[med-svn] [canu] 01/03: Imported Upstream version 1.5+dfsg
Afif Elghraoui
afif at moszumanska.debian.org
Thu May 18 05:30:12 UTC 2017
This is an automated email from the git hooks/post-receive script.
afif pushed a commit to branch master
in repository canu.
commit 15ffcc10d7d9504427a745dc66964692583fef82
Author: Afif Elghraoui <afif at debian.org>
Date: Thu May 18 01:12:00 2017 -0400
Imported Upstream version 1.5+dfsg
---
README.licenses | 43 +
addCopyrights-BuildData.pl | 1 +
addCopyrights.dat | 729 +++++
addCopyrights.pl | 41 +-
buildRelease.sh | 39 +
documentation/reST-markup-hints | 10 +
documentation/source/conf.py | 4 +-
documentation/source/faq.rst | 95 +-
documentation/source/parameter-reference.rst | 418 ++-
documentation/source/quick-start.rst | 55 +-
documentation/source/tutorial.rst | 114 +-
src/AS_UTL/AS_UTL_fileIO.C | 149 +-
src/AS_UTL/AS_UTL_fileIO.H | 8 +-
src/AS_UTL/AS_UTL_stackTrace.C | 376 ++-
src/AS_UTL/AS_UTL_stackTrace.H | 2 +-
src/AS_UTL/bitPackedFile.C | 2 +-
src/AS_UTL/libbacktrace/LICENSE | 29 +
src/AS_UTL/libbacktrace/README | 15 +
src/AS_UTL/libbacktrace/atomic.c | 113 +
src/AS_UTL/libbacktrace/backtrace-supported.h | 66 +
src/AS_UTL/libbacktrace/backtrace.c | 129 +
src/AS_UTL/libbacktrace/backtrace.h | 182 ++
src/AS_UTL/libbacktrace/config.h | 128 +
src/AS_UTL/libbacktrace/dwarf.c | 3123 ++++++++++++++++++++
src/AS_UTL/libbacktrace/elf.c | 983 ++++++
src/AS_UTL/libbacktrace/fileline.c | 194 ++
src/AS_UTL/libbacktrace/internal.h | 294 ++
src/AS_UTL/libbacktrace/make.out | 26 +
src/AS_UTL/libbacktrace/make.sh | 22 +
src/AS_UTL/libbacktrace/mmap.c | 303 ++
src/AS_UTL/libbacktrace/mmapio.c | 100 +
src/AS_UTL/libbacktrace/posix.c | 100 +
src/AS_UTL/libbacktrace/print.c | 92 +
src/AS_UTL/libbacktrace/simple.c | 108 +
src/AS_UTL/libbacktrace/sort.c | 108 +
src/AS_UTL/libbacktrace/state.c | 72 +
src/AS_UTL/libbacktrace/unknown.c | 68 +
src/AS_global.C | 9 +-
src/Makefile | 227 +-
src/bogart/AS_BAT_BestOverlapGraph.C | 90 +-
src/bogart/AS_BAT_BestOverlapGraph.H | 10 +-
src/bogart/AS_BAT_ChunkGraph.C | 16 +-
src/bogart/AS_BAT_CreateUnitigs.C | 2 +-
src/bogart/AS_BAT_MergeOrphans.H | 6 +-
src/bogart/AS_BAT_OverlapCache.C | 70 +-
src/bogart/AS_BAT_TigGraph.C | 28 +-
src/bogart/bogart.C | 25 +-
src/canu_version_update.pl | 58 +-
src/correction/generateCorrectionLayouts.C | 63 +-
src/falcon_sense/falcon_sense.C | 21 +
src/falcon_sense/falcon_sense.mk | 2 +-
src/falcon_sense/libfalcon/falcon.C | 240 +-
src/falcon_sense/libfalcon/falcon.H | 75 +-
src/falcon_sense/libfalcon/kmer_lookup.C | 622 ----
src/fastq-utilities/fastqSimulate.C | 4 +-
src/gfa/alignGFA.C | 464 +++
.../falcon_sense.mk => gfa/alignGFA.mk} | 8 +-
src/gfa/gfa.C | 345 +++
src/gfa/gfa.H | 103 +
src/main.mk | 43 +-
src/merTrim/merTrim.C | 183 +-
.../falcon_sense.mk => merTrim/merTrim.mk} | 10 +-
src/merTrim/merTrimResult.H | 9 +-
src/meryl/compare-counts.C | 4 +
src/meryl/estimate-mer-threshold.C | 310 +-
src/meryl/existDB.C | 252 ++
.../falcon_sense.mk => meryl/existDB.mk} | 12 +-
src/meryl/libkmer/existDB-create-from-fasta.C | 304 ++
src/meryl/libkmer/existDB-create-from-meryl.C | 261 ++
src/meryl/libkmer/existDB-create-from-sequence.C | 293 ++
src/meryl/libkmer/existDB-state.C | 233 ++
src/meryl/libkmer/existDB.C | 211 ++
src/meryl/libkmer/existDB.H | 190 ++
src/meryl/libkmer/existDB.mk | 13 +
src/meryl/libkmer/percentCovered.C | 95 +
src/meryl/libkmer/percentCovered.mk | 13 +
src/meryl/libkmer/posDB.mk | 13 +
src/meryl/libkmer/positionDB-access.C | 376 +++
src/meryl/libkmer/positionDB-dump.C | 79 +
src/meryl/libkmer/positionDB-file.C | 318 ++
src/meryl/libkmer/positionDB-mismatch.C | 412 +++
src/meryl/libkmer/positionDB-sort.C | 182 ++
src/meryl/libkmer/positionDB.C | 1186 ++++++++
src/meryl/libkmer/positionDB.H | 273 ++
src/meryl/maskMers.C | 4 +
src/meryl/meryl-build.C | 20 +
src/meryl/positionDB.C | 308 ++
.../falcon_sense.mk => meryl/positionDB.mk} | 12 +-
src/mhap/mhapConvert.C | 33 +-
src/minimap/mmapConvert.C | 36 +-
src/overlapInCore/libedlib/edlib.C | 65 +-
src/overlapInCore/libedlib/edlib.H | 327 +-
.../prefixEditDistance-allocateMoreSpace.C | 13 +-
.../liboverlap/prefixEditDistance-extend.C | 18 +-
.../liboverlap/prefixEditDistance-forward.C | 36 +-
.../liboverlap/prefixEditDistance-reverse.C | 36 +-
src/overlapInCore/liboverlap/prefixEditDistance.H | 21 +-
.../overlapInCore-Process_String_Overlaps.C | 2 +-
src/overlapInCore/overlapPair.C | 710 +++--
src/pipelines/canu-object-store.pl | 152 +
src/pipelines/canu.pl | 405 +--
src/pipelines/canu/Configure.pm | 193 +-
src/pipelines/canu/Consensus.pm | 465 ++-
src/pipelines/canu/CorrectReads.pm | 688 +++--
src/pipelines/canu/Defaults.pm | 407 ++-
src/pipelines/canu/ErrorEstimate.pm | 102 +-
src/pipelines/canu/Execution.pm | 508 +++-
src/pipelines/canu/Gatekeeper.pm | 369 +--
src/pipelines/canu/Grid_Cloud.pm | 347 +++
src/pipelines/canu/Grid_DNANexus.pm | 104 +
src/pipelines/canu/Grid_LSF.pm | 3 -
src/pipelines/canu/Grid_PBSTorque.pm | 19 +-
src/pipelines/canu/Grid_SGE.pm | 34 +-
src/pipelines/canu/Grid_Slurm.pm | 3 -
src/pipelines/canu/HTML.pm | 214 +-
src/pipelines/canu/Meryl.pm | 573 ++--
src/pipelines/canu/Output.pm | 150 +-
src/pipelines/canu/OverlapBasedTrimming.pm | 165 +-
src/pipelines/canu/OverlapErrorAdjustment.pm | 345 ++-
src/pipelines/canu/OverlapInCore.pm | 216 +-
src/pipelines/canu/OverlapMMap.pm | 298 +-
src/pipelines/canu/OverlapMhap.pm | 352 ++-
src/pipelines/canu/OverlapStore.pm | 428 +--
src/pipelines/canu/Report.pm | 172 ++
src/pipelines/canu/Unitig.pm | 169 +-
src/pipelines/parallel-ovl-store-test.sh | 68 +
src/pipelines/sanity/sanity-all-done.pl | 240 --
src/pipelines/sanity/sanity-merge-qc.pl | 121 -
src/pipelines/sanity/sanity.pl | 382 ---
src/pipelines/sanity/sanity.sh | 50 -
src/pipelines/simple-repeat-test.pl | 107 +
src/stores/gatekeeperPartition.C | 82 +-
src/stores/gkStore.C | 35 +-
src/stores/gkStore.H | 3 +-
src/stores/ovOverlap.C | 13 +-
src/stores/ovOverlap.H | 18 +-
src/stores/ovStore.C | 115 +-
src/stores/ovStore.H | 10 +-
src/stores/ovStoreBucketizer.C | 4 +-
src/stores/ovStoreBuild.C | 13 +-
src/stores/ovStoreFile.C | 202 +-
src/stores/ovStoreFilter.C | 23 +-
src/stores/ovStoreHistogram.C | 28 +-
src/stores/ovStoreHistogram.H | 4 +-
src/stores/ovStoreWriter.C | 12 +-
src/stores/tgStore.C | 8 +-
src/stores/tgStoreCoverageStat.C | 4 +-
src/stores/tgStoreDump.C | 16 +-
src/stores/tgStoreLoad.C | 41 +-
src/stores/tgTig.C | 32 +
src/stores/tgTig.H | 2 +
src/stores/tgTigMultiAlignDisplay.C | 2 +-
src/utgcns/libNDFalcon/dw.C | 10 +
src/utgcns/libcns/unitigConsensus.C | 134 +-
src/utgcns/libcns/unitigConsensus.H | 3 +-
src/utgcns/libpbutgcns/LICENSE | 72 +-
src/utgcns/utgcns.C | 5 +-
157 files changed, 20637 insertions(+), 5848 deletions(-)
diff --git a/README.licenses b/README.licenses
index a1092cc..10a1556 100644
--- a/README.licenses
+++ b/README.licenses
@@ -13,6 +13,11 @@ This software is based on the 'kmer package' (http://kmer.sourceforge.net) as
distributed by Applera Corporation under the GNU General Public License, version 2.
Canu branched from the kmer project at revision 1994.
+This software is based on 'pbutgcns' (https://github.com/pbjd/pbutgcns) and 'FALCON'
+(https://github.com/PacificBiosciences/FALCON) as released by Pacific Biosciences of
+California, Inc.
+
+
For the full software distribution and modifications made under the GNU General
@@ -33,6 +38,7 @@ Public License version 2:
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
For modifications made by the Battelle National Biodefense Institute
under the BSD 3-Clause License:
@@ -68,6 +74,7 @@ under the BSD 3-Clause License:
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
For modifications in the public domain:
PUBLIC DOMAIN NOTICE
@@ -87,3 +94,39 @@ For modifications in the public domain:
particular purpose.
Please cite the authors in any work or product based on this material.
+
+
+
+For software released by Pacific Biosciences of California, Inc.:
+
+ Copyright (c) 2011-2015, Pacific Biosciences of California, Inc.
+
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted (subject to the limitations in the
+ disclaimer below) provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name of Pacific Biosciences nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+ THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC BIOSCIENCES AND ITS
+ CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR
+ ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
diff --git a/addCopyrights-BuildData.pl b/addCopyrights-BuildData.pl
index fd1a618..41ff558 100644
--- a/addCopyrights-BuildData.pl
+++ b/addCopyrights-BuildData.pl
@@ -33,6 +33,7 @@ $stoppingCommits{"72c27c95d61cb8f37e859c4039456eb2acc5c55b"} = 1; # 19 NOV 20
$stoppingCommits{"b2df5790f77d38cc31fe77a7f65360e02389f92e"} = 1; # 04 MAR 2016
$stoppingCommits{"1ef335952342ef06ad1651a888f09c312f54dab8"} = 1; # 18 MAY 2016
$stoppingCommits{"bbbdcd063560e5f86006ee6b8b96d2d7b80bb750"} = 1; # 21 NOV 2016
+$stoppingCommits{"64459fe33f97f6d23fe036ba1395743d0cdd03e4"} = 1; # 17 APR 2017
open(F, "< logs") or die "Failed to open 'logs': $!\n";
diff --git a/addCopyrights.dat b/addCopyrights.dat
index cf41bea..21c0af7 100644
--- a/addCopyrights.dat
+++ b/addCopyrights.dat
@@ -11107,3 +11107,732 @@ A addCopyrights.dat nihh20160518Brian P. Walenz
A addCopyrights.pl nihh20160518Brian P. Walenz
D src/bogart/AS_BAT_MergeOrphans.C src/bogart/AS_BAT_PopBubbles.C
D src/bogart/AS_BAT_MergeOrphans.H src/bogart/AS_BAT_PopBubbles.H
+D src/libkmer/driver-existDB.C kmer/libkmer/driver-existDB.C
+D src/libkmer/driver-posDB.C kmer/libkmer/driver-posDB.C
+D src/libkmer/existDB-create-from-fasta.C kmer/libkmer/existDB-create-from-fasta.C
+D src/libkmer/existDB-create-from-meryl.C kmer/libkmer/existDB-create-from-meryl.C
+D src/libkmer/existDB-create-from-sequence.C kmer/libkmer/existDB-create-from-sequence.C
+D src/libkmer/existDB-state.C kmer/libkmer/existDB-state.C
+D src/libkmer/existDB.C kmer/libkmer/existDB.C
+D src/libkmer/existDB.H kmer/libkmer/existDB.H
+D src/libkmer/existDB.mk kmer/libkmer/existDB.mk
+D src/libkmer/main.mk kmer/libkmer/main.mk
+D src/libkmer/percentCovered.C kmer/libkmer/percentCovered.C
+D src/libkmer/percentCovered.mk kmer/libkmer/percentCovered.mk
+D src/libkmer/posDB.mk kmer/libkmer/posDB.mk
+D src/libkmer/positionDB-access.C kmer/libkmer/positionDB-access.C
+D src/libkmer/positionDB-dump.C kmer/libkmer/positionDB-dump.C
+D src/libkmer/positionDB-file.C kmer/libkmer/positionDB-file.C
+D src/libkmer/positionDB-mismatch.C kmer/libkmer/positionDB-mismatch.C
+D src/libkmer/positionDB-sort.C kmer/libkmer/positionDB-sort.C
+D src/libkmer/positionDB.C kmer/libkmer/positionDB.C
+D src/libkmer/positionDB.H kmer/libkmer/positionDB.H
+A src/stores/ovStore.C nihh20170417Brian P. Walenz
+A src/stores/ovStore.C nihh20170412Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170411Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170411Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170411Brian P. Walenz
+A src/AS_global.C nihh20170411Brian P. Walenz
+A src/canu_version_update.pl nihh20170411Brian P. Walenz
+A src/pipelines/canu/Grid_SGE.pm nihh20170407Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170407Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170407Brian P. Walenz
+A documentation/source/tutorial.rst nihh20170407Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170407Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170407Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170407Brian P. Walenz
+A src/pipelines/canu.pl nihh20170407Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170407Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170407Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170407Brian P. Walenz
+A src/pipelines/canu/Output.pm nihh20170407Brian P. Walenz
+A src/gfa/alignGFA.C nihh20170407Brian P. Walenz
+A src/stores/ovStore.C nihh20170406Brian P. Walenz
+A src/stores/ovStore.H nihh20170406Brian P. Walenz
+A documentation/source/faq.rst nihh20170405Brian P. Walenz
+A src/stores/ovStoreHistogram.C nihh20170405Brian P. Walenz
+A src/meryl/estimate-mer-threshold.C nihh20170405Brian P. Walenz
+A src/gfa/alignGFA.C nihh20170404Brian P. Walenz
+A src/gfa/gfa.C nihh20170404Brian P. Walenz
+A src/gfa/gfa.H nihh20170404Brian P. Walenz
+A src/gfa/alignGFA.C nihh20170404Brian P. Walenz
+A src/gfa/gfa.C nihh20170404Brian P. Walenz
+A src/gfa/gfa.H nihh20170404Brian P. Walenz
+A src/main.mk nihh20170404Brian P. Walenz
+A src/gfa/alignGFA.C nihh20170404Brian P. Walenz
+A src/gfa/alignGFA.mk nihh20170404Brian P. Walenz
+A src/main.mk nihh20170404Brian P. Walenz
+A src/AS_UTL/AS_UTL_fileIO.C nihh20170404Brian P. Walenz
+A src/AS_UTL/AS_UTL_fileIO.H nihh20170404Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170403Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170330Brian P. Walenz
+A documentation/source/faq.rst nihh20170330Brian P. Walenz
+A src/stores/gatekeeperPartition.C nihh20170330Brian P. Walenz
+A src/pipelines/canu.pl nihh20170329Brian P. Walenz
+A src/stores/gatekeeperPartition.C nihh20170329Brian P. Walenz
+A src/stores/gkStore.C nihh20170329Brian P. Walenz
+A src/stores/gkStore.H nihh20170329Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170329Brian P. Walenz
+A src/canu_version_update.pl nihh20170329Brian P. Walenz
+A src/main.mk nihh20170329Brian P. Walenz
+A src/utgcns/alignGFA.C nihh20170329Brian P. Walenz
+A src/utgcns/alignGFA.mk nihh20170329Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.C nihh20170329Brian P. Walenz
+A documentation/source/faq.rst nihh20170328Sergey Koren
+A documentation/source/conf.py nihh20170328Sergey Koren
+A src/pipelines/canu/Gatekeeper.pm nihh20170327Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20170327Brian P. Walenz
+A src/stores/tgTig.C nihh20170327Brian P. Walenz
+A src/stores/tgTig.H nihh20170327Brian P. Walenz
+A src/stores/tgTigMultiAlignDisplay.C nihh20170327Brian P. Walenz
+A src/pipelines/canu.pl nihh20170324Brian P. Walenz
+A src/AS_UTL/AS_UTL_stackTrace.C nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/LICENSE nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/README nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/atomic.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/backtrace-supported.h nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/backtrace.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/backtrace.h nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/config.h nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/dwarf.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/elf.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/fileline.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/internal.h nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/make.out nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/make.sh nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/mmap.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/mmapio.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/posix.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/print.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/simple.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/sort.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/state.c nihh20170322Brian P. Walenz
+A src/AS_UTL/libbacktrace/unknown.c nihh20170322Brian P. Walenz
+A src/Makefile nihh20170322Brian P. Walenz
+A src/main.mk nihh20170322Brian P. Walenz
+A src/Makefile nihh20170322Brian P. Walenz
+A src/AS_UTL/AS_UTL_stackTrace.C nihh20170322Brian P. Walenz
+A src/AS_UTL/AS_UTL_stackTrace.H nihh20170322Brian P. Walenz
+A src/AS_global.C nihh20170322Brian P. Walenz
+A src/AS_UTL/AS_UTL_stackTrace.C nihh20170322Brian P. Walenz
+A src/pipelines/canu.pl nihh20170321Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170321Brian P. Walenz
+A src/pipelines/canu.pl nihh20170321Brian P. Walenz
+A src/pipelines/canu.pl nihh20170321Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170321Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170321Brian P. Walenz
+A src/pipelines/canu.pl nihh20170320Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170320Brian P. Walenz
+A src/AS_global.C nihh20170320Brian P. Walenz
+A src/AS_global.C nihh20170320Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170320Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu.pl nihh20170320Brian P. Walenz
+A src/pipelines/canu/Report.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170320Brian P. Walenz
+A src/Makefile nihh20170320Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Report.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170320Brian P. Walenz
+A src/pipelines/canu/Grid_SGE.pm nihh20170316Brian P. Walenz
+A documentation/source/quick-start.rst nihh20170316Brian P. Walenz
+A documentation/source/faq.rst nihh20170316Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170315Sergey Koren
+A src/pipelines/canu/OverlapMhap.pm nihh20170315Sergey Koren
+A src/pipelines/sanity/sanity.sh nihh20170314Brian P. Walenz
+A src/pipelines/sanity/medium.arabidopsis_thaliana.pacbio.p4c2.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/medium.arabidopsis_thaliana.pacbio.p5c3.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/medium.caenorhabditis_elegans.pacbio.p6c4.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/medium.drosophila_melanogaster.pacbio.p5c3.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.h5-1000.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.h5-5000.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.sra-3000.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.pacbio.p6.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_ne92.pacbio.p4.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_ne92.pacbio.p5.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_o157_h7_str_f8092b.pacbio.p4c2.average.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_o157_h7_str_f8092b.pacbio.p4c2.long.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.francisella_tularensis.pacbio.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_glbrcy22-3.pacbio.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_glbrcy22-3.pacbio.sra.spec nihh20170314Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_s288c.pacbio.spec nihh20170314Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170314Brian P. Walenz
+A src/pipelines/canu-object-store.pl nihh20170314Brian P. Walenz
+A src/overlapInCore/overlapPair.C nihh20170310Brian P. Walenz
+A src/overlapInCore/libedlib/edlib.C nihh20170310Brian P. Walenz
+A src/overlapInCore/libedlib/edlib.H nihh20170310Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170310Brian P. Walenz
+A src/pipelines/sanity/medium.caenorhabditis_elegans.pacbio.p6c4.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/sanity.NOTES nihh20170310Brian P. Walenz
+A src/pipelines/sanity/sanity.sh nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.bacillus_anthracis_sterne.nanopore.34F2_NBI0483991.poretools.2D.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.all.2d.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-1.2d.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-2.2d.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-pcr-1.2d.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-pcr-2.2d.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.r9.4.superlong.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.r9.SpotOn.1d.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.pacbio.p6.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_ne92.pacbio.p4.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_ne92.pacbio.p5.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_o157_h7_str_f8092b.pacbio.p4c2.long.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_s288c.nanopore.r7.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_s288c.nanopore.r9.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_w303.nanopore.poretools.2D.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/small.yersinia_pestis.nanopore.NBI0499872.poretools.2D.spec nihh20170310Brian P. Walenz
+A src/pipelines/sanity/success.caenorhabditis_elegans.sh nihh20170310Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170309Sergey Koren
+A src/pipelines/canu/Defaults.pm nihh20170309Sergey Koren
+A src/pipelines/canu/Meryl.pm nihh20170307Sergey Koren
+A src/pipelines/canu/OverlapMMap.pm nihh20170307Sergey Koren
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170303Sergey Koren
+A src/pipelines/canu/Defaults.pm nihh20170302Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170223Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170302Sergey Koren
+A src/pipelines/canu/OverlapStore.pm nihh20170227Brian P. Walenz
+A src/main.mk nihh20170224Brian P. Walenz
+A src/merTrim/merTrim.C nihh20170224Brian P. Walenz
+A src/merTrim/merTrim.mk nihh20170224Brian P. Walenz
+A src/merTrim/merTrimResult.H nihh20170224Brian P. Walenz
+A src/main.mk nihh20170224Brian P. Walenz
+A src/meryl/existDB.C nihh20170224Brian P. Walenz
+A src/meryl/existDB.mk nihh20170224Brian P. Walenz
+A src/meryl/positionDB.C nihh20170224Brian P. Walenz
+A src/meryl/positionDB.mk nihh20170224Brian P. Walenz
+A src/main.mk nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-create-from-fasta.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-create-from-meryl.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-create-from-sequence.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-state.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB.H nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-access.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-dump.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-file.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-mismatch.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-sort.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB.H nihh20170224Brian P. Walenz
+A addCopyrights.dat nihh20170224Brian P. Walenz
+A src/meryl/libkmer/driver-existDB.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/driver-posDB.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-create-from-fasta.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-create-from-meryl.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-create-from-sequence.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB-state.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB.H nihh20170224Brian P. Walenz
+A src/meryl/libkmer/existDB.mk nihh20170224Brian P. Walenz
+A src/meryl/libkmer/main.mk nihh20170224Brian P. Walenz
+A src/meryl/libkmer/percentCovered.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/percentCovered.mk nihh20170224Brian P. Walenz
+A src/meryl/libkmer/posDB.mk nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-access.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-dump.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-file.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-mismatch.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB-sort.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB.C nihh20170224Brian P. Walenz
+A src/meryl/libkmer/positionDB.H nihh20170224Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170224Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170223Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170223Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170223Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170222Brian P. Walenz
+A src/pipelines/canu/Grid_Cloud.pm nihh20170222Brian P. Walenz
+A src/stores/tgStoreCoverageStat.C nihh20170222Brian P. Walenz
+A src/pipelines/canu.pl nihh20170222Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170222Brian P. Walenz
+A src/pipelines/canu/Grid_Cloud.pm nihh20170222Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170222Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170221Brian P. Walenz
+A src/pipelines/canu/Grid_DNANexus.pm nihh20170221Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170221Brian P. Walenz
+A src/stores/ovStoreBucketizer.C nihh20170221Brian P. Walenz
+A src/stores/ovStoreBuild.C nihh20170221Brian P. Walenz
+A src/stores/ovStoreWriter.C nihh20170221Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170221Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170221Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170221Brian P. Walenz
+A documentation/reST-markup-hints nihh20170221Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170221Brian P. Walenz
+A src/pipelines/canu/Grid_PBSTorque.pm nihh20170221Brian P. Walenz
+A src/pipelines/canu/Grid_Cloud.pm nihh20170221Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170220Brian P. Walenz
+A src/pipelines/canu/HTML.pm nihh20170220Brian P. Walenz
+A src/pipelines/canu/Grid_Cloud.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170215Brian P. Walenz
+A src/Makefile nihh20170215Brian P. Walenz
+A src/pipelines/canu.pl nihh20170215Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Grid_Cloud.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Output.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170215Brian P. Walenz
+A src/pipelines/canu.pl nihh20170213Brian P. Walenz
+A src/Makefile nihh20170211Brian P. Walenz
+A src/pipelines/canu.pl nihh20170211Brian P. Walenz
+A src/pipelines/canu/Grid_DNANexus.pm nihh20170211Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170204Brian P. Walenz
+A src/utgcns/libNDFalcon/dw.C nihh20170214Sergey Koren
+A src/utgcns/libcns/unitigConsensus.C nihh20170214Sergey Koren
+A src/utgcns/libcns/unitigConsensus.C nihh20170214Sergey Koren
+A src/utgcns/libcns/unitigConsensus.C nihh20170214Sergey Koren
+A src/pipelines/canu.pl nihh20170213Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170213Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170209Brian P. Walenz
+A src/utgcns/libcns/unitigConsensus.C nihh20170209Sergey Koren
+A src/pipelines/canu/Consensus.pm nihh20170203Sergey Koren
+A src/utgcns/libcns/unitigConsensus.C nihh20170203Sergey Koren
+A src/utgcns/libcns/unitigConsensus.H nihh20170203Sergey Koren
+A src/utgcns/utgcns.C nihh20170203Sergey Koren
+A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C nihh20170202Brian P. Walenz
+A src/overlapInCore/liboverlap/prefixEditDistance-extend.C nihh20170202Brian P. Walenz
+A src/overlapInCore/liboverlap/prefixEditDistance-forward.C nihh20170202Brian P. Walenz
+A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C nihh20170202Brian P. Walenz
+A src/overlapInCore/liboverlap/prefixEditDistance.H nihh20170202Brian P. Walenz
+A src/overlapInCore/overlapInCore-Process_String_Overlaps.C nihh20170202Brian P. Walenz
+A src/pipelines/canu.pl nihh20170202Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/ErrorEstimate.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/HTML.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Output.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170202Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170202Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170202Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170202Brian P. Walenz
+A src/stores/ovStoreBuild.C nihh20170201Brian P. Walenz
+A src/stores/ovStoreHistogram.C nihh20170201Brian P. Walenz
+A src/stores/ovStoreHistogram.H nihh20170201Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170201Brian P. Walenz
+A documentation/source/tutorial.rst nihh20170201Brian P. Walenz
+A src/bogart/bogart.C nihh20170127Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170124Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170124Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170124Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170124Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/ErrorEstimate.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/Gatekeeper.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/HTML.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/Output.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170117Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170130Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170127Brian P. Walenz
+A documentation/source/quick-start.rst nihh20170127Brian P. Walenz
+A documentation/source/tutorial.rst nihh20170127Brian P. Walenz
+A src/pipelines/canu.pl nihh20170127Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/ErrorEstimate.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170127Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170126Brian P. Walenz
+A src/pipelines/canu.pl nihh20170126Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170126Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170126Brian P. Walenz
+A src/falcon_sense/libfalcon/falcon.C nihh20170124Brian P. Walenz
+A src/falcon_sense/libfalcon/falcon.H nihh20170124Brian P. Walenz
+A src/falcon_sense/falcon_sense.C nihh20170124Brian P. Walenz
+A src/falcon_sense/libfalcon/falcon.C nihh20170124Brian P. Walenz
+A src/falcon_sense/libfalcon/falcon.H nihh20170124Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170123Brian P. Walenz
+A src/Makefile nihh20170123Brian P. Walenz
+A src/Makefile nihh20170123Brian P. Walenz
+A src/stores/ovStoreHistogram.C nihh20170123Brian P. Walenz
+A src/bogart/AS_BAT_OverlapCache.C nihh20170120Brian P. Walenz
+A src/bogart/AS_BAT_BestOverlapGraph.C nihh20170120Brian P. Walenz
+A src/pipelines/canu.pl nihh20170120Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20170120Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170120Brian P. Walenz
+A src/pipelines/canu.pl nihh20170119Brian P. Walenz
+A src/pipelines/canu/Consensus.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/ErrorEstimate.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/HTML.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/OverlapBasedTrimming.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/OverlapInCore.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170119Brian P. Walenz
+A src/pipelines/canu/Unitig.pm nihh20170119Brian P. Walenz
+A src/utgcns/libcns/unitigConsensus.C nihh20170116Brian P. Walenz
+A src/bogart/AS_BAT_ChunkGraph.C nihh20170113Brian P. Walenz
+A src/bogart/AS_BAT_BestOverlapGraph.C nihh20170112Brian P. Walenz
+A src/bogart/AS_BAT_BestOverlapGraph.H nihh20170112Brian P. Walenz
+A src/bogart/bogart.C nihh20170112Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170110Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170110Brian P. Walenz
+A src/stores/ovStore.H nihh20170110Brian P. Walenz
+A src/stores/ovStoreHistogram.C nihh20170110Brian P. Walenz
+A src/stores/ovStoreHistogram.H nihh20170110Brian P. Walenz
+A src/mhap/mhapConvert.C nihh20170110Brian P. Walenz
+A src/minimap/mmapConvert.C nihh20170110Brian P. Walenz
+A src/pipelines/canu/OverlapMMap.pm nihh20170110Brian P. Walenz
+A src/pipelines/canu/OverlapMhap.pm nihh20170110Brian P. Walenz
+A src/pipelines/parallel-ovl-store-test.sh nihh20170109Brian P. Walenz
+A documentation/reST-markup-hints nihh20170109Brian P. Walenz
+A src/pipelines/simple-repeat-test.pl nihh20170109Brian P. Walenz
+A documentation/source/parameter-reference.rst nihh20170109Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170107Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170107Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170107Brian P. Walenz
+A src/pipelines/canu/Grid_LSF.pm nihh20170107Brian P. Walenz
+A src/pipelines/canu/Grid_PBSTorque.pm nihh20170107Brian P. Walenz
+A src/pipelines/canu/Grid_SGE.pm nihh20170107Brian P. Walenz
+A src/pipelines/canu/Grid_Slurm.pm nihh20170107Brian P. Walenz
+A src/fastq-utilities/fastqSimulate.C nihh20170106Brian P. Walenz
+A src/falcon_sense/falcon_sense.mk nihh20170106Sergey Koren
+A src/falcon_sense/libfalcon/falcon.C nihh20170106Sergey Koren
+A src/falcon_sense/libfalcon/falcon.H nihh20170106Sergey Koren
+A src/main.mk nihh20170106Sergey Koren
+A src/overlapInCore/libedlib/edlib.C nihh20170106Sergey Koren
+A src/overlapInCore/libedlib/edlib.H nihh20170106Sergey Koren
+A documentation/source/parameter-reference.rst nihh20170106Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20170106Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170106Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20170106Brian P. Walenz
+A src/pipelines/canu.pl nihh20170106Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20170106Brian P. Walenz
+A src/pipelines/sanity/sanity.pl nihh20170104Brian P. Walenz
+A src/Makefile nihh20170104Brian P. Walenz
+A src/stores/ovOverlap.C nihh20170104Brian P. Walenz
+A src/stores/ovOverlap.H nihh20170104Brian P. Walenz
+A src/stores/ovStoreFile.C nihh20170104Brian P. Walenz
+A src/stores/ovStore.H nihh20170104Brian P. Walenz
+A src/correction/generateCorrectionLayouts.C nihh20170103Brian P. Walenz
+A src/stores/ovStoreBucketizer.C nihh20170103Brian P. Walenz
+A src/stores/ovStoreBuild.C nihh20170103Brian P. Walenz
+A src/stores/ovStoreFilter.C nihh20170103Brian P. Walenz
+A src/pipelines/sanity/sanity.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.arabidopsis_thaliana.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.bacillus_anthracis_sterne.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.bibersteinia_trehalosi.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.drosophila_melanogaster.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.escherichia_coli_k12.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.escherichia_coli_ne92.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.escherichia_coli_o157_h7_str_f8092b.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.francisella_tularensis.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.yersinia_pestis_i195.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/small.bacillus_anthracis_sterne.nanopore.34F2_NBI0483991.poretools.2D.spec nihh20161225Brian P. Walenz
+A src/pipelines/sanity/sanity.sh nihh20161225Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_glbrcy22-3.pacbio.spec nihh20161225Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_glbrcy22-3.pacbio.sra.spec nihh20161225Brian P. Walenz
+A src/pipelines/sanity/small.saccharomyces_cerevisiae_s288c.pacbio.spec nihh20161225Brian P. Walenz
+A src/pipelines/sanity/success.saccharomyces_cerevisiae_s288c.sh nihh20161225Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20161225Brian P. Walenz
+A src/pipelines/sanity/sanity.sh nihh20161222Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.h5-1000.spec nihh20161222Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.h5-5000.spec nihh20161222Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.sra-3000.spec nihh20161222Brian P. Walenz
+A src/pipelines/sanity/medium.arabidopsis_thaliana.pacbio.p4c2.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/medium.arabidopsis_thaliana.pacbio.p5c3.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/medium.drosophila_melanogaster.pacbio.p5c3.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/sanity.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.bacillus_anthracis_sterne.nanopore.34F2_NBI0483991.poretools.2D.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.h5-1000.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.h5-5000.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.bibersteinia_trehalosi.pacbio.sra-1000.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.all.2d.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-1.2d.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-2.2d.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-pcr-1.2d.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.map006-pcr-2.2d.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.nanopore.r9.SpotOn.1d.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_k12.pacbio.p6.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_ne92.pacbio.p4.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_ne92.pacbio.p5.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_o157_h7_str_f8092b.pacbio.p4c2.average.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.escherichia_coli_o157_h7_str_f8092b.pacbio.p4c2.long.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.francisella_tularensis.pacbio.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/small.yersinia_pestis.nanopore.NBI0499872.poretools.2D.spec nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.arabidopsis_thaliana.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.bacillus_anthracis_sterne.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.bibersteinia_trehalosi.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.drosophila_melanogaster.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.escherichia_coli_k12.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.escherichia_coli_ne92.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.escherichia_coli_o157_h7_str_f8092b.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.francisella_tularensis.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/success.yersinia_pestis_i195.sh nihh20161220Brian P. Walenz
+A src/pipelines/sanity/sanity.pl nihh20161220Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161219Brian P. Walenz
+A src/stores/tgStoreLoad.C nihh20161219Brian P. Walenz
+A src/stores/tgStore.C nihh20161219Brian P. Walenz
+A src/AS_UTL/AS_UTL_fileIO.C nihh20161219Brian P. Walenz
+A src/AS_UTL/AS_UTL_fileIO.H nihh20161219Brian P. Walenz
+A src/AS_UTL/bitPackedFile.C nihh20161219Brian P. Walenz
+A src/pipelines/canu/Configure.pm nihh20161219Brian P. Walenz
+A src/meryl/estimate-mer-threshold.C nihh20161216Brian P. Walenz
+A src/pipelines/canu/Meryl.pm nihh20161216Brian P. Walenz
+A src/pipelines/canu/Execution.pm nihh20161214Brian P. Walenz
+A src/pipelines/canu/Output.pm nihh20161214Brian P. Walenz
+A src/meryl/meryl-build.C nihh20161214Brian P. Walenz
+A src/correction/generateCorrectionLayouts.C nihh20161214Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20161214Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20161214Brian P. Walenz
+A src/pipelines/canu/CorrectReads.pm nihh20161214Brian P. Walenz
+A buildRelease.sh nihh20161213Brian P. Walenz
+A buildRelease.sh nihh20161213Brian P. Walenz
+A README.licenses nihh20161213Brian P. Walenz
+A src/utgcns/libpbutgcns/LICENSE nihh20161213Brian P. Walenz
+A buildRelease.sh nihh20161213Brian P. Walenz
+A src/pipelines/canu.pl nihh20161212Brian P. Walenz
+A src/pipelines/canu/Defaults.pm nihh20161212Brian P. Walenz
+A src/pipelines/canu/ErrorEstimate.pm nihh20161212Brian P. Walenz
+A src/stores/ovStoreDump.C nihh20161209Brian P. Walenz
+A src/stores/ovOverlap.C nihh20161208Brian P. Walenz
+A src/stores/ovOverlap.H nihh20161208Brian P. Walenz
+A src/stores/ovStoreFile.C nihh20161208Brian P. Walenz
+A addCopyrights.dat nihh20161207Brian P. Walenz
+A addCopyrights.pl nihh20161207Brian P. Walenz
+A src/bogart/AS_BAT_MergeOrphans.C nihh20161207Brian P. Walenz
+A src/bogart/AS_BAT_MergeOrphans.H nihh20161207Brian P. Walenz
+A src/bogart/bogart.C nihh20161207Brian P. Walenz
+A src/bogart/bogart.mk nihh20161207Brian P. Walenz
+A src/main.mk nihh20161206Brian P. Walenz
+A src/meryl/maskMers.C nihh20161206Brian P. Walenz
+A src/meryl/maskMers.mk nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_CreateUnitigs.C nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_Unitig.C nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_Unitig.H nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_PlaceReadUsingOverlaps.C nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_PlaceContains.C nihh20161206Brian P. Walenz
+A src/bogart/bogart.C nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_ReadInfo.C nihh20161206Brian P. Walenz
+A src/bogart/AS_BAT_ReadInfo.H nihh20161206Brian P. Walenz
+A src/pipelines/canu/OverlapStore.pm nihh20161205Brian P. Walenz
+A src/stores/ovStoreStats.C nihh20161205Brian P. Walenz
+A src/bogart/AS_BAT_CreateUnitigs.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_CreateUnitigs.H nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_SplitDiscontinuous.H nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.H nihh20161202Brian P. Walenz
+A src/bogart/bogart.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_CreateUnitigs.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_CreateUnitigs.H nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_SplitDiscontinuous.H nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.H nihh20161202Brian P. Walenz
+A src/bogart/bogart.C nihh20161202Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.C nihh20161201Brian P. Walenz
+A src/bogart/bogart.C nihh20161201Brian P. Walenz
+A src/pipelines/canu/Output.pm nihh20161201Brian P. Walenz
+A src/stores/gkStore.C nihh20161130Brian P. Walenz
+A src/overlapErrorAdjustment/findErrors.C nihh20161130Brian P. Walenz
+A src/stores/gkStore.C nihh20161130Brian P. Walenz
+A src/stores/ovStore.C nihh20161130Brian P. Walenz
+A src/fastq-utilities/fastqSimulate.C nihh20161130Brian P. Walenz
+A src/overlapInCore/overlapInCorePartition.C nihh20161130Brian P. Walenz
+A src/overlapInCore/overlapInCorePartition.C nihh20161130Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161130Brian P. Walenz
+A src/overlapInCore/overlapInCore-Build_Hash_Index.C nihh20161130Brian P. Walenz
+A src/overlapInCore/overlapInCore.C nihh20161130Brian P. Walenz
+A src/overlapInCore/overlapInCore.H nihh20161130Brian P. Walenz
+A src/stores/gkStore.C nihh20161130Brian P. Walenz
+A src/stores/ovOverlap.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_OverlapCache.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_Instrumentation.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_BestOverlapGraph.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_BestOverlapGraph.C nihh20161130Brian P. Walenz
+A src/overlapInCore/overlapImport.C nihh20161130Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.C nihh20161130Brian P. Walenz
+A src/erateEstimate/erateEstimate.C nihh20161130Brian P. Walenz
+A src/stores/ovStore.H nihh20161130Brian P. Walenz
+A src/stores/ovStoreWriter.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_PlaceReadUsingOverlaps.C nihh20161130Brian P. Walenz
+A src/stores/ovStore.H nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_ChunkGraph.C nihh20161130Brian P. Walenz
+A src/stores/ovStore.C nihh20161130Brian P. Walenz
+A src/bogart/AS_BAT_Instrumentation.C nihh20161129Brian P. Walenz
+A src/bogart/AS_BAT_OverlapCache.C nihh20161129Brian P. Walenz
+A src/fastq-utilities/fastqSample.C nihh20161129Brian P. Walenz
+A src/fastq-utilities/fastqSimulate.C nihh20161129Brian P. Walenz
+A src/meryl/leaff-partition.C nihh20161129Brian P. Walenz
+A src/overlapErrorAdjustment/findErrors-Read_Frags.C nihh20161129Brian P. Walenz
+A src/stores/gatekeeperCreate.C nihh20161129Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161129Brian P. Walenz
+A src/utgcns/stashContains.C nihh20161129Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161129Brian P. Walenz
+A src/overlapErrorAdjustment/correctOverlaps.H nihh20161129Brian P. Walenz
+A src/overlapErrorAdjustment/findErrors.H nihh20161129Brian P. Walenz
+A src/utgcns/libNDFalcon/dw.H nihh20161129Brian P. Walenz
+A src/overlapErrorAdjustment/findErrors.H nihh20161129Brian P. Walenz
+A src/main.mk nihh20161129Brian P. Walenz
+A src/stores/gkStore.H nihh20161129Brian P. Walenz
+A src/stores/gkStore.H nihh20161129Brian P. Walenz
+A src/bogart/AS_BAT_ChunkGraph.C nihh20161129Brian P. Walenz
+A src/bogart/AS_BAT_PopBubbles.C nihh20161129Brian P. Walenz
+A src/overlapInCore/overlapInCore.C nihh20161129Brian P. Walenz
+A src/overlapInCore/overlapInCore.H nihh20161129Brian P. Walenz
+A src/bogart/bogart.C nihh20161129Brian P. Walenz
+A src/stores/ovStoreDump.C nihh20161129Brian P. Walenz
+A src/stores/ovStoreWriter.C nihh20161129Brian P. Walenz
+A src/stores/ovStoreBucketizer.C nihh20161129Brian P. Walenz
+A src/stores/ovStoreIndexer.C nihh20161129Brian P. Walenz
+A src/stores/ovStoreSorter.C nihh20161129Brian P. Walenz
+A src/overlapInCore/overlapImport.C nihh20161129Brian P. Walenz
+A src/falcon_sense/falcon_sense.C nihh20161128Brian P. Walenz
+A src/meryl/libmeryl.C nihh20161123Brian P. Walenz
+A src/meryl/meryl-build.C nihh20161122Brian P. Walenz
+A src/stores/tgTig.C nihh20161122Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161122Brian P. Walenz
+A src/overlapInCore/overlapInCore-Output.C nihh20161122Brian P. Walenz
+A src/stores/gatekeeperDumpFASTQ.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_MarkRepeatReads.C nihh20161122Brian P. Walenz
+A src/meryl/leaff-statistics.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreDump.C nihh20161122Brian P. Walenz
+A src/meryl/libleaff/gkStoreFile.C nihh20161122Brian P. Walenz
+A src/meryl/libleaff/fastqStdin.C nihh20161122Brian P. Walenz
+A src/stores/tgTigMultiAlignDisplay.C nihh20161122Brian P. Walenz
+A src/stores/ovOverlap.H nihh20161122Brian P. Walenz
+A src/overlapErrorAdjustment/findErrors.C nihh20161122Brian P. Walenz
+A src/overlapErrorAdjustment/findErrors.H nihh20161122Brian P. Walenz
+A src/utgcns/libcns/abAbacus-refine.C nihh20161122Brian P. Walenz
+A src/AS_UTL/AS_UTL_fileIO.C nihh20161122Brian P. Walenz
+A src/AS_UTL/AS_UTL_stackTrace.C nihh20161122Brian P. Walenz
+A src/AS_global.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_AssemblyGraph.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_BestOverlapGraph.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_ChunkGraph.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_Instrumentation.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_Logging.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_MarkRepeatReads.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_Outputs.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_OverlapCache.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_PopBubbles.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_TigGraph.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_TigVector.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_Unitig.C nihh20161122Brian P. Walenz
+A src/bogart/bogart.C nihh20161122Brian P. Walenz
+A src/bogart/buildGraph.C nihh20161122Brian P. Walenz
+A src/bogus/bogus.C nihh20161122Brian P. Walenz
+A src/bogus/bogusness.C nihh20161122Brian P. Walenz
+A src/correction/filterCorrectionOverlaps.C nihh20161122Brian P. Walenz
+A src/correction/generateCorrectionLayouts.C nihh20161122Brian P. Walenz
+A src/erateEstimate/erateEstimate.C nihh20161122Brian P. Walenz
+A src/falcon_sense/createFalconSenseInputs.C nihh20161122Brian P. Walenz
+A src/fastq-utilities/fastqSample.C nihh20161122Brian P. Walenz
+A src/fastq-utilities/fastqSimulate.C nihh20161122Brian P. Walenz
+A src/merTrim/merTrim.C nihh20161122Brian P. Walenz
+A src/mercy/mercy.C nihh20161122Brian P. Walenz
+A src/meryl/compare-counts.C nihh20161122Brian P. Walenz
+A src/meryl/leaff-partition.C nihh20161122Brian P. Walenz
+A src/meryl/leaff.C nihh20161122Brian P. Walenz
+A src/meryl/libmeryl.C nihh20161122Brian P. Walenz
+A src/meryl/maskMers.C nihh20161122Brian P. Walenz
+A src/meryl/meryl-args.C nihh20161122Brian P. Walenz
+A src/meryl/meryl-build.C nihh20161122Brian P. Walenz
+A src/overlapBasedTrimming/splitReads.C nihh20161122Brian P. Walenz
+A src/overlapBasedTrimming/trimReads-bestEdge.C nihh20161122Brian P. Walenz
+A src/overlapBasedTrimming/trimReads.C nihh20161122Brian P. Walenz
+A src/overlapBasedTrimming/trimStat.H nihh20161122Brian P. Walenz
+A src/overlapInCore/overlapInCorePartition.C nihh20161122Brian P. Walenz
+A src/stores/gatekeeperCreate.C nihh20161122Brian P. Walenz
+A src/stores/gatekeeperDumpFASTQ.C nihh20161122Brian P. Walenz
+A src/stores/gatekeeperPartition.C nihh20161122Brian P. Walenz
+A src/stores/gkStore.C nihh20161122Brian P. Walenz
+A src/stores/ovStore.C nihh20161122Brian P. Walenz
+A src/stores/ovStore.H nihh20161122Brian P. Walenz
+A src/stores/ovStoreBucketizer.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreBuild.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreDump.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreHistogram.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreSorter.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreStats.C nihh20161122Brian P. Walenz
+A src/stores/ovStoreWriter.C nihh20161122Brian P. Walenz
+A src/stores/tgStore.C nihh20161122Brian P. Walenz
+A src/stores/tgStore.H nihh20161122Brian P. Walenz
+A src/stores/tgStoreCoverageStat.C nihh20161122Brian P. Walenz
+A src/stores/tgStoreDump.C nihh20161122Brian P. Walenz
+A src/stores/tgStoreFilter.C nihh20161122Brian P. Walenz
+A src/stores/tgTig.C nihh20161122Brian P. Walenz
+A src/stores/tgTigMultiAlignDisplay.C nihh20161122Brian P. Walenz
+A src/utgcns/utgcns.C nihh20161122Brian P. Walenz
+A src/stores/tgTig.C nihh20161122Brian P. Walenz
+A src/stores/tgTig.C nihh20161122Brian P. Walenz
+A src/AS_UTL/bitPackedFile.C nihh20161122Brian P. Walenz
+A src/AS_UTL/bitPackedFile.C nihh20161122Brian P. Walenz
+A src/AS_UTL/bitPackedFile.C nihh20161122Brian P. Walenz
+A src/AS_UTL/AS_UTL_fileIO.C nihh20161122Brian P. Walenz
+A src/bogart/AS_BAT_CreateUnitigs.C nihh20161121Brian P. Walenz
+A src/bogart/AS_BAT_Unitig.C nihh20161121Brian P. Walenz
+A addCopyrights-BuildData.pl nihh20161121Brian P. Walenz
+A addCopyrights.dat nihh20161121Brian P. Walenz
+A addCopyrights.pl nihh20161121Brian P. Walenz
diff --git a/addCopyrights.pl b/addCopyrights.pl
index 8f5607c..15a0c1f 100644
--- a/addCopyrights.pl
+++ b/addCopyrights.pl
@@ -258,6 +258,11 @@ foreach my $file (@filesToProcess) {
next if ($file =~ m/libboost/);
+ next if ($file =~ m/libedlib/);
+ next if ($file =~ m/libfalcon/);
+ next if ($file =~ m/libNDFalcon/);
+ next if ($file =~ m/libbacktrace/);
+
next if ($file =~ m/qsort_mt.c$/);
my $cb = "/";
@@ -273,29 +278,15 @@ foreach my $file (@filesToProcess) {
my $iskmer = 0;
$iskmer = 1 if ($file =~ m/^kmer/);
- $iskmer = 1 if ($file =~ m/meryl/);
-
- $iskmer = 1 if ($file =~ m/bitEncodings/);
- $iskmer = 1 if ($file =~ m/bitOperations/);
- $iskmer = 1 if ($file =~ m/bitPackedArray/);
- $iskmer = 1 if ($file =~ m/bitPackedFile/);
- $iskmer = 1 if ($file =~ m/bitPacking/);
- $iskmer = 1 if ($file =~ m/decodeBooleanString/);
- $iskmer = 1 if ($file =~ m/dnaAlphabets/);
- $iskmer = 1 if ($file =~ m/intervalList/);
- $iskmer = 1 if ($file =~ m/kMer/);
- $iskmer = 1 if ($file =~ m/kMerHuge/);
- $iskmer = 1 if ($file =~ m/kMerTiny/);
- $iskmer = 1 if ($file =~ m/memoryMappedFile/);
- $iskmer = 1 if ($file =~ m/memoryMappedFileTest/);
- $iskmer = 1 if ($file =~ m/readBuffer/);
- $iskmer = 1 if ($file =~ m/speedCounter/);
- $iskmer = 1 if ($file =~ m/splitToWords/);
- $iskmer = 1 if ($file =~ m/sweatShop/);
- $iskmer = 1 if ($file =~ m/testHashTable/);
- $iskmer = 1 if ($file =~ m/testRand/);
- $iskmer = 1 if ($file =~ m/testVar/);
- $iskmer = 1 if ($file =~ m/timeAndSize/);
+ #$iskmer = 1 if ($file =~ m/libmeryl/);
+ $iskmer = 1 if ($file =~ m/src\/meryl\/libkmer/);
+ $iskmer = 1 if ($file =~ m/src\/meryl\/existDB.C/);
+ $iskmer = 1 if ($file =~ m/src\/meryl\/positionDB.C/);
+
+ if ($iskmer) {
+ print STDERR "Won't process: '$file' - kmer copyrights screwed up\n";
+ next;
+ }
if (($file !~ m/\.[CHch]$/) && ($file !~ m/\.p[lm]/)) {
print STDERR "Won't process: '$file'\n";
@@ -380,7 +371,7 @@ foreach my $file (@filesToProcess) {
# If a single "/*" at the start of the line, assume this is NOT an old copyright block,
# but a copyright block (or just a comment) from some third party code.
- if ($_ eq "\/\*") {
+ if ($_ eq "/*") {
print STDERR "Foreign code found: '$file'\n";
$start = 0;
}
@@ -414,7 +405,7 @@ foreach my $file (@filesToProcess) {
if ($doForReal) {
my $perms = `stat -f %p $file`; chomp $perms; $perms = substr($perms, -3);
- rename "$file", "$file.ORIG";
+ #rename "$file", "$file.ORIG";
open(F, "> $file") or die "Failed to open '$file' for writing: $!\n";
print F @lines;
diff --git a/buildRelease.sh b/buildRelease.sh
new file mode 100644
index 0000000..a081df9
--- /dev/null
+++ b/buildRelease.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+version=$1
+
+if [ x$version = x ] ; then
+ echo usage: $0 numeric-version
+ exit
+fi
+
+git clone git at github.com:marbl/canu.git
+
+mv canu canu-$version
+
+cd canu-$version
+git tag v$version
+cd src
+gmake -j 12 > ../Darwin-amd64.out 2>&1
+cd ../..
+
+rm -f canu-$version/linux.sh
+
+echo >> canu-$version/linux.sh \#\!/bin/bash
+echo >> canu-$version/linux.sh yum install -y git
+echo >> canu-$version/linux.sh cd /build/canu-$version/src
+echo >> canu-$version/linux.sh gmake -j 12 \> ../Linux-amd64.out 2\>\&1
+echo >> canu-$version/linux.sh cd ../..
+echo >> canu-$version/linux.sh tar -cf canu-$version/README* canu-$version.Darwin-amd64.tar canu-$version/Darwin-amd64
+echo >> canu-$version/linux.sh tar -cf canu-$version/README* canu-$version.Linux-amd64.tar canu-$version/Linux-amd64
+
+chmod 755 canu-$version/linux.sh
+
+docker run -v `pwd`:/build -t -i --rm phusion/holy-build-box-64:latest /hbb_exe/activate-exec bash /build/canu-$version/linux.sh
+
+rm -rf canu-$version/*-amd64/obj
+
+xz -9v canu-$version.Darwin-amd64.tar
+xz -9v canu-$version.Linux-amd64.tar
+
+exit
diff --git a/documentation/reST-markup-hints b/documentation/reST-markup-hints
new file mode 100644
index 0000000..88ec8db
--- /dev/null
+++ b/documentation/reST-markup-hints
@@ -0,0 +1,10 @@
+
+
+http://docutils.sourceforge.net/rst.html
+
+http://docutils.sourceforge.net/docs/user/rst/quickstart.html
+http://docutils.sourceforge.net/docs/user/rst/quickref.html <- most useful
+http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#definition-lists
+
+http://rest-sphinx-memo.readthedocs.io/en/latest/ReST.html
+ - A very useful page with examples.
diff --git a/documentation/source/conf.py b/documentation/source/conf.py
index 25ceed3..35dddac 100644
--- a/documentation/source/conf.py
+++ b/documentation/source/conf.py
@@ -55,9 +55,9 @@ copyright = u'2015, Adam Phillippy, Sergey Koren, Brian Walenz'
# built documents.
#
# The short X.Y version.
-version = '1.3'
+version = '1.5'
# The full version, including alpha/beta/rc tags.
-release = '1.3'
+release = '1.5'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/documentation/source/faq.rst b/documentation/source/faq.rst
index 3863259..50cd03f 100644
--- a/documentation/source/faq.rst
+++ b/documentation/source/faq.rst
@@ -38,30 +38,42 @@ How do I run Canu on my SLURM / SGE / PBS / LSF / Torque system?
partition or queue.
To disable grid support and run only on the local machine, specify ``useGrid=false``
+
+My run stopped with the error ``'Failed to submit batch jobs'``
+-------------------------------------
+
+ The grid you run on must allow compute nodes to submit jobs. This means that if you are on a compute host, ``qsub/bsub/sbatch/etc`` must be available and working. You can test this by starting an interactive compute session and running the submit command manually (e.g. ``qsub`` on SGE, ``bsub`` on LSF, ``sbatch`` on SLURM).
+
+ If this is not the case, Canu **WILL NOT** work on your grid. You must then set ``useGrid=false`` and run on a single machine. Alternatively, you can run Canu with ``useGrid=remote`` which will stop at every submit command, list what should be submitted. You then submit these jobs manually, wait for them to complete, and run the Canu command again. This is a manual process but currently the only workaround for grids without submit support on the compute nodes.
-What parameters should I use for my genome? Sequencing type?
+What parameters should I use for my reads?
-------------------------------------
- Canu is designed to be universal on a large range of PacBio (C2-P6-C4) and Oxford Nanopore
- (R6-R9) data. You can adjust parameters to increase efficiency for your datatype:
+ Canu is designed to be universal on a large range of PacBio (C2, P4-C2, P5-C3, P6-C4) and Oxford Nanopore
+ (R6 through R9) data. Assembly quality and/or efficiency can be enhanced for specific datatypes:
**Nanopore R7 1D** and **Low Identity Reads**
- With R7 1D sequencing data, and generally for any reads lower than 80% identity, five to ten
- rounds of error correction are helpful. To run just the correction phase, use options
+ With R7 1D sequencing data, and generally for any raw reads lower than 80% identity, five to
+ ten rounds of error correction are helpful. To run just the correction phase, use options
``-correct corOutCoverage=500 corMinCoverage=0 corMhapSensitivity=high``. Use the output of
the previous run (in ``asm.correctedReads.fasta.gz``) as input to the next round.
- Once corrected, assemble with ``-nanopore-corrected <your data> errorRate=0.1 utgGraphDeviation=50``
+ Once corrected, assemble with ``-nanopore-corrected <your data> correctedErrorRate=0.3 utgGraphDeviation=50``
**Nanopore R7 2D** and **Nanopore R9 1D**
- ``errorRate=0.025``
+ Increase the maximum allowed difference in overlaps from the default of 4.5% to 7.5% with
+ ``correctedErrorRate=0.075``
**Nanopore R9 2D** and **PacBio P6**
- ``errorRate=0.013``
+ Slightly decrease the maximum allowed difference in overlaps from the default of 4.5% to 4.0%
+ with ``correctedErrorRate=0.040``
- **PacBio Sequel**
- Based on exactly one publically released `*A. thaliana* dataset <http://www.pacb.com/blog/sequel-system-data-release-arabidopsis-dataset-genome-assembly/>`_),
- ``errorRate=0.013 corMhapSensitivity=normal``
+ **Early PacBio Sequel**
+ Based on exactly one publically released *A. thaliana* `dataset
+ <http://www.pacb.com/blog/sequel-system-data-release-arabidopsis-dataset-genome-assembly/>`_,
+ slightly decrease the maximum allowed difference from the default of 4.5% to 4.0% with
+ ``correctedErrorRate=0.040 corMhapSensitivity=normal``. For recent Sequel data, the defaults
+ are appropriate.
My assembly continuity is not good, how can I improve it?
@@ -96,8 +108,12 @@ What parameters can I tweak?
-------------------------------------
For all stages:
- - ``errorRate`` is the expected error rate in _corrected_ reads. It is a meta-parameter that
- sets other parameters. It has been obsolesced and will eventually be removed.
+ - ``rawErrorRate`` is the maximum expected difference in an alignment of two _uncorrected_
+ reads. It is a meta-parameter that sets other parameters.
+
+ - ``correctedErrorRate`` is the maximum expected difference in an alignment of two _corrected_
+ reads. It is a meta-parameter that sets other parameters. (If you're used to the
+ ``errorRate`` parameter, multiply that by 3 and use it here.)
- ``minReadLength`` and ``minOverlapLength``. The defaults are to discard reads shorter than
1000bp and to not look for overlaps shorter than 500bp. Increasing ``minReadLength`` can
@@ -109,6 +125,7 @@ What parameters can I tweak?
- ``corOutCoverage`` controls how much coverage in corrected reads is generated. The default is
to target 40X, but, for various reasons, this results in 30X to 35X of reads being generated.
+
- ``corMinCoverage``, loosely, controls the quality of the corrected reads. It is the coverage
in evidence reads that is needed before a (portion of a) corrected read is reported.
Corrected reads are generated as a consensus of other reads; this is just the minimum ocverage
@@ -130,6 +147,47 @@ What parameters can I tweak?
contig is split. When this occurs, it isn't clear which overlap is 'true' - the longer one or
the slightly shorter one - and the contig is split to avoid misassemblies.
+ For polyploid genomes:
+
+ Generally, there's a couple of ways of dealing with the ploidy.
+
+ 1) **Avoid collapsing the genome** so you end up with double (assuming diploid) the genome
+ size as long as your divergence is above about 2% (for PacBio data). Below this
+ divergence, you'd end up collapsing the variations. We've used the following parameters
+ for polyploid populations (PacBio data):
+
+ ``corOutCoverage=200 correctedErrorRate=0.040 "batOptions=-dg 3 -db 3 -dr 1 -ca 500 -cp 50"``
+
+ This will output more corrected reads (than the default 40x). The latter option will be
+ more conservative at picking the error rate to use for the assembly to try to maintain
+ haplotype separation. If it works, you'll end up with an assembly >= 2x your haploid
+ genome size. Post-processing using gene information or other synteny information is
+ required to remove redunancy from this assembly.
+
+ 2) **Smash haplotypes together** and then do phasing using another approach (like HapCUT2 or
+ whatshap or others). In that case you want to do the opposite, increase the error rates
+ used for finding overlaps:
+
+ ``corOutCoverage=200 ovlErrorRate=0.15 obtErrorRate=0.15``
+
+ Error rates for trimming (``obtErrorRate``) and assembling (``batErrorRate``) can usually
+ be left as is. When trimming, reads will be trimmed using other reads in the same
+ chromosome (and probably some reads from other chromosomes). When assembling, overlaps
+ well outside the observed error rate distribution are discarded.
+
+ For low coverage:
+
+ - For less than 30X coverage, increase the alllowed difference in overlaps from 4.5% to 7.5%
+ (or more) with ``correctedErrorRate=0.075``, to adjust for inferior read correction. Canu
+ will automatically reduce ``corMinCoverage`` to zero to correct as many reads as possible.
+
+ For high coverage:
+
+ - For more than 60X coverage, decrease the allowed difference in overlaps from 4.5% to 4.0%
+ with ``correctedErrorRate=0.040``, so that only the better corrected reads are used. This is
+ primarily an optimization for speed and generally does not change assembly continuity.
+
+
My asm.contigs.fasta is empty, why?
-------------------------------------
@@ -187,11 +245,6 @@ Why do I get less corrected read data than I asked for?
What is the minimum coverage required to run Canu?
-------------------------------------
For eukaryotic genomes, coverage more than 20X is enough to outperform current hybrid methods.
- - For less than 30X coverage, we recommend using ``corMinCoverage=0 errorRate=0.035`` to correct
- as many reads as possible.
- - For more than 60X coverage, we recommend using ``errorRate=0.013`` to slightly decrease the
- error rate to use only the better reads. This is primarily an optimization for speed and
- generally does not improve (or degrade) assembly continuity.
My genome is AT (or GC) rich, do I need to adjust parameters? What about highly repetitive genomes?
@@ -205,3 +258,9 @@ My genome is AT (or GC) rich, do I need to adjust parameters? What about highly
In general, with high coverage repetitive genomes (such as plants) it can be beneficial to set
the above parameter anyway, as it will eliminate repetitive matches, speed up the assembly, and
sometime improve unitigs.
+
+
+How can I send data to you?
+-------------------------------------
+ FTP to ftp://ftp.cbcb.umd.edu/incoming/sergek. This is a write-only location that only the Canu
+ developers can see.
diff --git a/documentation/source/parameter-reference.rst b/documentation/source/parameter-reference.rst
index 27e6f26..8a0e20c 100644
--- a/documentation/source/parameter-reference.rst
+++ b/documentation/source/parameter-reference.rst
@@ -10,18 +10,77 @@ To get the most up-to-date options, run
The default values below will vary based on the input data type and genome size.
-.. _genomeSize:
+Boolean options accept true/false or 1/0.
+
+Memory sizes are assumed to be in gigabytes if no units are supplied. Values may be non-integer
+with or without a unit - 'k' for kilobytes, 'm' for megabytes, 'g' for gigabytes or 't' for
+terabytes. For example, "0.25t" is equivalent to "256g" (or simply "256").
Global Options
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The catch all category.
-errorRate <float=0.01>
- The expected error rate, as fraction error, in the corrected reads, set by default based on data type, typically not changed by the user.
+.. _errorRate:
+
+errorRate <float=unset> (OBSOLETE)
+ This parameter was removed on January 27th, 2016, and is valid only in Canu 1.4 or earlier.
+ Canu currently still accepts the :ref:`errorRate <errorRate>` parameter, but its use is strongly discouraged.
+
+ The expected error in a single corrected read. The seven error rates were then set to three times
+ this value (except for :ref:`corErrorRate <corErrorRate>`).
+
+.. _rawErrorRate:
+
+rawErrorRate <float=unset>
+ The allowed difference in an overlap between two uncorrected reads, expressed as fraction error.
+ Sets :ref:`corOvlErrorRate` and :ref:`corErrorRate`. The `rawErrorRate` typically does not need
+ to be modified. It might need to be increased if very early reads are being assembled. The
+ default is 0.300 For PacBio reads, and 0.500 for Nanopore reads.
+
+.. _correctedErrorRate:
+
+correctedErrorRate <float=unset>
+ The allowed difference in an overlap between two corrected reads, expressed as fraction error. Sets :ref:`obtOvlErrorRate`, :ref:`utgOvlErrorRate`, :ref:`obtErrorRate`, :ref:`utgErrorRate`, and :ref:`cnsErrorRate`.
+ The `correctedErrorRate` can be adjusted to account for the quality of read correction, for the amount of divergence in the sample being
+ assembled, and for the amount of sequence being assembled. The default is 0.045 for PacBio reads, and 0.144 for Nanopore reads.
+
+ For low coverage datasets (less than 30X), we recommend increasing `correctedErrorRate` slightly, by 1% or so.
-genomeSize <float=unset>
- An estimate of the size of the genome. Common suffices are allowed, for example, 3.7m or 2.8g. Required.
+ For high-coverage datasets (more than 60X), we recommend decreasing `correctedErrorRate` slighly, by 1% or so.
+
+ Raising the `correctedErrorRate` will increase run time. Likewise, decreasing `correctedErrorRate` will decrease run time, at the risk of missing overlaps and fracturing the assembly.
+
+.. _minReadLength:
+
+minReadLength <integer=1000>
+ Reads shorter than this are not loaded into the assembler. Reads output by correction and
+ trimming that are shorter than this are discarded.
+
+ Must be no smaller than minOverlapLength.
+
+ If set high enough, the gatekeeper module will halt as too many of the input reads have been
+ discarded. Set `stopOnReadQuality` to false to avoid this.
+
+.. _minOverlapLength:
+
+minOverlapLength <integer=500>
+ Overlaps shorter than this will not be discovered. Smaller values can be used to overcome lack of
+ read coverage, but will also lead to false overlaps and potential misassemblies. Larger values
+ will result in more correct assemblies, but more fragmented, assemblies.
+
+ Must be no bigger than minReadLength.
+
+.. _genomeSize:
+
+genomeSize <float=unset> *required*
+ An estimate of the size of the genome. Common suffices are allowed, for example, 3.7m or 2.8g.
+
+ The genome size estimate is used to decide how many reads to correct (via the corOutCoverage_
+ parameter) and how sensitive the mhap overlapper should be (via the mhapSensitivity_
+ parameter). It also impacts some logging, in particular, reports of NG50 sizes.
+
+.. _canuIteration:
canuIteration <internal parameter, do not use>
Which parallel iteration is being attempted.
@@ -30,13 +89,24 @@ canuIterationMax <integer=2>
Sometimes, jobs fail due to other jobs exhausting resources (memory), or by the node itself failing. In this case, canu will launch the jobs
again. This parameter controls how many times it tries.
+.. _onSuccess:
+
onSuccess <string=unset>
-onFailure <string=unset>
- On success or failure, execute the command supplied. The command will execute in the <assembly-directory> (the -d option to canu) and will be supplied with the name of the assembly (the -p option to canu) as its first and only parameter.
+ Execute the command supplied when Canu successfully completes an assembly. The command will
+ execute in the <assembly-directory> (the -d option to canu) and will be supplied with the name of
+ the assembly (the -p option to canu) as its first and only parameter.
+
+.. _onFailure:
- The 'onSuccess' command will run when canu finishes an assembly.
+onFailure <string=unset>
+ Execute the command supplied when Canu terminates abnormally. The command will execute in the
+ <assembly-directory> (the -d option to canu) and will be supplied with the name of the assembly
+ (the -p option to canu) as its first and only parameter.
- The 'onFailure' command will run when canu terminates abnormally. There are two exceptions: if a 'spec' file cannot be read, and if canu tries to access an invalid parameter. The former will be reported as a command line error, and canu will never start. The latter should never occur (and, in fact, has never occurred) except when developers are developing the software.
+ There are two exceptions when the command is not executed: if a 'spec' file cannot be read, or if
+ canu tries to access an invalid parameter. The former will be reported as a command line error,
+ and canu will never start. The latter should never occur except when developers are developing
+ the software.
Process Control
@@ -47,46 +117,22 @@ showNext <boolean=false>
command, for example, checking the output of the previous command or preparing inputs for the
next command, is still performed.
-stopBefore <string=undefined>
- Stop processing just before this stage would execute. The stage is configured, and the
- command logged to the standard output, before stopping. For grid-based stages, e.g., overlapper,
- the grid submit command is reported.
-
- If the stage has finished successfully, it will not stop.
-
- Only one stage may be supplied to stopBefore.
-
- Valid stages to stopBefore are:
- - gatekeeper
- - meryl
- - overlap
- - correction
- - overlapErrorAdjustment
- - trimReads
- - splitReads
- - unitig
- - consensusConfigure
- - consensus
- - output
-
- The default value is 'undef'.
-
stopAfter <string=undefined>
- Stop processing after this stage completes.
+ If set, Canu will stop processing after a specific stage in the pipeline finishes.
- The stage will typically stop BEFORE a summary of its processing is reported in the canu chatter.
+ Valid values for ``stopAfter`` are:
- Valid stages top stopAfter are:
- - gatekeeper
- - meryl
- - overlapConfigure
- - overlap
- - correction
- - unitig
- - consensusConfigure
- - consensus
- - consensusLoad
- - consensusFilter
+ - ``gatekeeper`` - stops after the reads are loaded into the assembler read database.
+ - ``meryl`` - stops after frequent kmers are tabulated.
+ - ``overlapConfigure`` - stops after overlap jobs are configured.
+ - ``overlap`` - stops after overlaps are generated, before they are loaded into the overlap database.
+ - ``overlapStoreConfigure`` - stops after the ``ovsMethod=parallel`` jobs are configured; has no impact for ``ovsMethod=sequential``.
+ - ``overlapStore`` - stops after overlaps are loaded into the overlap database.
+ - ``readCorrection`` - stops after corrected reads are generated.
+ - ``readTrimming`` - stops after trimmed reads are generated.
+ - ``unitig`` - stops after unitigs and contigs are created.
+ - ``consensusConfigure`` - stops after consensus jobs are configured.
+ - ``consensus`` - stops after consensus sequences are loaded into the databases.
General Options
@@ -118,6 +164,36 @@ gnuplotImageFormat <string="png">
gnuplotTested <boolean=false>
If set, skip the tests to determine if gnuplot will run, and to decide the image type to generate. This is used when gnuplot fails to run, or isn't even installed, and allows canu to continue execution without generating graphs.
+
+File Staging
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The correction stage of Canu requires random access to all the reads. Performance is greatly
+improved if the gkpStore database of reads is copied locally to each node that computes corrected
+read consensus sequences. This 'staging' is enabled by supplying a path name to fast local storage
+with the `stageDirectory` option, and, optionally, requesting access to that resource from the grid
+with the `gridEngineStageOption` option.
+
+stageDirectory <string=undefined>
+ A path to a directory local to each compute node. The directory should use an environment
+ variable specific to the grid engine to ensure that it is unique to each task.
+
+ For example, in Sun Grid Engine, `/scratch/$JOB_ID-$SGE_TASK_ID` will use both the numeric
+ job ID and the numeric task ID. In SLURM, `/scratch/$SLRUM_JOBID` accomplishes the same.
+
+ If specified on the command line, be sure to escape the dollar sign, otherwise the shell will try
+ to expand it before Canu sees the option: `stageDirectory=/scratch/\$JOB_ID-\$SGE_TASK_ID`.
+
+ If specified in a specFile, do not escape the dollar signs.
+
+gridEngineStageOption <string=undefined>
+ This string is passed to the job submission command, and is expected to request
+ local disk space on each node. It is highly grid specific. The string `DISK_SPACE`
+ will be replaced with the amount of disk space needed, in gigabytes.
+
+ On SLURM, an example is `--gres=lscratch:DISK_SPACE`
+
+
Cleanup Options
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -159,8 +235,17 @@ algorithm.
Overlapper Configuration, ovl Algorithm
---------------------------------------
+.. _corOvlErrorRate:
+.. _obtOvlErrorRate:
+.. _utgOvlErrorRate:
+.. _ovlErrorRate:
+
{prefix}OvlErrorRate <float=unset>
Overlaps above this error rate are not computed.
+ * `corOvlErrorRate` applies to overlaps generated for correcting reads;
+ * `obtOvlErrorRate` applied to overlaps generated for trimming reads;
+ * `utgOvlErrorRate` applies to overlaps generated for assembling reads.
+ These limits apply to the 'ovl' overlap algorithm and when alignments are computed for mhap overlaps with :ref:`mhapReAlign <mhapReAlign>`.
{prefix}OvlFrequentMers <string=undefined>
Do not seed overlaps with these kmers (fasta format).
@@ -201,12 +286,18 @@ Overlapper Configuration, mhap Algorithm
{prefix}MhapMerSize <integer=unset>
K-mer size for seeds in mhap.
+.. _mhapReAlign:
+
{prefix}ReAlign <boolean=false>
- Compute actual alignments from mhap overlaps; 'raw' from mhap output;
+ Compute actual alignments from mhap overlaps.
uses either obtErrorRate or ovlErrorRate, depending on which overlaps are computed)
+.. _mhapSensitivity:
+
{prefix}MhapSensitivity <string="normal">
- Coarse sensitivity level: 'normal' or 'high' or 'fast'.
+ Coarse sensitivity level: 'low', 'normal' or 'high'. Based on read coverage (which is impacted by
+ genomeSize), 'low' sensitivity is used if coverage is more than 60; 'normal' is used if coverage
+ is between 60 and 30, and 'high' is used for coverages less than 30.
Overlapper Configuration, mhap Algorithm
----------------------------------------
@@ -217,10 +308,6 @@ Overlapper Configuration, mhap Algorithm
{prefix}MMapMerSize <integer=unset>
K-mer size for seeds in minimap.
-{prefix}ReAlign <boolean=false>
- Compute actual alignments from minimap overlaps; 'raw' from mhap output;
- uses either obtErrorRate or ovlErrorRate, depending on which overlaps are computed)
-
Overlap Store
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -229,8 +316,8 @@ algorithms usually need to know all overlaps for a single read. The overlap sto
overlap, sorts them by the first ID, and stores them for quick retrieval of all overlaps for a
single read.
-ovsMemory <integer=2>
- How much memory, in gigabytes, to use for constructing overlap stores.
+ovsMemory <float>
+ How much memory, in gigabytes, to use for constructing overlap stores. Must be at least 256m or 0.25g.
ovsMethod <string="sequential">
Two construction algorithms are supported. One uses a single data stream, and is faster for small
@@ -255,8 +342,10 @@ merylThreads <integer=unset>
Overlap Based Trimming
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. _obtErrorRate:
+
obtErrorRate <float=unset>
- Stringency of overlaps to use for trimming
+ Stringency of overlaps to use for trimming reads.
trimReadsOverlap <integer=1>
Minimum overlap between evidence to make contiguous trim.
@@ -271,27 +360,42 @@ trimReadsCoverage <integer=1>
Grid Engine Support
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Canu supports Sun/Open/Univa Grid Engine (SGE), Portable Batch System (PBS), Platform Computing's Load
-Sharing Facility (LSF), and the Simple Linux Utility for Resource Management (SLURM). Most of the compute
-intensive stages can run under grid control.
+Canu directly supports most common grid scheduling systems. Under normal use, Canu will query the
+system for grid support, congigure itself for the machines available in the grid, then submit itself
+to the grid for execution. The Canu pipeline is a series of about a dozen steps that alternate
+between embarassingly parallel computations (e.g., overlap computation) and sequential bookkeeping
+steps (e.g., checking if all overlap jobs finished). This is entirely managed by Canu.
-The useGrid* options control which algorithms run in parallel on the grid.
+Canu has first class support for the various schedulers derived from Sun Grid Engine (Univa, Son of
+Grid Engine) and the Simple Linux Utility for Resource Management (SLURM), meaning that the
+devlopers have direct access to these systems. Platform Computing's Load Sharing Facility (LSF) and
+the various schedulers derived from the Portable Batch System (PBS, Torque and PBSPro) are supported
+as well, but without developer access bugs do creep in. As of Canu v1.5, support seems stable and
+working.
useGrid <boolean=true>
- Master control. If 'false', no algorithms will run under grid control. Does not change the value of the other useGrid options.
+ Master control. If 'false', no algorithms will run under grid control. Does not change the value
+ of the other useGrid options.
- If 'remote', jobs are configured for grid execution, but not submitted. A message, with commands to launch the job, is reported and canu halts execution.
+ If 'remote', jobs are configured for grid execution, but not submitted. A message, with commands
+ to launch the job, is reported and canu halts execution.
- Note that the host used to run canu for 'remote' execution must know about the grid, that is, it must be able to submit jobs to the grid.
+ Note that the host used to run canu for 'remote' execution must know about the grid, that is, it
+ must be able to submit jobs to the grid.
+
+It is also possible to enable/disable grid support for individual algorithms with options such as
+`useGridBAT`, `useGridCNS`, et cetera. This has been useful in the (far) past to prevent certain
+algorithms, notably overlap error adjustment, from running too many jobs concurrently and thrashing
+disk. Recent storage systems seem to be able to handle the load better -- computers have gotten
+faster quicker than genomes have gotten larger.
There are many options for configuring a new grid ('gridEngine*') and for configuring how canu
configures its computes to run under grid control ('gridOptions*'). The grid engine to use is
specified with the 'gridEngine' option.
gridEngine <string>
- Which grid engine to use. Auto-detected. Possible choices are 'sge', 'pbs', 'lsf' or 'slurm'.
-
- NOTE: 'lsf' support is untested.
+ Which grid engine to use. Auto-detected. Possible choices are 'sge', 'pbs', 'pbspro', 'lsf' or
+ 'slurm'.
.. _grid-engine-config:
@@ -347,6 +451,10 @@ gridOptions <string=unset>
Grid submission command options applied to all grid jobs
gridOptionsJobName <string=unset>
Grid submission command jobs name suffix
+gridOptionsBAT <string=unset>
+ Grid submission command options applied to unitig construction with the bogart algorithm
+gridOptionsGFA <string=unset>
+ Grid submission command options applied to gfa alignment and processing
gridOptionsCNS <string=unset>
Grid submission command options applied to unitig consensus jobs
gridOptionsCOR <string=unset>
@@ -385,85 +493,84 @@ assebmled, the type of processing desired, or the amount of comput resources ava
enableOEA <boolean=true>
Do overlap error adjustment - comprises two steps: read error detection (RED and overlap error adjustment (OEA
-WHERE IS OBT??
-
-
Algorithm Execution Method
--------------------------
-Each of the high compute stages can be computed either on a grid or in parallel on the local machine.
-Most algorithms will respect a given maximum memory usage.
-Most algorithms can support more than a single thread of computation.
-When the grid engine is not used, more than one task can be run at a time.
-
-BUG: Local execution doesn't pay attention to memory option.
-
-For execution locally, three parameters describe the task:
+Canu has a fairly sophisticated (or complicated, depending on if it is working or not) method for
+dividing large computes, such as read overlapping and consensus, into many smaller pieces and then
+running those pieces on a grid or in parallel on the local machine. The size of each piece is
+generally determined by the amount of memory the task is allowed to use, and this memory size --
+actually a range of memory sizes -- is set based on the genomeSize parameter, but can be set
+explicitly by the user. The same holds for the number of processors each task can use.
+For example, a genomeSize=5m would result in overlaps using between 4gb and
+8gb of memory, and between 1 and 8 processors.
+
+Given these requirements, Canu will pick a specific memory size and number of processors
+so that the maximum number of jobs will run at the same time. In the overlapper example,
+if we are running on a machine with 32gb memory and 8 processors, it is not possible to run
+8 concurrent jobs that each require 8gb memory, but it is possible to run 4 concurrent jobs
+each using 6gb memory and 2 processors.
+
+To completely specify how Canu runs algorithms, one needs to specify a maximum memory size, a
+maximum number of processors, and how many pieces to run at one time. Users can set these manually
+through the {prefix}Memory, {prefix}Threads and {prefix}Concurrency options. If they are not
+set, defaults are chosen based on genomeSize.
{prefix}Concurrency <integer=unset>
Set the number of tasks that can run at the same time, when running without grid support.
- Available prefixes are:
- - master
- - cns
- - cor
- - cormhap
- - obtmhap
- - utgmhap
- - corovl
- - obtovl
- - utgovl
- - cormmap
- - obtmmap
- - utgmmap
- - oea
- - ovb
- - ovs
- - red
-
{prefix}Threads <integer=unset>
Set the number of compute threads used per task.
- Available prefixes are:
- - master
- - bat
- - cns
- - cor
- - cormhap
- - obtmhap
- - utgmhap
- - corovl
- - obtovl
- - utgovl
- - cormmap
- - obtmmap
- - utgmmap
- - ovb
- - ovs
- - red
- - oea
-
{prefix}Memory <integer=unset>
- Set the amount of memory, in GB, to use for each job in a task.
-
- Available prefixes are:
- - master
- - bat
- - ovb
- - ovs
- - cns
- - cor
- - cormhap
- - obtmhap
- - utgmhap
- - corovl
- - obtovl
- - utgovl
- - cormmap
- - obtmmap
- - utgmmap
- - red
- - oea
+ Set the amount of memory, in gigabytes, to use for each job in a task.
+
+Available prefixes are:
+
++-------+-----------+----------------------------------------+
+| Prefix | Algorithm |
++=======+===========+========================================+
+| cor | | | Overlap generation using the |
++-------| | | 'mhap' algorithm for |
+| obt | mhap | | 'cor'=correction,, 'obt'=trimming |
++-------| | | or 'utg'=assembly. |
+| utg | | |
++-------+-----------+----------------------------------------+
+| cor | | | Overlap generation using the |
++-------| | | 'minimap' algorithm for |
+| obt | mmap | | 'cor'=correction,, 'obt'=trimming |
++-------| | | or 'utg'=assembly. |
+| utg | | |
++-------+-----------+----------------------------------------+
+| cor | | | Overlap generation using the |
++-------| | | 'overlapInCore' algorithm for |
+| obt | ovl | | 'cor'=correction,, 'obt'=trimming |
++-------| | | or 'utg'=assembly. |
+| utg | | |
++-------+-----------+----------------------------------------+
+| | ovb | Parallel overlap store bucketizing |
++-------+-----------+----------------------------------------+
+| | ovs | Parallel overlap store bucket sorting |
++-------+-----------+----------------------------------------+
+| | cor | Read correction |
++-------+-----------+----------------------------------------+
+| | red | Error detection in reads |
++-------+-----------+----------------------------------------+
+| | oea | Error adjustment in overlaps |
++-------+-----------+----------------------------------------+
+| | bat | Unitig/contig construction |
++-------+-----------+----------------------------------------+
+| | cns | Unitig/contig consensus |
++-------+-----------+----------------------------------------+
+
+For example, 'mhapMemory` would set the memory limit for computing overlaps with the mhap algorithm;
+'cormhapMemory' would set the memory limit only when mhap is used for generating overlaps used for
+correction.
+
+The 'minMemory', 'maxMemory', 'minThreads' and 'maxThreads' options will apply to all jobs, and
+can be used to artifically limit canu to a portion of the current machine. In the overlapper
+example above, setting maxThreads=4 would result in two concurrent jobs instead of four.
+
Overlap Error Adjustment
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -488,6 +595,11 @@ Unitigger
unitigger <string="bogart">
Which unitig construction algorithm to use. Only "bogart" is supported.
+.. _utgErrorRate:
+
+utgErrorRate <float=unset>
+ Stringency of overlaps used for constructing contigs. The `bogart` algorithm uses the distribution of overlap error rates to filter high error overlaps; `bogart` will never see overlaps with error higher than this parameter.
+
batOptions <unset>
Advanced options to bogart
@@ -508,28 +620,35 @@ cnsPartitionMin
cnsMaxCoverage
Limit unitig consensus to at most this coverage.
+.. _cnsErrorRate:
+
cnsErrorRate
- Possibly unused.
+ Inform the consensus genration algorithm of the amount of difference it should expect in a
+ read-to-read alignment. Typically set to :ref:`utgOvlErrorRate <utgOvlErrorRate>`. If set too
+ high, reads could be placed in an incorrect location, leading to errors in the consensus sequence.
+ If set too low, reads could be omitted from the consensus graph (or multialignment, depending on
+ algorithm), resulting in truncated consensus sequences.
+.. _correction:
Read Correction
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The first step in Canu is to find high-error overlaps and generate corrected sequences for subsequent assembly. This is currently the fastest step in Canu. By default, only the longest 40X of data (based on the specified genome size) is used for correction. Typically, some reads are trimmed during correction due to being chimeric or having erroneous sequence, resulting in a loss of 20-25% (30X output). You can force correction to be non-lossy by setting
+The first step in Canu is to find high-error overlaps and generate corrected sequences for
+subsequent assembly. This is currently the fastest step in Canu. By default, only the longest 40X of
+data (based on the specified genome size) is used for correction. Typically, some reads are trimmed
+during correction due to being chimeric or having erroneous sequence, resulting in a loss of 20-25%
+(30X output). You can force correction to be non-lossy by setting `corMinCoverage=0`, in which case
+the corrected reads output will be the same length as the input data, keeping any high-error
+unsupported bases. Canu will trim these in downstream steps before assembly.
-::
+If you have a dataset with uneven coverage or small plasmids, correcting the longest 40X may not
+give you sufficient coverage of your genome/plasmid. In these cases, you can set
+`corOutCoverage=999`, or any value greater than your total input coverage which will correct and
+assemble all input data, at the expense of runtime.
- corMinCoverage=0
-
-In which case the corrected reads output will be the same length as the input data, keeping any high-error unsupported bases. Canu will trim these in downstream steps before assembly.
-
-If you have a dataset with uneven coverage or small plasmids, correcting the longest 40X may not give you sufficient coverage of your genome/plasmid. In these cases, you can set
-
-::
-
- corOutCoverage=400
-
-Or any large value greater than your total input coverage which will correct and assemble all input data, at the expense of runtime.
+corErrorRate <integer=unset>
+ Do not use overlaps with error rate higher than this (estimated error rate for `mhap` and `minimap` overlaps).
corConsensus <string="falconpipe">
Which algorithm to use for computing read consensus sequences. Only 'falcon' and 'falconpipe' are supported.
@@ -551,15 +670,14 @@ corMaxEvidenceCoverageGlobal <string="1.0x">
corMaxEvidenceCoverageLocal <string="2.0x">
Limit reads being corrected to at most this much evidence coverage; default: 10 * estimated coverage
+.. _corOutCoverage:
+
corOutCoverage <integer=40>
Only correct the longest reads up to this coverage; default 40
corFilter <string="expensive">
Method to filter short reads from correction; 'quick' or 'expensive' or 'none'
-falconSense
- Path to fc_consensus.py or falcon_sense.bin
-
Output Filtering
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/documentation/source/quick-start.rst b/documentation/source/quick-start.rst
index 2149260..4819e93 100644
--- a/documentation/source/quick-start.rst
+++ b/documentation/source/quick-start.rst
@@ -4,44 +4,33 @@
Canu Quick Start
================
-Canu specializes in assembling PacBio or Oxford Nanopore sequences. Canu will correct the reads, then trim suspicious regions (such as remaining SMRTbell adapter), then
-assemble the corrected and cleaned reads into unitigs.
+Canu specializes in assembling PacBio or Oxford Nanopore sequences. Canu will correct the reads,
+trim suspicious regions (such as remaining SMRTbell adapter), and then assemble the corrected and
+cleaned reads into contigs and unitigs.
-Brief Introduction
--------------------
-Canu has been designed to auto-detect your resources and scale itself to fit. Two parameters let you restrict the resources used.
+For eukaryotic genomes, coverage more than 20x is enough to outperform current hybrid methods.
+Between 30x and 60x coverage is the recommended minimum. More coverage will let Canu use longer
+reads for assembly, which will result in better assemblies.
-::
+Input sequences can be FASTA or FASTQ format, uncompressed or compressed with gzip (.gz), bzip2
+(.bz2) or xz (.xz). Zip files (.zip) are not supported.
- maxMemory=XX
- maxThreads=XX
+Canu will auto-detect your resources and scale itself to fit, using all of the resources available
+(depending on the size of your assembly). You can limit memory and processors used with parameters
+:ref:`maxMemory` and :ref:`maxThreads`.
-Memory is specified in gigabytes. On a single machine, it will restrict Canu to at most this limit, on the grid, no single job will try to use more than the specified resources.
+Canu will take full advantage of any LSF/PBS/PBSPro/Torque/Slrum/SGE grid available, and do so
+automagically, even submitting itself for execution. For details, refer to the section on
+:ref:`execution`.
-The input sequences can be FASTA or FASTQ format, uncompressed, or compressed with gz, bz2 or xz.
-
-Running on the grid
-~~~~~~~~~~~~~~~~~~~~~~
-Canu is designed to run on grid environments (LSF/PBS/Torque/Slrum/SGE are supported). Currently, Canu will submit itself to the default queue with default time options. You can overwrite this behavior by providing any specific parameters you want to be used for submission as an option. Users should also specify a job name to use on the grid:
-
-::
-
- gridOptionsJobName=myassembly
- "gridOptions=--partition quick --time 2:00"
Assembling PacBio data
----------------------
-Pacific Biosciences released P6-C4 chemistry reads. You can download them
-`directly <https://s3.amazonaws.com/files.pacb.com/datasets/secondary-analysis/e-coli-k12-P6C4/p6c4_ecoli_RSII_DDR2_with_15kb_cut_E01_1.tar.gz>`_
-(7 GB) or from the
-`original page <https://github.com/PacificBiosciences/DevNet/wiki/E.-coli-Bacterial-Assembly>`_.
-You must have the Pac Bio SMRTpipe software installed to extract the reads as FASTQ.
+Pacific Biosciences released P6-C4 chemistry reads for Escherichia coli K12. You can download them
+`here <https://github.com/PacificBiosciences/DevNet/wiki/E.-coli-Bacterial-Assembly>`_, but note that you must have the `SMRTpipe software <http://www.pacb.com/support/software-downloads/>`_ installed to extract the reads as FASTQ.
-We made a 25X subset FASTQ available
-`here <http://gembox.cbcb.umd.edu/mhap/raw/ecoli_p6_25x.filtered.fastq>`_
-
-or use the following curl command:
+We made a 25X subset FASTQ available `here <http://gembox.cbcb.umd.edu/mhap/raw/ecoli_p6_25x.filtered.fastq>`_ (223MB), which can be downloaded with:
::
@@ -139,13 +128,13 @@ And finally, assemble the output of trimming, twice::
canu -assemble \
-p ecoli -d ecoli-erate-0.013 \
genomeSize=4.8m \
- errorRate=0.013 \
+ correctedErrorRate=0.039 \
-pacbio-corrected ecoli/trimming/ecoli.trimmedReads.fasta.gz
canu -assemble \
-p ecoli -d ecoli-erate-0.025 \
genomeSize=4.8m \
- errorRate=0.025 \
+ correctedErrorRate=0.075 \
-pacbio-corrected ecoli/trimming/ecoli.trimmedReads.fasta.gz
The directory layout for correction and trimming is exactly the same as when we ran all tasks in the same command.
@@ -205,18 +194,18 @@ or use the following curl command:
curl -L -o yeast.20x.fastq.gz http://gembox.cbcb.umd.edu/mhap/raw/yeast_filtered.20x.fastq.gz
-and run the assembler adding sensitive parameters (**errorRate=0.035**)::
+and run the assembler adding sensitive parameters (**correctedErrorRate=0.105**)::
canu \
-p asm -d yeast \
genomeSize=12.1m \
- errorRate=0.035 \
+ correctedErrorRate=0.105 \
-pacbio-raw yeast.20x.fastq.gz
After the run completes, we can check the assembly statistics::
- tgStoreDump -sizes -s 12100000 -T yeast/unitigging/asm.tigStore 2 -G yeast/unitigging/asm.gkpStore
+ tgStoreDump -sizes -s 12100000 -T yeast/unitigging/asm.ctgStore 2 -G yeast/unitigging/asm.gkpStore
::
diff --git a/documentation/source/tutorial.rst b/documentation/source/tutorial.rst
index be7660e..2e5ac73 100644
--- a/documentation/source/tutorial.rst
+++ b/documentation/source/tutorial.rst
@@ -32,23 +32,18 @@ such as reformatting files, but generally just executes other programs.
::
- canu \
- -d <working-directory> \
- -p <file-prefix> \
- [-s specifications] \
- [-correct | -trim | -assemble] \
- errorRate=<fraction-error> \
- genomeSize=<genome-size>\
- [parameters] \
- [-pacbio-raw <read-file>]
- [-pacbio-corrected <read-file>]
- [-nanopore-raw <read-file>]
- [-nanopore-corrected <read-file>]
-
-Two options are mandatory, -d, to set the working directory, and -p, to set the file name prefix.
-All work is performed and output appears in the working directory. The directory need not exist
-before starting. Most files in this directory have file names beginning with the file name
-prefix, however, running two canu commands in the same directory will probably lead to confusion.
+ canu [-correct | -trim | -assemble | -trim-assemble] \
+ [-s <assembly-specifications-file>] \
+ -p <assembly-prefix> \
+ -d <assembly-directory> \
+ genomeSize=<number>[g|m|k] \
+ [other-options] \
+ [-pacbio-raw | -pacbio-corrected | -nanopore-raw | -nanopore-corrected] *fastq
+
+The -p option, to set the file name prefix of intermediate and output files, is mandatory. If -d is
+not supplied, canu will run in the current directory, otherwise, Canu will create the
+`assembly-directory` and run in that directory. It is _not_ possible to run two different
+assemblies in the same directory.
The -s option will import a list of parameters from the supplied specification ('spec') file. These
parameters will be applied before any from the command line are used, providing a method for
@@ -57,6 +52,8 @@ setting commonly used parameters, but overriding them for specific assemblies.
By default, all three top-level tasks are performed. It is possible to run exactly one task by
using the -correct, -trim or -assemble options. These options can be useful if you want to correct
reads once and try many different assemblies. We do exactly that in the :ref:`quickstart`.
+Additionally, suppling pre-corrected reads with -pacbio-corrected or -nanopore-corrected
+will run only the trimming (-trim) and assembling (-assemble) stages.
Parameters are key=value pairs that configure the assembler. They set run time parameters (e.g.,
memory, threads, grid), algorithmic parameters (e.g., error rates, trimming aggressiveness), and
@@ -237,46 +234,49 @@ Fraction Error Percent Error
. .
============== =============
-Eventually, we want to have Canu take a single error rate, the error rate of a single corrected read,
-and derive all other rates from there. This is the parameter ``errorRate``. Currently, the defaults
-are 0.025 for PacBio sequences and 0.05 for Oxford Nanpore sequences. Typically, you should not need
-to modify this setting. However, the error rate does affect runtime and lowering it can
-significantly speed up your assembly. Thus, for low coverage datasets (<=30X) we recommend
-increasing the error rate slightly (by 1%, so errorRate=0.035 or PacBio) and for high-coverage
-(>=60X) datasets, we recommend decreasing it (by 1%, so errorRate=0.015 for PacBio).
-
-The following error rates are defined:
-
-errorRate
- The expected error rate in a corrected single read. This will set the remaining
- error rates implicitly.
-
-Recall there are three sets of overlaps generated: one for read correction, one for read trimming,
-and one for unitig construction. The read correction overlaps are typically generated by the 'mhap'
-overlapper, which doesn't need an error rate as it doesn't generate alignment based overlaps. The
-other two sets of overlaps are generated by overlapInCore, which is alignment based, and so requires
-a maximum error rate. Thus, out of the six possible tags, only two are currently useful for overlap
-generation:
-
-obtOvlErrorRate
- Do not compute overlaps used for trimming above this error rate. Applies to the
- standard overlapper, and realigning mhap overlaps.
-
-utgOvlErrorRate
- Do not compute overlaps used for unitig construction above this error rate. Applies
- to the standard overlapper, and realigning mhap overlaps.
-
-(ADVANCED) It is possible to convert the mhap or minimap overlaps to alignment based overlaps using
-``obtReAlign=true`` or ``ovlReAlign=true``. If so, the overlaps will be computed using
-either ``obtOvlErrorRate`` or ``utgOvlErrorRate``, depending on which overlaps are being generated.
-
-Be sure to not confuse ``obtOvlErrorRate`` with ``obtErrorRate``:
-
-obtErrorRate
- Filter overlaps during OBT's read trimming and read splitting.
-
-In `celera-assembler`_, consensus generation required an overlap. In canu, this is no longer used,
-but the ``cnsErrorRate`` option still exists.
+Canu error rates always refer to the percent difference in an alignment of two reads, not the
+percent error in a single read, and not the amount of variation in your reads. These error rates
+are used in two different ways: they are used to limit what overlaps are generated, e.g., don't
+compute overlaps that have more than 5% difference; and they are used to tell algorithms what
+overlaps to use, e.g., even though overlaps were computed to 5% difference, don't trust any above 3%
+difference.
+
+There are seven error rates. Three error rates control overlap creation (:ref:`corOvlErrorRate
+<ovlErrorRate>`, :ref:`obtOvlErrorRate <ovlErrorRate>` and :ref:`utgOvlErrorRate <ovlErrorRate>`),
+and four error rates control algorithms (:ref:`corErrorRate <corErrorRate>`, :ref:`obtErrorRate
+<obtErrorRate>`, :ref:`utgErrorRate <utgErrorRate>`, :ref:`cnsErrorRate <cnsErrorRate>`).
+
+The three error rates for overlap creation apply to the `ovl` overlap algorithm and the
+:ref:`mhapReAlign <mhapReAlign>` option used to generate alignments from `mhap` or `minimap`
+overlaps. Since `mhap` is used for generating correction overlaps, the :ref:`corOvlErrorRate
+<ovlErrorRate>` parameter is not used by default. Overlaps for trimming and assembling use the
+`ovl` algorithm, therefore, :ref:`obtOvlErrorRate <ovlErrorRate>` and :ref:`utgOvlErrorRate
+<ovlErrorRate>` are used.
+
+The four algoriothm error rates are used to select which overlaps can be used for correcting reads
+(:ref:`corErrorRate <corErrorRate>`); which overlaps can be used for trimming reads
+(:ref:`obtErrorRate <obtErrorRate>`); which overlaps can be used for assembling reads
+(:ref:`utgErrorRate <utgErrorRate>`). The last error rate, :ref:`cnsErrorRate <cnsErrorRate>`,
+tells the consensus algorithm to not trust read alignments above that value.
+
+For convenience, two meta options set the error rates used with uncorrected reads
+(:ref:`rawErrorRate <rawErrorRate>`) or used with corrected reads. (:ref:`correctedErrorRate
+<correctedErrorRate>`). The default depends on the type or read being assembled.
+
+================== ====== ========
+Parameter PacBio Nanopore
+================== ====== ========
+rawErrorRate 0.300 0.500
+correctedErrorRate 0.045 0.144
+================== ====== ========
+
+In practice, only the :ref:`correctedErrorRate <correctedErrorRate>` is usually changed.
+ * For low coverage datasets (less than 30X), we recommend increasing :ref:`correctedErrorRate <correctedErrorRate>` slightly, by 1% or so.
+ * For high-coverage datasets (more than 60X), we recommend decreasing :ref:`correctedErrorRate <correctedErrorRate>` slightly, by 1% or so.
+Raising the :ref:`correctedErrorRate <correctedErrorRate>` will increase run time. Likewise, decreasing :ref:`correctedErrorRate <correctedErrorRate>` will decrease run time, at the risk of missing overlaps and fracturing the assembly.
+
+Canu v1.4 and earlier used the :ref:`errorRate <errorRate>` parameter, which set the expected
+rate of error in a single corrected read.
.. _minimum-lengths:
diff --git a/src/AS_UTL/AS_UTL_fileIO.C b/src/AS_UTL/AS_UTL_fileIO.C
index 71fbaa5..3c3ad3c 100644
--- a/src/AS_UTL/AS_UTL_fileIO.C
+++ b/src/AS_UTL/AS_UTL_fileIO.C
@@ -163,78 +163,151 @@ AS_UTL_safeRead(FILE *file, void *buffer, const char *desc, size_t size, size_t
-// Ensure that directory 'dirname' exists. Returns true if the
-// directory needed to be created, false if it already exists.
-int
-AS_UTL_mkdir(const char *dirname) {
- struct stat st;
+#if 0
+// Reads a line, allocating space as needed. Alternate implementatioin, probably slower than the
+// getc() based one below.
+bool
+readLine(char *&L, uint32 &Llen, uint32 &Lmax, FILE *F) {
- errno = 0;
+ if ((L == NULL) || (Lmax == 0))
+ allocateArray(L, Lmax = 4, resizeArray_clearNew);
- // Stat the file. Don't fail if the file doesn't exist though.
+ L[Lmax-2] = 0;
+ L[Lmax-1] = 0;
- if ((stat(dirname, &st) != 0) && (errno != ENOENT)) {
- fprintf(stderr, "AS_UTL_mkdir()-- Couldn't stat '%s': %s\n", dirname, strerror(errno));
- exit(1);
+ fgets(L, Lmax, F);
+
+ Llen = strlen(L);
+
+ fprintf(stderr, "READ Llen %u\n", Llen);
+
+ // fgets() will always NUL-terminate the string. If the seocnd to last
+ // character exists and is not a newline, we didn't read the whole string.
+
+ while ((L[Lmax-2] != 0) && (L[Lmax-2] != '\n')) {
+ uint32 growth = 4;
+
+ assert(Llen == Lmax - 1);
+
+ resizeArray(L, Llen, Lmax, Lmax + growth); // Grow the array.
+ L[Lmax-2] = 0;
+ L[Lmax-1] = 0;
+
+ fgets(L + Llen, 1 + growth, F); // Read more bytes.
+
+ Llen += strlen(L + Llen); // How many more?
+
+ fprintf(stderr, "READ Llen %u Lmax %u '%s'\n", Llen, Lmax, L);
}
- // If file doesn't exist, make the directory, and fail horribly, or return success.
+ // Trim trailing whitespace.
- if (errno == ENOENT) {
- if (mkdir(dirname, S_IRWXU | S_IRWXG | S_IRWXO) != 0) {
- fprintf(stderr, "AS_UTL_mkdir()-- Couldn't create directory '%s': %s\n", dirname, strerror(errno));
- exit(1);
- }
+ while ((Llen > 0) && (isspace(L[Llen-1])))
+ L[--Llen] = 0;
- return(1);
+ return(true);
+}
+#endif
+
+
+
+// Reads a line of text from a file. Trims off trailing whitespace, including newlines.
+bool
+AS_UTL_readLine(char *&L, uint32 &Llen, uint32 &Lmax, FILE *F) {
+
+ if ((L == NULL) || (Lmax == 0))
+ allocateArray(L, Lmax = 4, resizeArray_clearNew);
+
+ Llen = 0;
+
+ int32 ch = getc(F);
+ uint32 growth = 4;
+
+ if (feof(F))
+ return(false);
+
+ while ((feof(F) == false) && (ch != '\n')) {
+ if (Llen >= Lmax)
+ resizeArray(L, Llen, Lmax, Lmax + growth, resizeArray_copyData | resizeArray_clearNew); // Grow the array.
+
+ L[Llen++] = ch;
+
+ ch = getc(F);
}
- // The file exists. Return that it already exists, or that it's a file and fail horribly.
+ // Terminate.
- if (S_ISDIR(st.st_mode))
- return(0);
+ L[Llen] = 0;
+
+ // Trim trailing whitespace.
+
+ while ((Llen > 0) && (isspace(L[Llen-1])))
+ L[--Llen] = 0;
- fprintf(stderr, "AS_UTL_mkdir()-- ERROR! '%s' is a file, and not a directory.\n", dirname);
- exit(1);
- return(1);
+ return(true);
}
-int
+// Ensure that directory 'dirname' exists.
+void
+AS_UTL_mkdir(const char *dirname) {
+ struct stat st;
+
+ // Stat the file. Don't fail if the file doesn't exist though.
+
+ errno = 0;
+ if ((stat(dirname, &st) != 0) && (errno != ENOENT))
+ fprintf(stderr, "AS_UTL_mkdir()-- Couldn't stat '%s': %s\n", dirname, strerror(errno)), exit(1);
+
+ // If file doesn't exist, make the directory. Fail horribly if it can't be made..
+ // Or, fail horribly if the 'directory' is a file instead.
+
+ if ((errno == ENOENT) && (mkdir(dirname, S_IRWXU | S_IRWXG | S_IRWXO) != 0))
+ fprintf(stderr, "AS_UTL_mkdir()-- Couldn't create directory '%s': %s\n", dirname, strerror(errno)), exit(1);
+
+ if ((errno != ENOENT) && (S_ISDIR(st.st_mode) == false))
+ fprintf(stderr, "AS_UTL_mkdir()-- ERROR! '%s' is a file, and not a directory.\n", dirname), exit(1);
+}
+
+
+
+void
AS_UTL_symlink(const char *pathToFile, const char *pathToLink) {
- if (AS_UTL_fileExists(pathToFile, FALSE, FALSE) == 0)
+ // Fail horribly if the file doesn't exist.
+
+ if (AS_UTL_fileExists(pathToFile, FALSE, FALSE) == false)
fprintf(stderr, "AS_UTL_symlink()-- Original file '%s' doesn't exist, won't make a link to nothing.\n",
pathToFile), exit(1);
+ // Succeed silently if the link already exists.
+
+ if (AS_UTL_fileExists(pathToLink, FALSE, FALSE) == true)
+ return;
+
+ // Nope? Make the link.
+
errno = 0;
symlink(pathToFile, pathToLink);
if (errno)
fprintf(stderr, "AS_UTL_symlink()-- Failed to make link '%s' pointing to file '%s': %s\n",
pathToLink, pathToFile, strerror(errno)), exit(1);
-
- return(0);
}
-// Remove a file, or do nothing if the file doesn't exist. Returns true if the file
-// was deleted, false if the file never existsed.
-int
+// Remove a file, or do nothing if the file doesn't exist.
+void
AS_UTL_unlink(const char *filename) {
- if (AS_UTL_fileExists(filename, FALSE, FALSE) == 0)
- return(0);
+ if (AS_UTL_fileExists(filename, FALSE, FALSE) == false)
+ return;
errno = 0;
unlink(filename);
- if (errno) {
- fprintf(stderr, "AS_UTL_unlink()-- Failed to remove file '%s': %s\n", filename, strerror(errno));
- exit(1);
- }
-
- return(1);
+ if (errno)
+ fprintf(stderr, "AS_UTL_unlink()-- Failed to remove file '%s': %s\n", filename, strerror(errno)), exit(1);
}
diff --git a/src/AS_UTL/AS_UTL_fileIO.H b/src/AS_UTL/AS_UTL_fileIO.H
index 7fddfcc..ac91c13 100644
--- a/src/AS_UTL/AS_UTL_fileIO.H
+++ b/src/AS_UTL/AS_UTL_fileIO.H
@@ -58,11 +58,13 @@ void AS_UTL_findBaseFileName(char *basename, const char *filename);
void AS_UTL_safeWrite(FILE *file, const void *buffer, const char *desc, size_t size, size_t nobj);
size_t AS_UTL_safeRead (FILE *file, void *buffer, const char *desc, size_t size, size_t nobj);
-int AS_UTL_mkdir(const char *dirname);
+bool AS_UTL_readLine(char *&L, uint32 &Llen, uint32 &Lmax, FILE *F);
-int AS_UTL_symlink(const char *pathToFile, const char *pathToLink);
+void AS_UTL_mkdir(const char *dirname);
-int AS_UTL_unlink(const char *filename);
+void AS_UTL_symlink(const char *pathToFile, const char *pathToLink);
+
+void AS_UTL_unlink(const char *filename);
int AS_UTL_fileExists(const char *path, int directory=false, int readwrite=false);
off_t AS_UTL_sizeOfFile(const char *path);
diff --git a/src/AS_UTL/AS_UTL_stackTrace.C b/src/AS_UTL/AS_UTL_stackTrace.C
index b764174..039da2e 100644
--- a/src/AS_UTL/AS_UTL_stackTrace.C
+++ b/src/AS_UTL/AS_UTL_stackTrace.C
@@ -33,11 +33,11 @@
#include "AS_global.H"
-#ifndef __CYGWIN__
- #ifndef _WIN32
- #include <execinfo.h> // backtrace
- #endif
+#if (!defined(__CYGWIN__) && !defined(_WIN31))
+#include <execinfo.h> // backtrace
#endif
+#include <execinfo.h> // backtrace
+
#include <signal.h>
#include <sys/types.h>
#include <sys/wait.h>
@@ -50,53 +50,82 @@
// Derived from
// http://sourceware.org/git/?p=glibc.git;a=blob;f=debug/segfault.c
-
+//
// Linux g++ -rdynamic -g3 -o st1 st1.C
// FreeBSD CC -o unwind unwind.C -I/usr/local/include -L/usr/local/lib -lunwind
-// (Not sure what this is about, but it was with the GDB commands at the end of this file)
-//
-// http://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html#Separate-Debug-Files
-//
-// objcopy --only-kep-debug foo foo.debug
-// strip -g foo
-// objcopy --add-gnu-debuglink=foo.debug foo
#define WRITE_STRING(S) write(2, S, strlen(S))
-void
-AS_UTL_envokeGDB(void) {
+// If set, a signal handler will be installed to call AS_UTL_catchCrash().
+// Defined by default, let the exceptions undef it.
+//
+#define INSTALL_HANDLER
-#if 0
- uint64 pid = getpid();
- uint64 cid = fork();
- char cmd[1024];
-
- if (cid != 0) {
- // Parent
- waitpid(cid, NULL, 0);
- return;
- }
- // Child
- snprintf(cmd, 1024, "gdb -quiet -silent -p " F_U64 " -batch -x commands", pid);
- system(cmd);
- exit(0);
-#endif
+#if defined(__CYGWIN__) || defined(NOBACKTRACE)
+
+// Do nothing.
+#undef INSTALL_HANDLER
+
+
+
+#elif defined(LIBBACKTRACE)
+#warning LIBBACKTRACE
+
+extern "C" {
+#include "libbacktrace/backtrace.h"
}
+backtrace_state *backtraceState = NULL;
+int
+AS_UTL_catchCrash_full(void *data,
+ uintptr_t pc,
+ const char *filename, int lineno,
+ const char *function) {
+ fprintf(stderr, "%s::%d in %s()\n", filename, lineno, function);
+ return(0); // to continue tracing
+ return(1); // to stop tracing
+}
-#ifdef LIBUNWIND
+void
+AS_UTL_catchCrash(int sig_num, siginfo_t *UNUSED(info), void *UNUSED(ctx)) {
+
+ WRITE_STRING("\nFailed with '");
+ WRITE_STRING(strsignal(sig_num));
+ WRITE_STRING("'; backtrace (libbacktrace):\n");
+ backtrace_full(backtraceState, 0, AS_UTL_catchCrash_full, NULL, NULL);
+
+ //WRITE_STRING("\nBacktrace (libbacktrace print):\n\n");
+ //backtrace_print(backtraceState, 0, stderr);
+
+ // Pass the signal through, only so a core file can get generated.
+
+ struct sigaction sa;
+
+ sa.sa_handler = SIG_DFL;
+ sigemptyset (&sa.sa_mask);
+ sa.sa_flags = 0;
+
+ sigaction(sig_num, &sa, NULL);
+
+ raise(sig_num);
+}
+
+
+
+#elif defined(LIBUNWIND)
+#warning LIBUNWIND
#include <libunwind.h>
void
-AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
+AS_UTL_catchCrash(int sig_num, siginfo_t *UNUSED(info), void *UNUSED(ctx)) {
WRITE_STRING("\nFailed with '");
WRITE_STRING(strsignal(sig_num));
@@ -131,10 +160,91 @@ AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
depth++;
}
- //WRITE_STRING("\nBacktrace (demangled):\n\n");
+ // Pass the signal through, only so a core file can get generated.
+
+ struct sigaction sa;
+
+ sa.sa_handler = SIG_DFL;
+ sigemptyset (&sa.sa_mask);
+ sa.sa_flags = 0;
+
+ sigaction(sig_num, &sa, NULL);
+
+ raise(sig_num);
+}
+
+
+
+#elif defined(BACKWARDCPP)
+#warning BACKWARDCPP
+
+#include "backward.hpp"
+
+//namespace backward {
+//backward::SignalHandling sh;
+//} // namespace backward
+
+void
+AS_UTL_catchCrash(int sig_num, siginfo_t *UNUSED(info), void *UNUSED(ctx)) {
+
+ // Report the signal we failed on, be careful to not allocate memory.
+
+ WRITE_STRING("\nFailed with '");
+ WRITE_STRING(strsignal(sig_num));
+ WRITE_STRING("'\n");
+
+ // Dump a full backtrace, even including the signal handler frames,
+ // instead of not dumping anything.
+
+ // The _fd version doesn't allocate memory, the standard backtrace_symbols
+ // does. Demangling names also allocates memory. We'll do the safe one
+ // first, just to get something, then try the others.
+
+ WRITE_STRING("\nBacktrace:\n\n");
+
+ backward::StackTrace st;
+ backward::TraceResolver tr;
+
+ st.load_here(32);
+ tr.load_stacktrace(st);
+
+ for (size_t tt=0; tt<st.size(); tt++) {
+ backward::ResolvedTrace trace = tr.resolve(st[tt]);
+
+ if (trace.source.line == 0) {
+ fprintf(stderr, "#" F_SIZE_T " in %s, function %s '%s' '%s'\n",
+ tt, trace.object_filename.c_str(), trace.object_function.c_str(),
+ trace.source.filename.c_str(), trace.source.function.c_str());
+ }
+
+ else if ((trace.source.function.c_str() == NULL) || (trace.source.function.c_str()[0] == 0)) {
+ fprintf(stderr, "#" F_SIZE_T " in %s:%d (%s)\n",
+ tt, trace.object_function.c_str(), trace.source.line, trace.object_filename.c_str());
+ }
+
+ else {
+ fprintf(stderr, "#" F_SIZE_T " in %s:%d\n",
+ tt, trace.source.function.c_str(), trace.source.line);
+ }
+
+ if (trace.inliners.size() > 0) {
+ fprintf(stderr, " " F_SIZE_T " inlined functions:\n", trace.inliners.size());
+ for (size_t ii=0; ii<trace.inliners.size(); ii++)
+ fprintf(stderr, " %s:%d\n", trace.inliners[ii].function.c_str(), trace.inliners[ii].line);
+ }
+
+ }
+
+#if 0
+ backward::Printer p;
- //WRITE_STRING("\nGDB:\n\n");
- //AS_UTL_envokeGDB();
+ p.snippet = true;
+ p.object = true;
+ p.color = false;
+ p.address = true;
+
+ p.print(st);
+#endif
// Pass the signal through, only so a core file can get generated.
@@ -149,13 +259,14 @@ AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
raise(sig_num);
}
-#elif defined(_GNU_SOURCE)
+
+
+#else
+#warning DEFAULT
void
-AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
+AS_UTL_catchCrash(int sig_num, siginfo_t *UNUSED(info), void *UNUSED(ctx)) {
void *arr[256];
-#ifndef __CYGWIN__
-#ifndef _WIN32
int32 cnt = backtrace(arr, 256);
// Report the signal we failed on, be careful to not allocate memory.
@@ -164,19 +275,20 @@ AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
WRITE_STRING(strsignal(sig_num));
WRITE_STRING("'\n");
- // Dump a full backtrace, even including the signal handler frames, instead of not dumping anything.
+ // Dump a full backtrace, even including the signal handler frames,
+ // instead of not dumping anything.
- // The _fd version doesn't allocate memory, the standard backtrace_symbols does. Demangling
- // names also allocates memory. We'll do the safe one first, just to get something, then try the
- // others.
+ // The _fd version doesn't allocate memory, the standard backtrace_symbols
+ // does. Demangling names also allocates memory. We'll do the safe one
+ // first, just to get something, then try the others.
- WRITE_STRING("\nBacktrace (mangled):\n\n");
+ WRITE_STRING("\nBacktrace (default) (mangled):\n\n");
backtrace_symbols_fd(arr, cnt, 2);
// Now that we have something, try to generate a pretty backtrace.
- WRITE_STRING("\nBacktrace (demangled):\n\n");
+ WRITE_STRING("\nBacktrace (default) (demangled):\n\n");
char **messages = backtrace_symbols(arr, cnt);
@@ -224,14 +336,6 @@ AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
}
}
- WRITE_STRING("\nGDB:\n\n");
-
- AS_UTL_envokeGDB();
-
- WRITE_STRING("\n");
-#endif
-#endif
-
// Pass the signal through, only so a core file can get generated.
struct sigaction sa;
@@ -248,16 +352,12 @@ AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) {
#endif
-void
-AS_UTL_installCrashCatcher(void) {
-
-#if (!defined(LIBUNWIND) && !defined(_GNU_SOURCE))
- // No handler, just get out of here.
- return;
-#else
+#ifdef INSTALL_HANDLER
+void
+AS_UTL_installCrashCatcher(const char *filename) {
struct sigaction sigact;
memset(&sigact, 0, sizeof(struct sigaction));
@@ -267,7 +367,7 @@ AS_UTL_installCrashCatcher(void) {
// Don't especially care if these fail or not.
- //sigaction(SIGINT, &sigact, NULL); // Interrupt
+ sigaction(SIGINT, &sigact, NULL); // Interrupt
sigaction(SIGILL, &sigact, NULL); // Illegal instruction
sigaction(SIGFPE, &sigact, NULL); // Floating point exception
@@ -275,157 +375,15 @@ AS_UTL_installCrashCatcher(void) {
sigaction(SIGBUS, &sigact, NULL); // Bus error
sigaction(SIGSEGV, &sigact, NULL); // Segmentation fault
+#ifdef LIBBACKTRACE
+ backtraceState = backtrace_create_state(filename, true, NULL, NULL);
#endif
}
+#else
+void
+AS_UTL_installCrashCatcher(const char *UNUSED(filename)) {
+}
-//
-// Proposed commands for GDB.
-//
-// print ""
-// print "info thread"
-// print ""
-// info thread
-// #
-// print ""
-// print "info proc all"
-// print ""
-// info proc all
-// #
-// print ""
-// print "maintenance info sections"
-// print ""
-// maintenance info sections
-// #
-// print ""
-// print "where"
-// print ""
-// where
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// #
-// #
-// print ""
-// print ""
-// print ""
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// #
-// print ""
-// print ""
-// print ""
-// info args
-// #
-// print ""
-// print ""
-// print ""
-// info locals
-// up
-// #
-// quit
-//
+#endif
diff --git a/src/AS_UTL/AS_UTL_stackTrace.H b/src/AS_UTL/AS_UTL_stackTrace.H
index ff163cf..9f8ce54 100644
--- a/src/AS_UTL/AS_UTL_stackTrace.H
+++ b/src/AS_UTL/AS_UTL_stackTrace.H
@@ -36,6 +36,6 @@ void
AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx);
void
-AS_UTL_installCrashCatcher(void);
+AS_UTL_installCrashCatcher(const char *filename);
#endif // AS_UTL_STACKTRACE_H
diff --git a/src/AS_UTL/bitPackedFile.C b/src/AS_UTL/bitPackedFile.C
index e77a9ea..7cf9c6f 100644
--- a/src/AS_UTL/bitPackedFile.C
+++ b/src/AS_UTL/bitPackedFile.C
@@ -132,7 +132,7 @@ bitPackedFile::bitPackedFile(char const *name, uint64 offset, bool forceTruncate
errno = 0;
lseek(_file, file_offset, SEEK_SET);
if (errno)
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' failed to seed to position %llu: %s\n", _name, file_offset, strerror(errno)), exit(1);
+ fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' failed to seed to position " F_U64 ": %s\n", _name, file_offset, strerror(errno)), exit(1);
}
// Deal with endianess. We write out some bytes (or read back some bytes) to the start of
diff --git a/src/AS_UTL/libbacktrace/LICENSE b/src/AS_UTL/libbacktrace/LICENSE
new file mode 100644
index 0000000..097d277
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/LICENSE
@@ -0,0 +1,29 @@
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+# (1) Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+
+# (2) Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+
+# (3) The name of the author may not be used to
+# endorse or promote products derived from this software without
+# specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/AS_UTL/libbacktrace/README b/src/AS_UTL/libbacktrace/README
new file mode 100644
index 0000000..bd0c420
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/README
@@ -0,0 +1,15 @@
+Ian Lance Taylor's libbacktrace
+https://github.com/ianlancetaylor/libbacktrace
+
+Minor modifications were made to make it compile without 'configure'.
+
+
+--From the original README.md:
+
+A C library that may be linked into a C/C++ program to produce symbolic backtraces
+
+Initially written by Ian Lance Taylor <iant at golang.org>
+
+The libbacktrace library is provided under a BSD license.
+See the source files for the exact license text.
+
diff --git a/src/AS_UTL/libbacktrace/atomic.c b/src/AS_UTL/libbacktrace/atomic.c
new file mode 100644
index 0000000..4f31ff3
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/atomic.c
@@ -0,0 +1,113 @@
+/* atomic.c -- Support for atomic functions if not present.
+ Copyright (C) 2013-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <sys/types.h>
+
+#include "backtrace.h"
+#include "backtrace-supported.h"
+#include "internal.h"
+
+/* This file holds implementations of the atomic functions that are
+ used if the host compiler has the sync functions but not the atomic
+ functions, as is true of versions of GCC before 4.7. */
+
+#if !defined (HAVE_ATOMIC_FUNCTIONS) && defined (HAVE_SYNC_FUNCTIONS)
+
+/* Do an atomic load of a pointer. */
+
+void *
+backtrace_atomic_load_pointer (void *arg)
+{
+ void **pp;
+ void *p;
+
+ pp = (void **) arg;
+ p = *pp;
+ while (!__sync_bool_compare_and_swap (pp, p, p))
+ p = *pp;
+ return p;
+}
+
+/* Do an atomic load of an int. */
+
+int
+backtrace_atomic_load_int (int *p)
+{
+ int i;
+
+ i = *p;
+ while (!__sync_bool_compare_and_swap (p, i, i))
+ i = *p;
+ return i;
+}
+
+/* Do an atomic store of a pointer. */
+
+void
+backtrace_atomic_store_pointer (void *arg, void *p)
+{
+ void **pp;
+ void *old;
+
+ pp = (void **) arg;
+ old = *pp;
+ while (!__sync_bool_compare_and_swap (pp, old, p))
+ old = *pp;
+}
+
+/* Do an atomic store of a size_t value. */
+
+void
+backtrace_atomic_store_size_t (size_t *p, size_t v)
+{
+ size_t old;
+
+ old = *p;
+ while (!__sync_bool_compare_and_swap (p, old, v))
+ old = *p;
+}
+
+/* Do an atomic store of a int value. */
+
+void
+backtrace_atomic_store_int (int *p, int v)
+{
+ size_t old;
+
+ old = *p;
+ while (!__sync_bool_compare_and_swap (p, old, v))
+ old = *p;
+}
+
+#endif
diff --git a/src/AS_UTL/libbacktrace/backtrace-supported.h b/src/AS_UTL/libbacktrace/backtrace-supported.h
new file mode 100644
index 0000000..7709fca
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/backtrace-supported.h
@@ -0,0 +1,66 @@
+/* backtrace-supported.h.in -- Whether stack backtrace is supported.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+/* The file backtrace-supported.h.in is used by configure to generate
+ the file backtrace-supported.h. The file backtrace-supported.h may
+ be #include'd to see whether the backtrace library will be able to
+ get a backtrace and produce symbolic information. */
+
+
+/* BACKTRACE_SUPPORTED will be #define'd as 1 if the backtrace library
+ should work, 0 if it will not. Libraries may #include this to make
+ other arrangements. */
+
+#define BACKTRACE_SUPPORTED 1
+
+/* BACKTRACE_USES_MALLOC will be #define'd as 1 if the backtrace
+ library will call malloc as it works, 0 if it will call mmap
+ instead. This may be used to determine whether it is safe to call
+ the backtrace functions from a signal handler. In general this
+ only applies to calls like backtrace and backtrace_pcinfo. It does
+ not apply to backtrace_simple, which never calls malloc. It does
+ not apply to backtrace_print, which always calls fprintf and
+ therefore malloc. */
+
+#define BACKTRACE_USES_MALLOC 0
+
+/* BACKTRACE_SUPPORTS_THREADS will be #define'd as 1 if the backtrace
+ library is configured with threading support, 0 if not. If this is
+ 0, the threaded parameter to backtrace_create_state must be passed
+ as 0. */
+
+#define BACKTRACE_SUPPORTS_THREADS 1
+
+/* BACKTRACE_SUPPORTS_DATA will be #defined'd as 1 if the backtrace_syminfo
+ will work for variables. It will always work for functions. */
+
+#define BACKTRACE_SUPPORTS_DATA 1
diff --git a/src/AS_UTL/libbacktrace/backtrace.c b/src/AS_UTL/libbacktrace/backtrace.c
new file mode 100644
index 0000000..7372a27
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/backtrace.c
@@ -0,0 +1,129 @@
+/* backtrace.c -- Entry point for stack backtrace library.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <sys/types.h>
+
+#include "unwind.h"
+#include "backtrace.h"
+#include "internal.h"
+
+/* The main backtrace_full routine. */
+
+/* Data passed through _Unwind_Backtrace. */
+
+struct backtrace_data
+{
+ /* Number of frames to skip. */
+ int skip;
+ /* Library state. */
+ struct backtrace_state *state;
+ /* Callback routine. */
+ backtrace_full_callback callback;
+ /* Error callback routine. */
+ backtrace_error_callback error_callback;
+ /* Data to pass to callback routines. */
+ void *data;
+ /* Value to return from backtrace_full. */
+ int ret;
+ /* Whether there is any memory available. */
+ int can_alloc;
+};
+
+/* Unwind library callback routine. This is passed to
+ _Unwind_Backtrace. */
+
+static _Unwind_Reason_Code
+unwind (struct _Unwind_Context *context, void *vdata)
+{
+ struct backtrace_data *bdata = (struct backtrace_data *) vdata;
+ uintptr_t pc;
+ int ip_before_insn = 0;
+
+#ifdef HAVE_GETIPINFO
+ pc = _Unwind_GetIPInfo (context, &ip_before_insn);
+#else
+ pc = _Unwind_GetIP (context);
+#endif
+
+ if (bdata->skip > 0)
+ {
+ --bdata->skip;
+ return _URC_NO_REASON;
+ }
+
+ if (!ip_before_insn)
+ --pc;
+
+ if (!bdata->can_alloc)
+ bdata->ret = bdata->callback (bdata->data, pc, NULL, 0, NULL);
+ else
+ bdata->ret = backtrace_pcinfo (bdata->state, pc, bdata->callback,
+ bdata->error_callback, bdata->data);
+ if (bdata->ret != 0)
+ return _URC_END_OF_STACK;
+
+ return _URC_NO_REASON;
+}
+
+/* Get a stack backtrace. */
+
+int
+backtrace_full (struct backtrace_state *state, int skip,
+ backtrace_full_callback callback,
+ backtrace_error_callback error_callback, void *data)
+{
+ struct backtrace_data bdata;
+ void *p;
+
+ bdata.skip = skip + 1;
+ bdata.state = state;
+ bdata.callback = callback;
+ bdata.error_callback = error_callback;
+ bdata.data = data;
+ bdata.ret = 0;
+
+ /* If we can't allocate any memory at all, don't try to produce
+ file/line information. */
+ p = backtrace_alloc (state, 4096, NULL, NULL);
+ if (p == NULL)
+ bdata.can_alloc = 0;
+ else
+ {
+ backtrace_free (state, p, 4096, NULL, NULL);
+ bdata.can_alloc = 1;
+ }
+
+ _Unwind_Backtrace (unwind, &bdata);
+ return bdata.ret;
+}
diff --git a/src/AS_UTL/libbacktrace/backtrace.h b/src/AS_UTL/libbacktrace/backtrace.h
new file mode 100644
index 0000000..fd0619d
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/backtrace.h
@@ -0,0 +1,182 @@
+/* backtrace.h -- Public header file for stack backtrace library.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#ifndef BACKTRACE_H
+#define BACKTRACE_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The backtrace state. This struct is intentionally not defined in
+ the public interface. */
+
+struct backtrace_state;
+
+/* The type of the error callback argument to backtrace functions.
+ This function, if not NULL, will be called for certain error cases.
+ The DATA argument is passed to the function that calls this one.
+ The MSG argument is an error message. The ERRNUM argument, if
+ greater than 0, holds an errno value. The MSG buffer may become
+ invalid after this function returns.
+
+ As a special case, the ERRNUM argument will be passed as -1 if no
+ debug info can be found for the executable, but the function
+ requires debug info (e.g., backtrace_full, backtrace_pcinfo). The
+ MSG in this case will be something along the lines of "no debug
+ info". Similarly, ERRNUM will be passed as -1 if there is no
+ symbol table, but the function requires a symbol table (e.g.,
+ backtrace_syminfo). This may be used as a signal that some other
+ approach should be tried. */
+
+typedef void (*backtrace_error_callback) (void *data, const char *msg,
+ int errnum);
+
+/* Create state information for the backtrace routines. This must be
+ called before any of the other routines, and its return value must
+ be passed to all of the other routines. FILENAME is the path name
+ of the executable file; if it is NULL the library will try
+ system-specific path names. If not NULL, FILENAME must point to a
+ permanent buffer. If THREADED is non-zero the state may be
+ accessed by multiple threads simultaneously, and the library will
+ use appropriate atomic operations. If THREADED is zero the state
+ may only be accessed by one thread at a time. This returns a state
+ pointer on success, NULL on error. If an error occurs, this will
+ call the ERROR_CALLBACK routine. */
+
+extern struct backtrace_state *backtrace_create_state (
+ const char *filename, int threaded,
+ backtrace_error_callback error_callback, void *data);
+
+/* The type of the callback argument to the backtrace_full function.
+ DATA is the argument passed to backtrace_full. PC is the program
+ counter. FILENAME is the name of the file containing PC, or NULL
+ if not available. LINENO is the line number in FILENAME containing
+ PC, or 0 if not available. FUNCTION is the name of the function
+ containing PC, or NULL if not available. This should return 0 to
+ continuing tracing. The FILENAME and FUNCTION buffers may become
+ invalid after this function returns. */
+
+typedef int (*backtrace_full_callback) (void *data, uintptr_t pc,
+ const char *filename, int lineno,
+ const char *function);
+
+/* Get a full stack backtrace. SKIP is the number of frames to skip;
+ passing 0 will start the trace with the function calling
+ backtrace_full. DATA is passed to the callback routine. If any
+ call to CALLBACK returns a non-zero value, the stack backtrace
+ stops, and backtrace returns that value; this may be used to limit
+ the number of stack frames desired. If all calls to CALLBACK
+ return 0, backtrace returns 0. The backtrace_full function will
+ make at least one call to either CALLBACK or ERROR_CALLBACK. This
+ function requires debug info for the executable. */
+
+extern int backtrace_full (struct backtrace_state *state, int skip,
+ backtrace_full_callback callback,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* The type of the callback argument to the backtrace_simple function.
+ DATA is the argument passed to simple_backtrace. PC is the program
+ counter. This should return 0 to continue tracing. */
+
+typedef int (*backtrace_simple_callback) (void *data, uintptr_t pc);
+
+/* Get a simple backtrace. SKIP is the number of frames to skip, as
+ in backtrace. DATA is passed to the callback routine. If any call
+ to CALLBACK returns a non-zero value, the stack backtrace stops,
+ and backtrace_simple returns that value. Otherwise
+ backtrace_simple returns 0. The backtrace_simple function will
+ make at least one call to either CALLBACK or ERROR_CALLBACK. This
+ function does not require any debug info for the executable. */
+
+extern int backtrace_simple (struct backtrace_state *state, int skip,
+ backtrace_simple_callback callback,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* Print the current backtrace in a user readable format to a FILE.
+ SKIP is the number of frames to skip, as in backtrace_full. Any
+ error messages are printed to stderr. This function requires debug
+ info for the executable. */
+
+extern void backtrace_print (struct backtrace_state *state, int skip, FILE *);
+
+/* Given PC, a program counter in the current program, call the
+ callback function with filename, line number, and function name
+ information. This will normally call the callback function exactly
+ once. However, if the PC happens to describe an inlined call, and
+ the debugging information contains the necessary information, then
+ this may call the callback function multiple times. This will make
+ at least one call to either CALLBACK or ERROR_CALLBACK. This
+ returns the first non-zero value returned by CALLBACK, or 0. */
+
+extern int backtrace_pcinfo (struct backtrace_state *state, uintptr_t pc,
+ backtrace_full_callback callback,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* The type of the callback argument to backtrace_syminfo. DATA and
+ PC are the arguments passed to backtrace_syminfo. SYMNAME is the
+ name of the symbol for the corresponding code. SYMVAL is the
+ value and SYMSIZE is the size of the symbol. SYMNAME will be NULL
+ if no error occurred but the symbol could not be found. */
+
+typedef void (*backtrace_syminfo_callback) (void *data, uintptr_t pc,
+ const char *symname,
+ uintptr_t symval,
+ uintptr_t symsize);
+
+/* Given ADDR, an address or program counter in the current program,
+ call the callback information with the symbol name and value
+ describing the function or variable in which ADDR may be found.
+ This will call either CALLBACK or ERROR_CALLBACK exactly once.
+ This returns 1 on success, 0 on failure. This function requires
+ the symbol table but does not require the debug info. Note that if
+ the symbol table is present but ADDR could not be found in the
+ table, CALLBACK will be called with a NULL SYMNAME argument.
+ Returns 1 on success, 0 on error. */
+
+extern int backtrace_syminfo (struct backtrace_state *state, uintptr_t addr,
+ backtrace_syminfo_callback callback,
+ backtrace_error_callback error_callback,
+ void *data);
+
+#ifdef __cplusplus
+} /* End extern "C". */
+#endif
+
+#endif
diff --git a/src/AS_UTL/libbacktrace/config.h b/src/AS_UTL/libbacktrace/config.h
new file mode 100644
index 0000000..20d31e1
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/config.h
@@ -0,0 +1,128 @@
+/* config.h. Generated from config.h.in by configure. */
+/* config.h.in. Generated from configure.ac by autoheader. */
+
+/* ELF size: 32 or 64 */
+#ifdef __APPLE__
+#define BACKTRACE_ELF_SIZE unused
+#else
+#define BACKTRACE_ELF_SIZE 64
+#endif
+
+/* Define to 1 if you have the __atomic functions */
+#define HAVE_ATOMIC_FUNCTIONS 1
+
+/* Define to 1 if you have the declaration of `strnlen', and to 0 if you
+ don't. */
+#define HAVE_DECL_STRNLEN 1
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define if dl_iterate_phdr is available. */
+#ifndef __APPLE__
+#define HAVE_DL_ITERATE_PHDR 1
+#endif
+
+/* Define to 1 if you have the fcntl function */
+#define HAVE_FCNTL 1
+
+/* Define if getexecname is available. */
+/* #undef HAVE_GETEXECNAME */
+
+/* Define if _Unwind_GetIPInfo is available. */
+#define HAVE_GETIPINFO 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <link.h> header file. */
+#ifndef __APPLE__
+#define HAVE_LINK_H 1
+#endif
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the __sync functions */
+#define HAVE_SYNC_FUNCTIONS 1
+
+/* Define to 1 if you have the <sys/mman.h> header file. */
+#define HAVE_SYS_MMAN_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#define LT_OBJDIR ".libs/"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "package-unused"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "package-unused version-unused"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "libbacktrace"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "version-unused"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Enable extensions on AIX 3, Interix. */
+#ifndef _ALL_SOURCE
+# define _ALL_SOURCE 1
+#endif
+/* Enable GNU extensions on systems that have them. */
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE 1
+#endif
+/* Enable threading extensions on Solaris. */
+#ifndef _POSIX_PTHREAD_SEMANTICS
+# define _POSIX_PTHREAD_SEMANTICS 1
+#endif
+/* Enable extensions on HP NonStop. */
+#ifndef _TANDEM_SOURCE
+# define _TANDEM_SOURCE 1
+#endif
+/* Enable general extensions on Solaris. */
+#ifndef __EXTENSIONS__
+# define __EXTENSIONS__ 1
+#endif
+
+
+/* Define to 1 if on MINIX. */
+/* #undef _MINIX */
+
+/* Define to 2 if the system does not provide POSIX.1 features except with
+ this defined. */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define to 1 if you need to in order for `stat' and other things to work. */
+/* #undef _POSIX_SOURCE */
diff --git a/src/AS_UTL/libbacktrace/dwarf.c b/src/AS_UTL/libbacktrace/dwarf.c
new file mode 100644
index 0000000..b182567
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/dwarf.c
@@ -0,0 +1,3123 @@
+/* dwarf.c -- Get file/line information from DWARF for backtraces.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+/* DWARF constants. */
+
+enum dwarf_tag {
+ DW_TAG_entry_point = 0x3,
+ DW_TAG_compile_unit = 0x11,
+ DW_TAG_inlined_subroutine = 0x1d,
+ DW_TAG_subprogram = 0x2e,
+};
+
+enum dwarf_form {
+ DW_FORM_addr = 0x1,
+ DW_FORM_block2 = 0x3,
+ DW_FORM_block4 = 0x4,
+ DW_FORM_data2 = 0x5,
+ DW_FORM_data4 = 0x6,
+ DW_FORM_data8 = 0x07,
+ DW_FORM_string = 0x08,
+ DW_FORM_block = 0x09,
+ DW_FORM_block1 = 0x0a,
+ DW_FORM_data1 = 0x0b,
+ DW_FORM_flag = 0x0c,
+ DW_FORM_sdata = 0x0d,
+ DW_FORM_strp = 0x0e,
+ DW_FORM_udata = 0x0f,
+ DW_FORM_ref_addr = 0x10,
+ DW_FORM_ref1 = 0x11,
+ DW_FORM_ref2 = 0x12,
+ DW_FORM_ref4 = 0x13,
+ DW_FORM_ref8 = 0x14,
+ DW_FORM_ref_udata = 0x15,
+ DW_FORM_indirect = 0x16,
+ DW_FORM_sec_offset = 0x17,
+ DW_FORM_exprloc = 0x18,
+ DW_FORM_flag_present = 0x19,
+ DW_FORM_ref_sig8 = 0x20,
+ DW_FORM_GNU_addr_index = 0x1f01,
+ DW_FORM_GNU_str_index = 0x1f02,
+ DW_FORM_GNU_ref_alt = 0x1f20,
+ DW_FORM_GNU_strp_alt = 0x1f21,
+};
+
+enum dwarf_attribute {
+ DW_AT_name = 0x3,
+ DW_AT_stmt_list = 0x10,
+ DW_AT_low_pc = 0x11,
+ DW_AT_high_pc = 0x12,
+ DW_AT_comp_dir = 0x1b,
+ DW_AT_abstract_origin = 0x31,
+ DW_AT_specification = 0x47,
+ DW_AT_ranges = 0x55,
+ DW_AT_call_file = 0x58,
+ DW_AT_call_line = 0x59,
+ DW_AT_linkage_name = 0x6e,
+ DW_AT_MIPS_linkage_name = 0x2007,
+};
+
+enum dwarf_line_number_op {
+ DW_LNS_extended_op = 0x0,
+ DW_LNS_copy = 0x1,
+ DW_LNS_advance_pc = 0x2,
+ DW_LNS_advance_line = 0x3,
+ DW_LNS_set_file = 0x4,
+ DW_LNS_set_column = 0x5,
+ DW_LNS_negate_stmt = 0x6,
+ DW_LNS_set_basic_block = 0x7,
+ DW_LNS_const_add_pc = 0x8,
+ DW_LNS_fixed_advance_pc = 0x9,
+ DW_LNS_set_prologue_end = 0xa,
+ DW_LNS_set_epilogue_begin = 0xb,
+ DW_LNS_set_isa = 0xc,
+};
+
+enum dwarf_extedned_line_number_op {
+ DW_LNE_end_sequence = 0x1,
+ DW_LNE_set_address = 0x2,
+ DW_LNE_define_file = 0x3,
+ DW_LNE_set_discriminator = 0x4,
+};
+
+#if defined(__MSDOS__) || defined(_WIN32) || defined(__OS2__) || defined (__CYGWIN__)
+# define IS_DIR_SEPARATOR(c) ((c) == '/' || (c) == '\\')
+#define HAS_DRIVE_SPEC(f) ((f)[0] && (f)[1] == ':')
+# define IS_ABSOLUTE_PATH(f) (IS_DIR_SEPARATOR(f[0]) || HAS_DRIVE_SPEC(f))
+#else
+# define IS_DIR_SEPARATOR(c) ((c) == '/')
+# define IS_ABSOLUTE_PATH(f) IS_DIR_SEPARATOR(f[0])
+#endif
+
+#if !defined(HAVE_DECL_STRNLEN) || !HAVE_DECL_STRNLEN
+
+/* If strnlen is not declared, provide our own version. */
+
+static size_t
+xstrnlen (const char *s, size_t maxlen)
+{
+ size_t i;
+
+ for (i = 0; i < maxlen; ++i)
+ if (s[i] == '\0')
+ break;
+ return i;
+}
+
+#define strnlen xstrnlen
+
+#endif
+
+/* A buffer to read DWARF info. */
+
+struct dwarf_buf
+{
+ /* Buffer name for error messages. */
+ const char *name;
+ /* Start of the buffer. */
+ const unsigned char *start;
+ /* Next byte to read. */
+ const unsigned char *buf;
+ /* The number of bytes remaining. */
+ size_t left;
+ /* Whether the data is big-endian. */
+ int is_bigendian;
+ /* Error callback routine. */
+ backtrace_error_callback error_callback;
+ /* Data for error_callback. */
+ void *data;
+ /* Non-zero if we've reported an underflow error. */
+ int reported_underflow;
+};
+
+/* A single attribute in a DWARF abbreviation. */
+
+struct attr
+{
+ /* The attribute name. */
+ enum dwarf_attribute name;
+ /* The attribute form. */
+ enum dwarf_form form;
+};
+
+/* A single DWARF abbreviation. */
+
+struct abbrev
+{
+ /* The abbrev code--the number used to refer to the abbrev. */
+ uint64_t code;
+ /* The entry tag. */
+ enum dwarf_tag tag;
+ /* Non-zero if this abbrev has child entries. */
+ int has_children;
+ /* The number of attributes. */
+ size_t num_attrs;
+ /* The attributes. */
+ struct attr *attrs;
+};
+
+/* The DWARF abbreviations for a compilation unit. This structure
+ only exists while reading the compilation unit. Most DWARF readers
+ seem to a hash table to map abbrev ID's to abbrev entries.
+ However, we primarily care about GCC, and GCC simply issues ID's in
+ numerical order starting at 1. So we simply keep a sorted vector,
+ and try to just look up the code. */
+
+struct abbrevs
+{
+ /* The number of abbrevs in the vector. */
+ size_t num_abbrevs;
+ /* The abbrevs, sorted by the code field. */
+ struct abbrev *abbrevs;
+};
+
+/* The different kinds of attribute values. */
+
+enum attr_val_encoding
+{
+ /* An address. */
+ ATTR_VAL_ADDRESS,
+ /* A unsigned integer. */
+ ATTR_VAL_UINT,
+ /* A sigd integer. */
+ ATTR_VAL_SINT,
+ /* A string. */
+ ATTR_VAL_STRING,
+ /* An offset to other data in the containing unit. */
+ ATTR_VAL_REF_UNIT,
+ /* An offset to other data within the .dwarf_info section. */
+ ATTR_VAL_REF_INFO,
+ /* An offset to data in some other section. */
+ ATTR_VAL_REF_SECTION,
+ /* A type signature. */
+ ATTR_VAL_REF_TYPE,
+ /* A block of data (not represented). */
+ ATTR_VAL_BLOCK,
+ /* An expression (not represented). */
+ ATTR_VAL_EXPR,
+};
+
+/* An attribute value. */
+
+struct attr_val
+{
+ /* How the value is stored in the field u. */
+ enum attr_val_encoding encoding;
+ union
+ {
+ /* ATTR_VAL_ADDRESS, ATTR_VAL_UINT, ATTR_VAL_REF*. */
+ uint64_t uint;
+ /* ATTR_VAL_SINT. */
+ int64_t sint;
+ /* ATTR_VAL_STRING. */
+ const char *string;
+ /* ATTR_VAL_BLOCK not stored. */
+ } u;
+};
+
+/* The line number program header. */
+
+struct line_header
+{
+ /* The version of the line number information. */
+ int version;
+ /* The minimum instruction length. */
+ unsigned int min_insn_len;
+ /* The maximum number of ops per instruction. */
+ unsigned int max_ops_per_insn;
+ /* The line base for special opcodes. */
+ int line_base;
+ /* The line range for special opcodes. */
+ unsigned int line_range;
+ /* The opcode base--the first special opcode. */
+ unsigned int opcode_base;
+ /* Opcode lengths, indexed by opcode - 1. */
+ const unsigned char *opcode_lengths;
+ /* The number of directory entries. */
+ size_t dirs_count;
+ /* The directory entries. */
+ const char **dirs;
+ /* The number of filenames. */
+ size_t filenames_count;
+ /* The filenames. */
+ const char **filenames;
+};
+
+/* Map a single PC value to a file/line. We will keep a vector of
+ these sorted by PC value. Each file/line will be correct from the
+ PC up to the PC of the next entry if there is one. We allocate one
+ extra entry at the end so that we can use bsearch. */
+
+struct line
+{
+ /* PC. */
+ uintptr_t pc;
+ /* File name. Many entries in the array are expected to point to
+ the same file name. */
+ const char *filename;
+ /* Line number. */
+ int lineno;
+ /* Index of the object in the original array read from the DWARF
+ section, before it has been sorted. The index makes it possible
+ to use Quicksort and maintain stability. */
+ int idx;
+};
+
+/* A growable vector of line number information. This is used while
+ reading the line numbers. */
+
+struct line_vector
+{
+ /* Memory. This is an array of struct line. */
+ struct backtrace_vector vec;
+ /* Number of valid mappings. */
+ size_t count;
+};
+
+/* A function described in the debug info. */
+
+struct function
+{
+ /* The name of the function. */
+ const char *name;
+ /* If this is an inlined function, the filename of the call
+ site. */
+ const char *caller_filename;
+ /* If this is an inlined function, the line number of the call
+ site. */
+ int caller_lineno;
+ /* Map PC ranges to inlined functions. */
+ struct function_addrs *function_addrs;
+ size_t function_addrs_count;
+};
+
+/* An address range for a function. This maps a PC value to a
+ specific function. */
+
+struct function_addrs
+{
+ /* Range is LOW <= PC < HIGH. */
+ uint64_t low;
+ uint64_t high;
+ /* Function for this address range. */
+ struct function *function;
+};
+
+/* A growable vector of function address ranges. */
+
+struct function_vector
+{
+ /* Memory. This is an array of struct function_addrs. */
+ struct backtrace_vector vec;
+ /* Number of address ranges present. */
+ size_t count;
+};
+
+/* A DWARF compilation unit. This only holds the information we need
+ to map a PC to a file and line. */
+
+struct unit
+{
+ /* The first entry for this compilation unit. */
+ const unsigned char *unit_data;
+ /* The length of the data for this compilation unit. */
+ size_t unit_data_len;
+ /* The offset of UNIT_DATA from the start of the information for
+ this compilation unit. */
+ size_t unit_data_offset;
+ /* DWARF version. */
+ int version;
+ /* Whether unit is DWARF64. */
+ int is_dwarf64;
+ /* Address size. */
+ int addrsize;
+ /* Offset into line number information. */
+ off_t lineoff;
+ /* Primary source file. */
+ const char *filename;
+ /* Compilation command working directory. */
+ const char *comp_dir;
+ /* Absolute file name, only set if needed. */
+ const char *abs_filename;
+ /* The abbreviations for this unit. */
+ struct abbrevs abbrevs;
+
+ /* The fields above this point are read in during initialization and
+ may be accessed freely. The fields below this point are read in
+ as needed, and therefore require care, as different threads may
+ try to initialize them simultaneously. */
+
+ /* PC to line number mapping. This is NULL if the values have not
+ been read. This is (struct line *) -1 if there was an error
+ reading the values. */
+ struct line *lines;
+ /* Number of entries in lines. */
+ size_t lines_count;
+ /* PC ranges to function. */
+ struct function_addrs *function_addrs;
+ size_t function_addrs_count;
+};
+
+/* An address range for a compilation unit. This maps a PC value to a
+ specific compilation unit. Note that we invert the representation
+ in DWARF: instead of listing the units and attaching a list of
+ ranges, we list the ranges and have each one point to the unit.
+ This lets us do a binary search to find the unit. */
+
+struct unit_addrs
+{
+ /* Range is LOW <= PC < HIGH. */
+ uint64_t low;
+ uint64_t high;
+ /* Compilation unit for this address range. */
+ struct unit *u;
+};
+
+/* A growable vector of compilation unit address ranges. */
+
+struct unit_addrs_vector
+{
+ /* Memory. This is an array of struct unit_addrs. */
+ struct backtrace_vector vec;
+ /* Number of address ranges present. */
+ size_t count;
+};
+
+/* The information we need to map a PC to a file and line. */
+
+struct dwarf_data
+{
+ /* The data for the next file we know about. */
+ struct dwarf_data *next;
+ /* The base address for this file. */
+ uintptr_t base_address;
+ /* A sorted list of address ranges. */
+ struct unit_addrs *addrs;
+ /* Number of address ranges in list. */
+ size_t addrs_count;
+ /* The unparsed .debug_info section. */
+ const unsigned char *dwarf_info;
+ size_t dwarf_info_size;
+ /* The unparsed .debug_line section. */
+ const unsigned char *dwarf_line;
+ size_t dwarf_line_size;
+ /* The unparsed .debug_ranges section. */
+ const unsigned char *dwarf_ranges;
+ size_t dwarf_ranges_size;
+ /* The unparsed .debug_str section. */
+ const unsigned char *dwarf_str;
+ size_t dwarf_str_size;
+ /* Whether the data is big-endian or not. */
+ int is_bigendian;
+ /* A vector used for function addresses. We keep this here so that
+ we can grow the vector as we read more functions. */
+ struct function_vector fvec;
+};
+
+/* Report an error for a DWARF buffer. */
+
+static void
+dwarf_buf_error (struct dwarf_buf *buf, const char *msg)
+{
+ char b[200];
+
+ snprintf (b, sizeof b, "%s in %s at %d",
+ msg, buf->name, (int) (buf->buf - buf->start));
+ buf->error_callback (buf->data, b, 0);
+}
+
+/* Require at least COUNT bytes in BUF. Return 1 if all is well, 0 on
+ error. */
+
+static int
+require (struct dwarf_buf *buf, size_t count)
+{
+ if (buf->left >= count)
+ return 1;
+
+ if (!buf->reported_underflow)
+ {
+ dwarf_buf_error (buf, "DWARF underflow");
+ buf->reported_underflow = 1;
+ }
+
+ return 0;
+}
+
+/* Advance COUNT bytes in BUF. Return 1 if all is well, 0 on
+ error. */
+
+static int
+advance (struct dwarf_buf *buf, size_t count)
+{
+ if (!require (buf, count))
+ return 0;
+ buf->buf += count;
+ buf->left -= count;
+ return 1;
+}
+
+/* Read one byte from BUF and advance 1 byte. */
+
+static unsigned char
+read_byte (struct dwarf_buf *buf)
+{
+ const unsigned char *p = buf->buf;
+
+ if (!advance (buf, 1))
+ return 0;
+ return p[0];
+}
+
+/* Read a signed char from BUF and advance 1 byte. */
+
+static signed char
+read_sbyte (struct dwarf_buf *buf)
+{
+ const unsigned char *p = buf->buf;
+
+ if (!advance (buf, 1))
+ return 0;
+ return (*p ^ 0x80) - 0x80;
+}
+
+/* Read a uint16 from BUF and advance 2 bytes. */
+
+static uint16_t
+read_uint16 (struct dwarf_buf *buf)
+{
+ const unsigned char *p = buf->buf;
+
+ if (!advance (buf, 2))
+ return 0;
+ if (buf->is_bigendian)
+ return ((uint16_t) p[0] << 8) | (uint16_t) p[1];
+ else
+ return ((uint16_t) p[1] << 8) | (uint16_t) p[0];
+}
+
+/* Read a uint32 from BUF and advance 4 bytes. */
+
+static uint32_t
+read_uint32 (struct dwarf_buf *buf)
+{
+ const unsigned char *p = buf->buf;
+
+ if (!advance (buf, 4))
+ return 0;
+ if (buf->is_bigendian)
+ return (((uint32_t) p[0] << 24) | ((uint32_t) p[1] << 16)
+ | ((uint32_t) p[2] << 8) | (uint32_t) p[3]);
+ else
+ return (((uint32_t) p[3] << 24) | ((uint32_t) p[2] << 16)
+ | ((uint32_t) p[1] << 8) | (uint32_t) p[0]);
+}
+
+/* Read a uint64 from BUF and advance 8 bytes. */
+
+static uint64_t
+read_uint64 (struct dwarf_buf *buf)
+{
+ const unsigned char *p = buf->buf;
+
+ if (!advance (buf, 8))
+ return 0;
+ if (buf->is_bigendian)
+ return (((uint64_t) p[0] << 56) | ((uint64_t) p[1] << 48)
+ | ((uint64_t) p[2] << 40) | ((uint64_t) p[3] << 32)
+ | ((uint64_t) p[4] << 24) | ((uint64_t) p[5] << 16)
+ | ((uint64_t) p[6] << 8) | (uint64_t) p[7]);
+ else
+ return (((uint64_t) p[7] << 56) | ((uint64_t) p[6] << 48)
+ | ((uint64_t) p[5] << 40) | ((uint64_t) p[4] << 32)
+ | ((uint64_t) p[3] << 24) | ((uint64_t) p[2] << 16)
+ | ((uint64_t) p[1] << 8) | (uint64_t) p[0]);
+}
+
+/* Read an offset from BUF and advance the appropriate number of
+ bytes. */
+
+static uint64_t
+read_offset (struct dwarf_buf *buf, int is_dwarf64)
+{
+ if (is_dwarf64)
+ return read_uint64 (buf);
+ else
+ return read_uint32 (buf);
+}
+
+/* Read an address from BUF and advance the appropriate number of
+ bytes. */
+
+static uint64_t
+read_address (struct dwarf_buf *buf, int addrsize)
+{
+ switch (addrsize)
+ {
+ case 1:
+ return read_byte (buf);
+ case 2:
+ return read_uint16 (buf);
+ case 4:
+ return read_uint32 (buf);
+ case 8:
+ return read_uint64 (buf);
+ default:
+ dwarf_buf_error (buf, "unrecognized address size");
+ return 0;
+ }
+}
+
+/* Return whether a value is the highest possible address, given the
+ address size. */
+
+static int
+is_highest_address (uint64_t address, int addrsize)
+{
+ switch (addrsize)
+ {
+ case 1:
+ return address == (unsigned char) -1;
+ case 2:
+ return address == (uint16_t) -1;
+ case 4:
+ return address == (uint32_t) -1;
+ case 8:
+ return address == (uint64_t) -1;
+ default:
+ return 0;
+ }
+}
+
+/* Read an unsigned LEB128 number. */
+
+static uint64_t
+read_uleb128 (struct dwarf_buf *buf)
+{
+ uint64_t ret;
+ unsigned int shift;
+ int overflow;
+ unsigned char b;
+
+ ret = 0;
+ shift = 0;
+ overflow = 0;
+ do
+ {
+ const unsigned char *p;
+
+ p = buf->buf;
+ if (!advance (buf, 1))
+ return 0;
+ b = *p;
+ if (shift < 64)
+ ret |= ((uint64_t) (b & 0x7f)) << shift;
+ else if (!overflow)
+ {
+ dwarf_buf_error (buf, "LEB128 overflows uint64_t");
+ overflow = 1;
+ }
+ shift += 7;
+ }
+ while ((b & 0x80) != 0);
+
+ return ret;
+}
+
+/* Read a signed LEB128 number. */
+
+static int64_t
+read_sleb128 (struct dwarf_buf *buf)
+{
+ uint64_t val;
+ unsigned int shift;
+ int overflow;
+ unsigned char b;
+
+ val = 0;
+ shift = 0;
+ overflow = 0;
+ do
+ {
+ const unsigned char *p;
+
+ p = buf->buf;
+ if (!advance (buf, 1))
+ return 0;
+ b = *p;
+ if (shift < 64)
+ val |= ((uint64_t) (b & 0x7f)) << shift;
+ else if (!overflow)
+ {
+ dwarf_buf_error (buf, "signed LEB128 overflows uint64_t");
+ overflow = 1;
+ }
+ shift += 7;
+ }
+ while ((b & 0x80) != 0);
+
+ if ((b & 0x40) != 0 && shift < 64)
+ val |= ((uint64_t) -1) << shift;
+
+ return (int64_t) val;
+}
+
+/* Return the length of an LEB128 number. */
+
+static size_t
+leb128_len (const unsigned char *p)
+{
+ size_t ret;
+
+ ret = 1;
+ while ((*p & 0x80) != 0)
+ {
+ ++p;
+ ++ret;
+ }
+ return ret;
+}
+
+/* Free an abbreviations structure. */
+
+static void
+free_abbrevs (struct backtrace_state *state, struct abbrevs *abbrevs,
+ backtrace_error_callback error_callback, void *data)
+{
+ size_t i;
+
+ for (i = 0; i < abbrevs->num_abbrevs; ++i)
+ backtrace_free (state, abbrevs->abbrevs[i].attrs,
+ abbrevs->abbrevs[i].num_attrs * sizeof (struct attr),
+ error_callback, data);
+ backtrace_free (state, abbrevs->abbrevs,
+ abbrevs->num_abbrevs * sizeof (struct abbrev),
+ error_callback, data);
+ abbrevs->num_abbrevs = 0;
+ abbrevs->abbrevs = NULL;
+}
+
+/* Read an attribute value. Returns 1 on success, 0 on failure. If
+ the value can be represented as a uint64_t, sets *VAL and sets
+ *IS_VALID to 1. We don't try to store the value of other attribute
+ forms, because we don't care about them. */
+
+static int
+read_attribute (enum dwarf_form form, struct dwarf_buf *buf,
+ int is_dwarf64, int version, int addrsize,
+ const unsigned char *dwarf_str, size_t dwarf_str_size,
+ struct attr_val *val)
+{
+ /* Avoid warnings about val.u.FIELD may be used uninitialized if
+ this function is inlined. The warnings aren't valid but can
+ occur because the different fields are set and used
+ conditionally. */
+ memset (val, 0, sizeof *val);
+
+ switch (form)
+ {
+ case DW_FORM_addr:
+ val->encoding = ATTR_VAL_ADDRESS;
+ val->u.uint = read_address (buf, addrsize);
+ return 1;
+ case DW_FORM_block2:
+ val->encoding = ATTR_VAL_BLOCK;
+ return advance (buf, read_uint16 (buf));
+ case DW_FORM_block4:
+ val->encoding = ATTR_VAL_BLOCK;
+ return advance (buf, read_uint32 (buf));
+ case DW_FORM_data2:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = read_uint16 (buf);
+ return 1;
+ case DW_FORM_data4:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = read_uint32 (buf);
+ return 1;
+ case DW_FORM_data8:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = read_uint64 (buf);
+ return 1;
+ case DW_FORM_string:
+ val->encoding = ATTR_VAL_STRING;
+ val->u.string = (const char *) buf->buf;
+ return advance (buf, strnlen ((const char *) buf->buf, buf->left) + 1);
+ case DW_FORM_block:
+ val->encoding = ATTR_VAL_BLOCK;
+ return advance (buf, read_uleb128 (buf));
+ case DW_FORM_block1:
+ val->encoding = ATTR_VAL_BLOCK;
+ return advance (buf, read_byte (buf));
+ case DW_FORM_data1:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = read_byte (buf);
+ return 1;
+ case DW_FORM_flag:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = read_byte (buf);
+ return 1;
+ case DW_FORM_sdata:
+ val->encoding = ATTR_VAL_SINT;
+ val->u.sint = read_sleb128 (buf);
+ return 1;
+ case DW_FORM_strp:
+ {
+ uint64_t offset;
+
+ offset = read_offset (buf, is_dwarf64);
+ if (offset >= dwarf_str_size)
+ {
+ dwarf_buf_error (buf, "DW_FORM_strp out of range");
+ return 0;
+ }
+ val->encoding = ATTR_VAL_STRING;
+ val->u.string = (const char *) dwarf_str + offset;
+ return 1;
+ }
+ case DW_FORM_udata:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = read_uleb128 (buf);
+ return 1;
+ case DW_FORM_ref_addr:
+ val->encoding = ATTR_VAL_REF_INFO;
+ if (version == 2)
+ val->u.uint = read_address (buf, addrsize);
+ else
+ val->u.uint = read_offset (buf, is_dwarf64);
+ return 1;
+ case DW_FORM_ref1:
+ val->encoding = ATTR_VAL_REF_UNIT;
+ val->u.uint = read_byte (buf);
+ return 1;
+ case DW_FORM_ref2:
+ val->encoding = ATTR_VAL_REF_UNIT;
+ val->u.uint = read_uint16 (buf);
+ return 1;
+ case DW_FORM_ref4:
+ val->encoding = ATTR_VAL_REF_UNIT;
+ val->u.uint = read_uint32 (buf);
+ return 1;
+ case DW_FORM_ref8:
+ val->encoding = ATTR_VAL_REF_UNIT;
+ val->u.uint = read_uint64 (buf);
+ return 1;
+ case DW_FORM_ref_udata:
+ val->encoding = ATTR_VAL_REF_UNIT;
+ val->u.uint = read_uleb128 (buf);
+ return 1;
+ case DW_FORM_indirect:
+ {
+ uint64_t form;
+
+ form = read_uleb128 (buf);
+ return read_attribute ((enum dwarf_form) form, buf, is_dwarf64,
+ version, addrsize, dwarf_str, dwarf_str_size,
+ val);
+ }
+ case DW_FORM_sec_offset:
+ val->encoding = ATTR_VAL_REF_SECTION;
+ val->u.uint = read_offset (buf, is_dwarf64);
+ return 1;
+ case DW_FORM_exprloc:
+ val->encoding = ATTR_VAL_EXPR;
+ return advance (buf, read_uleb128 (buf));
+ case DW_FORM_flag_present:
+ val->encoding = ATTR_VAL_UINT;
+ val->u.uint = 1;
+ return 1;
+ case DW_FORM_ref_sig8:
+ val->encoding = ATTR_VAL_REF_TYPE;
+ val->u.uint = read_uint64 (buf);
+ return 1;
+ case DW_FORM_GNU_addr_index:
+ val->encoding = ATTR_VAL_REF_SECTION;
+ val->u.uint = read_uleb128 (buf);
+ return 1;
+ case DW_FORM_GNU_str_index:
+ val->encoding = ATTR_VAL_REF_SECTION;
+ val->u.uint = read_uleb128 (buf);
+ return 1;
+ case DW_FORM_GNU_ref_alt:
+ val->encoding = ATTR_VAL_REF_SECTION;
+ val->u.uint = read_offset (buf, is_dwarf64);
+ return 1;
+ case DW_FORM_GNU_strp_alt:
+ val->encoding = ATTR_VAL_REF_SECTION;
+ val->u.uint = read_offset (buf, is_dwarf64);
+ return 1;
+ default:
+ dwarf_buf_error (buf, "unrecognized DWARF form");
+ return 0;
+ }
+}
+
+/* Compare function_addrs for qsort. When ranges are nested, make the
+ smallest one sort last. */
+
+static int
+function_addrs_compare (const void *v1, const void *v2)
+{
+ const struct function_addrs *a1 = (const struct function_addrs *) v1;
+ const struct function_addrs *a2 = (const struct function_addrs *) v2;
+
+ if (a1->low < a2->low)
+ return -1;
+ if (a1->low > a2->low)
+ return 1;
+ if (a1->high < a2->high)
+ return 1;
+ if (a1->high > a2->high)
+ return -1;
+ return strcmp (a1->function->name, a2->function->name);
+}
+
+/* Compare a PC against a function_addrs for bsearch. Note that if
+ there are multiple ranges containing PC, which one will be returned
+ is unpredictable. We compensate for that in dwarf_fileline. */
+
+static int
+function_addrs_search (const void *vkey, const void *ventry)
+{
+ const uintptr_t *key = (const uintptr_t *) vkey;
+ const struct function_addrs *entry = (const struct function_addrs *) ventry;
+ uintptr_t pc;
+
+ pc = *key;
+ if (pc < entry->low)
+ return -1;
+ else if (pc >= entry->high)
+ return 1;
+ else
+ return 0;
+}
+
+/* Add a new compilation unit address range to a vector. Returns 1 on
+ success, 0 on failure. */
+
+static int
+add_unit_addr (struct backtrace_state *state, uintptr_t base_address,
+ struct unit_addrs addrs,
+ backtrace_error_callback error_callback, void *data,
+ struct unit_addrs_vector *vec)
+{
+ struct unit_addrs *p;
+
+ /* Add in the base address of the module here, so that we can look
+ up the PC directly. */
+ addrs.low += base_address;
+ addrs.high += base_address;
+
+ /* Try to merge with the last entry. */
+ if (vec->count > 0)
+ {
+ p = (struct unit_addrs *) vec->vec.base + (vec->count - 1);
+ if ((addrs.low == p->high || addrs.low == p->high + 1)
+ && addrs.u == p->u)
+ {
+ if (addrs.high > p->high)
+ p->high = addrs.high;
+ return 1;
+ }
+ }
+
+ p = ((struct unit_addrs *)
+ backtrace_vector_grow (state, sizeof (struct unit_addrs),
+ error_callback, data, &vec->vec));
+ if (p == NULL)
+ return 0;
+
+ *p = addrs;
+ ++vec->count;
+ return 1;
+}
+
+/* Free a unit address vector. */
+
+static void
+free_unit_addrs_vector (struct backtrace_state *state,
+ struct unit_addrs_vector *vec,
+ backtrace_error_callback error_callback, void *data)
+{
+ struct unit_addrs *addrs;
+ size_t i;
+
+ addrs = (struct unit_addrs *) vec->vec.base;
+ for (i = 0; i < vec->count; ++i)
+ free_abbrevs (state, &addrs[i].u->abbrevs, error_callback, data);
+}
+
+/* Compare unit_addrs for qsort. When ranges are nested, make the
+ smallest one sort last. */
+
+static int
+unit_addrs_compare (const void *v1, const void *v2)
+{
+ const struct unit_addrs *a1 = (const struct unit_addrs *) v1;
+ const struct unit_addrs *a2 = (const struct unit_addrs *) v2;
+
+ if (a1->low < a2->low)
+ return -1;
+ if (a1->low > a2->low)
+ return 1;
+ if (a1->high < a2->high)
+ return 1;
+ if (a1->high > a2->high)
+ return -1;
+ if (a1->u->lineoff < a2->u->lineoff)
+ return -1;
+ if (a1->u->lineoff > a2->u->lineoff)
+ return 1;
+ return 0;
+}
+
+/* Compare a PC against a unit_addrs for bsearch. Note that if there
+ are multiple ranges containing PC, which one will be returned is
+ unpredictable. We compensate for that in dwarf_fileline. */
+
+static int
+unit_addrs_search (const void *vkey, const void *ventry)
+{
+ const uintptr_t *key = (const uintptr_t *) vkey;
+ const struct unit_addrs *entry = (const struct unit_addrs *) ventry;
+ uintptr_t pc;
+
+ pc = *key;
+ if (pc < entry->low)
+ return -1;
+ else if (pc >= entry->high)
+ return 1;
+ else
+ return 0;
+}
+
+/* Sort the line vector by PC. We want a stable sort here to maintain
+ the order of lines for the same PC values. Since the sequence is
+ being sorted in place, their addresses cannot be relied on to
+ maintain stability. That is the purpose of the index member. */
+
+static int
+line_compare (const void *v1, const void *v2)
+{
+ const struct line *ln1 = (const struct line *) v1;
+ const struct line *ln2 = (const struct line *) v2;
+
+ if (ln1->pc < ln2->pc)
+ return -1;
+ else if (ln1->pc > ln2->pc)
+ return 1;
+ else if (ln1->idx < ln2->idx)
+ return -1;
+ else if (ln1->idx > ln2->idx)
+ return 1;
+ else
+ return 0;
+}
+
+/* Find a PC in a line vector. We always allocate an extra entry at
+ the end of the lines vector, so that this routine can safely look
+ at the next entry. Note that when there are multiple mappings for
+ the same PC value, this will return the last one. */
+
+static int
+line_search (const void *vkey, const void *ventry)
+{
+ const uintptr_t *key = (const uintptr_t *) vkey;
+ const struct line *entry = (const struct line *) ventry;
+ uintptr_t pc;
+
+ pc = *key;
+ if (pc < entry->pc)
+ return -1;
+ else if (pc >= (entry + 1)->pc)
+ return 1;
+ else
+ return 0;
+}
+
+/* Sort the abbrevs by the abbrev code. This function is passed to
+ both qsort and bsearch. */
+
+static int
+abbrev_compare (const void *v1, const void *v2)
+{
+ const struct abbrev *a1 = (const struct abbrev *) v1;
+ const struct abbrev *a2 = (const struct abbrev *) v2;
+
+ if (a1->code < a2->code)
+ return -1;
+ else if (a1->code > a2->code)
+ return 1;
+ else
+ {
+ /* This really shouldn't happen. It means there are two
+ different abbrevs with the same code, and that means we don't
+ know which one lookup_abbrev should return. */
+ return 0;
+ }
+}
+
+/* Read the abbreviation table for a compilation unit. Returns 1 on
+ success, 0 on failure. */
+
+static int
+read_abbrevs (struct backtrace_state *state, uint64_t abbrev_offset,
+ const unsigned char *dwarf_abbrev, size_t dwarf_abbrev_size,
+ int is_bigendian, backtrace_error_callback error_callback,
+ void *data, struct abbrevs *abbrevs)
+{
+ struct dwarf_buf abbrev_buf;
+ struct dwarf_buf count_buf;
+ size_t num_abbrevs;
+
+ abbrevs->num_abbrevs = 0;
+ abbrevs->abbrevs = NULL;
+
+ if (abbrev_offset >= dwarf_abbrev_size)
+ {
+ error_callback (data, "abbrev offset out of range", 0);
+ return 0;
+ }
+
+ abbrev_buf.name = ".debug_abbrev";
+ abbrev_buf.start = dwarf_abbrev;
+ abbrev_buf.buf = dwarf_abbrev + abbrev_offset;
+ abbrev_buf.left = dwarf_abbrev_size - abbrev_offset;
+ abbrev_buf.is_bigendian = is_bigendian;
+ abbrev_buf.error_callback = error_callback;
+ abbrev_buf.data = data;
+ abbrev_buf.reported_underflow = 0;
+
+ /* Count the number of abbrevs in this list. */
+
+ count_buf = abbrev_buf;
+ num_abbrevs = 0;
+ while (read_uleb128 (&count_buf) != 0)
+ {
+ if (count_buf.reported_underflow)
+ return 0;
+ ++num_abbrevs;
+ // Skip tag.
+ read_uleb128 (&count_buf);
+ // Skip has_children.
+ read_byte (&count_buf);
+ // Skip attributes.
+ while (read_uleb128 (&count_buf) != 0)
+ read_uleb128 (&count_buf);
+ // Skip form of last attribute.
+ read_uleb128 (&count_buf);
+ }
+
+ if (count_buf.reported_underflow)
+ return 0;
+
+ if (num_abbrevs == 0)
+ return 1;
+
+ abbrevs->num_abbrevs = num_abbrevs;
+ abbrevs->abbrevs = ((struct abbrev *)
+ backtrace_alloc (state,
+ num_abbrevs * sizeof (struct abbrev),
+ error_callback, data));
+ if (abbrevs->abbrevs == NULL)
+ return 0;
+ memset (abbrevs->abbrevs, 0, num_abbrevs * sizeof (struct abbrev));
+
+ num_abbrevs = 0;
+ while (1)
+ {
+ uint64_t code;
+ struct abbrev a;
+ size_t num_attrs;
+ struct attr *attrs;
+
+ if (abbrev_buf.reported_underflow)
+ goto fail;
+
+ code = read_uleb128 (&abbrev_buf);
+ if (code == 0)
+ break;
+
+ a.code = code;
+ a.tag = (enum dwarf_tag) read_uleb128 (&abbrev_buf);
+ a.has_children = read_byte (&abbrev_buf);
+
+ count_buf = abbrev_buf;
+ num_attrs = 0;
+ while (read_uleb128 (&count_buf) != 0)
+ {
+ ++num_attrs;
+ read_uleb128 (&count_buf);
+ }
+
+ if (num_attrs == 0)
+ {
+ attrs = NULL;
+ read_uleb128 (&abbrev_buf);
+ read_uleb128 (&abbrev_buf);
+ }
+ else
+ {
+ attrs = ((struct attr *)
+ backtrace_alloc (state, num_attrs * sizeof *attrs,
+ error_callback, data));
+ if (attrs == NULL)
+ goto fail;
+ num_attrs = 0;
+ while (1)
+ {
+ uint64_t name;
+ uint64_t form;
+
+ name = read_uleb128 (&abbrev_buf);
+ form = read_uleb128 (&abbrev_buf);
+ if (name == 0)
+ break;
+ attrs[num_attrs].name = (enum dwarf_attribute) name;
+ attrs[num_attrs].form = (enum dwarf_form) form;
+ ++num_attrs;
+ }
+ }
+
+ a.num_attrs = num_attrs;
+ a.attrs = attrs;
+
+ abbrevs->abbrevs[num_abbrevs] = a;
+ ++num_abbrevs;
+ }
+
+ backtrace_qsort (abbrevs->abbrevs, abbrevs->num_abbrevs,
+ sizeof (struct abbrev), abbrev_compare);
+
+ return 1;
+
+ fail:
+ free_abbrevs (state, abbrevs, error_callback, data);
+ return 0;
+}
+
+/* Return the abbrev information for an abbrev code. */
+
+static const struct abbrev *
+lookup_abbrev (struct abbrevs *abbrevs, uint64_t code,
+ backtrace_error_callback error_callback, void *data)
+{
+ struct abbrev key;
+ void *p;
+
+ /* With GCC, where abbrevs are simply numbered in order, we should
+ be able to just look up the entry. */
+ if (code - 1 < abbrevs->num_abbrevs
+ && abbrevs->abbrevs[code - 1].code == code)
+ return &abbrevs->abbrevs[code - 1];
+
+ /* Otherwise we have to search. */
+ memset (&key, 0, sizeof key);
+ key.code = code;
+ p = bsearch (&key, abbrevs->abbrevs, abbrevs->num_abbrevs,
+ sizeof (struct abbrev), abbrev_compare);
+ if (p == NULL)
+ {
+ error_callback (data, "invalid abbreviation code", 0);
+ return NULL;
+ }
+ return (const struct abbrev *) p;
+}
+
+/* Add non-contiguous address ranges for a compilation unit. Returns
+ 1 on success, 0 on failure. */
+
+static int
+add_unit_ranges (struct backtrace_state *state, uintptr_t base_address,
+ struct unit *u, uint64_t ranges, uint64_t base,
+ int is_bigendian, const unsigned char *dwarf_ranges,
+ size_t dwarf_ranges_size,
+ backtrace_error_callback error_callback, void *data,
+ struct unit_addrs_vector *addrs)
+{
+ struct dwarf_buf ranges_buf;
+
+ if (ranges >= dwarf_ranges_size)
+ {
+ error_callback (data, "ranges offset out of range", 0);
+ return 0;
+ }
+
+ ranges_buf.name = ".debug_ranges";
+ ranges_buf.start = dwarf_ranges;
+ ranges_buf.buf = dwarf_ranges + ranges;
+ ranges_buf.left = dwarf_ranges_size - ranges;
+ ranges_buf.is_bigendian = is_bigendian;
+ ranges_buf.error_callback = error_callback;
+ ranges_buf.data = data;
+ ranges_buf.reported_underflow = 0;
+
+ while (1)
+ {
+ uint64_t low;
+ uint64_t high;
+
+ if (ranges_buf.reported_underflow)
+ return 0;
+
+ low = read_address (&ranges_buf, u->addrsize);
+ high = read_address (&ranges_buf, u->addrsize);
+
+ if (low == 0 && high == 0)
+ break;
+
+ if (is_highest_address (low, u->addrsize))
+ base = high;
+ else
+ {
+ struct unit_addrs a;
+
+ a.low = low + base;
+ a.high = high + base;
+ a.u = u;
+ if (!add_unit_addr (state, base_address, a, error_callback, data,
+ addrs))
+ return 0;
+ }
+ }
+
+ if (ranges_buf.reported_underflow)
+ return 0;
+
+ return 1;
+}
+
+/* Find the address range covered by a compilation unit, reading from
+ UNIT_BUF and adding values to U. Returns 1 if all data could be
+ read, 0 if there is some error. */
+
+static int
+find_address_ranges (struct backtrace_state *state, uintptr_t base_address,
+ struct dwarf_buf *unit_buf,
+ const unsigned char *dwarf_str, size_t dwarf_str_size,
+ const unsigned char *dwarf_ranges,
+ size_t dwarf_ranges_size,
+ int is_bigendian, backtrace_error_callback error_callback,
+ void *data, struct unit *u,
+ struct unit_addrs_vector *addrs)
+{
+ while (unit_buf->left > 0)
+ {
+ uint64_t code;
+ const struct abbrev *abbrev;
+ uint64_t lowpc;
+ int have_lowpc;
+ uint64_t highpc;
+ int have_highpc;
+ int highpc_is_relative;
+ uint64_t ranges;
+ int have_ranges;
+ size_t i;
+
+ code = read_uleb128 (unit_buf);
+ if (code == 0)
+ return 1;
+
+ abbrev = lookup_abbrev (&u->abbrevs, code, error_callback, data);
+ if (abbrev == NULL)
+ return 0;
+
+ lowpc = 0;
+ have_lowpc = 0;
+ highpc = 0;
+ have_highpc = 0;
+ highpc_is_relative = 0;
+ ranges = 0;
+ have_ranges = 0;
+ for (i = 0; i < abbrev->num_attrs; ++i)
+ {
+ struct attr_val val;
+
+ if (!read_attribute (abbrev->attrs[i].form, unit_buf,
+ u->is_dwarf64, u->version, u->addrsize,
+ dwarf_str, dwarf_str_size, &val))
+ return 0;
+
+ switch (abbrev->attrs[i].name)
+ {
+ case DW_AT_low_pc:
+ if (val.encoding == ATTR_VAL_ADDRESS)
+ {
+ lowpc = val.u.uint;
+ have_lowpc = 1;
+ }
+ break;
+
+ case DW_AT_high_pc:
+ if (val.encoding == ATTR_VAL_ADDRESS)
+ {
+ highpc = val.u.uint;
+ have_highpc = 1;
+ }
+ else if (val.encoding == ATTR_VAL_UINT)
+ {
+ highpc = val.u.uint;
+ have_highpc = 1;
+ highpc_is_relative = 1;
+ }
+ break;
+
+ case DW_AT_ranges:
+ if (val.encoding == ATTR_VAL_UINT
+ || val.encoding == ATTR_VAL_REF_SECTION)
+ {
+ ranges = val.u.uint;
+ have_ranges = 1;
+ }
+ break;
+
+ case DW_AT_stmt_list:
+ if (abbrev->tag == DW_TAG_compile_unit
+ && (val.encoding == ATTR_VAL_UINT
+ || val.encoding == ATTR_VAL_REF_SECTION))
+ u->lineoff = val.u.uint;
+ break;
+
+ case DW_AT_name:
+ if (abbrev->tag == DW_TAG_compile_unit
+ && val.encoding == ATTR_VAL_STRING)
+ u->filename = val.u.string;
+ break;
+
+ case DW_AT_comp_dir:
+ if (abbrev->tag == DW_TAG_compile_unit
+ && val.encoding == ATTR_VAL_STRING)
+ u->comp_dir = val.u.string;
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ if (abbrev->tag == DW_TAG_compile_unit
+ || abbrev->tag == DW_TAG_subprogram)
+ {
+ if (have_ranges)
+ {
+ if (!add_unit_ranges (state, base_address, u, ranges, lowpc,
+ is_bigendian, dwarf_ranges,
+ dwarf_ranges_size, error_callback,
+ data, addrs))
+ return 0;
+ }
+ else if (have_lowpc && have_highpc)
+ {
+ struct unit_addrs a;
+
+ if (highpc_is_relative)
+ highpc += lowpc;
+ a.low = lowpc;
+ a.high = highpc;
+ a.u = u;
+
+ if (!add_unit_addr (state, base_address, a, error_callback, data,
+ addrs))
+ return 0;
+ }
+
+ /* If we found the PC range in the DW_TAG_compile_unit, we
+ can stop now. */
+ if (abbrev->tag == DW_TAG_compile_unit
+ && (have_ranges || (have_lowpc && have_highpc)))
+ return 1;
+ }
+
+ if (abbrev->has_children)
+ {
+ if (!find_address_ranges (state, base_address, unit_buf,
+ dwarf_str, dwarf_str_size,
+ dwarf_ranges, dwarf_ranges_size,
+ is_bigendian, error_callback, data,
+ u, addrs))
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+/* Build a mapping from address ranges to the compilation units where
+ the line number information for that range can be found. Returns 1
+ on success, 0 on failure. */
+
+static int
+build_address_map (struct backtrace_state *state, uintptr_t base_address,
+ const unsigned char *dwarf_info, size_t dwarf_info_size,
+ const unsigned char *dwarf_abbrev, size_t dwarf_abbrev_size,
+ const unsigned char *dwarf_ranges, size_t dwarf_ranges_size,
+ const unsigned char *dwarf_str, size_t dwarf_str_size,
+ int is_bigendian, backtrace_error_callback error_callback,
+ void *data, struct unit_addrs_vector *addrs)
+{
+ struct dwarf_buf info;
+ struct abbrevs abbrevs;
+
+ memset (&addrs->vec, 0, sizeof addrs->vec);
+ addrs->count = 0;
+
+ /* Read through the .debug_info section. FIXME: Should we use the
+ .debug_aranges section? gdb and addr2line don't use it, but I'm
+ not sure why. */
+
+ info.name = ".debug_info";
+ info.start = dwarf_info;
+ info.buf = dwarf_info;
+ info.left = dwarf_info_size;
+ info.is_bigendian = is_bigendian;
+ info.error_callback = error_callback;
+ info.data = data;
+ info.reported_underflow = 0;
+
+ memset (&abbrevs, 0, sizeof abbrevs);
+ while (info.left > 0)
+ {
+ const unsigned char *unit_data_start;
+ uint64_t len;
+ int is_dwarf64;
+ struct dwarf_buf unit_buf;
+ int version;
+ uint64_t abbrev_offset;
+ int addrsize;
+ struct unit *u;
+
+ if (info.reported_underflow)
+ goto fail;
+
+ unit_data_start = info.buf;
+
+ is_dwarf64 = 0;
+ len = read_uint32 (&info);
+ if (len == 0xffffffff)
+ {
+ len = read_uint64 (&info);
+ is_dwarf64 = 1;
+ }
+
+ unit_buf = info;
+ unit_buf.left = len;
+
+ if (!advance (&info, len))
+ goto fail;
+
+ version = read_uint16 (&unit_buf);
+ if (version < 2 || version > 4)
+ {
+ dwarf_buf_error (&unit_buf, "unrecognized DWARF version");
+ goto fail;
+ }
+
+ abbrev_offset = read_offset (&unit_buf, is_dwarf64);
+ if (!read_abbrevs (state, abbrev_offset, dwarf_abbrev, dwarf_abbrev_size,
+ is_bigendian, error_callback, data, &abbrevs))
+ goto fail;
+
+ addrsize = read_byte (&unit_buf);
+
+ u = ((struct unit *)
+ backtrace_alloc (state, sizeof *u, error_callback, data));
+ if (u == NULL)
+ goto fail;
+ u->unit_data = unit_buf.buf;
+ u->unit_data_len = unit_buf.left;
+ u->unit_data_offset = unit_buf.buf - unit_data_start;
+ u->version = version;
+ u->is_dwarf64 = is_dwarf64;
+ u->addrsize = addrsize;
+ u->filename = NULL;
+ u->comp_dir = NULL;
+ u->abs_filename = NULL;
+ u->lineoff = 0;
+ u->abbrevs = abbrevs;
+ memset (&abbrevs, 0, sizeof abbrevs);
+
+ /* The actual line number mappings will be read as needed. */
+ u->lines = NULL;
+ u->lines_count = 0;
+ u->function_addrs = NULL;
+ u->function_addrs_count = 0;
+
+ if (!find_address_ranges (state, base_address, &unit_buf,
+ dwarf_str, dwarf_str_size,
+ dwarf_ranges, dwarf_ranges_size,
+ is_bigendian, error_callback, data,
+ u, addrs))
+ {
+ free_abbrevs (state, &u->abbrevs, error_callback, data);
+ backtrace_free (state, u, sizeof *u, error_callback, data);
+ goto fail;
+ }
+
+ if (unit_buf.reported_underflow)
+ {
+ free_abbrevs (state, &u->abbrevs, error_callback, data);
+ backtrace_free (state, u, sizeof *u, error_callback, data);
+ goto fail;
+ }
+ }
+ if (info.reported_underflow)
+ goto fail;
+
+ return 1;
+
+ fail:
+ free_abbrevs (state, &abbrevs, error_callback, data);
+ free_unit_addrs_vector (state, addrs, error_callback, data);
+ return 0;
+}
+
+/* Add a new mapping to the vector of line mappings that we are
+ building. Returns 1 on success, 0 on failure. */
+
+static int
+add_line (struct backtrace_state *state, struct dwarf_data *ddata,
+ uintptr_t pc, const char *filename, int lineno,
+ backtrace_error_callback error_callback, void *data,
+ struct line_vector *vec)
+{
+ struct line *ln;
+
+ /* If we are adding the same mapping, ignore it. This can happen
+ when using discriminators. */
+ if (vec->count > 0)
+ {
+ ln = (struct line *) vec->vec.base + (vec->count - 1);
+ if (pc == ln->pc && filename == ln->filename && lineno == ln->lineno)
+ return 1;
+ }
+
+ ln = ((struct line *)
+ backtrace_vector_grow (state, sizeof (struct line), error_callback,
+ data, &vec->vec));
+ if (ln == NULL)
+ return 0;
+
+ /* Add in the base address here, so that we can look up the PC
+ directly. */
+ ln->pc = pc + ddata->base_address;
+
+ ln->filename = filename;
+ ln->lineno = lineno;
+ ln->idx = vec->count;
+
+ ++vec->count;
+
+ return 1;
+}
+
+/* Free the line header information. If FREE_FILENAMES is true we
+ free the file names themselves, otherwise we leave them, as there
+ may be line structures pointing to them. */
+
+static void
+free_line_header (struct backtrace_state *state, struct line_header *hdr,
+ backtrace_error_callback error_callback, void *data)
+{
+ backtrace_free (state, hdr->dirs, hdr->dirs_count * sizeof (const char *),
+ error_callback, data);
+ backtrace_free (state, hdr->filenames,
+ hdr->filenames_count * sizeof (char *),
+ error_callback, data);
+}
+
+/* Read the line header. Return 1 on success, 0 on failure. */
+
+static int
+read_line_header (struct backtrace_state *state, struct unit *u,
+ int is_dwarf64, struct dwarf_buf *line_buf,
+ struct line_header *hdr)
+{
+ uint64_t hdrlen;
+ struct dwarf_buf hdr_buf;
+ const unsigned char *p;
+ const unsigned char *pend;
+ size_t i;
+
+ hdr->version = read_uint16 (line_buf);
+ if (hdr->version < 2 || hdr->version > 4)
+ {
+ dwarf_buf_error (line_buf, "unsupported line number version");
+ return 0;
+ }
+
+ hdrlen = read_offset (line_buf, is_dwarf64);
+
+ hdr_buf = *line_buf;
+ hdr_buf.left = hdrlen;
+
+ if (!advance (line_buf, hdrlen))
+ return 0;
+
+ hdr->min_insn_len = read_byte (&hdr_buf);
+ if (hdr->version < 4)
+ hdr->max_ops_per_insn = 1;
+ else
+ hdr->max_ops_per_insn = read_byte (&hdr_buf);
+
+ /* We don't care about default_is_stmt. */
+ read_byte (&hdr_buf);
+
+ hdr->line_base = read_sbyte (&hdr_buf);
+ hdr->line_range = read_byte (&hdr_buf);
+
+ hdr->opcode_base = read_byte (&hdr_buf);
+ hdr->opcode_lengths = hdr_buf.buf;
+ if (!advance (&hdr_buf, hdr->opcode_base - 1))
+ return 0;
+
+ /* Count the number of directory entries. */
+ hdr->dirs_count = 0;
+ p = hdr_buf.buf;
+ pend = p + hdr_buf.left;
+ while (p < pend && *p != '\0')
+ {
+ p += strnlen((const char *) p, pend - p) + 1;
+ ++hdr->dirs_count;
+ }
+
+ hdr->dirs = ((const char **)
+ backtrace_alloc (state,
+ hdr->dirs_count * sizeof (const char *),
+ line_buf->error_callback, line_buf->data));
+ if (hdr->dirs == NULL)
+ return 0;
+
+ i = 0;
+ while (*hdr_buf.buf != '\0')
+ {
+ if (hdr_buf.reported_underflow)
+ return 0;
+
+ hdr->dirs[i] = (const char *) hdr_buf.buf;
+ ++i;
+ if (!advance (&hdr_buf,
+ strnlen ((const char *) hdr_buf.buf, hdr_buf.left) + 1))
+ return 0;
+ }
+ if (!advance (&hdr_buf, 1))
+ return 0;
+
+ /* Count the number of file entries. */
+ hdr->filenames_count = 0;
+ p = hdr_buf.buf;
+ pend = p + hdr_buf.left;
+ while (p < pend && *p != '\0')
+ {
+ p += strnlen ((const char *) p, pend - p) + 1;
+ p += leb128_len (p);
+ p += leb128_len (p);
+ p += leb128_len (p);
+ ++hdr->filenames_count;
+ }
+
+ hdr->filenames = ((const char **)
+ backtrace_alloc (state,
+ hdr->filenames_count * sizeof (char *),
+ line_buf->error_callback,
+ line_buf->data));
+ if (hdr->filenames == NULL)
+ return 0;
+ i = 0;
+ while (*hdr_buf.buf != '\0')
+ {
+ const char *filename;
+ uint64_t dir_index;
+
+ if (hdr_buf.reported_underflow)
+ return 0;
+
+ filename = (const char *) hdr_buf.buf;
+ if (!advance (&hdr_buf,
+ strnlen ((const char *) hdr_buf.buf, hdr_buf.left) + 1))
+ return 0;
+ dir_index = read_uleb128 (&hdr_buf);
+ if (IS_ABSOLUTE_PATH (filename)
+ || (dir_index == 0 && u->comp_dir == NULL))
+ hdr->filenames[i] = filename;
+ else
+ {
+ const char *dir;
+ size_t dir_len;
+ size_t filename_len;
+ char *s;
+
+ if (dir_index == 0)
+ dir = u->comp_dir;
+ else if (dir_index - 1 < hdr->dirs_count)
+ dir = hdr->dirs[dir_index - 1];
+ else
+ {
+ dwarf_buf_error (line_buf,
+ ("invalid directory index in "
+ "line number program header"));
+ return 0;
+ }
+ dir_len = strlen (dir);
+ filename_len = strlen (filename);
+ s = ((char *)
+ backtrace_alloc (state, dir_len + filename_len + 2,
+ line_buf->error_callback, line_buf->data));
+ if (s == NULL)
+ return 0;
+ memcpy (s, dir, dir_len);
+ /* FIXME: If we are on a DOS-based file system, and the
+ directory or the file name use backslashes, then we
+ should use a backslash here. */
+ s[dir_len] = '/';
+ memcpy (s + dir_len + 1, filename, filename_len + 1);
+ hdr->filenames[i] = s;
+ }
+
+ /* Ignore the modification time and size. */
+ read_uleb128 (&hdr_buf);
+ read_uleb128 (&hdr_buf);
+
+ ++i;
+ }
+
+ if (hdr_buf.reported_underflow)
+ return 0;
+
+ return 1;
+}
+
+/* Read the line program, adding line mappings to VEC. Return 1 on
+ success, 0 on failure. */
+
+static int
+read_line_program (struct backtrace_state *state, struct dwarf_data *ddata,
+ struct unit *u, const struct line_header *hdr,
+ struct dwarf_buf *line_buf, struct line_vector *vec)
+{
+ uint64_t address;
+ unsigned int op_index;
+ const char *reset_filename;
+ const char *filename;
+ int lineno;
+
+ address = 0;
+ op_index = 0;
+ if (hdr->filenames_count > 0)
+ reset_filename = hdr->filenames[0];
+ else
+ reset_filename = "";
+ filename = reset_filename;
+ lineno = 1;
+ while (line_buf->left > 0)
+ {
+ unsigned int op;
+
+ op = read_byte (line_buf);
+ if (op >= hdr->opcode_base)
+ {
+ unsigned int advance;
+
+ /* Special opcode. */
+ op -= hdr->opcode_base;
+ advance = op / hdr->line_range;
+ address += (hdr->min_insn_len * (op_index + advance)
+ / hdr->max_ops_per_insn);
+ op_index = (op_index + advance) % hdr->max_ops_per_insn;
+ lineno += hdr->line_base + (int) (op % hdr->line_range);
+ add_line (state, ddata, address, filename, lineno,
+ line_buf->error_callback, line_buf->data, vec);
+ }
+ else if (op == DW_LNS_extended_op)
+ {
+ uint64_t len;
+
+ len = read_uleb128 (line_buf);
+ op = read_byte (line_buf);
+ switch (op)
+ {
+ case DW_LNE_end_sequence:
+ /* FIXME: Should we mark the high PC here? It seems
+ that we already have that information from the
+ compilation unit. */
+ address = 0;
+ op_index = 0;
+ filename = reset_filename;
+ lineno = 1;
+ break;
+ case DW_LNE_set_address:
+ address = read_address (line_buf, u->addrsize);
+ break;
+ case DW_LNE_define_file:
+ {
+ const char *f;
+ unsigned int dir_index;
+
+ f = (const char *) line_buf->buf;
+ if (!advance (line_buf, strnlen (f, line_buf->left) + 1))
+ return 0;
+ dir_index = read_uleb128 (line_buf);
+ /* Ignore that time and length. */
+ read_uleb128 (line_buf);
+ read_uleb128 (line_buf);
+ if (IS_ABSOLUTE_PATH (f))
+ filename = f;
+ else
+ {
+ const char *dir;
+ size_t dir_len;
+ size_t f_len;
+ char *p;
+
+ if (dir_index == 0)
+ dir = u->comp_dir;
+ else if (dir_index - 1 < hdr->dirs_count)
+ dir = hdr->dirs[dir_index - 1];
+ else
+ {
+ dwarf_buf_error (line_buf,
+ ("invalid directory index "
+ "in line number program"));
+ return 0;
+ }
+ dir_len = strlen (dir);
+ f_len = strlen (f);
+ p = ((char *)
+ backtrace_alloc (state, dir_len + f_len + 2,
+ line_buf->error_callback,
+ line_buf->data));
+ if (p == NULL)
+ return 0;
+ memcpy (p, dir, dir_len);
+ /* FIXME: If we are on a DOS-based file system,
+ and the directory or the file name use
+ backslashes, then we should use a backslash
+ here. */
+ p[dir_len] = '/';
+ memcpy (p + dir_len + 1, f, f_len + 1);
+ filename = p;
+ }
+ }
+ break;
+ case DW_LNE_set_discriminator:
+ /* We don't care about discriminators. */
+ read_uleb128 (line_buf);
+ break;
+ default:
+ if (!advance (line_buf, len - 1))
+ return 0;
+ break;
+ }
+ }
+ else
+ {
+ switch (op)
+ {
+ case DW_LNS_copy:
+ add_line (state, ddata, address, filename, lineno,
+ line_buf->error_callback, line_buf->data, vec);
+ break;
+ case DW_LNS_advance_pc:
+ {
+ uint64_t advance;
+
+ advance = read_uleb128 (line_buf);
+ address += (hdr->min_insn_len * (op_index + advance)
+ / hdr->max_ops_per_insn);
+ op_index = (op_index + advance) % hdr->max_ops_per_insn;
+ }
+ break;
+ case DW_LNS_advance_line:
+ lineno += (int) read_sleb128 (line_buf);
+ break;
+ case DW_LNS_set_file:
+ {
+ uint64_t fileno;
+
+ fileno = read_uleb128 (line_buf);
+ if (fileno == 0)
+ filename = "";
+ else
+ {
+ if (fileno - 1 >= hdr->filenames_count)
+ {
+ dwarf_buf_error (line_buf,
+ ("invalid file number in "
+ "line number program"));
+ return 0;
+ }
+ filename = hdr->filenames[fileno - 1];
+ }
+ }
+ break;
+ case DW_LNS_set_column:
+ read_uleb128 (line_buf);
+ break;
+ case DW_LNS_negate_stmt:
+ break;
+ case DW_LNS_set_basic_block:
+ break;
+ case DW_LNS_const_add_pc:
+ {
+ unsigned int advance;
+
+ op = 255 - hdr->opcode_base;
+ advance = op / hdr->line_range;
+ address += (hdr->min_insn_len * (op_index + advance)
+ / hdr->max_ops_per_insn);
+ op_index = (op_index + advance) % hdr->max_ops_per_insn;
+ }
+ break;
+ case DW_LNS_fixed_advance_pc:
+ address += read_uint16 (line_buf);
+ op_index = 0;
+ break;
+ case DW_LNS_set_prologue_end:
+ break;
+ case DW_LNS_set_epilogue_begin:
+ break;
+ case DW_LNS_set_isa:
+ read_uleb128 (line_buf);
+ break;
+ default:
+ {
+ unsigned int i;
+
+ for (i = hdr->opcode_lengths[op - 1]; i > 0; --i)
+ read_uleb128 (line_buf);
+ }
+ break;
+ }
+ }
+ }
+
+ return 1;
+}
+
+/* Read the line number information for a compilation unit. Returns 1
+ on success, 0 on failure. */
+
+static int
+read_line_info (struct backtrace_state *state, struct dwarf_data *ddata,
+ backtrace_error_callback error_callback, void *data,
+ struct unit *u, struct line_header *hdr, struct line **lines,
+ size_t *lines_count)
+{
+ struct line_vector vec;
+ struct dwarf_buf line_buf;
+ uint64_t len;
+ int is_dwarf64;
+ struct line *ln;
+
+ memset (&vec.vec, 0, sizeof vec.vec);
+ vec.count = 0;
+
+ memset (hdr, 0, sizeof *hdr);
+
+ if (u->lineoff != (off_t) (size_t) u->lineoff
+ || (size_t) u->lineoff >= ddata->dwarf_line_size)
+ {
+ error_callback (data, "unit line offset out of range", 0);
+ goto fail;
+ }
+
+ line_buf.name = ".debug_line";
+ line_buf.start = ddata->dwarf_line;
+ line_buf.buf = ddata->dwarf_line + u->lineoff;
+ line_buf.left = ddata->dwarf_line_size - u->lineoff;
+ line_buf.is_bigendian = ddata->is_bigendian;
+ line_buf.error_callback = error_callback;
+ line_buf.data = data;
+ line_buf.reported_underflow = 0;
+
+ is_dwarf64 = 0;
+ len = read_uint32 (&line_buf);
+ if (len == 0xffffffff)
+ {
+ len = read_uint64 (&line_buf);
+ is_dwarf64 = 1;
+ }
+ line_buf.left = len;
+
+ if (!read_line_header (state, u, is_dwarf64, &line_buf, hdr))
+ goto fail;
+
+ if (!read_line_program (state, ddata, u, hdr, &line_buf, &vec))
+ goto fail;
+
+ if (line_buf.reported_underflow)
+ goto fail;
+
+ if (vec.count == 0)
+ {
+ /* This is not a failure in the sense of a generating an error,
+ but it is a failure in that sense that we have no useful
+ information. */
+ goto fail;
+ }
+
+ /* Allocate one extra entry at the end. */
+ ln = ((struct line *)
+ backtrace_vector_grow (state, sizeof (struct line), error_callback,
+ data, &vec.vec));
+ if (ln == NULL)
+ goto fail;
+ ln->pc = (uintptr_t) -1;
+ ln->filename = NULL;
+ ln->lineno = 0;
+ ln->idx = 0;
+
+ if (!backtrace_vector_release (state, &vec.vec, error_callback, data))
+ goto fail;
+
+ ln = (struct line *) vec.vec.base;
+ backtrace_qsort (ln, vec.count, sizeof (struct line), line_compare);
+
+ *lines = ln;
+ *lines_count = vec.count;
+
+ return 1;
+
+ fail:
+ vec.vec.alc += vec.vec.size;
+ vec.vec.size = 0;
+ backtrace_vector_release (state, &vec.vec, error_callback, data);
+ free_line_header (state, hdr, error_callback, data);
+ *lines = (struct line *) (uintptr_t) -1;
+ *lines_count = 0;
+ return 0;
+}
+
+/* Read the name of a function from a DIE referenced by a
+ DW_AT_abstract_origin or DW_AT_specification tag. OFFSET is within
+ the same compilation unit. */
+
+static const char *
+read_referenced_name (struct dwarf_data *ddata, struct unit *u,
+ uint64_t offset, backtrace_error_callback error_callback,
+ void *data)
+{
+ struct dwarf_buf unit_buf;
+ uint64_t code;
+ const struct abbrev *abbrev;
+ const char *ret;
+ size_t i;
+
+ /* OFFSET is from the start of the data for this compilation unit.
+ U->unit_data is the data, but it starts U->unit_data_offset bytes
+ from the beginning. */
+
+ if (offset < u->unit_data_offset
+ || offset - u->unit_data_offset >= u->unit_data_len)
+ {
+ error_callback (data,
+ "abstract origin or specification out of range",
+ 0);
+ return NULL;
+ }
+
+ offset -= u->unit_data_offset;
+
+ unit_buf.name = ".debug_info";
+ unit_buf.start = ddata->dwarf_info;
+ unit_buf.buf = u->unit_data + offset;
+ unit_buf.left = u->unit_data_len - offset;
+ unit_buf.is_bigendian = ddata->is_bigendian;
+ unit_buf.error_callback = error_callback;
+ unit_buf.data = data;
+ unit_buf.reported_underflow = 0;
+
+ code = read_uleb128 (&unit_buf);
+ if (code == 0)
+ {
+ dwarf_buf_error (&unit_buf, "invalid abstract origin or specification");
+ return NULL;
+ }
+
+ abbrev = lookup_abbrev (&u->abbrevs, code, error_callback, data);
+ if (abbrev == NULL)
+ return NULL;
+
+ ret = NULL;
+ for (i = 0; i < abbrev->num_attrs; ++i)
+ {
+ struct attr_val val;
+
+ if (!read_attribute (abbrev->attrs[i].form, &unit_buf,
+ u->is_dwarf64, u->version, u->addrsize,
+ ddata->dwarf_str, ddata->dwarf_str_size,
+ &val))
+ return NULL;
+
+ switch (abbrev->attrs[i].name)
+ {
+ case DW_AT_name:
+ /* We prefer the linkage name if get one. */
+ if (val.encoding == ATTR_VAL_STRING)
+ ret = val.u.string;
+ break;
+
+ case DW_AT_linkage_name:
+ case DW_AT_MIPS_linkage_name:
+ if (val.encoding == ATTR_VAL_STRING)
+ return val.u.string;
+ break;
+
+ case DW_AT_specification:
+ if (abbrev->attrs[i].form == DW_FORM_ref_addr
+ || abbrev->attrs[i].form == DW_FORM_ref_sig8)
+ {
+ /* This refers to a specification defined in some other
+ compilation unit. We can handle this case if we
+ must, but it's harder. */
+ break;
+ }
+ if (val.encoding == ATTR_VAL_UINT
+ || val.encoding == ATTR_VAL_REF_UNIT)
+ {
+ const char *name;
+
+ name = read_referenced_name (ddata, u, val.u.uint,
+ error_callback, data);
+ if (name != NULL)
+ ret = name;
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ return ret;
+}
+
+/* Add a single range to U that maps to function. Returns 1 on
+ success, 0 on error. */
+
+static int
+add_function_range (struct backtrace_state *state, struct dwarf_data *ddata,
+ struct function *function, uint64_t lowpc, uint64_t highpc,
+ backtrace_error_callback error_callback,
+ void *data, struct function_vector *vec)
+{
+ struct function_addrs *p;
+
+ /* Add in the base address here, so that we can look up the PC
+ directly. */
+ lowpc += ddata->base_address;
+ highpc += ddata->base_address;
+
+ if (vec->count > 0)
+ {
+ p = (struct function_addrs *) vec->vec.base + vec->count - 1;
+ if ((lowpc == p->high || lowpc == p->high + 1)
+ && function == p->function)
+ {
+ if (highpc > p->high)
+ p->high = highpc;
+ return 1;
+ }
+ }
+
+ p = ((struct function_addrs *)
+ backtrace_vector_grow (state, sizeof (struct function_addrs),
+ error_callback, data, &vec->vec));
+ if (p == NULL)
+ return 0;
+
+ p->low = lowpc;
+ p->high = highpc;
+ p->function = function;
+ ++vec->count;
+ return 1;
+}
+
+/* Add PC ranges to U that map to FUNCTION. Returns 1 on success, 0
+ on error. */
+
+static int
+add_function_ranges (struct backtrace_state *state, struct dwarf_data *ddata,
+ struct unit *u, struct function *function,
+ uint64_t ranges, uint64_t base,
+ backtrace_error_callback error_callback, void *data,
+ struct function_vector *vec)
+{
+ struct dwarf_buf ranges_buf;
+
+ if (ranges >= ddata->dwarf_ranges_size)
+ {
+ error_callback (data, "function ranges offset out of range", 0);
+ return 0;
+ }
+
+ ranges_buf.name = ".debug_ranges";
+ ranges_buf.start = ddata->dwarf_ranges;
+ ranges_buf.buf = ddata->dwarf_ranges + ranges;
+ ranges_buf.left = ddata->dwarf_ranges_size - ranges;
+ ranges_buf.is_bigendian = ddata->is_bigendian;
+ ranges_buf.error_callback = error_callback;
+ ranges_buf.data = data;
+ ranges_buf.reported_underflow = 0;
+
+ while (1)
+ {
+ uint64_t low;
+ uint64_t high;
+
+ if (ranges_buf.reported_underflow)
+ return 0;
+
+ low = read_address (&ranges_buf, u->addrsize);
+ high = read_address (&ranges_buf, u->addrsize);
+
+ if (low == 0 && high == 0)
+ break;
+
+ if (is_highest_address (low, u->addrsize))
+ base = high;
+ else
+ {
+ if (!add_function_range (state, ddata, function, low + base,
+ high + base, error_callback, data, vec))
+ return 0;
+ }
+ }
+
+ if (ranges_buf.reported_underflow)
+ return 0;
+
+ return 1;
+}
+
+/* Read one entry plus all its children. Add function addresses to
+ VEC. Returns 1 on success, 0 on error. */
+
+static int
+read_function_entry (struct backtrace_state *state, struct dwarf_data *ddata,
+ struct unit *u, uint64_t base, struct dwarf_buf *unit_buf,
+ const struct line_header *lhdr,
+ backtrace_error_callback error_callback, void *data,
+ struct function_vector *vec_function,
+ struct function_vector *vec_inlined)
+{
+ while (unit_buf->left > 0)
+ {
+ uint64_t code;
+ const struct abbrev *abbrev;
+ int is_function;
+ struct function *function;
+ struct function_vector *vec;
+ size_t i;
+ uint64_t lowpc;
+ int have_lowpc;
+ uint64_t highpc;
+ int have_highpc;
+ int highpc_is_relative;
+ uint64_t ranges;
+ int have_ranges;
+
+ code = read_uleb128 (unit_buf);
+ if (code == 0)
+ return 1;
+
+ abbrev = lookup_abbrev (&u->abbrevs, code, error_callback, data);
+ if (abbrev == NULL)
+ return 0;
+
+ is_function = (abbrev->tag == DW_TAG_subprogram
+ || abbrev->tag == DW_TAG_entry_point
+ || abbrev->tag == DW_TAG_inlined_subroutine);
+
+ if (abbrev->tag == DW_TAG_inlined_subroutine)
+ vec = vec_inlined;
+ else
+ vec = vec_function;
+
+ function = NULL;
+ if (is_function)
+ {
+ function = ((struct function *)
+ backtrace_alloc (state, sizeof *function,
+ error_callback, data));
+ if (function == NULL)
+ return 0;
+ memset (function, 0, sizeof *function);
+ }
+
+ lowpc = 0;
+ have_lowpc = 0;
+ highpc = 0;
+ have_highpc = 0;
+ highpc_is_relative = 0;
+ ranges = 0;
+ have_ranges = 0;
+ for (i = 0; i < abbrev->num_attrs; ++i)
+ {
+ struct attr_val val;
+
+ if (!read_attribute (abbrev->attrs[i].form, unit_buf,
+ u->is_dwarf64, u->version, u->addrsize,
+ ddata->dwarf_str, ddata->dwarf_str_size,
+ &val))
+ return 0;
+
+ /* The compile unit sets the base address for any address
+ ranges in the function entries. */
+ if (abbrev->tag == DW_TAG_compile_unit
+ && abbrev->attrs[i].name == DW_AT_low_pc
+ && val.encoding == ATTR_VAL_ADDRESS)
+ base = val.u.uint;
+
+ if (is_function)
+ {
+ switch (abbrev->attrs[i].name)
+ {
+ case DW_AT_call_file:
+ if (val.encoding == ATTR_VAL_UINT)
+ {
+ if (val.u.uint == 0)
+ function->caller_filename = "";
+ else
+ {
+ if (val.u.uint - 1 >= lhdr->filenames_count)
+ {
+ dwarf_buf_error (unit_buf,
+ ("invalid file number in "
+ "DW_AT_call_file attribute"));
+ return 0;
+ }
+ function->caller_filename =
+ lhdr->filenames[val.u.uint - 1];
+ }
+ }
+ break;
+
+ case DW_AT_call_line:
+ if (val.encoding == ATTR_VAL_UINT)
+ function->caller_lineno = val.u.uint;
+ break;
+
+ case DW_AT_abstract_origin:
+ case DW_AT_specification:
+ if (abbrev->attrs[i].form == DW_FORM_ref_addr
+ || abbrev->attrs[i].form == DW_FORM_ref_sig8)
+ {
+ /* This refers to an abstract origin defined in
+ some other compilation unit. We can handle
+ this case if we must, but it's harder. */
+ break;
+ }
+ if (val.encoding == ATTR_VAL_UINT
+ || val.encoding == ATTR_VAL_REF_UNIT)
+ {
+ const char *name;
+
+ name = read_referenced_name (ddata, u, val.u.uint,
+ error_callback, data);
+ if (name != NULL)
+ function->name = name;
+ }
+ break;
+
+ case DW_AT_name:
+ if (val.encoding == ATTR_VAL_STRING)
+ {
+ /* Don't override a name we found in some other
+ way, as it will normally be more
+ useful--e.g., this name is normally not
+ mangled. */
+ if (function->name == NULL)
+ function->name = val.u.string;
+ }
+ break;
+
+ case DW_AT_linkage_name:
+ case DW_AT_MIPS_linkage_name:
+ if (val.encoding == ATTR_VAL_STRING)
+ function->name = val.u.string;
+ break;
+
+ case DW_AT_low_pc:
+ if (val.encoding == ATTR_VAL_ADDRESS)
+ {
+ lowpc = val.u.uint;
+ have_lowpc = 1;
+ }
+ break;
+
+ case DW_AT_high_pc:
+ if (val.encoding == ATTR_VAL_ADDRESS)
+ {
+ highpc = val.u.uint;
+ have_highpc = 1;
+ }
+ else if (val.encoding == ATTR_VAL_UINT)
+ {
+ highpc = val.u.uint;
+ have_highpc = 1;
+ highpc_is_relative = 1;
+ }
+ break;
+
+ case DW_AT_ranges:
+ if (val.encoding == ATTR_VAL_UINT
+ || val.encoding == ATTR_VAL_REF_SECTION)
+ {
+ ranges = val.u.uint;
+ have_ranges = 1;
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+ }
+
+ /* If we couldn't find a name for the function, we have no use
+ for it. */
+ if (is_function && function->name == NULL)
+ {
+ backtrace_free (state, function, sizeof *function,
+ error_callback, data);
+ is_function = 0;
+ }
+
+ if (is_function)
+ {
+ if (have_ranges)
+ {
+ if (!add_function_ranges (state, ddata, u, function, ranges,
+ base, error_callback, data, vec))
+ return 0;
+ }
+ else if (have_lowpc && have_highpc)
+ {
+ if (highpc_is_relative)
+ highpc += lowpc;
+ if (!add_function_range (state, ddata, function, lowpc, highpc,
+ error_callback, data, vec))
+ return 0;
+ }
+ else
+ {
+ backtrace_free (state, function, sizeof *function,
+ error_callback, data);
+ is_function = 0;
+ }
+ }
+
+ if (abbrev->has_children)
+ {
+ if (!is_function)
+ {
+ if (!read_function_entry (state, ddata, u, base, unit_buf, lhdr,
+ error_callback, data, vec_function,
+ vec_inlined))
+ return 0;
+ }
+ else
+ {
+ struct function_vector fvec;
+
+ /* Gather any information for inlined functions in
+ FVEC. */
+
+ memset (&fvec, 0, sizeof fvec);
+
+ if (!read_function_entry (state, ddata, u, base, unit_buf, lhdr,
+ error_callback, data, vec_function,
+ &fvec))
+ return 0;
+
+ if (fvec.count > 0)
+ {
+ struct function_addrs *faddrs;
+
+ if (!backtrace_vector_release (state, &fvec.vec,
+ error_callback, data))
+ return 0;
+
+ faddrs = (struct function_addrs *) fvec.vec.base;
+ backtrace_qsort (faddrs, fvec.count,
+ sizeof (struct function_addrs),
+ function_addrs_compare);
+
+ function->function_addrs = faddrs;
+ function->function_addrs_count = fvec.count;
+ }
+ }
+ }
+ }
+
+ return 1;
+}
+
+/* Read function name information for a compilation unit. We look
+ through the whole unit looking for function tags. */
+
+static void
+read_function_info (struct backtrace_state *state, struct dwarf_data *ddata,
+ const struct line_header *lhdr,
+ backtrace_error_callback error_callback, void *data,
+ struct unit *u, struct function_vector *fvec,
+ struct function_addrs **ret_addrs,
+ size_t *ret_addrs_count)
+{
+ struct function_vector lvec;
+ struct function_vector *pfvec;
+ struct dwarf_buf unit_buf;
+ struct function_addrs *addrs;
+ size_t addrs_count;
+
+ /* Use FVEC if it is not NULL. Otherwise use our own vector. */
+ if (fvec != NULL)
+ pfvec = fvec;
+ else
+ {
+ memset (&lvec, 0, sizeof lvec);
+ pfvec = &lvec;
+ }
+
+ unit_buf.name = ".debug_info";
+ unit_buf.start = ddata->dwarf_info;
+ unit_buf.buf = u->unit_data;
+ unit_buf.left = u->unit_data_len;
+ unit_buf.is_bigendian = ddata->is_bigendian;
+ unit_buf.error_callback = error_callback;
+ unit_buf.data = data;
+ unit_buf.reported_underflow = 0;
+
+ while (unit_buf.left > 0)
+ {
+ if (!read_function_entry (state, ddata, u, 0, &unit_buf, lhdr,
+ error_callback, data, pfvec, pfvec))
+ return;
+ }
+
+ if (pfvec->count == 0)
+ return;
+
+ addrs_count = pfvec->count;
+
+ if (fvec == NULL)
+ {
+ if (!backtrace_vector_release (state, &lvec.vec, error_callback, data))
+ return;
+ addrs = (struct function_addrs *) pfvec->vec.base;
+ }
+ else
+ {
+ /* Finish this list of addresses, but leave the remaining space in
+ the vector available for the next function unit. */
+ addrs = ((struct function_addrs *)
+ backtrace_vector_finish (state, &fvec->vec,
+ error_callback, data));
+ if (addrs == NULL)
+ return;
+ fvec->count = 0;
+ }
+
+ backtrace_qsort (addrs, addrs_count, sizeof (struct function_addrs),
+ function_addrs_compare);
+
+ *ret_addrs = addrs;
+ *ret_addrs_count = addrs_count;
+}
+
+/* See if PC is inlined in FUNCTION. If it is, print out the inlined
+ information, and update FILENAME and LINENO for the caller.
+ Returns whatever CALLBACK returns, or 0 to keep going. */
+
+static int
+report_inlined_functions (uintptr_t pc, struct function *function,
+ backtrace_full_callback callback, void *data,
+ const char **filename, int *lineno)
+{
+ struct function_addrs *function_addrs;
+ struct function *inlined;
+ int ret;
+
+ if (function->function_addrs_count == 0)
+ return 0;
+
+ function_addrs = ((struct function_addrs *)
+ bsearch (&pc, function->function_addrs,
+ function->function_addrs_count,
+ sizeof (struct function_addrs),
+ function_addrs_search));
+ if (function_addrs == NULL)
+ return 0;
+
+ while (((size_t) (function_addrs - function->function_addrs) + 1
+ < function->function_addrs_count)
+ && pc >= (function_addrs + 1)->low
+ && pc < (function_addrs + 1)->high)
+ ++function_addrs;
+
+ /* We found an inlined call. */
+
+ inlined = function_addrs->function;
+
+ /* Report any calls inlined into this one. */
+ ret = report_inlined_functions (pc, inlined, callback, data,
+ filename, lineno);
+ if (ret != 0)
+ return ret;
+
+ /* Report this inlined call. */
+ ret = callback (data, pc, *filename, *lineno, inlined->name);
+ if (ret != 0)
+ return ret;
+
+ /* Our caller will report the caller of the inlined function; tell
+ it the appropriate filename and line number. */
+ *filename = inlined->caller_filename;
+ *lineno = inlined->caller_lineno;
+
+ return 0;
+}
+
+/* Look for a PC in the DWARF mapping for one module. On success,
+ call CALLBACK and return whatever it returns. On error, call
+ ERROR_CALLBACK and return 0. Sets *FOUND to 1 if the PC is found,
+ 0 if not. */
+
+static int
+dwarf_lookup_pc (struct backtrace_state *state, struct dwarf_data *ddata,
+ uintptr_t pc, backtrace_full_callback callback,
+ backtrace_error_callback error_callback, void *data,
+ int *found)
+{
+ struct unit_addrs *entry;
+ struct unit *u;
+ int new_data;
+ struct line *lines;
+ struct line *ln;
+ struct function_addrs *function_addrs;
+ struct function *function;
+ const char *filename;
+ int lineno;
+ int ret;
+
+ *found = 1;
+
+ /* Find an address range that includes PC. */
+ entry = bsearch (&pc, ddata->addrs, ddata->addrs_count,
+ sizeof (struct unit_addrs), unit_addrs_search);
+
+ if (entry == NULL)
+ {
+ *found = 0;
+ return 0;
+ }
+
+ /* If there are multiple ranges that contain PC, use the last one,
+ in order to produce predictable results. If we assume that all
+ ranges are properly nested, then the last range will be the
+ smallest one. */
+ while ((size_t) (entry - ddata->addrs) + 1 < ddata->addrs_count
+ && pc >= (entry + 1)->low
+ && pc < (entry + 1)->high)
+ ++entry;
+
+ /* We need the lines, lines_count, function_addrs,
+ function_addrs_count fields of u. If they are not set, we need
+ to set them. When running in threaded mode, we need to allow for
+ the possibility that some other thread is setting them
+ simultaneously. */
+
+ u = entry->u;
+ lines = u->lines;
+
+ /* Skip units with no useful line number information by walking
+ backward. Useless line number information is marked by setting
+ lines == -1. */
+ while (entry > ddata->addrs
+ && pc >= (entry - 1)->low
+ && pc < (entry - 1)->high)
+ {
+ if (state->threaded)
+ lines = (struct line *) backtrace_atomic_load_pointer (&u->lines);
+
+ if (lines != (struct line *) (uintptr_t) -1)
+ break;
+
+ --entry;
+
+ u = entry->u;
+ lines = u->lines;
+ }
+
+ if (state->threaded)
+ lines = backtrace_atomic_load_pointer (&u->lines);
+
+ new_data = 0;
+ if (lines == NULL)
+ {
+ size_t function_addrs_count;
+ struct line_header lhdr;
+ size_t count;
+
+ /* We have never read the line information for this unit. Read
+ it now. */
+
+ function_addrs = NULL;
+ function_addrs_count = 0;
+ if (read_line_info (state, ddata, error_callback, data, entry->u, &lhdr,
+ &lines, &count))
+ {
+ struct function_vector *pfvec;
+
+ /* If not threaded, reuse DDATA->FVEC for better memory
+ consumption. */
+ if (state->threaded)
+ pfvec = NULL;
+ else
+ pfvec = &ddata->fvec;
+ read_function_info (state, ddata, &lhdr, error_callback, data,
+ entry->u, pfvec, &function_addrs,
+ &function_addrs_count);
+ free_line_header (state, &lhdr, error_callback, data);
+ new_data = 1;
+ }
+
+ /* Atomically store the information we just read into the unit.
+ If another thread is simultaneously writing, it presumably
+ read the same information, and we don't care which one we
+ wind up with; we just leak the other one. We do have to
+ write the lines field last, so that the acquire-loads above
+ ensure that the other fields are set. */
+
+ if (!state->threaded)
+ {
+ u->lines_count = count;
+ u->function_addrs = function_addrs;
+ u->function_addrs_count = function_addrs_count;
+ u->lines = lines;
+ }
+ else
+ {
+ backtrace_atomic_store_size_t (&u->lines_count, count);
+ backtrace_atomic_store_pointer (&u->function_addrs, function_addrs);
+ backtrace_atomic_store_size_t (&u->function_addrs_count,
+ function_addrs_count);
+ backtrace_atomic_store_pointer (&u->lines, lines);
+ }
+ }
+
+ /* Now all fields of U have been initialized. */
+
+ if (lines == (struct line *) (uintptr_t) -1)
+ {
+ /* If reading the line number information failed in some way,
+ try again to see if there is a better compilation unit for
+ this PC. */
+ if (new_data)
+ return dwarf_lookup_pc (state, ddata, pc, callback, error_callback,
+ data, found);
+ return callback (data, pc, NULL, 0, NULL);
+ }
+
+ /* Search for PC within this unit. */
+
+ ln = (struct line *) bsearch (&pc, lines, entry->u->lines_count,
+ sizeof (struct line), line_search);
+ if (ln == NULL)
+ {
+ /* The PC is between the low_pc and high_pc attributes of the
+ compilation unit, but no entry in the line table covers it.
+ This implies that the start of the compilation unit has no
+ line number information. */
+
+ if (entry->u->abs_filename == NULL)
+ {
+ const char *filename;
+
+ filename = entry->u->filename;
+ if (filename != NULL
+ && !IS_ABSOLUTE_PATH (filename)
+ && entry->u->comp_dir != NULL)
+ {
+ size_t filename_len;
+ const char *dir;
+ size_t dir_len;
+ char *s;
+
+ filename_len = strlen (filename);
+ dir = entry->u->comp_dir;
+ dir_len = strlen (dir);
+ s = (char *) backtrace_alloc (state, dir_len + filename_len + 2,
+ error_callback, data);
+ if (s == NULL)
+ {
+ *found = 0;
+ return 0;
+ }
+ memcpy (s, dir, dir_len);
+ /* FIXME: Should use backslash if DOS file system. */
+ s[dir_len] = '/';
+ memcpy (s + dir_len + 1, filename, filename_len + 1);
+ filename = s;
+ }
+ entry->u->abs_filename = filename;
+ }
+
+ return callback (data, pc, entry->u->abs_filename, 0, NULL);
+ }
+
+ /* Search for function name within this unit. */
+
+ if (entry->u->function_addrs_count == 0)
+ return callback (data, pc, ln->filename, ln->lineno, NULL);
+
+ function_addrs = ((struct function_addrs *)
+ bsearch (&pc, entry->u->function_addrs,
+ entry->u->function_addrs_count,
+ sizeof (struct function_addrs),
+ function_addrs_search));
+ if (function_addrs == NULL)
+ return callback (data, pc, ln->filename, ln->lineno, NULL);
+
+ /* If there are multiple function ranges that contain PC, use the
+ last one, in order to produce predictable results. */
+
+ while (((size_t) (function_addrs - entry->u->function_addrs + 1)
+ < entry->u->function_addrs_count)
+ && pc >= (function_addrs + 1)->low
+ && pc < (function_addrs + 1)->high)
+ ++function_addrs;
+
+ function = function_addrs->function;
+
+ filename = ln->filename;
+ lineno = ln->lineno;
+
+ ret = report_inlined_functions (pc, function, callback, data,
+ &filename, &lineno);
+ if (ret != 0)
+ return ret;
+
+ return callback (data, pc, filename, lineno, function->name);
+}
+
+
+/* Return the file/line information for a PC using the DWARF mapping
+ we built earlier. */
+
+static int
+dwarf_fileline (struct backtrace_state *state, uintptr_t pc,
+ backtrace_full_callback callback,
+ backtrace_error_callback error_callback, void *data)
+{
+ struct dwarf_data *ddata;
+ int found;
+ int ret;
+
+ if (!state->threaded)
+ {
+ for (ddata = (struct dwarf_data *) state->fileline_data;
+ ddata != NULL;
+ ddata = ddata->next)
+ {
+ ret = dwarf_lookup_pc (state, ddata, pc, callback, error_callback,
+ data, &found);
+ if (ret != 0 || found)
+ return ret;
+ }
+ }
+ else
+ {
+ struct dwarf_data **pp;
+
+ pp = (struct dwarf_data **) (void *) &state->fileline_data;
+ while (1)
+ {
+ ddata = backtrace_atomic_load_pointer (pp);
+ if (ddata == NULL)
+ break;
+
+ ret = dwarf_lookup_pc (state, ddata, pc, callback, error_callback,
+ data, &found);
+ if (ret != 0 || found)
+ return ret;
+
+ pp = &ddata->next;
+ }
+ }
+
+ /* FIXME: See if any libraries have been dlopen'ed. */
+
+ return callback (data, pc, NULL, 0, NULL);
+}
+
+/* Initialize our data structures from the DWARF debug info for a
+ file. Return NULL on failure. */
+
+static struct dwarf_data *
+build_dwarf_data (struct backtrace_state *state,
+ uintptr_t base_address,
+ const unsigned char *dwarf_info,
+ size_t dwarf_info_size,
+ const unsigned char *dwarf_line,
+ size_t dwarf_line_size,
+ const unsigned char *dwarf_abbrev,
+ size_t dwarf_abbrev_size,
+ const unsigned char *dwarf_ranges,
+ size_t dwarf_ranges_size,
+ const unsigned char *dwarf_str,
+ size_t dwarf_str_size,
+ int is_bigendian,
+ backtrace_error_callback error_callback,
+ void *data)
+{
+ struct unit_addrs_vector addrs_vec;
+ struct unit_addrs *addrs;
+ size_t addrs_count;
+ struct dwarf_data *fdata;
+
+ if (!build_address_map (state, base_address, dwarf_info, dwarf_info_size,
+ dwarf_abbrev, dwarf_abbrev_size, dwarf_ranges,
+ dwarf_ranges_size, dwarf_str, dwarf_str_size,
+ is_bigendian, error_callback, data, &addrs_vec))
+ return NULL;
+
+ if (!backtrace_vector_release (state, &addrs_vec.vec, error_callback, data))
+ return NULL;
+ addrs = (struct unit_addrs *) addrs_vec.vec.base;
+ addrs_count = addrs_vec.count;
+ backtrace_qsort (addrs, addrs_count, sizeof (struct unit_addrs),
+ unit_addrs_compare);
+
+ fdata = ((struct dwarf_data *)
+ backtrace_alloc (state, sizeof (struct dwarf_data),
+ error_callback, data));
+ if (fdata == NULL)
+ return NULL;
+
+ fdata->next = NULL;
+ fdata->base_address = base_address;
+ fdata->addrs = addrs;
+ fdata->addrs_count = addrs_count;
+ fdata->dwarf_info = dwarf_info;
+ fdata->dwarf_info_size = dwarf_info_size;
+ fdata->dwarf_line = dwarf_line;
+ fdata->dwarf_line_size = dwarf_line_size;
+ fdata->dwarf_ranges = dwarf_ranges;
+ fdata->dwarf_ranges_size = dwarf_ranges_size;
+ fdata->dwarf_str = dwarf_str;
+ fdata->dwarf_str_size = dwarf_str_size;
+ fdata->is_bigendian = is_bigendian;
+ memset (&fdata->fvec, 0, sizeof fdata->fvec);
+
+ return fdata;
+}
+
+/* Build our data structures from the DWARF sections for a module.
+ Set FILELINE_FN and STATE->FILELINE_DATA. Return 1 on success, 0
+ on failure. */
+
+int
+backtrace_dwarf_add (struct backtrace_state *state,
+ uintptr_t base_address,
+ const unsigned char *dwarf_info,
+ size_t dwarf_info_size,
+ const unsigned char *dwarf_line,
+ size_t dwarf_line_size,
+ const unsigned char *dwarf_abbrev,
+ size_t dwarf_abbrev_size,
+ const unsigned char *dwarf_ranges,
+ size_t dwarf_ranges_size,
+ const unsigned char *dwarf_str,
+ size_t dwarf_str_size,
+ int is_bigendian,
+ backtrace_error_callback error_callback,
+ void *data, fileline *fileline_fn)
+{
+ struct dwarf_data *fdata;
+
+ fdata = build_dwarf_data (state, base_address, dwarf_info, dwarf_info_size,
+ dwarf_line, dwarf_line_size, dwarf_abbrev,
+ dwarf_abbrev_size, dwarf_ranges, dwarf_ranges_size,
+ dwarf_str, dwarf_str_size, is_bigendian,
+ error_callback, data);
+ if (fdata == NULL)
+ return 0;
+
+ if (!state->threaded)
+ {
+ struct dwarf_data **pp;
+
+ for (pp = (struct dwarf_data **) (void *) &state->fileline_data;
+ *pp != NULL;
+ pp = &(*pp)->next)
+ ;
+ *pp = fdata;
+ }
+ else
+ {
+ while (1)
+ {
+ struct dwarf_data **pp;
+
+ pp = (struct dwarf_data **) (void *) &state->fileline_data;
+
+ while (1)
+ {
+ struct dwarf_data *p;
+
+ p = backtrace_atomic_load_pointer (pp);
+
+ if (p == NULL)
+ break;
+
+ pp = &p->next;
+ }
+
+ if (__sync_bool_compare_and_swap (pp, NULL, fdata))
+ break;
+ }
+ }
+
+ *fileline_fn = dwarf_fileline;
+
+ return 1;
+}
diff --git a/src/AS_UTL/libbacktrace/elf.c b/src/AS_UTL/libbacktrace/elf.c
new file mode 100644
index 0000000..9f0ff13
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/elf.c
@@ -0,0 +1,983 @@
+/* elf.c -- Get debug data from an ELF file for backtraces.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#if (BACKTRACE_ELF_SIZE == 32) || (BACKTRACE_ELF_SIZE == 64)
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+
+#ifdef HAVE_DL_ITERATE_PHDR
+#include <link.h>
+#endif
+
+#include "backtrace.h"
+#include "internal.h"
+
+#ifndef HAVE_DL_ITERATE_PHDR
+
+/* Dummy version of dl_iterate_phdr for systems that don't have it. */
+
+#define dl_phdr_info x_dl_phdr_info
+#define dl_iterate_phdr x_dl_iterate_phdr
+
+struct dl_phdr_info
+{
+ uintptr_t dlpi_addr;
+ const char *dlpi_name;
+};
+
+static int
+dl_iterate_phdr (int (*callback) (struct dl_phdr_info *,
+ size_t, void *) ATTRIBUTE_UNUSED,
+ void *data ATTRIBUTE_UNUSED)
+{
+ return 0;
+}
+
+#endif /* ! defined (HAVE_DL_ITERATE_PHDR) */
+
+/* The configure script must tell us whether we are 32-bit or 64-bit
+ ELF. We could make this code test and support either possibility,
+ but there is no point. This code only works for the currently
+ running executable, which means that we know the ELF mode at
+ configure mode. */
+
+#if BACKTRACE_ELF_SIZE != 32 && BACKTRACE_ELF_SIZE != 64
+#error "Unknown BACKTRACE_ELF_SIZE"
+#endif
+
+/* <link.h> might #include <elf.h> which might define our constants
+ with slightly different values. Undefine them to be safe. */
+
+#undef EI_NIDENT
+#undef EI_MAG0
+#undef EI_MAG1
+#undef EI_MAG2
+#undef EI_MAG3
+#undef EI_CLASS
+#undef EI_DATA
+#undef EI_VERSION
+#undef ELF_MAG0
+#undef ELF_MAG1
+#undef ELF_MAG2
+#undef ELF_MAG3
+#undef ELFCLASS32
+#undef ELFCLASS64
+#undef ELFDATA2LSB
+#undef ELFDATA2MSB
+#undef EV_CURRENT
+#undef ET_DYN
+#undef SHN_LORESERVE
+#undef SHN_XINDEX
+#undef SHN_UNDEF
+#undef SHT_SYMTAB
+#undef SHT_STRTAB
+#undef SHT_DYNSYM
+#undef STT_OBJECT
+#undef STT_FUNC
+
+/* Basic types. */
+
+typedef uint16_t b_elf_half; /* Elf_Half. */
+typedef uint32_t b_elf_word; /* Elf_Word. */
+typedef int32_t b_elf_sword; /* Elf_Sword. */
+
+#if BACKTRACE_ELF_SIZE == 32
+
+typedef uint32_t b_elf_addr; /* Elf_Addr. */
+typedef uint32_t b_elf_off; /* Elf_Off. */
+
+typedef uint32_t b_elf_wxword; /* 32-bit Elf_Word, 64-bit ELF_Xword. */
+
+#else
+
+typedef uint64_t b_elf_addr; /* Elf_Addr. */
+typedef uint64_t b_elf_off; /* Elf_Off. */
+typedef uint64_t b_elf_xword; /* Elf_Xword. */
+typedef int64_t b_elf_sxword; /* Elf_Sxword. */
+
+typedef uint64_t b_elf_wxword; /* 32-bit Elf_Word, 64-bit ELF_Xword. */
+
+#endif
+
+/* Data structures and associated constants. */
+
+#define EI_NIDENT 16
+
+typedef struct {
+ unsigned char e_ident[EI_NIDENT]; /* ELF "magic number" */
+ b_elf_half e_type; /* Identifies object file type */
+ b_elf_half e_machine; /* Specifies required architecture */
+ b_elf_word e_version; /* Identifies object file version */
+ b_elf_addr e_entry; /* Entry point virtual address */
+ b_elf_off e_phoff; /* Program header table file offset */
+ b_elf_off e_shoff; /* Section header table file offset */
+ b_elf_word e_flags; /* Processor-specific flags */
+ b_elf_half e_ehsize; /* ELF header size in bytes */
+ b_elf_half e_phentsize; /* Program header table entry size */
+ b_elf_half e_phnum; /* Program header table entry count */
+ b_elf_half e_shentsize; /* Section header table entry size */
+ b_elf_half e_shnum; /* Section header table entry count */
+ b_elf_half e_shstrndx; /* Section header string table index */
+} b_elf_ehdr; /* Elf_Ehdr. */
+
+#define EI_MAG0 0
+#define EI_MAG1 1
+#define EI_MAG2 2
+#define EI_MAG3 3
+#define EI_CLASS 4
+#define EI_DATA 5
+#define EI_VERSION 6
+
+#define ELFMAG0 0x7f
+#define ELFMAG1 'E'
+#define ELFMAG2 'L'
+#define ELFMAG3 'F'
+
+#define ELFCLASS32 1
+#define ELFCLASS64 2
+
+#define ELFDATA2LSB 1
+#define ELFDATA2MSB 2
+
+#define EV_CURRENT 1
+
+#define ET_DYN 3
+
+typedef struct {
+ b_elf_word sh_name; /* Section name, index in string tbl */
+ b_elf_word sh_type; /* Type of section */
+ b_elf_wxword sh_flags; /* Miscellaneous section attributes */
+ b_elf_addr sh_addr; /* Section virtual addr at execution */
+ b_elf_off sh_offset; /* Section file offset */
+ b_elf_wxword sh_size; /* Size of section in bytes */
+ b_elf_word sh_link; /* Index of another section */
+ b_elf_word sh_info; /* Additional section information */
+ b_elf_wxword sh_addralign; /* Section alignment */
+ b_elf_wxword sh_entsize; /* Entry size if section holds table */
+} b_elf_shdr; /* Elf_Shdr. */
+
+#define SHN_UNDEF 0x0000 /* Undefined section */
+#define SHN_LORESERVE 0xFF00 /* Begin range of reserved indices */
+#define SHN_XINDEX 0xFFFF /* Section index is held elsewhere */
+
+#define SHT_SYMTAB 2
+#define SHT_STRTAB 3
+#define SHT_DYNSYM 11
+
+#if BACKTRACE_ELF_SIZE == 32
+
+typedef struct
+{
+ b_elf_word st_name; /* Symbol name, index in string tbl */
+ b_elf_addr st_value; /* Symbol value */
+ b_elf_word st_size; /* Symbol size */
+ unsigned char st_info; /* Symbol binding and type */
+ unsigned char st_other; /* Visibility and other data */
+ b_elf_half st_shndx; /* Symbol section index */
+} b_elf_sym; /* Elf_Sym. */
+
+#else /* BACKTRACE_ELF_SIZE != 32 */
+
+typedef struct
+{
+ b_elf_word st_name; /* Symbol name, index in string tbl */
+ unsigned char st_info; /* Symbol binding and type */
+ unsigned char st_other; /* Visibility and other data */
+ b_elf_half st_shndx; /* Symbol section index */
+ b_elf_addr st_value; /* Symbol value */
+ b_elf_xword st_size; /* Symbol size */
+} b_elf_sym; /* Elf_Sym. */
+
+#endif /* BACKTRACE_ELF_SIZE != 32 */
+
+#define STT_OBJECT 1
+#define STT_FUNC 2
+
+/* An index of ELF sections we care about. */
+
+enum debug_section
+{
+ DEBUG_INFO,
+ DEBUG_LINE,
+ DEBUG_ABBREV,
+ DEBUG_RANGES,
+ DEBUG_STR,
+ DEBUG_MAX
+};
+
+/* Names of sections, indexed by enum elf_section. */
+
+static const char * const debug_section_names[DEBUG_MAX] =
+{
+ ".debug_info",
+ ".debug_line",
+ ".debug_abbrev",
+ ".debug_ranges",
+ ".debug_str"
+};
+
+/* Information we gather for the sections we care about. */
+
+struct debug_section_info
+{
+ /* Section file offset. */
+ off_t offset;
+ /* Section size. */
+ size_t size;
+ /* Section contents, after read from file. */
+ const unsigned char *data;
+};
+
+/* Information we keep for an ELF symbol. */
+
+struct elf_symbol
+{
+ /* The name of the symbol. */
+ const char *name;
+ /* The address of the symbol. */
+ uintptr_t address;
+ /* The size of the symbol. */
+ size_t size;
+};
+
+/* Information to pass to elf_syminfo. */
+
+struct elf_syminfo_data
+{
+ /* Symbols for the next module. */
+ struct elf_syminfo_data *next;
+ /* The ELF symbols, sorted by address. */
+ struct elf_symbol *symbols;
+ /* The number of symbols. */
+ size_t count;
+};
+
+/* A dummy callback function used when we can't find any debug info. */
+
+static int
+elf_nodebug (struct backtrace_state *state ATTRIBUTE_UNUSED,
+ uintptr_t pc ATTRIBUTE_UNUSED,
+ backtrace_full_callback callback ATTRIBUTE_UNUSED,
+ backtrace_error_callback error_callback, void *data)
+{
+ error_callback (data, "no debug info in ELF executable", -1);
+ return 0;
+}
+
+/* A dummy callback function used when we can't find a symbol
+ table. */
+
+static void
+elf_nosyms (struct backtrace_state *state ATTRIBUTE_UNUSED,
+ uintptr_t addr ATTRIBUTE_UNUSED,
+ backtrace_syminfo_callback callback ATTRIBUTE_UNUSED,
+ backtrace_error_callback error_callback, void *data)
+{
+ error_callback (data, "no symbol table in ELF executable", -1);
+}
+
+/* Compare struct elf_symbol for qsort. */
+
+static int
+elf_symbol_compare (const void *v1, const void *v2)
+{
+ const struct elf_symbol *e1 = (const struct elf_symbol *) v1;
+ const struct elf_symbol *e2 = (const struct elf_symbol *) v2;
+
+ if (e1->address < e2->address)
+ return -1;
+ else if (e1->address > e2->address)
+ return 1;
+ else
+ return 0;
+}
+
+/* Compare an ADDR against an elf_symbol for bsearch. We allocate one
+ extra entry in the array so that this can look safely at the next
+ entry. */
+
+static int
+elf_symbol_search (const void *vkey, const void *ventry)
+{
+ const uintptr_t *key = (const uintptr_t *) vkey;
+ const struct elf_symbol *entry = (const struct elf_symbol *) ventry;
+ uintptr_t addr;
+
+ addr = *key;
+ if (addr < entry->address)
+ return -1;
+ else if (addr >= entry->address + entry->size)
+ return 1;
+ else
+ return 0;
+}
+
+/* Initialize the symbol table info for elf_syminfo. */
+
+static int
+elf_initialize_syminfo (struct backtrace_state *state,
+ uintptr_t base_address,
+ const unsigned char *symtab_data, size_t symtab_size,
+ const unsigned char *strtab, size_t strtab_size,
+ backtrace_error_callback error_callback,
+ void *data, struct elf_syminfo_data *sdata)
+{
+ size_t sym_count;
+ const b_elf_sym *sym;
+ size_t elf_symbol_count;
+ size_t elf_symbol_size;
+ struct elf_symbol *elf_symbols;
+ size_t i;
+ unsigned int j;
+
+ sym_count = symtab_size / sizeof (b_elf_sym);
+
+ /* We only care about function symbols. Count them. */
+ sym = (const b_elf_sym *) symtab_data;
+ elf_symbol_count = 0;
+ for (i = 0; i < sym_count; ++i, ++sym)
+ {
+ int info;
+
+ info = sym->st_info & 0xf;
+ if ((info == STT_FUNC || info == STT_OBJECT)
+ && sym->st_shndx != SHN_UNDEF)
+ ++elf_symbol_count;
+ }
+
+ elf_symbol_size = elf_symbol_count * sizeof (struct elf_symbol);
+ elf_symbols = ((struct elf_symbol *)
+ backtrace_alloc (state, elf_symbol_size, error_callback,
+ data));
+ if (elf_symbols == NULL)
+ return 0;
+
+ sym = (const b_elf_sym *) symtab_data;
+ j = 0;
+ for (i = 0; i < sym_count; ++i, ++sym)
+ {
+ int info;
+
+ info = sym->st_info & 0xf;
+ if (info != STT_FUNC && info != STT_OBJECT)
+ continue;
+ if (sym->st_shndx == SHN_UNDEF)
+ continue;
+ if (sym->st_name >= strtab_size)
+ {
+ error_callback (data, "symbol string index out of range", 0);
+ backtrace_free (state, elf_symbols, elf_symbol_size, error_callback,
+ data);
+ return 0;
+ }
+ elf_symbols[j].name = (const char *) strtab + sym->st_name;
+ elf_symbols[j].address = sym->st_value + base_address;
+ elf_symbols[j].size = sym->st_size;
+ ++j;
+ }
+
+ backtrace_qsort (elf_symbols, elf_symbol_count, sizeof (struct elf_symbol),
+ elf_symbol_compare);
+
+ sdata->next = NULL;
+ sdata->symbols = elf_symbols;
+ sdata->count = elf_symbol_count;
+
+ return 1;
+}
+
+/* Add EDATA to the list in STATE. */
+
+static void
+elf_add_syminfo_data (struct backtrace_state *state,
+ struct elf_syminfo_data *edata)
+{
+ if (!state->threaded)
+ {
+ struct elf_syminfo_data **pp;
+
+ for (pp = (struct elf_syminfo_data **) (void *) &state->syminfo_data;
+ *pp != NULL;
+ pp = &(*pp)->next)
+ ;
+ *pp = edata;
+ }
+ else
+ {
+ while (1)
+ {
+ struct elf_syminfo_data **pp;
+
+ pp = (struct elf_syminfo_data **) (void *) &state->syminfo_data;
+
+ while (1)
+ {
+ struct elf_syminfo_data *p;
+
+ p = backtrace_atomic_load_pointer (pp);
+
+ if (p == NULL)
+ break;
+
+ pp = &p->next;
+ }
+
+ if (__sync_bool_compare_and_swap (pp, NULL, edata))
+ break;
+ }
+ }
+}
+
+/* Return the symbol name and value for an ADDR. */
+
+static void
+elf_syminfo (struct backtrace_state *state, uintptr_t addr,
+ backtrace_syminfo_callback callback,
+ backtrace_error_callback error_callback ATTRIBUTE_UNUSED,
+ void *data)
+{
+ struct elf_syminfo_data *edata;
+ struct elf_symbol *sym = NULL;
+
+ if (!state->threaded)
+ {
+ for (edata = (struct elf_syminfo_data *) state->syminfo_data;
+ edata != NULL;
+ edata = edata->next)
+ {
+ sym = ((struct elf_symbol *)
+ bsearch (&addr, edata->symbols, edata->count,
+ sizeof (struct elf_symbol), elf_symbol_search));
+ if (sym != NULL)
+ break;
+ }
+ }
+ else
+ {
+ struct elf_syminfo_data **pp;
+
+ pp = (struct elf_syminfo_data **) (void *) &state->syminfo_data;
+ while (1)
+ {
+ edata = backtrace_atomic_load_pointer (pp);
+ if (edata == NULL)
+ break;
+
+ sym = ((struct elf_symbol *)
+ bsearch (&addr, edata->symbols, edata->count,
+ sizeof (struct elf_symbol), elf_symbol_search));
+ if (sym != NULL)
+ break;
+
+ pp = &edata->next;
+ }
+ }
+
+ if (sym == NULL)
+ callback (data, addr, NULL, 0, 0);
+ else
+ callback (data, addr, sym->name, sym->address, sym->size);
+}
+
+/* Add the backtrace data for one ELF file. Returns 1 on success,
+ 0 on failure (in both cases descriptor is closed) or -1 if exe
+ is non-zero and the ELF file is ET_DYN, which tells the caller that
+ elf_add will need to be called on the descriptor again after
+ base_address is determined. */
+
+static int
+elf_add (struct backtrace_state *state, int descriptor, uintptr_t base_address,
+ backtrace_error_callback error_callback, void *data,
+ fileline *fileline_fn, int *found_sym, int *found_dwarf, int exe)
+{
+ struct backtrace_view ehdr_view;
+ b_elf_ehdr ehdr;
+ off_t shoff;
+ unsigned int shnum;
+ unsigned int shstrndx;
+ struct backtrace_view shdrs_view;
+ int shdrs_view_valid;
+ const b_elf_shdr *shdrs;
+ const b_elf_shdr *shstrhdr;
+ size_t shstr_size;
+ off_t shstr_off;
+ struct backtrace_view names_view;
+ int names_view_valid;
+ const char *names;
+ unsigned int symtab_shndx;
+ unsigned int dynsym_shndx;
+ unsigned int i;
+ struct debug_section_info sections[DEBUG_MAX];
+ struct backtrace_view symtab_view;
+ int symtab_view_valid;
+ struct backtrace_view strtab_view;
+ int strtab_view_valid;
+ off_t min_offset;
+ off_t max_offset;
+ struct backtrace_view debug_view;
+ int debug_view_valid;
+
+ *found_sym = 0;
+ *found_dwarf = 0;
+
+ shdrs_view_valid = 0;
+ names_view_valid = 0;
+ symtab_view_valid = 0;
+ strtab_view_valid = 0;
+ debug_view_valid = 0;
+
+ if (!backtrace_get_view (state, descriptor, 0, sizeof ehdr, error_callback,
+ data, &ehdr_view))
+ goto fail;
+
+ memcpy (&ehdr, ehdr_view.data, sizeof ehdr);
+
+ backtrace_release_view (state, &ehdr_view, error_callback, data);
+
+ if (ehdr.e_ident[EI_MAG0] != ELFMAG0
+ || ehdr.e_ident[EI_MAG1] != ELFMAG1
+ || ehdr.e_ident[EI_MAG2] != ELFMAG2
+ || ehdr.e_ident[EI_MAG3] != ELFMAG3)
+ {
+ error_callback (data, "executable file is not ELF", 0);
+ goto fail;
+ }
+ if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
+ {
+ error_callback (data, "executable file is unrecognized ELF version", 0);
+ goto fail;
+ }
+
+#if BACKTRACE_ELF_SIZE == 32
+#define BACKTRACE_ELFCLASS ELFCLASS32
+#else
+#define BACKTRACE_ELFCLASS ELFCLASS64
+#endif
+
+ if (ehdr.e_ident[EI_CLASS] != BACKTRACE_ELFCLASS)
+ {
+ error_callback (data, "executable file is unexpected ELF class", 0);
+ goto fail;
+ }
+
+ if (ehdr.e_ident[EI_DATA] != ELFDATA2LSB
+ && ehdr.e_ident[EI_DATA] != ELFDATA2MSB)
+ {
+ error_callback (data, "executable file has unknown endianness", 0);
+ goto fail;
+ }
+
+ /* If the executable is ET_DYN, it is either a PIE, or we are running
+ directly a shared library with .interp. We need to wait for
+ dl_iterate_phdr in that case to determine the actual base_address. */
+ if (exe && ehdr.e_type == ET_DYN)
+ return -1;
+
+ shoff = ehdr.e_shoff;
+ shnum = ehdr.e_shnum;
+ shstrndx = ehdr.e_shstrndx;
+
+ if ((shnum == 0 || shstrndx == SHN_XINDEX)
+ && shoff != 0)
+ {
+ struct backtrace_view shdr_view;
+ const b_elf_shdr *shdr;
+
+ if (!backtrace_get_view (state, descriptor, shoff, sizeof shdr,
+ error_callback, data, &shdr_view))
+ goto fail;
+
+ shdr = (const b_elf_shdr *) shdr_view.data;
+
+ if (shnum == 0)
+ shnum = shdr->sh_size;
+
+ if (shstrndx == SHN_XINDEX)
+ {
+ shstrndx = shdr->sh_link;
+
+ /* Versions of the GNU binutils between 2.12 and 2.18 did
+ not handle objects with more than SHN_LORESERVE sections
+ correctly. All large section indexes were offset by
+ 0x100. There is more information at
+ http://sourceware.org/bugzilla/show_bug.cgi?id-5900 .
+ Fortunately these object files are easy to detect, as the
+ GNU binutils always put the section header string table
+ near the end of the list of sections. Thus if the
+ section header string table index is larger than the
+ number of sections, then we know we have to subtract
+ 0x100 to get the real section index. */
+ if (shstrndx >= shnum && shstrndx >= SHN_LORESERVE + 0x100)
+ shstrndx -= 0x100;
+ }
+
+ backtrace_release_view (state, &shdr_view, error_callback, data);
+ }
+
+ /* To translate PC to file/line when using DWARF, we need to find
+ the .debug_info and .debug_line sections. */
+
+ /* Read the section headers, skipping the first one. */
+
+ if (!backtrace_get_view (state, descriptor, shoff + sizeof (b_elf_shdr),
+ (shnum - 1) * sizeof (b_elf_shdr),
+ error_callback, data, &shdrs_view))
+ goto fail;
+ shdrs_view_valid = 1;
+ shdrs = (const b_elf_shdr *) shdrs_view.data;
+
+ /* Read the section names. */
+
+ shstrhdr = &shdrs[shstrndx - 1];
+ shstr_size = shstrhdr->sh_size;
+ shstr_off = shstrhdr->sh_offset;
+
+ if (!backtrace_get_view (state, descriptor, shstr_off, shstr_size,
+ error_callback, data, &names_view))
+ goto fail;
+ names_view_valid = 1;
+ names = (const char *) names_view.data;
+
+ symtab_shndx = 0;
+ dynsym_shndx = 0;
+
+ memset (sections, 0, sizeof sections);
+
+ /* Look for the symbol table. */
+ for (i = 1; i < shnum; ++i)
+ {
+ const b_elf_shdr *shdr;
+ unsigned int sh_name;
+ const char *name;
+ int j;
+
+ shdr = &shdrs[i - 1];
+
+ if (shdr->sh_type == SHT_SYMTAB)
+ symtab_shndx = i;
+ else if (shdr->sh_type == SHT_DYNSYM)
+ dynsym_shndx = i;
+
+ sh_name = shdr->sh_name;
+ if (sh_name >= shstr_size)
+ {
+ error_callback (data, "ELF section name out of range", 0);
+ goto fail;
+ }
+
+ name = names + sh_name;
+
+ for (j = 0; j < (int) DEBUG_MAX; ++j)
+ {
+ if (strcmp (name, debug_section_names[j]) == 0)
+ {
+ sections[j].offset = shdr->sh_offset;
+ sections[j].size = shdr->sh_size;
+ break;
+ }
+ }
+ }
+
+ if (symtab_shndx == 0)
+ symtab_shndx = dynsym_shndx;
+ if (symtab_shndx != 0)
+ {
+ const b_elf_shdr *symtab_shdr;
+ unsigned int strtab_shndx;
+ const b_elf_shdr *strtab_shdr;
+ struct elf_syminfo_data *sdata;
+
+ symtab_shdr = &shdrs[symtab_shndx - 1];
+ strtab_shndx = symtab_shdr->sh_link;
+ if (strtab_shndx >= shnum)
+ {
+ error_callback (data,
+ "ELF symbol table strtab link out of range", 0);
+ goto fail;
+ }
+ strtab_shdr = &shdrs[strtab_shndx - 1];
+
+ if (!backtrace_get_view (state, descriptor, symtab_shdr->sh_offset,
+ symtab_shdr->sh_size, error_callback, data,
+ &symtab_view))
+ goto fail;
+ symtab_view_valid = 1;
+
+ if (!backtrace_get_view (state, descriptor, strtab_shdr->sh_offset,
+ strtab_shdr->sh_size, error_callback, data,
+ &strtab_view))
+ goto fail;
+ strtab_view_valid = 1;
+
+ sdata = ((struct elf_syminfo_data *)
+ backtrace_alloc (state, sizeof *sdata, error_callback, data));
+ if (sdata == NULL)
+ goto fail;
+
+ if (!elf_initialize_syminfo (state, base_address,
+ symtab_view.data, symtab_shdr->sh_size,
+ strtab_view.data, strtab_shdr->sh_size,
+ error_callback, data, sdata))
+ {
+ backtrace_free (state, sdata, sizeof *sdata, error_callback, data);
+ goto fail;
+ }
+
+ /* We no longer need the symbol table, but we hold on to the
+ string table permanently. */
+ backtrace_release_view (state, &symtab_view, error_callback, data);
+
+ *found_sym = 1;
+
+ elf_add_syminfo_data (state, sdata);
+ }
+
+ /* FIXME: Need to handle compressed debug sections. */
+
+ backtrace_release_view (state, &shdrs_view, error_callback, data);
+ shdrs_view_valid = 0;
+ backtrace_release_view (state, &names_view, error_callback, data);
+ names_view_valid = 0;
+
+ /* Read all the debug sections in a single view, since they are
+ probably adjacent in the file. We never release this view. */
+
+ min_offset = 0;
+ max_offset = 0;
+ for (i = 0; i < (int) DEBUG_MAX; ++i)
+ {
+ off_t end;
+
+ if (sections[i].size == 0)
+ continue;
+ if (min_offset == 0 || sections[i].offset < min_offset)
+ min_offset = sections[i].offset;
+ end = sections[i].offset + sections[i].size;
+ if (end > max_offset)
+ max_offset = end;
+ }
+ if (min_offset == 0 || max_offset == 0)
+ {
+ if (!backtrace_close (descriptor, error_callback, data))
+ goto fail;
+ return 1;
+ }
+
+ if (!backtrace_get_view (state, descriptor, min_offset,
+ max_offset - min_offset,
+ error_callback, data, &debug_view))
+ goto fail;
+ debug_view_valid = 1;
+
+ /* We've read all we need from the executable. */
+ if (!backtrace_close (descriptor, error_callback, data))
+ goto fail;
+ descriptor = -1;
+
+ for (i = 0; i < (int) DEBUG_MAX; ++i)
+ {
+ if (sections[i].size == 0)
+ sections[i].data = NULL;
+ else
+ sections[i].data = ((const unsigned char *) debug_view.data
+ + (sections[i].offset - min_offset));
+ }
+
+ if (!backtrace_dwarf_add (state, base_address,
+ sections[DEBUG_INFO].data,
+ sections[DEBUG_INFO].size,
+ sections[DEBUG_LINE].data,
+ sections[DEBUG_LINE].size,
+ sections[DEBUG_ABBREV].data,
+ sections[DEBUG_ABBREV].size,
+ sections[DEBUG_RANGES].data,
+ sections[DEBUG_RANGES].size,
+ sections[DEBUG_STR].data,
+ sections[DEBUG_STR].size,
+ ehdr.e_ident[EI_DATA] == ELFDATA2MSB,
+ error_callback, data, fileline_fn))
+ goto fail;
+
+ *found_dwarf = 1;
+
+ return 1;
+
+ fail:
+ if (shdrs_view_valid)
+ backtrace_release_view (state, &shdrs_view, error_callback, data);
+ if (names_view_valid)
+ backtrace_release_view (state, &names_view, error_callback, data);
+ if (symtab_view_valid)
+ backtrace_release_view (state, &symtab_view, error_callback, data);
+ if (strtab_view_valid)
+ backtrace_release_view (state, &strtab_view, error_callback, data);
+ if (debug_view_valid)
+ backtrace_release_view (state, &debug_view, error_callback, data);
+ if (descriptor != -1)
+ backtrace_close (descriptor, error_callback, data);
+ return 0;
+}
+
+/* Data passed to phdr_callback. */
+
+struct phdr_data
+{
+ struct backtrace_state *state;
+ backtrace_error_callback error_callback;
+ void *data;
+ fileline *fileline_fn;
+ int *found_sym;
+ int *found_dwarf;
+ int exe_descriptor;
+};
+
+/* Callback passed to dl_iterate_phdr. Load debug info from shared
+ libraries. */
+
+static int
+#ifdef __i386__
+__attribute__ ((__force_align_arg_pointer__))
+#endif
+phdr_callback (struct dl_phdr_info *info, size_t size ATTRIBUTE_UNUSED,
+ void *pdata)
+{
+ struct phdr_data *pd = (struct phdr_data *) pdata;
+ int descriptor;
+ int does_not_exist;
+ fileline elf_fileline_fn;
+ int found_dwarf;
+
+ /* There is not much we can do if we don't have the module name,
+ unless executable is ET_DYN, where we expect the very first
+ phdr_callback to be for the PIE. */
+ if (info->dlpi_name == NULL || info->dlpi_name[0] == '\0')
+ {
+ if (pd->exe_descriptor == -1)
+ return 0;
+ descriptor = pd->exe_descriptor;
+ pd->exe_descriptor = -1;
+ }
+ else
+ {
+ if (pd->exe_descriptor != -1)
+ {
+ backtrace_close (pd->exe_descriptor, pd->error_callback, pd->data);
+ pd->exe_descriptor = -1;
+ }
+
+ descriptor = backtrace_open (info->dlpi_name, pd->error_callback,
+ pd->data, &does_not_exist);
+ if (descriptor < 0)
+ return 0;
+ }
+
+ if (elf_add (pd->state, descriptor, info->dlpi_addr, pd->error_callback,
+ pd->data, &elf_fileline_fn, pd->found_sym, &found_dwarf, 0))
+ {
+ if (found_dwarf)
+ {
+ *pd->found_dwarf = 1;
+ *pd->fileline_fn = elf_fileline_fn;
+ }
+ }
+
+ return 0;
+}
+
+/* Initialize the backtrace data we need from an ELF executable. At
+ the ELF level, all we need to do is find the debug info
+ sections. */
+
+int
+backtrace_initialize (struct backtrace_state *state, int descriptor,
+ backtrace_error_callback error_callback,
+ void *data, fileline *fileline_fn)
+{
+ int ret;
+ int found_sym;
+ int found_dwarf;
+ fileline elf_fileline_fn = elf_nodebug;
+ struct phdr_data pd;
+
+ ret = elf_add (state, descriptor, 0, error_callback, data, &elf_fileline_fn,
+ &found_sym, &found_dwarf, 1);
+ if (!ret)
+ return 0;
+
+ pd.state = state;
+ pd.error_callback = error_callback;
+ pd.data = data;
+ pd.fileline_fn = &elf_fileline_fn;
+ pd.found_sym = &found_sym;
+ pd.found_dwarf = &found_dwarf;
+ pd.exe_descriptor = ret < 0 ? descriptor : -1;
+
+ dl_iterate_phdr (phdr_callback, (void *) &pd);
+
+ if (!state->threaded)
+ {
+ if (found_sym)
+ state->syminfo_fn = elf_syminfo;
+ else if (state->syminfo_fn == NULL)
+ state->syminfo_fn = elf_nosyms;
+ }
+ else
+ {
+ if (found_sym)
+ backtrace_atomic_store_pointer (&state->syminfo_fn, elf_syminfo);
+ else
+ (void) __sync_bool_compare_and_swap (&state->syminfo_fn, NULL,
+ elf_nosyms);
+ }
+
+ if (!state->threaded)
+ {
+ if (state->fileline_fn == NULL || state->fileline_fn == elf_nodebug)
+ *fileline_fn = elf_fileline_fn;
+ }
+ else
+ {
+ fileline current_fn;
+
+ current_fn = backtrace_atomic_load_pointer (&state->fileline_fn);
+ if (current_fn == NULL || current_fn == elf_nodebug)
+ *fileline_fn = elf_fileline_fn;
+ }
+
+ return 1;
+}
+
+#endif // __APPLE__
diff --git a/src/AS_UTL/libbacktrace/fileline.c b/src/AS_UTL/libbacktrace/fileline.c
new file mode 100644
index 0000000..503bbc6
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/fileline.c
@@ -0,0 +1,194 @@
+/* fileline.c -- Get file and line number information in a backtrace.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+#ifndef HAVE_GETEXECNAME
+#define getexecname() NULL
+#endif
+
+/* Initialize the fileline information from the executable. Returns 1
+ on success, 0 on failure. */
+
+static int
+fileline_initialize (struct backtrace_state *state,
+ backtrace_error_callback error_callback, void *data)
+{
+ int failed;
+ fileline fileline_fn;
+ int pass;
+ int called_error_callback;
+ int descriptor;
+
+ if (!state->threaded)
+ failed = state->fileline_initialization_failed;
+ else
+ failed = backtrace_atomic_load_int (&state->fileline_initialization_failed);
+
+ if (failed)
+ {
+ error_callback (data, "failed to read executable information", -1);
+ return 0;
+ }
+
+ if (!state->threaded)
+ fileline_fn = state->fileline_fn;
+ else
+ fileline_fn = backtrace_atomic_load_pointer (&state->fileline_fn);
+ if (fileline_fn != NULL)
+ return 1;
+
+ /* We have not initialized the information. Do it now. */
+
+ descriptor = -1;
+ called_error_callback = 0;
+ for (pass = 0; pass < 4; ++pass)
+ {
+ const char *filename;
+ int does_not_exist;
+
+ switch (pass)
+ {
+ case 0:
+ filename = state->filename;
+ break;
+ case 1:
+ filename = getexecname ();
+ break;
+ case 2:
+ filename = "/proc/self/exe";
+ break;
+ case 3:
+ filename = "/proc/curproc/file";
+ break;
+ default:
+ abort ();
+ }
+
+ if (filename == NULL)
+ continue;
+
+ descriptor = backtrace_open (filename, error_callback, data,
+ &does_not_exist);
+ if (descriptor < 0 && !does_not_exist)
+ {
+ called_error_callback = 1;
+ break;
+ }
+ if (descriptor >= 0)
+ break;
+ }
+
+ if (descriptor < 0)
+ {
+ if (!called_error_callback)
+ {
+ if (state->filename != NULL)
+ error_callback (data, state->filename, ENOENT);
+ else
+ error_callback (data,
+ "libbacktrace could not find executable to open",
+ 0);
+ }
+ failed = 1;
+ }
+
+ if (!failed)
+ {
+ if (!backtrace_initialize (state, descriptor, error_callback, data,
+ &fileline_fn))
+ failed = 1;
+ }
+
+ if (failed)
+ {
+ if (!state->threaded)
+ state->fileline_initialization_failed = 1;
+ else
+ backtrace_atomic_store_int (&state->fileline_initialization_failed, 1);
+ return 0;
+ }
+
+ if (!state->threaded)
+ state->fileline_fn = fileline_fn;
+ else
+ {
+ backtrace_atomic_store_pointer (&state->fileline_fn, fileline_fn);
+
+ /* Note that if two threads initialize at once, one of the data
+ sets may be leaked. */
+ }
+
+ return 1;
+}
+
+/* Given a PC, find the file name, line number, and function name. */
+
+int
+backtrace_pcinfo (struct backtrace_state *state, uintptr_t pc,
+ backtrace_full_callback callback,
+ backtrace_error_callback error_callback, void *data)
+{
+ if (!fileline_initialize (state, error_callback, data))
+ return 0;
+
+ if (state->fileline_initialization_failed)
+ return 0;
+
+ return state->fileline_fn (state, pc, callback, error_callback, data);
+}
+
+/* Given a PC, find the symbol for it, and its value. */
+
+int
+backtrace_syminfo (struct backtrace_state *state, uintptr_t pc,
+ backtrace_syminfo_callback callback,
+ backtrace_error_callback error_callback, void *data)
+{
+ if (!fileline_initialize (state, error_callback, data))
+ return 0;
+
+ if (state->fileline_initialization_failed)
+ return 0;
+
+ state->syminfo_fn (state, pc, callback, error_callback, data);
+ return 1;
+}
diff --git a/src/AS_UTL/libbacktrace/internal.h b/src/AS_UTL/libbacktrace/internal.h
new file mode 100644
index 0000000..aab4e2a
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/internal.h
@@ -0,0 +1,294 @@
+/* internal.h -- Internal header file for stack backtrace library.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#ifndef BACKTRACE_INTERNAL_H
+#define BACKTRACE_INTERNAL_H
+
+/* We assume that <sys/types.h> and "backtrace.h" have already been
+ included. */
+
+#ifndef GCC_VERSION
+# define GCC_VERSION (__GNUC__ * 1000 + __GNUC_MINOR__)
+#endif
+
+#if (GCC_VERSION < 2007)
+# define __attribute__(x)
+#endif
+
+#ifndef ATTRIBUTE_UNUSED
+# define ATTRIBUTE_UNUSED __attribute__ ((__unused__))
+#endif
+
+#ifndef ATTRIBUTE_MALLOC
+# if (GCC_VERSION >= 2096)
+# define ATTRIBUTE_MALLOC __attribute__ ((__malloc__))
+# else
+# define ATTRIBUTE_MALLOC
+# endif
+#endif
+
+#ifndef HAVE_SYNC_FUNCTIONS
+
+/* Define out the sync functions. These should never be called if
+ they are not available. */
+
+#define __sync_bool_compare_and_swap(A, B, C) (abort(), 1)
+#define __sync_lock_test_and_set(A, B) (abort(), 0)
+#define __sync_lock_release(A) abort()
+
+#endif /* !defined (HAVE_SYNC_FUNCTIONS) */
+
+#ifdef HAVE_ATOMIC_FUNCTIONS
+
+/* We have the atomic builtin functions. */
+
+#define backtrace_atomic_load_pointer(p) \
+ __atomic_load_n ((p), __ATOMIC_ACQUIRE)
+#define backtrace_atomic_load_int(p) \
+ __atomic_load_n ((p), __ATOMIC_ACQUIRE)
+#define backtrace_atomic_store_pointer(p, v) \
+ __atomic_store_n ((p), (v), __ATOMIC_RELEASE)
+#define backtrace_atomic_store_size_t(p, v) \
+ __atomic_store_n ((p), (v), __ATOMIC_RELEASE)
+#define backtrace_atomic_store_int(p, v) \
+ __atomic_store_n ((p), (v), __ATOMIC_RELEASE)
+
+#else /* !defined (HAVE_ATOMIC_FUNCTIONS) */
+#ifdef HAVE_SYNC_FUNCTIONS
+
+/* We have the sync functions but not the atomic functions. Define
+ the atomic ones in terms of the sync ones. */
+
+extern void *backtrace_atomic_load_pointer (void *);
+extern int backtrace_atomic_load_int (int *);
+extern void backtrace_atomic_store_pointer (void *, void *);
+extern void backtrace_atomic_store_size_t (size_t *, size_t);
+extern void backtrace_atomic_store_int (int *, int);
+
+#else /* !defined (HAVE_SYNC_FUNCTIONS) */
+
+/* We have neither the sync nor the atomic functions. These will
+ never be called. */
+
+#define backtrace_atomic_load_pointer(p) (abort(), (void *) NULL)
+#define backtrace_atomic_load_int(p) (abort(), 0)
+#define backtrace_atomic_store_pointer(p, v) abort()
+#define backtrace_atomic_store_size_t(p, v) abort()
+#define backtrace_atomic_store_int(p, v) abort()
+
+#endif /* !defined (HAVE_SYNC_FUNCTIONS) */
+#endif /* !defined (HAVE_ATOMIC_FUNCTIONS) */
+
+/* The type of the function that collects file/line information. This
+ is like backtrace_pcinfo. */
+
+typedef int (*fileline) (struct backtrace_state *state, uintptr_t pc,
+ backtrace_full_callback callback,
+ backtrace_error_callback error_callback, void *data);
+
+/* The type of the function that collects symbol information. This is
+ like backtrace_syminfo. */
+
+typedef void (*syminfo) (struct backtrace_state *state, uintptr_t pc,
+ backtrace_syminfo_callback callback,
+ backtrace_error_callback error_callback, void *data);
+
+/* What the backtrace state pointer points to. */
+
+struct backtrace_state
+{
+ /* The name of the executable. */
+ const char *filename;
+ /* Non-zero if threaded. */
+ int threaded;
+ /* The master lock for fileline_fn, fileline_data, syminfo_fn,
+ syminfo_data, fileline_initialization_failed and everything the
+ data pointers point to. */
+ void *lock;
+ /* The function that returns file/line information. */
+ fileline fileline_fn;
+ /* The data to pass to FILELINE_FN. */
+ void *fileline_data;
+ /* The function that returns symbol information. */
+ syminfo syminfo_fn;
+ /* The data to pass to SYMINFO_FN. */
+ void *syminfo_data;
+ /* Whether initializing the file/line information failed. */
+ int fileline_initialization_failed;
+ /* The lock for the freelist. */
+ int lock_alloc;
+ /* The freelist when using mmap. */
+ struct backtrace_freelist_struct *freelist;
+};
+
+/* Open a file for reading. Returns -1 on error. If DOES_NOT_EXIST
+ is not NULL, *DOES_NOT_EXIST will be set to 0 normally and set to 1
+ if the file does not exist. If the file does not exist and
+ DOES_NOT_EXIST is not NULL, the function will return -1 and will
+ not call ERROR_CALLBACK. On other errors, or if DOES_NOT_EXIST is
+ NULL, the function will call ERROR_CALLBACK before returning. */
+extern int backtrace_open (const char *filename,
+ backtrace_error_callback error_callback,
+ void *data,
+ int *does_not_exist);
+
+/* A view of the contents of a file. This supports mmap when
+ available. A view will remain in memory even after backtrace_close
+ is called on the file descriptor from which the view was
+ obtained. */
+
+struct backtrace_view
+{
+ /* The data that the caller requested. */
+ const void *data;
+ /* The base of the view. */
+ void *base;
+ /* The total length of the view. */
+ size_t len;
+};
+
+/* Create a view of SIZE bytes from DESCRIPTOR at OFFSET. Store the
+ result in *VIEW. Returns 1 on success, 0 on error. */
+extern int backtrace_get_view (struct backtrace_state *state, int descriptor,
+ off_t offset, size_t size,
+ backtrace_error_callback error_callback,
+ void *data, struct backtrace_view *view);
+
+/* Release a view created by backtrace_get_view. */
+extern void backtrace_release_view (struct backtrace_state *state,
+ struct backtrace_view *view,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* Close a file opened by backtrace_open. Returns 1 on success, 0 on
+ error. */
+
+extern int backtrace_close (int descriptor,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* Sort without using memory. */
+
+extern void backtrace_qsort (void *base, size_t count, size_t size,
+ int (*compar) (const void *, const void *));
+
+/* Allocate memory. This is like malloc. If ERROR_CALLBACK is NULL,
+ this does not report an error, it just returns NULL. */
+
+extern void *backtrace_alloc (struct backtrace_state *state, size_t size,
+ backtrace_error_callback error_callback,
+ void *data) ATTRIBUTE_MALLOC;
+
+/* Free memory allocated by backtrace_alloc. If ERROR_CALLBACK is
+ NULL, this does not report an error. */
+
+extern void backtrace_free (struct backtrace_state *state, void *mem,
+ size_t size,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* A growable vector of some struct. This is used for more efficient
+ allocation when we don't know the final size of some group of data
+ that we want to represent as an array. */
+
+struct backtrace_vector
+{
+ /* The base of the vector. */
+ void *base;
+ /* The number of bytes in the vector. */
+ size_t size;
+ /* The number of bytes available at the current allocation. */
+ size_t alc;
+};
+
+/* Grow VEC by SIZE bytes. Return a pointer to the newly allocated
+ bytes. Note that this may move the entire vector to a new memory
+ location. Returns NULL on failure. */
+
+extern void *backtrace_vector_grow (struct backtrace_state *state, size_t size,
+ backtrace_error_callback error_callback,
+ void *data,
+ struct backtrace_vector *vec);
+
+/* Finish the current allocation on VEC. Prepare to start a new
+ allocation. The finished allocation will never be freed. Returns
+ a pointer to the base of the finished entries, or NULL on
+ failure. */
+
+extern void* backtrace_vector_finish (struct backtrace_state *state,
+ struct backtrace_vector *vec,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* Release any extra space allocated for VEC. This may change
+ VEC->base. Returns 1 on success, 0 on failure. */
+
+extern int backtrace_vector_release (struct backtrace_state *state,
+ struct backtrace_vector *vec,
+ backtrace_error_callback error_callback,
+ void *data);
+
+/* Read initial debug data from a descriptor, and set the
+ fileline_data, syminfo_fn, and syminfo_data fields of STATE.
+ Return the fileln_fn field in *FILELN_FN--this is done this way so
+ that the synchronization code is only implemented once. This is
+ called after the descriptor has first been opened. It will close
+ the descriptor if it is no longer needed. Returns 1 on success, 0
+ on error. There will be multiple implementations of this function,
+ for different file formats. Each system will compile the
+ appropriate one. */
+
+extern int backtrace_initialize (struct backtrace_state *state,
+ int descriptor,
+ backtrace_error_callback error_callback,
+ void *data,
+ fileline *fileline_fn);
+
+/* Add file/line information for a DWARF module. */
+
+extern int backtrace_dwarf_add (struct backtrace_state *state,
+ uintptr_t base_address,
+ const unsigned char* dwarf_info,
+ size_t dwarf_info_size,
+ const unsigned char *dwarf_line,
+ size_t dwarf_line_size,
+ const unsigned char *dwarf_abbrev,
+ size_t dwarf_abbrev_size,
+ const unsigned char *dwarf_ranges,
+ size_t dwarf_range_size,
+ const unsigned char *dwarf_str,
+ size_t dwarf_str_size,
+ int is_bigendian,
+ backtrace_error_callback error_callback,
+ void *data, fileline *fileline_fn);
+
+#endif
diff --git a/src/AS_UTL/libbacktrace/make.out b/src/AS_UTL/libbacktrace/make.out
new file mode 100644
index 0000000..778a319
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/make.out
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+OPTS="-W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual -g -O2"
+
+gcc48 -I. -funwind-tables $OPTS -c atomic.c -o atomic.o
+gcc48 -I. -funwind-tables $OPTS -c dwarf.c -o dwarf.o
+gcc48 -I. -funwind-tables $OPTS -c fileline.c -o fileline.o
+gcc48 -I. -funwind-tables $OPTS -c posix.c -o posix.o
+gcc48 -I. -funwind-tables $OPTS -c print.c -o print.o
+gcc48 -I. -funwind-tables $OPTS -c sort.c -o sort.o
+gcc48 -I. -funwind-tables $OPTS -c state.c -o state.o
+gcc48 -I. -funwind-tables $OPTS -c backtrace.c -o backtrace.o
+gcc48 -I. -funwind-tables $OPTS -c simple.c -o simple.o
+gcc48 -I. -funwind-tables $OPTS -c elf.c -o elf.o
+gcc48 -I. -funwind-tables $OPTS -c mmapio.c -o mmapio.o
+gcc48 -I. -funwind-tables $OPTS -c mmap.c -o mmap.o
+
+#/bin/sh ./libtool --tag=CC --mode=link gcc48 -funwind-tables -frandom-seed=libbacktrace.la -W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual -g -O2 -o libbacktrace.la -rpath /work/canu-stack-trace/src/AS_UTL/libbacktrace/../../../FreeBSD-amd64/lib atomic.lo dwarf.lo fileline.lo posix.lo print.lo sort.lo state.lo backtrace.lo simple.lo elf.lo mmapio.lo mmap.lo
+
+ar cru libbacktrace.a atomic.o dwarf.o fileline.o posix.o print.o sort.o state.o backtrace.o simple.o elf.o mmapio.o mmap.o
+ranlib libbacktrace.a
+
+
+#
+#atomic.c dwarf.c fileline.c posix.c print.c sort.c state.c backtrace.c simple.c elf.c mmapio.c mmap.c
+#
\ No newline at end of file
diff --git a/src/AS_UTL/libbacktrace/make.sh b/src/AS_UTL/libbacktrace/make.sh
new file mode 100644
index 0000000..660de54
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/make.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+OPTS="-W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual -g -O2"
+
+gcc48 -I. -funwind-tables $OPTS -c atomic.c -o atomic.o
+gcc48 -I. -funwind-tables $OPTS -c dwarf.c -o dwarf.o
+gcc48 -I. -funwind-tables $OPTS -c fileline.c -o fileline.o
+gcc48 -I. -funwind-tables $OPTS -c posix.c -o posix.o
+gcc48 -I. -funwind-tables $OPTS -c print.c -o print.o
+gcc48 -I. -funwind-tables $OPTS -c sort.c -o sort.o
+gcc48 -I. -funwind-tables $OPTS -c state.c -o state.o
+gcc48 -I. -funwind-tables $OPTS -c backtrace.c -o backtrace.o
+gcc48 -I. -funwind-tables $OPTS -c simple.c -o simple.o
+gcc48 -I. -funwind-tables $OPTS -c elf.c -o elf.o
+gcc48 -I. -funwind-tables $OPTS -c mmapio.c -o mmapio.o
+gcc48 -I. -funwind-tables $OPTS -c mmap.c -o mmap.o
+
+#/bin/sh ./libtool --tag=CC --mode=link gcc48 -funwind-tables -frandom-seed=libbacktrace.la -W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual -g -O2 -o libbacktrace.la -rpath /work/canu-stack-trace/src/AS_UTL/libbacktrace/../../../FreeBSD-amd64/lib atomic.lo dwarf.lo fileline.lo posix.lo print.lo sort.lo state.lo backtrace.lo simple.lo elf.lo mmapio.lo mmap.lo
+
+ar cru libbacktrace.a atomic.o dwarf.o fileline.o posix.o print.o sort.o state.o backtrace.o simple.o elf.o mmapio.o mmap.o
+ranlib libbacktrace.a
+
diff --git a/src/AS_UTL/libbacktrace/mmap.c b/src/AS_UTL/libbacktrace/mmap.c
new file mode 100644
index 0000000..e30d1c1
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/mmap.c
@@ -0,0 +1,303 @@
+/* mmap.c -- Memory allocation with mmap.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+/* Memory allocation on systems that provide anonymous mmap. This
+ permits the backtrace functions to be invoked from a signal
+ handler, assuming that mmap is async-signal safe. */
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+#ifndef MAP_FAILED
+#define MAP_FAILED ((void *)-1)
+#endif
+
+/* A list of free memory blocks. */
+
+struct backtrace_freelist_struct
+{
+ /* Next on list. */
+ struct backtrace_freelist_struct *next;
+ /* Size of this block, including this structure. */
+ size_t size;
+};
+
+/* Free memory allocated by backtrace_alloc. */
+
+static void
+backtrace_free_locked (struct backtrace_state *state, void *addr, size_t size)
+{
+ /* Just leak small blocks. We don't have to be perfect. */
+ if (size >= sizeof (struct backtrace_freelist_struct))
+ {
+ struct backtrace_freelist_struct *p;
+
+ p = (struct backtrace_freelist_struct *) addr;
+ p->next = state->freelist;
+ p->size = size;
+ state->freelist = p;
+ }
+}
+
+/* Allocate memory like malloc. If ERROR_CALLBACK is NULL, don't
+ report an error. */
+
+void *
+backtrace_alloc (struct backtrace_state *state,
+ size_t size, backtrace_error_callback error_callback,
+ void *data)
+{
+ void *ret;
+ int locked;
+ struct backtrace_freelist_struct **pp;
+ size_t pagesize;
+ size_t asksize;
+ void *page;
+
+ ret = NULL;
+
+ /* If we can acquire the lock, then see if there is space on the
+ free list. If we can't acquire the lock, drop straight into
+ using mmap. __sync_lock_test_and_set returns the old state of
+ the lock, so we have acquired it if it returns 0. */
+
+ if (!state->threaded)
+ locked = 1;
+ else
+ locked = __sync_lock_test_and_set (&state->lock_alloc, 1) == 0;
+
+ if (locked)
+ {
+ for (pp = &state->freelist; *pp != NULL; pp = &(*pp)->next)
+ {
+ if ((*pp)->size >= size)
+ {
+ struct backtrace_freelist_struct *p;
+
+ p = *pp;
+ *pp = p->next;
+
+ /* Round for alignment; we assume that no type we care about
+ is more than 8 bytes. */
+ size = (size + 7) & ~ (size_t) 7;
+ if (size < p->size)
+ backtrace_free_locked (state, (char *) p + size,
+ p->size - size);
+
+ ret = (void *) p;
+
+ break;
+ }
+ }
+
+ if (state->threaded)
+ __sync_lock_release (&state->lock_alloc);
+ }
+
+ if (ret == NULL)
+ {
+ /* Allocate a new page. */
+
+ pagesize = getpagesize ();
+ asksize = (size + pagesize - 1) & ~ (pagesize - 1);
+ page = mmap (NULL, asksize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (page == MAP_FAILED)
+ {
+ if (error_callback)
+ error_callback (data, "mmap", errno);
+ }
+ else
+ {
+ size = (size + 7) & ~ (size_t) 7;
+ if (size < asksize)
+ backtrace_free (state, (char *) page + size, asksize - size,
+ error_callback, data);
+
+ ret = page;
+ }
+ }
+
+ return ret;
+}
+
+/* Free memory allocated by backtrace_alloc. */
+
+void
+backtrace_free (struct backtrace_state *state, void *addr, size_t size,
+ backtrace_error_callback error_callback ATTRIBUTE_UNUSED,
+ void *data ATTRIBUTE_UNUSED)
+{
+ int locked;
+
+ /* If we are freeing a large aligned block, just release it back to
+ the system. This case arises when growing a vector for a large
+ binary with lots of debug info. Calling munmap here may cause us
+ to call mmap again if there is also a large shared library; we
+ just live with that. */
+ if (size >= 16 * 4096)
+ {
+ size_t pagesize;
+
+ pagesize = getpagesize ();
+ if (((uintptr_t) addr & (pagesize - 1)) == 0
+ && (size & (pagesize - 1)) == 0)
+ {
+ /* If munmap fails for some reason, just add the block to
+ the freelist. */
+ if (munmap (addr, size) == 0)
+ return;
+ }
+ }
+
+ /* If we can acquire the lock, add the new space to the free list.
+ If we can't acquire the lock, just leak the memory.
+ __sync_lock_test_and_set returns the old state of the lock, so we
+ have acquired it if it returns 0. */
+
+ if (!state->threaded)
+ locked = 1;
+ else
+ locked = __sync_lock_test_and_set (&state->lock_alloc, 1) == 0;
+
+ if (locked)
+ {
+ backtrace_free_locked (state, addr, size);
+
+ if (state->threaded)
+ __sync_lock_release (&state->lock_alloc);
+ }
+}
+
+/* Grow VEC by SIZE bytes. */
+
+void *
+backtrace_vector_grow (struct backtrace_state *state,size_t size,
+ backtrace_error_callback error_callback,
+ void *data, struct backtrace_vector *vec)
+{
+ void *ret;
+
+ if (size > vec->alc)
+ {
+ size_t pagesize;
+ size_t alc;
+ void *base;
+
+ pagesize = getpagesize ();
+ alc = vec->size + size;
+ if (vec->size == 0)
+ alc = 16 * size;
+ else if (alc < pagesize)
+ {
+ alc *= 2;
+ if (alc > pagesize)
+ alc = pagesize;
+ }
+ else
+ {
+ alc *= 2;
+ alc = (alc + pagesize - 1) & ~ (pagesize - 1);
+ }
+ base = backtrace_alloc (state, alc, error_callback, data);
+ if (base == NULL)
+ return NULL;
+ if (vec->base != NULL)
+ {
+ memcpy (base, vec->base, vec->size);
+ backtrace_free (state, vec->base, vec->size + vec->alc,
+ error_callback, data);
+ }
+ vec->base = base;
+ vec->alc = alc - vec->size;
+ }
+
+ ret = (char *) vec->base + vec->size;
+ vec->size += size;
+ vec->alc -= size;
+ return ret;
+}
+
+/* Finish the current allocation on VEC. */
+
+void *
+backtrace_vector_finish (
+ struct backtrace_state *state ATTRIBUTE_UNUSED,
+ struct backtrace_vector *vec,
+ backtrace_error_callback error_callback ATTRIBUTE_UNUSED,
+ void *data ATTRIBUTE_UNUSED)
+{
+ void *ret;
+
+ ret = vec->base;
+ vec->base = (char *) vec->base + vec->size;
+ vec->size = 0;
+ return ret;
+}
+
+/* Release any extra space allocated for VEC. */
+
+int
+backtrace_vector_release (struct backtrace_state *state,
+ struct backtrace_vector *vec,
+ backtrace_error_callback error_callback,
+ void *data)
+{
+ size_t size;
+ size_t alc;
+ size_t aligned;
+
+ /* Make sure that the block that we free is aligned on an 8-byte
+ boundary. */
+ size = vec->size;
+ alc = vec->alc;
+ aligned = (size + 7) & ~ (size_t) 7;
+ alc -= aligned - size;
+
+ backtrace_free (state, (char *) vec->base + aligned, alc,
+ error_callback, data);
+ vec->alc = 0;
+ return 1;
+}
diff --git a/src/AS_UTL/libbacktrace/mmapio.c b/src/AS_UTL/libbacktrace/mmapio.c
new file mode 100644
index 0000000..8a9ba8e
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/mmapio.c
@@ -0,0 +1,100 @@
+/* mmapio.c -- File views using mmap.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+#ifndef MAP_FAILED
+#define MAP_FAILED ((void *)-1)
+#endif
+
+/* This file implements file views and memory allocation when mmap is
+ available. */
+
+/* Create a view of SIZE bytes from DESCRIPTOR at OFFSET. */
+
+int
+backtrace_get_view (struct backtrace_state *state ATTRIBUTE_UNUSED,
+ int descriptor, off_t offset, size_t size,
+ backtrace_error_callback error_callback,
+ void *data, struct backtrace_view *view)
+{
+ size_t pagesize;
+ unsigned int inpage;
+ off_t pageoff;
+ void *map;
+
+ pagesize = getpagesize ();
+ inpage = offset % pagesize;
+ pageoff = offset - inpage;
+
+ size += inpage;
+ size = (size + (pagesize - 1)) & ~ (pagesize - 1);
+
+ map = mmap (NULL, size, PROT_READ, MAP_PRIVATE, descriptor, pageoff);
+ if (map == MAP_FAILED)
+ {
+ error_callback (data, "mmap", errno);
+ return 0;
+ }
+
+ view->data = (char *) map + inpage;
+ view->base = map;
+ view->len = size;
+
+ return 1;
+}
+
+/* Release a view read by backtrace_get_view. */
+
+void
+backtrace_release_view (struct backtrace_state *state ATTRIBUTE_UNUSED,
+ struct backtrace_view *view,
+ backtrace_error_callback error_callback,
+ void *data)
+{
+ union {
+ const void *cv;
+ void *v;
+ } const_cast;
+
+ const_cast.cv = view->base;
+ if (munmap (const_cast.v, view->len) < 0)
+ error_callback (data, "munmap", errno);
+}
diff --git a/src/AS_UTL/libbacktrace/posix.c b/src/AS_UTL/libbacktrace/posix.c
new file mode 100644
index 0000000..be7357e
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/posix.c
@@ -0,0 +1,100 @@
+/* posix.c -- POSIX file I/O routines for the backtrace library.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
+#ifndef O_CLOEXEC
+#define O_CLOEXEC 0
+#endif
+
+#ifndef FD_CLOEXEC
+#define FD_CLOEXEC 1
+#endif
+
+/* Open a file for reading. */
+
+int
+backtrace_open (const char *filename, backtrace_error_callback error_callback,
+ void *data, int *does_not_exist)
+{
+ int descriptor;
+
+ if (does_not_exist != NULL)
+ *does_not_exist = 0;
+
+ descriptor = open (filename, (int) (O_RDONLY | O_BINARY | O_CLOEXEC));
+ if (descriptor < 0)
+ {
+ if (does_not_exist != NULL && errno == ENOENT)
+ *does_not_exist = 1;
+ else
+ error_callback (data, filename, errno);
+ return -1;
+ }
+
+#ifdef HAVE_FCNTL
+ /* Set FD_CLOEXEC just in case the kernel does not support
+ O_CLOEXEC. It doesn't matter if this fails for some reason.
+ FIXME: At some point it should be safe to only do this if
+ O_CLOEXEC == 0. */
+ fcntl (descriptor, F_SETFD, FD_CLOEXEC);
+#endif
+
+ return descriptor;
+}
+
+/* Close DESCRIPTOR. */
+
+int
+backtrace_close (int descriptor, backtrace_error_callback error_callback,
+ void *data)
+{
+ if (close (descriptor) < 0)
+ {
+ error_callback (data, "close", errno);
+ return 0;
+ }
+ return 1;
+}
diff --git a/src/AS_UTL/libbacktrace/print.c b/src/AS_UTL/libbacktrace/print.c
new file mode 100644
index 0000000..73b8abc
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/print.c
@@ -0,0 +1,92 @@
+/* print.c -- Print the current backtrace.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+/* Passed to callbacks. */
+
+struct print_data
+{
+ struct backtrace_state *state;
+ FILE *f;
+};
+
+/* Print one level of a backtrace. */
+
+static int
+print_callback (void *data, uintptr_t pc, const char *filename, int lineno,
+ const char *function)
+{
+ struct print_data *pdata = (struct print_data *) data;
+
+ fprintf (pdata->f, "0x%lx %s\n\t%s:%d\n",
+ (unsigned long) pc,
+ function == NULL ? "???" : function,
+ filename == NULL ? "???" : filename,
+ lineno);
+ return 0;
+}
+
+/* Print errors to stderr. */
+
+static void
+error_callback (void *data, const char *msg, int errnum)
+{
+ struct print_data *pdata = (struct print_data *) data;
+
+ if (pdata->state->filename != NULL)
+ fprintf (stderr, "%s: ", pdata->state->filename);
+ fprintf (stderr, "libbacktrace: %s", msg);
+ if (errnum > 0)
+ fprintf (stderr, ": %s", strerror (errnum));
+ fputc ('\n', stderr);
+}
+
+/* Print a backtrace. */
+
+void
+backtrace_print (struct backtrace_state *state, int skip, FILE *f)
+{
+ struct print_data data;
+
+ data.state = state;
+ data.f = f;
+ backtrace_full (state, skip + 1, print_callback, error_callback,
+ (void *) &data);
+}
diff --git a/src/AS_UTL/libbacktrace/simple.c b/src/AS_UTL/libbacktrace/simple.c
new file mode 100644
index 0000000..493fd6d
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/simple.c
@@ -0,0 +1,108 @@
+/* simple.c -- The backtrace_simple function.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include "unwind.h"
+#include "backtrace.h"
+
+/* The simple_backtrace routine. */
+
+/* Data passed through _Unwind_Backtrace. */
+
+struct backtrace_simple_data
+{
+ /* Number of frames to skip. */
+ int skip;
+ /* Library state. */
+ struct backtrace_state *state;
+ /* Callback routine. */
+ backtrace_simple_callback callback;
+ /* Error callback routine. */
+ backtrace_error_callback error_callback;
+ /* Data to pass to callback routine. */
+ void *data;
+ /* Value to return from backtrace. */
+ int ret;
+};
+
+/* Unwind library callback routine. This is passd to
+ _Unwind_Backtrace. */
+
+static _Unwind_Reason_Code
+simple_unwind (struct _Unwind_Context *context, void *vdata)
+{
+ struct backtrace_simple_data *bdata = (struct backtrace_simple_data *) vdata;
+ uintptr_t pc;
+ int ip_before_insn = 0;
+
+#ifdef HAVE_GETIPINFO
+ pc = _Unwind_GetIPInfo (context, &ip_before_insn);
+#else
+ pc = _Unwind_GetIP (context);
+#endif
+
+ if (bdata->skip > 0)
+ {
+ --bdata->skip;
+ return _URC_NO_REASON;
+ }
+
+ if (!ip_before_insn)
+ --pc;
+
+ bdata->ret = bdata->callback (bdata->data, pc);
+
+ if (bdata->ret != 0)
+ return _URC_END_OF_STACK;
+
+ return _URC_NO_REASON;
+}
+
+/* Get a simple stack backtrace. */
+
+int
+backtrace_simple (struct backtrace_state *state, int skip,
+ backtrace_simple_callback callback,
+ backtrace_error_callback error_callback, void *data)
+{
+ struct backtrace_simple_data bdata;
+
+ bdata.skip = skip + 1;
+ bdata.state = state;
+ bdata.callback = callback;
+ bdata.error_callback = error_callback;
+ bdata.data = data;
+ bdata.ret = 0;
+ _Unwind_Backtrace (simple_unwind, &bdata);
+ return bdata.ret;
+}
diff --git a/src/AS_UTL/libbacktrace/sort.c b/src/AS_UTL/libbacktrace/sort.c
new file mode 100644
index 0000000..f352fca
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/sort.c
@@ -0,0 +1,108 @@
+/* sort.c -- Sort without allocating memory
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <stddef.h>
+#include <sys/types.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+/* The GNU glibc version of qsort allocates memory, which we must not
+ do if we are invoked by a signal handler. So provide our own
+ sort. */
+
+static void
+swap (char *a, char *b, size_t size)
+{
+ size_t i;
+
+ for (i = 0; i < size; i++, a++, b++)
+ {
+ char t;
+
+ t = *a;
+ *a = *b;
+ *b = t;
+ }
+}
+
+void
+backtrace_qsort (void *basearg, size_t count, size_t size,
+ int (*compar) (const void *, const void *))
+{
+ char *base = (char *) basearg;
+ size_t i;
+ size_t mid;
+
+ tail_recurse:
+ if (count < 2)
+ return;
+
+ /* The symbol table and DWARF tables, which is all we use this
+ routine for, tend to be roughly sorted. Pick the middle element
+ in the array as our pivot point, so that we are more likely to
+ cut the array in half for each recursion step. */
+ swap (base, base + (count / 2) * size, size);
+
+ mid = 0;
+ for (i = 1; i < count; i++)
+ {
+ if ((*compar) (base, base + i * size) > 0)
+ {
+ ++mid;
+ if (i != mid)
+ swap (base + mid * size, base + i * size, size);
+ }
+ }
+
+ if (mid > 0)
+ swap (base, base + mid * size, size);
+
+ /* Recurse with the smaller array, loop with the larger one. That
+ ensures that our maximum stack depth is log count. */
+ if (2 * mid < count)
+ {
+ backtrace_qsort (base, mid, size, compar);
+ base += (mid + 1) * size;
+ count -= mid + 1;
+ goto tail_recurse;
+ }
+ else
+ {
+ backtrace_qsort (base + (mid + 1) * size, count - (mid + 1),
+ size, compar);
+ count = mid;
+ goto tail_recurse;
+ }
+}
diff --git a/src/AS_UTL/libbacktrace/state.c b/src/AS_UTL/libbacktrace/state.c
new file mode 100644
index 0000000..361a396
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/state.c
@@ -0,0 +1,72 @@
+/* state.c -- Create the backtrace state.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#include <string.h>
+#include <sys/types.h>
+
+#include "backtrace.h"
+#include "backtrace-supported.h"
+#include "internal.h"
+
+/* Create the backtrace state. This will then be passed to all the
+ other routines. */
+
+struct backtrace_state *
+backtrace_create_state (const char *filename, int threaded,
+ backtrace_error_callback error_callback,
+ void *data)
+{
+ struct backtrace_state init_state;
+ struct backtrace_state *state;
+
+#ifndef HAVE_SYNC_FUNCTIONS
+ if (threaded)
+ {
+ error_callback (data, "backtrace library does not support threads", 0);
+ return NULL;
+ }
+#endif
+
+ memset (&init_state, 0, sizeof init_state);
+ init_state.filename = filename;
+ init_state.threaded = threaded;
+
+ state = ((struct backtrace_state *)
+ backtrace_alloc (&init_state, sizeof *state, error_callback, data));
+ if (state == NULL)
+ return NULL;
+ *state = init_state;
+
+ return state;
+}
diff --git a/src/AS_UTL/libbacktrace/unknown.c b/src/AS_UTL/libbacktrace/unknown.c
new file mode 100644
index 0000000..23dd8f9
--- /dev/null
+++ b/src/AS_UTL/libbacktrace/unknown.c
@@ -0,0 +1,68 @@
+/* unknown.c -- used when backtrace configury does not know file format.
+ Copyright (C) 2012-2016 Free Software Foundation, Inc.
+ Written by Ian Lance Taylor, Google.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ (1) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ (2) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ (3) The name of the author may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE. */
+
+#include "config.h"
+
+#if (BACKTRACE_ELF_SIZE == unknown)
+
+#include <sys/types.h>
+
+#include "backtrace.h"
+#include "internal.h"
+
+/* A trivial routine that always fails to find fileline data. */
+
+static int
+unknown_fileline (struct backtrace_state *state ATTRIBUTE_UNUSED,
+ uintptr_t pc, backtrace_full_callback callback,
+ backtrace_error_callback error_callback ATTRIBUTE_UNUSED,
+ void *data)
+
+{
+ return callback (data, pc, NULL, 0, NULL);
+}
+
+/* Initialize the backtrace data when we don't know how to read the
+ debug info. */
+
+int
+backtrace_initialize (struct backtrace_state *state ATTRIBUTE_UNUSED,
+ int descriptor ATTRIBUTE_UNUSED,
+ backtrace_error_callback error_callback ATTRIBUTE_UNUSED,
+ void *data ATTRIBUTE_UNUSED, fileline *fileline_fn)
+{
+ state->fileline_data = NULL;
+ *fileline_fn = unknown_fileline;
+ return 1;
+}
+
+#endif // __APPLE__
diff --git a/src/AS_global.C b/src/AS_global.C
index 533f428..0b9304f 100644
--- a/src/AS_global.C
+++ b/src/AS_global.C
@@ -102,7 +102,7 @@ AS_configure(int argc, char **argv) {
// Install a signal handler to catch seg faults and errors.
- AS_UTL_installCrashCatcher();
+ AS_UTL_installCrashCatcher(argv[0]);
//
@@ -111,12 +111,7 @@ AS_configure(int argc, char **argv) {
for (int32 i=0; i<argc; i++) {
if (strcmp(argv[i], "--version") == 0) {
- fprintf(stderr, "Canu v%s.%s (+%s commits) r%s %s.\n",
- CANU_VERSION_MAJOR,
- CANU_VERSION_MINOR,
- CANU_VERSION_COMMITS,
- CANU_VERSION_REVISION,
- CANU_VERSION_HASH);
+ fputs(CANU_VERSION, stderr);
exit(0);
}
}
diff --git a/src/Makefile b/src/Makefile
index 05b2385..40ee044 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -97,7 +97,7 @@ define ADD_TARGET_RULE
$${TARGET_DIR}/${1}: $${${1}_OBJS} $${${1}_PREREQS}
@mkdir -p $$(dir $$@)
$$(strip $${${1}_LINKER} -o $$@ $${LDFLAGS} $${${1}_LDFLAGS} \
- $${${1}_OBJS} $${LDLIBS} $${${1}_LDLIBS})
+ $${${1}_OBJS} $${${1}_LDLIBS} $${LDLIBS})
$${${1}_POSTMAKE}
endif
endif
@@ -367,72 +367,67 @@ ifeq (${OSTYPE}, SunOS)
endif
endif
-# Set compiler and flags based on discovered hardware
+# Set compiler and flags based on discovered hardware
+#
+# By default, debug symbols are included in all builds (even optimized).
+#
+# BUILDOPTIMIZED will disable debug symbols (leaving it just optimized)
+# BUILDDEBUG will disable optimization (leaving it just with debug symbols)
+# BUILDSTACKTRACE will enable stack trace on crashes, only works for Linux
+# set to 0 on command line to disable (it's enabled by default for Linux)
+#
+# BUILDPROFILE used to add -pg to LDFLAGS, and remove -D_GLIBCXX_PARALLE from CXXFLAGS and LDFLAGS,
+# and remove -fomit-frame-pointer from CXXFLAGS. It added a bunch of complication and wasn't
+# really used.
ifeq (${OSTYPE}, Linux)
CC ?= gcc
CXX ?= g++
- CXXFLAGS := -pthread -fPIC -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare # -Wformat -Wall -Wextra
- LDFLAGS := -pthread -lm
- CXXFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
- LDFLAGS += -fopenmp
+ CXXFLAGS += -D_GLIBCXX_PARALLEL -pthread -fopenmp -fPIC
+ LDFLAGS += -D_GLIBCXX_PARALLEL -pthread -fopenmp -lm
CXXFLAGS += -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS +=
- LDFLAGS += -pg
+ BUILDSTACKTRACE ?= 1
+
+ ifeq ($(BUILDOPTIMIZED), 1)
else
- CXXFLAGS += -D_GLIBCXX_PARALLEL
- LDFLAGS += -D_GLIBCXX_PARALLEL
+ CXXFLAGS += -g3
endif
ifeq ($(BUILDDEBUG), 1)
- CXXFLAGS += -g
else
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions
- else
- CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
- endif
+ CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
endif
endif
ifeq (${OSTYPE}, Darwin)
- CLANG = $(shell echo `${CXX} --version 2>&1 | grep -c clang`)
-
CC ?= gcc
CXX ?= g++
+
+ CLANG = $(shell echo `${CXX} --version 2>&1 | grep -c clang`)
+
ifeq ($(CLANG), 0)
- CXXFLAGS := -pthread -fopenmp -fPIC -m64 -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat
- LDFLAGS := -pthread -fopenmp -lm
+ CXXFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
+ LDFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
else
- CXXFLAGS := -DBROKEN_CLANG_OpenMP -pthread -fPIC -m64 -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat
- LDFLAGS := -pthread -lm
+ CXXFLAGS += -DBROKEN_CLANG_OpenMP
+ LDFLAGS +=
endif
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS +=
- LDFLAGS += -pg
+ CXXFLAGS += -pthread -fPIC -m64 -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat
+ LDFLAGS += -pthread -lm
+
+ ifeq ($(BUILDOPTIMIZED), 1)
else
- CXXFLAGS += -D_GLIBCXX_PARALLEL
- LDFLAGS += -D_GLIBCXX_PARALLEL
+ CXXFLAGS += -g3
endif
- CXXFLAGS +=
- LDFLAGS +=
-
ifeq ($(BUILDDEBUG), 1)
- CXXFLAGS += -g
-# -fno-omit-frame-pointer
else
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS += -g3 -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fno-omit-frame-pointer
- else
- CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
- endif
+ CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
endif
endif
@@ -442,11 +437,10 @@ ifeq (${MACHINETYPE}, amd64)
CC ?= gcc48
CXX ?= g++48
- CXXFLAGS := -pthread -fPIC -Wall -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat -Wextra -Wno-parentheses
- LDFLAGS := -pthread -lm
+ CXXFLAGS += -I/usr/local/include -D_GLIBCXX_PARALLEL -pthread -fopenmp -fPIC
+ LDFLAGS += -L/usr/local/lib -D_GLIBCXX_PARALLEL -pthread -fopenmp -rpath /usr/local/lib/gcc48 -lm -lexecinfo
- CXXFLAGS += -fopenmp
- LDFLAGS += -fopenmp -rpath /usr/local/lib/gcc48
+ CXXFLAGS += -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat -Wno-parentheses
# Google Performance Tools malloc and heapchecker (HEAPCHECK=normal)
#CXXFLAGS +=
@@ -456,31 +450,17 @@ ifeq (${MACHINETYPE}, amd64)
#CXXFLAGS +=
#LDFLAGS += -lprofiler
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS +=
- LDFLAGS += -pg
- else
- CXXFLAGS += -D_GLIBCXX_PARALLEL
- LDFLAGS += -D_GLIBCXX_PARALLEL
- endif
-
- CXXFLAGS += -I/usr/local/include
- LDFLAGS += -L/usr/local/lib
-
# callgrind
#CXXFLAGS += -g3 -Wa,--gstabs -save-temps
- #CXXFLAGS += -DLIBUNWIND -I/usr/local/include
- #LDFLAGS += -L/usr/local/lib -lunwind -lunwind-x86_64
+ ifeq ($(BUILDOPTIMIZED), 1)
+ else
+ CXXFLAGS += -g3
+ endif
ifeq ($(BUILDDEBUG), 1)
- CXXFLAGS += -g3
else
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS += -g3 -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fno-omit-frame-pointer
- else
- CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
- endif
+ CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
endif
endif
endif
@@ -491,47 +471,117 @@ ifeq (${MACHINETYPE}, arm)
CC ?= gcc48
CXX ?= g++48
- CXXFLAGS := -pthread -fPIC -Wall -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat -Wextra
- LDFLAGS := -pthread -lm
+ CXXFLAGS += -I/usr/local/include -D_GLIBCXX_PARALLEL -pthread -fopenmp -fPIC
+ LDFLAGS += -L/usr/local/lib -D_GLIBCXX_PARALLEL -pthread -fopenmp -rpath /usr/local/lib/gcc48 -lm
+
+ CXXFLAGS += -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat -Wno-parentheses
+ CXXFLAGS += -funroll-loops -fomit-frame-pointer
+ LDFLAGS +=
- CXXFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
- LDFLAGS += -D_GLIBCXX_PARALLEL -fopenmp -rpath /usr/local/lib/gcc48
+ ifeq ($(BUILDOPTIMIZED), 1)
+ else
+ CXXFLAGS += -g3
+ endif
- CXXFLAGS += -I/usr/local/include -O3 -funroll-loops -fomit-frame-pointer
- LDFLAGS += -L/usr/local/lib
+ ifeq ($(BUILDDEBUG), 1)
+ else
+ CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
+ endif
endif
endif
+
ifneq (,$(findstring CYGWIN, ${OSTYPE}))
CC ?= gcc
CXX ?= g++
- CXXFLAGS := -pthread -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare
- LDFLAGS := -pthread -lm
- CXXFLAGS += -fopenmp
- LDFLAGS += -fopenmp
+ CXXFLAGS := -fopenmp -pthread
+ LDFLAGS := -fopenmp -pthread -lm
CXXFLAGS += -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS +=
- LDFLAGS += -pg
+ ifeq ($(BUILDOPTIMIZED), 1)
else
- CXXFLAGS +=
- LDFLAGS +=
+ CXXFLAGS += -g3
endif
ifeq ($(BUILDDEBUG), 1)
- CXXFLAGS += -g
else
- ifeq ($(BUILDPROFILE), 1)
- CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions
- else
- CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
- endif
+ CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
endif
endif
+
+# Stack tracing support. Wow, what a pain. Only Linux is supported. This is just documentation,
+# don't actually enable any of this stuff!
+#
+# backward-cpp looks very nice, only a single header file. But it needs libberty (-liberty) and
+# libbfd (-lbfd). The former should be installed with gcc, and the latter is in elfutils. On
+# three out of our three development machines, it fails for various reasons.
+#
+# libunwind is pretty basic.
+#
+# libbacktrace works (on Linux) and is simple enough to include in our tree.
+#
+# None of these give any useful information on BSDs (which includes OS X aka macOS).
+#
+#
+# Backtraces with libunwind. Not informative on FreeBSD.
+#CXXFLAGS += -DLIBUNWIND
+#LDFLAGS +=
+#LDLIBS += -lunwind -lunwind-x86_64
+#
+#
+# Backtraces with libbacktrace. FreeBSD works, but trace is empty.
+#BUILDSTACK = 1
+#CXXFLAGS += -DLIBBACKTRACE
+#LDFLAGS +=
+#LDLIBS +=
+#
+#
+# Backtraces with backward-cpp.
+#
+# Stack walking:
+# BACKWARD_HAS_UNWIND - used by gcc/clang for exception handling
+# BACKWARD_HAS_BACKTRACE - part of glib, not as accurate, more portable
+#
+# Stack interpretation:
+# BACKWARE_HAS_DW - most information, libdw, (elfutils or libdwarf)
+# BACKWARD_HAS_BFD - some information, libbfd
+# BACKWARD_HAS_BACKTRACE_SYMBOL - minimal information (file and function), portable
+#
+# helix fails with: cannot find -liberty
+# gryphon fails with: cannot find -lbfd
+# freebsd can't install a working elfutils, needed for libdw"
+# In file included from AS_UTL/AS_UTL_stackTrace.C:183:0:
+# AS_UTL/backward.hpp:241:30: fatal error: elfutils/libdw.h: No such file or directory
+# # include <elfutils/libdw.h>
+#
+#CXXFLAGS += -DBACKWARDCPP -DBACKWARD_HAS_BFD
+#LDFLAGS +=
+#LDLIBS += -lbfd -liberty -ldl -lz
+#
+# Needs libdw, elfutils
+#CXXFLAGS += -DBACKWARDCPP -DBACKWARD_HAS_DW
+#LDFLAGS +=
+#LDLIBS += -ldl -lz
+#
+# Generates nothing useful, no function names, just binary names
+#CXXFLAGS += -DBACKWARDCPP
+#LDFLAGS +=
+#LDLIBS += -ldl -lz
+#
+#
+# No backtrace support.
+#CXXFLAGS += -DNOBACKTRACE
+
+ifeq (${BUILDSTACKTRACE}, 1)
+CXXFLAGS += -DLIBBACKTRACE
+else
+CXXFLAGS += -DNOBACKTRACE
+endif
+
+
# Include the main user-supplied submakefile. This also recursively includes
# all other user-supplied submakefiles.
$(eval $(call INCLUDE_SUBMAKEFILE,main.mk))
@@ -555,6 +605,8 @@ all: UPDATE_VERSION MAKE_DIRS \
${TARGET_DIR}/lib/canu/Execution.pm \
${TARGET_DIR}/lib/canu/Gatekeeper.pm \
${TARGET_DIR}/lib/canu/Grid.pm \
+ ${TARGET_DIR}/lib/canu/Grid_Cloud.pm \
+ ${TARGET_DIR}/lib/canu/Grid_DNANexus.pm \
${TARGET_DIR}/lib/canu/Grid_LSF.pm \
${TARGET_DIR}/lib/canu/Grid_PBSTorque.pm \
${TARGET_DIR}/lib/canu/Grid_SGE.pm \
@@ -568,6 +620,7 @@ all: UPDATE_VERSION MAKE_DIRS \
${TARGET_DIR}/lib/canu/OverlapMhap.pm \
${TARGET_DIR}/lib/canu/OverlapMMap.pm \
${TARGET_DIR}/lib/canu/OverlapStore.pm \
+ ${TARGET_DIR}/lib/canu/Report.pm \
${TARGET_DIR}/lib/canu/Unitig.pm
@echo ""
@echo "Success!"
@@ -642,6 +695,12 @@ ${TARGET_DIR}/lib/canu/Gatekeeper.pm: pipelines/canu/Gatekeeper.pm
${TARGET_DIR}/lib/canu/Grid.pm: pipelines/canu/Grid.pm
cp -pf pipelines/canu/Grid.pm ${TARGET_DIR}/lib/canu/
+${TARGET_DIR}/lib/canu/Grid_Cloud.pm: pipelines/canu/Grid_Cloud.pm
+ cp -pf pipelines/canu/Grid_Cloud.pm ${TARGET_DIR}/lib/canu/
+
+${TARGET_DIR}/lib/canu/Grid_DNANexus.pm: pipelines/canu/Grid_DNANexus.pm
+ cp -pf pipelines/canu/Grid_DNANexus.pm ${TARGET_DIR}/lib/canu/
+
${TARGET_DIR}/lib/canu/Grid_LSF.pm: pipelines/canu/Grid_LSF.pm
cp -pf pipelines/canu/Grid_LSF.pm ${TARGET_DIR}/lib/canu/
@@ -681,10 +740,14 @@ ${TARGET_DIR}/lib/canu/OverlapMMap.pm: pipelines/canu/OverlapMMap.pm
${TARGET_DIR}/lib/canu/OverlapStore.pm: pipelines/canu/OverlapStore.pm
cp -pf pipelines/canu/OverlapStore.pm ${TARGET_DIR}/lib/canu/
+${TARGET_DIR}/lib/canu/Report.pm: pipelines/canu/Report.pm
+ cp -pf pipelines/canu/Report.pm ${TARGET_DIR}/lib/canu/
+
${TARGET_DIR}/lib/canu/Unitig.pm: pipelines/canu/Unitig.pm
cp -pf pipelines/canu/Unitig.pm ${TARGET_DIR}/lib/canu/
# Makefile processed. Report that we're starting the build.
${info Building for '${OSTYPE}' '${OSVERSION}' as '${MACHINETYPE}' into '${DESTDIR}${PREFIX}/$(OSTYPE)-$(MACHINETYPE)/{bin,obj}'}
+${info CC ${CC} CXX ${CXX}}
#${info Using LD_RUN_PATH '${LD_RUN_PATH}'}
diff --git a/src/bogart/AS_BAT_BestOverlapGraph.C b/src/bogart/AS_BAT_BestOverlapGraph.C
index 6d20b3d..6030cd7 100644
--- a/src/bogart/AS_BAT_BestOverlapGraph.C
+++ b/src/bogart/AS_BAT_BestOverlapGraph.C
@@ -56,8 +56,6 @@ BestOverlapGraph::removeSuspicious(const char *UNUSED(prefix)) {
uint32 numThreads = omp_get_max_threads();
uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- writeStatus("BestOverlapGraph()-- removing suspicious reads from graph, with %d thread%s.\n", numThreads, (numThreads == 1) ? "" : "s");
-
#pragma omp parallel for schedule(dynamic, blockSize)
for (uint32 fi=1; fi <= fiLimit; fi++) {
uint32 no = 0;
@@ -119,8 +117,6 @@ BestOverlapGraph::removeHighErrorBestEdges(void) {
uint32 numThreads = omp_get_max_threads();
uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- writeStatus("BestOverlapGraph()-- analyzing best edges to find useful edge error rate\n");
-
stdDev<double> edgeStats;
// Find the overlap for every best edge.
@@ -202,8 +198,6 @@ BestOverlapGraph::removeLopsidedEdges(const char *UNUSED(prefix)) {
uint32 numThreads = omp_get_max_threads();
uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- writeStatus("BestOverlapGraph()-- removing suspicious edges from graph, with %d thread%s.\n", numThreads, (numThreads == 1) ? "" : "s");
-
#pragma omp parallel for schedule(dynamic, blockSize)
for (uint32 fi=1; fi <= fiLimit; fi++) {
BestEdgeOverlap *this5 = getBestEdgeOverlap(fi, false);
@@ -219,6 +213,30 @@ BestOverlapGraph::removeLopsidedEdges(const char *UNUSED(prefix)) {
(this3->readId() == 0)))
continue;
+ // If there is a huge difference in error rates between the two best overlaps, that's a little
+ // suspicious. This kind-of worked, but it is very sensitive to the 'limit', and was only
+ // tested on one bacteria. It will also do very bad things in metagenomics.
+
+#if 0
+ double this5erate = this5->erate();
+ double this3erate = this3->erate();
+
+ double limit = 0.01;
+
+ if (fabs(this5erate - this3erate) > limit) {
+#pragma omp critical (suspInsert)
+ {
+ _suspicious.insert(fi);
+
+ writeStatus("Incompatible error rates on best edges for read %u -- %.4f %.4f.\n", fi, this5erate, this3erate);
+
+#warning NOT COUNTING ERATE DIFFS
+ //_ERateIncompatible++;
+ }
+ continue;
+ }
+#endif
+
// Find the overlap for this5 and this3.
int32 this5ovlLen = RI->overlapLength(fi, this5->readId(), this5->ahang(), this5->bhang());
@@ -303,8 +321,6 @@ BestOverlapGraph::removeSpurs(const char *prefix) {
if (errno)
F = NULL;
- writeStatus("BestOverlapGraph()-- detecting spur reads.\n");
-
_spur.clear();
for (uint32 fi=1; fi <= fiLimit; fi++) {
@@ -348,8 +364,6 @@ BestOverlapGraph::findEdges(void) {
memset(_bestA, 0, sizeof(BestOverlaps) * (fiLimit + 1));
memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1));
- writeStatus("BestOverlapGraph()-- analyzing %d reads for best contains, with %d thread%s.\n", fiLimit, numThreads, (numThreads == 1) ? "" : "s");
-
#pragma omp parallel for schedule(dynamic, blockSize)
for (uint32 fi=1; fi <= fiLimit; fi++) {
uint32 no = 0;
@@ -359,8 +373,6 @@ BestOverlapGraph::findEdges(void) {
scoreContainment(ovl[ii]);
}
- writeStatus("BestOverlapGraph()-- analyzing %d reads for best edges, with %d thread%s.\n", fiLimit, numThreads, (numThreads == 1) ? "" : "s");
-
#pragma omp parallel for schedule(dynamic, blockSize)
for (uint32 fi=1; fi <= fiLimit; fi++) {
uint32 no = 0;
@@ -382,8 +394,6 @@ void
BestOverlapGraph::removeContainedDovetails(void) {
uint32 fiLimit = RI->numReads();
- writeStatus("BestOverlapGraph()-- removing best edges for contained reads.\n");
-
for (uint32 fi=1; fi <= fiLimit; fi++) {
if (isContained(fi) == true) {
getBestEdgeOverlap(fi, false)->clear();
@@ -396,7 +406,11 @@ BestOverlapGraph::removeContainedDovetails(void) {
BestOverlapGraph::BestOverlapGraph(double erateGraph,
double deviationGraph,
- const char *prefix) {
+ const char *prefix,
+ bool filterSuspicious,
+ bool filterHighError,
+ bool filterLopsided,
+ bool filterSpur) {
writeStatus("\n");
writeStatus("BestOverlapGraph()-- allocating best edges (" F_SIZE_T "MB)\n",
@@ -432,21 +446,36 @@ BestOverlapGraph::BestOverlapGraph(double erateGraph,
// Find initial edges, only so we can report initial statistics on the graph
+ writeStatus("\n");
+ writeStatus("BestOverlapGraph()-- finding initial best edges.\n");
+
findEdges();
reportEdgeStatistics(prefix, "INITIAL");
// Mark reads as suspicious if they are not fully covered by overlaps.
- removeSuspicious(prefix);
- findEdges();
+ writeStatus("\n");
+ writeStatus("BestOverlapGraph()-- %sfiltering suspicious reads.\n",
+ (filterSuspicious == true) ? "" : "NOT ");
+
+ if (filterSuspicious) {
+ removeSuspicious(prefix);
+ findEdges();
+ }
if (logFileFlagSet(LOG_ALL_BEST_EDGES))
reportBestEdges(prefix, "best.0.initial");
// Analyze the current best edges to set a cutoff on overlap quality used for graph building.
- removeHighErrorBestEdges();
- findEdges();
+ writeStatus("\n");
+ writeStatus("BestOverlapGraph()-- %sfiltering high error edges.\n",
+ (filterHighError == true) ? "" : "NOT ");
+
+ if (filterHighError) {
+ removeHighErrorBestEdges();
+ findEdges();
+ }
if (logFileFlagSet(LOG_ALL_BEST_EDGES))
reportBestEdges(prefix, "best.1.filtered");
@@ -458,16 +487,28 @@ BestOverlapGraph::BestOverlapGraph(double erateGraph,
//
// This must come before removeSpurs().
- removeLopsidedEdges(prefix);
- findEdges();
+ writeStatus("\n");
+ writeStatus("BestOverlapGraph()-- %sfiltering reads with lopsided best edges.\n",
+ (filterLopsided == true) ? "" : "NOT ");
+
+ if (filterLopsided) {
+ removeLopsidedEdges(prefix);
+ findEdges();
+ }
if (logFileFlagSet(LOG_ALL_BEST_EDGES))
reportBestEdges(prefix, "best.2.cleaned");
// Mark reads as spurs, so we don't find best edges to them.
- removeSpurs(prefix);
- findEdges();
+ writeStatus("\n");
+ writeStatus("BestOverlapGraph()-- %sfiltering spur reads.\n",
+ (filterSpur == true) ? "" : "NOT ");
+
+ if (filterSpur) {
+ removeSpurs(prefix);
+ findEdges();
+ }
reportBestEdges(prefix, logFileFlagSet(LOG_ALL_BEST_EDGES) ? "best.3.final" : "best");
@@ -475,6 +516,9 @@ BestOverlapGraph::BestOverlapGraph(double erateGraph,
// Cleanup the contained reads. Why?
+ writeStatus("\n");
+ writeStatus("BestOverlapGraph()-- removing best edges for contained reads.\n");
+
removeContainedDovetails();
// Report filtering and final statistics.
diff --git a/src/bogart/AS_BAT_BestOverlapGraph.H b/src/bogart/AS_BAT_BestOverlapGraph.H
index f1b3730..5d14d29 100644
--- a/src/bogart/AS_BAT_BestOverlapGraph.H
+++ b/src/bogart/AS_BAT_BestOverlapGraph.H
@@ -187,9 +187,13 @@ private:
void removeContainedDovetails(void);
public:
- BestOverlapGraph(double erateGraph,
- double deviationGraph,
- const char *prefix);
+ BestOverlapGraph(double erateGraph,
+ double deviationGraph,
+ const char *prefix,
+ bool filterSuspicious,
+ bool filterHighError,
+ bool filterLopsided,
+ bool filterSpur);
~BestOverlapGraph() {
delete [] _bestA;
diff --git a/src/bogart/AS_BAT_ChunkGraph.C b/src/bogart/AS_BAT_ChunkGraph.C
index 3a9003a..fbc0d4b 100644
--- a/src/bogart/AS_BAT_ChunkGraph.C
+++ b/src/bogart/AS_BAT_ChunkGraph.C
@@ -147,12 +147,18 @@ ChunkGraph::countFullWidth(ReadEnd firstEnd) {
assert(firstIdx < _maxRead * 2 + 2);
- if (_pathLen[firstIdx] > 0)
+ if (_pathLen[firstIdx] > 0) {
+ if (_chunkLog)
+ fprintf(_chunkLog, "path from %d,%d'(length=%d)\n",
+ firstEnd.readId(),
+ (firstEnd.read3p()) ? 3 : 5,
+ _pathLen[firstIdx]);
return _pathLen[firstIdx];
+ }
uint32 length = 0;
- std::set<ReadEnd> seen;
- ReadEnd lastEnd = firstEnd;
+ std::set<ReadEnd> seen;
+ ReadEnd lastEnd = firstEnd;
uint64 lastIdx = firstIdx;
// Until we run off the chain, or we hit a read with a known length, compute the length FROM
@@ -205,13 +211,13 @@ ChunkGraph::countFullWidth(ReadEnd firstEnd) {
}
// Our return value is now whatever count we're at.
- uint32 lengthMax = length;
+ uint32 lengthMax = length;
// Traverse again, converting "path length from the start" into "path length from the end". Any
// cycle has had its length set correctly already, and we stop at either the start of the cycle,
// or at the start of any existing path.
//
- ReadEnd currEnd = firstEnd;
+ ReadEnd currEnd = firstEnd;
uint64 currIdx = firstIdx;
while (currEnd != lastEnd) {
diff --git a/src/bogart/AS_BAT_CreateUnitigs.C b/src/bogart/AS_BAT_CreateUnitigs.C
index 7b1c58e..ac168bb 100644
--- a/src/bogart/AS_BAT_CreateUnitigs.C
+++ b/src/bogart/AS_BAT_CreateUnitigs.C
@@ -463,7 +463,7 @@ createUnitigs(AssemblyGraph *AG,
unitigSource[id].uID = id;
}
}
-
+
// Reset for the next iteration.
ss = ee;
diff --git a/src/bogart/AS_BAT_MergeOrphans.H b/src/bogart/AS_BAT_MergeOrphans.H
index da8c37e..c626173 100644
--- a/src/bogart/AS_BAT_MergeOrphans.H
+++ b/src/bogart/AS_BAT_MergeOrphans.H
@@ -13,9 +13,13 @@
* Canu branched from Celera Assembler at its revision 4587.
* Canu branched from the kmer project at its revision 1994.
*
+ * This file is derived from:
+ *
+ * src/bogart/AS_BAT_PopBubbles.H
+ *
* Modifications by:
*
- * Brian P. Walenz beginning on 2016-MAR-11
+ * Brian P. Walenz beginning on 2016-DEC-07
* are a 'United States Government Work', and
* are released in the public domain
*
diff --git a/src/bogart/AS_BAT_OverlapCache.C b/src/bogart/AS_BAT_OverlapCache.C
index 97a9829..0e612db 100644
--- a/src/bogart/AS_BAT_OverlapCache.C
+++ b/src/bogart/AS_BAT_OverlapCache.C
@@ -418,8 +418,7 @@ OverlapCache::filterDuplicates(uint32 &no) {
nFiltered++;
- // If they're the same length, make the one with the higher evalue be length zero so it'll be
- // the shortest.
+ // Drop the shorter overlap, or the one with the higher erate.
uint32 iilen = RI->overlapLength(_ovs[ii].a_iid, _ovs[ii].b_iid, _ovs[ii].a_hang(), _ovs[ii].b_hang());
uint32 jjlen = RI->overlapLength(_ovs[jj].a_iid, _ovs[jj].b_iid, _ovs[jj].a_hang(), _ovs[jj].b_hang());
@@ -431,47 +430,60 @@ OverlapCache::filterDuplicates(uint32 &no) {
iilen = 0;
}
- // Drop the shorter overlap by forcing its erate to the maximum.
-
if (iilen < jjlen)
- _ovs[ii].evalue(AS_MAX_EVALUE);
+ _ovs[ii].a_iid = _ovs[ii].b_iid = 0;
else
- _ovs[jj].evalue(AS_MAX_EVALUE);
+ _ovs[jj].a_iid = _ovs[jj].b_iid = 0;
}
- // Now that all have been filtered, squeeze out the filtered overlaps. We used to just copy the
- // last element over any deleted ones, leaving the list unsorted, but we're now (Nov 2016) binary
- // searching on it, so can't do that.
+ // If nothing was filtered, return.
- if (nFiltered > 0) {
- // Needs to have it's own log. Lots of stuff here.
- //writeLog("OverlapCache()-- read %u filtered %u overlaps to the same read pair\n", _ovs[0].a_iid, nFiltered);
+ if (nFiltered == 0)
+ return(0);
- for (uint32 ii=0, jj=0; jj<no; ) {
- if (_ovs[jj].evalue() == AS_MAX_EVALUE) {
- jj++;
- continue;
- }
+ // Squeeze out the filtered overlaps. We used to just copy the last element over any deleted
+ // ones, leaving the list unsorted, but we're now (Nov 2016) binary searching on it, so can't do
+ // that.
- if (ii != jj) {
- _ovs[ii] = _ovs[jj];
- _ovs[jj].clear();
- }
+ // Needs to have it's own log. Lots of stuff here.
+ //writeLog("OverlapCache()-- read %u filtered %u overlaps to the same read pair\n", _ovs[0].a_iid, nFiltered);
- ii++;
+ for (uint32 ii=0, jj=0; jj<no; ) {
+ if (_ovs[jj].a_iid == 0) {
jj++;
+ continue;
}
- no -= nFiltered;
+ if (ii != jj)
+ _ovs[ii] = _ovs[jj];
- for (uint32 jj=0; jj<no; jj++) {
- assert(_ovs[jj].a_iid != 0);
- assert(_ovs[jj].b_iid != 0);
- assert(_ovs[jj].evalue() != AS_MAX_EVALUE);
- }
+ ii++;
+ jj++;
}
- return(nFiltered);
+ no -= nFiltered;
+
+ // Check for errors.
+
+ bool errors = false;
+
+ for (uint32 jj=0; jj<no; jj++)
+ if ((_ovs[jj].a_iid == 0) || (_ovs[jj].b_iid == 0))
+ errors = true;
+
+ if (errors == false)
+ return(nFiltered);
+
+ writeLog("ERROR: filtered overlap found in saved list for read %u. Filtered %u overlaps.\n", _ovs[0].a_iid, nFiltered);
+
+ for (uint32 jj=0; jj<no + nFiltered; jj++)
+ writeLog("OVERLAP %8d %8d hangs %5d %5d erate %.4f\n",
+ _ovs[jj].a_iid, _ovs[jj].b_iid, _ovs[jj].a_hang(), _ovs[jj].b_hang(), _ovs[jj].erate());
+
+ flushLog();
+
+ assert(errors == false);
+ return(0);
}
diff --git a/src/bogart/AS_BAT_TigGraph.C b/src/bogart/AS_BAT_TigGraph.C
index fee75b0..663cc45 100644
--- a/src/bogart/AS_BAT_TigGraph.C
+++ b/src/bogart/AS_BAT_TigGraph.C
@@ -33,6 +33,7 @@
#include "AS_BAT_TigGraph.H"
#undef SHOW_EDGES
+#undef SHOW_EDGES_UNPLACED // Generates a lot of noise
#undef SHOW_EDGES_VERBOSE
@@ -163,6 +164,16 @@ emitEdges(TigVector &tigs,
(double)placements[pp].errors / placements[pp].aligned);
#endif
+ // Decide the orientation of the second tig based on the orientation of the read and its
+ // alignment. If the orientations are the same, then the second tig doesn't need to be
+ // flipped.
+ //
+ // <-------------------------------------
+ // <--- read in first tig
+ //
+ // <--- alignment on second tig - so if not the same, the second tig needs to be
+ // -------------------> - flipped to make the alignment work
+
bool fwd = false;
if (((rdA->isForward() == true) && (placements[pp].verified.isForward() == true)) ||
@@ -212,7 +223,7 @@ emitEdges(TigVector &tigs,
(placements[pp].covered.end < rdAlen)) &&
(bgn > 100) &&
(end + 100 < tgBlen)) {
-#ifdef SHOW_EDGES
+#ifdef SHOW_EDGES_UNPLACED
writeLog("emitEdges()-- read %5u incomplete placement covering %5u-%-5u in at %5u-%-5u in tig %4u\n",
rdA->ident, placements[pp].covered.bgn, placements[pp].covered.end, bgn, end, tgBid);
#endif
@@ -243,7 +254,7 @@ emitEdges(TigVector &tigs,
if ((edges[ee].fwd == true) &&
(bgn - nbgn > nend - end)) { // If we decrease bgn more than we increased end, fail
-#ifdef SHOW_EDGES
+#ifdef SHOW_EDGES_UNPLACED
writeLog("emitEdges()-- edge %3u - extend from %5u-%-5u to %5u-%-5u -- placed read %5u at %5u-%-5u in tig %4u - wrong direction\n",
ee,
edges[ee].bgn, edges[ee].end,
@@ -258,7 +269,7 @@ emitEdges(TigVector &tigs,
if ((edges[ee].fwd == false) &&
(nend - end > bgn - nbgn)) { // If we increase end more than we decreased bgn, fail
-#ifdef SHOW_EDGES
+#ifdef SHOW_EDGES_UNPLACED
writeLog("emitEdges()-- edge %3u - extend from %5u-%-5u to %5u-%-5u -- placed read %5u at %5u-%-5u in tig %4u - wrong direction\n",
ee,
edges[ee].bgn, edges[ee].end,
@@ -287,6 +298,8 @@ emitEdges(TigVector &tigs,
// A better idea is to see if this read is overlapping with the first/last read
// in the other tig, and we're close enough to the end, instead of these silly 100bp thresholds.
+ // For edges making circles, when tgA == tgB, we need to flip tgB if tgA is flipped.
+
for (uint32 ee=0; ee<edges.size(); ee++) {
bool tgBflipped = (edges[ee].tigID == tgA->id()) && (tgAflipped);
@@ -304,8 +317,8 @@ emitEdges(TigVector &tigs,
edges[ee].end - edges[ee].bgn, edges[ee].bgn, edges[ee].end);
#endif
fprintf(BEG, "L\ttig%08u\t%c\ttig%08u\t%c\t%uM%s\n",
- tgA->id(), tgAflipped ? '-' : '+',
edges[ee].tigID, tgBflipped ? '+' : '-',
+ tgA->id(), tgAflipped ? '-' : '+',
edges[ee].end - edges[ee].bgn,
(sameContig == true) ? "\tcv:A:T" : "\tcv:A:F");
edges[ee].deleted = true;
@@ -320,8 +333,8 @@ emitEdges(TigVector &tigs,
edges[ee].end - edges[ee].bgn, edges[ee].bgn, edges[ee].end);
#endif
fprintf(BEG, "L\ttig%08u\t%c\ttig%08u\t%c\t%uM%s\n",
- tgA->id(), tgAflipped ? '-' : '+',
edges[ee].tigID, tgBflipped ? '-' : '+',
+ tgA->id(), tgAflipped ? '-' : '+',
edges[ee].end - edges[ee].bgn,
(sameContig == true) ? "\tcv:A:T" : "\tcv:A:F");
edges[ee].deleted = true;
@@ -337,6 +350,9 @@ emitEdges(TigVector &tigs,
if (edges[ee].fwd == false)
tgBflipped = !tgBflipped;
+ if (edges[ee].deleted == true)
+ continue;
+
if (edges[ee].extended == true)
continue;
@@ -351,7 +367,7 @@ emitEdges(TigVector &tigs,
edges[ee].deleted = true;
}
- // Compress the edges list (optional) to remove the deleted edges.
+ // Compress the edges list (optional, but messes up logging if not done) to remove the deleted edges.
uint32 oo = 0;
diff --git a/src/bogart/bogart.C b/src/bogart/bogart.C
index d750c06..ec0a35f 100644
--- a/src/bogart/bogart.C
+++ b/src/bogart/bogart.C
@@ -82,6 +82,11 @@ main (int argc, char * argv []) {
double erateGraph = 0.075;
double erateMax = 0.100;
+ bool filterSuspicious = true;
+ bool filterHighError = true;
+ bool filterLopsided = true;
+ bool filterSpur = true;
+
uint64 genomeSize = 0;
uint32 fewReadsNumber = 2; // Parameters for labeling of unassembled; also set in pipelines/canu/Defaults.pm
@@ -163,6 +168,13 @@ main (int argc, char * argv []) {
} else if (strcmp(argv[arg], "-dr") == 0) { // Deviations, repeat
deviationRepeat = atof(argv[++arg]);
+ } else if (strcmp(argv[arg], "-nofilter") == 0) {
+ ++arg;
+ filterSuspicious = ((arg >= argc) || (strcasestr(argv[arg], "suspicious") == NULL));
+ filterHighError = ((arg >= argc) || (strcasestr(argv[arg], "higherror") == NULL));
+ filterLopsided = ((arg >= argc) || (strcasestr(argv[arg], "lopsided") == NULL));
+ filterSpur = ((arg >= argc) || (strcasestr(argv[arg], "spur") == NULL));
+
} else if (strcmp(argv[arg], "-M") == 0) {
ovlCacheMemory = (uint64)(atof(argv[++arg]) * 1024 * 1024 * 1024);
@@ -247,6 +259,17 @@ main (int argc, char * argv []) {
fprintf(stderr, " -RL len Force reads below 'len' bases to be singletons.\n");
fprintf(stderr, " This WILL cause CGW to fail; diagnostic only.\n");
fprintf(stderr, "\n");
+ fprintf(stderr, " -nofilter [suspicious],[higherror],[lopsided],[spur]\n");
+ fprintf(stderr, " Disable filtering of:\n");
+ fprintf(stderr, " suspicious - reads that have a suspicious lack of overlaps\n");
+ fprintf(stderr, " higherror - overlaps that have error rates well outside the observed\n");
+ fprintf(stderr, " lopsided - reads that have unusually asymmetric best overlaps\n");
+ fprintf(stderr, " spur - reads that have no overlaps on one end\n");
+ fprintf(stderr, " The value supplied to -nofilter must be one word, order and punctuation\n");
+ fprintf(stderr, " do not matter. The following examples behave the same:\n");
+ fprintf(stderr, " '-nofilter suspicious,higherror'\n");
+ fprintf(stderr, " '-nofilter suspicious-and-higherror'\n");
+ fprintf(stderr, "\n");
fprintf(stderr, " -threads N Use N compute threads during repeat detection.\n");
fprintf(stderr, " 0 - use OpenMP default (default)\n");
fprintf(stderr, " 1 - use one thread\n");
@@ -320,7 +343,7 @@ main (int argc, char * argv []) {
RI = new ReadInfo(gkpStore, prefix, minReadLen);
OC = new OverlapCache(gkpStore, ovlStoreUniq, ovlStoreRept, prefix, MAX(erateMax, erateGraph), minOverlap, ovlCacheMemory, genomeSize, doSave);
- OG = new BestOverlapGraph(erateGraph, deviationGraph, prefix);
+ OG = new BestOverlapGraph(erateGraph, deviationGraph, prefix, filterSuspicious, filterHighError, filterLopsided, filterSpur);
CG = new ChunkGraph(prefix);
delete ovlStoreUniq; ovlStoreUniq = NULL;
diff --git a/src/canu_version_update.pl b/src/canu_version_update.pl
index ba3686a..a9ded13 100755
--- a/src/canu_version_update.pl
+++ b/src/canu_version_update.pl
@@ -30,24 +30,27 @@ use Cwd qw(getcwd);
my $cwd = getcwd();
-my $major = "0";
-my $minor = "0";
+my $label = "snapshot"; # Change this to 'release' just before making a release.
+my $major = "1"; # ...and this too.
+my $minor = "5"; # ...and this too.
+
my $commits = "0";
-my $hash = undef; # This from 'git describe'
-my $dirty = "";
-my $firstRev = undef; # This from 'git rev-list'
+my $hash1 = undef; # This from 'git describe'
+my $hash2 = undef; # This from 'git rev-list'
my $revCount = 0;
+my $dirty = undef;
# If in a git repo, we can get the actual values.
if (-d "../.git") {
+ $label = "snapshot";
open(F, "git rev-list HEAD |") or die "Failed to run 'git rev-list'.\n";
while (<F>) {
chomp;
- $firstRev = $_ if (!defined($firstRev));
+ $hash2 = $_ if (!defined($hash2));
$revCount++;
}
close(F);
@@ -59,7 +62,7 @@ if (-d "../.git") {
$major = $1;
$minor = $2;
$commits = $3;
- $hash = $4;
+ $hash1 = $4;
$dirty = $5;
} else {
die "Failed to parse describe string '$_'.\n";
@@ -77,36 +80,53 @@ if (-d "../.git") {
# If not in a git repo, we might be able to figure things out based on the directory name.
-elsif ((! -e "../.git") &&
- ($cwd =~ m/canu-(.{40})\/src/)) {
- $hash = $1;
- $firstRev = $1;
+elsif ($cwd =~ m/canu-(.{40})\/src/) {
+ $label = "snapshot";
+ $hash1 = $1;
+ $hash2 = $1;
}
+elsif ($cwd =~ m/canu-master\/src/) {
+ $label = "master-snapshot";
+}
-# Otherwise, we know absolutely nothing.
-
-else {
- $hash = "unknown-hash-tag-no-repository-available";
- $firstRev = "unknown-hash-tag-no-repository-available";
+elsif ($cwd =~ m/canu-(\d).(\d)\/src/) {
+ $label = "release";
+ $major = $1;
+ $minor = $2;
}
# Report what we found. This is really for the gmake output.
-print STDERR "Building v$major.$minor (+$commits commits) r$revCount $hash ($dirty)\n";
-print STDERR "\n";
+if ($commits > 0) {
+ print STDERR "Building snapshot v$major.$minor +$commits changes (r$revCount $hash1) ($dirty)\n";
+ print STDERR "\n";
+} else {
+ print STDERR "Building $label v$major.$minor\n";
+ print STDERR "\n";
+}
# Dump a new file, but don't overwrite the original.
open(F, "> canu_version.H.new") or die "Failed to open 'canu-version.H.new' for writing: $!\n";
print F "// Automagically generated by canu_version_update.pl! Do not commit!\n";
+print F "#define CANU_VERSION_LABEL \"$label\"\n";
print F "#define CANU_VERSION_MAJOR \"$major\"\n";
print F "#define CANU_VERSION_MINOR \"$minor\"\n";
print F "#define CANU_VERSION_COMMITS \"$commits\"\n";
print F "#define CANU_VERSION_REVISION \"$revCount\"\n";
-print F "#define CANU_VERSION_HASH \"$firstRev\"\n";
+print F "#define CANU_VERSION_HASH \"$hash1\"\n";
+
+if (defined($dirty)) {
+ print F "#define CANU_VERSION \"Canu snapshot v$major.$minor +$commits changes (r$revCount $hash1)\\n\"\n";
+} elsif (defined($hash1)) {
+ print F "#define CANU_VERSION \"Canu snapshot ($hash1)\\n\"\n";
+} else {
+ print F "#define CANU_VERSION \"Canu $label v$major.$minor\\n\"\n";
+}
+
close(F);
# If they're the same, don't replace the original.
diff --git a/src/correction/generateCorrectionLayouts.C b/src/correction/generateCorrectionLayouts.C
index 9067510..2779aff 100644
--- a/src/correction/generateCorrectionLayouts.C
+++ b/src/correction/generateCorrectionLayouts.C
@@ -41,6 +41,7 @@
#include "stashContains.H"
#include "splitToWords.H"
+#include "intervalList.H"
#include <set>
@@ -57,7 +58,7 @@ using namespace std;
tgTig *
generateLayout(gkStore *gkpStore,
uint64 *readScores,
- bool legacyScore,
+ bool legacyScore,
uint32 minEvidenceLength,
double maxEvidenceErate,
double maxEvidenceCoverage,
@@ -93,9 +94,10 @@ generateLayout(gkStore *gkpStore,
ovl[oo].b_end() - ovl[oo].b_bgn() :
ovl[oo].b_bgn() - ovl[oo].b_end());
uint64 ovlScore = 100 * ovlLength * (1 - ovl[oo].erate());
+
if (legacyScore) {
- ovlScore = ovlLength << AS_MAX_EVALUE_BITS;
- ovlScore |= (AS_MAX_EVALUE - ovl[oo].evalue());
+ ovlScore = ovlLength << AS_MAX_EVALUE_BITS;
+ ovlScore |= (AS_MAX_EVALUE - ovl[oo].evalue());
}
if (ovlLength > AS_MAX_READLEN) {
@@ -106,14 +108,14 @@ generateLayout(gkStore *gkpStore,
if (ovl[oo].erate() > maxEvidenceErate) {
if (flgFile)
- fprintf(flgFile, " filter read %9u at position %6u,%6u length %5lu erate %.3f - low quality (threshold %.2f)\n",
+ fprintf(flgFile, " filter read %9u at position %6u,%6u length %5llu erate %.3f - low quality (threshold %.2f)\n",
ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate(), maxEvidenceErate);
continue;
}
if (ovl[oo].a_end() - ovl[oo].a_bgn() < minEvidenceLength) {
if (flgFile)
- fprintf(flgFile, " filter read %9u at position %6u,%6u length %5lu erate %.3f - too short (threshold %u)\n",
+ fprintf(flgFile, " filter read %9u at position %6u,%6u length %5llu erate %.3f - too short (threshold %u)\n",
ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate(), minEvidenceLength);
continue;
}
@@ -121,20 +123,20 @@ generateLayout(gkStore *gkpStore,
if ((readScores != NULL) &&
(ovlScore < readScores[ovl[oo].b_iid])) {
if (flgFile)
- fprintf(flgFile, " filter read %9u at position %6u,%6u length %5lu erate %.3f - filtered by global filter (threshold " F_U64 ")\n",
+ fprintf(flgFile, " filter read %9u at position %6u,%6u length %5llu erate %.3f - filtered by global filter (threshold " F_U64 ")\n",
ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate(), readScores[ovl[oo].b_iid]);
continue;
}
if (children.find(ovl[oo].b_iid) != children.end()) {
if (flgFile)
- fprintf(flgFile, " filter read %9u at position %6u,%6u length %5lu erate %.3f - duplicate\n",
+ fprintf(flgFile, " filter read %9u at position %6u,%6u length %5llu erate %.3f - duplicate\n",
ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate());
continue;
}
if (flgFile)
- fprintf(flgFile, " allow read %9u at position %6u,%6u length %5lu erate %.3f\n",
+ fprintf(flgFile, " allow read %9u at position %6u,%6u length %5llu erate %.3f\n",
ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate());
tgPosition *pos = layout->addChild();
@@ -222,7 +224,7 @@ main(int argc, char **argv) {
FILE *flgFile = 0L;
uint32 minEvidenceOverlap = 40;
- uint32 minEvidenceCoverage = 1;
+ uint32 minEvidenceCoverage = 4;
uint32 iidMin = 0;
uint32 iidMax = UINT32_MAX;
@@ -236,7 +238,7 @@ main(int argc, char **argv) {
uint32 minCorLength = 0;
bool filterCorLength = false;
- bool legacyScore = false;
+ bool legacyScore = false;
argc = AS_configure(argc, argv);
@@ -263,7 +265,6 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-p") == 0) { // Output prefix, just logging and summary
outputPrefix = argv[++arg];
-
} else if (strcmp(argv[arg], "-b") == 0) { // Begin read range
iidMin = atoi(argv[++arg]);
@@ -273,17 +274,18 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-rl") == 0) { // List of reads to correct, will also apply -b/-e range
readListName = argv[++arg];
-
} else if (strcmp(argv[arg], "-L") == 0) { // Minimum length of evidence overlap
minEvidenceLength = atoi(argv[++arg]);
} else if (strcmp(argv[arg], "-E") == 0) { // Max error rate of evidence overlap
maxEvidenceErate = atof(argv[++arg]);
+ } else if (strcmp(argv[arg], "-c") == 0) { // Min coverage of evidence reads to consider the read corrected
+ minEvidenceCoverage = atof(argv[++arg]);
+
} else if (strcmp(argv[arg], "-C") == 0) { // Max coverage of evidence reads to emit.
maxEvidenceCoverage = atof(argv[++arg]);
-
} else if (strcmp(argv[arg], "-M") == 0) { // Minimum length of a corrected read
minCorLength = atoi(argv[++arg]);
@@ -319,7 +321,9 @@ main(int argc, char **argv) {
fprintf(stderr, " -rl file \n");
fprintf(stderr, "\n");
fprintf(stderr, " -L length minimum length of evidence overlaps\n");
- fprintf(stderr, " -E erate maxerror rate of evidence overlaps\n");
+ fprintf(stderr, " -E erate maximum error rate of evidence overlaps\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -c coverage minimum coverage needed in evidence reads\n");
fprintf(stderr, "\n");
fprintf(stderr, " -C coverage maximum coverage of evidence reads to emit\n");
fprintf(stderr, " -M length minimum length of a corrected read\n");
@@ -465,24 +469,33 @@ main(int argc, char **argv) {
skipIt = true;
}
- // Possibly filter by the length of the corrected read.
+ // Possibly filter by the length of the corrected read, taking into account depth of coverage.
- uint32 minPos = UINT32_MAX;
- uint32 maxPos = 0;
- uint32 corLen = 0;
+ intervalList<int32> coverage;
for (uint32 ii=0; ii<layout->numberOfChildren(); ii++) {
tgPosition *pos = layout->getChild(ii);
- if (pos->_min < minPos)
- minPos = pos->_min;
-
- if (maxPos < pos->_max)
- maxPos = pos->_max;
+ coverage.add(pos->_min, pos->_max - pos->_min);
}
- if (minPos != UINT32_MAX)
- corLen = maxPos - minPos;
+ intervalList<int32> depth(coverage);
+
+ int32 bgn = INT32_MAX;
+ int32 corLen = 0;
+
+ for (uint32 dd=0; dd<depth.numberOfIntervals(); dd++) {
+ if (depth.depth(dd) < minEvidenceCoverage) {
+ bgn = INT32_MAX;
+ continue;
+ }
+
+ if (bgn == INT32_MAX)
+ bgn = depth.lo(dd);
+
+ if (corLen < depth.hi(dd) - bgn)
+ corLen = depth.hi(dd) - bgn;
+ }
if (corLen < minCorLength) {
strcat(skipMsg, "\tcorrection_too_short");
diff --git a/src/falcon_sense/falcon_sense.C b/src/falcon_sense/falcon_sense.C
index f5fa694..cd30680 100644
--- a/src/falcon_sense/falcon_sense.C
+++ b/src/falcon_sense/falcon_sense.C
@@ -19,6 +19,10 @@
* are a 'United States Government Work', and
* are released in the public domain
*
+ * Brian P. Walenz beginning on 2016-NOV-28
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
* File 'README.licenses' in the root directory of this distribution contains
* full conditions and disclaimers for each license.
*/
@@ -107,11 +111,28 @@ main (int argc, char **argv) {
if (W[0][0] == '+') {
uint32 splitSeqID = 0;
FConsensus::consensus_data *consensus_data_ptr = FConsensus::generate_consensus( seqs, min_cov, K, min_idy, min_ovl_len, max_read_len );
+
+#ifdef TRACK_POSITIONS
+ //const std::string& sequenceToCorrect = seqs.at(0);
+ char * originalStringPointer = consensus_data_ptr->sequence;
+#endif
+
char * split = strtok(consensus_data_ptr->sequence, "acgt");
while (split != NULL) {
if (strlen(split) > min_len) {
AS_UTL_writeFastA(stdout, split, strlen(split), 60, ">%s_%d\n", seed.c_str(), splitSeqID);
splitSeqID++;
+
+#ifdef TRACK_POSITIONS
+ int distance_from_beginning = split - originalStringPointer;
+ std::vector<int> relevantOriginalPositions(consensus_data_ptr->originalPos.begin() + distance_from_beginning, consensus_data_ptr->originalPos.begin() + distance_from_beginning + strlen(split));
+ int firstRelevantPosition = relevantOriginalPositions.front();
+ int lastRelevantPosition = relevantOriginalPositions.back();
+
+ std::string relevantOriginalTemplate = seqs.at(0).substr(firstRelevantPosition, lastRelevantPosition - firstRelevantPosition + 1);
+
+ // store relevantOriginalTemplate along with corrected read - not implemented
+#endif
}
split = strtok(NULL, "acgt");
}
diff --git a/src/falcon_sense/falcon_sense.mk b/src/falcon_sense/falcon_sense.mk
index 4cb2f2e..1826aeb 100644
--- a/src/falcon_sense/falcon_sense.mk
+++ b/src/falcon_sense/falcon_sense.mk
@@ -10,7 +10,7 @@ endif
TARGET := falcon_sense
SOURCES := falcon_sense.C
-SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns/libNDFalcon libfalcon
+SRC_INCDIRS := .. ../AS_UTL ../stores ../overlapInCore/libedlib libfalcon
TGT_LDFLAGS := -L${TARGET_DIR}
TGT_LDLIBS := -lcanu
diff --git a/src/falcon_sense/libfalcon/falcon.C b/src/falcon_sense/libfalcon/falcon.C
index 05d4826..ef3f4c8 100644
--- a/src/falcon_sense/libfalcon/falcon.C
+++ b/src/falcon_sense/libfalcon/falcon.C
@@ -79,6 +79,7 @@
*/
#include "falcon.H"
+#include "edlib.H"
#include <stdlib.h>
#include <stdio.h>
@@ -87,14 +88,29 @@
#include <assert.h>
#include <stdint.h>
+#include <algorithm>
+
namespace FConsensus {
+#undef DEBUG
+
+
+typedef int32_t seq_coor_t;
+
+typedef struct {
+ seq_coor_t s1;
+ seq_coor_t e1;
+ seq_coor_t s2;
+ seq_coor_t e2;
+ long int score;
+} aln_range;
+
typedef struct {
seq_coor_t t_pos;
- uint8 delta;
+ uint16 delta;
char q_base;
seq_coor_t p_t_pos; // the tag position of the previous base
- uint8 p_delta; // the tag delta of the previous base
+ uint16 p_delta; // the tag delta of the previous base
char p_q_base; // the previous base
uint32 q_id;
} align_tag_t;
@@ -109,13 +125,13 @@ typedef struct {
uint16 size;
uint16 n_link;
seq_coor_t * p_t_pos; // the tag position of the previous base
- uint8 * p_delta; // the tag delta of the previous base
+ uint16 * p_delta; // the tag delta of the previous base
char * p_q_base; // the previous base
uint16 * link_count;
uint16 count;
seq_coor_t best_p_t_pos;
- uint8 best_p_delta;
- uint8 best_p_q_base; // encoded base
+ uint16 best_p_delta;
+ uint16 best_p_q_base; // encoded base
double score;
} align_tag_col_t;
@@ -124,8 +140,8 @@ typedef struct {
} msa_base_group_t;
typedef struct {
- uint8 size;
- uint8 max_delta;
+ uint16 size;
+ uint16 max_delta;
msa_base_group_t * delta;
} msa_delta_group_t;
@@ -136,7 +152,7 @@ align_tags_t * get_align_tags( char * aln_q_seq,
seq_coor_t aln_seq_len,
aln_range * range,
uint32 q_id,
- seq_coor_t t_offset) {
+ seq_coor_t t_offset, int a_len, int b_len) {
char p_q_base;
align_tags_t * tags;
seq_coor_t i, j, jj, k, p_j, p_jj;
@@ -160,10 +176,13 @@ align_tags_t * get_align_tags( char * aln_q_seq,
j ++;
jj = 0;
}
- //printf("t %d %d %d %c %c\n", q_id, j, jj, aln_t_seq[k], aln_q_seq[k]);
+ assert (i >= 0 && i < a_len && j >=0 && j < b_len);
+ #ifdef DEBUG
+ fprintf(stderr, "t %d %d %d %c %c\n", q_id, j, jj, aln_t_seq[k], aln_q_seq[k]);
+ #endif
- if ( j + t_offset >= 0 && jj < uint8MAX && p_jj < uint8MAX) {
+ if ( j + t_offset >= 0 && jj < uint16MAX && p_jj < uint16MAX) {
(tags->align_tags[k]).t_pos = j + t_offset;
(tags->align_tags[k]).delta = jj;
(tags->align_tags[k]).p_t_pos = p_j + t_offset;
@@ -181,7 +200,7 @@ align_tags_t * get_align_tags( char * aln_q_seq,
//k = aln_seq_len;
tags->len = k;
(tags->align_tags[k]).t_pos = uint32MAX;
- (tags->align_tags[k]).delta = uint8MAX;
+ (tags->align_tags[k]).delta = uint16MAX;
(tags->align_tags[k]).q_base = '.';
(tags->align_tags[k]).q_id = uint32MAX;
return tags;
@@ -195,14 +214,14 @@ void free_align_tags( align_tags_t * tags) {
void allocate_aln_col( align_tag_col_t * col) {
col->p_t_pos = ( seq_coor_t * ) calloc(col->size, sizeof( seq_coor_t ));
- col->p_delta = ( uint8 * ) calloc(col->size, sizeof( uint8 ));
+ col->p_delta = ( uint16 * ) calloc(col->size, sizeof( uint16 ));
col->p_q_base = ( char * )calloc(col->size, sizeof( char ));
col->link_count = ( uint16 * ) calloc(col->size, sizeof( uint16 ));
}
void realloc_aln_col( align_tag_col_t * col ) {
col->p_t_pos = (seq_coor_t *) realloc( col->p_t_pos, (col->size) * sizeof( seq_coor_t ));
- col->p_delta = ( uint8 *) realloc( col->p_delta, (col->size) * sizeof( uint8 ));
+ col->p_delta = ( uint16 *) realloc( col->p_delta, (col->size) * sizeof( uint16 ));
col->p_q_base = (char *) realloc( col->p_q_base, (col->size) * sizeof( char ));
col->link_count = ( uint16 *) realloc( col->link_count, (col->size) * sizeof( uint16 ));
}
@@ -255,7 +274,7 @@ void free_delta_group( msa_delta_group_t * g) {
free(g->delta);
}
-void update_col( align_tag_col_t * col, seq_coor_t p_t_pos, uint8 p_delta, char p_q_base) {
+void update_col( align_tag_col_t * col, seq_coor_t p_t_pos, uint16 p_delta, char p_q_base) {
int updated = 0;
int kk;
col->count += 1;
@@ -318,9 +337,9 @@ void clean_msa_working_space( msa_pos_t * msa_array, uint32 max_t_len) {
*/
col->n_link = 0;
col->count = 0;
- col->best_p_t_pos = 0;
- col->best_p_delta = 0;
- col->best_p_q_base = 0;
+ col->best_p_t_pos = -1;
+ col->best_p_delta = -1;
+ col->best_p_q_base = -1;
col->score = 0;
}
}
@@ -328,8 +347,8 @@ void clean_msa_working_space( msa_pos_t * msa_array, uint32 max_t_len) {
}
}
-#define STATIC_ALLOCATE
-//#undef STATIC_ALLOCATE
+
+
consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
uint32 n_tag_seqs,
@@ -340,7 +359,6 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
seq_coor_t t_pos = 0;
seq_coor_t t_count = 0;
uint32 * coverage;
- uint32 * local_nbase;
consensus_data * consensus;
align_tag_t * c_tag;
@@ -364,33 +382,18 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
}
coverage = (uint32 *)calloc( t_len, sizeof(uint32) );
- local_nbase = (uint32 *)calloc( t_len, sizeof(uint32) );
-
-#ifndef STATIC_ALLOCATE
-
- msa_array = (msa_pos_t *)calloc(t_len, sizeof(msa_pos_t));
-
- for (i = 0; i < t_len; i++) {
- msa_array[i] = (msa_delta_group_t *)calloc(1, sizeof(msa_delta_group_t));
- msa_array[i]->size = 8;
- allocate_delta_group(msa_array[i]);
- }
-
-#endif
-
-#ifdef STATIC_ALLOCATE
if ( msa_array == NULL) {
msa_array = get_msa_working_sapce( max_len );
+ clean_msa_working_space(msa_array, max_len);
}
assert(t_len < max_len);
-#endif
-
-
// loop through every alignment
- //printf("XX %d\n", n_tag_seqs);
+ #ifdef DEBUG
+ fprintf(stderr, "XX %d\n", n_tag_seqs);
+ #endif
for (i = 0; i < n_tag_seqs; i++) {
// for each alignment position, insert the alignment tag to msa_array
@@ -402,8 +405,13 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
t_pos = c_tag->t_pos;
coverage[ t_pos ] ++;
}
+ #ifdef DEBUG
+ fprintf(stderr, "Processing position %d in sequence %d (in msa it is column %d with cov %d) with delta %d and current size is %d\n", j, i, t_pos, coverage[t_pos], delta, msa_array[t_pos]->size);
+ #endif
+
// Assume t_pos was set on earlier iteration.
// (Otherwise, use its initial value, which might be an error. ~cd)
+ assert(delta < uint16MAX);
if (delta > msa_array[t_pos]->max_delta) {
msa_array[t_pos]->max_delta = delta;
if (msa_array[t_pos]->max_delta + 4 > msa_array[t_pos]->size ) {
@@ -421,8 +429,11 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
default : base = 4; break;
}
// Note: On bad input, base may be -1.
+ assert(c_tag->p_t_pos >= 0 || j == 0);
update_col( &(msa_array[t_pos]->delta[delta].base[base]), c_tag->p_t_pos, c_tag->p_delta, c_tag->p_q_base);
- local_nbase[ t_pos ] ++;
+ #ifdef DEBUG
+ fprintf(stderr, "Updating column from seq %d at position %d in column %d base pos %d base %d to be %c and max is %d\n", i, j, t_pos, base, c_tag->p_t_pos, c_tag->p_q_base, msa_array[t_pos]->max_delta);
+ #endif
}
}
@@ -448,7 +459,9 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
g_best_score = -1;
for (i = 0; i < t_len; i++) { //loop through every template base
- //printf("max delta: %d %d\n", i, msa_array[i]->max_delta);
+ #ifdef DEBUG
+ fprintf(stderr, "max delta: %d %d\n", i, msa_array[i]->max_delta);
+ #endif
for (j = 0; j <= msa_array[i]->max_delta; j++) { // loop through every delta position
for (kk = 0; kk < 5; kk++) { // loop through diff bases of the same delta posiiton
/*
@@ -466,6 +479,10 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
best_j = -1;
best_b = -1;
+ #ifdef DEBUG
+ fprintf(stderr, "Processing consensus template %d which as %d delta and on base %d i pulled up col %d with %d links and best %d %d %d\n", i, j, kk, aln_col, aln_col->n_link, aln_col->best_p_t_pos, aln_col->best_p_delta, aln_col->best_p_q_base);
+ #endif
+
for (ck = 0; ck < aln_col->n_link; ck++) { // loop through differnt link to previous column
int pi;
int pj;
@@ -498,14 +515,14 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
best_ck = ck;
// best_mark = '*';
}
- /*
- printf("X %d %d %d %c %d %d %d %c %d %lf %c\n", coverage[i], i, j, base, aln_col->count,
+ #ifdef DEBUG
+ fprintf(stderr, "X %d %d %d %d %d %d %c %d %lf\n", coverage[i], i, j, aln_col->count,
aln_col->p_t_pos[ck],
aln_col->p_delta[ck],
aln_col->p_q_base[ck],
aln_col->link_count[ck],
- score, best_mark);
- */
+ score);
+ #endif
}
aln_col->score = best_score;
if (best_score > g_best_score) {
@@ -513,7 +530,9 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
g_best_aln_col = aln_col;
g_best_ck = best_ck;
g_best_t_pos = i;
- //printf("GB %d %d %d %d\n", i, j, ck, g_best_aln_col);
+ #ifdef DEBUG
+ fprintf(stderr, "GB %d %d %d %d\n", i, j, ck, g_best_aln_col);
+ #endif
}
}
}
@@ -534,12 +553,18 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
consensus->eqv = (int32 *)calloc( t_len * 2 + 1, sizeof(int32) );
cns_str = consensus->sequence;
eqv = consensus->eqv;
+#ifdef TRACK_POSITIONS
+ consensus->originalPos.reserve(t_len * 2 + 1); // This is an over-generous pre-allocation
+#endif
index = 0;
ck = g_best_ck;
i = g_best_t_pos;
while (1) {
+#ifdef TRACK_POSITIONS
+ int originalI = i;
+#endif
if (coverage[i] > min_cov) {
switch (ck) {
case 0: bb = 'A'; break;
@@ -569,12 +594,22 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
if (bb != '-') {
cns_str[index] = bb;
eqv[index] = (int) score0 - (int) g_best_aln_col->score;
- //printf("C %d %d %c %lf %d %d\n", i, index, bb, g_best_aln_col->score, coverage[i], eqv[index] );
+ #ifdef DEBUG
+ fprintf(stderr, "C %d %d %c %lf %d %d\n", i, index, bb, g_best_aln_col->score, coverage[i], eqv[index] );
+ #endif
index ++;
+
+#ifdef TRACK_POSITIONS
+ consensus->originalPos.push_back(originalI);
+#endif
}
}
// reverse the sequence
+#ifdef TRACK_POSITIONS
+ std::reverse(consensus->originalPos.begin(), consensus->originalPos.end());
+#endif
+
for (i = 0; i < index/2; i++) {
cns_str[i] = cns_str[i] ^ cns_str[index-i-1];
cns_str[index-i-1] = cns_str[i] ^ cns_str[index-i-1];
@@ -586,21 +621,10 @@ consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs,
cns_str[index] = 0;
//printf("%s\n", cns_str);
-#ifndef STATIC_ALLOCATE
- for (i = 0; i < t_len; i++) {
- free_delta_group(msa_array[i]);
- free(msa_array[i]);
- }
-
- free(msa_array);
-#endif
-#ifdef STATIC_ALLOCATE
clean_msa_working_space(msa_array, t_len+1);
-#endif
free(coverage);
- free(local_nbase);
return consensus;
}
@@ -609,9 +633,6 @@ consensus_data * generate_consensus( vector<string> input_seq,
uint32 K,
double min_idt, uint32 min_len, uint32 max_len) {
uint32 seq_count;
- kmer_lookup * lk_ptr;
- seq_array sa_ptr;
- seq_addr_array sda_ptr;
align_tags_t ** tags_list;
consensus_data * consensus;
double max_diff;
@@ -621,60 +642,61 @@ consensus_data * generate_consensus( vector<string> input_seq,
fflush(stdout);
tags_list = (align_tags_t **)calloc( seq_count, sizeof(align_tags_t*) );
- lk_ptr = allocate_kmer_lookup( 1 << (K * 2) );
- sa_ptr = allocate_seq( (seq_coor_t) input_seq[0].length() );
- sda_ptr = allocate_seq_addr( (seq_coor_t) input_seq[0].length() );
- add_sequence( 0, K, input_seq[0].c_str(), input_seq[0].length(), sda_ptr, sa_ptr, lk_ptr);
-
#pragma omp parallel for schedule(dynamic)
for (uint32 j=0; j < seq_count; j++) {
-#define MAX_UNMASKED_LENGTH 500000
-#define MAX_KMER_REPEAT 1000
- if (input_seq[j].length() > MAX_UNMASKED_LENGTH) {
- mask_k_mer(1 << (K*2), lk_ptr, MAX_KMER_REPEAT);
- }
- kmer_match *kmer_match_ptr = find_kmer_pos_for_seq(input_seq[j].c_str(), input_seq[j].length(), K, sda_ptr, lk_ptr);
-#define INDEL_ALLOWENCE_0 6
-
- aln_range *arange = find_best_aln_range(kmer_match_ptr, K, K * INDEL_ALLOWENCE_0, 5); // narrow band to avoid aligning through big indels
-
- //fprintf(stderr, "1:read %d %ld %ld %ld %ld\n", j, arange->s1, arange->e1, arange->s2, arange->e2);
-
- //arange = find_best_aln_range2(kmer_match_ptr, K, K * INDEL_ALLOWENCE_0, 5); // narrow band to avoid aligning through big indels
+ int tolerance = (int)ceil((double)min(input_seq[j].length(), input_seq[0].length())*max_diff*1.1);
+ EdlibAlignResult align = edlibAlign(input_seq[j].c_str(), input_seq[j].size()-1, input_seq[0].c_str(), input_seq[0].size()-1, edlibNewAlignConfig(tolerance, EDLIB_MODE_HW, EDLIB_TASK_PATH));
+ if (align.numLocations >= 1 && align.endLocations[0] - align.startLocations[0] > min_len && ((float)align.editDistance / (align.endLocations[0]-align.startLocations[0]) < max_diff)) {
+ aln_range arange;
+ arange.s1 = 0;
+ arange.e1 = input_seq[j].length()-1;
+ arange.s2 = align.startLocations[0];
+ arange.e2 = align.endLocations[0];
+ #ifdef DEBUG
+ fprintf(stderr, "Found alignment for seq %d from %d - %d to %d - %d the dist %d length %d\n", j, arange.s1, arange.e1, arange.s2, arange.e2, align.editDistance, align.alignmentLength);
+ #endif
+
+ // convert edlib to expected
+ char *tgt_aln_str = (char *)calloc( align.alignmentLength+1, sizeof(char) );
+ char *qry_aln_str = (char *)calloc( align.alignmentLength+1, sizeof(char) );
+ edlibAlignmentToStrings(align.alignment, align.alignmentLength, arange.s2, arange.e2+1, arange.s1, arange.e1, input_seq[0].c_str(), input_seq[j].c_str(), tgt_aln_str, qry_aln_str);
+
+ // strip leading/trailing gaps on target
+ uint32_t first_pos = 0;
+ for (int i = 0; i < align.alignmentLength; i++) {
+ if (tgt_aln_str[i] != '-') {
+ first_pos=i;
+ break;
+ }
+ }
+ uint32_t last_pos = align.alignmentLength;
+ for (int i = align.alignmentLength-1; i >= 0; i--) {
+ if (tgt_aln_str[i] != '-') {
+ last_pos=i+1;
+ break;
+ }
+ }
+ arange.s1+= first_pos;
+ arange.e1-= (align.alignmentLength-last_pos);
+ arange.e2++;
+ qry_aln_str[last_pos]='\0';
+ tgt_aln_str[last_pos]='\0';
+
+ #ifdef DEBUG
+ fprintf(stderr, "Final positions to be %d %d for str %d and %d %d for str %d adjst %d %d %d\n", arange.s1, arange.e1, input_seq[j].length(), arange.s2, arange.e2, input_seq[0].length(), first_pos, last_pos, last_pos-first_pos);
+ fprintf(stderr, "Tgt string is %s %d\n", tgt_aln_str+first_pos, strlen(tgt_aln_str+first_pos));
+ fprintf(stderr, "Qry string is %s %d\n", qry_aln_str+first_pos, strlen(qry_aln_str+first_pos));
+ #endif
+ assert(arange.s1 >= 0 && arange.s2 >= 0 && arange.e1 <= input_seq[j].length() && arange.e2 <= input_seq[0].length());
+ tags_list[j] = get_align_tags(qry_aln_str+first_pos, tgt_aln_str+first_pos, last_pos-first_pos, &arange, j, 0, input_seq[j].length(), input_seq[0].length());
+ free(tgt_aln_str);
+ free(qry_aln_str);
+ }
+ edlibFreeAlignResult(align);
- //printf("2:%ld %ld %ld %ld\n\n", arange->s1, arange->e1, arange->s2, arange->e2);
-
-#define INDEL_ALLOWENCE_1 0.10
- if (arange->e1 - arange->s1 < 100 || arange->e2 - arange->s2 < 100 ||
- abs( (arange->e1 - arange->s1 ) - (arange->e2 - arange->s2) ) >
- (int) (0.5 * INDEL_ALLOWENCE_1 * (arange->e1 - arange->s1 + arange->e2 - arange->s2))) {
- free_kmer_match( kmer_match_ptr);
- free_aln_range(arange);
- continue;
- }
-
-
-#define INDEL_ALLOWENCE_2 150
- NDalignment::NDalignResult aln;
- align(input_seq[j].c_str()+arange->s1, arange->e1 - arange->s1 ,
- input_seq[0].c_str()+arange->s2, arange->e2 - arange->s2 ,
- INDEL_ALLOWENCE_2, 1, aln);
- if (aln._size > min_len && ((double) aln._dist / (double) aln._size) < max_diff) {
- tags_list[j] = get_align_tags( aln._qry_aln_str,
- aln._tgt_aln_str,
- aln._size,
- arange, j,
- 0);
- //fprintf(stderr, "Aligned seq %d to positions %d - %d and %d - %d with %d diffs size %d\n", j, aln._qry_bgn, aln._qry_end, aln._tgt_bgn, aln._tgt_end, aln._dist, aln._size);
- }
- free_aln_range(arange);
- free_kmer_match( kmer_match_ptr);
}
consensus = get_cns_from_align_tags( tags_list, seq_count, input_seq[0].length(), min_cov, max_len);
- free_seq_addr_array(sda_ptr);
- free_seq_array(sa_ptr);
- free_kmer_lookup(lk_ptr);
for (int j=0; j < seq_count; j++)
if (tags_list[j] != NULL)
free_align_tags(tags_list[j]);
diff --git a/src/falcon_sense/libfalcon/falcon.H b/src/falcon_sense/libfalcon/falcon.H
index 9aec526..e789195 100644
--- a/src/falcon_sense/libfalcon/falcon.H
+++ b/src/falcon_sense/libfalcon/falcon.H
@@ -79,89 +79,24 @@
*/
#include "AS_global.H"
-#include "dw.H"
#include <vector>
#include <string>
+#include <cstdlib>
+#include <stdint.h>
using namespace std;
-using namespace NDalignment;
namespace FConsensus {
-typedef struct {
- seq_coor_t start;
- seq_coor_t last;
- seq_coor_t count;
-} kmer_lookup;
-
-typedef unsigned char base;
-typedef base * seq_array;
-typedef seq_coor_t seq_addr;
-typedef seq_addr * seq_addr_array;
-
-
-typedef struct {
- seq_coor_t count;
- seq_coor_t * query_pos;
- seq_coor_t * target_pos;
-} kmer_match;
-
-
-typedef struct {
- seq_coor_t s1;
- seq_coor_t e1;
- seq_coor_t s2;
- seq_coor_t e2;
- long int score;
-} aln_range;
-
typedef struct {
char * sequence;
int32 * eqv;
+#ifdef TRACK_POSITIONS
+ vector<int> originalPos; // For tracking original read positions in the corrected read
+#endif
} consensus_data;
-kmer_lookup * allocate_kmer_lookup (seq_coor_t);
-void init_kmer_lookup ( kmer_lookup *, seq_coor_t );
-void free_kmer_lookup(kmer_lookup *);
-
-seq_array allocate_seq(seq_coor_t);
-void init_seq_array( seq_array, seq_coor_t);
-void free_seq_array(seq_array);
-
-seq_addr_array allocate_seq_addr(seq_coor_t size);
-
-void free_seq_addr_array(seq_addr_array);
-
-
-aln_range * find_best_aln_range(kmer_match *,
- seq_coor_t,
- seq_coor_t,
- seq_coor_t);
-
-void free_aln_range( aln_range *);
-
-kmer_match * find_kmer_pos_for_seq( const char *,
- seq_coor_t,
- uint32 K,
- seq_addr_array,
- kmer_lookup * );
-
-void free_kmer_match( kmer_match * ptr);
-void free_kmer_lookup(kmer_lookup * );
-
-
-
-void add_sequence ( seq_coor_t,
- uint32,
- const char *,
- seq_coor_t,
- seq_addr_array,
- seq_array,
- kmer_lookup *);
-
-void mask_k_mer(seq_coor_t, kmer_lookup *, seq_coor_t);
-
consensus_data * generate_consensus( vector<string> input_seq,
uint32 min_cov,
uint32 K,
diff --git a/src/falcon_sense/libfalcon/kmer_lookup.C b/src/falcon_sense/libfalcon/kmer_lookup.C
deleted file mode 100644
index eba9203..0000000
--- a/src/falcon_sense/libfalcon/kmer_lookup.C
+++ /dev/null
@@ -1,622 +0,0 @@
-
-/******************************************************************************
- *
- * This file is part of canu, a software program that assembles whole-genome
- * sequencing reads into contigs.
- *
- * This software is based on:
- * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
- * the 'kmer package' (http://kmer.sourceforge.net)
- * both originally distributed by Applera Corporation under the GNU General
- * Public License, version 2.
- *
- * Canu branched from Celera Assembler at its revision 4587.
- * Canu branched from the kmer project at its revision 1994.
- *
- * Modifications by:
- *
- * Sergey Koren beginning on 2016-FEB-24
- * are a 'United States Government Work', and
- * are released in the public domain
- *
- * File 'README.licenses' in the root directory of this distribution contains
- * full conditions and disclaimers for each license.
- */
-
-/*
- * =====================================================================================
- *
- * Filename: kmer_count.c
- *
- * Description:
- *
- * Version: 0.1
- * Created: 07/20/2013 17:00:00
- * Revision: none
- * Compiler: gcc
- *
- * Author: Jason Chin,
- * Company:
- *
- * =====================================================================================
-
- #################################################################################$$
- # Copyright (c) 2011-2014, Pacific Biosciences of California, Inc.
- #
- # All rights reserved.
- #
- # Redistribution and use in source and binary forms, with or without
- # modification, are permitted (subject to the limitations in the
- # disclaimer below) provided that the following conditions are met:
- #
- # * Redistributions of source code must retain the above copyright
- # notice, this list of conditions and the following disclaimer.
- #
- # * Redistributions in binary form must reproduce the above
- # copyright notice, this list of conditions and the following
- # disclaimer in the documentation and/or other materials provided
- # with the distribution.
- #
- # * Neither the name of Pacific Biosciences nor the names of its
- # contributors may be used to endorse or promote products derived
- # from this software without specific prior written permission.
- #
- # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
- # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
- # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
- # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS
- # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- # SUCH DAMAGE.
- #################################################################################$$
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <limits.h>
-#include "falcon.H"
-
-namespace FConsensus {
-const uint32 KMERMATCHINC = 10000;
-
-int compare_seq_coor(const void * a, const void * b) {
- const seq_coor_t * arg1 = (const seq_coor_t *)a;
- const seq_coor_t * arg2 = (const seq_coor_t *)b;
- return (* arg1) - (* arg2);
-}
-
-
-kmer_lookup * allocate_kmer_lookup ( seq_coor_t size ) {
- kmer_lookup * kl;
-
- kl = (kmer_lookup *) malloc( size * sizeof(kmer_lookup) );
- init_kmer_lookup( kl, size);
- return kl;
-}
-
-void init_kmer_lookup ( kmer_lookup * kl, seq_coor_t size ) {
- seq_coor_t i;
- for (i=0; i<size; i++) {
- kl[i].start = INT_MAX;
- kl[i].last = INT_MAX;
- kl[i].count = 0;
- }
-}
-
-
-void free_kmer_lookup( kmer_lookup * ptr) {
- free(ptr);
-}
-
-seq_array allocate_seq(seq_coor_t size) {
- seq_array sa;
- sa = (seq_array) malloc( size * sizeof(base) );
- init_seq_array( sa, size);
- return sa;
-}
-
-void init_seq_array( seq_array sa, seq_coor_t size) {
- seq_coor_t i;
- for (i=0; i<size; i++) {
- sa[i] = 0xff;
- }
-}
-
-void free_seq_array( seq_array sa) {
- free(sa);
-}
-
-seq_addr_array allocate_seq_addr(seq_coor_t size) {
- return (seq_addr_array) calloc( size, sizeof(seq_addr));
-}
-
-void free_seq_addr_array(seq_addr_array sda) {
- free(sda);
-}
-
-seq_coor_t get_kmer_bitvector(seq_array sa, uint32 K) {
- uint32 i;
- seq_coor_t kmer_bv = 0;
- seq_coor_t kmer_mask;
-
- kmer_mask = 0;
- for (i = 0; i < K; i++) {
- kmer_mask <<= 2;
- kmer_mask |= 0x00000003;
- }
-
- for (i = 0; i < K; i++) {
- kmer_bv <<= 2;
- kmer_bv |= (uint32) sa[i];
- }
-
- return kmer_bv;
-}
-
-void add_sequence ( seq_coor_t start,
- uint32 K,
- const char * seq,
- seq_coor_t seq_len,
- seq_addr_array sda,
- seq_array sa,
- kmer_lookup * lk ) {
-
- seq_coor_t i;
- seq_coor_t kmer_bv;
- seq_coor_t kmer_mask;
-
- kmer_mask = 0;
- for (i = 0; i < K; i++) {
- kmer_mask <<= 2;
- kmer_mask |= 0x00000003;
- }
-
- for (i = 0; i < seq_len; i++) {
- switch ( seq[i] ) {
- case 'A':
- sa[ start + i ] = 0;
- break;
- case 'C':
- sa[ start + i ] = 1;
- break;
- case 'G':
- sa[ start + i ] = 2;
- break;
- case 'T':
- sa[ start + i ] = 3;
- break;
- default:
- sa[ start + i ] = 0;
- }
- }
- kmer_bv = get_kmer_bitvector( sa + start, K);
- for (i = 0; i < seq_len - K; i++) {
- //fprintf(stderr, "%lu %lu\n", i, kmer_bv);
- //fprintf(stderr, "lk before init: %lu %lu %lu\n", kmer_bv, lk[kmer_bv].start, lk[kmer_bv].last);
- if (lk[kmer_bv].start == INT_MAX) {
- lk[kmer_bv].start = start + i;
- lk[kmer_bv].last = start + i;
- lk[kmer_bv].count += 1;
- //fprintf(stderr, "lk init: %lu %lu %lu\n", kmer_bv, lk[kmer_bv].start, lk[kmer_bv].last);
- } else {
- sda[ lk[kmer_bv].last ] = start + i;
- lk[kmer_bv].count += 1;
- lk[kmer_bv].last = start + i;
- //fprintf(stderr, "lk change: %lu %lu %lu\n", kmer_bv, lk[kmer_bv].start, lk[kmer_bv].last);
- }
- kmer_bv <<= 2;
- kmer_bv |= sa[ start + i + K];
- kmer_bv &= kmer_mask;
- }
-}
-
-
-void mask_k_mer(seq_coor_t size, kmer_lookup * kl, seq_coor_t threshold) {
- seq_coor_t i;
- for (i=0; i<size; i++) {
- if (kl[i].count > threshold) {
- kl[i].start = INT_MAX;
- kl[i].last = INT_MAX;
- //kl[i].count = 0;
- }
- }
-}
-
-
-kmer_match * find_kmer_pos_for_seq( const char * seq, seq_coor_t seq_len, uint32 K,
- seq_addr_array sda,
- kmer_lookup * lk) {
- seq_coor_t i;
- seq_coor_t kmer_bv;
- seq_coor_t kmer_mask;
- seq_coor_t kmer_pos;
- seq_coor_t next_kmer_pos;
- uint32 half_K;
- seq_coor_t kmer_match_rtn_allocation_size = KMERMATCHINC;
- kmer_match * kmer_match_rtn;
- base * sa;
-
- kmer_match_rtn = (kmer_match *) malloc( sizeof(kmer_match) );
- kmer_match_rtn->count = 0;
- kmer_match_rtn->query_pos = (seq_coor_t *) calloc( kmer_match_rtn_allocation_size, sizeof( seq_coor_t ) );
- kmer_match_rtn->target_pos = (seq_coor_t *) calloc( kmer_match_rtn_allocation_size, sizeof( seq_coor_t ) );
-
- sa = (base *)calloc( seq_len, sizeof(base) );
-
- kmer_mask = 0;
- for (i = 0; i < K; i++) {
- kmer_mask <<= 2;
- kmer_mask |= 0x00000003;
- }
-
- for (i = 0; i < seq_len; i++) {
- switch ( seq[i] ) {
- case 'A':
- sa[ i ] = 0;
- break;
- case 'C':
- sa[ i ] = 1;
- break;
- case 'G':
- sa[ i ] = 2;
- break;
- case 'T':
- sa[ i ] = 3;
- break;
- default:
- sa[ i ] = 0;
- }
- }
-
-
- kmer_bv = get_kmer_bitvector(sa, K);
- half_K = K >> 1;
- for (i = 0; i < seq_len - K; i += half_K) {
- kmer_bv = get_kmer_bitvector(sa + i, K);
- if (lk[kmer_bv].start == INT_MAX) { //for high count k-mers
- continue;
- }
- kmer_pos = lk[ kmer_bv ].start;
- next_kmer_pos = sda[ kmer_pos ];
- kmer_match_rtn->query_pos[ kmer_match_rtn->count ] = i;
- kmer_match_rtn->target_pos[ kmer_match_rtn->count ] = kmer_pos;
- kmer_match_rtn->count += 1;
- if (kmer_match_rtn->count > kmer_match_rtn_allocation_size - 1000) {
- kmer_match_rtn_allocation_size += KMERMATCHINC;
- kmer_match_rtn->query_pos = (seq_coor_t *) realloc( kmer_match_rtn->query_pos,
- kmer_match_rtn_allocation_size * sizeof(seq_coor_t) );
- kmer_match_rtn->target_pos = (seq_coor_t *) realloc( kmer_match_rtn->target_pos,
- kmer_match_rtn_allocation_size * sizeof(seq_coor_t) );
- }
- while ( next_kmer_pos > kmer_pos ){
- kmer_pos = next_kmer_pos;
- next_kmer_pos = sda[ kmer_pos ];
- kmer_match_rtn->query_pos[ kmer_match_rtn->count ] = i;
- kmer_match_rtn->target_pos[ kmer_match_rtn->count ] = kmer_pos;
- kmer_match_rtn->count += 1;
- if (kmer_match_rtn->count > kmer_match_rtn_allocation_size - 1000) {
- kmer_match_rtn_allocation_size += KMERMATCHINC;
- kmer_match_rtn->query_pos = (seq_coor_t *) realloc( kmer_match_rtn->query_pos,
- kmer_match_rtn_allocation_size * sizeof(seq_coor_t) );
- kmer_match_rtn->target_pos = (seq_coor_t *) realloc( kmer_match_rtn->target_pos,
- kmer_match_rtn_allocation_size * sizeof(seq_coor_t) );
- }
- }
- }
- free(sa);
- return kmer_match_rtn;
-}
-
-void free_kmer_match( kmer_match * ptr) {
- free(ptr->query_pos);
- free(ptr->target_pos);
- free(ptr);
-}
-
-aln_range* find_best_aln_range(kmer_match * km_ptr,
- seq_coor_t K,
- seq_coor_t bin_size,
- seq_coor_t count_th) {
- seq_coor_t i;
- seq_coor_t j;
- seq_coor_t q_min, q_max, t_min, t_max;
- seq_coor_t * d_count;
- seq_coor_t * q_coor;
- seq_coor_t * t_coor;
- aln_range * arange;
-
- long int d, d_min, d_max;
- long int cur_score;
- long int max_score;
- long int max_k_mer_count;
- long int max_k_mer_bin;
- seq_coor_t cur_start;
-
- arange = (aln_range *)calloc(1 , sizeof(aln_range));
-
- q_min = INT_MAX;
- q_max = 0;
- t_min = INT_MAX;
- t_max = 0;
-
- d_min = INT_MAX;
- d_max = LONG_MIN;
-
- for (i = 0; i < km_ptr->count; i++ ) {
- if ( km_ptr -> query_pos[i] < q_min) {
- q_min = km_ptr->query_pos[i];
- }
- if ( km_ptr -> query_pos[i] > q_max) {
- q_max = km_ptr->query_pos[i];
- }
- if ( km_ptr -> target_pos[i] < t_min) {
- t_min = km_ptr->target_pos[i];
- }
- if ( km_ptr -> target_pos[i] > t_max) {
- t_max = km_ptr->target_pos[i];
- }
- d = (long int) km_ptr->query_pos[i] - (long int) km_ptr->target_pos[i];
- if ( d < d_min ) {
- d_min = d;
- }
- if ( d > d_max ) {
- d_max = d;
- }
- }
-
- //printf("%lu %ld %ld\n" , km_ptr->count, d_min, d_max);
- if (km_ptr->count == 0)
- d_max = d_min = 0;
-
- d_count = (seq_coor_t *)calloc( (d_max - d_min)/bin_size + 1, sizeof(seq_coor_t) );
- q_coor = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) );
- t_coor = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) );
-
- for (i = 0; i < km_ptr->count; i++ ) {
- d = (long int) (km_ptr->query_pos[i]) - (long int) (km_ptr->target_pos[i]);
- d_count[ (d - d_min)/ (long int) bin_size ] += 1;
- q_coor[i] = INT_MAX;
- t_coor[i] = INT_MAX;
- }
-
- j = 0;
- max_k_mer_count = 0;
- max_k_mer_bin = INT_MAX;
- for (i = 0; i < km_ptr->count; i++ ) {
- d = (long int) (km_ptr->query_pos[i]) - (long int) (km_ptr->target_pos[i]);
- if ( d_count[ (d - d_min)/ (long int) bin_size ] > max_k_mer_count) {
- max_k_mer_count = d_count[ (d - d_min)/ (long int) bin_size ];
- max_k_mer_bin = (d - d_min)/ (long int) bin_size;
- }
- }
- //printf("k_mer: %lu %lu\n" , max_k_mer_count, max_k_mer_bin);
-
- if ( max_k_mer_bin != INT_MAX && max_k_mer_count > count_th ) {
- for (i = 0; i < km_ptr->count; i++ ) {
- d = (long int) (km_ptr->query_pos[i]) - (long int) (km_ptr->target_pos[i]);
- if ( abs( ( (d - d_min)/ (long int) bin_size ) - max_k_mer_bin ) > 5 ) {
- continue;
- }
- if (d_count[ (d - d_min)/ (long int) bin_size ] > count_th) {
- q_coor[j] = km_ptr->query_pos[i];
- t_coor[j] = km_ptr->target_pos[i];
- //printf("d_count: %lu %lu\n" ,i, d_count[(d - d_min)/ (long int) bin_size]);
- //printf("coor: %lu %lu\n" , q_coor[j], t_coor[j]);
- j ++;
- }
- }
- }
-
- if (j > 1) {
- arange->s1 = q_coor[0];
- arange->e1 = q_coor[0];
- arange->s2 = t_coor[0];
- arange->e2 = t_coor[0];
- arange->score = 0;
-
- max_score = 0;
- cur_score = 0;
- cur_start = 0;
-
- for (i = 1; i < j; i++) {
- cur_score += 32 - (q_coor[i] - q_coor[i-1]);
- //printf("deltaD, %lu %ld\n", q_coor[i] - q_coor[i-1], cur_score);
- if (cur_score < 0) {
- cur_score = 0;
- cur_start = i;
- } else if (cur_score > max_score) {
- arange->s1 = q_coor[cur_start];
- arange->s2 = t_coor[cur_start];
- arange->e1 = q_coor[i];
- arange->e2 = t_coor[i];
- max_score = cur_score;
- arange->score = max_score;
- //printf("%lu %lu %lu %lu\n", arange.s1, arange.e1, arange.s2, arange.e2);
- }
- }
-
- } else {
- arange->s1 = 0;
- arange->e1 = 0;
- arange->s2 = 0;
- arange->e2 = 0;
- arange->score = 0;
- }
-
- // printf("free\n");
-
- free(d_count);
- free(q_coor);
- free(t_coor);
- return arange;
-}
-
-aln_range* find_best_aln_range2(kmer_match * km_ptr,
- seq_coor_t K,
- seq_coor_t bin_width,
- seq_coor_t count_th) {
-
- seq_coor_t * d_coor;
- seq_coor_t * hit_score;
- seq_coor_t * hit_count;
- seq_coor_t * last_hit;
- seq_coor_t max_q, max_t;
- seq_coor_t s, e, max_s, max_e, max_span, d_s, d_e, delta, d_len;
- seq_coor_t px, py, cx, cy;
- seq_coor_t max_hit_idx;
- seq_coor_t max_hit_score, max_hit_count;
- seq_coor_t i, j;
- seq_coor_t candidate_idx, max_d, d;
-
- aln_range * arange;
-
- arange = (aln_range *)calloc(1 , sizeof(aln_range));
-
- d_coor = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) );
-
- max_q = -1;
- max_t = -1;
-
- for (i = 0; i < km_ptr->count; i++ ) {
- d_coor[i] = km_ptr->query_pos[i] - km_ptr->target_pos[i];
- max_q = max_q > km_ptr->query_pos[i] ? max_q : km_ptr->query_pos[i];
- max_t = max_t > km_ptr->target_pos[i] ? max_q : km_ptr->target_pos[i];
-
- }
-
- qsort(d_coor, km_ptr->count, sizeof(seq_coor_t), compare_seq_coor);
-
-
- s = 0;
- e = 0;
- max_s = -1;
- max_e = -1;
- max_span = -1;
- delta = (long int) ( 0.05 * ( max_q + max_t ) );
- d_len = km_ptr->count;
- d_s = -1;
- d_e = -1;
- while (1) {
- d_s = d_coor[s];
- d_e = d_coor[e];
- while (d_e < d_s + delta && e < d_len-1) {
- e += 1;
- d_e = d_coor[e];
- }
- if ( max_span == -1 || e - s > max_span ) {
- max_span = e - s;
- max_s = s;
- max_e = e;
- }
- s += 1;
- if (s == d_len || e == d_len) {
- break;
- }
- }
-
- if (max_s == -1 || max_e == -1 || max_e - max_s < 32) {
- arange->s1 = 0;
- arange->e1 = 0;
- arange->s2 = 0;
- arange->e2 = 0;
- arange->score = 0;
- free(d_coor);
- return arange;
- }
-
- last_hit = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) );
- hit_score = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) );
- hit_count = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) );
-
- for (i = 0; i < km_ptr->count; i++ ) {
- last_hit[i] = -1;
- hit_score[i] = 0;
- hit_count[i] = 0;
- }
- max_hit_idx = -1;
- max_hit_score = 0;
- for (i = 0; i < km_ptr->count; i ++) {
- cx = km_ptr->query_pos[i];
- cy = km_ptr->target_pos[i];
- d = cx - cy;
- if ( d < d_coor[max_s] || d > d_coor[max_e] ) continue;
-
- j = i - 1;
- candidate_idx = -1;
- max_d = 65535;
- while (1) {
- if ( j < 0 ) break;
- px = km_ptr->query_pos[j];
- py = km_ptr->target_pos[j];
- d = px - py;
- if ( d < d_coor[max_s] || d > d_coor[max_e] ) {
- j--;
- continue;
- }
- if (cx - px > 320) break; //the number here controling how big alignment gap to be considered
- if (cy > py && cx - px + cy - py < max_d && cy - py <= 320 ) {
- max_d = cx - px + cy - py;
- candidate_idx = j;
- }
- j--;
- }
- if (candidate_idx != -1) {
- last_hit[i] = candidate_idx;
- hit_score[i] = hit_score[candidate_idx] + (64 - max_d);
- hit_count[i] = hit_count[candidate_idx] + 1;
- if (hit_score[i] < 0) {
- hit_score[i] = 0;
- hit_count[i] = 0;
- }
- } else {
- hit_score[i] = 0;
- hit_count[i] = 0;
- }
- if (hit_score[i] > max_hit_score) {
- max_hit_score = hit_score[i];
- max_hit_count = hit_count[i];
- max_hit_idx = i;
- }
-
- }
- if (max_hit_idx == -1) {
- arange->s1 = 0;
- arange->e1 = 0;
- arange->s2 = 0;
- arange->e2 = 0;
- arange->score = 0;
- free(d_coor);
- free(last_hit);
- free(hit_score);
- free(hit_count);
- return arange;
- }
-
- arange->score = max_hit_count + 1;
- arange->e1 = km_ptr->query_pos[max_hit_idx];
- arange->e2 = km_ptr->target_pos[max_hit_idx];
- i = max_hit_idx;
- while (last_hit[i] != -1) {
- i = last_hit[i];
- }
- arange->s1 = km_ptr->query_pos[i];
- arange->s2 = km_ptr->target_pos[i];
-
- free(d_coor);
- free(last_hit);
- free(hit_score);
- free(hit_count);
- return arange;
-}
-
-void free_aln_range( aln_range * arange) {
- free(arange);
-}
-}
diff --git a/src/fastq-utilities/fastqSimulate.C b/src/fastq-utilities/fastqSimulate.C
index 50e946e..c4af92f 100644
--- a/src/fastq-utilities/fastqSimulate.C
+++ b/src/fastq-utilities/fastqSimulate.C
@@ -253,7 +253,7 @@ void
makeSE(char *seq,
int32 seqLen,
FILE *outputI,
- FILE *outputC,
+ FILE *UNUSED(outputC),
int32 readLen,
int32 numReads) {
char *s1 = new char [readLen + 1];
@@ -660,7 +660,7 @@ void
makeCC(char *seq,
int32 seqLen,
FILE *outputI,
- FILE *outputC,
+ FILE *UNUSED(outputC),
int32 readLen,
int32 numReads,
int32 ccJunkSize,
diff --git a/src/gfa/alignGFA.C b/src/gfa/alignGFA.C
new file mode 100644
index 0000000..f033a3c
--- /dev/null
+++ b/src/gfa/alignGFA.C
@@ -0,0 +1,464 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz beginning on 2017-APR-04
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "AS_global.H"
+#include "gkStore.H"
+#include "tgStore.H"
+
+#include "edlib.H"
+
+#include "splitToWords.H"
+#include "AS_UTL_reverseComplement.H"
+
+#include "gfa.H"
+
+
+class sequence {
+public:
+ sequence() {
+ seq = NULL;
+ len = 0;
+ };
+ ~sequence() {
+ delete [] seq;
+ };
+
+ void set(tgTig *tig) {
+ len = tig->length(false);
+ seq = new char [len + 1];
+
+ memcpy(seq, tig->bases(false), len);
+
+ seq[len] = 0;
+ };
+
+ char *seq;
+ uint32 len;
+};
+
+
+
+void
+dotplot(uint32 Aid, bool Afwd, char *Aseq,
+ uint32 Bid, bool Bfwd, char *Bseq) {
+ char Aname[128], Afile[128];
+ char Bname[128], Bfile[128];
+ char Pname[128], Pfile[128];
+ FILE *F;
+
+ sprintf(Aname, "tig%08u%c", Aid, (Afwd) ? '+' : '-');
+ sprintf(Afile, "tig%08u%c.fasta", Aid, (Afwd) ? '+' : '-');
+ sprintf(Bname, "tig%08u%c", Bid, (Bfwd) ? '+' : '-');
+ sprintf(Bfile, "tig%08u%c.fasta", Bid, (Bfwd) ? '+' : '-');
+ sprintf(Pname, "plot-%s-%s", Aname, Bname);
+ sprintf(Pfile, "plot-%s-%s.sh", Aname, Bname);
+
+ F = fopen(Pfile, "w");
+ fprintf(F, "#!/bin/sh\n");
+ fprintf(F, "\n");
+ fprintf(F, "nucmer --maxmatch --nosimplify -p %s %s.fasta %s.fasta\n", Pname, Aname, Bname);
+ fprintf(F, "show-coords -l -o -r -T %s.delta | expand -t 8 > %s.coords\n", Pname, Pname);
+ fprintf(F, "mummerplot --fat -t png -p %s %s.delta\n", Pname, Pname);
+ fprintf(F, "echo mummerplot --fat -p %s %s.delta\n", Pname, Pname);
+ fclose(F);
+
+ F = fopen(Afile, "w");
+ fprintf(F, ">%s\n%s\n", Aname, Aseq);
+ fclose(F);
+
+ F = fopen(Bfile, "w");
+ fprintf(F, ">%s\n%s\n", Bname, Bseq);
+ fclose(F);
+
+ sprintf(Pfile, "sh plot-%s-%s.sh", Aname, Bname);
+
+ system(Pfile);
+}
+
+
+
+
+bool
+checkLink(gfaLink *link,
+ sequence *seqs,
+ bool beVerbose,
+ bool doPlot) {
+
+ char *Aseq = seqs[link->_Aid].seq;
+ char *Bseq = seqs[link->_Bid].seq;
+
+ int32 Abgn, Aend, Alen = seqs[link->_Aid].len;
+ int32 Bbgn, Bend, Blen = seqs[link->_Bid].len;
+
+ EdlibAlignResult result = { 0, NULL, NULL, 0, NULL, 0, 0 };
+
+ int32 AalignLen = 0;
+ int32 BalignLen = 0;
+ int32 editDist = 0;
+ int32 alignLen = 0;
+ int32 maxEdit = 0;
+
+ // NOTE! edlibAlign calls the 'A' sequence the 'query' and
+ // the 'B' sequence the 'target' (aka, 'reference').
+
+ link->alignmentLength(AalignLen, BalignLen, alignLen);
+
+ // Regardless of if we find an new alignment or not, remove the old one.
+ // If we don't find a new one, we'll discard the link.
+
+ delete [] link->_cigar;
+ link->_cigar = NULL;
+
+ if (link->_Afwd == false)
+ reverseComplementSequence(Aseq, Alen);
+ if (link->_Bfwd == false)
+ reverseComplementSequence(Bseq, Blen);
+
+
+ // Ty to find the end coordinate on B. Align the last bits of A to B.
+ //
+ // -------(---------] v--??
+ // [------------)------
+ //
+
+ Abgn = max(Alen - AalignLen, 0);
+ Aend = Alen;
+
+ Bbgn = 0;
+ Bend = min(Blen, (int32)(1.10 * BalignLen)); // Allow 25% gaps over what the GFA said?
+
+ maxEdit = (int32)ceil(alignLen * 0.12);
+
+ if (beVerbose)
+ fprintf(stderr, "LINK tig%08u %c %17s tig%08u %c %17s Aalign %6u Balign %6u align %6u\n",
+ link->_Aid, (link->_Afwd) ? '+' : '-', "",
+ link->_Bid, (link->_Bfwd) ? '+' : '-', "",
+ AalignLen, BalignLen, alignLen);
+
+ if (beVerbose)
+ fprintf(stderr, "TEST tig%08u %c %8d-%-8d tig%08u %c %8d-%-8d maxEdit=%6d (extend B)",
+ link->_Aid, (link->_Afwd) ? '+' : '-', Abgn, Aend,
+ link->_Bid, (link->_Bfwd) ? '+' : '-', Bbgn, Bend,
+ maxEdit);
+
+ result = edlibAlign(Aseq + Abgn, Aend-Abgn, // The 'query'
+ Bseq + Bbgn, Bend-Bbgn, // The 'target'
+ edlibNewAlignConfig(maxEdit, EDLIB_MODE_HW, EDLIB_TASK_LOC));
+
+ if (result.numLocations > 0) {
+ if (beVerbose)
+ fprintf(stderr, "\n");
+ Bend = Bbgn + result.endLocations[0] + 1; // 0-based to space-based
+ edlibFreeAlignResult(result);
+ } else {
+ if (beVerbose)
+ fprintf(stderr, " - FAILED\n");
+ }
+
+ // Do the same for A. Aend and Bbgn never change; Bend was set above.
+ //
+ // ------(--------------]
+ // ^--?? [-------]-----------
+ //
+
+ Abgn = max(Alen - (int32)(1.10 * AalignLen), 0); // Allow 25% gaps over what the GFA said?
+
+ if (beVerbose)
+ fprintf(stderr, " tig%08u %c %8d-%-8d tig%08u %c %8d-%-8d maxEdit=%6d (extend A)",
+ link->_Aid, (link->_Afwd) ? '+' : '-', Abgn, Aend,
+ link->_Bid, (link->_Bfwd) ? '+' : '-', Bbgn, Bend,
+ maxEdit);
+
+ // NEEDS to be MODE_HW because we need to find the suffix alignment.
+
+ result = edlibAlign(Bseq + Bbgn, Bend-Bbgn, // The 'query'
+ Aseq + Abgn, Aend-Abgn, // The 'target'
+ edlibNewAlignConfig(maxEdit, EDLIB_MODE_HW, EDLIB_TASK_LOC));
+
+ if (result.numLocations > 0) {
+ if (beVerbose)
+ fprintf(stderr, "\n");
+ Abgn = Abgn + result.startLocations[0];
+ edlibFreeAlignResult(result);
+ } else {
+ if (beVerbose)
+ fprintf(stderr, " - FAILED\n");
+ }
+
+ // One more alignment, this time, with feeling - notice EDLIB_MODE_MW and EDLIB_TASK_PATH.
+
+ if (beVerbose)
+ fprintf(stderr, " tig%08u %c %8d-%-8d tig%08u %c %8d-%-8d maxEdit=%6d (final)",
+ link->_Aid, (link->_Afwd) ? '+' : '-', Abgn, Aend,
+ link->_Bid, (link->_Bfwd) ? '+' : '-', Bbgn, Bend,
+ maxEdit);
+
+ result = edlibAlign(Aseq + Abgn, Aend-Abgn,
+ Bseq + Bbgn, Bend-Bbgn,
+ edlibNewAlignConfig(2 * maxEdit, EDLIB_MODE_NW, EDLIB_TASK_PATH));
+
+
+ bool success = false;
+
+ if (result.numLocations > 0) {
+ if (beVerbose)
+ fprintf(stderr, "\n");
+
+ editDist = result.editDistance;
+ alignLen = ((Aend - Abgn) + (Bend - Bbgn) + (editDist)) / 2;
+ alignLen = result.alignmentLength; // Edlib 'alignmentLength' is populated only for TASK_PATH
+
+ link->_cigar = edlibAlignmentToCigar(result.alignment,
+ result.alignmentLength, EDLIB_CIGAR_STANDARD);
+
+ edlibFreeAlignResult(result);
+
+ success = true;
+ } else {
+ if (beVerbose)
+ fprintf(stderr, " - FAILED\n");
+ }
+
+ if (beVerbose)
+ fprintf(stderr, " tig%08u %c %8d-%-8d tig%08u %c %8d-%-8d %.4f\n",
+ link->_Aid, (link->_Afwd) ? '+' : '-', Abgn, Aend,
+ link->_Bid, (link->_Bfwd) ? '+' : '-', Bbgn, Bend,
+ (double)editDist / alignLen);
+
+ // Make a plot.
+
+ if ((success == false) && (doPlot == true))
+ dotplot(link->_Aid, link->_Afwd, Aseq,
+ link->_Bid, link->_Bfwd, Bseq);
+
+ // Cleanup for the next link->
+
+ if (link->_Afwd == false)
+ reverseComplementSequence(Aseq, Alen);
+ if (link->_Bfwd == false)
+ reverseComplementSequence(Bseq, Blen);
+
+ if (beVerbose)
+ fprintf(stderr, "\n");
+
+ return(success);
+}
+
+
+
+
+int
+main (int argc, char **argv) {
+ char *tigName = NULL;
+ uint32 tigVers = UINT32_MAX;
+
+ char *inGFA = NULL;
+ char *otGFA = NULL;
+
+ uint32 verbosity = 0;
+
+ argc = AS_configure(argc, argv);
+
+ int arg=1;
+ int err=0;
+ while (arg < argc) {
+ if (strcmp(argv[arg], "-T") == 0) {
+ tigName = argv[++arg];
+ tigVers = atoi(argv[++arg]);
+
+ if (tigVers == 0)
+ fprintf(stderr, "invalid tigStore version (-T store version partition) '-t %s %s %s'.\n", argv[arg-2], argv[arg-1], argv[arg]), exit(1);
+
+ } else if (strcmp(argv[arg], "-i") == 0) {
+ inGFA = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-o") == 0) {
+ otGFA = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-V") == 0) {
+ verbosity++;
+
+ } else if (strcmp(argv[arg], "-t") == 0) {
+ omp_set_num_threads(atoi(argv[++arg]));
+
+ } else {
+ fprintf(stderr, "%s: Unknown option '%s'\n", argv[0], argv[arg]);
+ err++;
+ }
+
+ arg++;
+ }
+
+ if (tigName == NULL)
+ err++;
+ if (inGFA == NULL)
+ err++;
+ if (otGFA == NULL)
+ err++;
+
+ if (err) {
+ fprintf(stderr, "usage: %s [opts]\n", argv[0]);
+ fprintf(stderr, " Validates a GFA by generating alignments.\n");
+ fprintf(stderr, " Optionally writes new GFA with updated CIGAR string (NOT IMPLEMENTED).\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -G g Load reads from gkStore 'g'\n");
+ fprintf(stderr, " -T t v Load tigs from tgStore 't', version 'v'.\n");
+ fprintf(stderr, " Consensus sequence must exist (usually in v=2)\n");
+ fprintf(stderr, " -i input.gfa\n");
+ fprintf(stderr, " -o output.gfa\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -V Increase chatter\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -t threads Use 'threads' computational threads.\n");
+ fprintf(stderr, "\n");
+
+ if (tigName == NULL)
+ fprintf(stderr, "ERROR: no tigStore (-T) supplied.\n");
+ if (inGFA == NULL)
+ fprintf(stderr, "ERROR: no input GFA (-i) supplied.\n");
+ if (otGFA == NULL)
+ fprintf(stderr, "ERROR: no output GFA (-o) supplied.\n");
+
+ exit(1);
+ }
+
+ fprintf(stderr, "-- Opening tigStore '%s' version %u.\n", tigName, tigVers);
+ tgStore *tigStore = new tgStore(tigName, tigVers);
+
+ // Load the GFA file.
+
+ fprintf(stderr, "-- Reading GFA '%s'.\n", inGFA);
+ gfaFile *gfa = new gfaFile(inGFA);
+
+ // Load all consensus sequences
+
+ uint32 b = 0;
+ uint32 e = tigStore->numTigs();
+
+ sequence *seqs = new sequence [e+1];
+
+ fprintf(stderr, "-- Loading tigs %u to %u.\n", b, e);
+
+ for (uint32 ti=b; ti < e; ti++) {
+ tgTig *tig = tigStore->loadTig(ti);
+
+ if (tig == NULL)
+ continue;
+
+ seqs[ti].set(tig);
+
+ tigStore->unloadTig(ti);
+ }
+
+ // Set GFA lengths based on the sequences we loaded.
+
+ fprintf(stderr, "-- Resetting sequence lengths.\n", inGFA);
+
+ for (uint32 ii=0; ii<gfa->_sequences.size(); ii++)
+ gfa->_sequences[ii]->_length = seqs[gfa->_sequences[ii]->_id].len;
+
+ // Done with the stores.
+
+ fprintf(stderr, "-- Closing tigStore '%s'.\n", tigName);
+
+ delete tigStore;
+
+ // Align!
+
+ uint32 passCircular = 0;
+ uint32 failCircular = 0;
+
+ uint32 passNormal = 0;
+ uint32 failNormal = 0;
+
+ uint32 iiLimit = gfa->_links.size();
+ uint32 iiNumThreads = omp_get_max_threads();
+ uint32 iiBlockSize = (iiLimit < 1000 * iiNumThreads) ? iiNumThreads : iiLimit / 999;
+
+ fprintf(stderr, "-- Aligning " F_U32 " links using " F_U32 " threads.\n", iiLimit, iiNumThreads);
+
+#pragma omp parallel for schedule(dynamic, iiBlockSize)
+ for (uint32 ii=0; ii<iiLimit; ii++) {
+ gfaLink *link = gfa->_links[ii];
+
+ if (link->_Aid == link->_Bid) {
+ if (verbosity > 0)
+ fprintf(stderr, "Processing circular link for tig %u\n", link->_Aid);
+
+ if (link->_Afwd != link->_Bfwd)
+ fprintf(stderr, "WARNING: %s %c %s %c -- circular to the same end!?\n",
+ link->_Aname, link->_Afwd ? '+' : '-',
+ link->_Bname, link->_Bfwd ? '+' : '-');
+
+ bool pN = checkLink(link, seqs, (verbosity > 0), false);
+
+ if (pN == true)
+ passCircular++;
+ else
+ failCircular++;
+ }
+
+ // Now the usual case.
+
+ else {
+ if (verbosity > 0)
+ fprintf(stderr, "Processing link between tig %u %s and tig %u %s\n",
+ link->_Aid, link->_Afwd ? "-->" : "<--",
+ link->_Bid, link->_Bfwd ? "-->" : "<--");
+
+ bool pN = checkLink(link, seqs, (verbosity > 0), false);
+
+ if (pN == true)
+ passNormal++;
+ else
+ failNormal++;
+ }
+
+ // If the cigar exists, we found an alignment. If not, delete the link.
+
+ if (link->_cigar == NULL) {
+ if (verbosity > 0)
+ fprintf(stderr, " Failed to find alignment.\n");
+ delete gfa->_links[ii];
+ gfa->_links[ii] = NULL;
+ }
+ }
+
+ fprintf(stderr, "-- Writing GFA '%s'.\n", otGFA);
+
+ gfa->saveFile(otGFA);
+
+ fprintf(stderr, "-- Cleaning up.\n");
+
+ delete [] seqs;
+ delete gfa;
+
+ fprintf(stderr, "-- Aligned %6u ciruclar tigs, failed %6u\n", passCircular, failCircular);
+ fprintf(stderr, "-- Aligned %6u linear tigs, failed %6u\n", passNormal, failNormal);
+ fprintf(stderr, "-- Bye.\n");
+
+ exit(0);
+}
diff --git a/src/falcon_sense/falcon_sense.mk b/src/gfa/alignGFA.mk
similarity index 71%
copy from src/falcon_sense/falcon_sense.mk
copy to src/gfa/alignGFA.mk
index 4cb2f2e..0d9f892 100644
--- a/src/falcon_sense/falcon_sense.mk
+++ b/src/gfa/alignGFA.mk
@@ -7,13 +7,13 @@ ifeq "$(strip ${TARGET_DIR})" ""
TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin
endif
-TARGET := falcon_sense
-SOURCES := falcon_sense.C
+TARGET := alignGFA
+SOURCES := alignGFA.C
-SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns/libNDFalcon libfalcon
+SRC_INCDIRS := .. ../AS_UTL ../stores ../overlapInCore/libedlib
TGT_LDFLAGS := -L${TARGET_DIR}
TGT_LDLIBS := -lcanu
-TGT_PREREQS := libcanu.a
+TGT_PREREQS := libcanu.a
SUBMAKEFILES :=
diff --git a/src/gfa/gfa.C b/src/gfa/gfa.C
new file mode 100644
index 0000000..ebef08a
--- /dev/null
+++ b/src/gfa/gfa.C
@@ -0,0 +1,345 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz beginning on 2017-APR-04
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "AS_global.H"
+#include "AS_UTL_fileIO.H"
+
+#include "gfa.H"
+
+
+
+template<typename TT>
+static
+bool
+findGFAtokenI(char *features, char *token, TT &value) {
+ char *p = NULL;
+
+ p = strstr(features, token);
+
+ if (p == NULL)
+ return(false);
+
+ p += strlen(token); // Skip over the token...
+
+ if (*p == ':') // ...and any :, if the user forgot to include it.
+ p++;
+
+ value = (TT)strtoll(p, NULL, 10);
+
+ //fprintf(stderr, "FOUND feature '%s' in '%s' -> '%s' %u\n", token, features, p, value);
+
+ return(true);
+}
+
+
+// Search for canu-specific names, and convert to tigID's.
+static
+uint32
+nameToCanuID(char *name) {
+ uint32 id = UINT32_MAX;
+
+ if ((name[0] == 't') &&
+ (name[1] == 'i') &&
+ (name[2] == 'g'))
+ id = strtoll(name + 3, NULL, 10);
+
+ return(id);
+}
+
+
+
+gfaSequence::gfaSequence() {
+ _name = NULL;
+ _sequence = NULL;
+ _features = NULL;
+ _length = 0;
+}
+
+
+gfaSequence::gfaSequence(char *inLine) {
+ load(inLine);
+}
+
+
+gfaSequence::~gfaSequence() {
+ delete [] _name;
+ delete [] _sequence;
+ delete [] _features;
+}
+
+
+void
+gfaSequence::load(char *inLine) {
+ splitToWords W(inLine);
+
+ _name = new char [strlen(W[1]) + 1];
+ _id = UINT32_MAX;
+ _sequence = new char [strlen(W[2]) + 1];
+ _features = new char [strlen(W[3]) + 1];
+
+ _length = 0;
+
+ strcpy(_name, W[1]);
+ strcpy(_sequence, W[2]);
+ strcpy(_features, W[3]);
+
+ // Scan the _features for a length.
+
+ findGFAtokenI(_features, "LN:i:", _length);
+
+ // And any canu ID
+
+ _id = nameToCanuID(_name);
+}
+
+
+void
+gfaSequence::save(FILE *outFile) {
+ fprintf(outFile, "S\t%s\t%s\tLN:i:%u\n", _name, _sequence, _length);
+}
+
+
+
+
+gfaLink::gfaLink() {
+ _Aname = NULL;
+ _Aid = UINT32_MAX;
+ _Afwd = false;
+
+ _Bname = NULL;
+ _Bid = UINT32_MAX;
+ _Bfwd = false;
+
+ _cigar = NULL;
+ _features = NULL;
+}
+
+
+gfaLink::gfaLink(char *inLine) {
+ load(inLine);
+}
+
+
+gfaLink::~gfaLink() {
+ delete [] _Aname;
+ delete [] _Bname;
+ delete [] _cigar;
+ delete [] _features;
+}
+
+
+void
+gfaLink::load(char *inLine) {
+ splitToWords W(inLine);
+
+ _Aname = new char [strlen(W[1]) + 1];
+ _Aid = UINT32_MAX;
+ _Afwd = W[2][0] == '+';
+
+ _Bname = new char [strlen(W[3]) + 1];
+ _Bid = UINT32_MAX;
+ _Bfwd = W[4][0] == '+';
+
+ _cigar = new char [strlen(W[5]) + 1];
+
+ _features = new char [(W[6]) ? strlen(W[6]) + 1 : 1];
+
+ strcpy(_Aname, W[1]);
+ strcpy(_Bname, W[3]);
+ strcpy(_cigar, W[5]);
+ strcpy(_features, (W[6]) ? W[6] : "");
+
+ _Aid = nameToCanuID(_Aname); // Search for canu-specific names, and convert to tigID's.
+ _Bid = nameToCanuID(_Bname);
+}
+
+
+void
+gfaLink::save(FILE *outFile) {
+ fprintf(outFile, "L\t%s\t%c\t%s\t%c\t%s\n",
+ _Aname, (_Afwd == true) ? '+' : '-',
+ _Bname, (_Bfwd == true) ? '+' : '-',
+ (_cigar == NULL) ? "*" : _cigar);
+}
+
+
+void
+gfaLink::alignmentLength(int32 &queryLen, int32 &refceLen, int32 &alignLen) {
+ char *cp = _cigar;
+
+ refceLen = 0; // Bases on the reference involved in the alignment
+ queryLen = 0; // Bases on the query involved in the alignment
+ alignLen = 0; // Length of the alignment
+
+ if (cp == NULL)
+ return;
+
+ if (*cp == '*')
+ return;
+
+ do {
+ int64 val = strtoll(cp, &cp, 10);
+ char code = *cp++;
+
+ switch (code) {
+ case 'M': // Alignment, either match or mismatch
+ refceLen += val;
+ queryLen += val;
+ alignLen += val;
+ break;
+ case 'I': // Insertion to the reference - gap in query
+ queryLen += val;
+ alignLen += val;
+ break;
+ case 'D': // Deletion from the reference - gap in reference
+ refceLen += val;
+ alignLen += val;
+ break;
+ case 'N': // Skipped in the reference (e.g., intron)
+ refceLen += val;
+ fprintf(stderr, "warning - unsupported CIGAR code '%c' in '%s'\n", code, _cigar);
+ break;
+ case 'S': // Soft-clipped from the query - not part of the alignment
+ fprintf(stderr, "warning - unsupported CIGAR code '%c' in '%s'\n", code, _cigar);
+ break;
+ case 'H': // Hard-clipped from the query - not part of the alignment, and removed from the read as input
+ fprintf(stderr, "warning - unsupported CIGAR code '%c' in '%s'\n", code, _cigar);
+ break;
+ case 'P': // Padding - "silent deletion from padded reference" - ???
+ fprintf(stderr, "warning - unsupported CIGAR code '%c' in '%s'\n", code, _cigar);
+ break;
+ case '=': // Alignment, match
+ refceLen += val;
+ queryLen += val;
+ alignLen += val;
+ break;
+ case 'X': // Alignment, mismatch
+ refceLen += val;
+ queryLen += val;
+ alignLen += val;
+ break;
+ default:
+ fprintf(stderr, "unknown CIGAR code '%c' in '%s'\n", code, _cigar);
+ break;
+ }
+
+ } while (*cp != 0);
+}
+
+
+
+
+gfaFile::gfaFile(char *inFile) {
+ _header = NULL;
+
+ loadFile(inFile);
+}
+
+
+gfaFile::~gfaFile() {
+ delete [] _header;
+
+ for (uint32 ii=0; ii<_sequences.size(); ii++)
+ delete _sequences[ii];
+
+ for (uint32 ii=0; ii<_links.size(); ii++)
+ delete _links[ii];
+}
+
+
+bool
+gfaFile::loadFile(char *inFile) {
+ FILE *F = NULL;
+ char *L = NULL;
+ uint32 Llen = 0;
+ uint32 Lmax = 0;
+
+ errno = 0;
+ F = fopen(inFile, "r");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s' for reading: %s\n", inFile, strerror(errno)), exit(1);
+
+
+ while (AS_UTL_readLine(L, Llen, Lmax, F)) {
+ char type = L[0];
+
+ if (L[1] != '\t')
+ fprintf(stderr, "gfaFile::loadFile()-- misformed file; second letter must be tab in line '%s'\n", L), exit(1);
+
+ if (type == 'H') {
+ delete [] _header;
+ _header = new char [Llen];
+ strcpy(_header, L+2);
+ }
+
+ else if (type == 'S') {
+ _sequences.push_back(new gfaSequence(L));
+ }
+
+ else if (type == 'L') {
+ _links.push_back(new gfaLink(L));
+ }
+
+ else {
+ fprintf(stderr, "gfaFile::loadFile()-- unrecognized line '%s'\n", L), exit(1);
+ }
+ }
+
+ fclose(F);
+
+ delete [] L;
+
+ fprintf(stderr, "gfa: Loaded " F_S64 " sequences and " F_S64 " links.\n", _sequences.size(), _links.size());
+
+ return(true);
+}
+
+
+
+
+bool
+gfaFile::saveFile(char *outFile) {
+ FILE *F = NULL;
+
+ errno = 0;
+ F = fopen(outFile, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s' for reading: %s\n", outFile, strerror(errno)), exit(1);
+
+ fprintf(F, "H\t%s\n", _header);
+
+ for (uint32 ii=0; ii<_sequences.size(); ii++)
+ if (_sequences[ii])
+ _sequences[ii]->save(F);
+
+ for (uint32 ii=0; ii<_links.size(); ii++)
+ if (_links[ii])
+ _links[ii]->save(F);
+
+ fclose(F);
+
+ return(true);
+}
+
diff --git a/src/gfa/gfa.H b/src/gfa/gfa.H
new file mode 100644
index 0000000..a54c518
--- /dev/null
+++ b/src/gfa/gfa.H
@@ -0,0 +1,103 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz beginning on 2017-APR-04
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#ifndef AS_UTL_GFA_H
+#define AS_UTL_GFA_H
+
+#include "AS_global.H"
+#include "splitToWords.H"
+
+
+
+// Features assumed to hold only the length, and we don't use it.
+
+class gfaSequence {
+public:
+ gfaSequence();
+ gfaSequence(char *inLine);
+ ~gfaSequence();
+
+ void load(char *inLine);
+ void save(FILE *outFile);
+
+public:
+ char *_name;
+ uint32 _id;
+ char *_sequence;
+ char *_features;
+
+ uint32 _length;
+};
+
+
+
+
+class gfaLink {
+public:
+ gfaLink();
+ gfaLink(char *inLine);
+ ~gfaLink();
+
+ void load(char *inLine);
+ void save(FILE *outFile);
+
+ void alignmentLength(int32 &queryLen, int32 &refceLen, int32 &alignLen);
+
+public:
+ char *_Aname;
+ uint32 _Aid; // Canu specific.
+ bool _Afwd;
+
+ char *_Bname;
+ uint32 _Bid; // Canu specific.
+ bool _Bfwd;
+
+ char *_cigar;
+
+ char *_features;
+};
+
+
+
+
+class gfaFile {
+public:
+ gfaFile(char *inFile);
+ ~gfaFile();
+
+ bool loadFile(char *inFile);
+ bool saveFile(char *outFile);
+
+public:
+ char *_header;
+
+ vector<gfaSequence *> _sequences;
+ vector<gfaLink *> _links;
+};
+
+
+
+
+#endif // AS_UTL_GFA_H
diff --git a/src/main.mk b/src/main.mk
index 740ab00..bf47b08 100644
--- a/src/main.mk
+++ b/src/main.mk
@@ -48,7 +48,6 @@ SOURCES := AS_global.C \
AS_UTL/kMer.C \
\
falcon_sense/libfalcon/falcon.C \
- falcon_sense/libfalcon/kmer_lookup.C \
\
stores/gkStore.C \
stores/gkStoreEncode.C \
@@ -107,7 +106,41 @@ SOURCES := AS_global.C \
utgcns/libcns/unitigConsensus.C \
utgcns/libpbutgcns/Alignment.C \
utgcns/libpbutgcns/AlnGraphBoost.C \
- utgcns/libNDFalcon/dw.C
+ utgcns/libNDFalcon/dw.C \
+ \
+ gfa/gfa.C \
+ \
+ meryl/libkmer/existDB-create-from-fasta.C \
+ meryl/libkmer/existDB-create-from-meryl.C \
+ meryl/libkmer/existDB-create-from-sequence.C \
+ meryl/libkmer/existDB-state.C \
+ meryl/libkmer/existDB.C \
+ meryl/libkmer/positionDB-access.C \
+ meryl/libkmer/positionDB-dump.C \
+ meryl/libkmer/positionDB-file.C \
+ meryl/libkmer/positionDB-mismatch.C \
+ meryl/libkmer/positionDB-sort.C \
+ meryl/libkmer/positionDB.C
+
+
+
+ifeq (${BUILDSTACKTRACE}, 1)
+SOURCES += AS_UTL/libbacktrace/atomic.c \
+ AS_UTL/libbacktrace/backtrace.c \
+ AS_UTL/libbacktrace/dwarf.c \
+ AS_UTL/libbacktrace/elf.c \
+ AS_UTL/libbacktrace/fileline.c \
+ AS_UTL/libbacktrace/mmap.c \
+ AS_UTL/libbacktrace/mmapio.c \
+ AS_UTL/libbacktrace/posix.c \
+ AS_UTL/libbacktrace/print.c \
+ AS_UTL/libbacktrace/simple.c \
+ AS_UTL/libbacktrace/sort.c \
+ AS_UTL/libbacktrace/state.c \
+ AS_UTL/libbacktrace/unknown.c
+endif
+
+
SRC_INCDIRS := . \
AS_UTL \
@@ -148,6 +181,10 @@ SUBMAKEFILES := stores/gatekeeperCreate.mk \
meryl/maskMers.mk \
meryl/simple.mk \
meryl/estimate-mer-threshold.mk \
+ meryl/existDB.mk \
+ meryl/positionDB.mk \
+ \
+ merTrim/merTrim.mk \
\
overlapInCore/overlapInCore.mk \
overlapInCore/overlapInCorePartition.mk \
@@ -185,6 +222,8 @@ SUBMAKEFILES := stores/gatekeeperCreate.mk \
\
utgcns/utgcns.mk \
\
+ gfa/alignGFA.mk \
+ \
fastq-utilities/fastqAnalyze.mk \
fastq-utilities/fastqSample.mk \
fastq-utilities/fastqSimulate.mk \
diff --git a/src/merTrim/merTrim.C b/src/merTrim/merTrim.C
index f005e00..da62815 100644
--- a/src/merTrim/merTrim.C
+++ b/src/merTrim/merTrim.C
@@ -40,21 +40,18 @@
*/
#include "AS_global.H"
+#include "ovStore.H"
+#include "gkStore.H"
#include "AS_UTL_reverseComplement.H"
-#include "AS_PER_gkpStore.H"
-#include "AS_PER_encodeSequenceQuality.H" // QUALITY_MAX, QV conversion
-#include "AS_OVS_overlapStore.H"
#include <algorithm>
-#include "AS_MER_gkpStore_to_FastABase.H"
-
-#include "bio++.H"
#include "sweatShop.H"
+
#include "existDB.H"
#include "positionDB.H"
+
#include "libmeryl.H"
-#include "logMsg.H"
#include "merTrimResult.H"
@@ -76,13 +73,13 @@ uint32 VERBOSE = 0;
#undef TEST_TESTBASE
-char *createAdapterString(bool adapIllumina, bool adap454);
-
+//char *createAdapterString(bool adapIllumina, bool adap454);
class mertrimGlobalData {
public:
mertrimGlobalData() {
gkpPath = 0L;
+ gkp = 0L;
fqInputPath = 0L;
fqOutputPath = 0L;
@@ -131,8 +128,6 @@ public:
trimImperfectCoverage = true;
- gkRead = NULL;
-
fqInput = NULL;
fqOutput = NULL;
fqVerify = NULL;
@@ -150,7 +145,8 @@ public:
};
~mertrimGlobalData() {
- delete gkRead;
+ gkp->gkStore_close();
+ gkp = NULL;
delete fqInput;
delete fqOutput;
@@ -170,11 +166,11 @@ public:
return;
fprintf(stderr, "opening gkStore '%s'\n", gkpPath);
- gkRead = gkStore::gkStore_open(gkpPath, FALSE, FALSE);
+ gkp = gkStore::gkStore_open(gkpPath);
if (gktBgn == 0) {
gktBgn = 1;
- gktEnd = gkRead->gkStore_getNumFragments();
+ gktEnd = gkp->gkStore_getNumReads();
}
gktCur = gktBgn;
@@ -182,9 +178,9 @@ public:
if (gktBgn > gktEnd)
fprintf(stderr, "ERROR: invalid range: -b (" F_U32 ") >= -e (" F_U32 ").\n",
gktBgn, gktEnd), exit(1);
- if (gktEnd > gkRead->gkStore_getNumFragments())
+ if (gktEnd > gkp->gkStore_getNumReads())
fprintf(stderr, "ERROR: invalid range: -e (" F_U32 ") > num frags (" F_U32 ").\n",
- gktEnd, gkRead->gkStore_getNumFragments()), exit(1);
+ gktEnd, gkp->gkStore_getNumReads()), exit(1);
errno = 0;
resFile = fopen(resPath, "w");
@@ -273,12 +269,14 @@ public:
} else if (adapIllumina || adap454) {
fprintf(stderr, "creating adapter mer database.\n");
+#if 0
char *adapter = createAdapterString(adapIllumina, adap454);
adapterDB = new existDB(adapter, merSize, existDBcanonical | existDBcounts);
//adapterDB->printState(stderr);
delete [] adapter;
+#endif
} else {
fprintf(stderr, "not searching for adapter.\n");
@@ -348,7 +346,7 @@ public:
// Global data
//
- gkStore *gkRead;
+ gkStore *gkp;
uint32 actualCoverage;
double minCorrectFraction;
@@ -392,7 +390,7 @@ public:
class mertrimComputation {
public:
- mertrimComputation() : log(false, 131072) {
+ mertrimComputation() {
readName = NULL;
origSeq = NULL;
@@ -439,8 +437,8 @@ public:
void initializeGatekeeper(mertrimGlobalData *g_) {
g = g_;
- readIID = fr.gkFragment_getReadIID();
- seqLen = fr.gkFragment_getSequenceLength();
+ readIID = fr.gkRead_readID();
+ seqLen = fr.gkRead_sequenceLength();
allocLen = seqLen + seqLen;
readName = NULL;
@@ -470,10 +468,15 @@ public:
eDB = NULL;
- strcpy(origSeq, fr.gkFragment_getSequence());
- strcpy(origQlt, fr.gkFragment_getQuality());
- strcpy(corrSeq, fr.gkFragment_getSequence());
- strcpy(corrQlt, fr.gkFragment_getQuality());
+#warning HORRIBLY NON OPTIMAL READING OF READS
+ gkReadData rd;
+
+ g->gkp->gkStore_loadReadData(&fr, &rd);
+
+ strcpy(origSeq, rd.gkReadData_getSequence());
+ strcpy(origQlt, rd.gkReadData_getQualities());
+ strcpy(corrSeq, rd.gkReadData_getSequence());
+ strcpy(corrQlt, rd.gkReadData_getQualities());
// Replace Ns with a random low-quality base. This is necessary, since the mer routines
// will not make a mer for N, and we never see it to correct it.
@@ -515,8 +518,7 @@ public:
readIID = g->gktCur++;
seqLen = 0;
- allocLen = AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 1; // Used for seq/qlt storage only
- allocLen = 65536;
+ allocLen = AS_MAX_READLEN + AS_MAX_READLEN + 1; // Used for seq/qlt storage only
readName = new char [1024];
@@ -644,7 +646,10 @@ public:
for (uint32 i=0; i<allocLen; i++)
seqMap[i] = i;
- if (numReplace >= AS_OVL_ERROR_RATE * seqLen) {
+#warning BOGUS ERROR RATE USED
+ double errorRate = 0.05;
+
+ if (numReplace >= errorRate * seqLen) {
garbageInInput = true;
clrBgn = 0;
clrEnd = 0;
@@ -682,12 +687,12 @@ public:
void dump(char *label);
// Public for the writer.
- gkFragment fr;
+ gkRead fr;
mertrimGlobalData *g;
mertrimThreadData *t;
- AS_IID readIID;
+ uint32 readIID;
uint32 seqLen;
uint32 allocLen;
@@ -742,8 +747,6 @@ public:
uint32 nConf; // Number of bases uncorrected because multiple answers found
char merstring[256];
-
- logMsg log;
};
@@ -755,7 +758,7 @@ uint32
mertrimComputation::evaluate(void) {
if (VERBOSE > 1)
- log.add("\nPROCESS read %d name %s\n", readIID, readName);
+ fprintf(stderr, "\nPROCESS read %d name %s\n", readIID, readName);
if (garbageInInput == true)
return(ALLCRAP);
@@ -782,7 +785,7 @@ mertrimComputation::evaluate(void) {
nMersTested++;
- //log.add("pos %d count %d\n",
+ //fprintf(stderr, "pos %d count %d\n",
// rMS->thePositionInSequence() + g->merSize - 1,
// eDB->count(rMS->theCMer()));
@@ -796,7 +799,7 @@ mertrimComputation::evaluate(void) {
}
if (VERBOSE > 0)
- log.add("INITIAL read %u %s len %u has %u mers, %u correct and %u trusted.\n",
+ fprintf(stderr, "INITIAL read %u %s len %u has %u mers, %u correct and %u trusted.\n",
readIID, readName, seqLen, nMersTested, nMersCorrect, nMersFound);
if (nMersCorrect == nMersExpected)
@@ -927,10 +930,10 @@ mertrimComputation::correctMismatch(uint32 pos, uint32 mNum, uint32 mExtra, bool
uint32 nR = 0;
if (VERBOSE > 2) {
- if (nA > mNum + mExtra) log.add("testA at %d -- %d req=%d\n", pos, nA, mNum + mExtra);
- if (nC > mNum + mExtra) log.add("testC at %d -- %d req=%d\n", pos, nC, mNum + mExtra);
- if (nG > mNum + mExtra) log.add("testG at %d -- %d req=%d\n", pos, nG, mNum + mExtra);
- if (nT > mNum + mExtra) log.add("testT at %d -- %d req=%d\n", pos, nT, mNum + mExtra);
+ if (nA > mNum + mExtra) fprintf(stderr, "testA at %d -- %d req=%d\n", pos, nA, mNum + mExtra);
+ if (nC > mNum + mExtra) fprintf(stderr, "testC at %d -- %d req=%d\n", pos, nC, mNum + mExtra);
+ if (nG > mNum + mExtra) fprintf(stderr, "testG at %d -- %d req=%d\n", pos, nG, mNum + mExtra);
+ if (nT > mNum + mExtra) fprintf(stderr, "testT at %d -- %d req=%d\n", pos, nT, mNum + mExtra);
} // VERBOSE
// If we found a single perfectly correct choice, ignore all the other solutions.
@@ -982,7 +985,7 @@ mertrimComputation::correctMismatch(uint32 pos, uint32 mNum, uint32 mExtra, bool
if (VERBOSE > 0) {
if (nR > 1)
- log.add("Correct read %d at position %d from %c (%u) to %c (%u) (QV %d) (%s) (multiple choices nA=%d nC=%d nG=%d nT=%d)\n",
+ fprintf(stderr, "Correct read %d at position %d from %c (%u) to %c (%u) (QV %d) (%s) (multiple choices nA=%d nC=%d nG=%d nT=%d)\n",
readIID,
(isReversed == false) ? pos : seqLen - pos,
corrSeq[pos], mNum,
@@ -991,7 +994,7 @@ mertrimComputation::correctMismatch(uint32 pos, uint32 mNum, uint32 mExtra, bool
(isReversed == false) ? "fwd" : "rev",
nA, nC, nG, nT);
else
- log.add("Correct read %d at position %d from %c (%u) to %c (%u) (QV %d) (%s)\n",
+ fprintf(stderr, "Correct read %d at position %d from %c (%u) to %c (%u) (QV %d) (%s)\n",
readIID,
(isReversed == false) ? pos : seqLen - pos,
corrSeq[pos], mNum,
@@ -1044,11 +1047,11 @@ mertrimComputation::correctIndel(uint32 pos, uint32 mNum, uint32 mExtra, bool is
if (nT > mNum + mExtra) { nR++; rB = 'T'; rV = nT; }
if (VERBOSE > 2) {
- if (nD > mNum + mExtra) log.add("test-- %d -- %d req=%d\n", pos, nD, mNum + mExtra);
- if (nA > mNum + mExtra) log.add("test+A %d -- %d req=%d\n", pos, nA, mNum + mExtra);
- if (nC > mNum + mExtra) log.add("test+C %d -- %d req=%d\n", pos, nC, mNum + mExtra);
- if (nG > mNum + mExtra) log.add("test+G %d -- %d req=%d\n", pos, nG, mNum + mExtra);
- if (nT > mNum + mExtra) log.add("test+T %d -- %d req=%d\n", pos, nT, mNum + mExtra);
+ if (nD > mNum + mExtra) fprintf(stderr, "test-- %d -- %d req=%d\n", pos, nD, mNum + mExtra);
+ if (nA > mNum + mExtra) fprintf(stderr, "test+A %d -- %d req=%d\n", pos, nA, mNum + mExtra);
+ if (nC > mNum + mExtra) fprintf(stderr, "test+C %d -- %d req=%d\n", pos, nC, mNum + mExtra);
+ if (nG > mNum + mExtra) fprintf(stderr, "test+G %d -- %d req=%d\n", pos, nG, mNum + mExtra);
+ if (nT > mNum + mExtra) fprintf(stderr, "test+T %d -- %d req=%d\n", pos, nT, mNum + mExtra);
} // VERBOSE
if (nR == 0)
@@ -1065,7 +1068,7 @@ mertrimComputation::correctIndel(uint32 pos, uint32 mNum, uint32 mExtra, bool is
if (nD > mNum + mExtra) {
if (VERBOSE > 0) {
- log.add("Correct read %d at position %d from %c (%u) to DELETE (%u) (QV %d) (%s)\n",
+ fprintf(stderr, "Correct read %d at position %d from %c (%u) to DELETE (%u) (QV %d) (%s)\n",
readIID,
(isReversed == false) ? pos : seqLen - pos,
corrSeq[pos], mNum,
@@ -1089,7 +1092,7 @@ mertrimComputation::correctIndel(uint32 pos, uint32 mNum, uint32 mExtra, bool is
} else {
if (VERBOSE > 0) {
- log.add("Correct read %d at position %d from . (%u) to INSERT %c (%u) (%s)\n",
+ fprintf(stderr, "Correct read %d at position %d from . (%u) to INSERT %c (%u) (%s)\n",
readIID,
(isReversed == false) ? pos : seqLen - pos,
mNum,
@@ -1229,7 +1232,7 @@ mertrimComputation::scoreAdapter(void) {
containsAdapterEnd = MAX(containsAdapterEnd, end + 1);
if (VERBOSE > 1)
- log.add("ADAPTER at " F_U32 "," F_U32 " [" F_U32 "," F_U32 "]\n",
+ fprintf(stderr, "ADAPTER at " F_U32 "," F_U32 " [" F_U32 "," F_U32 "]\n",
bgn, end, containsAdapterBgn, containsAdapterEnd);
for (uint32 a=bgn; a<=end; a++)
@@ -1262,7 +1265,7 @@ mertrimComputation::attemptCorrection(bool isReversed) {
uint32 pos = rMS->thePositionInSequence() + g->merSize - 1;
uint32 count = eDB->count(rMS->theCMer());
- //log.add("MER at %d is %s has count %d %s\n",
+ //fprintf(stderr, "MER at %d is %s has count %d %s\n",
// pos,
// rMS->theFMer().merToString(merstring),
// (count >= g->minCorrect) ? "CORRECT" : "ERROR",
@@ -1330,13 +1333,13 @@ mertrimComputation::testBases(char *bases, uint32 basesLen) {
kMer R(g->merSize);
for (uint32 i=1; i<g->merSize && offset<basesLen; i++, offset++) {
- F += letterToBits[bases[offset]];
- R -= letterToBits[complementSymbol[bases[offset]]];
+ F += alphabet.letterToBits(bases[offset]);
+ R -= alphabet.letterToBits(alphabet.complementSymbol(bases[offset]));
}
for (uint32 i=0; i<g->merSize && offset<basesLen; i++, offset++) {
- F += letterToBits[bases[offset]];
- R -= letterToBits[complementSymbol[bases[offset]]];
+ F += alphabet.letterToBits(bases[offset]);
+ R -= alphabet.letterToBits(alphabet.complementSymbol(bases[offset]));
F.mask(true);
R.mask(false);
@@ -1391,7 +1394,7 @@ mertrimComputation::testBaseChange(uint32 pos, char replacement) {
corrSeq[pos] = originalBase;
//if (numConfirmed > 0)
- // log.add("testBaseChange() pos=%d replacement=%c confirmed=%d\n",
+ // fprintf(stderr, "testBaseChange() pos=%d replacement=%c confirmed=%d\n",
// pos, replacement, numConfirmed);
return(numConfirmed);
@@ -1452,7 +1455,7 @@ mertrimComputation::testBaseIndel(uint32 pos, char replacement) {
#endif
//if (numConfirmed > 0)
- // log.add("testBaseIndel() pos=%d replacement=%c confirmed=%d\n",
+ // fprintf(stderr, "testBaseIndel() pos=%d replacement=%c confirmed=%d\n",
// pos, replacement, numConfirmed);
return(numConfirmed);
@@ -1482,7 +1485,7 @@ mertrimComputation::attemptTrimming5End(uint32 *errorPos, uint32 endWindow, uint
}
if (VERBOSE > 1)
- log.add("BGNTRIM found=%u pos=%u from %u to %u\n",
+ fprintf(stderr, "BGNTRIM found=%u pos=%u from %u to %u\n",
endFound, endTrimPos,
clrBgn, clrBgn + endWindow);
@@ -1520,7 +1523,7 @@ mertrimComputation::attemptTrimming3End(uint32 *errorPos, uint32 endWindow, uint
}
if (VERBOSE > 1)
- log.add("ENDTRIM found=%u pos=%u from %u to %u\n",
+ fprintf(stderr, "ENDTRIM found=%u pos=%u from %u to %u\n",
endFound, endTrimPos,
clrEnd - endWindow, clrEnd);
@@ -1562,7 +1565,7 @@ mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) {
clrEnd--;
}
- //log.add("TRIM: %d,%d (lop-off-ends)\n", clrBgn, clrEnd);
+ //fprintf(stderr, "TRIM: %d,%d (lop-off-ends)\n", clrBgn, clrEnd);
//dump("TRIM");
// Deal with adapter. We'll pick the biggest end that is adapter free for our sequence.
@@ -1599,7 +1602,7 @@ mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) {
clrEnd = 0;
}
- //log.add("TRIM: %d,%d (adapter)\n", clrBgn, clrEnd);
+ //fprintf(stderr, "TRIM: %d,%d (adapter)\n", clrBgn, clrEnd);
//dump("TRIM");
}
@@ -1614,7 +1617,7 @@ mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) {
while ((clrEnd > clrBgn) && (coverage[clrEnd-1] == 0))
clrEnd--;
- //log.add("TRIM: %d,%d (lop-off-ends-2)\n", clrBgn, clrEnd);
+ //fprintf(stderr, "TRIM: %d,%d (lop-off-ends-2)\n", clrBgn, clrEnd);
//dump("TRIM");
// True if there is an error at position i.
@@ -1661,7 +1664,7 @@ mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) {
uint32 endc = coverage[end];
if (VERBOSE)
- log.add("IMPERFECT: bgn=%u %u end=%u %u\n",
+ fprintf(stderr, "IMPERFECT: bgn=%u %u end=%u %u\n",
bgn, bgnc,
end, endc);
@@ -1725,7 +1728,7 @@ mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) {
}
if (VERBOSE)
- log.add("Reset clr from %d,%d to %d,%d\n", bgn, end, mbgn, mend);
+ fprintf(stderr, "Reset clr from %d,%d to %d,%d\n", bgn, end, mbgn, mend);
bgn = mbgn;
end = mend;
@@ -1756,7 +1759,7 @@ mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) {
}
if (VERBOSE > 1) {
- log.add("TRIM: %d,%d (post)\n", clrBgn, clrEnd);
+ fprintf(stderr, "TRIM: %d,%d (post)\n", clrBgn, clrEnd);
dump("TRIM");
} // VERBOSE
}
@@ -1769,7 +1772,7 @@ mertrimComputation::dump(char *label) {
uint32 logPos = 0;
uint32 bogus = (clrEnd == 0) ? 0 : UINT32_MAX;
- log.add("%s read %d %s len %d (trim %d-%d)\n", label, readIID, readName, seqLen, clrBgn, clrEnd);
+ fprintf(stderr, "%s read %d %s len %d (trim %d-%d)\n", label, readIID, readName, seqLen, clrBgn, clrEnd);
logPos = 0;
for (uint32 i=0; origSeq[i]; i++) {
@@ -1779,7 +1782,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (ORI)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
logPos = 0;
for (uint32 i=0; i<seqLen; i++) {
@@ -1789,7 +1792,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (SEQ)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
if (corrSeq && verifySeq) {
uint32 i=0;
@@ -1810,7 +1813,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (VAL)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
logPos = 0;
for (uint32 i=0; i<seqLen; i++) {
@@ -1820,7 +1823,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (VAL)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
}
logPos = 0;
@@ -1831,7 +1834,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (QLT)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
logPos = 0;
for (uint32 i=0; i<seqLen; i++) {
@@ -1841,7 +1844,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (COVERAGE)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
logPos = 0;
for (uint32 i=0; i<seqLen; i++) {
@@ -1851,7 +1854,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (CORRECTIONS)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
logPos = 0;
for (uint32 i=0; i<seqLen; i++) {
@@ -1861,7 +1864,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (DISCONNECTION)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
logPos = 0;
for (uint32 i=0; i<seqLen; i++) {
@@ -1871,7 +1874,7 @@ mertrimComputation::dump(char *label) {
if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; }
}
strcpy(logLine + logPos, " (ADAPTER)\n");
- log.add(logLine);
+ fprintf(stderr, logLine);
delete [] logLine;
}
@@ -1942,16 +1945,23 @@ mertrimReaderGatekeeper(mertrimGlobalData *g) {
(s == NULL)) {
s = new mertrimComputation();
- g->gkRead->gkStore_getFragment(g->gktCur, &s->fr, GKFRAGMENT_QLT);
+ g->gkp->gkStore_getRead(g->gktCur);
g->gktCur++;
+ // Original version used to check if the library was eligible for initial trimming
+ // based on kmers. That means nothing in canu.
+
+#if 1
+ s->initializeGatekeeper(g);
+#else
if ((g->forceCorrection) ||
- (g->gkRead->gkStore_getLibrary(s->fr.gkFragment_getLibraryIID())->doTrim_initialMerBased)) {
+ (g->gkp->gkStore_getLibrary(s->fr.gkRead_libraryID())->doTrim_initialMerBased)) {
s->initializeGatekeeper(g);
} else {
delete s;
s = NULL;
}
+#endif
}
return(s);
@@ -1976,7 +1986,7 @@ mertrimReader(void *G) {
mertrimGlobalData *g = (mertrimGlobalData *)G;
mertrimComputation *s = NULL;
- if (g->gkRead)
+ if (g->gkp)
s = mertrimReaderGatekeeper(g);
if (g->fqInput)
@@ -1991,10 +2001,13 @@ void
mertrimWriterGatekeeper(mertrimGlobalData *g, mertrimComputation *s) {
mertrimResult res;
- res.readIID = s->fr.gkFragment_getReadIID();
+ res.readIID = s->fr.gkRead_readID();
+
+#warning BOGUS MIN_READ_LENGTH USED
+ uint32 minReadLength = 500;
if ((s->getClrEnd() <= s->getClrBgn()) ||
- (s->getClrEnd() - s->getClrBgn() < AS_READ_MIN_LEN))
+ (s->getClrEnd() - s->getClrBgn() < minReadLength))
res.deleted = true;
else
res.deleted = false;
@@ -2021,6 +2034,9 @@ mertrimWriterFASTQ(mertrimGlobalData *g, mertrimComputation *s) {
label[0] = 0;
+#warning BOGUS MIN_READ_LENGTH USED
+ uint32 minReadLength = 500;
+
if (s->garbageInInput == true) {
strcat(label, "DEL-GARBAGE");
@@ -2063,7 +2079,7 @@ mertrimWriterFASTQ(mertrimGlobalData *g, mertrimComputation *s) {
if ((s->containsAdapter == false) &&
(s->suspectedChimer == true)) {
- if (s->suspectedChimerBgn - s->clrBgn >= AS_READ_MIN_LEN) {
+ if (s->suspectedChimerBgn - s->clrBgn >= minReadLength) {
strcpy(label, "MP"); // Junction read, longest portion would make an MP pair
seqOffset = s->clrBgn;
@@ -2071,7 +2087,7 @@ mertrimWriterFASTQ(mertrimGlobalData *g, mertrimComputation *s) {
s->corrSeq[s->suspectedChimerBgn] = 0;
s->corrQlt[s->suspectedChimerBgn] = 0;
- } else if (s->clrEnd - s->suspectedChimerEnd >= AS_READ_MIN_LEN) {
+ } else if (s->clrEnd - s->suspectedChimerEnd >= minReadLength) {
strcpy(label, "PE"); // Junction read, longest portion would make a PE pair
seqOffset = s->suspectedChimerEnd;
@@ -2189,8 +2205,7 @@ mertrimWriterFASTQ(mertrimGlobalData *g, mertrimComputation *s) {
s->corrQlt + seqOffset);
if (VERBOSE)
- s->log.add("RESULT read %d len %d (trim %d-%d) %s\n", s->readIID, s->seqLen, s->clrBgn, s->clrEnd, label);
- s->log.fwrite(stdout);
+ fprintf(stderr, "RESULT read %d len %d (trim %d-%d) %s\n", s->readIID, s->seqLen, s->clrBgn, s->clrEnd, label);
}
@@ -2376,11 +2391,9 @@ main(int argc, char **argv) {
exit(1);
}
- gkpStoreFile::registerFile();
-
g->initialize();
- gkFragment fr;
+ gkRead fr;
#if 0
// DEBUG, non-threaded version.
diff --git a/src/falcon_sense/falcon_sense.mk b/src/merTrim/merTrim.mk
similarity index 64%
copy from src/falcon_sense/falcon_sense.mk
copy to src/merTrim/merTrim.mk
index 4cb2f2e..ec89f14 100644
--- a/src/falcon_sense/falcon_sense.mk
+++ b/src/merTrim/merTrim.mk
@@ -7,13 +7,13 @@ ifeq "$(strip ${TARGET_DIR})" ""
TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin
endif
-TARGET := falcon_sense
-SOURCES := falcon_sense.C
+TARGET := merTrim
+SOURCES := merTrim.C
-SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns/libNDFalcon libfalcon
+SRC_INCDIRS := .. ../AS_UTL ../stores ../meryl ../meryl/libleaff ../meryl/libkmer
TGT_LDFLAGS := -L${TARGET_DIR}
-TGT_LDLIBS := -lcanu
-TGT_PREREQS := libcanu.a
+TGT_LDLIBS := -lleaff -lcanu
+TGT_PREREQS := libleaff.a libcanu.a
SUBMAKEFILES :=
diff --git a/src/merTrim/merTrimResult.H b/src/merTrim/merTrimResult.H
index 651be0d..3797318 100644
--- a/src/merTrim/merTrimResult.H
+++ b/src/merTrim/merTrimResult.H
@@ -35,9 +35,6 @@
* full conditions and disclaimers for each license.
*/
-#include <stdio.h>
-#include <stdlib.h>
-
#include "AS_global.H"
#ifndef MERTRIMRESULT_H
@@ -60,11 +57,11 @@ public:
if (F == NULL)
return;
if (chimer)
- fprintf(F, F_IID"\t" F_U32 "\t" F_U32 "\tchimer\t" F_U32 "\t" F_U32 "%s\n",
+ fprintf(F, F_U32"\t" F_U32 "\t" F_U32 "\tchimer\t" F_U32 "\t" F_U32 "%s\n",
readIID, clrBgn, clrEnd, chmBgn, chmEnd,
(deleted) ? "\tdeleted" : "");
else
- fprintf(F, F_IID"\t" F_U32 "\t" F_U32 "%s\n",
+ fprintf(F, F_U32"\t" F_U32 "\t" F_U32 "%s\n",
readIID, clrBgn, clrEnd,
(deleted) ? "\tdeleted" : "");
};
@@ -83,7 +80,7 @@ public:
return(feof(R) == false);
};
- AS_IID readIID;
+ uint32 readIID;
uint32 deleted;
uint32 clrBgn;
uint32 clrEnd;
diff --git a/src/meryl/compare-counts.C b/src/meryl/compare-counts.C
index d70cc68..5b372ec 100644
--- a/src/meryl/compare-counts.C
+++ b/src/meryl/compare-counts.C
@@ -27,6 +27,10 @@
* are Copyright 2014 Battelle National Biodefense Institute, and
* are subject to the BSD 3-Clause License
*
+ * Brian P. Walenz beginning on 2016-NOV-22
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
* File 'README.licenses' in the root directory of this distribution contains
* full conditions and disclaimers for each license.
*/
diff --git a/src/meryl/estimate-mer-threshold.C b/src/meryl/estimate-mer-threshold.C
index e2b696f..6ca6d6f 100644
--- a/src/meryl/estimate-mer-threshold.C
+++ b/src/meryl/estimate-mer-threshold.C
@@ -38,15 +38,117 @@
#include "AS_global.H"
#include "libmeryl.H"
+#include "splitToWords.H"
+
+// Try to deduce the X coverage we have. The pattern we should see in mer counts is an initial
+// spike for unique mers (these contain errors), then a drop into a valley, and a bump at the X
+// coverage.
+//
+// .
+// . ...
+// .. ..........
+// .................
+//
+double
+guessCoverage(uint32 *hist, uint32 histLen) {
+ uint32 i = 2;
+
+ while ((i < histLen) && (hist[i-1] > hist[i]))
+ i++;
+
+ uint32 iX = i - 1;
+
+ while (i < histLen) {
+ if (hist[iX] < hist[i])
+ iX = i;
+ i++;
+ }
+
+ fprintf(stderr, "Guessed X coverage is " F_U32 "\n", iX);
+
+ return(iX);
+}
+
+
+
+void
+loadHistogram(merylStreamReader *MF,
+ uint64 &nDistinct,
+ uint64 &nUnique,
+ uint64 &nTotal,
+ uint32 &histLen, uint32* &hist) {
+
+ nDistinct = MF->numberOfDistinctMers();
+ nUnique = MF->numberOfUniqueMers();
+ nTotal = MF->numberOfTotalMers();
+
+ histLen = MF->histogramLength();
+ hist = new uint32 [histLen];
+
+ for (uint32 hh=0; hh<histLen; hh++)
+ hist[hh] = MF->histogram(hh);
+}
+
+
+
+void
+loadHistogram(FILE *HF,
+ uint64 &nDistinct,
+ uint64 &nUnique,
+ uint64 &nTotal,
+ uint32 &histLen, uint32* &hist) {
+ char L[1024];
+ uint32 histMax;
+
+ nDistinct = 0;
+ nUnique = 0;
+ nTotal = 0;
+
+ histLen = 0;
+ histMax = 1048576;
+ hist = new uint32 [histMax];
+
+ memset(hist, 0, sizeof(uint32) * histMax);
+
+ fgets(L, 1024, HF);
+
+ while (!feof(HF)) {
+ splitToWords W(L);
+
+ uint32 h = W(0);
+ uint32 c = W(1);
+
+ while (h >= histMax)
+ resizeArray(hist, histLen, histMax, histMax * 2, resizeArray_copyData | resizeArray_clearNew);
+
+ hist[h] = c;
+
+ histLen = (histLen < h) ? h : histLen;
+
+ fgets(L, 1024, HF);
+ }
+
+ histLen++;
+
+ nUnique = hist[1];
+
+ for (uint32 hh=0; hh<histLen; hh++) {
+ nDistinct += hist[hh];
+ nTotal += hist[hh] * hh;
+ }
+}
+
+
+
int
main(int argc, char **argv) {
char *gkpPath = 0L;
char *merCountsFile = 0L;
+ char *histogramFile = 0L;
- merylStreamReader *MF = 0L;
-
- uint32 maxCount = 0;
+ double expectedCoverage = 0;
+ double guessedCoverage = 0;
argc = AS_configure(argc, argv);
@@ -56,129 +158,163 @@ main(int argc, char **argv) {
if (strcmp(argv[arg], "-m") == 0) {
merCountsFile = argv[++arg];
+ } else if (strcmp(argv[arg], "-h") == 0) {
+ histogramFile = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-c") == 0) {
+ expectedCoverage = atof(argv[++arg]);
+
} else {
fprintf(stderr, "unknown option '%s'\n", argv[arg]);
err++;
}
arg++;
}
- if ((merCountsFile == 0L) || (err)) {
- fprintf(stderr, "usage: %s -m mercounts\n", argv[0]);
- fprintf(stderr, " -m mercounts file of mercounts\n");
+ if (((merCountsFile == NULL) && (histogramFile == NULL)) || (err)) {
+ fprintf(stderr, "usage: %s [-c coverage] [-m mercounts] [-h histogram]\n", argv[0]);
+ fprintf(stderr, "INPUTS: (exactly one)\n");
+ fprintf(stderr, " -m mercounts file of mercounts from meryl\n");
+ fprintf(stderr, " -h histogram histogram from meryl\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "OPTIONS:\n");
+ fprintf(stderr, " -c coverage expected coverage of reads\n");
exit(1);
}
- MF = new merylStreamReader(merCountsFile);
+ uint64 nDistinct = 0;
+ uint64 nUnique = 0;
+ uint64 nTotal = 0;
+
+ uint32 histLen = 0;
+ uint32 *hist = NULL;
+
+ if (merCountsFile) {
+ merylStreamReader *MF = new merylStreamReader(merCountsFile);
+ loadHistogram(MF, nDistinct, nUnique, nTotal, histLen, hist);
+ delete MF;
+ }
+
+ if (histogramFile) {
+ FILE *HF = fopen(histogramFile, "r");
+ loadHistogram(HF, nDistinct, nUnique, nTotal, histLen, hist);
+ fclose(HF);
+ }
// Examine the counts, pick a reasonable upper limit.
- uint64 totalUsefulDistinct = MF->numberOfDistinctMers() - MF->numberOfUniqueMers();
- uint64 totalUsefulAll = MF->numberOfTotalMers() - MF->numberOfUniqueMers();
+ fprintf(stderr, "RAW MER COUNTS:\n");
+ fprintf(stderr, " distinct: %12" F_U64P " (different kmer sequences)\n", nDistinct);
+ fprintf(stderr, " unique: %12" F_U64P " (single-copy kmers)\n", nUnique);
+ fprintf(stderr, " total: %12" F_U64P " (kmers in sequences)\n", nTotal);
+ fprintf(stderr, "\n");
+
+ guessedCoverage = (expectedCoverage > 0) ? expectedCoverage : guessCoverage(hist, histLen);
+
+
+
+ // Pass 1: look for a reasonable limit, using %distinct and %total.
+ //
+ uint64 totalUsefulDistinct = nDistinct - nUnique;
+ uint64 totalUsefulAll = nTotal - nUnique;
uint64 distinct = 0;
uint64 total = 0;
- uint32 Xcoverage = 8;
- fprintf(stderr, "distinct: " F_U64 "\n", MF->numberOfDistinctMers());
- fprintf(stderr, "unique: " F_U64 "\n", MF->numberOfUniqueMers());
- fprintf(stderr, "total: " F_U64 "\n", MF->numberOfTotalMers());
+ uint32 maxCount = 0;
+ uint32 extCount = 0;
- // Pass 0: try to deduce the X coverage we have. The
- // pattern we should see in mer counts is an initial spike
- // for unique mers (these contain errors), then a drop into
- // a valley, and a bump at the X coverage.
- //
- // .
- // . ...
- // .. ..........
- // .................
+ uint32 kk = 2;
+
+ // If we cover 99% of all the distinct mers, that's reasonable.
//
- // If this pattern is not found, we fallback to the default
- // guess of 8x coverage.
+ // If we're a somewhat high count, and we're covering 2/3 of the total mers, assume that there
+ // are lots of errors (or polymorphism) that are preventing us from covering many distinct mers.
//
- uint32 i = 0;
- uint32 iX = 0;
+ for (; kk < histLen; kk++) {
+ distinct += hist[kk];
+ total += hist[kk] * kk;
- fprintf(stderr, "distinct: " F_U64 "\n", MF->numberOfDistinctMers());
- fprintf(stderr, "unique: " F_U64 "\n", MF->numberOfUniqueMers());
- fprintf(stderr, "total: " F_U64 "\n", MF->numberOfTotalMers());
+ if (((distinct / (double)totalUsefulDistinct) > 0.9975) ||
+ (((total / (double)totalUsefulAll) > 0.6667) && (kk > 50 * guessedCoverage))) {
+ maxCount = kk;
+ break;
+ }
+ }
- fprintf(stderr, "Xcoverage zero 1 0 " F_U64 "\n", MF->histogram(1));
+ fprintf(stderr, "Set maxCount to " F_U32 " (" F_U32 " kmers), which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
+ maxCount,
+ hist[maxCount],
+ 100.0 * distinct / totalUsefulDistinct,
+ 100.0 * total / totalUsefulAll);
- for (i=2; (i < MF->histogramLength()) && (MF->histogram(i-1) > MF->histogram(i)); i++)
- fprintf(stderr, "Xcoverage drop " F_U32 " " F_U64 " " F_U64 "\n", i, MF->histogram(i-1), MF->histogram(i));
+ // Compute an average number of kmers around this count. Skip the 1 count.
- iX = i - 1;
+ uint32 min = (maxCount < 27) ? 2 : maxCount - 25;
+ uint32 max = (maxCount + 26 > histLen) ? histLen : maxCount + 26;
+ uint64 avg = 0;
+ uint64 tot = 0;
- for (; i < MF->histogramLength(); i++) {
- if (MF->histogram(iX) < MF->histogram(i)) {
- fprintf(stderr, "Xcoverage incr " F_U32 " " F_U64 " " F_U64 "\n", i, MF->histogram(iX), MF->histogram(i));
- iX = i;
- } else {
- //fprintf(stderr, "Xcoverage drop " F_U32 " " F_U64 " " F_U64 "\n", i, MF->histogram(iX), MF->histogram(i));
- }
+ for (int32 ii=min; ii<max; ii++) {
+ avg += ii * hist[ii];
+ tot += ii * hist[ii];
}
- fprintf(stderr, "Guessed X coverage is " F_U32 "\n", iX);
+ avg /= (max - min);
- Xcoverage = iX;
+ fprintf(stderr, "Average number of kmers between count " F_U32 " and " F_U32 " is " F_U64 "\n", min, max, avg);
- // Pass 1: look for a reasonable limit, using %distinct and %total.
- //
- for (i=2; (i < MF->histogramLength()) && (maxCount == 0); i++) {
- distinct += MF->histogram(i);
- total += MF->histogram(i) * i;
-
- // If we cover 99% of all the distinct mers, that's reasonable.
- //
- if ((distinct / (double)totalUsefulDistinct) > 0.99)
- maxCount = i;
-
- // If we're a somewhat high count, and we're covering 2/3
- // of the total mers, assume that there are lots of
- // errors (or polymorphism) that are preventing us from
- // covering many distinct mers.
- //
- if ((i > 25 * Xcoverage) && ((total / (double)totalUsefulAll) > (2.0 / 3.0)))
- maxCount = i;
- }
+ // Scan forward until we find a big gap in kmers. This lets us ignore wildly over represented
+ // kmers in small assemblies.
- fprintf(stderr, "Set maxCount to " F_U32 ", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
- i, 100.0 * distinct / totalUsefulDistinct, 100.0 * total / totalUsefulAll);
+ while (max < histLen) {
+ if (tot == 0)
+ break;
+ tot -= min * hist[min]; // Subtract out the last min, move ahead one.
+ tot += max * hist[max]; // Add in the next max, move ahead one.
- // Pass 2: if the limit is relatively small compared to our
- // guessed Xcoverage, and %total is high, keep going to
- // close 75% of the gap in total coverage. So if the TC is
- // 90%, we'd keep going until TC is 97.5%.
- //
- // If we're WAY low compared to X coverage, close the gap
- // too, but not as much. This only happens if we're
- // covering 99% of the distinct, so we're already in good
- // shape. The genome doesn't appear to be very repetitive.
- //
- if (((maxCount < 5 * Xcoverage)) ||
- ((maxCount < 50 * Xcoverage) && (total / (double)totalUsefulAll > 0.90))) {
- double closeAmount = 0.75;
+ min++;
+ max++;
+ }
+
+ uint32 limit;
- if (total / (double)totalUsefulAll <= 0.90)
- closeAmount = 0.5;
+ if (tot > 0) {
+ limit = histLen;
+ fprintf(stderr, "No break in kmer coverage found.\n");
+ } else {
+ limit = min;
+ fprintf(stderr, "Found break in kmer coverage at %d\n", limit);
+ }
- // No, really. This is just 0.75 * (1-TC) + TC
- double desiredTC = closeAmount + (1 - closeAmount) * total / (double)totalUsefulAll;
+ // Scan forward until we see a 10x increase in the number of kmers, OR, until we cover 90% of the sequence and are at sufficient coverage.
- for (; (i < MF->histogramLength()) && (total / (double)totalUsefulAll < desiredTC); i++) {
- distinct += MF->histogram(i);
- total += MF->histogram(i) * i;
- }
+ for (; kk < limit; kk++) {
+ if (avg * 10 < kk * hist[kk])
+ break;
- maxCount = i;
+ if (((double)total / totalUsefulAll >= 0.9) && (kk > 50 * guessedCoverage))
+ break;
- fprintf(stderr, "Reset maxCount to " F_U32 ", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
- maxCount, 100.0 * distinct / totalUsefulDistinct, 100.0 * total / totalUsefulAll);
+ distinct += hist[kk];
+ total += hist[kk] * kk;
+
+ if (hist[kk] > 0)
+ extCount = kk;
}
+ if (extCount > 0)
+ maxCount = extCount;
+
+ fprintf(stderr, "Set maxCount to " F_U32 " (" F_U32 " kmers), which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
+ maxCount,
+ hist[maxCount],
+ 100.0 * distinct / totalUsefulDistinct,
+ 100.0 * total / totalUsefulAll);
+
fprintf(stdout, F_U32"\n", maxCount);
+ delete [] hist;
+
return(0);
}
diff --git a/src/meryl/existDB.C b/src/meryl/existDB.C
new file mode 100644
index 0000000..fed0bf0
--- /dev/null
+++ b/src/meryl/existDB.C
@@ -0,0 +1,252 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-FEB-20 to 2003-OCT-20
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-30 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-DEC-04 to 2014-APR-11
+ * are Copyright 2005,2007-2008,2011,2013-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "existDB.H"
+#include "speedCounter.H"
+#include "libmeryl.H"
+#include "merStream.H"
+
+// Driver for the existDB creation. Reads a sequence.fasta, builds
+// an existDB for the mers in the file, and then writes the internal
+// structures to disk.
+//
+// The existDB constructor is smart enough to read either a pre-built
+// image or a regular multi-fasta file.
+
+
+int
+testFiles(char *filename, char *prefix, uint32 merSize) {
+ char *prefixfilename = new char [strlen(prefix) + 32];
+
+ // Create existDB e and save it to disk
+ //
+ existDB *e = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~uint32ZERO);
+ sprintf(prefixfilename, "%s.1", prefix);
+ e->saveState(prefixfilename);
+
+ // Create existDB f by loading the saved copy from disk
+ //
+ existDB *f = new existDB(prefixfilename);
+
+ // Create a fresh existDB g (to check if we corrup the original when saved)
+ //
+ existDB *g = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~uint32ZERO);
+
+ speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
+ fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (uint64MASK(2 * merSize) + 1) / 1000000.0);
+
+ for (uint64 d=0, m=uint64MASK(2 * merSize); m--; ) {
+ bool ee = e->exists(m);
+ bool ef = f->exists(m);
+ bool eg = g->exists(m);
+
+ uint32 ce = e->count(m);
+ uint32 cf = f->count(m);
+ uint32 cg = g->count(m);
+
+ if ((ee != ef) || (ef != eg) || (ee != eg))
+ fprintf(stderr, "mer "F_X64" not found : e=%d f=%d g=%d\n", m, ee, ef, eg);
+
+ if ((ce != cf) || (cf != cg) || (ce != cg))
+ fprintf(stderr, "mer "F_X64" count differs : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee);
+
+ if ((m & 0xffffff) == 0) {
+ // Been a while since a report, so report.
+ d = 1;
+ }
+
+ if ((ce > 1) && (d == 1)) {
+ // Report anything not unique, to make sure that we're testing real counts and not just existence.
+ fprintf(stderr, "mer "F_X64" : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee);
+ d = 0;
+ }
+
+ C->tick();
+ }
+
+ delete e;
+ delete C;
+
+ return(0);
+}
+
+
+int
+testExistence(char *filename, uint32 merSize) {
+ existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~uint32ZERO);
+ merStream *M = new merStream(new kMerBuilder(merSize), new seqStream(filename), true, true);
+ uint64 tried = 0;
+ uint64 lost = 0;
+
+ while (M->nextMer()) {
+ tried++;
+ if (!E->exists(M->theFMer()))
+ lost++;
+ }
+
+ delete M;
+ delete E;
+
+ if (lost) {
+ fprintf(stderr, "Tried "F_U64", didn't find "F_U64" merStream mers in the existDB.\n", tried, lost);
+ return(1);
+ } else {
+ return(0);
+ }
+}
+
+
+
+int
+testExhaustive(char *filename, char *merylname, uint32 merSize) {
+ existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~uint32ZERO);
+ merylStreamReader *M = new merylStreamReader(merylname);
+ speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
+ uint64 found = uint64ZERO;
+ uint64 expected = uint64ZERO;
+
+ FILE *DUMP = 0L;
+
+ DUMP = fopen("testExhaustive.ms.dump", "w");
+
+ while (M->nextMer()) {
+ if (E->exists(M->theFMer())) {
+ expected++;
+ fprintf(DUMP, F_X64"\n", (uint64)M->theFMer());
+ } else {
+ fprintf(DUMP, F_X64" MISSED!\n", (uint64)M->theFMer());
+ }
+ }
+
+ fclose(DUMP);
+
+ fprintf(stderr, "Found "F_U64" mers in the meryl database.\n", expected);
+ fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (uint64MASK(2 * merSize) + 1) / 1000000.0);
+
+ DUMP = fopen("testExhaustive.ck.dump", "w");
+
+ for (uint64 m = uint64MASK(2 * merSize); m--; ) {
+ if (E->exists(m)) {
+ found++;
+ fprintf(DUMP, F_X64"\n", m);
+ }
+ C->tick();
+ }
+
+ fclose(DUMP);
+
+ delete C;
+ delete E;
+ delete M;
+
+ if (expected != found) {
+ fprintf(stderr, "Expected to find "F_U64" mers, but found "F_U64" instead.\n",
+ expected, found);
+ return(1);
+ } else {
+ return(0);
+ }
+}
+
+
+const char *usage =
+"usage: %s [stuff]\n"
+" -mersize mersize\n"
+" -- Use the specified mersize when building existDB tables.\n"
+"\n"
+" -build some.fasta prefix\n"
+" -- Build an existDB on all mers in some.fasta and save\n"
+" the tables into prefix.\n"
+"\n"
+" -describe prefix\n"
+" -- Reports the state of some existDB file.\n"
+"\n"
+" -testfiles some.fasta prefix\n"
+" -- Build an existDB table from some.fasta. Write that table to disk.\n"
+" Load the table back. Compare that each mer in some.fasta is present\n"
+" in all three existDB tables created earlier.\n"
+"\n"
+" -testexistence some.fasta\n"
+" -- Build an existDB table from some.fasta, check that every\n"
+" mer in some.fasta can be found in the table. Does not\n"
+" guarantee that every mer in the table is found in the file.\n"
+"\n"
+" -testexhaustive some.fasta some.meryl\n"
+" -- Build an existDB table from some.fasta, check _EVERY_ mer\n"
+" for existance. Complain if a mer exists in the table but\n"
+" not in the meryl database. Assumes 'some.meryl' is the\n"
+" mercount of some.fasta.\n"
+"\n";
+
+int
+main(int argc, char **argv) {
+ uint32 mersize = 20;
+
+ if (argc < 3) {
+ fprintf(stderr, usage, argv[0]);
+ exit(1);
+ }
+
+ int arg = 1;
+ while (arg < argc) {
+ if (strncmp(argv[arg], "-mersize", 2) == 0) {
+ arg++;
+ mersize = atoi(argv[arg]);
+
+ } else if (strncmp(argv[arg], "-describe", 2) == 0) {
+ existDB *e = new existDB(argv[argc-1], false);
+ e->printState(stdout);
+ delete e;
+ exit(0);
+
+ } else if (strncmp(argv[arg], "-testfiles", 8) == 0) {
+ exit(testFiles(argv[arg+1], argv[arg+2], mersize));
+
+ } else if (strncmp(argv[arg], "-testexistence", 8) == 0) {
+ exit(testExistence(argv[arg+1], mersize));
+
+ } else if (strncmp(argv[arg], "-testexhaustive", 8) == 0) {
+ exit(testExhaustive(argv[arg+1], argv[arg+2], mersize));
+
+ } else if (strncmp(argv[arg], "-build", 2) == 0) {
+ existDB *e = new existDB(argv[argc-2], mersize, existDBnoFlags, 0, ~uint32ZERO);
+ e->saveState(argv[argc-1]);
+ delete e;
+ exit(0);
+ }
+
+ arg++;
+ }
+
+ exit(0);
+}
diff --git a/src/falcon_sense/falcon_sense.mk b/src/meryl/existDB.mk
similarity index 66%
copy from src/falcon_sense/falcon_sense.mk
copy to src/meryl/existDB.mk
index 4cb2f2e..4edefaf 100644
--- a/src/falcon_sense/falcon_sense.mk
+++ b/src/meryl/existDB.mk
@@ -1,3 +1,4 @@
+
# If 'make' isn't run from the root directory, we need to set these to
# point to the upper level build directory.
ifeq "$(strip ${BUILD_DIR})" ""
@@ -7,13 +8,14 @@ ifeq "$(strip ${TARGET_DIR})" ""
TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin
endif
-TARGET := falcon_sense
-SOURCES := falcon_sense.C
+TARGET := existDB
+SOURCES := existDB.C
-SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns/libNDFalcon libfalcon
+SRC_INCDIRS := .. ../AS_UTL libleaff libkmer
TGT_LDFLAGS := -L${TARGET_DIR}
-TGT_LDLIBS := -lcanu
-TGT_PREREQS := libcanu.a
+TGT_LDLIBS := -lleaff -lcanu
+TGT_PREREQS := libleaff.a libcanu.a
SUBMAKEFILES :=
+
diff --git a/src/meryl/libkmer/existDB-create-from-fasta.C b/src/meryl/libkmer/existDB-create-from-fasta.C
new file mode 100644
index 0000000..c5da5de
--- /dev/null
+++ b/src/meryl/libkmer/existDB-create-from-fasta.C
@@ -0,0 +1,304 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-SEP-08 to 2003-OCT-20
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAR-20 to 2014-APR-11
+ * are Copyright 2005,2007-2008,2012-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz on 2014-AUG-31
+ * are Copyright 2014 Battelle National Biodefense Institute, and
+ * are subject to the BSD 3-Clause License
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "existDB.H"
+
+#include "seqCache.H"
+#include "seqStream.H"
+#include "merStream.H"
+
+bool
+existDB::createFromFastA(char const *filename,
+ uint32 merSize,
+ uint32 flags) {
+
+ bool beVerbose = false;
+ bool rebuilding = false;
+
+ _hashTable = 0L;
+ _buckets = 0L;
+ _counts = 0L;
+
+ _merSizeInBases = merSize;
+
+ _searchForDupe = true;
+
+ if ((flags & existDBcompressHash) ||
+ (flags & existDBcompressBuckets) ||
+ (flags & existDBcompressCounts))
+ fprintf(stderr, "existDB::createFromSequence: compression not supported.\n"), exit(1);
+
+ // This (at =22) eats up 16MB, and should allow a lot of mers at big sizes. Unfortunately, we
+ // know nothing about how man mers are going to be in the input.
+ //
+ // Setting this too high drastically reduces performance, suspected because of cache misses.
+ // Setting this too low will also reduce performance, by increasing the search time in a bucket.
+ //
+ uint32 tblBits = logBaseTwo64(AS_UTL_sizeOfFile(filename));
+
+ rebuild:
+ _shift1 = 2 * _merSizeInBases - tblBits;
+ _shift2 = _shift1 / 2;
+ _mask1 = uint64MASK(tblBits);
+ _mask2 = uint64MASK(_shift1);
+
+ _hshWidth = uint32ZERO;
+ _chkWidth = 2 * merSize - tblBits;
+ _cntWidth = 16;
+
+ uint64 tableSizeInEntries = uint64ONE << tblBits;
+ uint64 numberOfMers = uint64ZERO;
+ uint64 *countingTable = new uint64 [tableSizeInEntries + 1];
+
+ for (uint64 i=tableSizeInEntries+1; i--; )
+ countingTable[i] = 0;
+
+ _isCanonical = flags & existDBcanonical;
+ _isForward = flags & existDBforward;
+
+ assert(_isCanonical + _isForward == 1);
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 1) Count bucket sizes
+ //
+ merStream *M = new merStream(new kMerBuilder(_merSizeInBases),
+ new seqStream(filename),
+ true, true);
+
+ while (M->nextMer()) {
+ if (_isForward) {
+ countingTable[ HASH(M->theFMer()) ]++;
+ numberOfMers++;
+ }
+
+ if (_isCanonical) {
+ countingTable[ HASH(M->theCMer()) ]++;
+ numberOfMers++;
+ }
+ }
+
+ delete M;
+
+#ifdef STATS
+ uint64 dist[32] = {0};
+ uint64 maxcnt = 0;
+ for (uint64 i=tableSizeInEntries+1; i--; ) {
+ if (countingTable[i] > maxcnt)
+ maxcnt = countingTable[i];
+
+ if (countingTable[i] < 32)
+ dist[countingTable[i]]++;
+ }
+
+ for(uint64 i=0; i<32; i++)
+ fprintf(stderr, "existDB::usage[%2d] = %d\n", i, dist[i]);
+ fprintf(stderr, "existDB::maxcnt = %d\n", maxcnt);
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Determine how many bits we need to hold the value
+ // numberOfMers.....then....
+ //
+ // This is numberOfMers+1 because we need to store the
+ // first position after the last mer. That is, if there are two
+ // mers, we will store that the first mer is at position 0, the
+ // second mer is at position 1, and the end of the second mer is at
+ // position 2.
+ //
+ if (_compressedHash) {
+ _hshWidth = 1;
+ while ((numberOfMers+1) > (uint64ONE << _hshWidth))
+ _hshWidth++;
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 2) Allocate a hash table and some mer storage buckets.
+ //
+ _hashTableWords = tableSizeInEntries + 2;
+ if (_compressedHash)
+ _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
+
+ _bucketsWords = numberOfMers + 2;
+ if (_compressedBucket)
+ _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
+
+ _countsWords = numberOfMers + 2;
+ if (_compressedCounts)
+ _countsWords = _countsWords * _cntWidth / 64 + 1;
+
+ if (beVerbose) {
+ fprintf(stderr, "existDB::createFromFastA()-- hashTable is "F_U64"MB\n", _hashTableWords >> 17);
+ fprintf(stderr, "existDB::createFromFastA()-- buckets is "F_U64"MB\n", _bucketsWords >> 17);
+ if (flags & existDBcounts)
+ fprintf(stderr, "existDB::createFromFastA()-- counts is "F_U64"MB\n", _countsWords >> 17);
+ }
+
+ _hashTable = new uint64 [_hashTableWords];
+ _buckets = new uint64 [_bucketsWords];
+ _countsWords = (flags & existDBcounts) ? _countsWords : 0;
+ _counts = (flags & existDBcounts) ? new uint64 [_countsWords] : 0L;
+
+ // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
+ // is also cleared as it is initialized, but in the _compressedHash case, the last
+ // few words might be uninitialized. They're unused.
+ //
+ //memset(_hashTable, 0, sizeof(uint64) * _hashTableWords);
+ //memset(_buckets, 0, sizeof(uint64) * _bucketsWords); // buckets is cleared as it is built
+ //memset(_counts, 0, sizeof(uint64) * _countsWords);
+
+ _hashTable[_hashTableWords-1] = 0;
+ _hashTable[_hashTableWords-2] = 0;
+ _hashTable[_hashTableWords-3] = 0;
+ _hashTable[_hashTableWords-4] = 0;
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Make the hash table point to the start of the bucket, and reset
+ // the counting table -- we're going to use it to fill the buckets.
+ //
+ uint64 tmpPosition = 0;
+ uint64 begPosition = 0;
+ uint64 ptr = 0;
+
+ if (_compressedHash) {
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ tmpPosition = countingTable[i];
+ countingTable[i] = begPosition;
+
+ setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
+ ptr += _hshWidth;
+
+ begPosition += tmpPosition;
+ }
+
+ setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
+ } else {
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ tmpPosition = countingTable[i];
+ countingTable[i] = begPosition;
+
+ _hashTable[i] = begPosition;
+
+ begPosition += tmpPosition;
+ }
+
+ // Set the last position in the hash, but we don't care about
+ // the temporary counting table.
+ //
+ _hashTable[tableSizeInEntries] = begPosition;
+ }
+
+
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 3) Build list of mers, placed into buckets
+ //
+ M = new merStream(new kMerBuilder(_merSizeInBases),
+ new seqStream(filename),
+ true, true);
+
+ while (M->nextMer()) {
+ if (_isForward)
+ insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), 1, countingTable);
+
+ if (_isCanonical)
+ insertMer(HASH(M->theCMer()), CHECK(M->theCMer()), 1, countingTable);
+ }
+
+ delete M;
+
+ // Compress out the gaps we have from redundant kmers.
+
+ uint64 pos = 0;
+ uint64 frm = 0;
+ uint64 len = 0;
+
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ frm = _hashTable[i];
+ len = countingTable[i] - _hashTable[i];
+
+ _hashTable[i] = pos;
+
+ for (uint64 j=0; j<len; j++) {
+ if (_counts)
+ _counts[pos] = _counts[frm];
+
+ _buckets[pos++] = _buckets[frm++];
+ }
+ }
+
+ if (beVerbose)
+ fprintf(stderr, "Compressed from "F_U64" to "F_U64" ("F_U64" bits)\n",
+ _hashTable[tableSizeInEntries], pos, logBaseTwo64(pos));
+
+ while (pos < _bucketsWords)
+ _buckets[pos++] = 0;
+
+ _hashTable[tableSizeInEntries] = pos;
+
+ // All done. Delete temporary stuff
+ //
+ delete [] countingTable;
+
+ // But if we horribly screwed up the estimate of tblBits, reset and recompute
+
+ if ((logBaseTwo64(pos) < tblBits) &&
+ (rebuilding == false)) {
+ rebuilding = true;
+
+ delete [] _hashTable;
+ delete [] _buckets;
+ delete [] _counts;
+
+ _hashTable = 0L;
+ _buckets = 0L;
+ _counts = 0L;
+
+ tblBits = logBaseTwo64(pos);
+
+ goto rebuild;
+ }
+
+ return(true);
+}
diff --git a/src/meryl/libkmer/existDB-create-from-meryl.C b/src/meryl/libkmer/existDB-create-from-meryl.C
new file mode 100644
index 0000000..cd4c117
--- /dev/null
+++ b/src/meryl/libkmer/existDB-create-from-meryl.C
@@ -0,0 +1,261 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-SEP-08 to 2003-OCT-20
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz on 2004-APR-12
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAR-20 to 2014-APR-11
+ * are Copyright 2005,2007,2010-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "existDB.H"
+#warning YUCK RELATIVE INCLUDE OF libmeryl.H
+#include "../libmeryl.H"
+#include "speedCounter.H"
+
+
+bool
+existDB::createFromMeryl(char const *prefix,
+ uint32 merSize,
+ uint32 lo,
+ uint32 hi,
+ uint32 flags) {
+
+ merylStreamReader *M = new merylStreamReader(prefix);
+
+ bool beVerbose = false;
+
+ _hashTable = 0L;
+ _buckets = 0L;
+ _counts = 0L;
+
+ _merSizeInBases = M->merSize();
+
+ if (merSize != _merSizeInBases) {
+ fprintf(stderr, "createFromMeryl()-- ERROR: requested merSize ("F_U32") is different than merSize in meryl database ("F_U32").\n",
+ merSize, _merSizeInBases);
+ exit(1);
+ }
+
+ // We can set this exactly, but not memory optimal (see meryl/estimate.C:optimalNumberOfBuckets()).
+ // Instead, we just blindly use whatever meryl used.
+ //
+ uint32 tblBits = M->prefixSize();
+
+ // But it is faster to reset to this. Might use 2x the memory.
+ //uint32 tblBits = logBaseTwo64(M->numberOfDistinctMers() + 1);
+
+ _shift1 = 2 * _merSizeInBases - tblBits;
+ _shift2 = _shift1 / 2;
+ _mask1 = uint64MASK(tblBits);
+ _mask2 = uint64MASK(_shift1);
+
+ _hshWidth = uint32ZERO;
+ _chkWidth = 2 * _merSizeInBases - tblBits;
+ _cntWidth = 16;
+
+ uint64 tableSizeInEntries = uint64ONE << tblBits;
+ uint64 numberOfMers = uint64ZERO;
+ uint64 *countingTable = new uint64 [tableSizeInEntries + 1];
+
+ if (beVerbose) {
+ fprintf(stderr, "createFromMeryl()-- tableSizeInEntries "F_U64"\n", tableSizeInEntries);
+ fprintf(stderr, "createFromMeryl()-- count range "F_U32"-"F_U32"\n", lo, hi);
+ }
+
+ for (uint64 i=tableSizeInEntries+1; i--; )
+ countingTable[i] = 0;
+
+ _isCanonical = flags & existDBcanonical;
+ _isForward = flags & existDBforward;
+
+ if (beVerbose) {
+ fprintf(stderr, "createFromMeryl()-- canonical %c\n", (_isCanonical) ? 'T' : 'F');
+ fprintf(stderr, "createFromMeryl()-- forward %c\n", (_isForward) ? 'T' : 'F');
+ }
+
+ assert(_isCanonical + _isForward == 1);
+
+ // 1) Count bucket sizes
+ // While we don't know the bucket sizes right now, but we do know
+ // how many buckets and how many mers.
+ //
+ // Because we could be inserting both forward and reverse, we can't
+ // really move the direction testing outside the loop, unless we
+ // want to do two iterations over M.
+ //
+ speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
+
+ while (M->nextMer()) {
+ if ((lo <= M->theCount()) && (M->theCount() <= hi)) {
+ if (_isForward) {
+ countingTable[ HASH(M->theFMer()) ]++;
+ numberOfMers++;
+ }
+
+ if (_isCanonical) {
+ kMer r = M->theFMer();
+ r.reverseComplement();
+
+ if (M->theFMer() < r)
+ countingTable[ HASH(M->theFMer()) ]++;
+ else
+ countingTable[ HASH(r) ]++;
+ numberOfMers++;
+ }
+
+ C->tick();
+ }
+ }
+
+ if (beVerbose)
+ fprintf(stderr, "createFromMeryl()-- numberOfMers "F_U64"\n", numberOfMers);
+
+ delete C;
+ delete M;
+
+ if (_compressedHash) {
+ _hshWidth = 1;
+ while ((numberOfMers+1) > (uint64ONE << _hshWidth))
+ _hshWidth++;
+ }
+
+ if (beVerbose) {
+ fprintf(stderr, "existDB::createFromMeryl()-- Found "F_U64" mers between count of "F_U32" and "F_U32"\n",
+ numberOfMers, lo, hi);
+ }
+
+ // 2) Allocate hash table, mer storage buckets
+ //
+ _hashTableWords = tableSizeInEntries + 2;
+ if (_compressedHash)
+ _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
+
+ _bucketsWords = numberOfMers + 2;
+ if (_compressedBucket)
+ _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
+
+ _countsWords = numberOfMers + 2;
+ if (_compressedCounts)
+ _countsWords = _countsWords * _cntWidth / 64 + 1;
+
+ if (beVerbose) {
+ fprintf(stderr, "existDB::createFromMeryl()-- hashTable is "F_U64"MB\n", _hashTableWords >> 17);
+ fprintf(stderr, "existDB::createFromMeryl()-- buckets is "F_U64"MB\n", _bucketsWords >> 17);
+ if (flags & existDBcounts)
+ fprintf(stderr, "existDB::createFromMeryl()-- counts is "F_U64"MB\n", _countsWords >> 17);
+ }
+
+ _hashTable = new uint64 [_hashTableWords];
+ _buckets = new uint64 [_bucketsWords];
+ _countsWords = (flags & existDBcounts) ? _countsWords : 0;
+ _counts = (flags & existDBcounts) ? new uint64 [_countsWords] : 0L;
+
+ // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
+ // is also cleared as it is initialized, but in the _compressedHash case, the last
+ // few words might be uninitialized. They're unused.
+
+ //memset(_hashTable, 0, sizeof(uint64) * _hashTableWords);
+ //memset(_buckets, 0, sizeof(uint64) * _bucketsWords); // buckets is cleared as it is built
+ //memset(_counts, 0, sizeof(uint64) * _countsWords);
+
+ _hashTable[_hashTableWords-1] = 0;
+ _hashTable[_hashTableWords-2] = 0;
+ _hashTable[_hashTableWords-3] = 0;
+ _hashTable[_hashTableWords-4] = 0;
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Make the hash table point to the start of the bucket, and reset
+ // the counting table -- we're going to use it to fill the buckets.
+ //
+ uint64 tmpPosition = 0;
+ uint64 begPosition = 0;
+ uint64 ptr = 0;
+
+ if (_compressedHash) {
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ tmpPosition = countingTable[i];
+ countingTable[i] = begPosition;
+
+ setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
+ ptr += _hshWidth;
+
+ begPosition += tmpPosition;
+ }
+
+ setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
+ } else {
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ tmpPosition = countingTable[i];
+ countingTable[i] = begPosition;
+
+ _hashTable[i] = begPosition;
+
+ begPosition += tmpPosition;
+ }
+
+ // Set the last position in the hash, but we don't care about
+ // the temporary counting table.
+ //
+ _hashTable[tableSizeInEntries] = begPosition;
+ }
+
+
+ ///////////////////////////////////////////////////////////////////////////////
+ //
+ // 3) Build list of mers, placed into buckets
+ //
+ M = new merylStreamReader(prefix);
+ C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
+
+ while (M->nextMer()) {
+ if ((lo <= M->theCount()) && (M->theCount() <= hi)) {
+ if (_isForward)
+ insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable);
+
+ if (_isCanonical) {
+ kMer r = M->theFMer();
+ r.reverseComplement();
+
+ if (M->theFMer() < r)
+ insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable);
+ else
+ insertMer(HASH(r), CHECK(r), M->theCount(), countingTable);
+ numberOfMers++;
+ }
+
+
+ C->tick();
+ }
+ }
+
+ delete C;
+ delete M;
+ delete [] countingTable;
+
+ return(true);
+}
diff --git a/src/meryl/libkmer/existDB-create-from-sequence.C b/src/meryl/libkmer/existDB-create-from-sequence.C
new file mode 100644
index 0000000..e2dba06
--- /dev/null
+++ b/src/meryl/libkmer/existDB-create-from-sequence.C
@@ -0,0 +1,293 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2012-MAY-08 to 2014-APR-11
+ * are Copyright 2012-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz on 2014-AUG-31
+ * are Copyright 2014 Battelle National Biodefense Institute, and
+ * are subject to the BSD 3-Clause License
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "existDB.H"
+#include "merStream.H"
+
+bool
+existDB::createFromSequence(char const *sequence,
+ uint32 merSize,
+ uint32 flags) {
+
+ bool beVerbose = false;
+ bool rebuilding = false;
+
+ _hashTable = 0L;
+ _buckets = 0L;
+ _counts = 0L;
+
+ _merSizeInBases = merSize;
+
+ _searchForDupe = true;
+
+ if ((flags & existDBcompressHash) ||
+ (flags & existDBcompressBuckets) ||
+ (flags & existDBcompressCounts))
+ fprintf(stderr, "existDB::createFromSequence: compression not supported.\n"), exit(1);
+
+ // This (at =22) eats up 16MB, and should allow a lot of mers at big sizes. Unfortunately, we
+ // know nothing about how man mers are going to be in the input.
+ //
+ // Setting this too high drastically reduces performance, suspected because of cache misses.
+ // Setting this too low will also reduce performance, by increasing the search time in a bucket.
+ //
+ uint32 tblBits = logBaseTwo64(strlen(sequence));
+
+ rebuild:
+ _shift1 = 2 * _merSizeInBases - tblBits;
+ _shift2 = _shift1 / 2;
+ _mask1 = uint64MASK(tblBits);
+ _mask2 = uint64MASK(_shift1);
+
+ _hshWidth = uint32ZERO;
+ _chkWidth = 2 * merSize - tblBits;
+ _cntWidth = 16;
+
+ uint64 tableSizeInEntries = uint64ONE << tblBits;
+ uint64 numberOfMers = uint64ZERO;
+ uint64 *countingTable = new uint64 [tableSizeInEntries + 1];
+
+ for (uint64 i=tableSizeInEntries+1; i--; )
+ countingTable[i] = 0;
+
+ _isCanonical = flags & existDBcanonical;
+ _isForward = flags & existDBforward;
+
+ assert(_isCanonical + _isForward == 1);
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 1) Count bucket sizes
+ //
+ merStream *M = new merStream(new kMerBuilder(_merSizeInBases),
+ new seqStream(sequence, strlen(sequence)),
+ true, true);
+
+ while (M->nextMer()) {
+ if (_isForward) {
+ countingTable[ HASH(M->theFMer()) ]++;
+ numberOfMers++;
+ }
+
+ if (_isCanonical) {
+ countingTable[ HASH(M->theCMer()) ]++;
+ numberOfMers++;
+ }
+ }
+
+ delete M;
+
+#ifdef STATS
+ uint64 dist[32] = {0};
+ uint64 maxcnt = 0;
+ for (uint64 i=tableSizeInEntries+1; i--; ) {
+ if (countingTable[i] > maxcnt)
+ maxcnt = countingTable[i];
+
+ if (countingTable[i] < 32)
+ dist[countingTable[i]]++;
+ }
+
+ for(uint64 i=0; i<32; i++)
+ fprintf(stderr, "existDB::usage[%2d] = %d\n", i, dist[i]);
+ fprintf(stderr, "existDB::maxcnt = %d\n", maxcnt);
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Determine how many bits we need to hold the value
+ // numberOfMers.....then....
+ //
+ // This is numberOfMers+1 because we need to store the
+ // first position after the last mer. That is, if there are two
+ // mers, we will store that the first mer is at position 0, the
+ // second mer is at position 1, and the end of the second mer is at
+ // position 2.
+ //
+ if (_compressedHash) {
+ _hshWidth = 1;
+ while ((numberOfMers+1) > (uint64ONE << _hshWidth))
+ _hshWidth++;
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 2) Allocate a hash table and some mer storage buckets.
+ //
+ _hashTableWords = tableSizeInEntries + 2;
+ if (_compressedHash)
+ _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
+
+ _bucketsWords = numberOfMers + 2;
+ if (_compressedBucket)
+ _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
+
+ _countsWords = numberOfMers + 2;
+ if (_compressedCounts)
+ _countsWords = _countsWords * _cntWidth / 64 + 1;
+
+ if (beVerbose) {
+ fprintf(stderr, "existDB::createFromSequence()-- hashTable is "F_U64"MB\n", _hashTableWords >> 17);
+ fprintf(stderr, "existDB::createFromSequence()-- buckets is "F_U64"MB\n", _bucketsWords >> 17);
+ if (flags & existDBcounts)
+ fprintf(stderr, "existDB::createFromSequence()-- counts is "F_U64"MB\n", _countsWords >> 17);
+ }
+
+ _hashTable = new uint64 [_hashTableWords];
+ _buckets = new uint64 [_bucketsWords];
+ _countsWords = (flags & existDBcounts) ? _countsWords : 0;
+ _counts = (flags & existDBcounts) ? new uint64 [_countsWords] : 0L;
+
+ // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
+ // is also cleared as it is initialized, but in the _compressedHash case, the last
+ // few words might be uninitialized. They're unused.
+
+ //memset(_hashTable, 0, sizeof(uint64) * _hashTableWords);
+ //memset(_buckets, 0, sizeof(uint64) * _bucketsWords); // buckets is cleared as it is built
+ //memset(_counts, 0, sizeof(uint64) * _countsWords);
+
+ _hashTable[_hashTableWords-1] = 0;
+ _hashTable[_hashTableWords-2] = 0;
+ _hashTable[_hashTableWords-3] = 0;
+ _hashTable[_hashTableWords-4] = 0;
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Make the hash table point to the start of the bucket, and reset
+ // the counting table -- we're going to use it to fill the buckets.
+ //
+ uint64 tmpPosition = 0;
+ uint64 begPosition = 0;
+ uint64 ptr = 0;
+
+ if (_compressedHash) {
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ tmpPosition = countingTable[i];
+ countingTable[i] = begPosition;
+
+ setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
+ ptr += _hshWidth;
+
+ begPosition += tmpPosition;
+ }
+
+ setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
+ } else {
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ tmpPosition = countingTable[i];
+ countingTable[i] = begPosition;
+
+ _hashTable[i] = begPosition;
+
+ begPosition += tmpPosition;
+ }
+
+ // Set the last position in the hash, but we don't care about
+ // the temporary counting table.
+ //
+ _hashTable[tableSizeInEntries] = begPosition;
+ }
+
+
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 3) Build list of mers, placed into buckets
+ //
+ M = new merStream(new kMerBuilder(_merSizeInBases),
+ new seqStream(sequence, strlen(sequence)),
+ true, true);
+
+ while (M->nextMer()) {
+ if (_isForward)
+ insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), 1, countingTable);
+
+ if (_isCanonical)
+ insertMer(HASH(M->theCMer()), CHECK(M->theCMer()), 1, countingTable);
+ }
+
+ delete M;
+
+ // Compress out the gaps we have from redundant kmers.
+
+ uint64 pos = 0;
+ uint64 frm = 0;
+ uint64 len = 0;
+
+ for (uint64 i=0; i<tableSizeInEntries; i++) {
+ frm = _hashTable[i];
+ len = countingTable[i] - _hashTable[i];
+
+ _hashTable[i] = pos;
+
+ for (uint64 j=0; j<len; j++) {
+ if (_counts)
+ _counts[pos] = _counts[frm];
+
+ _buckets[pos++] = _buckets[frm++];
+ }
+ }
+
+ if (beVerbose)
+ fprintf(stderr, "Compressed from "F_U64" to "F_U64" ("F_U64" bits)\n",
+ _hashTable[tableSizeInEntries], pos, logBaseTwo64(pos));
+
+ while (pos < _bucketsWords)
+ _buckets[pos++] = 0;
+
+ _hashTable[tableSizeInEntries] = pos;
+
+ // All done. Delete temporary stuff
+ //
+ delete [] countingTable;
+
+ // But if we horribly screwed up the estimate of tblBits, reset and recompute
+
+ if ((logBaseTwo64(pos) < tblBits) &&
+ (rebuilding == false)) {
+ rebuilding = true;
+
+ delete [] _hashTable;
+ delete [] _buckets;
+ delete [] _counts;
+
+ _hashTable = 0L;
+ _buckets = 0L;
+ _counts = 0L;
+
+ tblBits = logBaseTwo64(pos);
+
+ goto rebuild;
+ }
+
+ return(true);
+}
diff --git a/src/meryl/libkmer/existDB-state.C b/src/meryl/libkmer/existDB-state.C
new file mode 100644
index 0000000..73d9abd
--- /dev/null
+++ b/src/meryl/libkmer/existDB-state.C
@@ -0,0 +1,233 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz on 2003-SEP-08
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-12 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAR-20 to 2014-APR-11
+ * are Copyright 2005,2007,2013-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "existDB.H"
+
+
+const char magic[16] = { 'e', 'x', 'i', 's', 't', 'D', 'B', '2',
+ ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' };
+
+
+void
+existDB::saveState(char const *filename) {
+ char cigam[16] = { 0 };
+
+ errno = 0;
+ FILE *F = fopen(filename, "wb");
+ if (errno) {
+ fprintf(stderr, "Can't open '%s' for writing\n%s\n", filename, strerror(errno));
+ exit(1);
+ }
+
+ strncpy(cigam, magic, 16);
+
+ if (_compressedHash)
+ cigam[8] = 'h';
+ if (_compressedBucket)
+ cigam[9] = 'b';
+ if (_compressedCounts)
+ cigam[10] = 'c';
+
+ if (_isForward)
+ cigam[11] = 'F';
+ if (_isCanonical)
+ cigam[11] = 'C';
+
+ fwrite(cigam, sizeof(char), 16, F);
+
+ fwrite(&_merSizeInBases, sizeof(uint32), 1, F);
+ fwrite(&_shift1, sizeof(uint32), 1, F);
+ fwrite(&_shift2, sizeof(uint32), 1, F);
+ fwrite(&_mask1, sizeof(uint64), 1, F);
+ fwrite(&_mask2, sizeof(uint64), 1, F);
+ fwrite(&_hshWidth, sizeof(uint32), 1, F); // only valid if _compressedHash
+ fwrite(&_chkWidth, sizeof(uint32), 1, F); // only valid if _compressedBucket
+ fwrite(&_cntWidth, sizeof(uint32), 1, F); // only valid if _compressedCounts
+
+ fwrite(&_hashTableWords, sizeof(uint64), 1, F);
+ fwrite(&_bucketsWords, sizeof(uint64), 1, F);
+ fwrite(&_countsWords, sizeof(uint64), 1, F);
+
+ fwrite(_hashTable, sizeof(uint64), _hashTableWords, F);
+ fwrite(_buckets, sizeof(uint64), _bucketsWords, F);
+ fwrite(_counts, sizeof(uint64), _countsWords, F);
+
+ fclose(F);
+
+ if (errno) {
+ fprintf(stderr, "existDB::saveState()-- Write failure.\n%s\n", strerror(errno));
+ exit(1);
+ }
+}
+
+
+
+bool
+existDB::loadState(char const *filename,
+ bool beNoisy,
+ bool loadData) {
+ char cigam[16];
+
+ errno = 0;
+ FILE *F = fopen(filename, "rb");
+ if (errno) {
+ //fprintf(stderr, "Can't open '%s' for reading pre-built existDB\n%s\n", strerror(errno));
+ return(false);
+ }
+
+ fread(cigam, sizeof(char), 16, F);
+
+ _compressedHash = false;
+ _compressedBucket = false;
+ _compressedCounts = false;
+ _isForward = false;
+ _isCanonical = false;
+
+ if (cigam[8] == 'h')
+ _compressedHash = true;
+ if (cigam[9] == 'b')
+ _compressedBucket = true;
+ if (cigam[10] == 'c')
+ _compressedCounts = true;
+
+ if (cigam[11] == 'F')
+ _isForward = true;
+ if (cigam[11] == 'C')
+ _isCanonical = true;
+
+ cigam[ 8] = ' ';
+ cigam[ 9] = ' ';
+ cigam[10] = ' ';
+ cigam[11] = ' ';
+
+ if (strncmp(magic, cigam, 16) != 0) {
+ if (beNoisy) {
+ fprintf(stderr, "existDB::loadState()-- Not an existDB binary file, maybe a sequence file?\n");
+ fprintf(stderr, "existDB::loadState()-- Read '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
+ cigam[0], cigam[1], cigam[2], cigam[3],
+ cigam[4], cigam[5], cigam[6], cigam[7],
+ cigam[8], cigam[9], cigam[10], cigam[11],
+ cigam[12], cigam[13], cigam[14], cigam[15]);
+ fprintf(stderr, "existDB::loadState()-- Expected '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
+ magic[0], magic[1], magic[2], magic[3],
+ magic[4], magic[5], magic[6], magic[7],
+ magic[8], magic[9], magic[10], magic[11],
+ magic[12], magic[13], magic[14], magic[15]);
+ }
+
+ fclose(F);
+ return(false);
+ }
+
+ fread(&_merSizeInBases, sizeof(uint32), 1, F);
+ fread(&_shift1, sizeof(uint32), 1, F);
+ fread(&_shift2, sizeof(uint32), 1, F);
+ fread(&_mask1, sizeof(uint64), 1, F);
+ fread(&_mask2, sizeof(uint64), 1, F);
+ fread(&_hshWidth, sizeof(uint32), 1, F); // only valid if _compressedHash
+ fread(&_chkWidth, sizeof(uint32), 1, F); // only valid if _compressedBucket
+ fread(&_cntWidth, sizeof(uint32), 1, F); // only valid if _compressedCounts
+
+ fread(&_hashTableWords, sizeof(uint64), 1, F);
+ fread(&_bucketsWords, sizeof(uint64), 1, F);
+ fread(&_countsWords, sizeof(uint64), 1, F);
+
+ _hashTable = 0L;
+ _buckets = 0L;
+ _counts = 0L;
+
+ if (loadData) {
+ _hashTable = new uint64 [_hashTableWords];
+ _buckets = new uint64 [_bucketsWords];
+
+ if (_countsWords > 0)
+ _counts = new uint64 [_countsWords];
+
+ fread(_hashTable, sizeof(uint64), _hashTableWords, F);
+ fread(_buckets, sizeof(uint64), _bucketsWords, F);
+
+ if (_countsWords > 0)
+ fread(_counts, sizeof(uint64), _countsWords, F);
+ }
+
+ fclose(F);
+
+ if (errno) {
+ fprintf(stderr, "existDB::loadState()-- Read failure.\n%s\n", strerror(errno));
+ exit(1);
+ }
+
+ return(true);
+}
+
+
+void
+existDB::printState(FILE *stream) {
+
+ fprintf(stream, "merSizeInBases: "F_U32"\n", _merSizeInBases);
+ fprintf(stream, "tableBits "F_U32"\n", 2 * _merSizeInBases - _shift1);
+ fprintf(stream, "-----------------\n");
+ fprintf(stream, "_hashTableWords "F_U64" ("F_U64" KB)\n", _hashTableWords, _hashTableWords >> 7);
+ fprintf(stream, "_bucketsWords "F_U64" ("F_U64" KB)\n", _bucketsWords, _bucketsWords >> 7);
+ fprintf(stream, "_countsWords "F_U64" ("F_U64" KB)\n", _countsWords, _countsWords >> 7);
+ fprintf(stream, "-----------------\n");
+ fprintf(stream, "_shift1: "F_U32"\n", _shift1);
+ fprintf(stream, "_shift2 "F_U32"\n", _shift2);
+ fprintf(stream, "_mask1 "F_X64"\n", _mask1);
+ fprintf(stream, "_mask2 "F_X64"\n", _mask2);
+
+ if (_compressedHash) {
+ fprintf(stream, "_compressedHash true\n");
+ fprintf(stream, "_hshWidth "F_U32"\n", _hshWidth);
+ } else {
+ fprintf(stream, "_compressedHash false\n");
+ fprintf(stream, "_hshWidth undefined\n");
+ }
+
+ if (_compressedBucket) {
+ fprintf(stream, "_compressedBucket true\n");
+ fprintf(stream, "_chkWidth "F_U32"\n", _chkWidth);
+ } else {
+ fprintf(stream, "_compressedBucket false\n");
+ fprintf(stream, "_chkWidth undefined\n");
+ }
+
+ if (_compressedCounts) {
+ fprintf(stream, "_compressedCount true\n");
+ fprintf(stream, "_cntWidth "F_U32"\n", _cntWidth);
+ } else {
+ fprintf(stream, "_compressedCount false\n");
+ fprintf(stream, "_cntWidth undefined\n");
+ }
+}
+
diff --git a/src/meryl/libkmer/existDB.C b/src/meryl/libkmer/existDB.C
new file mode 100644
index 0000000..ad87273
--- /dev/null
+++ b/src/meryl/libkmer/existDB.C
@@ -0,0 +1,211 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2003-OCT-20
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-12 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAR-20 to 2014-APR-11
+ * are Copyright 2005-2008,2012-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "existDB.H"
+#include "AS_UTL_fileIO.H"
+
+
+existDB::existDB(char const *filename,
+ bool loadData) {
+ clear();
+
+ _compressedHash = false;
+ _compressedBucket = false;
+
+ if (loadState(filename, true, loadData) == false) {
+ fprintf(stderr, "existDB::existDB()-- Tried to read state from '%s', but failed.\n", filename);
+ exit(1);
+ }
+}
+
+
+existDB::existDB(char const *filename,
+ uint32 merSize,
+ existDBflags flags,
+ uint32 lo,
+ uint32 hi) {
+ clear();
+
+ _compressedHash = flags & existDBcompressHash;
+ _compressedBucket = flags & existDBcompressBuckets;
+ _compressedCounts = flags & existDBcompressCounts;
+
+ _searchForDupe = false;
+
+ // Try to read state from the filename. If successful, make sure
+ // that the merSize is correct.
+ //
+ if (loadState(filename)) {
+ bool fail = false;
+
+ if (_merSizeInBases != merSize) {
+ fprintf(stderr, "existDB::existDB()-- Read state from '%s', but got different mer sizes\n", filename);
+ fprintf(stderr, "existDB::existDB()-- Got "F_U32", expected "F_U32"\n", _merSizeInBases, merSize);
+ fail = true;
+ }
+
+ if (fail)
+ exit(1);
+
+ return;
+ }
+
+ // If no direction flags are set, set the default direction of
+ // forward. Stupid precedence rules.
+ //
+ if ((flags & (existDBcanonical | existDBforward)) == uint32ZERO)
+ flags |= existDBforward;
+
+ // If we can open 'filename' for reading, then we assume the file
+ // is a multi-fasta, and we build an existDB/
+ //
+ // Otherwise, we assume that 'filename' is really the prefix for a
+ // meryl database.
+
+
+ if (AS_UTL_fileExists(filename))
+ createFromFastA(filename, merSize, flags);
+ else
+ createFromMeryl(filename, merSize, lo, hi, flags);
+}
+
+
+existDB::existDB(char const *sequence,
+ uint32 merSize,
+ existDBflags flags) {
+ clear();
+
+ _compressedHash = flags & existDBcompressHash;
+ _compressedBucket = flags & existDBcompressBuckets;
+ _compressedCounts = flags & existDBcompressCounts;
+
+ if ((flags & (existDBcanonical | existDBforward)) == uint32ZERO)
+ flags |= existDBforward;
+
+ createFromSequence(sequence, merSize, flags);
+}
+
+
+existDB::~existDB() {
+ delete [] _hashTable;
+ delete [] _buckets;
+ delete [] _counts;
+}
+
+
+
+
+
+bool
+existDB::exists(uint64 mer) {
+ uint64 c, h, st, ed;
+
+ if (_compressedHash) {
+ h = HASH(mer) * _hshWidth;
+ st = getDecodedValue(_hashTable, h, _hshWidth);
+ ed = getDecodedValue(_hashTable, h + _hshWidth, _hshWidth);
+ } else {
+ h = HASH(mer);
+ st = _hashTable[h];
+ ed = _hashTable[h+1];
+ }
+
+ if (st == ed)
+ return(false);
+
+ c = CHECK(mer);
+
+ if (_compressedBucket) {
+ st *= _chkWidth;
+ ed *= _chkWidth;
+
+ for (; st<ed; st += _chkWidth) {
+ if (getDecodedValue(_buckets, st, _chkWidth) == c)
+ return(true);
+ }
+ } else {
+ for (; st<ed; st++) {
+ if (_buckets[st] == c)
+ return(true);
+ }
+ }
+
+ return(false);
+}
+
+
+uint64
+existDB::count(uint64 mer) {
+ uint64 c, h, st, ed;
+
+ if (_counts == 0L)
+ return(0);
+
+ if (_compressedHash) {
+ h = HASH(mer) * _hshWidth;
+ st = getDecodedValue(_hashTable, h, _hshWidth);
+ ed = getDecodedValue(_hashTable, h + _hshWidth, _hshWidth);
+ } else {
+ h = HASH(mer);
+ st = _hashTable[h];
+ ed = _hashTable[h+1];
+ }
+
+ if (st == ed)
+ return(0);
+
+ c = CHECK(mer);
+
+ if (_compressedBucket) {
+ st *= _chkWidth;
+ ed *= _chkWidth;
+
+ for (; st<ed; st += _chkWidth) {
+ if (getDecodedValue(_buckets, st, _chkWidth) == c)
+ goto returncount;
+ }
+ } else {
+ for (; st<ed; st++) {
+ if (_buckets[st] == c)
+ goto returncount;
+ }
+ }
+
+ return(0);
+
+ returncount:
+ if (_compressedCounts)
+ return(getDecodedValue(_counts, st * _cntWidth, _cntWidth));
+ else
+ return(_counts[st]);
+}
diff --git a/src/meryl/libkmer/existDB.H b/src/meryl/libkmer/existDB.H
new file mode 100644
index 0000000..c23e03a
--- /dev/null
+++ b/src/meryl/libkmer/existDB.H
@@ -0,0 +1,190 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2003-OCT-20
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAR-20 to 2014-APR-11
+ * are Copyright 2005,2007,2012-2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Liliana Florea on 2010-DEC-06
+ * are Copyright 2010 Liliana Florea, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#ifndef EXISTDB_H
+#define EXISTDB_H
+
+#include "AS_global.H"
+
+#include "bitPacking.H"
+
+// Used by wgs-assembler, to determine if a rather serious bug was patched.
+#define EXISTDB_H_VERSION 1960
+
+// Takes as input a list of mers (in a file) and builds a searchable
+// structure listing those mers. Duplicate mers are not removed and
+// will be stored multiple times.
+//
+// Using a compressed hash is allowed, but somewhat useless -- it is
+// really slow and doesn't save that much.
+//
+// If existDBcanonical is requested, this will store only the
+// canonical mer. It is up to the client to be sure that is
+// appropriate! See positionDB.H for more.
+
+//#define STATS
+
+typedef uint32 existDBflags;
+const existDBflags existDBnoFlags = 0x0000;
+const existDBflags existDBcompressHash = 0x0001;
+const existDBflags existDBcompressBuckets = 0x0002;
+const existDBflags existDBcompressCounts = 0x0004;
+const existDBflags existDBcanonical = 0x0008;
+const existDBflags existDBforward = 0x0010;
+const existDBflags existDBcounts = 0x0020;
+
+class existDB {
+public:
+
+ // Read state from an existDB file
+ existDB(char const *filename,
+ bool loadData=true);
+
+ // Load mers from an existing existDB file, a fastafile, or a meryl database
+ existDB(char const *filename,
+ uint32 merSize,
+ existDBflags flags,
+ uint32 lo,
+ uint32 hi);
+
+ // Load mers from a character string
+ existDB(char const *sequence,
+ uint32 merSize,
+ existDBflags flags);
+
+ ~existDB();
+
+ void saveState(char const *filename);
+
+ void printState(FILE *stream);
+
+ bool isForward(void) { return(_isForward); };
+ bool isCanonical(void) { return(_isCanonical); };
+
+ bool exists(uint64 mer);
+ uint64 count(uint64 mer);
+
+private:
+ bool loadState(char const *filename, bool beNoisy=false, bool loadData=true);
+ bool createFromFastA(char const *filename,
+ uint32 merSize,
+ uint32 flags);
+ bool createFromMeryl(char const *filename,
+ uint32 merSize,
+ uint32 lo,
+ uint32 hi,
+ uint32 flags);
+ bool createFromSequence(char const *sequence,
+ uint32 merSize,
+ uint32 flags);
+
+ uint64 HASH(uint64 k) {
+ return(((k >> _shift1) ^ (k >> _shift2) ^ k) & _mask1);
+ };
+
+ uint64 CHECK(uint64 k) {
+ return(k & _mask2);
+ };
+
+ void insertMer(uint64 hsh, uint64 chk, uint64 cnt, uint64 *countingTable) {
+
+ // If the mer is already here, just update the count. This only
+ // works if not _compressedBucket, and only makes sense for loading from
+ // fasta or sequence.
+
+ if ((_compressedBucket == false) &&
+ (_searchForDupe)) {
+ uint64 st = _hashTable[hsh];
+ uint64 ed = countingTable[hsh];
+
+ for (; st<ed; st++) {
+ if (_buckets[st] == chk) {
+ if (_counts)
+ _counts[st] += cnt;
+ return;
+ }
+ }
+ }
+
+ if (_compressedBucket)
+ setDecodedValue(_buckets, countingTable[hsh] * _chkWidth, _chkWidth, chk);
+ else
+ _buckets[countingTable[hsh]] = chk;
+
+ if (_counts) {
+ if (_compressedCounts) {
+ setDecodedValue(_counts, countingTable[hsh] * _cntWidth, _cntWidth, cnt);
+ } else {
+ _counts[countingTable[hsh]] = cnt;
+ }
+ }
+
+ countingTable[hsh]++;
+ };
+
+ bool _compressedHash;
+ bool _compressedBucket;
+ bool _compressedCounts;
+ bool _isForward;
+ bool _isCanonical;
+
+ bool _searchForDupe;
+
+ uint32 _merSizeInBases;
+
+ uint32 _shift1;
+ uint32 _shift2;
+ uint64 _mask1;
+ uint64 _mask2;
+
+ uint32 _hshWidth; // Only for the compressed hash
+ uint32 _chkWidth; // Only for the compressed bucket
+ uint32 _cntWidth; // Only for the compressed counts
+
+ uint64 _hashTableWords;
+ uint64 _bucketsWords;
+ uint64 _countsWords;
+
+ uint64 *_hashTable;
+ uint64 *_buckets;
+ uint64 *_counts;
+
+ void clear(void) {
+ };
+};
+
+#endif // EXISTDB_H
diff --git a/src/meryl/libkmer/existDB.mk b/src/meryl/libkmer/existDB.mk
new file mode 100644
index 0000000..0ceacb4
--- /dev/null
+++ b/src/meryl/libkmer/existDB.mk
@@ -0,0 +1,13 @@
+#CXXFLAGS := -fopenmp -D_GLIBCXX_PARALLEL -O3 -fPIC -m64 -pipe -Wno-write-strings
+#LDFLAGS := -fopenmp -lm
+
+TARGET := existDB
+SOURCES := driver-existDB.C
+
+SRC_INCDIRS := ../libutil ../libbio ../libseq ../libkmer ../libmeryl
+
+TGT_LDFLAGS := -L${TARGET_DIR}
+TGT_LDLIBS := -lkmer -lmeryl -lseq -lbio -lutil
+TGT_PREREQS := libkmer.a libmeryl.a libseq.a libbio.a libutil.a
+
+SUBMAKEFILES :=
diff --git a/src/meryl/libkmer/percentCovered.C b/src/meryl/libkmer/percentCovered.C
new file mode 100644
index 0000000..35ee2c7
--- /dev/null
+++ b/src/meryl/libkmer/percentCovered.C
@@ -0,0 +1,95 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2006-MAY-09 to 2014-APR-11
+ * are Copyright 2006-2008,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz on 2014-OCT-07
+ * are Copyright 2014 Battelle National Biodefense Institute, and
+ * are subject to the BSD 3-Clause License
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "util++.H"
+#include "bio++.H"
+#include "existDB.H"
+
+#include "seqCache.H"
+#include "seqStream.H"
+#include "merStream.H"
+
+int
+main(int argc, char **argv) {
+ char *merFile = 0L;
+ char *queryFile = 0L;
+
+ int arg=1;
+ while (arg < argc) {
+ if (strcmp(argv[arg], "-m") == 0) {
+ merFile = argv[++arg];
+ } else if (strcmp(argv[arg], "-q") == 0) {
+ queryFile = argv[++arg];
+ } else {
+ fprintf(stderr, "Unknown arg '%s'\n", argv[arg]);
+ }
+ arg++;
+ }
+
+ existDB *E = new existDB(merFile, 22, existDBnoFlags, 0, ~uint32ZERO);
+ seqCache *Q = new seqCache(queryFile);
+ seqInCore *S = Q->getSequenceInCore();
+
+ intervalList<uint64> IL;
+ speedCounter SC(" %8f frags (%8.5f frags/sec)\r", 1, 1000, true);
+
+ while (S) {
+ merStream *MS = new merStream(new kMerBuilder(22),
+ new seqStream(S->sequence(), S->sequenceLength()),
+ true, true);
+
+ IL.clear();
+
+ while (MS->nextMer()) {
+ if (E->exists(MS->theFMer())) {
+ IL.add(MS->thePositionInSequence(), 22);
+ }
+ }
+
+ IL.merge();
+
+ if (IL.sumOfLengths() > 0) {
+ fprintf(stdout, "%5.2f\n",
+ 100.0 * IL.sumOfLengths() / (double)S->sequenceLength());
+ }
+
+ delete MS;
+ delete S;
+
+ SC.tick();
+
+ S = Q->getSequenceInCore();
+ }
+
+ delete Q;
+ delete E;
+
+ return(0);
+}
+
diff --git a/src/meryl/libkmer/percentCovered.mk b/src/meryl/libkmer/percentCovered.mk
new file mode 100644
index 0000000..b8a59d8
--- /dev/null
+++ b/src/meryl/libkmer/percentCovered.mk
@@ -0,0 +1,13 @@
+#CXXFLAGS := -fopenmp -D_GLIBCXX_PARALLEL -O3 -fPIC -m64 -pipe -Wno-write-strings
+#LDFLAGS := -fopenmp -lm
+
+TARGET := percentCovered
+SOURCES := percentCovered.C
+
+SRC_INCDIRS := ../libutil ../libbio ../libseq ../libkmer ../libmeryl
+
+TGT_LDFLAGS := -L${TARGET_DIR}
+TGT_LDLIBS := -lkmer -lmeryl -lseq -lbio -lutil
+TGT_PREREQS := libkmer.a libmeryl.a libseq.a libbio.a libutil.a
+
+SUBMAKEFILES :=
diff --git a/src/meryl/libkmer/posDB.mk b/src/meryl/libkmer/posDB.mk
new file mode 100644
index 0000000..7dacdbd
--- /dev/null
+++ b/src/meryl/libkmer/posDB.mk
@@ -0,0 +1,13 @@
+#CXXFLAGS := -fopenmp -D_GLIBCXX_PARALLEL -O3 -fPIC -m64 -pipe -Wno-write-strings
+#LDFLAGS := -fopenmp -lm
+
+TARGET := posDB
+SOURCES := driver-posDB.C
+
+SRC_INCDIRS := ../libutil ../libbio ../libseq ../libmeryl
+
+TGT_LDFLAGS := -L${TARGET_DIR}
+TGT_LDLIBS := -lkmer -lmeryl -lseq -lbio -lutil
+TGT_PREREQS := libkmer.a libmeryl.a libseq.a libbio.a libutil.a
+
+SUBMAKEFILES :=
diff --git a/src/meryl/libkmer/positionDB-access.C b/src/meryl/libkmer/positionDB-access.C
new file mode 100644
index 0000000..1eb7a3c
--- /dev/null
+++ b/src/meryl/libkmer/positionDB-access.C
@@ -0,0 +1,376 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2003-AUG-14
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-FEB-07 to 2014-APR-11
+ * are Copyright 2005-2008,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+
+
+void
+positionDB::reallocateSpace(uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen,
+ uint64 len) {
+
+ if (posnMax < posnLen + len) {
+ uint64 *pp;
+
+ posnMax = posnLen + len + (len >> 2);
+
+ if (posnMax == 0)
+ posnMax = 16384;
+
+ try {
+ pp = new uint64 [posnMax];
+ } catch (...) {
+ fprintf(stderr, "positionDB::get()-- Can't allocate space for more positions, requested "F_U64" uint64's.\n", posnMax);
+ abort();
+ }
+
+ memcpy(pp, posn, sizeof(uint64) * posnLen);
+
+ delete [] posn;
+ posn = pp;
+ }
+}
+
+
+
+void
+positionDB::loadPositions(uint64 J,
+ uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen,
+ uint64& count) {
+
+ uint64 sizs[3] = {_pptrWidth, 1, _sizeWidth};
+ uint64 vals[3] = {0, 0, 1};
+
+ getDecodedValues(_buckets, J + _chckWidth, (_sizeWidth == 0) ? 2 : 3, sizs, vals);
+
+ // If the size is stored, the count is updated to the correct
+ // thing. If it's not stored, the count is set to 1 by the default
+ // value of vals[2], and reset after we get the number of positions
+ // stored.
+ //
+ count = vals[2];
+
+ if (vals[1]) {
+ reallocateSpace(posn, posnMax, posnLen, 64);
+ posn[posnLen++] = vals[0];
+ } else {
+ uint64 ptr = vals[0] * _posnWidth;
+ uint64 len = getDecodedValue(_positions, ptr, _posnWidth);
+
+ if (_sizeWidth == 0)
+ count = len;
+
+ reallocateSpace(posn, posnMax, posnLen, len + 64);
+
+ for (ptr += _posnWidth; len > 0; ptr += _posnWidth, len--)
+ posn[posnLen++] = getDecodedValue(_positions, ptr, _posnWidth);
+ }
+}
+
+
+
+bool
+positionDB::getExact(uint64 mer,
+ uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen,
+ uint64& count) {
+ uint64 h = HASH(mer);
+ uint64 c = CHECK(mer);
+ uint64 st, ed;
+
+ if (_hashTable_BP) {
+ st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
+ ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
+ } else {
+ st = _hashTable_FW[h];
+ ed = _hashTable_FW[h+1];
+ }
+
+ posnLen = 0;
+
+ if (st == ed)
+ return(false);
+
+ for (uint64 i=st, J=st * _wFin; i<ed; i++, J += _wFin) {
+ if (c == getDecodedValue(_buckets, J, _chckWidth)) {
+ loadPositions(J, posn, posnMax, posnLen, count);
+ return(true);
+ }
+ }
+
+ return(false);
+}
+
+
+bool
+positionDB::existsExact(uint64 mer) {
+ uint64 h = HASH(mer);
+ uint64 c = CHECK(mer);
+ uint64 st, ed;
+
+ if (_hashTable_BP) {
+ st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
+ ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
+ } else {
+ st = _hashTable_FW[h];
+ ed = _hashTable_FW[h+1];
+ }
+
+ if (st == ed)
+ return(false);
+
+ for (uint64 i=st, J=st * _wFin; i<ed; i++, J += _wFin)
+ if (c == getDecodedValue(_buckets, J, _chckWidth))
+ return(true);
+
+ return(false);
+}
+
+
+uint64
+positionDB::countExact(uint64 mer) {
+ uint64 h = HASH(mer);
+ uint64 c = CHECK(mer);
+ uint64 st, ed;
+
+ if (_hashTable_BP) {
+ st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
+ ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
+ } else {
+ st = _hashTable_FW[h];
+ ed = _hashTable_FW[h+1];
+ }
+
+ if (st == ed)
+ return(0);
+
+ for (uint64 i=st, J=st * _wFin; i<ed; i++, J += _wFin) {
+ if (c == getDecodedValue(_buckets, J, _chckWidth)) {
+ uint64 sizs[3] = {_pptrWidth, 1, _sizeWidth};
+ uint64 vals[3] = {0};
+
+ getDecodedValues(_buckets, J + _chckWidth, 3, sizs, vals);
+
+ if (_sizeWidth > 0)
+ return(vals[2]);
+
+ if (vals[1])
+ return(1);
+
+ return(getDecodedValue(_positions, vals[0] * _posnWidth, _posnWidth));
+ }
+ }
+
+ return(0);
+}
+
+
+uint64
+positionDB::setCount(uint64 mer, uint64 count) {
+ uint64 h = HASH(mer);
+ uint64 c = CHECK(mer);
+ uint64 st, ed;
+
+ if (_hashTable_BP) {
+ st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
+ ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
+ } else {
+ st = _hashTable_FW[h];
+ ed = _hashTable_FW[h+1];
+ }
+
+ if (st == ed)
+ return(0);
+
+ for (uint64 i=st, J=st * _wFin; i<ed; i++, J += _wFin)
+ if (c == getDecodedValue(_buckets, J, _chckWidth)) {
+ setDecodedValue(_buckets, J + _chckWidth + _pptrWidth + 1, _sizeWidth, count);
+ return(count);
+ }
+
+ return(0);
+}
+
+
+
+void
+positionDB::filter(uint64 lo,
+ uint64 hi) {
+ uint64 st=0, ed=0; // iteration through buckets
+ uint64 nb=0; // bit position of the current (read) bucket and next (write) bucket
+ uint64 np=0; // bit position of the current (read) position and next (write) position
+ uint64 vv;
+
+ uint64 loCount = 0;
+ uint64 okCount = 0;
+ uint64 hiCount = 0;
+
+ uint64 sizs[4] = {_chckWidth, _pptrWidth, 1, _sizeWidth};
+ uint64 vals[4] = {0, 0, 0, 0};
+
+ //dump("posDB.before");
+
+ fprintf(stderr, "positionDB::filter()-- Filtering out kmers less than "F_U64" and more than "F_U64"\n", lo, hi);
+
+ if (_sizeWidth == 0) {
+ // Single copy mers in a table without counts can be multi-copy
+ // when combined with their reverse-complement mer.
+ fprintf(stderr, "positionDB::filter()-- ERROR!\n");
+ fprintf(stderr, "positionDB::filter()-- ERROR! No count information; filtering will break canonical assumptions.\n");
+ fprintf(stderr, "positionDB::filter()-- ERROR!\n");
+ exit(1);
+ }
+
+ // Grab the start of the first (current) bucket. We reset the
+ // hashTable at the end of the loop, forcing us to keep st
+ // up-to-date, instead of grabbing it anew each iteration.
+ //
+ if (_hashTable_BP)
+ st = getDecodedValue(_hashTable_BP, 0, _hashWidth);
+ else
+ st = _hashTable_FW[0];
+
+ // Over all buckets
+ //
+ for (uint64 h=0; h<_tableSizeInEntries; h++) {
+
+ // Grab the end of this bucket - the end is always for the
+ // current structure. This gets reset at the end of the loop.
+ //
+ if (_hashTable_BP)
+ ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
+ else
+ ed = _hashTable_FW[h+1];
+
+ // Over all entries in the bucket
+ //
+ while (st < ed) {
+ uint64 cb = st * _wFin;
+
+ getDecodedValues(_buckets, cb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
+
+ // Argh. Tricky. We need to grab the count stored in the
+ // table, but if it's a single mer, there is no count.
+
+ uint64 count = 1; // Real count over the whole data set
+ uint64 len = 1; // Number of times the mer occurs in this subset
+ uint64 cp = ~uint64ZERO; // current position pointer, used to copy position information
+
+ // If not a unique mer in this table, len and cp are defined.
+ if (vals[2] == 0) {
+ cp = vals[1] * _posnWidth;
+ len = getDecodedValue(_positions, cp, _posnWidth);
+ count = len;
+ }
+
+ // The size stored in the bucket is to be believed
+ if (_sizeWidth > 0)
+ count = vals[3];
+
+ // What happened here: By default, the count is 1. If it is
+ // NOT a unique mer in the table, we reset the count to the
+ // number of entries in the table. Then, if there is a count
+ // stored in the table, we reset the count again.
+
+ // Move on to copying the data, if in the correct range.
+
+ if (vals[2] == 1) {
+ // Is a single mer in our table. Copy if the actual count is
+ // acceptable.
+ if ((lo <= count) && (count < hi)) {
+ okCount++;
+ setDecodedValues(_buckets, nb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
+ nb += _wFin;
+ } else {
+ _numberOfDistinct--;
+ _numberOfMers--;
+ loCount++;
+ }
+ } else {
+ // Mer has more than one location in the table. Copy all
+ // locations if the count is acceptable.
+ if ((lo <= count) && (count < hi)) {
+ okCount++;
+
+ // Copy the bucket
+ vals[1] = np / _posnWidth;
+ setDecodedValues(_buckets, nb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
+ nb += _wFin;
+
+ // Copy length of the positions
+ if (cp != np)
+ setDecodedValue(_positions, np, _posnWidth, len);
+ np += _posnWidth;
+ cp += _posnWidth;
+
+ // Copy positions
+ while (len > 0) {
+ if (cp != np)
+ setDecodedValue(_positions, np, _posnWidth,
+ getDecodedValue(_positions, cp, _posnWidth));
+ np += _posnWidth;
+ cp += _posnWidth;
+ len--;
+ }
+ } else {
+ // Not acceptable count
+ _numberOfDistinct--;
+ _numberOfEntries -= len;
+ if (count < lo) loCount++;
+ if (count > hi) hiCount++;
+ }
+ }
+
+ // Move to the next entry
+ st++;
+ cb += _wFin;
+ } // Over all entries in the bucket
+
+ // Update the end position of this bucket
+ if (_hashTable_BP)
+ setDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth, nb / _wFin);
+ else
+ _hashTable_FW[h+1] = nb / _wFin;
+
+ } // Over all buckets
+
+ fprintf(stderr, "positionDB::filter()-- Filtered "F_U64" kmers less than "F_U64"\n", loCount, lo);
+ fprintf(stderr, "positionDB::filter()-- Filtered "F_U64" kmers more than "F_U64"\n", hiCount, hi);
+ fprintf(stderr, "positionDB::filter()-- Saved "F_U64" kmers with acceptable count\n", okCount);
+
+ //dump("posDB.after");
+}
diff --git a/src/meryl/libkmer/positionDB-dump.C b/src/meryl/libkmer/positionDB-dump.C
new file mode 100644
index 0000000..b29c975
--- /dev/null
+++ b/src/meryl/libkmer/positionDB-dump.C
@@ -0,0 +1,79 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2003-AUG-14
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2007-DEC-11 to 2014-APR-11
+ * are Copyright 2007-2008,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+
+
+void
+positionDB::dump(char *name) {
+ uint64 sizs[4] = {_chckWidth, _pptrWidth, 1, _sizeWidth};
+ uint64 vals[4] = {0, 0, 0, 0};
+ FILE *F = fopen(name, "w");
+
+ for (uint64 h=0; h<_tableSizeInEntries; h++) {
+ uint64 st, ed;
+
+ if (_hashTable_BP) {
+ st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
+ ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
+ } else {
+ st = _hashTable_FW[h];
+ ed = _hashTable_FW[h+1];
+ }
+
+ fprintf(F, "B "F_U64" "F_U64"-"F_U64"\n", h, st, ed);
+
+ while (st < ed) {
+ uint64 cb = st * _wFin;
+
+ getDecodedValues(_buckets, cb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
+
+ fprintf(F, "%c chk="F_X64" pos="F_U64" siz="F_U64,
+ (vals[2] == 0) ? 'D' : 'U', vals[0], vals[1], vals[3]);
+
+ if (vals[2] == 0) {
+ uint64 pos = vals[1] * _posnWidth;
+ uint64 len = getDecodedValue(_positions, pos, _posnWidth);
+
+ for (pos += _posnWidth; len > 0; pos += _posnWidth, len--)
+ fprintf(F, " "F_U64, getDecodedValue(_positions, pos, _posnWidth));
+ }
+
+ fprintf(F, "\n");
+
+ st++;
+ }
+ }
+
+ fclose(F);
+}
diff --git a/src/meryl/libkmer/positionDB-file.C b/src/meryl/libkmer/positionDB-file.C
new file mode 100644
index 0000000..3e3009e
--- /dev/null
+++ b/src/meryl/libkmer/positionDB-file.C
@@ -0,0 +1,318 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-AUG-14 to 2004-APR-01
+ * are Copyright 2003-2004 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz on 2004-APR-30
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAR-12 to 2014-APR-11
+ * are Copyright 2005,2007-2008,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+
+static
+char magic[16] = { 'p', 'o', 's', 'i', 't', 'i', 'o', 'n', 'D', 'B', '.', 'v', '1', ' ', ' ', ' ' };
+static
+char faild[16] = { 'p', 'o', 's', 'i', 't', 'i', 'o', 'n', 'D', 'B', 'f', 'a', 'i', 'l', 'e', 'd' };
+
+
+
+
+
+// These came from kmer/libutil/file.c and need to be replaced with more standard functions
+
+
+// Split writes/reads into smaller pieces, check the result of each
+// piece. Really needed by OSF1 (V5.1).
+//
+void
+safeWrite(int filedes, const void *buffer, const char *desc, size_t nbytes) {
+ size_t position = 0;
+ size_t length = 32 * 1024 * 1024;
+ size_t towrite = 0;
+ size_t written = 0;
+
+ while (position < nbytes) {
+ towrite = length;
+ if (position + towrite > nbytes)
+ towrite = nbytes - position;
+
+ errno = 0;
+ written = write(filedes, ((char *)buffer) + position, towrite);
+
+ if ((errno) || (towrite != written)) {
+ fprintf(stderr, "safeWrite()-- Write failure on %s: %s\n", desc, strerror(errno));
+ fprintf(stderr, "safeWrite()-- Wanted to write "F_S64" bytes, wrote "F_S64".\n", (int64)towrite, (int64)written);
+ exit(1);
+ }
+
+ position += written;
+ }
+}
+
+int
+safeRead(int filedes, const void *buffer, const char *desc, size_t nbytes) {
+ size_t position = 0;
+ size_t length = 32 * 1024 * 1024;
+ size_t toread = 0;
+ size_t written = 0; // readen?
+ int failed = 0;
+
+ while (position < nbytes) {
+ toread = length;
+ if (position + toread > nbytes)
+ toread = nbytes - position;
+
+ errno = 0;
+ written = read(filedes, ((char *)buffer) + position, toread);
+
+ failed = errno;
+#ifdef VERY_SAFE
+ if (toread != written)
+ failed = 1;
+#endif
+
+ if ((failed) && (errno != EINTR)) {
+ fprintf(stderr, "safeRead()-- Read failure on %s: %s.\n", desc, strerror(errno));
+ fprintf(stderr, "safeRead()-- Wanted to read "F_S64" bytes, read "F_S64".\n", (int64)toread, (int64)written);
+ exit(1);
+ }
+
+ if (written == 0)
+ break;
+
+ position += written;
+ }
+
+ return(position);
+}
+
+
+
+
+
+
+
+
+void
+positionDB::saveState(char const *filename) {
+
+ fprintf(stderr, "Saving positionDB to '%s'\n", filename);
+
+ errno = 0;
+ int F = open(filename, O_RDWR | O_CREAT | O_LARGEFILE,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+ if (errno) {
+ fprintf(stderr, "Can't open '%s' for writing positionDB.\n%s\n", filename, strerror(errno));
+ exit(1);
+ }
+
+ bool magicFirst = false;
+
+ // Test if this is a pipe. If so, we write the magic first,
+ // otherwise we write the magic last.
+ //
+ errno = 0;
+ lseek(F, 0, SEEK_SET);
+ if (errno == ESPIPE)
+ magicFirst = true;
+
+ if (magicFirst)
+ write(F, magic, sizeof(char) * 16);
+ else
+ write(F, faild, sizeof(char) * 16);
+
+ if (errno) {
+ fprintf(stderr, "positionDB::saveState()-- Write failure on magic first.\n%s\n", strerror(errno));
+ exit(1);
+ }
+
+ // If only to be completely annoying and anal, we clear the
+ // pointers before we write the data. Sure, we could just write
+ // the stuff we care about, but this is easier. This is easier.
+ // Before you go rip out this stuff, remember that you can now
+ // checksum the resulting files. So don't do it.
+ //
+ uint32 *bs = _bucketSizes;
+ uint64 *cb = _countingBuckets;
+ uint64 *hp = _hashTable_BP;
+ uint32 *hw = _hashTable_FW;
+ uint64 *bu = _buckets;
+ uint64 *ps = _positions;
+ uint64 *he = _hashedErrors;
+
+ _bucketSizes = 0L;
+ _countingBuckets = 0L;
+ _hashTable_BP = (uint64 *)((_hashTable_BP) ? uint64ONE : uint64ZERO);
+ _hashTable_FW = (uint32 *)((_hashTable_FW) ? uint64ONE : uint64ZERO);
+ _buckets = 0L;
+ _positions = 0L;
+ _hashedErrors = 0L;
+
+ safeWrite(F, this, "this", sizeof(positionDB) * 1);
+
+ _bucketSizes = bs;
+ _countingBuckets = cb;
+ _hashTable_BP = hp;
+ _hashTable_FW = hw;
+ _buckets = bu;
+ _positions = ps;
+ _hashedErrors = he;
+
+ if (_hashTable_BP) {
+ safeWrite(F, _hashTable_BP, "_hashTable_BP", sizeof(uint64) * (_tableSizeInEntries * _hashWidth / 64 + 1));
+ } else {
+ safeWrite(F, _hashTable_FW, "_hashTable_FW", sizeof(uint32) * (_tableSizeInEntries + 1));
+ }
+
+ safeWrite(F, _buckets, "_buckets", sizeof(uint64) * (_numberOfDistinct * _wFin / 64 + 1));
+ safeWrite(F, _positions, "_positions", sizeof(uint64) * (_numberOfEntries * _posnWidth / 64 + 1));
+ safeWrite(F, _hashedErrors, "_hashedErrors", sizeof(uint64) * (_hashedErrorsLen));
+
+ if (magicFirst == false) {
+ lseek(F, 0, SEEK_SET);
+ if (errno) {
+ fprintf(stderr, "positionDB::saveState()-- Failed to seek to start of file -- write failed.\n%s\n", strerror(errno));
+ exit(1);
+ }
+
+ write(F, magic, sizeof(char) * 16);
+ if (errno) {
+ fprintf(stderr, "positionDB::saveState()-- Write failure on magic last.\n%s\n", strerror(errno));
+ exit(1);
+ }
+ }
+
+ close(F);
+}
+
+
+bool
+positionDB::loadState(char const *filename, bool beNoisy, bool loadData) {
+ char cigam[16] = { 0 };
+
+ fprintf(stderr, "Loading positionDB from '%s'\n", filename);
+
+ errno = 0;
+ int F = open(filename, O_RDONLY | O_LARGEFILE, 0);
+ if (errno) {
+ fprintf(stderr, "Can't open '%s' for reading pre-built positionDB: %s\n", filename, strerror(errno));
+ return(false);
+ }
+
+ safeRead(F, cigam, "Magic Number", sizeof(char) * 16);
+
+ if (strncmp(faild, cigam, 16) == 0) {
+ if (beNoisy) {
+ fprintf(stderr, "positionDB::loadState()-- Incomplete positionDB binary file.\n");
+ fprintf(stderr, "positionDB::loadState()-- Read '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
+ cigam[0], cigam[1], cigam[2], cigam[3],
+ cigam[4], cigam[5], cigam[6], cigam[7],
+ cigam[8], cigam[9], cigam[10], cigam[11],
+ cigam[12], cigam[13], cigam[14], cigam[15]);
+ fprintf(stderr, "positionDB::loadState()-- Expected '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
+ magic[0], magic[1], magic[2], magic[3],
+ magic[4], magic[5], magic[6], magic[7],
+ magic[8], magic[9], magic[10], magic[11],
+ magic[12], magic[13], magic[14], magic[15]);
+ }
+ close(F);
+ return(false);
+ } else if (strncmp(magic, cigam, 16) != 0) {
+ if (beNoisy) {
+ fprintf(stderr, "positionDB::loadState()-- Not a positionDB binary file, maybe a sequence file?\n");
+ fprintf(stderr, "positionDB::loadState()-- Read '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
+ cigam[0], cigam[1], cigam[2], cigam[3],
+ cigam[4], cigam[5], cigam[6], cigam[7],
+ cigam[8], cigam[9], cigam[10], cigam[11],
+ cigam[12], cigam[13], cigam[14], cigam[15]);
+ fprintf(stderr, "positionDB::loadState()-- Expected '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
+ magic[0], magic[1], magic[2], magic[3],
+ magic[4], magic[5], magic[6], magic[7],
+ magic[8], magic[9], magic[10], magic[11],
+ magic[12], magic[13], magic[14], magic[15]);
+ }
+
+ close(F);
+ return(false);
+ }
+
+ safeRead(F, this, "positionDB", sizeof(positionDB) * 1);
+
+ _bucketSizes = 0L;
+ _countingBuckets = 0L;
+ _buckets = 0L;
+ _positions = 0L;
+ _hashedErrors = 0L;
+
+ if (loadData) {
+ uint64 hs = _tableSizeInEntries * _hashWidth / 64 + 1;
+ uint64 bs = _numberOfDistinct * _wFin / 64 + 1;
+ uint64 ps = _numberOfEntries * _posnWidth / 64 + 1;
+
+ if (_hashTable_BP) {
+ _hashTable_BP = new uint64 [hs];
+ _hashTable_FW = 0L;
+ safeRead(F, _hashTable_BP, "_hashTable_BP", sizeof(uint64) * hs);
+ } else {
+ _hashTable_BP = 0L;
+ _hashTable_FW = new uint32 [_tableSizeInEntries + 1];
+ safeRead(F, _hashTable_FW, "_hashTable_FW", sizeof(uint32) * (_tableSizeInEntries + 1));
+ }
+
+ _buckets = new uint64 [bs];
+ _positions = new uint64 [ps];
+ _hashedErrors = new uint64 [_hashedErrorsMax];
+
+ safeRead(F, _buckets, "_buckets", sizeof(uint64) * bs);
+ safeRead(F, _positions, "_positions", sizeof(uint64) * ps);
+ safeRead(F, _hashedErrors, "_hashedErrors", sizeof(uint64) * _hashedErrorsLen);
+ }
+
+ close(F);
+
+ return(true);
+}
+
+
+
+void
+positionDB::printState(FILE *stream) {
+ fprintf(stream, "merSizeInBases: "F_U32"\n", _merSizeInBases);
+ fprintf(stream, "merSkipInBases: "F_U32"\n", _merSkipInBases);
+ fprintf(stream, "tableSizeInBits: "F_U32"\n", _tableSizeInBits);
+ fprintf(stream, "tableSizeInEntries: "F_U64"\n", _tableSizeInEntries);
+ fprintf(stream, "hashWidth: "F_U32"\n", _hashWidth);
+ fprintf(stream, "chckWidth: "F_U32"\n", _chckWidth);
+ fprintf(stream, "posnWidth: "F_U32"\n", _posnWidth);
+ fprintf(stream, "numberOfMers: "F_U64"\n", _numberOfMers);
+ fprintf(stream, "numberOfPositions: "F_U64"\n", _numberOfPositions);
+ fprintf(stream, "numberOfDistinct: "F_U64"\n", _numberOfDistinct);
+ fprintf(stream, "numberOfUnique: "F_U64"\n", _numberOfUnique);
+ fprintf(stream, "numberOfEntries: "F_U64"\n", _numberOfEntries);
+ fprintf(stream, "maximumEntries: "F_U64"\n", _maximumEntries);
+}
+
diff --git a/src/meryl/libkmer/positionDB-mismatch.C b/src/meryl/libkmer/positionDB-mismatch.C
new file mode 100644
index 0000000..9d34b5e
--- /dev/null
+++ b/src/meryl/libkmer/positionDB-mismatch.C
@@ -0,0 +1,412 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2007-NOV-11 to 2014-APR-11
+ * are Copyright 2007-2008,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+
+
+static
+int
+stringscmp(const void *A, const void *B) {
+ uint64 const a = *(uint64 const *)A;
+ uint64 const b = *(uint64 const *)B;
+ if (a < b) return(-1);
+ if (a > b) return(1);
+ return(0);
+}
+
+
+static
+uint32
+makeUnique(uint64 *strings, uint32 stringsLen) {
+ qsort(strings, stringsLen, sizeof(uint64), stringscmp);
+ uint32 len = 0;
+ uint32 nxt = 1;
+ while (nxt < stringsLen) {
+ if (strings[len] != strings[nxt]) {
+ len++;
+ strings[len] = strings[nxt];
+ }
+ nxt++;
+ }
+ return(len+1);
+}
+
+
+#if 0
+// debug
+static
+void
+dumpPatterns(uint64 *strings, uint32 stringsLen, uint32 ts) {
+ for (uint32 i=0; i<stringsLen; i++) {
+ char str[1024] = {0};
+ uint32 cnt = 0;
+
+ for (uint32 b=0; b<ts; b++) {
+ if (strings[i] & (uint64ONE << b)) {
+ str[b] = '1';
+ cnt++;
+ } else {
+ str[b] = '0';
+ }
+ }
+
+ fprintf(stdout, "%s\t"F_U32"\n", str, cnt);
+ }
+}
+#endif
+
+
+double
+positionDB::setUpMismatchMatcher(uint32 nErrorsAllowed, uint64 approxMers) {
+
+ // Build an xor mask that will generate all errors for a given
+ // mersize.
+
+ _nErrorsAllowed = nErrorsAllowed;
+ _hashedErrorsLen = 0;
+ _hashedErrorsMax = 0;
+ _hashedErrors = 0L;
+
+ uint32 stringsMax = 128 * 1024 * 1024;
+ uint32 stringsLen = 0;
+ uint64 *strings = new uint64 [stringsMax];
+
+ uint64 totpat = 0;
+ uint64 toterr = 0;
+
+ uint64 m1, m2, m3, m4, m5, m6;
+ uint64 *e1, *e2, *e3, *e4, *e5, *e6;
+
+ {
+ // This can be trivially eliminated by replacing e1[x] with the err[] statement.
+ uint32 ne = 3;
+ for (uint32 x=1; x<_nErrorsAllowed; x++)
+ ne *= 3;
+
+ //fprintf(stderr, "Storing ne="F_U32" errors.\n", ne);
+
+ e1 = new uint64 [ne];
+ e2 = new uint64 [ne];
+ e3 = new uint64 [ne];
+ e4 = new uint64 [ne];
+ e5 = new uint64 [ne];
+ e6 = new uint64 [ne];
+
+ uint64 err[3] = { 0x5555555555555555llu, 0xaaaaaaaaaaaaaaaallu, 0xffffffffffffffffllu };
+
+ for (uint32 x=0; x<ne; x++) {
+ e1[x] = err[(x/ 1) % 3];
+ e2[x] = err[(x/ 3) % 3];
+ e3[x] = err[(x/ 9) % 3];
+ e4[x] = err[(x/ 27) % 3];
+ e5[x] = err[(x/ 81) % 3];
+ e6[x] = err[(x/243) % 3];
+ }
+ }
+
+
+ // Zero errors
+ strings[stringsLen++] = uint64ZERO;
+
+
+ // One error
+ if (1 <= _nErrorsAllowed) {
+ for (uint32 ai=0; ai<_merSizeInBases; ai++) {
+ totpat++;
+ toterr += 3;
+ m1 = 0x03llu << (ai * 2);
+
+ for (uint32 x=0; x<3; x++)
+ strings[stringsLen++] = HASH((m1 & e1[x]));
+ }
+
+ stringsLen = makeUnique(strings, stringsLen);
+ stringsLen = makeUnique(strings, stringsLen);
+ //dumpPatterns(strings, stringsLen, _tableSizeInBits);
+ //fprintf(stderr, "DONE1 totpat="F_U64" toterr="F_U64" stringsLen="F_U32"\n", totpat, toterr, stringsLen);
+ }
+
+
+ // Two errors
+ if (2 <= _nErrorsAllowed) {
+ for (uint32 ai=0; ai<_merSizeInBases; ai++)
+ for (uint32 bi=0; bi<ai; bi++) {
+ totpat++;
+ toterr += 9;
+ m1 = 0x03llu << (ai * 2);
+ m2 = 0x03llu << (bi * 2);
+
+ for (uint32 x=0; x<9; x++)
+ strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]));
+ }
+
+ stringsLen = makeUnique(strings, stringsLen);
+ stringsLen = makeUnique(strings, stringsLen);
+ //dumpPatterns(strings, stringsLen, _tableSizeInBits);
+ //fprintf(stderr, "DONE2 totpat="F_U64" toterr="F_U64" stringsLen="F_U32"\n", totpat, toterr, stringsLen);
+ }
+
+
+ // Three errors
+ if (3 <= _nErrorsAllowed) {
+ for (uint32 ai=0; ai<_merSizeInBases; ai++)
+ for (uint32 bi=0; bi<ai; bi++)
+ for (uint32 ci=0; ci<bi; ci++) {
+ totpat++;
+ toterr += 27;
+ m1 = 0x03llu << (ai * 2);
+ m2 = 0x03llu << (bi * 2);
+ m3 = 0x03llu << (ci * 2);
+
+ for (uint32 x=0; x<27; x++)
+ strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]));
+ }
+
+ stringsLen = makeUnique(strings, stringsLen);
+ stringsLen = makeUnique(strings, stringsLen);
+ //dumpPatterns(strings, stringsLen, _tableSizeInBits);
+ //fprintf(stderr, "DONE3 totpat="F_U64" toterr="F_U64" stringsLen="F_U32"\n", totpat, toterr, stringsLen);
+ }
+
+
+ // Four errors
+ if (4 <= _nErrorsAllowed) {
+ for (uint32 ai=0; ai<_merSizeInBases; ai++)
+ for (uint32 bi=0; bi<ai; bi++)
+ for (uint32 ci=0; ci<bi; ci++)
+ for (uint32 di=0; di<ci; di++) {
+ totpat++;
+ toterr += 81;
+ m1 = 0x03llu << (ai * 2);
+ m2 = 0x03llu << (bi * 2);
+ m3 = 0x03llu << (ci * 2);
+ m4 = 0x03llu << (di * 2);
+
+ for (uint32 x=0; x<81; x++)
+ strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]) ^ (m4 & e4[x]));
+ }
+
+ stringsLen = makeUnique(strings, stringsLen);
+ stringsLen = makeUnique(strings, stringsLen);
+ //dumpPatterns(strings, stringsLen, _tableSizeInBits);
+ //fprintf(stderr, "DONE4 totpat="F_U64" toterr="F_U64" stringsLen="F_U32"\n", totpat, toterr, stringsLen);
+ }
+
+
+ // Five errors
+ if (5 <= _nErrorsAllowed) {
+ for (uint32 ai=0; ai<_merSizeInBases; ai++)
+ for (uint32 bi=0; bi<ai; bi++)
+ for (uint32 ci=0; ci<bi; ci++)
+ for (uint32 di=0; di<ci; di++)
+ for (uint32 ei=0; ei<di; ei++) {
+ totpat++;
+ toterr += 243;
+ m1 = 0x03llu << (ai * 2);
+ m2 = 0x03llu << (bi * 2);
+ m3 = 0x03llu << (ci * 2);
+ m4 = 0x03llu << (di * 2);
+ m5 = 0x03llu << (ei * 2);
+
+ if (stringsLen + 32000 >= stringsMax)
+ stringsLen = makeUnique(strings, stringsLen);
+
+ for (uint32 x=0; x<243; x++)
+ strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]) ^ (m4 & e4[x]) ^ (m5 & e5[x]));
+ }
+
+ stringsLen = makeUnique(strings, stringsLen);
+ stringsLen = makeUnique(strings, stringsLen);
+ //dumpPatterns(strings, stringsLen, _tableSizeInBits);
+ //fprintf(stderr, "DONE5 totpat="F_U64" toterr="F_U64" stringsLen="F_U32"\n", totpat, toterr, stringsLen);
+ }
+
+
+ // Six errors
+ if (6 <= _nErrorsAllowed) {
+ for (uint32 ai=0; ai<_merSizeInBases; ai++)
+ for (uint32 bi=0; bi<ai; bi++)
+ for (uint32 ci=0; ci<bi; ci++)
+ for (uint32 di=0; di<ci; di++)
+ for (uint32 ei=0; ei<di; ei++)
+ for (uint32 fi=0; fi<ei; fi++) {
+ totpat++;
+ toterr += 729;
+ m1 = 0x03llu << (ai * 2);
+ m2 = 0x03llu << (bi * 2);
+ m3 = 0x03llu << (ci * 2);
+ m4 = 0x03llu << (di * 2);
+ m5 = 0x03llu << (ei * 2);
+ m6 = 0x03llu << (fi * 2);
+
+ if (stringsLen + 32000 >= stringsMax)
+ stringsLen = makeUnique(strings, stringsLen);
+
+ for (uint32 x=0; x<729; x++)
+ strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]) ^ (m4 & e4[x]) ^ (m5 & e5[x]) ^ (m6 & e6[x]));
+ }
+
+ stringsLen = makeUnique(strings, stringsLen);
+ stringsLen = makeUnique(strings, stringsLen);
+ //dumpPatterns(strings, stringsLen, _tableSizeInBits);
+ //fprintf(stderr, "DONE6 totpat="F_U64" toterr="F_U64" stringsLen="F_U32"\n", totpat, toterr, stringsLen);
+ }
+
+
+ if (7 <= _nErrorsAllowed) {
+ fprintf(stderr, "Only 6 errors allowed.\n");
+ exit(1);
+ }
+
+ for (uint32 i=1; i<stringsLen; i++) {
+ assert((strings[i] & ~_hashMask) == 0);
+ assert(strings[i] != 0);
+ }
+
+ delete [] e1;
+ delete [] e2;
+ delete [] e3;
+ delete [] e4;
+ delete [] e5;
+ delete [] e6;
+
+ delete [] _hashedErrors;
+
+ _hashedErrorsLen = stringsLen;
+ _hashedErrorsMax = stringsLen;
+ _hashedErrors = new uint64 [_hashedErrorsLen];
+
+ memcpy(_hashedErrors, strings, sizeof(uint64) * _hashedErrorsLen);
+
+ delete [] strings;
+
+#ifdef UNCOMPRESS_HASH_TABLE
+ // Cost is just bucket searching.
+ double work = (double)_hashedErrorsLen * approxMers / _tableSizeInEntries;
+#else
+ // Cost is bucket searching + hash table lookups.
+ double work = (double)_hashedErrorsLen * approxMers / _tableSizeInEntries + 2.0 * _hashedErrorsLen;
+#endif
+
+ //fprintf(stderr, "Built "F_U32" hashed errors at tableSize "F_U32" (work=%f.0).\n",
+ // _hashedErrorsLen,
+ // _tableSizeInBits,
+ // work);
+
+ //for (uint32 i=0; i<_hashedErrorsLen; i++)
+ // fprintf(stderr, "he["F_U32W(5)"] = "uint64HEX"\n", i, _hashedErrors[i]);
+
+ return(work);
+}
+
+
+
+// Returns hits with _AT_MOST_ numMismatches mistakes.
+bool
+positionDB::getUpToNMismatches(uint64 mer,
+ uint32 numMismatches,
+ uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen) {
+
+ PREFETCH(_hashedErrors); // Slightly better.
+
+ posnLen = 0;
+
+ if (_hashedErrors == 0L) {
+ fprintf(stderr, "ERROR: Nobody initialized getUpToNMismatches() by calling setUpMismatchMatcher().\n");
+ exit(1);
+ }
+
+ if (posnMax == 0) {
+ posnMax = 16384;
+ try {
+ posn = new uint64 [posnMax];
+ } catch (...) {
+ fprintf(stderr, "positionDB::getUpToNMismatches()-- Can't allocate space for initial positions, requested "F_U64" uint64's.\n", posnMax);
+ abort();
+ }
+ }
+
+ uint64 orig = HASH(mer);
+
+ // Optimization that didn't work. The idea was to compute all the
+ // hashes with errors, then sort to gain better cache locality in
+ // the lookups. The sort dominated.
+ //
+ // Another: Surprisingly, theq two getDecodedValue calls are faster
+ // than a single getDecodedValues.
+
+ for (uint32 e=0; e<_hashedErrorsLen; e++) {
+ uint64 hash = orig ^ _hashedErrors[e];
+ uint64 st, ed;
+
+ if (_hashTable_BP) {
+ st = getDecodedValue(_hashTable_BP, hash * _hashWidth, _hashWidth);
+ ed = getDecodedValue(_hashTable_BP, hash * _hashWidth + _hashWidth, _hashWidth);
+ } else {
+ st = _hashTable_FW[hash];
+ ed = _hashTable_FW[hash+1];
+ }
+
+ assert((_hashedErrors[e] & ~_hashMask) == 0);
+ assert((hash & ~_hashMask) == 0);
+
+ // Rebuild the mer from the hash and its check code.
+ //
+ // Compare the rebuilt mer and the original mer -- if there are
+ // exactly N errors, it's a hit! (if there are fewer than N,
+ // we'll find it when we look for N-1 errors).
+ //
+ // Before rebuilding, compute diffs on the chckBits only -- if
+ // things are wildly different (the usual case) we'll get
+ // enough difference here to abort. Remember, the chck bits
+ // are not encoded, they're an exact copy from the unhashed
+ // mer.
+
+ if (st != ed) {
+ for (uint64 i=ed-st, J=st * _wFin; i--; J += _wFin) {
+ uint64 chck = getDecodedValue(_buckets, J, _chckWidth);
+ uint64 diffs = chck ^ (mer & _mask2);
+ uint64 d1 = diffs & uint64NUMBER(0x5555555555555555);
+ uint64 d2 = diffs & uint64NUMBER(0xaaaaaaaaaaaaaaaa);
+ uint64 err = countNumberOfSetBits64(d1 | (d2 >> 1));
+
+ if (err <= numMismatches) {
+ diffs = REBUILD(hash, chck) ^ mer;
+ d1 = diffs & uint64NUMBER(0x5555555555555555);
+ d2 = diffs & uint64NUMBER(0xaaaaaaaaaaaaaaaa);
+ err = countNumberOfSetBits64(d1 | (d2 >> 1));
+
+ if (err <= numMismatches)
+ // err is junk, just need a parameter here
+ loadPositions(J, posn, posnMax, posnLen, err);
+ }
+ }
+ }
+ }
+
+ return(posnLen > 0);
+}
diff --git a/src/meryl/libkmer/positionDB-sort.C b/src/meryl/libkmer/positionDB-sort.C
new file mode 100644
index 0000000..2adc55e
--- /dev/null
+++ b/src/meryl/libkmer/positionDB-sort.C
@@ -0,0 +1,182 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2003-MAY-06
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2006-JUL-07 to 2014-APR-11
+ * are Copyright 2006-2008,2011,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+
+
+void
+adjustHeap(uint64 *C,
+ uint64 *P, int64 i, int64 n) {
+ uint64 c = C[i];
+ uint64 p = P[i];
+ int64 j = (i << 1) + 1; // let j be the left child
+
+ while (j < n) {
+ if (j<n-1 && C[j] < C[j+1])
+ j++; // j is the larger child
+
+ if (c >= C[j]) // a position for M[i] has been found
+ break;
+
+ C[(j-1)/2] = C[j]; // Move larger child up a level
+ P[(j-1)/2] = P[j];
+
+ j = (j << 1) + 1;
+ }
+
+ C[(j-1)/2] = c;
+ P[(j-1)/2] = p;
+}
+
+
+void
+positionDB::sortAndRepackBucket(uint64 b) {
+ uint64 st = _bucketSizes[b];
+ uint64 ed = _bucketSizes[b+1];
+ uint32 le = (uint32)(ed - st);
+
+ if (ed < st)
+ fprintf(stdout, "ERROR: Bucket "F_U64" starts at "F_U64" ends at "F_U64"?\n", b, st, ed);
+
+ if (le == 0)
+ return;
+
+ // One mer in the list? It's distinct and unique! (and doesn't
+ // contribute to the position list space count)
+ //
+ if (le == 1) {
+ _numberOfDistinct++;
+ _numberOfUnique++;
+ return;
+ }
+
+ // Allocate more space, if we need to.
+ //
+ if (_sortedMax <= le) {
+ _sortedMax = le + 1024;
+ delete [] _sortedChck;
+ delete [] _sortedPosn;
+ _sortedChck = new uint64 [_sortedMax];
+ _sortedPosn = new uint64 [_sortedMax];
+ }
+
+ // Unpack the bucket
+ //
+ uint64 lens[3] = {_chckWidth, _posnWidth, 1 + _sizeWidth};
+ uint64 vals[3] = {0};
+ for (uint64 i=st, J=st * _wCnt; i<ed; i++, J += _wCnt) {
+ getDecodedValues(_countingBuckets, J, 2, lens, vals);
+ _sortedChck[i-st] = vals[0];
+ _sortedPosn[i-st] = vals[1];
+ }
+
+ // Create the heap of lines.
+ //
+ int unsetBucket = 0;
+
+ for (int64 t=(le-2)/2; t>=0; t--) {
+ if (_sortedPosn[t] == uint64MASK(_posnWidth)) {
+ unsetBucket = 1;
+ fprintf(stdout, "ERROR: unset posn bucket="F_U64" t="F_S64" le="F_U32"\n", b, t, le);
+ }
+
+ adjustHeap(_sortedChck, _sortedPosn, t, le);
+ }
+
+ if (unsetBucket)
+ for (uint32 t=0; t<le; t++)
+ fprintf(stdout, "%4"F_U32P"] chck="F_X64" posn="F_U64"\n", t, _sortedChck[t], _sortedPosn[t]);
+
+ // Interchange the new maximum with the element at the end of the tree
+ //
+ for (int64 t=le-1; t>0; t--) {
+ uint64 tc = _sortedChck[t];
+ uint64 tp = _sortedPosn[t];
+
+ _sortedChck[t] = _sortedChck[0];
+ _sortedPosn[t] = _sortedPosn[0];
+
+ _sortedChck[0] = tc;
+ _sortedPosn[0] = tp;
+
+ adjustHeap(_sortedChck, _sortedPosn, 0, t);
+ }
+
+ // Scan the list of sorted mers, counting the number of distinct and unique,
+ // and the space needed in the position list.
+
+ uint64 entries = 1; // For t=0
+
+ for (uint32 t=1; t<le; t++) {
+ if (_sortedChck[t-1] > _sortedChck[t])
+ fprintf(stdout, "ERROR: bucket="F_U64" t="F_U32" le="F_U32": "F_X64" > "F_X64"\n",
+ b, t, le, _sortedChck[t-1], _sortedChck[t]);
+
+ if (_sortedChck[t-1] != _sortedChck[t]) {
+ _numberOfDistinct++;
+
+ if (_maximumEntries < entries)
+ _maximumEntries = entries;
+
+ if (entries == 1)
+ _numberOfUnique++;
+ else
+ _numberOfEntries += entries + 1; // +1 for the length
+
+ entries = 0;
+ }
+
+ entries++;
+ }
+
+ // Don't forget the last mer!
+ //
+ _numberOfDistinct++;
+ if (_maximumEntries < entries)
+ _maximumEntries = entries;
+ if (entries == 1)
+ _numberOfUnique++;
+ else
+ _numberOfEntries += entries + 1;
+
+
+ // Repack the sorted entries
+ //
+ for (uint64 i=st, J=st * _wCnt; i<ed; i++, J += _wCnt) {
+ vals[0] = _sortedChck[i-st];
+ vals[1] = _sortedPosn[i-st];
+ vals[2] = 0;
+ setDecodedValues(_countingBuckets, J, 3, lens, vals);
+ }
+}
+
diff --git a/src/meryl/libkmer/positionDB.C b/src/meryl/libkmer/positionDB.C
new file mode 100644
index 0000000..69cf582
--- /dev/null
+++ b/src/meryl/libkmer/positionDB.C
@@ -0,0 +1,1186 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2004-APR-05
+ * are Copyright 2003-2004 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-JAN-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Clark Mobarry on 2004-FEB-12
+ * are Copyright 2004 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAY-19 to 2014-APR-11
+ * are Copyright 2005-2008,2012,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+#include "existDB.H"
+#include "../libmeryl.H"
+
+#include "speedCounter.H"
+
+#undef ERROR_CHECK_COUNTING
+#undef ERROR_CHECK_COUNTING_ENCODING
+#undef ERROR_CHECK_EMPTY_BUCKETS
+
+// This tests Chunlin Xiao's discovered bug -- if there are a small
+// number of unique mers, compared to distinct mers (2 * #unique_mers
+// < #distinct_mers, we would overflow the position pointer in
+// buckets. This enables a check that it doesn't occur.
+//
+// This has a fixed allocation size, and crashes on larger inputs.
+//
+#undef TEST_NASTY_BUGS
+
+// Tests that mers are masked out properly. Doesn't handle canonical
+// mers though.
+//
+#undef MER_REMOVAL_TEST
+
+
+
+
+uint64
+reverseComplementMer(uint32 _merSize, uint64 _md) { // This came from kMerTiny.H
+
+ // Reverse the mer
+
+ _md = ((_md >> 2) & 0x3333333333333333llu) | ((_md << 2) & 0xccccccccccccccccllu);
+ _md = ((_md >> 4) & 0x0f0f0f0f0f0f0f0fllu) | ((_md << 4) & 0xf0f0f0f0f0f0f0f0llu);
+ _md = ((_md >> 8) & 0x00ff00ff00ff00ffllu) | ((_md << 8) & 0xff00ff00ff00ff00llu);
+ _md = ((_md >> 16) & 0x0000ffff0000ffffllu) | ((_md << 16) & 0xffff0000ffff0000llu);
+ _md = ((_md >> 32) & 0x00000000ffffffffllu) | ((_md << 32) & 0xffffffff00000000llu);
+
+ // Complement the bases
+
+ _md ^= 0xffffffffffffffffllu;
+
+ // Shift and mask out the bases not in the mer
+
+ _md >>= 64 - _merSize * 2;
+ _md &= uint64MASK(_merSize * 2);
+
+ return(_md);
+}
+
+
+
+
+positionDB::positionDB(char const *filename,
+ uint32 merSize,
+ uint32 merSkip,
+ uint32 maxMismatch,
+ bool loadData) {
+ memset(this, 0, sizeof(positionDB));
+
+ // loadData == false only for driver-posDB.C, and only so it can
+ // dump stats on a posDB file.
+
+ if (loadState(filename, true, false) == false) {
+ fprintf(stderr, "positionDB()-- Tried to read state from '%s', but failed.\n", filename);
+ exit(1);
+ }
+
+ if ((loadData) && (merSize != _merSizeInBases)) {
+ fprintf(stderr, "positionDB()-- Tried to read state from '%s', but mer size is wrong (found "F_U32", wanted "F_U32").\n",
+ filename, _merSizeInBases, merSize);
+ exit(1);
+ }
+
+ if ((loadData) && (merSkip != _merSkipInBases)) {
+ fprintf(stderr, "positionDB()-- Tried to read state from '%s', but mer skip is wrong (found "F_U32", wanted "F_U32").\n",
+ filename, _merSkipInBases, merSkip);
+ exit(1);
+ }
+
+if ((loadData) && (maxMismatch != _nErrorsAllowed)) {
+ fprintf(stderr, "positionDB()-- Tried to read state from '%s', but max number of mismatches is wrong (found "F_U32", wanted "F_U32").\n",
+ filename, _nErrorsAllowed, maxMismatch);
+ exit(1);
+ }
+
+ if (loadState(filename, true, loadData) == false) {
+ fprintf(stderr, "positionDB()-- Tried to read state from '%s', but failed.\n", filename);
+ exit(1);
+ }
+}
+
+
+positionDB::positionDB(merStream *MS,
+ uint32 merSize,
+ uint32 merSkip,
+ existDB *mask,
+ existDB *only,
+ merylStreamReader *counts,
+ uint32 minCount,
+ uint32 maxCount,
+ uint32 maxMismatch,
+ uint32 maxMemory,
+ bool beVerbose) {
+
+ memset(this, 0, sizeof(positionDB));
+
+ // Guesstimate a nice table size based on the number of input mers
+ // and the mersize, unless the user gave us a table size.
+ //
+ // We need to ensure that
+ // 2 * merSize + posnWidth + 1 - 64 <= tblBits <= 2 * merSize - 4
+ //
+ // The catch is that we don't exactly know posnWidth right now. We
+ // can overestimate it, though, based on the size of the sequence
+ // that is backing the merStream.
+ //
+ // The second catch is that we don't want to make tblBits too big
+ // or too small. If too big, we waste a lot of memory in the hash
+ // table pointers, and if too small, we waste even more memory in
+ // the data table (not to mention the algorithm dies because it
+ // assumed buckets in the data table are small).
+ //
+ // The memory size is (roughly):
+ //
+ // 2^tblBits * log(numDistinctMers) +
+ // numDistinctMers * (2*merSize - tblBits + 1 + log(numMers) +
+ // (numMers - numUniqieMers) * log(numMers)
+ //
+ // this is approximately proportional to:
+ //
+ // 2^tblBits * posnWidth +
+ // approxMers * (2*merSize - tblBits + 1 + posnWidth)
+ //
+ uint64 approxMers = MS->approximateNumberOfMers();
+ uint64 posnWidth = logBaseTwo64(approxMers + 1);
+
+ // Find the smallest and largest tblBits we could possibly use.
+ //
+ uint64 sm = 2 * merSize + posnWidth + 1 - 64;
+ uint64 lg = 2 * merSize - 4;
+
+ if (2 * merSize + posnWidth + 1 < 64)
+ sm = 2;
+
+ if (sm < 16)
+ sm = 16;
+
+ if (sm > lg) {
+ fprintf(stderr, "ERROR: too many mers for this mersize!\n");
+ fprintf(stderr, " sm = "F_U64"\n", sm);
+ fprintf(stderr, " lg = "F_U64"\n", lg);
+ fprintf(stderr, " merSize = "F_U32" bits\n", 2 * merSize);
+ fprintf(stderr, " approxMers = "F_U64" mers\n", approxMers);
+ fprintf(stderr, " posnWidth = "F_U64" bits\n", posnWidth);
+ exit(1);
+ }
+
+
+ // Iterate through all the choices, picking the one with the
+ // smallest expected footprint.
+ //
+ {
+
+ if (beVerbose) {
+ fprintf(stderr, "potential configurations for approximately "F_U64" "F_U32"-mers (posnW="F_U64").\n",
+ approxMers, merSize, posnWidth);
+ }
+
+ uint64 mini = 0; // tblSize of the smallest found
+ uint64 minm = ~mini; // memory size of the smallest found
+ double minw = 0.0; // work of the smallest found
+
+ uint64 memory = 0;
+ double effort = 0;
+
+ if (maxMemory == 0)
+ maxMemory = ~uint32ZERO;
+
+ for (uint64 i=sm; i<=lg; i++) {
+
+ // These are only needed if maxMismatch is set, but it's
+ // simpler to always set.
+ //
+ _merSizeInBases = merSize;
+ _merSizeInBits = 2 * _merSizeInBases;
+ _merSkipInBases = merSkip;
+ _tableSizeInBits = i;
+ _tableSizeInEntries = uint64ONE << _tableSizeInBits;
+ _hashWidth = uint32ZERO;
+ _hashMask = uint64MASK(_tableSizeInBits);
+ _chckWidth = _merSizeInBits - _tableSizeInBits;
+ _posnWidth = uint64ZERO;
+ _sizeWidth = 0;
+
+ _shift1 = _merSizeInBits - _tableSizeInBits;
+ _shift2 = _shift1 / 2;
+ _mask1 = uint64MASK(_tableSizeInBits);
+ _mask2 = uint64MASK(_shift1);
+
+ // Everyone wants to know the memory size (in MB).
+ //
+ memory = ((uint64ONE << i) * posnWidth + approxMers * (2*merSize - i + 1 + posnWidth)) >> 23;
+
+ // If we know we're looking for mismatches, we compute the amount
+ // of work needed per lookup, and use that, instead of strict
+ // memory sizing, to deicde the table size.
+ //
+ if (maxMismatch > 0)
+ effort = setUpMismatchMatcher(maxMismatch, approxMers);
+
+ // If our memory size is smaller than allowed, AND it's the
+ // smallest, or the work is smaller, save the table size.
+ //
+ if ((memory < maxMemory) &&
+ ((memory < minm) ||
+ (effort < minw))) {
+ mini = i;
+ minm = memory;
+ minw = effort;
+ }
+
+ if (beVerbose) {
+ fprintf(stderr, "tblBits=%02lu shifts=%02u,%02u -- size %8.3fGB -- work %8.3f%s\n",
+ i, _shift1, _shift2, memory / 1024.0, effort, (mini == i) ? " ***" : "");
+ }
+ }
+
+ _tableSizeInBits = mini;
+ }
+
+
+ if (_tableSizeInBits == 0) {
+ fprintf(stderr, "ERROR: No positionDB parameters within allowed memory limit.\n");
+ exit(1);
+ }
+
+
+ if (beVerbose) {
+ uint32 s1 = 2*merSize-_tableSizeInBits;
+ fprintf(stderr, "tblBits="F_U32" s1="F_U32" s2="F_U32" -- merSize="F_U32" bits + posnWidth="F_U64" bits (est "F_U64" mers) FINAL\n",
+ _tableSizeInBits, s1, s1/2, merSize, posnWidth, approxMers);
+ }
+
+
+ _merSizeInBases = merSize;
+ _merSizeInBits = 2 * _merSizeInBases;
+ _merSkipInBases = merSkip;
+ _tableSizeInEntries = uint64ONE << _tableSizeInBits;
+ _hashWidth = uint32ZERO;
+ _hashMask = uint64MASK(_tableSizeInBits);
+ _chckWidth = _merSizeInBits - _tableSizeInBits;
+ _posnWidth = uint64ZERO;
+ _sizeWidth = 0;
+
+ if (maxCount == 0)
+ maxCount = ~uint32ZERO;
+
+ if (counts)
+ _sizeWidth = (maxCount < ~uint32ZERO) ? logBaseTwo64(maxCount+1) : 32;
+
+ _shift1 = _merSizeInBits - _tableSizeInBits;
+ _shift2 = _shift1 / 2;
+ _mask1 = uint64MASK(_tableSizeInBits);
+ _mask2 = uint64MASK(_shift1);
+
+#if 0
+ fprintf(stderr, "merSizeInBits "F_U32"\n", _merSizeInBits);
+ fprintf(stderr, "hashWidth "F_U32"\n", _hashWidth);
+ fprintf(stderr, "chckWidth "F_U32"\n", _chckWidth);
+ fprintf(stderr, "shift1 "F_U32"\n", _shift1);
+ fprintf(stderr, "shift2 "F_U32"\n", _shift2);
+#endif
+
+ if (maxMismatch > 0)
+ setUpMismatchMatcher(maxMismatch, approxMers);
+
+ build(MS, mask, only, counts, minCount, maxCount, beVerbose);
+}
+
+
+
+void
+positionDB::build(merStream *MS,
+ existDB *mask,
+ existDB *only,
+ merylStreamReader *counts,
+ uint32 minCount,
+ uint32 maxCount,
+ bool beVerbose) {
+
+ _bucketSizes = 0L;
+ _countingBuckets = 0L;
+ _hashTable_BP = 0L;
+ _hashTable_FW = 0L;
+ _buckets = 0L;
+ _positions = 0L;
+
+ _wCnt = 0;
+ _wFin = 0;
+
+ // For get/setDecodedValues().
+ uint64 lensC[4] = {~uint64ZERO, ~uint64ZERO, ~uint64ZERO, ~uint64ZERO};
+ uint64 lensF[4] = {~uint64ZERO, ~uint64ZERO, ~uint64ZERO, ~uint64ZERO};
+ uint64 vals[4] = {0};
+ uint64 nval = (_sizeWidth == 0) ? 3 : 4;
+
+ _numberOfMers = uint64ZERO;
+ _numberOfPositions = uint64ZERO;
+ _numberOfDistinct = uint64ZERO;
+ _numberOfUnique = uint64ZERO;
+ _numberOfEntries = uint64ZERO;
+ _maximumEntries = uint64ZERO;
+
+ // We assume later that these are already allocated.
+ _sortedMax = 16384;
+ _sortedChck = new uint64 [_sortedMax];
+ _sortedPosn = new uint64 [_sortedMax];
+
+ if (MS == 0L) {
+ fprintf(stderr, "positionDB()-- ERROR: No merStream? Nothing to build a table with!\n");
+ exit(1);
+ }
+
+ MS->rewind();
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 1) Count bucket sizes
+ //
+
+ // We'll later want to reuse the _bucketSizes space for storing the
+ // hash table. To make it somewhat safe, we allocate the space as
+ // uint64, then cast it to be uint32.
+ //
+ // bktAllocIsJunk tells us if we should release this memory (if we
+ // need to allocate separate space for the hash table). We'd need
+ // to do this if the hashWidth is more than 32 bits, but we won't
+ // know that for a little bit.
+ //
+ // The _bucketSizes is offset by one from bktAlloc so that we don't
+ // overwrite _bucketSizes when we are constructing hash table.
+ //
+ uint64 *bktAlloc;
+ try {
+ bktAlloc = new uint64 [_tableSizeInEntries / 2 + 4];
+ } catch (std::bad_alloc) {
+ fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
+ fprintf(stderr, "positionDB()-- bktAlloc = new uint64 ["F_U64"]\n", _tableSizeInEntries / 2 + 4);
+ exit(1);
+ }
+ bool bktAllocIsJunk = false;
+
+ bzero(bktAlloc, sizeof(uint64) * (_tableSizeInEntries / 2 + 4));
+
+ // Why +2? We try to reuse the bktAlloc space for the hash table,
+ // which is constructed from the bucketSizes. The hashTable is
+ // built from the bucketSizes. It definitely needs to be +1, and
+ // so we use +2 just in case the human is being stupid again.
+ //
+ _bucketSizes = (uint32 *)(bktAlloc + 2);
+
+#ifdef ERROR_CHECK_COUNTING
+ fprintf(stdout, "ERROR_CHECK_COUNTING is defined.\n");
+ uint32 *_errbucketSizes = new uint32 [_tableSizeInEntries + 2];
+ for (uint64 i=0; i<_tableSizeInEntries + 2; i++)
+ _errbucketSizes[i] = uint32ZERO;
+#endif
+
+ if (beVerbose)
+ fprintf(stderr, " Allocated bucket size counting space with total size "F_U64" KB\n", _tableSizeInEntries >> 8);
+
+
+ speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
+
+ // Two choices here
+ //
+ // 1) No masking or onlying is done. Stream the mers and just
+ // count the positions. This is the original behavior.
+ //
+ // 2) Masking or onlying is done. Open the output stream file,
+ // stream the mers by, checking for mask/only of both
+ // forward and reverse mers. If either is found, push
+ // the (forward) mer and position onto the stream.
+ // close the output stream.
+ //
+ // Save the mer if it doesn't exist in the mask (both f and r),
+ // or does exist in the only (either f or r), add it.
+ //
+ // The input databases for mask and only are (currently) made
+ // using canonical mers. We halve the number of exists() by
+ // also using canonical mers here.
+ //
+
+ MS->rewind();
+
+ while (MS->nextMer(_merSkipInBases)) {
+ _bucketSizes[ HASH(MS->theFMer()) ]++;
+
+#ifdef ERROR_CHECK_COUNTING
+ _errbucketSizes[ HASH(MS->theFMer()) ]++;
+#endif
+
+ _numberOfMers++;
+ _numberOfPositions = MS->thePositionInStream();
+ assert((_numberOfPositions >> 60) == 0);
+ C->tick();
+ }
+
+
+ delete C;
+ C = 0L;
+
+ if (beVerbose)
+ fprintf(stderr, " Found "F_U64" mers (max position = "F_U64")\n", _numberOfMers, _numberOfPositions);
+
+ // This caught a nasty bug in merStream rewind(), and it's pretty
+ // cheap, so I left it in. Search for the other DEBUGnumPositions.
+ //
+ uint64 DEBUGnumPositions = _numberOfPositions + 1;
+
+ // This is _numberOfMers+1 because we need to store the first
+ // position after the last mer. That is, if there are two mers, we
+ // will store that the first mer is at position 0, the second mer
+ // is at position 1, and the end of the second mer is at position
+ // 2.
+ //
+ // In reality, it should be the number of distinct mers, not the
+ // total number of mers, but we don't know that yet. And so
+ // occasionally we'll make things too big and waste a bit of
+ // memory.
+ //
+ _hashWidth = logBaseTwo64(_numberOfMers+1);
+ _posnWidth = logBaseTwo64(_numberOfPositions+1);
+
+
+
+ ///////////////////////////////////////////////////////////////////////////////
+ //
+ // 2) Allocate buckets and make bucketSizes be a pointer into them
+ //
+ _wCnt = _chckWidth + _posnWidth + 1 + _sizeWidth;
+
+ lensC[0] = _chckWidth;
+ lensC[1] = _posnWidth;
+ lensC[2] = 1;
+ lensC[3] = _sizeWidth;
+
+ uint64 bucketsSpace = (_numberOfMers+1) * _wCnt / 64 + 1;
+ uint32 endPosition = 0;
+
+ if (beVerbose)
+ fprintf(stderr, " Allocated "F_U64"KB for buckets ("F_U64" 64-bit words)\n", bucketsSpace >> 7, bucketsSpace);
+ try {
+ _countingBuckets = new uint64 [bucketsSpace];
+ } catch (std::bad_alloc) {
+ fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
+ fprintf(stderr, "positionDB()-- _countingBuckets = new uint64 ["F_U64"]\n", bucketsSpace);
+ exit(1);
+ }
+
+ for (uint64 i=0; i<bucketsSpace; i++)
+ _countingBuckets[i] = ~uint64ZERO;
+
+ for (uint64 i=0; i<_tableSizeInEntries; i++) {
+ endPosition += _bucketSizes[i];
+ _bucketSizes[i] = endPosition;
+ }
+ _bucketSizes[_tableSizeInEntries] = endPosition;
+
+#ifdef ERROR_CHECK_COUNTING
+ if (endPosition != _numberOfMers)
+ fprintf(stdout, "ERROR_CHECK_COUNTING: BUCKETSIZE COUNTING PROBLEM -- endPos="F_U32" != numMers="F_U64"\n",
+ endPosition, _numberOfMers);
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 3) Build list of mers with positions
+ //
+ if (beVerbose)
+ fprintf(stderr, " Building lists with positions.\n");
+
+ C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
+
+#ifdef ERROR_CHECK_COUNTING_ENCODING
+ fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING is defined!\n");
+#endif
+
+
+ MS->rewind();
+
+ while (MS->nextMer(_merSkipInBases)) {
+ uint64 h = HASH(MS->theFMer());
+
+#ifdef ERROR_CHECK_COUNTING
+ if (_bucketSizes[h] == 0) {
+ char str[33];
+ fprintf(stderr, "positionDB()-- ERROR_CHECK_COUNTING: Bucket "F_U64" ran out of things! '%s'\n", h, MS->theFMer().merToString(str));
+ fprintf(stderr, "positionDB()-- ERROR_CHECK_COUNTING: Stream is at "F_U64"\n", MS->thePositionInStream());
+ }
+#endif
+
+ _bucketSizes[h]--;
+
+#ifdef ERROR_CHECK_COUNTING
+ _errbucketSizes[h]--;
+#endif
+
+
+#ifdef ERROR_CHECK_EMPTY_BUCKETS
+ // Check that everything is empty. Empty is defined as set to all 1's.
+ getDecodedValues(_countingBuckets, (uint64)_bucketSizes[h] * (uint64)_wCnt, nval, lensC, vals);
+
+ if (((~vals[0]) & uint64MASK(lensC[0])) ||
+ ((~vals[1]) & uint64MASK(lensC[1])) ||
+ ((~vals[2]) & uint64MASK(lensC[2])) ||
+ ((lensC[3] > 0) && ((~vals[3]) & uint64MASK(lensC[3]))))
+ fprintf(stdout, "ERROR_CHECK_EMPTY_BUCKETS: countingBucket not empty! pos=%lu 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
+ _bucketSizes[h] * _wCnt,
+ (~vals[0]) & uint64MASK(lensC[0]),
+ (~vals[1]) & uint64MASK(lensC[1]),
+ (~vals[2]) & uint64MASK(lensC[2]),
+ (~vals[3]) & uint64MASK(lensC[3]));
+#endif
+
+ vals[0] = CHECK(MS->theFMer());
+ vals[1] = MS->thePositionInStream();
+ vals[2] = 0;
+ vals[3] = 0;
+
+ setDecodedValues(_countingBuckets, (uint64)_bucketSizes[h] * (uint64)_wCnt, nval, lensC, vals);
+
+#ifdef ERROR_CHECK_COUNTING_ENCODING
+ getDecodedValues(_countingBuckets, (uint64)_bucketSizes[h] * (uint64)_wCnt, nval, lensC, vals);
+
+ if (vals[0] != CHECK(MS->theFMer()))
+ fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: CHCK corrupted! Wanted "uint64HEX" got "uint64HEX"\n",
+ CHECK(MS->theFMer()), vals[0]);
+ if (vals[1] != MS->thePositionInStream())
+ fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: POSN corrupted! Wanted "uint64HEX" got "uint64HEX"\n",
+ MS->thePositionInStream(), vals[1]);
+ if (vals[2] != 0)
+ fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: UNIQ corrupted.\n");
+ if (vals[3] != 0)
+ fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: SIZE corrupted.\n");
+#endif
+
+ C->tick();
+ }
+
+
+ delete C;
+ C = 0L;
+
+#ifdef ERROR_CHECK_COUNTING
+ for (uint64 i=0; i<_tableSizeInEntries; i++)
+ if (_errbucketSizes[i] != 0)
+ fprintf(stdout, "ERROR_CHECK_COUNTING: Bucket "F_U32" wasn't filled fully? "F_U32" left over.\n", i, _errbucketSizes[i]);
+
+ delete [] _errbucketSizes;
+ _errbucketSizes = 0L;
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 4) Sort each bucket -- count:
+ // 1) number of distinct mers
+ // 2) number of unique mers
+ // 3) number of entries in position table ( sum mercount+1 for all mercounts > 1)
+ // also need to repack the sorted things
+ //
+ if (beVerbose)
+ fprintf(stderr, " Sorting and repacking buckets ("F_U64" buckets).\n", _tableSizeInEntries);
+
+ C = new speedCounter(" %7.2f Mbuckets -- %5.2f Mbuckets/second\r", 1000000.0, 0x1ffffff, beVerbose);
+ for (uint64 i=0; i<_tableSizeInEntries; i++) {
+ sortAndRepackBucket(i);
+ C->tick();
+ }
+ delete C;
+ C = 0L;
+
+ if (beVerbose)
+ fprintf(stderr,
+ " Found %12lu total mers\n"
+ " Found %12lu distinct mers\n"
+ " Found %12lu unique mers\n"
+ " Need "F_U64" non-unique position list entries ("F_U64" maximum count)\n",
+ _numberOfMers, _numberOfDistinct, _numberOfUnique, _numberOfEntries, _maximumEntries);
+
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Compute the size of the final bucket position entry. It's
+ // either a position into the sequence, or a pointer into a list of
+ // positions. In rare cases, the pointer is larger than the
+ // sequence position, and we need to do extra work.
+ //
+ // The width of position pointers (in buckets) is the max of
+ // _posnWidth (a pointer to the sequence position) and
+ // _pptrWidth (a pointer to an entry in the positions table).
+ //
+ _pptrWidth = logBaseTwo64(_numberOfEntries+1);
+ if (_pptrWidth < _posnWidth)
+ _pptrWidth = _posnWidth;
+
+ _wFin = _chckWidth + _pptrWidth + 1 + _sizeWidth;
+
+ lensF[0] = _chckWidth;
+ lensF[1] = _pptrWidth;
+ lensF[2] = 1;
+ lensF[3] = _sizeWidth;
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 5) Allocate: real hash table, buckets and position table.
+ //
+
+ // XXXX how do we count the number of buckets/positions we never
+ // use because they are masked out??
+ //
+ // If we are just thresholding (ignore things with count > 100)
+ // it's easy, a simple loop over something.
+ //
+ // If we have an exist/only db....are they in the same order? Can
+ // we loop over both at the same time and count that way? That'd
+ // be cool! Mersize is the same, why can the table size be the
+ // same too -- OK, if the existDB has a small number of mers in it,
+ // then we don't need a large table.
+
+ uint64 hs = _tableSizeInEntries * _hashWidth / 64 + 1;
+ uint64 bs = _numberOfDistinct * _wFin / 64 + 1;
+ uint64 ps = _numberOfEntries * _posnWidth / 64 + 1;
+
+ if (_hashWidth <= 32) {
+ if (beVerbose)
+ fprintf(stderr, " Reusing bucket counting space for hash table.\n");
+
+#ifdef UNCOMPRESS_HASH_TABLE
+ _hashTable_BP = 0L;
+ _hashTable_FW = (uint32 *)bktAlloc;
+#else
+ _hashTable_BP = bktAlloc;
+ _hashTable_FW = 0L;
+#endif
+
+ bktAllocIsJunk = false;
+ } else {
+
+ // Can't use the full-width hash table, since the data size is >
+ // 32 bits -- we'd need to allocate 64-bit ints for it, and
+ // that'll likely be too big...and we'd need to have
+ // _hashTable_FW64 or something.
+
+ if (beVerbose)
+ fprintf(stderr, " Allocated "F_U64"KB for hash table ("F_U64" 64-bit words)\n", hs >> 7, hs);
+ try {
+ _hashTable_BP = new uint64 [hs];
+ _hashTable_FW = 0L;
+ } catch (std::bad_alloc) {
+ fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
+ fprintf(stderr, "positionDB()-- _hashTable_BP = new uint64 ["F_U64"]\n", hs);
+ exit(1);
+ }
+ bktAllocIsJunk = true;
+ }
+
+
+ // If we have enough space to reuse the counting space, reuse it.
+ // Else, allocate more space.
+ //
+ // We need to ensure that there are enough bits and that the size
+ // of a bucket didn't increase. If the bucket size did increase,
+ // and we see more unique buckets than total mers (up to some
+ // point) we overwrite data.
+ //
+ // Recall that bucketSpace ~= numberOfMers * wCnt
+ //
+ if ((bs < bucketsSpace) && (_wFin <= _wCnt)) {
+ if (beVerbose)
+ fprintf(stderr, " Reusing bucket space; Have: "F_U64" Need: "F_U64" (64-bit words)\n", bucketsSpace, bs);
+
+ _buckets = _countingBuckets;
+
+ bs = bucketsSpace; // for output at the end
+ } else {
+ if (beVerbose)
+ fprintf(stderr, " Allocated "F_U64"KB for buckets ("F_U64" 64-bit words)\n", bs >> 7, bs);
+ try {
+ _buckets = new uint64 [bs];
+ } catch (std::bad_alloc) {
+ fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
+ fprintf(stderr, "positionDB()-- _buckets = new uint64 ["F_U64"]\n", bs);
+ exit(1);
+ }
+ }
+
+ if (beVerbose)
+ fprintf(stderr, " Allocated "F_U64"KB for positions ("F_U64" 64-bit words)\n", ps >> 7, ps);
+ try {
+ _positions = new uint64 [ps];
+ } catch (std::bad_alloc) {
+ fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
+ fprintf(stderr, "positionDB()-- _positions = new uint64 ["F_U64"\n", ps);
+ exit(1);
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // 6) Transfer from the sorted buckets to the hash table.
+ //
+ if (beVerbose)
+ fprintf(stderr, " Transferring to final structure ("F_U64" buckets).\n", _tableSizeInEntries);
+
+ uint64 bucketStartPosition = 0;
+
+ // Current positions and bit positions in the buckets and position list.
+ //
+ uint64 currentBbit = uint64ZERO; // Bit position into bucket
+ uint64 currentPbit = uint64ZERO; // Bit position into positions
+ uint64 currentPpos = uint64ZERO; // Value position into positions
+
+#ifdef TEST_NASTY_BUGS
+ // Save the position array pointer of each bucket for debugging.
+ //
+ uint64 currentBpos = uint64ZERO; // Value position into bucket
+ uint32 *posPtrCheck = new uint32 [65826038];
+#endif
+
+ // We also take this opportunity to reset some statistics that are
+ // wrong.
+ //
+ _numberOfMers = 0;
+ _numberOfPositions = 0;
+ _numberOfDistinct = 0;
+ _numberOfUnique = 0;
+ _numberOfEntries = 0;
+ _maximumEntries = 0;
+
+ C = new speedCounter(" %7.2f Mbuckets -- %5.2f Mbuckets/second\r", 1000000.0, 0x1ffffff, beVerbose);
+
+ // We need b outside the loop!
+ //
+ uint64 b;
+ for (b=0; b<_tableSizeInEntries; b++) {
+ C->tick();
+
+ // Set the start of the bucket -- we took pains to ensure that
+ // we don't overwrite _bucketSizes[b], if we are reusing that
+ // space for the hash table.
+ //
+ if (_hashTable_BP)
+ setDecodedValue(_hashTable_BP, (uint64)b * (uint64)_hashWidth, _hashWidth, bucketStartPosition);
+ else
+ _hashTable_FW[b] = bucketStartPosition;
+
+ // Get the number of mers in the counting bucket. The error
+ // checking and sizing of _sortedChck and _sortedPosn was already
+ // done in the sort.
+ //
+ uint64 st = _bucketSizes[b];
+ uint64 ed = _bucketSizes[b+1];
+ uint32 le = ed - st;
+
+ // Unpack the check values
+ //
+ for (uint64 i=st, J=st * _wCnt; i<ed; i++, J += _wCnt) {
+ getDecodedValues(_countingBuckets, J, 2, lensC, vals);
+ _sortedChck[i-st] = vals[0];
+ _sortedPosn[i-st] = vals[1];
+ }
+
+
+ // Walk through the counting bucket, adding things to the real
+ // bucket as we see them. Mers with more than one position are
+ // inserted into the bucket, and the positions inserted into the
+ // position list.
+
+ // start and end locations of the mer. For mers with only
+ // one occurrance (unique mers), stM+1 == edM.
+ //
+ uint32 stM = uint32ZERO;
+ uint32 edM = uint32ZERO;
+
+ while (stM < le) {
+
+ // Move to the next mer.
+ //
+ edM++;
+
+ // Keep moving while the two mers are the same.
+ //
+ while ((edM < le) && (_sortedChck[stM] == _sortedChck[edM]))
+ edM++;
+
+ // edM is now the mer after the last. Write all mers from stM
+ // up to edM to the final structure. If there is one mer, put
+ // it in the bucket. If not, put a pointer to the position
+ // array there.
+
+ // We're in bucket b, looking at mer _sortedChck[stM]. Ask the
+ // only/mask if that exists, if so do/do not include the mer.
+ //
+ bool useMer = true;
+
+ if (edM - stM < minCount)
+ useMer = false;
+
+ if (edM - stM > maxCount)
+ useMer = false;
+
+ if ((useMer == true) && (mask || only)) {
+
+ // MER_REMOVAL_DURING_XFER. Great. The existDB has
+ // (usually) the canonical mer. We have the forward mer.
+ // Well, no, we have the forward mers' hash and check. So,
+ // we reconstruct the mer, reverse complement it, and then
+ // throw the mer out if either the forward or reverse exists
+ // (or doesn't exist).
+
+ uint64 m = REBUILD(b, _sortedChck[stM]);
+ uint64 r;
+
+ if (mask) {
+ if (mask->isCanonical()) {
+ r = reverseComplementMer(_merSizeInBases, m);
+ if (r < m)
+ m = r;
+ }
+ if (mask->exists(m))
+ useMer = false;
+ }
+
+ if (only) {
+ if (only->isCanonical()) {
+ r = reverseComplementMer(_merSizeInBases, m);
+ if (r < m)
+ m = r;
+ }
+ if (only->exists(m) == false)
+ useMer = false;
+ }
+ }
+
+ if (useMer) {
+ _numberOfMers += edM - stM;
+ _numberOfPositions += edM - stM;
+ _numberOfDistinct++;
+
+ if (stM+1 == edM) {
+ _numberOfUnique++;
+
+#ifdef TEST_NASTY_BUGS
+ posPtrCheck[currentBpos++] = _sortedPosn[stM];
+#endif
+
+ vals[0] = _sortedChck[stM];
+ vals[1] = _sortedPosn[stM];
+ vals[2] = 1;
+ vals[3] = 0;
+
+ currentBbit = setDecodedValues(_buckets, currentBbit, nval, lensF, vals);
+ bucketStartPosition++;
+ } else {
+ _numberOfEntries += edM - stM;
+ if (_maximumEntries < edM - stM)
+ _maximumEntries = edM - stM;
+
+#ifdef TEST_NASTY_BUGS
+ posPtrCheck[currentBpos++] = currentPpos;
+#endif
+
+ vals[0] = _sortedChck[stM];
+ vals[1] = currentPpos;
+ vals[2] = 0;
+ vals[3] = 0;
+
+ currentBbit = setDecodedValues(_buckets, currentBbit, nval, lensF, vals);
+ bucketStartPosition++;
+
+ // Store the positions. Store the number of positions
+ // here, then store all positions.
+ //
+ // The positions are in the proper place in _sortedPosn,
+ // and setDecodedValue masks out the extra crap, so no
+ // temporary needed. Probably should be done with
+ // setDecodedValues, but then we need another array telling
+ // the sizes of each piece.
+ //
+ setDecodedValue(_positions, currentPbit, _posnWidth, edM - stM);
+ currentPbit += _posnWidth;
+ currentPpos++;
+
+ for (; stM < edM; stM++) {
+ if (_sortedPosn[stM] >= DEBUGnumPositions) {
+ fprintf(stderr, "positionDB()-- ERROR: Got position "F_U64", but only "F_U64" available!\n",
+ _sortedPosn[stM], DEBUGnumPositions);
+ abort();
+ }
+ setDecodedValue(_positions, currentPbit, _posnWidth, _sortedPosn[stM]);
+ currentPbit += _posnWidth;
+ currentPpos++;
+ }
+ }
+ } // useMer
+
+ // All done with this mer.
+ //
+ stM = edM;
+ } // while (stM < le)
+ } // for each bucket
+
+ // Set the end of the last bucket
+ //
+ if (_hashTable_BP)
+ setDecodedValue(_hashTable_BP, b * _hashWidth, _hashWidth, bucketStartPosition);
+ else
+ _hashTable_FW[b] = bucketStartPosition;
+
+ delete C;
+
+ // Clear out the end of the arrays -- this is only so that we can
+ // checksum the result.
+ //
+ if (_hashTable_BP) {
+ b = b * _hashWidth + _hashWidth;
+ setDecodedValue(_hashTable_BP, b, 64 - (b % 64), uint64ZERO);
+ }
+ setDecodedValue(_buckets, currentBbit, 64 - (currentBbit % 64), uint64ZERO);
+ setDecodedValue(_positions, currentPbit, 64 - (currentPbit % 64), uint64ZERO);
+
+
+ if (beVerbose) {
+ fprintf(stderr, " Avail: Bucket %12lu Position %12lu (64-bit words)\n", bs, ps);
+ fprintf(stderr, " Avail: Bucket %12lu Position %12lu (entries)\n", _numberOfDistinct, _numberOfEntries);
+ fprintf(stderr, " Used: Bucket %12lu Position %12lu (64-bit words)\n", currentBbit / 64, currentPbit / 64);
+ }
+
+ // Reset the sizes to what we actually found. If we then
+ // dump/reload, we shrink our footprint.
+ //
+ _numberOfDistinct = currentBbit / _wFin;
+ _numberOfEntries = currentPbit / _posnWidth;
+
+ if (beVerbose) {
+ fprintf(stderr, " Used: Bucket %12lu Position %12lu (entries)\n", _numberOfDistinct, _numberOfEntries);
+ fprintf(stderr,
+ " Found %12lu total mers\n"
+ " Found %12lu distinct mers\n"
+ " Found %12lu unique mers\n"
+ " Need "F_U64" non-unique position list entries ("F_U64" maximum count)\n",
+ _numberOfMers, _numberOfDistinct, _numberOfUnique, _numberOfEntries, _maximumEntries);
+ }
+
+
+ // If we removed mers, there is a small chance that our hash table
+ // is too big -- we might have removed enoough mers to make the
+ // width smaller. If so, rebuild the hash table.
+ //
+ // Also, hooray, we finally know the number of distinct mers, so we
+ // can make this nice and tight
+ //
+ if (_hashTable_BP) {
+ uint32 newHashWidth = 1;
+ while ((_numberOfDistinct+1) > (uint64ONE << newHashWidth))
+ newHashWidth++;
+
+ if (newHashWidth != _hashWidth) {
+ uint64 npos = 0;
+ uint64 opos = 0;
+
+ if (beVerbose)
+ fprintf(stderr, " Rebuilding the hash table, from "F_U32" bits wide to "F_U32" bits wide.\n",
+ _hashWidth, newHashWidth);
+
+ for (uint64 z=0; z<_tableSizeInEntries+1; z++) {
+ setDecodedValue(_hashTable_BP,
+ npos,
+ newHashWidth,
+ getDecodedValue(_hashTable_BP, opos, _hashWidth));
+ npos += newHashWidth;
+ opos += _hashWidth;
+ }
+
+ // Clear the end again.
+ setDecodedValue(_hashTable_BP, npos, 64 - (npos % 64), uint64ZERO);
+ }
+
+ _hashWidth = newHashWidth;
+ }
+
+
+ // If supplied, add in any counts. The meryl table is, sadly, in
+ // the wrong order, and we must hash and search.
+ //
+ // Meryl _should_ be storing only forward mers, but we have no way
+ // of checking.
+ //
+ // After all counts are loaded, check if we can compress the counts
+ // space any. Check if the largestMerylCount is much smaller than
+ // the space it is stored in. If so, we can compress the table.
+ //
+ uint64 largestMerylCount = 0;
+ uint64 countsLoaded = 0;
+
+ if (counts) {
+ if (beVerbose)
+ fprintf(stderr, " Loading "F_U64" mercounts.\n", counts->numberOfDistinctMers());
+
+ C = new speedCounter(" %7.2f Mmercounts -- %5.2f Mmercounts/second\r", 1000000.0, 0x1fffff, beVerbose);
+
+ while (counts->nextMer()) {
+ kMer k = counts->theFMer();
+ uint64 c = counts->theCount();
+ uint64 f = setCount(k, c);
+ k.reverseComplement();
+ uint64 r = setCount(k, c);
+
+ if (f + r > 0) {
+ countsLoaded++;
+ if (largestMerylCount < c)
+ largestMerylCount = c;
+ }
+
+ C->tick();
+ }
+
+ delete C;
+
+ if (beVerbose)
+ fprintf(stderr, " Loaded "F_U64" mercounts; largest is "F_U64".\n", countsLoaded, largestMerylCount);
+
+ if (logBaseTwo64(largestMerylCount + 1) < _sizeWidth) {
+ if (beVerbose)
+ fprintf(stderr, " Compress sizes from "F_U32" bits to "F_U32" bits.\n",
+ _sizeWidth,
+ (uint32)logBaseTwo64(largestMerylCount + 1));
+
+ uint64 oSiz[4] = { _chckWidth, _pptrWidth, 1, _sizeWidth };
+ uint64 nSiz[4] = { _chckWidth, _pptrWidth, 1, logBaseTwo64(largestMerylCount + 1) };
+ uint64 tVal[4] = { 0, 0, 0, 0 };
+
+ uint64 oP = 0, oS = oSiz[0] + oSiz[1] + oSiz[2] + oSiz[3];
+ uint64 nP = 0, nS = nSiz[0] + nSiz[1] + nSiz[2] + nSiz[3];
+
+ assert(nS < oS);
+
+ C = new speedCounter(" %7.2f Mmercounts -- %5.2f Mmercounts/second\r", 1000000.0, 0x1fffff, beVerbose);
+
+ for (uint64 bu=0; bu<_numberOfDistinct; bu++) {
+ getDecodedValues(_buckets, oP, 4, oSiz, tVal);
+ setDecodedValues(_buckets, nP, 4, nSiz, tVal);
+
+ oP += oS;
+ nP += nS;
+
+ C->tick();
+ }
+
+ delete C;
+
+ _sizeWidth = nSiz[3];
+ _wFin = _chckWidth + _pptrWidth + 1 + _sizeWidth;
+ }
+ }
+
+
+#ifdef TEST_NASTY_BUGS
+ // Unpack the bucket positions and check. Report the first one
+ // that is broken.
+ //
+ for(uint64 bb=0; bb<currentBpos; bb++)
+ if (posPtrCheck[bb] != getDecodedValue(_buckets, bb * _wFin + _chckWidth, _pptrWidth))
+ fprintf(stderr, "Bucket %lu (at bitpos %lu) failed position check (wanted %lu got %lu)\n",
+ bb,
+ bb * _wFin,
+ posPtrCheck[bb],
+ getDecodedValue(_buckets, bb * _wFin + _chckWidth, _pptrWidth));
+ delete [] posPtrCheck;
+#endif
+
+
+#ifdef MER_REMOVAL_TEST
+#warning MER_REMOVAL_TEST was not updated to deal with canonical mers
+ if (beVerbose)
+ fprintf(stderr, "positionDB()-- TESTING MER REMOVAL\n");
+
+ MS->rewind();
+ if (mask) {
+ C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
+ uint32 extraMer = 0;
+ while (MS->nextMer(_merSkipInBases)) {
+ uint64 mer = MS->theFMer();
+ if (mask->exists(mer) && exists(mer))
+ extraMer++;
+ C->tick();
+ }
+ delete C;
+ fprintf(stderr, "positionDB()-- mask: "F_U32" mers extra!\n", extraMer);
+ } else if (only) {
+ C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
+ uint32 missingMer = 0;
+ while (MS->nextMer(_merSkipInBases)) {
+ uint64 mer = MS->theFMer();
+ if (only->exists(mer) && !exists(mer))
+ missingMer++;
+ C->tick();
+ }
+ delete C;
+ fprintf(stderr, "positionDB()-- only: "F_U32" mers missing!\n", missingMer);
+ }
+#endif
+
+ // Free the counting buckets if we aren't using the space for
+ // something else.
+ //
+ if (_buckets != _countingBuckets)
+ delete [] _countingBuckets;
+
+ // In theory, we could move these to be immediately after the data
+ // is useless.
+ //
+ _bucketSizes = 0L;
+ _countingBuckets = 0L;
+
+ delete [] _sortedChck;
+ delete [] _sortedPosn;
+
+ _sortedMax = 0;
+ _sortedChck = 0L;
+ _sortedPosn = 0L;
+
+ if (bktAllocIsJunk)
+ delete [] bktAlloc;
+}
+
+positionDB::~positionDB() {
+ delete [] _hashTable_BP;
+ delete [] _hashTable_FW;
+ delete [] _buckets;
+ delete [] _positions;
+ delete [] _hashedErrors;
+}
diff --git a/src/meryl/libkmer/positionDB.H b/src/meryl/libkmer/positionDB.H
new file mode 100644
index 0000000..b3e7120
--- /dev/null
+++ b/src/meryl/libkmer/positionDB.H
@@ -0,0 +1,273 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-JAN-02 to 2003-OCT-21
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-SEP-13 to 2014-APR-11
+ * are Copyright 2005-2008,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#ifndef POSITIONDB_H
+#define POSITIONDB_H
+
+#include "AS_global.H"
+#include "merStream.H"
+
+// The two existDB inputs can be either forward or canonical. If
+// canonical, we are smart enough to search exist/only with the
+// canonical mer.
+
+// Returns position in posn, resizing it if needed. Space is
+// allocated if none supplied. The following is valid:
+//
+// uint64 *posn = 0L;
+// uint64 posnMax = 0;
+// uint64 posnLen = 0;
+// if (get(somemer, posn, posnMax, posnLen)) {
+// do something with the positions
+// }
+//
+// exists() returns T/F if mer exists or not
+// count() returns the number of times that mer is present
+
+// Define this to use an uncompressed hash table when the width is 32
+// bits or less. Doing so is A LOT faster in mismatch lookups, but
+// does use more memory.
+#undef UNCOMPRESS_HASH_TABLE
+
+// Define this to leave out references to getTime(), speedCounter()
+// and make the positionDB build very quietly.
+#undef SILENTPOSITIONDB
+
+// Define these to enable some debugging methods
+#undef DEBUGPOSDB
+#undef DEBUGREBUILD
+
+class existDB;
+class merylStreamReader;
+
+class positionDB {
+public:
+ positionDB(char const *filename,
+ uint32 merSize,
+ uint32 merSkip,
+ uint32 maxMismatch,
+ bool loadData=true);
+
+ positionDB(merStream *MS,
+ uint32 merSize,
+ uint32 merSkip,
+ existDB *mask,
+ existDB *only,
+ merylStreamReader *counts,
+ uint32 minCount,
+ uint32 maxCount,
+ uint32 maxMismatch,
+ uint32 maxMemory,
+ bool beVerbose);
+
+ ~positionDB();
+
+private:
+ void build(merStream *MS,
+ existDB *mask,
+ existDB *only,
+ merylStreamReader *counts,
+ uint32 minCount,
+ uint32 maxCount,
+ bool beVerbose);
+
+private:
+ void reallocateSpace(uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen,
+ uint64 len);
+
+ void loadPositions(uint64 v,
+ uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen,
+ uint64& count);
+
+public:
+ bool getExact(uint64 mer,
+ uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen,
+ uint64& count);
+ bool existsExact(uint64 mer);
+ uint64 countExact(uint64 mer);
+
+public:
+ void filter(uint64 lo, uint64 hi);
+
+private:
+ double setUpMismatchMatcher(uint32 nErrorsAllowed, uint64 approxMers);
+public:
+ bool getUpToNMismatches(uint64 mer,
+ uint32 maxMismatches,
+ uint64*& posn,
+ uint64& posnMax,
+ uint64& posnLen);
+private:
+ uint64 setCount(uint64 mer, uint64 count);
+
+ // Save or load a built table
+ //
+public:
+ void saveState(char const *filename);
+ bool loadState(char const *filename, bool beNoisy=false, bool loadData=true);
+
+ void printState(FILE *stream);
+
+ // Only really useful for debugging. Don't use.
+ //
+ void dump(char *name);
+
+
+ bool checkREBUILD(uint64 m) {
+#define DEBUGREBUILD
+#ifdef DEBUGREBUILD
+ uint64 h = HASH(m);
+ uint64 c = CHECK(m);
+ uint64 r = REBUILD(h, c);
+ if (r != m) {
+ fprintf(stderr, "shift1 = "F_U32"\n", _shift1);
+ fprintf(stderr, "shift2 = "F_U32"\n", _shift2);
+ fprintf(stderr, "M = "F_X64"\n", m);
+ fprintf(stderr, "H = "F_X64"\n", h);
+ fprintf(stderr, "C = "F_X64"\n", c);
+ fprintf(stderr, "R = "F_X64"\n", r);
+ return(false);
+ }
+ return(true);
+#else
+ return(REBUILD(HASH(m), CHECK(m)) == m);
+#endif
+ };
+
+private:
+
+ uint64 HASH(uint64 k) {
+ return(((k >> _shift1) ^ (k >> _shift2) ^ k) & _mask1);
+ };
+
+ uint64 CHECK(uint64 k) {
+ return(k & _mask2);
+ };
+
+ uint64 REBUILD(uint64 h, uint64 c) {
+ // Decode a HASH and a CHECK to get back the mer. You'd better
+ // bloody PRAY you don't break this (test/test-rebuild.C). It
+ // was a headache++ to write.
+
+ uint64 sha = _shift1 - _shift2;
+ uint64 msk = uint64MASK(sha);
+
+ // The check is exactly the mer....just not all there.
+ uint64 mer = c;
+
+ uint64 shf = sha - (_tableSizeInBits % 2);
+ uint64 shg = 0;
+ uint64 shh = _shift1;
+
+ // Unrolling this is troublesome - we still need the tests,
+ // bizarre merSize, tblSize combinations use lots of iterations
+ // (when the merSize and tblSize are about the same, the CHECK is
+ // small, and so we need to do lots of iterations).
+
+ //fprintf(stderr, "shf="F_U64W(2)" shg="F_U64W(2)" shh="F_U64W(2)" mer="F_X64"\n", shf, shg, shh, mer);
+
+ do {
+ mer |= (((h >> shg) ^ (mer >> shg) ^ (mer >> shf)) & msk) << shh;
+ //fprintf(stderr, "shf="F_U64W(2)" shg="F_U64W(2)" shh="F_U64W(2)" mer="F_X64"\n", shf, shg, shh, mer);
+
+ shf += sha;
+ shg += sha;
+ shh += sha;
+ } while ((shf < _merSizeInBits) && (shh < 64));
+
+ mer &= uint64MASK(_merSizeInBits);
+
+ return(mer);
+ };
+
+ void sortAndRepackBucket(uint64 b);
+
+ uint32 *_bucketSizes;
+ uint64 *_countingBuckets;
+ uint64 *_hashTable_BP; // Bit packed
+ uint32 *_hashTable_FW; // Full width
+ uint64 *_buckets;
+
+ uint64 *_positions;
+
+ uint32 _merSizeInBases;
+ uint32 _merSizeInBits;
+
+ uint32 _merSkipInBases;
+
+ uint64 _tableSizeInEntries;
+ uint32 _tableSizeInBits;
+
+ uint32 _hashWidth; // Hash bith
+ uint32 _chckWidth; // Check bits
+ uint32 _posnWidth; // Positions in the sequence
+ uint32 _pptrWidth; // Pointers to positions
+ uint32 _sizeWidth; // Extra number in the table
+
+ uint64 _hashMask;
+
+ uint32 _wCnt;
+ uint32 _wFin;
+
+ uint32 _shift1;
+ uint32 _shift2;
+ uint64 _mask1;
+ uint64 _mask2;
+
+ uint64 _numberOfMers;
+ uint64 _numberOfPositions;
+ uint64 _numberOfDistinct;
+ uint64 _numberOfUnique;
+ uint64 _numberOfEntries;
+ uint64 _maximumEntries;
+
+ // For sorting the mers
+ //
+ uint32 _sortedMax;
+ uint64 *_sortedChck;
+ uint64 *_sortedPosn;
+
+ // For the mismatch matcher
+ uint32 _nErrorsAllowed;
+ uint32 _hashedErrorsLen;
+ uint32 _hashedErrorsMax;
+ uint64 *_hashedErrors;
+};
+
+#endif // POSITIONDB_H
diff --git a/src/meryl/maskMers.C b/src/meryl/maskMers.C
index afca643..2d5c5fa 100644
--- a/src/meryl/maskMers.C
+++ b/src/meryl/maskMers.C
@@ -27,6 +27,10 @@
* are Copyright 2014 Battelle National Biodefense Institute, and
* are subject to the BSD 3-Clause License
*
+ * Brian P. Walenz beginning on 2016-NOV-22
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
* File 'README.licenses' in the root directory of this distribution contains
* full conditions and disclaimers for each license.
*/
diff --git a/src/meryl/meryl-build.C b/src/meryl/meryl-build.C
index 3d67120..cb9195e 100644
--- a/src/meryl/meryl-build.C
+++ b/src/meryl/meryl-build.C
@@ -711,6 +711,26 @@ build(merylArgs *args) {
if (!args->countBatch && !args->mergeBatch)
prepareBatch(args);
+ // Since we write no output until after all the work is done, check that
+ // we can actually make output files before starting the work.
+
+ char N[FILENAME_MAX];
+
+ sprintf(N, "%s.existenceTest", args->outputFile);
+
+ if (AS_UTL_fileExists(args->outputFile, true) == true)
+ fprintf(stderr, "ERROR: output prefix -o cannot be a directory.\n"), exit(1);
+
+ if (AS_UTL_fileExists(N) == false) {
+ errno = 0;
+ FILE *F = fopen(N, "w");
+ if (errno)
+ fprintf(stderr, "ERROR: can't make outputs with prefix '%s': %s\n", args->outputFile, strerror(errno)), exit(1);
+
+ AS_UTL_unlink(N);
+ }
+
+
// Three choices:
//
// threaded -- start threads, launch pieces in each thread. This
diff --git a/src/meryl/positionDB.C b/src/meryl/positionDB.C
new file mode 100644
index 0000000..2fd4742
--- /dev/null
+++ b/src/meryl/positionDB.C
@@ -0,0 +1,308 @@
+
+/******************************************************************************
+ *
+ * This file is part of canu, a software program that assembles whole-genome
+ * sequencing reads into contigs.
+ *
+ * This software is based on:
+ * 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ * the 'kmer package' (http://kmer.sourceforge.net)
+ * both originally distributed by Applera Corporation under the GNU General
+ * Public License, version 2.
+ *
+ * Canu branched from Celera Assembler at its revision 4587.
+ * Canu branched from the kmer project at its revision 1994.
+ *
+ * Modifications by:
+ *
+ * Brian P. Walenz from 2003-AUG-14 to 2003-SEP-18
+ * are Copyright 2003 Applera Corporation, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2004-APR-30 to 2004-OCT-10
+ * are Copyright 2004 Brian P. Walenz, and
+ * are subject to the GNU General Public License version 2
+ *
+ * Brian P. Walenz from 2005-MAY-19 to 2014-APR-11
+ * are Copyright 2005,2007-2008,2011,2014 J. Craig Venter Institute, and
+ * are subject to the GNU General Public License version 2
+ *
+ * File 'README.licenses' in the root directory of this distribution contains
+ * full conditions and disclaimers for each license.
+ */
+
+#include "positionDB.H"
+#include "existDB.H"
+
+// Driver for the positionDB creation. Reads a sequence.fasta, builds
+// a positionDB for the mers in the file, and then writes the internal
+// structures to disk.
+//
+// The positionDB constructor is smart enough to read either a pre-built
+// image or a regular multi-fasta file.
+
+#define MERSIZE 20
+
+int
+test1(char *filename) {
+ merStream *T = new merStream(new kMerBuilder(MERSIZE), new seqStream(filename), true, true);
+ positionDB *M = new positionDB(T, MERSIZE, 0, 0L, 0L, 0L, 0, 0, 0, 0, true);
+ uint64 *posn = new uint64 [1024];
+ uint64 posnMax = 1024;
+ uint64 posnLen = uint64ZERO;
+ uint64 count = uint64ZERO;
+ uint32 missing = uint32ZERO;
+ uint32 failed = uint32ZERO;
+ char str[33];
+
+ T->rewind();
+
+ while (T->nextMer()) {
+ if (M->getExact(T->theFMer(),
+ posn,
+ posnMax,
+ posnLen,
+ count)) {
+
+ missing = uint32ZERO;
+ for (uint32 i=0; i<posnLen; i++)
+ if (posn[i] == T->thePositionInStream())
+ missing++;
+
+ if (missing != 1) {
+ failed++;
+
+ fprintf(stdout, "%s @ "F_U64"/"F_U64": Found "F_U64" table entries, and "F_U32" matching positions (",
+ T->theFMer().merToString(str), T->theSequenceNumber(), T->thePositionInStream(), posnLen, missing);
+
+ for (uint32 i=0; i<posnLen; i++) {
+ fprintf(stdout, F_U64, posn[i]);
+ if (i < posnLen - 1)
+ fprintf(stdout, " ");
+ else
+ fprintf(stdout, ")\n");
+ }
+ }
+ } else {
+ failed++;
+
+ fprintf(stdout, "Found no matches for mer=%s at pos="F_U64"\n",
+ T->theFMer().merToString(str), T->thePositionInStream());
+ }
+ }
+
+ delete M;
+ delete T;
+
+ return(failed != 0);
+}
+
+
+
+int
+test2(char *filename, char *query) {
+ merStream *T = new merStream(new kMerBuilder(MERSIZE), new seqStream(filename), true, true);
+ positionDB *M = new positionDB(T, MERSIZE, 0, 0L, 0L, 0L, 0, 0, 0, 0, true);
+ uint64 *posn = new uint64 [1024];
+ uint64 posnMax = 1024;
+ uint64 posnLen = uint64ZERO;
+ uint64 count = uint64ZERO;
+ char str[33];
+
+ delete T;
+
+ T = new merStream(new kMerBuilder(MERSIZE), new seqStream(query), true, true);
+
+ while (T->nextMer()) {
+ if (M->getExact(T->theFMer(),
+ posn,
+ posnMax,
+ posnLen,
+ count)) {
+ fprintf(stdout, "Got a F match for mer=%s at "F_U64"/"F_U64" (in mers), numMatches="F_U64"\n",
+ T->theFMer().merToString(str), T->theSequenceNumber(), T->thePositionInStream(), posnLen);
+ }
+
+ if (M->getExact(T->theRMer(),
+ posn,
+ posnMax,
+ posnLen,
+ count)) {
+ fprintf(stdout, "Got a R match for mer=%s at "F_U64"/"F_U64" (in mers), numMatches="F_U64"\n",
+ T->theRMer().merToString(str), T->theSequenceNumber(), T->thePositionInStream(), posnLen);
+ }
+ }
+
+ delete M;
+ delete T;
+
+ return(0);
+}
+
+
+
+// Builds a positionDB possibly using a subset of the file.
+//
+// Subset on entire sequences:
+// -use x-y,a,b
+//
+// Subset on a range of mers, in this case, use only the 1000th
+// through 1999th (inclusive) mer:
+// -merbegin 1000 -merend 2000
+//
+// Or do both, use the first 1000 mers from the 3rd sequence:
+// -use 3 -merbegin 0 -merend 1000
+
+
+
+int
+main(int argc, char **argv) {
+ uint32 mersize = 20;
+ uint32 merskip = 0;
+
+ char *maskF = 0L;
+ char *onlyF = 0L;
+
+ uint64 merBegin = ~uint64ZERO;
+ uint64 merEnd = ~uint64ZERO;
+
+ char *sequenceFile = 0L;
+ char *outputFile = 0L;
+
+ if (argc < 3) {
+ fprintf(stderr, "usage: %s [args]\n", argv[0]);
+ fprintf(stderr, " -mersize k The size of the mers, default=20.\n");
+ fprintf(stderr, " -merskip k The skip between mers, default=0\n");
+ fprintf(stderr, " -use a-b,c Specify which sequences to use, default=all\n");
+ fprintf(stderr, " -merbegin b Build on a subset of the mers, starting at mer #b, default=all mers\n");
+ fprintf(stderr, " -merend e Build on a subset of the mers, ending at mer #e, default=all mers\n");
+ fprintf(stderr, " -sequence s.fasta Input sequences.\n");
+ fprintf(stderr, " -output p.posDB Output filename.\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " To dump information about an image:\n");
+ fprintf(stderr, " -dump datafile\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " To run sanity tests:\n");
+ fprintf(stderr, " -buildonly [build opts] sequence.fasta\n");
+ fprintf(stderr, " -- just builds a table and exits\n");
+ fprintf(stderr, " -existence [build opts] sequence.fasta\n");
+ fprintf(stderr, " -- builds (or reads) a table reports if any mers\n");
+ fprintf(stderr, " in sequence.fasta cannot be found\n");
+ fprintf(stderr, " -extra [build opts] sequence.fasta\n");
+ fprintf(stderr, " -- builds (or reads) a table reports if any mers\n");
+ fprintf(stderr, " NOT in sequence.fasta are be found\n");
+ fprintf(stderr, " -test1 sequence.fasta\n");
+ fprintf(stderr, " -- Tests if each and every mer is found in the\n");
+ fprintf(stderr, " positionDB. Reports if it doesn't find a mer\n");
+ fprintf(stderr, " at the correct position. Doesn't report if table\n");
+ fprintf(stderr, " has too much stuff.\n");
+ fprintf(stderr, " -test2 db.fasta sequence.fasta\n");
+ fprintf(stderr, " -- Builds a positionDB from db.fasta, then searches\n");
+ fprintf(stderr, " the table for each mer in sequence.fasta. Reports\n");
+ fprintf(stderr, " all mers it finds.\n");
+ fprintf(stderr, " -- This is a silly test and you shouldn't do it.\n");
+ exit(1);
+ }
+
+ int arg = 1;
+ while (arg < argc) {
+ if (strcmp(argv[arg], "-mersize") == 0) {
+ mersize = strtouint32(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-merskip") == 0) {
+ merskip = strtouint32(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-mask") == 0) {
+ maskF = argv[++arg];
+ } else if (strcmp(argv[arg], "-only") == 0) {
+ onlyF = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-merbegin") == 0) {
+ merBegin = strtouint64(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-merend") == 0) {
+ merEnd = strtouint64(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-sequence") == 0) {
+ sequenceFile = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-output") == 0) {
+ outputFile = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-dump") == 0) {
+ positionDB *e = new positionDB(argv[++arg], 0, 0, 0, false);
+ e->printState(stdout);
+ delete e;
+ exit(0);
+ } else if (strcmp(argv[arg], "-test1") == 0) {
+ exit(test1(argv[arg+1]));
+ } else if (strcmp(argv[arg], "-test2") == 0) {
+ exit(test2(argv[arg+1], argv[arg+2]));
+ } else {
+ fprintf(stderr, "ERROR: unknown arg '%s'\n", argv[arg]);
+ exit(1);
+ }
+
+ arg++;
+ }
+
+ // Exit quickly if the output file exists.
+ //
+ if (AS_UTL_fileExists(outputFile)) {
+ fprintf(stderr, "Output file '%s' exists already!\n", outputFile);
+ exit(0);
+ }
+
+
+ merStream *MS = new merStream(new kMerBuilder(MERSIZE),
+ new seqStream(sequenceFile),
+ true, true);
+
+ // Approximate the number of mers in the sequences.
+ //
+ uint64 numMers = MS->approximateNumberOfMers();
+
+ // Reset the limits.
+ //
+ // XXX: If the user somehow knows how many mers are in the input
+ // file, and specifies an end between there and the amount of
+ // sequence, we'll pointlessly still make a merStreamFile, even
+ // though we shouldn't.
+ //
+ if (merBegin == ~uint64ZERO) merBegin = 0;
+ if (merEnd == ~uint64ZERO) merEnd = numMers;
+
+ if (merBegin >= merEnd) {
+ fprintf(stderr, "ERROR: merbegin="F_U64" and merend="F_U64" are incompatible.\n",
+ merBegin, merEnd);
+ exit(1);
+ }
+
+ if ((merBegin > 0) || (merEnd < numMers))
+ MS->setBaseRange(merBegin, merEnd);
+
+ existDB *maskDB = 0L;
+ if (maskF) {
+ fprintf(stderr, "Building maskDB from '%s'\n", maskF);
+ maskDB = new existDB(maskF, mersize, existDBnoFlags, 0, ~uint32ZERO);
+ }
+
+ existDB *onlyDB = 0L;
+ if (onlyF) {
+ fprintf(stderr, "Building onlyDB from '%s'\n", onlyF);
+ onlyDB = new existDB(onlyF, mersize, existDBnoFlags, 0, ~uint32ZERO);
+ }
+
+ fprintf(stderr, "Building table with merSize "F_U32", merSkip "F_U32"\n", mersize, merskip);
+
+ positionDB *positions = new positionDB(MS, mersize, merskip, maskDB, onlyDB, 0L, 0, 0, 0, 0, true);
+
+ fprintf(stderr, "Dumping positions table to '%s'\n", outputFile);
+
+ positions->saveState(outputFile);
+
+ delete MS;
+ delete positions;
+
+ exit(0);
+}
diff --git a/src/falcon_sense/falcon_sense.mk b/src/meryl/positionDB.mk
similarity index 66%
copy from src/falcon_sense/falcon_sense.mk
copy to src/meryl/positionDB.mk
index 4cb2f2e..01973d6 100644
--- a/src/falcon_sense/falcon_sense.mk
+++ b/src/meryl/positionDB.mk
@@ -1,3 +1,4 @@
+
# If 'make' isn't run from the root directory, we need to set these to
# point to the upper level build directory.
ifeq "$(strip ${BUILD_DIR})" ""
@@ -7,13 +8,14 @@ ifeq "$(strip ${TARGET_DIR})" ""
TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin
endif
-TARGET := falcon_sense
-SOURCES := falcon_sense.C
+TARGET := positionDB
+SOURCES := positionDB.C
-SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns/libNDFalcon libfalcon
+SRC_INCDIRS := .. ../AS_UTL libleaff libkmer
TGT_LDFLAGS := -L${TARGET_DIR}
-TGT_LDLIBS := -lcanu
-TGT_PREREQS := libcanu.a
+TGT_LDLIBS := -lleaff -lcanu
+TGT_PREREQS := libleaff.a libcanu.a
SUBMAKEFILES :=
+
diff --git a/src/mhap/mhapConvert.C b/src/mhap/mhapConvert.C
index 7f5daae..152c306 100644
--- a/src/mhap/mhapConvert.C
+++ b/src/mhap/mhapConvert.C
@@ -42,9 +42,8 @@ using namespace std;
int
main(int argc, char **argv) {
- bool asCoords = true;
-
char *outName = NULL;
+ char *gkpName = NULL;
uint32 baseIDhash = 0;
uint32 numIDhash = 0;
@@ -66,6 +65,9 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-q") == 0) {
baseIDquery = atoi(argv[++arg]) - 1;
+ } else if (strcmp(argv[arg], "-G") == 0) {
+ gkpName = argv[++arg];
+
} else if (AS_UTL_fileExists(argv[arg])) {
files.push_back(argv[arg]);
@@ -77,7 +79,7 @@ main(int argc, char **argv) {
arg++;
}
- if ((err) || (files.size() == 0)) {
+ if ((err) || (gkpName == NULL) || (outName == NULL) || (files.size() == 0)) {
fprintf(stderr, "usage: %s [options] file.mhap[.gz]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, " Converts mhap native output to ovb\n");
@@ -89,6 +91,8 @@ main(int argc, char **argv) {
fprintf(stderr, " -q id base id of query reads\n");
fprintf(stderr, " (mhap output IDs 'num+1' and higher)\n");
+ if (gkpName == NULL)
+ fprintf(stderr, "ERROR: no gkpStore (-G) supplied\n");
if (files.size() == 0)
fprintf(stderr, "ERROR: no overlap files supplied\n");
@@ -97,7 +101,8 @@ main(int argc, char **argv) {
char *ovStr = new char [1024];
- ovOverlap ov(NULL);
+ gkStore *gkpStore = gkStore::gkStore_open(gkpName);
+ ovOverlap ov(gkpStore);
ovFile *of = new ovFile(NULL, outName, ovFileFullWrite);
for (uint32 ff=0; ff<files.size(); ff++) {
@@ -144,6 +149,24 @@ main(int argc, char **argv) {
ov.erate(atof(W[2]));
+ // Check the overlap - the hangs must be less than the read length.
+
+ uint32 alen = gkpStore->gkStore_getRead(ov.a_iid)->gkRead_sequenceLength();
+ uint32 blen = gkpStore->gkStore_getRead(ov.b_iid)->gkRead_sequenceLength();
+
+ if ((alen < ov.dat.ovl.ahg5 + ov.dat.ovl.ahg3) ||
+ (blen < ov.dat.ovl.bhg5 + ov.dat.ovl.bhg3)) {
+ fprintf(stderr, "INVALID OVERLAP %8u (len %6d) %8u (len %6d) hangs %6lu %6lu - %6lu %6lu flip %lu\n",
+ ov.a_iid, alen,
+ ov.b_iid, blen,
+ ov.dat.ovl.ahg5, ov.dat.ovl.ahg3,
+ ov.dat.ovl.bhg5, ov.dat.ovl.bhg3,
+ ov.dat.ovl.flipped);
+ exit(1);
+ }
+
+ // Overlap looks good, write it!
+
of->writeOverlap(&ov);
}
@@ -155,5 +178,7 @@ main(int argc, char **argv) {
delete of;
delete [] ovStr;
+ gkpStore->gkStore_close();
+
exit(0);
}
diff --git a/src/minimap/mmapConvert.C b/src/minimap/mmapConvert.C
index f817d5d..1dab760 100644
--- a/src/minimap/mmapConvert.C
+++ b/src/minimap/mmapConvert.C
@@ -38,19 +38,20 @@ using namespace std;
int
main(int argc, char **argv) {
- bool asCoords = true;
-
- char *outName = NULL;
+ char *outName = NULL;
+ char *gkpName = NULL;
vector<char *> files;
-
int32 arg = 1;
int32 err = 0;
while (arg < argc) {
if (strcmp(argv[arg], "-o") == 0) {
outName = argv[++arg];
+ } else if (strcmp(argv[arg], "-G") == 0) {
+ gkpName = argv[++arg];
+
} else if (AS_UTL_fileExists(argv[arg])) {
files.push_back(argv[arg]);
@@ -62,7 +63,7 @@ main(int argc, char **argv) {
arg++;
}
- if ((err) || (files.size() == 0)) {
+ if ((err) || (gkpName == NULL) || (outName == NULL) || (files.size() == 0)) {
fprintf(stderr, "usage: %s [options] file.mhap[.gz]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, " Converts mhap native output to ovb\n");
@@ -70,6 +71,8 @@ main(int argc, char **argv) {
fprintf(stderr, " -o out.ovb output file\n");
fprintf(stderr, "\n");
+ if (gkpName == NULL)
+ fprintf(stderr, "ERROR: no gkpStore (-G) supplied\n");
if (files.size() == 0)
fprintf(stderr, "ERROR: no overlap files supplied\n");
@@ -78,7 +81,8 @@ main(int argc, char **argv) {
char *ovStr = new char [1024];
- ovOverlap ov(NULL);
+ gkStore *gkpStore = gkStore::gkStore_open(gkpName);
+ ovOverlap ov(gkpStore);
ovFile *of = new ovFile(NULL, outName, ovFileFullWrite);
for (uint32 ff=0; ff<files.size(); ff++) {
@@ -119,6 +123,24 @@ main(int argc, char **argv) {
ov.erate(1-((double)W(9)/W(10)));
+ // Check the overlap - the hangs must be less than the read length.
+
+ uint32 alen = gkpStore->gkStore_getRead(ov.a_iid)->gkRead_sequenceLength();
+ uint32 blen = gkpStore->gkStore_getRead(ov.b_iid)->gkRead_sequenceLength();
+
+ if ((alen < ov.dat.ovl.ahg5 + ov.dat.ovl.ahg3) ||
+ (blen < ov.dat.ovl.bhg5 + ov.dat.ovl.bhg3)) {
+ fprintf(stderr, "INVALID OVERLAP %8u (len %6d) %8u (len %6d) hangs %6lu %6lu - %6lu %6lu flip %lu\n",
+ ov.a_iid, alen,
+ ov.b_iid, blen,
+ ov.dat.ovl.ahg5, ov.dat.ovl.ahg3,
+ ov.dat.ovl.bhg5, ov.dat.ovl.bhg3,
+ ov.dat.ovl.flipped);
+ exit(1);
+ }
+
+ // Overlap looks good, write it!
+
of->writeOverlap(&ov);
}
@@ -128,5 +150,7 @@ main(int argc, char **argv) {
delete of;
delete [] ovStr;
+ gkpStore->gkStore_close();
+
exit(0);
}
diff --git a/src/overlapInCore/libedlib/edlib.C b/src/overlapInCore/libedlib/edlib.C
index 5f6d8b8..cc36091 100644
--- a/src/overlapInCore/libedlib/edlib.C
+++ b/src/overlapInCore/libedlib/edlib.C
@@ -203,14 +203,14 @@ EdlibAlignResult edlibAlign(const char* queryOriginal, const int queryLength,
if (result.editDistance >= 0) { // If there is solution.
// If NW mode, set end location explicitly.
if (config.mode == EDLIB_MODE_NW) {
- result.endLocations = (int *) malloc(sizeof(int) * 1);
+ result.endLocations = new int [1];
result.endLocations[0] = targetLength - 1;
result.numLocations = 1;
}
// Find starting locations.
if (config.task == EDLIB_TASK_LOC || config.task == EDLIB_TASK_PATH) {
- result.startLocations = (int*) malloc(result.numLocations * sizeof(int));
+ result.startLocations = new int [result.numLocations];
if (config.mode == EDLIB_MODE_HW) { // If HW, I need to calculate start locations.
const unsigned char* rTarget = createReverseCopy(target, targetLength);
const unsigned char* rQuery = createReverseCopy(query, queryLength);
@@ -260,9 +260,9 @@ EdlibAlignResult edlibAlign(const char* queryOriginal, const int queryLength,
//--- Free memory ---//
delete[] Peq;
- free(query);
- free(target);
- if (alignData) delete alignData;
+ delete[] query;
+ delete[] target;
+ delete alignData;
//-------------------//
return result;
@@ -313,13 +313,35 @@ char* edlibAlignmentToCigar(unsigned char* alignment, int alignmentLength,
}
}
cigar->push_back(0); // Null character termination.
- char* cigar_ = (char*) malloc(cigar->size() * sizeof(char));
+ char* cigar_ = new char [cigar->size()];
memcpy(cigar_, &(*cigar)[0], cigar->size() * sizeof(char));
delete cigar;
return cigar_;
}
+void edlibAlignmentToStrings(const unsigned char* alignment, int alignmentLength, int tgtStart, int tgtEnd, int qryStart, int qryEnd, const char *tgt, const char *qry, char *tgt_aln_str, char *qry_aln_str) {
+ for (int a = 0, qryPos=qryStart, tgtPos=tgtStart; a < alignmentLength; a++) {
+ assert(qryPos <= qryEnd && tgtPos <= tgtEnd);
+ if (alignment[a] == EDLIB_EDOP_MATCH || alignment[a] == EDLIB_EDOP_MISMATCH) { // match or mismatch
+ qry_aln_str[a] = qry[qryPos];
+ tgt_aln_str[a] = tgt[tgtPos];
+ qryPos++;
+ tgtPos++;
+ } else if (alignment[a] == EDLIB_EDOP_INSERT) { // insertion in target
+ tgt_aln_str[a] = '-';
+ qry_aln_str[a] = qry[qryPos];
+ qryPos++;
+ } else if (alignment[a] == EDLIB_EDOP_DELETE) { // insertion in query
+ tgt_aln_str[a] = tgt[tgtPos];
+ qry_aln_str[a] = '-';
+ tgtPos++;
+ }
+ }
+ qry_aln_str[alignmentLength] = tgt_aln_str[alignmentLength] = '\0';
+ assert(strlen(qry_aln_str) == alignmentLength && strlen(tgt_aln_str) == alignmentLength);
+}
+
/**
* Build Peq table for given query and alphabet.
* Peq is table of dimensions alphabetLength+1 x maxNumBlocks.
@@ -598,7 +620,7 @@ static int myersCalcEditDistanceSemiGlobal(Word* const Peq, const int W, const i
if (lastBlock < firstBlock) {
*bestScore_ = bestScore;
if (bestScore != -1) {
- *positions_ = (int *) malloc(sizeof(int) * positions.size());
+ *positions_ = new int [positions.size()];
*numPositions_ = positions.size();
copy(positions.begin(), positions.end(), *positions_);
}
@@ -647,7 +669,7 @@ static int myersCalcEditDistanceSemiGlobal(Word* const Peq, const int W, const i
*bestScore_ = bestScore;
if (bestScore != -1) {
- *positions_ = (int *) malloc(sizeof(int) * positions.size());
+ *positions_ = new int [positions.size()];
*numPositions_ = positions.size();
copy(positions.begin(), positions.end(), *positions_);
}
@@ -885,7 +907,7 @@ static int obtainAlignmentTraceback(const int queryLength, const int targetLengt
const int maxNumBlocks = ceilDiv(queryLength, WORD_SIZE);
const int W = maxNumBlocks * WORD_SIZE - queryLength;
- *alignment = (unsigned char*) malloc((queryLength + targetLength - 1) * sizeof(unsigned char));
+ *alignment = new unsigned char [queryLength + targetLength - 1];
*alignmentLength = 0;
int c = targetLength - 1; // index of column
int b = maxNumBlocks - 1; // index of block in column
@@ -1065,7 +1087,8 @@ static int obtainAlignmentTraceback(const int queryLength, const int targetLengt
//----------------------------------//
}
- *alignment = (unsigned char*) realloc(*alignment, (*alignmentLength) * sizeof(unsigned char));
+ // BPW suspects this is just releasing memory.
+ //*alignment = (unsigned char*) realloc(*alignment, (*alignmentLength) * sizeof(unsigned char));
reverse(*alignment, *alignment + (*alignmentLength));
return EDLIB_STATUS_OK;
}
@@ -1094,7 +1117,7 @@ static int obtainAlignment(const unsigned char* query, const unsigned char* rQue
// Handle special case when one of sequences has length of 0.
if (queryLength == 0 || targetLength == 0) {
*alignmentLength = targetLength + queryLength;
- *alignment = (unsigned char*) malloc((*alignmentLength) * sizeof(unsigned char));
+ *alignment = new unsigned char [*alignmentLength];
for (int i = 0; i < *alignmentLength; i++) {
(*alignment)[i] = queryLength == 0 ? EDLIB_EDOP_DELETE : EDLIB_EDOP_INSERT;
}
@@ -1302,19 +1325,19 @@ static int obtainAlignmentHirschberg(
target + ulWidth, rTarget, lrWidth,
alphabetLength, rightScore, &lrAlignment, &lrAlignmentLength);
if (ulStatusCode == EDLIB_STATUS_ERROR || lrStatusCode == EDLIB_STATUS_ERROR) {
- if (ulAlignment) free(ulAlignment);
- if (lrAlignment) free(lrAlignment);
+ delete[] ulAlignment;
+ delete[] lrAlignment;
return EDLIB_STATUS_ERROR;
}
// Build alignment by concatenating upper left alignment with lower right alignment.
*alignmentLength = ulAlignmentLength + lrAlignmentLength;
- *alignment = (unsigned char*) malloc((*alignmentLength) * sizeof(unsigned char));
+ *alignment = new unsigned char [*alignmentLength];
memcpy(*alignment, ulAlignment, ulAlignmentLength);
memcpy(*alignment + ulAlignmentLength, lrAlignment, lrAlignmentLength);
- free(ulAlignment);
- free(lrAlignment);
+ delete[] ulAlignment;
+ delete[] lrAlignment;
return EDLIB_STATUS_OK;
}
@@ -1343,8 +1366,8 @@ static int transformSequences(const char* queryOriginal, const int queryLength,
// Each letter is assigned an ordinal number, starting from 0 up to alphabetLength - 1,
// and new query and target are created in which letters are replaced with their ordinal numbers.
// This query and target are used in all the calculations later.
- *queryTransformed = (unsigned char *) malloc(sizeof(unsigned char) * queryLength);
- *targetTransformed = (unsigned char *) malloc(sizeof(unsigned char) * targetLength);
+ *queryTransformed = new unsigned char [queryLength];
+ *targetTransformed = new unsigned char [targetLength];
// Alphabet information, it is constructed on fly while transforming sequences.
unsigned char letterIdx[128]; //!< letterIdx[c] is index of letter c in alphabet
@@ -1388,7 +1411,7 @@ EdlibAlignConfig edlibDefaultAlignConfig() {
}
void edlibFreeAlignResult(EdlibAlignResult result) {
- if (result.endLocations) free(result.endLocations);
- if (result.startLocations) free(result.startLocations);
- if (result.alignment) free(result.alignment);
+ delete[] result.endLocations;
+ delete[] result.startLocations;
+ delete[] result.alignment;
}
diff --git a/src/overlapInCore/libedlib/edlib.H b/src/overlapInCore/libedlib/edlib.H
index ee9b320..c85d895 100644
--- a/src/overlapInCore/libedlib/edlib.H
+++ b/src/overlapInCore/libedlib/edlib.H
@@ -54,10 +54,6 @@
* @brief Main header file, containing all public functions and structures.
*/
-#ifdef __cplusplus
-extern "C" {
-#endif
-
// Status codes
#define EDLIB_STATUS_OK 0
#define EDLIB_STATUS_ERROR 1
@@ -66,40 +62,40 @@ extern "C" {
* Alignment methods - how should Edlib treat gaps before and after query?
*/
typedef enum {
- /**
- * Global method. This is the standard method.
- * Useful when you want to find out how similar is first sequence to second sequence.
- */
- EDLIB_MODE_NW,
- /**
- * Prefix method. Similar to global method, but with a small twist - gap at query end is not penalized.
- * What that means is that deleting elements from the end of second sequence is "free"!
- * For example, if we had "AACT" and "AACTGGC", edit distance would be 0, because removing "GGC" from the end
- * of second sequence is "free" and does not count into total edit distance. This method is appropriate
- * when you want to find out how well first sequence fits at the beginning of second sequence.
- */
- EDLIB_MODE_SHW,
- /**
- * Infix method. Similar as prefix method, but with one more twist - gaps at query end and start are
- * not penalized. What that means is that deleting elements from the start and end of second sequence is "free"!
- * For example, if we had ACT and CGACTGAC, edit distance would be 0, because removing CG from the start
- * and GAC from the end of second sequence is "free" and does not count into total edit distance.
- * This method is appropriate when you want to find out how well first sequence fits at any part of
- * second sequence.
- * For example, if your second sequence was a long text and your first sequence was a sentence from that text,
- * but slightly scrambled, you could use this method to discover how scrambled it is and where it fits in
- * that text. In bioinformatics, this method is appropriate for aligning read to a sequence.
- */
- EDLIB_MODE_HW
+ /**
+ * Global method. This is the standard method.
+ * Useful when you want to find out how similar is first sequence to second sequence.
+ */
+ EDLIB_MODE_NW,
+ /**
+ * Prefix method. Similar to global method, but with a small twist - gap at query end is not penalized.
+ * What that means is that deleting elements from the end of second sequence is "free"!
+ * For example, if we had "AACT" and "AACTGGC", edit distance would be 0, because removing "GGC" from the end
+ * of second sequence is "free" and does not count into total edit distance. This method is appropriate
+ * when you want to find out how well first sequence fits at the beginning of second sequence.
+ */
+ EDLIB_MODE_SHW,
+ /**
+ * Infix method. Similar as prefix method, but with one more twist - gaps at query end and start are
+ * not penalized. What that means is that deleting elements from the start and end of second sequence is "free"!
+ * For example, if we had ACT and CGACTGAC, edit distance would be 0, because removing CG from the start
+ * and GAC from the end of second sequence is "free" and does not count into total edit distance.
+ * This method is appropriate when you want to find out how well first sequence fits at any part of
+ * second sequence.
+ * For example, if your second sequence was a long text and your first sequence was a sentence from that text,
+ * but slightly scrambled, you could use this method to discover how scrambled it is and where it fits in
+ * that text. In bioinformatics, this method is appropriate for aligning read to a sequence.
+ */
+ EDLIB_MODE_HW
} EdlibAlignMode;
/**
* Alignment tasks - what do you want Edlib to do?
*/
typedef enum {
- EDLIB_TASK_DISTANCE, //!< Find edit distance and end locations.
- EDLIB_TASK_LOC, //!< Find edit distance, end locations and start locations.
- EDLIB_TASK_PATH //!< Find edit distance, end locations and start locations and alignment path.
+ EDLIB_TASK_DISTANCE, //!< Find edit distance and end locations.
+ EDLIB_TASK_LOC, //!< Find edit distance, end locations and start locations.
+ EDLIB_TASK_PATH //!< Find edit distance, end locations and start locations and alignment path.
} EdlibAlignTask;
/**
@@ -108,8 +104,8 @@ typedef enum {
* @see http://drive5.com/usearch/manual/cigar.html
*/
typedef enum {
- EDLIB_CIGAR_STANDARD, //!< Match: 'M', Insertion: 'I', Deletion: 'D', Mismatch: 'M'.
- EDLIB_CIGAR_EXTENDED //!< Match: '=', Insertion: 'I', Deletion: 'D', Mismatch: 'X'.
+ EDLIB_CIGAR_STANDARD, //!< Match: 'M', Insertion: 'I', Deletion: 'D', Mismatch: 'M'.
+ EDLIB_CIGAR_EXTENDED //!< Match: '=', Insertion: 'I', Deletion: 'D', Mismatch: 'X'.
} EdlibCigarFormat;
// Edit operations.
@@ -120,151 +116,148 @@ typedef enum {
- /**
- * @brief Configuration object for edlibAlign() function.
- */
- typedef struct {
- /**
- * Set k to non-negative value to tell edlib that edit distance is not larger than k.
- * Smaller k can significantly improve speed of computation.
- * If edit distance is larger than k, edlib will set edit distance to -1.
- * Set k to negative value and edlib will internally auto-adjust k until score is found.
- */
- int k;
-
- /**
- * Alignment method.
- * EDLIB_MODE_NW: global (Needleman-Wunsch)
- * EDLIB_MODE_SHW: prefix. Gap after query is not penalized.
- * EDLIB_MODE_HW: infix. Gaps before and after query are not penalized.
- */
- EdlibAlignMode mode;
+/**
+ * @brief Configuration object for edlibAlign() function.
+ */
+typedef struct {
+ /**
+ * Set k to non-negative value to tell edlib that edit distance is not larger than k.
+ * Smaller k can significantly improve speed of computation.
+ * If edit distance is larger than k, edlib will set edit distance to -1.
+ * Set k to negative value and edlib will internally auto-adjust k until score is found.
+ */
+ int k;
- /**
- * Alignment task - tells Edlib what to calculate. Less to calculate, faster it is.
- * EDLIB_TASK_DISTANCE - find edit distance and end locations of optimal alignment paths in target.
- * EDLIB_TASK_LOC - find edit distance and start and end locations of optimal alignment paths in target.
- * EDLIB_TASK_PATH - find edit distance, alignment path (and start and end locations of it in target).
- */
- EdlibAlignTask task;
- } EdlibAlignConfig;
+ /**
+ * Alignment method.
+ * EDLIB_MODE_NW: global (Needleman-Wunsch)
+ * EDLIB_MODE_SHW: prefix. Gap after query is not penalized.
+ * EDLIB_MODE_HW: infix. Gaps before and after query are not penalized.
+ */
+ EdlibAlignMode mode;
- /**
- * Helper method for easy construction of configuration object.
- * @return Configuration object filled with given parameters.
- */
- EdlibAlignConfig edlibNewAlignConfig(int k, EdlibAlignMode mode, EdlibAlignTask task);
+ /**
+ * Alignment task - tells Edlib what to calculate. Less to calculate, faster it is.
+ * EDLIB_TASK_DISTANCE - find edit distance and end locations of optimal alignment paths in target.
+ * EDLIB_TASK_LOC - find edit distance and start and end locations of optimal alignment paths in target.
+ * EDLIB_TASK_PATH - find edit distance, alignment path (and start and end locations of it in target).
+ */
+ EdlibAlignTask task;
+} EdlibAlignConfig;
- /**
- * @return Default configuration object, with following defaults:
- * k = -1, mode = EDLIB_MODE_NW, task = EDLIB_TASK_DISTANCE.
- */
- EdlibAlignConfig edlibDefaultAlignConfig();
+/**
+ * Helper method for easy construction of configuration object.
+ * @return Configuration object filled with given parameters.
+ */
+EdlibAlignConfig edlibNewAlignConfig(int k, EdlibAlignMode mode, EdlibAlignTask task);
+/**
+ * @return Default configuration object, with following defaults:
+ * k = -1, mode = EDLIB_MODE_NW, task = EDLIB_TASK_DISTANCE.
+ */
+EdlibAlignConfig edlibDefaultAlignConfig();
- /**
- * Container for results of alignment done by edlibAlign() function.
- */
- typedef struct {
- /**
- * -1 if k is non-negative and edit distance is larger than k.
- */
- int editDistance;
- /**
- * Array of zero-based positions in target where optimal alignment paths end.
- * If gap after query is penalized, gap counts as part of query (NW), otherwise not.
- * Set to NULL if edit distance is larger than k.
- * If you do not free whole result object using edlibFreeAlignResult(), do not forget to use free().
- */
- int* endLocations;
- /**
- * Array of zero-based positions in target where optimal alignment paths start,
- * they correspond to endLocations.
- * If gap before query is penalized, gap counts as part of query (NW), otherwise not.
- * Set to NULL if not calculated or if edit distance is larger than k.
- * If you do not free whole result object using edlibFreeAlignResult(), do not forget to use free().
- */
- int* startLocations;
- /**
- * Number of end (and start) locations.
- */
- int numLocations;
- /**
- * Alignment is found for first pair of start and end locations.
- * Set to NULL if not calculated.
- * Alignment is sequence of numbers: 0, 1, 2, 3.
- * 0 stands for match.
- * 1 stands for insertion to target.
- * 2 stands for insertion to query.
- * 3 stands for mismatch.
- * Alignment aligns query to target from begining of query till end of query.
- * If gaps are not penalized, they are not in alignment.
- * If you do not free whole result object using edlibFreeAlignResult(), do not forget to use free().
- */
- unsigned char* alignment;
- /**
- * Length of alignment.
- */
- int alignmentLength;
- /**
- * Number of different characters in query and target together.
- */
- int alphabetLength;
- } EdlibAlignResult;
- /**
- * Frees memory in EdlibAlignResult that was allocated by edlib.
- * If you do not use it, make sure to free needed members manually using free().
- */
- void edlibFreeAlignResult(EdlibAlignResult result);
+/**
+ * Container for results of alignment done by edlibAlign() function.
+ */
+typedef struct {
+ /**
+ * -1 if k is non-negative and edit distance is larger than k.
+ */
+ int editDistance;
+ /**
+ * Array of zero-based positions in target where optimal alignment paths end.
+ * If gap after query is penalized, gap counts as part of query (NW), otherwise not.
+ * Set to NULL if edit distance is larger than k.
+ * If you do not free whole result object using edlibFreeAlignResult(), do not forget to use free().
+ */
+ int* endLocations;
+ /**
+ * Array of zero-based positions in target where optimal alignment paths start,
+ * they correspond to endLocations.
+ * If gap before query is penalized, gap counts as part of query (NW), otherwise not.
+ * Set to NULL if not calculated or if edit distance is larger than k.
+ * If you do not free whole result object using edlibFreeAlignResult(), do not forget to use free().
+ */
+ int* startLocations;
+ /**
+ * Number of end (and start) locations.
+ */
+ int numLocations;
+ /**
+ * Alignment is found for first pair of start and end locations.
+ * Set to NULL if not calculated.
+ * Alignment is sequence of numbers: 0, 1, 2, 3.
+ * 0 stands for match.
+ * 1 stands for insertion to target.
+ * 2 stands for insertion to query.
+ * 3 stands for mismatch.
+ * Alignment aligns query to target from begining of query till end of query.
+ * If gaps are not penalized, they are not in alignment.
+ * If you do not free whole result object using edlibFreeAlignResult(), do not forget to use free().
+ */
+ unsigned char* alignment;
+ /**
+ * Length of alignment.
+ */
+ int alignmentLength;
+ /**
+ * Number of different characters in query and target together.
+ */
+ int alphabetLength;
+} EdlibAlignResult;
+/**
+ * Frees memory in EdlibAlignResult that was allocated by edlib.
+ * If you do not use it, make sure to free needed members manually using free().
+ */
+void edlibFreeAlignResult(EdlibAlignResult result);
- /**
- * Aligns two sequences (query and target) using edit distance (levenshtein distance).
- * Through config parameter, this function supports different alignment methods (global, prefix, infix),
- * as well as different modes of search (tasks).
- * It always returns edit distance and end locations of optimal alignment in target.
- * It optionally returns start locations of optimal alignment in target and alignment path,
- * if you choose appropriate tasks.
- * @param [in] query First sequence. Character codes should be in range [0, 127].
- * @param [in] queryLength Number of characters in first sequence.
- * @param [in] target Second sequence. Character codes should be in range [0, 127].
- * @param [in] targetLength Number of characters in second sequence.
- * @param [in] config Additional alignment parameters, like alignment method and wanted results.
- * @return Result of alignment, which can contain edit distance, start and end locations and alignment path.
- * Make sure to clean up the object using edlibFreeAlignResult() or by manually freeing needed members.
- */
- EdlibAlignResult edlibAlign(const char* query, const int queryLength,
- const char* target, const int targetLength,
- EdlibAlignConfig config);
+/**
+ * Aligns two sequences (query and target) using edit distance (levenshtein distance).
+ * Through config parameter, this function supports different alignment methods (global, prefix, infix),
+ * as well as different modes of search (tasks).
+ * It always returns edit distance and end locations of optimal alignment in target.
+ * It optionally returns start locations of optimal alignment in target and alignment path,
+ * if you choose appropriate tasks.
+ * @param [in] query First sequence. Character codes should be in range [0, 127].
+ * @param [in] queryLength Number of characters in first sequence.
+ * @param [in] target Second sequence. Character codes should be in range [0, 127].
+ * @param [in] targetLength Number of characters in second sequence.
+ * @param [in] config Additional alignment parameters, like alignment method and wanted results.
+ * @return Result of alignment, which can contain edit distance, start and end locations and alignment path.
+ * Make sure to clean up the object using edlibFreeAlignResult() or by manually freeing needed members.
+ */
+EdlibAlignResult edlibAlign(const char* query, const int queryLength,
+ const char* target, const int targetLength,
+ EdlibAlignConfig config);
- /**
- * Builds cigar string from given alignment sequence.
- * @param [in] alignment Alignment sequence.
- * 0 stands for match.
- * 1 stands for insertion to target.
- * 2 stands for insertion to query.
- * 3 stands for mismatch.
- * @param [in] alignmentLength
- * @param [in] cigarFormat Cigar will be returned in specified format.
- * @return Cigar string.
- * I stands for insertion.
- * D stands for deletion.
- * X stands for mismatch. (used only in extended format)
- * = stands for match. (used only in extended format)
- * M stands for (mis)match. (used only in standard format)
- * String is null terminated.
- * Needed memory is allocated and given pointer is set to it.
- * Do not forget to free it later using free()!
- */
- char* edlibAlignmentToCigar(unsigned char* alignment, int alignmentLength,
- EdlibCigarFormat cigarFormat);
+/**
+ * Builds cigar string from given alignment sequence.
+ * @param [in] alignment Alignment sequence.
+ * 0 stands for match.
+ * 1 stands for insertion to target.
+ * 2 stands for insertion to query.
+ * 3 stands for mismatch.
+ * @param [in] alignmentLength
+ * @param [in] cigarFormat Cigar will be returned in specified format.
+ * @return Cigar string.
+ * I stands for insertion.
+ * D stands for deletion.
+ * X stands for mismatch. (used only in extended format)
+ * = stands for match. (used only in extended format)
+ * M stands for (mis)match. (used only in standard format)
+ * String is null terminated.
+ * Needed memory is allocated and given pointer is set to it.
+ * Do not forget to free it later using free()!
+ */
+char* edlibAlignmentToCigar(unsigned char* alignment, int alignmentLength,
+ EdlibCigarFormat cigarFormat);
+void edlibAlignmentToStrings(const unsigned char* alignment, int alignmentLength, int tgtStart, int tgtEnd, int qryStart, int qryEnd, const char *tgt, const char *qry, char *tgt_aln_str, char *qry_aln_str);
-#ifdef __cplusplus
-}
-#endif
#endif // EDLIB_H
diff --git a/src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C b/src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C
index 35a29b0..ee29808 100644
--- a/src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C
+++ b/src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C
@@ -23,6 +23,10 @@
* are Copyright 2015 Battelle National Biodefense Institute, and
* are subject to the BSD 3-Clause License
*
+ * Brian P. Walenz beginning on 2017-FEB-02
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
* File 'README.licenses' in the root directory of this distribution contains
* full conditions and disclaimers for each license.
*/
@@ -44,7 +48,7 @@
uint32 EDIT_SPACE_SIZE = 1 * 1024 * 1024;
void
-prefixEditDistance::Allocate_More_Edit_Space(void) {
+prefixEditDistance::Allocate_More_Edit_Space(int32 ein) {
// Determine the last allocated block, and the last assigned block
@@ -94,6 +98,11 @@ prefixEditDistance::Allocate_More_Edit_Space(void) {
fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e);
assert(e != b);
- //fprintf(stderr, "WorkArea %d allocates space %d of size %d for array %d through %d\n", thread_id, a, Size, b, e-1);
+#ifdef DEBUG_EDIT_SPACE_ALLOC
+ Edit_Space_Lazy_Max = e-1;
+
+ fprintf(stdout, "WorkArea %2d allocates space %d (for e=%d) of size %d for array %d through %d\n",
+ omp_get_thread_num(), a, ein, Size, b, e-1);
+#endif
}
diff --git a/src/overlapInCore/liboverlap/prefixEditDistance-extend.C b/src/overlapInCore/liboverlap/prefixEditDistance-extend.C
index 30ccae3..3ee5127 100644
--- a/src/overlapInCore/liboverlap/prefixEditDistance-extend.C
+++ b/src/overlapInCore/liboverlap/prefixEditDistance-extend.C
@@ -69,7 +69,6 @@
#include "overlapInCore.H"
#include "prefixEditDistance.H"
-#undef DEBUG
// See how far the exact match in Match extends. The match
@@ -86,10 +85,10 @@
Overlap_t
prefixEditDistance::Extend_Alignment(Match_Node_t *Match,
- char *S, int32 S_Len,
- char *T, int32 T_Len,
- int32 &S_Lo, int32 &S_Hi,
- int32 &T_Lo, int32 &T_Hi,
+ char *S, uint32 S_ID, int32 S_Len,
+ char *T, uint32 T_ID, int32 T_Len,
+ int32 &S_Lo, int32 &S_Hi,
+ int32 &T_Lo, int32 &T_Hi,
int32 &Errors) {
int32 Right_Errors = 0;
int32 Left_Errors = 0;
@@ -112,10 +111,10 @@ prefixEditDistance::Extend_Alignment(Match_Node_t *Match,
int32 Error_Limit = Error_Bound[Total_Olap];
-#ifdef DEBUG
- fprintf(stderr, "prefixEditDistance::Extend_Alignment()-- limit olap of %u bases to %u errors - %f%%\n",
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "prefixEditDistance::Extend_Alignment()-- limit olap of %u bases to %u errors - %f%%\n",
Total_Olap, Error_Limit, 100.0 * Error_Limit / Total_Olap);
- fprintf(stderr, "prefixEditDistance::Extend_Alignment()-- S: %d-%d and %d-%d T: %d-%d and %d-%d\n",
+ fprintf(stdout, "prefixEditDistance::Extend_Alignment()-- S: %d-%d and %d-%d T: %d-%d and %d-%d\n",
0, S_Left_Begin, S_Right_Begin, S_Right_Begin + S_Right_Len,
0, T_Left_Begin, T_Right_Begin, T_Right_Begin + T_Right_Len);
#endif
@@ -198,6 +197,9 @@ prefixEditDistance::Extend_Alignment(Match_Node_t *Match,
assert(Errors <= Error_Limit);
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d OVERLAP %6d %6d - %5d + %5d = %5d errors out of %d possible\n", omp_get_thread_num(), S_ID, T_ID, Left_Errors, Right_Errors, Errors, Edit_Space_Lazy_Max);
+#endif
// No overlap if both right and left don't match to end, otherwise a branch point if only one.
// If both match to end, a dovetail overlap. Indenting is all screwed up here.
diff --git a/src/overlapInCore/liboverlap/prefixEditDistance-forward.C b/src/overlapInCore/liboverlap/prefixEditDistance-forward.C
index b50e57e..c6da505 100644
--- a/src/overlapInCore/liboverlap/prefixEditDistance-forward.C
+++ b/src/overlapInCore/liboverlap/prefixEditDistance-forward.C
@@ -23,13 +23,16 @@
* are Copyright 2015 Battelle National Biodefense Institute, and
* are subject to the BSD 3-Clause License
*
+ * Brian P. Walenz beginning on 2017-FEB-02
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
* File 'README.licenses' in the root directory of this distribution contains
* full conditions and disclaimers for each license.
*/
#include "prefixEditDistance.H"
-#undef DEBUG
@@ -126,7 +129,7 @@ prefixEditDistance::forward(char *A, int32 m,
;
if (Edit_Array_Lazy[0] == NULL)
- Allocate_More_Edit_Space();
+ Allocate_More_Edit_Space(0);
Edit_Array_Lazy[0][0] = Row;
@@ -134,8 +137,8 @@ prefixEditDistance::forward(char *A, int32 m,
// Exact match
A_End = T_End = m;
Match_To_End = TRUE;
-#ifdef DEBUG
- fprintf(stderr, "forward()- exact match\n");
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d FWD exact match\n", omp_get_thread_num());
#endif
return 0;
}
@@ -147,7 +150,7 @@ prefixEditDistance::forward(char *A, int32 m,
for (e = 1; e <= Error_Limit; e++) {
if (Edit_Array_Lazy[e] == NULL)
- Allocate_More_Edit_Space();
+ Allocate_More_Edit_Space(e);
Left = MAX (Left - 1, -e);
Right = MIN (Right + 1, e);
@@ -183,9 +186,9 @@ prefixEditDistance::forward(char *A, int32 m,
if ((doingPartialOverlaps == true) && (Score < Max_Score))
abort = true;
-#ifdef DEBUG
- fprintf(stderr, "prefixEditDistance()-- e=%d MIN=%d Tail_Len=%d Max_Score=%d Score=%d slope=%f SLOPE=%f\n",
- e, MIN_BRANCH_END_DIST, Tail_Len, Max_Score, Score, slope, MIN_BRANCH_TAIL_SLOPE);
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d FWD e=%d MIN=%d Tail_Len=%d Max_Score=%d Score=%d slope=%f SLOPE=%f\n",
+ omp_get_thread_num(), e, MIN_BRANCH_END_DIST, Tail_Len, Max_Score, Score, slope, MIN_BRANCH_TAIL_SLOPE);
#endif
if ((e > MIN_BRANCH_END_DIST / 2) &&
@@ -201,8 +204,8 @@ prefixEditDistance::forward(char *A, int32 m,
Match_To_End = FALSE;
-#ifdef DEBUG
- fprintf(stderr, "forward()- ABORT alignment\n");
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d FWD ABORT alignment at e=%d best_e=%d\n", omp_get_thread_num(), e, Max_Score_Best_e);
#endif
return(Max_Score_Best_e);
}
@@ -222,8 +225,8 @@ prefixEditDistance::forward(char *A, int32 m,
Match_To_End = TRUE;
-#ifdef DEBUG
- fprintf(stderr, "forward()- END alignment\n");
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d FWD END alignment at e=%d\n", omp_get_thread_num(), e);
#endif
return(e);
}
@@ -237,8 +240,8 @@ prefixEditDistance::forward(char *A, int32 m,
Left++;
if (Left > Right) {
-#ifdef DEBUG
- fprintf(stderr, "reverse()- Left=%d Right=%d BREAK\n", Left, Right);
+#ifdef SHOW_EXTEND_ALIGN
+ //fprintf(stdout, "WorkArea %2d FWD BREAK at Left=%d Right=%d\n", omp_get_thread_num(), Left, Right);
#endif
break;
}
@@ -270,9 +273,8 @@ prefixEditDistance::forward(char *A, int32 m,
}
}
-#ifdef DEBUG
- fprintf(stderr, "forward()- return e=%d Error_Limit=%d\n",
- e, Error_Limit);
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d FWD ERROR_LIMIT at e=%d Error_Limit=%d best_e=%d\n", omp_get_thread_num(), e, Error_Limit, Max_Score_Best_e);
#endif
A_End = Max_Score_Len;
diff --git a/src/overlapInCore/liboverlap/prefixEditDistance-reverse.C b/src/overlapInCore/liboverlap/prefixEditDistance-reverse.C
index a8184f7..9d05d7c 100644
--- a/src/overlapInCore/liboverlap/prefixEditDistance-reverse.C
+++ b/src/overlapInCore/liboverlap/prefixEditDistance-reverse.C
@@ -23,13 +23,16 @@
* are Copyright 2015 Battelle National Biodefense Institute, and
* are subject to the BSD 3-Clause License
*
+ * Brian P. Walenz beginning on 2017-FEB-02
+ * are a 'United States Government Work', and
+ * are released in the public domain
+ *
* File 'README.licenses' in the root directory of this distribution contains
* full conditions and disclaimers for each license.
*/
#include "prefixEditDistance.H"
-#undef DEBUG
// Put the delta encoding of the alignment represented in Edit_Array
@@ -148,7 +151,7 @@ prefixEditDistance::reverse(char *A, int32 m,
;
if (Edit_Array_Lazy[0] == NULL)
- Allocate_More_Edit_Space();
+ Allocate_More_Edit_Space(0);
Edit_Array_Lazy[0][0] = Row;
@@ -156,8 +159,8 @@ prefixEditDistance::reverse(char *A, int32 m,
A_End = T_End = - m;
Leftover = m;
Match_To_End = TRUE;
-#ifdef DEBUG
- fprintf(stderr, "reverse()- exact match\n");
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d REV exact match\n", omp_get_thread_num());
#endif
return 0;
}
@@ -169,7 +172,7 @@ prefixEditDistance::reverse(char *A, int32 m,
for (e = 1; e <= Error_Limit; e++) {
if (Edit_Array_Lazy[e] == NULL)
- Allocate_More_Edit_Space();
+ Allocate_More_Edit_Space(e);
Left = MAX (Left - 1, -e);
Right = MIN (Right + 1, e);
@@ -206,9 +209,9 @@ prefixEditDistance::reverse(char *A, int32 m,
if ((doingPartialOverlaps == true) && (Score < Max_Score))
abort = true;
-#ifdef DEBUG
- fprintf(stderr, "prefixEditDistance()-- e=%d MIN=%d Tail_Len=%d Max_Score=%d Score=%d slope=%f SLOPE=%f\n",
- e, MIN_BRANCH_END_DIST, Tail_Len, Max_Score, Score, slope, MIN_BRANCH_TAIL_SLOPE);
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d REV e=%d MIN=%d Tail_Len=%d Max_Score=%d Score=%d slope=%f SLOPE=%f\n",
+ omp_get_thread_num(), e, MIN_BRANCH_END_DIST, Tail_Len, Max_Score, Score, slope, MIN_BRANCH_TAIL_SLOPE);
#endif
if ((e > MIN_BRANCH_END_DIST / 2) &&
@@ -224,8 +227,8 @@ prefixEditDistance::reverse(char *A, int32 m,
Match_To_End = FALSE;
-#ifdef DEBUG
- fprintf(stderr, "reverse()- ABORT alignment\n");
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d REV ABORT alignment at e=%d best_e=%d\n", omp_get_thread_num(), e, Max_Score_Best_e);
#endif
return(Max_Score_Best_e);
}
@@ -237,8 +240,8 @@ prefixEditDistance::reverse(char *A, int32 m,
Match_To_End = TRUE;
-#ifdef DEBUG
- fprintf(stderr, "reverse()- END alignment\n");
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d REV END alignment at e=%d\n", omp_get_thread_num(), e);
#endif
return(e);
}
@@ -252,8 +255,8 @@ prefixEditDistance::reverse(char *A, int32 m,
Left++;
if (Left > Right) {
-#ifdef DEBUG
- fprintf(stderr, "reverse()- Left=%d Right=%d BREAK\n", Left, Right);
+#ifdef SHOW_EXTEND_ALIGN
+ //fprintf(stdout, "WorkArea %2d REV BREAK at Left=%d Right=%d\n", omp_get_thread_num(), Left, Right);
#endif
break;
}
@@ -285,9 +288,8 @@ prefixEditDistance::reverse(char *A, int32 m,
}
}
-#ifdef DEBUG
- fprintf(stderr, "reverse()- return e=%d Error_Limit=%d\n",
- e, Error_Limit);
+#ifdef SHOW_EXTEND_ALIGN
+ fprintf(stdout, "WorkArea %2d REV ERROR_LIMIT at e=%d Error_Limit=%d best_e=%d\n", omp_get_thread_num(), e, Error_Limit, Max_Score_Best_e);
#endif
A_End = - Max_Score_Len;
diff --git a/src/overlapInCore/liboverlap/prefixEditDistance.H b/src/overlapInCore/liboverlap/prefixEditDistance.H
index e073f58..dde9cba 100644
--- a/src/overlapInCore/liboverlap/prefixEditDistance.H
+++ b/src/overlapInCore/liboverlap/prefixEditDistance.H
@@ -39,6 +39,9 @@
#include "gkStore.H" // For AS_MAX_READLEN
+#undef DEBUG_EDIT_SPACE_ALLOC
+#undef SHOW_EXTEND_ALIGN
+
// Used in -forward and -reverse
#define Sign(a) ( ((a) > 0) - ((a) < 0) )
@@ -68,7 +71,7 @@ public:
prefixEditDistance(bool doingPartialOverlaps_, double maxErate_);
~prefixEditDistance();
- void Allocate_More_Edit_Space(void);
+ void Allocate_More_Edit_Space(int e);
void Set_Right_Delta(int32 e, int32 d);
int32 forward(char *A, int32 m,
@@ -89,12 +92,12 @@ public:
bool &Match_To_End);
- Overlap_t Extend_Alignment(Match_Node_t *Match,
- char *S, int32 S_Len,
- char *T, int32 T_Len,
- int32 &S_Lo, int32 &S_Hi,
- int32 &T_Lo, int32 &T_Hi,
- int32 &Errors);
+ Overlap_t Extend_Alignment(Match_Node_t *Match,
+ char *S, uint32 S_ID, int32 S_Len,
+ char *T, uint32 T_ID, int32 T_Len,
+ int32 &S_Lo, int32 &S_Hi,
+ int32 &T_Lo, int32 &T_Hi,
+ int32 &Errors);
public:
// The four below were global #defines, two depended on the error rate which is now local.
@@ -125,6 +128,10 @@ public:
int32 **Edit_Space_Lazy; // Array of pointers, if set, it was a new'd allocation
int32 **Edit_Array_Lazy; // Array of pointers, some are not new'd allocations
+#ifdef DEBUG_EDIT_SPACE_ALLOC
+ int32 Edit_Space_Lazy_Max; // Last allocated block, DEBUG ONLY
+#endif
+
// This array [e] is the minimum value of Edit_Array[e][d]
// to be worth pursuing in edit-distance computations between reads
const
diff --git a/src/overlapInCore/overlapInCore-Process_String_Overlaps.C b/src/overlapInCore/overlapInCore-Process_String_Overlaps.C
index 50cdef7..1f92a56 100644
--- a/src/overlapInCore/overlapInCore-Process_String_Overlaps.C
+++ b/src/overlapInCore/overlapInCore-Process_String_Overlaps.C
@@ -494,7 +494,7 @@ Process_Matches (int * Start,
// Longest_Match->Offset,
// Longest_Match->Start - Longest_Match->Offset,
// S_ID, S_Lo, S_Hi, T_ID, T_Lo, T_Hi);
- Kind_Of_Olap = WA->editDist->Extend_Alignment(Longest_Match, S, S_Len, T, t_len, S_Lo, S_Hi, T_Lo, T_Hi, Errors);
+ Kind_Of_Olap = WA->editDist->Extend_Alignment(Longest_Match, S, S_ID, S_Len, T, T_ID, t_len, S_Lo, S_Hi, T_Lo, T_Hi, Errors);
if (Kind_Of_Olap == DOVETAIL || G.Doing_Partial_Overlaps) {
diff --git a/src/overlapInCore/overlapPair.C b/src/overlapInCore/overlapPair.C
index bb7ac31..63571ea 100644
--- a/src/overlapInCore/overlapPair.C
+++ b/src/overlapInCore/overlapPair.C
@@ -60,17 +60,121 @@
#define BATCH_SIZE 1024 * 1024
#define THREAD_SIZE 128
-#define MHAP_SLOP 500
-//#define DEBUG 1
+// Does slightly better with 2550 than 500. Speed takes a slight hit.
+#define MHAP_SLOP 500
-overlapReadCache *rcache = NULL; // Used to be just 'cache', but that conflicted with -pg: /usr/lib/libc_p.a(msgcat.po):(.bss+0x0): multiple definition of `cache'
-uint32 batchPrtID = 0; // When to report progress
-uint32 batchPosID = 0; // The current position of the batch
-uint32 batchEndID = 0; // The end of the batch
-pthread_mutex_t balanceMutex;
-uint32 minOverlapLength = 0;
+class alignStats {
+public:
+ alignStats() {
+ startTime = getTime();
+ reportThreshold = 0;
+
+ clear();
+ };
+ ~alignStats() {
+ };
+
+ void clear(void) {
+ nSkipped = 0;
+ nPassed = 0;
+ nFailed = 0;
+
+ nFailExtA = 0;
+ nFailExtB = 0;
+ nFailExt = 0;
+
+ nExtendedA = 0;
+ nExtendedB = 0;
+
+ nPartial = 0;
+ nDovetail = 0;
+
+ nExt5a = 0;
+ nExt3a = 0;
+ nExt5b = 0;
+ nExt3b = 0;
+ };
+
+
+ alignStats &operator+=(alignStats &that) {
+ nSkipped += that.nSkipped;
+ nPassed += that.nPassed;
+ nFailed += that.nFailed;
+
+ nFailExtA += that.nFailExtA;
+ nFailExtB += that.nFailExtB;
+ nFailExt += that.nFailExt;
+
+ nExtendedA += that.nExtendedA;
+ nExtendedB += that.nExtendedB;
+
+ nPartial += that.nPartial;
+ nDovetail += that.nDovetail;
+
+ nExt5a += that.nExt5a;
+ nExt3a += that.nExt3a;
+ nExt5b += that.nExt5b;
+ nExt3b += that.nExt3b;
+
+ return(*this);
+ };
+
+ void reportStatus(void) {
+
+ if (nPassed + nFailed < reportThreshold)
+ return;
+
+ reportThreshold += 10000;
+
+ fprintf(stderr, "Tested %9lu olaps -- Skipped %8.4f%% -- Passed %8.4f%% -- %8.2f olaps/sec\n",
+ nPassed + nFailed,
+ 100.0 * nSkipped / (nPassed + nFailed),
+ 100.0 * nPassed / (nPassed + nFailed),
+ (nPassed + nFailed) / (getTime() - startTime));
+ };
+
+ void reportFinal(void) {
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -- %lu overlaps processed.\n", nPassed + nFailed);
+ fprintf(stderr, " -- %lu skipped.\n", nSkipped);
+ fprintf(stderr, " -- %lu failed %lu passed (%.4f%%).\n", nFailed, nPassed, 100.0 * nPassed / (nPassed + nFailed));
+ fprintf(stderr, " --\n");
+ fprintf(stderr, " -- %lu failed initial alignment, allowing A to extend\n", nFailExtA);
+ fprintf(stderr, " -- %lu failed initial alignment, allowing B to extend\n", nFailExtB);
+ fprintf(stderr, " -- %lu failed initial alignment\n", nFailExt);
+ fprintf(stderr, " --\n");
+ fprintf(stderr, " -- %lu partial overlaps (of any quality)\n", nPartial);
+ fprintf(stderr, " -- %lu dovetail overlaps (before extensions, of any quality)\n", nDovetail);
+ fprintf(stderr, " --\n");
+ fprintf(stderr, " -- %lu/%lu A read dovetail extensions\n", nExt5a, nExt3a);
+ fprintf(stderr, " -- %lu/%lu B read dovetail extensions\n", nExt5b, nExt3b);
+ };
+
+ double startTime;
+ uint64 reportThreshold;
+
+ uint64 nSkipped;
+ uint64 nPassed;
+ uint64 nFailed;
+
+ uint64 nFailExtA;
+ uint64 nFailExtB;
+ uint64 nFailExt;
+
+ uint64 nExtendedA;
+ uint64 nExtendedB;
+
+ uint64 nPartial;
+ uint64 nDovetail;
+
+ uint64 nExt5a;
+ uint64 nExt3a;
+ uint64 nExt5b;
+ uint64 nExt3b;
+};
+
class workSpace {
@@ -83,7 +187,6 @@ public:
invertOverlaps = false;
gkpStore = NULL;
- //analyze = NULL;
overlapsLen = 0;
overlaps = NULL;
readSeq = NULL;
@@ -108,6 +211,22 @@ public:
+
+overlapReadCache *rcache = NULL; // Used to be just 'cache', but that conflicted with -pg: /usr/lib/libc_p.a(msgcat.po):(.bss+0x0): multiple definition of `cache'
+uint32 batchPrtID = 0; // When to report progress
+uint32 batchPosID = 0; // The current position of the batch
+uint32 batchEndID = 0; // The end of the batch
+pthread_mutex_t balanceMutex;
+
+uint32 minOverlapLength = 0;
+
+alignStats globalStats;
+
+bool debug = false;
+
+
+
+
bool
getRange(uint32 &bgnID, uint32 &endID) {
@@ -130,6 +249,99 @@ getRange(uint32 &bgnID, uint32 &endID) {
+
+// Try to extend the overlap on the B read. If successful, returns new bbgn,bend and editDist and alignLen.
+//
+bool
+extendAlignment(char *aRead, int32 abgn, int32 aend, int32 UNUSED(alen), char *Alabel, uint32 Aid,
+ char *bRead, int32 &bbgn, int32 &bend, int32 blen, char *Blabel, uint32 Bid,
+ double maxErate,
+ int32 slop,
+ int32 &editDist,
+ int32 &alignLen) {
+ alignStats threadStats;
+ EdlibAlignResult result = { 0, NULL, NULL, 0, NULL, 0, 0 };
+ bool success = false;
+
+ // Find an alignment, allowing extensions on the B read.
+
+ int32 bbgnExt = max(0, bbgn - slop);
+ int32 bendExt = min(blen, bend + slop);
+
+ // This probably isn't exactly correct, but close enough.
+ int32 maxEdit = (int32)ceil(max(aend - abgn, bendExt - bbgnExt) * maxErate * 1.1);
+
+ if (debug)
+ fprintf(stderr, " align %s %6u %6d-%-6d to %s %6u %6d-%-6d", Alabel, Aid, abgn, aend, Blabel, Bid, bbgnExt, bendExt);
+
+ result = edlibAlign(aRead + abgn, aend - abgn,
+ bRead + bbgnExt, bendExt - bbgnExt,
+ edlibNewAlignConfig(maxEdit, EDLIB_MODE_HW, EDLIB_TASK_LOC));
+
+ // Change the overlap for any extension found.
+
+ if (result.numLocations > 0) {
+ bbgn = bbgnExt + result.startLocations[0];
+ bend = bbgnExt + result.endLocations[0] + 1; // Edlib returns 0-based positions, add one to end to get space-based.
+
+ editDist = result.editDistance;
+ alignLen = result.alignmentLength; // Edlib 'alignmentLength' isn't populated for TASK_LOC, so we approximate it.
+ alignLen = ((aend - abgn) + (bend - bbgn) + (editDist)) / 2;
+
+ if (debug)
+ fprintf(stderr, " aligned to %s at %6d-%-6d editDist %5d alignLen %6d qual %6.4f\n",
+ Blabel, bbgn, bend, editDist, alignLen, 1.0 - editDist / (double)alignLen);
+
+ success = true;
+ } else {
+ if (debug)
+ fprintf(stderr, "\n");
+ }
+
+ edlibFreeAlignResult(result);
+
+ return(success);
+}
+
+
+
+bool
+finalAlignment(char *aRead, int32 alen,// char *Alabel, uint32 Aid,
+ char *bRead, int32 blen,// char *Blabel, uint32 Bid,
+ ovOverlap *ovl,
+ double maxErate,
+ int32 &editDist,
+ int32 &alignLen) {
+ EdlibAlignResult result = { 0, NULL, NULL, 0, NULL, 0, 0 };
+ bool success = false;
+
+ int32 abgn = (int32) ovl->dat.ovl.ahg5;
+ int32 aend = (int32)alen - ovl->dat.ovl.ahg3;
+ int32 bbgn = (int32) ovl->dat.ovl.bhg5;
+ int32 bend = (int32)blen - ovl->dat.ovl.bhg3;
+
+ int32 maxEdit = (int32)ceil(max(aend - abgn, bend - bbgn) * maxErate * 1.1);
+
+ result = edlibAlign(aRead + abgn, aend - abgn,
+ bRead + bbgn, bend - bbgn,
+ edlibNewAlignConfig(maxEdit, EDLIB_MODE_NW, EDLIB_TASK_LOC)); // NOTE! Global alignment.
+
+ if (result.numLocations > 0) {
+ editDist = result.editDistance;
+ alignLen = result.alignmentLength; // Edlib 'alignmentLength' isn't populated for TASK_LOC, so we approximate it.
+ alignLen = ((aend - abgn) + (bend - bbgn) + (editDist)) / 2;
+
+ success = true;
+ } else {
+ }
+
+ edlibFreeAlignResult(result);
+
+ return(success);
+}
+
+
+
void *
recomputeOverlaps(void *ptr) {
workSpace *WA = (workSpace *)ptr;
@@ -137,27 +349,46 @@ recomputeOverlaps(void *ptr) {
uint32 bgnID = 0;
uint32 endID = 0;
- uint32 nPassed = 0;
- uint32 nFailed = 0;
- uint32 nTested = 0;
-
- // Lazy allocation of the prefixEditDistance structure; it's slow.
-
- //if (WA->analyze == NULL)
- // WA->analyze = new analyzeAlignment();
-
while (getRange(bgnID, endID)) {
- double startTime = getTime();
+ alignStats localStats;
for (uint32 oo=bgnID; oo<endID; oo++) {
ovOverlap *ovl = WA->overlaps + oo;
+ // Swap IDs if requested (why would anyone want to do this?)
+
if (WA->invertOverlaps) {
ovOverlap swapped = WA->overlaps[oo];
WA->overlaps[oo].swapIDs(swapped); // Needs to be from a temporary!
}
+ // Initialize early, just so we can use goto.
+
+ uint32 aID = ovl->a_iid;
+ char *aRead = rcache->getRead(aID);
+ int32 alen = (int32)rcache->getLength(aID);
+ int32 abgn = (int32) ovl->dat.ovl.ahg5;
+ int32 aend = (int32)alen - ovl->dat.ovl.ahg3;
+
+ uint32 bID = ovl->b_iid;
+ char *bRead = WA->readSeq;
+ int32 blen = (int32)rcache->getLength(bID);
+ int32 bbgn = (int32) ovl->dat.ovl.bhg5;
+ int32 bend = (int32)blen - ovl->dat.ovl.bhg3;
+
+ int32 alignLen = 1;
+ int32 editDist = INT32_MAX;
+
+ EdlibAlignResult result = { 0, NULL, NULL, 0, NULL, 0, 0 };
+
+ if (debug) {
+ fprintf(stderr, "--------\n");
+ fprintf(stderr, "OLAP A %7u %6d-%-6d\n", aID, abgn, aend);
+ fprintf(stderr, " B %7u %6d-%-6d %s\n", bID, bbgn, bend, (ovl->flipped() == false) ? "" : " flipped");
+ fprintf(stderr, "\n");
+ }
+
// Invalidate the overlap.
ovl->evalue(AS_MAX_EVALUE);
@@ -165,209 +396,267 @@ recomputeOverlaps(void *ptr) {
ovl->dat.ovl.forDUP = false;
ovl->dat.ovl.forUTG = false;
+ // Make some bad changes, for testing
+#if 0
+ abgn += 100;
+ aend -= 100;
+ bbgn += 100;
+ bend -= 100;
+#endif
- uint32 aID = ovl->a_iid;
- uint32 bID = ovl->b_iid;
+ // Too short? Don't bother doing anything.
+ //
+ // Warning! Edlib failed on a 10bp to 10bp (extended to 5kbp) alignment.
- // Compute the overlap
- if (ovl->a_end() - ovl->a_bgn() + 1 < minOverlapLength && ovl->b_end() - ovl->b_bgn() + 1 < minOverlapLength) {
- continue;
+ if ((aend - abgn < minOverlapLength) ||
+ (bend - bbgn < minOverlapLength)) {
+ localStats.nSkipped++;
+ goto finished;
}
-#ifdef DEBUG
-nTested++;
-if (nTested % 1000 == 0) {
- double deltaTime = getTime() - startTime;
- fprintf(stderr, "*******Thread %2u computed overlaps %7u - %7u in %7.3f seconds - %6.2f olaps per second (%8u fail %8u pass)\n",
- WA->threadID, bgnID, endID, deltaTime, (endID - bgnID) / deltaTime, nFailed, nPassed);
-}
-#endif
- char *bRead = WA->readSeq;
- int32 astart = (int32)ovl->a_bgn();
- int32 aend = (int32)ovl->a_end();
- int32 astartExtended = max((int32)0, (int32)ovl->a_bgn() - MHAP_SLOP);
- int32 aendExtended = min((int32)rcache->getLength(aID), (int32)ovl->a_end() + MHAP_SLOP);
- int32 bstart = (int32)ovl->b_bgn();
- int32 bend = (int32)ovl->b_end();
- int32 bstartExtended = max((int32)0, (int32)ovl->b_bgn() - MHAP_SLOP);
- int32 bendExtended = min((int32)rcache->getLength(bID), (int32)ovl->b_end() + MHAP_SLOP);
- strcpy(bRead, rcache->getRead(bID));
- if (ovl->flipped()) {
- reverseComplementSequence(bRead, rcache->getLength(bID));
- bstart = (int32)rcache->getLength(bID) - (int32)ovl->b_bgn();
- bend = (int32)rcache->getLength(bID) - (int32)ovl->b_end();
- bstartExtended = max((int32)0, (int32)rcache->getLength(bID) - (int32)ovl->b_bgn() - MHAP_SLOP);
- bendExtended = min((int32)rcache->getLength(bID), (int32)rcache->getLength(bID) - (int32)ovl->b_end() + MHAP_SLOP);
- }
+ // Grab the B read sequence.
- int tolerance = (int)ceil((double)max(aendExtended-astartExtended, bendExtended-bstartExtended)*WA->maxErate*1.1);
- EdlibAlignResult bQuery = edlibAlign(rcache->getRead(aID)+astart, aend-astart, bRead+bstartExtended, bendExtended-bstartExtended, edlibNewAlignConfig(tolerance, EDLIB_MODE_HW, EDLIB_TASK_LOC));
- EdlibAlignResult aQuery = edlibAlign(bRead+bstart, bend-bstart, rcache->getRead(aID)+astartExtended, aendExtended-astartExtended, edlibNewAlignConfig(tolerance, EDLIB_MODE_HW, EDLIB_TASK_LOC));
+ strcpy(bRead, rcache->getRead(bID));
- uint32 alignmentLength = 0;
- double dist = 0;
+ // If flipped, reverse complement the B read.
-#ifdef DEBUG
-fprintf(stderr, "Overlap between %d and %d at %d found %d %d hits\n", aID, bID, tolerance, bQuery.numLocations, aQuery.numLocations);
-#endif
- if (aQuery.numLocations >= 1 || bQuery.numLocations >= 1) {
- // if we couldn't find one of the options, try trimming and re-computing
- if (bQuery.numLocations == 0) {
- ovl->dat.ovl.ahg5 = aQuery.startLocations[0] + astartExtended;
- ovl->dat.ovl.ahg3 = rcache->getLength(aID) - (aQuery.endLocations[0] + astartExtended + 1);
- alignmentLength = max(alignmentLength, (uint32)(aQuery.endLocations[0] - aQuery.startLocations[0]));
- dist = min(aQuery.editDistance, (int)dist);
- edlibFreeAlignResult(bQuery);
- bQuery = edlibAlign(rcache->getRead(aID)+astart, aend-astart, bRead+bstartExtended, bendExtended-bstartExtended, edlibNewAlignConfig(tolerance, EDLIB_MODE_HW, EDLIB_TASK_LOC));
- }
- if (aQuery.numLocations == 0) {
- ovl->dat.ovl.bhg5 = bQuery.startLocations[0] + bstartExtended;
- ovl->dat.ovl.bhg3 = rcache->getLength(bID) - (bQuery.endLocations[0] + bstartExtended + 1);
- alignmentLength = bQuery.endLocations[0] - bQuery.startLocations[0];
- dist = bQuery.editDistance;
- edlibFreeAlignResult(aQuery);
- aQuery = edlibAlign(bRead+bstart, bend-bstart, rcache->getRead(aID)+astartExtended, aendExtended-astartExtended, edlibNewAlignConfig(tolerance, EDLIB_MODE_HW, EDLIB_TASK_LOC));
- }
-
- // now update the trim points based on where the overlapping broke
- // the aligner computes 0-based end positions so for matching ACGTA to ACTGTA positiosn are 0-4 so we need to adjust for that
- if (bQuery.numLocations >= 1) {
- ovl->dat.ovl.bhg5 = bQuery.startLocations[0] + bstartExtended;
- ovl->dat.ovl.bhg3 = rcache->getLength(bID) - (bQuery.endLocations[0] + bstartExtended + 1);
- alignmentLength = bQuery.endLocations[0] - bQuery.startLocations[0];
- dist = bQuery.editDistance;
- }
- if (aQuery.numLocations >= 1) {
- ovl->dat.ovl.ahg5 = aQuery.startLocations[0] + astartExtended;
- ovl->dat.ovl.ahg3 = rcache->getLength(aID) - (aQuery.endLocations[0] + astartExtended + 1);
- alignmentLength = max(alignmentLength, (uint32)(aQuery.endLocations[0] - aQuery.startLocations[0]));
- dist = min(aQuery.editDistance, (int)dist);
- }
- edlibFreeAlignResult(aQuery);
- edlibFreeAlignResult(bQuery);
-
-#ifdef DEBUG
-fprintf(stderr, "Expected overlap between %d and %d from %d - %d and %d - %d found overlap from %d - %d and %d - %d length %d dist %f\n", aID, bID, astart, aend, bstart, bend, ovl->a_bgn(), ovl->a_end(), ovl->b_bgn(), ovl->b_end(), alignmentLength, dist);
-#endif
+ if (ovl->flipped() == true)
+ reverseComplementSequence(bRead, blen);
- bool changed = true;
- tolerance = (int)(alignmentLength * WA->maxErate) + 1;
- EdlibAlignResult result;
+ //
+ // Find initial alignments, allowing one, then the other, sequence to be extended as needed.
+ //
- // extend to the ends if we are not looking for partial and we can, don't extend contains
- if (changed && WA->partialOverlaps == false && !ovl->overlapIsDovetail()) {
- bstart = ovl->flipped() ? rcache->getLength(bID) - ovl->b_bgn() : ovl->b_bgn();
- bend = ovl->flipped() ? rcache->getLength(bID) - ovl->b_end() : ovl->b_end();
-#ifdef DEBUG
-fprintf(stderr, "Overlap %d %d (%d %d) invert %d is %f error and not dovetail with %d %d and %d %d\n", aID, bID, rcache->getLength(aID) , rcache->getLength(bID), ovl->flipped(), dist/*result.editDistance*/, ovl->dat.ovl.ahg5, ovl->dat.ovl.ahg3, ovl->dat.ovl.bhg5, ovl->dat.ovl.bhg3);
-#endif
- // dist = result.editDistance;
- // check these cases one by one and extend both concordantly with each other
- // first is a contained in b
- if (rcache->getLength(aID) <= rcache->getLength(bID) && ovl->dat.ovl.ahg5 >= 0 && ovl->dat.ovl.ahg3 >= 0 && ovl->dat.ovl.bhg5 >= ovl->dat.ovl.ahg5 && ovl->dat.ovl.bhg3 >= ovl->dat.ovl.ahg3 && ((double)(ovl->dat.ovl.ahg5 + ovl->dat.ovl.ahg3 + dist) / ((double)(alignmentLength + ovl->dat.ovl.ahg5 + ovl->dat.ovl.ahg3))) <= WA->maxErate) {
- ovl->dat.ovl.bhg5 = max(0, ovl->dat.ovl.bhg5 - ovl->dat.ovl.ahg5); ovl->dat.ovl.ahg5 = 0;
- ovl->dat.ovl.bhg3 = max(0, ovl->dat.ovl.bhg3 - ovl->dat.ovl.ahg3); ovl->dat.ovl.ahg3 = 0;
- changed = true;
-#ifdef DEBUG
-fprintf(stderr, "Overlap %d %d case 1 acontained\n", aID, bID);
-#endif
- }
- // second is b contained (both b hangs can be extended)
- //
- else if (rcache->getLength(aID) >= rcache->getLength(bID) && ovl->dat.ovl.bhg5 >= 0 && ovl->dat.ovl.bhg3 >= 0 && ovl->dat.ovl.ahg5 >= ovl->dat.ovl.bhg5 && ovl->dat.ovl.ahg3 >= ovl->dat.ovl.bhg3 && ((double)(ovl->dat.ovl.bhg5 + ovl->dat.ovl.bhg3 + dist) / ((double)(alignmentLength + ovl->dat.ovl.bhg5 + ovl->dat.ovl.bhg3))) <= WA->maxErate) {
- ovl->dat.ovl.ahg5 = max(0, ovl->dat.ovl.ahg5 - ovl->dat.ovl.bhg5); ovl->dat.ovl.bhg5 = 0;
- ovl->dat.ovl.ahg3 = max(0, ovl->dat.ovl.ahg3 - ovl->dat.ovl.bhg3); ovl->dat.ovl.bhg3 = 0;
- changed = true;
-#ifdef DEBUG
-fprintf(stderr, "Overlap %d %d case 2 bconatined\n", aID, bID);
-#endif
+ if (extendAlignment(bRead, bbgn, bend, blen, "B", bID,
+ aRead, abgn, aend, alen, "A", aID,
+ WA->maxErate, MHAP_SLOP,
+ editDist,
+ alignLen) == false) {
+ localStats.nFailExtA++;
+ }
+
+ if (extendAlignment(aRead, abgn, aend, alen, "A", aID,
+ bRead, bbgn, bend, blen, "B", bID,
+ WA->maxErate, MHAP_SLOP,
+ editDist,
+ alignLen) == false) {
+ localStats.nFailExtB++;
+ }
+
+ // If no alignments were found, fail.
+
+ if (alignLen == 1) {
+ localStats.nFailExt++;
+ goto finished;
+ }
+
+ // Update the overlap.
+
+ ovl->dat.ovl.ahg5 = abgn;
+ ovl->dat.ovl.ahg3 = alen - aend;
+
+ ovl->dat.ovl.bhg5 = bbgn;
+ ovl->dat.ovl.bhg3 = blen - bend;
+
+ if (debug) {
+ fprintf(stderr, "\n");
+ fprintf(stderr, "init A %7u %6d-%-6d\n", aID, abgn, aend);
+ fprintf(stderr, " B %7u %6d-%-6d\n", bID, bbgn, bend);
+ fprintf(stderr, "\n");
+ }
+
+ // If we're just doing partial alignments or if we've found a dovetail, we're all done.
+
+ if (WA->partialOverlaps == true) {
+ localStats.nPartial++;
+ goto finished;
+ }
+
+ if (ovl->overlapIsDovetail() == true) {
+ localStats.nDovetail++;
+ goto finished;
+ }
+
+#warning do we need to check for contained too?
+
+
+
+ // Otherwise, try to extend the alignment to make a dovetail overlap.
+
+ {
+ int32 ahg5 = ovl->dat.ovl.ahg5;
+ int32 ahg3 = ovl->dat.ovl.ahg3;
+
+ int32 bhg5 = ovl->dat.ovl.bhg5;
+ int32 bhg3 = ovl->dat.ovl.bhg3;
+
+ int32 slop = 0;
+
+ if ((ahg5 >= bhg5) && (bhg5 > 0)) {
+ //fprintf(stderr, "extend 5' by B=%d\n", bhg5);
+ ahg5 -= bhg5;
+ bhg5 -= bhg5; // Now zero.
+ slop = bhg5 * WA->maxErate + 100;
+
+ abgn = (int32) ahg5;
+ aend = (int32)alen - ahg3;
+
+ bbgn = (int32) bhg5;
+ bend = (int32)blen - bhg3;
+
+ if (extendAlignment(bRead, bbgn, bend, blen, "Bb5", bID,
+ aRead, abgn, aend, alen, "Ab5", aID,
+ WA->maxErate, slop,
+ editDist,
+ alignLen) == true) {
+ ahg5 = abgn;
+ //ahg3 = alen - aend;
+ } else {
+ ahg5 = ovl->dat.ovl.ahg5;
+ bhg5 = ovl->dat.ovl.bhg5;
+ }
+ localStats.nExt5b++;
}
- // third is 5' dovetal ---------->
- // ---------->
- // or
- // <---------
- // bhg5 here is always first overhang on b read
- //
- else if (ovl->dat.ovl.ahg3 <= ovl->dat.ovl.bhg3 && (ovl->dat.ovl.ahg3 >= 0 && ((double)(ovl->dat.ovl.ahg3 + dist) / ((double)(alignmentLength + ovl->dat.ovl.ahg3))) <= WA->maxErate) &&
- (ovl->dat.ovl.bhg5 >= 0 && ((double)(ovl->dat.ovl.bhg5 + dist) / ((double)(alignmentLength + ovl->dat.ovl.bhg5))) <= WA->maxErate)) {
- ovl->dat.ovl.ahg5 = max(0, ovl->dat.ovl.ahg5 - ovl->dat.ovl.bhg5); ovl->dat.ovl.bhg5 = 0;
- ovl->dat.ovl.bhg3 = max(0, ovl->dat.ovl.bhg3 - ovl->dat.ovl.ahg3); ovl->dat.ovl.ahg3 = 0;
- changed = true;
-#ifdef DEBUG
-fprintf(stderr, "Overlap %d %d case 3 5' dovetail \n", aID, bID);
-#endif
+
+ if ((bhg5 >= ahg5) && (ahg5 > 0)) {
+ //fprintf(stderr, "extend 5' by A=%d\n", ahg5);
+ bhg5 -= ahg5;
+ ahg5 -= ahg5; // Now zero.
+ slop = ahg5 * WA->maxErate + 100;
+
+ abgn = (int32) ahg5;
+ aend = (int32)alen - ahg3;
+
+ bbgn = (int32) bhg5;
+ bend = (int32)blen - bhg3;
+
+ if (extendAlignment(aRead, abgn, aend, alen, "Aa5", aID,
+ bRead, bbgn, bend, blen, "Ba5", bID,
+ WA->maxErate, slop,
+ editDist,
+ alignLen) == true) {
+ bhg5 = bbgn;
+ //bhg3 = blen - bend;
+ } else {
+ bhg5 = ovl->dat.ovl.bhg5;
+ ahg5 = ovl->dat.ovl.ahg5;
+ }
+ localStats.nExt5a++;
}
- //
- // fourth is 3' dovetail ---------->
- // ---------->
- // or
- // <----------
- // bhg5 is always first overhang on b read
- else if (ovl->dat.ovl.ahg5 <= ovl->dat.ovl.bhg5 && (ovl->dat.ovl.ahg5 >= 0 && ((double)(ovl->dat.ovl.ahg5 + dist) / ((double)(alignmentLength + ovl->dat.ovl.ahg5))) <= WA->maxErate) &&
- (ovl->dat.ovl.bhg3 >= 0 && ((double)(ovl->dat.ovl.bhg3 + dist) / ((double)(alignmentLength + ovl->dat.ovl.bhg3))) <= WA->maxErate)) {
- ovl->dat.ovl.bhg5 = max(0, ovl->dat.ovl.bhg5 - ovl->dat.ovl.ahg5); ovl->dat.ovl.ahg5 = 0;
- ovl->dat.ovl.ahg3 = max(0, ovl->dat.ovl.ahg3 - ovl->dat.ovl.bhg3); ovl->dat.ovl.bhg3 = 0;
- changed = true;
-#ifdef DEBUG
-fprintf(stderr, "Overlap %d %d case 4 3' dovetail \n", aID, bID);
-#endif
+
+
+
+ if ((bhg3 >= ahg3) && (ahg3 > 0)) {
+ //fprintf(stderr, "extend 3' by A=%d\n", ahg3);
+ bhg3 -= ahg3;
+ ahg3 -= ahg3; // Now zero.
+ slop = ahg3 * WA->maxErate + 100;
+
+ abgn = (int32) ahg5;
+ aend = (int32)alen - ahg3;
+
+ bbgn = (int32) bhg5;
+ bend = (int32)blen - bhg3;
+
+ if (extendAlignment(aRead, abgn, aend, alen, "Aa3", aID,
+ bRead, bbgn, bend, blen, "Ba3", bID,
+ WA->maxErate, slop,
+ editDist,
+ alignLen) == true) {
+ //bhg5 = bbgn;
+ bhg3 = blen - bend;
+ } else {
+ bhg3 = ovl->dat.ovl.bhg3;
+ ahg3 = ovl->dat.ovl.ahg3;
+ }
+ localStats.nExt3a++;
}
- }
-#ifdef DEBUG
-fprintf(stderr, "Recomputed overlap from %d to %d is %d - %d and %d - %d\n", aID, bID, ovl->a_bgn(), ovl->a_end(), (ovl->flipped() ? rcache->getLength(bID) - ovl->b_bgn() : ovl->b_bgn()), (ovl->flipped() ? rcache->getLength(bID) - ovl->b_end() : ovl->b_end()));
-#endif
- // now compute the final
- if (changed) {
- bstart = ovl->flipped() ? rcache->getLength(bID) - ovl->b_bgn() : ovl->b_bgn();
- bend = ovl->flipped() ? rcache->getLength(bID) - ovl->b_end() : ovl->b_end();
- result = edlibAlign(rcache->getRead(aID)+ovl->a_bgn(), ovl->a_end()-ovl->a_bgn(), bRead+bstart, bend-bstart, edlibNewAlignConfig(tolerance, EDLIB_MODE_NW, EDLIB_TASK_LOC));
- if (result.numLocations >= 1) {
- dist = result.editDistance;
- alignmentLength = ovl->a_end() - ovl->a_bgn();
- } else {
- dist = ovl->a_end() - ovl->a_bgn();
- alignmentLength = 0;
+ if ((ahg3 >= bhg3) && (bhg3 > 0)) {
+ //fprintf(stderr, "extend 3' by B=%d\n", bhg3);
+ ahg3 -= bhg3;
+ bhg3 -= bhg3; // Now zero.
+ slop = bhg3 * WA->maxErate + 100;
+
+ abgn = (int32) ahg5;
+ aend = (int32)alen - ahg3;
+
+ bbgn = (int32) bhg5;
+ bend = (int32)blen - bhg3;
+
+ if (extendAlignment(bRead, bbgn, bend, blen, "Bb3", bID,
+ aRead, abgn, aend, alen, "Ab3", aID,
+ WA->maxErate, slop,
+ editDist,
+ alignLen) == true) {
+ //ahg5 = abgn;
+ ahg3 = alen - aend;
+ } else {
+ ahg3 = ovl->dat.ovl.ahg3;
+ bhg3 = ovl->dat.ovl.bhg3;
+ }
+ localStats.nExt3b++;
}
- edlibFreeAlignResult(result);
- }
-#ifdef DEBUG
-fprintf(stderr, "Done and error rate for this overlap between %d and %d is %d bp and %f errors is dovetail %d\n", aID, bID, alignmentLength, dist, ovl->overlapIsDovetail());
-#endif
- } else {
- edlibFreeAlignResult(aQuery);
- edlibFreeAlignResult(bQuery);
- }
- if (alignmentLength >= minOverlapLength && (dist / (double) (alignmentLength)) <= WA->maxErate) {
- nPassed++;
+ // Now reset the overlap.
- ovl->erate((double)dist/(alignmentLength));
- ovl->dat.ovl.forOBT = (WA->partialOverlaps == true);
- ovl->dat.ovl.forDUP = (WA->partialOverlaps == true);
- ovl->dat.ovl.forUTG = (WA->partialOverlaps == false) && (ovl->overlapIsDovetail() == true);
+ ovl->dat.ovl.ahg5 = ahg5;
+ ovl->dat.ovl.ahg3 = ahg3;
- } else {
- nFailed++;
- ovl->evalue(AS_MAX_EVALUE);
+ ovl->dat.ovl.bhg5 = bhg5;
+ ovl->dat.ovl.bhg3 = bhg3;
+ } // If not a contained overlap
+
+
+ // If we're still not dovetail, nothing more we want to do. Let the overlap be trashed.
+
+
+ if (debug) {
+ fprintf(stderr, "\n");
+ fprintf(stderr, "fini A %7u %6d-%-6d %d %d\n", aID, abgn, aend, ovl->a_bgn(), ovl->a_end());
+ fprintf(stderr, " B %7u %6d-%-6d %d %d %s\n", bID, bbgn, bend, ovl->b_bgn(), ovl->b_end(), (ovl->flipped() == false) ? "" : " flipped");
+ fprintf(stderr, "\n");
+ }
+
+ finalAlignment(aRead, alen,// "A", aID,
+ bRead, blen,// "B", bID,
+ ovl, WA->maxErate, editDist, alignLen);
+
+
+ finished:
+
+ // Trash the overlap if it's junky quality.
+
+ double eRate = editDist / (double)alignLen;
+
+ if ((alignLen < minOverlapLength) ||
+ (eRate > WA->maxErate)) {
+ localStats.nFailed++;
+ ovl->evalue(AS_MAX_EVALUE);
ovl->dat.ovl.forOBT = false;
ovl->dat.ovl.forDUP = false;
ovl->dat.ovl.forUTG = false;
+
+ } else {
+ localStats.nPassed++;
+ ovl->erate(eRate);
+ ovl->dat.ovl.forOBT = (WA->partialOverlaps == true);
+ ovl->dat.ovl.forDUP = (WA->partialOverlaps == true);
+ ovl->dat.ovl.forUTG = (WA->partialOverlaps == false) && (ovl->overlapIsDovetail() == true);
}
- }
-#ifdef DEBUG
- double deltaTime = getTime() - startTime;
- fprintf(stderr, "Thread %2u computed overlaps %7u - %7u in %7.3f seconds - %6.2f olaps per second (%8u fail %8u pass)\n",
- WA->threadID, bgnID, endID, deltaTime, (endID - bgnID) / deltaTime, nFailed, nPassed);
-#endif
- }
+ } // Over all overlaps in this range
- // Report. The last batch has no work to do.
- if (nFailed + nPassed > 0)
- fprintf(stderr, "Thread %u finished -- %u failed %u passed.\n", WA->threadID, nFailed, nPassed);
+ // Log that we've done stuff
+
+ pthread_mutex_lock(&balanceMutex);
+ globalStats += localStats;
+ globalStats.reportStatus();
+ localStats.clear();
+ pthread_mutex_unlock(&balanceMutex);
+ } // Over all ranges
return(NULL);
}
@@ -430,7 +719,7 @@ main(int argc, char **argv) {
memLimit = atoi(argv[++arg]);
} else if (strcmp(argv[arg], "-len") == 0) {
- minOverlapLength = atoi(argv[++arg]);
+ minOverlapLength = atoi(argv[++arg]);
} else {
err++;
@@ -513,18 +802,18 @@ main(int argc, char **argv) {
// Initialize thread work areas. Mirrored from overlapInCore.C
for (uint32 tt=0; tt<numThreads; tt++) {
- fprintf(stderr, "Initialize thread %u\n", tt);
+ fprintf(stderr, "Initialize thread %u\n", tt);
- WA[tt].threadID = tt;
- WA[tt].maxErate = maxErate;
- WA[tt].partialOverlaps = partialOverlaps;
- WA[tt].invertOverlaps = invertOverlaps;
+ WA[tt].threadID = tt;
+ WA[tt].maxErate = maxErate;
+ WA[tt].partialOverlaps = partialOverlaps;
+ WA[tt].invertOverlaps = invertOverlaps;
- WA[tt].gkpStore = gkpStore;
- WA[tt].overlaps = NULL;
+ WA[tt].gkpStore = gkpStore;
+ WA[tt].overlaps = NULL;
- // preallocate some work thread memory for common tasks to avoid allocation
- WA[tt].readSeq = new char[AS_MAX_READLEN+1];
+ // preallocate some work thread memory for common tasks to avoid allocation
+ WA[tt].readSeq = new char[AS_MAX_READLEN+1];
}
@@ -644,6 +933,10 @@ main(int argc, char **argv) {
rcache->purgeReads();
}
+ // Report. The last batch has no work to do.
+
+ globalStats.reportFinal();
+
// Goodbye.
delete rcache;
@@ -662,6 +955,9 @@ main(int argc, char **argv) {
delete [] WA;
delete [] tID;
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Bye.\n");
+
return(0);
}
diff --git a/src/pipelines/canu-object-store.pl b/src/pipelines/canu-object-store.pl
new file mode 100755
index 0000000..09be0c5
--- /dev/null
+++ b/src/pipelines/canu-object-store.pl
@@ -0,0 +1,152 @@
+#!/usr/bin/env perl
+
+###############################################################################
+ #
+ # This file is part of canu, a software program that assembles whole-genome
+ # sequencing reads into contigs.
+ #
+ # This software is based on:
+ # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ # the 'kmer package' (http://kmer.sourceforge.net)
+ # both originally distributed by Applera Corporation under the GNU General
+ # Public License, version 2.
+ #
+ # Canu branched from Celera Assembler at its revision 4587.
+ # Canu branched from the kmer project at its revision 1994.
+ #
+ # Modifications by:
+ #
+ # Brian P. Walenz beginning on 2017-MAR-14
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
+ # File 'README.licenses' in the root directory of this distribution contains
+ # full conditions and disclaimers for each license.
+ ##
+
+use strict;
+use File::Basename;
+
+# A simple wrapper to emulate an object store. Implements:
+# describe --name <name> - prints <name> if the store has the named object, nothing otherwise
+# upload --path <name> <file> - uploads local <file> to the store as object <name>
+# download --output <file> <name> - downloads object <name> into <file>
+#
+# As seen immediately below, requires a magic hardcoded path. Files are copied to this directory
+# to become the objects.
+
+my $STASH;
+
+$STASH = "/assembly/STASH" if (-d "/assembly/STASH");
+$STASH = "/Users/walenzbp/STASH" if (-d "/Users/walenzbp/STASH");
+
+die "No STASH found\n" if (!defined($STASH));
+
+my $task = shift @ARGV;
+
+
+
+if ($task eq "describe") {
+ my $file;
+ my $path = "";
+ my $wait = 0;
+
+ while (scalar(@ARGV) > 0) {
+ my $arg = shift @ARGV;
+
+ if ($arg eq "--name") {
+ $path = shift @ARGV;
+ }
+
+ else {
+ die "Unknown option $arg\n";
+ }
+ }
+
+ if (-e "$STASH/$path") {
+ print "$path\n";
+ }
+}
+
+
+
+if ($task eq "upload") {
+ my $file;
+ my $path;
+ my $wait = 0;
+
+ while (scalar(@ARGV) > 0) {
+ my $arg = shift @ARGV;
+
+ if ($arg eq "--path") {
+ $path = shift @ARGV;
+ }
+
+ elsif ($arg eq "--wait") {
+ $wait = 1;
+ }
+
+ elsif (!defined($file)) {
+ $file = $arg;
+ }
+
+ else {
+ die "Unknown option $arg\n";
+ }
+ }
+
+ # Copy local file $file, assumed to be in this directory, to the stash as $path.
+
+ die "dx upload - no stash path supplied.\n" if (!defined($path));
+ die "dx upload - no input file supplied.\n" if (!defined($file));
+ die "dx upload - input file $file not found.\n" if (($file ne "-") && (! -e $file));
+
+ system("mkdir -p $STASH/" . dirname($path));
+
+ if ($file eq "-") {
+ system("dd status=none of=$STASH/$path");
+ } else {
+ system("cp -fp $file $STASH/$path");
+ }
+}
+
+
+
+if ($task eq "download") {
+ my $file;
+ my $path;
+ my $wait = 0;
+
+ while (scalar(@ARGV) > 0) {
+ my $arg = shift @ARGV;
+
+ if ($arg eq "--output") {
+ $file = shift @ARGV;
+ }
+
+ elsif (!defined($path)) {
+ $path = $arg;
+ }
+
+ else {
+ die "Unknown option $arg\n";
+ }
+ }
+
+ # Copy local file $file, assumed to be in this directory, to the stash as $path.
+
+ die "dx download - no stash path supplied.\n" if (!defined($path));
+ die "dx download - no output file supplied.\n" if (!defined($file));
+ #die "dx download - stash path $path not found.\n" if (! -e "$STASH/$path");
+
+ exit(0) if (! -e "$STASH/$path");
+
+ if ($file eq "-") {
+ system("dd status=none if=$STASH/$path");
+ } else {
+ system("cp -fp $STASH/$path $file");
+ }
+}
+
+
+exit(0);
diff --git a/src/pipelines/canu.pl b/src/pipelines/canu.pl
index b0c4e9e..e15d27a 100644
--- a/src/pipelines/canu.pl
+++ b/src/pipelines/canu.pl
@@ -39,13 +39,13 @@
use strict;
use FindBin;
-use Cwd;
+use Cwd qw(getcwd abs_path);
use lib "$FindBin::RealBin/lib";
use lib "$FindBin::RealBin/lib/canu/lib/perl5";
use lib "$FindBin::RealBin/lib/canu/lib64/perl5";
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use Carp;
@@ -55,10 +55,12 @@ use canu::Execution;
use canu::Configure;
use canu::Grid;
+use canu::Grid_Cloud;
use canu::Grid_SGE;
use canu::Grid_Slurm;
use canu::Grid_PBSTorque;
use canu::Grid_LSF;
+use canu::Grid_DNANexus;
use canu::Gatekeeper;
use canu::Meryl;
@@ -92,12 +94,21 @@ setDefaults();
my $bin = getBinDirectory(); # Path to binaries, reset later.
my $cmd = undef; # Temporary string passed to system().
-my $wrk = undef; # Path to our assembly directory.
my $asm = undef; # Name of our assembly.
+# What a mess. We can't set the version string until after we have a bin directory, and
+# Defaults.pm can't call stuff in Execution.pm. So, we need to special case setting the version
+# string.
+
+setVersion($bin);
+
# Check for the presence of a -options switch BEFORE we do any work.
# This lets us print the default values of options.
+if (scalar(@ARGV) == 0) {
+ printHelp(1);
+}
+
foreach my $arg (@ARGV) {
if (($arg eq "-options") ||
($arg eq "-defaults")) {
@@ -107,7 +118,7 @@ foreach my $arg (@ARGV) {
if (($arg eq "-version") ||
($arg eq "--version")) {
- system("$bin/gatekeeperCreate --version");
+ print getGlobal("version") . "\n";
exit(0);
}
}
@@ -119,32 +130,33 @@ foreach my $arg (@ARGV) {
# to use these when we resubmit ourself to the grid. We can't simply dump
# all of @ARGV into here, because we need to fix up relative paths first.
-my $mode = undef;
-my $step = "run";
-my $haveRaw = 0;
+my $rootdir = undef;
+my $readdir = undef;
+my $mode = undef;
+my $step = "run";
+my $haveRaw = 0;
my $haveCorrected = 0;
while (scalar(@ARGV)) {
my $arg = shift @ARGV;
- if ($arg =~ m/^-d/) {
- $wrk = shift @ARGV;
- $wrk = "$ENV{'PWD'}/$wrk" if ($wrk !~ m!^/!);
- addCommandLineOption("-d \"$wrk\"");
- setGlobal("onExitDir", $wrk);
+ if (($arg eq "-h") || ($arg eq "-help") || ($arg eq "--help")) {
+ printHelp(1);
+
+ } elsif ($arg eq "-d") {
+ $rootdir = shift @ARGV;
} elsif ($arg eq "-p") {
$asm = shift @ARGV;
- addCommandLineOption("-p \"$asm\"");
- setGlobal("onExitNam", $asm);
+ addCommandLineOption("-p '$asm'");
} elsif ($arg eq "-s") {
my $spec = shift @ARGV;
- $spec = "$ENV{'PWD'}/$spec" if ($spec !~ m!^/!);
+ $spec = abs_path($spec);
push @specFiles, $spec;
- addCommandLineOption("-s \"$spec\"");
+ addCommandLineOption("-s '$spec'");
} elsif ($arg eq "-correct") {
$mode = $step = "correct";
@@ -162,19 +174,26 @@ while (scalar(@ARGV)) {
$mode = $step = "trim-assemble";
addCommandLineOption("-trim-assemble");
+ } elsif ($arg eq "-readdir") {
+ $readdir = shift @ARGV;
+ addCommandLineOption("-readdir '$readdir'");
+
} elsif (($arg eq "-pacbio-raw") || # File handling is also present in
($arg eq "-pacbio-corrected") || # Defaults.pm around line 438
($arg eq "-nanopore-raw") ||
($arg eq "-nanopore-corrected")) {
- addCommandLineError("ERROR: File '$ARGV[0]' not found.\n") if (! -e $ARGV[0]);
- while (-e $ARGV[0]) {
- my $file = shift @ARGV;
+ my $file = $ARGV[0];
+ my $fopt = addSequenceFile($readdir, $file, 1);
- $file = "$ENV{'PWD'}/$file" if ($file !~ m!^/!);
+ while (defined($fopt)) {
+ push @inputFiles, "$arg\0$fopt";
+ addCommandLineOption("$arg '$fopt'");
- push @inputFiles, "$arg\0$file";
- addCommandLineOption("$arg \"$file\"");
+ shift @ARGV;
+
+ $file = $ARGV[0];
+ $fopt = addSequenceFile($readdir, $file);
}
} elsif (-e $arg) {
@@ -182,9 +201,10 @@ while (scalar(@ARGV)) {
} elsif ($arg =~ m/=/) {
push @specOpts, $arg;
- addCommandLineOption("\"$arg\"");
+ addCommandLineOption("'$arg'");
} else {
+ print STDERR "INVALID $arg\n";
addCommandLineError("ERROR: Invalid command line option '$arg'. Did you forget quotes around options with spaces?\n");
}
}
@@ -192,24 +212,23 @@ while (scalar(@ARGV)) {
# Fail if some obvious things aren't set.
addCommandLineError("ERROR: Assembly name prefix not supplied with -p.\n") if (!defined($asm));
-addCommandLineError("ERROR: Directory not supplied with -d.\n") if (!defined($wrk));
# If the mode isn't set - which is allowed only if a gkpStore exists somewhere - be a little smart
# and figure out which store exists.
-$mode = "run" if (!defined($mode) && (-d "$wrk/correction/$asm.gkpStore"));
-$mode = "trim-assemble" if (!defined($mode) && (-d "$wrk/trimming/$asm.gkpStore"));
-$mode = "assemble" if (!defined($mode) && (-d "$wrk/unitigging/$asm.gkpStore"));
+$mode = "run" if (!defined($mode) && (-d "correction/$asm.gkpStore"));
+$mode = "trim-assemble" if (!defined($mode) && (-d "trimming/$asm.gkpStore"));
+$mode = "assemble" if (!defined($mode) && (-d "unitigging/$asm.gkpStore"));
# Load paramters from the defaults files
- at inputFiles = setParametersFromFile("$bin/canu.defaults", @inputFiles) if (-e "$bin/canu.defaults");
- at inputFiles = setParametersFromFile("$ENV{'HOME'}/.canu", @inputFiles) if (-e "$ENV{'HOME'}/.canu");
+ at inputFiles = setParametersFromFile("$bin/canu.defaults", $readdir, @inputFiles) if (-e "$bin/canu.defaults");
+ at inputFiles = setParametersFromFile("$ENV{'HOME'}/.canu", $readdir, @inputFiles) if (-e "$ENV{'HOME'}/.canu");
# For each of the spec files, parse it, setting parameters and remembering any input files discovered.
foreach my $specFile (@specFiles) {
- @inputFiles = setParametersFromFile($specFile, @inputFiles);
+ @inputFiles = setParametersFromFile($specFile, $readdir, @inputFiles);
}
# Set parameters from the command line.
@@ -218,64 +237,77 @@ setParametersFromCommandLine(@specOpts);
# Set parameters based on file types supplied.
+my $setUpForPacBio = 0;
+my $setUpForNanopore = 0;
+
foreach my $typefile (@inputFiles) {
my ($type, $file) = split '\0', $typefile;
- $mode = "trim-assemble" if (!defined($mode) && ($type =~ m/corrected/));
- $mode = "run" if (!defined($mode) && ($type =~ m/raw/));
-
- $haveCorrected = 1 if ($type =~ m/corrected/);
- $haveRaw = 1 if ($type =~ m/raw/);
+ $mode = "trim-assemble" if (!defined($mode) && ($type =~ m/corrected/));
+ $mode = "run" if (!defined($mode) && ($type =~ m/raw/));
- setErrorRate(0.015, 0) if ($type =~ m/pacbio/);
- setGlobal("corErrorRate", "0.30") if ($type =~ m/pacbio/);
+ $haveCorrected = 1 if ($type =~ m/corrected/);
+ $haveRaw = 1 if ($type =~ m/raw/);
- setErrorRate(0.048, 0) if ($type =~ m/nanopore/);
- setGlobal("corErrorRate", "0.50") if ($type =~ m/nanopore/);
+ $setUpForPacBio++ if ($type =~ m/pacbio/);
+ $setUpForNanopore++ if ($type =~ m/nanopore/);
}
# Fail if both raw and corrected are supplied.
addCommandLineError("ERROR: Canu does not currently support mixing raw and corrected sequences.\n") if ($haveRaw && $haveCorrected);
+# If anything complained (invalid option, missing file, etc) printHelp() will trigger and exit.
+
+printHelp();
+
# When resuming a run without input files, set the error rates based on library type in the
-# gkpStore. If the user set the error rate already, do nothing.
-#
-# Also, check if we have gkpStores but no input files and reset error rates based on gkpStore.
+# gkpStore.
-if (scalar(@inputFiles) == 0 && ! defined(getGlobal("errorRate"))) {
+if (scalar(@inputFiles) == 0) {
my $gkpStore = undef;
- $gkpStore = "$wrk/correction/$asm.gkpStore" if -e "$wrk/correction/$asm.gkpStore/libraries.txt";
- $gkpStore = "$wrk/trimming/$asm.gkpStore" if -e "$wrk/trimming/$asm.gkpStore/libraries.txt";
- $gkpStore = "$wrk/unitigging/$asm.gkpStore" if -e "$wrk/unitigging/$asm.gkpStore/libraries.txt";
- # set to the default if we can't find anything
- if (!defined($gkpStore)) {
- setErrorRate(0.01, 0);
- } else {
- my $numPacBioRaw = 0;
- my $numPacBioCorrected = 0;
- my $numNanoporeRaw = 0;
- my $numNanoporeCorrected = 0;
-
- open(L, "< $gkpStore/libraries.txt") or caExit("can't open '$gkpStore/libraries.txt' for reading: $!", undef);
- while (<L>) {
- $numPacBioRaw++ if (m/pacbio-raw/);
- $numPacBioCorrected++ if (m/pacbio-corrected/);
- $numNanoporeRaw++ if (m/nanopore-raw/);
- $numNanoporeCorrected++ if (m/nanopore-corrected/);
- }
- if ($numPacBioRaw > 0 || $numPacBioCorrected > 0) {
- setErrorRate(0.015, 0);
- setGlobal("corErrorRate", "0.30");
- setGlobal("cnsMaxCoverage", 40);
- }
- if ($numNanoporeRaw > 0 || $numNanoporeCorrected > 0) {
- setErrorRate(0.048, 0);
- setGlobal("corErrorRate", "0.50");
- setGlobal("cnsMaxCoverage", 40);
- }
+ $gkpStore = "correction/$asm.gkpStore" if (-e "correction/$asm.gkpStore/libraries.txt");
+ $gkpStore = "trimming/$asm.gkpStore" if (-e "trimming/$asm.gkpStore/libraries.txt");
+ $gkpStore = "unitigging/$asm.gkpStore" if (-e "unitigging/$asm.gkpStore/libraries.txt");
+
+ caExit("ERROR: no reads supplied, and can't find any library information in gkpStore", undef) if (!defined($gkpStore));
+
+ my $numPacBioRaw = 0;
+ my $numPacBioCorrected = 0;
+ my $numNanoporeRaw = 0;
+ my $numNanoporeCorrected = 0;
+
+ open(L, "< $gkpStore/libraries.txt") or caExit("can't open '$gkpStore/libraries.txt' for reading: $!", undef);
+ while (<L>) {
+ $numPacBioRaw++ if (m/pacbio-raw/);
+ $numPacBioCorrected++ if (m/pacbio-corrected/);
+ $numNanoporeRaw++ if (m/nanopore-raw/);
+ $numNanoporeCorrected++ if (m/nanopore-corrected/);
}
+
+ $setUpForPacBio++ if ($numPacBioRaw + $numPacBioCorrected > 0);
+ $setUpForNanopore++ if ($numNanoporeRaw + $numNanoporeCorrected > 0);
+}
+
+# Now set error rates (if not set already) based on the dominant read type.
+
+if ($setUpForNanopore > 0) {
+ setGlobalIfUndef("corOvlErrorRate", 0.320);
+ setGlobalIfUndef("obtOvlErrorRate", 0.144);
+ setGlobalIfUndef("utgOvlErrorRate", 0.144);
+ setGlobalIfUndef("corErrorRate", 0.500);
+ setGlobalIfUndef("obtErrorRate", 0.144);
+ setGlobalIfUndef("utgErrorRate", 0.144);
+ setGlobalIfUndef("cnsErrorRate", 0.144);
+} else {
+ setGlobalIfUndef("corOvlErrorRate", 0.240);
+ setGlobalIfUndef("obtOvlErrorRate", 0.045);
+ setGlobalIfUndef("utgOvlErrorRate", 0.045);
+ setGlobalIfUndef("corErrorRate", 0.300);
+ setGlobalIfUndef("obtErrorRate", 0.045);
+ setGlobalIfUndef("utgErrorRate", 0.045);
+ setGlobalIfUndef("cnsErrorRate", 0.045);
}
# Finish setting parameters, then reset the bin directory using pathMap.
@@ -291,7 +323,7 @@ printHelp();
# Now that we know the bin directory, print the version so those pesky users
# will (hopefully) include it when they paste in logs.
-printVersion($bin);
+print "-- " . getGlobal("version") . "\n";
# Check java and gnuplot.
@@ -308,11 +340,14 @@ printHelp();
# when there isn't a grid.
print STDERR "-- Detected ", getNumberOfCPUs(), " CPUs and ", getPhysicalMemorySize(), " gigabytes of memory.\n";
+print STDERR "-- Limited to ", getGlobal("maxMemory"), " gigabytes from maxMemory option.\n" if (defined(getGlobal("maxMemory")));
+print STDERR "-- Limited to ", getGlobal("maxThreads"), " CPUs from maxThreads option.\n" if (defined(getGlobal("maxThreads")));
detectSGE();
detectSlurm();
detectPBSTorque();
detectLSF();
+detectDNANexus();
# Report if no grid engine found, or if the user has disabled grid support.
@@ -333,17 +368,56 @@ configureSlurm();
configurePBSTorque();
configureLSF();
configureRemote();
+configureDNANexus();
-# Based on genomeSize, configure the execution of every component. This needs to be done AFTER the grid is setup!
+# Based on genomeSize, configure the execution of every component.
+# This needs to be done AFTER the grid is setup!
configureAssembler();
+# And, finally, move to the assembly directory, finish setting things up, and report the critical
+# parameters.
+
+setWorkDirectory();
+
+if (defined($rootdir)) {
+ make_path($rootdir) if (! -d $rootdir);
+ chdir($rootdir);
+}
+
+setGlobal("onExitDir", getcwd());
+setGlobal("onExitNam", $asm);
+
+setGlobalIfUndef("objectStoreNameSpace", $asm); # No good place to put this.
+
+printf STDERR "--\n";
+printf STDERR "-- Generating assembly '$asm' in '" . getcwd() . "'\n";
+printf STDERR "--\n";
+printf STDERR "-- Parameters:\n";
+printf STDERR "--\n";
+printf STDERR "-- genomeSize %s\n", getGlobal("genomeSize");
+printf STDERR "--\n";
+printf STDERR "-- Overlap Generation Limits:\n";
+printf STDERR "-- corOvlErrorRate %6.4f (%6.2f%%)\n", getGlobal("corOvlErrorRate"), getGlobal("corOvlErrorRate") * 100.0;
+printf STDERR "-- obtOvlErrorRate %6.4f (%6.2f%%)\n", getGlobal("obtOvlErrorRate"), getGlobal("obtOvlErrorRate") * 100.0;
+printf STDERR "-- utgOvlErrorRate %6.4f (%6.2f%%)\n", getGlobal("utgOvlErrorRate"), getGlobal("utgOvlErrorRate") * 100.0;
+printf STDERR "--\n";
+printf STDERR "-- Overlap Processing Limits:\n";
+printf STDERR "-- corErrorRate %6.4f (%6.2f%%)\n", getGlobal("corErrorRate"), getGlobal("corErrorRate") * 100.0;
+printf STDERR "-- obtErrorRate %6.4f (%6.2f%%)\n", getGlobal("obtErrorRate"), getGlobal("obtErrorRate") * 100.0;
+printf STDERR "-- utgErrorRate %6.4f (%6.2f%%)\n", getGlobal("utgErrorRate"), getGlobal("utgErrorRate") * 100.0;
+printf STDERR "-- cnsErrorRate %6.4f (%6.2f%%)\n", getGlobal("cnsErrorRate"), getGlobal("cnsErrorRate") * 100.0;
+
+if (defined(getGlobal('errorRateUsed'))) {
+ print STDERR getGlobal('errorRateUsed');
+}
+
# Fail immediately if we run the script on the grid, and the gkpStore directory doesn't exist and
# we have no input files. Without this check we'd fail only after being scheduled on the grid.
-my $cor = (-e "$wrk/correction/$asm.gkpStore") || sequenceFileExists("$wrk/$asm.correctedReads") || (-e "$wrk/$asm.correctedReads.gkp");
-my $obt = (-e "$wrk/trimming/$asm.gkpStore") || sequenceFileExists("$wrk/$asm.trimmedReads") || (-e "$wrk/$asm.trimmedReads.gkp");
-my $utg = (-e "$wrk/unitigging/$asm.gkpStore");
+my $cor = (-e "correction/$asm.gkpStore") || fileExists("correction/$asm.gkpStore.tar") || sequenceFileExists("$asm.correctedReads") || (-e "$asm.correctedReads.gkp");
+my $obt = (-e "trimming/$asm.gkpStore") || fileExists("trimming/$asm.gkpStore.tar") || sequenceFileExists("$asm.trimmedReads") || (-e "$asm.trimmedReads.gkp");
+my $utg = (-e "unitigging/$asm.gkpStore") || fileExists("unitigging/$asm.gkpStore.tar");
if (($cor + $obt + $utg == 0) &&
(scalar(@inputFiles) == 0)) {
@@ -352,26 +426,21 @@ if (($cor + $obt + $utg == 0) &&
# Check that we were supplied a work directory, and that it exists, or we can create it.
-caExit("no run directory (-d option) specified", undef) if (!defined($wrk));
-
-make_path("$wrk") if (! -d "$wrk");
-make_path("$wrk/canu-logs") if (! -d "$wrk/canu-logs");
-make_path("$wrk/canu-scripts") if (! -d "$wrk/canu-scripts");
-
-caExit("run directory (-d option) '$wrk' doesn't exist and couldn't be created", undef) if (! -d $wrk);
+make_path("canu-logs") if (! -d "canu-logs");
+make_path("canu-scripts") if (! -d "canu-scripts");
# This environment variable tells the binaries to log their execution in canu-logs/
-$ENV{'CANU_DIRECTORY'} = $wrk;
+$ENV{'CANU_DIRECTORY'} = getcwd();
# Report the parameters used.
-writeLog($wrk);
+writeLog();
# Submit ourself for grid execution? If not grid enabled, or already running on the grid, this
# call just returns. The arg MUST be undef.
-submitScript($wrk, $asm, undef);
+submitScript($asm, undef);
#
# When doing 'run', this sets options for each stage.
@@ -399,9 +468,9 @@ sub setOptions ($$) {
# Create directories for the step, if needed.
- make_path("$wrk/correction") if ((! -d "$wrk/correction") && ($step eq "correct"));
- make_path("$wrk/trimming") if ((! -d "$wrk/trimming") && ($step eq "trim"));
- make_path("$wrk/unitigging") if ((! -d "$wrk/unitigging") && ($step eq "assemble"));
+ make_path("correction") if ((! -d "correction") && ($step eq "correct"));
+ make_path("trimming") if ((! -d "trimming") && ($step eq "trim"));
+ make_path("unitigging") if ((! -d "unitigging") && ($step eq "assemble"));
# Return that we want to run this step.
@@ -412,77 +481,74 @@ sub setOptions ($$) {
# Pipeline piece
#
-sub overlap ($$$) {
- my $wrk = shift @_;
+sub overlap ($$) {
my $asm = shift @_;
my $tag = shift @_;
my $ovlType = ($tag eq "utg") ? "normal" : "partial";
if (getGlobal("${tag}overlapper") eq "mhap") {
- mhapConfigure($wrk, $asm, $tag, $ovlType);
+ mhapConfigure($asm, $tag, $ovlType);
- mhapPrecomputeCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
+ mhapPrecomputeCheck($asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
# this also does mhapReAlign
- mhapCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
+ mhapCheck($asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
} elsif (getGlobal("${tag}overlapper") eq "minimap") {
- mmapConfigure($wrk, $asm, $tag, $ovlType);
+ mmapConfigure($asm, $tag, $ovlType);
- mmapPrecomputeCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
+ mmapPrecomputeCheck($asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
- mmapCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
+ mmapCheck($asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
} else {
- overlapConfigure($wrk, $asm, $tag, $ovlType);
+ overlapConfigure($asm, $tag, $ovlType);
- overlapCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
+ overlapCheck($asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1);
}
- createOverlapStore($wrk, $asm, $tag, getGlobal("ovsMethod"));
+ createOverlapStore($asm, $tag, getGlobal("ovsMethod"));
}
#
# Begin pipeline
#
-
-if (getGlobal("canuIteration") > 0) {
- print STDERR "--\n";
- print STDERR "-- This is canu parallel iteration #" . getGlobal("canuIteration") . ", out of a maximum of " . getGlobal("canuIterationMax") . " attempts.\n";
-}
-
-print STDERR "--\n";
-print STDERR "-- Final error rates before starting pipeline:\n";
-
-showErrorRates("-- ");
+# The checks for sequenceFileExists() at the start aren't needed except for
+# object storage mode. Gatekeeper has no way of knowing, inside
+# gatekeeper(), that this stage is completed and it shouldn't fetch the
+# store. In 'normal' operation, the store exists already, and we just
+# return.
+#
if (setOptions($mode, "correct") eq "correct") {
- print STDERR "--\n";
- print STDERR "--\n";
- print STDERR "-- BEGIN CORRECTION\n";
- print STDERR "--\n";
+ if (sequenceFileExists("$asm.correctedReads") eq undef) {
+ print STDERR "--\n";
+ print STDERR "--\n";
+ print STDERR "-- BEGIN CORRECTION\n";
+ print STDERR "--\n";
- gatekeeper($wrk, $asm, "cor", @inputFiles);
+ gatekeeper($asm, "cor", @inputFiles);
- merylConfigure($wrk, $asm, "cor");
- merylCheck($wrk, $asm, "cor") foreach (1..getGlobal("canuIterationMax") + 1);
- merylProcess($wrk, $asm, "cor");
+ merylConfigure($asm, "cor");
+ merylCheck($asm, "cor") foreach (1..getGlobal("canuIterationMax") + 1);
+ merylProcess($asm, "cor");
- overlap($wrk, $asm, "cor");
+ overlap($asm, "cor");
- buildCorrectionLayouts($wrk, $asm);
- generateCorrectedReads($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1);
- dumpCorrectedReads($wrk, $asm);
+ buildCorrectionLayouts($asm);
+ generateCorrectedReads($asm) foreach (1..getGlobal("canuIterationMax") + 1);
+ dumpCorrectedReads($asm);
- estimateCorrectedError($wrk, $asm, "cor");
+ estimateCorrectedError($asm, "cor");
- buildHTML($wrk, $asm, "cor");
+ buildHTML($asm, "cor");
+ }
- my $correctedReads = sequenceFileExists("$wrk/$asm.correctedReads");
+ my $correctedReads = sequenceFileExists("$asm.correctedReads");
- caExit("can't find corrected reads in '$wrk/$asm.correctedReads*'", undef) if (!defined($correctedReads));
+ caExit("can't find corrected reads '$asm.correctedReads*' in directory '" . getcwd() . "'", undef) if (!defined($correctedReads));
undef @inputFiles;
push @inputFiles, "-pacbio-corrected\0$correctedReads";
@@ -490,29 +556,31 @@ if (setOptions($mode, "correct") eq "correct") {
if (setOptions($mode, "trim") eq "trim") {
- print STDERR "--\n";
- print STDERR "--\n";
- print STDERR "-- BEGIN TRIMMING\n";
- print STDERR "--\n";
+ if (sequenceFileExists("$asm.trimmedReads") eq undef) {
+ print STDERR "--\n";
+ print STDERR "--\n";
+ print STDERR "-- BEGIN TRIMMING\n";
+ print STDERR "--\n";
- gatekeeper($wrk, $asm, "obt", @inputFiles);
+ gatekeeper($asm, "obt", @inputFiles);
- merylConfigure($wrk, $asm, "obt");
- merylCheck($wrk, $asm, "obt") foreach (1..getGlobal("canuIterationMax") + 1);
- merylProcess($wrk, $asm, "obt");
+ merylConfigure($asm, "obt");
+ merylCheck($asm, "obt") foreach (1..getGlobal("canuIterationMax") + 1);
+ merylProcess($asm, "obt");
- overlap($wrk, $asm, "obt");
+ overlap($asm, "obt");
- trimReads ($wrk, $asm);
- splitReads($wrk, $asm);
- dumpReads ($wrk, $asm);
- #summarizeReads($wrk, $asm);
+ trimReads ($asm);
+ splitReads($asm);
+ dumpReads ($asm);
+ #summarizeReads($asm);
- buildHTML($wrk, $asm, "obt");
+ buildHTML($asm, "obt");
+ }
- my $trimmedReads = sequenceFileExists("$wrk/$asm.trimmedReads");
+ my $trimmedReads = sequenceFileExists("$asm.trimmedReads");
- caExit("can't find trimmed reads in '$wrk/$asm.trimmedReads*'", undef) if (!defined($trimmedReads));
+ caExit("can't find trimmed reads '$asm.trimmedReads*' in directory '" . getcwd() . "'", undef) if (!defined($trimmedReads));
undef @inputFiles;
push @inputFiles, "-pacbio-corrected\0$trimmedReads";
@@ -520,39 +588,46 @@ if (setOptions($mode, "trim") eq "trim") {
if (setOptions($mode, "assemble") eq "assemble") {
- print STDERR "--\n";
- print STDERR "--\n";
- print STDERR "-- BEGIN ASSEMBLY\n";
- print STDERR "--\n";
+ if (sequenceFileExists("$asm.contigs") eq undef) {
+ print STDERR "--\n";
+ print STDERR "--\n";
+ print STDERR "-- BEGIN ASSEMBLY\n";
+ print STDERR "--\n";
+
+ gatekeeper($asm, "utg", @inputFiles);
- gatekeeper($wrk, $asm, "utg", @inputFiles);
+ merylConfigure($asm, "utg");
+ merylCheck($asm, "utg") foreach (1..getGlobal("canuIterationMax") + 1);
+ merylProcess($asm, "utg");
- merylConfigure($wrk, $asm, "utg");
- merylCheck($wrk, $asm, "utg") foreach (1..getGlobal("canuIterationMax") + 1);
- merylProcess($wrk, $asm, "utg");
+ overlap($asm, "utg");
- overlap($wrk, $asm, "utg");
+ #readErrorDetection($asm);
- #readErrorDetection($wrk, $asm);
+ readErrorDetectionConfigure($asm);
+ readErrorDetectionCheck($asm) foreach (1..getGlobal("canuIterationMax") + 1);
- readErrorDetectionConfigure($wrk, $asm);
- readErrorDetectionCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1);
+ overlapErrorAdjustmentConfigure($asm);
+ overlapErrorAdjustmentCheck($asm) foreach (1..getGlobal("canuIterationMax") + 1);
- overlapErrorAdjustmentConfigure($wrk, $asm);
- overlapErrorAdjustmentCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1);
+ updateOverlapStore($asm);
- updateOverlapStore($wrk, $asm);
+ unitig($asm);
+ unitigCheck($asm) foreach (1..getGlobal("canuIterationMax") + 1);
- unitig($wrk, $asm);
- unitigCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1);
+ foreach (1..getGlobal("canuIterationMax") + 1) { # Consensus wants to change the script between the first and
+ consensusConfigure($asm); # second iterations. The script is rewritten in
+ consensusCheck($asm); # consensusConfigure(), so we need to add that to the loop.
+ }
- consensusConfigure($wrk, $asm);
- consensusCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1);
+ consensusLoad($asm);
+ consensusAnalyze($asm);
- consensusLoad($wrk, $asm);
- consensusAnalyze($wrk, $asm);
+ alignGFA($asm) foreach (1..getGlobal("canuIterationMax") + 1);
- generateOutputs($wrk, $asm);
+ generateOutputs($asm);
+ }
}
+
exit(0);
diff --git a/src/pipelines/canu/Configure.pm b/src/pipelines/canu/Configure.pm
index c03d4aa..45c268e 100644
--- a/src/pipelines/canu/Configure.pm
+++ b/src/pipelines/canu/Configure.pm
@@ -39,7 +39,7 @@ use Carp qw(cluck);
use Sys::Hostname;
use canu::Defaults;
-
+use canu::Execution;
# This is called to expand parameter ranges for memory and thread parameters.
# Examples of valid ranges:
@@ -145,20 +145,43 @@ sub expandRange ($$) {
}
+sub findGridMaxMemoryAndThreads () {
+ my @grid = split '\0', getGlobal("availableHosts");
+ my $maxmem = 0;
+ my $maxcpu = 0;
+
+ foreach my $g (@grid) {
+ my ($cpu, $mem, $num) = split '-', $g;
+
+ $maxmem = ($maxmem < $mem) ? $mem : $maxmem;
+ $maxcpu = ($maxcpu < $cpu) ? $cpu : $maxcpu;
+ }
+
+ return($maxmem, $maxcpu);
+}
+
+
# Side effect! This will RESET the $global{} parameters to the computed value. This lets
# the rest of canu - in particular, the part that runs the jobs - use the correct value. Without
# resetting, I'd be making code changes all over the place to support the values returned.
-sub getAllowedResources ($$$$) {
+sub getAllowedResources ($$$$@) {
my $tag = shift @_; # Variant, e.g., "cor", "utg"
my $alg = shift @_; # Algorithm, e.g., "mhap", "ovl"
my $err = shift @_; # Report of things we can't run.
my $all = shift @_; # Report of things we can run.
+ my $dbg = shift @_; # Optional, report debugging stuff
# If no grid, or grid not enabled, everything falls under 'lcoal'.
my $class = ((getGlobal("useGrid") ne "0") && (defined(getGlobal("gridEngine")))) ? "grid" : "local";
+ # If grid, but no hosts, fail.
+
+ if (($class eq "grid") && (!defined(getGlobal("availableHosts")))) {
+ caExit("invalid useGrid (" . getGlobal("useGrid") . ") and gridEngine (" . getGlobal("gridEngine") . "); found no execution hosts - is grid available from this host?", undef);
+ }
+
# Figure out limits.
my $maxMemory = getGlobal("maxMemory");
@@ -175,8 +198,20 @@ sub getAllowedResources ($$$$) {
# If the maximum limits aren't set, default to 'unlimited' (for the grid; we'll effectively filter
# by the number of jobs we can fit on the hosts) or to the current hardware limits.
- $maxMemory = (($class eq "grid") ? 1024 * 1024 : getPhysicalMemorySize()) if (!defined($maxMemory)); # 1 PB memory!
- $maxThreads = (($class eq "grid") ? 1024 : getNumberOfCPUs()) if (!defined($maxThreads)); # 1 k cores!
+ if ($dbg) {
+ print STDERR "--\n";
+ print STDERR "-- DEBUG\n";
+ print STDERR "-- DEBUG Limited to $maxMemory GB memory via maxMemory option\n" if (defined($maxMemory));
+ print STDERR "-- DEBUG Limited to $maxThreads threads via maxThreads option\n" if (defined($maxThreads));
+ }
+
+ # Figure out the largest memory and threads that could ever be supported. This lets us short-circuit
+ # the loop below.
+
+ my ($gridMaxMem, $gridMaxThr) = findGridMaxMemoryAndThreads();
+
+ $maxMemory = (($class eq "grid") ? $gridMaxMem : getPhysicalMemorySize()) if (!defined($maxMemory));
+ $maxThreads = (($class eq "grid") ? $gridMaxThr : getNumberOfCPUs()) if (!defined($maxThreads));
# Build a list of the available hardware configurations we can run on. If grid, we get this
# from the list of previously discovered hosts. If local, it's just this machine.
@@ -188,10 +223,6 @@ sub getAllowedResources ($$$$) {
if ($class eq "grid") {
my @grid = split '\0', getGlobal("availableHosts");
- if (scalar(@grid) == 0) {
- caExit("invalid useGrid (" . getGlobal("useGrid") . ") and gridEngine (" . getGlobal("gridEngine") . "); found no execution hosts - is grid available from this host?", undef);
- }
-
foreach my $g (@grid) {
my ($cpu, $mem, $num) = split '-', $g;
@@ -207,6 +238,15 @@ sub getAllowedResources ($$$$) {
push @gridNum, 1;
}
+ if ($dbg) {
+ print STDERR "-- DEBUG\n";
+ print STDERR "-- DEBUG Have ", scalar(@gridCor), " configurations; largest memory size $maxMemory GB; most cores $maxThreads:\n";
+ for (my $ii=0; $ii<scalar(@gridCor); $ii++) {
+ print STDERR "-- DEBUG class$ii - $gridNum[$ii] machines with $gridCor[$ii] cores with $gridMem[$ii]GB memory each.\n";
+ }
+ print STDERR "-- DEBUG\n";
+ }
+
# The task usually has multiple choices, and we have a little optimization problem to solve. For each
# pair of memory/threads, compute three things:
# a) how many processes we can get running
@@ -218,39 +258,6 @@ sub getAllowedResources ($$$$) {
my @taskMemory = expandRange("${tag}${alg}Memory", $taskMemory);
my @taskThreads = expandRange("${tag}${alg}Threads", $taskThreads);
- # Filter out task settings that can't be run based on the gridMemory/gridThreads or masterMemory/masterThreads setting.
- # (actually, this just reports those that would be filtered; the actual filtering is inline in the algorithm)
-
- my $ignoreM;
- my $ignoreT;
-
- foreach my $m (@taskMemory) {
- $m = adjustMemoryValue($m);
- }
-
- foreach my $m (@taskMemory) {
- next if ($m <= $maxMemory);
- $ignoreM .= "," if (defined($ignoreM));
- $ignoreM .= "${m}g";
- }
- foreach my $t (@taskThreads) {
- next if ($t <= $maxThreads);
- $ignoreT .= "," if (defined($ignoreT));
- $ignoreT .= "$t";
- }
-
- # Too verbose with long value lists
- #
- #if (defined($ignoreM) && defined($ignoreT)) {
- # $err .= "-- Can't use ${tag}${alg}Memory=$ignoreM and ${tag}${alg}Threads=$ignoreT because of maxMemory=${maxMemory}g and maxThreads=$maxThreads limits.\n";
- #
- #} elsif (defined($ignoreM)) {
- # $err .= "-- Can't use ${tag}${alg}Memory=$ignoreM because of maxMemory=${maxMemory}g limit.\n";
- #
- #} elsif (defined($ignoreT)) {
- # $err .= "-- Can't use ${tag}${alg}Threads=$ignoreT because of maxThreads=$maxThreads limit.\n";
- #}
-
# Find task memory/thread settings that will maximize the number of cores running. This used
# to also compute best as 'cores * memory' but that is better handled by ordering the task
# settings parameters. The example below will pick the largest (last) configuration that
@@ -263,7 +270,9 @@ sub getAllowedResources ($$$$) {
foreach my $m (@taskMemory) {
foreach my $t (@taskThreads) {
-
+ #if ($dbg && (($m > $maxMemory) || ($t > $maxThreads))) {
+ # print STDERR "-- DEBUG Tested $tag$alg requesting $t cores and ${m}GB memory - rejected: limited to ${maxMemory}GB and $maxThreads cores.\n";
+ #}
next if ($m > $maxMemory); # Bail if either of the suggest settings are
next if ($t > $maxThreads); # larger than the maximum allowed.
@@ -287,11 +296,19 @@ sub getAllowedResources ($$$$) {
my $np = ($np_cpu < $np_mem) ? $np_cpu : $np_mem;
+ if ($dbg) {
+ print STDERR "-- DEBUG for $t threads and $m memory - class$ii can support $np_cpu jobs(cores) and $np_mem jobs(memory), so $np jobs.\n";
+ }
+
$processes += $np;
$cores += $np * $t;
$memory += $np * $m;
}
+ if ($dbg) {
+ print STDERR "-- DEBUG Tested $tag$alg requesting $t cores and ${m}GB memory and found $cores could be used.\n";
+ }
+
# If no cores, then all machines were too small.
next if ($cores == 0);
@@ -307,6 +324,8 @@ sub getAllowedResources ($$$$) {
}
if (!defined($bestCoresM)) {
+ getAllowedResources($tag, $alg, $err, $all, 1) if (!defined($dbg));
+
print STDERR "--\n";
print STDERR "-- Task $tag$alg can't run on any available machines.\n";
print STDERR "-- It is requesting ", getGlobal("${tag}${alg}Memory"), " GB memory and ", getGlobal("${tag}${alg}Threads"), " threads.\n";
@@ -362,26 +381,28 @@ sub getAllowedResources ($$$$) {
my $nam;
- if ($alg eq "bat") { $nam = "bogart (unitigger)"; }
- elsif ($alg eq "cns") { $nam = "utgcns (consensus)"; }
- elsif ($alg eq "cor") { $nam = "falcon_sense (read correction)"; }
- elsif ($alg eq "meryl") { $nam = "meryl (k-mer counting)"; }
+ if ($alg eq "bat") { $nam = "bogart"; }
+ elsif ($alg eq "cns") { $nam = "consensus"; }
+ elsif ($alg eq "gfa") { $nam = "GFA alignment and processing"; }
+ elsif ($alg eq "cor") { $nam = "falcon_sense"; }
+ elsif ($alg eq "meryl") { $nam = "meryl"; }
elsif ($alg eq "oea") { $nam = "overlap error adjustment"; }
- elsif ($alg eq "ovb") { $nam = "overlap store parallel bucketizer"; }
- elsif ($alg eq "ovs") { $nam = "overlap store parallel sorting"; }
- elsif ($alg eq "red") { $nam = "read error detection (overlap error adjustment)"; }
- elsif ($alg eq "mhap") { $nam = "mhap (overlapper)"; }
- elsif ($alg eq "mmap") { $nam = "minimap (overlapper)"; }
- elsif ($alg eq "ovl") { $nam = "overlapper"; }
+ elsif ($alg eq "ovb") { $nam = "ovStore bucketizer"; }
+ elsif ($alg eq "ovs") { $nam = "ovStore sorting"; }
+ elsif ($alg eq "red") { $nam = "read error detection"; }
+ elsif ($alg eq "mhap") { $nam = "mhap ($tag)"; }
+ elsif ($alg eq "mmap") { $nam = "minimap ($tag)"; }
+ elsif ($alg eq "ovl") { $nam = "overlapper ($tag)"; }
else {
caFailure("unknown task '$alg' in getAllowedResources().", undef);
}
- $all .= "-- Allowed to";
- $all .= " run " . substr(" $concurrent", -3) . " job" . (($concurrent == 1) ? " " : "s") . " concurrently," if (defined($concurrent));
- $all .= " run under grid control," if (!defined($concurrent));
- $all .= " and use up to " . substr(" $taskThreads", -3) . " compute thread" . (($taskThreads == 1) ? " " : "s");
- $all .= " and " . substr(" $taskMemory", -4) . " GB memory for stage '$nam'.\n";
+ my $job = substr(" $concurrent", -3) . " job" . (($concurrent == 1) ? " " : "s");
+ my $thr = substr(" $taskThreads", -3) . " CPU" . (($taskThreads == 1) ? " " : "s");
+ my $mem = substr(" $taskMemory", -4) . " GB";
+
+ $all .= "-- Run $job concurrently using $mem and $thr for stage '$nam'.\n" if ( defined($concurrent));
+ $all .= "-- Run under grid control using $mem and $thr for stage '$nam'.\n" if (!defined($concurrent));
return($err, $all);
}
@@ -635,22 +656,29 @@ sub configureAssembler () {
setGlobalIfUndef("oeaMemory", "4"); setGlobalIfUndef("oeaThreads", "1");
}
- # And bogart.
+ # And bogart and GFA alignment/processing.
+ #
+ # GFA for genomes less than 40m is run in the canu process itself.
if (getGlobal("genomeSize") < adjustGenomeSize("40m")) {
setGlobalIfUndef("batMemory", "2-16"); setGlobalIfUndef("batThreads", "1-4");
+ setGlobalIfUndef("gfaMemory", "2-4"); setGlobalIfUndef("gfaThreads", "1");
} elsif (getGlobal("genomeSize") < adjustGenomeSize("500m")) {
- setGlobalIfUndef("batMemory", "16-64"); setGlobalIfUndef("batThreads", "2-8");
+ setGlobalIfUndef("batMemory", "16-64"); setGlobalIfUndef("batThreads", "2-8");
+ setGlobalIfUndef("gfaMemory", "2-4"); setGlobalIfUndef("gfaThreads", "2-4");
} elsif (getGlobal("genomeSize") < adjustGenomeSize("2g")) {
setGlobalIfUndef("batMemory", "32-256"); setGlobalIfUndef("batThreads", "4-16");
+ setGlobalIfUndef("gfaMemory", "4-8"); setGlobalIfUndef("gfaThreads", "4-8");
} elsif (getGlobal("genomeSize") < adjustGenomeSize("5g")) {
setGlobalIfUndef("batMemory", "128-512"); setGlobalIfUndef("batThreads", "8-32");
+ setGlobalIfUndef("gfaMemory", "8-16"); setGlobalIfUndef("gfaThreads", "8-16");
} else {
setGlobalIfUndef("batMemory", "256-1024"); setGlobalIfUndef("batThreads", "16-64");
+ setGlobalIfUndef("gfaMemory", "16-32"); setGlobalIfUndef("gfaThreads", "16-64");
}
# Finally, use all that setup to pick actual values for each component.
@@ -662,23 +690,42 @@ sub configureAssembler () {
my $err;
my $all;
- ($err, $all) = getAllowedResources("", "bat", $err, $all);
- ($err, $all) = getAllowedResources("cor", "mhap", $err, $all);
- ($err, $all) = getAllowedResources("obt", "mhap", $err, $all);
- ($err, $all) = getAllowedResources("utg", "mhap", $err, $all);
+ ($err, $all) = getAllowedResources("", "meryl", $err, $all);
+
+ ($err, $all) = getAllowedResources("cor", "mhap", $err, $all) if (getGlobal("corOverlapper") eq "mhap");
+ ($err, $all) = getAllowedResources("cor", "mmap", $err, $all) if (getGlobal("corOverlapper") eq "minimap");
+ ($err, $all) = getAllowedResources("cor", "ovl", $err, $all) if (getGlobal("corOverlapper") eq "ovl");
+
+ ($err, $all) = getAllowedResources("obt", "mhap", $err, $all) if (getGlobal("obtOverlapper") eq "mhap");
+ ($err, $all) = getAllowedResources("obt", "mmap", $err, $all) if (getGlobal("obtOverlapper") eq "minimap");
+ ($err, $all) = getAllowedResources("obt", "ovl", $err, $all) if (getGlobal("obtOverlapper") eq "ovl");
+
+ ($err, $all) = getAllowedResources("utg", "mhap", $err, $all) if (getGlobal("utgOverlapper") eq "mhap");
+ ($err, $all) = getAllowedResources("utg", "mmap", $err, $all) if (getGlobal("utgOverlapper") eq "minimap");
+ ($err, $all) = getAllowedResources("utg", "ovl", $err, $all) if (getGlobal("utgOverlapper") eq "ovl");
+
+ ($err, $all) = getAllowedResources("", "cor", $err, $all);
+
+ ($err, $all) = getAllowedResources("", "ovb", $err, $all);
+ ($err, $all) = getAllowedResources("", "ovs", $err, $all);
+
($err, $all) = getAllowedResources("", "red", $err, $all);
($err, $all) = getAllowedResources("", "oea", $err, $all);
+
+ ($err, $all) = getAllowedResources("", "bat", $err, $all);
+
+ ($err, $all) = getAllowedResources("", "gfa", $err, $all);
+
($err, $all) = getAllowedResources("", "cns", $err, $all);
- ($err, $all) = getAllowedResources("", "ovb", $err, $all);
- ($err, $all) = getAllowedResources("", "ovs", $err, $all);
- ($err, $all) = getAllowedResources("cor", "ovl", $err, $all);
- ($err, $all) = getAllowedResources("obt", "ovl", $err, $all);
- ($err, $all) = getAllowedResources("utg", "ovl", $err, $all);
- ($err, $all) = getAllowedResources("", "meryl", $err, $all);
- ($err, $all) = getAllowedResources("", "cor", $err, $all);
- ($err, $all) = getAllowedResources("cor", "mmap", $err, $all);
- ($err, $all) = getAllowedResources("obt", "mmap", $err, $all);
- ($err, $all) = getAllowedResources("utg", "mmap", $err, $all);
+
+ # Check some minimums.
+
+ if ((getGlobal("ovsMemory") =~ m/^([0123456789.]+)-*[0123456789.]*$/) &&
+ ($1 < 0.25)) {
+ caExit("ovsMemory must be at least 0.25g or 256m", undef);
+ }
+
+ # 2017-02-21 -- not sure why $err is being reported here if it doesn't stop. What's in it?
print STDERR "--\n" if (defined($err));
print STDERR $err if (defined($err));
diff --git a/src/pipelines/canu/Consensus.pm b/src/pipelines/canu/Consensus.pm
index 7f03b6a..21e7869 100644
--- a/src/pipelines/canu/Consensus.pm
+++ b/src/pipelines/canu/Consensus.pm
@@ -40,32 +40,38 @@ package canu::Consensus;
require Exporter;
@ISA = qw(Exporter);
- at EXPORT = qw(consensusConfigure consensusCheck consensusLoad consensusAnalyze);
+ at EXPORT = qw(consensusConfigure consensusCheck consensusLoad consensusAnalyze alignGFA);
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
use canu::Unitig;
use canu::HTML;
+use canu::Grid_Cloud;
-sub utgcns ($$$$) {
- my $wrk = shift @_; # Local work directory
+sub utgcns ($$$) {
my $asm = shift @_;
my $ctgjobs = shift @_;
my $utgjobs = shift @_;
my $jobs = $ctgjobs + $utgjobs;
- open(F, "> $wrk/5-consensus/consensus.sh") or caExit("can't open '$wrk/5-consensus/consensus.sh' for writing: $!", undef);
+ my $path = "unitigging/5-consensus";
+
+ open(F, "> $path/consensus.sh") or caExit("can't open '$path/consensus.sh' for writing: $!", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
print F "if [ \$jobid -gt $jobs ]; then\n";
print F " echo Error: Only $jobs partitions, you asked for \$jobid.\n";
print F " exit 1\n";
@@ -80,93 +86,116 @@ sub utgcns ($$$$) {
print F "\n";
print F "jobid=`printf %04d \$jobid`\n";
print F "\n";
- print F "if [ ! -d $wrk/5-consensus/\${tag}cns ] ; then\n";
- print F " mkdir -p $wrk/5-consensus/\${tag}cns\n";
+ print F "if [ ! -d ./\${tag}cns ] ; then\n";
+ print F " mkdir -p ./\${tag}cns\n";
print F "fi\n";
print F "\n";
- print F "if [ -e $wrk/5-consensus/\${tag}cns/\$jobid.cns ] ; then\n";
+ print F "if [ -e ./\${tag}cns/\$jobid.cns ] ; then\n";
print F " exit 0\n";
print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
+ print F fetchFileShellCode("unitigging/$asm.\${tag}Store", "seqDB.v001.dat", "");
+ print F fetchFileShellCode("unitigging/$asm.\${tag}Store", "seqDB.v001.tig", "");
+ print F "\n";
+ print F fetchStoreShellCode("unitigging/$asm.\${tag}Store/partitionedReads.gkpStore", $path, "");
print F "\n";
print F "\$bin/utgcns \\\n";
- print F " -G $wrk/$asm.\${tag}Store/partitionedReads.gkpStore \\\n"; # Optional; utgcns will default to this
- print F " -T $wrk/$asm.\${tag}Store 1 \$jobid \\\n";
- print F " -O $wrk/5-consensus/\${tag}cns/\$jobid.cns.WORKING \\\n";
+ print F " -G ../$asm.\${tag}Store/partitionedReads.gkpStore \\\n"; # Optional; utgcns will default to this
+ print F " -T ../$asm.\${tag}Store 1 \$jobid \\\n";
+ print F " -O ./\${tag}cns/\$jobid.cns.WORKING \\\n";
print F " -maxcoverage " . getGlobal('cnsMaxCoverage') . " \\\n";
print F " -e " . getGlobal("cnsErrorRate") . " \\\n";
print F " -quick \\\n" if (getGlobal("cnsConsensus") eq "quick");
print F " -pbdagcon \\\n" if (getGlobal("cnsConsensus") eq "pbdagcon");
+ print F " -edlib \\\n" if (getGlobal("canuIteration") > 0);
print F " -utgcns \\\n" if (getGlobal("cnsConsensus") eq "utgcns");
print F " -threads " . getGlobal("cnsThreads") . " \\\n";
print F "&& \\\n";
- print F "mv $wrk/5-consensus/\${tag}cns/\$jobid.cns.WORKING $wrk/5-consensus/\${tag}cns/\$jobid.cns \\\n";
+ print F "mv ./\${tag}cns/\$jobid.cns.WORKING ./\${tag}cns/\$jobid.cns \\\n";
+ print F "\n";
+ print F stashFileShellCode("unitigging/5-consensus", "\${tag}cns/\$jobid.cns", "");
print F "\n";
print F "exit 0\n";
+ if (getGlobal("canuIteration") == 0) {
+ print STDERR "-- Using fast alignment for consensus (iteration '", getGlobal("canuIteration"), "').\n";
+ } else {
+ print STDERR "-- Using slow alignment for consensus (iteration '", getGlobal("canuIteration"), "').\n";
+ }
+
close(F);
+
+ stashFile("$path/consensus.sh");
}
-sub cleanupPartitions ($$$) {
- my $wrk = shift @_;
+sub cleanupPartitions ($$) {
my $asm = shift @_;
my $tag = shift @_;
- return if (! -e "$wrk/$asm.${tag}Store/partitionedReads.gkpStore/partitions/map");
+ return if (! -e "unitigging/$asm.${tag}Store/partitionedReads.gkpStore/partitions/map");
- my $gkpTime = -M "$wrk/$asm.${tag}Store/partitionedReads.gkpStore/partitions/map";
- my $tigTime = -M "$wrk/$asm.ctgStore/seqDB.v001.tig";
+ my $gkpTime = -M "unitigging/$asm.${tag}Store/partitionedReads.gkpStore/partitions/map";
+ my $tigTime = -M "unitigging/$asm.ctgStore/seqDB.v001.tig";
return if ($gkpTime <= $tigTime);
print STDERR "-- Partitioned gkpStore is older than tigs, rebuild partitioning (gkpStore $gkpTime days old; ctgStore $tigTime days old).\n";
- if (runCommandSilently($wrk, "rm -rf $wrk/$asm.${tag}Store/partitionedReads.gkpStore", 1)) {
- caExit("failed to remove old partitions ($wrk/$asm.${tag}Store/partitionedReads.gkpStore/partitions), can't continue until these are removed", undef);
+ if (runCommandSilently("unitigging", "rm -rf ./$asm.${tag}Store/partitionedReads.gkpStore", 1)) {
+ caExit("failed to remove old partitions (unitigging/$asm.${tag}Store/partitionedReads.gkpStore/partitions), can't continue until these are removed", undef);
}
}
-sub partitionReads ($$$) {
- my $wrk = shift @_;
+sub partitionReads ($$) {
my $asm = shift @_;
my $tag = shift @_;
my $bin = getBinDirectory();
my $cmd;
- return if (-e "$wrk/$asm.${tag}Store/partitionedReads.gkpStore/partitions/map");
+ return if (-e "unitigging/$asm.${tag}Store/partitionedReads.gkpStore/partitions/map");
+ return if (fileExists("unitigging/$asm.${tag}Store/partitionedReads.gkpStore.tar"));
+
+ fetchStore("unitigging/$asm.gkpStore");
+
+ fetchFile("unitigging/$asm.${tag}Store/seqDB.v001.dat");
+ fetchFile("unitigging/$asm.${tag}Store/seqDB.v001.tig");
$cmd = "$bin/gatekeeperPartition \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.${tag}Store 1 \\\n";
+ $cmd .= " -G ./$asm.gkpStore \\\n";
+ $cmd .= " -T ./$asm.${tag}Store 1 \\\n";
$cmd .= " -b " . getGlobal("cnsPartitionMin") . " \\\n" if (defined(getGlobal("cnsPartitionMin")));
$cmd .= " -p " . getGlobal("cnsPartitions") . " \\\n" if (defined(getGlobal("cnsPartitions")));
- $cmd .= "> $wrk/$asm.${tag}Store/partitionedReads.err 2>&1";
+ $cmd .= "> ./$asm.${tag}Store/partitionedReads.log 2>&1";
- stopBefore("consensusConfigure", $cmd);
-
- if (runCommand("$wrk", $cmd)) {
- caExit("failed to partition the reads", "$wrk/$asm.${tag}Store/partitionedReads.err");
+ if (runCommand("unitigging", $cmd)) {
+ caExit("failed to partition the reads", "unitigging/$asm.${tag}Store/partitionedReads.log");
}
+
+ stashStore("unitigging/$asm.${tag}Store/partitionedReads.gkpStore");
+ stashFile ("unitigging/$asm.${tag}Store/partitionedReads.log");
}
-sub computeNumberOfConsensusJobs ($$$) {
- my $wrk = shift @_;
+sub computeNumberOfConsensusJobs ($$) {
my $asm = shift @_;
my $tag = shift @_;
- my $jobs = 0;
+ my $jobs = "0001";
my $bin = getBinDirectory();
- open(F, "ls $wrk/$asm.${tag}Store/partitionedReads.gkpStore/partitions/blobs.* |") or caExit("failed to find partitioned files in '$wrk/$asm.${tag}Store/partitionedReads.gkpStore/partitions/blobs.*': $!", undef);
- while (<F>) {
- if (m/blobs.(\d+)$/) {
- $jobs = int($1);
+ fetchFile("unitigging/$asm.${tag}Store/partitionedReads.log");
+
+ open(F, "< unitigging/$asm.${tag}Store/partitionedReads.log") or caExit("can't open 'unitigging/$asm.${tag}Store/partitionedReads.log' for reading: $!", undef);
+ while(<F>) {
+ #if (m/^partition (\d+) has \d+ reads$/) {
+ # $jobs = $1;
+ #}
+ if (m/^Found \d+ unpartitioned reads and maximum partition of (\d+)$/) {
+ $jobs = $1;
}
}
close(F);
@@ -176,55 +205,60 @@ sub computeNumberOfConsensusJobs ($$$) {
-sub consensusConfigure ($$) {
- my $WRK = shift @_; # Root work directory
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub consensusConfigure ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/5-consensus";
+ my $path = "unitigging/5-consensus";
- goto allDone if (skipStage($WRK, $asm, "consensusConfigure") == 1);
- goto allDone if ((-e "$wrk/$asm.ctgStore/seqDB.v002.tig") &&
- (-e "$wrk/$asm.utgStore/seqDB.v002.tig"));
+ goto allDone if (skipStage($asm, "consensusConfigure") == 1);
+ goto allDone if ((fileExists("unitigging/$asm.ctgStore/seqDB.v002.tig")) &&
+ (fileExists("unitigging/$asm.utgStore/seqDB.v002.tig")));
- make_path("$path") if (! -d "$path");
+ make_path($path) if (! -d $path);
# If the gkpStore partitions are older than the ctgStore unitig output, assume the unitigs have
# changed and remove the gkpStore partition. -M is (annoyingly) 'file age', so we need to
# rebuild if gkp is older (larger) than tig.
- cleanupPartitions($wrk, $asm, "ctg");
- cleanupPartitions($wrk, $asm, "utg");
+ cleanupPartitions($asm, "ctg");
+ cleanupPartitions($asm, "utg");
# Partition gkpStore if needed. Yeah, we could create both at the same time, with significant
# effort in coding it up.
- partitionReads($wrk, $asm, "ctg");
- partitionReads($wrk, $asm, "utg");
+ partitionReads($asm, "ctg");
+ partitionReads($asm, "utg");
# Set up the consensus compute. It's in a useless if chain because there used to be
# different executables; now they're all rolled into utgcns itself.
- my $ctgjobs = computeNumberOfConsensusJobs($wrk, $asm, "ctg");
- my $utgjobs = computeNumberOfConsensusJobs($wrk, $asm, "utg");
+ my $ctgjobs = computeNumberOfConsensusJobs($asm, "ctg");
+ my $utgjobs = computeNumberOfConsensusJobs($asm, "utg");
+
+ # This configure is an odd-ball. Unlike all the other places that write scripts,
+ # we'll rewrite this one every time, so that we can change the alignment algorithm
+ # on the second attempt.
+
+ my $firstTime = (! -e "$path/consensus.sh");
if ((getGlobal("cnsConsensus") eq "quick") ||
(getGlobal("cnsConsensus") eq "pbdagcon") ||
(getGlobal("cnsConsensus") eq "utgcns")) {
- utgcns($wrk, $asm, $ctgjobs, $utgjobs);
+ utgcns($asm, $ctgjobs, $utgjobs);
} else {
caFailure("unknown consensus style '" . getGlobal("cnsConsensus") . "'", undef);
}
+ print STDERR "-- Configured $ctgjobs contig and $utgjobs unitig consensus jobs.\n";
+
finishStage:
- emitStage($WRK, $asm, "consensusConfigure");
- buildHTML($WRK, $asm, "utg");
- stopAfter("consensusConfigure");
+ emitStage($asm, "consensusConfigure") if ($firstTime);
+ buildHTML($asm, "utg");
allDone:
- print STDERR "-- Configured $ctgjobs contig and $utgjobs unitig consensus jobs.\n";
+ stopAfter("consensusConfigure");
}
@@ -233,23 +267,26 @@ sub consensusConfigure ($$) {
# Checks that all consensus jobs are complete, loads them into the store.
#
-sub consensusCheck ($$) {
- my $WRK = shift @_; # Root work directory
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub consensusCheck ($) {
my $asm = shift @_;
my $attempt = getGlobal("canuIteration");
- my $path = "$wrk/5-consensus";
+ my $path = "unitigging/5-consensus";
- goto allDone if (skipStage($WRK, $asm, "consensusCheck", $attempt) == 1);
- goto allDone if ((-e "$path/ctgcns.files") && (-e "$path/utgcns.files"));
- goto allDone if (-e "$wrk/$asm.ctgStore/seqDB.v002.tig");
+ goto allDone if (skipStage($asm, "consensusCheck", $attempt) == 1);
+ goto allDone if ((fileExists("$path/ctgcns.files")) &&
+ (fileExists("$path/utgcns.files")));
+ goto allDone if (fileExists("unitigging/$asm.ctgStore/seqDB.v002.tig"));
+
+ fetchFile("$path/consensus.sh");
# Figure out if all the tasks finished correctly.
- my $ctgjobs = computeNumberOfConsensusJobs($wrk, $asm, "ctg");
- my $utgjobs = computeNumberOfConsensusJobs($wrk, $asm, "utg");
+ my $ctgjobs = computeNumberOfConsensusJobs($asm, "ctg");
+ my $utgjobs = computeNumberOfConsensusJobs($asm, "utg");
my $jobs = $ctgjobs + $utgjobs;
+ caExit("no consensus jobs found?", undef) if ($jobs == 0);
+
my $currentJobID = "0001";
my $tag = "ctgcns";
@@ -259,24 +296,24 @@ sub consensusCheck ($$) {
my $failureMessage = "";
for (my $job=1; $job <= $jobs; $job++) {
- if (-e "$path/$tag/$currentJobID.cns") {
- push @ctgSuccessJobs, "$path/$tag/$currentJobID.cns\n" if ($tag eq "ctgcns");
- push @utgSuccessJobs, "$path/$tag/$currentJobID.cns\n" if ($tag eq "utgcns");
+ if (fileExists("$path/$tag/$currentJobID.cns")) {
+ push @ctgSuccessJobs, "5-consensus/$tag/$currentJobID.cns\n" if ($tag eq "ctgcns");
+ push @utgSuccessJobs, "5-consensus/$tag/$currentJobID.cns\n" if ($tag eq "utgcns");
- } elsif (-e "$path/$tag/$currentJobID.cns.gz") {
- push @ctgSuccessJobs, "$path/$tag/$currentJobID.cns.gz\n" if ($tag eq "ctgcns");
- push @utgSuccessJobs, "$path/$tag/$currentJobID.cns.gz\n" if ($tag eq "utgcns");
+ } elsif (fileExists("$path/$tag/$currentJobID.cns.gz")) {
+ push @ctgSuccessJobs, "5-consensus/$tag/$currentJobID.cns.gz\n" if ($tag eq "ctgcns");
+ push @utgSuccessJobs, "5-consensus/$tag/$currentJobID.cns.gz\n" if ($tag eq "utgcns");
- } elsif (-e "$path/$tag/$currentJobID.cns.bz2") {
- push @ctgSuccessJobs, "$path/$tag/$currentJobID.cns.bz2\n" if ($tag eq "ctgcns");
- push @utgSuccessJobs, "$path/$tag/$currentJobID.cns.bz2\n" if ($tag eq "utgcns");
+ } elsif (fileExists("$path/$tag/$currentJobID.cns.bz2")) {
+ push @ctgSuccessJobs, "5-consensus/$tag/$currentJobID.cns.bz2\n" if ($tag eq "ctgcns");
+ push @utgSuccessJobs, "5-consensus/$tag/$currentJobID.cns.bz2\n" if ($tag eq "utgcns");
- } elsif (-e "$path/$tag/$currentJobID.cns.xz") {
- push @ctgSuccessJobs, "$path/$tag/$currentJobID.cns.xz\n" if ($tag eq "ctgcns");
- push @utgSuccessJobs, "$path/$tag/$currentJobID.cns.xz\n" if ($tag eq "utgcns");
+ } elsif (fileExists("$path/$tag/$currentJobID.cns.xz")) {
+ push @ctgSuccessJobs, "5-consensus/$tag/$currentJobID.cns.xz\n" if ($tag eq "ctgcns");
+ push @utgSuccessJobs, "5-consensus/$tag/$currentJobID.cns.xz\n" if ($tag eq "utgcns");
} else {
- $failureMessage .= "-- job $path/$tag/$currentJobID.cns FAILED.\n";
+ $failureMessage .= "-- job $tag/$currentJobID.cns FAILED.\n";
push @failedJobs, $job;
}
@@ -307,12 +344,10 @@ sub consensusCheck ($$) {
# Otherwise, run some jobs.
- print STDERR "-- Consensus attempt $attempt begins with ", scalar(@ctgSuccessJobs) + scalar(@utgSuccessJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
+ emitStage($asm, "consensusCheck", $attempt);
+ buildHTML($asm, "utg");
- emitStage($WRK, $asm, "consensusCheck", $attempt);
- buildHTML($WRK, $asm, "utg");
-
- submitOrRunParallelJob($WRK, $asm, "cns", $path, "consensus", @failedJobs);
+ submitOrRunParallelJob($asm, "cns", $path, "consensus", @failedJobs);
return;
}
@@ -323,28 +358,31 @@ sub consensusCheck ($$) {
print L @ctgSuccessJobs;
close(L);
+ stashFile("$path/ctgcns.files");
+
open(L, "> $path/utgcns.files") or caExit("can't open '$path/utgcns.files' for writing: $!", undef);
print L @utgSuccessJobs;
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "consensusCheck");
- buildHTML($WRK, $asm, "utg");
- stopAfter("consensusCheck");
+ stashFile("$path/utgcns.files");
+
+ emitStage($asm, "consensusCheck");
+ buildHTML($asm, "utg");
allDone:
}
-sub purgeFiles ($$$$$$) {
- my $path = shift @_;
+sub purgeFiles ($$$$$) {
my $tag = shift @_;
my $Ncns = shift @_;
my $Nfastq = shift @_;
my $Nlayout = shift @_;
my $Nlog = shift @_;
+ my $path = "unitigging/5-consensus";
+
open(F, "< $path/$tag.files") or caExit("can't open '$path/$tag.files' for reading: $!\n", undef);
while (<F>) {
chomp;
@@ -385,42 +423,74 @@ sub purgeFiles ($$$$$$) {
-sub consensusLoad ($$) {
- my $WRK = shift @_; # Root work directory
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub consensusLoad ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/5-consensus";
+ my $path = "unitigging/5-consensus";
- goto allDone if (skipStage($WRK, $asm, "consensusLoad") == 1);
- goto allDone if ((-e "$wrk/$asm.ctgStore/seqDB.v002.tig") && (-e "$wrk/$asm.utgStore/seqDB.v002.tig"));
+ goto allDone if (skipStage($asm, "consensusLoad") == 1);
+ goto allDone if ((fileExists("unitigging/$asm.ctgStore/seqDB.v002.tig")) &&
+ (fileExists("unitigging/$asm.utgStore/seqDB.v002.tig")));
# Expects to have a list of output files from the consensusCheck() function.
+ fetchFile("$path/ctgcns.files");
+ fetchFile("$path/utgcns.files");
+
caExit("can't find '$path/ctgcns.files' for loading tigs into store: $!", undef) if (! -e "$path/ctgcns.files");
caExit("can't find '$path/utgcns.files' for loading tigs into store: $!", undef) if (! -e "$path/utgcns.files");
# Now just load them.
- $cmd = "$bin/tgStoreLoad \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.ctgStore 2 \\\n";
- $cmd .= " -L $path/ctgcns.files \\\n";
- $cmd .= "> $path/ctgcns.files.ctgStoreLoad.err 2>&1";
+ if (! fileExists("unitigging/$asm.ctgStore/seqDB.v002.tig")) {
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v001.dat");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v001.tig");
+
+ open(F, "< $path/ctgcns.files");
+ while (<F>) {
+ chomp;
+ fetchFile("unitigging/$_");
+ }
+ close(F);
- if (runCommand($path, $cmd)) {
- caExit("failed to load unitig consensus into ctgStore", "$path/ctgcns.files.ctgStoreLoad.err");
+ $cmd = "$bin/tgStoreLoad \\\n";
+ $cmd .= " -G ./$asm.gkpStore \\\n";
+ $cmd .= " -T ./$asm.ctgStore 2 \\\n";
+ $cmd .= " -L ./5-consensus/ctgcns.files \\\n";
+ $cmd .= "> ./5-consensus/ctgcns.files.ctgStoreLoad.err 2>&1";
+
+ if (runCommand("unitigging", $cmd)) {
+ caExit("failed to load unitig consensus into ctgStore", "$path/ctgcns.files.ctgStoreLoad.err");
+ }
+
+ stashFile("unitigging/$asm.ctgStore/seqDB.v002.dat");
+ stashFile("unitigging/$asm.ctgStore/seqDB.v002.tig");
}
- $cmd = "$bin/tgStoreLoad \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.utgStore 2 \\\n";
- $cmd .= " -L $path/utgcns.files \\\n";
- $cmd .= "> $path/utgcns.files.utgStoreLoad.err 2>&1";
+ if (! fileExists("unitigging/$asm.utgStore/seqDB.v002.tig")) {
+ fetchFile("unitigging/$asm.utgStore/seqDB.v001.dat");
+ fetchFile("unitigging/$asm.utgStore/seqDB.v001.tig");
- if (runCommand($path, $cmd)) {
- caExit("failed to load unitig consensus into utgStore", "$path/utgcns.files.utgStoreLoad.err");
+ open(F, "< $path/utgcns.files");
+ while (<F>) {
+ chomp;
+ fetchFile("unitigging/$_");
+ }
+ close(F);
+
+ $cmd = "$bin/tgStoreLoad \\\n";
+ $cmd .= " -G ./$asm.gkpStore \\\n";
+ $cmd .= " -T ./$asm.utgStore 2 \\\n";
+ $cmd .= " -L ./5-consensus/utgcns.files \\\n";
+ $cmd .= "> ./5-consensus/utgcns.files.utgStoreLoad.err 2>&1";
+
+ if (runCommand("unitigging", $cmd)) {
+ caExit("failed to load unitig consensus into utgStore", "$path/utgcns.files.utgStoreLoad.err");
+ }
+
+ stashFile("unitigging/$asm.utgStore/seqDB.v002.dat");
+ stashFile("unitigging/$asm.utgStore/seqDB.v002.tig");
}
# Remvoe consensus outputs
@@ -434,8 +504,8 @@ sub consensusLoad ($$) {
my $Nlayout = 0;
my $Nlog = 0;
- ($Ncns, $Nfastq, $Nlayout, $Nlog) = purgeFiles($path, "ctgcns", $Ncns, $Nfastq, $Nlayout, $Nlog);
- ($Ncns, $Nfastq, $Nlayout, $Nlog) = purgeFiles($path, "utgcns", $Ncns, $Nfastq, $Nlayout, $Nlog);
+ ($Ncns, $Nfastq, $Nlayout, $Nlog) = purgeFiles("ctgcns", $Ncns, $Nfastq, $Nlayout, $Nlog);
+ ($Ncns, $Nfastq, $Nlayout, $Nlog) = purgeFiles("utgcns", $Ncns, $Nfastq, $Nlayout, $Nlog);
print STDERR "-- Purged $Ncns .cns outputs.\n" if ($Ncns > 0);
print STDERR "-- Purged $Nfastq .fastq outputs.\n" if ($Nfastq > 0);
@@ -443,45 +513,172 @@ sub consensusLoad ($$) {
print STDERR "-- Purged $Nlog .err log outputs.\n" if ($Nlog > 0);
}
+ reportUnitigSizes($asm, 2, "after consensus generation");
+
finishStage:
- emitStage($WRK, $asm, "consensusLoad");
- buildHTML($WRK, $asm, "utg");
- stopAfter("consensusLoad");
+ emitStage($asm, "consensusLoad");
+ buildHTML($asm, "utg");
allDone:
- reportUnitigSizes($wrk, $asm, 2, "after consenss generation");
}
-sub consensusAnalyze ($$) {
- my $WRK = shift @_; # Root work directory
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub consensusAnalyze ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/5-consensus";
- goto allDone if (skipStage($WRK, $asm, "consensusAnalyze") == 1);
- goto allDone if (-e "$wrk/$asm.ctgStore/status.coverageStat");
+ goto allDone if (skipStage($asm, "consensusAnalyze") == 1);
+ goto allDone if (fileExists("unitigging/$asm.ctgStore.coverageStat.log"));
+
+ fetchStore("unitigging/$asm.gkpStore");
+
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v001.dat"); # Shouldn't need this, right?
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v001.tig"); # So why does it?
+
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v002.dat");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v002.tig");
$cmd = "$bin/tgStoreCoverageStat \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.ctgStore 2 \\\n";
- $cmd .= " -s " . getGlobal("genomeSize") . " \\\n";
- $cmd .= " -o $wrk/$asm.ctgStore.coverageStat \\\n";
- $cmd .= "> $wrk/$asm.ctgStore.coverageStat.err 2>&1";
-
- if (runCommand($path, $cmd)) {
- caExit("failed to compute coverage statistics", "$wrk/$asm.ctgStore.coverageStat.err");
+ $cmd .= " -G ./$asm.gkpStore \\\n";
+ $cmd .= " -T ./$asm.ctgStore 2 \\\n";
+ $cmd .= " -s " . getGlobal("genomeSize") . " \\\n";
+ $cmd .= " -o ./$asm.ctgStore.coverageStat \\\n";
+ $cmd .= "> ./$asm.ctgStore.coverageStat.err 2>&1";
+
+ if (runCommand("unitigging", $cmd)) {
+ caExit("failed to compute coverage statistics", "unitigging/$asm.ctgStore.coverageStat.err");
+ }
+
+ unlink "unitigging/$asm.ctgStore.coverageStat.err";
+
+ stashFile("unitigging/$asm.ctgStore.coverageStat.stats");
+ stashFile("unitigging/$asm.ctgStore.coverageStat.log");
+
+ finishStage:
+ emitStage($asm, "consensusAnalyze");
+ buildHTML($asm, "utg");
+
+ allDone:
+ stopAfter("consensus");
+}
+
+
+
+
+sub alignGFA ($) {
+ my $asm = shift @_;
+ my $attempt = getGlobal("canuIteration");
+ my $path = "unitigging/4-unitigger";
+
+ # Decide if this is small enough to run right now, or if we should submit to the grid.
+
+ #my $bin = getBinDirectory();
+
+ # This is just big enough to not fit comfortably in the canu process itself.
+
+ goto allDone if (skipStage($asm, "alignGFA") == 1);
+ goto allDone if (fileExists("unitigging/4-unitigger/$asm.contigs.aligned.gfa") &&
+ fileExists("unitigging/4-unitigger/$asm.unitigs.aligned.gfa"));
+
+ fetchFile("$path/alignGFA.sh");
+
+ if (! -e "$path/alignGFA.sh") {
+ open(F, "> $path/alignGFA.sh") or caExit("can't open '$path/alignGFA.sh.sh' for writing: $!\n", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F "\n";
+ print F fetchFileShellCode("unitigging/$asm.utgStore", "seqDB.v001.dat", "");
+ print F fetchFileShellCode("unitigging/$asm.utgStore", "seqDB.v001.tig", "");
+ print F "\n";
+ print F fetchFileShellCode("unitigging/$asm.utgStore", "seqDB.v002.dat", "");
+ print F fetchFileShellCode("unitigging/$asm.utgStore", "seqDB.v002.tig", "");
+ print F "\n";
+ print F fetchFileShellCode("unitigging/$asm.ctgStore", "seqDB.v001.dat", "");
+ print F fetchFileShellCode("unitigging/$asm.ctgStore", "seqDB.v001.tig", "");
+ print F "\n";
+ print F fetchFileShellCode("unitigging/$asm.ctgStore", "seqDB.v002.dat", "");
+ print F fetchFileShellCode("unitigging/$asm.ctgStore", "seqDB.v002.tig", "");
+ print F "\n";
+ print F "\n";
+ print F "if [ ! -e ./$asm.unitigs.aligned.gfa ] ; then\n";
+ print F " \$bin/alignGFA \\\n";
+ print F " -T ../$asm.utgStore 2 \\\n";
+ print F " -i ./$asm.unitigs.gfa \\\n";
+ print F " -o ./$asm.unitigs.aligned.gfa \\\n";
+ print F " -t " . getGlobal("gfaThreads") . " \\\n";
+ print F " > ./$asm.unitigs.aligned.gfa.err 2>&1";
+ print F "\n";
+ print F stashFileShellCode("$path", "$asm.unitigs.aligned.gfa", " ");
+ print F "fi\n";
+ print F "\n";
+ print F "\n";
+ print F "if [ ! -e ./$asm.contigs.aligned.gfa ] ; then\n";
+ print F " \$bin/alignGFA \\\n";
+ print F " -T ../$asm.ctgStore 2 \\\n";
+ print F " -i ./$asm.contigs.gfa \\\n";
+ print F " -o ./$asm.contigs.aligned.gfa \\\n";
+ print F " -t " . getGlobal("gfaThreads") . " \\\n";
+ print F " > ./$asm.contigs.aligned.gfa.err 2>&1";
+ print F "\n";
+ print F stashFileShellCode("$path", "$asm.contigs.aligned.gfa", " ");
+ print F "fi\n";
+ print F "\n";
+ print F "\n";
+ print F "if [ -e ./$asm.unitigs.aligned.gfa -a \\\n";
+ print F " -e ./$asm.contigs.aligned.gfa ] ; then\n";
+ print F " echo GFA alignments updated.\n";
+ print F " exit 0\n";
+ print F "else\n";
+ print F " echo GFA alignments failed.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ close(F);
+
+ system("chmod +x $path/alignGFA.sh");
+
+ stashFile("$path/alignGFA.sh");
+ }
+
+ # Since there is only one job, if we get here, we're not done. Any other 'check' function
+ # shows how to process multiple jobs. This only checks for the existence of the final outputs.
+ # (meryl and unitig are the same)
+
+ # If not the first attempt, report the jobs that failed, and that we're recomputing.
+
+ if ($attempt > 1) {
+ print STDERR "--\n";
+ print STDERR "-- GFA alignment failed.\n";
+ print STDERR "--\n";
+ }
+
+ # If too many attempts, give up.
+
+ if ($attempt > getGlobal("canuIterationMax")) {
+ caExit("failed to align GFA links. Made " . ($attempt-1) . " attempts, jobs still failed", undef);
}
- unlink "$wrk/$asm.ctgStore.coverageStat.err";
+ # Otherwise, run some jobs. If the genome is small, just do it here and now, otherwise,
+ # run on the grid.
+
+ emitStage($asm, "alignGFA", $attempt);
+
+ if (getGlobal("genomeSize") < 40000000) {
+ if (runCommand("$path", "./alignGFA.sh")) {
+ caExit("failed to align contigs", "./$asm.contigs.aligned.gfa.err");
+ }
+ } else {
+ submitOrRunParallelJob($asm, "gfa", $path, "alignGFA", (1));
+ }
+
+ return;
finishStage:
- emitStage($WRK, $asm, "consensusAnalyze");
- buildHTML($WRK, $asm, "utg");
- touch("$wrk/$asm.ctgStore/status.coverageStat");
- stopAfter("consensusAnalyze");
+ emitStage($asm, "alignGFA");
+
allDone:
}
diff --git a/src/pipelines/canu/CorrectReads.pm b/src/pipelines/canu/CorrectReads.pm
index fee4d3f..5c738f6 100644
--- a/src/pipelines/canu/CorrectReads.pm
+++ b/src/pipelines/canu/CorrectReads.pm
@@ -44,25 +44,27 @@ require Exporter;
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
+use canu::Report;
use canu::HTML;
+use canu::Grid_Cloud;
+
# Returns a coverage:
# If $cov not defined, default to desired output coverage * 1.0.
# Otherwise, if defined but ends in an 'x', that's desired output coverage * whatever
# Otherwise, the coverage is as defined.
#
-sub getCorCov ($$$) {
- my $wrk = shift @_; # Local work directory
+sub getCorCov ($$) {
my $asm = shift @_;
my $typ = shift @_;
my $cov = getGlobal("corMaxEvidenceCoverage$typ");
- my $exp = getExpectedCoverage($wrk, $asm);
+ my $exp = getExpectedCoverage("correction", $asm);
my $des = getGlobal("corOutCoverage");
if (!defined($cov)) {
@@ -77,8 +79,7 @@ sub getCorCov ($$$) {
# Query gkpStore to find the read types involved. Return an error rate that is appropriate for
# aligning reads of that type to each other.
-sub getCorErrorRate ($$) {
- my $wrk = shift @_; # Local work directory
+sub getCorErrorRate ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $erate = getGlobal("corErrorRate");
@@ -93,7 +94,7 @@ sub getCorErrorRate ($$) {
my $numNanoporeRaw = 0;
my $numNanoporeCorrected = 0;
- open(L, "< $wrk/$asm.gkpStore/libraries.txt") or caExit("can't open '$wrk/$asm.gkpStore/libraries.txt' for reading: $!", undef);
+ open(L, "< correction/$asm.gkpStore/libraries.txt") or caExit("can't open 'correction/$asm.gkpStore/libraries.txt' for reading: $!", undef);
while (<L>) {
$numPacBioRaw++ if (m/pacbio-raw/);
$numPacBioCorrected++ if (m/pacbio-corrected/);
@@ -117,14 +118,14 @@ sub getCorErrorRate ($$) {
# Return the number of jobs for 'falcon', 'falconpipe' or 'utgcns'
#
-sub computeNumberOfCorrectionJobs ($$) {
- my $wrk = shift @_; # Local work directory
+sub computeNumberOfCorrectionJobs ($) {
my $asm = shift @_;
my $nJobs = 0;
my $nPerJob = 0;
+ my $path = "correction/2-correction";
- if (getGlobal("corConsensus") eq "falcon" && -e "$wrk/2-correction/correction_inputs" ) {
- open(F, "ls $wrk/2-correction/correction_inputs/ |") or caExit("can't find list of correction_inputs: $!", undef);
+ if (getGlobal("corConsensus") eq "falcon" && -e "$path/correction_inputs" ) {
+ open(F, "ls $path/correction_inputs/ |") or caExit("can't find list of correction_inputs: $!", undef);
while (<F>) {
$nJobs++ if (m/^\d\d\d\d$/);
}
@@ -137,7 +138,9 @@ sub computeNumberOfCorrectionJobs ($$) {
(getGlobal("corConsensus") eq "falconpipe") ||
(getGlobal("corConsensus") eq "falcon")) {
my $nPart = getGlobal("corPartitions");
- my $nReads = getNumberOfReadsInStore($wrk, $asm);
+ my $nReads = getNumberOfReadsInStore("correction", $asm);
+
+ caExit("didn't find any reads in store 'correction/$asm.gkpStore'?", undef) if ($nReads == 0);
$nPerJob = int($nReads / $nPart + 1);
$nPerJob = getGlobal("corPartitionMin") if ($nPerJob < getGlobal("corPartitionMin"));
@@ -153,60 +156,60 @@ sub computeNumberOfCorrectionJobs ($$) {
# Generate a corStore, dump files for falcon to process, generate a script to run falcon.
#
-sub buildCorrectionLayouts_direct ($$) {
- my $wrk = shift @_; # Local work directory
+sub buildCorrectionLayouts_direct ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/2-correction";
+ my $base = "correction";
+ my $path = "correction/2-correction";
# Outer level buildCorrectionLayouts() ensures the task is not finished.
- my $maxCov = getCorCov($wrk, $asm, "Local");
+ my $maxCov = getCorCov($asm, "Local");
- if (! -e "$wrk/$asm.corStore") {
+ if (! -e "correction/$asm.corStore") {
$cmd = "$bin/generateCorrectionLayouts \\\n";
- $cmd .= " -rl $path/$asm.readsToCorrect \\\n" if (-e "$path/$asm.readsToCorrect");
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
- $cmd .= " -S $path/$asm.globalScores \\\n" if (-e "$path/$asm.globalScores");
- $cmd .= " -T $wrk/$asm.corStore.WORKING \\\n";
+ $cmd .= " -rl ./$asm.readsToCorrect \\\n" if (-e "$path/$asm.readsToCorrect");
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -O ../$asm.ovlStore \\\n";
+ $cmd .= " -S ./$asm.globalScores \\\n" if (-e "$path/$asm.globalScores");
+ $cmd .= " -T ../$asm.corStore.WORKING \\\n";
$cmd .= " -L " . getGlobal("corMinEvidenceLength") . " \\\n" if (defined(getGlobal("corMinEvidenceLength")));
$cmd .= " -E " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate")));
$cmd .= " -C $maxCov \\\n" if (defined($maxCov));
$cmd .= " -legacy \\\n" if (defined(getGlobal("corLegacyFilter")));
- $cmd .= "> $wrk/$asm.corStore.err 2>&1";
+ $cmd .= "> ../$asm.corStore.err 2>&1";
- if (runCommand($wrk, $cmd)) {
- caExit("failed to generate layouts for correction", "$wrk/$asm.corStore.err");
+ if (runCommand($path, $cmd)) {
+ caExit("failed to generate layouts for correction", "correction/$asm.corStore.err");
}
- rename "$wrk/$asm.corStore.WORKING", "$wrk/$asm.corStore";
+ rename "correction/$asm.corStore.WORKING", "correction/$asm.corStore";
}
# first we call this function to compute partioning
- my ($jobs, $nPer) = computeNumberOfCorrectionJobs($wrk, $asm);
+ my ($jobs, $nPer) = computeNumberOfCorrectionJobs($asm);
- make_path("$path/correction_inputs") if (! -d "$path/correction_inputs");
+ make_path("$path/correction_inputs") if (! -d "$path/correction_inputs");
make_path("$path/correction_outputs") if (! -d "$path/correction_outputs");
if (getGlobal("corConsensus") eq "falcon") {
$cmd = "$bin/createFalconSenseInputs \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.corStore 1 \\\n";
- $cmd .= " -o $path/correction_inputs/ \\\n";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -T ../$asm.corStore 1 \\\n";
+ $cmd .= " -o ./correction_inputs/ \\\n";
$cmd .= " -p " . $jobs . " \\\n";
- $cmd .= "> $path/correction_inputs.err 2>&1";
+ $cmd .= "> ./correction_inputs.err 2>&1";
- if (runCommand($wrk, $cmd)) {
+ if (runCommand($path, $cmd)) {
caExit("failed to generate falcon inputs", "$path/correction_inputs.err");
}
}
if (getGlobal("corConsensus") eq "falcon") {
#the second call will confirm we have the proper number of output files and set jobs
- ($jobs, $nPer) = computeNumberOfCorrectionJobs($wrk, $asm);
+ ($jobs, $nPer) = computeNumberOfCorrectionJobs($asm);
}
#getAllowedResources("", "cor");
@@ -215,6 +218,11 @@ sub buildCorrectionLayouts_direct ($$) {
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/2-correction", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
print F "if [ \$jobid -gt $jobs ]; then\n";
@@ -229,19 +237,40 @@ sub buildCorrectionLayouts_direct ($$) {
}
print F "jobid=`printf %04d \$jobid`\n";
print F "\n";
- print F "if [ -e \"$path/correction_outputs/\$jobid.fasta\" ] ; then\n";
+ print F "if [ -e \"./correction_outputs/\$jobid.fasta\" ] ; then\n";
print F " echo Job finished successfully.\n";
print F " exit 0\n";
print F "fi\n";
print F "\n";
- print F "if [ ! -d \"$path/correction_outputs\" ] ; then\n";
- print F " mkdir -p \"$path/correction_outputs\"\n";
+ print F "if [ ! -d \"./correction_outputs\" ] ; then\n";
+ print F " mkdir -p \"./correction_outputs\"\n";
print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
+ print F "gkpStore=\"../$asm.gkpStore\"\n";
+ print F "\n";
+
+ my $stageDir = getGlobal("stageDirectory");
- my $erate = getCorErrorRate($wrk, $asm);
+ if (defined($stageDir)) {
+ print F "if [ ! -d $stageDir ] ; then\n";
+ print F " mkdir -p $stageDir\n";
+ print F "fi\n";
+ print F "\n";
+ print F "mkdir -p $stageDir/$asm.gkpStore\n";
+ print F "\n";
+ print F "echo Start copy at `date`\n";
+ print F "cp -p \$gkpStore/info $stageDir/$asm.gkpStore/info\n";
+ print F "cp -p \$gkpStore/libraries $stageDir/$asm.gkpStore/libraries\n";
+ print F "cp -p \$gkpStore/reads $stageDir/$asm.gkpStore/reads\n";
+ print F "cp -p \$gkpStore/blobs $stageDir/$asm.gkpStore/blobs\n";
+ print F "echo Finished at `date`\n";
+ print F "\n";
+ print F "gkpStore=\"$stageDir/$asm.gkpStore\"\n";
+ print F "\n";
+ }
+
+ my $erate = getCorErrorRate($asm);
my $minidt = 1 - $erate;
# UTGCNS for correction is writing FASTQ, but needs to write FASTA. The names below were changed to fasta preemptively.
@@ -252,39 +281,49 @@ sub buildCorrectionLayouts_direct ($$) {
print F "\$bin/utgcns \\\n";
print F " -u \$bgn-\$end \\\n";
print F " -e $erate \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -T $wrk/$asm.corStore 1 . \\\n";
- print F " -O $path/correction_outputs/\$jobid.cns.WORKING \\\n";
- print F " -L $path/correction_outputs/\$jobid.layout.WORKING \\\n";
- print F " -F $path/correction_outputs/\$jobid.fasta.WORKING \\\n";
+ print F " -G \$gkpStore \\\n";
+ print F " -T ../$asm.corStore 1 . \\\n";
+ print F " -O ./correction_outputs/\$jobid.cns.WORKING \\\n";
+ print F " -L ./correction_outputs/\$jobid.layout.WORKING \\\n";
+ print F " -F ./correction_outputs/\$jobid.fasta.WORKING \\\n";
print F "&& \\\n";
- print F "mv $path/correction_outputs/\$jobid.cns.WORKING $path/correction_outputs/\$jobid.cns \\\n";
+ print F "mv ./correction_outputs/\$jobid.cns.WORKING ./correction_outputs/\$jobid.cns \\\n";
print F "&& \\\n";
- print F "mv $path/correction_outputs/\$jobid.layout.WORKING $path/correction_outputs/\$jobid.layout \\\n";
+ print F "mv ./correction_outputs/\$jobid.layout.WORKING ./correction_outputs/\$jobid.layout \\\n";
print F "&& \\\n";
- print F "mv $path/correction_outputs/\$jobid.fasta.WORKING $path/correction_outputs/\$jobid.fasta \\\n";
+ print F "mv ./correction_outputs/\$jobid.fasta.WORKING ./correction_outputs/\$jobid.fasta \\\n";
print F "\n";
}
if (getGlobal("corConsensus") eq "falcon") {
print F "\n";
- print F getGlobal("falconSense") . " \\\n" if ( defined(getGlobal("falconSense")));
- print F "\$bin/falcon_sense \\\n" if (!defined(getGlobal("falconSense")));
+ print F "\$bin/falcon_sense \\\n";
print F " --min_idt $minidt \\\n";
print F " --min_len " . getGlobal("minReadLength") . "\\\n";
- print F " --max_read_len " . 2 * getMaxReadInStore($wrk, $asm) . "\\\n";
+ print F " --max_read_len " . 2 * getMaxReadLengthInStore($base, $asm) . " \\\n";
print F " --min_ovl_len " . getGlobal("minOverlapLength") . "\\\n";
print F " --min_cov " . getGlobal("corMinCoverage") . " \\\n";
print F " --n_core " . getGlobal("corThreads") . " \\\n";
- print F " < $path/correction_inputs/\$jobid \\\n";
- print F " > $path/correction_outputs/\$jobid.fasta.WORKING \\\n";
- print F " 2> $path/correction_outputs/\$jobid.err \\\n";
+ print F " < ./correction_inputs/\$jobid \\\n";
+ print F " > ./correction_outputs/\$jobid.fasta.WORKING \\\n";
+ print F " 2> ./correction_outputs/\$jobid.err \\\n";
print F "&& \\\n";
- print F "mv $path/correction_outputs/\$jobid.fasta.WORKING $path/correction_outputs/\$jobid.fasta \\\n";
+ print F "mv ./correction_outputs/\$jobid.fasta.WORKING ./correction_outputs/\$jobid.fasta \\\n";
}
+ if (defined($stageDir)) {
+ print F "\n";
+ print F "rm -rf $stageDir/$asm.gkpStore\n"; # Prevent accidents of 'rm -rf /' if stageDir = "/".
+ print F "rmdir $stageDir\n";
+ }
+
+ print F "\n";
+ print F "exit 0\n";
+
close(F);
+ stashFile("$path/correctReads.sh");
+
finishStage:
;
allDone:
@@ -296,21 +335,22 @@ sub buildCorrectionLayouts_direct ($$) {
# For falcon_sense, using a pipe and no intermediate files
#
-sub buildCorrectionLayouts_piped ($$) {
- my $wrk = shift @_; # Local work directory
+sub buildCorrectionLayouts_piped ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/2-correction";
+
+ my $base = "correction";
+ my $path = "correction/2-correction";
# Outer level buildCorrectionLayouts() ensures the task is not finished.
- make_path("$path/correction_inputs") if (! -d "$path/correction_inputs");
+ make_path("$path/correction_inputs") if (! -d "$path/correction_inputs");
make_path("$path/correction_outputs") if (! -d "$path/correction_outputs");
- my ($nJobs, $nPerJob) = computeNumberOfCorrectionJobs($wrk, $asm); # Does math based on number of reads and parameters.
+ my ($nJobs, $nPerJob) = computeNumberOfCorrectionJobs($asm); # Does math based on number of reads and parameters.
- my $nReads = getNumberOfReadsInStore($wrk, $asm);
+ my $nReads = getNumberOfReadsInStore("correction", $asm);
#getAllowedResources("", "cor");
@@ -318,6 +358,11 @@ sub buildCorrectionLayouts_piped ($$) {
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/2-correction", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
print F "if [ \$jobid -gt $nJobs ]; then\n";
@@ -346,21 +391,51 @@ sub buildCorrectionLayouts_piped ($$) {
print F "\n";
print F "jobid=`printf %04d \$jobid`\n";
print F "\n";
- print F "if [ -e \"$path/correction_outputs/\$jobid.fasta\" ] ; then\n";
+ print F "if [ -e \"./correction_outputs/\$jobid.fasta\" ] ; then\n";
print F " echo Job finished successfully.\n";
print F " exit 0\n";
print F "fi\n";
print F "\n";
- print F "if [ ! -d \"$path/correction_outputs\" ] ; then\n";
- print F " mkdir -p \"$path/correction_outputs\"\n";
+ print F "if [ ! -d \"./correction_outputs\" ] ; then\n";
+ print F " mkdir -p \"./correction_outputs\"\n";
print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
+
+ print F fetchStoreShellCode("correction/$asm.gkpStore", "correction/3-correction", "");
+ print F "\n";
+ print F fetchStoreShellCode("correction/$asm.ovlStore", "correction/3-correction", "");
+ print F "\n";
+ print F fetchFileShellCode("correction/2-correction", "$asm.readsToCorrect", "");
+ print F "\n";
+ print F fetchFileShellCode("correction/2-correction", "$asm.globalScores", "");
print F "\n";
- my $maxCov = getCorCov($wrk, $asm, "Local");
+ print F "gkpStore=\"../$asm.gkpStore\"\n";
+ print F "\n";
+
+ my $stageDir = getGlobal("stageDirectory");
+
+ if (defined($stageDir)) {
+ print F "if [ ! -d $stageDir ] ; then\n";
+ print F " mkdir -p $stageDir\n";
+ print F "fi\n";
+ print F "\n";
+ print F "mkdir -p $stageDir/$asm.gkpStore\n";
+ print F "\n";
+ print F "echo Start copy at `date`\n";
+ print F "cp -p \$gkpStore/info $stageDir/$asm.gkpStore/info\n";
+ print F "cp -p \$gkpStore/libraries $stageDir/$asm.gkpStore/libraries\n";
+ print F "cp -p \$gkpStore/reads $stageDir/$asm.gkpStore/reads\n";
+ print F "cp -p \$gkpStore/blobs $stageDir/$asm.gkpStore/blobs\n";
+ print F "echo Finished at `date`\n";
+ print F "\n";
+ print F "gkpStore=\"$stageDir/$asm.gkpStore\"\n";
+ print F "\n";
+ }
+
+ my $maxCov = getCorCov($asm, "Local");
- my $erate = getCorErrorRate($wrk, $asm);
+ my $erate = getCorErrorRate($asm);
my $minidt = 1 - $erate;
print F "\n";
@@ -370,41 +445,52 @@ sub buildCorrectionLayouts_piped ($$) {
print F "\n";
print F "( \\\n";
print F "\$bin/generateCorrectionLayouts -b \$bgn -e \$end \\\n";
- print F " -rl $path/$asm.readsToCorrect \\\n" if (-e "$path/$asm.readsToCorrect");
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $wrk/$asm.ovlStore \\\n";
- print F " -S $path/$asm.globalScores \\\n" if (-e "$path/$asm.globalScores");
+ print F " -rl ./$asm.readsToCorrect \\\n" if (-e "$path/$asm.readsToCorrect");
+ print F " -G \$gkpStore \\\n";
+ print F " -O ../$asm.ovlStore \\\n";
+ print F " -S ./$asm.globalScores \\\n" if (-e "$path/$asm.globalScores");
print F " -L " . getGlobal("corMinEvidenceLength") . " \\\n" if (defined(getGlobal("corMinEvidenceLength")));
print F " -E " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate")));
print F " -C $maxCov \\\n" if (defined($maxCov));
print F " -legacy \\\n" if (defined(getGlobal("corLegacyFilter")));
print F " -F \\\n";
print F "&& \\\n";
- print F " touch $path/correction_outputs/\$jobid.dump.success \\\n";
+ print F " touch ./correction_outputs/\$jobid.dump.success \\\n";
print F ") \\\n";
print F "| \\\n";
- print F getGlobal("falconSense") . " \\\n" if ( defined(getGlobal("falconSense")));
- print F "\$bin/falcon_sense \\\n" if (!defined(getGlobal("falconSense")));
+ print F "\$bin/falcon_sense \\\n";
print F " --min_idt $minidt \\\n";
print F " --min_len " . getGlobal("minReadLength") . "\\\n";
- print F " --max_read_len " . 2 * getMaxReadInStore($wrk, $asm) . "\\\n";
+ print F " --max_read_len " . 2 * getMaxReadLengthInStore($base, $asm) . " \\\n";
print F " --min_ovl_len " . getGlobal("minOverlapLength") . "\\\n";
print F " --min_cov " . getGlobal("corMinCoverage") . " \\\n";
print F " --n_core " . getGlobal("corThreads") . " \\\n";
- print F " > $path/correction_outputs/\$jobid.fasta.WORKING \\\n";
- print F " 2> $path/correction_outputs/\$jobid.err \\\n";
+ print F " > ./correction_outputs/\$jobid.fasta.WORKING \\\n";
+ print F " 2> ./correction_outputs/\$jobid.err \\\n";
print F "&& \\\n";
- print F "mv $path/correction_outputs/\$jobid.fasta.WORKING $path/correction_outputs/\$jobid.fasta \\\n";
+ print F "mv ./correction_outputs/\$jobid.fasta.WORKING ./correction_outputs/\$jobid.fasta \\\n";
print F "\n";
- print F "if [ ! -e \"$path/correction_outputs/\$jobid.dump.success\" ] ; then\n";
+ print F "if [ ! -e \"./correction_outputs/\$jobid.dump.success\" ] ; then\n";
print F " echo Read layout generation failed.\n";
- print F " mv $path/correction_outputs/\$jobid.fasta $path/correction_outputs/\$jobid.fasta.INCOMPLETE\n";
+ print F " mv ./correction_outputs/\$jobid.fasta ./correction_outputs/\$jobid.fasta.INCOMPLETE\n";
print F "fi\n";
print F "\n";
+
+ if (defined($stageDir)) {
+ print F "rm -rf $stageDir/$asm.gkpStore\n"; # Prevent accidents of 'rm -rf /' if stageDir = "/".
+ print F "rmdir $stageDir\n";
+ print F "\n";
+ }
+
+ print F stashFileShellCode("$path", "correction_outputs/\$jobid.fasta", "");
+
+ print F "\n";
print F "exit 0\n";
close(F);
+ stashFile("$path/correctReads.sh");
+
finishStage:
;
allDone:
@@ -419,30 +505,32 @@ sub lengthStats (@) {
my $mean = 0;
my $n50 = 0;
- foreach my $v (@v) {
- $total += $v;
- $n50 = $v if ($total < getGlobal("genomeSize") / 2);
- }
+ if (scalar(@v) > 0) {
+ foreach my $v (@v) {
+ $total += $v;
+ $n50 = $v if ($total < getGlobal("genomeSize") / 2);
+ }
- $mean = int($total / scalar(@v) + 0.5);
+ $mean = int($total / scalar(@v) + 0.5);
+ }
return($mean, $n50);
}
-sub quickFilter ($$$) {
- my $wrk = shift @_; # Local work directory
- my $asm = shift @_;
+sub quickFilter ($$) {
+ my $asm = shift @_;
my $minTotal = shift @_;
- my $bin = getBinDirectory();
- my $path = "$wrk/2-correction";
+ my $bin = getBinDirectory();
+
+ my $path = "correction/2-correction";
my $totCorLengthIn = 0;
my $totCorLengthOut = 0;
my $minCorLength = 0;
open(O, "> $path/$asm.readsToCorrect.WORKING") or caExit("can't open '$path/$asm.readsToCorrect.WORKING' for writing: $!\n", undef);
- open(F, "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads | sort -T . -k3nr | ") or caExit("can't dump gatekeeper for read lengths: $!\n", undef);
+ open(F, "$bin/gatekeeperDumpMetaData -G correction/$asm.gkpStore -reads | sort -T . -k3nr | ") or caExit("can't dump gatekeeper for read lengths: $!\n", undef);
print O "read\toriginalLength\tcorrectedLength\n";
@@ -463,46 +551,63 @@ sub quickFilter ($$$) {
close(O);
rename "$path/$asm.readsToCorrect.WORKING", "$path/$asm.readsToCorrect";
+
+ stashFile("$path/$asm.readsToCorrect");
}
-sub expensiveFilter ($$) {
- my $wrk = shift @_; # Local work directory
- my $asm = shift @_;
- my $bin = getBinDirectory();
+sub expensiveFilter ($) {
+ my $asm = shift @_;
+ my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/2-correction";
- my $maxCov = getCorCov($wrk, $asm, "Local");
+ my $path = "correction/2-correction";
+
+ my $minCov = getGlobal("corMinCoverage");
+ my $maxCov = getCorCov($asm, "Local");
+
+ if (! fileExists("$path/$asm.estimate.log")) {
+ print STDERR "-- Computing expected corrected read lengths '$path/$asm.estimate.log'.\n";
- if (! -e "$path/$asm.estimate.log") {
$cmd = "$bin/generateCorrectionLayouts \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
- $cmd .= " -S $path/$asm.globalScores \\\n" if (-e "$path/$asm.globalScores");
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -O ../$asm.ovlStore \\\n";
+ $cmd .= " -S ./$asm.globalScores \\\n" if (-e "$path/$asm.globalScores");
$cmd .= " -L " . getGlobal("corMinEvidenceLength") . " \\\n" if (defined(getGlobal("corMinEvidenceLength")));
$cmd .= " -E " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate")));
+ $cmd .= " -c $minCov \\\n" if (defined($minCov));
$cmd .= " -C $maxCov \\\n" if (defined($maxCov));
$cmd .= " -legacy \\\n" if (defined(getGlobal("corLegacyFilter")));
- $cmd .= " -p $path/$asm.estimate.WORKING";
+ $cmd .= " -p ./$asm.estimate.WORKING";
- if (runCommand($wrk, $cmd)) {
+ if (runCommand($path, $cmd)) {
rename "$path/$asm.estimate.log", "$path/$asm.estimate.log.FAILED";
- caExit("failed to generate estimated lengths of corrected reads", "$wrk/$asm.corStore.err");
+ caExit("failed to generate estimated lengths of corrected reads", "correction/$asm.corStore.err");
}
rename "$path/$asm.estimate.WORKING.filter.log", "$path/$asm.estimate";
rename "$path/$asm.estimate.WORKING.summary", "$path/$asm.estimate.stats";
rename "$path/$asm.estimate.WORKING.log", "$path/$asm.estimate.log";
+
+ unlink "$path/$asm.estimate.correctedLength.log";
+ unlink "$path/$asm.estimate.originalLength.log";
+ } else {
+ print STDERR "-- Expected corrected read lengths found in '$path/$asm.estimate.log'.\n";
}
- if (runCommandSilently($path, "sort -T . -k4nr -k2nr < $path/$asm.estimate.log > $path/$asm.estimate.correctedLength.log", 1)) {
- caExit("failed to sort by corrected read length", undef);
+ if (! -e "$path/$asm.estimate.correctedLength.log") {
+ print STDERR "-- Sorting reads by expected corrected length.\n";
+ if (runCommandSilently($path, "sort -T . -k4nr -k2nr < ./$asm.estimate.log > ./$asm.estimate.correctedLength.log", 1)) {
+ caExit("failed to sort by corrected read length", undef);
+ }
}
- if (runCommandSilently($path, "sort -T . -k2nr -k4nr < $path/$asm.estimate.log > $path/$asm.estimate.originalLength.log", 1)) {
- caExit("failed to sort by original read length", undef);
+ if (! -e "$path/$asm.estimate.originalLength.log") {
+ print STDERR "-- Sorting reads by uncorrected length.\n";
+ if (runCommandSilently($path, "sort -T . -k2nr -k4nr < ./$asm.estimate.log > ./$asm.estimate.originalLength.log", 1)) {
+ caExit("failed to sort by original read length", undef);
+ }
}
my $totRawLengthIn = 0; # Bases in raw reads we correct
@@ -517,9 +622,17 @@ sub expensiveFilter ($$) {
# Lists of reads to correct if we use the raw length or the corrected length as a filter.
- my %rawReads;
- my %corReads;
- my %corReadLen;
+ my $nReads = getNumberOfReadsInStore("correction", $asm);
+
+ my @rawReads;
+ my @corReads;
+ my @corReadLen;
+
+ for (my $ii=0; $ii<=$nReads; $ii++) {
+ $rawReads[$ii] = undef;
+ $corReads[$ii] = undef;
+ $corReadLen[$ii] = undef;
+ }
# The expected length of the corrected reads, based on the filter
@@ -528,15 +641,19 @@ sub expensiveFilter ($$) {
# Filter!
+ print STDERR "-- Loading expected corrected read lengths.\n";
+
open(F, "< $path/$asm.estimate.originalLength.log");
while (<F>) {
my @v = split '\s+', $_;
next if ($v[0] eq "read");
- $corReadLen{$v[0]} = $v[3];
+ $corReadLen[int($v[0])] = int($v[3]);
}
close(F);
+ print STDERR "-- Picking longest corrected reads.\n";
+
open(F, "< $path/$asm.estimate.originalLength.log");
while (<F>) {
my @v = split '\s+', $_;
@@ -547,7 +664,7 @@ sub expensiveFilter ($$) {
#print O "$v[0]\t$v[1]\t$v[3]\n";
- $rawReads{$v[0]} = 1;
+ $rawReads[int($v[0])] = 1;
push @corLengthRawFilter, $v[3];
@@ -557,6 +674,8 @@ sub expensiveFilter ($$) {
}
}
+ print STDERR "-- Writing longest corrected reads to '$path/$asm.readsToCorrect'.\n";
+
open(F, "< $path/$asm.estimate.correctedLength.log");
open(O, "| sort -T . -k1n > $path/$asm.readsToCorrect.WORKING") or caExit("can't open sort -k1n > '$path/$asm.readsToCorrect.WORKING' for writing: $!\n", undef);
@@ -571,7 +690,7 @@ sub expensiveFilter ($$) {
print O "$v[0]\t$v[1]\t$v[3]\n";
- $corReads{$v[0]} = 1;
+ $corReads[int($v[0])] = 1;
push @corLengthCorFilter, $v[3];
@@ -585,9 +704,12 @@ sub expensiveFilter ($$) {
rename "$path/$asm.readsToCorrect.WORKING", "$path/$asm.readsToCorrect";
+ stashFile("$path/$asm.readsToCorrect");
# Generate true/false positive/negative lists.
+ print STDERR "-- Summarizing filter.\n";
+
open(F, "< $path/$asm.estimate.correctedLength.log") or die;
open(TN, "> $path/$asm.estimate.tn.log") or die;
@@ -604,15 +726,17 @@ sub expensiveFilter ($$) {
my @v = split '\s+', $_;
next if ($v[0] eq "read");
- my $er = exists($rawReads{$v[0]});
- my $ec = exists($corReads{$v[0]});
+ my $er = defined($rawReads[int($v[0])]);
+ my $ec = defined($corReads[int($v[0])]);
- if (($er == 0) && ($ec == 0)) { print TN $_; $tnReads++; $tnBasesR += $v[1]; $tnBasesC += $corReadLen{$v[0]}; } # True negative, yay!
- if (($er == 0) && ($ec == 1)) { print FN $_; $fnReads++; $fnBasesR += $v[1]; $fnBasesC += $corReadLen{$v[0]}; } # False negative. Bad.
- if (($er == 1) && ($ec == 0)) { print FP $_; $fpReads++; $fpBasesR += $v[1]; $fpBasesC += $corReadLen{$v[0]}; } # False positive. Bad.
- if (($er == 1) && ($ec == 1)) { print TP $_; $tpReads++; $tpBasesR += $v[1]; $tpBasesC += $corReadLen{$v[0]}; } # True positive, yay!
+ if (($er == 0) && ($ec == 0)) { print TN $_; $tnReads++; $tnBasesR += $v[1]; $tnBasesC += $corReadLen[$v[0]]; } # True negative, yay!
+ if (($er == 0) && ($ec == 1)) { print FN $_; $fnReads++; $fnBasesR += $v[1]; $fnBasesC += $corReadLen[$v[0]]; } # False negative. Bad.
+ if (($er == 1) && ($ec == 0)) { print FP $_; $fpReads++; $fpBasesR += $v[1]; $fpBasesC += $corReadLen[$v[0]]; } # False positive. Bad.
+ if (($er == 1) && ($ec == 1)) { print TP $_; $tpReads++; $tpBasesR += $v[1]; $tpBasesC += $corReadLen[$v[0]]; } # True positive, yay!
}
+ undef @corReadLen;
+
close(TP);
close(FP);
close(FN);
@@ -637,10 +761,24 @@ sub expensiveFilter ($$) {
my ($rawFilterMean, $rawFilterN50) = lengthStats(@corLengthRawFilter);
my ($corFilterMean, $corFilterN50) = lengthStats(@corLengthCorFilter);
+ undef @corLengthRawFilter;
+ undef @corLengthCorFilter;
+
+ my $nCorReads = 0;
+ my $nRawReads = 0;
+
+ for (my $ii=0; $ii<=$nReads; $ii++) {
+ $nCorReads++ if (defined($corReads[$ii]));
+ $nRawReads++ if (defined($rawReads[$ii]));
+ }
+
+ undef @corReads;
+ undef @rawReads;
+
open(F, "> $path/$asm.readsToCorrect.summary") or caExit("can't open '$path/$asm.readsToCorrect.summary' for writing: $!\n", undef);
print F "Corrected read length filter:\n";
print F "\n";
- print F " nReads ", scalar(keys %corReads), "\n";
+ print F " nReads $nCorReads\n";
print F " nBases $totCorLengthIn (input bases)\n";
print F " nBases $totCorLengthOut (corrected bases)\n";
print F " Mean $corFilterMean\n";
@@ -648,7 +786,7 @@ sub expensiveFilter ($$) {
print F "\n";
print F "Raw read length filter:\n";
print F "\n";
- print F " nReads ", scalar(keys %rawReads), "\n";
+ print F " nReads $nRawReads\n";
print F " nBases $totRawLengthIn (input bases)\n";
print F " nBases $totRawLengthOut (corrected bases)\n";
print F " Mean $rawFilterMean\n";
@@ -660,10 +798,27 @@ sub expensiveFilter ($$) {
printf F "TP %9d reads %13d raw bases (%6d ave) %13d corrected bases (%6d ave)\n", $tpReads, $tpBasesR, $tpBasesRave, $tpBasesC, $tpBasesCave;
close(F);
+ stashFile("$path/$asm.readsToCorrect.summary");
+
+ my $report;
+
+ $report = "--\n";
+ $report .= "-- Reads to be corrected:\n";
+ $report .= "-- $nCorReads reads longer than $minRawLength bp\n";
+ $report .= "-- $totCorLengthIn bp\n";
+ $report .= "-- Expected corrected reads:\n";
+ $report .= "-- $nCorReads reads\n";
+ $report .= "-- $totCorLengthOut bp\n";
+ $report .= "-- $minCorLength bp minimum length\n";
+ $report .= "-- $corFilterMean bp mean length\n";
+ $report .= "-- $corFilterN50 bp n50 length\n";
+
+ addToReport("corrections", $report);
+
# Plot a scatter plot of the original vs the expected corrected read lengths. Early versions
# also plotted the sorted length vs the other length, but those were not interesting.
- if (! -e "$path/$asm.estimate.original-x-correctedLength.gp") {
+ if (! fileExists("$path/$asm.estimate.original-x-correctedLength.gp")) {
my $gnuplot = getGlobal("gnuplot");
my $format = getGlobal("gnuplotImageFormat");
@@ -674,35 +829,37 @@ sub expensiveFilter ($$) {
print F "set pointsize 0.25\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$path/$asm.estimate.original-x-corrected.lg.$format'\n";
- print F "plot '$path/$asm.estimate.tn.log' using 2:4 title 'tn', \\\n";
- print F " '$path/$asm.estimate.fn.log' using 2:4 title 'fn', \\\n";
- print F " '$path/$asm.estimate.fp.log' using 2:4 title 'fp', \\\n";
- print F " '$path/$asm.estimate.tp.log' using 2:4 title 'tp'\n";
+ print F "set output './$asm.estimate.original-x-corrected.lg.$format'\n";
+ print F "plot './$asm.estimate.tn.log' using 2:4 title 'tn', \\\n";
+ print F " './$asm.estimate.fn.log' using 2:4 title 'fn', \\\n";
+ print F " './$asm.estimate.fp.log' using 2:4 title 'fp', \\\n";
+ print F " './$asm.estimate.tp.log' using 2:4 title 'tp'\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$path/$asm.estimate.original-x-corrected.sm.$format'\n";
+ print F "set output './$asm.estimate.original-x-corrected.sm.$format'\n";
print F "replot\n";
close(F);
- if (runCommandSilently($path, "$gnuplot $path/$asm.estimate.original-x-correctedLength.gp > /dev/null 2>&1", 0)) {
+ if (runCommandSilently($path, "$gnuplot ./$asm.estimate.original-x-correctedLength.gp > /dev/null 2>&1", 0)) {
print STDERR "--\n";
print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n";
print STDERR "--\n";
print STDERR "----------------------------------------\n";
}
+
+ stashFile("$path/$asm.estimate.original-x-correctedLength.gp");
+ stashFile("$path/$asm.estimate.original-x-corrected.lg.$format");
+ stashFile("$path/$asm.estimate.original-x-corrected.sm.$format");
}
}
-sub buildCorrectionLayouts ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/correction"; # Local work directory
+sub buildCorrectionLayouts ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/2-correction";
+ my $path = "correction/2-correction";
# All we do here is decide if the job is finished, and delegate
# to the correct function if not finished.
@@ -716,29 +873,47 @@ sub buildCorrectionLayouts ($$) {
# A one-pass algorithm would write layouts to tigStore, computing stats as before, then delete
# tigs that shouldn't be corrected. I suspect this will be slower.
- goto allDone if (skipStage($WRK, $asm, "cor-buildCorrectionLayouts") == 1);
- goto allDone if (sequenceFileExists("$WRK/$asm.correctedReads")); # Output exists
- goto allDone if (-e "$path/cnsjob.files"); # Jobs all finished
- goto allDone if (-e "$path/correctReads.sh"); # Jobs created
+ goto allDone if (skipStage($asm, "cor-buildCorrectionLayouts") == 1);
+ goto allDone if (sequenceFileExists("$asm.correctedReads")); # Output exists
+ goto allDone if (fileExists("$path/cnsjob.files")); # Jobs all finished
+ goto allDone if (fileExists("$path/correctReads.sh")); # Jobs created
make_path("$path") if (! -d "$path");
+ # Set the minimum coverage for a corrected read based on coverage in input reads.
+
+ if (!defined(getGlobal("corMinCoverage"))) {
+ my $cov = getExpectedCoverage("correction", $asm);
+
+ setGlobal("corMinCoverage", 4);
+ setGlobal("corMinCoverage", 4) if ($cov < 60);
+ setGlobal("corMinCoverage", 0) if ($cov <= 20);
+
+ print STDERR "-- Set corMinCoverage=", getGlobal("corMinCoverage"), " based on read coverage of $cov.\n";
+ }
+
# This will eventually get rolled into overlap store creation. Generate a list of scores for
# 'global' overlap filtering.
- if (! -e "$path/$asm.globalScores") {
- my $maxCov = getCorCov($wrk, $asm, "Global");
+ fetchFile("$path/$asm.globalScores");
+
+ if (! fileExists("$path/$asm.globalScores")) {
+ print STDERR "-- Computing global filter scores '$path/$asm.globalScores'.\n";
+
+ fetchStore("./correction/$asm.ovlStore");
+
+ my $maxCov = getCorCov($asm, "Global");
my $minLen = (defined(getGlobal("corMinEvidenceLength"))) ? getGlobal("corMinEvidenceLength") : 0;
$cmd = "$bin/filterCorrectionOverlaps \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
- $cmd .= " -S $path/$asm.globalScores.WORKING \\\n";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -O ../$asm.ovlStore \\\n";
+ $cmd .= " -S ./$asm.globalScores.WORKING \\\n";
$cmd .= " -c $maxCov \\\n";
$cmd .= " -l $minLen \\\n";
$cmd .= " -e " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate")));
$cmd .= " -legacy \\\n" if (defined(getGlobal("corLegacyFilter")));
- $cmd .= "> $path/$asm.globalScores.err 2>&1";
+ $cmd .= "> ./$asm.globalScores.err 2>&1";
if (runCommand($path, $cmd)) {
caExit("failed to globally filter overlaps for correction", "$path/$asm.globalScores.err");
@@ -748,7 +923,22 @@ sub buildCorrectionLayouts ($$) {
rename "$path/$asm.globalScores.WORKING.stats", "$path/$asm.globalScores.stats";
rename "$path/$asm.globalScores.WORKING.log", "$path/$asm.globalScores.log";
unlink "$path/$asm.globalScores.err";
+
+ stashFile("$path/$asm.globalScores");
+ } else {
+ print STDERR "-- Global filter scores found in '$path/$asm.globalScores'.\n";
+ }
+
+ my $report;
+
+#FORMAT
+ open(F, "< $path/$asm.globalScores.stats") or caExit("can't open '$path/$asm.globalScores.stats' for reading: $!", undef);
+ while(<F>) {
+ $report .= "-- $_";
}
+ close(F);
+
+ addToReport("filtering", $report);
# For 'quick' filtering, but more reads to correct, sort the reads by length, and correct the
# longest Nx of reads.
@@ -758,40 +948,34 @@ sub buildCorrectionLayouts ($$) {
#
# Both are required to create a file $asm.readsToCorrect, containing a list of IDs to correct.
- if (getGlobal("corFilter") eq "quick") {
- quickFilter($wrk, $asm, (getGlobal("genomeSize") * getGlobal("corOutCoverage")));
+ fetchFile("$path/$asm.readsToCorrect");
- } elsif (getGlobal("corFilter") eq "expensive") {
- expensiveFilter($wrk, $asm);
+ if (! fileExists("$path/$asm.readsToCorrect")) {
+ if (getGlobal("corFilter") eq "quick") {
+ quickFilter($asm, (getGlobal("genomeSize") * getGlobal("corOutCoverage")));
- } elsif (getGlobal("corFilter") eq "none" ) {
- quickFilter($wrk, $asm, 0);
+ } elsif (getGlobal("corFilter") eq "expensive") {
+ expensiveFilter($asm);
- } else {
- caFailure("unknown corFilter '" . getGlobal("corFilter") . "'", undef);
- }
-
- # Set the minimum coverage for a corrected read based on coverage in input reads.
-
- if (!defined(getGlobal("corMinCoverage"))) {
- my $cov = getExpectedCoverage($wrk, $asm);
+ } elsif (getGlobal("corFilter") eq "none" ) {
+ quickFilter($asm, 0);
- setGlobal("corMinCoverage", 4);
- setGlobal("corMinCoverage", 4) if ($cov < 60);
- setGlobal("corMinCoverage", 0) if ($cov <= 20);
+ } else {
+ caFailure("unknown corFilter '" . getGlobal("corFilter") . "'", undef);
+ }
- print STDERR "-- Set corMinCoverage=", getGlobal("corMinCoverage"), " based on read coverage of $cov.\n";
+ caExit("failed to create list of reads to correct", undef) if (! -e "$path/$asm.readsToCorrect");
+ } else {
+ print STDERR "-- Filtered list of reads found in '$path/$asm.readsToCorrect'.\n";
}
- caExit("failed to create list of reads to correct", undef) if (! -e "$path/$asm.readsToCorrect");
-
- buildCorrectionLayouts_direct($wrk, $asm) if (getGlobal("corConsensus") eq "utgcns");
- buildCorrectionLayouts_direct($wrk, $asm) if (getGlobal("corConsensus") eq "falcon");
- buildCorrectionLayouts_piped($wrk, $asm) if (getGlobal("corConsensus") eq "falconpipe");
+ buildCorrectionLayouts_direct($asm) if (getGlobal("corConsensus") eq "utgcns");
+ buildCorrectionLayouts_direct($asm) if (getGlobal("corConsensus") eq "falcon");
+ buildCorrectionLayouts_piped($asm) if (getGlobal("corConsensus") eq "falconpipe");
finishStage:
- emitStage($WRK, $asm, "cor-buildCorrectionLayouts");
- buildHTML($WRK, $asm, "cor");
+ emitStage($asm, "cor-buildCorrectionLayouts");
+ buildHTML($asm, "cor");
allDone:
}
@@ -799,21 +983,36 @@ sub buildCorrectionLayouts ($$) {
-sub generateCorrectedReads ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/correction"; # Local work directory
+sub generateCorrectedReads ($) {
my $asm = shift @_;
my $attempt = getGlobal("canuIteration");
my $bin = getBinDirectory();
- my $path = "$wrk/2-correction";
+ my $path = "correction/2-correction";
+
+ goto allDone if (skipStage($asm, "cor-generateCorrectedReads", $attempt) == 1);
+ goto allDone if (sequenceFileExists("$asm.correctedReads"));
+
+ # Compute the size of gkpStore for staging
- goto allDone if (skipStage($WRK, $asm, "cor-generateCorrectedReads", $attempt) == 1);
- goto allDone if (sequenceFileExists("$WRK/$asm.correctedReads"));
+ {
+ my $size = 0;
+
+ $size += -s "correction/$asm.gkpStore/info";
+ $size += -s "correction/$asm.gkpStore/libraries";
+ $size += -s "correction/$asm.gkpStore/reads";
+ $size += -s "correction/$asm.gkpStore/blobs";
+
+ $size = int($size / 1024 / 1024 / 1024 + 1.5);
+
+ setGlobal("corStageSpace", $size);
+ }
# Figure out if all the tasks finished correctly.
- my ($jobs, undef) = computeNumberOfCorrectionJobs($wrk, $asm);
+ fetchFile("$path/correctReads.sh");
+
+ my ($jobs, undef) = computeNumberOfCorrectionJobs($asm);
my $currentJobID = "0001";
my @successJobs;
@@ -821,7 +1020,7 @@ sub generateCorrectedReads ($$) {
my $failureMessage = "";
for (my $job=1; $job <= $jobs; $job++) {
- if (-e "$path/correction_outputs/$currentJobID.fasta") {
+ if (fileExists("$path/correction_outputs/$currentJobID.fasta")) {
push @successJobs, "$path/correction_outputs/$currentJobID.fasta\n";
} else {
@@ -852,12 +1051,10 @@ sub generateCorrectedReads ($$) {
# Otherwise, run some jobs.
- print STDERR "-- generate corrected reads attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
+ emitStage($asm, "cor-generateCorrectedReads", $attempt);
+ buildHTML($asm, "cor");
- emitStage($WRK, $asm, "cor-generateCorrectedReads", $attempt);
- buildHTML($WRK, $asm, "cor");
-
- submitOrRunParallelJob($WRK, $asm, "cor", $path, "correctReads", @failedJobs);
+ submitOrRunParallelJob($asm, "cor", $path, "correctReads", @failedJobs);
return;
}
@@ -868,40 +1065,42 @@ sub generateCorrectedReads ($$) {
print L @successJobs;
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "cor-generateCorrectedReads");
- buildHTML($WRK, $asm, "cor");
- stopAfter("readCorrection");
+ stashFile("$path/corjob.files");
+
+ emitStage($asm, "cor-generateCorrectedReads");
+ buildHTML($asm, "cor");
allDone:
}
-sub dumpCorrectedReads ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/correction"; # Local work directory
+sub dumpCorrectedReads ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
- my $path = "$wrk/2-correction";
+ my $path = "correction/2-correction";
- goto allDone if (skipStage($WRK, $asm, "cor-dumpCorrectedReads") == 1);
- goto allDone if (sequenceFileExists("$WRK/$asm.correctedReads"));
+ goto allDone if (skipStage($asm, "cor-dumpCorrectedReads") == 1);
+ goto allDone if (sequenceFileExists("$asm.correctedReads"));
print STDERR "-- Concatenating correctReads output.\n";
my $files = 0;
my $reads = 0;
- open(F, "< $path/corjob.files") or caExit("can't open '$path/corjob.files' for reading: $!", undef);
- open(N, "< $wrk/$asm.gkpStore/readNames.txt") or caExit("can't open '$wrk/$asm.gkpStore/readNames.txt' for reading: $!", undef);
- open(O, "| gzip -1c > $WRK/$asm.correctedReads.fasta.gz") or caExit("can't open '$WRK/$asm.correctedReads.fasta.gz' for writing: $!", undef);
- #open(O, "> $WRK/$asm.correctedReads.fasta") or caExit("can't open '$WRK/$asm.correctedReads.fasta' for writing: $!", undef);
- open(L, "> $WRK/$asm.correctedReads.length") or caExit("can't open '$WRK/$asm.correctedReads.length' for writing: $!", undef);
+ stashFile("$path/corjob.files");
+
+ open(F, "< $path/corjob.files") or caExit("can't open '$path/corjob.files' for reading: $!", undef);
+ open(N, "< correction/$asm.gkpStore/readNames.txt") or caExit("can't open 'correction/$asm.gkpStore/readNames.txt' for reading: $!", undef);
+ open(O, "| gzip -1c > $asm.correctedReads.fasta.gz") or caExit("can't open '$asm.correctedReads.fasta.gz' for writing: $!", undef);
+ open(L, "> $asm.correctedReads.length") or caExit("can't open '$asm.correctedReads.length' for writing: $!", undef);
while (<F>) {
chomp;
+
+ fetchFile($_);
+
open(R, "< $_") or caExit("can't open correction output '$_' for reading: $!\n", undef);
my $h; # Current header line
@@ -985,6 +1184,9 @@ sub dumpCorrectedReads ($$) {
close(O);
close(F);
+ stashFile("$asm.correctedReads.fasta.gz");
+ stashFile("$asm.correctedReads.length");
+
# Analyze the results.
print STDERR "-- Analyzing correctReads output.\n";
@@ -1012,10 +1214,13 @@ sub dumpCorrectedReads ($$) {
my $minDiff = 0;
my $maxDiff = 0;
- open(R, "< $wrk/2-correction/$asm.readsToCorrect") or caExit("can't open '$wrk/2-correction/$asm.readsToCorrect' for reading: $!", undef);
- open(L, "< $WRK/$asm.correctedReads.length") or caExit("can't open '$WRK./$asm.correctedReads.length' for reading: $!", undef);
+ fetchFile("$path/$asm.readsToCorrect");
+ fetchFile("$asm.correctedReads.length"); # Already here, we just wrote it.
- open(S, "> $wrk/2-correction/$asm.original-expected-corrected-length.dat") or caExit("", undef);
+ open(R, "< $path/$asm.readsToCorrect") or caExit("can't open '$path/$asm.readsToCorrect' for reading: $!", undef);
+ open(L, "< $asm.correctedReads.length") or caExit("can't open '$asm.correctedReads.length' for reading: $!", undef);
+
+ open(S, "> $path/$asm.original-expected-corrected-length.dat") or caExit("", undef);
my $moreData = 1;
@@ -1102,9 +1307,26 @@ sub dumpCorrectedReads ($$) {
close(R);
close(L);
+ stashFile("$path/$asm.original-expected-corrected-length.dat");
+
# Write a summary of the corrections.
- open(F, "> $wrk/2-correction/$asm.correction.summary") or caExit("", undef);
+ my $report = getFromReport("corrections");
+
+ $report .= "-- Actual corrected reads:\n";
+ $report .= "-- $correctedReads reads\n";
+ $report .= "-- $correctedBases bp\n";
+
+ for (my $ii=0; $ii<scalar(@corrPiecesHist); $ii++) {
+ $corrPiecesHist[$ii] += 0;
+ $report .= "-- $corrPiecesHist[$ii] reads with $ii corrected block" . (($ii == 1) ? "" : "s") . "\n";
+ }
+
+ addToReport("corrections", $report);
+
+
+#OBSOLETE
+ open(F, "> $path/$asm.correction.summary") or caExit("", undef);
print F "CORRECTION INPUTS:\n";
print F "-----------------\n";
print F "$inputReads (input reads)\n";
@@ -1125,12 +1347,14 @@ sub dumpCorrectedReads ($$) {
}
close(F);
+ stashFile("$path/$asm.correction.summary");
+
# Scatterplot of lengths.
my $gnuplot = getGlobal("gnuplot");
my $format = getGlobal("gnuplotImageFormat");
- open(F, "> $wrk/2-correction/$asm.originalLength-vs-correctedLength.gp") or caExit("", undef);
+ open(F, "> $path/$asm.originalLength-vs-correctedLength.gp") or caExit("", undef);
print F "\n";
print F "set pointsize 0.25\n";
print F "\n";
@@ -1139,10 +1363,10 @@ sub dumpCorrectedReads ($$) {
print F "set ylabel 'expected corrected read length'\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$wrk/2-correction/$asm.originalLength-vs-expectedLength.lg.$format'\n";
- print F "plot [0:$maxReadLen] [0:$maxReadLen] '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using 2:3 title 'original (x) vs expected (y)'\n";
+ print F "set output './$asm.originalLength-vs-expectedLength.lg.$format'\n";
+ print F "plot [0:$maxReadLen] [0:$maxReadLen] './$asm.original-expected-corrected-length.dat' using 2:3 title 'original (x) vs expected (y)'\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$wrk/2-correction/$asm.originalLength-vs-expectedLength.sm.$format'\n";
+ print F "set output './$asm.originalLength-vs-expectedLength.sm.$format'\n";
print F "replot\n";
print F "\n";
print F "set title 'original read length vs sum of corrected read lengths'\n";
@@ -1150,10 +1374,10 @@ sub dumpCorrectedReads ($$) {
print F "set ylabel 'sum of corrected read lengths'\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$wrk/2-correction/$asm.originalLength-vs-correctedLength.lg.$format'\n";
- print F "plot [0:$maxReadLen] [0:$maxReadLen] '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using 2:4 title 'original (x) vs corrected (y)'\n";
+ print F "set output './$asm.originalLength-vs-correctedLength.lg.$format'\n";
+ print F "plot [0:$maxReadLen] [0:$maxReadLen] './$asm.original-expected-corrected-length.dat' using 2:4 title 'original (x) vs corrected (y)'\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$wrk/2-correction/$asm.originalLength-vs-correctedLength.sm.$format'\n";
+ print F "set output './$asm.originalLength-vs-correctedLength.sm.$format'\n";
print F "replot\n";
print F "\n";
print F "set title 'expected read length vs sum of corrected read lengths'\n";
@@ -1161,25 +1385,33 @@ sub dumpCorrectedReads ($$) {
print F "set ylabel 'sum of corrected read lengths'\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$wrk/2-correction/$asm.expectedLength-vs-correctedLength.lg.$format'\n";
- print F "plot [0:$maxReadLen] [0:$maxReadLen] '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using 3:4 title 'expected (x) vs corrected (y)'\n";
+ print F "set output './$asm.expectedLength-vs-correctedLength.lg.$format'\n";
+ print F "plot [0:$maxReadLen] [0:$maxReadLen] './$asm.original-expected-corrected-length.dat' using 3:4 title 'expected (x) vs corrected (y)'\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$wrk/2-correction/$asm.expectedLength-vs-correctedLength.sm.$format'\n";
+ print F "set output './$asm.expectedLength-vs-correctedLength.sm.$format'\n";
print F "replot\n";
close(F);
- if (runCommandSilently("$wrk/2-correction", "$gnuplot $wrk/2-correction/$asm.originalLength-vs-correctedLength.gp > /dev/null 2>&1", 0)) {
+ if (runCommandSilently($path, "$gnuplot ./$asm.originalLength-vs-correctedLength.gp > /dev/null 2>&1", 0)) {
print STDERR "--\n";
print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n";
print STDERR "--\n";
print STDERR "----------------------------------------\n";
}
+ stashFile("$path/$asm.originalLength-vs-correctedLength.gp");
+ stashFile("$asm.originalLength-vs-expectedLength.lg.$format");
+ stashFile("$asm.originalLength-vs-expectedLength.sm.$format");
+ stashFile("$asm.originalLength-vs-correctedLength.lg.$format");
+ stashFile("$asm.originalLength-vs-correctedLength.sm.$format");
+ stashFile("$asm.expectedLength-vs-correctedLength.lg.$format");
+ stashFile("$asm.expectedLength-vs-correctedLength.sm.$format");
+
# Histograms of lengths, including the difference between expected and actual corrected (the
# other two difference plots weren't interesting; original-expected was basically all zero, and
# so original-actual was nearly the same as expected-actual.
- open(F, "> $wrk/2-correction/$asm.length-histograms.gp") or caExit("", undef);
+ open(F, "> $path/$asm.length-histograms.gp") or caExit("", undef);
print F "set title 'read length'\n";
print F "set ylabel 'number of reads'\n";
print F "set xlabel 'read length, bin width = 250'\n";
@@ -1189,33 +1421,39 @@ sub dumpCorrectedReads ($$) {
print F "bin(x,width) = width*floor(x/width) + binwidth/2.0\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$wrk/2-correction/$asm.length-histograms.lg.$format'\n";
+ print F "set output './$asm.length-histograms.lg.$format'\n";
print F "plot [1:$maxReadLen] [0:] \\\n";
- print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$2,binwidth)):(1.0) smooth freq with boxes title 'original', \\\n";
- print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$3,binwidth)):(1.0) smooth freq with boxes title 'expected', \\\n";
- print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$4,binwidth)):(1.0) smooth freq with boxes title 'corrected'\n";
+ print F " './$asm.original-expected-corrected-length.dat' using (bin(\$2,binwidth)):(1.0) smooth freq with boxes title 'original', \\\n";
+ print F " './$asm.original-expected-corrected-length.dat' using (bin(\$3,binwidth)):(1.0) smooth freq with boxes title 'expected', \\\n";
+ print F " './$asm.original-expected-corrected-length.dat' using (bin(\$4,binwidth)):(1.0) smooth freq with boxes title 'corrected'\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$wrk/2-correction/$asm.length-histograms.sm.$format'\n";
+ print F "set output './$asm.length-histograms.sm.$format'\n";
print F "replot\n";
print F "\n";
print F "set xlabel 'difference between expected and corrected read length, bin width = 250, min=$minDiff, max=$maxDiff'\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$wrk/2-correction/$asm.length-difference-histograms.lg.$format'\n";
+ print F "set output './$asm.length-difference-histograms.lg.$format'\n";
print F "plot [$minDiff:$maxDiff] [0:] \\\n";
- print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$7,binwidth)):(1.0) smooth freq with boxes title 'expected - corrected'\n";
+ print F " './$asm.original-expected-corrected-length.dat' using (bin(\$7,binwidth)):(1.0) smooth freq with boxes title 'expected - corrected'\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$wrk/2-correction/$asm.length-difference-histograms.sm.$format'\n";
+ print F "set output './$asm.length-difference-histograms.sm.$format'\n";
print F "replot\n";
close(F);
- if (runCommandSilently("$wrk/2-correction", "$gnuplot $wrk/2-correction/$asm.length-histograms.gp > /dev/null 2>&1", 0)) {
+ if (runCommandSilently($path, "$gnuplot ./$asm.length-histograms.gp > /dev/null 2>&1", 0)) {
print STDERR "--\n";
print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n";
print STDERR "--\n";
print STDERR "----------------------------------------\n";
}
+ stashFile("$path/$asm.length-histograms.gp");
+ stashFile("$asm.length-histograms.lg.$format");
+ stashFile("$asm.length-histograms.sm.$format");
+ stashFile("$asm.length-difference-histograms.lg.$format");
+ stashFile("$asm.length-difference-histograms.sm.$format");
+
# Now that all outputs are (re)written, cleanup the job outputs.
print STDERR "--\n";
@@ -1272,10 +1510,12 @@ sub dumpCorrectedReads ($$) {
print STDERR "-- Purged $Nlog .out job log outputs.\n" if ($Nlog > 0);
finishStage:
- emitStage($WRK, $asm, "cor-dumpCorrectedReads");
- buildHTML($WRK, $asm, "cor");
+ emitStage($asm, "cor-dumpCorrectedReads");
+ buildHTML($asm, "cor");
allDone:
print STDERR "--\n";
- print STDERR "-- Corrected reads saved in '", sequenceFileExists("$WRK/$asm.correctedReads"), "'.\n";
+ print STDERR "-- Corrected reads saved in '", sequenceFileExists("$asm.correctedReads"), "'.\n";
+
+ stopAfter("readCorrection");
}
diff --git a/src/pipelines/canu/Defaults.pm b/src/pipelines/canu/Defaults.pm
index 0c3d69e..6e91966 100644
--- a/src/pipelines/canu/Defaults.pm
+++ b/src/pipelines/canu/Defaults.pm
@@ -40,18 +40,20 @@ package canu::Defaults;
require Exporter;
@ISA = qw(Exporter);
- at EXPORT = qw(getCommandLineOptions addCommandLineOption addCommandLineError writeLog getNumberOfCPUs getPhysicalMemorySize getAllowedResources diskSpace printOptions printVersion printHelp setParametersFromFile setParametersFromCommandLine checkJava checkGnuplot checkParameters getGlobal setGlobal setGlobalIfUndef showErrorRates setErrorRate setDefaults);
+ at EXPORT = qw(getCommandLineOptions addCommandLineOption addCommandLineError writeLog getNumberOfCPUs getPhysicalMemorySize getAllowedResources diskSpace printOptions printHelp addSequenceFile setParametersFromFile setParametersFromCommandLine checkJava checkGnuplot checkParameters getGlobal setGlobal setGlobalIfUndef setDefaults setVersion);
use strict;
+use Cwd qw(getcwd abs_path);
use Carp qw(cluck);
use Sys::Hostname;
use Text::Wrap;
+use File::Basename; # dirname
my %global; # Parameter value
my %synops; # Parameter description (for -defaults)
my %synnam; # Parameter name (beacuse the key is lowercase)
-my $cLineOpts = "";
+my $cLineOpts = undef;
my $specLog = "";
@@ -95,7 +97,9 @@ sub setGlobal ($$) {
my $set = 0;
$var =~ tr/A-Z/a-z/;
- $val = undef if ($val eq ""); # Set to undefined, the default for many of the options.
+
+ $val = undef if ($val eq "undef"); # Set to undefined, the default for many of the options.
+ $val = undef if ($val eq "");
# Map 'true'/'false' et al. to 0/1.
@@ -114,7 +118,7 @@ sub setGlobal ($$) {
}
}
- foreach my $opt ("overlapper") {
+ foreach my $opt ("overlapper", "realign") {
$set += setGlobalSpecialization($val, ("cor${opt}", "obt${opt}", "utg${opt}")) if ($var eq "${opt}");
}
@@ -128,17 +132,43 @@ sub setGlobal ($$) {
$set += setGlobalSpecialization($val, ("cor${opt}", "obt${opt}", "utg${opt}")) if ($var eq "${opt}");
}
- return if ($set > 0);
+ # Handle the two error rate aliases. Note 'errorRateUsed' must be lowercase. setGlobal/getGlobal do that for us.
if ($var eq "errorrate") {
- setErrorRate($val, 1);
+ $var = "correctederrorrate";
+ $val = 3 * $val;
+
+ $global{"errorrateused"} = "--\n";
+ $global{"errorrateused"} .= "-- WARNING: Obsolete 'errorRate' used, replace with 'correctedErrorRate', set to three times the value.\n";
+ $global{"errorrateused"} .= "-- WARNING: errorRate was the expected error rate in a single corrected read; correctedErrorRate is the\n";
+ $global{"errorrateused"} .= "-- WARNING: allowed difference in an alignment of two corrected reads.\n";
+ }
+
+ if ($var eq "rawerrorrate") {
+ setGlobalIfUndef("corOvlErrorRate", $val);
+ setGlobalIfUndef("corErrorRate", $val);
+ return;
+ }
+
+ if ($var eq "correctederrorrate") {
+ setGlobalIfUndef("obtOvlErrorRate", $val);
+ setGlobalIfUndef("obtErrorRate", $val);
+ setGlobalIfUndef("utgOvlErrorRate", $val);
+ setGlobalIfUndef("utgErrorRate", $val);
+ setGlobalIfUndef("cnsErrorRate", $val);
return;
}
- # If we got a parameter we don't understand, we should be parsing command line options or
+ return if ($set > 0);
+
+ #if ($var eq "canuiteration") {
+ # print STDERR "-- WARNING: set canuIteration to $val\n";
+ #}
+
+ # If we get a parameter we don't understand, we should be parsing command line options or
# reading spec files, and we can let the usual error handling handle it.
- addCommandLineError("ERROR: Paramter '$VAR' is not known.\n") if (!exists($global{$var}));
+ addCommandLineError("ERROR: Parameter '$VAR' is not known.\n") if (!exists($global{$var}));
$global{$var} = $val;
}
@@ -166,11 +196,12 @@ sub getCommandLineOptions () {
sub addCommandLineOption ($) {
- if ($cLineOpts =~ m/\s$/) {
- $cLineOpts .= "$_[0]";
- } else {
- $cLineOpts .= " $_[0]";
- }
+ my $opt = shift @_;
+
+ return if ($opt =~ m/canuIteration=/); # Ignore canu resetting canuIteration
+
+ $cLineOpts .= " " if (defined($cLineOpts) && ($cLineOpts !~ m/\s$/));
+ $cLineOpts .= $opt;
}
@@ -181,14 +212,12 @@ sub addCommandLineError($) {
-sub writeLog ($) {
- my $wrk = shift @_;
-
+sub writeLog () {
my $time = time();
my $host = hostname();
my $pid = $$;
- open(F, "> $wrk/canu-logs/${time}_${host}_${pid}_canu");
+ open(F, "> canu-logs/${time}_${host}_${pid}_canu");
print F $specLog;
close(F);
}
@@ -249,26 +278,11 @@ sub getPhysicalMemorySize () {
-sub dirname ($) {
- my $d = shift @_;
-
- return($d) if (-d $d);
-
- my @d = split '/', $d;
- pop @d;
-
- $d = join('/', @d);
-
- return($d);
-}
-
-
-
sub diskSpace ($) {
- my $wrk = dirname($_[0]);
+ my $dir = dirname($_[0]);
my ($total, $used, $free, $avail) = (0, 0, 0, 0);
- open(DF, "df -P -k $wrk |");
+ open(DF, "df -P -k $dir |");
while (<DF>) {
chomp;
@@ -311,33 +325,18 @@ sub printOptions () {
}
-sub printVersion ($) {
- my $bin = shift @_;
- my $version;
-
- open(F, "$bin/gatekeeperCreate --version 2>&1 |");
- while (<F>) {
- $version = $_; chomp $version;
- }
- close(F);
-
- if (length($version) > 0) {
- print "-- $version\n";
- }
-}
-
-
-sub printHelp () {
+sub printHelp (@) {
+ my $force = shift @_;
- return if (!exists($global{'errors'}));
+ return if (!defined($force) && !defined($global{"errors"}));
print "\n";
- print "usage: canu [-correct | -trim | -assemble | -trim-assemble] \\\n";
+ print "usage: canu [-version] \\\n";
+ print " [-correct | -trim | -assemble | -trim-assemble] \\\n";
print " [-s <assembly-specifications-file>] \\\n";
print " -p <assembly-prefix> \\\n";
print " -d <assembly-directory> \\\n";
print " genomeSize=<number>[g|m|k] \\\n";
- print " errorRate=0.X \\\n";
print " [other-options] \\\n";
print " [-pacbio-raw | -pacbio-corrected | -nanopore-raw | -nanopore-corrected] *fastq\n";
print "\n";
@@ -355,9 +354,6 @@ sub printHelp () {
print " It is used mostly to compute coverage in reads. Fractional values are allowed: '4.7m'\n";
print " is the same as '4700k' and '4700000'\n";
print "\n";
- print " The errorRate is not used correctly (we're working on it). Don't set it\n";
- print " If you want to change the defaults, use the various utg*ErrorRate options.\n";
- print "\n";
print " A full list of options can be printed with '-options'. All options\n";
print " can be supplied in an optional sepc file.\n";
print "\n";
@@ -371,8 +367,11 @@ sub printHelp () {
print "\n";
print "Complete documentation at http://canu.readthedocs.org/en/latest/\n";
print "\n";
- print "$global{'errors'}";
- print "\n";
+
+ if (defined($global{'errors'})) {
+ print "$global{'errors'}";
+ print "\n";
+ }
exit(1);
}
@@ -382,15 +381,16 @@ sub printHelp () {
sub makeAbsolute ($) {
my $var = shift @_;
my $val = getGlobal($var);
- if (defined($val) && ($val !~ m!^/!)) {
- $val = "$ENV{'PWD'}/$val";
- setGlobal($var, $val);
+ my $abs = abs_path($val);
+
+ if (defined($val) && ($val != $abs)) {
+ setGlobal($var, $abs);
$val =~ s/\\\"/\"/g;
$val =~ s/\"/\\\"/g;
$val =~ s/\\\$/\$/g;
$val =~ s/\$/\\\$/g;
- addCommandLineOption("\"$var=$val\"");
+ addCommandLineOption("'$var=$val'");
}
}
@@ -408,8 +408,29 @@ sub fixCase ($) {
-sub setParametersFromFile ($@) {
+sub addSequenceFile ($$@) {
+ my $dir = shift @_;
+ my $file = shift @_;
+ my $err = shift @_;
+
+ return(undef) if (!defined($file)); # No file name? Nothing to do.
+ $file = "$dir/$file" if (defined($dir)); # If $dir defined, assume file is in there.
+ return(abs_path($file)) if (-e $file); # If found, return the full path.
+
+ # And if not found, report an error, unless told not to. This is because on the command
+ # line, the first word after -pacbio-raw must exist, but all the other words could
+ # be files or options.
+
+ addCommandLineError("ERROR: Input read file '$file' not found.\n") if (defined($err));
+
+ return(undef);
+}
+
+
+
+sub setParametersFromFile ($$@) {
my $specFile = shift @_;
+ my $readdir = shift @_;
my @fragFiles = @_;
# Client should be ensuring that the file exists before calling this function.
@@ -435,28 +456,54 @@ sub setParametersFromFile ($@) {
next if (m/^#/);
next if (length($_) eq 0);
- # File handling is also present in canu.pl around line 165.
- if (m/^-(pacbio|nanopore)-(corrected|raw)\s+(.*)$/) {
- my $arg = "-$1-$2";
- my $file = $3;
+ # First, figure out the two words.
- $file = "$ENV{'PWD'}/$file" if ($file !~ m!^/!);
+ my $one;
+ my $two;
+ my $opt;
- push @fragFiles, "$arg\0$file";
- addCommandLineOption("$arg \"$file\"");
+ if (m/^-(pacbio|nanopore)-(corrected|raw)\s+(.*)\s*$/) { # Comments not allowed, because then we can't decide
+ $one = "-$1-$2"; # if the # is a comment, or part of the file!
+ $two = $3; # e.g., this_is_file_#1 vs
+ $opt = 0; # this_is_the_only_file#no more data
}
- elsif (m/\s*(\w*)\s*=([^#]*)#*.*$/) {
- my ($var, $val) = ($1, $2);
- $var =~ s/^\s+//; $var =~ s/\s+$//;
- $val =~ s/^\s+//; $val =~ s/\s+$//;
- undef $val if ($val eq "undef");
- setGlobal($var, $val);
- }
+ elsif (m/^(\w*)\s*=\s*([^#]*)\s*#*.*?$/) { # Word two won't match a #, but will gobble up spaces at the end.
+ $one = $1; # Then, we can match a #, and any amount of comment, minimally.
+ $two = $2; # If word two is made non-greedy, it will shrink to nothing, as
+ $opt = 1; # the last bit will gobble up everything, since we're allowed
+ } # to match zero #'s in between.
else {
addCommandLineError("ERROR: File not found or unknown specFile option line '$_'.\n");
}
+
+ # Now, clean up the second word to handle quotes.
+
+ $two =~ s/^\s+//; # There can be spaces from the greedy match.
+ $two =~ s/\s+$//;
+
+ $two = $1 if ($two =~ m/^'(.+)'$/); # Remove single quotes | But don't allowed mixed quotes; users
+ $two = $1 if ($two =~ m/^"(.+)"$/); # Remove double quotes | should certainly know better
+
+ # And do something.
+
+ if ($opt == 1) {
+ $two =~ s/^\s+//; # Remove spaces again. They'll just confuse our option processing.
+ $two =~ s/\s+$//;
+
+ setGlobal($one, $two);
+ }
+
+ else {
+ my $file = addSequenceFile($readdir, $two, 1); # Don't remove spaces. File could be " file ", for some stupid reason.
+
+ if (defined($file)) {
+ push @fragFiles, "$one\0$file";
+ } else {
+ addCommandLineError("ERROR: File not found in spec file option '$_'\n");
+ }
+ }
}
close(F);
@@ -505,62 +552,15 @@ sub setExecDefaults ($$) {
$global{"${tag}Threads"} = undef;
$synops{"${tag}Threads"} = "Number of threads to use for $name jobs";
+ $global{"${tag}StageSpace"} = undef;
+ $synops{"${tag}StageSpace"} = "Amount of local disk space needed to stage data for $name jobs";
+
$global{"${tag}Concurrency"} = undef;
$synops{"${tag}Concurrency"} = "If grid not enabled, number of $name jobs to run at the same time; default is n_proc / n_threads";
}
-sub showErrorRates ($) {
- my $prefix = shift @_;
-
- print STDERR "${prefix}\n";
- print STDERR "${prefix}genomeSize -- ", getGlobal("genomeSize"), "\n";
- print STDERR "${prefix}errorRate -- ", getGlobal("errorRate"), "\n";
- print STDERR "${prefix}\n";
- print STDERR "${prefix}corOvlErrorRate -- ", getGlobal("corOvlErrorRate"), "\n";
- print STDERR "${prefix}obtOvlErrorRate -- ", getGlobal("obtOvlErrorRate"), "\n";
- print STDERR "${prefix}utgOvlErrorRate -- ", getGlobal("utgOvlErrorRate"), "\n";
- print STDERR "${prefix}\n";
- print STDERR "${prefix}obtErrorRate -- ", getGlobal("obtErrorRate"), "\n";
- print STDERR "${prefix}\n";
- #print STDERR "${prefix}corErrorRate -- ", getGlobal("corErrorRate"), "\n";
- print STDERR "${prefix}cnsErrorRate -- ", getGlobal("cnsErrorRate"), "\n";
-}
-
-
-# Defaults are set for yeast:
-# trimming errorRate = 0.009 obtOvlErrorRate = 0.06 obtErrorRate = 0.035
-# assembly errorRate = 0.009 utgOvlErrorRate = 0.06 bogart 0.035
-#
-sub setErrorRate ($$) {
- my $er = shift @_;
- my $force = shift @_;
-
- if (($force == 0) && (defined($global{"errorrate"}))) {
- #print STDERR "-- Can't change error rate from ", getGlobal('errorRate'), " to $er - not allowed.\n";
- return;
- }
-
- #print STDERR "-- Set errorRate to $er\n";
-
- # Can NOT call setGlobal() for this, because it calls setErrorRate()!.
- $global{"errorrate"} = $er;
- setGlobal("corOvlErrorRate", $er * 3); # Not used, except for realigning
- setGlobal("obtOvlErrorRate", $er * 3); # Generally must be smaller than utgGraphErrorRate
- setGlobal("utgOvlErrorRate", $er * 3);
-
- setGlobal("obtErrorRate", $er * 3);
-
- # Removed, is usually set in CorrectReads, can be set from command line directly.
- #setGlobal("corErrorRate", $er * 10); # Erorr rate used for raw sequence alignment/consensus
- setGlobal("cnsErrorRate", $er * 3);
-
- #showErrorRates("-- ");
-}
-
-
-
sub setOverlapDefaults ($$$) {
my $tag = shift @_; # If 'cor', some parameters are loosened for raw pacbio reads
my $name = shift @_;
@@ -611,7 +611,7 @@ sub setOverlapDefaults ($$$) {
$global{"${tag}MhapVersion"} = "2.1.2";
$synops{"${tag}MhapVersion"} = "Version of the MHAP jar file to use";
- $global{"${tag}MhapFilterThreshold"} = "0.000005";
+ $global{"${tag}MhapFilterThreshold"} = "0.000005"; # Needs to be a string, else it is printed as 5e-06.
$synops{"${tag}MhapFilterThreshold"} = "Value between 0 and 1. kmers which comprise more than this percentage of the input are downweighted";
$global{"${tag}MhapFilterUnique"} = undef;
@@ -633,7 +633,7 @@ sub setOverlapDefaults ($$$) {
$synops{"${tag}MhapOrderedMerSize"} = "K-mer size for second-stage filter in mhap";
$global{"${tag}MhapSensitivity"} = undef;
- $synops{"${tag}MhapSensitivity"} = "Coarse sensitivity level: 'low', 'normal' or 'high'. Usually set automatically based on coverage; 'high' <= 30x < 'normal' < 60x <= 'low'";
+ $synops{"${tag}MhapSensitivity"} = "Coarse sensitivity level: 'low', 'normal' or 'high'. Set automatically based on coverage; 'high' <= 30x < 'normal' < 60x <= 'low'";
$global{"${tag}MMapBlockSize"} = 6000;
$synops{"${tag}MMapBlockSize"} = "Number of reads per 1GB; memory * blockSize = the size of block loaded into memory per job";
@@ -643,24 +643,31 @@ sub setOverlapDefaults ($$$) {
$synops{"${tag}MMapMerSize"} = "K-mer size for seeds in minmap";
# shared parameters for alignment-free overlappers
- $global{"${tag}ReAlign"} = undef;
- $synops{"${tag}ReAlign"} = "Compute actual alignments from overlaps; 'raw' from output, 'final' from overlap store; uses either obtErrorRate or ovlErrorRate, depending on which overlaps are computed";
+ $global{"${tag}ReAlign"} = 0;
+ $synops{"${tag}ReAlign"} = "Refine mhap/minimap overlaps by computing the actual alignment: 'true' or 'false'. Uses ${tag}OvlErrorRate";
}
sub setDefaults () {
+ ##### Internal stuff
+
+ $global{"errors"} = undef; # Command line errors
+ $global{"errorRateUsed"} = undef; # A warning if obsolete 'errorRate' parameter is used. This lets us print the error in a useful place, instead of at the very start of the output.
+
+ $global{"version"} = undef; # Reset at the end of this function, once we know where binaries are.
+
##### General Configuration Options (aka miscellany)
- $global{"canuIteration"} = 1; # See documentation in Execution.pm
+ $global{"canuIteration"} = 0; # See documentation in Execution.pm
$global{"canuIterationMax"} = 2;
$global{"showNext"} = undef;
$synops{"showNext"} = "Don't run any commands, just report what would run";
$global{"pathMap"} = undef;
- $synops{"pathMap"} = "File with a hostname to binary directory map";
+ $synops{"pathMap"} = "File with a hostname to binary directory map; binary directories must be absolute paths";
$global{"shell"} = "/bin/sh";
$synops{"shell"} = "Command interpreter to use; sh-compatible (e.g., bash), NOT C-shell (csh or tcsh); default '/bin/sh'";
@@ -677,6 +684,9 @@ sub setDefaults () {
$global{"gnuplotTested"} = 0;
$synops{"gnuplotTested"} = "If set, skip the initial testing of gnuplot";
+ $global{"stageDirectory"} = undef;
+ $synops{"stageDirectory"} = "If set, copy heavily used data to this node-local location";
+
##### Cleanup and Termination options
$global{"saveOverlaps"} = 0;
@@ -699,9 +709,6 @@ sub setDefaults () {
##### Error Rates
- $global{"errorRate"} = undef;
- $synops{"errorRate"} = "The expected error rate in the corrected reads, typically set based on sequencing type. Set to 0 to try to estimate dynamically. (EXPERIMENTAL)";
-
$global{"corOvlErrorRate"} = undef;
$synops{"corOvlErrorRate"} = "Overlaps above this error rate are not computed";
@@ -711,17 +718,17 @@ sub setDefaults () {
$global{"utgOvlErrorRate"} = undef;
$synops{"utgOvlErrorRate"} = "Overlaps at or below this error rate are used to trim reads";
- #$global{"utgErrorRate"} = undef;
- #$synops{"utgErrorRate"} = "Overlaps at or below this error rate are used to construct unitigs (BOG and UTG)";
+ $global{"utgErrorRate"} = undef;
+ $synops{"utgErrorRate"} = "Overlaps at or below this error rate are used to construct contigs";
$global{"utgGraphDeviation"} = 6;
- $synops{"utgGraphDeviation"} = "Overlaps this much above median will not be used for initial graph construction (BOGART)";
+ $synops{"utgGraphDeviation"} = "Overlaps this much above median will not be used for initial graph construction";
$global{"utgRepeatDeviation"} = 3;
- $synops{"utgRepeatDeviation"} = "Overlaps this much above mean unitig error rate will not be used for repeat splitting (BOGART)";
+ $synops{"utgRepeatDeviation"} = "Overlaps this much above mean unitig error rate will not be used for repeat splitting";
$global{"utgRepeatConfusedBP"} = 2100;
- $synops{"utgRepeatConfusedBP"} = "Repeats where the next best edge is at least this many bp shorter will not be split (BOGART)";
+ $synops{"utgRepeatConfusedBP"} = "Repeats where the next best edge is at least this many bp shorter will not be split";
$global{"corErrorRate"} = undef;
$synops{"corErrorRate"} = "Only use raw alignments below this error rate to construct corrected reads";
@@ -752,9 +759,6 @@ sub setDefaults () {
$global{"stopOnReadQuality"} = 1;
$synops{"stopOnReadQuality"} = "Stop if a significant portion of the input data is too short or has quality value or base composition errors";
- $global{"stopBefore"} = undef;
- $synops{"stopBefore"} = "Tell canu when to halt execution";
-
$global{"stopAfter"} = undef;
$synops{"stopAfter"} = "Tell canu when to halt execution";
@@ -764,9 +768,6 @@ sub setDefaults () {
$global{"gridEngine"} = undef;
$global{"gridEngineSubmitCommand"} = undef;
- $global{"gridEngineHoldOption"} = undef;
- $global{"gridEngineHoldOptionNoArray"} = undef;
- $global{"gridEngineSyncOption"} = undef;
$global{"gridEngineNameOption"} = undef;
$global{"gridEngineArrayOption"} = undef;
$global{"gridEngineArrayName"} = undef;
@@ -778,6 +779,7 @@ sub setDefaults () {
$global{"gridEngineMemoryUnits"} = undef;
$global{"gridEngineNameToJobIDCommand"} = undef;
$global{"gridEngineNameToJobIDCommandNoArray"} = undef;
+ $global{"gridEngineStageOption"} = undef;
$global{"gridEngineTaskID"} = undef;
$global{"gridEngineArraySubmitID"} = undef;
$global{"gridEngineJobID"} = undef;
@@ -787,7 +789,7 @@ sub setDefaults () {
$global{"useGrid"} = 1;
$synops{"useGrid"} = "If 'true', enable grid-based execution; if 'false', run all jobs on the local machine; if 'remote', create jobs for grid execution but do not submit; default 'true'";
- foreach my $c (qw(BAT CNS COR MERYL CORMHAP CORMMAP COROVL OBTMHAP OBTOVL OEA OVB OVS RED UTGMHAP UTGMMAP UTGOVL)) {
+ foreach my $c (qw(BAT GFA CNS COR MERYL CORMHAP CORMMAP COROVL OBTMHAP OBTMMAP OBTOVL OEA OVB OVS RED UTGMHAP UTGMMAP UTGOVL)) {
$global{"useGrid$c"} = 1;
$synops{"useGrid$c"} = "If 'true', run module $c under grid control; if 'false' run locally.";
}
@@ -829,6 +831,18 @@ sub setDefaults () {
setExecDefaults("bat", "unitig construction");
setExecDefaults("cns", "unitig consensus");
+ setExecDefaults("gfa", "graph alignment and processing");
+
+ ##### Object Storage
+
+ $global{"objectStore"} = undef;
+ $synops{"objectStore"} = "Type of object storage used; not ready for production yet";
+
+ $global{"objectStoreClient"} = undef;
+ $synops{"objectStoreClient"} = "Path to the command line client used to access the object storage";
+
+ $global{"objectStoreNameSpace"} = undef;
+ $synops{"objectStoreNameSpace"} = "Object store parameters; specific to the type of objectStore used";
##### Overlapper
@@ -954,9 +968,6 @@ sub setDefaults () {
$global{"corConsensus"} = "falconpipe";
$synops{"corConsensus"} = "Which consensus algorithm to use; only 'falcon' and 'falconpipe' are supported; default 'falconpipe'";
- $global{"falconSense"} = undef;
- $synops{"falconSense"} = "Path to fc_consensus.py or falcon_sense.bin";
-
$global{"corLegacyFilter"} = undef;
$synops{"corLegacyFilter"} = "Expert option: global filter, length * identity (default) or length with broken by identity (if on)";
@@ -987,6 +998,21 @@ sub setDefaults () {
}
+# Get the version information. Needs to be last so that pathMap can be defined.
+
+sub setVersion ($) {
+ my $bin = shift @_;
+ my $version;
+
+ open(F, "$bin/gatekeeperCreate --version 2>&1 |");
+ while (<F>) {
+ $version = $_; chomp $version;
+ }
+ close(F);
+
+ $global{'version'} = $version;
+}
+
sub checkJava () {
return if ((getGlobal("corOverlapper") ne "mhap") &&
@@ -1156,8 +1182,6 @@ sub checkParameters () {
# Fiddle with filenames to make them absolute paths.
#
- makeAbsolute("pathMap");
-
makeAbsolute("corOvlFrequentMers");
makeAbsolute("obtOvlFrequentMers");
makeAbsolute("utgOvlFrequentMers");
@@ -1176,10 +1200,15 @@ sub checkParameters () {
fixCase("corFilter");
fixCase("unitigger");
- fixCase("stopBefore");
fixCase("stopAfter");
#
+ # Well, crud. 'gridEngine' wants to be uppercase, not lowercase like fixCase() would do.
+ #
+
+ $global{"gridengine"} =~ tr/a-z/A-Z/; # NOTE: lowercase 'gridengine'
+
+ #
# Check for inconsistent parameters
#
@@ -1189,6 +1218,10 @@ sub checkParameters () {
{
my $gs = getGlobal("genomeSize");
+ if (!defined($gs)) {
+ addCommandLineError("ERROR: Required parameter 'genomeSize' not set.\n");
+ }
+
if (($gs =~ m/^(\d+)$/) ||
($gs =~ m/^(\d+\.\d+)$/)) {
if ($gs < 1000) {
@@ -1197,7 +1230,7 @@ sub checkParameters () {
}
}
- foreach my $var ("corOvlErrorRate", "obtOvlErrorRate", "utgOvlErrorRate", "corErrorRate", "cnsErrorRate", "obtErrorRate") {
+ foreach my $var ("corOvlErrorRate", "obtOvlErrorRate", "utgOvlErrorRate", "corErrorRate", "obtErrorRate", "utgErrorRate", "cnsErrorRate") {
if (!defined(getGlobal($var))) {
addCommandLineError("ERROR: Invalid '$var' specified; must be set\n");
}
@@ -1322,34 +1355,6 @@ sub checkParameters () {
addCommandLineError("ERROR: Invalid 'useGrid' specified (" . getGlobal("useGrid") . "); must be 'true', 'false' or 'remote'\n");
}
- if (defined(getGlobal("stopBefore"))) {
- my $ok = 0;
- my $st = getGlobal("stopBefore");
- $st =~ tr/A-Z/a-z/;
-
- my $failureString = "ERROR: Invalid stopBefore specified (" . getGlobal("stopBefore") . "); must be one of:\n";
-
- my @stopBefore = ("gatekeeper",
- "meryl",
- "trimReads",
- "splitReads",
- "red", "oea",
- "unitig",
- "consensusConfigure",
- "cns");
-
- foreach my $sb (@stopBefore) {
- $failureString .= "ERROR: '$sb'\n";
- $sb =~ tr/A-Z/a-z/;
- if ($st eq $sb) {
- $ok++;
- setGlobal('stopBefore', $st);
- }
- }
-
- addCommandLineError($failureString) if ($ok == 0);
- }
-
if (defined(getGlobal("stopAfter"))) {
my $ok = 0;
my $st = getGlobal("stopAfter");
@@ -1359,14 +1364,17 @@ sub checkParameters () {
my @stopAfter = ("gatekeeper",
"meryl",
- "mhapConfigure",
+ "overlapConfigure",
+ "overlap",
"overlapStoreConfigure",
"overlapStore",
+ "readCorrection",
+ "readTrimming",
"unitig",
"consensusConfigure",
"consensusCheck",
"consensusLoad",
- "consensusFilter");
+ "consensusAnalyze");
foreach my $sa (@stopAfter) {
$failureString .= "ERROR: '$sa'\n";
@@ -1380,9 +1388,6 @@ sub checkParameters () {
addCommandLineError($failureString) if ($ok == 0);
}
- addCommandLineError("ERROR: Required parameter 'errorRate' is not set\n") if (! defined(getGlobal("errorRate")));
- addCommandLineError("ERROR: Required parameter 'genomeSize' is not set\n") if (! defined(getGlobal("genomeSize")));
-
#
# Minimap, no valid identities, set legacy
#
@@ -1390,30 +1395,6 @@ sub checkParameters () {
if (getGlobal("corOverlapper") eq "minimap") {
setGlobalIfUndef("corLegacyFilter", 1);
}
-
- #
- # Falcon? Need to find it.
- #
-
- if ((getGlobal("corConsensus") eq "falcon") ||
- (getGlobal("corConsensus") eq "falconpipe")) {
- my $falcon = getGlobal("falconSense");
-
- addCommandLineError("ERROR: Didn't find falcon program with option falconSense='$falcon'") if ((defined($falcon)) && (! -e $falcon));
- }
-
- #
- # Set default error rates based on the per-read error rate.
- #
-
- setGlobalIfUndef("corOvlErrorRate", 3.0 * getGlobal("errorRate"));
- setGlobalIfUndef("obtOvlErrorRate", 3.0 * getGlobal("errorRate"));
- setGlobalIfUndef("utgOvlErrorRate", 3.0 * getGlobal("errorRate"));
-
- setGlobalIfUndef("ovlErrorRate", 2.5 * getGlobal("errorRate"));
-
- setGlobalIfUndef("corsErrorRate", 10.0 * getGlobal("errorRate"));
- setGlobalIfUndef("cnsErrorRate", 2.5 * getGlobal("errorRate"));
}
diff --git a/src/pipelines/canu/ErrorEstimate.pm b/src/pipelines/canu/ErrorEstimate.pm
index 8809838..ab76fd0 100644
--- a/src/pipelines/canu/ErrorEstimate.pm
+++ b/src/pipelines/canu/ErrorEstimate.pm
@@ -19,6 +19,10 @@
# are a 'United States Government Work', and
# are released in the public domain
#
+ # Brian P. Walenz beginning on 2016-DEC-12
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
# File 'README.licenses' in the root directory of this distribution contains
# full conditions and disclaimers for each license.
##
@@ -33,13 +37,19 @@ require Exporter;
use strict;
use POSIX qw(floor);
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
use canu::HTML;
+#
+# NOTE!!! This was made to compile ONLY. All the paths will be wrong. runCommand() is expecting
+# commands to be run in the directory they operate on, but this was written to run with absolute
+# paths.
+#
+
sub fac($) {
my $x = shift @_;
@@ -56,15 +66,15 @@ sub poisson_pdf ($$) {
}
sub uniqueKmerThreshold($$$$) {
- my $wrk = shift @_;
+ my $path = shift @_;
my $asm = shift @_;
my $merSize = shift @_;
my $loss = shift @_;
my $bin = getBinDirectory();
- my $errorRate = estimateRawError($wrk, $asm, "cor", $merSize);
+ my $errorRate = estimateRawError($path, $asm, "cor", $merSize);
- my $readLength = getNumberOfBasesInStore($wrk, $asm) / getNumberOfReadsInStore ($wrk, $asm);
- my $effective_coverage = getExpectedCoverage($wrk, $asm) * ( ($readLength - $merSize + 1)/$readLength ) * (1 - $errorRate) ** $merSize;
+ my $readLength = getNumberOfBasesInStore($path, $asm) / getNumberOfReadsInStore ($path, $asm);
+ my $effective_coverage = getExpectedCoverage($path, $asm) * ( ($readLength - $merSize + 1)/$readLength ) * (1 - $errorRate) ** $merSize;
my $threshold = 0;
my $kMer_loss = poisson_pdf($effective_coverage, 0);
@@ -88,7 +98,7 @@ sub uniqueKmerThreshold($$$$) {
}
sub computeSampleSize($$$$$) {
- my $wrk = shift @_;
+ my $path = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $percent = shift @_;
@@ -99,10 +109,10 @@ sub computeSampleSize($$$$$) {
my $maxSampleSize = getGlobal("${tag}MhapBlockSize") * 4;
if (defined($percent)) {
- $sampleSize = int($percent * getNumberOfReadsInStore ($wrk, $asm))+1;
+ $sampleSize = int($percent * getNumberOfReadsInStore ($path, $asm))+1;
$sampleSize++ if ($sampleSize % 2 != 0);
} elsif (defined($coverage)) {
- $sampleSize = int(($coverage * getGlobal("genomeSize")) / (getNumberOfBasesInStore($wrk, $asm) / getNumberOfReadsInStore ($wrk, $asm))) + 1;
+ $sampleSize = int(($coverage * getGlobal("genomeSize")) / (getNumberOfBasesInStore($path, $asm) / getNumberOfReadsInStore ($path, $asm))) + 1;
}
$sampleSize = $maxSampleSize if (defined($percent) && $sampleSize > $maxSampleSize);
@@ -110,7 +120,7 @@ sub computeSampleSize($$$$$) {
}
sub runMHAP($$$$$$$$$$$$) {
- my ($wrk, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize, $hash, $query, $out, $err) = @_;
+ my ($path, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize, $hash, $query, $out, $err) = @_;
my $filterThreshold = getGlobal("${tag}MhapFilterThreshold");
my $merSize = getGlobal("${tag}MhapMerSize");
@@ -124,21 +134,21 @@ sub runMHAP($$$$$$$$$$$$) {
my $cmd = "$javaPath -d64 -server -Xmx4g -jar $bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar ";
$cmd .= " --no-self --repeat-weight 0.9 -k $merSize --num-hashes $numHashes --num-min-matches $minNumMatches --ordered-sketch-size $ordSketch --ordered-kmer-size $ordSketchMer --threshold $threshold --filter-threshold $filterThreshold --num-threads " . getGlobal("${tag}mhapThreads");
$cmd .= " -s $hash -q $query 2> /dev/null | awk '{if (\$1 != \$2+$sampleSize) { print \$0}}' | $bin/errorEstimate -d 2 -m 0.95 -S - > $out 2> $err";
- runCommand($wrk, $cmd);
+ runCommand($path, $cmd);
}
sub estimateRawError($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+ my $base = "correction";
+ my $path = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $merSize = shift @_;
my $bin = getBinDirectory();
- my $numReads = getNumberOfReadsInStore ($wrk, $asm);
+ my $numReads = getNumberOfReadsInStore ($path, $asm);
- goto allDone if (skipStage($WRK, $asm, "errorEstimate") == 1);
- goto allDone if (-e "$wrk/asm.gkpStore/raw.estimate.out");
+ goto allDone if (skipStage($asm, "errorEstimate") == 1);
+ goto allDone if (-e "$path/asm.gkpStore/raw.estimate.out");
goto allDone if (getGlobal("errorrate") > 0);
my ($numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer);
@@ -150,28 +160,28 @@ sub estimateRawError($$$$) {
$ordSketchMer = getGlobal("${tag}MhapOrderedMerSize");
# subsample raw reads
- my $sampleSize = computeSampleSize($wrk, $asm, $tag, 0.01, undef);
+ my $sampleSize = computeSampleSize($path, $asm, $tag, 0.01, undef);
$sampleSize /= 2;
- my $cmd = "$bin/gatekeeperDumpFASTQ -G $wrk/$asm.gkpStore -nolibname -fasta -r 1-$sampleSize -o - > $wrk/$asm.gkpStore/subset.fasta 2> /dev/null";
- runCommandSilently($wrk, $cmd, 1);
+ my $cmd = "$bin/gatekeeperDumpFASTQ -G $base/$asm.gkpStore -nolibname -fasta -r 1-$sampleSize -o - > $base/$asm.gkpStore/subset.fasta 2> /dev/null";
+ runCommandSilently($path, $cmd, 1);
my $min = $numReads - $sampleSize + 1;
- my $cmd = "$bin/gatekeeperDumpFASTQ -G $wrk/$asm.gkpStore -nolibname -fasta -r $min-$numReads -o - >> $wrk/$asm.gkpStore/subset.fasta 2> /dev/null";
- runCommandSilently($wrk, $cmd, 1);
- my $querySize = computeSampleSize($wrk, $asm, $tag, undef, 2);
- my $cmd = "$bin/gatekeeperDumpFASTQ -G $wrk/$asm.gkpStore -nolibname -fasta -r 1-$querySize -o - > $wrk/$asm.gkpStore/reads.fasta 2> /dev/null";
- runCommandSilently($wrk, $cmd, 1);
+ my $cmd = "$bin/gatekeeperDumpFASTQ -G $base/$asm.gkpStore -nolibname -fasta -r $min-$numReads -o - >> $base/$asm.gkpStore/subset.fasta 2> /dev/null";
+ runCommandSilently($path, $cmd, 1);
+ my $querySize = computeSampleSize($path, $asm, $tag, undef, 2);
+ my $cmd = "$bin/gatekeeperDumpFASTQ -G $base/$asm.gkpStore -nolibname -fasta -r 1-$querySize -o - > $base/$asm.gkpStore/reads.fasta 2> /dev/null";
+ runCommandSilently($path, $cmd, 1);
print STDERR "--\n";
print STDERR "-- ESTIMATOR (mhap) (raw) (hash sample size=". ($sampleSize*2) . ") (query sample size=$querySize)\n";
- runMHAP($wrk, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize*2, "$wrk/$asm.gkpStore/subset.fasta", "$wrk/$asm.gkpStore/reads.fasta", "$wrk/$asm.gkpStore/raw.estimate.out", "$wrk/$asm.gkpStore/raw.estimate.err");
- unlink("$wrk/$asm.gkpStore/subset.fasta");
- unlink("$wrk/$asm.gkpStore/reads.fasta");
+ runMHAP($path, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize*2, "$base/$asm.gkpStore/subset.fasta", "$base/$asm.gkpStore/reads.fasta", "$base/$asm.gkpStore/raw.estimate.out", "$base/$asm.gkpStore/raw.estimate.err");
+ unlink("$base/$asm.gkpStore/subset.fasta");
+ unlink("$base/$asm.gkpStore/reads.fasta");
allDone:
- return 0.15 if (! -e "$wrk/$asm.gkpStore/raw.estimate.out");
+ return 0.15 if (! -e "$base/$asm.gkpStore/raw.estimate.out");
my $errorRate = 0;
- open(L, "< $wrk/$asm.gkpStore/raw.estimate.out") or caExit("can't open '$wrk/$asm.gkpStore/raw.estimate.out' for reading: $!", undef);
+ open(L, "< $base/$asm.gkpStore/raw.estimate.out") or caExit("can't open '$base/$asm.gkpStore/raw.estimate.out' for reading: $!", undef);
while (<L>) {
$errorRate = sprintf "%.3f", ($_ / 2);
$errorRate = 0.15 if ($errorRate <= 0.005);
@@ -184,20 +194,24 @@ sub estimateRawError($$$$) {
# Map subset of reads to long reads with mhap.
# Compute resulting distribution and estimate error rate
-sub estimateCorrectedError ($$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub estimateCorrectedError ($$) {
my $asm = shift @_;
my $tag = shift @_;
my $bin = getBinDirectory();
- $wrk = "$wrk/correction";
+ # DISABLED 2016 JAN 27 when 'errorRate' was replaced with 'correctedErrorRate'.
+ #
+ # A new option needs to be added to enable this explicitly.
+ # This doesn't work on grid either (the set error rates are lost on the next restart).
+
+ return;
- my $path = "$wrk/3-estimator";
+ my $base = "correction";
+ my $path = "correction/3-estimator";
# only run if we aren't done and were asked to
- goto allDone if (skipStage($WRK, $asm, "errorEstimate") == 1);
- goto allDone if (-e "$path/$asm.estimate.out");
+ goto allDone if (skipStage($asm, "errorEstimate") == 1);
+ goto allDone if (-e "$base/$asm.estimate.out");
goto allDone if (getGlobal("errorrate") > 0);
# Mhap parameters - filterThreshold needs to be a string, else it is printed as 5e-06.
@@ -214,29 +228,29 @@ sub estimateCorrectedError ($$$) {
make_path("$path");
# subsample corrected reads, this assumes the fasta records are on a single line. We take some reads from the top and bottom of file to avoid sampling one library
- my $sampleSize = computeSampleSize($wrk, $asm, $tag, 0.01, undef);
- my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |head -n $sampleSize > $path/subset.fasta";
+ my $sampleSize = computeSampleSize("correction", $asm, $tag, 0.01, undef);
+ my $cmd = "gunzip -c correction/asm.correctedReads.fasta.gz |head -n $sampleSize > $path/subset.fasta";
runCommandSilently($path, $cmd, 1);
- my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |tail -n $sampleSize >> $path/subset.fasta";
+ my $cmd = "gunzip -c correction/asm.correctedReads.fasta.gz |tail -n $sampleSize >> $path/subset.fasta";
runCommandSilently($path, $cmd, 1);
- my $querySize = computeSampleSize($wrk, $asm, $tag, undef, 2);
- my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |head -n $querySize > $path/reads.fasta";
+ my $querySize = computeSampleSize("correction", $asm, $tag, undef, 2);
+ my $cmd = "gunzip -c correction/asm.correctedReads.fasta.gz |head -n $querySize > $path/reads.fasta";
runCommandSilently($path, $cmd, 1);
- my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |tail -n $querySize >> $path/reads.fasta";
+ my $cmd = "gunzip -c correction/asm.correctedReads.fasta.gz |tail -n $querySize >> $path/reads.fasta";
runCommandSilently($path, $cmd, 1);
# now compute the overlaps
print STDERR "--\n";
print STDERR "-- ESTIMATOR (mhap) (corrected) (hash sample size=$sampleSize) (query sample size=$querySize)\n";
- runMHAP($wrk, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize, "$path/subset.fasta", "$path/reads.fasta", "$path/$asm.estimate.out", "$path/$asm.estimate.err");
+ runMHAP("correction", $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize, "$path/subset.fasta", "$path/reads.fasta", "$base/$asm.estimate.out", "$base/$asm.estimate.err");
unlink("$path/subset.fasta");
unlink("$path/reads.fasta");
allDone:
- return if (! -e "$path/$asm.estimate.out");
+ return if (! -e "$base/$asm.estimate.out");
my $errorRate = 0;
- open(L, "< $path/$asm.estimate.out") or caExit("can't open '$path/$asm.estimate.out' for reading: $!", undef);
+ open(L, "< $base/$asm.estimate.out") or caExit("can't open '$base/$asm.estimate.out' for reading: $!", undef);
while (<L>) {
$errorRate = sprintf "%.3f", ($_ / 2);
}
diff --git a/src/pipelines/canu/Execution.pm b/src/pipelines/canu/Execution.pm
index 3da3977..be1fe77 100644
--- a/src/pipelines/canu/Execution.pm
+++ b/src/pipelines/canu/Execution.pm
@@ -55,19 +55,39 @@ package canu::Execution;
require Exporter;
@ISA = qw(Exporter);
- at EXPORT = qw(stopBefore stopAfter skipStage emitStage touch getInstallDirectory getJobIDShellCode getLimitShellCode getBinDirectory getBinDirectoryShellCode submitScript submitOrRunParallelJob runCommand runCommandSilently findCommand findExecutable caExit caFailure);
+ at EXPORT = qw(stopAfter
+ skipStage
+ emitStage
+ touch
+ getInstallDirectory
+ getJobIDShellCode
+ getLimitShellCode
+ getBinDirectory
+ getBinDirectoryShellCode
+ setWorkDirectory
+ setWorkDirectoryShellCode
+ submitScript
+ submitOrRunParallelJob
+ runCommand
+ runCommandSilently
+ findCommand
+ findExecutable
+ caExit
+ caFailure);
use strict;
use Config; # for @signame
use Cwd qw(getcwd);
-use Carp qw(cluck);
+use Carp qw(longmess);
use POSIX ":sys_wait_h"; # For waitpid(..., &WNOHANG)
use List::Util qw(min max);
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use File::Spec;
use canu::Defaults;
+use canu::Report qw(generateReport);
+
#
@@ -162,6 +182,10 @@ sub schedulerFinish ($) {
print STDERR "-- Starting concurrent execution on ", scalar(localtime()), " with $diskfree GB free disk space ($remain processes; $numberOfProcesses concurrently)\n" if (defined($dir));
print STDERR "-- Starting concurrent execution on ", scalar(localtime()), " ($remain processes; $numberOfProcesses concurrently)\n" if (!defined($dir));
print STDERR "\n";
+ print STDERR " cd $dir\n";
+
+ my $cwd = getcwd(); # Remember where we are.
+ chdir($dir); # So we can root the jobs in the correct location.
# Run all submitted jobs
#
@@ -188,6 +212,8 @@ sub schedulerFinish ($) {
waitpid(shift @processesRunning, 0);
}
+ chdir($cwd);
+
$diskfree = (defined($dir)) ? (diskSpace($dir)) : (0);
my $warning = " !!! WARNING !!!" if ($diskfree < 10);
@@ -220,23 +246,6 @@ sub touch ($@) {
# State management
#
-sub stopBefore ($$) {
- my $stopBefore = shift @_;
- my $cmd = shift @_;
-
- $stopBefore =~ tr/A-Z/a-z/;
-
- if ((defined($stopBefore)) &&
- (defined(getGlobal("stopBefore"))) &&
- (getGlobal("stopBefore") eq $stopBefore)) {
- print STDERR "\n";
- print STDERR "Stop requested before '$stopBefore'.\n";
- print STDERR "\n";
- print STDERR "Command:\n $cmd\n" if (defined($cmd));
- exit(0);
- }
-}
-
sub stopAfter ($) {
my $stopAfter = shift @_;
@@ -251,12 +260,21 @@ sub stopAfter ($) {
}
-sub emitStage ($$$@) {
- return;
+sub emitStage ($$@) {
+ my $asm = shift @_;
+ my $stage = shift @_;
+ my $attempt = shift @_;
+
+ generateReport($asm);
+
+ if (!defined($attempt)) {
+ print STDERR "-- Finished stage '$stage', reset canuIteration.\n";
+ setGlobal("canuIteration", 0);
+ }
}
-sub skipStage ($$$@) {
+sub skipStage ($$@) {
return(0);
}
@@ -447,6 +465,64 @@ sub getBinDirectoryShellCode () {
+#
+# If running on a cloud system, shell scripts are started in some random location.
+# setWorkDirectory() will create the directory the script is supposed to run in (e.g.,
+# correction/0-mercounts) and move into it. This will keep the scripts compatible with the way
+# they are run from within canu.pl.
+#
+# If you're fine running in 'some random location' do nothing here.
+#
+# Note that canu does minimal cleanup.
+#
+
+sub setWorkDirectory () {
+
+ if ((getGlobal("objectStore") eq "TEST") && (defined($ENV{"JOB_ID"}))) {
+ my $jid = $ENV{'JOB_ID'};
+ my $tid = $ENV{'SGE_TASK_ID'};
+
+ make_path("/assembly/COMPUTE/job-$jid-$tid");
+ chdir ("/assembly/COMPUTE/job-$jid-$tid");
+ }
+
+ elsif (getGlobal("objectStore") eq "DNANEXUS") {
+ }
+
+ elsif (getGlobal("gridEngine") eq "PBSPRO") {
+ chdir($ENV{"PBS_O_WORKDIR"}) if (exists($ENV{"PBS_O_WORKDIR"}));
+ }
+}
+
+
+
+sub setWorkDirectoryShellCode ($) {
+ my $path = shift @_;
+ my $code = "";
+
+ if (getGlobal("objectStore") eq "TEST") {
+ $code .= "if [ z\$SGE_TASK_ID != z ] ; then\n";
+ $code .= " jid=\$JOB_ID\n";
+ $code .= " tid=\$SGE_TASK_ID\n";
+ $code .= " mkdir -p /assembly/COMPUTE/job-\$jid-\$tid/$path\n";
+ $code .= " cd /assembly/COMPUTE/job-\$jid-\$tid/$path\n";
+ $code .= " echo IN /assembly/COMPUTE/job-\$jid-\$tid/$path\n";
+ $code .= "fi\n";
+ }
+ elsif (getGlobal("objectStore") eq "DNANEXUS") {
+ # You're probably fine running in some random location, but if there is faster disk
+ # available, move there.
+ }
+ elsif (getGlobal("gridEngine") eq "PBSPRO") {
+ $code .= "if [ z\$PBS_O_WORKDIR != z ] ; then\n";
+ $code .= " cd \$PBS_O_WORKDIR\n";
+ $code .= "fi\n";
+ }
+
+ return($code);
+}
+
+
# Spend too much effort ensuring that the name is unique in the system. For 'canu' jobs, we don't
# care.
@@ -498,6 +574,9 @@ sub makeUniqueJobName ($$) {
if (uc(getGlobal("gridEngine")) eq "LSF") {
}
+ if (uc(getGlobal("gridEngine")) eq "DNANEXUS") {
+ }
+
# If the jobName doesn't exist, we can use it.
return($jobName) if (! exists($jobs{$jobName}));
@@ -526,36 +605,34 @@ sub makeUniqueJobName ($$) {
# The previous version (CA) would use "gridPropagateHold" to reset holds on existing jobs so that
# they would also hold on this job.
#
-sub submitScript ($$$) {
- my $wrk = shift @_;
+sub submitScript ($$) {
my $asm = shift @_;
- my $jobToWaitOn = shift @_;
+ my $jobHold = shift @_;
return if (getGlobal("useGrid") ne "1"); # If not requested to run on the grid,
return if (getGlobal("gridEngine") eq undef); # or can't run on the grid, don't run on the grid.
- # If no job to wait on, and we are already on the grid, do NOT resubmit ourself.
+ # If no job hold, and we are already on the grid, do NOT resubmit ourself.
#
# When the user launches canu on the head node, a call to submitScript() is made to launch canu
# under grid control. That results in a restart of canu, and another call to submitScript(),
# but this time, the envorinment variable is set, we we can skip the resubmission, and continue
# with canu execution.
- return if (($jobToWaitOn eq undef) && (exists($ENV{getGlobal("gridEngineJobID")})));
+ return if (($jobHold eq undef) && (exists($ENV{getGlobal("gridEngineJobID")})));
# Find the next available output file.
- make_path("$wrk/canu-scripts") if (! -d "$wrk/canu-scripts"); # Done in canu.pl, just being paranoid
+ make_path("canu-scripts") if (! -d "canu-scripts"); # Done in canu.pl, just being paranoid
my $idx = "01";
- while (-e "$wrk/canu-scripts/canu.$idx.out") {
+ while (-e "canu-scripts/canu.$idx.out") {
$idx++;
}
- my $output = "$wrk/canu-scripts/canu.$idx.out";
- my $script = "$wrk/canu-scripts/canu.$idx.sh";
- my $iteration = getGlobal("canuIteration");
+ my $outName = "canu-scripts/canu.$idx.out";
+ my $script = "canu-scripts/canu.$idx.sh";
# Make a script for us to submit.
@@ -577,8 +654,13 @@ sub submitScript ($$$) {
print F "\n";
print F getBinDirectoryShellCode();
print F "\n";
+ print F setWorkDirectoryShellCode(".");
+ print F "\n";
+ print F "rm -f canu.out\n";
+ print F "ln -s canu-scripts/canu.$idx.out canu.out\n";
+ print F "\n";
print F "/usr/bin/env perl \\\n";
- print F "\$bin/canu " . getCommandLineOptions() . " canuIteration=$iteration\n";
+ print F "\$bin/canu " . getCommandLineOptions() . " canuIteration=" . getGlobal("canuIteration") . "\n";
close(F);
system("chmod +x $script");
@@ -604,7 +686,9 @@ sub submitScript ($$$) {
$memOption = buildMemoryOption($mem, 1);
$thrOption = buildThreadOption($thr);
- $gridOpts = $memOption if (defined($memOption));
+ $gridOpts = $jobHold;
+ $gridOpts .= " " if (defined($gridOpts));
+ $gridOpts .= $memOption if (defined($memOption));
$gridOpts .= " " if (defined($gridOpts));
$gridOpts .= $thrOption if (defined($thrOption));
$gridOpts .= " " if (defined($gridOpts));
@@ -612,70 +696,19 @@ sub submitScript ($$$) {
$gridOpts .= " " if (defined($gridOpts));
$gridOpts .= getGlobal("gridOptionsExecutive") if (defined(getGlobal("gridOptionsExecutive")));
- # If the jobToWaitOn is defined, make the script wait for that to complete. LSF might need to
- # query jobs in the queue and figure out the job ID (or IDs) for the jobToWaitOn. Reading LSF
- # docs online (for bsub.1) claim that we can still use jobToWaitOn.
-
- if (defined($jobToWaitOn)) {
- my $hold = getGlobal("gridEngineHoldOption");
-
- # most grid engines don't understand job names to hold on, only IDs
- if ((uc(getGlobal("gridEngine")) eq "PBS") ||
- (uc(getGlobal("gridEngine")) eq "PBSPRO") ||
- (uc(getGlobal("gridEngine")) eq "SLURM")){
- my $tcmd = getGlobal("gridEngineNameToJobIDCommand");
- $tcmd =~ s/WAIT_TAG/$jobToWaitOn/g;
- my $propJobCount = `$tcmd |wc -l`;
- chomp $propJobCount;
- if ($propJobCount == 0) {
- $tcmd = getGlobal("gridEngineNameToJobIDCommandNoArray");
- $tcmd =~ s/WAIT_TAG/$jobToWaitOn/g;
- $hold = getGlobal("gridEngineHoldOptionNoArray");
- $propJobCount = `$tcmd |wc -l`;
- }
- if ($propJobCount != 1) {
- print STDERR "Warning: multiple IDs for job $jobToWaitOn got $propJobCount and should have been 1.\n";
- }
- my $jobID = undef;
- open(F, "$tcmd |awk '{print \$1}' |");
- while (<F>) {
- chomp $_;
- if (defined($jobID)) {
- $jobID = "$jobID:$_";
- } else {
- $jobID = $_;
- }
- }
- close(F);
- $hold =~ s/WAIT_TAG/$jobID/g;
- } else {
- $hold =~ s/WAIT_TAG/$jobToWaitOn/;
- }
- $gridOpts .= " " . $hold;
- }
-
-
my $submitCommand = getGlobal("gridEngineSubmitCommand");
my $nameOption = getGlobal("gridEngineNameOption");
my $outputOption = getGlobal("gridEngineOutputOption");
- my $qcmd = "$submitCommand $gridOpts $nameOption \"$jobName\" $outputOption $output $script";
+ my $qcmd = "$submitCommand $gridOpts $nameOption '$jobName' $outputOption $outName $script";
- runCommand($wrk, $qcmd) and caFailure("Failed to submit script", undef);
+ runCommand(getcwd(), $qcmd) and caFailure("Failed to submit script", undef);
exit(0);
}
-
-
-
-# Expects
-# job name
-# number of jobs
-# global pattern for option
-#
sub buildGridArray ($$$$) {
my ($name, $bgn, $end, $opt) = @_;
my $off = 0;
@@ -715,19 +748,54 @@ sub buildGridArray ($$$$) {
}
-
sub buildOutputName ($$$) {
my $path = shift @_;
my $script = shift @_;
- my $tid = shift @_;
+ my $tid = substr("000000" . (shift @_), -6);
+ my $o;
+
+ # When this function is called, canu.pl is running in the assembly directory.
+ # But, when the script is executed, it is rooted in '$path'. To get the
+ # 'logs' working, we need to check if the directory relative to the assembly root exists,
+ # but set it relative to $path (which is also where $script is relative to).
+
+ $o = "$script.$tid.out";
+ $o = "logs/$1.$tid.out" if ((-e "$path/logs") && ($script =~ m/scripts\/(.*)/));
+
+ return($o);
+}
- my $outName = "$path/$script.$tid.out";
- if ((-e "$path/logs") && ($script =~ m/scripts\/(.*)/)) {
- $outName = "$path/logs/$1.$tid.out";
+sub buildOutputOption ($$) {
+ my $path = shift @_;
+ my $script = shift @_;
+ my $tid = getGlobal("gridEngineArraySubmitID");
+ my $opt = getGlobal("gridEngineOutputOption");
+
+ if (defined($tid) && defined($opt)) {
+ my $o;
+
+ $o = "$script.$tid.out";
+ $o = "logs/$1.$tid.out" if ((-e "$path/logs") && ($script =~ m/scripts\/(.*)/));
+
+ return("$opt $o");
}
- return($outName);
+ return(undef);
+}
+
+
+sub buildStageOption ($$) {
+ my $t = shift @_;
+ my $d = shift @_;
+ my $r;
+
+ if ($t eq "cor") {
+ $r = getGlobal("gridEngineStageOption");
+ $r =~ s/DISK_SPACE/${d}/g;
+ }
+
+ return($r);
}
@@ -776,13 +844,14 @@ sub buildThreadOption ($) {
}
-sub buildGridJob ($$$$$$$$) {
+sub buildGridJob ($$$$$$$$$) {
my $asm = shift @_;
my $jobType = shift @_;
my $path = shift @_;
my $script = shift @_;
my $mem = shift @_;
my $thr = shift @_;
+ my $dsk = shift @_;
my $bgnJob = shift @_;
my $endJob = shift @_;
@@ -807,41 +876,43 @@ sub buildGridJob ($$$$$$$$) {
my ($jobName, $jobOff) = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayName"));
my ($arrayOpt, $arrayOff) = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayOption"));
- my $outputOption = getGlobal("gridEngineOutputOption");
- my $outName = buildOutputName($path, $script, getGlobal("gridEngineArraySubmitID"));
+ my $outputOption = buildOutputOption($path, $script);
+ my $stageOption = buildStageOption($jobType, $dsk);
my $memOption = buildMemoryOption($mem, $thr);
my $thrOption = buildThreadOption($thr);
+ my $globalOptions = getGlobal("gridOptions");
+ my $jobOptions = getGlobal("gridOptions$jobType");
- my $gridOpts;
+ my $opts;
- $gridOpts = getGlobal("gridOptions") if (defined(getGlobal("gridOptions")));
- $gridOpts .= " " if (defined($gridOpts));
- $gridOpts .= getGlobal("gridOptions$jobType") if (defined(getGlobal("gridOptions$jobType")));
- $gridOpts .= " " if (defined($gridOpts));
- $gridOpts .= $memOption if (defined($memOption));
- $gridOpts .= " " if (defined($gridOpts));
- $gridOpts .= $thrOption if (defined($thrOption));
+ $opts = "$stageOption " if (defined($stageOption));
+ $opts .= "$memOption " if (defined($memOption));
+ $opts .= "$thrOption " if (defined($thrOption));
+ $opts .= "$globalOptions " if (defined($globalOptions));
+ $opts .= "$jobOptions " if (defined($jobOptions));
+ $opts .= "$outputOption " if (defined($outputOption));
+ $opts =~ s/\s+$//;
- # Build the command line.
+ # Build and save the command line. Return the command PREFIX (we'll be adding .sh and .out as
+ # appropriate), and the job name it will be submitted with (which isn't expected to be used).
my $cmd;
- $cmd = " $submitCommand \\\n";
- $cmd .= " $gridOpts \\\n" if (defined($gridOpts));
- $cmd .= " $nameOption \"$jobName\" \\\n";
- $cmd .= " $arrayOpt \\\n";
- $cmd .= " $outputOption $outName \\\n";
- $cmd .= " $path/$script.sh $arrayOff\n";
-
- # Save it, just because.
open(F, "> $path/$script.jobSubmit.sh") or die;
- print F $cmd;
+ print F "#!/bin/sh\n";
+ print F "\n";
+ print F "$submitCommand \\\n";
+ print F " $opts \\\n" if (defined($opts));
+ print F " $nameOption \"$jobName\" \\\n";
+ print F " $arrayOpt \\\n";
+ print F " ./$script.sh $arrayOff \\\n";
+ print F "> ./$script.jobSubmit.out 2>&1\n";
close(F);
- # Return the command and the job name it will be submitted with.
+ chmod 0755, "$path/$script.jobSubmit.sh";
- return($cmd, $jobName);
+ return("$script.jobSubmit", $jobName);
}
@@ -941,8 +1012,7 @@ sub convertToJobRange (@) {
#
# If under grid control, submit grid jobs. Otherwise, run in parallel locally.
#
-sub submitOrRunParallelJob ($$$$$@) {
- my $wrk = shift @_; # Root of the assembly (NOT wrk/correction or wrk/trimming)
+sub submitOrRunParallelJob ($$$$@) {
my $asm = shift @_; # Name of the assembly
my $jobType = shift @_; # E.g., ovl, cns, ... - populates 'gridOptionsXXX
@@ -953,6 +1023,7 @@ sub submitOrRunParallelJob ($$$$$@) {
my $mem = getGlobal("${jobType}Memory");
my $thr = getGlobal("${jobType}Threads");
+ my $dsk = getGlobal("${jobType}StageSpace");
my @jobs = convertToJobRange(@_);
@@ -966,10 +1037,6 @@ sub submitOrRunParallelJob ($$$$$@) {
#print STDERR "----------------------------------------GRIDSTART $t\n";
#print STDERR "$path/$script.sh with $mem gigabytes memory and $thr threads.\n";
- # Check stopping rules.
-
- stopBefore($jobType, "$path/$script.sh");
-
# Break infinite loops. If the grid jobs keep failing, give up after a few attempts.
#
# submitScript() passes canuIteration on to the next call.
@@ -985,8 +1052,27 @@ sub submitOrRunParallelJob ($$$$$@) {
# If the jobs succeed in Iteration 2, the canu in iteration 3 will pass the Check(), never call
# this function, and continue the pipeline.
- caExit("canu iteration count too high, stopping pipeline (most likely a problem in the grid-based computes)", undef)
- if (getGlobal("canuIteration") > getGlobal("canuIterationMax"));
+ my $iter = getGlobal("canuIteration");
+ my $max = getGlobal("canuIterationMax");
+
+ if ($iter >= $max) {
+ caExit("canu iteration count too high, stopping pipeline (most likely a problem in the grid-based computes)", undef);
+ } elsif ($iter == 0) {
+ $iter = "First";
+ } elsif ($iter == 1) {
+ $iter = "Second";
+ } elsif ($iter == 2) {
+ $iter = "Third";
+ } elsif ($iter == 3) {
+ $iter = "Fourth";
+ } elsif ($iter == 4) {
+ $iter = "Fifth";
+ } else {
+ $iter = "${iter}th";
+ }
+
+ print STDERR "--\n";
+ print STDERR "-- Running jobs. $iter attempt out of $max.\n";
setGlobal("canuIteration", getGlobal("canuIteration") + 1);
@@ -1001,16 +1087,104 @@ sub submitOrRunParallelJob ($$$$$@) {
(getGlobal("useGrid") eq "1") &&
(getGlobal("useGrid$jobType") eq "1") &&
(exists($ENV{getGlobal("gridEngineJobID")}))) {
- my $cmd;
- my $jobName;
+ my @jobsSubmitted;
+
+ print STDERR "--\n";
foreach my $j (@jobs) {
- ($cmd, $jobName) = buildGridJob($asm, $jobType, $path, $script, $mem, $thr, $j, undef);
+ my ($cmd, $jobName) = buildGridJob($asm, $jobType, $path, $script, $mem, $thr, $dsk, $j, undef);
+
+ runCommandSilently($path, "./$cmd.sh", 0) and caFailure("Failed to submit batch jobs", "$path/$cmd.out");
+
+ # Parse the stdout/stderr from the submit command to find the id of the job
+ # we just submitted. We'll use this to hold the next iteration until all these
+ # jobs have completed.
+
+ open(F, "< $path/$cmd.out");
+ while (<F>) {
+ chomp;
+
+ if (uc(getGlobal("gridEngine")) eq "SGE") {
+ # Your job 148364 ("canu_asm") has been submitted
+ if (m/Your\sjob\s(\d+)\s/) {
+ $jobName = $1;
+ }
+ # Your job-array 148678.1500-1534:1 ("canu_asm") has been submitted
+ if (m/Your\sjob-array\s(\d+).\d+-\d+:\d\s/) {
+ $jobName = $1;
+ }
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "LSF") {
+ # Job <759810> is submitted to queue <14>.
+ if (m/Job\s<(\d+)>\sis/) {
+ $jobName = "ended($1)";
+ }
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "PBS") {
+ # 123456.qm2
+ $jobName = $_;
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "PBSPRO") {
+ # ??
+ $jobName = $_;
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "SLURM") {
+ if (m/Submitted\sbatch\sjob\s(\d+)/) {
+ $jobName = $1;
+ } else {
+ $jobName = $_;
+ }
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "DNANEXUS") {
+ }
+ }
+ close(F);
+
+ if ($j =~ m/^\d+$/) {
+ print STDERR "-- '$cmd.sh' -> job $jobName task $j.\n";
+ } else {
+ print STDERR "-- '$cmd.sh' -> job $jobName tasks $j.\n";
+ }
+
+ push @jobsSubmitted, $jobName;
+ }
+
+ print STDERR "--\n";
+
+ # All jobs submitted. Make an option to hold the executive on those jobs.
- runCommand($path, $cmd) and caFailure("Failed to submit batch jobs", undef);
+ my $jobHold;
+
+ if (uc(getGlobal("gridEngine")) eq "SGE") {
+ $jobHold = "-hold_jid " . join ",", @jobsSubmitted;
}
- submitScript($wrk, $asm, $jobName);
+ if (uc(getGlobal("gridEngine")) eq "LSF") {
+ $jobHold = "-w \"" . (join "&&", @jobsSubmitted) . "\"";
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "PBS") {
+ $jobHold = "-W depend=afteranyarray:" . join ":", @jobsSubmitted;
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "PBSPRO") {
+ $jobHold = "-W depend=afterany:" . join ":", @jobsSubmitted;
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "SLURM") {
+ $jobHold = "--depend=afterany:" . join ":", @jobsSubmitted;
+ }
+
+ if (uc(getGlobal("gridEngine")) eq "DNANEXUS") {
+ $jobHold = "...whatever magic needed to hold the job until all jobs in @jobsSubmitted are done...";
+ }
+
+ submitScript($asm, $jobHold);
# submitScript() should never return. If it does, then a parallel step was attempted too many time.
@@ -1024,26 +1198,25 @@ sub submitOrRunParallelJob ($$$$$@) {
(getGlobal("useGrid$jobType") eq "1") &&
(! exists($ENV{getGlobal("gridEngineJobID")}))) {
print STDERR "\n";
- print STDERR "Please submit the following jobs to the grid for execution using $mem gigabytes memory and $thr threads:\n";
+ print STDERR "Please run the following commands to submit jobs to the grid for execution using $mem gigabytes memory and $thr threads:\n";
print STDERR "\n";
foreach my $j (@jobs) {
- my ($cmd, $jobName) = buildGridJob($asm, $jobType, $path, $script, $mem, $thr, $j, undef);
+ my $cwd = getcwd();
+ my ($cmd, $jobName) = buildGridJob($asm, $jobType, $path, $script, $mem, $thr, $dsk, $j, undef);
- print $cmd;
+ print " $cwd/$path/$cmd.sh\n";
}
print STDERR "\n";
print STDERR "When all jobs complete, restart canu as before.\n";
+ print STDERR "\n";
exit(0);
}
# Standard jobs, run locally.
- my $cwd = getcwd(); # Remember where we are.
- chdir($path); # So we can root the jobs in the correct location.
-
foreach my $j (@jobs) {
my $st;
my $ed;
@@ -1056,13 +1229,10 @@ sub submitOrRunParallelJob ($$$$$@) {
}
for (my $i=$st; $i<=$ed; $i++) {
- my $outName = buildOutputName($path, $script, substr("000000" . $i, -6));
-
- schedulerSubmit("$path/$script.sh $i > $outName 2>&1");
+ schedulerSubmit("./$script.sh $i > ./" . buildOutputName($path, $script, $i) . " 2>&1");
}
}
-
# compute limit based on # of cpus
my $nCParallel = getGlobal("${jobType}Concurrency");
$nCParallel = int(getGlobal("maxThreads") / $thr) if ((!defined($nCParallel)) || ($nCParallel == 0));
@@ -1078,8 +1248,6 @@ sub submitOrRunParallelJob ($$$$$@) {
schedulerSetNumberOfProcesses($nParallel);
schedulerFinish($path);
-
- chdir($cwd);
}
@@ -1163,18 +1331,21 @@ sub runCommand ($$) {
# Log that we're starting, and show the pretty-ified command.
+ my $cwd = getcwd(); # Remember where we are.
+ chdir($dir); # So we can root the jobs in the correct location.
+
my $startsecs = time();
- my $diskfree = (defined($dir)) ? (diskSpace($dir)) : (0);
+ my $diskfree = diskSpace(".");
print STDERR "----------------------------------------\n";
- print STDERR "-- Starting command on ", scalar(localtime()), " with $diskfree GB free disk space\n" if (defined($dir));
- print STDERR "-- Starting command on ", scalar(localtime()), "\n" if (!defined($dir));
+ print STDERR "-- Starting command on ", scalar(localtime()), " with $diskfree GB free disk space\n";
print STDERR "\n";
+ print STDERR " cd $dir\n";
print STDERR "$dis\n";
- my $rc = 0xffff & system("cd $dir && $cmd");
+ my $rc = 0xffff & system($cmd);
- $diskfree = (defined($dir)) ? (diskSpace($dir)) : (0);
+ $diskfree = diskSpace(".");
my $warning = " !!! WARNING !!!" if ($diskfree < 10);
my $elapsed = time() - $startsecs;
@@ -1187,6 +1358,8 @@ sub runCommand ($$) {
print STDERR "-- Finished on ", scalar(localtime()), " ($elapsed) with $diskfree GB free disk space$warning\n";
print STDERR "----------------------------------------\n";
+ chdir($cwd);
+
# Pretty much copied from Programming Perl page 230
return(0) if ($rc == 0);
@@ -1198,6 +1371,8 @@ sub runCommand ($$) {
+# Duplicated in Grid_Cloud.pm to get around recursive 'use' statements.
+
sub runCommandSilently ($$$) {
my $dir = shift @_;
my $cmd = shift @_;
@@ -1206,11 +1381,12 @@ sub runCommandSilently ($$$) {
return(0) if ($cmd eq "");
- if (! -d $dir) {
- caFailure("Directory '$dir' doesn't exist, can't run command", "");
- }
+ my $cwd = getcwd(); # Remember where we are.
+ chdir($dir); # So we can root the jobs in the correct location.
+
+ my $rc = 0xffff & system($cmd);
- my $rc = 0xffff & system("cd $dir && $cmd");
+ chdir($cwd);
return(0) if ($rc == 0); # No errors, return no error.
return(1) if ($critical == 0); # If not critical, return that it failed, otherwise, report error and fail.
@@ -1254,16 +1430,14 @@ sub findExecutable ($) {
# Use caExit() for transient errors, like not opening files, processes that die, etc.
sub caExit ($$) {
- my $wrk = getGlobal("onExitDir");
my $asm = getGlobal("onExitNam");
my $msg = shift @_;
my $log = shift @_;
print STDERR "================================================================================\n";
- print STDERR "Don't panic, but a mostly harmless error occurred and canu failed.\n";
+ print STDERR "Don't panic, but a mostly harmless error occurred and Canu stopped.\n";
print STDERR "\n";
- # Really should pass in $wrk
if (defined($log)) {
my $df = diskSpace($log);
@@ -1278,12 +1452,15 @@ sub caExit ($$) {
print STDERR "\n";
}
- print STDERR "canu failed with '$msg'.\n";
+ my $version = getGlobal("version");
+
+ print STDERR "$version failed with:\n";
+ print STDERR " $msg\n";
print STDERR "\n";
my $fail = getGlobal('onFailure');
if (defined($fail)) {
- runCommandSilently($wrk, "$fail $asm", 0);
+ runCommandSilently(getGlobal("onExitDir"), "$fail $asm", 0);
}
exit(1);
@@ -1292,17 +1469,18 @@ sub caExit ($$) {
# Use caFailure() for errors that definitely will require code changes to fix.
sub caFailure ($$) {
- my $wrk = getGlobal("onExitDir");
- my $asm = getGlobal("onExitNam");
- my $msg = shift @_;
- my $log = shift @_;
+ my $asm = getGlobal("onExitNam");
+ my $msg = shift @_;
+ my $log = shift @_;
+ my $version = getGlobal("version");
+ my $trace = longmess(undef);
print STDERR "================================================================================\n";
- print STDERR "Please panic. canu failed, and it shouldn't have.\n";
+ print STDERR "Please panic. Canu failed, and it shouldn't have.\n";
print STDERR "\n";
print STDERR "Stack trace:\n";
print STDERR "\n";
- cluck;
+ print STDERR "$trace\n";
print STDERR "\n";
if (-e $log) {
@@ -1312,11 +1490,13 @@ sub caFailure ($$) {
}
print STDERR "\n";
- print STDERR "canu failed with '$msg'.\n";
+ print STDERR "$version failed with:\n";
+ print STDERR " $msg\n";
+ print STDERR "\n";
my $fail = getGlobal('onFailure');
if (defined($fail)) {
- runCommandSilently($wrk, "$fail $asm", 0);
+ runCommandSilently(getGlobal("onExitDir"), "$fail $asm", 0);
}
exit(1);
diff --git a/src/pipelines/canu/Gatekeeper.pm b/src/pipelines/canu/Gatekeeper.pm
index 4ce67e2..bf28144 100644
--- a/src/pipelines/canu/Gatekeeper.pm
+++ b/src/pipelines/canu/Gatekeeper.pm
@@ -40,56 +40,46 @@ package canu::Gatekeeper;
require Exporter;
@ISA = qw(Exporter);
- at EXPORT = qw(getMaxReadInStore getNumberOfReadsInStore getNumberOfBasesInStore getExpectedCoverage sequenceFileExists gatekeeper);
+ at EXPORT = qw(getMaxReadLengthInStore getNumberOfReadsInStore getNumberOfBasesInStore getExpectedCoverage sequenceFileExists gatekeeper);
use strict;
+use Cwd qw(getcwd);
+
use canu::Defaults;
use canu::Execution;
use canu::HTML;
+use canu::Report;
+use canu::Grid_Cloud;
-sub storeExists ($$) {
- my $wrk = shift @_;
- my $asm = shift @_;
-
- return (-e "$wrk/$asm.gkpStore");
-}
-
-sub getMaxReadInStore ($$) {
- my $wrk = shift @_; # Local work directory
+sub getMaxReadLengthInStore ($$) {
+ my $base = shift @_;
my $asm = shift @_;
- my $nr = 0;
+ my $ml = 0;
- # No file, no reads.
+ open(L, "< $base/$asm.gkpStore/maxreadlength.txt") or caExit("can't open '$base/$asm.gkpStore/maxreadlength.txt' for reading: $!", undef);
+ $ml = <L>;
+ close(L);
- return($nr) if (! -e "$wrk/$asm.gkpStore/readlengthhistogram.txt");
+ return(int($ml));
+}
- # Read the info file. gatekeeperCreate creates this at the end.
- open(F, "< $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for reading: $!", undef);
- while (<F>) {
- my @v = split '\s+', $_;
- $nr = $v[1];
- }
- close(F);
-
- return($nr);
-}
sub getNumberOfReadsInStore ($$) {
- my $wrk = shift @_; # Local work directory
+ my $base = shift @_;
my $asm = shift @_;
my $nr = 0;
# No file, no reads.
- return($nr) if (! -e "$wrk/$asm.gkpStore/info.txt");
+ return($nr) if (! -e "$base/$asm.gkpStore/info.txt");
# Read the info file. gatekeeperCreate creates this at the end.
- open(F, "< $wrk/$asm.gkpStore/info.txt") or caExit("can't open '$wrk/$asm.gkpStore/info.txt' for reading: $!", undef);
+ open(F, "< $base/$asm.gkpStore/info.txt") or caExit("can't open '$base/$asm.gkpStore/info.txt' for reading: $!", undef);
while (<F>) {
if (m/numReads\s+=\s+(\d+)/) {
$nr = $1;
@@ -103,34 +93,45 @@ sub getNumberOfReadsInStore ($$) {
sub getNumberOfBasesInStore ($$) {
- my $wrk = shift @_; # Local work directory
+ my $base = shift @_;
my $asm = shift @_;
my $nb = 0;
- my $bin = getBinDirectory();
# No file, no bases.
- return($nb) if (! -e "$wrk/$asm.gkpStore/info.txt");
+ return($nb) if (! -e "$base/$asm.gkpStore/info.txt");
# Read the info file. gatekeeperCreate creates this at the end.
- open(F, "< $wrk/$asm.gkpStore/reads.txt") or caExit("can't open '$wrk/$asm.gkpStore/reads.txt' for reading: $!", undef);
+ open(F, "< $base/$asm.gkpStore/reads.txt") or caExit("can't open '$base/$asm.gkpStore/reads.txt' for reading: $!", undef);
while (<F>) {
my @v = split '\s+', $_;
$nb += $v[2];
}
close(F);
+ # An alternate, used in Meryl::getGenomeCoverage (deceased) was to parse the stats output.
+ #
+ #open(F, "$bin/gatekeeperDumpMetaData -stats -G $base/$asm.gkpStore | ") or caFailure("failed to read gatekeeper stats fromfrom '$base/$asm.gkpStore'", undef);
+ #while (<F>) {
+ # my ($junk1, $library, $junk2, $reads, $junk3, $junk4, $bases, $junk5, $average, $junk6, $min, $junk7, $max) = split '\s+', $_;
+ # if ($library == 0) {
+ # $gs = $bases / getGlobal("genomeSize");
+ # last;
+ # }
+ #}
+ #close(F);
+
return($nb);
}
sub getExpectedCoverage ($$) {
- my $wrk = shift @_; # Local work directory
- my $asm = shift @_;
+ my $base = shift @_;
+ my $asm = shift @_;
- return(int(getNumberOfBasesInStore($wrk, $asm) / getGlobal("genomeSize")));
+ return(int(getNumberOfBasesInStore($base, $asm) / getGlobal("genomeSize")));
}
@@ -143,7 +144,7 @@ sub sequenceFileExists ($) {
foreach my $s ("", ".fasta", ".fastq", ".fa", ".fq") {
foreach my $c ("", ".gz", ".xz") {
- return("$p$s$c") if (-e "$p$s$c");
+ return("$p$s$c") if (fileExists("$p$s$c"));
}
}
@@ -152,18 +153,18 @@ sub sequenceFileExists ($) {
-sub gatekeeperCreateStore ($$$@) {
- my $wrk = shift @_; # Local work directory
+sub gatekeeperCreateStore ($$@) {
+ my $base = shift @_;
my $asm = shift @_;
- my $tag = shift @_;
my $bin = getBinDirectory();
my @inputs = @_;
# If the store failed to build because of input errors and warnings, rename the store and continue.
+ # Not sure how to support this in DNANexus.
- if (-e "$wrk/$asm.gkpStore.ACCEPTED") {
- rename("$wrk/$asm.gkpStore.ACCEPTED", "$wrk/$asm.gkpStore");
- rename("$wrk/$asm.gkpStore.BUILDING.err", "$wrk/$asm.gkpStore.err");
+ if (-e "$base/$asm.gkpStore.ACCEPTED") {
+ rename("$base/$asm.gkpStore.ACCEPTED", "$base/$asm.gkpStore");
+ rename("$base/$asm.gkpStore.BUILDING.err", "$base/$asm.gkpStore.err");
return;
}
@@ -172,29 +173,38 @@ sub gatekeeperCreateStore ($$$@) {
caExit("no input files specified, and store not already created, I have nothing to work on!", undef)
if (scalar(@inputs) == 0);
- # Make sure all the inputs are here.
+ # Convert the canu-supplied reads into correct relative paths. This is made complicated by
+ # gatekeeperCreate being run in a directory one below where we are now.
+
+ # At the same time, check that all files exist.
- my $failedFiles = undef;
+ my $ff = undef;
foreach my $iii (@inputs) {
- my $file = $iii; # This stupid foreach works by reference!
+ my ($type, $file) = split '\0', $iii;
- $file = $2 if ($file =~ m/^(.*)\0(.*)/); # Handle the raw sequence inputs.
+ if (($file =~ m/\.correctedReads\./) ||
+ ($file =~ m/\.trimmedReads\./)) {
+ fetchFile($file);
- if (! -e $file) {
- if (defined($failedFiles)) {
- $failedFiles .= "; '$file' not found in gatekeeper()";
- } else {
- $failedFiles = "'$file' not found in gatekeeper()";
- }
+ chdir($base); # Move to where we run the command
+ $file = "../$file" if (-e "../$file"); # If file exists up one dir, it's our file
+ $iii = "$type\0$file"; # Rewrite the option
+ chdir(".."); # ($file is used below too)
}
+
+ chdir($base);
+ $ff .= (defined($ff) ? "\n " : "") . "reads '$file' not found." if (! -e $file);
+ chdir("..");
}
- caExit($failedFiles, undef) if defined($failedFiles);
+
+ caExit($ff, undef) if defined($ff);
+
# Build a gkp file for all the raw sequence inputs. For simplicity, we just copy in any gkp
# files as is. This documents what gatekeeper was built with, etc.
- open(F, "> $wrk/$asm.gkpStore.gkp") or caExit("cant' open '$wrk/$asm.gkpStore.gkp' for writing: $0", undef);
+ open(F, "> $base/$asm.gkpStore.gkp") or caExit("cant' open '$base/$asm.gkpStore.gkp' for writing: $0", undef);
foreach my $iii (@inputs) {
if ($iii =~ m/^-(.*)\0(.*)$/) {
@@ -238,27 +248,25 @@ sub gatekeeperCreateStore ($$$@) {
my $cmd;
$cmd .= "$bin/gatekeeperCreate \\\n";
$cmd .= " -minlength " . getGlobal("minReadLength") . " \\\n";
- $cmd .= " -o $wrk/$asm.gkpStore.BUILDING \\\n";
- $cmd .= " $wrk/$asm.gkpStore.gkp \\\n";
- $cmd .= "> $wrk/$asm.gkpStore.BUILDING.err 2>&1";
-
- stopBefore("gatekeeper", $cmd);
+ $cmd .= " -o ./$asm.gkpStore.BUILDING \\\n";
+ $cmd .= " ./$asm.gkpStore.gkp \\\n";
+ $cmd .= "> ./$asm.gkpStore.BUILDING.err 2>&1";
# A little funny business to make gatekeeper not fail on read quality issues.
# A return code of 0 is total success.
# A return code of 1 means it found errors in the inputs, but finished.
# Anything larger is a crash.
- if (runCommand($wrk, $cmd) > 1) {
- caExit("gatekeeper failed", "$wrk/$asm.gkpStore.BUILDING.err");
+ if (runCommand($base, $cmd) > 1) {
+ caExit("gatekeeper failed", "$base/$asm.gkpStore.BUILDING.err");
}
# Check for quality issues.
- if (-e "$wrk/$asm.gkpStore.BUILDING.err") {
+ if (-e "$base/$asm.gkpStore.BUILDING.err") {
my $nProblems = 0;
- open(F, "< $wrk/$asm.gkpStore.BUILDING.err");
+ open(F, "< $base/$asm.gkpStore.BUILDING.err");
while (<F>) {
$nProblems++ if (m/Check\syour\sreads/);
}
@@ -266,16 +274,16 @@ sub gatekeeperCreateStore ($$$@) {
if ($nProblems > 0) {
print STDERR "Gatekeeper detected problems in your input reads. Please review the logging in files:\n";
- print STDERR " $wrk/$asm.gkpStore.BUILDING.err\n";
- print STDERR " $wrk/$asm.gkpStore.BUILDING/errorLog\n";
+ print STDERR " ", getcwd(), "/$base/$asm.gkpStore.BUILDING.err\n";
+ print STDERR " ", getcwd(), "/$base/$asm.gkpStore.BUILDING/errorLog\n";
if (getGlobal("stopOnReadQuality")) {
print STDERR "If you wish to proceed, rename the store with the following commands and restart canu.\n";
print STDERR "\n";
- print STDERR " mv $wrk/$asm.gkpStore.BUILDING \\\n";
- print STDERR " $wrk/$asm.gkpStore.ACCEPTED\n";
+ print STDERR " mv ", getcwd(), "/$base/$asm.gkpStore.BUILDING \\\n";
+ print STDERR " ", getcwd(), "/$base/$asm.gkpStore.ACCEPTED\n";
print STDERR "\n";
- print STDERR "Or remove $wrk and re-run with stopOnReadQuality=false\n";
+ print STDERR "Or remove '", getcwd(), "/$base/' and re-run with stopOnReadQuality=false\n";
print STDERR "\n";
exit(1);
} else {
@@ -284,88 +292,103 @@ sub gatekeeperCreateStore ($$$@) {
}
}
- rename "$wrk/$asm.gkpStore.BUILDING", "$wrk/$asm.gkpStore";
- rename "$wrk/$asm.gkpStore.BUILDING.err", "$wrk/$asm.gkpStore.err";
+ rename "$base/$asm.gkpStore.BUILDING", "$base/$asm.gkpStore";
+ rename "$base/$asm.gkpStore.BUILDING.err", "$base/$asm.gkpStore.err";
}
-sub gatekeeperGenerateReadsList ($$$) {
- my $wrk = shift @_; # Local work directory
+sub gatekeeperGenerateReadsList ($$) {
+ my $base = shift @_;
my $asm = shift @_;
- my $tag = shift @_;
my $bin = getBinDirectory();
- if (runCommandSilently("$wrk/$asm.gkpStore", "$bin/gatekeeperDumpMetaData -G . -reads > reads.txt 2> /dev/null", 1)) {
+ if (runCommandSilently($base, "$bin/gatekeeperDumpMetaData -G ./$asm.gkpStore -reads > ./$asm.gkpStore/reads.txt 2> /dev/null", 1)) {
caExit("failed to generate list of reads in store", undef);
}
}
-sub gatekeeperGenerateLibrariesList ($$$) {
- my $wrk = shift @_; # Local work directory
+
+
+sub gatekeeperGenerateLibrariesList ($$) {
+ my $base = shift @_;
my $asm = shift @_;
- my $tag = shift @_;
my $bin = getBinDirectory();
- if (runCommandSilently("$wrk/$asm.gkpStore", "$bin/gatekeeperDumpMetaData -G . -libs > libraries.txt 2> /dev/null", 1)) {
+ if (runCommandSilently($base, "$bin/gatekeeperDumpMetaData -G ./$asm.gkpStore -libs > ./$asm.gkpStore/libraries.txt 2> /dev/null", 1)) {
caExit("failed to generate list of libraries in store", undef);
}
}
-sub gatekeeperGenerateReadLengths ($$$) {
- my $wrk = shift @_; # Local work directory
+
+
+sub gatekeeperGenerateReadLengths ($$) {
+ my $base = shift @_;
my $asm = shift @_;
- my $tag = shift @_;
- my $bin = getBinDirectory();
- my $nb = 0;
- my @rl;
- my @hi;
- my $mm;
+ my $nb = 0;
+ my @rl;
+ my @hi;
+ my $mm;
+ my $minLen = 999999;
+ my $maxLen = 0;
- open(F, "< $wrk/$asm.gkpStore/reads.txt") or caExit("can't open '$wrk/$asm.gkpStore/reads.txt' for reading: $!", undef);
- while (<F>) {
- my @v = split '\s+', $_;
+ open(F, "< $base/$asm.gkpStore/reads.txt") or caExit("can't open '$base/$asm.gkpStore/reads.txt' for reading: $!", undef);
+ while (<F>) {
+ my @v = split '\s+', $_;
- push @rl, $v[2]; # Save the length
- $nb += $v[2]; # Sum the bases
- $hi[int($v[2] / 1000)]++; # Add to the histogram (int truncates)
- }
- close(F);
+ push @rl, $v[2]; # Save the length
+ $nb += $v[2]; # Sum the bases
- @rl = sort { $a <=> $b } @rl;
- $mm = int($rl[scalar(@rl)-1] / 1000); # max histogram value
+ $minLen = ($minLen < $v[2]) ? $minLen : $v[2];
+ $maxLen = ($v[2] < $maxLen) ? $maxLen : $v[2];
+ }
+ close(F);
- open(F, "> $wrk/$asm.gkpStore/readlengths.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengths.txt' for writing: $!", undef);
- foreach my $rl (@rl) {
- print F "$rl\n";
- }
- close(F);
+ @rl = sort { $a <=> $b } @rl;
- open(F, "> $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for writing: $!", undef);
- for (my $ii=0; $ii<=$mm; $ii++) {
- my $s = $ii * 1000;
- my $e = $ii * 1000 + 999;
+ # Buckets of size 1000 are easy to interpret, but sometimes not ideal.
- $hi[$ii] += 0; # Otherwise, cells with no count print as null.
+ my $bucketSize = 0;
- print F "$s\t$e\t$hi[$ii]\n";
- }
- close(F);
-}
+ if ($maxLen - $minLen < 10000) {
+ $bucketSize = 100;
+ } elsif ($maxLen - $minLen < 100000) {
+ $bucketSize = 1000;
+ } elsif ($maxLen - $minLen < 1000000) {
+ $bucketSize = 5000;
+ } else {
+ $bucketSize = 10000;
+ }
+ # Generate the histogram (int truncates)
+ foreach my $rl (@rl) {
+ my $b = int($rl / $bucketSize);
-sub gatekeeperGenerateReadLengthPlot ($$$) {
- my $wrk = shift @_; # Local work directory
- my $asm = shift @_;
- my $tag = shift @_;
- my $bin = getBinDirectory();
+ $hi[$b]++;
+ }
+
+ $mm = int($maxLen / $bucketSize); # Max histogram value
+
+ # Write the sorted read lengths (for gnuplot) and the maximum read length (for correction consensus)
+
+ open(F, "> $base/$asm.gkpStore/readlengths.txt") or caExit("can't open '$base/$asm.gkpStore/readlengths.txt' for writing: $!", undef);
+ foreach my $rl (@rl) {
+ print F "$rl\n";
+ }
+ close(F);
+
+ open(F, "> $base/$asm.gkpStore/maxreadlength.txt") or caExit("can't open '$base/$asm.gkpStore/maxreadlength.txt' for writing: $!", undef);
+ print F "$maxLen\n";
+ close(F);
+
+ # Generate PNG histograms
my $gnuplot = getGlobal("gnuplot");
my $format = getGlobal("gnuplotImageFormat");
- open(F, "> $wrk/$asm.gkpStore/readlengths.gp") or caExit("can't open '$wrk/$asm.gkpStore/readlengths.gp' for writing: $!", undef);
+ open(F, "> $base/$asm.gkpStore/readlengths.gp") or caExit("can't open '$base/$asm.gkpStore/readlengths.gp' for writing: $!", undef);
print F "set title 'read length'\n";
print F "set xlabel 'read length, bin width = 250'\n";
print F "set ylabel 'number of reads'\n";
@@ -375,101 +398,107 @@ sub gatekeeperGenerateReadLengthPlot ($$$) {
print F "bin(x,width) = width*floor(x/width) + binwidth/2.0\n";
print F "\n";
print F "set terminal $format size 1024,1024\n";
- print F "set output '$wrk/$asm.gkpStore/readlengths.lg.$format'\n";
- print F "plot [] '$wrk/$asm.gkpStore/readlengths.txt' using (bin(\$1,binwidth)):(1.0) smooth freq with boxes title ''\n";
+ print F "set output './$asm.gkpStore/readlengths.lg.$format'\n";
+ print F "plot [] './$asm.gkpStore/readlengths.txt' using (bin(\$1,binwidth)):(1.0) smooth freq with boxes title ''\n";
print F "\n";
print F "set terminal $format size 256,256\n";
- print F "set output '$wrk/$asm.gkpStore/readlengths.sm.$format'\n";
- print F "plot [] '$wrk/$asm.gkpStore/readlengths.txt' using (bin(\$1,binwidth)):(1.0) smooth freq with boxes title ''\n";
+ print F "set output './$asm.gkpStore/readlengths.sm.$format'\n";
+ print F "plot [] './$asm.gkpStore/readlengths.txt' using (bin(\$1,binwidth)):(1.0) smooth freq with boxes title ''\n";
close(F);
- if (runCommandSilently("$wrk/$asm.gkpStore", "$gnuplot $wrk/$asm.gkpStore/readlengths.gp > /dev/null 2>&1", 0)) {
+ if (runCommandSilently($base, "$gnuplot ./$asm.gkpStore/readlengths.gp > /dev/null 2>&1", 0)) {
print STDERR "--\n";
print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n";
print STDERR "--\n";
print STDERR "----------------------------------------\n";
}
-}
-
+ # Generate the ASCII histogram
-sub gatekeeperReportReadLengthHistogram ($$$) {
- my $wrk = shift @_; # Local work directory
- my $asm = shift @_;
- my $tag = shift @_;
- my $bin = getBinDirectory();
-
- my $reads = getNumberOfReadsInStore($wrk, $asm);
- my $bases = getNumberOfBasesInStore($wrk, $asm);
+ my $reads = getNumberOfReadsInStore($base, $asm);
+ my $bases = getNumberOfBasesInStore($base, $asm);
my $coverage = int(100 * $bases / getGlobal("genomeSize")) / 100;
- my $maxhist = 0;
+ my $scale = 0;
+ my $hist;
- open(F, "< $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for reading: $!", undef);
- while (<F>) {
- my @v = split '\s+', $_;
- $maxhist = ($maxhist < $v[2]) ? $v[2] : $maxhist;
+ for (my $ii=0; $ii<=$mm; $ii++) { # Scale the *'s so that the longest has 70 of 'em
+ $scale = $hi[$ii] / 70 if ($scale < $hi[$ii] / 70);
}
- close(F);
- my $scale = $maxhist / 70;
+ $hist = "--\n";
+ $hist .= "-- In gatekeeper store '$base/$asm.gkpStore':\n";
+ $hist .= "-- Found $reads reads.\n";
+ $hist .= "-- Found $bases bases ($coverage times coverage).\n";
+ $hist .= "--\n";
+ $hist .= "-- Read length histogram (one '*' equals " . int(100 * $scale) / 100 . " reads):\n";
- print STDERR "--\n";
- print STDERR "-- In gatekeeper store '$wrk/$asm.gkpStore':\n";
- print STDERR "-- Found $reads reads.\n";
- print STDERR "-- Found $bases bases ($coverage times coverage).\n";
- print STDERR "--\n";
- print STDERR "-- Read length histogram (one '*' equals ", int(100 * $scale) / 100, " reads):\n";
+ for (my $ii=0; $ii<=$mm; $ii++) {
+ my $s = $ii * $bucketSize;
+ my $e = $ii * $bucketSize + $bucketSize - 1;
- open(F, "< $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for reading: $!", undef);
- while (<F>) {
- my @v = split '\s+', $_;
+ $hi[$ii] += 0; # Otherwise, cells with no count print as null.
- printf STDERR "-- %6d %6d %6d %s\n", $v[0], $v[1], $v[2], "*" x int($v[2] / $scale);
+ $hist .= sprintf("-- %6d %6d %6d %s\n", $s, $e, $hi[$ii], "*" x int($hi[$ii] / $scale));
}
- close(F);
+
+ return($hist);
}
-sub gatekeeper ($$$@) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub gatekeeper ($$@) {
my $asm = shift @_;
my $tag = shift @_;
- my $bin = getBinDirectory();
my @inputs = @_;
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base;
+
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ # Try fetching the store from object storage. This might not be needed in all cases (e.g.,
+ # between mhap precompute and mhap compute), but it greatly simplifies stuff, like immediately
+ # here needing to check if the store exists.
+
+ fetchStore("$base/$asm.gkpStore");
# An empty store? Remove it and try again.
- if ((storeExists($wrk, $asm)) && (getNumberOfReadsInStore($wrk, $asm) == 0)) {
- print STDERR "-- Removing empty or incomplate gkpStore '$wrk/$asm.gkpStore'\n";
- runCommandSilently($wrk, "rm -rf $wrk/$asm.gkpStore", 1);
+ if ((-e "$base/$asm.gkpStore/info") && (getNumberOfReadsInStore($base, $asm) == 0)) {
+ print STDERR "-- Removing empty or incomplate gkpStore '$base/$asm.gkpStore'\n";
+ runCommandSilently($base, "rm -rf ./$asm.gkpStore", 1);
}
# Store with reads? Yay! Report it, then skip.
- goto allDone if (skipStage($WRK, $asm, "$tag-gatekeeper") == 1);
- goto allDone if (getNumberOfReadsInStore($wrk, $asm) > 0);
+ goto allDone if (skipStage($asm, "$tag-gatekeeper") == 1);
+ goto allDone if (getNumberOfReadsInStore($base, $asm) > 0);
- gatekeeperCreateStore($wrk, $asm, $tag, @inputs) if (! -e "$wrk/$asm.gkpStore");
+ # Create the store. If all goes well, we get asm.gkpStore. If not, we could end up with
+ # asm.BUILDING.gkpStore and ask the user to examine it and rename it to asm.ACCEPTED.gkpStore
+ # and restart. On the restart, gatekeeperCreateStore() detects the 'ACCPETED' store and
+ # renames to asm.gkpStore.
- caExit("gatekeeper store exists, but contains no reads", undef) if (getNumberOfReadsInStore($wrk, $asm) == 0);
+ gatekeeperCreateStore($base, $asm, @inputs) if (! -e "$base/$asm.gkpStore");
- gatekeeperGenerateReadsList($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/reads.txt");
- gatekeeperGenerateLibrariesList($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/libraries.txt");
- gatekeeperGenerateReadLengths($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/readlengths.txt");
- gatekeeperGenerateReadLengthPlot($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/readlengths.gp");
+ caExit("gatekeeper store exists, but contains no reads", undef) if (getNumberOfReadsInStore($base, $asm) == 0);
- finishStage:
- gatekeeperReportReadLengthHistogram($wrk, $asm, $tag);
+ gatekeeperGenerateReadsList($base, $asm) if (! -e "$base/$asm.gkpStore/reads.txt");
+ gatekeeperGenerateLibrariesList($base, $asm) if (! -e "$base/$asm.gkpStore/libraries.txt");
- emitStage($WRK, $asm, "$tag-gatekeeper");
- buildHTML($WRK, $asm, $tag);
- stopAfter("gatekeeper");
+ my $hist = gatekeeperGenerateReadLengths($base, $asm) if (! -e "$base/$asm.gkpStore/readlengths.txt");
+
+ addToReport("${tag}GkpStore", $hist);
+
+ # Now that all the extra data is generated, stash the store.
+
+ stashStore("$base/$asm.gkpStore");
+
+ finishStage:
+ emitStage($asm, "$tag-gatekeeper");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("gatekeeper");
}
diff --git a/src/pipelines/canu/Grid_Cloud.pm b/src/pipelines/canu/Grid_Cloud.pm
new file mode 100644
index 0000000..2a30570
--- /dev/null
+++ b/src/pipelines/canu/Grid_Cloud.pm
@@ -0,0 +1,347 @@
+
+###############################################################################
+ #
+ # This file is part of canu, a software program that assembles whole-genome
+ # sequencing reads into contigs.
+ #
+ # This software is based on:
+ # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ # the 'kmer package' (http://kmer.sourceforge.net)
+ # both originally distributed by Applera Corporation under the GNU General
+ # Public License, version 2.
+ #
+ # Canu branched from Celera Assembler at its revision 4587.
+ # Canu branched from the kmer project at its revision 1994.
+ #
+ # Modifications by:
+ #
+ # Brian P. Walenz beginning on 2017-FEB-15
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
+ # File 'README.licenses' in the root directory of this distribution contains
+ # full conditions and disclaimers for each license.
+ ##
+
+package canu::Grid_Cloud;
+
+require Exporter;
+
+ at ISA = qw(Exporter);
+ at EXPORT = qw(fileExists
+ fileExistsShellCode
+ fetchFile
+ fetchFileShellCode
+ stashFile
+ stashFileShellCode
+ fetchStore
+ fetchStoreShellCode
+ stashStore
+ stashStoreShellCode);
+
+use strict;
+
+use File::Path qw(make_path);
+use File::Basename;
+
+use Cwd qw(getcwd);
+
+use canu::Defaults;
+use canu::Grid;
+use canu::Execution qw(runCommand runCommandSilently);
+
+
+# This file contains most of the magic needed to access an object store. Two flavors of each
+# function are needed: one that runs in the canu.pl process (rooted in the base assembly directory,
+# where the 'correction', 'trimming' and 'unitigging' directories exist) and one that is
+# used in shell scripts (rooted where the shell script is run from).
+
+
+# Convert a/path/to/file to ../../../..
+sub pathToDots ($) {
+ return(join("/", map("..", (1..scalar(split '/', $_[0])))));
+}
+
+# True if we're using an object store.
+sub isOS () {
+ return(getGlobal("objectStore"));
+}
+
+
+#
+# fileExists() returns true if the file exists on disk or in the object store. It does not fetch
+# the file. It returns undef if the file doesn't exist. The second argument to
+# fileExistsShellCode() is an optional indent level (a whitespace string).
+#
+# The shellCode version should emit the if test for file existence, but nothing else (not even the
+# endif).
+#
+
+sub fileExists ($) {
+ my $file = shift @_;
+ my $exists = "";
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ return(1) if (-e $file); # If file exists, it exists.
+
+ if (isOS() eq "TEST") {
+ $exists = `$client describe --name $ns/$file`;
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ $exists = "";
+ }
+
+ $exists =~ s/^\s+//;
+ $exists =~ s/\s+$//;
+
+ return(($exists ne "") ? 1 : undef);
+}
+
+
+
+sub fileExistsShellCode ($@) {
+ my $file = shift @_;
+ my $indent = shift @_;
+ my $code = "";
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ if (isOS() eq "TEST") {
+ $code .= "${indent}if [ ! -e $file ] ; then\n";
+ $code .= "${indent} exists=`$client describe --name $ns/$file`\n";
+ $code .= "${indent}fi\n";
+ $code .= "${indent}if [ -e $file -o x\$exists != x ] ; then\n";
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ $code .= "${indent}if [ -e $file ]; then\n";
+ }
+
+ return($code);
+}
+
+
+
+#
+# fetchFile() and stashFile() both expect to be called from the assembly root directory, and have
+# the path to the file passed in, e.g., "correction/0-mercounts/whatever.histogram".
+#
+# The shellCode versions expect the same, but need the path from the assembly root to the location
+# the shell script is running split. A meryl script would give "correction/0-mercounts" for the
+# first arg, and could give "some/directory/file" for the file.
+#
+
+sub fetchFile ($) {
+ my $file = shift @_;
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ return if (-e $file); # If it exists, we don't need to fetch it.
+
+ if (isOS() eq "TEST") {
+ make_path(dirname($file));
+ runCommandSilently(".", "$client download --output $file $ns/$file", 1);
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ # Nothing we can be obnoxious about here, I suppose we could log...
+ }
+}
+
+
+
+sub fetchFileShellCode ($$$) {
+ my $path = shift @_;
+ my $dots = pathToDots($path);
+ my $file = shift @_;
+ my $indent = shift @_;
+ my $code = "";
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ # We definitely need to be able to fetch files from places that are
+ # parallel to us, e.g., from 0-mercounts when we're in 1-overlapper.
+ #
+ # To get a file, we first go up to the assembly root, then check if the
+ # file exists, and fetch it if not.
+ #
+ # The call needs to be something like:
+ # stashFileShellCode("correction/0-mercounts", "whatever", "");
+
+ if (isOS() eq "TEST") {
+ $code .= "${indent}if [ ! -e $dots/$path/$file ] ; then\n";
+ $code .= "${indent} mkdir -p $dots/$path\n";
+ $code .= "${indent} cd $dots/$path\n";
+ $code .= "${indent} $client download --output $file $ns/$path/$file\n";
+ $code .= "${indent} cd -\n";
+ $code .= "${indent}fi\n";
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ $code .= "# File must exist: $file\n";
+ }
+
+ return($code);
+}
+
+
+
+sub stashFile ($) {
+ my $file = shift @_;
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ return if (! -e $file);
+
+ if (isOS() eq "TEST") {
+ runCommandSilently(".", "$client upload --path $ns/$file $file", 1);
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ # Nothing we can be obnoxious about here, I suppose we could log...
+ }
+
+}
+
+
+
+sub stashFileShellCode ($$$) {
+ my $path = shift @_;
+ my $dots = pathToDots($path);
+ my $file = shift @_;
+ my $indent = shift @_;
+ my $code = "";
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ # Just like for fetching, we allow stashing files from parallel
+ # directories (even though that should never happen).
+
+ if (isOS() eq "TEST") {
+ $code .= "${indent}if [ -e $dots/$path/$file ] ; then\n";
+ $code .= "${indent} cd $dots/$path\n";
+ $code .= "${indent} $client upload --path $ns/$path/$file $file\n";
+ $code .= "${indent} cd -\n";
+ $code .= "${indent}fi\n";
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ $code .= "# File is important: $file\n";
+ }
+
+ return($code);
+}
+
+
+
+#
+# Given $base/$asm.gkpStore, fetch or stash it.
+#
+# The non-shell versions are assumed to be running in the assembly directory, that is, where
+# $base/$asm.gkpStore would exist naturally. This is consistent with canu.pl - it runs in the
+# assembly directory, and then chdir to subdirectories to run binaries.
+#
+# The shell versions usually run within a subdirectory (e.g., in correction/0-mercounts). They
+# need to know this location, so they can go up to the assembly directory to fetch and unpack the
+# store. After fetching, they chdir back to the subdirectory.
+#
+
+sub fetchStore ($) {
+ my $store = shift @_; # correction/asm.gkpStore
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ return if (-e "$store/info"); # Store exists on disk
+ return if (! fileExists("$store.tar")); # Store doesn't exist in object store
+
+ if (isOS() eq "TEST") {
+ runCommandSilently(".", "$client download --output - $ns/$store.tar | tar -xf -", 1);
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ }
+}
+
+
+
+sub stashStore ($) {
+ my $store = shift @_; # correction/asm.gkpStore
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ return if (! -e "$store/info"); # Store doesn't exist on disk
+
+ if (isOS() eq "TEST") {
+ runCommandSilently(".", "tar -cf - $store | $client upload --path $ns/$store.tar -", 1);
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ }
+}
+
+
+
+sub fetchStoreShellCode ($$@) {
+ my $store = shift @_; # correction/asm.gkpStore - store we're trying to get
+ my $root = shift @_; # correction/1-overlapper - place the script is running in
+ my $indent = shift @_; #
+ my $base = dirname($store); # correction
+ my $basep = pathToDots($root); # ../..
+ my $name = basename($store); # asm.gkpStore
+ my $code;
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ if (isOS() eq "TEST") {
+ $code .= "${indent}if [ ! -e $basep/$store/info ] ; then\n";
+ $code .= "${indent} echo Fetching $ns/$store\n";
+ $code .= "${indent} $client download --output - $ns/$store.tar | tar -C $basep -xf -\n";
+ $code .= "${indent}fi\n";
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ $code .= "# Store must exist: $store\n";
+ }
+
+ return($code);
+}
+
+
+
+sub stashStoreShellCode ($$@) {
+ my $store = shift @_; # correction/asm.gkpStore - store we're trying to get
+ my $root = shift @_; # correction/1-overlapper - place the script is running in
+ my $indent = shift @_; #
+ my $base = dirname($store); # correction
+ my $basep = pathToDots($root); # ../..
+ my $name = basename($store); # asm.gkpStore
+ my $code;
+ my $client = getGlobal("objectStoreClient");
+ my $ns = getGlobal("objectStoreNameSpace");
+
+ if (isOS() eq "TEST") {
+ $code .= "${indent}if [ -e $basep/$store/info ] ; then\n";
+ $code .= "${indent} echo Stashing $ns/$store\n";
+ $code .= "${indent} tar -C $basep -cf - $store | $client upload --path $ns/$store.tar -\n";
+ $code .= "${indent}fi\n";
+ }
+ elsif (isOS() eq "DNANEXUS") {
+ }
+ else {
+ $code .= "# Store is important: $store\n";
+ }
+
+ return($code);
+}
+
diff --git a/src/pipelines/canu/Grid_DNANexus.pm b/src/pipelines/canu/Grid_DNANexus.pm
new file mode 100644
index 0000000..ef896a3
--- /dev/null
+++ b/src/pipelines/canu/Grid_DNANexus.pm
@@ -0,0 +1,104 @@
+
+###############################################################################
+ #
+ # This file is part of canu, a software program that assembles whole-genome
+ # sequencing reads into contigs.
+ #
+ # This software is based on:
+ # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ # the 'kmer package' (http://kmer.sourceforge.net)
+ # both originally distributed by Applera Corporation under the GNU General
+ # Public License, version 2.
+ #
+ # Canu branched from Celera Assembler at its revision 4587.
+ # Canu branched from the kmer project at its revision 1994.
+ #
+ # Modifications by:
+ #
+ # Brian P. Walenz beginning on 2017-FEB-11
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
+ # File 'README.licenses' in the root directory of this distribution contains
+ # full conditions and disclaimers for each license.
+ ##
+
+package canu::Grid_DNANexus;
+
+require Exporter;
+
+ at ISA = qw(Exporter);
+ at EXPORT = qw(detectDNANexus configureDNANexus);
+
+use strict;
+
+use canu::Defaults;
+use canu::Grid;
+use canu::Execution;
+
+sub detectDNANexus () {
+
+ return if ( defined(getGlobal("gridEngine"))); # Grid not requested.
+ return if (!defined($ENV{'DNA_NEXUS'})); # Not a DNA Nexus grid
+
+ print STDERR "-- Detected DNA Nexus '...some-version...'.\n";
+ setGlobal("gridEngine", "DNANEXUS");
+
+ # DNANexus mode doesn't support (easily) the gatekeeper check on short reads.
+ # The issue is that we'd need to save the store, ask the user to accept it (and rename),
+ # then continue. Nothing super tricky, just not done.
+
+ setGlobal("stopOnReadQuality", 0);
+}
+
+
+sub configureDNANexus () {
+
+ return if (uc(getGlobal("gridEngine")) ne "DNANEXUS");
+
+ my $maxArraySize = 65535;
+
+ # Probe for the maximum array job size
+
+ setGlobalIfUndef("gridEngineSubmitCommand", "");
+ setGlobalIfUndef("gridEngineNameOption", "");
+ setGlobalIfUndef("gridEngineArrayOption", "");
+ setGlobalIfUndef("gridEngineArrayName", "");
+ setGlobalIfUndef("gridEngineArrayMaxJobs", $maxArraySize);
+ setGlobalIfUndef("gridEngineOutputOption", "");
+ setGlobalIfUndef("gridEnginePropagateCommand", "");
+ setGlobalIfUndef("gridEngineThreadsOption", undef);
+ setGlobalIfUndef("gridEngineMemoryOption", undef);
+ setGlobalIfUndef("gridEngineNameToJobIDCommand", undef);
+ setGlobalIfUndef("gridEngineNameToJobIDCommandNoArray", undef);
+ setGlobalIfUndef("gridEngineTaskID", "");
+ setGlobalIfUndef("gridEngineArraySubmitID", "");
+ setGlobalIfUndef("gridEngineJobID", "");
+
+
+ my %hosts;
+
+ # Probe for how to request multiple CPUs on each node, set
+
+ # .
+ # .
+ # .
+
+ # Probe for how to reserve memory on each node
+
+ # .
+ # .
+ # .
+
+ # Build a list of the resources available in the grid. This will contain a list with keys of
+ # "#CPUs-#GBs" and values of the number of nodes With such a config. Later on, we'll use this
+ # to figure out what specific settings to use for each algorithm.
+ #
+ # The list is saved in global{"availableHosts"}
+
+ # .
+ # $hosts{"4-32"} = 15; # 15 machines with 4 CPUs and 32gb memory
+ # .
+
+ setGlobal("availableHosts", formatAllowedResources(%hosts, "DNA Nexus"));
+}
diff --git a/src/pipelines/canu/Grid_LSF.pm b/src/pipelines/canu/Grid_LSF.pm
index b287d7c..22c6f00 100644
--- a/src/pipelines/canu/Grid_LSF.pm
+++ b/src/pipelines/canu/Grid_LSF.pm
@@ -58,9 +58,6 @@ sub configureLSF () {
return if (uc(getGlobal("gridEngine")) ne "LSF");
setGlobalIfUndef("gridEngineSubmitCommand", "bsub");
- setGlobalIfUndef("gridEngineHoldOption", "-w \"ended\(\"WAIT_TAG\"\)\"");
- setGlobalIfUndef("gridEngineHoldOptionNoArray", "-w \"done\(\"WAIT_TAG\"\)\"");
- setGlobalIfUndef("gridEngineSyncOption", "-K");
setGlobalIfUndef("gridEngineNameOption", "-J");
setGlobalIfUndef("gridEngineArrayOption", "");
setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
diff --git a/src/pipelines/canu/Grid_PBSTorque.pm b/src/pipelines/canu/Grid_PBSTorque.pm
index 186526f..50c1621 100644
--- a/src/pipelines/canu/Grid_PBSTorque.pm
+++ b/src/pipelines/canu/Grid_PBSTorque.pm
@@ -158,18 +158,19 @@ sub configurePBSTorque () {
my $isPro = (uc(getGlobal("gridEngine")) eq "PBSPRO");
- setGlobalIfUndef("gridEngineSubmitCommand", "qsub");
- setGlobalIfUndef("gridEngineHoldOption", "-W depend=afteranyarray:WAIT_TAG") if ($isPro == 0);
- setGlobalIfUndef("gridEngineHoldOption", "-W depend=afterany:WAIT_TAG") if ($isPro == 1);
- setGlobalIfUndef("gridEngineHoldOptionNoArray", "-W depend=afterany:WAIT_TAG");
- setGlobalIfUndef("gridEngineSyncOption", "");
- setGlobalIfUndef("gridEngineNameOption", "-d `pwd` -N") if ($isPro == 0);
- setGlobalIfUndef("gridEngineNameOption", "-N") if ($isPro == 1);
+ # PBSPro, again, throws a curve ball at us. There is no way to set the output of array jobs
+ # to someting reasonable like name.TASK_ID.err, even though that is basically the default.
+ # So, we unset gridEngineArraySubmitID to get the default name, but then need to move the '-j oe'
+ # somewhere else - and put it in the submit command.
+
+ setGlobalIfUndef("gridEngineSubmitCommand", "qsub -j oe -d `pwd`") if ($isPro == 0);
+ setGlobalIfUndef("gridEngineSubmitCommand", "qsub -j oe") if ($isPro == 1);
+ setGlobalIfUndef("gridEngineNameOption", "-N");
setGlobalIfUndef("gridEngineArrayOption", "-t ARRAY_JOBS") if ($isPro == 0);
setGlobalIfUndef("gridEngineArrayOption", "-J ARRAY_JOBS") if ($isPro == 1);
setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME");
setGlobalIfUndef("gridEngineArrayMaxJobs", 268435456); # Effectively unlimited.
- setGlobalIfUndef("gridEngineOutputOption", "-j oe -o");
+ setGlobalIfUndef("gridEngineOutputOption", "-o");
setGlobalIfUndef("gridEngineThreadsOption", "-l nodes=1:ppn=THREADS");
setGlobalIfUndef("gridEngineMemoryOption", "-l mem=MEMORY");
setGlobalIfUndef("gridEnginePropagateCommand", "qalter -W depend=afterany:\"WAIT_TAG\"");
@@ -178,7 +179,7 @@ sub configurePBSTorque () {
setGlobalIfUndef("gridEngineTaskID", "PBS_ARRAYID") if ($isPro == 0);
setGlobalIfUndef("gridEngineTaskID", "PBS_ARRAY_INDEX") if ($isPro == 1);
setGlobalIfUndef("gridEngineArraySubmitID", "\\\$PBS_ARRAYID") if ($isPro == 0);
- setGlobalIfUndef("gridEngineArraySubmitID", "\\\$PBS_ARRAY_INDEX") if ($isPro == 1);
+ setGlobalIfUndef("gridEngineArraySubmitID", undef) if ($isPro == 1); # Was "\\\$PBS_ARRAY_INDEX"
setGlobalIfUndef("gridEngineJobID", "PBS_JOBID");
# Build a list of the resources available in the grid. This will contain a list with keys
diff --git a/src/pipelines/canu/Grid_SGE.pm b/src/pipelines/canu/Grid_SGE.pm
index 777f252..cb18c2d 100644
--- a/src/pipelines/canu/Grid_SGE.pm
+++ b/src/pipelines/canu/Grid_SGE.pm
@@ -54,20 +54,21 @@ sub configureSGE () {
return if (uc(getGlobal("gridEngine")) ne "SGE");
- my $maxArraySize = 65535;
+ my $maxArraySize = getGlobal("gridEngineArrayMaxJobs");
- open(F, "qconf -sconf |") or caExit("can't run 'qconf' to get SGE config", undef);
- while (<F>) {
- if (m/max_aj_tasks\s+(\d+)/) {
- $maxArraySize = $1;
+ if (!defined($maxArraySize)) {
+ $maxArraySize = 65535;
+
+ open(F, "qconf -sconf |") or caExit("can't run 'qconf' to get SGE config", undef);
+ while (<F>) {
+ if (m/max_aj_tasks\s+(\d+)/) {
+ $maxArraySize = $1;
+ }
}
+ close(F);
}
- close(F);
setGlobalIfUndef("gridEngineSubmitCommand", "qsub");
- setGlobalIfUndef("gridEngineHoldOption", "-hold_jid \"WAIT_TAG\"");
- setGlobalIfUndef("gridEngineHoldOptionNoArray", undef);
- setGlobalIfUndef("gridEngineSyncOption", "-sync y");
setGlobalIfUndef("gridEngineNameOption", "-cwd -N");
setGlobalIfUndef("gridEngineArrayOption", "-t ARRAY_JOBS");
setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME");
@@ -306,5 +307,20 @@ sub configureSGE () {
}
close(F);
+ if (scalar(keys(%hosts)) == 0) {
+ my $mm = getGlobal("maxMemory");
+ my $mt = getGlobal("maxThreads");
+
+ print STDERR "--\n";
+ print STDERR "-- WARNING: No hosts found in 'qhost' report.\n";
+ print STDERR "-- WARNING: Will use maxMemory=$mm and maxThreads=$mt instead.\n";
+ print STDERR "-- ERROR: maxMemory not defined!\n" if (!defined($mm));
+ print STDERR "-- ERROR: maxThreads not defined!\n" if (!defined($mt));
+
+ caExit("maxMemory or maxThreads not defined", undef) if (!defined($mm) || !defined($mt));
+
+ $hosts{"$mt-$mm"}++;
+ }
+
setGlobal("availableHosts", formatAllowedResources(%hosts, "Sun Grid Engine"));
}
diff --git a/src/pipelines/canu/Grid_Slurm.pm b/src/pipelines/canu/Grid_Slurm.pm
index 6cdb0c9..a0df707 100644
--- a/src/pipelines/canu/Grid_Slurm.pm
+++ b/src/pipelines/canu/Grid_Slurm.pm
@@ -76,9 +76,6 @@ sub configureSlurm () {
close(F);
setGlobalIfUndef("gridEngineSubmitCommand", "sbatch");
- setGlobalIfUndef("gridEngineHoldOption", "--depend=afterany:WAIT_TAG");
- setGlobalIfUndef("gridEngineHoldOptionNoArray", "--depend=afterany:WAIT_TAG");
- setGlobalIfUndef("gridEngineSyncOption", ""); ## TODO: SLURM may not support w/out wrapper; See LSF bsub manpage to compare
setGlobalIfUndef("gridEngineNameOption", "-D `pwd` -J");
setGlobalIfUndef("gridEngineArrayOption", "-a ARRAY_JOBS");
setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME");
diff --git a/src/pipelines/canu/HTML.pm b/src/pipelines/canu/HTML.pm
index 4c309fc..20852cc 100644
--- a/src/pipelines/canu/HTML.pm
+++ b/src/pipelines/canu/HTML.pm
@@ -33,7 +33,7 @@ require Exporter;
use strict;
use File::Copy;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
@@ -93,7 +93,7 @@ sub simpleFigure ($$$$) {
sub buildGatekeeperHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -103,14 +103,14 @@ sub buildGatekeeperHTML ($$$$$$) {
push @$body, "<h2>Input Reads</h2>\n";
push @$body, "\n";
- if (! -e "$wrk/$asm.gkpStore/load.dat") {
+ if (! -e "$base/$asm.gkpStore/load.dat") {
push @$body, "<p>None loaded.</p>\n";
return;
}
push @$body, "<table>\n";
- open(F, "< $wrk/$asm.gkpStore/load.dat") or caExit("can't open '$wrk/$asm.gkpStore/load.dat' for reading: $!", undef);
+ open(F, "< $base/$asm.gkpStore/load.dat") or caExit("can't open '$base/$asm.gkpStore/load.dat' for reading: $!", undef);
while (<F>) {
# nam blocks show up once per file.
@@ -124,7 +124,7 @@ sub buildGatekeeperHTML ($$$$$$) {
push @$scripts, "document.getElementById('gkpload$idx').style = 'cursor: pointer;';\n";
}
- # lib blocks show up once per file, all paramters are on the same line
+ # lib blocks show up once per file, all parameters are on the same line
elsif (m/^lib\s/) {
my @libs = split '\s+', $_;
my ($param, $np, $var, $val);
@@ -183,28 +183,28 @@ sub buildGatekeeperHTML ($$$$$$) {
push @$body, "<h2>Final Store</h2>\n";
push @$body, "\n";
push @$body, "<table>\n";
- push @$body, "<tr><td colspan='2'>$wrk/$asm.gkpStore</td></tr>\n";
+ push @$body, "<tr><td colspan='2'>$base/$asm.gkpStore</td></tr>\n";
push @$body, "<tr><td>readsLoaded</td><td>$nLOADED reads ($bLOADED bp)</td></tr>\n";
push @$body, "<tr><td>readsSkipped</td><td>$nSKIPPED reads ($bSKIPPED bp) (read was too short)</td></tr>\n";
push @$body, "<tr><td>warnings</td><td>$nWARNS warnings (invalid base or quality value)</td></tr>\n";
push @$body, "</table>\n";
} else {
- caExit("failed to read '$wrk/$asm.gkpStore/load.log': invalid format", undef);
+ caExit("failed to read '$base/$asm.gkpStore/load.log': invalid format", undef);
}
}
close(F);
push @$body, "<h3>Read Length Histogram</h3>\n";
simpleFigure($body,
- "$wrk/$asm.gkpStore/readlengths",
- "$wrk.html.files/readlengths",
+ "$base/$asm.gkpStore/readlengths",
+ "$base.html.files/readlengths",
"");
}
sub buildMerylHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -214,14 +214,14 @@ sub buildMerylHTML ($$$$$$) {
push @$body, "<h2>k-Mer Counts</h2>\n";
push @$body, "\n";
- if (! -d "$wrk/0-mercounts") {
- push @$body, "<p>Stage not computed. ($wrk/0-mercounts)</p>\n";
+ if (! -d "$base/0-mercounts") {
+ push @$body, "<p>Stage not computed. ($base/0-mercounts)</p>\n";
return;
}
my %merSizes;
- open(F, "ls $wrk/0-mercounts/ |") or caExit("can't find files in '$wrk/0-mercounts': $!", undef);
+ open(F, "ls $base/0-mercounts/ |") or caExit("can't find files in '$base/0-mercounts': $!", undef);
while (<F>) {
if (m/\.ms(\d+)\./) {
$merSizes{$1}++;
@@ -235,8 +235,8 @@ sub buildMerylHTML ($$$$$$) {
my $numUnique = 0;
my $largest = 0;
- if (-e "$wrk/0-mercounts/$asm.ms$ms.histogram.info") {
- open(F, "< $wrk/0-mercounts/$asm.ms$ms.histogram.info") or caExit("can't open '$wrk/0-mercounts/$asm.ms$ms.histogram.info' for reading: $!", undef);
+ if (-e "$base/0-mercounts/$asm.ms$ms.histogram.info") {
+ open(F, "< $base/0-mercounts/$asm.ms$ms.histogram.info") or caExit("can't open '$base/0-mercounts/$asm.ms$ms.histogram.info' for reading: $!", undef);
while (<F>) {
$numTotal = $1 if (m/Found\s(\d+)\s+mers./);
$numDistinct = $1 if (m/Found\s(\d+)\s+distinct\smers./);
@@ -246,16 +246,16 @@ sub buildMerylHTML ($$$$$$) {
close(F);
simpleFigure($body,
- "$wrk/0-mercounts/$asm.ms$ms.histogram",
- "$wrk.html.files/$asm.ms$ms.histogram",
+ "$base/0-mercounts/$asm.ms$ms.histogram",
+ "$base.html.files/$asm.ms$ms.histogram",
"Histogram for k=$ms with $numTotal mers, $numDistinct distinct mers and $numUnique single-copy mers. Largest count is $largest.");
}
- elsif ((-e "$wrk/0-mercounts/$asm.ms$ms.ignore") && (-z "$wrk/0-mercounts/$asm.ms$ms.ignore")) {
+ elsif ((-e "$base/0-mercounts/$asm.ms$ms.ignore") && (-z "$base/0-mercounts/$asm.ms$ms.ignore")) {
push @$body, "Threshold zero. No mers reported.\n";
}
- elsif ((-e "$wrk/0-mercounts/$asm.ms$ms.fasta") && (-z "$wrk/0-mercounts/$asm.ms$ms.fasta")) {
+ elsif ((-e "$base/0-mercounts/$asm.ms$ms.fasta") && (-z "$base/0-mercounts/$asm.ms$ms.fasta")) {
push @$body, "Threshold zero. No mers reported.\n";
}
@@ -268,7 +268,7 @@ sub buildMerylHTML ($$$$$$) {
sub buildCorrectionHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -284,12 +284,12 @@ sub buildCorrectionHTML ($$$$$$) {
push @$body, "<h2>Overlap Filtering</h2>\n";
push @$body, "\n";
- if (-e "$wrk/2-correction/$asm.globalScores.stats") {
+ if (-e "$base/2-correction/$asm.globalScores.stats") {
my $rh; # 'row header', for labeling a set of rows with a common cell
push @$body, "<table>\n";
- open(F, "< $wrk/2-correction/$asm.globalScores.stats") or caExit("can't open '$wrk/2-correction/$asm.globalScores.stats' for reading: $!", undef);
+ open(F, "< $base/2-correction/$asm.globalScores.stats") or caExit("can't open '$base/2-correction/$asm.globalScores.stats' for reading: $!", undef);
while (<F>) {
chomp;
@@ -318,7 +318,7 @@ sub buildCorrectionHTML ($$$$$$) {
push @$body, "</table>\n";
} else {
- push @$body, "<p>Stage not computed or results file removed ($wrk/2-correction/$asm.globalScores.stats).</p>\n";
+ push @$body, "<p>Stage not computed or results file removed ($base/2-correction/$asm.globalScores.stats).</p>\n";
}
@@ -333,8 +333,8 @@ sub buildCorrectionHTML ($$$$$$) {
my $nBasesIn = undef;
my $nBasesOut = undef;
- if (-e "$wrk/2-correction/$asm.readsToCorrect.summary") {
- open(F, "< $wrk/2-correction/$asm.readsToCorrect.summary") or caExit("can't open '$wrk/2-correction/$asm.readsToCorrect.summary' for reading: $!", undef);
+ if (-e "$base/2-correction/$asm.readsToCorrect.summary") {
+ open(F, "< $base/2-correction/$asm.readsToCorrect.summary") or caExit("can't open '$base/2-correction/$asm.readsToCorrect.summary' for reading: $!", undef);
while (<F>) {
$nReads = $1 if ((m/nReads\s+(\d+)/) && (!defined($nReads)));
$nBasesIn = $1 if ((m/nBasds\s+(\d+).*input/) && (!defined($nBasesIn)));
@@ -350,17 +350,17 @@ sub buildCorrectionHTML ($$$$$$) {
}
- # $wrk/2-correction/$asm.readsToCorrect has 'readID', 'originalLength' and 'expectedCorrectedLength'.
- # $WRK/$asm.correctedReads.length has 'readID', 'pieceID', 'length'.
+ # $base/2-correction/$asm.readsToCorrect has 'readID', 'originalLength' and 'expectedCorrectedLength'.
+ # $BASE/$asm.correctedReads.length has 'readID', 'pieceID', 'length'.
#
# Both files should be sorted by increasing ID, so a simple merge sufficies.
- if (-e "$wrk/2-correction/$asm.correction.summary") {
+ if (-e "$base/2-correction/$asm.correction.summary") {
my $rh;
push @$body, "<table>\n";
- open(F, "< $wrk/2-correction/$asm.correction.summary") or caExit("can't open '$wrk/2-correction/$asm.correction.summary' for reading: $!", undef);
+ open(F, "< $base/2-correction/$asm.correction.summary") or caExit("can't open '$base/2-correction/$asm.correction.summary' for reading: $!", undef);
while (<F>) {
chomp;
@@ -393,8 +393,8 @@ sub buildCorrectionHTML ($$$$$$) {
# Simple vs Expensive filter true/false positive
simpleFigure($body,
- "$wrk/2-correction/$asm.estimate.original-x-corrected",
- "$wrk.html.files/$asm.estimate.original-x-corrected",
+ "$base/2-correction/$asm.estimate.original-x-corrected",
+ "$base.html.files/$asm.estimate.original-x-corrected",
"Scatter plot of the original read length (X axis) against the expected corrected read length (Y axis).\n" .
"Colors show a comparison of the simple filter (which doesn't use overlaps) to the expensive filter (which does).\n" .
"A large green triangle (false negatives) hints that there could be abnormally low quality regions in the reads.\n");
@@ -403,33 +403,33 @@ sub buildCorrectionHTML ($$$$$$) {
# Original vs expected shown above.
simpleFigure($body,
- "$wrk/2-correction/$asm.originalLength-vs-expectedLength",
- "$wrk.html.files/$asm.originalLength-vs-expectedLength",
+ "$base/2-correction/$asm.originalLength-vs-expectedLength",
+ "$base.html.files/$asm.originalLength-vs-expectedLength",
"Scatter plot of original vs expected read length. Shown in filter plot above.");
simpleFigure($body,
- "$wrk/2-correction/$asm.originalLength-vs-correctedLength",
- "$wrk.html.files/$asm.originalLength-vs-correctedLength",
+ "$base/2-correction/$asm.originalLength-vs-correctedLength",
+ "$base.html.files/$asm.originalLength-vs-correctedLength",
"Scatter plot of original vs corrected read length.");
simpleFigure($body,
- "$wrk/2-correction/$asm.expectedLength-vs-correctedLength",
- "$wrk.html.files/$asm.expectedLength-vs-correctedLength",
+ "$base/2-correction/$asm.expectedLength-vs-correctedLength",
+ "$base.html.files/$asm.expectedLength-vs-correctedLength",
"Scatter plot of expected vs corrected read length.");
# Histogram - expected vs corrected lengths NEEDS TO SHOW NEGATIVES!?
simpleFigure($body,
- "$wrk/2-correction/$asm.length-difference-histograms",
- "$wrk.html.files/$asm.length-difference-histograms",
+ "$base/2-correction/$asm.length-difference-histograms",
+ "$base.html.files/$asm.length-difference-histograms",
"Histogram of the difference between the expected and corrected read lengths.\n" .
"Note that a negative difference means the corrected read is larger than expected.\n");
# Histogram - original, expected, corrected lengths
simpleFigure($body,
- "$wrk/2-correction/$asm.length-histograms",
- "$wrk.html.files/$asm.length-histograms",
+ "$base/2-correction/$asm.length-histograms",
+ "$base.html.files/$asm.length-histograms",
"Histogram of original (red), expected (green) and actual corrected (blue) read lengths.\n");
}
@@ -437,7 +437,7 @@ sub buildCorrectionHTML ($$$$$$) {
sub buildTrimmingHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -448,10 +448,10 @@ sub buildTrimmingHTML ($$$$$$) {
push @$body, "\n";
- if (-e "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats") {
+ if (-e "$base/3-overlapbasedtrimming/$asm.1.trimReads.stats") {
my $rh; # 'row header', for labeling a set of rows with a common cell
- # Read once to make a paramters table. We could have embedded this in the loop below, but it's cleaner here.
+ # Read once to make a parameters table. We could have embedded this in the loop below, but it's cleaner here.
#push @$body, "<table>\n";
#push @$body, "</table>\n";
@@ -460,7 +460,7 @@ sub buildTrimmingHTML ($$$$$$) {
push @$body, "<table>\n";
- open(F, "< $wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats") or caExit("can't open '$wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats' for reading: $!", undef);
+ open(F, "< $base/3-overlapbasedtrimming/$asm.1.trimReads.stats") or caExit("can't open '$base/3-overlapbasedtrimming/$asm.1.trimReads.stats' for reading: $!", undef);
while (<F>) {
chomp;
@@ -496,26 +496,26 @@ sub buildTrimmingHTML ($$$$$$) {
push @$body, "</table>\n";
} else {
- push @$body, "<p>Stage not computed or results file removed ($wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats).</p>\n";
+ push @$body, "<p>Stage not computed or results file removed ($base/3-overlapbasedtrimming/$asm.1.trimReads.stats).</p>\n";
}
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.inputDeletedReads", "$wrk.html.files/$asm.1.trimReads.inputDeletedReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.inputNoTrimReads", "$wrk.html.files/$asm.1.trimReads.inputNoTrimReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.inputReads", "$wrk.html.files/$asm.1.trimReads.inputReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputDeletedReads", "$wrk.html.files/$asm.1.trimReads.outputDeletedReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputNoOvlReads", "$wrk.html.files/$asm.1.trimReads.outputNoOvlReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputTrimmedReads", "$wrk.html.files/$asm.1.trimReads.outputTrimmedReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputUnchangedReads", "$wrk.html.files/$asm.1.trimReads.outputUnchangedReads", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.trim3", "$wrk.html.files/$asm.1.trimReads.trim3", "");
- simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.trim5", "$wrk.html.files/$asm.1.trimReads.trim5", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.inputDeletedReads", "$base.html.files/$asm.1.trimReads.inputDeletedReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.inputNoTrimReads", "$base.html.files/$asm.1.trimReads.inputNoTrimReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.inputReads", "$base.html.files/$asm.1.trimReads.inputReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.outputDeletedReads", "$base.html.files/$asm.1.trimReads.outputDeletedReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.outputNoOvlReads", "$base.html.files/$asm.1.trimReads.outputNoOvlReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.outputTrimmedReads", "$base.html.files/$asm.1.trimReads.outputTrimmedReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.outputUnchangedReads", "$base.html.files/$asm.1.trimReads.outputUnchangedReads", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.trim3", "$base.html.files/$asm.1.trimReads.trim3", "");
+ simpleFigure($body, "$base/3-overlapbasedtrimming/$asm.1.trimReads.trim5", "$base.html.files/$asm.1.trimReads.trim5", "");
push @$body, "<h2>Splitting</h2>\n";
push @$body, "\n";
- if (-e "$wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats") {
+ if (-e "$base/3-overlapbasedtrimming/$asm.2.splitReads.stats") {
my $rh; # 'row header', for labeling a set of rows with a common cell
- # Read once to make a paramters table. We could have embedded this in the loop below, but it's cleaner here.
+ # Read once to make a parameters table. We could have embedded this in the loop below, but it's cleaner here.
#push @$body, "<table>\n";
#push @$body, "</table>\n";
@@ -524,7 +524,7 @@ sub buildTrimmingHTML ($$$$$$) {
push @$body, "<table>\n";
- open(F, "< $wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats") or caExit("can't open '$wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats' for reading: $!", undef);
+ open(F, "< $base/3-overlapbasedtrimming/$asm.2.splitReads.stats") or caExit("can't open '$base/3-overlapbasedtrimming/$asm.2.splitReads.stats' for reading: $!", undef);
while (<F>) {
chomp;
@@ -570,11 +570,11 @@ sub buildTrimmingHTML ($$$$$$) {
push @$body, "</table>\n";
} else {
- push @$body, "<p>Stage not computed or results file removed ($wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats).</p>\n";
+ push @$body, "<p>Stage not computed or results file removed ($base/3-overlapbasedtrimming/$asm.2.splitReads.stats).</p>\n";
}
- #buildGatekeeperHTML($wrk, $asm, $tag, $css, $body, $scripts);
+ #buildGatekeeperHTML($base, $asm, $tag, $css, $body, $scripts);
# Analyzes the output fastq
}
@@ -582,7 +582,7 @@ sub buildTrimmingHTML ($$$$$$) {
sub buildOverlapperHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -592,13 +592,13 @@ sub buildOverlapperHTML ($$$$$$) {
push @$body, "<h2>Overlaps</h2>\n";
push @$body, "\n";
- if (! -d "$wrk/$asm.ovlStore") {
+ if (! -d "$base/$asm.ovlStore") {
push @$body, "<p>Overlaps not computed.</p>\n";
return;
}
- if (! -e "$wrk/$asm.ovlStore.summary") {
- push @$body, "<p>No statistics available for store '$wrk/$asm.ovlStore'.</p>\n";
+ if (! -e "$base/$asm.ovlStore.summary") {
+ push @$body, "<p>No statistics available for store '$base/$asm.ovlStore'.</p>\n";
return;
}
@@ -607,7 +607,7 @@ sub buildOverlapperHTML ($$$$$$) {
my ($category, $reads, $readsP, $length, $lengthsd, $size, $sizesd, $analysis);
- open(F, "< $wrk/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$wrk/$asm.ovlStore': $!", undef);
+ open(F, "< $base/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$base/$asm.ovlStore': $!", undef);
$_ = <F>;
$_ = <F>;
while (<F>) {
@@ -639,7 +639,7 @@ sub buildOverlapperHTML ($$$$$$) {
} else {
chomp;
- caExit("failed to parse line '$_' in file '$wrk/$asm.ovlStore.summary'", undef);
+ caExit("failed to parse line '$_' in file '$base/$asm.ovlStore.summary'", undef);
}
}
close(F);
@@ -649,7 +649,7 @@ sub buildOverlapperHTML ($$$$$$) {
sub buildOverlapErrorCorrectionHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -698,37 +698,37 @@ sub reportSizeStatistics ($$$) {
sub buildUnitiggerHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
my $body = shift @_; # Array reference
my $scripts = shift @_; # Array reference
- return if (! -d "$wrk/4-unitigger");
+ return if (! -d "$base/4-unitigger");
my @logs;
- push @logs, "$wrk/4-unitigger/unitigger.err";
+ push @logs, "$base/4-unitigger/unitigger.err";
- open(F, "ls $wrk/4-unitigger |");
+ open(F, "ls $base/4-unitigger |");
while (<F>) {
chomp;
- push @logs, "$wrk/4-unitigger/$_" if (m/log$/);
+ push @logs, "$base/4-unitigger/$_" if (m/log$/);
}
close(F);
push @$body, "<h2>Unitigs</h2>\n";
push @$body, "\n";
- if (-e "$wrk/4-unitigger/unitigger.err") {
+ if (-e "$base/4-unitigger/unitigger.err") {
my $all = 0;
my $some = 0;
my $someL = 0;
my $olaps = 0;
- open(F, "< $wrk/4-unitigger/unitigger.err");
+ open(F, "< $base/4-unitigger/unitigger.err");
while (<F>) {
chomp;
@@ -753,7 +753,7 @@ sub buildUnitiggerHTML ($$$$$$) {
push @$body, "Loaded $olaps overlaps in total.<br>\n";
}
- if (-e "$wrk/4-unitigger/$asm.001.filterOverlaps.thr000.num000.log") {
+ if (-e "$base/4-unitigger/$asm.001.filterOverlaps.thr000.num000.log") {
push @$body, "<h3>Edges</h3>\n";
push @$body, "\n";
@@ -785,7 +785,7 @@ sub buildUnitiggerHTML ($$$$$$) {
my $finalBest1Mutual = 0;
my $finalBest2Mutual = 0;
- open(F, "$wrk/4-unitigger/$asm.001.filterOverlaps.thr000.num000.log");
+ open(F, "$base/4-unitigger/$asm.001.filterOverlaps.thr000.num000.log");
$_ = <F>; chomp;
my $block = "none";
@@ -862,16 +862,16 @@ sub buildUnitiggerHTML ($$$$$$) {
push @$body, "<h3>Initial Tig Sizes</h3>\n";
- if (-e "$wrk/4-unitigger/$asm.003.buildUnitigs.sizes") {
- open(F, "< $wrk/4-unitigger/$asm.003.buildUnitigs.sizes");
+ if (-e "$base/4-unitigger/$asm.003.buildUnitigs.sizes") {
+ open(F, "< $base/4-unitigger/$asm.003.buildUnitigs.sizes");
reportSizeStatistics($css, $body, $scripts);
close(F);
}
push @$body, "<h3>Final Tig Sizes</h3>\n";
- if (-e "$wrk/4-unitigger/$asm.008.generateOutputs.sizes") {
- open(F, "< $wrk/4-unitigger/$asm.008.generateOutputs.sizes");
+ if (-e "$base/4-unitigger/$asm.008.generateOutputs.sizes") {
+ open(F, "< $base/4-unitigger/$asm.008.generateOutputs.sizes");
reportSizeStatistics($css, $body, $scripts);
close(F);
}
@@ -881,7 +881,7 @@ sub buildUnitiggerHTML ($$$$$$) {
sub buildConsensusHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -894,7 +894,7 @@ sub buildConsensusHTML ($$$$$$) {
sub buildOutputHTML ($$$$$$) {
- my $wrk = shift @_;
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
my $css = shift @_; # Array reference
@@ -906,62 +906,62 @@ sub buildOutputHTML ($$$$$$) {
}
-sub buildHTML ($$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub buildHTML ($$) {
my $asm = shift @_;
my $tag = shift @_;
my @css;
my @body;
my @scripts;
- $wrk = "$WRK/correction" if ($tag eq "cor");
- $wrk = "$WRK/trimming" if ($tag eq "obt");
- $wrk = "$WRK/unitigging" if ($tag eq "utg");
+ my $base;
- make_path("$wrk.html.files") if (! -e "$wrk.html.files");
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ make_path("$base.html.files") if (! -e "$base.html.files");
# For correction runs
if ($tag eq "cor") {
push @body, "<h1>Correction</h1>\n";
- buildGatekeeperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildMerylHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildOverlapperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildCorrectionHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
+ buildGatekeeperHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildMerylHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildOverlapperHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildCorrectionHTML($base, $asm, $tag, \@css, \@body, \@scripts);
}
# For trimming runs
if ($tag eq "obt") {
push @body, "<h1>Trimming</h1>\n";
- buildGatekeeperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildMerylHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildOverlapperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildTrimmingHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
+ buildGatekeeperHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildMerylHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildOverlapperHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildTrimmingHTML($base, $asm, $tag, \@css, \@body, \@scripts);
}
# For assembly runs
if ($tag eq "utg") {
push @body, "<h1>Assembly</h1>\n";
- buildGatekeeperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildMerylHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildOverlapperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildOverlapErrorCorrectionHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildUnitiggerHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildConsensusHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
- buildOutputHTML($wrk, $asm, $tag, \@css, \@body, \@scripts);
+ buildGatekeeperHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildMerylHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildOverlapperHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildOverlapErrorCorrectionHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildUnitiggerHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildConsensusHTML($base, $asm, $tag, \@css, \@body, \@scripts);
+ buildOutputHTML($base, $asm, $tag, \@css, \@body, \@scripts);
}
- #print STDERR "WRITING '$wrk/$asm-summary.html'\n";
+ #print STDERR "WRITING '$base/$asm-summary.html'\n";
- open(F, "> $wrk.html") or die "can't open '$wrk.html' for writing: $!\n";
+ open(F, "> $base.html") or die "can't open '$base.html' for writing: $!\n";
print F "<!DOCTYPE html>\n";
print F "\n";
print F "<html>\n";
print F "\n";
print F "<head>\n";
- print F "<title>canu analysis for assembly '$asm' in directory '$wrk'</title>\n";
+ print F "<title>canu analysis for assembly '$asm' in directory '$base'</title>\n";
print F "<style type='text/css'>\n";
print F "body { font-family: Helvetica, Verdana, sans-serif; }\n";
print F "h1, h2, h3 { color: #ee3e80; }\n";
diff --git a/src/pipelines/canu/Meryl.pm b/src/pipelines/canu/Meryl.pm
index 6b5c98e..c6068ea 100644
--- a/src/pipelines/canu/Meryl.pm
+++ b/src/pipelines/canu/Meryl.pm
@@ -40,81 +40,203 @@ package canu::Meryl;
require Exporter;
@ISA = qw(Exporter);
- at EXPORT = qw(getGenomeCoverage merylConfigure merylCheck merylProcess);
+ at EXPORT = qw(merylConfigure merylCheck merylProcess);
use strict;
-use File::Path qw(make_path);
+use File::Path 2.08 qw(make_path remove_tree);
+use File::Basename;
+use POSIX qw(ceil);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
use canu::ErrorEstimate;
use canu::HTML;
+use canu::Report;
+use canu::Grid_Cloud;
-sub getGenomeCoverage($$$) {
- my $wrk = shift @_;
+
+
+sub merylGenerateHistogram ($$) {
my $asm = shift @_;
- my $merSize = shift @_;
- my $bin = getBinDirectory();
+ my $tag = shift @_;
+ my $hist;
- my $gs=`cat $wrk/0-mercounts/$asm.ms$merSize.estMerThresh.err | grep "Guessed X coverage"|awk '{print \$NF}'`;
- chomp $gs;
+ # We don't know $ofile from where merylGenerateHistogram is typically called (Report.pm)
+ # and so we're forced to configure every time.
- # if we couldn't find the coverage, just take # bases divided by user supplied genome size as a guestimate
- if ($gs == "") {
- open(F, "$bin/gatekeeperDumpMetaData -stats -G $wrk/$asm.gkpStore | ") or caFailure("failed to read gatekeeper stats fromfrom '$wrk/$asm.gkpStore'", undef);
- while (<F>) {
- my ($junk1, $library, $junk2, $reads, $junk3, $junk4, $bases, $junk5, $average, $junk6, $min, $junk7, $max) = split '\s+', $_;
- if ($library == 0) {
- $gs = $bases / getGlobal("genomeSize");
- last;
- }
+ my ($base, $path, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($asm, $tag);
+
+ return(undef) if (! -e "$path/$ofile.histogram");
+ return(undef) if (! -e "$path/$ofile.histogram.info");
+
+ # Load the statistics
+
+ my $numTotal = 0;
+ my $numDistinct = 0;
+ my $numUnique = 0;
+ my $largest = 0;
+
+ open(F, "< $path/$ofile.histogram.info") or caFailure("can't open meryl histogram information file '$path/$ofile.histogram.info' for reading: $!\n", undef);
+ while (<F>) {
+ $numTotal = $1 if (m/Found\s(\d+)\s+mers./);
+ $numDistinct = $1 if (m/Found\s(\d+)\s+distinct\smers./);
+ $numUnique = $1 if (m/Found\s(\d+)\s+unique\smers./);
+ $largest = $1 if (m/Largest\smercount\sis\s(\d+)/);
+ }
+ close(F);
+
+ # Load histogram data
+
+ my @tc; # Total count
+ my @fu; # Fraction unique
+ my @ft; # Fraction total
+ my $mc;
+
+ open(F, "< $path/$ofile.histogram");
+ while (<F>) {
+ my @v = split '\s+', $_;
+ $tc[$v[0]] = $v[1];
+ $fu[$v[0]] = $v[2];
+ $ft[$v[0]] = $v[3];
+ $mc = $v[0]; # histogram should be sorted
+ }
+ close(F);
+
+ # Prune the high-count kmers
+ #
+ # In blocks of 40, extend the histogram until the average of the next block is nearly the same
+ # as the average of this block.
+if (0) {
+ my $lo = 2;
+ my $hi = 3;
+ my $st = 1;
+ my $aveLast = 0;
+ my $aveThis = 0;
+
+ for (my $ii=$lo; $ii<$hi; $ii++) {
+ $aveThis += $tc[$ii];
+ }
+ $aveThis /= ($hi - $lo);
+ $aveLast = 0;
+
+ print STDERR "aveLast $aveLast aveThis $aveThis $lo $hi INITIAL\n";
+
+ while (($hi < $mc) &&
+ ($aveThis > 2) &&
+ (($aveThis < 0.90 * $aveLast) ||
+ ($aveLast < 0.90 * $aveThis))) {
+ $lo += $st;
+ $hi += $st;
+ $st += 1;
+
+ $aveLast = $aveThis;
+ $aveThis = 0;
+
+ for (my $ii=$lo; $ii<$hi; $ii++) {
+ $aveThis += $tc[$ii];
}
- close(F);
+ $aveThis /= ($hi - $lo);
+ print STDERR "aveLast $aveLast aveThis $aveThis $lo $hi\n";
}
- return $gs;
+ print STDERR "aveLast $aveLast aveThis $aveThis $lo $hi FINAL\n";
}
+ my @TC;
+ my @FU;
+ my @FT;
+
+ my $TCmax = 0;
+ my $lo = 1;
+ my $hi = 2;
+ my $st = 1;
+ for (my $ii=0; $ii <= 40; $ii++) {
+ for (my $jj=$lo; $jj < $hi; $jj++) {
+ $TC[$ii] += $tc[$jj]; # Sum the counts
+ $FU[$ii] = ($fu[$ii] < $FU[$ii]) ? $FU[$ii] : $fu[$jj]; # But the fractions are already cumulative,
+ $FT[$ii] = ($ft[$ii] < $FT[$ii]) ? $FT[$ii] : $ft[$jj]; # we just need to skip zeros.
+ }
-# Generates
-# meryl mcdat/mcidx
-# mer histogram file
-# mer histogram plots
-# mer threshold
-# frequent mers
-#
+ if ($ii > 0) {
+ $TCmax = ($TCmax < $TC[$ii]) ? $TC[$ii] : $TCmax;
+ }
-# Threshold: Three methods to pick it.
-# Threshold - 'auto', 'auto * X', 'auto / X', or an integer value
-# Distinct - by the fraction distinct retained
-# Total - by the fraction total retained
-#
+ $lo = $hi;
+ $hi += $st;
+ $st += 1;
+ }
+
+ my $maxY = $lo;
+ my $Xscale = $TCmax / 70;
-# We always compute canonical mers, that are not compressed.
+ # Now just draw the histogram
+ $hist .= "--\n";
+ $hist .= "-- $merSize-mers Fraction\n";
+ $hist .= "-- Occurrences NumMers Unique Total\n";
-# Generates $wrk/0-mercounts/$asm.ms$merSize.frequentMers.fasta
-# stopBefore meryl (stops before meryl itself runs)
-# stopAfter meryl (stops after output is generated, even if it is just a symlink)
+ $lo = 1;
+ $hi = 2;
+ $st = 1;
+ for (my $ii=0; $ii<=40; $ii++) {
+ my $numXs = int($TC[$ii] / $Xscale);
-sub plotHistogram ($$$$) {
- my $wrk = shift @_; # Local work directory
+ if ($numXs <= 70) {
+ $hist .= sprintf("-- %6d-%6d %9d %s%s %.4f %.4f\n",
+ $lo, $hi-1, $TC[$ii],
+ "*" x ($numXs),
+ " " x (70 - $numXs), $FU[$ii], $FT[$ii]);
+ } else {
+ $hist .= sprintf("-- %6d-%6d %9d %s%s %.4f %.4f\n",
+ $lo, $hi-1, $TC[$ii],
+ "*" x 67,
+ "-->", $FU[$ii], $FT[$ii]);
+ }
+
+ last if ($hi >= $maxY);
+
+ $lo = $hi;
+ $hi += $st;
+ $st += 1;
+ }
+
+ $hist .= sprintf("--\n");
+ $hist .= sprintf("-- %11d (max occurrences)\n", $largest);
+ $hist .= sprintf("-- %11d (total mers, non-unique)\n", $numTotal - $numUnique);
+ $hist .= sprintf("-- %11d (distinct mers, non-unique)\n", $numDistinct - $numUnique);
+ $hist .= sprintf("-- %11d (unique mers)\n", $numUnique);
+
+ return($hist);
+}
+
+
+
+
+# Threshold: Three methods to pick it.
+# Threshold - 'auto', 'auto * X', 'auto / X', or an integer value
+# Distinct - by the fraction distinct retained
+# Total - by the fraction total retained
+
+sub merylPlotHistogram ($$$$) {
+ my $path = shift @_;
my $ofile = shift @_;
my $suffix = shift @_;
- my $size = shift @_;
+ my $size = shift @_; # Size of image, not merSize!
- return if (-e "$ofile.histogram.$suffix.gp");
+ return if (fileExists("$path/$ofile.histogram.$suffix.gp"));
my $gnuplot = getGlobal("gnuplot");
my $format = getGlobal("gnuplotImageFormat");
- open(F, "> $ofile.histogram.$suffix.gp");
+ fetchFile("$path/$ofile.histogram");
+
+ open(F, "> $path/$ofile.histogram.$suffix.gp");
print F "\n";
print F "unset multiplot\n";
print F "\n";
@@ -166,29 +288,32 @@ sub plotHistogram ($$$$) {
print F "plot [0:200] '$ofile.histogram' using 1:2 with lines title 'Histogram'\n";
close(F);
- if (runCommandSilently("$wrk/0-mercounts", "$gnuplot $ofile.histogram.$suffix.gp > /dev/null 2>&1", 0)) {
+ if (runCommandSilently($path, "$gnuplot ./$ofile.histogram.$suffix.gp > /dev/null 2>&1", 0)) {
print STDERR "--\n";
print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n";
print STDERR "--\n";
print STDERR "----------------------------------------\n";
}
+
+ stashFile("$path/$ofile.histogram.$suffix.gp");
+ stashFile("$path/$ofile.histogram.$suffix.$format");
}
-sub merylParameters ($$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- ## $wrk = $WRK; # Local work directory
+sub merylParameters ($$) {
my $asm = shift @_;
my $tag = shift @_;
- my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile);
+ my ($base, $path, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile);
# Find a place to run stuff.
- $wrk = "$WRK/correction" if ($tag eq "cor");
- $wrk = "$WRK/trimming" if ($tag eq "obt");
- $wrk = "$WRK/unitigging" if ($tag eq "utg");
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ $path = "$base/0-mercounts";
# Decide on which set of parameters we need to be using, and make output file names.
@@ -199,8 +324,8 @@ sub merylParameters ($$$) {
$merDistinct = getGlobal("${tag}OvlMerDistinct");
$merTotal = getGlobal("${tag}OvlMerTotal");
- $ffile = "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.fasta"; # The fasta file we should be creating (ends in FASTA).
- $ofile = "$wrk/0-mercounts/$asm.ms$merSize"; # The meryl database 'intermediate file'.
+ $ffile = "$asm.ms$merSize.frequentMers.fasta"; # The fasta file we should be creating (ends in FASTA).
+ $ofile = "$asm.ms$merSize"; # The meryl database 'intermediate file'.
} elsif (getGlobal("${tag}Overlapper") eq "mhap") {
$merSize = getGlobal("${tag}mhapMerSize");
@@ -209,14 +334,19 @@ sub merylParameters ($$$) {
$merDistinct = undef;
$merTotal = undef;
- $ffile = "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz"; # The mhap-specific file we should be creating (ends in IGNORE).
- $ofile = "$wrk/0-mercounts/$asm.ms$merSize"; # The meryl database 'intermediate file'.
+ $ffile = "$asm.ms$merSize.frequentMers.ignore.gz"; # The mhap-specific file we should be creating (ends in IGNORE).
+ $ofile = "$asm.ms$merSize"; # The meryl database 'intermediate file'.
} elsif (getGlobal("${tag}Overlapper") eq "minimap") {
- # do nothing
- $ffile = "$wrk/0-mercounts/$asm.ms$merSize.skip";
- make_path("$wrk/0-mercounts") if (! -d "$wrk/0-mercounts");
- touch($ffile);
+ $merSize = 0;
+ $merThresh = 0;
+ $merScale = 1.0;
+ $merDistinct = undef;
+ $merTotal = undef;
+
+ $ffile = undef;
+ $ofile = undef;
+
} else {
caFailure("unknown ${tag}Overlapper '" . getGlobal("${tag}Overlapper") . "'", undef);
}
@@ -235,222 +365,226 @@ sub merylParameters ($$$) {
# Return all this goodness.
- #print STDERR "wrk - '$wrk'\n";
- #print STDERR "merSize - '$merSize'\n";
- #print STDERR "merThresh - '$merThresh'\n";
- #print STDERR "merScale - '$merScale'\n";
- #print STDERR "merDistinct - '$merDistinct'\n";
- #print STDERR "merTotal - '$merTotal'\n";
- #print STDERR "ffile - '$ffile'\n";
- #print STDERR "ofile - '$ofile'\n";
-
- return($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile);
+ return($base, $path, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile);
}
-sub merylConfigure ($$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- ## $wrk = $WRK; # Local work directory
+sub merylConfigure ($$) {
my $asm = shift @_;
my $tag = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($WRK, $asm, $tag);
+ my ($base, $path, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($asm, $tag);
- goto allDone if (skipStage($WRK, $asm, "$tag-merylConfigure") == 1);
- goto allDone if (-e "$ffile");
- goto allDone if ((-e "$ofile.mcidx") && (-e "$ofile.mcdat"));
+ goto allDone if (skipStage($asm, "$tag-merylConfigure") == 1);
+ goto allDone if (fileExists("$path/meryl.sh"));
+ goto allDone if (!defined($ffile));
+ goto allDone if (fileExists("$path/$ffile"));
+ goto allDone if (fileExists("$path/$ofile.mcidx") && fileExists("$path/$ofile.mcdat"));
- make_path("$wrk/0-mercounts") if (! -d "$wrk/0-mercounts");
+ make_path($path) if (! -d $path);
- # User supplied mers? Just symlink to them.
+ # User supplied mers? Copy them to the proper location and exit.
- if (defined(getGlobal("${tag}OvlFrequentMers"))) {
- #my $ffile = "$wrk/0-mercounts/$asm.frequentMers.fasta";
- my $sfile = getGlobal("${tag}OvlFrequentMers");
+ my $sfile = getGlobal("${tag}OvlFrequentMers");
- if (! -e $ffile) {
- caFailure("${tag}OvlFrequentMers '$sfile' not found", undef) if (! -e $sfile);
- symlink $sfile, $ffile;
- }
+ if (defined($sfile) && ! -e "$path/$ffile") {
+ caFailure("${tag}OvlFrequentMers '$sfile' not found", undef) if (! -e $sfile);
+ copy($sfile, "$path/$ffile");
+ stashFile("$path/$ffile");
+ goto allDone;
+ }
+
+ # No filtering? Make an empty file and exit.
+ if ((defined($merThresh)) &&
+ ($merThresh ne "auto") &&
+ ($merThresh == 0) &&
+ (!defined($merDistinct)) &&
+ (!defined($merTotal))) {
+ touch("$path/$ffile");
+ stashFile("$path/$ffile");
goto allDone;
}
- # Nope, build a script.
+ # Nope, build a script for computing kmer counts.
my $mem = int(getGlobal("merylMemory") * 1024 * 0.8); # Because meryl expects megabytes, not gigabytes.
my $thr = getGlobal("merylThreads");
+ my $cov = getExpectedCoverage($base, $asm);
caExit("merylMemory isn't defined?", undef) if (!defined($mem));
caExit("merylThreads isn't defined?", undef) if (!defined($thr));
- open(F, "> $wrk/0-mercounts/meryl.sh") or caExit("can't open '$wrk/0-mercounts/meryl.sh' for writing: $1", undef);
+ open(F, "> $path/meryl.sh") or caExit("can't open '$path/meryl.sh' for writing: $1", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
- print F "if [ -e $wrk/$asm.ctgStore/seqDB.v001.tig ] ; then\n";
- print F " exit 0\n";
- print F "fi\n";
- print F "\n";
print F getBinDirectoryShellCode();
print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", $path);
+ print F "\n";
print F "# Purge any previous intermediate result. Possibly not needed, but safer.\n";
print F "\n";
- print F "rm -f $ofile.WORKING*\n";
+ print F "rm -f ./$ofile.WORKING*\n";
print F "\n";
print F "\$bin/meryl \\\n";
print F " -B -C -L 2 -v -m $merSize -threads $thr -memory $mem \\\n";
- print F " -s $wrk/$asm.gkpStore \\\n";
- print F " -o $ofile.WORKING \\\n";
+ print F " -s ../$asm.gkpStore \\\n";
+ print F " -o ./$ofile.WORKING \\\n";
print F "&& \\\n";
- print F "mv $ofile.WORKING.mcdat $ofile.FINISHED.mcdat \\\n";
+ print F "mv ./$ofile.WORKING.mcdat ./$ofile.mcdat \\\n";
print F "&& \\\n";
- print F "mv $ofile.WORKING.mcidx $ofile.FINISHED.mcidx\n";
+ print F "mv ./$ofile.WORKING.mcidx ./$ofile.mcidx\n";
+ print F "\n";
+ print F stashFileShellCode("$path", "$ofile.mcdat", "");
+ print F "\n";
+ print F stashFileShellCode("$path", "$ofile.mcidx", "");
+ print F "\n";
+ print F "\n";
+ print F "# Dump a histogram\n";
+ print F "\n";
+ print F "\$bin/meryl \\\n";
+ print F " -Dh -s ./$ofile \\\n";
+ print F "> ./$ofile.histogram.WORKING \\\n";
+ print F "2> ./$ofile.histogram.info \\\n";
+ print F "&& \\\n";
+ print F "mv -f ./$ofile.histogram.WORKING ./$ofile.histogram\n";
+ print F "\n";
+ print F stashFileShellCode("$path", "$ofile.histogram", "");
+ print F "\n";
+ print F stashFileShellCode("$path", "$ofile.histogram.info", "");
+ print F "\n";
+ print F "\n";
+ print F "# Compute a nice kmer threshold.\n";
+ print F "\n";
+ print F "\$bin/estimate-mer-threshold \\\n";
+ print F " -h ./$ofile.histogram \\\n";
+ print F " -c $cov \\\n";
+ print F "> ./$ofile.estMerThresh.out.WORKING \\\n";
+ print F "2> ./$ofile.estMerThresh.err \\\n";
+ print F "&& \\\n";
+ print F "mv ./$ofile.estMerThresh.out.WORKING ./$ofile.estMerThresh.out\n";
+ print F "\n";
+ print F stashFileShellCode("$path", "$ofile.estMerThresh.out", "");
+ print F "\n";
+ print F stashFileShellCode("$path", "$ofile.estMerThresh.err", "");
+ print F "\n";
print F "\n";
print F "exit 0\n";
close(F);
+ stashFile("$path/meryl.sh");
+
finishStage:
- emitStage($WRK, $asm, "merylConfigure");
- buildHTML($WRK, $asm, $tag);
- stopAfter("merylConfigure");
+ emitStage($asm, "merylConfigure");
+ buildHTML($asm, $tag);
allDone:
}
-sub merylCheck ($$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- ## $wrk = $WRK; # Local work directory
+sub merylCheck ($$) {
my $asm = shift @_;
my $tag = shift @_;
my $attempt = getGlobal("canuIteration");
- my $bin = getBinDirectory();
+ my $bin = getBinDirectory();
my $cmd;
- my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($WRK, $asm, $tag);
+ my ($base, $path, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($asm, $tag);
# If the frequent mer file exists, don't bother running meryl. We don't really need the
# databases.
- goto allDone if (skipStage($WRK, $asm, "$tag-meryl") == 1);
- goto allDone if (-e "$ffile");
- goto allDone if ((-e "$ofile.mcidx") && (-e "$ofile.mcdat"));
-
- # If FINISHED exists, meryl finished successfully. If WORKING, it failed during output. If
- # nothing, it failed before output.
+ goto allDone if (skipStage($asm, "$tag-meryl") == 1);
+ goto allDone if (fileExists("$path/meryl.success"));
+ goto finishStage if (!defined($ffile));
+ goto finishStage if (fileExists("$path/$ffile"));
+ goto finishStage if (fileExists("$path/$ofile.mcidx") && fileExists("$path/$ofile.mcdat"));
- if ((! -e "$ofile.FINISHED.mcdat") ||
- (! -e "$ofile.FINISHED.mcidx")) {
+ fetchFile("$path/meryl.sh");
- # If not the first attempt, report the jobs that failed, and that we're recomputing.
+ # Since there is only one job, if we get here, we're not done. Any other 'check' function
+ # shows how to process multiple jobs. This only checks for the existence of the final outputs.
+ # (unitigger is the same)
- if ($attempt > 1) {
- print STDERR "--\n";
- print STDERR "-- meryl failed.\n";
- print STDERR "--\n";
- }
+ # If not the first attempt, report the jobs that failed, and that we're recomputing.
- # If too many attempts, give up.
+ if ($attempt > 1) {
+ print STDERR "--\n";
+ print STDERR "-- meryl failed.\n";
+ print STDERR "--\n";
+ }
- if ($attempt > getGlobal("canuIterationMax")) {
- caExit("failed to generate mer counts. Made " . ($attempt-1) . " attempts, jobs still failed", undef);
- }
+ # If too many attempts, give up.
- # Otherwise, run some jobs.
+ if ($attempt > getGlobal("canuIterationMax")) {
+ caExit("failed to generate mer counts. Made " . ($attempt-1) . " attempts, jobs still failed", undef);
+ }
- print STDERR "-- Meryl attempt $attempt begins.\n";
+ # Otherwise, run some jobs.
- emitStage($WRK, $asm, "merylCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
+ emitStage($asm, "merylCheck", $attempt);
+ buildHTML($asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "meryl", "$wrk/0-mercounts", "meryl", (1));
- return;
- }
+ submitOrRunParallelJob($asm, "meryl", $path, "meryl", (1));
+ return;
finishStage:
print STDERR "-- Meryl finished successfully.\n";
- rename("$ofile.FINISHED.mcdat", "$ofile.mcdat");
- rename("$ofile.FINISHED.mcidx", "$ofile.mcidx");
+ make_path($path); # With object storage, we might not have this directory!
+
+ open(F, "> $path/meryl.success") or caExit("can't open '$path/meryl.success' for writing: $!", undef);
+ close(F);
+
+ stashFile("$path/meryl.success");
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "merylCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("merylCheck");
+ emitStage($asm, "merylCheck");
+ buildHTML($asm, $tag);
allDone:
}
-sub merylProcess ($$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- ## $wrk = $WRK; # Local work directory
+sub merylProcess ($$) {
my $asm = shift @_;
my $tag = shift @_;
- my $bin = getBinDirectory();
+ my $bin = getBinDirectory();
my $cmd;
- my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($WRK, $asm, $tag);
+ my ($base, $path, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($asm, $tag);
- goto allDone if (skipStage($WRK, $asm, "$tag-meryl") == 1);
- goto allDone if (-e "$ffile");
+ # ffile exists if we've already output it here, or if user supplied a file, or if user wants no masking.
- # A special case; if the threshold is zero, we can skip the rest.
+ goto allDone if (skipStage($asm, "$tag-meryl") == 1);
+ goto allDone if (fileExists("$path/$ffile"));
- if ((defined($merThresh)) &&
- ($merThresh ne "auto") &&
- ($merThresh == 0) &&
- (!defined($merDistinct)) &&
- (!defined($merTotal))) {
- touch($ffile);
- goto allDone;
- }
-
- # Dump a histogram.
-
- if (! -e "$ofile.histogram") {
- $cmd = "$bin/meryl -Dh -s $ofile > $ofile.histogram 2> $ofile.histogram.info";
-
- if (runCommand("$wrk/0-mercounts", $cmd)) {
- rename "$ofile.histogram", "$ofile.histogram.FAILED";
- caFailure("meryl histogram failed", "$ofile.histogram.info");
- }
- }
-
- # Compute a threshold, if needed
+ # Compute a threshold, if needed.
if ($merThresh eq "auto") {
- if (! -e "$ofile.estMerThresh.out") {
- $cmd = "$bin/estimate-mer-threshold ";
- $cmd .= " -m $ofile ";
- $cmd .= " > $ofile.estMerThresh.out ";
- $cmd .= "2> $ofile.estMerThresh.err";
-
- if (runCommand("$wrk/0-mercounts", $cmd)) {
- rename "$ofile.estMerThresh.out", "$ofile.estMerThresh.out.FAILED";
- caFailure("estimate-mer-threshold failed", "$ofile.estMerThresh.err");
- }
- }
+ fetchFile("$path/$ofile.estMerThresh.out");
- open(F, "< $ofile.estMerThresh.out") or caFailure("failed to read estimated mer threshold from '$ofile.estMerThresh.out'", undef);
+ open(F, "< $path/$ofile.estMerThresh.out") or caFailure("failed to read estimated mer threshold from '$path/$ofile.estMerThresh.out'", undef);
$merThresh = <F>;
- $merThresh = int($merThresh * $merScale);
+ $merThresh = int($merThresh * $merScale) + 1;
close(F);
}
# Compute a threshold based on the fraction distinct or total.
if (defined($merDistinct) || defined($merTotal)) {
- open(F, "< $ofile.histogram") or caFailure("failed to read mer histogram from '$ofile.histogram'", undef);
+ fetchFile("$path/$ofile.histogram");
+
+ open(F, "< $path/$ofile.histogram") or caFailure("failed to read mer histogram from '$path/$ofile.histogram'", undef);
while (<F>) {
my ($threshold, $num, $distinct, $total) = split '\s+', $_;
@@ -476,21 +610,32 @@ sub merylProcess ($$$) {
# Plot the histogram - annotated with the thesholds
- plotHistogram($wrk, $ofile, "lg", 1024);
- plotHistogram($wrk, $ofile, "sm", 256);
+ merylPlotHistogram($path, $ofile, "lg", 1024); # $ofile has merSize encoded in it
+ merylPlotHistogram($path, $ofile, "sm", 256);
+
+ # Display the histogram, and save to the report. Shouldn't this (and the plots above)
+ # go in finishStage?
+
+ addToReport("${tag}Meryl", merylGenerateHistogram($asm, $tag));
# Generate the frequent mers for overlapper
- if ((getGlobal("${tag}Overlapper") eq "ovl") &&
- (! -e $ffile)) {
- $cmd = "$bin/meryl -Dt -n $merThresh -s $ofile > $ffile 2> $ffile.err";
+ if (getGlobal("${tag}Overlapper") eq "ovl") {
+ fetchFile("$path/$ofile.mcdat");
+ fetchFile("$path/$ofile.mcidx");
+
+ if ((! -e "$path/$ofile.mcdat") ||
+ (! -e "$path/$ofile.mcdat")) {
+ caFailure("meryl can't dump frequent mers, databases don't exist. Remove $path/meryl.success to try again.", undef);
+ }
- if (runCommand("$wrk/0-mercounts", $cmd)) {
- unlink $ffile;
- caFailure("meryl failed to dump frequent mers", "$ffile.err");
+ if (runCommand($path, "$bin/meryl -Dt -n $merThresh -s ./$ofile > ./$ffile 2> ./$ffile.err")) {
+ unlink "$path/$ffile";
+ caFailure("meryl failed to dump frequent mers", "$path/$ffile.err");
}
+ unlink "$path/$ffile.err";
- unlink "$ffile.err";
+ stashFile("$path/$ffile");
}
# Generate the frequent mers for mhap
@@ -500,16 +645,17 @@ sub merylProcess ($$$) {
#
# The fraction is just $3/$4. I assume this is used with "--filter-threshold 0.000005".
- if ((getGlobal("${tag}Overlapper") eq "mhap") &&
- (! -e $ffile)) {
-
+ if (getGlobal("${tag}Overlapper") eq "mhap") {
my $totalMers = 0;
my $maxCount = 0;
+ fetchFile("$path/$ofile.histogram");
+ fetchFile("$path/$ofile.histogram.info");
+
# Meryl reports number of distinct canonical mers, we multiply by two to get the
# (approximate) number of distinct mers. Palindromes are counted twice, oh well.
- open(F, "< $ofile.histogram.info") or die "Failed to open '$ofile.histogram.info' for reading: $!\n";
+ open(F, "< $path/$ofile.histogram.info") or die "Failed to open '$path/$ofile.histogram.info' for reading: $!\n";
while (<F>) {
if (m/Found\s+(\d+)\s+mers./) {
$totalMers = 2 * $1;
@@ -519,15 +665,19 @@ sub merylProcess ($$$) {
}
}
close(F);
- caFailure("didn't find any mers?", "$ofile.histogram.info") if ($totalMers == 0);
+ caFailure("didn't find any mers?", "$path/$ofile.histogram.info") if ($totalMers == 0);
+
+ my $filterThreshold = getGlobal("${tag}MhapFilterThreshold");
+ my $misRate = 0.1;
+ my $minCount = int($filterThreshold * $totalMers);
+ my $totalToOutput = 0;
+ my $totalFiltered = 0;
- my $filterThreshold = (getGlobal("${tag}MhapSensitivity") eq "normal") ? getGlobal("${tag}MhapFilterThreshold") : getGlobal("${tag}MhapFilterThreshold"); # Also set in Meryl.pm
+ if (defined(getGlobal("${tag}MhapFilterUnique"))) {
+ $minCount = uniqueKmerThreshold($base, $asm, $merSize, $misRate) + 1;
+ }
- my $misRate = 0.1;
- my $minCount = defined(getGlobal("${tag}MhapFilterUnique")) ? uniqueKmerThreshold($wrk, $asm, $merSize, $misRate)+1 : int($filterThreshold * $totalMers);
- my $totalToOutput = 0;
- my $totalFiltered = 0;
- open(F, "< $ofile.histogram") or die "Failed to open '$ofile.histogram' for reading: $!\n";
+ open(F, "< $path/$ofile.histogram") or die "Failed to open '$path/$ofile.histogram' for reading: $!\n";
while (<F>) {
my ($kCount, $occurences, $cumsum, $faction) = split '\s+', $_;
if ($kCount < $minCount) {
@@ -540,8 +690,12 @@ sub merylProcess ($$$) {
close(F);
$totalToOutput *= 2; # for the reverse complement
- open(F, "$bin/meryl -Dt -n $minCount -s $ofile | ") or die "Failed to run meryl to generate frequent mers $!\n";
- open(O, "| gzip -c > $ofile.frequentMers.ignore.gz") or die "Failed to open '$ofile.frequentMers.ignore.gz' for writing: $!\n";
+ fetchFile("$path/$ofile.mcdat");
+ fetchFile("$path/$ofile.mcidx");
+
+ open(F, "$bin/meryl -Dt -n $minCount -s $path/$ofile | ") or die "Failed to run meryl to generate frequent mers $!\n";
+ open(O, "| gzip -c > $path/$ofile.frequentMers.ignore.gz") or die "Failed to open '$path/$ofile.frequentMers.ignore.gz' for writing: $!\n";
+
printf(O "%d\n", $totalToOutput);
while (!eof(F)) {
@@ -559,8 +713,10 @@ sub merylProcess ($$$) {
close(O);
close(F);
+ stashFile("$path/$ffile");
+
if (defined(getGlobal("${tag}MhapFilterUnique"))) {
- printf STDERR "-- For %s overlapping, filtering low-occurence k-mers < %d (%.2f\%) based on estimated error of %.2f\%.\n", getGlobal("${tag}Overlapper"), $minCount, $totalFiltered, 100*estimateRawError($wrk, $asm, $tag, $merSize);
+ printf STDERR "-- For %s overlapping, filtering low-occurence k-mers < %d (%.2f\%) based on estimated error of %.2f\%.\n", getGlobal("${tag}Overlapper"), $minCount, $totalFiltered, 100*estimateRawError($base, $asm, $tag, $merSize);
}
printf STDERR "-- For %s overlapping, set repeat k-mer threshold to %d.\n", getGlobal("${tag}Overlapper"), int($filterThreshold * $totalMers);
}
@@ -573,13 +729,16 @@ sub merylProcess ($$$) {
}
finishStage:
- if (-e "$ofile.histogram.info") {
+ fetchFile("$path/$ofile.histogram.info");
+ fetchFile("$path/$ffile");
+
+ if (-e "$path/$ofile.histogram.info") {
my $numTotal = 0;
my $numDistinct = 0;
my $numUnique = 0;
my $largest = 0;
- open(F, "< $ofile.histogram.info") or caFailure("can't open meryl histogram information file '$ofile.histogram.info' for reading: $!\n", undef);
+ open(F, "< $path/$ofile.histogram.info") or caFailure("can't open meryl histogram information file '$path/$ofile.histogram.info' for reading: $!\n", undef);
while (<F>) {
$numTotal = $1 if (m/Found\s(\d+)\s+mers./);
$numDistinct = $1 if (m/Found\s(\d+)\s+distinct\smers./);
@@ -591,21 +750,21 @@ sub merylProcess ($$$) {
print STDERR "--\n";
print STDERR "-- Found $numTotal $merSize-mers; $numDistinct distinct and $numUnique unique. Largest count $largest.\n";
- } elsif (-z $ffile) {
+ } elsif (-z "$path/$ffile") {
print STDERR "--\n";
- print STDERR "-- Threshold zero. No mers will be to find.\n";
+ print STDERR "-- Threshold zero. No mers will be masked.\n";
} else {
print STDERR "--\n";
print STDERR "-- Using frequent mers in '", getGlobal("${tag}OvlFrequentMers"), "'\n";
}
- unlink "$ofile.mcidx" if (getGlobal("saveMerCounts") == 0);
- unlink "$ofile.mcdat" if (getGlobal("saveMerCounts") == 0);
+ unlink "$path/$ofile.mcidx" if (getGlobal("saveMerCounts") == 0);
+ unlink "$path/$ofile.mcdat" if (getGlobal("saveMerCounts") == 0);
- emitStage($WRK, $asm, "$tag-meryl");
- buildHTML($WRK, $asm, $tag);
- stopAfter("meryl");
+ emitStage($asm, "$tag-meryl");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("meryl");
}
diff --git a/src/pipelines/canu/Output.pm b/src/pipelines/canu/Output.pm
index 4512842..8269bb0 100644
--- a/src/pipelines/canu/Output.pm
+++ b/src/pipelines/canu/Output.pm
@@ -49,141 +49,167 @@ use File::Copy;
use canu::Defaults;
use canu::Execution;
use canu::HTML;
+use canu::Grid_Cloud;
-# In this module, the inputs are read from $wrk (the local work directory), and the output are
-# written to $WRK (the root work directory). This is a change from CA8 where outputs
-# were written to the 9-terminator directory.
-
-sub generateOutputs ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub generateOutputs ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
my $type = "fasta"; # Should probably be an option.
- goto allDone if (skipStage($WRK, $asm, "generateOutputs") == 1);
+ goto allDone if (skipStage($asm, "generateOutputs") == 1);
# Layouts
- if (! -e "$WRK/$asm.contigs.layout") {
+ if (! fileExists("$asm.contigs.layout")) {
+ fetchStore("unitigging/$asm.gkpStore");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v002.dat");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v002.tig");
+
$cmd = "$bin/tgStoreDump \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.ctgStore 2 \\\n";
- $cmd .= " -o $WRK/$asm.contigs \\\n";
+ $cmd .= " -G ./unitigging/$asm.gkpStore \\\n";
+ $cmd .= " -T ./unitigging/$asm.ctgStore 2 \\\n";
+ $cmd .= " -o ./$asm.contigs \\\n";
$cmd .= " -layout \\\n";
- $cmd .= "> $WRK/$asm.contigs.layout.err 2>&1";
+ $cmd .= "> ./$asm.contigs.layout.err 2>&1";
- if (runCommand($wrk, $cmd)) {
- caExit("failed to output contig layouts", "$WRK/$asm.contigs.layout.err");
+ if (runCommand(".", $cmd)) {
+ caExit("failed to output contig layouts", "$asm.contigs.layout.err");
}
- unlink "$WRK/$asm.contigs.layout.err";
+ unlink "$asm.contigs.layout.err";
+
+ stashFile("$asm.contigs.layout");
}
- if (! -e "$WRK/$asm.unitigs.layout") {
+ if (! fileExists("$asm.unitigs.layout")) {
+ fetchStore("unitigging/$asm.gkpStore");
+
+ fetchFile("unitigging/$asm.utgStore/seqDB.v001.dat"); # Why is this needed?
+ fetchFile("unitigging/$asm.utgStore/seqDB.v001.tig");
+
+ fetchFile("unitigging/$asm.utgStore/seqDB.v002.dat");
+ fetchFile("unitigging/$asm.utgStore/seqDB.v002.tig");
+
$cmd = "$bin/tgStoreDump \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.utgStore 2 \\\n";
- $cmd .= " -o $WRK/$asm.unitigs \\\n";
+ $cmd .= " -G ./unitigging/$asm.gkpStore \\\n";
+ $cmd .= " -T ./unitigging/$asm.utgStore 2 \\\n";
+ $cmd .= " -o ./$asm.unitigs \\\n";
$cmd .= " -layout \\\n";
- $cmd .= "> $WRK/$asm.unitigs.layout.err 2>&1";
+ $cmd .= "> ./$asm.unitigs.layout.err 2>&1";
- if (runCommand($wrk, $cmd)) {
- caExit("failed to output unitig layouts", "$WRK/$asm.unitigs.layout.err");
+ if (runCommand(".", $cmd)) {
+ caExit("failed to output unitig layouts", "$asm.unitigs.layout.err");
}
- unlink "$WRK/$asm.unitigs.layout.err";
+ unlink "$asm.unitigs.layout.err";
+
+ stashFile("$asm.unitigs.layout");
}
# Sequences
- foreach my $tt ("unassembled", "bubbles", "contigs") {
- if (! -e "$WRK/$asm.$tt.$type") {
+ foreach my $tt ("unassembled", "contigs") {
+ if (! fileExists("$asm.$tt.$type")) {
+ fetchStore("unitigging/$asm.gkpStore");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v002.dat");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v002.tig");
+
$cmd = "$bin/tgStoreDump \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.ctgStore 2 \\\n";
+ $cmd .= " -G ./unitigging/$asm.gkpStore \\\n";
+ $cmd .= " -T ./unitigging/$asm.ctgStore 2 \\\n";
$cmd .= " -consensus -$type \\\n";
$cmd .= " -$tt \\\n";
- $cmd .= "> $WRK/$asm.$tt.$type\n";
- $cmd .= "2> $WRK/$asm.$tt.err";
+ $cmd .= "> ./$asm.$tt.$type\n";
+ $cmd .= "2> ./$asm.$tt.err";
- if (runCommand($WRK, $cmd)) {
- caExit("failed to output $tt consensus sequences", "$WRK/$asm.$tt.err");
+ if (runCommand(".", $cmd)) {
+ caExit("failed to output $tt consensus sequences", "$asm.$tt.err");
}
- unlink "$WRK/$asm.$tt.err";
+ unlink "$asm.$tt.err";
+
+ stashFile("$asm.$tt.$type");
}
}
- if (! -e "$WRK/$asm.unitigs.$type") {
+ if (! fileExists("$asm.unitigs.$type")) {
+ fetchStore("unitigging/$asm.gkpStore");
+ fetchFile("unitigging/$asm.utgStore/seqDB.v002.dat");
+ fetchFile("unitigging/$asm.utgStore/seqDB.v002.tig");
+
$cmd = "$bin/tgStoreDump \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.utgStore 2 \\\n";
+ $cmd .= " -G ./unitigging/$asm.gkpStore \\\n";
+ $cmd .= " -T ./unitigging/$asm.utgStore 2 \\\n";
$cmd .= " -consensus -$type \\\n";
$cmd .= " -contigs \\\n";
- $cmd .= "> $WRK/$asm.unitigs.$type\n";
- $cmd .= "2> $WRK/$asm.unitigs.err";
+ $cmd .= "> ./$asm.unitigs.$type\n";
+ $cmd .= "2> ./$asm.unitigs.err";
- if (runCommand($WRK, $cmd)) {
- caExit("failed to output unitig consensus sequences", "$WRK/$asm.unitigs.err");
+ if (runCommand(".", $cmd)) {
+ caExit("failed to output unitig consensus sequences", "$asm.unitigs.err");
}
- unlink "$WRK/$asm.unitigs.err";
+ unlink "$asm.unitigs.err";
+
+ stashFile("$asm.unitigs.$type");
}
# Graphs
- if ((! -e "$WRK/$asm.contigs.gfa") &&
- ( -e "$wrk/4-unitigger/$asm.contigs.gfa")) {
- copy("$wrk/4-unitigger/$asm.contigs.gfa", "$WRK/$asm.contigs.gfa");
+ if ((!fileExists("$asm.contigs.gfa")) &&
+ ( fileExists("unitigging/4-unitigger/$asm.contigs.aligned.gfa"))) {
+ fetchFile("unitigging/4-unitigger/$asm.contigs.aligned.gfa");
+ copy("unitigging/4-unitigger/$asm.contigs.aligned.gfa", "$asm.contigs.gfa");
+ stashFile("$asm.contigs.gfa");
}
- if ((! -e "$WRK/$asm.unitigs.gfa") &&
- ( -e "$wrk/4-unitigger/$asm.unitigs.gfa")) {
- copy("$wrk/4-unitigger/$asm.unitigs.gfa", "$WRK/$asm.unitigs.gfa");
+ if ((! fileExists("$asm.unitigs.gfa")) &&
+ ( fileExists("unitigging/4-unitigger/$asm.unitigs.aligned.gfa"))) {
+ fetchFile("unitigging/4-unitigger/$asm.unitigs.aligned.gfa");
+ copy("unitigging/4-unitigger/$asm.unitigs.aligned.gfa", "$asm.unitigs.gfa");
+ stashFile("$asm.unitigs.gfa");
}
# User-supplied termination command.
if (defined(getGlobal("onSuccess"))) {
print STDERR "-- Running user-supplied termination command.\n";
- runCommand($WRK, getGlobal("onSuccess") . " $asm");
+ runCommand(getGlobal("onExitDir"), getGlobal("onSuccess") . " $asm");
}
finishStage:
- emitStage($WRK, $asm, "generateOutputs");
- buildHTML($WRK, $asm, "utg");
+ emitStage($asm, "generateOutputs");
+ buildHTML($asm, "utg");
allDone:
print STDERR "--\n";
- print STDERR "-- Assembly finished.\n";
+ print STDERR "-- Assembly '", getGlobal("onExitNam"), "' finished in '", getGlobal("onExitDir"), "'.\n";
print STDERR "--\n";
- print STDERR "-- Summary saved in '$WRK/unitigging.html'.\n";
+ print STDERR "-- Summary saved in 'unitigging.html'.\n";
print STDERR "--\n";
print STDERR "-- Sequences saved:\n";
- print STDERR "-- Contigs -> '$WRK/$asm.contigs.$type'\n";
- print STDERR "-- Bubbles -> '$WRK/$asm.bubbles.$type' (DEPRECATED)\n";
- print STDERR "-- Unassembled -> '$WRK/$asm.unassembled.$type'\n";
- print STDERR "-- Unitigs -> '$WRK/$asm.unitigs.$type'\n";
+ print STDERR "-- Contigs -> '$asm.contigs.$type'\n";
+ print STDERR "-- Unassembled -> '$asm.unassembled.$type'\n";
+ print STDERR "-- Unitigs -> '$asm.unitigs.$type'\n";
print STDERR "--\n";
print STDERR "-- Read layouts saved:\n";
- print STDERR "-- Contigs -> '$WRK/$asm.contigs.layout'.\n";
- print STDERR "-- Unitigs -> '$WRK/$asm.unitigs.layout'.\n";
+ print STDERR "-- Contigs -> '$asm.contigs.layout'.\n";
+ print STDERR "-- Unitigs -> '$asm.unitigs.layout'.\n";
print STDERR "--\n";
print STDERR "-- Graphs saved:\n";
- print STDERR "-- Contigs -> '$WRK/$asm.contigs.gfa'.\n";
- print STDERR "-- Unitigs -> '$WRK/$asm.unitigs.gfa'.\n";
+ print STDERR "-- Contigs -> '$asm.contigs.gfa'.\n";
+ print STDERR "-- Unitigs -> '$asm.unitigs.gfa'.\n";
print STDERR "--\n";
print STDERR "-- Bye.\n";
finishStage:
- emitStage($WRK, $asm, "outputSequence");
- buildHTML($WRK, $asm, "utg");
+ emitStage($asm, "outputSequence");
+ buildHTML($asm, "utg");
allDone:
}
diff --git a/src/pipelines/canu/OverlapBasedTrimming.pm b/src/pipelines/canu/OverlapBasedTrimming.pm
index c67f85f..4abf45a 100644
--- a/src/pipelines/canu/OverlapBasedTrimming.pm
+++ b/src/pipelines/canu/OverlapBasedTrimming.pm
@@ -27,6 +27,10 @@
# are a 'United States Government Work', and
# are released in the public domain
#
+ # Sergey Koren beginning on 2017-MAR-03
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
# File 'README.licenses' in the root directory of this distribution contains
# full conditions and disclaimers for each license.
##
@@ -40,42 +44,43 @@ require Exporter;
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
+use canu::Gatekeeper;
+use canu::Report;
use canu::HTML;
+use canu::Grid_Cloud;
-sub trimReads ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/trimming"; # Local work directory
+sub trimReads ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/3-overlapbasedtrimming";
+ my $path = "trimming/3-overlapbasedtrimming";
- goto allDone if (skipStage($WRK, $asm, "obt-trimReads") == 1);
- goto allDone if (-e "$path/trimmed");
+ goto allDone if (skipStage($asm, "obt-trimReads") == 1);
+ goto allDone if (fileExists("trimming/3-overlapbasedtrimming/$asm.1.trimReads.clear"));
make_path($path) if (! -d $path);
+ fetchStore("./trimming/$asm.ovlStore");
+
# Previously, we'd pick the error rate used by unitigger. Now, we don't know unitigger here,
# and require an obt specific error rate.
$cmd = "$bin/trimReads \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
- $cmd .= " -Co $path/$asm.1.trimReads.clear \\\n";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -O ../$asm.ovlStore \\\n";
+ $cmd .= " -Co ./$asm.1.trimReads.clear \\\n";
$cmd .= " -e " . getGlobal("obtErrorRate") . " \\\n";
$cmd .= " -minlength " . getGlobal("minReadLength") . " \\\n";
- #$cmd .= " -Cm $path/$asm.max.clear \\\n" if (-e "$path/$asm.max.clear");
+ #$cmd .= " -Cm ./$asm.max.clear \\\n" if (-e "./$asm.max.clear");
$cmd .= " -ol " . getGlobal("trimReadsOverlap") . " \\\n";
$cmd .= " -oc " . getGlobal("trimReadsCoverage") . " \\\n";
- $cmd .= " -o $path/$asm.1.trimReads \\\n";
- $cmd .= "> $path/$asm.1.trimReads.err 2>&1";
-
- stopBefore("trimReads", $cmd);
+ $cmd .= " -o ./$asm.1.trimReads \\\n";
+ $cmd .= "> ./$asm.1.trimReads.err 2>&1";
if (runCommand($path, $cmd)) {
caFailure("trimReads failed", "$path/$asm.1.trimReads.err");
@@ -85,57 +90,69 @@ sub trimReads ($$) {
unlink("$path/$asm.1.trimReads.err");
+ stashFile("./trimming/3-overlapbasedtrimming/$asm.1.trimReads.clear");
+
+ my $report;
+
+#FORMAT
+ open(F, "< trimming/3-overlapbasedtrimming/$asm.1.trimReads.stats") or caExit("can't open 'trimming/3-overlapbasedtrimming/$asm.1.trimReads.stats' for reading: $!", undef);
+ while (<F>) {
+ $report .= "-- $_";
+ }
+ close(F);
+
+ addToReport("trimming", $report);
+
+
if (0) {
$cmd = "$bin/gatekeeperDumpFASTQ \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -c $path/$asm.1.trimReads.clear \\\n";
- $cmd .= " -o $path/$asm.1.trimReads.trimmed \\\n";
- $cmd .= "> $path/$asm.1.trimReads.trimmed.err 2>&1";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -c ./$asm.1.trimReads.clear \\\n";
+ $cmd .= " -o ./$asm.1.trimReads.trimmed \\\n";
+ $cmd .= "> ./$asm.1.trimReads.trimmed.err 2>&1";
if (runCommand($path, $cmd)) {
- caFailure("dumping trimmed reads failed", "$wrk/$asm.1.trimReads.trimmed.err");
+ caFailure("dumping trimmed reads failed", "$path/$asm.1.trimReads.trimmed.err");
}
}
finishStage:
- touch("$path/trimmed");
- emitStage($WRK, $asm, "obt-trimReads");
- buildHTML($WRK, $asm, "obt");
+ emitStage($asm, "obt-trimReads");
+ buildHTML($asm, "obt");
allDone:
}
-sub splitReads ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/trimming"; # Local work directory
+sub splitReads ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/3-overlapbasedtrimming";
+ my $path = "trimming/3-overlapbasedtrimming";
- goto allDone if (skipStage($WRK, $asm, "obt-splitReads") == 1);
- goto allDone if (-e "$path/splitted"); # Splitted?
+ goto allDone if (skipStage($asm, "obt-splitReads") == 1);
+ goto allDone if (fileExists("trimming/3-overlapbasedtrimming/$asm.2.splitReads.clear"));
make_path($path) if (! -d $path);
+ fetchStore("./trimming/$asm.ovlStore");
+ fetchFile("./trimming/3-overlapbasedtrimming/$asm.1.trimReads.clear");
+
my $erate = getGlobal("obtErrorRate"); # Was this historically
#$cmd .= " -mininniepair 0 -minoverhanging 0 \\\n" if (getGlobal("doChimeraDetection") eq "aggressive");
$cmd = "$bin/splitReads \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
- $cmd .= " -Ci $path/$asm.1.trimReads.clear \\\n" if (-e "$path/$asm.1.trimReads.clear");
- #$cmd .= " -Cm $path/$asm.max.clear \\\n" if (-e "$path/$asm.max.clear");
- $cmd .= " -Co $path/$asm.2.splitReads.clear \\\n";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -O ../$asm.ovlStore \\\n";
+ $cmd .= " -Ci ./$asm.1.trimReads.clear \\\n" if (-e "trimming/3-overlapbasedtrimming/$asm.1.trimReads.clear");
+ #$cmd .= " -Cm ./$asm.max.clear \\\n" if (-e "trimming/3-overlapbasedtrimming/$asm.max.clear");
+ $cmd .= " -Co ./$asm.2.splitReads.clear \\\n";
$cmd .= " -e $erate \\\n";
$cmd .= " -minlength " . getGlobal("minReadLength") . " \\\n";
- $cmd .= " -o $path/$asm.2.splitReads \\\n";
- $cmd .= "> $path/$asm.2.splitReads.err 2>&1";
-
- stopBefore("splitReads", $cmd);
+ $cmd .= " -o ./$asm.2.splitReads \\\n";
+ $cmd .= "> ./$asm.2.splitReads.err 2>&1";
if (runCommand($path, $cmd)) {
caFailure("splitReads failed", "$path/$asm.2.splitReads.err");
@@ -145,68 +162,84 @@ sub splitReads ($$) {
unlink("$path/$asm.2.splitReads.err");
+ stashFile("./trimming/3-overlapbasedtrimming/$asm.2.splitReads.clear");
+
+ my $report;
+
+#FORMAT
+ open(F, "< trimming/3-overlapbasedtrimming/$asm.2.splitReads.stats") or caExit("can't open 'trimming/3-overlapbasedtrimming/$asm.2.splitReads.stats' for reading: $!", undef);
+ while (<F>) {
+ $report .= "-- $_";
+ }
+ close(F);
+
+ addToReport("splitting", $report);
+
if (0) {
$cmd = "$bin/gatekeeperDumpFASTQ \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -c $path/$asm.2.splitReads.clear \\\n";
- $cmd .= " -o $path/$asm.2.splitReads.trimmed \\\n";
- $cmd .= "> $path/$asm.2.splitReads.trimmed.err 2>&1";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -c ./$asm.2.splitReads.clear \\\n";
+ $cmd .= " -o ./$asm.2.splitReads.trimmed \\\n";
+ $cmd .= "> ./$asm.2.splitReads.trimmed.err 2>&1";
if (runCommand($path, $cmd)) {
- caFailure("dumping trimmed reads failed", "$wrk/$asm.2.splitReads.trimmed.err");
+ caFailure("dumping trimmed reads failed", "$path/$asm.2.splitReads.trimmed.err");
}
}
finishStage:
- touch("$path/splitted", "Splitted? Is that even a word?");
- emitStage($WRK, $asm, "obt-splitReads");
- buildHTML($WRK, $asm, "obt");
+ emitStage($asm, "obt-splitReads");
+ buildHTML($asm, "obt");
allDone:
}
-sub dumpReads ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/trimming"; # Local work directory
+sub dumpReads ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/3-overlapbasedtrimming";
+ my $path = "trimming/3-overlapbasedtrimming";
my $inp;
- goto allDone if (skipStage($WRK, $asm, "obt-dumpReads") == 1);
- goto allDone if (-e "$WRK/$asm.trimmedReads.fasta");
- goto allDone if (-e "$WRK/$asm.trimmedReads.fasta.gz");
+ goto allDone if (skipStage($asm, "obt-dumpReads") == 1);
+ goto allDone if (sequenceFileExists("$asm.trimmedReads"));
make_path($path) if (! -d $path);
- $inp = "$path/$asm.1.trimReads.clear" if (-e "$path/$asm.1.trimReads.clear");
- $inp = "$path/$asm.2.splitReads.clear" if (-e "$path/$asm.2.splitReads.clear");
+ fetchFile("./trimming/3-overlapbasedtrimming/$asm.1.trimReads.clear");
+ fetchFile("./trimming/3-overlapbasedtrimming/$asm.2.splitReads.clear");
+
+ $inp = "./3-overlapbasedtrimming/$asm.1.trimReads.clear" if (-e "$path/$asm.1.trimReads.clear");
+ $inp = "./3-overlapbasedtrimming/$asm.2.splitReads.clear" if (-e "$path/$asm.2.splitReads.clear");
- caFailure("dumping trimmed reads failed; no 'clear' input", "$WRK/$asm.trimmedReads.err") if (!defined($inp));
+ caFailure("dumping trimmed reads failed; no 'clear' input", "trimming/$asm.trimmedReads.err") if (!defined($inp));
$cmd = "$bin/gatekeeperDumpFASTQ -fasta -nolibname \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -G ./$asm.gkpStore \\\n";
$cmd .= " -c $inp \\\n";
- $cmd .= " -o $WRK/$asm.trimmedReads.gz \\\n";
- $cmd .= "> $WRK/$asm.trimmedReads.err 2>&1";
+ $cmd .= " -o ../$asm.trimmedReads.gz \\\n"; # Adds .fasta
+ $cmd .= "> ../$asm.trimmedReads.err 2>&1";
- if (runCommand($wrk, $cmd)) {
- caFailure("dumping trimmed reads failed", "$WRK/$asm.trimmedReads.err");
+ if (runCommand("trimming", $cmd)) {
+ caFailure("dumping trimmed reads failed", "./$asm.trimmedReads.err");
}
- unlink("$WRK/$asm.trimmedReads.err");
+ unlink("./$asm.trimmedReads.err");
+
+ stashFile("./$asm.trimmedReads.fasta.gz");
# Need gatekeeperDumpFASTQ to also write a gkp input file
- #touch("$wrk/$asm.trimmedReads.gkp");
+ #touch("../$asm.trimmedReads.gkp");
finishStage:
- emitStage($WRK, $asm, "obt-dumpReads");
- buildHTML($WRK, $asm, "obt");
+ emitStage($asm, "obt-dumpReads");
+ buildHTML($asm, "obt");
allDone:
print STDERR "--\n";
- print STDERR "-- Trimmed reads saved in '$WRK/$asm.trimmedReads.fasta.gz'\n";
+ print STDERR "-- Trimmed reads saved in 'trimming/$asm.trimmedReads.fasta.gz'\n";
+
+ stopAfter("readTrimming");
}
diff --git a/src/pipelines/canu/OverlapErrorAdjustment.pm b/src/pipelines/canu/OverlapErrorAdjustment.pm
index 07c5164..5db2253 100644
--- a/src/pipelines/canu/OverlapErrorAdjustment.pm
+++ b/src/pipelines/canu/OverlapErrorAdjustment.pm
@@ -44,64 +44,33 @@ require Exporter;
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
+use canu::Report;
use canu::HTML;
+use canu::Grid_Cloud;
# Hardcoded to use utgOvlErrorRate
-sub concatOutput ($@) {
- my $outName = shift @_;
- my @successJobs = @_;
-
- open(O, "> $outName");
- binmode(O);
-
- foreach my $f (@successJobs) {
- open(F, "< $f");
- binmode(F);
-
- my $buf;
- my $len = sysread(F, $buf, 1024 * 1024);
-
- while ($len > 0) {
- syswrite(O, $buf, $len);
-
- $len = sysread(F, $buf, 1024 * 1024);
- }
-
- close(F);
- }
-
- close(O);
-
- foreach my $f (@successJobs) {
- unlink $f;
- }
-}
-
-
-
-
-sub readErrorDetectionConfigure ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub readErrorDetectionConfigure ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
- my $path = "$wrk/3-overlapErrorAdjustment";
+ my $path = "unitigging/3-overlapErrorAdjustment";
return if (getGlobal("enableOEA") == 0);
- goto allDone if (skipStage($wrk, $asm, "readErrorDetectionConfigure") == 1);
- goto allDone if (-e "$path/red.red");
+ goto allDone if (fileExists("$path/red.sh")); # Script exists
+ goto allDone if (fileExists("$path/red.red")); # Result exists
+
+ goto allDone if (skipStage($asm, "readErrorDetectionConfigure") == 1);
- goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues");
- goto allDone if (-d "$wrk/$asm.ctgStore");
+ goto allDone if (fileExists("unitigging/$asm.ovlStore/evalues")); # Stage entrely finished
+ goto allDone if (-d "unitigging/$asm.ctgStore"); # Assembly finished
make_path("$path") if (! -d "$path");
@@ -110,16 +79,20 @@ sub readErrorDetectionConfigure ($$) {
my @readLengths;
my @numOlaps;
- #print STDERR "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads\n";
- open(F, "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads |");
+ #print STDERR "$bin/gatekeeperDumpMetaData -G unitigging/$asm.gkpStore -reads\n";
+ open(F, "$bin/gatekeeperDumpMetaData -G unitigging/$asm.gkpStore -reads |");
while (<F>) {
my @v = split '\s+', $_;
$readLengths[$v[0]] = $v[2];
}
close(F);
- #print STDERR "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts\n";
- open(F, "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts |");
+ # NEEDS OPTIMIZE - only need counts here, not the whole store
+
+ fetchStore("unitigging/$asm.ovlStore");
+
+ #print STDERR "$bin/ovStoreDump -G unitigging/$asm.gkpStore -O unitigging/$asm.ovlStore -d -counts\n";
+ open(F, "$bin/ovStoreDump -G unitigging/$asm.gkpStore -O unitigging/$asm.ovlStore -d -counts |");
while (<F>) {
my @v = split '\s+', $_;
$numOlaps[$v[0]] = $v[1];
@@ -134,7 +107,7 @@ sub readErrorDetectionConfigure ($$) {
#getAllowedResources("", "red");
- my $maxID = getNumberOfReadsInStore($wrk, $asm);
+ my $maxID = getNumberOfReadsInStore("unitigging", $asm);
my $maxMem = getGlobal("redMemory") * 1024 * 1024 * 1024;
my $maxReads = getGlobal("redBatchSize");
my $maxBases = getGlobal("redBatchLength");
@@ -147,7 +120,7 @@ sub readErrorDetectionConfigure ($$) {
my $bases = 0;
my $olaps = 0;
- my $coverage = getExpectedCoverage($wrk, $asm);
+ my $coverage = getExpectedCoverage("unitigging", $asm);
push @bgn, 1;
@@ -189,12 +162,18 @@ sub readErrorDetectionConfigure ($$) {
my $batchSize = getGlobal("redBatchSize");
my $numThreads = getGlobal("redThreads");
- my $numReads = getNumberOfReadsInStore($wrk, $asm);
+ my $numReads = getNumberOfReadsInStore("unitigging", $asm);
open(F, "> $path/red.sh") or caExit("can't open '$path/red.sh' for writing: $!", undef);
print F "#!" . getGlobal("shell") . "\n\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("unitigging/$asm.gkpStore", $path, "");
+ print F fetchStoreShellCode("unitigging/$asm.ovlStore", $path, "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
@@ -207,32 +186,33 @@ sub readErrorDetectionConfigure ($$) {
print F "jobid=`printf %04d \$jobid`\n";
print F "\n";
- print F "if [ -e $path/\$jobid.red ] ; then\n";
+ print F "if [ -e ./\$jobid.red ] ; then\n";
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
-
- print F getBinDirectoryShellCode();
-
- print F "if [ ! -e $path/\$jobid.red ] ; then\n";
- print F " \$bin/findErrors \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $wrk/$asm.ovlStore \\\n";
- print F " -R \$minid \$maxid \\\n";
- print F " -e " . getGlobal("utgOvlErrorRate") . " -l " . getGlobal("minOverlapLength") . " \\\n";
- print F " -o $path/\$jobid.red.WORKING \\\n";
- print F " -t $numThreads \\\n";
- print F " && \\\n";
- print F " mv $path/\$jobid.red.WORKING $path/\$jobid.red\n";
- print F "fi\n";
+ print F "\n";
+ print F "\$bin/findErrors \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -O ../$asm.ovlStore \\\n";
+ print F " -R \$minid \$maxid \\\n";
+ print F " -e " . getGlobal("utgOvlErrorRate") . " -l " . getGlobal("minOverlapLength") . " \\\n";
+ print F " -o ./\$jobid.red.WORKING \\\n";
+ print F " -t $numThreads \\\n";
+ print F "&& \\\n";
+ print F "mv ./\$jobid.red.WORKING ./\$jobid.red\n";
+ print F "\n";
+ print F stashFileShellCode("$path", "\$jobid.red", "");
+ print F "\n";
close(F);
chmod 0755, "$path/red.sh";
+ stashFile("$path/red.sh");
+
finishStage:
- emitStage($WRK, $asm, "readErrorDetectionConfigure");
- buildHTML($WRK, $asm, "utg");
+ emitStage($asm, "readErrorDetectionConfigure");
+ buildHTML($asm, "utg");
allDone:
}
@@ -241,19 +221,21 @@ sub readErrorDetectionConfigure ($$) {
-sub readErrorDetectionCheck ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub readErrorDetectionCheck ($) {
my $asm = shift @_;
my $attempt = getGlobal("canuIteration");
- my $path = "$wrk/3-overlapErrorAdjustment";
+ my $path = "unitigging/3-overlapErrorAdjustment";
return if (getGlobal("enableOEA") == 0);
- goto allDone if (skipStage($wrk, $asm, "readErrorDetectionCheck", $attempt) == 1);
- goto allDone if (-e "$path/red.red");
- goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues");
- goto allDone if (-d "$wrk/$asm.ctgStore");
+ goto allDone if (fileExists("$path/red.red")); # Output exists
+
+ goto allDone if (skipStage($asm, "readErrorDetectionCheck", $attempt) == 1);
+
+ goto allDone if (fileExists("unitigging/$asm.ovlStore/evalues")); # Stage entrely finished
+ goto allDone if (-d "unitigging/$asm.ctgStore"); # Assembly finished
+
+ fetchFile("$path/red.sh");
# Figure out if all the tasks finished correctly.
@@ -265,10 +247,10 @@ sub readErrorDetectionCheck ($$) {
while (<A>) {
if (m/if.*jobid\s+=\s+(\d+)\s+.*then/) {
my $ji = substr("0000" . $1, -4);
- my $jn = "$path/$ji.red";
+ my $jn = "unitigging/3-overlapErrorAdjustment/$ji.red";
- if (! -e $jn) {
- $failureMessage .= "-- job $jn FAILED.\n";
+ if (! fileExists($jn)) {
+ $failureMessage .= "-- job $ji.red FAILED.\n";
push @failedJobs, $1;
} else {
push @successJobs, $jn;
@@ -298,12 +280,10 @@ sub readErrorDetectionCheck ($$) {
# Otherwise, run some jobs.
- print STDERR "-- read error detection attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "readErrorDetectionCheck", $attempt);
- buildHTML($WRK, $asm, "utg");
+ emitStage($asm, "readErrorDetectionCheck", $attempt);
+ buildHTML($asm, "utg");
- submitOrRunParallelJob($WRK, $asm, "red", "$path", "red", @failedJobs);
+ submitOrRunParallelJob($asm, "red", $path, "red", @failedJobs);
return;
}
@@ -315,12 +295,37 @@ sub readErrorDetectionCheck ($$) {
# hacking correctOverlaps to handle multiple corrections files. Plus, it is now really just a
# concat; before, the files needed to be parsed to strip off a header.
- concatOutput("$path/red.red", @successJobs);
+ open(O, "> $path/red.red") or caExit("can't open '$path/red.red' for writing: $!", undef);
+ binmode(O);
+
+ foreach my $f (@successJobs) {
+ fetchFile($f);
+
+ open(F, "< $f") or caExit("can't open '$f' for reading: $!", undef);
+ binmode(F);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "readErrorDetectionCheck");
- buildHTML($WRK, $asm, "utg");
- stopAfter("red");
+ my $buf;
+ my $len = sysread(F, $buf, 1024 * 1024);
+
+ while ($len > 0) {
+ syswrite(O, $buf, $len);
+
+ $len = sysread(F, $buf, 1024 * 1024);
+ }
+
+ close(F);
+ }
+
+ close(O);
+
+ stashFile("$path/red.red");
+
+ foreach my $f (@successJobs) {
+ unlink $f;
+ }
+
+ emitStage($asm, "readErrorDetectionCheck");
+ buildHTML($asm, "utg");
allDone:
}
@@ -329,20 +334,19 @@ sub readErrorDetectionCheck ($$) {
-sub overlapErrorAdjustmentConfigure ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub overlapErrorAdjustmentConfigure ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
- my $path = "$wrk/3-overlapErrorAdjustment";
+ my $path = "unitigging/3-overlapErrorAdjustment";
return if (getGlobal("enableOEA") == 0);
- goto allDone if (skipStage($wrk, $asm, "overlapErrorAdjustmentConfigure") == 1);
- goto allDone if (-e "$path/oea.sh");
+ goto allDone if (fileExists("$path/oea.sh")); # Script exists
+
+ goto allDone if (skipStage($asm, "overlapErrorAdjustmentConfigure") == 1);
- goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues");
- goto allDone if (-d "$wrk/$asm.ctgStore");
+ goto allDone if (fileExists("unitigging/$asm.ovlStore/evalues")); # Stage entrely finished
+ goto allDone if (-d "unitigging/$asm.ctgStore"); # Assembly finished
# OEA uses 1 byte/base + 8 bytes/adjustment + 28 bytes/overlap. We don't know the number of adjustments, but that's
# basically error rate. No adjustment is output for mismatches.
@@ -350,16 +354,16 @@ sub overlapErrorAdjustmentConfigure ($$) {
my @readLengths;
my @numOlaps;
- #print STDERR "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads\n";
- open(F, "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads |");
+ #print STDERR "$bin/gatekeeperDumpMetaData -G unitigging/$asm.gkpStore -reads\n";
+ open(F, "$bin/gatekeeperDumpMetaData -G unitigging/$asm.gkpStore -reads |");
while (<F>) {
my @v = split '\s+', $_;
$readLengths[$v[0]] = $v[2];
}
close(F);
- #print STDERR "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts\n";
- open(F, "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts |");
+ #print STDERR "$bin/ovStoreDump -G unitigging/$asm.gkpStore -O unitigging/$asm.ovlStore -d -counts\n";
+ open(F, "$bin/ovStoreDump -G unitigging/$asm.gkpStore -O unitigging/$asm.ovlStore -d -counts |");
while (<F>) {
my @v = split '\s+', $_;
$numOlaps[$v[0]] = $v[1];
@@ -375,7 +379,7 @@ sub overlapErrorAdjustmentConfigure ($$) {
my $nj = 0;
- my $maxID = getNumberOfReadsInStore($wrk, $asm);
+ my $maxID = getNumberOfReadsInStore("unitigging", $asm);
my $maxMem = getGlobal("oeaMemory") * 1024 * 1024 * 1024;
my $maxReads = getGlobal("oeaBatchSize");
my $maxBases = getGlobal("oeaBatchLength");
@@ -387,7 +391,9 @@ sub overlapErrorAdjustmentConfigure ($$) {
my $bases = 0;
my $olaps = 0;
- my $coverage = getExpectedCoverage($wrk, $asm);
+ fetchFile("$path/red.red");
+
+ my $coverage = getExpectedCoverage("unitigging", $asm);
my $corrSize = (-s "$path/red.red");
my $smallJobs = 0;
@@ -474,6 +480,12 @@ sub overlapErrorAdjustmentConfigure ($$) {
print F "#!" . getGlobal("shell") . "\n\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("unitigging/$asm.gkpStore", $path, "");
+ print F fetchStoreShellCode("unitigging/$asm.ovlStore", $path, "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
@@ -486,32 +498,35 @@ sub overlapErrorAdjustmentConfigure ($$) {
print F "jobid=`printf %04d \$jobid`\n";
print F "\n";
- print F "if [ -e $path/\$jobid.oea ] ; then\n";
+ print F "if [ -e ./\$jobid.oea ] ; then\n";
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
-
- print F getBinDirectoryShellCode();
-
- print F "if [ ! -e $path/\$jobid.oea ] ; then\n";
- print F " \$bin/correctOverlaps \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $wrk/$asm.ovlStore \\\n";
- print F " -R \$minid \$maxid \\\n";
- print F " -e " . getGlobal("utgOvlErrorRate") . " -l " . getGlobal("minOverlapLength") . " \\\n";
- print F " -c $path/red.red \\\n";
- print F " -o $path/\$jobid.oea.WORKING \\\n";
- print F " && \\\n";
- print F " mv $path/\$jobid.oea.WORKING $path/\$jobid.oea\n";
- print F "fi\n";
+ print F "\n";
+ print F fetchFileShellCode("unitigging/3-overlapErrorAdjustment", "red.red", "");
+ print F "\n";
+ print F "\$bin/correctOverlaps \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -O ../$asm.ovlStore \\\n";
+ print F " -R \$minid \$maxid \\\n";
+ print F " -e " . getGlobal("utgOvlErrorRate") . " -l " . getGlobal("minOverlapLength") . " \\\n";
+ print F " -c ./red.red \\\n";
+ print F " -o ./\$jobid.oea.WORKING \\\n";
+ print F "&& \\\n";
+ print F "mv ./\$jobid.oea.WORKING ./\$jobid.oea\n";
+ print F "\n";
+ print F stashFileShellCode("$path", "\$jobid.oea", "");
+ print F "\n";
close(F);
chmod 0755, "$path/oea.sh";
+ stashFile("$path/oea.sh");
+
finishStage:
- emitStage($WRK, $asm, "overlapErrorAdjustmentConfigure");
- buildHTML($WRK, $asm, "utg");
+ emitStage($asm, "overlapErrorAdjustmentConfigure");
+ buildHTML($asm, "utg");
allDone:
}
@@ -520,23 +535,25 @@ sub overlapErrorAdjustmentConfigure ($$) {
-sub overlapErrorAdjustmentCheck ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub overlapErrorAdjustmentCheck ($) {
my $asm = shift @_;
my $attempt = getGlobal("canuIteration");
- my $path = "$wrk/3-overlapErrorAdjustment";
+ my $path = "unitigging/3-overlapErrorAdjustment";
return if (getGlobal("enableOEA") == 0);
- goto allDone if (skipStage($wrk, $asm, "overlapErrorAdjustmentCheck", $attempt) == 1);
- goto allDone if (-e "$path/oea.files");
+ goto allDone if (fileExists("$path/oea.files")); # Output exists
+
+ goto allDone if (skipStage($asm, "overlapErrorAdjustmentCheck", $attempt) == 1);
+
+ goto allDone if (fileExists("unitigging/$asm.ovlStore/evalues")); # Stage entrely finished
+ goto allDone if (-d "unitigging/$asm.ctgStore"); # Assembly finished
# Figure out if all the tasks finished correctly.
my $batchSize = getGlobal("oeaBatchSize");
my $failedJobs = 0;
- my $numReads = getNumberOfReadsInStore($wrk, $asm);
+ my $numReads = getNumberOfReadsInStore("unitigging", $asm);
my $numJobs = 0; #int($numReads / $batchSize) + (($numReads % $batchSize == 0) ? 0 : 1);
# Need to read script to find number of jobs!
@@ -545,17 +562,18 @@ sub overlapErrorAdjustmentCheck ($$) {
my @failedJobs;
my $failureMessage = "";
+ fetchFile("$path/oea.sh");
+
open(A, "< $path/oea.sh") or caExit("can't open '$path/oea.sh' for reading: $!", undef);
while (<A>) {
if (m/if.*jobid\s+=\s+(\d+)\s+.*then/) {
my $ji = substr("0000" . $1, -4);
- my $jn = "$path/$ji.oea";
- if (! -e $jn) {
- $failureMessage .= "-- job $jn FAILED.\n";
+ if (! fileExists("unitigging/3-overlapErrorAdjustment/$ji.oea")) {
+ $failureMessage .= "-- job $ji.oea FAILED.\n";
push @failedJobs, $1;
} else {
- push @successJobs, $jn;
+ push @successJobs, "./$ji.oea";
}
}
}
@@ -582,12 +600,10 @@ sub overlapErrorAdjustmentCheck ($$) {
# Otherwise, run some jobs.
- print STDERR "-- overlap error adjustment attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
+ emitStage($asm, "overlapErrorAdjustmentCheck", $attempt);
+ buildHTML($asm, "utg");
- emitStage($WRK, $asm, "overlapErrorAdjustmentCheck", $attempt);
- buildHTML($WRK, $asm, "utg");
-
- submitOrRunParallelJob($WRK, $asm, "oea", "$path", "oea", @failedJobs);
+ submitOrRunParallelJob($asm, "oea", $path, "oea", @failedJobs);
return;
}
@@ -600,10 +616,10 @@ sub overlapErrorAdjustmentCheck ($$) {
}
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "overlapErrorAdjustmentCheck");
- buildHTML($WRK, $asm, "utg");
- stopAfter("oea");
+ stashFile("$path/oea.files");
+
+ emitStage($asm, "overlapErrorAdjustmentCheck");
+ buildHTML($asm, "utg");
allDone:
}
@@ -611,39 +627,58 @@ sub overlapErrorAdjustmentCheck ($$) {
-sub updateOverlapStore ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub updateOverlapStore ($) {
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
- my $path = "$wrk/3-overlapErrorAdjustment";
+ my $path = "unitigging/3-overlapErrorAdjustment";
return if (getGlobal("enableOEA") == 0);
- goto allDone if (skipStage($wrk, $asm, "updateOverlapStore") == 1);
- goto allDone if (-e "$wrk/$asm.ovlStore/evalues");
+ goto allDone if (skipStage($asm, "updateOverlapStore") == 1);
+
+ goto allDone if (fileExists("unitigging/$asm.ovlStore/evalues")); # Stage entrely finished
+ goto allDone if (-d "unitigging/$asm.ctgStore"); # Assembly finished
- goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues");
- goto allDone if (-d "$wrk/$asm.ctgStore");
+ fetchFile("unitigging/3-overlapErrorAdjustment/oea.files");
+
+ caExit("didn't find '$path/oea.files' to add to store, yet jobs claim to be finished", undef) if (! -e "$path/oea.files");
+
+ open(F, "< $path/oea.files");
+ while (<F>) {
+ chomp;
+ fetchFile("$path/$_");
+ }
+ close(F);
- caExit("didn't find '$path/oea.files' to add to store, yet overlapper finished", undef) if (! -e "$path/oea.files");
+ fetchStore("unitigging/$asm.ovlStore");
$cmd = "$bin/ovStoreBuild \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
+ $cmd .= " -G ../$asm.gkpStore \\\n";
+ $cmd .= " -O ../$asm.ovlStore \\\n";
$cmd .= " -evalues \\\n";
- $cmd .= " -L $path/oea.files \\\n";
- $cmd .= "> $path/oea.apply.err 2>&1";
+ $cmd .= " -L ./oea.files \\\n";
+ $cmd .= "> ./oea.apply.err 2>&1";
- if (runCommand("$path", $cmd)) {
- unlink "$wrk/$asm.ovlStore/evalues";
+ if (runCommand($path, $cmd)) {
+ unlink "unitigging/$asm.ovlStore/evalues";
caExit("failed to add error rates to overlap store", "$path/oea.apply.err");
}
+ stashFile("unitigging/$asm.ovlStore/evalues");
+
+ my $report = "-- No report available.\n";
+
+ #open(F, "< $path/oea.apply.stats") or caExit("Failed to open error rate adjustment statistics in '$path/oea.apply.stats': $!", undef);
+ #while (<F>) {
+ #}
+ #close(F);
+
+ addToReport("adjustments", $report);
+
finishStage:
- emitStage($WRK, $asm, "updateOverlapStore");
- buildHTML($WRK, $asm, "utg");
+ emitStage($asm, "updateOverlapStore");
+ buildHTML($asm, "utg");
allDone:
}
diff --git a/src/pipelines/canu/OverlapInCore.pm b/src/pipelines/canu/OverlapInCore.pm
index 1d2d661..4997172 100644
--- a/src/pipelines/canu/OverlapInCore.pm
+++ b/src/pipelines/canu/OverlapInCore.pm
@@ -44,16 +44,16 @@ require Exporter;
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
+use canu::Report;
use canu::HTML;
+use canu::Grid_Cloud;
-sub overlapConfigure ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub overlapConfigure ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $type = shift @_;
@@ -61,23 +61,30 @@ sub overlapConfigure ($$$$) {
my $bin = getBinDirectory();
my $cmd;
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base;
+ my $path;
- my $path = "$wrk/1-overlapper";
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ $path = "$base/1-overlapper";
caFailure("invalid type '$type'", undef) if (($type ne "partial") && ($type ne "normal"));
- goto allDone if (skipStage($WRK, $asm, "$tag-overlapConfigure") == 1);
- goto allDone if (-e "$path/$asm.partition.ovlopt");
- goto allDone if (-e "$path/ovljob.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ fetchFile("$path/overlap.sh");
+ fetchFile("$path/ovljob.files");
+
+ goto allDone if (skipStage($asm, "$tag-overlapConfigure") == 1);
+ goto allDone if (fileExists("$path/overlap.sh") && fileExists("$path/$asm.partition.ovlbat") && fileExists("$path/$asm.partition.ovljob") && fileExists("$path/$asm.partition.ovlopt"));
+ goto allDone if (fileExists("$path/ovljob.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
print STDERR "--\n";
- print STDERR "-- OVERLAPPER (normal) (correction) erate=", getGlobal("${tag}OvlErrorRate"), "\n" if ($tag eq "cor");
- print STDERR "-- OVERLAPPER (normal) (trimming) erate=", getGlobal("${tag}OvlErrorRate"), "\n" if ($tag eq "obt");
- print STDERR "-- OVERLAPPER (normal) (assembly) erate=", getGlobal("${tag}OvlErrorRate"), "\n" if ($tag eq "utg");
+ print STDERR "-- OVERLAPPER (normal) (correction) erate=", getGlobal("corOvlErrorRate"), "\n" if ($tag eq "cor");
+ print STDERR "-- OVERLAPPER (normal) (trimming) erate=", getGlobal("obtOvlErrorRate"), "\n" if ($tag eq "obt");
+ print STDERR "-- OVERLAPPER (normal) (assembly) erate=", getGlobal("utgOvlErrorRate"), "\n" if ($tag eq "utg");
print STDERR "--\n";
make_path("$path") if (! -d "$path");
@@ -86,7 +93,13 @@ sub overlapConfigure ($$$$) {
# version right before it exits. All we need to do here is check for existence of
# the output, and exit if the command fails.
- if (! -e "$path/$asm.partition.ovlopt") {
+ fetchFile("$path/$asm.partition.ovlbat"); # Don't know where to put these. Before the tests above?
+ fetchFile("$path/$asm.partition.ovljob"); # Here? Just before we use them?
+ fetchFile("$path/$asm.partition.ovlopt");
+
+ if ((! -e "$path/$asm.partition.ovlbat") ||
+ (! -e "$path/$asm.partition.ovljob") ||
+ (! -e "$path/$asm.partition.ovlopt")) {
# These used to be runCA options, but were removed in canu. They were used mostly for
# illumina-pacbio correction, but were also used (or could have been used) during the
@@ -108,7 +121,7 @@ sub overlapConfigure ($$$$) {
}
$cmd = "$bin/overlapInCorePartition \\\n";
- $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -g ../$asm.gkpStore \\\n";
$cmd .= " -bl $hashBlockLength \\\n";
$cmd .= " -bs $hashBlockSize \\\n";
$cmd .= " -rs $refBlockSize \\\n";
@@ -117,12 +130,18 @@ sub overlapConfigure ($$$$) {
#$cmd .= " -R $refLibrary \\\n" if ($refLibrary ne "0");
#$cmd .= " -C \\\n" if (!$checkLibrary);
$cmd .= " -ol $minOlapLength \\\n";
- $cmd .= " -o $path/$asm.partition \\\n";
- $cmd .= "> $path/$asm.partition.err 2>&1";
+ $cmd .= " -o ./$asm.partition \\\n";
+ $cmd .= "> ./$asm.partition.err 2>&1";
- if (runCommand($wrk, $cmd)) {
+ if (runCommand($path, $cmd)) {
caExit("failed partition for overlapper", undef);
}
+
+ stashFile("$path/$asm.partition.ovlbat");
+ stashFile("$path/$asm.partition.ovljob");
+ stashFile("$path/$asm.partition.ovlopt");
+
+ unlink "$path/overlap.sh";
}
open(BAT, "< $path/$asm.partition.ovlbat") or caExit("can't open '$path/$asm.partition.ovlbat' for reading: $!", undef);
@@ -137,6 +156,8 @@ sub overlapConfigure ($$$$) {
close(JOB);
close(OPT);
+ fetchFile("$path/overlap.sh");
+
if (! -e "$path/overlap.sh") {
my $merSize = getGlobal("${tag}OvlMerSize");
@@ -154,6 +175,11 @@ sub overlapConfigure ($$$$) {
print F "\n";
print F "perl='/usr/bin/env perl'\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/1-overlapper", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
@@ -167,63 +193,69 @@ sub overlapConfigure ($$$$) {
}
print F "\n";
- print F "if [ ! -d $path/\$bat ]; then\n";
- print F " mkdir $path/\$bat\n";
+ print F "if [ ! -d ./\$bat ]; then\n";
+ print F " mkdir ./\$bat\n";
print F "fi\n";
print F "\n";
- print F "if [ -e $path/\$job.ovb ]; then\n";
+ print F fileExistsShellCode("./\$job.ovb");
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
+ print F fetchFileShellCode("$base/0-mercounts", "$asm.ms$merSize.frequentMers.fasta", "");
print F "\n";
print F "\$bin/overlapInCore \\\n";
print F " -G \\\n" if ($type eq "partial");
print F " -t ", getGlobal("${tag}OvlThreads"), " \\\n";
print F " -k $merSize \\\n";
- print F " -k $wrk/0-mercounts/$asm.ms$merSize.frequentMers.fasta \\\n";
+ print F " -k ../0-mercounts/$asm.ms$merSize.frequentMers.fasta \\\n";
print F " --hashbits $hashBits \\\n";
print F " --hashload $hashLoad \\\n";
- print F " --maxerate ", getGlobal("${tag}OvlErrorRate"), " \\\n";
+ print F " --maxerate ", getGlobal("corOvlErrorRate"), " \\\n" if ($tag eq "cor"); # Explicitly using proper name for grepability.
+ print F " --maxerate ", getGlobal("obtOvlErrorRate"), " \\\n" if ($tag eq "obt");
+ print F " --maxerate ", getGlobal("utgOvlErrorRate"), " \\\n" if ($tag eq "utg");
print F " --minlength ", getGlobal("minOverlapLength"), " \\\n";
print F " --minkmers \\\n" if (defined(getGlobal("${tag}OvlFilter")) && getGlobal("${tag}OvlFilter")==1);
print F " \$opt \\\n";
- print F " -o $path/\$job.ovb.WORKING \\\n";
- print F " -s $path/\$job.stats \\\n";
+ print F " -o ./\$job.ovb.WORKING \\\n";
+ print F " -s ./\$job.stats \\\n";
#print F " -H $hashLibrary \\\n" if ($hashLibrary ne "0");
#print F " -R $refLibrary \\\n" if ($refLibrary ne "0");
- print F " $wrk/$asm.gkpStore \\\n";
+ print F " ../$asm.gkpStore \\\n";
print F "&& \\\n";
- print F "mv $path/\$job.ovb.WORKING $path/\$job.ovb\n";
+ print F "mv ./\$job.ovb.WORKING ./\$job.ovb\n";
+ print F "\n";
+ print F stashFileShellCode("$base/1-overlapper/", "\$job.ovb", "");
+ print F stashFileShellCode("$base/1-overlapper/", "\$job.counts", "");
+ print F stashFileShellCode("$base/1-overlapper/", "\$job.stats", "");
print F "\n";
print F "exit 0\n";
close(F);
system("chmod +x $path/overlap.sh");
+
+ stashFile("$path/overlap.sh");
}
my $jobs = scalar(@job);
my $batchName = $bat[$jobs-1]; chomp $batchName;
my $jobName = $job[$jobs-1]; chomp $jobName;
- if (-e "$path/overlap.sh") {
- my $numJobs = 0;
- open(F, "< $path/overlap.sh") or caExit("can't open '$path/overlap.sh' for reading: $!", undef);
- while (<F>) {
- $numJobs++ if (m/^\s+job=/);
- }
- close(F);
- print STDERR "--\n";
- print STDERR "-- Configured $numJobs overlapInCore jobs.\n";
+ my $numJobs = 0;
+ open(F, "< $path/overlap.sh") or caExit("can't open '$path/overlap.sh' for reading: $!", undef);
+ while (<F>) {
+ $numJobs++ if (m/^\s+job=/);
}
+ close(F);
+ print STDERR "--\n";
+ print STDERR "-- Configured $numJobs overlapInCore jobs.\n";
finishStage:
- emitStage($WRK, $asm, "$tag-overlapConfigure");
- buildHTML($WRK, $asm, $tag);
- stopAfter("overlapConfigure");
+ emitStage($asm, "$tag-overlapConfigure");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlapConfigure");
}
@@ -249,7 +281,7 @@ sub reportSumMeanStdDev (@) {
sub reportOverlapStats ($$@) {
- my $wrk = shift @_; # Local work directory
+ my $base = shift @_;
my $asm = shift @_;
my @statsJobs = @_;
@@ -263,7 +295,9 @@ sub reportOverlapStats ($$@) {
my @longReject;
foreach my $s (@statsJobs) {
- open(F, "< $s") or caExit("can't open '$s' for reading: $!", undef);
+ fetchFile("$base/$s");
+
+ open(F, "< $base/$s") or caExit("can't open '$base/$s' for reading: $!", undef);
$_ = <F>; push @hitsWithoutOlaps, $1 if (m/^\s*Kmer\shits\swithout\solaps\s=\s(\d+)$/);
$_ = <F>; push @hitsWithOlaps, $1 if (m/^\s*Kmer\shits\swith\solaps\s=\s(\d+)$/);
@@ -278,7 +312,7 @@ sub reportOverlapStats ($$@) {
}
printf STDERR "--\n";
- printf STDERR "-- overlapInCore compute '$wrk/1-overlapper':\n";
+ printf STDERR "-- overlapInCore compute '$base/1-overlapper':\n";
printf STDERR "-- kmer hits\n";
printf STDERR "-- with no overlap %12d %s\n", reportSumMeanStdDev(@hitsWithoutOlaps);
printf STDERR "-- with an overlap %12d %s\n", reportSumMeanStdDev(@hitsWithOlaps);
@@ -297,23 +331,25 @@ sub reportOverlapStats ($$@) {
# Check that the overlapper jobs properly executed. If not,
# complain, but don't help the user fix things.
#
-sub overlapCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub overlapCheck ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $type = shift @_;
my $attempt = getGlobal("canuIteration");
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base;
+ my $path;
+
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
- my $path = "$wrk/1-overlapper";
+ $path = "$base/1-overlapper";
- goto allDone if (skipStage($WRK, $asm, "$tag-overlapCheck", $attempt) == 1);
- goto allDone if (-e "$path/ovljob.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ goto allDone if (skipStage($asm, "$tag-overlapCheck", $attempt) == 1);
+ goto allDone if (fileExists("$path/ovljob.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
# Figure out if all the tasks finished correctly.
@@ -324,36 +360,38 @@ sub overlapCheck ($$$$) {
my @failedJobs;
my $failureMessage = "";
+ fetchFile("$path/overlap.sh");
+
open(F, "< $path/overlap.sh") or caExit("can't open '$path/overlap.sh' for reading: $!", undef);
while (<F>) {
if (m/^\s+job=\"(\d+\/\d+)\"$/) {
- if (-e "$path/$1.ovb.gz") {
- push @successJobs, "$path/$1.ovb.gz\n"; # Dumped to a file, so include \n
- push @statsJobs, "$path/$1.stats"; # Used here, don't include \n
- push @miscJobs, "$path/$1.stats\n";
- push @miscJobs, "$path/$1.counts\n";
-
- } elsif (-e "$path/$1.ovb") {
- push @successJobs, "$path/$1.ovb\n";
- push @statsJobs, "$path/$1.stats";
- push @miscJobs, "$path/$1.stats\n";
- push @miscJobs, "$path/$1.counts\n";
-
- } elsif (-e "$path/$1.ovb.bz2") {
- push @successJobs, "$path/$1.ovb.bz2\n";
- push @statsJobs, "$path/$1.stats";
- push @miscJobs, "$path/$1.stats\n";
- push @miscJobs, "$path/$1.counts\n";
-
- } elsif (-e "$path/$1.ovb.xz") {
- push @successJobs, "$path/$1.ovb.xz\n";
- push @statsJobs, "$path/$1.stats";
- push @miscJobs, "$path/$1.stats\n";
- push @miscJobs, "$path/$1.counts\n";
+ if (fileExists("$path/$1.ovb.gz")) {
+ push @successJobs, "1-overlapper/$1.ovb.gz\n"; # Dumped to a file, so include \n
+ push @statsJobs, "1-overlapper/$1.stats"; # Used here, don't include \n
+ push @miscJobs, "1-overlapper/$1.stats\n";
+ push @miscJobs, "1-overlapper/$1.counts\n";
+
+ } elsif (fileExists("$path/$1.ovb")) {
+ push @successJobs, "1-overlapper/$1.ovb\n";
+ push @statsJobs, "1-overlapper/$1.stats";
+ push @miscJobs, "1-overlapper/$1.stats\n";
+ push @miscJobs, "1-overlapper/$1.counts\n";
+
+ } elsif (fileExists("$path/$1.ovb.bz2")) {
+ push @successJobs, "1-overlapper/$1.ovb.bz2\n";
+ push @statsJobs, "1-overlapper/$1.stats";
+ push @miscJobs, "1-overlapper/$1.stats\n";
+ push @miscJobs, "1-overlapper/$1.counts\n";
+
+ } elsif (fileExists("$path/$1.ovb.xz")) {
+ push @successJobs, "1-overlapper/$1.ovb.xz\n";
+ push @statsJobs, "1-overlapper/$1.stats";
+ push @miscJobs, "1-overlapper/$1.stats\n";
+ push @miscJobs, "1-overlapper/$1.counts\n";
} else {
- $failureMessage .= "-- job $path/$1 FAILED.\n";
+ $failureMessage .= "-- job 1-overlapper/$1.ovb FAILED.\n";
push @failedJobs, $currentJobID;
}
@@ -384,12 +422,10 @@ sub overlapCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- overlapInCore attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-overlapCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
+ emitStage($asm, "$tag-overlapCheck", $attempt);
+ buildHTML($asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "${tag}ovl", $path, "overlap", @failedJobs);
+ submitOrRunParallelJob($asm, "${tag}ovl", $path, "overlap", @failedJobs);
return;
}
@@ -404,13 +440,15 @@ sub overlapCheck ($$$$) {
print L @miscJobs;
close(L);
- reportOverlapStats($wrk, $asm, @statsJobs);
+ stashFile("$path/ovljob.files");
+ stashFile("$path/ovljob.more.files");
+
+ reportOverlapStats($base, $asm, @statsJobs);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-overlapCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("overlapper");
+ emitStage($asm, "$tag-overlapCheck");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlap");
}
diff --git a/src/pipelines/canu/OverlapMMap.pm b/src/pipelines/canu/OverlapMMap.pm
index 91aebd7..a190189 100644
--- a/src/pipelines/canu/OverlapMMap.pm
+++ b/src/pipelines/canu/OverlapMMap.pm
@@ -36,35 +36,38 @@ require Exporter;
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
use canu::HTML;
+use canu::Grid_Cloud;
# Map long reads to long reads with minimap.
-sub mmapConfigure ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub mmapConfigure ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $typ = shift @_;
my $bin = getBinDirectory();
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base; # e.g., $base/1-overlapper/mhap.sh
+ my $path; # e.g., $path/mhap.sh
- my $path = "$wrk/1-overlapper";
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ $path = "$base/1-overlapper";
caFailure("invalid type '$typ'", undef) if (($typ ne "partial") && ($typ ne "normal"));
- goto allDone if (skipStage($WRK, $asm, "$tag-mmapConfigure") == 1);
- goto allDone if (-e "$path/mmap.sh");
- goto allDone if (-e "$path/ovljob.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ goto allDone if (skipStage($asm, "$tag-mmapConfigure") == 1);
+ goto allDone if (fileExists("$path/precompute.sh")) && (fileExists("$path/mmap.sh"));
+ goto allDone if (fileExists("$path/ovljob.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
print STDERR "--\n";
print STDERR "-- OVERLAPPER (mmap) (correction)\n" if ($tag eq "cor");
@@ -72,13 +75,13 @@ sub mmapConfigure ($$$$) {
print STDERR "-- OVERLAPPER (mmap) (assembly)\n" if ($tag eq "utg");
print STDERR "--\n";
- make_path("$path") if (! -d "$path");
+ make_path($path) if (! -d $path);
# Constants.
my $merSize = getGlobal("${tag}MMapMerSize");
- my $numReads = getNumberOfReadsInStore($wrk, $asm);
+ my $numReads = getNumberOfReadsInStore($base, $asm);
my $memorySize = getGlobal("${tag}mmapMemory");
my $blockPerGb = getGlobal("${tag}MMapBlockSize");
my $blockSize = int($blockPerGb * $memorySize);
@@ -174,7 +177,7 @@ sub mmapConfigure ($$$$) {
for (my $qid=$qbgn; $qid <= $qend; $qid++) {
my $qry = substr("000000" . $qid, -6); # Name for the query block
- symlink("$path/blocks/$qry.fasta", "$path/queries/$job/$qry.fasta");
+ symlink("../../blocks/$qry.fasta", "$path/queries/$job/$qry.fasta");
}
} else {
@@ -190,12 +193,27 @@ sub mmapConfigure ($$$$) {
close(L);
+ # Tar up the queries directory. Only useful for cloud support.
+
+ runCommandSilently($path, "tar -cf queries.tar queries", 1);
+ stashFile("$path/queries.tar");
+
# Create a script to generate precomputed blocks, including extracting the reads from gkpStore.
+ #OPTIMIZE
+ #OPTIMIZE Probably a big optimization for cloud assemblies, the block fasta inputs can be
+ #OPTIMIZE computed ahead of time, stashed, and then fetched to do the actual precompute.
+ #OPTIMIZE
+
open(F, "> $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for writing: $!", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/1-overlapper", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
for (my $ii=1; $ii < scalar(@blocks); $ii++) {
@@ -211,27 +229,26 @@ sub mmapConfigure ($$$$) {
print F " exit 1\n";
print F "fi\n";
print F "\n";
- print F "if [ ! -d $path/blocks ]; then\n";
- print F " mkdir $path/blocks\n";
+ print F "if [ ! -d ./blocks ]; then\n";
+ print F " mkdir -p ./blocks\n";
print F "fi\n";
print F "\n";
- print F "if [ -e $path/blocks/\$job.fasta ]; then\n";
+ print F fileExistsShellCode("./blocks/\$job.fasta");
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
- print F "\n";
print F "\$bin/gatekeeperDumpFASTQ \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
print F " \$rge \\\n";
print F " -nolibname \\\n";
print F " -noreadname \\\n";
print F " -fasta \\\n";
- print F " -o $path/blocks/\$job \\\n";
- print F "|| \\\n";
- print F "mv -f $path/blocks/\$job.fasta $path/blocks/\$job.fasta.FAILED\n";
+ print F " -o ./blocks/\$job.input \\\n";
+ print F "&& \\\n";
+ print F "mv -f ./blocks/\$job.input.fasta ./blocks/\$job.fasta\n";
print F "\n";
+ print F stashFileShellCode("$base/1-overlapper/blocks", "\$job.fasta", "");
print F "\n";
print F "exit 0\n";
@@ -243,6 +260,11 @@ sub mmapConfigure ($$$$) {
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/1-overlapper", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
for (my $ii=1; $ii < scalar(@hashes); $ii++) {
@@ -260,25 +282,37 @@ sub mmapConfigure ($$$$) {
print F " exit 1\n";
print F "fi\n";
print F "\n";
- print F "if [ -e $path/results/\$qry.ovb ]; then\n";
+ print F "if [ -e ./results/\$qry.ovb ]; then\n";
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
print F "\n";
- print F "if [ ! -d $path/results ]; then\n";
- print F " mkdir $path/results\n";
+
+ print F fetchFileShellCode("$path", "queries.tar", "");
+ print F "\n";
+ print F "if [ -e ./queries.tar -a ! -d ./queries ] ; then\n";
+ print F " tar -xf ./queries.tar\n";
print F "fi\n";
print F "\n";
- print F "echo Running block \$blk in query \$qry\n";
-
+ print F "if [ ! -d ./results ]; then\n";
+ print F " mkdir -p ./results\n";
+ print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
+
+ print F fetchFileShellCode("$path", "blocks/\$blk.fasta", "");
+
+ print F "for ii in `ls ./queries/\$qry` ; do\n";
+ print F " echo Fetch blocks/\$ii\n";
+ print F fetchFileShellCode("$path", "blocks/\$ii", " ");
+ print F "done\n";
print F "\n";
- # begin comparison, we loop through query and compare current block to it, if we need to do self first compare to self, otherwise initialize as empty
+ # Begin comparison, we loop through query and compare current block to it, if we need to do
+ # self first compare to self, otherwise initialize as empty
+
print F "if [ x\$slf = x ]; then\n";
- print F " > $path/results/\$qry.mmap.WORKING\n";
+ print F " > ./results/\$qry.mmap.WORKING\n";
print F "else\n";
print F " \$bin/minimap \\\n";
print F " -k $merSize \\\n";
@@ -286,66 +320,69 @@ sub mmapConfigure ($$$$) {
print F " -L100 \\\n";
print F " -m0 \\\n";
print F " -t ", getGlobal("${tag}mmapThreads"), " \\\n";
- print F " $path/blocks/\$blk.fasta \\\n";
- print F " $path/blocks/\$blk.fasta \\\n";
- print F " > $path/results/\$qry.mmap.WORKING \n";
+ print F " ./blocks/\$blk.fasta \\\n";
+ print F " ./blocks/\$blk.fasta \\\n";
+ print F " > ./results/\$qry.mmap.WORKING \n";
print F " \n";
print F "fi\n";
print F "\n";
- print F "for file in `ls $path/queries/\$qry/*.fasta`; do\n";
+ print F "for file in `ls queries/\$qry/*.fasta`; do\n";
print F " \$bin/minimap \\\n";
print F " -k $merSize \\\n";
print F " -Sw5 \\\n";
print F " -L100 \\\n";
print F " -m0 \\\n";
print F " -t ", getGlobal("${tag}mmapThreads"), " \\\n";
- print F " $path/blocks/\$blk.fasta \\\n";
+ print F " ./blocks/\$blk.fasta \\\n";
print F " \$file \\\n";
- print F " >> $path/results/\$qry.mmap.WORKING \n";
+ print F " >> ./results/\$qry.mmap.WORKING \n";
print F "done\n";
print F "\n";
- print F "mv $path/results/\$qry.mmap.WORKING $path/results/\$qry.mmap\n";
+ print F "mv ./results/\$qry.mmap.WORKING ./results/\$qry.mmap\n";
print F "\n";
- print F "if [ -e \"$path/results/\$qry.mmap\" -a \\\n";
- print F " ! -e \"$path/results/\$qry.ovb\" ] ; then\n";
+ print F "if [ -e ./results/\$qry.mmap -a \\\n";
+ print F " ! -e ./results/\$qry.ovb ] ; then\n";
print F " \$bin/mmapConvert \\\n";
- print F " -o $path/results/\$qry.mmap.ovb.WORKING \\\n";
- print F " $path/results/\$qry.mmap \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -o ./results/\$qry.mmap.ovb.WORKING \\\n";
+ print F " ./results/\$qry.mmap \\\n";
print F " && \\\n";
- print F " mv $path/results/\$qry.mmap.ovb.WORKING $path/results/\$qry.mmap.ovb\n";
+ print F " mv ./results/\$qry.mmap.ovb.WORKING ./results/\$qry.mmap.ovb\n";
print F "fi\n";
print F "\n";
if (getGlobal('saveOverlaps') eq "0") {
- print F "if [ -e \"$path/results/\$qry.mmap\" -a \\\n";
- print F " -e \"$path/results/\$qry.mmap.ovb\" ] ; then\n";
- print F " rm -f $path/results/\$qry.mmap\n";
+ print F "if [ -e ./results/\$qry.mmap -a \\\n";
+ print F " -e ./results/\$qry.mmap.ovb ] ; then\n";
+ print F " rm -f ./results/\$qry.mmap\n";
print F "fi\n";
print F "\n";
}
- print F "if [ -e \"$path/results/\$qry.mmap.ovb\" ] ; then\n";
- if (getGlobal("${tag}ReAlign") eq "raw") {
+ print F "if [ -e ./results/\$qry.mmap.ovb ] ; then\n";
+ if (getGlobal("${tag}ReAlign") eq "1") {
print F " \$bin/overlapPair \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $path/results/\$qry.mmap.ovb \\\n";
- print F " -o $path/results/\$qry.ovb \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -O ./results/\$qry.mmap.ovb \\\n";
+ print F " -o ./results/\$qry.ovb \\\n";
print F " -partial \\\n" if ($typ eq "partial");
- print F " -erate ", getGlobal("corErrorRate"), " \\\n" if ($tag eq "cor");
+ print F " -erate ", getGlobal("corOvlErrorRate"), " \\\n" if ($tag eq "cor"); # Explicitly using proper name for grepability.
print F " -erate ", getGlobal("obtOvlErrorRate"), " \\\n" if ($tag eq "obt");
print F " -erate ", getGlobal("utgOvlErrorRate"), " \\\n" if ($tag eq "utg");
print F " -memory " . getGlobal("${tag}mmapMemory") . " \\\n";
print F " -t " . getGlobal("${tag}mmapThreads") . " \n";
} else {
- print F " mv -f \"$path/results/\$qry.mmap.ovb\" \"$path/results/\$qry.ovb\"\n";
+ print F " mv -f ./results/\$qry.mmap.ovb ./results/\$qry.ovb\n";
+ print F " mv -f ./results/\$qry.mmap.counts ./results/\$qry.counts\n";
}
print F "fi\n";
+ print F stashFileShellCode("$path", "results/\$qry.ovb", "");
+ print F stashFileShellCode("$path", "results/\$qry.counts", "");
print F "\n";
- print F "\n";
- #print F "rm -rf $path/queries/\$qry\n";
+
print F "\n";
print F "exit 0\n";
@@ -366,39 +403,46 @@ sub mmapConfigure ($$$$) {
my $numJobs = 0;
open(F, "< $path/mmap.sh") or caFailure("can't open '$path/mmap.sh' for reading: $!", undef);
while (<F>) {
- $numJobs++ if (m/^\s+qry=\"(\d+)\"$/);
+ $numJobs++ if (m/^\s+qry=/);
}
close(F);
print STDERR "-- Configured $numJobs mmap overlap jobs.\n";
}
+ stashFile("$path/precompute.sh");
+ stashFile("$path/mhap.sh");
+
finishStage:
- emitStage($WRK, $asm, "$tag-mmapConfigure");
- buildHTML($WRK, $asm, $tag);
- stopAfter("mmapConfigure");
+ emitStage($asm, "$tag-mmapConfigure");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlapConfigure");
}
-sub mmapPrecomputeCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub mmapPrecomputeCheck ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $typ = shift @_;
my $attempt = getGlobal("canuIteration");
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base; # e.g., $base/1-overlapper/mhap.sh
+ my $path; # e.g., $path/mhap.sh
+
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
- my $path = "$wrk/1-overlapper";
+ $path = "$base/1-overlapper";
- goto allDone if (skipStage($WRK, $asm, "$tag-mmapPrecomputeCheck", $attempt) == 1);
- goto allDone if (-e "$path/precompute.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ goto allDone if (skipStage($asm, "$tag-mmapPrecomputeCheck", $attempt) == 1);
+ goto allDone if (fileExists("$path/precompute.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
+
+ fetchFile("$path/precompute.sh");
# Figure out if all the tasks finished correctly.
@@ -410,10 +454,10 @@ sub mmapPrecomputeCheck ($$$$) {
open(F, "< $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for reading: $!", undef);
while (<F>) {
if (m/^\s+job=\"(\d+)\"$/) {
- if (-e "$path/blocks/$1.fasta") {
- push @successJobs, "$path/blocks/$1.fasta\n";
+ if (fileExists("$path/blocks/$1.fasta")) {
+ push @successJobs, "1-overlapper/blocks/$1.fasta\n";
} else {
- $failureMessage .= "-- job $path/blocks/$1.fasta FAILED.\n";
+ $failureMessage .= "-- job 1-overlapper/blocks/$1.fasta FAILED.\n";
push @failedJobs, $currentJobID;
}
@@ -443,12 +487,10 @@ sub mmapPrecomputeCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- mmap precompute attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-mmapPrecomputeCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
+ emitStage($asm, "$tag-mmapPrecomputeCheck", $attempt);
+ buildHTML($asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "${tag}mmap", $path, "precompute", @failedJobs);
+ submitOrRunParallelJob($asm, "${tag}mmap", $path, "precompute", @failedJobs);
return;
}
@@ -459,32 +501,37 @@ sub mmapPrecomputeCheck ($$$$) {
print L @successJobs;
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-mmapPrecomputeCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("mmapPrecompute");
+ stashFile("$path/precompute.files");
+
+ emitStage($asm, "$tag-mmapPrecomputeCheck");
+ buildHTML($asm, $tag);
allDone:
}
-sub mmapCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+
+sub mmapCheck ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $typ = shift @_;
my $attempt = getGlobal("canuIteration");
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base; # e.g., $base/1-overlapper/mhap.sh
+ my $path; # e.g., $path/mhap.sh
- my $path = "$wrk/1-overlapper";
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
- goto allDone if (skipStage($WRK, $asm, "$tag-mmapCheck", $attempt) == 1);
- goto allDone if (-e "$path/mmap.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ $path = "$base/1-overlapper";
+
+ goto allDone if (skipStage($asm, "$tag-mmapCheck", $attempt) == 1);
+ goto allDone if (fileExists("$path/mmap.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
+
+ fetchFile("$path/mmap.sh");
# Figure out if all the tasks finished correctly.
@@ -498,32 +545,32 @@ sub mmapCheck ($$$$) {
open(F, "< $path/mmap.sh") or caExit("failed to open '$path/mmap.sh'", undef);
while (<F>) {
if (m/^\s+qry=\"(\d+)\"$/) {
- if (-e "$path/results/$1.ovb.gz") {
- push @mmapJobs, "$path/results/$1.mmap\n";
- push @successJobs, "$path/results/$1.ovb.gz\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
-
- } elsif (-e "$path/results/$1.ovb") {
- push @mmapJobs, "$path/results/$1.mmap\n";
- push @successJobs, "$path/results/$1.ovb\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
-
- } elsif (-e "$path/results/$1.ovb.bz2") {
- push @mmapJobs, "$path/results/$1.mmap\n";
- push @successJobs, "$path/results/$1.ovb.bz2\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
-
- } elsif (-e "$path/results/$1.ovb.xz") {
- push @mmapJobs, "$path/results/$1.mmap\n";
- push @successJobs, "$path/results/$1.ovb.xz\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
+ if (fileExists("$path/results/$1.ovb.gz")) {
+ push @mmapJobs, "1-overlapper/results/$1.mmap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb.gz\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
+
+ } elsif (fileExists("$path/results/$1.ovb")) {
+ push @mmapJobs, "1-overlapper/results/$1.mmap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
+
+ } elsif (fileExists("$path/results/$1.ovb.bz2")) {
+ push @mmapJobs, "1-overlapper/results/$1.mmap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb.bz2\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
+
+ } elsif (fileExists("$path/results/$1.ovb.xz")) {
+ push @mmapJobs, "1-overlapper/results/$1.mmap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb.xz\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
} else {
- $failureMessage .= "-- job $path/results/$1.ovb FAILED.\n";
+ $failureMessage .= "-- job 1-overlapper/results/$1.ovb FAILED.\n";
push @failedJobs, $currentJobID;
}
@@ -562,11 +609,9 @@ sub mmapCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- mmap attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-mmapCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "${tag}mmap", $path, "mmap", @failedJobs);
+ emitStage($asm, "$tag-mmapCheck", $attempt);
+ buildHTML($asm, $tag);
+ submitOrRunParallelJob($asm, "${tag}mmap", $path, "mmap", @failedJobs);
return;
}
@@ -585,11 +630,14 @@ sub mmapCheck ($$$$) {
print L @miscJobs;
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-mmapCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("mmapOverlap");
+ stashFile("$path/mmap.files");
+ stashFile("$path/ovljob.files");
+ stashFile("$path/ovljob.more.files");
+
+ emitStage($asm, "$tag-mmapCheck");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlap");
}
diff --git a/src/pipelines/canu/OverlapMhap.pm b/src/pipelines/canu/OverlapMhap.pm
index 7d51772..caee4bb 100644
--- a/src/pipelines/canu/OverlapMhap.pm
+++ b/src/pipelines/canu/OverlapMhap.pm
@@ -45,12 +45,13 @@ require Exporter;
use strict;
use POSIX qw(floor);
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use canu::Defaults;
use canu::Execution;
use canu::Gatekeeper;
use canu::HTML;
+use canu::Grid_Cloud;
# Map long reads to long reads with mhap.
@@ -60,26 +61,28 @@ use canu::HTML;
-sub mhapConfigure ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub mhapConfigure ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $typ = shift @_;
my $bin = getBinDirectory();
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base; # e.g., $base/1-overlapper/mhap.sh
+ my $path; # e.g., $path/mhap.sh
- my $path = "$wrk/1-overlapper";
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ $path = "$base/1-overlapper";
caFailure("invalid type '$typ'", undef) if (($typ ne "partial") && ($typ ne "normal"));
- goto allDone if (skipStage($WRK, $asm, "$tag-mhapConfigure") == 1);
- goto allDone if (-e "$path/precompute.sh") && (-e "$path/mhap.sh");
- goto allDone if (-e "$path/ovljob.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ goto allDone if (skipStage($asm, "$tag-mhapConfigure") == 1);
+ goto allDone if (fileExists("$path/precompute.sh")) && (fileExists("$path/mhap.sh"));
+ goto allDone if (fileExists("$path/ovljob.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
print STDERR "--\n";
print STDERR "-- OVERLAPPER (mhap) (correction)\n" if ($tag eq "cor");
@@ -87,14 +90,14 @@ sub mhapConfigure ($$$$) {
print STDERR "-- OVERLAPPER (mhap) (assembly)\n" if ($tag eq "utg");
print STDERR "--\n";
- make_path("$path") if (! -d "$path");
+ make_path($path) if (! -d $path);
- # Mhap parameters - filterThreshold needs to be a string, else it is printed as 5e-06.
+ # Mhap parameters
my ($numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer);
if (!defined(getGlobal("${tag}MhapSensitivity"))) {
- my $cov = getExpectedCoverage($wrk, $asm);
+ my $cov = getExpectedCoverage($base, $asm);
setGlobal("${tag}MhapSensitivity", "low"); # Yup, super inefficient. The code is
setGlobal("${tag}MhapSensitivity", "normal") if ($cov < 60); # compact and clear and runs once.
@@ -128,13 +131,23 @@ sub mhapConfigure ($$$$) {
caFailure("invalid ${tag}MhapSensitivity=" . getGlobal("${tag}MhapSensitivity"), undef);
}
+ # due to systematic bias in nanopore data, adjust threshold up by 5%
+ my $numNanoporeRaw = 0;
+ open(L, "< $base/$asm.gkpStore/libraries.txt") or caExit("can't open '$base/$asm.gkpStore/libraries.txt' for reading: $!", undef);
+ while (<L>) {
+ $numNanoporeRaw++ if (m/nanopore-raw/);
+ }
+ close(L);
+
+ $threshold += 0.05 if ($numNanoporeRaw > 0);
+
my $filterThreshold = getGlobal("${tag}MhapFilterThreshold");
# Constants.
my $merSize = getGlobal("${tag}MhapMerSize");
- my $numReads = getNumberOfReadsInStore($wrk, $asm);
+ my $numReads = getNumberOfReadsInStore($base, $asm);
my $memorySize = getGlobal("${tag}mhapMemory");
my $blockPerGb = getGlobal("${tag}MhapBlockSize");
if ($numHashes >= 768) {
@@ -259,7 +272,7 @@ sub mhapConfigure ($$$$) {
for (my $qid=$qbgn; $qid <= $qend; $qid++) {
my $qry = substr("000000" . $qid, -6); # Name for the query block
- symlink("$path/blocks/$qry.dat", "$path/queries/$job/$qry.dat");
+ symlink("../../blocks/$qry.dat", "$path/queries/$job/$qry.dat");
}
} else {
@@ -284,20 +297,33 @@ sub mhapConfigure ($$$$) {
close(L);
- # The ignore file is created in Meryl.pm
+ # Tar up the queries directory. Only useful for cloud support.
+ runCommandSilently($path, "tar -cf queries.tar queries", 1);
+ stashFile("$path/queries.tar");
# Create a script to generate precomputed blocks, including extracting the reads from gkpStore.
- #getAllowedResources($tag, "mhap");
+ #OPTIMIZE
+ #OPTIMIZE Probably a big optimization for cloud assemblies, the block fasta inputs can be
+ #OPTIMIZE computed ahead of time, stashed, and then fetched to do the actual precompute.
+ #OPTIMIZE
- my $javaPath = getGlobal("java");
+ my $javaPath = getGlobal("java");
my $javaMemory = int(getGlobal("${tag}mhapMemory") * 1024 + 0.5);
+ my $cygA = ($^O ne "cygwin") ? "" : "\$(cygpath -w "; # Some baloney to convert nice POSIX paths into
+ my $cygB = ($^O ne "cygwin") ? "" : ")"; # whatever abomination Windows expects.
+
open(F, "> $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for writing: $!", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/1-overlapper", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
for (my $ii=1; $ii < scalar(@blocks); $ii++) {
@@ -313,44 +339,46 @@ sub mhapConfigure ($$$$) {
print F " exit 1\n";
print F "fi\n";
print F "\n";
- print F "if [ ! -d $path/blocks ]; then\n";
- print F " mkdir $path/blocks\n";
+ print F "if [ ! -d ./blocks ]; then\n";
+ print F " mkdir -p ./blocks\n";
print F "fi\n";
print F "\n";
- print F "if [ -e $path/blocks/\$job.dat ]; then\n";
+ print F fileExistsShellCode("./blocks/\$job.dat");
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
print F "\n";
- print F "# If the fasta exists, our job failed, and we should try again.\n";
- print F "if [ -e \"$path/blocks/\$job.fasta\" ] ; then\n";
- print F " rm -f $path/blocks/\$job.dat\n";
- print F "fi\n";
+ print F "# Remove any previous result.\n";
+ print F "rm -f ./blocks/\$job.input.dat\n";
print F "\n";
- print F getBinDirectoryShellCode();
+ #print F fetchFileShellCode("./blocks/\$job.input.fasta");
print F "\n";
print F "\$bin/gatekeeperDumpFASTQ \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
print F " \$rge \\\n";
print F " -nolibname \\\n";
print F " -fasta \\\n";
- print F " -o $path/blocks/\$job \\\n";
+ print F " -o ./blocks/\$job.input \\\n";
print F "|| \\\n";
- print F "mv -f $path/blocks/\$job.fasta $path/blocks/\$job.fasta.FAILED\n";
+ print F "mv -f ./blocks/\$job.input.fasta ./blocks/\$job.input.fasta.FAILED\n";
print F "\n";
print F "\n";
- print F "if [ ! -e \"$path/blocks/\$job.fasta\" ] ; then\n";
+ print F "if [ ! -e ./blocks/\$job.input.fasta ] ; then\n";
print F " echo Failed to extract fasta.\n";
print F " exit 1\n";
print F "fi\n";
print F "\n";
+ print F fetchFileShellCode("$base/0-mercounts", "$asm.ms$merSize.frequentMers.ignore.gz", "");
+ print F "\n";
+ print F "echo \"\"\n";
print F "echo Starting mhap precompute.\n";
+ print F "echo \"\"\n";
print F "\n";
print F "# So mhap writes its output in the correct spot.\n";
- print F "cd $path/blocks\n";
+ print F "cd ./blocks\n";
print F "\n";
print F "$javaPath -d64 -server -Xmx", $javaMemory, "m \\\n";
- print F " -jar " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "\$bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar " . ($^O eq "cygwin" ? ")" : "") . "\\\n";
+ print F " -jar $cygA \$bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar $cygB \\\n";
print F " --repeat-weight 0.9 --repeat-idf-scale 10 -k $merSize \\\n";
print F " --supress-noise 2 \\\n" if (defined(getGlobal("${tag}MhapFilterUnique")) && getGlobal("${tag}MhapFilterUnique") == 1);
print F " --no-tf \\\n" if (defined(getGlobal("${tag}MhapNoTf")) && getGlobal("${tag}MhapNoTf") == 1);
@@ -362,19 +390,21 @@ sub mhapConfigure ($$$$) {
print F " --filter-threshold $filterThreshold \\\n";
print F " --num-threads ", getGlobal("${tag}mhapThreads"), " \\\n";
print F " " . getGlobal("${tag}MhapOptions") . " \\\n" if (defined(getGlobal("${tag}MhapOptions")));
- print F " -f " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz" . ($^O eq "cygwin" ? ") " : "") . "\\\n" if (-e "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz");
- print F " -p " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/blocks/\$job.fasta" . ($^O eq "cygwin" ? ") " : "") . "\\\n";
- print F " -q " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/blocks" .($^O eq "cygwin" ? ") " : "") . "\\\n";
- print F "|| \\\n";
- print F "mv -f $path/blocks/\$job.dat $path/blocks/\$job.dat.FAILED\n";
+ print F " -f $cygA ../../0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz $cygB \\\n" if (-e "$base/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz");
+ print F " -p $cygA ./\$job.input.fasta $cygB \\\n";
+ print F " -q $cygA . $cygB \\\n";
+ print F "&& \\\n";
+ print F "mv -f ./\$job.input.dat ./\$job.dat\n";
print F "\n";
- print F "if [ ! -e \"$path/blocks/\$job.dat\" ] ; then\n";
+ print F "if [ ! -e ./\$job.dat ] ; then\n";
print F " echo Mhap failed.\n";
print F " exit 1\n";
print F "fi\n";
print F "\n";
print F "# Clean up, remove the fasta input\n";
- print F "rm -f $path/blocks/\$job.fasta\n";
+ print F "rm -f ./\$job.input.fasta\n";
+ print F "\n";
+ print F stashFileShellCode("$base/1-overlapper/blocks", "\$job.dat", "");
print F "\n";
print F "exit 0\n";
@@ -386,6 +416,11 @@ sub mhapConfigure ($$$$) {
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F fetchStoreShellCode("$base/$asm.gkpStore", "$base/1-overlapper", "");
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
for (my $ii=1; $ii < scalar(@hashes); $ii++) {
@@ -404,24 +439,45 @@ sub mhapConfigure ($$$$) {
print F " exit 1\n";
print F "fi\n";
print F "\n";
- print F "if [ -e $path/results/\$qry.ovb ]; then\n";
+ print F "if [ -e ./results/\$qry.ovb ]; then\n";
print F " echo Job previously completed successfully.\n";
print F " exit\n";
print F "fi\n";
print F "\n";
- print F "if [ ! -d $path/results ]; then\n";
- print F " mkdir $path/results\n";
+
+ print F fetchFileShellCode("$path", "queries.tar", "");
+ print F "\n";
+ print F "if [ -e ./queries.tar -a ! -d ./queries ] ; then\n";
+ print F " tar -xf ./queries.tar\n";
print F "fi\n";
print F "\n";
- print F "echo Running block \$blk in query \$qry\n";
+ print F "if [ ! -d ./results ]; then\n";
+ print F " mkdir -p ./results\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ ! -d ./blocks ] ; then\n";
+ print F " mkdir -p ./blocks\n";
+ print F "fi\n";
+
+ print F fetchFileShellCode("$path", "blocks/\$blk.dat", "");
+ print F "for ii in `ls ./queries/\$qry` ; do\n";
+ print F " echo Fetch blocks/\$ii\n";
+ print F fetchFileShellCode("$path", "blocks/\$ii", " ");
+ print F "done\n";
print F "\n";
- print F getBinDirectoryShellCode();
+
+ print F fetchFileShellCode("$base/0-mercounts", "$asm.ms$merSize.frequentMers.ignore.gz", "");
print F "\n";
- print F "if [ ! -e \"$path/results/\$qry.mhap\" ] ; then\n";
+
+ print F "echo \"\"\n";
+ print F "echo Running block \$blk in query \$qry\n";
+ print F "echo \"\"\n";
+ print F "\n";
+ print F "if [ ! -e ./results/\$qry.mhap ] ; then\n";
print F " $javaPath -d64 -server -Xmx", $javaMemory, "m \\\n";
- print F " -jar " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "\$bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar " . ($^O eq "cygwin" ? ")" : "") . "\\\n";
+ print F " -jar $cygA \$bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar $cygB \\\n";
print F " --repeat-weight 0.9 --repeat-idf-scale 10 -k $merSize \\\n";
print F " --supress-noise 2 \\\n" if (defined(getGlobal("${tag}MhapFilterUnique")) && getGlobal("${tag}MhapFilterUnique") == 1);
print F " --no-tf \\\n" if (defined(getGlobal("${tag}MhapNoTf")) && getGlobal("${tag}MhapNoTf") == 1);
@@ -433,57 +489,59 @@ sub mhapConfigure ($$$$) {
print F " --ordered-kmer-size $ordSketchMer \\\n";
print F " --num-threads ", getGlobal("${tag}mhapThreads"), " \\\n";
print F " " . getGlobal("${tag}MhapOptions") . " \\\n" if (defined(getGlobal("${tag}MhapOptions")));
- print F " -f " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz" . ($^O eq "cygwin" ? ")" : "") . "\\\n" if (-e "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz");
- print F " -s " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/blocks/\$blk.dat \$slf" . ($^O eq "cygwin" ? ")" : "") . "\\\n";
- print F " -q " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/queries/\$qry" . ($^O eq "cygwin" ? ")" : "") . "\\\n";
- print F " > $path/results/\$qry.mhap.WORKING \\\n";
+ print F " -s $cygA ./blocks/\$blk.dat \$slf $cygB \\\n";
+ print F " -q $cygA queries/\$qry $cygB \\\n";
+ print F " > ./results/\$qry.mhap.WORKING \\\n";
print F " && \\\n";
- print F " mv -f $path/results/\$qry.mhap.WORKING $path/results/\$qry.mhap\n";
+ print F " mv -f ./results/\$qry.mhap.WORKING ./results/\$qry.mhap\n";
print F "fi\n";
print F "\n";
- print F "if [ -e \"$path/results/\$qry.mhap\" -a \\\n";
- print F " ! -e \"$path/results/\$qry.ovb\" ] ; then\n";
+ print F "if [ -e ./results/\$qry.mhap -a \\\n";
+ print F " ! -e ./results/\$qry.ovb ] ; then\n";
print F " \$bin/mhapConvert \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
print F " \$cvt \\\n";
- print F " -o $path/results/\$qry.mhap.ovb.WORKING \\\n";
- print F " $path/results/\$qry.mhap \\\n";
+ print F " -o ./results/\$qry.mhap.ovb.WORKING \\\n";
+ print F " ./results/\$qry.mhap \\\n";
print F " && \\\n";
- print F " mv $path/results/\$qry.mhap.ovb.WORKING $path/results/\$qry.mhap.ovb\n";
+ print F " mv ./results/\$qry.mhap.ovb.WORKING ./results/\$qry.mhap.ovb\n";
print F "fi\n";
print F "\n";
if (getGlobal('saveOverlaps') eq "0") {
- print F "if [ -e \"$path/results/\$qry.mhap\" -a \\\n";
- print F " -e \"$path/results/\$qry.mhap.ovb\" ] ; then\n";
- print F " rm -f $path/results/\$qry.mhap\n";
+ print F "if [ -e ./results/\$qry.mhap -a \\\n";
+ print F " -e ./results/\$qry.mhap.ovb ] ; then\n";
+ print F " rm -f ./results/\$qry.mhap\n";
print F "fi\n";
print F "\n";
}
- print F "if [ -e \"$path/results/\$qry.mhap.ovb\" ] ; then\n";
- if (getGlobal("${tag}ReAlign") eq "raw") {
+ if (getGlobal("${tag}ReAlign") eq "1") {
+ print F "if [ -e ./results/\$qry.mhap.ovb ] ; then\n";
print F " \$bin/overlapPair \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $path/results/\$qry.mhap.ovb \\\n";
- print F " -o $path/results/\$qry.WORKING.ovb \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -O ./results/\$qry.mhap.ovb \\\n";
+ print F " -o ./results/\$qry.WORKING.ovb \\\n";
print F " -partial \\\n" if ($typ eq "partial");
print F " -len " , getGlobal("minOverlapLength"), " \\\n";
- print F " -erate ", getGlobal("corErrorRate"), " \\\n" if ($tag eq "cor");
+ print F " -erate ", getGlobal("corOvlErrorRate"), " \\\n" if ($tag eq "cor"); # Explicitly using proper name for grepability.
print F " -erate ", getGlobal("obtOvlErrorRate"), " \\\n" if ($tag eq "obt");
print F " -erate ", getGlobal("utgOvlErrorRate"), " \\\n" if ($tag eq "utg");
print F " -memory " . getGlobal("${tag}mhapMemory") . " \\\n";
print F " -t " . getGlobal("${tag}mhapThreads") . " \\\n";
print F " && \\\n";
- print F " mv -f $path/results/\$qry.WORKING.ovb $path/results/\$qry.ovb\n";
+ print F " mv -f ./results/\$qry.WORKING.ovb ./results/\$qry.ovb\n";
+ print F "fi\n";
} else {
- print F " mv -f \"$path/results/\$qry.mhap.ovb\" \"$path/results/\$qry.ovb\"\n";
+ print F "if [ -e ./results/\$qry.mhap.ovb ] ; then\n";
+ print F " mv -f ./results/\$qry.mhap.ovb ./results/\$qry.ovb\n";
+ print F "fi\n";
}
- print F "fi\n";
print F "\n";
- print F "\n";
- #print F "rm -rf $path/queries/\$qry\n";
+ print F stashFileShellCode("$path", "results/\$qry.ovb", "");
+ print F stashFileShellCode("$path", "results/\$qry.counts", "");
print F "\n";
print F "exit 0\n";
@@ -504,41 +562,48 @@ sub mhapConfigure ($$$$) {
my $numJobs = 0;
open(F, "< $path/mhap.sh") or caFailure("can't open '$path/mhap.sh' for reading: $!", undef);
while (<F>) {
- $numJobs++ if (m/^\s+qry=\"(\d+)\"$/);
+ $numJobs++ if (m/^\s+qry=/);
}
close(F);
print STDERR "-- Configured $numJobs mhap overlap jobs.\n";
}
+ stashFile("$path/precompute.sh");
+ stashFile("$path/mhap.sh");
+
finishStage:
- emitStage($WRK, $asm, "$tag-mhapConfigure");
- buildHTML($WRK, $asm, $tag);
- stopAfter("mhapConfigure");
+ emitStage($asm, "$tag-mhapConfigure");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlapConfigure");
}
-sub mhapPrecomputeCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub mhapPrecomputeCheck ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $typ = shift @_;
my $attempt = getGlobal("canuIteration");
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base; # e.g., $base/1-overlapper/mhap.sh
+ my $path; # e.g., $path/mhap.sh
+
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
- my $path = "$wrk/1-overlapper";
+ $path = "$base/1-overlapper";
- goto allDone if (skipStage($WRK, $asm, "$tag-mhapPrecomputeCheck", $attempt) == 1);
- goto allDone if (-e "$path/precompute.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ goto allDone if (skipStage($asm, "$tag-mhapPrecomputeCheck", $attempt) == 1);
+ goto allDone if (fileExists("$path/precompute.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
+
+ fetchFile("$path/precompute.sh");
# Figure out if all the tasks finished correctly.
@@ -549,11 +614,11 @@ sub mhapPrecomputeCheck ($$$$) {
open(F, "< $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for reading: $!", undef);
while (<F>) {
- if (m/^\s+job=\"(\d+)\"$/) {
- if (-e "$path/blocks/$1.dat") {
- push @successJobs, "$path/blocks/$1.dat\n";
+ if (m/^\s+job=\"(\d+)\"$/) {
+ if (fileExists("$path/blocks/$1.dat")) {
+ push @successJobs, "1-overlapper/blocks/$1.dat\n";
} else {
- $failureMessage .= "-- job $path/blocks/$1.dat FAILED.\n";
+ $failureMessage .= "-- job 1-overlapper/blocks/$1.dat FAILED.\n";
push @failedJobs, $currentJobID;
}
@@ -583,12 +648,10 @@ sub mhapPrecomputeCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- mhap precompute attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-mhapPrecomputeCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
+ emitStage($asm, "$tag-mhapPrecomputeCheck", $attempt);
+ buildHTML($asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "${tag}mhap", $path, "precompute", @failedJobs);
+ submitOrRunParallelJob($asm, "${tag}mhap", $path, "precompute", @failedJobs);
return;
}
@@ -599,10 +662,10 @@ sub mhapPrecomputeCheck ($$$$) {
print L @successJobs;
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-mhapPrecomputeCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("mhapPrecompute");
+ stashFile("$path/precompute.files");
+
+ emitStage($asm, "$tag-mhapPrecomputeCheck");
+ buildHTML($asm, $tag);
allDone:
}
@@ -612,23 +675,27 @@ sub mhapPrecomputeCheck ($$$$) {
-sub mhapCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub mhapCheck ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $typ = shift @_;
my $attempt = getGlobal("canuIteration");
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base; # e.g., $base/1-overlapper/mhap.sh
+ my $path; # e.g., $path/mhap.sh
+
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
+
+ $path = "$base/1-overlapper";
- my $path = "$wrk/1-overlapper";
+ goto allDone if (skipStage($asm, "$tag-mhapCheck", $attempt) == 1);
+ goto allDone if (fileExists("$path/mhap.files"));
+ goto allDone if (-e "$base/$asm.ovlStore");
+ goto allDone if (fileExists("$base/$asm.ovlStore.tar"));
- goto allDone if (skipStage($WRK, $asm, "$tag-mhapCheck", $attempt) == 1);
- goto allDone if (-e "$path/mhap.files");
- goto allDone if (-e "$wrk/$asm.ovlStore");
+ fetchFile("$path/mhap.sh");
# Figure out if all the tasks finished correctly.
@@ -642,32 +709,32 @@ sub mhapCheck ($$$$) {
open(F, "< $path/mhap.sh") or caExit("failed to open '$path/mhap.sh'", undef);
while (<F>) {
if (m/^\s+qry=\"(\d+)\"$/) {
- if (-e "$path/results/$1.ovb.gz") {
- push @mhapJobs, "$path/results/$1.mhap\n";
- push @successJobs, "$path/results/$1.ovb.gz\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
-
- } elsif (-e "$path/results/$1.ovb") {
- push @mhapJobs, "$path/results/$1.mhap\n";
- push @successJobs, "$path/results/$1.ovb\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
-
- } elsif (-e "$path/results/$1.ovb.bz2") {
- push @mhapJobs, "$path/results/$1.mhap\n";
- push @successJobs, "$path/results/$1.ovb.bz2\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
-
- } elsif (-e "$path/results/$1.ovb.xz") {
- push @mhapJobs, "$path/results/$1.mhap\n";
- push @successJobs, "$path/results/$1.ovb.xz\n";
- push @miscJobs, "$path/results/$1.stats\n";
- push @miscJobs, "$path/results/$1.counts\n";
+ if (fileExists("$path/results/$1.ovb.gz")) {
+ push @mhapJobs, "1-overlapper/results/$1.mhap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb.gz\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
+
+ } elsif (fileExists("$path/results/$1.ovb")) {
+ push @mhapJobs, "1-overlapper/results/$1.mhap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
+
+ } elsif (fileExists("$path/results/$1.ovb.bz2")) {
+ push @mhapJobs, "1-overlapper/results/$1.mhap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb.bz2\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
+
+ } elsif (fileExists("$path/results/$1.ovb.xz")) {
+ push @mhapJobs, "1-overlapper/results/$1.mhap\n";
+ push @successJobs, "1-overlapper/results/$1.ovb.xz\n";
+ push @miscJobs, "1-overlapper/results/$1.stats\n";
+ push @miscJobs, "1-overlapper/results/$1.counts\n";
} else {
- $failureMessage .= "-- job $path/results/$1.ovb FAILED.\n";
+ $failureMessage .= "-- job 1-overlapper/results/$1.ovb FAILED.\n";
push @failedJobs, $currentJobID;
}
@@ -706,11 +773,9 @@ sub mhapCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- mhap attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-mhapCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "${tag}mhap", $path, "mhap", @failedJobs);
+ emitStage($asm, "$tag-mhapCheck", $attempt);
+ buildHTML($asm, $tag);
+ submitOrRunParallelJob($asm, "${tag}mhap", $path, "mhap", @failedJobs);
return;
}
@@ -729,11 +794,14 @@ sub mhapCheck ($$$$) {
print L @miscJobs;
close(L);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-mhapCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("mhapOverlap");
+ stashFile("$path/mhap.files");
+ stashFile("$path/ovljob.files");
+ stashFile("$path/ovljob.more.files");
+
+ emitStage($asm, "$tag-mhapCheck");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlap");
}
diff --git a/src/pipelines/canu/OverlapStore.pm b/src/pipelines/canu/OverlapStore.pm
index 807d815..6579e9e 100644
--- a/src/pipelines/canu/OverlapStore.pm
+++ b/src/pipelines/canu/OverlapStore.pm
@@ -43,11 +43,14 @@ require Exporter;
@EXPORT = qw(createOverlapStore);
use strict;
+use File::Basename; # dirname
use POSIX qw(ceil);
use canu::Defaults;
use canu::Execution;
+use canu::Report;
use canu::HTML;
+use canu::Grid_Cloud;
# Parallel documentation: Each overlap job is converted into a single bucket of overlaps. Within
@@ -58,18 +61,29 @@ use canu::HTML;
# NOT FILTERING overlaps by error rate when building the parallel store.
-sub createOverlapStoreSequential ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub createOverlapStoreSequential ($$$) {
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
- my $files = shift @_;
my $bin = getBinDirectory();
my $cmd;
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ # Fetch inputs. If you're not cloud-based, this does nothing. Really. Trust me.
+
+ fetchFile("$base/1-overlapper/ovljob.files");
+
+ open(F, "< $base/1-overlapper/ovljob.files") or caExit("failed to open overlapper output list in '$base/1-overlapper/ovljob.files'", undef);
+ while (<F>) {
+ chomp;
+
+ if (m/^(.*).ovb$/) {
+ fetchFile("$base/$1.ovb");
+ fetchFile("$base/$1.counts");
+ } else {
+ caExit("didn't recognize ovljob.files line '$_'", undef);
+ }
+ }
+ close(F);
# This is running in the canu process itself. Execution.pm has special case code
# to submit canu to grids using the maximum of 4gb and this memory limit.
@@ -80,19 +94,21 @@ sub createOverlapStoreSequential ($$$$) {
# runs out of open file handles first (meaning it has never run out of processes yet).
$cmd = "$bin/ovStoreBuild \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore.BUILDING \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -O ./$asm.ovlStore.BUILDING \\\n";
+ $cmd .= " -G ./$asm.gkpStore \\\n";
$cmd .= " -M $memSize \\\n";
- $cmd .= " -L $files \\\n";
- $cmd .= " > $wrk/$asm.ovlStore.err 2>&1";
+ $cmd .= " -L ./1-overlapper/ovljob.files \\\n";
+ $cmd .= " > ./$asm.ovlStore.err 2>&1";
- if (runCommand($wrk, $cmd)) {
- caExit("failed to create the overlap store", "$wrk/$asm.ovlStore.err");
+ if (runCommand($base, $cmd)) {
+ caExit("failed to create the overlap store", "$base/$asm.ovlStore.err");
}
- unlink "$wrk/$asm.ovlStore.err";
+ unlink("$base/$asm.ovlStore.err");
- rename "$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore";
+ rename("$base/$asm.ovlStore.BUILDING", "$base/$asm.ovlStore");
+
+ stashStore("$base/$asm.ovlStore");
}
@@ -100,15 +116,19 @@ sub createOverlapStoreSequential ($$$$) {
# Count the number of inputs. We don't expect any to be missing (they were just checked
# by overlapCheck()) but feel silly not checking again.
+#
+# Note that these files are rooted in '$base' (because that's where we run the overlap store
+# building) but canu.pl itself is rooted in the same directory as '$base', so we need to
+# add in '$base'.
sub countOverlapStoreInputs ($) {
- my $inputs = shift @_;
+ my $base = shift @_;
my $numInputs = 0;
- open(F, "< $inputs") or die "Failed to open overlap store input file '$inputs': $0\n";
+ open(F, "< $base/1-overlapper/ovljob.files") or caExit("Failed to open overlap store input file '$base/1-overlapper/ovljob.files': $0", undef);
while (<F>) {
chomp;
- die "overlapper output '$_' not found\n" if (! -e $_);
+ caExit("overlapper output '$_' not found", undef) if (! -e "$base/$_");
$numInputs++;
}
close(F);
@@ -120,14 +140,15 @@ sub countOverlapStoreInputs ($) {
sub getNumOlapsAndSlices ($$) {
- my $wrk = shift @_;
- my $asm = shift @_;
+ my $base = shift @_;
+ my $asm = shift @_;
+ my $path = "$base/$asm.ovlStore.BUILDING";
my $numOlaps = undef;
my $numSlices = undef;
my $memLimit = undef;
- open(F, "< $wrk/$asm.ovlStore.BUILDING/config.err") or caExit("can't open '$wrk/$asm.ovlStore.BUILDING/config.err' for reading: $!\n", undef);
+ open(F, "< $path/config.err") or caExit("can't open '$path/config.err' for reading: $!\n", undef);
while (<F>) {
if (m/Will sort (\d+.\d+) million overlaps per bucket, using (\d+) buckets (\d+.\d+) GB per bucket./) {
$numOlaps = $1;
@@ -141,94 +162,108 @@ sub getNumOlapsAndSlices ($$) {
caExit("Failed to find any overlaps ($numOlaps) or slices ($numSlices) or memory limit ($memLimit)", undef);
}
+ # Bump up the memory limit on grid jobs a bit.
+
+ setGlobal("ovsMemory", ceil($memLimit + 0.5));
+
+ # The memory limit returned is used to tell ovStoreSorter itself how much space to reserve.
+
return($numOlaps, $numSlices, $memLimit);
}
-sub overlapStoreConfigure ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub overlapStoreConfigure ($$$) {
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
- my $files = shift @_;
my $bin = getBinDirectory();
my $cmd;
+ my $path = "$base/$asm.ovlStore.BUILDING";
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ goto allDone if (skipStage($asm, "$tag-overlapStoreConfigure") == 1);
+ goto allDone if ((-e "$path/scripts/0-config.sh") &&
+ (-e "$path/scripts/1-bucketize.sh") &&
+ (-e "$path/scripts/2-sort.sh") &&
+ (-e "$path/scripts/3-index.sh"));
+ goto allDone if (-d "$base/$asm.ovlStore");
- goto allDone if (skipStage($WRK, $asm, "$tag-overlapStoreConfigure") == 1);
- goto allDone if (-d "$wrk/$asm.ovlStore");
-
- my $numInputs = countOverlapStoreInputs($files);
+ my $numInputs = countOverlapStoreInputs($base);
# Create an output directory, and populate it with more directories and scripts
- system("mkdir -p $wrk/$asm.ovlStore.BUILDING") if (! -d "$wrk/$asm.ovlStore.BUILDING");
- system("mkdir -p $wrk/$asm.ovlStore.BUILDING/scripts") if (! -d "$wrk/$asm.ovlStore.BUILDING/scripts");
- system("mkdir -p $wrk/$asm.ovlStore.BUILDING/logs") if (! -d "$wrk/$asm.ovlStore.BUILDING/logs");
+ system("mkdir -p $path/scripts") if (! -d "$path/scripts");
+ system("mkdir -p $path/logs") if (! -d "$path/logs");
# Run the normal store build, but just to get the partitioning. ovStoreBuild internally
# writes to config.WORKING, then renames when it is finished. No need for the script
# to be overly careful about incomplete files.
- if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/0-config.sh") {
- open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/0-config.sh") or die;
+ if (! -e "$path/scripts/0-config.sh") {
+ open(F, "> $path/scripts/0-config.sh") or die;
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
- print F getLimitShellCode("processes");
- print F getLimitShellCode("files");
+ print F setWorkDirectoryShellCode($path);
print F "\n";
+ #print F getJobIDShellCode();
+ #print F "\n";
print F getBinDirectoryShellCode();
print F "\n";
+ print F getLimitShellCode("processes");
+ print F getLimitShellCode("files");
+ print F "\n";
print F "\$bin/ovStoreBuild \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $wrk/$asm.ovlStore \\\n"; # NOT created!
+ print F " -G ./$asm.gkpStore \\\n";
+ print F " -O ./$asm.ovlStore \\\n"; # NOT created!
print F " -M " . getGlobal("ovsMemory") . " \\\n";
- print F " -config $wrk/$asm.ovlStore.BUILDING/config \\\n";
- print F " -L $files \\\n";
+ print F " -config ./$asm.ovlStore.BUILDING/config \\\n";
+ print F " -L ./1-overlapper/ovljob.files \\\n";
close(F);
}
- system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/0-config.sh");
+ system("chmod +x $path/scripts/0-config.sh");
- if (! -e "$wrk/$asm.ovlStore.BUILDING/config") {
- $cmd = "$wrk/$asm.ovlStore.BUILDING/scripts/0-config.sh \\\n";
- $cmd .= "> $wrk/$asm.ovlStore.BUILDING/config.err 2>&1\n";
+ if (! -e "$path/config") {
+ $cmd = "./$asm.ovlStore.BUILDING/scripts/0-config.sh \\\n";
+ $cmd .= "> ./$asm.ovlStore.BUILDING/config.err 2>&1\n";
- if (runCommand("$wrk/$asm.ovlStore.BUILDING/scripts", $cmd)) {
- caExit("failed to generate configuration for building overlap store", "$wrk/$asm.ovlStore.BUILDING/config.err");
+ if (runCommand($base, $cmd)) {
+ caExit("failed to generate configuration for building overlap store", "$path/config.err");
}
}
# Parse the output to find the number of jobs we need to sort and the memory
# ovs store memory is left as a range (e.g. 4-16) so building can scale itself to (hopefully) fit both into memory and into max system open files
- my ($numOlaps, $numSlices, $memLimit) = getNumOlapsAndSlices($wrk, $asm);
- setGlobal("ovsMemory", ceil($memLimit * 1.1)); # request more memory to avoid memory issues on machines with no swap
+ my ($numOlaps, $numSlices, $memLimit) = getNumOlapsAndSlices($base, $asm);
# Parallel jobs for bucketizing. This should really be part of overlap computation itself.
#getAllowedResources("", "ovb");
- if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/1-bucketize.sh") {
- open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/1-bucketize.sh") or die;
+ if (! -e "$path/scripts/1-bucketize.sh") {
+ open(F, "> $path/scripts/1-bucketize.sh") or die;
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
print F "bn=`printf %04d \$jobid`\n";
print F "jn=\"undefined\"\n";
print F "\n";
+ print F "# This script runs in $path/, but the overlap file list\n";
+ print F "# is relative to $base/, so we need to add a few dots to make things work.\n";
+ print F "\n";
my $tstid = 1;
- open(I, "< $files") or die "Failed to open '$files': $0\n";
+ open(I, "< $base/1-overlapper/ovljob.files") or die "Failed to open '$base/1-overlapper/ovljob.files': $0\n";
while (<I>) {
chomp;
- print F "if [ \"\$jobid\" -eq \"$tstid\" ] ; then jn=\"$_\"; fi\n";
+ print F "if [ \"\$jobid\" -eq \"$tstid\" ] ; then jn=\"../$_\"; fi\n";
$tstid++;
}
@@ -240,28 +275,34 @@ sub overlapStoreConfigure ($$$$) {
print F " exit\n";
print F "fi\n";
print F "\n";
- print F "if [ -e \"$wrk/$asm.ovlStore.BUILDING/bucket\$bn/sliceSizes\" ] ; then\n";
- print F " echo \"Bucket $wrk/$asm.ovlStore.BUILDING/bucket\$bn finished successfully.\"\n";
+ print F "if [ -e \"./bucket\$bn/sliceSizes\" ] ; then\n";
+ print F " echo \"Bucket bucket\$bn finished successfully.\"\n";
print F " exit\n";
print F "fi\n";
print F "\n";
- print F "if [ -e \"$wrk/$asm.ovlStore.BUILDING/create\$bn\" ] ; then\n";
- print F " echo \"Removing incomplete bucket $wrk/$asm.ovlStore.BUILDING/create\$bn\"\n";
- print F " rm -rf \"$wrk/$asm.ovlStore.BUILDING/create\$bn\"\n";
+ print F "if [ -e \"./create\$bn\" ] ; then\n";
+ print F " echo \"Removing incomplete bucket create\$bn\"\n";
+ print F " rm -rf \"./create\$bn\"\n";
print F "fi\n";
print F "\n";
print F getLimitShellCode("processes");
print F getLimitShellCode("files");
print F "\n";
- print F getBinDirectoryShellCode();
- print F "\n";
print F "\$bin/ovStoreBucketizer \\\n";
- print F " -O $wrk/$asm.ovlStore.BUILDING \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -C $wrk/$asm.ovlStore.BUILDING/config \\\n";
+ print F " -O . \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -C ./config \\\n";
#print F " -e " . getGlobal("") . " \\\n" if (defined(getGlobal("")));
print F " -job \$jobid \\\n";
print F " -i \$jn\n";
+ print F "\n";
+ print F "if [ \$? = 0 ] ; then\n";
+ print F " echo Success.\n";
+ print F " exit 0\n";
+ print F "else\n";
+ print F " echo Failure.\n";
+ print F " exit 1\n";
+ print F "fi\n";
close(F);
}
@@ -269,87 +310,94 @@ sub overlapStoreConfigure ($$$$) {
#getAllowedResources("", "ovs");
- if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/2-sort.sh") {
- open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/2-sort.sh") or die;
+ if (! -e "$path/scripts/2-sort.sh") {
+ open(F, "> $path/scripts/2-sort.sh") or die;
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F "\n";
print F getJobIDShellCode();
print F "\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
print F getLimitShellCode("processes");
print F getLimitShellCode("files");
print F "\n";
- print F getBinDirectoryShellCode();
- print F "\n";
print F "\$bin/ovStoreSorter \\\n";
print F " -deletelate \\\n"; # Choices -deleteearly -deletelate or nothing
print F " -M $memLimit \\\n";
- print F " -O $wrk/$asm.ovlStore.BUILDING \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -O . \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
print F " -F $numSlices \\\n";
print F " -job \$jobid $numInputs\n";
print F "\n";
print F "if [ \$? = 0 ] ; then\n";
print F " echo Success.\n";
+ print F " exit 0\n";
print F "else\n";
print F " echo Failure.\n";
+ print F " exit 1\n";
print F "fi\n";
close(F);
}
# A final job to merge the indices.
- if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh") {
- open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh") or die;
+ if (! -e "$path/scripts/3-index.sh") {
+ open(F, "> $path/scripts/3-index.sh") or die;
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F "\n";
+ #print F getJobIDShellCode();
+ #print F "\n";
print F getBinDirectoryShellCode();
print F "\n";
print F "\$bin/ovStoreIndexer \\\n";
#print F " -nodelete \\\n"; # Choices -nodelete or nothing
- print F " -O $wrk/$asm.ovlStore.BUILDING \\\n";
+ print F " -O . \\\n";
print F " -F $numSlices\n";
print F "\n";
print F "if [ \$? = 0 ] ; then\n";
print F " echo Success.\n";
+ print F " exit 0\n";
print F "else\n";
print F " echo Failure.\n";
+ print F " exit 1\n";
print F "fi\n";
close(F);
}
- system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/1-bucketize.sh");
- system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/2-sort.sh");
- system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh");
+ system("chmod +x $path/scripts/1-bucketize.sh");
+ system("chmod +x $path/scripts/2-sort.sh");
+ system("chmod +x $path/scripts/3-index.sh");
finishStage:
- emitStage($WRK, $asm, "$tag-overlapStoreConfigure");
- buildHTML($WRK, $asm, $tag);
- stopAfter("overlapStoreConfigure");
+ emitStage($asm, "$tag-overlapStoreConfigure");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlapStoreConfigure");
}
-sub overlapStoreBucketizerCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub overlapStoreBucketizerCheck ($$$) {
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
- my $files = shift @_;
my $attempt = getGlobal("canuIteration");
+ my $path = "$base/$asm.ovlStore.BUILDING";
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ goto allDone if (skipStage($asm, "$tag-overlapStoreBucketizerCheck", $attempt) == 1);
+ goto allDone if (-d "$base/$asm.ovlStore");
+ goto allDone if (-e "$path/1-bucketize.success");
- goto allDone if (skipStage($WRK, $asm, "$tag-overlapStoreBucketizerCheck", $attempt) == 1);
- goto allDone if (-d "$wrk/$asm.ovlStore");
- goto allDone if (-e "$wrk/$asm.ovlStore.BUILDING/1-bucketize.success");
+ fetchFile("scripts/1-bucketize/1-bucketize.sh");
# Figure out if all the tasks finished correctly.
- my $numInputs = countOverlapStoreInputs($files);
+ my $numInputs = countOverlapStoreInputs($base);
my $currentJobID = 1;
my @successJobs;
my @failedJobs;
@@ -361,13 +409,13 @@ sub overlapStoreBucketizerCheck ($$$$) {
# exists. The compute is done in a 'create' directory, which is renamed to 'bucket' just
# before the job completes.
- open(F, "< $files") or caExit("can't open '$files' for reading: $!", undef);
+ open(F, "< $base/1-overlapper/ovljob.files") or caExit("can't open '$base/1-overlapper/ovljob.files' for reading: $!", undef);
while (<F>) {
chomp;
- if (! -e "$wrk/$asm.ovlStore.BUILDING/bucket$bucketID") {
- $failureMessage .= "-- job $wrk/$asm.ovlStore.BUILDING/bucket$bucketID FAILED.\n";
+ if (! -e "$path/bucket$bucketID") {
+ $failureMessage .= "-- job $path/bucket$bucketID FAILED.\n";
push @failedJobs, $currentJobID;
} else {
push @successJobs, $currentJobID;
@@ -400,24 +448,20 @@ sub overlapStoreBucketizerCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- overlap store bucketizer attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-overlapStoreBucketizerCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
+ emitStage($asm, "$tag-overlapStoreBucketizerCheck", $attempt);
+ buildHTML($asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "ovB", "$wrk/$asm.ovlStore.BUILDING", "scripts/1-bucketize", @failedJobs);
+ submitOrRunParallelJob($asm, "ovB", $path, "scripts/1-bucketize", @failedJobs);
return;
}
finishStage:
print STDERR "-- Overlap store bucketizer finished.\n";
- touch("$wrk/$asm.ovlStore.BUILDING/1-bucketize.success");
+ touch("$path/1-bucketize.success");
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-overlapStoreBucketizerCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("overlapBucketizer");
+ emitStage($asm, "$tag-overlapStoreBucketizerCheck");
+ buildHTML($asm, $tag);
allDone:
}
@@ -426,25 +470,22 @@ sub overlapStoreBucketizerCheck ($$$$) {
-sub overlapStoreSorterCheck ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub overlapStoreSorterCheck ($$$) {
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
- my $files = shift @_;
my $attempt = getGlobal("canuIteration");
+ my $path = "$base/$asm.ovlStore.BUILDING";
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ goto allDone if (skipStage($asm, "$tag-overlapStoreSorterCheck", $attempt) == 1);
+ goto allDone if (-d "$base/$asm.ovlStore");
+ goto allDone if (-e "$path/2-sorter.success");
- goto allDone if (skipStage($WRK, $asm, "$tag-overlapStoreSorterCheck", $attempt) == 1);
- goto allDone if (-d "$wrk/$asm.ovlStore");
- goto allDone if (-e "$wrk/$asm.ovlStore.BUILDING/2-sorter.success");
+ fetchFile("scripts/1-bucketize/2-sort.sh");
# Figure out if all the tasks finished correctly.
- my ($numOlaps, $numSlices) = getNumOlapsAndSlices($wrk, $asm);
+ my ($numOlaps, $numSlices, $memLimit) = getNumOlapsAndSlices($base, $asm);
my $currentJobID = 1;
my @successJobs;
@@ -453,25 +494,25 @@ sub overlapStoreSorterCheck ($$$$) {
my $sortID = "0001";
- open(F, "< $files") or caExit("can't open '$files' for reading: $!", undef);
+ open(F, "< $base/1-overlapper/ovljob.files") or caExit("can't open '$base/1-overlapper/ovljob.files' for reading: $!", undef);
# A valid result has three files:
- # $wrk/$asm.ovlStore.BUILDING/$sortID
- # $wrk/$asm.ovlStore.BUILDING/$sortID.index
- # $wrk/$asm.ovlStore.BUILDING/$sortID.info
+ # $path/$sortID
+ # $path/$sortID.index
+ # $path/$sortID.info
#
# A crashed result has one file, if it crashes before output
- # $wrk/$asm.ovlStore.BUILDING/$sortID.ovs
+ # $path/$sortID.ovs
#
# On out of disk, the .info is missing. It's the last thing created.
#
while ($currentJobID <= $numSlices) {
- if ((! -e "$wrk/$asm.ovlStore.BUILDING/$sortID") ||
- (! -e "$wrk/$asm.ovlStore.BUILDING/$sortID.info") ||
- ( -e "$wrk/$asm.ovlStore.BUILDING/$sortID.ovs")) {
- $failureMessage .= "-- job $wrk/$asm.ovlStore.BUILDING/$sortID FAILED.\n";
- unlink "$wrk/$asm.ovlStore.BUILDING/$sortID.ovs";
+ if ((! -e "$path/$sortID") ||
+ (! -e "$path/$sortID.info") ||
+ ( -e "$path/$sortID.ovs")) {
+ $failureMessage .= "-- job $path/$sortID FAILED.\n";
+ unlink "$path/$sortID.ovs";
push @failedJobs, $currentJobID;
} else {
push @successJobs, $currentJobID;
@@ -504,24 +545,20 @@ sub overlapStoreSorterCheck ($$$$) {
# Otherwise, run some jobs.
- print STDERR "-- overlap store sorter attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n";
-
- emitStage($WRK, $asm, "$tag-overlapStoreSorterCheck", $attempt);
- buildHTML($WRK, $asm, $tag);
+ emitStage($asm, "$tag-overlapStoreSorterCheck", $attempt);
+ buildHTML($asm, $tag);
- submitOrRunParallelJob($WRK, $asm, "ovS", "$wrk/$asm.ovlStore.BUILDING", "scripts/2-sort", @failedJobs);
+ submitOrRunParallelJob($asm, "ovS", $path, "scripts/2-sort", @failedJobs);
return;
}
finishStage:
print STDERR "-- Overlap store sorter finished.\n";
- touch("$wrk/$asm.ovlStore.BUILDING/2-sorter.success");
+ touch("$path/2-sorter.success");
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "$tag-overlapStoreSorterCheck");
- buildHTML($WRK, $asm, $tag);
- stopAfter("overlapSorter");
+ emitStage($asm, "$tag-overlapStoreSorterCheck");
+ buildHTML($asm, $tag);
allDone:
}
@@ -529,80 +566,85 @@ sub overlapStoreSorterCheck ($$$$) {
-sub createOverlapStoreParallel ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub createOverlapStoreParallel ($$$) {
+ my $base = shift @_;
my $asm = shift @_;
my $tag = shift @_;
- my $files = shift @_;
+ my $path = "$base/$asm.ovlStore.BUILDING";
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ overlapStoreConfigure ($base, $asm, $tag);
+ overlapStoreBucketizerCheck($base, $asm, $tag) foreach (1..getGlobal("canuIterationMax") + 1);
+ overlapStoreSorterCheck ($base, $asm, $tag) foreach (1..getGlobal("canuIterationMax") + 1);
- overlapStoreConfigure($WRK, $asm, $tag, $files);
- overlapStoreBucketizerCheck($WRK, $asm, $tag, $files) foreach (1..getGlobal("canuIterationMax") + 1);
- overlapStoreSorterCheck($WRK, $asm, $tag, $files) foreach (1..getGlobal("canuIterationMax") + 1);
-
- if (runCommand("$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh > $wrk/$asm.ovlStore.BUILDING/logs/3-index.err 2>&1")) {
- caExit("failed to build index for overlap store", "$wrk/$asm.ovlStore.BUILDING/logs/3-index.err");
+ if (runCommand($path, "./scripts/3-index.sh > ./logs/3-index.err 2>&1")) {
+ caExit("failed to build index for overlap store", "$base/$asm.ovlStore.BUILDING/logs/3-index.err");
}
- rename "$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore";
+ rename $path, "$base/$asm.ovlStore";
}
sub generateOverlapStoreStats ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+ my $base = shift @_;
my $asm = shift @_;
my $bin = getBinDirectory();
my $cmd;
$cmd = "$bin/ovStoreStats \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -O $wrk/$asm.ovlStore \\\n";
- $cmd .= " -o $wrk/$asm.ovlStore \\\n";
- $cmd .= " > $wrk/$asm.ovlStore.summary.err 2>&1";
+ $cmd .= " -G ./$asm.gkpStore \\\n";
+ $cmd .= " -O ./$asm.ovlStore \\\n";
+ $cmd .= " -o ./$asm.ovlStore \\\n";
+ $cmd .= " > ./$asm.ovlStore.summary.err 2>&1";
- if (runCommand($wrk, $cmd)) {
+ if (runCommand($base, $cmd)) {
print STDERR "--\n";
- print STDERR "-- WARNING: failed to generate statistics for the overlap store; no summary will appear in HTML output.\n";
+ print STDERR "-- WARNING: failed to generate statistics for the overlap store; no summary will appear in report.\n";
print STDERR "--\n";
print STDERR "----------------------------------------\n";
+ return;
+ }
+
+ unlink "$base/$asm.ovlStore.summary.err";
+
+ my $report;
+
+ open(F, "< $base/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$base/$asm.ovlStore.summary': $!", undef);
+ while (<F>) {
+ $report .= "-- $_";
}
+ close(F);
+
+ addToReport("overlaps", $report);
}
-sub createOverlapStore ($$$$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = $WRK; # Local work directory
+sub createOverlapStore ($$$) {
my $asm = shift @_;
my $tag = shift @_;
my $seq = shift @_;
- $wrk = "$wrk/correction" if ($tag eq "cor");
- $wrk = "$wrk/trimming" if ($tag eq "obt");
- $wrk = "$wrk/unitigging" if ($tag eq "utg");
+ my $base;
- my $path = "$wrk/1-overlapper";
+ $base = "correction" if ($tag eq "cor");
+ $base = "trimming" if ($tag eq "obt");
+ $base = "unitigging" if ($tag eq "utg");
- goto allDone if (skipStage($WRK, $asm, "$tag-createOverlapStore") == 1);
- goto allDone if (-d "$wrk/$asm.ovlStore");
- goto allDone if (-d "$wrk/$asm.ctgStore");
+ goto allDone if (skipStage($asm, "$tag-createOverlapStore") == 1);
+ goto allDone if ((-d "$base/$asm.ovlStore") || (fileExists("$base/$asm.ovlStore.tar")));
+ goto allDone if ((-d "$base/$asm.ctgStore") || (fileExists("$base/$asm.ctgStore.tar")));
# Did we _really_ complete?
- caExit("overlapper claims to be finished, but no job list found in '$path/ovljob.files'", undef) if (! -e "$path/ovljob.files");
+ caExit("overlapper claims to be finished, but no job list found in '$base/1-overlapper/ovljob.files'", undef) if (! fileExists("$base/1-overlapper/ovljob.files"));
# Then just build the store! Simple!
- createOverlapStoreSequential($WRK, $asm, $tag, "$path/ovljob.files") if ($seq eq "sequential");
- createOverlapStoreParallel ($WRK, $asm, $tag, "$path/ovljob.files") if ($seq eq "parallel");
+ createOverlapStoreSequential($base, $asm, $tag) if ($seq eq "sequential");
+ createOverlapStoreParallel ($base, $asm, $tag) if ($seq eq "parallel");
print STDERR "--\n";
- print STDERR "-- Overlap store '$wrk/$asm.ovlStore' successfully constructed.\n";
+ print STDERR "-- Overlap store '$base/$asm.ovlStore' successfully constructed.\n";
goto finishStage if (getGlobal("saveOverlaps") eq "1");
@@ -613,7 +655,11 @@ sub createOverlapStore ($$$$) {
my $bytes = 0;
my $files = 0;
- foreach my $file ("$path/ovljob.files", "$path/ovljob.more.files", "$path/mhap.files", "$path/mmap.files", "$path/precompute.files") {
+ foreach my $file ("$base/1-overlapper/ovljob.files",
+ "$base/1-overlapper/ovljob.more.files",
+ "$base/1-overlapper/mhap.files",
+ "$base/1-overlapper/mmap.files",
+ "$base/1-overlapper/precompute.files") {
next if (! -e $file);
open(F, "< $file") or caExit("can't open '$file' for reading: $!\n", undef);
@@ -623,27 +669,19 @@ sub createOverlapStore ($$$$) {
# Decide what to do. Directories - register for later deletion. Files - sum size and
# delete.
- if (-d $_) {
- $directories{$_}++;
+ if (-d "$base/$_") {
+ $directories{"$base/$_"}++;
- } elsif (-e $_) {
- $bytes += -s $_;
+ } elsif (-e "$base/$_") {
+ $bytes += -s "$base/$_";
$files += 1;
- unlink $_;
+ unlink "$base/$_";
}
- # If the file isn't a directory -- the file itself could not exist above -- register
- # the directory it is in for deletion.
-
- if (! -d $_) {
- my @components = split '/', $_;
- pop @components;
- my $dir = join '/', @components;
-
- $directories{$dir}++;
- }
+ # If the path isn't a directory register the directory it is in for deletion.
+ $directories{dirname("$base/$_")}++ if (! -d "$base/$_");
}
close(F);
@@ -678,27 +716,27 @@ sub createOverlapStore ($$$$) {
finishStage:
if ($tag eq "utg") {
- generateOverlapStoreStats($wrk, $asm);
+ generateOverlapStoreStats($base, $asm);
}
- if (-e "$wrk/$asm.ovlStore.summary") {
+ if (-e "$base/$asm.ovlStore.summary") {
print STDERR "--\n";
- print STDERR "-- Overlap store '$wrk/$asm.ovlStore' contains:\n";
+ print STDERR "-- Overlap store '$base/$asm.ovlStore' contains:\n";
print STDERR "--\n";
- open(F, "< $wrk/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$wrk/$asm.ovlStore': $!", undef);
+ open(F, "< $base/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$base/$asm.ovlStore': $!", undef);
while (<F>) {
print STDERR "-- $_";
}
close(F);
} else {
- print STDERR "-- Overlap store '$wrk/$asm.ovlStore' statistics not available (skipped in correction and trimming stages).\n";
+ print STDERR "-- Overlap store '$base/$asm.ovlStore' statistics not available (skipped in correction and trimming stages).\n";
}
- emitStage($WRK, $asm, "$tag-createOverlapStore");
- buildHTML($WRK, $asm, $tag);
- stopAfter("overlapStore");
+ emitStage($asm, "$tag-createOverlapStore");
+ buildHTML($asm, $tag);
allDone:
+ stopAfter("overlapStore");
}
diff --git a/src/pipelines/canu/Report.pm b/src/pipelines/canu/Report.pm
new file mode 100644
index 0000000..5b75d65
--- /dev/null
+++ b/src/pipelines/canu/Report.pm
@@ -0,0 +1,172 @@
+
+###############################################################################
+ #
+ # This file is part of canu, a software program that assembles whole-genome
+ # sequencing reads into contigs.
+ #
+ # This software is based on:
+ # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ # the 'kmer package' (http://kmer.sourceforge.net)
+ # both originally distributed by Applera Corporation under the GNU General
+ # Public License, version 2.
+ #
+ # Canu branched from Celera Assembler at its revision 4587.
+ # Canu branched from the kmer project at its revision 1994.
+ #
+ # Modifications by:
+ #
+ # Brian P. Walenz beginning on 2017-MAR-20
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
+ # File 'README.licenses' in the root directory of this distribution contains
+ # full conditions and disclaimers for each license.
+ ##
+
+package canu::Report;
+
+require Exporter;
+
+ at ISA = qw(Exporter);
+ at EXPORT = qw(generateReport addToReport getFromReport);
+
+use strict;
+
+#use File::Copy;
+#use File::Path 2.08 qw(make_path remove_tree);
+
+#use canu::Defaults;
+use canu::Grid_Cloud;
+
+
+# Holds the report we have so far (loaded from disk) and is then populated with
+# any new results and written back to disk.
+#
+# For a rather technical reason, some keys will exist in the hash (*Meryl, for example)
+# but have an undef value.
+#
+my %report;
+
+
+
+# Parse an existing report to load the data present in it.
+
+sub loadReport ($) {
+ my $asm = shift @_;
+ my $tag;
+ my $rpt;
+
+ fetchFile("$asm.report");
+
+ return if (! -e "$asm.report");
+
+ # Parse the file, loading each section into the proper key.
+
+ open(F, "< $asm.report") or caExit("can't open '$asm.report' for reading: $!", undef);
+ while (<F>) {
+ if (m/^\[CORRECTION/) {
+ $tag = "cor";
+ } elsif (m/^\[TRIMMING/) {
+ $tag = "obt";
+ } elsif (m/^\[UNITIGGING/) {
+ $tag = "utg";
+ }
+
+ if (m/READS\]$/) { $rpt = "${tag}GkpStore"; $report{$rpt} = undef;
+ } elsif (m/MERS\]$/) { $rpt = "${tag}Meryl"; $report{$rpt} = undef;
+
+ } elsif (m/FILTERING\]$/) { $rpt = "filtering"; $report{$rpt} = undef;
+ } elsif (m/CORRECTIONS\]$/) { $rpt = "corrections"; $report{$rpt} = undef;
+
+ } elsif (m/TRIMMING\]$/) { $rpt = "trimming"; $report{$rpt} = undef;
+ } elsif (m/SPLITTING\]$/) { $rpt = "splitting"; $report{$rpt} = undef;
+
+ } elsif (m/OVERLAPS\]$/) { $rpt = "overlaps"; $report{$rpt} = undef;
+ } elsif (m/ADJUSTMENT\]$/) { $rpt = "adjustments"; $report{$rpt} = undef;
+ } elsif (m/UNITIGS\]$/) { $rpt = "unitigs"; $report{$rpt} = undef;
+ } elsif (m/CONTIGS\]$/) { $rpt = "contigs"; $report{$rpt} = undef;
+ } elsif (m/CONSENSUS\]$/) { $rpt = "consensus"; $report{$rpt} = undef;
+
+ } else {
+ $report{$rpt} .= $_;
+ }
+ }
+ close(F);
+
+ # Ignore blank lines at the start and end.
+
+ foreach my $k (keys %report) {
+ $report{$k} =~ s/^\s+//;
+ $report{$k} =~ s/\s+$//;
+ $report{$k} .= "\n";
+ }
+}
+
+
+
+sub saveReportItem ($$) {
+ my $title = shift @_;
+ my $item = shift @_;
+
+ print F "[$title]\n$item\n" if (defined($item));
+}
+
+
+
+sub saveReport ($) {
+ my $asm = shift @_;
+ my $tag;
+
+ open(F, "> $asm.report") or caExit("can't open '$asm.report' for writing: $!", undef);
+
+ saveReportItem("CORRECTION/READS", $report{"corGkpStore"});
+ saveReportItem("CORRECTION/MERS", $report{"corMeryl"});
+ saveReportItem("CORRECTION/FILTERING", $report{"correctionFiltering"});
+ saveReportItem("CORRECTION/CORRECTIONS", $report{"corrections"});
+
+ saveReportItem("TRIMMING/READS", $report{"obtGkpStore"});
+ saveReportItem("TRIMMING/MERS", $report{"obtMeryl"});
+ saveReportItem("TRIMMING/TRIMMING", $report{"trimming"});
+ saveReportItem("TRIMMING/SPLITTING", $report{"splitting"});
+
+ saveReportItem("UNITIGGING/READS", $report{"utgGkpStore"});
+ saveReportItem("UNITIGGING/MERS", $report{"utgMeryl"});
+ saveReportItem("UNITIGGING/OVERLAPS", $report{"overlaps"});
+ saveReportItem("UNITIGGING/ADJUSTMENT", $report{"adjustments"});
+ saveReportItem("UNITIGGING/UNITIGS", $report{"unitigs"});
+ saveReportItem("UNITIGGING/CONTIGS", $report{"contigs"});
+ saveReportItem("UNITIGGING/CONSENSUS", $report{"consensus"});
+
+ close(F);
+
+ stashFile("$asm.report");
+}
+
+
+
+sub generateReport ($) {
+ my $asm = shift @_;
+
+ loadReport($asm);
+ saveReport($asm);
+}
+
+
+
+sub addToReport ($$) {
+ my $item = shift @_;
+ my $text = shift @_;
+
+ print STDERR $text; # Client code is GREATLY simplified if this dumps to the screen too.
+
+ $report{$item} = $text;
+}
+
+
+
+sub getFromReport ($) {
+ return($report{$_[0]});
+}
+
+
+1;
diff --git a/src/pipelines/canu/Unitig.pm b/src/pipelines/canu/Unitig.pm
index 13d0245..fdc8194 100644
--- a/src/pipelines/canu/Unitig.pm
+++ b/src/pipelines/canu/Unitig.pm
@@ -44,21 +44,22 @@ require Exporter;
use strict;
-use File::Path qw(make_path remove_tree);
+use File::Path 2.08 qw(make_path remove_tree);
use POSIX qw(ceil);
use canu::Defaults;
use canu::Execution;
use canu::Configure; # For displayGenomeSize
use canu::Gatekeeper;
+use canu::Report;
use canu::HTML;
use canu::Meryl;
+use canu::Grid_Cloud;
-sub reportUnitigSizes ($$$$) {
- my $wrk = shift @_;
+sub reportUnitigSizes ($$$) {
my $asm = shift @_;
my $version = shift @_;
my $label = shift @_;
@@ -82,24 +83,33 @@ sub reportUnitigSizes ($$$$) {
my $gs = getGlobal("genomeSize");
my $V = substr("000000" . $version, -3);
- my $N = "$wrk/$asm.ctgStore/seqDB.v$V.sizes.txt";
+ my $N = "$asm.ctgStore/seqDB.v$V.sizes.txt";
- if (! -e $N) {
- $cmd = "$bin/tgStoreDump \\\n";
- $cmd .= " -G $wrk/$asm.gkpStore \\\n";
- $cmd .= " -T $wrk/$asm.ctgStore $version \\\n";
+ fetchFile("unitigging/$N");
+
+ if (! -e "unitigging/$N") {
+ fetchStore("unitigging/$asm.gkpStore");
+
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v$V.dat");
+ fetchFile("unitigging/$asm.ctgStore/seqDB.v$V.tig");
+
+ $cmd = "$bin/tgStoreDump \\\n"; # Duplicated at the end of unitigger.sh
+ $cmd .= " -G ./$asm.gkpStore \\\n";
+ $cmd .= " -T ./$asm.ctgStore $version \\\n";
$cmd .= " -sizes -s " . getGlobal("genomeSize") . " \\\n";
- $cmd .= "> $N";
+ $cmd .= "> ./$N";
- if (runCommand($wrk, $cmd)) {
+ if (runCommand("unitigging", $cmd)) {
caExit("failed to generate unitig sizes", undef);
}
+
+ stashFile("unitigging/$N");
}
$ctgSizes .= "-- NG (bp) LG (contigs) sum (bp)\n";
$ctgSizes .= "-- ---------- ------------ ----------\n";
- open(F, "< $N") or caExit("failed to open '$N' for reading: $!\n", undef);
+ open(F, "< unitigging/$N") or caExit("failed to open 'unitigging/$N' for reading: $!", undef);
while (<F>) {
$rptBases = $1 if (m/lenSuggestRepeat\s+sum\s+(\d+)/);
$rptNum = $1 if (m/lenSuggestRepeat\s+num\s+(\d+)/);
@@ -129,28 +139,36 @@ sub reportUnitigSizes ($$$$) {
}
close(F);
- print STDERR "-- Found, in version $version, $label:\n";
- print STDERR "-- contigs: $ctgNum sequences, total length $ctgBases bp (including $rptNum repeats of total length $rptBases bp).\n";
- print STDERR "-- bubbles: $bubNum sequences, total length $bubBases bp.\n";
- print STDERR "-- unassembled: $usmNum sequences, total length $usmBases bp.\n";
- print STDERR "--\n";
- print STDERR "-- Contig sizes based on genome size ", displayGenomeSize($gs), "bp:\n";
- print STDERR "--\n";
- print STDERR "$ctgSizes";
- print STDERR "--\n";
+ my $report;
+
+ $report .= "-- Found, in version $version, $label:\n";
+ $report .= "-- contigs: $ctgNum sequences, total length $ctgBases bp (including $rptNum repeats of total length $rptBases bp).\n";
+ $report .= "-- bubbles: $bubNum sequences, total length $bubBases bp.\n";
+ $report .= "-- unassembled: $usmNum sequences, total length $usmBases bp.\n";
+ $report .= "--\n";
+ $report .= "-- Contig sizes based on genome size ", displayGenomeSize($gs), "bp:\n";
+ $report .= "--\n";
+ $report .= $ctgSizes;
+ $report .= "--\n";
+
+ # Hmmm. Report wants a small tag, but $label is a bit verbose.
+
+ addToReport("contigs", $report) if ($label eq "after unitig construction");
+ addToReport("consensus", $report) if ($label eq "after consensus generation");
}
-sub unitig ($$) {
- my $WRK = shift @_; # Root work directory (the -d option to canu)
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub unitig ($) {
my $asm = shift @_;
+ my $path = "unitigging/4-unitigger";
- goto allDone if (skipStage($wrk, $asm, "unitig") == 1);
- goto allDone if ((-d "$wrk/$asm.ctgStore") && (-d "$wrk/$asm.utgStore"));
+ goto allDone if (skipStage($asm, "unitig") == 1);
+ goto allDone if (fileExists("unitigging/4-unitigger/unitigger.sh"));
+ goto allDone if (fileExists("unitigging/$asm.utgStore/seqDB.v001.tig") &&
+ fileExists("unitigging/$asm.ctgStore/seqDB.v001.tig"));
- make_path("$wrk/4-unitigger") if (! -d "$wrk/4-unitigger");
+ make_path($path) if (! -d $path);
# How many reads per partition? This will change - it'll move to be after unitigs are constructed.
@@ -158,25 +176,34 @@ sub unitig ($$) {
# Dump a script to run the unitigger.
- open(F, "> $wrk/4-unitigger/unitigger.sh") or caExit("can't open '$wrk/4-unitigger/unitigger.sh' for writing: $!\n", undef);
+ open(F, "> $path/unitigger.sh") or caExit("can't open '$path/unitigger.sh' for writing: $!\n", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
- print F "if [ -e $wrk/$asm.ctgStore/seqDB.v001.tig -a -e $wrk/$asm.utgStore/seqDB.v001.tig ] ; then\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F setWorkDirectoryShellCode($path);
+ print F "\n";
+ print F fetchStoreShellCode("unitigging/$asm.gkpStore", $path, "");
+ print F fetchStoreShellCode("unitigging/$asm.ovlStore", $path, "");
+ print F "\n";
+ print F fetchFileShellCode("unitigging/$asm.ovlStore", "evalues", "");
+ print F "\n";
+ print F getJobIDShellCode();
+ print F "\n";
+ print F "if [ -e unitigging/$asm.ctgStore/seqDB.v001.tig -a -e unitigging/$asm.utgStore/seqDB.v001.tig ] ; then\n";
print F " exit 0\n";
print F "fi\n";
print F "\n";
- print F getBinDirectoryShellCode();
- print F "\n";
if (getGlobal("unitigger") eq "bogart") {
print F "\$bin/bogart \\\n";
- print F " -G $wrk/$asm.gkpStore \\\n";
- print F " -O $wrk/$asm.ovlStore \\\n";
- print F " -o $wrk/4-unitigger/$asm \\\n";
+ print F " -G ../$asm.gkpStore \\\n";
+ print F " -O ../$asm.ovlStore \\\n";
+ print F " -o ./$asm \\\n";
print F " -gs " . getGlobal("genomeSize") . " \\\n";
- print F " -eg " . getGlobal("utgOvlErrorRate") . " \\\n";
- print F " -eM " . getGlobal("utgOvlErrorRate") . " \\\n";
+ print F " -eg " . getGlobal("utgErrorRate") . " \\\n";
+ print F " -eM " . getGlobal("utgErrorRate") . " \\\n";
print F " -el " . $overlapLength . " \\\n";
print F " -dg " . getGlobal("utgGraphDeviation") . " \\\n";
print F " -db " . getGlobal("utgGraphDeviation") . " \\\n";
@@ -187,24 +214,42 @@ sub unitig ($$) {
print F " -M " . getGlobal("batMemory") . " \\\n" if (defined(getGlobal("batMemory")));
print F " -unassembled " . getGlobal("contigFilter") . " \\\n" if (defined(getGlobal("contigFilter")));
print F " " . getGlobal("batOptions") . " \\\n" if (defined(getGlobal("batOptions")));
- print F " > $wrk/4-unitigger/unitigger.err 2>&1 \\\n";
+ print F " > ./unitigger.err 2>&1 \\\n";
print F "&& \\\n";
- print F "mv $wrk/4-unitigger/$asm.ctgStore $wrk/$asm.ctgStore \\\n";
+ print F "mv ./$asm.ctgStore ../$asm.ctgStore \\\n";
print F "&& \\\n";
- print F "mv $wrk/4-unitigger/$asm.utgStore $wrk/$asm.utgStore\n";
+ print F "mv ./$asm.utgStore ../$asm.utgStore\n";
} else {
caFailure("unknown unitigger '" . getGlobal("unitigger") . "'", undef);
}
print F "\n";
+ print F stashFileShellCode("unitigging/4-unitigger", "$asm.unitigs.gfa", "");
+ print F stashFileShellCode("unitigging/4-unitigger", "$asm.contigs.gfa", "");
+ print F "\n";
+ print F stashFileShellCode("unitigging/$asm.ctgStore", "seqDB.v001.dat", "");
+ print F stashFileShellCode("unitigging/$asm.ctgStore", "seqDB.v001.tig", "");
+ print F "\n";
+ print F stashFileShellCode("unitigging/$asm.utgStore", "seqDB.v001.dat", "");
+ print F stashFileShellCode("unitigging/$asm.utgStore", "seqDB.v001.tig", "");
+ print F "\n";
+ print F "\$bin/tgStoreDump \\\n"; # Duplicated in reportUnitigSizes()
+ print F " -G ../$asm.gkpStore \\\n"; # Done here so we don't need another
+ print F " -T ../$asm.ctgStore 1 \\\n"; # pull of gkpStore and ctgStore
+ print F " -sizes -s " . getGlobal("genomeSize") . " \\\n";
+ print F "> ../$asm.ctgStore/seqDB.v001.sizes.txt";
+ print F "\n";
+ print F stashFileShellCode("unitigging/$asm.ctgStore", "seqDB.v001.sizes.txt", "");
+ print F "\n";
print F "exit 0\n";
close(F);
+ stashFile("$path/unitigger.sh");
+
finishStage:
- emitStage($WRK, $asm, "unitig");
- buildHTML($WRK, $asm, "utg");
- stopAfter("unitig");
+ emitStage($asm, "unitig");
+ buildHTML($asm, "utg");
allDone:
}
@@ -212,18 +257,21 @@ sub unitig ($$) {
-sub unitigCheck ($$) {
- my $WRK = shift @_; # Root work directory
- my $wrk = "$WRK/unitigging"; # Local work directory
+sub unitigCheck ($) {
my $asm = shift @_;
my $attempt = getGlobal("canuIteration");
- my $path = "$wrk/4-unitigger";
+ my $path = "unitigging/4-unitigger";
+
+ goto allDone if (skipStage($asm, "unitigCheck", $attempt) == 1);
+ goto allDone if (fileExists("$path/unitigger.success"));
+ goto finishStage if (fileExists("unitigging/$asm.utgStore/seqDB.v001.tig") &&
+ fileExists("unitigging/$asm.ctgStore/seqDB.v001.tig"));
- goto allDone if (skipStage($WRK, $asm, "unitigCheck", $attempt) == 1);
- goto allDone if ((-e "$wrk/$asm.ctgStore/seqDB.v001.tig") && (-e "$wrk/$asm.utgStore/seqDB.v001.tig"));
+ fetchFile("$path/unitigger.sh");
# Since there is only one job, if we get here, we're not done. Any other 'check' function
# shows how to process multiple jobs. This only checks for the existence of the final outputs.
+ # (meryl is the same)
# If not the first attempt, report the jobs that failed, and that we're recomputing.
@@ -241,26 +289,27 @@ sub unitigCheck ($$) {
# Otherwise, run some jobs.
- print STDERR "-- Unitigger attempt $attempt begins.\n";
+ emitStage($asm, "unitigCheck", $attempt);
+ buildHTML($asm, "utg");
- emitStage($WRK, $asm, "unitigCheck", $attempt);
- buildHTML($WRK, $asm, "utg");
-
- submitOrRunParallelJob($WRK, $asm, "bat", $path, "unitigger", (1));
+ submitOrRunParallelJob($asm, "bat", $path, "unitigger", (1));
return;
- # If onGrid, the submitOrRun() has submitted parallel jobs to the grid, and resubmitted the
- # executive. # The parallel version never gets here
-
finishStage:
print STDERR "-- Unitigger finished successfully.\n";
- reportUnitigSizes($wrk, $asm, 1, "after unitig construction");
+ make_path($path); # With object storage, we might not have this directory!
+
+ open(F, "> $path/unitigger.success") or caExit("can't open '$path/unitigger.success' for writing: $!", undef);
+ close(F);
- setGlobal("canuIteration", 1);
- emitStage($WRK, $asm, "unitigCheck");
- buildHTML($WRK, $asm, "utg");
- stopAfter("unitigCheck");
+ stashFile("$path/unitigger.success");
+
+ reportUnitigSizes($asm, 1, "after unitig construction");
+
+ emitStage($asm, "unitigCheck");
+ buildHTML($asm, "utg");
allDone:
+ stopAfter("unitig");
}
diff --git a/src/pipelines/parallel-ovl-store-test.sh b/src/pipelines/parallel-ovl-store-test.sh
new file mode 100644
index 0000000..fee59f7
--- /dev/null
+++ b/src/pipelines/parallel-ovl-store-test.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+
+bin="/work/canu/FreeBSD-amd64/bin"
+
+if [ ! -e test.gkpStore ] ; then
+ echo Didn\'t find \'test.gkpStore\', can\'t make fake overlaps.
+ exit
+fi
+
+rm -rf *ovb *counts test.ovlStore test.ovlStore.seq
+
+jobs=50
+size=100000000
+
+# Create a bunch of overlap outputs, one for each job.
+
+echo ""
+echo "Creating overlaps"
+echo ""
+
+for ii in `seq 1 $jobs` ; do
+ name=`printf %03d $ii`
+ $bin/overlapImport -G test.gkpStore -o $name.ovb -random $size
+done
+
+# Configure.
+
+$bin/ovStoreBuild -G test.gkpStore -O test.ovlStore -M 0.26 -config config *ovb > config.err 2>&1
+
+buckets=`grep Will config.err | grep buckets | awk '{ print $9 }'`
+
+echo ""
+echo "Using $buckets buckets for sorting"
+echo ""
+
+# Bucketize each job.
+
+for ii in `seq 1 $jobs` ; do
+ name=`printf %03d $ii`
+ $bin/ovStoreBucketizer -G test.gkpStore -O test.ovlStore -C config -i $name.ovb -job $ii
+done
+
+# Sort each bucket.
+
+echo ""
+echo "Sorting each bucket"
+echo ""
+
+for ii in `seq 1 $buckets` ; do
+ echo ""
+ echo "Sorting bucket $ii"
+ echo ""
+ $bin/ovStoreSorter -deletelate -G test.gkpStore -O test.ovlStore -F $jobs -job $ii $jobs
+done
+
+# And build the index
+
+$bin/ovStoreIndexer -O test.ovlStore -F $buckets
+
+# And a sequential store?
+
+exit
+
+echo ""
+echo "Building sequential store"
+echo ""
+
+$bin/ovStoreBuild -G test.gkpStore -O test.ovlStore.seq *ovb > test.ovlStore.seq.err 2>&1
diff --git a/src/pipelines/sanity/sanity-all-done.pl b/src/pipelines/sanity/sanity-all-done.pl
deleted file mode 100644
index 5e7e4d9..0000000
--- a/src/pipelines/sanity/sanity-all-done.pl
+++ /dev/null
@@ -1,240 +0,0 @@
-#!/usr/bin/env perl
-
-###############################################################################
- #
- # This file is part of canu, a software program that assembles whole-genome
- # sequencing reads into contigs.
- #
- # This software is based on:
- # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
- # the 'kmer package' (http://kmer.sourceforge.net)
- # both originally distributed by Applera Corporation under the GNU General
- # Public License, version 2.
- #
- # Canu branched from Celera Assembler at its revision 4587.
- # Canu branched from the kmer project at its revision 1994.
- #
- # Modifications by:
- #
- # Brian P. Walenz beginning on 2015-OCT-12
- # are a 'United States Government Work', and
- # are released in the public domain
- #
- # File 'README.licenses' in the root directory of this distribution contains
- # full conditions and disclaimers for each license.
- ##
-
-use MIME::QuotedPrint;
-use MIME::Base64;
-use HTML::Entities;
-use strict;
-
-if (scalar(@ARGV) != 5) {
- die "wrong args.\n";
-}
-
-my $wrkdir = shift @ARGV;
-my $thisdate = shift @ARGV;
-my $label = shift @ARGV;
-my $addresses = shift @ARGV;
-my $assemblies = shift @ARGV;
-my @assemblies = split ',', $assemblies;
-my $timenow = localtime();
-
-print "Subject: CAtest $thisdate $label\n";
-print "To: $addresses\n";
-print "Content-Type: multipart/mixed; boundary=Bri_Says_This_Is_The_Boundary\n"; # alternative if no atachments
-print "\n";
-
-
-########################################
-#
-# Dump the HTML part
-#
-print "--Bri_Says_This_Is_The_Boundary\n";
-print "Content-Type: text/html; charset=\"iso-8859-1\"\n";
-print "Content-Transfer-Encoding: quoted-printable\n";
-print "\n";
-
-print "<HTML>\n";
-print "<P>\n";
-print "<font size=4>" . encode_entities("Results for $thisdate (Finished at $timenow).") . "</font>\n";
-print "</P>\n";
-print "\n";
-
-print "<P>\n";
-
-foreach my $asm (@assemblies) {
- open(R, "< $wrkdir/$thisdate/$asm/sanity-result.out");
- $_ = <R>;
- close(R);
-
- chomp;
- # Green!
- s!SUCCESS!<span style=\"color: rgb(0, 255, 0);\">SUCCESS</span>!g;
- s!same!<span style=\"color: rgb(0, 255, 0);\">same</span>!g;
-
- # Red
- s!FAILURE!<span style=\"color: rgb(255, 0, 0);\">FAILURE</span>!g;
- s!differs!<span style=\"color: rgb(255, 0, 0);\">differs</span>!g;
-
- print "$_<BR>\n";
-}
-
-print "</P>\n";
-
-print "<HR>\n";
-if ((! -e "$wrkdir/$thisdate/wgs/kmer.updates") || (-z "$wrkdir/$thisdate/wgs/kmer.updates")) {
- print "<P>\n";
- print "<font size=4>No changes to kmer.</font>\n";
- print "</P>\n";
-} else {
- print "<P>\n";
- print "<font size=4>Changes for kmer.</font>\n";
- print "</P>\n";
- print "<P>\n";
- open(F, "< $wrkdir/$thisdate/wgs/kmer.updates");
- while (<F>) {
- chomp;
- print encode_entities($_) . "<BR>\n";
- }
- close(F);
- print "</P>\n";
-}
-
-print "<HR>\n";
-if ((! -e "$wrkdir/$thisdate/wgs/src.updates") || (-z "$wrkdir/$thisdate/wgs/src.updates")) {
- print "<P>\n";
- print "<font size=+2>No chages to wgs-assembler.</font>\n";
- print "</P>\n";
-} else {
- print "<P>\n";
- print "<font size=+2>Changes for wgs-assembler.</font>\n";
- print "</P>\n";
- print "<P>\n";
- open(F, "< $wrkdir/$thisdate/wgs/src.updates");
- while (<F>) {
- chomp;
- print encode_entities($_) . "<BR>\n";
- }
- close(F);
- print "</P>\n";
-}
-
-#print "<HR>\n";
-#print "<P>\n";
-#print "<font size=+2>Build Results for kmer.</font>\n";
-#print "</P>\n";
-#open(F, "< $wrkdir/$thisdate/wgs/kmer/make.err");
-#while (<F>) {
-# chomp;
-# print encode_entities($_) . "<BR>\n";
-#}
-#close(F);
-
-#print "<HR>\n";
-#print "<P>\n";
-#print "<font size=+2>Build Results for wgs-assembler.</font>\n";
-#print "</P>\n";
-#open(F, "< $wrkdir/$thisdate/wgs/src/make.err");
-#while (<F>) {
-# chomp;
-# print encode_entities($_) . "<BR>\n";
-#}
-#close(F);
-
-print "</HTML>\n";
-print "\n";
-
-
-
-########################################
-#
-# Dump the ASCII part -- not dumped; outlook shows this as another attachment
-#
-#print "--Bri_Says_This_Is_The_Boundary\n";
-#print "Content-Type: text/plain; charset=\"iso-8859-1\"\n";
-#print "Content-Transfer-Encoding: quoted-printable\n";
-#print "\n";
-
-
-
-########################################
-#
-# Dump the checkout and build results
-#
-attachFile("$wrkdir/$thisdate/wgs/kmer.updates", "kmerChanges.txt");
-attachFile("$wrkdir/$thisdate/wgs/src.updates", "wgsChanges.txt");
-attachFile("$wrkdir/$thisdate/wgs/kmer/make.err", "kmerBuildError.txt");
-attachFile("$wrkdir/$thisdate/wgs/src/make.err", "wgsBuildError.txt");
-
-
-########################################
-#
-# Dump the QC attachments
-#
-foreach my $asm (@assemblies) {
- my $errfile = "$wrkdir/$thisdate/$asm/sanity-error.out";
- my $qcfile = "$wrkdir/$thisdate/$asm/sanity-qc.out";
-
- if ((-e $qcfile) && (! -z $qcfile)) {
- my @n = split "/", $qcfile;
- my $l = scalar(@n);
- my $name = "$n[$l-3]-$n[$l-2]-qc.txt";
-
- attachFile($qcfile, $name);
- }
-
- if ((-e $errfile) && (! -z $errfile)) {
- my @n = split "/", $errfile;
- my $l = scalar(@n);
- my $name = "$n[$l-3]-$n[$l-2]-error.txt";
-
- attachFile($errfile, $name);
- }
-}
-
-
-########################################
-#
-# All done. DO NOT add another boundary - this results in a bogus empty attachment.
-#
-#print "--Bri_Says_This_Is_The_Boundary\n";
-
-
-
-
-
-
-
-
-sub attachFile ($$) {
- my $fileName = shift @_; # Name of file on disk
- my $attachName = shift @_; # Name of file in email
-
- return if (! -e $fileName);
- return if (-z $fileName);
-
- print "--Bri_Says_This_Is_The_Boundary\n";
- print "Content-Type: application/octet-stream; name=\"$attachName\"\n";
- print "Content-Transfer-Encoding: base64\n";
- print "Content-Disposition: attachment; filename=\"$attachName\"\n";
- print "\n";
-
- my @message;
- my $message;
-
- # Winblows wants \r\n, and since that's the primary consumer of these emails, we put it in.
-
- open(F, "< $fileName");
- while (<F>) {
- chomp;
- push @message, "$_\r\n";
- }
- close(F);
-
- $message = join "", @message;
-
- local($/) = undef;
- print encode_base64($message);
-}
diff --git a/src/pipelines/sanity/sanity-merge-qc.pl b/src/pipelines/sanity/sanity-merge-qc.pl
deleted file mode 100644
index deb5c70..0000000
--- a/src/pipelines/sanity/sanity-merge-qc.pl
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/usr/bin/env perl
-
-###############################################################################
- #
- # This file is part of canu, a software program that assembles whole-genome
- # sequencing reads into contigs.
- #
- # This software is based on:
- # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
- # the 'kmer package' (http://kmer.sourceforge.net)
- # both originally distributed by Applera Corporation under the GNU General
- # Public License, version 2.
- #
- # Canu branched from Celera Assembler at its revision 4587.
- # Canu branched from the kmer project at its revision 1994.
- #
- # Modifications by:
- #
- # Brian P. Walenz beginning on 2015-OCT-12
- # are a 'United States Government Work', and
- # are released in the public domain
- #
- # File 'README.licenses' in the root directory of this distribution contains
- # full conditions and disclaimers for each license.
- ##
-
-use strict;
-
-# Simple hack to merge multiple QC reports into one. Uses the first
-# QC on the command line to figure out what fields exist, then writes
-# a multiple column QC using those fields and data from all QC files
-# on the command line.
-
-my @labels;
-my %values;
-
-my $s;
-my $l;
-my $v;
-
-my $firstFile = 1;
-
-while (scalar(@ARGV) > 0) {
- if (open(F, "< $ARGV[0]")) {
- if ($ARGV[0] =~ m!.*/NIGHTLY/(\d\d\d\d-\d\d-\d\d-\d\d\d\d)/(.*)/9-terminator/.*.qc$!) {
- print "$1/$2\n";
- } else {
- print "$ARGV[0]\n";
- }
-
- while (<F>) {
- $_ =~ s/^\s+//;
- $_ =~ s/\s+$//;
-
- if ($_ =~ m/^\s+$/) {
- next;
- }
-
- if ($_ =~ m/^\[(.*)\]$/) {
- $s = $1;
- next;
- }
-
- if ($_ =~ m/^(.*)=(.*)$/) {
- $l = "$s\0$1";
- $v = $2;
- } else {
- next;
- }
-
- if ($firstFile) {
- push @labels, $l;
- }
-
- $values{$l} .= substr(" $v ", 0, 16);
- $values{$l} .= "BRIWASHERE";
- }
- close(F);
-
- my @k = keys %values;
- foreach my $l (@k) {
- if ($values{$l} =~ m/^(.*)BRIWASHERE$/) {
- $values{$l} = $1;
- } else {
- $values{$l} .= substr(" N/A ", 0, 16);
- }
- }
-
- $firstFile = 0;
- }
-
- shift @ARGV;
-}
-
-
-my $lasts;
-foreach my $xx (@labels) {
- ($s, $l) = split '\0', $xx;
-
- if ($s ne $lasts) {
- print "\n[$s]\n";
- $lasts = $s;
- }
-
- $l = substr("$l ", 0, 20);
-
- my $d = " ";
-
- my @v = split '\s+', $values{$xx};
- foreach my $a (@v) {
- foreach my $b (@v) {
- # Because $xx is whitespace justified, the first
- # thing from the split can be empty.
- if (($a ne "") && ($b ne "") && ($a ne $b)) {
- $d = " ** ";
- }
- }
- }
-
- print "$l$d$values{$xx}\n";
-}
diff --git a/src/pipelines/sanity/sanity.pl b/src/pipelines/sanity/sanity.pl
deleted file mode 100755
index 52904b8..0000000
--- a/src/pipelines/sanity/sanity.pl
+++ /dev/null
@@ -1,382 +0,0 @@
-#!/usr/bin/env perl
-
-###############################################################################
- #
- # This file is part of canu, a software program that assembles whole-genome
- # sequencing reads into contigs.
- #
- # This software is based on:
- # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
- # the 'kmer package' (http://kmer.sourceforge.net)
- # both originally distributed by Applera Corporation under the GNU General
- # Public License, version 2.
- #
- # Canu branched from Celera Assembler at its revision 4587.
- # Canu branched from the kmer project at its revision 1994.
- #
- # Modifications by:
- #
- # Brian P. Walenz beginning on 2015-OCT-12
- # are a 'United States Government Work', and
- # are released in the public domain
- #
- # File 'README.licenses' in the root directory of this distribution contains
- # full conditions and disclaimers for each license.
- ##
-
-use strict;
-use Config; # for @signame
-use Time::Local;
-
-# fetch
-# update the local repositories from github.
-#
-# checkout date
-# Date must be of the form yyyy-mm-dd-hhmm. This will checkout
-# the assembler current at that time. If not present, the current
-# time is assumed. A change log is generated from the last
-# date present.
-#
-# build date
-# Builds the assembler checked out into 'date'.
-#
-# assemble date specFile .... email ...
-# Launches canu on each specFile, running it in directory 'date'.
-# Each specFile must have a unique name; the assembly is named
-# after the specFile.
-# At the end, a diff is generated to POINTERS/$prefix.last and
-# POINTERS/$prefix.reference. If the assembly finished, POINTERS/$prefix.last
-# is updated.
-#
-# submit date
-# Submit ourself for execution at date.
-#
-
-my $oper = shift @ARGV;
-
-my $site = undef;
-my $wrkdir = undef;
-my $gitrepo = undef;
-
-my $tz = `date +%z`; chomp $tz;
-
-
-# Set up for the machine we're on.
-
-
-if (-d "/gryphon") {
- $site = "gryphon";
- $wrkdir = "/data/projects/phillippy/scratch/NIGHTLY";
- $gitrepo = "/data/projects/phillippy/scratch/NIGHTLY/canu";
-}
-
-elsif (-d "/data/walenzbp/NIGHTLY/") {
- $site = "BPWI";
- $wrkdir = "/data/walenzbp/NIGHTLY";
- $gitrepo = "/data/walenzbp/NIGHTLY/canu";
-}
-
-elsif (-d "/assembly/NIGHTLY/") {
- $site = "BPWI";
- $wrkdir = "/assembly/NIGHTLY";
- $gitrepo = "/assembly/NIGHTLY/canu";
-}
-
-elsif (-d "/work/NIGHTLY/") {
- $site = "BPWI";
- $wrkdir = "/work/NIGHTLY";
- $gitrepo = "/work/NIGHTLY/canu";
-}
-
-else {
- die "Unknown site configuration.\n";
-}
-
-
-# Now do something.
-
-
-if ($oper eq "fetch") {
- fetch();
- exit(0);
-}
-
-
-if ($oper eq "checkout") {
- checkout(@ARGV);
- exit(0);
-}
-
-
-if ($oper eq "build") {
- build(@ARGV);
- exit(0);
-}
-
-
-if ($oper eq "assemble") {
- # if canu.merge.out is "Already up-to-date." we can skip running stuff.
- assemble(@ARGV);
- exit(0);
-}
-
-
-if ($oper eq "submit") {
- submit(@ARGV);
- exit(0);
-}
-
-
-die "$0: unknown action '$oper'\n";
-exit(0);
-
-
-
-
-
-
-sub parseDate ($) {
- my $pathdate = shift @_;
- my $thisdate;
- my $lastdate;
-
- # Until our Makefile is fixed, the ":" cannot be used, and so we need
- # to adopt a slightly goofy timestamp name.
-
- if (defined($pathdate)) {
- if ($pathdate =~ m/^\d\d\d\d-\d\d-\d\d-\d\d\d\d$/) {
- $thisdate = $pathdate;
- } else {
- die "Malformed ddir '$pathdate'\n";
- }
- } else {
- my @v = localtime;
- $v[5] += 1900;
- $v[4]++;
-
- $v[5] = substr("0000$v[5]", -4);
- $v[4] = substr("0000$v[4]", -2);
- $v[3] = substr("0000$v[3]", -2);
- $v[2] = substr("0000$v[2]", -2);
- $v[1] = substr("0000$v[1]", -2);
-
- $thisdate = "$v[5]-$v[4]-$v[3]-$v[2]$v[1]";
- }
-
- open(F, "ls $wrkdir |");
- while (<F>) {
- chomp;
- if (m/^\d\d\d\d-\d\d-\d\d-\d\d\d\d$/) {
- if ($_ lt $thisdate) {
- $lastdate = $_;
- }
- }
- }
- close(F);
-
- #print STDERR "Working on '$thisdate' -- last found is '$lastdate'.\n";
-
- return($thisdate, $lastdate);
-}
-
-
-
-sub gitDate ($) {
- my $date = shift @_;
-
- if ((!defined($date)) || ($date eq "")) {
- return($date);
- }
-
- if ($date =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) {
- $date = "$1-$2-$3T$4:$5$tz";
- } else {
- die "Malformed date '$date' to gitDate().\n";
- }
-
- return($date);
-}
-
-
-
-sub fetch () {
-
- print STDERR "\n";
- print STDERR "SANITY FETCH\n";
- print STDERR "\n";
-
- if (! -d "$gitrepo") {
- print STDERR " Cloning new repo '$gitrepo'.\n";
- system("mkdir -p $gitrepo") if (! -d "$gitrepo");
- system("cd $gitrepo/.. ; git clone git\@github.com:marbl/canu.git > canu.clone.out 2>&1");
- }
- else {
- print STDERR " Updating existing repo '$gitrepo'.\n";
- system("cd $gitrepo ; git fetch > ../canu.fetch.out 2>&1");
- system("cd $gitrepo ; git merge > ../canu.merge.out 2>&1");
- }
-}
-
-
-
-sub checkout (@) {
- my ($thisdate, $lastdate) = parseDate($_[0]);
-
- my $ldate = gitDate($lastdate);
- my $tdate = gitDate($thisdate);
-
- print STDERR "\n";
- print STDERR "SANITY CHECKOUT $wrkdir/$thisdate\n";
- print STDERR "\n";
-
- if (-d "$wrkdir/$thisdate/canu") {
- print STDERR " $wrkdir/$thisdate/canu already exists. Please remove to rerun.\n";
- return;
- }
-
- system("mkdir -p $wrkdir/$thisdate/canu") if (! -d "$wrkdir/$thisdate/canu");
-
- # Clone the clone.
-
- system("cd $wrkdir/$thisdate/canu && rsync -a $gitrepo/ .");
-
- # Find a git hash to grab the bits we want.
-
- my $hash = `cd $wrkdir/$thisdate/canu && git rev-list -n 1 --first-parent --before=\"$tdate\" master`; chomp $hash;
-
- # Checkout that hash.
-
- system("cd $wrkdir/$thisdate/canu && git checkout -b SANITY-$thisdate $hash");
-
- # git clone $gitrepo $wrkdir/$thisdate/canu
-
- if ($ldate ne "") {
- print sTDERR "log\n";
- system("cd $wrkdir/$thisdate/canu && git log --after=\"$ldate\" --until=\"$tdate\" > $wrkdir/$thisdate/canu.updates");
- }
-
- print STDERR " $thisdate checked out!\n";
-}
-
-
-
-sub build (@) {
- my ($thisdate, $lastdate) = parseDate($_[0]);
-
- print STDERR "\n";
- print STDERR "SANITY BUILD $wrkdir/$thisdate\n";
- print STDERR "\n";
-
- if (-e "$wrkdir/$thisdate/canu/src/make.err") {
- print STDERR " $wrkdir/$thisdate/canu was already built once (successuflly or not). Please cleanup first.\n";
- return;
- }
-
- system("cd $wrkdir/$thisdate/canu/src && gmake -j 12 > make.out 2> make.err");
-}
-
-
-
-sub assemble (@) {
- my ($thisdate, $lastdate) = parseDate(shift @_);
-
- print STDERR "\n";
- print STDERR "ASSEMBLE\n";
- print STDERR "\n";
-
- my $syst = `uname -s`; chomp $syst; # OS implementation
- my $arch = `uname -m`; chomp $arch; # Hardware platform
- my $name = `uname -n`; chomp $name; # Name of the system
-
- $arch = "amd64" if ($arch eq "x86_64");
- $arch = "ppc" if ($arch eq "Power Macintosh");
-
- my @spec;
- my $addresses;
-
- foreach my $arg (@_) {
- if ($arg =~ m/\@/) {
- $addresses = $arg if (!defined($addresses));
- $addresses .= ",$arg" if ( defined($addresses));
- }
- else {
- $arg = "$ENV{PWD}/$arg" if ($arg !~ m/^\//);
- push @spec, $arg;
- }
- }
-
- #open(F, "> $wrkdir/$thisdate/asm-done.sh");
- #print F "#!/bin/sh\n";
- #print F "\n";
- #print F "perl $wrkdir/sanity-asm-done.pl $wrkdir \$1 $thisdate\n";
- #close(F);
-
- #open(F, "> $wrkdir/$thisdate/all-done.sh");
- #print F "#!/bin/sh\n";
- #print F "\n";
- #print F "perl $wrkdir/sanity-all-done.pl $wrkdir $thisdate \$1 $addresses \$2 \\\n";
- #print F "| \\\n";
- #print F "tee \"$wrkdir/$thisdate/sanity-all-done.\$1\" \\\n";
- #print F "| \\\n";
- #print F "/usr/sbin/sendmail -i -t -f thebri\@gmail.com\n" if ($site eq "JCVI");
- #print F "/usr/local/sbin/ssmtp thebri\@gmail.com\n" if ($site eq "BPWI");
- #close(F);
-
- foreach my $s (@spec) {
- my @c = split '/', $s;
- my $n = $c[scalar(@c) - 1];
-
- $n =~ s/.spec$//;
- $n =~ s/.specFile$//;
-
- print STDERR "Submitting assembly '$n' for spec '$s'.\n";
-
- print STDERR "cd $wrkdir/$thisdate \\\n";
- print STDERR "&& \\\n";
- print STDERR "$wrkdir/$thisdate/canu/$syst-$arch/bin/canu -p $n -d $n -s $s";
-
- system("cd $wrkdir/$thisdate && $wrkdir/$thisdate/canu/$syst-$arch/bin/canu -p $n -d $n -s $s");
- }
-}
-
-
-
-sub submit (@) {
- my ($thisdate, $lastdate) = parseDate(shift @ARGV);
- my $seconds;
-
- if ($thisdate =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) {
- $seconds = timelocal(0, $5, $4, $3, $2 - 1, $1);
- }
-
- $seconds += 604800; # one week
- $seconds += 86400; # one day
- $seconds += 14400; # four hours
- $seconds += 7200; # two hours
- $seconds += 21600; # six hours
- $seconds += 43200; # twelve hours
-
- my @v = localtime($seconds);
-
- $v[5] += 1900;
- $v[4]++;
-
- $v[5] = substr("0000$v[5]", -4);
- $v[4] = substr("0000$v[4]", -2);
- $v[3] = substr("0000$v[3]", -2);
- $v[2] = substr("0000$v[2]", -2);
- $v[1] = substr("0000$v[1]", -2);
-
- #$v[2] = "00";
- #$v[1] = "01";
-
- my $nextdate = "$v[5]-$v[4]-$v[3]-$v[2]$v[1]";
- my $nexthold = "$v[5]$v[4]$v[3]$v[2]$v[1].00";
- my $nextname = "$v[4]$v[3]-$v[2]$v[1]";
-
- print STDERR "Submit next at date='$nextdate' hold='$nexthold' name='$nextname'\n";
-
- #system("qsub -cwd -j y -o $nextdate.err -A assembly-nightly -N snty$nextname -a $nexthold -b n sanity.sh $nextdate grid");
-}
-
-
diff --git a/src/pipelines/sanity/sanity.sh b/src/pipelines/sanity/sanity.sh
deleted file mode 100644
index cf3d7bd..0000000
--- a/src/pipelines/sanity/sanity.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/sh
-
-# (re)Load the sge config.
-. $SGE_ROOT/$SGE_CELL/common/settings.sh
-
-#
-# Master controller of nightly sanity checks. Optional date on command line.
-#
-# To start the next nightly, launch with something like:
-# qsub -cwd -j y -o init.err -A assembly-nightly -l fast -a 200907180001.00 -b n sanity.sh 2009-07-18-0001 grid
-#
-# To start a previous nightly, the sge hold isn't needed, well, neither is sge for that matter.
-# Just run sanity.sh with the date/time you want to start at.
-#
-# CAREFUL! Submitting something in the past will automagically submit every nightly since that date!
-#
-
-date=$1
-bins=/work/canu/src/pipelines/sanity
-spec=/work/canu/src/pipelines/sanity
-
-if [ x$date = x ] ; then
- date=`date +%Y-%m-%d-%H%M`
-fi
-
-echo "SANITY BEGINS for $date at `date`"
-
-# Remove old versions
-#perl $bins/sanity-purge-old.pl purge
-#rm -rf DEL
-
-perl $bins/sanity.pl fetch # Update the local repo.
-perl $bins/sanity.pl checkout $date # Checkout from that repo.
-perl $bins/sanity.pl build $date # Compile.
-perl $bins/sanity.pl submit $date # Submit the next iteration.
-
-# Run small stuff daily.
-
-perl $bins/sanity.pl assemble $date $spec/small1.spec
-perl $bins/sanity.pl assemble $date $spec/small2.spec
-perl $bins/sanity.pl assemble $date $spec/small3.spec
-perl $bins/sanity.pl assemble $date $spec/small4.spec
-
-# Run big stuff weekly.
-
-#if [ `date +%u` = 6] ; then
-# sh sanity-weekly-dros.sh $date
-# sh sanity-weekly-moore.sh $date
-#fi
-
diff --git a/src/pipelines/simple-repeat-test.pl b/src/pipelines/simple-repeat-test.pl
new file mode 100644
index 0000000..55cae63
--- /dev/null
+++ b/src/pipelines/simple-repeat-test.pl
@@ -0,0 +1,107 @@
+#!/usr/bin/env perl
+
+###############################################################################
+ #
+ # This file is part of canu, a software program that assembles whole-genome
+ # sequencing reads into contigs.
+ #
+ # This software is based on:
+ # 'Celera Assembler' (http://wgs-assembler.sourceforge.net)
+ # the 'kmer package' (http://kmer.sourceforge.net)
+ # both originally distributed by Applera Corporation under the GNU General
+ # Public License, version 2.
+ #
+ # Canu branched from Celera Assembler at its revision 4587.
+ # Canu branched from the kmer project at its revision 1994.
+ #
+ # Modifications by:
+ #
+ # Brian P. Walenz beginning on 2017-JAN-09
+ # are a 'United States Government Work', and
+ # are released in the public domain
+ #
+ # File 'README.licenses' in the root directory of this distribution contains
+ # full conditions and disclaimers for each license.
+ ##
+
+use strict;
+
+# Generate a circular 'genome' with nUnique pieces and nUnique+1 repeats, the pair at the start/end
+# forming the circularization. Assemble it, and compare against 'reference'.
+
+my $repeatSize = 10000;
+my $uniqueSize = 90000;
+my $nUnique = 4;
+
+my $readLenL = 4000; # Long reads
+my $readLenM = 3500; # Medium reads
+my $readLenS = 3000; # Short reads
+
+my $coverageL = 30;
+my $coverageM = 1;
+my $coverageS = 1;
+
+sub randomSequence ($) {
+ my $L = shift @_;
+
+ open(F, "leaff -G 1 $L $L |");
+ $_ = <F>; chomp;
+ $_ = <F>; chomp;
+ close(F);
+
+ return($_);
+}
+
+if (! -e "reads.genome.fasta") {
+ my $len = 0;
+
+ my $R = randomSequence($repeatSize);
+ my $Rlen = length($R);
+
+ open(O, "> reads.genome.fasta");
+
+ print O ">G\n";
+
+ for (my $r=0; $r < $nUnique; $r++) {
+ print STDERR "REPEAT $len ", $len + $Rlen, "\n";
+ print O "$R\n";
+ $len += $Rlen;
+
+ my $U = randomSequence($uniqueSize);
+ my $Ulen = length($U);
+
+ print STDERR "UNIQUE $len ", $len + $Ulen, "\n";
+ print O "$U\n";
+ $len += $Ulen;
+ }
+
+ print STDERR "REPEAT $len ", $len + $Rlen, "\n";
+ print O "$R\n";
+ $len += $Rlen;
+
+ close(O);
+}
+
+
+if (! -e "reads.s.fastq") {
+ system("fastqSimulate -f reads.genome.fasta -o readsL -l $readLenL -x $coverageL -em 0.005 -ei 0 -ed 0 -se");
+ system("fastqSimulate -f reads.genome.fasta -o readsM -l $readLenM -x $coverageM -em 0.005 -ei 0 -ed 0 -se");
+ system("fastqSimulate -f reads.genome.fasta -o readsS -l $readLenS -x $coverageS -em 0.005 -ei 0 -ed 0 -se");
+}
+
+my $canu;
+
+$canu = "canu -assemble -p test -d test";
+$canu .= " ovlMerThreshold=0";
+$canu .= " useGrid=0 genomeSize=1m ovlThreads=24";
+$canu .= " -pacbio-corrected readsL.s.fastq";
+$canu .= " -pacbio-corrected readsM.s.fastq";
+$canu .= " -pacbio-corrected readsS.s.fastq";
+
+system($canu);
+
+system("dotplot.sh UTG reads.genome.fasta test/test.unitigs.fasta");
+system("dotplot.sh CTG reads.genome.fasta test/test.contigs.fasta");
+
+exit(0);
+
diff --git a/src/stores/gatekeeperPartition.C b/src/stores/gatekeeperPartition.C
index ac89892..97c259a 100644
--- a/src/stores/gatekeeperPartition.C
+++ b/src/stores/gatekeeperPartition.C
@@ -34,6 +34,8 @@
//#include "AS_UTL_fileIO.H"
+#include <libgen.h>
+
uint32 *
buildPartition(char *tigStoreName,
@@ -117,9 +119,8 @@ buildPartition(char *tigStoreName,
int
main(int argc, char **argv) {
- char *gkpStoreName = NULL;
- char *tigStoreName = NULL;
- char gkpCloneName[FILENAME_MAX];
+ char *gkpStorePath = NULL;
+ char *tigStorePath = NULL;
uint32 tigStoreVers = 0;
uint32 readCountTarget = 2500; // No partition smaller than this
uint32 partCountTarget = 200; // No more than this many partitions
@@ -130,10 +131,10 @@ main(int argc, char **argv) {
int arg = 1;
while (arg < argc) {
if (strcmp(argv[arg], "-G") == 0) {
- gkpStoreName = argv[++arg];
+ gkpStorePath = argv[++arg];
} else if (strcmp(argv[arg], "-T") == 0) {
- tigStoreName = argv[++arg];
+ tigStorePath = argv[++arg];
tigStoreVers = atoi(argv[++arg]);
} else if (strcmp(argv[arg], "-b") == 0) {
@@ -151,15 +152,28 @@ main(int argc, char **argv) {
arg++;
}
- if (gkpStoreName == NULL) err.push_back("ERROR: no gkpStore (-G) supplied.\n");
- if (tigStoreName == NULL) err.push_back("ERROR: no partition input (-P) supplied.\n");
+ if (gkpStorePath == NULL) err.push_back("ERROR: no gkpStore (-G) supplied.\n");
+ if (tigStorePath == NULL) err.push_back("ERROR: no partition input (-P) supplied.\n");
if (err.size() > 0) {
- fprintf(stderr, "usage: %s -G <gkpStore> -T <tigStore v>\n", argv[0]);
+ fprintf(stderr, "usage: %s -G <gkpStore> -T <tigStore> <v>\n", argv[0]);
fprintf(stderr, " -G <gkpStore> path to gatekeeper store\n");
fprintf(stderr, " -T <tigStore> <v> path to tig store and version to be partitioned\n");
+ fprintf(stderr, "\n");
fprintf(stderr, " -b <nReads> minimum number of reads per partition (50000)\n");
fprintf(stderr, " -p <nPartitions> number of partitions (200)\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Create a partitioned copy of <gkpStore> and place it in <tigStore>/partitionedReads.gkpStore\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "NOTE: Path handling in this is probably quite brittle. Due to an implementation\n");
+ fprintf(stderr, " detail, the new store must have symlinks back to the original store. Canu \n");
+ fprintf(stderr, " wants to use relative paths, and this program tries to adjust <gkpStore> to be\n");
+ fprintf(stderr, " relative to <tigStore/partitionedReads.gkpStore. If it fails to do this correctly,\n");
+ fprintf(stderr, " one of two (seen so far) errors will occur:\n");
+ fprintf(stderr, " Original file '.../partitionedReads.gkpStore/info' doesn't exist, won't make a link to nothing.\n");
+ fprintf(stderr, " Couldn't open '.../partitionedReads.gkpStore/libraries' for mmap: No such file or directoryn");
+ fprintf(stderr, " In both cases, try to simplify <tigStore> -- in particular, remove any '..' or '.' components -- or\n");
+ fprintf(stderr, " run this from a higher/lower directory.\n");
for (uint32 ii=0; ii<err.size(); ii++)
if (err[ii])
@@ -168,30 +182,56 @@ main(int argc, char **argv) {
exit(1);
}
+ char gkpSourcePath[FILENAME_MAX] = {0};
+ char gkpClonePath[FILENAME_MAX] = {0};
- // Open a READ ONLY store. This prevents us from mucking with the non-partitioned reads
- // (like, by changing the read ID or pointer to the blob). We don't need it opened
- // for writing anyway.
+ // We're making a clone of the master gkpStore in the tigStore directory.
- gkStore *gkpStore = gkStore::gkStore_open(gkpStoreName, gkStore_readOnly);
- uint32 numReads = gkpStore->gkStore_getNumReads();
+ snprintf(gkpClonePath, FILENAME_MAX, "%s/partitionedReads.gkpStore", tigStorePath);
- // Clone that store into the tigStore directory.
+ // The path to the gkpStore that we want to use in the link is a wee-bit more complicated.
+ // If it's an absolute path, there's nothing we need to do.
- snprintf(gkpCloneName, FILENAME_MAX, "%s/partitionedReads.gkpStore", tigStoreName);
+ if (gkpStorePath[0] == '/') {
+ strcpy(gkpSourcePath, gkpStorePath);
+ }
- gkpStore->gkStore_clone(gkpCloneName);
+ // But if it's a relative path, we need to add a bunch of dots. One pair to account
+ // for the directory we added above, and then more dots for each component in tigStorePath.
- // Then close the original store and open the clone.
+ else {
+ char t[FILENAME_MAX]; // Copy command line tigStorePath to a
+ char *p = t; // local, and modifiable, space.
- gkpStore->gkStore_close();
+ strcpy(p, tigStorePath);
+
+ strcat(gkpSourcePath, "../"); // One for the directory we created above
+
+ while ((p[0] != '.') || (p[1] != 0)) { // Many for each component in the tigStorePath.
+ strcat(gkpSourcePath, "../");
+ p = dirname(p);
+ }
+
+ if ((gkpStorePath[0] == '.') && (gkpStorePath[1] == '/')) // Finally, append the supplied
+ strcat(gkpSourcePath, gkpStorePath + 2); // gkpStorePath, possibly
+ else // stripping off any ./ at the
+ strcat(gkpSourcePath, gkpStorePath); // start.
+ }
+
+ // Make the clone.
+
+ gkStore::gkStore_clone(gkpSourcePath, gkpClonePath);
+
+ // Open the clone.
- gkpStore = gkStore::gkStore_open(gkpCloneName, gkStore_readOnly);
+ gkStore *gkpStore = gkStore::gkStore_open(gkpClonePath, gkStore_readOnly);
// Scan all the tigs to build a map from read to partition.
- uint32 *partition = buildPartition(tigStoreName, tigStoreVers,
- readCountTarget, partCountTarget, numReads);
+ uint32 *partition = buildPartition(tigStorePath, tigStoreVers,
+ readCountTarget,
+ partCountTarget,
+ gkpStore->gkStore_getNumReads());
// Dump the partition data to the store, let it build partitions.
diff --git a/src/stores/gkStore.C b/src/stores/gkStore.C
index bdcb322..814fffd 100644
--- a/src/stores/gkStore.C
+++ b/src/stores/gkStore.C
@@ -1304,34 +1304,29 @@ gkStore::gkStore_buildPartitions(uint32 *partitionMap) {
void
-gkStore::gkStore_clone(char *clonePath) {
- char rPath[FILENAME_MAX];
+gkStore::gkStore_clone(char *originalPath, char *clonePath) {
+ char cPath[FILENAME_MAX];
char sPath[FILENAME_MAX];
- char dPath[FILENAME_MAX];
+
+ getcwd(cPath, FILENAME_MAX);
AS_UTL_mkdir(clonePath);
- errno = 0;
- realpath(gkStore_path(), rPath);
- if (errno)
- fprintf(stderr, "gkStore::gkStore_clone()- failed to find path of '%s': %s\n",
- gkStore_path(), strerror(errno)), exit(1);
+ chdir(clonePath);
+
+ snprintf(sPath, FILENAME_MAX, "%s/info", originalPath);
+ AS_UTL_symlink(sPath, "info");
- snprintf(sPath, FILENAME_MAX, "%s/info", rPath);
- snprintf(dPath, FILENAME_MAX, "%s/info", clonePath);
- AS_UTL_symlink(sPath, dPath);
+ snprintf(sPath, FILENAME_MAX, "%s/libraries", originalPath);
+ AS_UTL_symlink(sPath, "libraries");
- snprintf(sPath, FILENAME_MAX, "%s/libraries", rPath);
- snprintf(dPath, FILENAME_MAX, "%s/libraries", clonePath);
- AS_UTL_symlink(sPath, dPath);
+ snprintf(sPath, FILENAME_MAX, "%s/reads", originalPath);
+ AS_UTL_symlink(sPath, "reads");
- snprintf(sPath, FILENAME_MAX, "%s/reads", rPath);
- snprintf(dPath, FILENAME_MAX, "%s/reads", clonePath);
- AS_UTL_symlink(sPath, dPath);
+ snprintf(sPath, FILENAME_MAX, "%s/blobs", originalPath);
+ AS_UTL_symlink(sPath, "blobs");
- snprintf(sPath, FILENAME_MAX, "%s/blobs", rPath);
- snprintf(dPath, FILENAME_MAX, "%s/blobs", clonePath);
- AS_UTL_symlink(sPath, dPath);
+ chdir(cPath);
}
diff --git a/src/stores/gkStore.H b/src/stores/gkStore.H
index 3dcb424..84b4241 100644
--- a/src/stores/gkStore.H
+++ b/src/stores/gkStore.H
@@ -486,7 +486,8 @@ public:
void gkStore_buildPartitions(uint32 *partitionMap);
- void gkStore_clone(char *clonePath);
+ static
+ void gkStore_clone(char *originalPath, char *clonePath);
void gkStore_delete(void); // Deletes the files in the store.
void gkStore_deletePartitions(void); // Deletes the files for a partition.
diff --git a/src/stores/ovOverlap.C b/src/stores/ovOverlap.C
index 801246f..15a9a65 100644
--- a/src/stores/ovOverlap.C
+++ b/src/stores/ovOverlap.C
@@ -118,17 +118,8 @@ ovOverlap::swapIDs(ovOverlap const &orig) {
// Copy the overlap as is, then fix it for the ID swap.
-#if (ovOverlapNWORDS == 5)
- dat.dat[0] = orig.dat.dat[0];
- dat.dat[1] = orig.dat.dat[1];
- dat.dat[2] = orig.dat.dat[2];
- dat.dat[3] = orig.dat.dat[3];
- dat.dat[4] = orig.dat.dat[4];
-#else
- dat.dat[0] = orig.dat.dat[0];
- dat.dat[1] = orig.dat.dat[1];
- dat.dat[2] = orig.dat.dat[2];
-#endif
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++)
+ dat.dat[ii] = orig.dat.dat[ii];
// Swap the A and B hangs. If the overlap is flipped, we also need to reverse 5' and 3' hangs to
// make the now-A read forward oriented.
diff --git a/src/stores/ovOverlap.H b/src/stores/ovOverlap.H
index 362495e..8db42ff 100644
--- a/src/stores/ovOverlap.H
+++ b/src/stores/ovOverlap.H
@@ -314,18 +314,12 @@ public:
if (a_iid > that.a_iid) return(false);
if (b_iid < that.b_iid) return(true);
if (b_iid > that.b_iid) return(false);
- if (dat.dat[0] < that.dat.dat[0]) return(true);
- if (dat.dat[0] > that.dat.dat[0]) return(false);
- if (dat.dat[1] < that.dat.dat[1]) return(true);
- if (dat.dat[1] > that.dat.dat[1]) return(false);
- if (dat.dat[2] < that.dat.dat[2]) return(true);
- if (dat.dat[2] > that.dat.dat[2]) return(false);
-#if (ovOverlapNWORDS == 5)
- if (dat.dat[3] < that.dat.dat[3]) return(true);
- if (dat.dat[3] > that.dat.dat[3]) return(false);
- if (dat.dat[4] < that.dat.dat[4]) return(true);
- if (dat.dat[4] > that.dat.dat[4]) return(false);
-#endif
+
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++) {
+ if (dat.dat[ii] < that.dat.dat[ii]) return(true);
+ if (dat.dat[ii] > that.dat.dat[ii]) return(false);
+ }
+
return(false);
};
diff --git a/src/stores/ovStore.C b/src/stores/ovStore.C
index f484a67..83e7bde 100644
--- a/src/stores/ovStore.C
+++ b/src/stores/ovStore.C
@@ -529,54 +529,22 @@ ovStore::numOverlapsPerFrag(uint32 &firstFrag, uint32 &lastFrag) {
void
ovStore::addEvalues(vector<char *> &fileList) {
-
- for (uint32 i=0; i<fileList.size(); i++) {
- errno = 0;
- FILE *fp = fopen(fileList[i], "r");
- if (errno)
- fprintf(stderr, "Failed to open evalues file '%s': %s\n", fileList[i], strerror(errno)), exit(1);
-
- uint32 bgnID = 0;
- uint32 endID = 0;
- uint64 len = 0;
-
- AS_UTL_safeRead(fp, &bgnID, "loid", sizeof(uint32), 1);
- AS_UTL_safeRead(fp, &endID, "hiid", sizeof(uint32), 1);
- AS_UTL_safeRead(fp, &len, "len", sizeof(uint64), 1);
-
- uint16 *evalues = new uint16 [len];
-
- AS_UTL_safeRead(fp, evalues, "evalues", sizeof(uint16), len);
-
- fclose(fp);
-
- fprintf(stderr, "- Loading evalues from '%s' -- ID range " F_U32 "-" F_U32 " with " F_U64 " overlaps\n",
- fileList[i], bgnID, endID, len);
-
- addEvalues(bgnID, endID, evalues, len);
-
- delete [] evalues;
- }
-}
-
-
-
-void
-ovStore::addEvalues(uint32 bgnID, uint32 endID, uint16 *evalues, uint64 evaluesLen) {
-
char name[FILENAME_MAX];
snprintf(name, FILENAME_MAX, "%s/evalues", _storePath);
- // If we have an opened memory mapped file, and it isn't open for writing, close it.
+ // If we have an opened memory mapped file, close it.
- if ((_evaluesMap) && (_evaluesMap->type() == memoryMappedFile_readOnly)) {
- fprintf(stderr, "WARNING: closing read-only evalues file.\n");
+ if (_evaluesMap) {
delete _evaluesMap;
_evaluesMap = NULL;
_evalues = NULL;
}
+ // Allocate space for the evalues.
+
+ _evalues = new uint16 [_info.numOverlaps()];
+
// Remove a bogus evalues file if one exists.
if ((AS_UTL_fileExists(name) == true) &&
@@ -586,56 +554,63 @@ ovStore::addEvalues(uint32 bgnID, uint32 endID, uint16 *evalues, uint64 evaluesL
AS_UTL_unlink(name);
}
- // Make a new evalues file if one doesn't exist.
+ // Clear the evalues.
+
+ for (uint64 ii=0; ii<_info.numOverlaps(); ii++)
+ _evalues[ii] = UINT16_MAX;
+
+ // For each file in the fileList, open it, read the header (bgnID, endID and
+ // number of values), load the evalues, then copy this data to the actual
+ // evalues file.
- if (AS_UTL_fileExists(name) == false) {
- fprintf(stderr, "Creating evalues file for " F_U64 " overlaps.\r", _info.numOverlaps());
+ for (uint32 i=0; i<fileList.size(); i++) {
+ uint32 bgnID = 0;
+ uint32 endID = 0;
+ uint64 len = 0;
errno = 0;
- FILE *F = fopen(name, "w");
+ FILE *fp = fopen(fileList[i], "r");
if (errno)
- fprintf(stderr, "Failed to make evalues file '%s': %s\n", name, strerror(errno)), exit(1);
+ fprintf(stderr, "Failed to open evalues file '%s': %s\n", fileList[i], strerror(errno)), exit(1);
- uint16 *Z = new uint16 [1048576];
- uint64 Zn = 0;
+ AS_UTL_safeRead(fp, &bgnID, "loid", sizeof(uint32), 1);
+ AS_UTL_safeRead(fp, &endID, "hiid", sizeof(uint32), 1);
+ AS_UTL_safeRead(fp, &len, "len", sizeof(uint64), 1);
- memset(Z, 0, sizeof(uint16) * 1048576);
+ // Figure out the overlap ID for the first overlap associated with bgnID
- while (Zn < _info.numOverlaps()) {
- uint64 S = (Zn + 1048576 < _info.numOverlaps()) ? 1048576 : _info.numOverlaps() - Zn;
+ setRange(bgnID, endID);
- AS_UTL_safeWrite(F, Z, "zero evalues", sizeof(uint16), S);
+ // Load data directly into the evalue array
- Zn += S;
+ fprintf(stderr, "- Loading evalues from '%s' -- ID range " F_U32 "-" F_U32 " with " F_U64 " overlaps\n",
+ fileList[i], bgnID, endID, len);
- fprintf(stderr, "Creating evalues file for " F_U64 " overlaps....%07.3f%%\r",
- _info.numOverlaps(), 100.0 * Zn / _info.numOverlaps());
- }
+ AS_UTL_safeRead(fp, _evalues + _offt._overlapID, "evalues", sizeof(uint16), len);
- delete [] Z;
+ fclose(fp);
+ }
- fclose(F);
+ // Write the evalues to disk.
- fprintf(stderr, "Creating evalues file for " F_U64 " overlaps....%07.3f%%\n",
- _info.numOverlaps(), 100.0 * Zn / _info.numOverlaps());
- }
+ fprintf(stderr, "Saving evalues file for " F_U64 " overlaps.\n", _info.numOverlaps());
- // Open the evalues file if it isn't already opened
+ errno = 0;
+ FILE *F = fopen(name, "w");
+ if (errno)
+ fprintf(stderr, "Failed to make evalues file '%s': %s\n", name, strerror(errno)), exit(1);
- if (_evalues == NULL) {
- _evaluesMap = new memoryMappedFile(name, memoryMappedFile_readWrite);
- _evalues = (uint16 *)_evaluesMap->get(0);
- }
+ AS_UTL_safeWrite(F, _evalues, "evalues", sizeof(uint16), _info.numOverlaps());
- // Figure out the overlap ID for the first overlap associated with bgnID
+ fclose(F);
- setRange(bgnID, endID);
+ // Clean up, and reopen the file. Usually, we just delete the store after
+ // values are loaded, so this is pointless.
- // Load the evalues from 'evalues'
+ delete [] _evalues;
- for (uint64 ii=0; ii<evaluesLen; ii++)
- _evalues[_offt._overlapID + ii] = evalues[ii];
+ // Open the evalues file if it isn't already opened
- // That's it. Deleting the ovStore object will close the memoryMappedFile. It's left open
- // for more updates.
+ _evaluesMap = new memoryMappedFile(name, memoryMappedFile_readOnly);
+ _evalues = (uint16 *)_evaluesMap->get(0);
}
diff --git a/src/stores/ovStore.H b/src/stores/ovStore.H
index d86a6b1..fdaee47 100644
--- a/src/stores/ovStore.H
+++ b/src/stores/ovStore.H
@@ -322,12 +322,11 @@ public:
// of evalues must agree.
void addEvalues(vector<char *> &fileList);
- void addEvalues(uint32 bgnID, uint32 endID, uint16 *evalues, uint64 evaluesLen);
// Return the statistics associated with this store
ovStoreHistogram *getHistogram(void) {
- return(new ovStoreHistogram(_gkp, _storePath));
+ return(new ovStoreHistogram(_storePath));
};
private:
@@ -362,11 +361,10 @@ public:
ovStoreFilter(gkStore *gkp_, double maxErate);
~ovStoreFilter();
- void filterOverlap(ovOverlap &foverlap,
- ovOverlap &roverlap);
+ void filterOverlap(ovOverlap &foverlap,
+ ovOverlap &roverlap);
- //void reportFate(void);
- void resetCounters(void);
+ void resetCounters(void);
uint64 savedUnitigging(void) { return(saveUTG); };
uint64 savedTrimming(void) { return(saveOBT); };
diff --git a/src/stores/ovStoreBucketizer.C b/src/stores/ovStoreBucketizer.C
index afa4e66..a9227b4 100644
--- a/src/stores/ovStoreBucketizer.C
+++ b/src/stores/ovStoreBucketizer.C
@@ -62,7 +62,7 @@ writeToFile(gkStore *gkp,
if (sliceFile[df] == NULL) {
char name[FILENAME_MAX];
- snprintf(name, FILENAME_MAX, "%s/create%04d/slice%03d%s", ovlName, jobIndex, df, (useGzip) ? ".gz" : "");
+ snprintf(name, FILENAME_MAX, "%s/create%04d/slice%04d%s", ovlName, jobIndex, df, (useGzip) ? ".gz" : "");
sliceFile[df] = new ovFile(gkp, name, ovFileFullWriteNoCounts);
sliceSize[df] = 0;
}
@@ -250,7 +250,7 @@ main(int argc, char **argv) {
//AS_OVS_setBinaryOverlapFileBufferSize(2 * 1024 * 1024);
while (inputFile->readOverlap(&foverlap)) {
- filter->filterOverlap(foverlap, roverlap); // The filter copies f into r
+ filter->filterOverlap(foverlap, roverlap); // The filter copies f into r, and checks IDs
// If all are skipped, don't bother writing the overlap.
diff --git a/src/stores/ovStoreBuild.C b/src/stores/ovStoreBuild.C
index a6f1a08..4d9bf40 100644
--- a/src/stores/ovStoreBuild.C
+++ b/src/stores/ovStoreBuild.C
@@ -166,7 +166,7 @@ computeIIDperBucket(uint32 fileLimit,
allocateArray(oPR, maxIID);
for (uint32 i=0; i<fileList.size(); i++)
- hist->loadData(fileList[i]);
+ hist->loadData(fileList[i], maxIID);
uint64 numOverlaps = hist->getOverlapsPerRead(oPR, maxIID);
@@ -293,13 +293,13 @@ computeIIDperBucket(uint32 fileLimit,
iidToBucket[ii] = bucket;
if (olaps >= olapsPerBucketMax) {
- fprintf(stderr, " bucket %3d has " F_U64 " olaps.\n", bucket, olaps);
+ fprintf(stderr, " bucket %4d has " F_U64 " olaps.\n", bucket, olaps);
olaps = 0;
bucket++;
}
}
- fprintf(stderr, " bucket %3d has " F_U64 " olaps.\n", bucket, olaps);
+ fprintf(stderr, " bucket %4d has " F_U64 " olaps.\n", bucket, olaps);
}
fprintf(stderr, "Will sort %.3f million overlaps per bucket, using %u buckets %.2f GB per bucket.\n",
@@ -329,7 +329,7 @@ writeToDumpFile(gkStore *gkp,
if (dumpFile[df] == NULL) {
char name[FILENAME_MAX];
- snprintf(name, FILENAME_MAX, "%s/tmp.sort.%03d", ovlName, df);
+ snprintf(name, FILENAME_MAX, "%s/tmp.sort.%04d", ovlName, df);
fprintf(stderr, "-- Create bucket '%s'\n", name);
dumpFile[df] = new ovFile(gkp, name, ovFileFullWriteNoCounts);
dumpLength[df] = 0;
@@ -442,7 +442,8 @@ main(int argc, char **argv) {
fprintf(stderr, " -config out.dat don't build a store, just dump a binary partitioning file for ovStoreBucketizer\n");
fprintf(stderr, "\n");
fprintf(stderr, "Sizes and Limits:\n");
- fprintf(stderr, " ovOverlapSortSize " F_S32 " bytes\n", (int32)ovOverlapSortSize);
+ fprintf(stderr, " ovOverlap " F_S32 " words of " F_S32 " bits each.\n", (int32)ovOverlapNWORDS, (int32)ovOverlapWORDSZ);
+ fprintf(stderr, " ovOverlapSortSize " F_S32 " bits\n", (int32)ovOverlapSortSize * 8);
fprintf(stderr, " SC_CHILD_MAX " F_S32 " processes\n", (int32)sysconf(_SC_CHILD_MAX));
fprintf(stderr, " SC_OPEN_MAX " F_S32 " files\n", (int32)sysconf(_SC_OPEN_MAX));
fprintf(stderr, "\n");
@@ -593,7 +594,7 @@ main(int argc, char **argv) {
// directly....BUT....we can't do that because the AS_OVS interface is rearranging the data to
// make sure the store is cross-platform compatible.
- snprintf(name, FILENAME_MAX, "%s/tmp.sort.%03d", ovlName, i);
+ snprintf(name, FILENAME_MAX, "%s/tmp.sort.%04d", ovlName, i);
fprintf(stderr, "- Loading '%s'\n", name);
bof = new ovFile(gkp, name, ovFileFull);
diff --git a/src/stores/ovStoreFile.C b/src/stores/ovStoreFile.C
index 46a3fe0..8b74288 100644
--- a/src/stores/ovStoreFile.C
+++ b/src/stores/ovStoreFile.C
@@ -208,46 +208,16 @@ ovFile::writeOverlap(ovOverlap *overlap) {
_buffer[_bufferLen++] = overlap->b_iid;
-#if (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 3)
- _buffer[_bufferLen++] = overlap->dat.dat[0];
- _buffer[_bufferLen++] = overlap->dat.dat[1];
- _buffer[_bufferLen++] = overlap->dat.dat[2];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 5)
- _buffer[_bufferLen++] = overlap->dat.dat[0];
- _buffer[_bufferLen++] = overlap->dat.dat[1];
- _buffer[_bufferLen++] = overlap->dat.dat[2];
- _buffer[_bufferLen++] = overlap->dat.dat[3];
- _buffer[_bufferLen++] = overlap->dat.dat[4];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 6)
- _buffer[_bufferLen++] = overlap->dat.dat[0];
- _buffer[_bufferLen++] = overlap->dat.dat[1];
- _buffer[_bufferLen++] = overlap->dat.dat[2];
- _buffer[_bufferLen++] = overlap->dat.dat[3];
- _buffer[_bufferLen++] = overlap->dat.dat[4];
- _buffer[_bufferLen++] = overlap->dat.dat[5];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 8)
- _buffer[_bufferLen++] = overlap->dat.dat[0];
- _buffer[_bufferLen++] = overlap->dat.dat[1];
- _buffer[_bufferLen++] = overlap->dat.dat[2];
- _buffer[_bufferLen++] = overlap->dat.dat[3];
- _buffer[_bufferLen++] = overlap->dat.dat[4];
- _buffer[_bufferLen++] = overlap->dat.dat[5];
- _buffer[_bufferLen++] = overlap->dat.dat[6];
- _buffer[_bufferLen++] = overlap->dat.dat[7];
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 2)
- _buffer[_bufferLen++] = (overlap->dat.dat[0] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[0] >> 0) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[1] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[1] >> 0) & 0xffffffff;
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 3)
- _buffer[_bufferLen++] = (overlap->dat.dat[0] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[0] >> 0) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[1] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[1] >> 0) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[2] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlap->dat.dat[2] >> 0) & 0xffffffff;
-#else
-#error unknown ovOverlapNWORDS
+#if (ovOverlapWORDSZ == 32)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++)
+ _buffer[_bufferLen++] = overlap->dat.dat[ii];
+#endif
+
+#if (ovOverlapWORDSZ == 64)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++) {
+ _buffer[_bufferLen++] = (overlap->dat.dat[ii] >> 32) & 0xffffffff;
+ _buffer[_bufferLen++] = (overlap->dat.dat[ii]) & 0xffffffff;
+ }
#endif
assert(_bufferLen <= _bufferMax);
@@ -273,46 +243,16 @@ ovFile::writeOverlaps(ovOverlap *overlaps, uint64 overlapsLen) {
_buffer[_bufferLen++] = overlaps[nWritten].b_iid;
-#if (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 3)
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[0];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[1];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[2];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 5)
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[0];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[1];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[2];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[3];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[4];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 6)
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[0];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[1];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[2];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[3];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[4];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[5];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 8)
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[0];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[1];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[2];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[3];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[4];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[5];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[6];
- _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[7];
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 2)
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[0] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[0] >> 0) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[1] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[1] >> 0) & 0xffffffff;
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 3)
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[0] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[0] >> 0) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[1] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[1] >> 0) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[2] >> 32) & 0xffffffff;
- _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[2] >> 0) & 0xffffffff;
-#else
-#error unknown ovOverlapNWORDS
+#if (ovOverlapWORDSZ == 32)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++)
+ _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[ii];
+#endif
+
+#if (ovOverlapWORDSZ == 64)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++) {
+ _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[ii] >> 32) & 0xffffffff;
+ _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[ii]) & 0xffffffff;
+ }
#endif
nWritten++;
@@ -386,46 +326,17 @@ ovFile::readOverlap(ovOverlap *overlap) {
overlap->b_iid = _buffer[_bufferPos++];
-#if (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 3)
- overlap->dat.dat[0] = _buffer[_bufferPos++];
- overlap->dat.dat[1] = _buffer[_bufferPos++];
- overlap->dat.dat[2] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 5)
- overlap->dat.dat[0] = _buffer[_bufferPos++];
- overlap->dat.dat[1] = _buffer[_bufferPos++];
- overlap->dat.dat[2] = _buffer[_bufferPos++];
- overlap->dat.dat[3] = _buffer[_bufferPos++];
- overlap->dat.dat[4] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 6)
- overlap->dat.dat[0] = _buffer[_bufferPos++];
- overlap->dat.dat[1] = _buffer[_bufferPos++];
- overlap->dat.dat[2] = _buffer[_bufferPos++];
- overlap->dat.dat[3] = _buffer[_bufferPos++];
- overlap->dat.dat[4] = _buffer[_bufferPos++];
- overlap->dat.dat[5] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 8)
- overlap->dat.dat[0] = _buffer[_bufferPos++];
- overlap->dat.dat[1] = _buffer[_bufferPos++];
- overlap->dat.dat[2] = _buffer[_bufferPos++];
- overlap->dat.dat[3] = _buffer[_bufferPos++];
- overlap->dat.dat[4] = _buffer[_bufferPos++];
- overlap->dat.dat[5] = _buffer[_bufferPos++];
- overlap->dat.dat[6] = _buffer[_bufferPos++];
- overlap->dat.dat[7] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 2)
- overlap->dat.dat[0] = _buffer[_bufferPos++]; overlap->dat.dat[0] <<= 32;
- overlap->dat.dat[0] |= _buffer[_bufferPos++];
- overlap->dat.dat[1] = _buffer[_bufferPos++]; overlap->dat.dat[1] <<= 32;
- overlap->dat.dat[1] |= _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 3)
- overlap->dat.dat[0] = _buffer[_bufferPos++]; overlap->dat.dat[0] <<= 32;
- overlap->dat.dat[0] |= _buffer[_bufferPos++];
- overlap->dat.dat[1] = _buffer[_bufferPos++]; overlap->dat.dat[1] <<= 32;
- overlap->dat.dat[1] |= _buffer[_bufferPos++];
- overlap->dat.dat[2] = _buffer[_bufferPos++]; overlap->dat.dat[2] <<= 32;
- overlap->dat.dat[2] |= _buffer[_bufferPos++];
-#else
-#error unknown ovOverlapNWORDS
+#if (ovOverlapWORDSZ == 32)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++)
+ overlap->dat.dat[ii] = _buffer[_bufferPos++];
+#endif
+
+#if (ovOverlapWORDSZ == 64)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++) {
+ overlap->dat.dat[ii] = _buffer[_bufferPos++];
+ overlap->dat.dat[ii] <<= 32;
+ overlap->dat.dat[ii] |= _buffer[_bufferPos++];
+ }
#endif
assert(_bufferPos <= _bufferLen);
@@ -454,46 +365,17 @@ ovFile::readOverlaps(ovOverlap *overlaps, uint64 overlapsLen) {
overlaps[nLoaded].b_iid = _buffer[_bufferPos++];
-#if (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 3)
- overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 5)
- overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[3] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[4] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 6)
- overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[3] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[4] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[5] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 32) && (ovOverlapNWORDS == 8)
- overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[3] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[4] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[5] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[6] = _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[7] = _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 2)
- overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[0] <<= 32;
- overlaps[nLoaded].dat.dat[0] |= _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[1] <<= 32;
- overlaps[nLoaded].dat.dat[1] |= _buffer[_bufferPos++];
-#elif (ovOverlapWORDSZ == 64) && (ovOverlapNWORDS == 3)
- overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[0] <<= 32;
- overlaps[nLoaded].dat.dat[0] |= _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[1] <<= 32;
- overlaps[nLoaded].dat.dat[1] |= _buffer[_bufferPos++];
- overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[2] <<= 32;
- overlaps[nLoaded].dat.dat[2] |= _buffer[_bufferPos++];
-#else
-#error unknown ovOverlapNWORDS
+#if (ovOverlapWORDSZ == 32)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++)
+ overlaps[nLoaded].dat.dat[ii] = _buffer[_bufferPos++];
+#endif
+
+#if (ovOverlapWORDSZ == 64)
+ for (uint32 ii=0; ii<ovOverlapNWORDS; ii++) {
+ overlaps[nLoaded].dat.dat[ii] = _buffer[_bufferPos++];
+ overlaps[nLoaded].dat.dat[ii] <<= 32;
+ overlaps[nLoaded].dat.dat[ii] |= _buffer[_bufferPos++];
+ }
#endif
nLoaded++;
diff --git a/src/stores/ovStoreFilter.C b/src/stores/ovStoreFilter.C
index 89bc2fe..0f8842f 100644
--- a/src/stores/ovStoreFilter.C
+++ b/src/stores/ovStoreFilter.C
@@ -38,37 +38,24 @@
ovStoreFilter::ovStoreFilter(gkStore *gkp_, double maxErate) {
gkp = gkp_;
+ maxID = gkp->gkStore_getNumReads() + 1;
+ maxEvalue = AS_OVS_encodeEvalue(maxErate);
resetCounters();
- maxID = gkp->gkStore_getNumReads() + 1;
- maxEvalue = AS_OVS_encodeEvalue(maxErate);
-
skipReadOBT = new char [maxID];
skipReadDUP = new char [maxID];
- memset(skipReadOBT, 0, sizeof(char) * maxID);
- memset(skipReadDUP, 0, sizeof(char) * maxID);
-
-
uint32 numSkipOBT = 0;
uint32 numSkipDUP = 0;
-#if 0
- fprintf(stderr, "Marking fragments to skip overlap based trimming.\n");
-
- fprintf(stderr, "LIB 1 - dup=%d trim=%d spur=%d chimera=%d subreads=%d\n",
- gkp->gkStore_getLibrary(1)->gkLibrary_removeDuplicateReads(),
- gkp->gkStore_getLibrary(1)->gkLibrary_finalTrim(),
- gkp->gkStore_getLibrary(1)->gkLibrary_removeSpurReads(),
- gkp->gkStore_getLibrary(1)->gkLibrary_removeChimericReads(),
- gkp->gkStore_getLibrary(1)->gkLibrary_checkForSubReads());
-#endif
-
for (uint64 iid=0; iid<maxID; iid++) {
uint32 Lid = gkp->gkStore_getRead(iid)->gkRead_libraryID();
gkLibrary *L = gkp->gkStore_getLibrary(Lid);
+ skipReadOBT[iid] = false;
+ skipReadDUP[iid] = false;
+
if ((L->gkLibrary_removeDuplicateReads() == false) &&
(L->gkLibrary_finalTrim() == GK_FINALTRIM_NONE) &&
(L->gkLibrary_removeSpurReads() == false) &&
diff --git a/src/stores/ovStoreHistogram.C b/src/stores/ovStoreHistogram.C
index 1a1f2f0..702b9a2 100644
--- a/src/stores/ovStoreHistogram.C
+++ b/src/stores/ovStoreHistogram.C
@@ -46,8 +46,7 @@ ovStoreHistogram::ovStoreHistogram() {
}
-
-ovStoreHistogram::ovStoreHistogram(gkStore *gkp, char *path) {
+ovStoreHistogram::ovStoreHistogram(char *path) {
_gkp = NULL;
@@ -169,7 +168,25 @@ ovStoreHistogram::addOverlap(ovOverlap *overlap) {
memset(_opel[ev], 0, sizeof(uint32) * _opelLen);
}
- _opel[ev][len]++;
+ int32 alen = _gkp->gkStore_getRead(overlap->a_iid)->gkRead_sequenceLength();
+ int32 blen = _gkp->gkStore_getRead(overlap->b_iid)->gkRead_sequenceLength();
+
+ if (len < _opelLen) {
+ //fprintf(stderr, "overlap %8u (len %6d) %8u (len %6d) hangs %6" F_U64P " %6d %6" F_U64P " - %6" F_U64P " %6d %6" F_U64P " flip " F_U64 "\n",
+ // overlap->a_iid, alen,
+ // overlap->b_iid, blen,
+ // overlap->dat.ovl.ahg5, (int32)alen - (int32)overlap->dat.ovl.ahg5 - (int32)overlap->dat.ovl.ahg3, overlap->dat.ovl.ahg3,
+ // overlap->dat.ovl.bhg5, (int32)blen - (int32)overlap->dat.ovl.bhg5 - (int32)overlap->dat.ovl.bhg3, overlap->dat.ovl.bhg3,
+ // overlap->dat.ovl.flipped);
+ _opel[ev][len]++;
+ } else {
+ fprintf(stderr, "overlap %8u (len %6d) %8u (len %6d) hangs %6" F_U64P " %6d %6" F_U64P " - %6" F_U64P " %6d %6" F_U64P " flip " F_U64 " -- BOGUS\n",
+ overlap->a_iid, alen,
+ overlap->b_iid, blen,
+ overlap->dat.ovl.ahg5, (int32)alen - (int32)overlap->dat.ovl.ahg5 - (int32)overlap->dat.ovl.ahg3, overlap->dat.ovl.ahg3,
+ overlap->dat.ovl.bhg5, (int32)blen - (int32)overlap->dat.ovl.bhg5 - (int32)overlap->dat.ovl.bhg3, overlap->dat.ovl.bhg3,
+ overlap->dat.ovl.flipped);
+ }
}
}
@@ -257,7 +274,7 @@ ovStoreHistogram::saveData(char *prefix) {
void
-ovStoreHistogram::loadData(char *prefix) {
+ovStoreHistogram::loadData(char *prefix, uint32 maxIID) {
char name[FILENAME_MAX];
// Add in any overlaps-per-read data.
@@ -277,6 +294,9 @@ ovStoreHistogram::loadData(char *prefix) {
if (_oprMax < inLen) // Resize to fit those values
resizeArray(_opr, _oprLen, _oprMax, inLen + inLen/2, resizeArray_copyData | resizeArray_clearNew);
+ if (maxIID < inLen)
+ fprintf(stderr, "WARNING: histogram file '%s' has data for %u reads, but only %u reads known.\n", name, inLen, maxIID);
+
if (_oprLen < inLen) // Remember the new length
_oprLen = inLen;
diff --git a/src/stores/ovStoreHistogram.H b/src/stores/ovStoreHistogram.H
index 917d07d..c8fa89a 100644
--- a/src/stores/ovStoreHistogram.H
+++ b/src/stores/ovStoreHistogram.H
@@ -38,7 +38,7 @@
class ovStoreHistogram {
public:
ovStoreHistogram(); // Used when loading data, user must loadData() later
- ovStoreHistogram(gkStore *gkp, char *path); // Used when loading data, calls loadData() for you
+ ovStoreHistogram(char *path); // Used when loading data, calls loadData() for you
ovStoreHistogram(gkStore *gkp, ovFileType type); // Used when writing ovFile
~ovStoreHistogram();
@@ -74,7 +74,7 @@ public:
// In an ovStore, load the histogram saved in a file, and add it to our current data.
void saveData(char *prefix);
- void loadData(char *prefix);
+ void loadData(char *prefix, uint32 maxIID=UINT32_MAX);
// Remove data associated with some prefix.
diff --git a/src/stores/ovStoreWriter.C b/src/stores/ovStoreWriter.C
index cc6a5ca..dcddf3c 100644
--- a/src/stores/ovStoreWriter.C
+++ b/src/stores/ovStoreWriter.C
@@ -607,8 +607,8 @@ ovStoreWriter::loadBucketSizes(uint64 *bucketSizes) {
for (uint32 i=0; i<=_jobIdxMax; i++) {
bucketSizes[i] = 0;
- snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%03d", _storePath, i, _fileID);
- snprintf(namz, FILENAME_MAX, "%s/bucket%04d/slice%03d.gz", _storePath, i, _fileID);
+ snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%04d", _storePath, i, _fileID);
+ snprintf(namz, FILENAME_MAX, "%s/bucket%04d/slice%04d.gz", _storePath, i, _fileID);
// If no file, there are no overlaps. Skip loading the bucketSizes file.
// With snappy compression, we expect the file to be not gzip compressed, but will happily
@@ -654,10 +654,10 @@ ovStoreWriter::loadOverlapsFromSlice(uint32 slice, uint64 expectedLen, ovOverlap
if (expectedLen == 0)
return;
- snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%03d", _storePath, slice, _fileID);
+ snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%04d", _storePath, slice, _fileID);
if (AS_UTL_fileExists(name, FALSE, FALSE) == false) {
- snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%03d.gz", _storePath, slice, _fileID);
+ snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%04d.gz", _storePath, slice, _fileID);
if (AS_UTL_fileExists(name, FALSE, FALSE) == false)
fprintf(stderr, "ERROR: " F_U64 " overlaps claim to exist in bucket '%s', but file not found.\n",
@@ -688,8 +688,8 @@ ovStoreWriter::removeOverlapSlice(void) {
char name[FILENAME_MAX];
for (uint32 i=0; i<=_jobIdxMax; i++) {
- snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%03d.gz", _storePath, i, _fileID); AS_UTL_unlink(name);
- snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%03d", _storePath, i, _fileID); AS_UTL_unlink(name);
+ snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%04d.gz", _storePath, i, _fileID); AS_UTL_unlink(name);
+ snprintf(name, FILENAME_MAX, "%s/bucket%04d/slice%04d", _storePath, i, _fileID); AS_UTL_unlink(name);
}
}
diff --git a/src/stores/tgStore.C b/src/stores/tgStore.C
index 55b326f..7538a01 100644
--- a/src/stores/tgStore.C
+++ b/src/stores/tgStore.C
@@ -129,12 +129,8 @@ tgStore::tgStore(const char *path_,
break;
case tgStoreReadOnly:
- if (_tigLen == 0) {
- fprintf(stderr, "tgStore::tgStore()-- ERROR, didn't find any tigs in the store.\n");
- fprintf(stderr, "tgStore::tgStore()-- asked for store '%s', correct?\n", _path);
- fprintf(stderr, "tgStore::tgStore()-- asked for version '%d', correct?\n", _originalVersion);
- exit(1);
- }
+ if (_tigLen == 0)
+ fprintf(stderr, "tgStore::tgStore()-- WARNING: no tigs in store '%s' version '%d'.\n", _path, _originalVersion);
break;
case tgStoreWrite:
diff --git a/src/stores/tgStoreCoverageStat.C b/src/stores/tgStoreCoverageStat.C
index ededf2f..36efb7a 100644
--- a/src/stores/tgStoreCoverageStat.C
+++ b/src/stores/tgStoreCoverageStat.C
@@ -551,6 +551,8 @@ main(int argc, char **argv) {
fprintf(stderr, "Computing coverage stat for tigs %u-%u.\n", bgnID, endID-1);
+ fprintf(outLOG, "# tigID rho covStat arrDist\n");
+
for (uint32 i=bgnID; i<endID; i++) {
tgTig *tig = tigStore->loadTig(i);
@@ -573,8 +575,6 @@ main(int argc, char **argv) {
(globalRate > 0.0))
covStat = (rho * globalRate) - (ln2 * (numRandom - 1));
- if (i == bgnID)
- fprintf(outLOG, " tigID rho covStat arrDist\n");
fprintf(outLOG, "%10u %10.2f %10.2f %10.2f\n", tig->tigID(), rho, covStat, arrDist);
#undef ADJUST_FOR_PARTIAL_EXCESS
diff --git a/src/stores/tgStoreDump.C b/src/stores/tgStoreDump.C
index ff45a7c..b0c0910 100644
--- a/src/stores/tgStoreDump.C
+++ b/src/stores/tgStoreDump.C
@@ -304,7 +304,7 @@ dumpTigs(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool us
void
-dumpConsensus(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, char cnsFormat) {
+dumpConsensus(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, bool useReverse, char cnsFormat) {
for (uint32 ti=0; ti<tigStore->numTigs(); ti++) {
if (tigStore->isDeleted(ti))
@@ -323,6 +323,9 @@ dumpConsensus(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bo
continue;
}
+ if (useReverse)
+ tig->reverseComplement();
+
switch (cnsFormat) {
case 'A':
tig->dumpFASTA(stdout, useGapped);
@@ -1011,6 +1014,7 @@ main (int argc, char **argv) {
uint32 dumpType = DUMP_UNSET;
bool useGapped = false;
+ bool useReverse = false;
char cnsFormat = 'A'; // Or 'Q' for FASTQ
@@ -1120,6 +1124,9 @@ main (int argc, char **argv) {
else if (strcmp(argv[arg], "-gapped") == 0)
useGapped = true;
+ else if (strcmp(argv[arg], "-reverse") == 0)
+ useReverse = true;
+
else if (strcmp(argv[arg], "-fasta") == 0)
cnsFormat = 'A';
else if (strcmp(argv[arg], "-fastq") == 0)
@@ -1188,6 +1195,7 @@ main (int argc, char **argv) {
fprintf(stderr, "\n");
fprintf(stderr, " -consensus [opts] the consensus sequence, with options:\n");
fprintf(stderr, " -gapped report the gapped (multialignment) consensus sequence\n");
+ fprintf(stderr, " -reverse reverse complement the sequence\n");
fprintf(stderr, " -fasta report sequences in FASTA format (the default)\n");
fprintf(stderr, " -fastq report sequences in FASTQ format\n");
fprintf(stderr, "\n");
@@ -1249,7 +1257,7 @@ main (int argc, char **argv) {
if (filter.tigIDend == UINT32_MAX)
filter.tigIDend = nTigs-1;
- if (nTigs <= filter.tigIDend) {
+ if ((nTigs > 0) && (nTigs <= filter.tigIDend)) {
fprintf(stderr, "WARNING: adjusting tig ID range from " F_U32 "-" F_U32 " to " F_U32 "-" F_U32 " as there are only " F_U32 " tigs in the store.\n",
filter.tigIDbgn, filter.tigIDend, filter.tigIDbgn, nTigs-1, nTigs);
filter.tigIDend = nTigs - 1;
@@ -1263,7 +1271,7 @@ main (int argc, char **argv) {
filter.tigIDbgn = x;
}
- if (nTigs <= filter.tigIDbgn)
+ if ((nTigs > 0) && (nTigs <= filter.tigIDbgn))
fprintf(stderr, "ERROR: only " F_U32 " tigs in the store (IDs 0-" F_U32 " inclusive); can't dump requested range -t " F_U32 "-" F_U32 "\n",
nTigs,
nTigs-1,
@@ -1279,7 +1287,7 @@ main (int argc, char **argv) {
dumpTigs(gkpStore, tigStore, filter, useGapped);
break;
case DUMP_CONSENSUS:
- dumpConsensus(gkpStore, tigStore, filter, useGapped, cnsFormat);
+ dumpConsensus(gkpStore, tigStore, filter, useGapped, useReverse, cnsFormat);
break;
case DUMP_LAYOUT:
dumpLayout(gkpStore, tigStore, filter, useGapped, outPrefix);
diff --git a/src/stores/tgStoreLoad.C b/src/stores/tgStoreLoad.C
index 027c7ae..efc55b9 100644
--- a/src/stores/tgStoreLoad.C
+++ b/src/stores/tgStoreLoad.C
@@ -83,16 +83,17 @@ operationBuild(char *buildName,
int
main (int argc, char **argv) {
- char *gkpName = NULL;
- char *tigName = NULL;
- int32 tigVers = -1;
+ char *gkpName = NULL;
+ char *tigName = NULL;
+ int32 tigVers = -1;
vector<char *> tigInputs;
- tgStoreType tigType = tgStoreModify;
+ char *tigInputsFile = NULL;
+ tgStoreType tigType = tgStoreModify;
argc = AS_configure(argc, argv);
- int arg=1;
- int err=0;
+ vector<char *> err;
+ int arg = 1;
while (arg < argc) {
if (strcmp(argv[arg], "-G") == 0) {
gkpName = argv[++arg];
@@ -102,7 +103,8 @@ main (int argc, char **argv) {
tigVers = atoi(argv[++arg]);
} else if (strcmp(argv[arg], "-L") == 0) {
- AS_UTL_loadFileList(argv[++arg], tigInputs);
+ tigInputsFile = argv[++arg];
+ AS_UTL_loadFileList(tigInputsFile, tigInputs);
} else if (strcmp(argv[arg], "-n") == 0) {
tigType = tgStoreReadOnly;
@@ -111,19 +113,29 @@ main (int argc, char **argv) {
tigInputs.push_back(argv[arg]);
} else {
- fprintf(stderr, "%s: unknown option '%s'\n", argv[0], argv[arg]);
- err++;
+ char *s = new char [1024];
+ snprintf(s, 1024, "ERROR: Unknown option '%s'.\n", argv[arg]);
+ err.push_back(s);
}
arg++;
}
- if ((err) || (gkpName == NULL) || (tigName == NULL) || (tigInputs.size() == 0)) {
+
+ if (gkpName == NULL)
+ err.push_back("ERROR: no gatekeeper store (-G) supplied.\n");
+ if (tigName == NULL)
+ err.push_back("ERROR: no tig store (-T) supplied.\n");
+ if ((tigInputs.size() == 0) && (tigInputsFile == NULL))
+ err.push_back("ERROR: no input tigs supplied on command line and no -L file supplied.\n");
+
+ if (err.size() > 0) {
fprintf(stderr, "usage: %s -G <gkpStore> -T <tigStore> <v> [input.cns]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, " -G <gkpStore> Path to the gatekeeper store\n");
fprintf(stderr, " -T <tigStore> <v> Path to the tigStore and version to add tigs to\n");
fprintf(stderr, "\n");
fprintf(stderr, " -L <file-of-files> Load the tig(s) from files listed in 'file-of-files'\n");
+ fprintf(stderr, " (WARNING: program will succeed if this file is empty)\n");
fprintf(stderr, "\n");
fprintf(stderr, " -n Don't replace, just report what would have happened\n");
fprintf(stderr, "\n");
@@ -140,12 +152,9 @@ main (int argc, char **argv) {
fprintf(stderr, " To delete a tig, remove all children, and set the number of them to zero.\n");
fprintf(stderr, "\n");
- if (gkpName == NULL)
- fprintf(stderr, "ERROR: no gatekeeper store (-G) supplied.\n");
- if (tigName == NULL)
- fprintf(stderr, "ERROR: no tig store (-T) supplied.\n");
- if (tigInputs.size() == 0)
- fprintf(stderr, "ERROR: no input tigs (-R) supplied.\n");
+ for (uint32 ii=0; ii<err.size(); ii++)
+ if (err[ii])
+ fputs(err[ii], stderr);
exit(1);
}
diff --git a/src/stores/tgTig.C b/src/stores/tgTig.C
index 3f7643e..eae55c0 100644
--- a/src/stores/tgTig.C
+++ b/src/stores/tgTig.C
@@ -36,6 +36,8 @@
#include "AS_UTL_fileIO.H"
#include "AS_UTL_fasta.C"
+#include "AS_UTL_reverseComplement.H"
+
#include "splitToWords.H"
#include "intervalList.H"
@@ -666,6 +668,36 @@ tgTig::loadLayout(FILE *F) {
void
+tgTig::reverseComplement(void) {
+
+ // Primary data is in _gapped and _children.
+
+ ::reverseComplement(_gappedBases, _gappedQuals, _gappedLen);
+
+ // Remove _ungapped and _gappedToUngapped, let it be rebuilt if needed.
+
+ delete [] _ungappedBases; _ungappedBases = NULL;
+ delete [] _ungappedQuals; _ungappedQuals = NULL;
+
+ _ungappedLen = 0;
+ _ungappedMax = 0;
+
+ delete [] _gappedToUngapped; _gappedToUngapped = NULL;
+
+ // _anchor, and the hangs, are now invalid.
+
+ for (uint32 ii=0; ii<_childrenLen; ii++) {
+ int32 bgn = _gappedLen - _children[ii].bgn();
+ int32 end = _gappedLen - _children[ii].end();
+
+ _children[ii].set(_children[ii].ident(), 0, 0, 0, bgn, end);
+ }
+
+ // _childDeltas are also invalid.
+}
+
+
+void
tgTig::dumpFASTA(FILE *F, bool useGapped) {
AS_UTL_writeFastA(F,
bases(useGapped), length(useGapped), 100,
diff --git a/src/stores/tgTig.H b/src/stores/tgTig.H
index 9c2fd9e..5085ba6 100644
--- a/src/stores/tgTig.H
+++ b/src/stores/tgTig.H
@@ -309,6 +309,8 @@ public:
void dumpLayout(FILE *F);
bool loadLayout(FILE *F);
+ void reverseComplement(void); // Does NOT update childDeltas
+
void dumpFASTA(FILE *F, bool useGapped);
void dumpFASTQ(FILE *F, bool useGapped);
diff --git a/src/stores/tgTigMultiAlignDisplay.C b/src/stores/tgTigMultiAlignDisplay.C
index 708c0d3..d82b5ec 100644
--- a/src/stores/tgTigMultiAlignDisplay.C
+++ b/src/stores/tgTigMultiAlignDisplay.C
@@ -183,7 +183,7 @@ tgTig::display(FILE *F,
node->quals[ii] += '!';
if (node->read->isReverse())
- reverseComplement(node->bases, node->quals, node->readLen);
+ ::reverseComplement(node->bases, node->quals, node->readLen);
//fprintf(stderr, "NODE READ %u at %d %d tigLength %d with %u deltas at offset %u\n",
// i, node->read->bgn(), node->read->end(), gappedLength(), node->read->deltaLength(), node->read->deltaOffset());
diff --git a/src/utgcns/libNDFalcon/dw.C b/src/utgcns/libNDFalcon/dw.C
index e12e89a..ef9e7c2 100644
--- a/src/utgcns/libNDFalcon/dw.C
+++ b/src/utgcns/libNDFalcon/dw.C
@@ -177,6 +177,16 @@ bool align(const char * query_seq, seq_coor_t q_len,
aln_path = (path_point *)calloc( q_len + t_len + 1, sizeof(path_point) );
+ if (d_path == NULL || aln_path == NULL) {
+ fprintf(stderr, "generatePBDAG()-- Failed memory allocation max_d %d band_size %d.\n",
+ max_d, band_size);
+ free(V);
+ free(U);
+ free(d_path);
+ free(aln_path);
+ return aligned;
+ }
+
if (get_aln_str) {
align_rtn._tgt_aln_str = (char *)calloc( q_len + t_len + 1, sizeof(char));
align_rtn._qry_aln_str = (char *)calloc( q_len + t_len + 1, sizeof(char));
diff --git a/src/utgcns/libcns/unitigConsensus.C b/src/utgcns/libcns/unitigConsensus.C
index 3762ed2..d8c58e8 100644
--- a/src/utgcns/libcns/unitigConsensus.C
+++ b/src/utgcns/libcns/unitigConsensus.C
@@ -75,6 +75,7 @@
// for pbdagcon
#include "Alignment.H"
#include "AlnGraphBoost.H"
+#include "edlib.H"
#include "dw.H"
#include "NDalign.H"
@@ -258,7 +259,8 @@ unitigConsensus::generate(tgTig *tig_,
bool
-unitigConsensus::generatePBDAG(tgTig *tig_,
+unitigConsensus::generatePBDAG(char aligner,
+ tgTig *tig_,
map<uint32, gkRead *> *inPackageRead_,
map<uint32, gkReadData *> *inPackageReadData_) {
tig = tig_;
@@ -361,42 +363,61 @@ unitigConsensus::generatePBDAG(tgTig *tig_,
#endif
dagcon::Alignment aln;
-
- aln.start = utgpos[i].min();
- aln.end = utgpos[i].max();
+ NDalignment::NDalignResult ndaln;
+ EdlibAlignResult align;
+ int32 padding = (aligner == 'E' ? (int32)round((double)(utgpos[i].max() - utgpos[i].min()) * errorRate) + 1 : 0);
+ aln.start = max((int32)0, (int32)utgpos[i].min() - padding);
+ aln.end = min((int32)utg.seq.size(), (int32)utgpos[i].max() + padding);
aln.frgid = utgpos[i].ident();
aln.qstr = string(fragment);
- aln.tstr = utg.seq.substr(aln.start, aln.end-aln.start);
- NDalignment::NDalignResult ndaln;
+ aln.tstr = utg.seq.substr(aln.start, aln.end-aln.start);
uint32 aLen = aln.qstr.size();
uint32 bLen = aln.tstr.size();
uint32 bandTolerance = 150;
- bool aligned = NDalignment::align(aln.qstr.c_str(), aln.qstr.size(),
+ bool aligned = false;
+ if (aligner == 'E') {
+ align = edlibAlign(aln.qstr.c_str(), aln.qstr.size()-1, aln.tstr.c_str(), aln.tstr.size()-1, edlibNewAlignConfig(bandTolerance, EDLIB_MODE_HW, EDLIB_TASK_PATH));
+ aligned = (align.numLocations >= 1);
+ } else {
+ aligned = NDalignment::align(aln.qstr.c_str(), aln.qstr.size(),
aln.tstr.c_str(), aln.tstr.size(),
bandTolerance,
true,
ndaln);
+ }
while ((aligned == false) && (bandTolerance < errorRate * (aLen + bLen))) {
- bandTolerance *= 4;
- fprintf(stderr, "generatePBDAG()-- retry with bandTolerance = %d\n",
+ bandTolerance *= 2;
+ if (aligner == 'E')
+ edlibFreeAlignResult(align);
+
+ fprintf(stderr, "generatePBDAG()-- retry with bandTolerance = %d\n",
bandTolerance);
- aligned = NDalignment::align(aln.qstr.c_str(), aln.qstr.size(),
- aln.tstr.c_str(), aln.tstr.size(),
- bandTolerance,
- true,
- ndaln);
+ if (aligner == 'E') {
+ align = edlibAlign(aln.qstr.c_str(), aln.qstr.size()-1, aln.tstr.c_str(), aln.tstr.size()-1, edlibNewAlignConfig(bandTolerance, EDLIB_MODE_HW, EDLIB_TASK_PATH));
+ aligned = (align.numLocations >= 1);
+ } else {
+ aligned = NDalignment::align(aln.qstr.c_str(), aln.qstr.size(),
+ aln.tstr.c_str(), aln.tstr.size(),
+ bandTolerance,
+ true,
+ ndaln);
+ }
}
- double errorRateAln = (ndaln._size > 0) ? ((double)ndaln._dist / ndaln._size) : 1.0;
+ double errorRateAln = 0;
+ if (aligner == 'E')
+ errorRateAln = (align.alignmentLength > 0) ? ((double)align.editDistance / align.alignmentLength) : 1.0;
+ else
+ errorRateAln = (ndaln._size > 0) ? ((double)ndaln._dist / ndaln._size) : 1.0;
if ((aligned == true) && (errorRateAln > errorRate)) {
fprintf(stderr, "generatePBDAG()-- error rate too high distance=%5d size=%5d, %f > %f\n",
- ndaln._dist, ndaln._size, errorRateAln, errorRate);
+ align.editDistance, align.alignmentLength, errorRateAln, errorRate);
aligned = false;
}
@@ -410,25 +431,39 @@ unitigConsensus::generatePBDAG(tgTig *tig_,
i, utgpos[i].ident(), utgpos[i].min(), utgpos[i].max());
cnspos[i].setMinMax(0, 0);
-
+ if (aligner == 'E')
+ edlibFreeAlignResult(align);
continue;
}
-
- fprintf(stderr, "generatePBDAG()-- aligned distance=%5d size=%5d, %f < %f\n",
- ndaln._dist, ndaln._size,
- (double) ndaln._dist / ndaln._size,
+ if (aligner == 'E') {
+ fprintf(stderr, "generatePBDAG()-- aligned distance=%5d size=%5d, %f < %f\n",
+ align.editDistance, align.alignmentLength,
+ (double) align.editDistance / align.alignmentLength,
errorRate);
- aln.start += ndaln._tgt_bgn;
- aln.end = aln.start + ndaln._tgt_end;
- aln.start++;
- aln.qstr = std::string(ndaln._qry_aln_str);
- aln.tstr = std::string(ndaln._tgt_aln_str);
+ char *tgt_aln_str = new char[align.alignmentLength+1];
+ char *qry_aln_str = new char[align.alignmentLength+1];
+ edlibAlignmentToStrings(align.alignment, align.alignmentLength, align.startLocations[0], align.endLocations[0]+1, 0, aln.qstr.length(), aln.tstr.c_str(), aln.qstr.c_str(), tgt_aln_str, qry_aln_str);
- assert(aln.qstr.length() == aln.tstr.length());
+ aln.start += align.startLocations[0];
+ aln.end = aln.start + (align.endLocations[0] - align.startLocations[0]) + 1;
+ aln.qstr = std::string(qry_aln_str);
+ aln.tstr = std::string(tgt_aln_str);
+ edlibFreeAlignResult(align);
+ delete[] tgt_aln_str;
+ delete[] qry_aln_str;
+ } else {
+ aln.start += ndaln._tgt_bgn;
+ aln.end = aln.start + (ndaln._tgt_end - ndaln._tgt_bgn) - 1;
+ aln.qstr = std::string(ndaln._qry_aln_str);
+ aln.tstr = std::string(ndaln._tgt_aln_str);
+ }
+ aln.start++;
cnspos[i].setMinMax(aln.start, aln.end);
+ assert(aln.qstr.length() == aln.tstr.length());
+ assert(aln.end < utg.seq.size());
dagcon::Alignment norm = normalizeGaps(aln);
@@ -437,11 +472,49 @@ unitigConsensus::generatePBDAG(tgTig *tig_,
}
// Merge the nodes and call consensus
-
ag.mergeNodes();
std::string cns = ag.consensus(1);
+#ifdef REALIGN
+ // update positions, this requires remapping but this time to the final consensus, turned off for now
+ uint32 minPos = cns.size();
+ uint32 maxPos = 0;
+
+#pragma omp parallel for schedule(dynamic)
+ for (uint32 i=0; i<numfrags; i++) {
+ abSequence *seq = abacus->getSequence(i);
+
+ uint32 bandTolerance = (int32)round((double)(seq->length() * errorRate)) * 2;
+ uint32 maxExtend = (int32)round((double)seq->length() * 0.01) + 1;
+ int32 padding = bandTolerance;
+ uint32 start = max((int32)0, (int32)utgpos[i].min() - padding);
+ uint32 end = min((int32)cns.size(), (int32)utgpos[i].max() + padding);
+
+ EdlibAlignResult align = edlibAlign(seq->getBases(), seq->length()-1, cns.c_str()+start, end-start+1, edlibNewAlignConfig(bandTolerance, EDLIB_MODE_HW, EDLIB_TASK_LOC));
+ if (align.numLocations > 0) {
+ cnspos[i].setMinMax(align.startLocations[0]+start, align.endLocations[0]+start+1);
+ // when we are very close to end extend
+ if (cnspos[i].max() < cns.size() && cns.size() - cnspos[i].max() <= maxExtend && (align.editDistance + cns.size() - cnspos[i].max()) < bandTolerance) {
+ cnspos[i].setMinMax(cnspos[i].min(), cns.size());
+ }
+#pragma omp critical (trackMin)
+ if (cnspos[i].min() < minPos) minPos = cnspos[i].min();
+#pragma omp critical (trackMax)
+ if (cnspos[i].max() > maxPos) maxPos = cnspos[i].max();
+ } else {
+}
+ edlibFreeAlignResult(align);
+ }
+ memcpy(tig->getChild(0), cnspos, sizeof(tgPosition) * numfrags);
+
+ // trim consensus if needed
+ if (maxPos < cns.size())
+ cns = cns.substr(0, maxPos);
+ assert(minPos == 0);
+ assert(maxPos == cns.size());
+#endif
+
// Save consensus
resizeArrayPair(tig->_gappedBases, tig->_gappedQuals, 0, tig->_gappedMax, (uint32) cns.length() + 1, resizeArray_doNothing);
@@ -452,7 +525,6 @@ unitigConsensus::generatePBDAG(tgTig *tig_,
tig->_gappedBases[len] = cns[len];
tig->_gappedQuals[len] = CNS_MIN_QV;
}
-
// Terminate the string.
tig->_gappedBases[len] = 0;
@@ -842,7 +914,7 @@ unitigConsensus::computePositionFromAlignment(void) {
if (foundAlign == false) {
- if (oaPartial == false)
+ if (oaPartial == NULL)
oaPartial = new NDalign(pedLocal, errorRate, 17); // partial allowed!
oaPartial->initialize(0, abacus->bases(), abacus->numberOfColumns(), 0, abacus->numberOfColumns(),
@@ -1115,7 +1187,7 @@ unitigConsensus::alignFragment(bool forceAlignment) {
// Create new aligner object. 'Global' in this case just means to not stop early, not a true global alignment.
- if (oaFull == false)
+ if (oaFull == NULL)
oaFull = new NDalign(pedGlobal, errorRate, 17);
oaFull->initialize(0, aseq, cnsEnd - cnsBgn, 0, cnsEnd - cnsBgn,
diff --git a/src/utgcns/libcns/unitigConsensus.H b/src/utgcns/libcns/unitigConsensus.H
index 72a42a2..5a0a286 100644
--- a/src/utgcns/libcns/unitigConsensus.H
+++ b/src/utgcns/libcns/unitigConsensus.H
@@ -84,7 +84,8 @@ public:
map<uint32, gkRead *> *inPackageRead = NULL,
map<uint32, gkReadData *> *inPackageReadData = NULL);
- bool generatePBDAG(tgTig *tig,
+ bool generatePBDAG(char aligner,
+ tgTig *tig,
map<uint32, gkRead *> *inPackageRead = NULL,
map<uint32, gkReadData *> *inPackageReadData = NULL);
diff --git a/src/utgcns/libpbutgcns/LICENSE b/src/utgcns/libpbutgcns/LICENSE
index 11fc3d0..94e4fd5 100644
--- a/src/utgcns/libpbutgcns/LICENSE
+++ b/src/utgcns/libpbutgcns/LICENSE
@@ -1,36 +1,36 @@
-#################################################################################$$
-# Copyright (c) 2011-2015, Pacific Biosciences of California, Inc.
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted (subject to the limitations in the
-# disclaimer below) provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following
-# disclaimer in the documentation and/or other materials provided
-# with the distribution.
-#
-# * Neither the name of Pacific Biosciences nor the names of its
-# contributors may be used to endorse or promote products derived
-# from this software without specific prior written permission.
-#
-# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
-# GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
-# BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#################################################################################$$
+#################################################################################$$
+# Copyright (c) 2011-2015, Pacific Biosciences of California, Inc.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted (subject to the limitations in the
+# disclaimer below) provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+#
+# * Neither the name of Pacific Biosciences nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
+# GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
+# BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#################################################################################$$
diff --git a/src/utgcns/utgcns.C b/src/utgcns/utgcns.C
index 7c37214..bdf03be 100644
--- a/src/utgcns/utgcns.C
+++ b/src/utgcns/utgcns.C
@@ -84,6 +84,7 @@ main (int argc, char **argv) {
char *inPackageName = NULL;
char algorithm = 'P';
+ char aligner = 'N';
uint32 numThreads = 0;
bool forceCompute = false;
@@ -146,6 +147,8 @@ main (int argc, char **argv) {
algorithm = 'P';
} else if (strcmp(argv[arg], "-utgcns") == 0) {
algorithm = 'U';
+ } else if (strcmp(argv[arg], "-edlib") == 0) {
+ aligner = 'E';
} else if (strcmp(argv[arg], "-threads") == 0) {
numThreads = atoi(argv[++arg]);
@@ -503,7 +506,7 @@ main (int argc, char **argv) {
break;
case 'P':
default:
- success = utgcns->generatePBDAG(tig, inPackageRead, inPackageReadData);
+ success = utgcns->generatePBDAG(aligner, tig, inPackageRead, inPackageReadData);
break;
case 'U':
success = utgcns->generate(tig, inPackageRead, inPackageReadData);
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/canu.git
More information about the debian-med-commit
mailing list