[med-svn] [wgs-assembler] 05/12: Imported Upstream version 8.3rc2+ds1
Afif Elghraoui
afif-guest at moszumanska.debian.org
Sat Aug 8 07:38:06 UTC 2015
This is an automated email from the git hooks/post-receive script.
afif-guest pushed a commit to branch master
in repository wgs-assembler.
commit 7cbc871cf44f7b26bc0739e244bb0989ee38d221
Author: Afif Elghraoui <afif at ghraoui.name>
Date: Fri Aug 7 22:37:07 2015 -0700
Imported Upstream version 8.3rc2+ds1
---
README | 785 ++-
kmer/Make.include | 30 -
kmer/Make.rules | 316 -
kmer/Makefile | 194 -
kmer/configure.sh | 382 --
kmer/libbio/Make.include | 65 -
kmer/libbio/alphabet-acgtspace.c | 149 -
kmer/libbio/alphabet-colorspace.c | 121 -
kmer/libbio/alphabet-generate.c | 134 -
kmer/libbio/alphabet.c | 271 -
kmer/libbio/alphabet.h | 25 -
kmer/libbio/bio++.H | 23 -
kmer/libbio/bio.h | 47 -
kmer/libbio/halign.c | 467 --
kmer/libbio/kmer.C | 497 --
kmer/libbio/kmer.H | 160 -
kmer/libbio/kmerhuge.H | 396 --
kmer/libbio/kmeriface.H | 83 -
kmer/libbio/kmertiny.H | 147 -
kmer/libbio/merCovering.H | 353 --
kmer/libbio/merList.H | 94 -
kmer/libbio/mers.h | 63 -
kmer/libbio/reversecomplement.c | 44 -
kmer/libbio/test/Makefile | 98 -
kmer/libbio/test/halign-test.C | 48 -
kmer/libbio/test/test-bigmer-msf.C | 222 -
kmer/libbio/test/test-setbits.C | 28 -
kmer/libkmer/Make.include | 35 -
kmer/libkmer/driver-existDB.C | 227 -
kmer/libkmer/driver-posDB.C | 287 -
kmer/libkmer/existDB-create-from-fasta.C | 270 -
kmer/libkmer/existDB-create-from-meryl.C | 230 -
kmer/libkmer/existDB-create-from-sequence.C | 272 -
kmer/libkmer/existDB-state.C | 205 -
kmer/libkmer/existDB.C | 182 -
kmer/libkmer/existDB.H | 151 -
kmer/libkmer/iterate.C | 64 -
kmer/libkmer/mask.C | 283 -
kmer/libkmer/merTable.H | 76 -
kmer/libkmer/percentCovered.C | 66 -
kmer/libkmer/positionDB-access.C | 344 --
kmer/libkmer/positionDB-dump.C | 50 -
kmer/libkmer/positionDB-file.C | 211 -
kmer/libkmer/positionDB-mismatch.C | 388 --
kmer/libkmer/positionDB-sort.C | 150 -
kmer/libkmer/positionDB.C | 1125 ----
kmer/libkmer/positionDB.H | 241 -
kmer/libkmer/test/Makefile | 115 -
kmer/libkmer/test/test-maskonly.C | 110 -
kmer/libkmer/test/test-mertable.C | 15 -
kmer/libkmer/test/test-rebuild.C | 50 -
kmer/libmeryl/Make.include | 15 -
kmer/libmeryl/libmeryl.C | 490 --
kmer/libmeryl/libmeryl.H | 185 -
kmer/libseq/Make.include | 33 -
kmer/libseq/fastaFile.C | 585 --
kmer/libseq/fastaFile.H | 66 -
kmer/libseq/fastaStdin.C | 179 -
kmer/libseq/fastaStdin.H | 43 -
kmer/libseq/fastqFile.C | 593 --
kmer/libseq/fastqFile.H | 66 -
kmer/libseq/merStream.C | 84 -
kmer/libseq/merStream.H | 99 -
kmer/libseq/selftest.C | 53 -
kmer/libseq/seqCache.C | 197 -
kmer/libseq/seqCache.H | 104 -
kmer/libseq/seqFactory.C | 58 -
kmer/libseq/seqFactory.H | 33 -
kmer/libseq/seqFile.H | 50 -
kmer/libseq/seqStore.C | 622 --
kmer/libseq/seqStore.H | 120 -
kmer/libseq/seqStream.C | 396 --
kmer/libseq/seqStream.H | 124 -
kmer/libseq/sffFile.C | 208 -
kmer/libseq/sffFile.H | 104 -
kmer/libseq/test-correctSequence.H | 151 -
kmer/libseq/test-merStream.C | 284 -
kmer/libseq/test-seqCache.C | 181 -
kmer/libseq/test-seqStream.C | 287 -
kmer/libseq/test/Makefile | 23 -
kmer/libseq/test/test-merstream-speed.C | 52 -
kmer/libutil/Make.include | 63 -
kmer/libutil/NOTES | 10 -
kmer/libutil/bigQueue.C | 343 --
kmer/libutil/bigQueue.H | 150 -
kmer/libutil/bitOperations.h | 157 -
kmer/libutil/bitPackedArray.C | 100 -
kmer/libutil/bitPackedArray.H | 318 -
kmer/libutil/bitPackedFile.C | 473 --
kmer/libutil/bitPackedFile.H | 127 -
kmer/libutil/bitPacking.h | 510 --
kmer/libutil/bzipBuffer.C | 238 -
kmer/libutil/bzipBuffer.H | 92 -
kmer/libutil/eliasDeltaEncoding.h | 33 -
kmer/libutil/eliasGammaEncoding.h | 33 -
kmer/libutil/endianess.H | 64 -
kmer/libutil/fibonacciEncoding.h | 171 -
kmer/libutil/fibonacciNumbers.C | 108 -
kmer/libutil/file.c | 446 --
kmer/libutil/generalizedUnaryEncoding.h | 116 -
kmer/libutil/intervalList.C | 438 --
kmer/libutil/intervalList.H | 112 -
kmer/libutil/kazlib/Make.include | 27 -
kmer/libutil/kazlib/blast.pl | 33 -
kmer/libutil/kazlib/dict.c | 1238 ----
kmer/libutil/kazlib/dict.h | 142 -
kmer/libutil/kazlib/docs/CHANGES | 290 -
kmer/libutil/kazlib/docs/MUST_READ | 25 -
kmer/libutil/kazlib/docs/README | 66 -
kmer/libutil/kazlib/docs/docs.ist | 4 -
kmer/libutil/kazlib/docs/docs.ltx | 4155 -------------
kmer/libutil/kazlib/drivers/dict-main.c | 300 -
kmer/libutil/kazlib/drivers/except-main.c | 57 -
kmer/libutil/kazlib/drivers/hash-main.c | 187 -
kmer/libutil/kazlib/drivers/list-main.c | 152 -
kmer/libutil/kazlib/drivers/sfx-main.c | 41 -
kmer/libutil/kazlib/except.c | 347 --
kmer/libutil/kazlib/except.h | 147 -
kmer/libutil/kazlib/hash.c | 837 ---
kmer/libutil/kazlib/hash.h | 238 -
kmer/libutil/kazlib/list.c | 766 ---
kmer/libutil/kazlib/list.h | 152 -
kmer/libutil/kazlib/sfx.c | 1138 ----
kmer/libutil/kazlib/sfx.h | 46 -
kmer/libutil/logMsg.H | 115 -
kmer/libutil/md5.c | 441 --
kmer/libutil/mt19937ar/Make.include | 23 -
kmer/libutil/mt19937ar/mt19937ar-test.c | 38 -
kmer/libutil/mt19937ar/mt19937ar.c | 189 -
kmer/libutil/mt19937ar/mt19937ar.h | 47 -
kmer/libutil/mt19937ar/mt19937ar.out | 736 ---
kmer/libutil/mt19937ar/mt19937ar.readme | 74 -
kmer/libutil/mt19937ar/tt800.c | 64 -
kmer/libutil/palloc.c | 236 -
kmer/libutil/qsort_mt.c | 406 --
kmer/libutil/readBuffer.C | 284 -
kmer/libutil/readBuffer.H | 86 -
kmer/libutil/recordFile.C | 320 -
kmer/libutil/recordFile.H | 65 -
kmer/libutil/speedCounter.C | 61 -
kmer/libutil/speedCounter.H | 77 -
kmer/libutil/splitToWords.H | 117 -
kmer/libutil/sweatShop.C | 587 --
kmer/libutil/sweatShop.H | 81 -
kmer/libutil/test/Makefile | 106 -
kmer/libutil/test/atomic.C | 62 -
kmer/libutil/test/endianess.c | 124 -
kmer/libutil/test/order.C | 84 -
kmer/libutil/test/tcat.C | 86 -
kmer/libutil/test/test-bigQueue.C | 72 -
kmer/libutil/test/test-bitPackedArray.C | 152 -
kmer/libutil/test/test-bitPackedFile.C | 271 -
kmer/libutil/test/test-bitPacking.C | 337 --
kmer/libutil/test/test-bzipBuffer.C | 110 -
kmer/libutil/test/test-freeDiskSpace.c | 16 -
kmer/libutil/test/test-intervalList.C | 276 -
kmer/libutil/test/test-logMsg.C | 27 -
kmer/libutil/test/test-md5.c | 47 -
kmer/libutil/test/test-mmap.c | 80 -
kmer/libutil/test/test-palloc.c | 65 -
kmer/libutil/test/test-readBuffer.C | 135 -
kmer/libutil/test/test-recordFile.C | 64 -
kmer/libutil/test/test-types.c | 34 -
kmer/libutil/u32bitList.H | 62 -
kmer/libutil/unaryEncoding.h | 76 -
kmer/libutil/unaryEncodingTester.C | 199 -
kmer/libutil/util++.H | 46 -
kmer/libutil/util.c | 115 -
kmer/libutil/util.h | 356 --
kmer/meryl/Make.include | 43 -
kmer/meryl/args.C | 589 --
kmer/meryl/asmMerQC-regions.C | 1023 ----
kmer/meryl/asmMerQC.C | 396 --
kmer/meryl/asmMerQC.sh | 166 -
kmer/meryl/binaryOp.C | 176 -
kmer/meryl/build-threads.C | 94 -
kmer/meryl/build.C | 842 ---
kmer/meryl/compare-counts.C | 233 -
kmer/meryl/dump.C | 156 -
kmer/meryl/estimate.C | 182 -
kmer/meryl/m-heap.H | 152 -
kmer/meryl/m.C | 118 -
kmer/meryl/mapMers-depth.C | 131 -
kmer/meryl/mapMers.C | 210 -
kmer/meryl/maskMers.C | 591 --
kmer/meryl/merge.C | 240 -
kmer/meryl/merge.listmerge.C | 447 --
kmer/meryl/merge.qsort.C | 471 --
kmer/meryl/mervin.C | 793 ---
kmer/meryl/meryl.C | 72 -
kmer/meryl/meryl.H | 128 -
kmer/meryl/simple.C | 164 -
kmer/meryl/test/Makefile | 37 -
kmer/meryl/test/exhaustive.C | 171 -
kmer/meryl/test/kmerlite.H | 133 -
kmer/meryl/test/stupidcount.C | 38 -
kmer/meryl/test/test-seq1.fasta | 8 -
kmer/meryl/test/test-seq2.fasta | 18 -
kmer/meryl/test/test-seq3.fasta | 2 -
kmer/meryl/testPositionBias.C | 117 -
kmer/meryl/unaryOp.C | 59 -
samtools | 1 -
samtools-0.1.19/.gitignore | 4 -
samtools-0.1.19/AUTHORS | 20 -
samtools-0.1.19/COPYING | 21 -
samtools-0.1.19/ChangeLog.old | 3875 ------------
samtools-0.1.19/INSTALL | 30 -
samtools-0.1.19/Makefile | 101 -
samtools-0.1.19/Makefile.mingw | 63 -
samtools-0.1.19/NEWS | 836 ---
samtools-0.1.19/bam.c | 474 --
samtools-0.1.19/bam.h | 793 ---
samtools-0.1.19/bam2bcf.c | 467 --
samtools-0.1.19/bam2bcf.h | 67 -
samtools-0.1.19/bam2bcf_indel.c | 498 --
samtools-0.1.19/bam2depth.c | 143 -
samtools-0.1.19/bam_aux.c | 217 -
samtools-0.1.19/bam_cat.c | 185 -
samtools-0.1.19/bam_color.c | 145 -
samtools-0.1.19/bam_endian.h | 42 -
samtools-0.1.19/bam_import.c | 489 --
samtools-0.1.19/bam_index.c | 726 ---
samtools-0.1.19/bam_lpileup.c | 198 -
samtools-0.1.19/bam_mate.c | 128 -
samtools-0.1.19/bam_md.c | 389 --
samtools-0.1.19/bam_pileup.c | 437 --
samtools-0.1.19/bam_plcmd.c | 606 --
samtools-0.1.19/bam_reheader.c | 62 -
samtools-0.1.19/bam_rmdup.c | 206 -
samtools-0.1.19/bam_rmdupse.c | 159 -
samtools-0.1.19/bam_sort.c | 571 --
samtools-0.1.19/bam_stat.c | 77 -
samtools-0.1.19/bam_tview.c | 368 --
samtools-0.1.19/bam_tview.h | 75 -
samtools-0.1.19/bam_tview_curses.c | 297 -
samtools-0.1.19/bam_tview_html.c | 349 --
samtools-0.1.19/bamshuf.c | 141 -
samtools-0.1.19/bamtk.c | 119 -
samtools-0.1.19/bcftools/Makefile | 51 -
samtools-0.1.19/bcftools/README | 36 -
samtools-0.1.19/bcftools/bcf.c | 396 --
samtools-0.1.19/bcftools/bcf.h | 197 -
samtools-0.1.19/bcftools/bcf.tex | 77 -
samtools-0.1.19/bcftools/bcf2qcall.c | 91 -
samtools-0.1.19/bcftools/bcfutils.c | 504 --
samtools-0.1.19/bcftools/call1.c | 633 --
samtools-0.1.19/bcftools/em.c | 310 -
samtools-0.1.19/bcftools/fet.c | 112 -
samtools-0.1.19/bcftools/index.c | 336 --
samtools-0.1.19/bcftools/kfunc.c | 162 -
samtools-0.1.19/bcftools/kmin.c | 209 -
samtools-0.1.19/bcftools/kmin.h | 46 -
samtools-0.1.19/bcftools/main.c | 191 -
samtools-0.1.19/bcftools/mut.c | 127 -
samtools-0.1.19/bcftools/prob1.c | 988 ---
samtools-0.1.19/bcftools/prob1.h | 49 -
samtools-0.1.19/bcftools/vcf.c | 249 -
samtools-0.1.19/bcftools/vcfutils.pl | 567 --
samtools-0.1.19/bedcov.c | 127 -
samtools-0.1.19/bedidx.c | 162 -
samtools-0.1.19/bgzf.c | 694 ---
samtools-0.1.19/bgzf.h | 207 -
samtools-0.1.19/bgzip.c | 206 -
samtools-0.1.19/cut_target.c | 193 -
samtools-0.1.19/errmod.c | 130 -
samtools-0.1.19/errmod.h | 24 -
samtools-0.1.19/examples/00README.txt | 23 -
samtools-0.1.19/examples/Makefile | 50 -
samtools-0.1.19/examples/bam2bed.c | 51 -
samtools-0.1.19/examples/calDepth.c | 62 -
samtools-0.1.19/examples/chk_indel.c | 83 -
samtools-0.1.19/examples/ex1.fa | 56 -
samtools-0.1.19/examples/ex1.sam.gz | Bin 114565 -> 0 bytes
samtools-0.1.19/examples/toy.fa | 4 -
samtools-0.1.19/examples/toy.sam | 14 -
samtools-0.1.19/faidx.c | 437 --
samtools-0.1.19/faidx.h | 103 -
samtools-0.1.19/kaln.c | 486 --
samtools-0.1.19/kaln.h | 67 -
samtools-0.1.19/khash.h | 528 --
samtools-0.1.19/klist.h | 96 -
samtools-0.1.19/knetfile.c | 632 --
samtools-0.1.19/knetfile.h | 75 -
samtools-0.1.19/kprobaln.c | 280 -
samtools-0.1.19/kprobaln.h | 49 -
samtools-0.1.19/kseq.h | 235 -
samtools-0.1.19/ksort.h | 285 -
samtools-0.1.19/kstring.c | 212 -
samtools-0.1.19/kstring.h | 169 -
samtools-0.1.19/misc/HmmGlocal.java | 178 -
samtools-0.1.19/misc/Makefile | 69 -
samtools-0.1.19/misc/ace2sam.c | 249 -
samtools-0.1.19/misc/bamcheck.c | 1521 -----
samtools-0.1.19/misc/blast2sam.pl | 92 -
samtools-0.1.19/misc/bowtie2sam.pl | 92 -
samtools-0.1.19/misc/export2sam.pl | 545 --
samtools-0.1.19/misc/interpolate_sam.pl | 125 -
samtools-0.1.19/misc/maq2sam.c | 173 -
samtools-0.1.19/misc/md5.c | 296 -
samtools-0.1.19/misc/md5.h | 57 -
samtools-0.1.19/misc/md5fa.c | 58 -
samtools-0.1.19/misc/novo2sam.pl | 281 -
samtools-0.1.19/misc/plot-bamcheck | 882 ---
samtools-0.1.19/misc/psl2sam.pl | 65 -
samtools-0.1.19/misc/r2plot.lua | 83 -
samtools-0.1.19/misc/sam2vcf.pl | 270 -
samtools-0.1.19/misc/samtools.pl | 528 --
samtools-0.1.19/misc/soap2sam.pl | 109 -
samtools-0.1.19/misc/varfilter.py | 205 -
samtools-0.1.19/misc/vcfutils.lua | 694 ---
samtools-0.1.19/misc/wgsim.c | 419 --
samtools-0.1.19/misc/wgsim_eval.pl | 91 -
samtools-0.1.19/misc/zoom2sam.pl | 97 -
samtools-0.1.19/padding.c | 479 --
samtools-0.1.19/phase.c | 687 ---
samtools-0.1.19/razf.c | 853 ---
samtools-0.1.19/razf.h | 134 -
samtools-0.1.19/razip.c | 141 -
samtools-0.1.19/sam.c | 186 -
samtools-0.1.19/sam.h | 99 -
samtools-0.1.19/sam_header.c | 810 ---
samtools-0.1.19/sam_header.h | 48 -
samtools-0.1.19/sam_view.c | 441 --
samtools-0.1.19/sample.c | 107 -
samtools-0.1.19/sample.h | 17 -
samtools-0.1.19/samtools.1 | 1066 ----
samtools-0.1.19/win32/libcurses.a | Bin 114764 -> 0 bytes
samtools-0.1.19/win32/libz.a | Bin 74266 -> 0 bytes
samtools-0.1.19/win32/xcurses.h | 1377 -----
samtools-0.1.19/win32/zconf.h | 332 -
samtools-0.1.19/win32/zlib.h | 1357 -----
src/._AS_ALN | Bin 0 -> 311 bytes
src/._AS_ARD | Bin 0 -> 311 bytes
src/._AS_BAT | Bin 0 -> 311 bytes
src/._AS_BOG | Bin 0 -> 311 bytes
src/._AS_CGB | Bin 0 -> 311 bytes
src/._AS_CGW | Bin 0 -> 311 bytes
src/._AS_CNS | Bin 0 -> 311 bytes
src/._AS_ENV | Bin 0 -> 311 bytes
src/._AS_GKP | Bin 0 -> 311 bytes
src/._AS_LIN | Bin 0 -> 311 bytes
src/._AS_MER | Bin 0 -> 311 bytes
src/._AS_MSG | Bin 0 -> 311 bytes
src/._AS_OBT | Bin 0 -> 311 bytes
src/._AS_OVL | Bin 0 -> 311 bytes
src/._AS_OVM | Bin 0 -> 311 bytes
src/._AS_OVS | Bin 0 -> 311 bytes
src/._AS_PBR | Bin 0 -> 311 bytes
src/._AS_PER | Bin 0 -> 311 bytes
src/._AS_PL5 | Bin 0 -> 311 bytes
src/._AS_REF | Bin 0 -> 311 bytes
src/._AS_REZ | Bin 0 -> 311 bytes
src/._AS_RUN | Bin 0 -> 311 bytes
src/._AS_TER | Bin 0 -> 311 bytes
src/._AS_UID | Bin 0 -> 311 bytes
src/._AS_UTL | Bin 0 -> 311 bytes
src/._AS_VWR | Bin 0 -> 311 bytes
src/._AS_global.C | Bin 0 -> 280 bytes
src/._AS_global.H | Bin 0 -> 280 bytes
src/._Makefile | Bin 0 -> 280 bytes
src/._c_make.as | Bin 0 -> 280 bytes
src/._c_make.gen | Bin 0 -> 280 bytes
src/._site_name.as | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_aligners.H | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_bruteforcedp.C | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_bruteforcedp.H | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_dpaligner.C | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_forcns.C | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_loverlapper.C | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_pieceOlap.C | Bin 0 -> 280 bytes
src/AS_ALN/._AS_ALN_qvaligner.C | Bin 0 -> 280 bytes
src/AS_ALN/._CA_ALN_local.C | Bin 0 -> 280 bytes
src/AS_ALN/._CA_ALN_local.H | Bin 0 -> 280 bytes
src/AS_ALN/._CA_ALN_overlap.C | Bin 0 -> 280 bytes
src/AS_ALN/._Makefile | Bin 0 -> 280 bytes
src/AS_ALN/AS_ALN_bruteforcedp.C | 32 +-
src/AS_ALN/AS_ALN_forcns.C | 12 +-
src/AS_ARD/._AS_ARD_database.cc | Bin 0 -> 280 bytes
src/AS_ARD/._AS_ARD_database.hh | Bin 0 -> 280 bytes
src/AS_ARD/._BCPOutput.cc | Bin 0 -> 280 bytes
src/AS_ARD/._BCPOutput.hh | Bin 0 -> 280 bytes
src/AS_ARD/._DBSchema.sql | Bin 0 -> 280 bytes
src/AS_ARD/._DBTextOutput.cc | Bin 0 -> 280 bytes
src/AS_ARD/._DBTextOutput.hh | Bin 0 -> 280 bytes
src/AS_ARD/._IAssemblyDB.hh | Bin 0 -> 280 bytes
src/AS_ARD/._IDBConnection.hh | Bin 0 -> 280 bytes
src/AS_ARD/._IDBOutput.hh | Bin 0 -> 280 bytes
src/AS_ARD/._Makefile | Bin 0 -> 280 bytes
src/AS_ARD/._SQLOutput.cc | Bin 0 -> 280 bytes
src/AS_ARD/._SQLOutput.hh | Bin 0 -> 280 bytes
src/AS_ARD/._Sybase.cc | Bin 0 -> 280 bytes
src/AS_ARD/._Sybase.hh | Bin 0 -> 280 bytes
src/AS_ARD/._getAssembly.cc | Bin 0 -> 280 bytes
src/AS_ARD/._loadAssembly.cc | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_BestOverlapGraph.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_BestOverlapGraph.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Breaking.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Breaking.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_ChunkGraph.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_ChunkGraph.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Datatypes.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_EvaluateMates.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_EvaluateMates.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_ExtendByMates.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_FragmentInfo.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_InsertSizes.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_InsertSizes.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Instrumentation.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Instrumentation.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_IntersectBubble.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_IntersectBubble.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_IntersectSplit.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_IntersectSplit.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Joining.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Joining.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Logging.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Logging.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MateBubble.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MateChecker.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MateChecker.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MateLocation.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MateLocation.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MergeSplitJoin.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MergeSplitJoin.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MoveContains.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_MoveContains.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Outputs.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Outputs.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_OverlapCache.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_OverlapCache.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PlaceContains.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PlaceContains.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PlaceZombies.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PlaceZombies.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PopulateUnitig.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PopulateUnitig.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_PromoteToSingleton.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_ReconstructRepeats.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_RepeatJunctionEvidence.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_SetParentAndHang.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_SetParentAndHang.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_SplitDiscontinuous.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_SplitDiscontinuous.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Unitig.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Unitig.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Unitig_AddAndPlaceFrag.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Unitig_AddFrag.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_Unitig_PlaceFragUsingEdges.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_bogusUtil.C | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_bogusUtil.H | Bin 0 -> 280 bytes
src/AS_BAT/._AS_BAT_findEdges.C | Bin 0 -> 280 bytes
src/AS_BAT/._Makefile | Bin 0 -> 280 bytes
src/AS_BAT/._analyze-mapped-unitigs-for-joins.pl | Bin 0 -> 280 bytes
src/AS_BAT/._analyze-nucmer-gaps.pl | Bin 0 -> 280 bytes
src/AS_BAT/._bogart-build-fasta.pl | Bin 0 -> 280 bytes
src/AS_BAT/._bogart.C | Bin 0 -> 280 bytes
src/AS_BAT/._bogus-genome.css | Bin 0 -> 280 bytes
src/AS_BAT/._bogus-genome.json | Bin 0 -> 280 bytes
src/AS_BAT/._bogus-run.sh | Bin 0 -> 280 bytes
src/AS_BAT/._bogus.C | Bin 0 -> 280 bytes
src/AS_BAT/._bogus.json.README | Bin 0 -> 280 bytes
src/AS_BAT/._bogusness-run.pl | Bin 0 -> 280 bytes
src/AS_BAT/._bogusness.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-BFS.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-DFS.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-RFS.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-Suspicious.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-globalData.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-globalData.H | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-runningTime.H | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates-saveDistance.H | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMates.H | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMatesApply.C | Bin 0 -> 280 bytes
src/AS_BAT/._classifyMatesPairwise.C | Bin 0 -> 280 bytes
src/AS_BAT/._computeCoverageStat.C | Bin 0 -> 280 bytes
src/AS_BAT/._count-bubbles.pl | Bin 0 -> 280 bytes
.../._erate-estimate-plot-per-base-estimate.pl | Bin 0 -> 280 bytes
.../._erate-estimate-test-based-on-mapping.pl | Bin 0 -> 280 bytes
src/AS_BAT/._erate-estimate.C | Bin 0 -> 280 bytes
src/AS_BAT/._examine-mapping-ideal.pl | Bin 0 -> 280 bytes
src/AS_BAT/._examine-mapping.pl | Bin 0 -> 280 bytes
.../._locate-read-in-unitig-based-on-olaps.pl | Bin 0 -> 280 bytes
src/AS_BAT/._markRepeatUnique.C | Bin 0 -> 280 bytes
src/AS_BAT/._memoryMappedFile.H | Bin 0 -> 280 bytes
src/AS_BAT/._petey.C | Bin 0 -> 280 bytes
src/AS_BAT/._plot-break-points.pl | Bin 0 -> 280 bytes
src/AS_BAT/._rewriteCache.C | Bin 0 -> 280 bytes
src/AS_BAT/._show-false-best-edges-from-mapping.pl | Bin 0 -> 280 bytes
src/AS_BAT/._splitUnitigs.C | Bin 0 -> 280 bytes
src/AS_BAT/AS_BAT_BestOverlapGraph.C | 896 ++-
src/AS_BAT/AS_BAT_BestOverlapGraph.H | 27 +-
src/AS_BAT/AS_BAT_Datatypes.H | 23 +-
src/AS_BAT/AS_BAT_ExtendByMates.C | 4 +-
src/AS_BAT/AS_BAT_Logging.H | 4 +-
src/AS_BAT/AS_BAT_MergeSplitJoin.C | 88 +-
src/AS_BAT/AS_BAT_Outputs.C | 17 +-
src/AS_BAT/AS_BAT_OverlapCache.C | 113 +-
src/AS_BAT/AS_BAT_OverlapCache.H | 5 +-
src/AS_BAT/AS_BAT_PlaceContains.C | 104 +-
src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C | 28 +-
src/AS_BAT/AS_BAT_bogusUtil.C | 57 +-
src/AS_BAT/AS_BAT_bogusUtil.H | 7 +-
src/AS_BAT/AS_BAT_findEdges.C | 151 +
src/AS_BAT/Makefile | 51 +-
src/AS_BAT/analyze-mapped-unitigs-for-joins.pl | 269 +
src/AS_BAT/analyze-nucmer-gaps.pl | 260 +
src/AS_BAT/bogart.C | 44 +-
src/AS_BAT/bogus.C | 18 +-
src/AS_BAT/classifyMates.C | 4 +-
src/AS_BAT/count-bubbles.pl | 204 +
.../erate-estimate-plot-per-base-estimate.pl | 113 +
src/AS_BAT/erate-estimate-test-based-on-mapping.pl | 225 +
src/AS_BAT/erate-estimate.C | 688 +++
src/AS_BAT/examine-mapping-ideal.pl | 365 ++
src/AS_BAT/examine-mapping.pl | 368 ++
src/AS_BAT/locate-read-in-unitig-based-on-olaps.pl | 199 +
src/AS_BAT/markRepeatUnique.C | 601 ++
src/AS_BAT/plot-break-points.pl | 162 +
src/AS_BAT/show-false-best-edges-from-mapping.pl | 87 +
src/AS_BOG/._AS_BOG_BestOverlapGraph.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_BestOverlapGraph.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Breaking.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_ChunkGraph.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_ChunkGraph.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Datatypes.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_EvaluateMates.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_FragmentInfo.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_InsertSizes.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_InsertSizes.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Instrumentation.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_IntersectBubble.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_IntersectSplit.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Joining.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_MateBubble.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_MateChecker.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_MateLocation.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_MateLocation.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_MoveContains.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Outputs.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_OverlapBubble.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_PlaceContains.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_PlaceFragUsingOverlaps.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_PlaceZombies.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_PopulateUnitig.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_SetParentAndHang.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_SplitDiscontinuous.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Unitig.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Unitig.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_UnitigGraph.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_UnitigGraph.H | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Unitig_AddAndPlaceFrag.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Unitig_AddFrag.C | Bin 0 -> 280 bytes
src/AS_BOG/._AS_BOG_Unitig_PlaceFragUsingEdges.C | Bin 0 -> 280 bytes
src/AS_BOG/._BuildUnitigs.C | Bin 0 -> 280 bytes
src/AS_BOG/._Makefile | Bin 0 -> 280 bytes
src/AS_BOG/._analyze-unitig-mates-from-tigStore.pl | Bin 0 -> 280 bytes
src/AS_BOG/._analyzeBest.C | Bin 0 -> 280 bytes
src/AS_BOG/._find-breaks.pl | Bin 0 -> 280 bytes
src/AS_BOG/._fixUnitigs.C | Bin 0 -> 280 bytes
src/AS_BOG/._fixUnitigs.cases | Bin 0 -> 280 bytes
src/AS_BOG/._mate-based-splitting | Bin 0 -> 280 bytes
src/AS_BOG/._plot-break-points.pl | Bin 0 -> 280 bytes
src/AS_BOG/._plot-log-length.pl | Bin 0 -> 280 bytes
src/AS_BOG/AS_BOG_Breaking.C | 4 +-
src/AS_BOG/AS_BOG_Datatypes.H | 5 +-
src/AS_BOG/AS_BOG_IntersectSplit.C | 4 +-
src/AS_BOG/AS_BOG_Outputs.C | 17 +-
src/AS_BOG/AS_BOG_PlaceFragUsingOverlaps.C | 12 +-
src/AS_BOG/analyzeBest.C | 4 +-
src/AS_CGB/._AS_CGB_Bubble.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_Graph.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_Graph.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_Popper.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_Popper.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_PopperMethods.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_PopperMethods.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_VertexSet.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_VertexSet.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_Bubble_dfs.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_all.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_cga.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_cgb.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_chimeras.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_classify.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_edgemate.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_fga.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_fgb.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_histo.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_histo.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_main.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_methods.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_traversal.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_unitigger.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_unitigger_globals.H | Bin 0 -> 280 bytes
src/AS_CGB/._AS_CGB_walk.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_FGB_contained.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_FGB_hanging_fragment.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_FGB_io.C | Bin 0 -> 280 bytes
src/AS_CGB/._AS_FGB_main.C | Bin 0 -> 280 bytes
src/AS_CGB/._Makefile | Bin 0 -> 280 bytes
src/AS_CGB/AS_CGB_unitigger.C | 11 +-
src/AS_CGW/._AS_CGW_dataTypes.H | Bin 0 -> 280 bytes
src/AS_CGW/._AS_CGW_histo.C | Bin 0 -> 280 bytes
src/AS_CGW/._AS_CGW_histo.H | Bin 0 -> 280 bytes
src/AS_CGW/._AS_CGW_main.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIEdgeT_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Analysis.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Analysis.H | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Biconnected_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Cleanup_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_MergeScaffolds.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_MergeScaffolds.H | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.H | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Merge_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Merge_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._CIScaffoldT_Merge_Interleaved.C | Bin 0 -> 280 bytes
src/AS_CGW/._Celamy_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._ChiSquareTest_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._ChunkOverlap_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._ChunkOverlap_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._ContigT_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._DemoteUnitigsWithRBP_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Globals_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Globals_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._GraphCGW_T.C | Bin 0 -> 280 bytes
src/AS_CGW/._GraphCGW_T.H | Bin 0 -> 280 bytes
src/AS_CGW/._GraphEdgeIterator.C | Bin 0 -> 280 bytes
src/AS_CGW/._GraphEdgeIterator.H | Bin 0 -> 280 bytes
src/AS_CGW/._InputDataTypes_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._Input_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Input_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._Instrument_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Instrument_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._InterleavedMerging.C | Bin 0 -> 280 bytes
src/AS_CGW/._InterleavedMerging.H | Bin 0 -> 280 bytes
src/AS_CGW/._LeastSquaresGaps_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Makefile | Bin 0 -> 280 bytes
src/AS_CGW/._MarkInternalEdgeStatus.C | Bin 0 -> 280 bytes
src/AS_CGW/._MergeEdges_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Output_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Output_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._SEdgeT_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._ScaffoldGraphIterator_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._ScaffoldGraph_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._ScaffoldGraph_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._ShatterScaffolds_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._SplitScaffolds_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Stats_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._Stats_CGW.H | Bin 0 -> 280 bytes
src/AS_CGW/._TransitiveReduction_CGW.C | Bin 0 -> 280 bytes
src/AS_CGW/._analyzeScaffolds.C | Bin 0 -> 280 bytes
src/AS_CGW/._cgwDump.C | Bin 0 -> 280 bytes
src/AS_CGW/._dumpCloneMiddles.C | Bin 0 -> 280 bytes
src/AS_CGW/._dumpSingletons.C | Bin 0 -> 280 bytes
src/AS_CGW/._eCR-diagnostic.C | Bin 0 -> 280 bytes
src/AS_CGW/._eCR-examineGap.C | Bin 0 -> 280 bytes
src/AS_CGW/._eCR-partition.C | Bin 0 -> 280 bytes
src/AS_CGW/._eCR.C | Bin 0 -> 280 bytes
src/AS_CGW/._eCR.H | Bin 0 -> 280 bytes
src/AS_CGW/._find-big-gaps-in-cgw.out.pl | Bin 0 -> 280 bytes
src/AS_CGW/._find-large-gaps-in-cgw-out.pl | Bin 0 -> 280 bytes
src/AS_CGW/._fixZLFContigs.H | Bin 0 -> 280 bytes
src/AS_CGW/._fragmentPlacement.C | Bin 0 -> 280 bytes
src/AS_CGW/._fragmentPlacement.H | Bin 0 -> 280 bytes
src/AS_CGW/._frgs2clones.C | Bin 0 -> 280 bytes
src/AS_CGW/._parse-cgw-out-for-merges.pl | Bin 0 -> 280 bytes
src/AS_CGW/._resolveSurrogates.C | Bin 0 -> 280 bytes
src/AS_CGW/AS_CGW_main.C | 10 +-
src/AS_CGW/CIScaffoldT_CGW.C | 6 +-
src/AS_CGW/ContigT_CGW.C | 6 +-
src/AS_CGW/DemoteUnitigsWithRBP_CGW.C | 6 +-
src/AS_CGW/Globals_CGW.C | 4 +-
src/AS_CGW/Globals_CGW.H | 4 +-
src/AS_CGW/GraphCGW_T.C | 4 +-
src/AS_CGW/GraphCGW_T.H | 4 +-
src/AS_CGW/Input_CGW.C | 154 +-
src/AS_CGW/TransitiveReduction_CGW.C | 4 +-
src/AS_CGW/find-big-gaps-in-cgw.out.pl | 32 +
src/AS_CGW/find-large-gaps-in-cgw-out.pl | 56 +
src/AS_CGW/frgs2clones.C | 17 +-
src/AS_CGW/parse-cgw-out-for-merges.pl | 195 +
src/AS_CNS/._AbacusRefine.C | Bin 0 -> 280 bytes
src/AS_CNS/._ApplyAlignment.C | Bin 0 -> 280 bytes
src/AS_CNS/._BaseCall.C | Bin 0 -> 280 bytes
src/AS_CNS/._GetAlignmentTrace.C | Bin 0 -> 280 bytes
src/AS_CNS/._Makefile | Bin 0 -> 280 bytes
src/AS_CNS/._MergeMultiAligns.C | Bin 0 -> 280 bytes
src/AS_CNS/._MergeRefine.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlign.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlign.H | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignContig.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignMatePairAnalysis.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignMatePairAnalysis.H | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignPrint.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignSizeAnalysis.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignSizeAnalysis.H | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignStore.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignStore.H | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignUnitig.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignment_CNS.C | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignment_CNS.H | Bin 0 -> 280 bytes
src/AS_CNS/._MultiAlignment_CNS_private.H | Bin 0 -> 280 bytes
src/AS_CNS/._PrintAlignment.C | Bin 0 -> 280 bytes
src/AS_CNS/._RefreshMANode.C | Bin 0 -> 280 bytes
src/AS_CNS/._ReplaceEndUnitigInContig.C | Bin 0 -> 280 bytes
src/AS_CNS/._SeqAn_CNS.C | Bin 0 -> 280 bytes
src/AS_CNS/._addReadsToUnitigs.C | Bin 0 -> 280 bytes
src/AS_CNS/._ctgcns.C | Bin 0 -> 280 bytes
src/AS_CNS/._submit-parallel.pl | Bin 0 -> 280 bytes
src/AS_CNS/._tigStore.C | Bin 0 -> 280 bytes
src/AS_CNS/._tigStoreDelete.pl | Bin 0 -> 280 bytes
src/AS_CNS/._utgcns.C | Bin 0 -> 280 bytes
src/AS_CNS/._utgcnsfix.C | Bin 0 -> 280 bytes
src/AS_CNS/MultiAlign.C | 107 +-
src/AS_CNS/MultiAlign.H | 9 +-
src/AS_CNS/MultiAlignMatePairAnalysis.C | 12 +-
src/AS_CNS/MultiAlignStore.C | 3 +-
src/AS_CNS/MultiAlignStore.H | 81 +-
src/AS_CNS/addReadsToUnitigs.C | 261 +-
src/AS_CNS/tigStore.C | 367 +-
src/AS_CNS/utgcns.C | 128 +-
src/AS_ENV/._Makefile | Bin 0 -> 280 bytes
src/AS_ENV/._chimChe.C | Bin 0 -> 280 bytes
src/AS_ENV/._greedyFragmentTiling.C | Bin 0 -> 280 bytes
src/AS_ENV/._greedy_layout_to_IUM.pl | Bin 0 -> 280 bytes
src/AS_ENV/._kmer_utils.H | Bin 0 -> 280 bytes
src/AS_ENV/._metagenomics_ovl_analyses.C | Bin 0 -> 280 bytes
src/AS_ENV/._run_greedy.csh | Bin 0 -> 280 bytes
src/AS_ENV/._utg2fasta.pl | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_bench.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_buildPartition.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_checkFrag.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_checkLibrary.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_checkLink.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_checkPlace.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_dump.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_edit.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_errors.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_illumina.C | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_include.H | Bin 0 -> 280 bytes
src/AS_GKP/._AS_GKP_main.C | Bin 0 -> 280 bytes
src/AS_GKP/._Makefile | Bin 0 -> 280 bytes
src/AS_GKP/._build-ordered-reads.sh | Bin 0 -> 280 bytes
src/AS_GKP/._clearRangeHistogram.C | Bin 0 -> 280 bytes
src/AS_GKP/._fastqAnalyze.C | Bin 0 -> 280 bytes
src/AS_GKP/._fastqSample.C | Bin 0 -> 280 bytes
src/AS_GKP/._fastqSimulate-checkCoverage.pl | Bin 0 -> 280 bytes
src/AS_GKP/._fastqSimulate-perfectSep.pl | Bin 0 -> 280 bytes
src/AS_GKP/._fastqSimulate-sort.C | Bin 0 -> 280 bytes
src/AS_GKP/._fastqSimulate.C | Bin 0 -> 280 bytes
src/AS_GKP/._fastqToCA.C | Bin 0 -> 280 bytes
src/AS_GKP/._frg-to-fastq.pl | Bin 0 -> 280 bytes
src/AS_GKP/._gkpStoreCreate.C | Bin 0 -> 280 bytes
src/AS_GKP/._gkpStoreDumpFASTQ.C | Bin 0 -> 280 bytes
src/AS_GKP/._sffToCA-evaluateLinker.pl | Bin 0 -> 280 bytes
src/AS_GKP/._sffToCA.C | Bin 0 -> 280 bytes
src/AS_GKP/._upgrade-v6-to-v7.C | Bin 0 -> 280 bytes
src/AS_GKP/._upgrade-v7-to-v8.C | Bin 0 -> 280 bytes
src/AS_GKP/._upgrade-v8-to-v9.C | Bin 0 -> 280 bytes
src/AS_GKP/._upgrade-v9-to-v10.C | Bin 0 -> 280 bytes
src/AS_GKP/AS_GKP_dump.C | 194 +-
src/AS_GKP/AS_GKP_illumina.C | 96 +-
src/AS_GKP/AS_GKP_include.H | 7 +-
src/AS_GKP/AS_GKP_main.C | 27 +-
src/AS_GKP/build-ordered-reads.sh | 413 ++
src/AS_GKP/fastqAnalyze.C | 49 +-
src/AS_GKP/fastqSample.C | 450 +-
src/AS_GKP/fastqSimulate-checkCoverage.pl | 104 +
src/AS_GKP/fastqSimulate.C | 232 +-
src/AS_GKP/gkpStoreCreate.C | 4 +-
src/AS_GKP/gkpStoreDumpFASTQ.C | 16 +-
src/AS_GKP/sffToCA.C | 17 +-
src/AS_LIN/._Makefile | Bin 0 -> 280 bytes
src/AS_LIN/._daxpy.c | Bin 0 -> 280 bytes
src/AS_LIN/._dcopy.c | Bin 0 -> 280 bytes
src/AS_LIN/._ddot.c | Bin 0 -> 280 bytes
src/AS_LIN/._dgbtf2.c | Bin 0 -> 280 bytes
src/AS_LIN/._dgbtrf.c | Bin 0 -> 280 bytes
src/AS_LIN/._dgbtrs.c | Bin 0 -> 280 bytes
src/AS_LIN/._dgemm.c | Bin 0 -> 280 bytes
src/AS_LIN/._dgemv.c | Bin 0 -> 280 bytes
src/AS_LIN/._dger.c | Bin 0 -> 280 bytes
src/AS_LIN/._disnan.c | Bin 0 -> 280 bytes
src/AS_LIN/._dlaisnan.c | Bin 0 -> 280 bytes
src/AS_LIN/._dlamch.c | Bin 0 -> 280 bytes
src/AS_LIN/._dlaswp.c | Bin 0 -> 280 bytes
src/AS_LIN/._dpbtf2.c | Bin 0 -> 280 bytes
src/AS_LIN/._dpbtrf.c | Bin 0 -> 280 bytes
src/AS_LIN/._dpbtrs.c | Bin 0 -> 280 bytes
src/AS_LIN/._dpotf2.c | Bin 0 -> 280 bytes
src/AS_LIN/._dptrfs.c | Bin 0 -> 280 bytes
src/AS_LIN/._dpttrs.c | Bin 0 -> 280 bytes
src/AS_LIN/._dscal.c | Bin 0 -> 280 bytes
src/AS_LIN/._dswap.c | Bin 0 -> 280 bytes
src/AS_LIN/._dsyr.c | Bin 0 -> 280 bytes
src/AS_LIN/._dsyrk.c | Bin 0 -> 280 bytes
src/AS_LIN/._dtbsv.c | Bin 0 -> 280 bytes
src/AS_LIN/._dtrsm.c | Bin 0 -> 280 bytes
src/AS_LIN/._f2c.h | Bin 0 -> 280 bytes
src/AS_LIN/._idamax.c | Bin 0 -> 280 bytes
src/AS_LIN/._ieeeck.c | Bin 0 -> 280 bytes
src/AS_LIN/._ilaenv.c | Bin 0 -> 280 bytes
src/AS_LIN/._iparmq.c | Bin 0 -> 280 bytes
src/AS_LIN/._lsame.c | Bin 0 -> 280 bytes
src/AS_LIN/._s_cmp.c | Bin 0 -> 280 bytes
src/AS_LIN/._s_copy.c | Bin 0 -> 280 bytes
src/AS_LIN/._xerbla.c | Bin 0 -> 280 bytes
src/AS_MER/._AS_MER_gkpStoreChain.C | Bin 0 -> 280 bytes
src/AS_MER/._AS_MER_gkpStoreChain.H | Bin 0 -> 280 bytes
src/AS_MER/._AS_MER_gkpStore_to_FastABase.C | Bin 0 -> 280 bytes
src/AS_MER/._AS_MER_gkpStore_to_FastABase.H | Bin 0 -> 280 bytes
src/AS_MER/._AS_MER_meryl.cc | Bin 0 -> 280 bytes
src/AS_MER/._Makefile | Bin 0 -> 280 bytes
src/AS_MER/._estimate-mer-threshold.C | Bin 0 -> 280 bytes
src/AS_MER/._gkrpt.pl | Bin 0 -> 280 bytes
src/AS_MER/._merTrim-compare-logs.pl | Bin 0 -> 280 bytes
src/AS_MER/._merTrim.C | Bin 0 -> 280 bytes
src/AS_MER/._merTrimAdapter.C | Bin 0 -> 280 bytes
src/AS_MER/._merTrimApply.C | Bin 0 -> 280 bytes
src/AS_MER/._merTrimResult.H | Bin 0 -> 280 bytes
src/AS_MER/._mercy-regions.C | Bin 0 -> 280 bytes
src/AS_MER/._mercy.C | Bin 0 -> 280 bytes
src/AS_MER/._mercy.sh | Bin 0 -> 280 bytes
src/AS_MER/._meryl.C | Bin 0 -> 280 bytes
src/AS_MER/._overmerry.C | Bin 0 -> 280 bytes
src/AS_MER/AS_MER_gkpStoreChain.C | 42 +-
src/AS_MER/AS_MER_gkpStoreChain.H | 32 +-
src/AS_MER/AS_MER_gkpStore_to_FastABase.C | 12 +-
src/AS_MER/AS_MER_gkpStore_to_FastABase.H | 20 +-
src/AS_MER/Makefile | 91 +-
src/AS_MER/estimate-mer-threshold.C | 22 +-
src/AS_MER/merTrim-compare-logs.pl | 86 +
src/AS_MER/merTrim.C | 16 +-
src/AS_MER/merTrimApply.C | 4 +-
src/AS_MER/mercy-regions.C | 70 +-
src/AS_MER/mercy.C | 84 +-
src/AS_MER/overmerry.C | 112 +-
src/AS_MSG/._AS_MSG_pmesg.C | Bin 0 -> 280 bytes
src/AS_MSG/._AS_MSG_pmesg.H | Bin 0 -> 280 bytes
src/AS_MSG/._AS_MSG_pmesg1.C | Bin 0 -> 280 bytes
src/AS_MSG/._AS_MSG_pmesg2.C | Bin 0 -> 280 bytes
src/AS_MSG/._AS_MSG_pmesg_documentation.txt | Bin 0 -> 280 bytes
src/AS_MSG/._AS_MSG_pmesg_internal.H | Bin 0 -> 280 bytes
src/AS_MSG/._AS_MSG_types.H | Bin 0 -> 280 bytes
src/AS_MSG/._ExtractMessages.C | Bin 0 -> 280 bytes
src/AS_MSG/._Makefile | Bin 0 -> 280 bytes
src/AS_MSG/._convert-fasta-to-v2.pl | Bin 0 -> 280 bytes
src/AS_MSG/._convert-v1-to-v2.pl | Bin 0 -> 280 bytes
src/AS_MSG/._generate-random-fragments.sh | Bin 0 -> 280 bytes
src/AS_MSG/._remove_fragment.C | Bin 0 -> 280 bytes
src/AS_MSG/._tracedb-to-frg.pl | Bin 0 -> 280 bytes
src/AS_MSG/AS_MSG_pmesg.H | 8 +-
src/AS_MSG/AS_MSG_pmesg1.C | 6 +-
src/AS_MSG/AS_MSG_types.H | 10 +-
src/AS_MSG/tracedb-to-frg.pl | 8 +-
src/AS_OBT/._AS_OBT_acceptableOverlap.C | Bin 0 -> 280 bytes
src/AS_OBT/._AS_OBT_acceptableOverlap.H | Bin 0 -> 280 bytes
src/AS_OBT/._AS_OBT_overlaps.C | Bin 0 -> 280 bytes
src/AS_OBT/._AS_OBT_overlaps.H | Bin 0 -> 280 bytes
src/AS_OBT/._Makefile | Bin 0 -> 280 bytes
src/AS_OBT/._chimera.C | Bin 0 -> 280 bytes
src/AS_OBT/._deduplicate.C | Bin 0 -> 280 bytes
.../._detect-unsplit-subreads-in-overlaps.pl | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim-bestEdge.C | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim-consolidate.H | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim-evidenceBased.C | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim-largestCovered.C | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim-stats.pl | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim.C | Bin 0 -> 280 bytes
src/AS_OBT/._finalTrim.H | Bin 0 -> 280 bytes
src/AS_OBT/._generate-random-chimeric-fragments.pl | Bin 0 -> 280 bytes
src/AS_OBT/._initialTrim.C | Bin 0 -> 280 bytes
src/AS_OBT/._removeMateOverlap.C | Bin 0 -> 280 bytes
src/AS_OBT/._test-random-chimeric-fragments.pl | Bin 0 -> 280 bytes
src/AS_OBT/._trim.C | Bin 0 -> 280 bytes
src/AS_OBT/._trim.H | Bin 0 -> 280 bytes
src/AS_OBT/AS_OBT_overlaps.C | 50 +-
src/AS_OBT/chimera.C | 44 +-
src/AS_OBT/detect-unsplit-subreads-in-overlaps.pl | 95 +
src/AS_OBT/finalTrim-bestEdge.C | 4 +-
src/AS_OBT/finalTrim-largestCovered.C | 20 +-
src/AS_OBT/finalTrim.C | 39 +-
src/AS_OBT/finalTrim.H | 4 +-
src/AS_OBT/trim.C | 4 +-
src/AS_OVL/._AS_OVL_olapstats.H | Bin 0 -> 280 bytes
src/AS_OVL/._CatCorrectsOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._CatEratesOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._CorrectOlapsOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._FragCorrectOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._FragCorrectOVL.H | Bin 0 -> 280 bytes
src/AS_OVL/._Makefile | Bin 0 -> 280 bytes
src/AS_OVL/._OlapFromSeedsOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._OlapFromSeedsOVL.H | Bin 0 -> 280 bytes
src/AS_OVL/._SharedOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._SharedOVL.H | Bin 0 -> 280 bytes
src/AS_OVL/._ShowCorrectsOVL.C | Bin 0 -> 280 bytes
src/AS_OVL/._delcher.sty | Bin 0 -> 280 bytes
src/AS_OVL/._olap-seeds-doc.tex | Bin 0 -> 280 bytes
src/AS_OVL/._overlap_partition.C | Bin 0 -> 280 bytes
src/AS_OVL/AS_OVL_delcher.C | 188 -
src/AS_OVL/AS_OVL_delcher.H | 78 -
src/AS_OVL/CorrectOlapsOVL.C | 242 +-
src/AS_OVL/FragCorrectOVL.C | 175 +-
src/AS_OVL/Makefile | 18 +-
src/AS_OVL/OlapFromSeedsOVL.C | 47 +-
src/AS_OVL/OlapFromSeedsOVL.H | 9 +-
src/AS_OVL/SharedOVL.C | 5 +-
src/AS_OVL/SharedOVL.H | 8 +-
src/AS_OVL/ShowCorrectsOVL.C | 10 +-
src/AS_OVM/._Makefile | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore-Build_Hash_Index.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore-Extend_Alignment.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore-Find_Overlaps.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore-Output.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore-Process_Overlaps.C | Bin 0 -> 280 bytes
.../._overlapInCore-Process_String_Overlaps.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore-Read_Next_Frag.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore.C | Bin 0 -> 280 bytes
src/AS_OVM/._overlapInCore.H | Bin 0 -> 280 bytes
src/AS_OVM/overlapInCore-Extend_Alignment.C | 184 +-
src/AS_OVM/overlapInCore.C | 25 +-
src/AS_OVM/overlapInCore.H | 8 +-
src/AS_OVS/._AS_OVS_overlap.C | Bin 0 -> 280 bytes
src/AS_OVS/._AS_OVS_overlap.H | Bin 0 -> 280 bytes
src/AS_OVS/._AS_OVS_overlapFile.C | Bin 0 -> 280 bytes
src/AS_OVS/._AS_OVS_overlapFile.H | Bin 0 -> 280 bytes
src/AS_OVS/._AS_OVS_overlapStore.C | Bin 0 -> 280 bytes
src/AS_OVS/._AS_OVS_overlapStore.H | Bin 0 -> 280 bytes
src/AS_OVS/._AS_UTL_histogram.H | Bin 0 -> 280 bytes
src/AS_OVS/._Makefile | Bin 0 -> 280 bytes
src/AS_OVS/._README.GES | Bin 0 -> 280 bytes
src/AS_OVS/._analyze-true-vs-test.pl | Bin 0 -> 280 bytes
.../._check-ordered-reads-for-missed-overlaps.pl | Bin 0 -> 280 bytes
src/AS_OVS/._convertOverlap.C | Bin 0 -> 280 bytes
src/AS_OVS/._filterOverlap.C | Bin 0 -> 280 bytes
src/AS_OVS/._filterTrue.pl | Bin 0 -> 280 bytes
src/AS_OVS/._find-missed-true-overlaps.pl | Bin 0 -> 280 bytes
src/AS_OVS/._infer-obt-from-genomic-blasr.pl | Bin 0 -> 280 bytes
src/AS_OVS/._infer-olaps-from-genomic-coords.pl | Bin 0 -> 280 bytes
src/AS_OVS/._infer-olaps-from-pairwise-blasr.pl | Bin 0 -> 280 bytes
src/AS_OVS/._infer-olaps-from-pairwise-coords.pl | Bin 0 -> 280 bytes
src/AS_OVS/._infer-ovl-from-genomic-blasr.pl | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStats.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStatsBoringStuff.H | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStore.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStore.H | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStoreBucketizer.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStoreBuild.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStoreIndexer.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStoreSorter.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStore_dump.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStore_erates.C | Bin 0 -> 280 bytes
src/AS_OVS/._overlapStore_genomeLength.C | Bin 0 -> 280 bytes
src/AS_OVS/._rewrite-obt-dump-with-names.pl | Bin 0 -> 280 bytes
src/AS_OVS/._rewrite-overlap-dump-with-names.pl | Bin 0 -> 280 bytes
src/AS_OVS/AS_OVS_overlap.C | 42 +-
src/AS_OVS/AS_OVS_overlap.H | 4 +-
src/AS_OVS/analyze-true-vs-test.pl | 552 ++
.../check-ordered-reads-for-missed-overlaps.pl | 132 +
src/AS_OVS/filterTrue.pl | 549 ++
src/AS_OVS/find-missed-true-overlaps.pl | 91 +
src/AS_OVS/infer-obt-from-genomic-blasr.pl | 43 +
src/AS_OVS/infer-olaps-from-genomic-coords.pl | 316 +
src/AS_OVS/infer-olaps-from-pairwise-blasr.pl | 152 +
src/AS_OVS/infer-olaps-from-pairwise-coords.pl | 184 +
src/AS_OVS/infer-ovl-from-genomic-blasr.pl | 150 +
src/AS_OVS/overlapStore.C | 17 +-
src/AS_OVS/overlapStore.H | 20 +-
src/AS_OVS/overlapStoreBucketizer.C | 34 +-
src/AS_OVS/overlapStoreBuild.C | 32 +-
src/AS_OVS/overlapStore_dump.C | 252 +-
src/AS_OVS/rewrite-obt-dump-with-names.pl | 24 +
src/AS_OVS/rewrite-overlap-dump-with-names.pl | 26 +
src/AS_PBR/._AS_PBR_correct.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_correct.hh | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_dumpStore.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_filter.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_filter.hh | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_mates.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_mates.hh | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_output.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_output.hh | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_store.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_store.hh | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_util.cc | Bin 0 -> 280 bytes
src/AS_PBR/._AS_PBR_util.hh | Bin 0 -> 280 bytes
src/AS_PBR/._ConvertSamToCA.cc | Bin 0 -> 280 bytes
src/AS_PBR/._CorrectPacBio.cc | Bin 0 -> 280 bytes
src/AS_PBR/._Makefile | Bin 0 -> 280 bytes
src/AS_PBR/._OutputLayout.cc | Bin 0 -> 280 bytes
src/AS_PBR/._PBcR.pl | Bin 0 -> 280 bytes
src/AS_PBR/._TrimFastqByQVWindow.cc | Bin 0 -> 280 bytes
src/AS_PBR/._addCNSToStore.pl | Bin 0 -> 280 bytes
src/AS_PBR/._bamcat.c | Bin 0 -> 280 bytes
src/AS_PBR/._bamcat.h | Bin 0 -> 280 bytes
src/AS_PBR/._blasr.Linux-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._boost | Bin 0 -> 311 bytes
src/AS_PBR/._convertToPBCNS.pl | Bin 0 -> 280 bytes
src/AS_PBR/._falcon_sense.Darwin-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._falcon_sense.Linux-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._jellyfish.Darwin-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._jellyfish.Linux-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._libgcc_s.1.dylib | Bin 0 -> 280 bytes
src/AS_PBR/._libstdc++.6.dylib | Bin 0 -> 280 bytes
src/AS_PBR/._mhap.tar.gz | Bin 0 -> 280 bytes
src/AS_PBR/._pbdagcon.Linux-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._pbutgcns.Darwin-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/._pbutgcns.Linux-amd64.bin | Bin 0 -> 280 bytes
src/AS_PBR/AS_PBR_correct.cc | 51 +-
src/AS_PBR/AS_PBR_dumpStore.cc | 6 +-
src/AS_PBR/AS_PBR_filter.cc | 31 +-
src/AS_PBR/AS_PBR_output.cc | 188 +-
src/AS_PBR/AS_PBR_output.hh | 5 +-
src/AS_PBR/AS_PBR_util.cc | 29 +-
src/AS_PBR/AS_PBR_util.hh | 29 +-
src/AS_PBR/ConvertSamToCA.cc | 2 +-
src/AS_PBR/CorrectPacBio.cc | 13 +-
src/AS_PBR/Makefile | 119 +-
src/AS_PBR/OutputLayout.cc | 16 +-
src/AS_PBR/PBcR.pl | 2992 +++++++++
src/AS_PBR/TrimFastqByQVWindow.cc | 0
src/AS_PBR/addCNSToStore.pl | 345 ++
src/AS_PBR/bamcat.c | 2605 ++++++++
src/AS_PBR/bamcat.h | 2134 +++++++
src/AS_PBR/blasr.Linux-amd64.bin | Bin 0 -> 7963062 bytes
src/AS_PBR/boost/._LICENSE_1_0.txt | Bin 0 -> 280 bytes
src/AS_PBR/boost/._config | Bin 0 -> 311 bytes
src/AS_PBR/boost/._config.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/._detail | Bin 0 -> 311 bytes
src/AS_PBR/boost/._dynamic_bitset | Bin 0 -> 311 bytes
src/AS_PBR/boost/._dynamic_bitset.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/._dynamic_bitset_fwd.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/._limits.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/._pending | Bin 0 -> 311 bytes
src/AS_PBR/boost/._static_assert.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._abi | Bin 0 -> 311 bytes
src/AS_PBR/boost/config/._abi_prefix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._abi_suffix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._auto_link.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._compiler | Bin 0 -> 311 bytes
src/AS_PBR/boost/config/._no_tr1 | Bin 0 -> 311 bytes
src/AS_PBR/boost/config/._platform | Bin 0 -> 311 bytes
src/AS_PBR/boost/config/._posix_features.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._requires_threads.hpp | Bin 0 -> 280 bytes
.../boost/config/._select_compiler_config.hpp | Bin 0 -> 280 bytes
.../boost/config/._select_platform_config.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._select_stdlib_config.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._stdlib | Bin 0 -> 311 bytes
src/AS_PBR/boost/config/._suffix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._user.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/._warning_disable.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/abi/._borland_prefix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/abi/._borland_suffix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/abi/._msvc_prefix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/abi/._msvc_suffix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._borland.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._clang.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._codegear.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._comeau.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._common_edg.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._compaq_cxx.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._cray.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._digitalmars.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._gcc.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._gcc_xml.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._greenhills.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._hp_acc.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._intel.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._kai.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._metrowerks.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._mpw.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._nvcc.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._pathscale.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._pgi.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._sgi_mipspro.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._sunpro_cc.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._vacpp.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/compiler/._visualc.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/no_tr1/._cmath.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/no_tr1/._complex.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/no_tr1/._functional.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/no_tr1/._memory.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/no_tr1/._utility.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._aix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._amigaos.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._beos.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._bsd.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._cray.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._cygwin.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._hpux.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._irix.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._linux.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._macos.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._qnxnto.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._solaris.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._symbian.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._vms.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._vxworks.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/platform/._win32.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._dinkumware.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._libcomo.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._libcpp.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._libstdcpp3.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._modena.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._msl.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._roguewave.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._sgi.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._stlport.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/config/stdlib/._vacpp.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/detail/._dynamic_bitset.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/detail/._iterator.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/detail/._workaround.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/dynamic_bitset/._config.hpp | Bin 0 -> 280 bytes
.../boost/dynamic_bitset/._dynamic_bitset.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/pending/._integer_log2.hpp | Bin 0 -> 280 bytes
src/AS_PBR/boost/pending/._lowest_bit.hpp | Bin 0 -> 280 bytes
src/AS_PBR/convertToPBCNS.pl | 381 +-
src/AS_PBR/falcon_sense.Darwin-amd64.bin | Bin 0 -> 5470817 bytes
src/AS_PBR/falcon_sense.Linux-amd64.bin | Bin 0 -> 7521743 bytes
src/AS_PBR/jellyfish.Darwin-amd64.bin | Bin 0 -> 851752 bytes
src/AS_PBR/jellyfish.Linux-amd64.bin | Bin 0 -> 989208 bytes
src/AS_PBR/libgcc_s.1.dylib | Bin 0 -> 82640 bytes
src/AS_PBR/libstdc++.6.dylib | Bin 0 -> 906176 bytes
src/AS_PBR/mhap.tar.gz | Bin 0 -> 2289816 bytes
src/AS_PBR/pacBioToCA.pl | 2056 -------
src/AS_PBR/pbdagcon.Linux-amd64.bin | Bin 0 -> 2956633 bytes
src/AS_PBR/pbutgcns.Darwin-amd64.bin | Bin 0 -> 1607332 bytes
src/AS_PBR/pbutgcns.Linux-amd64.bin | Bin 0 -> 1975552 bytes
src/AS_PER/._AS_PER_encodeSequenceQuality.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_encodeSequenceQuality.H | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_genericStore.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_genericStore.H | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_genericStore.doc.txt | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkLibrary.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_IID.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_PLC.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_UID.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_clearRange.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_fragments.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_load.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_partition.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStore_stats.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkStream.C | Bin 0 -> 280 bytes
src/AS_PER/._AS_PER_gkpStore.H | Bin 0 -> 280 bytes
src/AS_PER/._Makefile | Bin 0 -> 280 bytes
src/AS_PER/._gkClearRange.H | Bin 0 -> 280 bytes
src/AS_PER/._gkFragment.H | Bin 0 -> 280 bytes
src/AS_PER/._gkLibrary.H | Bin 0 -> 280 bytes
src/AS_PER/._gkPlacement.H | Bin 0 -> 280 bytes
src/AS_PER/._gkStore.H | Bin 0 -> 280 bytes
src/AS_PER/._gkStream.H | Bin 0 -> 280 bytes
src/AS_PER/._size.C | Bin 0 -> 280 bytes
src/AS_PL5/._AS_MSG | Bin 0 -> 311 bytes
src/AS_PL5/._AS_PER | Bin 0 -> 311 bytes
src/AS_PL5/._README | Bin 0 -> 280 bytes
src/AS_PL5/._caqc.pl | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._MSG.C | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._MSG.h | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._MSG.swig | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._Makefile.PL | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._Parser.pm | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._acronyms.h | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._parser.h | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._pmesg.h | Bin 0 -> 280 bytes
src/AS_PL5/AS_MSG/._test.pl | Bin 0 -> 280 bytes
src/AS_PL5/AS_PER/._GateKeeperStore.pm | Bin 0 -> 280 bytes
src/AS_PL5/AS_PER/._Makefile.PL | Bin 0 -> 280 bytes
src/AS_PL5/AS_PER/._PER.swig | Bin 0 -> 280 bytes
src/AS_REF/._Makefile | Bin 0 -> 280 bytes
src/AS_REF/._NOTES | Bin 0 -> 280 bytes
src/AS_REF/._buildRefContigs.C | Bin 0 -> 280 bytes
src/AS_REF/._buildRefOverlaps.C | Bin 0 -> 280 bytes
src/AS_REF/._frg-to-fasta.pl | Bin 0 -> 280 bytes
src/AS_REF/._refAlignment.H | Bin 0 -> 280 bytes
src/AS_REF/buildRefContigs.C | 8 +-
src/AS_REF/buildRefOverlaps.C | 4 +-
src/AS_REZ/._CommonREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._ConsistencyChecksREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._ConsistencyChecksREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._DataTypesREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._GapFillREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._GapWalkerREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._GapWalkerREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._Makefile | Bin 0 -> 280 bytes
src/AS_REZ/._MicroHetREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._MicroHetREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._PartitionsREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._RepeatRez.H | Bin 0 -> 280 bytes
src/AS_REZ/._SUBDELREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._StatisticsREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._StatisticsREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._UpdateREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._UpdateREZ.H | Bin 0 -> 280 bytes
src/AS_REZ/._UtilsREZ.C | Bin 0 -> 280 bytes
src/AS_REZ/._UtilsREZ.H | Bin 0 -> 280 bytes
src/AS_RUN/._Makefile | Bin 0 -> 280 bytes
src/AS_RUN/._README | Bin 0 -> 280 bytes
src/AS_RUN/._asmQC | Bin 0 -> 311 bytes
src/AS_RUN/._fragmentDepth.C | Bin 0 -> 280 bytes
src/AS_RUN/._get-commit-logs.pl | Bin 0 -> 280 bytes
src/AS_RUN/._markUniqueUnique.C | Bin 0 -> 280 bytes
src/AS_RUN/._new.pl | Bin 0 -> 280 bytes
src/AS_RUN/._replaceIIDwithName-overlapDump.pl | Bin 0 -> 280 bytes
src/AS_RUN/._replaceUIDwithName-fastq.pl | Bin 0 -> 280 bytes
src/AS_RUN/._replaceUIDwithName-posmap.pl | Bin 0 -> 280 bytes
src/AS_RUN/._runCA-dedupe.pl | Bin 0 -> 280 bytes
src/AS_RUN/._runCA-overlapStoreBuild.pl | Bin 0 -> 280 bytes
src/AS_RUN/._runCA.pl | Bin 0 -> 280 bytes
src/AS_RUN/._sanity | Bin 0 -> 311 bytes
src/AS_RUN/Makefile | 13 +-
src/AS_RUN/asmQC/._TIGR | Bin 0 -> 311 bytes
src/AS_RUN/asmQC/._asmToAGP.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._asmToFasta.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._ca2ace.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._caqc-diff.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._caqc.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._caqc_help.ini | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._checkMates.rb | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._compare-asm.sh | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._compare-qc.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._compare-trim-against-map.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._find-chimeric-contigs.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._find-chimeric-reads.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._mergeqc.pl | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._statsFromAGP.rb | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._unitigGraph.rb | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/._utgCoverageGaps.rb | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/TIGR/._AsmLib.pm | Bin 0 -> 280 bytes
src/AS_RUN/asmQC/TIGR/._Foundation.pm | Bin 0 -> 280 bytes
src/AS_RUN/markUniqueUnique.C | 7 +-
src/AS_RUN/new.pl | 6325 ++++++++++++++++++++
src/AS_RUN/replaceUIDwithName-fastq.pl | 117 +
src/AS_RUN/replaceUIDwithName-posmap.pl | 62 +
src/AS_RUN/replaceUIDwithName-simple.pl | 67 -
src/AS_RUN/replaceUIDwithName.pl | 117 -
src/AS_RUN/runCA-overlapStoreBuild.pl | 98 +-
src/AS_RUN/runCA.pl | 1114 ++--
src/AS_RUN/sanity/._build-all-kmer-revisions.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._build-all-wgs-revisions.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._compile-all-wgs-revisions.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity-all-done.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity-asm-done.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity-get-next-date.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity-merge-qc.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity-purge-old.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity-update-reference.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity.pl | Bin 0 -> 280 bytes
src/AS_RUN/sanity/._sanity.sh | Bin 0 -> 280 bytes
src/AS_RUN/sanity/build-all-kmer-revisions.pl | 26 +
src/AS_RUN/sanity/build-all-wgs-revisions.pl | 153 +
src/AS_RUN/sanity/compile-all-wgs-revisions.pl | 177 +
src/AS_RUN/sanity/sanity.pl | 163 +-
src/AS_TER/._Makefile | Bin 0 -> 280 bytes
src/AS_TER/._analyzePosMap-gapFillProbability.C | Bin 0 -> 280 bytes
src/AS_TER/._analyzePosMap-libraryFate.C | Bin 0 -> 280 bytes
src/AS_TER/._analyzePosMap-load.C | Bin 0 -> 280 bytes
src/AS_TER/._analyzePosMap.C | Bin 0 -> 280 bytes
src/AS_TER/._analyzePosMap.H | Bin 0 -> 280 bytes
src/AS_TER/._asmOutputFasta.C | Bin 0 -> 280 bytes
src/AS_TER/._asmOutputStatistics.C | Bin 0 -> 280 bytes
src/AS_TER/._buildPosMap.C | Bin 0 -> 280 bytes
src/AS_TER/._demotePosMap.C | Bin 0 -> 280 bytes
src/AS_TER/._fragsInVars.C | Bin 0 -> 280 bytes
src/AS_TER/._posmap-to-gff3.pl | Bin 0 -> 280 bytes
src/AS_TER/._terminator.C | Bin 0 -> 280 bytes
src/AS_TER/analyzePosMap.H | 6 +-
src/AS_TER/buildPosMap.C | 13 +-
src/AS_TER/demotePosMap.C | 4 +-
src/AS_TER/posmap-to-gff3.pl | 109 +
src/AS_UID/._Makefile | Bin 0 -> 280 bytes
src/AS_UID/._SYS_UIDclient.H | Bin 0 -> 280 bytes
src/AS_UID/._SYS_UIDclient_JTC.C | Bin 0 -> 280 bytes
src/AS_UID/._SYS_UIDclient_LOCAL.C | Bin 0 -> 280 bytes
src/AS_UID/._SYS_UIDclient_SERVER.C | Bin 0 -> 280 bytes
src/AS_UID/._uidclient.C | Bin 0 -> 280 bytes
src/AS_UID/._uidserver.C | Bin 0 -> 280 bytes
src/AS_UID/._uidserver_common.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_GPL.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_GPL.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_Hash.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_Hash.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_IID.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_UID.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_UID.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_Var.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_Var.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_alloc.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_alloc.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_decodeRange.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_decodeRange.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_extractMSG | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_extractMSG.awk | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_fasta.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_fasta.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_fileIO.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_fileIO.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_heap.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_heap.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_histo.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_histo.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_interval.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_interval.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_param_proc.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_param_proc.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_rand.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_rand.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_reverseComplement.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_reverseComplement.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_skiplist.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_skiplist.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_skiplist_test.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_stackTrace.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_stackTrace.H | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_stackTraceTest.C | Bin 0 -> 280 bytes
src/AS_UTL/._AS_UTL_testhisto.C | Bin 0 -> 280 bytes
src/AS_UTL/._Makefile | Bin 0 -> 280 bytes
src/AS_UTL/._UnionFind_AS.C | Bin 0 -> 280 bytes
src/AS_UTL/._UnionFind_AS.H | Bin 0 -> 280 bytes
src/AS_UTL/._extractMSG | Bin 0 -> 280 bytes
src/AS_UTL/._testHashTable.C | Bin 0 -> 280 bytes
src/AS_UTL/._testRand.C | Bin 0 -> 280 bytes
src/AS_UTL/._testVar.C | Bin 0 -> 280 bytes
src/AS_UTL/AS_UTL_alloc.H | 6 +-
src/AS_UTL/AS_UTL_fileIO.C | 3 +-
src/AS_UTL/AS_UTL_intervalList.C | 510 --
src/AS_UTL/AS_UTL_intervalList.H | 173 -
src/AS_UTL/AS_UTL_intervalListTest.C | 63 -
src/AS_UTL/AS_UTL_splitToWords.H | 139 -
src/AS_UTL/Makefile | 1 -
src/AS_VWR/._Makefile | Bin 0 -> 280 bytes
src/AS_VWR/._agrep.c | Bin 0 -> 280 bytes
src/AS_VWR/._agrep.h | Bin 0 -> 280 bytes
src/AS_VWR/._button.c | Bin 0 -> 280 bytes
src/AS_VWR/._button.h | Bin 0 -> 280 bytes
src/AS_VWR/._celagram.c | Bin 0 -> 280 bytes
src/AS_VWR/._celamy.c | Bin 0 -> 280 bytes
src/AS_VWR/._gendata.c | Bin 0 -> 280 bytes
src/AS_VWR/._hash.c | Bin 0 -> 280 bytes
src/AS_VWR/._hash.h | Bin 0 -> 280 bytes
src/AS_VWR/._layout.c | Bin 0 -> 280 bytes
src/AS_VWR/._layout.h | Bin 0 -> 280 bytes
src/AS_VWR/._layout_uint.c | Bin 0 -> 280 bytes
src/AS_VWR/._nata1 | Bin 0 -> 280 bytes
src/AS_VWR/._nata2.5 | Bin 0 -> 280 bytes
src/AS_VWR/._nata2.7 | Bin 0 -> 280 bytes
src/AS_VWR/._nata3 | Bin 0 -> 280 bytes
src/AS_VWR/._nata4 | Bin 0 -> 280 bytes
src/AS_VWR/._query_celamy.c | Bin 0 -> 280 bytes
src/AS_VWR/._wpack.c | Bin 0 -> 280 bytes
src/AS_VWR/._wpack.h | Bin 0 -> 280 bytes
src/AS_global.C | 2 +-
src/AS_global.H | 64 +-
src/c_make.as | 72 +-
1360 files changed, 27870 insertions(+), 90786 deletions(-)
diff --git a/README b/README
index 8ad2688..bb8499d 100644
--- a/README
+++ b/README
@@ -1,56 +1,34 @@
-These are release notes for Celera Assembler version 8.1, which was released on
-December 16th, 2013.
+These are release notes for Celera Assembler version 8.3rc2, which was released on May 24, 2015.
-This distribution package provides a stable, tested, documented version of the
-software. The distribution is usable on most Unix-like platforms, and some
-platforms have pre-compiled binary distributions ready for installation.
+This distribution package provides a stable, tested, documented version of the software. The distribution is usable on most Unix-like platforms, and some platforms have pre-compiled binary distributions ready for installation.
-The source code package includes full source code (revision r4490), Makefiles,
-and scripts. A subset of the kmer package (http://kmer.sourceforge.net/,
-revision r1969), used by some modules of Celera Assembler, is included.
-
-This package was prepared by scientists at the J. Craig Venter Institute
-(http://www.jcvi.org/) with funding provided by the National Institutes of
-Health (http://www.nih.gov/).
+The source code package includes full source code (revision 4627), Makefiles, and scripts. A subset of the kmer package (http://kmer.sourceforge.net/, version r1994), used by some modules of Celera Assembler, is included. This distribution includes [http://samtools.sourceforge.net/ SAMtools], [http://www.cbcb.umd.edu/software/jellyfish/ Jellyfish 2.0], [https://github.com/pbjd/pbutgcns PBUTGCNS], [https://github.com/PacificBiosciences/pbdagcon PBDAGCON], [https://github.com/PacificBios [...]
Full documentation can be found online at http://wgs-assembler.sourceforge.net/.
Citation
-Please cite Celera Assembler in publications that refer to its algorithm or its
-output. The standard citation is the original paper [Myers et al. (2000) A
-Whole-Genome Assembly of Drosophila. Science 287 2196-2204]. More recent papers
-describe modifications for human genome assembly [Istrail et al. 2004; Levy et
-al. 2007], metagenomics assembly [Venter et al. 2004; Rusch et al. 2007],
-haplotype separation [Levy et al. 2007; Denisov et al. 2008], a
-Sanger+pyrosequencing hybrid pipeline [Goldberg et al. 2006] and native assembly
-of 454 data [Miller et al. 2008]. There are links to these papers, and more, in
-the on-line documentation (http://wgs-assembler.sourceforge.net/).
+Please cite Celera Assembler in publications that refer to its algorithm or its output. The standard citation is the original paper [Myers et al. (2000) A Whole-Genome Assembly of Drosophila. Science 287 2196-2204]. More recent papers describe modifications for human genome assembly [Istrail et al. 2004; Levy et al. 2007], metagenomics assembly [Venter et al. 2004; Rusch et al. 2007], haplotype separation [Levy et al. 2007; Denisov et al. 2008], a Sanger+pyrosequencing hybrid pipeline [G [...]
Compilation and Installation
-Users can download Celera Assembler as source code or as pre-compiled
-binaries. The source code package needs to be compiled and installed before it
-can be used. The binary distributions need only be unpacked, but they are not
-available for all platforms.
+Users can download Celera Assembler as source code or as pre-compiled binaries. The source code package needs to be compiled and installed before it can be used. The binary distributions need only be unpacked, but they are not available for all platforms.
To use the source code, execute these commands on any unix-like platform:
- bzip2 -dc wgs-8.1.tar.bz2 | tar -xf -
- cd wgs-8.1
- cd kmer && make install && cd ..
- cd samtools && make && cd ..
- cd src && make && cd ..
- cd ..
+ bzip2 -dc wgs-8.3rc2.tar.bz2 | tar -xf -
+ cd wgs-8.3rc2
+ cd kmer && make install && cd ..
+ cd src && make && cd ..
+ cd ..
-To use the binary distributions, choose a platform, download that package, then
-unpack it with some unix command like this:
+To use the binary distributions, choose a platform, download that package, then unpack it with some unix command like this:
- bzip2 -dc wgs-8.1-*.tar.bz2 | tar -xf -
+ bzip2 -dc wgs-8.3rc2-*.tar.bz2 | tar -xf -
In both cases, you can run the assembler with:
- wgs-8.1/*/bin/runCA
+ wgs-8.3rc2/*/bin/runCA
Legal
@@ -58,3 +36,740 @@ Copyright 1999-2004 by Applera Corporation. Copyright 2005-2013 by the J. Craig
Venter Institute. The Celera Assembler software, also known as the wgs-assembler
and CABOG, is open-source and available free of charge subject to the GNU
General Public License, version 2.
+
+SAMtools Licence:
+The MIT License
+
+Copyright (c) 2008-2009 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+Jellyfish licence:
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ <program> Copyright (C) <year> <name of author>
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License. But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+
+PBUTGCNS and Falcon license:
+#################################################################################$$
+# Copyright (c) 2011-2014, Pacific Biosciences of California, Inc.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted (subject to the limitations in the
+# disclaimer below) provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+#
+# * Neither the name of Pacific Biosciences nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
+# GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
+# BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#################################################################################
diff --git a/kmer/Make.include b/kmer/Make.include
deleted file mode 100644
index bfef44b..0000000
--- a/kmer/Make.include
+++ /dev/null
@@ -1,30 +0,0 @@
-# -*- makefile -*-
-
-LIBBIO/ :=$(realpath $/libbio/)/
-LIBSEQ/ :=$(realpath $/libseq/)/
-LIBKMER/ :=$(realpath $/libkmer/)/
-LIBMERYL/ :=$(realpath $/libmeryl/)/
-LIBSIM4/ :=$(realpath $/libsim4/)/
-LIBUTL/ :=$(realpath $/libutl/)/
-
-$(eval $(call Include,$/ESTmapper/))
-
-$(eval $(call Include,$/atac-driver/))
-$(eval $(call Include,$/seatac/))
-
-$(eval $(call Include,$/leaff/))
-$(eval $(call Include,$/meryl/))
-$(eval $(call Include,$/seagen/))
-$(eval $(call Include,$/sim4dbutils/))
-$(eval $(call Include,$/sim4db/))
-$(eval $(call Include,$/snapper/))
-$(eval $(call Include,$/tapper/))
-
-$(eval $(call Include,${LIBSIM4/}))
-$(eval $(call Include,${LIBKMER/}))
-$(eval $(call Include,${LIBMERYL/}))
-$(eval $(call Include,${LIBBIO/}))
-$(eval $(call Include,${LIBSEQ/}))
-$(eval $(call Include,${LIBUTL/}))
-
-$/.REAL-CLEAN := $/Make.compilers
diff --git a/kmer/Make.rules b/kmer/Make.rules
deleted file mode 100644
index f904b13..0000000
--- a/kmer/Make.rules
+++ /dev/null
@@ -1,316 +0,0 @@
-# -*- makefile -*-
-
-# this might be useful for some future work if we want to make
-# actions more variable.
-define .Make-rule
-$1:
- $2
-
-endef
-
-############################################################
-# useful functions for the install methods mentioned below
-############################################################
-# $(call .FUN-install-copy,file1 file2 file3,dir1/ dir2/)
-# copies whichever file exists into each of ${INSTALL/}dir
-# mkdir-ing as necessary.
-define .FUN-install-copy
- @ files='$$(strip $1)'; dirs='$$(strip $2)'; \
- if [ -n "$$$${files}" -a -n "$$$${dirs}" ] ; then \
- for F in $$$${files} ; do \
- if [ -f $$$${F} ] ; then \
- for D in $$$${dirs} ; do \
- Fout=$${INSTALL/}$$$${D}`basename $$$${F}` ; \
- mkdir -p `dirname $$$${Fout}` && \
- rm -f $$$${Fout} && cp -fp $$$${F} $$$${Fout} ; \
- done ; \
- fi ; \
- done ; \
- fi
-endef
-# because SOME PLATFORMS (like cygwin) use a special .exe extension
-# in executables, we have to do a little hack here. We assume that
-# Make.compilers has set a variable called .EXE
-define .FUN-install-copy-exe
- @ files='$$(strip $1)'; dirs='$$(strip $2)'; \
- if [ -n "$$$${files}" -a -n "$$$${dirs}" ] ; then \
- for F in $$$${files} ; do \
- if [ "${.EXE}" != "" -a -f $$$${F}${.EXE} ] ; then \
- for D in $$$${dirs} ; do \
- Fout=$${INSTALL/}$$$${D}`basename $$$${F}` ; \
- mkdir -p `dirname $$$${Fout}` && \
- rm -f $$$${Fout}${.EXE} && cp -fp $$$${F}${.EXE} $$$${Fout}${.EXE} ; \
- done ; \
- fi ; \
- if [ -f $$$${F} ] ; then \
- for D in $$$${dirs} ; do \
- Fout=$${INSTALL/}$$$${D}`basename $$$${F}` ; \
- mkdir -p `dirname $$$${Fout}` && \
- rm -f $$$${Fout} && cp -fp $$$${F} $$$${Fout} ; \
- done ; \
- fi ; \
- done ; \
- fi
-endef
-# we do another cygwin inspired hack to deal with that fact that
-# .so shlibs need to be turned into .dll files.
-define .FUN-install-copy-shlib
- @ files='$$(strip $1)'; dirs='$$(strip $2)'; \
- if [ -n "$$$${files}" -a -n "$$$${dirs}" ] ; then \
- for F in $$$${files} ; do \
- if [ -f $$$${F} ] ; then \
- for D in $$$${dirs} ; do \
- Fout=$${INSTALL/}$$$${D}`basename $$$${F} .so`${.SO} ; \
- mkdir -p `dirname $$$${Fout}` && \
- rm -f $$$${Fout} && cp -fp $$$${F} $$$${Fout} ; \
- done ; \
- fi ; \
- done ; \
- fi
-endef
-# use this one for executable scripts with #! substitution
-# echo ":Mangling $$$${F} to $$$${Fout}:" ;
-define .FUN-install-copy-script
- @ files='$$(strip $1)'; dirs='$$(strip $2)'; sheb='$$(strip $3)'; \
- if [ -n "$$$${files}" -a -n "$$$${dirs}" ] ; then \
- for F in $$$${files} ; do \
- if [ -f $$$${F} ] ; then \
- for D in $$$${dirs} ; do \
- Fout=$${INSTALL/}$$$${D}`basename $$$${F}` ; \
- mkdir -p `dirname $$$${Fout}` && \
- rm -f $$$${Fout} && cp -fp $$$${F} $$$${Fout} ; \
- chmod ugo+x $$$${Fout} && \
- ${PERL} -npi \
- -e"if(0==\$$$$i++){s|^#!.*|#!$$$${sheb}|}" $$$${Fout}; \
- done ; \
- fi ; \
- done ; \
- fi
-endef
-
-
-############################################################
-# C and C++ stuff
-############################################################
-# Building depends goals for C/CXX things
-
-# C_SRCS and CXX_SRCS are collected together and turned into
-# associated *.d dependency files.
-
-# WISHLIST: does not propagate failure to the parent make
-# for some reason. It really should. I think the pipe to
-# sed masks the exit code.
-__DEPGOALS__ += $$(patsubst %,%.d,$${${1:.=.C_SRCS}})
-ALL_C_DEPS :=$(foreach x,${//},$(patsubst %,%.d,${${x:.=.C_SRCS}}))
-${ALL_C_DEPS}:%.d:%
- @ echo "making $@"
- @ dir=`echo $< | sed -e's~[^/]*$$~~'`; \
- ${CCDEP} ${CDEPFLAGS} ${CFLAGS} $< | \
- sed -e"/:/s!^!$${dir}!" > $@
-
-__DEPGOALS__ += $$(patsubst %,%.d,$${${1:.=.CXX_SRCS}})
-ALL_CXX_DEPS :=$(foreach x,${//},$(patsubst %,%.d,${${x:.=.CXX_SRCS}}))
-${ALL_CXX_DEPS}:%.d:%
- @ echo "making $@"
- @ dir=`echo $< | sed -e's~[^/]*$$~~'`; \
- ${CXXDEP} ${CXXDEPFLAGS} ${CXXFLAGS} $< | \
- sed -e"/:/s!^!$${dir}!" > $@
-
-
-###### generic pattern rules for subgoals
-# don't want .o's getting deleted as intermediates
-.PRECIOUS: %${.O}
-.SUFFIXES: ${.O}
-
-%${.O}: %.c
- ${-CC} ${CC} ${CFLAGS} ${CFLAGS_COMPILE} -o $@ -c $<
-
-%${.O}: %.cc
- ${-CXX} ${CXX} ${CXXFLAGS} ${CXXFLAGS_COMPILE} -o $@ -c $<
-
-%${.O}: %.cpp
- ${-CXX} ${CXX} ${CXXFLAGS} ${CXXFLAGS_COMPILE} -o $@ -c $<
-
-%${.O}: %.C
- ${-CXX} ${CXX} ${CXXFLAGS} ${CXXFLAGS_COMPILE} -o $@ -c $<
-
-
-# linking commands use the $+ to get duplicated prereqs for linking
-## EXE targets
-ALL_C_EXES :=$(strip $(foreach x,${//},${${x:.=.C_EXES}}))
-${ALL_C_EXES}:
- ${-CC} ${CLD} ${CLDFLAGS} -o $@ $+ ${CLIBS}
-__SUBGOALS__+=$${${1:.=.C_EXES}}
-
-ALL_CXX_EXES :=$(strip $(foreach x,${//},${${x:.=.CXX_EXES}}))
-${ALL_CXX_EXES}:
- ${-CXX} ${CXXLD} ${CXXLDFLAGS} -o $@ $+ ${CXXLIBS}
-__SUBGOALS__+=$${${1:.=.CXX_EXES}}
-
-define .RULE-install-copy-C-CXX-EXES
-${1:.=.install-copy}: ${1:.=.install-copy-C-CXX-EXES}
-${1:.=.install-copy-C-CXX-EXES}:
- $(call .FUN-install-copy-exe,$${${1:.=.C_EXES}} $${${1:.=.CXX_EXES}},bin/)
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-C-CXX-EXES,$x)))
-
-
-## LIB targets
-ALL_C_LIBS :=$(strip $(foreach x,${//},${${x:.=.C_LIBS}}))
-${ALL_C_LIBS}:
- ${-CC} ${RM} $@ && ${AR} ${ARFLAGS} $@ $^
-__SUBGOALS__+=$${${1:.=.C_LIBS}}
-
-ALL_CXX_LIBS :=$(strip $(foreach x,${//},${${x:.=.CXX_LIBS}}))
-${ALL_CXX_LIBS}:
- ${-CXX} ${RM} $@ && ${AR} ${ARFLAGS} $@ $^
-__SUBGOALS__+=$${${1:.=.CXX_LIBS}}
-
-## Shared targets
-# AIX has really weird shared lib building flags. Unfortunately, I could
-# not think of a way out of this hack.
-${_OS_}_SHLIB_FLAGS:=
-ALL_C_SHLIBS :=$(strip $(foreach x,${//},${${x:.=.C_SHLIBS}}))
-${ALL_C_SHLIBS}:
- ${-CC} ${RM} $@ && ${CC} ${CLDFLAGS} ${SHLIB_FLAGS} ${${_OS_}_SHLIB_FLAGS} -o $@ $^ ${CLIBS}
-
-ALL_CXX_SHLIBS :=$(strip $(foreach x,${//},${${x:.=.CXX_SHLIBS}}))
-${ALL_CXX_SHLIBS}:
- ${-CXX} ${RM} $@ && ${CXX} ${CXXLDFLAGS} ${SHLIB_FLAGS} ${${_OS_}_SHLIB_FLAGS} -o $@ $^ ${CXXLIBS}
-__SUBGOALS__+=$${${1:.=.C_SHLIBS}} $${${1:.=.CXX_SHLIBS}}
-
-define .RULE-install-copy-C-CXX-LIBS
-${1:.=.install-copy}: ${1:.=.install-copy-C-CXX-LIBS}
-${1:.=.install-copy-C-CXX-LIBS}:
- $(call .FUN-install-copy,$${${1:.=.C_LIBS}} $${${1:.=.CXX_LIBS}}, lib/$${${1.=.LIB/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-C-CXX-LIBS,$x)))
-
-define .RULE-install-copy-C-CXX-SHLIBS
-${1:.=.install-copy}: ${1:.=.install-copy-C-CXX-SHLIBS}
-${1:.=.install-copy-C-CXX-SHLIBS}:
- $(call .FUN-install-copy-shlib, $${${1:.=.C_SHLIBS}} $${${1:.=.CXX_SHLIBS}}, lib/$${${1.=.LIB/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-C-CXX-SHLIBS,$x)))
-
-
-
-define .RULE-install-copy-C-CXX-INCS
-${1:.=.install-copy}: ${1:.=.install-copy-C-CXX-INCS}
-${1:.=.install-copy-C-CXX-INCS}:
- $(call .FUN-install-copy,$${${1:.=.C_INCS}} $${${1:.=.CXX_INCS}}, include/$${${1:.=.INCLUDE/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-C-CXX-INCS,$x)))
-
-
-
-############################################################
-# latex and some ps/pdf stuff
-############################################################
-
-# I'm not sure if I should mark these as precious or not
-#.PRECIOUS: %.pdf %.ps %.dvi %.aux %.bbl
-
-%.dvi: %.tex
- ${-LATEX} cd `dirname $<` && ${LATEX} `basename $<` && ${LATEX} `basename $<`
-
-%.aux: %.tex
- ${-LATEX} cd `dirname $<` && ${LATEX} `basename $<` && ${LATEX} `basename $<`
-
-%.bbl: %.aux
- ${-LATEX} cd `dirname $<` && ${BIBTEX} `basename ${<:.aux=}`
-
-
-ALL_TEX_PS :=$(strip $(foreach x,${//},${${x:.=.TEX_PS}}))
-ALL_TEX_PDF :=$(strip $(foreach x,${//},${${x:.=.TEX_PDF}}))
-
-${ALL_TEX_PS}: %.ps: %.dvi
- ${-LATEX} cd `dirname $<` && ${DVIPS} -o `basename $@` `basename $<`
-
-${ALL_TEX_PDF}: %.pdf: %.tex %.aux
- ${-LATEX} cd `dirname $<` && ${PDFLATEX} `basename $<` && ${PDFLATEX} `basename $<`
-__SUBGOALS__+=$${${1:.=.TEX_PS}} $${${1:.=.TEX_PDF}}
-
- # install rules
-define .RULE-install-copy-TEX_PSPDF
-${1:.=.install-copy}: ${1:.=.install-copy-TEX_PSPDF}
-${1:.=.install-copy-TEX_PSPDF}:
-# TEX_PS go to doc/
- $(call .FUN-install-copy,$${${1:.=.TEX_PS}},doc/$${${1:.=.DOC/}})
-# TEX_PDF go to doc/
- $(call .FUN-install-copy,$${${1:.=.TEX_PDF}},doc/$${${1:.=.DOC/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-TEX_PSPDF,$x)))
-
-
-
-############################################################
-# Python
-############################################################
-
-# python exes and libs are not subgoals.
-# Otherwise, they'd be deleted when we cleaned.
-# if we ever introduce some notion of 'file prep', beyond
-# adding the she-bang line, then we might want to do something
-# different here.
-#__SUBGOALS__+=$${${1:.=.PY_EXES}} $${${1:.=.PY_LIBS}}
-
-define .RULE-install-copy-PYTHON
-${1:.=.install-copy}: ${1:.=.install-copy-PYTHON}
-${1:.=.install-copy-PYTHON}:
- $(call .FUN-install-copy-script, $${${1:.=.PY_EXES}}, bin/, ${PYTHON} ${PYTHON_FLAGS})
- $(call .FUN-install-copy, $${${1:.=.PY_LIBS}}, lib/$${${1:.=.PY_LIB/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-PYTHON,$x)))
-
-
-############################################################
-# Perl
-############################################################
-# Same sort of spiel as python
-#__SUBGOALS__+=$${${1:.=.PERL_EXES}} $${${1:.=.PERL_LIBS}}
-
-define .RULE-install-copy-PERL
-${1:.=.install-copy}: ${1:.=.install-copy-PERL}
-${1:.=.install-copy-PERL}:
- $(call .FUN-install-copy-script, $${${1:.=.PERL_EXES}}, bin/, ${PERL} ${PERL_FLAGS})
- $(call .FUN-install-copy, $${${1:.=.PERL_LIBS}}, lib/$${${1:.=.PERL_LIB/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-PERL,$x)))
-
-############################################################
-# sh
-############################################################
-# Same sort of spiel as python
-#__SUBGOALS__+=$${${1:.=.SH_EXES}} $${${1:.=.SH_LIBS}}
-
-define .RULE-install-copy-SH
-${1:.=.install-copy}: ${1:.=.install-copy-SH}
-${1:.=.install-copy-SH}:
- $(call .FUN-install-copy-script, $${${1:.=.SH_EXES}}, bin/, ${SH} ${SH_FLAGS})
- $(call .FUN-install-copy, $${${1:.=.SH_LIBS}}, lib/$${${1:.=.SH_LIB/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-SH,$x)))
-
-
-############################################################
-# share -- a random catchall for scripts and whatnot that
-# we should have real rules for but we don't right now
-############################################################
-#__SUBGOALS__+=$${${1:.=.SHARES}}
-
-define .RULE-install-copy-SHARE
-${1:.=.install-copy}: ${1:.=.install-copy-SHARE}
-${1:.=.install-copy-SHARE}:
- $(call .FUN-install-copy,$${${1:.=.SHARES}}, share/$${${1:.=.SHARE/}})
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy-SHARE,$x)))
diff --git a/kmer/Makefile b/kmer/Makefile
deleted file mode 100644
index 822b810..0000000
--- a/kmer/Makefile
+++ /dev/null
@@ -1,194 +0,0 @@
-# -*- makefile -*-
-
-### NOTE: make-3.80 has a bug in eval which causes it not to return
-### an empty target, but instead it sometimes returns very weird
-### stuff that makes make segv, I think.
-### You need to get it patched at:
-### http://savannah.gnu.org/bugs/?func=detailbug&bug_id=1517&group_id=71
-
-default: all
-
-MAKEFILE/ :=$(dir $(firstword $(MAKEFILE_LIST)))
-
-ifneq ($(shell ls -d ${MAKEFILE/}/Make.compilers 2> /dev/null), ${MAKEFILE/}/Make.compilers)
- MAKECOMPILERSNOTHING := $(shell ${MAKEFILE/}configure.sh)
-endif
-
-include ${MAKEFILE/}Make.compilers
-
-
-##### non-recursive make magic
-# all directories which have been included
-// :=
-# current path (empty)
-/ :=
-
-# recursive directories to be filtered out of //
-# and handled differently
-//-RECURSIVE :=
-define MakeRecursive
-//-RECURSIVE :=$$/.
-endef
-
-## Include -hack
-# Include is wrapped in something which will push and pop /
-# properly while adding newly discovered directories to //
-# and keeping track of who is who's children (.SUBS).
-#
-# Each directory so included has its Make.include file included.
-# Those Make.include files can use $/ prepended to local names
-# to prevent name pollution, and define their own subtargets.
-#
-define Include
- $(foreach x,$(strip ${1}),$(call Include_File,$x))
-endef
-
-define Include_File
- ifeq ($(filter ${1}.,${WITHOUT_}),)
- ifeq ($(wildcard ${1}Make.include),${1}Make.include)
- $/.SUBS +=${1}.
- // +=${1}.
- ${1}.SUBS :=
- / :=${1}
- include ${1}Make.include
- / :=$/
- endif
- endif
-
-endef
-
-##### System specific includes
-
-## WITHOUT
-# If the user specifies a WITHOUT, then those paths are not
-# followed.
-ifndef WITHOUT
- WITHOUT:=
-endif
-WITHOUT_:=$(patsubst %,%.,$(strip ${WITHOUT}))
-
-## First Make.include inclusion
-# invoke the toplevel include file.
-# We use 'Include_File' instead of 'Include' since $/ is empty
-$(eval $(call Include_File,$/))
-
-#### Targets which have been declared RECURSIVE are removed
-# from // and processed separately.
-// :=$(filter-out ${//-RECURSIVE},${//})
-
-# //-RECURSIVE now holds the paths which are being done legacy style
-# and // holds the paths which are going to be part of the system.
-
-### Building subgoals in Make.rules
-# subgoals and depends are done with the deferred '=' not the ':='.
-# This is because we want dynamic scoping.
-__SUBGOALS__=
-__DEPGOALS__=
-
--include ${MAKEFILE/}Make.rules
-
-# now we bring in the depends files as defined by the Includes
-# and the patterns in Make.rules
-$(eval DEPENDS:=$(foreach x,${//},$(call __DEPGOALS__,$x)))
-ifneq ($(strip ${DEPENDS}),)
- # this conditional gets us a way out if things go way wrong
- ifeq ($(filter %-clean,${MAKECMDGOALS}),)
- -include ${DEPENDS}
- endif
-endif
-
-### the standard make targets, applied to all subdirectory targets
-# We define the basic form of the all, clean, ... rules on a
-# per-path basis (the $/.all, $..clean, ... targets). This allows
-# selective targeting.
-## rules for each subtarget
-# current subtargets are :
-# all, ls, clean, real-clean, depends-clean, install
-# with all, clean, real-clean, install being required targets
-# of recursive makes.
-
-clean: ${//-RECURSIVE:.=.clean} $/.clean
-define .RULE-clean
-${1:.=.clean}: $${${1:.=.SUBS}:.=.clean}
- ${RM} $${${1:.=.CLEAN}} ${__SUBGOALS__}
-ifneq ($(strip ${C_TMP_COMPILE} ${CXX_TMP_COMPILE}),)
- (cd $1 && ${RM} -r ${C_TMP_COMPILE} ${CXX_TMP_COMPILE})
-endif
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-clean,$x)))
-
-
-depends-clean: $/.depends-clean
-${//-RECURSIVE:.=.depends-clean}:
-define .RULE-depends-clean
-${1:.=.depends-clean}: $${${1:.=.SUBS}:.=.depends-clean}
- ${RM} ${1:.=Make.depends} ${__DEPGOALS__}
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-depends-clean,$x)))
-
-real-clean: ${//-RECURSIVE:.=.real-clean} $/.real-clean
-define .RULE-real-clean
-${1:.=.real-clean}: $${${1:.=.SUBS}:.=.real-clean}
- ${RM} $${${1:.=.CLEAN}} ${__SUBGOALS__} ${1:.=Make.depends} ${__DEPGOALS__} $${${1:.=.REAL-CLEAN}}
-#ifneq ($(strip ${INSTALL/}),)
-# ${RM} -r ${INSTALL/}
-#endif
-ifneq ($(strip ${C_TMP_COMPILE} ${CXX_TMP_COMPILE}),)
- (cd $1 && ${RM} -r ${C_TMP_COMPILE} ${CXX_TMP_COMPILE})
-endif
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-real-clean,$x)))
-
-all: ${//-RECURSIVE:.=.all} $/.all
-define .RULE-all
-${1:.=.all}: $${${1:.=.SUBS}:.=.all} ${__SUBGOALS__}
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-all,$x)))
-
-## INSTALL:
-# Define targets which give a basic recursive traversal to hang
-# things from for anything install related that I haven't thought of
-
-# this does the copy part of the install
-${//-RECURSIVE:.=.install-copy}:
-install-copy: ${//-RECURSIVE:.=.install-copy} $/.install-copy
-define .RULE-install-copy
-${1:.=.install-copy}: $${${1:.=.SUBS}:.=.install-copy}
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install-copy,$x)))
-
-# define the actual install target as being a combination of the
-# all target plus the pre-/install-copy targets
-
-install: ${//-RECURSIVE:.=.install} $/.install
-define .RULE-install
-${1:.=.install}: ${1:.=.all} ${1:.=.install-copy}
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-install,$x)))
-
-
-
-# a sort of debugging thing. Let's me check on which targets
-# actually didn't get made, or got partially made
-LSOPTS:=-l
-ls: $/.ls
-${//-RECURSIVE:.=.ls}:
-define .RULE-ls
-${1:.=.ls}: $${${1:.=.SUBS}:.=.ls}
- @ files='$$(strip ${__SUBGOALS__})'; \
- if [ -n "$$$${files}" ] ; then \
- ls ${LSOPTS} $$$${files} ; \
- fi ; exit 0;
-
-endef
-$(eval $(foreach x,${//},$(call .RULE-ls,$x)))
-
-
-
-
diff --git a/kmer/configure.sh b/kmer/configure.sh
deleted file mode 100755
index 1383579..0000000
--- a/kmer/configure.sh
+++ /dev/null
@@ -1,382 +0,0 @@
-#!/bin/sh
-
-# Set up the build system -- need some symlinks to the build
-# directory.
-#
-if [ ! -e Makefile ] ; then
- if [ -e ../build/Makefile ] ; then
- ln -s ../build/Make.rules .
- ln -s ../build/Makefile .
- elif [ -e build/Makefile ] ; then
- ln -s build/Make.rules .
- ln -s build/Makefile .
- else
- echo "ERROR: Couldn't find the Makefile!"
- exit 1
- fi
-fi
-
-
-# If no target, try to figure out one based on uname. This defaults to
-# the optimized target below. If it works well, we can always use this
-# mechanism, and extend with "debug" or "profile" (e.g., "./configure.sh debug")
-#
-target=$1
-
-if [ "x$target" = "xdebug" ] ; then
- opts="-debug";
- target=""
-fi
-if [ "x$target" = "xprofile" ] ; then
- opts="-profile";
- target=""
-fi
-
-if [ "x$target" = "x" ] ; then
- case `uname` in
- Darwin)
- target="Darwin-i386$opts"
- if [ "`uname -m`" = "Power Macintosh" ] ; then
- target="Darwin-ppc$opts"
- fi
- if [ `uname -m` = "x86_64" ] ; then
- target="Darwin-amd64$opts"
- fi
- ;;
- FreeBSD)
- target="FreeBSD-i386$opts"
- if [ `uname -m` = "amd64" ] ; then
- target="FreeBSD-amd64$opts"
- fi
- ;;
- AIX)
- target="AIX$opts"
- ;;
- OSF1)
- target="OSF1$opts"
- ;;
- Linux)
- target="Linux-i686$opts"
- if [ `uname -m` = "x86_64" ] ; then
- target="Linux-amd64$opts"
- fi
- if [ `uname -m` = "ia64" ] ; then
- target="Linux-ia64$opts"
- fi
- ;;
- SunOS)
- target="solaris$opts"
- ;;
- *)
- echo "ERROR: Unknown uname of `uname` -- try manual configuration."
- exit 1
- ;;
- esac
-fi
-
-
-#
-# Look for the python headers. We don't need the libraries. This is
-# used by atac-driver/chainer only.
-#
-
-PYTHON=${PYTHON:-`which python`}
-
-if [ ! -x $PYTHON ] ; then
- echo "WARNING: Python program not found at '$PYTHON'. Try setting environment variable PYTHON to the location of the python interpreter."
- WITHOUT_ATAC="atac-driver/ seatac/"
-else
- echo "Python executable found in '$PYTHON'"
- CFLAGS_PYTHON=`$PYTHON -c "from distutils import sysconfig; print sysconfig.get_python_inc()"`
-
- if [ -z "$CFLAGS_PYTHON" -o ! -d "$CFLAGS_PYTHON" ] ; then
- echo "WARNING: Python development environment not found."
- WITHOUT_ATAC="atac-driver/ seatac/"
- else
- echo "Python libraries found in '$CFLAGS_PYTHON'"
- CFLAGS_PYTHON="-I$CFLAGS_PYTHON"
- fi
-fi
-
-if [ ! -z "$WITHOUT_ATAC" ] ; then
- echo "WARNING: Will not build ATAC."
-fi
-
-
-#
-# Decide on compilers to use. Unfortunately, all the options are tuned for gcc/g++.
-# In particular, -m64 and -W* and -f* aren't liked by Intel compilers.
-#
-
-if [ x$CC = x ] ; then
- CC="gcc"
-fi
-
-if [ x$CXX = x ] ; then
- CXX="g++"
-fi
-
-#
-# Emit architecture specific configurations.
-#
-
-case $target in
- Darwin-i386|Darwin-amd64)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# OS-X, optimized
-#
-CC := $CC
-SHLIB_FLAGS := -dynamiclib -undefined dynamic_lookup
-CFLAGS_COMPILE := -fast -fPIC -m64 -fmessage-length=0 -D_REENTRANT -D_THREAD_SAFE -Wall -Wno-char-subscripts
-CLDFLAGS := -m64
-CLIBS :=
-CXX := $CXX
-CXXFLAGS_COMPILE := -fast -fPIC -m64 -fmessage-length=0 -D_REENTRANT -D_THREAD_SAFE -Wall -Wno-char-subscripts
-CXXLDFLAGS := -m64
-CXXLIBS :=
-LDFLAGS_PYTHON := -bundle -framework CoreFoundation -framework Python -dynamic
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- Darwin-i386-debug|Darwin-amd64-debug)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# OS-X, debug
-#
-CC := $CC
-SHLIB_FLAGS := -dynamiclib -undefined dynamic_lookup
-CFLAGS_COMPILE := -g3 -m64 -fmessage-length=0 -D_REENTRANT -D_THREAD_SAFE -Wall -Wno-char-subscripts
-CLDFLAGS := -m64
-CLIBS :=
-CXX := $CXX
-CXXFLAGS_COMPILE := -g3 -m64 -fmessage-length=0 -D_REENTRANT -D_THREAD_SAFE -Wall -Wno-char-subscripts
-CXXLDFLAGS := -m64
-CXXLIBS :=
-LDFLAGS_PYTHON := -bundle -framework CoreFoundation -framework Python -dynamic
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- FreeBSD-amd64)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# FreeBSD, optimized
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -O3 -fPIC -pthread -D_REENTRANT -Wall -Wno-char-subscripts -mtune=native -march=native -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -lthr
-CXX := $CXX
-CXXFLAGS_COMPILE := -O3 -fPIC -pthread -D_REENTRANT -Wall -Wno-char-subscripts -mtune=native -march=native -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -lthr
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- FreeBSD-amd64-debug)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# FreeBSD, debug, warnings
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -g -pthread -D_REENTRANT -fPIC -Wall -Wno-char-subscripts -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings -Wconversion -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wnested-externs
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -lthr
-CXX := $CXX
-CXXFLAGS_COMPILE := -g -pthread -D_REENTRANT -fPIC -Wall -Wno-char-subscripts -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings -Wconversion
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -lthr
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- FreeBSD-amd64-profile)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# FreeBSD, debug, warnings
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -pg -O3 -pthread -D_REENTRANT -fPIC -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions
-CLDFLAGS := -pg -L/usr/local/lib
-CLIBS := -pthread -lthr
-CXX := $CXX
-CXXFLAGS_COMPILE := -pg -O3 -pthread -D_REENTRANT -fPIC -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions
-CXXLDFLAGS := -pg -L/usr/local/lib
-CXXLIBS := -pthread -lthr
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- Linux-i686)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# Linux, optimized
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -ldl
-CXX := $CXX
-CXXFLAGS_COMPILE := -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -ldl
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- Linux-amd64)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# Linux64, optimized
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -m64 -fPIC -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -ldl
-CXX := $CXX
-CXXFLAGS_COMPILE := -m64 -fPIC -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -ldl
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- Linux-amd64-debug)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# Linux64, optimized
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -m64 -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_REENTRANT -g -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -ldl
-CXX := $CXX
-CXXFLAGS_COMPILE := -m64 -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_REENTRANT -g -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -ldl
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- Linux-amd64-profile)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# Linux64, optimized
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -pg -m64 -fPIC -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -ldl
-CXX := $CXX
-CXXFLAGS_COMPILE := -pg -m64 -fPIC -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -ldl
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
- Linux-ia64)
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# Linux64, optimized
-CC := $CC
-SHLIB_FLAGS := -shared
-CFLAGS_COMPILE := -m64 -fPIC -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CLDFLAGS := -L/usr/local/lib
-CLIBS := -pthread -ldl
-CXX := $CXX
-CXXFLAGS_COMPILE := -m64 -fPIC -D_REENTRANT -O3 -D_THREAD_SAFE -pthread -fmessage-length=0 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CXXLDFLAGS := -L/usr/local/lib
-CXXLIBS := -pthread -ldl
-ARFLAGS := ruvs
-INSTALL/ := $target/
-EOF
- ;;
-
-
-
- # SUNLF needs to be set to allow for large file support on Solaris. It
- # should be whatever the following getconf's say.
- #
- # getconf LFS_CFLAGS = "-D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
- # getconf LFS_LDFLAGS = ""
- # getconf LFS_LFS_LIBS = ""
- #
- solaris)
- echo "Solaris is UNTESTED!"
- rm -f Make.compilers
- cat <<EOF > Make.compilers
-# -*- makefile -*-
-# Solaris, gcc optimized
-#
-CC := $CC -m64
-SHLIB_FLAGS := -G #untested
-CFLAGS_COMPILE := -D_REENTRANT -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CLDFLAGS :=
-CLIBS := -lpthread -lrt
-CXX := $CXX -m64
-CXXFLAGS_COMPILE := -D_REENTRANT -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -Wall -Wno-char-subscripts -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer
-CXXLDFLAGS :=
-CXXLIBS := -lpthread -lrt
-ARFLAGS := ruv
-INSTALL/ := $target/
-EOF
- ;;
- *)
- echo "usage: $0 <configuration>"
- echo " osx OS-X, optimized"
- echo " osx-debug OS-X, debug"
- echo ""
- echo " freebsd FreeBSD, optimized"
- echo " freebsd-profile FreeBSD, optimized, profiled"
- echo " freebsd-debug FreeBSD, debug and warnings"
- echo ""
- echo " aix AIX, optimized"
- echo " aix-profile AIX, optimized, profiled (NOT TESTED)"
- echo " aix-debug AIX, debug"
- echo ""
- echo " tru64, compaq Tru64, optimized"
- echo " tru64-debug, compaq-debug Tru64, debug, warnings, trapuv"
- echo ""
- echo " linux Linux, i686, optimized"
- echo " linux64 Linux, Opteron, optimized"
- echo ""
- echo " solaris Solaris, gcc, optimized (STALE)"
- exit
- ;;
-esac
-
-
-cat <<EOF >> Make.compilers
-PERL := /usr/bin/perl
-.EXE :=
-.SO := .so
-.A := .a
-.O := .o
-CLD := \${CC}
-CXXLD := \${CXX}
-CCDEP := gcc -MM -MG
-CXXDEP := g++ -MM -MG
-CLIBS += -lm -lbz2
-CXXLIBS += -lm -lbz2
-PYTHON := $PYTHON
-CFLAGS_PYTHON := $CFLAGS_PYTHON
-WITHOUT := $WITHOUT_ATAC
-EOF
-
-echo "Configured."
-
-#cat Make.compilers
diff --git a/kmer/libbio/Make.include b/kmer/libbio/Make.include
deleted file mode 100644
index 581274c..0000000
--- a/kmer/libbio/Make.include
+++ /dev/null
@@ -1,65 +0,0 @@
-# -*- makefile -*-
-
-LIBUTL/ :=$(realpath $/../libutil/)/
-LIBSEQ/ :=$(realpath $/../libseq/)/
-
-src := $/alphabet.c \
- $/alphabet.h \
- $/alphabet-acgtspace.c \
- $/alphabet-colorspace.c \
- $/bio++.H \
- $/bio.h \
- $/halign.c \
- $/kmer.C \
- $/kmer.H \
- $/kmerhuge.H \
- $/kmeriface.H \
- $/kmertiny.H \
- $/merCovering.H \
- $/merList.H \
- $/mers.h \
- $/reversecomplement.c
-
-old := $/fasta-accessor.H \
- $/fasta-c.C \
- $/fasta-c.h \
- $/fasta-cache.C \
- $/fasta-cache.H \
- $/fasta-simple.c \
- $/fasta-simple.h \
- $/fasta.C \
- $/fasta.H \
- $/merstream.C \
- $/merstream.H \
- $/seq.C \
- $/seq.H \
- $/seqFactory.H \
- $/seqFile.H \
- $/seqInCore.H \
- $/seqOnDisk.H \
- $/seqStore.H \
- $/seqStream.H \
- $/sff.H \
- $/sff.C
-
-
-$/.C_SRCS :=$(filter %.c,${src})
-$/.C_INCS :=$(filter %.h,${src})
-$/.CXX_SRCS :=$(filter %.C,${src})
-$/.CXX_INCS :=$(filter %.H,${src})
-$/.CXX_LIBS :=$/libbio.a
-
-$/.CLEAN := $/*.o
-
-$(eval $/%.d $/%.o: CFLAGS += -I${LIBUTL/})
-$(eval $/%.d $/%.o: CXXFLAGS += -I${LIBUTL/})
-
-$/reversecomplement.c.d: $/alphabet.h
-$/merstream.C.d: $/alphabet.h
-
-$/libbio.a: ${$/.C_SRCS:.c=.o} ${$/.CXX_SRCS:.C=.o} $/alphabet.o
-
-$/alphabet.c: $/alphabet.h
-$/alphabet.h: $/alphabet-generate.c $/alphabet-acgtspace.c $/alphabet-colorspace.c
- $(CC) $(CFLAGS) $(CFLAGS_COMPILE) -o `dirname $@`/a.out $<
- cd `dirname $@` ; ./a.out && rm -f ./a.out
diff --git a/kmer/libbio/alphabet-acgtspace.c b/kmer/libbio/alphabet-acgtspace.c
deleted file mode 100644
index b8f73d2..0000000
--- a/kmer/libbio/alphabet-acgtspace.c
+++ /dev/null
@@ -1,149 +0,0 @@
-#include <stdio.h>
-#include <ctype.h>
-#include "alphabet.h"
-
-void
-initCompressionTablesForACGTSpace(void) {
- int i, j;
-
- for (i=0; i<256; i++) {
- whitespaceSymbol[i] = isspace(i) ? 1 : 0;
- toLower[i] = tolower(i);
- toUpper[i] = toupper(i);
- letterToBits[i] = (unsigned char)0xff;
- bitsToLetter[i] = (unsigned char)'?';
- bitsToColor[i] = (unsigned char)'?';
- complementSymbol[i] = (unsigned char)'?';
- }
-
- for (i=0; i<128; i++)
- for (j=0; j<128; j++)
- IUPACidentity[i][j] = 0;
-
- letterToBits['a'] = letterToBits['A'] = (unsigned char)0x00;
- letterToBits['c'] = letterToBits['C'] = (unsigned char)0x01;
- letterToBits['g'] = letterToBits['G'] = (unsigned char)0x02;
- letterToBits['t'] = letterToBits['T'] = (unsigned char)0x03;
-
- letterToBits['0'] = (unsigned char)0x00;
- letterToBits['1'] = (unsigned char)0x01;
- letterToBits['2'] = (unsigned char)0x02;
- letterToBits['3'] = (unsigned char)0x03;
-
- bitsToLetter[0x00] = 'A';
- bitsToLetter[0x01] = 'C';
- bitsToLetter[0x02] = 'G';
- bitsToLetter[0x03] = 'T';
-
- bitsToColor[0x00] = '0';
- bitsToColor[0x01] = '1';
- bitsToColor[0x02] = '2';
- bitsToColor[0x03] = '3';
-
- complementSymbol['a'] = 't'; // a
- complementSymbol['t'] = 'a'; // t
- complementSymbol['u'] = 'a'; // u, Really, only for RNA
- complementSymbol['g'] = 'c'; // g
- complementSymbol['c'] = 'g'; // c
- complementSymbol['y'] = 'r'; // c t
- complementSymbol['r'] = 'y'; // a g
- complementSymbol['s'] = 'w'; // g c
- complementSymbol['w'] = 's'; // a t
- complementSymbol['k'] = 'm'; // t/u g
- complementSymbol['m'] = 'k'; // a c
- complementSymbol['b'] = 'v'; // c g t
- complementSymbol['d'] = 'h'; // a g t
- complementSymbol['h'] = 'd'; // a c t
- complementSymbol['v'] = 'b'; // a c g
- complementSymbol['n'] = 'n'; // a c g t
-
- complementSymbol['A'] = 'T'; // a
- complementSymbol['T'] = 'A'; // t
- complementSymbol['U'] = 'A'; // u, Really, only for RNA
- complementSymbol['G'] = 'C'; // g
- complementSymbol['C'] = 'G'; // c
- complementSymbol['Y'] = 'R'; // c t
- complementSymbol['R'] = 'Y'; // a g
- complementSymbol['S'] = 'W'; // g c
- complementSymbol['W'] = 'S'; // a t
- complementSymbol['K'] = 'M'; // t/u g
- complementSymbol['M'] = 'K'; // a c
- complementSymbol['B'] = 'V'; // c g t
- complementSymbol['D'] = 'H'; // a g t
- complementSymbol['H'] = 'D'; // a c t
- complementSymbol['V'] = 'B'; // a c g
- complementSymbol['N'] = 'N'; // a c g t
-
- complementSymbol['0'] = '0'; // ColorSpace is self-complementing
- complementSymbol['1'] = '1';
- complementSymbol['2'] = '2';
- complementSymbol['3'] = '3';
-
- IUPACidentity['A']['A'] = 1;
- IUPACidentity['C']['C'] = 1;
- IUPACidentity['G']['G'] = 1;
- IUPACidentity['T']['T'] = 1;
- IUPACidentity['M']['A'] = 1;
- IUPACidentity['M']['C'] = 1;
- IUPACidentity['R']['A'] = 1;
- IUPACidentity['R']['G'] = 1;
- IUPACidentity['W']['A'] = 1;
- IUPACidentity['W']['T'] = 1;
- IUPACidentity['S']['C'] = 1;
- IUPACidentity['S']['G'] = 1;
- IUPACidentity['Y']['C'] = 1;
- IUPACidentity['Y']['T'] = 1;
- IUPACidentity['K']['G'] = 1;
- IUPACidentity['K']['T'] = 1;
- IUPACidentity['V']['A'] = 1;
- IUPACidentity['V']['C'] = 1;
- IUPACidentity['V']['G'] = 1;
- IUPACidentity['H']['A'] = 1;
- IUPACidentity['H']['C'] = 1;
- IUPACidentity['H']['T'] = 1;
- IUPACidentity['D']['A'] = 1;
- IUPACidentity['D']['G'] = 1;
- IUPACidentity['D']['T'] = 1;
- IUPACidentity['B']['C'] = 1;
- IUPACidentity['B']['G'] = 1;
- IUPACidentity['B']['T'] = 1;
-
- IUPACidentity['N']['A'] = 1;
- IUPACidentity['N']['C'] = 1;
- IUPACidentity['N']['G'] = 1;
- IUPACidentity['N']['T'] = 1;
-
- IUPACidentity['M']['M'] = 1;
- IUPACidentity['R']['R'] = 1;
- IUPACidentity['W']['W'] = 1;
- IUPACidentity['S']['S'] = 1;
- IUPACidentity['Y']['Y'] = 1;
- IUPACidentity['K']['K'] = 1;
- IUPACidentity['V']['V'] = 1;
- IUPACidentity['H']['W'] = 1;
- IUPACidentity['D']['D'] = 1;
- IUPACidentity['B']['B'] = 1;
- IUPACidentity['N']['N'] = 1;
-
- // Order isn't important
- //
- for (i='A'; i<'Z'; i++)
- for (j='A'; j<'Z'; j++) {
- if (IUPACidentity[j][i])
- IUPACidentity[i][j] = 1;
- }
-
- // Case isn't important
- //
- for (i='A'; i<'Z'; i++)
- for (j='A'; j<'Z'; j++) {
- if (IUPACidentity[j][i]) {
- IUPACidentity[tolower(i)][tolower(j)] = 1;
- IUPACidentity[tolower(i)][j ] = 1;
- IUPACidentity[i ][tolower(j)] = 1;
- }
- }
-}
-
-
-
diff --git a/kmer/libbio/alphabet-colorspace.c b/kmer/libbio/alphabet-colorspace.c
deleted file mode 100644
index 98d28f6..0000000
--- a/kmer/libbio/alphabet-colorspace.c
+++ /dev/null
@@ -1,121 +0,0 @@
-#include <stdio.h>
-#include <ctype.h>
-#include "alphabet.h"
-
-void
-initCompressionTablesForColorSpace(void) {
- int i, j;
-
- for (i=0; i<128; i++)
- for (j=0; j<128; j++)
- baseToColor[i][j] = '.'; // Invalid
-
- // Supports transforming a base sequence to a color sequence.
-
- // Not sure how valid this is; treat every letter like it's a gap.
- // We then override ACGT to be the correct encoding.
- for (i='a'; i<='z'; i++) {
- baseToColor['a'][i] = '4';
- baseToColor['c'][i] = '4';
- baseToColor['g'][i] = '4';
- baseToColor['t'][i] = '4';
- baseToColor['n'][i] = '4';
- }
- for (i='a'; i<='z'; i++) {
- baseToColor[i]['a'] = '0';
- baseToColor[i]['c'] = '1';
- baseToColor[i]['g'] = '2';
- baseToColor[i]['t'] = '3';
- baseToColor[i]['n'] = '4';
- }
-
- baseToColor['a']['a'] = '0';
- baseToColor['a']['c'] = '1';
- baseToColor['a']['g'] = '2';
- baseToColor['a']['t'] = '3';
- baseToColor['a']['n'] = '4';
-
- baseToColor['c']['a'] = '1';
- baseToColor['c']['c'] = '0';
- baseToColor['c']['g'] = '3';
- baseToColor['c']['t'] = '2';
- baseToColor['c']['n'] = '4';
-
- baseToColor['g']['a'] = '2';
- baseToColor['g']['c'] = '3';
- baseToColor['g']['g'] = '0';
- baseToColor['g']['t'] = '1';
- baseToColor['g']['n'] = '4';
-
- baseToColor['t']['a'] = '3';
- baseToColor['t']['c'] = '2';
- baseToColor['t']['g'] = '1';
- baseToColor['t']['t'] = '0';
- baseToColor['t']['n'] = '4';
-
- for (i='a'; i<='z'; i++)
- for (j='a'; j<='z'; j++) {
- baseToColor[toupper(i)][toupper(j)] = baseToColor[i][j];
- baseToColor[tolower(i)][toupper(j)] = baseToColor[i][j];
- baseToColor[toupper(i)][tolower(j)] = baseToColor[i][j];
- baseToColor[tolower(i)][tolower(j)] = baseToColor[i][j];
- }
-
- // Supports composing colors
-
- baseToColor['0']['0'] = '0';
- baseToColor['0']['1'] = '1';
- baseToColor['0']['2'] = '2';
- baseToColor['0']['3'] = '3';
- baseToColor['0']['4'] = '4';
-
- baseToColor['1']['0'] = '1';
- baseToColor['1']['1'] = '0';
- baseToColor['1']['2'] = '3';
- baseToColor['1']['3'] = '2';
- baseToColor['1']['4'] = '4';
-
- baseToColor['2']['0'] = '2';
- baseToColor['2']['1'] = '3';
- baseToColor['2']['2'] = '0';
- baseToColor['2']['3'] = '1';
- baseToColor['2']['4'] = '4';
-
- baseToColor['3']['0'] = '3';
- baseToColor['3']['1'] = '2';
- baseToColor['3']['2'] = '1';
- baseToColor['3']['3'] = '0';
- baseToColor['3']['4'] = '4';
-
- // Supports transforming color sequence to base sequence.
-
- baseToColor['a']['0'] = baseToColor['A']['0'] = 'a';
- baseToColor['a']['1'] = baseToColor['A']['1'] = 'c';
- baseToColor['a']['2'] = baseToColor['A']['2'] = 'g';
- baseToColor['a']['3'] = baseToColor['A']['3'] = 't';
- baseToColor['a']['4'] = baseToColor['A']['4'] = 'n';
-
- baseToColor['c']['0'] = baseToColor['C']['0'] = 'c';
- baseToColor['c']['1'] = baseToColor['C']['1'] = 'a';
- baseToColor['c']['2'] = baseToColor['C']['2'] = 't';
- baseToColor['c']['3'] = baseToColor['C']['3'] = 'g';
- baseToColor['c']['4'] = baseToColor['C']['4'] = 'n';
-
- baseToColor['g']['0'] = baseToColor['G']['0'] = 'g';
- baseToColor['g']['1'] = baseToColor['G']['1'] = 't';
- baseToColor['g']['2'] = baseToColor['G']['2'] = 'a';
- baseToColor['g']['3'] = baseToColor['G']['3'] = 'c';
- baseToColor['g']['4'] = baseToColor['G']['4'] = 'n';
-
- baseToColor['t']['0'] = baseToColor['T']['0'] = 't';
- baseToColor['t']['1'] = baseToColor['T']['1'] = 'g';
- baseToColor['t']['2'] = baseToColor['T']['2'] = 'c';
- baseToColor['t']['3'] = baseToColor['T']['3'] = 'a';
- baseToColor['t']['4'] = baseToColor['T']['4'] = 'n';
-
- baseToColor['n']['0'] = baseToColor['N']['0'] = 'a';
- baseToColor['n']['1'] = baseToColor['N']['1'] = 'c';
- baseToColor['n']['2'] = baseToColor['N']['2'] = 'g';
- baseToColor['n']['3'] = baseToColor['N']['3'] = 't';
- baseToColor['n']['4'] = baseToColor['N']['4'] = 'n';
-}
diff --git a/kmer/libbio/alphabet-generate.c b/kmer/libbio/alphabet-generate.c
deleted file mode 100644
index 20b2dc2..0000000
--- a/kmer/libbio/alphabet-generate.c
+++ /dev/null
@@ -1,134 +0,0 @@
-#include <stdio.h>
-#include <ctype.h>
-
-// Instead of forcing client applications to explicitly call
-// initCompressionTables(), static tables are now generated.
-
-unsigned char whitespaceSymbol[256];
-unsigned char toLower[256];
-unsigned char toUpper[256];
-
-unsigned char letterToBits[256];
-unsigned char bitsToLetter[256];
-unsigned char bitsToColor[256];
-
-unsigned char complementSymbol[256];
-unsigned char validCompressedSymbol[256];
-
-unsigned char IUPACidentity[128][128];
-unsigned char baseToColor[128][128];
-
-void initCompressionTablesForACGTSpace(void);
-void initCompressionTablesForColorSpace(void);
-
-#include "alphabet-acgtspace.c"
-#include "alphabet-colorspace.c"
-
-int
-main(int argc, char **argv) {
- int i, j;
-
- FILE *C = fopen("alphabet.c", "w");
- FILE *H = fopen("alphabet.h", "w");
-
- initCompressionTablesForACGTSpace();
- initCompressionTablesForColorSpace();
-
- fprintf(H, "//\n");
- fprintf(H, "// Automagically generated -- DO NOT EDIT!\n");
- fprintf(H, "// See libbri/alphabet-generate.c for details.\n");
- fprintf(H, "//\n");
- fprintf(H, "\n");
- fprintf(H, "#ifdef __cplusplus\n");
- fprintf(H, "extern \"C\" {\n");
- fprintf(H, "#endif\n");
- fprintf(H, "\n");
-
- fprintf(C, "//\n");
- fprintf(C, "// Automagically generated -- DO NOT EDIT!\n");
- fprintf(C, "// See %s for details.\n", __FILE__);
- fprintf(C, "//\n");
-
- fprintf(H, "extern unsigned char whitespaceSymbol[256];\n");
- fprintf(C, "unsigned char whitespaceSymbol[256] = { %d", whitespaceSymbol[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", whitespaceSymbol[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char toLower[256];\n");
- fprintf(C, "unsigned char toLower[256] = { %d", toLower[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", toLower[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char toUpper[256];\n");
- fprintf(C, "unsigned char toUpper[256] = { %d", toUpper[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", toUpper[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char letterToBits[256];\n");
- fprintf(C, "unsigned char letterToBits[256] = { %d", letterToBits[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", letterToBits[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char bitsToLetter[256];\n");
- fprintf(C, "unsigned char bitsToLetter[256] = { %d", bitsToLetter[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", bitsToLetter[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char bitsToColor[256];\n");
- fprintf(C, "unsigned char bitsToColor[256] = { %d", bitsToColor[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", bitsToColor[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char complementSymbol[256];\n");
- fprintf(C, "unsigned char complementSymbol[256] = { %d", complementSymbol[0]);
- for (i=1; i<256; i++)
- fprintf(C, ",%d", complementSymbol[i]);
- fprintf(C, " };\n");
-
- fprintf(H, "extern unsigned char IUPACidentity[128][128];\n");
- fprintf(C, "unsigned char IUPACidentity[128][128] = {\n");
- for (i=0; i<128; i++) {
- fprintf(C, " {");
- if (IUPACidentity[i][0])
- fprintf(C, "1");
- else
- fprintf(C, "0");
- for (j=1;j<128; j++) {
- if (IUPACidentity[i][j])
- fprintf(C, ",1");
- else
- fprintf(C, ",0");
- }
- fprintf(C, "},\n");
- }
- fprintf(C, "};\n");
-
-
- fprintf(H, "extern unsigned char baseToColor[128][128];\n");
- fprintf(C, "unsigned char baseToColor[128][128] = {\n");
- for (i=0; i<128; i++) {
- fprintf(C, " {%d", baseToColor[i][0]);
- for (j=1;j<128; j++)
- fprintf(C, ",%d", baseToColor[i][j]);
- fprintf(C, "},\n");
- }
- fprintf(C, "};\n");
-
-
- fprintf(H, "\n");
- fprintf(H, "void initCompressionTablesForACGTSpace(void);\n");
- fprintf(H, "void initCompressionTablesForColorSpace(void);\n");
-
- fprintf(H, "\n");
- fprintf(H, "#ifdef __cplusplus\n");
- fprintf(H, "}\n");
- fprintf(H, "#endif\n");
-
- return(0);
-}
diff --git a/kmer/libbio/alphabet.c b/kmer/libbio/alphabet.c
deleted file mode 100644
index 2fbe1e0..0000000
--- a/kmer/libbio/alphabet.c
+++ /dev/null
@@ -1,271 +0,0 @@
-//
-// Automagically generated -- DO NOT EDIT!
-// See ../kmer/libbio/alphabet-generate.c for details.
-//
-unsigned char whitespaceSymbol[256] = { 0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, [...]
-unsigned char toLower[256] = { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,13 [...]
-unsigned char toUpper[256] = { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147, [...]
-unsigned char letterToBits[256] = { 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,1,2,3,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,1,255,255,255,2,255,255,255,255,255,255,255,255,255,255,255,255,3,255,255,255,255,255,255,255,255,255,255,255,255,0,255,1,255,255,255,2,255,255,255,255,255,255,255,255,255,255,255,255,3,255,255,25 [...]
-unsigned char bitsToLetter[256] = { 65,67,71,84,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,6 [...]
-unsigned char bitsToColor[256] = { 48,49,50,51,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63 [...]
-unsigned char complementSymbol[256] = { 63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,48,49,50,51,63,63,63,63,63,63,63,63,63,63,63,63,63,84,86,71,72,63,63,67,68,63,63,77,63,75,78,63,63,63,89,87,65,65,66,83,63,82,63,63,63,63,63,63,63,116,118,103,104,63,63,99,100,63,63,109,63,107,110,63,63,63,121,119,97,97,98,115,63,114,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63, [...]
-unsigned char IUPACidentity[128][128] = {
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
- {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-};
-unsigned char baseToColor[128][128] = {
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,49,50,51,52,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,49,48,51,50,52,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,50,51,48,49,52,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,51,50,49,48,52,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,97,99,103,116,110,46,46,46,46,46,46,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,99,97,116,103,110,46,46,46,46,46,46,46,46,46,46,46,46,49,52,48,52,52,52,51,52,52,52,52,52,52,52,52,52,52,52,52,50,52,52,52,52,52,52,46,46,46,46,46,46,49,52,48,52,52,52,51,52,52,52,52,52,52,52,52,52,52,52,52,50,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,103,116,97,99,110,46,46,46,46,46,46,46,46,46,46,46,46,50,52,51,52,52,52,48,52,52,52,52,52,52,52,52,52,52,52,52,49,52,52,52,52,52,52,46,46,46,46,46,46,50,52,51,52,52,52,48,52,52,52,52,52,52,52,52,52,52,52,52,49,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,97,99,103,116,110,46,46,46,46,46,46,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,116,103,99,97,110,46,46,46,46,46,46,46,46,46,46,46,46,51,52,50,52,52,52,49,52,52,52,52,52,52,52,52,52,52,52,52,48,52,52,52,52,52,52,46,46,46,46,46,46,51,52,50,52,52,52,49,52,52,52,52,52,52,52,52,52,52,52,52,48,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,97,99,103,116,110,46,46,46,46,46,46,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,99,97,116,103,110,46,46,46,46,46,46,46,46,46,46,46,46,49,52,48,52,52,52,51,52,52,52,52,52,52,52,52,52,52,52,52,50,52,52,52,52,52,52,46,46,46,46,46,46,49,52,48,52,52,52,51,52,52,52,52,52,52,52,52,52,52,52,52,50,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,103,116,97,99,110,46,46,46,46,46,46,46,46,46,46,46,46,50,52,51,52,52,52,48,52,52,52,52,52,52,52,52,52,52,52,52,49,52,52,52,52,52,52,46,46,46,46,46,46,50,52,51,52,52,52,48,52,52,52,52,52,52,52,52,52,52,52,52,49,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,97,99,103,116,110,46,46,46,46,46,46,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46,46,48,52,49,52,52,52,50,52,52,52,52,52,52,52,52,52,52,52,52,51,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,116,103,99,97,110,46,46,46,46,46,46,46,46,46,46,46,46,51,52,50,52,52,52,49,52,52,52,52,52,52,52,52,52,52,52,52,48,52,52,52,52,52,52,46,46,46,46,46,46,51,52,50,52,52,52,49,52,52,52,52,52,52,52,52,52,52,52,52,48,52,52,52,52,52,52,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46,46,48,46,49,46,46,46,50,46,46,46,46,46,46,52,46,46,46,46,46,51,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
- {46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46},
-};
diff --git a/kmer/libbio/alphabet.h b/kmer/libbio/alphabet.h
deleted file mode 100644
index 4243a9a..0000000
--- a/kmer/libbio/alphabet.h
+++ /dev/null
@@ -1,25 +0,0 @@
-//
-// Automagically generated -- DO NOT EDIT!
-// See libbri/alphabet-generate.c for details.
-//
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern unsigned char whitespaceSymbol[256];
-extern unsigned char toLower[256];
-extern unsigned char toUpper[256];
-extern unsigned char letterToBits[256];
-extern unsigned char bitsToLetter[256];
-extern unsigned char bitsToColor[256];
-extern unsigned char complementSymbol[256];
-extern unsigned char IUPACidentity[128][128];
-extern unsigned char baseToColor[128][128];
-
-void initCompressionTablesForACGTSpace(void);
-void initCompressionTablesForColorSpace(void);
-
-#ifdef __cplusplus
-}
-#endif
diff --git a/kmer/libbio/bio++.H b/kmer/libbio/bio++.H
deleted file mode 100644
index ceeca6c..0000000
--- a/kmer/libbio/bio++.H
+++ /dev/null
@@ -1,23 +0,0 @@
-#ifndef BIO_PLUS_PLUS_H
-#define BIO_PLUS_PLUS_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <string.h>
-#include <ctype.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include "bio.h"
-#include "util++.H"
-
-#include "mers.h"
-#include "kmer.H"
-#include "merCovering.H"
-#include "merList.H"
-
-#endif // BIO_PLUS_PLUS_H
-
diff --git a/kmer/libbio/bio.h b/kmer/libbio/bio.h
deleted file mode 100644
index 809d203..0000000
--- a/kmer/libbio/bio.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#ifndef BIO_H
-#define BIO_H
-
-#include "util.h"
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-////////////////////////////////////////
-//
-// alphabet
-//
-#include "alphabet.h"
-
-
-////////////////////////////////////////
-//
-// reversecomplement.c
-//
-char *reverseComplementSequence(char *seq, u32bit seqlen);
-char *reverseString(char *seq, u32bit seqlen);
-
-
-// halign
-//
-// N.B. align() (aka halign) was switched over to palloc() -- this
-// fixed any memory leaks, and gives a 30%-ish speed increase. This
-// is thread safe (unless someone breaks palloc2()).
-//
-void
-halign(const char *string1,
- const char *string2,
- const int len1,
- const int len2,
- char *alnline1,
- char *alnline2);
-
-
-#ifdef __cplusplus
-}
-#endif
-
-
-#endif // BIO_H
diff --git a/kmer/libbio/halign.c b/kmer/libbio/halign.c
deleted file mode 100644
index 86aeb89..0000000
--- a/kmer/libbio/halign.c
+++ /dev/null
@@ -1,467 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <limits.h>
-
-#include "bio.h"
-
-// Liliana Florea's halign (a sim4-derivitive).
-
-#define DEL 0
-#define INS 1
-#define SUB 2
-
-#ifdef min
-#undef min
-#endif
-#define min(x,y) ((x)<=(y) ? (x):(y))
-
-#ifdef max
-#undef max
-#endif
-#define max(x,y) ((x)>=(y) ? (x):(y))
-
-
-typedef struct edit_script {
- int op_type; /* SUB, INS or DEL */
- int num; /* Number of operations */
- struct edit_script *next;
-} edit_script;
-
-typedef struct edit_script_list {
- int offset1, offset2;
- int len1, len2;
- int score;
- int first;
- edit_script *script;
-} edit_script_list;
-
-
-
-static
-int
-snake(const char *seq1, const char *seq2, int k, int x, int endx, int endy) {
- int y;
-
- if (x<0) return x;
- y = x+k;
- while ((x < endx) &&
- (y < endy) &&
- (toUpper[seq1[x]] == toUpper[seq2[y]])) {
- ++x;
- ++y;
- }
- return x;
-}
-
-
-static
-int
-rsnake(const char *seq1, const char *seq2, int k, int x, int startx, int starty, int M, int N) {
- int y;
-
- if (x>M) return x;
- if ((startx<0) || (starty<0))
- fprintf(stderr, "halign::rsnake()-- TROUBLE!!! startx: %5d, starty: %5d\n",startx, starty);
- if ((x>M) || (x+k>N))
- fprintf(stderr, "halign::rsnake()-- TROUBLE!!! x: %5d, y: %5d\n",x,x+k);
-
- y = x+k;
- while ((x>startx) &&
- (y>starty) &&
- (toUpper[seq1[x-1]] == toUpper[seq2[y-1]])) {
- --x;
- --y;
- }
- return x;
-}
-
-
-static
-int
-align_get_dist(const char *seq1,
- const char *seq2,
- int i1, int j1,
- int i2, int j2,
- int limit,
- void *ph) {
- int *last_d, *temp_d;
- int goal_diag, ll, uu;
- int c, k, row;
- int start;
- int lower, upper;
-
- /* Compute the boundary diagonals */
- start = j1 - i1;
- lower = max(j1-i2, start-limit);
- upper = min(j2-i1, start+limit);
- goal_diag = j2-i2;
-
- if (goal_diag > upper || goal_diag < lower) {
- fprintf(stderr, "The two sequences are not really similar.\n");
- fprintf(stderr, "Please try an exact aligning method.\n");
- exit(1);
- }
-
- /* Allocate space for forward vectors */
- last_d = (int *)palloc2((upper-lower+1)*sizeof(int), ph) - lower;
- temp_d = (int *)palloc2((upper-lower+1)*sizeof(int), ph) - lower;
-
- /* Initialization */
- for (k=lower; k<=upper; ++k) last_d[k] = INT_MIN;
- last_d[start] = snake(seq1, seq2, start, i1, i2, j2);
-
- if (last_d[goal_diag] >= i2)
- return 0;
-
- for (c=1; c<=limit; ++c) {
- ll = max(lower,start-c); uu = min(upper, start+c);
- for (k=ll; k<=uu; ++k) {
- if (k == ll)
- row = last_d[k+1]+1; /* DELETE */
- else if (k == uu)
- row = last_d[k-1]; /* INSERT */
- else if ((last_d[k]>=last_d[k+1]) &&
- (last_d[k]+1>=last_d[k-1]))
- row = last_d[k]+1; /*SUBSTITUTE */
- else if ((last_d[k+1]+1>=last_d[k-1]) &&
- (last_d[k+1]>=last_d[k]))
- row = last_d[k+1]+1; /* DELETE */
- else
- row = last_d[k-1]; /* INSERT */
-
- temp_d[k] = snake(seq1,seq2,k,row,i2,j2);
- }
-
- for (k=ll; k<=uu; ++k) last_d[k] = temp_d[k];
-
- if (last_d[goal_diag] >= i2)
- return c;
- }
-
- /* Ran out of distance limit */
- return -1;
-}
-
-
-static
-void
-align_path(const char *seq1,
- const char *seq2,
- int i1, int j1,
- int i2, int j2,
- int dist,
- edit_script **head,
- edit_script **tail,
- int M,
- int N,
- void *ph) {
-
- int *last_d, *temp_d; /* forward vectors */
- int *rlast_d, *rtemp_d; /* backward vectors */
-
- edit_script *head1, *tail1, *head2, *tail2;
- int midc, rmidc;
- int start;
- int lower, upper;
- int rstart, rlower, rupper;
- int c, k, row;
- int mi, mj, tmp, ll, uu;
- char flag;
-
- *head = *tail = NULL;
-
- /* Boundary cases */
- if (i1 == i2) {
- if (j1 == j2) *head = NULL;
- else {
- head1 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head1->op_type = INS;
- head1->num = j2-j1;
- head1->next = NULL;
- *head = *tail = head1;
- }
- return;
- }
-
- if (j1 == j2) {
- head1 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head1->op_type = DEL;
- head1->num = i2-i1;
- head1->next = NULL;
- *head = *tail = head1;
- return;
- }
-
- if (dist <= 1) {
- start = j1-i1;
- if (j2-i2 == j1-i1) {
- head1 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head1->op_type = SUB;
- head1->num = i2-i1;
- head1->next = NULL;
- *head = *tail = head1;
- } else if (j2-j1 == i2-i1+1) {
-
- tmp = snake(seq1,seq2,start,i1,i2,j2);
- if (tmp>i1) {
- head1 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head1->op_type = SUB;
- head1->num = tmp-i1;
- *head = head1;
- }
- head2 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head2->op_type = INS;
- head2->num = 1;
-
- if (*head) head1->next = head2;
- else *head = head2;
- *tail = head2;
- head2->next = NULL;
-
- if (i2-tmp) {
- head1 = head2;
- *tail = head2 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head2->op_type = SUB;
- head2->num = i2-tmp;
- head2->next = NULL;
- head1->next = head2;
- }
- } else if (j2-j1+1 == i2-i1) {
-
- tmp = snake(seq1,seq2,start,i1,i2,j2);
- if (tmp>i1) {
- head1 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head1->op_type = SUB;
- head1->num = tmp-i1;
- *head = head1;
- }
- head2 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head2->op_type = DEL;
- head2->num = 1;
-
- if (*head) head1->next = head2;
- else *head = head2;
- *tail = head2;
- head2->next = NULL;
-
- if (i2>tmp+1) {
- head1 = head2;
- *tail = head2 = (edit_script *)palloc2(sizeof(edit_script), ph);
- head2->op_type = SUB;
- head2->num = i2-tmp-1;
- head2->next = NULL;
- head1->next = head2;
- }
- } else {
- fprintf(stderr, "halign::align_path()-- warning: something wrong when aligning.");
- }
- return;
- }
-
- /* Divide the problem at the middle cost */
- midc = dist/2;
- rmidc = dist - midc;
-
- /* Compute the boundary diagonals */
- start = j1 - i1;
- lower = max(j1-i2, start-midc);
- upper = min(j2-i1, start+midc);
- rstart = j2-i2;
- rlower = max(j1-i2, rstart-rmidc);
- rupper = min(j2-i1, rstart+rmidc);
-
- /* Allocate space for forward vectors */
- last_d = (int *)palloc2((upper-lower+1)*sizeof(int), ph) - lower;
- temp_d = (int *)palloc2((upper-lower+1)*sizeof(int), ph) - lower;
-
- for (k=lower; k<=upper; k++) last_d[k] = -1;
- last_d[start] = snake(seq1,seq2,start,i1,i2,j2);
-
- /* Forward computation */
- for (c=1; c<=midc; ++c) {
- ll = max(lower,start-c);
- uu = min(upper,start+c);
- for (k=ll; k<=uu; ++k) {
- if (k == ll) {
- /* DELETE : down from (k+1,c-1) */
- row = last_d[k+1]+1;
- } else if (k == uu) {
- /* INSERT : right from (k-1,c-1) */
- row = last_d[k-1];
- } else if ((last_d[k]>=last_d[k+1]) &&
- (last_d[k]+1>=last_d[k-1])) {
- /* SUBSTITUTE */
- row = last_d[k]+1;
- } else if ((last_d[k+1]+1>=last_d[k-1]) &&
- (last_d[k+1]>=last_d[k])) {
- /* DELETE */
- row = last_d[k+1]+1;
- } else {
- /* INSERT */
- row = last_d[k-1];
- }
-
- temp_d[k] = snake(seq1,seq2,k,row,i2,j2);
- }
- for (k=ll; k<=uu; ++k)
- last_d[k] = temp_d[k];
- }
-
- /* Allocate space for backward vectors */
- rlast_d = (int *)palloc2((rupper-rlower+1)*sizeof(int), ph) - rlower;
- rtemp_d = (int *)palloc2((rupper-rlower+1)*sizeof(int), ph) - rlower;
-
- for (k=rlower; k<=rupper; k++) rlast_d[k] = i2+1;
- rlast_d[rstart] = rsnake(seq1,seq2,rstart,i2,i1,j1,M,N);
-
- /* Backward computation */
- for (c=1; c<=rmidc; ++c) {
- ll = max(rlower,rstart-c);
- uu = min(rupper,rstart+c);
- for (k=ll; k<=uu; ++k) {
- if (k == ll) {
- /* INSERT : left from (k+1,c-1) */
- row = rlast_d[k+1];
- } else if (k == uu) {
- /* DELETE : up from (k-1,c-1) */
- row = rlast_d[k-1]-1;
- } else if ((rlast_d[k]-1<=rlast_d[k+1]) &&
- (rlast_d[k]-1<=rlast_d[k-1]-1)) {
- /* SUBSTITUTE */
- row = rlast_d[k]-1;
- } else if ((rlast_d[k-1]-1<=rlast_d[k+1]) &&
- (rlast_d[k-1]-1<=rlast_d[k]-1)) {
- /* DELETE */
- row = rlast_d[k-1]-1;
- } else {
- /* INSERT */
- row = rlast_d[k+1];
- }
-
- rtemp_d[k] = rsnake(seq1,seq2,k,row,i1,j1,M,N);
- }
- for (k=ll; k<=uu; ++k)
- rlast_d[k] = rtemp_d[k];
- }
-
- /* Find (mi, mj) such that the distance from (i1, j1) to (mi, mj) is
- midc and the distance from (mi, mj) to (i2, j2) is rmidc.
- */
-
- flag = 0;
- mi = i1; mj = j1;
- ll = max(lower,rlower);
- uu = min(upper,rupper);
-
- for (k=ll; k<=uu; ++k) {
- if (last_d[k]>=rlast_d[k]) {
- if (last_d[k]-i1>=i2-rlast_d[k]) {
- mi = last_d[k]; mj = k+mi;
- } else {
- mi = rlast_d[k]; mj = k+mi;
- }
- flag = 1;
-
- break;
- }
- }
-
- if (flag) {
- /* Find a path from (i1,j1) to (mi,mj) */
- align_path(seq1,seq2,i1,j1,mi,mj,midc,&head1,&tail1,M,N,ph);
-
- /* Find a path from (mi,mj) to (i2,j2) */
- align_path(seq1,seq2,mi,mj,i2,j2,rmidc,&head2,&tail2,M,N,ph);
-
- /* Join these two paths together */
- if (head1) tail1->next = head2;
- else head1 = head2;
- } else {
- fprintf(stderr, "halign::align_path()-- warning: something wrong when dividing\n");
- head1 = NULL;
- }
- *head = head1;
- if (head2) *tail = tail2;
- else *tail = tail1;
-}
-
-
-
-
-
-void
-halign(const char *seq1,
- const char *seq2,
- const int len1,
- const int len2,
- char *alnline1,
- char *alnline2) {
- edit_script *head, *tail, *tp;
- int i;
- void *ph;
-
- ph = pallochandle(0);
-
- align_path(seq1, seq2,
- 0, 0,
- len1, len2,
- align_get_dist(seq1, seq2, 0, 0, len1, len2, len1+len2, ph),
- &head, &tail,
- len1, len2,
- ph);
-
- /* generate the alignment(s) */
-
- *alnline1 = 0;
- *alnline2 = 0;
-
- for (tp=head; tp; tp=tp->next) {
- switch (tp->op_type) {
- case SUB:
- for (i=0; i<tp->num; i++) {
- if (toUpper[*seq1] == toUpper[*seq2]) {
- *alnline1 = toLower[*seq1];
- *alnline2 = toLower[*seq2];
- } else {
- *alnline1 = toUpper[*seq1];
- *alnline2 = toUpper[*seq2];
- }
- seq1++;
- seq2++;
- alnline1++;
- alnline2++;
- }
- break;
-
- case INS:
- for (i=0; i<tp->num; i++) {
- *alnline1 = '-';
- *alnline2 = toUpper[*seq2];
- seq2++;
- alnline1++;
- alnline2++;
- }
- break;
-
- case DEL:
- for (i=0; i<tp->num; i++) {
- *alnline2 = '-';
- *alnline1 = toUpper[*seq1];
- seq1++;
- alnline1++;
- alnline2++;
- }
- break;
-
- default:
- fprintf(stderr, "halign::halign()-- unrecognized op_type in script. %d\n", tp->op_type);
- exit(0);
- }
- }
- *alnline1 = 0;
- *alnline2 = 0;
-
- pfree2(ph);
- pfreehandle(ph);
-}
diff --git a/kmer/libbio/kmer.C b/kmer/libbio/kmer.C
deleted file mode 100644
index c938dd4..0000000
--- a/kmer/libbio/kmer.C
+++ /dev/null
@@ -1,497 +0,0 @@
-#include "kmer.H"
-
-kMerBuilder::kMerBuilder(u32bit ms, u32bit cm, char *tm) {
- _style = 0;
-
- _merSize = 0;
- _merSizeValid = 0L;
- _merSizeValidIs = 0;
- _merSizeValidZero = 0;
-
- _merStorage = 0L;
- _fMer = 0L;
- _rMer = 0L;
-
- _compression = 0;
- _compressionIndex = 0;
- _compressionFirstIndex = 0;
- _compressionLength = 0L;
- _compressionCurrentLength = 0;
-
- _templateSpan = 0;
- _templateLength = 0;
- _template = 0L;
- _templatePos = 0;
- _templateMer = 0;
- _templateFirst = 0;
-
- if (ms) {
- _style = 0;
- _merSize = ms;
- _merSizeValidIs = _merSize + _merSize;
- _merSizeValidZero = _merSize;
- }
-
- if (cm) {
- _style = 1;
-
- _merSize = ms;
- _merSizeValidIs = _merSize + _merSize;
- _merSizeValidZero = _merSize;
-
- _compression = cm;
- _compressionIndex = 0;
- _compressionFirstIndex = 0;
- _compressionLength = 0L;
- _compressionCurrentLength = 0;
-
- assert(_compression < _merSize);
- }
-
- if (tm) {
- _style = 2;
- _merSize = 0;
- _templateSpan = strlen(tm);
- _templateLength = 0;
- _template = new char [_templateSpan + 1];
- _templatePos = 0;
- _templateMer = 0;
- _templateFirst = 1;
-
- // Templates cannot begin or end in zero -- they MUST begin/end
- // with a letter. We silently fix these problems. Unless there
- // are no 1's in the string, then we bail.
-
- u32bit i=0, t=0;
- while ((i < _templateSpan) && (tm[i] == '0'))
- i++;
-
- if (i == _templateSpan) {
- fprintf(stderr, "invalid kMerBuilder template '%s' -- its empty!\n", tm);
- exit(1);
- }
-
- while (i < _templateSpan) {
- _template[t] = 0;
-
- if (tm[i] == '1') {
- _template[t] = 1;
- _merSize++;
- }
-
- i++;
- t++;
- }
-
- while (_template[--t] == 0)
- ;
-
- _templateSpan = t + 1;
-
-#ifdef DEBUGSPACE
- for (i=0; i<_templateSpan; i++)
- fprintf(stderr, "%d", _template[i]);
- fprintf(stderr, " -- %d\n", _templateSpan);
-#endif
-
- // Look for patterns in the template, set templateLength to be the
- // size of the pattern.
-
- _templateLength = _templateSpan;
-
- // Finally, we can set what valid and zero mersizes are.
- _merSizeValidIs = _templateLength + _merSize;
- _merSizeValidZero = _templateLength;
- }
-
- if (cm && tm) {
- _style = 3;
- assert(0);
- }
-
-
- if (_merSize > KMER_WORDS * 32)
- fprintf(stderr, "kMer size too large; increase KMER_WORDS in libbio/kmer.H\n"), exit(1);
-
-
- _compressionLength = new u32bit [_merSize];
-
- for (u32bit z=0; z<_merSize; z++)
- _compressionLength[z] = (cm) ? 0 : 1;
-
-
-
- if (tm) {
- _merStorage = new kMer [_templateLength * 2];
- _merSizeValid = new u32bit [_templateLength];
-
- for (u32bit i=0; i<2*_templateLength; i++) {
- _merStorage[i].setMerSize(_merSize);
- _merStorage[i].setMerSpan(_templateSpan);
- }
-
- // VERY IMPORTANT! Offset the valid length to adjust for the
- // template that every mer except the first is starting in the
- // middle of.
- //
- for (u32bit i=0; i<_templateLength; i++)
- _merSizeValid[i] = _merSize - i;
- } else {
- _merStorage = new kMer [2];
- _merSizeValid = new u32bit [1];
-
- _merStorage[0].setMerSize(_merSize);
- _merStorage[1].setMerSize(_merSize);
-
- _merSizeValid[0] = _merSizeValidZero;
-
- if (cm) {
- _merStorage[0].setMerSpan(0);
- _merStorage[1].setMerSpan(0);
- }
- }
-
- _fMer = _merStorage + 0;
- _rMer = _merStorage + 1;
-}
-
-
-
-kMerBuilder::~kMerBuilder() {
- delete [] _merSizeValid;
- delete [] _merStorage;
- delete [] _compressionLength;
- delete [] _template;
-}
-
-
-
-void
-kMerBuilder::clear(bool clearMer) {
-
- // Contiguous mers
- _merSizeValid[0] = _merSizeValidZero;
-
- // Compressed mers
- if (_compression) {
- _compressionIndex = 0;
- _compressionFirstIndex = 0;
- _compressionCurrentLength = 0;
-
- for (u32bit z=0; z<_merSize; z++)
- _compressionLength[z] = 0;
-
- _merStorage[0].setMerSpan(0);
- _merStorage[1].setMerSpan(0);
- }
-
- // Spaced mers
- if (_template) {
- for (u32bit i=0; i<2*_templateLength; i++)
- _merStorage[i].clear();
-
- for (u32bit i=0; i<_templateLength; i++)
- _merSizeValid[i] = _merSize - i;
-
- _templatePos = 0;
- _templateMer = 0;
- _templateFirst = 1;
- }
-
- if (clearMer) {
- _fMer->clear();
- _rMer->clear();
- }
-}
-
-
-
-
-//
-// The addBase methods add a single base (cf - forward, cr - complemented) to
-// the mer. The return true if another base is needed to finish the mer, and
-// false if the mer is complete.
-//
-
-
-
-
-
-bool
-kMerBuilder::addBaseContiguous(u64bit cf, u64bit cr) {
-
- // Not a valid base, reset the mer to empty, and request more bases
- // (this is a slightly optimized version of clear()).
- if (cf & (unsigned char)0xfc) {
- clear(false);
- //_merSizeValid[0] = _merSizeValidZero;
- return(true);
- }
-
- // Add the base to both mers.
- *_fMer += cf;
- *_rMer -= cr;
-
- // If there aren't enough bases, request another one.
- if (_merSizeValid[0] + 1 < _merSizeValidIs) {
- _merSizeValid[0]++;
- return(true);
- }
-
- return(false); // Good! Don't need another letter.
-}
-
-
-
-
-
-
-bool
-kMerBuilder::addBaseCompressed(u64bit cf, u64bit cr) {
-
- // Not a valid base, reset the mer to empty, and request more bases.
- //
- if (cf & (unsigned char)0xfc) {
- clear();
- return(true);
- }
-
- u64bit lb = theFMer().endOfMer(2); // Last base in the mer
- u32bit ms = theFMer().getMerSpan(); // Span BEFORE adding the mer
-
- if (_merSizeValid[0] <= _merSizeValidZero)
- lb = 9; // No valid last base (should probably be ~u64bitZERO, but that screws up diagnostic output)
-
-#ifdef DEBUGCOMP
- fprintf(stderr, "kMerBuilder::addBaseCompressed()-- lb="u64bitFMT" cf="u64bitFMT" ms="u32bitFMT" ccl="u32bitFMT" lvl="u32bitFMT"\n",
- lb, cf, ms, _compressionCurrentLength, _compression);
-#endif
-
- // Always add one to the current length. When we started, it
- // was 0. This represents the length AFTER adding the base.
- //
- _compressionCurrentLength++;
-
- // If the lastbase is the same as the one we want to add (and
- // there IS a last base), and we've seen too many of these,
- // remember we've seen another letter in the run, and don't add
- // it. Request another letter.
- //
- if ((lb == cf) && // last is the same as this
- (_compressionCurrentLength > _compression)) { // run is already too big
- _compressionLength[_compressionIndex]++;
-
- _fMer->setMerSpan(ms + 1);
- _rMer->setMerSpan(ms + 1);
-
-#ifdef DEBUGCOMP
- fprintf(stderr, "kMerBuilder::addBaseCompressed()-- COMPRESSED currentIdx=%u first=%u",
- _compressionIndex, _compressionFirstIndex);
- for (u32bit x=0, y=_compressionFirstIndex; x<_merSize; x++) {
- fprintf(stderr, " %u(%d)", _compressionLength[y], y);
- y = (y + 1) % _merSize;
- }
- fprintf(stderr, "\n");
-#endif
- return(true);
- }
-
- // Else, it's a new run (a different letter) or our run isn't
- // big enough to compress and we need to add the duplicate
- // letter.
-
- *_fMer += cf;
- *_rMer -= cr;
-
- // If this is a new letter, propagate the current length to the first letter in this run. That
- // way, when that letter is popped off the mer, we automagically update our span to include only
- // as many letters as are here.
- //
- // 01234567890
- //
- // E.g. For sequence TATTTTTTAGT (that's 6 T's) with a mersize of 3 and compression 2, we'd have
- // mers with position:
- //
- // TATTTTTTAGT
- // #1 TAT position 0 (with lengths 1, 1, 1) uncompressed mer TAT
- // #2 ATT position 1 (with lengths 1, 1, 1) ATT
- // #3 TTA position 6 (with lengths 5, 1, 1) TTTTTTA
- // #4 TAG position 7 TAG
- // #5 AGT position 8 AGT
- //
- // In #2, because the length so far (1) is not >= the compression (2) we add a new base and
- // return.
- //
- // In #3, the current length is >= the compression, so we keep stuffing on T's and incrementing
- // the last length, stopping when we get the A. We now propagate the current length to the first
- // letter in the run. Special case, if the first letter in the run is the first letter in the
- // mer, we need to immediately update the span.
-
-#ifdef DEBUGCOMP
- fprintf(stderr, "kMerBuilder::addBaseCompressed()-- ADDNEWBASE currentIdx=%u first=%u",
- _compressionIndex, _compressionFirstIndex);
- for (u32bit x=0, y=_compressionFirstIndex; x<_merSize; x++) {
- fprintf(stderr, " %u(%d)", _compressionLength[y], y);
- y = (y + 1) % _merSize;
- }
- fprintf(stderr, "\n");
-#endif
-
- // If we added a new letter, transfer the run-length count to the first letter in the previous
- // run. In the above example, when we built the run, the lengths are (1, 1, 5). That is, all
- // compression occurred on the last letter. When we shift off that first letter, we want to
- // remove as much of the run as possible.
-
- if (lb != cf) {
- if (_compressionFirstIndex != _compressionIndex) {
- _compressionLength[_compressionFirstIndex] += _compressionLength[_compressionIndex] - 1;
- _compressionLength[_compressionIndex] = 1;
- }
- _compressionFirstIndex = (_compressionIndex + 1) % _merSize;
- _compressionCurrentLength = 1;
- }
-
- _compressionIndex = (_compressionIndex + 1) % _merSize;
- ms -= _compressionLength[_compressionIndex]; // subtract the count for the letter we just shifted out
-
-#ifdef DEBUGCOMP
- fprintf(stderr, "kMerBuilder::addBaseCompressed()-- ADDNEWBASE shifted out at idx="u32bitFMT" with "u32bitFMT" positions; final span "u32bitFMT"\n",
- _compressionIndex,
- _compressionLength[_compressionIndex],
- ms + 1);
-#endif
-
- _compressionLength[_compressionIndex] = 1; // one letter at this position
-
- _fMer->setMerSpan(ms + 1);
- _rMer->setMerSpan(ms + 1);
-
- // If there aren't enough bases, request another one.
- if (_merSizeValid[0] + 1 < _merSizeValidIs) {
- _merSizeValid[0]++;
- return(true);
- }
-
- return(false); // Good! Don't need another letter.
-}
-
-
-
-
-
-
-bool
-kMerBuilder::addBaseSpaced(u64bit cf, u64bit cr) {
-#ifdef DEBUGSPACE
- fprintf(stderr, "add %c templatePos=%u templateMer=%u\n", ch, _templatePos, _templateMer);
-#endif
-
- // We always advance the templatePos, unfortunately, we need to
- // use the current value throughout this function. If there
- // was a single return point, we could advance immediately
- // before returning.
- //
- u32bit tp = _templatePos;
- _templatePos = (_templatePos + 1) % _templateLength;
-
- // If we get an invalid letter, set all mers that would have
- // had a letter added to be broken.
- //
- if (cf & (unsigned char)0xfc) {
-
- for (u32bit m=0; m<_templateLength; m++) {
- u32bit tppos = (tp + _templateLength - m) % _templateLength;
-
- if (_template[tppos] == 1) {
-
- // Reset to 'zero', but make it skip over any remaining
- // positions in the current template.
- //
- _merSizeValid[m] = _merSizeValidZero + tppos - _templateLength + 1;
-
-#ifdef DEBUGSPACE
- fprintf(stderr, "-- invalid letter, reset mer %u to valid %u (mersizevalidzero=%u ttpos=%u templatelength=%u)\n",
- m, _merSizeValid[m], _merSizeValidZero, tppos, _templateLength);
-#endif
- }
- }
-
- if (_templateFirst == 0)
- _templateMer = (_templateMer + 1) % _templateLength;
-
- return(true);
- }
-
- // We have a valid letter, and add it to all the mers that the
- // template allows.
- //
- for (u32bit m=0; m<_templateLength; m++) {
- u32bit tppos = (tp + _templateLength - m) % _templateLength;
-
- if (_template[tppos] == 1) {
- _merStorage[2*m+0] += cf;
- _merStorage[2*m+1] -= cr;
-
- if (_merSizeValid[m] < _merSizeValidIs)
- _merSizeValid[m]++;
-
-#ifdef DEBUGSPACE
- fprintf(stderr, "push %c onto %d (at template %u) length = %u %s\n",
- ch, m, (tp + _templateLength - m) % _templateLength,
- _merSizeValid[m],
- (_merSizeValid[m] >= _merSizeValidIs) ? "complete" : "");
-#endif
- } else if (_merSizeValid[m] <= _merSizeValidZero) {
-
- // The template doesn't want us to add a letter to the mer,
- // but we're adjusting for an aborted template, and we're
- // counting template positions (not just non-zero template
- // positions) when adjusting.
- //
- _merSizeValid[m]++;
- }
- }
-
- // If the current mer isn't long enough, we move to the next mer,
- // and request another letter.
- //
- if (_merSizeValid[_templateMer] < _merSizeValidIs) {
- if (_templateFirst == 0)
- _templateMer = (_templateMer + 1) % _templateLength;
-#ifdef DEBUGSPACE
- fprintf(stderr, "-- too short -- need more templateMer=%u templateFirst=%u\n", _templateMer, _templateFirst);
-#endif
- return(true);
- }
-
- // On startup, _templateMer is always 0 (the first mer) until
- // it is long enough to be a valid mer. Then, we clear
- // _templateFirst so that we can start advancing through mers.
-
- // Update the f and r pointers to the correct mers, advance our
- // template to the next, and terminate.
- //
- _fMer = _merStorage + 2 * _templateMer + 0;
- _rMer = _merStorage + 2 * _templateMer + 1;
-
-#ifdef DEBUGSPACE
- fprintf(stderr, "-- valid! (templateMer = %u)\n", _templateMer);
-#endif
-
- _templateFirst = 0;
- _templateMer = (_templateMer + 1) % _templateLength;
-
- return(false); // Good! Don't need another letter.
-}
-
-
-
-
-
-
-bool
-kMerBuilder::addBaseCompressedSpaced(u64bit cf, u64bit cr) {
- fprintf(stderr, "kMerBuilder::addBaseCompressedSpace()-- Compressed and spaced mers not supported.\n");
- exit(1);
-}
-
diff --git a/kmer/libbio/kmer.H b/kmer/libbio/kmer.H
deleted file mode 100644
index 517598e..0000000
--- a/kmer/libbio/kmer.H
+++ /dev/null
@@ -1,160 +0,0 @@
-// Copyright (c) 2005 J. Craig Venter Institute
-// Author: Brian Walenz
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received (LICENSE.txt) a copy of the GNU General Public
-// License along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-// A 'simple' kMer datastructure.
-
-#ifndef BIO_KMER_H
-#define BIO_KMER_H
-
-// The maximum size of a mer. You get 32 bases per word, so
-// KMER_WORDS=4 will get you up to a 128-mer.
-//
-#define KMER_WORDS 1
-
-#include "util.h"
-#include "util++.H"
-#include "bio.h"
-#include "bio++.H"
-#include "kmeriface.H"
-
-#if KMER_WORDS == 1
-#include "kmertiny.H"
-typedef kMerTiny kMer;
-#else
-#include "kmerhuge.H"
-typedef kMerHuge kMer;
-#endif
-
-
-#undef DEBUGADDBASE
-#undef DEBUGCOMP
-#undef DEBUGSPACE
-
-
-class kMerBuilder {
-public:
- kMerBuilder(u32bit ms=0, u32bit cm=0, char *tm=0L);
- ~kMerBuilder();
-
- // Clear all mer data, reset state to as just after construction.
- void clear(bool clearMer=true);
-
- // Returns true if we need another base to finish the mer. This
- // only occurs for compressed mers, if we are in a homopolymer run.
- //
-private:
- bool addBaseContiguous(u64bit cf, u64bit cr);
- bool addBaseCompressed(u64bit cf, u64bit cr);
- bool addBaseSpaced(u64bit cf, u64bit cr);
- bool addBaseCompressedSpaced(u64bit cf, u64bit cr);
-
-public:
- bool addBase(char ch) {
- u64bit cf = letterToBits[ch];
- u64bit cr = letterToBits[complementSymbol[ch]];
-
-#ifdef DEBUGADDBASE
- fprintf(stderr, "addBase() %c\n", ch);
-#endif
-
- if (_style == 0)
- return(addBaseContiguous(cf, cr));
-
- if (_style == 1)
- return(addBaseCompressed(cf, cr));
-
- if (_style == 2)
- return(addBaseSpaced(cf, cr));
-
- if (_style == 3)
- return(addBaseCompressedSpaced(cf, cr));
-
- fprintf(stderr, "kMerBuilder::addBase()-- Invalid mer type %d.\n", _style);
- exit(1);
-
- return(false);
- }
-
- void mask(void) {
- _fMer->mask(true);
- _rMer->mask(false);
- };
-
- kMer const &theFMer(void) { return(*_fMer); };
- kMer const &theRMer(void) { return(*_rMer); };
- kMer const &theCMer(void) { return((theFMer() < theRMer()) ? theFMer() : theRMer()); };
-
- u32bit merSize(void) { return(_merSize); };
- u32bit templateSpan(void) { return(_templateSpan); };
-
- u32bit baseSpan(u32bit b) {
- return(_compressionLength[(_compressionIndex + 1 + b) % _merSize]);;
- };
-
-private:
-
- // Style of builder we are
- u32bit _style;
-
- // Amount of the mer that has valid sequence. Sigh. I really needed a signed value here --
- // where negative values mean that we first have to get to the end of the template that was
- // invalid, then we need to build a new mer.
- //
- // And, yes, just simply making it signed leads to all sortes of compiler warnings about
- // comparing signed and unsigned. And I've been here before, and those warnings just propate
- // endlessly. Just go away, Mr. Smartypants.
- //
- // Details: when building spaced seeds, if we hit an N in the middle of the template, we need to
- // invalidate the mer, but not start building a new mer until we exhaust the current template.
- // The example is template=1101. Suppose we hit an N at the second 1. We set the merSizeValid
- // to 0, and proceed. When we push on the base for the last 1 in the template, we'd increment
- // the merSizeValid. The first two 1's in the template would now create a mer big enough to be
- // valid, and we'd return it -- but now the template we're using is 0111.
- //
- // _merSizeValid is offset by _merSize (e.g., the true valid size is _merSizeValid - _merSize).
- // _merSizeValidIs is the size _merSizeValid needs to be in order for it to be valid.
- // Similarily, _merSizeValidZero is the value of zero (currently this is equal to _merSize).
- //
- u32bit _merSize; // Desired number of bases in the mer
- u32bit *_merSizeValid; // Actual number of bases in the mer
- u32bit _merSizeValidZero; // Definition of 'zero' bases in the mer
- u32bit _merSizeValidIs; // Definition of 'full' bases in the mer
-
- // An array of mers, we allocate all mers in one block
- kMer *_merStorage;
-
- // Pointer to the currently active mer
- kMer *_fMer;
- kMer *_rMer;
-
- // For compression
- u32bit _compression;
- u32bit _compressionIndex; // index into cL[] that is the last base in the mer
- u32bit _compressionFirstIndex; // index into cL[] that is the first base in a run
- u32bit *_compressionLength; // one per base
- u32bit _compressionCurrentLength;
-
- // For templates
- u32bit _templateSpan; // # of 0's and 1's in the template
- u32bit _templateLength; // length of the pattern in the template
- char *_template; // character string template
- u32bit _templatePos; // position we are building in the template
- u32bit _templateMer; // the mer we should output next
- u32bit _templateFirst; // if true, we're still building the initial mer
-};
-
-#endif // BIO_KMER_H
diff --git a/kmer/libbio/kmerhuge.H b/kmer/libbio/kmerhuge.H
deleted file mode 100644
index e26ab47..0000000
--- a/kmer/libbio/kmerhuge.H
+++ /dev/null
@@ -1,396 +0,0 @@
-
-#define MERWORD(N) _md[N]
-
-
-class kMerHuge {
-public:
- kMerHuge(u32bit ms=u32bitZERO) {
- setMerSize(ms);
- clear();
- };
- ~kMerHuge() {
- };
-
- void setMerSize(u32bit ms);
- u32bit getMerSize(void) const { return(_merSize); };
-
- void setMerSpan(u32bit ms) { _merSpan = ms; };
- u32bit getMerSpan(void) const { return(_merSpan); };
-
- kMerHuge &reverseComplement(void) {
- for (u32bit i=0, j=KMER_WORDS-1; i<KMER_WORDS/2; i++, j--) {
- u64bit t = MERWORD(i);
- MERWORD(i) = MERWORD(j);
- MERWORD(j) = t;
- }
-
- for (u32bit i=0; i<KMER_WORDS; i++) {
- MERWORD(i) = ((MERWORD(i) >> 2) & 0x3333333333333333llu) | ((MERWORD(i) << 2) & 0xccccccccccccccccllu);
- MERWORD(i) = ((MERWORD(i) >> 4) & 0x0f0f0f0f0f0f0f0fllu) | ((MERWORD(i) << 4) & 0xf0f0f0f0f0f0f0f0llu);
- MERWORD(i) = ((MERWORD(i) >> 8) & 0x00ff00ff00ff00ffllu) | ((MERWORD(i) << 8) & 0xff00ff00ff00ff00llu);
- MERWORD(i) = ((MERWORD(i) >> 16) & 0x0000ffff0000ffffllu) | ((MERWORD(i) << 16) & 0xffff0000ffff0000llu);
- MERWORD(i) = ((MERWORD(i) >> 32) & 0x00000000ffffffffllu) | ((MERWORD(i) << 32) & 0xffffffff00000000llu);
- MERWORD(i) ^= 0xffffffffffffffffllu;
- }
-
- *this >>= KMER_WORDS * 64 - 2 * _merSize;
-
- return(*this);
- };
-
-
- void clear(void) {
- for (u32bit i=0; i<KMER_WORDS; i++)
- MERWORD(i) = u64bitZERO;
- };
- void smallest(void) {
- clear();
- };
- void largest(void) {
- clear();
- reverseComplement();
- };
-
-private:
- void operator>>=(u32bit x) {
-
- // thisWord, the word we shift bits into
- // thatWord, the word we shift bits out of
- // shift, the number of bits we shift
- //
- u32bit thisWord = 0;
- u32bit thatWord = x >> 6;
- u32bit shift = x & u32bitMASK(6);
-
- // Do an initial word-size shift, to reduce the shift amount to
- // be less than wordsize. Fill any shifted-out words with zero.
- //
- if (thatWord) {
- while (thatWord < KMER_WORDS)
- MERWORD(thisWord++) = MERWORD(thatWord++);
- while (thisWord < KMER_WORDS)
- MERWORD(thisWord++) = 0;
- }
-
- // Do bit-size shift, of adjacent words
- //
- thisWord = 0;
- thatWord = 1;
- MERWORD(thisWord) >>= shift;
- while (thatWord < KMER_WORDS) {
- MERWORD(thisWord++) |= MERWORD(thatWord) << (64 - shift);
- MERWORD(thatWord++) >>= shift;
- }
- };
-
- void operator<<=(u32bit x) {
- u32bit thisWord = KMER_WORDS;
- u32bit thatWord = KMER_WORDS - (x >> 6);
- u32bit shift = x & u32bitMASK(6);
-
- if (thatWord != KMER_WORDS) {
- while (thatWord > 0)
- MERWORD(--thisWord) = MERWORD(--thatWord);
- while (thisWord > 0)
- MERWORD(--thisWord) = 0;
- }
-
- thisWord = KMER_WORDS;
- thatWord = KMER_WORDS - 1;
- MERWORD(thisWord-1) <<= shift;
- while (thatWord > 0) {
- --thisWord;
- --thatWord;
- MERWORD(thisWord) |= MERWORD(thatWord) >> (64 - shift);
- MERWORD(thatWord) <<= shift;
- }
- };
-
-
-public:
- void operator+=(u64bit x) {
- *this <<= 2;
- assert((x & 0xfc) == 0);
- MERWORD(0) |= x & u64bitNUMBER(0x3);
- };
- void operator-=(u64bit x) {
- *this >>= 2;
- assert((x & 0xfc) == 0);
- MERWORD(_lastWord) |= (x & u64bitNUMBER(0x3)) << _lastShift;
- };
-
- void mask(bool full) {
- MERWORD(_maskWord) &= _mask;
- if (full)
- for (u32bit x=_maskWord+1; x<KMER_WORDS; x++)
- MERWORD(x) = u64bitZERO;
- };
-
-public:
- bool operator!=(kMerHuge const &r) const {
- u64bit res = u64bitZERO;
- for (u32bit i=KMER_WORDS; i--; )
- res |= MERWORD(i) ^ r.MERWORD(i);
- return(res != u64bitZERO);
- };
- bool operator==(kMerHuge const &r) const {
- u64bit res = u64bitZERO;
- for (u32bit i=KMER_WORDS; i--; )
- res |= MERWORD(i) ^ r.MERWORD(i);
- return(res == u64bitZERO);
- };
-
- bool operator<(kMerHuge const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (MERWORD(i) < r.MERWORD(i)) return(true);
- if (MERWORD(i) > r.MERWORD(i)) return(false);
- }
- return(false);
- };
- bool operator>(kMerHuge const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (MERWORD(i) > r.MERWORD(i)) return(true);
- if (MERWORD(i) < r.MERWORD(i)) return(false);
- }
- return(false);
- };
- bool operator<=(kMerHuge const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (MERWORD(i) < r.MERWORD(i)) return(true);
- if (MERWORD(i) > r.MERWORD(i)) return(false);
- }
- return(true);
- };
- bool operator>=(kMerHuge const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (MERWORD(i) > r.MERWORD(i)) return(true);
- if (MERWORD(i) < r.MERWORD(i)) return(false);
- }
- return(true);
- };
- int qsort_less(kMerHuge const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (MERWORD(i) < r.MERWORD(i)) return(-1);
- if (MERWORD(i) > r.MERWORD(i)) return(1);
- }
- return(0);
- };
-
-
-public:
- operator u64bit () const {return(MERWORD(0));};
-
-
-public:
- // these should work generically for both big and small
-
- void writeToBitPackedFile(bitPackedFile *BPF, u32bit numBits=0) const {
- if (numBits == 0)
- numBits = _merSize << 1;
-
- u32bit lastWord = numBits >> 6;
-
- if ((numBits & u32bitMASK(6)) == 0)
- lastWord++;
-
- if (numBits & u32bitMASK(6))
- BPF->putBits(MERWORD(lastWord), numBits & u32bitMASK(6));
- while (lastWord > 0) {
- lastWord--;
- BPF->putBits(MERWORD(lastWord), 64);
- }
- };
- void readFromBitPackedFile(bitPackedFile *BPF, u32bit numBits=0) {
- if (numBits == 0)
- numBits = _merSize << 1;
-
- u32bit lastWord = numBits >> 6;
-
- if ((numBits & u32bitMASK(6)) == 0)
- lastWord++;
-
- if (numBits & u32bitMASK(6))
- MERWORD(lastWord) = BPF->getBits(numBits & u32bitMASK(6));
- while (lastWord > 0) {
- lastWord--;
- MERWORD(lastWord) = BPF->getBits(64);
- }
- };
-
-
-public:
- // these should work generically for both big and small
-
- void setBits(u32bit pos, u32bit numbits, u64bit val) {
- u32bit wrd = pos >> 6;
- u32bit bit = pos & 0x3f;
-
- val &= u64bitMASK(numbits);
-
- if (wrd >= KMER_WORDS) {
- fprintf(stderr, "kMer::setBits()-- ERROR: tried to set pos="u32bitFMT" numbits="u32bitFMT" larger than KMER_WORDS=%d\n",
- pos, numbits, KMER_WORDS), exit(1);
- }
-
- // If we have enough space in the word for the bits, replace
- // those bits in the word. Otherwise we need to split the value
- // into two pieces, and add to the end of the first word and the
- // start of the second.
-
- if (64 - bit >= numbits) {
- MERWORD(wrd) &= ~(u64bitMASK(numbits) << bit);
- MERWORD(wrd) |= val << bit;
- } else {
- if (wrd+1 >= KMER_WORDS) {
- fprintf(stderr, "kMer::setBits()-- ERROR: tried to set pos="u32bitFMT" numbits="u32bitFMT" larger than KMER_WORDS=%d\n",
- pos, numbits, KMER_WORDS), exit(1);
- }
-
- u32bit b1 = 64 - bit; // bits in the first word
- u32bit b2 = numbits - b1; // bits in the second word
-
- MERWORD(wrd) &= ~(u64bitMASK(b1) << bit);
- MERWORD(wrd) |= (val & u64bitMASK(b1)) << bit;
-
- MERWORD(wrd+1) &= ~(u64bitMASK(b2));
- MERWORD(wrd+1) |= (val >> b1) & u64bitMASK(b2);
- }
- };
-
- u64bit getBits(u32bit pos, u32bit numbits) const {
- u64bit val = u64bitZERO;
- u32bit wrd = pos >> 6;
- u32bit bit = pos & 0x3f;
-
- if (wrd >= KMER_WORDS) {
- fprintf(stderr, "kMer::getBits()-- ERROR: tried to get pos="u32bitFMT" numbits="u32bitFMT" larger than KMER_WORDS=%d\n",
- pos, numbits, KMER_WORDS), exit(1);
- }
-
- if (64 - bit >= numbits) {
- val = MERWORD(wrd) >> bit;
- } else {
- if (wrd+1 >= KMER_WORDS) {
- fprintf(stderr, "kMer::getBits()-- ERROR: tried to get pos="u32bitFMT" numbits="u32bitFMT" larger than KMER_WORDS=%d\n",
- pos, numbits, KMER_WORDS), exit(1);
- }
-
- u32bit b1 = 64 - bit; // bits in the first word
- u32bit b2 = numbits - b1; // bits in the second word
-
- val = MERWORD(wrd) >> (64-b1);
- val |= (MERWORD(wrd+1) & u64bitMASK(b2)) << b1;
- }
-
- val &= u64bitMASK(numbits);
- return(val);
- };
-
-
-public:
- // these should work generically for both big and small
-
- u64bit startOfMer(u32bit bits) const {
- return(getBits((_merSize << 1) - bits, bits));
- };
- u64bit endOfMer(u32bit bits) const {
- return(MERWORD(0) & u64bitMASK(bits));
- };
-
-public:
- // these should work generically for both big and small
- u64bit getWord(u32bit wrd) const { return(MERWORD(wrd)); };
- void setWord(u32bit wrd, u64bit val) { MERWORD(wrd) = val; };
-
-public:
- char *merToString(char *instr) const;
-
-private:
- u64bit _md[KMER_WORDS];
-
- // The _merSize is always the number of letters in the mer -- if we
- // are a spaced seed, it is the weight.
- //
- u32bit _merSize;
- u32bit _merSpan;
-
- // The mask is used to make sure the mer has only _merSize bases
- // set -- we can get more than that if we shift to the left. The
- // _maskWord is the word that we want to mask:
- //
- u64bit _mask;
- u32bit _maskWord;
-
- // For operator-=() (add a base to the left end) we need to know
- // what the last word is, and how far to shift the bits.
- //
- // _lastWord -- the last word that contains bases
- // _lastShift -- the amount we need to shift left to put bits 0 and 1
- // into the last base
- u32bit _lastWord;
- u32bit _lastShift;
-};
-
-
-
-inline
-void
-kMerHuge::setMerSize(u32bit ms) {
- _merSize = ms;
- _merSpan = ms;
- _lastWord = (2 * ms - 2) / 64;
- _lastShift = (2 * ms - 2) % 64;
-
- _mask = u64bitZERO;
- _maskWord = _merSize / 32;
-
- // Filled whole words with the mer, the mask is special-cased
- // to clear the whole next word, unless there is no whole next
- // word, then it does nothing on the last word.
- //
- // Otherwise, we can construct the mask as usual.
- //
- if ((_merSize % 32) == 0) {
- if (_maskWord >= KMER_WORDS) {
- _maskWord = KMER_WORDS - 1;
- _mask = ~u64bitZERO;
- } else {
- _maskWord = _merSize / 32;
- _mask = u64bitZERO;
- }
- } else {
- _mask = u64bitMASK((_merSize % 32) << 1);
- }
-
- if (_maskWord >= KMER_WORDS) {
- fprintf(stderr, "kMer::setMerSize()-- ERROR! Desired merSize of "u32bitFMT" larger than\n", _merSize);
- fprintf(stderr, " available storage space (KMER_WORDS=%d, max merSize %d).\n", KMER_WORDS, KMER_WORDS*32);
- exit(1);
- }
-}
-
-
-
-
-inline
-char *
-kMerHuge::merToString(char *instr) const {
- u32bit lastWord = _merSize >> 5;
- char *str = instr;
-
- if ((_merSize & u32bitMASK(6)) == 0)
- lastWord++;
-
- if (_merSize & u32bitMASK(5)) {
- u64bitToMerString(_merSize & u32bitMASK(5), MERWORD(lastWord), str);
- str += _merSize & u32bitMASK(5);
- }
-
- while (lastWord > 0) {
- lastWord--;
- u64bitToMerString(32, MERWORD(lastWord), str);
- str += 32;
- }
-
- return(instr);
-};
diff --git a/kmer/libbio/kmeriface.H b/kmer/libbio/kmeriface.H
deleted file mode 100644
index 93dff56..0000000
--- a/kmer/libbio/kmeriface.H
+++ /dev/null
@@ -1,83 +0,0 @@
-
-#if 0
-
-// Documentation, really.
-
-// Incomplete too.
-
-class kMerInterface {
- kMerInterface() {};
- virtual ~kMerInterface() {};
-
- // Reverse all the words, reverse and complement the bases in
- // each word, then shift right to align the edge.
- //
- virtual kMerInterface &reverseComplement(void) = 0;
- virtual void clear(void);
-
- // Construct a mer by shifting bases onto the end:
- // += shifts onto the right end
- // -= shifts onto the left end
- //
- virtual void operator+=(u64bit x) = 0;
- virtual void operator-=(u64bit x) = 0;
-
- // used by merStream at least
- //
- virtual void mask(bool) = 0;
-
- // Return the mer, as a 64-bit integer. If the mer is more than
- // 32-bases long, then the left-most (the earliest, the start, etc)
- // bases are used.
- //
- virtual operator u64bit () const = 0;
-
- // These are written/read in 5'endian, which isn't the most natural
- // implementation. It's done this way to keep the sequence in
- // order (e.g., the merStreamFile). Don't change the order.
- //
- // On the otherhand, the implementation (of write anyway) is
- // basically the same as merToString().
- //
- // Takes an optional number of BITS to write, pulled from the
- // END of the mer.
- //
- virtual void writeToBitPackedFile(bitPackedFile *BPF, u32bit numBits=0) const = 0;
- virtual void readFromBitPackedFile(bitPackedFile *BPF, u32bit numBits=0) = 0;
-
- // Returns a sub-mer from either the start (left end) or the end
- // (right end) of the mer. The sub-mer must be at most 64 bits
- // long. Yes, BITS.
- //
- // The start is difficult, because it can span multiple words. The
- // end is always in the first word.
- //
- virtual u64bit startOfMer(u32bit bits) const = 0;
- virtual u64bit endOfMer(u32bit bits) const = 0;
-
- // Set 'numbits' bits from (the end of) 'val' at bit position 'pos'
- // in the mer. This is wildly low-level, but merylStreamReader
- // needs it.
- //
- // The position is measured from the right end.
- // (0, 8, X) would copy the bits 7 to 0 of X to bits 7 to 0 of the mer.
- //
- // Argh! Can't use set/getDecodedValue because that is doing things in the wrong order.
- //
- // Meryl
- //
- virtual u64bit getWord(u32bit wrd) const = 0; // { return(MERWORD(wrd)); };
- virtual void setWord(u32bit wrd, u64bit val) = 0; // { MERWORD(wrd) = val; };
-
- // Show the mer as ascii
- //
- // Doesn't print the last full word, if it's on the word boundary
- //
- // We build the string right to left, print any partial word first,
- // then print whole words until we run out of words to print.
- //
- virtual char *merToString(char *instr) const = 0;
-};
-
-
-#endif
diff --git a/kmer/libbio/kmertiny.H b/kmer/libbio/kmertiny.H
deleted file mode 100644
index 162cd20..0000000
--- a/kmer/libbio/kmertiny.H
+++ /dev/null
@@ -1,147 +0,0 @@
-
-class kMerTiny {
-public:
- kMerTiny(u32bit ms=u32bitZERO) {
- setMerSize(ms);
- clear();
- };
- ~kMerTiny() {
- };
-
- void setMerSize(u32bit ms);
- u32bit getMerSize(void) const { return(_merSize); };
-
- void setMerSpan(u32bit ms) { _merSpan = ms; };
- u32bit getMerSpan(void) const { return(_merSpan); };
-
- kMerTiny &reverseComplement(void) {
- _md = reverseComplementMer(_merSize, _md);
- return(*this);
- };
-
- void clear(void) {
- _md = u64bitZERO;
- };
- void smallest(void) {
- clear();
- };
- void largest(void) {
- clear();
- reverseComplement();
- };
-
-private:
- void operator>>=(u32bit x) {
- _md >>= x;
- };
- void operator<<=(u32bit x) {
- _md <<= x;
- };
-
-public:
- void operator+=(u64bit x) {
- *this <<= 2;
- assert((x & 0xfc) == 0);
- _md |= x & u64bitNUMBER(0x3);
- };
- void operator-=(u64bit x) {
- *this >>= 2;
- assert((x & 0xfc) == 0);
- _md |= (x & u64bitNUMBER(0x3)) << _lastShift;
- };
-
-public:
- void mask(bool) {
- _md &= _mask;
- };
-
-public:
- bool operator!=(kMerTiny const &r) const { return(_md != r._md); };
- bool operator==(kMerTiny const &r) const { return(_md == r._md); };
- bool operator< (kMerTiny const &r) const { return(_md < r._md); };
- bool operator> (kMerTiny const &r) const { return(_md > r._md); };
- bool operator<=(kMerTiny const &r) const { return(_md <= r._md); };
- bool operator>=(kMerTiny const &r) const { return(_md >= r._md); };
- int qsort_less(kMerTiny const &r) const {
- if (_md < r._md) return(-1);
- if (_md > r._md) return( 1);
- return(0);
- };
-public:
- operator u64bit () const {return(_md);};
-
-public:
- void writeToBitPackedFile(bitPackedFile *BPF, u32bit numBits=0) const {
- BPF->putBits(_md, _merSize << 1);
- };
- void readFromBitPackedFile(bitPackedFile *BPF, u32bit numBits=0) {
- _md = BPF->getBits(_merSize << 1);
- };
-
-public:
- void setBits(u32bit pos, u32bit numbits, u64bit val) {
- _md &= ~(u64bitMASK(numbits) << pos);
- _md |= val << pos;
- };
-
- u64bit getBits(u32bit pos, u32bit numbits) const {
- return((_md >> pos) & u64bitMASK(numbits));
- };
-
-public:
- u64bit startOfMer(u32bit bits) const {
- return(getBits((_merSize << 1) - bits, bits));
- };
- u64bit endOfMer(u32bit bits) const {
- return(_md & u64bitMASK(bits));
- };
-
-public:
- u64bit getWord(u32bit wrd) const { return(_md); };
- void setWord(u32bit wrd, u64bit val) { _md = val; };
-
-public:
- char *merToString(char *instr) const;
-
-private:
- u64bit _md;
-
- // The _merSize is always the number of letters in the mer -- if we
- // are a spaced seed, it is the weight.
- //
- u32bit _merSize;
- u32bit _merSpan;
-
- // The mask is used to make sure the mer has only _merSize bases
- // set -- we can get more than that if we shift to the left. The
- //
- u64bit _mask;
-
- // For operator-=() (add a base to the left end) we need to know
- // what the last word is, and how far to shift the bits.
- //
- u32bit _lastShift;
-};
-
-
-
-
-
-inline
-void
-kMerTiny::setMerSize(u32bit ms) {
- _merSize = ms;
- _merSpan = ms;
- _lastShift = (2 * ms - 2) % 64;
- _mask = u64bitMASK(_merSize << 1);
-}
-
-
-inline
-char *
-kMerTiny::merToString(char *str) const {
- for (u32bit i=0; i<_merSize; i++)
- str[_merSize-i-1] = bitsToLetter[(_md >> (2*i)) & 0x03];
- str[_merSize] = 0;
- return(str);
-}
diff --git a/kmer/libbio/merCovering.H b/kmer/libbio/merCovering.H
deleted file mode 100644
index e8c8c57..0000000
--- a/kmer/libbio/merCovering.H
+++ /dev/null
@@ -1,353 +0,0 @@
-#ifndef MER_COVERING_H
-#define MER_COVERING_H
-
-// This is an interval list, where the intervals are built using
-// fixed size pieces.
-//
-// It's designed to accept pieces in roughly sorted order.
-//
-// Intervals are stored c-style.
-//
-
-#include <stdio.h>
-#include <stdlib.h>
-
-class merCovering {
-private:
- class interval {
- public:
- u32bit _lo;
- u32bit _hi;
- interval *_next;
-
- interval(u32bit lo, u32bit hi, interval *n) {
- _lo = lo;
- _hi = hi;
- _next = n;
- }
- };
-
- interval *_intervals;
- u32bit _width;
- u32bit _pieces;
-#ifdef TEST_MERCOVERING
- u32bit _test[TEST_SIZE];
-#endif
-
-public:
- merCovering(u32bit w) {
- _intervals = 0L;
- _width = w;
- _pieces = 0;
-
-#ifdef TEST_MERCOVERING
- for (u32bit i=0; i<TEST_SIZE; i++)
- _test[i] = 0;
-#endif
- };
-
- ~merCovering() {
- clear();
- };
-
- void clear(void) {
- interval *i = _intervals;
-
- while (i) {
- _intervals = i->_next;
- delete i;
- i = _intervals;
- }
-
- _intervals = 0L;
- _pieces = 0;
- };
-
- u32bit sumOfLengths(void) {
- u32bit s=0;
-
- for (interval *i=_intervals; i; i = i->_next)
- s += i->_hi - i->_lo;
-
- return(s);
- };
-
- u32bit numberOfPieces(void) {
- return(_pieces);
- };
-
- void addMer(u32bit lo) {
- _pieces++;
-
- u32bit hi = lo + _width;
-
- interval *c;
-
-#ifdef TEST_MERCOVERING
- for (u32bit i=lo; i<hi; i++)
- _test[i] = 1;
-#endif
-
- // Case: No existing intervals, or the new interval extends
- // the low range of the first interval
- //
- if ((_intervals == 0L) ||
- (hi < _intervals->_lo)) {
- _intervals = new interval(lo, hi, _intervals);
- return;
- }
-
- c = _intervals;
-
- while (c) {
-
- // Case: New interval is completely contained in the current interval.
- //
- if ((c->_lo <= lo) && (hi <= c->_hi))
- return;
-
- // Case: New interval overlaps the low end of the current interval,
- // or is completely contained in an existing interval.
- //
- if ((lo <= c->_lo) && (hi <= c->_hi)) {
- c->_lo = lo;
- return;
- }
-
- if (c->_next) {
-
- // Case: New interval overlaps the high end of the current interval...
- //
- if (lo <= c->_hi) {
-
- if (hi < c->_next->_lo) {
- // but does not intersect the next interval.
- //
- c->_hi = hi;
- return;
- } else {
- // and does intersect the next interval.
- //
- interval *p = c->_next;
-
- c->_hi = c->_next->_hi;
- c->_next = c->_next->_next;
-
- delete p;
- return;
- }
- } else {
- // Case: New interval is between two existing intervals
- //
- // (lo > c->_hi) is given
- //
- if (hi < c->_next->_lo) {
- c->_next = new interval(lo, hi, c->_next);
- return;
- }
- }
- } else {
- // Case: New interval overlaps the high end of the current interval
- //
- if (lo <= c->_hi) {
- c->_hi = hi;
- return;
- } else {
- // Otherwise, we just fell off the end of all intervals.
- // Add one at the end.
- //
- c->_next = new interval(lo, hi,0L);
- return;
- }
- }
-
- c = c->_next;
- }
-
-#ifdef TEST_MERCOVERING
- fprintf(stderr, "ERROR IN addInterval!\n");
-#endif
- };
-
-#ifdef TEST_MERCOVERING
- void test(void) {
- for (u32bit i=0; i<TEST_SIZE; i++) {
- if (_test[i])
- _test[i] = 2;
- }
- for (interval *z=_intervals; z; z = z->_next) {
- for (u32bit i=z->_lo; i<z->_hi; i++) {
- if (_test[i] == 0) {
- fprintf(stderr, "INTERVAL CONTAINS SOMETHING NOT IN ARRAY! (%d)\n", i);
- exit(1);
- }
- if (_test[i] == 1) {
- fprintf(stderr, "INTERVAL HIT SOMETHING TWICE! (%d)\n", i);
- exit(1);
- }
- _test[i] = 1;
- }
- }
- for (u32bit i=0; i<TEST_SIZE; i++) {
- if (_test[i] == 2) {
- fprintf(stderr, "ARRAY CONTAINED SOMETHING NOT IN INTERVAL! (%d)\n", i);
- exit(1);
- }
- }
- };
-#endif
-
-
-
- // Incorporates the intervals in B into our list.
- //
- void merge(merCovering *I = 0L) {
- interval *A, *B, *N, *L;
-
- if (I == 0L)
- return;
-
- A = _intervals;
- B = I->_intervals;
- N = 0L;
- L = 0L;
-
- while (A || B) {
- u32bit lo = 0;
- u32bit hi = 0;
-
- // if either list is zero, we can just zip down the other list
- // and add things.
- //
- if (!B) {
- while (A) {
- L->_next = new interval(A->_lo, A->_hi, 0L);
- L = L->_next;
- A = A->_next;
- }
- }
-
- if (!A) {
- while (B) {
- L->_next = new interval(B->_lo, B->_hi, 0L);
- L = L->_next;
- B = B->_next;
- }
- }
-
- if (A && B) {
- if (A->_lo == B->_lo) {
- // A and B start at the same position
- //
- lo = A->_lo;
- hi = A->_hi;
- if (hi < B->_hi)
- hi = B->_hi;
-
- A = A->_next;
- B = B->_next;
- } else {
- // A and B start at different positions. Pick the first one.
- //
- if (A->_lo < B->_lo) {
- lo = A->_lo;
- hi = A->_hi;
- A = A->_next;
- } else {
- lo = B->_lo;
- hi = B->_hi;
- B = B->_next;
- }
- }
-
- // We have an initial interval. Add more stuff, while there
- // are overlaps.
-
- bool modified = true;
-
- while ((A || B) && (modified)) {
- modified = false;
-
- if ((A) && (hi >= A->_lo)) {
- if (hi < A->_hi)
- hi = A->_hi;
- A = A->_next;
- modified = true;
- }
-
- if ((B) && (hi >= B->_lo)) {
- if (hi < B->_hi)
- hi = B->_hi;
- B = B->_next;
- modified = true;
- }
- }
-
- // OK, got the new interval. Save it.
- //
- if (N) {
- L->_next = new interval(lo, hi, 0L);
- L = L->_next;
- } else {
- N = L = new interval(lo, hi, 0L);
- }
- }
- }
-
- // Save the number of mers in both intervals
- //
- u32bit p = _pieces + I->_pieces;
-
- clear();
-
- _intervals = N;
- _pieces = p;
- }
-
-
-
-
-#ifdef TEST_MERCOVERING
- void dump(void) {
- for (interval *i=_intervals; i; i = i->_next)
- fprintf(stderr, "%5d-%5d ", i->_lo, i->_hi);
- fprintf(stderr, "\n");
- };
-
- void compare(merCovering *B) {
- interval *i = _intervals;
- interval *j = B->_intervals;
-
- if (_pieces != B->_pieces) {
- fprintf(stderr, "Pieces differ (this=%d that=%d).\n", _pieces, B->_pieces);
- exit(1);
- }
-
- while (i && j) {
- if ((i->_lo != j->_lo) || (i->_hi != j->_hi)) {
- fprintf(stderr, "ERROR!\n");
- exit(1);
- }
-
- i = i->_next;
- j = j->_next;
- }
-
- if (i) {
- fprintf(stderr, "ERROR (i still exists)!\n");
- exit(1);
- }
-
- if (j) {
- fprintf(stderr, "ERROR (i still exists)!\n");
- exit(1);
- }
- };
-#endif
-
-};
-
-#endif // MERCOVERING_H
-
-
-
diff --git a/kmer/libbio/merList.H b/kmer/libbio/merList.H
deleted file mode 100644
index 21f0eaa..0000000
--- a/kmer/libbio/merList.H
+++ /dev/null
@@ -1,94 +0,0 @@
-#ifndef MER_LIST_H
-#define MER_LIST_H
-
-// A simple list of mers. Implemented as a list of lists.
-
-class merList {
-public:
- merList() {
- _ptrsMax = 8;
- _ptrsLen = 0;
- _ptrs = new coord * [_ptrsMax];
-
- _mersWid = 12;
- _mersMax = u32bitMASK(_mersWid);
- _mersLen = 0;
- _ptrs[0] = new coord [_mersMax+1];
- };
- ~merList() {
- for (u32bit x=0; x<_ptrsLen+1; x++)
- delete [] _ptrs[x];
- delete [] _ptrs;
- };
-
- void addMer(u32bit x, u32bit y) {
- if (_mersLen > _mersMax) {
- _ptrsLen++;
-
- if (_ptrsLen >= _ptrsMax) {
- _ptrsMax *= 2;
- coord **p = new coord * [_ptrsMax];
- memcpy(p, _ptrs, sizeof(coord*) * _ptrsLen);
- delete [] _ptrs;
- _ptrs = p;
- }
-
- _ptrs[_ptrsLen] = new coord [_mersMax+1];
- _mersLen = 0;
- }
-
- _ptrs[_ptrsLen][_mersLen]._qPos = x;
- _ptrs[_ptrsLen][_mersLen]._gPos = y;
-
- _mersLen++;
- };
-
- bool getMer(u32bit i, u32bit &x, u32bit &y) {
- u32bit p = i >> _mersWid;
- u32bit a = i & _mersMax;
-
- if ((p > _ptrsLen) || ((p == _ptrsLen) && (a >= _mersLen)))
- return(false);
-
- x = _ptrs[(i >> _mersWid)][i & _mersMax]._qPos;
- y = _ptrs[(i >> _mersWid)][i & _mersMax]._gPos;
-
- return(true);
- };
-
- void clear(void) {
- // Don't delete the first guy! We write into it blindly!
- for (u32bit x=1; x<_ptrsLen; x++)
- delete [] _ptrs[x];
- _ptrsLen = 0;
- _mersLen = 0;
- };
-
- void merge(merList *ML) {
- u32bit i, x, y;
- for (i=0; ML->getMer(i, x, y); i++)
- addMer(x, y);
- };
-
-private:
- struct coord {
- u32bit _qPos;
- u32bit _gPos;
- };
-
- // The number of mer blocks we have space for, and the current mer
- // block.
- //
- u32bit _ptrsMax;
- u32bit _ptrsLen;
- coord **_ptrs;
-
- // The number of mers available in each block, and the current mer
- // we are at in the current block (for adding new mers).
- //
- u32bit _mersWid;
- u32bit _mersMax;
- u32bit _mersLen;
-};
-
-#endif // MER_LIST_H
diff --git a/kmer/libbio/mers.h b/kmer/libbio/mers.h
deleted file mode 100644
index 3503012..0000000
--- a/kmer/libbio/mers.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#ifndef BIO_MERS_H
-#define BIO_MERS_H
-
-
-
-inline
-u64bit
-reverseComplementMer(u32bit ms, u64bit fmer) {
-
- // The interested reader shall consult bri-bits.h
-
- // Reverse the mer
- //
- u64bit rmer = fmer;
- rmer = ((rmer >> 2) & 0x3333333333333333llu) | ((rmer << 2) & 0xccccccccccccccccllu);
- rmer = ((rmer >> 4) & 0x0f0f0f0f0f0f0f0fllu) | ((rmer << 4) & 0xf0f0f0f0f0f0f0f0llu);
- rmer = ((rmer >> 8) & 0x00ff00ff00ff00ffllu) | ((rmer << 8) & 0xff00ff00ff00ff00llu);
- rmer = ((rmer >> 16) & 0x0000ffff0000ffffllu) | ((rmer << 16) & 0xffff0000ffff0000llu);
- rmer = ((rmer >> 32) & 0x00000000ffffffffllu) | ((rmer << 32) & 0xffffffff00000000llu);
-
- // Complement the bases
- //
- rmer ^= 0xffffffffffffffffllu;
-
- // Shift and mask out the bases not in the mer
- //
- rmer >>= 64 - ms * 2;
- rmer &= u64bitMASK(ms * 2);
- return(rmer);
-}
-
-
-// Used for in seagen/encodedQuery.C (diagnostics) and
-// libbio/kmerhuge.H (in its merToString method).
-inline
-char *
-u64bitToMerString(u32bit ms, u64bit mer, char *str) {
- for (u32bit i=0; i<ms; i++)
- str[ms-i-1] = bitsToLetter[(mer >> (2*i)) & 0x03];
- str[ms] = 0;
- return(str);
-}
-
-
-#if 0
-#error this is not used anywhere
-inline
-u64bit
-stringToMer(u32bit ms, char *str) {
- u64bit mer = 0L;
-
- for (u32bit i=0; i<ms; i++) {
- mer <<= 2;
- mer |= compressSymbol[str[i]];
- }
-
- return(mer);
-}
-#endif
-
-
-
-#endif // BIO_MERS_H
diff --git a/kmer/libbio/reversecomplement.c b/kmer/libbio/reversecomplement.c
deleted file mode 100644
index 0cd139b..0000000
--- a/kmer/libbio/reversecomplement.c
+++ /dev/null
@@ -1,44 +0,0 @@
-#include "bio.h"
-
-#include <string.h>
-
-// Inplace reverse-complement an ACGT sequence. A pointer the the
-// string is returned.
-//
-char *
-reverseComplementSequence(char *seq, u32bit seqlen) {
- char *s = seq;
- char *e = seq + seqlen - 1;
- char t;
- u32bit c = seqlen / 2;
-
- while (c--) {
- t = complementSymbol[*s];
- *(s++) = complementSymbol[*e];
- *(e--) = t;
- }
-
- if (s == e)
- *s = complementSymbol[*s];
-
- return(seq);
-}
-
-
-// Inplace reverse a string. A pointer the the string is returned.
-//
-char *
-reverseString(char *seq, u32bit seqlen) {
- char *s = seq;
- char *e = seq + seqlen - 1;
- char t;
- u32bit c = seqlen / 2;
-
- while (c--) {
- t = *s;
- *(s++) = *e;
- *(e--) = t;
- }
-
- return(seq);
-}
diff --git a/kmer/libbio/test/Makefile b/kmer/libbio/test/Makefile
deleted file mode 100644
index 445ffdb..0000000
--- a/kmer/libbio/test/Makefile
+++ /dev/null
@@ -1,98 +0,0 @@
-
-
-#PROG = test-merstream-from-seqstore
-
-PROG = dump-merstreamfile \
- test-seqStream \
- test-chainedSequence \
- test-fasta-accessor \
- test-merstream \
- test-merstreamfile \
- test-setbits \
- halign-test
-
-DEAD = test-merstream-speed \
- test-bigmer-msf \
-
-
-INCLUDE = -I.. -I../../libutil
-LIBS = -L.. -L../../libutil -lbio -lutil -lm
-OBJS =
-
-include ../../Make.compilers
-
-all: $(PROG)
- @echo Tests passed!
-
-dump-merstreamfile: dump-merstreamfile.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o dump-merstreamfile.o dump-merstreamfile.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o dump-merstreamfile dump-merstreamfile.o $(LIBS)
-
-test-merstream-from-seqstore: test-merstream-from-seqstore.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-merstream-from-seqstore.o test-merstream-from-seqstore.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-merstream-from-seqstore test-merstream-from-seqstore.o $(LIBS)
-
-test-seqStream: test-seqStream.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-seqStream.o test-seqStream.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-seqStream test-seqStream.o $(LIBS)
- ../../leaff/leaff -G 3 30 40 > junk2.fasta
- ./test-seqStream junk2.fasta
- rm -f junk*
-
-test-chainedSequence: test-chainedSequence.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-chainedSequence.o test-chainedSequence.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-chainedSequence test-chainedSequence.o $(LIBS)
- ./test-chainedSequence
- ../../leaff/leaff -G 1000 1000 3000 > junk2.fasta
- ./test-chainedSequence junk2.fasta
- rm -f junk*
-
-test-fasta-accessor: test-fasta-accessor.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-fasta-accessor.o test-fasta-accessor.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-fasta-accessor test-fasta-accessor.o $(LIBS)
- ./test-fasta-accessor
-
-test-merstream: test-merstream.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-merstream.o test-merstream.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-merstream test-merstream.o $(LIBS)
- ../../leaff/leaff -G 1 10000 30000 > junk.fasta
- ./test-merstream junk.fasta
- rm -f junk*
- ../../leaff/leaff -G 1000 10000 30000 > junk.fasta
- ./test-merstream junk.fasta
- rm -f junk*
-
-test-merstreamfile: test-merstreamfile.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-merstreamfile.o test-merstreamfile.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-merstreamfile test-merstreamfile.o $(LIBS)
- #../../leaff/leaff -G 2 50 50 > junk.fasta
- ../../leaff/leaff -G 100000 10 600 > junk.fasta
- #../../leaff/leaff -G 10000 10 10000 > junk.fasta
- #../../leaff/leaff -G 30000 10000 10000 > junk.fasta
- ./test-merstreamfile junk.fasta
- #rm -f junk.fasta junk.fastaidx
-
-test-merstream-speed: test-merstream-speed.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-merstream-speed.o test-merstream-speed.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-merstream-speed test-merstream-speed.o $(LIBS)
- ../../leaff/leaff -G 10000 1000 10000 > junk.fasta
- cat junk.fasta > /dev/null
- ./test-merstream-speed junk.fasta
- rm -f junk*
-
-test-setbits: test-setbits.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-setbits.o test-setbits.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-setbits test-setbits.o $(LIBS)
- ./test-setbits
-
-test-bigmer-msf: test-bigmer-msf.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-bigmer-msf.o test-bigmer-msf.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-bigmer-msf test-bigmer-msf.o $(LIBS)
- ./test-bigmer-msf
-
-halign-test: halign-test.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o halign-test.o halign-test.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o halign-test halign-test.o $(LIBS)
-
-clean:
- rm -f $(PROG) *.o *junk*
diff --git a/kmer/libbio/test/halign-test.C b/kmer/libbio/test/halign-test.C
deleted file mode 100644
index f7e380b..0000000
--- a/kmer/libbio/test/halign-test.C
+++ /dev/null
@@ -1,48 +0,0 @@
-#include "bio++.H"
-
-int
-main(int argc, char **argv) {
-
- const char *s1 = "gattcatggctgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctaaaattaaccgtcagaatatttatgcctgattcatggctgaaattgtgtttgaccagctatgtgtgtctcttaatccactcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcacgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctgaaattaaccatcagaatatttatgcctgattcatggctgaaatttcaggatgaaagctatgaaatctctatttgtgtttgtgtatctattaatgtatgttatgtatatgtgatattttcttaactcc [...]
- const char *s2 = "gattcatggctgaaatcatgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcgtgtttgaccagctatgtgtgtctcttactccactcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcatgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctgaaattaaccatcagaatatttatgcctgattcatggctgaaatttcaggatgacagctatgaaatctctatttgtgtttgtatatctattaatgtatgttatgtatatgtgatattttcttaactcc [...]
-
- s1 = "gATTCATGGCTgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctaaaattaaccgtcaGAATATTTATGCCTGATTCATGGCTgaaattgtgtttgaccagctatgtgtgtctcttaatccactcaagtagatgtctaaaattaaccatcaGAATATTTATGCCTGATTCATGGCTgaaatcacgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcaGAATATTTATGCCTGATTCATGGCTgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctgaaattaaccatcaGAAtatttatgcctgattcatggctgaaatttcaggatgaaagctatgaaatctctatttgtgtttgtgtatctattaatgtatgttatgtatatgtgatattttcttaactcc [...]
- s2 = "gATTCATGGCTGAAATCATGTTTGACCAGCTATGTGTGTCTCTTAATCCAGTCAAGTAGATGTCTAAAATTAACCATCAGAATATTTATGCCTGATTCATGGCTGAAATCGTGTTTGACCAGCTATGTGTGTCTCTTACTCCACTCAAGTAGATGTCTAAAATTAACCATCAGAATATTTATGCCTGATTCATGGCTGAAATCATGTTTGACCAGCTATGTGTGTCTCTTAATCCAGTCAAGTAGATGTCTAAAATTAACCATCAGAATATTTATGCCTGATTCATGGCTGAAATCGTGTTTGACCAGCTATGTGTGTCTCTCAATCCGATCAAGTAGATGTCTGAAATTAACCATCAGAATATTTATGCCTGATTCATGGCTGAAATTTCAGGATGACAGCTATGAAATCTCTATTTGTGTTTGTATATCTATTAATGTATGTTATGTATATGTGATATTTTCTTAACTCC [...]
-
- s1 = "gattcatggctgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctaaaattaaccgtcagaatatttatgcctgattcatggctgaaattgtgtttgaccagctatgtgtgtctcttaatccactcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcacgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctgaaattaaccatcagaatatttatgcctgattcatggctgaaatttcaggatgaaagctatgaaatctctatttgtgtttgtgtatctattaatgtatgttatgtatatgtgatattttcttaactcc [...]
- s2 = "gattcatggctgaaatcatgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcgtgtttgaccagctatgtgtgtctcttactccactcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcatgtttgaccagctatgtgtgtctcttaatccagtcaagtagatgtctaaaattaaccatcagaatatttatgcctgattcatggctgaaatcgtgtttgaccagctatgtgtgtctctcaatccgatcaagtagatgtctgaaattaaccatcagaatatttatgcctgattcatggctgaaatttcaggatgacagctatgaaatctctatttgtgtttgtatatctattaatgtatgttatgtatatgtgatattttcttaactcc [...]
-
- char *a1 = new char [10240];
- char *a2 = new char [10240];
-
- halign(s1, s2,
- strlen(s1), strlen(s2),
- a1, a2);
-
- int match = 0;
- int mismatch = 0;
- int gap = 0;
-
- for (int i=0; a1[i]; i++) {
- if ((a1[i] == '-') || (a2[i] == '-')) {
- gap++;
- a1[i] = toupper(a1[i]);
- a2[i] = toupper(a2[i]);
- } else if (a1[i] != a2[i]) {
- mismatch++;
- a1[i] = toupper(a1[i]);
- a2[i] = toupper(a2[i]);
- } else {
- match++;
- a1[i] = tolower(a1[i]);
- a2[i] = tolower(a2[i]);
- }
- }
-
- fprintf(stdout, "a1 = %s\n", a1);
- fprintf(stdout, "a2 = %s\n", a2);
-
- fprintf(stdout, "mismatch=%d gap=%d match=%d\n", mismatch, gap, match);
-}
-
-
diff --git a/kmer/libbio/test/test-bigmer-msf.C b/kmer/libbio/test/test-bigmer-msf.C
deleted file mode 100644
index 8300174..0000000
--- a/kmer/libbio/test/test-bigmer-msf.C
+++ /dev/null
@@ -1,222 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <math.h>
-#include <time.h>
-
-#include "bio++.H"
-
-// Build a merStreamFile using small mers, read it back using bigger mers.
-//
-// construct a fasta sequence:
-// small sequence
-// big sequence, multiple of mersize
-// small sequence
-// big sequence
-// etc
-// small sequence
-//
-// Then we reconstruct the sequences using mers. All three merstream
-// sources are tested (the character string source is not tested).
-// The merStreamFile is tested both forwards (nextMer(), via the
-// merstream interface) and backwards (setIterationStart()).
-
-#define BUILD_SIZE 88
-#define TEST_SIZE 403
-#define MERS_PER_SEQ 37
-#define TEST_ITERATIONS 300
-
-#define MSF_FILENAME "junk.bigmer"
-#define FASTA_FILENAME "junk.bigmer.fasta"
-
-// construct a multi-fasta sequence, alternating short and long
-// sequences. Short sequences are less than TEST_SIZE long (most,
-// not all, longer than BUILD_SIZE). Long sequences are exactly
-// TEST_SIZE * MERS_PER_SEQ long -- this lets us dump the mers to
-// reconstruct the sequence.
-//
-void
-buildFastA(void) {
- mt_s *mtctx = mtInit(time(0L));
- char *seq = new char [TEST_SIZE * MERS_PER_SEQ + 1];
- char dna[4] = { 'A', 'C', 'G', 'T' };
-
- FILE *F = fopen(FASTA_FILENAME, "w");
-
- for (u32bit i=0; i<TEST_ITERATIONS; i++) {
- u32bit len;
-
- fprintf(F, ">"u32bitFMT"short\n", i);
- len = mtRandom32(mtctx) % (TEST_SIZE-1) + 1;
- for (u32bit s=0; s<len; s++)
- seq[s] = dna[ mtRandom32(mtctx) % 4 ];
- seq[len] = 0;
- fprintf(F, "%s\n", seq);
-
- fprintf(F, ">"u32bitFMT"long\n", i);
- len = TEST_SIZE * MERS_PER_SEQ;
- for (u32bit s=0; s<len; s++)
- seq[s] = dna[ mtRandom32(mtctx) % 4 ];
- seq[len] = 0;
- fprintf(F, "%s\n", seq);
- }
-
- fclose(F);
-}
-
-
-// Uses the merStreamFile directly to read mers, chains mers
-// into a sequence, compares against the correct sequence.
-//
-void
-test1(u32bit style) {
- seqCache *fasta = new seqCache(FASTA_FILENAME);
- seqInCore *sseq = fasta->getSequenceInCore();
- seqInCore *lseq = fasta->getSequenceInCore();
-
- char mseq[TEST_SIZE * MERS_PER_SEQ + 1];
-
- // Construct a reader, and load the first mer.
-
- merStream *MS = 0L;
- merStreamFileReader *RD = 0L;
- chainedSequence *CS = 0L;
-
- switch (style) {
- case 0:
- fprintf(stderr, "test1(0)-- Testing merStreamFileReader -> merStream\n");
- RD = new merStreamFileReader(MSF_FILENAME, TEST_SIZE);
- MS = new merStream(RD);
- break;
- case 1:
- fprintf(stderr, "test1(2)-- Testing chainedSequence -> merStream\n");
- CS = new chainedSequence();
- CS->setSource(FASTA_FILENAME);
- CS->finish();
- MS = new merStream(TEST_SIZE, CS);
- break;
- case 2:
- fprintf(stderr, "test1(3)-- Testing merStreamFileReader (backwards)\n");
- RD = new merStreamFileReader(MSF_FILENAME, TEST_SIZE);
- break;
- default:
- break;
- }
-
-
- for (u32bit s=0; fasta->eof() == false; s++) {
- for (u32bit i=0; i<TEST_SIZE * MERS_PER_SEQ + 1; i++)
- mseq[i] = 0;
-
- switch (style) {
- case 0:
- case 1:
- // Fill the sequence using non-overlapping mers, skipping
- // intermediate mers (there aren't intermediate mers if we're the
- // last mer in the sequence!)
- //
- MS->nextMer();
- for (u32bit i=0; i<MERS_PER_SEQ; i++) {
- MS->theFMer().merToString(mseq + i * TEST_SIZE);
- if (i != MERS_PER_SEQ-1)
- MS->nextMer(TEST_SIZE - 1);
- }
- break;
- case 2:
- // Same thing, but read the mers backwards -- we could read
- // the sequences backwards, too, but that doesn't gain us
- // anything (we still seek to every location).
- //
- for (u32bit i=MERS_PER_SEQ; i--; ) {
- char copy[TEST_SIZE + 1];
- RD->setIterationStart(s * (MERS_PER_SEQ * TEST_SIZE - TEST_SIZE + 1) + i * (TEST_SIZE));
- RD->nextMer();
- RD->theFMer().merToString(copy);
- strncpy(mseq + i * TEST_SIZE, copy, TEST_SIZE);
-
- // Aww, what the hell! Test reverse complement stuff too!
- //
- kMer f = RD->theFMer();
- kMer r = RD->theRMer();
- f.reverseComplement();
-
- if (f != r) {
- char str[1025];
- fprintf(stderr, "Reverse Complement mismatch:\n");
- fprintf(stderr, " reversed fwd = '%s'\n", f.merToString(str));
- fprintf(stderr, " rev = '%s'\n", r.merToString(str));
- exit(1);
- }
-
- f = RD->theFMer();
- r = RD->theRMer();
- r.reverseComplement();
-
- if (f != r) {
- char str[1025];
- fprintf(stderr, "Reverse Complement mismatch:\n");
- fprintf(stderr, " fwd = '%s'\n", f.merToString(str));
- fprintf(stderr, " reversed rev = '%s'\n", r.merToString(str));
- exit(1);
- }
-
-
- }
- mseq[MERS_PER_SEQ * TEST_SIZE] = 0;
- break;
- default:
- break;
- }
-
- // Compare our mer-constructed sequence to the long sequence in
- // the file
- //
- if (strcmp(mseq, lseq->sequence()) != 0) {
- fprintf(stderr, "FAIL: seq="u32bitFMT"\nmseq=%s\nlseq=%s\n", s, mseq, lseq->sequence());
- exit(1);
- }
-
- delete sseq;
- delete lseq;
-
- sseq = fasta->getSequenceInCore();
- lseq = fasta->getSequenceInCore();
- }
-
- delete sseq;
- delete lseq;
-
- delete CS;
- delete RD;
- delete MS;
-
- fprintf(stderr, " OK!\n");
-}
-
-
-
-
-int
-main(int argc, char **argv) {
-
- // Minimum KMER_WORDS is 13 -- mersizes up to 416 bases
- if (KMER_WORDS < 13) {
- fprintf(stderr, "I need at least KMER_WORDS == 13; test not run.\n");
- exit(0);
- }
-
- buildFastA();
-
- merStreamFileBuilder *B = new merStreamFileBuilder(BUILD_SIZE, FASTA_FILENAME, MSF_FILENAME);
- B->build(true);
- delete B;
-
- test1(0);
- test1(1);
- test1(2);
-
- unlink(FASTA_FILENAME);
- unlink(FASTA_FILENAME "idx");
- unlink(MSF_FILENAME ".merStream");
-
- exit(0);
-}
diff --git a/kmer/libbio/test/test-setbits.C b/kmer/libbio/test/test-setbits.C
deleted file mode 100644
index bca3702..0000000
--- a/kmer/libbio/test/test-setbits.C
+++ /dev/null
@@ -1,28 +0,0 @@
-#include "bio++.H"
-
-//g++ -o test-setbits test-setbits.C -I../libutil -I. -L../libutil -L. -lbio -lutil
-
-int
-main(int argc, char **argv) {
-
- kMer x(96);
- char str[256];
-
- if (KMER_WORDS < 3) {
- fprintf(stderr, "I need at least KMER_WORDS == 3; test not run.\n");
- exit(0);
- }
-
- for (u32bit i=0; i<168; i++) {
- x.clear();
- x.setBits(i, 24, 0x535);
- fprintf(stderr, u32bitFMTW(3)" -- %s -- "u64bitHEX"\n", i, x.merToString(str), x.getBits(i, 16));
-
- if (x.getBits(i, 16) != 0x535) {
- fprintf(stderr, "decode error.\n");
- exit(1);
- }
- }
-
- exit(0);
-}
diff --git a/kmer/libkmer/Make.include b/kmer/libkmer/Make.include
deleted file mode 100644
index 58941c0..0000000
--- a/kmer/libkmer/Make.include
+++ /dev/null
@@ -1,35 +0,0 @@
-# -*- makefile -*-
-
-LIBUTL/ :=$(realpath $/../libutil/)/
-LIBBIO/ :=$(realpath $/../libbio/)/
-LIBSEQ/ :=$(realpath $/../libseq/)/
-LIBMERYL/ :=$(realpath $/../libmeryl/)/
-
-src := $/existDB-create-from-fasta.C \
- $/existDB-create-from-meryl.C \
- $/existDB-create-from-sequence.C \
- $/existDB-state.C \
- $/existDB.C \
- $/existDB.H \
- $/merTable.H \
- $/positionDB-access.C \
- $/positionDB-dump.C \
- $/positionDB-file.C \
- $/positionDB-mismatch.C \
- $/positionDB-sort.C \
- $/positionDB.C \
- $/positionDB.H
-
-$/.CXX_SRCS := $(filter %.C,${src}) $/driver-existDB.C $/driver-posDB.C $/percentCovered.C $/mask.C
-$/.CXX_INCS := $(filter %.H,${src})
-$/.CXX_EXES := $/existDB $/positionDB $/percentCovered $/mask
-$/.CXX_LIBS := $/libkmer.a
-$/.CLEAN := $/*.o
-
-$/libkmer.a: $(filter %.o,${src:.C=.o})
-$/existDB: $/driver-existDB.o $/libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/positionDB: $/driver-posDB.o $/libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/percentCovered: $/percentCovered.o $/libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/mask: $/mask.o $/libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-
-$(eval $/%.d $/%.o: CXXFLAGS+= -I${LIBMERYL/} -I${LIBBIO/} -I${LIBSEQ/} -I${LIBUTL/})
diff --git a/kmer/libkmer/driver-existDB.C b/kmer/libkmer/driver-existDB.C
deleted file mode 100644
index 4522532..0000000
--- a/kmer/libkmer/driver-existDB.C
+++ /dev/null
@@ -1,227 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-
-#include "bio++.H"
-#include "existDB.H"
-#include "libmeryl.H"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-// Driver for the existDB creation. Reads a sequence.fasta, builds
-// an existDB for the mers in the file, and then writes the internal
-// structures to disk.
-//
-// The existDB constructor is smart enough to read either a pre-built
-// image or a regular multi-fasta file.
-
-
-int
-testFiles(char *filename, char *prefix, u32bit merSize) {
- char *prefixfilename = new char [strlen(prefix) + 32];
-
- // Create existDB e and save it to disk
- //
- existDB *e = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~u32bitZERO);
- sprintf(prefixfilename, "%s.1", prefix);
- e->saveState(prefixfilename);
-
- // Create existDB f by loading the saved copy from disk
- //
- existDB *f = new existDB(prefixfilename);
-
- // Create a fresh existDB g (to check if we corrup the original when saved)
- //
- existDB *g = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~u32bitZERO);
-
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
- fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (u64bitMASK(2 * merSize) + 1) / 1000000.0);
-
- for (u64bit d=0, m=u64bitMASK(2 * merSize); m--; ) {
- bool ee = e->exists(m);
- bool ef = f->exists(m);
- bool eg = g->exists(m);
-
- u32bit ce = e->count(m);
- u32bit cf = f->count(m);
- u32bit cg = g->count(m);
-
- if ((ee != ef) || (ef != eg) || (ee != eg))
- fprintf(stderr, "mer "u64bitHEX" not found : e=%d f=%d g=%d\n", m, ee, ef, eg);
-
- if ((ce != cf) || (cf != cg) || (ce != cg))
- fprintf(stderr, "mer "u64bitHEX" count differs : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee);
-
- if ((m & 0xffffff) == 0) {
- // Been a while since a report, so report.
- d = 1;
- }
-
- if ((ce > 1) && (d == 1)) {
- // Report anything not unique, to make sure that we're testing real counts and not just existence.
- fprintf(stderr, "mer "u64bitHEX" : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee);
- d = 0;
- }
-
- C->tick();
- }
-
- delete e;
- delete C;
-
- return(0);
-}
-
-
-int
-testExistence(char *filename, u32bit merSize) {
- existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~u32bitZERO);
- merStream *M = new merStream(new kMerBuilder(merSize), new seqStream(filename), true, true);
- u64bit tried = 0;
- u64bit lost = 0;
-
- while (M->nextMer()) {
- tried++;
- if (!E->exists(M->theFMer()))
- lost++;
- }
-
- delete M;
- delete E;
-
- if (lost) {
- fprintf(stderr, "Tried "u64bitFMT", didn't find "u64bitFMT" merStream mers in the existDB.\n", tried, lost);
- return(1);
- } else {
- return(0);
- }
-}
-
-
-
-int
-testExhaustive(char *filename, char *merylname, u32bit merSize) {
- existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~u32bitZERO);
- merylStreamReader *M = new merylStreamReader(merylname);
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
- u64bit found = u64bitZERO;
- u64bit expected = u64bitZERO;
-
- FILE *DUMP = 0L;
-
- DUMP = fopen("testExhaustive.ms.dump", "w");
-
- while (M->nextMer()) {
- if (E->exists(M->theFMer())) {
- expected++;
- fprintf(DUMP, u64bitHEX"\n", (u64bit)M->theFMer());
- } else {
- fprintf(DUMP, u64bitHEX" MISSED!\n", (u64bit)M->theFMer());
- }
- }
-
- fclose(DUMP);
-
- fprintf(stderr, "Found "u64bitFMT" mers in the meryl database.\n", expected);
- fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (u64bitMASK(2 * merSize) + 1) / 1000000.0);
-
- DUMP = fopen("testExhaustive.ck.dump", "w");
-
- for (u64bit m = u64bitMASK(2 * merSize); m--; ) {
- if (E->exists(m)) {
- found++;
- fprintf(DUMP, u64bitHEX"\n", m);
- }
- C->tick();
- }
-
- fclose(DUMP);
-
- delete C;
- delete E;
- delete M;
-
- if (expected != found) {
- fprintf(stderr, "Expected to find "u64bitFMT" mers, but found "u64bitFMT" instead.\n",
- expected, found);
- return(1);
- } else {
- return(0);
- }
-}
-
-
-const char *usage =
-"usage: %s [stuff]\n"
-" -mersize mersize\n"
-" -- Use the specified mersize when building existDB tables.\n"
-"\n"
-" -build some.fasta prefix\n"
-" -- Build an existDB on all mers in some.fasta and save\n"
-" the tables into prefix.\n"
-"\n"
-" -describe prefix\n"
-" -- Reports the state of some existDB file.\n"
-"\n"
-" -testfiles some.fasta prefix\n"
-" -- Build an existDB table from some.fasta. Write that table to disk.\n"
-" Load the table back. Compare that each mer in some.fasta is present\n"
-" in all three existDB tables created earlier.\n"
-"\n"
-" -testexistence some.fasta\n"
-" -- Build an existDB table from some.fasta, check that every\n"
-" mer in some.fasta can be found in the table. Does not\n"
-" guarantee that every mer in the table is found in the file.\n"
-"\n"
-" -testexhaustive some.fasta some.meryl\n"
-" -- Build an existDB table from some.fasta, check _EVERY_ mer\n"
-" for existance. Complain if a mer exists in the table but\n"
-" not in the meryl database. Assumes 'some.meryl' is the\n"
-" mercount of some.fasta.\n"
-"\n";
-
-int
-main(int argc, char **argv) {
- u32bit mersize = 20;
-
- if (argc < 3) {
- fprintf(stderr, usage, argv[0]);
- exit(1);
- }
-
- int arg = 1;
- while (arg < argc) {
- if (strncmp(argv[arg], "-mersize", 2) == 0) {
- arg++;
- mersize = atoi(argv[arg]);
-
- } else if (strncmp(argv[arg], "-describe", 2) == 0) {
- existDB *e = new existDB(argv[argc-1], false);
- e->printState(stdout);
- delete e;
- exit(0);
-
- } else if (strncmp(argv[arg], "-testfiles", 8) == 0) {
- exit(testFiles(argv[arg+1], argv[arg+2], mersize));
-
- } else if (strncmp(argv[arg], "-testexistence", 8) == 0) {
- exit(testExistence(argv[arg+1], mersize));
-
- } else if (strncmp(argv[arg], "-testexhaustive", 8) == 0) {
- exit(testExhaustive(argv[arg+1], argv[arg+2], mersize));
-
- } else if (strncmp(argv[arg], "-build", 2) == 0) {
- existDB *e = new existDB(argv[argc-2], mersize, existDBnoFlags, 0, ~u32bitZERO);
- e->saveState(argv[argc-1]);
- delete e;
- exit(0);
- }
-
- arg++;
- }
-
- exit(0);
-}
diff --git a/kmer/libkmer/driver-posDB.C b/kmer/libkmer/driver-posDB.C
deleted file mode 100644
index 206e4f7..0000000
--- a/kmer/libkmer/driver-posDB.C
+++ /dev/null
@@ -1,287 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-
-#include "bio++.H"
-#include "existDB.H"
-#include "positionDB.H"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-// Driver for the positionDB creation. Reads a sequence.fasta, builds
-// a positionDB for the mers in the file, and then writes the internal
-// structures to disk.
-//
-// The positionDB constructor is smart enough to read either a pre-built
-// image or a regular multi-fasta file.
-
-
-#define MERSIZE 20
-
-
-int
-test1(char *filename) {
- merStream *T = new merStream(new kMerBuilder(MERSIZE), new seqStream(filename), true, true);
- positionDB *M = new positionDB(T, MERSIZE, 0, 0L, 0L, 0L, 0, 0, 0, 0, true);
- u64bit *posn = new u64bit [1024];
- u64bit posnMax = 1024;
- u64bit posnLen = u64bitZERO;
- u64bit count = u64bitZERO;
- u32bit missing = u32bitZERO;
- u32bit failed = u32bitZERO;
- char str[33];
-
- T->rewind();
-
- while (T->nextMer()) {
- if (M->getExact(T->theFMer(),
- posn,
- posnMax,
- posnLen,
- count)) {
-
- missing = u32bitZERO;
- for (u32bit i=0; i<posnLen; i++)
- if (posn[i] == T->thePositionInStream())
- missing++;
-
- if (missing != 1) {
- failed++;
-
- fprintf(stdout, "%s @ "u64bitFMT"/"u64bitFMT": Found "u64bitFMT" table entries, and "u32bitFMT" matching positions (",
- T->theFMer().merToString(str), T->theSequenceNumber(), T->thePositionInStream(), posnLen, missing);
-
- for (u32bit i=0; i<posnLen; i++) {
- fprintf(stdout, u64bitFMT, posn[i]);
- if (i < posnLen - 1)
- fprintf(stdout, " ");
- else
- fprintf(stdout, ")\n");
- }
- }
- } else {
- failed++;
-
- fprintf(stdout, "Found no matches for mer=%s at pos="u64bitFMT"\n",
- T->theFMer().merToString(str), T->thePositionInStream());
- }
- }
-
- delete M;
- delete T;
-
- return(failed != 0);
-}
-
-
-
-int
-test2(char *filename, char *query) {
- merStream *T = new merStream(new kMerBuilder(MERSIZE), new seqStream(filename), true, true);
- positionDB *M = new positionDB(T, MERSIZE, 0, 0L, 0L, 0L, 0, 0, 0, 0, true);
- u64bit *posn = new u64bit [1024];
- u64bit posnMax = 1024;
- u64bit posnLen = u64bitZERO;
- u64bit count = u64bitZERO;
- char str[33];
-
- delete T;
-
- T = new merStream(new kMerBuilder(MERSIZE), new seqStream(query), true, true);
-
- while (T->nextMer()) {
- if (M->getExact(T->theFMer(),
- posn,
- posnMax,
- posnLen,
- count)) {
- fprintf(stdout, "Got a F match for mer=%s at "u64bitFMT"/"u64bitFMT" (in mers), numMatches="u64bitFMT"\n",
- T->theFMer().merToString(str), T->theSequenceNumber(), T->thePositionInStream(), posnLen);
- }
-
- if (M->getExact(T->theRMer(),
- posn,
- posnMax,
- posnLen,
- count)) {
- fprintf(stdout, "Got a R match for mer=%s at "u64bitFMT"/"u64bitFMT" (in mers), numMatches="u64bitFMT"\n",
- T->theRMer().merToString(str), T->theSequenceNumber(), T->thePositionInStream(), posnLen);
- }
- }
-
- delete M;
- delete T;
-
- return(0);
-}
-
-
-
-// Builds a positionDB possibly using a subset of the file.
-//
-// Subset on entire sequences:
-// -use x-y,a,b
-//
-// Subset on a range of mers, in this case, use only the 1000th
-// through 1999th (inclusive) mer:
-// -merbegin 1000 -merend 2000
-//
-// Or do both, use the first 1000 mers from the 3rd sequence:
-// -use 3 -merbegin 0 -merend 1000
-
-
-
-int
-main(int argc, char **argv) {
- u32bit mersize = 20;
- u32bit merskip = 0;
-
- char *maskF = 0L;
- char *onlyF = 0L;
-
- u64bit merBegin = ~u64bitZERO;
- u64bit merEnd = ~u64bitZERO;
-
- char *sequenceFile = 0L;
- char *outputFile = 0L;
-
- if (argc < 3) {
- fprintf(stderr, "usage: %s [args]\n", argv[0]);
- fprintf(stderr, " -mersize k The size of the mers, default=20.\n");
- fprintf(stderr, " -merskip k The skip between mers, default=0\n");
- fprintf(stderr, " -use a-b,c Specify which sequences to use, default=all\n");
- fprintf(stderr, " -merbegin b Build on a subset of the mers, starting at mer #b, default=all mers\n");
- fprintf(stderr, " -merend e Build on a subset of the mers, ending at mer #e, default=all mers\n");
- fprintf(stderr, " -sequence s.fasta Input sequences.\n");
- fprintf(stderr, " -output p.posDB Output filename.\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " To dump information about an image:\n");
- fprintf(stderr, " -dump datafile\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " To run sanity tests:\n");
- fprintf(stderr, " -buildonly [build opts] sequence.fasta\n");
- fprintf(stderr, " -- just builds a table and exits\n");
- fprintf(stderr, " -existence [build opts] sequence.fasta\n");
- fprintf(stderr, " -- builds (or reads) a table reports if any mers\n");
- fprintf(stderr, " in sequence.fasta cannot be found\n");
- fprintf(stderr, " -extra [build opts] sequence.fasta\n");
- fprintf(stderr, " -- builds (or reads) a table reports if any mers\n");
- fprintf(stderr, " NOT in sequence.fasta are be found\n");
- fprintf(stderr, " -test1 sequence.fasta\n");
- fprintf(stderr, " -- Tests if each and every mer is found in the\n");
- fprintf(stderr, " positionDB. Reports if it doesn't find a mer\n");
- fprintf(stderr, " at the correct position. Doesn't report if table\n");
- fprintf(stderr, " has too much stuff.\n");
- fprintf(stderr, " -test2 db.fasta sequence.fasta\n");
- fprintf(stderr, " -- Builds a positionDB from db.fasta, then searches\n");
- fprintf(stderr, " the table for each mer in sequence.fasta. Reports\n");
- fprintf(stderr, " all mers it finds.\n");
- fprintf(stderr, " -- This is a silly test and you shouldn't do it.\n");
- exit(1);
- }
-
- int arg = 1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-mersize") == 0) {
- mersize = strtou32bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-merskip") == 0) {
- merskip = strtou32bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-mask") == 0) {
- maskF = argv[++arg];
- } else if (strcmp(argv[arg], "-only") == 0) {
- onlyF = argv[++arg];
-
- } else if (strcmp(argv[arg], "-merbegin") == 0) {
- merBegin = strtou64bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-merend") == 0) {
- merEnd = strtou64bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-sequence") == 0) {
- sequenceFile = argv[++arg];
-
- } else if (strcmp(argv[arg], "-output") == 0) {
- outputFile = argv[++arg];
-
- } else if (strcmp(argv[arg], "-dump") == 0) {
- positionDB *e = new positionDB(argv[++arg], 0, 0, 0, false);
- e->printState(stdout);
- delete e;
- exit(0);
- } else if (strcmp(argv[arg], "-test1") == 0) {
- exit(test1(argv[arg+1]));
- } else if (strcmp(argv[arg], "-test2") == 0) {
- exit(test2(argv[arg+1], argv[arg+2]));
- } else {
- fprintf(stderr, "ERROR: unknown arg '%s'\n", argv[arg]);
- exit(1);
- }
-
- arg++;
- }
-
- // Exit quickly if the output file exists.
- //
- if (fileExists(outputFile)) {
- fprintf(stderr, "Output file '%s' exists already!\n", outputFile);
- exit(0);
- }
-
-
- merStream *MS = new merStream(new kMerBuilder(MERSIZE),
- new seqStream(sequenceFile),
- true, true);
-
- // Approximate the number of mers in the sequences.
- //
- u64bit numMers = MS->approximateNumberOfMers();
-
- // Reset the limits.
- //
- // XXX: If the user somehow knows how many mers are in the input
- // file, and specifies an end between there and the amount of
- // sequence, we'll pointlessly still make a merStreamFile, even
- // though we shouldn't.
- //
- if (merBegin == ~u64bitZERO) merBegin = 0;
- if (merEnd == ~u64bitZERO) merEnd = numMers;
-
- if (merBegin >= merEnd) {
- fprintf(stderr, "ERROR: merbegin="u64bitFMT" and merend="u64bitFMT" are incompatible.\n",
- merBegin, merEnd);
- exit(1);
- }
-
- if ((merBegin > 0) || (merEnd < numMers))
- MS->setBaseRange(merBegin, merEnd);
-
- existDB *maskDB = 0L;
- if (maskF) {
- fprintf(stderr, "Building maskDB from '%s'\n", maskF);
- maskDB = new existDB(maskF, mersize, existDBnoFlags, 0, ~u32bitZERO);
- }
-
- existDB *onlyDB = 0L;
- if (onlyF) {
- fprintf(stderr, "Building onlyDB from '%s'\n", onlyF);
- onlyDB = new existDB(onlyF, mersize, existDBnoFlags, 0, ~u32bitZERO);
- }
-
- fprintf(stderr, "Building table with merSize "u32bitFMT", merSkip "u32bitFMT"\n", mersize, merskip);
-
- positionDB *positions = new positionDB(MS, mersize, merskip, maskDB, onlyDB, 0L, 0, 0, 0, 0, true);
-
- fprintf(stderr, "Dumping positions table to '%s'\n", outputFile);
-
- positions->saveState(outputFile);
-
- delete MS;
- delete positions;
-
- exit(0);
-}
diff --git a/kmer/libkmer/existDB-create-from-fasta.C b/kmer/libkmer/existDB-create-from-fasta.C
deleted file mode 100644
index 2fc7921..0000000
--- a/kmer/libkmer/existDB-create-from-fasta.C
+++ /dev/null
@@ -1,270 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include "existDB.H"
-#include "bio++.H"
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-bool
-existDB::createFromFastA(char const *filename,
- u32bit merSize,
- u32bit flags) {
-
- bool beVerbose = false;
- bool rebuilding = false;
-
- _hashTable = 0L;
- _buckets = 0L;
- _counts = 0L;
-
- _merSizeInBases = merSize;
-
- _searchForDupe = true;
-
- if ((flags & existDBcompressHash) ||
- (flags & existDBcompressBuckets) ||
- (flags & existDBcompressCounts))
- fprintf(stderr, "existDB::createFromSequence: compression not supported.\n"), exit(1);
-
- // This (at =22) eats up 16MB, and should allow a lot of mers at big sizes. Unfortunately, we
- // know nothing about how man mers are going to be in the input.
- //
- // Setting this too high drastically reduces performance, suspected because of cache misses.
- // Setting this too low will also reduce performance, by increasing the search time in a bucket.
- //
- u32bit tblBits = logBaseTwo64(sizeOfFile(filename));
-
- rebuild:
- _shift1 = 2 * _merSizeInBases - tblBits;
- _shift2 = _shift1 / 2;
- _mask1 = u64bitMASK(tblBits);
- _mask2 = u64bitMASK(_shift1);
-
- _hshWidth = u32bitZERO;
- _chkWidth = 2 * merSize - tblBits;
- _cntWidth = 16;
-
- u64bit tableSizeInEntries = u64bitONE << tblBits;
- u64bit numberOfMers = u64bitZERO;
- u64bit *countingTable = new u64bit [tableSizeInEntries + 1];
-
- for (u64bit i=tableSizeInEntries+1; i--; )
- countingTable[i] = 0;
-
- _isCanonical = flags & existDBcanonical;
- _isForward = flags & existDBforward;
-
- assert(_isCanonical + _isForward == 1);
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 1) Count bucket sizes
- //
- merStream *M = new merStream(new kMerBuilder(_merSizeInBases),
- new seqStream(filename),
- true, true);
-
- while (M->nextMer()) {
- if (_isForward) {
- countingTable[ HASH(M->theFMer()) ]++;
- numberOfMers++;
- }
-
- if (_isCanonical) {
- countingTable[ HASH(M->theCMer()) ]++;
- numberOfMers++;
- }
- }
-
- delete M;
-
-#ifdef STATS
- u64bit dist[32] = {0};
- u64bit maxcnt = 0;
- for (u64bit i=tableSizeInEntries+1; i--; ) {
- if (countingTable[i] > maxcnt)
- maxcnt = countingTable[i];
-
- if (countingTable[i] < 32)
- dist[countingTable[i]]++;
- }
-
- for(u64bit i=0; i<32; i++)
- fprintf(stderr, "existDB::usage[%2d] = %d\n", i, dist[i]);
- fprintf(stderr, "existDB::maxcnt = %d\n", maxcnt);
-#endif
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Determine how many bits we need to hold the value
- // numberOfMers.....then....
- //
- // This is numberOfMers+1 because we need to store the
- // first position after the last mer. That is, if there are two
- // mers, we will store that the first mer is at position 0, the
- // second mer is at position 1, and the end of the second mer is at
- // position 2.
- //
- if (_compressedHash) {
- _hshWidth = 1;
- while ((numberOfMers+1) > (u64bitONE << _hshWidth))
- _hshWidth++;
- }
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 2) Allocate a hash table and some mer storage buckets.
- //
- _hashTableWords = tableSizeInEntries + 2;
- if (_compressedHash)
- _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
-
- _bucketsWords = numberOfMers + 2;
- if (_compressedBucket)
- _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
-
- _countsWords = numberOfMers + 2;
- if (_compressedCounts)
- _countsWords = _countsWords * _cntWidth / 64 + 1;
-
- if (beVerbose) {
- fprintf(stderr, "existDB::createFromFastA()-- hashTable is "u64bitFMT"MB\n", _hashTableWords >> 17);
- fprintf(stderr, "existDB::createFromFastA()-- buckets is "u64bitFMT"MB\n", _bucketsWords >> 17);
- if (flags & existDBcounts)
- fprintf(stderr, "existDB::createFromFastA()-- counts is "u64bitFMT"MB\n", _countsWords >> 17);
- }
-
- _hashTable = new u64bit [_hashTableWords];
- _buckets = new u64bit [_bucketsWords];
- _countsWords = (flags & existDBcounts) ? _countsWords : 0;
- _counts = (flags & existDBcounts) ? new u64bit [_countsWords] : 0L;
-
- // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
- // is also cleared as it is initialized, but in the _compressedHash case, the last
- // few words might be uninitialized. They're unused.
- //
- //memset(_hashTable, 0, sizeof(u64bit) * _hashTableWords);
- //memset(_buckets, 0, sizeof(u64bit) * _bucketsWords); // buckets is cleared as it is built
- //memset(_counts, 0, sizeof(u64bit) * _countsWords);
-
- _hashTable[_hashTableWords-1] = 0;
- _hashTable[_hashTableWords-2] = 0;
- _hashTable[_hashTableWords-3] = 0;
- _hashTable[_hashTableWords-4] = 0;
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Make the hash table point to the start of the bucket, and reset
- // the counting table -- we're going to use it to fill the buckets.
- //
- u64bit tmpPosition = 0;
- u64bit begPosition = 0;
- u64bit ptr = 0;
-
- if (_compressedHash) {
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- tmpPosition = countingTable[i];
- countingTable[i] = begPosition;
-
- setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
- ptr += _hshWidth;
-
- begPosition += tmpPosition;
- }
-
- setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
- } else {
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- tmpPosition = countingTable[i];
- countingTable[i] = begPosition;
-
- _hashTable[i] = begPosition;
-
- begPosition += tmpPosition;
- }
-
- // Set the last position in the hash, but we don't care about
- // the temporary counting table.
- //
- _hashTable[tableSizeInEntries] = begPosition;
- }
-
-
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 3) Build list of mers, placed into buckets
- //
- M = new merStream(new kMerBuilder(_merSizeInBases),
- new seqStream(filename),
- true, true);
-
- while (M->nextMer()) {
- if (_isForward)
- insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), 1, countingTable);
-
- if (_isCanonical)
- insertMer(HASH(M->theCMer()), CHECK(M->theCMer()), 1, countingTable);
- }
-
- delete M;
-
- // Compress out the gaps we have from redundant kmers.
-
- u64bit pos = 0;
- u64bit frm = 0;
- u64bit len = 0;
-
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- frm = _hashTable[i];
- len = countingTable[i] - _hashTable[i];
-
- _hashTable[i] = pos;
-
- for (u64bit j=0; j<len; j++) {
- if (_counts)
- _counts[pos] = _counts[frm];
-
- _buckets[pos++] = _buckets[frm++];
- }
- }
-
- fprintf(stderr, "Compressed from "u64bitFMT" to "u64bitFMT" ("u64bitFMT" bits)\n",
- _hashTable[tableSizeInEntries], pos, logBaseTwo64(pos));
-
- while (pos < _bucketsWords)
- _buckets[pos++] = 0;
-
- _hashTable[tableSizeInEntries] = pos;
-
- // All done. Delete temporary stuff
- //
- delete [] countingTable;
-
- // But if we horribly screwed up the estimate of tblBits, reset and recompute
-
- if ((logBaseTwo64(pos) < tblBits) &&
- (rebuilding == false)) {
- rebuilding = true;
-
- delete [] _hashTable;
- delete [] _buckets;
- delete [] _counts;
-
- _hashTable = 0L;
- _buckets = 0L;
- _counts = 0L;
-
- tblBits = logBaseTwo64(pos);
-
- goto rebuild;
- }
-
- return(true);
-}
diff --git a/kmer/libkmer/existDB-create-from-meryl.C b/kmer/libkmer/existDB-create-from-meryl.C
deleted file mode 100644
index bcbe172..0000000
--- a/kmer/libkmer/existDB-create-from-meryl.C
+++ /dev/null
@@ -1,230 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include "existDB.H"
-#include "libmeryl.H"
-
-
-bool
-existDB::createFromMeryl(char const *prefix,
- u32bit merSize,
- u32bit lo,
- u32bit hi,
- u32bit flags) {
-
- merylStreamReader *M = new merylStreamReader(prefix);
-
- bool beVerbose = false;
-
- _hashTable = 0L;
- _buckets = 0L;
- _counts = 0L;
-
- _merSizeInBases = M->merSize();
-
- if (merSize != _merSizeInBases) {
- fprintf(stderr, "createFromMeryl()-- ERROR: requested merSize ("u32bitFMT") is different than merSize in meryl database ("u32bitFMT").\n",
- merSize, _merSizeInBases);
- exit(1);
- }
-
- // We can set this exactly, but not memory optimal (see meryl/estimate.C:optimalNumberOfBuckets()).
- // Instead, we just blindly use whatever meryl used.
- //
- u32bit tblBits = M->prefixSize();
-
- // But it is faster to reset to this. Might use 2x the memory.
- //u32bit tblBits = logBaseTwo64(M->numberOfDistinctMers() + 1);
-
- _shift1 = 2 * _merSizeInBases - tblBits;
- _shift2 = _shift1 / 2;
- _mask1 = u64bitMASK(tblBits);
- _mask2 = u64bitMASK(_shift1);
-
- _hshWidth = u32bitZERO;
- _chkWidth = 2 * _merSizeInBases - tblBits;
- _cntWidth = 16;
-
- u64bit tableSizeInEntries = u64bitONE << tblBits;
- u64bit numberOfMers = u64bitZERO;
- u64bit *countingTable = new u64bit [tableSizeInEntries + 1];
-
- if (beVerbose) {
- fprintf(stderr, "createFromMeryl()-- tableSizeInEntries "u64bitFMT"\n", tableSizeInEntries);
- fprintf(stderr, "createFromMeryl()-- count range "u32bitFMT"-"u32bitFMT"\n", lo, hi);
- }
-
- for (u64bit i=tableSizeInEntries+1; i--; )
- countingTable[i] = 0;
-
- _isCanonical = flags & existDBcanonical;
- _isForward = flags & existDBforward;
-
- if (beVerbose) {
- fprintf(stderr, "createFromMeryl()-- canonical %c\n", (_isCanonical) ? 'T' : 'F');
- fprintf(stderr, "createFromMeryl()-- forward %c\n", (_isForward) ? 'T' : 'F');
- }
-
- assert(_isCanonical + _isForward == 1);
-
- // 1) Count bucket sizes
- // While we don't know the bucket sizes right now, but we do know
- // how many buckets and how many mers.
- //
- // Because we could be inserting both forward and reverse, we can't
- // really move the direction testing outside the loop, unless we
- // want to do two iterations over M.
- //
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
-
- while (M->nextMer()) {
- if ((lo <= M->theCount()) && (M->theCount() <= hi)) {
- if (_isForward) {
- countingTable[ HASH(M->theFMer()) ]++;
- numberOfMers++;
- }
-
- if (_isCanonical) {
- kMer r = M->theFMer();
- r.reverseComplement();
-
- if (M->theFMer() < r)
- countingTable[ HASH(M->theFMer()) ]++;
- else
- countingTable[ HASH(r) ]++;
- numberOfMers++;
- }
-
- C->tick();
- }
- }
-
- if (beVerbose)
- fprintf(stderr, "createFromMeryl()-- numberOfMers "u64bitFMT"\n", numberOfMers);
-
- delete C;
- delete M;
-
- if (_compressedHash) {
- _hshWidth = 1;
- while ((numberOfMers+1) > (u64bitONE << _hshWidth))
- _hshWidth++;
- }
-
- if (beVerbose) {
- fprintf(stderr, "existDB::createFromMeryl()-- Found "u64bitFMT" mers between count of "u32bitFMT" and "u32bitFMT"\n",
- numberOfMers, lo, hi);
- }
-
- // 2) Allocate hash table, mer storage buckets
- //
- _hashTableWords = tableSizeInEntries + 2;
- if (_compressedHash)
- _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
-
- _bucketsWords = numberOfMers + 2;
- if (_compressedBucket)
- _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
-
- _countsWords = numberOfMers + 2;
- if (_compressedCounts)
- _countsWords = _countsWords * _cntWidth / 64 + 1;
-
- if (beVerbose) {
- fprintf(stderr, "existDB::createFromMeryl()-- hashTable is "u64bitFMT"MB\n", _hashTableWords >> 17);
- fprintf(stderr, "existDB::createFromMeryl()-- buckets is "u64bitFMT"MB\n", _bucketsWords >> 17);
- if (flags & existDBcounts)
- fprintf(stderr, "existDB::createFromMeryl()-- counts is "u64bitFMT"MB\n", _countsWords >> 17);
- }
-
- _hashTable = new u64bit [_hashTableWords];
- _buckets = new u64bit [_bucketsWords];
- _countsWords = (flags & existDBcounts) ? _countsWords : 0;
- _counts = (flags & existDBcounts) ? new u64bit [_countsWords] : 0L;
-
- // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
- // is also cleared as it is initialized, but in the _compressedHash case, the last
- // few words might be uninitialized. They're unused.
-
- //memset(_hashTable, 0, sizeof(u64bit) * _hashTableWords);
- //memset(_buckets, 0, sizeof(u64bit) * _bucketsWords); // buckets is cleared as it is built
- //memset(_counts, 0, sizeof(u64bit) * _countsWords);
-
- _hashTable[_hashTableWords-1] = 0;
- _hashTable[_hashTableWords-2] = 0;
- _hashTable[_hashTableWords-3] = 0;
- _hashTable[_hashTableWords-4] = 0;
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Make the hash table point to the start of the bucket, and reset
- // the counting table -- we're going to use it to fill the buckets.
- //
- u64bit tmpPosition = 0;
- u64bit begPosition = 0;
- u64bit ptr = 0;
-
- if (_compressedHash) {
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- tmpPosition = countingTable[i];
- countingTable[i] = begPosition;
-
- setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
- ptr += _hshWidth;
-
- begPosition += tmpPosition;
- }
-
- setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
- } else {
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- tmpPosition = countingTable[i];
- countingTable[i] = begPosition;
-
- _hashTable[i] = begPosition;
-
- begPosition += tmpPosition;
- }
-
- // Set the last position in the hash, but we don't care about
- // the temporary counting table.
- //
- _hashTable[tableSizeInEntries] = begPosition;
- }
-
-
- ///////////////////////////////////////////////////////////////////////////////
- //
- // 3) Build list of mers, placed into buckets
- //
- M = new merylStreamReader(prefix);
- C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
-
- while (M->nextMer()) {
- if ((lo <= M->theCount()) && (M->theCount() <= hi)) {
- if (_isForward)
- insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable);
-
- if (_isCanonical) {
- kMer r = M->theFMer();
- r.reverseComplement();
-
- if (M->theFMer() < r)
- insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable);
- else
- insertMer(HASH(r), CHECK(r), M->theCount(), countingTable);
- numberOfMers++;
- }
-
-
- C->tick();
- }
- }
-
- delete C;
- delete M;
- delete [] countingTable;
-
- return(true);
-}
diff --git a/kmer/libkmer/existDB-create-from-sequence.C b/kmer/libkmer/existDB-create-from-sequence.C
deleted file mode 100644
index 593e9ff..0000000
--- a/kmer/libkmer/existDB-create-from-sequence.C
+++ /dev/null
@@ -1,272 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include "existDB.H"
-#include "bio++.H"
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-bool
-existDB::createFromSequence(char const *sequence,
- u32bit merSize,
- u32bit flags) {
-
- bool beVerbose = false;
- bool rebuilding = false;
-
- _hashTable = 0L;
- _buckets = 0L;
- _counts = 0L;
-
- _merSizeInBases = merSize;
-
- _searchForDupe = true;
-
- if ((flags & existDBcompressHash) ||
- (flags & existDBcompressBuckets) ||
- (flags & existDBcompressCounts))
- fprintf(stderr, "existDB::createFromSequence: compression not supported.\n"), exit(1);
-
- // This (at =22) eats up 16MB, and should allow a lot of mers at big sizes. Unfortunately, we
- // know nothing about how man mers are going to be in the input.
- //
- // Setting this too high drastically reduces performance, suspected because of cache misses.
- // Setting this too low will also reduce performance, by increasing the search time in a bucket.
- //
- u32bit tblBits = logBaseTwo64(strlen(sequence));
-
- rebuild:
- fprintf(stderr, "tblBits %d seqlen %d\n", tblBits, strlen(sequence));
-
- _shift1 = 2 * _merSizeInBases - tblBits;
- _shift2 = _shift1 / 2;
- _mask1 = u64bitMASK(tblBits);
- _mask2 = u64bitMASK(_shift1);
-
- _hshWidth = u32bitZERO;
- _chkWidth = 2 * merSize - tblBits;
- _cntWidth = 16;
-
- u64bit tableSizeInEntries = u64bitONE << tblBits;
- u64bit numberOfMers = u64bitZERO;
- u64bit *countingTable = new u64bit [tableSizeInEntries + 1];
-
- for (u64bit i=tableSizeInEntries+1; i--; )
- countingTable[i] = 0;
-
- _isCanonical = flags & existDBcanonical;
- _isForward = flags & existDBforward;
-
- assert(_isCanonical + _isForward == 1);
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 1) Count bucket sizes
- //
- merStream *M = new merStream(new kMerBuilder(_merSizeInBases),
- new seqStream(sequence, strlen(sequence)),
- true, true);
-
- while (M->nextMer()) {
- if (_isForward) {
- countingTable[ HASH(M->theFMer()) ]++;
- numberOfMers++;
- }
-
- if (_isCanonical) {
- countingTable[ HASH(M->theCMer()) ]++;
- numberOfMers++;
- }
- }
-
- delete M;
-
-#ifdef STATS
- u64bit dist[32] = {0};
- u64bit maxcnt = 0;
- for (u64bit i=tableSizeInEntries+1; i--; ) {
- if (countingTable[i] > maxcnt)
- maxcnt = countingTable[i];
-
- if (countingTable[i] < 32)
- dist[countingTable[i]]++;
- }
-
- for(u64bit i=0; i<32; i++)
- fprintf(stderr, "existDB::usage[%2d] = %d\n", i, dist[i]);
- fprintf(stderr, "existDB::maxcnt = %d\n", maxcnt);
-#endif
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Determine how many bits we need to hold the value
- // numberOfMers.....then....
- //
- // This is numberOfMers+1 because we need to store the
- // first position after the last mer. That is, if there are two
- // mers, we will store that the first mer is at position 0, the
- // second mer is at position 1, and the end of the second mer is at
- // position 2.
- //
- if (_compressedHash) {
- _hshWidth = 1;
- while ((numberOfMers+1) > (u64bitONE << _hshWidth))
- _hshWidth++;
- }
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 2) Allocate a hash table and some mer storage buckets.
- //
- _hashTableWords = tableSizeInEntries + 2;
- if (_compressedHash)
- _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
-
- _bucketsWords = numberOfMers + 2;
- if (_compressedBucket)
- _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
-
- _countsWords = numberOfMers + 2;
- if (_compressedCounts)
- _countsWords = _countsWords * _cntWidth / 64 + 1;
-
- if (beVerbose) {
- fprintf(stderr, "existDB::createFromSequence()-- hashTable is "u64bitFMT"MB\n", _hashTableWords >> 17);
- fprintf(stderr, "existDB::createFromSequence()-- buckets is "u64bitFMT"MB\n", _bucketsWords >> 17);
- if (flags & existDBcounts)
- fprintf(stderr, "existDB::createFromSequence()-- counts is "u64bitFMT"MB\n", _countsWords >> 17);
- }
-
- _hashTable = new u64bit [_hashTableWords];
- _buckets = new u64bit [_bucketsWords];
- _countsWords = (flags & existDBcounts) ? _countsWords : 0;
- _counts = (flags & existDBcounts) ? new u64bit [_countsWords] : 0L;
-
- // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
- // is also cleared as it is initialized, but in the _compressedHash case, the last
- // few words might be uninitialized. They're unused.
-
- //memset(_hashTable, 0, sizeof(u64bit) * _hashTableWords);
- //memset(_buckets, 0, sizeof(u64bit) * _bucketsWords); // buckets is cleared as it is built
- //memset(_counts, 0, sizeof(u64bit) * _countsWords);
-
- _hashTable[_hashTableWords-1] = 0;
- _hashTable[_hashTableWords-2] = 0;
- _hashTable[_hashTableWords-3] = 0;
- _hashTable[_hashTableWords-4] = 0;
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Make the hash table point to the start of the bucket, and reset
- // the counting table -- we're going to use it to fill the buckets.
- //
- u64bit tmpPosition = 0;
- u64bit begPosition = 0;
- u64bit ptr = 0;
-
- if (_compressedHash) {
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- tmpPosition = countingTable[i];
- countingTable[i] = begPosition;
-
- setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
- ptr += _hshWidth;
-
- begPosition += tmpPosition;
- }
-
- setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
- } else {
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- tmpPosition = countingTable[i];
- countingTable[i] = begPosition;
-
- _hashTable[i] = begPosition;
-
- begPosition += tmpPosition;
- }
-
- // Set the last position in the hash, but we don't care about
- // the temporary counting table.
- //
- _hashTable[tableSizeInEntries] = begPosition;
- }
-
-
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 3) Build list of mers, placed into buckets
- //
- M = new merStream(new kMerBuilder(_merSizeInBases),
- new seqStream(sequence, strlen(sequence)),
- true, true);
-
- while (M->nextMer()) {
- if (_isForward)
- insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), 1, countingTable);
-
- if (_isCanonical)
- insertMer(HASH(M->theCMer()), CHECK(M->theCMer()), 1, countingTable);
- }
-
- delete M;
-
- // Compress out the gaps we have from redundant kmers.
-
- u64bit pos = 0;
- u64bit frm = 0;
- u64bit len = 0;
-
- for (u64bit i=0; i<tableSizeInEntries; i++) {
- frm = _hashTable[i];
- len = countingTable[i] - _hashTable[i];
-
- _hashTable[i] = pos;
-
- for (u64bit j=0; j<len; j++) {
- if (_counts)
- _counts[pos] = _counts[frm];
-
- _buckets[pos++] = _buckets[frm++];
- }
- }
-
- fprintf(stderr, "Compressed from "u64bitFMT" to "u64bitFMT" ("u64bitFMT" bits)\n",
- _hashTable[tableSizeInEntries], pos, logBaseTwo64(pos));
-
- while (pos < _bucketsWords)
- _buckets[pos++] = 0;
-
- _hashTable[tableSizeInEntries] = pos;
-
- // All done. Delete temporary stuff
- //
- delete [] countingTable;
-
- // But if we horribly screwed up the estimate of tblBits, reset and recompute
-
- if ((logBaseTwo64(pos) < tblBits) &&
- (rebuilding == false)) {
- rebuilding = true;
-
- delete [] _hashTable;
- delete [] _buckets;
- delete [] _counts;
-
- _hashTable = 0L;
- _buckets = 0L;
- _counts = 0L;
-
- tblBits = logBaseTwo64(pos);
-
- goto rebuild;
- }
-
- return(true);
-}
diff --git a/kmer/libkmer/existDB-state.C b/kmer/libkmer/existDB-state.C
deleted file mode 100644
index e991de8..0000000
--- a/kmer/libkmer/existDB-state.C
+++ /dev/null
@@ -1,205 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include "existDB.H"
-#include "bio++.H"
-
-
-const char magic[16] = { 'e', 'x', 'i', 's', 't', 'D', 'B', '2',
- ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' };
-
-
-void
-existDB::saveState(char const *filename) {
- char cigam[16] = { 0 };
-
- errno = 0;
- FILE *F = fopen(filename, "wb");
- if (errno) {
- fprintf(stderr, "Can't open '%s' for writing\n%s\n", filename, strerror(errno));
- exit(1);
- }
-
- strncpy(cigam, magic, 16);
-
- if (_compressedHash)
- cigam[8] = 'h';
- if (_compressedBucket)
- cigam[9] = 'b';
- if (_compressedCounts)
- cigam[10] = 'c';
-
- if (_isForward)
- cigam[11] = 'F';
- if (_isCanonical)
- cigam[11] = 'C';
-
- fwrite(cigam, sizeof(char), 16, F);
-
- fwrite(&_merSizeInBases, sizeof(u32bit), 1, F);
- fwrite(&_shift1, sizeof(u32bit), 1, F);
- fwrite(&_shift2, sizeof(u32bit), 1, F);
- fwrite(&_mask1, sizeof(u64bit), 1, F);
- fwrite(&_mask2, sizeof(u64bit), 1, F);
- fwrite(&_hshWidth, sizeof(u32bit), 1, F); // only valid if _compressedHash
- fwrite(&_chkWidth, sizeof(u32bit), 1, F); // only valid if _compressedBucket
- fwrite(&_cntWidth, sizeof(u32bit), 1, F); // only valid if _compressedCounts
-
- fwrite(&_hashTableWords, sizeof(u64bit), 1, F);
- fwrite(&_bucketsWords, sizeof(u64bit), 1, F);
- fwrite(&_countsWords, sizeof(u64bit), 1, F);
-
- fwrite(_hashTable, sizeof(u64bit), _hashTableWords, F);
- fwrite(_buckets, sizeof(u64bit), _bucketsWords, F);
- fwrite(_counts, sizeof(u64bit), _countsWords, F);
-
- fclose(F);
-
- if (errno) {
- fprintf(stderr, "existDB::saveState()-- Write failure.\n%s\n", strerror(errno));
- exit(1);
- }
-}
-
-
-
-bool
-existDB::loadState(char const *filename,
- bool beNoisy,
- bool loadData) {
- char cigam[16];
-
- errno = 0;
- FILE *F = fopen(filename, "rb");
- if (errno) {
- //fprintf(stderr, "Can't open '%s' for reading pre-built existDB\n%s\n", strerror(errno));
- return(false);
- }
-
- fread(cigam, sizeof(char), 16, F);
-
- _compressedHash = false;
- _compressedBucket = false;
- _compressedCounts = false;
- _isForward = false;
- _isCanonical = false;
-
- if (cigam[8] == 'h')
- _compressedHash = true;
- if (cigam[9] == 'b')
- _compressedBucket = true;
- if (cigam[10] == 'c')
- _compressedCounts = true;
-
- if (cigam[11] == 'F')
- _isForward = true;
- if (cigam[11] == 'C')
- _isCanonical = true;
-
- cigam[ 8] = ' ';
- cigam[ 9] = ' ';
- cigam[10] = ' ';
- cigam[11] = ' ';
-
- if (strncmp(magic, cigam, 16) != 0) {
- if (beNoisy) {
- fprintf(stderr, "existDB::loadState()-- Not an existDB binary file, maybe a sequence file?\n");
- fprintf(stderr, "existDB::loadState()-- Read '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
- cigam[0], cigam[1], cigam[2], cigam[3],
- cigam[4], cigam[5], cigam[6], cigam[7],
- cigam[8], cigam[9], cigam[10], cigam[11],
- cigam[12], cigam[13], cigam[14], cigam[15]);
- fprintf(stderr, "existDB::loadState()-- Expected '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
- magic[0], magic[1], magic[2], magic[3],
- magic[4], magic[5], magic[6], magic[7],
- magic[8], magic[9], magic[10], magic[11],
- magic[12], magic[13], magic[14], magic[15]);
- }
-
- fclose(F);
- return(false);
- }
-
- fread(&_merSizeInBases, sizeof(u32bit), 1, F);
- fread(&_shift1, sizeof(u32bit), 1, F);
- fread(&_shift2, sizeof(u32bit), 1, F);
- fread(&_mask1, sizeof(u64bit), 1, F);
- fread(&_mask2, sizeof(u64bit), 1, F);
- fread(&_hshWidth, sizeof(u32bit), 1, F); // only valid if _compressedHash
- fread(&_chkWidth, sizeof(u32bit), 1, F); // only valid if _compressedBucket
- fread(&_cntWidth, sizeof(u32bit), 1, F); // only valid if _compressedCounts
-
- fread(&_hashTableWords, sizeof(u64bit), 1, F);
- fread(&_bucketsWords, sizeof(u64bit), 1, F);
- fread(&_countsWords, sizeof(u64bit), 1, F);
-
- _hashTable = 0L;
- _buckets = 0L;
- _counts = 0L;
-
- if (loadData) {
- _hashTable = new u64bit [_hashTableWords];
- _buckets = new u64bit [_bucketsWords];
-
- if (_countsWords > 0)
- _counts = new u64bit [_countsWords];
-
- fread(_hashTable, sizeof(u64bit), _hashTableWords, F);
- fread(_buckets, sizeof(u64bit), _bucketsWords, F);
-
- if (_countsWords > 0)
- fread(_counts, sizeof(u64bit), _countsWords, F);
- }
-
- fclose(F);
-
- if (errno) {
- fprintf(stderr, "existDB::loadState()-- Read failure.\n%s\n", strerror(errno));
- exit(1);
- }
-
- return(true);
-}
-
-
-void
-existDB::printState(FILE *stream) {
-
- fprintf(stream, "merSizeInBases: "u32bitFMT"\n", _merSizeInBases);
- fprintf(stream, "tableBits "u32bitFMT"\n", 2 * _merSizeInBases - _shift1);
- fprintf(stream, "-----------------\n");
- fprintf(stream, "_hashTableWords "u64bitFMT" ("u64bitFMT" KB)\n", _hashTableWords, _hashTableWords >> 7);
- fprintf(stream, "_bucketsWords "u64bitFMT" ("u64bitFMT" KB)\n", _bucketsWords, _bucketsWords >> 7);
- fprintf(stream, "_countsWords "u64bitFMT" ("u64bitFMT" KB)\n", _countsWords, _countsWords >> 7);
- fprintf(stream, "-----------------\n");
- fprintf(stream, "_shift1: "u32bitFMT"\n", _shift1);
- fprintf(stream, "_shift2 "u32bitFMT"\n", _shift2);
- fprintf(stream, "_mask1 "u64bitHEX"\n", _mask1);
- fprintf(stream, "_mask2 "u64bitHEX"\n", _mask2);
-
- if (_compressedHash) {
- fprintf(stream, "_compressedHash true\n");
- fprintf(stream, "_hshWidth "u32bitFMT"\n", _hshWidth);
- } else {
- fprintf(stream, "_compressedHash false\n");
- fprintf(stream, "_hshWidth undefined\n");
- }
-
- if (_compressedBucket) {
- fprintf(stream, "_compressedBucket true\n");
- fprintf(stream, "_chkWidth "u32bitFMT"\n", _chkWidth);
- } else {
- fprintf(stream, "_compressedBucket false\n");
- fprintf(stream, "_chkWidth undefined\n");
- }
-
- if (_compressedCounts) {
- fprintf(stream, "_compressedCount true\n");
- fprintf(stream, "_cntWidth "u32bitFMT"\n", _cntWidth);
- } else {
- fprintf(stream, "_compressedCount false\n");
- fprintf(stream, "_cntWidth undefined\n");
- }
-}
-
diff --git a/kmer/libkmer/existDB.C b/kmer/libkmer/existDB.C
deleted file mode 100644
index 4d767df..0000000
--- a/kmer/libkmer/existDB.C
+++ /dev/null
@@ -1,182 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include "existDB.H"
-#include "bio++.H"
-
-
-existDB::existDB(char const *filename,
- bool loadData) {
- clear();
-
- _compressedHash = false;
- _compressedBucket = false;
-
- if (loadState(filename, true, loadData) == false) {
- fprintf(stderr, "existDB::existDB()-- Tried to read state from '%s', but failed.\n", filename);
- exit(1);
- }
-}
-
-
-existDB::existDB(char const *filename,
- u32bit merSize,
- existDBflags flags,
- u32bit lo,
- u32bit hi) {
- clear();
-
- _compressedHash = flags & existDBcompressHash;
- _compressedBucket = flags & existDBcompressBuckets;
- _compressedCounts = flags & existDBcompressCounts;
-
- _searchForDupe = false;
-
- // Try to read state from the filename. If successful, make sure
- // that the merSize is correct.
- //
- if (loadState(filename)) {
- bool fail = false;
-
- if (_merSizeInBases != merSize) {
- fprintf(stderr, "existDB::existDB()-- Read state from '%s', but got different mer sizes\n", filename);
- fprintf(stderr, "existDB::existDB()-- Got "u32bitFMT", expected "u32bitFMT"\n", _merSizeInBases, merSize);
- fail = true;
- }
-
- if (fail)
- exit(1);
-
- return;
- }
-
- // If no direction flags are set, set the default direction of
- // forward. Stupid precedence rules.
- //
- if ((flags & (existDBcanonical | existDBforward)) == u32bitZERO)
- flags |= existDBforward;
-
- // If we can open 'filename' for reading, then we assume the file
- // is a multi-fasta, and we build an existDB/
- //
- // Otherwise, we assume that 'filename' is really the prefix for a
- // meryl database.
-
-
- if (fileExists(filename))
- createFromFastA(filename, merSize, flags);
- else
- createFromMeryl(filename, merSize, lo, hi, flags);
-}
-
-
-existDB::existDB(char const *sequence,
- u32bit merSize,
- existDBflags flags) {
- clear();
-
- _compressedHash = flags & existDBcompressHash;
- _compressedBucket = flags & existDBcompressBuckets;
- _compressedCounts = flags & existDBcompressCounts;
-
- if ((flags & (existDBcanonical | existDBforward)) == u32bitZERO)
- flags |= existDBforward;
-
- createFromSequence(sequence, merSize, flags);
-}
-
-
-existDB::~existDB() {
- delete [] _hashTable;
- delete [] _buckets;
- delete [] _counts;
-}
-
-
-
-
-
-bool
-existDB::exists(u64bit mer) {
- u64bit c, h, st, ed;
-
- if (_compressedHash) {
- h = HASH(mer) * _hshWidth;
- st = getDecodedValue(_hashTable, h, _hshWidth);
- ed = getDecodedValue(_hashTable, h + _hshWidth, _hshWidth);
- } else {
- h = HASH(mer);
- st = _hashTable[h];
- ed = _hashTable[h+1];
- }
-
- if (st == ed)
- return(false);
-
- c = CHECK(mer);
-
- if (_compressedBucket) {
- st *= _chkWidth;
- ed *= _chkWidth;
-
- for (; st<ed; st += _chkWidth) {
- if (getDecodedValue(_buckets, st, _chkWidth) == c)
- return(true);
- }
- } else {
- for (; st<ed; st++) {
- if (_buckets[st] == c)
- return(true);
- }
- }
-
- return(false);
-}
-
-
-u64bit
-existDB::count(u64bit mer) {
- u64bit c, h, st, ed;
-
- if (_counts == 0L)
- return(0);
-
- if (_compressedHash) {
- h = HASH(mer) * _hshWidth;
- st = getDecodedValue(_hashTable, h, _hshWidth);
- ed = getDecodedValue(_hashTable, h + _hshWidth, _hshWidth);
- } else {
- h = HASH(mer);
- st = _hashTable[h];
- ed = _hashTable[h+1];
- }
-
- if (st == ed)
- return(0);
-
- c = CHECK(mer);
-
- if (_compressedBucket) {
- st *= _chkWidth;
- ed *= _chkWidth;
-
- for (; st<ed; st += _chkWidth) {
- if (getDecodedValue(_buckets, st, _chkWidth) == c)
- goto returncount;
- }
- } else {
- for (; st<ed; st++) {
- if (_buckets[st] == c)
- goto returncount;
- }
- }
-
- return(0);
-
- returncount:
- if (_compressedCounts)
- return(getDecodedValue(_counts, st * _cntWidth, _cntWidth));
- else
- return(_counts[st]);
-}
diff --git a/kmer/libkmer/existDB.H b/kmer/libkmer/existDB.H
deleted file mode 100644
index e0c0ee1..0000000
--- a/kmer/libkmer/existDB.H
+++ /dev/null
@@ -1,151 +0,0 @@
-#ifndef EXISTDB_H
-#define EXISTDB_H
-
-// Used by wgs-assembler, to determine if a rather serious bug was patched.
-#define EXISTDB_H_VERSION 1960
-
-#include "bio++.H"
-
-// Takes as input a list of mers (in a file) and builds a searchable
-// structure listing those mers. Duplicate mers are not removed and
-// will be stored multiple times.
-//
-// Using a compressed hash is allowed, but somewhat useless -- it is
-// really slow and doesn't save that much.
-//
-// If existDBcanonical is requested, this will store only the
-// canonical mer. It is up to the client to be sure that is
-// appropriate! See positionDB.H for more.
-
-//#define STATS
-
-typedef u32bit existDBflags;
-const existDBflags existDBnoFlags = 0x0000;
-const existDBflags existDBcompressHash = 0x0001;
-const existDBflags existDBcompressBuckets = 0x0002;
-const existDBflags existDBcompressCounts = 0x0004;
-const existDBflags existDBcanonical = 0x0008;
-const existDBflags existDBforward = 0x0010;
-const existDBflags existDBcounts = 0x0020;
-
-class existDB {
-public:
-
- // Read state from an existDB file
- existDB(char const *filename,
- bool loadData=true);
-
- // Load mers from an existing existDB file, a fastafile, or a meryl database
- existDB(char const *filename,
- u32bit merSize,
- existDBflags flags,
- u32bit lo,
- u32bit hi);
-
- // Load mers from a character string
- existDB(char const *sequence,
- u32bit merSize,
- existDBflags flags);
-
- ~existDB();
-
- void saveState(char const *filename);
-
- void printState(FILE *stream);
-
- bool isForward(void) { return(_isForward); };
- bool isCanonical(void) { return(_isCanonical); };
-
- bool exists(u64bit mer);
- u64bit count(u64bit mer);
-
-private:
- bool loadState(char const *filename, bool beNoisy=false, bool loadData=true);
- bool createFromFastA(char const *filename,
- u32bit merSize,
- u32bit flags);
- bool createFromMeryl(char const *filename,
- u32bit merSize,
- u32bit lo,
- u32bit hi,
- u32bit flags);
- bool createFromSequence(char const *sequence,
- u32bit merSize,
- u32bit flags);
-
- u64bit HASH(u64bit k) {
- return(((k >> _shift1) ^ (k >> _shift2) ^ k) & _mask1);
- };
-
- u64bit CHECK(u64bit k) {
- return(k & _mask2);
- };
-
- void insertMer(u64bit hsh, u64bit chk, u64bit cnt, u64bit *countingTable) {
-
- // If the mer is already here, just update the count. This only
- // works if not _compressedBucket, and only makes sense for loading from
- // fasta or sequence.
-
- if ((_compressedBucket == false) &&
- (_searchForDupe)) {
- u64bit st = _hashTable[hsh];
- u64bit ed = countingTable[hsh];
-
- for (; st<ed; st++) {
- if (_buckets[st] == chk) {
- if (_counts)
- _counts[st] += cnt;
- return;
- }
- }
- }
-
- if (_compressedBucket)
- setDecodedValue(_buckets, countingTable[hsh] * _chkWidth, _chkWidth, chk);
- else
- _buckets[countingTable[hsh]] = chk;
-
- if (_counts) {
- if (_compressedCounts) {
- setDecodedValue(_counts, countingTable[hsh] * _cntWidth, _cntWidth, cnt);
- } else {
- _counts[countingTable[hsh]] = cnt;
- }
- }
-
- countingTable[hsh]++;
- };
-
- bool _compressedHash;
- bool _compressedBucket;
- bool _compressedCounts;
- bool _isForward;
- bool _isCanonical;
-
- bool _searchForDupe;
-
- u32bit _merSizeInBases;
-
- u32bit _shift1;
- u32bit _shift2;
- u64bit _mask1;
- u64bit _mask2;
-
- u32bit _hshWidth; // Only for the compressed hash
- u32bit _chkWidth; // Only for the compressed bucket
- u32bit _cntWidth; // Only for the compressed counts
-
- u64bit _hashTableWords;
- u64bit _bucketsWords;
- u64bit _countsWords;
-
- u64bit *_hashTable;
- u64bit *_buckets;
- u64bit *_counts;
-
- void clear(void) {
- };
-};
-
-#endif // EXISTDB_H
diff --git a/kmer/libkmer/iterate.C b/kmer/libkmer/iterate.C
deleted file mode 100644
index 0820582..0000000
--- a/kmer/libkmer/iterate.C
+++ /dev/null
@@ -1,64 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-
-#include "../libutil/util++.H"
-#include "hashIterator.H"
-
-
-int
-main(int argc, char **argv) {
-
- u64bit merorig = 0;
-
- u64bit hashorig = 0;
- u64bit hasherrors = 0;
-
- u32bit skipped = 0;
- u32bit usable = 0;
-
- // 1,000,000 hashes with three errors in 156u
-
- for (hashorig = 0; hashorig < 1000000; hashorig++) {
- hashIterator iter(26, 3);
-
- while (iter.next()) {
-
- // 40 == 01000000 in binary
- for (u64bit errors=0; errors < 0x00000040; errors++) {
-
- // Build the hash with errors
- hasherrors = hashorig;
- hasherrors &= ~(iter.error1() | iter.error2() | iter.error3());
-
- hasherrors |= (errors & 0x00000003) << (iter.shift1() - 0);
- hasherrors |= (errors & 0x0000000c) << (iter.shift2() - 2);
- hasherrors |= (errors & 0x00000030) << (iter.shift3() - 4);
-
- // If any of those errors aren't errors, we don't want to use
- // this hash, as it represents a hash with fewer errors that we
- // expected. The problem is that other error patterns would do
- // the same work; for example, if the error mask is 011001, and
- // the last 1 isn't an error, the mask 011010 does the same
- // work all over again.
-
- if (((hasherrors & iter.error1()) == (hashorig & iter.error1())) ||
- ((hasherrors & iter.error2()) == (hashorig & iter.error2())) ||
- ((hasherrors & iter.error3()) == (hashorig & iter.error3()))) {
- // Skip it.
- skipped++;
- //fprintf(stdout, "hash:"u64bitHEX" errs:"u64bitHEX" e1:"u64bitHEX" e2:"u64bitHEX" e3:"u64bitHEX" SKIP\n",
- // hasherrors, errors, iter.error1(), iter.error2(), iter.error3());
- } else {
- usable++;
- //fprintf(stdout, "hash:"u64bitHEX" errs:"u64bitHEX" e1:"u64bitHEX" e2:"u64bitHEX" e3:"u64bitHEX"\n",
- // hasherrors, errors, iter.error1(), iter.error2(), iter.error3());
- }
- }
- }
- }
-
- fprintf(stderr, "skipped "u32bitFMT" usable "u32bitFMT"\n", skipped, usable);
-
- return(0);
-}
diff --git a/kmer/libkmer/mask.C b/kmer/libkmer/mask.C
deleted file mode 100644
index 8836af2..0000000
--- a/kmer/libkmer/mask.C
+++ /dev/null
@@ -1,283 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "util++.H"
-#include "bio++.H"
-#include "libmeryl.H"
-#include "existDB.H"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-// 1) Use meryl to find the list of mers in common between HMISSING and NCBI.
-// 2) Read mers from that meryl database into an existDB.
-// 3) Stream each sequence from HMISSING. Mask out any mer in HMISSING.
-
-#define DEBUG
-
-int
-main(int argc, char **argv) {
- const char *merName = "/project/huref4/assembly-mapping/missing/missing0/HMISSING-and-B35LC";
- const char *seqName = "/project/huref4/assembly-mapping/missing/missing0/HMISSING-ge64.uid.fasta";
- u32bit merSize = 28;
-
- fprintf(stderr, "Build existDB.\n");
-
- existDB *exist = 0L;
- if (0) {
- exist = new existDB(merName, merSize, existDBnoFlags, 0, ~u32bitZERO);
- exist->saveState("/project/huref4/assembly-mapping/missing/missing0/HMISSING-and-B35LC.existDB");
- } else {
- exist = new existDB("/project/huref4/assembly-mapping/missing/missing0/HMISSING-and-B35LC.existDB");
- }
-
- seqCache *F = new seqCache(seqName);
- seqInCore *S = 0L;
-
- u32bit maskLen = 1048576;
- bool *mask = new bool [maskLen];
- char *maskSeq = new char [maskLen];
- char *maskBit = new char [maskLen];
-
- fprintf(stderr, "Begin.\n");
-
- while ((S = F->getSequenceInCore()) != 0L) {
- //fprintf(stderr, "iid="u32bitFMT" len="u32bitFMT" %s\n", S->getIID(), S->sequenceLength(), S->header());
-
- if (maskLen <= S->sequenceLength() + 1024) {
- maskLen = S->sequenceLength() + S->sequenceLength() + 1024;
-
- delete [] mask;
- delete [] maskSeq;
- delete [] maskBit;
-
- mask = new bool [maskLen];
- maskSeq = new char [maskLen];
- maskBit = new char [maskLen];
- }
-
- for (u32bit i=0; i<S->sequenceLength(); i++)
- mask[i] = false;
-
- // Build the initial masking
- //
- merStream *MS = new merStream(new kMerBuilder(merSize),
- new seqStream(S->sequence(), S->sequenceLength()),
- true, true);
- while (MS->nextMer())
- if (exist->exists(MS->theFMer()) || exist->exists(MS->theRMer()))
- mask[MS->thePositionInSequence()] = true;
- delete MS;
-
-
-#ifdef PRINT_BITS
- for (u32bit i=0; i<S->sequenceLength(); i++) {
- if (mask[i])
- maskBit[i] = '1';
- else
- maskBit[i] = '0';
- }
- maskBit[S->sequenceLength()] = 0;
- fprintf(stdout, "%s\n", maskBit);
-#endif
-
-
- // Clean up the masking.
- //
- // If there are no spurious mer matchs, a real mismatch (and also an insert) should look like:
- //
- // AAAAAAAAAAAAAAAAAAAAAAAAAXAAAAAAAA.....
- // 1111111111111111111110000011111111
- // -----
- // -----
- // -----
- // -----
- // -----
- // -----
- // -----
- // -----
- // The mers cover all letters, except for the mismatch.
- //
- //
- // A break in contiguous sequence (or a deletion) is very similar, except
- // mers cover all letters, they just don't span the gap.
- // AAAAAAAAAAAAAAAAAAAAAAAAACCCCCCCCCCCCCC....
- // 111111111111111111111000011111111
- // -----
- // -----
- // -----
- // -----
- // -----
- // -----
- // -----
- // -----
- //
- //
- // Thus, A block of fewer than merSize 0's indicates that we have
- // spurious mer hits, and don't find any contiguous sequence
- // match.
- //
- // This block marks the zeros and ms-1 bases on either side as questionable
- // sequence.
- //
- // I'm not sure if the blocks outside are questionable -- we do
- // find the mer, and overlapping mers on the other side. So
- // possibly the unmapped sequence is just:
- // 11111111111111100110010001111111111111111111
- // ----------
- //
- //
- // Alg is now to look for blocks of less than merSize zeros,
- // and....do something. Certainly, chain blocks together if
- // there is no intervening block of merSize 1's in between.
- // Possibly extend the block merSize-1 bases to either side.
- //
- //
- for (u32bit i=0; i<S->sequenceLength(); i++) {
-
- // Now we're potentially at the start of a small block. The
- // goal is to see if this is an isolated block of spurious
- // match, and if so, merge it into the surrounding blocks of
- // mismatch.
- //
- // If it's a small block of 1's, we extend the block into the
- // next small blocks, stopping at the start of a large block.
- // e.g.: (the --'s are the end of the small blocks):
- //
- // ...0000000--1111110000011111--000000000000000000000...
- // ...0000000--1110001100110010--111111111111111111111...
- // ...1111111--0000111100100000--111111111111111111111...
- // ...1111111--0000111100100011--000000000000000000000...
- //
-
- bool isIsolated = true;
- u32bit blockEnd = i;
- u32bit bs = 0;
- u32bit nextStart = i;
-
- while (isIsolated) {
- u32bit j = blockEnd;
-
- while ((j < S->sequenceLength()) &&
- (mask[j] == mask[blockEnd]))
- j++;
-
- bs = j - blockEnd;
-
- // Remember how far we got.
- nextStart += j;
-
- // If a big block terminate.
- //
- if (bs >= merSize)
- isIsolated = false;
- else
- blockEnd += bs;
-
- // If now at the end of the sequence, terminate.
- //
- if (blockEnd >= S->sequenceLength())
- isIsolated = false;
- }
-
-
- // But if we stop because we find a big block of zeros, then
- // the previous block of ones is still valid. Our big block of
- // zeros indicates a true mismatch.
- //
- // .....11111000110000111100000000000000000000000000111
- //
-
- // If we did find a small block
- if (i != blockEnd) {
-
- // And that block doesn't run up to the end
- if (blockEnd < S->sequenceLength()) {
-
- // And the last thing in that block is 1's, move back to the previous block of zeros.
- //
- // EXCEPT, that this big block of zeros might REALLY be
- // unmatched sequence, not just an isolated error. So we
- // don't reverse if the big block of zeros is big.
- //
- if (bs < merSize + 6)
- while ((blockEnd > 0) && (mask[blockEnd-1] == true))
- blockEnd--;
- }
-
- fprintf(stdout, "mask "u32bitFMT" "u32bitFMT"\n", i, blockEnd);
-
- // Supposedly, our block of unmasked sequence is now from i to
- // blockEnd. Flip all that stuff to unmasked.
- //
- for (u32bit j=i; j<blockEnd; j++)
- mask[j] = false;
- }
-
-
- // And move up to the next starting position.
- i = nextStart - 1;
- }
-
-
-#ifdef PRINT_BITS
- for (u32bit i=0; i<S->sequenceLength(); i++) {
- if (mask[i])
- maskBit[i] = '1';
- else
- maskBit[i] = '0';
- }
- maskBit[S->sequenceLength()] = 0;
- fprintf(stdout, "%s\n", maskBit);
-#endif
-
-
-
- // Convert the mer-markings to base-markings
- //
- u32bit isMasking = 0;
- for (u32bit i=0; i<S->sequenceLength(); i++) {
- if (mask[i])
- isMasking = merSize;
-
- if (isMasking > 0) {
- mask[i] = true;
- isMasking--;
- }
- }
-
-
-#ifdef PRINT_BITS
- for (u32bit i=0; i<S->sequenceLength(); i++) {
- if (mask[i])
- maskBit[i] = '1';
- else
- maskBit[i] = '0';
- }
- maskBit[S->sequenceLength()] = 0;
- fprintf(stdout, "%s\n", maskBit);
-#endif
-
-
-
- char *seq = S->sequence();
-
- for (u32bit i=0; i<S->sequenceLength(); i++) {
- if (mask[i]) {
- maskSeq[i] = 'n';
- } else {
- maskSeq[i] = seq[i];
- }
- }
-
- maskSeq[S->sequenceLength()] = 0;
-
- fprintf(stdout, "%s\n%s\n", S->header(), maskSeq);
-
- delete S;
- }
-
- delete [] maskSeq;
- delete [] mask;
-}
diff --git a/kmer/libkmer/merTable.H b/kmer/libkmer/merTable.H
deleted file mode 100644
index 96bf01f..0000000
--- a/kmer/libkmer/merTable.H
+++ /dev/null
@@ -1,76 +0,0 @@
-#ifndef MERTABLE_H
-#define MERTABLE_H
-
-// The obvious simple small mer table, appropriate for large sequences
-
-#error merTable should be unused
-
-class merTable {
-public:
- merTable() {
- };
- ~merTable() {
- delete [] merToPositions;
- delete [] positions;
- };
-
- void build(seqStream *CS, u32bit merSize, u32bit merSkip=0) {
-
- // Allocate the mer table
- //
- u32bit tableSize = u32bitONE << (2*merSize);
- fprintf(stderr, "allocate "u32bitFMT" entries for a merTable.\n", tableSize);
-
- merToPositions = new u32bit [tableSize+1];
-
- // First pass, count the number of times we see each mer
- //
- for (u32bit i=0; i<=tableSize; i++)
- merToPositions[i] = 0;
-
- merStream MS(merSize, CS);
-
- while (MS.nextMer(merSkip)) {
- u64bit m = (u64bit)MS.theFMer();
- //fprintf(stderr, "add mer "u64bitFMT"\n", m);
- merToPositions[m]++;
- }
-
- // Convert those to indexes into positions - m[i] is the start of
- // the locations in positions[] where positions are stored.
- //
- for (u32bit pos=0, val=0, i=0; i<=tableSize; i++) {
- val = merToPositions[i];
- merToPositions[i] = pos;
- pos += val;
- }
-
- // Allocate space
- //
- fprintf(stderr, "allocate "u32bitFMT" entries for positions\n", merToPositions[tableSize]);
- positions = new u32bit [merToPositions[tableSize]];
-
- // Second pass, fill in positions
- //
- MS.rewind();
-
- while (MS.nextMer(merSkip))
- positions[ merToPositions[(u64bit)MS.theFMer()]++ ] = MS.thePositionInStream();
- };
-
- u32bit numberOfPositions(u64bit mer) {
- return(merToPositions[mer+1] - merToPositions[mer]);
- };
-
- u32bit getPosition(u64bit mer, u32bit index) {
- if (index >= merToPositions[mer+1] - merToPositions[mer])
- return(~u32bitZERO);
- return(merToPositions[mer] + index);
- };
-
-private:
- u32bit *merToPositions; // index into positions[]; merToPositions[mer] is the first base in the mer
- u32bit *positions; // list of positions for mers, sorted by mer
-};
-
-#endif // MERTABLE_H
diff --git a/kmer/libkmer/percentCovered.C b/kmer/libkmer/percentCovered.C
deleted file mode 100644
index ce0da4a..0000000
--- a/kmer/libkmer/percentCovered.C
+++ /dev/null
@@ -1,66 +0,0 @@
-#include "util++.H"
-#include "bio++.H"
-#include "existDB.H"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-int
-main(int argc, char **argv) {
- char *merFile = 0L;
- char *queryFile = 0L;
-
- int arg=1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-m") == 0) {
- merFile = argv[++arg];
- } else if (strcmp(argv[arg], "-q") == 0) {
- queryFile = argv[++arg];
- } else {
- fprintf(stderr, "Unknown arg '%s'\n", argv[arg]);
- }
- arg++;
- }
-
- existDB *E = new existDB(merFile, 22, existDBnoFlags, 0, ~u32bitZERO);
- seqCache *Q = new seqCache(queryFile);
- seqInCore *S = Q->getSequenceInCore();
-
- intervalList IL;
- speedCounter SC(" %8f frags (%8.5f frags/sec)\r", 1, 1000, true);
-
- while (S) {
- merStream *MS = new merStream(new kMerBuilder(22),
- new seqStream(S->sequence(), S->sequenceLength()),
- true, true);
-
- IL.clear();
-
- while (MS->nextMer()) {
- if (E->exists(MS->theFMer())) {
- IL.add(MS->thePositionInSequence(), 22);
- }
- }
-
- IL.merge();
-
- if (IL.sumOfLengths() > 0) {
- fprintf(stdout, "%5.2f\n",
- 100.0 * IL.sumOfLengths() / (double)S->sequenceLength());
- }
-
- delete MS;
- delete S;
-
- SC.tick();
-
- S = Q->getSequenceInCore();
- }
-
- delete Q;
- delete E;
-
- return(0);
-}
-
diff --git a/kmer/libkmer/positionDB-access.C b/kmer/libkmer/positionDB-access.C
deleted file mode 100644
index 2e63ef1..0000000
--- a/kmer/libkmer/positionDB-access.C
+++ /dev/null
@@ -1,344 +0,0 @@
-#include "bio++.H"
-#include "positionDB.H"
-
-
-void
-positionDB::reallocateSpace(u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen,
- u64bit len) {
-
- if (posnMax < posnLen + len) {
- u64bit *pp;
-
- posnMax = posnLen + len + (len >> 2);
-
- if (posnMax == 0)
- posnMax = 16384;
-
- try {
- pp = new u64bit [posnMax];
- } catch (...) {
- fprintf(stderr, "positionDB::get()-- Can't allocate space for more positions, requested "u64bitFMT" u64bit's.\n", posnMax);
- abort();
- }
-
- memcpy(pp, posn, sizeof(u64bit) * posnLen);
-
- delete [] posn;
- posn = pp;
- }
-}
-
-
-
-void
-positionDB::loadPositions(u64bit J,
- u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen,
- u64bit& count) {
-
- u64bit sizs[3] = {_pptrWidth, 1, _sizeWidth};
- u64bit vals[3] = {0, 0, 1};
-
- getDecodedValues(_buckets, J + _chckWidth, (_sizeWidth == 0) ? 2 : 3, sizs, vals);
-
- // If the size is stored, the count is updated to the correct
- // thing. If it's not stored, the count is set to 1 by the default
- // value of vals[2], and reset after we get the number of positions
- // stored.
- //
- count = vals[2];
-
- if (vals[1]) {
- reallocateSpace(posn, posnMax, posnLen, 64);
- posn[posnLen++] = vals[0];
- } else {
- u64bit ptr = vals[0] * _posnWidth;
- u64bit len = getDecodedValue(_positions, ptr, _posnWidth);
-
- if (_sizeWidth == 0)
- count = len;
-
- reallocateSpace(posn, posnMax, posnLen, len + 64);
-
- for (ptr += _posnWidth; len > 0; ptr += _posnWidth, len--)
- posn[posnLen++] = getDecodedValue(_positions, ptr, _posnWidth);
- }
-}
-
-
-
-bool
-positionDB::getExact(u64bit mer,
- u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen,
- u64bit& count) {
- u64bit h = HASH(mer);
- u64bit c = CHECK(mer);
- u64bit st, ed;
-
- if (_hashTable_BP) {
- st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
- ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
- } else {
- st = _hashTable_FW[h];
- ed = _hashTable_FW[h+1];
- }
-
- posnLen = 0;
-
- if (st == ed)
- return(false);
-
- for (u64bit i=st, J=st * _wFin; i<ed; i++, J += _wFin) {
- if (c == getDecodedValue(_buckets, J, _chckWidth)) {
- loadPositions(J, posn, posnMax, posnLen, count);
- return(true);
- }
- }
-
- return(false);
-}
-
-
-bool
-positionDB::existsExact(u64bit mer) {
- u64bit h = HASH(mer);
- u64bit c = CHECK(mer);
- u64bit st, ed;
-
- if (_hashTable_BP) {
- st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
- ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
- } else {
- st = _hashTable_FW[h];
- ed = _hashTable_FW[h+1];
- }
-
- if (st == ed)
- return(false);
-
- for (u64bit i=st, J=st * _wFin; i<ed; i++, J += _wFin)
- if (c == getDecodedValue(_buckets, J, _chckWidth))
- return(true);
-
- return(false);
-}
-
-
-u64bit
-positionDB::countExact(u64bit mer) {
- u64bit h = HASH(mer);
- u64bit c = CHECK(mer);
- u64bit st, ed;
-
- if (_hashTable_BP) {
- st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
- ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
- } else {
- st = _hashTable_FW[h];
- ed = _hashTable_FW[h+1];
- }
-
- if (st == ed)
- return(0);
-
- for (u64bit i=st, J=st * _wFin; i<ed; i++, J += _wFin) {
- if (c == getDecodedValue(_buckets, J, _chckWidth)) {
- u64bit sizs[3] = {_pptrWidth, 1, _sizeWidth};
- u64bit vals[3] = {0};
-
- getDecodedValues(_buckets, J + _chckWidth, 3, sizs, vals);
-
- if (_sizeWidth > 0)
- return(vals[2]);
-
- if (vals[1])
- return(1);
-
- return(getDecodedValue(_positions, vals[0] * _posnWidth, _posnWidth));
- }
- }
-
- return(0);
-}
-
-
-u64bit
-positionDB::setCount(u64bit mer, u64bit count) {
- u64bit h = HASH(mer);
- u64bit c = CHECK(mer);
- u64bit st, ed;
-
- if (_hashTable_BP) {
- st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
- ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
- } else {
- st = _hashTable_FW[h];
- ed = _hashTable_FW[h+1];
- }
-
- if (st == ed)
- return(0);
-
- for (u64bit i=st, J=st * _wFin; i<ed; i++, J += _wFin)
- if (c == getDecodedValue(_buckets, J, _chckWidth)) {
- setDecodedValue(_buckets, J + _chckWidth + _pptrWidth + 1, _sizeWidth, count);
- return(count);
- }
-
- return(0);
-}
-
-
-
-void
-positionDB::filter(u64bit lo,
- u64bit hi) {
- u64bit st=0, ed=0; // iteration through buckets
- u64bit nb=0; // bit position of the current (read) bucket and next (write) bucket
- u64bit np=0; // bit position of the current (read) position and next (write) position
- u64bit vv;
-
- u64bit loCount = 0;
- u64bit okCount = 0;
- u64bit hiCount = 0;
-
- u64bit sizs[4] = {_chckWidth, _pptrWidth, 1, _sizeWidth};
- u64bit vals[4] = {0, 0, 0, 0};
-
- //dump("posDB.before");
-
- fprintf(stderr, "positionDB::filter()-- Filtering out kmers less than "u64bitFMT" and more than "u64bitFMT"\n", lo, hi);
-
- if (_sizeWidth == 0) {
- // Single copy mers in a table without counts can be multi-copy
- // when combined with their reverse-complement mer.
- fprintf(stderr, "positionDB::filter()-- ERROR!\n");
- fprintf(stderr, "positionDB::filter()-- ERROR! No count information; filtering will break canonical assumptions.\n");
- fprintf(stderr, "positionDB::filter()-- ERROR!\n");
- exit(1);
- }
-
- // Grab the start of the first (current) bucket. We reset the
- // hashTable at the end of the loop, forcing us to keep st
- // up-to-date, instead of grabbing it anew each iteration.
- //
- if (_hashTable_BP)
- st = getDecodedValue(_hashTable_BP, 0, _hashWidth);
- else
- st = _hashTable_FW[0];
-
- // Over all buckets
- //
- for (u64bit h=0; h<_tableSizeInEntries; h++) {
-
- // Grab the end of this bucket - the end is always for the
- // current structure. This gets reset at the end of the loop.
- //
- if (_hashTable_BP)
- ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
- else
- ed = _hashTable_FW[h+1];
-
- // Over all entries in the bucket
- //
- while (st < ed) {
- u64bit cb = st * _wFin;
-
- getDecodedValues(_buckets, cb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
-
- // Argh. Tricky. We need to grab the count stored in the
- // table, but if it's a single mer, there is no count.
-
- u64bit count = 1; // Real count over the whole data set
- u64bit len = 1; // Number of times the mer occurs in this subset
- u64bit cp = ~u64bitZERO; // current position pointer, used to copy position information
-
- // If not a unique mer in this table, len and cp are defined.
- if (vals[2] == 0) {
- cp = vals[1] * _posnWidth;
- len = getDecodedValue(_positions, cp, _posnWidth);
- count = len;
- }
-
- // The size stored in the bucket is to be believed
- if (_sizeWidth > 0)
- count = vals[3];
-
- // What happened here: By default, the count is 1. If it is
- // NOT a unique mer in the table, we reset the count to the
- // number of entries in the table. Then, if there is a count
- // stored in the table, we reset the count again.
-
- // Move on to copying the data, if in the correct range.
-
- if (vals[2] == 1) {
- // Is a single mer in our table. Copy if the actual count is
- // acceptable.
- if ((lo <= count) && (count < hi)) {
- okCount++;
- setDecodedValues(_buckets, nb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
- nb += _wFin;
- } else {
- _numberOfDistinct--;
- _numberOfMers--;
- loCount++;
- }
- } else {
- // Mer has more than one location in the table. Copy all
- // locations if the count is acceptable.
- if ((lo <= count) && (count < hi)) {
- okCount++;
-
- // Copy the bucket
- vals[1] = np / _posnWidth;
- setDecodedValues(_buckets, nb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
- nb += _wFin;
-
- // Copy length of the positions
- if (cp != np)
- setDecodedValue(_positions, np, _posnWidth, len);
- np += _posnWidth;
- cp += _posnWidth;
-
- // Copy positions
- while (len > 0) {
- if (cp != np)
- setDecodedValue(_positions, np, _posnWidth,
- getDecodedValue(_positions, cp, _posnWidth));
- np += _posnWidth;
- cp += _posnWidth;
- len--;
- }
- } else {
- // Not acceptable count
- _numberOfDistinct--;
- _numberOfEntries -= len;
- if (count < lo) loCount++;
- if (count > hi) hiCount++;
- }
- }
-
- // Move to the next entry
- st++;
- cb += _wFin;
- } // Over all entries in the bucket
-
- // Update the end position of this bucket
- if (_hashTable_BP)
- setDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth, nb / _wFin);
- else
- _hashTable_FW[h+1] = nb / _wFin;
-
- } // Over all buckets
-
- fprintf(stderr, "positionDB::filter()-- Filtered "u64bitFMT" kmers less than "u64bitFMT"\n", loCount, lo);
- fprintf(stderr, "positionDB::filter()-- Filtered "u64bitFMT" kmers more than "u64bitFMT"\n", hiCount, hi);
- fprintf(stderr, "positionDB::filter()-- Saved "u64bitFMT" kmers with acceptable count\n", okCount);
-
- //dump("posDB.after");
-}
diff --git a/kmer/libkmer/positionDB-dump.C b/kmer/libkmer/positionDB-dump.C
deleted file mode 100644
index 546dd40..0000000
--- a/kmer/libkmer/positionDB-dump.C
+++ /dev/null
@@ -1,50 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "positionDB.H"
-#include "bio++.H"
-
-
-void
-positionDB::dump(char *name) {
- u64bit sizs[4] = {_chckWidth, _pptrWidth, 1, _sizeWidth};
- u64bit vals[4] = {0, 0, 0, 0};
- FILE *F = fopen(name, "w");
-
- for (u64bit h=0; h<_tableSizeInEntries; h++) {
- u64bit st, ed;
-
- if (_hashTable_BP) {
- st = getDecodedValue(_hashTable_BP, h * _hashWidth, _hashWidth);
- ed = getDecodedValue(_hashTable_BP, h * _hashWidth + _hashWidth, _hashWidth);
- } else {
- st = _hashTable_FW[h];
- ed = _hashTable_FW[h+1];
- }
-
- fprintf(F, "B "u64bitFMT" "u64bitFMT"-"u64bitFMT"\n", h, st, ed);
-
- while (st < ed) {
- u64bit cb = st * _wFin;
-
- getDecodedValues(_buckets, cb, (_sizeWidth == 0) ? 3 : 4, sizs, vals);
-
- fprintf(F, "%c chk="u64bitHEX" pos="u64bitFMT" siz="u64bitFMT,
- (vals[2] == 0) ? 'D' : 'U', vals[0], vals[1], vals[3]);
-
- if (vals[2] == 0) {
- u64bit pos = vals[1] * _posnWidth;
- u64bit len = getDecodedValue(_positions, pos, _posnWidth);
-
- for (pos += _posnWidth; len > 0; pos += _posnWidth, len--)
- fprintf(F, " "u64bitFMT, getDecodedValue(_positions, pos, _posnWidth));
- }
-
- fprintf(F, "\n");
-
- st++;
- }
- }
-
- fclose(F);
-}
diff --git a/kmer/libkmer/positionDB-file.C b/kmer/libkmer/positionDB-file.C
deleted file mode 100644
index cfccb5c..0000000
--- a/kmer/libkmer/positionDB-file.C
+++ /dev/null
@@ -1,211 +0,0 @@
-#include "positionDB.H"
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include <errno.h>
-
-static
-char magic[16] = { 'p', 'o', 's', 'i', 't', 'i', 'o', 'n', 'D', 'B', '.', 'v', '1', ' ', ' ', ' ' };
-static
-char faild[16] = { 'p', 'o', 's', 'i', 't', 'i', 'o', 'n', 'D', 'B', 'f', 'a', 'i', 'l', 'e', 'd' };
-
-void
-positionDB::saveState(char const *filename) {
-
- fprintf(stderr, "Saving positionDB to '%s'\n", filename);
-
- errno = 0;
- int F = open(filename, O_RDWR | O_CREAT | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno) {
- fprintf(stderr, "Can't open '%s' for writing positionDB.\n%s\n", filename, strerror(errno));
- exit(1);
- }
-
- bool magicFirst = false;
-
- // Test if this is a pipe. If so, we write the magic first,
- // otherwise we write the magic last.
- //
- errno = 0;
- lseek(F, 0, SEEK_SET);
- if (errno == ESPIPE)
- magicFirst = true;
-
- if (magicFirst)
- write(F, magic, sizeof(char) * 16);
- else
- write(F, faild, sizeof(char) * 16);
-
- if (errno) {
- fprintf(stderr, "positionDB::saveState()-- Write failure on magic first.\n%s\n", strerror(errno));
- exit(1);
- }
-
- // If only to be completely annoying and anal, we clear the
- // pointers before we write the data. Sure, we could just write
- // the stuff we care about, but this is easier. This is easier.
- // Before you go rip out this stuff, remember that you can now
- // checksum the resulting files. So don't do it.
- //
- u32bit *bs = _bucketSizes;
- u64bit *cb = _countingBuckets;
- u64bit *hp = _hashTable_BP;
- u32bit *hw = _hashTable_FW;
- u64bit *bu = _buckets;
- u64bit *ps = _positions;
- u64bit *he = _hashedErrors;
-
- _bucketSizes = 0L;
- _countingBuckets = 0L;
- _hashTable_BP = (u64bit *)((_hashTable_BP) ? u64bitONE : u64bitZERO);
- _hashTable_FW = (u32bit *)((_hashTable_FW) ? u32bitONE : u32bitZERO);
- _buckets = 0L;
- _positions = 0L;
- _hashedErrors = 0L;
-
- safeWrite(F, this, "this", sizeof(positionDB) * 1);
-
- _bucketSizes = bs;
- _countingBuckets = cb;
- _hashTable_BP = hp;
- _hashTable_FW = hw;
- _buckets = bu;
- _positions = ps;
- _hashedErrors = he;
-
- if (_hashTable_BP) {
- safeWrite(F, _hashTable_BP, "_hashTable_BP", sizeof(u64bit) * (_tableSizeInEntries * _hashWidth / 64 + 1));
- } else {
- safeWrite(F, _hashTable_FW, "_hashTable_FW", sizeof(u32bit) * (_tableSizeInEntries + 1));
- }
-
- safeWrite(F, _buckets, "_buckets", sizeof(u64bit) * (_numberOfDistinct * _wFin / 64 + 1));
- safeWrite(F, _positions, "_positions", sizeof(u64bit) * (_numberOfEntries * _posnWidth / 64 + 1));
- safeWrite(F, _hashedErrors, "_hashedErrors", sizeof(u64bit) * (_hashedErrorsLen));
-
- if (magicFirst == false) {
- lseek(F, 0, SEEK_SET);
- if (errno) {
- fprintf(stderr, "positionDB::saveState()-- Failed to seek to start of file -- write failed.\n%s\n", strerror(errno));
- exit(1);
- }
-
- write(F, magic, sizeof(char) * 16);
- if (errno) {
- fprintf(stderr, "positionDB::saveState()-- Write failure on magic last.\n%s\n", strerror(errno));
- exit(1);
- }
- }
-
- close(F);
-}
-
-
-bool
-positionDB::loadState(char const *filename, bool beNoisy, bool loadData) {
- char cigam[16] = { 0 };
-
- fprintf(stderr, "Loading positionDB from '%s'\n", filename);
-
- errno = 0;
- int F = open(filename, O_RDONLY | O_LARGEFILE, 0);
- if (errno) {
- fprintf(stderr, "Can't open '%s' for reading pre-built positionDB: %s\n", filename, strerror(errno));
- return(false);
- }
-
- safeRead(F, cigam, "Magic Number", sizeof(char) * 16);
-
- if (strncmp(faild, cigam, 16) == 0) {
- if (beNoisy) {
- fprintf(stderr, "positionDB::loadState()-- Incomplete positionDB binary file.\n");
- fprintf(stderr, "positionDB::loadState()-- Read '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
- cigam[0], cigam[1], cigam[2], cigam[3],
- cigam[4], cigam[5], cigam[6], cigam[7],
- cigam[8], cigam[9], cigam[10], cigam[11],
- cigam[12], cigam[13], cigam[14], cigam[15]);
- fprintf(stderr, "positionDB::loadState()-- Expected '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
- magic[0], magic[1], magic[2], magic[3],
- magic[4], magic[5], magic[6], magic[7],
- magic[8], magic[9], magic[10], magic[11],
- magic[12], magic[13], magic[14], magic[15]);
- }
- close(F);
- return(false);
- } else if (strncmp(magic, cigam, 16) != 0) {
- if (beNoisy) {
- fprintf(stderr, "positionDB::loadState()-- Not a positionDB binary file, maybe a sequence file?\n");
- fprintf(stderr, "positionDB::loadState()-- Read '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
- cigam[0], cigam[1], cigam[2], cigam[3],
- cigam[4], cigam[5], cigam[6], cigam[7],
- cigam[8], cigam[9], cigam[10], cigam[11],
- cigam[12], cigam[13], cigam[14], cigam[15]);
- fprintf(stderr, "positionDB::loadState()-- Expected '%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c'\n",
- magic[0], magic[1], magic[2], magic[3],
- magic[4], magic[5], magic[6], magic[7],
- magic[8], magic[9], magic[10], magic[11],
- magic[12], magic[13], magic[14], magic[15]);
- }
-
- close(F);
- return(false);
- }
-
- safeRead(F, this, "positionDB", sizeof(positionDB) * 1);
-
- _bucketSizes = 0L;
- _countingBuckets = 0L;
- _buckets = 0L;
- _positions = 0L;
- _hashedErrors = 0L;
-
- if (loadData) {
- u64bit hs = _tableSizeInEntries * _hashWidth / 64 + 1;
- u64bit bs = _numberOfDistinct * _wFin / 64 + 1;
- u64bit ps = _numberOfEntries * _posnWidth / 64 + 1;
-
- if (_hashTable_BP) {
- _hashTable_BP = new u64bit [hs];
- _hashTable_FW = 0L;
- safeRead(F, _hashTable_BP, "_hashTable_BP", sizeof(u64bit) * hs);
- } else {
- _hashTable_BP = 0L;
- _hashTable_FW = new u32bit [_tableSizeInEntries + 1];
- safeRead(F, _hashTable_FW, "_hashTable_FW", sizeof(u32bit) * (_tableSizeInEntries + 1));
- }
-
- _buckets = new u64bit [bs];
- _positions = new u64bit [ps];
- _hashedErrors = new u64bit [_hashedErrorsMax];
-
- safeRead(F, _buckets, "_buckets", sizeof(u64bit) * bs);
- safeRead(F, _positions, "_positions", sizeof(u64bit) * ps);
- safeRead(F, _hashedErrors, "_hashedErrors", sizeof(u64bit) * _hashedErrorsLen);
- }
-
- close(F);
-
- return(true);
-}
-
-
-
-void
-positionDB::printState(FILE *stream) {
- fprintf(stream, "merSizeInBases: "u32bitFMT"\n", _merSizeInBases);
- fprintf(stream, "merSkipInBases: "u32bitFMT"\n", _merSkipInBases);
- fprintf(stream, "tableSizeInBits: "u32bitFMT"\n", _tableSizeInBits);
- fprintf(stream, "tableSizeInEntries: "u64bitFMT"\n", _tableSizeInEntries);
- fprintf(stream, "hashWidth: "u32bitFMT"\n", _hashWidth);
- fprintf(stream, "chckWidth: "u32bitFMT"\n", _chckWidth);
- fprintf(stream, "posnWidth: "u32bitFMT"\n", _posnWidth);
- fprintf(stream, "numberOfMers: "u64bitFMT"\n", _numberOfMers);
- fprintf(stream, "numberOfPositions: "u64bitFMT"\n", _numberOfPositions);
- fprintf(stream, "numberOfDistinct: "u64bitFMT"\n", _numberOfDistinct);
- fprintf(stream, "numberOfUnique: "u64bitFMT"\n", _numberOfUnique);
- fprintf(stream, "numberOfEntries: "u64bitFMT"\n", _numberOfEntries);
- fprintf(stream, "maximumEntries: "u64bitFMT"\n", _maximumEntries);
-}
-
diff --git a/kmer/libkmer/positionDB-mismatch.C b/kmer/libkmer/positionDB-mismatch.C
deleted file mode 100644
index bbdfb4b..0000000
--- a/kmer/libkmer/positionDB-mismatch.C
+++ /dev/null
@@ -1,388 +0,0 @@
-#include "positionDB.H"
-#include "bio++.H"
-
-
-static
-int
-stringscmp(const void *A, const void *B) {
- u64bit const a = *(u64bit const *)A;
- u64bit const b = *(u64bit const *)B;
- if (a < b) return(-1);
- if (a > b) return(1);
- return(0);
-}
-
-
-static
-u32bit
-makeUnique(u64bit *strings, u32bit stringsLen) {
- qsort(strings, stringsLen, sizeof(u64bit), stringscmp);
- u32bit len = 0;
- u32bit nxt = 1;
- while (nxt < stringsLen) {
- if (strings[len] != strings[nxt]) {
- len++;
- strings[len] = strings[nxt];
- }
- nxt++;
- }
- return(len+1);
-}
-
-
-#if 0
-// debug
-static
-void
-dumpPatterns(u64bit *strings, u32bit stringsLen, u32bit ts) {
- for (u32bit i=0; i<stringsLen; i++) {
- char str[1024] = {0};
- u32bit cnt = 0;
-
- for (u32bit b=0; b<ts; b++) {
- if (strings[i] & (u64bitONE << b)) {
- str[b] = '1';
- cnt++;
- } else {
- str[b] = '0';
- }
- }
-
- fprintf(stdout, "%s\t"u32bitFMT"\n", str, cnt);
- }
-}
-#endif
-
-
-double
-positionDB::setUpMismatchMatcher(u32bit nErrorsAllowed, u64bit approxMers) {
-
- // Build an xor mask that will generate all errors for a given
- // mersize.
-
- _nErrorsAllowed = nErrorsAllowed;
- _hashedErrorsLen = 0;
- _hashedErrorsMax = 0;
- _hashedErrors = 0L;
-
- u32bit stringsMax = 128 * 1024 * 1024;
- u32bit stringsLen = 0;
- u64bit *strings = new u64bit [stringsMax];
-
- u64bit totpat = 0;
- u64bit toterr = 0;
-
- u64bit m1, m2, m3, m4, m5, m6;
- u64bit *e1, *e2, *e3, *e4, *e5, *e6;
-
- {
- // This can be trivially eliminated by replacing e1[x] with the err[] statement.
- u32bit ne = 3;
- for (u32bit x=1; x<_nErrorsAllowed; x++)
- ne *= 3;
-
- //fprintf(stderr, "Storing ne="u32bitFMT" errors.\n", ne);
-
- e1 = new u64bit [ne];
- e2 = new u64bit [ne];
- e3 = new u64bit [ne];
- e4 = new u64bit [ne];
- e5 = new u64bit [ne];
- e6 = new u64bit [ne];
-
- u64bit err[3] = { 0x5555555555555555llu, 0xaaaaaaaaaaaaaaaallu, 0xffffffffffffffffllu };
-
- for (u32bit x=0; x<ne; x++) {
- e1[x] = err[(x/ 1) % 3];
- e2[x] = err[(x/ 3) % 3];
- e3[x] = err[(x/ 9) % 3];
- e4[x] = err[(x/ 27) % 3];
- e5[x] = err[(x/ 81) % 3];
- e6[x] = err[(x/243) % 3];
- }
- }
-
-
- // Zero errors
- strings[stringsLen++] = u64bitZERO;
-
-
- // One error
- if (1 <= _nErrorsAllowed) {
- for (u32bit ai=0; ai<_merSizeInBases; ai++) {
- totpat++;
- toterr += 3;
- m1 = 0x03llu << (ai * 2);
-
- for (u32bit x=0; x<3; x++)
- strings[stringsLen++] = HASH((m1 & e1[x]));
- }
-
- stringsLen = makeUnique(strings, stringsLen);
- stringsLen = makeUnique(strings, stringsLen);
- //dumpPatterns(strings, stringsLen, _tableSizeInBits);
- //fprintf(stderr, "DONE1 totpat="u64bitFMT" toterr="u64bitFMT" stringsLen="u32bitFMT"\n", totpat, toterr, stringsLen);
- }
-
-
- // Two errors
- if (2 <= _nErrorsAllowed) {
- for (u32bit ai=0; ai<_merSizeInBases; ai++)
- for (u32bit bi=0; bi<ai; bi++) {
- totpat++;
- toterr += 9;
- m1 = 0x03llu << (ai * 2);
- m2 = 0x03llu << (bi * 2);
-
- for (u32bit x=0; x<9; x++)
- strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]));
- }
-
- stringsLen = makeUnique(strings, stringsLen);
- stringsLen = makeUnique(strings, stringsLen);
- //dumpPatterns(strings, stringsLen, _tableSizeInBits);
- //fprintf(stderr, "DONE2 totpat="u64bitFMT" toterr="u64bitFMT" stringsLen="u32bitFMT"\n", totpat, toterr, stringsLen);
- }
-
-
- // Three errors
- if (3 <= _nErrorsAllowed) {
- for (u32bit ai=0; ai<_merSizeInBases; ai++)
- for (u32bit bi=0; bi<ai; bi++)
- for (u32bit ci=0; ci<bi; ci++) {
- totpat++;
- toterr += 27;
- m1 = 0x03llu << (ai * 2);
- m2 = 0x03llu << (bi * 2);
- m3 = 0x03llu << (ci * 2);
-
- for (u32bit x=0; x<27; x++)
- strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]));
- }
-
- stringsLen = makeUnique(strings, stringsLen);
- stringsLen = makeUnique(strings, stringsLen);
- //dumpPatterns(strings, stringsLen, _tableSizeInBits);
- //fprintf(stderr, "DONE3 totpat="u64bitFMT" toterr="u64bitFMT" stringsLen="u32bitFMT"\n", totpat, toterr, stringsLen);
- }
-
-
- // Four errors
- if (4 <= _nErrorsAllowed) {
- for (u32bit ai=0; ai<_merSizeInBases; ai++)
- for (u32bit bi=0; bi<ai; bi++)
- for (u32bit ci=0; ci<bi; ci++)
- for (u32bit di=0; di<ci; di++) {
- totpat++;
- toterr += 81;
- m1 = 0x03llu << (ai * 2);
- m2 = 0x03llu << (bi * 2);
- m3 = 0x03llu << (ci * 2);
- m4 = 0x03llu << (di * 2);
-
- for (u32bit x=0; x<81; x++)
- strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]) ^ (m4 & e4[x]));
- }
-
- stringsLen = makeUnique(strings, stringsLen);
- stringsLen = makeUnique(strings, stringsLen);
- //dumpPatterns(strings, stringsLen, _tableSizeInBits);
- //fprintf(stderr, "DONE4 totpat="u64bitFMT" toterr="u64bitFMT" stringsLen="u32bitFMT"\n", totpat, toterr, stringsLen);
- }
-
-
- // Five errors
- if (5 <= _nErrorsAllowed) {
- for (u32bit ai=0; ai<_merSizeInBases; ai++)
- for (u32bit bi=0; bi<ai; bi++)
- for (u32bit ci=0; ci<bi; ci++)
- for (u32bit di=0; di<ci; di++)
- for (u32bit ei=0; ei<di; ei++) {
- totpat++;
- toterr += 243;
- m1 = 0x03llu << (ai * 2);
- m2 = 0x03llu << (bi * 2);
- m3 = 0x03llu << (ci * 2);
- m4 = 0x03llu << (di * 2);
- m5 = 0x03llu << (ei * 2);
-
- if (stringsLen + 32000 >= stringsMax)
- stringsLen = makeUnique(strings, stringsLen);
-
- for (u32bit x=0; x<243; x++)
- strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]) ^ (m4 & e4[x]) ^ (m5 & e5[x]));
- }
-
- stringsLen = makeUnique(strings, stringsLen);
- stringsLen = makeUnique(strings, stringsLen);
- //dumpPatterns(strings, stringsLen, _tableSizeInBits);
- //fprintf(stderr, "DONE5 totpat="u64bitFMT" toterr="u64bitFMT" stringsLen="u32bitFMT"\n", totpat, toterr, stringsLen);
- }
-
-
- // Six errors
- if (6 <= _nErrorsAllowed) {
- for (u32bit ai=0; ai<_merSizeInBases; ai++)
- for (u32bit bi=0; bi<ai; bi++)
- for (u32bit ci=0; ci<bi; ci++)
- for (u32bit di=0; di<ci; di++)
- for (u32bit ei=0; ei<di; ei++)
- for (u32bit fi=0; fi<ei; fi++) {
- totpat++;
- toterr += 729;
- m1 = 0x03llu << (ai * 2);
- m2 = 0x03llu << (bi * 2);
- m3 = 0x03llu << (ci * 2);
- m4 = 0x03llu << (di * 2);
- m5 = 0x03llu << (ei * 2);
- m6 = 0x03llu << (fi * 2);
-
- if (stringsLen + 32000 >= stringsMax)
- stringsLen = makeUnique(strings, stringsLen);
-
- for (u32bit x=0; x<729; x++)
- strings[stringsLen++] = HASH((m1 & e1[x]) ^ (m2 & e2[x]) ^ (m3 & e3[x]) ^ (m4 & e4[x]) ^ (m5 & e5[x]) ^ (m6 & e6[x]));
- }
-
- stringsLen = makeUnique(strings, stringsLen);
- stringsLen = makeUnique(strings, stringsLen);
- //dumpPatterns(strings, stringsLen, _tableSizeInBits);
- //fprintf(stderr, "DONE6 totpat="u64bitFMT" toterr="u64bitFMT" stringsLen="u32bitFMT"\n", totpat, toterr, stringsLen);
- }
-
-
- if (7 <= _nErrorsAllowed) {
- fprintf(stderr, "Only 6 errors allowed.\n");
- exit(1);
- }
-
- for (u32bit i=1; i<stringsLen; i++) {
- assert((strings[i] & ~_hashMask) == 0);
- assert(strings[i] != 0);
- }
-
- delete [] e1;
- delete [] e2;
- delete [] e3;
- delete [] e4;
- delete [] e5;
- delete [] e6;
-
- delete [] _hashedErrors;
-
- _hashedErrorsLen = stringsLen;
- _hashedErrorsMax = stringsLen;
- _hashedErrors = new u64bit [_hashedErrorsLen];
-
- memcpy(_hashedErrors, strings, sizeof(u64bit) * _hashedErrorsLen);
-
- delete [] strings;
-
-#ifdef UNCOMPRESS_HASH_TABLE
- // Cost is just bucket searching.
- double work = (double)_hashedErrorsLen * approxMers / _tableSizeInEntries;
-#else
- // Cost is bucket searching + hash table lookups.
- double work = (double)_hashedErrorsLen * approxMers / _tableSizeInEntries + 2.0 * _hashedErrorsLen;
-#endif
-
- //fprintf(stderr, "Built "u32bitFMT" hashed errors at tableSize "u32bitFMT" (work=%f.0).\n",
- // _hashedErrorsLen,
- // _tableSizeInBits,
- // work);
-
- //for (u32bit i=0; i<_hashedErrorsLen; i++)
- // fprintf(stderr, "he["u32bitFMTW(5)"] = "u64bitHEX"\n", i, _hashedErrors[i]);
-
- return(work);
-}
-
-
-
-// Returns hits with _AT_MOST_ numMismatches mistakes.
-bool
-positionDB::getUpToNMismatches(u64bit mer,
- u32bit numMismatches,
- u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen) {
-
- PREFETCH(_hashedErrors); // Slightly better.
-
- posnLen = 0;
-
- if (_hashedErrors == 0L) {
- fprintf(stderr, "ERROR: Nobody initialized getUpToNMismatches() by calling setUpMismatchMatcher().\n");
- exit(1);
- }
-
- if (posnMax == 0) {
- posnMax = 16384;
- try {
- posn = new u64bit [posnMax];
- } catch (...) {
- fprintf(stderr, "positionDB::getUpToNMismatches()-- Can't allocate space for initial positions, requested "u64bitFMT" u64bit's.\n", posnMax);
- abort();
- }
- }
-
- u64bit orig = HASH(mer);
-
- // Optimization that didn't work. The idea was to compute all the
- // hashes with errors, then sort to gain better cache locality in
- // the lookups. The sort dominated.
- //
- // Another: Surprisingly, theq two getDecodedValue calls are faster
- // than a single getDecodedValues.
-
- for (u32bit e=0; e<_hashedErrorsLen; e++) {
- u64bit hash = orig ^ _hashedErrors[e];
- u64bit st, ed;
-
- if (_hashTable_BP) {
- st = getDecodedValue(_hashTable_BP, hash * _hashWidth, _hashWidth);
- ed = getDecodedValue(_hashTable_BP, hash * _hashWidth + _hashWidth, _hashWidth);
- } else {
- st = _hashTable_FW[hash];
- ed = _hashTable_FW[hash+1];
- }
-
- assert((_hashedErrors[e] & ~_hashMask) == 0);
- assert((hash & ~_hashMask) == 0);
-
- // Rebuild the mer from the hash and its check code.
- //
- // Compare the rebuilt mer and the original mer -- if there are
- // exactly N errors, it's a hit! (if there are fewer than N,
- // we'll find it when we look for N-1 errors).
- //
- // Before rebuilding, compute diffs on the chckBits only -- if
- // things are wildly different (the usual case) we'll get
- // enough difference here to abort. Remember, the chck bits
- // are not encoded, they're an exact copy from the unhashed
- // mer.
-
- if (st != ed) {
- for (u64bit i=ed-st, J=st * _wFin; i--; J += _wFin) {
- u64bit chck = getDecodedValue(_buckets, J, _chckWidth);
- u64bit diffs = chck ^ (mer & _mask2);
- u64bit d1 = diffs & u64bitNUMBER(0x5555555555555555);
- u64bit d2 = diffs & u64bitNUMBER(0xaaaaaaaaaaaaaaaa);
- u64bit err = countNumberOfSetBits64(d1 | (d2 >> 1));
-
- if (err <= numMismatches) {
- diffs = REBUILD(hash, chck) ^ mer;
- d1 = diffs & u64bitNUMBER(0x5555555555555555);
- d2 = diffs & u64bitNUMBER(0xaaaaaaaaaaaaaaaa);
- err = countNumberOfSetBits64(d1 | (d2 >> 1));
-
- if (err <= numMismatches)
- // err is junk, just need a parameter here
- loadPositions(J, posn, posnMax, posnLen, err);
- }
- }
- }
- }
-
- return(posnLen > 0);
-}
diff --git a/kmer/libkmer/positionDB-sort.C b/kmer/libkmer/positionDB-sort.C
deleted file mode 100644
index f5bdf6c..0000000
--- a/kmer/libkmer/positionDB-sort.C
+++ /dev/null
@@ -1,150 +0,0 @@
-#include "positionDB.H"
-#include "bio++.H"
-
-
-void
-adjustHeap(u64bit *C,
- u64bit *P, s64bit i, s64bit n) {
- u64bit c = C[i];
- u64bit p = P[i];
- s64bit j = (i << 1) + 1; // let j be the left child
-
- while (j < n) {
- if (j<n-1 && C[j] < C[j+1])
- j++; // j is the larger child
-
- if (c >= C[j]) // a position for M[i] has been found
- break;
-
- C[(j-1)/2] = C[j]; // Move larger child up a level
- P[(j-1)/2] = P[j];
-
- j = (j << 1) + 1;
- }
-
- C[(j-1)/2] = c;
- P[(j-1)/2] = p;
-}
-
-
-void
-positionDB::sortAndRepackBucket(u64bit b) {
- u64bit st = _bucketSizes[b];
- u64bit ed = _bucketSizes[b+1];
- u32bit le = (u32bit)(ed - st);
-
- if (ed < st)
- fprintf(stdout, "ERROR: Bucket "u64bitFMT" starts at "u64bitFMT" ends at "u64bitFMT"?\n", b, st, ed);
-
- if (le == 0)
- return;
-
- // One mer in the list? It's distinct and unique! (and doesn't
- // contribute to the position list space count)
- //
- if (le == 1) {
- _numberOfDistinct++;
- _numberOfUnique++;
- return;
- }
-
- // Allocate more space, if we need to.
- //
- if (_sortedMax <= le) {
- _sortedMax = le + 1024;
- delete [] _sortedChck;
- delete [] _sortedPosn;
- _sortedChck = new u64bit [_sortedMax];
- _sortedPosn = new u64bit [_sortedMax];
- }
-
- // Unpack the bucket
- //
- u64bit lens[3] = {_chckWidth, _posnWidth, 1 + _sizeWidth};
- u64bit vals[3] = {0};
- for (u64bit i=st, J=st * _wCnt; i<ed; i++, J += _wCnt) {
- getDecodedValues(_countingBuckets, J, 2, lens, vals);
- _sortedChck[i-st] = vals[0];
- _sortedPosn[i-st] = vals[1];
- }
-
- // Create the heap of lines.
- //
- int unsetBucket = 0;
-
- for (s64bit t=(le-2)/2; t>=0; t--) {
- if (_sortedPosn[t] == u64bitMASK(_posnWidth)) {
- unsetBucket = 1;
- fprintf(stdout, "ERROR: unset posn bucket="u64bitFMT" t="s64bitFMT" le="u32bitFMT"\n", b, t, le);
- }
-
- adjustHeap(_sortedChck, _sortedPosn, t, le);
- }
-
- if (unsetBucket)
- for (u32bit t=0; t<le; t++)
- fprintf(stdout, u32bitFMTW(4)"] chck="u64bitHEX" posn="u64bitFMT"\n", t, _sortedChck[t], _sortedPosn[t]);
-
- // Interchange the new maximum with the element at the end of the tree
- //
- for (s64bit t=le-1; t>0; t--) {
- u64bit tc = _sortedChck[t];
- u64bit tp = _sortedPosn[t];
-
- _sortedChck[t] = _sortedChck[0];
- _sortedPosn[t] = _sortedPosn[0];
-
- _sortedChck[0] = tc;
- _sortedPosn[0] = tp;
-
- adjustHeap(_sortedChck, _sortedPosn, 0, t);
- }
-
- // Scan the list of sorted mers, counting the number of distinct and unique,
- // and the space needed in the position list.
-
- u64bit entries = 1; // For t=0
-
- for (u32bit t=1; t<le; t++) {
- if (_sortedChck[t-1] > _sortedChck[t])
- fprintf(stdout, "ERROR: bucket="u64bitFMT" t="u32bitFMT" le="u32bitFMT": "u64bitHEX" > "u64bitHEX"\n",
- b, t, le, _sortedChck[t-1], _sortedChck[t]);
-
- if (_sortedChck[t-1] != _sortedChck[t]) {
- _numberOfDistinct++;
-
- if (_maximumEntries < entries)
- _maximumEntries = entries;
-
- if (entries == 1)
- _numberOfUnique++;
- else
- _numberOfEntries += entries + 1; // +1 for the length
-
- entries = 0;
- }
-
- entries++;
- }
-
- // Don't forget the last mer!
- //
- _numberOfDistinct++;
- if (_maximumEntries < entries)
- _maximumEntries = entries;
- if (entries == 1)
- _numberOfUnique++;
- else
- _numberOfEntries += entries + 1;
-
-
- // Repack the sorted entries
- //
- for (u64bit i=st, J=st * _wCnt; i<ed; i++, J += _wCnt) {
- vals[0] = _sortedChck[i-st];
- vals[1] = _sortedPosn[i-st];
- vals[2] = 0;
- setDecodedValues(_countingBuckets, J, 3, lens, vals);
- }
-}
-
diff --git a/kmer/libkmer/positionDB.C b/kmer/libkmer/positionDB.C
deleted file mode 100644
index 7d14f60..0000000
--- a/kmer/libkmer/positionDB.C
+++ /dev/null
@@ -1,1125 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <new>
-
-#include "bio++.H"
-#include "positionDB.H"
-#include "existDB.H"
-#include "libmeryl.H"
-
-#undef ERROR_CHECK_COUNTING
-#undef ERROR_CHECK_COUNTING_ENCODING
-#undef ERROR_CHECK_EMPTY_BUCKETS
-
-// This tests Chunlin Xiao's discovered bug -- if there are a small
-// number of unique mers, compared to distinct mers (2 * #unique_mers
-// < #distinct_mers, we would overflow the position pointer in
-// buckets. This enables a check that it doesn't occur.
-//
-// This has a fixed allocation size, and crashes on larger inputs.
-//
-#undef TEST_NASTY_BUGS
-
-// Tests that mers are masked out properly. Doesn't handle canonical
-// mers though.
-//
-#undef MER_REMOVAL_TEST
-
-
-positionDB::positionDB(char const *filename,
- u32bit merSize,
- u32bit merSkip,
- u32bit maxMismatch,
- bool loadData) {
- memset(this, 0, sizeof(positionDB));
-
- // loadData == false only for driver-posDB.C, and only so it can
- // dump stats on a posDB file.
-
- if (loadState(filename, true, false) == false) {
- fprintf(stderr, "positionDB()-- Tried to read state from '%s', but failed.\n", filename);
- exit(1);
- }
-
- if ((loadData) && (merSize != _merSizeInBases)) {
- fprintf(stderr, "positionDB()-- Tried to read state from '%s', but mer size is wrong (found "u32bitFMT", wanted "u32bitFMT").\n",
- filename, _merSizeInBases, merSize);
- exit(1);
- }
-
- if ((loadData) && (merSkip != _merSkipInBases)) {
- fprintf(stderr, "positionDB()-- Tried to read state from '%s', but mer skip is wrong (found "u32bitFMT", wanted "u32bitFMT").\n",
- filename, _merSkipInBases, merSkip);
- exit(1);
- }
-
-if ((loadData) && (maxMismatch != _nErrorsAllowed)) {
- fprintf(stderr, "positionDB()-- Tried to read state from '%s', but max number of mismatches is wrong (found "u32bitFMT", wanted "u32bitFMT").\n",
- filename, _nErrorsAllowed, maxMismatch);
- exit(1);
- }
-
- if (loadState(filename, true, loadData) == false) {
- fprintf(stderr, "positionDB()-- Tried to read state from '%s', but failed.\n", filename);
- exit(1);
- }
-}
-
-
-positionDB::positionDB(merStream *MS,
- u32bit merSize,
- u32bit merSkip,
- existDB *mask,
- existDB *only,
- merylStreamReader *counts,
- u32bit minCount,
- u32bit maxCount,
- u32bit maxMismatch,
- u32bit maxMemory,
- bool beVerbose) {
-
- memset(this, 0, sizeof(positionDB));
-
- // Guesstimate a nice table size based on the number of input mers
- // and the mersize, unless the user gave us a table size.
- //
- // We need to ensure that
- // 2 * merSize + posnWidth + 1 - 64 <= tblBits <= 2 * merSize - 4
- //
- // The catch is that we don't exactly know posnWidth right now. We
- // can overestimate it, though, based on the size of the sequence
- // that is backing the merStream.
- //
- // The second catch is that we don't want to make tblBits too big
- // or too small. If too big, we waste a lot of memory in the hash
- // table pointers, and if too small, we waste even more memory in
- // the data table (not to mention the algorithm dies because it
- // assumed buckets in the data table are small).
- //
- // The memory size is (roughly):
- //
- // 2^tblBits * log(numDistinctMers) +
- // numDistinctMers * (2*merSize - tblBits + 1 + log(numMers) +
- // (numMers - numUniqieMers) * log(numMers)
- //
- // this is approximately proportional to:
- //
- // 2^tblBits * posnWidth +
- // approxMers * (2*merSize - tblBits + 1 + posnWidth)
- //
- u64bit approxMers = MS->approximateNumberOfMers();
- u64bit posnWidth = logBaseTwo64(approxMers + 1);
-
- // Find the smallest and largest tblBits we could possibly use.
- //
- u64bit sm = 2 * merSize + posnWidth + 1 - 64;
- u64bit lg = 2 * merSize - 4;
-
- if (2 * merSize + posnWidth + 1 < 64)
- sm = 2;
-
- if (sm < 16)
- sm = 16;
-
- if (sm > lg) {
- fprintf(stderr, "ERROR: too many mers for this mersize!\n");
- fprintf(stderr, " sm = "u64bitFMT"\n", sm);
- fprintf(stderr, " lg = "u64bitFMT"\n", lg);
- fprintf(stderr, " merSize = "u32bitFMT" bits\n", 2 * merSize);
- fprintf(stderr, " approxMers = "u64bitFMT" mers\n", approxMers);
- fprintf(stderr, " posnWidth = "u64bitFMT" bits\n", posnWidth);
- exit(1);
- }
-
-
- // Iterate through all the choices, picking the one with the
- // smallest expected footprint.
- //
- {
-
- if (beVerbose) {
- fprintf(stderr, "potential configurations for approximately "u64bitFMT" "u32bitFMT"-mers (posnW="u64bitFMT").\n",
- approxMers, merSize, posnWidth);
- }
-
- u64bit mini = 0; // tblSize of the smallest found
- u64bit minm = ~mini; // memory size of the smallest found
- double minw = 0.0; // work of the smallest found
-
- u64bit memory = 0;
- double effort = 0;
-
- if (maxMemory == 0)
- maxMemory = ~u32bitZERO;
-
- for (u64bit i=sm; i<=lg; i++) {
-
- // These are only needed if maxMismatch is set, but it's
- // simpler to always set.
- //
- _merSizeInBases = merSize;
- _merSizeInBits = 2 * _merSizeInBases;
- _merSkipInBases = merSkip;
- _tableSizeInBits = i;
- _tableSizeInEntries = u64bitONE << _tableSizeInBits;
- _hashWidth = u32bitZERO;
- _hashMask = u64bitMASK(_tableSizeInBits);
- _chckWidth = _merSizeInBits - _tableSizeInBits;
- _posnWidth = u64bitZERO;
- _sizeWidth = 0;
-
- _shift1 = _merSizeInBits - _tableSizeInBits;
- _shift2 = _shift1 / 2;
- _mask1 = u64bitMASK(_tableSizeInBits);
- _mask2 = u64bitMASK(_shift1);
-
- // Everyone wants to know the memory size (in MB).
- //
- memory = ((u64bitONE << i) * posnWidth + approxMers * (2*merSize - i + 1 + posnWidth)) >> 23;
-
- // If we know we're looking for mismatches, we compute the amount
- // of work needed per lookup, and use that, instead of strict
- // memory sizing, to deicde the table size.
- //
- if (maxMismatch > 0)
- effort = setUpMismatchMatcher(maxMismatch, approxMers);
-
- // If our memory size is smaller than allowed, AND it's the
- // smallest, or the work is smaller, save the table size.
- //
- if ((memory < maxMemory) &&
- ((memory < minm) ||
- (effort < minw))) {
- mini = i;
- minm = memory;
- minw = effort;
- }
-
- if (beVerbose) {
- fprintf(stderr, "tblBits="u64bitFMTW(2)" shifts="u32bitFMTW(02)","u32bitFMTW(02)" -- size %8.3fGB -- work %8.3f%s\n",
- i, _shift1, _shift2, memory / 1024.0, effort, (mini == i) ? " ***" : "");
- }
- }
-
- _tableSizeInBits = mini;
- }
-
-
- if (_tableSizeInBits == 0) {
- fprintf(stderr, "ERROR: No positionDB parameters within allowed memory limit.\n");
- exit(1);
- }
-
-
- if (beVerbose) {
- u32bit s1 = 2*merSize-_tableSizeInBits;
- fprintf(stderr, "tblBits="u32bitFMT" s1="u32bitFMT" s2="u32bitFMT" -- merSize="u32bitFMT" bits + posnWidth="u64bitFMT" bits (est "u64bitFMT" mers) FINAL\n",
- _tableSizeInBits, s1, s1/2, merSize, posnWidth, approxMers);
- }
-
-
- _merSizeInBases = merSize;
- _merSizeInBits = 2 * _merSizeInBases;
- _merSkipInBases = merSkip;
- _tableSizeInEntries = u64bitONE << _tableSizeInBits;
- _hashWidth = u32bitZERO;
- _hashMask = u64bitMASK(_tableSizeInBits);
- _chckWidth = _merSizeInBits - _tableSizeInBits;
- _posnWidth = u64bitZERO;
- _sizeWidth = 0;
-
- if (maxCount == 0)
- maxCount = ~u32bitZERO;
-
- if (counts)
- _sizeWidth = (maxCount < ~u32bitZERO) ? logBaseTwo64(maxCount+1) : 32;
-
- _shift1 = _merSizeInBits - _tableSizeInBits;
- _shift2 = _shift1 / 2;
- _mask1 = u64bitMASK(_tableSizeInBits);
- _mask2 = u64bitMASK(_shift1);
-
-#if 0
- fprintf(stderr, "merSizeInBits "u32bitFMT"\n", _merSizeInBits);
- fprintf(stderr, "hashWidth "u32bitFMT"\n", _hashWidth);
- fprintf(stderr, "chckWidth "u32bitFMT"\n", _chckWidth);
- fprintf(stderr, "shift1 "u32bitFMT"\n", _shift1);
- fprintf(stderr, "shift2 "u32bitFMT"\n", _shift2);
-#endif
-
- if (maxMismatch > 0)
- setUpMismatchMatcher(maxMismatch, approxMers);
-
- build(MS, mask, only, counts, minCount, maxCount, beVerbose);
-}
-
-
-
-void
-positionDB::build(merStream *MS,
- existDB *mask,
- existDB *only,
- merylStreamReader *counts,
- u32bit minCount,
- u32bit maxCount,
- bool beVerbose) {
-
- _bucketSizes = 0L;
- _countingBuckets = 0L;
- _hashTable_BP = 0L;
- _hashTable_FW = 0L;
- _buckets = 0L;
- _positions = 0L;
-
- _wCnt = 0;
- _wFin = 0;
-
- // For get/setDecodedValues().
- u64bit lensC[4] = {~u64bitZERO, ~u64bitZERO, ~u64bitZERO, ~u64bitZERO};
- u64bit lensF[4] = {~u64bitZERO, ~u64bitZERO, ~u64bitZERO, ~u64bitZERO};
- u64bit vals[4] = {0};
- u64bit nval = (_sizeWidth == 0) ? 3 : 4;
-
- _numberOfMers = u64bitZERO;
- _numberOfPositions = u64bitZERO;
- _numberOfDistinct = u64bitZERO;
- _numberOfUnique = u64bitZERO;
- _numberOfEntries = u64bitZERO;
- _maximumEntries = u64bitZERO;
-
- // We assume later that these are already allocated.
- _sortedMax = 16384;
- _sortedChck = new u64bit [_sortedMax];
- _sortedPosn = new u64bit [_sortedMax];
-
- if (MS == 0L) {
- fprintf(stderr, "positionDB()-- ERROR: No merStream? Nothing to build a table with!\n");
- exit(1);
- }
-
- MS->rewind();
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 1) Count bucket sizes
- //
-
- // We'll later want to reuse the _bucketSizes space for storing the
- // hash table. To make it somewhat safe, we allocate the space as
- // u64bit, then cast it to be u32bit.
- //
- // bktAllocIsJunk tells us if we should release this memory (if we
- // need to allocate separate space for the hash table). We'd need
- // to do this if the hashWidth is more than 32 bits, but we won't
- // know that for a little bit.
- //
- // The _bucketSizes is offset by one from bktAlloc so that we don't
- // overwrite _bucketSizes when we are constructing hash table.
- //
- u64bit *bktAlloc;
- try {
- bktAlloc = new u64bit [_tableSizeInEntries / 2 + 4];
- } catch (std::bad_alloc) {
- fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
- fprintf(stderr, "positionDB()-- bktAlloc = new u64bit ["u64bitFMT"]\n", _tableSizeInEntries / 2 + 4);
- exit(1);
- }
- bool bktAllocIsJunk = false;
-
- bzero(bktAlloc, sizeof(u64bit) * (_tableSizeInEntries / 2 + 4));
-
- // Why +2? We try to reuse the bktAlloc space for the hash table,
- // which is constructed from the bucketSizes. The hashTable is
- // built from the bucketSizes. It definitely needs to be +1, and
- // so we use +2 just in case the human is being stupid again.
- //
- _bucketSizes = (u32bit *)(bktAlloc + 2);
-
-#ifdef ERROR_CHECK_COUNTING
- fprintf(stdout, "ERROR_CHECK_COUNTING is defined.\n");
- u32bit *_errbucketSizes = new u32bit [_tableSizeInEntries + 2];
- for (u64bit i=0; i<_tableSizeInEntries + 2; i++)
- _errbucketSizes[i] = u32bitZERO;
-#endif
-
- if (beVerbose)
- fprintf(stderr, " Allocated bucket size counting space with total size "u64bitFMT" KB\n", _tableSizeInEntries >> 8);
-
-
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
-
- // Two choices here
- //
- // 1) No masking or onlying is done. Stream the mers and just
- // count the positions. This is the original behavior.
- //
- // 2) Masking or onlying is done. Open the output stream file,
- // stream the mers by, checking for mask/only of both
- // forward and reverse mers. If either is found, push
- // the (forward) mer and position onto the stream.
- // close the output stream.
- //
- // Save the mer if it doesn't exist in the mask (both f and r),
- // or does exist in the only (either f or r), add it.
- //
- // The input databases for mask and only are (currently) made
- // using canonical mers. We halve the number of exists() by
- // also using canonical mers here.
- //
-
- MS->rewind();
-
- while (MS->nextMer(_merSkipInBases)) {
- _bucketSizes[ HASH(MS->theFMer()) ]++;
-
-#ifdef ERROR_CHECK_COUNTING
- _errbucketSizes[ HASH(MS->theFMer()) ]++;
-#endif
-
- _numberOfMers++;
- _numberOfPositions = MS->thePositionInStream();
- assert((_numberOfPositions >> 60) == 0);
- C->tick();
- }
-
-
- delete C;
- C = 0L;
-
- if (beVerbose)
- fprintf(stderr, " Found "u64bitFMT" mers (max position = "u64bitFMT")\n", _numberOfMers, _numberOfPositions);
-
- // This caught a nasty bug in merStream rewind(), and it's pretty
- // cheap, so I left it in. Search for the other DEBUGnumPositions.
- //
- u64bit DEBUGnumPositions = _numberOfPositions + 1;
-
- // This is _numberOfMers+1 because we need to store the first
- // position after the last mer. That is, if there are two mers, we
- // will store that the first mer is at position 0, the second mer
- // is at position 1, and the end of the second mer is at position
- // 2.
- //
- // In reality, it should be the number of distinct mers, not the
- // total number of mers, but we don't know that yet. And so
- // occasionally we'll make things too big and waste a bit of
- // memory.
- //
- _hashWidth = logBaseTwo64(_numberOfMers+1);
- _posnWidth = logBaseTwo64(_numberOfPositions+1);
-
-
-
- ///////////////////////////////////////////////////////////////////////////////
- //
- // 2) Allocate buckets and make bucketSizes be a pointer into them
- //
- _wCnt = _chckWidth + _posnWidth + 1 + _sizeWidth;
-
- lensC[0] = _chckWidth;
- lensC[1] = _posnWidth;
- lensC[2] = 1;
- lensC[3] = _sizeWidth;
-
- u64bit bucketsSpace = (_numberOfMers+1) * _wCnt / 64 + 1;
- u32bit endPosition = 0;
-
- if (beVerbose)
- fprintf(stderr, " Allocated "u64bitFMT"KB for buckets ("u64bitFMT" 64-bit words)\n", bucketsSpace >> 7, bucketsSpace);
- try {
- _countingBuckets = new u64bit [bucketsSpace];
- } catch (std::bad_alloc) {
- fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
- fprintf(stderr, "positionDB()-- _countingBuckets = new u64bit ["u64bitFMT"]\n", bucketsSpace);
- exit(1);
- }
-
- for (u64bit i=0; i<bucketsSpace; i++)
- _countingBuckets[i] = ~u64bitZERO;
-
- for (u64bit i=0; i<_tableSizeInEntries; i++) {
- endPosition += _bucketSizes[i];
- _bucketSizes[i] = endPosition;
- }
- _bucketSizes[_tableSizeInEntries] = endPosition;
-
-#ifdef ERROR_CHECK_COUNTING
- if (endPosition != _numberOfMers)
- fprintf(stdout, "ERROR_CHECK_COUNTING: BUCKETSIZE COUNTING PROBLEM -- endPos="u32bitFMT" != numMers="u64bitFMT"\n",
- endPosition, _numberOfMers);
-#endif
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 3) Build list of mers with positions
- //
- if (beVerbose)
- fprintf(stderr, " Building lists with positions.\n");
-
- C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
-
-#ifdef ERROR_CHECK_COUNTING_ENCODING
- fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING is defined!\n");
-#endif
-
-
- MS->rewind();
-
- while (MS->nextMer(_merSkipInBases)) {
- u64bit h = HASH(MS->theFMer());
-
-#ifdef ERROR_CHECK_COUNTING
- if (_bucketSizes[h] == 0) {
- char str[33];
- fprintf(stderr, "positionDB()-- ERROR_CHECK_COUNTING: Bucket "u64bitFMT" ran out of things! '%s'\n", h, MS->theFMer().merToString(str));
- fprintf(stderr, "positionDB()-- ERROR_CHECK_COUNTING: Stream is at "u64bitFMT"\n", MS->thePositionInStream());
- }
-#endif
-
- _bucketSizes[h]--;
-
-#ifdef ERROR_CHECK_COUNTING
- _errbucketSizes[h]--;
-#endif
-
-
-#ifdef ERROR_CHECK_EMPTY_BUCKETS
- // Check that everything is empty. Empty is defined as set to all 1's.
- getDecodedValues(_countingBuckets, (u64bit)_bucketSizes[h] * (u64bit)_wCnt, nval, lensC, vals);
-
- if (((~vals[0]) & u64bitMASK(lensC[0])) ||
- ((~vals[1]) & u64bitMASK(lensC[1])) ||
- ((~vals[2]) & u64bitMASK(lensC[2])) ||
- ((lensC[3] > 0) && ((~vals[3]) & u64bitMASK(lensC[3]))))
- fprintf(stdout, "ERROR_CHECK_EMPTY_BUCKETS: countingBucket not empty! pos=%lu 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
- _bucketSizes[h] * _wCnt,
- (~vals[0]) & u64bitMASK(lensC[0]),
- (~vals[1]) & u64bitMASK(lensC[1]),
- (~vals[2]) & u64bitMASK(lensC[2]),
- (~vals[3]) & u64bitMASK(lensC[3]));
-#endif
-
- vals[0] = CHECK(MS->theFMer());
- vals[1] = MS->thePositionInStream();
- vals[2] = 0;
- vals[3] = 0;
-
- setDecodedValues(_countingBuckets, (u64bit)_bucketSizes[h] * (u64bit)_wCnt, nval, lensC, vals);
-
-#ifdef ERROR_CHECK_COUNTING_ENCODING
- getDecodedValues(_countingBuckets, (u64bit)_bucketSizes[h] * (u64bit)_wCnt, nval, lensC, vals);
-
- if (vals[0] != CHECK(MS->theFMer()))
- fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: CHCK corrupted! Wanted "u64bitHEX" got "u64bitHEX"\n",
- CHECK(MS->theFMer()), vals[0]);
- if (vals[1] != MS->thePositionInStream())
- fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: POSN corrupted! Wanted "u64bitHEX" got "u64bitHEX"\n",
- MS->thePositionInStream(), vals[1]);
- if (vals[2] != 0)
- fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: UNIQ corrupted.\n");
- if (vals[3] != 0)
- fprintf(stdout, "ERROR_CHECK_COUNTING_ENCODING error: SIZE corrupted.\n");
-#endif
-
- C->tick();
- }
-
-
- delete C;
- C = 0L;
-
-#ifdef ERROR_CHECK_COUNTING
- for (u64bit i=0; i<_tableSizeInEntries; i++)
- if (_errbucketSizes[i] != 0)
- fprintf(stdout, "ERROR_CHECK_COUNTING: Bucket "u32bitFMT" wasn't filled fully? "u32bitFMT" left over.\n", i, _errbucketSizes[i]);
-
- delete [] _errbucketSizes;
- _errbucketSizes = 0L;
-#endif
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 4) Sort each bucket -- count:
- // 1) number of distinct mers
- // 2) number of unique mers
- // 3) number of entries in position table ( sum mercount+1 for all mercounts > 1)
- // also need to repack the sorted things
- //
- if (beVerbose)
- fprintf(stderr, " Sorting and repacking buckets ("u64bitFMT" buckets).\n", _tableSizeInEntries);
-
- C = new speedCounter(" %7.2f Mbuckets -- %5.2f Mbuckets/second\r", 1000000.0, 0x1ffffff, beVerbose);
- for (u64bit i=0; i<_tableSizeInEntries; i++) {
- sortAndRepackBucket(i);
- C->tick();
- }
- delete C;
- C = 0L;
-
- if (beVerbose)
- fprintf(stderr,
- " Found "u64bitFMTW(12)" total mers\n"
- " Found "u64bitFMTW(12)" distinct mers\n"
- " Found "u64bitFMTW(12)" unique mers\n"
- " Need "u64bitFMT" non-unique position list entries ("u64bitFMT" maximum count)\n",
- _numberOfMers, _numberOfDistinct, _numberOfUnique, _numberOfEntries, _maximumEntries);
-
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Compute the size of the final bucket position entry. It's
- // either a position into the sequence, or a pointer into a list of
- // positions. In rare cases, the pointer is larger than the
- // sequence position, and we need to do extra work.
- //
- // The width of position pointers (in buckets) is the max of
- // _posnWidth (a pointer to the sequence position) and
- // _pptrWidth (a pointer to an entry in the positions table).
- //
- _pptrWidth = logBaseTwo64(_numberOfEntries+1);
- if (_pptrWidth < _posnWidth)
- _pptrWidth = _posnWidth;
-
- _wFin = _chckWidth + _pptrWidth + 1 + _sizeWidth;
-
- lensF[0] = _chckWidth;
- lensF[1] = _pptrWidth;
- lensF[2] = 1;
- lensF[3] = _sizeWidth;
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 5) Allocate: real hash table, buckets and position table.
- //
-
- // XXXX how do we count the number of buckets/positions we never
- // use because they are masked out??
- //
- // If we are just thresholding (ignore things with count > 100)
- // it's easy, a simple loop over something.
- //
- // If we have an exist/only db....are they in the same order? Can
- // we loop over both at the same time and count that way? That'd
- // be cool! Mersize is the same, why can the table size be the
- // same too -- OK, if the existDB has a small number of mers in it,
- // then we don't need a large table.
-
- u64bit hs = _tableSizeInEntries * _hashWidth / 64 + 1;
- u64bit bs = _numberOfDistinct * _wFin / 64 + 1;
- u64bit ps = _numberOfEntries * _posnWidth / 64 + 1;
-
- if (_hashWidth <= 32) {
- if (beVerbose)
- fprintf(stderr, " Reusing bucket counting space for hash table.\n");
-
-#ifdef UNCOMPRESS_HASH_TABLE
- _hashTable_BP = 0L;
- _hashTable_FW = (u32bit *)bktAlloc;
-#else
- _hashTable_BP = bktAlloc;
- _hashTable_FW = 0L;
-#endif
-
- bktAllocIsJunk = false;
- } else {
-
- // Can't use the full-width hash table, since the data size is >
- // 32 bits -- we'd need to allocate 64-bit ints for it, and
- // that'll likely be too big...and we'd need to have
- // _hashTable_FW64 or something.
-
- if (beVerbose)
- fprintf(stderr, " Allocated "u64bitFMTW(10)"KB for hash table ("u64bitFMT" 64-bit words)\n", hs >> 7, hs);
- try {
- _hashTable_BP = new u64bit [hs];
- _hashTable_FW = 0L;
- } catch (std::bad_alloc) {
- fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
- fprintf(stderr, "positionDB()-- _hashTable_BP = new u64bit ["u64bitFMT"]\n", hs);
- exit(1);
- }
- bktAllocIsJunk = true;
- }
-
-
- // If we have enough space to reuse the counting space, reuse it.
- // Else, allocate more space.
- //
- // We need to ensure that there are enough bits and that the size
- // of a bucket didn't increase. If the bucket size did increase,
- // and we see more unique buckets than total mers (up to some
- // point) we overwrite data.
- //
- // Recall that bucketSpace ~= numberOfMers * wCnt
- //
- if ((bs < bucketsSpace) && (_wFin <= _wCnt)) {
- if (beVerbose)
- fprintf(stderr, " Reusing bucket space; Have: "u64bitFMT" Need: "u64bitFMT" (64-bit words)\n", bucketsSpace, bs);
-
- _buckets = _countingBuckets;
-
- bs = bucketsSpace; // for output at the end
- } else {
- if (beVerbose)
- fprintf(stderr, " Allocated "u64bitFMTW(10)"KB for buckets ("u64bitFMT" 64-bit words)\n", bs >> 7, bs);
- try {
- _buckets = new u64bit [bs];
- } catch (std::bad_alloc) {
- fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
- fprintf(stderr, "positionDB()-- _buckets = new u64bit ["u64bitFMT"]\n", bs);
- exit(1);
- }
- }
-
- if (beVerbose)
- fprintf(stderr, " Allocated "u64bitFMTW(10)"KB for positions ("u64bitFMT" 64-bit words)\n", ps >> 7, ps);
- try {
- _positions = new u64bit [ps];
- } catch (std::bad_alloc) {
- fprintf(stderr, "positionDB()-- caught std::bad_alloc in %s at line %d\n", __FILE__, __LINE__);
- fprintf(stderr, "positionDB()-- _positions = new u64bit ["u64bitFMT"\n", ps);
- exit(1);
- }
-
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // 6) Transfer from the sorted buckets to the hash table.
- //
- if (beVerbose)
- fprintf(stderr, " Transferring to final structure ("u64bitFMT" buckets).\n", _tableSizeInEntries);
-
- u64bit bucketStartPosition = 0;
-
- // Current positions and bit positions in the buckets and position list.
- //
- u64bit currentBbit = u64bitZERO; // Bit position into bucket
- u64bit currentPbit = u64bitZERO; // Bit position into positions
- u64bit currentPpos = u64bitZERO; // Value position into positions
-
-#ifdef TEST_NASTY_BUGS
- // Save the position array pointer of each bucket for debugging.
- //
- u64bit currentBpos = u64bitZERO; // Value position into bucket
- u32bit *posPtrCheck = new u32bit [65826038];
-#endif
-
- // We also take this opportunity to reset some statistics that are
- // wrong.
- //
- _numberOfMers = 0;
- _numberOfPositions = 0;
- _numberOfDistinct = 0;
- _numberOfUnique = 0;
- _numberOfEntries = 0;
- _maximumEntries = 0;
-
- C = new speedCounter(" %7.2f Mbuckets -- %5.2f Mbuckets/second\r", 1000000.0, 0x1ffffff, beVerbose);
-
- // We need b outside the loop!
- //
- u64bit b;
- for (b=0; b<_tableSizeInEntries; b++) {
- C->tick();
-
- // Set the start of the bucket -- we took pains to ensure that
- // we don't overwrite _bucketSizes[b], if we are reusing that
- // space for the hash table.
- //
- if (_hashTable_BP)
- setDecodedValue(_hashTable_BP, (u64bit)b * (u64bit)_hashWidth, _hashWidth, bucketStartPosition);
- else
- _hashTable_FW[b] = bucketStartPosition;
-
- // Get the number of mers in the counting bucket. The error
- // checking and sizing of _sortedChck and _sortedPosn was already
- // done in the sort.
- //
- u64bit st = _bucketSizes[b];
- u64bit ed = _bucketSizes[b+1];
- u32bit le = ed - st;
-
- // Unpack the check values
- //
- for (u64bit i=st, J=st * _wCnt; i<ed; i++, J += _wCnt) {
- getDecodedValues(_countingBuckets, J, 2, lensC, vals);
- _sortedChck[i-st] = vals[0];
- _sortedPosn[i-st] = vals[1];
- }
-
-
- // Walk through the counting bucket, adding things to the real
- // bucket as we see them. Mers with more than one position are
- // inserted into the bucket, and the positions inserted into the
- // position list.
-
- // start and end locations of the mer. For mers with only
- // one occurrance (unique mers), stM+1 == edM.
- //
- u32bit stM = u32bitZERO;
- u32bit edM = u32bitZERO;
-
- while (stM < le) {
-
- // Move to the next mer.
- //
- edM++;
-
- // Keep moving while the two mers are the same.
- //
- while ((edM < le) && (_sortedChck[stM] == _sortedChck[edM]))
- edM++;
-
- // edM is now the mer after the last. Write all mers from stM
- // up to edM to the final structure. If there is one mer, put
- // it in the bucket. If not, put a pointer to the position
- // array there.
-
- // We're in bucket b, looking at mer _sortedChck[stM]. Ask the
- // only/mask if that exists, if so do/do not include the mer.
- //
- bool useMer = true;
-
- if (edM - stM < minCount)
- useMer = false;
-
- if (edM - stM > maxCount)
- useMer = false;
-
- if ((useMer == true) && (mask || only)) {
-
- // MER_REMOVAL_DURING_XFER. Great. The existDB has
- // (usually) the canonical mer. We have the forward mer.
- // Well, no, we have the forward mers' hash and check. So,
- // we reconstruct the mer, reverse complement it, and then
- // throw the mer out if either the forward or reverse exists
- // (or doesn't exist).
-
- u64bit m = REBUILD(b, _sortedChck[stM]);
- u64bit r;
-
- if (mask) {
- if (mask->isCanonical()) {
- r = reverseComplementMer(_merSizeInBases, m);
- if (r < m)
- m = r;
- }
- if (mask->exists(m))
- useMer = false;
- }
-
- if (only) {
- if (only->isCanonical()) {
- r = reverseComplementMer(_merSizeInBases, m);
- if (r < m)
- m = r;
- }
- if (only->exists(m) == false)
- useMer = false;
- }
- }
-
- if (useMer) {
- _numberOfMers += edM - stM;
- _numberOfPositions += edM - stM;
- _numberOfDistinct++;
-
- if (stM+1 == edM) {
- _numberOfUnique++;
-
-#ifdef TEST_NASTY_BUGS
- posPtrCheck[currentBpos++] = _sortedPosn[stM];
-#endif
-
- vals[0] = _sortedChck[stM];
- vals[1] = _sortedPosn[stM];
- vals[2] = 1;
- vals[3] = 0;
-
- currentBbit = setDecodedValues(_buckets, currentBbit, nval, lensF, vals);
- bucketStartPosition++;
- } else {
- _numberOfEntries += edM - stM;
- if (_maximumEntries < edM - stM)
- _maximumEntries = edM - stM;
-
-#ifdef TEST_NASTY_BUGS
- posPtrCheck[currentBpos++] = currentPpos;
-#endif
-
- vals[0] = _sortedChck[stM];
- vals[1] = currentPpos;
- vals[2] = 0;
- vals[3] = 0;
-
- currentBbit = setDecodedValues(_buckets, currentBbit, nval, lensF, vals);
- bucketStartPosition++;
-
- // Store the positions. Store the number of positions
- // here, then store all positions.
- //
- // The positions are in the proper place in _sortedPosn,
- // and setDecodedValue masks out the extra crap, so no
- // temporary needed. Probably should be done with
- // setDecodedValues, but then we need another array telling
- // the sizes of each piece.
- //
- setDecodedValue(_positions, currentPbit, _posnWidth, edM - stM);
- currentPbit += _posnWidth;
- currentPpos++;
-
- for (; stM < edM; stM++) {
- if (_sortedPosn[stM] >= DEBUGnumPositions) {
- fprintf(stderr, "positionDB()-- ERROR: Got position "u64bitFMT", but only "u64bitFMT" available!\n",
- _sortedPosn[stM], DEBUGnumPositions);
- abort();
- }
- setDecodedValue(_positions, currentPbit, _posnWidth, _sortedPosn[stM]);
- currentPbit += _posnWidth;
- currentPpos++;
- }
- }
- } // useMer
-
- // All done with this mer.
- //
- stM = edM;
- } // while (stM < le)
- } // for each bucket
-
- // Set the end of the last bucket
- //
- if (_hashTable_BP)
- setDecodedValue(_hashTable_BP, b * _hashWidth, _hashWidth, bucketStartPosition);
- else
- _hashTable_FW[b] = bucketStartPosition;
-
- delete C;
-
- // Clear out the end of the arrays -- this is only so that we can
- // checksum the result.
- //
- if (_hashTable_BP) {
- b = b * _hashWidth + _hashWidth;
- setDecodedValue(_hashTable_BP, b, 64 - (b % 64), u64bitZERO);
- }
- setDecodedValue(_buckets, currentBbit, 64 - (currentBbit % 64), u64bitZERO);
- setDecodedValue(_positions, currentPbit, 64 - (currentPbit % 64), u64bitZERO);
-
-
- if (beVerbose) {
- fprintf(stderr, " Avail: Bucket "u64bitFMTW(12)" Position "u64bitFMTW(12)" (64-bit words)\n", bs, ps);
- fprintf(stderr, " Avail: Bucket "u64bitFMTW(12)" Position "u64bitFMTW(12)" (entries)\n", _numberOfDistinct, _numberOfEntries);
- fprintf(stderr, " Used: Bucket "u64bitFMTW(12)" Position "u64bitFMTW(12)" (64-bit words)\n", currentBbit / 64, currentPbit / 64);
- }
-
- // Reset the sizes to what we actually found. If we then
- // dump/reload, we shrink our footprint.
- //
- _numberOfDistinct = currentBbit / _wFin;
- _numberOfEntries = currentPbit / _posnWidth;
-
- if (beVerbose) {
- fprintf(stderr, " Used: Bucket "u64bitFMTW(12)" Position "u64bitFMTW(12)" (entries)\n", _numberOfDistinct, _numberOfEntries);
- fprintf(stderr,
- " Found "u64bitFMTW(12)" total mers\n"
- " Found "u64bitFMTW(12)" distinct mers\n"
- " Found "u64bitFMTW(12)" unique mers\n"
- " Need "u64bitFMT" non-unique position list entries ("u64bitFMT" maximum count)\n",
- _numberOfMers, _numberOfDistinct, _numberOfUnique, _numberOfEntries, _maximumEntries);
- }
-
-
- // If we removed mers, there is a small chance that our hash table
- // is too big -- we might have removed enoough mers to make the
- // width smaller. If so, rebuild the hash table.
- //
- // Also, hooray, we finally know the number of distinct mers, so we
- // can make this nice and tight
- //
- if (_hashTable_BP) {
- u32bit newHashWidth = 1;
- while ((_numberOfDistinct+1) > (u64bitONE << newHashWidth))
- newHashWidth++;
-
- if (newHashWidth != _hashWidth) {
- u64bit npos = 0;
- u64bit opos = 0;
-
- if (beVerbose)
- fprintf(stderr, " Rebuilding the hash table, from "u32bitFMT" bits wide to "u32bitFMT" bits wide.\n",
- _hashWidth, newHashWidth);
-
- for (u64bit z=0; z<_tableSizeInEntries+1; z++) {
- setDecodedValue(_hashTable_BP,
- npos,
- newHashWidth,
- getDecodedValue(_hashTable_BP, opos, _hashWidth));
- npos += newHashWidth;
- opos += _hashWidth;
- }
-
- // Clear the end again.
- setDecodedValue(_hashTable_BP, npos, 64 - (npos % 64), u64bitZERO);
- }
-
- _hashWidth = newHashWidth;
- }
-
-
- // If supplied, add in any counts. The meryl table is, sadly, in
- // the wrong order, and we must hash and search.
- //
- // Meryl _should_ be storing only forward mers, but we have no way
- // of checking.
- //
- // After all counts are loaded, check if we can compress the counts
- // space any. Check if the largestMerylCount is much smaller than
- // the space it is stored in. If so, we can compress the table.
- //
- u64bit largestMerylCount = 0;
- u64bit countsLoaded = 0;
-
- if (counts) {
- if (beVerbose)
- fprintf(stderr, " Loading "u64bitFMT" mercounts.\n", counts->numberOfDistinctMers());
-
- C = new speedCounter(" %7.2f Mmercounts -- %5.2f Mmercounts/second\r", 1000000.0, 0x1fffff, beVerbose);
-
- while (counts->nextMer()) {
- kMer k = counts->theFMer();
- u64bit c = counts->theCount();
- u64bit f = setCount(k, c);
- k.reverseComplement();
- u64bit r = setCount(k, c);
-
- if (f + r > 0) {
- countsLoaded++;
- if (largestMerylCount < c)
- largestMerylCount = c;
- }
-
- C->tick();
- }
-
- delete C;
-
- if (beVerbose)
- fprintf(stderr, " Loaded "u64bitFMT" mercounts; largest is "u64bitFMT".\n", countsLoaded, largestMerylCount);
-
- if (logBaseTwo64(largestMerylCount + 1) < _sizeWidth) {
- if (beVerbose)
- fprintf(stderr, " Compress sizes from "u32bitFMT" bits to "u32bitFMT" bits.\n",
- _sizeWidth,
- (u32bit)logBaseTwo64(largestMerylCount + 1));
-
- u64bit oSiz[4] = { _chckWidth, _pptrWidth, 1, _sizeWidth };
- u64bit nSiz[4] = { _chckWidth, _pptrWidth, 1, logBaseTwo64(largestMerylCount + 1) };
- u64bit tVal[4] = { 0, 0, 0, 0 };
-
- u64bit oP = 0, oS = oSiz[0] + oSiz[1] + oSiz[2] + oSiz[3];
- u64bit nP = 0, nS = nSiz[0] + nSiz[1] + nSiz[2] + nSiz[3];
-
- assert(nS < oS);
-
- C = new speedCounter(" %7.2f Mmercounts -- %5.2f Mmercounts/second\r", 1000000.0, 0x1fffff, beVerbose);
-
- for (u64bit bu=0; bu<_numberOfDistinct; bu++) {
- getDecodedValues(_buckets, oP, 4, oSiz, tVal);
- setDecodedValues(_buckets, nP, 4, nSiz, tVal);
-
- oP += oS;
- nP += nS;
-
- C->tick();
- }
-
- delete C;
-
- _sizeWidth = nSiz[3];
- _wFin = _chckWidth + _pptrWidth + 1 + _sizeWidth;
- }
- }
-
-
-#ifdef TEST_NASTY_BUGS
- // Unpack the bucket positions and check. Report the first one
- // that is broken.
- //
- for(u64bit bb=0; bb<currentBpos; bb++)
- if (posPtrCheck[bb] != getDecodedValue(_buckets, bb * _wFin + _chckWidth, _pptrWidth))
- fprintf(stderr, "Bucket %lu (at bitpos %lu) failed position check (wanted %lu got %lu)\n",
- bb,
- bb * _wFin,
- posPtrCheck[bb],
- getDecodedValue(_buckets, bb * _wFin + _chckWidth, _pptrWidth));
- delete [] posPtrCheck;
-#endif
-
-
-#ifdef MER_REMOVAL_TEST
-#warning MER_REMOVAL_TEST was not updated to deal with canonical mers
- if (beVerbose)
- fprintf(stderr, "positionDB()-- TESTING MER REMOVAL\n");
-
- MS->rewind();
- if (mask) {
- C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
- u32bit extraMer = 0;
- while (MS->nextMer(_merSkipInBases)) {
- u64bit mer = MS->theFMer();
- if (mask->exists(mer) && exists(mer))
- extraMer++;
- C->tick();
- }
- delete C;
- fprintf(stderr, "positionDB()-- mask: "u32bitFMT" mers extra!\n", extraMer);
- } else if (only) {
- C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
- u32bit missingMer = 0;
- while (MS->nextMer(_merSkipInBases)) {
- u64bit mer = MS->theFMer();
- if (only->exists(mer) && !exists(mer))
- missingMer++;
- C->tick();
- }
- delete C;
- fprintf(stderr, "positionDB()-- only: "u32bitFMT" mers missing!\n", missingMer);
- }
-#endif
-
- // Free the counting buckets if we aren't using the space for
- // something else.
- //
- if (_buckets != _countingBuckets)
- delete [] _countingBuckets;
-
- // In theory, we could move these to be immediately after the data
- // is useless.
- //
- _bucketSizes = 0L;
- _countingBuckets = 0L;
-
- delete [] _sortedChck;
- delete [] _sortedPosn;
-
- _sortedMax = 0;
- _sortedChck = 0L;
- _sortedPosn = 0L;
-
- if (bktAllocIsJunk)
- delete [] bktAlloc;
-}
-
-positionDB::~positionDB() {
- delete [] _hashTable_BP;
- delete [] _hashTable_FW;
- delete [] _buckets;
- delete [] _positions;
- delete [] _hashedErrors;
-}
diff --git a/kmer/libkmer/positionDB.H b/kmer/libkmer/positionDB.H
deleted file mode 100644
index 2d0c725..0000000
--- a/kmer/libkmer/positionDB.H
+++ /dev/null
@@ -1,241 +0,0 @@
-#ifndef POSITIONDB_H
-#define POSITIONDB_H
-
-#include <stdio.h>
-#include "bio++.H"
-#include "merStream.H"
-
-// The two existDB inputs can be either forward or canonical. If
-// canonical, we are smart enough to search exist/only with the
-// canonical mer.
-
-// Returns position in posn, resizing it if needed. Space is
-// allocated if none supplied. The following is valid:
-//
-// u64bit *posn = 0L;
-// u64bit posnMax = 0;
-// u64bit posnLen = 0;
-// if (get(somemer, posn, posnMax, posnLen)) {
-// do something with the positions
-// }
-//
-// exists() returns T/F if mer exists or not
-// count() returns the number of times that mer is present
-
-// Define this to use an uncompressed hash table when the width is 32
-// bits or less. Doing so is A LOT faster in mismatch lookups, but
-// does use more memory.
-#undef UNCOMPRESS_HASH_TABLE
-
-// Define this to leave out references to getTime(), speedCounter()
-// and make the positionDB build very quietly.
-#undef SILENTPOSITIONDB
-
-// Define these to enable some debugging methods
-#undef DEBUGPOSDB
-#undef DEBUGREBUILD
-
-class existDB;
-class merylStreamReader;
-
-class positionDB {
-public:
- positionDB(char const *filename,
- u32bit merSize,
- u32bit merSkip,
- u32bit maxMismatch,
- bool loadData=true);
-
- positionDB(merStream *MS,
- u32bit merSize,
- u32bit merSkip,
- existDB *mask,
- existDB *only,
- merylStreamReader *counts,
- u32bit minCount,
- u32bit maxCount,
- u32bit maxMismatch,
- u32bit maxMemory,
- bool beVerbose);
-
- ~positionDB();
-
-private:
- void build(merStream *MS,
- existDB *mask,
- existDB *only,
- merylStreamReader *counts,
- u32bit minCount,
- u32bit maxCount,
- bool beVerbose);
-
-private:
- void reallocateSpace(u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen,
- u64bit len);
-
- void loadPositions(u64bit v,
- u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen,
- u64bit& count);
-
-public:
- bool getExact(u64bit mer,
- u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen,
- u64bit& count);
- bool existsExact(u64bit mer);
- u64bit countExact(u64bit mer);
-
-public:
- void filter(u64bit lo, u64bit hi);
-
-private:
- double setUpMismatchMatcher(u32bit nErrorsAllowed, u64bit approxMers);
-public:
- bool getUpToNMismatches(u64bit mer,
- u32bit maxMismatches,
- u64bit*& posn,
- u64bit& posnMax,
- u64bit& posnLen);
-private:
- u64bit setCount(u64bit mer, u64bit count);
-
- // Save or load a built table
- //
-public:
- void saveState(char const *filename);
- bool loadState(char const *filename, bool beNoisy=false, bool loadData=true);
-
- void printState(FILE *stream);
-
- // Only really useful for debugging. Don't use.
- //
- void dump(char *name);
-
-
- bool checkREBUILD(u64bit m) {
-#define DEBUGREBUILD
-#ifdef DEBUGREBUILD
- u64bit h = HASH(m);
- u64bit c = CHECK(m);
- u64bit r = REBUILD(h, c);
- if (r != m) {
- fprintf(stderr, "shift1 = "u32bitFMT"\n", _shift1);
- fprintf(stderr, "shift2 = "u32bitFMT"\n", _shift2);
- fprintf(stderr, "M = "u64bitHEX"\n", m);
- fprintf(stderr, "H = "u64bitHEX"\n", h);
- fprintf(stderr, "C = "u64bitHEX"\n", c);
- fprintf(stderr, "R = "u64bitHEX"\n", r);
- return(false);
- }
- return(true);
-#else
- return(REBUILD(HASH(m), CHECK(m)) == m);
-#endif
- };
-
-private:
-
- u64bit HASH(u64bit k) {
- return(((k >> _shift1) ^ (k >> _shift2) ^ k) & _mask1);
- };
-
- u64bit CHECK(u64bit k) {
- return(k & _mask2);
- };
-
- u64bit REBUILD(u64bit h, u64bit c) {
- // Decode a HASH and a CHECK to get back the mer. You'd better
- // bloody PRAY you don't break this (test/test-rebuild.C). It
- // was a headache++ to write.
-
- u64bit sha = _shift1 - _shift2;
- u64bit msk = u64bitMASK(sha);
-
- // The check is exactly the mer....just not all there.
- u64bit mer = c;
-
- u64bit shf = sha - (_tableSizeInBits % 2);
- u64bit shg = 0;
- u64bit shh = _shift1;
-
- // Unrolling this is troublesome - we still need the tests,
- // bizarre merSize, tblSize combinations use lots of iterations
- // (when the merSize and tblSize are about the same, the CHECK is
- // small, and so we need to do lots of iterations).
-
- //fprintf(stderr, "shf="u64bitFMTW(2)" shg="u64bitFMTW(2)" shh="u64bitFMTW(2)" mer="u64bitHEX"\n", shf, shg, shh, mer);
-
- do {
- mer |= (((h >> shg) ^ (mer >> shg) ^ (mer >> shf)) & msk) << shh;
- //fprintf(stderr, "shf="u64bitFMTW(2)" shg="u64bitFMTW(2)" shh="u64bitFMTW(2)" mer="u64bitHEX"\n", shf, shg, shh, mer);
-
- shf += sha;
- shg += sha;
- shh += sha;
- } while ((shf < _merSizeInBits) && (shh < 64));
-
- mer &= u64bitMASK(_merSizeInBits);
-
- return(mer);
- };
-
- void sortAndRepackBucket(u64bit b);
-
- u32bit *_bucketSizes;
- u64bit *_countingBuckets;
- u64bit *_hashTable_BP; // Bit packed
- u32bit *_hashTable_FW; // Full width
- u64bit *_buckets;
-
- u64bit *_positions;
-
- u32bit _merSizeInBases;
- u32bit _merSizeInBits;
-
- u32bit _merSkipInBases;
-
- u64bit _tableSizeInEntries;
- u32bit _tableSizeInBits;
-
- u32bit _hashWidth; // Hash bith
- u32bit _chckWidth; // Check bits
- u32bit _posnWidth; // Positions in the sequence
- u32bit _pptrWidth; // Pointers to positions
- u32bit _sizeWidth; // Extra number in the table
-
- u64bit _hashMask;
-
- u32bit _wCnt;
- u32bit _wFin;
-
- u32bit _shift1;
- u32bit _shift2;
- u64bit _mask1;
- u64bit _mask2;
-
- u64bit _numberOfMers;
- u64bit _numberOfPositions;
- u64bit _numberOfDistinct;
- u64bit _numberOfUnique;
- u64bit _numberOfEntries;
- u64bit _maximumEntries;
-
- // For sorting the mers
- //
- u32bit _sortedMax;
- u64bit *_sortedChck;
- u64bit *_sortedPosn;
-
- // For the mismatch matcher
- u32bit _nErrorsAllowed;
- u32bit _hashedErrorsLen;
- u32bit _hashedErrorsMax;
- u64bit *_hashedErrors;
-};
-
-#endif // POSITIONDB_H
diff --git a/kmer/libkmer/test/Makefile b/kmer/libkmer/test/Makefile
deleted file mode 100644
index d548a32..0000000
--- a/kmer/libkmer/test/Makefile
+++ /dev/null
@@ -1,115 +0,0 @@
-include ../../Make.compilers
-
-# Bigger tblsize makes existDB much faster, but uses more memory (not
-# much, really). 23 is nice.
-
-all: test-maskonly-passed position-passed
- @echo "existDB has expensive tests. They take:"
- @echo " 17 minutes on 1.8GHz Quadxeon (with KMER=1)"
- @echo " 60 minutes on 2.8GHz P4 (with KMER=1)"
- @echo " 120 minutes on 2.0GHz G5 (with KMER=8)"
- @echo "If you really want to run them, do 'make exist-passed'."
-
-# ../../meryl/meryl -M equal 1 -s xp -o xp1
-# ../../meryl/meryl -Dt -n 1 -s xp1 > xp.uni.fasta
-
-# Dead code, removed.
-test-mertable:
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-mertable.o test-mertable.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-mertable test-mertable.o $(LIBS)
- ../../leaff/leaff -G 1000 5000 8000 > xp.fasta
- ./test-mertable xp.fasta
- echo test-mertable PASSED
- rm xp* junk*
-
-test-maskonly-passed:
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-maskonly.o test-maskonly.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-maskonly test-maskonly.o $(LIBS)
- ../../leaff/leaff -G 1000 5000 8000 > xp.fasta
- ../../meryl/meryl -B -f -m 14 -s xp.fasta -o xp
- ../../meryl/meryl -Dt -n 2 -s xp > xp.dup.fasta
- ./test-maskonly xp.fasta xp.dup.fasta xp.dup.fasta
- echo test-maskonky-passed PASSED
- rm xp* junk*
- touch test-maskonly-passed
-
-test-rebuild: test-rebuild.C ../positionDB.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-rebuild.o test-rebuild.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-rebuild test-rebuild.o $(LIBS)
- ./test-rebuild
- @echo test-rebuild-passed PASSED
- rm -f xp* junk*
-
-position-passed: position-passed1 position-passed2 exist-fast-passed
- touch position-passed
-xp.fasta: ../../leaff/leaff
- #../../leaff/leaff -G 1000 5000 8000 > xp.fasta
- ../../leaff/leaff -G 1 500 800 > xp.fasta
-yp.fasta: ../../leaff/leaff
- #../../leaff/leaff -G 100 500 1000 > yp.fasta
- ../../leaff/leaff -G 1 50 100 > yp.fasta
-position-passed1: testerp xp.fasta
- ./testerp -test1 xp.fasta
- touch position-passed1
-position-passed2: testerp xp.fasta yp.fasta
- ./testerp -test2 xp.fasta yp.fasta
- touch position-passed2
-exist-fast-passed: ../existDB xe.fasta
- ../existDB -mersize 14 -tblsize 21 -testfiles xe.fasta junk
- ../existDB -mersize 14 -tblsize 21 -testexistence xe.fasta
- rm -f xe.mcdat xe.mcidx
- ../../meryl/meryl -B -f -m 14 -s xe.fasta -o xe
- ../existDB -mersize 14 -tblsize 21 -testexhaustive xe.fasta xe
- touch exist-fast-passed
-
-
-exist-passed: exist-passed1 exist-passed2 exist-passed3
- touch exist-passed
-xe.fasta: ../../leaff/leaff
- #../../leaff/leaff -G 1000 5000 8000 > xe.fasta
- ../../leaff/leaff -G 1 500 800 > xe.fasta
-exist-passed1: ../existDB xe.fasta
- ../existDB -mersize 17 -tblsize 23 -testfiles xe.fasta junk
- rm -f junk*
- touch exist-passed1
-exist-passed2: ../existDB xe.fasta
- ../existDB -mersize 17 -tblsize 23 -testexistence xe.fasta
- rm -f junk*
- touch exist-passed2
-exist-passed3: ../existDB xe.fasta ../../meryl/meryl
- rm -f xe.mcdat xe.mcidx
- ../../meryl/meryl -B -f -m 17 -s xe.fasta -o xe
- ../existDB -mersize 17 -tblsize 23 -testexhaustive xe.fasta xe
- rm -f junk*
- touch exist-passed3
-
-
-INCLUDE = -I../../libbio -I../../libseq -I../../libutil -I../../libmeryl -I..
-LIBS = -L../../libbio -L../../libseq -L../../libutil -L../../libmeryl -L.. -lkmer -lmeryl -lbio -lutil
-DBGOPT = -DERROR_CHECK_COUNTING -DERROR_CHECK_COUNTING_ENCODING -DERROR_CHECK_EMPTY_BUCKETS
-
-testerp: ../positionDB.C ../positionDB.H ../positionDB-access.C ../positionDB-dump.C ../positionDB-sort.C ../positionDB-file.C
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o driverp.o ../driver-posDB.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o positionDB.o ../positionDB.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o positionDB-access.o ../positionDB-access.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o positionDB-dump.o ../positionDB-dump.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o positionDB-sort.o ../positionDB-sort.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o positionDB-file.o ../positionDB-file.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o testerp driverp.o positionDB.o positionDB-access.o positionDB-dump.o positionDB-sort.o positionDB-file.o $(LIBS) -lm
-
-# XXX: There isn't any reason we need to build testere, we could
-# just use ../existDB (as it did before!)
-testere: ../existDB.C ../existDB-create-from-fasta.C ../existDB-create-from-meryl.C ../existDB-state.C
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o drivere.o ../driver-existDB.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o existDB.o ../existDB.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o existDB-create-from-fasta.o ../existDB-create-from-fasta.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o existDB-create-from-meryl.o ../existDB-create-from-meryl.C $(INCLUDE)
- $(CXX) $(DBGOPT) $(CXXFLAGS_COMPILE) -c -o existDB-state.o ../existDB-state.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o testere drivere.o existDB.o existDB-create-from-fasta.o existDB-create-from-meryl.o existDB-state.o $(LIBS) -lm
-
-clean:
- rm -f *passed*
- rm -f testerp *.o xp.fasta* yp.fasta*
- rm -f testere junk* xe.fasta* xe.mcidx xe.mcdat xe.merStream
- rm -f test-maskonly xp.dup.fasta xp.mcidx xp.mcdat
- rm -f test-rebuild
diff --git a/kmer/libkmer/test/test-maskonly.C b/kmer/libkmer/test/test-maskonly.C
deleted file mode 100644
index 368a90f..0000000
--- a/kmer/libkmer/test/test-maskonly.C
+++ /dev/null
@@ -1,110 +0,0 @@
-#include "bio++.H"
-#include "existDB.H"
-#include "positionDB.H"
-
-// Tests a positionDB when using an existDB for masking.
-//
-// existDB can be either include or exclude
-// positionDB can use include, exclude or threshold
-//
-
-#define MERSIZE 14
-
-int
-main(int argc, char **argv) {
- existDB *include;
- existDB *exclude;
- positionDB *full;
- positionDB *incl;
- positionDB *excl;
- positionDB *thrs;
-
- if (argc != 4) {
- fprintf(stderr, "usage: %s seq.fasta mask.fasta incl.fasta\n", argv[0]);
- exit(1);
- }
-
- char *seqName = argv[1];
- char *mskName = argv[2];
- char *incName = argv[3];
-
- fprintf(stderr, "BUILDING EXCLUDE\n");
- exclude = new existDB(mskName, MERSIZE, existDBnoFlags, u32bitZERO, ~u32bitZERO);
-
- fprintf(stderr, "BUILDING INCLUDE\n");
- include = new existDB(incName, MERSIZE, existDBnoFlags, u32bitZERO, ~u32bitZERO);
-
- seqStream *F = new seqStream(seqName, true);
- merStream *T = new merStream(new kMerBuilder(MERSIZE), F);
-
- fprintf(stderr, "BUILDING FULL\n");
- full = new positionDB(T, MERSIZE, 0, 0L, 0L, 0L, 0, 0, 0, 0, true);
- full->saveState("junk-full");
- delete full;
-
- fprintf(stderr, "BUILDING INCL\n");
- incl = new positionDB(T, MERSIZE, 0, 0L, include, 0L, 0, 0, 0, 0, true);
- incl->saveState("junk-incl");
- delete incl;
-
- fprintf(stderr, "BUILDING EXCL\n");
- excl = new positionDB(T, MERSIZE, 0, exclude, 0L, 0L, 0, 0, 0, 0, true);
- excl->saveState("junk-excl");
- delete excl;
-
- fprintf(stderr, "BUILDING THRS\n");
- thrs = new positionDB(T, MERSIZE, 0, 0L, 0L, 0L, 1, 0, 0, 0, true);
- thrs->saveState("junk-thrs");
- delete thrs;
-
- full = new positionDB("junk-full", MERSIZE, 0, 0);
- incl = new positionDB("junk-incl", MERSIZE, 0, 0);
- excl = new positionDB("junk-excl", MERSIZE, 0, 0);
- thrs = new positionDB("junk-thrs", MERSIZE, 0, 0);
-
- char themer[1000];
- u32bit mernum = 0;
-
- u32bit err = 0;
-
- // Check everything looks ok
- T->rewind();
- while (T->nextMer()) {
-
- if (!full->existsExact(T->theFMer())) {
- fprintf(stderr, "Didn't find mer "u32bitFMT" %s in full.\n", mernum, T->theFMer().merToString(themer));
- err++;
- }
-
- if (include->exists(T->theFMer())) {
- if (!incl->existsExact(T->theFMer())) {
- fprintf(stderr, "Didn't find mer "u32bitFMT" %s in incl.\n", mernum, T->theFMer().merToString(themer));
- err++;
- }
- } else {
- if (incl->existsExact(T->theFMer())) {
- fprintf(stderr, "Found extra mer "u32bitFMT" %s in incl.\n", mernum, T->theFMer().merToString(themer));
- err++;
- }
- }
-
- if (exclude->exists(T->theFMer())) {
- if (excl->existsExact(T->theFMer())) {
- fprintf(stderr, "Found extra mer "u32bitFMT" %s in excl.\n", mernum, T->theFMer().merToString(themer));
- err++;
- }
- } else {
- if (!excl->existsExact(T->theFMer())) {
- fprintf(stderr, "Didn't find mer "u32bitFMT" %s in excl.\n", mernum, T->theFMer().merToString(themer));
- err++;
- }
- }
-
- mernum++;
- }
-
- delete T;
- delete F;
-
- exit(err > 0);
-}
diff --git a/kmer/libkmer/test/test-mertable.C b/kmer/libkmer/test/test-mertable.C
deleted file mode 100644
index 2d40603..0000000
--- a/kmer/libkmer/test/test-mertable.C
+++ /dev/null
@@ -1,15 +0,0 @@
-#include "bio++.H"
-#include "merTable.H"
-
-
-int
-main(int argc, char **argv) {
- merTable X;
-
- chainedSequence *CS = new chainedSequence();
- CS->setSource(argv[1]);
- CS->finish();
-
- X.build(CS, 8);
-}
-
diff --git a/kmer/libkmer/test/test-rebuild.C b/kmer/libkmer/test/test-rebuild.C
deleted file mode 100644
index a943baf..0000000
--- a/kmer/libkmer/test/test-rebuild.C
+++ /dev/null
@@ -1,50 +0,0 @@
-#include "bio++.H"
-#include "existDB.H"
-#include "positionDB.H"
-
-// Tests a positionDB when using an existDB for masking.
-//
-// existDB can be either include or exclude
-// positionDB can use include, exclude or threshold
-//
-
-int
-main(int argc, char **argv) {
-
- u64bit maxMers = u64bitONE << 25;
-
- for (u32bit merSize=8; merSize<33; merSize++) {
- fprintf(stderr, "Testing "u64bitFMT" Mmers at merSize "u32bitFMT".\n", maxMers, merSize);
-
- kMerBuilder *K = new kMerBuilder(merSize);
- merStream *T = new merStream(K, "acgcgactcgagctacgagcgatcacgacgactacgagca", 40);
- positionDB *P = new positionDB(T, merSize, 0, 0L, 0L, 0L, 0, 0, false, true);
-
- u64bit p = 0;
- u64bit f = 0;
-
- mt_s *mts = mtInit(3492);
- u64bit msk = u64bitMASK(2*merSize);
- u64bit cnt = maxMers;
-
- while (cnt--) {
- if (P->checkREBUILD(mtRandom64(mts) & msk) == false) {
- f++;
- } else {
- p++;
- }
- }
-
- if (f) {
- fprintf(stderr, "PASS: "u64bitFMT" FAIL: "u64bitFMT"\n", p, f);
- exit(1);
- }
-
- free(mts);
-
- delete P;
- delete T;
- }
-
- exit(0);
-}
diff --git a/kmer/libmeryl/Make.include b/kmer/libmeryl/Make.include
deleted file mode 100644
index fb6b89b..0000000
--- a/kmer/libmeryl/Make.include
+++ /dev/null
@@ -1,15 +0,0 @@
-# -*- makefile -*-
-
-LIBUTL/ :=$(realpath $/../libutil/)/
-LIBBIO/ :=$(realpath $/../libbio/)/
-LIBSEQ/ :=$(realpath $/../libseq/)/
-
-$/.CXX_SRCS := $/libmeryl.C
-$/.CXX_INCS := $/libmeryl.H
-$/.CXX_LIBS := $/libmeryl.a
-$/.CLEAN := $/*.o
-
-$/libmeryl.a : $/libmeryl.o
-
-$(eval $/%.d $/%.o: CXXFLAGS+=-I${LIBBIO/} -I${LIBSEQ/} -I${LIBUTL/})
-
diff --git a/kmer/libmeryl/libmeryl.C b/kmer/libmeryl/libmeryl.C
deleted file mode 100644
index b41885c..0000000
--- a/kmer/libmeryl/libmeryl.C
+++ /dev/null
@@ -1,490 +0,0 @@
-#include "libmeryl.H"
-
-#define LIBMERYL_HISTOGRAM_MAX 1048576
-
-// 0123456789012345
-static char *ImagicV = "merylStreamIv03\n";
-static char *ImagicX = "merylStreamIvXX\n";
-static char *DmagicV = "merylStreamDv03\n";
-static char *DmagicX = "merylStreamDvXX\n";
-static char *PmagicV = "merylStreamPv03\n";
-static char *PmagicX = "merylStreamPvXX\n";
-
-merylStreamReader::merylStreamReader(const char *fn, u32bit ms) {
-
- if (fn == 0L) {
- fprintf(stderr, "ERROR - no counted database file specified.\n");
- exit(1);
- }
-
- // Open the files
- //
- char *inpath = new char [strlen(fn) + 8];
-
- sprintf(inpath, "%s.mcidx", fn);
- _IDX = new bitPackedFile(inpath);
-
- sprintf(inpath, "%s.mcdat", fn);
- _DAT = new bitPackedFile(inpath);
-
- sprintf(inpath, "%s.mcpos", fn);
- if (fileExists(inpath))
- _POS = new bitPackedFile(inpath);
- else
- _POS = 0L;
-
- delete [] inpath;
-
- // Verify that they are what they should be, and read in the header
- //
- char Imagic[16] = {0};
- char Dmagic[16] = {0};
- char Pmagic[16] = {0};
- bool fail = false;
-
- for (u32bit i=0; i<16; i++) {
- Imagic[i] = _IDX->getBits(8);
- Dmagic[i] = _DAT->getBits(8);
- if (_POS)
- Pmagic[i] = _POS->getBits(8);
- }
- if (strncmp(Imagic, ImagicX, 16) == 0) {
- fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcidx is an INCOMPLETE merylStream index file!\n", fn);
- fail = true;
- }
- if (strncmp(Imagic, ImagicX, 13) != 0) {
- fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcidx is not a merylStream index file!\n", fn);
- fail = true;
- }
- if (strncmp(Dmagic, DmagicX, 16) == 0) {
- fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcdat is an INCOMPLETE merylStream data file!\n", fn);
- fail = true;
- }
- if (strncmp(Dmagic, DmagicX, 13) != 0) {
- fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcdat is not a merylStream data file!\n", fn);
- fail = true;
- }
- if ((Imagic[13] != Dmagic[13]) ||
- (Imagic[14] != Dmagic[14])) {
- fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcidx and %s.mcdat are different versions!\n", fn, fn);
- fail = true;
- }
-#warning not checking pmagic
-
- if (fail)
- exit(1);
-
- _idxIsPacked = _IDX->getBits(32);
- _datIsPacked = _IDX->getBits(32);
- _posIsPacked = _IDX->getBits(32);
-
- _merSizeInBits = _IDX->getBits(32) << 1;
- _merCompression = _IDX->getBits(32);
- _prefixSize = _IDX->getBits(32);
- _merDataSize = _merSizeInBits - _prefixSize;
-
- _numUnique = _IDX->getBits(64);
- _numDistinct = _IDX->getBits(64);
- _numTotal = _IDX->getBits(64);
-
- _histogramHuge = 0;
- _histogramLen = 0;
- _histogramMaxValue = 0;
- _histogram = 0L;
-
- u32bit version = atoi(Imagic + 13);
-
- if (version > 1) {
- _histogramHuge = _IDX->getBits(64);
- _histogramLen = _IDX->getBits(64);
- _histogramMaxValue = _IDX->getBits(64);
- _histogram = new u64bit [_histogramLen];
-
- for (u32bit i=0; i<_histogramLen; i++)
- _histogram[i] = _IDX->getBits(64);
- }
-
- _thisBucket = u64bitZERO;
- _thisBucketSize = getIDXnumber();
- _numBuckets = u64bitONE << _prefixSize;
-
- _thisMer.setMerSize(_merSizeInBits >> 1);
- _thisMer.clear();
- _thisMerCount = u64bitZERO;
-
- _thisMerPositionsMax = 0;
- _thisMerPositions = 0L;
-
- _validMer = true;
-
-#ifdef SHOW_VARIABLES
- fprintf(stderr, "_merSizeInBits = "u32bitFMT"\n", _merSizeInBits);
- fprintf(stderr, "_merCompression = "u32bitFMT"\n", _merCompression);
- fprintf(stderr, "_prefixSize = "u32bitFMT"\n", _prefixSize);
- fprintf(stderr, "_merDataSize = "u32bitFMT"\n", _merDataSize);
- fprintf(stderr, "_numUnique = "u64bitFMT"\n", _numUnique);
- fprintf(stderr, "_numDistinct = "u64bitFMT"\n", _numDistinct);
- fprintf(stderr, "_numTotal = "u64bitFMT"\n", _numTotal);
- fprintf(stderr, "_thisBucket = "u64bitFMT"\n", _thisBucket);
- fprintf(stderr, "_thisBucketSize = "u64bitFMT"\n", _thisBucketSize);
- fprintf(stderr, "_thisMerCount = "u64bitFMT"\n", _thisMerCount);
-#endif
-
- if ((ms > 0) && (_merSizeInBits >> 1 != ms)) {
- fprintf(stderr, "merylStreamReader()-- ERROR: User requested mersize "u32bitFMT" but '%s' is mersize "u32bitFMT"\n",
- ms, fn, _merSizeInBits >> 1);
- exit(1);
- }
-}
-
-
-merylStreamReader::~merylStreamReader() {
- delete _IDX;
- delete _DAT;
- delete _POS;
- delete [] _thisMerPositions;
- delete [] _histogram;
-}
-
-
-
-bool
-merylStreamReader::nextMer(void) {
-
- // Use a while here, so that we skip buckets that are empty
- //
- while ((_thisBucketSize == 0) && (_thisBucket < _numBuckets)) {
- _thisBucketSize = getIDXnumber();
- _thisBucket++;
- }
-
- if (_thisBucket >= _numBuckets)
- return(_validMer = false);
-
- // Before you get rid of the clear() -- if, say, the list of mers
- // is sorted and we can shift the mer to make space for the new
- // stuff -- make sure that nobody is calling reverseComplement()!
- //
- _thisMer.clear();
- _thisMer.readFromBitPackedFile(_DAT, _merDataSize);
- _thisMer.setBits(_merDataSize, _prefixSize, _thisBucket);
-
- _thisMerCount = getDATnumber();
-
- _thisBucketSize--;
-
- if (_POS) {
- if (_thisMerPositionsMax < _thisMerCount) {
- delete [] _thisMerPositions;
- _thisMerPositionsMax = _thisMerCount + 1024;
- _thisMerPositions = new u32bit [_thisMerPositionsMax];
- }
- for (u32bit i=0; i<_thisMerCount; i++) {
- _thisMerPositions[i] = _POS->getBits(32);
- }
- }
-
- return(true);
-}
-
-
-
-
-
-
-merylStreamWriter::merylStreamWriter(const char *fn,
- u32bit merSize,
- u32bit merComp,
- u32bit prefixSize,
- bool positionsEnabled) {
-
- char *outpath = new char [strlen(fn) + 17];
-
- sprintf(outpath, "%s.mcidx", fn);
- _IDX = new bitPackedFile(outpath, 0, true);
-
- sprintf(outpath, "%s.mcdat", fn);
- _DAT = new bitPackedFile(outpath, 0, true);
-
- if (positionsEnabled) {
- sprintf(outpath, "%s.mcpos", fn);
- _POS = new bitPackedFile(outpath, 0, true);
- } else {
- _POS = 0L;
- }
-
- delete [] outpath;
-
- // Save really important stuff
-
- // unpacked --> write 0.42M mers/sec on 8 threads, merge 3.3M mers/sec
- // packed --> write 0.77M mers/sec on 8 threads, merge 3.9M mers/sec
- //
- // This sucks.
- //
- _idxIsPacked = 1;
- _datIsPacked = 1;
- _posIsPacked = 0;
-
- _merSizeInBits = merSize * 2;
- _merCompression = merComp;
- _prefixSize = prefixSize;
- _merDataSize = _merSizeInBits - _prefixSize;
-
- _thisBucket = u64bitZERO;
- _thisBucketSize = u64bitZERO;
- _numBuckets = u64bitONE << _prefixSize;
-
- _numUnique = u64bitZERO;
- _numDistinct = u64bitZERO;
- _numTotal = u64bitZERO;
-
- _thisMerIsBits = false;
- _thisMerIskMer = false;
-
- _thisMer.setMerSize(_merSizeInBits >> 1);
- _thisMer.clear();
-
- _thisMerPre = u64bitZERO;
- _thisMerMer = u64bitZERO;
-
- _thisMerPreSize = prefixSize;
- _thisMerMerSize = 2 * merSize - prefixSize;
-
- _thisMerCount = u64bitZERO;
-
- for (u32bit i=0; i<16; i++)
- _IDX->putBits(ImagicX[i], 8);
-
- _IDX->putBits(_idxIsPacked, 32);
- _IDX->putBits(_datIsPacked, 32);
- _IDX->putBits(_posIsPacked, 32);
-
- _IDX->putBits(_merSizeInBits >> 1, 32);
- _IDX->putBits(_merCompression, 32);
- _IDX->putBits(_prefixSize, 32);
- _IDX->putBits(_numUnique, 64);
- _IDX->putBits(_numDistinct, 64);
- _IDX->putBits(_numTotal, 64);
-
- _histogramHuge = 0;
- _histogramLen = LIBMERYL_HISTOGRAM_MAX;
- _histogramMaxValue = 0;
- _histogram = new u64bit [_histogramLen];
-
- for (u32bit i=0; i<_histogramLen; i++)
- _histogram[i] = 0;
-
- _IDX->putBits(_histogramHuge, 64);
- _IDX->putBits(_histogramLen, 64);
- _IDX->putBits(_histogramMaxValue, 64);
- for (u32bit i=0; i<_histogramLen; i++)
- _IDX->putBits(_histogram[i], 64);
-
- for (u32bit i=0; i<16; i++)
- _DAT->putBits(DmagicX[i], 8);
-
- if (_POS)
- for (u32bit i=0; i<16; i++)
- _POS->putBits(PmagicX[i], 8);
-}
-
-
-merylStreamWriter::~merylStreamWriter() {
-
- writeMer();
-
- // Finish writing the buckets.
- //
- while (_thisBucket < _numBuckets + 2) {
- setIDXnumber(_thisBucketSize);
- _thisBucketSize = 0;
- _thisBucket++;
- }
-
- // Seek back to the start and rewrite the magic numbers
- //
- _IDX->seek(0);
- _DAT->seek(0);
-
- for (u32bit i=0; i<16; i++)
- _IDX->putBits(ImagicV[i], 8);
-
- _IDX->putBits(_idxIsPacked, 32);
- _IDX->putBits(_datIsPacked, 32);
- _IDX->putBits(_posIsPacked, 32);
-
- _IDX->putBits(_merSizeInBits >> 1, 32);
- _IDX->putBits(_merCompression, 32);
- _IDX->putBits(_prefixSize, 32);
- _IDX->putBits(_numUnique, 64);
- _IDX->putBits(_numDistinct, 64);
- _IDX->putBits(_numTotal, 64);
-
- _IDX->putBits(_histogramHuge, 64);
- _IDX->putBits(_histogramLen, 64);
- _IDX->putBits(_histogramMaxValue, 64);
- for (u32bit i=0; i<_histogramLen; i++)
- _IDX->putBits(_histogram[i], 64);
- delete _IDX;
-
- delete [] _histogram;
-
- for (u32bit i=0; i<16; i++)
- _DAT->putBits(DmagicV[i], 8);
- delete _DAT;
-
- if (_POS) {
- for (u32bit i=0; i<16; i++)
- _POS->putBits(PmagicV[i], 8);
- delete _POS;
- }
-}
-
-
-void
-merylStreamWriter::writeMer(void) {
-
- if (_thisMerCount == 0)
- return;
-
- _numTotal += _thisMerCount;
- _numDistinct++;
-
- if (_thisMerCount < LIBMERYL_HISTOGRAM_MAX)
- _histogram[_thisMerCount]++;
- else
- _histogramHuge++;
- if (_histogramMaxValue < _thisMerCount)
- _histogramMaxValue = _thisMerCount;
-
- assert((_thisMerIsBits == false) || (_thisMerIskMer == false));
-
- if (_thisMerIsBits) {
- if (_thisMerCount == 1) {
- _DAT->putBits(_thisMerMer, _thisMerMerSize);
- setDATnumber(1);
- _thisBucketSize++;
- _numUnique++;
- } else {
- _DAT->putBits(_thisMerMer, _thisMerMerSize);
- setDATnumber(_thisMerCount);
- _thisBucketSize++;
- }
-
- } else {
- if (_thisMerCount == 1) {
- _thisMer.writeToBitPackedFile(_DAT, _merDataSize);
- setDATnumber(1);
- _thisBucketSize++;
- _numUnique++;
- } else if (_thisMerCount > 1) {
- _thisMer.writeToBitPackedFile(_DAT, _merDataSize);
- setDATnumber(_thisMerCount);
- _thisBucketSize++;
- }
- }
-}
-
-
-
-void
-merylStreamWriter::addMer(kMer &mer, u32bit count, u32bit *positions) {
- u64bit val;
-
- if (_thisMerIskMer == false) {
- _thisMerIskMer = true;
- assert(_thisMerIsBits == false);
- }
-
- // Fail if we see a smaller mer than last time.
- //
- if (mer < _thisMer) {
- char str[1024];
- fprintf(stderr, "merylStreamWriter::addMer()-- ERROR: your mer stream isn't sorted increasingly!\n");
- fprintf(stderr, "merylStreamWriter::addMer()-- last: %s\n", _thisMer.merToString(str));
- fprintf(stderr, "merylStreamWriter::addMer()-- this: %s\n", mer.merToString(str));
- exit(1);
- }
-
- // If there was a position given, write it.
- //
- if (positions && _POS)
- for (u32bit i=0; i<count; i++)
- _POS->putBits(positions[i], 32);
-
- // If the new mer is the same as the last one just increase the
- // count.
- //
- if (mer == _thisMer) {
- _thisMerCount += count;
- return;
- }
-
- // Write thisMer to disk. If the count is zero, we don't write
- // anything. The count is zero for the first mer (all A) unless we
- // add that mer, and if the silly user gives us a mer with zero
- // count.
- //
- writeMer();
-
- // If the new mer is in a different bucket from the last mer, write
- // out some bucket counts. We need a while loop (opposed to just
- // writing one bucket) because we aren't guaranteed that the mers
- // are in adjacent buckets.
- //
- val = mer.startOfMer(_prefixSize);
-
- while (_thisBucket < val) {
- setIDXnumber(_thisBucketSize);
- _thisBucketSize = 0;
- _thisBucket++;
- }
-
- // Remember the new mer for the next time
- //
- _thisMer = mer;
- _thisMerCount = count;
-}
-
-
-
-void
-merylStreamWriter::addMer(u64bit prefix, u32bit prefixBits,
- u64bit mer, u32bit merBits,
- u32bit count,
- u32bit *positions) {
-
- if (_thisMerIsBits == false) {
- _thisMerIsBits = true;
- assert(_thisMerIskMer == false);
- }
-
- assert(prefixBits == _prefixSize);
- assert(prefixBits == _thisMerPreSize);
- assert(merBits == _thisMerMerSize);
- assert(prefixBits + merBits == _merSizeInBits);
-
- if ((prefix < _thisMerPre) ||
- (prefix <= _thisMerPre) && (mer < _thisMerMer)) {
- assert(0);
- }
-
- if ((prefix == _thisMerPre) &&
- (mer == _thisMerMer)) {
- _thisMerCount += count;
- return;
- }
-
- writeMer();
-
- while (_thisBucket < prefix) {
- setIDXnumber(_thisBucketSize);
- _thisBucketSize = 0;
- _thisBucket++;
- }
-
- _thisMerPre = prefix;
- _thisMerMer = mer;
- _thisMerCount = count;
-}
diff --git a/kmer/libmeryl/libmeryl.H b/kmer/libmeryl/libmeryl.H
deleted file mode 100644
index 51f8464..0000000
--- a/kmer/libmeryl/libmeryl.H
+++ /dev/null
@@ -1,185 +0,0 @@
-#ifndef LIBMERYL_H
-#define LIBMERYL_H
-
-#include "bio++.H"
-
-// A merStream reader/writer for meryl mercount data.
-//
-// merSize is used to check that the meryl file is the correct size.
-// If it isn't the code fails.
-//
-// The reader returns mers in lexicographic order. No random access.
-// The writer assumes that mers come in sorted increasingly.
-//
-// numUnique the total number of mers with count of one
-// numDistinct the total number of distinct mers in this file
-// numTotal the total number of mers in this file
-
-
-class merylStreamReader {
-public:
- merylStreamReader(const char *fn, u32bit ms=0);
- ~merylStreamReader();
-
- kMer &theFMer(void) { return(_thisMer); };
- u64bit theCount(void) { return(_thisMerCount); };
-
- bool hasPositions(void) { return(_POS != 0L); };
- u32bit *thePositions(void) { return(_thisMerPositions); };
- u32bit getPosition(u32bit i) { return(((_POS) && (i < _thisMerCount)) ? _thisMerPositions[i] : ~u32bitZERO); };
-
- u32bit merSize(void) { return(_merSizeInBits >> 1); };
- u32bit merCompression(void) { return(_merCompression); };
-
- u32bit prefixSize(void) { return(_prefixSize); };
-
- u64bit numberOfUniqueMers(void) { return(_numUnique); };
- u64bit numberOfDistinctMers(void) { return(_numDistinct); };
- u64bit numberOfTotalMers(void) { return(_numTotal); };
-
- u64bit histogram(u32bit i) { return((i < _histogramLen) ? _histogram[i] : ~u64bitZERO); };
- u64bit histogramLength(void) { return(_histogramLen); };
- u64bit histogramHuge(void) { return(_histogramHuge); };
- u64bit histogramMaximumCount(void) { return(_histogramMaxValue); };
-
- bool nextMer(void);
- bool validMer(void) { return(_validMer); };
-private:
- bitPackedFile *_IDX;
- bitPackedFile *_DAT;
- bitPackedFile *_POS;
-
- u64bit getIDXnumber(void) {
- u64bit n = 1;
-
- if (_idxIsPacked)
- n = _IDX->getNumber();
- else
- n = _IDX->getBits(32);
-
- return(n);
- };
- u64bit getDATnumber(void) {
- u64bit n = 1;
-
- if (_datIsPacked) {
- if (_DAT->getBits(1))
- n = _DAT->getNumber() + 2;
- } else {
- n = _DAT->getBits(32);
- }
-
- return(n);
- };
-
- // Why not bool? Seems like the bitPackedFile is incompatible
- // with bools.
- u32bit _idxIsPacked;
- u32bit _datIsPacked;
- u32bit _posIsPacked;
-
- u32bit _merSizeInBits;
- u32bit _merCompression;
- u32bit _prefixSize;
- u32bit _merDataSize;
- u64bit _thisBucket;
- u64bit _thisBucketSize;
- u64bit _numBuckets;
-
- kMer _thisMer;
- u64bit _thisMerCount;
-
- u32bit _thisMerPositionsMax;
- u32bit *_thisMerPositions;
-
- u64bit _numUnique;
- u64bit _numDistinct;
- u64bit _numTotal;
-
- u64bit _histogramHuge; // number that are bigger than Len
- u64bit _histogramLen; // number of entries in the histo
- u64bit _histogramMaxValue; // highest count ever seen
- u64bit *_histogram;
-
- bool _validMer;
-};
-
-
-class merylStreamWriter {
-public:
- merylStreamWriter(const char *filePrefix,
- u32bit merSize, // In bases
- u32bit merComp, // A length, bases
- u32bit prefixSize, // In bits
- bool positionsEnabled);
- ~merylStreamWriter();
-
- void addMer(kMer &mer, u32bit count=1, u32bit *positions=0L);
- void addMer(u64bit prefix, u32bit prefixBits,
- u64bit mer, u32bit merBits,
- u32bit count=1,
- u32bit *positions=0L);
-
-private:
- void writeMer(void);
-
- void setIDXnumber(u64bit n) {
- if (_idxIsPacked)
- _IDX->putNumber(n);
- else
- _IDX->putBits(n, 32);
- };
- void setDATnumber(u64bit n) {
- if (_datIsPacked) {
- if (n == 1) {
- _DAT->putBits(u64bitZERO, 1);
- } else {
- _DAT->putBits(u64bitONE, 1);
- _DAT->putNumber(n-2);
- }
- } else {
- _DAT->putBits(n, 32);
- }
- };
-
-
- bitPackedFile *_IDX;
- bitPackedFile *_DAT;
- bitPackedFile *_POS;
-
- u32bit _idxIsPacked;
- u32bit _datIsPacked;
- u32bit _posIsPacked;
-
- u32bit _merSizeInBits;
- u32bit _merCompression;
- u32bit _prefixSize;
- u32bit _merDataSize;
- u64bit _thisBucket;
- u64bit _thisBucketSize;
- u64bit _numBuckets;
-
- u64bit _numUnique;
- u64bit _numDistinct;
- u64bit _numTotal;
-
- u64bit _histogramHuge; // number that are bigger than Len
- u64bit _histogramLen; // number of entries in the histo
- u64bit _histogramMaxValue; // highest count ever seen
- u64bit *_histogram;
-
- bool _thisMerIsBits;
- bool _thisMerIskMer;
-
- kMer _thisMer;
-
- u64bit _thisMerPre;
- u64bit _thisMerMer;
-
- u32bit _thisMerPreSize;
- u32bit _thisMerMerSize;
-
- u64bit _thisMerCount;
-};
-
-#endif // LIBMERYL_H
diff --git a/kmer/libseq/Make.include b/kmer/libseq/Make.include
deleted file mode 100644
index 3ffa519..0000000
--- a/kmer/libseq/Make.include
+++ /dev/null
@@ -1,33 +0,0 @@
-# -*- makefile -*-
-
-LIBUTL/ :=$(realpath $/../libutil/)/
-LIBBIO/ :=$(realpath $/../libbio/)/
-LIBSEQ/ :=$(realpath $/../libseq/)/
-
-src := $/seqFile.H \
- $/fastaFile.H $/fastaFile.C \
- $/fastqFile.H $/fastqFile.C \
- $/fastaStdin.H $/fastaStdin.C \
- $/seqStore.H $/seqStore.C \
- $/sffFile.H $/sffFile.C \
- $/seqFactory.H $/seqFactory.C \
- $/seqCache.H $/seqCache.C \
- $/seqStream.H $/seqStream.C \
- $/merStream.H $/merStream.C
-
-$/.CXX_SRCS :=$(filter %.C,${src}) $/test-seqCache.C $/test-seqStream.C $/test-merStream.C
-$/.CXX_INCS :=$(filter %.H,${src})
-$/.CXX_EXES :=$/test-seqCache $/test-seqStream $/test-merStream
-$/.CXX_LIBS :=$/libseq.a
-
-$/.CLEAN := $/*.o
-
-$/libseq.a : ${$/.C_SRCS:.c=.o} ${$/.CXX_SRCS:.C=.o}
-
-$/test-seqCache : $/test-seqCache.o ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/test-seqStream : $/test-seqStream.o ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/test-merStream : $/test-merStream.o ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-
-$(eval $/%.d $/%.o: CFLAGS += -I${LIBUTL/} -I${LIBBIO/} -I${LIBSEQ/})
-$(eval $/%.d $/%.o: CXXFLAGS += -I${LIBUTL/} -I${LIBBIO/} -I${LIBSEQ/})
-
diff --git a/kmer/libseq/fastaFile.C b/kmer/libseq/fastaFile.C
deleted file mode 100644
index baace04..0000000
--- a/kmer/libseq/fastaFile.C
+++ /dev/null
@@ -1,585 +0,0 @@
-#include "fastaFile.H"
-#include "alphabet.h"
-
-
-#undef DEBUG
-#undef DEBUGINDEX
-
-// Says 'kmerFastaFileIdx'
-#define FASTA_MAGICNUMBER1 0x7473614672656d6bULL
-#define FASTA_MAGICNUMBER2 0x786449656c694661ULL
-
-
-fastaFile::fastaFile(const char *filename) {
- clear();
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::fastaFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER");
-#endif
-
- strcpy(_filename, filename);
-
- constructIndex();
-
- _rb = new readBuffer(_filename);
-
- _numberOfSequences = _header._numberOfSequences;
-}
-
-
-
-fastaFile::fastaFile() {
- clear();
-}
-
-
-
-fastaFile::~fastaFile() {
- delete _rb;
- delete [] _index;
- delete [] _names;
-}
-
-
-
-
-
-seqFile *
-fastaFile::openFile(const char *filename) {
- struct stat st;
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::openFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER");
-#endif
-
- if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) ||
- ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0)))
- return(0L);
-
- errno = 0;
- stat(filename, &st);
- if (errno)
- return(0L);
- if ((st.st_mode & S_IFREG) == 0)
- return(0L);
-
- // Otherwise, open and see if we can get the first sequence. We
- // assume it's fasta if we find a '>' denoting a defline the first
- // thing in the file.
- //
- // Use of a readBuffer here is a bit heavyweight, but it's safe and
- // easy. Opening a fastaFile isn't, after all, lightweight anyway.
- //
- fastaFile *f = 0L;
- readBuffer *r = new readBuffer(filename);
- char x = r->read();
-
- while ((r->eof() == false) && (whitespaceSymbol[x] == true))
- x = r->read();
-
- // If we get a fasta record separator assume it's a fasta file. If
- // it's eof, the file is empty, and we might as well return this
- // fasta file and let the client deal with the lack of sequence.
- //
- if ((x == '>') || (r->eof() == true))
- f = new fastaFile(filename);
-
- delete r;
-
- return(f);
-}
-
-
-
-u32bit
-fastaFile::find(const char *sequencename) {
- char *ptr = _names;
-
- // If this proves far too slow, rewrite the _names string to
- // separate IDs with 0xff, then use strstr on the whole thing. To
- // find the ID, scan down the string counting the number of 0xff's.
- //
- // Similar code is used for seqStore::find()
-
- for (u32bit iid=0; iid < _header._numberOfSequences; iid++) {
- //fprintf(stderr, "fastaFile::find()-- '%s' vs '%s'\n", sequencename, ptr);
- if (strcmp(sequencename, ptr) == 0)
- return(iid);
-
- while (*ptr)
- ptr++;
- ptr++;
- }
-
- return(~u32bitZERO);
-}
-
-
-
-u32bit
-fastaFile::getSequenceLength(u32bit iid) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::getSequenceLength()-- "u32bitFMT"\n", iid);
-#endif
-
- return((iid < _numberOfSequences) ? _index[iid]._seqLength : 0);
-}
-
-
-
-bool
-fastaFile::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::getSequence(full)-- "u32bitFMT"\n", iid);
-#endif
-
- // Assume there is no index. Without being horribly complicated
- // (as in the previous versions of this codebase) all we'd get from
- // having an index around is the length of the sequence.
- //
- // Previous versions used to use the index to tell if the sequence
- // was squeezed (and so a direct copy to the output), if it was
- // fixed width (mostly direct copies) or unknown. Now we just
- // assume it's unknown and go byte by byte. If speed is a concern,
- // use the seqFile instead.
-
- if (iid >= _header._numberOfSequences) {
- fprintf(stderr, "fastaFile::getSequence(full)-- iid "u32bitFMT" more than number of sequences "u32bitFMT"\n",
- iid, _header._numberOfSequences);
- return(false);
- }
-
- if (sMax == 0) {
- sMax = 2048;
- s = new char [sMax];
- }
-
- if (hMax == 0) {
- hMax = 2048;
- h = new char [hMax];
- }
-
- if ((_index) && (sMax < _index[iid]._seqLength)) {
- sMax = _index[iid]._seqLength;
- delete [] s;
- s = new char [sMax];
- }
-
- hLen = 0;
- sLen = 0;
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::getSequence(full)-- seek to iid="u32bitFMT" at pos="u32bitFMT"\n",
- iid, _index[iid]._seqPosition);
-#endif
- _rb->seek(_index[iid]._seqPosition);
-
- char x = _rb->read();
-
- // Skip whitespace at the start of the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // We should be at a '>' character now. Fail if not.
- if (_rb->eof())
- return(false);
- if (x != '>')
- fprintf(stderr, "fastaFile::getSequence(full)-- ERROR1: In %s, expected '>' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Skip the '>' in the defline
- x = _rb->read();
-
- // Skip whitespace between the '>' and the defline
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true) && (x != '\r') && (x != '\n'))
- x = _rb->read();
-
- // Copy the defline, until the first newline.
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) {
- h[hLen++] = x;
- if (hLen >= hMax) {
- hMax += 2048;
- char *H = new char [hMax];
- memcpy(H, h, hLen);
- delete [] h;
- h = H;
- }
- x = _rb->read();
- }
- h[hLen] = 0;
-
- // Skip whitespace between the defline and the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // Copy the sequence, until EOF or the next '>'.
- while ((_rb->eof() == false) && (_rb->peek() != '>')) {
- if (whitespaceSymbol[x] == false) {
- s[sLen++] = x;
- if (sLen >= sMax) {
- if (sMax == 4294967295) // 4G - 1
- fprintf(stderr, "fastaFile::getSequence()-- ERROR: sequence is too long; must be less than 4 Gbp.\n"), exit(1);
- if (sMax >= 2147483648) // 2G
- sMax = 4294967295;
- else
- sMax *= 2;
- char *S = new char [sMax];
- memcpy(S, s, sLen);
- delete [] s;
- s = S;
- }
- }
- x = _rb->read();
- }
- s[sLen] = 0;
-
- _nextID++;
-
- return(true);
-}
-
-
-// slow
-bool
-fastaFile::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
-
- if (iid >= _header._numberOfSequences) {
- fprintf(stderr, "fastaFile::getSequence(part)-- iid "u32bitFMT" more than number of sequences "u32bitFMT"\n",
- iid, _header._numberOfSequences);
- return(false);
- }
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::getSequence(part)-- "u32bitFMT"\n", iid);
-#endif
-
- // It is impossible to be efficient here; see the big comment in
- // the other getSequence() above.
- //
- // We can't even guess where to start scanning the sequence; we
- // just don't have any information about how much whitespace is in
- // the sequence.
-
- _rb->seek(_index[iid]._seqPosition);
-
- u32bit pos = 0;
- char x = _rb->read();
-
- // Skip whitespace at the start of the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // We should be at a '>' character now. Fail if not.
- if (_rb->eof())
- return(false);
- if (x != '>')
- fprintf(stderr, "fastaFile::getSequence(part)-- ERROR2: In %s, expected '>' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Skip the defline.
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n'))
- x = _rb->read();
-
- // Skip whitespace between the defline and the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // Skip sequence up until bgn.
- while ((_rb->eof() == false) && (pos < bgn)) {
- if (whitespaceSymbol[x] == false)
- pos++;
-
- x = _rb->read();
- }
-
- // Copy sequence
- while ((_rb->eof() == false) && (pos < end)) {
- if (whitespaceSymbol[x] == false)
- s[pos++ - bgn] = x;
-
- x = _rb->read();
- }
- s[pos - bgn] = 0;
-
- // Fail if we didn't copy enough stuff.
- return((pos == end) ? true : false);
-}
-
-
-
-
-void
-fastaFile::clear(void) {
- memset(_filename, 0, FILENAME_MAX);
- memset(_typename, 0, FILENAME_MAX);
-
- strcpy(_typename, "FastA");
-
- _numberOfSequences = 0;
-
- _rb = 0L;
- memset(&_header, 0, sizeof(fastaFileHeader));
- _index = 0L;
- _names = 0L;
- _nextID = 0;
-}
-
-
-
-void
-fastaFile::loadIndex(char *indexname) {
- struct stat fastastat;
-
- if (fileExists(indexname) == false)
- return;
-
- errno = 0;
- if (stat(_filename, &fastastat)) {
- fprintf(stderr, "fastaFile::constructIndex()-- stat of file '%s' failed: %s\n",
- _filename, strerror(errno));
- return;
- }
-
- FILE *I = fopen(indexname, "r");
- if (errno) {
- fprintf(stderr, "fastaFile::constructIndex()-- open of file '%s' failed: %s\n",
- indexname, strerror(errno));
- return;
- }
-
- fread(&_header, sizeof(fastaFileHeader), 1, I);
-
- if ((_header._magic[0] != FASTA_MAGICNUMBER1) &&
- (_header._magic[1] != FASTA_MAGICNUMBER2)) {
- fprintf(stderr, "fastaFile::constructIndex()-- magic mismatch.\n");
- fclose(I);
- return;
- }
-
-#if 0
- (_header._fastaModificationTime != (u64bit)fastastat.st_mtime)
- (_header._fastaCreationTime != (u64bit)fastastat.st_ctime)
-#endif
- if (_header._fastaFileSize != (u64bit)fastastat.st_size) {
- fprintf(stderr, "fastaFile::constructIndex()-- stat mismatch.\n");
- fclose(I);
- return;
- }
-
- _index = new fastaFileIndex [_header._numberOfSequences];
- _names = new char [_header._namesLength];
-
- fread(_index, sizeof(fastaFileIndex), _header._numberOfSequences, I);
- fread(_names, sizeof(char), _header._namesLength, I);
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::constructIndex()-- '%s' LOADED\n", _filename);
-#endif
-
- fclose(I);
- return;
-}
-
-
-void
-fastaFile::constructIndex(void) {
-
- if (_index)
- return;
-
- // If the filename ends in '.fasta' then append a 'idx',
- // otherwise, append '.fastaidx'.
-
- char indexname[FILENAME_MAX];
-
- strcpy(indexname, _filename);
- u32bit l = strlen(_filename);
- if ((l > 5) && (strcmp(_filename + l - 6, ".fasta") == 0))
- strcat(indexname, "idx");
- else
- strcat(indexname, ".fastaidx");
-
- // If the index exists, suck it in and return.
-
- loadIndex(indexname);
-
- if (_index)
- return;
-
-#ifdef DEBUG
- fprintf(stderr, "fastaFile::constructIndex()-- '%s' BUILDING\n", _filename);
-#endif
-
- // Allocate some space for the index structures.
-
- u32bit indexMax = 64 * 1024 * 1024 / sizeof(fastaFileIndex);
- u32bit indexLen = 0;
-
- _index = new fastaFileIndex [indexMax];
-
- u32bit namesMax = 32 * 1024 * 1024;
- u32bit namesLen = 0;
-
- _names = new char [namesMax];
-
- // Some local storage
-
- u64bit seqStart;
- u32bit seqLen;
- u32bit seqLenMax = ~u32bitZERO;
- u32bit namePos;
-
- readBuffer ib(_filename);
- char x = ib.read();
-
-#ifdef DEBUGINDEX
- fprintf(stderr, "readBuffer '%s' eof=%d x=%c %d\n", _filename, ib.eof(), x, x);
-#endif
-
- // Build it.
-
- // Skip whitespace at the start of the sequence.
- while ((ib.eof() == false) && (whitespaceSymbol[x] == true)) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "skip '%c' %d\n", x, x);
-#endif
- x = ib.read();
- }
-
- while (ib.eof() == false) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "index\n");
-#endif
-
- // We should be at a '>' character now. Fail if not.
- if (x != '>')
- fprintf(stderr, "fastaFile::constructIndex()-- ERROR3: In %s, expected '>' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Save info - ib's position is correctly at the first letter in
- // the defline (which might be whitespace), but the reader
- // expects our position to be at the '>' -- hence the -1.
- seqStart = ib.tell() - 1;
- seqLen = 0;
- namePos = namesLen;
-
- // Read that first letter
- x = ib.read();
-
- // Copy the name to the names
- while ((ib.eof() == false) && (whitespaceSymbol[x] == false)) {
- if (namesLen + 1 >= namesMax) {
- namesMax += 32 * 1024 * 1024;
- char *nt = new char [namesMax];
- memcpy(nt, _names, namesLen);
- delete [] _names;
- _names = nt;
- }
-
- _names[namesLen++] = x;
-#ifdef DEBUGINDEX
- fprintf(stderr, "name += %c\n", x);
-#endif
- x = ib.read();
- }
-
- if (namesLen + 1 >= namesMax) {
- namesMax += 32 * 1024 * 1024;
- char *nt = new char [namesMax];
- memcpy(nt, _names, namesLen);
- delete [] _names;
- _names = nt;
- }
- _names[namesLen++] = 0;
-
- // Skip the rest of the defline
- while ((ib.eof() == false) && (x != '\r') && (x != '\n')) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "skip let %c\n", x);
-#endif
- x = ib.read();
- }
-
- // Skip whitespace between the defline and the sequence.
- while ((ib.eof() == false) && (whitespaceSymbol[x] == true)) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "skip num %d\n", x);
-#endif
- x = ib.read();
- }
-
-#ifdef DEBUGINDEX
- fprintf(stderr, "x=%c peek=%c\n", x, ib.peek());
-#endif
-
- // Count sequence length
- while ((ib.eof() == false) && (ib.peek() != '>')) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "seqlen %s %c\n", (whitespaceSymbol[x] == false) ? "save" : "skip", x);
-#endif
- if (whitespaceSymbol[x] == false)
- seqLen++;
- if (seqLen >= seqLenMax)
- fprintf(stderr, "fastaFile::constructIndex()-- ERROR: In %s, sequence '%s' is too long. Maximum length is %u bases.\n",
- _filename, _names + namePos, seqLenMax), exit(1);
- x = ib.read();
- }
-
- // Save to the index.
-
- if (indexLen >= indexMax) {
- indexMax *= 2;
- fastaFileIndex *et = new fastaFileIndex[indexMax];
- memcpy(et, _index, sizeof(fastaFileIndex) * indexLen);
- delete [] _index;
- _index = et;
- }
-
- _index[indexLen]._seqPosition = seqStart;
- _index[indexLen]._seqLength = seqLen;
-
-#ifdef DEBUG
- fprintf(stderr, "INDEX iid="u32bitFMT" len="u32bitFMT" pos="u64bitFMT"\n",
- indexLen, seqLen, seqStart);
-#endif
-
- indexLen++;
-
- // Load the '>' for the next iteration.
- x = ib.read();
- }
-
- // Fill out the index meta data
-
- struct stat fastastat;
- errno = 0;
- if (stat(_filename, &fastastat))
- fprintf(stderr, "fastaFile::constructIndex()-- stat() of file '%s' failed: %s\n",
- _filename, strerror(errno)), exit(1);
-
- _header._magic[0] = FASTA_MAGICNUMBER1;
- _header._magic[1] = FASTA_MAGICNUMBER2;
- _header._numberOfSequences = indexLen;
- _header._namesLength = namesLen;
- _header._fastaFileSize = fastastat.st_size;
- _header._fastaModificationTime = fastastat.st_mtime;
- _header._fastaCreationTime = fastastat.st_ctime;
-
- // Dump the index, if possible.
-
- errno = 0;
- FILE *I = fopen(indexname, "w");
- if (errno)
- return;
-
- fwrite(&_header, sizeof(fastaFileHeader), 1, I);
- fwrite( _index, sizeof(fastaFileIndex), _header._numberOfSequences, I);
- fwrite( _names, sizeof(char), _header._namesLength, I);
-
- fclose(I);
-}
diff --git a/kmer/libseq/fastaFile.H b/kmer/libseq/fastaFile.H
deleted file mode 100644
index 42b8f14..0000000
--- a/kmer/libseq/fastaFile.H
+++ /dev/null
@@ -1,66 +0,0 @@
-#ifndef FASTAFILE_H
-#define FASTAFILE_H
-
-#include "util++.H"
-#include "bio++.H"
-
-#include "seqFile.H"
-
-struct fastaFileHeader {
- u64bit _magic[2];
- u32bit _numberOfSequences; // Number of sequences in the file
- u32bit _namesLength; // Bytes in the names
- u64bit _fastaFileSize; // st_size - size of file in bytes
- u64bit _fastaModificationTime; // st_mtime - time of last data modification
- u64bit _fastaCreationTime; // st_ctime - time of last file status change
-};
-
-
-struct fastaFileIndex {
- u64bit _seqPosition; // Position of the sequence in the file
- u32bit _seqLength; // Length of the sequence (no whitespace counted)
-};
-
-
-class fastaFile : public seqFile {
-protected:
- fastaFile(const char *filename);
- fastaFile();
-
-public:
- ~fastaFile();
-
-protected:
- seqFile *openFile(const char *filename);
-
-public:
- u32bit find(const char *sequencename);
-
- u32bit getSequenceLength(u32bit iid);
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
-
-private:
- void clear(void);
- void loadIndex(char *indexname);
- void constructIndex(void);
-
- readBuffer *_rb;
-
- fastaFileHeader _header;
- fastaFileIndex *_index;
- char *_names;
-
- u32bit _nextID; // Next sequence in the read buffer
-
- u32bit _gs_iid;
- u32bit _gs_pos;
-
- friend class seqFactory;
-};
-
-
-#endif // FASTAFILE_H
diff --git a/kmer/libseq/fastaStdin.C b/kmer/libseq/fastaStdin.C
deleted file mode 100644
index 0135878..0000000
--- a/kmer/libseq/fastaStdin.C
+++ /dev/null
@@ -1,179 +0,0 @@
-#include "fastaStdin.H"
-#include "alphabet.h"
-
-fastaStdin::fastaStdin(const char *filename) {
- clear();
-
-#ifdef DEBUG
- fprintf(stderr, "fastaStdin::fastaStdin()-- '%s'\n", (filename) ? filename : "NULLPOINTER");
-#endif
-
- strcpy(_filename, "(stdin)");
-
- _rb = new readBuffer("-");
-}
-
-
-
-fastaStdin::fastaStdin() {
- clear();
-}
-
-
-
-fastaStdin::~fastaStdin() {
- delete _rb;
-}
-
-
-
-seqFile *
-fastaStdin::openFile(const char *filename) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastaStdin::openFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER");
-#endif
-
- if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) ||
- ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0)))
- return(new fastaStdin(0L));
-
- return(0L);
-}
-
-
-
-u32bit
-fastaStdin::find(const char *sequencename) {
- fprintf(stderr, "fastaStdin::find()-- WARNING! Used for random access.\n");
- return(~u32bitZERO);
-}
-
-
-
-u32bit
-fastaStdin::getSequenceLength(u32bit iid) {
- fprintf(stderr, "fastaStdin::getSequenceLength()-- WARNING! Used for random access.\n");
- return(0);
-}
-
-
-
-bool
-fastaStdin::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastaStdin::getSequence(full)-- "u32bitFMT"\n", iid);
-#endif
-
- if (iid != _thisIID)
- fprintf(stderr, "fastaStdin::getSequence(full)-- WARNING! Used for random access.\n"), exit(1);
-
- if (sMax == 0) {
- sMax = 2048;
- s = new char [sMax];
- }
-
- if (hMax == 0) {
- hMax = 2048;
- h = new char [hMax];
- }
-
- return(loadNextSequence(h, hLen, hMax, s, sLen, sMax));
-}
-
-
-
-bool
-fastaStdin::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastaStdin::getSequence(part)-- "u32bitFMT"\n", iid);
-#endif
- assert(0);
- return(false);
-}
-
-
-
-void
-fastaStdin::clear(void) {
- memset(_filename, 0, FILENAME_MAX);
- memset(_typename, 0, FILENAME_MAX);
-
- strcpy(_typename, "FastAstream");
-
- _numberOfSequences = ~u32bitZERO;
-
- _rb = 0L;
- _thisIID = 0;
-}
-
-
-
-bool
-fastaStdin::loadNextSequence(char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
- char x = _rb->read();
-
- // Skip whitespace at the start of the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // We should be at a '>' character now. Fail if not.
- if (_rb->eof() == true)
- return(false);
- if (x != '>')
- fprintf(stderr, "fastaStdin::loadNextSequence(part)-- ERROR: In %s, expected '>' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Skip the '>' in the defline
- x = _rb->read();
-
- // Skip whitespace between the '>' and the defline
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true) && (x != '\r') && (x != '\n'))
- x = _rb->read();
-
- // Copy the defline, until the first newline.
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) {
- h[hLen++] = x;
- if (hLen >= hMax) {
- //fprintf(stderr, "realloc header\n");
- hMax += 2048;
- char *H = new char [hMax];
- memcpy(H, h, hLen);
- delete [] h;
- h = H;
- }
- x = _rb->read();
- }
- h[hLen] = 0;
-
- // Skip whitespace between the defline and the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // Copy the sequence, until EOF or the next '>'.
- while ((_rb->eof() == false) && (_rb->peek() != '>')) {
- if (whitespaceSymbol[x] == false) {
- s[sLen++] = x;
- if (sLen >= sMax) {
- //fprintf(stderr, "realloc sequence\n");
- sMax *= 2;
- char *S = new char [sMax];
- memcpy(S, s, sLen);
- delete [] s;
- s = S;
- }
- }
- x = _rb->read();
- }
- s[sLen] = 0;
-
- _thisIID++;
-
- return(true);
-}
diff --git a/kmer/libseq/fastaStdin.H b/kmer/libseq/fastaStdin.H
deleted file mode 100644
index e12b56e..0000000
--- a/kmer/libseq/fastaStdin.H
+++ /dev/null
@@ -1,43 +0,0 @@
-#ifndef FASTASTDIN_H
-#define FASTASTDIN_H
-
-#include "util++.H"
-#include "bio++.H"
-
-#include "seqFile.H"
-
-
-class fastaStdin : public seqFile {
-protected:
- fastaStdin(const char *filename);
- fastaStdin();
-
-public:
- ~fastaStdin();
-
-protected:
- seqFile *openFile(const char *filename);
-
-public:
- u32bit find(const char *sequencename);
-
- u32bit getSequenceLength(u32bit iid);
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
-
-private:
- void clear(void);
- bool loadNextSequence(char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
-
- readBuffer *_rb;
- u32bit _thisIID;
-
- friend class seqFactory;
-};
-
-
-#endif // FASTASTDIN_H
diff --git a/kmer/libseq/fastqFile.C b/kmer/libseq/fastqFile.C
deleted file mode 100644
index 82094a3..0000000
--- a/kmer/libseq/fastqFile.C
+++ /dev/null
@@ -1,593 +0,0 @@
-#include "fastqFile.H"
-#include "alphabet.h"
-
-
-#undef DEBUG
-#undef DEBUGINDEX
-
-// Says 'kmerFastaFileIdx'
-#define FASTQ_MAGICNUMBER1 0x7473614672656d6bULL
-#define FASTQ_MAGICNUMBER2 0x786449656c694661ULL
-
-
-fastqFile::fastqFile(const char *filename) {
- clear();
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::fastqFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER");
-#endif
-
- strcpy(_filename, filename);
-
- constructIndex();
-
- _rb = new readBuffer(_filename);
-
- _numberOfSequences = _header._numberOfSequences;
-}
-
-
-
-fastqFile::fastqFile() {
- clear();
-}
-
-
-
-fastqFile::~fastqFile() {
- delete _rb;
- delete [] _index;
- delete [] _names;
-}
-
-
-
-
-
-seqFile *
-fastqFile::openFile(const char *filename) {
- struct stat st;
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::openFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER");
-#endif
-
- if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) ||
- ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0)))
- return(0L);
-
- errno = 0;
- stat(filename, &st);
- if (errno)
- return(0L);
- if ((st.st_mode & S_IFREG) == 0)
- return(0L);
-
- // Otherwise, open and see if we can get the first sequence. We
- // assume it's fastq if we find a '>' denoting a defline the first
- // thing in the file.
- //
- // Use of a readBuffer here is a bit heavyweight, but it's safe and
- // easy. Opening a fastqFile isn't, after all, lightweight anyway.
- //
- fastqFile *f = 0L;
- readBuffer *r = new readBuffer(filename);
- char x = r->read();
-
- while ((r->eof() == false) && (whitespaceSymbol[x] == true))
- x = r->read();
-
- // If we get a fastq record separator assume it's a fastq file. If
- // it's eof, the file is empty, and we might as well return this
- // fastq file and let the client deal with the lack of sequence.
- //
- if ((x == '@') || (r->eof() == true))
- f = new fastqFile(filename);
-
- delete r;
-
- return(f);
-}
-
-
-
-u32bit
-fastqFile::find(const char *sequencename) {
- char *ptr = _names;
-
- // If this proves far too slow, rewrite the _names string to
- // separate IDs with 0xff, then use strstr on the whole thing. To
- // find the ID, scan down the string counting the number of 0xff's.
- //
- // Similar code is used for seqStore::find()
-
- for (u32bit iid=0; iid < _header._numberOfSequences; iid++) {
- //fprintf(stderr, "fastqFile::find()-- '%s' vs '%s'\n", sequencename, ptr);
- if (strcmp(sequencename, ptr) == 0)
- return(iid);
-
- while (*ptr)
- ptr++;
- ptr++;
- }
-
- return(~u32bitZERO);
-}
-
-
-
-u32bit
-fastqFile::getSequenceLength(u32bit iid) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::getSequenceLength()-- "u32bitFMT"\n", iid);
-#endif
-
- return((iid < _numberOfSequences) ? _index[iid]._seqLength : 0);
-}
-
-
-
-bool
-fastqFile::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::getSequence(full)-- "u32bitFMT"\n", iid);
-#endif
-
- if (iid >= _header._numberOfSequences) {
- fprintf(stderr, "fastqFile::getSequence(full)-- iid "u32bitFMT" more than number of sequences "u32bitFMT"\n",
- iid, _header._numberOfSequences);
- return(false);
- }
-
- if (sMax == 0) {
- sMax = 2048;
- s = new char [sMax];
- }
-
- if (hMax == 0) {
- hMax = 2048;
- h = new char [hMax];
- }
-
- if ((_index) && (sMax < _index[iid]._seqLength)) {
- sMax = _index[iid]._seqLength;
- delete [] s;
- s = new char [sMax];
- }
-
- hLen = 0;
- sLen = 0;
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::getSequence(full)-- seek to iid="u32bitFMT" at pos="u32bitFMT"\n",
- iid, _index[iid]._seqPosition);
-#endif
- _rb->seek(_index[iid]._seqPosition);
-
- char x = _rb->read();
-
- // Skip whitespace at the start of the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // We should be at a '@' character now. Fail if not.
- if (_rb->eof())
- return(false);
- if (x != '@')
- fprintf(stderr, "fastqFile::getSequence(full)-- ERROR1: In %s, expected '@' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Skip the '@' in the defline
- x = _rb->read();
-
- // Skip whitespace between the '@' and the defline
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true) && (x != '\r') && (x != '\n'))
- x = _rb->read();
-
- // Copy the defline, until the first newline.
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) {
- h[hLen++] = x;
- if (hLen >= hMax) {
- hMax += 2048;
- char *H = new char [hMax];
- memcpy(H, h, hLen);
- delete [] h;
- h = H;
- }
- x = _rb->read();
- }
- h[hLen] = 0;
-
- // Skip whitespace between the defline and the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // Copy the sequence, until EOF or the start of the QV bases.
- while ((_rb->eof() == false) && (x != '+')) {
- if (whitespaceSymbol[x] == false) {
- s[sLen++] = x;
- if (sLen >= sMax) {
- if (sMax == 4294967295) // 4G - 1
- fprintf(stderr, "fastqFile::getSequence()-- ERROR: sequence is too long; must be less than 4 Gbp.\n"), exit(1);
- if (sMax >= 2147483648) // 2G
- sMax = 4294967295;
- else
- sMax *= 2;
- char *S = new char [sMax];
- memcpy(S, s, sLen);
- delete [] s;
- s = S;
- }
- }
- x = _rb->read();
- }
- s[sLen] = 0;
-
- // Skip the rest of the QV id line and then the entire QV line.
-
- //x = _rb->read();
- assert((_rb->eof() == true) || (x == '+'));
-
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n'))
- x = _rb->read();
- x = _rb->read();
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n'))
- x = _rb->read();
-
- _nextID++;
-
- return(true);
-}
-
-
-// slow
-bool
-fastqFile::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
-
- if (iid >= _header._numberOfSequences) {
- fprintf(stderr, "fastqFile::getSequence(part)-- iid "u32bitFMT" more than number of sequences "u32bitFMT"\n",
- iid, _header._numberOfSequences);
- return(false);
- }
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::getSequence(part)-- "u32bitFMT"\n", iid);
-#endif
-
- // Unlike the fasta version of this, we know that all the sequence is on one line. However, we
- // expect fastq sequences to be small, and we still do the same processing -- character by character.
-
- _rb->seek(_index[iid]._seqPosition);
-
- u32bit pos = 0;
- char x = _rb->read();
-
- // Skip whitespace at the start of the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // We should be at a '@' character now. Fail if not.
- if (_rb->eof())
- return(false);
- if (x != '@')
- fprintf(stderr, "fastqFile::getSequence(part)-- ERROR2: In %s, expected '@' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Skip the defline.
- while ((_rb->eof() == false) && (x != '\r') && (x != '\n'))
- x = _rb->read();
-
- // Skip whitespace between the defline and the sequence.
- while ((_rb->eof() == false) && (whitespaceSymbol[x] == true))
- x = _rb->read();
-
- // Skip sequence up until bgn.
- while ((_rb->eof() == false) && (pos < bgn)) {
- if (whitespaceSymbol[x] == false)
- pos++;
-
- x = _rb->read();
- }
-
- // Copy sequence
- while ((_rb->eof() == false) && (pos < end)) {
- if (whitespaceSymbol[x] == false)
- s[pos++ - bgn] = x;
-
- x = _rb->read();
- }
- s[pos - bgn] = 0;
-
- // Fail if we didn't copy enough stuff.
- return((pos == end) ? true : false);
-}
-
-
-
-
-void
-fastqFile::clear(void) {
- memset(_filename, 0, FILENAME_MAX);
- memset(_typename, 0, FILENAME_MAX);
-
- strcpy(_typename, "Fastq");
-
- _numberOfSequences = 0;
-
- _rb = 0L;
- memset(&_header, 0, sizeof(fastqFileHeader));
- _index = 0L;
- _names = 0L;
- _nextID = 0;
-}
-
-
-
-void
-fastqFile::loadIndex(char *indexname) {
- struct stat fastqstat;
-
- if (fileExists(indexname) == false)
- return;
-
- errno = 0;
- if (stat(_filename, &fastqstat)) {
- fprintf(stderr, "fastqFile::constructIndex()-- stat of file '%s' failed: %s\n",
- _filename, strerror(errno));
- return;
- }
-
- FILE *I = fopen(indexname, "r");
- if (errno) {
- fprintf(stderr, "fastqFile::constructIndex()-- open of file '%s' failed: %s\n",
- indexname, strerror(errno));
- return;
- }
-
- fread(&_header, sizeof(fastqFileHeader), 1, I);
-
- if ((_header._magic[0] != FASTQ_MAGICNUMBER1) &&
- (_header._magic[1] != FASTQ_MAGICNUMBER2)) {
- fprintf(stderr, "fastqFile::constructIndex()-- magic mismatch.\n");
- fclose(I);
- return;
- }
-
- if ((_header._fastqFileSize != (u64bit)fastqstat.st_size) ||
- (_header._fastqModificationTime != (u64bit)fastqstat.st_mtime) ||
- (_header._fastqCreationTime != (u64bit)fastqstat.st_ctime)) {
- fprintf(stderr, "fastqFile::constructIndex()-- stat mismatch.\n");
- fclose(I);
- return;
- }
-
- _index = new fastqFileIndex [_header._numberOfSequences];
- _names = new char [_header._namesLength];
-
- fread(_index, sizeof(fastqFileIndex), _header._numberOfSequences, I);
- fread(_names, sizeof(char), _header._namesLength, I);
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::constructIndex()-- '%s' LOADED\n", _filename);
-#endif
-
- fclose(I);
- return;
-}
-
-
-void
-fastqFile::constructIndex(void) {
-
- if (_index)
- return;
-
- // If the filename ends in '.fastq' then append a 'idx',
- // otherwise, append '.fastqidx'.
-
- char indexname[FILENAME_MAX];
-
- strcpy(indexname, _filename);
- u32bit l = strlen(_filename);
- if ((l > 5) && (strcmp(_filename + l - 6, ".fastq") == 0))
- strcat(indexname, "idx");
- else
- strcat(indexname, ".fastqidx");
-
- // If the index exists, suck it in and return.
-
- loadIndex(indexname);
-
- if (_index)
- return;
-
-#ifdef DEBUG
- fprintf(stderr, "fastqFile::constructIndex()-- '%s' BUILDING\n", _filename);
-#endif
-
- // Allocate some space for the index structures.
-
- u32bit indexMax = 64 * 1024 * 1024 / sizeof(fastqFileIndex);
- u32bit indexLen = 0;
-
- _index = new fastqFileIndex [indexMax];
-
- u32bit namesMax = 32 * 1024 * 1024;
- u32bit namesLen = 0;
-
- _names = new char [namesMax];
-
- // Some local storage
-
- u64bit seqStart;
- u32bit seqLen;
- u32bit seqLenMax = ~u32bitZERO;
- u32bit namePos;
-
- readBuffer ib(_filename);
- char x = ib.read();
-
-#ifdef DEBUGINDEX
- fprintf(stderr, "readBuffer '%s' eof=%d x=%c %d\n", _filename, ib.eof(), x, x);
-#endif
-
- // Build it.
-
- // Skip whitespace at the start of the sequence.
- while ((ib.eof() == false) && (whitespaceSymbol[x] == true)) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "skip '%c' %d\n", x, x);
-#endif
- x = ib.read();
- }
-
- while (ib.eof() == false) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "index\n");
-#endif
-
- // We should be at a '@' character now. Fail if not.
- if (x != '@')
- fprintf(stderr, "fastqFile::constructIndex()-- ERROR3: In %s, expected '@' at beginning of defline, got '%c' instead.\n",
- _filename, x), exit(1);
-
- // Save info - ib's position is correctly at the first letter in
- // the defline (which might be whitespace), but the reader
- // expects our position to be at the '@' -- hence the -1.
- seqStart = ib.tell() - 1;
- seqLen = 0;
- namePos = namesLen;
-
- // Read that first letter
- x = ib.read();
-
- // Copy the name to the names
- while ((ib.eof() == false) && (whitespaceSymbol[x] == false)) {
- if (namesLen + 1 >= namesMax) {
- namesMax += 32 * 1024 * 1024;
- char *nt = new char [namesMax];
- memcpy(nt, _names, namesLen);
- delete [] _names;
- _names = nt;
- }
-
- _names[namesLen++] = x;
-#ifdef DEBUGINDEX
- fprintf(stderr, "name += %c\n", x);
-#endif
- x = ib.read();
- }
-
- if (namesLen + 1 >= namesMax) {
- namesMax += 32 * 1024 * 1024;
- char *nt = new char [namesMax];
- memcpy(nt, _names, namesLen);
- delete [] _names;
- _names = nt;
- }
- _names[namesLen++] = 0;
-
- // Skip the rest of the defline
- while ((ib.eof() == false) && (x != '\r') && (x != '\n')) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "skip let %c\n", x);
-#endif
- x = ib.read();
- }
-
- // Skip whitespace between the defline and the sequence.
- while ((ib.eof() == false) && (whitespaceSymbol[x] == true)) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "skip num %d\n", x);
-#endif
- x = ib.read();
- }
-
-#ifdef DEBUGINDEX
- fprintf(stderr, "x=%c peek=%c\n", x, ib.peek());
-#endif
-
- // Count sequence length
- while ((ib.eof() == false) && (x != '+')) {
-#ifdef DEBUGINDEX
- fprintf(stderr, "seqlen %s %c\n", (whitespaceSymbol[x] == false) ? "save" : "skip", x);
-#endif
- if (whitespaceSymbol[x] == false)
- seqLen++;
- if (seqLen >= seqLenMax)
- fprintf(stderr, "fastqFile::constructIndex()-- ERROR: In %s, sequence '%s' is too long. Maximum length is %u bases.\n",
- _filename, _names + namePos, seqLenMax), exit(1);
- x = ib.read();
- }
-
- // Save to the index.
-
- if (indexLen >= indexMax) {
- fprintf(stderr, "REALLOC len="u32bitFMT" from "u32bitFMT" to "u32bitFMT"\n", indexLen, indexMax, indexMax * 2);
- indexMax *= 2;
- fastqFileIndex *et = new fastqFileIndex[indexMax];
- memcpy(et, _index, sizeof(fastqFileIndex) * indexLen);
- delete [] _index;
- _index = et;
- }
-
- _index[indexLen]._seqPosition = seqStart;
- _index[indexLen]._seqLength = seqLen;
-
-#if 0
- if ((indexLen * sizeof(fastqFileIndex) > 131000) &&
- (indexLen * sizeof(fastqFileIndex) < 131200))
- fprintf(stderr, "INDEX pos="u64bitFMT" iid="u32bitFMT" len="u32bitFMT" pos="u64bitFMT"\n",
- indexLen * sizeof(fastqFileIndex), indexLen, seqLen, seqStart);
-#endif
-
- indexLen++;
-
- // Skip the rest of the QV def line, then the entire QV line, then load the '@' for the next sequence.
-
- //x = ib.read();
- assert((ib.eof() == true) || (x == '+'));
-
- while ((ib.eof() == false) && (x != '\r') && (x != '\n'))
- x = ib.read();
- x = ib.read();
- while ((ib.eof() == false) && (x != '\r') && (x != '\n'))
- x = ib.read();
- while ((ib.eof() == false) && (x != '@'))
- x = ib.read();
- }
-
- // Fill out the index meta data
-
- struct stat fastqstat;
- errno = 0;
- if (stat(_filename, &fastqstat))
- fprintf(stderr, "fastqFile::constructIndex()-- stat() of file '%s' failed: %s\n",
- _filename, strerror(errno)), exit(1);
-
- _header._magic[0] = FASTQ_MAGICNUMBER1;
- _header._magic[1] = FASTQ_MAGICNUMBER2;
- _header._numberOfSequences = indexLen;
- _header._namesLength = namesLen;
- _header._fastqFileSize = fastqstat.st_size;
- _header._fastqModificationTime = fastqstat.st_mtime;
- _header._fastqCreationTime = fastqstat.st_ctime;
-
- // Dump the index, if possible.
-
- errno = 0;
- FILE *I = fopen(indexname, "w");
- if (errno)
- return;
-
- fwrite(&_header, sizeof(fastqFileHeader), 1, I);
- fwrite( _index, sizeof(fastqFileIndex), _header._numberOfSequences, I);
- fwrite( _names, sizeof(char), _header._namesLength, I);
-
- fclose(I);
-}
diff --git a/kmer/libseq/fastqFile.H b/kmer/libseq/fastqFile.H
deleted file mode 100644
index b2ba7c2..0000000
--- a/kmer/libseq/fastqFile.H
+++ /dev/null
@@ -1,66 +0,0 @@
-#ifndef FASTQFILE_H
-#define FASTQFILE_H
-
-#include "util++.H"
-#include "bio++.H"
-
-#include "seqFile.H"
-
-struct fastqFileHeader {
- u64bit _magic[2];
- u32bit _numberOfSequences; // Number of sequences in the file
- u32bit _namesLength; // Bytes in the names
- u64bit _fastqFileSize; // st_size - size of file in bytes
- u64bit _fastqModificationTime; // st_mtime - time of last data modification
- u64bit _fastqCreationTime; // st_ctime - time of last file status change
-};
-
-
-struct fastqFileIndex {
- u64bit _seqPosition; // Position of the sequence in the file
- u32bit _seqLength; // Length of the sequence (no whitespace counted)
-};
-
-
-class fastqFile : public seqFile {
-protected:
- fastqFile(const char *filename);
- fastqFile();
-
-public:
- ~fastqFile();
-
-protected:
- seqFile *openFile(const char *filename);
-
-public:
- u32bit find(const char *sequencename);
-
- u32bit getSequenceLength(u32bit iid);
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
-
-private:
- void clear(void);
- void loadIndex(char *indexname);
- void constructIndex(void);
-
- readBuffer *_rb;
-
- fastqFileHeader _header;
- fastqFileIndex *_index;
- char *_names;
-
- u32bit _nextID; // Next sequence in the read buffer
-
- u32bit _gs_iid;
- u32bit _gs_pos;
-
- friend class seqFactory;
-};
-
-
-#endif // FASTQFILE_H
diff --git a/kmer/libseq/merStream.C b/kmer/libseq/merStream.C
deleted file mode 100644
index e17a623..0000000
--- a/kmer/libseq/merStream.C
+++ /dev/null
@@ -1,84 +0,0 @@
-#include "merStream.H"
-
-
-merStream::merStream(kMerBuilder *kb, seqStream *ss, bool kbown, bool ssown) {
- _kb = kb;
- _ss = ss;
-
- _kbdelete = kbown;
- _ssdelete = ssown;
-
- _beg = u64bitZERO;
- _end = ~u64bitZERO;
-
- _kb->clear();
-
- _invalid = true;
-}
-
-
-merStream::~merStream() {
- if (_kbdelete) delete _kb;
- if (_ssdelete) delete _ss;
-}
-
-
-void
-merStream::rewind(void) {
- _ss->rewind();
- _kb->clear();
- _invalid = true;
-}
-
-
-void
-merStream::rebuild(void) {
- _ss->setPosition(_ss->strPos() - _kb->theFMer().getMerSpan());
- _kb->clear();
- _invalid = true;
-}
-
-
-void
-merStream::setBaseRange(u64bit beg, u64bit end) {
-
- assert(beg < end);
-
- //fprintf(stderr, "merStream::setBaseRange()-- from "u64bitFMT" to "u64bitFMT".\n", beg, end);
-
- // We can't tell the seqStore when to stop; while we could compute the span of a spaced seed, we
- // cannot compute it for a compressed seed. We need to stop iterating when the beginning of the
- // mer reaches the requested end.
-
- _ss->setRange(beg, ~u64bitZERO);
-
- _beg = beg;
- _end = end;
-
- _kb->clear();
-
- _invalid = true;
-}
-
-
-u64bit
-merStream::approximateNumberOfMers(void) {
- u64bit approx = _end - _beg;
- u64bit k = _kb->merSize();
-
- // If we don't know the range, sum all the sequence lengths, otherwise, it's just the length from
- // begin to end.
-
- if (_end == ~u64bitZERO) {
- approx = u64bitZERO;
-
- for (u32bit s=0; s<_ss->numberOfSequences(); s++) {
- u32bit l = _ss->lengthOf(s);
-
- if (l > k)
- approx += l - k + 1;
- }
- }
-
- return(approx);
-}
diff --git a/kmer/libseq/merStream.H b/kmer/libseq/merStream.H
deleted file mode 100644
index cc0b17e..0000000
--- a/kmer/libseq/merStream.H
+++ /dev/null
@@ -1,99 +0,0 @@
-#ifndef MERSTREAM_H
-#define MERSTREAM_H
-
-#include "util++.H"
-#include "bio++.H"
-
-#include "seqFile.H"
-#include "seqStream.H"
-
-//
-// merStream needs exclusive use of a kMerBuilder and a seqStream.
-//
-// The kMerBuilder can be used over and over. I think snapper is the
-// only one that does this though.
-//
-// The seqStream can be used elsewhere, but ONLY for looking up
-// positions.
-//
-// The merStream does NOT assume ownership of either of these, unless
-// the own flags are set.
-//
-// The stream is not valid until nextMer is called; allowing loops of
-// while (MS->nextMer()) {
-// process(MS->theFMer());
-// }
-//
-// setRange() positions refer to ACGT letters in the input, NOT mers.
-// rewind() repositions the file to the start of the range.
-//
-
-class merStream {
-public:
- merStream(kMerBuilder *kb, seqStream *ss, bool kbown=false, bool ssown=false);
- ~merStream();
-
- kMer const & theFMer(void) { assert(_invalid == false); return(_kb->theFMer()); };
- kMer const & theRMer(void) { assert(_invalid == false); return(_kb->theRMer()); };
- kMer const & theCMer(void) { assert(_invalid == false); return(_kb->theCMer()); };
-
- bool nextMer(u32bit skip=0) {
- char ch;
-
- do {
- ch = _ss->get();
- if (ch == 0)
- return(false);
- } while ((_kb->addBase(ch) == true) || (skip-- > 0));
-
- _kb->mask();
- _invalid = false;
-
-#if 0
- char merstring[256];
-
- fprintf(stderr, "merStream::nextMer()-- seqPos="u64bitFMT" merPos="u64bitFMT" span="u32bitFMT" base0span="u32bitFMT" end="u64bitFMT" %s %s\n",
- _ss->strPos(),
- _ss->strPos() - theFMer().getMerSpan(),
- theFMer().getMerSpan(),
- _kb->baseSpan(0),
- _end,
- _kb->theFMer().merToString(merstring),
- (_ss->strPos() - theFMer().getMerSpan() < _end) ? "" : "STOP");
-#endif
-
- // The mer is out of range if:
- // o it begins at or past the _end
- // o the span of the first base ends at or past the _end
- //
- // If the mer isn't spaced, the base span is always 1. If it is spaced, the span will be
- // between 1 and ... who knows.
-
- return(_ss->strPos() - theFMer().getMerSpan() + _kb->baseSpan(0) - 1 < _end);
- };
-
- void rewind(void);
- void rebuild(void);
- void setBaseRange(u64bit beg, u64bit end);
-
- u64bit thePositionInSequence(void) { assert(_invalid == false); return(_ss->seqPos() - theFMer().getMerSpan()); };
- u64bit thePositionInStream(void) { assert(_invalid == false); return(_ss->strPos() - theFMer().getMerSpan()); };
- u64bit theSequenceNumber(void) { assert(_invalid == false); return(_ss->seqIID()); };
-
- u64bit approximateNumberOfMers(void);
-
-private:
- kMerBuilder *_kb;
- seqStream *_ss;
-
- bool _kbdelete;
- bool _ssdelete;
-
- bool _invalid;
-
- u64bit _beg;
- u64bit _end;
-};
-
-
-#endif // MERSTREAM_H
diff --git a/kmer/libseq/selftest.C b/kmer/libseq/selftest.C
deleted file mode 100644
index 299f0b6..0000000
--- a/kmer/libseq/selftest.C
+++ /dev/null
@@ -1,53 +0,0 @@
-
-
- {
- seqFile *SF = openSeqFile(argv[1]);
-
- fprintf(stdout, "source '%s' of type '%s' has "u32bitFMT" sequences.\n",
- SF->getSourceName(), SF->getFileTypeName(), SF->getNumberOfSequences());
-
- fprintf(stdout, "getSequenceLength() vs getSequence(full)\n");
- {
- char *h = 0L;
- char *s = 0L;
- u32bit hLen=0, hMax=0;
- u32bit sLen=0, sMax=0;
-
- for (u32bit sid=0; sid<SF->getNumberOfSequences(); sid++) {
- SF->getSequence(sid, h, hLen, hMax, s, sLen, sMax);
-
- if ((strlen(s) != SF->getSequenceLength(sid)) ||
- (strlen(s) != sLen) ||
- (SF->getSequenceLength(sid) != sLen)) {
- fprintf(stdout, "length differ for sid="u32bitFMT" h='%s' strlen(s)=%d sLen="u32bitFMT" getSequenceLength()="u32bitFMT"\n",
- sid, h, strlen(s), sLen, SF->getSequenceLength(sid));
- }
- }
-
- delete [] h;
- delete [] s;
- }
-
-
- fprintf(stdout, "getSequenceLength() vs getSequence(part)\n");
- {
- char *p = new char [128 * 1024 * 1024];
-
- for (u32bit sid=0; sid<SF->getNumberOfSequences(); sid++) {
- SF->getSequence(sid, 0, SF->getSequenceLength(sid), p);
-
- if (strlen(p) != SF->getSequenceLength(sid)) {
- fprintf(stdout, "length differ for sid="u32bitFMT" strlen(s)=%d getSequenceLength()="u32bitFMT"\n",
- sid, strlen(p), SF->getSequenceLength(sid));
- }
- }
-
- delete [] p;
- }
-
-
-
-
- return(0);
-}
-
diff --git a/kmer/libseq/seqCache.C b/kmer/libseq/seqCache.C
deleted file mode 100644
index a1ce8a0..0000000
--- a/kmer/libseq/seqCache.C
+++ /dev/null
@@ -1,197 +0,0 @@
-#include "seqCache.H"
-#include "seqFactory.H"
-#include "alphabet.h"
-
-#undef DEBUG
-
-
-seqCache::seqCache(const char *filename, u32bit cachesize, bool verbose) {
-
- _fb = openSeqFile(filename);
- _idToGetNext = 0;
-
- _allSequencesLoaded = false;
- _reportLoading = verbose;
-
- _cacheMap = 0L;
- _cacheSize = 0;
- _cacheNext = 0;
- _cache = 0L;
-
- setCacheSize(cachesize);
-}
-
-
-
-seqCache::~seqCache() {
- flushCache();
- delete _fb;
- delete [] _cacheMap;
- delete [] _cache;
-}
-
-
-
-u32bit
-seqCache::getSequenceIID(char *name) {
- u32bit iid = ~u32bitZERO;
-
- // If the name is all integers, AND below the number of sequences
- // we have, return that, otherwise, look it up.
- //
- bool isInt = true;
- char *x = name;
-
- while (*x) {
- if ((*x < '0') || ('9' < *x))
- isInt = false;
- x++;
- }
-
- if (isInt)
- iid = strtou32bit(name, 0L);
-
- if (iid >= _fb->getNumberOfSequences())
- iid = _fb->find(name);
-
-#ifdef DEBUG
- fprintf(stderr, "seqCache::getSequenceIID()-- '%s' -> "u32bitFMT"\n", name, iid);
-#endif
-
- return(iid);
-}
-
-
-
-seqInCore *
-seqCache::getSequenceInCore(u32bit iid) {
- u32bit cacheID = ~u32bitZERO;
- seqInCore *retSeq = 0L;
-
- if (iid >= _fb->getNumberOfSequences())
- return(0L);
-
-
- if (_allSequencesLoaded == true) {
- cacheID = iid;
-
- } else if ((_cacheSize > 0) && (_cacheMap[iid] != ~u32bitZERO)) {
- cacheID = _cacheMap[iid];
-
- } else {
- u32bit hLen=0, hMax=0, sLen=0, sMax=0;
- char *h=0L, *s=0L;
-
- if (_fb->getSequence(iid, h, hLen, hMax, s, sLen, sMax) == false)
- return(0L);
-
- retSeq = new seqInCore(iid, h, hLen, s, sLen, true);
-
- // Remove any old cached sequence, then store the one we just made
-
- if (_cache) {
- if (_cache[_cacheNext]) {
- _cacheMap[_cache[_cacheNext]->getIID()] = ~u32bitZERO;
- delete _cache[_cacheNext];
- }
-
- _cache[_cacheNext] = retSeq;
- _cacheMap[iid] = _cacheNext;
-
- cacheID = _cacheNext;
- retSeq = 0L;
-
- _cacheNext = (_cacheNext + 1) % _cacheSize;
- }
- }
-
- // If no retSeq set, make a copy of the one we have in the cache.
-
- if ((retSeq == 0L) && (cacheID != ~u32bitZERO))
- retSeq = new seqInCore(iid,
- _cache[cacheID]->header(), _cache[cacheID]->headerLength(),
- _cache[cacheID]->sequence(), _cache[cacheID]->sequenceLength(),
- false);
-
- return(retSeq);
-}
-
-
-
-void
-seqCache::setCacheSize(u32bit cachesize) {
- u32bit ns = _fb->getNumberOfSequences();
-
- flushCache();
-
- if (cachesize == 0) {
- _cacheMap = 0L;
- _cacheSize = 0;
- _cacheNext = 0;
- _cache = 0L;
- return;
- }
-
- _cacheMap = new u32bit [ns];
- _cacheSize = cachesize;
- _cacheNext = 0;
- _cache = new seqInCore * [_cacheSize];
-
- for (u32bit i=0; i<ns; i++)
- _cacheMap[i] = ~u32bitZERO;
-
- for (u32bit i=0; i<_cacheSize; i++)
- _cache[i] = 0L;
-}
-
-
-
-void
-seqCache::loadAllSequences(void) {
-
- if (_allSequencesLoaded)
- return;
-
- flushCache();
-
- delete [] _cacheMap;
- delete [] _cache;
-
- _cacheMap = 0L;
- _cacheSize = _fb->getNumberOfSequences();
- _cacheNext = 0;
- _cache = new seqInCore * [_cacheSize];
-
-
- for (u32bit iid=0; iid<_cacheSize; iid++) {
- u32bit hLen=0, hMax=0, sLen=0, sMax=0;
- char *h=0L, *s=0L;
-
- if (_fb->getSequence(iid, h, hLen, hMax, s, sLen, sMax) == false)
- fprintf(stderr, "seqCache::loadAllSequences()-- Failed to load iid "u32bitFMT".\n",
- iid), exit(1);
-
- _cache[iid] = new seqInCore(iid, h, hLen, s, sLen, true);
- }
-
- _allSequencesLoaded = true;
-}
-
-void
-seqCache::flushCache(void) {
-
- if (_fb == 0L)
- return;
-
- if (_cacheMap) {
- u32bit ns = _fb->getNumberOfSequences();
- for (u32bit i=0; i<ns; i++)
- _cacheMap[i] = ~u32bitZERO;
- }
-
- if (_cache)
- for (u32bit i=0; i<_cacheSize; i++) {
- delete _cache[i];
- _cache[i] = 0L;
- }
-}
diff --git a/kmer/libseq/seqCache.H b/kmer/libseq/seqCache.H
deleted file mode 100644
index 9f243e9..0000000
--- a/kmer/libseq/seqCache.H
+++ /dev/null
@@ -1,104 +0,0 @@
-#ifndef SEQCACHE_H
-#define SEQCACHE_H
-
-#include "util++.H"
-#include "seqFile.H"
-
-
-class seqInCore {
-private:
- seqInCore(u32bit iid, char *hdr, u32bit hdrlen, char *seq, u32bit seqlen, bool deletable) {
- _idx = iid;
-
- _deletable = deletable;
-
- _headerLen = hdrlen;
- _header = hdr;
-
- _seqLen = seqlen;
- _seq = seq;
- };
-
- friend class seqCache;
-
-public:
- ~seqInCore() {
- if (_deletable) {
- delete [] _header; _header = 0L;
- delete [] _seq; _seq = 0L;
- }
- };
-
- char *header(void) const { return(_header); };
- u32bit headerLength(void) const { return(_headerLen); };
-
- char *sequence(void) const { return(_seq); };
- u32bit sequenceLength(void) const { return(_seqLen); };
-
- u32bit getIID(void) const { return(_idx); };
-
- // Used only by searchGENOME (as far as I know)
- seqInCore *copy(void) {
- char *h = new char [_headerLen + 1];
- char *s = new char [_seqLen + 1];
-
- memcpy(h, _header, _headerLen + 1);
- memcpy(s, _seq, _seqLen + 1);
-
- return(new seqInCore(_idx, h, _headerLen, s, _seqLen, true));
- };
-
-private:
- u32bit _idx;
-
- bool _deletable;
-
- u32bit _headerLen;
- char *_header;
-
- u32bit _seqLen;
- char *_seq;
-};
-
-
-
-class seqCache {
-public:
- seqCache(const char *filename, u32bit cachesize=0, bool verbose=false);
- ~seqCache();
-
- // Returns IID for a name, either the first word on the defline, or
- // the ascii IID.
- u32bit getSequenceIID(char *name);
-
- seqInCore *getSequenceInCore(u32bit iid);
- seqInCore *getSequenceInCore(char *name) { return(getSequenceInCore(getSequenceIID(name))); };
- seqInCore *getSequenceInCore(void) { return(getSequenceInCore(_idToGetNext++)); };
-
- const char *getSourceName(void) { return(_fb->getSourceName()); };
- const char *getFileTypeName(void) { return(_fb->getFileTypeName()); };
-
- u32bit getNumberOfSequences(void) { return(_fb->getNumberOfSequences()); };
-
- u32bit getSequenceLength(u32bit iid) { return(_fb->getSequenceLength(iid)); };
-
- void setCacheSize(u32bit cachesize);
-
- void loadAllSequences(void);
- void flushCache(void);
-
-private:
- seqFile *_fb;
- u32bit _idToGetNext;
-
- bool _allSequencesLoaded;
- bool _reportLoading;
-
- u32bit *_cacheMap; // Maps ID to cache entry
- u32bit _cacheSize; // Size of cache
- u32bit _cacheNext; // Next cache spot to use
- seqInCore **_cache; // Cache of sequences
-};
-
-
-#endif // SEQCACHE_H
diff --git a/kmer/libseq/seqFactory.C b/kmer/libseq/seqFactory.C
deleted file mode 100644
index ed24d6c..0000000
--- a/kmer/libseq/seqFactory.C
+++ /dev/null
@@ -1,58 +0,0 @@
-#include "seqFactory.H"
-
-#include "fastaStdin.H"
-#include "fastaFile.H"
-#include "fastqFile.H"
-#include "seqStore.H"
-
-seqFactory *seqFactory::me = 0L;
-
-
-seqFactory::seqFactory() {
- _filesNum = 0;
- _filesMax = 16;
- _files = new seqFile * [_filesMax];
-
- registerFile(new fastaStdin);
- registerFile(new fastaFile);
- registerFile(new fastqFile);
- registerFile(new seqStore);
- //registerFile(new sffFile);
-}
-
-
-seqFactory::~seqFactory() {
- for (u32bit i=0; i<_filesNum; i++)
- delete _files[i];
- delete [] _files;
-}
-
-
-void
-seqFactory::registerFile(seqFile *f) {
- if (_filesNum >= _filesMax) {
- fprintf(stderr, "seqFactory::registerFile()-- Wow! You registered lots of files! Now fix %s at line %d.\n", __FILE__, __LINE__);
- exit(1);
- }
- _files[_filesNum++] = f;
-}
-
-
-seqFile *
-seqFactory::openFile(const char *name) {
- seqFile *n = 0L;
-
- for (u32bit i=0; i<_filesNum; i++) {
- n = _files[i]->openFile(name);
- if (n)
- return(n);
- }
-
- fprintf(stderr, "seqFactory::registerFile()-- Cannot determine type of file '%s'. Tried:\n", name);
-
- for (u32bit i=0; i<_filesNum; i++)
- fprintf(stderr, "seqFactory::registerFile()-- '%s'\n", _files[i]->getFileTypeName());
-
- exit(1);
- return(n);
-}
diff --git a/kmer/libseq/seqFactory.H b/kmer/libseq/seqFactory.H
deleted file mode 100644
index 29d3df0..0000000
--- a/kmer/libseq/seqFactory.H
+++ /dev/null
@@ -1,33 +0,0 @@
-#ifndef SEQFACTORY_H
-#define SEQFACTORY_H
-
-#include "util.h"
-#include "seqFile.H"
-
-class seqFactory {
-protected:
- seqFactory();
- ~seqFactory();
-
-public:
- static seqFactory *instance(void) {
- if (me == 0L)
- me = new seqFactory;
- return(me);
- };
-
- void registerFile(seqFile *f);
- seqFile *openFile(const char *name);
-private:
- static seqFactory *me;
-
- u32bit _filesNum;
- u32bit _filesMax;
- seqFile **_files;
-};
-
-
-#define openSeqFile(S) seqFactory::instance()->openFile((S))
-
-
-#endif // SEQFACTORY_H
diff --git a/kmer/libseq/seqFile.H b/kmer/libseq/seqFile.H
deleted file mode 100644
index ca1100d..0000000
--- a/kmer/libseq/seqFile.H
+++ /dev/null
@@ -1,50 +0,0 @@
-#ifndef SEQFILE_H
-#define SEQFILE_H
-
-#include "util.h"
-
-// General flow of the constructors is:
-// Clear all data
-// Open the file
-// Set _filename, _typename
-// Read/build the index structure
-// Position the file to the first read
-// Set _numberOfSequences (IMPORTANT, and subtle)
-
-class seqFile {
-protected:
- seqFile(const char *filename) {};
- seqFile() {};
-
-public:
- virtual ~seqFile() {};
-
-protected:
- virtual seqFile *openFile(const char *filename) = 0;
-
-public:
- virtual u32bit find(const char *sequencename) = 0;
-
- virtual const char *getSourceName(void) { return(_filename); };
- virtual const char *getFileTypeName(void) { return(_typename); };
-
- // Needs to go. It is unreliable for stdin fasta files.
- virtual u32bit getNumberOfSequences(void) { return(_numberOfSequences); };
-
- virtual u32bit getSequenceLength(u32bit id) = 0;
- virtual bool getSequence(u32bit id,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) = 0;
- virtual bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) = 0;
-
-protected:
- char _filename[FILENAME_MAX];
- char _typename[FILENAME_MAX];
-
- u32bit _numberOfSequences;
-
- friend class seqFactory;
-};
-
-#endif // SEQFILE_H
diff --git a/kmer/libseq/seqStore.C b/kmer/libseq/seqStore.C
deleted file mode 100644
index 4906519..0000000
--- a/kmer/libseq/seqStore.C
+++ /dev/null
@@ -1,622 +0,0 @@
-
-#include "seqStore.H"
-#include "seqCache.H"
-#include "alphabet.h"
-
-// Says 'kmerSeqStoreFile'
-#define SEQSTORE_MAGICNUMBER1 0x5371655372656d6bULL
-#define SEQSTORE_MAGICNUMBER2 0x656c694665726f74ULL
-
-
-seqStore::seqStore(const char *filename) {
- clear();
-
- strcpy(_filename, filename);
-
- errno = 0;
- FILE *F = fopen(_filename, "r");
- if (errno)
- fprintf(stderr, "seqStore::seqStore()-- Failed to open '%s': %s\n",
- _filename, strerror(errno)), exit(1);
- fread(&_header, sizeof(seqStoreHeader), 1, F);
- fclose(F);
-
- //_indexBPF = new bitPackedFile(_filename, _header._indexStart);
- //_blockBPF = new bitPackedFile(_filename, _header._blockStart);
- //_namesBPF = new bitPackedFile(_filename, _header._namesStart);
-
- _bpf = new bitPackedFile(_filename, sizeof(seqStoreHeader));
-
- _numberOfSequences = _header._numberOfSequences;
-}
-
-
-
-seqStore::seqStore() {
- clear();
-}
-
-
-
-seqStore::~seqStore() {
- //if ((_filename) && (_filename[0] != 0))
- // fprintf(stderr, "Closing seqStore '%s'\n", _filename);
- delete _bpf;
- delete [] _index;
- delete [] _block;
- delete [] _names;
- delete _indexBPF;
- delete _blockBPF;
- delete _namesBPF;
-}
-
-
-
-seqFile *
-seqStore::openFile(const char *filename) {
- u64bit magic1, magic2;
- struct stat st;
-
- errno = 0;
- stat(filename, &st);
- if (errno)
- return(0L);
- if ((st.st_mode & S_IFREG) == 0)
- return(0L);
-
- // Check the magic. Fail if not correct.
-
- errno = 0;
- FILE *F = fopen(filename, "r");
- if (errno)
- return(0L);
- fread(&magic1, sizeof(u64bit), 1, F);
- fread(&magic2, sizeof(u64bit), 1, F);
- fclose(F);
- if ((magic1 != SEQSTORE_MAGICNUMBER1) || (magic2 != SEQSTORE_MAGICNUMBER2))
- return(0L);
-
- return(new seqStore(filename));
-}
-
-
-
-// If this proves far too slow, rewrite the _names string to separate IDs with 0xff, then use
-// strstr on the whole thing. To find the ID, scan down the string counting the number of 0xff's.
-//
-// Similar code is used for fastaFile::find()
-//
-u32bit
-seqStore::find(const char *sequencename) {
-
- if (_names == NULL)
- loadIndex();
-
- char *ptr = _names;
-
- for (u32bit iid=0; iid < _header._numberOfSequences; iid++) {
- if (strcmp(sequencename, ptr) == 0)
- return(iid);
-
- while (*ptr)
- ptr++;
- ptr++;
- }
-
- return(~u32bitZERO);
-}
-
-
-
-u32bit
-seqStore::getSequenceLength(u32bit iid) {
- if (_index == NULL)
- loadIndex();
- return((iid < _header._numberOfSequences) ? _index[iid]._seqLength : 0);
-}
-
-
-
-bool
-seqStore::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
-
- if (_index == NULL)
- loadIndex();
-
- if (iid >= _header._numberOfSequences) {
- fprintf(stderr, "seqStore::getSequence(full)-- iid "u32bitFMT" more than number of sequences "u32bitFMT"\n",
- iid, _header._numberOfSequences);
- return(false);
- }
-
- if (sMax == 0) s = 0L; // So the delete below doesn't bomb
- if (hMax == 0) h = 0L;
-
- if (sMax < _index[iid]._seqLength + 1) {
- sMax = _index[iid]._seqLength + 1024;
- delete [] s;
- s = new char [sMax];
- }
-
- if (hMax < _index[iid]._hdrLength + 1) {
- hMax = _index[iid]._hdrLength + 1024;
- delete [] h;
- h = new char [hMax];
- }
-
- hLen = 0;
- sLen = 0;
-
- // Copy the defline into h
-
- memcpy(h, _names + _index[iid]._hdrPosition, _index[iid]._hdrLength);
-
- h[_index[iid]._hdrLength] = 0;
-
- // Decode and copy the sequence into s
-
- u32bit seqLen = _index[iid]._seqLength;
- u32bit block = _index[iid]._block;
- u64bit seekpos = _index[iid]._seqPosition * 2;
-
- _bpf->seek(seekpos);
-
- while (sLen < seqLen) {
- assert(_bpf->tell() == _block[block]._bpf * 2);
- assert(sLen == _block[block]._pos);
-
- if (_block[block]._isACGT == 0) {
- memset(s + sLen, 'N', _block[block]._len);
- sLen += _block[block]._len;
- } else {
- for (u32bit xx=0; xx<_block[block]._len; xx++) {
- s[sLen++] = bitsToLetter[_bpf->getBits(2)];
- }
- }
-
- block++;
- }
-
- s[sLen] = 0;
-
- return(true);
-}
-
-
-
-bool
-seqStore::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
-
- if (_index == NULL)
- loadIndex();
-
- if (iid >= _header._numberOfSequences) {
- fprintf(stderr, "seqStore::getSequence(part)-- iid "u32bitFMT" more than number of sequences "u32bitFMT"\n",
- iid, _header._numberOfSequences);
- return(false);
- }
-
- if (bgn >= end) {
- fprintf(stderr, "seqStore::getSequence(part)-- for iid "u32bitFMT"; invalid bgn="u32bitFMT" end="u32bitFMT"; seqLen="u32bitFMT"\n",
- iid, bgn, end, _index[iid]._seqLength);
- return(false);
- }
-
- // Decode and copy the sequence into s
-
- u32bit block = _index[iid]._block;
- u32bit sLen = 0; // length of sequence we've copied
- u32bit sPos = 0; // position in the sequence
-
- // Skip blocks before we care.
- //
- while (sPos + _block[block]._len < bgn) {
- sPos += _block[block]._len;
- block++;
- }
-
- assert(sPos == _block[block]._pos);
-
- // Move into the block (we could just set sPos = bgn...).
- sPos += bgn - _block[block]._pos;
-
- // Handle the partial block. Copy what is left in the block, or
- // the requested size, whichever is smaller.
-
- u32bit partLen = MIN((_block[block]._pos + _block[block]._len - bgn),
- (end - bgn));
-
- if (_block[block]._isACGT == 0) {
- memset(s, 'N', partLen);
- sLen += partLen;
- _bpf->seek(_block[block+1]._bpf * 2);
- } else {
- _bpf->seek((_block[block]._bpf + bgn - _block[block]._pos) * 2);
-
- for (u32bit xx=0; xx<partLen; xx++)
- s[sLen++] = bitsToLetter[_bpf->getBits(2)];
- }
-
- sPos += partLen;
-
- block++;
-
- while (sPos < end) {
- assert(_bpf->tell() == _block[block]._bpf * 2);
- assert(sPos == _block[block]._pos);
-
- // Like the partial block above, pick how much to copy as the
- // smaller of the block size and what is left to fill.
-
- partLen = MIN((_block[block]._len), (end - sPos));
-
- if (_block[block]._isACGT == 0) {
- memset(s + sLen, 'N', partLen);
- sLen += partLen;
- } else {
- for (u32bit xx=0; xx<partLen; xx++)
- s[sLen++] = bitsToLetter[_bpf->getBits(2)];
- }
-
- sPos += partLen;
-
- block++;
- }
-
- s[sLen] = 0;
-
- return(true);
-}
-
-
-
-void
-seqStore::clear(void) {
- memset(_filename, 0, FILENAME_MAX);
- memset(_typename, 0, FILENAME_MAX);
-
- strcpy(_typename, "seqStore");
-
- _numberOfSequences = 0;
-
- _bpf = 0L;
-
- memset(&_header, 0, sizeof(seqStoreHeader));
-
- _index = 0L;
- _block = 0L;
- _names = 0L;
-
- _indexBPF = 0L;
- _blockBPF = 0L;
- _namesBPF = 0L;
-
- _lastIIDloaded = ~u32bitZERO;
-}
-
-
-
-void
-seqStore::loadIndex(void) {
-
- if (_index)
- return;
-
- delete _indexBPF; _indexBPF = 0L;
- delete _blockBPF; _blockBPF = 0L;
- delete _namesBPF; _namesBPF = 0L;
-
- errno = 0;
- FILE *F = fopen(_filename, "r");
- if (errno)
- fprintf(stderr, "seqStore::seqStore()-- Failed to open '%s': %s\n",
- _filename, strerror(errno)), exit(1);
-
- fread(&_header, sizeof(seqStoreHeader), 1, F);
-
- //fprintf(stderr, "seqStore::seqStore()-- Allocating space for "u32bitFMT" sequences ("u64bitFMT"MB)\n", _header._numberOfSequences, _header._numberOfSequences * sizeof(seqStoreIndex) / 1024 / 1024);
- //fprintf(stderr, "seqStore::seqStore()-- Allocating space for "u32bitFMT" blocks ("u64bitFMT"MB)\n", _header._numberOfBlocks, _header._numberOfBlocks * sizeof(seqStoreBlock) / 1024 / 1024);
- //fprintf(stderr, "seqStore::seqStore()-- Allocating space for "u32bitFMT" labels ("u64bitFMT"MB)\n", _header._namesLength, _header._namesLength * sizeof(char) / 1024 / 1024);
-
- _index = new seqStoreIndex [_header._numberOfSequences];
- _block = new seqStoreBlock [_header._numberOfBlocks];
- _names = new char [_header._namesLength];
-
- fseeko(F, _header._indexStart, SEEK_SET);
- fread( _index, sizeof(seqStoreIndex), _header._numberOfSequences, F);
-
-#if 0
- for (u32bit i=0; i<_header._numberOfSequences; i++)
- fprintf(stderr, "IDX[%4u] hdrPos=%u hdrLen=%u seqPos=%llu seqLen=%u block=%u\n",
- i,
- _index[i]._hdrPosition,
- _index[i]._hdrLength,
- _index[i]._seqPosition,
- _index[i]._seqLength,
- _index[i]._block);
-#endif
-
- fseeko(F, _header._blockStart, SEEK_SET);
- fread( _block, sizeof(seqStoreBlock), _header._numberOfBlocks, F);
-
- fseeko(F, _header._namesStart, SEEK_SET);
-
- fread( _names, sizeof(char), _header._namesLength, F);
- if (errno)
- fprintf(stderr, "seqStore::seqStore()-- Failed to read index from '%s': %s\n",
- _filename, strerror(errno)), exit(1);
-
- fclose(F);
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-static
-void
-addSeqStoreBlock(u32bit &BLOKmax,
- u32bit &BLOKlen,
- seqStoreBlock* &BLOK,
- seqStoreBlock &b,
- u32bit &nBlockACGT,
- u32bit &nBlockGAP,
- u64bit &nACGT) {
-
- //fprintf(stderr, "addSeqStoreBlock()-- BLOK max=%u len=%u ACGT=%u GAP=%u nACGT=%lu\n",
- // BLOKmax, BLOKlen, nBlockACGT, nBlockGAP, nACGT);
-
- if (b._len == 0)
- return;
-
- if (b._isACGT == 1) {
- nBlockACGT++;
- nACGT += b._len;
- } else {
- nBlockGAP++;
- }
-
- BLOK[BLOKlen++] = b;
-
- if (BLOKlen >= BLOKmax) {
- BLOKmax *= 2;
- seqStoreBlock *nb = new seqStoreBlock [BLOKmax];
- memcpy(nb, BLOK, BLOKlen * sizeof(seqStoreBlock));
- delete [] BLOK;
- BLOK = nb;
- }
-}
-
-
-
-void
-constructSeqStore(char *filename, seqCache *inputseq) {
-
- fprintf(stderr, "constructSeqStore()-- constructing seqStore '%s' from seqCache '%s' of type '%s'.\n",
- filename, inputseq->getSourceName(), inputseq->getFileTypeName());
-
- seqStoreHeader HEAD;
- memset(&HEAD, sizeof(seqStoreHeader), 0);
-
- bitPackedFile *DATA = new bitPackedFile(filename, sizeof(seqStoreHeader), true);
-
- u32bit INDXmax = 1048576;
- seqStoreIndex *INDX = new seqStoreIndex [INDXmax];
-
- u32bit BLOKmax = 1048576;
- u32bit BLOKlen = 0;
- seqStoreBlock *BLOK = new seqStoreBlock [BLOKmax];
-
- u32bit NAMEmax = 32 * 1024 * 1024;
- u32bit NAMElen = 0;
- char *NAME = new char [NAMEmax];
-
- seqInCore *sic = inputseq->getSequenceInCore();
-
- u64bit nACGT = 0;
- u32bit nBlockACGT = 0;
- u32bit nBlockGAP = 0;
- u32bit nSequences = 0;
-
- speedCounter C(" reading sequences %7.0f sequences -- %5.0f sequences/second\r", 1.0, 0x1ffff, true);
-
- while (sic != NULL) {
- if (sic->sequence()) {
- char *seq = sic->sequence();
- seqStoreBlock b;
-
- if (nSequences >= INDXmax) {
- seqStoreIndex *I = new seqStoreIndex[INDXmax * 2];
- memcpy(I, INDX, sizeof(seqStoreIndex) * nSequences);
- delete [] INDX;
- INDXmax *= 2;
- INDX = I;
- }
-
- INDX[nSequences]._hdrPosition = NAMElen;
- INDX[nSequences]._hdrLength = sic->headerLength();
- INDX[nSequences]._seqPosition = DATA->tell() / 2;
- INDX[nSequences]._seqLength = sic->sequenceLength();
- INDX[nSequences]._block = BLOKlen;
-
-#if 0
- fprintf(stderr, "ADD SEQUENCE hdr pos=%u len=%u seq pos=%u len=%u blok=%u\n",
- INDX[nSequences]._hdrPosition,
- INDX[nSequences]._hdrLength,
- INDX[nSequences]._seqPosition,
- INDX[nSequences]._seqLength,
- INDX[nSequences]._block);
-#endif
-
- if (sic->sequenceLength() > SEQSTOREBLOCK_MAXPOS)
- fprintf(stderr, "constructSeqStore()-- sequence %s too long, must be shorter than "u64bitFMT" Gbp.\n",
- sic->header(), SEQSTOREBLOCK_MAXPOS / 1024 / 1024 / 1024), exit(1);
-
- if (sic->getIID() > SEQSTOREBLOCK_MAXPOS)
- fprintf(stderr, "constructSeqStore()-- too many sequences, must be fewer than "u64bitFMT".\n",
- SEQSTOREBLOCK_MAXIID), exit(1);
-
- if (NAMElen + sic->headerLength() + 1 > NAMEmax) {
- NAMEmax += 32 * 1024 * 1024;
- char *nm = new char [NAMEmax];
- memcpy(nm, NAME, sizeof(char) * NAMElen);
- delete [] NAME;
- NAME = nm;
- }
- strcpy(NAME + NAMElen, sic->header());
- NAMElen += sic->headerLength() + 1;
-
- b._isACGT = 0;
- b._iid = sic->getIID();
- b._pos = 0;
- b._len = 0;
- b._bpf = DATA->tell() / 2;
-
- for (u32bit p=0; p<sic->sequenceLength(); p++) {
- u64bit bits = letterToBits[seq[p]];
-
- // If the length of the current block is too big (which would
- // soon overflow the bit field storing length) write out a
- // block and reset the length.
- //
- if (b._len == SEQSTOREBLOCK_MAXLEN) {
- addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT);
-
- b._pos = p;
- b._len = 0;
- b._bpf = DATA->tell() / 2;
- }
-
-
- if (bits == 0xff) {
- // This letter is NOT ACGT. If the current block is an ACGT block, write it
- // and reset.
- //
- if (b._isACGT == 1) {
- addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT);
-
- b._isACGT = 0;
- b._iid = sic->getIID();
- b._pos = p;
- b._len = 0;
- b._bpf = DATA->tell() / 2;
- }
-
- } else {
-
- // This letter is ACGT. If the current block is NOT an ACGT block, write it
- // and reset.
- //
- if (b._isACGT == 0) {
- addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT);
-
- b._isACGT = 1;
- b._iid = sic->getIID();
- b._pos = p;
- b._len = 0;
- b._bpf = DATA->tell() / 2;
- }
- }
-
- // Always add one to the length of the current block, and
- // write out the base if the letter is ACGT.
- //
- b._len++;
-
- if (bits != 0xff)
- DATA->putBits(bits, 2);
- }
-
- // Emit the last block
- //
- addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT);
- }
-
- // If there is no sequence, the index record for this sequence is left blank.
- //
- nSequences++;
-
- C.tick();
-
- delete sic;
- sic = inputseq->getSequenceInCore();
- }
-
- // And a sentinel EOF block -- gets the last position in the file,
- // useful for the binary search. We always have a space block at
- // the end of the list, but we don't care if we just used the last
- // block (and so we don't bother to reallocate the array if it is
- // full).
-
- BLOK[BLOKlen]._isACGT = 0;
- BLOK[BLOKlen]._iid = u32bitMASK(32);
- BLOK[BLOKlen]._pos = u32bitMASK(31);
- BLOK[BLOKlen]._len = 0;
- BLOK[BLOKlen]._bpf = DATA->tell() / 2;
-
- BLOKlen++;
-
- // Update the header, assemble the final file.
-
- delete DATA;
-
- HEAD._magic[0] = SEQSTORE_MAGICNUMBER1;
- HEAD._magic[1] = SEQSTORE_MAGICNUMBER2;
- HEAD._pad = u32bitZERO;
- HEAD._numberOfSequences = nSequences;
- HEAD._numberOfACGT = nACGT;
- HEAD._numberOfBlocksACGT = nBlockACGT;
- HEAD._numberOfBlocksGAP = nBlockGAP;
- HEAD._numberOfBlocks = BLOKlen;
- HEAD._namesLength = NAMElen;
- HEAD._indexStart = u64bitZERO;
- HEAD._blockStart = u64bitZERO;
- HEAD._namesStart = u64bitZERO;
-
- errno = 0;
- FILE *F = fopen(filename, "r+");
- if (errno)
- fprintf(stderr, "constructSeqStore()-- Failed to reopen '%s' to write data: %s\n",
- filename, strerror(errno)), exit(1);
-
- fseeko(F, 0, SEEK_END);
- HEAD._indexStart = ftello(F);
- fwrite(INDX, sizeof(seqStoreIndex), HEAD._numberOfSequences, F);
-
- fseeko(F, 0, SEEK_END);
- HEAD._blockStart = ftello(F);
- fwrite(BLOK, sizeof(seqStoreBlock), HEAD._numberOfBlocks, F);
-
- fseeko(F, 0, SEEK_END);
- HEAD._namesStart = ftello(F);
- fwrite(NAME, sizeof(char), HEAD._namesLength, F);
-
- fseeko(F, 0, SEEK_SET);
- fwrite(&HEAD, sizeof(seqStoreHeader), 1, F);
-
- fclose(F);
-
- if (errno)
- fprintf(stderr, "constructSeqStore()-- Failed to write data to '%s': %s\n",
- filename, strerror(errno)), exit(1);
-
- delete [] INDX;
- delete [] BLOK;
- delete [] NAME;
-
- // ESTmapper depends on this output.
-
- fprintf(stderr, "constructSeqStore()-- seqStore '%s' constructed ("u32bitFMT" sequences, "u64bitFMT" ACGT letters, "u32bitFMT" ACGT blocks, "u32bitFMT" GAP blocks).\n",
- filename, HEAD._numberOfSequences, HEAD._numberOfACGT, HEAD._numberOfBlocksACGT, HEAD._numberOfBlocksGAP);
-}
diff --git a/kmer/libseq/seqStore.H b/kmer/libseq/seqStore.H
deleted file mode 100644
index 1ef2334..0000000
--- a/kmer/libseq/seqStore.H
+++ /dev/null
@@ -1,120 +0,0 @@
-#ifndef SEQSTORE_H
-#define SEQSTORE_H
-
-#include "util++.H"
-#include "seqCache.H"
-
-// A binary fasta file.
-//
-// HEADER
-// magic number
-// number of sequences
-// optional - alphabet size
-// optional - alphabet map (0x00 -> 'a', etc)
-// position of index start
-// position of data start
-// DATA
-// INDEX
-// position of sequence start in DATA
-// header length
-// sequence length
-// MAP
-// name to IID mapping
-
-struct seqStoreHeader {
- u64bit _magic[2];
- u32bit _pad;
- u32bit _numberOfSequences;
- u64bit _numberOfACGT;
- u32bit _numberOfBlocksACGT;
- u32bit _numberOfBlocksGAP;
- u32bit _numberOfBlocks;
- u32bit _namesLength;
-
- u64bit _indexStart;
- u64bit _blockStart;
- u64bit _namesStart;
-};
-
-
-// This index allows us to return a complete sequence
-//
-struct seqStoreIndex {
- u32bit _hdrPosition; // Offset into _names for the defline
- u32bit _hdrLength; // Length of the defline
- u64bit _seqPosition; // Offset into _bpf for the sequence data
- u32bit _seqLength; // Length, in bases, of the sequence
- u32bit _block; // The seqStoreBlock that starts this sequence
-};
-
-
-// This index allows us to seek to a specific base in the
-// file of sequences. Each block is either:
-// ACGT - and has data
-// N - no data
-// It will map a specific ACGT location to the sequence, and the ID
-// of that sequence (seq ID and location in that sequence).
-//
-struct seqStoreBlock {
- u64bit _isACGT:1; // block is acgt
- u64bit _pos:32; // position in sequence
- u64bit _iid:32; // iid of the sequence we are in
- u64bit _len:23; // length of block
- u64bit _bpf:40; // position in the bit file of sequence
-};
-
-#define SEQSTOREBLOCK_MAXPOS u64bitMASK(32)
-#define SEQSTOREBLOCK_MAXIID u64bitMASK(32)
-#define SEQSTOREBLOCK_MAXLEN u64bitMASK(23)
-
-class seqStore : public seqFile {
-protected:
- seqStore(const char *filename);
- seqStore();
-
-public:
- ~seqStore();
-
-protected:
- seqFile *openFile(const char *filename);
-
-public:
- u32bit find(const char *sequencename);
-
- u32bit getSequenceLength(u32bit iid);
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
-
-private:
- void clear(void);
- void loadIndex(void);
-
- bitPackedFile *_bpf;
-
- seqStoreHeader _header;
-
- seqStoreIndex *_index;
- seqStoreBlock *_block;
- char *_names;
-
- bitPackedFile *_indexBPF;
- bitPackedFile *_blockBPF;
- bitPackedFile *_namesBPF;
-
- u32bit _lastIIDloaded;
-
- friend class seqFactory;
-};
-
-
-// Construct a new seqStore 'filename' from input file 'inputseq'.
-//
-void
-constructSeqStore(char *filename,
- seqCache *inputseq);
-
-
-#endif // SEQSTORE_H
diff --git a/kmer/libseq/seqStream.C b/kmer/libseq/seqStream.C
deleted file mode 100644
index e18c6ef..0000000
--- a/kmer/libseq/seqStream.C
+++ /dev/null
@@ -1,396 +0,0 @@
-#include "seqFactory.H"
-#include "seqStream.H"
-
-
-seqStream::seqStream(const char *filename) {
- _file = openSeqFile(filename);
- _string = 0L;
-
- _currentIdx = 0;
- _currentPos = 0;
- _streamPos = 0;
-
- _bufferMax = 1048576;
- _bufferLen = 0;
- _bufferPos = 0;
- _bufferSep = 0;
- _buffer = new char [_bufferMax + 1];
-
- _idxLen = _file->getNumberOfSequences();
- _idx = new seqStreamIndex [_idxLen + 1];
-
- //fprintf(stderr, "seqStream::seqStream()-- Allocating "u64bitFMT"MB for seqStreamIndex on "u64bitFMT" sequences.\n",
- // _idxLen * sizeof(seqStreamIndex) / 1024 / 1024, _idxLen);
-
- _seqNumOfPos = 0L;
-
- _lengthOfSequences = 0;
-
- _eof = false;
-
- _separator = '.';
- _separatorLength = 2;
-
- setSeparator('.', 2);
-
- _bgn = 0;
- _end = _lengthOfSequences;
-}
-
-
-
-seqStream::seqStream(const char *sequence, u32bit length) {
- _file = 0L;
- _string = (char *)sequence;
-
- _currentIdx = 0;
- _currentPos = 0;
- _streamPos = 0;
-
- _bufferMax = length;
- _bufferLen = length;
- _bufferPos = 0;
- _bufferSep = 0;
- _buffer = _string;
-
- _idxLen = 1;
- _idx = new seqStreamIndex [_idxLen + 1];
-
- _seqNumOfPos = 0L;
-
- _idx[0]._iid = 0;
- _idx[0]._len = length;
- _idx[0]._bgn = 0;
-
- _idx[1]._iid = ~u32bitZERO;
- _idx[1]._len = 0;
- _idx[1]._bgn = length;
-
- _lengthOfSequences = length;
-
- _eof = false;
-
- _separator = '.';
- _separatorLength = 20;
-
- _bgn = 0;
- _end = length;
-}
-
-
-
-seqStream::~seqStream() {
- if (_file) {
- delete _file;
- delete [] _buffer;
- }
- delete [] _idx;
- delete [] _seqNumOfPos;
-}
-
-
-
-void
-seqStream::setSeparator(char sep, u32bit len) {
-
- // Special case; no separator needed for string backed sequences.
- if (_string)
- return;
-
- // Bizarre signedness issue with sep=255
- // ST->get() == sep FAILS
- // x=ST->get(); x == sep SUCCEEDS
- //
- // Not suggested to use non-printable ascii.
-
- if ((isprint(sep) == 0) || (tolower(sep) == 'a') || (tolower(sep) == 'c') || (tolower(sep) == 'g') || (tolower(sep) == 't')) {
- fprintf(stderr, "seqStream::setSeparator()-- ERROR! Separator letter must be printable ASCII and not [ACGTacgt].\n");
- exit(1);
- }
- if (len == 0) {
- fprintf(stderr, "seqStream::setSeparator()-- ERROR! Separator length cannot be zero.\n");
- exit(1);
- }
-
- _lengthOfSequences = 0;
-
- _separator = sep;
- _separatorLength = len;;
-
- for (u32bit s=0; s<_idxLen; s++) {
- _idx[s]._iid = s;
- _idx[s]._len = _file->getSequenceLength(s);
- _idx[s]._bgn = _lengthOfSequences;
-
- _lengthOfSequences += _idx[s]._len;
- }
-
- _idx[_idxLen]._iid = ~u32bitZERO;
- _idx[_idxLen]._len = 0;
- _idx[_idxLen]._bgn = _lengthOfSequences;
-
- // Rebuild our sequence number of position map, if it exists.
- //
- if (_seqNumOfPos) {
- delete [] _seqNumOfPos;
- tradeSpaceForTime();
- }
-}
-
-
-
-void
-seqStream::tradeSpaceForTime(void) {
- u32bit i = 0;
- u32bit s = 0;
-
- //fprintf(stderr, "Allocating "u32bitFMT" u32bits for seqNumOfPos.\n", _lengthOfSequences);
-
- _seqNumOfPos = new u32bit [_lengthOfSequences];
-
- for (i=0; i<_lengthOfSequences; i++) {
-
- // Increment the sequence number until we enter into the next
- // sequence. Zero length sequences require the use of a 'while'
- // here.
- //
- while (i >= _idx[s+1]._bgn)
- s++;
-
- _seqNumOfPos[i] = s;
- }
-}
-
-
-
-unsigned char
-seqStream::get(void) {
- if (_streamPos >= _end)
- _eof = true;
- if ((_eof == false) && (_bufferPos >= _bufferLen))
- fillBuffer();
- if (_eof)
- return(0);
- if (_bufferSep == 0) {
- _currentPos++;
- _streamPos++;
- } else {
- _bufferSep--;
- }
- return(_buffer[_bufferPos++]);
-}
-
-
-
-void
-seqStream::rewind(void){
-
- // Search for the correct spot. Uncommon operation, be inefficient
- // but simple. The range was checked to be good by setRange().
-
- u32bit s = 0;
- u64bit l = 0;
-
- while ((s < _idxLen) && (l + _idx[s]._len < _bgn))
- l += _idx[s++]._len;
-
- _eof = false;
-
- // (_bgn - l) is a 32-bit quanitity because of the second half of
- // the while above. Although _bgn is a 64-bit value, the value
- // used to set _bufferPos will be for that of a string constructor,
- // and so _bgn will be 32-bits. fillBuffer() resets _bufferPos if
- // we're backed by a file.
-
- _currentIdx = s;
- _currentPos = _bgn - l;
- _streamPos = _bgn;
- _bufferPos = _bgn;
-
- //fprintf(stderr, "seqStream::rewind()-- 1 currentIdx="u32bitFMT" currentPos="u32bitFMT" streamPos="u32bitFMT" bufferPos="u32bitFMT"\n",
- // _currentIdx, _currentPos, _streamPos, _bufferPos);
-
- fillBuffer();
-
- //fprintf(stderr, "seqStream::rewind()-- 2 currentIdx="u32bitFMT" currentPos="u32bitFMT" streamPos="u32bitFMT" bufferPos="u32bitFMT"\n",
- // _currentIdx, _currentPos, _streamPos, _bufferPos);
-}
-
-
-
-void
-seqStream::setRange(u64bit bgn, u64bit end) {
-
- assert(bgn < end);
-
- u32bit s = 0;
- u64bit l = 0;
-
- while (s < _idxLen)
- l += _idx[s++]._len;
-
- if (end == ~u64bitZERO)
- end = l;
-
- if ((bgn > l) || (end > l))
- fprintf(stderr, "seqStream::setRange()-- ERROR: range ("u64bitFMT","u64bitFMT") too big; only "u64bitFMT" positions.\n",
- bgn, end, l), exit(1);
-
- _bgn = bgn;
- _end = end;
-
- rewind();
-}
-
-
-void
-seqStream::setPosition(u64bit pos) {
-
- assert(_bgn <= pos);
- assert( pos < _end);
-
- u64bit old = _bgn;
-
- _bgn = pos;
- rewind();
- _bgn = old;
-}
-
-
-u32bit
-seqStream::sequenceNumberOfPosition(u64bit p) {
- u32bit s = ~u32bitZERO;
-
- // binary search on our list of start positions, to find the
- // sequence that p is in.
-
- if (_lengthOfSequences <= p) {
- fprintf(stderr, "seqStream::sequenceNumberOfPosition()-- WARNING: position p="u64bitFMT" too big; only "u64bitFMT" positions.\n",
- p, _lengthOfSequences);
- return(s);
- }
-
- if (_seqNumOfPos)
- return(_seqNumOfPos[p]);
-
- if (_idxLen < 16) {
- for (s=0; s<_idxLen; s++)
- if ((_idx[s]._bgn <= p) && (p < _idx[s+1]._bgn))
- break;
- } else {
- u32bit lo = 0;
- u32bit hi = _idxLen;
- u32bit md = 0;
-
- while (lo <= hi) {
- md = (lo + hi) / 2;
-
- if (p < _idx[md]._bgn) {
- // This block starts after the one we're looking for.
- hi = md;
-
- } else if ((_idx[md]._bgn <= p) && (p < _idx[md+1]._bgn)) {
- // Got it!
- lo = md + 1;
- hi = md;
- s = md;
-
- } else {
- // By default, then, the block is too low.
- lo = md;
- }
- }
- }
-
- return(s);
-}
-
-
-
-void
-seqStream::fillBuffer(void) {
-
- // Special case for when we're backed by a character string; there
- // is no need to fill the buffer.
- //
- if (_file == 0L) {
- if (_currentPos >= _end)
- _eof = true;
- return;
- }
-
- // Read bytes from the _file, stuff them into the buffer. Assumes
- // there is nothing in the buffer to save.
-
- _bufferLen = 0;
- _bufferPos = 0;
-
- // Still more stuff in the sequence? Get it.
-
- if (_currentPos < _idx[_currentIdx]._len) {
-#ifdef DEBUG
- fprintf(stderr, "seqStream::fillBuffer()-- More Seq currentPos="u32bitFMT" len="u32bitFMT"\n", _currentPos, _idx[_currentIdx]._len);
-#endif
- _bufferLen = MIN(_idx[_currentIdx]._len - _currentPos, _bufferMax);
-
- if (_file->getSequence(_idx[_currentIdx]._iid,
- _currentPos,
- _currentPos + _bufferLen,
- _buffer) == false)
- fprintf(stderr, "seqStream::fillBuffer()-- Failed to getSequence(part) #1 iid="u32bitFMT" bgn="u32bitFMT" end="u32bitFMT"\n",
- _idx[_currentIdx]._iid, _currentPos, _currentPos + _bufferLen), exit(1);
-
- return;
- }
-
- // We've finished a sequence. Load the next.
-
- _currentPos = 0;
- _currentIdx++;
-
- while ((_currentIdx < _idxLen) && (_idx[_currentIdx]._len == 0))
- _currentIdx++;
-
-#ifdef DEBUG
- fprintf(stderr, "seqStream::fillBuffer()-- New Seq currentPos="u32bitFMT" len="u32bitFMT"\n", _currentPos, _idx[_currentIdx]._len);
-#endif
-
- // All done if there is no more sequence.
-
- if (_currentIdx >= _idxLen) {
- _eof = true;
- return;
- }
-
- // Insert a separator.
-
- for (_bufferLen = 0; _bufferLen < _separatorLength; _bufferLen++)
- _buffer[_bufferLen] = _separator;
-
- // Keep track of the separator - this is used to make sure we don't
- // advance the sequence/stream position while the separator is
- // being returned.
- //
- _bufferSep = _bufferLen;
-
- // How much to get; minimum of what is left in the sequence, and
- // the buffer size. Don't forget about the separator we already
- // inserted!
- //
- u32bit bl = MIN(_idx[_currentIdx]._len - _currentPos, _bufferMax - _bufferLen);
-
- if (_file->getSequence(_idx[_currentIdx]._iid,
- _currentPos,
- _currentPos + bl,
- _buffer + _bufferLen) == false)
- fprintf(stderr, "seqStream::fillBuffer()-- Failed to getSequence(part) #2 iid="u32bitFMT" bgn="u32bitFMT" end="u32bitFMT"\n",
- _idx[_currentIdx]._iid, _currentPos, _currentPos + bl), exit(1);
-
- _bufferLen += bl;
-
- // Load more, until buffer is full. Not really needed, and won't
- // improve performance much. AND it adds a lot of complexity to
- // track which sequence is current (_currentIdx).
-
- return;
-}
diff --git a/kmer/libseq/seqStream.H b/kmer/libseq/seqStream.H
deleted file mode 100644
index 0acf5b3..0000000
--- a/kmer/libseq/seqStream.H
+++ /dev/null
@@ -1,124 +0,0 @@
-#ifndef SEQSTREAM_H
-#define SEQSTREAM_H
-
-#include "util++.H"
-#include "bio++.H"
-
-#include "seqFile.H"
-
-struct seqStreamIndex {
- u32bit _iid; // seqFile IID
- u32bit _len; // length of the sequence
- u64bit _bgn; // begin position in the stream
-};
-
-
-class seqStream {
-public:
- seqStream(const char *filename);
- seqStream(const char *sequence, u32bit length);
- ~seqStream();
-
- // Separate sequences with this letter. Non-ACGT is always
- // returned as 'N'. Changing the length of the separator AFTER
- // setting the range will result in the wrong range being used.
- //
- void setSeparator(char sep, u32bit len);
-
- // get() returns one letter per input letter -- a gap of size n
- // will return n gap symbols.
- //
- unsigned char get(void);
- bool eof(void) { return(_eof); };
-
- // Returns to the start of the range.
- //
- void rewind(void);
-
- // Set the range of ACGT sequence we will return. Coordinates are
- // space-based. Example:
- //
- // >one
- // AAA
- // >two
- // C
- // >three
- // GGG
- //
- // We separate these sequences with three '-' letters.
- //
- // strPos 012...3...456
- // AAA---C---GGG
- //
- // range(0,0) -> nothing
- // range(0,1) -> A
- // range(0,3) -> AAA
- // range(0,4) -> AAAnnnC
- // range(0,5) -> AAAnnnCnnnG
- //
- void setRange(u64bit bgn, u64bit end);
- void setPosition(u64bit pos);
-
- // seqPos() is the position we are at in the current sequence;
- // seqIID() is the iid of that sequence;
- // strPos() is the position we are at in the chained sequence
- //
- // Values are not defined if the letter is a separator.
- //
- u32bit seqPos(void) { return(_currentPos); };
- u32bit seqIID(void) { return(_idx[_currentIdx]._iid); };
- u64bit strPos(void) { return(_streamPos); };
-
- u32bit numberOfSequences(void) { return(_idxLen); };
-
- // Return the length of, position of (in the chain) and IID of the
- // (s)th sequence in the chain.
- //
- u32bit lengthOf(u32bit s) { return((s >= _idxLen) ? ~u32bitZERO : _idx[s]._len); };
- u32bit IIDOf(u32bit s) { return((s >= _idxLen) ? ~u32bitZERO : _idx[s]._iid); };
- u64bit startOf(u32bit s) { return((s >= _idxLen) ? ~u64bitZERO : _idx[s]._bgn); };
-
- // For a chain position p, returns the s (above) for that position.
- //
- u32bit sequenceNumberOfPosition(u64bit p);
- void tradeSpaceForTime(void);
-
-private:
- void fillBuffer(void);
-
- seqFile *_file; // Backed by a seqFile.
- char *_string; // Backed by a character string.
-
- u64bit _bgn; // Begin/End position in chained sequence
- u64bit _end;
-
- u32bit _currentIdx; // index into _idx of the current sequence
- u32bit _currentPos; // position in the current sequence
- u64bit _streamPos; // position in the chained sequence
-
- // Buffer for holding sequence from the seqFile.
-
- u32bit _bufferMax; // Max size of the buffer
- u32bit _bufferLen; // Actual size of the buffer
- u32bit _bufferPos; // Position we are at in the buffer
- u32bit _bufferSep; // How much of the buffer is separator
- char *_buffer;
-
- // Info about the raw sequences
-
- u32bit _idxLen;
- seqStreamIndex *_idx;
-
- u32bit *_seqNumOfPos;
-
- u64bit _lengthOfSequences;
-
- bool _eof;
-
- char _separator;
- u32bit _separatorLength;
-};
-
-
-
-#endif // SEQSTREAM_H
diff --git a/kmer/libseq/sffFile.C b/kmer/libseq/sffFile.C
deleted file mode 100644
index c73eba9..0000000
--- a/kmer/libseq/sffFile.C
+++ /dev/null
@@ -1,208 +0,0 @@
-#include "sffFile.H"
-
-// Lots of ths came from AS_GKP_sff.c
-
-
-
-sffFile::sffFile() {
- clear();
-}
-
-sffFile::sffFile(const char *name) {
-
- clear();
-
- strcpy(_filename, name);
-
- _rb = new readBuffer(name);
-
- _rb->read(&_header, 31);
-
- if (_header.magic_number != 0x2e736666) {
- _header.swap_endianess = 1;
- _header.magic_number = u32bitSwap(_header.magic_number);
- _header.index_offset = u64bitSwap(_header.index_offset);
- _header.index_length = u32bitSwap(_header.index_length);
- _header.number_of_reads = u32bitSwap(_header.number_of_reads);
- _header.header_length = u16bitSwap(_header.header_length);
- _header.key_length = u16bitSwap(_header.key_length);
- _header.number_of_flows_per_read = u16bitSwap(_header.number_of_flows_per_read);
- }
-
- assert(_header.magic_number == 0x2e736666);
- assert(_header.number_of_flows_per_read < SFF_NUMBER_OF_FLOWS_MAX);
- assert(_header.key_length < SFF_KEY_SEQUENCE_MAX);
-
- _rb->read(_header.flow_chars, sizeof(char) * _header.number_of_flows_per_read);
- _rb->read(_header.key_sequence, sizeof(char) * _header.key_length);
-
- _firstReadLocation = _header.header_length;
-
- // The spec says the index might be here, however, all files I've
- // seen have the index at the end of the file.
- //
- if ((_header.index_length > 0) && (_header.index_offset == _header.header_length))
- _firstReadLocation += _header.index_length;
-
- // Index
- //
- _index = new sffIndex [_header.number_of_reads];
-
-
- for (u64bit i=0; i<_header.number_of_reads; i++) {
- u64bit pos = _rb->tell();
-
- _rb->read(&_read, 16);
-
- if (_header.swap_endianess) {
- _read.read_header_length = u16bitSwap(_read.read_header_length);
- _read.name_length = u16bitSwap(_read.name_length);
- _read.number_of_bases = u32bitSwap(_read.number_of_bases);
- }
-
- _index[i]._seqPos = pos;
- _index[i]._seqLen = _read.number_of_bases;
- _index[i]._namLen = _read.name_length;
-
- pos += _read.read_header_length;
- pos += sizeof(u16bit) * _header.number_of_flows_per_read;
- pos += sizeof(u8bit) * _read.number_of_bases;
- pos += sizeof(char) * _read.number_of_bases;
- pos += sizeof(u8bit) * _read.number_of_bases;
-
- pos += (_header.number_of_flows_per_read * sizeof(u16bit) +
- _read.number_of_bases * sizeof(u8bit) +
- _read.number_of_bases * sizeof(char) +
- _read.number_of_bases * sizeof(u8bit)) % 8;
-
- _rb->seek(pos);
- }
- //
- // Index
-
- _rb->seek(_firstReadLocation);
-
- _numberOfSequences = _header.number_of_reads;
-}
-
-sffFile::~sffFile() {
- delete _rb;
- delete [] _index;
-}
-
-////////////////////////////////////////
-
-seqFile *
-sffFile::openFile(const char *name) {
- struct stat st;
-
- // Open the file, return if it matches the SFF magic_number.
-
- errno = 0;
- stat(name, &st);
- if (errno)
- return(0L);
- if ((st.st_mode & S_IFREG) == 0)
- return(0L);
-
- FILE *F = fopen(name, "r");
- if (errno) {
- fprintf(stderr, "sffFile::openFile()- failed to open '%s': %s\n", name, strerror(errno));
- return(0L);
- }
-
- u32bit magic_number = 0;
- safeRead(fileno(F), &magic_number, "sff magic_number", sizeof(u32bit));
-
- fclose(F);
-
- if ((magic_number == 0x2e736666) || (u32bitSwap(magic_number) == 0x2e736666))
- return(new sffFile(name));
-
- return(0L);
-}
-
-
-
-bool
-sffFile::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
-
- if (iid > _header.number_of_reads)
- return(false);
-
- memset(&_read, 0, sizeof(sffRead));
-
- _rb->seek(_index[iid]._seqPos);
-
- _rb->read(&_read, 16);
-
- if (_header.swap_endianess) {
- _read.read_header_length = u16bitSwap(_read.read_header_length);
- _read.name_length = u16bitSwap(_read.name_length);
- _read.number_of_bases = u32bitSwap(_read.number_of_bases);
- _read.clip_quality_left = u16bitSwap(_read.clip_quality_left);
- _read.clip_quality_right = u16bitSwap(_read.clip_quality_right);
- _read.clip_adapter_left = u16bitSwap(_read.clip_adapter_left);
- _read.clip_adapter_right = u16bitSwap(_read.clip_adapter_right);
- }
-
- assert(_read.read_header_length < SFF_NAME_LENGTH_MAX);
- assert(_read.number_of_bases < SFF_NUMBER_OF_BASES_MAX);
-
- _rb->read(_read.name, sizeof(char) * _read.name_length);
- _read.name[_read.name_length] = 0;
-
- u64bit pos = _rb->tell();
-
- pos += _read.read_header_length;
- pos += sizeof(u16bit) * _header.number_of_flows_per_read;
- pos += sizeof(u8bit) * _read.number_of_bases;
-
- _rb->seek(pos);
-
- _rb->read(_read.bases, sizeof(char) * _read.number_of_bases);
- _read.bases[_read.number_of_bases] = 0;
-
- return(true);
-}
-
-
-
-
-
-bool
-sffFile::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
-
- if (iid > _header.number_of_reads)
- return(false);
-
- // Same as above, mostly.
-
- return(false);
-}
-
-
-
-void
-sffFile::clear(void) {
-
- memset(_filename, 0, FILENAME_MAX);
- memset(_typename, 0, FILENAME_MAX);
-
- strcpy(_typename, "SFF");
-
- _numberOfSequences = 0;
-
- _rb = 0L;
-
- memset(&_header, 0, sizeof(sffHeader));
- memset(&_read, 0, sizeof(sffRead));
-
- _index = 0L;
-
- _firstReadLocation = 0;
- _readIID = 0;
-}
diff --git a/kmer/libseq/sffFile.H b/kmer/libseq/sffFile.H
deleted file mode 100644
index 56bf987..0000000
--- a/kmer/libseq/sffFile.H
+++ /dev/null
@@ -1,104 +0,0 @@
-#ifndef SFF_H
-#define SFF_H
-
-#include "util++.H"
-#include "bio++.H"
-
-#include "seqFile.H"
-
-#define SFF_KEY_SEQUENCE_MAX 64
-
-#define SFF_NAME_LENGTH_MAX 256
-#define SFF_NUMBER_OF_FLOWS_MAX 512
-#define SFF_NUMBER_OF_BASES_MAX 2048 // The assembler itself cannot handle longer
-
-
-struct sffHeader {
- // The next block is read in one swoop from the sff file. DO NOT MODIFY!
- u32bit magic_number;
- char version[4];
- u64bit index_offset;
- u32bit index_length;
- u32bit number_of_reads;
- u16bit header_length;
- u16bit key_length;
- u16bit number_of_flows_per_read;
- u8bit flowgram_format_code;
-
- char flow_chars[SFF_NUMBER_OF_FLOWS_MAX]; // h->number_of_flows_per_read
- char key_sequence[SFF_KEY_SEQUENCE_MAX]; // h->key_length
-
- u32bit swap_endianess;
-};
-
-
-struct sffRead {
- // The next block is read in one swoop from the sff file. DO NOT MODIFY!
- u16bit read_header_length;
- u16bit name_length;
- u32bit number_of_bases;
- u16bit clip_quality_left;
- u16bit clip_quality_right;
- u16bit clip_adapter_left;
- u16bit clip_adapter_right;
-
- char name[SFF_NAME_LENGTH_MAX]; // r->name_length
-
- u16bit flowgram_values[SFF_NUMBER_OF_FLOWS_MAX]; // h->number_of_flows_per_read
- u8bit flow_index_per_base[SFF_NUMBER_OF_BASES_MAX]; // r->number_of_bases
- char bases[SFF_NUMBER_OF_BASES_MAX]; // r->number_of_bases
- u8bit quality_scores[SFF_NUMBER_OF_BASES_MAX]; // r->number_of_bases
-
- char quality[SFF_NUMBER_OF_BASES_MAX]; // quality_scores converted to CA-format qv
-};
-
-struct sffIndex {
- u64bit _seqPos;
- u32bit _seqLen;
- u32bit _namLen;
-};
-
-
-class sffFile : public seqFile {
-protected:
- sffFile(const char *filename);
- sffFile();
-
-public:
- ~sffFile();
-
-protected:
- seqFile *openFile(const char *name);
-
-public:
- u32bit find(const char *sequencename) {
- assert(0);
- return(0);
- };
-
- u32bit getSequenceLength(u32bit iid) { return(_index[iid]._seqLen); };
-
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
-
-private:
- void clear(void);
-
- readBuffer *_rb;
-
- sffHeader _header;
- sffRead _read;
-
- sffIndex *_index;
-
- u64bit _firstReadLocation;
- u64bit _readIID;
-
- friend class seqFactory;
-};
-
-
-#endif // SFF_H
diff --git a/kmer/libseq/test-correctSequence.H b/kmer/libseq/test-correctSequence.H
deleted file mode 100644
index cf5ee67..0000000
--- a/kmer/libseq/test-correctSequence.H
+++ /dev/null
@@ -1,151 +0,0 @@
-#ifndef TEST_CORRECTSEQUENCE_H
-#define TEST_CORRECTSEQUENCE_H
-
-//#define WITH_WHITESPACE
-
-struct correctSequence_t {
- char header[256];
- u32bit headerLength;
- char *sequence;
- u32bit sequenceLength;
-};
-
-
-correctSequence_t *correctSequence = 0L;
-mt_s *mtctx = 0L;
-
-char *chainSeq;
-u32bit *chainSeqPos;
-u32bit *chainSeqIID;
-u64bit *chainStrPos;
-
-
-void
-generateCorrectSequence(u32bit minLen, u32bit maxLen, u32bit numSeq) {
- char bases[4] = {'A', 'C', 'G', 'T'};
-
- u32bit n = numSeq;
- u32bit s = minLen;
- u32bit l = maxLen;
-
- u32bit seed = (u32bit)(getTime() * 1000);
-
- fprintf(stderr, "generateCorrectSequence()-- Using seed "u32bitFMT"\n", seed);
- fprintf(stderr, "generateCorrectSequence()-- Generating "u32bitFMT" sequences of length "u32bitFMT" to "u32bitFMT"\n", numSeq, minLen, maxLen);
-
- correctSequence = new correctSequence_t [n];
- mtctx = mtInit(seed);
-
- FILE *F = fopen("test-correctSequence.fasta", "w");
-
- for (u32bit i=0; i<n; i++) {
- u32bit j = s + (mtRandom32(mtctx) % (l-s));
- u32bit p = 0;
-
- sprintf(correctSequence[i].header, "sequence%d", i);
-
- correctSequence[i].headerLength = strlen(correctSequence[i].header);
- correctSequence[i].sequence = new char [j+1];
- correctSequence[i].sequenceLength = j;
-
- while (p < j)
- correctSequence[i].sequence[p++] = bases[mtRandom32(mtctx) & 0x3];
-
- correctSequence[i].sequence[p] = 0;
-
- // Spend lots of pain adding whitespace to the file.
-
-#ifdef WITH_WHITESPACE
- for (u32bit r=mtRandom32(mtctx) % 4; r--; )
- fprintf(F, "\n");
-
- fprintf(F, ">%s\n", correctSequence[i].header);
-
- for (u32bit r=mtRandom32(mtctx) % 4; r--; )
- fprintf(F, "\n");
-
- for (u32bit p=0; p<correctSequence[i].sequenceLength; p++) {
- fprintf(F, "%c", correctSequence[i].sequence[p]);
- if ((mtRandom32(mtctx) % 100) == 0)
- for (u32bit r=mtRandom32(mtctx) % 4; r--; )
- fprintf(F, "\n");
- }
-#else
- fprintf(F, ">%s\n", correctSequence[i].header);
- fprintf(F, "%s\n", correctSequence[i].sequence);
-#endif
-
- }
-
- for (u32bit r=mtRandom32(mtctx) % 4; r--; )
- fprintf(F, "\n");
-
- fclose(F);
-}
-
-
-void
-generateChainedAnswer(u32bit numSeq, char sep, u32bit sepLen) {
- u32bit maxLen = 0;
-
- for (u32bit i=0; i<numSeq; i++)
- maxLen += correctSequence[i].sequenceLength + sepLen;
-
- maxLen -= sepLen;
-
- delete [] chainSeq;
- delete [] chainSeqPos;
- delete [] chainSeqIID;
- delete [] chainStrPos;
-
- chainSeq = new char [maxLen + 1];
- chainSeqPos = new u32bit [maxLen];
- chainSeqIID = new u32bit [maxLen];
- chainStrPos = new u64bit [maxLen];
-
- u32bit p = 0;
- u64bit strpos = 0;
-
- for (u32bit sid=0; sid<numSeq; sid++) {
- for (u32bit ppp=0; ppp<correctSequence[sid].sequenceLength; ppp++, p++, strpos++) {
- chainSeq[p] = correctSequence[sid].sequence[ppp];
- chainSeqPos[p] = ppp;
- chainSeqIID[p] = sid;
- chainStrPos[p] = strpos;
- }
- if (sid+1 < numSeq) {
- for (u32bit ppp=0; ppp<sepLen; ppp++, p++) {
- chainSeq[p] = sep;
- chainSeqPos[p] = ~u32bitZERO;
- chainSeqIID[p] = ~u32bitZERO;
- chainStrPos[p] = ~u64bitZERO;
- }
- }
- }
-
- assert(p == maxLen);
-
- // Used to get the length of the string.
- chainSeq[maxLen] = 0;
-}
-
-
-void
-removeCorrectSequence(u32bit numSeq) {
-
- unlink("test-correctSequence.fasta");
- unlink("test-correctSequence.fastaidx");
-
- for (u32bit i=0; i<numSeq; i++)
- delete [] correctSequence[i].sequence;
-
- delete [] correctSequence;
-
- delete [] chainSeq;
- delete [] chainSeqPos;
- delete [] chainSeqIID;
- delete [] chainStrPos;
-}
-
-
-#endif // TEST_CORRECTSEQUENCE_H
diff --git a/kmer/libseq/test-merStream.C b/kmer/libseq/test-merStream.C
deleted file mode 100644
index 57a36f6..0000000
--- a/kmer/libseq/test-merStream.C
+++ /dev/null
@@ -1,284 +0,0 @@
-#include "util.h"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-#include "test-correctSequence.H"
-
-#define FAIL() { err++; assert(0); }
-
-#warning HOW DO WE TEST IF WE GET ALL THE MERS?
-
-
-u32bit
-testMerStreamSimple(merStream *MS, u32bit merSize, char *seq, u32bit *SP) {
- u32bit err = 0;
- u32bit pos = 0;
- char testmer[32];
- bool verbose = true;
- bool nm = false;
-
- if (verbose)
- fprintf(stdout, "testMSsimple() begins.\n");
-
- // Until we have no more mers in the input
-
- while (seq[pos + merSize - 1] != 0) {
- nm = MS->nextMer();
-
- MS->theFMer().merToString(testmer);
-
- if (verbose) {
- fprintf(stdout, "MS pos="u32bitFMT" posInSeq="u64bitFMT" posInStr="u64bitFMT" seqNum="u64bitFMT"\n",
- pos,
- MS->thePositionInSequence(),
- MS->thePositionInStream(),
- MS->theSequenceNumber());
- if (strncmp(testmer, seq + pos, merSize))
- fprintf(stdout, "MS pos="u32bitFMT" failed '%s' != '%s'.\n", pos, testmer, seq + pos);
- }
-
- assert(nm == true);
- assert(MS->thePositionInSequence() == SP[pos]);
- assert(MS->thePositionInStream() == SP[pos]);
- assert(MS->theSequenceNumber() == 0);
- assert(strncmp(testmer, seq + pos, merSize) == 0);
-
- pos++;
- }
-
- // Should have no more mers
-
- nm = MS->nextMer();
- assert(nm == false);
-
- return(err);
-}
-
-
-
-u32bit
-testMerStreamOperation(merStream *MS, u32bit beg, u32bit end, u32bit sepLen) {
- u32bit err = 0;
-
- char fmerstr[256];
- char rmerstr[256];
- char cmerstr[256];
- char tmerstr[256];
-
- while (MS->nextMer()) {
- MS->theFMer().merToString(fmerstr);
- MS->theRMer().merToString(rmerstr);
- MS->theCMer().merToString(cmerstr);
-
- if ((strcmp(fmerstr, cmerstr) != 0) && ((strcmp(rmerstr, cmerstr) != 0))) {
- fprintf(stderr, "mer strings disagree; F:%s R:%s C:%s\n", fmerstr, rmerstr, cmerstr);
- FAIL();
- }
-
- reverseComplementSequence(rmerstr, strlen(rmerstr));
-
- if (strcmp(fmerstr, rmerstr) != 0) {
- fprintf(stderr, "mer strings disagree after reverse; F:%s R:%s\n", fmerstr, rmerstr);
- FAIL();
- }
-
- u32bit pseq = MS->thePositionInSequence();
- u32bit pstr = MS->thePositionInStream();
- u32bit piid = MS->theSequenceNumber();
-
- u32bit mersize = MS->theFMer().getMerSize();
- u32bit merspan = MS->theFMer().getMerSpan();
-
-#if 0
- if (beg > 10) {
- u32bit pp = pstr + piid * sepLen - 10;
- u32bit xx = 0;
-
- fprintf(stderr, "beg="u32bitFMT" pstr="u32bitFMT" '", beg, pstr);
-
- for (xx=0; xx<10; xx++, pp++)
- fprintf(stderr, "%c", chainSeq[pp]);
- fprintf(stderr, ":");
- for (xx=0; xx<merspan; xx++, pp++)
- fprintf(stderr, "%c", chainSeq[pp]);
- fprintf(stderr, ":");
- for (xx=0; xx<10; xx++, pp++)
- fprintf(stderr, "%c", chainSeq[pp]);
-
- fprintf(stderr, "'\n");
- }
-#endif
-
- if (mersize == merspan) {
- strncpy(tmerstr, correctSequence[piid].sequence + pseq, mersize);
- tmerstr[mersize] = 0;
-
- if (strcmp(fmerstr, tmerstr) != 0) {
- fprintf(stderr, "mer string doesn't agree with sequence; '%s' vs '%s'.\n", fmerstr, tmerstr);
- FAIL();
- }
-
- if ((pstr < beg) || (end < pstr)) {
- fprintf(stderr, "mer stream position out of range; at "u32bitFMT", range "u32bitFMT"-"u32bitFMT"\n",
- pstr, beg, end);
- FAIL();
- }
-
- // The pstr returned above is the ACGT position, not the
- // chainSeq position we expect. Trusting that the IID is
- // correct (if not, the previous strcmp() would have failed) we
- // can add in the missing separators to get a chainSeq
- // position.
-
- strncpy(tmerstr, chainSeq + pstr + piid * sepLen, mersize);
- tmerstr[mersize] = 0;
-
- if (strcmp(fmerstr, tmerstr) != 0) {
- fprintf(stderr, "mer string doesn't agree with stream; '%s' vs '%s'.\n", fmerstr, tmerstr);
- FAIL();
- }
- }
- }
-
- return(err);
-}
-
-
-
-
-u32bit
-testMerStream(kMerBuilder *KB, u32bit numSeq, char sep, u32bit sepLen) {
- u32bit err = 0;
- seqStream *ST = 0L;
- merStream *MS = 0L;
-
- generateChainedAnswer(numSeq, sep, sepLen);
-
- if (numSeq > 1) {
- ST = new seqStream("test-correctSequence.fasta");
- ST->setSeparator(sep, sepLen);
- } else {
- ST = new seqStream(correctSequence[0].sequence, correctSequence[0].sequenceLength);
- }
-
- MS = new merStream(KB, ST, true, true);
-
- u32bit maxLen = ST->startOf(numSeq-1) + ST->lengthOf(numSeq-1);
-
- // Whole thing, rewind, whole thing
-
- fprintf(stderr, "whole thing.\n");
-
- err += testMerStreamOperation(MS, 0, maxLen, sepLen);
- MS->rewind();
- err += testMerStreamOperation(MS, 0, maxLen, sepLen);
-
-
- // Random subsets - we're not terribly interested in streaming,
- // just getting the start/end correct.
-
- fprintf(stderr, "subsets.\n");
-
- for (u32bit iter=0; iter<500; iter++) {
- u32bit beg = mtRandom32(mtctx) % maxLen;
- u32bit end = (beg + 10000 < maxLen) ? (beg + 10000) : maxLen;
-
- //fprintf(stderr, "subsets - "u32bitFMT"-"u32bitFMT"\n", beg, end);
-
- MS->setBaseRange(beg, end);
-
- err += testMerStreamOperation(MS, beg, end, sepLen);
- MS->rewind();
- err += testMerStreamOperation(MS, beg, end, sepLen);
- }
-
- delete MS;
-
- return(err);
-}
-
-
-
-
-
-int
-main(int argc, char **argv) {
- u32bit minLen = 1000;
- u32bit maxLen = 200000;
- u32bit numSeq = 1000;
- u32bit err = 0;
-
- // Very simple merStream test
-
- {
- fprintf(stdout, "merStream(kMerBuilder(20), ...)\n");
-
- merStream *MS = new merStream(new kMerBuilder(20),
- new seqStream("GGGTCAACTCCGCCCGCACTCTAGC", 25),
- true, true);
- u32bit SP[10] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
-
- testMerStreamSimple(MS, 20, "GGGTCAACTCCGCCCGCACTCTAGC", SP);
- MS->rewind();
- testMerStreamSimple(MS, 20, "GGGTCAACTCCGCCCGCACTCTAGC", SP);
- MS->rewind();
- MS->rewind();
- testMerStreamSimple(MS, 20, "GGGTCAACTCCGCCCGCACTCTAGC", SP);
-
- delete MS;
-
- fprintf(stdout, "merStream(kMerBuilder(20), ...) - PASSED\n");
- }
-
- {
- fprintf(stdout, "merStream(kMerBuilder(20, 1), ...)\n");
-
- merStream *MS = new merStream(new kMerBuilder(20, 1),
- new seqStream("GGGAATTTTCAACTCCGCCCGCACTCTAGCCCAAA", 35),
- true, true);
- u32bit SP[10] = { 0, 3, 5, 9, 10, 12 };
-
- testMerStreamSimple(MS, 20, "GATCACTCGCGCACTCTAGCA", SP);
- MS->rewind();
- testMerStreamSimple(MS, 20, "GATCACTCGCGCACTCTAGCA", SP);
- MS->rewind();
- MS->rewind();
- testMerStreamSimple(MS, 20, "GATCACTCGCGCACTCTAGCA", SP);
-
- delete MS;
-
- fprintf(stdout, "merStream(kMerBuilder(20, 1), ...) - PASSED\n");
- }
-
- // Move on to harder tests
-
- generateCorrectSequence(minLen, maxLen, numSeq);
-
- // Tests seqStream(string, strlen) construction method
-
- fprintf(stderr, "err += testMerStream(new kMerBuilder(20, 0, 0L), 1, '.', 1);\n");
- err += testMerStream(new kMerBuilder(20, 0, 0L), 1, '.', 1);
-
- fprintf(stderr, "err += testMerStream(new kMerBuilder(22, 1, 0L), 1, '.', 1);\n");
- err += testMerStream(new kMerBuilder(22, 1, 0L), 1, '.', 1);
-
- // Tests seqStream(filename) construction method
-
- fprintf(stderr, "err += testMerStream(new kMerBuilder(20, 0, 0L), numSeq, '.', 1);\n");
- err += testMerStream(new kMerBuilder(20, 0, 0L), numSeq, '.', 1);
-
- fprintf(stderr, "err += testMerStream(new kMerBuilder(28, 0, 0L), numSeq, '.', 100);\n");
- err += testMerStream(new kMerBuilder(28, 0, 0L), numSeq, '.', 100);
-
- fprintf(stderr, "err += testMerStream(new kMerBuilder(24, 4, 0L), numSeq, '.', 100);\n");
- err += testMerStream(new kMerBuilder(24, 4, 0L), numSeq, '.', 100);
-
- removeCorrectSequence(numSeq);
-
- if (err == 0)
- fprintf(stderr, "Success!\n");
-
- exit(err > 0);
-}
diff --git a/kmer/libseq/test-seqCache.C b/kmer/libseq/test-seqCache.C
deleted file mode 100644
index 30aad28..0000000
--- a/kmer/libseq/test-seqCache.C
+++ /dev/null
@@ -1,181 +0,0 @@
-#include "util.h"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-#include "test-correctSequence.H"
-
-
-u32bit
-testSeqVsCorrect(seqInCore *S, u32bit testID) {
- u32bit err = 0;
-
- if (S == 0L) {
- fprintf(stderr, "testID:"u32bitFMT" - empty sequence\n", testID);
- return(1);
- }
-
- u32bit sid = S->getIID();
-
- if (strcmp(S->header(), correctSequence[sid].header) != 0) {
- fprintf(stderr, "testID:"u32bitFMT" - header differs '%s' vs '%s'\n", testID, S->header(), correctSequence[sid].header);
- err++;
- }
- if (S->headerLength() != correctSequence[sid].headerLength) {
- fprintf(stderr, "testID:"u32bitFMT" - header length differs "u32bitFMT" vs "u32bitFMT"\n", testID, S->headerLength(), correctSequence[sid].headerLength);
- err++;
- }
- if (strcmp(S->sequence(), correctSequence[sid].sequence) != 0) {
- fprintf(stderr, "testID:"u32bitFMT" - sequence differs\n", testID);
- err++;
- }
- if (strlen(S->sequence()) != correctSequence[sid].sequenceLength) {
- fprintf(stderr, "testID:"u32bitFMT" - sequence length differs strlen "u32bitFMT" vs "u32bitFMT"\n", testID, (u32bit)strlen(S->sequence()), correctSequence[sid].sequenceLength);
- err++;
- }
- if (S->sequenceLength() != correctSequence[sid].sequenceLength) {
- fprintf(stderr, "testID:"u32bitFMT" - sequence length differs "u32bitFMT" vs "u32bitFMT"\n", testID, S->sequenceLength(), correctSequence[sid].sequenceLength);
- err++;
- }
-
- return(err);
-}
-
-
-u32bit
-testSeqCacheIDLookups(seqCache *SC) {
- u32bit err = 0;
- u32bit numSeq = SC->getNumberOfSequences();
- double start = getTime();
-
- // 1 - getSequenceIID()
- fprintf(stderr, "1 - getSequenceIID()\n");
- for (u32bit sid=0; sid<numSeq; sid++) {
- if (sid != SC->getSequenceIID(correctSequence[sid].header)) {
- fprintf(stderr, "2 - failed to find name '%s'\n", correctSequence[sid].header);
- err++;
- }
- }
-
- fprintf(stderr, "Test took %f seconds.\n", getTime() - start);
-
- return(err);
-}
-
-
-u32bit
-testSeqCache(seqCache *SC) {
- u32bit err = 0;
- u32bit numSeq = SC->getNumberOfSequences();
- seqInCore *S = 0L;
- double start = getTime();
-
- // 0 - getSequenceLength()
- fprintf(stderr, "0 - getSequenceLength()\n");
- for (u32bit sid=0; sid<numSeq; sid++)
- if (SC->getSequenceLength(sid) != correctSequence[sid].sequenceLength) {
- fprintf(stderr, "1 - length differs.\n");
- err++;
- }
-
- // 2 - stream with getSequenceInCore()
- fprintf(stderr, "2 - stream with getSequenceInCore()\n");
- S = SC->getSequenceInCore();
- while (S != 0L) {
- err += testSeqVsCorrect(S, 2);
- delete S;
- S = SC->getSequenceInCore();
- }
-
- // 3 - iterate with getSequenceInCore(sid++)
- fprintf(stderr, "3 - iterate with getSequenceInCore(sid++)\n");
- for (u32bit sid=0; sid<numSeq; sid++) {
- S = SC->getSequenceInCore(sid);
- err += testSeqVsCorrect(S, 3);
- delete S;
- }
-
- // 4 - random with getSequenceInCore(sid)
- fprintf(stderr, "4 - random with getSequenceInCore(sid)\n");
- for (u32bit cnt=0; cnt<4*numSeq; cnt++) {
- u32bit sid = mtRandom32(mtctx) % numSeq;
- S = SC->getSequenceInCore(sid);
- err += testSeqVsCorrect(S, 4);
- delete S;
- }
-
- fprintf(stderr, "Test took %f seconds.\n", getTime() - start);
-
- return(err);
-}
-
-
-int
-main(int argc, char **argv) {
- u32bit minLen = 100;
- u32bit maxLen = 2000;
- u32bit numSeq = 100000;
- seqCache *SC = 0L;
- u32bit err = 0;
-
- generateCorrectSequence(minLen, maxLen, numSeq);
-
- fprintf(stderr, "seqCache(file, 0, true) (ID lookups)\n");
- SC = new seqCache("test-correctSequence.fasta", 0, true);
- //err += testSeqCacheIDLookups(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 0, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 0, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 1, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 1, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 2, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 2, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 4, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 4, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 8, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 8, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 32, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 32, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 200, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 200, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 1000000, true)\n");
- SC = new seqCache("test-correctSequence.fasta", 1000000, true);
- err += testSeqCache(SC);
- delete SC;
-
- fprintf(stderr, "seqCache(file, 0, true) -- loadAllSequence\n");
- SC = new seqCache("test-correctSequence.fasta", 0, true);
- SC->loadAllSequences();
- err += testSeqCache(SC);
- delete SC;
-
- removeCorrectSequence(numSeq);
-
- if (err == 0)
- fprintf(stderr, "Success!\n");
-
- exit(err > 0);
-}
diff --git a/kmer/libseq/test-seqStream.C b/kmer/libseq/test-seqStream.C
deleted file mode 100644
index 94bd3e3..0000000
--- a/kmer/libseq/test-seqStream.C
+++ /dev/null
@@ -1,287 +0,0 @@
-#include "util.h"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-#include "test-correctSequence.H"
-
-#define FAIL() { err++; assert(0); }
-
-
-u32bit
-testIndexing(u32bit numSeq, char sep, u32bit sepLen) {
- u32bit err = 0;
- seqStream *ST = 0L;
-
- fprintf(stderr, "testIndexing()-- numSeq="u32bitFMT" sep=%c sepLen="u32bitFMT"\n", numSeq, sep, sepLen);
-
- generateChainedAnswer(numSeq, sep, sepLen);
-
- if (numSeq > 1) {
- ST = new seqStream("test-correctSequence.fasta");
- ST->setSeparator(sep, sepLen);
- } else {
- ST = new seqStream(correctSequence[0].sequence, correctSequence[0].sequenceLength);
- }
-
- u32bit maxLen = ST->startOf(numSeq-1) + ST->lengthOf(numSeq-1);
-
- // Basic checks on the reverse lookup - this is state independent;
- // it changes only based on the separator length. In other words,
- // there is no need to check this while iterating through the
- // seqStream.
-
- fprintf(stderr, "IGNORE THIS WARNING: ");
- if (ST->sequenceNumberOfPosition(maxLen) != ~u32bitZERO) {
- fprintf(stderr, "maxLen too small.\n");
- FAIL();
- }
- if (ST->sequenceNumberOfPosition(maxLen - 1) == ~u32bitZERO) {
- fprintf(stderr, "maxLen too big.\n");
- FAIL();
- }
-
- // Check all lookups - lengthOf() and IIDOf() are implicitly
- // checked by the operation of seqStream (get() mostly). startOf()
- // isn't, but inserting errors in setRange() led to
- // infinite-looking loops.
-
- u64bit pos = 0;
- u64bit sta = 0;
-
- for (u32bit sid=0; sid<numSeq; sid++) {
- if (ST->lengthOf(sid) != correctSequence[sid].sequenceLength) {
- fprintf(stderr, "lengthOf "u32bitFMT" returned "u32bitFMT", not correct "u32bitFMT"\n",
- sid, ST->lengthOf(sid), correctSequence[sid].sequenceLength);
- FAIL();
- }
- if (ST->startOf(sid) != sta) {
- fprintf(stderr, "startOf "u32bitFMT" returned "u64bitFMT", not correct "u64bitFMT"\n",
- sid, ST->startOf(sid), sta);
- FAIL();
- }
- if (ST->IIDOf(sid) != sid) {
- fprintf(stderr, "IIDOf "u32bitFMT" returned "u32bitFMT", not correct "u32bitFMT"\n",
- sid, ST->IIDOf(sid), sid);
- FAIL();
- }
-
- sta += correctSequence[sid].sequenceLength;
-
- for (u32bit ppp=0; ppp<correctSequence[sid].sequenceLength; ppp++, pos++) {
- if (ST->sequenceNumberOfPosition(pos) != sid) {
- fprintf(stderr, "sequenceNumberOfPosition "u64bitFMT" returned "u32bitFMT", not correct "u32bitFMT".\n",
- pos, ST->sequenceNumberOfPosition(pos), sid);
- FAIL();
- }
- }
- }
- if (pos != maxLen) {
- fprintf(stderr, "maxLen wrong.\n");
- FAIL();
- }
-
- // Check the separator. Seek to a spot right before one, and count
- // that we have the correct length. More rigorously tested in
- // testChaining().
-
- for (u32bit sid=0; sid<numSeq-1; sid++) {
- ST->setRange(ST->startOf(sid) + ST->lengthOf(sid)-1, ~u64bitZERO);
- ST->get();
- for (u32bit x=0; x<sepLen; x++) {
- char s = ST->get();
- if (s != sep) {
- fprintf(stderr, "wrong separator at sep "u32bitFMT" got %d expected %d\n", x, s, sep);
- FAIL();
- }
- }
- if (ST->get() == sep) {
- fprintf(stderr, "too many separators!\n");
- FAIL();
- }
- }
-
- delete ST;
-
- return(err);
-}
-
-
-
-u32bit
-testSeqStream(seqStream *ST, u32bit sib, u32bit sie, char sep) {
- u32bit err = 0;
-
- while (ST->eof() == false) {
- u32bit sp = ST->seqPos();
- u32bit si = ST->seqIID();
- u64bit st = ST->strPos();
- char ch = ST->get();
-
- if (ch != 0) {
- if (ch != chainSeq[sib]) {
- fprintf(stderr, "sp="u32bitFMT" si="u32bitFMT" st="u64bitFMT" ch=%c -- letter wrong got'%c'\n", sp, si, st, ch, chainSeq[sib]);
- FAIL();
- }
- if ((ch != sep) && (sp != chainSeqPos[sib])) {
- fprintf(stderr, "sp="u32bitFMT" si="u32bitFMT" st="u64bitFMT" ch=%c -- seqPos wrong got "u32bitFMT"\n", sp, si, st, ch, chainSeqPos[sib]);
- FAIL();
- }
- if ((ch != sep) && (si != chainSeqIID[sib])) {
- fprintf(stderr, "sp="u32bitFMT" si="u32bitFMT" st="u64bitFMT" ch=%c -- seqIID wrong got"u32bitFMT"\n", sp, si, st, ch, chainSeqIID[sib]);
- FAIL();
- }
- if ((ch != sep) && (st != chainStrPos[sib])) {
- fprintf(stderr, "sp="u32bitFMT" si="u32bitFMT" st="u64bitFMT" ch=%c -- strPos wrong got "u64bitFMT"\n", sp, si, st, ch, chainStrPos[sib]);
- FAIL();
- }
-
- sib++;
- }
- }
-
- if (sib != sie) {
- fprintf(stderr, "iterated length wrong; sib="u32bitFMT" sie="u32bitFMT"\n", sib, sie);
- FAIL();
- }
-
- return(err);
-}
-
-
-
-u32bit
-testChaining(u32bit numSeq, char sep, u32bit sepLen) {
- u32bit err = 0;
- seqStream *ST = 0L;
-
- fprintf(stderr, "testChaining()-- numSeq="u32bitFMT" sep=%c sepLen="u32bitFMT"\n", numSeq, sep, sepLen);
-
- generateChainedAnswer(numSeq, sep, sepLen);
-
- if (numSeq > 1) {
- ST = new seqStream("test-correctSequence.fasta");
- ST->setSeparator(sep, sepLen);
- } else {
- ST = new seqStream(correctSequence[0].sequence, correctSequence[0].sequenceLength);
- }
-
- // Do a test on the whole thing.
-
- {
- u32bit sib = 0;
- u32bit sie = strlen(chainSeq);
-
- fprintf(stderr, "initial test with full range\n");
- testSeqStream(ST, sib, sie, sep);
-
- fprintf(stderr, "initial test with full range (rewind)\n");
- ST->rewind();
- testSeqStream(ST, sib, sie, sep);
- }
-
-
- // Set the range to random values, and check all the results.
- // We've already verified the index works, so we're free to use
- // that (but we currently don't).
-
- u32bit maxLen = ST->startOf(numSeq-1) + ST->lengthOf(numSeq-1);
-
- fprintf(stderr, "test on subranges\n");
-
- for (u32bit iter=0; iter<500; iter++) {
- u32bit beg = mtRandom32(mtctx) % maxLen;
- u32bit end = mtRandom32(mtctx) % maxLen;
- if (beg > end) {
- u32bit t = end;
- end = beg;
- beg = t;
- }
-
- ST->setRange(beg, end);
-
- // Compute the position in our stream for the ACGT based beg and
- // end. The quirk here is that our stream includes the
- // separator.
-
- u32bit sib = 0; // chainSeq position
- u32bit sie = 0;
-
- for (u32bit ppp=0, sid=0; sid<numSeq; sid++) {
- u32bit len = correctSequence[sid].sequenceLength;
-
- if ((ppp <= beg) && (beg < ppp + len)) {
- sib += beg - ppp;
- break;
- }
-
- ppp += len;
- sib += len + sepLen;
- }
-
- for (u32bit ppp=0, sid=0; sid<numSeq; sid++) {
- u32bit len = correctSequence[sid].sequenceLength;
-
- if ((ppp <= end) && (end < ppp + len)) {
- sie += end - ppp;
- break;
- }
-
- ppp += len;
- sie += len + sepLen;
- }
-
- // Optionally do a rewind in the middle
-
- if (iter % 2) {
- //fprintf(stderr, "Random iter "u32bitFMT" (with rewind)\n", iter);
- while (ST->eof() == false)
- ST->get();
- ST->rewind();
- } else {
- //fprintf(stderr, "Random iter "u32bitFMT"\n", iter);
- }
-
-
- testSeqStream(ST, sib, sie, sep);
- }
-
- return(err > 0);
-}
-
-
-
-int
-main(int argc, char **argv) {
- u32bit minLen = 100;
- u32bit maxLen = 20000;
- u32bit numSeq = 1000;
- u32bit err = 0;
-
- generateCorrectSequence(minLen, maxLen, numSeq);
-
- // Tests seqStream(string, strlen) construction method
-
- err += testIndexing(1, '.', 1);
- err += testChaining(1, '.', 1);
-
- // Tests seqStream(filename) construction method
-
- err += testIndexing(numSeq, '.', 1);
- err += testIndexing(numSeq, ':', 10);
- err += testIndexing(numSeq, 'z', 100);
- err += testIndexing(numSeq, '-', 1000);
-
- err += testChaining(numSeq, '.', 1);
- err += testChaining(numSeq, ':', 10);
- err += testChaining(numSeq, 'z', 100);
- err += testChaining(numSeq, '-', 1000);
-
- removeCorrectSequence(numSeq);
-
- if (err == 0)
- fprintf(stderr, "Success!\n");
-
- exit(err > 0);
-}
diff --git a/kmer/libseq/test/Makefile b/kmer/libseq/test/Makefile
deleted file mode 100644
index bc03f4b..0000000
--- a/kmer/libseq/test/Makefile
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
-PROG = test-merstream-speed
-
-INCLUDE = -I.. -I../../libutil -I../../libbio -I../../libseq
-LIBS = -L.. -L../../libutil -L../../libbio -L../../libseq -lseq -lbio -lutil -lm
-OBJS =
-
-include ../../Make.compilers
-
-all: $(PROG)
- @echo Tests passed!
-
-test-merstream-speed: test-merstream-speed.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-merstream-speed.o test-merstream-speed.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-merstream-speed test-merstream-speed.o $(LIBS)
- ../../leaff/leaff -G 10000 1000 10000 > junk.fasta
- cat junk.fasta > /dev/null
- ./test-merstream-speed junk.fasta
- rm -f junk*
-
-clean:
- rm -f $(PROG) *.o *junk*
diff --git a/kmer/libseq/test/test-merstream-speed.C b/kmer/libseq/test/test-merstream-speed.C
deleted file mode 100644
index 7eac3d1..0000000
--- a/kmer/libseq/test/test-merstream-speed.C
+++ /dev/null
@@ -1,52 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "bio++.H"
-
-#include "seqCache.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-int
-main(int argc, char **argv) {
- speedCounter *C = 0L;
- FILE *F = 0L;
- seqStream *S = 0L;
- merStream *M = 0L;
-
- if (argc != 2) {
- fprintf(stderr, "usage: %s some.fasta\n", argv[0]);
- fprintf(stderr, "Reads some.fasta using fgetc(), the seqStream and the merStream,\n");
- fprintf(stderr, "reporting the speed of each method.\n");
- exit(1);
- }
-
- ////////////////////////////////////////
- F = fopen(argv[1], "r");
- C = new speedCounter("fgetc(): %7.2f Mthings -- %5.2f Mthings/second\r", 1000000.0, 0x3fffff, true);
- while (!feof(F))
- fgetc(F), C->tick();
- delete C;
- fclose(F);
-
- ////////////////////////////////////////
- S = new seqStream(argv[1]);
- C = new speedCounter("seqStream: %7.2f Mthings -- %5.2f Mthings/second\r", 1000000.0, 0x3fffff, true);
- while (S->get())
- C->tick();
- delete C;
- delete S;
-
- ////////////////////////////////////////
- M = new merStream(new kMerBuilder(20),
- new seqStream(argv[1]),
- true, true);
- C = new speedCounter("seqStream -> merStream: %7.2f Mthings -- %5.2f Mthings/second\r", 1000000.0, 0x3fffff, true);
- while (M->nextMer())
- C->tick();
- delete C;
- delete M;
-
- exit(0);
-}
-
diff --git a/kmer/libutil/Make.include b/kmer/libutil/Make.include
deleted file mode 100644
index be5a515..0000000
--- a/kmer/libutil/Make.include
+++ /dev/null
@@ -1,63 +0,0 @@
-# -*- makefile -*-
-
-$(eval $(call Include,$/mt19937ar/))
-$(eval $(call Include,$/kazlib/))
-
-src := $/bigQueue.C \
- $/bigQueue.H \
- $/bitOperations.h \
- $/bitPackedArray.C \
- $/bitPackedArray.H \
- $/bitPackedFile.C \
- $/bitPackedFile.H \
- $/bitPacking.h \
- $/eliasDeltaEncoding.h \
- $/eliasGammaEncoding.h \
- $/endianess.H \
- $/fibonacciEncoding.h \
- $/fibonacciNumbers.C \
- $/file.c \
- $/generalizedUnaryEncoding.h \
- $/intervalList.C \
- $/intervalList.H \
- $/logMsg.H \
- $/md5.c \
- $/palloc.c \
- $/qsort_mt.c \
- $/readBuffer.C \
- $/readBuffer.H \
- $/recordFile.C \
- $/recordFile.H \
- $/speedCounter.C \
- $/speedCounter.H \
- $/splitToWords.H \
- $/sweatShop.C \
- $/sweatShop.H \
- $/u32bitList.H \
- $/unaryEncoding.h \
- $/util++.H \
- $/util.c \
- $/util.h
-
-# Broken
-# $/bzipBuffer.C
-# $/bzipBuffer.H
-
-# Executables
-# $/unaryEncodingTester.C
-
-
-$/.C_SRCS :=$(filter %.c,${src})
-$/.CXX_SRCS :=$(filter %.C,${src})
-$/.CXX_INCS :=$(filter %.H,${src}) $(filter %.h,${src})
-$/.CXX_LIBS :=$/libutil.a
-
-$/.CLEAN := $/*.o
-
-$/libutil.a: ${$/.C_SRCS:.c=.o} ${$/.CXX_SRCS:.C=.o} \
- $/mt19937ar/mt19937ar.o \
- $/kazlib/dict.o \
- $/kazlib/except.o \
- $/kazlib/hash.o \
- $/kazlib/list.o \
- $/kazlib/sfx.o
diff --git a/kmer/libutil/NOTES b/kmer/libutil/NOTES
deleted file mode 100644
index cb1d6f2..0000000
--- a/kmer/libutil/NOTES
+++ /dev/null
@@ -1,10 +0,0 @@
-Various notes that should be turned into real documentation
-
-merStream
- (need to check this)
- posInSeq() is relative to the start of the current sequence.
- posInStream() is relative to the start of the source file.
-
-Whitespace in the sequence / source files mess up position calculations.
-
-The FastAstream positions are NOT sequence positions.
diff --git a/kmer/libutil/bigQueue.C b/kmer/libutil/bigQueue.C
deleted file mode 100644
index 16b83c8..0000000
--- a/kmer/libutil/bigQueue.C
+++ /dev/null
@@ -1,343 +0,0 @@
-#include "bigQueue.H"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-
-// Kaz Kylheku <kaz at ashi.footprints.net> library.
-#include "kazlib/dict.h"
-#include "kazlib/except.h"
-#include "kazlib/hash.h"
-#include "kazlib/list.h"
-#include "kazlib/sfx.h"
-
-// qsort and kazlib are incombatible. qsort passes a pointer to the data, kaz lib passes
-// the data (which it assumes is a pointer to begin with).
-
-
-void
-bigQueue::_initialize(int (*sortfcn)(const void *a, const void *b),
- bool (*readfcn)(FILE *f, void *a),
- bool (*writfcn)(FILE *f, void *a),
- void (*killfcn)(void *a),
- u32bit objectSize,
- u32bit memoryToUse,
- char *tmppath,
- char *filename) {
- _saveFile = 0L;
- _tmpPath = 0L;
-
- if (filename) {
- _saveFile = new char [strlen(filename) + 1];
- strcpy(_saveFile, filename);
- }
- if (tmppath) {
- _tmpPath = new char [strlen(tmppath) + 1];
- strcpy(_tmpPath, tmppath);
- }
-
- _sortFunction = sortfcn;
- _writFunction = writfcn;
- _readFunction = readfcn;
- _killFunction = killfcn;
-
- _objectSize = objectSize;
- _memoryToUse = memoryToUse;
-
- _maxOpenFiles = getdtablesize() - 8;
- _numTemporaryFiles = 0;
- _numMergeFiles = 0;
-
- _temporaryFiles = new FILE* [_maxOpenFiles];
-
- for (u32bit i=0; i<_maxOpenFiles; i++)
- _temporaryFiles[i] = 0L;
-
- // Open the first temporary file for writing.
- //
- _temporaryFiles[_numTemporaryFiles++] = makeTempFile(_tmpPath);
-
- // XXX: It would be rather convenient if we could get another file
- // handle given an existing handle (no, dup(2) doesn't do that).
- // In particular, we want two file pointers, one for read, one for
- // write.
- //
- //_inputFile = fdopen(dup(fileno(_temporaryFiles[0])), "w+");
-
- _thingBuffer = new u64bit [_objectSize / 8 + 1];
-
- _bufferMax = 0;
- _bufferLen = 0;
- _buffer = 0L;
-
- if (_sortFunction) {
- _bufferMax = (u64bit)memoryToUse * 1024 * 1024 / ((u64bit)sizeof(void *) + objectSize);
- _bufferLen = 0;
- _buffer = new void* [_bufferMax];
- }
-}
-
-
-
-bigQueue::~bigQueue() {
- delete [] _saveFile;
- delete [] _tmpPath;
-
- for (u32bit i=0; i<_numTemporaryFiles; i++)
- fclose(_temporaryFiles[i]);
-
- delete [] _temporaryFiles;
-
- //fclose(_inputFile);
-
- clearBuffer();
-}
-
-
-
-
-
-
-// Add elements to the end of the array.
-void
-bigQueue::add(void *thing) {
-
- if (_buffer == 0L) {
- if (_writFunction)
- (*_writFunction)(_temporaryFiles[_numTemporaryFiles-1], thing);
- else
- fwrite(thing, _objectSize, 1, _temporaryFiles[_numTemporaryFiles-1]);
- } else {
-
- // No space in the buffer? Sort it, write it out and make a new
- // one.
- //
- if (_bufferLen >= _bufferMax) {
- sortAndWriteBuffer();
-
- if (_numTemporaryFiles+1 >= _maxOpenFiles)
- mergeTemporaryFiles();
-
- _temporaryFiles[_numTemporaryFiles++] = makeTempFile(_tmpPath);
- }
-
- _buffer[_bufferLen++] = thing;
- }
-}
-
-
-
-void
-bigQueue::sortAndWriteBuffer(void) {
-
- if (_bufferLen > 0) {
-
- // Sort!
- //
- qsort(_buffer, _bufferLen, sizeof(void *), _sortFunction);
-
- // Write!
- //
- if (_writFunction) {
- for (u32bit i=0; i<_bufferLen; i++)
- (*_writFunction)(_temporaryFiles[_numTemporaryFiles-1], _buffer[i]);
- } else {
- for (u32bit i=0; i<_bufferLen; i++)
- fwrite(_buffer[i], _objectSize, 1, _temporaryFiles[_numTemporaryFiles-1]);
- }
-
- // Flush and rewind the file!
- //
- fflush(_temporaryFiles[_numTemporaryFiles-1]);
- ::rewind(_temporaryFiles[_numTemporaryFiles-1]);
-
- clearBuffer();
- }
-}
-
-
-void
-bigQueue::clearBuffer(void) {
-
- if (_killFunction)
- for (u32bit i=0; i<_bufferLen; i++)
- (*_killFunction)(_buffer[i]);
- else
- for (u32bit i=0; i<_bufferLen; i++)
- free(_buffer[i]);
-
- _bufferLen = 0;
-}
-
-
-void
-bigQueue::mergeTemporaryFiles(void) {
-
- if (_numTemporaryFiles > 1) {
- dict_t *sorted;
- dnode_t *nodes = new dnode_t [_maxOpenFiles];
-
- // To be efficient, we need to maintain a sorted queue of the head
- // elements of each temporary file. A red-black tree would do
- // nicely, eh?
- //
- sorted = dict_create(DICTCOUNT_T_MAX, _sortFunction);
-
- // Grab the first thing off each file, insert it into the dictionary.
- // The 'key' is our chunk of data, and the 'value' is the file number
- // it came from.
- //
- for (u32bit i=0; i<_numTemporaryFiles; i++) {
- if (_temporaryFiles[i]) {
-
- // Rewind all the temporary files. XXXX This is probably done
- // already.
- //
- ::rewind(_temporaryFiles[i]);
-
- void *thing = malloc(_objectSize);
-
- if (_readFunction)
- (*_readFunction)(_temporaryFiles[i], thing);
- else
- fread(thing, _objectSize, 1, _temporaryFiles[i]);
-
- if (feof(_temporaryFiles[i])) {
- fclose(_temporaryFiles[i]);
- _temporaryFiles[i] = 0L;
- } else {
- // initialize the node with the value
- dnode_init(&nodes[i], (void *)(unsigned long)i);
-
- // insert the node into the tree using the key
- dict_insert(sorted, &nodes[i], thing);
- }
- }
- }
-
- FILE *mergeFile = makeTempFile(_tmpPath);
-
- // while there is stuff in the tree
-
- while (dict_isempty(sorted) == 0) {
-
- // pop the head element off, and print it
- dnode_t *head = dict_first(sorted);
-
- // XXX: should be const thing
-
- void *thing = (void *)dnode_getkey(head);
- long fileid = (long)dnode_get(head);
-
- if (_writFunction)
- (*_writFunction)(mergeFile, thing);
- else
- fwrite(thing, _objectSize, 1, mergeFile);
-
- // delete the node from the tree
- dict_delete(sorted, head);
-
- // destroy the thing
- if (_killFunction)
- (*_killFunction)(thing);
- else
- free(thing);
-
- // load the next element from the same file that the head was
- // from (that's stored as the value of the head element)
-
- thing = malloc(_objectSize);
-
- if (_readFunction)
- (*_readFunction)(_temporaryFiles[fileid], thing);
- else
- fread(thing, _objectSize, 1, _temporaryFiles[fileid]);
-
- // if there was a next element in that file, insert it
- // into the tree. if not, close the temporary file.
- //
- if (feof(_temporaryFiles[fileid])) {
- fclose(_temporaryFiles[fileid]);
- _temporaryFiles[fileid] = 0;
- free(thing);
- } else {
- // initialize the node with the value
- dnode_init(&nodes[fileid], (void *)fileid);
-
- // insert the node into the tree using the key
- dict_insert(sorted, &nodes[fileid], thing);
- }
- }
-
- dict_free(sorted);
- delete [] nodes;
-
- _numTemporaryFiles = 1;
- _temporaryFiles[0] = mergeFile;
- }
-
- ::rewind(_temporaryFiles[0]);
-
-#if 0
- fclose(_inputFile);
- errno = 0;
- _inputFile = fdopen(dup(fileno(_temporaryFiles[0])), "w+");
- if (errno)
- fprintf(stderr, "bigQueue::mergeTemporaryFiles()-- _inputFile = fdopen() failed: %s\n", strerror(errno)), exit(1);
-
- ::rewind(_inputFile);
-#endif
-}
-
-
-bool
-bigQueue::next(void) {
-
- if (_readFunction) {
- //(*_readFunction)(_inputFile, _thingBuffer);
- (*_readFunction)(_temporaryFiles[0], _thingBuffer);
- } else {
- //fread(_thingBuffer, _objectSize, 1, _inputFile);
- fread(_thingBuffer, _objectSize, 1, _temporaryFiles[0]);
- }
-
-#if 0
- if (feof(_inputFile))
- return(false);
-#endif
-
- if (feof(_temporaryFiles[0]))
- return(false);
-
- return(true);
-}
-
-
-void*
-bigQueue::get(void) {
- return(_thingBuffer);
-}
-
-void
-bigQueue::rewind(void) {
- //::rewind(_inputFile);
- ::rewind(_temporaryFiles[0]);
- next();
-}
-
-void
-bigQueue::save(char *filepath) {
- fprintf(stderr, "bigQueue::save()-- not implemented.\n");
-}
-
-void
-bigQueue::sort(void) {
- sortAndWriteBuffer();
- mergeTemporaryFiles();
-}
-
-void
-bigQueue::flush(void) {
- fflush(_temporaryFiles[_numTemporaryFiles-1]);
-}
diff --git a/kmer/libutil/bigQueue.H b/kmer/libutil/bigQueue.H
deleted file mode 100644
index 3a7ebb9..0000000
--- a/kmer/libutil/bigQueue.H
+++ /dev/null
@@ -1,150 +0,0 @@
-#ifndef BIGQUEUE_H
-#define BIGQUEUE_H
-
-#include "util++.H"
-
-// A disk-backed list of user-defined objects.
-//
-// At creation time, you can opt to have it sorted, using a
-// user-defined function.
-
-// An list based on a variable length object (let alone a sort!)
-// must use some form of dereferencing scheme. So, if you want to
-// use variable length records, you have to use pointers, and supply
-// functions to do everything (compare, read, write).
-//
-// On the otherhand, it would be quite more convenient (to use) if we
-// used objects (would need copy, compare, read, write).
-//
-// 1) Restrict to void*, fixed block size, functions for compare,
-// destroy. read, write and copy done with fread(), fwrite() and
-// memcpy().
-//
-// 2) Restrict to void*, functions for compare, read, write and
-// destroy. I allocate an array of pointers. Assume shallow copies
-// are ok (qsort will be used). On construct, we need to know the
-// size of the data so we know how many objects to buffer before
-// sorting and writing. It's also possible to use fread() and
-// fwrite().
-//
-// 3) Restrict to objects, operators for copy, compare, read, write,
-// default construct, destroy. I allocate an array of objects.
-//
-// 1 is the easiest to write, 2 and 3 are conceptually the same. 1
-// cannot write out deep data (pointer to string). 2 is a trivial
-// extenstion to 1, and fixes that. 3 is the correct version, but I
-// don't want to deal with streams io. So, 2 it is.
-//
-
-class bigQueue {
-public:
- // Initialize the bigQueue for anonymous storage, with an
- // option to later save the array.
- //
- bigQueue(bool (*readfcn)(FILE *, void *),
- bool (*writfcn)(FILE *, void *),
- void (*killfcn)(void *),
- u32bit objectSize,
- char *tmpPath) {
- _initialize(0L, readfcn, writfcn, killfcn, objectSize, 0, tmpPath, 0L);
- };
-
-
- // Initialize the bigQueue with a file of objects, presumabely from
- // a previous invocation of bigQueue.
- //
- bigQueue(bool (*readfcn)(FILE *, void *),
- bool (*writfcn)(FILE *, void *),
- void (*killfcn)(void *),
- u32bit objectSize,
- char *tmpPath,
- char *filename) {
- _initialize(0L, readfcn, writfcn, killfcn, objectSize, 0, tmpPath, filename);
- };
-
-
- // Initialize the bigQueue for sorting.
- //
- bigQueue(int (*sortfcn)(const void *a, const void *b),
- bool (*readfcn)(FILE *, void *),
- bool (*writfcn)(FILE *, void *),
- void (*killfcn)(void *),
- u32bit objectSize,
- u32bit memoryToUse,
- char *tmpPath) {
- _initialize(sortfcn, readfcn, writfcn, killfcn, objectSize, memoryToUse, tmpPath, 0L);
- };
-
-private:
- void _initialize(int (*sortfcn)(const void *a, const void *b),
- bool (*readfcn)(FILE *f, void *a),
- bool (*writfcn)(FILE *f, void *a),
- void (*killfcn)(void *),
- u32bit objectSize,
- u32bit memoryToUse,
- char *tmppath,
- char *filename);
-
-public:
- ~bigQueue();
-
- // Add elements to the end of the array.
- void add(void *);
-
- // We are designed for streaming access.
- bool next(void);
- void *get(void);
-
- // Rewind to the start. Sortable must be sorted.
- void rewind(void);
-
- // Save the anonymous array into a real file.
- void save(char *filepath);
-
- // Sort the sortable. Flush the flushable.
- void sort(void);
- void flush(void);
-
-private:
- void sortAndWriteBuffer(void);
- void clearBuffer(void);
- void mergeTemporaryFiles(void);
-
- char *_saveFile;
- char *_tmpPath;
-
- int (*_sortFunction)(const void *a, const void *b);
- bool (*_writFunction)(FILE *f, void *a);
- bool (*_readFunction)(FILE *f, void *a);
- void (*_killFunction)(void *a);
-
- u32bit _objectSize;
- u32bit _memoryToUse;
-
- u32bit _maxOpenFiles;
- u32bit _numTemporaryFiles;
- u32bit _numMergeFiles;
-
- // _temporaryFiles is all the opened output files. If we aren't
- // sorting, then only the first one is opened.
- //
- // _inputFile is a dup of the first temporary file. If we are
- // sorting, and you start reading before you sort, then you'll get
- // a very short read.
- //
- FILE **_temporaryFiles;
- FILE *_inputFile;
-
- // Stores things read back from disk, for return to the user.
- // Currently just one, but should be extended to many.
- //
- void *_thingBuffer;
-
- u32bit _bufferMax;
- u32bit _bufferLen;
- void **_buffer;
-};
-
-
-
-#endif // BIGQUEUE_H
diff --git a/kmer/libutil/bitOperations.h b/kmer/libutil/bitOperations.h
deleted file mode 100644
index 0466d38..0000000
--- a/kmer/libutil/bitOperations.h
+++ /dev/null
@@ -1,157 +0,0 @@
-#ifndef BRI_BITS_H
-#define BRI_BITS_H
-
-// For dealing with the bits in bytes.
-
-// I wish I could claim these.
-//
-// Freed, Edwin E. 1983. "Binary Magic Number" Dr. Dobbs Journal
-// Vol. 78 (April) pp. 24-37
-//
-// Supposedly tells us how to reverse the bits in a word, count the number
-// of set bits in a words and more.
-//
-// A bit of verbage on counting the number of set bits. The naive way
-// is to loop and shift:
-//
-// u32bit r = u32bitZERO;
-// while (x) {
-// r++;
-// x >>= 1;
-// }
-// return(r);
-//
-// http://remus.rutgers.edu/~rhoads/Code/bitcount3.c has an optimized
-// method:
-//
-// x -= (0xaaaaaaaa & x) >> 1;
-// x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
-// x += x >> 4;
-// x &= 0x0f0f0f0f;
-// x += x >> 8;
-// x += x >> 16;
-// x &= 0x000000ff;
-// return(x);
-//
-// No loops!
-//
-// Freed's methods are easier to understand, and just as fast.
-//
-// Using our bit counting routines, Ross Lippert suggested a nice
-// way of computing log2 -- use log2 shifts to fill up the lower
-// bits, then count bits. See logBaseTwo*()
-//
-
-
-inline
-u32bit
-reverseBits32(u32bit x) {
- x = ((x >> 1) & u32bitNUMBER(0x55555555)) | ((x << 1) & u32bitNUMBER(0xaaaaaaaa));
- x = ((x >> 2) & u32bitNUMBER(0x33333333)) | ((x << 2) & u32bitNUMBER(0xcccccccc));
- x = ((x >> 4) & u32bitNUMBER(0x0f0f0f0f)) | ((x << 4) & u32bitNUMBER(0xf0f0f0f0));
- x = ((x >> 8) & u32bitNUMBER(0x00ff00ff)) | ((x << 8) & u32bitNUMBER(0xff00ff00));
- x = ((x >> 16) & u32bitNUMBER(0x0000ffff)) | ((x << 16) & u32bitNUMBER(0xffff0000));
- return(x);
-}
-
-inline
-u64bit
-reverseBits64(u64bit x) {
- x = ((x >> 1) & u64bitNUMBER(0x5555555555555555)) | ((x << 1) & u64bitNUMBER(0xaaaaaaaaaaaaaaaa));
- x = ((x >> 2) & u64bitNUMBER(0x3333333333333333)) | ((x << 2) & u64bitNUMBER(0xcccccccccccccccc));
- x = ((x >> 4) & u64bitNUMBER(0x0f0f0f0f0f0f0f0f)) | ((x << 4) & u64bitNUMBER(0xf0f0f0f0f0f0f0f0));
- x = ((x >> 8) & u64bitNUMBER(0x00ff00ff00ff00ff)) | ((x << 8) & u64bitNUMBER(0xff00ff00ff00ff00));
- x = ((x >> 16) & u64bitNUMBER(0x0000ffff0000ffff)) | ((x << 16) & u64bitNUMBER(0xffff0000ffff0000));
- x = ((x >> 32) & u64bitNUMBER(0x00000000ffffffff)) | ((x << 32) & u64bitNUMBER(0xffffffff00000000));
- return(x);
-}
-
-
-#if (__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)
-#define PREFETCH(x) __builtin_prefetch((x), 0, 0)
-#else
-#define PREFETCH(x)
-#endif
-
-
-
-
-// Amazingingly, this is slower. From what I can google, the builtin
-// is using the 2^16 lookup table method - so a 64-bit popcount does
-// 4 lookups in the table and sums. Bad cache performance in codes
-// that already have bad cache performance, I'd guess.
-//
-//#if (__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)
-//#define BUILTIN_POPCOUNT
-//#endif
-
-#ifdef BUILTIN_POPCOUNT
-
-inline
-u32bit
-countNumberOfSetBits32(u32bit x) {
- return(__builtin_popcount(x));
-}
-
-inline
-u64bit
-countNumberOfSetBits64(u64bit x) {
- return(__builtin_popcountll(x));
-}
-
-#else
-
-inline
-u32bit
-countNumberOfSetBits32(u32bit x) {
- x = ((x >> 1) & u32bitNUMBER(0x55555555)) + (x & u32bitNUMBER(0x55555555));
- x = ((x >> 2) & u32bitNUMBER(0x33333333)) + (x & u32bitNUMBER(0x33333333));
- x = ((x >> 4) & u32bitNUMBER(0x0f0f0f0f)) + (x & u32bitNUMBER(0x0f0f0f0f));
- x = ((x >> 8) & u32bitNUMBER(0x00ff00ff)) + (x & u32bitNUMBER(0x00ff00ff));
- x = ((x >> 16) & u32bitNUMBER(0x0000ffff)) + (x & u32bitNUMBER(0x0000ffff));
- return(x);
-}
-
-inline
-u64bit
-countNumberOfSetBits64(u64bit x) {
- x = ((x >> 1) & u64bitNUMBER(0x5555555555555555)) + (x & u64bitNUMBER(0x5555555555555555));
- x = ((x >> 2) & u64bitNUMBER(0x3333333333333333)) + (x & u64bitNUMBER(0x3333333333333333));
- x = ((x >> 4) & u64bitNUMBER(0x0f0f0f0f0f0f0f0f)) + (x & u64bitNUMBER(0x0f0f0f0f0f0f0f0f));
- x = ((x >> 8) & u64bitNUMBER(0x00ff00ff00ff00ff)) + (x & u64bitNUMBER(0x00ff00ff00ff00ff));
- x = ((x >> 16) & u64bitNUMBER(0x0000ffff0000ffff)) + (x & u64bitNUMBER(0x0000ffff0000ffff));
- x = ((x >> 32) & u64bitNUMBER(0x00000000ffffffff)) + (x & u64bitNUMBER(0x00000000ffffffff));
- return(x);
-}
-
-#endif
-
-
-
-inline
-u32bit
-logBaseTwo32(u32bit x) {
- x |= x >> 1;
- x |= x >> 2;
- x |= x >> 4;
- x |= x >> 8;
- x |= x >> 16;
- return(countNumberOfSetBits32(x));
-}
-
-inline
-u64bit
-logBaseTwo64(u64bit x) {
- x |= x >> 1;
- x |= x >> 2;
- x |= x >> 4;
- x |= x >> 8;
- x |= x >> 16;
- x |= x >> 32;
- return(countNumberOfSetBits64(x));
-}
-
-
-
-
-#endif // BRI_BITS_H
diff --git a/kmer/libutil/bitPackedArray.C b/kmer/libutil/bitPackedArray.C
deleted file mode 100644
index df54837..0000000
--- a/kmer/libutil/bitPackedArray.C
+++ /dev/null
@@ -1,100 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <errno.h>
-#include <string.h>
-#include <strings.h>
-
-#include "util++.H"
-
-bitPackedArray::bitPackedArray(u32bit valueWidth, u32bit segmentSize) {
- _valueWidth = valueWidth;
- _segmentSize = segmentSize;
- _nextElement = 0;
- _valuesPerSegment = (u64bit)_segmentSize * 1024 * 8 / (u64bit)_valueWidth;
-
- _numSegments = 0;
- _maxSegments = 16;
- _segments = new u64bit * [_maxSegments];
-}
-
-
-bitPackedArray::~bitPackedArray() {
- for (u32bit i=0; i<_numSegments; i++)
- delete [] _segments[i];
- delete [] _segments;
-}
-
-
-u64bit
-bitPackedArray::get(u64bit idx) {
- u64bit s = idx / _valuesPerSegment;
- u64bit p = _valueWidth * (idx % _valuesPerSegment);
-
- if (idx >= _nextElement) {
- fprintf(stderr, "bitPackedArray::get()-- element index "u64bitFMT" is out of range, only "u64bitFMT" elements.\n",
- idx, _nextElement-1);
- return(0xdeadbeefdeadbeefULL);
- }
-
- return(getDecodedValue(_segments[s], p, _valueWidth));
-}
-
-
-void
-bitPackedArray::set(u64bit idx, u64bit val) {
- u64bit s = idx / _valuesPerSegment;
- u64bit p = _valueWidth * (idx % _valuesPerSegment);
-
- //fprintf(stderr, "s="u64bitFMT" p="u64bitFMT" segments="u64bitFMT"/"u64bitFMT"\n", s, p, _numSegments, _maxSegments);
-
- if (idx >= _nextElement)
- _nextElement = idx+1;
-
- if (s >= _maxSegments) {
- _maxSegments = s + 16;
- u64bit **S = new u64bit * [_maxSegments];
- for (u32bit i=0; i<_numSegments; i++)
- S[i] = _segments[i];
- delete [] _segments;
- _segments = S;
- }
-
- while (_numSegments <= s)
- _segments[_numSegments++] = new u64bit [_segmentSize * 1024 / 8];
-
- setDecodedValue(_segments[s], p, _valueWidth, val);
-}
-
-
-void
-bitPackedArray::clear(void) {
- for (u32bit s=0; s<_numSegments; s++)
- bzero(_segments[s], _segmentSize * 1024);
-}
-
-
-////////////////////////////////////////
-
-bitArray::bitArray(u32bit segmentSize) {
- _segmentSize = segmentSize;
- _valuesPerSegment = (u64bit)_segmentSize * 1024 * 8;
-
- _numSegments = 0;
- _maxSegments = 16;
- _segments = new u64bit * [_maxSegments];
-}
-
-
-bitArray::~bitArray() {
- for (u32bit i=0; i<_numSegments; i++)
- delete [] _segments[i];
- delete [] _segments;
-}
-
-
-void
-bitArray::clear(void) {
- for (u32bit s=0; s<_numSegments; s++)
- bzero(_segments[s], _segmentSize * 1024);
-}
diff --git a/kmer/libutil/bitPackedArray.H b/kmer/libutil/bitPackedArray.H
deleted file mode 100644
index 896c334..0000000
--- a/kmer/libutil/bitPackedArray.H
+++ /dev/null
@@ -1,318 +0,0 @@
-#ifndef BITPACKEDARRAY_H
-#define BITPACKEDARRAY_H
-
-#undef DEBUG_BPH_ADD
-#undef DEBUG_BPH_GET
-
-////////////////////////////////////////
-//
-// bitPackedArray
-//
-// implements an integer array using bit-widths less than word-sizes,
-// e.g., a memory efficient way to store 23 bit numbers. Numbers may
-// be up to 64 bits wide.
-//
-// The array is variable length, and it is implemented as an array,
-// not a list or tree -- accessing element 1,000,000 will allocate
-// elements 0 through 999,999.
-//
-class bitPackedArray {
-public:
-
- // Create a bitpacked array with elements of width 'width' using
- // 'segmentSize' KB per segment. If you know your array is going
- // to be much bigger or smaller, crank this value.
- //
- bitPackedArray(u32bit valueWidth, u32bit segmentSize = 1024);
- ~bitPackedArray();
-
- // No array operator is provided, because we cannot return a
- // reference to a value that is split across two words (or even a
- // reference to a value that is not bit aligned in the word).
- //
- u64bit get(u64bit idx);
- void set(u64bit idx, u64bit val);
-
- // Clear the array. Since the array is variable sized, you must add
- // things to a new array before clearing it.
- void clear(void);
-
-private:
- u32bit _valueWidth;
- u32bit _segmentSize;
- u64bit _nextElement; // the first invalid element
- u64bit _valuesPerSegment;
-
- u64bit _numSegments;
- u64bit _maxSegments;
- u64bit **_segments;
-};
-
-
-// An array of bits. Exactly the same as the bitPackedArray, but
-// optimized for width=1.
-//
-class bitArray {
-public:
-
- bitArray(u32bit segmentSize = 1024);
- ~bitArray();
-
- u64bit get(u64bit idx);
-
- u64bit getAndSet(u64bit idx);
-
- void set(u64bit idx);
- void clr(u64bit idx);
-
- void clear(void);
-
-private:
- void resize(u64bit s);
-
- u32bit _segmentSize;
- u64bit _valuesPerSegment;
-
- u64bit _numSegments;
- u64bit _maxSegments;
- u64bit **_segments;
-};
-
-
-// Uses the bitPackedArray to implement a heap. The bitPackedArray is dynamically sized,
-// so this can be too.
-//
-class bitPackedHeap {
-public:
- bitPackedHeap(u32bit width, u64bit size=16) {
- _array = new bitPackedArray(width, size);
- _array->set(0, 0);
- _lastVal = 0;
- };
-
- ~bitPackedHeap() {
- delete _array;
- };
-
- u64bit get(void) {
- u64bit biggestVal = ~u64bitZERO;
-
- if (_lastVal == 0)
- return(biggestVal);
-
- biggestVal = _array->get(0);
- _lastVal--;
-
- if (_lastVal == 0)
- return(biggestVal);
-
- u64bit t = _array->get(_lastVal);
-
- _array->set(0, t);
-
- u64bit pidx = 0;
- u64bit pval = t;
- u64bit cidx = 1;
- u64bit cval = 0; // set below
-
- while (cidx < _lastVal) {
- // Set cval here, so we can first test if cidx is in range.
- cval = _array->get(cidx);
-
- // Pick the smallest of the two kids
- if (cidx+1 < _lastVal) {
- t = _array->get(cidx+1);
- if (cval > t) {
- cidx++;
- cval = t;
- }
- }
-
-#ifdef DEBUG_BPH_GET
- fprintf(stderr, "test c="u64bitFMT" and p="u64bitFMT" lastVal="u64bitFMT"\n",
- cidx, pidx, _lastVal);
- fprintf(stderr, "test c="u64bitFMT"="u64bitFMT"\n",
- cidx, cval);
- fprintf(stderr, "test p="u64bitFMT"="u64bitFMT"\n",
- pidx, pval);
- fprintf(stderr, "test c="u64bitFMT"="u64bitFMT" and p="u64bitFMT"="u64bitFMT"\n",
- cidx, cval, pidx, pval);
-#endif
-
- if (cval < pval) {
-
-#ifdef DEBUG_BPH_GET
- fprintf(stderr, "swap c="u64bitFMT"="u64bitFMT" and p="u64bitFMT"="u64bitFMT"\n",
- cidx, cval, pidx, pval);
-#endif
-
- // Swap p and c
- _array->set(pidx, cval);
- _array->set(cidx, pval);
-
- // Move down the tree -- pval doesn't change, we moved it into cidx!
- pidx = cidx;
- cidx = cidx * 2 + 1;
- } else {
- cidx = _lastVal;
- }
- }
-
- return(biggestVal);
- };
-
- void add(u64bit value) {
- u64bit cidx = _lastVal;
- u64bit cval = value;
- u64bit pidx = 0;
- u64bit pval = 0;
- bool more = false;
-
-#ifdef DEBUG_BPH_ADD
- fprintf(stderr, "add c="u64bitFMT"="u64bitFMT" -- lastVal="u64bitFMT"\n",
- cidx, cval, _lastVal);
-#endif
-
- _array->set(cidx, cval);
-
- if (cidx > 0)
- more = true;
-
- while (more) {
- pidx = (cidx-1) / 2;
-
-#ifdef DEBUG_BPH_ADD
- fprintf(stderr, "more c="u64bitFMT" and p="u64bitFMT"\n", cidx, pidx);
-#endif
-
- pval = _array->get(pidx);
-
-#ifdef DEBUG_BPH_ADD
- fprintf(stderr, "test c="u64bitFMT"="u64bitFMT" and p="u64bitFMT"="u64bitFMT"\n",
- cidx, cval, pidx, pval);
-#endif
-
- if (pval > cval) {
-
-#ifdef DEBUG_BPH_ADD
- fprintf(stderr, "swap c="u64bitFMT"="u64bitFMT" and p="u64bitFMT"="u64bitFMT"\n",
- cidx, cval, pidx, pval);
-#endif
-
- // Swap p and c
- _array->set(cidx, pval);
- _array->set(pidx, cval);
-
- // Move up the tree -- cval doesn't change, we moved it into pidx!
- cidx = pidx;
- } else {
- more = false;
- }
- if (cidx == 0)
- more = false;
- }
-
- _lastVal++;
-
- //dump();
- };
-
- void dump(void) {
- for (u32bit i=0; i<_lastVal; i++)
- fprintf(stderr, "HEAP["u32bitFMT"]="u64bitFMT"\n", i, _array->get(i));
- }
-
- void clear(void) {
- _array->clear();
- _lastVal = 0;
- };
-
-private:
- bitPackedArray *_array;
- u64bit _lastVal;
-};
-
-
-
-inline
-u64bit
-bitArray::get(u64bit idx) {
- u64bit s = idx / _valuesPerSegment;
- u64bit p = idx % _valuesPerSegment;
-
- u64bit wrd = (p >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (p ) & 0x000000000000003fllu;
-
- return((_segments[s][wrd] >> bit) & 0x0000000000000001llu);
-}
-
-
-inline
-void
-bitArray::resize(u64bit s) {
-
- if (s < _numSegments)
- return;
-
- if (s > _maxSegments) {
- _maxSegments = s + 16;
- u64bit **S = new u64bit * [_maxSegments];
- for (u32bit i=0; i<_numSegments; i++)
- S[i] = _segments[i];
- delete [] _segments;
- _segments = S;
- }
-
- while (_numSegments <= s)
- _segments[_numSegments++] = new u64bit [_segmentSize * 1024 / 8];
-}
-
-
-inline
-u64bit
-bitArray::getAndSet(u64bit idx) {
- u64bit s = idx / _valuesPerSegment;
- u64bit p = idx % _valuesPerSegment;
-
- u64bit wrd = (p >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (p ) & 0x000000000000003fllu;
-
- u64bit ret = (_segments[s][wrd] >> bit) & 0x0000000000000001llu;
-
- _segments[s][wrd] |= u64bitONE << bit;
-
- return(ret);
-}
-
-
-inline
-void
-bitArray::set(u64bit idx) {
- u64bit s = idx / _valuesPerSegment;
- u64bit p = idx % _valuesPerSegment;
-
- resize(s);
-
- u64bit wrd = (p >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (p ) & 0x000000000000003fllu;
-
- _segments[s][wrd] |= u64bitONE << bit;
-}
-
-
-inline
-void
-bitArray::clr(u64bit idx) {
- u64bit s = idx / _valuesPerSegment;
- u64bit p = idx % _valuesPerSegment;
-
- resize(s);
-
- u64bit wrd = (p >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (p ) & 0x000000000000003fllu;
-
- _segments[s][wrd] &= ~(0x0000000000000001llu << bit);
-}
-
-
-#endif // BITPACKEDARRAY_H
diff --git a/kmer/libutil/bitPackedFile.C b/kmer/libutil/bitPackedFile.C
deleted file mode 100644
index 4d2eec3..0000000
--- a/kmer/libutil/bitPackedFile.C
+++ /dev/null
@@ -1,473 +0,0 @@
-#include "util++.H"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <fcntl.h>
-
-// N.B. any read() / write() pair (either order) must have a seek (or
-// a fflush) in between.
-
-bitPackedFile::bitPackedFile(char const *name, u64bit offset, bool forceTruncate) {
-
- _file = 0;
- _name = new char [strlen(name) + 1];
- strcpy(_name, name);
-
-#ifdef WITH_BZIP2
- _bzFILE = 0L;
- _bzerr = 0;
- _bzfile = 0L;
-#endif
-
- _bfrmax = 1048576 / 8;
- _bfr = new u64bit [_bfrmax];
- _pos = u64bitZERO;
- _bit = u64bitZERO;
-
- memset(_bfr, 0, sizeof(u64bit) * _bfrmax);
-
- _inCore = false;
- _bfrDirty = false;
- _forceFirstLoad = false;
- _isReadOnly = false;
- _isBzip2 = false;
-
- stat_seekInside = u64bitZERO;
- stat_seekOutside = u64bitZERO;
- stat_dirtyFlushes = u64bitZERO;
-
- file_offset = 0;
- endianess_offset = 0;
- endianess_flipped = false;
-
-
- // Try to open the original name -- we don't support compressed
- // files for rewrite. We just fail with a can't open message.
- //
- // To get read/write and create we have to use open(2), as mode
- // "r+" of fopen(3) will not create. (Yes, but w+ does, sigh.)
- //
- if (forceTruncate) {
- errno = 0;
- _file = open(_name,
- O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno)
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open and truncate '%s': %s\n",
- _name, strerror(errno)), exit(1);
- } else if (fileExists(_name)) {
- errno = 0;
- _file = open(_name,
- O_RDONLY | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno)
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open '%s': %s\n",
- _name, strerror(errno)), exit(1);
- _isReadOnly = true;
- } else {
- errno = 0;
- _file = open(_name,
- O_RDWR | O_CREAT | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno)
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open '%s': %s\n",
- _name, strerror(errno)), exit(1);
- }
-
- // Move to the correct position in the file.
- //
- file_offset = offset;
- if (file_offset > 0)
- lseek(_file, file_offset, SEEK_SET);
-
- // Deal with endianess. We write out some bytes (or read back some bytes) to the start of
- // the file, and then hide them from the user.
- //
- endianess_offset = 32 + file_offset;
- endianess_flipped = false;
-
- char t[16] = { 'b', 'i', 't', 'P', 'a', 'c', 'k', 'e', 'd', 'F', 'i', 'l', 'e', 0, 0, 1 };
- char c[16] = { 0 };
- u64bit at = u64bitNUMBER(0xdeadbeeffeeddada );
- u64bit bt = u64bitNUMBER(0x0abeadedbabed8f8);
- u64bit ac = u64bitNUMBER(0);
- u64bit bc = u64bitNUMBER(0);
- size_t nr = 0;
-
- errno = 0;
- nr += read(_file, c, sizeof(char) * 16);
- nr += read(_file, &ac, sizeof(u64bit));
- nr += read(_file, &bc, sizeof(u64bit));
-
- if (nr == 0) {
- // Empty file! Write the magic number and our endianess check.
-
- errno = 0;
- write(_file, t, sizeof(char) * 16);
- write(_file, &at, sizeof(u64bit));
- write(_file, &bt, sizeof(u64bit));
- if (errno)
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' failed to write the header: %s\n", _name, strerror(errno)), exit(1);
-
- return;
- }
-
-
- if ((c[0] == 'B') && (c[1] == 'Z') && (c[2] == 'h')) {
-#ifdef WITH_BZIP2
- // Looks like a bzip2 file!
-
- errno = 0;
- _bzFILE = fopen(_name, "r");
- if (errno) {
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open bzip2 file '%s'\n", _name);
- exit(1);
- }
-
- _bzerr = 0;
- _bzfile = BZ2_bzReadOpen(&_bzerr, _bzFILE, 0, 0, 0L, 0);
- if ((_bzfile == 0L) || (_bzerr != BZ_OK)) {
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to init bzip2 file '%s'\n", _name);
- exit(1);
- }
-
- BZ2_bzRead(&_bzerr, _bzfile, c, sizeof(char) * 16);
- BZ2_bzRead(&_bzerr, _bzfile, &ac, sizeof(u64bit));
- BZ2_bzRead(&_bzerr, _bzfile, &bc, sizeof(u64bit));
-
- // XXX should check bzerr!
-
- _isReadOnly = true;
- _isBzip2 = true;
-#else
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' looks like a bzip2 file, but bzip2 support not available!\n", _name);
- exit(1);
-#endif
- }
-
-
- // Check the magic number, decide on an endianess to use.
- //
- if (strncmp(t, c, 16) == 0) {
- if ((at == ac) && (bt == bc)) {
- endianess_flipped = false;
- } else if ((at == u64bitSwap(ac)) && (bt == u64bitSwap(bc))) {
- endianess_flipped = true;
- } else {
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' looked like a bitPackedFile, but failed the endianess check, not opened.\n", _name);
- exit(1);
- }
- } else {
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' doesn't appear to be a bitPackedFile, not opened.\n", _name);
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- found ");
- for (u32bit i=0; i<16; i++)
- fprintf(stderr, "%c", isascii(c[i]) ? c[i] : '.');
- fprintf(stderr, " at position "u64bitHEX"\n", file_offset);
- exit(1);
- }
-
- _forceFirstLoad = true;
- seek(0);
-}
-
-
-bitPackedFile::~bitPackedFile() {
- flushDirty();
- delete [] _bfr;
- delete [] _name;
- close(_file);
-
-#ifdef WITH_BZIP2
- if (_bzFILE)
- fclose(_bzFILE);
-
- if (_bzfile)
- BZ2_bzReadClose(&_bzerr, _bzfile);
-#endif
-}
-
-
-
-// If the page is dirty, flush it to disk
-//
-void
-bitPackedFile::flushDirty(void) {
-
- if (_bfrDirty == false)
- return;
-
- if (_isReadOnly) {
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' is readonly, but is dirty!\n", _name);
- exit(1);
- }
-
- stat_dirtyFlushes++;
-
- errno = 0;
- lseek(_file, _pos * sizeof(u64bit) + endianess_offset, SEEK_SET);
- if (errno) {
- fprintf(stderr, "bitPackedFile::seek()-- '%s' failed: %s\n",
- _name, strerror(errno));
- exit(1);
- }
-
- // If we need to, flip all the words we are going to write
- //
- if (endianess_flipped)
- for (u32bit i=0; i<_bfrmax; i++)
- _bfr[i] = u64bitSwap(_bfr[i]);
-
- // We should only write bits up to _bit, the position we are
- // currently at. However, we don't know if the block is being
- // flushed because we're totally finished with it, or because we
- // are moving on to the next block. If we're done with it, we
- // want to flush the word that contains _bit, and if we're moving
- // on to the next one, we'll flush that word again. So, in
- // either case, we flush the word that contains _bit.
- //
- errno = 0;
- write(_file, _bfr, sizeof(u64bit) * _bfrmax);
- if (errno) {
- fprintf(stderr, "bitPackedFile::write()-- '%s' failed: %s\n",
- _name, strerror(errno));
- exit(1);
- }
-
- // And then flip them back
- //
- if (endianess_flipped)
- for (u32bit i=0; i<_bfrmax; i++)
- _bfr[i] = u64bitSwap(_bfr[i]);
-
- _bfrDirty = false;
-}
-
-
-
-void
-bitPackedFile::seekBzip2(u64bit bitpos) {
-
-#ifdef WITH_BZIP2
- // All we can do here is check that bitpos is
- // a) in our current buffer
- // b) would be in the next buffer once we read it
-
- u64bit newpos = bitpos >> 6;
-
- if (_pos + _bfrmax < newpos) {
- // nope, not in the buffer -- we could probably handle this by just reading and
- // discarding from the file until we get to the correct bitpos.
- fprintf(stderr, "bitPackedFile::seekBzip2()-- '%s' seek was not contiguous!\n", _name);
- exit(1);
- }
-
- // Copy the remaining bits of the current buffer to the start. Or
- // not, if this is the first load.
-
- u64bit lastpos = _bit >> 6; // The word we are currently in
- u64bit lastlen = (_bfrmax - lastpos); // The number of words left in the buffer
-
- if (_forceFirstLoad == true) {
- lastpos = 0;
- lastlen = 0;
- } else {
- memcpy(_bfr, _bfr + lastpos, sizeof(u64bit) * lastlen);
- }
-
- // Update _bit and _pos -- lastlen is now the first invalid word
- //
- _bit = bitpos & 0x3f; // 64 * lastlen;
- _pos = bitpos >> 6;
-
- // Fill the buffer
-
- size_t wordsread = 0;
-
- if (_bzfile) {
- _bzerr = 0;
- wordsread = BZ2_bzRead(&_bzerr, _bzfile, _bfr + lastlen, sizeof(u64bit) * (_bfrmax - lastlen));
- if (_bzerr == BZ_STREAM_END) {
- //fprintf(stderr, "bitPackedFile::seekBzip2() file ended.\n");
- BZ2_bzReadClose(&_bzerr, _bzfile);
- fclose(_bzFILE);
- _bzfile = 0L;
- _bzFILE = 0L;
- } else if (_bzerr != BZ_OK) {
- fprintf(stderr, "bitPackedFile::seekBzip2() '%s' read failed.\n", _name);
- exit(1);
- }
- }
-
- //fprintf(stderr, "Filled buffer with %d words!\n", wordsread);
-
- // Adjust to make wordsread be the index of the last word we actually read.
- //
- wordsread += lastlen;
-
- // Flip all the words we just read, if needed
- //
- if (endianess_flipped)
- for (u32bit i=lastlen; i<wordsread; i++)
- _bfr[i] = u64bitSwap(_bfr[i]);
-
- // Clear any words that we didn't read (supposedly, because we hit
- // EOF).
- //
- while (wordsread < _bfrmax)
- _bfr[wordsread++] = u64bitZERO;
-#else
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s'\n", _name);
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- bzip2 support not present, but still tried to read it??\n");
- exit(1);
-#endif
-}
-
-
-
-void
-bitPackedFile::seekNormal(u64bit bitpos) {
-
- if (_inCore) {
- fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' is in core, but still needed to seek??\n",
- _name);
- exit(1);
- }
-
- // Somewhat of a gross hack to allow sequential access backwards.
- //
- // If the new position (bitpos >> 6) is just before the old
- // position (_pos), assume that we are being accessed iteratively
- // backwards and load a full buffer so that the position we want to
- // access is at the end.
- //
- // Easy to think of bone-headed ways to break this (e.g., seek to
- // the second element in a structure, access the first, then access
- // the third). Not so easy to think of a logical reason someone
- // would want to do that.
- //
- if (((bitpos >> 6) < _pos) && (_pos <= (bitpos >> 6) + 32)) {
- _pos = bitpos >> 6;
- if (_pos > _bfrmax)
- _pos = _pos - _bfrmax + 32;
- else
- _pos = 0;
- } else {
- _pos = bitpos >> 6;
- }
-
- _bit = bitpos - (_pos << 6);
-
-
- errno = 0;
- lseek(_file, _pos * 8 + endianess_offset, SEEK_SET);
- if (errno) {
- fprintf(stderr, "bitPackedFile::seekNormal() '%s' seek to pos="u64bitFMT" failed: %s\n",
- _name,
- _pos * 8 + endianess_offset, strerror(errno));
- exit(1);
- }
-
- errno = 0;
- size_t wordsread = read(_file, _bfr, sizeof(u64bit) * _bfrmax);
- if (errno) {
- fprintf(stderr, "bitPackedFile::seekNormal() '%s' read of "u64bitFMT" bytes failed': %s\n",
- _name,
- sizeof(u64bit) * _bfrmax,
- strerror(errno));
- exit(1);
- }
-
- // Flip all the words we just read, if needed
- //
- if (endianess_flipped)
- for (u32bit i=0; i<wordsread; i++)
- _bfr[i] = u64bitSwap(_bfr[i]);
-
- // Clear any words that we didn't read (supposedly, because we hit
- // EOF).
- //
- while (wordsread < _bfrmax)
- _bfr[wordsread++] = u64bitZERO;
-}
-
-
-
-
-
-// Seeks to bitposition pos in the file, reads in a new block.
-//
-void
-bitPackedFile::seek(u64bit bitpos) {
-
- // If we are seeking to somewhere in the current block, don't do a
- // real seek, just move our position within the block.
- //
- if (_forceFirstLoad == false) {
- u64bit np = bitpos >> 6;
-
- if ((_pos <= np) && (np <= _pos + _bfrmax - 32)) {
- _bit = bitpos - (_pos << 6);
- stat_seekInside++;
- //fprintf(stderr, "SEEK INSIDE to _bit="u64bitFMT"\n", _bit);
- return;
- }
- }
-
- if (_inCore) {
- fprintf(stderr, "bitPackedFile::seek()-- file '%s' is in core, but still needed to seek??\n",
- _name);
- exit(1);
- }
-
- stat_seekOutside++;
-
- flushDirty();
-
- if (_isBzip2)
- seekBzip2(bitpos);
- else
- seekNormal(bitpos);
-
- _forceFirstLoad = false;
-
- //fprintf(stderr, "SEEK OUTSIDE to _pos="u64bitFMT" _bit="u64bitFMT"\n", _pos, _bit);
-}
-
-
-
-
-u64bit
-bitPackedFile::loadInCore(void) {
- struct stat sb;
-
- // Convert this disk-based, read/write bitPackedFile to memory-based read-only.
-
- flushDirty();
-
- fstat(_file, &sb);
-
- // The extra 1024 words is to keep seek() from attempting to grab
- // the next block (there isn't a next block, we've got it all!)
- // when we're near the end of this block. We just make the block
- // a little bigger than it really is.
-
- delete [] _bfr;
-
- _bfrmax = sb.st_size / 8 + 1024;
- _bfr = new u64bit [_bfrmax];
- _pos = 0;
- _bit = 0;
-
- // Tada! All we need to do now is load the block!
-
- _forceFirstLoad = true;
-
- seek(0);
-
- _inCore = true;
-
- return(_bfrmax * 8);
-}
diff --git a/kmer/libutil/bitPackedFile.H b/kmer/libutil/bitPackedFile.H
deleted file mode 100644
index 2cf5ef1..0000000
--- a/kmer/libutil/bitPackedFile.H
+++ /dev/null
@@ -1,127 +0,0 @@
-#ifndef BITPACKEDFILE_H
-#define BITPACKEDFILE_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include "util.h"
-
-//#define WITH_BZIP2
-
-#ifdef WITH_BZIP2
-#include <bzlib.h>
-#endif
-
-class bitPackedFile {
-public:
- bitPackedFile(char const *name, u64bit offset=0, bool forceTruncate=false);
- ~bitPackedFile();
-
- u64bit getBits(u32bit size);
- u64bit getNumber(void);
-
- void putBits(u64bit bits, u32bit size);
- void putNumber(u64bit val);
-
- u64bit tell(void) { return((_pos << 6) + _bit); };
- void seek(u64bit pos);
-
- u64bit loadInCore(void);
-
- void showStats(FILE *f) {
- fprintf(f, "inside: "u64bitFMT" outside: "u64bitFMT"\n", stat_seekInside, stat_seekOutside);
- fflush(f);
- };
-private:
-
- // Ensure that the buffer has enough space for any future
- // operation. This constant, currently 31 bytes, must be strictly
- // less than the constant used in deciding if seek() is moving
- // forward or backwards.
- //
- void sync(void) {
- if (((_bit >> 6) + 31) >= _bfrmax)
- seek((_pos << 6) + _bit);
- };
-
- void flushDirty(void);
- void seekBzip2(u64bit bitpos);
- void seekNormal(u64bit bitpos);
-
- int _file;
- char *_name;
-
-#ifdef WITH_BZIP2
- FILE *_bzFILE;
- int _bzerr;
- BZFILE *_bzfile;
-#endif
-
- u64bit _bfrmax; // Number of words in the buffer
- u64bit *_bfr; // A chunk of the bitPackedFile in core
- u64bit _pos; // The location this chunk is from (in words)
- u64bit _bit; // The bit position we are modifying relative to _pos
-
- bool _inCore;
- bool _bfrDirty;
- bool _forceFirstLoad;
- bool _isReadOnly;
- bool _isBzip2;
-
- // For collecting statistics on our usage
- //
- u64bit stat_seekInside;
- u64bit stat_seekOutside;
- u64bit stat_dirtyFlushes;
-
- // For converting between hardware of different endianess.
- //
- u64bit file_offset;
- u64bit endianess_offset;
- bool endianess_flipped;
-};
-
-
-inline
-u64bit
-bitPackedFile::getBits(u32bit siz) {
- sync();
- u64bit ret = getDecodedValue(_bfr, _bit, siz);
- _bit += siz;
- return(ret);
-}
-
-inline
-u64bit
-bitPackedFile::getNumber(void) {
- sync();
- u64bit siz = 0;
- u64bit ret = getFibonacciEncodedNumber(_bfr, _bit, &siz);
- _bit += siz;
- return(ret);
-}
-
-
-inline
-void
-bitPackedFile::putBits(u64bit bits, u32bit siz) {
- assert(_isReadOnly == false);
- sync();
- setDecodedValue(_bfr, _bit, siz, bits);
- _bit += siz;
- _bfrDirty = true;
-}
-
-inline
-void
-bitPackedFile::putNumber(u64bit val) {
- assert(_isReadOnly == false);
- sync();
- u64bit siz = 0;
- setFibonacciEncodedNumber(_bfr, _bit, &siz, val);
- _bit += siz;
- _bfrDirty = true;
-}
-
-
-#endif // BITPACKEDFILE_H
diff --git a/kmer/libutil/bitPacking.h b/kmer/libutil/bitPacking.h
deleted file mode 100644
index 59cd272..0000000
--- a/kmer/libutil/bitPacking.h
+++ /dev/null
@@ -1,510 +0,0 @@
-#ifndef BRI_BITPACKING_H
-#define BRI_BITPACKING_H
-
-#include <stdio.h>
-#include <assert.h>
-
-// Routines used for stuffing bits into a word array.
-
-// Define this to enable testing that the width of the data element
-// is greater than zero. The u64bitMASK() macro (bri.h) does not
-// generate a mask for 0. Compiler warnings are issued, because you
-// shouldn't use this in production code.
-//
-//#define CHECK_WIDTH
-
-// As CHECK_WIDTH is kind of expensive, we'll warn.
-#ifdef CHECK_WIDTH
-#warning libutil/bitPacking.h defined CHECK_WIDTH
-#endif
-
-// Returns 'siz' bits from the stream based at 'ptr' and currently at
-// location 'pos'. The position of the stream is not changed.
-//
-// Retrieves a collection of values; the number of bits advanced in
-// the stream is returned.
-//
-// Copies the lowest 'siz' bits in 'val' to the stream based at 'ptr'
-// and currently at 'pos'. The position of the stream is not
-// changed.
-//
-// Sets a collection of values; the number of bits advanced in the
-// stream is returned.
-//
-u64bit getDecodedValue (u64bit *ptr, u64bit pos, u64bit siz);
-u64bit getDecodedValues(u64bit *ptr, u64bit pos, u64bit num, u64bit *sizs, u64bit *vals);
-void setDecodedValue (u64bit *ptr, u64bit pos, u64bit siz, u64bit val);
-u64bit setDecodedValues(u64bit *ptr, u64bit pos, u64bit num, u64bit *sizs, u64bit *vals);
-
-
-// Like getDecodedValue() but will pre/post increment/decrement the
-// value stored in the stream before in addition to returning the
-// value.
-//
-// preIncrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz) + 1;
-// setDecodedValue(ptr, pos, siz, x);
-//
-// preDecrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz) - 1;
-// setDecodedValue(ptr, pos, siz, x);
-//
-// postIncrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz);
-// setDecodedValue(ptr, pos, siz, x + 1);
-//
-// postDecrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz);
-// setDecodedValue(ptr, pos, siz, x - 1);
-//
-u64bit preIncrementDecodedValue(u64bit *ptr, u64bit pos, u64bit siz);
-u64bit preDecrementDecodedValue(u64bit *ptr, u64bit pos, u64bit siz);
-u64bit postIncrementDecodedValue(u64bit *ptr, u64bit pos, u64bit siz);
-u64bit postDecrementDecodedValue(u64bit *ptr, u64bit pos, u64bit siz);
-
-
-
-// N.B. - I assume the bits in words are big-endian, which is
-// backwards from the way we shift things around.
-//
-// I define the "addresses" of bits in two consectuve words as
-// [0123][0123]. When adding words to the bit array, they're added
-// from left to right:
-//
-// setDecodedValue(bitstream, %0abc, 3)
-// setDecodedValue(bitstream, %0def, 3)
-//
-// results in [abcd][ef00]
-//
-// But when shifting things around, we typically do it from the right
-// side, since that is where the machine places numbers.
-//
-// A picture or two might help.
-//
-//
-// |----b1-----|
-// |-bit-||-sz-|
-// XXXXXX
-// [0---------------63]
-// ^
-// pos
-//
-//
-// If the bits span two words, it'll look like this; b1 is smaller
-// than siz, and we update bit to be the "uncovered" piece of XXX
-// (all the stuff in word2). The first word is masked, then those
-// bits are shifted onto the result in the correct place. The second
-// word has the correct bits shifted to the right, then those are
-// appended to the result.
-//
-// |b1-|
-// |-----bit-----||---sz---|
-// XXXXXXXXXX
-// [0------------word1][0-------------word2]
-// ^
-// pos
-//
-
-
-inline
-u64bit
-getDecodedValue(u64bit *ptr,
- u64bit pos,
- u64bit siz) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- //PREFETCH(ptr + wrd); makes it worse
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 64 - bit;
- u64bit ret = 0;
-
-#ifdef CHECK_WIDTH
- if (siz == 0) {
- fprintf(stderr, "ERROR: getDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= siz) {
- ret = ptr[wrd] >> (b1 - siz);
- } else {
- bit = siz - b1;
- ret = (ptr[wrd] & u64bitMASK(b1)) << bit;
- wrd++;
- ret |= (ptr[wrd] >> (64 - bit)) & u64bitMASK(bit);
- }
-
- ret &= u64bitMASK(siz);
-
- return(ret);
-}
-
-
-inline
-void
-setDecodedValue(u64bit *ptr,
- u64bit pos,
- u64bit siz,
- u64bit val) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 64 - bit;
-
-#ifdef CHECK_WIDTH
- if (siz == 0) {
- fprintf(stderr, "ERROR: setDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- val &= u64bitMASK(siz);
-
- if (b1 >= siz) {
- ptr[wrd] &= ~( u64bitMASK(siz) << (b1 - siz) );
- ptr[wrd] |= val << (b1 - siz);
- } else {
- bit = siz - b1;
- ptr[wrd] &= ~u64bitMASK(b1);
- ptr[wrd] |= (val & (u64bitMASK(b1) << (bit))) >> (bit);
- wrd++;
- ptr[wrd] &= ~(u64bitMASK(bit) << (64 - bit));
- ptr[wrd] |= (val & (u64bitMASK(bit))) << (64 - bit);
- }
-}
-
-
-inline
-u64bit
-getDecodedValues(u64bit *ptr,
- u64bit pos,
- u64bit num,
- u64bit *sizs,
- u64bit *vals) {
-
- // compute the location of the start of the encoded words, then
- // just walk through to get the remaining words.
-
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- //PREFETCH(ptr + wrd); makes it worse
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 0;
-
- for (u64bit i=0; i<num; i++) {
- b1 = 64 - bit;
-
-#ifdef CHECK_WIDTH
- if (siz[i] == 0) {
- fprintf(stderr, "ERROR: postDecrementDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz[i] > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= sizs[i]) {
- //fprintf(stderr, "get-single pos=%d b1=%d bit=%d wrd=%d\n", pos, b1, bit, wrd);
- vals[i] = ptr[wrd] >> (b1 - sizs[i]);
- bit += sizs[i];
- } else {
- //fprintf(stderr, "get-double pos=%d b1=%d bit=%d wrd=%d bitafter=%d\n", pos, b1, bit, wrd, sizs[i]-b1);
- bit = sizs[i] - b1;
- vals[i] = (ptr[wrd] & u64bitMASK(b1)) << bit;
- wrd++;
- vals[i] |= (ptr[wrd] >> (64 - bit)) & u64bitMASK(bit);
- }
-
- if (bit == 64) {
- wrd++;
- bit = 0;
- }
-
- assert(bit < 64);
-
- vals[i] &= u64bitMASK(sizs[i]);
- pos += sizs[i];
- }
-
- return(pos);
-}
-
-
-inline
-u64bit
-setDecodedValues(u64bit *ptr,
- u64bit pos,
- u64bit num,
- u64bit *sizs,
- u64bit *vals) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 0;
-
- for (u64bit i=0; i<num; i++) {
- vals[i] &= u64bitMASK(sizs[i]);
-
- b1 = 64 - bit;
-
-#ifdef CHECK_WIDTH
- if (siz[i] == 0) {
- fprintf(stderr, "ERROR: postDecrementDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz[i] > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= sizs[i]) {
- //fprintf(stderr, "set-single pos=%d b1=%d bit=%d wrd=%d\n", pos, b1, bit, wrd);
- ptr[wrd] &= ~( u64bitMASK(sizs[i]) << (b1 - sizs[i]) );
- ptr[wrd] |= vals[i] << (b1 - sizs[i]);
- bit += sizs[i];
- } else {
- //fprintf(stderr, "set-double pos=%d b1=%d bit=%d wrd=%d bitafter=%d\n", pos, b1, bit, wrd, sizs[i]-b1);
- bit = sizs[i] - b1;
- ptr[wrd] &= ~u64bitMASK(b1);
- ptr[wrd] |= (vals[i] & (u64bitMASK(b1) << (bit))) >> (bit);
- wrd++;
- ptr[wrd] &= ~(u64bitMASK(bit) << (64 - bit));
- ptr[wrd] |= (vals[i] & (u64bitMASK(bit))) << (64 - bit);
- }
-
- if (bit == 64) {
- wrd++;
- bit = 0;
- }
-
- assert(bit < 64);
-
- pos += sizs[i];
- }
-
- return(pos);
-}
-
-
-
-
-
-
-
-
-
-
-
-
-inline
-u64bit
-preIncrementDecodedValue(u64bit *ptr,
- u64bit pos,
- u64bit siz) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 64 - bit;
- u64bit ret = 0;
-
-#ifdef CHECK_WIDTH
- if (siz == 0) {
- fprintf(stderr, "ERROR: preIncrementDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= siz) {
- ret = ptr[wrd] >> (b1 - siz);
-
- ret++;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~( u64bitMASK(siz) << (b1 - siz) );
- ptr[wrd] |= ret << (b1 - siz);
- } else {
- bit = siz - b1;
-
- ret = (ptr[wrd] & u64bitMASK(b1)) << bit;
- ret |= (ptr[wrd+1] >> (64 - bit)) & u64bitMASK(bit);
-
- ret++;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~u64bitMASK(b1);
- ptr[wrd] |= (ret & (u64bitMASK(b1) << (bit))) >> (bit);
- wrd++;
- ptr[wrd] &= ~(u64bitMASK(bit) << (64 - bit));
- ptr[wrd] |= (ret & (u64bitMASK(bit))) << (64 - bit);
- }
-
- return(ret);
-}
-
-
-
-inline
-u64bit
-preDecrementDecodedValue(u64bit *ptr,
- u64bit pos,
- u64bit siz) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 64 - bit;
- u64bit ret = 0;
-
-#ifdef CHECK_WIDTH
- if (siz == 0) {
- fprintf(stderr, "ERROR: preDecrementDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= siz) {
- ret = ptr[wrd] >> (b1 - siz);
-
- ret--;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~( u64bitMASK(siz) << (b1 - siz) );
- ptr[wrd] |= ret << (b1 - siz);
- } else {
- bit = siz - b1;
-
- ret = (ptr[wrd] & u64bitMASK(b1)) << bit;
- ret |= (ptr[wrd+1] >> (64 - bit)) & u64bitMASK(bit);
-
- ret--;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~u64bitMASK(b1);
- ptr[wrd] |= (ret & (u64bitMASK(b1) << (bit))) >> (bit);
- wrd++;
- ptr[wrd] &= ~(u64bitMASK(bit) << (64 - bit));
- ptr[wrd] |= (ret & (u64bitMASK(bit))) << (64 - bit);
- }
-
- return(ret);
-}
-
-
-
-inline
-u64bit
-postIncrementDecodedValue(u64bit *ptr,
- u64bit pos,
- u64bit siz) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 64 - bit;
- u64bit ret = 0;
-
-#ifdef CHECK_WIDTH
- if (siz == 0) {
- fprintf(stderr, "ERROR: postIncrementDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= siz) {
- ret = ptr[wrd] >> (b1 - siz);
-
- ret++;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~( u64bitMASK(siz) << (b1 - siz) );
- ptr[wrd] |= ret << (b1 - siz);
- } else {
- bit = siz - b1;
-
- ret = (ptr[wrd] & u64bitMASK(b1)) << bit;
- ret |= (ptr[wrd+1] >> (64 - bit)) & u64bitMASK(bit);
-
- ret++;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~u64bitMASK(b1);
- ptr[wrd] |= (ret & (u64bitMASK(b1) << (bit))) >> (bit);
- wrd++;
- ptr[wrd] &= ~(u64bitMASK(bit) << (64 - bit));
- ptr[wrd] |= (ret & (u64bitMASK(bit))) << (64 - bit);
- }
-
- ret--;
- ret &= u64bitMASK(siz);
-
- return(ret);
-}
-
-
-
-
-
-inline
-u64bit
-postDecrementDecodedValue(u64bit *ptr,
- u64bit pos,
- u64bit siz) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit bit = (pos ) & 0x000000000000003fllu;
- u64bit b1 = 64 - bit;
- u64bit ret = 0;
-
-#ifdef CHECK_WIDTH
- if (siz == 0) {
- fprintf(stderr, "ERROR: postDecrementDecodedValue() called with zero size!\n");
- abort();
- }
- if (siz > 64) {
- fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("u64bitFMT")!\n", siz);
- abort();
- }
-#endif
-
- if (b1 >= siz) {
- ret = ptr[wrd] >> (b1 - siz);
-
- ret--;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~( u64bitMASK(siz) << (b1 - siz) );
- ptr[wrd] |= ret << (b1 - siz);
- } else {
- bit = siz - b1;
-
- ret = (ptr[wrd] & u64bitMASK(b1)) << bit;
- ret |= (ptr[wrd+1] >> (64 - bit)) & u64bitMASK(bit);
-
- ret--;
- ret &= u64bitMASK(siz);
-
- ptr[wrd] &= ~u64bitMASK(b1);
- ptr[wrd] |= (ret & (u64bitMASK(b1) << (bit))) >> (bit);
- wrd++;
- ptr[wrd] &= ~(u64bitMASK(bit) << (64 - bit));
- ptr[wrd] |= (ret & (u64bitMASK(bit))) << (64 - bit);
- }
-
- ret++;
- ret &= u64bitMASK(siz);
-
- return(ret);
-}
-
-
-
-#endif // BRI_BITPACKING_H
diff --git a/kmer/libutil/bzipBuffer.C b/kmer/libutil/bzipBuffer.C
deleted file mode 100644
index 76d37f0..0000000
--- a/kmer/libutil/bzipBuffer.C
+++ /dev/null
@@ -1,238 +0,0 @@
-#include "util++.H"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <fcntl.h>
-
-
-// This is probably correct, it just cannot read a normal *.bz file;
-// it probably reads an unpackaged raw bzip stream.
-
-
-bzipBuffer::bzipBuffer(const char *filename, u32bit bufferMax) {
-
- _filename = new char [strlen(filename) + 1];
- strcpy(_filename, filename);
-
- if (bufferMax == 0)
- bufferMax = 32 * 1024;
-
- errno = 0;
- _file = open(filename, O_RDONLY | O_LARGEFILE);
- if (errno) {
- fprintf(stderr, "bzipBuffer()-- couldn't open the file '%s': %s\n",
- filename, strerror(errno));
- exit(1);
- }
-
- _filePos = 0;
- _eof = false;
-
- _bzip2bufferMax = bufferMax;
- _bzip2inPos = 0;
- _bzip2outPos = 0;
-
- _bzip2in = new char [_bzip2bufferMax];
- _bzip2out = new char [_bzip2bufferMax];
-
- _bzip2streamEnd = false;
-
- _bzip2stream.next_in = _bzip2in;
- _bzip2stream.avail_in = 0;
- _bzip2stream.total_in_lo32 = 0;
- _bzip2stream.total_in_hi32 = 0;
- _bzip2stream.next_out = _bzip2out;
- _bzip2stream.avail_out = 0;
- _bzip2stream.total_out_lo32 = 0;
- _bzip2stream.total_out_hi32 = 0;
- _bzip2stream.state = 0L;
- _bzip2stream.bzalloc = 0L;
- _bzip2stream.bzfree = 0L;
- _bzip2stream.opaque = 0L;
-
- int res = BZ2_bzDecompressInit(&_bzip2stream, 0, 0);
- if (res != BZ_OK) {
- // BZ_CONFIG_ERROR, BZ_PARAM_ERROR, BZ_MEM_ERROR
- fprintf(stderr, "bzipBuffer::bzipBuffer()-- Failed to initialize the decompressor.\n");
- exit(1);
- }
-
- fillBuffer();
-}
-
-
-bzipBuffer::~bzipBuffer() {
- delete [] _bzip2in;
- delete [] _bzip2out;
- close(_file);
-}
-
-
-void
-bzipBuffer::fillBuffer(void) {
-
- if (_bzip2streamEnd) {
- _eof = true;
- return;
- }
-
- // Scream and holler if the bzip2 buffer isn't exhausted!
- //
- if (_bzip2outPos < _bzip2stream.avail_out) {
- fprintf(stderr, "bzipBuffer::fillBuffer()-- Buffer isn't empty! Still %d bytes!\n",
- (int)(_bzip2stream.avail_out - _bzip2outPos));
- return;
- }
-
- _bzip2outPos = 0;
-
- again:
-
- // If there is stuff in the input, run the decompressor. If it
- // decompresses anything, return.
- //
- if (_bzip2stream.avail_in > 0) {
-
- fprintf(stderr, "about to decompress %d bytes in input\n", (int)_bzip2stream.avail_in);
- fprintf(stderr, "in is bzip2:%p and real:%p (diff %d)\n", _bzip2stream.next_in, _bzip2in, _bzip2stream.next_in - _bzip2in);
- fprintf(stderr, "out is bzip2:%p and real:%p (diff %d)\n", _bzip2stream.next_out, _bzip2out, _bzip2stream.next_out - _bzip2out);
-
-
- int res = BZ2_bzDecompress(&_bzip2stream);
- if (res == BZ_STREAM_END) {
- fprintf(stderr, "GOT STREAM END!\n");
-
- BZ2_bzDecompressEnd(&_bzip2stream);
-
- _bzip2streamEnd = true;
- res = BZ_OK;
- }
- if (res != BZ_OK) {
- fprintf(stderr, "bzipBuffer::fillBuffer()-- Failed to decompress.\n"), exit(1);
- }
-
- fprintf(stderr, "decompressed %d bytes; still have %d in input\n", (int)_bzip2stream.avail_out, (int)_bzip2stream.avail_in);
- fprintf(stderr, "in is bzip2:%p and real:%p (diff %d)\n", _bzip2stream.next_in, _bzip2in, _bzip2stream.next_in - _bzip2in);
- fprintf(stderr, "out is bzip2:%p and real:%p (diff %d)\n", _bzip2stream.next_out, _bzip2out, _bzip2stream.next_out - _bzip2out);
-
- if (_bzip2stream.avail_out > 0) {
- fprintf(stderr, "----------------------------------------\n");
- fwrite(_bzip2stream.next_out, sizeof(char), _bzip2stream.avail_out, stderr);
- fprintf(stderr, "\n----------------------------------------\n");
- return;
- }
- }
-
- // If we're here and _bzip2streamEnd is true, we hit the end of the
- // stream at the same time we hit the end of the input data.
- //
- if (_bzip2streamEnd) {
- _eof = true;
- return;
- }
-
- // Otherwise, we need to read some input.
- //
- errno = 0;
- _bzip2stream.next_in = _bzip2in;
- _bzip2stream.avail_in = (u32bit)::read(_file, _bzip2in, sizeof(char) * _bzip2bufferMax);
- _bzip2stream.next_out = _bzip2out;
- _bzip2stream.avail_out = _bzip2bufferMax;
- if (errno) {
- fprintf(stderr, "bzipBuffer::fillBuffer()-- read failed: %s\n", strerror(errno));
- exit(1);
- }
-
- fprintf(stderr, "read %d bytes\n", (int)_bzip2stream.avail_in);
-
- if (_bzip2stream.avail_in == 0) {
- fprintf(stderr, "bzipBuffer::fillBuffer()-- hit end of file?\n");
- _eof = true;
- return;
- }
-
- // And now try to decompress it again
- //
- goto again;
-}
-
-
-bool
-bzipBuffer::seek(off_t pos) {
- fprintf(stderr, "bzipBuffer()-- seek() not available for file '%s'.\n", _filename);
- return(false);
-}
-
-
-size_t
-bzipBuffer::read(char *buf, size_t len) {
-
-#if 0
- if (_fileType == 2) {
- size_t c = 0;
-
- while ((_bufferPos < _bufferLen) && (c < len))
- buf[c++] = _buffer[_bufferPos++];
-
- return(c);
- } else {
- // The trick here is to use the existing buffered input first,
- // then do a direct read to get the rest.
- //
- // We fill the buffer again if it is empty.
- //
- // The number of bytes actually put into buf is returned.
-
- size_t bCopied = 0; // Number of bytes copied into the buffer
- size_t bRead = 0; // Number of bytes read into the buffer
- size_t bAct = 0; // Number of bytes actually read from disk
-
- // Easy case; the next len bytes are already in the buffer; just
- // copy and move the position.
- //
- // XXX: Check the zero-left-in-buffer case
- //
- if (_bufferLen - _bufferPos > len) {
- bCopied = len;
- bRead = 0;
-
- memcpy(buf, _buffer + _bufferPos, sizeof(char) * len);
- _bufferPos += (u32bit)len;
- } else {
-
- // Existing buffer not big enough. Copy what's there, then finish
- // with a read.
- //
- memcpy(buf, _buffer + _bufferPos, (_bufferLen - _bufferPos) * sizeof(char));
- bCopied = _bufferLen - _bufferPos;
- _bufferPos = _bufferLen;
-
- while (bCopied + bRead < len) {
- errno = 0;
- bAct = (u32bit)::read(_file, buf + bCopied + bRead, (len - bCopied - bRead) * sizeof(char));
- if (errno) {
- fprintf(stderr, "bzipBuffer()-- couldn't read %d bytes from '%s': n%s\n",
- (u32bit)len * sizeof(char), _filename, strerror(errno));
- exit(1);
- }
-
- // If we hit EOF, return a short read
- if (bAct == 0) {
- len = 0;
- }
- bRead += bAct;
- }
- }
-
- if (_bufferPos == _bufferLen)
- fillBuffer();
-
- return(bCopied + bRead);
- }
-#endif
-
- return(0);
-}
diff --git a/kmer/libutil/bzipBuffer.H b/kmer/libutil/bzipBuffer.H
deleted file mode 100644
index b5e0e47..0000000
--- a/kmer/libutil/bzipBuffer.H
+++ /dev/null
@@ -1,92 +0,0 @@
-#ifndef BZIP_BUFFER_H
-#define BZIP_BUFFER_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <bzlib.h>
-
-#include "util.h"
-
-class bzipBuffer {
-public:
- bzipBuffer(const char *filename, u32bit bufferMax = 32 * 1024);
- ~bzipBuffer();
-
- bool eof(void);
- bool next(void);
- char get(void);
- char getnext(void);
- bool seek(off_t pos);
- size_t read(char *buf, size_t len); // read the next len bytes into the user buffer buf
- off_t tell(void);
-
-private:
- void fillBuffer(void);
- void init(int fileptr, const char *filename, u32bit bufferMax);
-
- char *_filename;
-
- int _file;
- off_t _filePos;
- bool _eof;
-
- u32bit _bzip2bufferMax;
- u32bit _bzip2inPos;
- u32bit _bzip2outPos;
-
- char *_bzip2in;
- char *_bzip2out;
-
- bool _bzip2streamEnd;
- bz_stream _bzip2stream;
-};
-
-
-inline
-bool
-bzipBuffer::eof(void) {
- return(_eof);
-}
-
-
-inline
-bool
-bzipBuffer::next(void) {
-
- if (_eof)
- return(true);
-
- _bzip2outPos++;
- _filePos++;
-
- if (_bzip2outPos >= _bzip2stream.avail_out)
- fillBuffer();
-
- return(_eof);
-}
-
-
-inline
-char
-bzipBuffer::get(void) {
- return(_bzip2out[_bzip2outPos]);
-}
-
-
-inline
-char
-bzipBuffer::getnext(void) {
- char x = _bzip2out[_bzip2outPos];
- next();
- return(x);
-}
-
-
-inline
-off_t
-bzipBuffer::tell(void) {
- return(_filePos);
-}
-
-
-#endif // BZIP_BUFFER_H
diff --git a/kmer/libutil/eliasDeltaEncoding.h b/kmer/libutil/eliasDeltaEncoding.h
deleted file mode 100644
index 8edff1e..0000000
--- a/kmer/libutil/eliasDeltaEncoding.h
+++ /dev/null
@@ -1,33 +0,0 @@
-#ifndef ELIAS_DELTA_ENCODING_H
-#define ELIAS_DELTA_ENCODING_H
-
-#include "bitPacking.h"
-
-inline
-void
-setEliasDeltaEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz,
- u64bit val) {
- u64bit b = logBaseTwo64(val);
- setEliasGammaEncodedNumber(ptr, pos, siz, b);
- pos += *siz;
- setDecodedValue(ptr, pos, b-1, val);
- *siz += b-1;
-}
-
-
-inline
-u64bit
-getEliasDeltaEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz) {
- u64bit b = getEliasGammaEncodedNumber(ptr, pos, siz) - 1;
- pos += *siz;
- *siz += b;
- return(u64bitONE << b | getDecodedValue(ptr, pos, b));
-}
-
-
-
-#endif // ELIAS_DELTA_ENCODING_H
diff --git a/kmer/libutil/eliasGammaEncoding.h b/kmer/libutil/eliasGammaEncoding.h
deleted file mode 100644
index 2b12c15..0000000
--- a/kmer/libutil/eliasGammaEncoding.h
+++ /dev/null
@@ -1,33 +0,0 @@
-#ifndef ELIAS_GAMMA_ENCODING_H
-#define ELIAS_GAMMA_ENCODING_H
-
-#include "bitPacking.h"
-
-inline
-void
-setEliasGammaEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz,
- u64bit val) {
- u64bit b = logBaseTwo64(val);
- setUnaryEncodedNumber(ptr, pos, siz, b);
- pos += *siz;
- setDecodedValue(ptr, pos, b, val);
- *siz += b;
-}
-
-
-inline
-u64bit
-getEliasGammaEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz) {
- u64bit b = getUnaryEncodedNumber(ptr, pos, siz);
- pos += *siz;
- *siz += b;
- return(getDecodedValue(ptr, pos, b));
-}
-
-
-
-#endif // ELIAS_GAMMA_ENCODING_H
diff --git a/kmer/libutil/endianess.H b/kmer/libutil/endianess.H
deleted file mode 100644
index 761ffc7..0000000
--- a/kmer/libutil/endianess.H
+++ /dev/null
@@ -1,64 +0,0 @@
-#ifndef ENDIANESS_H
-#define ENDIANESS_H
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "util.h"
-
-// We need to test how to swap off_t and size_t
-
-// See also test/endianess.c
-
-// If we wanted to convert to network order for everything, rather
-// than convert only when needed, this would be useful.
-//
-#if 0
-bool
-checkEndianessSwapNeeded(void) {
-
- union u64 {
- u64bit u;
- unsigned char c[8];
- };
- union u32 {
- u32bit u;
- unsigned char c[4];
- };
- union u16 {
- u16bit u;
- unsigned char c[2];
- };
-
- u64 u64t.u = u64bitNUMBER(0x0123456789abcdef);
-
- return(u64t.c[0] != 0x0f)
-}
-#endif
-
-
-inline
-u64bit
-u64bitSwap(u64bit x) {
- x = ((x >> 8) & u64bitNUMBER(0x00ff00ff00ff00ff)) | ((x << 8) & u64bitNUMBER(0xff00ff00ff00ff00));
- x = ((x >> 16) & u64bitNUMBER(0x0000ffff0000ffff)) | ((x << 16) & u64bitNUMBER(0xffff0000ffff0000));
- x = ((x >> 32) & u64bitNUMBER(0x00000000ffffffff)) | ((x << 32) & u64bitNUMBER(0xffffffff00000000));
- return(x);
-}
-
-inline
-u32bit
-u32bitSwap(u32bit x) {
- x = ((x >> 8) & u32bitNUMBER(0x00ff00ff)) | ((x << 8) & u32bitNUMBER(0xff00ff00));
- x = ((x >> 16) & u32bitNUMBER(0x0000ffff)) | ((x << 16) & u32bitNUMBER(0xffff0000));
- return(x);
-}
-
-inline
-u16bit
-u16bitSwap(u16bit x) {
- x = ((x >> 8) & 0x00ff) | ((x << 8) & 0xff00);
- return(x);
-}
-
-#endif // ENDIANESS_H
diff --git a/kmer/libutil/fibonacciEncoding.h b/kmer/libutil/fibonacciEncoding.h
deleted file mode 100644
index 08f639d..0000000
--- a/kmer/libutil/fibonacciEncoding.h
+++ /dev/null
@@ -1,171 +0,0 @@
-#ifndef FIBONACCI_ENCODING_H
-#define FIBONACCI_ENCODING_H
-
-#include "bitPacking.h"
-
-// Routines to store and retrieve a Fibonacci encoded number to/from a
-// bit packed word array based at 'ptr' and currently at location
-// 'pos'. Both routines return the size of the encoded number in
-// 'siz'.
-//
-// FibEncoding can store values up to 17,167,680,177,565 (slightly
-// below 2^45, so at most a 44-bit number) in a 64-bit quantity.
-//
-// 93 bits (92 + 1) are needed to store up to 64-bit values.
-//
-// Remember that since we can't store 0, we increment all incoming
-// values, so the actual space used is:
-//
-// #### bits
-// 0 2
-// 1 3
-// 2 4
-// 3 4
-// 4 5
-// 5 5
-// 6 5
-// 7 6
-// 8 6
-// 9 6
-// 10 6
-// 11 6
-// 12 7
-// 20 8
-// 33 9
-// 54 10
-// 88 11
-// 143 12
-// 232 13
-// 376 14
-// 609 15
-// 986 16
-// 1596 17
-// 2583 18
-// 4180 19
-// 6764 20
-// 10945 21
-// 17710 22
-// 28656 23
-// 46387 24
-// 75024 25
-// 121392 26
-
-extern u32bit fibonacciValuesLen;
-extern u64bit fibonacciValues[92];
-
-inline
-void
-setFibonacciEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz,
- u64bit val) {
- u64bit out1 = u64bitZERO;
- u64bit out2 = u64bitZERO;
- u32bit fib = fibonacciValuesLen;
- u32bit fibmax = u64bitZERO;
-
- // We cannot store zero as a fibonacci number, so we simply
- // increase everything by one.
- //
- val++;
-
- // Estimate a starting point for our search; we need a function
- // that is always slightly more than fib()
- //
- // Find the highest bit set, do a lookup
- //
- // XXX: Still need this!
-
- while (fib-- > 0) {
- if (val >= fibonacciValues[fib]) {
- if (fib >= 64)
- out2 |= u64bitONE << (127 - fib);
- else
- out1 |= u64bitONE << (63 - fib);
-
- val -= fibonacciValues[fib];
-
- if (fibmax == u64bitZERO) {
- fibmax = fib + 1;
- if (fibmax >= 64)
- out2 |= u64bitONE << (127 - fibmax);
- else
- out1 |= u64bitONE << (63 - fibmax);
- }
- }
- }
-
- fibmax++;
-
- // Write the encoded numbers to the stream
- //
- if (fibmax > 64) {
- setDecodedValue(ptr, pos, 64, out1);
- pos += 64;
- out2 >>= (128 - fibmax);
- setDecodedValue(ptr, pos, fibmax - 64, out2);
- } else {
- out1 >>= (64 - fibmax);
- setDecodedValue(ptr, pos, fibmax, out1);
- }
-
- *siz = fibmax;
-}
-
-
-
-
-
-inline
-u64bit
-getFibonacciEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz) {
- u64bit wrd = (pos >> 6) & 0x0000cfffffffffffllu;
- u64bit sft = 0x8000000000000000llu >> (pos & 0x000000000000003fllu);
- u64bit val = 0;
- u32bit fib = 0;
- u64bit newbit;
- u64bit oldbit;
-
- oldbit = ptr[wrd] & sft;
- sft >>= 1;
- if (sft == u64bitZERO) {
- wrd++;
- sft = 0x8000000000000000llu;
- }
-
- newbit = ptr[wrd] & sft;
- sft >>= 1;
- if (sft == u64bitZERO) {
- wrd++;
- sft = 0x8000000000000000llu;
- }
-
- while (!oldbit || !newbit) {
- if (oldbit)
- val += fibonacciValues[fib];
-
- fib++;
-
- oldbit = newbit;
- newbit = ptr[wrd] & sft;
- sft >>= 1;
- if (sft == u64bitZERO) {
- wrd++;
- sft = 0x8000000000000000llu;
- }
- }
-
- val += fibonacciValues[fib];
-
- (*siz) = fib + 2;
-
- // We stored val+1, remember? Probably not, because the encoder is
- // next.
- //
- return(val - 1);
-}
-
-
-#endif // FIBONACCI_ENCODING_H
diff --git a/kmer/libutil/fibonacciNumbers.C b/kmer/libutil/fibonacciNumbers.C
deleted file mode 100644
index 3ada6ec..0000000
--- a/kmer/libutil/fibonacciNumbers.C
+++ /dev/null
@@ -1,108 +0,0 @@
-#include "util.h"
-
-//
-// Argh, 64-bit guys use LU as their modifier, but 32-bit guys use LLU.
-//
-
-#ifdef TRUE64BIT
-#define _(VAL) VAL ## LU
-#else
-#define _(VAL) VAL ## LLU
-#endif
-
-u32bit
-fibonacciValuesLen = 92;
-
-u64bit
-fibonacciValues[92] = { _(1),
- _(2),
- _(3),
- _(5),
- _(8),
- _(13),
- _(21),
- _(34),
- _(55),
- _(89),
- _(144),
- _(233),
- _(377),
- _(610),
- _(987),
- _(1597),
- _(2584),
- _(4181),
- _(6765),
- _(10946),
- _(17711),
- _(28657),
- _(46368),
- _(75025),
- _(121393),
- _(196418),
- _(317811),
- _(514229),
- _(832040),
- _(1346269),
- _(2178309),
- _(3524578),
- _(5702887),
- _(9227465),
- _(14930352),
- _(24157817),
- _(39088169),
- _(63245986),
- _(102334155),
- _(165580141),
- _(267914296),
- _(433494437),
- _(701408733),
- _(1134903170),
- _(1836311903),
- _(2971215073),
- _(4807526976),
- _(7778742049),
- _(12586269025),
- _(20365011074),
- _(32951280099),
- _(53316291173),
- _(86267571272),
- _(139583862445),
- _(225851433717),
- _(365435296162),
- _(591286729879),
- _(956722026041),
- _(1548008755920),
- _(2504730781961),
- _(4052739537881),
- _(6557470319842),
- _(10610209857723),
- _(17167680177565),
- _(27777890035288),
- _(44945570212853),
- _(72723460248141),
- _(117669030460994),
- _(190392490709135),
- _(308061521170129),
- _(498454011879264),
- _(806515533049393),
- _(1304969544928657),
- _(2111485077978050),
- _(3416454622906707),
- _(5527939700884757),
- _(8944394323791464),
- _(14472334024676221),
- _(23416728348467685),
- _(37889062373143906),
- _(61305790721611591),
- _(99194853094755497),
- _(160500643816367088),
- _(259695496911122585),
- _(420196140727489673),
- _(679891637638612258),
- _(1100087778366101931),
- _(1779979416004714189),
- _(2880067194370816120),
- _(4660046610375530309),
- _(7540113804746346429),
- _(12200160415121876738) };
diff --git a/kmer/libutil/file.c b/kmer/libutil/file.c
deleted file mode 100644
index 0547f03..0000000
--- a/kmer/libutil/file.c
+++ /dev/null
@@ -1,446 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-
-#include <sys/statvfs.h>
-
-#include "util.h"
-
-
-int
-isHuman(FILE *F) {
- return(isatty(fileno(F)));
-}
-
-
-#ifdef __alpha
-unsigned long __sbrk_override = 1; // See malloc(3) for details.
-
-#define MMAPFLAGS (MAP_FILE | MAP_VARIABLE | MAP_SHARED)
-#endif
-
-#ifdef _AIX
-#define MMAPFLAGS (MAP_FILE | MAP_VARIABLE | MAP_SHARED)
-#endif
-
-#ifdef __CYGWIN__
-#define MMAPFLAGS (MAP_FILE | MAP_SHARED)
-#endif
-
-#ifdef __linux
-#define MMAPFLAGS (MAP_FILE | MAP_SHARED)
-#endif
-
-#ifdef __FreeBSD__
-#define MMAPFLAGS (MAP_FILE | MAP_SHARED)
-#endif
-
-#ifdef __sun
-#define MMAPFLAGS (MAP_SHARED)
-#endif
-
-#ifdef __APPLE__
-#define MMAPFLAGS (MAP_FILE | MAP_SHARED)
-#endif
-
-
-
-
-FILE*
-makeTempFile(char *path) {
- char template[PATH_MAX + 1];
- int fildes;
- FILE *F;
-
- if (path) {
- strcpy(template, path);
- strcat(template, "/XXXXXX");
- } else {
- strcpy(template, "XXXXXX");
- }
-
- errno = 0;
- fildes = mkstemp(template);
- if (errno) {
- fprintf(stderr, "Failed to create temporary file '%s': %s\n", template, strerror(errno));
- exit(1);
- }
-
- errno = 0;
- F = fdopen(fildes, "w+");
- if (errno) {
- fprintf(stderr, "Failed to open temporary file '%s': %s\n", template, strerror(errno));
- exit(1);
- }
-
- errno = 0;
- unlink(template);
- if (errno) {
- fprintf(stderr, "Failed to hide temporary file '%s': %s\n", template, strerror(errno));
- exit(1);
- }
-
- return(F);
-}
-
-
-
-
-
-
-
-
-
-
-void*
-mapFile(const char *filename, u64bit *length, char mode) {
- void *ptr = 0L;
- struct stat sb;
- int f;
- int openMode = O_RDONLY | O_LARGEFILE;
- int mapMode = O_RDWR | O_LARGEFILE;
-
- switch (mode) {
- case 'r':
- openMode = O_RDONLY | O_LARGEFILE;
- mapMode = PROT_READ;
- break;
- case 'w':
- openMode = O_RDWR | O_LARGEFILE;
- mapMode = PROT_READ | PROT_WRITE;
- break;
- default:
- fprintf(stderr, "Invalid mode to mapFile; must be 'r' or 'w'\n");
- exit(1);
- break;
- }
-
- errno = 0;
- f = open(filename, openMode);
- if (errno) {
- fprintf(stderr, "Couldn't open() '%s'\n%s\n", filename, strerror(errno));
- exit(1);
- }
-
- fstat(f, &sb);
- if (errno) {
- fprintf(stderr, "Couldn't fstat() '%s'\n%s\n", filename, strerror(errno));
- exit(1);
- }
-
- *length = sb.st_size;
-
- ptr = mmap(0L, *length, mapMode, MMAPFLAGS, f, (off_t)0);
- if (errno) {
- fprintf(stderr, "Couldn't mmap() '%s'\n%s\n", filename, strerror(errno));
- exit(1);
- }
-
- close(f);
-
- return(ptr);
-}
-
-
-
-void
-unmapFile(void *addr, u64bit length) {
-#ifdef __sun
- // This might work in general, but sun definitely needs the cast.
- //
- (void)munmap((caddr_t)addr, length);
-#else
- (void)munmap(addr, length);
-#endif
-}
-
-
-
-
-
-// Copies all of srcFile to dstFile, returns the number of bytes written
-//
-off_t
-copyFile(char *srcName, FILE *dstFile) {
- off_t srcSize = 0;
- off_t bytesRemain = 0;
- off_t bytesRead = 0;
- int bufferSize = 1024 * 1024;
- char *buffer = 0L;
- FILE *srcFile = 0L;
-
- buffer = (char *)malloc(sizeof(char) * bufferSize);
- if (buffer == 0L) {
- fprintf(stderr, "copyFile()-- Can't allocate buffer.\n");
- exit(1);
- }
-
- srcSize = sizeOfFile(srcName);
- bytesRemain = srcSize;
-
- errno = 0;
- srcFile = fopen(srcName, "r");
- if (errno) {
- fprintf(stderr, "copyFile()-- failed to open the '%s' during merge: %s\n", srcName, strerror(errno));
- exit(1);
- }
-
- while (bytesRemain > 0) {
-
- errno = 0;
-
- if (bytesRemain > bufferSize)
- bytesRead = fread(buffer, sizeof(char), (size_t)bufferSize, srcFile);
- else
- bytesRead = fread(buffer, sizeof(char), (size_t)bytesRemain, srcFile);
-
- if (errno) {
- fprintf(stderr, "copyFile()-- Error reading source: %s\n", strerror(errno));
- exit(1);
- }
-
- if (bytesRead == 0) {
- fprintf(stderr, "copyFile()-- Short read (%d bytes) on source: %s\n", (int)bytesRead, strerror(errno));
- exit(1);
- }
-
- if (bytesRead > 0) {
- fwrite(buffer, sizeof(char), (size_t)bytesRead, dstFile);
-
- if (errno) {
- fprintf(stderr, "copyFile()-- Error writing %d bytes to destination: %s\n", (int)bytesRead, strerror(errno));
- exit(1);
- }
- }
-
- bytesRemain -= bytesRead;
- }
-
- fclose(srcFile);
- free(buffer);
-
- return(srcSize);
-}
-
-
-
-
-
-// Takes a path to a file (that possibly doesn't exist) and returns
-// the number of MB (1048576 bytes) free in the directory of that
-// file.
-//
-u32bit
-freeDiskSpace(char *path) {
- char *p, *t;
- struct statvfs dst;
- struct stat fst;
- u64bit ret = 0;
-
- // Stat the path; if it exists, we're golden.
- //
- if (stat(path, &fst) == 0) {
- if (statvfs(path, &dst) == -1) {
- perror("statvfs");
- exit(1);
- }
- } else {
- // Doesn't exist. Try to find the directory that the file goes into.
- //
- // Copy the input path to a temporary string. Strip off
- // the last component (probably a file prefix, but it could also
- // be a directory -- see below) and return the free space on
- // that device.
- //
- p = (char *)malloc(sizeof(char) * (strlen(path) + 1));
- strcpy(p, path);
- t = strrchr(p, '/');
-
- if (t) {
- *t = 0;
- } else {
- p[0] = '.';
- p[1] = 0;
- }
-
- if (statvfs(p, &dst) == -1) {
- perror("statvfs");
- exit(1);
- }
-
- free(p);
- }
-
- ret = dst.f_frsize;
- ret *= dst.f_bavail;
- ret >>= 20;
-
- return((u32bit)ret);
-}
-
-
-
-
-
-
-// Split writes/reads into smaller pieces, check the result of each
-// piece. Really needed by OSF1 (V5.1).
-//
-void
-safeWrite(int filedes, const void *buffer, const char *desc, size_t nbytes) {
- size_t position = 0;
- size_t length = 32 * 1024 * 1024;
- size_t towrite = 0;
- size_t written = 0;
-
- while (position < nbytes) {
- towrite = length;
- if (position + towrite > nbytes)
- towrite = nbytes - position;
-
- errno = 0;
- written = write(filedes, ((char *)buffer) + position, towrite);
-
- if ((errno) || (towrite != written)) {
- fprintf(stderr, "safeWrite()-- Write failure on %s: %s\n", desc, strerror(errno));
- fprintf(stderr, "safeWrite()-- Wanted to write "s64bitFMT" bytes, wrote "s64bitFMT".\n", (s64bit)towrite, (s64bit)written);
- exit(1);
- }
-
- position += written;
- }
-}
-
-int
-safeRead(int filedes, const void *buffer, const char *desc, size_t nbytes) {
- size_t position = 0;
- size_t length = 32 * 1024 * 1024;
- size_t toread = 0;
- size_t written = 0; // readen?
- int failed = 0;
-
- while (position < nbytes) {
- toread = length;
- if (position + toread > nbytes)
- toread = nbytes - position;
-
- errno = 0;
- written = read(filedes, ((char *)buffer) + position, toread);
-
- failed = errno;
-#ifdef VERY_SAFE
- if (toread != written)
- failed = 1;
-#endif
-
- if ((failed) && (errno != EINTR)) {
- fprintf(stderr, "safeRead()-- Read failure on %s: %s.\n", desc, strerror(errno));
- fprintf(stderr, "safeRead()-- Wanted to read "s64bitFMT" bytes, read "s64bitFMT".\n", (s64bit)toread, (s64bit)written);
- exit(1);
- }
-
- if (written == 0)
- break;
-
- position += written;
- }
-
- return(position);
-}
-
-
-
-void
-closeFile(FILE *F, const char *path) {
-
- // If we're given the path name, see if we need to pclose(),
- // otherwise just fclose() the file.
-
- if ((path) &&
- ((strcmp(path + strlen(path) - 4, ".bz2") == 0) ||
- (strcmp(path + strlen(path) - 3, ".gz") == 0))) {
- pclose(F);
- } else {
- fclose(F);
- }
-}
-
-FILE*
-openFile(const char *path, const char *mode) {
- FILE *F = 0L;
- int isBz = 0;
- int isGz = 0;
- int isRead = 0;
- int isWrite = 0;
- int isRW = 1;
- char cmd[1024] = { 0 };;
-
- // Yes, one could make this significantly simpler by saving the
- // compression command into a variable, instead of the isBz and
- // isGz flags. Maybe instead we should find a compression command
- // that uses different flags.
-
- if (strcmp(path + strlen(path) - 4, ".bz2") == 0)
- isBz = 1;
- if (strcmp(path + strlen(path) - 3, ".gz") == 0)
- isGz = 1;
-
- if (strcmp(mode, "w") == 0) {
- isRead = 0;
- isWrite = 1;
- isRW = 0;
- }
- if (strcmp(mode, "r") == 0) {
- isRead = 1;
- isWrite = 0;
- isRW = 0;
- }
-
- if (isBz) {
- if (isRead) {
- sprintf(cmd, "bzip2 -dc %s", path);
- } else if (isWrite) {
- sprintf(cmd, "bzip2 -9c > %s", path);
- } else {
- fprintf(stderr, "openFile()-- Error! Requested mode '%s' unavailable for bzip2 file '%s'\n", mode, path);
- exit(1);
- }
- } else if (isGz) {
- if (isRead) {
- sprintf(cmd, "gzip -dc %s", path);
- } else if (isWrite) {
- sprintf(cmd, "gzip -9c > %s", path);
- } else {
- fprintf(stderr, "openFile()-- Error! Requested mode '%s' unavailable for gzip file '%s'\n", mode, path);
- exit(1);
- }
- } else {
- // Must be a normal file!
- }
-
-
- if (cmd[0]) {
- errno = 0;
- F = popen(cmd, mode);
- // popen doesn't reliably set errnoman
- //if (errno)
- // fprintf(stderr, "openFile()-- Failed to open pipe '%s': %s\n", cmd, strerror(errno)), exit(1);
- if (F == 0L)
- fprintf(stderr, "openFile()-- Failed to open pipe '%s'\n", cmd), exit(1);
- } else {
- errno = 0;
- F = fopen(path, mode);
- if (errno)
- fprintf(stderr, "openFile()-- Failed to open '%s': %s\n", path, strerror(errno)), exit(1);
- }
-
- return(F);
-}
-
diff --git a/kmer/libutil/generalizedUnaryEncoding.h b/kmer/libutil/generalizedUnaryEncoding.h
deleted file mode 100644
index 8d6531f..0000000
--- a/kmer/libutil/generalizedUnaryEncoding.h
+++ /dev/null
@@ -1,116 +0,0 @@
-#ifndef GENERALIZED_UNARY_ENCODING_H
-#define GENERALIZED_UNARY_ENCODING_H
-
-#include "bitPacking.h"
-
-// Lots and lots of semi-useless debugging information
-//#define DEBUG_GENERALIZEDUNARYENCODING
-
-
-// Generalized unary encodings. Defined by (start, step, stop).
-// This implementation uses stop=infinity to encode all possible
-// numbers. If you know the highest number possible, you'll get a
-// slight decrease in space used ...
-
-// The method:
-//
-// The mth code word consists of 'm' unary encoded, followed by w =
-// start + m * step binary encoded bits. If a == stop, then the
-// terminator in the unary code is dropped.
-//
-// Encoding is tricky. Take the 3,2,9 example:
-// m w template # vals #'s
-// 0 3 1xxx 8 0- 7
-// 1 5 01xxxxx 32 8- 39
-// 2 7 001xxxxxxx 128 40-167
-// 3 9 000xxxxxxxxx 512 168-679
-//
-// I don't see a nice way of mapping our number n to the prefix m,
-// short of some sort of search. The implementation below is
-// probably very slow.
-//
-// On the bright side, decoding is trivial. Read back the unary
-// encoded number, then read that many bits to get the value.
-//
-
-static const u64bit _genunary_start = 3;
-static const u64bit _genunary_step = 2;
-//static const u64bit _genunary_stop = ~u64bitZERO;
-
-
-inline
-void
-setGeneralizedUnaryEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz,
- u64bit val) {
- u64bit m = u64bitZERO;
- u64bit w = _genunary_start;
- u64bit n = u64bitONE << w;
-
- // Search for the prefix m, given our number 'val'.
- // While doing this, we get rid of all the implicitly stored values from 'val'.
- //
-#ifdef DEBUG_GENERALIZEDUNARYENCODING
- fprintf(stderr, " val="u64bitFMT" try n="u64bitFMT" for m="u64bitFMT"\n", val, n, m);
-#endif
-
- while (n <= val) {
- val -= n;
- w += _genunary_step;
- n = u64bitONE << w;
- m++;
-#ifdef DEBUG_GENERALIZEDUNARYENCODING
- fprintf(stderr, " val="u64bitFMT" try n="u64bitFMT" for m="u64bitFMT"\n", val, n, m);
-#endif
- }
-
-#ifdef DEBUG_GENERALIZEDUNARYENCODING
- fprintf(stderr, "val="u64bitFMT" found m="u64bitFMT"\n", val, m);
-#endif
-
- // Now just encode the number
- // m - the unary encoded prefix
- // w - the size of the binary encoded number
-
- setUnaryEncodedNumber(ptr, pos, siz, m);
- setDecodedValue(ptr, pos+*siz, w, val);
- *siz = m + 1 + w;
-}
-
-
-
-inline
-u64bit
-getGeneralizedUnaryEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz) {
- u64bit val = u64bitZERO;
- u64bit m = u64bitZERO;
- u64bit w = u64bitZERO;
-
- // Comments in the encoder apply here too.
-
- m = getUnaryEncodedNumber(ptr, pos, siz);
- w = _genunary_start + m * _genunary_step;
- val = getDecodedValue(ptr, pos + *siz, w);
- *siz = m + 1 + w;
-
-#ifdef DEBUG_GENERALIZEDUNARYENCODING
- fprintf(stderr, "m="u64bitFMT" w="u64bitFMT" val="u64bitFMT"\n", m, w, val);
-#endif
-
- // Add in the implcitly stored pieces of the number
- //
- while (m--) {
- w -= _genunary_step;
- val += u64bitONE << w;
- }
-
- return(val);
-}
-
-
-
-
-#endif // GENERALIZED_UNARY_ENCODING_H
diff --git a/kmer/libutil/intervalList.C b/kmer/libutil/intervalList.C
deleted file mode 100644
index 07cb19b..0000000
--- a/kmer/libutil/intervalList.C
+++ /dev/null
@@ -1,438 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "util++.H"
-
-
-intervalList::intervalList(u32bit initialSize) {
- _isSorted = true;
- _isMerged = true;
- _listLen = 0;
- _listMax = initialSize;
- _list = new _intervalPair [_listMax];
-}
-
-
-intervalList::~intervalList() {
- delete [] _list;
-}
-
-
-intervalList &
-intervalList::operator=(intervalList &src) {
- _isSorted = src._isSorted;
- _isMerged = src._isMerged;
- _listLen = src._listLen;
-
- if (_listMax < src._listMax) {
- delete [] _list;
- _listMax = src._listMax;
- _list = new _intervalPair [_listMax];
- }
-
- memcpy(_list, src._list, _listLen * sizeof(_intervalPair));
-
- return(*this);
-}
-
-
-void
-intervalList::add(intervalNumber position, intervalNumber length) {
-
- if (_listLen >= _listMax) {
- _listMax *= 2;
- _intervalPair *l = new _intervalPair [_listMax];
- memcpy(l, _list, sizeof(_intervalPair) * _listLen);
- delete [] _list;
- _list = l;
- }
-
- _list[_listLen].lo = position;
- _list[_listLen].hi = position + length;
- _list[_listLen].ct = 1;
-
-#if 0
- // Aborted attempt to add a data field here. Got stuck
- // deciding how to handle merges lightweight
-
- _list[_listLen].data = 0L;
-
- if (data != ~u64bitZERO) {
- _list[_listLen].dataLen = 1;
- _list[_listLen].dataMax = 4;
- _list[_listLen].data = new u64bit [_list[_listLen].dataMax];
- _list[_listLen].data[0] = data;
- }
-#endif
-
- if ((_listLen > 0) &&
- (_list[_listLen-1].lo > _list[_listLen].lo)) {
- _isSorted = false;
- _isMerged = false;
- }
-
- _listLen++;
-}
-
-
-static
-int
-intervalList_sort_helper(const void *a, const void *b) {
- _intervalPair *A = (_intervalPair *)a;
- _intervalPair *B = (_intervalPair *)b;
-
- if (A->lo < B->lo) return(-1);
- if (A->lo > B->lo) return(1);
- if (A->hi < B->hi) return(-1);
- if (A->hi > B->hi) return(1);
- return(0);
-}
-
-
-void
-intervalList::sort(void) {
-
- if (_isSorted)
- return;
-
- if (_listLen > 1)
- qsort(_list, _listLen, sizeof(_intervalPair), intervalList_sort_helper);
-
- _isSorted = true;
-}
-
-
-void
-intervalList::merge(u32bit minOverlap) {
- u32bit thisInterval = 0;
- u32bit nextInterval = 1;
-
- if (_listLen < 2)
- return;
-
- sort();
-
- while (nextInterval < _listLen) {
-
- if ((_list[thisInterval].lo == 0) &&
- (_list[thisInterval].hi == 0)) {
-
- // Our interval is empty. Copy in the interval we are
- // examining and move to the next.
-
- // XXX This is probably useless, thisInterval should always be
- // valid.
-
- _list[thisInterval].lo = _list[nextInterval].lo;
- _list[thisInterval].hi = _list[nextInterval].hi;
- _list[thisInterval].ct = _list[nextInterval].ct;
-
- _list[nextInterval].lo = 0;
- _list[nextInterval].hi = 0;
- nextInterval++;
- } else {
-
- // This interval is valid. See if it overlaps with the next
- // interval.
-
- bool intersects = false;
-
- if ((_list[thisInterval].lo <= _list[nextInterval].lo) &&
- (_list[nextInterval].hi <= _list[thisInterval].hi))
- // next is contained in this
- intersects = true;
-
- if (_list[thisInterval].hi - minOverlap >= _list[nextInterval].lo)
- // next has thick overlap to this
- intersects = true;
-
-
- if (intersects) {
-
- // Got an intersection.
-
- // Merge nextInterval into thisInterval -- the hi range
- // is extended if the nextInterval range is larger.
- //
- if (_list[thisInterval].hi < _list[nextInterval].hi)
- _list[thisInterval].hi = _list[nextInterval].hi;
-
- _list[thisInterval].ct += _list[nextInterval].ct;
-
- // Clear the just merged nextInterval and move to the next one.
- //
- _list[nextInterval].lo = 0;
- _list[nextInterval].hi = 0;
- _list[nextInterval].ct = 0;
- nextInterval++;
- } else {
-
- // No intersection. Move along. Nothing to see here.
-
- // If there is a gap between the target and the examine (we
- // must have merged sometime in the past), copy examine to
- // the next target.
-
- thisInterval++;
-
- if (thisInterval != nextInterval) {
- _list[thisInterval].lo = _list[nextInterval].lo;
- _list[thisInterval].hi = _list[nextInterval].hi;
- _list[thisInterval].ct = _list[nextInterval].ct;
- }
-
- nextInterval++;
- }
- }
- }
-
- if (thisInterval+1 < _listLen)
- _listLen = thisInterval + 1;
-
- _isMerged = true;
-}
-
-
-void
-intervalList::invert(intervalNumber lo, intervalNumber hi) {
-
- if (!_isSorted || !_isMerged) {
- fprintf(stderr, "intervalList::invert()-- ERROR! List is not sorted or not merged!\n");
- exit(1);
- }
-
- // Create a new list to store the inversion
- //
- u32bit invLen = 0;
- u32bit invMax = _listLen + 2;
- _intervalPair *inv = new _intervalPair [invMax];
-
- // Add the first
- //
- if (lo < _list[0].lo) {
- inv[invLen].lo = lo;
- inv[invLen].hi = _list[0].lo;
- invLen++;
- }
-
- // Add the pieces
- for (u32bit i=1; i<_listLen; i++) {
- if (_list[i-1].hi < _list[i].lo) {
- inv[invLen].lo = _list[i-1].hi;
- inv[invLen].hi = _list[i].lo;
- invLen++;
- }
- }
-
- // Add the last
- if (_list[_listLen-1].hi < hi) {
- inv[invLen].lo = _list[_listLen-1].hi;
- inv[invLen].hi = hi;
- invLen++;
- }
-
- // Nuke the old list, swap in the new one
- delete [] _list;
-
- _list = inv;
- _listLen = invLen;
- _listMax = invMax;
-}
-
-
-
-void
-intervalList::merge(intervalList *IL) {
- //bool isSorted = _isSorted;
- //bool isMerged = _isMerged;
-
- for (u32bit i=0; i<IL->_listLen; i++)
- add(IL->_list[i].lo, IL->_list[i].hi - IL->_list[i].lo);
-
- //if (isSorted) sort();
- //if (isMerged) merge();
-}
-
-
-
-void
-intervalList::intersect(intervalList &A,
- intervalList &B) {
- A.merge();
- B.merge();
-
- u32bit ai = 0;
- u32bit bi = 0;
-
- while ((ai < A.numberOfIntervals()) &&
- (bi < B.numberOfIntervals())) {
- u32bit al = A.lo(ai);
- u32bit ah = A.hi(ai);
- u32bit bl = B.lo(bi);
- u32bit bh = B.hi(bi);
- u32bit nl = 0;
- u32bit nh = 0;
-
- // If they intersect, make a new region
- //
- if ((al <= bl) && (bl < ah)) {
- nl = bl;
- nh = (ah < bh) ? ah : bh;
- }
-
- if ((bl <= al) && (al < bh)) {
- nl = al;
- nh = (ah < bh) ? ah : bh;
- }
-
- if (nl < nh)
- add(nl, nh - nl);
-
- // Advance the list with the earlier region.
- //
- if (ah < bh) {
- // A ends before B
- ai++;
- } else if (ah > bh) {
- // B ends before A
- bi++;
- } else {
- // Exactly the same ending!
- ai++;
- bi++;
- }
- }
-}
-
-
-
-// Populates an array with the intervals that are within the supplied interval.
-//
-// Naive implementation that is easy to verify (and that works on an unsorted list).
-//
-u32bit
-intervalList::overlapping(intervalNumber rangelo,
- intervalNumber rangehi,
- u32bit *&intervals,
- u32bit &intervalsLen,
- u32bit &intervalsMax) {
-
- if (intervals == 0L) {
- intervalsMax = 256;
- intervals = new u32bit [intervalsMax];
- }
-
- intervalsLen = 0;
-
- for (u32bit i=0; i<_listLen; i++) {
- if ((rangelo <= _list[i].hi) &&
- (rangehi >= _list[i].lo)) {
- if (intervalsLen >= intervalsMax) {
- intervalsMax *= 2;
- u32bit *X = new u32bit [intervalsMax];
- memcpy(X, intervals, sizeof(u32bit) * intervalsLen);
- delete [] intervals;
- intervals = X;
- }
-
- intervals[intervalsLen++] = i;
- }
- }
-
- return(intervalsLen);
-}
-
-
-
-static
-int
-intervalDepth_sort_helper(const void *a, const void *b) {
- intervalDepthRegions *A = (intervalDepthRegions *)a;
- intervalDepthRegions *B = (intervalDepthRegions *)b;
-
- if (A->pos < B->pos) return(-1);
- if (A->pos > B->pos) return(1);
- return(0);
-}
-
-
-intervalDepth::intervalDepth(intervalList &IL) {
- u32bit idlen = IL.numberOfIntervals() * 2;
- intervalDepthRegions *id = new intervalDepthRegions [idlen];
-
- for (u32bit i=0; i<IL.numberOfIntervals(); i++) {
- id[2*i ].pos = IL.lo(i);
- id[2*i ].cha = 1;
- id[2*i+1].pos = IL.hi(i);
- id[2*i+1].cha = -1;
- }
-
- qsort(id, idlen, sizeof(intervalDepthRegions), intervalDepth_sort_helper);
- computeIntervals(id, idlen);
-
- delete [] id;
-}
-
-
-intervalDepth::intervalDepth(intervalDepthRegions *id, u32bit idlen) {
- qsort(id, idlen, sizeof(intervalDepthRegions), intervalDepth_sort_helper);
- computeIntervals(id, idlen);
-}
-
-
-void
-intervalDepth::computeIntervals(intervalDepthRegions *id, u32bit idlen) {
-
- // Scan the list, counting how many times we change depth.
- //
- _listMax = 1;
- for (u32bit i=1; i<idlen; i++) {
- if (id[i-1].pos != id[i].pos)
- _listMax++;
- }
-
- // Allocate the real depth of coverage intervals
- //
- _listLen = 0;
- _list = new _intervalDepth [_listMax];
-
- // Build new intervals
-
- // Init first interval.
- _list[_listLen].lo = id[0].pos;
- _list[_listLen].hi = id[0].pos;
- _list[_listLen].de = id[0].cha;
-
- for (u32bit i=1; i<idlen; i++) {
- // Update the end of the current interval.
- _list[_listLen].hi = id[i].pos;
-
- //fprintf(stderr, "ID %d %d - cur %d %d-%d %d\n",
- // id[i].pos, id[i].cha,
- // _listLen, _list[_listLen].lo, _list[_listLen].hi, _list[_listLen].de);
-
- // If the position is different than the last, make a new interval
- if (id[i-1].pos != id[i].pos) {
- _listLen++;
-
- _list[_listLen].lo = id[i].pos;
- _list[_listLen].de = _list[_listLen-1].de;
- }
-
- // Finally, update the depth and end of the current interval
- _list[_listLen].hi = id[i].pos;
- _list[_listLen].de += id[i].cha;
-
- //fprintf(stderr, "ID %d %d - cur %d %d-%d %d POST\n",
- // id[i].pos, id[i].cha,
- // _listLen, _list[_listLen].lo, _list[_listLen].hi, _list[_listLen].de);
- }
-
- assert(_listLen > 0);
- assert(_listLen < _listMax);
-}
-
-intervalDepth::~intervalDepth() {
- delete [] _list;
-}
diff --git a/kmer/libutil/intervalList.H b/kmer/libutil/intervalList.H
deleted file mode 100644
index 35ceaab..0000000
--- a/kmer/libutil/intervalList.H
+++ /dev/null
@@ -1,112 +0,0 @@
-#ifndef INTERVALLIST_H
-#define INTERVALLIST_H
-
-
-// I don't want this to be public, but the sort function
-// in intervalList.C needs it so.
-//
-typedef u64bit intervalNumber;
-
-struct _intervalPair {
- intervalNumber lo;
- intervalNumber hi;
- u32bit ct;
-};
-
-struct _intervalDepth {
- intervalNumber lo;
- intervalNumber hi;
- u32bit de;
-};
-
-struct intervalDepthRegions {
- intervalNumber pos; // Position of the change in depth
- s32bit cha; // The change in depth, e.g, +1, -1
-};
-
-
-class intervalList {
-public:
- intervalList(u32bit initialSize=32);
- ~intervalList();
-
- intervalList &operator=(intervalList &src);
-
- void clear(void) {
- _isSorted = true;
- _isMerged = true;
- _listLen = 0;
- }
-
- void add(intervalNumber position, intervalNumber length);
- void sort(void);
- void merge(u32bit minOverlap=0); // Merge overlapping regions
- void merge(intervalList *IL); // Insert IL into this list
-
- void intersect(intervalList &A,
- intervalList &B);
-
- u32bit overlapping(intervalNumber lo,
- intervalNumber hi,
- u32bit *&intervals,
- u32bit &intervalsLen,
- u32bit &intervalsMax);
-
- void invert(intervalNumber lo, intervalNumber hi);
-
- u32bit numberOfIntervals(void) { return(_listLen); };
-
- intervalNumber sumOfLengths(void) {
- intervalNumber len = 0;
- u32bit i = numberOfIntervals();
-
- if (i > 0)
- while (i--)
- len += _list[i].hi - _list[i].lo;
-
- return(len);
- };
-
-
- intervalNumber lo(u32bit i) { return(_list[i].lo); };
- intervalNumber hi(u32bit i) { return(_list[i].hi); };
- u32bit ct(u32bit i) { return(_list[i].ct); };
-
-private:
- bool _isSorted;
- bool _isMerged;
- u32bit _listLen;
- u32bit _listMax;
- _intervalPair *_list;
-};
-
-
-
-// Takes as input an intervalList, computes the number of intervals
-// covering every position in there, stores this as a new set of
-// intervals, annotated with the depth.
-//
-// This is a static object, initialized once by the intervalList.
-//
-class intervalDepth {
-public:
- intervalDepth(intervalList &IL);
- intervalDepth(intervalDepthRegions *id, u32bit idlen);
- ~intervalDepth();
-
- u32bit numberOfIntervals(void) { return(_listLen); };
-
- intervalNumber lo(u32bit i) { return(_list[i].lo); };
- intervalNumber hi(u32bit i) { return(_list[i].hi); };
- u32bit de(u32bit i) { return(_list[i].de); };
-
-private:
- void computeIntervals(intervalDepthRegions *id, u32bit idlen);
-
- u32bit _listLen;
- u32bit _listMax;
- _intervalDepth *_list;
-};
-
-
-#endif // INTERVALLIST_H
diff --git a/kmer/libutil/kazlib/Make.include b/kmer/libutil/kazlib/Make.include
deleted file mode 100644
index 7ecb369..0000000
--- a/kmer/libutil/kazlib/Make.include
+++ /dev/null
@@ -1,27 +0,0 @@
-# -*- makefile -*-
-
-src := $/dict.c \
- $/dict.h \
- $/except.c \
- $/except.h \
- $/hash.c \
- $/hash.h \
- $/list.c \
- $/list.h \
- $/sfx.c \
- $/sfx.h
-
-tst := $/dict-main.c \
- $/except-main.c \
- $/hash-main.c \
- $/list-main.c \
- $/sfx-main.c
-
-$/.C_SRCS :=$(filter %.c,${src})
-$/.CXX_SRCS :=$(filter %.C,${src})
-$/.CXX_LIBS :=$/libkaz.a
-
-$/.CLEAN := $/*.o
-
-$/libkaz.a: ${$/.C_SRCS:.c=.o} ${$/.CXX_SRCS:.C=.o}
-
diff --git a/kmer/libutil/kazlib/blast.pl b/kmer/libutil/kazlib/blast.pl
deleted file mode 100755
index 63351c9..0000000
--- a/kmer/libutil/kazlib/blast.pl
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/perl
-
-#
-# This is a program whose output can be piped to the test drivers for
-# hash.c and dict.c. It inserts a bunch of data and then deletes it all.
-#
-# The $modulus should be a prime number. This ensures that the $modulus - 1
-# generated keys are all distinct. The $factor_i and $factor_d values need not
-# be prime, but it should not be a multiple of $modulus (including zero),
-# otherwise a sequence of duplicate keys will be generated: choose numbers
-# in the range [1, $modulus - 1]. Choosing 1 means that
-# insertions (or deletions) will take place in order.
-# The purpose of using the prime modulus number is to generate a repeatable
-# sequence of unique keys that is (possibly) not in sorted order.
-#
-
-# $modulus = 200003;
-# $factor_i = 100;
-# $factor_d = 301;
-
-$modulus = 6113;
-$factor_i = 1669;
-$factor_d = 2036;
-
-for ($i = 1; $i < $modulus; $i++) {
- printf("a %d %d\n", ($i * $factor_i) % $modulus, $i);
-}
-
-for ($i = 1; $i < $modulus; $i++) {
- printf("d %d\n", ($i * $factor_d) % $modulus);
-}
-
-print "t\nq\n"
diff --git a/kmer/libutil/kazlib/dict.c b/kmer/libutil/kazlib/dict.c
deleted file mode 100644
index cd98498..0000000
--- a/kmer/libutil/kazlib/dict.c
+++ /dev/null
@@ -1,1238 +0,0 @@
-/*
- * Dictionary Abstract Data Type
- * Copyright (C) 1997 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#define NDEBUG
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <assert.h>
-#define DICT_IMPLEMENTATION
-#include "dict.h"
-
-// bpw 20050309 define this to use a qsort(3) compatible sort function,
-// requiring two dereferences to get the data instead of one.
-//
-#define BE_QSORT_COMPATIBLE
-
-/*
- * These macros provide short convenient names for structure members,
- * which are embellished with dict_ prefixes so that they are
- * properly confined to the documented namespace. It's legal for a
- * program which uses dict to define, for instance, a macro called ``parent''.
- * Such a macro would interfere with the dnode_t struct definition.
- * In general, highly portable and reusable C modules which expose their
- * structures need to confine structure member names to well-defined spaces.
- * The resulting identifiers aren't necessarily convenient to use, nor
- * readable, in the implementation, however!
- */
-
-#define left dict_left
-#define right dict_right
-#define parent dict_parent
-#define color dict_color
-#define key dict_key
-#define data dict_data
-
-#define nilnode dict_nilnode
-#define nodecount dict_nodecount
-#define maxcount dict_maxcount
-#define compare dict_compare
-#define allocnode dict_allocnode
-#define freenode dict_freenode
-#define context dict_context
-#define dupes dict_dupes
-
-#define dictptr dict_dictptr
-
-#define dict_root(D) ((D)->nilnode.left)
-#define dict_nil(D) (&(D)->nilnode)
-#define DICT_DEPTH_MAX 64
-
-static dnode_t *dnode_alloc(void *context);
-static void dnode_free(dnode_t *node, void *context);
-
-/*
- * Perform a ``left rotation'' adjustment on the tree. The given node P and
- * its right child C are rearranged so that the P instead becomes the left
- * child of C. The left subtree of C is inherited as the new right subtree
- * for P. The ordering of the keys within the tree is thus preserved.
- */
-
-static void rotate_left(dnode_t *upper)
-{
- dnode_t *lower, *lowleft, *upparent;
-
- lower = upper->right;
- upper->right = lowleft = lower->left;
- lowleft->parent = upper;
-
- lower->parent = upparent = upper->parent;
-
- /* don't need to check for root node here because root->parent is
- the sentinel nil node, and root->parent->left points back to root */
-
- if (upper == upparent->left) {
- upparent->left = lower;
- } else {
- assert (upper == upparent->right);
- upparent->right = lower;
- }
-
- lower->left = upper;
- upper->parent = lower;
-}
-
-/*
- * This operation is the ``mirror'' image of rotate_left. It is
- * the same procedure, but with left and right interchanged.
- */
-
-static void rotate_right(dnode_t *upper)
-{
- dnode_t *lower, *lowright, *upparent;
-
- lower = upper->left;
- upper->left = lowright = lower->right;
- lowright->parent = upper;
-
- lower->parent = upparent = upper->parent;
-
- if (upper == upparent->right) {
- upparent->right = lower;
- } else {
- assert (upper == upparent->left);
- upparent->left = lower;
- }
-
- lower->right = upper;
- upper->parent = lower;
-}
-
-/*
- * Do a postorder traversal of the tree rooted at the specified
- * node and free everything under it. Used by dict_free().
- */
-
-static void free_nodes(dict_t *dict, dnode_t *node, dnode_t *nil)
-{
- if (node == nil)
- return;
- free_nodes(dict, node->left, nil);
- free_nodes(dict, node->right, nil);
- dict->freenode(node, dict->context);
-}
-
-/*
- * This procedure performs a verification that the given subtree is a binary
- * search tree. It performs an inorder traversal of the tree using the
- * dict_next() successor function, verifying that the key of each node is
- * strictly lower than that of its successor, if duplicates are not allowed,
- * or lower or equal if duplicates are allowed. This function is used for
- * debugging purposes.
- */
-
-static int verify_bintree(dict_t *dict)
-{
- dnode_t *first, *next;
-
- first = dict_first(dict);
-
- if (dict->dupes) {
- while (first && (next = dict_next(dict, first))) {
-#ifdef BE_QSORT_COMPATIBLE
- if (dict->compare(&first->key, &next->key) > 0)
- return 0;
-#else
- if (dict->compare(first->key, next->key) > 0)
- return 0;
-#endif
- first = next;
- }
- } else {
- while (first && (next = dict_next(dict, first))) {
-#ifdef BE_QSORT_COMPATIBLE
- if (dict->compare(&first->key, &next->key) >= 0)
- return 0;
-#else
- if (dict->compare(first->key, next->key) >= 0)
- return 0;
-#endif
- first = next;
- }
- }
- return 1;
-}
-
-
-/*
- * This function recursively verifies that the given binary subtree satisfies
- * three of the red black properties. It checks that every red node has only
- * black children. It makes sure that each node is either red or black. And it
- * checks that every path has the same count of black nodes from root to leaf.
- * It returns the blackheight of the given subtree; this allows blackheights to
- * be computed recursively and compared for left and right siblings for
- * mismatches. It does not check for every nil node being black, because there
- * is only one sentinel nil node. The return value of this function is the
- * black height of the subtree rooted at the node ``root'', or zero if the
- * subtree is not red-black.
- */
-
-static unsigned int verify_redblack(dnode_t *nil, dnode_t *root)
-{
- unsigned height_left, height_right;
-
- if (root != nil) {
- height_left = verify_redblack(nil, root->left);
- height_right = verify_redblack(nil, root->right);
- if (height_left == 0 || height_right == 0)
- return 0;
- if (height_left != height_right)
- return 0;
- if (root->color == dnode_red) {
- if (root->left->color != dnode_black)
- return 0;
- if (root->right->color != dnode_black)
- return 0;
- return height_left;
- }
- if (root->color != dnode_black)
- return 0;
- return height_left + 1;
- }
- return 1;
-}
-
-/*
- * Compute the actual count of nodes by traversing the tree and
- * return it. This could be compared against the stored count to
- * detect a mismatch.
- */
-
-static dictcount_t verify_node_count(dnode_t *nil, dnode_t *root)
-{
- if (root == nil)
- return 0;
- else
- return 1 + verify_node_count(nil, root->left)
- + verify_node_count(nil, root->right);
-}
-
-/*
- * Verify that the tree contains the given node. This is done by
- * traversing all of the nodes and comparing their pointers to the
- * given pointer. Returns 1 if the node is found, otherwise
- * returns zero. It is intended for debugging purposes.
- */
-
-static int verify_dict_has_node(dnode_t *nil, dnode_t *root, dnode_t *node)
-{
- if (root != nil) {
- return root == node
- || verify_dict_has_node(nil, root->left, node)
- || verify_dict_has_node(nil, root->right, node);
- }
- return 0;
-}
-
-
-/*
- * Dynamically allocate and initialize a dictionary object.
- */
-
-dict_t *dict_create(dictcount_t maxcount, dict_comp_t comp)
-{
- dict_t *new = malloc(sizeof *new);
-
- if (new) {
- new->compare = comp;
- new->allocnode = dnode_alloc;
- new->freenode = dnode_free;
- new->context = NULL;
- new->nodecount = 0;
- new->maxcount = maxcount;
- new->nilnode.left = &new->nilnode;
- new->nilnode.right = &new->nilnode;
- new->nilnode.parent = &new->nilnode;
- new->nilnode.color = dnode_black;
- new->dupes = 0;
- }
- return new;
-}
-
-/*
- * Select a different set of node allocator routines.
- */
-
-void dict_set_allocator(dict_t *dict, dnode_alloc_t al,
- dnode_free_t fr, void *context)
-{
- assert (dict_count(dict) == 0);
- assert ((al == NULL && fr == NULL) || (al != NULL && fr != NULL));
-
- dict->allocnode = al ? al : dnode_alloc;
- dict->freenode = fr ? fr : dnode_free;
- dict->context = context;
-}
-
-/*
- * Free a dynamically allocated dictionary object. Removing the nodes
- * from the tree before deleting it is required.
- */
-
-void dict_destroy(dict_t *dict)
-{
- assert (dict_isempty(dict));
- free(dict);
-}
-
-/*
- * Free all the nodes in the dictionary by using the dictionary's
- * installed free routine. The dictionary is emptied.
- */
-
-void dict_free_nodes(dict_t *dict)
-{
- dnode_t *nil = dict_nil(dict), *root = dict_root(dict);
- free_nodes(dict, root, nil);
- dict->nodecount = 0;
- dict->nilnode.left = &dict->nilnode;
- dict->nilnode.right = &dict->nilnode;
-}
-
-/*
- * Obsolescent function, equivalent to dict_free_nodes
- */
-
-void dict_free(dict_t *dict)
-{
-#ifdef KAZLIB_OBSOLESCENT_DEBUG
- assert ("call to obsolescent function dict_free()" && 0);
-#endif
- dict_free_nodes(dict);
-}
-
-/*
- * Initialize a user-supplied dictionary object.
- */
-
-dict_t *dict_init(dict_t *dict, dictcount_t maxcount, dict_comp_t comp)
-{
- dict->compare = comp;
- dict->allocnode = dnode_alloc;
- dict->freenode = dnode_free;
- dict->context = NULL;
- dict->nodecount = 0;
- dict->maxcount = maxcount;
- dict->nilnode.left = &dict->nilnode;
- dict->nilnode.right = &dict->nilnode;
- dict->nilnode.parent = &dict->nilnode;
- dict->nilnode.color = dnode_black;
- dict->dupes = 0;
- return dict;
-}
-
-/*
- * Initialize a dictionary in the likeness of another dictionary
- */
-
-void dict_init_like(dict_t *dict, const dict_t *template)
-{
- dict->compare = template->compare;
- dict->allocnode = template->allocnode;
- dict->freenode = template->freenode;
- dict->context = template->context;
- dict->nodecount = 0;
- dict->maxcount = template->maxcount;
- dict->nilnode.left = &dict->nilnode;
- dict->nilnode.right = &dict->nilnode;
- dict->nilnode.parent = &dict->nilnode;
- dict->nilnode.color = dnode_black;
- dict->dupes = template->dupes;
-
- assert (dict_similar(dict, template));
-}
-
-/*
- * Remove all nodes from the dictionary (without freeing them in any way).
- */
-
-static void dict_clear(dict_t *dict)
-{
- dict->nodecount = 0;
- dict->nilnode.left = &dict->nilnode;
- dict->nilnode.right = &dict->nilnode;
- dict->nilnode.parent = &dict->nilnode;
- assert (dict->nilnode.color == dnode_black);
-}
-
-
-/*
- * Verify the integrity of the dictionary structure. This is provided for
- * debugging purposes, and should be placed in assert statements. Just because
- * this function succeeds doesn't mean that the tree is not corrupt. Certain
- * corruptions in the tree may simply cause undefined behavior.
- */
-
-int dict_verify(dict_t *dict)
-{
- dnode_t *nil = dict_nil(dict), *root = dict_root(dict);
-
- /* check that the sentinel node and root node are black */
- if (root->color != dnode_black)
- return(0 * fprintf(stderr, "dict_verify()-- Root node not black!\n"));
- if (nil->color != dnode_black)
- return(0 * fprintf(stderr, "dict_verify()-- Nil node not black!\n"));
- if (nil->right != nil)
- return(0 * fprintf(stderr, "dict_verify()-- Nul->right not Nil!\n"));
- /* nil->left is the root node; check that its parent pointer is nil */
- if (nil->left->parent != nil)
- return(0 * fprintf(stderr, "dict_verify()-- Nul->left->parent is not Nil!\n"));
- /* perform a weak test that the tree is a binary search tree */
- if (!verify_bintree(dict))
- return(0 * fprintf(stderr, "dict_verify()-- Not a binary search tree!\n"));
- /* verify that the tree is a red-black tree */
- if (!verify_redblack(nil, root))
- return(0 * fprintf(stderr, "dict_verify()-- Not a red-black tree!\n"));
- if (verify_node_count(nil, root) != dict_count(dict))
- return(0 * fprintf(stderr, "dict_verify()-- Node count is wrong!\n"));
- return 1;
-}
-
-/*
- * Determine whether two dictionaries are similar: have the same comparison and
- * allocator functions, and same status as to whether duplicates are allowed.
- */
-
-int dict_similar(const dict_t *left, const dict_t *right)
-{
- if (left->compare != right->compare)
- return 0;
-
- if (left->allocnode != right->allocnode)
- return 0;
-
- if (left->freenode != right->freenode)
- return 0;
-
- if (left->context != right->context)
- return 0;
-
- if (left->dupes != right->dupes)
- return 0;
-
- return 1;
-}
-
-/*
- * Locate a node in the dictionary having the given key.
- * If the node is not found, a null a pointer is returned (rather than
- * a pointer that dictionary's nil sentinel node), otherwise a pointer to the
- * located node is returned.
- */
-
-dnode_t *dict_lookup(dict_t *dict, const void *key)
-{
- dnode_t *root = dict_root(dict);
- dnode_t *nil = dict_nil(dict);
- dnode_t *saved;
- int result;
-
- /* simple binary search adapted for trees that contain duplicate keys */
-
- while (root != nil) {
-#ifdef BE_QSORT_COMPATIBLE
- result = dict->compare(&key, &root->key);
-#else
- result = dict->compare(key, root->key);
-#endif
- if (result < 0)
- root = root->left;
- else if (result > 0)
- root = root->right;
- else {
- if (!dict->dupes) { /* no duplicates, return match */
- return root;
- } else { /* could be dupes, find leftmost one */
- do {
- saved = root;
- root = root->left;
-#ifdef BE_QSORT_COMPATIBLE
- while (root != nil && dict->compare(&key, &root->key))
- root = root->right;
-#else
- while (root != nil && dict->compare(key, root->key))
- root = root->right;
-#endif
- } while (root != nil);
- return saved;
- }
- }
- }
-
- return NULL;
-}
-
-/*
- * Look for the node corresponding to the lowest key that is equal to or
- * greater than the given key. If there is no such node, return null.
- */
-
-dnode_t *dict_lower_bound(dict_t *dict, const void *key)
-{
- dnode_t *root = dict_root(dict);
- dnode_t *nil = dict_nil(dict);
- dnode_t *tentative = 0;
-
- while (root != nil) {
-#ifdef BE_QSORT_COMPATIBLE
- int result = dict->compare(&key, &root->key);
-#else
- int result = dict->compare(key, root->key);
-#endif
-
- if (result > 0) {
- root = root->right;
- } else if (result < 0) {
- tentative = root;
- root = root->left;
- } else {
- if (!dict->dupes) {
- return root;
- } else {
- tentative = root;
- root = root->left;
- }
- }
- }
-
- return tentative;
-}
-
-/*
- * Look for the node corresponding to the greatest key that is equal to or
- * lower than the given key. If there is no such node, return null.
- */
-
-dnode_t *dict_upper_bound(dict_t *dict, const void *key)
-{
- dnode_t *root = dict_root(dict);
- dnode_t *nil = dict_nil(dict);
- dnode_t *tentative = 0;
-
- while (root != nil) {
-#ifdef BE_QSORT_COMPATIBLE
- int result = dict->compare(&key, &root->key);
-#else
- int result = dict->compare(key, root->key);
-#endif
-
- if (result < 0) {
- root = root->left;
- } else if (result > 0) {
- tentative = root;
- root = root->right;
- } else {
- if (!dict->dupes) {
- return root;
- } else {
- tentative = root;
- root = root->right;
- }
- }
- }
-
- return tentative;
-}
-
-/*
- * Insert a node into the dictionary. The node should have been
- * initialized with a data field. All other fields are ignored.
- * The behavior is undefined if the user attempts to insert into
- * a dictionary that is already full (for which the dict_isfull()
- * function returns true).
- */
-
-void dict_insert(dict_t *dict, dnode_t *node, const void *key)
-{
- dnode_t *where = dict_root(dict), *nil = dict_nil(dict);
- dnode_t *parent = nil, *uncle, *grandpa;
- int result = -1;
-
- node->key = key;
-
- assert (!dict_isfull(dict));
- assert (!dict_contains(dict, node));
- assert (!dnode_is_in_a_dict(node));
-
- /* basic binary tree insert */
-
- while (where != nil) {
- parent = where;
-#ifdef BE_QSORT_COMPATIBLE
- result = dict->compare(&key, &where->key);
-#else
- result = dict->compare(key, where->key);
-#endif
- /* trap attempts at duplicate key insertion unless it's explicitly allowed */
- assert (dict->dupes || result != 0);
- if (result < 0)
- where = where->left;
- else
- where = where->right;
- }
-
- assert (where == nil);
-
- if (result < 0)
- parent->left = node;
- else
- parent->right = node;
-
- node->parent = parent;
- node->left = nil;
- node->right = nil;
-
- dict->nodecount++;
-
- /* red black adjustments */
-
- node->color = dnode_red;
-
- while (parent->color == dnode_red) {
- grandpa = parent->parent;
- if (parent == grandpa->left) {
- uncle = grandpa->right;
- if (uncle->color == dnode_red) { /* red parent, red uncle */
- parent->color = dnode_black;
- uncle->color = dnode_black;
- grandpa->color = dnode_red;
- node = grandpa;
- parent = grandpa->parent;
- } else { /* red parent, black uncle */
- if (node == parent->right) {
- rotate_left(parent);
- parent = node;
- assert (grandpa == parent->parent);
- /* rotation between parent and child preserves grandpa */
- }
- parent->color = dnode_black;
- grandpa->color = dnode_red;
- rotate_right(grandpa);
- break;
- }
- } else { /* symmetric cases: parent == parent->parent->right */
- uncle = grandpa->left;
- if (uncle->color == dnode_red) {
- parent->color = dnode_black;
- uncle->color = dnode_black;
- grandpa->color = dnode_red;
- node = grandpa;
- parent = grandpa->parent;
- } else {
- if (node == parent->left) {
- rotate_right(parent);
- parent = node;
- assert (grandpa == parent->parent);
- }
- parent->color = dnode_black;
- grandpa->color = dnode_red;
- rotate_left(grandpa);
- break;
- }
- }
- }
-
- dict_root(dict)->color = dnode_black;
-
- assert (dict_verify(dict));
-}
-
-/*
- * Delete the given node from the dictionary. If the given node does not belong
- * to the given dictionary, undefined behavior results. A pointer to the
- * deleted node is returned.
- */
-
-dnode_t *dict_delete(dict_t *dict, dnode_t *delete)
-{
- dnode_t *nil = dict_nil(dict), *child, *delparent = delete->parent;
-
- /* basic deletion */
-
- assert (!dict_isempty(dict));
- assert (dict_contains(dict, delete));
-
- /*
- * If the node being deleted has two children, then we replace it with its
- * successor (i.e. the leftmost node in the right subtree.) By doing this,
- * we avoid the traditional algorithm under which the successor's key and
- * value *only* move to the deleted node and the successor is spliced out
- * from the tree. We cannot use this approach because the user may hold
- * pointers to the successor, or nodes may be inextricably tied to some
- * other structures by way of embedding, etc. So we must splice out the
- * node we are given, not some other node, and must not move contents from
- * one node to another behind the user's back.
- */
-
- if (delete->left != nil && delete->right != nil) {
- dnode_t *next = dict_next(dict, delete);
- dnode_t *nextparent = next->parent;
- dnode_color_t nextcolor = next->color;
-
- assert (next != nil);
- assert (next->parent != nil);
- assert (next->left == nil);
-
- /*
- * First, splice out the successor from the tree completely, by
- * moving up its right child into its place.
- */
-
- child = next->right;
- child->parent = nextparent;
-
- if (nextparent->left == next) {
- nextparent->left = child;
- } else {
- assert (nextparent->right == next);
- nextparent->right = child;
- }
-
- /*
- * Now that the successor has been extricated from the tree, install it
- * in place of the node that we want deleted.
- */
-
- next->parent = delparent;
- next->left = delete->left;
- next->right = delete->right;
- next->left->parent = next;
- next->right->parent = next;
- next->color = delete->color;
- delete->color = nextcolor;
-
- if (delparent->left == delete) {
- delparent->left = next;
- } else {
- assert (delparent->right == delete);
- delparent->right = next;
- }
-
- } else {
- assert (delete != nil);
- assert (delete->left == nil || delete->right == nil);
-
- child = (delete->left != nil) ? delete->left : delete->right;
-
- child->parent = delparent = delete->parent;
-
- if (delete == delparent->left) {
- delparent->left = child;
- } else {
- assert (delete == delparent->right);
- delparent->right = child;
- }
- }
-
- delete->parent = NULL;
- delete->right = NULL;
- delete->left = NULL;
-
- dict->nodecount--;
-
- assert (verify_bintree(dict));
-
- /* red-black adjustments */
-
- if (delete->color == dnode_black) {
- dnode_t *parent, *sister;
-
- dict_root(dict)->color = dnode_red;
-
- while (child->color == dnode_black) {
- parent = child->parent;
- if (child == parent->left) {
- sister = parent->right;
- assert (sister != nil);
- if (sister->color == dnode_red) {
- sister->color = dnode_black;
- parent->color = dnode_red;
- rotate_left(parent);
- sister = parent->right;
- assert (sister != nil);
- }
- if (sister->left->color == dnode_black
- && sister->right->color == dnode_black) {
- sister->color = dnode_red;
- child = parent;
- } else {
- if (sister->right->color == dnode_black) {
- assert (sister->left->color == dnode_red);
- sister->left->color = dnode_black;
- sister->color = dnode_red;
- rotate_right(sister);
- sister = parent->right;
- assert (sister != nil);
- }
- sister->color = parent->color;
- sister->right->color = dnode_black;
- parent->color = dnode_black;
- rotate_left(parent);
- break;
- }
- } else { /* symmetric case: child == child->parent->right */
- assert (child == parent->right);
- sister = parent->left;
- assert (sister != nil);
- if (sister->color == dnode_red) {
- sister->color = dnode_black;
- parent->color = dnode_red;
- rotate_right(parent);
- sister = parent->left;
- assert (sister != nil);
- }
- if (sister->right->color == dnode_black
- && sister->left->color == dnode_black) {
- sister->color = dnode_red;
- child = parent;
- } else {
- if (sister->left->color == dnode_black) {
- assert (sister->right->color == dnode_red);
- sister->right->color = dnode_black;
- sister->color = dnode_red;
- rotate_left(sister);
- sister = parent->left;
- assert (sister != nil);
- }
- sister->color = parent->color;
- sister->left->color = dnode_black;
- parent->color = dnode_black;
- rotate_right(parent);
- break;
- }
- }
- }
-
- child->color = dnode_black;
- dict_root(dict)->color = dnode_black;
- }
-
- assert (dict_verify(dict));
-
- return delete;
-}
-
-/*
- * Allocate a node using the dictionary's allocator routine, give it
- * the data item.
- */
-
-int dict_alloc_insert(dict_t *dict, const void *key, void *data)
-{
- dnode_t *node = dict->allocnode(dict->context);
-
- if (node) {
- dnode_init(node, data);
- dict_insert(dict, node, key);
- return 1;
- }
- return 0;
-}
-
-void dict_delete_free(dict_t *dict, dnode_t *node)
-{
- dict_delete(dict, node);
- dict->freenode(node, dict->context);
-}
-
-/*
- * Return the node with the lowest (leftmost) key. If the dictionary is empty
- * (that is, dict_isempty(dict) returns 1) a null pointer is returned.
- */
-
-dnode_t *dict_first(dict_t *dict)
-{
- dnode_t *nil = dict_nil(dict), *root = dict_root(dict), *left;
-
- if (root != nil)
- while ((left = root->left) != nil)
- root = left;
-
- return (root == nil) ? NULL : root;
-}
-
-/*
- * Return the node with the highest (rightmost) key. If the dictionary is empty
- * (that is, dict_isempty(dict) returns 1) a null pointer is returned.
- */
-
-dnode_t *dict_last(dict_t *dict)
-{
- dnode_t *nil = dict_nil(dict), *root = dict_root(dict), *right;
-
- if (root != nil)
- while ((right = root->right) != nil)
- root = right;
-
- return (root == nil) ? NULL : root;
-}
-
-/*
- * Return the given node's successor node---the node which has the
- * next key in the the left to right ordering. If the node has
- * no successor, a null pointer is returned rather than a pointer to
- * the nil node.
- */
-
-dnode_t *dict_next(dict_t *dict, dnode_t *curr)
-{
- dnode_t *nil = dict_nil(dict), *parent, *left;
-
- if (curr->right != nil) {
- curr = curr->right;
- while ((left = curr->left) != nil)
- curr = left;
- return curr;
- }
-
- parent = curr->parent;
-
- while (parent != nil && curr == parent->right) {
- curr = parent;
- parent = curr->parent;
- }
-
- return (parent == nil) ? NULL : parent;
-}
-
-/*
- * Return the given node's predecessor, in the key order.
- * The nil sentinel node is returned if there is no predecessor.
- */
-
-dnode_t *dict_prev(dict_t *dict, dnode_t *curr)
-{
- dnode_t *nil = dict_nil(dict), *parent, *right;
-
- if (curr->left != nil) {
- curr = curr->left;
- while ((right = curr->right) != nil)
- curr = right;
- return curr;
- }
-
- parent = curr->parent;
-
- while (parent != nil && curr == parent->left) {
- curr = parent;
- parent = curr->parent;
- }
-
- return (parent == nil) ? NULL : parent;
-}
-
-void dict_allow_dupes(dict_t *dict)
-{
- dict->dupes = 1;
-}
-
-#undef dict_count
-#undef dict_isempty
-#undef dict_isfull
-#undef dnode_get
-#undef dnode_put
-#undef dnode_getkey
-
-dictcount_t dict_count(dict_t *dict)
-{
- return dict->nodecount;
-}
-
-int dict_isempty(dict_t *dict)
-{
- return dict->nodecount == 0;
-}
-
-int dict_isfull(dict_t *dict)
-{
- return dict->nodecount == dict->maxcount;
-}
-
-int dict_contains(dict_t *dict, dnode_t *node)
-{
- return verify_dict_has_node(dict_nil(dict), dict_root(dict), node);
-}
-
-static dnode_t *dnode_alloc(void *context)
-{
- return malloc(sizeof *dnode_alloc(NULL));
-}
-
-static void dnode_free(dnode_t *node, void *context)
-{
- free(node);
-}
-
-dnode_t *dnode_create(void *data)
-{
- dnode_t *new = malloc(sizeof *new);
- if (new) {
- new->data = data;
- new->parent = NULL;
- new->left = NULL;
- new->right = NULL;
- }
- return new;
-}
-
-dnode_t *dnode_init(dnode_t *dnode, void *data)
-{
- dnode->data = data;
- dnode->parent = NULL;
- dnode->left = NULL;
- dnode->right = NULL;
- return dnode;
-}
-
-void dnode_destroy(dnode_t *dnode)
-{
- assert (!dnode_is_in_a_dict(dnode));
- free(dnode);
-}
-
-void *dnode_get(dnode_t *dnode)
-{
- return dnode->data;
-}
-
-const void *dnode_getkey(dnode_t *dnode)
-{
- return dnode->key;
-}
-
-void dnode_put(dnode_t *dnode, void *data)
-{
- dnode->data = data;
-}
-
-int dnode_is_in_a_dict(dnode_t *dnode)
-{
- return (dnode->parent && dnode->left && dnode->right);
-}
-
-void dict_process(dict_t *dict, void *context, dnode_process_t function)
-{
- dnode_t *node = dict_first(dict), *next;
-
- while (node != NULL) {
- /* check for callback function deleting */
- /* the next node from under us */
- assert (dict_contains(dict, node));
- next = dict_next(dict, node);
- function(dict, node, context);
- node = next;
- }
-}
-
-static void load_begin_internal(dict_load_t *load, dict_t *dict)
-{
- load->dictptr = dict;
- load->nilnode.left = &load->nilnode;
- load->nilnode.right = &load->nilnode;
-}
-
-void dict_load_begin(dict_load_t *load, dict_t *dict)
-{
- assert (dict_isempty(dict));
- load_begin_internal(load, dict);
-}
-
-void dict_load_next(dict_load_t *load, dnode_t *newnode, const void *key)
-{
- dict_t *dict = load->dictptr;
- dnode_t *nil = &load->nilnode;
-
- assert (!dnode_is_in_a_dict(newnode));
- assert (dict->nodecount < dict->maxcount);
-
-#ifndef NDEBUG
- if (dict->nodecount > 0) {
-#ifdef BE_QSORT_COMPATIBLE
- if (dict->dupes)
- assert (dict->compare(&nil->left->key, &key) <= 0);
- else
- assert (dict->compare(&nil->left->key, &key) < 0);
-#else
- if (dict->dupes)
- assert (dict->compare(nil->left->key, key) <= 0);
- else
- assert (dict->compare(nil->left->key, key) < 0);
-#endif
- }
-#endif
-
- newnode->key = key;
- nil->right->left = newnode;
- nil->right = newnode;
- newnode->left = nil;
- dict->nodecount++;
-}
-
-void dict_load_end(dict_load_t *load)
-{
- dict_t *dict = load->dictptr;
- dnode_t *tree[DICT_DEPTH_MAX] = { 0 };
- dnode_t *curr, *dictnil = dict_nil(dict), *loadnil = &load->nilnode, *next;
- dnode_t *complete = 0;
- dictcount_t fullcount = DICTCOUNT_T_MAX, nodecount = dict->nodecount;
- dictcount_t botrowcount;
- unsigned baselevel = 0, level = 0, i;
-
- assert (dnode_red == 0 && dnode_black == 1);
-
- while (fullcount >= nodecount && fullcount)
- fullcount >>= 1;
-
- botrowcount = nodecount - fullcount;
-
- for (curr = loadnil->left; curr != loadnil; curr = next) {
- next = curr->left;
-
- if (complete == NULL && botrowcount-- == 0) {
- assert (baselevel == 0);
- assert (level == 0);
- baselevel = level = 1;
- complete = tree[0];
-
- if (complete != 0) {
- tree[0] = 0;
- complete->right = dictnil;
- while (tree[level] != 0) {
- tree[level]->right = complete;
- complete->parent = tree[level];
- complete = tree[level];
- tree[level++] = 0;
- }
- }
- }
-
- if (complete == NULL) {
- curr->left = dictnil;
- curr->right = dictnil;
- curr->color = level % 2;
- complete = curr;
-
- assert (level == baselevel);
- while (tree[level] != 0) {
- tree[level]->right = complete;
- complete->parent = tree[level];
- complete = tree[level];
- tree[level++] = 0;
- }
- } else {
- curr->left = complete;
- curr->color = (level + 1) % 2;
- complete->parent = curr;
- tree[level] = curr;
- complete = 0;
- level = baselevel;
- }
- }
-
- if (complete == NULL)
- complete = dictnil;
-
- for (i = 0; i < DICT_DEPTH_MAX; i++) {
- if (tree[i] != 0) {
- tree[i]->right = complete;
- complete->parent = tree[i];
- complete = tree[i];
- }
- }
-
- dictnil->color = dnode_black;
- dictnil->right = dictnil;
- complete->parent = dictnil;
- complete->color = dnode_black;
- dict_root(dict) = complete;
-
- assert (dict_verify(dict));
-}
-
-void dict_merge(dict_t *dest, dict_t *source)
-{
- dict_load_t load;
- dnode_t *leftnode = dict_first(dest), *rightnode = dict_first(source);
-
- assert (dict_similar(dest, source));
-
- if (source == dest)
- return;
-
- dest->nodecount = 0;
- load_begin_internal(&load, dest);
-
- for (;;) {
- if (leftnode != NULL && rightnode != NULL) {
-#ifdef BE_QSORT_COMPATIBLE
- if (dest->compare(&leftnode->key, &rightnode->key) < 0)
- goto copyleft;
- else
- goto copyright;
-#else
- if (dest->compare(leftnode->key, rightnode->key) < 0)
- goto copyleft;
- else
- goto copyright;
-#endif
- } else if (leftnode != NULL) {
- goto copyleft;
- } else if (rightnode != NULL) {
- goto copyright;
- } else {
- assert (leftnode == NULL && rightnode == NULL);
- break;
- }
-
- copyleft:
- {
- dnode_t *next = dict_next(dest, leftnode);
- #ifndef NDEBUG
- leftnode->left = NULL; /* suppress assertion in dict_load_next */
- #endif
- dict_load_next(&load, leftnode, leftnode->key);
- leftnode = next;
- continue;
- }
-
- copyright:
- {
- dnode_t *next = dict_next(source, rightnode);
-#ifndef NDEBUG
- rightnode->left = NULL;
-#endif
- dict_load_next(&load, rightnode, rightnode->key);
- rightnode = next;
- continue;
- }
- }
-
- dict_clear(source);
- dict_load_end(&load);
-}
diff --git a/kmer/libutil/kazlib/dict.h b/kmer/libutil/kazlib/dict.h
deleted file mode 100644
index 2bab634..0000000
--- a/kmer/libutil/kazlib/dict.h
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Dictionary Abstract Data Type
- * Copyright (C) 1997 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#ifndef DICT_H
-#define DICT_H
-
-#include <limits.h>
-#ifdef KAZLIB_SIDEEFFECT_DEBUG
-#include "sfx.h"
-#endif
-
-/*
- * Blurb for inclusion into C++ translation units
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef unsigned long dictcount_t;
-#define DICTCOUNT_T_MAX ULONG_MAX
-
-/*
- * The dictionary is implemented as a red-black tree
- */
-
-typedef enum { dnode_red, dnode_black } dnode_color_t;
-
-typedef struct dnode_t {
-#if defined(DICT_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- struct dnode_t *dict_left;
- struct dnode_t *dict_right;
- struct dnode_t *dict_parent;
- dnode_color_t dict_color;
- const void *dict_key;
- void *dict_data;
-#else
- int dict_dummy;
-#endif
-} dnode_t;
-
-typedef int (*dict_comp_t)(const void *, const void *);
-typedef dnode_t *(*dnode_alloc_t)(void *);
-typedef void (*dnode_free_t)(dnode_t *, void *);
-
-typedef struct dict_t {
-#if defined(DICT_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- dnode_t dict_nilnode;
- dictcount_t dict_nodecount;
- dictcount_t dict_maxcount;
- dict_comp_t dict_compare;
- dnode_alloc_t dict_allocnode;
- dnode_free_t dict_freenode;
- void *dict_context;
- int dict_dupes;
-#else
- int dict_dummmy;
-#endif
-} dict_t;
-
-typedef void (*dnode_process_t)(dict_t *, dnode_t *, void *);
-
-typedef struct dict_load_t {
-#if defined(DICT_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- dict_t *dict_dictptr;
- dnode_t dict_nilnode;
-#else
- int dict_dummmy;
-#endif
-} dict_load_t;
-
-extern dict_t *dict_create(dictcount_t, dict_comp_t);
-extern void dict_set_allocator(dict_t *, dnode_alloc_t, dnode_free_t, void *);
-extern void dict_destroy(dict_t *);
-extern void dict_free_nodes(dict_t *);
-extern void dict_free(dict_t *);
-extern dict_t *dict_init(dict_t *, dictcount_t, dict_comp_t);
-extern void dict_init_like(dict_t *, const dict_t *);
-extern int dict_verify(dict_t *);
-extern int dict_similar(const dict_t *, const dict_t *);
-extern dnode_t *dict_lookup(dict_t *, const void *);
-extern dnode_t *dict_lower_bound(dict_t *, const void *);
-extern dnode_t *dict_upper_bound(dict_t *, const void *);
-extern void dict_insert(dict_t *, dnode_t *, const void *);
-extern dnode_t *dict_delete(dict_t *, dnode_t *);
-extern int dict_alloc_insert(dict_t *, const void *, void *);
-extern void dict_delete_free(dict_t *, dnode_t *);
-extern dnode_t *dict_first(dict_t *);
-extern dnode_t *dict_last(dict_t *);
-extern dnode_t *dict_next(dict_t *, dnode_t *);
-extern dnode_t *dict_prev(dict_t *, dnode_t *);
-extern dictcount_t dict_count(dict_t *);
-extern int dict_isempty(dict_t *);
-extern int dict_isfull(dict_t *);
-extern int dict_contains(dict_t *, dnode_t *);
-extern void dict_allow_dupes(dict_t *);
-extern int dnode_is_in_a_dict(dnode_t *);
-extern dnode_t *dnode_create(void *);
-extern dnode_t *dnode_init(dnode_t *, void *);
-extern void dnode_destroy(dnode_t *);
-extern void *dnode_get(dnode_t *);
-extern const void *dnode_getkey(dnode_t *);
-extern void dnode_put(dnode_t *, void *);
-extern void dict_process(dict_t *, void *, dnode_process_t);
-extern void dict_load_begin(dict_load_t *, dict_t *);
-extern void dict_load_next(dict_load_t *, dnode_t *, const void *);
-extern void dict_load_end(dict_load_t *);
-extern void dict_merge(dict_t *, dict_t *);
-
-#if defined(DICT_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
-#ifdef KAZLIB_SIDEEFFECT_DEBUG
-#define dict_isfull(D) (SFX_CHECK(D)->dict_nodecount == (D)->dict_maxcount)
-#else
-#define dict_isfull(D) ((D)->dict_nodecount == (D)->dict_maxcount)
-#endif
-#define dict_count(D) ((D)->dict_nodecount)
-#define dict_isempty(D) ((D)->dict_nodecount == 0)
-#define dnode_get(N) ((N)->dict_data)
-#define dnode_getkey(N) ((N)->dict_key)
-#define dnode_put(N, X) ((N)->dict_data = (X))
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/kmer/libutil/kazlib/docs/CHANGES b/kmer/libutil/kazlib/docs/CHANGES
deleted file mode 100644
index 3c949eb..0000000
--- a/kmer/libutil/kazlib/docs/CHANGES
+++ /dev/null
@@ -1,290 +0,0 @@
-New in 1.20
-
- 1. Bugfix in except.h. Modified non-volatile auto variables were
- being accessed after longjmp.
-
-New in 1.19
-
- 1. Rewrite of broken dict_free.
- 2. Fixed embarassing build breakages that accidentally went into 1.18
- 3. Function hash_scan_delete_free renamed to hash_scan_delfree to be
- distinct from hash_scan_delete in the first 14 characters.
- 4. To resolve inconsistencies between hash_free and dict_free,
- and a difference between the actual behavior of hash_free and
- the documented behavior, these two functions are marked obsolescent.
- The functions dict_free_nodes and hash_free_nodes are provided.
- The obsolescent functions continue to work as before, for now.
- 5. Documentation of hash_free is fixed to say that it also subjects
- the hash to hash_destroy, which is what the implementation does.
- 6. Documentation states what release it is for.
-
-New in 1.18
-
- 1. Error in assert expression in list_merge fixed.
- 2. Semantics of list_merge extended to allow list to be merged
- onto itself (which is a noop).
- 3. Clarified interface specification of list_transfer and list_extract;
- the source and destination list may be the same object.
- 4. New functions:
- dict_init_like: create a dictionary similar to another one;
- dict_similar: determine whether two dictionaries are similar;
- dict_merge: merge contents of one dictionary to another.
- 5. Dictionary test main can juggle multiple dictionaries, and test
- dict_merge.
- 6. If a hash node is inserted into some hash, it is a now a constraint
- violation to insert it again into some hash.
- 7. The hash_scan_delete_free function has been implemented; it is to
- hash_scan_delete what hash_delete_free is to hash_delete.
-
-New in 1.17
-
- Carl van Tast <vanTast at netway.at>:
- 1. Removed references to ``safe malloc'' from some comments.
- 2. Swapped ``allowed'' and ``not allowed'' in comment to
- verify_bintree.
- 3. Fixed comment to list_next: this function never returns the
- sentinel.
- 4. lnode_pool_init: nodes[i].prev = nodes instead of nodes + 1. This
- saves one or two CPU cycles :-) and it gives a valid address even
- if we have a (somewhat pathological) pool with just one element.
-
- Kaz:
- 5. Dropped extra parameter from tree rotation functions in dict.c. Should
- shave a few cycles.
- 6. Fixed error in the duplicate key iteration idiom example in the
- documentation (see the section on dict_upper_bound).
- 7. Forgotten #include <string.h> added to hash.c
-
-New in 1.16
-
- 1. Added an interface for loading the contents of a dictionary from an
- ordered sequence. This is done in O(n) time by a direct bottom-up
- construction of the red-black tree, making it much faster than
- the O(n log n) process of inserting each element.
- 2. Miscellaneous cleanup: missing const qualifiers were added
- to key pointer parameters, some incorrect comments fixed;
- spelling errors corrected in documentation.
-
-New in 1.15
-
- 1. Another potential exception handling memory leak fixed. This one
- has to do with throwing an exception from within a try-catch region
- in which an exception was just caught. The new exception replaces
- the old without the old's dynamic memory being disposed of.
- 2. Restrictions added on except_rethrow.
- 3. Exception module must now be explicitly initialized with except_init.
- 4. Structure members in exception header renamed to adhere to documented
- namespace.
- 5. The exwrap.[ch] source files are gone. There is support for memory
- allocation with exception handling in except.c, which supports user
- defined allocators.
- 6. Three bugfixes to sfx parser. First, unary operators take a cast
- expression, not a unary expression. Secondly, sizeof doesn't throw a syntax
- error anymore on things that look like casts, but maybe are not.
- Thirdly, empty parentheses weren't handled right in treatment of
- ambiguous expressions, e.g. (a)() was declared a syntax error.
- 7. Changed the representation of hash table chains. They are now
- singly linked lists, which means that the overhead of managing
- back pointers is gone. Only deletion is slightly more complicated
- now because it has to search from the beginning of the chain.
- [Rationale: this is okay, since chains are supposed to be short
- in a hash table!]
- 8. Rewritten test main() in list.c. It's now more like the others
- with a menu. Previously it was essentially a file sorting program.
- 9. New function: list_find. Exhaustively searches the list for a
- matching entry, returns pointer to node if found.
-
-New in 1.14
-
- 1. Got rid of some overbearing copyright restrictions. There is no need for
- executables to contain copyright notices. In fact, there are no
- restrictions on the use, or distribution in executable form.
- 2. Tiny tweak in red-black fixup code of dict_insert.
- 3. Keys in hash and dict are declared const void * now in all functions
- rather than plain void *. This means that casts are no longer
- necessary when calling insert or lookup functions with const
- data as the key. But casts of the return value of hnode_getkey
- or dnode_getkey may be required.
- 4. Fixed compile breakage of except.c when posix thread support enabled.
- 5. Side effect assertion interface now performs caching, to avoid
- parsing the same expressions over and over again. Thus debugging with
- KAZLIB_SIDEEFFECT_DEBUG incurs a smaller performance hit.
- 6. Major bugfix to sfx expression parser. The function dealing with
- disambiguating casts had to be rewritten to do more sophisticated
- lookahead and backtracking. It all started with Mark Brady discovered
- that (a++)+b was being incorrectly diagnosed as a syntax error.
- 7. Added documentation. more examples for uses of dictionaries, and
- exception handling. Some documentation about the internals
- of exception handling added. Changed document format for narrower
- margins, reducing page count and increasing readability.
- 8. Bugfix in except_rethrow. It was freeing the dynamic data of the
- exception even though it's not handled yet.
-
-New in 1.13
-
- 1. Fixed some potential memory leaks in except.c.
- 2. Finished all interface documentation. All that is left now
- is to flesh out the implementation notes.
- 3. Fixed a bug in POSIX threaded variant of except.c. Null
- function pointer dereference in unhandled exception case.
- 4. Macros beginning with E[A-Z] have been renamed to stay out
- of space reserved for <errno.h>.
- 5. Identifiers in exwrap.[ch] have been renamed from having
- ex_ prefixed to having exwrap_ prefixes.
-
-New in 1.12
-
- 1. COOL! New module for detecting side effects in C expressions.
- 2. Serious bugfix in hash_init(). The computation of the initial hash
- mask was completely botched up. Historically this code has seen little
- testing because hashing over a user supplied table is not extendible.
- Users of hash_create() are not affected.
- 3. Tried to make computation of hash_val_t_bit more threadsafe. It should
- be okay if writes to int objects are atomic, and concurrent writes of
- the same int value to a given object are safe.
- 4. Makefile renamed to Makefile.gcc. Makevile.vc added. The rename
- is retroactive to all prior releases.
- 5. OPAQUE_DEBUG becomes KAZLIB_OPAQUE_DEBUG and TEST_MAIN becomes
- KAZLIB_TEST_MAIN. In general, macros that affect how the modules
- build should be confined to a special namespace.
- 6. New KAZLIB_SIDEEFFECT_DEBUG feature to enable diagnosis of side
- effect expressions being passed to macros that evaluate their arguments
- more than once.
-
-New in 1.11
-
- 1. Improvements in experimental exception handling module:
- except_throwf has been added which takes printf-like arguments;
- except_checked_cleanup_pop has been added to provide a measure
- of safety; there is now a way to pass arbitrary data from the throw site
- to the catch.
- 2. Improvements in dict_insert. A redundant call to the comparison function
- has been eliminated, resulting in one fewer comparisons per insert
- operation! Also a redundant test has been removed from the controlling
- expression of the fixup loop, taking advantage of the fact that nil
- is always black, and hence the root node always has a black parent.
- 3. Small change in dict_delete. A test in the fixup loop has been eliminated
- by temporarily coloring the root node red. See comment and diff between
- dict.c revision 1.25 and 1.26.
- 4. Test program blast.pl deletes keys out of order; to get in order
- delete, initialize $factor_d to 1.
-
-New in 1.10
-
- 1. The dict_init function now correctly initializes allocator-related
- members of the dict structure.
- 2. Tiny optimization in dict_lookup---less frequent cases tested last.
- 3. Added list_extract, for extracting list slices (more general than
- list_transfer).
- 4. Incorporated changes from Loic Dachary: hash_free() has been
- added for deleting all nodes; hash and compare functions
- from the hash.c test code are now available to the user as
- defaults if null pointers are given to hash_init() or
- hash_create(); and hash_set_allocator restores the default
- allocator routines if null pointers are given to it.
- 5. Changes to dict analogous to hash: dict_free() added, etc.
- 6. New exception handling module added (experimental).
- 7. Much new documentation.
-
-New in 1.9
-
- 1. Third argument of list_transfer may be null, in which case no nodes
- are transferred. [Rationale: allows empty source list to be treated
- without special case testing when all nodes are being transferred.]
- 2. Two new functions added to dict: dict_upper_bound and dict_lower_bound.
- These allow for inexact and range searches.
-
-New in 1.8
-
- 1. New improved hashing function in the hash.c test code. It turns out that
- when I changed the hash table algorithm, the blast.pl testcase was
- hashing all to a single chain due to the pathologically bad hashing
- function. The new hashing function should be good enough for general use.
- It uses each nybble of the key to index a table of 16 random 32 bit integers.
- These integers are XOR-ed into the hash value which is rotated after each
- XOR.
- 2. Spurious semicolon removed from the #define of HASH_VAL_T_BIT.
- 3. I fixed some incorrect comments in hash.c which still talked about the
- old algorithm from release 1.5 and older.
- 4. The smalloc.c module is no longer supported. It's still in RCS but it's not
- tagged as being part of release 1.8, and is not used by any of the other
- sources. The standard library memory allocation functions are now used
- directly. [Rationale: smalloc.c is overkill and interferes with
- integration of the other source files into projects. Conscientious programmer
- already ahve their own tools for debugging allocator corruption, anyway.]
-
-New in 1.7
-
- 1. Missing #include <stdlib.h> added to smalloc.h
- 2. The dict_delete() functions internals have been changed to make it much
- more sane. This function no longer has the potential to return a node
- other than the one that is passed to it.
- 3. The changes to dict_delete() also fix a serious bug in dict_process().
- The dict_process computes a pointer to a node's successor before
- invoking the user callback to process a node. If the user callback calls
- dict_delete() on the node, under the old dict_delete() semantics it was
- possible for the successor to get deleted instead. Thus dict_process()
- could end up with an invalid pointer.
- 4. The changes to dict_delete() also mean that key and value information will
- never be relocated from one node to another. User code can now rely on this
- convenient assumption.
-
-New in 1.6
-
- 1. The extendible hashing algorithm internals have changed. This
- has a potential impact on the behavior with respect to hashing functions
- which were written to work well specifically with the old hashing
- scheme. For a silly reason, in the old hashing scheme, the top N bits
- were always taken from the results of a hashing function, for a hash
- table size of 2^N chains. In the new scheme, the bottom N bits are taken
- instead. [Rationale: This is change makes it easier to write portable
- hashing functions and simplifies the functions that expand or contract
- the table, making them more efficient.]
- 2. Added const qualifiers to the rcsid[] and right[] char arrays,
- which shuts up the GCC compiler from complaining that these are
- unused statics.
-
-New in 1.5
-
- 1. First two arguments to list_prune_graft() are reversed. The leftmost
- argument is now the destination list. Moreover, the function has been
- renamed list_transfer(). [Rationale: this ordering of parameters is
- consistent with list_merge(), and the standard C <string.h> functions
- also pass destination pointers on the left. Renaming the function
- protects against incorrect use.]
-
- 2. Red-Black tree dictionaries now support duplicate keys. [Rationale:
- duplicate keys could be useful in some applications.] When a dictionary
- is created or initialized, it does not allow duplicate keys. The
- function dict_allow_dupes() is used to set a flag in a dictionary to
- henceforth allow duplicates. Once made, the decision to allow
- duplicates cannot be reversed. [Rationale: toggling between allowing
- and disallowing duplicates does not seem useful. Once duplicates are
- admitted, there is no point in disallowing duplicates.] When a key is
- sought in tree that currently allows duplicates, the leftmost node
- containing that key is chosen from among the nodes that contain
- duplicates of the key. Then dict_next() can be used to fetch the
- remaining duplicates one by one. No particular order among the
- duplicates may be assumed. However, for what it may be worth, the order
- between any two duplicates is preserved for as long as they both remain
- in the dictionary.
-
- 3. The function prototypes in the header files have been modified to eliminate
- parameter names. [Rationale: parameter names in prototypes have only
- documentary value, and may clash with macro identifiers defined in other
- headers.]
-
- 4. Dictionary and hash table now has support for automatic allocation of
- nodes in the insert and delete operations, which means that the user
- can add items in one operation instead of the two operations of
- allocating a node and inserting it. [Rationale: ease of use.] There is
- support for user-defined allocators; the default allocators use the
- smalloc.c routines. For any instance of a dict_t or hash_t object, the
- user can override the allocator functions by supplying his or her
- own pointers to suitable functions, and a context pointer that
- will be passed to these functions when they are called through that
- particular dict_t or hash_t instance. [Rationale: flexibility, ease of
- use, promotes good design.] The funtion pointers can only be set when
- the data structure is empty. [Rationale: it is undesirable to switch to
- a different allocator when there are nodes in the dictionary; it might
- lead to the error of freeing a node with an incorrect allocator.]
diff --git a/kmer/libutil/kazlib/docs/MUST_READ b/kmer/libutil/kazlib/docs/MUST_READ
deleted file mode 100644
index 20ca12e..0000000
--- a/kmer/libutil/kazlib/docs/MUST_READ
+++ /dev/null
@@ -1,25 +0,0 @@
-Greetings, Programmer!
-
-I gather that because you are reading this, you are probably considering using
-the C language translation units included here in your own software. If that
-is the case, I would like to know who you are and urge you to contact me.
-
-Here is why: I rove over this code periodically looking for defects. In fact,
-I use it in my own programming projects. If I discover a defect, I will
-notify everyone who I know is a user of this software. If there is a serious
-defect in some code that you are using in your software project, wouldn't you
-want to be informed? In fact, there is no question that you _need_ to be
-informed!
-
-Here is what you do: simply send an e-mail message to kaz at ashi.footprints.net
-with the subject "kazlib" and the body "I am a user". Be sure that your message
-has a good return address. I will manually add your e-mail address to a list
-which I will use only for the purpose of notifications regarding Kazlib. You
-will receive a reply to the effect that you are added.
-
-If ever you should wish to be removed from this list, simply ask and it shall
-be done.
-
-Yours in earnest,
-
- Kaz Kylheku
diff --git a/kmer/libutil/kazlib/docs/README b/kmer/libutil/kazlib/docs/README
deleted file mode 100644
index 08f14a1..0000000
--- a/kmer/libutil/kazlib/docs/README
+++ /dev/null
@@ -1,66 +0,0 @@
-This collection of data structures is maintained by
-Kaz Kylheku <kaz at ashi.footprints.net>
-
-INSTRUCTIONS
-
-Simply add the necessary .c and .h files to your project. Include the
-appropriate .h file in any translation unit that interfaces with one or more of
-the kazlib modules. Then compile and link the modules together with your program.
-
-To use kazlib in a C++ project, don't compile them with a C++ compiler.
-Compile with a C compiler, and include the header files in
-your C++ translation units. Then link together the translated C and C++.
-As of release 1.2, the header files should work with C++.
-
-IMPORTANT NOTES
-
-1. Self checks
-
-The modules in this collection perform extensive self-checks, some of
-which make the performance really poor (by actually raising the overall
-asymptotic complexity of an operation, for example from O(log N) to O(N). The
-instrumentation assertions can be disabled by compiling with the NDEBUG macro
-defined.
-
-You can check that your project does not violate the principles of
-implementation hiding in connection with its use of the kazlib modules. This
-is accomplished by defining the macro KAZLIB_OPAQUE_DEBUG at the beginning of
-any translation unit which includes the kazlib header files. Note that
-whereas this will detect violations, it will not result in a translation
-that can be linked against the kazlib. When you are done checking, turn
-off KAZLIB_OPAQUE_DEBUG and recompile. If your compiler has a special ``check only''
-mode which enables it to perform syntax and type checking without doing
-an actual translation (similar to lint), it may be a time-saving idea to
-use it in conjunction with KAZLIB_OPAQUE_DEBUG.
-
-2. Macros with side effects
-
-Some of the kazlib header files define macros that evaluate their arguments
-more than once. This means that if expressions with side effects are passed
-to these macros, undesirable and undefined behavior will happen. There is
-support in Kazlib for catching these kinds of bugs: compile with
-KAZLIB_SIDEEFFECT_DEBUG, and add the except.c and sfx.c modules to your
-object. The macros will now parse their expressions at run time to diagnose
-the presence of side effects and function calls. It's easy to add this support
-to your own code!
-
-3. Thread support
-
-POSIX thread support is enabled by predefining KAZLIB_POSIX_THREADS. Currently
-only the exception-handling module has any need for this. When compiled that
-way, it provides thread-safe exception handling. Threads can independently
-throw exceptions and each thread can install its own specific catcher
-for unhandled exceptions. Moreover, each thread can register its own
-memory allocator functions.
-
-Note: this variant of the code also depends on the ability to cast between void
-* and function pointers, which is a common language extension.
-
-4. CVS identification
-
-The source files contain declarations of a static char array variable called
-rcsid. This contains an expansion of the CVS identification of each module,
-making it possible to determine the ``bill of materials'' that went into an
-executable build. I have now wrapped the declarations of these rcsid[] arrays
-so they are conditional on KAZLIB_RCSID being defined. For many users, these
-are just a waste of space.
diff --git a/kmer/libutil/kazlib/docs/docs.ist b/kmer/libutil/kazlib/docs/docs.ist
deleted file mode 100644
index 808c029..0000000
--- a/kmer/libutil/kazlib/docs/docs.ist
+++ /dev/null
@@ -1,4 +0,0 @@
-preamble
-"\\begin{theindex}\n\\addcontentsline{toc}{section}{Index}\n"
-postamble
-"\n\\end{theindex}\n"
diff --git a/kmer/libutil/kazlib/docs/docs.ltx b/kmer/libutil/kazlib/docs/docs.ltx
deleted file mode 100644
index 139f212..0000000
--- a/kmer/libutil/kazlib/docs/docs.ltx
+++ /dev/null
@@ -1,4155 +0,0 @@
-\documentclass{article}
-\usepackage{makeidx}
-\usepackage[margin=1.0in]{geometry}
-\makeatletter
-\newcommand{\defsubsection}{\@startsection
- {subsection}
- {2}
- {0pt}
- {2.0ex plus 0.1ex minus 0.05ex}
- {-0pt}
- {\normalfont\normalsize\bfseries}}
-\newcommand{\defsubsubsection}{\@startsection
- {subsection}
- {3}
- {0ex}
- {2.0ex plus 0.1ex minus 0.05ex}
- {1.0ex}
- {\normalfont\normalsize\bfseries}}
-\renewcommand{\paragraph}{\@startsection
- {paragraph}
- {4}
- {0ex}
- {2.0ex plus 0.1ex minus 0.05ex}
- {1.0ex}
- {\normalsize\bfseries}}
-\makeatother
-\title{Kazlib---Reusable Components\\for C Programming}
-\author{Kaz Kylheku}
-\date{Release 1.20\\July 24, 2001}
-\makeindex
-\setcounter{tocdepth}{1}
-\setcounter{secnumdepth}{4}
-\begin{document}
-\catcode`\_=11
-\def\indextype#1{\index{#1@{\tt #1} type}}
-\def\indexmacro#1{\index{#1@{\tt #1} macro}}
-\def\indexobject#1{\index{#1@{\tt #1} object}}
-\def\indexfunc#1{\index{#1@{\tt #1} function}}
-\def\indexenum#1{\index{#1@{\tt #1} enum constant}}
-\def\synopsis{\paragraph*{Synopsis}}
-\def\constraints{\paragraph*{Constraints}}
-\def\description{\paragraph*{Description}}
-\def\example{\paragraph*{Example}}
-\maketitle
-\abstract{The aim of the Kazlib project is to provide a well-documented
-programming interface featuring commonly needed programming abstractions,
-accompanied by a high quality, portable reference implementation.
-Kazlib consists of four independent components: a list module, a hash table
-module, a dictionary module and an exception handling module. The reference
-implementations of the first three of these are based on, respectively, the
-following algorithms: doubly linked circular list with sentinel node,
-extendible hashing, and red-black tree.}
-\tableofcontents
-\section{Introduction}
-This document establishes the provisions required of an implementation of the
-Kazlib library, and describes a reference implementation thereof.
-This document specifies
-\begin{itemize}
-\item the names and types of identifiers and preprocessor symbols made
- available by each component;
-\item identifier name spaces reserved for future use by each component;
-\item the interface syntax and semantics of each component operation;
-\item the conditions required for the well-defined execution of each operation;
-\item the externally visible behavior of each component, including global
- side effects and the effects on the subject data structures;
-
-\item and the implementation language of Kazlib.
-\end{itemize}
-Furthermore, this document describes, but does not specify
-\begin{itemize}
-\item the implementation details of structure objects manipulated by the
- operations of each component;
-\item objects and functions that are defined by the implementation of
- each component but are not externally visible;
-\item the algorithms and implementation details of the operations.
-\end{itemize}
-Finally, this document does {\em not\/} specify or describe
-\begin{itemize}
-\item the specific choices for parameters which may be adjusted by an
- installation or implementation of Kazlib.
-\item the size of any data structure which will exceed the capacity of
- a particular installation.
-\item the mechanisms or procedures for the translation of Kazlib and
- their integration with other translation units.
-\end{itemize}
-
-\section{References}
-\label{sec:references}
-
-\begin{trivlist}
-\item ISO 9899:1990, {\it Programming Languages---C.}
-\item {\it Introduction to Algorithms}, Thomas H. Cormen, Charles E.
-Leiserson, Ronald L. Rivest, eighth printing, 1992.
-\end{trivlist}
-
-\section{Definitions and conventions}
-The following terms shall be interpreted in accordance with the definitions
-below. Other terms appearing in this document shall be defined upon their
-first mention, indicated by {\it italic\/} type. Any terms not explicitly
-defined in this document should be interpreted according to ISO 9899-1990,
-clause 3. Failing that, they should be interpreted according to other works
-listed in section \ref{sec:references}.
-\nobreak
-\defsubsection{implementation}: A library and set of C language headers
-which conforms to the specifications of this Document.
-\index{production mode}
-\indexmacro{NDEBUG}
-\defsubsection{production mode}: A mode of operating the implementation
-in such a way that maximum efficiency of execution is achieved at the expense
-of the verification of constraints. An implementation shall provide
-a production mode, which is enabled in an implementation-defined
-manner.\footnote{An implementation may have to supply a separate set of
-libraries for production and for verification use, for instance. The
-manner of selecting libraries varies with each programming environment.} Each
-translation unit of the program which includes a Kazlib header shall ensure that the macro {\tt
-NDEBUG} is defined prior to the inclusion of that header, otherwise the
-implementation is not said to be operated in production mode.
-\index{verification mode}
-\defsubsection{verification mode}: A mode of operating the implementation in
-such a way that maximum error checking is obtained at the cost of
-execution efficiency. An implementation shall provide a verification mode, which
-is enabled in an implementation-defined manner. If any translation unit which
-includes a Kazlib header defines the macro name {\tt NDEBUG}\footnote{The
-intent is that the standard {\tt assert} macro may be exploited
-by the implementation's headers for the purpose of provisioning verification
-mode.} prior to including that header, the implementation is not said to be in
-verification mode. The least requirements of a Kazlib implementation operated
-in verification mode, is that it shall stop translation or execution of any
-program which violates a constraint.
-\index{undefined behavior}
-\defsubsection{undefined behavior}: Behavior of a program, upon violation of a
-requirement with respect to the use of Kazlib, or upon use of corrupt or
-incorrect data, for which this document does not impose any requirements.
-Additional undefined behaviors are:
-\begin{itemize}
-\item any behavior that is undefined by the C language standard;
-\item evaluation of an object whose contents are indeterminate;
-\item a violation of any explicit constraint stated in
-this document, if that program was built using Kazlib in production
-mode;\footnote{The intent is that violations of constraints are diagnosed by
-the implementation in verification mode, and hence do not lead to undefined
-behavior.}
-\item a violation of any requirement stated in this document that
-is not designated as a constraint, and is introduced using the word
-{\it shall}; and
-\item any other construct for which no definition of behavior can be deduced
-from this document.
-\end{itemize}
-If a program invokes undefined behavior of any kind, the Kazlib implementation
-is absolved from any requirements as to what events should ensue. The
-implementation may respond by invoking undefined behavior in the C language
-sense, or it may detect the behavior and terminate with a diagnostic message.
-\defsubsection{implementation-defined}: An adjective which, when appearing
-in the description of a feature, represents a requirement that the
-implementor must supply a definition, and document that
-definition. This adjective is applied to both behavior and to results.
-Implementation-defined behavior is behavior which depends on the
-characteristics of an implementation.\footnote{It is not considered adequate
-for the implementor to allow implementation-defined behavior to produce
-unpredictable effects or to terminate the program when such behavior is
-invoked.} When said of a result,
-implementation-defined means that a value is successfully computed, but depends
-on the characteristics of the implementation. It is possible for the presence of a
-requirement on a program to be described as implementation-defined, giving the
-implementor a choice whether to make that requirement or not. If a program
-violates a requirement whose presence is implementation-defined, that program's
-behavior is undefined in any implementation which elects to in fact impose that
-requirement.
-\index{implementation-defined}
-\defsubsection{unpredictable result}: A successfully computed value which is
-unreliable because some procedure or data failed to satisfy a property required
-by the computation.
-\defsubsection{constraint}: A semantic restriction with which a program must
-comply. Some sections of this Document contain paragraphs under the heading
-{\it Constraints\/} which list all constraints pertaining to the described
-feature. When operated in production mode, the Kazlib implementation
-is not required to diagnose constraint violations. When operated in
-verification mode, the Kazlib implementation must halt translation or
-execution of a program which violates a constraint.
-\index{constraint}
-\defsubsection{comparison function}: A function which accepts two arguments
-\index{comparison function}
-of type \verb|const void *| and returns a value of type int based on
-a ranking comparison of these arguments, and which satisfies the following
-additional semantic properties. If the two arguments are deemed to be equal, the
-function must return zero. If the first argument is determined to have a
-greater rank than the second, a positive value is returned. Otherwise if the
-first argument is determined to have a lesser rank than the second, a negative
-value is returned. The rank is computed as if each value has associated with it
-an integer, not necessarily unique, and as if these integers are compared for ordinary equality or
-inequality when values are said to be compared. The assignment of integers is
-up to the designer of the comparison function, and does not change between
-successive invocations of the function.\footnote{Of course, an actual
-comparison function need not assign actual integer ranks to data items, but it
-must behave as if such ranks were assigned.}
-If a comparison function is invoked in the context of an operation on some data
-structure, it shall not invoke any operation on any component of that same
-structure.\footnote{Thus, if a comparison function is invoked from, for
-instance, {\tt list_sort}, it must not call any list operations that
-inspect or modify the list being sorted, or any of its constituent nodes.}
-\defsubsection{opaque data type}: A data type whose precise definition is
-not documented, and which is intended to be manipulated only using the
-documented interface, which consists of a set of functions. Many data types in
-Kazlib are described as opaque. A program which bypasses the documented
-interfaces in inspecting or manipulating these data types invokes undefined
-behavior, and is not portable among Kazlib implementations.
-\defsubsection{user}: \index{user} The program which uses Kazlib.
-\defsubsection{user data}: \index{user data} Data provided by the program
-to which Kazlib stores a pointer, but otherwise does not inspect or modify.
-
-\section{Environment}
-\label{sec:environment}
-
-The translation and use of Kazlib requires a conforming, hosted implementation
-of the C language which meets the following additional minimal requirements:
-\begin{enumerate}
-\item The C implementation distinguishes external names by at least their
-initial 15 characters\footnote{The ISO 9899:1990 standard demands only that
-external names be distinguished by their initial six characters.}. External
-names that are distinct in their first 15 characters are treated by the
-implementation as distinct names. Upper and lower case letters in external
-identifiers need not be treated as distinct.
-\item The C implementation does not claim the identifier \verb|__cplusplus|
-for its internal use as a preprocessor symbol or keyword.
-\end{enumerate}
-If Kazlib headers are used by a C++ program, the C++ implementation
-meets these additional requirements:
-\begin{enumerate}
-\item the C++ implementation identifies itself by predefining the preprocessor
-symbol \verb|__cplusplus|;
-\item the C++ implementation is be capable of linkage against
-the C implementation with which the Kazlib source files units were translated.
-\end{enumerate}
-The Kazlib headers shall not make use of any names that are claimed
-by the C++ programming language, and shall ensure that the \verb|extern "C"|
-mechanism is used for all declarations when they are included into a C++
-translation unit, or otherwise provide compatibility with C++.\footnote{The
-intent is that the Kazlib implementation could, in principle, provide
-a separate set of headers for use with each language.}
-
-In programming environments that support the programming mechanism of multiple
-threads of execution an implementation of Kazlib may be designated as {\it
-thread safe}. To be called thread safe, it must guarantee that the use of an
-object by one thread cannot visibly interact or interfere with the concurrent
-or interleaved use of another object by another thread. If a Kazlib
-implementation that is not thread safe is provided for an environment which
-supports threads, it shall be accompanied by documentation which describes
-the extent of this limitation.
-
-A Kazlib implementation can also be designated as being {\it async safe}.
-The minimum requirement for this designation is that an operation on an object
-can be interrupted by delivery of an asynchronous signal and from within the
-catching function for that signal, it is safe to perform an operation on
-another object. An implementation shall document that it is async safe,
-or the extent to which it fails to be async safe.
-
-\section{General restrictions}
-
-\subsection{Headers}
-
-The Kazlib headers may be included in any order, and may be included more than
-once. Prior to the inclusion of a Kazlib header, the translation unit shall not
-define any macro name that has the same spelling as a C language keyword. The
-Kazlib headers may behave as though they include arbitrary standard C headers,
-so any requirements related to the inclusion of standard headers apply to
-Kazlib headers. A header shall be included before the first reference to any
-of the functions, types or macros that it defines.
-
-If one or more preprocessor symbols whose names begin with the sequence
-\verb|KAZLIB_| are defined prior to the inclusion of a Kazlib header,
-the behavior is implementation-defined.
-
-\subsection{Reserved macros}
-
-A Kazlib header defines all of the macros explicitly listed in the section of
-this document that defines the contents of that header. It may also define
-additional macros that belong to the macro namespace reserved by that header.
-The translation unit that includes the header shall not \verb|#define| or
-\verb|#undef| any of these macros.
-
-A header may define function-like macros that supplement existing functions,
-provided that such macros do not cause multiple evaluation of arguments except
-as explicitly permitted, and are safe to use wherever the corresponding
-function call would be. These function-like macros may be subject to
-\verb|#undef|.\footnote{In principle, an implementation may provide, within the
-reserved namespaces, additional functions not specified in this document, and
-function-like macro equivalents of these functions. A program that uses such
-identifiers in a block or function scope should use {\tt \#undef} on these
-identifiers prior to their use.}
-
-\subsection{Reserved symbols}
-
-Each Kazlib header provides file scope declarations for the typedef names,
-struct tags, enum constants and function names listed in its corresponding
-section in this document. Moreover, each header may define additional such
-names that fall into the documented reserved namespaces.
-
-The behavior is undefined if a translation unit that includes a Kazlib header
-defines any identifier that is the same as an identifier reserved by the header
-in the same scope and namespace.\footnote{Therefore, it is permitted to redeclare
-or redefine the identifiers reserved by a previously included Kazlib header,
-provided that the declarations or definitions are in a different namespace or
-scope. Reserved names may be redeclared in a block scope, or used as
-statement labels which have function scope and are in their own namespace.}
-
-The behavior is also undefined if the program contains a definition of an
-object or function with external linkage whose name matches an external object
-of unction defined by Kazlib component that is used as part of the
-program, or whose name is in a namespace reserved by that
-component.\footnote{This restriction exists whether or not the corresponding Kazlib
-header is included.}
-
-Lastly, the behavior is undefined if a translation unit defines a macro whose
-name is in the space of reserved symbols of a Kazlib header that is included in
-that translation unit.
-
-\subsection{Argument aliasing}
-
-Kazlib provides functions that operate on objects of various types. Pointers
-to objects are passed to these functions, thereby giving rise to the
-possibility of {\it aliasing}---passing of objects that wholly or partially
-overlap. The program shall not present aliased objects to any Kazlib function.
-Objects of distinct types shall not be aliased in a function call under any
-circumstances.
-The aliasing of two or more objects of compatible type is permitted only as
-explicitly documented in the description of a function; in all such
-circumstances, only exact overlapping is permitted.\footnote{That is to say,
-where explicitly allowed, a pointer to the same object may be specified for two
-(or more) parameters of like type.}
-
-\subsection{Object initialization}
-
-The Kazlib opaque data types can only be initialized with the initialization
-functions provided by the Kazlib library, or by implementation-defined
-initialization functions.\footnote{Of course, the use of implementation-defined
-functions results in programs that are not portable among library
-implementations.} An opaque object that is initialized by a method other than
-by being passed to an appropriate initialization function, or that is not
-initialized at all, has indeterminate contents. A pointer to an object having
-indeterminate contents may be passed to an initialization function; the object
-then has well-determined contents.
-
-An object whose initialization function is capable of indicating failure is
-considered indeterminate if the attempt to initialize that object using that
-function does in fact fail. The program shall not attempt to deinitialize such
-an object. The implementation shall reclaim any resources that were allocated
-for an object whose initialization failed. This reclamation need
-not be immediate, but may be delayed; however, the delay shall not
-give rise to the possibility of resource leaks in any correct program.
-
-Those objects for which deinitialization operations are defined should be
-subject to these operations when these objects are no longer needed. Failure
-to apply the deinitialization functions may result in the leakage of resources.
-
-\subsection{Object copying}
-
-Certain data types may be sensitive to their own location in memory. This
-means that copying their values by assignment or \verb|memcpy| results in the
-copy having an indeterminate value which cannot be used. All opaque types in
-Kazlib are assumed to have this property; copying the value of an opaquely
-typed object to another suitably typed object causes the destination
-object to have indeterminate contents.
-
-\section{List component}
-
-The List component provides a set of functions, macros and type declarations
-which together provide a library for maintaining a possibly empty ordered set
-of elements, called a {\it list}. This list has the following properties:
-\index{List}\begin{enumerate}
-\item If the list is not empty, a first and last element can be identified.
- In a list having only one element, that one element is both the first and
- last element.
-\item Each element that is not the last element has another element as its
- {\it successor}.
- \index{successor!of a list element}
- \index{List!successor of an element}
-\item Each element that is not the first element has a {\it
- predecessor}.
- \index{predecessor!of a list element}
- \index{List!predecessor of an element}
-\item No element is the predecessor or successor of more than one element.
-\item If one element is the successor of another, the other is necessarily the
- predecessor of the first.
-\item Each element is associated with arbitrary {\it satellite\/} data.
-\end{enumerate}
-The {\it size} of a list, also known as the {\it list count}, is simply the
-number of elements contained in it.\index{size!of a list}\index{List!count}
-
-A list imposes a maximum value on the number of nodes that may be in it
-simultaneously. This is known as the list's {\it capacity}. A list that
-has the maximum number of nodes is said to be full.
-
-\subsection{Interface}
-
-\subsubsection{The {\tt list.h} header}
-
-Each C or C++ translation unit that is to use the functionality of
-the List component shall include the header \verb|list.h|. This header
-shall contain declarations of types and external functions, and definitions of
-macros.
-The following typedef names shall be defined:\index{List!typedef names}
-\index{typedefs!defined by List}
-\begin{verbatim}
- list_t listcount_t
- lnode_t lnodepool_t
-\end{verbatim}
-In addition, the following structure tags may be defined:\index{List!tag names}
-\index{tags!defined by List}
-\begin{verbatim}
- struct list_t
- struct lnode_t
- struct lnodepool_t
-\end{verbatim}
-The following external function names shall be declared:
-\index{List!function names}\index{functions!defined by List}
-\begin{verbatim}
- list_append list_prev
- list_contains list_process
- list_count list_return_nodes
- list_create list_sort
- list_del_first list_find
- list_del_last list_transfer
- list_delete list_verify
- list_destroy lnode_borrow
- list_destroy_nodes lnode_create
- list_extract lnode_destroy
- list_first lnode_get
- list_init lnode_init
- list_ins_after lnode_is_in_a_list
- list_ins_before lnode_pool_create
- list_is_sorted lnode_pool_destroy
- list_isempty lnode_pool_init
- list_isfull lnode_pool_isempty
- list_last lnode_pool_isfrom
- list_merge lnode_put
- list_next lnode_return
- list_prepend
-\end{verbatim}
-The following preprocessor symbols (macros) shall be defined:
-\index{List!macro names}\index{macros!defined by List}
-\indexmacro{LISTCOUNT_T_MAX}
-\indexmacro{LIST_H}
-\begin{verbatim}
- LISTCOUNT_T_MAX
- LIST_H\end{verbatim}
-\index{symbols!reserved by List}\index{List!reserved symbols}
-Macro identifiers which begin with the upper-case prefix \verb|LIST| are
-reserved for future extensions to the \verb|list.h| header, as are
-names in the ordinary and tag namespaces which begin with
-\verb|list_| or \verb|lnode_|. External names which begin with \verb|list_| or
-\verb|lnode_| are reserved by the Kazlib library regardless of what header
-files are included.
-
-\subsubsection{The {\tt list_t} type}
-
-\indextype{list_t}
-The type \verb|list_t| is an opaque data type which maintains information about the
-current state of a single list. A list consists of an instance of the
-\verb|list_t| type, plus zero or more instances of the type \verb|lnode_t|. An
-instance of the \verb|list_t| type can be dynamically created using the
-\verb|list_create| function, and destroyed by the \verb|list_destroy| function.
-Alternately, the program can declare an object of type \verb|list_t| and have
-it initialized via the \verb|list_init| function.
-
-\subsubsection{The {\tt listcount_t} type}
-
-\indextype{listcount_t}
-\indexmacro{LISTCOUNT_T_MAX}
-The type \verb|listcount_t| is an unsigned integral type which represents
-the number of nodes in a list. The specific choice of unsigned integral type
-is implementation defined. The \verb|LISTCOUNT_T_MAX| macro expands to a
-constant expression of type \verb|listcount_t| which specifies the maximum
-value of that type.\footnote{For example, if the implementation defines
-{\tt listcount_t} as an alias for the type unsigned long, then
-{\tt LISTCOUNT_T_MAX} must have the same value as {\tt ULONG_MAX}.}
-
-\subsubsection{The {\tt lnode_t} type}
-
-\indextype{lnode_t}
-The type \verb|lnode_t| is an opaque type that represents a single node of a
-list. A node contains a a reference to satellite data provided by the user,
-and also stores the key that is associated with the node when it is inserted.
-Nodes may be dynamically created by the \verb|lnode_create| function.
-Alternately, the program may supply an \verb|lnode_t| object that can be
-initialized by the \verb|lnode_init| function.
-
-\subsubsection{The {\tt lnodepool_t} type}
-
-\indextype{lnodepool_t}
-The \verb|lnodepool_t| type provides an alternate method for supplying list
-nodes to the application. A user-supplied or dynamically allocated fixed size
-array of nodes is converted into a a {\it pool\/} of nodes from which free
-nodes may be obtained and to which they may be returned. A user-supplied node
-pool is created by the function \verb|lnode_pool_init| which requires a pointer
-to an object of type \verb|lnode_pool_t|, a pointer to the first element of an
-array of \verb|lnode_t| objects, as well as an integer representing the size of
-the array. Alternately, the function \verb|lnode_pool_create| will dynamically
-allocate an object of type \verb|lnode_pool_t| containing the specified number
-of list nodes.
-
-\subsubsection{The {\tt list_append} function}
-
- \indexfunc{list_append}
- \index{List!appending a node}
- \index{append node to list}
- \synopsis
- \begin{verbatim}
- void list_append(list_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The second argument shall not refer to a node that is already in a list
- or in a list node pool. The first argument shall not refer to a list
- that is full.
-
- \description
- The append operation causes the node pointed at by the second
- argument to become the last node in the list pointed at by the first
- argument.\footnote{That is to say, after the operation, the
- {\tt list_last} function, when applied to the list, shall return a pointer
- to that node.}
-
- If the first argument is an expression with side effects, the behavior
- is undefined.\footnote{Thus, the implementation may provide a macro
- version of {\tt list_append} which evaluates the first argument
- more than once.}
- \index{macros!and side effects}
-
-\subsubsection{The {\tt list_contains} function}
-
- \indexfunc{list_contains}
- \index{List!testing for presence of node}
- \nobreak
- \synopsis
- \begin{verbatim}
- int list_contains(list_t *, lnode_t *node);\end{verbatim}
- \nobreak
- \description
- \nobreak
- The \verb|list_contains| function shall return 1 if the node
- pointed at by the second argument is in the list pointed at by the first
- argument. Otherwise, it shall return 0.
-
-\subsubsection{The {\tt list_count} function}
-
- \indexfunc{list_count}
- \index{List!count}
- \index{List!size}
- \synopsis
- \begin{verbatim}
- listcount_t list_count(list_t *);\end{verbatim}
-
- \description
-
- The \verb|list_count| function returns a value which represents the number
- of nodes currently stored in the list pointed at by the argument.
-
-\subsubsection{The {\tt list_create} function}
-
- \indexfunc{list_create}
- \index{List!creation of}
- \index{create!list object}
- \synopsis
- \begin{verbatim}
- list_t *list_create(listcount_t);\end{verbatim}
-
- \description
- The \verb|list_create| function instantiates and initializes an object of
- type \verb|list_t|, and returns a pointer to it unless insufficient
- resources exist for the creation of the object, in which case a null
- pointer is returned.
-
- The value of the function's argument establishes, for the entire duration
- of the list object, its capacity.
-
- The newly created list object is empty.
-
-\subsubsection{The {\tt list_del_first} function}
-
- \index{List!first node}
- \indexfunc{list_del_first}
- \index{List!deletion}
- \index{delete!first node of a list}
- \synopsis
- \begin{verbatim}
- lnode_t *list_del_first(list_t *);\end{verbatim}
-
- \constraints
- The argument shall not point to an empty list.
-
- \description
- The \verb|list_del_first| function removes the first node from the
- list pointed at by the argument and returns a pointer to that
- node.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_del_last} function}
-
- \index{List!last node}
- \indexfunc{list_del_last}
- \index{List!deletion}
- \index{delete!last node of a list}
- \synopsis
- \begin{verbatim}
- lnode_t *list_del_last(list_t *);\end{verbatim}
-
- \constraints
- The argument shall not point to an empty list.
-
- \description
- The \verb|list_del_last| function removes the last node from the list
- specified by the argument, and returns a pointer to that node. If,
- prior to the operation, that node had a predecessor, that predecessor
- shall become the new last node of the list. Otherwise, the list
- shall become empty.
-
- The new value of the list count shall be one less than its value
- prior to the call to this function.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_delete} function}
-
- \indexfunc{list_delete}
- \index{List!deletion}
- \index{delete!arbitrary node of a list}
- \synopsis
- \begin{verbatim}
- lnode_t *list_delete(list_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The second argument shall point to a node that is inside the list
- pointed at by the first argument.
-
- \description
- The \verb|list_delete| function removes the node pointed at by its
- second argument from the list pointed at by its first argument.
- A pointer to the deleted node is returned.
-
-\subsubsection{The {\tt list_destroy} function}
-
- \indexfunc{list_destroy}
- \index{List!destruction of}
- \synopsis
- \begin{verbatim}
- void list_destroy(list_t *);\end{verbatim}
-
- \constraints
- The argument shall point to an empty list.
-
- \description
- The empty list pointed at by the argument is destroyed. If the list has
- not been created by a call to the \verb|list_create| function, the
- behavior is undefined.
-
- A pointer that previously referred to a list that has been disposed by
- \verb|list_destroy| has an indeterminate value.
-
-\subsubsection{The {\tt list_destroy_nodes} function}
-
- \indexfunc{list_destroy_nodes}
- \synopsis
- \begin{verbatim}
- void list_destroy_nodes(list_t *);\end{verbatim}
-
- \description
- The nodes, if any, contained in the list pointed at by the argument are
- disposed of as if by a call to the \verb|lnode_destroy| function. If any
- node contained in the list was created by means other than the
- \verb|lnode_create| function, the behavior is undefined.
-
- After the operation, the list is empty.
-
- Any pointer that referred to any of the destroyed nodes takes on an
- indeterminate value.
-
-\subsubsection{The {\tt list_extract} function}
-
- \index{List!node range extraction}
- \indexfunc{list_extract}
- \synopsis
- \begin{verbatim}
- void list_extract(list_t *, list_t *, lnode_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The second argument points to the {\it source list}. The third
- argument is either null, or points to a node that is an occupant
- of the source list. This node is called the {\it starting node}.
- The fourth argument is either null, or points to a node that is
- an occupant of the source list. This node is called the {\it ending
- node}. If the starting node and ending node are both specified, and are
- distinct nodes, then the starting node shall appear earlier in the source
- list than the ending node.
-
- The transfer request shall not call for the capacity of the destination
- list to be exceeded.
-
- \description
- The \verb|list_extract| function moves nodes from the source
- list to the {\it destination list\/} pointed at by the first
- argument.\footnote{This right-to-left direction of transfer is consistent
- with the semantics of standard C library functions such as {\tt memmove} or
- {\tt strcpy}.}
-
- If the third and fourth arguments are not null, the entire range of nodes
- from the starting node and to the ending node, inclusive, is transferred
- from the source list to the end of the destination list, where they appear
- in their original order. Other nodes in the source list, if any, are
- unaffected.
-
- If the third and fourth arguments both point to the same node, that
- node alone is transferred to the end of the destination list.
-
- If either the third argument or the fourth argument is null, or both are null,
- no transfer of nodes takes place.
-
- The source and destination list may be the same object.
-
-\subsubsection{The {\tt list_first} function}
-
- \index{List!first node}
- \indexfunc{list_first}
- \synopsis
- \begin{verbatim}
- lnode_t *list_first(list_t *);\end{verbatim}
-
- \description
- If the list pointed at by the argument is an empty list, a null pointer
- is returned. Otherwise, a pointer to the first node in that list is
- returned.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_init} function}
-
- \indexfunc{list_init}
- \synopsis
- \begin{verbatim}
- list_t *list_init(list_t *, listcount_t);\end{verbatim}
-
- \constraints
- The second argument shall not have a zero value.
-
- \description
- The \verb|list_init| function initializes the list object pointed at by the
- first argument, turning it into a valid, empty list. If the object is an
- already initialized list, the behavior is undefined. A list returned by
- \verb|list_create| is considered initialized. The second argument
- specifies the maximum number of nodes that may simultaneously occupy the
- list.
-
- The value returned is that of the first argument.
-
-\subsubsection{The {\tt list_ins_after} function}
-
- \indexfunc{list_ins_after}
- \index{insert!node into list}
- \index{List!insertion}
- \synopsis
- \begin{verbatim}
- void list_ins_after(list_t *, lnode_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The first argument shall point to a list that is not already full. The
- second argument shall point to a node, called the {\it new node}, that is not
- already an occupant of the list pointed at by the first argument, nor
- of any other list or node pool object. The third
- argument shall point to a node, called the {\it reference node}, that is an
- occupant of the list.
-
- \description
- The new node becomes an occupant of the list, such that its predecessor
- is the reference node. If the reference node has a successor, the
- new node is inserted between the reference node and that successor.
- Otherwise, the new node becomes the last node of the list.
-
-\subsubsection{The {\tt list_ins_before} function}
-
- \indexfunc{list_ins_before}
- \index{insert!node into list}
- \index{List!insertion}
- \synopsis
- \begin{verbatim}
- void list_ins_before(list_t *, lnode_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The first argument shall point to a list that is not already full. The
- second argument shall point to a node, called the {\it new node}, that is not
- already an occupant of the list pointed at by the first argument, nor
- of any other list or node pool object. The third
- argument shall point to a node, called the {\it reference node}, that is an
- occupant of the list.
-
- \description
- The new node becomes an occupant of the list, such that its successor
- is the reference node. If the reference node has a predecessor, the
- new node is inserted between the reference node and that predecessor.
- Otherwise, the new node becomes the first node of the list.
-
-\subsubsection{The {\tt list_is_sorted} function}
-\label{list:is:sorted}
- \indexfunc{list_is_sorted}
-
- \synopsis
- \begin{verbatim}
- int list_is_sorted(list_t *,
- int (const void *, const void *));\end{verbatim}
-
- \description
- The first argument points to a list object. The second is assumed to
- point to a comparison function.
-
- If the list has exactly one node or is empty, $1$ is returned
- unconditionally. Otherwise, nodes of the list are examined to
- determine whether they are in a sorted order according to the comparison
- function. This is true if the integer ranks of their data items,
- examined from the first node of the list through to the last node, form a
- monotonically increasing sequence. If the nodes are in order, the value $1$
- is returned. Otherwise $0$ is returned.
-
- If the list has two or more nodes, and the second argument is a pointer to
- a function that has the correct type, but does not satisfy the semantic
- properties of a comparison function, the result is unpredictable, but is
- guaranteed to be one of the values~$0$~or~$1$.
-
-\subsubsection{The {\tt list_isempty} function}
-
- \indexfunc{list_isempty}
- \synopsis
- \begin{verbatim}
- int list_isempty(list_t *);\end{verbatim}
-
- \description
- The \verb|list_isempty| function returns $1$ if the list pointed at by
- the first argument is empty. Otherwise it returns $0$.
-
-\subsubsection{The {\tt list_isfull} function}
-
- \indexfunc{list_isfull}
- \synopsis
- \begin{verbatim}
- int list_isfull(list_t *);\end{verbatim}
-
- \description
- The \verb|list_isfull| function returns $1$ if the list pointed at by
- the first argument is full. Otherwise it returns $0$.
- A list is considered full when it contains the maximum number of nodes
- that was specified upon its initialization.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_last} function}
-
- \index{List!last node}
- \indexfunc{list_last}
- \synopsis
- \begin{verbatim}
- lnode_t *list_last(list_t *);\end{verbatim}
-
- \description
- If the list pointed at by its first argument is empty, the \verb|list_last|
- function returns a null pointer. Otherwise it returns a pointer to the
- last node.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_merge} function}
-
- \index{List!merge operation}
- \indexfunc{list_merge}
- \synopsis
- \begin{verbatim}
- void list_merge(list_t *, list_t *,
- int (const void *, const void *));\end{verbatim}
-
- \constraints
- The list pointed at by the first argument is called the {\it destination
- list}. The second argument points to the {\it source list}. The third
- argument points to a comparison function. The sum of the number of nodes
- occupying the source list and the destination list shall not exceed the
- maximum number of nodes that are permitted to occupy the destination list.
- Furthermore, both the source and destination list shall be sorted such that
- a call to \verb|list_is_sorted| given a pointer to either list as a first
- argument, and the pointer to the comparison function as its second
- argument, shall yield the value $1$.
-
- \description
- Nodes from the sorted source list are merged into the sorted destination
- list. After the operation, the source list is empty and the destination
- list contains all of the nodes it contained prior to the operation, as well
- as all of the nodes that the source list contained. The nodes are in sorted
- order according to the comparison function.
-
- If the third argument is a pointer to a function that has the correct type,
- but does not fulfill the semantic properties of a comparison function, the
- order of the nodes in the destination list is unpredictable.
-
- If the source and destination list are the same object, the
- \verb|list_merge| operation has no effect.
-
-\subsubsection{The {\tt list_next} function}
-
- \indexfunc{list_next}
- \synopsis
- \begin{verbatim}
- lnode_t *list_next(list_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The node pointed at by the second argument is an occupant of the list pointed
- at by the first argument.
-
- \description
- If the node pointed at by the second argument has a successor, a pointer to
- that successor is returned. Otherwise, a null pointer is returned.
-
- If the second argument is an expression which has side effects, the behavior
- is undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_prepend} function}
-
- \indexfunc{list_prepend}
- \index{List!prepending a node}
- \index{prepend node to list}
- \synopsis
- \begin{verbatim}
- void list_prepend(list_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The second argument shall not refer to a node that is already in a list
- or in a list node pool. The first argument shall not refer to a list
- that is full.
-
- \description
- The prepend operation causes the node pointed at by the second
- argument to become the first node in the list pointed at by the first
- argument. After the operation, the \verb|list_first| function, when
- applied to the list, shall return a pointer to that node.
- If, prior to to the operation, the list is empty, then the prepended node
- shall become the first node in that list, otherwise, the prepended node
- becomes the predecessor of what was previously the first node.
-
- If the first argument is an expression with side effects, the behavior
- is undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt list_prev} function}
-
- \indexfunc{list_prev}
- \synopsis
- \begin{verbatim}
- lnode_t *list_prev(list_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The node pointed at by the second argument is an occupant of the list pointed
- at by the first argument.
-
- \description
- If the node pointed at by the second argument has a predecessor, a pointer to
- that predecessor is returned. Otherwise, a null pointer is returned.
-
- If the second argument is an expression which has side effects, the behavior
- \index{macros!and side effects}
- is undefined.
-
-\subsubsection{The {\tt list_process} function}
-
- \indexfunc{list_process}
- \synopsis
- \begin{verbatim}
- void list_process(list_t *, void *,
- void (*)(list_t *, lnode_t *, void *));\end{verbatim}
- \nobreak
- \description
- The \verb|list_process| function iterates over the nodes of a list,
- and for each node invokes a callback function.\footnote{In most cases,
- it is more convenient and preferable to
- iterate over the list using explicit calls to {\tt list_first}
- and {\tt list_next}.}
- The second argument is a {\it context pointer\/} which can have any value.
- The third argument of
- \verb|list_process| shall be a pointer to a function which is compatible
- with the specified type. If the list contains one or more nodes,
- then the function is invoked once for each node, in order from first
- to last. On each invocation, the first argument of the callback is a
- pointer to the list; the second argument is a pointer to a node, called
- the {\it subject node}; and the third argument repeats the context pointer
- value that was originally passed to \verb|list_process|.
-
- The callback function may delete the subject node by, for instance, calling
- \verb|list_delete|. It may insert new nodes to any place in the list;
- however, if such an insertion causes the subject node to acquire
- a new successor, it is implementation-defined whether upon returning
- from the callback function, the traversal shall continue with the
- new successor, or with the original successor.
-
- The callback function, and any function invoked from the callback
- function, shall not destroy the list or make any modifications
- other than the insertion of new nodes, or the deletion of the
- subject node.
-
- The callback function may recursively invoke \verb|list_process| for the
- same list or for a different list; the callback invocations arising out of
- the nested call inherit all of the restrictions of the outer callback in
- addition to being subject to the usual restrictions.\footnote{This means,
- for instance, that if two callbacks are in progress for different
- subject nodes from the same list, the inner callback may not delete
- its subject node, because it inherits the restriction that the only
- permitted deletion is the outer callback's subject node.}
-
- The callback function may freely operate on a different list,
- subject to any inherited restrictions.
-
-\subsubsection{The {\tt list_return_nodes} function}
-
- \indexfunc{list_return_nodes}
- \synopsis
- \begin{verbatim}
- void list_return_nodes(list_t *, lnodepool_t *);\end{verbatim}
-
- \description
-
- Every node in the list specified by the first argument
- is returned to the node pool specified by the second argument
- If the list contains a node that has not been allocated
- from that node pool, the behavior is undefined.
-
-\subsubsection{The {\tt list_sort} function}
-
- \index{List!sort operation}
- \indexfunc{list_sort}
- \synopsis
- \begin{verbatim}
- void list_sort(list_t *, int (const void *, const void *));\end{verbatim}
-
- \description
-
- The \verb|list_sort| function changes the order of the nodes of the list
- specified by the first argument according to the comparison function
- pointed at by the second argument.
-
- If the list is empty, or contains only one node, the comparison function is
- not called.
-
- Whenever the comparison function is invoked, its arguments are are the data
- pointers stored in two distinct nodes of the list.
-
-\subsubsection{The {\tt list_find} function}
-
- \index{List!find operation}
- \indexfunc{list_find}
- \synopsis
- \begin{verbatim}
- lnode_t *list_find(list_t *,
- const void *, int (const void *, const void *));\end{verbatim}
-
- \description
-
- The \verb|list_find| function exhaustively searches the key for a node
- whose satellite data matches a search key according to the comparison
- function. The first argument is the list to be searched, the second
- argument specifies the search key and the third argument is a pointer
- to the comparison function.
-
- The comparison function is invoked to compare the key against the
- satellite data of successive nodes of the list, starting with the first
- node. A pointer to the first node for which the comparison function returns
- zero is returned.
-
- If the list is empty, or the comparison function returns non-zero for
- each item, a null pointer is returned.
-
-\subsubsection{The {\tt list_transfer} function}
-
- \index{List!node transfer}
- \indexfunc{list_transfer}
- \synopsis
- \begin{verbatim}
- void list_transfer(list_t *, list_t *, lnode_t *);\end{verbatim}
-
- \constraints
- The third argument is either null, or it points at a node which is an
- occupant of the list pointed at by the second argument.
-
- The transfer request shall not call for the capacity of the destination
- list to be exceeded.
-
- \description
- The \verb|list_transfer| function moves nodes from the list
- pointed at by the second argument to the list pointed at by
- the first argument.
-
- If the third argument is not null, it specifies the node in the source list
- at which the transfer begins. That node, its successor, and all
- subsequent nodes, are transferred to the end of the destination list where
- they appear in their original order. Other nodes in the source list are
- unaffected.
-
- If the third argument is null, no transfer of nodes takes place.
-
- The source and destination list may be the same object.
-
- If \verb|DL|, \verb|SL| and \verb|SN| are appropriately typed expressions,
- the function call
-
-\begin{verbatim}
- void list_transfer(DL, SL, SN);
-\end{verbatim}
- is equivalent to
-\begin{verbatim}
- list_extract(DL, SL, SN, list_last(SL));
-\end{verbatim}
- except that \verb|SL| is evaluated only once.
-
-\subsubsection{The {\tt list_verify} function}
-
- \indexfunc{list_verify}
- \synopsis
- \begin{verbatim}
- int list_verify(list_t *list);\end{verbatim}
-
- \description
- The intent of the \verb|list_verify| function is to perform a verification
- on the list object, regardless of whether the Kazlib implementation is
- operated in verification or production mode. If the list objects
- and its constituent nodes have been correctly manipulated, and the
- program has not caused any undefined behaviors, the value $1$ is returned.
- Otherwise, the function may be able to, but is not guaranteed to, detect
- corruption, and return the value zero.
-
-\subsubsection{The {\tt lnode_borrow} function}
-
- \indexfunc{lnode_borrow}
- \synopsis
- \begin{verbatim}
- lnode_t *lnode_borrow(lnodepool_t *, void *);\end{verbatim}
-
- \description
-
- The \verb|lnode_borrow| function allocates a node from
- the pool managed by the given \verb|lnodepool_t| object.
- If the request succeeds, a pointer to the node is returned. If the object
- has run out of nodes, the return value is a null pointer.
-
-\subsubsection{The {\tt lnode_create} function}
-
- \indexfunc{lnode_create}
- \synopsis
- \begin{verbatim}
- lnode_t *lnode_create(void *);\end{verbatim}
-
- \description
-
- The \verb|lnode_create| function dynamically allocates a list node,
- stores in it the data value specified in the argument and
- returns a pointer to it. The allocation is performed by a call to the
- standard \verb|malloc| function. If the allocation fails, a null
- pointer is returned.
-
-\subsubsection{The {\tt lnode_destroy} function}
-
- \indexfunc{lnode_destroy}
- \synopsis
- \begin{verbatim}
- void lnode_destroy(lnode_t *);\end{verbatim}
-
- \description
-
- The \verb|lnode_destroy| function destroys a list node that has been
- allocated with the \verb|lnode_create| function. The value of any pointer
- that referred to the node that was thus freed is indeterminate.
-
- If the node is currently the occupant of a list, the behavior is undefined
- if the list is subsequently used.
-
-\subsubsection{The {\tt lnode_get} function}
-
- \indexfunc{lnode_get}
- \synopsis
- \begin{verbatim}
- void *lnode_get(lnode_t *);\end{verbatim}
-
- \description
-
- The \verb|lnode_get| function retrieves the \verb|void *| data value
- associated with a node.\footnote{This is the {\bf only} interface for
- retrieving the data element.}
-
-\subsubsection{The {\tt lnode_init} function}
-
- \indexfunc{lnode_init}
- \synopsis
- \begin{verbatim}
- lnode_t *lnode_init(lnode_t *, void *);\end{verbatim}
-
- The \verb|lnode_init| function initializes the contents
- of the specified list node object, assigning it the
- data value specified as the second argument.
- The first argument is a pointer which refers to
- a data object that has a suitable size and alignment
- for the representation of an \verb|lnode_t| type.
- After initialization with \verb|lnode_init|, the object is subsequently
- eligible as an operand to the functions of the List component.
-
-\subsubsection{The {\tt lnode_is_in_a_list} function}
-
- \indexfunc{lnode_is_in_a_list}
- \synopsis
- \begin{verbatim}
- int lnode_is_in_a_list(lnode_t *);\end{verbatim}
-
- \description
-
- The \verb|lnode_is_in_a_list| function determines whether the given node is
- an occupant of some list. If the node is in a list, the function returns
- the value $1$. If the node is not in any list, the return value is zero.
-
-\subsubsection{The {\tt lnode_pool_create} function}
-
- \indexfunc{lnode_pool_create}
- \synopsis
- \begin{verbatim}
- lnodepool_t *lnode_pool_create(listcount_t);\end{verbatim}
-
- \constraints
-
- The value of the argument shall not be zero.
-
- \description
-
- The \verb|lnode_pool_create| function dynamically allocates,
- by means of the standard library function \verb|malloc|
- a node pool object containing the number of nodes specified
- as the first argument. If not enough resources are available,
- a null pointer is returned, otherwise a pointer to the
- \verb|lnodepool_t| object is returned.
-
-\subsubsection{The {\tt lnode_pool_destroy} function}
-
- \indexfunc{lnode_pool_destroy}
- \synopsis
- \begin{verbatim}
- void lnode_pool_destroy(lnodepool_t *);\end{verbatim}
-
- \description
-
- The \verb|lnode_pool_destroy| function deallocates a
- node pool that was allocated by \verb|lnode_pool_create|.
- The value of any pointer which referred to the
- node pool object becomes indeterminate.
-
-\subsubsection{The {\tt lnode_pool_init} function}
-
- \indexfunc{lnode_pool_init}
- \synopsis
- \begin{verbatim}
- lnodepool_t *lnode_pool_init(lnodepool_t *,
- lnode_t *, listcount_t);\end{verbatim}
-
- \constraints
-
- The third argument, which specifies the node count, shall not be zero.
-
- \description
-
- The \verb|lnode_pool_init| function initializes a data object
- that has a suitable size and alignment to represent an
- \verb|lnodepool_t| type. A pointer to this object is passed
- as the first argument. The node pool thus created draws nodes
- from an array specified by the second argument, which shall be a pointer to
- an object that can behave like an array of \verb|lnode_t| objects.
- The third argument specifies the number of elements in this array.
-
- After this function, the object pointed at by the \verb|lnodepool_t *|
- argument is eligible for use with the node pool management functions
- of the List component. Nodes may be drawn from the pool and returned to it.
-
- As long as the pool continues to be used, the program should not directly
- manipulate the node array. In particular, if the program modifies any
- part of the array, then the behavior is undefined if the
- \verb|lnodepool_t| object or any nodes drawn from it are subsequently
- passed to a List function. The program shall not directly use the array
- elements as independent \verb|lnode_t| objects while the array is
- associated with the pool; in particular, it shall not pass these elements
- to Kazlib functions that operate on \verb|lnode_t|.
-
- The behavior is undefined if the same array is associated with more than
- one node pool object, or if two node pool objects are given overlapping
- arrays.
-
- The node array is managed in an manner that is specific to the
- implementation; the intent is that each element of the array represents a
- distinct node object, a pointer to which can be returned in response to an
- allocation request.
-
- The \verb|lnode_pool_init| function returns a copy of the first argument.
-
-\subsubsection{The {\tt lnode_pool_isempty} function}
-
- \indexfunc{lnode_pool_isempty}
- \synopsis
- \begin{verbatim}
- int lnode_pool_isempty(lnodepool_t *);\end{verbatim}
-
- \description
-
- The \verb|lnode_pool_isempty| function tests the
- specified \verb|lnodepool_t| object for ability to supply nodes.
- If the object has been
- subject to so many requests that it is no longer capable of
- of supplying additional list nodes, the value $1$ is returned.
- Otherwise the return value returned is zero.
-
-\subsubsection{The {\tt lnode_pool_isfrom} function}
-
- \indexfunc{lnode_pool_isfrom}
- \synopsis
- \begin{verbatim}
- int lnode_pool_isfrom(lnodepool_t *, lnode_t *);\end{verbatim}
-
- \description
-
- The function \verb|lnode_pool_isfrom|, intended to serve as a software
- verification aid, determines whether a list node originates from
- a particular node pool. The return value is $1$ if this relationship is
- true, otherwise zero.
-
-\subsubsection{The {\tt lnode_put} function}
-
- \indexfunc{lnode_put}
- \synopsis
- \begin{verbatim}
- void lnode_put(lnode_t *, void *);\end{verbatim}
-
- \description
-
- The function \verb|lnode_put| replaces the data element
- associated with the list node.
-
-\subsubsection{The {\tt lnode_return} function}
-
- \indexfunc{lnode_return}
- \synopsis
- \begin{verbatim}
- void lnode_return(lnodepool_t *, lnode_t *);\end{verbatim}
-
- \constraints
-
- The node pointed at by the second argument was derived by an allocation
- request from the pool pointed at by the first argument.\footnote{In
- other words, the {\tt lnode_pool_isfrom} function, were it called with
- the same two arguments, would return $1$ if this constraint is met.}
-
- Furthermore, the node must not be the occupant of a list.
-
- \description
-
- The \verb|lnode_return| function returns a node back to the node pool from
- which it came. The node must not be subsequently used as an argument to any
- List functions, until it happens to be allocated again. The pointer to
- the node object remains valid, and may be returned by a subsequent
- allocation request from the same node pool.
-
-\subsection{Implementation}
-\index{List!reference implementation}
-
-This section describes the elements of the reference implementation of the
-List component. No requirement is imposed that an implementation should
-follow the reference implementation. The same is true of the
-implementation notes for the other components.
-
-\subsubsection{Types}
-\index{implementation!List types}
-\index{typedefs!implementation of List}
-
-The reference List implementation is a doubly-linked circular list
-\index{sentinel node!of linked list}
-with a {\it sentinel node}. The node structure type is defined like this:
-\begin{verbatim}
- typedef struct lnode_t {
- struct lnode_t *list_next;
- struct lnode_t *list_prev;
- void *list_data;
- } lnode_t;
-\end{verbatim}
-and the list structure is defined like this:
-\begin{verbatim}
- typedef struct list_t {
- lnode_t list_nilnode;
- listcount_t list_nodecount;
- listcount_t list_maxcount;
- } list_t;
-\end{verbatim}
-The \verb|list_nilnode| member of the list object is the sentinel. It is
-always present in the list, never deleted. When the list is empty, the sentinel
-node's \verb|list_next| and \verb|list_prev| pointers simply point back at the sentinel
-node. The \verb|list_maxcount| member of the list tells how many nodes may be
-inserted and \verb|list_nodecount| keeps track of the actual count.
-
-The reason the sentinel node is called \verb|list_nilnode| is that it
-acts as the successor of a list's tail node, if there is one,
-and as the predecessor of the first node. In a linked list implementation
-that does not use a sentinel node, the \verb|list_next| pointer of
-the the tail node and the \verb|list_prev| pointer of the first node would
-be null.
-
-Note that prefixed names are used for all of the structure members. This is so
-that the header file conforms to the documented namespace. If, for example, the
-\verb|list_nilnode| member were simply called \verb|nilnode|, then
-if the program contained somewhere a macro called \verb|nilnode|, there would
-be a potential clash. If the program defined \verb|nilnode| prior to including
-the \verb|list.h| header, the declaration of \verb|struct list_t| would
-be confounded. If the program defined \verb|nilnode| after
-including \verb|list.h|, the definition would interfere with \verb|list.h|
-macros whose replacement text refers to the \verb|nilnode| member.
-
-For programming convenience, the list implementation source file defines short
-macro names for the structure members:
-\begin{verbatim}
- #define next list_next
- #define prev list_prev
- #define data list_data
-\end{verbatim}
-... and so forth. These names are private to the translation unit, which
-includes only standard ANSI C headers. Some of the examples in this section
-make use of the short names; it is assumed that these macros are in effect.
-
-\subsubsection{Selected operations}
-\index{implementation!List operations}
-
-\paragraph{Retrieving the first node}
-\index{List!first node}
-
-Given a pointer \verb|P| to a \verb|list_t| type, the \verb|list_first|
-function examines the value of \verb|P->nilnode.next| which points
-at the head node if the list is not empty. If the list is empty,
-then this expression points back at the sentinel node. In
-other words, the comparison
-\begin{verbatim}
- P->nilnode.next == &P->nilnode
-\end{verbatim}
-yields true when the list is empty. In this case, the interface requires that
-a null pointer be returned by \verb|list_first|. The implementation actually
-uses the above test, through a test for \verb|P->nodecount| being equal to
-zero is also possible.
-
-In general, any operation which produces a pointer to the nilnode that must be
-returned back to the calling program must test for that case and return a null
-pointer instead to satisfy the interface requirements.
-
-\paragraph{Node deletion}
-\index{List!deletion}
-
-Thanks to the use of the sentinel node, the list deletion operation doesn't
-have to test for special cases. A node in the middle of the list is
-deleted in exactly the same way as the first or the last node:
-\begin{verbatim}
- lnode_t *list_delete(list_t *list, lnode_t *del)
- {
- lnode_t *next = del->next;
- lnode_t *prev = del->prev;
-
- assert (list_contains(list, del));
-
- prev->next = next;
- next->prev = prev;
- list->nodecount--;
-
- del->next = del->prev = NULL;
-
- return del;
- }
-\end{verbatim}
-Quite simply, the successor and predecessor of the deleted node are connected
-together so that the deleted node is spliced out from the list. If the node is
-the last remaining one, then the sentinel node serves as both the successor and
-the predecessor. The effect of the deletion then is to set the sentinel's next
-and previous links to point to itself, as they did initially when the list was
-previously empty.
-
-The next and prev pointers are set to null not only for enhanced error checking
-in language implementations that trap dereferences of null pointers,
-but also to indicate that the node is not on any list. The interface
-function \verb|lnode_is_in_a_list| makes use of this.
-
-It's worth discussing in some detail why the values of expressions
-\verb|del->next| and \verb|del->prev| are cached in local variables. The
-actual statements that splice the node out of the list could instead have been
-written:
-\begin{verbatim}
- del->prev->next = del->next;
- del->next->prev = del->prev;
-\end{verbatim}
-However, this causes some compilers to generate less than optimal code because
-they fail to apply common subexpression elimination to the double
-occurrence of \verb|del->next|. Caching this expression in a local variable
-helps to get better code by making the semantics more obvious. In any case,
-modern compilers tend to do a good job of caching locals in high speed storage,
-particularly on architectures generously endowed with registers, so using a few
-extra locals is unlikely to lead to worse target code. The principle of using
-local variables to perform ``manual CSE'' is applied throughout the Kazlib
-reference implementation.
-
-\paragraph{Node insertion}
-Node insertion is also simple, thanks to the sentinel node which makes
-the doubly linked list circular. All insertions are done using
-the functions \verb|list_ins_before| and \verb|list_ins_after|.
-These are very similar, so it suffices to show \verb|list_ins_before|:
-\begin{verbatim}
- void list_ins_before(list_t *list, lnode_t *new, lnode_t *this)
- {
- lnode_t *that = this->prev;
-
- assert (new != NULL);
- assert (!list_contains(list, new));
- assert (!lnode_is_in_a_list(new));
- assert (this == list_nil(list) || list_contains(list, this));
- assert (list->nodecount + 1 > list->nodecount);
-
- new->next = this;
- new->prev = that;
- that->next = new;
- this->prev = new;
- list->nodecount++;
-
- assert (list->nodecount <= list->maxcount);
- }
-\end{verbatim}
-The node \verb|this| is the one before which the new node is being
-inserted. Internally, the pointer \verb|that| points to the
-node after which the insertion takes place. In other words, the function
-inserts the node \verb|new| in between \verb|this| and \verb|that|.
-
-Note the copious assertions which verify all of the documented constraints:
-that the node is not already on the list, or any other list, that the reference
-node \verb|this| is in the list, and that the list capacity won't be exceeded,
-and that the node count doesn't overflow its type.
-
-\index{List!insertion}
-
-\section{Hash component}
-
-The Hash component provides a means to manage collections of elements, called
-hashes, that are not ordered. Each element in the collection has a unique key,
-which is used for searching and inserting. The intent is that the
-implementation is based on extendible hashing, and the interface allows for
-user-defined hashing functions. The number of elements that can be stored
-in a hash is limited; maximum number of entries in a hash is known as its
-{\it capacity}.
-
-\subsection{Interface}
-
-\subsubsection{The {\tt hash.h} header}
-
-Each C or C++ translation unit that is to use the functionality of the Hash
-component shall include the header \verb|hash.h|. This header shall
-contain declarations of types and external functions, and definitions of
-macros. The following typedef names shall be
-defined:\index{Hash!typedef names}
-\index{typedefs!defined by Hash}
-\begin{verbatim}
- hash_t hashcount_t
- hnode_t hash_val_t
- hash_comp_t hnode_alloc_t
- hscan_t hnode_free_t
- hash_fun_t
-\end{verbatim}
-In addition, the following structure tags may be defined:\index{Hash!tag names}
-\index{tags!defined by Hash}
-\begin{verbatim}
- struct hash_t
- struct hnode_t
- struct hscan_t
-\end{verbatim}
-The following external function names shall be declared:
-\index{Hash!function names}\index{functions!defined by Hash}
-\begin{verbatim}
- hash_create hash_count
- hash_set_allocator hash_size
- hash_destroy hash_isfull
- hash_free_nodes hash_isempty
- hash_init hash_scan_begin
- hash_insert hash_scan_next
- hash_lookup hash_scan_delete
- hash_delete hash_scan_delfree
- hash_alloc_insert hash_verify
- hash_delete_free hnode_create
- hnode_put hnode_init
- hnode_get hnode_destroy
- hnode_getkey hash_free
-\end{verbatim}
-\index{Hash!external objects}
-In addition, the external object name
-\begin{verbatim}
- hash_val_t_bit
-\end{verbatim}
-shall be declared. The following preprocessor symbols (macros) shall be
-defined: \index{Hash!macro names}\index{macros!defined by Hash}
-\indexmacro{HASHCOUNT_T_MAX}
-\indexmacro{HASH_VAL_T_BIT}
-\indexmacro{HASH_VAL_T_MAX}
-\indexmacro{HASH_H}
-\begin{verbatim}
- HASHCOUNT_T_MAX
- HASH_VAL_T_BIT
- HASH_H\end{verbatim}
-\index{symbols!reserved by Hash}\index{Hash!reserved symbols}
-Macro identifiers which begin with the upper-case prefix \verb|HASH| are
-reserved for future extensions to the \verb|hash.h| header, as are
-names in the ordinary and tag namespaces which begin with \verb|hash_|,
-\verb|hnode_| or \verb|hscan_|. External names which begin with \verb|hash_|,
-\verb|hnode_| or \verb|hscan_| are reserved by the Kazlib library regardless of
-what headers are included.
-
-\subsubsection{The {\tt hash_t} type}
-
-\indextype{hash_t}
-The type \verb|hash_t| is an opaque data type which maintains information about
-the current state of a single hash. From the programmer's viewpoint, a hash
-consists of an instance of the \verb|hash_t| type, plus zero or more instances
-of the type \verb|hnode_t|. An instance of the \verb|hash_t| type can be
-dynamically created using the \verb|hash_create| function, and destroyed by the
-\verb|hash_destroy| function. Alternately, the program can declare an object
-of type \verb|hash_t| and have it initialized via the \verb|hash_init|
-function. When initializing a hash this way, the user must also provide
-a fixed-size array of \verb|hnode_t *| objects which serves as the hash table.
-\footnote{A hash initialized this way does not support extendible hashing,
-because there is no mechanism for growing the user-supplied array.}
-
-\subsubsection{The {\tt hnode_t} type}
-
-\indextype{hnode_t}
-The \verb|hnode_t| type is an opaque type that represents a single element
-that can be inserted into a hash. A hash node contains a a reference to
-satellite data provided by the user. Nodes may be dynamically created by the
-\verb|hnode_create| function. Alternately, the program may supply an
-\verb|hnode_t| object that can be initialized by the \verb|hnode_init|
-function.
-
-\subsubsection{The {\tt hash_comp_t} type}
-
-\indextype{hash_comp_t}
-The \verb|hash_comp_t| type is a typedef name for the pointer-to-function type
-\begin{verbatim}
- int (*)(const void *, const void *);
-\end{verbatim}
-In the context of the Hash component, this type denotes pointers to
-comparison functions.
-
-\subsubsection{The {\tt hscan_t} type}
-
-\indextype{hscan_t}
-The \verb|hscan_t| typedef stands for an opaque type which represents
-context information for traversing a hash. It is initialized by the
-\verb|hash_scan_begin| function, which specifies a hash to be
-traversed. Successive elements are retrieved using the \verb|hash_scan_next|
-function, which eventually indicates that no more elements
-remain. Inserting to, or deleting from a hash other than using
-the function \verb|hash_scan_delete| causes any \verb|hscan_t|
-objects that refer to it to become indeterminate.
-
-\subsubsection{The {\tt hashcount_t} type}
-
-\indextype{hashcount_t}
-\indexmacro{HASHCOUNT_T_MAX}
-This is an unsigned integral type which is capable of representing the number
-of nodes in a hash.
-The \verb|HASHCOUNT_T_MAX| macro expands to a
-constant expression of type \verb|hashcount_t| which specifies the maximum
-value of that type.
-
-\subsubsection{The {\tt hash_val_t} type}
-
-\indextype{hash_val_t}
-\indexmacro{HASH_VAL_T_MAX}
-The \verb|hash_val_t| type is an unsigned integral type capable of
-holding at least 32 bits. The purpose of this type is to represent the
-output values of hashing functions.
-The \verb|HASH_VAL_T_MAX| macro expands to a
-constant expression of type \verb|hash_val_t| which specifies the maximum
-value of that type.
-
-\subsubsection{The {\tt hnode_alloc_t} type}
-
-\index{Hash!allocator function}
-The \verb|hnode_alloc_t| identifier is a typedef name for the pointer-to-function
-type
-\begin{verbatim}
- hnode_t *(*)(void *);
-\end{verbatim}
-In other words, a pointer to a function that takes a \verb|void *|
-argument and returns a pointer to \verb|hnode_t|.
-A function of this type which meets certain behavior criteria may be
-registered with a \verb|hash_t| object as node allocator, together
-with a compatible deallocator function. The \verb|void *| argument
-passes user-specified context information through to the
-allocator routines (see section \ref{section:hash_set_allocator}).
-
-\subsubsection{The {\tt hnode_free_t} type}
-
-\index{Hash!deallocator function}
-The \verb|hnode_free_t| identifier is a typedef name for the
-pointer-to-function type
-\begin{verbatim}
- void (*)(hnode_t *, void *);
-\end{verbatim}
-A function of this type which meets certain behavior criteria may be
-registered with a \verb|hash_t| object as node deallocator
-together with a compatible allocator function.
-
-\subsubsection{The {\tt hash_fun_t} type}
-
-\index{hashing function}
-The \verb|hash_fun_t| identifier is a typedef name for the
-pointer-to-function type
-\begin{verbatim}
- hash_val_t (*hash_fun_t)(const void *);
-\end{verbatim}
-A function of this type which behaves a certain way is called
-a {\it hashing function}. To be a viable hashing function, such
-a function must take a pointer to a key object, and produce
-an integer value that depends only on the contents of the key,
-and possibly on information that does not change over the lifetime of any hash
-for which that hashing function is used. Additional requirements for hashing
-functions are introduced later.
-
-\subsubsection{The {\tt hash_val_t_bit} object}
-
- \indexobject{hash_val_t_bit}
- \synopsis
- \begin{verbatim}
- extern int hash_val_t_bit;\end{verbatim}
-
- \description
-
- The \verb|hash_val_t_bit| object of type int has a fixed value
- which counts the number of bits in the \verb|hash_val_t| object.
- The program shall not store a value into this object.
-
- The value of \verb|hash_val_t_bit| need not be correct until the
- first successful call to \verb|hash_create| or to \verb|hash_init|
- completes.
-
- The implementation shall provide the macro \verb|HASH_VAL_T_BIT| which
- expands to a non-lvalue expression that has the same value and type as the
- object, but which may be a constant expression.\footnote{The intent of
- providing these values is to ease the implementation of portable hashing
- functions that take advantage of all of the available bits of a given
- Kazlib implementation. Alternately, hashing functions may be constructed to
- only use the lower 32 bits of the type.}
-
-\subsubsection{The {\tt hash_create} function}
-
- \indexfunc{hash_create}
- \index{Hash!creation of}
- \index{create!hash object}
- \synopsis
- \begin{verbatim}
- hash_t *hash_create(hashcount_t, hash_comp_t, hash_fun_t);\end{verbatim}
-
- \description
-
- If sufficient resources exist, the \verb|hash_create| function instantiates
- and initializes an object of type \verb|hash_t| and returns a pointer to
- it. Otherwise it returns a null pointer.
-
- The first argument establishes the capacity of the hash, which is
- initially empty.
-
- The second argument is a pointer to a comparison function that will be
- associated with the \verb|hash_t| object for its entire duration.
-
- \index{hashing function}
- The third argument is either null or a pointer to a hashing function
- that is permanently associated with the object. If it is null, a {\it default
- hashing function\/} is assigned by the implementation.
-
- The hashing function shall be invoked with an argument that is one
- of the keys that are being inserted into, or sought after, in the
- hash. The hashing function must produce the same value each time it
- is called for a given key. It is up to the hash user to define the
- representation of keys, to manage their storage, and to provide a matching
- hashing function. The hash stores only generic \verb|void *| pointers to
- keys.
-
- The default hashing function assumes that keys are null terminated
- strings. That is to say, it behaves as though its \verb|void *|
- argument points to the first elements of an array of \verb|unsigned|
- \verb|char|, the last of which is a null character. The use of
- the default hashing function with keys that do not have this representation
- results in undefined behavior.
-
-\subsubsection{The {\tt hash_set_allocator} function}
-
- \indexfunc{hash_set_allocator}
- \label{section:hash_set_allocator}
-
- \synopsis
- \begin{verbatim}
- void hash_set_allocator(hash_t *, hnode_alloc_t,
- hnode_free_t, void *);\end{verbatim}
-
- \constraints
-
- The second and third arguments---the function pointers---shall either
- both be null, or both be non-null. The hash pointed at by the first
- argument shall be empty.
-
- \description
-
- When a hash is initialized, it is outfitted with a pair of default
- node allocation functions. These functions may be replaced with functions
- supplied by the program by calling the \verb|hash_set_allocator| function
- and specifying two suitable pointers. If these pointers are null, the
- default functions are restored.
-
- These functions are called to allocate and free \verb|hnode_t|
- objects by the functions \verb|hash_alloc_insert|
- and \verb|hash_delete_free| (see sections
- \ref{section:hash_delete_free} and \ref{section:hash_alloc_insert}).
-
- If sufficient resources exist, the allocation function shall
- return a pointer to a unique storage object that is large enough
- and suitably aligned to represent an object of type \verb|dnode_t|.
- Otherwise, the function shall return a null pointer.
-
- The deallocation function shall be capable of disposing of the
- objects created by the matching allocator function.
-
-
-\subsubsection{The {\tt hash_destroy} function}
-
- \indexfunc{hash_destroy}
- \synopsis
- \begin{verbatim}
- void hash_destroy(hash_t *);\end{verbatim}
-
- \constraints
-
- The hash pointed at by the first argument shall be empty.
-
- \description
-
- The \verb|hash_destroy| function deinitializes and deallocates a hash
- that was created with \verb|hash_create|.
- All pointers and \verb|hscan_t| objects that referred to the hash become
- indeterminate.
-
-\subsubsection{The {\tt hash_free_nodes} function}
-
- \indexfunc{hash_free_nodes}
- \synopsis
- \begin{verbatim}
- void hash_free_nodes(hash_t *);\end{verbatim}
-
- \description
-
- The \verb|hash_free_nodes| function removes each node from
- the hash and destroys it as if by calling \verb|hash_delete_free|
- (Section \ref{section:hash_delete_free}). The order in which
- the nodes are destroyed is unspecified.
-
-\subsubsection{The {\tt hash_free} function}
-
- \indexfunc{hash_free}
- \synopsis
- \begin{verbatim}
- void hash_free(hash_t *);\end{verbatim}
-
- \description
-
- Every node in the hash is removed from the hash and is then subject to the
- deallocation function. The overall effect is as if the function
- \verb|hash_delete_free| (Section \ref{section:hash_delete_free}) were
- invoked on each node, and then \verb|hash_destroy| invoked on the
- hash itself.
-
- This function is obsolescent, and will be removed from some future revision
- of this document.
-
-\subsubsection{The {\tt hash_init} function}
-
- \indexfunc{hash_init}
- \synopsis
- \begin{verbatim}
- hash_t *hash_init(hash_t *, hashcount_t, hash_comp_t,
- hash_fun_t, hnode_t **, hashcount_t);
- \end{verbatim}
-
- \constraints
-
- The last argument, which specifies the size of the program-supplied table,
- shall be integral power of two that is greater than one---that is to say, an
- integer of the form $2^k$ where $k$ is a positive integer.
-
- \description
-
- The \verb|hash_init| function configures the specified \verb|hash_t| object
- to use a specified array of \verb|hnode_t *| pointer objects as a table.
- The user is responsible for providing storage for the \verb|hash_t|
- object and the array. As in the \verb|hash_create| interface,
- the second parameter specifies the capacity, and the subsequent
- arguments specify the comparison and hashing function, respectively.
- The last two arguments specify the table of pointers. The array object
- shall have at least as many elements as indicated by the last parameter,
- otherwise the behavior is undefined. The call to \verb|hash_init| is said
- to register the array with the hash.
-
- The program shall not register the same array with more than one hash.
- More specifically, once the program modifies a registered array, or
- registers it with another hash, it must discontinue use of the first hash.
- \footnote{Note that no explicit deinitialization function is provided to
- dissociate the array. A program disposes of a hash created by
- {\tt hash_init} by discontinuing its use.}
-
-\subsubsection{The {\tt hash_insert} function}
-
- \indexfunc{hash_insert}
- \label{section:hash_insert}
- \synopsis
- \begin{verbatim}
- void hash_insert(hash_t *, hnode_t *, const void *);\end{verbatim}
-
- \constraints
- The hash is not full. The key specified by the \verb|void *| parameter
- does not already exist in the specified hash. The node specified
- by the second parameter is not already inserted into a hash.
-
- \description
- The \verb|hash_insert| function adds a new node to a hash. The user
- must supply a node object that was initialized with \verb|hnode_init|
- or dynamically created with \verb|hnode_create|. If the node is
- already inserted into the same hash or any other hash, the behavior
- is undefined.
-
- A program may modify a key or node that has been inserted into a hash, or
- cause the storage of the key or the node to become invalid. However, any
- subsequent use of the hash invokes undefined behavior, with the following
- exception: the data pointer stored within a node may be modified using the
- \verb|hnode_put| function.
-
- The \verb|hash_insert| function invokes the hashing function callback with
- the key pointer as the argument.
-
- The \verb|hash_insert| function may need to acquire additional storage in
- order to support hash table growth. If the storage allocation fails, the
- function shall fully recover, and insert the node without growing the
- table.
-
- The Hash implementation shall not modify the storage referenced by a key,
- and shall not access it other than indirectly through the supplied hashing
- and comparison functions.
-
-\subsubsection{The {\tt hash_lookup} function}
-
- \indexfunc{hash_lookup}
- \synopsis
- \begin{verbatim}
- hnode_t *hash_lookup(hash_t *, const void *);\end{verbatim}
-
- \description
-
- The \verb|hash_lookup| function searches the given hash for a node
- matching the given key. Unless the hash is empty, the key shall be
- compared against one or more keys that are already in the hash,
- using the comparison function. The key pointer may
- be identical to one that has already been inserted into the
- hash.\footnote {In that case, the comparison function must correctly
- cope with aliased parameters}.
-
- If the key is found in the hash, a pointer to the corresponding node
- is returned.\footnote{The corresponding node is the one that was specified
- in the call to {\tt hash_insert} together with the matching key.}
-
- If the key is not found, a null pointer is returned.
-
-\subsubsection{The {\tt hash_delete} function}
-
- \indexfunc{hash_delete}
- \synopsis
- \begin{verbatim}
- hnode_t *hash_delete(hash_t *, hnode_t *);\end{verbatim}
-
- \constraints
- The specified node is an occupant of the given hash.
-
- \description
- The \verb|hash_delete| function removes from the given hash a
- node that has previously been inserted into it. The key under
- which the node was inserted is also removed from the hash.\footnote{Thus
- the program may arbitrarily manipulate the removed key without destroying
- the integrity of the hash.}
-
- Any existing \verb|hscan_t| iterator which is associated with the
- hash becomes indeterminate.\footnote{To delete the current node during hash
- table traversal, the {\tt hash_scan_delete} function must be used
- instead.}
-
-
-\subsubsection{The {\tt hash_alloc_insert} function}
-
- \label{section:hash_alloc_insert}
- \indexfunc{hash_alloc_insert}
-
- \synopsis
- \begin{verbatim}
- int hash_alloc_insert(hash_t *, const void *, void *);\end{verbatim}
-
- \constraints
-
- The second argument specifies the insertion key. The hash shall not
- already contain this key.
-
- \description
-
- The \verb|hash_alloc_insert| function dynamically allocates and
- initializes a \verb|hnode_t| object and inserts it into the
- given hash. The second argument and third arguments are pointers
- to user data and key objects, either of which may be null.
-
- The allocation is performed by a call to the default allocation
- function, or to the function that was configured using
- \verb|hash_set_allocator| (Section \ref{section:hash_set_allocator}).
-
- If the allocation succeeds, the insertion is performed and
- the value 1 is returned. If the allocation fails, no insertion is
- performed and 0 is returned.
-
-\subsubsection{The {\tt hash_delete_free} function}
-
- \label{section:hash_delete_free}
- \indexfunc{hash_delete_free}
-
- \synopsis
- \begin{verbatim}
- void hash_delete_free(hash_t *, hnode_t *)
- \end{verbatim}
-
- \constraints
- The given node can be found within the given hash.
-
- \description
- The \verb|hash_delete_free| function is the reverse of
- \verb|hash_alloc_insert|. It removes the given node form the
- hash as if by a call to \verb|hash_delete| and then deletes it using the
- default or user-defined allocator (Section
- \ref{section:hash_set_allocator}). If the given node had not been created
- using \verb|hash_alloc_insert|, the behavior is undefined.
-
-\subsubsection{The {\tt hnode_put} function}
-
- \indexfunc{hnode_put}
- \synopsis
- \begin{verbatim}
- void hnode_put(hnode_t *, void *);\end{verbatim}
-
- \description
- The function \verb|hnode_put| replaces the data element
- associated with the hash node.
-
-\subsubsection{The {\tt hnode_get} function}
-
- \indexfunc{hnode_get}
- \synopsis
- \begin{verbatim}
- void *hnode_get(hnode_t *);\end{verbatim}
-
- \description
- The \verb|hnode_get| function retrieves the \verb|void * | data value
- associated with the given hash node.
-
-\subsubsection{The {\tt hnode_getkey} function}
-
- \indexfunc{hnode_getkey}
- \synopsis
- \begin{verbatim}
- const void *hnode_getkey(hnode_t *);\end{verbatim}
-
- \description
-
- The \verb|hnode_getkey| function retrieves the \verb|void *| key value
- associated with the given node. A node acquires an associated key
- when it is inserted into a hash (see section \ref{section:hash_insert}).
- Invoking \verb|hnode_getkey| on a node that has not been inserted
- into a hash results in undefined behavior.
-
-\subsubsection{The {\tt hash_count} function}
-
- \indexfunc{hash_count}
- \synopsis
- \begin{verbatim}
- hashcount_t hash_count(hash_t *);\end{verbatim}
-
- \description
- The \verb|hash_count| function returns a value which represents the number
- of nodes currently stored in the hash pointed at by the argument.
-
-\subsubsection{The {\tt hash_size} function}
-
- \indexfunc{hash_size}
- \synopsis
- \begin{verbatim}
- hashcount_t hash_size(hash_t *hash)\end{verbatim}
-
- \description
- The \verb|hash_size| function returns an implementation-defined value that
- depends on the number of entries in the given hash. The intent is that the
- value represent the size of the internal hash table managed by the given
- hash.
-
-\subsubsection{The {\tt hash_isfull} function}
-
- \indexfunc{hash_isfull}
- \synopsis
- \begin{verbatim}
- int hash_isfull(hash_t *);\end{verbatim}
-
- \description
- The \verb|hash_isfull| function returns 1 if the hash is full,
- otherwise it returns 0.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-
-\subsubsection{The {\tt hash_isempty} function}
-
- \indexfunc{hash_isempty}
- \synopsis
- \begin{verbatim}
- int hash_isempty(hash_t *);\end{verbatim}
-
- \description
- The \verb|hash_isempty| function returns 1 if the given hash is empty,
- otherwise it returns 0.
-
-\subsubsection{The {\tt hash_scan_begin} function}
-
- \indexfunc{hash_scan_begin}
- \synopsis
- \begin{verbatim}
- void hash_scan_begin(hscan_t *, hash_t *);\end{verbatim}
-
- \description
- The \verb|hash_scan_begin| initializes the \verb|hscan_t| iterator object,
- preparing it for a traversal of the given hash.
-
- After this initialization, if the hash is modified in any way by
- the performance of an insertion or deletion operation, the
- value of the \verb|hscan_t| object becomes indeterminate,
- with one exception: the \verb|hash_scan_delete| function or the
- \verb|hash_scan_delfree| function may be used to delete the current
- node.
-
-\subsubsection{The {\tt hash_scan_next} function}
-
- \indexfunc{hash_scan_next}
- \synopsis
- \begin{verbatim}
- hnode_t *hash_scan_next(hscan_t *);\end{verbatim}
-
- \description
- If any unvisited nodes remain, the \verb|hash_scan_next| function advances
- to the next one and returns a pointer to it. Otherwise, it returns a null
- pointer. Repeated invocations of \verb|hash_scan_next| return a pointer to
- every node that has been inserted into the table, in no particular order,
- such that no node is reported twice.
-
-\subsubsection{The {\tt hash_scan_delete} function}
-
- \indexfunc{hash_scan_delete}
- \synopsis
- \begin{verbatim}
- hnode_t *hash_scan_delete(hash_t *, hnode_t *);
- \end{verbatim}
-
- \constraints
- The specified node is an occupant of the given hash.
-
- \description
- This function is almost exactly like \verb|hash_delete| except that it may
- be used to delete a node that has been most recently obtained from
- \verb|hash_scan_next| without destroying the validity of the \verb|hscan_t|
- iterator from which the node was obtained.
-
-\subsubsection{The {\tt hash_scan_delfree} function}
-
- \label{section:hash_scan_delfree}
- \indexfunc{hash_scan_delfree}
-
- \synopsis
- \begin{verbatim}
- void hash_scan_delfree(hash_t *, hnode_t *)
- \end{verbatim}
-
- \constraints
- The given node can be found within the given hash.
-
- \description
- The \verb|hash_scan_delfree| function is similar to
- \verb|hash_delete_free|. It removes the given node form the
- hash and then deletes it using the default or user-defined allocator
- (Section \ref{section:hash_set_allocator}). If the given node
- had not been created using \verb|hash_alloc_insert|, the behavior
- is undefined.
-
- The deletion from the hash is performed as if by a call to
- \verb|hash_scan_delete|, thus it is safe to delete a node that
- was most recently obtained from a \verb|hash_scan_next| without
- destroying the validity of the \verb|hscan_t| iterator.
-
-\subsubsection{The {\tt hash_verify} function}
-
- \indexfunc{hash_verify}
- \synopsis
- \begin{verbatim}
- int hash_verify(hash_t *hash);\end{verbatim}
-
- \description
- The intent of the \verb|hash_verify| function is to perform a verification
- on the hash object, regardless of whether the Kazlib implementation is
- operated in verification or production mode. If the hash object
- and its constituent nodes have been correctly manipulated, and the
- program has not caused any undefined behaviors, the value $1$ is returned.
- Otherwise, the function may be able to, but is not guaranteed to, detect
- corruption, and return the value zero.
-
-\subsubsection{The {\tt hnode_create} function}
-
- \indexfunc{hnode_create}
- \synopsis
- \begin{verbatim}
- hnode_t *hnode_create(void *);\end{verbatim}
-
- \description
- The \verb|hnode_create| function dynamically allocates a hash node,
- stores in it the data value specified in the argument and
- returns a pointer to it. The allocation is performed by a call to the
- standard \verb|malloc| function. If the allocation fails, a null
- pointer is returned.
-
- The node's key pointer remains indeterminate until it is the subject of a
- \verb|hash_insert| operation.
-
-\subsubsection{The {\tt hnode_init} function}
-
- \indexfunc{hnode_init}
- \synopsis
- \begin{verbatim}
- hnode_t *hnode_init(hnode_t *, void *);\end{verbatim}
-
- \description
- The \verb|hnode_init| function initializes the contents
- of the specified hash node object, assigning it the
- data value specified as the second argument.
- The first argument is a pointer which refers to
- a data object that has a suitable size and alignment
- for the representation of an \verb|hnode_t| type.
- After initialization with \verb|hnode_init|, the object is subsequently
- eligible as an operand to the functions of the hash component,
- other than \verb|hnode_getkey|.
-
- The node's key pointer remains indeterminate until it is the subject of a
- \verb|hash_insert| operation.
-
-\subsubsection{The {\tt hnode_destroy} function}
-
- \indexfunc{hnode_destroy}
- \synopsis
- \begin{verbatim}
- void hnode_destroy(hnode_t *);\end{verbatim}
-
- \description
- The \verb|hnode_destroy| function destroys a hash node that has been
- allocated with the \verb|hnode_create| function. The value of any pointer
- that referred to the node that was thus freed is indeterminate.
-
- If the node is currently the occupant of a hash, the behavior is undefined
- if the hash is subsequently used.
-
-\subsection{Implementation}
-
-TODO
-
-\section{Dictionary component}
-
-\index{Dictionary}
-The Dictionary component provides a means to manage ordered sequences of
-elements, having the following properties:
-\begin{enumerate}
-\item If the dictionary is not empty, a first and last element can be identified.
- In a dictionary having only one element, that one element is both the first and
- last element.
-\item Each element that is not the last element has another element as its
- {\it successor}.
- \index{successor!of a dictionary element}
- \index{Dictionary!successor of an element}
-\item Each element that is not the first element has a {\it predecessor}.
- \index{predecessor!of a dictionary element}
- \index{Dictionary!predecessor of an element}
-\item No element is the predecessor or successor of more than one element.
-\item If one element is the successor of another, the other is necessarily the
- predecessor of the first.
-\item Each element is associated with a piece of information known as
- the key. The sequence is ordered according to the relation imposed
- by the comparison function: the key of an element compares
- greater than or equal to the key of its predecessor.
-\item If duplicate keys are present, then elements
- having the same key form a subsequence with no other keys in it, which
- follows from the previous property. No additional ordering is imposed
- within such subsequences.
-\item Each element is associated with arbitrary satellite data.
-\end{enumerate}
-
-The Dictionary component supports efficient operations over such ordered
-sequences: such as insertion, deletion, ordered traversal, as well as exact and
-range searches.\footnote{The implicit association of keys and satellite data,
-together with the ability of efficiently search by key to retrieve data, gives
-rise to the term {\it dictionary}. A dictionary need not be ordered; a hash can
-therefore also be considered to be a kind of dictionary; the Kazlib
-nomenclature is somewhat unfortunate in that regard.}
-
-The number of elements that can be stored in a dictionary is limited; maximum
-number of entries in a dictionary is known as its {\it capacity}.
-
-\subsection{Interface}
-
-\subsubsection{The {\tt dict.h} header}
-
-Each C or C++ translation unit that is to use the functionality of the Dict
-component shall include the header \verb|dict.h|. This header shall
-contain declarations of types and external functions, and definitions of
-macros. The following typedef names shall be
-defined:\index{Dict!typedef names}
-\index{typedefs!defined by Dict}
-\begin{verbatim}
- dict_t dnode_process_t
- dnode_t dnode_alloc_t
- dictcount_t dnode_free_t
- dict_comp_t dict_load_t
-\end{verbatim}
-In addition, the following structure tags may be defined:\index{Dict!tag names}
-\index{tags!defined by Dict}
-\begin{verbatim}
- struct dict_t
- struct dnode_t
-\end{verbatim}
-The following external function names shall be declared:
-\index{Dict!function names}\index{functions!defined by Dict}
-\begin{verbatim}
- dict_create dict_count
- dict_set_allocator dict_isempty
- dict_destroy dict_isfull
- dict_free_nodes dict_contains
- dict_init dict_allow_dupes
- dict_verify dnode_is_in_a_dict
- dict_lookup dnode_create
- dict_lower_bound dnode_init
- dict_upper_bound dnode_destroy
- dict_insert dnode_get
- dict_delete dnode_getkey
- dict_alloc_insert dnode_put
- dict_delete_free dict_process
- dict_first dict_load_begin
- dict_last dict_load_next
- dict_next dict_load_end
- dict_prev dict_free
-\end{verbatim}
-The following preprocessor symbols shall be
-defined: \index{Dict!macro names}\index{macros!defined by Dict}
-\indexmacro{DICTCOUNT_T_MAX}
-\indexmacro{DICT_H}
-\begin{verbatim}
- DICTCOUNT_T_MAX
- DICT_H\end{verbatim}
-\index{symbols!reserved by Dict}\index{Dict!reserved symbols}
-Macro identifiers which begin with the upper-case prefix \verb|DICT| are
-reserved for future extensions to the \verb|dict.h| header, as are
-names in the ordinary and tag namespaces which begin with \verb|dict_|
-or \verb|dnode_|. External names which begin with \verb|dict_|
-or \verb|dnode_| are reserved by the Kazlib library regardless of
-what headers are included.
-
-\subsubsection{The {\tt dict_t} type}
-
-\indextype{dict_t}
-The type \verb|dict_t| is an opaque data type which represents a single
-dictionary. A dictionary consists of an instance of the \verb|dict_t| type,
-plus zero or more instances of the type \verb|dnode_t|. An object of type
-\verb|dict_t| can be initialized by the \verb|dict_init| function. Alternately,
-the \verb|dict_create| function will dynamically allocate and initialize a
-dictionary. An empty dictionary created by \verb|dict_create| may be disposed
-of using \verb|dict_destroy|.
-
-\subsubsection{The {\tt dnode_t} type}
-
-\indextype{dnode_t}
-The \verb|dnode_t| type represents a single entry in a dictionary called a
-dictionary node. The object stores a pointer to user data, and a key pointer
-that is assigned to the dictionary node at the time when it is inserted into
-the dictionary. A \verb|dnode_t| may be dynamically created using
-\verb|dnode_create| and destroyed using \verb|dnode_destroy|. Alternately,
-the program may supply storage for a \verb|dnode_t| object and initialize
-it using the \verb|dnode_init| function.
-
-\subsubsection{The {\tt dictcount_t} type}
-
-\indextype{dictcount_t}
-\indexmacro{DICTCOUNT_T_MAX}
-This is an unsigned integral type which is capable of representing the number
-of nodes in a dictionary. The \verb|DICTCOUNT_T_MAX| macro expands to a
-constant expression of type \verb|dictcount_t| which specifies the maximum
-value of that type.
-
-\subsubsection{The {\tt dict_comp_t} type}
-
-\indextype{dict_comp_t}
-The \verb|dict_comp_t| type is a typedef name for the pointer-to-function type
-\begin{verbatim}
- int (*)(const void *, const void *);
-\end{verbatim}
-In the context of the Dictionary component, this type denotes pointers to
-comparison functions.
-
-\subsubsection{The {\tt dnode_process_t} type}
-
-\indextype{dnode_process_t}
-The type \verb|dnode_process_t| is a typedef name for the pointer-to-function type
-\begin{verbatim}
- void (*)(dict_t *, dnode_t *, void *);
-\end{verbatim}
-In the context of the Dictionary component, this is the type of a
-dictionary node processing function (See section \ref{section:dict_process}).
-The first two parameters identify a dictionary and the node within that
-dictionary that is being processed. The third argument is a context pointer.
-
-\subsubsection{The {\tt dnode_alloc_t} type}
-
-\indextype{dnode_alloc_t}
-The type \verb|dnode_alloc_t| is a typedef name for the pointer-to-function type
-\begin{verbatim}
- dnode_t *(*)(void *);
-\end{verbatim}
-A function compatible with this type which meets certain other criteria may be
-registered with a \verb|dict_t| object as a node allocator function
-(See section \ref{section:dict_set_allocator}).
-
-\subsubsection{The {\tt dnode_free_t} type}
-
-\indextype{dnode_free_t}
-The type \verb|dnode_free_t| is a typedef name for the pointer-to-function type
-\begin{verbatim}
- void (*)(dnode_t *, void *);
-\end{verbatim}
-A function compatible with this type which meets certain other criteria may be
-registered with a \verb|dict_t| object as a node deallocator function.
-(See section \ref{section:dict_set_allocator}).
-
-\subsubsection{The {\tt dict_load_t} type}
-
-\indextype{dict_load_t}
-
-The \verb|dict_load_t| type is opaque, and represents a context structure
-used during the process of constructing a dictionary from an ordered list
-of nodes. (See sections \ref{section:dict_load_begin} to
-\ref{section:dict_load_end}).
-
-\subsubsection{The {\tt dict_create} function}
-
- \indexfunc{dict_create}
- \index{Dictionary!creation of}
- \index{create!dictionary object}
-
- \synopsis
- \begin{verbatim}
- dict_t *dict_create(dictcount_t, dict_comp_t);\end{verbatim}
-
- \description
- The \verb|dict_create| function allocates a new
- object of type \verb|dict_t| and initializes it to act as
- a dictionary.
-
- If insufficient resources exist for the allocation,
- a null pointer is returned, otherwise a pointer to the dictionary
- is returned.
-
- The first argument specifies the capacity of the dictionary,
- which is initially empty.
-
- The second argument is a comparison function that is used for comparing
- keys during insertion and searching operations, and is associated
- with the dictionary for its entire duration.
-
-\subsubsection{The {\tt dict_set_allocator} function}
-
- \label{section:dict_set_allocator}
- \indexfunc{dict_set_allocator}
-
- \synopsis
- \begin{verbatim}
- void dict_set_allocator(dict_t *, dnode_alloc_t,
- dnode_free_t, void *);\end{verbatim}
-
- \constraints
-
- The second and third arguments---the function pointers---shall either
- both be null, or both be non-null. The dictionary pointed at by the first
- argument shall be empty.
-
- \description
-
- When a dictionary is initialized, it is outfitted with a pair of default
- node allocation functions. These functions may be replaced with functions
- supplied by the program by calling the \verb|dict_set_allocator| function
- and specifying two suitable pointers. If these pointers are null, the
- default functions are restored.
-
- These functions are called to allocate and free \verb|dnode_t|
- objects by the functions \verb|dict_alloc_insert|
- and \verb|dict_delete_free| (see sections
- \ref{section:dict_delete_free} and \ref{section:dict_alloc_insert}).
-
- If sufficient resources exist, the allocation function shall
- return a pointer to a unique storage object that is large enough
- and suitably aligned to represent an object of type \verb|dnode_t|.
- Otherwise, the function shall return a null pointer.
-
- The deallocation function shall be capable of disposing of the
- objects created by the matching allocator function.
-
-\subsubsection{The {\tt dict_destroy} function}
-
- \indexfunc{dict_destroy}
-
- \synopsis
- \begin{verbatim}
- void dict_destroy(dict_t *);\end{verbatim}
-
- \constraints
-
- The dictionary pointed at by the first argument shall be empty.
-
- \description
-
- The \verb|dict_destroy| function deinitializes and deallocates a dictionary
- object that was created by \verb|dict_create|. All pointers that
- referred to the dictionary become indeterminate.
-
-\subsubsection{The {\tt dict_free_nodes} function}
-
- \indexfunc{dict_free_nodes}
-
- \synopsis
- \begin{verbatim}
- void dict_free_nodes(dict_t *);\end{verbatim}
-
- \description
-
- Every node in the dictionary is removed from the dictionary and is then
- subject to the deallocation function, as if the function
- \verb|dict_delete_free| (Section \ref{section:dict_delete_free}) were
- invoked on each node, in some unspecified order.
-
-\subsubsection{The {\tt dict_free} function}
-
- \indexfunc{dict_free}
-
- \synopsis
- \begin{verbatim}
- void dict_free(dict_t *);\end{verbatim}
-
- \description
-
- This function is obsolescent, and will be removed from some future revision
- of this document. It is equivalent to \verb|dict_free_nodes|.
-
-\subsubsection{The {\tt dict_init} function}
-
- \indexfunc{dict_init}
-
- \synopsis
- \begin{verbatim}
- dict_t *dict_init(dict_t *, dictcount_t, dict_comp_t);\end{verbatim}
-
- \description
-
- The \verb|dict_init| function prepares specified \verb|dict_t| object
- to behave as a dictionary that may subsequently be used with the other
- dictionary functions.
-
- The first argument points to the \verb|dict_t| object to be initialized.
- The second argument specifies the capacity of the dictionary. The third
- argument is a pointer to the comparison function which shall be associated
- with the dictionary for its entire duration.
-
-\subsubsection{The {\tt dict_verify} function}
-
- \indexfunc{dict_verify}
-
- \synopsis
- \begin{verbatim}
- int dict_verify(dict_t *);\end{verbatim}
-
- \description
-
- The intent of the \verb|dict_verify| function is to perform a verification
- on the dictionary object, regardless of whether the Kazlib implementation
- is operated in verification or production mode. If the dictionary object
- and its constituent nodes have been correctly manipulated, and the program
- has not caused any undefined behaviors, the value $1$ is returned.
- Otherwise, the function may be able to, but is not guaranteed to, detect
- corruption, and return the value zero.
-
-\subsubsection{The {\tt dict_lookup} function}
-
- \indexfunc{dict_lookup}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_lookup(dict_t *, const void *);\end{verbatim}
-
- \description
- The \verb|dict_lookup| function searches the given dictionary for a node
- matching the given key. Unless the dictionary is empty, the key shall be
- compared against one or more keys that are already in the dictionary, using
- the comparison function. The key pointer may be identical to one that has
- already been inserted into the dictionary.
-
- If the key is found in the dictionary, a pointer to the corresponding node
- is returned.
-
- If the key is not found, a null pointer is returned.
-
- If the dictionary contains more than one key which matches the search
- key, then the first key in the subsequence of duplicate keys is returned.
-
-\subsubsection{The {\tt dict_lower_bound} function}
-
- \indexfunc{dict_lower_bound}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_lower_bound(dict_t *, const void *);\end{verbatim}
-
- \description
-
- The \verb|dict_lower_bound| function searches the dictionary in a manner
- similar to \verb|dict_lookup|.
-
- If the given key exists in the dictionary, the behavior is exactly the same
- as \verb|dict_lookup|.
-
- However, if the key is not found, then the node which has the smallest key
- that is greater than the search key is returned. If no such key exists
- (because the search key is higher than any other key in the dictionary
- or the dictionary is empty) then a null pointer is returned.
-
- \example
- Suppose that pointer \verb|d| refers to a dictionary whose registered
- comparison function performs lexicographic comparisons on ordinary
- C strings, similar to \verb|strcmp|. To iterate over all keys that
- begin with the letter \verb|d|, the following idiom can be used:
- \begin{verbatim}
- dict_t *d;
- dnode_t *n, *start, *end;
- /*...*/
- start = dict_lower_bound(d, "d");
- end = dict_lower_bound(d, "e");
- for (n = start; n != end; n = dict_next(d, n)) {
- /* n points to each node in turn whose
- key starts with 'd' */
- }
- \end{verbatim}
- Note that if the dictionary is empty, or has keys which are all lower
- than \verb|"d"|, then both \verb|start| and \verb|end| shall be null
- pointers, and the loop body will never execute since the two are equal.
- Also note that if there are keys that begin with \verb|d| and the
- dictionary's last node has a key that starts with \verb|d|, then \verb|end|
- is null, otherwise \verb|end| points to the first key that doesn't begin
- with \verb|d|. In both cases, the loop will terminate after processing the
- last \verb|d| key, because \verb|dict_next| shall produce a pointer that is
- equal to \verb|end|.
-
-\subsubsection{The {\tt dict_upper_bound} function}
-
- \indexfunc{dict_upper_bound}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_upper_bound(dict_t *, const void *);\end{verbatim}
-
- \description
-
- The \verb|dict_upper_bound| function searches the dictionary in a manner
- similar to \verb|dict_lookup|.
-
- If the given key exists in the dictionary, the behavior is exactly the same
- as \verb|dict_lookup| with one difference:
- If the dictionary contains more than one key which matches the search
- key, then the last key in the sequence of duplicates is returned,
- rather than the first.
-
- However, if the key is not found, then the node which has the greatest key
- that is lower than the search key is returned. If no such key exists
- (because the search key is lower than any other key in the dictionary
- or the dictionary is empty) then a null pointer is returned.
-
- \example
- The following idiom can be used to iterate over a sequence of duplicate
- keys without the overhead of performing a full comparison before each
- iteration to detect the first non-matching key.
- \begin{verbatim}
- dict_t *d;
- void *key;
- dnode_t *n, *start, *end;
-
- /* ... Initialize d, and key. ...*/
- start = dict_lower_bound(d, key);
- end = dict_upper_bound(d, key);
-
- /* advance end to first non-matching key */
- if (end != 0)
- end = dict_next(d, end);
- else
- end = start; /* start == dict_first(d) in this case */
-
- for (n = start; n != end; n = dict_next(d, n)) {
- /* n points to duplicate keys in turn */
- }
- \end{verbatim}
- Immediately prior to the execution of the if statement, exactly one of the
- following conditions is true:
- \begin{itemize}
- \item The key was found in the dictionary; \verb|start| points to the
- first duplicate node and \verb|end| points to the last.
- \item The dictionary has only higher keys than the search key; \verb|start|
- points to the first node in the dictionary and \verb|end| is null.
- \item The dictionary has only lower keys than the search key; \verb|end|
- points to the last node in the dictionary, and \verb|start| is null.
- \item The dictionary has both lower and higher keys; \verb|end| and \verb|start|
- point to two consecutive nodes, respectively, such that the node
- pointed at by \verb|end| has a lower key than the search key and
- the node pointed at by \verb|start| has a higher key.
- \item The dictionary is empty; \verb|start| and \verb|end| are null.
- \end{itemize}
- The if statement ensures that if the dictionary contains no matching
- keys, than \verb|start| and \verb|end| are equal, and if the dictionary
- contains one or more matching keys, than \verb|end| points to the first
- non-matching node, or is null if there is no such node. Thus the loop
- performs correctly in all circumstances.
-
-\subsubsection{The {\tt dict_insert} function}
-
- \label{section:dict_insert}
- \indexfunc{dict_insert}
-
- \synopsis
- \begin{verbatim}
- void dict_insert(dict_t *, dnode_t *, const void *);\end{verbatim}
-
- \constraints
- The dictionary is not full. If the dictionary has not been configured
- to allow duplicate keys, the key specified by the \verb|void *| parameter
- does not already exist in the dictionary.
-
- \description
- The \verb|dict_insert| function adds a new node to a dictionary. The user
- must supply a node object that was initialized with \verb|dnode_init| or
- dynamically created with \verb|dnode_create|. If the node is already
- inserted into the same dictionary or any other dictionary, the behavior is
- undefined.
-
- Duplicate keys may be inserted into a dictionary only if the dictionary
- has been configured to permit duplicate keys (see section
- \ref{section:dict_allow_dupes}). If this is the case, it is also
- permissible to insert the same key more than once: the implementation shall
- not distinguish between distinct keys that are declared equal by a
- correctly designed comparison function, and two key pointers that refer to
- the same key.
-
- A program may modify a key or node that has been inserted into a
- dictionary, or cause the storage of the key or the node to become invalid.
- However, any subsequent use of the dictionary invokes undefined behavior, with
- the following exception: the data pointer stored within a node may be
- modified using the \verb|dnode_put| function.
-
- The Dictionary implementation shall not modify the storage referenced by a
- key, and shall not access it other than indirectly through the supplied
- comparison function.
-
-\subsubsection{The {\tt dict_delete} function}
-
- \indexfunc{dict_delete}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_delete(dict_t *, dnode_t *);\end{verbatim}
-
- \constraints
- The specified node is an occupant of the given dictionary.
-
- \description
- The \verb|dict_delete| function removes from the given dictionary a
- node that has previously been inserted into it. The key under
- which the node was inserted is also removed from the dictionary.
-
-\subsubsection{The {\tt dict_alloc_insert} function}
-
- \label{section:dict_alloc_insert}
- \indexfunc{dict_alloc_insert}
-
- \synopsis
- \begin{verbatim}
- int dict_alloc_insert(dict_t *, const void *, void *);\end{verbatim}
-
- \constraints
-
- The second argument specifies the insertion key. The dictionary shall not
- already contain this key unless it has been configured as allowing
- duplicates.
-
- \description
-
- The \verb|dict_alloc_insert| function dynamically allocates and
- initializes a \verb|dnode_t| object and inserts it into the
- given dictionary. The second argument and third arguments are pointers
- to user data and key objects, either of which may be null.
-
- The allocation is performed by a call to the default allocation
- function, or to the function that was configured using
- \verb|dict_set_allocator| (Section \ref{section:dict_set_allocator}).
-
- If the allocation succeeds, the insertion is performed and
- the value 1 is returned. If the allocation fails, no insertion is
- performed and 0 is returned.
-
-\subsubsection{The {\tt dict_delete_free} function}
-
- \label{section:dict_delete_free}
- \indexfunc{dict_delete_free}
-
- \synopsis
- \begin{verbatim}
- void dict_delete_free(dict_t *, dnode_t *);\end{verbatim}
-
- \constraints
- The given node can be found within the given dictionary.
-
- \description
- The \verb|dict_delete_free| function is the reverse of
- \verb|dict_alloc_insert|. It removes the given node form the
- dictionary and then deletes it using the default or user-defined allocator
- (Section \ref{section:dict_set_allocator}). If the given node
- had not been created using \verb|dict_alloc_insert|, the behavior
- is undefined.
-
-\subsubsection{The {\tt dict_first} function}
-
- \indexfunc{dict_first}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_first(dict_t *);\end{verbatim}
-
- \description
- If the dictionary pointed at by the argument is empty, a null pointer
- is returned. Otherwise, a pointer to the first node in that dictionary is
- returned.
-
-\subsubsection{The {\tt dict_last} function}
-
- \indexfunc{dict_last}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_last(dict_t *);\end{verbatim}
-
- \description
- If the dictionary pointed at by the argument is empty, a null pointer
- is returned. Otherwise, a pointer to the last node in that dictionary is
- returned.
-
-
-\subsubsection{The {\tt dict_next} function}
-
- \indexfunc{dict_next}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_next(dict_t *, dnode_t *);\end{verbatim}
-
- \constraints
- The node pointed at by the second argument is an occupant of the dictionary
- pointed at by the first argument.
-
- \description
- If the node pointed at by the second argument has a successor, a pointer to
- that successor is returned. Otherwise, a null pointer is returned.
-
- \example
- The \verb|dict_first| and \verb|dict_next| functions can be used together
- to iterate over all of the elements of the dictionary, as in the following
- idiom:
- \begin{verbatim}
- dict_t *d;
- dnode_t *n;
- /*...*/
- for (n = dict_first(d); n != 0; n = dict_next(d, n)) {
- /* n points to each node in turn */
- }
- \end{verbatim}
-
-\subsubsection{The {\tt dict_prev} function}
-
- \indexfunc{dict_prev}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dict_prev(dict_t *, dnode_t *);\end{verbatim}
-
- \constraints
- The node pointed at by the second argument is an occupant of the dictionary
- pointed at by the first argument.
-
- \description
- If the node pointed at by the second argument has a predecessor, a pointer
- to that predecessor is returned. Otherwise, a null pointer is returned.
-
-\subsubsection{The {\tt dict_count} function}
-
- \indexfunc{dict_count}
-
- \synopsis
- \begin{verbatim}
- dictcount_t dict_count(dict_t *);\end{verbatim}
-
- \description
- The \verb|dict_count| function returns a value which represents the number
- of nodes currently stored in the dictionary pointed at by the argument.
-
-\subsubsection{The {\tt dict_isempty} function}
-
- \indexfunc{dict_isempty}
-
- \synopsis
- \begin{verbatim}
- int dict_isempty(dict_t *);\end{verbatim}
-
- \description
- The \verb|dict_isempty| function returns 1 if the given dictionary is
- empty, otherwise it returns 0.
-
-\subsubsection{The {\tt dict_isfull} function}
-
- \indexfunc{dict_isfull}
-
- \synopsis
- \begin{verbatim}
- int dict_isfull(dict_t *);\end{verbatim}
-
- \description
- The \verb|dict_isfull| function returns 1 if the dictionary is full,
- otherwise it returns 0.
-
- If the argument is an expression with side effects, the behavior is
- undefined.\index{macros!and side effects}
-
-\subsubsection{The {\tt dict_contains} function}
-
- \indexfunc{dict_contains}
-
- \synopsis
- \begin{verbatim}
- int dict_contains(dict_t *, dnode_t *);\end{verbatim}
-
- \description
- The \verb|dict_contains| function searches the given dictionary to
- determine whether the given node is an occupant. If the node is found, 1 is
- returned, otherwise 0 is returned.\footnote{The intent is to support
- verification. The search may be inefficient compared to {\tt
- dict_lookup}.}
-
-\subsubsection{The {\tt dict_allow_dupes} function}
-
- \label{section:dict_allow_dupes}
- \indexfunc{dict_allow_dupes}
-
- \synopsis
- \begin{verbatim}
- void dict_allow_dupes(dict_t *);\end{verbatim}
-
- \constraints
- The dictionary specified by the first argument shall be empty.
-
- \description
- The \verb|dict_allow_dupes| function configures the given dictionary to
- support duplicate keys. This can only be done when the dictionary is empty,
- and the change cannot be reverted.
-
-\subsubsection{The {\tt dnode_is_in_a_dict} function}
-
- \indexfunc{dnode_is_in_a_dict}
-
- \synopsis
- \begin{verbatim}
- int dnode_is_in_a_dict(dnode_t *);\end{verbatim}
-
- \description
- The \verb|dnode_is_in_a_dict| function reports whether the given node
- is currently the occupant of some dictionary. If so, 1 is returned.
- Otherwise 0 is returned.
-
-\subsubsection{The {\tt dnode_create} function}
-
- \indexfunc{dnode_create}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dnode_create(void *);\end{verbatim}
-
- \description
- The \verb|dnode_create| function dynamically allocates a dictionary node,
- stores in it the data value specified in the argument and
- returns a pointer to it. The allocation is performed by a call to the
- standard \verb|malloc| function. If the allocation fails, a null
- pointer is returned.
-
- The node's key pointer remains indeterminate until it is the subject of a
- \verb|dict_insert| operation.
-
-\subsubsection{The {\tt dnode_init} function}
-
- \indexfunc{dnode_init}
-
- \synopsis
- \begin{verbatim}
- dnode_t *dnode_init(dnode_t *, void *);\end{verbatim}
-
- \description
- The \verb|dnode_init| function initializes the contents
- of the specified dictionary node object, assigning it the
- data value specified as the second argument.
- The first argument is a pointer which refers to
- a data object that has a suitable size and alignment
- for the representation of an \verb|dnode_t| type.
- After initialization with \verb|dnode_init|, the object is subsequently
- eligible as an operand to the functions of the dictionary component,
- other than \verb|dnode_getkey|.
-
- The node's key pointer remains indeterminate until it is the subject of a
- \verb|dict_insert| operation.
-
-\subsubsection{The {\tt dnode_destroy} function}
-
- \indexfunc{dnode_destroy}
-
- \synopsis
- \begin{verbatim}
- void dnode_destroy(dnode_t *);\end{verbatim}
-
- \description
- The \verb|dnode_destroy| function destroys a dictionary node that has been
- allocated with \verb|dnode_create|. The value of any pointer
- that referred to the node that was thus freed is indeterminate.
-
- If the node is currently the occupant of a dictionary, the behavior is
- undefined if the hash is subsequently used.
-
-\subsubsection{The {\tt dnode_get} function}
-
- \indexfunc{dnode_get}
-
- \synopsis
- \begin{verbatim}
- void *dnode_get(dnode_t *);\end{verbatim}
-
- \description
- The \verb|dnode_get| function retrieves the \verb|void * | data value
- associated with the given dictionary node.
-
-\subsubsection{The {\tt dnode_getkey} function}
-
- \indexfunc{dnode_getkey}
-
- \synopsis
- \begin{verbatim}
- const void *dnode_getkey(dnode_t *);\end{verbatim}
-
- \description
-
- The \verb|dnode_getkey| function retrieves the \verb|void *| key value
- associated with the given node. A node acquires an associated key
- when it is inserted into a dictionary (see section \ref{section:dict_insert}).
- Invoking \verb|dnode_getkey| on a node that has not been inserted
- into a dictionary results in undefined behavior.
-
-\subsubsection{The {\tt dnode_put} function}
-
- \indexfunc{dnode_put}
-
- \synopsis
- \begin{verbatim}
- void dnode_put(dnode_t *, void *);\end{verbatim}
-
- \description
- The function \verb|dnode_put| replaces the data element
- associated with the dictionary node.
-
-\subsubsection{The {\tt dict_process} function}
-
- \label{section:dict_process}
- \indexfunc{dict_process}
-
- \synopsis
- \begin{verbatim}
- void dict_process(dict_t *, void *, dnode_process_t);\end{verbatim}
-
- \description
- The \verb|dict_process| function iterates over the nodes of a dict,
- and for each node invokes a callback function.\footnote{In most cases,
- it is more convenient and preferable to
- iterate over the dict using explicit calls to {\tt dict_first}
- and {\tt dict_next}.}
- The second argument is a {\it context pointer\/} which can have any value.
- The third argument of
- \verb|dict_process| shall be a pointer to a function which is compatible
- with the specified type. If the dict contains one or more nodes,
- then the function is invoked once for each node, in order from first
- to last. On each invocation, the first argument of the callback is a
- pointer to the dict; the second argument is a pointer to a node, called
- the {\it subject node}; and the third argument repeats the context pointer
- value that was originally passed to \verb|dict_process|.
-
- The callback function may delete the subject node by, for instance, calling
- \verb|dict_delete|. It may insert new nodes into the dictionary;
- however, if such an insertion causes the subject node to acquire
- a new successor, it is implementation-defined whether upon returning
- from the callback function, the traversal shall continue with the
- new successor, or with the original successor.
-
- The callback function, and any function invoked from the callback
- function, shall not destroy the dictionary or make any modifications
- other than the insertion of new nodes, or the deletion of the
- subject node.
-
- The callback function may recursively invoke \verb|dict_process| for the
- same dictionary or for a different dictionary; the callback invocations arising out of
- the nested call inherit all of the restrictions of the outer callback in
- addition to being subject to the usual restrictions.\footnote{This means,
- for instance, that if two callbacks are in progress for different
- subject nodes from the same dictionary, the inner callback may not delete
- its subject node, because it inherits the restriction that the only
- permitted deletion is the outer callback's subject node.}
-
- The callback function may freely operate on a different dictionary,
- subject to any inherited restrictions.
-
-\subsubsection{The {\tt dict_load_begin} function}
-
- \label{section:dict_load_begin}
- \indexfunc{dict_load_begin}
-
- \synopsis
- \begin{verbatim}
- void dict_load_begin(dict_load_t *, dict_t *);\end{verbatim}
-
- \constraints
- The dictionary specified by the second argument is empty.
-
- \description
- The \verb|dict_load_begin| function prepares a context object
- for the task of constructing the contents of a dictionary out of
- a sequence of elements which is already sorted according to the
- sorting function of the dictionary.\footnote{This process is more efficient
- than inserting all of the elements into a dictionary using {\tt dict_insert}.
- In the reference implementation, this process runs in linear time, or $O(n)$
- whereas construction by repeated insertions runs in $O(n\log n)$ time.}
- The actual construction is performed
- by zero or more calls to \verb|dict_load_next| and is finalized by
- \verb|dict_load_end|.
-
- The \verb|dict_load_begin| function is said to bind the dictionary
- and context object together; the only way to unbind the two
- is by calling \verb|dict_load_end| on the context object.
-
- The program shall not manipulate a dictionary that is bound to
- a context object, other than by calling \verb|dict_load_next|.
-
- The program shall not attempt to bind a dictionary to more than one context
- object simultaneously, or a context object to more than one dictionary
- simultaneously.
-
-\subsubsection{The {\tt dict_load_next} function}
-
- \label{section:dict_load_next}
- \indexfunc{dict_load_next}
-
- \synopsis
- \begin{verbatim}
- void dict_load_next(dict_load_t *, dnode_t *, const void *);\end{verbatim}
-
- \constraints
- The node pointed at by the second argument is not an occupant of
- any dictionary. The key specified by the third argument is greater
- than or equal to all keys specified in previous calls to
- \verb|dict_load_next| in the context of the same construction,
- according to the comparison function of the dictionary that is
- being constructed. That is to say, successive calls specify monotonically
- increasing keys.
- The dictionary is not full.
-
- \description
- The \verb|dict_load_next| function continues the construction of a
- dictionary from an ordered list of elements by specifying the next
- node in the sequence, along with its key. After this call, the node
- is considered to be inserted into the dictionary as if by
- \verb|dict_insert|.
-
-\subsubsection{The {\tt dict_load_end} function}
-
- \label{section:dict_load_end}
- \indexfunc{dict_load_end}
-
- \synopsis
- \begin{verbatim}
- void dict_load_end(dict_load_t *);\end{verbatim}
-
- \description
- The \verb|dict_load_end| function finalizes the construction of
- a dictionary from a ordered sequence. It breaks the binding between
- the \verb|dict_load_t| context object and the dictionary.
-
-\subsection{Implementation}
-
-TODO
-
-\section{Exception component}
-\label{section:exception_component}
-\index{Exception}
-
-The Exception component provides distributed error handling in the form of
-exceptions, behind an interface designed to be implementable using only the
-portable features of standard C. The features of this interface are:
-\begin{itemize}
-\item the ability to set up nested try-catch regions which declare specific
-exceptions that they can handle;
-\item grouped exceptions, allowing handlers to catch specific exceptions,
-or any exception within a group;
-\item the ability to designate a function that is called in the event
-that an exception is thrown that has no handler.
-\item a mechanism for releasing resources acquired by code that is terminated
-by an exception;
-\item the ability to pass dynamically allocated data from the throw site to the
-catch site.
-\end{itemize}
-
-An exception is simply a means of returning to a prior place in the program's
-execution. The ANSI C language provides crude, but portable, exception handling
-consisting of the \verb|jmp_buf| type, the \verb|setjmp| macro and the
-\verb|longjmp| function. The Kazlib Exception component can be implemented in
-terms of these primitives. The constraint to implementability in standard C
-leads to a number of concessions:
-\begin{itemize}
-\item A program can leave cleanup regions and try-catch regions by improper
-means, such as using \verb|goto|, \verb|return| or \verb|break|. This is
-difficult to diagnose, and is simply documented as undefined behavior.
-There is no support in the standard language for designating code that is
-executed whenever a statement block terminates by any means.
-\item For the same reason, the exception handling interface described here
-has an explicit mechanism for deallocation of resources associated with
-statement blocks that are terminated by exceptions. This interface is
-not as convenient as language support for automatic cleanup. Correct
-management of temporary dynamic resources using this interface requires
-programmer discipline.
-\item The requirement to be able to use \verb|setjmp| to save a context
-to be later returned to during exception processing brings in restrictions
-related to non-volatile objects. If non-volatile objects are modified
-between the time an exception handling region is initiated and the time
-an exception is caught in the region, these objects have indeterminate
-values.\footnote{This liberty in ANSI C allows compiler
-or library writers to implement {\tt setjmp} as a simple mechanism that
-takes a snapshot of the machine context. Objects that are optimized into
-special storage---such as registers---and whose values change since the
-context saving operation will be clobbered when the context is restored
-by {\tt longjmp}.}
-\end{itemize}
-
-\subsection{Interface}
-
-\subsubsection{The {\tt except.h} header}
-
-Each C or C++ translation unit that is to use the functionality of the Exception
-component shall include the header \verb|except.h|. This header shall
-contain declarations of types and external functions, and definitions of
-macros. The following typedef names shall be
-defined:\index{Exception!typedef names}
-\begin{verbatim}
- except_id_t
- except_t
-\end{verbatim}
-The following external function names shall be declared:
-\index{Exception!function names}\index{functions!defined by Exception}
-\begin{verbatim}
- except_init except_group
- except_deinit except_message
- except_rethrow except_data
- except_throw except_take_data
- except_throwd except_set_allocator
- except_throwf except_alloc
- except_unhandled_catcher except_free
- except_code
-\end{verbatim}
-The following preprocessor symbols shall be
-defined: \index{Exception!macro names}\index{macros!defined by Exception}
-\indexmacro{XCEPT_H}
-\begin{verbatim}
- XCEPT_H except_cleanup_pop
- XCEPT_GROUP_ANY except_checked_cleanup_pop
- XCEPT_CODE_ANY except_try_push
- XCEPT_BAD_ALLOC except_try_pop
- except_cleanup_push
-\end{verbatim}
-Finally, these two enum constants are defined:
-\begin{verbatim}
- except_no_call
- except_call
-\end{verbatim}
-\index{symbols!reserved by Exception}\index{Exception!reserved symbols} Macro
-identifiers which begin with the upper-case prefix \verb|XCEPT|\footnote{The
-prefix {\tt XCEPT} is used rather than {\tt EXCEPT} because ISO 9899 reserves
-preprocessor symbols beginning with {\tt E} followed by a digit or
-capital letter for future extensions to the {\tt <errno.h>} header.}
-are reserved for future extensions to the \verb|except.h|
-header, as are names in the ordinary and tag namespaces which begin with
-\verb|except_|. External names which begin with \verb|except_| are reserved by
-the Kazlib library regardless of what headers are included.
-
-\subsubsection{The {\tt except_id_t} type}
-
-\label{section:except_id_t}
-\indextype{except_id_t}
-\indexmacro{XCEPT_GROUP_ANY}
-\indexmacro{XCEPT_CODE_ANY}
-The type \verb|except_id_t| is an aggregate consisting of two unsigned long
-values which represent an {\it exception group\/} and {\it exception code},
-respectively, in that order.\footnote{Thus, the program may initialize
-an {\tt except_id_t} object using two brace-enclosed initializers which
-specify the group and code.} An exception group is a value which identifies a
-group of related exceptions. An exception code is a value which identifies a
-specific exception uniquely within a group. The codes are assigned by the
-program designer. The Exception component reserves only the group and code
-values of zero, which, when used to specify a catch, match any value.
-
-The preprocessor symbols \verb|XCEPT_GROUP_ANY| and
-\verb|XCEPT_CODE_ANY| each expand to a constant integral expression having the
-value zero. These symbols are intended, in a catch specification, to clearly
-convey that any exception or any group is being caught.
-
-The preprocessor symbol \verb|XCEPT_BAD_ALLOC| expands to an integral constant
-expression having the value 1. This symbol is intended to represent the
-standard exception group for failed memory allocations.
-(See section \ref{section:except_throwf}).
-
-The exception groups from 1 to 15 are reserved for implementation use.
-
-\subsubsection{The {\tt except_t} type}
-
-\indextype{except_t}
-An object of type \verb|except_t| keeps track of all of the information that is
-passed when an exception is thrown, and is known as an {\it exception
-descriptor}. The type is opaque, hence the program shall manipulate this type
-using only the interface functions provided.
-
-\subsubsection{The {\tt except_init} function}
-
- \indexfunc{except_init}
-
- \synopsis
- \begin{verbatim}
- int except_init(void);\end{verbatim}
-
- \description
- The \verb|except_init| function allocates resources needed by the
- Exception component. Before using any of the other exception interface
- functions or macros, the program shall perform at least one successful call
- to \verb|except_init|.
-
- If the initialization succeeds, \verb|except_init| returns 1. Otherwise
- it returns 0.
-
- The \verb|except_init| function may be called more than once. After a
- successful call, every subsequent call shall be successful up to an
- implementation-defined maximum number of repetitions, which shall be at least
- as large as the \verb|INT_MAX| from \verb|limits.h|. \footnote{
- The intent is to support, but not enforce, a style of global initialization
- whereby each module which requires the use of another module calls its
- initialization function from its own initialization function. Only the
- first such call performs the initialization of the module; subsequent calls
- merely increment a counter. During deinitialization, the counter is
- decremented and cleanup takes place when the counter reaches zero.}
-
-\subsubsection{The {\tt except_deinit} function}
-
- \indexfunc{except_deinit}
-
- \synopsis
- \begin{verbatim}
- void except_deinit(void);\end{verbatim}
-
- \description
- The \verb|except_deinit| function releases the resources
- that were allocated by \verb|except_init|.
-
- For the resource deallocation to actually take place, the
- \verb|except_deinit| must be called as many times as the
- number of times \verb|except_init| was successfully called.
-
- If \verb|except_deinit| is called more times than \verb|except_init| is
- successfully called, the behavior is undefined.
-
-\subsubsection{The {\tt except_rethrow} function}
-
- \indexfunc{except_rethrow}
-
- \synopsis
- \begin{verbatim}
- void except_rethrow(except_t *);\end{verbatim}
-
- \description
- The rethrow function is used to rethrow a caught exception. The argument
- shall not be null. An exception shall not be rethrown from outside of the
- {\it try-catch region\/} in which it was caught. An exception shall not be
- rethrown from a try-catch region other than the one in which it was caught.
- It shall not be rethrown from a try-catch or cleanup region enclosed within
- the one in which it was caught.
-
- When an exception is rethrown, the search for a handler does not begin with
- the region in which the exception was caught. Instead, this region is
- terminated, and the search continues with the enclosing one, if one
- exists.
-
-\subsubsection{The {\tt except_throw} function}
-
- \indexfunc{except_throw}
-
- \synopsis
- \begin{verbatim}
- void except_throw(long, long, const char *);\end{verbatim}
-
- \constraints
- The first two arguments specify the exception group and code,
- respectively. Neither of these arguments shall be zero.
-
- \description
- The \verb|except_throw| function causes an exception to be thrown.
-
- If the throw takes place in a try-catch region where an exception
- was just caught, this original exception is considered handled. In
- this case, the new exception is still eligible for handling by the
- same try-catch region.
-
- The third argument points to the first character of a string
- which becomes the {\it exception message}. Because the throwing of
- the exception may cause the current statement block to terminate,
- this string data shall be non-local. It may be a string literal, since the
- implementation shall not modify the message, or it may be an ordinary
- object of static duration. If it is dynamic data, it becomes the handler's
- responsibility to extract the message from the caught exception and
- free the data.\footnote{The programmer should consider using
- {\tt except_throwd} to pass arbitrary dynamic data from the throw
- site to the try-catch region.}
-
- The \verb|except_throw| function does not return. The implementation
- searches for a suitable try-catch region starting with the one
- initiated by the most recent \verb|except_try_push|. If there
- is no enclosing region, the search fails. Otherwise if a match is found,
- execution continues at the start of the target try-catch region, appearing
- to be a second return from \verb|except_try_push| distinguished by a non-null
- value of the \verb|except_t *| object.
-
- If no match is found during exception processing, the exception is
- handled internally by the implementation. The implementation then
- calls the currently registered function for catching unhandled
- exceptions (see section \ref{section:except_unhandled_catcher}).
-
- The default catcher for unhandled exceptions shall terminate the program
- with a diagnostic which identifies the code, group and exception message.
-
- During the search for an exception handler, cleanup handlers may be
- encountered. They are removed from the inside out and called with
- their registered arguments. This process is called {\it unwinding}.
- \index{unwinding}
-
-\subsubsection{The {\tt except_throwd} function}
-
- \indexfunc{except_throwd}
-
- \synopsis
- \begin{verbatim}
- void except_throwd(long, long, const char *, void *);\end{verbatim}
-
- \constraints
- The first two arguments specify the exception group and code,
- respectively. Neither of these arguments shall be zero.
-
- \description
- The \verb|except_throwd| function is the same as \verb|except_throw| in
- every respect except that it has an additional \verb|void *| parameter. A
- null argument may be used for this parameter, or it may be any valid
- pointer value.
-
- When the exception is handled, and the handler does not remove this pointer
- using \verb|except_take_data| then the implementation shall automatically
- invoke the function \verb|except_free| on this pointer.
-
-\subsubsection{The {\tt except_throwf} function}
-
- \indexfunc{except_throwf}
- \label{section:except_throwf}
-
- \synopsis
- \begin{verbatim}
- void except_throwf(long, long, const char *, ...);\end{verbatim}
-
- \constraints
- The first two arguments specify the exception group and code,
- respectively. Neither of these arguments shall be zero.
-
- \description
-
- This function is almost exactly the same as \verb|except_throw|
- except that the exception message is not directly specified.
- Instead, the \verb|char *| argument specifies a format string which may be
- followed by trailing arguments. The format string and trailing arguments
- are interpreted as the format string and arguments of the standard C
- function \verb|printf| and are subject to the same requirements.
-
- The format string is interpreted, and the results of formatting are placed into
- buffer provided by the implementation. The implementation shall provide
- space for at least 1024 bytes of storage for the result of the formatting,
- including the null terminator byte. If the formatting requires more space
- than the implementation provides, the behavior is undefined.
-
- The results of the formatted print shall become the exception message
- of the thrown exception.
-
- If the implementation is unable to allocate resources for the formatted
- message, it shall throw a code 1 exception having an unspecified code in
- group \verb|XCEPT_BAD_ALLOC| with an implementation-defined message.
- (See section \ref{section:except_id_t}).
-
-\subsubsection{The {\tt except_unhandled_catcher} function}
-
- \label{section:except_unhandled_catcher}
- \indexfunc{except_unhandled_catcher}
-
- \synopsis
- \begin{verbatim}
- void (*except_unhandled_catcher(void (*)(except_t *)))
- (except_t *);\end{verbatim}
-
- \description
- The \verb|except_unhandled_catcher| function installs a new
- function for catching unhandled exceptions. The argument is a
- pointer to a catching function that returns nothing, and accepts a pointer
- of type \verb|except_t *|. A pointer to the previously installed
- catching function is returned. If the program did not previously
- install a catching function, then a pointer to the default catching
- function is returned. The program may retain this pointer and
- use it to reinstall the default function.
-
- A function for catching unhandled exceptions should not return. If it
- returns, the implementation shall terminate the program with a diagnostic.
-
-\subsubsection{The {\tt except_code} function}
-
- \indexfunc{except_code}
-
- \synopsis
- \begin{verbatim}
- unsigned long except_code(except_t *);\end{verbatim}
-
- \description
- The \verb|except_code| is an accessor function which returns the
- exception code of the given exception descriptor.
-
-\subsubsection{The {\tt except_group} function}
-
- \indexfunc{except_group}
-
- \synopsis
- \begin{verbatim}
- unsigned long except_group(except_t *);\end{verbatim}
-
- \description
- The \verb|except_group| is an accessor function which returns the
- exception group of the given exception descriptor.
-
-\subsubsection{The {\tt except_message} function}
-
- \indexfunc{except_message}
-
- \synopsis
- \begin{verbatim}
- const char *except_message(except_t *);\end{verbatim}
-
- \description
- The \verb|except_group| is an accessor function which returns
- a pointer to the string of text that was specified when the
- exception was thrown (the exception message).
-
-\subsubsection{The {\tt except_data} function}
-
- \indexfunc{except_data}
-
- \synopsis
- \begin{verbatim}
- void *except_data(except_t *);\end{verbatim}
-
- \description
- The \verb|except_group| returns the data pointer that
- was specified in the \verb|except_throwd| call.
- If the exception was not thrown by \verb|except_throwd|
- the return value is unspecified.
-
-
-\subsubsection{The {\tt except_take_data} function}
-
- \indexfunc{except_take_data}
-
- \synopsis
- \begin{verbatim}
- void *except_take_data(except_t *);\end{verbatim}
-
- \description
- The \verb|except_take_data| returns the data pointer that
- was specified in the \verb|except_throwd| call, and
- updates the exception descriptor so that the pointer is
- set to null.
-
- If the exception was not thrown by \verb|except_throwd|
- the result is unspecified.
-
-\subsubsection{The {\tt except_cleanup_push} macro}
-
- \indexmacro{except_cleanup_push}
-
- \synopsis
- \begin{verbatim}
- void except_cleanup_push(void (*)(void *), void *);\end{verbatim}
-
- \description
- The call to \verb|except_cleanup_push| shall be matched with a call to
- \verb|except_cleanup_pop| which must occur in the same statement block at
- the same level of nesting.\footnote{This requirement allows an implementation
- to provide an {\tt except_cleanup_push} macro which opens up a statement
- block and a {\tt except_cleanup_pop} which closes the statement block.
- The space for the registered pointers can then be efficiently allocated
- from automatic storage.}
-
- The \verb|except_cleanup_push| macro registers a cleanup handler that will
- be called if an exception subsequently occurs before the matching
- \verb|except_cleanup_pop| is executed, and is not intercepted and handled by
- a try-catch region that is nested between the two.
-
- The first argument to \verb|except_cleanup_push| is a pointer
- to the cleanup handler, a function that returns nothing and takes
- a single argument of type \verb|void *|. The second argument
- is a \verb|void *| value that is registered along with the handler.
- This value is what is passed to the registered handler, should it
- be called.
-
- Cleanup handlers are called in the reverse order of their nesting: inner
- handlers are called before outer handlers.
-
- The program shall not leave the cleanup region between the call to the macro
- \verb|except_cleanup_push| and the matching call to
- \verb|except_cleanup_pop| by means other than throwing an exception, or
- calling \verb|except_cleanup_pop|.
-
- Within the call to the cleanup handler, it is possible that new exceptions
- may happen. Such exceptions must be handled before the cleanup handler
- terminates. If the call to the cleanup handler is terminated by an
- exception, the behavior is undefined.\footnote{The exception which triggered
- the cleanup is not yet caught; thus the program would be effectively trying
- to replace an exception with one that isn't in a well-defined state.}
-
-\subsubsection{The {\tt except_cleanup_pop} macro}
-
- \indexmacro{except_cleanup_pop}
- \label{section:except_cleanup_pop}
-
- \synopsis
- \begin{verbatim}
- void except_cleanup_pop(int);\end{verbatim}
-
- \description
- A call to the \verb|except_cleanup_pop| macro shall match each
- call to \verb|except_cleanup_push| which shall be in the
- same statement block at the same nesting level. It shall
- match the most recent such a call that is not matched
- by a previous \verb|except_cleanup_pop| at the same level.
-
- This macro causes the registered cleanup handler to be removed. If, and
- only if the argument is other than zero, the cleanup handler is called.
- In that case, the registered context pointer is passed to the cleanup
- handler.
-
- \indexenum{except_no_call}
- \indexenum{except_call}
- The enumeration constants \verb|except_no_call| and \verb|except_call|
- may be used as arguments to this function instead of
- the equivalent constants \verb|0| and \verb|1|.
-
- The program shall not leave the region between the call to the macro
- \verb|except_cleanup_push| and the matching call to
- \verb|except_cleanup_pop| other than by throwing an exception, or
- by executing the \verb|except_cleanup_pop|.
-
-\subsubsection{The {\tt except_checked_cleanup_pop} macro}
-
- \indexmacro{except_checked_cleanup_pop}
-
- \synopsis
- \begin{verbatim}
- void except_checked_cleanup_pop(void (*)(void *), int);\end{verbatim}
-
- \constraints
- The first pointer-to-function argument shall match the pointer value that
- was registered by the matching \verb|except_cleanup_push| macro.
-
- \description
- The \verb|except_checked_cleanup_pop| macro may be used as an alternative to
- \verb|except_cleanup_pop|. In verification mode, the constraint serves to
- provide additional safety by making an explicit declaration regarding which
- handler is being called (or ignored, as the case may be).
-
- The program shall not leave the region between the call to the macro
- \verb|except_cleanup_push| and the call to
- \verb|except_checked_cleanup_pop| by means other than throwing an
- exception, or executing the latter macro.
-
-\subsubsection{The {\tt except_try_push} macro}
-
- \indexmacro{except_try_push}
- \label{section:except_try_push}
-
- \synopsis
- \begin{verbatim}
- void except_try_push(const except_id_t [],
- size_t, except_t **);\end{verbatim}
-
- \description
- The \verb|except_try_push| marks the beginning of a try-catch region
- of the program. It must be matched by a \verb|except_try_pop| written in
- the same statement block at the same level of nesting, which
- terminates the try-catch region. Regions may be nested.
-
- The program shall not leave a try-catch region other than by throwing
- an exception or by executing the \verb|except_try_pop|.\footnote{Thus,
- leaving the try-catch region using {\tt goto}, {\tt return},
- {\tt break} or {\tt continue} leads to undefined behavior.}
-
- The first argument is a pointer to the first element of an array of
- \verb|except_id_t| objects, the number of elements of which is specified by
- the second argument. The array specifies which exceptions are caught.
- The implementation shall treat this array as read-only.\footnote{Thus,
- the program may allocate the array in static storage.}
-
- The third argument of \verb|except_try_push| shall point to an object
- of type \verb|except_t *|. After the call to \verb|except_try_push|,
- the program shall inspect the value of this object. A null value indicates
- that no exception has been thrown. A non-null value indicates that an
- exception was thrown, and is now caught. In other words, when an exception
- is caught by a try-catch region, then control passes from the throw site
- back to the first statement after the \verb|except_try_push| statement of
- the try-catch region. This case is distinguished from an ordinary return by
- the non-null value of the pointer object that was specified by the third
- argument of the earlier call to \verb|except_try_push|.
-
- An exception is considered handled if it is caught in a try-catch region
- which subsequently terminates by executing its \verb|except_try_pop| or by
- throwing another exception. When an exception is considered handled, any
- dynamic data that was associated with that exception is
- freed.\footnote{Dynamic data may be explicitly associated with an exception
- using {\tt except_throwd}. Other types of throw may associate unspecified
- dynamic data.} It's possible for more than one exception to be active
- at once. During the processing of one exception, a try-catch region
- which catches the exception may execute a nested try-catch region
- in which independent exception processing takes place. Provided that
- no exception escapes from the inner try-catch region, the original
- exception remains pending. But if an exception escapes from the inner
- region, it causes the original exception to be handled.\footnote{Thus, a
- given try-catch region cannot catch multiple exceptions concurrently.}
-
- The caught exception may be rethrown by calling \verb|except_rethrow|,
- specifying the the value of the caught exception descriptor as the
- argument. Rethrowing a caught exception causes the innermost try-catch
- region to terminate, but the exception is not considered handled. The
- search for a handler continues with the second most enclosing region.
-
- Throwing a new exception during the handling of a caught exception may
- cause the {\it same\/} try-catch region to catch that exception; the
- try-catch region is not terminated until it is determined that it doesn't
- catch the new exception.
-
- Each entry in the array of \verb|except_id_t| objects specifies what
- exceptions are caught by the try-catch region. When an exception is
- thrown, the implementation searches for the inner-most try-catch region
- which has at least one match for the thrown exception in its catch
- specification array.
-
- A match occurs when a specification exactly matches the group and code of
- the thrown exception. If a catch specification is for group 0, then it
- matches any group. If a catch specification is for code 0, then it matches
- any exception code. A catch specification of group 0 and code 0 catches all
- exceptions.
-
- Non-volatile automatic variables that are local to the function containing
- the try-catch region, and that are modified after \verb|except_try_push|
- begins the try-catch region have indeterminate values when an exception is
- caught.
-
- Once a caught exception is handled or re-thrown, the value of the
- \verb|except_t *| pointer which referenced it becomes indeterminate.
- If a re-thrown exception is caught again, the implementation shall
- produce a valid \verb|except_t *| pointer.
-
- \example
- The following example illustrates the use of \verb|except_try_push| and
- related macros and functions.
- \begin{verbatim}
- #include <stdlib.h>
- #include <assert.h>
- #include "except.h"
-
- #define MY_GROUP 42
- #define MY_CODE 1
-
- static void func_that_throws(void)
- {
- except_throw(MY_GROUP, MY_CODE, "this is an exception");
- }
-
- static void func_that_cleans_up(void)
- {
- void *local_data = malloc(10);
-
- except_cleanup_push(free, local_data);
- func_that_throws();
- except_checked_cleanup_pop(free, except_call);
- }
-
- void func_that_catches(void)
- {
- /* catch specification */
- static const except_id_t catch_spec[] = {
- { MY_GROUP, XCEPT_CODE_ANY }
- };
- /* exception handle */
- except_t *exc;
-
- except_try_push(catch_spec, 1, &exc);
-
- /*
- * Start of try-catch region: when exception is
- * thrown, control returns here.
- */
-
- if (exc == 0) {
- /* try code that may throw an exception */
-
- func_that_cleans_up();
- } else {
- /* handle exception that was thrown */
-
- assert (except_group(exc) == MY_GROUP);
- printf("exception caught: %s %ld %ld\n",
- except_message(exc),
- except_group(exc), except_code(exc));
-
- goto terminate; /* ERROR! jumping out of try-catch */
- }
-
- /* end of try-catch region */
-
- except_try_pop();
- terminate:
- ;
- }
- \end{verbatim}
- In this example, the function \verb|func_that_catches| is intended to be
- called first. It sets up a try-catch region which traps exceptions having
- the group identification \verb|MY_GROUP| (or 42). Any code within that
- group is caught because the code catch was specified as
- \verb|XCEPT_CODE_ANY|. When the \verb|except_try_push| macro is executed,
- it sets the value of \verb|exc| to null. Then \verb|func_that_cleans_up| is
- called, which throws an exception in the \verb|MY_GROUP| group. This
- exception is caught, so control resumes at the top of the try-catch region,
- with \verb|exc| set to a non-null value. Thus the else clause of the if
- statement is now executed. The handling code simply prints the exception
- message on standard output, as well as the numeric group and code. The
- subsequent goto statement demonstrates a serious programming error.
-
- The \verb|func_that_cleans_up| function illustrates the use of cleanup
- regions. Dynamic memory is allocated which must not be allowed to leak
- when an exception is thrown, so a cleanup handler is set up to free the
- memory in that event. The standard C function \verb|free| happens to have,
- the right type signature and semantics that it can be used directly as a
- cleanup handler. Should no exception be thrown, the cleanup pop macro
- will perform the call to the cleanup handler, because it is invoked with
- argument \verb|except_call|.
-
-\subsubsection{The {\tt except_try_pop} macro}
-
- \indexmacro{except_try_pop}
-
- \synopsis
- \begin{verbatim}
- void except_try_pop(void);\end{verbatim}
-
- \description
-
- The \verb|except_try_pop| macro terminates a try-catch region. It must
- match a previous \verb|except_try_push| macro in the same statement
- block at the same level of nesting which is not already matched by an
- earlier \verb|except_try_pop|.
-
-\subsubsection{The {\tt except_set_allocator} function}
-
- \indexfunc{except_set_allocator}
- \label{section:except_set_allocator}
-
- \synopsis
- \begin{verbatim}
- void except_set_allocator(void *(*)(size_t), void (*)(void *));\end{verbatim}
-
- \description
- The \verb|except_set_allocator| function installs a pair of allocator
- routines that will be used by the Exception component for future allocation
- and deallocation requests.
-
- The first argument points to a function that resembles the standard C
- \verb|malloc| in type and semantics. The second argument points to a
- function that similarly resembles the standard C function \verb|free|.
-
- The default allocators are \verb|malloc| and \verb|free|.
- The call
- \begin{verbatim}
- except_set_allocator(malloc, free);
- \end{verbatim}
- may be used to restore these default allocator functions.
-
- The program shall not call \verb|except_set_allocator| if an exception
- was thrown and has not yet been handled.\footnote{Doing so could, for example,
- create a mismatch whereby a pointer to data allocated with the previously installed
- allocator function would be passed to the new deallocator function.}
-
- The allocator function shall create a unique object consisting of at least
- as many bytes of storage as indicated by the value of the argument.
- The pointer returned shall be suitably aligned to represent an object
- of any type. If insufficient resources exist, the pointer returned shall be
- null. Requesting an object of zero size may produce a unique pointer
- that shall be acceptable to the deallocator function, or a null pointer.
-
- The deallocator function shall be capable of destroying objects created
- by the corresponding allocator function. Passing a null pointer to the
- deallocator shall have no effect.
-
-\subsubsection{The {\tt except_alloc} function}
-
- \indexfunc{except_alloc}
-
- \synopsis
- \begin{verbatim}
- void *except_alloc(size_t);\end{verbatim}
-
- \description
- The \verb|except_alloc| function allocates memory using the default
- memory allocator or one installed by the program.
- (See section \ref{section:except_set_allocator}).
-
- If the allocation succeeds, a non-null pointer to the allocated object is
- returned.
-
- If the allocator indicates failure by returning a null pointer,
- then instead of returning, \verb|except_alloc| throws exception code 1
- in the group \verb|XCEPT_BAD_ALLOC| (See section \ref{section:except_id_t}).
-
- If a zero size request is specified, then an exception is thrown or
- a non-null pointer is returned, depending on the treatment of such
- requests by the underlying allocator.
-
-\subsubsection{The {\tt except_free} function}
-
- \indexfunc{except_free}
-
- \synopsis
- \begin{verbatim}
- void *except_free(void *);\end{verbatim}
-
- \description
-
- The \verb|except_free| function releases memory that was allocated
- using \verb|except_alloc|. The deallocation is performed using the
- default allocator or one installed by the program.
-
- If an object is allocated by \verb|except_alloc|, then a
- different allocator is installed, and the object is freed using
- \verb|except_free|, the behavior is undefined.
-
-\subsection{Implementation}
-\index{Exception component!reference implementation}
-
-Described here is a reference implementation of the exception handling
-interface that is covered in section \ref{section:exception_component}
-The reference implementation requires only a conforming ANSI C implementation.
-In particular, the actual mechanism for passing control from an exception throw
-to a catch handler is based on the standard C \verb|setjmp| macro and
-\verb|longjmp| function.
-
-\subsubsection{Overview}
-
-The core structure in the exception handling implementation is a stack that is
-composed of a mixture of two types of nodes: cleanup nodes and catch nodes.
-When an exception is thrown, the stack nodes are popped and processed starting
-with the topmost one.
-
-The nodes are efficiently allocated in automatic storage by the macros
-\verb|except_cleanup_push| and \verb|except_try_push|. These macros
-open up a new statement block and declare the node information in automatic
-storage. These objects are then pushed onto the stack. The corresponding macros
-\verb|except_cleanup_pop| and \verb|except_try_pop| pop the node off the stack
-and close the statement block.
-
-An static variable keeps track of the stack top. In the multi-threaded variant
-of the code which is based on the POSIX threading interface, there is a
-thread-specific stack top created using the thread-specific function
-pthread_key_create. Using global variables is a compromise that simplifies the
-interface; the throw functions simply ``know'' where the thread's exception
-stack is, so the context information doesn't have to be passed around.
-
-\subsubsection{Stack nodes}
-
-A node in the exception handling stack contains a pointer to the next
-node below, followed by a type field and a union which together keep
-track of the appropriate type-specific data:
-\begin{verbatim}
- enum except_stacktype {
- XCEPT_CLEANUP, XCEPT_CATCHER
- };
-
- struct except_stacknode {
- struct except_stacknode *except_down;
- enum except_stacktype except_type;
- union {
- struct except_catch *except_catcher;
- struct except_cleanup *except_cleanup;
- } except_info;
- };
-\end{verbatim}
-The union overlaps pointers to structures instead of structures in order to
-save space: there is a disparity in size between a cleanup node and a catch
-node, so making them both use the same amount of space would be wasteful.
-The space saving comes at a price, because the pointers themselves take up
-extra space and time is spent initializing them. Some casting trickery
-could be used to create a stack having two different kinds of structures
-without the use of unions.
-
-\paragraph{Cleanup nodes}
-
-Cleanup nodes act as placeholders for a pointer to a cleanup handler function
-and a context pointer to be passed to that function. The type-dependent
-component of the cleanup node is declared like this:
-\begin{verbatim}
- struct except_cleanup {
- void (*except_func)(void *);
- void *except_context;
- };
-\end{verbatim}
-The cleanup handler is invoked when the node is popped during exception
-processing. A cleanup handler may also be invoked when the cleanup node is
-removed by executing \verb|except_cleanup_pop| or
-\verb|except_checked_cleanup_pop|. Whether or not this happens depends on the
-integer parameter that is documented in section
-\ref{section:except_cleanup_pop}.
-
-\paragraph{Catch nodes}
-
-The catch node structure is more complicated than the cleanup node.
-Its definition depends on two additional types, \verb|except_id_t|
-and \verb|except_t|, both of which also make play a role in the exception
-component's interface.
-\begin{verbatim}
- typedef struct {
- unsigned long except_group;
- unsigned long except_code;
- } except_id_t;
-
- typedef struct {
- except_id_t except_id;
- const char *except_message;
- void *except_dyndata;
- } except_t;
-
- struct except_catch {
- const except_id_t *except_id;
- size_t except_size;
- except_t except_obj;
- jmp_buf except_jmp;
- };
-\end{verbatim}
-The \verb|except_id| member of the \verb|except_catch| structure is a pointer to the
-array of \verb|except_id_t| objects which specify what exceptions the node
-catches. The \verb|except_size| member specifies the number of elements in the array.
-Both of these values are derived directly from the arguments of the
-\verb|except_try_push| macro (see section \ref{section:except_try_push}). The
-\verb|except_obj| member provides storage for the caught exception. This member is
-the means by which the thrown exception is communicated to the try-catch region
-where it is caught. It contains the group and code identifiers, the exception
-message and, optionally, the pointer to arbitrary exception data. The
-\verb|except_jmp| member is the standard C \verb|jmp_buf|---a place for saving the
-execution context so that it's possible to pass control, via \verb|longjmp|
-from the place where an exception is thrown to the place where it is caught.
-
-If, during the search for an exception handler, a catch node is encountered
-which matches the thrown exception, the node remains the stack. The exception
-information is stored into into the node's \verb|except_obj| member and a
-\verb|longjmp| is executed to return to the try-catch region in which the node
-was allocated and pushed. Because the node is still on the stack, it's possible
-to throw another exception which is caught again by the same node. When an
-exception is thus caught, control resumes just after the \verb|except_throw|
-which placed the node onto the stack. The pointer passed into \verb|except_throw|
-is updated to point to the \verb|except_obj| member of the catch structure.
-The program can then use the portable accessor functions such as
-\verb|except_code| to gain information about the caught exception and handle it
-accordingly.
-
-\index{external names|see {functions}}
-\index{reference implementation|see {implementation}}
-\index{names|see {symbols}}
-\index{identifiers|see {symbols}}
-\index{structure names|see{tags}}
-\index{preprocessor symbols|see{macros}}
-\index{defines|see{macros}}
-\index{reserved symbols|see{symbols}}
-\index{symbols!preprocessor|see{macros}}
-\index{symbols!type names|see{typedefs}}
-\index{symbols!function names|see{functions}}
-\printindex
-
-\end{document}
diff --git a/kmer/libutil/kazlib/drivers/dict-main.c b/kmer/libutil/kazlib/drivers/dict-main.c
deleted file mode 100644
index 08f2e7a..0000000
--- a/kmer/libutil/kazlib/drivers/dict-main.c
+++ /dev/null
@@ -1,300 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <stdarg.h>
-
-typedef char input_t[256];
-
-static int tokenize(char *string, ...)
-{
- char **tokptr;
- va_list arglist;
- int tokcount = 0;
-
- va_start(arglist, string);
- tokptr = va_arg(arglist, char **);
- while (tokptr) {
- while (*string && isspace((unsigned char) *string))
- string++;
- if (!*string)
- break;
- *tokptr = string;
- while (*string && !isspace((unsigned char) *string))
- string++;
- tokptr = va_arg(arglist, char **);
- tokcount++;
- if (!*string)
- break;
- *string++ = 0;
- }
- va_end(arglist);
-
- return tokcount;
-}
-
-static int comparef(const void *key1, const void *key2)
-{
- return strcmp(key1, key2);
-}
-
-static char *dupstring(char *str)
-{
- int sz = strlen(str) + 1;
- char *new = malloc(sz);
- if (new)
- memcpy(new, str, sz);
- return new;
-}
-
-static dnode_t *new_node(void *c)
-{
- static dnode_t few[5];
- static int count;
-
- if (count < 5)
- return few + count++;
-
- return NULL;
-}
-
-static void del_node(dnode_t *n, void *c)
-{
-}
-
-static int prompt = 0;
-
-static void construct(dict_t *d)
-{
- input_t in;
- int done = 0;
- dict_load_t dl;
- dnode_t *dn;
- char *tok1, *tok2, *val;
- const char *key;
- char *help =
- "p turn prompt on\n"
- "q finish construction\n"
- "a <key> <val> add new entry\n";
-
- if (!dict_isempty(d))
- puts("warning: dictionary not empty!");
-
- dict_load_begin(&dl, d);
-
- while (!done) {
- if (prompt)
- putchar('>');
- fflush(stdout);
-
- if (!fgets(in, sizeof(input_t), stdin))
- break;
-
- switch (in[0]) {
- case '?':
- puts(help);
- break;
- case 'p':
- prompt = 1;
- break;
- case 'q':
- done = 1;
- break;
- case 'a':
- if (tokenize(in+1, &tok1, &tok2, (char **) 0) != 2) {
- puts("what?");
- break;
- }
- key = dupstring(tok1);
- val = dupstring(tok2);
- dn = dnode_create(val);
-
- if (!key || !val || !dn) {
- puts("out of memory");
- free((void *) key);
- free(val);
- if (dn)
- dnode_destroy(dn);
- }
-
- dict_load_next(&dl, dn, key);
- break;
- default:
- putchar('?');
- putchar('\n');
- break;
- }
- }
-
- dict_load_end(&dl);
-}
-
-int main(void)
-{
- input_t in;
- dict_t darray[10];
- dict_t *d = &darray[0];
- dnode_t *dn;
- int i;
- char *tok1, *tok2, *val;
- const char *key;
-
- char *help =
- "a <key> <val> add value to dictionary\n"
- "d <key> delete value from dictionary\n"
- "l <key> lookup value in dictionary\n"
- "( <key> lookup lower bound\n"
- ") <key> lookup upper bound\n"
- "# <num> switch to alternate dictionary (0-9)\n"
- "j <num> <num> merge two dictionaries\n"
- "f free the whole dictionary\n"
- "k allow duplicate keys\n"
- "c show number of entries\n"
- "t dump whole dictionary in sort order\n"
- "m make dictionary out of sorted items\n"
- "p turn prompt on\n"
- "s switch to non-functioning allocator\n"
- "q quit";
-
- for (i = 0; i < sizeof darray / sizeof *darray; i++)
- dict_init(&darray[i], DICTCOUNT_T_MAX, comparef);
-
- for (;;) {
- if (prompt)
- putchar('>');
- fflush(stdout);
-
- if (!fgets(in, sizeof(input_t), stdin))
- break;
-
- switch(in[0]) {
- case '?':
- puts(help);
- break;
- case 'a':
- if (tokenize(in+1, &tok1, &tok2, (char **) 0) != 2) {
- puts("what?");
- break;
- }
- key = dupstring(tok1);
- val = dupstring(tok2);
-
- if (!key || !val) {
- puts("out of memory");
- free((void *) key);
- free(val);
- }
-
- if (!dict_alloc_insert(d, key, val)) {
- puts("dict_alloc_insert failed");
- free((void *) key);
- free(val);
- break;
- }
- break;
- case 'd':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- dn = dict_lookup(d, tok1);
- if (!dn) {
- puts("dict_lookup failed");
- break;
- }
- val = dnode_get(dn);
- key = dnode_getkey(dn);
- dict_delete_free(d, dn);
-
- free(val);
- free((void *) key);
- break;
- case 'f':
- dict_free(d);
- break;
- case 'l':
- case '(':
- case ')':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- dn = 0;
- switch (in[0]) {
- case 'l':
- dn = dict_lookup(d, tok1);
- break;
- case '(':
- dn = dict_lower_bound(d, tok1);
- break;
- case ')':
- dn = dict_upper_bound(d, tok1);
- break;
- }
- if (!dn) {
- puts("lookup failed");
- break;
- }
- val = dnode_get(dn);
- puts(val);
- break;
- case 'm':
- construct(d);
- break;
- case 'k':
- dict_allow_dupes(d);
- break;
- case 'c':
- printf("%lu\n", (unsigned long) dict_count(d));
- break;
- case 't':
- for (dn = dict_first(d); dn; dn = dict_next(d, dn)) {
- printf("%s\t%s\n", (char *) dnode_getkey(dn),
- (char *) dnode_get(dn));
- }
- break;
- case 'q':
- exit(0);
- break;
- case '\0':
- break;
- case 'p':
- prompt = 1;
- break;
- case 's':
- dict_set_allocator(d, new_node, del_node, NULL);
- break;
- case '#':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- } else {
- int dictnum = atoi(tok1);
- if (dictnum < 0 || dictnum > 9) {
- puts("invalid number");
- break;
- }
- d = &darray[dictnum];
- }
- break;
- case 'j':
- if (tokenize(in+1, &tok1, &tok2, (char **) 0) != 2) {
- puts("what?");
- break;
- } else {
- int dict1 = atoi(tok1), dict2 = atoi(tok2);
- if (dict1 < 0 || dict1 > 9 || dict2 < 0 || dict2 > 9) {
- puts("invalid number");
- break;
- }
- dict_merge(&darray[dict1], &darray[dict2]);
- }
- break;
- default:
- putchar('?');
- putchar('\n');
- break;
- }
- }
-
- return 0;
-}
diff --git a/kmer/libutil/kazlib/drivers/except-main.c b/kmer/libutil/kazlib/drivers/except-main.c
deleted file mode 100644
index fdb64db..0000000
--- a/kmer/libutil/kazlib/drivers/except-main.c
+++ /dev/null
@@ -1,57 +0,0 @@
-#include <stdio.h>
-#include <ctype.h>
-
-static void cleanup(void *arg)
-{
- printf("cleanup(\"%s\") called\n", (char *) arg);
-}
-
-static void bottom_level(void)
-{
- char buf[256];
- printf("throw exception? "); fflush(stdout);
- fgets(buf, sizeof buf, stdin);
-
- if (buf[0] >= 0 && toupper(buf[0]) == 'Y')
- except_throw(1, 1, "nasty exception");
-}
-
-static void top_level(void)
-{
- except_cleanup_push(cleanup, "argument");
- bottom_level();
- except_cleanup_pop(0);
-}
-
-int main(int argc, char **argv)
-{
- static const except_id_t catch[] = { { 1, 1 }, { 1, 2 } };
- except_t *ex;
-
- /*
- * Nested exception ``try blocks''
- */
-
- /* outer */
- except_try_push(catch, 2, &ex);
- if (!ex) {
- /* inner */
- except_try_push(catch, 2, &ex);
- if (!ex) {
- top_level();
- } else {
- /* inner catch */
- printf("caught exception (inner): \"%s\", s=%ld, c=%ld\n",
- except_message(ex), except_group(ex), except_code(ex));
- except_rethrow(ex);
- }
- except_try_pop();
- } else {
- /* outer catch */
- printf("caught exception (outer): \"%s\", s=%ld, c=%ld\n",
- except_message(ex), except_group(ex), except_code(ex));
- }
- except_try_pop();
- except_throw(99, 99, "exception in main");
- return 0;
-}
diff --git a/kmer/libutil/kazlib/drivers/hash-main.c b/kmer/libutil/kazlib/drivers/hash-main.c
deleted file mode 100644
index 0a08542..0000000
--- a/kmer/libutil/kazlib/drivers/hash-main.c
+++ /dev/null
@@ -1,187 +0,0 @@
-#include <stdio.h>
-#include <ctype.h>
-#include <stdarg.h>
-
-typedef char input_t[256];
-
-static int tokenize(char *string, ...)
-{
- char **tokptr;
- va_list arglist;
- int tokcount = 0;
-
- va_start(arglist, string);
- tokptr = va_arg(arglist, char **);
- while (tokptr) {
- while (*string && isspace((unsigned char) *string))
- string++;
- if (!*string)
- break;
- *tokptr = string;
- while (*string && !isspace((unsigned char) *string))
- string++;
- tokptr = va_arg(arglist, char **);
- tokcount++;
- if (!*string)
- break;
- *string++ = 0;
- }
- va_end(arglist);
-
- return tokcount;
-}
-
-static char *dupstring(char *str)
-{
- int sz = strlen(str) + 1;
- char *new = malloc(sz);
- if (new)
- memcpy(new, str, sz);
- return new;
-}
-
-static hnode_t *new_node(void *c)
-{
- static hnode_t few[5];
- static int count;
-
- if (count < 5)
- return few + count++;
-
- return NULL;
-}
-
-static void del_node(hnode_t *n, void *c)
-{
-}
-
-int main(void)
-{
- input_t in;
- hash_t *h = hash_create(HASHCOUNT_T_MAX, 0, 0);
- hnode_t *hn;
- hscan_t hs;
- char *tok1, *tok2, *val;
- const char *key;
- int prompt = 0;
-
- char *help =
- "a <key> <val> add value to hash table\n"
- "d <key> delete value from hash table\n"
- "l <key> lookup value in hash table\n"
- "n show size of hash table\n"
- "c show number of entries\n"
- "t dump whole hash table\n"
- "+ increase hash table (private func)\n"
- "- decrease hash table (private func)\n"
- "b print hash_t_bit value\n"
- "p turn prompt on\n"
- "s switch to non-functioning allocator\n"
- "q quit";
-
- if (!h)
- puts("hash_create failed");
-
- for (;;) {
- if (prompt)
- putchar('>');
- fflush(stdout);
-
- if (!fgets(in, sizeof(input_t), stdin))
- break;
-
- switch(in[0]) {
- case '?':
- puts(help);
- break;
- case 'b':
- printf("%d\n", hash_val_t_bit);
- break;
- case 'a':
- if (tokenize(in+1, &tok1, &tok2, (char **) 0) != 2) {
- puts("what?");
- break;
- }
- key = dupstring(tok1);
- val = dupstring(tok2);
-
- if (!key || !val) {
- puts("out of memory");
- free((void *) key);
- free(val);
- }
-
- if (!hash_alloc_insert(h, key, val)) {
- puts("hash_alloc_insert failed");
- free((void *) key);
- free(val);
- break;
- }
- break;
- case 'd':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- hn = hash_lookup(h, tok1);
- if (!hn) {
- puts("hash_lookup failed");
- break;
- }
- val = hnode_get(hn);
- key = hnode_getkey(hn);
- hash_scan_delfree(h, hn);
- free((void *) key);
- free(val);
- break;
- case 'l':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- hn = hash_lookup(h, tok1);
- if (!hn) {
- puts("hash_lookup failed");
- break;
- }
- val = hnode_get(hn);
- puts(val);
- break;
- case 'n':
- printf("%lu\n", (unsigned long) hash_size(h));
- break;
- case 'c':
- printf("%lu\n", (unsigned long) hash_count(h));
- break;
- case 't':
- hash_scan_begin(&hs, h);
- while ((hn = hash_scan_next(&hs)))
- printf("%s\t%s\n", (char*) hnode_getkey(hn),
- (char*) hnode_get(hn));
- break;
- case '+':
- grow_table(h); /* private function */
- break;
- case '-':
- shrink_table(h); /* private function */
- break;
- case 'q':
- exit(0);
- break;
- case '\0':
- break;
- case 'p':
- prompt = 1;
- break;
- case 's':
- hash_set_allocator(h, new_node, del_node, NULL);
- break;
- default:
- putchar('?');
- putchar('\n');
- break;
- }
- }
-
- return 0;
-}
diff --git a/kmer/libutil/kazlib/drivers/list-main.c b/kmer/libutil/kazlib/drivers/list-main.c
deleted file mode 100644
index 6f462e4..0000000
--- a/kmer/libutil/kazlib/drivers/list-main.c
+++ /dev/null
@@ -1,152 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <stdarg.h>
-
-typedef char input_t[256];
-
-static int tokenize(char *string, ...)
-{
- char **tokptr;
- va_list arglist;
- int tokcount = 0;
-
- va_start(arglist, string);
- tokptr = va_arg(arglist, char **);
- while (tokptr) {
- while (*string && isspace((unsigned char) *string))
- string++;
- if (!*string)
- break;
- *tokptr = string;
- while (*string && !isspace((unsigned char) *string))
- string++;
- tokptr = va_arg(arglist, char **);
- tokcount++;
- if (!*string)
- break;
- *string++ = 0;
- }
- va_end(arglist);
-
- return tokcount;
-}
-
-static int comparef(const void *key1, const void *key2)
-{
- return strcmp(key1, key2);
-}
-
-static char *dupstring(char *str)
-{
- int sz = strlen(str) + 1;
- char *new = malloc(sz);
- if (new)
- memcpy(new, str, sz);
- return new;
-}
-
-int main(void)
-{
- input_t in;
- list_t *l = list_create(LISTCOUNT_T_MAX);
- lnode_t *ln;
- char *tok1, *val;
- int prompt = 0;
-
- char *help =
- "a <val> append value to list\n"
- "d <val> delete value from list\n"
- "l <val> lookup value in list\n"
- "s sort list\n"
- "c show number of entries\n"
- "t dump whole list\n"
- "p turn prompt on\n"
- "q quit";
-
- if (!l)
- puts("list_create failed");
-
- for (;;) {
- if (prompt)
- putchar('>');
- fflush(stdout);
-
- if (!fgets(in, sizeof(input_t), stdin))
- break;
-
- switch(in[0]) {
- case '?':
- puts(help);
- break;
- case 'a':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- val = dupstring(tok1);
- ln = lnode_create(val);
-
- if (!val || !ln) {
- puts("allocation failure");
- if (ln)
- lnode_destroy(ln);
- free(val);
- break;
- }
-
- list_append(l, ln);
- break;
- case 'd':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- ln = list_find(l, tok1, comparef);
- if (!ln) {
- puts("list_find failed");
- break;
- }
- list_delete(l, ln);
- val = lnode_get(ln);
- lnode_destroy(ln);
- free(val);
- break;
- case 'l':
- if (tokenize(in+1, &tok1, (char **) 0) != 1) {
- puts("what?");
- break;
- }
- ln = list_find(l, tok1, comparef);
- if (!ln)
- puts("list_find failed");
- else
- puts("found");
- break;
- case 's':
- list_sort(l, comparef);
- break;
- case 'c':
- printf("%lu\n", (unsigned long) list_count(l));
- break;
- case 't':
- for (ln = list_first(l); ln != 0; ln = list_next(l, ln))
- puts(lnode_get(ln));
- break;
- case 'q':
- exit(0);
- break;
- case '\0':
- break;
- case 'p':
- prompt = 1;
- break;
- default:
- putchar('?');
- putchar('\n');
- break;
- }
- }
-
- return 0;
-}
diff --git a/kmer/libutil/kazlib/drivers/sfx-main.c b/kmer/libutil/kazlib/drivers/sfx-main.c
deleted file mode 100644
index fda683b..0000000
--- a/kmer/libutil/kazlib/drivers/sfx-main.c
+++ /dev/null
@@ -1,41 +0,0 @@
-#include <stdlib.h>
-
-int main(int argc, char **argv)
-{
- char expr_buf[256];
- char *expr, *ptr;
- sfx_rating_t eff;
-
- for (;;) {
- if (argc < 2) {
- expr = expr_buf;
- if (fgets(expr_buf, sizeof expr_buf, stdin) == 0)
- break;
- if ((ptr = strchr(expr_buf, '\n')) != 0)
- *ptr = 0;
- } else {
- expr = (argv++)[1];
- if (!expr)
- break;
- }
-
- if (!sfx_determine(expr, &eff)) {
- printf("expression '%s' has a syntax error\n", expr);
- return EXIT_FAILURE;
- }
-
- switch (eff) {
- case sfx_none:
- printf("expression '%s' has no side effects\n", expr);
- break;
- case sfx_potential:
- printf("expression '%s' may have side effects\n", expr);
- break;
- case sfx_certain:
- printf("expression '%s' has side effects\n", expr);
- break;
- }
- }
-
- return 0;
-}
diff --git a/kmer/libutil/kazlib/except.c b/kmer/libutil/kazlib/except.c
deleted file mode 100644
index c915dda..0000000
--- a/kmer/libutil/kazlib/except.c
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Portable Exception Handling for ANSI C.
- * Copyright (C) 1999 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#include <assert.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <limits.h>
-#include "except.h"
-
-#define XCEPT_BUFFER_SIZE 1024
-
-#define group except_group
-#define code except_code
-#define id except_id
-#define message except_message
-#define dyndata except_dyndata
-#define func except_func
-#define context except_context
-#define id except_id
-#define size except_size
-#define obj except_obj
-#define jmp except_jmp
-#define down except_down
-#define type except_type
-#define catcher except_catcher
-#define cleanup except_cleanup
-#define info except_info
-
-#ifdef KAZLIB_POSIX_THREADS
-
-#include <pthread.h>
-
-static pthread_mutex_t init_mtx = PTHREAD_MUTEX_INITIALIZER;
-static int init_counter;
-static pthread_key_t top_key;
-static pthread_key_t uh_key;
-static pthread_key_t alloc_key;
-static pthread_key_t dealloc_key;
-static void unhandled_catcher(except_t *);
-
-#define get_top() ((struct except_stacknode *) pthread_getspecific(top_key))
-#define set_top(T) (pthread_setspecific(top_key, (T)), (void)((T) == (struct except_stacknode *) 0))
-#define set_catcher(C) (pthread_setspecific(uh_key, (void *) (C)), (void)((C) == (void (*)(except_t *)) 0))
-#define set_alloc(A) (pthread_setspecific(alloc_key, (void *) (A)), (void)((A) == (void *(*)(size_t)) 0))
-#define set_dealloc(D) (pthread_setspecific(dealloc_key, (void *) (D)), (void)((D) == (void (*)(void *)) 0))
-
-static void (*get_catcher(void))(except_t *)
-{
- void (*catcher)(except_t *) = (void (*)(except_t *)) pthread_getspecific(uh_key);
- return (catcher == 0) ? unhandled_catcher : catcher;
-}
-
-static void *(*get_alloc(void))(size_t)
-{
- void *(*alloc)(size_t) = (void *(*)(size_t)) pthread_getspecific(alloc_key);
- return (alloc == 0) ? malloc : alloc;
-}
-
-static void (*get_dealloc(void))(void *)
-{
- void (*dealloc)(void *) = (void (*)(void *)) pthread_getspecific(dealloc_key);
- return (dealloc == 0) ? free : dealloc;
-}
-
-int except_init(void)
-{
- int retval = 1;
-
- pthread_mutex_lock(&init_mtx);
-
- assert (init_counter < INT_MAX);
-
- if (init_counter++ == 0) {
- int top_ok = (pthread_key_create(&top_key, 0) == 0);
- int uh_ok = (pthread_key_create(&uh_key, 0) == 0);
- int alloc_ok = (pthread_key_create(&alloc_key, 0) == 0);
- int dealloc_ok = (pthread_key_create(&dealloc_key, 0) == 0);
-
- if (!top_ok || !uh_ok || !alloc_ok || !dealloc_ok) {
- retval = 0;
- init_counter = 0;
- if (top_ok)
- pthread_key_delete(top_key);
- if (uh_ok)
- pthread_key_delete(uh_key);
- if (alloc_ok)
- pthread_key_delete(alloc_key);
- if (dealloc_ok)
- pthread_key_delete(dealloc_key);
- }
- }
-
- pthread_mutex_unlock(&init_mtx);
-
- return retval;
-}
-
-void except_deinit(void)
-{
- pthread_mutex_lock(&init_mtx);
-
- assert (init_counter > 0);
-
- if (--init_counter == 0) {
- pthread_key_delete(top_key);
- pthread_key_delete(uh_key);
- pthread_key_delete(alloc_key);
- pthread_key_delete(dealloc_key);
- }
-
- pthread_mutex_unlock(&init_mtx);
-}
-
-#else /* no thread support */
-
-static int init_counter;
-static void unhandled_catcher(except_t *);
-static void (*uh_catcher_ptr)(except_t *) = unhandled_catcher;
-static void *(*allocator)(size_t) = malloc;
-static void (*deallocator)(void *) = free;
-static struct except_stacknode *stack_top;
-
-#define get_top() (stack_top)
-#define set_top(T) (stack_top = (T))
-#define get_catcher() (uh_catcher_ptr)
-#define set_catcher(C) (uh_catcher_ptr = (C))
-#define get_alloc() (allocator)
-#define set_alloc(A) (allocator = (A))
-#define get_dealloc() (deallocator)
-#define set_dealloc(D) (deallocator = (D))
-
-int except_init(void)
-{
- assert (init_counter < INT_MAX);
- init_counter++;
- return 1;
-}
-
-void except_deinit(void)
-{
- assert (init_counter > 0);
- init_counter--;
-}
-
-#endif
-
-
-static int match(const volatile except_id_t *thrown, const except_id_t *caught)
-{
- int group_match = (caught->group == XCEPT_GROUP_ANY || caught->group == thrown->group);
- int code_match = (caught->code == XCEPT_CODE_ANY || caught->code == thrown->code);
-
- return group_match && code_match;
-}
-
-static void do_throw(except_t *except)
-{
- struct except_stacknode *top;
-
- assert (except->id.group != 0 && except->id.code != 0);
-
- for (top = get_top(); top != 0; top = top->down) {
- if (top->type == XCEPT_CLEANUP) {
- top->info.cleanup->func(top->info.cleanup->context);
- } else {
- struct except_catch *catcher = top->info.catcher;
- const except_id_t *pi = catcher->id;
- size_t i;
-
- assert (top->type == XCEPT_CATCHER);
- except_free(catcher->obj.dyndata);
-
- for (i = 0; i < catcher->size; pi++, i++) {
- if (match(&except->id, pi)) {
- catcher->obj = *except;
- set_top(top);
- longjmp(catcher->jmp, 1);
- }
- }
- }
- }
-
- set_top(top);
- get_catcher()(except); /* unhandled exception */
- abort();
-}
-
-static void unhandled_catcher(except_t *except)
-{
- fprintf(stderr, "Unhandled exception (\"%s\", group=%ld, code=%ld)\n",
- except->message, except->id.group, except->id.code);
- abort();
-}
-
-static void stack_push(struct except_stacknode *node)
-{
- node->down = get_top();
- set_top(node);
-}
-
-void except_setup_clean(struct except_stacknode *esn,
- struct except_cleanup *ecl, void (*cleanf)(void *), void *context)
-{
- esn->type = XCEPT_CLEANUP;
- ecl->func = cleanf;
- ecl->context = context;
- esn->info.cleanup = ecl;
- stack_push(esn);
-}
-
-void except_setup_try(struct except_stacknode *esn,
- struct except_catch *ech, const except_id_t id[], size_t size)
-{
- ech->id = id;
- ech->size = size;
- ech->obj.dyndata = 0;
- esn->type = XCEPT_CATCHER;
- esn->info.catcher = ech;
- stack_push(esn);
-}
-
-struct except_stacknode *except_pop(void)
-{
- struct except_stacknode *top = get_top();
- set_top(top->down);
- return top;
-}
-
-void except_rethrow(except_t *except)
-{
- struct except_stacknode *top = get_top();
- assert (top != 0);
- assert (top->type == XCEPT_CATCHER);
- assert (&top->info.catcher->obj == except);
- set_top(top->down);
- do_throw(except);
-}
-
-void except_throw(long group, long code, const char *msg)
-{
- except_t except;
-
- except.id.group = group;
- except.id.code = code;
- except.message = msg;
- except.dyndata = 0;
-
- do_throw(&except);
-}
-
-void except_throwd(long group, long code, const char *msg, void *data)
-{
- except_t except;
-
- except.id.group = group;
- except.id.code = code;
- except.message = msg;
- except.dyndata = data;
-
- do_throw(&except);
-}
-
-void except_throwf(long group, long code, const char *fmt, ...)
-{
- char *buf = except_alloc(XCEPT_BUFFER_SIZE);
- va_list vl;
-
- va_start (vl, fmt);
- vsprintf(buf, fmt, vl);
- va_end (vl);
- except_throwd(group, code, buf, buf);
-}
-
-void (*except_unhandled_catcher(void (*new_catcher)(except_t *)))(except_t *)
-{
- void (*old_catcher)(except_t *) = get_catcher();
- set_catcher(new_catcher);
- return old_catcher;
-}
-
-#undef except_code
-#undef except_group
-#undef except_message
-#undef except_data
-
-unsigned long except_code(except_t *ex)
-{
- return ex->id.code;
-}
-
-unsigned long except_group(except_t *ex)
-{
- return ex->id.group;
-}
-
-const char *except_message(except_t *ex)
-{
- return ex->message;
-}
-
-void *except_data(except_t *ex)
-{
- return ex->dyndata;
-}
-
-void *except_take_data(except_t *ex)
-{
- void *data = ex->dyndata;
- ex->dyndata = 0;
- return data;
-}
-
-void except_set_allocator(void *(*alloc)(size_t), void (*dealloc)(void *))
-{
- set_alloc(alloc);
- set_dealloc(dealloc);
-}
-
-void *except_alloc(size_t size)
-{
- void *ptr = get_alloc()(size);
-
- if (ptr == 0)
- except_throw(XCEPT_BAD_ALLOC, 0, "out of memory");
- return ptr;
-}
-
-void except_free(void *ptr)
-{
- get_dealloc()(ptr);
-}
diff --git a/kmer/libutil/kazlib/except.h b/kmer/libutil/kazlib/except.h
deleted file mode 100644
index 3131fb9..0000000
--- a/kmer/libutil/kazlib/except.h
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Portable Exception Handling for ANSI C.
- * Copyright (C) 1999 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#ifndef XCEPT_H
-#define XCEPT_H
-
-#include <setjmp.h>
-#include <stdlib.h>
-#include <assert.h>
-
-#define XCEPT_GROUP_ANY 0
-#define XCEPT_CODE_ANY 0
-#define XCEPT_BAD_ALLOC 1
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum { except_no_call, except_call };
-
-typedef struct {
- unsigned long except_group;
- unsigned long except_code;
-} except_id_t;
-
-typedef struct {
- except_id_t volatile except_id;
- const char *volatile except_message;
- void *volatile except_dyndata;
-} except_t;
-
-struct except_cleanup {
- void (*except_func)(void *);
- void *except_context;
-};
-
-struct except_catch {
- const except_id_t *except_id;
- size_t except_size;
- except_t except_obj;
- jmp_buf except_jmp;
-};
-
-enum except_stacktype {
- XCEPT_CLEANUP, XCEPT_CATCHER
-};
-
-struct except_stacknode {
- struct except_stacknode *except_down;
- enum except_stacktype except_type;
- union {
- struct except_catch *except_catcher;
- struct except_cleanup *except_cleanup;
- } except_info;
-};
-
-/* private functions made external so they can be used in macros */
-void except_setup_clean(struct except_stacknode *,
- struct except_cleanup *, void (*)(void *), void *);
-void except_setup_try(struct except_stacknode *,
- struct except_catch *, const except_id_t [], size_t);
-struct except_stacknode *except_pop(void);
-
-/* public interface functions */
-int except_init(void);
-void except_deinit(void);
-void except_rethrow(except_t *);
-void except_throw(long, long, const char *);
-void except_throwd(long, long, const char *, void *);
-void except_throwf(long, long, const char *, ...);
-void (*except_unhandled_catcher(void (*)(except_t *)))(except_t *);
-unsigned long except_code(except_t *);
-unsigned long except_group(except_t *);
-const char *except_message(except_t *);
-void *except_data(except_t *);
-void *except_take_data(except_t *);
-void except_set_allocator(void *(*)(size_t), void (*)(void *));
-void *except_alloc(size_t);
-void except_free(void *);
-
-#define except_code(E) ((E)->except_id.except_code)
-#define except_group(E) ((E)->except_id.except_group)
-#define except_message(E) ((E)->except_message)
-#define except_data(E) ((E)->except_dyndata)
-
-#ifdef __cplusplus
-}
-#endif
-
-/*
- * void except_cleanup_push(void (*)(void *), void *);
- * void except_cleanup_pop(int);
- * void except_checked_cleanup_pop(void (*)(void *), int);
- * void except_try_push(const except_id_t [], size_t, except_t **);
- * void except_try_pop(void);
- */
-
-#define except_cleanup_push(F, C) \
- { \
- struct except_stacknode except_sn; \
- struct except_cleanup except_cl; \
- except_setup_clean(&except_sn, &except_cl, F, C)
-
-#define except_cleanup_pop(E) \
- except_pop(); \
- if (E) \
- except_cl.except_func(except_cl.except_context); \
- }
-
-#define except_checked_cleanup_pop(F, E) \
- except_pop(); \
- assert (except_cl.except_func == (F)); \
- if (E) \
- except_cl.except_func(except_cl.except_context); \
- }
-
-#define except_try_push(ID, NUM, PPE) \
- { \
- struct except_stacknode except_sn; \
- struct except_catch except_ch; \
- except_setup_try(&except_sn, &except_ch, ID, NUM); \
- if (setjmp(except_ch.except_jmp)) \
- *(PPE) = &except_ch.except_obj; \
- else \
- *(PPE) = 0
-
-#define except_try_pop() \
- except_free(except_ch.except_obj.except_dyndata); \
- except_pop(); \
- }
-
-#endif
diff --git a/kmer/libutil/kazlib/hash.c b/kmer/libutil/kazlib/hash.c
deleted file mode 100644
index 2140e66..0000000
--- a/kmer/libutil/kazlib/hash.c
+++ /dev/null
@@ -1,837 +0,0 @@
-/*
- * Hash Table Data Type
- * Copyright (C) 1997 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#include <stdlib.h>
-#include <stddef.h>
-#include <assert.h>
-#include <string.h>
-#define HASH_IMPLEMENTATION
-#include "hash.h"
-
-#define INIT_BITS 6
-#define INIT_SIZE (1UL << (INIT_BITS)) /* must be power of two */
-#define INIT_MASK ((INIT_SIZE) - 1)
-
-#define next hash_next
-#define key hash_key
-#define data hash_data
-#define hkey hash_hkey
-
-#define table hash_table
-#define nchains hash_nchains
-#define nodecount hash_nodecount
-#define maxcount hash_maxcount
-#define highmark hash_highmark
-#define lowmark hash_lowmark
-#define compare hash_compare
-#define function hash_function
-#define allocnode hash_allocnode
-#define freenode hash_freenode
-#define context hash_context
-#define mask hash_mask
-#define dynamic hash_dynamic
-
-#define table hash_table
-#define chain hash_chain
-
-static hnode_t *hnode_alloc(void *context);
-static void hnode_free(hnode_t *node, void *context);
-static hash_val_t hash_fun_default(const void *key);
-static int hash_comp_default(const void *key1, const void *key2);
-
-int hash_val_t_bit;
-
-/*
- * Compute the number of bits in the hash_val_t type. We know that hash_val_t
- * is an unsigned integral type. Thus the highest value it can hold is a
- * Mersenne number (power of two, less one). We initialize a hash_val_t
- * object with this value and then shift bits out one by one while counting.
- * Notes:
- * 1. HASH_VAL_T_MAX is a Mersenne number---one that is one less than a power
- * of two. This means that its binary representation consists of all one
- * bits, and hence ``val'' is initialized to all one bits.
- * 2. While bits remain in val, we increment the bit count and shift it to the
- * right, replacing the topmost bit by zero.
- */
-
-static void compute_bits(void)
-{
- hash_val_t val = HASH_VAL_T_MAX; /* 1 */
- int bits = 0;
-
- while (val) { /* 2 */
- bits++;
- val >>= 1;
- }
-
- hash_val_t_bit = bits;
-}
-
-/*
- * Verify whether the given argument is a power of two.
- */
-
-static int is_power_of_two(hash_val_t arg)
-{
- if (arg == 0)
- return 0;
- while ((arg & 1) == 0)
- arg >>= 1;
- return (arg == 1);
-}
-
-/*
- * Compute a shift amount from a given table size
- */
-
-static hash_val_t compute_mask(hashcount_t size)
-{
- assert (is_power_of_two(size));
- assert (size >= 2);
-
- return size - 1;
-}
-
-/*
- * Initialize the table of pointers to null.
- */
-
-static void clear_table(hash_t *hash)
-{
- hash_val_t i;
-
- for (i = 0; i < hash->nchains; i++)
- hash->table[i] = NULL;
-}
-
-/*
- * Double the size of a dynamic table. This works as follows. Each chain splits
- * into two adjacent chains. The shift amount increases by one, exposing an
- * additional bit of each hashed key. For each node in the original chain, the
- * value of this newly exposed bit will decide which of the two new chains will
- * receive the node: if the bit is 1, the chain with the higher index will have
- * the node, otherwise the lower chain will receive the node. In this manner,
- * the hash table will continue to function exactly as before without having to
- * rehash any of the keys.
- * Notes:
- * 1. Overflow check.
- * 2. The new number of chains is twice the old number of chains.
- * 3. The new mask is one bit wider than the previous, revealing a
- * new bit in all hashed keys.
- * 4. Allocate a new table of chain pointers that is twice as large as the
- * previous one.
- * 5. If the reallocation was successful, we perform the rest of the growth
- * algorithm, otherwise we do nothing.
- * 6. The exposed_bit variable holds a mask with which each hashed key can be
- * AND-ed to test the value of its newly exposed bit.
- * 7. Now loop over each chain in the table and sort its nodes into two
- * chains based on the value of each node's newly exposed hash bit.
- * 8. The low chain replaces the current chain. The high chain goes
- * into the corresponding sister chain in the upper half of the table.
- * 9. We have finished dealing with the chains and nodes. We now update
- * the various bookeeping fields of the hash structure.
- */
-
-static void grow_table(hash_t *hash)
-{
- hnode_t **newtable;
-
- assert (2 * hash->nchains > hash->nchains); /* 1 */
-
- newtable = realloc(hash->table,
- sizeof *newtable * hash->nchains * 2); /* 4 */
-
- if (newtable) { /* 5 */
- hash_val_t mask = (hash->mask << 1) | 1; /* 3 */
- hash_val_t exposed_bit = mask ^ hash->mask; /* 6 */
- hash_val_t chain;
-
- assert (mask != hash->mask);
-
- for (chain = 0; chain < hash->nchains; chain++) { /* 7 */
- hnode_t *low_chain = 0, *high_chain = 0, *hptr, *next;
-
- for (hptr = newtable[chain]; hptr != 0; hptr = next) {
- next = hptr->next;
-
- if (hptr->hkey & exposed_bit) {
- hptr->next = high_chain;
- high_chain = hptr;
- } else {
- hptr->next = low_chain;
- low_chain = hptr;
- }
- }
-
- newtable[chain] = low_chain; /* 8 */
- newtable[chain + hash->nchains] = high_chain;
- }
-
- hash->table = newtable; /* 9 */
- hash->mask = mask;
- hash->nchains *= 2;
- hash->lowmark *= 2;
- hash->highmark *= 2;
- }
- assert (hash_verify(hash));
-}
-
-/*
- * Cut a table size in half. This is done by folding together adjacent chains
- * and populating the lower half of the table with these chains. The chains are
- * simply spliced together. Once this is done, the whole table is reallocated
- * to a smaller object.
- * Notes:
- * 1. It is illegal to have a hash table with one slot. This would mean that
- * hash->shift is equal to hash_val_t_bit, an illegal shift value.
- * Also, other things could go wrong, such as hash->lowmark becoming zero.
- * 2. Looping over each pair of sister chains, the low_chain is set to
- * point to the head node of the chain in the lower half of the table,
- * and high_chain points to the head node of the sister in the upper half.
- * 3. The intent here is to compute a pointer to the last node of the
- * lower chain into the low_tail variable. If this chain is empty,
- * low_tail ends up with a null value.
- * 4. If the lower chain is not empty, we simply tack the upper chain onto it.
- * If the upper chain is a null pointer, nothing happens.
- * 5. Otherwise if the lower chain is empty but the upper one is not,
- * If the low chain is empty, but the high chain is not, then the
- * high chain is simply transferred to the lower half of the table.
- * 6. Otherwise if both chains are empty, there is nothing to do.
- * 7. All the chain pointers are in the lower half of the table now, so
- * we reallocate it to a smaller object. This, of course, invalidates
- * all pointer-to-pointers which reference into the table from the
- * first node of each chain.
- * 8. Though it's unlikely, the reallocation may fail. In this case we
- * pretend that the table _was_ reallocated to a smaller object.
- * 9. Finally, update the various table parameters to reflect the new size.
- */
-
-static void shrink_table(hash_t *hash)
-{
- hash_val_t chain, nchains;
- hnode_t **newtable, *low_tail, *low_chain, *high_chain;
-
- assert (hash->nchains >= 2); /* 1 */
- nchains = hash->nchains / 2;
-
- for (chain = 0; chain < nchains; chain++) {
- low_chain = hash->table[chain]; /* 2 */
- high_chain = hash->table[chain + nchains];
- for (low_tail = low_chain; low_tail && low_tail->next; low_tail = low_tail->next)
- ; /* 3 */
- if (low_chain != 0) /* 4 */
- low_tail->next = high_chain;
- else if (high_chain != 0) /* 5 */
- hash->table[chain] = high_chain;
- else
- assert (hash->table[chain] == NULL); /* 6 */
- }
- newtable = realloc(hash->table,
- sizeof *newtable * nchains); /* 7 */
- if (newtable) /* 8 */
- hash->table = newtable;
- hash->mask >>= 1; /* 9 */
- hash->nchains = nchains;
- hash->lowmark /= 2;
- hash->highmark /= 2;
- assert (hash_verify(hash));
-}
-
-
-/*
- * Create a dynamic hash table. Both the hash table structure and the table
- * itself are dynamically allocated. Furthermore, the table is extendible in
- * that it will automatically grow as its load factor increases beyond a
- * certain threshold.
- * Notes:
- * 1. If the number of bits in the hash_val_t type has not been computed yet,
- * we do so here, because this is likely to be the first function that the
- * user calls.
- * 2. Allocate a hash table control structure.
- * 3. If a hash table control structure is successfully allocated, we
- * proceed to initialize it. Otherwise we return a null pointer.
- * 4. We try to allocate the table of hash chains.
- * 5. If we were able to allocate the hash chain table, we can finish
- * initializing the hash structure and the table. Otherwise, we must
- * backtrack by freeing the hash structure.
- * 6. INIT_SIZE should be a power of two. The high and low marks are always set
- * to be twice the table size and half the table size respectively. When the
- * number of nodes in the table grows beyond the high size (beyond load
- * factor 2), it will double in size to cut the load factor down to about
- * about 1. If the table shrinks down to or beneath load factor 0.5,
- * it will shrink, bringing the load up to about 1. However, the table
- * will never shrink beneath INIT_SIZE even if it's emptied.
- * 7. This indicates that the table is dynamically allocated and dynamically
- * resized on the fly. A table that has this value set to zero is
- * assumed to be statically allocated and will not be resized.
- * 8. The table of chains must be properly reset to all null pointers.
- */
-
-hash_t *hash_create(hashcount_t maxcount, hash_comp_t compfun,
- hash_fun_t hashfun)
-{
- hash_t *hash;
-
- if (hash_val_t_bit == 0) /* 1 */
- compute_bits();
-
- hash = malloc(sizeof *hash); /* 2 */
-
- if (hash) { /* 3 */
- hash->table = malloc(sizeof *hash->table * INIT_SIZE); /* 4 */
- if (hash->table) { /* 5 */
- hash->nchains = INIT_SIZE; /* 6 */
- hash->highmark = INIT_SIZE * 2;
- hash->lowmark = INIT_SIZE / 2;
- hash->nodecount = 0;
- hash->maxcount = maxcount;
- hash->compare = compfun ? compfun : hash_comp_default;
- hash->function = hashfun ? hashfun : hash_fun_default;
- hash->allocnode = hnode_alloc;
- hash->freenode = hnode_free;
- hash->context = NULL;
- hash->mask = INIT_MASK;
- hash->dynamic = 1; /* 7 */
- clear_table(hash); /* 8 */
- assert (hash_verify(hash));
- return hash;
- }
- free(hash);
- }
-
- return NULL;
-}
-
-/*
- * Select a different set of node allocator routines.
- */
-
-void hash_set_allocator(hash_t *hash, hnode_alloc_t al,
- hnode_free_t fr, void *context)
-{
- assert (hash_count(hash) == 0);
- assert ((al == 0 && fr == 0) || (al != 0 && fr != 0));
-
- hash->allocnode = al ? al : hnode_alloc;
- hash->freenode = fr ? fr : hnode_free;
- hash->context = context;
-}
-
-/*
- * Free every node in the hash using the hash->freenode() function pointer, and
- * cause the hash to become empty.
- */
-
-void hash_free_nodes(hash_t *hash)
-{
- hscan_t hs;
- hnode_t *node;
- hash_scan_begin(&hs, hash);
- while ((node = hash_scan_next(&hs))) {
- hash_scan_delete(hash, node);
- hash->freenode(node, hash->context);
- }
- hash->nodecount = 0;
- clear_table(hash);
-}
-
-/*
- * Obsolescent function for removing all nodes from a table,
- * freeing them and then freeing the table all in one step.
- */
-
-void hash_free(hash_t *hash)
-{
-#ifdef KAZLIB_OBSOLESCENT_DEBUG
- assert ("call to obsolescent function hash_free()" && 0);
-#endif
- hash_free_nodes(hash);
- hash_destroy(hash);
-}
-
-/*
- * Free a dynamic hash table structure.
- */
-
-void hash_destroy(hash_t *hash)
-{
- assert (hash_val_t_bit != 0);
- assert (hash_isempty(hash));
- free(hash->table);
- free(hash);
-}
-
-/*
- * Initialize a user supplied hash structure. The user also supplies a table of
- * chains which is assigned to the hash structure. The table is static---it
- * will not grow or shrink.
- * 1. See note 1. in hash_create().
- * 2. The user supplied array of pointers hopefully contains nchains nodes.
- * 3. See note 7. in hash_create().
- * 4. We must dynamically compute the mask from the given power of two table
- * size.
- * 5. The user supplied table can't be assumed to contain null pointers,
- * so we reset it here.
- */
-
-hash_t *hash_init(hash_t *hash, hashcount_t maxcount,
- hash_comp_t compfun, hash_fun_t hashfun, hnode_t **table,
- hashcount_t nchains)
-{
- if (hash_val_t_bit == 0) /* 1 */
- compute_bits();
-
- assert (is_power_of_two(nchains));
-
- hash->table = table; /* 2 */
- hash->nchains = nchains;
- hash->nodecount = 0;
- hash->maxcount = maxcount;
- hash->compare = compfun ? compfun : hash_comp_default;
- hash->function = hashfun ? hashfun : hash_fun_default;
- hash->dynamic = 0; /* 3 */
- hash->mask = compute_mask(nchains); /* 4 */
- clear_table(hash); /* 5 */
-
- assert (hash_verify(hash));
-
- return hash;
-}
-
-/*
- * Reset the hash scanner so that the next element retrieved by
- * hash_scan_next() shall be the first element on the first non-empty chain.
- * Notes:
- * 1. Locate the first non empty chain.
- * 2. If an empty chain is found, remember which one it is and set the next
- * pointer to refer to its first element.
- * 3. Otherwise if a chain is not found, set the next pointer to NULL
- * so that hash_scan_next() shall indicate failure.
- */
-
-void hash_scan_begin(hscan_t *scan, hash_t *hash)
-{
- hash_val_t nchains = hash->nchains;
- hash_val_t chain;
-
- scan->table = hash;
-
- /* 1 */
-
- for (chain = 0; chain < nchains && hash->table[chain] == 0; chain++)
- ;
-
- if (chain < nchains) { /* 2 */
- scan->chain = chain;
- scan->next = hash->table[chain];
- } else { /* 3 */
- scan->next = NULL;
- }
-}
-
-/*
- * Retrieve the next node from the hash table, and update the pointer
- * for the next invocation of hash_scan_next().
- * Notes:
- * 1. Remember the next pointer in a temporary value so that it can be
- * returned.
- * 2. This assertion essentially checks whether the module has been properly
- * initialized. The first point of interaction with the module should be
- * either hash_create() or hash_init(), both of which set hash_val_t_bit to
- * a non zero value.
- * 3. If the next pointer we are returning is not NULL, then the user is
- * allowed to call hash_scan_next() again. We prepare the new next pointer
- * for that call right now. That way the user is allowed to delete the node
- * we are about to return, since we will no longer be needing it to locate
- * the next node.
- * 4. If there is a next node in the chain (next->next), then that becomes the
- * new next node, otherwise ...
- * 5. We have exhausted the current chain, and must locate the next subsequent
- * non-empty chain in the table.
- * 6. If a non-empty chain is found, the first element of that chain becomes
- * the new next node. Otherwise there is no new next node and we set the
- * pointer to NULL so that the next time hash_scan_next() is called, a null
- * pointer shall be immediately returned.
- */
-
-
-hnode_t *hash_scan_next(hscan_t *scan)
-{
- hnode_t *next = scan->next; /* 1 */
- hash_t *hash = scan->table;
- hash_val_t chain = scan->chain + 1;
- hash_val_t nchains = hash->nchains;
-
- assert (hash_val_t_bit != 0); /* 2 */
-
- if (next) { /* 3 */
- if (next->next) { /* 4 */
- scan->next = next->next;
- } else {
- while (chain < nchains && hash->table[chain] == 0) /* 5 */
- chain++;
- if (chain < nchains) { /* 6 */
- scan->chain = chain;
- scan->next = hash->table[chain];
- } else {
- scan->next = NULL;
- }
- }
- }
- return next;
-}
-
-/*
- * Insert a node into the hash table.
- * Notes:
- * 1. It's illegal to insert more than the maximum number of nodes. The client
- * should verify that the hash table is not full before attempting an
- * insertion.
- * 2. The same key may not be inserted into a table twice.
- * 3. If the table is dynamic and the load factor is already at >= 2,
- * grow the table.
- * 4. We take the bottom N bits of the hash value to derive the chain index,
- * where N is the base 2 logarithm of the size of the hash table.
- */
-
-void hash_insert(hash_t *hash, hnode_t *node, const void *key)
-{
- hash_val_t hkey, chain;
-
- assert (hash_val_t_bit != 0);
- assert (node->next == NULL);
- assert (hash->nodecount < hash->maxcount); /* 1 */
- assert (hash_lookup(hash, key) == NULL); /* 2 */
-
- if (hash->dynamic && hash->nodecount >= hash->highmark) /* 3 */
- grow_table(hash);
-
- hkey = hash->function(key);
- chain = hkey & hash->mask; /* 4 */
-
- node->key = key;
- node->hkey = hkey;
- node->next = hash->table[chain];
- hash->table[chain] = node;
- hash->nodecount++;
-
- assert (hash_verify(hash));
-}
-
-/*
- * Find a node in the hash table and return a pointer to it.
- * Notes:
- * 1. We hash the key and keep the entire hash value. As an optimization, when
- * we descend down the chain, we can compare hash values first and only if
- * hash values match do we perform a full key comparison.
- * 2. To locate the chain from among 2^N chains, we look at the lower N bits of
- * the hash value by anding them with the current mask.
- * 3. Looping through the chain, we compare the stored hash value inside each
- * node against our computed hash. If they match, then we do a full
- * comparison between the unhashed keys. If these match, we have located the
- * entry.
- */
-
-hnode_t *hash_lookup(hash_t *hash, const void *key)
-{
- hash_val_t hkey, chain;
- hnode_t *nptr;
-
- hkey = hash->function(key); /* 1 */
- chain = hkey & hash->mask; /* 2 */
-
- for (nptr = hash->table[chain]; nptr; nptr = nptr->next) { /* 3 */
- if (nptr->hkey == hkey && hash->compare(nptr->key, key) == 0)
- return nptr;
- }
-
- return NULL;
-}
-
-/*
- * Delete the given node from the hash table. Since the chains
- * are singly linked, we must locate the start of the node's chain
- * and traverse.
- * Notes:
- * 1. The node must belong to this hash table, and its key must not have
- * been tampered with.
- * 2. If this deletion will take the node count below the low mark, we
- * shrink the table now.
- * 3. Determine which chain the node belongs to, and fetch the pointer
- * to the first node in this chain.
- * 4. If the node being deleted is the first node in the chain, then
- * simply update the chain head pointer.
- * 5. Otherwise advance to the node's predecessor, and splice out
- * by updating the predecessor's next pointer.
- * 6. Indicate that the node is no longer in a hash table.
- */
-
-hnode_t *hash_delete(hash_t *hash, hnode_t *node)
-{
- hash_val_t chain;
- hnode_t *hptr;
-
- assert (hash_lookup(hash, node->key) == node); /* 1 */
- assert (hash_val_t_bit != 0);
-
- if (hash->dynamic && hash->nodecount <= hash->lowmark
- && hash->nodecount > INIT_SIZE)
- shrink_table(hash); /* 2 */
-
- chain = node->hkey & hash->mask; /* 3 */
- hptr = hash->table[chain];
-
- if (hptr == node) { /* 4 */
- hash->table[chain] = node->next;
- } else {
- while (hptr->next != node) { /* 5 */
- assert (hptr != 0);
- hptr = hptr->next;
- }
- assert (hptr->next == node);
- hptr->next = node->next;
- }
-
- hash->nodecount--;
- assert (hash_verify(hash));
-
- node->next = NULL; /* 6 */
- return node;
-}
-
-int hash_alloc_insert(hash_t *hash, const void *key, void *data)
-{
- hnode_t *node = hash->allocnode(hash->context);
-
- if (node) {
- hnode_init(node, data);
- hash_insert(hash, node, key);
- return 1;
- }
- return 0;
-}
-
-void hash_delete_free(hash_t *hash, hnode_t *node)
-{
- hash_delete(hash, node);
- hash->freenode(node, hash->context);
-}
-
-/*
- * Exactly like hash_delete, except does not trigger table shrinkage. This is to be
- * used from within a hash table scan operation. See notes for hash_delete.
- */
-
-hnode_t *hash_scan_delete(hash_t *hash, hnode_t *node)
-{
- hash_val_t chain;
- hnode_t *hptr;
-
- assert (hash_lookup(hash, node->key) == node);
- assert (hash_val_t_bit != 0);
-
- chain = node->hkey & hash->mask;
- hptr = hash->table[chain];
-
- if (hptr == node) {
- hash->table[chain] = node->next;
- } else {
- while (hptr->next != node)
- hptr = hptr->next;
- hptr->next = node->next;
- }
-
- hash->nodecount--;
- assert (hash_verify(hash));
- node->next = NULL;
-
- return node;
-}
-
-/*
- * Like hash_delete_free but based on hash_scan_delete.
- */
-
-void hash_scan_delfree(hash_t *hash, hnode_t *node)
-{
- hash_scan_delete(hash, node);
- hash->freenode(node, hash->context);
-}
-
-/*
- * Verify whether the given object is a valid hash table. This means
- * Notes:
- * 1. If the hash table is dynamic, verify whether the high and
- * low expansion/shrinkage thresholds are powers of two.
- * 2. Count all nodes in the table, and test each hash value
- * to see whether it is correct for the node's chain.
- */
-
-int hash_verify(hash_t *hash)
-{
- hashcount_t count = 0;
- hash_val_t chain;
- hnode_t *hptr;
-
- if (hash->dynamic) { /* 1 */
- if (hash->lowmark >= hash->highmark)
- return 0;
- if (!is_power_of_two(hash->highmark))
- return 0;
- if (!is_power_of_two(hash->lowmark))
- return 0;
- }
-
- for (chain = 0; chain < hash->nchains; chain++) { /* 2 */
- for (hptr = hash->table[chain]; hptr != 0; hptr = hptr->next) {
- if ((hptr->hkey & hash->mask) != chain)
- return 0;
- count++;
- }
- }
-
- if (count != hash->nodecount)
- return 0;
-
- return 1;
-}
-
-/*
- * Test whether the hash table is full and return 1 if this is true,
- * 0 if it is false.
- */
-
-#undef hash_isfull
-int hash_isfull(hash_t *hash)
-{
- return hash->nodecount == hash->maxcount;
-}
-
-/*
- * Test whether the hash table is empty and return 1 if this is true,
- * 0 if it is false.
- */
-
-#undef hash_isempty
-int hash_isempty(hash_t *hash)
-{
- return hash->nodecount == 0;
-}
-
-static hnode_t *hnode_alloc(void *context)
-{
- return malloc(sizeof *hnode_alloc(NULL));
-}
-
-static void hnode_free(hnode_t *node, void *context)
-{
- free(node);
-}
-
-
-/*
- * Create a hash table node dynamically and assign it the given data.
- */
-
-hnode_t *hnode_create(void *data)
-{
- hnode_t *node = malloc(sizeof *node);
- if (node) {
- node->data = data;
- node->next = NULL;
- }
- return node;
-}
-
-/*
- * Initialize a client-supplied node
- */
-
-hnode_t *hnode_init(hnode_t *hnode, void *data)
-{
- hnode->data = data;
- hnode->next = NULL;
- return hnode;
-}
-
-/*
- * Destroy a dynamically allocated node.
- */
-
-void hnode_destroy(hnode_t *hnode)
-{
- free(hnode);
-}
-
-#undef hnode_put
-void hnode_put(hnode_t *node, void *data)
-{
- node->data = data;
-}
-
-#undef hnode_get
-void *hnode_get(hnode_t *node)
-{
- return node->data;
-}
-
-#undef hnode_getkey
-const void *hnode_getkey(hnode_t *node)
-{
- return node->key;
-}
-
-#undef hash_count
-hashcount_t hash_count(hash_t *hash)
-{
- return hash->nodecount;
-}
-
-#undef hash_size
-hashcount_t hash_size(hash_t *hash)
-{
- return hash->nchains;
-}
-
-static hash_val_t hash_fun_default(const void *key)
-{
- static unsigned long randbox[] = {
- 0x49848f1bU, 0xe6255dbaU, 0x36da5bdcU, 0x47bf94e9U,
- 0x8cbcce22U, 0x559fc06aU, 0xd268f536U, 0xe10af79aU,
- 0xc1af4d69U, 0x1d2917b5U, 0xec4c304dU, 0x9ee5016cU,
- 0x69232f74U, 0xfead7bb3U, 0xe9089ab6U, 0xf012f6aeU,
- };
-
- const unsigned char *str = key;
- hash_val_t acc = 0;
-
- while (*str) {
- acc ^= randbox[(*str + acc) & 0xf];
- acc = (acc << 1) | (acc >> 31);
- acc &= 0xffffffffU;
- acc ^= randbox[((*str++ >> 4) + acc) & 0xf];
- acc = (acc << 2) | (acc >> 30);
- acc &= 0xffffffffU;
- }
- return acc;
-}
-
-static int hash_comp_default(const void *key1, const void *key2)
-{
- return strcmp(key1, key2);
-}
diff --git a/kmer/libutil/kazlib/hash.h b/kmer/libutil/kazlib/hash.h
deleted file mode 100644
index e8213f7..0000000
--- a/kmer/libutil/kazlib/hash.h
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Hash Table Data Type
- * Copyright (C) 1997 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#ifndef HASH_H
-#define HASH_H
-
-#include <limits.h>
-#ifdef KAZLIB_SIDEEFFECT_DEBUG
-#include "sfx.h"
-#endif
-
-/*
- * Blurb for inclusion into C++ translation units
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef unsigned long hashcount_t;
-#define HASHCOUNT_T_MAX ULONG_MAX
-
-typedef unsigned long hash_val_t;
-#define HASH_VAL_T_MAX ULONG_MAX
-
-extern int hash_val_t_bit;
-
-#ifndef HASH_VAL_T_BIT
-#define HASH_VAL_T_BIT ((int) hash_val_t_bit)
-#endif
-
-/*
- * Hash chain node structure.
- * Notes:
- * 1. This preprocessing directive is for debugging purposes. The effect is
- * that if the preprocessor symbol KAZLIB_OPAQUE_DEBUG is defined prior to the
- * inclusion of this header, then the structure shall be declared as having
- * the single member int __OPAQUE__. This way, any attempts by the
- * client code to violate the principles of information hiding (by accessing
- * the structure directly) can be diagnosed at translation time. However,
- * note the resulting compiled unit is not suitable for linking.
- * 2. This is a pointer to the next node in the chain. In the last node of a
- * chain, this pointer is null.
- * 3. The key is a pointer to some user supplied data that contains a unique
- * identifier for each hash node in a given table. The interpretation of
- * the data is up to the user. When creating or initializing a hash table,
- * the user must supply a pointer to a function for comparing two keys,
- * and a pointer to a function for hashing a key into a numeric value.
- * 4. The value is a user-supplied pointer to void which may refer to
- * any data object. It is not interpreted in any way by the hashing
- * module.
- * 5. The hashed key is stored in each node so that we don't have to rehash
- * each key when the table must grow or shrink.
- */
-
-typedef struct hnode_t {
-#if defined(HASH_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG) /* 1 */
- struct hnode_t *hash_next; /* 2 */
- const void *hash_key; /* 3 */
- void *hash_data; /* 4 */
- hash_val_t hash_hkey; /* 5 */
-#else
- int hash_dummy;
-#endif
-} hnode_t;
-
-/*
- * The comparison function pointer type. A comparison function takes two keys
- * and produces a value of -1 if the left key is less than the right key, a
- * value of 0 if the keys are equal, and a value of 1 if the left key is
- * greater than the right key.
- */
-
-typedef int (*hash_comp_t)(const void *, const void *);
-
-/*
- * The hashing function performs some computation on a key and produces an
- * integral value of type hash_val_t based on that key. For best results, the
- * function should have a good randomness properties in *all* significant bits
- * over the set of keys that are being inserted into a given hash table. In
- * particular, the most significant bits of hash_val_t are most significant to
- * the hash module. Only as the hash table expands are less significant bits
- * examined. Thus a function that has good distribution in its upper bits but
- * not lower is preferrable to one that has poor distribution in the upper bits
- * but not the lower ones.
- */
-
-typedef hash_val_t (*hash_fun_t)(const void *);
-
-/*
- * allocator functions
- */
-
-typedef hnode_t *(*hnode_alloc_t)(void *);
-typedef void (*hnode_free_t)(hnode_t *, void *);
-
-/*
- * This is the hash table control structure. It keeps track of information
- * about a hash table, as well as the hash table itself.
- * Notes:
- * 1. Pointer to the hash table proper. The table is an array of pointers to
- * hash nodes (of type hnode_t). If the table is empty, every element of
- * this table is a null pointer. A non-null entry points to the first
- * element of a chain of nodes.
- * 2. This member keeps track of the size of the hash table---that is, the
- * number of chain pointers.
- * 3. The count member maintains the number of elements that are presently
- * in the hash table.
- * 4. The maximum count is the greatest number of nodes that can populate this
- * table. If the table contains this many nodes, no more can be inserted,
- * and the hash_isfull() function returns true.
- * 5. The high mark is a population threshold, measured as a number of nodes,
- * which, if exceeded, will trigger a table expansion. Only dynamic hash
- * tables are subject to this expansion.
- * 6. The low mark is a minimum population threshold, measured as a number of
- * nodes. If the table population drops below this value, a table shrinkage
- * will occur. Only dynamic tables are subject to this reduction. No table
- * will shrink beneath a certain absolute minimum number of nodes.
- * 7. This is the a pointer to the hash table's comparison function. The
- * function is set once at initialization or creation time.
- * 8. Pointer to the table's hashing function, set once at creation or
- * initialization time.
- * 9. The current hash table mask. If the size of the hash table is 2^N,
- * this value has its low N bits set to 1, and the others clear. It is used
- * to select bits from the result of the hashing function to compute an
- * index into the table.
- * 10. A flag which indicates whether the table is to be dynamically resized. It
- * is set to 1 in dynamically allocated tables, 0 in tables that are
- * statically allocated.
- */
-
-typedef struct hash_t {
-#if defined(HASH_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- struct hnode_t **hash_table; /* 1 */
- hashcount_t hash_nchains; /* 2 */
- hashcount_t hash_nodecount; /* 3 */
- hashcount_t hash_maxcount; /* 4 */
- hashcount_t hash_highmark; /* 5 */
- hashcount_t hash_lowmark; /* 6 */
- hash_comp_t hash_compare; /* 7 */
- hash_fun_t hash_function; /* 8 */
- hnode_alloc_t hash_allocnode;
- hnode_free_t hash_freenode;
- void *hash_context;
- hash_val_t hash_mask; /* 9 */
- int hash_dynamic; /* 10 */
-#else
- int hash_dummy;
-#endif
-} hash_t;
-
-/*
- * Hash scanner structure, used for traversals of the data structure.
- * Notes:
- * 1. Pointer to the hash table that is being traversed.
- * 2. Reference to the current chain in the table being traversed (the chain
- * that contains the next node that shall be retrieved).
- * 3. Pointer to the node that will be retrieved by the subsequent call to
- * hash_scan_next().
- */
-
-typedef struct hscan_t {
-#if defined(HASH_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- hash_t *hash_table; /* 1 */
- hash_val_t hash_chain; /* 2 */
- hnode_t *hash_next; /* 3 */
-#else
- int hash_dummy;
-#endif
-} hscan_t;
-
-extern hash_t *hash_create(hashcount_t, hash_comp_t, hash_fun_t);
-extern void hash_set_allocator(hash_t *, hnode_alloc_t, hnode_free_t, void *);
-extern void hash_destroy(hash_t *);
-extern void hash_free_nodes(hash_t *);
-extern void hash_free(hash_t *);
-extern hash_t *hash_init(hash_t *, hashcount_t, hash_comp_t,
- hash_fun_t, hnode_t **, hashcount_t);
-extern void hash_insert(hash_t *, hnode_t *, const void *);
-extern hnode_t *hash_lookup(hash_t *, const void *);
-extern hnode_t *hash_delete(hash_t *, hnode_t *);
-extern int hash_alloc_insert(hash_t *, const void *, void *);
-extern void hash_delete_free(hash_t *, hnode_t *);
-
-extern void hnode_put(hnode_t *, void *);
-extern void *hnode_get(hnode_t *);
-extern const void *hnode_getkey(hnode_t *);
-extern hashcount_t hash_count(hash_t *);
-extern hashcount_t hash_size(hash_t *);
-
-extern int hash_isfull(hash_t *);
-extern int hash_isempty(hash_t *);
-
-extern void hash_scan_begin(hscan_t *, hash_t *);
-extern hnode_t *hash_scan_next(hscan_t *);
-extern hnode_t *hash_scan_delete(hash_t *, hnode_t *);
-extern void hash_scan_delfree(hash_t *, hnode_t *);
-
-extern int hash_verify(hash_t *);
-
-extern hnode_t *hnode_create(void *);
-extern hnode_t *hnode_init(hnode_t *, void *);
-extern void hnode_destroy(hnode_t *);
-
-#if defined(HASH_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
-#ifdef KAZLIB_SIDEEFFECT_DEBUG
-#define hash_isfull(H) (SFX_CHECK(H)->hash_nodecount == (H)->hash_maxcount)
-#else
-#define hash_isfull(H) ((H)->hash_nodecount == (H)->hash_maxcount)
-#endif
-#define hash_isempty(H) ((H)->hash_nodecount == 0)
-#define hash_count(H) ((H)->hash_nodecount)
-#define hash_size(H) ((H)->hash_nchains)
-#define hnode_get(N) ((N)->hash_data)
-#define hnode_getkey(N) ((N)->hash_key)
-#define hnode_put(N, V) ((N)->hash_data = (V))
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/kmer/libutil/kazlib/list.c b/kmer/libutil/kazlib/list.c
deleted file mode 100644
index 818b427..0000000
--- a/kmer/libutil/kazlib/list.c
+++ /dev/null
@@ -1,766 +0,0 @@
-/*
- * List Abstract Data Type
- * Copyright (C) 1997 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-
-#include <stdlib.h>
-#include <stddef.h>
-#include <assert.h>
-#define LIST_IMPLEMENTATION
-#include "list.h"
-
-#define next list_next
-#define prev list_prev
-#define data list_data
-
-#define pool list_pool
-#define fre list_free
-#define size list_size
-
-#define nilnode list_nilnode
-#define nodecount list_nodecount
-#define maxcount list_maxcount
-
-#define list_nil(L) (&(L)->nilnode)
-#define list_first_priv(L) ((L)->nilnode.next)
-#define list_last_priv(L) ((L)->nilnode.prev)
-#define lnode_next(N) ((N)->next)
-#define lnode_prev(N) ((N)->prev)
-
-/*
- * Initialize a list object supplied by the client such that it becomes a valid
- * empty list. If the list is to be ``unbounded'', the maxcount should be
- * specified as LISTCOUNT_T_MAX, or, alternately, as -1. The value zero
- * is not permitted.
- */
-
-list_t *list_init(list_t *list, listcount_t maxcount)
-{
- assert (maxcount != 0);
- list->nilnode.next = &list->nilnode;
- list->nilnode.prev = &list->nilnode;
- list->nodecount = 0;
- list->maxcount = maxcount;
- return list;
-}
-
-/*
- * Dynamically allocate a list object using malloc(), and initialize it so that
- * it is a valid empty list. If the list is to be ``unbounded'', the maxcount
- * should be specified as LISTCOUNT_T_MAX, or, alternately, as -1.
- */
-
-list_t *list_create(listcount_t maxcount)
-{
- list_t *new = malloc(sizeof *new);
- if (new) {
- assert (maxcount != 0);
- new->nilnode.next = &new->nilnode;
- new->nilnode.prev = &new->nilnode;
- new->nodecount = 0;
- new->maxcount = maxcount;
- }
- return new;
-}
-
-/*
- * Destroy a dynamically allocated list object.
- * The client must remove the nodes first.
- */
-
-void list_destroy(list_t *list)
-{
- assert (list_isempty(list));
- free(list);
-}
-
-/*
- * Free all of the nodes of a list. The list must contain only
- * dynamically allocated nodes. After this call, the list
- * is empty.
- */
-
-void list_destroy_nodes(list_t *list)
-{
- lnode_t *lnode = list_first_priv(list), *nil = list_nil(list), *tmp;
-
- while (lnode != nil) {
- tmp = lnode->next;
- lnode->next = NULL;
- lnode->prev = NULL;
- lnode_destroy(lnode);
- lnode = tmp;
- }
-
- list_init(list, list->maxcount);
-}
-
-/*
- * Return all of the nodes of a list to a node pool. The nodes in
- * the list must all have come from the same pool.
- */
-
-void list_return_nodes(list_t *list, lnodepool_t *pool)
-{
- lnode_t *lnode = list_first_priv(list), *tmp, *nil = list_nil(list);
-
- while (lnode != nil) {
- tmp = lnode->next;
- lnode->next = NULL;
- lnode->prev = NULL;
- lnode_return(pool, lnode);
- lnode = tmp;
- }
-
- list_init(list, list->maxcount);
-}
-
-/*
- * Insert the node ``new'' into the list immediately after ``this'' node.
- */
-
-void list_ins_after(list_t *list, lnode_t *new, lnode_t *this)
-{
- lnode_t *that = this->next;
-
- assert (new != NULL);
- assert (!list_contains(list, new));
- assert (!lnode_is_in_a_list(new));
- assert (this == list_nil(list) || list_contains(list, this));
- assert (list->nodecount + 1 > list->nodecount);
-
- new->prev = this;
- new->next = that;
- that->prev = new;
- this->next = new;
- list->nodecount++;
-
- assert (list->nodecount <= list->maxcount);
-}
-
-/*
- * Insert the node ``new'' into the list immediately before ``this'' node.
- */
-
-void list_ins_before(list_t *list, lnode_t *new, lnode_t *this)
-{
- lnode_t *that = this->prev;
-
- assert (new != NULL);
- assert (!list_contains(list, new));
- assert (!lnode_is_in_a_list(new));
- assert (this == list_nil(list) || list_contains(list, this));
- assert (list->nodecount + 1 > list->nodecount);
-
- new->next = this;
- new->prev = that;
- that->next = new;
- this->prev = new;
- list->nodecount++;
-
- assert (list->nodecount <= list->maxcount);
-}
-
-/*
- * Delete the given node from the list.
- */
-
-lnode_t *list_delete(list_t *list, lnode_t *del)
-{
- lnode_t *next = del->next;
- lnode_t *prev = del->prev;
-
- assert (list_contains(list, del));
-
- prev->next = next;
- next->prev = prev;
- list->nodecount--;
-
- del->next = del->prev = NULL;
-
- return del;
-}
-
-/*
- * For each node in the list, execute the given function. The list,
- * current node and the given context pointer are passed on each
- * call to the function.
- */
-
-void list_process(list_t *list, void *context,
- void (* function)(list_t *list, lnode_t *lnode, void *context))
-{
- lnode_t *node = list_first_priv(list), *next, *nil = list_nil(list);
-
- while (node != nil) {
- /* check for callback function deleting */
- /* the next node from under us */
- assert (list_contains(list, node));
- next = node->next;
- function(list, node, context);
- node = next;
- }
-}
-
-/*
- * Dynamically allocate a list node and assign it the given piece of data.
- */
-
-lnode_t *lnode_create(void *data)
-{
- lnode_t *new = malloc(sizeof *new);
- if (new) {
- new->data = data;
- new->next = NULL;
- new->prev = NULL;
- }
- return new;
-}
-
-/*
- * Initialize a user-supplied lnode.
- */
-
-lnode_t *lnode_init(lnode_t *lnode, void *data)
-{
- lnode->data = data;
- lnode->next = NULL;
- lnode->prev = NULL;
- return lnode;
-}
-
-/*
- * Destroy a dynamically allocated node.
- */
-
-void lnode_destroy(lnode_t *lnode)
-{
- assert (!lnode_is_in_a_list(lnode));
- free(lnode);
-}
-
-/*
- * Initialize a node pool object to use a user-supplied set of nodes.
- * The ``nodes'' pointer refers to an array of lnode_t objects, containing
- * ``n'' elements.
- */
-
-lnodepool_t *lnode_pool_init(lnodepool_t *pool, lnode_t *nodes, listcount_t n)
-{
- listcount_t i;
-
- assert (n != 0);
-
- pool->pool = nodes;
- pool->fre = nodes;
- pool->size = n;
- for (i = 0; i < n - 1; i++) {
- nodes[i].next = nodes + i + 1;
- }
- nodes[i].next = NULL;
- nodes[i].prev = nodes; /* to make sure node is marked ``on list'' */
- return pool;
-}
-
-/*
- * Create a dynamically allocated pool of n nodes.
- */
-
-lnodepool_t *lnode_pool_create(listcount_t n)
-{
- lnodepool_t *pool;
- lnode_t *nodes;
-
- assert (n != 0);
-
- pool = malloc(sizeof *pool);
- if (!pool)
- return NULL;
- nodes = malloc(n * sizeof *nodes);
- if (!nodes) {
- free(pool);
- return NULL;
- }
- lnode_pool_init(pool, nodes, n);
- return pool;
-}
-
-/*
- * Determine whether the given pool is from this pool.
- */
-
-int lnode_pool_isfrom(lnodepool_t *pool, lnode_t *node)
-{
- listcount_t i;
-
- /* this is carefully coded this way because ANSI C forbids pointers
- to different objects from being subtracted or compared other
- than for exact equality */
-
- for (i = 0; i < pool->size; i++) {
- if (pool->pool + i == node)
- return 1;
- }
- return 0;
-}
-
-/*
- * Destroy a dynamically allocated pool of nodes.
- */
-
-void lnode_pool_destroy(lnodepool_t *p)
-{
- free(p->pool);
- free(p);
-}
-
-/*
- * Borrow a node from a node pool. Returns a null pointer if the pool
- * is exhausted.
- */
-
-lnode_t *lnode_borrow(lnodepool_t *pool, void *data)
-{
- lnode_t *new = pool->fre;
- if (new) {
- pool->fre = new->next;
- new->data = data;
- new->next = NULL;
- new->prev = NULL;
- }
- return new;
-}
-
-/*
- * Return a node to a node pool. A node must be returned to the pool
- * from which it came.
- */
-
-void lnode_return(lnodepool_t *pool, lnode_t *node)
-{
- assert (lnode_pool_isfrom(pool, node));
- assert (!lnode_is_in_a_list(node));
-
- node->next = pool->fre;
- node->prev = node;
- pool->fre = node;
-}
-
-/*
- * Determine whether the given list contains the given node.
- * According to this function, a list does not contain its nilnode.
- */
-
-int list_contains(list_t *list, lnode_t *node)
-{
- lnode_t *n, *nil = list_nil(list);
-
- for (n = list_first_priv(list); n != nil; n = lnode_next(n)) {
- if (node == n)
- return 1;
- }
-
- return 0;
-}
-
-/*
- * A more generalized variant of list_transfer. This one removes a
- * ``slice'' from the source list and appends it to the destination
- * list.
- */
-
-void list_extract(list_t *dest, list_t *source, lnode_t *first, lnode_t *last)
-{
- listcount_t moved = 1;
-
- assert (first == NULL || list_contains(source, first));
- assert (last == NULL || list_contains(source, last));
-
- if (first == NULL || last == NULL)
- return;
-
- /* adjust the destination list so that the slice is spliced out */
-
- first->prev->next = last->next;
- last->next->prev = first->prev;
-
- /* graft the splice at the end of the dest list */
-
- last->next = &dest->nilnode;
- first->prev = dest->nilnode.prev;
- dest->nilnode.prev->next = first;
- dest->nilnode.prev = last;
-
- while (first != last) {
- first = first->next;
- assert (first != list_nil(source)); /* oops, last before first! */
- moved++;
- }
-
- /* assert no overflows */
- assert (source->nodecount - moved <= source->nodecount);
- assert (dest->nodecount + moved >= dest->nodecount);
-
- /* assert no weirdness */
- assert (moved <= source->nodecount);
-
- source->nodecount -= moved;
- dest->nodecount += moved;
-
- /* assert list sanity */
- assert (list_verify(source));
- assert (list_verify(dest));
-}
-
-
-/*
- * Split off a trailing sequence of nodes from the source list and relocate
- * them to the tail of the destination list. The trailing sequence begins
- * with node ``first'' and terminates with the last node of the source
- * list. The nodes are added to the end of the new list in their original
- * order.
- */
-
-void list_transfer(list_t *dest, list_t *source, lnode_t *first)
-{
- listcount_t moved = 1;
- lnode_t *last;
-
- assert (first == NULL || list_contains(source, first));
-
- if (first == NULL)
- return;
-
- last = source->nilnode.prev;
-
- source->nilnode.prev = first->prev;
- first->prev->next = &source->nilnode;
-
- last->next = &dest->nilnode;
- first->prev = dest->nilnode.prev;
- dest->nilnode.prev->next = first;
- dest->nilnode.prev = last;
-
- while (first != last) {
- first = first->next;
- moved++;
- }
-
- /* assert no overflows */
- assert (source->nodecount - moved <= source->nodecount);
- assert (dest->nodecount + moved >= dest->nodecount);
-
- /* assert no weirdness */
- assert (moved <= source->nodecount);
-
- source->nodecount -= moved;
- dest->nodecount += moved;
-
- /* assert list sanity */
- assert (list_verify(source));
- assert (list_verify(dest));
-}
-
-void list_merge(list_t *dest, list_t *sour,
- int compare (const void *, const void *))
-{
- lnode_t *dn, *sn, *tn;
- lnode_t *d_nil = list_nil(dest), *s_nil = list_nil(sour);
-
- /* Nothing to do if source and destination list are the same. */
- if (dest == sour)
- return;
-
- /* overflow check */
- assert (list_count(sour) + list_count(dest) >= list_count(sour));
-
- /* lists must be sorted */
- assert (list_is_sorted(sour, compare));
- assert (list_is_sorted(dest, compare));
-
- dn = list_first_priv(dest);
- sn = list_first_priv(sour);
-
- while (dn != d_nil && sn != s_nil) {
- if (compare(lnode_get(dn), lnode_get(sn)) >= 0) {
- tn = lnode_next(sn);
- list_delete(sour, sn);
- list_ins_before(dest, sn, dn);
- sn = tn;
- } else {
- dn = lnode_next(dn);
- }
- }
-
- if (dn != d_nil)
- return;
-
- if (sn != s_nil)
- list_transfer(dest, sour, sn);
-}
-
-void list_sort(list_t *list, int compare(const void *, const void *))
-{
- list_t extra;
- listcount_t middle;
- lnode_t *node;
-
- if (list_count(list) > 1) {
- middle = list_count(list) / 2;
- node = list_first_priv(list);
-
- list_init(&extra, list_count(list) - middle);
-
- while (middle--)
- node = lnode_next(node);
-
- list_transfer(&extra, list, node);
- list_sort(list, compare);
- list_sort(&extra, compare);
- list_merge(list, &extra, compare);
- }
- assert (list_is_sorted(list, compare));
-}
-
-lnode_t *list_find(list_t *list, const void *key, int compare(const void *, const void *))
-{
- lnode_t *node;
-
- for (node = list_first_priv(list); node != list_nil(list); node = node->next) {
- if (compare(lnode_get(node), key) == 0)
- return node;
- }
-
- return 0;
-}
-
-
-/*
- * Return 1 if the list is in sorted order, 0 otherwise
- */
-
-int list_is_sorted(list_t *list, int compare(const void *, const void *))
-{
- lnode_t *node, *next, *nil;
-
- next = nil = list_nil(list);
- node = list_first_priv(list);
-
- if (node != nil)
- next = lnode_next(node);
-
- for (; next != nil; node = next, next = lnode_next(next)) {
- if (compare(lnode_get(node), lnode_get(next)) > 0)
- return 0;
- }
-
- return 1;
-}
-
-/*
- * Get rid of macro functions definitions so they don't interfere
- * with the actual definitions
- */
-
-#undef list_isempty
-#undef list_isfull
-#undef lnode_pool_isempty
-#undef list_append
-#undef list_prepend
-#undef list_first
-#undef list_last
-#undef list_next
-#undef list_prev
-#undef list_count
-#undef list_del_first
-#undef list_del_last
-#undef lnode_put
-#undef lnode_get
-
-/*
- * Return 1 if the list is empty, 0 otherwise
- */
-
-int list_isempty(list_t *list)
-{
- return list->nodecount == 0;
-}
-
-/*
- * Return 1 if the list is full, 0 otherwise
- * Permitted only on bounded lists.
- */
-
-int list_isfull(list_t *list)
-{
- return list->nodecount == list->maxcount;
-}
-
-/*
- * Check if the node pool is empty.
- */
-
-int lnode_pool_isempty(lnodepool_t *pool)
-{
- return (pool->fre == NULL);
-}
-
-/*
- * Add the given node at the end of the list
- */
-
-void list_append(list_t *list, lnode_t *node)
-{
- list_ins_before(list, node, &list->nilnode);
-}
-
-/*
- * Add the given node at the beginning of the list.
- */
-
-void list_prepend(list_t *list, lnode_t *node)
-{
- list_ins_after(list, node, &list->nilnode);
-}
-
-/*
- * Retrieve the first node of the list
- */
-
-lnode_t *list_first(list_t *list)
-{
- if (list->nilnode.next == &list->nilnode)
- return NULL;
- return list->nilnode.next;
-}
-
-/*
- * Retrieve the last node of the list
- */
-
-lnode_t *list_last(list_t *list)
-{
- if (list->nilnode.prev == &list->nilnode)
- return NULL;
- return list->nilnode.prev;
-}
-
-/*
- * Retrieve the count of nodes in the list
- */
-
-listcount_t list_count(list_t *list)
-{
- return list->nodecount;
-}
-
-/*
- * Remove the first node from the list and return it.
- */
-
-lnode_t *list_del_first(list_t *list)
-{
- return list_delete(list, list->nilnode.next);
-}
-
-/*
- * Remove the last node from the list and return it.
- */
-
-lnode_t *list_del_last(list_t *list)
-{
- return list_delete(list, list->nilnode.prev);
-}
-
-
-/*
- * Associate a data item with the given node.
- */
-
-void lnode_put(lnode_t *lnode, void *data)
-{
- lnode->data = data;
-}
-
-/*
- * Retrieve the data item associated with the node.
- */
-
-void *lnode_get(lnode_t *lnode)
-{
- return lnode->data;
-}
-
-/*
- * Retrieve the node's successor. If there is no successor,
- * NULL is returned.
- */
-
-lnode_t *list_next(list_t *list, lnode_t *lnode)
-{
- assert (list_contains(list, lnode));
-
- if (lnode->next == list_nil(list))
- return NULL;
- return lnode->next;
-}
-
-/*
- * Retrieve the node's predecessor. See comment for lnode_next().
- */
-
-lnode_t *list_prev(list_t *list, lnode_t *lnode)
-{
- assert (list_contains(list, lnode));
-
- if (lnode->prev == list_nil(list))
- return NULL;
- return lnode->prev;
-}
-
-/*
- * Return 1 if the lnode is in some list, otherwise return 0.
- */
-
-int lnode_is_in_a_list(lnode_t *lnode)
-{
- return (lnode->next != NULL || lnode->prev != NULL);
-}
-
-
-int list_verify(list_t *list)
-{
- lnode_t *node = list_first_priv(list), *nil = list_nil(list);
- listcount_t count = list_count(list);
-
- if (node->prev != nil)
- return 0;
-
- if (count > list->maxcount)
- return 0;
-
- while (node != nil && count--) {
- if (node->next->prev != node)
- return 0;
- node = node->next;
- }
-
- if (count != 0 || node != nil)
- return 0;
-
- return 1;
-}
diff --git a/kmer/libutil/kazlib/list.h b/kmer/libutil/kazlib/list.h
deleted file mode 100644
index 97abc2f..0000000
--- a/kmer/libutil/kazlib/list.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * List Abstract Data Type
- * Copyright (C) 1997 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#ifndef LIST_H
-#define LIST_H
-
-#include <limits.h>
-
-#ifdef KAZLIB_SIDEEFFECT_DEBUG
-#include "sfx.h"
-#define LIST_SFX_CHECK(E) SFX_CHECK(E)
-#else
-#define LIST_SFX_CHECK(E) (E)
-#endif
-
-/*
- * Blurb for inclusion into C++ translation units
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef unsigned long listcount_t;
-#define LISTCOUNT_T_MAX ULONG_MAX
-
-typedef struct lnode_t {
-#if defined(LIST_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- struct lnode_t *list_next;
- struct lnode_t *list_prev;
- void *list_data;
-#else
- int list_dummy;
-#endif
-} lnode_t;
-
-typedef struct lnodepool_t {
-#if defined(LIST_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- struct lnode_t *list_pool;
- struct lnode_t *list_free;
- listcount_t list_size;
-#else
- int list_dummy;
-#endif
-} lnodepool_t;
-
-typedef struct list_t {
-#if defined(LIST_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
- lnode_t list_nilnode;
- listcount_t list_nodecount;
- listcount_t list_maxcount;
-#else
- int list_dummy;
-#endif
-} list_t;
-
-lnode_t *lnode_create(void *);
-lnode_t *lnode_init(lnode_t *, void *);
-void lnode_destroy(lnode_t *);
-void lnode_put(lnode_t *, void *);
-void *lnode_get(lnode_t *);
-int lnode_is_in_a_list(lnode_t *);
-
-#if defined(LIST_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
-#define lnode_put(N, D) ((N)->list_data = (D))
-#define lnode_get(N) ((N)->list_data)
-#endif
-
-lnodepool_t *lnode_pool_init(lnodepool_t *, lnode_t *, listcount_t);
-lnodepool_t *lnode_pool_create(listcount_t);
-void lnode_pool_destroy(lnodepool_t *);
-lnode_t *lnode_borrow(lnodepool_t *, void *);
-void lnode_return(lnodepool_t *, lnode_t *);
-int lnode_pool_isempty(lnodepool_t *);
-int lnode_pool_isfrom(lnodepool_t *, lnode_t *);
-
-list_t *list_init(list_t *, listcount_t);
-list_t *list_create(listcount_t);
-void list_destroy(list_t *);
-void list_destroy_nodes(list_t *);
-void list_return_nodes(list_t *, lnodepool_t *);
-
-listcount_t list_count(list_t *);
-int list_isempty(list_t *);
-int list_isfull(list_t *);
-int list_contains(list_t *, lnode_t *);
-
-void list_append(list_t *, lnode_t *);
-void list_prepend(list_t *, lnode_t *);
-void list_ins_before(list_t *, lnode_t *, lnode_t *);
-void list_ins_after(list_t *, lnode_t *, lnode_t *);
-
-lnode_t *list_first(list_t *);
-lnode_t *list_last(list_t *);
-lnode_t *list_next(list_t *, lnode_t *);
-lnode_t *list_prev(list_t *, lnode_t *);
-
-lnode_t *list_del_first(list_t *);
-lnode_t *list_del_last(list_t *);
-lnode_t *list_delete(list_t *, lnode_t *);
-
-void list_process(list_t *, void *, void (*)(list_t *, lnode_t *, void *));
-
-int list_verify(list_t *);
-
-#if defined(LIST_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
-#define lnode_pool_isempty(P) ((P)->list_free == 0)
-#define list_count(L) ((L)->list_nodecount)
-#define list_isempty(L) ((L)->list_nodecount == 0)
-#define list_isfull(L) (LIST_SFX_CHECK(L)->list_nodecount == (L)->list_maxcount)
-#define list_next(L, N) (LIST_SFX_CHECK(N)->list_next == &(L)->list_nilnode ? NULL : (N)->list_next)
-#define list_prev(L, N) (LIST_SFX_CHECK(N)->list_prev == &(L)->list_nilnode ? NULL : (N)->list_prev)
-#define list_first(L) list_next(LIST_SFX_CHECK(L), &(L)->list_nilnode)
-#define list_last(L) list_prev(LIST_SFX_CHECK(L), &(L)->list_nilnode)
-#endif
-
-#if defined(LIST_IMPLEMENTATION) || !defined(KAZLIB_OPAQUE_DEBUG)
-#define list_append(L, N) list_ins_before(LIST_SFX_CHECK(L), N, &(L)->list_nilnode)
-#define list_prepend(L, N) list_ins_after(LIST_SFX_CHECK(L), N, &(L)->list_nilnode)
-#define list_del_first(L) list_delete(LIST_SFX_CHECK(L), list_first(L))
-#define list_del_last(L) list_delete(LIST_SFX_CHECK(L), list_last(L))
-#endif
-
-/* destination list on the left, source on the right */
-
-void list_extract(list_t *, list_t *, lnode_t *, lnode_t *);
-void list_transfer(list_t *, list_t *, lnode_t *first);
-void list_merge(list_t *, list_t *, int (const void *, const void *));
-void list_sort(list_t *, int (const void *, const void *));
-lnode_t *list_find(list_t *, const void *, int (const void *, const void *));
-int list_is_sorted(list_t *, int (const void *, const void *));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/kmer/libutil/kazlib/sfx.c b/kmer/libutil/kazlib/sfx.c
deleted file mode 100644
index 829a53d..0000000
--- a/kmer/libutil/kazlib/sfx.c
+++ /dev/null
@@ -1,1138 +0,0 @@
-/*
- * SFX---A utility which tries to determine whether a given C expression
- * is free of side effects. This can be used for verifying that macros which
- * expand their arguments more than once are not being accidentally misused.
- *
- * Copyright (C) 1999 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#include <ctype.h>
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-#include "except.h"
-#include "sfx.h"
-#include "hash.h"
-#ifdef KAZLIB_POSIX_THREADS
-#include <pthread.h>
-#endif
-
-/*
- * Exceptions
- */
-
-#define SFX_EX 0x34DB9C4A
-#define SFX_SYNERR 1
-
-/*
- * Cache entry
- */
-
-typedef struct {
- hnode_t node;
- const char *expr;
- sfx_rating_t eff;
-} sfx_entry_t;
-
-/*
- * Parsing context structure
- */
-
-typedef struct {
- const unsigned char *start;
- const unsigned char *input;
- size_t size;
- sfx_rating_t eff;
-} context_t;
-
-/*
- * Declarator type: abstract, concrete or both
- */
-
-typedef enum {
- decl_abstract, decl_concrete, decl_both
-} decl_t;
-
-static void init_context(context_t *ctx, const unsigned char *expr)
-{
- ctx->input = ctx->start = expr;
- ctx->size = strlen((const char *) expr) + 1;
- ctx->eff = sfx_none;
-}
-
-static void assign_context(context_t *copy, context_t *orig)
-{
- *copy = *orig;
-}
-
-static void set_effect(context_t *ctx, sfx_rating_t eff)
-{
- assert (eff == sfx_none || eff == sfx_potential || eff == sfx_certain);
-
- if (eff > ctx->eff)
- ctx->eff = eff;
-}
-
-static void reset_effect(context_t *ctx)
-{
- ctx->eff = sfx_none;
-}
-
-static sfx_rating_t get_effect(context_t *ctx)
-{
- return ctx->eff;
-}
-
-static int skip_ws(context_t *expr)
-{
- while (*expr->input != 0 && isspace(*expr->input))
- expr->input++;
-
- return (*expr->input == 0);
-}
-
-static int get_next(context_t *expr)
-{
- int ret = *expr->input;
- if (ret)
- expr->input++;
- return ret;
-}
-
-static int get_next_skip_ws(context_t *expr)
-{
- if (!skip_ws(expr))
- return *expr->input++;
- return 0;
-}
-
-static const unsigned char *get_ptr(context_t *expr)
-{
- return expr->input;
-}
-
-static void skip_n(context_t *ctx, size_t n)
-{
- assert ((size_t) (ctx->input - ctx->start) <= ctx->size - n);
- ctx->input += n;
-}
-
-static void put_back(context_t *expr, int ch)
-{
- if (ch)
- expr->input--;
-}
-
-static int peek_next(context_t *expr)
-{
- return *expr->input;
-}
-
-static void syntax_error(void)
-{
- except_throw(SFX_EX, SFX_SYNERR, "syntax_error");
-}
-
-static void match_hard(context_t *expr, int match)
-{
- int ch = get_next(expr);
- if (ch != match)
- syntax_error();
-}
-
-static void chk_comma(context_t *);
-
-static void skip_ident(context_t *expr)
-{
- int ch = get_next(expr);
-
- if (!isalpha(ch) && ch != '_')
- syntax_error();
-
- do {
- ch = get_next(expr);
- } while (isalnum(ch) || ch == '_');
-
- put_back(expr, ch);
-}
-
-static void skip_constant(context_t *expr)
-{
- int ch = get_next(expr);
-
- assert (isdigit(ch) || ch == '.');
-
- do {
- ch = get_next(expr);
- if (ch == 'e' || ch == 'E') {
- ch = get_next(expr);
- if (ch == '+' || ch == '-') {
- ch = get_next(expr);
- if (!isdigit(ch))
- syntax_error();
- }
- }
- } while (ch != 0 && (isalnum(ch) || ch == '.'));
-
- put_back(expr, ch);
-}
-
-static void skip_strlit(context_t *expr)
-{
- int ch = get_next(expr);
-
- assert (ch == '"');
-
- do {
- ch = get_next(expr);
- if (ch == '\\') {
- get_next(expr);
- continue;
- }
- } while (ch != 0 && ch != '"');
-
- if (ch != '"')
- syntax_error();
-}
-
-static void skip_charlit(context_t *expr)
-{
- int ch = get_next(expr);
-
- assert (ch == '\'');
-
- do {
- ch = get_next(expr);
- if (ch == '\\') {
- get_next(expr);
- continue;
- }
- } while (ch != 0 && ch != '\'');
-
- if (ch != '\'')
- syntax_error();
-}
-
-static void chk_spec_qual_list(context_t *expr)
-{
- skip_ws(expr);
- skip_ident(expr);
-
- for (;;) {
- int ch;
-
- skip_ws(expr);
- ch = peek_next(expr);
-
- if (!isalpha(ch) && ch != '_')
- break;
-
- skip_ident(expr);
- }
-}
-
-static int speculate(void (*chk_func)(context_t *), context_t *expr, context_t *copy, int nextchar)
-{
- static const except_id_t catch[] = { { SFX_EX, XCEPT_CODE_ANY } };
- except_t *ex;
- volatile int result = 0;
- assign_context(copy, expr);
-
- except_try_push(catch, 1, &ex);
-
- if (ex == 0) {
- chk_func(copy);
- if (nextchar) {
- skip_ws(copy);
- match_hard(copy, nextchar);
- }
- result = 1;
- }
-
- except_try_pop();
-
- return result;
-}
-
-static void chk_pointer_opt(context_t *expr)
-{
- for (;;) {
- int ch = get_next_skip_ws(expr);
-
- if (ch != '*') {
- put_back(expr, ch);
- break;
- }
-
- skip_ws(expr);
-
- ch = peek_next(expr);
-
- if (ch == '*')
- continue;
- if (!isalpha(ch) && ch != '_')
- break;
-
- skip_ident(expr);
- }
-}
-
-static void chk_decl(context_t *, decl_t);
-
-static void chk_parm_decl(context_t *expr)
-{
- chk_spec_qual_list(expr);
- chk_decl(expr, decl_both);
-}
-
-static void chk_parm_type_list(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_parm_decl(expr);
-
- ch = get_next_skip_ws(expr);
-
- if (ch != ',') {
- put_back(expr, ch);
- break;
- }
-
- ch = get_next_skip_ws(expr);
-
- if (ch == '.') {
- match_hard(expr, '.');
- match_hard(expr, '.');
- break;
- }
-
- put_back(expr, ch);
- }
-}
-
-static void chk_conditional(context_t *);
-
-static void chk_direct_decl(context_t *expr, decl_t type)
-{
- for (;;) {
- int ch = get_next_skip_ws(expr);
-
- if (ch == '(') {
- skip_ws(expr);
- ch = peek_next(expr);
- if (ch == '*' || ch == '(' || ch == '[')
- chk_decl(expr, type);
- else if (isalpha(ch) || ch == '_')
- chk_parm_type_list(expr);
- match_hard(expr, ')');
- } else if (ch == '[') {
- skip_ws(expr);
- ch = peek_next(expr);
- if (ch != ']')
- chk_conditional(expr);
- match_hard(expr, ']');
- } else if ((type == decl_concrete || type == decl_both) && (isalpha(ch) || ch == '_')) {
- put_back(expr, ch);
- skip_ident(expr);
- break;
- } else {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_decl(context_t *expr, decl_t type)
-{
- int ch;
- chk_pointer_opt(expr);
- skip_ws(expr);
- ch = peek_next(expr);
- if (ch == '[' || ch == '(' || ((type == decl_concrete || type == decl_both) && (isalpha(ch) || ch == '_'))) {
- chk_direct_decl(expr, type);
- }
-}
-
-static void chk_typename(context_t *expr)
-{
- chk_spec_qual_list(expr);
- chk_decl(expr, decl_abstract);
-}
-
-static void chk_primary(context_t *expr)
-{
- int ch = peek_next(expr);
-
- if (ch == 'L') {
- get_next(expr);
- ch = peek_next(expr);
-
- if (ch == '\'') {
- skip_charlit(expr);
- return;
- }
-
- if (ch == '"') {
- skip_strlit(expr);
- return;
- }
-
- put_back(expr, 'L');
- ch = 'L';
- }
-
- if (isalpha(ch) || ch == '_') {
- skip_ident(expr);
- return;
- }
-
- if (isdigit(ch) || ch == '.') {
- skip_constant(expr);
- return;
- }
-
- if (ch == '(') {
- get_next(expr);
- chk_comma(expr);
- match_hard(expr, ')');
- return;
- }
-
- if (ch == '\'') {
- skip_charlit(expr);
- return;
- }
-
- if (ch == '"') {
- skip_strlit(expr);
- return;
- }
-
- syntax_error();
-}
-
-static void chk_postfix(context_t *expr)
-{
- chk_primary(expr);
-
- for (;;) {
- int ch = get_next_skip_ws(expr);
-
- switch (ch) {
- case '[':
- chk_comma(expr);
- skip_ws(expr);
- match_hard(expr, ']');
- continue;
- case '(':
- set_effect(expr, sfx_potential);
- ch = get_next_skip_ws(expr);
-
- if (ch != ')') {
- put_back(expr, ch);
- /* clever hack: parse non-empty argument list as comma expression */
- chk_comma(expr);
- ch = get_next_skip_ws(expr);
- }
-
- if (ch != ')')
- syntax_error();
-
- continue;
- case '.':
- skip_ws(expr);
- skip_ident(expr);
- continue;
- case '-':
- ch = get_next(expr);
-
- if (ch != '-' && ch != '>') {
- put_back(expr, ch);
- put_back(expr, '-');
- break;
- }
-
- if (ch == '>') {
- skip_ws(expr);
- skip_ident(expr);
- continue;
- }
-
- set_effect(expr, sfx_certain);
- continue;
- case '+':
- ch = get_next(expr);
- if (ch != '+') {
- put_back(expr, ch);
- put_back(expr, '+');
- break;
- }
-
- set_effect(expr, sfx_certain);
- continue;
- default:
- put_back(expr, ch);
- break;
- }
- break;
- }
-}
-
-static void chk_cast(context_t *);
-
-static void chk_unary(context_t *expr)
-{
- for (;;) {
- int nscan, ch = get_next_skip_ws(expr);
-
- switch (ch) {
- case '+':
- ch = get_next(expr);
- if (ch == '+')
- set_effect(expr, sfx_certain);
- else
- put_back(expr, ch);
- chk_cast(expr);
- break;
- case '-':
- ch = get_next(expr);
- if (ch == '-')
- set_effect(expr, sfx_certain);
- else
- put_back(expr, ch);
- chk_cast(expr);
- break;
- case '&': case '*': case '~': case '!':
- chk_cast(expr);
- break;
- case 's':
- put_back(expr, ch);
- nscan = 0;
- sscanf((const char *) get_ptr(expr), "sizeof%*1[^a-z0-9_]%n", &nscan);
-
- if (nscan == 7 || strcmp((const char *) get_ptr(expr), "sizeof") == 0) {
- sfx_rating_t eff = get_effect(expr);
-
- skip_n(expr, 6);
-
- ch = get_next_skip_ws(expr);
-
- if (ch == '(') {
- context_t comma, type;
- int iscomma = speculate(chk_comma, expr, &comma, ')');
- int istype = speculate(chk_typename, expr, &type, ')');
-
- if (!iscomma && !istype)
- syntax_error();
-
- if (iscomma) {
- context_t unary;
- put_back(expr, ch);
- if (speculate(chk_unary, expr, &unary, 0)) {
- assign_context(expr, &unary);
- istype = 0;
- }
- }
-
- if (istype)
- assign_context(expr, &type);
- } else {
- put_back(expr, ch);
- chk_unary(expr);
- }
-
- reset_effect(expr);
- set_effect(expr, eff);
- break;
- }
- chk_postfix(expr);
- break;
- default:
- put_back(expr, ch);
- chk_postfix(expr);
- break;
- }
-
- break;
- }
-}
-
-static void chk_cast(context_t *expr)
-{
- enum {
- parexpr, /* parenthesized expression */
- partype, /* parenthesized type name */
- parambig, /* ambiguity between paren expr and paren type name */
- unary, /* unary expression */
- plunary, /* unary expression with leading plus or minus */
- other /* none of the above, or even end of input */
- } curr = partype, old = partype, peek = partype;
-
- /* history for backtracking: two cast expression elements back */
- context_t old_expr = { 0 }, cur_expr = { 0 };
-
- for (;;) {
- context_t type, comma, unr;
- int ch = get_next_skip_ws(expr);
-
- /*
- * Determine what the next bit of input is: parenthesized type name,
- * expression, unary expression or what? Speculative parsing is used
- * to test several hypotheses. For example, something like
- * (X)(Y) ^ 1 is seen, it will be turned, by subsequent iterations of
- * this loop, into the codes: parambig, parambig, other.
- */
-
- if (ch == '(') {
- int istype = speculate(chk_typename, expr, &type, ')');
- int iscomma = speculate(chk_comma, expr, &comma, ')');
-
- switch (istype << 1 | iscomma) {
- case 0:
- ch = get_next_skip_ws(expr);
- if (ch == ')')
- peek = other; /* empty parentheses */
- else
- syntax_error();
- break;
- case 1:
- peek = parexpr;
- break;
- case 2:
- peek = partype;
- break;
- case 3:
- peek = parambig;
- break;
- }
- put_back(expr, ch);
- } else if (ch == 0) {
- peek = other;
- } else {
- put_back(expr, ch);
- if (speculate(chk_unary, expr, &unr, 0)) {
- peek = (ch == '+' || ch == '-' || ch == '*' || ch == '&') ? plunary : unary;
- } else {
- peek = other;
- }
- }
-
- /*
- * Okay, now we have an idea what is coming in the input. We make some
- * sensible decision based on this and the thing we parsed previously.
- * Either the parsing continues to grab more parenthesized things, or
- * some decision is made to parse out the suffix material sensibly and
- * terminate. Backtracking is used up to two elements back. For
- * example in the case of (X)(Y) ^ 1 (parambig, parambig, other) it's
- * necessary, upon seeing ^ 1 (other) to go back to second to last
- * ambigous parenthesized element (X) and terminate by parsing the
- * (X)(Y) as a postfix expression. It cannot be a cast, because ^1
- * isn't an expression. Unary expressions that start with + or -
- * create an interesting ambiguity. Is (X)(Y) + 1 the addition of 1 to
- * the result of the call to function X with parameter Y? Or is it the
- * unary expression + 1 cast to type Y and X? The safer assumption is
- * to go with the function call hypothesis, since that's the
- * interpretation that may have side effects.
- */
-
- switch (curr) {
- case parexpr: /* impossible cases */
- case other:
- case unary:
- case plunary:
- assert (0);
- syntax_error();
- /* notreached */
- case partype:
- switch (peek) {
- case parexpr: /* cast in front of parenthesized expression */
- chk_postfix(expr);
- return;
- case partype: /* compounding cast: keep looping */
- break;
- case parambig: /* type or expr: keep looping */
- break;
- case unary:
- case plunary:
- chk_unary(expr);
- return;
- case other: /* cast in front of non-expression! */
- syntax_error();
- /* notreached */
- }
- break;
- case parambig:
- switch (peek) {
- case parexpr: /* function call */
- assign_context(expr, &cur_expr);
- chk_postfix(expr);
- return;
- case partype: /* compounding cast: keep looping */
- break;
- case parambig: /* type or expr: keep looping */
- break;
- case unary:
- chk_unary(expr);
- return;
- case plunary: /* treat unary expr with + or - as additive */
- case other:
- if (old == parambig) {
- /* reparse two expression-like things in a row as call */
- assign_context(expr, &old_expr);
- chk_postfix(expr);
- return;
- }
- /* reparse expression followed by non-parenthesized
- stuff as postfix expression */
- assign_context(expr, &cur_expr);
- chk_postfix(expr);
- return; /* need more context */
- }
- break;
- }
-
- old = curr;
- curr = peek;
- assign_context(&old_expr, &cur_expr);
- assign_context(&cur_expr, expr);
- assign_context(expr, &type);
- }
-}
-
-static void chk_multiplicative(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_cast(expr);
- ch = get_next_skip_ws(expr);
-
- if ((ch != '*' && ch != '/' && ch != '%') || peek_next(expr) == '=') {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_additive(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_multiplicative(expr);
- ch = get_next_skip_ws(expr);
-
- if ((ch != '+' && ch != '-') || peek_next(expr) == '=') {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_shift(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_additive(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '<' && ch != '>') {
- put_back(expr, ch);
- break;
- }
-
- if (ch == '<' && peek_next(expr) != '<') {
- put_back(expr, ch);
- break;
- }
-
- if (ch == '>' && peek_next(expr) != '>') {
- put_back(expr, ch);
- break;
- }
-
- get_next(expr);
-
- if (peek_next(expr) == '=') {
- put_back(expr, ch);
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_relational(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_shift(expr);
- ch = get_next_skip_ws(expr);
-
-
- if (ch != '<' && ch != '>') {
- put_back(expr, ch);
- break;
- }
-
- if (ch == '<' && peek_next(expr) == '<') {
- put_back(expr, ch);
- break;
- }
-
- if (ch == '>' && peek_next(expr) == '>') {
- put_back(expr, ch);
- break;
- }
-
- if (peek_next(expr) == '=')
- get_next(expr);
- }
-}
-
-static void chk_equality(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_relational(expr);
- ch = get_next_skip_ws(expr);
-
- if ((ch != '!' && ch != '=') || peek_next(expr) != '=') {
- put_back(expr, ch);
- break;
- }
-
- match_hard(expr, '=');
- }
-}
-
-static void chk_and(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_equality(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '&' || peek_next(expr) == '&' || peek_next(expr) == '=') {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_exclusive_or(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_and(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '^' || peek_next(expr) == '=') {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_inclusive_or(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_exclusive_or(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '|' || peek_next(expr) == '|' || peek_next(expr) == '=') {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-static void chk_logical_and(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_inclusive_or(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '&' || peek_next(expr) != '&') {
- put_back(expr, ch);
- break;
- }
-
- match_hard(expr, '&');
- }
-}
-
-static void chk_logical_or(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_logical_and(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '|' || peek_next(expr) != '|') {
- put_back(expr, ch);
- break;
- }
-
- match_hard(expr, '|');
- }
-}
-
-static void chk_conditional(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_logical_or(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != '?') {
- put_back(expr, ch);
- break;
- }
-
- chk_comma(expr);
-
- skip_ws(expr);
- match_hard(expr, ':');
- }
-}
-
-static void chk_assignment(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_conditional(expr);
- ch = get_next_skip_ws(expr);
-
- switch (ch) {
- case '=':
- break;
- case '*': case '/': case '%':
- case '+': case '-': case '&':
- case '^': case '|':
- match_hard(expr, '=');
- break;
- case '<':
- match_hard(expr, '<');
- match_hard(expr, '=');
- break;
- case '>':
- match_hard(expr, '>');
- match_hard(expr, '=');
- break;
- case 0:
- default:
- put_back(expr, ch);
- return;
- }
- set_effect(expr, sfx_certain);
- }
-}
-
-static void chk_comma(context_t *expr)
-{
- for (;;) {
- int ch;
-
- chk_assignment(expr);
- ch = get_next_skip_ws(expr);
-
- if (ch != ',') {
- put_back(expr, ch);
- break;
- }
- }
-}
-
-/*
- * This function returns 1 if the expression is successfully parsed,
- * or 0 if there is a syntax error.
- *
- * The object pointed to by eff is set to indicate the side effect ranking of
- * the parsed expression: sfx_none, sfx_potential and sfx_certain. These
- * rankins mean, respectively, that there are no side effects, that there are
- * potential side effects, or that there certainly are side effects.
- */
-
-int sfx_determine(const char *expr, sfx_rating_t *eff)
-{
- static const except_id_t catch[] = { { SFX_EX, XCEPT_CODE_ANY } };
- except_t *ex;
- context_t ctx;
- volatile int retval = 1;
-
- if (!except_init())
- return 0;
-
- init_context(&ctx, (const unsigned char *) expr);
-
- except_try_push(catch, 1, &ex);
-
- if (ex == 0) {
- chk_comma(&ctx);
- skip_ws(&ctx);
- if (peek_next(&ctx) != 0)
- syntax_error();
- } else {
- /* exception caught */
- retval = 0;
- }
-
- except_try_pop();
-
- *eff = ctx.eff;
-
- except_deinit();
-
- return retval;
-}
-
-
-#ifdef KAZLIB_POSIX_THREADS
-
-static pthread_once_t cache_init;
-static pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-#define init_once(X, Y) pthread_once(X, Y)
-#define lock_cache() pthread_mutex_lock(&cache_mutex)
-#define unlock_cache() pthread_mutex_unlock(&cache_mutex)
-
-#else
-static int cache_init;
-
-static void init_once(int *once, void (*func)(void))
-{
- if (*once == 0) {
- func();
- *once = 1;
- }
-}
-
-#define lock_cache()
-#define unlock_cache()
-#endif
-
-static hash_t *cache;
-
-extern hash_t *hash_create(hashcount_t, hash_comp_t, hash_fun_t);
-
-static void init_cache(void)
-{
- cache = hash_create(HASHCOUNT_T_MAX, 0, 0);
-}
-
-static int lookup_cache(const char *expr, sfx_rating_t *rating)
-{
- hnode_t *cache_node;
- init_once(&cache_init, init_cache);
-
- lock_cache();
-
- cache_node = hash_lookup(cache, expr);
-
- unlock_cache();
-
- if (cache_node != 0) {
- sfx_entry_t *cache_entry = hnode_get(cache_node);
- *rating = cache_entry->eff;
- return 1;
- }
-
- return 0;
-}
-
-static int cache_result(const char *expr, sfx_rating_t rating)
-{
- int result = 0;
- hnode_t *cache_node;
-
- init_once(&cache_init, init_cache);
-
- if (cache == 0)
- goto bail;
-
- lock_cache();
-
- cache_node = hash_lookup(cache, expr);
-
- if (!cache_node) {
- sfx_entry_t *cache_entry = malloc(sizeof *cache_entry);
-
- if (cache_entry == 0)
- goto bail_unlock;
-
- hnode_init(&cache_entry->node, cache_entry);
- cache_entry->expr = expr;
- cache_entry->eff = rating;
- hash_insert(cache, &cache_entry->node, expr);
- } else {
- sfx_entry_t *cache_entry = hnode_get(cache_node);
- cache_entry->eff = rating;
- result = 1;
- }
-
- result = 1;
-
-
-bail_unlock:
- unlock_cache();
-
-bail:
- return result;
-}
-
-
-void sfx_check(const char *expr, const char *file, unsigned long line)
-{
- sfx_rating_t eff;
- int success = lookup_cache(expr, &eff);
-
- if (!success) {
- success = sfx_determine(expr, &eff);
- cache_result(expr, eff);
- }
-
- if (!success) {
- fprintf(stderr, "%s:%ld: syntax error in expression \"%s\"\n",
- file, line, expr);
- } else if (eff == sfx_potential) {
- fprintf(stderr, "%s:%ld: expression \"%s\" may have side effects\n",
- file, line, expr);
- } else if (eff == sfx_certain) {
- fprintf(stderr, "%s:%ld: expression \"%s\" has side effects\n",
- file, line, expr);
- } else {
- return;
- }
-}
-
-int sfx_declare(const char *expr, sfx_rating_t eff)
-{
- return cache_result(expr, eff);
-}
-
diff --git a/kmer/libutil/kazlib/sfx.h b/kmer/libutil/kazlib/sfx.h
deleted file mode 100644
index b2a485c..0000000
--- a/kmer/libutil/kazlib/sfx.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * SideChk---A utility which tries to determine whether a given C expression
- * is free of side effects. This can be used for verifying that macros which
- * expand their arguments more than once are not being accidentally misused.
- *
- * Copyright (C) 1999 Kaz Kylheku <kaz at ashi.footprints.net>
- *
- * Free Software License:
- *
- * All rights are reserved by the author, with the following exceptions:
- * Permission is granted to freely reproduce and distribute this software,
- * possibly in exchange for a fee, provided that this copyright notice appears
- * intact. Permission is also granted to adapt this software to produce
- * derivative works, as long as the modified versions carry this copyright
- * notice and additional notices stating that the work has been modified.
- * This source code may be translated into executable form and incorporated
- * into proprietary software; there is no requirement for such software to
- * contain a copyright notice related to this source.
- *
- */
-
-#ifndef SFX_H
-#define SFX_H
-
-#include <assert.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef enum {
- sfx_none, sfx_potential, sfx_certain
-} sfx_rating_t;
-
-int sfx_determine(const char *, sfx_rating_t *);
-int sfx_declare(const char *, sfx_rating_t);
-void sfx_check(const char *, const char *, unsigned long);
-
-#ifdef __cplusplus
-}
-#endif
-
-#define SFX_CHECK(E) (sfx_check(#E, __FILE__, __LINE__), (E))
-#define SFX_STRING(E) #E
-
-#endif
diff --git a/kmer/libutil/logMsg.H b/kmer/libutil/logMsg.H
deleted file mode 100644
index 049f10c..0000000
--- a/kmer/libutil/logMsg.H
+++ /dev/null
@@ -1,115 +0,0 @@
-#ifndef LOGMSG_H
-#define LOGMSG_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <unistd.h>
-#include <string.h>
-#include <stdarg.h>
-#include <errno.h>
-
-#include "util.h"
-
-class logMsg {
-public:
- logMsg(bool toScreen=false, u32bit r=8192) {
- _logLen = 0;
- _logMax = r;
- _log = new char [_logMax];
- _resize = r;
- _toScreenToo = toScreen;
- };
-
-
- ~logMsg() {
- delete [] _log;
- };
-
-
- void setResize(u32bit r) {
- _resize = r;
- };
-
-
- // Ensure that the string has at least 'moreSpace' available.
- //
- void resize(u32bit moreSpace) {
- if (_logLen + moreSpace < _logMax)
- return;
-
- _logMax += _logMax + moreSpace + 1;
- char *ll = new char [_logMax];
- memcpy(ll, _log, sizeof(char) * _logLen);
- delete [] _log;
- _log = ll;
- };
-
-
- // Add a message to the log, assume the message is less than 8192 bytes. Would be nice to parse
- // the fmt string (and any args) but that's a lot of work (and already done if you have
- // vsnprintf.
- //
- // It warns if you overwrote memory.
- //
- void add(char const *fmt, ...) {
- va_list ap;
-
- resize(_resize);
-
- if (_toScreenToo) {
- va_start(ap, fmt);
- vfprintf(stderr, fmt, ap);
- va_end(ap);
- }
-
- // Reinit the ap, since it seems to get 'used up' if _toScreenToo is set.
-
- va_start(ap, fmt);
- _logLen += vsprintf(_log + _logLen, fmt, ap);
- va_end(ap);
-
- if (_logLen > _logMax)
- fprintf(stderr,
- "logMsg::add()-- HEY! I wrote "u32bitFMT" bytes beyond the end of the buffer!\n"
- "logMsg::add()-- This program will probably crash soon....\n\n%s\n\n",
- _logLen - _logMax, _log);
- };
-
-
- // Dump the message to a file, taking care of errors.
- //
- void write(int file, char const *name=0L) {
- errno = 0;
- ::write(file, _log, sizeof(char) * _logLen);
- if (errno) {
- fprintf(stderr, "logMsg::write()-- Couldn't write to the log message file '%s': %s\n",
- name ? name : "(unknown)",
- strerror(errno));
- exit(1);
- }
- };
-
- void fwrite(FILE *file, char const *name=0L) {
- errno = 0;
- ::fwrite(_log, sizeof(char), _logLen, file);
- if (errno) {
- fprintf(stderr, "logMsg::fwrite()-- Couldn't write to the log message file '%s': %s\n",
- name ? name : "(unknown)",
- strerror(errno));
- exit(1);
- }
- };
-
-
-private:
- u32bit _logLen;
- u32bit _logMax;
- char *_log;
- u32bit _resize;
- bool _toScreenToo;
-};
-
-
-#endif // LOGMSG_H
diff --git a/kmer/libutil/md5.c b/kmer/libutil/md5.c
deleted file mode 100644
index 9bf827f..0000000
--- a/kmer/libutil/md5.c
+++ /dev/null
@@ -1,441 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "util.h"
-
-// The RSA MD5 implementation. Functions md5_* (at the end) are glue
-// to kmer libutil.
-
-// See RFC1321, "The MD5 Message-Digest Algorithm", R. Rivest.
-
-// Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
-// rights reserved.
-//
-// License to copy and use this software is granted provided that it
-// is identified as the "RSA Data Security, Inc. MD5 Message-Digest
-// Algorithm" in all material mentioning or referencing this software
-// or this function.
-//
-// License is also granted to make and use derivative works provided
-// that such works are identified as "derived from the RSA Data
-// Security, Inc. MD5 Message-Digest Algorithm" in all material
-// mentioning or referencing the derived work.
-//
-// RSA Data Security, Inc. makes no representations concerning either
-// the merchantability of this software or the suitability of this
-// software for any particular purpose. It is provided "as is"
-// without express or implied warranty of any kind.
-//
-// These notices must be retained in any copies of any part of this
-// documentation and/or software.
-
-
-typedef struct {
- u32bit state[4]; // state (ABCD)
- u32bit count[2]; // number of bits, modulo 2^64 (lsb first)
- unsigned char buffer[64]; // input buffer
-} MD5_CTX;
-
-
-static void MD5Init(MD5_CTX *);
-static void MD5Update(MD5_CTX *, unsigned char const *, size_t);
-static void MD5Final(unsigned char [16], MD5_CTX *);
-
-static void MD5Transform(u32bit [4], unsigned char const [64]);
-static void Encode(unsigned char *, u32bit *, unsigned int);
-static void Decode(u32bit *, unsigned char const *, unsigned int);
-
-// Constants for MD5Transform routine.
-#define S11 7
-#define S12 12
-#define S13 17
-#define S14 22
-#define S21 5
-#define S22 9
-#define S23 14
-#define S24 20
-#define S31 4
-#define S32 11
-#define S33 16
-#define S34 23
-#define S41 6
-#define S42 10
-#define S43 15
-#define S44 21
-
-static unsigned char PADDING[64] = {
- 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-};
-
-// F, G, H and I are basic MD5 functions.
-#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
-#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
-#define H(x, y, z) ((x) ^ (y) ^ (z))
-#define I(x, y, z) ((y) ^ ((x) | (~z)))
-
-// ROTATE_LEFT rotates x left n bits.
-#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
-
-// FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
-// Rotation is separate from addition to prevent recomputation.
-#define FF(a, b, c, d, x, s, ac) { \
- (a) += F ((b), (c), (d)) + (x) + (u32bit)(ac); \
- (a) = ROTATE_LEFT ((a), (s)); \
- (a) += (b); \
- }
-#define GG(a, b, c, d, x, s, ac) { \
- (a) += G ((b), (c), (d)) + (x) + (u32bit)(ac); \
- (a) = ROTATE_LEFT ((a), (s)); \
- (a) += (b); \
- }
-#define HH(a, b, c, d, x, s, ac) { \
- (a) += H ((b), (c), (d)) + (x) + (u32bit)(ac); \
- (a) = ROTATE_LEFT ((a), (s)); \
- (a) += (b); \
- }
-#define II(a, b, c, d, x, s, ac) { \
- (a) += I ((b), (c), (d)) + (x) + (u32bit)(ac); \
- (a) = ROTATE_LEFT ((a), (s)); \
- (a) += (b); \
- }
-
-// MD5 initialization. Begins an MD5 operation, writing a new context.
-//
-void MD5Init (MD5_CTX *context) {
- context->count[0] = context->count[1] = 0;
- // Load magic initialization constants.
- context->state[0] = 0x67452301;
- context->state[1] = 0xefcdab89;
- context->state[2] = 0x98badcfe;
- context->state[3] = 0x10325476;
-}
-
-// MD5 block update operation. Continues an MD5 message-digest
-// operation, processing another message block, and updating the
-// context.
-//
-void MD5Update (MD5_CTX *context, unsigned char const *input, size_t inputLen) {
- unsigned int i, index, partLen;
-
- // Compute number of bytes mod 64
- index = (unsigned int)((context->count[0] >> 3) & 0x3F);
-
- // Update number of bits
- if ((context->count[0] += ((u32bit)inputLen << 3))
- < ((u32bit)inputLen << 3))
- context->count[1]++;
- context->count[1] += ((u32bit)inputLen >> 29);
-
- partLen = 64 - index;
-
- // Transform as many times as possible.
- if (inputLen >= partLen) {
- memcpy(&context->buffer[index], input, partLen);
- MD5Transform(context->state, context->buffer);
-
- for (i = partLen; i + 63 < inputLen; i += 64)
- MD5Transform(context->state, &input[i]);
-
- index = 0;
- }
- else
- i = 0;
-
- // Buffer remaining input
- memcpy(&context->buffer[index], &input[i], inputLen-i);
-}
-
-// MD5 finalization. Ends an MD5 message-digest operation, writing the
-// the message digest and zeroizing the context.
-//
-void MD5Final (unsigned char digest[16], MD5_CTX *context) {
- unsigned char bits[8];
- unsigned int index, padLen;
-
- // Save number of bits
- Encode (bits, context->count, 8);
-
- // Pad out to 56 mod 64.
- index = (unsigned int)((context->count[0] >> 3) & 0x3f);
- padLen = (index < 56) ? (56 - index) : (120 - index);
- MD5Update (context, PADDING, padLen);
-
- // Append length (before padding)
- MD5Update (context, bits, 8);
- // Store state in digest
- Encode (digest, context->state, 16);
-
- // Zeroize sensitive information.
- memset(context, 0, sizeof(*context));
-}
-
-// MD5 basic transformation. Transforms state based on block.
-//
-static void MD5Transform(u32bit state[4], unsigned char const block[64]) {
- u32bit a = state[0], b = state[1], c = state[2], d = state[3], x[16];
-
- Decode(x, block, 64);
-
- // Round 1
- FF (a, b, c, d, x[ 0], S11, 0xd76aa478); // 1
- FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); // 2
- FF (c, d, a, b, x[ 2], S13, 0x242070db); // 3
- FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); // 4
- FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); // 5
- FF (d, a, b, c, x[ 5], S12, 0x4787c62a); // 6
- FF (c, d, a, b, x[ 6], S13, 0xa8304613); // 7
- FF (b, c, d, a, x[ 7], S14, 0xfd469501); // 8
- FF (a, b, c, d, x[ 8], S11, 0x698098d8); // 9
- FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); // 10
- FF (c, d, a, b, x[10], S13, 0xffff5bb1); // 11
- FF (b, c, d, a, x[11], S14, 0x895cd7be); // 12
- FF (a, b, c, d, x[12], S11, 0x6b901122); // 13
- FF (d, a, b, c, x[13], S12, 0xfd987193); // 14
- FF (c, d, a, b, x[14], S13, 0xa679438e); // 15
- FF (b, c, d, a, x[15], S14, 0x49b40821); // 16
-
- // Round 2
- GG (a, b, c, d, x[ 1], S21, 0xf61e2562); // 17
- GG (d, a, b, c, x[ 6], S22, 0xc040b340); // 18
- GG (c, d, a, b, x[11], S23, 0x265e5a51); // 19
- GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); // 20
- GG (a, b, c, d, x[ 5], S21, 0xd62f105d); // 21
- GG (d, a, b, c, x[10], S22, 0x2441453); // 22
- GG (c, d, a, b, x[15], S23, 0xd8a1e681); // 23
- GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); // 24
- GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); // 25
- GG (d, a, b, c, x[14], S22, 0xc33707d6); // 26
- GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); // 27
- GG (b, c, d, a, x[ 8], S24, 0x455a14ed); // 28
- GG (a, b, c, d, x[13], S21, 0xa9e3e905); // 29
- GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); // 30
- GG (c, d, a, b, x[ 7], S23, 0x676f02d9); // 31
- GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); // 32
-
- // Round 3
- HH (a, b, c, d, x[ 5], S31, 0xfffa3942); // 33
- HH (d, a, b, c, x[ 8], S32, 0x8771f681); // 34
- HH (c, d, a, b, x[11], S33, 0x6d9d6122); // 35
- HH (b, c, d, a, x[14], S34, 0xfde5380c); // 36
- HH (a, b, c, d, x[ 1], S31, 0xa4beea44); // 37
- HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); // 38
- HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); // 39
- HH (b, c, d, a, x[10], S34, 0xbebfbc70); // 40
- HH (a, b, c, d, x[13], S31, 0x289b7ec6); // 41
- HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); // 42
- HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); // 43
- HH (b, c, d, a, x[ 6], S34, 0x4881d05); // 44
- HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); // 45
- HH (d, a, b, c, x[12], S32, 0xe6db99e5); // 46
- HH (c, d, a, b, x[15], S33, 0x1fa27cf8); // 47
- HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); // 48
-
- // Round 4
- II (a, b, c, d, x[ 0], S41, 0xf4292244); // 49
- II (d, a, b, c, x[ 7], S42, 0x432aff97); // 50
- II (c, d, a, b, x[14], S43, 0xab9423a7); // 51
- II (b, c, d, a, x[ 5], S44, 0xfc93a039); // 52
- II (a, b, c, d, x[12], S41, 0x655b59c3); // 53
- II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); // 54
- II (c, d, a, b, x[10], S43, 0xffeff47d); // 55
- II (b, c, d, a, x[ 1], S44, 0x85845dd1); // 56
- II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); // 57
- II (d, a, b, c, x[15], S42, 0xfe2ce6e0); // 58
- II (c, d, a, b, x[ 6], S43, 0xa3014314); // 59
- II (b, c, d, a, x[13], S44, 0x4e0811a1); // 60
- II (a, b, c, d, x[ 4], S41, 0xf7537e82); // 61
- II (d, a, b, c, x[11], S42, 0xbd3af235); // 62
- II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); // 63
- II (b, c, d, a, x[ 9], S44, 0xeb86d391); // 64
-
- state[0] += a;
- state[1] += b;
- state[2] += c;
- state[3] += d;
-
- // Zeroize sensitive information.
- memset (x, 0, sizeof(x));
-}
-
-// Encodes input (u32bit) into output (unsigned char). Assumes len is
-// a multiple of 4.
-//
-static void Encode (unsigned char *output, u32bit *input, unsigned int len) {
- unsigned int i, j;
-
- for (i = 0, j = 0; j < len; i++, j += 4) {
- output[j] = (unsigned char)(input[i] & 0xff);
- output[j+1] = (unsigned char)((input[i] >> 8) & 0xff);
- output[j+2] = (unsigned char)((input[i] >> 16) & 0xff);
- output[j+3] = (unsigned char)((input[i] >> 24) & 0xff);
- }
-}
-
-// Decodes input (unsigned char) into output (u32bit). Assumes len is
-// a multiple of 4.
-//
-static void Decode (u32bit *output, unsigned char const *input, unsigned int len) {
- unsigned int i, j;
-
- for (i = 0, j = 0; j < len; i++, j += 4)
- output[i] = ((u32bit)input[j]) | (((u32bit)input[j+1]) << 8) |
- (((u32bit)input[j+2]) << 16) | (((u32bit)input[j+3]) << 24);
-}
-
-
-////////////////////////////////////////////////////////////////////////////////
-//
-// kmer glue functions
-//
-////////////////////////////////////////////////////////////////////////////////
-
-
-int
-md5_compare(void const *a, void const *b) {
- md5_s const *A = (md5_s const *)a;
- md5_s const *B = (md5_s const *)b;
-
- if (A->a < B->a) return(-1);
- if (A->a > B->a) return(1);
- if (A->b < B->b) return(-1);
- if (A->b > B->b) return(1);
- return(0);
-}
-
-static const char *md5_letters = "0123456789abcdef";
-
-char*
-md5_toascii(md5_s *m, char *s) {
- int i;
- for (i=0; i<16; i++) {
- s[15-i ] = md5_letters[(m->a >> 4*i) & 0x0f];
- s[15-i+16] = md5_letters[(m->b >> 4*i) & 0x0f];
- }
- s[32] = 0;
-
- return(s);
-}
-
-md5_s*
-md5_string(md5_s *m, char *s, u32bit l) {
- MD5_CTX ctx;
- unsigned char dig[16];
- int i = 0;
-
- if (m == NULL) {
- errno = 0;
- m = (md5_s *)malloc(sizeof(md5_s));
- if (errno) {
- fprintf(stderr, "md5_string()-- Can't allocate a md5_s.\n%s\n", strerror(errno));
- exit(1);
- }
- }
-
- MD5Init(&ctx);
- MD5Update(&ctx, (unsigned char*)s, l);
- MD5Final(dig, &ctx);
-
- m->a = dig[0];
- while (i<8) {
- m->a <<= 8;
- m->a |= dig[i++];
- }
-
- m->b = dig[i++];
- while (i<16) {
- m->b <<= 8;
- m->b |= dig[i++];
- }
-
- return(m);
-}
-
-static
-md5_increment_s*
-md5_increment_initialize(void) {
- md5_increment_s *m;
-
- errno = 0;
- m = (md5_increment_s *)malloc(sizeof(md5_increment_s));
- if (errno) {
- fprintf(stderr, "md5_increment_*()-- Can't allocate a md5_increment_s.\n%s\n", strerror(errno));
- exit(1);
- }
-
- m->context = (MD5_CTX *)malloc(sizeof(MD5_CTX));
- if (errno) {
- fprintf(stderr, "md5_increment_*()-- Can't allocate a md5 context.\n%s\n", strerror(errno));
- exit(1);
- }
- MD5Init((MD5_CTX *)m->context);
-
- m->bufferPos = 0;
-
- return(m);
-}
-
-md5_increment_s*
-md5_increment_char(md5_increment_s *m, char s) {
-
- if (m == NULL)
- m = md5_increment_initialize();
-
- m->buffer[m->bufferPos++] = s;
-
- if (m->bufferPos == MD5_BUFFER_SIZE) {
- MD5Update((MD5_CTX *)m->context, m->buffer, m->bufferPos);
- m->bufferPos = 0;
- }
-
- return(m);
-}
-
-md5_increment_s*
-md5_increment_block(md5_increment_s *m, char *s, u32bit l) {
-
- if (m == NULL)
- m = md5_increment_initialize();
-
- MD5Update((MD5_CTX *)m->context, (unsigned char*)s, l);
-
- return(m);
-}
-
-void
-md5_increment_finalize(md5_increment_s *m) {
- MD5_CTX *ctx = (MD5_CTX *)m->context;
- unsigned char dig[16];
- int i = 0;
-
- if (m->bufferPos > 0) {
- MD5Update((MD5_CTX *)m->context, m->buffer, m->bufferPos);
- m->bufferPos = 0;
- }
-
- MD5Final(dig, ctx);
-
- m->a = dig[0];
- while (i<8) {
- m->a <<= 8;
- m->a |= dig[i++];
- }
-
- m->b = dig[i++];
- while (i<16) {
- m->b <<= 8;
- m->b |= dig[i++];
- }
-
- m->context = 0L;
-
- free(ctx);
-}
-
-void
-md5_increment_destroy(md5_increment_s *m) {
- free(m);
-}
diff --git a/kmer/libutil/mt19937ar/Make.include b/kmer/libutil/mt19937ar/Make.include
deleted file mode 100644
index a9ead6d..0000000
--- a/kmer/libutil/mt19937ar/Make.include
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- makefile -*-
-
-LIBUTL/ :=$(realpath $/../../libutil/)/
-MTDIR/ :=${LIBUTL/}mt19937ar/
-
-$/.C_EXES := $/mt19937ar-test
-$/.C_SRCS := $/mt19937ar.c $/test.c
-$/.C_LIBS := $/libmt19937ar.a
-
-$/.CLEAN := $/*.o $/test.c $/diffs
-$/.REAL-CLEAN := $/*.o $/test.c $/diffs $/mt19937ar-test
-
-$/libmt19937ar.a: $/mt19937ar.o $/test.o
-
-$/mt19937ar-test: $/mt19937ar.o $/mt19937ar-test.o
-
-$/test.c: $/mt19937ar-test
- ${MTDIR/}mt19937ar-test | diff - ${MTDIR/}mt19937ar.out > ${MTDIR/}diffs 2>&1
- if test -s ${MTDIR/}diffs ; then echo 'MT19937: TEST FAILED'; else echo 'MT19937: Test Passed'; fi
- touch ${MTDIR/}test.c
- ${MTDIR/}mt19937ar-test | diff - ${MTDIR/}mt19937ar.out
-
-#$(eval $/%.d $/%.o: CFLAGS+= -I..)
diff --git a/kmer/libutil/mt19937ar/mt19937ar-test.c b/kmer/libutil/mt19937ar/mt19937ar-test.c
deleted file mode 100644
index 98bf30c..0000000
--- a/kmer/libutil/mt19937ar/mt19937ar-test.c
+++ /dev/null
@@ -1,38 +0,0 @@
-#include "mt19937ar.h"
-
-// The MD5 checksum of the correct output is
-// cb33e6acc162cbe20f7fcac26adddd02
-// and it is 22465 bytes long.
-//
-// but we cannot use md5, as it's in libbri, and
-// so is this...
-
-int main(void) {
- int i;
- u32bit init[4] = {0x123, 0x234, 0x345, 0x456};
- u32bit length = 4;
- mt_s *ctx = mtInitArray(init, length);
-
- printf("1000 outputs of genrand_int32()\n");
-
- for (i=0; i<1000; i++) {
- printf(u32bitFMTW(10)" ", mtRandom32(ctx));
- if (i%5==4) printf("\n");
- }
-
- printf("\n1000 outputs of genrand_real2()\n");
-
- for (i=0; i<1000; i++) {
- printf("%10.8f ", mtRandomRealOpen(ctx));
- if (i%5==4) printf("\n");
- }
-
-
-
- for (i=0; i<999; i++) {
- printf(u64bitHEX" ", mtRandom64(ctx));
- if (i%3==2) printf("\n");
- }
-
- return 0;
-}
diff --git a/kmer/libutil/mt19937ar/mt19937ar.c b/kmer/libutil/mt19937ar/mt19937ar.c
deleted file mode 100644
index fb230ed..0000000
--- a/kmer/libutil/mt19937ar/mt19937ar.c
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- A C-program for MT19937, with initialization improved 2002/1/26.
- Coded by Takuji Nishimura and Makoto Matsumoto.
-
- Before using, initialize the state by using init_genrand(seed)
- or init_by_array(init_key, key_length).
-
- Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- 1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. The names of its contributors may not be used to endorse or promote
- products derived from this software without specific prior written
- permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
- Any feedback is very welcome.
- http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html
- email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space)
-*/
-
-#include "mt19937ar.h"
-
-#include <stdlib.h>
-#include <math.h>
-
-
-// Buried in genrand_in32 was this:
-// if init_genrand() has not been called,
-// a default initial seed is used
-//
-// if (ctx->mti == N+1)
-// init_genrand(5489UL);
-//
-// But we don't need that anymore, as we require for
-// thread-safety that init_genrand be called.
-
-
-
-
-// initialize with a single seed
-mt_s*
-mtInit(u32bit s) {
- mt_s *ctx = (mt_s *)malloc(sizeof(mt_s));
- if (ctx == NULL)
- return(NULL);
-
- ctx->mt[0] = s;
-
- // See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier.
- // In the previous versions, MSBs of the seed affect
- // only MSBs of the array mt[].
- // 2002/01/09 modified by Makoto Matsumoto
-
- for (ctx->mti=1; ctx->mti<MT_N; ctx->mti++)
- ctx->mt[ctx->mti] = (1812433253UL * (ctx->mt[ctx->mti-1] ^ (ctx->mt[ctx->mti-1] >> 30)) + ctx->mti);
-
- ctx->mag01[0] = u32bitZERO;
- ctx->mag01[1] = MT_MATRIX_A;
-
- return(ctx);
-}
-
-
-
-
-/* initialize by an array with array-length */
-/* init_key is the array for initializing keys */
-/* key_length is its length */
-/* slight change for C++, 2004/2/26 */
-mt_s*
-mtInitArray(u32bit *init_key, u32bit key_length) {
-
- mt_s *ctx = mtInit(19650218UL);
- int i = 1;
- int j = 0;
- int k = (MT_N > key_length ? MT_N : key_length);
-
- for (; k; k--) {
- ctx->mt[i] = (ctx->mt[i] ^ ((ctx->mt[i-1] ^ (ctx->mt[i-1] >> 30)) * 1664525UL)) + init_key[j] + j; /* non linear */
- i++;
- j++;
- if (i >= MT_N) {
- ctx->mt[0] = ctx->mt[MT_N-1];
- i=1;
- }
- if (j >= key_length)
- j=0;
- }
- for (k=MT_N-1; k; k--) {
- ctx->mt[i] = (ctx->mt[i] ^ ((ctx->mt[i-1] ^ (ctx->mt[i-1] >> 30)) * 1566083941UL)) - i; /* non linear */
- i++;
- if (i>=MT_N) {
- ctx->mt[0] = ctx->mt[MT_N-1];
- i=1;
- }
- }
-
- ctx->mt[0] = 0x80000000UL; /* MSB is 1; assuring non-zero initial array */
- return(ctx);
-}
-
-
-
-/* generates a random number on [0,0xffffffff]-interval */
-u32bit
-mtRandom32(mt_s *ctx) {
- u32bit y;
-
- // generate MT_N words at one time
- //
- if (ctx->mti >= MT_N) {
- int kk;
-
- for (kk=0; kk < MT_N - MT_M; kk++) {
- y = (ctx->mt[kk] & MT_UPPER_MASK) | (ctx->mt[kk+1] & MT_LOWER_MASK);
- ctx->mt[kk] = ctx->mt[kk + MT_M] ^ (y >> 1) ^ ctx->mag01[y & u32bitONE];
- }
- for (; kk < MT_N-1; kk++) {
- y = (ctx->mt[kk] & MT_UPPER_MASK) | (ctx->mt[kk + 1] & MT_LOWER_MASK);
- ctx->mt[kk] = ctx->mt[kk + (MT_M - MT_N)] ^ (y >> 1) ^ ctx->mag01[y & u32bitONE];
- }
- y = (ctx->mt[MT_N-1] & MT_UPPER_MASK) | (ctx->mt[0] & MT_LOWER_MASK);
- ctx->mt[MT_N-1] = ctx->mt[MT_M-1] ^ (y >> 1) ^ ctx->mag01[y & u32bitONE];
-
- ctx->mti = 0;
- }
-
- y = ctx->mt[ctx->mti++];
-
- /* Tempering */
- y ^= (y >> 11);
- y ^= (y << 7) & 0x9d2c5680UL;
- y ^= (y << 15) & 0xefc60000UL;
- y ^= (y >> 18);
-
- return y;
-}
-
-
-// generates a random number on gaussian distribution with 0 median and 1 std.dev.
-double
-mtRandomGaussian(mt_s *mt) {
- double x1=0, x2=0, w=0, y1=0, y2=0;
-
- // from http://www.taygeta.com/random/gaussian.html
- //
- // supposedly equivalent to
- //
- // y1 = sqrt(-2*ln(x1)) cos(2*pi*x2)
- // y2 = sqrt(-2*ln(x1)) sin(2*pi*x2)
- //
- // but stable when x1 close to zero
-
- do {
- x1 = 2.0 * mtRandomRealClosed(mt) - 1.0;
- x2 = 2.0 * mtRandomRealClosed(mt) - 1.0;
- w = x1 * x1 + x2 * x2;
- } while (w >= 1.0);
-
- w = sqrt( (-2.0 * log(w)) / w);
-
- y1 = x1 * w;
- y2 = x2 * w;
-
- return(y1);
-}
diff --git a/kmer/libutil/mt19937ar/mt19937ar.h b/kmer/libutil/mt19937ar/mt19937ar.h
deleted file mode 100644
index 82a73dc..0000000
--- a/kmer/libutil/mt19937ar/mt19937ar.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#ifndef MT19937AR_H
-#define MT19937AR_H
-
-// Refactoring of
-//
-// A C-program for MT19937, with initialization improved 2002/1/26.
-// Coded by Takuji Nishimura and Makoto Matsumoto.
-//
-// to make it thread safe and (hopefully) more portable.
-//
-// 20040421, bpw
-
-// bri.h contains the function prototypes, but we hide the structure and
-// implementation here.
-//
-#include "../util.h"
-
-/* Period parameters */
-#define MT_N 624
-#define MT_M 397
-#define MT_MATRIX_A 0x9908b0dfUL /* constant vector a */
-#define MT_UPPER_MASK 0x80000000UL /* most significant w-r bits */
-#define MT_LOWER_MASK 0x7fffffffUL /* least significant r bits */
-
-
-struct mtctx {
- // The array for the state vector
- //
- u32bit mt[MT_N];
-
- // The ordinal of the first uninitialized element --
- // mti = N+1 -> element N is uninitialized
- //
- u32bit mti;
-
- // Something
- // mag01[x] = x * MT_MATRIX_A for x=0,1
- //
- u32bit mag01[2];
-};
-
-// This is declared in util.h
-//
-//typedef struct mt mt_s;
-
-
-#endif // MT19937AR_H
diff --git a/kmer/libutil/mt19937ar/mt19937ar.out b/kmer/libutil/mt19937ar/mt19937ar.out
deleted file mode 100644
index 03834bd..0000000
--- a/kmer/libutil/mt19937ar/mt19937ar.out
+++ /dev/null
@@ -1,736 +0,0 @@
-1000 outputs of genrand_int32()
-1067595299 955945823 477289528 4107218783 4228976476
-3344332714 3355579695 227628506 810200273 2591290167
-2560260675 3242736208 646746669 1479517882 4245472273
-1143372638 3863670494 3221021970 1773610557 1138697238
-1421897700 1269916527 2859934041 1764463362 3874892047
-3965319921 72549643 2383988930 2600218693 3237492380
-2792901476 725331109 605841842 271258942 715137098
-3297999536 1322965544 4229579109 1395091102 3735697720
-2101727825 3730287744 2950434330 1661921839 2895579582
-2370511479 1004092106 2247096681 2111242379 3237345263
-4082424759 219785033 2454039889 3709582971 835606218
-2411949883 2735205030 756421180 2175209704 1873865952
-2762534237 4161807854 3351099340 181129879 3269891896
- 776029799 2218161979 3001745796 1866825872 2133627728
- 34862734 1191934573 3102311354 2916517763 1012402762
-2184831317 4257399449 2899497138 3818095062 3030756734
-1282161629 420003642 2326421477 2741455717 1278020671
-3744179621 271777016 2626330018 2560563991 3055977700
-4233527566 1228397661 3595579322 1077915006 2395931898
-1851927286 3013683506 1999971931 3006888962 1049781534
-1488758959 3491776230 104418065 2448267297 3075614115
-3872332600 891912190 3936547759 2269180963 2633455084
-1047636807 2604612377 2709305729 1952216715 207593580
-2849898034 670771757 2210471108 467711165 263046873
-3569667915 1042291111 3863517079 1464270005 2758321352
-3790799816 2301278724 3106281430 7974801 2792461636
- 555991332 621766759 1322453093 853629228 686962251
-1455120532 957753161 1802033300 1021534190 3486047311
-1902128914 3701138056 4176424663 1795608698 560858864
-3737752754 3141170998 1553553385 3367807274 711546358
-2475125503 262969859 251416325 2980076994 1806565895
- 969527843 3529327173 2736343040 2987196734 1649016367
-2206175811 3048174801 3662503553 3138851612 2660143804
-1663017612 1816683231 411916003 3887461314 2347044079
-1015311755 1203592432 2170947766 2569420716 813872093
-1105387678 1431142475 220570551 4243632715 4179591855
-2607469131 3090613241 282341803 1734241730 1391822177
-1001254810 827927915 1886687171 3935097347 2631788714
-3905163266 110554195 2447955646 3717202975 3304793075
-3739614479 3059127468 953919171 2590123714 1132511021
-3795593679 2788030429 982155079 3472349556 859942552
-2681007391 2299624053 647443547 233600422 608168955
-3689327453 1849778220 1608438222 3968158357 2692977776
-2851872572 246750393 3582818628 3329652309 4036366910
-1012970930 950780808 3959768744 2538550045 191422718
-2658142375 3276369011 2927737484 1234200027 1920815603
-3536074689 1535612501 2184142071 3276955054 428488088
-2378411984 4059769550 3913744741 2732139246 64369859
-3755670074 842839565 2819894466 2414718973 1010060670
-1839715346 2410311136 152774329 3485009480 4102101512
-2852724304 879944024 1785007662 2748284463 1354768064
-3267784736 2269127717 3001240761 3179796763 895723219
- 865924942 4291570937 89355264 1471026971 4114180745
-3201939751 2867476999 2460866060 3603874571 2238880432
-3308416168 2072246611 2755653839 3773737248 1709066580
-4282731467 2746170170 2832568330 433439009 3175778732
- 26248366 2551382801 183214346 3893339516 1928168445
-1337157619 3429096554 3275170900 1782047316 4264403756
-1876594403 4289659572 3223834894 1728705513 4068244734
-2867840287 1147798696 302879820 1730407747 1923824407
-1180597908 1569786639 198796327 560793173 2107345620
-2705990316 3448772106 3678374155 758635715 884524671
- 486356516 1774865603 3881226226 2635213607 1181121587
-1508809820 3178988241 1594193633 1235154121 326117244
-2304031425 937054774 2687415945 3192389340 2003740439
-1823766188 2759543402 10067710 1533252662 4132494984
- 82378136 420615890 3467563163 541562091 3535949864
-2277319197 3330822853 3215654174 4113831979 4204996991
-2162248333 3255093522 2219088909 2978279037 255818579
-2859348628 3097280311 2569721123 1861951120 2907080079
-2719467166 998319094 2521935127 2404125338 259456032
-2086860995 1839848496 1893547357 2527997525 1489393124
-2860855349 76448234 2264934035 744914583 2586791259
-1385380501 66529922 1819103258 1899300332 2098173828
-1793831094 276463159 360132945 4178212058 595015228
- 177071838 2800080290 1573557746 1548998935 378454223
-1460534296 1116274283 3112385063 3709761796 827999348
-3580042847 1913901014 614021289 4278528023 1905177404
- 45407939 3298183234 1184848810 3644926330 3923635459
-1627046213 3677876759 969772772 1160524753 1522441192
- 452369933 1527502551 832490847 1003299676 1071381111
-2891255476 973747308 4086897108 1847554542 3895651598
-2227820339 1621250941 2881344691 3583565821 3510404498
- 849362119 862871471 797858058 2867774932 2821282612
-3272403146 3997979905 209178708 1805135652 6783381
-2823361423 792580494 4263749770 776439581 3798193823
-2853444094 2729507474 1071873341 1329010206 1289336450
-3327680758 2011491779 80157208 922428856 1158943220
-1667230961 2461022820 2608845159 387516115 3345351910
-1495629111 4098154157 3156649613 3525698599 4134908037
- 446713264 2137537399 3617403512 813966752 1157943946
-3734692965 1680301658 3180398473 3509854711 2228114612
-1008102291 486805123 863791847 3189125290 1050308116
-3777341526 4291726501 844061465 1347461791 2826481581
- 745465012 2055805750 4260209475 2386693097 2980646741
- 447229436 2077782664 1232942813 4023002732 1399011509
-3140569849 2579909222 3794857471 900758066 2887199683
-1720257997 3367494931 2668921229 955539029 3818726432
-1105704962 3889207255 2277369307 2746484505 1761846513
-2413916784 2685127085 4240257943 1166726899 4215215715
-3082092067 3960461946 1663304043 2087473241 4162589986
-2507310778 1579665506 767234210 970676017 492207530
-1441679602 1314785090 3262202570 3417091742 1561989210
-3011406780 1146609202 3262321040 1374872171 1634688712
-1280458888 2230023982 419323804 3262899800 39783310
-1641619040 1700368658 2207946628 2571300939 2424079766
- 780290914 2715195096 3390957695 163151474 2309534542
-1860018424 555755123 280320104 1604831083 2713022383
-1728987441 3639955502 623065489 3828630947 4275479050
-3516347383 2343951195 2430677756 635534992 3868699749
- 808442435 3070644069 4282166003 2093181383 2023555632
-1568662086 3422372620 4134522350 3016979543 3259320234
-2888030729 3185253876 4258779643 1267304371 1022517473
- 815943045 929020012 2995251018 3371283296 3608029049
-2018485115 122123397 2810669150 1411365618 1238391329
-1186786476 3155969091 2242941310 1765554882 279121160
-4279838515 1641578514 3796324015 13351065 103516986
-1609694427 551411743 2493771609 1316337047 3932650856
-4189700203 463397996 2937735066 1855616529 2626847990
- 55091862 3823351211 753448970 4045045500 1274127772
-1124182256 92039808 2126345552 425973257 386287896
-2589870191 1987762798 4084826973 2172456685 3366583455
-3602966653 2378803535 2901764433 3716929006 3710159000
-2653449155 3469742630 3096444476 3932564653 2595257433
- 318974657 3146202484 853571438 144400272 3768408841
- 782634401 2161109003 570039522 1886241521 14249488
-2230804228 1604941699 3928713335 3921942509 2155806892
- 134366254 430507376 1924011722 276713377 196481886
-3614810992 1610021185 1785757066 851346168 3761148643
-2918835642 3364422385 3012284466 3735958851 2643153892
-3778608231 1164289832 205853021 2876112231 3503398282
-3078397001 3472037921 1748894853 2740861475 316056182
-1660426908 168885906 956005527 3984354789 566521563
-1001109523 1216710575 2952284757 3834433081 3842608301
-2467352408 3974441264 3256601745 1409353924 1329904859
-2307560293 3125217879 3622920184 3832785684 3882365951
-2308537115 2659155028 1450441945 3532257603 3186324194
-1225603425 1124246549 175808705 3009142319 2796710159
-3651990107 160762750 1902254979 1698648476 1134980669
- 497144426 3302689335 4057485630 3603530763 4087252587
- 427812652 286876201 823134128 1627554964 3745564327
-2589226092 4202024494 62878473 3275585894 3987124064
-2791777159 1916869511 2585861905 1375038919 1403421920
- 60249114 3811870450 3021498009 2612993202 528933105
-2757361321 3341402964 2621861700 273128190 4015252178
-3094781002 1621621288 2337611177 1796718448 1258965619
-4241913140 2138560392 3022190223 4174180924 450094611
-3274724580 617150026 2704660665 1469700689 1341616587
- 356715071 1188789960 2278869135 1766569160 2795896635
- 57824704 2893496380 1235723989 1630694347 3927960522
- 428891364 1814070806 2287999787 4125941184 3968103889
-3548724050 1025597707 1404281500 2002212197 92429143
-2313943944 2403086080 3006180634 3561981764 1671860914
-1768520622 1803542985 844848113 3006139921 1410888995
-1157749833 2125704913 1789979528 1799263423 741157179
-2405862309 767040434 2655241390 3663420179 2172009096
-2511931187 1680542666 231857466 1154981000 157168255
-1454112128 3505872099 1929775046 2309422350 2143329496
-2960716902 407610648 2938108129 2581749599 538837155
-2342628867 430543915 740188568 1937713272 3315215132
-2085587024 4030765687 766054429 3517641839 689721775
-1294158986 1753287754 4202601348 1974852792 33459103
-3568087535 3144677435 1686130825 4134943013 3005738435
-3599293386 426570142 754104406 3660892564 1964545167
- 829466833 821587464 1746693036 1006492428 1595312919
-1256599985 1024482560 1897312280 2902903201 691790057
-1037515867 3176831208 1968401055 2173506824 1089055278
-1748401123 2941380082 968412354 1818753861 2973200866
-3875951774 1119354008 3988604139 1647155589 2232450826
-3486058011 3655784043 3759258462 847163678 1082052057
- 989516446 2871541755 3196311070 3929963078 658187585
-3664944641 2175149170 2203709147 2756014689 2456473919
-3890267390 1293787864 2830347984 3059280931 4158802520
-1561677400 2586570938 783570352 1355506163 31495586
-3789437343 3340549429 2092501630 896419368 671715824
-3530450081 3603554138 1055991716 3442308219 1499434728
-3130288473 3639507000 17769680 2259741420 487032199
-4227143402 3693771256 1880482820 3924810796 381462353
-4017855991 2452034943 2736680833 2209866385 2128986379
- 437874044 595759426 641721026 1636065708 3899136933
- 629879088 3591174506 351984326 2638783544 2348444281
-2341604660 2123933692 143443325 1525942256 364660499
- 599149312 939093251 1523003209 106601097 376589484
-1346282236 1297387043 764598052 3741218111 933457002
-1886424424 3219631016 525405256 3014235619 323149677
-2038881721 4100129043 2851715101 2984028078 1888574695
-2014194741 3515193880 4180573530 3461824363 2641995497
-3179230245 2902294983 2217320456 4040852155 1784656905
-3311906931 87498458 2752971818 2635474297 2831215366
-3682231106 2920043893 3772929704 2816374944 309949752
-2383758854 154870719 385111597 1191604312 1840700563
- 872191186 2925548701 1310412747 2102066999 1504727249
-3574298750 1191230036 3330575266 3180292097 3539347721
- 681369118 3305125752 3648233597 950049240 4173257693
-1760124957 512151405 681175196 580563018 1169662867
-4015033554 2687781101 699691603 2673494188 1137221356
- 123599888 472658308 1053598179 1012713758 3481064843
-3759461013 3981457956 3830587662 1877191791 3650996736
- 988064871 3515461600 4089077232 2225147448 1249609188
-2643151863 3896204135 2416995901 1397735321 3460025646
-
-1000 outputs of genrand_real2()
-0.76275443 0.99000644 0.98670464 0.10143112 0.27933125
-0.69867227 0.94218740 0.03427201 0.78842173 0.28180608
-0.92179002 0.20785655 0.54534773 0.69644020 0.38107718
-0.23978165 0.65286910 0.07514568 0.22765211 0.94872929
-0.74557914 0.62664415 0.54708246 0.90959343 0.42043116
-0.86334511 0.19189126 0.14718544 0.70259889 0.63426346
-0.77408121 0.04531601 0.04605807 0.88595519 0.69398270
-0.05377184 0.61711170 0.05565708 0.10133577 0.41500776
-0.91810699 0.22320679 0.23353705 0.92871862 0.98897234
-0.19786706 0.80558809 0.06961067 0.55840445 0.90479405
-0.63288060 0.95009721 0.54948447 0.20645042 0.45000959
-0.87050869 0.70806991 0.19406895 0.79286390 0.49332866
-0.78483914 0.75145146 0.12341941 0.42030252 0.16728160
-0.59906494 0.37575460 0.97815160 0.39815952 0.43595080
-0.04952478 0.33917805 0.76509902 0.61034321 0.90654701
-0.92915732 0.85365931 0.18812377 0.65913428 0.28814566
-0.59476081 0.27835931 0.60722542 0.68310435 0.69387186
-0.03699800 0.65897714 0.17527003 0.02889304 0.86777366
-0.12352068 0.91439461 0.32022990 0.44445731 0.34903686
-0.74639273 0.65918367 0.92492794 0.31872642 0.77749724
-0.85413832 0.76385624 0.32744211 0.91326300 0.27458185
-0.22190155 0.19865383 0.31227402 0.85321225 0.84243342
-0.78544200 0.71854080 0.92503892 0.82703064 0.88306297
-0.47284073 0.70059042 0.48003761 0.38671694 0.60465770
-0.41747204 0.47163243 0.72750808 0.65830223 0.10955369
-0.64215401 0.23456345 0.95944940 0.72822249 0.40888451
-0.69980355 0.26677428 0.57333635 0.39791582 0.85377858
-0.76962816 0.72004885 0.90903087 0.51376506 0.37732665
-0.12691640 0.71249738 0.81217908 0.37037313 0.32772374
-0.14238259 0.05614811 0.74363008 0.39773267 0.94859135
-0.31452454 0.11730313 0.62962618 0.33334237 0.45547255
-0.10089665 0.56550662 0.60539371 0.16027624 0.13245301
-0.60959939 0.04671662 0.99356286 0.57660859 0.40269560
-0.45274629 0.06699735 0.85064246 0.87742744 0.54508392
-0.87242982 0.29321385 0.67660627 0.68230715 0.79052073
-0.48592054 0.25186266 0.93769755 0.28565487 0.47219067
-0.99054882 0.13155240 0.47110470 0.98556600 0.84397623
-0.12875246 0.90953202 0.49129015 0.23792727 0.79481194
-0.44337770 0.96564297 0.67749118 0.55684872 0.27286897
-0.79538393 0.61965356 0.22487929 0.02226018 0.49248200
-0.42247006 0.91797788 0.99250134 0.23449967 0.52531508
-0.10246337 0.78685622 0.34310922 0.89892996 0.40454552
-0.68608407 0.30752487 0.83601319 0.54956031 0.63777550
-0.82199797 0.24890696 0.48801123 0.48661910 0.51223987
-0.32969635 0.31075073 0.21393155 0.73453207 0.15565705
-0.58584522 0.28976728 0.97621478 0.61498701 0.23891470
-0.28518540 0.46809591 0.18371914 0.37597910 0.13492176
-0.66849449 0.82811466 0.56240330 0.37548956 0.27562998
-0.27521910 0.74096121 0.77176757 0.13748143 0.99747138
-0.92504502 0.09175241 0.21389176 0.21766512 0.31183245
-0.23271221 0.21207367 0.57903312 0.77523344 0.13242613
-0.31037988 0.01204835 0.71652949 0.84487594 0.14982178
-0.57423142 0.45677888 0.48420169 0.53465428 0.52667473
-0.46880526 0.49849733 0.05670710 0.79022476 0.03872047
-0.21697212 0.20443086 0.28949326 0.81678186 0.87629474
-0.92297064 0.27373097 0.84625273 0.51505586 0.00582792
-0.33295971 0.91848412 0.92537226 0.91760033 0.07541125
-0.71745848 0.61158698 0.00941650 0.03135554 0.71527471
-0.24821915 0.63636652 0.86159918 0.26450229 0.60160194
-0.35557725 0.24477500 0.07186456 0.51757096 0.62120362
-0.97981062 0.69954667 0.21065616 0.13382753 0.27693186
-0.59644095 0.71500764 0.04110751 0.95730081 0.91600724
-0.47704678 0.26183479 0.34706971 0.07545431 0.29398385
-0.93236070 0.60486023 0.48015011 0.08870451 0.45548581
-0.91872718 0.38142712 0.10668643 0.01397541 0.04520355
-0.93822273 0.18011940 0.57577277 0.91427606 0.30911399
-0.95853475 0.23611214 0.69619891 0.69601980 0.76765372
-0.58515930 0.49479057 0.11288752 0.97187699 0.32095365
-0.57563608 0.40760618 0.78703383 0.43261152 0.90877651
-0.84686346 0.10599030 0.72872803 0.19315490 0.66152912
-0.10210518 0.06257876 0.47950688 0.47062066 0.72701157
-0.48915116 0.66110261 0.60170685 0.24516994 0.12726050
-0.03451185 0.90864994 0.83494878 0.94800035 0.91035206
-0.14480751 0.88458997 0.53498312 0.15963215 0.55378627
-0.35171349 0.28719791 0.09097957 0.00667896 0.32309622
-0.87561479 0.42534520 0.91748977 0.73908457 0.41793223
-0.99279792 0.87908370 0.28458072 0.59132853 0.98672190
-0.28547393 0.09452165 0.89910674 0.53681109 0.37931425
-0.62683489 0.56609740 0.24801549 0.52948179 0.98328855
-0.66403523 0.55523786 0.75886666 0.84784685 0.86829981
-0.71448906 0.84670080 0.43922919 0.20771016 0.64157936
-0.25664246 0.73055695 0.86395782 0.65852932 0.99061803
-0.40280575 0.39146298 0.07291005 0.97200603 0.20555729
-0.59616495 0.08138254 0.45796388 0.33681125 0.33989127
-0.18717090 0.53545811 0.60550838 0.86520709 0.34290701
-0.72743276 0.73023855 0.34195926 0.65019733 0.02765254
-0.72575740 0.32709576 0.03420866 0.26061893 0.56997511
-0.28439072 0.84422744 0.77637570 0.55982168 0.06720327
-0.58449067 0.71657369 0.15819609 0.58042821 0.07947911
-0.40193792 0.11376012 0.88762938 0.67532159 0.71223735
-0.27829114 0.04806073 0.21144026 0.58830274 0.04140071
-0.43215628 0.12952729 0.94668759 0.87391019 0.98382450
-0.27750768 0.90849647 0.90962737 0.59269720 0.96102026
-0.49544979 0.32007095 0.62585546 0.03119821 0.85953001
-0.22017528 0.05834068 0.80731217 0.53799961 0.74166948
-0.77426600 0.43938444 0.54862081 0.58575513 0.15886492
-0.73214332 0.11649057 0.77463977 0.85788827 0.17061997
-0.66838056 0.96076133 0.07949296 0.68521946 0.89986254
-0.05667410 0.12741385 0.83470977 0.63969104 0.46612929
-0.10200126 0.01194925 0.10476340 0.90285217 0.31221221
-0.32980614 0.46041971 0.52024973 0.05425470 0.28330912
-0.60426543 0.00598243 0.97244013 0.21135841 0.78561597
-0.78428734 0.63422849 0.32909934 0.44771136 0.27380750
-0.14966697 0.18156268 0.65686758 0.28726350 0.97074787
-0.63676171 0.96649494 0.24526295 0.08297372 0.54257548
-0.03166785 0.33735355 0.15946671 0.02102971 0.46228045
-0.11892296 0.33408336 0.29875681 0.29847692 0.73767569
-0.02080745 0.62980060 0.08082293 0.22993106 0.25031439
-0.87787525 0.45150053 0.13673441 0.63407612 0.97907688
-0.52241942 0.50580158 0.06273902 0.05270283 0.77031811
-0.05113352 0.24393329 0.75036441 0.37436336 0.22877652
-0.59975358 0.85707591 0.88691457 0.85547165 0.36641027
-0.58720133 0.45462835 0.09243817 0.32981586 0.07820411
-0.25421519 0.36004706 0.60092307 0.46192412 0.36758683
-0.98424170 0.08019934 0.68594024 0.45826386 0.29962317
-0.79365413 0.89231296 0.49478547 0.87645944 0.23590734
-0.28106737 0.75026285 0.08136314 0.79582424 0.76010628
-0.82792971 0.27947652 0.72482861 0.82191216 0.46171689
-0.79189752 0.96043686 0.51609668 0.88995725 0.28998963
-0.55191845 0.03934737 0.83033700 0.49553013 0.98009549
-0.19017594 0.98347750 0.33452066 0.87144372 0.72106301
-0.71272114 0.71465963 0.88361677 0.85571283 0.73782329
-0.20920458 0.34855153 0.46766817 0.02780062 0.74898344
-0.03680650 0.44866557 0.77426312 0.91025891 0.25195236
-0.87319953 0.63265037 0.25552148 0.27422476 0.95217406
-0.39281839 0.66441573 0.09158900 0.94515992 0.07800798
-0.02507888 0.39901462 0.17382573 0.12141278 0.85502334
-0.19902911 0.02160210 0.44460522 0.14688742 0.68020336
-0.71323733 0.60922473 0.95400380 0.99611159 0.90897777
-0.41073520 0.66206647 0.32064685 0.62805003 0.50677209
-0.52690101 0.87473387 0.73918362 0.39826974 0.43683919
-0.80459118 0.32422684 0.01958019 0.95319576 0.98326137
-0.83931735 0.69060863 0.33671416 0.68062550 0.65152380
-0.33392969 0.03451730 0.95227244 0.68200635 0.85074171
-0.64721009 0.51234433 0.73402047 0.00969637 0.93835057
-0.80803854 0.31485260 0.20089527 0.01323282 0.59933780
-0.31584602 0.20209563 0.33754800 0.68604181 0.24443049
-0.19952227 0.78162632 0.10336988 0.11360736 0.23536740
-0.23262256 0.67803776 0.48749791 0.74658435 0.92156640
-0.56706407 0.36683221 0.99157136 0.23421374 0.45183767
-0.91609720 0.85573315 0.37706276 0.77042618 0.30891908
-0.40709595 0.06944866 0.61342849 0.88817388 0.58734506
-0.98711323 0.14744128 0.63242656 0.87704136 0.68347125
-0.84446569 0.43265239 0.25146321 0.04130111 0.34259839
-0.92697368 0.40878778 0.56990338 0.76204273 0.19820348
-0.66314909 0.02482844 0.06669207 0.50205581 0.26084093
-0.65139159 0.41650223 0.09733904 0.56344203 0.62651696
-0.67332139 0.58037374 0.47258086 0.21010758 0.05713135
-0.89390629 0.10781246 0.32037450 0.07628388 0.34227964
-0.42190597 0.58201860 0.77363549 0.49595133 0.86031236
-0.83906769 0.81098161 0.26694195 0.14215941 0.88210306
-0.53634237 0.12090720 0.82480459 0.75930318 0.31847147
-0.92768077 0.01037616 0.56201727 0.88107122 0.35925856
-0.85860762 0.61109408 0.70408301 0.58434977 0.92192494
-0.62667915 0.75988365 0.06858761 0.36156496 0.58057195
-0.13636150 0.57719713 0.59340255 0.63530602 0.22976282
-0.71915530 0.41162531 0.63979565 0.09931342 0.79344045
-0.10893790 0.84450224 0.23122236 0.99485593 0.73637397
-0.17276368 0.13357764 0.74965804 0.64991737 0.61990341
-0.41523170 0.05878239 0.05687301 0.05497131 0.42868366
-0.42571090 0.25810502 0.89642955 0.30439758 0.39310223
-0.11357431 0.04288255 0.23397550 0.11200634 0.85621396
-0.89733974 0.37508865 0.42077265 0.68597384 0.72781399
-0.19296476 0.61699087 0.31667128 0.67756410 0.00177323
-0.05725176 0.79474693 0.18885238 0.06724856 0.68193156
-0.42202167 0.22082041 0.28554673 0.64995708 0.87851940
-0.29124547 0.61009521 0.87374537 0.05743712 0.69902994
-0.81925115 0.45653873 0.37236821 0.31118709 0.52734307
-0.39672836 0.38185294 0.30163915 0.17374510 0.04913278
-0.90404879 0.25742801 0.58266467 0.97663209 0.79823377
-0.36437958 0.15206043 0.26529938 0.22690047 0.05839021
-0.84721160 0.18622435 0.37809403 0.55706977 0.49828704
-0.47659049 0.24289680 0.88477595 0.07807463 0.56245739
-0.73490635 0.21099431 0.13164942 0.75840044 0.66877037
-0.28988183 0.44046090 0.24967434 0.80048356 0.26029740
-0.30416821 0.64151867 0.52067892 0.12880774 0.85465381
-0.02690525 0.19149288 0.49630295 0.79682619 0.43566145
-0.00288078 0.81484193 0.03763639 0.68529083 0.01339574
-0.38405386 0.30537067 0.22994703 0.44000045 0.27217985
-0.53831243 0.02870435 0.86282045 0.61831306 0.09164956
-0.25609707 0.07445781 0.72185784 0.90058883 0.30070608
-0.94476583 0.56822213 0.21933909 0.96772793 0.80063440
-0.26307906 0.31183306 0.16501252 0.55436179 0.68562285
-0.23829083 0.86511559 0.57868991 0.81888344 0.20126869
-0.93172350 0.66028129 0.21786948 0.78515828 0.10262106
-0.35390326 0.79303876 0.63427924 0.90479631 0.31024934
-0.60635447 0.56198079 0.63573813 0.91854197 0.99701497
-0.83085849 0.31692291 0.01925964 0.97446405 0.98751283
-0.60944293 0.13751018 0.69519957 0.68956636 0.56969015
-0.46440193 0.88341765 0.36754434 0.89223647 0.39786427
-0.85055280 0.12749961 0.79452122 0.89449784 0.14567830
-0.45716830 0.74822309 0.28200437 0.42546044 0.17464886
-0.68308746 0.65496587 0.52935411 0.12736159 0.61523955
-0.81590528 0.63107864 0.39786553 0.20102294 0.53292914
-0.75485590 0.59847044 0.32861691 0.12125866 0.58917183
-0.07638293 0.86845380 0.29192617 0.03989733 0.52180460
-0.32503407 0.64071852 0.69516575 0.74254998 0.54587026
-0.48713246 0.32920155 0.08719954 0.63497059 0.54328459
-0.64178757 0.45583809 0.70694291 0.85212760 0.86074305
-0.33163422 0.85739792 0.59908488 0.74566046 0.72157152
-0x67405e6c328fecdf 0x3c8b2c35482ec8c9 0x3250533bca1940c7
-0xf2d983e5b3262520 0xe5b759c591be1fda 0x8242a4458c0654ef
-0xb04d83e5cb5b6017 0xb6ad8ae702c9d964 0xbcf18ae96331a2da
-0x1cc1d152497d4674 0xc89cf1de59189442 0x398b33c171e4c16c
-0xe1ef8b20e9581f1f 0xbcf3922d01c3c4c3 0x3fb925af371e20d8
-0x3788696c8a091e68 0x98a8edcd8a199268 0x2b2bf18b86a1d357
-0x474017009e18d034 0x0f5914833849cde7 0x5f04574352379c0d
-0x8f5ca9b0d749b8d0 0x75b973eb6c039cde 0x69d4a24d0386aca4
-0xfe82fe22f8c3715d 0x007e2b70611c98e7 0xf45e29c72b9f1786
-0x7694fd07e82e529d 0x0d374894c5b55c9c 0xc8e6005052a38ac7
-0xdafb054cec6083d7 0x625a22c66cd3bf85 0xc0af2ff40f2b0074
-0xd6489630d188c4c3 0x7f034dbaf566f42b 0xfa47383d871e8dae
-0x1e9bc6524bbc99df 0xc97e66d2eef0793f 0x45760d27aade8dc5
-0xd9a5a1454582602e 0xb340cc9b522bb2b1 0xc449fec5c8359c3b
-0xdba9d20c2b809802 0xe414bdd2089cd4a9 0x714def09cabd0d3d
-0xf9755125bdca9539 0xa5d9bfb951aed29c 0x5c7e2d33c83ddf92
-0x8a59ea07850aa835 0x401f067db97b5427 0xefe4b17bb713b9c7
-0xee193cc8a0f16596 0x0c6b1f2a9ea778e5 0xbcacdc1567f07bef
-0x713ddb4b58672888 0xee8075374182b161 0x03ec2941a3da86d6
-0x675aebbee50e893c 0xd225931684eb5291 0xe477c127c0a105e7
-0xefcf3d05ef772a45 0xc4ef9941734f83b1 0x87e4942af85e11e8
-0xbf5fcadc377de765 0x079d9de562268b5f 0xea04faeba98f3e5c
-0xb930c6da6ddba5f7 0x478236b93821b9ba 0xa9cfc9be294b0ff7
-0x6079f977b05fb70e 0xe9e2dbe839ffb6e8 0xb3170798eddd9379
-0x0ef96e0ec793a92f 0x0524e13897de842d 0x465b56ac3b31cdda
-0x890bd07a90444c77 0xe234eff01af4dfd2 0xe84848232dbbfdd3
-0x3c4efcfca0456b81 0x670edacde7eeec34 0x2b36fde828c7b1ec
-0x9b6da65be00fdbd5 0xa3c761de25e1f4ba 0x7377fd171c85c139
-0x1b128eb2dad95dd2 0x3537b8e218bd4ec7 0xa0101eb1b3e29a86
-0x528f6b866eb20175 0x5473da9172d03fbd 0x9c77153ec299763c
-0x47ba00f901873acb 0x137df82e07c009b0 0x3c61915b065d0aa6
-0xe49e7299af8e6c5c 0x257f9436560d2208 0x78b7d974999c611a
-0x0d8adaad822c4965 0xa20c4d15c9573034 0x6f95460c6e9d822d
-0x2f853e287ea74d01 0xab53596fa5dc9c89 0x527171fc57868fc3
-0xab0a91e3a225f47f 0x5fe417b6624ce303 0x8ae42059c66c39dd
-0xb60de6ddff69c310 0xc83f24d03dbfe608 0xc5165efd988fd170
-0x12c3eb7ad4274929 0x1f8cee33476428d2 0x1ddea9364463c3ff
-0x1667501fcad4dedb 0x62c7365a9bb30ad9 0x8899b942b7573303
-0xc887f6d9377ee390 0x402af03474f37acc 0x9da72866ac0d1cc5
-0xa986abc55d6af4f4 0xe8a71cb1151dbf43 0x0ec056bba0bc3215
-0xda14a58e848fcf79 0x491f5bd20198aa9b 0xf42da0476fcd8832
-0x627c1873be8fb51e 0xa454d9b0317b861a 0x40fc96f80cd3bbc0
-0x9e2ff393daf483cb 0xfc758a65c59b083e 0xe859fd51d3a08556
-0x5efef07d19820724 0xe279ece74937813f 0x110eb92574a2084e
-0x3df42d82245bff2b 0x9ca473630575feb8 0x15c70084fb66d585
-0xbee7c870394194f7 0x9cc0a4f51f369867 0x6afbf2cb6ec861cf
-0xa5b91a04835cae3a 0x7ddc222f0834f7eb 0xa9dfb4a5b06e5b6b
-0xb5810c3cc04cd424 0xc868b4afb4e9dacc 0x42c02254bac2d4d8
-0xf0a9eece4466ef85 0x4871a4148a47ea38 0x12576ffdb6fdad9b
-0x0e9505d10fc16302 0x5631bf2c70323f0e 0x28b179d03e01feab
-0x9dc0dd12b00dac9b 0x8cf7134a0d2f5d0e 0xeb08c68420fa5985
-0x70c0a74d1d12661a 0xc66142602dd41863 0xb20965edb4a3adb9
-0x319530ddf3890df6 0x5b78a9a93cbc723c 0x4cb9b467b0b55068
-0x8542da469019ce95 0xc091429ae362663a 0x9ef9dc529dccab6e
-0x96da56d02c5e2eda 0xbd065629a65356b8 0x86ab31cf8072eca6
-0xc95dff79f3ee122c 0x9044db614b167618 0x2b53b3ada5a7095d
-0x039b2a4c565ecffa 0xebe357d7d10d5f2d 0x64b694ddcdffd60a
-0x2f1a1613ac6c9f3d 0xc79da221329d4ef9 0x2bdb226b6cd38813
-0xca9767b9d3a2b084 0x66f6512c1f7dc7fd 0x91a73f962a2f07e0
-0x885bbddc2ff3debf 0xbd67554077fd4c70 0x99013399c758bd15
-0x5cf5b0f828128900 0x210a832780dbd783 0x9e3a38661b53eda9
-0xaae6b882c58412c3 0x2082626df4ca895b 0x5cf5f76bb7b48682
-0x6c0cae2d715e434e 0xef16816391c305dc 0x87b979af394695e4
-0xcd2a272a1a805492 0x60f7e95280c303c4 0x321c98d44c5f4e05
-0xadf6dce5480a0aa2 0x72c9e5996c3339ad 0xbf1d29e6f0b41a0d
-0x274f3d2dd5f0c37e 0x914b57f6ef5243e9 0xed1817876768b412
-0x05b4432aa26c8866 0x87ca3f2c01af6814 0x03116c1030ac3ab8
-0xf5cacecb81781b7c 0x131f0c45877769e0 0x3eabeb0bd69e2872
-0x7b276bc3d7ecbd7f 0xccb9143a415b647d 0xe84d77e693e184c0
-0x780d77c885a12891 0xcbfefe34f9a928ce 0xd5c2487c43c47678
-0x2eb49b0cff9b2b0c 0xd31248757f950b80 0xac0f5e6b333b69f3
-0x3bc0db5d4cd64c13 0x6e7b83ef32960445 0xa503015d8f6deb6c
-0x5b8daa355e155964 0x2f734ee67c567191 0xedbf27328a640c45
-0xa7d67ea4920db6eb 0x581d6a00e15dd86f 0xc50a59f87c672f01
-0x6bbb37d607050b9c 0xbe1ed4bf962c3f9c 0xb9e18300cc6a0292
-0xbaf3a963ebf66bb1 0xcc4a11a37f3f042c 0x78c18be41d2173a5
-0xd624936215d43e75 0x123cf1fcbd9a720f 0xebd61ce176c98627
-0x78f872dffb1a12cc 0x58e45fdbe9434f31 0x1fb28683306e8c21
-0x994bdbccb69dbb36 0xca605fc87b2b873a 0xfdef6f060b8f4c19
-0xbf7b96a4d8039aa5 0xcd017cc08e0ef146 0x109f34ce10506ff9
-0x79a06caf3d6c91c8 0x2ed9f6c0e43a7a0a 0xda890a93647404d9
-0x8f495cdc9571b6db 0xbceda30aec45acf5 0xe2f8e38ec63c6eb3
-0x86637eb61e70775c 0x656a79f5182dcd27 0x0d6ac797d29627af
-0xcfa36f230d63e8f5 0x0fe19f20f9309ac4 0x069fb30d45ab4af4
-0xb5ba378ef4a205b2 0xe51b1280dc04e530 0x6efd94972a8be0a7
-0x334fc5b584a1bad7 0xbdd5a374520f4a9b 0x706b706b5bc348c8
-0x836dc7d17c04caac 0x356ddab9d49ddfe6 0xd159ed5840ddd0ae
-0xadac31b4fef8e091 0x8d89032b3845859f 0x3bd971e2a8cadd2c
-0x65304014021337bf 0xb7d974ac484afeaf 0xdbc7da0068d2f636
-0x16eb1156c1b04b4d 0x8616a5dea534ad0e 0x67f74216c5383c94
-0x66b2b1aa56ad4087 0x29c2484d55c3489d 0x150215c70d49f395
-0xf9d5c84babe4b95c 0x5190562945d45db4 0x422706f447c8cb26
-0xd836deaaa1c2de48 0x422d6a90be18aada 0x47be7c3ce1d2d478
-0xf67eca6f67bb3775 0x93b171e4234cfe76 0x284d0008372ef07c
-0x3698d967c82e6cf2 0xe2f6a86325a83871 0xa05b6bb8d5a8f19d
-0x8c3c8c0932b47315 0xcb17cef7ae59b502 0x03cfa20ca46fa2b2
-0xe6b973298c8e9cf9 0x3dc8b542ce244d7c 0x24e151f91a603c98
-0xbabc1fe6edf224d0 0x03057b07c7da7488 0x9893f09f3a05d9d4
-0x8d7c9ede4bbb4625 0x0a3e483d53be86ca 0xaeaa80fa118aa1be
-0x5db21e05baa52d1a 0x7e6646013c3c76bd 0x6e02586aab7c0b75
-0xc599811f3381e84e 0x8584e334f66cfd11 0x2ddab189180628de
-0x798e7628ad4a602b 0xfb4dd579277e2008 0x6600c85ce8ec6a2e
-0x0aa55f130589b854 0xe45ab335eac329a3 0xa5325407038e4e11
-0xcf996346624b521c 0x4d0274a34c9589bd 0x8d0cad0b87392d57
-0x2d9646a64bcd2d2f 0xad7a1ec0ae4a2151 0x3f376577014f8dca
-0x19deb4a855f350e3 0x4998df888b984424 0xceebb951fbc934ca
-0x506ef0f671ceb9c1 0xc5ef09726ea704bf 0x5189b998448a5e11
-0x675f92fbc1a4c11e 0x530c4265087a7c4a 0xe18e8069e8b707e1
-0x3c2779074e4a00a2 0x95370af830c9dc3a 0x416b062811f2175f
-0x1fa8fe3935e37c73 0xde540bcb853484b0 0xcb601ed072199738
-0x83c2ac01d2ca8f62 0x64f81e49fdcaf6a7 0xb7e5e765429cee4c
-0xcec327d993653b95 0xd21eb799eac60217 0x3d1f583dcea5cb99
-0x2f780a3c356d4f5d 0x0d25a447cc72a00c 0x333fcbb7b46e275c
-0xd30674469bf820aa 0x434434c3882a96aa 0x3c8b8d389fa5647b
-0xa1af5ecc309ff07c 0x39e85842caf91bcc 0x383eac8cbb8c7b53
-0xddc66b6b38b1f273 0x7b0e539b24202231 0xf65bd3d8c61484a8
-0x8ffa69be80f9b321 0x22b46595c09272e4 0x5393806eb788160e
-0x891be7c411c10b15 0x9278dccf0fe4813b 0x86b245936eaa77d7
-0x4a6b28b10796ab38 0x1ea2db227acc5ae6 0x7b54f85812e5900b
-0xe5f912551ddf84a8 0x0f4fea8cd744faf7 0x35aa34911651bab2
-0x3302457b0da9f478 0x7e8980bb045d7552 0xb17000de13ee8a17
-0x15eef5ca5c5209d4 0x776f92bce4b4bf98 0xb8176052fe1587e7
-0x9437869fefbbe7ea 0xeee20c003aea2c86 0xe2c48da4fdc5426a
-0x084429586e8a01ac 0x51156e0189a6be85 0x48fa8721bf77762c
-0x2b0b33a3c4fc2e60 0x57d068040d700132 0xdb1dbc6158eb9ea0
-0xc097e39e74ba26af 0x4a1c702bd0ab338b 0xc64fb41bb2e7242a
-0x397ce664cd1bd8b9 0x10998a8b432e89d5 0x6641eb4947f78de5
-0x90b623890c63938b 0xb70c700b9de39c7d 0x28bdd272e47f9e88
-0x9539798ec4e3ea34 0x3ef0763c6f6c3108 0x26ac5d6724e40a1b
-0x6195a673243088f6 0xdb5aba293e070ca4 0x98800aea4e5f0e4e
-0x82c4134187537bc8 0x4ba979d8911b8e0e 0x943e41ad3cd01884
-0x88e65d80db067bd7 0x7869b39765369333 0xaf50316f4c694b29
-0xc451835f04a402e0 0x60dcb558ee7e6e2b 0x65234e499ff85e2a
-0xe3dc8b2a11bd5edb 0x2a21e5d894243627 0x3ed8d69bd926c4cc
-0x15bf0f6060684012 0x50ce7ee65b3c64a6 0x11cf02e2af1452d3
-0xd8c84781d09f2c5c 0xd0bcaa5424f39d22 0xb251707659f54392
-0x2bce134f4cc2db80 0xac8d00764ecd9b1e 0xd39e3d7e7c50dcd9
-0x3bc9595caaa45365 0x4a8f4c6e9fb55bfe 0x3acabfc0bbaeab42
-0xe8dc0d62f62fce39 0x87db2dfde390bb6b 0x21733320e5683477
-0x382b5edf45b76468 0x0942a6f469259049 0x69d0104109910df2
-0xcd97e90a0b92882f 0xccbc893007f0ce9e 0xc7f15af4b343b733
-0x8452042dbf83c7b6 0x4cd332f6dcfcf88c 0x2af26987bfce6905
-0xe906963392926ca5 0xe497fc1e943fa9f2 0x98a7949ac41b7f42
-0x468f4c6bd9e09993 0xb7aa4263eacbfa1c 0x1046dafd21831081
-0xdce0a46e7f40131a 0x02fb82dfbc2bc81f 0x24fb0ae41a50b4ce
-0xb5f3c95c66a0eb55 0x4f6cb9ad702e23eb 0x53de1d08a184b063
-0x6ec2daf2a3b654ad 0x0e1dce6a1f2683d8 0xfde0cc4dc717cb77
-0x15ab25c3e644d3b3 0x4e4b23569f93d996 0xa081f0ade3ca6df5
-0x49427fdef4f200ee 0xcf635cbfae34ebe2 0xd898e8639bb74469
-0x4a86c97e8ee0654d 0x108f1bd0715a571f 0x47513cd35aecd66c
-0xb9c0324084cdcc91 0x53b651c7d391dd3c 0x3752b5626ec39e99
-0xb4b374a39bea99bb 0xa21b1a5d19d2a41c 0x8276063bed6548c5
-0x71bc6f337c67476c 0x9c40e276168f4a94 0x4b445b3a5789d7c2
-0xaf355b6b8f0c93cb 0xb4f15704ffb1c015 0x9aebd68bc4601ee9
-0x7274be877312d407 0x1a8eda0149fb3c1e 0xac96a19dbddb1f43
-0x289d606d06643c34 0x16dc1e37a3f591b5 0xe67f91eac997ba12
-0x545f62407cfe3487 0x9ad438a940a78009 0x5cbc0cff8ea73089
-0xe21a9f995af5be74 0x85802cfeb24a44d6 0xf31f35e62429f5cb
-0x47b870132f0f2527 0x3bef8c28c62b7933 0xf9862f1954334aaa
-0x6142043b9d7b8b81 0xc78a907965972287 0xe14b1d2a8d083d05
-0xb1a871510b4f3f75 0x59a00d2aac09a4ef 0x7c97edc49af98314
-0xdd4aab86f1152b2b 0x88bd0a6cd1b0307d 0xfdefc6d97ed5c95a
-0xe28c7cca7215417d 0x7d2120fe52be9b7f 0xa02e5ee452ffad28
-0x8dab828a23b62644 0x25b3328373eb47de 0xde0169b28c909cf4
-0xb9b3c51169a7a836 0x4541eb9d97eea0d6 0x1066c5b6d33156b5
-0xaf4e1758f7c6645f 0xf436e47c3b2de674 0xd8debdd7895e6d80
-0x615d5dded5a0383b 0x4834c108bc72e7b0 0x5ac8150a4d88bf44
-0xff249baacfc83e7a 0x1adbe96930658b73 0x0aa6f579acb30710
-0x30ec7277223022a2 0x39aa3d32743f07b4 0x8b822e40dfbcefa9
-0x8119c3984a6d6fc4 0x228746c8f461f4ca 0xa7588248e15d4fb6
-0xa900b1fe9351661d 0xfd6ee310a38cfc88 0x5cc2716dde3a6486
-0x2a430535315c47c6 0xec8f741aa91d097a 0x325170f292f4d4b7
-0xf71ad5c55a97aafa 0x3051eca81bdefbdc 0xc55f411439234e97
-0x10fe11b12e91e37a 0xf081df2576233eec 0x9eb8940ca99473c9
-0x5268bb96f025f00d 0x0281ee36f301d7ed 0xa0dfe16d6dafba1a
-0x2c1414da9a9b5d5a 0x9c44677222d40889 0xae4747a3134dd86d
-0x2d8892a98f291495 0xfb877e659c3ba71c 0xddf2cd76c64c3eae
-0x86998597390f3524 0xed28981f85bb3386 0x55b153bfec5beeac
-0x070f3b6614925bee 0x4eb1f4056f2ffa7f 0x78c6fd1608716443
-0x9a7662241ec5ee22 0x271ec60c7e87c370 0xf06fd9db9054c3d2
-0xa2723d1d26721f97 0xb078f617085673e5 0xdc78f1d0abc15908
-0x943849291f25e178 0xbae22ce64af1206e 0x11c78544941df857
-0x770ade0044d5f1fc 0xbc2878e716899306 0x94790cb9dea053b5
-0x6bbde24988ce4f85 0xfeb8786c8d39aa85 0x1f97f351d463dcf9
-0x7b4e291afc0e51d5 0x1faae3799dba6603 0x25815979ba54f03b
-0xb02a9b623c9400f0 0xc4a07da834cbf427 0x8f5cc1076ea09031
-0x2dea3ec81ab08515 0xe98683d1fb2b46e0 0xd03c1f8e1d3ab77d
-0xf2c2c6d3e5f609ab 0x2a4de6d7e059318d 0x5ade1e2f78a73769
-0x2662dcf712c0a5f9 0x14207cc1750e2a61 0xa02d796041e7f4c5
-0x996adc6d965c2445 0x5dd4cb5847526843 0x0612d104e4e52c29
-0x627c637c39b6587a 0xc04c4f4dc793d508 0xdd6cc43e981b9b46
-0xec8c58354fba1cfb 0xbc57a1e2281000bf 0x856fe5ec82652f66
-0x2350e203ab340e42 0xb74978ac55eb75fa 0x844067a42c59d22a
-0x5bb67e9025bcb14b 0xc6882f42a876775d 0x537deec5107af383
-0xa1b9e0408bb030ae 0x96f5422b40d40266 0xe6cd16792e1a8f4f
-0xd0994934ced6fb04 0x48924c4026066397 0xe6554877dbefffdc
-0x8acdae02c1b72b73 0x7002431784308714 0xec6113b8ebf9216d
-0xccd7d92eb94a654d 0x55bf89d61ea45655 0x40dfe822e6fc1ea4
-0xd871edcfa00e4eae 0x2b162c2c595b2ce0 0xfe78e58f4fa84c74
-0x8761656ac9573dc6 0xfb1c05d5c05ad042 0xcd6868216c342eae
-0x438466743ae36016 0x090c7ae0a96ce094 0x84c585a68e7a3fd3
-0x0f6b9eec0252f718 0x8144d103959c9cc8 0x9228d0f9530fe13e
-0x8182289a02b0f3b8 0xc71701f3dc02224e 0xd4fea5e16b0d5abc
-0x1bf89249443455f9 0x4d55abb33ea80876 0x30983ba4fc324bb0
-0xbdbe2ca26b4c48f5 0x110c8c2a0ce2403f 0x28fb71fe1421a58e
-0x897ca36224a0aeeb 0x873de78eae3414fa 0x5fc8368456d0a9a5
-0x730c72bf20e283f8 0x7582ea92afd96933 0x109dc798de62815f
-0x1334281c26f62ce5 0xc7351153d0631deb 0x0c7023b82da633bf
-0xab1f57a9402eb30f 0xbb43ef1a4abfed2f 0xbb13ff409efeb1fc
-0x16c77c66e400694e 0x5426e09c1deea6a8 0x79166753b5237b34
-0x4dac916d35ab84db 0xb2ddcede1bc5dccf 0x2ca000aed2faada6
-0x07b93902e0f10ca3 0xb181252e99021c2e 0x2a7cbcd596a23023
-0x1d135b910a22e3cd 0x50cac25266319226 0xd58ef5433e09ed3d
-0x2ba2a95166295246 0x104b0b90d7f54fcb 0x581e62e6e1effb88
-0x3b45c52fb3a61216 0x9b49976a6f98f4eb 0x5ecfaa723c68195f
-0x8bbec72e20caaf68 0xc6bcc3dd73e2ad16 0xeba0ca03b8cb6ce6
-0x77ea36fa204a7dbc 0x032d39bedbe56a62 0xf602ba75320469bb
-0x75ab379ae76f35d0 0xde954c6d2bc62abc 0xc67dc6587d5cb845
-0x6dd3c792d70f1d9f 0xda648f505baf81a4 0x6db4fb04bf05696b
-0x171a2898c06495a2 0x70f7573328116d96 0xa1b113a37b65e83b
-0xb7dc61716efbe386 0x44ff43b2d0f72216 0xb7572d3ef3679377
-0x01aa46678e35f96e 0x4793e9cefb9c00c5 0xb1b1cef3fa3e69da
-0xd6d5ce0496318fda 0x69d94c713ea5f8ff 0xb472156b44138e70
-0xa02166bb98773ebb 0xb91af940dff04058 0x9ceb3ab0aa02a266
-0xe2100434f0fc2e43 0xb80cdd4bbf69e7d3 0xe7c922fe9b3e7dfd
-0x3efad5419796a29b 0x87066640dc12ad77 0x395836ee2932bf23
-0x8e31b88ecced3d2e 0x4961ec2d6c7a75bf 0x6cd4f08d12c54e2b
-0x9f4dffcf1ddaa230 0xde123965919adba9 0x85e8c4d2146b9a09
-0x9e08deb3621df068 0xe0fd112de4b1269a 0xce27bb4d2a13cc8f
-0x32da8b6a6be75911 0x74e5dc1240d07647 0x99f0a90a8e1744c6
-0x51f94d8711994a47 0x690ca685dce349e9 0xda0c25b35ec3c056
-0x1ddea693f42c7d0f 0x23ab3c219c040475 0x3117b89a22075ecf
-0x8b729c188838cc11 0xf70c2e398725c02c 0xb5bf35fe12a37678
-0x1f0d26466bf81f25 0x20035f8d7e67489e 0x536e07502c93e26d
-0x0289b3b250176742 0xf0808a555109f37a 0x57ad69a0b71ab441
-0x1c81fa7d03800bdf 0xdf4febd9d519ccce 0x3b197c4de921087e
-0xda7c01969094f8f2 0x1bc17b9c04f6c5a5 0x89e4e1442e1d96d2
-0x05ae8ea263a5551a 0x6a01bac34ab09948 0x0b3e4f6e4a88c0f0
-0x4ecd511935dc55fe 0xa86534eb3311fa12 0x5b30a6c3a1e4141f
-0xa7cb6e8360649ab6 0xac9f884e6cfdacf4 0xf91bedc58f459fbc
-0x0fc06df57546cd8d 0xbe67f7c136080899 0x56abb03af17d6abf
-0x724d1581df36e444 0x6d348658a786cf13 0x15657b71facd5394
-0x870833f43ea4f44c 0x0c6b401a4ac5a9cf 0xac11311d9fdc1be0
-0x476db893090dbf50 0x6413363a40a0cc4d 0x656747030824b882
-0xc7763af198a3a915 0xc75b4c22db7f00e4 0xae0908746889482f
-0x105715af45acd194 0x484b10f834c570c8 0x29ac2c4625e5ebe7
-0x1ceda07f07739b02 0x25b8f899688c1e3e 0x9dffd257982a8139
-0x6ab245d75fd640ad 0xfd4a57561629ffa3 0xc6d726b0652e522f
-0xbcee4526af1030dd 0x43ea89479729db21 0xf0293df6e9672dec
-0xd4bc658e845060e1 0xd3a76977a87cc56a 0x3e9d83e4f81d87ba
-0x37f417d10048bc8d 0x14813d8b4553c5ae 0x574426b7bcdf149a
-0x9f791183f3df9a17 0xb155609334da4198 0xb4c3a5805d905e94
-0x12be857025a4a3eb 0x987445ae6fa8f849 0x428543effdb537ca
-0x05184d6f0c8af720 0x5f5a26958fc4f110 0x327feea9b8040719
-0x20f1368a9c267469 0x142bd97ddce3145f 0xb6dde259f3962e5b
-0xa5b7dfcf9c74d178 0x4a18aeb68cb7ee39 0xb7cb562ea710998c
-0x192c6a2440aa4799 0x262f549ff36ca087 0x00f1af86f7765419
-0x4fcaee4d2099f234 0xcb4823b5e460d68f 0xb8739346ee15f7c7
-0xe43f60dfcb722680 0xe588a07930826597 0xf11726b1fc2109e9
-0x2424847c52585435 0x594ca4d5eeaee507 0x960a8b2c5b3f2315
-0xf967389af1bc6066 0x9f5e1091a412689b 0x62c846564646bc24
-0x700fc081c9ad8622 0x054e53d585fed025 0x8393936ba2f979b0
-0xb7c891fc56a0e600 0x2055bd49358e42d3 0x10cc9952ec1e0244
-0x5dec53aadd0f64cf 0x5e2b688880e60374 0x26e5823522a3d1f7
-0x59263a46b0645a04 0xf8ea1b64930f2e4b 0x36e9fd75c4e84705
-0x4d196437dc1e9d07 0xc5c13dbebb428a60 0x0dcfbb9e0dd9357a
-0x7431271fe51ae20f 0x7578310058c9bc41 0xedab646a7a08e4e7
-0x1f098e77ac40fb45 0xdeb6b54c5f31ee6a 0x7d4953f9398fdf64
-0x9f28504dddbf678f 0xe3a56dee1a79e9f0 0x5cc098e3879b9e87
-0xc0b3e036a7160df6 0x345202b60f7d5fb9 0xde5f13f2f373b5d5
-0x09be9e199ea0e9d9 0x750064ba7fef6ab6 0x8393d8eed970a861
-0x90b992785db1e2ce 0x66a200d30926cb63 0x08d2a52a1be49bcf
-0xb1d396ed4bd1e92d 0x0067b3f585de736c 0x3e203ee543d7bc8e
-0x7f660ac01586461f 0x2ef0b714fe7da812 0xd02030c778ab097c
-0xbe9b971cd60bc342 0xb1cf6f572000cd53 0x7090e8f533244d92
-0x4b3d3eb42bc0616f 0x4abcb0dc9345cec3 0x3315735b2eaa1dc1
-0x1968b53c2948c9fe 0x6f832e2ab85716fb 0xf680b4690f452fe7
-0x35634e189261c27a 0xb1655320b6881a39 0x3aa4f712d8d74eb6
-0xe7f8db6934680789 0x31a395a5ef322e71 0x9e8108f1e7bfbf9b
-0x32dc1fc02c568ef1 0x1c7e39ba8e98e717 0x98bb5d9f1719de9c
-0x64fad9fd4e9e04ac 0xc43c4ef84c1a749a 0xf172bfdcb082ae79
-0x775944b64825cc94 0x6752a18cc5cbd881 0x0bcd3e25a4cd6344
-0xdc00a7d88f1be5a6 0xfdf241d4b98b15c7 0x6ffccf1d3347e63e
-0xb4985dd04e81f326 0x798f4cfb661bbd4b 0xa2013d7415eb3df7
-0x879a90fdc3315936 0xdf037725e8829def 0x3eee6d747de55fd1
-0x1950ae14c16199ba 0xf32d6b0bbb81943f 0x6b813e655734bbf8
-0x63f5368932cfe7b8 0xf4ebf4ca577ec930 0xe7a5ec5bab21afba
-0x804feff47c04c4b7 0x2e3da596c444bf41 0x21be1d62146c81da
-0xfaa880de93886ac0 0x88987a63ea1750e6 0xce299bb9e0c40dd8
-0x6d2d3a48162f4f0d 0xfb873ecc6261d540 0x665184a16cbfeca9
-0x99f08162522947e1 0xb678afe1ae26f80e 0x81438967c30cec52
-0x7b84c7088847f470 0xdf8e250fe4392e3d 0xde420f611e212a49
-0x247e7a9bc296553f 0xcda4431f6214d257 0x0fe8ce1a5e7264ac
-0x7e2e9d87db4e18fc 0x12e9b0a508d5e4e0 0xdef99c379a602fc6
-0x772b91fb215e3f6c 0x29bb880dcd669c11 0x604e2a3e80d1a980
-0x62db806e1dacccb7 0x5a9925e5d386b369 0x464807efe2c001cb
-0x0681067ba9e69114 0xc4d7c8c2a7123d6a 0x4a3285f39878a215
-0x7432ddf4653a9cb8 0xd007940d70c24b13 0x5608cb9f87571fc8
-0x7a2c5b232b41ebdd 0x99245a3f8c434fd8 0x8acbdd231982f5a2
-0xfa50e2c5460ba07f 0xb2b5383501d97388 0x91722d48b0a05a3e
-0xe92bd4a4ad9bd471 0xf8b226909751d1ac 0x3a84feeb4efe53ea
-0x1bf2c0769fe54fae 0x1f06a43bf2b2bb23 0x6d89b57008409736
-0x68d2563f7fc1319a 0x4cca7c28306c60d5 0x45532d245acdef1b
-0xda535f9dec96bbab 0x25451d82b9aa12b3 0xc2a354b5d8c63228
-0x1c1f97d0851becc0 0x9324fa1d5e1b4a44 0xdb312686295300fa
-0x92e8ee2945f76afc 0x102b3df2dea6e8b2 0x309d7a9e07174ebf
-0xa0d2445e0bebc266 0xdfb983a73c6afd26 0x52cfe364f3957e2d
-0x72b6f2ad68342515 0x1383fb2184f4ab50 0xfc50740a6dc0d7a0
-0x12c03f1b30b409a3 0x195f3359d8d3d697 0xb1a696de263a9206
-0xbfa9d96984833e72 0xbc6b844babe41595 0xd72507fee5d57c9b
-0x48ad095be6f55861 0x76942ad5903bb97c 0xd4b006bd9e4de5eb
-0x0ae6d222ed88e74c 0x37aa525e18213b95 0x3e87311c62589252
-0x9db5c12f91a6f728 0xe9dd60e4310fa419 0xe5ab03d0f0a1f978
-0xc699639ec932afb5 0xc2e90bc8cddee0ab 0xafa68bbcb8d7ddd0
-0x7a8c80def2c5c00f 0xb6caf5c351a3f9e2 0x149e219d648cff15
-0x55e3b95cfb7941c9 0x445df7728165470e 0x246e3b9dde51bb60
-0x7f47c1e9e9e105cc 0x2a78b27625fe4e3e 0xf7e647125a81184b
-0x35615c5ac93e3ee2 0xab3d5ee5f30c6542 0x66adaa81be80a255
-0xba3dee726484d20a 0x81fd51257a971385 0xb6a79592d5164e85
-0xd3333835d170e9bd 0x5debdffeff4d63b0 0xe405e71c9c7fe247
-0x64df8c009c73b84e 0x850fb06b2c70c144 0x1619917459f3cff2
-0x50738716b1e7b305 0xa361ad73caf95820 0x6cbee1e292279728
-0xd3a60974113b480f 0x1333cc2da08bd7d0 0x9e983992c88d0c05
-0xf3885e54ee8d8864 0xbb72960373ad2100 0x7fce723754efe9c5
-0x0d04185678b2aa93 0x2bd575cf4c798b50 0x03deee9bed8c238b
-0x4dd3e83aef99b423 0x49575eddb23bcd0c 0xa86925782bd40519
-0x61604398fc35ed8b 0x0030c4447d5843d7 0x97f557e742d44f66
-0xe478030253f33ed2 0xb5b8b264f2499aab 0xa66da3eef7c6a8b7
-0x48c5802dde4e9c48 0x892fe0c182a44518 0x40da78f7fc26965f
-0xef69f38439b57f01 0x41ce76b97047c1f5 0x11d6df8a7e933468
-0x0e68110350ac736e 0x4993e7a6fe133ef1 0x3ee9280e32bf67ad
-0x9dd5df301cd57953 0x1e541a2e250db81e 0x193c71118501bda9
-0xef943420618b4a08 0x88496f019fcee0bc 0xbea7b0911223aabe
diff --git a/kmer/libutil/mt19937ar/mt19937ar.readme b/kmer/libutil/mt19937ar/mt19937ar.readme
deleted file mode 100755
index c3a9c41..0000000
--- a/kmer/libutil/mt19937ar/mt19937ar.readme
+++ /dev/null
@@ -1,74 +0,0 @@
-This is a Mersenne Twister pseudorandom number generator
-with period 2^19937-1 with improved initialization scheme,
-modified on 2002/1/26 by Takuji Nishimura and Makoto Matsumoto.
-
-Contents of this tar ball:
-readme-mt.txt this file
-mt19937ar.c the C source (ar: initialize by ARray)
-mt19937ar.out Test outputs of six types generators. 1000 for each
-
-1. Initialization
- The initialization scheme for the previous versions of MT
-(e.g. 1999/10/28 version or earlier) has a tiny problem, that
-the most significant bits of the seed is not well reflected
-to the state vector of MT.
-
-This version (2002/1/26) has two initialization schemes:
-init_genrand(seed) and init_by_array(init_key, key_length).
-
-init_genrand(seed) initializes the state vector by using
-one unsigned 32-bit integer "seed", which may be zero.
-
-init_by_array(init_key, key_length) initializes the state vector
-by using an array init_key[] of unsigned 32-bit integers
-of length key_kength. If key_length is smaller than 624,
-then each array of 32-bit integers gives distinct initial
-state vector. This is useful if you want a larger seed space
-than 32-bit word.
-
-2. Generation
-After initialization, the following type of pseudorandom numbers
-are available.
-
-genrand_int32() generates unsigned 32-bit integers.
-genrand_int31() generates unsigned 31-bit integers.
-genrand_real1() generates uniform real in [0,1] (32-bit resolution).
-genrand_real2() generates uniform real in [0,1) (32-bit resolution).
-genrand_real3() generates uniform real in (0,1) (32-bit resolution).
-genrand_res53() generates uniform real in [0,1) with 53-bit resolution.
-
-Note: the last five functions call the first one.
-if you need more speed for these five functions, you may
-suppress the function call by copying genrand_int32() and
-replacing the last return(), following to these five functions.
-
-3. main()
-main() is an example to initialize with an array of length 4,
-then 1000 outputs of unsigned 32-bit integers,
-then 1000 outputs of real [0,1) numbers.
-
-4. The outputs
-The output of the mt19937ar.c is in the file mt19937ar.out.
-If you revise or translate the code, check the output
-by using this file.
-
-5. Cryptography
-This generator is not cryptoraphically secure.
-You need to use a one-way (or hash) function to obtain
-a secure random sequence.
-
-6. Correspondence
-See:
-URL http://www.math.keio.ac.jp/matumoto/emt.html
-email matumoto at math.keio.ac.jp, nisimura at sci.kj.yamagata-u.ac.jp
-
-7. Reference
-M. Matsumoto and T. Nishimura,
-"Mersenne Twister: A 623-Dimensionally Equidistributed Uniform
-Pseudo-Random Number Generator",
-ACM Transactions on Modeling and Computer Simulation,
-Vol. 8, No. 1, January 1998, pp 3--30.
-
--------
-Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
-All rights reserved.
diff --git a/kmer/libutil/mt19937ar/tt800.c b/kmer/libutil/mt19937ar/tt800.c
deleted file mode 100644
index d4f110b..0000000
--- a/kmer/libutil/mt19937ar/tt800.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* http://random.mat.sbg.ac.at/ftp/pub/data/tt800.c */
-
-/* A C-program for TT800 : July 8th 1996 Version */
-/* by M. Matsumoto, email: matumoto at math.keio.ac.jp */
-/* genrand() generate one pseudorandom number with double precision */
-/* which is uniformly distributed on [0,1]-interval */
-/* for each call. One may choose any initial 25 seeds */
-/* except all zeros. */
-
-/* See: ACM Transactions on Modelling and Computer Simulation, */
-/* Vol. 4, No. 3, 1994, pages 254-266. */
-
-#include <stdio.h>
-#define N 25
-#define M 7
-
-double
-genrand()
-{
- unsigned long y;
- static int k = 0;
- static unsigned long x[N]={ /* initial 25 seeds, change as you wish */
- 0x95f24dab, 0x0b685215, 0xe76ccae7, 0xaf3ec239, 0x715fad23,
- 0x24a590ad, 0x69e4b5ef, 0xbf456141, 0x96bc1b7b, 0xa7bdf825,
- 0xc1de75b7, 0x8858a9c9, 0x2da87693, 0xb657f9dd, 0xffdc8a9f,
- 0x8121da71, 0x8b823ecb, 0x885d05f5, 0x4e20cd47, 0x5a9ad5d9,
- 0x512c0c03, 0xea857ccd, 0x4cc1d30f, 0x8891a8a1, 0xa6b7aadb
- };
- static unsigned long mag01[2]={
- 0x0, 0x8ebfd028 /* this is magic vector `a', don't change */
- };
- if (k==N) { /* generate N words at one time */
- int kk;
- for (kk=0;kk<N-M;kk++) {
- x[kk] = x[kk+M] ^ (x[kk] >> 1) ^ mag01[x[kk] % 2];
- }
- for (; kk<N;kk++) {
- x[kk] = x[kk+(M-N)] ^ (x[kk] >> 1) ^ mag01[x[kk] % 2];
- }
- k=0;
- }
- y = x[k];
- y ^= (y << 7) & 0x2b5b2500; /* s and b, magic vectors */
- y ^= (y << 15) & 0xdb8b0000; /* t and c, magic vectors */
- y &= 0xffffffff; /* you may delete this line if word size = 32 */
-/*
- the following line was added by Makoto Matsumoto in the 1996 version
- to improve lower bit's corellation.
- Delete this line to o use the code published in 1994.
-*/
- y ^= (y >> 16); /* added to the 1994 version */
- k++;
- return( (double) y / (unsigned long) 0xffffffff);
-}
-
-/* this main() output first 50 generated numbers */
-main()
-{ int j;
- for (j=0; j<100000; j++) {
- printf("%5f ", genrand());
- if (j%8==7) printf("\n");
- }
- printf("\n");
-}
diff --git a/kmer/libutil/palloc.c b/kmer/libutil/palloc.c
deleted file mode 100644
index db848ad..0000000
--- a/kmer/libutil/palloc.c
+++ /dev/null
@@ -1,236 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "util.h"
-
-typedef struct pallocroot pallocroot;
-typedef struct pallocnode pallocnode;
-
-// _dbg: 0 -- print nothing
-// 1 -- print block allocation
-// 2 -- print all allocations
-
-struct pallocroot {
- size_t _bs; // size of block
- pallocnode *_nl; // nodeList
- pallocnode *_cn; // currentNode
- int _dbg; // if set, debug information is printed
-};
-
-struct pallocnode {
- size_t _cp; // cuurentPosition
- char *_dt; // data
- pallocnode *_nx; // next pallocnode
-};
-
-extern pallocroot _palloc_stuff;
-
-pallocroot _palloc_stuff = { 128 * 1024 * 1024, NULL, NULL, 0 };
-
-static
-void *
-really_allocate(size_t size) {
- void *ret = malloc(size);
- if (ret == 0L) {
- fprintf(stderr, "palloc()-- can't allocate "sizetFMT" bytes: %s.\n", size, strerror(errno));
- exit(1);
- }
- return(ret);
-}
-
-void
-psetblocksize(size_t size) {
- if (_palloc_stuff._nl == 0L)
- _palloc_stuff._bs = size;
-}
-
-size_t
-pgetblocksize(void) {
- return(_palloc_stuff._bs);
-}
-
-void
-psetdebug(int on) {
- _palloc_stuff._dbg = on;
-}
-
-void*
-pallochandle(size_t size) {
- pallocroot *root = (pallocroot *)malloc(sizeof(pallocroot));
- if (root == NULL)
- fprintf(stderr, "pallochandle()-- can't allocate a handle!\n"), exit(1);
- if (size == 0)
- size = 128 * 1024 * 1024;
- root->_bs = size;
- root->_nl = NULL;
- root->_cn = NULL;
- root->_dbg = 0;
- return(root);
-}
-
-
-// Release a palloc handle, does not release the memory in the handle!
-void
-pfreehandle(void *handle) {
- free((pallocroot *)handle);
-}
-
-// Clear out memory inside the handle. The handle remains valid after this.
-void
-pfree2(void *handle) {
- pallocroot *root = (pallocroot *)handle;
- pallocnode *n;
- size_t r = 0;
- size_t b = 0;
-
- if (root == NULL)
- root = &_palloc_stuff;
-
- while ((n = root->_nl) != 0L) {
- r += n->_cp;
- b++;
- root->_nl = n->_nx;
- free(n->_dt);
- free(n);
- }
-
- if (root->_dbg > 0)
- fprintf(stderr, "palloc()-- "sizetFMT" bytes in "sizetFMT" blocks returned to free store.\n", r, b);
-
- root->_nl = 0L;
- root->_cn = 0L;
-}
-
-void
-pfree(void) {
- pfree2(&_palloc_stuff);
-}
-
-
-void *
-palloc2(size_t size, void *handle) {
- pallocroot *root = (pallocroot *)handle;
-
- if (root == NULL)
- root = &_palloc_stuff;
-
- // Make size a multiple of 8
- //
- if (size & 0x7) {
- size >>= 3;
- size++;
- size <<= 3;
- }
- if (size == 0)
- return(0L);
-
- // Allocate the initial block if it doesn't exist.
- //
- if (root->_nl == NULL) {
- root->_nl = (pallocnode *)really_allocate(sizeof(pallocnode));
- root->_cn = root->_nl;
-
- if (root->_dbg > 0)
- fprintf(stderr, "palloc()-- Inital block of "sizetFMT" bytes at %p.\n", root->_bs, root->_cn);
-
- root->_cn->_cp = 0;
- root->_cn->_dt = (char *)really_allocate(root->_bs);
- root->_cn->_nx = NULL;
- }
-
-
- // If the requested space is larger than our block size, allocate a
- // new node with the required amount of space. The new node is
- // placed on the start of the alloc'd list.
- //
- // We also place blocks that are bigger than the amount free in the
- // current block, AND bigger than the amount used in the current
- // block here. Since the new block is larger than the free space,
- // it won't fit in the current block. Since the new block is
- // larger than the current block, it is wasteful to throw out the
- // current block and replace it with a new block.
- //
- // The tests read:
- // new block is bigger than our block size
- // new block won't fit in current block
- // new block is larger than current block
- //
- if ((size > root->_bs) ||
- ((size > root->_bs - root->_cn->_cp) &&
- (size > root->_cn->_cp))) {
- pallocnode *n;
-
- n = (pallocnode *)really_allocate(sizeof(pallocnode));
- n->_cp = size;
- n->_dt = (char *)really_allocate(size);
- n->_nx = root->_nl;
-
- if (root->_dbg > 0)
- fprintf(stderr, "palloc()-- New needs "sizetFMT" bytes: custom new block at %p.\n",
- size,
- n);
-
- root->_nl = n;
- if (root->_cn == 0L)
- root->_cn = n;
-
- return(n->_dt);
- }
-
-
- // Need more space?
- //
- if (size + root->_cn->_cp > root->_bs) {
- root->_cn->_nx = (pallocnode *)really_allocate(sizeof(pallocnode));
-
- if (root->_dbg > 0)
- fprintf(stderr, "palloc()-- Old block %.3f%% used ("sizetFMT" bytes remaining), new needs "sizetFMT" bytes: new block of "sizetFMT" bytes at %p.\n",
- 100.0 * root->_cn->_cp / root->_bs,
- root->_bs - root->_cn->_cp,
- size,
- root->_bs,
- root->_cn->_nx);
-
- root->_cn = root->_cn->_nx;
- root->_cn->_cp = 0;
- root->_cn->_dt = (char *)really_allocate(root->_bs);
- root->_cn->_nx = NULL;
- }
-
-
- // OK, grab the space, and return it.
- //
- root->_cn->_cp += size;
-
- if (root->_dbg > 1)
- fprintf(stderr, "palloc()-- Old block %.3f%% used ("sizetFMT" bytes remaining): returning "sizetFMT" bytes at %p.\n",
- 100.0 * root->_cn->_cp / root->_bs,
- root->_bs - root->_cn->_cp,
- size, root->_cn->_dt + root->_cn->_cp - size);
-
- return(root->_cn->_dt + root->_cn->_cp - size);
-}
-
-
-
-void *
-palloc(size_t size) {
- return(palloc2(size, &_palloc_stuff));
-}
-
-
-void
-pdumppalloc(void *handle) {
- pallocroot *root = (pallocroot *)handle;
- pallocnode *n = root->_nl;
- fprintf(stderr, "palloc dump\n");
- fprintf(stderr, ""sizetFMT" bytes per block\n", root->_bs);
- while (n != 0L) {
- fprintf(stderr, "%p: currentPosition: "sizetFMT" bytes used%s\n",
- n, n->_cp, (n == root->_cn) ? ", current block" : "");
- n = n->_nx;
- }
-}
-
diff --git a/kmer/libutil/qsort_mt.c b/kmer/libutil/qsort_mt.c
deleted file mode 100644
index bb0b722..0000000
--- a/kmer/libutil/qsort_mt.c
+++ /dev/null
@@ -1,406 +0,0 @@
-/*-
- * Copyright (c) 1992, 1993
- * The Regents of the University of California. All rights reserved.
- * Multithread implementation Copyright (c) 2006, 2007 Diomidis Spinellis.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-//static char sccsid[] = "@(#)qsort.c 8.1 (Berkeley) 6/4/93";
-//__FBSDID("$FreeBSD: src/lib/libc/stdlib/qsort.c,v 1.12 2002/09/10 02:04:49 wollman Exp $");
-
-//#include <sys/cdefs.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <sys/types.h>
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#ifdef __FreeBSD__
-#include <pmc.h>
-#endif
-
-typedef int cmp_t(const void *, const void *);
-
-static inline char *med3(char *, char *, char *, cmp_t *);
-static inline void swapfunc(char *, char *, int, int);
-
-#define min(a, b) (a) < (b) ? a : b
-
-/*
- * Qsort routine from Bentley & McIlroy's "Engineering a Sort Function".
- */
-#define swapcode(TYPE, parmi, parmj, n) { \
- long i = (n) / sizeof (TYPE); \
- TYPE *pi = (TYPE *) (parmi); \
- TYPE *pj = (TYPE *) (parmj); \
- do { \
- TYPE t = *pi; \
- *pi++ = *pj; \
- *pj++ = t; \
- } while (--i > 0); \
- }
-
-
-static inline void
-swapfunc(a, b, n, swaptype)
- char *a, *b;
- int n, swaptype;
-{
- if(swaptype <= 1)
- swapcode(long, a, b, n)
- else
- swapcode(char, a, b, n)
- }
-
-#define swap(a, b) \
- if (swaptype == 0) { \
- long t = *(long *)(a); \
- *(long *)(a) = *(long *)(b); \
- *(long *)(b) = t; \
- } else \
- swapfunc(a, b, es, swaptype)
-
-#define vecswap(a, b, n) if ((n) > 0) swapfunc(a, b, n, swaptype)
-
-#define CMP(x, y) (cmp((x), (y)))
-
-static inline char *
-med3(char *a, char *b, char *c, cmp_t *cmp)
-{
- return CMP(a, b) < 0 ?
- (CMP(b, c) < 0 ? b : (CMP(a, c) < 0 ? c : a ))
- :(CMP(b, c) > 0 ? b : (CMP(a, c) < 0 ? a : c ));
-}
-
-/*
- * We use some elaborate condition variables and signalling
- * to ensure a bound of the number of active threads at
- * 2 * maxthreads and the size of the thread data structure
- * to maxthreads.
- */
-
-/* Condition of starting a new thread. */
-enum thread_state {
- ts_idle, /* Idle, waiting for instructions. */
- ts_work, /* Has work to do. */
- ts_term /* Asked to terminate. */
-};
-
-/* Variant part passed to qsort invocations. */
-struct qsort {
- enum thread_state st; /* For coordinating work. */
- struct common *common; /* Common shared elements. */
- void *a; /* Array base. */
- size_t n; /* Number of elements. */
- pthread_t id; /* Thread id. */
- pthread_mutex_t mtx_st; /* For signalling state change. */
- pthread_cond_t cond_st; /* For signalling state change. */
-};
-
-/* Invariant common part, shared across invocations. */
-struct common {
- int swaptype; /* Code to use for swapping */
- size_t es; /* Element size. */
- cmp_t *cmp; /* Comparison function */
- int nthreads; /* Total number of pool threads. */
- int idlethreads; /* Number of idle threads in pool. */
- int forkelem; /* Minimum number of elements for a new thread. */
- struct qsort *pool; /* Fixed pool of threads. */
- pthread_mutex_t mtx_al; /* For allocating threads in the pool. */
-};
-
-static void *qsort_thread(void *p);
-
-/* The multithreaded qsort public interface */
-
-void
-qsort_mt(void *a, size_t n, size_t es, cmp_t *cmp, int maxthreads, int forkelem)
-{
- struct qsort *qs;
- struct common c;
- int i, islot;
- int bailout = 1;
-
- if (n < forkelem)
- goto f1;
- errno = 0;
-
- if (maxthreads <= 1)
- goto f1;
-
- /* Try to initialize the resources we need. */
- if (pthread_mutex_init(&c.mtx_al, NULL) != 0)
- goto f1;
- if ((c.pool = (struct qsort *)calloc(maxthreads, sizeof(struct qsort))) ==NULL)
- goto f2;
- for (islot = 0; islot < maxthreads; islot++) {
- qs = &c.pool[islot];
- if (pthread_mutex_init(&qs->mtx_st, NULL) != 0)
- goto f3;
- if (pthread_cond_init(&qs->cond_st, NULL) != 0) {
- pthread_mutex_destroy(&qs->mtx_st);
- goto f3;
- }
- qs->st = ts_idle;
- qs->common = &c;
- if (pthread_create(&qs->id, NULL, qsort_thread, qs) != 0) {
- pthread_mutex_destroy(&qs->mtx_st);
- pthread_cond_destroy(&qs->cond_st);
- goto f3;
- }
- }
-
- /* All systems go. */
- bailout = 0;
-
- /* Initialize common elements. */
- c.swaptype = ((char *)a - (char *)0) % sizeof(long) || \
- es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1;
- c.es = es;
- c.cmp = cmp;
- c.forkelem = forkelem;
- c.idlethreads = c.nthreads = maxthreads;
-
- /* Hand out the first work batch. */
- qs = &c.pool[0];
- pthread_mutex_lock(&qs->mtx_st);
- qs->a = a;
- qs->n = n;
- qs->st = ts_work;
- c.idlethreads--;
- pthread_cond_signal(&qs->cond_st);
- pthread_mutex_unlock(&qs->mtx_st);
-
- /*
- * Wait for all threads to finish, and
- * free acquired resources.
- */
- f3: for (i = 0; i < islot; i++) {
- qs = &c.pool[i];
- if (bailout) {
- pthread_mutex_lock(&qs->mtx_st);
- qs->st = ts_term;
- pthread_cond_signal(&qs->cond_st);
- pthread_mutex_unlock(&qs->mtx_st);
- }
- pthread_join(qs->id, NULL);
- pthread_mutex_destroy(&qs->mtx_st);
- pthread_cond_destroy(&qs->cond_st);
- }
- free(c.pool);
- f2: pthread_mutex_destroy(&c.mtx_al);
- if (bailout) {
- /* XXX should include a syslog call here */
- fprintf(stderr, "Resource initialization failed; bailing out.\n");
- f1: qsort(a, n, es, cmp);
- }
-}
-
-
-/*
- * Allocate an idle thread from the pool, lock its
- * mutex, change its state to work, decrease the number
- * of idle threads, and return a
- * pointer to its data area.
- * Return NULL, if no thread is available.
- */
-static struct qsort *
-allocate_thread(struct common *c)
-{
- int i;
-
- pthread_mutex_lock(&c->mtx_al);
- for (i = 0; i < c->nthreads; i++)
- if (c->pool[i].st == ts_idle) {
- c->idlethreads--;
- c->pool[i].st = ts_work;
- pthread_mutex_lock(&c->pool[i].mtx_st);
- pthread_mutex_unlock(&c->mtx_al);
- return (&c->pool[i]);
- }
- pthread_mutex_unlock(&c->mtx_al);
- return (NULL);
-}
-
-/* Thread-callable quicksort. */
-static void
-qsort_algo(struct qsort *qs)
-{
- char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
- long d, r, swaptype, swap_cnt;
- void *a; /* Array of elements. */
- size_t n, es; /* Number of elements; size. */
- cmp_t *cmp;
- long nl, nr;
- struct common *c;
- struct qsort *qs2;
- pthread_t id;
-
- /* Initialize qsort arguments. */
- id = qs->id;
- c = qs->common;
- es = c->es;
- cmp = c->cmp;
- swaptype = c->swaptype;
- a = qs->a;
- n = qs->n;
- top:
-
- /* From here on qsort(3) business as usual. */
- swap_cnt = 0;
- if (n < 7) {
- for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
- for (pl = pm;
- pl > (char *)a && CMP(pl - es, pl) > 0;
- pl -= es)
- swap(pl, pl - es);
- return;
- }
- pm = (char *)a + (n / 2) * es;
- if (n > 7) {
- pl = a;
- pn = (char *)a + (n - 1) * es;
- if (n > 40) {
- d = (n / 8) * es;
- pl = med3(pl, pl + d, pl + 2 * d, cmp);
- pm = med3(pm - d, pm, pm + d, cmp);
- pn = med3(pn - 2 * d, pn - d, pn, cmp);
- }
- pm = med3(pl, pm, pn, cmp);
- }
- swap(a, pm);
- pa = pb = (char *)a + es;
-
- pc = pd = (char *)a + (n - 1) * es;
- for (;;) {
- while (pb <= pc && (r = CMP(pb, a)) <= 0) {
- if (r == 0) {
- swap_cnt = 1;
- swap(pa, pb);
- pa += es;
- }
- pb += es;
- }
- while (pb <= pc && (r = CMP(pc, a)) >= 0) {
- if (r == 0) {
- swap_cnt = 1;
- swap(pc, pd);
- pd -= es;
- }
- pc -= es;
- }
- if (pb > pc)
- break;
- swap(pb, pc);
- swap_cnt = 1;
- pb += es;
- pc -= es;
- }
- if (swap_cnt == 0) { /* Switch to insertion sort */
- for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
- for (pl = pm;
- pl > (char *)a && CMP(pl - es, pl) > 0;
- pl -= es)
- swap(pl, pl - es);
- return;
- }
-
- pn = (char *)a + n * es;
- r = min(pa - (char *)a, pb - pa);
- vecswap(a, pb - r, r);
- r = min(pd - pc, pn - pd - es);
- vecswap(pb, pn - r, r);
-
- nl = (pb - pa) / es;
- nr = (pd - pc) / es;
-
- /* Now try to launch subthreads. */
- if (nl > c->forkelem && nr > c->forkelem &&
- (qs2 = allocate_thread(c)) != NULL) {
- qs2->a = a;
- qs2->n = nl;
- pthread_cond_signal(&qs2->cond_st);
- pthread_mutex_unlock(&qs2->mtx_st);
- } else if (nl > 0) {
- qs->a = a;
- qs->n = nl;
- qsort_algo(qs);
- }
- if (nr > 0) {
- a = pn - nr * es;
- n = nr;
- goto top;
- }
-}
-
-/* Thread-callable quicksort. */
-static void *
-qsort_thread(void *p)
-{
- struct qsort *qs, *qs2;
- int i;
- struct common *c;
- pthread_t id;
-
- qs = p;
- id = qs->id;
- c = qs->common;
- again:
- /* Wait for work to be allocated. */
- pthread_mutex_lock(&qs->mtx_st);
- while (qs->st == ts_idle)
- pthread_cond_wait(&qs->cond_st, &qs->mtx_st);
- pthread_mutex_unlock(&qs->mtx_st);
- if (qs->st == ts_term) {
- return(NULL);
- }
- assert(qs->st == ts_work);
-
- qsort_algo(qs);
-
- pthread_mutex_lock(&c->mtx_al);
- qs->st = ts_idle;
- c->idlethreads++;
- if (c->idlethreads == c->nthreads) {
- for (i = 0; i < c->nthreads; i++) {
- qs2 = &c->pool[i];
- if (qs2 == qs)
- continue;
- pthread_mutex_lock(&qs2->mtx_st);
- qs2->st = ts_term;
- pthread_cond_signal(&qs2->cond_st);
- pthread_mutex_unlock(&qs2->mtx_st);
- }
- pthread_mutex_unlock(&c->mtx_al);
- return(NULL);
- }
- pthread_mutex_unlock(&c->mtx_al);
- goto again;
-}
diff --git a/kmer/libutil/readBuffer.C b/kmer/libutil/readBuffer.C
deleted file mode 100644
index 44b6b38..0000000
--- a/kmer/libutil/readBuffer.C
+++ /dev/null
@@ -1,284 +0,0 @@
-#include "util++.H"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <fcntl.h>
-
-
-// If bufferMax is zero, then the file is accessed using memory
-// mapped I/O. Otherwise, a small buffer is used.
-//
-readBuffer::readBuffer(const char *filename, u64bit bufferMax) {
-
- _filename = 0L;
- _file = 0;
- _filePos = 0;
- _mmap = false;
- _stdin = false;
- _eof = false;
- _bufferPos = 0;
- _bufferLen = 0;
- _bufferMax = 0;
- _buffer = 0L;
-
- if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) ||
- ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0))) {
- _filename = new char [32];
- strcpy(_filename, "(stdin)");
-
- _stdin = true;
-
- if (bufferMax == 0)
- bufferMax = 32 * 1024;
- } else if (filename == 0L) {
- fprintf(stderr, "readBuffer()-- no filename supplied, and I will not use the terminal for input.\n"), exit(1);
- } else {
- _filename = new char [strlen(filename) + 1];
- strcpy(_filename, filename);
- }
-
- if (bufferMax == 0) {
- _mmap = true;
- _buffer = (char *)mapFile(_filename, &_bufferLen, 'r');
- } else {
- errno = 0;
- _file = (_stdin) ? fileno(stdin) : open(_filename, O_RDONLY | O_LARGEFILE);
- if (errno)
- fprintf(stderr, "readBuffer()-- couldn't open the file '%s': %s\n",
- _filename, strerror(errno)), exit(1);
-
- _bufferMax = bufferMax;
- _buffer = new char [_bufferMax];
- }
-
- fillBuffer();
-
- if (_bufferLen == 0)
- _eof = true;
-}
-
-
-readBuffer::readBuffer(FILE *file, u64bit bufferMax) {
-
- if (bufferMax == 0)
- fprintf(stderr, "readBuffer()-- WARNING: mmap() not supported in readBuffer(FILE *)\n");
-
- _filename = new char [32];
- _file = fileno(file);
- _filePos = 0;
- _mmap = false;
- _stdin = false;
- _eof = false;
- _bufferPos = 0;
- _bufferLen = 0;
- _bufferMax = (bufferMax == 0) ? 32 * 1024 : bufferMax;
- _buffer = new char [_bufferMax];
-
- strcpy(_filename, "(hidden file)");
-
- // Just be sure that we are at the start of the file.
- errno = 0;
- lseek(_file, 0, SEEK_SET);
- if (errno)
- fprintf(stderr, "readBuffer()-- '%s' couldn't seek to position 0: %s\n",
- _filename, strerror(errno)), exit(1);
-
- fillBuffer();
-
- if (_bufferLen == 0)
- _eof = true;
-}
-
-
-readBuffer::~readBuffer() {
-
- delete [] _filename;
-
- if (_mmap)
- unmapFile(_buffer, _bufferLen);
- else
- delete [] _buffer;
-
- if (_stdin == false)
- close(_file);
-}
-
-
-void
-readBuffer::fillBuffer(void) {
-
- // If there is still stuff in the buffer, no need to fill.
- if (_bufferPos < _bufferLen)
- return;
-
- // No more stuff in the buffer. But if mmap'd, ths means we're EOF.
- if (_mmap) {
- _eof = true;
- return;
- }
-
- _bufferPos = 0;
- _bufferLen = 0;
-
- again:
- errno = 0;
- _bufferLen = (u64bit)::read(_file, _buffer, _bufferMax);
- if (errno == EAGAIN)
- goto again;
- if (errno)
- fprintf(stderr, "readBuffer::fillBuffer()-- only read "u64bitFMT" bytes, couldn't read "u64bitFMT" bytes from '%s': %s\n",
- _bufferLen, _bufferMax, _filename, strerror(errno)), exit(1);
-
- if (_bufferLen == 0)
- _eof = true;
-}
-
-
-void
-readBuffer::seek(u64bit pos) {
-
- if (_stdin == true) {
- if (_filePos < _bufferLen) {
- _filePos = 0;
- _bufferPos = 0;
- return;
- } else {
- fprintf(stderr, "readBuffer()-- seek() not available for file 'stdin'.\n");
- exit(1);
- }
-
- return;
- }
-
- assert(_stdin == false);
-
- if (_mmap) {
- _bufferPos = pos;
- _filePos = pos;
- } else {
- errno = 0;
- lseek(_file, pos, SEEK_SET);
- if (errno)
- fprintf(stderr, "readBuffer()-- '%s' couldn't seek to position "s64bitFMT": %s\n",
- _filename, pos, strerror(errno)), exit(1);
-
- _bufferLen = 0;
- _bufferPos = 0;
- _filePos = pos;
-
- fillBuffer();
- }
-
- _eof = (_bufferPos >= _bufferLen);
-}
-
-
-u64bit
-readBuffer::read(void *buf, u64bit len) {
- char *bufchar = (char *)buf;
-
- // Handle the mmap'd file first.
-
- if (_mmap) {
- u64bit c = 0;
-
- while ((_bufferPos < _bufferLen) && (c < len)) {
- bufchar[c++] = _buffer[_bufferPos++];
- _filePos++;
- }
-
- if (c == 0)
- _eof = true;
-
- return(c);
- }
-
- // Easy case; the next len bytes are already in the buffer; just
- // copy and move the position.
-
- if (_bufferLen - _bufferPos > len) {
- memcpy(bufchar, _buffer + _bufferPos, len);
- _bufferPos += len;
-
- fillBuffer();
-
- _filePos += len;
-
- return(len);
- }
-
- // Existing buffer not big enough. Copy what's there, then finish
- // with a read.
-
- u64bit bCopied = 0; // Number of bytes copied into the buffer
- u64bit bRead = 0; // Number of bytes read into the buffer
- u64bit bAct = 0; // Number of bytes actually read from disk
-
- memcpy(bufchar, _buffer + _bufferPos, _bufferLen - _bufferPos);
- bCopied = _bufferLen - _bufferPos;
- _bufferPos = _bufferLen;
-
- while (bCopied + bRead < len) {
- errno = 0;
- bAct = (u64bit)::read(_file, bufchar + bCopied + bRead, len - bCopied - bRead);
- if (errno)
- fprintf(stderr, "readBuffer()-- couldn't read "u64bitFMT" bytes from '%s': n%s\n",
- len, _filename, strerror(errno)), exit(1);
-
- // If we hit EOF, return a short read
- if (bAct == 0)
- len = 0;
-
- bRead += bAct;
- }
-
- fillBuffer();
-
- _filePos += bCopied + bRead;
-
- return(bCopied + bRead);
-}
-
-
-u64bit
-readBuffer::read(void *buf, u64bit maxlen, char stop) {
- char *bufchar = (char *)buf;
- u64bit c = 0;
-
- // We will copy up to 'maxlen'-1 bytes into 'buf', or stop at the first occurrence of 'stop'.
- // This will reserve space at the end of any string for a zero-terminating byte.
- maxlen--;
-
- if (_mmap) {
- // Handle the mmap'd file first.
- while ((_bufferPos < _bufferLen) &&
- (c < maxlen)) {
- bufchar[c++] = _buffer[_bufferPos++];
-
- if (bufchar[c-1] == stop)
- break;
- }
-
- if (_bufferPos >= _bufferLen)
- _eof = true;
-
- } else {
- // And the usual case.
- while ((_eof == false) && (c < maxlen)) {
- bufchar[c++] = _buffer[_bufferPos++];
-
- if (_bufferPos >= _bufferLen)
- fillBuffer();
-
- if (bufchar[c-1] == stop)
- break;
- }
- }
-
- bufchar[c] = 0;
-
- return(c);
-}
diff --git a/kmer/libutil/readBuffer.H b/kmer/libutil/readBuffer.H
deleted file mode 100644
index 1cf13f7..0000000
--- a/kmer/libutil/readBuffer.H
+++ /dev/null
@@ -1,86 +0,0 @@
-#ifndef READ_BUFFER_H
-#define READ_BUFFER_H
-
-
-class readBuffer {
-public:
- readBuffer(const char *filename, u64bit bufferMax = 32 * 1024);
- readBuffer(FILE *F, u64bit bufferMax = 32 * 1024);
- ~readBuffer();
-
- bool eof(void) { return(_eof); };
-
- char peek(void);
-
- char read(void);
- u64bit read(void *buf, u64bit len);
- u64bit read(void *buf, u64bit maxlen, char stop);
-
- void seek(u64bit pos);
- u64bit tell(void) { return(_filePos); };
-
- const char *filename(void) { return(_filename); };
-
-private:
- void fillBuffer(void);
- void init(int fileptr, const char *filename, u64bit bufferMax);
-
- char *_filename;
-
- int _file;
- u64bit _filePos;
-
- bool _mmap;
- bool _stdin;
-
- bool _eof;
-
- // If bufferMax is zero, then we are using the mmapped interface, otherwise,
- // we are using a open()/read() and a small buffer.
-
- u64bit _bufferPos;
- u64bit _bufferLen;
- u64bit _bufferMax;
- char *_buffer;
-};
-
-
-// Returns the next letter in the buffer, but DOES NOT advance past
-// it. Might have some wierd interaction with EOF -- if you peek()
-// and the next thing is eof , the _eof flag might get set.
-//
-inline
-char
-readBuffer::peek(void) {
-
- if ((_eof == false) && (_bufferPos >= _bufferLen))
- fillBuffer();
-
- if (_eof)
- return(0);
-
- return(_buffer[_bufferPos]);
-}
-
-
-// Returns the next letter in the buffer. Returns EOF (0) if there
-// is no next letter.
-//
-inline
-char
-readBuffer::read(void) {
-
- if ((_eof == false) && (_bufferPos >= _bufferLen))
- fillBuffer();
-
- if (_eof)
- return(0);
-
- _bufferPos++;
- _filePos++;
-
- return(_buffer[_bufferPos-1]);
-}
-
-
-#endif // READ_BUFFER_H
diff --git a/kmer/libutil/recordFile.C b/kmer/libutil/recordFile.C
deleted file mode 100644
index ff27795..0000000
--- a/kmer/libutil/recordFile.C
+++ /dev/null
@@ -1,320 +0,0 @@
-#include "util++.H"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <fcntl.h>
-
-// N.B. any read() / write() pair (either order) must have a seek (or
-// a fflush) in between.
-
-u64bit recordFileMagic1 = 0x694664726f636572llu;
-u64bit recordFileMagic2 = 0x000000000000656cllu;
-
-recordFile::recordFile(char const *name,
- u32bit headerSize,
- u32bit recordSize,
- char mode) {
-
- _file = 0;
- _name = new char [strlen(name) + 1];
- strcpy(_name, name);
-
- _numRecords = 0;
- _recordSize = recordSize;
-
- _headerSize = headerSize;
- _header = new char [_headerSize];
-
- memset(_header, 0, sizeof(char) * _headerSize);
-
- _bfrmax = MAX(1048576 / _recordSize, 16);
- _bfr = new char [_bfrmax * _recordSize];
-
- _limit = ~u32bitZERO;
-
- _pos = u64bitZERO;
- _rec = 0;
-
- memset(_bfr, 0, sizeof(char) * _bfrmax * _recordSize);
-
- _bfrDirty = false;
- _isReadOnly = true;
-
- if ((mode != 'r') && (mode != 'w') && (mode |= 'a')) {
- fprintf(stderr, "recordFile::recordFile()-- Invalid mode '%c'.\n", mode);
- exit(1);
- }
-
- // If the file doesn't exist, or we're opening for write, we're
- // basically done. Do that first.
- // Write the magic.
- // Write the metadata.
- // Write the header.
-
- if (((mode == 'w')) ||
- ((mode == 'a') && (fileExists(_name) == false))) {
- errno = 0;
- _file = open(_name,
- O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno)
- fprintf(stderr, "recordFile::recordFile()-- failed to open '%s': %s\n",
- _name, strerror(errno)), exit(1);
- _isReadOnly = false;
-
- write(_file, &recordFileMagic1, sizeof(u64bit));
- write(_file, &recordFileMagic2, sizeof(u64bit));
- write(_file, &_numRecords, sizeof(u64bit));
- write(_file, &_recordSize, sizeof(u32bit));
- write(_file, &_headerSize, sizeof(u32bit));
- write(_file, _header, sizeof(char) * _headerSize);
-
- if (errno)
- fprintf(stderr, "recordFile::recordFile()-- failed to write header to '%s': %s\n",
- _name, strerror(errno)), exit(1);
-
- return;
- }
-
- // File does exist. If we're not appending, open it read-only.
- // Otherwise, open read-write.
-
- if (mode == 'r') {
- errno = 0;
- _file = open(_name,
- O_RDONLY | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno)
- fprintf(stderr, "recordFile::recordFile()-- failed to open '%s': %s\n",
- _name, strerror(errno)), exit(1);
- _isReadOnly = true;
- } else {
- errno = 0;
- _file = open(_name,
- O_RDWR | O_LARGEFILE,
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
- if (errno)
- fprintf(stderr, "recordFile::recordFile()-- failed to open for write '%s': %s\n",
- _name, strerror(errno)), exit(1);
- _isReadOnly = false;
- }
-
- // Read the magic, metadata and header.
-
- {
- u64bit m1, m2;
-
- errno = 0;
-
- read(_file, &m1, sizeof(u64bit));
- read(_file, &m2, sizeof(u64bit));
- read(_file, &_numRecords, sizeof(u64bit));
- read(_file, &_recordSize, sizeof(u32bit));
- read(_file, &_headerSize, sizeof(u32bit));
- read(_file, _header, sizeof(char) * _headerSize);
-
- if (errno)
- fprintf(stderr, "recordFile::recordFile()-- failed to read header from '%s': %s\n",
- _name, strerror(errno)), exit(1);
-
- if ((m1 != recordFileMagic1) || (m2 != recordFileMagic2))
- fprintf(stderr, "recordFile::recordFile()-- magic number disagreement; '%s' not a recordFile?\n",
- _name), exit(1);
- }
-
- if (mode == 'a') {
- _pos = _numRecords;
- _rec = 0;
-
- errno = 0;
- lseek(_file, 0, SEEK_END);
- if (errno)
- fprintf(stderr, "recordFile::recordFile()-- seek to end of '%s' failed: %s\n", _name, strerror(errno)), exit(1);
- } else {
- seek(0, true);
- }
-}
-
-
-recordFile::~recordFile() {
- flushDirty();
-
- if (_isReadOnly == false) {
- errno = 0;
- lseek(_file, 0, SEEK_SET);
- if (errno)
- fprintf(stderr, "recordFile::~recordFile()-- seek to start of '%s' failed: %s\n", _name, strerror(errno)), exit(1);
-
- write(_file, &recordFileMagic1, sizeof(u64bit));
- write(_file, &recordFileMagic2, sizeof(u64bit));
- write(_file, &_numRecords, sizeof(u64bit));
- write(_file, &_recordSize, sizeof(u32bit));
- write(_file, &_headerSize, sizeof(u32bit));
- write(_file, _header, sizeof(char) * _headerSize);
-
- if (errno)
- fprintf(stderr, "recordFile::~recordFile()-- failed to write header to '%s': %s\n",
- _name, strerror(errno)), exit(1);
- }
-
- close(_file);
-
- if (errno)
- fprintf(stderr, "recordFile::~recordFile()-- failed to close '%s': %s\n",
- _name, strerror(errno)), exit(1);
-
- delete [] _bfr;
- delete [] _name;
- delete [] _header;
-}
-
-
-
-// If the page is dirty, flush it to disk
-//
-void
-recordFile::flushDirty(void) {
-
- if (_bfrDirty == false)
- return;
-
- if (_isReadOnly)
- fprintf(stderr, "recordFile::recordFile()-- '%s' is readonly, but is dirty!\n", _name), exit(1);
-
- errno = 0;
- lseek(_file, 32 + _headerSize + _pos * _recordSize, SEEK_SET);
- if (errno)
- fprintf(stderr, "recordFile::seek()-- '%s' failed: %s\n", _name, strerror(errno)), exit(1);
-
- // Write records up to, not including, _rec. Unlike the
- // bitPackedFile, there is no issue with partially filled words
- // here.
- //
- errno = 0;
- write(_file, _bfr, _recordSize * _rec);
- if (errno)
- fprintf(stderr, "recordFile::write()-- '%s' failed: %s\n", _name, strerror(errno)), exit(1);
-
- _bfrDirty = false;
-}
-
-
-
-// Seeks to rec in the file, reads in a new block.
-//
-void
-recordFile::seek(u64bit rec, bool forced) {
-
- // If we are seeking to somewhere in the current block, don't do a
- // real seek, just move our position within the block.
- //
- if ((forced == false) && (_pos <= rec) && (rec < _pos + _bfrmax)) {
- _rec = rec - _pos;
- return;
- }
-
- flushDirty();
-
- _pos = rec; // Root of buffer is now here
- _rec = 0; // See?
-
- errno = 0;
- lseek(_file, 32 + _headerSize + _pos * _recordSize, SEEK_SET);
- if (errno)
- fprintf(stderr, "recordFile::seek() '%s' seek to record="u64bitFMT" at fileposition="u64bitFMT" failed: %s\n",
- _name, _pos, _headerSize + _pos * _recordSize, strerror(errno)), exit(1);
-
- errno = 0;
- read(_file, _bfr, _recordSize * _bfrmax);
- if (errno)
- fprintf(stderr, "recordFile::seek() '%s' read of "u64bitFMT" bytes failed at record "u64bitFMT", fileposition "u64bitFMT"': %s\n",
- _name, _recordSize * _bfrmax, _pos, _headerSize + _pos * _recordSize, strerror(errno)), exit(1);
-}
-
-
-
-u32bit
-recordFile::getRecord(void *record, u32bit num) {
- u32bit maxnum = _bfrmax / 2;
-
- // Reading large blocks -- bigger than the in-core size? Loop and
- // recurse.
- //
- if (num > maxnum) {
- u32bit numread = 0;
- u32bit pos = 0;
- u32bit len = 0;
-
- while (num > 0) {
- len = MIN(maxnum, num);
- len = getRecord((char *)record + pos * _recordSize, len);
-
- if (len == 0)
- return(numread);
-
- num -= len;
- pos += len;
- numread += len;
- }
-
- return(numread);
- }
-
- // If asked to read too many records, read whatever is left.
- //
- if (_numRecords < _pos + _rec + num)
- num = _numRecords - _pos - _rec;
- if (_limit < _pos + _rec + num)
- num = _limit - _pos - _rec;
-
- // If the current position is already past eof, return without
- // reading. The previous 'if' ensures we will never read a block
- // past eof.
- //
- if ((_numRecords < _pos + _rec) || (_limit < _pos + _rec))
- return(0);
-
- if (_bfrmax < _rec + num + 1)
- seek(_pos + _rec, true);
-
- memcpy(record, _bfr + _rec * _recordSize, _recordSize * num);
-
- _rec += num;
-
- return(num);
-}
-
-
-
-void
-recordFile::putRecord(void *record, u32bit num) {
- u32bit maxnum = _bfrmax / 2;
-
- if (num > maxnum) {
- u32bit pos = 0;
- u32bit len = 0;
-
- while (num > 0) {
- len = MIN(maxnum, num);
-
- putRecord((char *)record + pos * _recordSize, len);
-
- num -= len;
- pos += len;
- }
-
- } else {
- if (_bfrmax < _rec + num + 1)
- seek(_pos + _rec, true);
-
- memcpy(_bfr + _rec * _recordSize, record, _recordSize * num);
-
- _rec += num;
- _numRecords += num;
-
- _bfrDirty = true;
- }
-}
diff --git a/kmer/libutil/recordFile.H b/kmer/libutil/recordFile.H
deleted file mode 100644
index 2d61f29..0000000
--- a/kmer/libutil/recordFile.H
+++ /dev/null
@@ -1,65 +0,0 @@
-#ifndef RECORDFILE_H
-#define RECORDFILE_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include "util.h"
-
-// A file of fixed size records, with an optional header at the
-// start. Derived from the bitPackedFile at SVN-1533, but heavily
-// modified. Records can only be added, not updated (probably
-// trivial to fix). Records must be dense (also probably trivial to
-// fix).
-
-class recordFile {
-public:
- recordFile(char const *name,
- u32bit headerSize,
- u32bit recordSize,
- char mode);
- ~recordFile();
-
- void *header(void) { return(_header); };
-
- u64bit numRecords(void) { return(_numRecords); };
-
- // Read/write records.
- u32bit getRecord(void *record, u32bit num=1);
- void putRecord(void *record, u32bit num=1);
-
- // Seek to record rec, optionally repositioning the buffer to that
- // record.
- void seek(u64bit rec, bool forced=false);
-
- // Set an artificial EOF at record rec.
- void limit(u64bit rec) { _limit = rec; };
-
-private:
- void flushDirty(void);
-
- int _file;
- char *_name;
-
- u64bit _numRecords;
- u32bit _recordSize;
-
- u32bit _headerSize;
- char *_header;
-
- u64bit _bfrmax; // Number of records in the buffer
- char *_bfr; // A chunk of the bitPackedFile in core
-
- u64bit _limit; // An artificial EOF
-
- u64bit _pos; // The location this chunk is from (in records)
- u64bit _rec; // The record we're modifying relative to _pos
-
- bool _bfrDirty;
- bool _isReadOnly;
-};
-
-#endif // RECORDFILE_H
-
-
-
diff --git a/kmer/libutil/speedCounter.C b/kmer/libutil/speedCounter.C
deleted file mode 100644
index 75a9371..0000000
--- a/kmer/libutil/speedCounter.C
+++ /dev/null
@@ -1,61 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "util++.H"
-
-const char*
-speedCounter::_spinr[4] = { "[|]", "[/]", "[-]", "[\\]" };
-
-const char*
-speedCounter::_liner[19] = { "[- ]",
- "[-- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ --]",
- "[ -]",
- "[ --]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]",
- "[ -- ]" };
-
-
-speedCounter::speedCounter(char const *fmt,
- double unit,
- u64bit freq,
- bool enabled) {
- _count = 0;
- _draws = 0;
- _unit = unit;
- _freq = freq;
- _startTime = getTime();
- _fmt = fmt;
- _spin = false;
- _line = false;
- _enabled = enabled;
-
- // We use _draws instead of shifting _count just because it's
- // simpler, and both methods need another variable anyway.
-
- // Set all the bits below the hightest set in _freq --
- // this allows us to do a super-fast test in tick().
- //
- _freq |= _freq >> 1;
- _freq |= _freq >> 2;
- _freq |= _freq >> 4;
- _freq |= _freq >> 8;
- _freq |= _freq >> 16;
- _freq |= _freq >> 32;
-}
-
-speedCounter::~speedCounter() {
- finish();
-}
diff --git a/kmer/libutil/speedCounter.H b/kmer/libutil/speedCounter.H
deleted file mode 100644
index 547f01e..0000000
--- a/kmer/libutil/speedCounter.H
+++ /dev/null
@@ -1,77 +0,0 @@
-#ifndef SPEEDCOUNTER_H
-#define SPEEDCOUNTER_H
-
-#include <stdio.h>
-
-class speedCounter {
-public:
- // fmt specifies the status format. An example:
- // " %8f [unit]things (%8.5f [unit]things/sec)\r"
- //
- speedCounter(char const *fmt,
- double unit,
- u64bit freq,
- bool enabled=true);
- ~speedCounter();
-
- void enableSpinner(void) { _spin = true; };
- void enableLiner(void) { _line = true; };
-
- bool tick(void) {
- if (_enabled && ((++_count & _freq) == u64bitZERO)) {
- double v = _count / _unit;
- if (_spin) fputs(_spinr[_draws % 4], stderr);
- if (_line) fputs(_liner[_draws % 19], stderr);
- _draws++;
- fprintf(stderr, _fmt, v, v / (getTime() - _startTime));
- fflush(stderr);
- return(true);
- }
- return(false);
- };
-
- bool tick(u64bit increment) {
- if (_enabled == false)
- return(false);
-
- _count += increment;
- if ((_count & _freq) == u64bitZERO) {
- double v = _count / _unit;
- if (_spin) fputs(_spinr[_draws % 4], stderr);
- if (_line) fputs(_liner[_draws % 19], stderr);
- _draws++;
- fprintf(stderr, _fmt, v, v / (getTime() - _startTime));
- fflush(stderr);
- return(true);
- }
- return(false);
- };
-
- void finish(void) {
- if (_enabled && (_count >= _freq)) {
- double v = _count / _unit;
- if (_spin) fputs(_spinr[_draws % 4], stderr);
- if (_line) fputs(_liner[_draws % 19], stderr);
- fprintf(stderr, _fmt, v, v / (getTime() - _startTime));
- fprintf(stderr, "\n");
- fflush(stderr);
- }
- _count = 0;
- };
-
-private:
- static const char *_spinr[4];
- static const char *_liner[19];
- u64bit _count;
- u64bit _draws;
- double _unit;
- u64bit _freq;
- double _startTime;
- char const *_fmt;
- bool _spin;
- bool _line;
- bool _enabled;
-};
-
-
-#endif // SPEEDCOUNTER_H
diff --git a/kmer/libutil/splitToWords.H b/kmer/libutil/splitToWords.H
deleted file mode 100644
index 90306ff..0000000
--- a/kmer/libutil/splitToWords.H
+++ /dev/null
@@ -1,117 +0,0 @@
-#ifndef SPLITTOWORDS_H
-#define SPLITTOWORDS_H
-
-
-class splitToWords {
-public:
- splitToWords() {
- _argWords = 0;
- _maxWords = 0;
- _arg = 0L;
- _maxChars = 0;
- _cmd = 0L;
- };
- splitToWords(char *cmd) {
- _argWords = 0;
- _maxWords = 0;
- _arg = 0L;
- _maxChars = 0;
- _cmd = 0L;
-
- split(cmd);
- };
- ~splitToWords() {
- delete [] _cmd;
- delete [] _arg;
- };
-
-
- void split(char *cmd) {
-
- // Step Zero:
- //
- // Count the length of the string, in words and in characters.
- // For simplicity, we overcount words, by just counting white-space.
- //
- // Then, allocate space for a temporary copy of the string, and a
- // set of pointers into the temporary copy (much like argv).
- //
- u32bit cmdChars = 1; // 1 == Space for terminating 0
- u32bit cmdWords = 2; // 2 == Space for first word and terminating 0L
-
- for (char *tmp=cmd; *tmp; tmp++) {
- cmdWords += *tmp == ' ';
- cmdWords += *tmp == '\t';
- cmdChars++;
- }
-
- if (cmdChars > _maxChars) {
- delete [] _cmd;
- _cmd = new char [cmdChars];
- _maxChars = cmdChars;
- }
- if (cmdWords > _maxWords) {
- delete [] _arg;
- _arg = new char * [cmdWords];
- _maxWords = cmdWords;
- }
-
- _argWords = 0;
-
- // Step One:
- //
- // Determine where the words are in the command string, copying the
- // string to _cmd and storing words in _arg.
- //
- bool isFirst = true;
- char *cmdI = cmd;
- char *cmdO = _cmd;
-
- while (*cmdI) {
-
- // If we are at a non-space character, we are in a word. If
- // this is the first character in the word, save the word in
- // the args list.
- //
- // Otherwise we are at a space and thus not in a word. Make
- // all spaces be string terminators, and declare that we are
- // at the start of a word.
- //
- if ((*cmdI != ' ') && (*cmdI != '\t')) {
- *cmdO = *cmdI;
-
- if (isFirst) {
- _arg[_argWords++] = cmdO;
- isFirst = false;
- }
- } else {
- *cmdO = 0;
- isFirst = true;
- }
-
- cmdI++;
- cmdO++;
- }
-
- // Finish off the list by terminating the last arg, and
- // terminating the list of args.
- //
- *cmdO = 0;
- _arg[_argWords] = 0L;
- };
-
-
- u32bit numWords(void) { return(_argWords); };
- char *getWord(u32bit i) { return(_arg[i]); };
- char *operator[](u32bit i) { return(_arg[i]); };
- s64bit operator()(u32bit i) { return(strtoull(_arg[i], NULL, 10)); };
-private:
- u32bit _argWords;
- u32bit _maxWords;
- char **_arg;
- u32bit _maxChars;
- char *_cmd;
-};
-
-
-#endif // SPLITTOWORDS_H
diff --git a/kmer/libutil/sweatShop.C b/kmer/libutil/sweatShop.C
deleted file mode 100644
index 5faaf65..0000000
--- a/kmer/libutil/sweatShop.C
+++ /dev/null
@@ -1,587 +0,0 @@
-#include "sweatShop.H"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <sys/time.h>
-#include <time.h>
-
-#include <sched.h> // pthread scheduling stuff
-
-
-
-class sweatShopWorker {
-public:
- sweatShopWorker() {
- shop = 0L;
- threadUserData = 0L;
- numComputed = 0;
- workerQueue = 0L;
- workerQueueLen = 0L;
- };
-
- sweatShop *shop;
- void *threadUserData;
- pthread_t threadID;
- u32bit numComputed;
- sweatShopState **workerQueue;
- u32bit workerQueueLen;
-};
-
-
-// This gets created by the loader, passed to the worker, and printed
-// by the writer. userData is controlled by the user.
-//
-class sweatShopState {
-public:
- sweatShopState(void *userData) {
- _user = userData;
- _computed = false;
- _next = 0L;
- };
- ~sweatShopState() {
- };
-
- void *_user;
- bool _computed;
- sweatShopState *_next;
-};
-
-
-
-
-// Simply forwards control to the class
-void*
-_sweatshop_loaderThread(void *ss_) {
- sweatShop *ss = (sweatShop *)ss_;
- return(ss->loader());
-}
-
-void*
-_sweatshop_workerThread(void *sw_) {
- sweatShopWorker *sw = (sweatShopWorker *)sw_;
- return(sw->shop->worker(sw));
-}
-
-void*
-_sweatshop_writerThread(void *ss_) {
- sweatShop *ss = (sweatShop *)ss_;
- return(ss->writer());
-}
-
-void*
-_sweatshop_statusThread(void *ss_) {
- sweatShop *ss = (sweatShop *)ss_;
- return(ss->status());
-}
-
-
-
-sweatShop::sweatShop(void*(*loaderfcn)(void *G),
- void (*workerfcn)(void *G, void *T, void *S),
- void (*writerfcn)(void *G, void *S)) {
-
- _userLoader = loaderfcn;
- _userWorker = workerfcn;
- _userWriter = writerfcn;
-
- _globalUserData = 0L;
-
- _writerP = 0L;
- _workerP = 0L;
- _loaderP = 0L;
-
- _showStatus = false;
-
- _loaderQueueSize = 1024;
- _loaderQueueMax = 10240;
- _loaderQueueMin = 4; // _numberOfWorkers * 2, reset when that changes
- _loaderBatchSize = 1;
- _workerBatchSize = 1;
- _writerQueueSize = 4096;
- _writerQueueMax = 10240;
-
- _numberOfWorkers = 2;
-
- _workerData = 0L;
-
- _numberLoaded = 0;
- _numberComputed = 0;
- _numberOutput = 0;
-}
-
-
-sweatShop::~sweatShop() {
- delete [] _workerData;
-}
-
-
-
-void
-sweatShop::setThreadData(u32bit t, void *x) {
- if (_workerData == 0L)
- _workerData = new sweatShopWorker [_numberOfWorkers];
-
- if (t >= _numberOfWorkers)
- fprintf(stderr, "sweatShop::setThreadData()-- worker ID "u32bitFMT" more than number of workers="u32bitFMT"\n", t, _numberOfWorkers), exit(1);
-
- _workerData[t].threadUserData = x;
-}
-
-
-
-// Build a list of states to add in one swoop
-//
-void
-sweatShop::loaderSave(sweatShopState *&tail, sweatShopState *&head, sweatShopState *thisState) {
-
- thisState->_next = 0L;
-
- if (tail) {
- head->_next = thisState;
- head = thisState;
- } else {
- tail = head = thisState;
- }
- _numberLoaded++;
-}
-
-
-// Add a bunch of new states to the queue.
-//
-void
-sweatShop::loaderAppend(sweatShopState *&tail, sweatShopState *&head) {
- int err;
-
- if ((tail == 0L) || (head == 0L))
- return;
-
- err = pthread_mutex_lock(&_stateMutex);
- if (err != 0)
- fprintf(stderr, "sweatShop::loaderAppend()-- Failed to lock mutex (%d). Fail.\n", err), exit(1);
-
- if (_loaderP == 0L) {
- _writerP = tail;
- _workerP = tail;
- _loaderP = head;
- } else {
- _loaderP->_next = tail;
- }
- _loaderP = head;
-
- err = pthread_mutex_unlock(&_stateMutex);
- if (err != 0)
- fprintf(stderr, "sweatShop::loaderAppend()-- Failed to unlock mutex (%d). Fail.\n", err), exit(1);
-
- tail = 0L;
- head = 0L;
-}
-
-
-
-void*
-sweatShop::loader(void) {
-
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 166666666ULL; // 1/6 second
-
- // We can batch several loads together before we push them onto the
- // queue, this should reduce the number of times the loader needs to
- // lock the queue.
- //
- // But it also increases the latency, so it's disabled by default.
- //
- sweatShopState *tail = 0L; // The first thing loaded
- sweatShopState *head = 0L; // The last thing loaded
- u32bit numLoaded = 0;
-
- bool moreToLoad = true;
-
- while (moreToLoad) {
-
- // Zzzzzzz....
- while (_numberLoaded > _numberComputed + _loaderQueueSize)
- nanosleep(&naptime, 0L);
-
- sweatShopState *thisState = new sweatShopState((*_userLoader)(_globalUserData));
-
- // If we actually loaded a new state, add it
- //
- if (thisState->_user) {
- loaderSave(tail, head, thisState);
- numLoaded++;
- if (numLoaded >= _loaderBatchSize)
- loaderAppend(tail, head);
- } else {
- // Didn't read, must be all done! Push on the end-of-input marker state.
- //
- loaderSave(tail, head, new sweatShopState(0L));
- loaderAppend(tail, head);
-
- moreToLoad = false;
- delete thisState;
- }
- }
-
- //fprintf(stderr, "sweatShop::reader exits.\n");
- return(0L);
-}
-
-
-
-void*
-sweatShop::worker(sweatShopWorker *workerData) {
-
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 50000000ULL;
-
- bool moreToCompute = true;
- int err;
-
- while (moreToCompute) {
-
- // Usually beacuse some worker is taking a long time, and the
- // output queue isn't big enough.
- //
- while (_numberOutput + _writerQueueSize < _numberComputed)
- nanosleep(&naptime, 0L);
-
- // Grab the next state. We don't grab it if it's the last in the
- // queue (else we would fall off the end) UNLESS it really is the
- // last one.
- //
- err = pthread_mutex_lock(&_stateMutex);
- if (err != 0)
- fprintf(stderr, "sweatShop::worker()-- Failed to lock mutex (%d). Fail.\n", err), exit(1);
-
- for (workerData->workerQueueLen = 0; ((workerData->workerQueueLen < _workerBatchSize) &&
- (_workerP) &&
- ((_workerP->_next != 0L) || (_workerP->_user == 0L))); workerData->workerQueueLen++) {
- workerData->workerQueue[workerData->workerQueueLen] = _workerP;
- _workerP = _workerP->_next;
- }
-
- if (_workerP == 0L)
- moreToCompute = false;
-
- err = pthread_mutex_unlock(&_stateMutex);
- if (err != 0)
- fprintf(stderr, "sweatShop::worler()-- Failed to lock mutex (%d). Fail.\n", err), exit(1);
-
-
- if (workerData->workerQueueLen == 0) {
- // No work, sleep a bit to prevent thrashing the mutex and resume.
- nanosleep(&naptime, 0L);
- continue;
- }
-
- // Execute
- //
- for (u32bit x=0; x<workerData->workerQueueLen; x++) {
- sweatShopState *ts = workerData->workerQueue[x];
-
- if (ts && ts->_user) {
- (*_userWorker)(_globalUserData, workerData->threadUserData, ts->_user);
- ts->_computed = true;
- workerData->numComputed++;
- } else {
- // When we really do run out of stuff to do, we'll end up here
- // (only one thread will end up in the other case, with
- // something to do and moreToCompute=false). If it's actually
- // the end, skip the sleep and just get outta here.
- //
- if (moreToCompute == true) {
- fprintf(stderr, "WARNING! Worker is sleeping because the reader is slow!\n");
- nanosleep(&naptime, 0L);
- }
- }
- }
- }
-
- //fprintf(stderr, "sweatShop::worker exits.\n");
- return(0L);
-}
-
-
-void*
-sweatShop::writer(void) {
- sweatShopState *deleteState = 0L;
-
- // Wait for output to appear, then write.
- //
- while (_writerP && _writerP->_user) {
-
- if (_writerP->_computed == false) {
- // Wait for a slow computation.
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 5000000ULL;
-
- //fprintf(stderr, "Writer waits for slow thread at "u64bitFMT".\n", _numberOutput);
- nanosleep(&naptime, 0L);
- } else if (_writerP->_next == 0L) {
- // Wait for the input.
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 5000000ULL;
-
- //fprintf(stderr, "Writer waits for all threads at "u64bitFMT".\n", _numberOutput);
- nanosleep(&naptime, 0L);
- } else {
- (*_userWriter)(_globalUserData, _writerP->_user);
- _numberOutput++;
-
- deleteState = _writerP;
- _writerP = _writerP->_next;
- delete deleteState;
- }
- }
-
- // Tell status to stop.
- _writerP = 0L;
-
- //fprintf(stderr, "sweatShop::writer exits.\n");
- return(0L);
-}
-
-
-// This thread not only shows a status message, but it also updates the critical shared variable
-// _numberComputed. Worker threads use this to throttle themselves. Thus, even if _showStatus is
-// not set, and this thread doesn't _appear_ to be doing anything useful....it is.
-//
-void*
-sweatShop::status(void) {
-
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 250000000ULL;
-
- double startTime = getTime() - 0.001;
- double thisTime = 0;
-
- u64bit deltaOut = 0;
- u64bit deltaCPU = 0;
-
- double cpuPerSec = 0;
-
- u64bit readjustAt = 16384;
-
- while (_writerP) {
- u32bit nc = 0;
- for (u32bit i=0; i<_numberOfWorkers; i++)
- nc += _workerData[i].numComputed;
- _numberComputed = nc;
-
- deltaOut = deltaCPU = 0;
-
- thisTime = getTime();
-
- if (_numberComputed > _numberOutput)
- deltaOut = _numberComputed - _numberOutput;
- if (_numberLoaded > _numberComputed)
- deltaCPU = _numberLoaded - _numberComputed;
-
- cpuPerSec = _numberComputed / (thisTime - startTime);
-
- if (_showStatus) {
- fprintf(stderr, " %6.1f/s - "u64bitFMTW(8)" loaded; "u64bitFMTW(8)" queued for compute; "u64bitFMTW(8)" finished; "u64bitFMTW(8)" written; "u64bitFMTW(8)" queued for output)\r",
- cpuPerSec, _numberLoaded, deltaCPU, _numberComputed, _numberOutput, deltaOut);
- fflush(stderr);
- }
-
- // Readjust queue sizes based on current performance, but don't let it get too big or small.
- // In particular, don't let it get below 2*numberOfWorkers.
- //
- if (_numberComputed > readjustAt) {
- readjustAt += (u64bit)(2 * cpuPerSec);
- _loaderQueueSize = (u32bit)(5 * cpuPerSec);
- }
-
- if (_loaderQueueSize < _loaderQueueMin)
- _loaderQueueSize = _loaderQueueMin;
-
- if (_loaderQueueSize < 2 * _numberOfWorkers)
- _loaderQueueSize = 2 * _numberOfWorkers;
-
- if (_loaderQueueSize > _loaderQueueMax)
- _loaderQueueSize = _loaderQueueMax;
-
- nanosleep(&naptime, 0L);
- }
-
- if (_showStatus) {
- thisTime = getTime();
-
- if (_numberComputed > _numberOutput)
- deltaOut = _numberComputed - _numberOutput;
- if (_numberLoaded > _numberComputed)
- deltaCPU = _numberLoaded - _numberComputed;
-
- cpuPerSec = _numberComputed / (thisTime - startTime);
-
- fprintf(stderr, " %6.1f/s - "u64bitFMTW(8)" queued for compute; "u64bitFMTW(8)" finished; "u64bitFMTW(8)" queued for output)\n",
- cpuPerSec, deltaCPU, _numberComputed, deltaOut);
- }
-
- //fprintf(stderr, "sweatShop::status exits.\n");
- return(0L);
-}
-
-
-
-
-
-void
-sweatShop::run(void *user, bool beVerbose) {
- pthread_attr_t threadAttr;
- pthread_t threadIDloader;
- pthread_t threadIDwriter;
- pthread_t threadIDstats;
-#if 0
- int threadSchedPolicy = 0;
- struct sched_param threadSchedParamDef;
- struct sched_param threadSchedParamMax;
-#endif
- int err = 0;
-
- _globalUserData = user;
- _showStatus = beVerbose;
-
- // Configure everything ahead of time.
-
- if (_workerBatchSize < 1)
- _workerBatchSize = 1;
-
- if (_workerData == 0L)
- _workerData = new sweatShopWorker [_numberOfWorkers];
-
- for (u32bit i=0; i<_numberOfWorkers; i++) {
- _workerData[i].shop = this;
- _workerData[i].workerQueue = new sweatShopState * [_workerBatchSize];
- }
-
- // Open the doors.
-
- errno = 0;
-
- err = pthread_mutex_init(&_stateMutex, NULL);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (state mutex): %s.\n", strerror(err)), exit(1);
-
- err = pthread_attr_init(&threadAttr);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (attr init): %s.\n", strerror(err)), exit(1);
-
- err = pthread_attr_setscope(&threadAttr, PTHREAD_SCOPE_SYSTEM);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (set scope): %s.\n", strerror(err)), exit(1);
-
- err = pthread_attr_setdetachstate(&threadAttr, PTHREAD_CREATE_JOINABLE);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (joinable): %s.\n", strerror(err)), exit(1);
-
-#if 0
- err = pthread_attr_getschedparam(&threadAttr, &threadSchedParamDef);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (get default param): %s.\n", strerror(err)), exit(1);
-
- err = pthread_attr_getschedparam(&threadAttr, &threadSchedParamMax);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (get max param): %s.\n", strerror(err)), exit(1);
-#endif
-
- // SCHED_RR needs root privs to run on FreeBSD.
- //
- //err = pthread_attr_setschedpolicy(&threadAttr, SCHED_RR);
- //if (err)
- // fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (sched policy): %s.\n", strerror(err)), exit(1);
-
-#if 0
- err = pthread_attr_getschedpolicy(&threadAttr, &threadSchedPolicy);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (sched policy): %s.\n", strerror(err)), exit(1);
-
- errno = 0;
- threadSchedParamMax.sched_priority = sched_get_priority_max(threadSchedPolicy);
- if (errno)
- fprintf(stderr, "sweatShop::run()-- WARNING: Failed to configure pthreads (set max param priority): %s.\n", strerror(errno));
-
- // Fire off the loader
-
- err = pthread_attr_setschedparam(&threadAttr, &threadSchedParamMax);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to set loader priority: %s.\n", strerror(err)), exit(1);
-#endif
-
- err = pthread_create(&threadIDloader, &threadAttr, _sweatshop_loaderThread, this);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to launch loader thread: %s.\n", strerror(err)), exit(1);
-
- // Wait for it to actually load something (otherwise all the
- // workers immediately go home)
-
- while (!_writerP && !_workerP && !_loaderP) {
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 250000ULL;
- nanosleep(&naptime, 0L);
- }
-
- // Start the statistics and writer
-
-#if 0
- err = pthread_attr_setschedparam(&threadAttr, &threadSchedParamMax);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to set status and writer priority: %s.\n", strerror(err)), exit(1);
-#endif
-
- err = pthread_create(&threadIDstats, &threadAttr, _sweatshop_statusThread, this);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to launch status thread: %s.\n", strerror(err)), exit(1);
-
- err = pthread_create(&threadIDwriter, &threadAttr, _sweatshop_writerThread, this);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to launch writer thread: %s.\n", strerror(err)), exit(1);
-
- // And some labor
-
-#if 0
- err = pthread_attr_setschedparam(&threadAttr, &threadSchedParamDef);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to set worker priority: %s.\n", strerror(err)), exit(1);
-#endif
-
- for (u32bit i=0; i<_numberOfWorkers; i++) {
- err = pthread_create(&_workerData[i].threadID, &threadAttr, _sweatshop_workerThread, _workerData + i);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to launch worker thread "u32bitFMT": %s.\n", i, strerror(err)), exit(1);
- }
-
- // Now sit back and relax.
-
- err = pthread_join(threadIDloader, 0L);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to join loader thread: %s.\n", strerror(err)), exit(1);
-
- err = pthread_join(threadIDwriter, 0L);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to join writer thread: %s.\n", strerror(err)), exit(1);
-
- err = pthread_join(threadIDstats, 0L);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to join status thread: %s.\n", strerror(err)), exit(1);
-
- for (u32bit i=0; i<_numberOfWorkers; i++) {
- err = pthread_join(_workerData[i].threadID, 0L);
- if (err)
- fprintf(stderr, "sweatShop::run()-- Failed to join worker thread "u32bitFMT": %s.\n", i, strerror(err)), exit(1);
- }
-
- // Cleanup.
-
- delete _loaderP;
- _loaderP = _workerP = _writerP = 0L;
-}
diff --git a/kmer/libutil/sweatShop.H b/kmer/libutil/sweatShop.H
deleted file mode 100644
index d525934..0000000
--- a/kmer/libutil/sweatShop.H
+++ /dev/null
@@ -1,81 +0,0 @@
-#ifndef SWEATSHOP_H
-#define SWEATSHOP_H
-
-#include <pthread.h>
-#include <semaphore.h>
-
-#include "util++.H"
-
-class sweatShopWorker;
-class sweatShopState;
-
-class sweatShop {
-public:
- sweatShop(void*(*loaderfcn)(void *G),
- void (*workerfcn)(void *G, void *T, void *S),
- void (*writerfcn)(void *G, void *S));
- ~sweatShop();
-
- void setNumberOfWorkers(u32bit x) {
- _numberOfWorkers = x;
- _loaderQueueMin = x * 2;
- };
-
- void setThreadData(u32bit t, void *x);
-
- void setLoaderBatchSize(u32bit batchSize) { _loaderBatchSize = batchSize; };
- void setLoaderQueueSize(u32bit queueSize) { _loaderQueueSize = queueSize; _loaderQueueMax = queueSize; };
-
- void setWorkerBatchSize(u32bit batchSize) { _workerBatchSize = batchSize; };
-
- void setWriterQueueSize(u32bit queueSize) { _writerQueueSize = queueSize; _writerQueueMax = queueSize; };
-
- void run(void *user=0L, bool beVerbose=false);
-private:
-
- // Stubs that forward control from the c-based pthread to this class
- friend void *_sweatshop_loaderThread(void *ss);
- friend void *_sweatshop_workerThread(void *ss);
- friend void *_sweatshop_writerThread(void *ss);
- friend void *_sweatshop_statusThread(void *ss);
-
- // The threaded routines
- void *loader(void);
- void *worker(sweatShopWorker *workerData);
- void *writer(void);
- void *status(void);
-
- // Utilities for the loader thread
- //void loaderAdd(sweatShopState *thisState);
- void loaderSave(sweatShopState *&tail, sweatShopState *&head, sweatShopState *thisState);
- void loaderAppend(sweatShopState *&tail, sweatShopState *&head);
-
- pthread_mutex_t _stateMutex;
-
- void *(*_userLoader)(void *global);
- void (*_userWorker)(void *global, void *thread, void *thing);
- void (*_userWriter)(void *global, void *thing);
-
- void *_globalUserData;
-
- sweatShopState *_writerP; // Where output takes stuff from, the tail
- sweatShopState *_workerP; // Where computes happen, the middle
- sweatShopState *_loaderP; // Where input is put, the head
-
- bool _showStatus;
-
- u32bit _loaderQueueSize, _loaderQueueMin, _loaderQueueMax;
- u32bit _loaderBatchSize;
- u32bit _workerBatchSize;
- u32bit _writerQueueSize, _writerQueueMax;
-
- u32bit _numberOfWorkers;
-
- sweatShopWorker *_workerData;
-
- u64bit _numberLoaded;
- u64bit _numberComputed;
- u64bit _numberOutput;
-};
-
-#endif // SWEATSHOP_H
diff --git a/kmer/libutil/test/Makefile b/kmer/libutil/test/Makefile
deleted file mode 100644
index c686ee1..0000000
--- a/kmer/libutil/test/Makefile
+++ /dev/null
@@ -1,106 +0,0 @@
-PROG = test-bigQueue \
- test-bitPackedArray \
- test-bitPackedFile \
- test-bitPacking \
- test-freeDiskSpace \
- test-intervalList \
- test-logMsg \
- test-md5 \
- test-mmap \
- test-palloc \
- test-readBuffer \
- test-recordFile \
- test-types \
- tcat
-
-# Broken, don't test.
-#test-bzipBuffer
-
-INCLUDE = -I..
-LIBS = -L.. -lutil -lm
-OBJS =
-
-include ../../Make.compilers
-
-all: $(PROG)
- @echo Tests passed!
-
-test-bigQueue: test-bigQueue.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-bigQueue.o test-bigQueue.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-bigQueue test-bigQueue.o $(LIBS)
- time ./test-bigQueue
- time sort -k2n < junk-bigQueue-out-2 > junk-bigQueue-out-2.s
- diff junk-bigQueue-out-1 junk-bigQueue-out-2.s
-
-test-bitPackedArray: test-bitPackedArray.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-bitPackedArray.o test-bitPackedArray.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-bitPackedArray test-bitPackedArray.o $(LIBS)
- ./test-bitPackedArray
-
-test-bitPackedFile: test-bitPackedFile.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-bitPackedFile.o test-bitPackedFile.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-bitPackedFile test-bitPackedFile.o $(LIBS) -lbz2
- ./test-bitPackedFile
-
-test-bitPacking: test-bitPacking.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-bitPacking.o test-bitPacking.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-bitPacking test-bitPacking.o $(LIBS)
- ./test-bitPacking
-
-test-freeDiskSpace: test-freeDiskSpace.c ../libutil.a
- $(CC) $(CFLAGS_COMPILE) -c -o test-freeDiskSpace.o test-freeDiskSpace.c $(INCLUDE)
- $(CC) $(CLDFLAGS) -o test-freeDiskSpace test-freeDiskSpace.o $(LIBS)
- ./test-freeDiskSpace test-freeDiskSpace.c
-
-test-intervalList: test-intervalList.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-intervalList.o test-intervalList.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-intervalList test-intervalList.o $(LIBS)
- ./test-intervalList
-
-test-logMsg: test-logMsg.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-logMsg.o test-logMsg.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-logMsg test-logMsg.o $(LIBS)
- ./test-logMsg | wc
-
-test-md5: test-md5.c ../libutil.a ../util.h ../util++.H
- $(CC) $(CFLAGS_COMPILE) -c -o test-md5.o test-md5.c $(INCLUDE)
- $(CC) $(CLDFLAGS) -o test-md5 test-md5.o $(LIBS)
- ./test-md5
-
-test-mmap: test-mmap.c ../libutil.a ../util.h ../util++.H
- $(CC) $(CFLAGS_COMPILE) -c -o test-mmap.o test-mmap.c $(INCLUDE)
- $(CC) $(CLDFLAGS) -o test-mmap test-mmap.o $(LIBS)
- ./test-mmap 16
-
-test-palloc: test-palloc.c ../libutil.a ../util.h ../util++.H
- $(CC) $(CFLAGS_COMPILE) -c -o test-palloc.o test-palloc.c $(INCLUDE)
- $(CC) $(CLDFLAGS) -o test-palloc test-palloc.o $(LIBS)
- ./test-palloc
-
-test-readBuffer: test-readBuffer.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-readBuffer.o test-readBuffer.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-readBuffer test-readBuffer.o $(LIBS)
- ./test-readBuffer test-readBuffer
-
-test-recordFile: test-recordFile.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-recordFile.o test-recordFile.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-recordFile test-recordFile.o $(LIBS)
- ./test-recordFile
-
-test-bzipBuffer: test-bzipBuffer.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o test-bzipBuffer.o test-bzipBuffer.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o test-bzipBuffer test-bzipBuffer.o -lbz2 $(LIBS)
- bzip2 -9vc ./test-bzipBuffer > ./test-bzipBuffer.bz2
- ./test-bzipBuffer ./test-bzipBuffer.bz2
-
-test-types: test-types.c ../libutil.a ../util.h ../util++.H
- $(CC) $(CFLAGS_COMPILE) -c -o test-types.o test-types.c $(INCLUDE)
- $(CC) $(CLDFLAGS) -o test-types test-types.o $(LIBS)
- ./test-types
-
-tcat: tcat.C ../libutil.a ../util.h ../util++.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o tcat.o tcat.C $(INCLUDE) $(THREADS)
- $(CXX) $(CXXLDFLAGS) -o tcat tcat.o $(LIBS) $(THREADL)
-
-clean:
- rm -f $(PROG) *.o *junk*
diff --git a/kmer/libutil/test/atomic.C b/kmer/libutil/test/atomic.C
deleted file mode 100644
index c9acd3e..0000000
--- a/kmer/libutil/test/atomic.C
+++ /dev/null
@@ -1,62 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <math.h>
-#include <time.h>
-#include <sys/time.h>
-
-#include <pthread.h>
-#include <semaphore.h>
-
-// Tests if add is atomic.
-
-long int count = 0;
-long int counts[8] = { 0 };
-pthread_t threadID[8];
-
-double
-getTime(void) {
- struct timeval tp;
- gettimeofday(&tp, NULL);
- return(tp.tv_sec + (double)tp.tv_usec / 1000000.0);
-}
-
-void*
-workerThread(void *idx) {
- long int &c = (*(long int *)idx);
-
- double start = getTime();
- while (getTime() - start < 5) {
- c++; count++;
- c++; count++;
- c++; count++;
- c++; count++;
- c++; count++;
- c++; count++;
- }
-}
-
-int
-main(int argc, char **argv) {
- pthread_attr_t threadAttr;
-
- pthread_attr_init(&threadAttr);
- pthread_attr_setscope(&threadAttr, PTHREAD_SCOPE_SYSTEM);
- pthread_attr_setschedpolicy(&threadAttr, SCHED_OTHER);
-
- int numThreads = 5;
- int sum = 0;
-
- for (int i=0; i<numThreads; i++)
- pthread_create(threadID+i, &threadAttr, workerThread, (void *)(counts + i));
-
- for (int i=0; i<numThreads; i++) {
- pthread_join(threadID[i], 0L);
- fprintf(stderr, "thread %2ld %ld\n", i, counts[i]);
- sum += counts[i];
- }
-
- fprintf(stderr, "total sum: %ld total global: %ld\n", sum, count);
-}
-
-
diff --git a/kmer/libutil/test/endianess.c b/kmer/libutil/test/endianess.c
deleted file mode 100644
index bb1dba0..0000000
--- a/kmer/libutil/test/endianess.c
+++ /dev/null
@@ -1,124 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/param.h> // BYTE_ORDER
-
-#include "/home/work/src/genomics/libutil/util.h"
-
-// Reports the byte order, writes words to files for testing.
-
-#if 0
-
-begin 644 test-alpha
-.`@$$`P(!"`<&!00#`@$`
-`
-end
-begin 644 test-i386
-.`@$$`P(!"`<&!00#`@$`
-`
-end
-begin 644 test-opteron
-.`@$$`P(!"`<&!00#`@$`
-`
-end
-begin 644 test-power
-.`0(!`@,$`0(#!`4&!P@`
-`
-end
-
-#endif
-
-
-int
-isBig1(void) {
- u64bit l = u64bitONE;
-
- if (*((char *)(&l)) == 1)
- return(0);
- return(1);
-}
-
-
-// supposedly due to Harbison and Steele
-int
-isBig2(void) {
- union {
- u64bit l;
- char c[sizeof(u64bit)];
- } u;
-
- u.l = u64bitONE;
-
-#if 0
- fprintf(stderr, "%d%d%d%d%d%d%d%d\n",
- u.c[0], u.c[1], u.c[2], u.c[3],
- u.c[4], u.c[5], u.c[6], u.c[7]);
-#endif
-
- if (u.c[0] == 1) // LSB is first
- return(0);
- return(1); // MSB is first
-}
-
-
-
-int
-main(int argc, char **argv) {
- u16bit u16 = 0x0102;
- u32bit u32 = u32bitNUMBER(0x01020304);
- u64bit u64 = u64bitNUMBER(0x0102030405060708);
-
- fprintf(stderr, "BYTE_ORDER = %d\n", BYTE_ORDER);
-
- fprintf(stderr, " BIG_ENDIAN = %d\n", BIG_ENDIAN);
- fprintf(stderr, " LITTLE_ENDIAN = %d\n", LITTLE_ENDIAN);
- fprintf(stderr, " PDP_ENDIAN = %d\n", PDP_ENDIAN);
-
- fprintf(stderr, "isBig1() = %d\n", isBig1());
- fprintf(stderr, "isBig2() = %d\n", isBig2());
-
- if (argc == 1) {
- fprintf(stderr, "usage: %s [ write | read ] < source > check\n", argv[0]);
- exit(1);
- }
-
- if (strcmp(argv[1], "write") == 0) {
- fwrite(&u16, sizeof(u16bit), 1, stdout);
- fwrite(&u32, sizeof(u32bit), 1, stdout);
- fwrite(&u64, sizeof(u64bit), 1, stdout);
- return(0);
- }
-
- fread(&u16, sizeof(u16bit), 1, stdin);
- fread(&u32, sizeof(u32bit), 1, stdin);
- fread(&u64, sizeof(u64bit), 1, stdin);
-
-#if 0
- // swap bytes to convert u16
- u16 = (((u16 >> 8) & 0x00ff) |
- ((u16 << 8) & 0xff00));
-
- // swap bytes, then swap words to convert u32
- u32 = (((u32 >> 24) & 0x000000ff) |
- ((u32 >> 8) & 0x0000ff00) |
- ((u32 << 8) & 0x00ff0000) |
- ((u32 << 24) & 0xff000000));
-
- // swap bytes, then flip words [0<->3, 1<->2] to convert u64
- u64 = (((u64 >> 24) & u64bitNUMBER(0x000000ff000000ff)) |
- ((u64 >> 8) & u64bitNUMBER(0x0000ff000000ff00)) |
- ((u64 << 8) & u64bitNUMBER(0x00ff000000ff0000)) |
- ((u64 << 24) & u64bitNUMBER(0xff000000ff000000)));
- u64 = (((u64 >> 32) & u64bitNUMBER(0x00000000ffffffff)) |
- ((u64 << 32) & u64bitNUMBER(0xffffffff00000000)));
-#endif
-
- if (u16 != 0x1234)
- fprintf(stderr, "u16 -- 0x%04x correct=0x%04x\n", u16, 0x1234);
- if (u32 != 0x12345678)
- fprintf(stderr, "u32 -- "u32bitHEX" correct="u32bitHEX"\n", u32, 0x12345678);
- if (u64 != u64bitNUMBER(0x1234567890abcdef))
- fprintf(stderr, "u64 -- "u64bitHEX" correct="u64bitHEX"\n", u64, u64bitNUMBER(0x1234567890abcdef));
-
- return(0);
-}
-
diff --git a/kmer/libutil/test/order.C b/kmer/libutil/test/order.C
deleted file mode 100644
index 5be9d0d..0000000
--- a/kmer/libutil/test/order.C
+++ /dev/null
@@ -1,84 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "../util.h"
-
-//#include <sys/param.h>
-
-union u64 {
- u64bit u;
- unsigned char c[8];
-};
-
-union u32 {
- u32bit u;
- unsigned char c[4];
-};
-
-union u16 {
- u16bit u;
- unsigned char c[2];
-};
-
-
-u64bit
-u64bitSwap(u64bit x) {
- x = ((x >> 8) & u64bitNUMBER(0x00ff00ff00ff00ff)) | ((x << 8) & u64bitNUMBER(0xff00ff00ff00ff00));
- x = ((x >> 16) & u64bitNUMBER(0x0000ffff0000ffff)) | ((x << 16) & u64bitNUMBER(0xffff0000ffff0000));
- x = ((x >> 32) & u64bitNUMBER(0x00000000ffffffff)) | ((x << 32) & u64bitNUMBER(0xffffffff00000000));
- return(x);
-}
-
-u32bit
-u32bitSwap(u32bit x) {
- x = ((x >> 8) & u32bitNUMBER(0x00ff00ff)) | ((x << 8) & u32bitNUMBER(0xff00ff00));
- x = ((x >> 16) & u32bitNUMBER(0x0000ffff)) | ((x << 16) & u32bitNUMBER(0xffff0000));
- return(x);
-}
-
-u16bit
-u16bitSwap(u16bit x) {
- x = ((x >> 8) & 0x00ff) | ((x << 8) & 0xff00);
- return(x);
-}
-
-
-
-int
-main(int argc, char **argv) {
- u64 u64v;
- u32 u32v;
- u16 u16v;
-
- u64v.u = 0x1234567890abcdefLLU;
- u32v.u = 0x12345678;
- u16v.u = 0x1234;
-
- for (int i=0; i<8; i++)
- fprintf(stderr, "%02x", u64v.c[i]);
- fprintf(stderr, "\n");
-
- for (int i=0; i<4; i++)
- fprintf(stderr, "%02x", u32v.c[i]);
- fprintf(stderr, "\n");
-
- for (int i=0; i<2; i++)
- fprintf(stderr, "%02x", u16v.c[i]);
- fprintf(stderr, "\n");
-
- u64v.u = u64bitSwap(u64v.u);
- u32v.u = u32bitSwap(u32v.u);
- u16v.u = u16bitSwap(u16v.u);
-
- for (int i=0; i<8; i++)
- fprintf(stderr, "%02x", u64v.c[i]);
- fprintf(stderr, "\n");
-
- for (int i=0; i<4; i++)
- fprintf(stderr, "%02x", u32v.c[i]);
- fprintf(stderr, "\n");
-
- for (int i=0; i<2; i++)
- fprintf(stderr, "%02x", u16v.c[i]);
- fprintf(stderr, "\n");
-}
diff --git a/kmer/libutil/test/tcat.C b/kmer/libutil/test/tcat.C
deleted file mode 100644
index b2bcdc2..0000000
--- a/kmer/libutil/test/tcat.C
+++ /dev/null
@@ -1,86 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <math.h>
-#include <time.h>
-#include "sweatShop.H"
-
-// Reads stdin, writes stdout. Uses threads.
-
-int blockSize = 8192;
-
-struct tcat_s {
- int dataLen;
- char *data;
-};
-
-void*
-tcatReader(void *) {
- tcat_s *s = new tcat_s;
-
- s->data = new char [blockSize];
- s->dataLen = safeRead(STDIN_FILENO, s->data, "tcatReader", sizeof(char) * blockSize);
-
- if (s->dataLen == 0) {
- delete [] s->data;
- delete s;
- return(0L);
- }
-
- return(s);
-}
-
-void
-tcatWorker(void *, void *, void *) {
- // Noop!
-}
-
-void
-tcatWriter(void *, void *S) {
- tcat_s *s = (tcat_s *)S;
-
- safeWrite(STDOUT_FILENO, s->data, "tcatWriter", sizeof(char) * s->dataLen);
-
- delete [] s->data;
- delete s;
-}
-
-
-int
-main(int argc, char **argv) {
- int readBuf = 64;
- int writBuf = 64;
-
- int arg = 1;
- int err = 0;
- while (arg < argc) {
- if (strcmp(argv[arg], "-r") == 0) {
- readBuf = atoi(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-w") == 0) {
- writBuf = atoi(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-b") == 0) {
- blockSize = atoi(argv[++arg]);
-
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- err++;
- }
-
- arg++;
- }
- if (err) {
- fprintf(stderr, "usage: %s [-b blockSizeBytes] [-r readBufferSizeMB] [-w writeBufferSizeMB]\n", argv[0]);
- exit(1);
- }
-
- sweatShop *ss = new sweatShop(tcatReader, tcatWorker, tcatWriter);
-
- ss->setLoaderQueueSize(readBuf * 1024 * 1024 / blockSize);
- ss->setWriterQueueSize(writBuf * 1024 * 1024 / blockSize);
-
- ss->run();
-
- exit(0);
-}
diff --git a/kmer/libutil/test/test-bigQueue.C b/kmer/libutil/test/test-bigQueue.C
deleted file mode 100644
index b62a1ae..0000000
--- a/kmer/libutil/test/test-bigQueue.C
+++ /dev/null
@@ -1,72 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "util++.H"
-
-//
-// mbri && CC -g -o test-bigQueue test-bigQueue.C -L. -lutil && ./bigQueue-test | & more
-//
-
-struct thing_s {
- int a;
- int b;
- double c;
- int d;
-};
-
-
-int
-sortthing(const void *a, const void *b) {
- thing_s *A = *((thing_s **)a);
- thing_s *B = *((thing_s **)b);
-
- if (A->a < B->a)
- return(-1);
- if (A->a > B->a)
- return(1);
- if (A->b < B->b)
- return(-1);
- if (A->b > B->b)
- return(1);
- return(0);
-}
-
-
-int
-main(int argc, char **argv) {
- bigQueue *T = new bigQueue(sortthing, 0L, 0L, 0L, sizeof(thing_s), 1, 0L);
-
- mt_s *mtctx = mtInit(3);
-
- int testSize = 2000000;
-
- FILE *out = fopen("junk-bigQueue-out-1", "w");
-
- for (int i=0; i<testSize; i++) {
- thing_s *t = new thing_s;
- t->a = mtRandom32(mtctx) / 4;
- t->b = i;
- t->c = (double)i;
- t->d = -i;
-
- fprintf(out, "%012d %08d %12.3f %08d\n", t->a, t->b, t->c, t->d);
-
- T->add(t);
- }
-
- fclose(out);
- out = fopen("junk-bigQueue-out-2", "w");
-
- T->sort();
-
- while (T->next()) {
- thing_s *t = (thing_s *)T->get();
-
- fprintf(out, "%012d %08d %12.3f %08d\n", t->a, t->b, t->c, t->d);
- }
-
- delete T;
-
- fclose(out);
-}
diff --git a/kmer/libutil/test/test-bitPackedArray.C b/kmer/libutil/test/test-bitPackedArray.C
deleted file mode 100644
index dc8a5f9..0000000
--- a/kmer/libutil/test/test-bitPackedArray.C
+++ /dev/null
@@ -1,152 +0,0 @@
-#include <unistd.h>
-#include <time.h>
-#include <math.h>
-
-#include "util++.H"
-
-u32bit wordSize = 41;
-u32bit testSize = 1 * 1024 * 1024;
-u32bit arrySize = 1 * 1024 * 1024;
-
-int
-u64bitcompare(const void *a, const void *b) {
- const u64bit A = *(const u64bit *)a;
- const u64bit B = *(const u64bit *)b;
- if (A<B) return(-1);
- if (A>B) return(1);
- return(0);
-}
-
-int
-main(int argc, char **argv) {
-
- mt_s *mtctx = mtInit(time(NULL));
-
- // Test the bitPackedArray by writing a bunch of random gibberish
- // to it, and see if it's the same.
-
- u32bit *pos = new u32bit [testSize];
- u64bit *val = new u64bit [testSize];
- u64bit *ans = new u64bit [arrySize];
-
- bitPackedArray *ARR = new bitPackedArray(wordSize, 16);
- u32bit fail = u32bitZERO;
-
-#if 1
- fprintf(stderr, "Touching the end of the array and clearing.\n");
- //ARR->set(arrySize, 0);
- //ARR->clear();
-
- fprintf(stderr, "Generating random test data.\n");
-
- // Hit every element first, just to do it
- for (u32bit i=0; i<arrySize; i++) {
- pos[i] = i;
- val[i] = mtRandom64(mtctx);
- val[i] &= u64bitMASK(wordSize);
- ans[pos[i]] = val[i];
- }
-
- // Then hit random elements, with replacement, looking for bugs
- for (u32bit i=arrySize; i<testSize; i++) {
- pos[i] = mtRandom32(mtctx) % arrySize;
- val[i] = mtRandom64(mtctx);
- val[i] &= u64bitMASK(wordSize);
- ans[pos[i]] = val[i];
- }
-
- fprintf(stderr, "Filling array.\n");
-
- for (u32bit i=0; i<testSize; i++)
- ARR->set(pos[i], val[i]);
-
- fprintf(stderr, "Validating array.\n");
-
- for (u32bit i=0; i<arrySize; i++)
- if (ARR->get(i) != ans[i]) {
- fprintf(stderr, "FAIL at i="u32bitFMT"\n", i);
- fail++;
-
- if (fail > 1024) {
- fprintf(stderr, "bitPackedArray has errors, aborting!\n");
- return(1);
- }
- }
-
- if (fail) {
- fprintf(stderr, "bitPackedArray had "u32bitFMT" errors.\n", fail);
- return(1);
- }
-
- fprintf(stderr, "OK!\n");
-#endif
-
- delete ARR;
- delete [] pos;
- delete [] val;
- delete [] ans;
-
- //
- //
- //
-
- for (u32bit testNum=0; testNum<32; testNum++) {
- u32bit thisTestSize = 0;
- u32bit thisWordSize = 0;
-
- // Test a BIG heap the first iteration.
- if (testNum == 0) {
- thisTestSize = 857353; //23987153;
- thisWordSize = 63;
-
- fprintf(stderr, "Building heap "u32bitFMT" (wordsize="u32bitFMT" testsize="u32bitFMT").\n",
- testNum, thisWordSize, thisTestSize);
- } else {
- thisTestSize = (mtRandom64(mtctx) % (2 * testNum)) * 1024 + 1024;
- thisWordSize = (mtRandom64(mtctx) % 63) + 1;
- }
-
- u32bit blockSize = mtRandom64(mtctx) % 32 + 1;
- bitPackedHeap *HEAP = new bitPackedHeap(thisWordSize, blockSize);
-
- val = new u64bit [thisTestSize];
- for (u32bit i=0; i<thisTestSize; i++) {
- val[i] = mtRandom64(mtctx);
- val[i] &= u64bitMASK(thisWordSize);
- HEAP->add(val[i]);
- }
-
- fprintf(stderr, "Testing heap "u32bitFMT" (wordsize="u32bitFMT" testsize="u32bitFMT").\n",
- testNum, thisWordSize, thisTestSize);
-
- qsort(val, thisTestSize, sizeof(u64bit), u64bitcompare);
-
- for (u32bit i=0; i<thisTestSize; i++) {
- u64bit h = HEAP->get();
-
- //fprintf(stderr, "val["u32bitFMT"]="u64bitFMT" -- HEAP="u64bitFMT"\n", i, val[i], h);
-
- if (val[i] != h) {
- fprintf(stderr, "val["u32bitFMT"]="u64bitFMT" !! HEAP="u64bitFMT"\n", i, val[i], h);
- fail++;
- if (fail > 25) {
- fprintf(stderr, "bitPackedHeap has errors, aborting!\n");
- return(1);
- }
- }
- }
-
- if (fail) {
- fprintf(stderr, "bitPackedHeap had "u32bitFMT" errors.!\n", fail);
- return(1);
- }
-
- delete HEAP;
- delete [] val;
- }
-
- fprintf(stderr, "OK!\n");
-
- return(fail);
-}
-
diff --git a/kmer/libutil/test/test-bitPackedFile.C b/kmer/libutil/test/test-bitPackedFile.C
deleted file mode 100644
index 58a1afe..0000000
--- a/kmer/libutil/test/test-bitPackedFile.C
+++ /dev/null
@@ -1,271 +0,0 @@
-#include <unistd.h>
-#include <time.h>
-#include <math.h>
-
-#include "util++.H"
-
-// This will perform various tests on the bitPackedFile class,
-// returning 0 if OK and 1 if error.
-//
-// testSize -- the number of words to use in a write then read test
-// testIter -- the number of random access tests to do
-
-u32bit testSize = 2000000;
-u32bit testIter = 50;
-
-mt_s *mtctx;
-
-// Generate a list of random 64-bit numbers, remember the number and the size
-//
-void
-generateRandom(u32bit *siz, u64bit *val) {
-
- for (u32bit i=0; i<testSize; i++) {
-#if 0
- // For debugging
- siz[i] = 13;
- val[i] = (i % 2) ? u64bitZERO : ~u64bitZERO;
-#else
- siz[i] = (mtRandom32(mtctx) % 63) + 1;
- val[i] = mtRandom64(mtctx);
-#endif
- val[i] &= u64bitMASK(siz[i]);
- }
-}
-
-
-void
-testStreaming(void) {
- bitPackedFile *F = 0L;
- u32bit i;
- u32bit *siz = new u32bit [testSize];
- u64bit *val = new u64bit [testSize];
- u32bit errs = 0;
-
- generateRandom(siz, val);
-
- // Write those numbers to a bitPackedFile, both binary encoded and
- // fibonacci encoded.
- //
- F = new bitPackedFile("bittest.junk");
- for (i=0; i<testSize; i++) {
- F->putBits(val[i], siz[i]);
- F->putNumber(val[i]);
- }
- delete F;
-
- // Open the file and check what we just wrote.
- //
- F = new bitPackedFile("bittest.junk");
- for (i=0; i<testSize; i++) {
- u64bit v;
-
- v = F->getBits(siz[i]);
- if (v != val[i]) {
- fprintf(stderr, u32bitFMT"] ERROR in getBits() -- retrieved "u64bitHEX" != expected "u64bitHEX" ("u32bitFMT" bits).\n", i, v, val[i], siz[i]);
- errs++;
- }
-
- v = F->getNumber();
- if (v != val[i]) {
- fprintf(stderr, u32bitFMT"] ERROR in getNumber() -- retrieved "u64bitHEX" != expected "u64bitHEX".\n", i, v, val[i]);
- errs++;
- }
- }
- delete F;
-
- delete [] val;
- delete [] siz;
-
- if (errs > 0) {
- fprintf(stderr, "There are "u32bitFMT" errors in the stream test.\n", errs);
- exit(1);
- } else {
- fprintf(stderr, "The stream test PASSED.\n");
- }
-
- unlink("bittest.junk");
-}
-
-
-void
-testRandomReading(bool inCore) {
- bitPackedFile *F = 0L;
- u32bit i;
- u32bit *siz = new u32bit [testSize + 1];
- u64bit *val = new u64bit [testSize];
- u32bit errs = 0;
-
- fprintf(stderr, "BUILDING random test set.\n");
- generateRandom(siz, val);
-
- // Create a new bitpacked file, writing just numbers as binary encoded.
- //
- fprintf(stderr, "SAVING random test set.\n");
- F = new bitPackedFile("bittest.junk");
- for (i=0; i<testSize; i++)
- F->putBits(val[i], siz[i]);
- delete F;
-
- // Covert the siz[] into offsets
- //
- u32bit t = siz[0];
- siz[0] = 0;
- for (u32bit i=1; i<testSize; i++) {
- u32bit x = siz[i];
- siz[i] = t;
- t += x;
- }
- siz[testSize] = t;
-
- // Attempt to flush memory
- //
- {
- u32bit ll = 400 * 1024 * 1024 / 8;
- u64bit *xx = new u64bit [ll];
- xx[0] = 1;
- xx[1] = 1;
- for (u32bit i=2; i<ll; i++)
- xx[i] = xx[i-1] + xx[i-2];
- fprintf(stdout, "FLUSHED: "u32bitFMT"\n", xx[ll-1]);
- delete [] xx;
- }
-
- // Do several seek tests. Seek to a random element, and read it.
- //
- F = new bitPackedFile("bittest.junk");
-
- if (inCore) {
- F->loadInCore();
- fprintf(stderr, "Begin INCORE seek test!\n");
- } else {
- fprintf(stderr, "Begin DISKBASED seek test!\n");
- }
-
- double startTime = getTime();
-
- for (i=0; i<testIter; i++) {
- u32bit idx = (u32bit)lrand48() % testSize;
-
- F->seek(siz[idx]);
- u64bit r = F->getBits(siz[idx+1] - siz[idx]);
-
- if (r != val[idx]) {
- fprintf(stderr, u32bitFMT"] ERROR in seek()/getBits() -- retrieved "u64bitHEX" != expected "u64bitHEX" ("u32bitFMT" bits).\n", i, r, val[i], siz[i]);
- errs++;
- }
- }
- delete F;
-
- if (errs > 0) {
- fprintf(stderr, "There are "u32bitFMT" errors in the %s random access.\n", errs, (inCore) ? "inCore" : "disk");
- exit(1);
- } else {
- fprintf(stderr, "The %s seek test PASSED (%f seconds).\n",
- (inCore) ? "inCore" : "disk",
- getTime() - startTime);
- }
-
- delete [] val;
- delete [] siz;
-
- unlink("bittest.junk");
-}
-
-
-
-
-
-void
-testReWrite(void) {
- bitPackedFile *F = 0L;
- u32bit i;
- u32bit *siz = new u32bit [testSize];
- u64bit *val = new u64bit [testSize];
- u32bit errs = 0;
- u64bit pos = u64bitZERO;
-
- generateRandom(siz, val);
-
- // First, write zeros to the file
- //
- F = new bitPackedFile("bittest.junk");
- for (i=0; i<testSize; i++)
- F->putBits(u64bitZERO, siz[i]);
- delete F;
-
- fprintf(stderr, "WRITING FORWARDS!\n");
-
- // Now, write every other number to the file
- //
- F = new bitPackedFile("bittest.junk");
- for (i=0; i<testSize; i++) {
- if ((i % 2) == 1) {
- F->seek(pos);
- F->putBits(val[i], siz[i]);
- }
- pos += siz[i];
- }
- F->showStats(stderr);
- delete F;
-
- fprintf(stderr, "WRITING BACKWARDS!\n");
-
- // And go backwards and write the other set of numbers to the file
- //
- F = new bitPackedFile("bittest.junk");
- for (i=testSize; i--; ) {
- pos -= siz[i];
- if ((i % 2) == 0) {
- F->seek(pos);
- F->putBits(val[i], siz[i]);
- }
- }
- F->showStats(stderr);
- delete F;
-
- // Now, stream through the file and see if we wrote what we should have
- //
- F = new bitPackedFile("bittest.junk");
- for (i=0; i<testSize; i++) {
- u64bit v;
-
- v = F->getBits(siz[i]);
- if (v != val[i]) {
- fprintf(stderr, u32bitFMT"] ERROR in seekstream/getBits() -- retrieved "u64bitHEX" != expected "u64bitHEX" ("u32bitFMT" bits).\n", i, v, val[i], siz[i]);
- errs++;
- }
- }
- F->showStats(stderr);
- delete F;
-
- delete [] val;
- delete [] siz;
-
- if (errs > 0) {
- fprintf(stderr, "There are "u32bitFMT" errors in the rewrite test.\n", errs);
- exit(1);
- } else {
- fprintf(stderr, "The rewrite test PASSED.\n");
- }
-
- unlink("bittest.junk");
-}
-
-
-
-int
-main(int argc, char **argv) {
-
- mtctx = mtInit(time(NULL));
-
- testSize = 30000000;
- testIter = 2000;
- //testStreaming();
- //testReWrite();
-
- testSize = 40000000;
- testIter = 10000;
- testRandomReading(false);
- testRandomReading(true);
-}
diff --git a/kmer/libutil/test/test-bitPacking.C b/kmer/libutil/test/test-bitPacking.C
deleted file mode 100644
index d4eaebb..0000000
--- a/kmer/libutil/test/test-bitPacking.C
+++ /dev/null
@@ -1,337 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-
-#include "util++.H"
-
-// An integer multiplier on the test length. 1 is pretty quick, but
-// 10 is the default.
-//
-#define TEST_LENGTH (10 * 1024 * 1024)
-
-// We test
-//
-// 1) binary encoding/decoding
-//
-// 2) pre/post increment of binary encoding
-//
-// 3) Perform some testing on the fibonacci encoded bit-packed stream
-// -- encode a bunch of random 64-bit numbers, make sure we can
-// decode back to the same number.
-//
-// NOTES: pre/post increment/decrement work modulo whatever size they
-// are. So, if you have a 6-bit value of zero, and you decrement,
-// you end up with a 6-bit value of all 1's, or 63.
-
-
-void
-testBinaryEncoding(void) {
- time_t mtseed = time(0L);
- mt_s *mtctx = 0L;
-
- u32bit iterations = TEST_LENGTH;
-
- u64bit *bits = new u64bit [iterations + 2];
- u64bit bpos = u64bitZERO;
-
- u64bit *V = new u64bit [iterations];
- u64bit *C = new u64bit [iterations];
- u64bit *S = new u64bit [iterations];
-
- u32bit failed = 0;
- u32bit errors = 0;
-
- fprintf(stderr, "Starting test of binary encoding\n");
-
- bpos = u64bitZERO;
- mtctx = mtInit(mtseed);
-
- // Build some values to stuff into the bits
-
- for (u32bit j=0; j < iterations; j++) {
- S[j] = (mtRandom32(mtctx) % 63) + 1;
- V[j] = mtRandom64(mtctx) & u64bitMASK(S[j]);
- //fprintf(stderr, "[%2d] S="u64bitFMT" V="u64bitHEX"\n", j, S[j], V[j]);
- }
-
- // Stuff them in, in blocks of some size. At the same time, decode
- // (this has found bugs in the past).
-
- failed = 0;
- for (u32bit j=0; j < iterations; ) {
- u64bit num = (mtRandom32(mtctx) % 8);
-
- if (j + num > iterations)
- num = iterations - j;
-
- if (num == 0) {
- setDecodedValue(bits, bpos, S[j], V[j]);
- C[j] = getDecodedValue(bits, bpos, S[j]);
- //fprintf(stderr, "[%2d] V="u64bitHEX" C="u64bitHEX" single\n", j, V[j], C[j]);
- bpos += S[j];
- } else {
- u64bit newp1 = setDecodedValues(bits, bpos, num, S+j, V+j);
- u64bit newp2 = getDecodedValues(bits, bpos, num, S+j, C+j);
-
- if (newp1 != newp2) {
- // not perfect; we should be checking the values too, but we do that later.
- for (u32bit x=0; x<num; x++)
- fprintf(stderr, "[%2d] #1 V="u64bitHEX" C="u64bitHEX" multiple "u32bitFMT" %s\n",
- j+x, V[j+x], C[j+x], num, (V[j+x] == C[j+x]) ? "" : "FAILED");
- failed++;
- }
-
- bpos = newp2;
- }
-
- j += num;
- if (num == 0)
- j++;
- }
- if (failed) {
- fprintf(stderr, "binEncoding #1 failed encoding "u32bitFMT" times.\n", failed);
- errors++;
- }
-
- // Check that V == C
-
- failed = 0;
- for (u32bit j=0; j<iterations; j++) {
- if (V[j] != C[j]) {
- fprintf(stderr, "[%2d] #2 V="u64bitHEX" C="u64bitHEX" S="u32bitFMT"\n",
- j, V[j], C[j], S[j]);
- failed++;
- }
- }
- if (failed) {
- fprintf(stderr, "binEncoding #2 failed encode/decode "u32bitFMT" times.\n", failed);
- errors++;
- }
-
-
- // Decode independently, with different nums
-
- bpos = 0; // reset to start of bits
-
- for (u32bit j=0; j < iterations; ) {
- u64bit num = (mtRandom32(mtctx) % 8);
-
- if (j + num > iterations)
- num = iterations - j;
-
- if (num == 0) {
- C[j] = getDecodedValue(bits, bpos, S[j]);
- bpos += S[j];
- } else {
- bpos = getDecodedValues(bits, bpos, num, S+j, C+j);
- }
-
- j += num;
- if (num == 0)
- j++;
- }
-
- // Check that V == C
-
- failed = 0;
- for (u32bit j=0; j<iterations; j++) {
- if (V[j] != C[j]) {
- fprintf(stderr, "[%2d] #3 V="u64bitHEX" C="u64bitHEX" S="u32bitFMT"\n",
- j, V[j], C[j], S[j]);
- failed++;
- }
- }
- if (failed) {
- fprintf(stderr, "binEncoding #3 failed decoding "u32bitFMT" times.\n", failed);
- errors++;
- }
-
- // Clean.
-
- delete [] bits;
- delete [] V;
- delete [] C;
- delete [] S;
-
- if (errors)
- exit(1);
-}
-
-
-
-
-void
-testBinaryEncodingPrePost(void) {
- time_t mtseed = time(0L);
- mt_s *mtctx = 0L;
-
- u32bit iterations = TEST_LENGTH;
-
- u64bit *bits = new u64bit [2 * iterations];
- u64bit bpos = u64bitZERO;
- u32bit siz1 = u64bitZERO;
- u64bit val1 = u64bitZERO;
- u64bit val2 = u64bitZERO;
-
- fprintf(stderr, "Starting test of binary encoding pre/post increment\n");
-
- bpos = u64bitZERO;
- mtctx = mtInit(mtseed);
-
- for (u32bit j=0; j < iterations; j++) {
- siz1 = (mtRandom32(mtctx) % 63) + 1;
- val1 = mtRandom64(mtctx) & u64bitMASK(siz1);
-
- setDecodedValue(bits, bpos, siz1, val1);
-
- val2 = postDecrementDecodedValue(bits, bpos, siz1);
- if (val2 != val1) {
- fprintf(stderr, "postDec1 failed: got "u64bitFMT" expected "u64bitFMT" siz="u32bitFMT"\n",
- val2, val1, siz1);
- exit(1);
- }
- val2 = getDecodedValue(bits, bpos, siz1) + 1;
- val2 &= u64bitMASK(siz1);
- if (val2 != val1) {
- fprintf(stderr, "postDec2 failed: got "u64bitFMT" expected "u64bitFMT" siz="u32bitFMT"\n",
- val2, val1, siz1);
- exit(1);
- }
-
- val2 = preDecrementDecodedValue(bits, bpos, siz1) + 2;
- val2 &= u64bitMASK(siz1);
- if (val2 != val1) {
- fprintf(stderr, "preDec failed: got "u64bitFMT" expected "u64bitFMT" siz="u32bitFMT"\n",
- val2, val1, siz1);
- exit(1);
- }
-
- val2 = postIncrementDecodedValue(bits, bpos, siz1) + 2;
- val2 &= u64bitMASK(siz1);
- if (val2 != val1) {
- fprintf(stderr, "postInc failed: got "u64bitFMT" expected "u64bitFMT"\n", val2+2, val1-2);
- exit(1);
- }
- val2 = getDecodedValue(bits, bpos, siz1) + 1;
- val2 &= u64bitMASK(siz1);
- if (val2 != val1) {
- fprintf(stderr, "postInc2 failed: got "u64bitFMT" expected "u64bitFMT" siz="u32bitFMT"\n",
- val2, val1, siz1);
- exit(1);
- }
-
- val2 = preIncrementDecodedValue(bits, bpos, siz1);
- // Should be back to original value, so no mask
- if (val2 != val1) {
- fprintf(stderr, "preInc failed: got "u64bitFMT" expected "u64bitFMT"\n", val2, val1);
- exit(1);
- }
-
- switch (j % 4) {
- case 0:
- val2 = postDecrementDecodedValue(bits, bpos, siz1);
- break;
- case 1:
- val2 = preDecrementDecodedValue(bits, bpos, siz1);
- break;
- case 2:
- val2 = postIncrementDecodedValue(bits, bpos, siz1);
- break;
- case 3:
- val2 = preIncrementDecodedValue(bits, bpos, siz1);
- break;
- }
-
- bpos += siz1;
- }
-
- bpos = u64bitZERO;
- mtctx = mtInit(mtseed);
-
- //for (j=0; j < iterations; j++) {
- //}
-
- delete [] bits;
-}
-
-
-
-
-
-
-void
-testFibonacciEncoding(void) {
- time_t mtseed = time(0L);
- mt_s *mtctx = 0L;
-
- u32bit iterations = TEST_LENGTH / 4;
-
- u64bit *bits = new u64bit [3 * iterations];
- u64bit bpos = u64bitZERO;
-
- u32bit failed = 0;
- u32bit errors = 0;
-
- fprintf(stderr, "Starting test of fibonacci encoding\n");
-
- bpos = u64bitZERO;
- mtctx = mtInit(mtseed);
- failed = 0;
-
- for (u32bit j=0; j < iterations; j++) {
- u64bit siz1 = (mtRandom32(mtctx) % 63) + 1;
- u64bit val1 = mtRandom64(mtctx) & u64bitMASK(siz1);
- u64bit siz2 = siz1;
-
- setFibonacciEncodedNumber(bits, bpos, &siz1, val1);
-
- u64bit val2 = getFibonacciEncodedNumber(bits, bpos, &siz2);
-
- if ((val1 != val2) || (siz1 != siz2)) {
- fprintf(stderr, "fibEnc #1 failed on "u32bitFMT": got "u64bitFMT" expected "u64bitFMT"\n", j, val2, val1);
- failed++;
- }
- bpos += siz1;
- }
- if (failed) {
- fprintf(stderr, "fibEnc #1 failed "u32bitFMT" times.\n", failed);
- errors++;
- }
-
- bpos = u64bitZERO;
- mtctx = mtInit(mtseed);
- failed = 0;
-
- for (u32bit j=0; j < iterations; j++) {
- u64bit siz1 = (mtRandom32(mtctx) % 63) + 1;
- u64bit val1 = mtRandom64(mtctx) & u64bitMASK(siz1);
- u64bit val2 = getFibonacciEncodedNumber(bits, bpos, &siz1);
-
- if (val1 != val2) {
- fprintf(stderr, "fibEnc #2 failed on "u32bitFMT": got "u64bitFMT" expected "u64bitFMT"\n", j, val2, val1);
- failed++;
- }
- bpos += siz1;
- }
- if (failed) {
- fprintf(stderr, "fibEnc #2 failed "u32bitFMT" times.\n", failed);
- errors++;
- }
-
- delete [] bits;
-
- if (errors)
- exit(1);
-}
-
-
-
-
-
-int
-main(int argc, char **argv) {
- testBinaryEncoding();
- testBinaryEncodingPrePost();
- testFibonacciEncoding();
- return(0);
-}
diff --git a/kmer/libutil/test/test-bzipBuffer.C b/kmer/libutil/test/test-bzipBuffer.C
deleted file mode 100644
index 02437a0..0000000
--- a/kmer/libutil/test/test-bzipBuffer.C
+++ /dev/null
@@ -1,110 +0,0 @@
-#include <stdio.h>
-
-#include "util++.H"
-
-char *filename = 0L;
-md5_s *correct = 0L;
-
-int
-doTest(bzipBuffer *B, char *description) {
- int error = 0;
- md5_increment_s *testing = 0L;
-
- while (!B->eof())
- testing = md5_increment_char(testing, B->getnext());
-
- md5_increment_finalize(testing);
-
- if ((testing->a != correct->a) || (testing->b != correct->b)) {
- fprintf(stderr, "bzipBuffer test %s failed.\n", description);
- fprintf(stderr, "Got correct md5 of "u64bitHEX" "u64bitHEX"\n", correct->a, correct->b);
- fprintf(stderr, "Got testing md5 of "u64bitHEX" "u64bitHEX"\n", testing->a, testing->b);
- error = 1;
- }
-
- md5_increment_destroy(testing);
-
- return(error);
-}
-
-int
-doTestRead(bzipBuffer *B, size_t bufferSize, char *description) {
- int error = 0;
- char *buffer = new char [bufferSize];
- size_t bufferLen = 0;
-
- md5_increment_s *testing = 0L;
-
- while (!B->eof()) {
- bufferLen = B->read(buffer, bufferSize);
- testing = md5_increment_block(testing, buffer, bufferLen);
- }
-
- md5_increment_finalize(testing);
-
- if ((testing->a != correct->a) || (testing->b != correct->b)) {
- fprintf(stderr, "bzipBuffer test %s failed.\n", description);
- fprintf(stderr, "Got correct md5 of "u64bitHEX" "u64bitHEX"\n", correct->a, correct->b);
- fprintf(stderr, "Got testing md5 of "u64bitHEX" "u64bitHEX"\n", testing->a, testing->b);
- error = 1;
- }
-
- md5_increment_destroy(testing);
-
- return(error);
-}
-
-
-int
-main(int argc, char **argv) {
- int error = 0;
- bzipBuffer *B = 0L;
-
- // If we are given a file, use that, otherwise, use ourself.
- //
- filename = argv[argc-1];
-
-
- // Suck in the whole file, compute the correct md5 checksum on it
- //
- char *c = new char [sizeOfFile(filename)];
- FILE *F = fopen(filename, "r");
- fread(c, sizeof(char), sizeOfFile(filename), F);
- fclose(F);
- correct = md5_string(0L, c, sizeOfFile(filename));
- delete [] c;
-
-
- // Test just reading, with a small buffer
- //
- B = new bzipBuffer(filename, 999);
- error += doTest(B, "#1 (read)");
-
-
- exit(1);
-
-
- // Test read() with a small buffer, reading large chunks
- //
- B = new bzipBuffer(filename, 100);
- error += doTestRead(B, 10000, "#4 (read)");
- delete B;
-
-
- // Test read() with a small buffer, reading small chunks that are a
- // factor of the buffersize.
- //
- B = new bzipBuffer(filename, 2000);
- error += doTestRead(B, 1000, "#4 (read)");
- delete B;
-
-
- // Test read() with a large buffer, reading even larger pieces
- //
- B = new bzipBuffer(filename, sizeOfFile(filename));
- error += doTestRead(B, sizeOfFile(filename) + 100000, "#5 (read)");
- delete B;
-
- return(error);
-}
-
diff --git a/kmer/libutil/test/test-freeDiskSpace.c b/kmer/libutil/test/test-freeDiskSpace.c
deleted file mode 100644
index 9e4b9b9..0000000
--- a/kmer/libutil/test/test-freeDiskSpace.c
+++ /dev/null
@@ -1,16 +0,0 @@
-#include "util.h"
-
-int
-main(int argc, char **argv) {
- int i;
-
- if (argc == 1) {
- fprintf(stderr, "usage: %s file [...]\n", argv[0]);
- exit(1);
- }
-
- for (i=1; i<argc; i++)
- fprintf(stderr, "%s: %d\n", argv[i], (int)freeDiskSpace(argv[i]));
-
- return(0);
-}
diff --git a/kmer/libutil/test/test-intervalList.C b/kmer/libutil/test/test-intervalList.C
deleted file mode 100644
index eef06e6..0000000
--- a/kmer/libutil/test/test-intervalList.C
+++ /dev/null
@@ -1,276 +0,0 @@
-#include <stdio.h>
-
-#include "util++.H"
-
-mt_s *mt = 0L;
-
-void
-test(void) {
- int e = 0;
- intervalList I;
-
- I.add(71, 3);
- I.add( 5, 3);
- I.add(32, 5);
- I.add(73, 3);
- I.add(55, 10);
- I.add( 5, 3);
- I.add(10, 5);
- I.add(20, 10);
- I.add(30, 10);
- I.add(50, 10);
- I.add(70, 3);
- I.add(72, 3);
- I.add( 5, 3);
- I.add(15, 5);
-
-#if 0
- for (u32bit i=0; i<I.numberOfIntervals(); i++)
- fprintf(stderr, "%2d] %2d %2d\n", i, I.lo(i), I.hi(i));
-#endif
-
- I.sort();
- I.merge();
-
- if (I.sumOfLengths() != 54)
- fprintf(stderr, "Lengths don't add up.\n"), e++;
-
- if (I.numberOfIntervals() != 4)
- fprintf(stderr, "Wrong number of intervals.\n"), e++;
-
- if ((I.lo(0) != 5) || (I.hi(0) != 8))
- fprintf(stderr, "Interval 0 is wrong.\n"), e++;
- if ((I.lo(1) != 10) || (I.hi(1) != 40))
- fprintf(stderr, "Interval 1 is wrong.\n"), e++;
- if ((I.lo(2) != 50) || (I.hi(2) != 65))
- fprintf(stderr, "Interval 2 is wrong.\n"), e++;
- if ((I.lo(3) != 70) || (I.hi(3) != 76))
- fprintf(stderr, "Interval 3 is wrong.\n"), e++;
-
- if (e)
- exit(e);
-}
-
-
-
-void
-testIntersect(u32bit type) {
- u32bit numTests = 1000000;
- u32bit *beg = new u32bit [numTests];
- u32bit *len = new u32bit [numTests];
- u32bit *end = new u32bit [numTests];
- u32bit *abegh = new u32bit [numTests];
- u32bit *aendh = new u32bit [numTests];
- u32bit *bbegh = new u32bit [numTests];
- u32bit *bendh = new u32bit [numTests];
- u32bit errors = 0;
- u32bit passed = 0;
-
- intervalList A;
- intervalList B;
-
- //
- // Build two interval lists
- //
- // type == 0 --> all pairwise
- // type == 1 --> A sequence is solid
- // type == 2 --> B sequence is solid
- //
-
- if (type == 1)
- A.add(1, 1500000000);
- if (type == 2)
- B.add(1, 1500000000);
-
- for (u32bit i=0; i<numTests; i++) {
-
- // Compute the result we want to get
- //
- len[i] = mtRandom32(mt) % 200;
- if (len[i] < 100) {
- beg[i] = end[i] = mtRandom32(mt) % 100 + 100;
- } else {
- beg[i] = mtRandom32(mt) % 100 + 100;
- end[i] = beg[i] + len[i];
- }
-
- // Reset if the type is 1 or 2.
- //
- if ((type == 1) || (type == 2)) {
- len[i] = mtRandom32(mt) % 100 + 100;
- beg[i] = mtRandom32(mt) % 100 + 100;
- end[i] = beg[i] + len[i];
- }
-
- // Extend it to an interval -- we can extend exactly one end, or
- // two opposite ends.
- //
- abegh[i] = 0;
- aendh[i] = 0;
- bbegh[i] = 0;
- bendh[i] = 0;
-
- if (type == 0) {
- switch (mtRandom32(mt) % 8) {
- case 0:
- abegh[i] = mtRandom32(mt) % 50;
- break;
- case 1:
- aendh[i] = mtRandom32(mt) % 50;
- break;
- case 2:
- bbegh[i] = mtRandom32(mt) % 50;
- break;
- case 3:
- bendh[i] = mtRandom32(mt) % 50;
- break;
- case 4:
- abegh[i] = mtRandom32(mt) % 50;
- aendh[i] = mtRandom32(mt) % 50;
- break;
- case 5:
- bbegh[i] = mtRandom32(mt) % 50;
- bendh[i] = mtRandom32(mt) % 50;
- break;
- case 6:
- abegh[i] = mtRandom32(mt) % 50;
- bendh[i] = mtRandom32(mt) % 50;
- break;
- case 7:
- aendh[i] = mtRandom32(mt) % 50;
- bbegh[i] = mtRandom32(mt) % 50;
- break;
- }
- }
-
- // Add it to the lists -- if type == 1 or 2, these should then
- // get merged into the one big thing.
- //
- A.add(1000 * i + beg[i] - abegh[i], abegh[i] + end[i] - beg[i] + aendh[i]);
- B.add(1000 * i + beg[i] - bbegh[i], bbegh[i] + end[i] - beg[i] + bendh[i]);
- }
-
- intervalList I;
- I.intersect(A, B);
-
- //
- // Check the result.
- //
-
- for (u32bit i=0, j=0; i<numTests; i++) {
- u32bit b = I.lo(j) - 1000 * i;
- u32bit e = I.hi(j) - 1000 * i;
-
- if (len[i] < 100) {
- //
- // Expect no result here. We ca only test that the stuff that
- // should be intersecting is correct, and if all that is, then
- // I guess the non-intersection stuff is correct too.
- //
- } else {
- if ((b != beg[i]) || (e != end[i])) {
- fprintf(stderr, "FAILED[%4d]: "u32bitFMT"-"u32bitFMT" X "u32bitFMT"-"u32bitFMT" -> "u32bitFMT","u32bitFMT" ("u32bitFMT","u32bitFMT") (should have been "u32bitFMT","u32bitFMT")\n",
- i,
- beg[i] - abegh[i], beg[i] - abegh[i] + abegh[i] + end[i] - beg[i] + aendh[i],
- beg[i] - bbegh[i], beg[i] - bbegh[i] + bbegh[i] + end[i] - beg[i] + bendh[i],
- b, e, (u32bit)I.lo(j), (u32bit)I.hi(j),
- beg[i], end[i]);
- errors++;
- } else {
- passed++;
- }
- j++;
- }
- }
-
- fprintf(stderr, "intersection test had "u32bitFMT" successes and "u32bitFMT" errors.\n", passed, errors);
-}
-
-
-
-
-void
-testMerge(void) {
- intervalList IL;
-
- // Test 1: one long sequence containing lots of little non-overlapping sequences
- // Test 2: three long overlapping sequences, containing lots of non-overlapping sequences
- // Test 3: dense random
- // Test 4: special cases
-
- fprintf(stderr, "Merge test 1\n");
- IL.clear();
- IL.add(0, 100000);
- for (u32bit i=0; i<999; i++)
- IL.add(100 + 100 * i, 50);
- IL.merge();
- for (u32bit i=0; i<IL.numberOfIntervals(); i++)
- fprintf(stderr, "IL["u32bitFMTW(3)"] "u64bitFMT" "u64bitFMT"\n", i, IL.lo(i), IL.hi(i));
-
- fprintf(stderr, "Merge test 2\n");
- IL.clear();
- IL.add(0, 25000);
- IL.add(25000, 25000);
- IL.add(50000, 50000);
- for (u32bit i=0; i<999; i++)
- IL.add(100 + 100 * i, 50);
- IL.merge();
- for (u32bit i=0; i<IL.numberOfIntervals(); i++)
- fprintf(stderr, "IL["u32bitFMTW(3)"] "u64bitFMT" "u64bitFMT"\n", i, IL.lo(i), IL.hi(i));
-
- fprintf(stderr, "Merge test 3\n");
- IL.clear();
- u32bit lo = 200;
- u32bit hi = 0;
- for (u32bit i=0; i<999; i++) {
- u32bit beg = mtRandom32(mt) % 100;
- u32bit end = mtRandom32(mt) % 100 + 100;
- if (beg < lo) lo = beg;
- if (end > hi) hi = end;
- IL.add(beg, end - beg);
- }
- IL.merge();
- if ((IL.lo(0) != lo) || (IL.hi(0) != hi))
- fprintf(stderr, "ERROR!\n");
- for (u32bit i=0; i<IL.numberOfIntervals(); i++)
- fprintf(stderr, "IL["u32bitFMTW(3)"] "u64bitFMT" "u64bitFMT"\n", i, IL.lo(i), IL.hi(i));
-
- fprintf(stderr, "Merge test 4a\n");
- IL.clear();
- IL.add(0, 25000);
- IL.add(25000, 25000);
- IL.add(50000, 50000);
- IL.merge();
- for (u32bit i=0; i<IL.numberOfIntervals(); i++)
- fprintf(stderr, "IL["u32bitFMTW(3)"] "u64bitFMT" "u64bitFMT"\n", i, IL.lo(i), IL.hi(i));
-
- fprintf(stderr, "Merge test 4b\n");
- IL.clear();
- IL.add(0, 25000);
- IL.add(25000, 25000);
- IL.add(50000, 50000);
- IL.add(20000, 5000);
- IL.add(45000, 5000);
- IL.add(95000, 5000);
- IL.merge();
- for (u32bit i=0; i<IL.numberOfIntervals(); i++)
- fprintf(stderr, "IL["u32bitFMTW(3)"] "u64bitFMT" "u64bitFMT"\n", i, IL.lo(i), IL.hi(i));
-}
-
-
-
-int
-main(int argc, char **argv) {
-
- mt = mtInit(time(NULL));
-
- test();
-
- testIntersect(0);
- testIntersect(1);
- testIntersect(2);
-
- testMerge();
-
- exit(0);
-}
diff --git a/kmer/libutil/test/test-logMsg.C b/kmer/libutil/test/test-logMsg.C
deleted file mode 100644
index a20697c..0000000
--- a/kmer/libutil/test/test-logMsg.C
+++ /dev/null
@@ -1,27 +0,0 @@
-#include "util++.H"
-
-int
-main(int argc, char **argv) {
- logMsg M;
-
- M.add("this is a simple test\n");
- M.add("%s %s %s %s %s\n", "1", "2", "3", "4", "5");
-
- M.add("%s%s%s%s%s",
- "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n",
- "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n",
- "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\n",
- "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\n",
- "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\n");
-
- for (int a=0; a<1024; a++) {
- M.add("%s%s%s%s%s",
- "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa [...]
- "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb [...]
- "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc [...]
- "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd [...]
- "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee [...]
- }
-
- M.fwrite(stdout);
-}
diff --git a/kmer/libutil/test/test-md5.c b/kmer/libutil/test/test-md5.c
deleted file mode 100644
index 5713b08..0000000
--- a/kmer/libutil/test/test-md5.c
+++ /dev/null
@@ -1,47 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "util.h"
-
-// Performs the md5 test suite using libbri. MD5 itself is tested in
-// external/md5.
-//
-// Appendix 5 of RFC 1321;
-//
-// MD5 test suite:
-// MD5 ("") = d41d8cd98f00b204e9800998ecf8427e
-// MD5 ("a") = 0cc175b9c0f1b6a831c399e269772661
-// MD5 ("abc") = 900150983cd24fb0d6963f7d28e17f72
-// MD5 ("message digest") = f96b697d7cb7938d525a2f31aaf161d0
-// MD5 ("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
-// MD5 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
-// MD5 ("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
-//
-
-int
-testit(char *str, char *ans) {
- md5_s m;
- char r[33];
- int ret = 0;
-
- md5_toascii(md5_string(&m, str, strlen(str)), r);
- ret = strcmp(r, ans);
- if (ret)
- printf("ERROR: expect %s, got %s for %s\n", ans, r, str);
- return(ret == 0);
-}
-
-int
-main(int argc, char **argv) {
- int ret = 7;
-
- ret -= testit("", "d41d8cd98f00b204e9800998ecf8427e");
- ret -= testit("a", "0cc175b9c0f1b6a831c399e269772661");
- ret -= testit("abc", "900150983cd24fb0d6963f7d28e17f72");
- ret -= testit("message digest", "f96b697d7cb7938d525a2f31aaf161d0");
- ret -= testit("abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b");
- ret -= testit("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "d174ab98d277d9f5a5611c2c9f419d9f");
- ret -= testit("12345678901234567890123456789012345678901234567890123456789012345678901234567890", "57edf4a22be3c955ac49da2e2107b67a");
- exit(ret);
-}
diff --git a/kmer/libutil/test/test-mmap.c b/kmer/libutil/test/test-mmap.c
deleted file mode 100644
index 15cb4c1..0000000
--- a/kmer/libutil/test/test-mmap.c
+++ /dev/null
@@ -1,80 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-
-#include "util.h"
-
-// Does a quick test of memory mapped files. First, writes a small
-// file, then it reads it back, checking the data.
-//
-// Takes one optional argument, the size in MB of the file to map.
-
-int
-main(int argc, char **argv) {
- size_t lw;
- u32bit *ww = 0L;
- u32bit idx = 0;
- u32bit err = 0;
- FILE *out;
- u32bit blockSize = 1048576;
- u32bit numBlocks = 32;
-
- if (argc == 2)
- numBlocks = strtou32bit(argv[1], 0L);
-
- // The file must exist, and it must be large enough to contain all
- // that we want to write. So, we create the file and fill it with
- // junk.
- //
- ww = (u32bit *)malloc(sizeof(u32bit) * blockSize);
- if (ww == NULL) {
- fprintf(stderr, "can't allocate %d u32bit's for clearing the file.\n", blockSize);
- exit(1);
- }
- errno = 0;
- out = fopen("mmap.test.junk", "w");
- if (errno) {
- fprintf(stderr, "can't open 'mmap.test.junk' to fill with junk: %s\n", strerror(errno));
- exit(1);
- }
- for (idx=0; idx<numBlocks; idx++) {
- fprintf(stderr, "Writing initial blocks: "u32bitFMT"/"u32bitFMT"\r", idx, numBlocks), fflush(stderr);
- fwrite(ww, sizeof(u32bit), 1048576, out);
- if (errno) {
- fprintf(stderr, "can't write to 'mmap.test.junk': %s\n", strerror(errno));
- exit(1);
- }
- }
- fclose(out);
- free(ww);
- fprintf(stderr, "\n");
-
- // Now, map it, and fill it with real data.
- //
- ww = (u32bit *)mapFile("mmap.test.junk", &lw, 'w');
- for (idx=0; idx<numBlocks * blockSize; idx++) {
- if ((idx & 0xfff) == 0)
- fprintf(stderr, "Writing: "u32bitFMT"/"u32bitFMT"\r", idx, numBlocks * blockSize), fflush(stderr);
- ww[idx] = idx;
- }
- unmapFile(ww, lw);
- fprintf(stderr, "\n");
-
- // Map again, and check the data.
- //
- ww = mapFile("mmap.test.junk", &lw, 'r');
- for (idx=0; idx<numBlocks * blockSize; idx++) {
- if ((idx & 0xfff) == 0)
- fprintf(stderr, "Verifying: "u32bitFMT"/"u32bitFMT"\r", idx, numBlocks * blockSize), fflush(stderr);
- if (ww[idx] != idx)
- err++;
- }
- unmapFile(ww, lw);
- fprintf(stderr, "\n");
-
- unlink("mmap.test.junk");
-
- return (err != 0);
-}
diff --git a/kmer/libutil/test/test-palloc.c b/kmer/libutil/test/test-palloc.c
deleted file mode 100644
index 9e9e792..0000000
--- a/kmer/libutil/test/test-palloc.c
+++ /dev/null
@@ -1,65 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "util.h"
-
-int
-main(int argc, char **argv) {
- mt_s *mtctx;
- int i;
-
- psetdebug(2);
- psetblocksize(1024);
-
- palloc(2048);
- palloc(128);
- palloc(999);
- palloc(1);
- palloc(2);
- palloc(3);
- palloc(4);
- palloc(2056);
- palloc(8);
- palloc(2064);
- palloc(8);
- palloc(2072);
- palloc(8);
-
- pdumppalloc();
-
- pfree();
-
- fprintf(stderr, "----------------------------------------\n");
-
- psetblocksize(10240);
-
- palloc(2048);
- palloc(128);
- palloc(999);
- palloc(8);
- palloc(8);
- palloc(8);
- palloc(8);
- palloc(2056);
- palloc(8);
- palloc(2064);
- palloc(8);
- palloc(2072);
- palloc(8);
-
- pdumppalloc();
-
- pfree();
-
- psetdebug(0);
- psetblocksize(16 * 1024 * 1024);
-
- mtctx = mtInit(time(NULL));
- for (i=0; i<512 * 1024; i++)
- palloc(mtRandom32(mtctx) & 0xfff);
- psetdebug(1);
- pfree();
-
- return(0);
-}
-
diff --git a/kmer/libutil/test/test-readBuffer.C b/kmer/libutil/test/test-readBuffer.C
deleted file mode 100644
index f6f1164..0000000
--- a/kmer/libutil/test/test-readBuffer.C
+++ /dev/null
@@ -1,135 +0,0 @@
-#include <stdio.h>
-
-#include "util++.H"
-
-char *filename = 0L;
-md5_s *full = 0L;
-md5_s *part = 0L;
-
-
-int
-doTest(readBuffer *B, md5_s *correct, const char *description) {
- int error = 0;
- md5_increment_s *testing = 0L;
- int bread = 0;
-
- fprintf(stderr, "readBuffer test %s.\n", description);
-
- for (char x = B->read(); !B->eof(); x = B->read()) {
- testing = md5_increment_char(testing, x);
- bread++;
- }
-
- md5_increment_finalize(testing);
-
- if ((testing->a != correct->a) || (testing->b != correct->b)) {
- fprintf(stderr, "readBuffer test %s failed (read %d bytes).\n", description, bread);
- fprintf(stderr, "Got correct md5 of "u64bitHEX" "u64bitHEX"\n", correct->a, correct->b);
- fprintf(stderr, "Got testing md5 of "u64bitHEX" "u64bitHEX"\n", testing->a, testing->b);
- error = 1;
- }
-
- md5_increment_destroy(testing);
-
- return(error);
-}
-
-
-int
-doTestRead(readBuffer *B, md5_s *correct, size_t bufferSize, const char *description) {
- int error = 0;
- char *buffer = new char [bufferSize];
- size_t bufferLen = 0;
-
- md5_increment_s *testing = 0L;
-
- fprintf(stderr, "readBuffer test %s.\n", description);
-
- while (!B->eof()) {
- bufferLen = B->read(buffer, bufferSize);
- //fprintf(stderr, "Read bufferLen=%d bufferSize=%d\n", bufferLen, bufferSize);
- testing = md5_increment_block(testing, buffer, bufferLen);
- }
-
- md5_increment_finalize(testing);
-
- if ((testing->a != correct->a) || (testing->b != correct->b)) {
- fprintf(stderr, "readBuffer test %s failed.\n", description);
- fprintf(stderr, "Got correct md5 of "u64bitHEX" "u64bitHEX"\n", correct->a, correct->b);
- fprintf(stderr, "Got testing md5 of "u64bitHEX" "u64bitHEX"\n", testing->a, testing->b);
- error = 1;
- }
-
- md5_increment_destroy(testing);
-
- return(error);
-}
-
-
-int
-main(int argc, char **argv) {
- int error = 0;
- readBuffer *B = 0L;
-
- size_t L = 0;
- size_t H = 0;
- size_t R = 0;
-
- // If we are given a file, use that, otherwise, use ourself.
- //
- filename = argv[argc-1];
-
- L = sizeOfFile(filename);
- H = L/2;
- R = L - H;
-
- fprintf(stderr, "L=%d H=%d R=%d\n", L, H, R);
-
- // Suck in the whole file, compute the correct md5 checksum on it
- //
- char *c = new char [L];
-
- FILE *F = fopen(filename, "r");
- fread(c, sizeof(char), L, F);
- fclose(F);
- full = md5_string(0L, c, L);
- part = md5_string(0L, c+H, R);
-
- delete [] c;
-
-
- B = new readBuffer(filename, 999);
- error += doTest(B, full, "#1 (read)");
- B->seek(0);
- error += doTest(B, full, "#2 (seek)");
- B->seek(H);
- error += doTest(B, part, "#2 (seek half)");
- delete B;
-
- B = new readBuffer(filename, 0);
- error += doTest(B, full, "#3 (mmap)");
- B->seek(0);
- error += doTest(B, full, "#2 (mmap seek)");
- B->seek(H);
- error += doTest(B, part, "#2 (mmap seek half)");
- delete B;
-
- B = new readBuffer(filename, 0);
- error += doTestRead(B, full, 10000, "#4 (read buffer=mmap readsize=10000)");
- delete B;
-
- B = new readBuffer(filename, 100);
- error += doTestRead(B, full, 10000, "#4 (read buffer=100 readsize=10000)");
- delete B;
-
- B = new readBuffer(filename, 2000);
- error += doTestRead(B, full, 1000, "#4 (read buffer=2000 readsize=1000)");
- delete B;
-
- B = new readBuffer(filename, L);
- error += doTestRead(B, full, L+1000, "#5 (read buffer=filesize readsize=filesize+1000)");
- delete B;
-
- return(error);
-}
-
diff --git a/kmer/libutil/test/test-recordFile.C b/kmer/libutil/test/test-recordFile.C
deleted file mode 100644
index ae26544..0000000
--- a/kmer/libutil/test/test-recordFile.C
+++ /dev/null
@@ -1,64 +0,0 @@
-#include <unistd.h>
-#include <time.h>
-#include <math.h>
-
-#include "util++.H"
-
-struct header_s {
- u64bit t1;
- char s1[570];
- u64bit t2;
-};
-
-struct record_s {
- u64bit t1;
- char s1[123];
-};
-
-int
-main(int argc, char **argv) {
- header_s h;
- record_s r;
-
- h.t1 = 0x0123456789abcdefllu;
- memset(h.s1, 0x66, 570);
- strcpy(h.s1, "this is the header");
- h.t2 = 0xdeadbeefdeadbeefllu;
-
- recordFile *RF = new recordFile("test", sizeof(header_s), sizeof(record_s), 'w');
-
- memcpy(RF->header(), &h, sizeof(header_s));
-
- r.t1 = 1; memset(r.s1, 0x66, 123); strcpy(r.s1, "record1");
- RF->putRecord(&r);
-
- r.t1 = 2; memset(r.s1, 0x66, 123); strcpy(r.s1, "record2");
- RF->putRecord(&r);
-
- r.t1 = 3; memset(r.s1, 0x66, 123); strcpy(r.s1, "record3");
- RF->putRecord(&r);
-
- r.t1 = 4; memset(r.s1, 0x66, 123); strcpy(r.s1, "record4");
- RF->putRecord(&r);
-
- r.t1 = 5; memset(r.s1, 0x66, 123); strcpy(r.s1, "record5");
- RF->putRecord(&r);
-
- delete RF;
-
- RF = new recordFile("test", sizeof(header_s), sizeof(record_s), 'r');
-
- header_s *hh = (header_s *)RF->header();
-
- fprintf(stderr, "header t1 "u64bitHEX" '%s' t2 "u64bitHEX"\n", hh->t1, hh->s1, hh->t2);
- RF->getRecord(&r); fprintf(stderr, "record "u64bitFMT" '%s'\n", r.t1, r.s1);
- RF->getRecord(&r); fprintf(stderr, "record "u64bitFMT" '%s'\n", r.t1, r.s1);
- RF->getRecord(&r); fprintf(stderr, "record "u64bitFMT" '%s'\n", r.t1, r.s1);
- RF->getRecord(&r); fprintf(stderr, "record "u64bitFMT" '%s'\n", r.t1, r.s1);
- RF->getRecord(&r); fprintf(stderr, "record "u64bitFMT" '%s'\n", r.t1, r.s1);
- RF->getRecord(&r); fprintf(stderr, "record "u64bitFMT" '%s'\n", r.t1, r.s1);
-
- delete RF;
-
- return(0);
-}
diff --git a/kmer/libutil/test/test-types.c b/kmer/libutil/test/test-types.c
deleted file mode 100644
index 0814e3b..0000000
--- a/kmer/libutil/test/test-types.c
+++ /dev/null
@@ -1,34 +0,0 @@
-#include <stdio.h>
-
-#include "util.h"
-
-int
-main(void) {
- u32bit errors = 0;
- u32bit u3 = -1;
- s32bit s3 = -1;
- u64bit u6 = -1;
- s64bit s6 = -1;
-
- if (sizeof(u32bit) != 4)
- fprintf(stderr, "u32bit has %d bytes (should be 4)!\n", (int)sizeof(u32bit)), errors++;
-
- if (sizeof(u64bit) != 8)
- fprintf(stderr, "u64bit has %d bytes (should be 8)!\n", (int)sizeof(u64bit)), errors++;
-
- if (u3 < 0)
- fprintf(stderr, "u32bit is signed (should be unsigned)!\n"), errors++;
-
- if (s3 > 0)
- fprintf(stderr, "s32bit is unsigned (should be signed)!\n"), errors++;
-
- if (u6 < 0)
- fprintf(stderr, "u64bit is signed (should be unsigned)!\n"), errors++;
-
- if (s6 > 0)
- fprintf(stderr, "s64bit is unsigned (should be signed)!\n"), errors++;
-
- return(errors);
-}
-
-
diff --git a/kmer/libutil/u32bitList.H b/kmer/libutil/u32bitList.H
deleted file mode 100644
index b758ef1..0000000
--- a/kmer/libutil/u32bitList.H
+++ /dev/null
@@ -1,62 +0,0 @@
-#ifndef U32BITLIST_H
-#define U32BITLIST_H
-
-#include <string.h>
-
-// A very simple integer list. Hopefully lighter weight than a
-// vector.
-
-// It might be useful to extend this to have 'undef' values,
-// and to allow shift(), pop().
-
-class u32bitList {
-public:
- u32bitList(u32bit max=16) {
- _len = 0;
- _max = max;
- _lst = new u32bit [_max];
- };
- ~u32bitList() {
- delete [] _lst;
- };
-
-private:
- void resize(u32bit idx) {
- if (idx >= _max) {
- _max *= 2;
- u32bit *L = new u32bit [_max];
- memcpy(L, _lst, sizeof(u32bit) * _len);
- delete [] _lst;
- _lst = L;
- }
- if (idx >= _len)
- _len = idx + 1;
- }
-
-public:
- u32bit &operator[](u32bit idx) {
- resize(idx);
- return(_lst[idx]);
- }
-
- void push(u32bit val) {
- resize(_len);
- _lst[_len++] = val;
- }
-
- u32bit length(void) {
- return(_len);
- };
-
- void clear(void) {
- _len = 0;
- }
-
-private:
- u32bit _len;
- u32bit _max;
- u32bit *_lst;
-};
-
-
-#endif // U32BITLIST_H
diff --git a/kmer/libutil/unaryEncoding.h b/kmer/libutil/unaryEncoding.h
deleted file mode 100644
index 1405ade..0000000
--- a/kmer/libutil/unaryEncoding.h
+++ /dev/null
@@ -1,76 +0,0 @@
-#ifndef UNARY_ENCODING_H
-#define UNARY_ENCODING_H
-
-#include "bitPacking.h"
-
-
-// Routines to store and retrieve a unary encoded number to/from a
-// bit packed word array based at 'ptr' and currently at location
-// 'pos'. Both routines return the size of the encoded number in
-// 'siz'.
-
-
-
-// The usual unary encoding. Store the number n as n 0 bits followed
-// by a single 1 bit.
-//
-// 0 -> 1
-// 1 -> 01
-// 2 -> 001
-// 3 -> 0001
-// 4 -> 00001
-//
-// See the decoder as to why we use 0 instead of 1 for the count.
-
-
-inline
-void
-setUnaryEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz,
- u64bit val) {
-
- *siz = val + 1;
-
- while (val >= 64) {
- setDecodedValue(ptr, pos, 64, u64bitZERO);
- pos += 64;
- val -= 64;
- siz += 64;
- }
-
- setDecodedValue(ptr, pos, val + 1, u64bitONE);
- pos += val + 1;
-}
-
-
-
-inline
-u64bit
-getUnaryEncodedNumber(u64bit *ptr,
- u64bit pos,
- u64bit *siz) {
- u64bit val = u64bitZERO;
- u64bit enc = u64bitZERO;
-
- // How many whole words are zero?
- //
- enc = getDecodedValue(ptr, pos, 64);
- while (enc == u64bitZERO) {
- val += 64;
- pos += 64;
- enc = getDecodedValue(ptr, pos, 64);
- }
-
- // This word isn't zero. Count how many bits are zero (see, the
- // choice of 0 or 1 for the encoding wasn't arbitrary!)
- //
- val += 64 - logBaseTwo64(enc);
-
- *siz = val + 1;
-
- return(val);
-}
-
-
-#endif // UNARY_ENCODING_H
diff --git a/kmer/libutil/unaryEncodingTester.C b/kmer/libutil/unaryEncodingTester.C
deleted file mode 100644
index 149797b..0000000
--- a/kmer/libutil/unaryEncodingTester.C
+++ /dev/null
@@ -1,199 +0,0 @@
-#include "util++.H"
-
-
-u64bit numLoops = 1;
-u64bit numNums = 4000000;
-u64bit numSize = 300;
-
-// The space in bits that we can play with, and the pointer to said space.
-//
-u64bit spa = 128 * 1024 * 1024 * 8;
-u64bit *ptr = 0L;
-u64bit *rnd = 0L;
-
-void
-testUnary(void) {
- u64bit pos = u64bitZERO;
- u64bit siz = u64bitZERO;
- u64bit val = u64bitZERO;
- u64bit i = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- setUnaryEncodedNumber(ptr, pos, &siz, rnd[i]);
- pos += siz;
- if (pos + 1000 >= spa) {
- fprintf(stderr, "ERROR: Ran out of space in testUnary at number "u64bitFMT" out of "u64bitFMT"\n", i, numNums);
- exit(1);
- }
- }
-
- //fprintf(stderr, "unaryEncodedNumbers used "u64bitFMT"MB of storage out of "u64bitFMT"MB.\n", pos >> 23, spa >> 23);
-
- pos = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- val = getUnaryEncodedNumber(ptr, pos, &siz);
- if (val != rnd[i]) {
- fprintf(stderr, "Number "u64bitFMT" at bitpos "u64bitFMT" failed. Desired "u64bitFMT" got "u64bitFMT"\n", i, pos, rnd[i], val);
- exit(1);
- }
- pos += siz;
- }
-
- fprintf(stderr, "unary encoded numbers OK!\n");
-}
-
-
-
-void
-testGeneralizedUnary(void) {
- u64bit pos = u64bitZERO;
- u64bit siz = u64bitZERO;
- u64bit val = u64bitZERO;
- u64bit i = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- setGeneralizedUnaryEncodedNumber(ptr, pos, &siz, rnd[i]);
- pos += siz;
- if (pos + 1000 >= spa) {
- fprintf(stderr, "ERROR: Ran out of space in testGeneralizedUnary at number "u64bitFMT" out of "u64bitFMT"\n", i, numNums);
- exit(1);
- }
- }
-
- //fprintf(stderr, "generalizedUnaryEncodedNumbers used "u64bitFMT"MB of storage out of "u64bitFMT"MB.\n", pos >> 23, spa >> 23);
-
- pos = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- val = getGeneralizedUnaryEncodedNumber(ptr, pos, &siz);
- if (val != rnd[i]) {
- fprintf(stderr, "Number "u64bitFMT" at bitpos "u64bitFMT" failed. Desired "u64bitFMT" got "u64bitFMT"\n", i, pos, rnd[i], val);
- exit(1);
- }
- pos += siz;
- }
-
- fprintf(stderr, "generalized unary encoded numbers OK!\n");
-}
-
-
-
-
-void
-testEliasGamma(void) {
- u64bit pos = u64bitZERO;
- u64bit siz = u64bitZERO;
- u64bit val = u64bitZERO;
- u64bit i = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- setEliasGammaEncodedNumber(ptr, pos, &siz, rnd[i]);
- pos += siz;
- if (pos + 1000 >= spa) {
- fprintf(stderr, "ERROR: Ran out of space in testGeneralizedUnary at number "u64bitFMT" out of "u64bitFMT"\n", i, numNums);
- exit(1);
- }
- }
-
- //fprintf(stderr, "eliasGammaEncodedNumbers used "u64bitFMT"MB of storage out of "u64bitFMT"MB.\n", pos >> 23, spa >> 23);
-
- pos = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- val = getEliasGammaEncodedNumber(ptr, pos, &siz);
- if (val != rnd[i]) {
- fprintf(stderr, "Number "u64bitFMT" at bitpos "u64bitFMT" failed. Desired "u64bitFMT" got "u64bitFMT"\n", i, pos, rnd[i], val);
- exit(1);
- }
- pos += siz;
- }
-
- fprintf(stderr, "Elias gamma encoded numbers OK!\n");
-}
-
-
-
-void
-testEliasDelta(void) {
- u64bit pos = u64bitZERO;
- u64bit siz = u64bitZERO;
- u64bit val = u64bitZERO;
- u64bit i = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- setEliasDeltaEncodedNumber(ptr, pos, &siz, rnd[i]);
- pos += siz;
- if (pos + 1000 >= spa) {
- fprintf(stderr, "ERROR: Ran out of space in testGeneralizedUnary at number "u64bitFMT" out of "u64bitFMT"\n", i, numNums);
- exit(1);
- }
- }
-
- //fprintf(stderr, "eliasDeltaEncodedNumbers used "u64bitFMT"MB of storage out of "u64bitFMT"MB.\n", pos >> 23, spa >> 23);
-
- pos = u64bitZERO;
-
- for (i=0; i<numNums; i++) {
- val = getEliasDeltaEncodedNumber(ptr, pos, &siz);
- if (val != rnd[i]) {
- fprintf(stderr, "Number "u64bitFMT" at bitpos "u64bitFMT" failed. Desired "u64bitFMT" got "u64bitFMT"\n", i, pos, rnd[i], val);
- exit(1);
- }
- pos += siz;
- }
-
- fprintf(stderr, "Elias delta encoded numbers OK!\n");
-}
-
-
-
-
-
-int
-main(int argc, char **argv) {
-
- if (argc != 3) {
- fprintf(stderr, "usage: %s <num-loops> <num-nums-per-loop>\n", argv[0]);
- fprintf(stderr, " -> DEFAULTS USED <-\n");
- } else {
- numLoops = strtou32bit(argv[1], 0L);
- numNums = strtou32bit(argv[2], 0L);
- }
-
- rnd = new u64bit [numNums];
- ptr = new u64bit [spa >> 6];
-
- mt_s *ctx = mtInit(time(NULL));
-
- // Generate some random numbers to store
- //
- while (numLoops--) {
-
- // Test out unary encodings on small numbers
- //
- for (u64bit i=0; i<numNums; i++)
- rnd[i] = mtRandom32(ctx) % numSize;
- testUnary();
-
- // Generalized unary encoding can handle larger numbers
- //
- for (u64bit i=0; i<numNums; i++)
- rnd[i] = mtRandom32(ctx);
- testGeneralizedUnary();
-
- // Elias Gamma and Delta codes are probably pretty good
- //
- for (u64bit i=0; i<numNums; i++)
- rnd[i] = mtRandom64(ctx);
- testEliasGamma();
- testEliasDelta();
- }
-
- delete [] rnd;
- delete [] ptr;
-
- exit(0);
-}
-
-
diff --git a/kmer/libutil/util++.H b/kmer/libutil/util++.H
deleted file mode 100644
index 4397d23..0000000
--- a/kmer/libutil/util++.H
+++ /dev/null
@@ -1,46 +0,0 @@
-#ifndef UTIL_PLUS_PLUS_H
-#define UTIL_PLUS_PLUS_H
-
-#include "util.h"
-
-// These are all inlined, and C doesn't want to listen to that, so
-// they're here.
-//
-#include "bitOperations.h"
-#include "bitPacking.h"
-#include "endianess.H"
-
-// Various methods for encoding numbers into a bitstream.
-//
-// Still missing:
-// minimal binary
-// golomb (actually rice, since power of two)
-// teuhola exponential golomb
-//
-// And a nice way of getting parameters to those (and generalizedUnary)
-//
-#include "unaryEncoding.h"
-#include "generalizedUnaryEncoding.h"
-#include "eliasGammaEncoding.h"
-#include "eliasDeltaEncoding.h"
-#include "fibonacciEncoding.h"
-
-// Lists?
-#include "u32bitList.H"
-
-// Now the good stuff!
-//
-#include "speedCounter.H"
-//#include "bzipBuffer.H"
-#include "readBuffer.H"
-#include "splitToWords.H"
-#include "bitPackedArray.H"
-#include "bitPackedFile.H"
-#include "recordFile.H"
-#include "intervalList.H"
-#include "bigQueue.H"
-#include "sweatShop.H"
-#include "logMsg.H"
-
-#endif // UTIL_PLUS_PLUS_H
-
diff --git a/kmer/libutil/util.c b/kmer/libutil/util.c
deleted file mode 100644
index 0c0d65f..0000000
--- a/kmer/libutil/util.c
+++ /dev/null
@@ -1,115 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/stat.h>
-
-#include <sys/types.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-
-
-#include "util.h"
-
-
-double
-getTime(void) {
- struct timeval tp;
- gettimeofday(&tp, NULL);
- return(tp.tv_sec + (double)tp.tv_usec / 1000000.0);
-}
-
-
-u64bit
-getProcessSizeCurrent(void) {
- struct rusage ru;
- u64bit sz = 0;
-
- errno = 0;
- if (getrusage(RUSAGE_SELF, &ru) == -1) {
- fprintf(stderr, "getProcessSizeCurrent()-- getrusage(RUSAGE_SELF, ...) failed: %s\n",
- strerror(errno));
- } else {
- sz = ru.ru_maxrss;
- sz *= 1024;
- }
-
- return(sz);
-}
-
-
-u64bit
-getProcessSizeLimit(void) {
- struct rlimit rlp;
- u64bit sz = ~u64bitZERO;
-
- errno = 0;
- if (getrlimit(RLIMIT_DATA, &rlp) == -1) {
- fprintf(stderr, "getProcessSizeLimit()-- getrlimit(RLIMIT_DATA, ...) failed: %s\n",
- strerror(errno));
- } else {
- sz = rlp.rlim_cur;
- }
-
- return(sz);
-}
-
-
-
-
-void *
-memdup(const void *orig, size_t size) {
- void *rslt = NULL;
-
- if ((orig != NULL) && (size > 0)) {
- errno = 0;
- rslt = malloc(size);
- if (errno) {
- // Some ugliness to print out a size_t. This might be useless,
- // as it might be determined by TRUE64BIT.
- //
- if (sizeof(size_t) == 8)
- fprintf(stderr, "memdup()-- can't allocate "s64bitFMT" bytes.\n%s\n", (s64bit)size, strerror(errno));
- else
- fprintf(stderr, "memdup()-- can't allocate "u32bitFMT" bytes.\n%s\n", (u32bit)size, strerror(errno));
- exit(1);
- }
- memcpy(rslt, orig, size);
- }
- return(rslt);
-}
-
-
-int
-fileExists(const char *path) {
- struct stat s;
-
- return(stat(path, &s) == 0);
-}
-
-
-off_t
-sizeOfFile(const char *path) {
- struct stat s;
-
- errno = 0;
- if (stat(path, &s) != 0)
- fprintf(stderr, "Couldn't stat() '%s'\n%s\n", path, strerror(errno)), exit(1);
-
- return(s.st_size);
-}
-
-
-u64bit
-timeOfFile(const char *path) {
- struct stat s;
-
- errno = 0;
- if (stat(path, &s) != 0)
- fprintf(stderr, "Couldn't stat() '%s'\n%s\n", path, strerror(errno)), exit(1);
-
- return(s.st_mtime);
-}
diff --git a/kmer/libutil/util.h b/kmer/libutil/util.h
deleted file mode 100644
index 15d8928..0000000
--- a/kmer/libutil/util.h
+++ /dev/null
@@ -1,356 +0,0 @@
-#ifndef UTIL_H
-#define UTIL_H
-
-// ISO C99 says that to get INT32_MAX et al, these must be defined. (7.18.2, 7.18.4, 7.8.1)
-#ifndef __STDC_CONSTANT_MACROS
-#define __STDC_CONSTANT_MACROS
-#endif
-#ifndef __STDC_LIMIT_MACROS
-#define __STDC_LIMIT_MACROS
-#endif
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <ctype.h>
-
-#include <inttypes.h>
-
-// Useful types.
-//
-// *MASK(x) is only defined for unsigned types, with x != 0 and less
-// than the datawidth.
-
-typedef uint64_t u64bit;
-typedef uint32_t u32bit;
-typedef uint16_t u16bit;
-typedef uint8_t u8bit;
-
-typedef int64_t s64bit;
-typedef int32_t s32bit;
-typedef int16_t s16bit;
-typedef int8_t s8bit;
-
-
-#if defined(__alpha) || defined(_AIX) || defined(__LP64__) || defined(_LP64)
-#define TRUE64BIT
-#define u64bitNUMBER(X) X ## LU
-#define u32bitNUMBER(X) X ## U
-#else
-#define u64bitNUMBER(X) X ## LLU
-#define u32bitNUMBER(X) X ## LU
-#endif
-
-
-#define sizetFMT "%zd"
-
-#define u64bitZERO u64bitNUMBER(0x0000000000000000)
-#define u64bitONE u64bitNUMBER(0x0000000000000001)
-#define u64bitMAX u64bitNUMBER(0xffffffffffffffff)
-#define u64bitMASK(X) ((~u64bitZERO) >> (64 - (X)))
-#define u64bitFMTW(X) "%" #X PRIu64
-#define u64bitFMT "%"PRIu64
-#define u64bitHEX "0x%016"PRIx64
-#define s64bitFMTW(X) "%" #X PRId64
-#define s64bitFMT "%"PRId64
-
-#define u32bitZERO u32bitNUMBER(0x00000000)
-#define u32bitONE u32bitNUMBER(0x00000001)
-#define u32bitMAX u32bitNUMBER(0xffffffff)
-#define u32bitMASK(X) ((~u32bitZERO) >> (32 - (X)))
-#define u32bitFMTW(X) "%" #X PRIu32
-#define u32bitFMT "%"PRIu32
-#define u32bitHEX "0x%08"PRIx32
-#define s32bitFMTW(X) "%" #X PRId32
-#define s32bitFMT "%"PRId32
-
-#define u16bitZERO (0x0000)
-#define u16bitONE (0x0001)
-#define u16bitMAX (0xffff)
-#define u16bitMASK(X) ((~u16bitZERO) >> (16 - (X)))
-#define u16bitFMTW(X) "%" #X PRIu16
-#define u16bitFMT "%"PRIu16
-
-#define u8bitZERO (0x00)
-#define u8bitONE (0x01)
-#define u8bitMAX (0xff)
-#define u8bitMASK(X) ((~u8bitZERO) >> (8 - (X)))
-
-#define strtou32bit(N,O) (u32bit)strtoul(N, O, 10)
-#define strtou64bit(N,O) (u64bit)strtoul(N, O, 10)
-
-
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-
-
-////////////////////////////////////////
-//
-// time
-//
-double getTime(void);
-
-
-
-////////////////////////////////////////
-//
-// file
-//
-
-// Create the O_LARGEFILE type for open(), if it doesn't already
-// exist (FreeBSD, Tru64). We assume that by including the stuff
-// needed for open(2) we'll get any definition of O_LARGEFILE.
-//
-#ifndef O_LARGEFILE
-#define O_LARGEFILE 0
-#endif
-
-
-u64bit getProcessSizeCurrent(void);
-u64bit getProcessSizeLimit(void);
-
-
-// Useful routines for dealing with the existence of files
-
-int isHuman(FILE *F);
-
-// Handles mmap() of files. Write is not tested -- in particluar,
-// the test main() in mmap.c fails.
-//
-void*
-mapFile(const char *filename,
- u64bit *length,
- char mode);
-
-void
-unmapFile(void *addr,
- u64bit length);
-
-
-
-// Creates a hidden temporary file. If path is given, the temporary
-// file is created in that directory. The temoprary file is unlinked
-// after it is created, so once you close the file, it's gone.
-//
-FILE *makeTempFile(char *path);
-
-
-// Copies all of srcFile to dstFile, returns the number of bytes written
-//
-off_t copyFile(char *srcName, FILE *dstFile);
-
-
-// Takes a path to a file (that possibly doesn't exist) and returns
-// the number of MB (1048576 bytes) free in the directory of that
-// file.
-//
-u32bit freeDiskSpace(char *path);
-
-// Safer read(2) and write(2).
-//
-void safeWrite(int filedes, const void *buffer, const char *desc, size_t nbytes);
-int safeRead(int filedes, const void *buffer, const char *desc, size_t nbytes);
-
-
-
-////////////////////////////////////////
-//
-int fileExists(const char *path);
-off_t sizeOfFile(const char *path);
-u64bit timeOfFile(const char *path);
-
-// Open a file, read/write, using compression based on the file name
-//
-FILE *openFile(const char *path, const char *mode);
-void closeFile(FILE *F, const char *path);
-
-////////////////////////////////////////
-//
-void *memdup(const void *orig, size_t size);
-
-
-////////////////////////////////////////
-//
-// Pac-Man's memory allocator.
-//
-// Grabs big chunks of memory, then gives out little pieces. You can
-// only free ALL memory, not single blocks.
-//
-// This is useful when one needs to malloc() tens of millions of
-// things, at which point the overhead of finding a free block is
-// large.
-//
-void *palloc(size_t size);
-void pfree(void);
-
-// A thread-safe(r) implementation just forces the user to use a
-// handle. This also lets us use palloc() for collections of things
-// -- e.g., twice in a program. If you don't give a handle, the
-// default one is used.
-//
-void *palloc2(size_t size, void *handle);
-void pfree2(void *handle);
-
-// Get a new handle, release a used one. The size is the same
-// as for psetblocksize().
-//
-void *pallochandle(size_t size);
-void pfreehandle(void *handle);
-
-// The block size can only be changed before the first call to
-// palloc(). Calling psetblocksize() after that has no effect.
-//
-void psetblocksize(size_t size);
-size_t pgetblocksize(void);
-
-// Not generally useful - just dumps the allocated blocks to stdout.
-// Uses internal structures, and used in the test routine.
-//
-// psetdebug() enables reporting of allocations.
-//
-void pdumppalloc(void *handle);
-void psetdebug(int on);
-
-
-////////////////////////////////////////
-//
-// md5
-//
-
-
-typedef struct {
- u64bit a;
- u64bit b;
- u32bit i; // the iid, used in leaff
- u32bit pad; // keep us size compatible between 32- and 64-bit machines.
-} md5_s;
-
-#define MD5_BUFFER_SIZE 32*1024
-
-typedef struct {
- u64bit a;
- u64bit b;
- void *context;
- int bufferPos;
- unsigned char buffer[MD5_BUFFER_SIZE];
-} md5_increment_s;
-
-
-// Returns -1, 0, 1 depending on if a <, ==, > b. Suitable for
-// qsort().
-//
-int md5_compare(void const *a, void const *b);
-
-
-// Converts an md5_s into a character string. s must be at least
-// 33 bytes long.
-//
-char *md5_toascii(md5_s *m, char *s);
-
-
-// Computes the md5 checksum on the string s.
-//
-md5_s *md5_string(md5_s *m, char *s, u32bit l);
-
-
-// Computes an md5 checksum piece by piece.
-//
-// If m is NULL, a new md5_increment_s is allocated and returned.
-//
-md5_increment_s *md5_increment_char(md5_increment_s *m, char s);
-md5_increment_s *md5_increment_block(md5_increment_s *m, char *s, u32bit l);
-void md5_increment_finalize(md5_increment_s *m);
-void md5_increment_destroy(md5_increment_s *m);
-
-
-////////////////////////////////////////
-//
-// Matsumoto and Nichimura's Mersenne Twister pseudo random number
-// generator. The struct and functions are defined in external/mt19937ar.[ch]
-//
-typedef struct mtctx mt_s;
-
-mt_s *mtInit(u32bit s);
-mt_s *mtInitArray(u32bit *init_key, u32bit key_length);
-u32bit mtRandom32(mt_s *mt);
-
-// A u64bit random number
-//
-#define mtRandom64(MT) ( (((u64bit)mtRandom32(MT)) << 32) | (u64bit)mtRandom32(MT) )
-
-// Real valued randomness
-// mtRandomRealOpen() -- on [0,1) real interval
-// mtRandomRealClosed() -- on [0,1] real interval
-// mrRandomRealOpen53() -- on [0,1) real interval, using 53 bits
-//
-// "These real versions are due to Isaku Wada, 2002/01/09 added" and were taken from
-// the mt19937ar.c distribution (but they had actual functions, not macros)
-//
-// They also had
-// random number in (0,1) as (mtRandom32() + 0.5) * (1.0 / 4294967296.0)
-//
-#define mtRandomRealOpen(MT) ( (double)mtRandom32(MT) * (1.0 / 4294967296.0) )
-#define mtRandomRealClosed(MT) ( (double)mtRandom32(MT) * (1.0 / 4294967295.0) )
-#define mtRandomRealOpen53(MT) ( ((mtRandom32(MT) >> 5) * 67108864.0 + (mtRandom32(MT) >> 6)) * (1.0 / 9007199254740992.0) )
-
-// returns a random number with gaussian distribution, mean of zero and std.dev. of 1
-//
-double mtRandomGaussian(mt_s *mt);
-
-
-////////////////////////////////////////
-//
-// FreeBSD's multithreaded qsort.
-//
-void
-qsort_mt(void *a,
- size_t n,
- size_t es,
- int (*cmp)(const void *, const void *),
- int maxthreads,
- int forkelem);
-
-//#define qsort(A, N, ES, CMP) qsort_mt((A), (N), (ES), (CMP), 4, 64 * 1024)
-
-
-
-////////////////////////////////////////
-//
-// perl's chomp is pretty nice
-//
-#ifndef chomp
-#define chomp(S) { char *t=S; while (*t) t++; t--; while (isspace(*t)) { *t--=0; } }
-#define chompL(S,L) { char *t=S; while (*t) t++; t--; while (isspace(*t)) { *t--=0; L--; } }
-#endif
-
-#ifndef munch
-#define munch(S) { while (*(S) && isspace(*(S))) (S)++; }
-#endif
-
-#ifndef crunch
-#define crunch(S) { while (*(S) && !isspace(*(S))) (S)++; }
-#endif
-
-
-#ifndef MIN
-#define MIN(x,y) (((x) > (y)) ? (y) : (x))
-#endif
-
-#ifndef MAX
-#define MAX(x,y) (((x) < (y)) ? (y) : (x))
-#endif
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // UTIL_H
diff --git a/kmer/meryl/Make.include b/kmer/meryl/Make.include
deleted file mode 100644
index 494d4a2..0000000
--- a/kmer/meryl/Make.include
+++ /dev/null
@@ -1,43 +0,0 @@
-# -*- makefile -*-
-
-LIBUTL/ :=$(realpath $/../libutil/)/
-LIBBIO/ :=$(realpath $/../libbio/)/
-LIBSEQ/ :=$(realpath $/../libseq/)/
-LIBMERYL/ :=$(realpath $/../libmeryl/)/
-LIBKMER/ :=$(realpath $/../libkmer/)/
-
-merylsrc := $/args.C \
- $/binaryOp.C \
- $/build.C \
- $/build-threads.C \
- $/dump.C \
- $/estimate.C \
- $/merge.C \
- $/unaryOp.C
-
-# Removed m.C from SRCS. and m from EXES -- what's it do??
-
-# meryl.H is exported only for celera-assembler.
-
-$/.CXX_SRCS := ${merylsrc} $/meryl.C $/mervin.C $/asmMerQC.C $/mapMers.C $/mapMers-depth.C $/maskMers.C $/compare-counts.C $/simple.C
-$/.CXX_INCS := $/meryl.H
-$/.CXX_LIBS := $/libmerylguts.a
-$/.CXX_EXES := $/meryl $/mervin $/simple $/asmMerQC $/mapMers $/mapMers-depth $/testPositionBias $/maskMers $/compare-counts
-$/.CLEAN := $/*.o
-
-$/libmerylguts.a : ${merylsrc:.C=.o}
-
-$/meryl : $/meryl.o $/libmerylguts.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/mervin : $/mervin.o ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/simple : $/simple.o ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/asmMerQC : $/asmMerQC.o ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/mapMers : $/mapMers.o ${LIBKMER/}libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/mapMers-depth : $/mapMers-depth.o ${LIBKMER/}libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/m : $/m.o ${LIBKMER/}libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/testPositionBias : $/testPositionBias.o ${LIBKMER/}libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/maskMers : $/maskMers.o ${LIBKMER/}libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-$/compare-counts : $/compare-counts.o ${LIBKMER/}libkmer.a ${LIBMERYL/}libmeryl.a ${LIBSEQ/}libseq.a ${LIBBIO/}libbio.a ${LIBUTL/}libutil.a
-
-$(eval $/%.d $/%.o: CXXFLAGS+=-I${LIBMERYL/} -I${LIBKMER/} -I${LIBBIO/} -I${LIBSEQ/} -I${LIBUTL/})
-
-$/%.d: ${LIBBIO/}alphabet.h
diff --git a/kmer/meryl/args.C b/kmer/meryl/args.C
deleted file mode 100644
index c632ba5..0000000
--- a/kmer/meryl/args.C
+++ /dev/null
@@ -1,589 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "bio++.H"
-#include "meryl.H"
-
-
-// Some string handling utilities.
-//
-bool
-writeString(const char *str, FILE *F) {
- errno = 0;
-
- u32bit len = 0;
- if (str) {
- len = (u32bit)strlen(str) + 1;
- fwrite(&len, sizeof(u32bit), 1, F);
- fwrite( str, sizeof(char), len, F);
- } else {
- fwrite(&len, sizeof(u32bit), 1, F);
- }
-
- if (errno) {
- fprintf(stderr, "writeString()-- Failed to write string of length "u32bitFMT": %s\n", len, strerror(errno));
- fprintf(stderr, "writeString()-- First 80 bytes of string is:\n");
- fprintf(stderr, "%80.80s\n", str);
- return(false);
- }
-
- return(true);
-}
-
-char*
-readString(FILE *F) {
- errno = 0;
-
- u32bit len = 0;
- fread(&len, sizeof(u32bit), 1, F);
- if (errno) {
- fprintf(stderr, "readString()-- Failed to read string: %s\n", strerror(errno));
- exit(1);
- }
-
- char *str = 0L;
-
- if (len > 0) {
- str = new char [len];
- fread(str, sizeof(char), len, F);
- if (errno) {
- fprintf(stderr, "readString()-- Failed to read string: %s\n", strerror(errno));
- exit(1);
- }
- }
-
- return(str);
-}
-
-char*
-duplString(char *str) {
- char *dupstr = 0L;
- if (str) {
- u32bit len = (u32bit)strlen(str);
- dupstr = new char [len+1];
- strcpy(dupstr, str);
- }
- return(dupstr);
-}
-
-
-
-void
-merylArgs::usage(void) {
- fprintf(stderr, "usage: %s [personality] [global options] [options]\n", execName);
- fprintf(stderr, "\n");
- fprintf(stderr, "where personality is:\n");
- fprintf(stderr, " -P -- compute parameters\n");
- fprintf(stderr, " -B -- build table\n");
- fprintf(stderr, " -S -- scan table\n");
- fprintf(stderr, " -M -- \"math\" operations\n");
- fprintf(stderr, " -D -- dump table\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "-P: Given a sequence file (-s) or an upper limit on the\n");
- fprintf(stderr, " number of mers in the file (-n), compute the table size\n");
- fprintf(stderr, " (-t in build) to minimize the memory usage.\n");
- fprintf(stderr, " -m # (size of a mer; required)\n");
- fprintf(stderr, " -c # (homopolymer compression; optional)\n");
- fprintf(stderr, " -p (enable positions)\n");
- fprintf(stderr, " -s seq.fasta (seq.fasta is scanned to determine the number of mers)\n");
- fprintf(stderr, " -n # (compute params assuming file with this many mers in it)\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Only one of -s, -n need to be specified. If both are given\n");
- fprintf(stderr, " -s takes priority.\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "-B: Given a sequence file (-s) and lots of parameters, compute\n");
- fprintf(stderr, " the mer-count tables. By default, both strands are processed.\n");
- fprintf(stderr, " -f (only build for the forward strand)\n");
- fprintf(stderr, " -r (only build for the reverse strand)\n");
- fprintf(stderr, " -C (use canonical mers, assumes both strands)\n");
- fprintf(stderr, " -L # (DON'T save mers that occur less than # times)\n");
- fprintf(stderr, " -U # (DON'T save mers that occur more than # times)\n");
- fprintf(stderr, " -m # (size of a mer; required)\n");
- fprintf(stderr, " -c # (homopolymer compression; optional)\n");
- fprintf(stderr, " -p (enable positions)\n");
- fprintf(stderr, " -s seq.fasta (sequence to build the table for)\n");
- fprintf(stderr, " -o tblprefix (output table prefix)\n");
- fprintf(stderr, " -v (entertain the user)\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " By default, the computation is done as one large sequential process.\n");
- fprintf(stderr, " Multi-threaded operation is possible, at additional memory expense, as\n");
- fprintf(stderr, " is segmented operation, at additional I/O expense.\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Threaded operation: Split the counting in to n almost-equally sized\n");
- fprintf(stderr, " pieces. This uses an extra h MB (from -P) per thread.\n");
- fprintf(stderr, " -threads n (use n threads to build)\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Segmented, sequential operation: Split the counting into pieces that\n");
- fprintf(stderr, " will fit into no more than m MB of memory, or into n equal sized pieces.\n");
- fprintf(stderr, " Each piece is computed sequentially, and the results are merged at the end.\n");
- fprintf(stderr, " Only one of -memory and -segments is needed.\n");
- fprintf(stderr, " -memory mMB (use at most m MB of memory per segment)\n");
- fprintf(stderr, " -segments n (use n segments)\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Segmented, batched operation: Same as sequential, except this allows\n");
- fprintf(stderr, " each segment to be manually executed in parallel.\n");
- fprintf(stderr, " Only one of -memory and -segments is needed.\n");
- fprintf(stderr, " -memory mMB (use at most m MB of memory per segment)\n");
- fprintf(stderr, " -segments n (use n segments)\n");
- fprintf(stderr, " -configbatch (create the batches)\n");
- fprintf(stderr, " -countbatch n (run batch number n)\n");
- fprintf(stderr, " -mergebatch (merge the batches)\n");
- fprintf(stderr, " Initialize the compute with -configbatch, which needs all the build options.\n");
- fprintf(stderr, " Execute all -countbatch jobs, then -mergebatch to complete.\n");
- fprintf(stderr, " meryl -configbatch -B [options] -o file\n");
- fprintf(stderr, " meryl -countbatch 0 -o file\n");
- fprintf(stderr, " meryl -countbatch 1 -o file\n");
- fprintf(stderr, " ...\n");
- fprintf(stderr, " meryl -countbatch N -o file\n");
- fprintf(stderr, " meryl -mergebatch N -o file\n");
- fprintf(stderr, " Batched mode can run on the grid.\n");
- fprintf(stderr, " -sge jobname unique job name for this execution. Meryl will submit\n");
- fprintf(stderr, " jobs with name mpjobname, ncjobname, nmjobname, for\n");
- fprintf(stderr, " phases prepare, count and merge.\n");
- fprintf(stderr, " -sgebuild \"options\" any additional options to sge, e.g.,\n");
- fprintf(stderr, " -sgemerge \"options\" \"-p -153 -pe thread 2 -A merylaccount\"\n");
- fprintf(stderr, " N.B. - -N will be ignored\n");
- fprintf(stderr, " N.B. - be sure to quote the options\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "-M: Given a list of tables, perform a math, logical or threshold operation.\n");
- fprintf(stderr, " Unless specified, all operations take any number of databases.\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Math operations are:\n");
- fprintf(stderr, " min count is the minimum count for all databases. If the mer\n");
- fprintf(stderr, " does NOT exist in all databases, the mer has a zero count, and\n");
- fprintf(stderr, " is NOT in the output.\n");
- fprintf(stderr, " minexist count is the minimum count for all databases that contain the mer\n");
- fprintf(stderr, " max count is the maximum count for all databases\n");
- fprintf(stderr, " add count is sum of the counts for all databases\n");
- fprintf(stderr, " sub count is the first minus the second (binary only)\n");
- fprintf(stderr, " abs count is the absolute value of the first minus the second (binary only)\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Logical operations are:\n");
- fprintf(stderr, " and outputs mer iff it exists in all databases\n");
- fprintf(stderr, " nand outputs mer iff it exists in at least one, but not all, databases\n");
- fprintf(stderr, " or outputs mer iff it exists in at least one database\n");
- fprintf(stderr, " xor outputs mer iff it exists in an odd number of databases\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " Threshold operations are:\n");
- fprintf(stderr, " lessthan x outputs mer iff it has count < x\n");
- fprintf(stderr, " lessthanorequal x outputs mer iff it has count <= x\n");
- fprintf(stderr, " greaterthan x outputs mer iff it has count > x\n");
- fprintf(stderr, " greaterthanorequal x outputs mer iff it has count >= x\n");
- fprintf(stderr, " equal x outputs mer iff it has count == x\n");
- fprintf(stderr, " Threshold operations work on exactly one database.\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " -s tblprefix (use tblprefix as a database)\n");
- fprintf(stderr, " -o tblprefix (create this output)\n");
- fprintf(stderr, " -v (entertain the user)\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " NOTE: Multiple tables are specified with multiple -s switches; e.g.:\n");
- fprintf(stderr, " %s -M add -s 1 -s 2 -s 3 -s 4 -o all\n", execName);
- fprintf(stderr, " NOTE: It is NOT possible to specify more than one operation:\n");
- fprintf(stderr, " %s -M add -s 1 -s 2 -sub -s 3\n", execName);
- fprintf(stderr, " will NOT work.\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "-D: Dump the table (not all of these work).\n");
- fprintf(stderr, "\n");
- fprintf(stderr, " -Dd Dump a histogram of the distance between the same mers.\n");
- fprintf(stderr, " -Dt Dump mers >= a threshold. Use -n to specify the threshold.\n");
- fprintf(stderr, " -Dc Count the number of mers, distinct mers and unique mers.\n");
- fprintf(stderr, " -Dh Dump (to stdout) a histogram of mer counts.\n");
- fprintf(stderr, " -s Read the count table from here (leave off the .mcdat or .mcidx).\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "\n");
-}
-
-
-
-void
-merylArgs::clear(void) {
-
- execName = 0L;
- options = 0L;
-
- beVerbose = false;
- doForward = true;
- doReverse = false;
- doCanonical = false;
-
- inputFile = 0L;
- outputFile = 0L;
- queryFile = 0L;
-
- merSize = 20;
- merComp = 0;
- positionsEnabled = false;
-
- numMersEstimated = 0;
- numMersActual = 0;
-
- numBasesActual = 0;
-
- mersPerBatch = 0;
- basesPerBatch = 0;
-
- numBuckets = 0;
- numBuckets_log2 = 0;
- merDataWidth = 0;
- merDataMask = u64bitZERO;
- bucketPointerWidth = 0;
-
- numThreads = 0;
- memoryLimit = 0;
- segmentLimit = 0;
- configBatch = false;
- countBatch = false;
- mergeBatch = false;
- batchNumber = 0;
-
- sgeJobName = 0L;
- sgeBuildOpt = 0L;
- sgeMergeOpt = 0L;
- isOnGrid = false;
-
- lowCount = 0;
- highCount = ~lowCount;
- desiredCount = 0;
-
- outputCount = 0;
- outputAll = 0;
- outputPosition = 0;
-
- mergeFilesMax = 0;
- mergeFilesLen = 0;
- mergeFiles = 0L;
-
- personality = 0;
-}
-
-
-
-
-merylArgs::merylArgs(int argc, char **argv) {
-
- clear();
-
- execName = duplString(argv[0]);
-
- if (argc == 1) {
- usage();
- exit(1);
- }
-
- // Count how many '-s' switches there are, then allocate space
- // for them in mergeFiles. We also sum the length of all options,
- // so we can copy them into an 'options' string used when we
- // resubmit to the grid.
- //
- u32bit optionsLen = 0;
- for (int arg=1; arg < argc; arg++) {
- optionsLen += strlen(argv[arg]) + 1;
- if (strcmp(argv[arg], "-s") == 0)
- mergeFilesMax++;
- }
-
- mergeFiles = new char * [mergeFilesMax];
- options = new char [2 * optionsLen + 1];
- options[0] = 0;
-
- bool fail = false;
-
- char *optptr = options;
-
- for (int arg=1; arg < argc; arg++) {
- if (arg > 1)
- *optptr++ = ' ';
-
- // Arg! If the arg has spaces or other stuff that the shell
- // needs escaped we need to escape them again. So, we copy byte
- // by byte and insert escapes at the right points.
-
- for (char *op=argv[arg]; *op; op++, optptr++) {
- if (isspace(*op) || !isalnum(*op))
- if ((*op != '-') && (*op != '_') && (*op != '.') && (*op != '/'))
- *optptr++ = '\\';
- *optptr = *op;
- }
-
- //strcat(options, argv[arg]);
- }
-
-
- // Parse the options
- //
- for (int arg=1; arg < argc; arg++) {
- if (strncmp(argv[arg], "-V", 2) == 0) {
- fprintf(stdout, "meryl the Mighty Mer Counter version (no version)\n");
- exit(0);
- } else if (strcmp(argv[arg], "-m") == 0) {
- arg++;
- merSize = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-c") == 0) {
- arg++;
- merComp = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-p") == 0) {
- positionsEnabled = true;
- } else if (strcmp(argv[arg], "-s") == 0) {
- arg++;
- delete [] inputFile;
- inputFile = duplString(argv[arg]);
- mergeFiles[mergeFilesLen++] = duplString(argv[arg]);
- } else if (strcmp(argv[arg], "-n") == 0) {
- arg++;
- numMersEstimated = strtou64bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-f") == 0) {
- doForward = true;
- doReverse = false;
- doCanonical = false;
- } else if (strcmp(argv[arg], "-r") == 0) {
- doForward = false;
- doReverse = true;
- doCanonical = false;
- } else if (strcmp(argv[arg], "-C") == 0) {
- doForward = false;
- doReverse = false;
- doCanonical = true;
- } else if (strcmp(argv[arg], "-L") == 0) {
- arg++;
- lowCount = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-U") == 0) {
- arg++;
- highCount = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-o") == 0) {
- arg++;
- delete [] outputFile;
- outputFile = duplString(argv[arg]);
- } else if (strcmp(argv[arg], "-v") == 0) {
- beVerbose = true;
-
- } else if (strcmp(argv[arg], "-P") == 0) {
- personality = 'P';
- } else if (strcmp(argv[arg], "-B") == 0) {
- personality = 'B';
- } else if (strcmp(argv[arg], "-S") == 0) {
- personality = 'S';
- } else if (strcmp(argv[arg], "-M") == 0) {
- arg++;
- if (strcmp(argv[arg], "merge") == 0) {
- personality = PERSONALITY_MERGE;
- } else if (strcmp(argv[arg], "min") == 0) {
- personality = PERSONALITY_MIN;
- } else if (strcmp(argv[arg], "minexist") == 0) {
- personality = PERSONALITY_MINEXIST;
- } else if (strcmp(argv[arg], "max") == 0) {
- personality = PERSONALITY_MAX;
- } else if (strcmp(argv[arg], "maxexist") == 0) {
- personality = PERSONALITY_MAXEXIST;
- } else if (strcmp(argv[arg], "add") == 0) {
- personality = PERSONALITY_ADD;
- } else if (strcmp(argv[arg], "sub") == 0) {
- personality = PERSONALITY_SUB;
- } else if (strcmp(argv[arg], "abs") == 0) {
- personality = PERSONALITY_ABS;
- } else if (strcmp(argv[arg], "divide") == 0) {
- personality = PERSONALITY_DIVIDE;
- } else if (strcmp(argv[arg], "and") == 0) {
- personality = PERSONALITY_AND;
- } else if (strcmp(argv[arg], "nand") == 0) {
- personality = PERSONALITY_NAND;
- } else if (strcmp(argv[arg], "or") == 0) {
- personality = PERSONALITY_OR;
- } else if (strcmp(argv[arg], "xor") == 0) {
- personality = PERSONALITY_XOR;
- } else if (strcmp(argv[arg], "lessthan") == 0) {
- personality = PERSONALITY_LEQ;
- arg++;
- desiredCount = strtou32bit(argv[arg], 0L) - 1;
- } else if (strcmp(argv[arg], "lessthanorequal") == 0) {
- personality = PERSONALITY_LEQ;
- arg++;
- desiredCount = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "greaterthan") == 0) {
- personality = PERSONALITY_GEQ;
- arg++;
- desiredCount = strtou32bit(argv[arg], 0L) + 1;
- } else if (strcmp(argv[arg], "greaterthanorequal") == 0) {
- personality = PERSONALITY_GEQ;
- arg++;
- desiredCount = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "equal") == 0) {
- personality = PERSONALITY_EQ;
- arg++;
- desiredCount = strtou32bit(argv[arg], 0L);
- } else {
- fprintf(stderr, "ERROR: unknown math personality %s\n", argv[arg]);
- exit(1);
- }
- } else if (strcmp(argv[arg], "-Dd") == 0) {
- personality = 'd';
- } else if (strcmp(argv[arg], "-Dt") == 0) {
- personality = 't';
- } else if (strcmp(argv[arg], "-Dp") == 0) {
- personality = 'p';
- } else if (strcmp(argv[arg], "-Dc") == 0) {
- personality = 'c';
- } else if (strcmp(argv[arg], "-Dh") == 0) {
- personality = 'h';
- } else if (strcmp(argv[arg], "-memory") == 0) {
- arg++;
- memoryLimit = strtou64bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-segments") == 0) {
- arg++;
- segmentLimit = strtou64bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-threads") == 0) {
- arg++;
- numThreads = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-configbatch") == 0) {
- personality = 'B';
- configBatch = true;
- countBatch = false;
- mergeBatch = false;
- batchNumber = u32bitZERO;
- } else if (strcmp(argv[arg], "-countbatch") == 0) {
- arg++;
- personality = 'B';
- configBatch = false;
- countBatch = true;
- mergeBatch = false;
- batchNumber = strtou32bit(argv[arg], 0L);
- } else if (strcmp(argv[arg], "-mergebatch") == 0) {
- personality = 'B';
- configBatch = false;
- countBatch = false;
- mergeBatch = true;
- batchNumber = u32bitZERO;
- } else if (strcmp(argv[arg], "-sge") == 0) {
- sgeJobName = argv[++arg];
- } else if (strcmp(argv[arg], "-sgebuild") == 0) {
- sgeBuildOpt = argv[++arg];
- } else if (strcmp(argv[arg], "-sgemerge") == 0) {
- sgeMergeOpt = argv[++arg];
- } else if (strcmp(argv[arg], "-forcebuild") == 0) {
- isOnGrid = true;
- } else {
- fprintf(stderr, "Unknown option '%s'.\n", argv[arg]);
- fail = true;
- }
- }
-
- // Using threads is only useful if we are not a batch.
- //
- if ((numThreads > 0) && (configBatch || countBatch || mergeBatch)) {
- if (configBatch)
- fprintf(stderr, "WARNING: -threads has no effect with -configbatch, disabled.\n");
- if (countBatch)
- fprintf(stderr, "WARNING: -threads has no effect with -countbatch, disabled.\n");
- if (mergeBatch)
- fprintf(stderr, "WARNING: -threads has no effect with -mergebatch, disabled.\n");
- numThreads = 0;
- }
-
- // SGE is not useful unless we are in batch mode.
- //
- if (sgeJobName && !configBatch && !countBatch && !mergeBatch) {
- fprintf(stderr, "ERROR: -sge not useful unless in batch mode (replace -B with -configbatch)\n");
- exit(1);
- }
-
- if (fail)
- exit(1);
-}
-
-
-
-merylArgs::merylArgs(const char *prefix) {
-
- clear();
-
- char *filename = new char [strlen(prefix) + 17];
- sprintf(filename, "%s.merylArgs", prefix);
-
- errno = 0;
- FILE *F = fopen(filename, "rb");
- if (errno) {
- fprintf(stderr, "merylArgs::readConfig()-- Failed to open '%s': %s\n", filename, strerror(errno));
- exit(1);
- }
-
- char magic[17] = {0};
- fread(magic, sizeof(char), 16, F);
- if (strncmp(magic, "merylBatcherv02", 16) != 0) {
- fprintf(stderr, "merylArgs::readConfig()-- '%s' doesn't appear to be a merylArgs file.\n", filename);
- exit(1);
- }
-
- // Load the config, then reset the pointers.
-
- fread(this, sizeof(merylArgs), 1, F);
-
- execName = readString(F);
- options = 0L;
- inputFile = readString(F);
- outputFile = readString(F);
- queryFile = 0L;
- sgeJobName = readString(F);
- sgeBuildOpt = readString(F);
- sgeMergeOpt = readString(F);
-
- mergeFiles = new char* [mergeFilesLen];
-
- for (u32bit i=0; i<mergeFilesLen; i++)
- mergeFiles[i] = readString(F);
-
- fclose(F);
-
- delete [] filename;
-}
-
-
-
-merylArgs::~merylArgs() {
- delete [] execName;
- delete [] options;
- delete [] inputFile;
- delete [] outputFile;
-
- for (u32bit i=0; i<mergeFilesLen; i++)
- delete [] mergeFiles[i];
-
- delete [] mergeFiles;
-}
-
-
-
-bool
-merylArgs::writeConfig(void) {
- char *filename;
-
- filename = new char [strlen(outputFile) + 17];
- sprintf(filename, "%s.merylArgs", outputFile);
-
- errno = 0;
- FILE *F = fopen(filename, "wb");
- if (errno) {
- fprintf(stderr, "merylArgs::writeConfig()-- Failed to open '%s': %s\n", filename, strerror(errno));
- exit(1);
- }
-
- fwrite("merylBatcherv02", sizeof(char), 16, F);
-
- fwrite(this, sizeof(merylArgs), 1, F);
-
- writeString(execName, F);
- writeString(inputFile, F);
- writeString(outputFile, F);
- writeString(sgeJobName, F);
- writeString(sgeBuildOpt, F);
- writeString(sgeMergeOpt, F);
-
- for (u32bit i=0; i<mergeFilesLen; i++)
- writeString(mergeFiles[i], F);
-
- fclose(F);
-
- return(true);
-}
diff --git a/kmer/meryl/asmMerQC-regions.C b/kmer/meryl/asmMerQC-regions.C
deleted file mode 100644
index 8470f3e..0000000
--- a/kmer/meryl/asmMerQC-regions.C
+++ /dev/null
@@ -1,1023 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-
-// This reads the assembly frgctg, varctg and merQC badmers, computes
-// the number and location of bad-mer, bad-var regions, and their
-// depth, in contig space.
-//
-// File paths are hardcoded.
-// This code ONLY works on 64-bit hardware, but it's easy to fix.
-
-using namespace std;
-#include <map>
-
-//
-// Begin code from Bri's intervalList.H, intervalList.C, splitToWords.H
-//
-typedef unsigned long u64bit;
-typedef unsigned int u32bit;
-
-#define u64bitFMT "%lu"
-#define u32bitFMT "%u"
-#define u32bitFMTW(X) "%" #X "u"
-
-#define strtou32bit(N,O) (u32bit)strtoul(N, O, 10)
-#define strtou64bit(N,O) (u64bit)strtoul(N, O, 10)
-
-class splitToWords {
-public:
- splitToWords() {
- _argWords = 0;
- _maxWords = 0;
- _arg = 0L;
- _maxChars = 0;
- _cmd = 0L;
- };
- splitToWords(char *cmd) {
- _argWords = 0;
- _maxWords = 0;
- _arg = 0L;
- _maxChars = 0;
- _cmd = 0L;
-
- split(cmd);
- };
- ~splitToWords() {
- delete [] _cmd;
- delete [] _arg;
- };
-
-
- void split(char *cmd) {
-
- // Step Zero:
- //
- // Count the length of the string, in words and in characters.
- // For simplicity, we overcount words, by just counting white-space.
- //
- // Then, allocate space for a temporary copy of the string, and a
- // set of pointers into the temporary copy (much like argv).
- //
- u32bit cmdChars = 1; // 1 == Space for terminating 0
- u32bit cmdWords = 2; // 2 == Space for first word and terminating 0L
-
- for (char *tmp=cmd; *tmp; tmp++) {
- cmdWords += *tmp == ' ';
- cmdWords += *tmp == '\t';
- cmdChars++;
- }
-
- if (cmdChars > _maxChars) {
- delete [] _cmd;
- _cmd = new char [cmdChars];
- _maxChars = cmdChars;
- }
- if (cmdWords > _maxWords) {
- delete [] _arg;
- _arg = new char * [cmdWords];
- _maxWords = cmdWords;
- }
-
- _argWords = 0;
-
- // Step One:
- //
- // Determine where the words are in the command string, copying the
- // string to _cmd and storing words in _arg.
- //
- bool isFirst = true;
- char *cmdI = cmd;
- char *cmdO = _cmd;
-
- while (*cmdI) {
-
- // If we are at a non-space character, we are in a word. If
- // this is the first character in the word, save the word in
- // the args list.
- //
- // Otherwise we are at a space and thus not in a word. Make
- // all spaces be string terminators, and declare that we are
- // at the start of a word.
- //
- if ((*cmdI != ' ') && (*cmdI != '\t')) {
- *cmdO = *cmdI;
-
- if (isFirst) {
- _arg[_argWords++] = cmdO;
- isFirst = false;
- }
- } else {
- *cmdO = 0;
- isFirst = true;
- }
-
- cmdI++;
- cmdO++;
- }
-
- // Finish off the list by terminating the last arg, and
- // terminating the list of args.
- //
- *cmdO = 0;
- _arg[_argWords] = 0L;
- };
-
-
- u32bit numWords(void) { return(_argWords); };
- char *getWord(u32bit i) { return(_arg[i]); };
- char *operator[](u32bit i) { return(_arg[i]); };
-private:
- u32bit _argWords;
- u32bit _maxWords;
- char **_arg;
- u32bit _maxChars;
- char *_cmd;
-};
-
-
-
-
-typedef u64bit intervalNumber;
-
-struct _intervalPair {
- intervalNumber lo;
- intervalNumber hi;
-};
-
-struct _intervalDepth {
- intervalNumber lo;
- intervalNumber hi;
- u32bit de;
-};
-
-
-class intervalList {
-public:
- intervalList();
- ~intervalList();
-
- intervalList &operator=(intervalList &src);
-
- // Clear a list
- void clear(void) {
- _isSorted = true;
- _isMerged = true;
- _listLen = 0;
- }
-
- // Insert a new interval into the list
- void add(intervalNumber position, intervalNumber length);
-
- // Sort the set of intervals by the lo value
- void sort(void);
-
- // Merge overlapping or adjacent intervals together.
- void merge(void);
-
- void invert(intervalNumber lo, intervalNumber hi);
-
- // Returns the number of intervals
- u32bit numberOfIntervals(void) {
- return(_listLen);
- };
-
- // Returns the sum of the length of all intervals
- intervalNumber sumOfLengths(void) {
- intervalNumber len = 0;
- u32bit i = numberOfIntervals();
-
- if (i > 0)
- while (i--)
- len += _list[i].hi - _list[i].lo;
-
- return(len);
- };
-
- // Populates an array with the intervals that are within the
- // supplied interval. Return
- //
- u32bit overlapping(intervalNumber lo,
- intervalNumber hi,
- u32bit *&intervals,
- u32bit &intervalsLen,
- u32bit &intervalsMax);
-
- // Populates this intervalList with the intersection of A and B.
- // This intervalList is not cleared prior to adding new intervals.
- //
- // Both A and B call merge().
- //
- void intersect(intervalList &A,
- intervalList &B);
-
- // Populates this intervalList with regions in A that are completely
- // contained in a region in B.
- //
- // Both A and B call merge().
- //
- void contained(intervalList &A,
- intervalList &B);
-
-
- intervalNumber lo(u32bit i) { return(_list[i].lo); };
- intervalNumber hi(u32bit i) { return(_list[i].hi); };
-
-private:
- bool _isSorted;
- bool _isMerged;
- u32bit _listLen;
- u32bit _listMax;
- _intervalPair *_list;
-};
-
-
-
-// Takes as input an intervalList, computes the number of intervals
-// covering every position in there, stores this as a new set of
-// intervals, annotated with the depth.
-//
-// This is a static object, initialized once by the intervalList.
-//
-class intervalDepth {
-public:
- intervalDepth(intervalList &IL);
- ~intervalDepth();
-
- // Returns the number of intervals
- u32bit numberOfIntervals(void) {
- return(_listLen);
- };
-
- intervalNumber lo(u32bit i) { return(_list[i].lo); };
- intervalNumber hi(u32bit i) { return(_list[i].hi); };
- u32bit de(u32bit i) { return(_list[i].de); };
-
-private:
- u32bit _listLen;
- u32bit _listMax;
- _intervalDepth *_list;
-};
-
-
-intervalList::intervalList() {
- _isSorted = true;
- _isMerged = true;
- _listLen = 0;
- _listMax = 16;
- _list = new _intervalPair [_listMax];
-}
-
-
-intervalList::~intervalList() {
- delete [] _list;
-}
-
-
-intervalList &
-intervalList::operator=(intervalList &src) {
- _isSorted = src._isSorted;
- _isMerged = src._isMerged;
- _listLen = src._listLen;
-
- if (_listMax < src._listMax) {
- delete [] _list;
- _listMax = src._listMax;
- _list = new _intervalPair [_listMax];
- }
-
- memcpy(_list, src._list, _listLen * sizeof(_intervalPair));
-
- return(*this);
-}
-
-
-void
-intervalList::add(intervalNumber position, intervalNumber length) {
-
- if (_listLen >= _listMax) {
- _listMax *= 2;
- _intervalPair *l = new _intervalPair [_listMax];
- memcpy(l, _list, sizeof(_intervalPair) * _listLen);
- delete [] _list;
- _list = l;
- }
-
- _list[_listLen].lo = position;
- _list[_listLen].hi = position + length;
-
-#if 0
- // Aborted attempt to add a data field here. Got stuck
- // deciding how to handle merges lightweight
-
- _list[_listLen].data = 0L;
-
- if (data != ~u64bitZERO) {
- _list[_listLen].dataLen = 1;
- _list[_listLen].dataMax = 4;
- _list[_listLen].data = new u64bit [_list[_listLen].dataMax];
- _list[_listLen].data[0] = data;
- }
-#endif
-
- if ((_listLen > 0) &&
- (_list[_listLen-1].lo > _list[_listLen].lo)) {
- _isSorted = false;
- _isMerged = false;
- }
-
- _listLen++;
-}
-
-
-static
-int
-intervalList_sort_helper(const void *a, const void *b) {
- _intervalPair *A = (_intervalPair *)a;
- _intervalPair *B = (_intervalPair *)b;
-
- if (A->lo < B->lo) return(-1);
- if (A->lo > B->lo) return(1);
- if (A->hi < B->hi) return(-1);
- if (A->hi > B->hi) return(1);
- return(0);
-}
-
-
-void
-intervalList::sort(void) {
-
- if (_isSorted)
- return;
-
- if (_listLen > 1)
- qsort(_list, _listLen, sizeof(_intervalPair), intervalList_sort_helper);
-
- _isSorted = true;
-}
-
-
-void
-intervalList::merge(void) {
- u32bit thisInterval = 0;
- u32bit nextInterval = 1;
-
- if (_listLen < 2)
- return;
-
- sort();
-
- while (nextInterval < _listLen) {
-
- if ((_list[thisInterval].lo == 0) &&
- (_list[thisInterval].hi == 0)) {
-
- // Our interval is empty. Copy in the interval we are
- // examining and move to the next.
-
- // XXX This is probably useless, thisInterval should always be
- // valid.
-
- _list[thisInterval].lo = _list[nextInterval].lo;
- _list[thisInterval].hi = _list[nextInterval].hi;
-
- _list[nextInterval].lo = 0;
- _list[nextInterval].hi = 0;
- nextInterval++;
- } else {
-
- // This interval is valid. See if it overlaps with the next
- // interval.
-
- if (_list[thisInterval].hi >= _list[nextInterval].lo) {
-
- // Got an intersection.
-
- // Merge nextInterval into thisInterval -- the hi range
- // is extended if the nextInterval range is larger.
- //
- if (_list[thisInterval].hi < _list[nextInterval].hi)
- _list[thisInterval].hi = _list[nextInterval].hi;
-
- // Clear the just merged nextInterval and move to the next one.
- //
- _list[nextInterval].lo = 0;
- _list[nextInterval].hi = 0;
- nextInterval++;
- } else {
-
- // No intersection. Move along. Nothing to see here.
-
- // If there is a gap between the target and the examine (we
- // must have merged sometime in the past), copy examine to
- // the next target.
-
- thisInterval++;
-
- if (thisInterval != nextInterval) {
- _list[thisInterval].lo = _list[nextInterval].lo;
- _list[thisInterval].hi = _list[nextInterval].hi;
- }
-
- nextInterval++;
- }
- }
- }
-
- if (thisInterval+1 < _listLen)
- _listLen = thisInterval + 1;
-
- _isMerged = true;
-}
-
-
-void
-intervalList::invert(intervalNumber lo, intervalNumber hi) {
-
- if (!_isSorted || !_isMerged) {
- fprintf(stderr, "intervalList::invert()-- ERROR! List is not sorted or not merged!\n");
- exit(1);
- }
-
- // Create a new list to store the inversion
- //
- u32bit invLen = 0;
- u32bit invMax = _listLen + 2;
- _intervalPair *inv = new _intervalPair [invMax];
-
- // Add the first
- //
- if (lo < _list[0].lo) {
- inv[invLen].lo = lo;
- inv[invLen].hi = _list[0].lo;
- invLen++;
- }
-
- // Add the pieces
- for (u32bit i=1; i<_listLen; i++) {
- if (_list[i-1].hi < _list[i].lo) {
- inv[invLen].lo = _list[i-1].hi;
- inv[invLen].hi = _list[i].lo;
- invLen++;
- }
- }
-
- // Add the last
- if (_list[_listLen-1].hi < hi) {
- inv[invLen].lo = _list[_listLen-1].hi;
- inv[invLen].hi = hi;
- invLen++;
- }
-
- // Nuke the old list, swap in the new one
- delete [] _list;
-
- _list = inv;
- _listLen = invLen;
- _listMax = invMax;
-}
-
-
-
-u32bit
-intervalList::overlapping(intervalNumber rangelo,
- intervalNumber rangehi,
- u32bit *&intervals,
- u32bit &intervalsLen,
- u32bit &intervalsMax) {
-
-
- // XXX: Naive implementation that is easy to verify (and that works
- // on an unsorted list).
-
- if (intervals == 0L) {
- intervalsMax = 256;
- intervals = new u32bit [intervalsMax];
- }
-
- intervalsLen = 0;
-
- for (u32bit i=0; i<_listLen; i++) {
- if ((rangelo <= _list[i].hi) &&
- (rangehi >= _list[i].lo)) {
- if (intervalsLen >= intervalsMax) {
- intervalsMax *= 2;
- u32bit *X = new u32bit [intervalsMax];
- memcpy(X, intervals, sizeof(u32bit) * intervalsLen);
- delete [] intervals;
- intervals = X;
- }
-
- intervals[intervalsLen++] = i;
- }
- }
-
- return(intervalsLen);
-}
-
-
-
-void
-intervalList::intersect(intervalList &A,
- intervalList &B) {
- A.merge();
- B.merge();
-
- u32bit ai = 0;
- u32bit bi = 0;
-
- while ((ai < A.numberOfIntervals()) &&
- (bi < B.numberOfIntervals())) {
- u32bit al = A.lo(ai);
- u32bit ah = A.hi(ai);
- u32bit bl = B.lo(bi);
- u32bit bh = B.hi(bi);
- u32bit nl = 0;
- u32bit nh = 0;
-
- // If they intersect, make a new region
- //
- if ((al <= bl) && (bl < ah)) {
- nl = bl;
- nh = (ah < bh) ? ah : bh;
- }
-
- if ((bl <= al) && (al < bh)) {
- nl = al;
- nh = (ah < bh) ? ah : bh;
- }
-
- if (nl < nh)
- add(nl, nh - nl);
-
- // Advance the list with the earlier region.
- //
- if (ah < bh) {
- // A ends before B
- ai++;
- } else if (ah > bh) {
- // B ends before A
- bi++;
- } else {
- // Exactly the same ending!
- ai++;
- bi++;
- }
- }
-}
-
-void
-intervalList::contained(intervalList &A,
- intervalList &B) {
- A.merge();
- B.merge();
-
- u32bit ai = 0;
- u32bit bi = 0;
-
- while ((ai < A.numberOfIntervals()) &&
- (bi < B.numberOfIntervals())) {
- u32bit al = A.lo(ai);
- u32bit ah = A.hi(ai);
- u32bit bl = B.lo(bi);
- u32bit bh = B.hi(bi);
-
- // If A is contained in B, make a new region.
- //
- if ((bl <= al) && (ah <= bh))
- add(bl, bh - bl);
-
-#if 0
- if ((al <= bl) && (bh <= ah))
- add(al, ah - al);
-#endif
-
- // Advance the list with the earlier region.
- //
- if (ah < bh) {
- // A ends before B
- ai++;
- } else if (ah > bh) {
- // B ends before A
- bi++;
- } else {
- // Exactly the same ending!
- ai++;
- bi++;
- }
- }
-}
-
-
-
-
-
-
-static
-int
-intervalDepth_sort_helper(const void *a, const void *b) {
- _intervalDepth *A = (_intervalDepth *)a;
- _intervalDepth *B = (_intervalDepth *)b;
-
- if (A->lo < B->lo) return(-1);
- if (A->lo > B->lo) return(1);
- return(0);
-}
-
-
-intervalDepth::intervalDepth(intervalList &IL) {
-
- u32bit idlen = IL.numberOfIntervals() * 2;
- _intervalDepth *id = new _intervalDepth [idlen];
-
- for (u32bit i=0; i<IL.numberOfIntervals(); i++) {
- id[2*i ].lo = IL.lo(i);
- id[2*i ].hi = 0;
- id[2*i ].de = 1;
- id[2*i+1].lo = IL.hi(i);
- id[2*i+1].hi = 0;
- id[2*i+1].de = 0;
- }
-
- qsort(id, idlen, sizeof(_intervalDepth), intervalDepth_sort_helper);
-
- // Scan the list, counting how many times we change depth.
- //
- _listMax = 1;
- for (u32bit i=1; i<idlen; i++) {
- if (id[i-1].lo != id[i].lo)
- _listMax++;
- }
-
- // Allocate the real depth of coverage intervals
- //
- _listLen = 0;
- _list = new _intervalDepth [_listMax];
-
- // Build new intervals
- //
- // Initialize the first interval
- //
- _list[_listLen].lo = id[0].lo;
- _list[_listLen].hi = id[0].lo;
- _list[_listLen].de = 1;
-
- for (u32bit i=1; i<idlen; i++) {
-
- if (_list[_listLen].de == 0) {
- // Update the start position if the current interval is at zero
- // depth.
- //
- _list[_listLen].lo = id[i].lo;
- } else {
-
- // If we are at a position different from the start, we need to
- // close out the current interval and make a new one.
- //
- if (id[i-1].lo != id[i].lo) {
- _list[_listLen].hi = id[i].lo;
-
- _listLen++;
-
- _list[_listLen].lo = id[i].lo;
- _list[_listLen].hi = id[i].lo;
- _list[_listLen].de = _list[_listLen-1].de;
- }
- }
-
- // Finally, update the depth of the current interval
- //
- if (id[i].de)
- _list[_listLen].de++;
- else
- _list[_listLen].de--;
- }
-
- // Toss out the last one if it's zero length -- I think it's always
- // zero length, just can convince myself.
- //
- if (_list[_listLen].lo == _list[_listLen].hi)
- _listLen--;
-
- delete [] id;
-}
-
-intervalDepth::~intervalDepth() {
- delete [] _list;
-}
-
-//
-// End code from Bri's libutil/util++.H
-//
-
-
-
-void
-readDepth(char *depthname, map<u64bit,intervalDepth*> &lowCoverage) {
- char line[1024] = {0};
- map<u64bit,intervalList*> ILs;
-
- fprintf(stderr, "Reading depth from '%s'\n", depthname);
-
- errno = 0;
- FILE *F = fopen(depthname, "r");
- if (errno)
- fprintf(stderr, "failed to open '%s': %s\n", depthname, strerror(errno)), exit(1);
-
- u32bit i=0;
-
- fgets(line, 1024, F);
- while (!feof(F)) {
- splitToWords W(line);
-
- u64bit uid = strtou64bit(W[1], 0L);
- u32bit beg = strtou32bit(W[2], 0L);
- u32bit end = strtou32bit(W[3], 0L);
-
- if (beg > end)
- fprintf(stderr, "ERROR: l="u32bitFMT" h="u32bitFMT"\n", beg, end);
-
- if (ILs[uid] == 0L)
- ILs[uid] = new intervalList();
- ILs[uid]->add(beg, end - beg);
-
- i++;
-
- fgets(line, 1024, F);
- }
-
- fclose(F);
- fprintf(stderr, " "u32bitFMT" lines.\n", i);
-
- map<u64bit,intervalList*>::iterator it = ILs.begin();
- map<u64bit,intervalList*>::iterator ed = ILs.end();
-
- while (it != ed) {
- lowCoverage[it->first] = new intervalDepth(*it->second);
- delete it->second;
- it->second = 0L;
- it++;
- }
-}
-
-
-void
-readVariation(char *depthname, map<u64bit,intervalList*> &variation) {
- char line[1024 * 1024] = {0};
-
- fprintf(stderr, "Reading variation from '%s'\n", depthname);
-
- errno = 0;
- FILE *F = fopen(depthname, "r");
- if (errno)
- fprintf(stderr, "failed to open '%s': %s\n", depthname, strerror(errno)), exit(1);
-
- u32bit i=0;
-
- fgets(line, 1024 * 1024, F);
- while (!feof(F)) {
- splitToWords W(line);
-
- u64bit uid = strtou64bit(W[1], 0L);
- u32bit beg = strtou32bit(W[2], 0L);
- u32bit end = strtou32bit(W[3], 0L);
-
- if (variation[uid] == 0L)
- variation[uid] = new intervalList();
- variation[uid]->add(beg, end - beg);
-
- i++;
-
- fgets(line, 1024 * 1024, F);
- }
-
- fclose(F);
- fprintf(stderr, " "u32bitFMT" lines.\n", i);
-}
-
-
-void
-readBadMers(char *depthname, map<u64bit,intervalList*> &badMers) {
- char line[1024] = {0};
-
- fprintf(stderr, "Reading badMers from '%s'\n", depthname);
-
- errno = 0;
- FILE *F = fopen(depthname, "r");
- if (errno)
- fprintf(stderr, "failed to open '%s': %s\n", depthname, strerror(errno)), exit(1);
-
- u32bit i=0;
-
- fgets(line, 1024, F);
- while (!feof(F)) {
- splitToWords W(line);
-
- // Change every non-digit to a space in the first word.
- for (u32bit z=strlen(W[0])-1; z--; )
- if (!isdigit(W[0][z]))
- W[0][z] = ' ';
-
- u64bit uid = strtou64bit(W[0], 0L);
- u32bit beg = strtou32bit(W[3], 0L);
- u32bit end = strtou32bit(W[4], 0L);
-
- if (badMers[uid] == 0L)
- badMers[uid] = new intervalList();
- badMers[uid]->add(beg, end - beg);
-
- i++;
-
- fgets(line, 1024, F);
- }
-
- fclose(F);
- fprintf(stderr, " "u32bitFMT" lines.\n", i);
-}
-
-
-
-int
-main(int argc, char **argv) {
- map<u64bit,intervalList*> badMers;
- map<u64bit,intervalList*> variation;
- map<u64bit,intervalDepth*> lowCoverage;
-
- bool showDepthIntersect = false;
- bool showVariantIntersect = false;
- bool showVarDepthIntersect = false;
-
- int arg=1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-D") == 0) {
-
- } else if (strcmp(argv[arg], "-pd") == 0) {
- showDepthIntersect = true;
- } else if (strcmp(argv[arg], "-pv") == 0) {
- showVariantIntersect = true;
- } else if (strcmp(argv[arg], "-pvd") == 0) {
- showVarDepthIntersect = true;
- } else {
- fprintf(stderr, "usage: %s [-D debugfile] [-pd] [-pv] [-pvd]\n", argv[0]);
- fprintf(stderr, " -pd print bad mers regions isect depth\n");
- fprintf(stderr, " -pv print bad mers regions isect variants\n");
- fprintf(stderr, " -pvd print bad mers regions isect both variants and depth\n");
- exit(1);
- }
- arg++;
- }
-
-#if 1
- // HuRef6, in the assembly directory.
- //
- readDepth ("/project/huref6/assembly/h6/9-terminator/h6.posmap.frgctg", lowCoverage);
- readVariation("/project/huref6/assembly/h6/9-terminator/h6.posmap.varctg", variation);
- readBadMers ("/project/huref6/assembly/h6-mer-validation/h6-ms22-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers", badMers);
-#endif
-
-#if 0
- // HuRef6, ws=25, in the assembly directory.
- //
- readDepth ("/project/huref6/assembly/h6/9-terminator-ws25/h6.posmap.frgctg", lowCoverage);
- readVariation("/project/huref6/assembly/h6/9-terminator-ws25/h6.posmap.varctg", variation);
- readBadMers ("/project/huref6/assembly/h6-mer-validation/h6-version4-ws25/h6-ms22-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers", badMers);
-#endif
-
-#if 0
- // Our scratch huref
- //
- readDepth ("/project/huref6/redo_consensus-gennady/mer-validation/h6tmp.posmap.frgctg", lowCoverage);
- readVariation("/project/huref6/redo_consensus-gennady/mer-validation/h6tmp.posmap.varctg", variation);
- readBadMers ("/project/huref6/redo_consensus-gennady/mer-validation/h6tmp-ms22-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers", badMers);
-#endif
-
- u32bit badBegDepth[1024] = {0};
- u32bit badEndDepth[1024] = {0};
-
- u32bit badDepth[32][32];
- for (u32bit i=0; i<32; i++)
- for (u32bit j=0; j<32; j++)
- badDepth[i][j] = 0;
-
- map<u64bit,intervalList*>::iterator it = badMers.begin();
- map<u64bit,intervalList*>::iterator ed = badMers.end();
- while (it != ed) {
- u64bit uid = it->first;
-
- intervalList *Iv = variation[uid];
- intervalList *Ib = badMers[uid];
- intervalList *Ii = 0L;
- intervalDepth *Id = lowCoverage[uid];
-
- if (Iv)
- Iv->merge();
- if (Ib)
- Ib->merge();
-
- if (Iv && Ib) {
- Ii = new intervalList();
- Ii->intersect(*Iv, *Ib);
- }
-
-
- if (Ii) {
- u32bit ii = 0;
- u32bit id = 0;
-
- while ((ii < Ii->numberOfIntervals()) &&
- (id < Id->numberOfIntervals())) {
-
- // We want to count the number of times a badmer region
- // begins/ends in some depth.
-
- //fprintf(stderr, "testing beg "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT"\n",
- // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id));
-
- u32bit beg = 0;
- u32bit end = 0;
-
- // Low points are not allowed to be equal to high points, skip to the next
- while ((id < Id->numberOfIntervals()) &&
- (Id->hi(id) <= Ii->lo(ii))) {
- id++;
- //fprintf(stderr, "testing beg (m) "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT"\n",
- // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id));
- }
- if (id < Id->numberOfIntervals()) {
- u32bit lo = Id->lo(id);
- u32bit hi = Id->hi(id);
-
- // Low points are not allowed to be equal to high points.
- if ((lo <= Ii->lo(ii)) && (Ii->lo(ii) < hi)) {
- beg = Id->de(id);
- } else {
- fprintf(stderr, "failed to find begin "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT" "u32bitFMT"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->de(id));
- if (id > 0)
- fprintf(stderr, " "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT" "u32bitFMT"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->de(id-1));
- //exit(1);
- }
- }
-
- //fprintf(stderr, "testing end "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT"\n",
- // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id));
-
- // High points can be equal.
- while ((id < Id->numberOfIntervals()) &&
- (Id->hi(id) < Ii->hi(ii))) {
- id++;
- //fprintf(stderr, "testing end (m) "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT"\n",
- // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id));
- }
- if (id < Id->numberOfIntervals()) {
- u32bit lo = Id->lo(id);
- u32bit hi = Id->hi(id);
-
- // High points aren't allowed to be equal to lo, but can be equal to hi.
- if ((lo < Ii->hi(ii)) && (Ii->hi(ii) <= hi)) {
- end = Id->de(id);
- } else {
- fprintf(stderr, "failed to find end "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT" "u32bitFMT"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->de(id));
- if (id > 0)
- fprintf(stderr, " "u32bitFMT" "u32bitFMT" -- "u32bitFMT" "u32bitFMT" "u32bitFMT"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->de(id-1));
- //exit(1);
- }
- }
-
- badBegDepth[beg]++;
- badEndDepth[end]++;
-
- fprintf(stdout, u64bitFMT"\t"u32bitFMT"\t"u32bitFMT"\tdepth="u32bitFMT","u32bitFMT"\n",
- uid, Ii->lo(ii), Ii->hi(ii), beg, end);
-
- if ((beg < 32) && (end < 32))
- badDepth[beg][end]++;
-
- ii++;
- }
- }
-
- it++;
- }
-
- u32bit bb = 0;
- u32bit be = 0;
- for (u32bit x=0; x<32; x++) {
- fprintf(stdout, u32bitFMT"\t"u32bitFMT"\t"u32bitFMT"\n", x, badBegDepth[x], badEndDepth[x]);
- bb += badBegDepth[x];
- be += badEndDepth[x];
- }
- fprintf(stdout, "total\t"u32bitFMT"\t"u32bitFMT"\n", bb, be);
-
- for (u32bit i=0; i<30; i++) {
- for (u32bit j=0; j<30; j++)
- fprintf(stdout, u32bitFMTW(5), badDepth[i][j]);
- fprintf(stdout, "\n");
- }
-
- return(0);
-}
diff --git a/kmer/meryl/asmMerQC.C b/kmer/meryl/asmMerQC.C
deleted file mode 100644
index 9e8e903..0000000
--- a/kmer/meryl/asmMerQC.C
+++ /dev/null
@@ -1,396 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "libmeryl.H"
-
-// The categories depend on the type of input (fragments or contigs):
-//
-// 0 -- no count, mer not present
-// 1 -- single copy
-// 2 -- 2 -> 10 copies (contigs) -- 2 -> 2mode copies (frags)
-// 3 -- 11 -> 100 copies (contigs) -- -> 10mode copies (frags)
-// 4 -- 101+ copies (contigs) -- -> 100mode copies (frags)
-// 5 -- -> infinity copies (frags)
-
-// You'll also need to modify compare() and output() if you change this.
-#define NUMCATEGORIES 6
-
-// The output files are global for convenience. Otherwise, we'd be passing
-// them to compare() for every single mer.
-//
-bool dumpFlag = false;
-FILE *dumpSCZF = 0L;
-FILE *dumpMCZF = 0L;
-FILE *dumpMCSF = 0L;
-FILE *dumpMCMF = 0L;
-char merstring[1024];
-
-u32bit
-findMode(char *name) {
- merylStreamReader *M = new merylStreamReader(name);
- u32bit *H = new u32bit [16384];
-
- fprintf(stderr, "Finding mode of '%s'\n", name);
-
- for (u32bit i=0; i<16384; i++)
- H[i] = 0;
-
- while (M->validMer()) {
- if (M->theCount() < 16384)
- H[M->theCount()]++;
- M->nextMer();
- }
-
- u32bit mi = 2;
- for (u32bit i=2; i<16384; i++)
- if (H[i] > H[mi])
- mi = i;
-
- fprintf(stderr, "Mode of '%s' is "u32bitFMT"\n", name, mi);
-
- return(mi);
-}
-
-
-void
-compare(merylStreamReader *F,
- merylStreamReader *C,
- kMer &minmer,
- u32bit mode,
- u32bit R[NUMCATEGORIES][NUMCATEGORIES]) {
- u32bit Ftype = 0;
- u32bit Ctype = 0;
- kMer Fmer = F->theFMer();
- kMer Cmer = C->theFMer();
- u32bit Fcnt = F->theCount();
- u32bit Ccnt = C->theCount();
-
- if (Fcnt == 0)
- Ftype = 0;
- else if (Fcnt == 1)
- Ftype = 1;
- else if (Fcnt <= 2*mode)
- Ftype = 2;
- else if (Fcnt <= 10*mode)
- Ftype = 3;
- else if (Fcnt <= 100*mode)
- Ftype = 4;
- else
- Ftype = 5;
-
- if (Ccnt == 0)
- Ctype = 0;
- else if (Ccnt == 1)
- Ctype = 1;
- else if (Ccnt <= 10)
- Ctype = 2;
- else if (Ccnt <= 100)
- Ctype = 3;
- else
- Ctype = 4;
-
- // If the mer isn't valid, we hit the end of the file, and the mer
- // thus (obviously) isn't in the file.
- //
- if (F->validMer() == false)
- Ftype = 0;
- if (C->validMer() == false)
- Ctype = 0;
-
- // If either type is 0, we're done, but only increment the count if
- // this mer is the minmer.
- //
- if ((Ftype == 0) || (Ctype == 0)) {
- if (((Ftype == 0) && (Cmer == minmer)) ||
- ((Ctype == 0) && (Fmer == minmer))) {
- R[Ftype][Ctype]++;
-
- // Save the mer if it's in contigs, but not fragments.
- if (dumpFlag)
- if (Ftype == 0)
- if (Ctype == 1)
- fprintf(dumpSCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
- else
- fprintf(dumpMCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
- }
- return;
- }
-
- // If the mers don't agree, we're also done. If either is the
- // minmer, note that we saw it.
- //
- if (Fmer != Cmer) {
- if (Fmer == minmer)
- R[Ftype][0]++;
- if (Cmer == minmer) {
- R[0][Ctype]++;
-
- // Again, save the mer since it's in contigs, but not fragments.
- if (dumpFlag)
- if (Ctype == 1)
- fprintf(dumpSCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
- else
- fprintf(dumpMCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
- }
-
- return;
- }
-
- // If we're not the minmer, we're done.
- if (Fmer != minmer)
- return;
-
- // Otherwise, the mers are in both inputs
- R[Ftype][Ctype]++;
-
- // Save the mer if it's in contigs "more" than if in fragments.
- if (dumpFlag) {
- if (Ftype < Ctype)
- if (Ctype == 2)
- fprintf(dumpMCSF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
- else
- fprintf(dumpMCMF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
-
- if ((Ftype == 0) && (Ctype == 1))
- fprintf(dumpSCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
- }
-}
-
-
-void
-output(char *title,
- u32bit mode,
- u32bit R[NUMCATEGORIES][NUMCATEGORIES]) {
-
- fprintf(stdout, "\n\n%s\n", title);
- fprintf(stdout, "(frags) | zero | one | <= 10 | <= 100 | <= inf | (contigs)\n");
-
- for (u32bit i=0; i<6; i++) {
- switch (i) {
- case 0: fprintf(stdout, "zero "); break;
- case 1: fprintf(stdout, "one "); break;
- case 2: fprintf(stdout, "<= 2mode "); break;
- case 3: fprintf(stdout, "<= 10mode "); break;
- case 4: fprintf(stdout, "<= 100mode "); break;
- case 5: fprintf(stdout, "<= inf "); break;
- default: fprintf(stdout, "????????? "); break;
- }
- for (u32bit j=0; j<5; j++)
- fprintf(stdout, u32bitFMTW(12), R[i][j]);
- fprintf(stdout, "\n");
- }
-}
-
-
-
-
-int
-main(int argc, char **argv) {
- merylStreamReader *AF = 0L;
- merylStreamReader *TF = 0L;
- merylStreamReader *AC = 0L;
- merylStreamReader *DC = 0L;
- merylStreamReader *CO = 0L;
-
- u32bit AFmode = 0;
- u32bit TFmode = 0;
-
- char dumpSCZFname[1024] = {0}; // single contig, zero frags
- char dumpMCZFname[1024] = {0}; // low contig, zero frags
- char dumpMCSFname[1024] = {0}; // medium contig, low frags
- char dumpMCMFname[1024] = {0}; // everything else, contig > frags
-
- bool beVerbose = false;
-
- //fprintf(stderr, "using cached modes for testing!\n");
-
- int arg=1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-af") == 0) { // All frags
- ++arg;
- AFmode = findMode(argv[arg]);
- //AFmode = 8;
- AF = new merylStreamReader(argv[arg]);
- AF->nextMer();
- } else if (strcmp(argv[arg], "-tf") == 0) { // Trimmed frags
- ++arg;
- TFmode = findMode(argv[arg]);
- //TFmode = 8;
- TF = new merylStreamReader(argv[arg]);
- TF->nextMer();
- } else if (strcmp(argv[arg], "-ac") == 0) { // All contigs
- AC = new merylStreamReader(argv[++arg]);
- AC->nextMer();
- } else if (strcmp(argv[arg], "-dc") == 0) { // Degenerate contigs
- DC = new merylStreamReader(argv[++arg]);
- DC->nextMer();
- } else if (strcmp(argv[arg], "-co") == 0) { // Contigs
- CO = new merylStreamReader(argv[++arg]);
- CO->nextMer();
- } else if (strcmp(argv[arg], "-dump") == 0) {
- arg++;
- dumpFlag = true;
- sprintf(dumpSCZFname, "%s.0.singlecontig.zerofrag.fasta", argv[arg]);
- sprintf(dumpMCZFname, "%s.1.multiplecontig.zerofrag.fasta", argv[arg]);
- sprintf(dumpMCSFname, "%s.2.multiplecontig.lowfrag.fasta", argv[arg]);
- sprintf(dumpMCMFname, "%s.3.multiplecontig.multiplefrag.fasta", argv[arg]);
- } else if (strcmp(argv[arg], "-v") == 0) {
- beVerbose = true;
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- }
- arg++;
- }
-
- if ((AF == 0L) && (TF == 0L) && (AC == 0L) && (DC == 0L) && (CO == 0L)) {
- fprintf(stderr, "usage: %s [opts] [-v] [-dump prefix]\n", argv[0]);
- fprintf(stderr, "At least one fragcounts and one contigcounts are needed.\n");
- fprintf(stderr, " -af | -tf fragcounts\n");
- fprintf(stderr, " -ac | -dc | -co contigcounts \n");
- fprintf(stderr, "Dumping is probably only useful with exactly one frag and\n");
- fprintf(stderr, "one contig, but I'll let you do it with any number.\n");
- exit(1);
- }
- if ((AF == 0L) && (TF == 0L)) {
- fprintf(stderr, "ERROR - need at least one of -af, -tf\n");
- exit(1);
- }
- if ((AC == 0L) && (DC == 0L) && (CO == 0L)) {
- fprintf(stderr, "ERROR - need at least one of -ac, -dc, -co\n");
- exit(1);
- }
-
- // Check mersizes.
- //
- u32bit merSize = 0;
- u32bit ms[5] = { 0 };
-
- if (AF) merSize = ms[0] = AF->merSize();
- if (TF) merSize = ms[1] = TF->merSize();
- if (AC) merSize = ms[2] = AC->merSize();
- if (DC) merSize = ms[3] = DC->merSize();
- if (CO) merSize = ms[4] = CO->merSize();
-
- bool differ = false;
-
- if ((ms[0] > 0) && (ms[0] != merSize)) differ = true;
- if ((ms[1] > 0) && (ms[1] != merSize)) differ = true;
- if ((ms[2] > 0) && (ms[2] != merSize)) differ = true;
- if ((ms[3] > 0) && (ms[3] != merSize)) differ = true;
- if ((ms[4] > 0) && (ms[4] != merSize)) differ = true;
-
- if (differ) {
- fprintf(stderr, "error: mer size differ.\n");
- fprintf(stderr, " AF - "u32bitFMT"\n", ms[0]);
- fprintf(stderr, " TF - "u32bitFMT"\n", ms[1]);
- fprintf(stderr, " AC - "u32bitFMT"\n", ms[2]);
- fprintf(stderr, " DC - "u32bitFMT"\n", ms[3]);
- fprintf(stderr, " CO - "u32bitFMT"\n", ms[4]);
- exit(1);
- }
-
- if (dumpFlag) {
- errno = 0;
- dumpSCZF = fopen(dumpSCZFname, "w");
- dumpMCZF = fopen(dumpMCZFname, "w");
- dumpMCSF = fopen(dumpMCSFname, "w");
- dumpMCMF = fopen(dumpMCMFname, "w");
- if (errno)
- fprintf(stderr, "Failed to open the dump files: %s\n", strerror(errno)), exit(1);
- }
-
- u32bit AFvsAC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit AFvsDC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit AFvsCO[NUMCATEGORIES][NUMCATEGORIES];
- u32bit TFvsAC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit TFvsDC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit TFvsCO[NUMCATEGORIES][NUMCATEGORIES];
- for (u32bit i=0; i<NUMCATEGORIES; i++)
- for (u32bit j=0; j<NUMCATEGORIES; j++) {
- AFvsAC[i][j] = 0;
- AFvsDC[i][j] = 0;
- AFvsCO[i][j] = 0;
- TFvsAC[i][j] = 0;
- TFvsDC[i][j] = 0;
- TFvsCO[i][j] = 0;
- }
-
- // The default constructor for kMer sets the mer to size 0, all A.
- // We need it to be the proper size, and all T.
- kMer minmer(merSize);
-
- // Don't care what we pick, as long as it's a mer in the set.
- //
- if (AF && AF->validMer()) minmer = AF->theFMer();
- if (TF && TF->validMer()) minmer = TF->theFMer();
- if (AC && AC->validMer()) minmer = AC->theFMer();
- if (DC && DC->validMer()) minmer = DC->theFMer();
- if (CO && CO->validMer()) minmer = CO->theFMer();
-
- speedCounter *C = new speedCounter(" Examining: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
-
- bool morestuff = true;
- while (morestuff) {
-
- // Find any mer in our set
- if (AF && AF->validMer()) minmer = AF->theFMer();
- if (TF && TF->validMer()) minmer = TF->theFMer();
- if (AC && AC->validMer()) minmer = AC->theFMer();
- if (DC && DC->validMer()) minmer = DC->theFMer();
- if (CO && CO->validMer()) minmer = CO->theFMer();
-
- // Find the smallest mer in our set
- if (AF && AF->validMer() && (AF->theFMer() < minmer)) minmer = AF->theFMer();
- if (TF && TF->validMer() && (TF->theFMer() < minmer)) minmer = TF->theFMer();
- if (AC && AC->validMer() && (AC->theFMer() < minmer)) minmer = AC->theFMer();
- if (DC && DC->validMer() && (DC->theFMer() < minmer)) minmer = DC->theFMer();
- if (CO && CO->validMer() && (CO->theFMer() < minmer)) minmer = CO->theFMer();
-
- // We need to do up to six comparisons here.
- if (AF && AC) compare(AF, AC, minmer, AFmode, AFvsAC);
- if (AF && DC) compare(AF, DC, minmer, AFmode, AFvsDC);
- if (AF && CO) compare(AF, CO, minmer, AFmode, AFvsCO);
- if (TF && AC) compare(TF, AC, minmer, TFmode, TFvsAC);
- if (TF && DC) compare(TF, DC, minmer, TFmode, TFvsDC);
- if (TF && CO) compare(TF, CO, minmer, TFmode, TFvsCO);
-
- C->tick();
-#if 0
- if (C->tick()) {
- char stringjunk[256];
- fprintf(stderr, "\nMM %s\n", minmer.merToString(stringjunk));
- if (AF) fprintf(stderr, "AF %s\n", AF->theFMer().merToString(stringjunk));
- if (TF) fprintf(stderr, "TF %s\n", TF->theFMer().merToString(stringjunk));
- if (AC) fprintf(stderr, "AC %s\n", AC->theFMer().merToString(stringjunk));
- if (DC) fprintf(stderr, "DC %s\n", DC->theFMer().merToString(stringjunk));
- if (CO) fprintf(stderr, "CO %s\n", CO->theFMer().merToString(stringjunk));
- }
-#endif
-
- // Advance to the next mer, if we were just used
- morestuff = false;
- if ((AF) && (AF->theFMer() == minmer)) morestuff |= AF->nextMer();
- if ((TF) && (TF->theFMer() == minmer)) morestuff |= TF->nextMer();
- if ((AC) && (AC->theFMer() == minmer)) morestuff |= AC->nextMer();
- if ((DC) && (DC->theFMer() == minmer)) morestuff |= DC->nextMer();
- if ((CO) && (CO->theFMer() == minmer)) morestuff |= CO->nextMer();
- }
-
- delete C;
-
- // output
-
- if ((AF) && (AC)) output("all frags vs all contigs", AFmode, AFvsAC);
- if ((AF) && (DC)) output("all frags vs deg. contigs", AFmode, AFvsDC);
- if ((AF) && (CO)) output("all frags vs non-deg. contigs", AFmode, AFvsCO);
- if ((TF) && (AC)) output("trimmed frags vs all contigs", TFmode, TFvsAC);
- if ((TF) && (DC)) output("trimmed frags vs deg. contigs", TFmode, TFvsDC);
- if ((TF) && (CO)) output("trimmed frags vs non-deg. contigs", TFmode, TFvsCO);
-
- delete AF;
- delete TF;
- delete AC;
- delete DC;
- delete CO;
-}
diff --git a/kmer/meryl/asmMerQC.sh b/kmer/meryl/asmMerQC.sh
deleted file mode 100644
index 584fb39..0000000
--- a/kmer/meryl/asmMerQC.sh
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/bin/sh
-
-# Test if the mers in the consensus sequence are supported by mers in
-# the fragments.
-
-# If we count just the clear, we get a clearer (ha, ha) picture of
-# the assembly quality, while if we count all reads we get a picture
-# of trimming.
-#
-onlyClear=1
-onlyReal=1
-
-mem=8192
-mem=16384
-mem=24576
-
-ms=22
-
-binroot=/bioinfo/assembly/walenz/src/genomics
-asmMerQC=$binroot/meryl/asmMerQC
-mapMers=$binroot/meryl/mapMers
-
-dir=/scratch/drosnightly
-asm=willi
-
-dir=/project/huref6/redo-consensus_gennady
-asm=h6tmp
-
-dir=/project/huref6/assembly/h6
-asm=h6
-
-
-# Count mers in reads
-#
-if [ ! -e $asm-ms$ms-clr-frags.mcidx ] ; then
- bin/dumpFragStoreAsFasta -frg $dir/$asm.frgStore | \
- meryl -B -C -m $ms -s - -o $asm-ms$ms-clr-frags -threads 4 -memory $mem -v
-fi
-
-if [ ! -e $asm-ms$ms-all-frags.mcidx ] ; then
- bin/dumpFragStoreAsFasta -allbases -allfrags -frg $dir/$asm.frgStore | \
- meryl -B -C -m $ms -s - -o $asm-ms$ms-all-frags -threads 4 -memory $mem -v
-fi
-
-echo Finding contigs.
-
-if [ ! -e $asm.normalcontigs.fasta ] ; then
- bin/asmOutputContigsFasta < $dir/9-terminator/$asm.asm > $asm.normalcontigs.fasta &
-fi
-if [ ! -e $asm.degeneratecontigs.fasta ] ; then
- bin/asmOutputContigsFasta -D < $dir/9-terminator/$asm.asm > $asm.degeneratecontigs.fasta &
-fi
-if [ ! -e $asm.allcontigs.fasta ] ; then
- bin/asmOutputContigsFasta -d < $dir/9-terminator/$asm.asm > $asm.allcontigs.fasta &
-fi
-
-# Count mers in contigs
-#
-if [ ! -e $asm-ms$ms-normal-contigs.mcidx ] ; then
- meryl -B -C -m $ms -s $asm.normalcontigs.fasta -o $asm-ms$ms-normal-contigs -threads 4 -segments 4 -v &
-fi
-if [ ! -e $asm-ms$ms-degenerate-contigs.mcidx ] ; then
- meryl -B -C -m $ms -s $asm.degeneratecontigs.fasta -o $asm-ms$ms-degenerate-contigs -threads 4 -segments 4 -v &
-fi
-if [ ! -e $asm-ms$ms-all-contigs.mcidx ] ; then
- meryl -B -C -m $ms -s $asm.allcontigs.fasta -o $asm-ms$ms-all-contigs -threads 4 -segments 4 -v &
-fi
-
-if [ ! -e $asm-ms$ms.asmMerQC ] ; then
- $asmMerQC -af $asm-ms$ms-all-frags \
- -tf $asm-ms$ms-clr-frags \
- -co $asm-ms$ms-normal-contigs \
- -ac $asm-ms$ms-all-contigs \
- -dc $asm-ms$ms-degenerate-contigs \
- > $asm-ms$ms.asmMerQC &
-fi
-
-echo Finding badmers.
-
-if [ ! -e $asm-ms$ms-allfrags-normalcontigs.badmers.asmMerQC ] ; then
- $asmMerQC -af $asm-ms$ms-all-frags \
- -co $asm-ms$ms-normal-contigs \
- -dump $asm-ms$ms-allfrags-normalcontigs.badmers \
- > $asm-ms$ms-allfrags-normalcontigs.badmers.asmMerQC &
-fi
-if [ ! -e $asm-ms$ms-allfrags-allcontigs.badmers.asmMerQC ] ; then
- $asmMerQC -af $asm-ms$ms-all-frags \
- -ac $asm-ms$ms-all-contigs \
- -dump $asm-ms$ms-allfrags-allcontigs.badmers \
- > $asm-ms$ms-allfrags-allcontigs.badmers.asmMerQC &
-fi
-if [ ! -e $asm-ms$ms-allfrags-degeneratecontigs.badmers.asmMerQC ] ; then
- $asmMerQC -af $asm-ms$ms-all-frags \
- -dc $asm-ms$ms-degenerate-contigs \
- -dump $asm-ms$ms-allfrags-degeneratecontigs.badmers \
- > $asm-ms$ms-allfrags-degeneratecontigs.badmers.asmMerQC &
-fi
-
-if [ ! -e $asm-ms$ms-clrfrags-normalcontigs.badmers.asmMerQC ] ; then
- $asmMerQC -tf $asm-ms$ms-clr-frags \
- -co $asm-ms$ms-normal-contigs \
- -dump $asm-ms$ms-clrfrags-normalcontigs.badmers \
- > $asm-ms$ms-clrfrags-normalcontigs.badmers.asmMerQC &
-fi
-if [ ! -e $asm-ms$ms-clrfrags-allcontigs.badmers.asmMerQC ] ; then
- $asmMerQC -tf $asm-ms$ms-clr-frags \
- -ac $asm-ms$ms-all-contigs \
- -dump $asm-ms$ms-clrfrags-allcontigs.badmers \
- > $asm-ms$ms-clrfrags-allcontigs.badmers.asmMerQC &
-fi
-if [ ! -e $asm-ms$ms-clrfrags-degeneratecontigs.badmers.asmMerQC ] ; then
- $asmMerQC -tf $asm-ms$ms-clr-frags \
- -dc $asm-ms$ms-degenerate-contigs \
- -dump $asm-ms$ms-clrfrags-degeneratecontigs.badmers \
- > $asm-ms$ms-clrfrags-degeneratecontigs.badmers.asmMerQC &
-fi
-
-echo Mapping.
-
-if [ ! -e $asm-ms$ms-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then
- $mapMers -m 22 \
- -mers $asm-ms$ms-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.fasta \
- -seq $asm.normalcontigs.fasta \
- > $asm-ms$ms-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers &
-fi
-if [ ! -e $asm-ms$ms-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then
- $mapMers -m 22 \
- -mers $asm-ms$ms-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.fasta \
- -seq $asm.allcontigs.fasta \
- > $asm-ms$ms-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers &
-fi
-if [ ! -e $asm-ms$ms-allfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then
- $mapMers -m 22 \
- -mers $asm-ms$ms-allfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.fasta \
- -seq $asm.degeneratecontigs.fasta \
- > $asm-ms$ms-allfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers &
-fi
-
-if [ ! -e $asm-ms$ms-clrfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then
- $mapMers -m 22 \
- -mers $asm-ms$ms-clrfrags-normalcontigs.badmers.0.singlecontig.zerofrag.fasta \
- -seq $asm.normalcontigs.fasta \
- > $asm-ms$ms-clrfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers &
-fi
-if [ ! -e $asm-ms$ms-clrfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then
- $mapMers -m 22 \
- -mers $asm-ms$ms-clrfrags-allcontigs.badmers.0.singlecontig.zerofrag.fasta \
- -seq $asm.allcontigs.fasta \
- > $asm-ms$ms-clrfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers &
-fi
-if [ ! -e $asm-ms$ms-clrfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then
- $mapMers -m 22 \
- -mers $asm-ms$ms-clrfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.fasta \
- -seq $asm.degeneratecontigs.fasta \
- > $asm-ms$ms-clrfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers &
-fi
-
-if [ ! -e $asm-ms$ms-allfrags-normalcontigs.badmers.5.all.badmers ] ; then
- cat $asm-ms$ms-allfrags-normalcontigs.badmers.[01].*.fasta > $asm-ms$ms-allfrags-normalcontigs.badmers.5.allzero.fasta
- $mapMers -m 22 \
- -mers $asm-ms$ms-allfrags-normalcontigs.badmers.5.allzero.fasta \
- -seq $asm.normalcontigs.fasta \
- > $asm-ms$ms-allfrags-normalcontigs.badmers.5.allzero.badmers &
-fi
-
-date
diff --git a/kmer/meryl/binaryOp.C b/kmer/meryl/binaryOp.C
deleted file mode 100644
index 767c8ff..0000000
--- a/kmer/meryl/binaryOp.C
+++ /dev/null
@@ -1,176 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-
-#include "meryl.H"
-#include "libmeryl.H"
-
-
-void
-binaryOperations(merylArgs *args) {
-
- if (args->mergeFilesLen != 2) {
- fprintf(stderr, "ERROR - must have exactly two files!\n");
- exit(1);
- }
- if (args->outputFile == 0L) {
- fprintf(stderr, "ERROR - no output file specified.\n");
- exit(1);
- }
- if ((args->personality != PERSONALITY_SUB) &&
- (args->personality != PERSONALITY_ABS) &&
- (args->personality != PERSONALITY_DIVIDE)) {
- fprintf(stderr, "ERROR - only personalities sub and abs\n");
- fprintf(stderr, "ERROR - are supported in binaryOperations().\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- }
-
- // Open the input files, read in the first mer
- //
- merylStreamReader *A = new merylStreamReader(args->mergeFiles[0]);
- merylStreamReader *B = new merylStreamReader(args->mergeFiles[1]);
-
- A->nextMer();
- B->nextMer();
-
- // Make sure that the mersizes agree, and pick a prefix size for
- // the output
- //
- if (A->merSize() != B->merSize()) {
- fprintf(stderr, "ERROR - mersizes are different!\n");
- fprintf(stderr, "ERROR - mersize of '%s' is "u32bitFMT"\n", args->mergeFiles[0], A->merSize());
- fprintf(stderr, "ERROR - mersize of '%s' is "u32bitFMT"\n", args->mergeFiles[1], B->merSize());
- exit(1);
- }
-
- // Open the output file, using the larger of the two prefix sizes
- //
- merylStreamWriter *W = new merylStreamWriter(args->outputFile,
- A->merSize(),
- A->merCompression(),
- (A->prefixSize() > B->prefixSize()) ? A->prefixSize() : B->prefixSize(),
- A->hasPositions());
-
-
-
-
- // SUB - report A - B
- // ABS - report the absolute difference between the two files
- //
- // These two operations are very similar (SUB was derived from ABS), so
- // any bug found in one is probably in the other.
- //
- kMer Amer;
- u32bit Acnt = u32bitZERO;
- kMer Bmer;
- u32bit Bcnt = u32bitZERO;
-
- switch (args->personality) {
- case PERSONALITY_SUB:
- while (A->validMer() || B->validMer()) {
- Amer = A->theFMer();
- Acnt = A->theCount();
- Bmer = B->theFMer();
- Bcnt = B->theCount();
-
- // If the A stream is all out of mers, set Amer to be the
- // same as Bmer, and set Acnt to zero. Similar for B.
- //
- if (!A->validMer()) {
- Amer = Bmer;
- Acnt = u32bitZERO;
- }
- if (!B->validMer()) {
- Bmer = Amer;
- Bcnt = u32bitZERO;
- }
-
- //fprintf(stderr, "sub A="u64bitHEX" B="u64bitHEX"\n", Amer, Bmer);
-
- if (Amer == Bmer) {
- W->addMer(Amer, (Acnt > Bcnt) ? Acnt - Bcnt : 0);
- A->nextMer();
- B->nextMer();
- } else if (Amer < Bmer) {
- W->addMer(Amer, Acnt);
- A->nextMer();
- } else {
- B->nextMer();
- }
- }
- break;
- case PERSONALITY_ABS:
- while (A->validMer() || B->validMer()) {
- Amer = A->theFMer();
- Acnt = A->theCount();
- Bmer = B->theFMer();
- Bcnt = B->theCount();
-
- // If the A stream is all out of mers, set Amer to be the
- // same as Bmer, and set Acnt to zero. Similar for B.
- //
- if (!A->validMer()) {
- Amer = Bmer;
- Acnt = u32bitZERO;
- }
- if (!B->validMer()) {
- Bmer = Amer;
- Bcnt = u32bitZERO;
- }
-
- if (Amer == Bmer) {
- W->addMer(Amer, (Acnt > Bcnt) ? Acnt - Bcnt : Bcnt - Acnt);
- A->nextMer();
- B->nextMer();
- } else if (Amer < Bmer) {
- W->addMer(Amer, Acnt);
- A->nextMer();
- } else {
- W->addMer(Bmer, Bcnt);
- B->nextMer();
- }
- }
- break;
- case PERSONALITY_DIVIDE:
- while (A->validMer() || B->validMer()) {
- Amer = A->theFMer();
- Acnt = A->theCount();
- Bmer = B->theFMer();
- Bcnt = B->theCount();
-
- // If the A stream is all out of mers, set Amer to be the
- // same as Bmer, and set Acnt to zero. Similar for B.
- //
- if (!A->validMer()) {
- Amer = Bmer;
- Acnt = u32bitZERO;
- }
- if (!B->validMer()) {
- Bmer = Amer;
- Bcnt = u32bitZERO;
- }
-
- if (Amer == Bmer) {
- if ((Acnt > 0) && (Bcnt > 0)) {
- double d = 1000.0 * (double)Acnt / (double)Bcnt;
- if (d > 4096.0 * 1024.0 * 1024.0)
- d = 4096.0 * 1024.0 * 1024.0;
- W->addMer(Amer, (u32bit)floor(d));
- }
- A->nextMer();
- B->nextMer();
- } else if (Amer < Bmer) {
- A->nextMer();
- } else {
- B->nextMer();
- }
- }
- break;
- }
-
- delete A;
- delete B;
- delete W;
-}
diff --git a/kmer/meryl/build-threads.C b/kmer/meryl/build-threads.C
deleted file mode 100644
index 3776f26..0000000
--- a/kmer/meryl/build-threads.C
+++ /dev/null
@@ -1,94 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <pthread.h>
-
-#include "bio++.H"
-#include "meryl.H"
-#include "libmeryl.H"
-
-void
-runSegment(merylArgs *args, u64bit segment);
-
-pthread_mutex_t segmentMutex;
-u64bit segmentNext;
-u64bit segmentMax;
-u32bit *segmentDone;
-
-
-void*
-buildThread(void *U) {
- u64bit segment = u32bitZERO;
- merylArgs *args = (merylArgs *)U;
-
- while (segment < segmentMax) {
- pthread_mutex_lock(&segmentMutex);
- segment = segmentNext++;
- pthread_mutex_unlock(&segmentMutex);
-
- if (segment < segmentMax) {
- runSegment(args, segment);
- segmentDone[segment]++;
- }
- }
-
- if (args->beVerbose)
- fprintf(stderr, "Thread exits.\n");
-
- return(0L);
-}
-
-
-void
-runThreaded(merylArgs *args) {
-
- // Clear stuff
- //
- segmentNext = u64bitZERO;
- segmentMax = args->segmentLimit;
- segmentDone = new u32bit [segmentMax];
- for (u64bit s=0; s<segmentMax; s++)
- segmentDone[s] = u32bitZERO;
-
- // Initialize threads
- //
- pthread_attr_t threadAttr;
- pthread_t threadID;
-
- pthread_mutex_init(&segmentMutex, NULL);
-
- pthread_attr_init(&threadAttr);
- pthread_attr_setscope(&threadAttr, PTHREAD_SCOPE_SYSTEM);
- pthread_attr_setdetachstate(&threadAttr, PTHREAD_CREATE_DETACHED);
- pthread_attr_setschedpolicy(&threadAttr, SCHED_OTHER);
-
- // Start the threads
- //
- for (u64bit i=0; i<args->numThreads; i++)
- pthread_create(&threadID, &threadAttr, buildThread, (void *)args);
-
- // Wait for the threads to complete
- //
- struct timespec shortNap;
- shortNap.tv_sec = 1;
- shortNap.tv_nsec = 0;
-
- u64bit s=0;
- while (s < segmentMax) {
- if (segmentDone[s] == 0)
- nanosleep(&shortNap, 0L);
- else
- s++;
- }
-
- if (args->beVerbose)
- fprintf(stderr, "Threads all done, cleaning up.\n");
-
- // Cleanup
- //
- pthread_attr_destroy(&threadAttr);
- pthread_mutex_destroy(&segmentMutex);
-
- delete [] segmentDone;
-}
diff --git a/kmer/meryl/build.C b/kmer/meryl/build.C
deleted file mode 100644
index e6f790a..0000000
--- a/kmer/meryl/build.C
+++ /dev/null
@@ -1,842 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <math.h>
-
-#include "bio++.H"
-#include "meryl.H"
-#include "libmeryl.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-void runThreaded(merylArgs *args);
-
-// You probably want this to be the same as KMER_WORDS, but in rare
-// cases, it can be less.
-//
-#define SORTED_LIST_WIDTH KMER_WORDS
-
-// to make the sorted list be wider, we also need to store wide
-// things in the bitpackedarray buckets. probably easy (do multiple
-// adds of data, each at most 64 bits) but not braindead.
-
-#if SORTED_LIST_WIDTH == 1
-
-class sortedList_t {
-public:
- u64bit _w;
- u32bit _p;
-
- bool operator<(sortedList_t &that) {
- return(_w < that._w);
- };
-
- bool operator>=(sortedList_t &that) {
- return(_w >= that._w);
- };
-
- sortedList_t &operator=(sortedList_t &that) {
- _w = that._w;
- _p = that._p;
- return(*this);
- };
-};
-
-#else
-
-class sortedList_t {
-public:
- u64bit _w[SORTED_LIST_WIDTH];
- u32bit _p;
-
- bool operator<(sortedList_t &that) {
- for (u32bit i=SORTED_LIST_WIDTH; i--; ) {
- if (_w[i] < that._w[i]) return(true);
- if (_w[i] > that._w[i]) return(false);
- }
- return(false);
- };
-
- bool operator>=(sortedList_t &that) {
- for (u32bit i=SORTED_LIST_WIDTH; i--; ) {
- if (_w[i] > that._w[i]) return(true);
- if (_w[i] < that._w[i]) return(false);
- }
- return(true);
- };
-
- sortedList_t &operator=(sortedList_t &that) {
- for (u32bit i=SORTED_LIST_WIDTH; i--; )
- _w[i] = that._w[i];
- _p = that._p;
- return(*this);
- };
-};
-
-#endif
-
-
-
-
-void
-adjustHeap(sortedList_t *M, s64bit i, s64bit n) {
- sortedList_t m = M[i];
- s64bit j = (i << 1) + 1; // let j be the left child
-
- while (j < n) {
- if (j<n-1 && M[j] < M[j+1])
- j++; // j is the larger child
-
- if (m >= M[j]) // a position for M[i] has been found
- break;
-
- M[(j-1)/2] = M[j]; // Move larger child up a level
-
- j = (j << 1) + 1;
- }
-
- M[(j-1)/2] = m;
-}
-
-
-void
-submitPrepareBatch(merylArgs *args) {
- FILE *F;
- char nam[1024];
- char cmd[1024];
-
- sprintf(nam, "%s-prepare.sh", args->outputFile);
-
- errno = 0;
- F = fopen(nam, "w");
- if (errno)
- fprintf(stderr, "Failed to open '%s': %s\n", nam, strerror(errno)), exit(1);
-
- fprintf(F, "#!/bin/sh\n\n");
- fprintf(F, ". $SGE_ROOT/$SGE_CELL/common/settings.sh\n");
- fprintf(F, "%s -forcebuild %s\n", args->execName, args->options);
- fclose(F);
-
- if (args->sgeMergeOpt)
- sprintf(cmd, "qsub -cwd -b n -j y -o %s-prepare.err %s -N mp%s %s-prepare.sh",
- args->outputFile, args->sgeMergeOpt, args->sgeJobName, args->outputFile);
- else
- sprintf(cmd, "qsub -cwd -b n -j y -o %s-prepare.err -N mp%s %s-prepare.sh",
- args->outputFile, args->sgeJobName, args->outputFile);
- fprintf(stderr, "%s\n", cmd);
- if (system(cmd))
- fprintf(stderr, "%s\nFailed to execute qsub command: %s\n", cmd, strerror(errno)), exit(1);
-}
-
-
-void
-submitCountBatches(merylArgs *args) {
- FILE *F;
- char nam[1024];
- char cmd[1024];
-
- sprintf(nam, "%s-count.sh", args->outputFile);
-
- errno = 0;
- F = fopen(nam, "w");
- if (errno)
- fprintf(stderr, "Failed to open '%s': %s\n", nam, strerror(errno)), exit(1);
-
- fprintf(F, "#!/bin/sh\n\n");
- fprintf(F, ". $SGE_ROOT/$SGE_CELL/common/settings.sh\n");
- fprintf(F, "batchnum=`expr $SGE_TASK_ID - 1`\n");
- fprintf(F, "%s -v -countbatch $batchnum -o %s\n", args->execName, args->outputFile);
- fclose(F);
-
- if (args->sgeBuildOpt)
- sprintf(cmd, "qsub -t 1-"u64bitFMT" -cwd -b n -j y -o %s-count-\\$TASK_ID.err %s -N mc%s %s-count.sh",
- args->segmentLimit, args->outputFile, args->sgeBuildOpt, args->sgeJobName, args->outputFile);
- else
- sprintf(cmd, "qsub -t 1-"u64bitFMT" -cwd -b n -j y -o %s-count-\\$TASK_ID.err -N mc%s %s-count.sh",
- args->segmentLimit, args->outputFile, args->sgeJobName, args->outputFile);
- fprintf(stderr, "%s\n", cmd);
- if (system(cmd))
- fprintf(stderr, "%s\nFailed to execute qsub command: %s\n", cmd, strerror(errno)), exit(1);
-
- // submit the merge
-
- sprintf(nam, "%s-merge.sh", args->outputFile);
-
- errno = 0;
- F = fopen(nam, "w");
- if (errno)
- fprintf(stderr, "Failed to open '%s': %s\n", nam, strerror(errno)), exit(1);
-
- fprintf(F, "#!/bin/sh\n\n");
- fprintf(F, ". $SGE_ROOT/$SGE_CELL/common/settings.sh\n");
- fprintf(F, "%s -mergebatch -o %s\n", args->execName, args->outputFile);
- fclose(F);
-
- if (args->sgeMergeOpt)
- sprintf(cmd, "qsub -hold_jid mc%s -cwd -b n -j y -o %s-merge.err %s -N mm%s %s-merge.sh",
- args->sgeJobName, args->outputFile, args->sgeMergeOpt, args->sgeJobName, args->outputFile);
- else
- sprintf(cmd, "qsub -hold_jid mc%s -cwd -b n -j y -o %s-merge.err -N mm%s %s-merge.sh",
- args->sgeJobName, args->outputFile, args->sgeJobName, args->outputFile);
- fprintf(stderr, "%s\n", cmd);
- if (system(cmd))
- fprintf(stderr, "%s\nFailed to execute qsub command: %s\n", cmd, strerror(errno)), exit(1);
-}
-
-
-void
-prepareBatch(merylArgs *args) {
- bool fatalError = false;
-
- if (args->inputFile == 0L)
- fprintf(stderr, "ERROR - no input file specified.\n"), fatalError = true;
-
- if (args->outputFile == 0L)
- fprintf(stderr, "ERROR - no output file specified.\n"), fatalError = true;
-
- if ((args->doForward == false) &&
- (args->doReverse == false) &&
- (args->doCanonical == false))
- fprintf(stderr, "ERROR - need to specify at least one of -f, -r, -C\n"), fatalError = true;
-
- if ((args->doForward && args->doReverse) ||
- (args->doForward && args->doCanonical) ||
- (args->doReverse && args->doCanonical))
- fprintf(stderr, "ERROR - only one of -f, -r and -C may be specified!\n"), fatalError = true;
-
- if (args->lowCount > args->highCount)
- fprintf(stderr, "ERROR - lowCount > highCount??\n"), fatalError = true;
-
- if (args->segmentLimit && args->memoryLimit)
- fprintf(stderr, "ERROR: Only one of -memory and -segments can be specified.\n"), fatalError=true;
-
- if (fatalError)
- exit(1);
-
- if (args->numThreads > 0) {
- // If we were given no segment or memory limit, but threads, we
- // really want to create n segments.
- //
- if ((args->segmentLimit == 0) && (args->memoryLimit == 0)) {
- args->segmentLimit = args->numThreads;
- }
-
- // If we are given a memory limit and threads, we want to use that much memory
- // total, not per thread.
- //
- if ((args->memoryLimit > 0) && (args->numThreads > 0)) {
- args->segmentLimit = 0;
- args->memoryLimit /= args->numThreads;
- }
- }
-
- {
- seqStream *seqstr = new seqStream(args->inputFile);
-
- args->numBasesActual = 0;
- for (u32bit i=0; i<seqstr->numberOfSequences(); i++)
- args->numBasesActual += seqstr->lengthOf(i);
-
- merStream *merstr = new merStream(new kMerBuilder(args->merSize), seqstr, true, true);
-
- args->numMersActual = merstr->approximateNumberOfMers() + 1;
-
- delete merstr;
- }
-
-#warning not submitting prepareBatch to grid
-#if 0
- if ((args->isOnGrid) || (args->sgeJobName == 0L)) {
- } else {
-
- // Shucks, we need to build the merstream file. Lets do it
- // on the grid!
- //
- submitPrepareBatch(args);
- exit(0);
- }
-#endif
-
-
- // If there is a memory limit, ignore the total number of mers and
- // pick a value that fits in memory.
- //
- // Otherwise, if there is a segment limit, split the total number
- // of mers into n pieces. Remember, there cannot be both a
- // memoryLimit and a segmentLimit.
- //
- // Otherwise, we must be doing it all in one fell swoop.
- //
- if (args->memoryLimit) {
- args->mersPerBatch = estimateNumMersInMemorySize(args->merSize, args->memoryLimit, args->positionsEnabled, args->beVerbose);
- if (args->mersPerBatch > args->numMersActual)
- args->mersPerBatch = args->numMersActual;
- args->segmentLimit = (u64bit)ceil((double)args->numMersActual / (double)args->mersPerBatch);
- if (args->beVerbose)
- fprintf(stderr, "Have a memory limit: mersPerBatch="u64bitFMT" segmentLimit="u64bitFMT"\n", args->mersPerBatch, args->segmentLimit);
- } else if (args->segmentLimit) {
- args->mersPerBatch = (u64bit)ceil((double)args->numMersActual / (double)args->segmentLimit);
- if (args->beVerbose)
- fprintf(stderr, "Have a segment limit: mersPerBatch="u64bitFMT" segmentLimit="u64bitFMT"\n", args->mersPerBatch, args->segmentLimit);
- } else {
- args->mersPerBatch = args->numMersActual;
- args->segmentLimit = 1;
- if (args->beVerbose)
- fprintf(stderr, "Have NO LIMITS!: mersPerBatch="u64bitFMT" segmentLimit="u64bitFMT"\n", args->mersPerBatch, args->segmentLimit);
- }
-
- args->basesPerBatch = (u64bit)ceil((double)args->numBasesActual / (double)args->segmentLimit);
- if (args->beVerbose)
- fprintf(stderr, "basesPerBatch = "u64bitFMT"\n", args->basesPerBatch);
-
- // Choose the optimal number of buckets to reduce memory usage.
- // Yes, this is already done in estimateNumMersInMemorySize() (but
- // not saved) and we need to do it for the other cases anyway.
- //
- // We use the number of mers per batch + 1 because we need to store
- // the first position after the last mer. That is, if there are
- // two mers, we will store that the first mer is at position 0, the
- // second mer is at position 1, and the end of the second mer is at
- // position 2.
- //
- args->bucketPointerWidth = logBaseTwo64(args->basesPerBatch + 1);
- args->numBuckets_log2 = optimalNumberOfBuckets(args->merSize, args->basesPerBatch, args->positionsEnabled);
- args->numBuckets = (u64bitONE << args->numBuckets_log2);
- args->merDataWidth = args->merSize * 2 - args->numBuckets_log2;
- //args->bucketPointerMask = u64bitMASK(args->numBuckets_log2);
-
-
- if (args->merDataWidth > SORTED_LIST_WIDTH * 64) {
- fprintf(stderr, " numMersActual = "u64bitFMT"\n", args->numMersActual);
- fprintf(stderr, " mersPerBatch = "u64bitFMT"\n", args->mersPerBatch);
- fprintf(stderr, " basesPerBatch = "u64bitFMT"\n", args->basesPerBatch);
- fprintf(stderr, " numBuckets = "u64bitFMT" ("u32bitFMT" bits)\n", args->numBuckets, args->numBuckets_log2);
- fprintf(stderr, " bucketPointerWidth = "u32bitFMT"\n", args->bucketPointerWidth);
- fprintf(stderr, " merDataWidth = "u32bitFMT"\n", args->merDataWidth);
- fprintf(stderr, "Sorry! merSize too big! Increase KMER_WORDS in libbio.kmer.H\n");
- exit(1);
- }
-
- if (args->beVerbose) {
- if (args->memoryLimit)
- fprintf(stderr, "Computing "u64bitFMT" segments using "u64bitFMT"MB memory each.\n",
- args->segmentLimit, args->memoryLimit);
- else
- fprintf(stderr, "Computing "u64bitFMT" segments using AS MUCH MEMORY AS NEEDED.\n",
- args->segmentLimit);
- fprintf(stderr, " numMersActual = "u64bitFMT"\n", args->numMersActual);
- fprintf(stderr, " mersPerBatch = "u64bitFMT"\n", args->mersPerBatch);
- fprintf(stderr, " basesPerBatch = "u64bitFMT"\n", args->basesPerBatch);
- fprintf(stderr, " numBuckets = "u64bitFMT" ("u32bitFMT" bits)\n", args->numBuckets, args->numBuckets_log2);
- fprintf(stderr, " bucketPointerWidth = "u32bitFMT"\n", args->bucketPointerWidth);
- fprintf(stderr, " merDataWidth = "u32bitFMT"\n", args->merDataWidth);
- }
-}
-
-
-
-
-void
-runSegment(merylArgs *args, u64bit segment) {
- merStream *M = 0L;
- merylStreamWriter *W = 0L;
- speedCounter *C = 0L;
- u32bit *bucketSizes = 0L;
- u64bit *bucketPointers = 0L;
- u64bit *merDataArray[SORTED_LIST_WIDTH] = { 0L };
- u32bit *merPosnArray = 0L;
-
- // If this segment exists already, skip it.
- //
- // XXX: This should be a command line option.
- // XXX: This should check that the files are complete meryl files.
- //
- char *filename = new char [strlen(args->outputFile) + 17];
- sprintf(filename, "%s.batch"u64bitFMT".mcdat", args->outputFile, segment);
-
- if (fileExists(filename)) {
- if (args->beVerbose)
- fprintf(stderr, "Found result for batch "u64bitFMT" in %s.\n", segment, filename);
- delete [] filename;
- return;
- }
-
- if ((args->beVerbose) && (args->segmentLimit > 1))
- fprintf(stderr, "Computing segment "u64bitFMT" of "u64bitFMT".\n", segment+1, args->segmentLimit);
-
- delete [] filename;
-
-
-
- //
- // We can do all allocations up front:
- // mer data storage (the buckets themselves, plus 64 for slop)
- // bucket pointers (plus an extra bucket at the end and a little for slop)
- // bucket size counting space, last because we toss it out quickly
- //
- if (args->beVerbose)
- fprintf(stderr, " Allocating "u64bitFMT"MB for mer storage ("u32bitFMT" bits wide).\n",
- (args->basesPerBatch * args->merDataWidth + 64) >> 23, args->merDataWidth);
-
- // Mer storage - if mers are bigger than 32, we allocate full
- // words. The last allocation is always a bitPacked array.
-
- for (u64bit mword=0, width=args->merDataWidth; width > 0; ) {
- if (width >= 64) {
- merDataArray[mword] = new u64bit [ args->basesPerBatch + 1 ];
- width -= 64;
- mword++;
- } else {
- merDataArray[mword] = new u64bit [ (args->basesPerBatch * width + 64) >> 6 ];
- width = 0;
- }
- }
-
- if (args->positionsEnabled) {
- if (args->beVerbose)
- fprintf(stderr, " Allocating "u64bitFMT"MB for mer position storage.\n",
- (args->basesPerBatch * 32 + 32) >> 23);
- merPosnArray = new u32bit [ args->basesPerBatch + 1 ];
- }
-
- if (args->beVerbose)
- fprintf(stderr, " Allocating "u64bitFMT"MB for bucket pointer table ("u32bitFMT" bits wide).\n",
- (args->numBuckets * args->bucketPointerWidth + 128) >> 23, args->bucketPointerWidth);
- bucketPointers = new u64bit [(args->numBuckets * args->bucketPointerWidth + 128) >> 6];
-
-
- if (args->beVerbose)
- fprintf(stderr, " Allocating "u64bitFMT"MB for counting the size of each bucket.\n", args->numBuckets >> 18);
- bucketSizes = new u32bit [ args->numBuckets ];
- for (u64bit i=args->numBuckets; i--; )
- bucketSizes[i] = u32bitZERO;
-
-
- // Position the mer stream at the start of this segments' mers.
- // The last segment goes until the stream runs out of mers,
- // everybody else does args->basesPerBatch mers.
-
- C = new speedCounter(" Counting mers in buckets: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
- M = new merStream(new kMerBuilder(args->merSize, args->merComp),
- new seqStream(args->inputFile),
- true, true);
- M->setBaseRange(args->basesPerBatch * segment, args->basesPerBatch * segment + args->basesPerBatch);
-
- char mstring[256];
-
- if (args->doForward) {
- while (M->nextMer()) {
- //fprintf(stderr, "FMER %s\n", M->theFMer().merToString(mstring));
- bucketSizes[ args->hash(M->theFMer()) ]++;
- C->tick();
- }
- }
-
- if (args->doReverse) {
- while (M->nextMer()) {
- //fprintf(stderr, "RMER %s\n", M->theRMer().merToString(mstring));
- bucketSizes[ args->hash(M->theRMer()) ]++;
- C->tick();
- }
- }
-
- if (args->doCanonical) {
- while (M->nextMer()) {
- if (M->theFMer() <= M->theRMer()) {
- //fprintf(stderr, "FMER %s\n", M->theFMer().merToString(mstring));
- bucketSizes[ args->hash(M->theFMer()) ]++;
- } else {
- //fprintf(stderr, "RMER %s\n", M->theRMer().merToString(mstring));
- bucketSizes[ args->hash(M->theRMer()) ]++;
- }
- C->tick();
- }
- }
-
- delete C;
- delete M;
-
- // Create the hash index using the counts. The hash points
- // to the end of the bucket; when we add a word, we move the
- // hash bucket pointer down one.
- //
- // When done, we can deallocate the counting table.
- //
- if (args->beVerbose)
- fprintf(stderr, " Creating bucket pointers.\n");
-
- {
- u64bit mi=0;
- u64bit mj=0;
- u64bit mc=0;
-
- while (mi < args->numBuckets) {
- mc += bucketSizes[mi++];
- setDecodedValue(bucketPointers, mj, args->bucketPointerWidth, mc);
- mj += args->bucketPointerWidth;
- }
-
- // Add the location of the end of the table. This is not
- // modified when adding words, but is used to determine
- // the size of the last bucket.
- //
- setDecodedValue(bucketPointers, mj, args->bucketPointerWidth, mc);
- }
-
-
- // All done with the counting table, get rid of it.
- //
- if (args->beVerbose)
- fprintf(stderr, " Releasing "u64bitFMT"MB from counting the size of each bucket.\n", args->numBuckets >> 18);
- delete [] bucketSizes;
-
-
- C = new speedCounter(" Filling mers into list: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
- M = new merStream(new kMerBuilder(args->merSize, args->merComp),
- new seqStream(args->inputFile),
- true, true);
- M->setBaseRange(args->basesPerBatch * segment, args->basesPerBatch * segment + args->basesPerBatch);
-
- while (M->nextMer()) {
-
- kMer const &m = ((args->doReverse) || (args->doCanonical && (M->theFMer() > M->theRMer()))) ?
- M->theRMer()
- :
- M->theFMer();
-
- u64bit element = preDecrementDecodedValue(bucketPointers,
- args->hash(m) * args->bucketPointerWidth,
- args->bucketPointerWidth);
-
-#if SORTED_LIST_WIDTH == 1
- // Even though this would work in the general loop below, we
- // special case one word mers to avoid the loop overhead.
- //
- setDecodedValue(merDataArray[0],
- element * args->merDataWidth,
- args->merDataWidth,
- m.endOfMer(args->merDataWidth));
-#else
- for (u64bit mword=0, width=args->merDataWidth; width>0; ) {
- if (width >= 64) {
- merDataArray[mword][element] = m.getWord(mword);
- width -= 64;
- mword++;
- } else {
- setDecodedValue(merDataArray[mword],
- element * width,
- width,
- m.getWord(mword) & u64bitMASK(width));
- width = 0;
- }
- }
-#endif
-
- if (args->positionsEnabled)
- merPosnArray[element] = M->thePositionInStream();
-
- C->tick();
- }
-
- delete C;
- delete M;
-
- char *batchOutputFile = new char [strlen(args->outputFile) + 33];
- sprintf(batchOutputFile, "%s.batch"u64bitFMT, args->outputFile, segment);
-
- C = new speedCounter(" Writing output: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
- W = new merylStreamWriter((args->segmentLimit == 1) ? args->outputFile : batchOutputFile,
- args->merSize, args->merComp,
- args->numBuckets_log2,
- args->positionsEnabled);
-
- // Sort each bucket into sortedList, then output the mers
- //
- sortedList_t *sortedList = 0L;
- u32bit sortedListMax = 0;
- u32bit sortedListLen = 0;
-
- for (u64bit bucket=0, bucketPos=0; bucket < args->numBuckets; bucket++) {
- u64bit st = getDecodedValue(bucketPointers, bucketPos, args->bucketPointerWidth);
- bucketPos += args->bucketPointerWidth;
- u64bit ed = getDecodedValue(bucketPointers, bucketPos, args->bucketPointerWidth);
-
- if (ed < st) {
- fprintf(stderr, "ERROR: In segment "u64bitFMT"\n", segment);
- fprintf(stderr, "ERROR: Bucket "u64bitFMT" (out of "u64bitFMT") ends before it starts!\n",
- bucket, args->numBuckets);
- fprintf(stderr, "ERROR: start="u64bitFMT"\n", st);
- fprintf(stderr, "ERROR: end ="u64bitFMT"\n", ed);
- }
- assert(ed >= st);
-
- if ((ed - st) > (u64bitONE << 30)) {
- fprintf(stderr, "ERROR: In segment "u64bitFMT"\n", segment);
- fprintf(stderr, "ERROR: Bucket "u64bitFMT" (out of "u64bitFMT") is HUGE!\n",
- bucket, args->numBuckets);
- fprintf(stderr, "ERROR: start="u64bitFMT"\n", st);
- fprintf(stderr, "ERROR: end ="u64bitFMT"\n", ed);
- }
-
- // Nothing here? Keep going.
- if (ed == st)
- continue;
-
- sortedListLen = (u32bit)(ed - st);
-
- // Allocate more space, if we need to.
- //
- if (sortedListLen > sortedListMax) {
- delete [] sortedList;
- sortedList = new sortedList_t [2 * sortedListLen + 1];
- sortedListMax = 2 * sortedListLen;
- }
-
- // Clear out the sortedList -- if we don't, we leave the high
- // bits unset which will probably make the sort random.
- //
- bzero(sortedList, sizeof(sortedList_t) * sortedListLen);
-
- // Unpack the mers into the sorting array
- //
- if (args->positionsEnabled)
- for (u64bit i=st; i<ed; i++)
- sortedList[i-st]._p = merPosnArray[i];
-
-#if SORTED_LIST_WIDTH == 1
- for (u64bit i=st, J=st*args->merDataWidth; i<ed; i++, J += args->merDataWidth)
- sortedList[i-st]._w = getDecodedValue(merDataArray[0], J, args->merDataWidth);
-#else
- for (u64bit i=st; i<ed; i++) {
- for (u64bit mword=0, width=args->merDataWidth; width>0; ) {
- if (width >= 64) {
- sortedList[i-st]._w[mword] = merDataArray[mword][i];
- width -= 64;
- mword++;
- } else {
- sortedList[i-st]._w[mword] = getDecodedValue(merDataArray[mword], i * width, width);
- width = 0;
- }
- }
- }
-#endif
-
- // Sort if there is more than one item
- //
- if (sortedListLen > 1) {
- for (s64bit t=(sortedListLen-2)/2; t>=0; t--)
- adjustHeap(sortedList, t, sortedListLen);
-
- for (s64bit t=sortedListLen-1; t>0; t--) {
- sortedList_t tv = sortedList[t];
- sortedList[t] = sortedList[0];
- sortedList[0] = tv;
-
- adjustHeap(sortedList, 0, t);
- }
- }
-
- // Dump the list of mers to the file.
- //
- kMer mer(args->merSize);
-
- for (u32bit t=0; t<sortedListLen; t++) {
- C->tick();
-
- // Build the complete mer
- //
-#if SORTED_LIST_WIDTH == 1
- mer.setWord(0, sortedList[t]._w);
-#else
- for (u64bit mword=0; mword < SORTED_LIST_WIDTH; mword++)
- mer.setWord(mword, sortedList[t]._w[mword]);
-#endif
- mer.setBits(args->merDataWidth, args->numBuckets_log2, bucket);
-
- // Add it
- if (args->positionsEnabled)
- W->addMer(mer, 1, &sortedList[t]._p);
- else
- W->addMer(mer, 1, 0L);
-
- }
- }
-
- delete [] sortedList;
-
- delete C;
- delete W;
-
- delete [] batchOutputFile;
-
- for (u32bit x=0; x<SORTED_LIST_WIDTH; x++)
- delete [] merDataArray[x];
-
- delete [] merPosnArray;
-
- delete [] bucketPointers;
-
- if (args->beVerbose)
- fprintf(stderr, "Segment "u64bitFMT" finished.\n", segment);
-}
-
-
-
-void
-build(merylArgs *args) {
-
- if (!args->countBatch && !args->mergeBatch)
- prepareBatch(args);
-
- // Three choices:
- //
- // threaded -- start threads, launch pieces in each thread. This
- // thread waits for completion and then merges the results.
- //
- // batched -- write info file and exit. Compute and merge is done
- // on separate invocations.
- //
- // segmented -- write info file, then do each piece sequentially.
- // After all pieces finished, do a merge.
- //
- //
-
- bool doMerge = false;
-
- if (args->configBatch) {
-
- // Write out our configuration and exit if we are -configbatch
- //
- args->writeConfig();
-
- if (args->sgeJobName) {
- fprintf(stdout, "Batch prepared. Submitting to the grid.\n");
- submitCountBatches(args);
- } else {
- fprintf(stdout, "Batch prepared. Please run:\n");
- for (u64bit s=0; s<args->segmentLimit; s++)
- fprintf(stdout, "%s -countbatch "u64bitFMT" -o %s\n", args->execName, s, args->outputFile);
- fprintf(stdout, "%s -mergebatch -o %s\n", args->execName, args->outputFile);
- }
- } else if (args->countBatch) {
-
- // Read back the configuration, run the segment and exit if we
- // are -countbatch
- //
- merylArgs *savedArgs = new merylArgs(args->outputFile);
- savedArgs->beVerbose = args->beVerbose;
- runSegment(savedArgs, args->batchNumber);
- delete savedArgs;
- } else if (args->mergeBatch) {
-
- // Check that all the files exist if we are -mergebatch and
- // continue with execution
- //
- // MEMORY LEAK! We should delete this at the end of the
- // function, but it's a pain, and who cares?
- //
- merylArgs *savedArgs = new merylArgs(args->outputFile);
- savedArgs->beVerbose = args->beVerbose;
-
- args = savedArgs;
-
- doMerge = true;
- } else {
-
- if (args->numThreads > 1)
-
- // Run, using threads. There is a lot of baloney needed, so it's
- // all in a separate function.
- //
- runThreaded(args);
- else
- // No special options given, do all the work here and now
- //
- for (u64bit s=0; s<args->segmentLimit; s++)
- runSegment(args, s);
-
- // Either case, we want to merge now.
- //
- doMerge = true;
- }
-
-
- // If there is more than one segment, merge them to get the output.
- //
- // We do this by contructing a meryl command line and recursively
- // (effectively) calling meryl.
- //
- // The command line is
- //
- // ./meryl -M merge [-v] -s batch1 -s batch2 ... -s batchN -o outputFile
- //
- if ((doMerge) && (args->segmentLimit > 1)) {
-
- if (args->beVerbose)
- fprintf(stderr, "Merge results.\n");
-
- int argc = 0;
- char **argv = new char* [7 + 2 * args->segmentLimit];
- bool *arga = new bool [7 + 2 * args->segmentLimit];
-
- arga[argc] = false; argv[argc++] = "meryl-build-merge";
- arga[argc] = false; argv[argc++] = "-M";
- arga[argc] = false; argv[argc++] = "merge";
-
- if (args->beVerbose) {
- arga[argc] = false;
- argv[argc++] = "-v";
- }
-
- for (u32bit i=0; i<args->segmentLimit; i++) {
- arga[argc] = false;
- argv[argc++] = "-s";
- arga[argc] = true;
- argv[argc] = new char [strlen(args->outputFile) + 33];
- sprintf(argv[argc], "%s.batch"u32bitFMT, args->outputFile, i);
- argc++;
- }
-
- arga[argc] = false; argv[argc++] = "-o";
- arga[argc] = false; argv[argc++] = args->outputFile;
-
- merylArgs *addArgs = new merylArgs(argc, argv);
- multipleOperations(addArgs);
-
- // Cleanup the memory leak.
- //
- delete addArgs;
- for (int i=0; i<argc; i++)
- if (arga[i])
- delete [] argv[i];
- delete [] argv;
- delete [] arga;
-
- // Remove temporary files
- //
- char *filename = new char [strlen(args->outputFile) + 17];
-
- for (u32bit i=0; i<args->segmentLimit; i++) {
- sprintf(filename, "%s.batch"u32bitFMT".mcidx", args->outputFile, i);
- unlink(filename);
- sprintf(filename, "%s.batch"u32bitFMT".mcdat", args->outputFile, i);
- unlink(filename);
- sprintf(filename, "%s.batch"u32bitFMT".mcpos", args->outputFile, i);
- unlink(filename);
- }
-
- delete [] filename;
- }
-
- // If we just merged, delete the merstream file
- //
- if (doMerge) {
- char *filename = new char [strlen(args->outputFile) + 17];
-
- sprintf(filename, "%s.merStream", args->outputFile);
- unlink(filename);
-
- delete [] filename;
- }
-}
diff --git a/kmer/meryl/compare-counts.C b/kmer/meryl/compare-counts.C
deleted file mode 100644
index c1d47de..0000000
--- a/kmer/meryl/compare-counts.C
+++ /dev/null
@@ -1,233 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-
-#include "libmeryl.H"
-
-
-
-
-#if 0
-void
-heatMap() {
- speedCounter *C = new speedCounter(" Examining: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1ffffff, false);
-
-#define MAXA 150
-#define MAXB 150
-
- double heatraw[MAXA][MAXB];
- double heatsca[MAXA][MAXB];
-
- for (u32bit i=0; i<MAXA; i++)
- for (u32bit j=0; j<MAXB; j++)
- heatraw[i][j] = heatsca[i][j] = 0;
-
- A->nextMer();
- B->nextMer();
-
- while ((A->validMer()) ||
- (B->validMer())) {
- kMer &a = A->theFMer();
- kMer &b = B->theFMer();
-
- u32bit ac = A->theCount();
- u32bit bc = B->theCount();
-
- if (ac >= MAXA)
- ac = MAXA-1;
-
- if (bc >= MAXB)
- bc = MAXB-1;
-
- if (A->validMer() == false) {
- ac = 0;
- heatraw[ac][bc]++;
- B->nextMer();
- continue;
- }
-
- if (B->validMer() == false) {
- bc = 0;
- heatraw[ac][bc]++;
- A->nextMer();
- continue;
- }
-
- if (a == b) {
- heatraw[ac][bc]++;
- A->nextMer();
- B->nextMer();
-
- } else if (a < b) {
- heatraw[ac][0]++;
- A->nextMer();
-
- } else {
- heatraw[0][bc]++;
- B->nextMer();
- }
-
- C->tick();
- }
-
- delete C;
- delete A;
- delete B;
-
- // Scale each row to be between 0 and 1
-
-#if 0
- for (u32bit j=0; j<MAXB; j++) {
- double mina = heatraw[0][j];
- double maxa = heatraw[0][j];
-
- for (u32bit ii=0; ii<MAXA; ii++) {
- if (maxa < heatraw[ii][j])
- maxa = heatraw[ii][j];
- if (heatraw[ii][j] < mina)
- mina = heatraw[ii][j];
- }
-
- for (u32bit i=0; i<MAXA; i++)
- heatsca[i][j] = (heatraw[i][j] - mina) / (maxa - mina);
- }
-#endif
-
-
- for (u32bit i=0; i<MAXA; i++)
- for (u32bit j=0; j<MAXB; j++)
- fprintf(stdout, u32bitFMT"\t"u32bitFMT"\t%f\n", i, j, log(heatraw[i][j]));
-}
-#endif
-
-
-
-
-
-int
-main(int argc, char **argv) {
- merylStreamReader *T = 0L;
- merylStreamReader *S = 0L;
- char *outputPrefix = NULL;
- char *plotTitle = NULL;
- int arg = 1;
- int err = 0;
-
- while (arg < argc) {
- if (strcmp(argv[arg], "-truth") == 0) {
- T = new merylStreamReader(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-sample") == 0) {
- S = new merylStreamReader(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-output") == 0) {
- outputPrefix = argv[++arg];
-
- } else if (strcmp(argv[arg], "-title") == 0) {
- plotTitle = argv[++arg];
-
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- err++;
- }
-
- arg++;
- }
-
- if ((T == 0L) || (S == 0L) || (outputPrefix == 0L) || (plotTitle == 0L) || (err)) {
- fprintf(stderr, "usage: %s [opts]\n", argv[0]);
- fprintf(stderr, " -truth k-mers from reference\n");
- fprintf(stderr, " -sample k-mers from sample\n");
- fprintf(stderr, " -output output prefix\n");
- fprintf(stderr, " -title plot label\n");
- exit(1);
- }
-
- u32bit kmerSize = T->merSize();
-
-#define HMAX 64 * 1024
-
- u32bit *Htrue = new u32bit [HMAX];
- u32bit *Hnoise = new u32bit [HMAX];
-
- for (u32bit i=0; i<HMAX; i++)
- Htrue[i] = Hnoise[i] = 0;
-
- T->nextMer();
- S->nextMer();
-
- while ((T->validMer()) ||
- (S->validMer())) {
- kMer &t = T->theFMer();
- kMer &s = S->theFMer();
-
- u32bit tc = T->theCount();
- u32bit sc = S->theCount();
-
- if (tc >= HMAX) tc = HMAX-1;
- if (sc >= HMAX) sc = HMAX-1;
-
- // If we're out of truth kmers, the sample is noise.
- if (T->validMer() == false) {
- Hnoise[sc]++;
- S->nextMer();
- continue;
- }
-
- // If we're out of sample kmers, do nothing but go to the next truth kmer.
- if (S->validMer() == false) {
- T->nextMer();
- continue;
- }
-
- // If the kmers are equal, this is a true kmer
- if (t == s) {
- Htrue[sc]++;
- T->nextMer();
- S->nextMer();
- }
-
- // If the truth kmer is the lesser, get the next truth.
- else if (t < s) {
- T->nextMer();
- }
-
- // Else the sample kmer is smaller, add it to the noise pile, and get the next.
- else {
- Hnoise[sc]++;
- S->nextMer();
- }
- }
-
- delete T;
- delete S;
-
- char outputName[FILENAME_MAX];
-
- sprintf(outputName, "%s.gp", outputPrefix);
- FILE *outputGP = fopen(outputName, "w");
-
- sprintf(outputName, "%s.dat", outputPrefix);
- FILE *outputDAT = fopen(outputName, "w");
-
- fprintf(outputGP, "set terminal png\n");
- fprintf(outputGP, "set output \"%s.png\"\n", outputPrefix);
- fprintf(outputGP, "set title \"%s true/false %d-mers\"\n", plotTitle, kmerSize);
- fprintf(outputGP, "set xlabel \"k-mer count\"\n");
- fprintf(outputGP, "set ylabel \"number of kmers\"\n");
- fprintf(outputGP, "plot [0:100] [0:1000000] \"%s.dat\" using 1:2 with lines title \"true\", \"%s.dat\" using 1:3 with lines title \"false\"\n",
- outputPrefix, outputPrefix);
-
- fclose(outputGP);
-
- for (u32bit i=0; i<HMAX; i++)
- fprintf(outputDAT, u32bitFMT"\t"u32bitFMT"\t"u32bitFMT"\n", i, Htrue[i], Hnoise[i]);
-
- fclose(outputDAT);
-
- sprintf(outputName, "gnuplot < %s.gp", outputPrefix);
- system(outputName);
-
- exit(0);
-}
diff --git a/kmer/meryl/dump.C b/kmer/meryl/dump.C
deleted file mode 100644
index ac80192..0000000
--- a/kmer/meryl/dump.C
+++ /dev/null
@@ -1,156 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "meryl.H"
-#include "libmeryl.H"
-
-#include <algorithm>
-
-void
-dumpThreshold(merylArgs *args) {
- merylStreamReader *M = new merylStreamReader(args->inputFile);
- char str[1025];
-
- while (M->nextMer()) {
- if (M->theCount() >= args->numMersEstimated)
- fprintf(stdout, ">"u64bitFMT"\n%s\n",
- M->theCount(),
- M->theFMer().merToString(str));
- }
-
- delete M;
-}
-
-
-void
-dumpPositions(merylArgs *args) {
- merylStreamReader *M = new merylStreamReader(args->inputFile);
- char str[1025];
-
- if (M->hasPositions() == false) {
- fprintf(stderr, "File '%s' contains no position information.\n", args->inputFile);
- } else {
- while (M->nextMer()) {
- fprintf(stdout, ">"u64bitFMT, M->theCount());
- for (u32bit i=0; i<M->theCount(); i++)
- fprintf(stdout, " "u32bitFMT, M->getPosition(i));
- fprintf(stdout, "\n%s\n", M->theFMer().merToString(str));
- }
- }
-
- delete M;
-}
-
-
-void
-countUnique(merylArgs *args) {
- merylStreamReader *M = new merylStreamReader(args->inputFile);
-
-#warning make this a test
-#if 0
- u64bit numDistinct = 0;
- u64bit numUnique = 0;
- u64bit numMers = 0;
- u64bit c = 0;
-
- while (M->nextMer()) {
- c = M->theCount();
-
- numDistinct++;
- if (c == 1)
- numUnique++;
- numMers += c;
- }
-
- assert(numMers == M->numberOfTotalMers());
- assert(numDistinct == M->numberOfDistinctMers());
- assert(numUnique == M->numberOfUniqueMers());
- fprintf(stderr, "OK\n");
-#endif
-
- fprintf(stdout, "Found "u64bitFMT" mers.\n", M->numberOfTotalMers());
- fprintf(stdout, "Found "u64bitFMT" distinct mers.\n", M->numberOfDistinctMers());
- fprintf(stdout, "Found "u64bitFMT" unique mers.\n", M->numberOfUniqueMers());
-
- delete M;
-}
-
-
-void
-plotHistogram(merylArgs *args) {
- u64bit distinct = 0;
- u64bit total = 0;
-
- merylStreamReader *M = new merylStreamReader(args->inputFile);
-
- fprintf(stderr, "Found "u64bitFMT" mers.\n", M->numberOfTotalMers());
- fprintf(stderr, "Found "u64bitFMT" distinct mers.\n", M->numberOfDistinctMers());
- fprintf(stderr, "Found "u64bitFMT" unique mers.\n", M->numberOfUniqueMers());
-
- fprintf(stderr, "Largest mercount is "u64bitFMT"; "u64bitFMT" mers are too big for histogram.\n",
- M->histogramMaximumCount(), M->histogramHuge());
-
- for (u32bit i=1; i<M->histogramLength(); i++) {
- u64bit hist = M->histogram(i);
-
- if (hist > 0) {
- distinct += hist;
- total += hist * i;
-
- fprintf(stdout, u32bitFMT"\t"u64bitFMT"\t%.4f\t%.4f\n",
- i,
- hist,
- distinct / (double)M->numberOfDistinctMers(),
- total / (double)M->numberOfTotalMers());
- }
- }
-
- delete M;
-}
-
-
-
-void
-dumpDistanceBetweenMers(merylArgs *args) {
- merylStreamReader *M = new merylStreamReader(args->inputFile);
-
- // This is now tough because we don't know where the sequences end,
- // and our positions encode position in the chain.
-
- u32bit histMax = 64 * 1024 * 1024;
- u64bit *hist = new u64bit [histMax];
- u64bit histHuge = 0;
-
- if (M->hasPositions() == false) {
- fprintf(stderr, "File '%s' contains no position information.\n", args->inputFile);
- } else {
- while (M->nextMer()) {
- std::sort(M->thePositions(), M->thePositions() + M->theCount());
-
- for (u32bit i=1; i<M->theCount(); i++) {
- u32bit d = M->getPosition(i) - M->getPosition(i-1);
- if (d < histMax)
- hist[d]++;
- else
- histHuge++;
- }
- }
-
- u32bit maxd = 0;
-
- for (u32bit d=0; d<histMax; d++)
- if (hist[d])
- maxd = d+1;
-
- for (u32bit d=0; d<maxd; d++)
- if (hist[d])
- fprintf(stderr, u32bitFMT"\t"u64bitFMT"\n", d, hist[d]);
-
- if (histHuge)
- fprintf(stderr, "huge\t"u64bitFMT"\n", histHuge);
- }
-
- delete [] hist;
- delete M;
-}
diff --git a/kmer/meryl/estimate.C b/kmer/meryl/estimate.C
deleted file mode 100644
index 5c690d8..0000000
--- a/kmer/meryl/estimate.C
+++ /dev/null
@@ -1,182 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "bio++.H"
-#include "seqStream.H"
-#include "merStream.H"
-#include "libmeryl.H"
-#include "meryl.H"
-
-// Takes a memory limit in MB, returns the number of mers that we can
-// fit in that memory size, assuming optimalNumberOfBuckets() below
-// uses the same algorithm.
-//
-u64bit
-estimateNumMersInMemorySize(u32bit merSize,
- u32bit mem,
- bool positionsEnabled,
- bool beVerbose) {
- u64bit maxN = 0;
- u64bit bestT = 0;
-
-
- // For each possible number of buckets, try all poissible pointer
- // widths. First we compute the number of mers that fit in a
- // bucket pointer table of size 2^t storing N bits in the mer data
- // table, then we check that the number of mers in the mer data
- // table agrees with the width of the pointer table.
-
-
- // This is the memory size we are trying to fill, in bits.
- //
- u64bit memLimt = ((u64bit)mem) << 23;
-
- // Positions consume space too, but only if enabled.
- //
- u64bit posPerMer = 0;
- if (positionsEnabled)
- posPerMer = 32;
-
- // Limit the number of entries in the bucket pointer table to
- // 50 bits -- thus, the prefix of each mer is at most 25.
- //
- u32bit tMax = 2*merSize - 2;
- if (tMax > 50)
- tMax = 50;
-
- for (u64bit t=2; t < tMax; t++) {
-
- // We need to try all possibilities of N, the width of the
- // bucket pointer table === log2(numMers).
- //
- // Increased to 40 bits, so we're valid up to 1 trillion mers.
- //
- for (u64bit N=1; N<40; N++) {
- u64bit Nmin = u64bitONE << (N - 1);
- u64bit Nmax = u64bitONE << (N);
-
- // The size in bits of the bucket pointer table.
- //
- u64bit bucketsize = (u64bitONE << t) * N;
-
- // If our bucket pointer table size hasn't already blown our
- // memory limit, compute the number of mers that we can stuff
- // into the list.
- //
- if (memLimt > bucketsize) {
-
- // The number of mers we can then fit into the mer data table
- // is easy to compute.
- //
- // Even though we allocate merDataArray, bucketPointers,
- // bucketSizes, we don't use merDataArray until after we
- // release bucketSizes, and so we only estimate the maximum
- // in core (not allocated) size.
- //
- u64bit n = (memLimt - bucketsize) / (2*merSize - t + posPerMer);
-
- // We can stop now if our computed number of mers is outside the range that
- // the bucket pointer table can address.
- //
- if ((Nmin <= n) && (n <= Nmax)) {
-
- //fprintf(stderr, "prefixSize="u64bitFMTW(2)" numMers="u64bitFMTW(9)" memory=%.3fMB\n",
- // t, n,
- // (((u64bitONE << t) * logBaseTwo64(n) + n * (2*merSize - t + posPerMer)) >> 3) / 1048576.0);
-
- // Remember the settings with the highest number of mers, if
- // more than zero mers.
- //
- if ((n > 0) &&
- (maxN < n)) {
- maxN = n;
- bestT = t;
- }
-
- }
- }
- }
- }
-
- if (beVerbose)
- fprintf(stdout, "Can fit "u64bitFMT" mers into table with prefix of "u64bitFMT" bits, using %8.3fMB (%8.3fMB for positions)\n",
- maxN,
- bestT,
- (((u64bitONE << bestT) * logBaseTwo64(maxN) + maxN * (2*merSize - bestT + posPerMer)) >> 3) / 1048576.0,
- ((maxN * posPerMer) >> 3) / 1048576.0);
-
- return(maxN);
-}
-
-
-
-u32bit
-optimalNumberOfBuckets(u32bit merSize,
- u64bit numMers,
- bool positionsEnabled) {
- u64bit opth = ~u64bitZERO;
- u64bit opts = ~u64bitZERO;
- u64bit h = 0;
- u64bit s = 0;
- u64bit hwidth = logBaseTwo64(numMers);
-
- // Positions consume space too, but only if enabled. Probably
- // doesn't matter here.
- //
- u64bit posPerMer = 0;
- if (positionsEnabled)
- posPerMer = 32;
-
- // Find the table size (in bits, h) that minimizes memory usage
- // for the given merSize and numMers
- //
- // We have two tables:
- // the bucket pointers num buckets * pointer width == 2 << h * hwidth
- // the mer data: num mers * (mersize - hwidth)
- //
- u64bit hmax = 64 - logBaseTwo64(hwidth + numMers * (2 * merSize - h));
- for (h=2; h<=hmax && h<2*merSize; h++) {
- s = (u64bitONE << h) * hwidth + numMers * (2 * merSize - h + posPerMer);
-
- //fprintf(stderr, "optimalNumberOfBuckets()-- h="u64bitFMT" s="u64bitFMT"\n", h, s);
-
- if (s < opts) {
- opth = h;
- opts = s;
- }
- }
-
- return((u32bit)opth);
-}
-
-
-
-void
-estimate(merylArgs *args) {
-
- if (args->inputFile) {
- merStream M(new kMerBuilder(args->merSize, args->merComp),
- new seqStream(args->inputFile),
- true, true);
- speedCounter C(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
-
- if (args->beVerbose)
- fprintf(stderr, "Counting mers in '%s'\n", args->inputFile);
-
- args->numMersEstimated = 0;
-
- while (M.nextMer()) {
- C.tick();
- args->numMersEstimated++;
- }
-
- C.finish();
- }
-
- u32bit opth = optimalNumberOfBuckets(args->merSize, args->numMersEstimated, args->positionsEnabled);
- u64bit memu = ((u64bitONE << opth) * logBaseTwo64(args->numMersEstimated+1) +
- args->numMersEstimated * (2 * args->merSize - opth));
-
- fprintf(stderr, u64bitFMT" "u32bitFMT"-mers can be computed using "u64bitFMT"MB memory.\n",
- args->numMersEstimated, args->merSize, memu >> 23);
-}
diff --git a/kmer/meryl/m-heap.H b/kmer/meryl/m-heap.H
deleted file mode 100644
index 0c44b7f..0000000
--- a/kmer/meryl/m-heap.H
+++ /dev/null
@@ -1,152 +0,0 @@
-#ifndef M_HEAP_H
-#define M_HEAP_H
-
-#include "util++.H"
-#include "bio++.H"
-
-//
-// This is a bit packed heap, derived from bitPackedHeap. It is
-// customized to decode a kmer from a merStream, given the location
-// of the kmer in the stream. This kmer is used for the value of the
-// data item in the heap, instead of the value stored in the heap.
-//
-
-class bitPackedMerHeap {
-public:
- bitPackedMerHeap(seqStore *SS, u32bit width, u64bit size=16) {
- _array = new bitPackedArray(width, size);
- _array->set(0, 0);
- _lastVal = 0;
- _mers = SS;
- };
-
- ~bitPackedMerHeap() {
- delete _array;
- };
-
- // Get the mer with index idx in the merStream
- //
- kMer const &getMer(u64bit idx) {
- _mers->setIterationStart(idx);
- _mers->nextMer();
- if (_mers->theRMer() < _mers->theFMer())
- return(_mers->theRMer());
- return(_mers->theFMer());
- }
-
- u64bit get(kMer &mer) {
- u64bit pos = ~u64bitZERO;
-
- if (_lastVal == 0)
- return(pos);
-
- pos = _array->get(0);
- mer = getMer(pos);
-
- if (--_lastVal == 0)
- return(pos);
-
- // Rebalance the heap
-
- u64bit tval = _array->get(_lastVal);
- kMer tmer;
-
- _array->set(0, tval);
-
- u64bit pidx = 0;
- u64bit pval = tval;
- kMer pmer = getMer(pval);
- u64bit cidx = 1;
- u64bit cval = 0; // set below
- kMer cmer;
-
- while (cidx < _lastVal) {
- // Set cval here, so we can first test if cidx is in range.
- cval = _array->get(cidx);
- cmer = getMer(cval);
-
- // Pick the smallest of the two kids
- if (cidx+1 < _lastVal) {
- tval = _array->get(cidx+1);
- tmer = getMer(tval);
-
- if (cmer > tmer) {
- cidx++;
- cval = tval;
- cmer = tmer;
- }
- }
-
- if (cmer < pmer) {
-
- // Swap p and c
- _array->set(pidx, cval);
- _array->set(cidx, pval);
-
- // Move down the tree -- pval doesn't change, we moved it into cidx!
- pidx = cidx;
-
- cidx = cidx * 2 + 1;
- } else {
- cidx = _lastVal;
- }
- }
-
- return(pos);
- };
-
- void add(u64bit value) {
- u64bit cidx = _lastVal++;
- u64bit cval = value;
- kMer cmer;
- u64bit pidx = 0;
- u64bit pval = 0;
- kMer pmer;
- bool more = true;
-
- _array->set(cidx, cval);
-
- if (cidx == 0)
- return;
-
- cmer = getMer(cval);
-
- while (more) {
- pidx = (cidx-1) / 2;
- pval = _array->get(pidx);
- pmer = getMer(pval);
-
- if (pmer > cmer) {
-
- // Swap p and c
- _array->set(cidx, pval);
- _array->set(pidx, cval);
-
- // Move up the tree -- cval doesn't change, we moved it into pidx!
- cidx = pidx;
- } else {
- more = false;
- }
- if (cidx == 0)
- more = false;
- }
- };
-
- void dump(void) {
- for (u32bit i=0; i<_lastVal; i++)
- fprintf(stderr, "HEAP["u32bitFMT"]="u64bitFMT"\n", i, _array->get(i));
- }
-
- void clear(void) {
- _array->clear();
- _lastVal = 0;
- };
-
-private:
- bitPackedArray *_array;
- u64bit _lastVal;
- seqStore *_mers;
-};
-
-
-#endif // M_HEAP_H
diff --git a/kmer/meryl/m.C b/kmer/meryl/m.C
deleted file mode 100644
index e649e28..0000000
--- a/kmer/meryl/m.C
+++ /dev/null
@@ -1,118 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "util++.H"
-#include "bio++.H"
-#include "meryl.H"
-
-#include "m-heap.H"
-
-int
-main(int argc, char **argv) {
- bool beVerbose = false;
- u64bit merSize = 20;
- u64bit memLimit = 768;
- char *inName = 0L;
- char *outName = 0L;
-
- int arg=1;
- while (arg < argc) {
- if (strncmp(argv[arg], "-verbose", 2) == 0) {
- beVerbose = true;
- } else if (strncmp(argv[arg], "-mersize", 4) == 0) {
- merSize = strtou64bit(argv[++arg], 0L);
- } else if (strncmp(argv[arg], "-memory", 4) == 0) {
- memLimit = strtou64bit(argv[++arg], 0L) * 1024 * 1024;
- } else if (strncmp(argv[arg], "-sequence", 2) == 0) {
- inName = argv[++arg];
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- }
- arg++;
- }
-
- if (inName == 0L) {
- fprintf(stderr, "usage: %s [-v] [-m mersize] [-memory Nmb] [-s seq.fasta]\n", argv[0]);
- exit(1);
- }
-
- outName = new char [strlen(inName) + 1];
- strcpy(outName, inName);
-
- seqStream *seqstr = new seqStream(inName);
- seqStore *seqsto = new seqStore(outName, seqstr);
-
- u64bit memUsed = seqsto->loadStoreInCore();
- u64bit numMers = seqsto->numberOfACGT();
-
-#warning needed exact number of mers here
-
- fprintf(stderr, "Found "u64bitFMT" mers in file of size "u64bitFMT"\n", numMers, memUsed);
-
- if (memUsed > memLimit) {
- fprintf(stderr, "ERROR: two-bit encoded sequence file is bigger than allowed memory usage.\n");
- exit(1);
- }
-
- // Allocate a heap to fill up the rest of space
-
- // Allocate a bitPackedHeap to store N merSize*2 integeers.
- // N = (memLimit - memUsed) * 8 / (merSize * 2)
- //
- // The bitPackedHeap doesn't care about the maximum size, only
- // about the block size.
- //
- u64bit pointerWidth = logBaseTwo64(numMers);
- bitPackedMerHeap *heap = new bitPackedMerHeap(seqsto, pointerWidth, 8 * 1024);
-
- speedCounter *S;
-
- u64bit N = (memLimit - memUsed) * 8 / pointerWidth;
- u64bit M = 0;
-
- fprintf(stderr, "Can store "u64bitFMT" mer pointers of size "u64bitFMT" in the heap.\n", N, pointerWidth);
-
- kMer mer;
-
- if (N > numMers)
- N = numMers;
-
- // Initialize the heap with some numbers
- //
- S = new speedCounter(" Loading heap: %7.2f Mmers -- %8.1f mers/second\r", 1.0, 0x1ffff, beVerbose);
- while (M < N) {
-
-#if 0
- heap->add(M);
- heap->get(mer);
- fprintf(stdout, "ADD "u64bitFMT" -- %s\n", M, mer.merToString(str));
-#endif
-
- heap->add(M++);
- S->tick();
- }
- delete S;
-
- // Until we run out of mers, write things out of the heap.
- //
- S = new speedCounter(" Cycling heap: %7.2f Mmers -- %8.1f Mmers/second\r", 1.0, 0x1fff, beVerbose);
- while (M < numMers) {
- heap->add(M++);
- heap->get(mer);
- //fprintf(stdout, "GOT "u64bitFMT" -- %s\n", M, mer.merToString(str));
- S->tick();
- }
- delete S;
-
- // And finally, flush the heap.
- //
- S = new speedCounter(" Dumping heap: %7.2f Mmers -- %8.1f Mmers/second\r", 1.0, 0x1fff, beVerbose);
- u64bit idx = heap->get(mer);
- while (idx != ~u64bitZERO) {
- //fprintf(stdout, "OUT "u64bitFMT" -- %s\n", idx, mer.merToString(str));
- idx = heap->get(mer);
- S->tick();
- }
- delete S;
-}
diff --git a/kmer/meryl/mapMers-depth.C b/kmer/meryl/mapMers-depth.C
deleted file mode 100644
index a8ba66c..0000000
--- a/kmer/meryl/mapMers-depth.C
+++ /dev/null
@@ -1,131 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "bio++.H"
-#include "seqCache.H"
-#include "merStream.H"
-#include "libmeryl.H"
-#include "existDB.H"
-
-
-int
-main(int argc, char **argv) {
- u32bit merSize = 16;
- char *merylFile = 0L;
- char *fastaFile = 0L;
- bool beVerbose = false;
- u32bit loCount = 0;
- u32bit hiCount = ~u32bitZERO;
- u32bit windowsize = 0;
- u32bit skipsize = 0;
-
- int arg=1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-m") == 0) {
- merSize = strtou32bit(argv[++arg], 0L);
- } else if (strcmp(argv[arg], "-mers") == 0) {
- merylFile = argv[++arg];
- } else if (strcmp(argv[arg], "-seq") == 0) {
- fastaFile = argv[++arg];
- } else if (strcmp(argv[arg], "-v") == 0) {
- beVerbose = true;
- } else if (strcmp(argv[arg], "-lo") == 0) {
- loCount = strtou32bit(argv[++arg], 0L);
- } else if (strcmp(argv[arg], "-hi") == 0) {
- hiCount = strtou32bit(argv[++arg], 0L);
- } else if (strcmp(argv[arg], "-w") == 0) {
- windowsize = strtou32bit(argv[++arg], 0L);
- } else if (strcmp(argv[arg], "-s") == 0) {
- skipsize = strtou32bit(argv[++arg], 0L);
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- }
- arg++;
- }
-
- if ((merylFile == 0L) || (fastaFile == 0L)) {
- fprintf(stderr, "usage: %s -m mersize -mers mers -seq fasta > output\n", argv[0]);
- exit(1);
- }
-
- existDB *E = new existDB(merylFile, merSize, existDBcounts | existDBcompressCounts | existDBcompressBuckets, loCount, hiCount);
- seqCache *F = new seqCache(fastaFile);
-
- for (u32bit Sid=0; Sid < F->getNumberOfSequences(); Sid++) {
- seqInCore *S = F->getSequenceInCore(Sid);
- merStream *MS = new merStream(new kMerBuilder(merSize),
- new seqStream(S->sequence(), S->sequenceLength()),
- true, true);
-
- u32bit idlen = 0;
- intervalDepthRegions *id = new intervalDepthRegions [S->sequenceLength() * 2 + 2];
-
- while (MS->nextMer()) {
- s32bit cnt = (s32bit)E->count(MS->theFMer()) + (s32bit)E->count(MS->theRMer());
-
- id[idlen].pos = MS->thePositionInSequence();
- id[idlen].cha = cnt;
- idlen++;
-
- id[idlen].pos = MS->thePositionInSequence() + merSize;
- id[idlen].cha = -cnt;
- idlen++;
- }
-
- intervalDepth ID(id, idlen);
- u32bit x = 0;
-
- u32bit len = S->sequenceLength();
-
- // Default case, report un-averaged depth at every single location.
- //
- if ((windowsize == 0) && (skipsize == 0)) {
- for (u32bit i=0; i < ID.numberOfIntervals(); i++) {
- for (; x < ID.lo(i); x++)
- fprintf(stdout, u32bitFMTW(7)"\t"u32bitFMTW(6)"\n", x, 0);
- for (; x < ID.hi(i); x++)
- fprintf(stdout, u32bitFMTW(7)"\t"u32bitFMTW(6)"\n", x, ID.de(i));
- }
- for (; x < len; x++)
- fprintf(stdout, u32bitFMTW(7)"\t"u32bitFMTW(6)"\n", x, 0);
-
- } else {
- u32bit *depth = new u32bit [len];
- for (x=0; x < len; x++)
- depth[x] = 0;
-
- for (u32bit i=0; i < ID.numberOfIntervals(); i++)
- for (x=ID.lo(i); x < ID.hi(i); x++)
- depth[x] = ID.de(i);
-
- u32bit avedepth = 0;
-
- for (x=0; x < windowsize; x++)
- avedepth += depth[x];
-
- while (x < len) {
- u32bit avepos = (x - 1) - (windowsize - 1) / 2;
- if ((avepos % skipsize) == 0)
- fprintf(stdout, u32bitFMT"\t%.4f\n",
- avepos,
- (double)avedepth / (double)windowsize);
-
- avedepth = avedepth + depth[x] - depth[x-windowsize];
-
- x++;
- }
-
- delete [] depth;
- }
-
- delete [] id;
-
- delete MS;
- delete S;
- }
-
-
- delete F;
- delete E;
-}
diff --git a/kmer/meryl/mapMers.C b/kmer/meryl/mapMers.C
deleted file mode 100644
index 91f9610..0000000
--- a/kmer/meryl/mapMers.C
+++ /dev/null
@@ -1,210 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "bio++.H"
-#include "seqCache.H"
-#include "merStream.H"
-#include "libmeryl.H"
-#include "existDB.H"
-
-#define OP_NONE 0
-#define OP_STATS 1
-#define OP_REGIONS 2
-#define OP_DETAILS 3
-
-int
-main(int argc, char **argv) {
- u32bit merSize = 16;
- char *merylFile = 0L;
- char *fastaFile = 0L;
- bool beVerbose = false;
- u32bit loCount = 0;
- u32bit hiCount = ~u32bitZERO;
- u32bit operation = OP_NONE;
-
- // For OP_STATS
-
- u32bit Clen = 0;
- u32bit Cmax = 4 * 1024 * 1024;
- u32bit *C = new u32bit [Cmax];
-
- int arg=1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-m") == 0) {
- merSize = strtou32bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-mers") == 0) {
- merylFile = argv[++arg];
-
- } else if (strcmp(argv[arg], "-seq") == 0) {
- fastaFile = argv[++arg];
-
- } else if (strcmp(argv[arg], "-v") == 0) {
- beVerbose = true;
-
- } else if (strcmp(argv[arg], "-lo") == 0) {
- loCount = strtou32bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-hi") == 0) {
- hiCount = strtou32bit(argv[++arg], 0L);
-
- } else if (strcmp(argv[arg], "-stats") == 0) {
- operation = OP_STATS;
-
- } else if (strcmp(argv[arg], "-regions") == 0) {
- operation = OP_REGIONS;
-
- } else if (strcmp(argv[arg], "-details") == 0) {
- operation = OP_DETAILS;
-
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- }
- arg++;
- }
-
- if ((operation == OP_NONE) || (merylFile == 0L) || (fastaFile == 0L)) {
- fprintf(stderr, "usage: %s [-stats | -regions | -details] -m mersize -mers mers -seq fasta > output\n", argv[0]);
- exit(1);
- }
-
-#if 0
- existDB *E = NULL;
-
- if (fileExists("junk.existDB")) {
- fprintf(stderr, "loading from junk.existDB\n");
- E = new existDB("junk.existDB");
- fprintf(stderr, "loaded\n");
- } else {
- exit(1);
- E = new existDB(merylFile, merSize, existDBcounts, loCount, hiCount);
- E->saveState("junk.existDB");
- }
-#endif
-
- existDB *E = new existDB(merylFile, merSize, existDBcounts, loCount, hiCount);
- seqCache *F = new seqCache(fastaFile);
-
- fprintf(stderr, "Begin.\n");
-
-
- for (u32bit Sid=0; Sid < F->getNumberOfSequences(); Sid++) {
- seqInCore *S = F->getSequenceInCore(Sid);
- merStream *MS = new merStream(new kMerBuilder(merSize),
- new seqStream(S->sequence(), S->sequenceLength()),
- true, true);
-
- // with counts, report mean, mode, median, min, max for each frag.
- if (operation == OP_STATS) {
- Clen = 0;
- for (u32bit i=0; i<Cmax; i++)
- C[i] = 0;
-
- while (MS->nextMer()) {
- u64bit cnt = E->count(MS->theFMer()) + E->count(MS->theRMer());
-
- if (cnt > 0)
- C[Clen++] = cnt;
- }
-
- u64bit mean = u64bitZERO;
- u64bit min = ~u64bitZERO;
- u64bit max = u64bitZERO;
- u64bit hist[16] = { 0 };
-
- // Histogram values are powers of two, e.g., <=1, <=2, <=4, <=8, <=16, <=32, <=64, <=128, <=256, <=512, <=1024, <=4096, <=8192, <=328768
-
- for (u32bit i=0; i<Clen; i++) {
- mean += C[i];
-
- if ((min > C[i]) && (C[i] > 1))
- min = C[i];
- if (max < C[i])
- max = C[i];
-
- hist[ logBaseTwo64(C[i]) ]++;
- }
-
- mean /= Clen;
-
- fprintf(stdout,
- "%s\t"
- u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"
- u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"
- u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\n",
- S->header(),
- mean, min, max,
- hist[ 0], hist[ 1], hist[ 2], hist[ 3], hist[ 4], hist[ 5], hist[ 6], hist[ 7],
- hist[ 8], hist[ 9], hist[10], hist[11], hist[12], hist[13], hist[14], hist[15]);
- }
-
-
- // without counts, reports regions with mer coverage.
- // Orientation tells us nothing, since the mers are probably canonical
- if (operation == OP_REGIONS) {
- u64bit beg = ~u64bitZERO;
- u64bit end = ~u64bitZERO;
- u64bit pos = ~u64bitZERO;
-
- u64bit numCovReg = 0;
- u64bit lenCovReg = 0;
-
- while (MS->nextMer()) {
- if (E->exists(MS->theFMer()) || E->exists(MS->theRMer())) {
- pos = MS->thePositionInSequence();
-
- if (beg == ~u64bitZERO)
- beg = end = pos;
-
- if (pos <= end + merSize) {
- end = pos;
- } else {
- fprintf(stdout, "%s\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\n", S->header(), beg, end+merSize, end+merSize - beg);
- numCovReg++;
- lenCovReg += end+merSize - beg;
- beg = end = pos;
- }
- } else {
- fprintf(stdout, "%s\t"u64bitFMT"\tuncovered\n", S->header(), MS->thePositionInSequence());
- }
- }
-
- if (beg != ~u64bitZERO)
- fprintf(stdout, "%s\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\n", S->header(), beg, end+merSize, end+merSize - beg);
-
- fprintf(stderr, "numCovReg: "u64bitFMT"\n", numCovReg);
- fprintf(stderr, "lenCovReg: "u64bitFMT"\n", lenCovReg);
- }
-
-
-
- if (operation == OP_DETAILS) {
- char merString[256];
-
- while (MS->nextMer()) {
- u64bit beg = MS->thePositionInSequence();
- u64bit end = beg + merSize;
- u64bit fnt = E->count(MS->theFMer());
- u64bit rnt = E->count(MS->theRMer());
-
- fprintf(stdout, "%s\t%s\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\t"u64bitFMT"\n",
- S->header(),
- MS->theFMer().merToString(merString),
- beg,
- end,
- fnt,
- rnt,
- fnt + rnt);
- }
- }
-
-
- delete MS;
- delete S;
- }
-
-
- delete F;
- delete E;
-}
diff --git a/kmer/meryl/maskMers.C b/kmer/meryl/maskMers.C
deleted file mode 100644
index 2aa8954..0000000
--- a/kmer/meryl/maskMers.C
+++ /dev/null
@@ -1,591 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-
-#include "bio++.H"
-#include "seqStream.H"
-#include "libmeryl.H"
-
-#include <algorithm>
-
-#define MAX_COVERAGE 51
-
-class mateRescueData {
-public:
- mateRescueData() {
- _mean = 0;
- _stddev = 0;
- _coverage = 0;
- _normal = 0L;
- _normalZero = 0;
- };
-
- void init(s32bit mean_, s32bit stddev_, u32bit coverage_) {
- _mean = mean_;
- _stddev = stddev_;
- _coverage = coverage_;
-
- assert(_mean > 3 * _stddev);
-
- double a = 1.0 / (_stddev * sqrt(2 * M_PI));
- double c = 2 * _stddev * _stddev;
-
- s32bit b1l = (s32bit)floor(-3 * _stddev);
- s32bit b1h = (s32bit)ceil ( 3 * _stddev);
-
- _normal = new double [b1h - b1l + 1];
- _normalZero = -b1l;
-
- for (s32bit l=0; l<b1h - b1l + 1; l++)
- _normal[l] = 0.0;
-
- for (s32bit l=b1l; l<b1h; l++)
- _normal[l + _normalZero] = a * exp(- l*l / c);
- };
- ~mateRescueData() {
- };
-
- s32bit mean(void) { return(_mean); };
- s32bit stddev(void) { return(_stddev); };
- u32bit coverage(void) { return(_coverage); };
-
- double normal(s32bit p) { return(_normal[p + _normalZero]); };
-
-private:
- s32bit _mean;
- s32bit _stddev;
- u32bit _coverage;
-
- double *_normal;
- s32bit _normalZero;
-};
-
-
-class merMaskedSequence {
-public:
- merMaskedSequence(char *fastaName_, char *merylName_, u32bit onlySeqIID_=~u32bitZERO) {
- _numSeq = 0;
- _seqLen = 0L;
- _masking = 0L;
- _repeatID = 0L;
- _merSize = 0;
-
- strcpy(_fastaName, fastaName_);
- strcpy(_merylName, merylName_);
-
- strcpy(_maskMersName, _merylName);
- strcat(_maskMersName, ".maskMers");
-
- if (fileExists(_maskMersName))
- loadMasking(onlySeqIID_);
- else
- buildMasking();
- };
- ~merMaskedSequence() {
- delete [] _seqLen;
- for (u32bit i=0; i<_numSeq; i++) {
- delete [] _masking[i];
- delete [] _repeatID[i];
- }
- delete [] _masking;
- delete [] _repeatID;
- };
-
-public:
- u32bit numSeq(void) { return(_numSeq); };
- s32bit seqLen(u32bit i) { return(_seqLen[i]); };
- char masking(u32bit s, u32bit p) { return(_masking[s][p]); };
- u32bit repeatID(u32bit s, u32bit p) { return(_repeatID[s][p]); };
-
- u32bit merSize(void) { return(_merSize); };
-
-private:
- void loadMasking(u32bit onlySeqIID_=~u32bitZERO); // Read the masking from the saved file
- void saveMasking(void); // Write the masking to a file
- void buildMasking(void); // Read the mers to build the masking
-
- u32bit _numSeq;
- s32bit *_seqLen; // signed just for convenience later (positions are signed for same reason)
- char **_masking;
- u32bit **_repeatID;
-
- u32bit _merSize;
-
- char _fastaName[FILENAME_MAX];
- char _merylName[FILENAME_MAX];
- char _maskMersName[FILENAME_MAX];
-};
-
-
-void
-merMaskedSequence::loadMasking(u32bit onlySeqIID_) {
- FILE *maskMersFile = fopen(_maskMersName, "r");
-
- fread(&_numSeq, sizeof(u32bit), 1, maskMersFile);
- fread(&_merSize, sizeof(u32bit), 1, maskMersFile);
-
- _seqLen = new s32bit [_numSeq];
- _masking = new char * [_numSeq];
- _repeatID = new u32bit * [_numSeq];
-
- fprintf(stderr, u32bitFMT" sequences in '%s'\n", _numSeq, _fastaName);
-
- fread( _seqLen, sizeof(u32bit), _numSeq, maskMersFile);
-
- for (u32bit i=0; i<_numSeq; i++) {
- _masking[i] = 0L;
- _repeatID[i] = 0L;
-
- if ((onlySeqIID_ >= _numSeq) || (onlySeqIID_ == i)) {
- fprintf(stderr, "Loading sequence "u32bitFMT" of length "u32bitFMT"\n", i, _seqLen[i]);
-
- _masking[i] = new char [_seqLen[i]];
- _repeatID[i] = new u32bit [_seqLen[i]];
-
- //memset(_masking[i], 'g', sizeof(char) * _seqLen[i]);
- //memset(_repeatID[i], 0, sizeof(u32bit) * _seqLen[i]);
-
- fread(_masking[i], sizeof(char), _seqLen[i], maskMersFile);
- fread(_repeatID[i], sizeof(u32bit), _seqLen[i], maskMersFile);
- } else {
- fseek(maskMersFile, sizeof(char) * _seqLen[i], SEEK_CUR);
- fseek(maskMersFile, sizeof(u32bit) * _seqLen[i], SEEK_CUR);
- _seqLen[i] = 0;
- }
- }
-
- fclose(maskMersFile);
-}
-
-
-void
-merMaskedSequence::saveMasking(void) {
- FILE *maskMersFile = fopen(_maskMersName, "w");
-
- fwrite(&_numSeq, sizeof(u32bit), 1, maskMersFile);
- fwrite(&_merSize, sizeof(u32bit), 1, maskMersFile);
- fwrite( _seqLen, sizeof(u32bit), _numSeq, maskMersFile);
-
- for (u32bit i=0; i<_numSeq; i++) {
- fwrite(_masking[i], sizeof(char), _seqLen[i], maskMersFile);
- fwrite(_repeatID[i], sizeof(u32bit), _seqLen[i], maskMersFile);
- }
-
- fclose(maskMersFile);
-}
-
-
-void
-merMaskedSequence::buildMasking(void) {
- seqStream *STR = new seqStream(_fastaName);
-
- _numSeq = STR->numberOfSequences();
-
- _seqLen = new s32bit [_numSeq];
- _masking = new char * [_numSeq];
- _repeatID = new u32bit * [_numSeq];
-
- _merSize = 0;
-
- fprintf(stderr, u32bitFMT" sequences in '%s'\n", _numSeq, _fastaName);
-
- for (u32bit i=0; i<_numSeq; i++) {
- _seqLen[i] = STR->lengthOf(i);
-
- _masking[i] = new char [_seqLen[i]];
- _repeatID[i] = new u32bit [_seqLen[i]];
-
- memset(_masking[i], 'g', sizeof(char) * _seqLen[i]);
- memset(_repeatID[i], 0, sizeof(u32bit) * _seqLen[i]);
- }
-
- // g -> gap in sequence
- // u -> unique mer
- // r -> repeat mer
- //
- // For all the r's we also need to remember the other locations
- // that repeat is at. We annotate the map with a repeat id, set if
- // another copy of the repeat is nearby.
-
- merylStreamReader *MS = new merylStreamReader(_merylName);
- speedCounter *CT = new speedCounter(" Masking mers in sequence: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
-
- u32bit rid = 0;
-
- _merSize = MS->merSize();
-
- while (MS->nextMer()) {
- //fprintf(stderr, "mer count="u64bitFMT" pos="u32bitFMT"\n", MS->theCount(), MS->getPosition(0));
-
- if (MS->theCount() == 1) {
- u32bit p = MS->getPosition(0);
- u32bit s = STR->sequenceNumberOfPosition(p);
- p -= STR->startOf(s);
-
- _masking[s][p] = 'u';
- } else {
- std::sort(MS->thePositions(), MS->thePositions() + MS->theCount());
-
- u32bit lastS = ~u32bitZERO;
- u32bit lastP = 0;
-
- rid++;
-
- for (u32bit i=0; i<MS->theCount(); i++) {
- u32bit p = MS->getPosition(i);
- u32bit s = STR->sequenceNumberOfPosition(p);
- p -= STR->startOf(s);
-
- // Always set the masking.
- _masking[s][p] = 'r';
-
- // If there is a repeat close by, set the repeat ID.
- if ((s == lastS) && (lastP + 40000 > p)) {
- _repeatID[s][lastP] = rid;
- _repeatID[s][p] = rid;
- }
-
- lastS = s;
- lastP = p;
- }
- }
-
- CT->tick();
- }
-
- delete CT;
-
- delete MS;
-
- delete STR;
-
- saveMasking();
-}
-
-
-void
-computeDensity(merMaskedSequence *S, char *outputPrefix) {
- char outputName[FILENAME_MAX];
- FILE *outputFile;
- u32bit windowSizeMax = 10000;
-
- for (u32bit s=0; s<S->numSeq(); s++) {
-
- // seqLen == 0 iff that sequence is not loaded.
- if (S->seqLen(s) == 0)
- continue;
-
- sprintf(outputName, "%s.density.seq"u32bitFMTW(02), outputPrefix, s);
- outputFile = fopen(outputName, "w");
-
- fprintf(stderr, "Starting '%s'\n", outputName);
-
- fprintf(outputFile, "#window\tunique\trepeat\tgaps\n");
-
- // Not the most efficient, but good enough for us right now.
-
- for (s32bit p=0; p<S->seqLen(s); ) {
- u32bit windowSize = 0;
- u32bit uniqueSum = 0;
- u32bit repeatSum = 0;
- u32bit gapSum = 0;
-
- while ((windowSize < windowSizeMax) &&
- (p < S->seqLen(s))) {
- char m = S->masking(s, p);
-
- if (m == 'u') uniqueSum++;
- if (m == 'g') gapSum++;
- if (m == 'r') repeatSum++;
-
- windowSize++;
- p++;
- }
-
- fprintf(outputFile, u32bitFMT"\t%f\t%f\t%f\n",
- p - windowSize,
- (double)uniqueSum / windowSize,
- (double)repeatSum / windowSize,
- (double)gapSum / windowSize);
- }
-
- fclose(outputFile);
- }
-}
-
-
-// For each 'r' mer, compute the number of 'u' mers
-// that are within some mean +- stddev range.
-//
-// We count for two blocks:
-//
-// | <- mean -> | <- mean -> |
-// ---[block1]---------------mer---------------[block2]---
-//
-// Once we know that, we can compute the probability that
-// a repeat mer can be rescued.
-//
-// p1 = uniq/total -- for 1 X coverage
-// pn = 1 - (1-p1)^n -- for n X coverage
-
-
-void
-computeMateRescue(merMaskedSequence *S, char *outputPrefix, mateRescueData *lib, u32bit libLen) {
- char outputName[FILENAME_MAX];
- FILE *outputFile;
- FILE *outputData;
-
- u32bit closeRepeatsLen = 0;
- u32bit closeRepeatsMax = 80000;
- s32bit *closeRepeats = new s32bit [closeRepeatsMax];
-
- speedCounter *CT = new speedCounter(" Examining repeats: %7.2f Kbases -- %5.2f Kbases/second\r", 1000.0, 0x1ffff, true);
-
- u32bit totalDepth = 0;
- for (u32bit l=0; l<libLen; l++)
- totalDepth += lib[l].coverage();
-
- for (u32bit s=0; s<S->numSeq(); s++) {
-
- // seqLen == 0 iff that sequence is not loaded.
- if (S->seqLen(s) == 0)
- continue;
-
- fprintf(stderr, "Starting sequence "u32bitFMT"\n", s);
-
- sprintf(outputName, "%s.mateRescue.seq"u32bitFMTW(02)".out", outputPrefix, s);
- outputFile = fopen(outputName, "w");
-
- sprintf(outputName, "%s.mateRescue.seq"u32bitFMTW(02)".dat", outputPrefix, s);
- outputData = fopen(outputName, "w");
-
- double numRR[MAX_COVERAGE] = {0}; // num repeats rescued (expected) for [] X coverage
- double numNR[MAX_COVERAGE] = {0}; // num repeats nonrescuable (expected) for [] X coverage
-
- u32bit numRT = 0; // num repeats total
-
- for (s32bit p=0; p<S->seqLen(s); p++) {
- CT->tick();
-
- double pRtot = 0.0;
- double pFtot = 0.0;
-
- if ((S->masking(s, p) != 'g') &&
- (S->masking(s, p) != 'u') &&
- (S->masking(s, p) != 'r'))
- fprintf(stderr, "INVALID MASKING - got %d = %c\n", S->masking(s, p), S->masking(s, p));
-
-
- if (S->masking(s, p) == 'r') {
- numRT++;
-
- // Index over x-coverage in libraries. MUST BE 1.
- u32bit ridx = 1;
-
- for (u32bit l=0; l<libLen; l++) {
- s32bit mean = lib[l].mean();
- s32bit stddev = lib[l].stddev();
-
- // Build a list of the same repeat close to this guy.
- closeRepeatsLen = 0;
-
- if (S->repeatID(s, p) > 0) {
- s32bit pl = (s32bit)floor(p - 3 * stddev);
- s32bit ph = (s32bit)ceil (p + 3 * stddev);
-
- if (pl < 0) pl = 0;
- if (ph > S->seqLen(s)) ph = S->seqLen(s);
-
- for (s32bit pi=pl; pi<ph; pi++)
- if ((S->repeatID(s, pi) == S->repeatID(s, p)) && (pi != p))
- closeRepeats[closeRepeatsLen++] = pi;
- }
-
-
- s32bit b1l = (s32bit)floor(p - mean - 3 * stddev);
- s32bit b1h = (s32bit)ceil (p - mean + 3 * stddev);
-
- s32bit b2l = (s32bit)floor(p + mean - 3 * stddev);
- s32bit b2h = (s32bit)ceil (p + mean + 3 * stddev);
-
- if (b1l < 0) b1l = 0;
- if (b1h < 0) b1h = 0;
- if (b1h > S->seqLen(s)) b1h = S->seqLen(s);
-
- if (b2l < 0) b2l = 0;
- if (b2h > S->seqLen(s)) b2h = S->seqLen(s);
- if (b2l > S->seqLen(s)) b2l = S->seqLen(s);
-
- //fprintf(stderr, "b1: %d-%d b2:%d-%d\n", b1l, b1h, b2l, b2h);
-
- // probability we can rescue this repeat with this mate pair
- double pRescue = 0.0;
- double pFailed = 0.0;
-
- if (closeRepeatsLen == 0) {
- // No close repeats, use the fast method.
- for (s32bit b=b1l; b<b1h; b++) {
- if (S->masking(s, b) == 'u')
- pRescue += lib[l].normal(b - p + mean);
- }
-
- for (s32bit b=b2l; b<b2h; b++) {
- if (S->masking(s, b) == 'u')
- pRescue += lib[l].normal(b - p - mean);
- }
- } else {
- // Close repeats, gotta be slow.
- for (s32bit b=b1l; b<b1h; b++) {
- if (S->masking(s, b) == 'u') {
- s32bit mrl = b + mean - 3 * stddev;
- s32bit mrh = b + mean + 3 * stddev;
-
- bool rescuable = true;
-
- for (u32bit cri=0; rescuable && cri<closeRepeatsLen; cri++)
- if ((mrl <= closeRepeats[cri]) && (closeRepeats[cri] <= mrh))
- rescuable = false;
-
- if (rescuable)
- pRescue += lib[l].normal(b - p + mean);
- else
- pFailed += lib[l].normal(b - p + mean);
- }
- }
-
- for (s32bit b=b2l; b<b2h; b++) {
- if (S->masking(s, b) == 'u') {
- s32bit mrl = b - mean - 3 * stddev;
- s32bit mrh = b - mean + 3 * stddev;
-
- bool rescuable = true;
-
- for (u32bit cri=0; rescuable && cri<closeRepeatsLen; cri++)
- if ((mrl <= closeRepeats[cri]) && (closeRepeats[cri] <= mrh))
- rescuable = false;
-
- if (rescuable)
- pRescue += lib[l].normal(b - p - mean);
- else
- pFailed += lib[l].normal(b - p - mean);
- }
- }
- }
-
- // We're summing over two distributions.
- pRescue /= 2.0;
- pFailed /= 2.0;
-
- // Compute probability of rescuing with libraries we've
- // seen already, and the expected number of repeats
- // rescued.
- //
- // We keep track of the probability we rescue this repeat
- // with additional coverage of libraries. First 1x of the
- // first lib, then 2x of the first, etc, etc.
- //
- {
- double pR = 1.0;
- double pF = 1.0;
- for (u32bit x=0; x<lib[l].coverage(); x++) {
- // Makes it here. pRescue != 1.0
- pR *= (1.0 - pRescue);
- numRR[ridx] += 1 - pR;
- pRtot += 1 - pR;
-
- pF *= (1.0 - pFailed);
- numNR[ridx] += 1 - pF;
- pFtot += 1 - pF;
-
- ridx++;
- }
- }
- } // over all libs
-
- fprintf(outputData, s32bitFMT"\t%f\t%f\n", p, pRtot / totalDepth, pFtot / totalDepth);
-
- } // if masking is r
- } // over all positions
-
- fprintf(outputFile, "seqIID\tmerSize\ttRepeat\teRescue\teFailed\tXcov\tmean\tstddev\n");
-
- for (u32bit x=1, l=0, n=0; l<libLen; x++) {
- fprintf(outputFile, u32bitFMT"\t"u32bitFMT"\t"u32bitFMT"\t%.0f\t%.0f\t"u32bitFMT"\t"s32bitFMT"\t"s32bitFMT"\n",
- s, S->merSize(), numRT, numRR[x], numNR[x], x, lib[l].mean(), lib[l].stddev());
- n++;
- if (n >= lib[l].coverage()) {
- l++;
- n = 0;
- }
- }
-
- fclose(outputFile);
- fclose(outputData);
- }
-
- delete CT;
-}
-
-
-
-int
-main(int argc, char **argv) {
- char *merylName = 0L;
- char *fastaName = 0L;
- char *outputPrefix = 0L;
-
- u32bit onlySeqIID = ~u32bitZERO;
-
- bool doDensity = false;
- bool doRescue = false;
-
- mateRescueData lib[MAX_COVERAGE];
- u32bit libLen = 0;
-
- int arg=1;
- int err=0;
- while (arg < argc) {
- if (strcmp(argv[arg], "-mers") == 0) {
- merylName = argv[++arg];
-
- } else if (strcmp(argv[arg], "-seq") == 0) {
- fastaName = argv[++arg];
-
- } else if (strcmp(argv[arg], "-only") == 0) {
- onlySeqIID = atoi(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-output") == 0) {
- outputPrefix = argv[++arg];
-
- } else if (strcmp(argv[arg], "-d") == 0) {
- doDensity = true;
-
- } else if (strcmp(argv[arg], "-r") == 0) {
- if (atoi(argv[arg+3]) > 0) {
- doRescue = true;
- lib[libLen++].init(atoi(argv[arg+1]), atoi(argv[arg+2]), atoi(argv[arg+3]));
- }
- arg += 3;
-
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- err++;
- }
- arg++;
- }
- if ((err) || (merylName == 0L) || (fastaName == 0L) || (outputPrefix == 0L)) {
- fprintf(stderr, "usage: %s -mers mers -seq fasta -output prefix [-d] [-r mean stddev coverage]\n", argv[0]);
- exit(1);
- }
-
- merMaskedSequence *S = new merMaskedSequence(fastaName, merylName, onlySeqIID);
-
- if (doDensity)
- computeDensity(S, outputPrefix);
-
- if (doRescue)
- computeMateRescue(S, outputPrefix, lib, libLen);
-
- return(0);
-}
diff --git a/kmer/meryl/merge.C b/kmer/meryl/merge.C
deleted file mode 100644
index 0a90cb8..0000000
--- a/kmer/meryl/merge.C
+++ /dev/null
@@ -1,240 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "meryl.H"
-#include "libmeryl.H"
-
-
-
-void
-multipleOperations(merylArgs *args) {
-
- if (args->mergeFilesLen < 2) {
- fprintf(stderr, "ERROR - must have at least two databases (you gave "u32bitFMT")!\n", args->mergeFilesLen);
- exit(1);
- }
- if (args->outputFile == 0L) {
- fprintf(stderr, "ERROR - no output file specified.\n");
- exit(1);
- }
- if ((args->personality != PERSONALITY_MERGE) &&
- (args->personality != PERSONALITY_MIN) &&
- (args->personality != PERSONALITY_MINEXIST) &&
- (args->personality != PERSONALITY_MAX) &&
- (args->personality != PERSONALITY_MAXEXIST) &&
- (args->personality != PERSONALITY_ADD) &&
- (args->personality != PERSONALITY_AND) &&
- (args->personality != PERSONALITY_NAND) &&
- (args->personality != PERSONALITY_OR) &&
- (args->personality != PERSONALITY_XOR)) {
- fprintf(stderr, "ERROR - only personalities min, minexist, max, maxexist, add, and, nand, or, xor\n");
- fprintf(stderr, "ERROR - are supported in multipleOperations(). (%d)\n", args->personality);
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- }
-
- merylStreamReader **R = new merylStreamReader* [args->mergeFilesLen];
- merylStreamWriter *W = 0L;
-
- // Open the input files, read in the first mer
- //
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- R[i] = new merylStreamReader(args->mergeFiles[i]);
- R[i]->nextMer();
- }
-
- // Verify that the mersizes are all the same
- //
- bool fail = false;
- u32bit merSize = R[0]->merSize();
- u32bit merComp = R[0]->merCompression();
-
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- fail |= (merSize != R[i]->merSize());
- fail |= (merComp != R[i]->merCompression());
- }
-
- if (fail)
- fprintf(stderr, "ERROR: mer sizes (or compression level) differ.\n"), exit(1);
-
- // Open the output file, using the largest prefix size found in the
- // input/mask files.
- //
- u32bit prefixSize = 0;
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- if (prefixSize < R[i]->prefixSize())
- prefixSize = R[i]->prefixSize();
-
- W = new merylStreamWriter(args->outputFile, merSize, merComp, prefixSize, args->positionsEnabled);
-
- // We will find the smallest mer in any file, and count the number of times
- // it is present in the input files.
-
- bool moreInput = true;
-
- kMer currentMer; // The current mer we're operating on
- u32bit currentCount = u32bitZERO; // The count (operation dependent) of this mer
- u32bit currentTimes = u32bitZERO; // Number of files it's in
-
- u32bit currentPositionsMax = 0;
- u32bit *currentPositions = 0L;
-
- kMer thisMer; // The mer we just read
- u32bit thisFile = ~u32bitZERO; // The file we read it from
- u32bit thisCount = u32bitZERO; // The count of the mer we just read
-
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
-
- currentMer.setMerSize(merSize);
- thisMer.setMerSize(merSize);
-
- while (moreInput) {
-
- // Find the smallest mer present in any input file.
- //
- moreInput = false;
- thisMer.clear();
- thisFile = ~u32bitZERO;
- thisCount = u32bitZERO;
-
- // Load thisMer with the first valid mer
- for (u32bit i=0; i<args->mergeFilesLen && !moreInput; i++)
- if (R[i]->validMer()) {
- moreInput = true;
- thisCount = R[i]->theCount();
- thisFile = i;
- thisMer = R[i]->theFMer();
- }
-
- // Now find the smallest one
- if (moreInput) {
- for (u32bit i=thisFile+1; i<args->mergeFilesLen; i++)
- if ((R[i]->validMer()) && (R[i]->theFMer()) < thisMer) {
- moreInput = true;
- thisCount = R[i]->theCount();
- thisFile = i;
- thisMer = R[i]->theFMer();
- }
- }
-
- // If we've hit a different mer, write out the last one
- if ((moreInput == false) || (thisMer != currentMer)) {
- switch (args->personality) {
- case PERSONALITY_MIN:
- case PERSONALITY_MAX:
- if (currentTimes == args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_MERGE:
- case PERSONALITY_MINEXIST:
- case PERSONALITY_MAXEXIST:
- case PERSONALITY_ADD:
- W->addMer(currentMer, currentCount, currentPositions);
- break;
- case PERSONALITY_AND:
- if (currentTimes == args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_NAND:
- if (currentTimes != args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_OR:
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_XOR:
- if ((currentTimes % 2) == 1)
- W->addMer(currentMer, currentCount);
- break;
- default:
- fprintf(stderr, "ERROR - invalid personality in multipleOperations::write\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- break;
- }
-
- currentMer = thisMer;
-
- currentCount = u32bitZERO;
- currentTimes = u32bitZERO;
-
- C->tick();
- }
-
- // All done? Exit.
- if (moreInput == false)
- continue;
-
- // Perform the operation
- switch (args->personality) {
- case PERSONALITY_MERGE:
- if (R[thisFile]->thePositions()) {
- if (currentPositionsMax == 0) {
- currentPositionsMax = 1048576;
- currentPositions = new u32bit [currentPositionsMax];
- }
-
- if (currentPositionsMax < currentCount + thisCount) {
- while (currentPositionsMax < currentCount + thisCount)
- currentPositionsMax *= 2;
-
- u32bit *t = new u32bit [currentPositionsMax];
- memcpy(t, currentPositions, sizeof(u32bit) * currentCount);
- delete [] currentPositions;
- currentPositions = t;
- }
-
- if (thisCount < 16) {
- u32bit *p = R[thisFile]->thePositions();
- for (u32bit i=0; i<thisCount; i++)
- currentPositions[currentCount + i] = p[i];
- } else {
- memcpy(currentPositions + currentCount, R[thisFile]->thePositions(), sizeof(u32bit) * thisCount);
- }
- }
- // Otherwise, we're the same as ADD.
- currentCount += thisCount;
- break;
- case PERSONALITY_MIN:
- case PERSONALITY_MINEXIST:
- if (currentTimes == 0) {
- currentCount = thisCount;
- } else {
- if (currentCount > thisCount)
- currentCount = thisCount;
- }
- break;
- case PERSONALITY_MAX:
- case PERSONALITY_MAXEXIST:
- if (currentCount < thisCount)
- currentCount = thisCount;
- break;
- case PERSONALITY_ADD:
- currentCount += thisCount;
- break;
- case PERSONALITY_AND:
- case PERSONALITY_NAND:
- case PERSONALITY_OR:
- case PERSONALITY_XOR:
- currentCount = 1;
- break;
- default:
- fprintf(stderr, "ERROR - invalid personality in multipleOperations::operate\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- break;
- }
-
- currentTimes++;
-
- // Move the file we just read from to the next mer
- R[thisFile]->nextMer();
- }
-
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- delete R[i];
- delete R;
- delete W;
- delete C;
-}
diff --git a/kmer/meryl/merge.listmerge.C b/kmer/meryl/merge.listmerge.C
deleted file mode 100644
index d0fbaa2..0000000
--- a/kmer/meryl/merge.listmerge.C
+++ /dev/null
@@ -1,447 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "meryl.H"
-#include "libmeryl.H"
-
-
-using namespace std;
-
-#include <algorithm>
-
-struct mMer {
- kMer _mer;
- u32bit _cnt;
- u32bit _off;
- u32bit _nxt;
- u32bit _stp;
-};
-
-
-class mMerList {
-public:
- mMerList(u32bit maxSize) {
- _posLen = 0;
- _posMax = 2 * maxSize;
- _pos = new u32bit [_posMax];
-
- _mmmLen = 0;
- _mmmMax = maxSize;
- _mmm = new mMer [_mmmMax];
-
- _tip = ~u32bitZERO;
- _fre = 0;
-
- for (u32bit i=0; i<_mmmMax; i++) {
- _mmm[i]._cnt = 0;
- _mmm[i]._off = 0;
- _mmm[i]._nxt = i+1;
- _mmm[i]._stp = 0;
- }
-
- _mmm[_mmmMax-1]._nxt = ~u32bitZERO;
- };
- ~mMerList() {
- delete [] _pos;
- delete [] _mmm;
- };
-
- bool loadMore(void) { return((_mmmMax < _tip) || (_mmm[_tip]._stp == 1)); };
-
- u32bit length(void) { return(_mmmLen); };
-
- kMer *pop(u32bit &cnt, u32bit* &pos) {
- kMer *ret = 0L;
-
- //fprintf(stderr, "POP tip="u32bitFMT"\n", _tip);
-
- if (_tip < _mmmMax) {
- u32bit f = _tip;
-
- ret = &_mmm[f]._mer;
- cnt = _mmm[f]._cnt;
- pos = (_mmm[f]._off != ~u32bitZERO) ? _pos + _mmm[f]._off : 0L;
-
- // Move tip to the next thing
- _tip = _mmm[f]._nxt;
-
- // And append this one to the free list.
- _mmm[f]._nxt = _fre;
- _fre = f;
-
- _mmmLen--;
-
- //fprintf(stderr, "POP f="u32bitFMT" tip="u32bitFMT" len="u32bitFMT"\n", f, _tip, _mmmLen);
- }
-
- return(ret);
- };
-
-
- // rebuild the position list, squeezes out empty items
- void rebuild(void) {
- if (_posLen > 0) {
- assert(0);
- u32bit *np = new u32bit [_posMax];
-
- _posLen = 0;
-
- for (u32bit i=0; i<_mmmLen; i++) {
- mMer *m = _mmm + i;
-
- if (m->_off != ~u32bitZERO) {
- _mmm[_mmmLen]._off = _posLen;
-
- for (u32bit p=0; p<m->_cnt; p++, _posLen++)
- np[_posLen] = _pos[p];
- }
- }
-
- delete [] _pos;
- _pos = np;
- }
- };
-
-
-
- // Read more mers from the file
- void read(merylStreamReader *R, u32bit num, bool loadAll) {
- u32bit xxx = 0;
- u32bit las = ~u32bitZERO;
- u32bit pos = _tip;
- bool stop = false;
-
- //fprintf(stderr, "read()- loading "u32bitFMT"\n", num);
-
- assert(_mmmLen + num < _mmmMax);
-
- // Load until we hit the sentinal.
- if (loadAll == false)
- num = ~u32bitZERO;
-
- for (xxx=0; (xxx < num) && (stop == false) && (R->nextMer()); xxx++) {
-
- // Insert into a free node
- u32bit fre = _fre;
- _fre = _mmm[fre]._nxt;
-
- _mmm[fre]._mer = R->theFMer();
- _mmm[fre]._cnt = R->theCount();
- _mmm[fre]._off = ~u32bitZERO;
- _mmm[fre]._stp = 0;
-
- u32bit *ppp = R->thePositions();
- if (ppp) {
- _mmm[fre]._off = _posLen;
-
- if (_posMax <= _posLen + _mmm[fre]._cnt) {
- fprintf(stderr, "Reallocate _pos\n");
- _posMax *= 2;
- u32bit *tmp = new u32bit [_posMax];
- memcpy(tmp, _pos, sizeof(u32bit) * _posLen);
- delete [] _pos;
- _pos = tmp;
- }
-
- for (u32bit i=0; i<_mmm[fre]._cnt; i++, _posLen++)
- _pos[_posLen] = ppp[i];
- }
-
- // Keep count
- _mmmLen++;
-
- // Figure out where to put it in the list. New duplicates must
- // go AFTER the existing -- that's the job of <=.
-
- while ((pos < _mmmMax) && (_mmm[pos]._mer <= R->theFMer())) {
- las = pos;
- pos = _mmm[pos]._nxt;
- }
-
- if (_mmmMax < _tip) {
- // No tip, make new list.
- _mmm[fre]._nxt = _tip;
- _tip = fre;
- las = ~u32bitZERO;
- pos = _tip;
- } else if (_mmmMax < las) {
- // Valid list, but we want to insert before the start
- _mmm[fre]._nxt = _tip;
- _tip = fre;
- las = ~u32bitZERO;
- pos = _tip;
- } else if (pos < _mmmMax) {
- // Valid pos, insert in the middle (after las, before pos)
- _mmm[fre]._nxt = _mmm[las]._nxt;
- _mmm[las]._nxt = fre;
- las = fre;
- //pos = _mmm[las]._nxt;
- } else {
- // Have a list, but we ran off the end, append (after las)
- _mmm[fre]._nxt = ~u32bitZERO;
- _mmm[las]._nxt = fre;
- pos = fre;
-
- if (loadAll == false)
- stop = true;
- }
- }
-
- // Set the sentinal. This forces us to load more mers.
- //
- if (loadAll == true) {
- //fprintf(stderr, "read()-- stop on tip = "u32bitFMT"\n", las);
- _mmm[las]._stp = 1;
- }
-
- //fprintf(stderr, "read()-- now up to "u32bitFMT" mers ("u32bitFMT" pos); loaded "u32bitFMT" out of "u32bitFMT" requested.\n", _mmmLen, _posLen, xxx, num);
- };
-
-private:
- u32bit _posLen;
- u32bit _posMax;
- u32bit *_pos;
-
- u32bit _mmmLen;
- u32bit _mmmMax;
- mMer *_mmm;
-
- u32bit _tip;
- u32bit _fre;
-};
-
-
-
-
-void
-multipleOperations(merylArgs *args) {
-
- if (args->mergeFilesLen < 2) {
- fprintf(stderr, "ERROR - must have at least two databases (you gave "u32bitFMT")!\n", args->mergeFilesLen);
- exit(1);
- }
- if (args->outputFile == 0L) {
- fprintf(stderr, "ERROR - no output file specified.\n");
- exit(1);
- }
- if ((args->personality != PERSONALITY_MERGE) &&
- (args->personality != PERSONALITY_MIN) &&
- (args->personality != PERSONALITY_MINEXIST) &&
- (args->personality != PERSONALITY_MAX) &&
- (args->personality != PERSONALITY_ADD) &&
- (args->personality != PERSONALITY_AND) &&
- (args->personality != PERSONALITY_NAND) &&
- (args->personality != PERSONALITY_OR) &&
- (args->personality != PERSONALITY_XOR)) {
- fprintf(stderr, "ERROR - only personalities min, minexist, max, add, and, nand, or, xor\n");
- fprintf(stderr, "ERROR - are supported in multipleOperations().\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- }
-
- u32bit maxSize = 64 * 1024 * 1024;
-
- merylStreamReader **R = new merylStreamReader* [args->mergeFilesLen];
- merylStreamWriter *W = 0L;
- mMerList *M = new mMerList(maxSize + maxSize / 4);
-
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- R[i] = new merylStreamReader(args->mergeFiles[i]);
-
- // Verify that the mersizes are all the same
- //
- bool fail = false;
- u32bit merSize = R[0]->merSize();
- u32bit merComp = R[0]->merCompression();
-
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- fail |= (merSize != R[i]->merSize());
- fail |= (merComp != R[i]->merCompression());
- }
-
- if (fail)
- fprintf(stderr, "ERROR: mer size or compression level differ.\n"), exit(1);
-
- // Open the output file, using the largest prefix size found in the
- // input/mask files.
- //
- u32bit prefixSize = 0;
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- if (prefixSize < R[i]->prefixSize())
- prefixSize = R[i]->prefixSize();
-
- W = new merylStreamWriter(args->outputFile, merSize, merComp, prefixSize);
-
- // Load mers from all files, remember the largest mer we load.
- //
- bool loadAll = true;
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- M->read(R[i], maxSize / args->mergeFilesLen, loadAll);
- loadAll = false;
- }
-
- fprintf(stderr, "Initial load: length="u32bitFMT"\n", M->length());
-
- bool moreStuff = true;
-
- kMer currentMer; // The current mer we're operating on
- u32bit currentCount = u32bitZERO; // The count (operation dependent) of this mer
- u32bit currentTimes = u32bitZERO; // Number of files it's in
-
- u32bit currentPositionsMax = 0;
- u32bit *currentPositions = 0L;
-
- kMer *thisMer; // The mer we just read
- u32bit thisCount = u32bitZERO; // The count of the mer we just read
- u32bit *thisPositions = 0L;
-
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
-
- currentMer.setMerSize(merSize);
-
- while (moreStuff) {
-
- // Load more stuff if needed.
- //
- if (M->loadMore() == true) {
- M->rebuild();
-
- u32bit additionalLoading = 8192;
-
- if (maxSize / args->mergeFilesLen > M->length())
- additionalLoading = maxSize / args->mergeFilesLen - M->length();
-
- loadAll = true;
-
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- if (R[i]->validMer()) {
- M->read(R[i], additionalLoading, loadAll);
- loadAll = false;
- }
- }
- }
-
- // All done? Exit.
- if (M->length() == 0)
- moreStuff = false;
-
- thisMer = M->pop(thisCount, thisPositions);
-
- // If we've hit a different mer, write out the last one
- if ((M->length() == 0) || (*thisMer != currentMer)) {
- switch (args->personality) {
- case PERSONALITY_MIN:
- if (currentTimes == args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_MERGE:
- case PERSONALITY_MINEXIST:
- case PERSONALITY_MAX:
- case PERSONALITY_ADD:
- W->addMer(currentMer, currentCount, currentPositions);
- break;
- case PERSONALITY_AND:
- if (currentTimes == args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_NAND:
- if (currentTimes != args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_OR:
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_XOR:
- if ((currentTimes % 2) == 1)
- W->addMer(currentMer, currentCount);
- break;
- default:
- fprintf(stderr, "ERROR - invalid personality in multipleOperations::write\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- break;
- }
-
- currentMer = *thisMer;
-
- currentCount = u32bitZERO;
- currentTimes = u32bitZERO;
-
- C->tick();
- }
-
-
- if (moreStuff == false)
- break;
-
-
- // Perform the operation
- switch (args->personality) {
- case PERSONALITY_MERGE:
- if (thisPositions) {
-
- if (currentPositionsMax == 0) {
- currentPositionsMax = 1048576;
- currentPositions = new u32bit [currentPositionsMax];
- }
-
- if (currentPositionsMax < currentCount + thisCount) {
- while (currentPositionsMax < currentCount + thisCount)
- currentPositionsMax *= 2;
-
- u32bit *t = new u32bit [currentPositionsMax];
- memcpy(t, currentPositions, sizeof(u32bit) * currentCount);
- delete [] currentPositions;
- currentPositions = t;
- }
-
- if (thisCount < 16) {
- for (u32bit i=0; i<thisCount; i++)
- currentPositions[currentCount + i] = thisPositions[i];
- } else {
- memcpy(currentPositions + currentCount, thisPositions, sizeof(u32bit) * thisCount);
- }
- }
- // Otherwise, we're the same as ADD.
- currentCount += thisCount;
- break;
- case PERSONALITY_MIN:
- case PERSONALITY_MINEXIST:
- if (currentTimes == 0) {
- currentCount = thisCount;
- } else {
- if (currentCount > thisCount)
- currentCount = thisCount;
- }
- break;
- case PERSONALITY_MAX:
- if (currentCount < thisCount)
- currentCount = thisCount;
- break;
- case PERSONALITY_ADD:
- currentCount += thisCount;
- break;
- case PERSONALITY_AND:
- case PERSONALITY_NAND:
- case PERSONALITY_OR:
- case PERSONALITY_XOR:
- currentCount = 1;
- break;
- default:
- fprintf(stderr, "ERROR - invalid personality in multipleOperations::operate\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- break;
- }
-
- currentTimes++;
- }
-
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- delete R[i];
- delete R;
- delete W;
- delete M;
- delete C;
-}
diff --git a/kmer/meryl/merge.qsort.C b/kmer/meryl/merge.qsort.C
deleted file mode 100644
index a71f62b..0000000
--- a/kmer/meryl/merge.qsort.C
+++ /dev/null
@@ -1,471 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "meryl.H"
-#include "libmeryl.H"
-
-
-using namespace std;
-
-#include <algorithm>
-
-struct mMer {
- kMer _mer;
- u32bit _cnt;
- u32bit _off;
-};
-
-static
-int
-mMerGreaterThan(void const *a, void const *b) {
- mMer const *A = (mMer const *)a;
- mMer const *B = (mMer const *)b;
- return(B->_mer.qsort_less(A->_mer));
-}
-
-
-
-class mMerList {
-public:
- mMerList(u32bit maxSize) {
- _posLen = 0;
- _posMax = 2 * maxSize;
- _pos = new u32bit [_posMax];
-
- _mmmLen = 0;
- _mmmMax = maxSize;
- _mmm = new mMer [_mmmMax];
- };
- ~mMerList() {
- delete [] _pos;
- delete [] _mmm;
- };
-
- u32bit length(void) { return(_mmmLen); };
-
- // Until we sort, first() is the last thing loaded.
- // After we sort, first() is the lowest mer in the set.
-
- kMer &first(void) { return(_mmm[_mmmLen-1]._mer); };
- //kMer &last(void) { return(_mmm[0]._mer); };
- //kMer &get(u32bit i) { return(_mmm[i]._mer); };
-
- // Return the first (sorted order) thing in the list -- it's the last on the list.
- kMer *pop(u32bit &cnt, u32bit* &pos) {
- if (_mmmLen == 0)
- return(0L);
-
- _mmmLen--;
-
- assert(_sorted);
-
- cnt = _mmm[_mmmLen]._cnt;
- pos = 0L;
-
- if (_mmm[_mmmLen]._off != ~u32bitZERO)
- pos = _pos + _mmm[_mmmLen]._off;
-
- return(&_mmm[_mmmLen]._mer);
- }
-
-
- // rebuild the position list, squeezes out empty items
- void rebuild(void) {
- if (_posLen > 0) {
- u32bit *np = new u32bit [_posMax];
-
- _posLen = 0;
-
- for (u32bit i=0; i<_mmmLen; i++) {
- mMer *m = _mmm + i;
-
- if (m->_off != ~u32bitZERO) {
- _mmm[_mmmLen]._off = _posLen;
-
- for (u32bit p=0; p<m->_cnt; p++, _posLen++)
- np[_posLen] = _pos[p];
- }
- }
-
- delete [] _pos;
- _pos = np;
- }
- };
-
-
-
- // Read more mers from the file
- void read(merylStreamReader *R, u32bit num) {
- u32bit xxx = 0;
-
- if (_mmmLen + num >= _mmmMax) {
- fprintf(stderr, "Reallocate _mmm\n");
- _mmmMax = _mmmMax + 2 * num;
- mMer *tmp = new mMer [_mmmMax];
- memcpy(tmp, _mmm, sizeof(mMer) * _mmmLen);
- delete [] _mmm;
- _mmm = tmp;
- }
-
- _sorted = false;
-
- R->nextMer();
-
- for (xxx=0; (xxx < num) && (R->validMer()); xxx++) {
- if (_mmmMax <= _mmmLen) {
- fprintf(stderr, "Reallocate _mmm\n");
- _mmmMax *= 2;
- mMer *tmp = new mMer [_mmmMax];
- memcpy(tmp, _mmm, sizeof(mMer) * _mmmLen);
- delete [] _mmm;
- _mmm = tmp;
- }
-
- _mmm[_mmmLen]._mer = R->theFMer();
- _mmm[_mmmLen]._cnt = R->theCount();
- _mmm[_mmmLen]._off = ~u32bitZERO;
-
- u32bit *pos = R->thePositions();
- if (pos) {
- _mmm[_mmmLen]._off = _posLen;
-
- if (_posMax <= _posLen + _mmm[_mmmLen]._cnt) {
- fprintf(stderr, "Reallocate _pos\n");
- _posMax *= 2;
- u32bit *tmp = new u32bit [_posMax];
- memcpy(tmp, _pos, sizeof(u32bit) * _posLen);
- delete [] _pos;
- _pos = tmp;
- }
-
- for (u32bit i=0; i<_mmm[_mmmLen]._cnt; i++, _posLen++)
- _pos[_posLen] = pos[i];
- }
-
- _mmmLen++;
-
- R->nextMer();
- }
-
- //fprintf(stderr, "read()-- now up to "u32bitFMT" mers ("u32bitFMT" pos); loaded "u32bitFMT" out of "u32bitFMT" requested.\n", _mmmLen, _posLen, xxx, num);
- };
-
-
- // Sort our list of mers
- void sort(void) {
- if (_sorted == false) {
- //fprintf(stderr, "SORT BEG\n");
- qsort_mt(_mmm, _mmmLen, sizeof(mMer), mMerGreaterThan, 8, 32 * 1024);
- _sorted = true;
- //fprintf(stderr, "SORT END\n");
- }
- };
-
-
-private:
- bool _sorted;
-
- u32bit _posLen;
- u32bit _posMax;
- u32bit *_pos;
-
- u32bit _mmmLen;
- u32bit _mmmMax;
- mMer *_mmm;
-};
-
-
-
-
-void
-multipleOperations(merylArgs *args) {
-
- char debugstring[256];
- char debugstring2[256];
-
- if (args->mergeFilesLen < 2) {
- fprintf(stderr, "ERROR - must have at least two databases (you gave "u32bitFMT")!\n", args->mergeFilesLen);
- exit(1);
- }
- if (args->outputFile == 0L) {
- fprintf(stderr, "ERROR - no output file specified.\n");
- exit(1);
- }
- if ((args->personality != PERSONALITY_MERGE) &&
- (args->personality != PERSONALITY_MIN) &&
- (args->personality != PERSONALITY_MINEXIST) &&
- (args->personality != PERSONALITY_MAX) &&
- (args->personality != PERSONALITY_ADD) &&
- (args->personality != PERSONALITY_AND) &&
- (args->personality != PERSONALITY_NAND) &&
- (args->personality != PERSONALITY_OR) &&
- (args->personality != PERSONALITY_XOR)) {
- fprintf(stderr, "ERROR - only personalities min, minexist, max, add, and, nand, or, xor\n");
- fprintf(stderr, "ERROR - are supported in multipleOperations().\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- }
-
- merylStreamReader **R = new merylStreamReader* [args->mergeFilesLen];
- merylStreamWriter *W = 0L;
-
- u32bit maxSize = 512 * 1024;
-
- mMerList *M = new mMerList(maxSize + maxSize / 4);
-
-
- // Open the input files and load some mers - we need to do this
- // just so we can check the mersizes/compression next.
- //
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- R[i] = new merylStreamReader(args->mergeFiles[i]);
- M->read(R[i], 1 + i);
- }
-
- // Verify that the mersizes are all the same
- //
- bool fail = false;
- u32bit merSize = R[0]->merSize();
- u32bit merComp = R[0]->merCompression();
-
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- fail |= (merSize != R[i]->merSize());
- fail |= (merComp != R[i]->merCompression());
- }
-
- if (fail)
- fprintf(stderr, "ERROR: mer sizes (or compression level) differ.\n"), exit(1);
-
- // Open the output file, using the largest prefix size found in the
- // input/mask files.
- //
- u32bit prefixSize = 0;
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- if (prefixSize < R[i]->prefixSize())
- prefixSize = R[i]->prefixSize();
-
- W = new merylStreamWriter(args->outputFile, merSize, merComp, prefixSize);
-
-
- kMer lastLoaded;
-
- lastLoaded.setMerSize(merSize);
- lastLoaded.smallest();
-
- // Load mers from all files, remember the largest mer we load.
- //
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- M->read(R[i], maxSize / args->mergeFilesLen);
- if (lastLoaded < M->first())
- lastLoaded = M->first();
- }
-
- // Make sure all files have at least that largest mer loaded.
- //
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- while (R[i]->validMer() && (R[i]->theFMer() <= lastLoaded))
- M->read(R[i], 2 * 1024);
-
- fprintf(stderr, "Initial load: length="u32bitFMT" lastLoaded=%s\n",
- M->length(), lastLoaded.merToString(debugstring));
-
- M->sort();
-
- bool allLoaded = false;
- bool moreStuff = true;
-
- kMer currentMer; // The current mer we're operating on
- u32bit currentCount = u32bitZERO; // The count (operation dependent) of this mer
- u32bit currentTimes = u32bitZERO; // Number of files it's in
-
- u32bit currentPositionsMax = 0;
- u32bit *currentPositions = 0L;
-
- kMer *thisMer; // The mer we just read
- u32bit thisCount = u32bitZERO; // The count of the mer we just read
- u32bit *thisPositions = 0L;
-
- speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose);
-
- currentMer.setMerSize(merSize);
-
- while (moreStuff) {
-
- // Load more stuff if needed. M is sorted, so first() is the
- // smallest mer in the set - we're good up to and including
- // lastLoaded.
- //
- if ((allLoaded == false) &&
- ((M->length() == 0) || (lastLoaded < M->first()))) {
-
-#if 0
- if (M->length() > 0)
- fprintf(stderr, "LOADMORE length="u32bitFMT" lastLoaded=%s first=%s\n",
- M->length(), lastLoaded.merToString(debugstring2), M->first().merToString(debugstring));
- else
- fprintf(stderr, "LOADMORE length="u32bitFMT" lastLoaded=%s first=EMPTY\n",
- M->length(), lastLoaded.merToString(debugstring2));
-#endif
-
- // We need to copy all the mers currently loaded into fresh
- // storage, so we can deallocate the position storage. Yucky.
- //
- M->rebuild();
-
- allLoaded = true;
-
- // Load more stuff to give us a large collection of mers
- //
- u32bit additionalLoading = 8192;
-
- if (maxSize / args->mergeFilesLen > M->length())
- additionalLoading = maxSize / args->mergeFilesLen - M->length();
-
- //fprintf(stderr, "LOADMORE adding "u32bitFMT" from each file\n", additionalLoading);
-
- lastLoaded.setMerSize(merSize);
- lastLoaded.smallest();
-
- for (u32bit i=0; i<args->mergeFilesLen; i++) {
- if (R[i]->validMer()) {
- M->read(R[i], additionalLoading);
- if (lastLoaded < M->first())
- lastLoaded = M->first();
- allLoaded = false;
- }
- }
-
- // Make sure all files have at least that largest mer loaded.
- //
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- while (R[i]->validMer() && (R[i]->theFMer() <= lastLoaded))
- M->read(R[i], 2 * 1024);
-
- M->sort();
- }
-
-
- // All done? Exit.
- if (M->length() == 0)
- moreStuff = false;
-
- thisMer = M->pop(thisCount, thisPositions);
-
- // If we've hit a different mer, write out the last one
- if ((M->length() == 0) || (*thisMer != currentMer)) {
- switch (args->personality) {
- case PERSONALITY_MIN:
- if (currentTimes == args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_MERGE:
- case PERSONALITY_MINEXIST:
- case PERSONALITY_MAX:
- case PERSONALITY_ADD:
- W->addMer(currentMer, currentCount, currentPositions);
- break;
- case PERSONALITY_AND:
- if (currentTimes == args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_NAND:
- if (currentTimes != args->mergeFilesLen)
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_OR:
- W->addMer(currentMer, currentCount);
- break;
- case PERSONALITY_XOR:
- if ((currentTimes % 2) == 1)
- W->addMer(currentMer, currentCount);
- break;
- default:
- fprintf(stderr, "ERROR - invalid personality in multipleOperations::write\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- break;
- }
-
- currentMer = *thisMer;
-
- currentCount = u32bitZERO;
- currentTimes = u32bitZERO;
-
- C->tick();
- }
-
-
- if (moreStuff == false)
- break;
-
-
- // Perform the operation
- switch (args->personality) {
- case PERSONALITY_MERGE:
- if (thisPositions) {
-
- if (currentPositionsMax == 0) {
- currentPositionsMax = 1048576;
- currentPositions = new u32bit [currentPositionsMax];
- }
-
- if (currentPositionsMax < currentCount + thisCount) {
- while (currentPositionsMax < currentCount + thisCount)
- currentPositionsMax *= 2;
-
- u32bit *t = new u32bit [currentPositionsMax];
- memcpy(t, currentPositions, sizeof(u32bit) * currentCount);
- delete [] currentPositions;
- currentPositions = t;
- }
-
- if (thisCount < 16) {
- for (u32bit i=0; i<thisCount; i++)
- currentPositions[currentCount + i] = thisPositions[i];
- } else {
- memcpy(currentPositions + currentCount, thisPositions, sizeof(u32bit) * thisCount);
- }
- }
- // Otherwise, we're the same as ADD.
- currentCount += thisCount;
- break;
- case PERSONALITY_MIN:
- case PERSONALITY_MINEXIST:
- if (currentTimes == 0) {
- currentCount = thisCount;
- } else {
- if (currentCount > thisCount)
- currentCount = thisCount;
- }
- break;
- case PERSONALITY_MAX:
- if (currentCount < thisCount)
- currentCount = thisCount;
- break;
- case PERSONALITY_ADD:
- currentCount += thisCount;
- break;
- case PERSONALITY_AND:
- case PERSONALITY_NAND:
- case PERSONALITY_OR:
- case PERSONALITY_XOR:
- currentCount = 1;
- break;
- default:
- fprintf(stderr, "ERROR - invalid personality in multipleOperations::operate\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- break;
- }
-
- currentTimes++;
- }
-
- for (u32bit i=0; i<args->mergeFilesLen; i++)
- delete R[i];
- delete R;
- delete W;
- delete M;
- delete C;
-}
diff --git a/kmer/meryl/mervin.C b/kmer/meryl/mervin.C
deleted file mode 100644
index ec7e06b..0000000
--- a/kmer/meryl/mervin.C
+++ /dev/null
@@ -1,793 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "bio++.H"
-#include "sweatShop.H"
-
-#include "libmeryl.H"
-
-#include <algorithm>
-
-using namespace std;
-
-// var, old, new -- returns true if "(var == old) and var <- new"
-//
-// CAS - #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
-
-const u32bit pileMax = 32768;
-
-const u32bit kmerSize = 22;
-const u32bit kmerBits = 2 * kmerSize;
-
-const u32bit pilePreSize = 6;
-const u32bit pilePreBits = 2 * pilePreSize;
-
-const u32bit sortPreSize = 10;
-const u32bit sortPreBits = 2 * sortPreSize;
-
-
-class kmerPile {
-public:
- kmerPile(u32bit prefix) {
- pileLen = 0;
- pilePrefix = prefix;
- };
- ~kmerPile() {
- };
-
- void initialize(u32bit prefix) {
- pileLen = 0;
- pilePrefix = prefix;
- };
-
- void addMer(u64bit mer) {
- pileDat[pileLen++] = mer;
- };
-
- void sort(void) {
- ::sort(pileDat, pileDat + pileLen);
- };
-
- u32bit pileLen;
- u32bit pilePrefix;
-
- u64bit pileDat[pileMax];
-};
-
-
-
-
-class kmerSorter {
-public:
- kmerSorter() {
- sorterLocked = 0;
- sorterLen = 0;
- sorterMax = 4;
- sorterMer = new u64bit [sorterMax];
- sorterCnt = new u32bit [sorterMax];
- };
- ~kmerSorter() {
- delete [] sorterMer;
- delete [] sorterCnt;
- };
-
- void merge(u64bit *pileDat, u32bit pileLen) {
- u32bit nmax = MAX(16, sorterLen + pileLen / 4);
- u64bit *nmer = new u64bit [nmax];
- u32bit *ncnt = new u32bit [nmax];
- u32bit npos = 0;
-
- assert(nmax > 0);
-
- u32bit spos = 0;
- u32bit ppos = 0;
-
- bool useSorterFirst = false;
-
- if ((sorterLen > 0) && (pileLen > 0)) {
- useSorterFirst = (sorterMer[0] < pileDat[0]);
-
- } else if (spos < sorterLen) {
- useSorterFirst = true;
-
- } else if (ppos < pileLen) {
- useSorterFirst = false;
-
- } else {
- assert(0);
- }
-
- if (useSorterFirst) {
- nmer[0] = sorterMer[spos];
- ncnt[0] = sorterCnt[spos];
- spos++;
- } else {
- nmer[0] = pileDat[ppos];
- ncnt[0] = 1;
- ppos++;
- }
-
- while ((spos < sorterLen) && (ppos < pileLen)) {
-
- if (nmax <= npos + 1) {
- nmax += (pileLen - ppos) + (sorterLen - spos) + 1;
-
- u64bit *nmermore = new u64bit [nmax];
- u32bit *ncntmore = new u32bit [nmax];
-
- memcpy(nmermore, nmer, sizeof(u64bit) * (npos + 1));
- memcpy(ncntmore, ncnt, sizeof(u32bit) * (npos + 1));
-
- delete [] nmer; nmer = nmermore;
- delete [] ncnt; ncnt = ncntmore;
- }
-
- if (nmer[npos] == sorterMer[spos]) {
- ncnt[npos] += sorterCnt[spos];
- spos++;
-
- } else if (nmer[npos] == pileDat[ppos]) {
- ncnt[npos] += 1;
- ppos++;
-
- } else if (sorterMer[spos] < pileDat[ppos]) {
- npos++;
- nmer[npos] = sorterMer[spos];
- ncnt[npos] = sorterCnt[spos];
- spos++;
-
- } else {
- npos++;
- nmer[npos] = pileDat[ppos];
- ncnt[npos] = 1;
- ppos++;
- }
- }
-
- u32bit remain = (sorterLen - spos) + (pileLen - ppos);
-
- if (nmax < npos + 1 + remain) {
- nmax = npos + 1 + remain;
-
- u64bit *nmermore = new u64bit [nmax];
- u32bit *ncntmore = new u32bit [nmax];
-
- memcpy(nmermore, nmer, sizeof(u64bit) * (npos + 1));
- memcpy(ncntmore, ncnt, sizeof(u32bit) * (npos + 1));
-
- delete [] nmer; nmer = nmermore;
- delete [] ncnt; ncnt = ncntmore;
- }
-
-
-
- while (spos < sorterLen) {
- if (nmer[npos] == sorterMer[spos]) {
- ncnt[npos] += sorterCnt[spos];
- } else {
- npos++;
- nmer[npos] = sorterMer[spos];
- ncnt[npos] = sorterCnt[spos];
- }
-
- spos++;
- }
-
-
- while (ppos < pileLen) {
- if (nmer[npos] == pileDat[ppos]) {
- ncnt[npos] += 1;
- } else {
- npos++;
- nmer[npos] = pileDat[ppos];
- ncnt[npos] = 1;
- }
-
- ppos++;
- }
-
- delete [] sorterMer;
- delete [] sorterCnt;
-
- sorterMer = nmer;
- sorterCnt = ncnt;
- sorterLen = npos + 1;
- sorterMax = nmax;
-
-#if 1
- bool broken = false;
-
- for (u32bit i=1; i<sorterLen; i++) {
- assert(sorterMer[i-1] < sorterMer[i]);
- if (sorterMer[i-1] >= sorterMer[i])
- broken = true;
- }
-#endif
-
- };
-
- void write(u32bit prefix, FILE *F, merylStreamWriter *W) {
- char km[64] = {0};
- u32bit kp = pilePreSize;
- u32bit np = 0;
-
- {
- u32bit pre = prefix;
-
- for (u32bit pp=0; pp<pilePreSize; pp++) {
- km[--kp] = bitsToLetter[pre & 0x03];
- pre >>= 2;
- }
- }
-
- np = kmerSize - pilePreSize;
-
- for (u32bit ii=0; ii<sorterLen; ii++) {
- u64bit mer = sorterMer[ii];
-
- kp = kmerSize;
-
- for (u32bit pp=0; pp<np; pp++) {
- km[--kp] = bitsToLetter[mer & 0x03];
- mer >>= 2;
- }
-
- fprintf(F, ">"u32bitFMT"\n%s\n", sorterCnt[ii], km);
-
- if (W)
- W->addMer(prefix, pilePreBits,
- sorterMer[ii], kmerBits - pilePreBits,
- sorterCnt[ii],
- 0L);
- }
- };
-
- volatile u32bit sorterLocked;
- u32bit sorterLen;
- u32bit sorterMax;
-
- u64bit *sorterMer;
- u32bit *sorterCnt;
-};
-
-
-
-
-class kmerGlobal {
-public:
- kmerGlobal() {
- inName = NULL;
- inFile = NULL;
-
-#if 0
- inputBufferMax = 131072;
- inputBufferLen = 0;
- inputBufferPos = 0;
- inputBuffer = new char [inputBufferMax];
-#endif
-
- inputBufferMax = 0;
- inputBufferLen = 0;
- inputBufferPos = 0;
- inputBuffer = NULL;
-
- outPrefix = NULL;
- outFile = NULL;
-
- fkPre = 0;
- fkMer = 0;
-
- rkPre = 0;
- rkMer = 0;
-
- kLen = 0;
-
- pilesFreeLock = 0;
- pilesFreeLen = 2048;
- pilesFreeMax = 2 << pilePreBits;
- pilesFree = new kmerPile * [pilesFreeMax];
-
- memset(pilesFree, 0, sizeof(kmerPile *) * pilesFreeMax);
-
- piles = new kmerPile * [1 << pilePreBits];
- sorters = new kmerSorter [1 << sortPreBits];
-
- memset(piles, 0, sizeof(kmerPile *) * (1 << pilePreBits));
-
- for (u32bit i=0; i<pilesFreeLen; i++)
- pilesFree[i] = new kmerPile(0);
-
- for (u32bit i=0; i< (1 << pilePreBits); i++)
- piles[i] = new kmerPile(i);
-
- pilesToSortLen = 0;
- pilesToSortMax = 2 * (1 << pilePreBits);
- pilesToSort = new kmerPile * [pilesToSortMax];
- };
- ~kmerGlobal() {
- delete [] piles;
- delete [] sorters;
- delete [] pilesToSort;
- //delete [] inputBuffer;
- };
-
-
- void initialize(void) {
- //inBuffer = new readBuffer(inName, 0);
-
-#if 1
- inputBufferMax = 0;
- inputBufferLen = 0;
- inputBufferPos = 0;
- inputBuffer = (char *)mapFile(inName, &inputBufferLen, 'r');
-#endif
-
- naptime.tv_sec = 0;
- naptime.tv_nsec = 166666666ULL; // 1/6 second
- naptime.tv_nsec = 250000ULL;
- };
-
- kmerPile *getFreePile(u32bit prefix) {
- kmerPile *pp;
-
- while (__sync_bool_compare_and_swap(&pilesFreeLock, 0, 1) == false)
- nanosleep(&naptime, 0L);
-
- assert(pilesFreeLock == 1);
-
- if (pilesFreeLen == 0) {
- pilesFreeLock = 0;
- //fprintf(stderr, "ALLOCATE PILE!\n");
- pp = new kmerPile(prefix);
-
- } else {
- pp = pilesFree[--pilesFreeLen];
- pilesFreeLock = 0;
- }
-
- pp->initialize(prefix);
-
- return(pp);
- };
-
- void releasePile(kmerPile *pile) {
-
- if (pilesFreeLen >= pilesFreeMax) {
- //fprintf(stderr, "DELETE PILE!\n");
- delete pile;
-
- } else {
- while (__sync_bool_compare_and_swap(&pilesFreeLock, 0, 1) == false)
- nanosleep(&naptime, 0L);
-
- assert(pilesFreeLock == 1);
-
- pilesFree[pilesFreeLen++] = pile;
-
- pilesFreeLock = 0;
- }
-
- };
-
-
- void addToPile(u64bit pre, u64bit mer) {
-
- assert(piles[pre] != NULL);
- //if (piles[pre] == NULL)
- // piles[pre] = getFreePile(pre);
-
- if (piles[pre]->pileLen < pileMax) {
- piles[pre]->addMer(mer);
- return;
- }
-
- if (pilesToSortMax <= pilesToSortLen) {
- fprintf(stderr, "realloc\n");
- exit(1);
- }
-
- pilesToSort[pilesToSortLen++] = piles[pre];
-
- piles[pre] = getFreePile(pre);
- piles[pre]->addMer(mer);
- };
-
-
- kmerPile *getFullPile(void) {
- if (pilesToSortLen == 0)
- return(NULL);
-
- //fprintf(stderr, "return pile "u32bitFMT"\n", pilesToSort[pilesToSortLen-1]->pilePrefix);
- return(pilesToSort[--pilesToSortLen]);
- };
-
-
- kmerPile *allDataLoaded(void) {
-
- for (u32bit pp=0; pp < (1 << pilePreBits); pp++) {
- if ((piles[pp] != NULL) &&
- (piles[pp]->pileLen > 0)) {
- //fprintf(stderr, "Add pile "u32bitFMT" to list.\n", pp);
- pilesToSort[pilesToSortLen++] = piles[pp];
- } else {
- delete piles[pp];
- }
-
- piles[pp] = NULL;
- }
-
- fprintf(stderr, "allDataLoaded()-- pilesToSortLen = "u32bitFMT"\n", pilesToSortLen);
-
- return(getFullPile());
- };
-
-
- void addBases(u32bit bgn, u32bit len) {
- u32bit kp2 = kmerBits - pilePreBits - 2;
- u32bit pp2 = pilePreBits - 2;
-
- u64bit mpp = u64bitMASK(pilePreBits);
- u64bit mkp = u64bitMASK(kmerBits - pilePreBits);
-
- for (u32bit pos=0; pos<len; pos++) {
- u64bit bt = letterToBits[ inputBuffer[bgn+pos] ];
-
- if (bt > 4) {
- kLen = 0;
- continue;
- }
-
- u64bit tm = 0;
-
- tm = fkMer >> kp2;
- tm &= 0x00000003;
-
- fkPre <<= 2;
- fkPre |= tm;
-
- fkMer <<= 2;
- fkMer |= bt;
-
- tm = rkMer & 0x00000003;
-
- rkPre >>= 2;
- rkPre |= tm << pp2;
-
- rkMer >>= 2;
- rkMer |= bt << kp2;
-
- kLen++;
-
- if (kLen < kmerSize)
- continue;
-
- kLen = kmerSize;
-
- fkPre &= mpp;
- fkMer &= mkp;
-
- rkPre &= mpp;
- rkMer &= mkp;
-
- addToPile(fkPre, fkMer);
- addToPile(rkPre, rkMer);
- }
- }
-
- bool addBaseToKmer(char base) {
- u64bit bt = letterToBits[base];
-
- if (bt > 4) {
- kLen = 0;
- return(false);
- }
-
- u64bit tm = 0;
-
- tm = fkMer >> (kmerBits - pilePreBits - 2);
- tm &= 0x00000003;
-
- fkPre <<= 2;
- fkPre |= tm;
-
- fkMer <<= 2;
- fkMer |= bt;
-
- tm = rkMer & 0x00000003;
-
- rkPre >>= 2;
- rkPre |= tm << (pilePreBits - 2);
-
- rkMer >>= 2;
- rkMer |= bt << (kmerBits - pilePreBits - 2);
-
- kLen++;
-
- if (kLen < kmerSize) {
- return(false);
- }
-
- kLen = kmerSize;
-
- fkPre &= u64bitMASK(pilePreBits);
- fkMer &= u64bitMASK(kmerBits - pilePreBits);
-
- rkPre &= u64bitMASK(pilePreBits);
- rkMer &= u64bitMASK(kmerBits - pilePreBits);
-
- addToPile(fkPre, fkMer);
- addToPile(rkPre, rkMer);
-
- return(true);
- };
-
-
- void write(void) {
- char outName[FILENAME_MAX];
-
- sprintf(outName, "%s.fasta", outPrefix);
-
- errno = 0;
- FILE *F = fopen(outName, "w");
- if (errno)
- fprintf(stderr, "Failed to open '%s' for writing: %s\n", outName, strerror(errno)), exit(1);
-
- //merylStreamWriter *W = new merylStreamWriter(outPrefix, kmerSize, 0, sortPreBits, false);
-
- for (u32bit ss=0; ss < (1 << sortPreBits); ss++)
- sorters[ss].write(ss, F, NULL);
-
- fclose(F);
- //delete W;
- }
-
- char *inName;
- FILE *inFile;
-
- readBuffer *inBuffer;
-
- u64bit inputBufferMax;
- u64bit inputBufferLen;
- u64bit inputBufferPos;
- char *inputBuffer;
-
- char *outPrefix;
- FILE *outFile;
-
- u64bit fkPre; // Forward loaded kmer
- u64bit fkMer;
-
- u64bit rkPre; // Reverse loaded kmer
- u64bit rkMer;
-
- u32bit kLen;
-
- u32bit pilesFreeLock;
- u32bit pilesFreeLen;
- u32bit pilesFreeMax;
- kmerPile **pilesFree;
-
- kmerPile **piles;
- kmerSorter *sorters;
-
- struct timespec naptime;
-
- u32bit pilesToSortLen;
- u32bit pilesToSortMax;
- kmerPile **pilesToSort;
-};
-
-
-
-
-
-u64bit bytesLoaded = 0;
-u64bit basesLoaded = 0;
-speedCounter bytes(" bytes %8.0f Mbytes (%8.5f Mbytes/sec\r", 1048576, 1048576, true);
-
- // Reads input, constructs kmers, adds kmers to piles of kmers.
-void*
-sifterThread(void *global) {
- kmerGlobal *glob = (kmerGlobal *)global;
- kmerPile *pile = glob->getFullPile();
-
- if (pile)
- return(pile);
-
- //if ((glob->inFile == NULL) && (glob->inBuffer == NULL))
- // return(NULL);
-
- anotherBase:
- //bytesLoaded++;
- //if ((bytesLoaded % (16 * 1048576)) == 0)
- // fprintf(stderr, "sifterThread()-- loaded "u64bitFMT" MB.\n", bytesLoaded >> 20);
-
-#if 0
- // Uses the readBuffer in char-by-char mode
- //
- char ch = glob->inBuffer->read();
- bytes.tick();
-
- if (glob->inBuffer->eof()) {
- delete glob->inBuffer;
- glob->inBuffer = NULL;
- return(glob->allDataLoaded());
- }
-
- if (glob->addBaseToKmer(ch) == false)
- goto anotherBase;
-
-#endif
-
-
-#if 0
- // Uses the readBuffer in block-copy mode
- //
- u32bit len = glob->inBuffer->read(glob->inputBuffer, glob->inputBufferMax);
-
- if (len == 0) {
- delete glob->inBuffer;
- glob->inBuffer = NULL;
- return(glob->allDataLoaded());
- }
-
- glob->addBases(0, len);
- bytes.tick(len);
-
-#endif
-
-#if 1
- // Uses a direct mmap'd file
- //
- u64bit len = glob->inputBufferLen - glob->inputBufferPos;
-
- if (len == 0)
- return(NULL);
-
- if (len > 16 * 1048576)
- len = 16 * 1048576;
-
- //fprintf(stderr, "Add "u64bitFMT" bases.\n", len);
-
- glob->addBases(glob->inputBufferPos, len);
- bytes.tick(len);
-
- glob->inputBufferPos += len;
-#endif
-
- pile = glob->getFullPile();
-
- if (pile == NULL)
- goto anotherBase;
-
- return(pile);
-}
-
-
-
- // Takes a pile of kmers, sorts it, and them merges into the appropriate kmerSorter objects.
-void
-sorterThread(void *global, void *thread, void *thing) {
- kmerGlobal *glob = (kmerGlobal *)global;
- kmerPile *pile = (kmerPile *)thing;
-
- struct timespec naptime;
- naptime.tv_sec = 0;
- naptime.tv_nsec = 166666666ULL; // 1/6 second
- naptime.tv_nsec = 250000ULL;
-
- if (pile->pileLen == 0)
- // Nothing to add.
- return;
-
- pile->sort();
-
- u32bit pileBgn = 0;
- u32bit pileEnd = 1;
-
- u32bit pileMaskShift = sortPreBits - pilePreBits;
- u32bit pileDataShift = kmerBits - sortPreBits;
-
- u64bit pileToSortPreMask = u64bitMASK(sortPreBits - pilePreBits) << (kmerBits - sortPreBits);
- u64bit pileToSortMask = u64bitMASK(kmerBits - sortPreBits);
-
- u32bit sortPre = 0;
- u64bit pileToSort = 0;
-
- while (pileBgn < pile->pileLen) {
- sortPre = (pile->pilePrefix << pileMaskShift) | (pile->pileDat[pileBgn] >> pileDataShift);
- pileToSort = pile->pileDat[pileBgn] & pileToSortPreMask;
-
- //fprintf(stderr, "0x"u64bitHEX"\n", pileToSortPreMask);
- //fprintf(stderr, "0x"u64bitHEX"\n", pileToSortMask);
-
- while ((pileEnd < pile->pileLen) &&
- ((pile->pileDat[pileEnd] & pileToSortPreMask) == pileToSort)) {
- //fprintf(stderr, "0x"u64bitHEX" -> 0x"u64bitHEX" "u64bitFMT"\n",
- // pile->pileDat[pileEnd],
- // pile->pileDat[pileEnd] & pileToSortMask,
- // pile->pileDat[pileEnd] & pileToSortMask);
- pile->pileDat[pileEnd] &= pileToSortMask;
- pileEnd++;
- }
-
- while (__sync_bool_compare_and_swap(&glob->sorters[sortPre].sorterLocked, 0, 1) == false)
- nanosleep(&naptime, 0L);
-
- assert(glob->sorters[sortPre].sorterLocked == 1);
-
- glob->sorters[sortPre].merge(pile->pileDat + pileBgn, pileEnd - pileBgn);
-
- glob->sorters[sortPre].sorterLocked = 0;
-
- pileBgn = pileEnd;
- }
-}
-
-
-
-// Does nothing but delete the pile object. We don't output till the end.
-void
-nullThread(void *global, void *thing) {
- kmerGlobal *glob = (kmerGlobal *)global;
- kmerPile *pile = (kmerPile *)thing;
-
- glob->releasePile(pile);
-}
-
-
-
-int
-main(int argc, char **argv) {
- kmerGlobal *kg = new kmerGlobal;
-
- int arg=1;
- int err=0;
-
- while (arg < argc) {
- if (strcmp(argv[arg], "-i") == 0)
- kg->inName = argv[++arg];
-
- else if (strcmp(argv[arg], "-o") == 0)
- kg->outPrefix = argv[++arg];
-
- else
- err++;
-
- arg++;
- }
- if (kg->inName == NULL)
- err++;
- if (kg->outPrefix == NULL)
- err++;
- if (err) {
- fprintf(stderr, "usage: %s -i in.sequence -i prefix\n", argv[0]);
- exit(1);
- }
-
- kg->initialize();
-
- sweatShop *ss = new sweatShop(sifterThread,
- sorterThread,
- nullThread);
-
- ss->setLoaderBatchSize(512);
-
- ss->setNumberOfWorkers(1);
- ss->setWriterQueueSize(16384);
-
- //for (u32bit i=0; i<config._numSearchThreads; i++)
- // ss->setThreadData(i, new searcherState(i));
-
- ss->run(kg, true);
-
- delete ss;
-
- kg->write();
-
- delete kg;
-
- exit(0);
-}
diff --git a/kmer/meryl/meryl.C b/kmer/meryl/meryl.C
deleted file mode 100644
index a2a5f98..0000000
--- a/kmer/meryl/meryl.C
+++ /dev/null
@@ -1,72 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "bio++.H"
-#include "meryl.H"
-
-int
-main(int argc, char **argv) {
- merylArgs *args = new merylArgs(argc, argv);
-
- switch (args->personality) {
- case 'P':
- estimate(args);
- break;
-
- case 'B':
- build(args);
- break;
-
- case 'd':
- dumpDistanceBetweenMers(args);
- break;
- case 't':
- dumpThreshold(args);
- break;
- case 'p':
- dumpPositions(args);
- break;
- case 'c':
- countUnique(args);
- break;
- case 'h':
- plotHistogram(args);
- break;
-
- case PERSONALITY_MIN:
- case PERSONALITY_MINEXIST:
- case PERSONALITY_MAX:
- case PERSONALITY_MAXEXIST:
- case PERSONALITY_ADD:
- case PERSONALITY_AND:
- case PERSONALITY_NAND:
- case PERSONALITY_OR:
- case PERSONALITY_XOR:
- multipleOperations(args);
- break;
-
- case PERSONALITY_SUB:
- case PERSONALITY_ABS:
- case PERSONALITY_DIVIDE:
- binaryOperations(args);
- break;
-
- case PERSONALITY_LEQ:
- case PERSONALITY_GEQ:
- case PERSONALITY_EQ:
- unaryOperations(args);
- break;
-
- default:
- args->usage();
- fprintf(stderr, "%s: unknown personality. Specify -P, -B, -S or -M!\n", args->execName);
- exit(1);
- break;
- }
-
- delete args;
-
- return(0);
-}
diff --git a/kmer/meryl/meryl.H b/kmer/meryl/meryl.H
deleted file mode 100644
index 60bd610..0000000
--- a/kmer/meryl/meryl.H
+++ /dev/null
@@ -1,128 +0,0 @@
-#ifndef MERYL_H
-#define MERYL_H
-
-#include "bio++.H"
-
-#define PERSONALITY_MERGE 0xff
-
-#define PERSONALITY_MIN 0x01
-#define PERSONALITY_MINEXIST 0x02
-#define PERSONALITY_MAX 0x03
-#define PERSONALITY_MAXEXIST 0x04
-#define PERSONALITY_ADD 0x05
-#define PERSONALITY_SUB 0x06
-#define PERSONALITY_DIVIDE 0x07
-#define PERSONALITY_ABS 0x08
-
-#define PERSONALITY_AND 0x10
-#define PERSONALITY_NAND 0x11
-#define PERSONALITY_OR 0x12
-#define PERSONALITY_XOR 0x13
-#define PERSONALITY_LEQ 0x14
-#define PERSONALITY_GEQ 0x15
-#define PERSONALITY_EQ 0x16
-
-class merylArgs {
-public:
- merylArgs(int argc, char **argv);
- merylArgs(const char *prefix);
- ~merylArgs();
-
- void usage(void);
- void clear(void);
-
- u64bit hash(kMer const &mer) {
- return(mer.startOfMer(numBuckets_log2));
- };
-
- bool writeConfig(void);
- bool readConfig(const char *prefix);
-public:
- char *execName;
- char *options;
-
- bool beVerbose;
- bool doForward;
- bool doReverse;
- bool doCanonical;
-
- char *inputFile;
- char *outputFile;
- char *queryFile;
-
- u32bit merSize;
- u32bit merComp;
- bool positionsEnabled;
-
- u64bit numMersEstimated;
- u64bit numMersActual;
-
- u64bit numBasesActual;
-
- u64bit mersPerBatch;
- u64bit basesPerBatch;
-
- u64bit numBuckets;
- u32bit numBuckets_log2;
- u32bit merDataWidth;
- u64bit merDataMask;
- u32bit bucketPointerWidth;
-
- u32bit numThreads;
- u64bit memoryLimit;
- u64bit segmentLimit;
- bool configBatch;
- bool countBatch;
- bool mergeBatch;
- u32bit batchNumber;
-
- char *sgeJobName;
- char *sgeBuildOpt;
- char *sgeMergeOpt;
- bool isOnGrid;
-
- u32bit lowCount;
- u32bit highCount;
- u32bit desiredCount;
-
- bool outputCount;
- bool outputAll;
- bool outputPosition;
-
- bool includeDefLine;
- bool includeMer;
-
- u32bit mergeFilesMax;
- u32bit mergeFilesLen;
- char **mergeFiles;
-
- u32bit personality;
-};
-
-
-u64bit
-estimateNumMersInMemorySize(u32bit merSize,
- u32bit mem,
- bool positionsEnabled,
- bool beVerbose);
-
-u32bit
-optimalNumberOfBuckets(u32bit merSize,
- u64bit numMers,
- bool positionsEnabled);
-
-void estimate(merylArgs *args);
-void build(merylArgs *args);
-
-void multipleOperations(merylArgs *args);
-void binaryOperations(merylArgs *args);
-void unaryOperations(merylArgs *args);
-
-void dump(merylArgs *args);
-void dumpThreshold(merylArgs *args);
-void dumpPositions(merylArgs *args);
-void countUnique(merylArgs *args);
-void dumpDistanceBetweenMers(merylArgs *args);
-void plotHistogram(merylArgs *args);
-
-#endif // MERYL_H
diff --git a/kmer/meryl/simple.C b/kmer/meryl/simple.C
deleted file mode 100644
index c4bf019..0000000
--- a/kmer/meryl/simple.C
+++ /dev/null
@@ -1,164 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include <algorithm>
-
-#include "bio++.H"
-#include "meryl.H"
-
-#include "libmeryl.H"
-#include "seqStream.H"
-#include "merStream.H"
-
-using namespace std;
-
-// A very simple mer counter. Allocates a gigantic 32-bit array,
-// populates the array with mers, sorts, writes output.
-
-int
-main(int argc, char **argv) {
- char *inName = 0L;
- char *otName = 0L;
- u32bit merSize = 22;
- u32bit merCompression = 1;
-
- bool doForward = true;
- bool doReverse = false;
- bool doCanonical = false;
-
- speedCounter *C = 0L;
- merStream *M = 0L;
- merylStreamWriter *W = 0L;
-
- u64bit numMers = 0;
-
- u64bit *theMers = 0L;
- u64bit theMersMax = 0;
- u64bit theMersLen = 0;
-
- int arg = 1;
- int err = 0;
- while (arg < argc) {
- if (strcmp(argv[arg], "-i") == 0) {
- inName = argv[++arg];
-
- } else if (strcmp(argv[arg], "-o") == 0) {
- otName = argv[++arg];
-
- } else if (strcmp(argv[arg], "-m") == 0) {
- merSize = atoi(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-f") == 0) {
- doForward = true;
- doReverse = false;
- doCanonical = false;
-
- } else if (strcmp(argv[arg], "-r") == 0) {
- doForward = false;
- doReverse = true;
- doCanonical = false;
-
- } else if (strcmp(argv[arg], "-C") == 0) {
- doForward = false;
- doReverse = false;
- doCanonical = true;
-
- } else if (strcmp(argv[arg], "-c") == 0) {
- merCompression = atoi(argv[++arg]);
-
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- err++;
- }
-
- arg++;
- }
- if (inName == 0L) {
- fprintf(stderr, "no input given with '-i'\n");
- err++;
- }
- if (otName == 0L) {
- fprintf(stderr, "no output given with '-o'\n");
- err++;
- }
- if (err)
- exit(1);
-
-
- {
- M = new merStream(new kMerBuilder(merSize, merCompression),
- new seqStream(inName),
- true, true);
- numMers = M->approximateNumberOfMers();
- delete M;
- }
-
- fprintf(stderr, "Guessing "u64bitFMT" mers in input '%s'\n", numMers, inName);
- fprintf(stderr, "Allocating "u64bitFMT"MB for mer storage.\n", numMers * 8 >> 20);
-
- theMers = new u64bit [numMers];
- theMersLen = 0;
- theMersMax = numMers;
-
- ////////////////////////////////////////
-
- C = new speedCounter(" Counting mers in buckets: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, 1);
- M = new merStream(new kMerBuilder(merSize, merCompression),
- new seqStream(inName),
- true, true);
- //M->setRange(args->mersPerBatch * segment, args->mersPerBatch * segment + args->mersPerBatch);
-
- while (M->nextMer()) {
- if (doForward)
- theMers[theMersLen++] = M->theFMer();
-
- if (doReverse)
- theMers[theMersLen++] = M->theRMer();
-
- if (doCanonical)
- theMers[theMersLen++] = (M->theFMer() <= M->theRMer()) ? M->theFMer() : M->theRMer();
-
- C->tick();
- }
-
- delete C;
- delete M;
-
- fprintf(stderr, "Found "u64bitFMT" mers in input '%s'\n", theMersLen, inName);
-
- if (theMersLen > theMersMax)
- fprintf(stderr, "ERROR: too many mers in input!\n"), exit(1);
-
- ////////////////////////////////////////
-
- fprintf(stderr, "sorting\n");
-
- sort(theMers, theMers + theMersLen);
-
- ////////////////////////////////////////
-
- C = new speedCounter(" Writing output: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, 1);
- W = new merylStreamWriter(otName,
- merSize, merCompression,
- 16,
- false);
-
- kMer mer(merSize);
-
- for (u64bit i=0; i<theMersLen; i++) {
- mer.setWord(0, theMers[i]);
- W->addMer(mer, 1, 0L);
- C->tick();
- }
-
- delete C;
- delete W;
-
- ////////////////////////////////////////
-
- delete [] theMers;
-
- exit(0);
-}
diff --git a/kmer/meryl/test/Makefile b/kmer/meryl/test/Makefile
deleted file mode 100644
index 567f874..0000000
--- a/kmer/meryl/test/Makefile
+++ /dev/null
@@ -1,37 +0,0 @@
-PROG = stupidcount exhaustive
-INCLUDE = -I.. -I../../libutil -I../../libbio -I../../libmeryl
-LIBS = -L.. -L../../libutil -L../../libbio -L../../libmeryl -lmeryl -lbio -lutil -lm
-
-MERSIZE = 26
-
-include ../../Make.compilers
-
-all: $(PROG) test-reduce
-
-stupidcount: stupidcount.C
- $(CXX) $(CXXFLAGS_COMPILE) -c -o stupidcount.o stupidcount.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o stupidcount stupidcount.o $(LIBS)
-
-exhaustive: exhaustive.C kmerlite.H
- $(CXX) $(CXXFLAGS_COMPILE) -c -o exhaustive.o exhaustive.C $(INCLUDE)
- $(CXX) $(CXXLDFLAGS) -o exhaustive exhaustive.o $(LIBS)
-
-test-exhaustive: exhaustive ../meryl ../../leaff/leaff
- ../../leaff/leaff -G 1000 10000 40000 > g.fasta
- ../meryl -B -s g.fasta -o s -m $(MERSIZE) -threads 7
- ./exhaustive -m s -f g.fasta
-
-test-reduce: ../meryl
- ../meryl -B -f -m 20 -s test-seq1.fasta -o 1 # Build the initial table
- ../meryl -Dt -n 0 -s 1 > 2.reduce.fasta # Dump the initial table as fasta
- ../meryl -B -f -m 20 -s 2.reduce.fasta -o 2 # Build a new table on the dumped fasta
- ../meryl -M sub -s 1 -s 2 -o 3 # Remove one copy of each mer
- ../meryl -Dt -n 1 -s 3 # Dump the resulting file
- echo 1 10 9 1 is correct
- touch test-reduce
-
-test:
- ../meryl -B -s test-seq1.fasta -o t -m 20
-
-clean:
- rm -f $(PROG) *.o *.mc??? test-reduce *.seqStore* g.fasta 2.reduce.fasta *.fastaidx
diff --git a/kmer/meryl/test/exhaustive.C b/kmer/meryl/test/exhaustive.C
deleted file mode 100644
index 4c5c9a5..0000000
--- a/kmer/meryl/test/exhaustive.C
+++ /dev/null
@@ -1,171 +0,0 @@
-#include "bio++.H"
-#include "libmeryl.H"
-#include "kmerlite.H"
-
-// This tests that all the mers in an input fasta file are counted
-// properly. It does not test that the meryl output contains exactly
-// those mers, just that those mers are there.
-//
-// If you can fit into one batch, then it _will_ verift that the
-// meryl output is exactly correct.
-//
-// Reads a meryl-format kmer count in chunks. Each chunk is stored
-// in a searchable structure (we should be using, say, an extended
-// existDB, but we're using a balanced binary tree). The entire
-// source fasta file is then streamed against the kmer chunk,
-// decrementing the count for each mer. When the whole file is
-// streamed, any kmers with positive count are reported.
-
-
-// NB: My hacked kazlib returns a pointer to whatever we give it.
-// Since we gave it a pointer to an object, it gives us back a
-// pointer to "a pointer to an object". Hence, this ugliness.
-//
-int
-kMerLiteSort(void const *a, void const *b) {
- kMerLite const *A = *((kMerLite * const *)a);
- kMerLite const *B = *((kMerLite * const *)b);
-
- if (*A < *B) return(-1);
- if (*A > *B) return(1);
- return(0);
-}
-
-
-
-int
-main(int argc, char **argv) {
-
- char *merylCount = 0L;
- char *fastaName = 0L;
-
- int arg=1;
- while (arg < argc) {
-
- if (strcmp(argv[arg], "-m") == 0) {
- merylCount = argv[++arg];
- } else if (strcmp(argv[arg], "-f") == 0) {
- fastaName = argv[++arg];
- } else {
- fprintf(stderr, "unknown option '%s'\n", argv[arg]);
- }
- arg++;
- }
-
- if ((merylCount == 0L) || (fastaName == 0L)) {
- fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]);
- exit(1);
- }
-
-
- // Open the count files
- //
- merylStreamReader *MSR = new merylStreamReader(merylCount);
-
- fprintf(stderr, "Mers are "u32bitFMT" bases.\n", MSR->merSize());
- fprintf(stderr, "There are "u64bitFMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers());
- fprintf(stderr, "There are "u64bitFMT" distinct mers.\n", MSR->numberOfDistinctMers());
- fprintf(stderr, "There are "u64bitFMT" mers total.\n", MSR->numberOfTotalMers());
-
- // Guess how many mers we can fit into 512MB, then report how many chunks we need to do.
-
- u32bit merSize = MSR->merSize();
- u64bit memoryLimit = 700 * 1024 * 1024;
- u64bit perMer = sizeof(kMerLite) + sizeof(dnode_t);
- u64bit mersPerBatch = memoryLimit / perMer;
- u32bit numBatches = MSR->numberOfDistinctMers() / mersPerBatch;
- u32bit batch = 0;
-
- dnode_t *nodes = new dnode_t [mersPerBatch];
- kMerLite *mers = new kMerLite [mersPerBatch];
-
- if (MSR->numberOfDistinctMers() % mersPerBatch)
- numBatches++;
-
- fprintf(stderr, "perMer: "u64bitFMT" bytes ("u64bitFMT" for kMerLite, "u64bitFMT" for dnode_t.\n",
- perMer, (u64bit)sizeof(kMerLite), (u64bit)sizeof(dnode_t));
- fprintf(stderr, "We can fit "u64bitFMT" mers into "u64bitFMT"MB.\n", mersPerBatch, memoryLimit >> 20);
- fprintf(stderr, "So we need "u32bitFMT" batches to verify the count.\n", numBatches);
-
- while (MSR->validMer()) {
- u64bit mersRemain = mersPerBatch;
- dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort);
-
- batch++;
-
- // STEP 1: Insert mersPerBatch into the merDict
- //
- fprintf(stderr, "STEP 1 BATCH "u32bitFMTW(2)": Insert into merDict\n", batch);
- while (MSR->nextMer() && mersRemain) {
- mersRemain--;
-
- mers[mersRemain] = MSR->theFMer();
-
- // initialize the node with the value, then insert the node
- // into the tree using the key
-
- s32bit val = (s32bit)MSR->theCount();
- dnode_init(&nodes[mersRemain], (void *)val);
- dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]);
- }
-
- // STEP 2: Stream the original file, decrementing the count
- //
- fprintf(stderr, "STEP 2 BATCH "u32bitFMTW(2)": Stream fasta\n", batch);
- seqStream *CS = new seqStream(fastaName, true);
- merStream *MS = new merStream(new kMerBuilder(merSize), CS);
-
- kMerLite mer;
- dnode_t *nod;
-
- while (MS->nextMer()) {
- mer = MS->theFMer();
-
- nod = dict_lookup(merDict, &mer);
-
- if (nod != 0L) {
- s32bit val = (s32bit)dnode_get(nod);
- val--;
- dnode_put(nod, (void *)val);
- } else {
- // Unless the whole meryl file fit into our merDict, we cannot warn if
- // we don't find mers.
- //
- if (numBatches == 1) {
- char str[1024];
- fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str));
- }
- }
- }
-
- delete MS;
- delete CS;
-
- // STEP 3: Check every node in the tree to make sure that the counts
- // are exactly zero.
- //
- fprintf(stderr, "STEP 3 BATCH "u32bitFMTW(2)": Check\n", batch);
- nod = dict_first(merDict);
- while (nod) {
- s32bit val = (s32bit)dnode_get(nod);
- kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod);
-
- if (val != 0) {
- char str[1024];
- fprintf(stderr, "Got count "s32bitFMT" for mer '%s'\n",
- val,
- nodmer->merToString(merSize, str));
- }
-
- nod = dict_next(merDict, nod);
- }
-
-
- // STEP 4: Destroy the dictionary.
- //
- fprintf(stderr, "STEP 4 BATCH "u32bitFMTW(2)": Destroy\n", batch);
- while ((nod = dict_first(merDict)))
- dict_delete(merDict, nod);
- dict_destroy(merDict);
- }
-}
diff --git a/kmer/meryl/test/kmerlite.H b/kmer/meryl/test/kmerlite.H
deleted file mode 100644
index 40de26b..0000000
--- a/kmer/meryl/test/kmerlite.H
+++ /dev/null
@@ -1,133 +0,0 @@
-#include "bio++.H"
-
-#ifndef KMER_LITE_H
-#define KMER_LITE_H
-
-////////////////////////////////////////
-//
-// This is kMerLite -- derived from kMer.H, removing
-// most of the accessors.
-//
-// Assumes that KMER_WORDS is already defined.
-
-
-class kMerLite {
-public:
-
- // Used by some of the test routines.
- void dump(void) const {
- for (u32bit i=0; i<KMER_WORDS; i++)
- fprintf(stderr, "kMerLite["u32bitFMTW(2)"] = "u64bitHEX"\n", i, _wd[i]);
- };
-
-
-public:
- void copy(kMer const &that) {
- for (u32bit i=0; i<KMER_WORDS; i++)
- _wd[i] = that.getWord(i);
- };
- void clear(void) {
- for (u32bit i=0; i<KMER_WORDS; i++)
- _wd[i] = u64bitZERO;
- };
-
-public:
- kMerLite() {
- clear();
- };
- kMerLite(kMer const &that) {
- copy(that);
- };
- ~kMerLite() {
- };
-
- kMerLite &operator=(kMer const &that) {
- copy(that);
- return(*this);
- };
-
- char *merToString(u32bit merSize, char *instr) const {
- u32bit lastWord = merSize >> 5;
- char *str = instr;
-
- if ((merSize & u32bitMASK(6)) == 0)
- lastWord++;
-
- // We build the string right to left, print any partial word
- // first, then print whole words until we run out of words to
- // print.
-
- if (merSize & u32bitMASK(5)) {
- ::merToString(merSize & u32bitMASK(5), _wd[lastWord], str);
- str += merSize & u32bitMASK(5);
- }
-
- while (lastWord > 0) {
- lastWord--;
- ::merToString(32, _wd[lastWord], str);
- str += 32;
- }
-
- return(instr);
- };
-
-#if KMER_WORDS == 1
-
- bool operator!=(kMerLite const &r) const { return(_wd[0] != r._wd[0]); };
- bool operator==(kMerLite const &r) const { return(_wd[0] == r._wd[0]); };
- bool operator<(kMerLite const &r) const { return(_wd[0] < r._wd[0]); };
- bool operator>(kMerLite const &r) const { return(_wd[0] > r._wd[0]); };
- bool operator<=(kMerLite const &r) const { return(_wd[0] <= r._wd[0]); };
- bool operator>=(kMerLite const &r) const { return(_wd[0] >= r._wd[0]); };
-
-#else
-
- bool operator!=(kMerLite const &r) const {
- u64bit res = u64bitZERO;
- for (u32bit i=KMER_WORDS; i--; )
- res |= _wd[i] ^ r._wd[i];
- return(res != u64bitZERO);
- };
- bool operator==(kMerLite const &r) const {
- u64bit res = u64bitZERO;
- for (u32bit i=KMER_WORDS; i--; )
- res |= _wd[i] ^ r._wd[i];
- return(res == u64bitZERO);
- };
-
- bool operator<(kMerLite const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (_wd[i] < r._wd[i]) return(true);
- if (_wd[i] > r._wd[i]) return(false);
- }
- return(false);
- };
- bool operator>(kMerLite const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (_wd[i] > r._wd[i]) return(true);
- if (_wd[i] < r._wd[i]) return(false);
- }
- return(false);
- };
- bool operator<=(kMerLite const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (_wd[i] < r._wd[i]) return(true);
- if (_wd[i] > r._wd[i]) return(false);
- }
- return(true);
- };
- bool operator>=(kMerLite const &r) const {
- for (u32bit i=KMER_WORDS; i--; ) {
- if (_wd[i] > r._wd[i]) return(true);
- if (_wd[i] < r._wd[i]) return(false);
- }
- return(true);
- };
-#endif
-
-private:
- u64bit _wd[KMER_WORDS];
-};
-
-
-#endif // KMER_LITE_H
diff --git a/kmer/meryl/test/stupidcount.C b/kmer/meryl/test/stupidcount.C
deleted file mode 100644
index 3d9d3c5..0000000
--- a/kmer/meryl/test/stupidcount.C
+++ /dev/null
@@ -1,38 +0,0 @@
-#include "bio++.H"
-
-// Reads a sequence file, outputs a list of the mers in it. You can
-// then pipe this to unix sort and uniq to do a mercount. You
-// probably don't want to count large things this way...
-
-int
-main(int argc, char **argv) {
- char *seqName = 0L;
- u32bit merSize = 20;
-
- int arg=1;
- while (arg < argc) {
- if (strcmp(argv[arg], "-s") == 0) {
- seqName = argv[++arg];
- } else if (strcmp(argv[arg], "-m") == 0) {
- merSize = strtou32bit(argv[++arg], 0L);
- }
- arg++;
- }
-
- if (seqName == 0L) {
- fprintf(stderr, "usage: %s [-m mersize] -s seqfile.fasta\n", argv[0]);
- exit(1);
- }
-
- seqStream *CS = new seqStream(seqName, true);
- merStream *MS = new merStream(new kMerBuilder(merSize), CS);
- char str[1024];
-
- while (MS->nextMer())
- fprintf(stdout, "%s\n", MS->theFMer().merToString(str));
-
- delete MS;
- delete CS;
-
- exit(0);
-}
diff --git a/kmer/meryl/test/test-seq1.fasta b/kmer/meryl/test/test-seq1.fasta
deleted file mode 100644
index 4e2c60d..0000000
--- a/kmer/meryl/test/test-seq1.fasta
+++ /dev/null
@@ -1,8 +0,0 @@
-> 1 A 20 CG 0 T
-AAAAAAAAAAAAAAAAAAAAGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCG
-> 1 A 1 CG 0 T ----not-a-mer------ -----is-a-mer-------
-AAAAAAAAAAAAAAAAAAAANGCGCGCGCGCGCGCGCGCGNCGCGCGCGCGCGCGCGCGCG
-> (zero 63 bases)
-ATNCGGATYCGATCGASCHJAGSVHYWERIGHWEEIRVHSDKFVHWIERVHIWRVHKSDFVKS
-> 20 T
-NNNNTTTTTTTTTTTTTTTTTTTTNNNNTTTTTTTTTTTTTTTTTTTTNNNN
diff --git a/kmer/meryl/test/test-seq2.fasta b/kmer/meryl/test/test-seq2.fasta
deleted file mode 100644
index 4d53253..0000000
--- a/kmer/meryl/test/test-seq2.fasta
+++ /dev/null
@@ -1,18 +0,0 @@
-> 2 A 20 CG 0 T
-AAAAAAAAAAAAAAAAAAAAN
-AAAAAAAAAAAAAAAAAAAAN
-AAAAAAAAAAAAAAAAAAAAN
-AAAAAAAAAAAAAAAAAAAAN
-AAAAAAAAAAAAAAAAAAAAN
-AAAAAAAAAAAAAAAAAAACN
-AAAAAAAAAAAAAAAAAAACN
-AAAAAAAAAAAAAAAAAAACN
-AAAAAAAAAAAAAAAAAAACN
-AAAAAAAAAAAAAAAAAAACN
-AAAAAAAAAAAAAAAAAAAAAAAGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCG
-> 1 A 1 CG 0 T ----not-a-mer------ -----is-a-mer-------
-AAAAAAAAAAAAAAAAAAAANGCGCGCGCGCGCGCGCGCGNCGCGCGCGCGCGCGCGCGCG
-> (zero 63 bases)
-ATNCGGATYCGATCGASCHJAGSVHYWERIGHWEEIRVHSDKFVHWIERVHIWRVHKSDFVKS
-> 20 T
-NNNNTTTTTTTTTTTTTTTTTTTTNNNNTTTTTTTTTTTTTTTTTTTTNNNN
diff --git a/kmer/meryl/test/test-seq3.fasta b/kmer/meryl/test/test-seq3.fasta
deleted file mode 100644
index 5b42b3e..0000000
--- a/kmer/meryl/test/test-seq3.fasta
+++ /dev/null
@@ -1,2 +0,0 @@
->
-ACGCTCAGCTACTACGACTTAGAGAAAATAGCGATATAGCGATCGATCGATTAGAGA
diff --git a/kmer/meryl/testPositionBias.C b/kmer/meryl/testPositionBias.C
deleted file mode 100644
index 7e17f72..0000000
--- a/kmer/meryl/testPositionBias.C
+++ /dev/null
@@ -1,117 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-
-#include "bio++.H"
-#include "libmeryl.H"
-
-
-u32bit *
-collectCounts(char *name, u32bit base) {
- merylStreamReader *A = new merylStreamReader(name);
- u32bit *C = new u32bit [4*4*4];
- char S[32];
- u32bit code = 0;
-
- for (u32bit i=0; i<4*4*4; i++)
- C[i] = 0;
-
- while (A->nextMer()) {
- A->theFMer().merToString(S);
-
- code = 0;
- code |= letterToBits[S[base]];
- code <<= 2;
- code |= letterToBits[S[base+1]];
- code <<= 2;
- code |= letterToBits[S[base+2]];
-
- C[code] += A->theCount();
-
- kMer R = A->theFMer();
- R.reverseComplement();
- R.merToString(S);
-
- code = 0;
- code |= letterToBits[S[base]];
- code <<= 2;
- code |= letterToBits[S[base+1]];
- code <<= 2;
- code |= letterToBits[S[base+2]];
-
- C[code] += A->theCount();
- }
-
- delete A;
-
- return(C);
-}
-
-
-void
-showBias(u32bit base=5) {
- u32bit *A = collectCounts("CNPT3", base);
- u32bit *B = collectCounts("25.errorless", base);
- u32bit *C = collectCounts("25.errorless.simulated", base);
-
- for (u32bit i=0; i<4*4*4; i++) {
- double bp = 0.0;
- double cp = 0.0;
-
- if (A[i] > 0) {
- bp = (double)B[i] / (double)A[i];
- cp = (double)C[i] / (double)A[i];
- }
-
- fprintf(stdout, "%c%c%c "u32bitFMTW(3)" A "u32bitFMTW(6)" B "u32bitFMTW(6)" %.5f C "u32bitFMTW(6)" %.5f\n",
- bitsToLetter[(i >> 4) & 0x00000003],
- bitsToLetter[(i >> 2) & 0x00000003],
- bitsToLetter[(i >> 0) & 0x00000003],
- i,
- A[i],
- B[i],
- bp,
- C[i],
- cp);
- }
-}
-
-
-double
-computeRMSD(u32bit base) {
- u32bit *A = collectCounts("CNPT3", base);
- u32bit *B = collectCounts("25.errorless", base);
- u32bit *C = collectCounts("25.errorless.simulated", base);
-
- double rmsd = 0;
-
- for (u32bit i=0; i<4*4*4; i++) {
- double bp = 0.0;
- double cp = 0.0;
-
- if (A[i] > 0) {
- bp = (double)B[i] / (double)A[i];
- cp = (double)C[i] / (double)A[i];
- }
-
- rmsd += (bp - cp) * (bp - cp);
- }
-
- rmsd /= 4*4*4;
-
- return(sqrt(rmsd));
-}
-
-
-
-
-
-int
-main(int argc, char **argv) {
-
- showBias(5);
-
- //for (u32bit i=0; i<23; i++)
- // fprintf(stdout, "rmsd "u32bitFMTW(2)" %f\n", i, computeRMSD(i));
-}
diff --git a/kmer/meryl/unaryOp.C b/kmer/meryl/unaryOp.C
deleted file mode 100644
index 06c7446..0000000
--- a/kmer/meryl/unaryOp.C
+++ /dev/null
@@ -1,59 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "meryl.H"
-#include "libmeryl.H"
-
-
-void
-unaryOperations(merylArgs *args) {
-
- if (args->mergeFilesLen != 1) {
- fprintf(stderr, "ERROR - must have exactly one file!\n");
- exit(1);
- }
- if (args->outputFile == 0L) {
- fprintf(stderr, "ERROR - no output file specified.\n");
- exit(1);
- }
- if ((args->personality != PERSONALITY_LEQ) &&
- (args->personality != PERSONALITY_GEQ) &&
- (args->personality != PERSONALITY_EQ)) {
- fprintf(stderr, "ERROR - only personalities lessthan, lessthanorequal,\n");
- fprintf(stderr, "ERROR - greaterthan, greaterthanorequal, and equal\n");
- fprintf(stderr, "ERROR - are supported in unaryOperations().\n");
- fprintf(stderr, "ERROR - this is a coding error, not a user error.\n");
- exit(1);
- }
-
- // Open the input and output files -- we don't know the number
- // unique, distinct, and total until after the operation, so we
- // leave them zero.
- //
- merylStreamReader *R = new merylStreamReader(args->mergeFiles[0]);
- merylStreamWriter *W = new merylStreamWriter(args->outputFile, R->merSize(), R->merCompression(), R->prefixSize(), R->hasPositions());
-
- switch (args->personality) {
- case PERSONALITY_LEQ:
- while (R->nextMer())
- if (R->theCount() <= args->desiredCount)
- W->addMer(R->theFMer(), R->theCount(), R->thePositions());
- break;
-
- case PERSONALITY_GEQ:
- while (R->nextMer())
- if (R->theCount() >= args->desiredCount)
- W->addMer(R->theFMer(), R->theCount(), R->thePositions());
- break;
-
- case PERSONALITY_EQ:
- while (R->nextMer())
- if (R->theCount() == args->desiredCount)
- W->addMer(R->theFMer(), R->theCount(), R->thePositions());
- break;
- }
-
- delete R;
- delete W;
-}
diff --git a/samtools b/samtools
deleted file mode 120000
index 3bfd233..0000000
--- a/samtools
+++ /dev/null
@@ -1 +0,0 @@
-samtools-0.1.19
\ No newline at end of file
diff --git a/samtools-0.1.19/.gitignore b/samtools-0.1.19/.gitignore
deleted file mode 100644
index bb605d4..0000000
--- a/samtools-0.1.19/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-*.o
-.*.swp
-*.a
-*.dSYM
diff --git a/samtools-0.1.19/AUTHORS b/samtools-0.1.19/AUTHORS
deleted file mode 100644
index 95afabb..0000000
--- a/samtools-0.1.19/AUTHORS
+++ /dev/null
@@ -1,20 +0,0 @@
-Heng Li from the Sanger Institute wrote most of the initial source codes
-of SAMtools and various converters.
-
-Bob Handsaker from the Broad Institute is a major contributor to the
-SAM/BAM specification. He designed and implemented the BGZF format, the
-underlying indexable compression format for the BAM format. BGZF does
-not support arithmetic between file offsets.
-
-Jue Ruan for the Beijing Genome Institute designed and implemented the
-RAZF format, an alternative indexable compression format. RAZF supports
-arithmetic between file offsets, at the cost of increased index file
-size and the full compatibility with gzip. RAZF is optional and only
-used in `faidx' for indexing RAZF compressed fasta files.
-
-Colin Hercus updated novo2sam.pl to support gapped alignment by
-novoalign.
-
-Petr Danecek contributed the header parsing library sam_header.c and
-sam2vcf.pl script and added knet support to the RAZF library.
-
diff --git a/samtools-0.1.19/COPYING b/samtools-0.1.19/COPYING
deleted file mode 100644
index 82fa2f4..0000000
--- a/samtools-0.1.19/COPYING
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License
-
-Copyright (c) 2008-2009 Genome Research Ltd.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
\ No newline at end of file
diff --git a/samtools-0.1.19/ChangeLog.old b/samtools-0.1.19/ChangeLog.old
deleted file mode 100644
index 19aefae..0000000
--- a/samtools-0.1.19/ChangeLog.old
+++ /dev/null
@@ -1,3875 +0,0 @@
-commit db2ad3e19068cbafde72ecde75d0638bbb3598ba
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 16 14:45:17 2012 -0500
-
- removed downsample.c
-
-commit 6c55c576903992c6fef148fe3b606fbc8bd10655
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 16 14:45:06 2012 -0500
-
- print to output
-
-commit db1044a34e6049c87eaa63c39ed6e56f03e7d4c1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 16 14:39:34 2012 -0500
-
- removed sample
-
- Downsampling already exists in "view". View also keeps pairing while "sample" does not.
-
-commit ffdeed3e5d4a530bfdf6f9ba97fff0ba7add6cba
-Merge: 2daad7b accf026
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 16 14:22:15 2012 -0500
-
- Merge branch 'master' of github.com:lh3/samtools
-
-commit accf0260fd1117e10047344345d40b31a9ec31bb
-Merge: 9134e0d c554160
-Author: Heng Li <lh3 at me.com>
-Date: Thu Feb 16 11:21:14 2012 -0800
-
- Merge pull request #8 from nh13/master
-
- Patches
-
-commit c554160df16ec7748cfdda4c7b54c641be7b809f
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 14:06:52 2012 -0500
-
- * more README.md work
-
-commit 2a81ffe349208d917666808fbc9f3041e0cb57de
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 14:06:10 2012 -0500
-
- * more README work
-
-commit fb3125f732715f62cded8685a23a002a96ce009b
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 14:05:19 2012 -0500
-
- * more README work
-
-commit 444d41002c37e1c3d0f9208b4a88126c47276386
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 14:02:13 2012 -0500
-
- * updating README
-
-commit dec53cb1043fe7efadfde75fa2fd39b76de22e54
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 13:55:01 2012 -0500
-
- updating the README for markdown syntax
-
-commit 798da18c346dca8ec6005582a0ddb1d5420b04ca
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 13:48:35 2012 -0500
-
- adding a README with the current differences between this repository and
- the official one
-
-commit 4d22d86c0f28636662f2144a88cd168e104c4275
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 13:35:03 2012 -0500
-
- adding "samtools sample" to the main
-
-commit 893c25a37c21005dc42f45d45e9ad78ddc5f29bb
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 13:33:51 2012 -0500
-
- * removing some compile flags to work with OS X
-
-commit 7ac22f72fdc32edd5c24af6baebfa7db5faf8e7b
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:47:14 2012 -0500
-
- Check write filehandle after opening for write. tamw/tamr is a union type, so change is only semantic.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit fef53330416631690f60fdff42b6e43d764170dc
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:44:59 2012 -0500
-
- Catch and report invalid BAM header, instead of segfaulting later on.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 5cc013fe4930bf9b6e7963aab1cd4a3c94f695bc
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:44:16 2012 -0500
-
- Add downsample to examples.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit b3fa9e7071532905a81dc7aa48eadc24b8c8846b
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:43:48 2012 -0500
-
- Adjust for leading hard clip on colorspace reads.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 1a9296c1389469d1c1db5b8069f0e11ffcc8abb2
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:42:52 2012 -0500
-
- Add samtools sample command, contributed by Davide Cittaro <davide.cittaro at ifom-ieo-campus.it>.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 2a804f3379748aeba944f1dec306dd726ff3235e
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:42:07 2012 -0500
-
- Add samtools qa command, contributed by Roman Valls Guimera <roman.valls.guimera at scilifelab.se>.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 0f3207fe8fd93e44d40fcf57204079c8c06d24a6
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:39:08 2012 -0500
-
- Makefile cleanup - allow CC, CFLAGS, LDFLAGS to be passed on make command line. Use LDFLAGS in samtools compile.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 6e7df604025f6a86881bf7f4a16f30e15d31538a
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:31:15 2012 -0500
-
- Allow max_mem for sort to be specified with units.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit f12ebcaf6e60d34180a27d70e09b743cef140b98
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:29:11 2012 -0500
-
- Allow user defined [lowercase] tags in header elements.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 50b931fa3312dc109537a4260698ddecd0f06a05
-Author: Jonathan Manning <jonathan.manning at lifetech.com>
-Date: Thu Feb 16 10:27:11 2012 -0500
-
- Check lowerbound in text entry box to avoid segfault in tview. Remove redundant call to bam_aux_get.
-
- Signed-off-by: Nils Homer <nils.homer at lifetech.com>
-
-commit 5e729da5190949a813d20d329eab7ddb661816bd
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 10:31:48 2012 -0500
-
- * fixing overflow/underflow in integer parsing
-
-commit fa50a4330b9abedaf07c26e13d31f05e57f1d319
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 10:30:40 2012 -0500
-
- * updating help message for samtools depth
-
-commit 79e52c9624b6dd3bdfdf439f4b4bc6f774c230a4
-Author: Nils Homer <nils.homer at lifetech.com>
-Date: Thu Feb 16 10:29:32 2012 -0500
-
- * adding support for outputting a circos histogram file in "samtools depth". Use
- the "-c/-B" options.
-
-commit 2daad7b52daa86561c0fb65fe366691fad9f5ed3
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 16 09:31:57 2012 -0500
-
- bugfix: wrong SP; missing DV in the VCF hdr
-
-commit 9134e0d5047c281ef3bd53da91771d4814a5131c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 8 11:19:12 2012 -0500
-
- missing support of DV
-
-commit 34ebf12078c1d1015a0b8b9a9221243a60b22893
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 8 11:08:56 2012 -0500
-
- new BCF DV format: number of variant reads
-
-commit 9589d3312fa2d076f48bdd68e2a5edd419c8070c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jan 10 10:30:27 2012 -0500
-
- scale depth to quality (hidden option)
-
-commit 704473e14668333ecaca5fb7b238af405c43e3b1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jan 10 10:18:17 2012 -0500
-
- really nothing
-
-commit 01b307fd287962372bbf07461c88b54f41636817
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Dec 7 13:07:42 2011 -0500
-
- added an example containing 'B'
-
-commit c678791f0451ceb9205c1ab5c52c84641863c99a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Dec 3 12:10:30 2011 -0500
-
- 'B' now moves backward w.r.t. the query
-
-commit 152119bc06a073933ca830e8e1407538e44626cc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Dec 2 10:50:12 2011 -0500
-
- better consensus; a little more robust
-
-commit 454da4754ac503edda5b1329b67757d797e46e07
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Dec 2 00:20:22 2011 -0500
-
- in pileup call remove_B()
-
-commit ff2bcac1cc078ba1879f18c89cfae314439d7086
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Dec 2 00:17:32 2011 -0500
-
- working on a few toy examples
-
-commit 745ca7260158d6df7897b52598033ffb055a9e4f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Dec 1 22:55:39 2011 -0500
-
- bam_remove_B(); not tested
-
-commit 07e4cdc7300abfcc82e03105b4689f95cab551cd
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Nov 10 12:58:55 2011 -0500
-
- baseQ threshold on plain pipleup; removed -E
-
-commit 322ebf2082dfa91df44b3a996d26c85357e5d5a2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Oct 19 09:28:04 2011 -0400
-
- fixed two gcc warnings
-
-commit a632457b4c4adc50d833b56b5a5231feafaf8193
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Oct 4 10:13:23 2011 -0400
-
- change size_t to uint32_t in bam_header_t
-
- This may cause issues on 64-bit big-endian machines. Reported and fixed by Paolo Emilio Mazzon.
-
-commit af31bf5a78aea03baf6eb90fe50076549d499f6e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Sep 26 20:17:57 2011 -0400
-
- rename pad2unpad to depad
-
-commit 77b198b73dfad1048e5d1c5a64aa75ee7b90f596
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Sep 23 01:22:40 2011 -0400
-
- convert padded BAM to unpadded BAM
-
-commit adb9e2342b7b7501d9527d3c23afab10469ae2c6
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Sep 7 11:40:50 2011 -0400
-
- generate template cigar with "fixmate"
-
-commit 46e5ab445a0fe880216cbc0daf1225725b569d7a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Sep 2 12:50:18 2011 -0400
-
- update kseq.h to the latest version
-
-commit 68e9e4a73eb91405bb3e56bf0cdaf12d1b487abb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Sep 2 12:44:45 2011 -0400
-
- Release samtools-0.1.18
-
-commit aa06bdadb2d109a79f927f478102f96a1f5fd258
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Sep 2 12:14:17 2011 -0400
-
- updated the revision number
-
-commit 267e1e1b6e54c0ab24f94cd9aee9cbd2d1923f9f
-Merge: 19ff1d3 aebab30
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Sep 2 12:13:08 2011 -0400
-
- Merge https://github.com/lh3/samtools into reduce
-
- Conflicts:
- bam_md.c
-
- Fixed a few typos in the merge
-
-commit aebab302399c24eaa6c5ab79d13d6bd5e2e9ea9a
-Merge: c2c63d0 da62663
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Sep 2 09:03:49 2011 -0700
-
- Merge pull request #4 from peterjc/x_equals2
-
- Implement basic support for =/X CIGAR operations
-
-commit 19ff1d3d7f47d7e61b121292aefe5a74bb8a18d2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Aug 25 16:38:12 2011 -0400
-
- reduce BAM size (experimental)
-
-commit da626630fd98fd4e07ceb4d58c5c9a42d312a85d
-Author: peterjc <p.j.a.cock at googlemail.com>
-Date: Mon Aug 22 06:58:08 2011 +0100
-
- Support =/X CIGAR operations (treated like M)
-
-commit 461d8003529db77a4d5ecbd108312e868b051a3d
-Author: peterjc <p.j.a.cock at googlemail.com>
-Date: Mon Aug 22 05:52:56 2011 +0100
-
- Define CIGAR equals and X operationss (7 and 8)
-
-commit c2c63d067113baab41f3bc35fb28f4f00578accb
-Merge: 7ab3ef3 9a0ed9a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Aug 18 17:21:54 2011 -0700
-
- Merge pull request #3 from peterjc/x_equals
-
- Accept SAM files using = in CIGAR (treats X and = as M)
-
-commit 9a0ed9a6b85c7981465f459300208dbd93e3c6f5
-Author: peterjc <p.j.a.cock at googlemail.com>
-Date: Thu Aug 18 19:28:52 2011 +0100
-
- Accept SAM files using = in CIGAR (treats X and = as M)
-
-commit 7ab3ef388c1eb34d7912fd70cc5656c955240263
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Aug 8 10:22:22 2011 -0400
-
- bugfix: indexing takes huge memory
-
- This happens when an unmapped mate has coordinate 1. Thank Joel Martin for the fix.
-
-commit a3f6738593e944354a8f75306687d8b3acf08bf1
-Merge: a8bdca9 bc67ea2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Aug 8 09:52:26 2011 -0400
-
- Merge branch 'master' of github.com:lh3/samtools
-
-commit bc67ea225da653f36a70b38382d6111dd494f659
-Author: Petr Danecek <pd3 at sanger.ac.uk>
-Date: Thu Jul 28 20:03:16 2011 +0100
-
- Variant Distance Bias
-
-commit deb578f0c49d0b7d8c3bc6be220b4d67e2e7dfdf
-Author: Petr Danecek <pd3 at sanger.ac.uk>
-Date: Tue Jul 26 09:57:37 2011 +0100
-
- If there is only one RG tag present in the header and reads are not annotated, don't refuse to work but use the tag instead.
-
-commit a8bdca9cf482a637b89ee4f98469a93e0ab5e69b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jul 25 10:10:55 2011 -0400
-
- bugfix: LRT2=nan
-
-commit 0afe33137d046a3e849eeb4a54590f27cbad4228
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 22 21:55:38 2011 -0400
-
- fixed a bug/typo
-
-commit 62d5849658c10222d40308c6b53ab4f99a448494
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 15 16:04:19 2011 -0400
-
- allow to set see in subsampling
-
-commit 5f46243824cc9435b167973e1d51e13128794ea1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 15 15:54:47 2011 -0400
-
- support subsampling
-
-commit 5e55b6f34fc86cba7cf98d52ccaed405c3ffabbc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 15 15:53:38 2011 -0400
-
- support indels
-
-commit f31c162926d6f43e8b60171789a258d02e1f9be5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jul 7 17:02:33 2011 -0400
-
- do not count indel with "view -Y"
-
-commit e412dae587883b4c17e5fbf4b7c33f38bfa8458a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jul 7 00:35:25 2011 -0400
-
- for WIN32 compatibility
-
-commit 70a52501bcfa63824749893a5ab8ed3c38e34958
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jul 7 00:32:46 2011 -0400
-
- for WIN32 compatibility
-
-commit 00438f14ed5984f08e8f7645a9b95644a812f969
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jul 6 23:41:45 2011 -0400
-
- fixed an uninitialized variable
-
-commit 7609c4a01059c326544b3d0142dfe9c4229d68c6
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jul 6 23:39:31 2011 -0400
-
- fixed an uninitialized variable
-
-commit cec7189a412f80ccb068a73bd28528915c16b0bf
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jul 6 22:53:19 2011 -0400
-
- Release samtools-0.1.17
-
-commit 93c06a249de3bb666029bf07b66de5e8e5e314fa
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jul 6 09:46:09 2011 -0400
-
- bugfix: incorrect idxstats for the last seq
-
- Again, this bug is caused by 3rd-party code for the sorting order checking.
-
-commit 84f6ca62db6e27b8c4c711e7b5f3ca704bf27b4f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jul 5 23:30:23 2011 -0400
-
- output mapping quality in the old pileup format
-
-commit 362e05fd670886acaede69b864903d730b9db3ca
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jul 5 21:59:22 2011 -0400
-
- added a brief description of the VCF format
-
-commit e690a696468205e0cc4560016361c997660dd496
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jul 5 16:23:10 2011 -0400
-
- improved samtools manual page
-
-commit 362b4a1408ef3c32311d638aa8d85ce39c1c7b2d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jul 5 15:58:29 2011 -0400
-
- merge bcftools.1 to samtools.1
-
-commit 643e0e61ba7266efbc9e5bfcb8e41f369ba2ce0a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Jul 5 13:39:02 2011 -0400
-
- mpileup: when region set, set reference properly
-
-commit 613e4d67624a94f62563935fbd5cc294df69605a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jul 4 23:29:02 2011 -0400
-
- compute the min PL diff
-
-commit 5b7d5d3f52b97ca42c8500eede808dab88a46a53
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jul 4 22:57:48 2011 -0400
-
- rename trio.c to mut.c
-
-commit 84fe96ad64b0365ead93a4115d1684b9bebb98fc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Jul 3 15:38:51 2011 -0400
-
- added pair caller interface; not tested
-
-commit 2f2867b87b84c35319cc416d6173819d5c8a4e8c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Jul 3 15:24:23 2011 -0400
-
- inital implementation of a pair caller
-
-commit e97653cf2ad653c95886933c42a2b5492ccab5ff
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Jul 3 00:06:28 2011 -0400
-
- convert bam to single-end fastq
-
-commit e8013e11f7a8db0a8d18c60d130169cca39bf2bd
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Jul 2 14:39:18 2011 -0400
-
- improve BED parsing
-
-commit 1025714325fdc636aeee47a76db8dafbbbfde64b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 1 14:19:54 2011 -0400
-
- update the manual page
-
-commit 8022d0039dff47b1c11b2421357d510c1f28ae15
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 1 14:17:03 2011 -0400
-
- output the best constrained genotypes in trio
-
-commit 18c87295e12f5bebafdcae00d52000fb94c8a566
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jul 1 11:18:14 2011 -0400
-
- added documentations for view -T
-
-commit daf7a8d96bd495296bf7c7d99cddb808a3ced7d5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 22:45:20 2011 -0400
-
- fixed a bug in writing SP
-
-commit e5c32bf9b28c6e3e861db88de56b5dbe11058b61
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 22:35:25 2011 -0400
-
- optionally output read positions in mpileup
-
-commit 1008051155ec994c1901e18f3eb03ea32a62e5d7
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 22:17:25 2011 -0400
-
- make faidx works with <2GB lines
-
-commit 2daebb63762425dd3074ddf71582ad189001e394
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 17:28:58 2011 -0400
-
- fixed an issue in the trio caller and the indel caller
-
-commit 9fdd52cf0716fb342a94946433d564b28b230835
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 13:34:01 2011 -0400
-
- Added trio caller; NOT tested yet
-
-commit ea22a8ed83625e9c82382b56acc42a2d9cfd17e5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 11:42:29 2011 -0400
-
- convert PL to 10-likelihood GL
-
-commit 10d7065267b0d12c2bfcb6c70204fb6944cd395d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jun 30 10:49:05 2011 -0400
-
- fix a compatibility issue with the new bcftools
-
-commit d340f01f609c61b719d38a6a55629a3fc899e1cd
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Jun 26 23:41:20 2011 -0400
-
- allow to ignore RG
-
-commit d6321faf98ebfe899b9409fb23c90a4aa8c6b542
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Jun 5 23:05:21 2011 -0400
-
- fixed a bug in SO checking due to a recent change
-
-commit bc995abf666d0c9ab4258f6c1b3518a45a89209f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jun 3 14:45:36 2011 -0400
-
- update the version number
-
-commit 9e7cd83a08383858d008e0ccb2238a2b93831d6c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Jun 3 14:43:12 2011 -0400
-
- smarter way to parse a region string
-
-commit e58a90a0fde54053dac65352b34c13c3fea815fc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jun 1 14:36:22 2011 -0400
-
- output LRT2 instead of LRT1
-
-commit 08f78c9af3e5661f04f80bef424232de721dba03
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jun 1 14:02:28 2011 -0400
-
- genotype test, but assuming 1-degree
-
-commit 587b852340d7e60f6f7cf474a92ef77aeab46018
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jun 1 12:55:19 2011 -0400
-
- perform 2-degree test by default
-
-commit 3d38e403c5c830478b7eb157a484776997440501
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jun 1 12:44:34 2011 -0400
-
- fixed a typo; but the result is still not good
-
-commit 06291624f7dcc57445676f3be25d0bc355dd7110
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jun 1 12:24:18 2011 -0400
-
- fixed a typo
-
-commit 63b98aa33636b0d82a435bf49153c8c1502e7d42
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jun 1 12:23:37 2011 -0400
-
- added HWE+F<0 filter
-
-commit 37d926e8999999b593d0637ab7dc379dbd3d6006
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed May 4 10:11:59 2011 -0400
-
- improved sorting order checking in index
-
- Patches from Jonathan Manning
-
-commit 1c2dc6762c5f7cd946046b53346513f2f9761dbf
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue May 3 23:09:05 2011 -0400
-
- added r^2 estimate; added Brent's method
-
-commit c2d3bcd8f98e31668b5f1321222fbc6fd6336e75
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun May 1 23:45:23 2011 -0400
-
- combine several utilites into vcfutils.lua
-
-commit be2e7362d7593ea4d03fb33cdb6af2aa096ca6c4
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 27 21:09:22 2011 -0400
-
- minor warning
-
-commit 683ef0443860813d743cf84fa86dda9bfaf5445a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 27 10:10:38 2011 -0400
-
- added versioning
-
-commit ed72f25ec85671f7646dbc92fa7b5b1dda427f7d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 27 10:04:02 2011 -0400
-
- Output ML allele count
-
-commit 2a9e36d2d6c405b2411ca47458f028ada8fe1000
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Apr 26 16:14:20 2011 -0400
-
- use ar -s
-
-commit 7a4f54e6dbcd7c94acbb3f1050a93f94b8a07949
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Apr 23 01:22:31 2011 -0400
-
- added another type of LRT
-
-commit b9c5e84762a4aacce3a3771b51ea80967c79a2e5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 22 16:00:31 2011 -0400
-
- added version
-
-commit 8fad6677c5952efd67391581d64e67e02e7f6e68
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 22 00:30:19 2011 -0400
-
- remove the pileup command
-
-commit 3a962fb6ebf779de70f9e6effb2d8701a9aa3dd9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 21 23:10:45 2011 -0400
-
- Release 0.1.16 (r963:234)
-
-commit b4d683cffbd98c43f05aff8610b37d63dd7e54aa
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 21 12:44:44 2011 -0400
-
- fixed a bug when coordinate-less reads are on the reverse strand
-
-commit c5ec45a128f409debc6a56a798024f53004037dc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 20 11:36:52 2011 -0400
-
- added option '-f' to merge to avoid overwritting
-
-commit 68d431531370d24907c01a27f166f2341d7c4d35
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 20 10:26:58 2011 -0400
-
- do not print a warning
-
-commit 32922607e51ad2260c337eb022b9e4aedacb049f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 20 10:21:06 2011 -0400
-
- Added ldpair to compute LD between requested pairs
-
-commit b8d6fa71b91678fa02338257e0707d1e5ca098dd
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 17 21:51:43 2011 -0400
-
- On a toy sample, type "B" seems to be accepted
-
-commit 0e7ee9a6bb4029184202aa6e6738105ba0c0510b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 17 21:21:20 2011 -0400
-
- added type "B"; not tested yet
-
-commit a513dfad0ac0062b03871eb6ecf26cb8d18dc895
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 17 19:25:54 2011 -0400
-
- fixed a bug in bedidx.c: input BED not sorted
-
-commit de1e192bb0a8a762a54a6eee81d882fab01c3d32
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 17 18:51:08 2011 -0400
-
- by default, always perform posterior chi^2
-
-commit df6e0d1099895fc6cd7a19dc89fba95ed6654d35
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Apr 16 12:33:28 2011 -0400
-
- added debugging
-
-commit 8ce52e024dc2ef361dbd5399c232163055057e70
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Apr 16 00:59:05 2011 -0400
-
- avoid a segfault given wrong input
-
-commit e66b6684fc9a397f91ec29fdeecae9f8eb986a55
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 19:55:39 2011 -0400
-
- do not segfault when there is no PL
-
-commit 9ce3c584ec0cebfa45576f2ef538df4dad2b7e55
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 11:59:55 2011 -0400
-
- remove another unused part
-
-commit f53a051d68bf312ac8d5865210fae7a9808c0fb9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 10:41:25 2011 -0400
-
- print G3 if HWE is small
-
-commit 4b2c08bb86ca4ed4959e4cb77a28f7d6fc19f5c9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 10:04:34 2011 -0400
-
- fixed a bug
-
- actually not fix, but hide it
-
-commit 088e13c32453fb533b7bb1c65a573f9b90a23625
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 09:48:47 2011 -0400
-
- added LRT based permutation; not used though
-
-commit 1e3c2001afcb80b5eaa4c3f88df9da7b01b62524
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 09:28:55 2011 -0400
-
- Perform posterior contrast for small LRT
-
- Posterior contrast is much slower than LRT. Nonetheless, posterior P-value is
- more robust to sequencing artifacts. Thus we may combine the two to achieve a
- balance between speed and low FPR.
-
-commit 6f1b066270902198a7175ff6c1b05ebc8d1919be
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 15 01:36:06 2011 -0400
-
- Added Brent's method
-
-commit 3d061e5db25b67b25f6ff87afe4162e121354232
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 23:30:10 2011 -0400
-
- fixed a typo in printing
-
-commit 7fd14ceb5990bb350b8e97346ef3537d80058def
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 23:14:23 2011 -0400
-
- fixed a stupid bug
-
-commit f5b2c3459ec098b3cafd9619b9077132516baf58
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 22:42:35 2011 -0400
-
- separate EM and posterior
-
- Now, constrast is not performed unless -C is in use. EM can be invoked
- independently with -e without computing the posterior.
-
-commit 9eefcac963697fae554789b11ae3cb2c23f224d0
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 22:00:19 2011 -0400
-
- further code cleanup; prepare to add EM interface
-
-commit c2cce52355262743711e4742b0c8542bfcab1cdd
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 21:44:03 2011 -0400
-
- drop EM from prob1
-
-commit 24016f04bd3bdffb7eeb50cb25854f5007feb70f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 21:08:33 2011 -0400
-
- drop posterior LRT; prepare for clean up
-
-commit 3670d8bd88c3eb22873f0a80e2a5913f64ca8c9a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 20:57:43 2011 -0400
-
- better initial values for LD
-
-commit d48a8873c060b18b57799cfe3a0e5496ba069457
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Apr 14 20:36:25 2011 -0400
-
- finished EM
-
-commit b101f2db476188a950c23f5c1b6185fdb7f8f40b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 13 01:19:04 2011 -0400
-
- genotype frequency estimate
-
-commit d79bdcbf6242ecfb8accba9ac9a22fbcbd543cf2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 13 00:37:22 2011 -0400
-
- prepare for code clean up
-
-commit e0ce416abfc094f0c090957080b1404fd0edf752
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 13 00:34:15 2011 -0400
-
- rename ld.c to em.c
-
-commit 45ede3ad181f35c1be24bed5d75841e472357ab7
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Apr 13 00:22:10 2011 -0400
-
- implemeted EM likelihood ratio test
-
- The idea is learned from a brief chat with Rasmus Nielsen.
-
-commit 0454a346b60e42b75a2f742272089810279c7131
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Apr 12 15:45:52 2011 -0400
-
- added likelihood-ratio test (idea from Nick)
-
-commit f6287c8646c690440a1554c8958e7268f4134dc2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 10 18:24:37 2011 -0400
-
- Release samtools-0.1.15 (r949:203)
-
-commit de6023f38f4d652438557cf7a0ac6eec324e7416
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 10 15:54:58 2011 -0400
-
- improved help information
-
-commit d3b337f2b7eda1e6f8f5575a19d1b5ed55cae279
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Apr 9 16:28:01 2011 -0400
-
- fixed a minor issue
-
-commit 82f6e4f49247e75fbd8ec08c285b8d3047b3d235
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Apr 9 15:49:04 2011 -0400
-
- separate QC-pass and QC-fail reads
-
-commit 8362b4a255081ee7ca0a4ca2eabc8c76758b6863
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 8 17:45:19 2011 -0400
-
- added verbose level
-
-commit f7bf419c290462be7d289249a4a6d28f825b4c93
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 8 16:08:14 2011 -0400
-
- fixed a bug
-
-commit 890cbb1ac93b3004fb6cf42ff47195077dcfc8ad
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 8 16:00:37 2011 -0400
-
- drop unrelated @RG when "-R" is in use
-
-commit a62dc929c950fb51311b705f5b5bfba8e3f704d7
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 8 16:00:14 2011 -0400
-
- skip header validation
-
-commit 39da810e2c56c8f0eff1ab726600b41f26d3d8e9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Apr 5 23:52:22 2011 -0400
-
- change error message
-
-commit c0c50a34df250ef8a7a29b172058cd229be582b5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Apr 5 23:50:46 2011 -0400
-
- fixed a bug caused by recent modifications
-
-commit 25226e8c468404cb5e1b5272efcea57e4193c762
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Apr 5 13:31:19 2011 -0400
-
- reduce the indel filtering window
-
-commit 5e18d7014437734f9dac9ab45a95e43ec2526101
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Apr 4 13:56:20 2011 -0400
-
- only output hwe if it is small enough
-
-commit 614941fb7dd276de662e7820eb8c7bae871a18cc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Apr 4 13:34:02 2011 -0400
-
- added HWE back
-
-commit 7abe8825aa0bacccdeb38125934ae94d18f0ad4d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Apr 4 12:46:24 2011 -0400
-
- EM estimate of genotype frequency
-
-commit 2bfeff9c645d177416664f1cb811e85cac3ff9e3
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Apr 4 11:29:12 2011 -0400
-
- minor
-
-commit 401e40647e7e3abbac6e4ec3d8bb68eb6f2d401b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Apr 4 11:24:04 2011 -0400
-
- Added genotype freq estimate and association test
-
-commit 6cc226df6e3b480f1bd6e763ce8ef47f785bbb74
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Apr 3 20:57:23 2011 -0400
-
- minor changes
-
-commit 7e47a39630e812f09b80369f14606245976f687e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 1 15:21:59 2011 -0400
-
- print the grayscale
-
-commit 2f675d9c0dde3c166c99e335fa17c7873a5ae8d5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 1 08:55:16 2011 -0400
-
- change to comment
-
-commit 0592bb514994544ed84f51e509b233cf8821e0cf
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Apr 1 08:54:35 2011 -0400
-
- added base quality filtering
-
-commit fc1b47e04a7b94f6362c45856cbeb89d9d0b5ca5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 31 23:31:14 2011 -0400
-
- fixed a few typos in comments
-
-commit 60be79bc8f0d24656e5e8a329af7e9b5b91d4c8b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 31 23:13:23 2011 -0400
-
- comments
-
-commit 2432864acc25ebe5cee4217dbb0120439077a7f8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 31 22:42:46 2011 -0400
-
- added bam2depth.c, a demo program
-
-commit 39625f7c6bea9ccbfd9af0feb22348d52079f012
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 31 16:37:22 2011 -0400
-
- added bgzf_check_bgzf() (used by tabix)
-
-commit 6de6bd3fb67fd22753a5f07d4cc25bf94e1b5a8c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 31 16:37:08 2011 -0400
-
- fixed a bug in bedidx.c
-
-commit 3b9e257d25b2e81eed1625bc5d2882ed486ef20e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 30 13:27:15 2011 -0400
-
- added bed support to bcftools
-
-commit 47bcce3d14ec4d205283b61e5e653803996c42e0
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 30 12:56:40 2011 -0400
-
- Added BED support to "samtools view"
-
-commit a812386017faedfc86c0e6562adbb2138329cfeb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 30 12:47:04 2011 -0400
-
- support BED file
-
-commit 3052dddc929f1825e6e7f7f6f6724d9465d6cf9a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Mar 28 15:51:55 2011 -0400
-
- relax RG matching; proper mismatching message
-
-commit f86d60c8fe25785523f01fae1486d2a6df4ee6ef
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Mar 26 10:38:23 2011 -0400
-
- Avoid reporting association when something unexpected, which I do not understand, happens.
-
-commit dd41e6b26fd9fe30218748b9a0a1f49bdb1862b9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Mar 26 10:38:01 2011 -0400
-
- Added -1 to merge
-
-commit 4a0364b0d7f87f1c88d71ec5857a1f1d40710681
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 23 16:56:55 2011 -0400
-
- plot pairwise r^2
-
-commit 452629a711582e612bec22b3b082e234bd37039b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 23 14:31:01 2011 -0400
-
- pairwise LD; case-control AF2
-
-commit 52862951adcaecde26ba8f0d9c1897944640a674
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Mar 21 23:03:14 2011 -0400
-
- Release samtools-0.1.14 (r933:170)
-
-commit 59a5a8ba8e2940f0e38238f9339f02c91a8a0ce4
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Mar 21 13:52:55 2011 -0400
-
- optionally skip loci with too low sample coverage
-
-commit 6434264b5c69514d4fafe62cbd30b3bbaddc1d41
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Mar 19 14:38:25 2011 -0400
-
- mpileup support Illumina1.3+ quality; skip non-variant sites when "view -v" is in use
-
-commit 5f59e01987e1d5eca7d6359cae64a9734b18beea
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 18 17:19:18 2011 -0400
-
- update version to r933:167
-
-commit 4d2c3c950910aa3d2c87760c3532e458fe01c0fa
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 18 16:25:01 2011 -0400
-
- added "-1" to the command-line help
-
-commit 55313a015a7bd6369cf5a66fed7fab2333201dc9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 18 16:22:12 2011 -0400
-
- added the "cat" command (by Chris Saunders)
-
-commit b670272cadf3efa4dc456ac4c76104f73477d60d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 18 15:59:46 2011 -0400
-
- support varying the compression level
-
-commit c5dd3c9ca5f75f880e52c8cd2beae983bcb8d3b1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 16 14:33:45 2011 -0400
-
- update the manual pages
-
-commit 12fb4b596dc51bccd154fc4bd0593442f7937a46
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 16 12:49:26 2011 -0400
-
- update changelog
-
-commit e7fe4fd66e02d60a1ca7952ad1938809e77729a9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 16 12:10:05 2011 -0400
-
- do not call indels when the depth is very high
-
-commit 7455eeaa32b949bb3856f75810890aabf7cacb18
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 16 11:56:56 2011 -0400
-
- code clean up
-
-commit 5f16679e54ced8e67a75d949f9175c50480b914e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 15 14:45:24 2011 -0400
-
- when -s is specified, change the sample order
-
-commit 7ba95adee09d3b06a7eaf797d25efef837e592f5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 15 14:11:42 2011 -0400
-
- compute the rank in permutation
-
-commit d219783cea7643fc7e10e1bd3a98e9b3165b4506
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Mar 13 21:35:13 2011 -0400
-
- I have found a SERIOUS BUG!!!
-
-commit 8e20d04ecdac1a7788eef71c4bb91b8479cf7150
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Mar 13 17:04:04 2011 -0400
-
- optionally shuffle samples in a BCF (debugging)
-
-commit fc7b261f181f2a411427bc9ee5d586c883ca9cdc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 11 09:34:20 2011 -0500
-
- fixed a bug
-
-commit b3bbcc3d40994ae85705ab6fef9866ec8c142201
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 10 20:25:59 2011 -0500
-
- use mode instead of mean
-
-commit f1161262d137098a19143b5cb0de810e5db3243e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 10 20:09:16 2011 -0500
-
- start from the mean instead of the mode
-
-commit 2ba56f5e99e90674855c4ffc8bf583340b932e1e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 10 17:13:34 2011 -0500
-
- fixed an error in Chi^2 test
-
-commit b4ce7ae400290bc43dd287240479667f99b3b11e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Mar 10 00:23:39 2011 -0500
-
- minor
-
-commit 8487fa5d3a73a43443964e731ea2a4c873c9d4e5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 21:33:19 2011 -0500
-
- added -F to accept BCFs generated by old samtools
-
-commit fd51d2093f7fd775a7eaaeea57fa34716ab59ac2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 17:39:09 2011 -0500
-
- update version
-
-commit b6da54335df943015a998a934075331b467abb5b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 17:37:14 2011 -0500
-
- compute pseudo-chi2 probability
-
-commit 9f73cefdb8935421d872b989dd98fbc8e1295029
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 15:54:04 2011 -0500
-
- remove a comment which is wrong
-
-commit b10b1e47ece522e97ab8ef23417bcb6454f8b9db
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 15:51:12 2011 -0500
-
- clean up
-
-commit 353bfae2c6ff59205bd9223db04084cf7f507f01
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 15:45:29 2011 -0500
-
- for backup
-
-commit 53915d1c6410c2537d18bfa8eb8c657a2233c35e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 15:27:56 2011 -0500
-
- having debugging code
-
-commit 0d0dbf66995b1511390d593981eae7b5d36fe17b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Mar 9 14:58:23 2011 -0500
-
- temporary backup
-
-commit 5b74a174a8b637dee43b7f30250df6fb96580e12
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 8 15:46:11 2011 -0500
-
- the output makes sense, but there may be a typo...
-
-commit d81ec654b6c0c1eef6b0625d96f14b3155cee7c6
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 8 15:19:09 2011 -0500
-
- added contrast2(); fixed a bug in haploid mode
-
-commit 0cfd896fad5f7737cca49efa94a11892dafcd812
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Mar 7 21:40:17 2011 -0500
-
- fixed a bug in haploid genotyping
-
-commit ccd52155ef61273f2b42ad9c7b31ff1915f81b24
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Mar 5 18:10:35 2011 -0500
-
- fixed a few bugs; still not fully working
-
-commit edc3af753f96f831968ae32f2e0f915b74f31e6e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 4 17:31:33 2011 -0500
-
- drop HWE calculation
-
-commit 92dac194debb66ca0718c21c871822dda2dd5bc1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 4 17:28:35 2011 -0500
-
- implemented hap/dipoind mode; probably BUGGY!
-
-commit 7f26804bc27937e36fdc967e5c76514653ea40f5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 4 16:01:27 2011 -0500
-
- read ploidy
-
-commit e7b7213475b5e61a69aab77ffb02b4983c8e7678
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 4 14:12:14 2011 -0500
-
- added math notes
-
-commit 46023e2f21321da83fc8e83e9229757a4e821acb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Mar 4 13:34:10 2011 -0500
-
- update BCF spec
-
-commit 13190c49eeb006ad7013b7f1e9fc1b3beca3ae78
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 1 14:45:19 2011 -0500
-
- Release samtools-0.1.13 (r926:134)
-
-commit be8fabbb6001d9fd5263a70a3e21ed6dfe5a9837
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 1 14:07:15 2011 -0500
-
- prepare to finalize 0.1.13
-
-commit 1e8c753660978bed7e9289fe50becd596d9314bb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Mar 1 09:40:17 2011 -0500
-
- allow to change whether to drop ambiguous reads
-
-commit 412210bfdb46606023f2e4b9086f2787f0cf1c62
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 22:01:29 2011 -0500
-
- revert to the old behavior of phase
-
-commit 46035589518cf84738de8666b866e2619457c1fb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 16:46:23 2011 -0500
-
- change version number
-
-commit 7f40c33e37fc16fcb0a375ce46ae1d09cafb6d50
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 16:37:42 2011 -0500
-
- bugfix in indel calling: interger overflow
-
-commit 75849470efbe30042e5ddd516f9bcbe3b9bf6062
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 15:35:47 2011 -0500
-
- fixed a typo
-
-commit 9e6fb569885f906fabaab7fc2f02eae82f4bd602
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 15:34:09 2011 -0500
-
- minor changes to heuristic rules
-
-commit 30a799a91f5e2c10b761aa5437f902c6649fceb3
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 15:20:26 2011 -0500
-
- fixed a bug in the latest change
-
-commit e21ba9df950ea37f5c1b35c2af9ba9a4e0bba02a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 12:47:06 2011 -0500
-
- put version in bam.h
-
-commit 918b14780c1dceb39c7010638ecd61c626e17166
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 12:00:38 2011 -0500
-
- frag_t::phased==0 reads are dumped to chimera.bam
-
-commit 657293c7bdba3ac69f53cd1ffa2874ed8756475e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 11:05:29 2011 -0500
-
- change default -q to 37 (previously 40)
-
-commit 33d8d3bea76e466798ea322d68d34deb8d2dff06
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 28 10:39:57 2011 -0500
-
- fixed a minor bug in BAM reading
-
-commit daa25d426d42465d76c7317c95772bbb36bb3f47
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Feb 26 21:07:24 2011 -0500
-
- suppress gzopen64() warning
-
-commit 9cec4256eb9e7848d4711adb67b540659c141e32
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 22:14:52 2011 -0500
-
- fixed a long existing bug in vcf2fq
-
-commit 304487c83067a733add71cbc3886fa8c49f7ef2a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 16:37:40 2011 -0500
-
- change version number
-
-commit 10ba6bf4f16692760f696f7b17f3719065786f77
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 16:34:08 2011 -0500
-
- Change the order of PL; change SP to int32_t
-
-commit c5cc2a8036a9c3579fbfde651efec4f6763b0228
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 14:52:03 2011 -0500
-
- claim X defined in the header
-
-commit 4ee8cb29f6092fd14a89f0cc5d3575112a204f39
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 14:40:24 2011 -0500
-
- minor changes
-
-commit 00065e9336a2831dc53bee7da2f4719845be1a2a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 11:39:06 2011 -0500
-
- fixed an error in the BCF spec
-
-commit 1e2a73afcb72a02aa448718cb017c0438de89f90
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 11:36:40 2011 -0500
-
- update BCF spec
-
-commit dbf8eedaa38a405cb2fba5b3952b85776f51d035
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 11:28:43 2011 -0500
-
- update BCF spec
-
-commit eed1d91af9fad3c9d965333a55e623757f9c4e9d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 09:51:39 2011 -0500
-
- fixed a flaw in targetcut
-
-commit 59bc980bb832b92a8b0cc244cf106e6150e4db6f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 00:54:35 2011 -0500
-
- update manual page
-
-commit fcc4738c4abdca79e3de159e21208df1b98ac76c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 00:45:39 2011 -0500
-
- update version format
-
-commit 5748639ae542b7f6b853562edc2bb3faf43030e4
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 00:45:12 2011 -0500
-
- update version number
-
-commit 06b44cc366cf27ce8976ee6a05810a0b3c48b56d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 00:44:21 2011 -0500
-
- update version number
-
-commit ab7f4529d12739ff66fd4c09af9d992ab59c53ef
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 25 00:42:55 2011 -0500
-
- various help message
-
-commit a092e1f6f963272f8bb23616986ddaf604fd0f82
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 24 23:43:13 2011 -0500
-
- disable unfinished functionality
-
-commit f00a78db72b14ee4c6689fc13f20ed31aeaecd40
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 24 10:04:56 2011 -0500
-
- added "const" to bcf_p1_cal()
-
-commit 91049c4a8db3bf50dcc9d07506f22fa4ca5b5a96
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 23 11:53:47 2011 -0500
-
- randomly allocate unphased reads
-
-commit f4405354a8d4cb3441141fa734573031059d7f57
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 15:36:07 2011 -0500
-
- fixed a typo
-
-commit 3075e4dc5c7c9d954426aabda6a73fa788357100
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 15:33:40 2011 -0500
-
- make output more informative
-
-commit 628cf3235e2815a40acf089fb1d3357be6437787
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 14:50:06 2011 -0500
-
- change the scoring rule; change default k to 13
-
-commit f22fd99831e4b5c74f898719216f359dbe987bbf
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 14:45:15 2011 -0500
-
- update scoring in masking
-
-commit 2f23547b81984555032aa0eefd064b8e07986fdc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 14:37:17 2011 -0500
-
- remove dropreg()
-
-commit 4d8b6b1f1f331ca9041983c66e34a857c3b8f1bb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 13:10:16 2011 -0500
-
- accept files from stdin
-
-commit 9b50c5038e6fc0185e29ca5b50fe0806a9a939b9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 22 11:16:57 2011 -0500
-
- fixed a bug in consensus generation
-
-commit 1332ab32fb788fdc81b2ba8653b905d106238fad
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 22:53:23 2011 -0500
-
- print dropped fragments
-
-commit a288761b4ca1584e51076a71cbc4d72fe923dda1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 22:37:04 2011 -0500
-
- bugfix: singletons are not phased
-
-commit 683365f534c0223dea7d72532015ac16a45ba22b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 17:27:10 2011 -0500
-
- output singleton blocks
-
-commit 841a4609084d81f1bc81e0b00dd806002461e7d9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 15:58:55 2011 -0500
-
- fixed a bug; not working with -l right now
-
-commit fdd57ea31732b5516dc212d72174b60206952636
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 15:17:00 2011 -0500
-
- skip mapQ==0 reads
-
-commit 4eb6ba75c23c1c9be5f76814fa1b93a2e304b2af
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 14:03:03 2011 -0500
-
- print the "targetcut" command
-
-commit 0123d9559ba58b026c0dfd15bc26019a193cd21a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 11:22:13 2011 -0500
-
- allow to set the maximum depth
-
-commit 0f92eb248a4d06645b2c3d736a0faea8a7a9f731
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 09:56:41 2011 -0500
-
- use a proper error model to call hets
-
-commit 587a01504af5aea6288740d121dccf48fb8a75f4
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 21 09:16:38 2011 -0500
-
- phase is UNFINISHED; strip RG when merging
-
-commit 723bf3cd79e4f4a558373d4c707fa6b3db0fb357
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Feb 19 23:38:11 2011 -0500
-
- use a proper model to compute consensus
-
-commit 891a6b02d4a9af2ed98fbaac4915bf1f0da4f6c8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Feb 19 22:14:19 2011 -0500
-
- added comment
-
-commit 8b55e0a581ecc9e4ba754d1f3c8784f3038b6e48
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 18 17:23:39 2011 -0500
-
- change the output format
-
-commit 75c36e8c563eddd0a362ba3b38cf0aea21aafb1f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 15 20:31:00 2011 -0500
-
- fixed a bug in writing BAM
-
-commit bb0ce52f066cfebaa35a125d57b353bb717a5165
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 23:39:09 2011 -0500
-
- skip uncovered; unknown alleles taken as X
-
-commit ba67f4d119c7d06907db3015d337d9a01a3fc9fe
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 23:21:19 2011 -0500
-
- fixed a bug
-
-commit e4448d49e6129a5e1ee9c7f04f43612f12d6aad6
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 22:43:09 2011 -0500
-
- prepare to read hets from a list; unfinished
-
-commit 129ea29c1f12177c0a7c3e21676f6210370fc59b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 16:32:22 2011 -0500
-
- updated khash.h to 0.2.5
-
-commit 15b44ed93bd949dffcf79ac8dbea6d9b7dfcb58c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 16:15:04 2011 -0500
-
- use the latest version of khash
-
-commit 486c05f06f44d981dfb2069bcb43e4b35fd8389c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 15:04:40 2011 -0500
-
- change the default -k to 11
-
-commit 07cf9d1e443d73cf053de38dd01671e3781f6e29
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 14:50:51 2011 -0500
-
- sort fragments by vpos instead of by beg
-
-commit d0d3e7faabf5cbb7e5ff7b294f7e220da807c4c0
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 14:45:41 2011 -0500
-
- shuffling the two haplotypes for better randomness
-
-commit 3be28eaf5f6033229aedf12ddb11a0084ba01cd8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 14:09:17 2011 -0500
-
- write chimeras to a separate BAM
-
-commit 80ccbc26f43918fe42be123cc1da9d3d7ce30816
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 13:54:13 2011 -0500
-
- no mem leak/violation on small files; correctness is not checked
-
-commit 5c923867432fa14c26a19e3782e7f48d4080f6ac
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 14 13:50:25 2011 -0500
-
- bam separation; at least not immediate segfault
-
-commit cea2643ec30a59735bf89b2f562b563bf7263e79
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Feb 13 23:24:11 2011 -0500
-
- on the way to implement BAM separation; unfinished
-
-commit 964269cd15036a470ca89e43d0952201a0825671
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Feb 13 18:07:56 2011 -0500
-
- keep singletons in the hash table
-
-commit 2d4aa649bd670d5e038a1acaefd33c5fe24ae0e8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Feb 13 17:42:24 2011 -0500
-
- Revert "prepare to add bam separation"
-
- This reverts commit ed6957e5211c2c4cf684dcb8bbb661052c74df6f.
-
-commit ed6957e5211c2c4cf684dcb8bbb661052c74df6f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Feb 13 00:24:28 2011 -0500
-
- prepare to add bam separation
-
-commit d211e652d93791d2e112d334added243ffe5fc3e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Feb 12 18:50:20 2011 -0500
-
- accelerate kstrtok
-
-commit 2d6af49d331ff5afe7b9e9b102e79d7d4512fdbe
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 11 21:08:21 2011 -0500
-
- split unlinked blocks
-
-commit 68e4cd1b560b0a6fd4c77e5e51eadde9fda26ea4
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 11 10:47:58 2011 -0500
-
- remove heading and tailing ambiguous positions
-
-commit d2b685141426a902ae76660c1fbe8020da150cf8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 11 10:02:21 2011 -0500
-
- code clean up for further features
-
-commit c6980e062d55928b59f287c03e599dd5a37ed509
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 11 08:00:08 2011 -0500
-
- change /64 to >>6; the latter is faster
-
-commit 91635b9c2687f24d72ee6a8aad2050a79bb8400f
-Merge: 41d4df2 9a7e155
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 11 01:22:55 2011 -0500
-
- Merge branch 'master' into devel
-
-commit 9a7e155cc591c1b6c9f7f9cb939364a6becb65b2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 11 01:21:07 2011 -0500
-
- output an unrecognized field as '.'; autofix GL/PL
-
-commit 41d4df2e9545e9abe97151cfe5d6c763f3d00db1
-Merge: c00c41c aacce0c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 23:00:14 2011 -0500
-
- Merge branch 'master' into devel
-
-commit aacce0ce7276f451e4fddf81832f9e5f7f65198b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 22:57:53 2011 -0500
-
- finished VCF->BCF conversion
-
-commit 0e875df643e41d848b709e2fa877de8ae53cdd4c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 21:57:28 2011 -0500
-
- fixed a bug in reading VCF files
-
-commit c00c41c2a5da69cccea64adb542a0b365e56b4fc
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 16:28:37 2011 -0500
-
- suppres one-allele blocks
-
-commit 2e2354b673722e2f00d72970a043f80a66270da1
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 16:06:56 2011 -0500
-
- fixed the bug in filtering
-
-commit d971e1fe24de4ecaf94055efffc5f641e2bdb563
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 12:24:23 2011 -0500
-
- prepare to add filtering; buggy right now
-
-commit a0a5a3fbf504c3b02f7b9212e72315c1047cc249
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 11:55:02 2011 -0500
-
- make masking optional
-
-commit 28db71ccd95054a5f8a47c2332794f8968f6a822
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 10 11:40:47 2011 -0500
-
- routine to mask poorly called regions
-
-commit a3f6c439262bc10a4067860440f4d4dde9e0c515
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 17:18:33 2011 -0500
-
- code clean up: remove globals
-
-commit 0b711978492f6ad39d459d78723c299468906818
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 16:52:54 2011 -0500
-
- output more information
-
-commit f69d217ae5b691bf42ad07a97f29a7cc6456046f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 16:11:54 2011 -0500
-
- fixed another bug in flipping
-
-commit d47882d549337fbcc251597508a2c7faf1bb92e2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 16:01:35 2011 -0500
-
- fixed a stupid bug in flipping
-
-commit e33f89de499496537f5fbde396a66557f0353f1b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 15:54:42 2011 -0500
-
- fix chimeras; a little weird...
-
-commit 03d3c1d0b945245108ce0942d4772536a32212c7
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 13:27:35 2011 -0500
-
- no effective change; prepare to fix chimera
-
-commit 6bc0a4676dd2252085a6e67bb06daa5ae05a554f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 11:52:58 2011 -0500
-
- better count output
-
-commit dcac515439d25f71125d6de8111da417776ab9ce
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 9 10:31:07 2011 -0500
-
- prepare for another way of filtering
-
-commit ca7e4f1899b86d2e077994c789e8f69d699b3cd9
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 8 16:10:08 2011 -0500
-
- fixed the bug; I can do better.
-
-commit 0733f77b98af121bdcb198cea6151d159831bb9c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 8 15:55:38 2011 -0500
-
- fixed two bugs; still not working...
-
-commit 80f18cba9ba73c9592380fc1ecd53c351d294782
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 8 15:42:58 2011 -0500
-
- filter false SNPs; NOT working right now
-
-commit 69a66e2f96d5b102cd712ff1527a3802fa84c590
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 8 14:39:09 2011 -0500
-
- write sequence in the SAM format for debugging
-
-commit b6f1c9d160822af2b713be206f37bd6dde00546a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 7 11:51:21 2011 -0500
-
- fixed two bugs
-
-commit 400aa5c06100af9c47cd5e4ce8b95b7deb84f54b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 7 11:22:38 2011 -0500
-
- Optionally apply BAQ
-
-commit 4c82e0e19682e424f5cdb8381364114c307b329e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Feb 7 01:23:31 2011 -0500
-
- improved output; the result makes sense at a glance
-
-commit dc7853a581ab24bcc496e96b123ccf637e32ed1d
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Feb 6 14:12:43 2011 -0500
-
- process per linked block instead of per chr
-
-commit e867d9c6c2e61d9e748e78163e5481dca5697a36
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sun Feb 6 00:45:46 2011 -0500
-
- DP seems to work on toy examples
-
-commit 445ad72fc43d4354d56f5f759790e8ae0be73d02
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Feb 5 01:24:42 2011 -0500
-
- implemented backtrack; not tested
-
-commit ba38e180b9cd545956583b22e97e09b4bb12073e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 4 23:55:23 2011 -0500
-
- More "correct" DP; backtrack not implemented
-
-commit d69761fd9351273ccd37ea431b10509add91e7cf
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 4 17:22:31 2011 -0500
-
- scratch of dynamic programming; unfinished...
-
-commit 769ffcb44e26e59300791658801d321559b33858
-Author: Heng Li <lh3 at live.co.uk>
-Date: Fri Feb 4 16:29:55 2011 -0500
-
- UNFINISHED commit.
-
-commit 9adab9591317c3467f3d8cdf2d19ec1f65d1b5b7
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 3 16:20:59 2011 -0500
-
- another way of counting; can be even faster
-
-commit bbafbdc01ed1ceaab44927def1ad47c4c78aeb9c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Feb 3 14:48:20 2011 -0500
-
- for backup
-
-commit eba7446389cad62a19133bced1386a4334dcab79
-Merge: a44a98e f01a593
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 2 14:06:07 2011 -0500
-
- Merge branch 'master' into devel
-
-commit f01a5930445b5fda7e6b5b813ed63c652160ada2
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 2 11:31:54 2011 -0500
-
- Better truncation warning when EOF is absent
-
-commit dd3ee5ed26c8bbef4a62fa5b2bfb0a75833f2c31
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 2 10:38:28 2011 -0500
-
- fixed a typo in BCF/VCF headers
-
-commit b9d1137c55f401387113d1ad8a387489afe741db
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Feb 2 09:13:44 2011 -0500
-
- fixed an out-of-boundary bug (fixed by Roel Kluin)
-
-commit a44a98e16559b9672e8a3492c8f8c640074b7ee2
-Merge: ef68a14 d0443d5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 1 21:54:48 2011 -0500
-
- Merge branch 'master' into devel
-
-commit d0443d5c2f648e0f69bd4c56eaac7868e501c18b
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 1 17:31:52 2011 -0500
-
- improved sorting order checking
-
-commit ef68a14fab91399b2ecd38345936c3d6e7391cf3
-Merge: 1e597b3 1a39a2e
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 1 15:12:37 2011 -0500
-
- Merge branch 'master' into devel
-
-commit 1a39a2eb08a270e20a34a0983e8bed6ffb3e2008
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 1 15:12:14 2011 -0500
-
- more precise error message
-
-commit e028e7a47c02232e06a9dd3009262c00dede1060
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 1 14:48:01 2011 -0500
-
- improved sorting order validation in index
-
-commit 1e597b3356744e2b791b12c9187f91c8054511d5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Tue Feb 1 14:44:27 2011 -0500
-
- testing only; not working
-
-commit 5753ace1e54228822d8ee95f69943f586e42f6e8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jan 31 17:37:08 2011 -0500
-
- reduce the effect of seq errors at the cost of SN
-
-commit 6f239ce5e0abd47babee33174476d48b723260d8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jan 31 17:29:34 2011 -0500
-
- added testing code
-
-commit 3db42fe22d27d61ab5735cd2308f73d93def8ebe
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jan 31 14:33:21 2011 -0500
-
- routine for phasing fosmid resequencing (incomplete)
-
-commit ed88f2797323229ae8f38fbcd107b231007956a8
-Author: Heng Li <lh3 at live.co.uk>
-Date: Mon Jan 31 10:12:53 2011 -0500
-
- SAM output
-
-commit abc6acae28dc4794f6422255f077cf370d34e414
-Merge: f1985a9 b133dbf
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Jan 29 22:56:10 2011 -0500
-
- Merge branch 'master' into devel
-
-commit b133dbf82de4e8cea5eb56e5bbf0c4b3e9368fd5
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Jan 29 22:37:11 2011 -0500
-
- fixed a bug in tview on big-endian by Nathan Weeks
-
-commit 9d3fdaef29f91e21dbfcb9ff0165b9573e7c1042
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Jan 29 22:24:00 2011 -0500
-
- update INSTALL
-
-commit 9d074a38bde53961f96157b6fb3683b6dded38d7
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Jan 29 21:56:25 2011 -0500
-
- avoid a segfault when network connect fails
-
-commit f1985a93f7455b3ea1b0ef9b959d50b896ccd620
-Author: Heng Li <lh3 at live.co.uk>
-Date: Sat Jan 29 21:53:18 2011 -0500
-
- fixed a bug about bit ordering
-
-commit d09797db6fef648a6823cbe718d67664660c6ebe
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 16:53:19 2011 -0500
-
- point out there are 4 or fewer free parameters
-
-commit 5fd1717650ed68ab6c55d094d1648c16a054891a
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 16:09:18 2011 -0500
-
- updated .gitignore
-
-commit fccb19fbe8f9de91f59d85bb49a248683dc6266c
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 16:08:14 2011 -0500
-
- fixed a bug; better scoring
-
-commit b4dcb844bde3d09eedcd9f6832186ece60ae5afd
-Merge: ffc3e89 6f502de
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 14:50:30 2011 -0500
-
- Merge branch 'master' into devel
-
-commit 6f502dec46b18dae4bb5b2319715d028b5e193d0
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 14:47:31 2011 -0500
-
- skip unmapped and ref-skip reads in indel calling
-
-commit 3639f37dd8257b24560c35effcc3b6c16c3c1bcb
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 14:19:15 2011 -0500
-
- fixed an out-of-boundary bug in rare cases
-
-commit ffc3e89678ab9052b84f403da1e43044b045e73f
-Author: Heng Li <lh3 at live.co.uk>
-Date: Thu Jan 27 14:00:17 2011 -0500
-
- targetcut can be compiled, though probably buggy
-
-commit f452b3ac51306865ddde31a8d715b155d4d3e6e6
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jan 26 18:58:43 2011 -0500
-
- this is for a very special application...
-
-commit ca1451c6406c7ee757cb31349ea0b8de70db0656
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jan 26 18:48:09 2011 -0500
-
- fixed compiling errors
-
-commit 085b87a7642865f17239fb6a436e626e25417838
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jan 26 18:45:09 2011 -0500
-
- This script was put in a wrong place...
-
-commit 090d360828622520de60385af4928ce1aebe0e48
-Author: Heng Li <lh3 at live.co.uk>
-Date: Wed Jan 26 18:33:58 2011 -0500
-
- Imported from samtools-r902
-------------------------------------------------------------------------
-r108 | lh3lh3 | 2009-01-20 11:56:45 +0000 (Tue, 20 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/examples/Makefile
-
-made it a little more convenient
-
-------------------------------------------------------------------------
-r107 | lh3lh3 | 2009-01-20 11:53:30 +0000 (Tue, 20 Jan 2009) | 2 lines
-Changed paths:
- A /branches/dev/samtools/examples/Makefile
-
-added a Makefile
-
-------------------------------------------------------------------------
-r106 | lh3lh3 | 2009-01-20 11:25:05 +0000 (Tue, 20 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/maq2sam.c
-
-support RG tag
-
-------------------------------------------------------------------------
-r105 | lh3lh3 | 2009-01-18 17:37:20 +0000 (Sun, 18 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/ChangeLog
-
-update changelog
-
-------------------------------------------------------------------------
-r104 | lh3lh3 | 2009-01-18 17:31:21 +0000 (Sun, 18 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_lpileup.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-18
- * fixed a bug in bam_lpileup.c: segment start and end are not correctly recognized
-
-------------------------------------------------------------------------
-r103 | lh3lh3 | 2009-01-18 16:34:03 +0000 (Sun, 18 Jan 2009) | 5 lines
-Changed paths:
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-17
- * fixed a bug when there are reads without coordinates
- * also recognize type 'c' as 'A'
- * found a bug in bam_lpileup.c; NOT fixed yet
-
-------------------------------------------------------------------------
-r102 | lh3lh3 | 2009-01-17 19:46:49 +0000 (Sat, 17 Jan 2009) | 2 lines
-Changed paths:
- A /branches/dev/samtools/INSTALL
-
-Instruction for compilation
-
-------------------------------------------------------------------------
-r101 | lh3lh3 | 2009-01-17 19:31:36 +0000 (Sat, 17 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- A /branches/dev/samtools/Makefile.lite
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/faidx.c
- M /branches/dev/samtools/misc/Makefile
- M /branches/dev/samtools/razf.c
-
- * replaced HAVE_RAZF with _NO_RAZF
- * added Makefile.lite for people who have trouble with razf.c
-
-------------------------------------------------------------------------
-r100 | lh3lh3 | 2009-01-16 10:03:37 +0000 (Fri, 16 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_mate.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/misc/wgsim.c
-
- * samtools-0.1.1-15
- * fixed another bug in fixmate: unmapped pair has non-zero isize
-
-------------------------------------------------------------------------
-r99 | lh3lh3 | 2009-01-16 09:13:36 +0000 (Fri, 16 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/ChangeLog
- M /branches/dev/samtools/bam_mate.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-14
- * fixed a bug in fixmate: isize not equal to zero if two ends mapped to
- different chr
-
-------------------------------------------------------------------------
-r98 | lh3lh3 | 2009-01-15 16:47:41 +0000 (Thu, 15 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-13
- * fixed the prior for hom indels (Richard pointed this out)
-
-------------------------------------------------------------------------
-r97 | lh3lh3 | 2009-01-15 16:38:47 +0000 (Thu, 15 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/COPYING
- M /branches/dev/samtools/bam_sort.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/source.dot
-
- * samtools-0.1.1-12
- * fixed a bug in sort
- * update source file graph and copyright information
-
-------------------------------------------------------------------------
-r96 | lh3lh3 | 2009-01-14 21:46:14 +0000 (Wed, 14 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/glf.c
-
-fixed a typo
-
-------------------------------------------------------------------------
-r95 | lh3lh3 | 2009-01-14 21:44:53 +0000 (Wed, 14 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/glf.c
-
-added a main function for glf.c
-
-------------------------------------------------------------------------
-r94 | lh3lh3 | 2009-01-14 17:14:59 +0000 (Wed, 14 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/bgzf.h
- A /branches/dev/samtools/glf.c
- M /branches/dev/samtools/glf.h
-
- * samtools-0.1.1-11
- * generate binary GLFv2
- * added glfview command to dump GLFv2 binary file
-
-------------------------------------------------------------------------
-r93 | lh3lh3 | 2009-01-14 15:07:44 +0000 (Wed, 14 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_rmdup.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/glf.h
-
- * samtools-0.1.1-10
- * fixed several bugs in rmdup
- * prepare to generate GLF2
-
-------------------------------------------------------------------------
-r92 | lh3lh3 | 2009-01-14 13:27:44 +0000 (Wed, 14 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_import.c
- A /branches/dev/samtools/bam_rmdup.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-9
- * implemented rmdup; NOT tested yet
-
-------------------------------------------------------------------------
-r91 | lh3lh3 | 2009-01-13 20:15:43 +0000 (Tue, 13 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/examples/00README.txt
-
-update README for typos
-
-------------------------------------------------------------------------
-r90 | lh3lh3 | 2009-01-13 19:57:50 +0000 (Tue, 13 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/examples/ex1.sam.gz
-
-update example
-
-------------------------------------------------------------------------
-r89 | lh3lh3 | 2009-01-13 17:21:38 +0000 (Tue, 13 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam.c
- A /branches/dev/samtools/bam_mate.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-8
- * added fixmate command
-
-------------------------------------------------------------------------
-r88 | lh3lh3 | 2009-01-13 10:48:23 +0000 (Tue, 13 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-7
- * change the reported indel position to the previous way
-
-------------------------------------------------------------------------
-r87 | lh3lh3 | 2009-01-12 22:12:12 +0000 (Mon, 12 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-6
- * addd glt output
- * allow to change indel calling parameters at the command line
-
-------------------------------------------------------------------------
-r86 | lh3lh3 | 2009-01-12 21:16:48 +0000 (Mon, 12 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_pileup.c
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-5
- * added two more flags
- * allowed to select reads shown in pileup with a mask
-
-------------------------------------------------------------------------
-r85 | lh3lh3 | 2009-01-12 20:47:51 +0000 (Mon, 12 Jan 2009) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-4
- * fixed a bug in indexing (linear index)
- * prepare to add glt output from pileup
-
-------------------------------------------------------------------------
-r84 | lh3lh3 | 2009-01-12 09:22:35 +0000 (Mon, 12 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-3
- * fixed a bug in outputing the coordinate of an indel
-
-------------------------------------------------------------------------
-r83 | lh3lh3 | 2009-01-11 15:18:01 +0000 (Sun, 11 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-2
- * pileup: allows to output indel sites only
-
-------------------------------------------------------------------------
-r82 | lh3lh3 | 2009-01-10 23:34:31 +0000 (Sat, 10 Jan 2009) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bam_maqcns.h
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.1-1
- * implemented a Bayesian indel caller
-
-------------------------------------------------------------------------
-r81 | lh3lh3 | 2009-01-09 09:54:28 +0000 (Fri, 09 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/examples/00README.txt
- D /branches/dev/samtools/examples/ex1.fa.fai
-
-Let users generate ex1.fa.fai.
-
-------------------------------------------------------------------------
-r80 | lh3lh3 | 2009-01-08 16:10:08 +0000 (Thu, 08 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/bowtie2sam.pl
-
-make the bowtie converter works for "-k 2"
-
-------------------------------------------------------------------------
-r78 | lh3lh3 | 2009-01-03 17:25:24 +0000 (Sat, 03 Jan 2009) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/export2sam.pl
-
-fixed a bug for "QC" reads
-
-------------------------------------------------------------------------
-r77 | lh3lh3 | 2009-01-01 18:32:06 +0000 (Thu, 01 Jan 2009) | 3 lines
-Changed paths:
- A /branches/dev/samtools/misc/bowtie2sam.pl
- M /branches/dev/samtools/misc/soap2sam.pl
-
- * soap2sam.pl: added NM tag
- * bowtie2sam.pl: converter for bowtie
-
-------------------------------------------------------------------------
-r76 | lh3lh3 | 2008-12-31 23:24:24 +0000 (Wed, 31 Dec 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/misc/soap2sam.pl
-
-soap2sam.pl: convert soap output to SAM
-
-------------------------------------------------------------------------
-r75 | lh3lh3 | 2008-12-31 17:54:32 +0000 (Wed, 31 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/misc/wgsim_eval.pl
-
- * wgsim_eval.pl-0.1.1
- * fixed a bug for a contig name like "NT_012345"
-
-------------------------------------------------------------------------
-r74 | lh3lh3 | 2008-12-31 16:38:21 +0000 (Wed, 31 Dec 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/misc/wgsim_eval.pl
-
- * evaluate alignment for reads generated by wgsim
-
-------------------------------------------------------------------------
-r73 | lh3lh3 | 2008-12-31 15:11:22 +0000 (Wed, 31 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/Makefile
- M /branches/dev/samtools/misc/wgsim.c
-
-fixed compiling warnings for wgsim
-
-------------------------------------------------------------------------
-r72 | lh3lh3 | 2008-12-31 13:40:51 +0000 (Wed, 31 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/bam_tview.c
-
-remove an unused variable (a compiler warning only)
-
-------------------------------------------------------------------------
-r71 | lh3lh3 | 2008-12-31 13:37:16 +0000 (Wed, 31 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/Makefile
- A /branches/dev/samtools/misc/wgsim.c
-
-wgsim: Paired-end reads simulator
-
-------------------------------------------------------------------------
-r70 | bhandsaker | 2008-12-29 20:27:16 +0000 (Mon, 29 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bam_tview.c
-
-Move definition of bam_nt16_nt4_table so we can build without curses.
-
-------------------------------------------------------------------------
-r62 | lh3lh3 | 2008-12-22 15:55:13 +0000 (Mon, 22 Dec 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/NEWS
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/samtools.1
-
-Release samtools-0.1.1
-
-------------------------------------------------------------------------
-r61 | lh3lh3 | 2008-12-22 15:46:08 +0000 (Mon, 22 Dec 2008) | 10 lines
-Changed paths:
- M /branches/dev/samtools/bam_aux.c
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bam_tview.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/razf.c
- M /branches/dev/samtools/samtools.1
-
- * samtools-0.1.0-66
- * fixed a bug in razf.c: reset z_eof when razf_seek() is called
- * fixed a memory leak in parsing a region
- * changed pileup a little bit when -s is in use: output ^ and $
- * when a bam is not indexed, output more meaningful error message
- * fixed a bug in indexing for small alignment
- * fixed a bug in the viewer when we come to the end of a reference file
- * updated documentation
- * prepare to release 0.1.1
-
-------------------------------------------------------------------------
-r60 | lh3lh3 | 2008-12-22 15:10:16 +0000 (Mon, 22 Dec 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/examples
- A /branches/dev/samtools/examples/00README.txt
- A /branches/dev/samtools/examples/ex1.fa
- A /branches/dev/samtools/examples/ex1.fa.fai
- A /branches/dev/samtools/examples/ex1.sam.gz
-
-example
-
-------------------------------------------------------------------------
-r59 | lh3lh3 | 2008-12-22 09:38:15 +0000 (Mon, 22 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/ChangeLog
-
-update ChangeLog
-
-------------------------------------------------------------------------
-r58 | lh3lh3 | 2008-12-20 23:06:00 +0000 (Sat, 20 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/misc/export2sam.pl
-
- * added comments
- * fixed several bugs
-
-------------------------------------------------------------------------
-r57 | lh3lh3 | 2008-12-20 15:44:20 +0000 (Sat, 20 Dec 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/misc/export2sam.pl
-
-convert Export format to SAM; not thoroughly tested
-
-------------------------------------------------------------------------
-r56 | lh3lh3 | 2008-12-19 22:13:28 +0000 (Fri, 19 Dec 2008) | 6 lines
-Changed paths:
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bam_tview.c
- M /branches/dev/samtools/bamtk.c
- A /branches/dev/samtools/source.dot
-
- * samtools-0.1.0-65
- * pileup: generate maq-like simple output
- * pileup: allow to output pileup at required sites
- * source.dot: source file relationship graph
- * tview: fixed a minor bug
-
-------------------------------------------------------------------------
-r55 | lh3lh3 | 2008-12-19 20:10:26 +0000 (Fri, 19 Dec 2008) | 2 lines
-Changed paths:
- D /branches/dev/samtools/misc/all2sam.pl
-
-remove all2sam.pl
-
-------------------------------------------------------------------------
-r54 | lh3lh3 | 2008-12-16 22:34:25 +0000 (Tue, 16 Dec 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/COPYING
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/faidx.h
- M /branches/dev/samtools/khash.h
- M /branches/dev/samtools/kseq.h
- M /branches/dev/samtools/ksort.h
- M /branches/dev/samtools/samtools.1
-
-Added copyright information and a bit more documentation. No code change.
-
-------------------------------------------------------------------------
-r53 | lh3lh3 | 2008-12-16 13:40:18 +0000 (Tue, 16 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam.c
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-64
- * improved efficiency of the indel caller for spliced alignments
-
-------------------------------------------------------------------------
-r52 | lh3lh3 | 2008-12-16 10:28:20 +0000 (Tue, 16 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam.c
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_aux.c
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-63
- * a bit code cleanup: reduce the dependency between source files
-
-------------------------------------------------------------------------
-r51 | lh3lh3 | 2008-12-15 14:29:32 +0000 (Mon, 15 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-62
- * fixed a memory leak
-
-------------------------------------------------------------------------
-r50 | lh3lh3 | 2008-12-15 14:00:13 +0000 (Mon, 15 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/ChangeLog
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/samtools.1
-
-update documentation, ChangeLog and a comment
-
-------------------------------------------------------------------------
-r49 | lh3lh3 | 2008-12-15 13:36:43 +0000 (Mon, 15 Dec 2008) | 6 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bam_maqcns.h
- M /branches/dev/samtools/bam_pileup.c
- A /branches/dev/samtools/bam_plcmd.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/samtools.1
-
- * samtools-0.1.0-61
- * moved pileup command to a separate source file
- * added indel caller
- * added bam_cal_segend(). (NOT WORKING for spliced alignment!!!)
- * updated documentation
-
-------------------------------------------------------------------------
-r48 | lh3lh3 | 2008-12-12 13:55:36 +0000 (Fri, 12 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-60
- * fixed another bug in maqcns when there is a nearby deletion
-
-------------------------------------------------------------------------
-r47 | lh3lh3 | 2008-12-12 13:42:16 +0000 (Fri, 12 Dec 2008) | 5 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bam_pileup.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-59
- * pileup: outputing consensus is now optional
- * fixed a bug in glfgen. This bug also exists in maq's glfgen. However,
- I am not quite sure why the previous version may have problem.
-
-------------------------------------------------------------------------
-r46 | lh3lh3 | 2008-12-12 11:44:56 +0000 (Fri, 12 Dec 2008) | 6 lines
-Changed paths:
- M /branches/dev/samtools/bam_pileup.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-58
- * add maq consensus to pileup. However, I will move this part to a new
- command as strictly speaking, consensus callin is not part of pileup,
- and imposing it would make it harder to generate for other language
- bindings.
-
-------------------------------------------------------------------------
-r45 | bhandsaker | 2008-12-11 20:43:56 +0000 (Thu, 11 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/bgzf.c
-
-Fix bug in tell() after reads that consume to the exact end of a block.
-
-------------------------------------------------------------------------
-r44 | lh3lh3 | 2008-12-11 09:36:53 +0000 (Thu, 11 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/samtools.1
-
-update manual
-
-------------------------------------------------------------------------
-r43 | lh3lh3 | 2008-12-11 09:25:36 +0000 (Thu, 11 Dec 2008) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-57
- * fixed a bug in parser when there is auxiliary fields
- * made the parser a bit more robust
-
-------------------------------------------------------------------------
-r42 | lh3lh3 | 2008-12-10 14:57:29 +0000 (Wed, 10 Dec 2008) | 5 lines
-Changed paths:
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/bgzf.c
-
- * samtools-0.1.0-56
- * fixed a bug in bgzf (only reading is affected)
- * fixed a typo in bam_index.c
- * in bam_index.c, check potential bugs in the underlying I/O library
-
-------------------------------------------------------------------------
-r41 | lh3lh3 | 2008-12-10 12:53:08 +0000 (Wed, 10 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/samtools.1
-
-update manual
-
-------------------------------------------------------------------------
-r40 | lh3lh3 | 2008-12-10 11:52:10 +0000 (Wed, 10 Dec 2008) | 5 lines
-Changed paths:
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_pileup.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-55
- * tried to make pileup work with clipping (previously not), though NOT tested
- * removed -v from pileup
- * made pileup take the reference sequence
-
-------------------------------------------------------------------------
-r39 | lh3lh3 | 2008-12-09 11:59:28 +0000 (Tue, 09 Dec 2008) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/samtools.1
-
- * samtools-0.1.0-54
- * in parser, recognize "=", rather than ",", as a match
- * in parser, correctl parse "=" at the MRNM field.
-
-------------------------------------------------------------------------
-r38 | lh3lh3 | 2008-12-09 11:39:07 +0000 (Tue, 09 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/maq2sam.c
-
-fixed a bug in handling maq flag 64 and 192
-
-------------------------------------------------------------------------
-r37 | lh3lh3 | 2008-12-09 09:53:46 +0000 (Tue, 09 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/md5fa.c
-
-also calculate unordered md5sum check
-
-------------------------------------------------------------------------
-r36 | lh3lh3 | 2008-12-09 09:46:21 +0000 (Tue, 09 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/md5fa.c
-
-fixed a minor bug when there are space in the sequence
-
-------------------------------------------------------------------------
-r35 | lh3lh3 | 2008-12-09 09:40:45 +0000 (Tue, 09 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/md5fa.c
-
-fixed a potential memory leak
-
-------------------------------------------------------------------------
-r34 | lh3lh3 | 2008-12-08 14:52:17 +0000 (Mon, 08 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bamtk.c
-
- * fixed a bug in import: bin is wrongly calculated
-
-------------------------------------------------------------------------
-r33 | lh3lh3 | 2008-12-08 14:08:01 +0000 (Mon, 08 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/misc/all2sam.pl
-
-nothing, really
-
-------------------------------------------------------------------------
-r32 | lh3lh3 | 2008-12-08 12:56:02 +0000 (Mon, 08 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/kseq.h
- M /branches/dev/samtools/misc/Makefile
- A /branches/dev/samtools/misc/md5.c
- A /branches/dev/samtools/misc/md5.h
- A /branches/dev/samtools/misc/md5fa.c
-
- * fixed two warnings in kseq.h
- * added md5sum utilities
-
-------------------------------------------------------------------------
-r31 | lh3lh3 | 2008-12-08 11:35:29 +0000 (Mon, 08 Dec 2008) | 5 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bamtk.c
- A /branches/dev/samtools/kseq.h
- D /branches/dev/samtools/kstream.h
-
- * samtools-0.1.0-52
- * replace kstream with kseq. kseq is a superset of kstream. I need the
- extra functions in kseq.h.
- * also compile stand-alone faidx
-
-------------------------------------------------------------------------
-r30 | lh3lh3 | 2008-12-08 11:17:04 +0000 (Mon, 08 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_sort.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-51
- * sorting by read names is available
-
-------------------------------------------------------------------------
-r29 | lh3lh3 | 2008-12-08 10:29:02 +0000 (Mon, 08 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam.c
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bam_pileup.c
- M /branches/dev/samtools/bam_sort.c
- M /branches/dev/samtools/bam_tview.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/misc/maq2sam.c
-
- * samtools-0.1.0-50
- * format change to meet the latest specification
-
-------------------------------------------------------------------------
-r28 | lh3lh3 | 2008-12-04 16:09:21 +0000 (Thu, 04 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/misc/maq2sam.c
-
- * minor change in maqcns: special care when n==0
- * change maq2sam to meet the latest specification
-
-------------------------------------------------------------------------
-r27 | lh3lh3 | 2008-12-04 15:55:44 +0000 (Thu, 04 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/razf.c
- M /branches/dev/samtools/razf.h
-
-considerable code clean up in razf
-
-------------------------------------------------------------------------
-r26 | lh3lh3 | 2008-12-04 15:08:18 +0000 (Thu, 04 Dec 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/ChangeLog
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/faidx.c
-
-make RAZF optional in faidx.c
-
-------------------------------------------------------------------------
-r25 | lh3lh3 | 2008-12-01 15:27:22 +0000 (Mon, 01 Dec 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/bam_aux.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/samtools.1
-
- * samtools-0.1.0-49
- * added routines for retrieving aux data, NOT TESTED YET!
-
-------------------------------------------------------------------------
-r24 | lh3lh3 | 2008-12-01 14:29:43 +0000 (Mon, 01 Dec 2008) | 5 lines
-Changed paths:
- M /branches/dev/samtools/bam.c
- M /branches/dev/samtools/bam_import.c
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/bgzf.c
- M /branches/dev/samtools/samtools.1
-
- * samtools-0.1.0-48
- * bgzf: fixed a potential integer overflow on 32-it machines
- * maqcns: set the minimum combined quality as 0
- * supporting hex strings
-
-------------------------------------------------------------------------
-r23 | lh3lh3 | 2008-11-27 17:14:37 +0000 (Thu, 27 Nov 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/bam_maqcns.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-47
- * fixed the bug in maqcns
-
-------------------------------------------------------------------------
-r22 | lh3lh3 | 2008-11-27 17:08:11 +0000 (Thu, 27 Nov 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam.h
- A /branches/dev/samtools/bam_maqcns.c
- A /branches/dev/samtools/bam_maqcns.h
- M /branches/dev/samtools/bam_tview.c
- M /branches/dev/samtools/bamtk.c
- A /branches/dev/samtools/glf.h
-
- * samtools-0.1.0-46
- * add MAQ consensus caller, currently BUGGY!
-
-------------------------------------------------------------------------
-r21 | lh3lh3 | 2008-11-27 13:51:28 +0000 (Thu, 27 Nov 2008) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_pileup.c
- M /branches/dev/samtools/bam_tview.c
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-45
- * tview: display padded alignment (but not P operation)
- * better coordinates and reference sequence
-
-------------------------------------------------------------------------
-r19 | lh3lh3 | 2008-11-27 09:26:05 +0000 (Thu, 27 Nov 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/ChangeLog
-
-new ChangeLog
-
-------------------------------------------------------------------------
-r18 | lh3lh3 | 2008-11-27 09:24:45 +0000 (Thu, 27 Nov 2008) | 3 lines
-Changed paths:
- D /branches/dev/samtools/ChangeLog
- A /branches/dev/samtools/ChangeLog.old (from /branches/dev/samtools/ChangeLog:6)
-
-Rename ChangeLog to ChangeLog.old. This old ChangeLog is generated from
-the log of my personal SVN repository.
-
-------------------------------------------------------------------------
-r17 | lh3lh3 | 2008-11-27 09:22:55 +0000 (Thu, 27 Nov 2008) | 6 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/bgzf.c
-
- * samtools-0.1.0-44
- * declare fseeko and ftello as some Linux may not do this by default and
- missing these declarations will make bgzf buggy
- * get rid of some harmless warings
- * use BGZF by default, now
-
-------------------------------------------------------------------------
-r16 | lh3lh3 | 2008-11-26 21:19:11 +0000 (Wed, 26 Nov 2008) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bam_index.c
- M /branches/dev/samtools/bamtk.c
- M /branches/dev/samtools/razf.c
-
- * samtools-0.1.0-43
- * fixed a bug in razf_read()
- * give more warnings when the file is truncated (or due to bugs in I/O library)
-
-------------------------------------------------------------------------
-r15 | lh3lh3 | 2008-11-26 20:41:39 +0000 (Wed, 26 Nov 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/bgzf.c
-
-fixed a bug in bgzf.c at the end of the file
-
-------------------------------------------------------------------------
-r14 | lh3lh3 | 2008-11-26 17:05:18 +0000 (Wed, 26 Nov 2008) | 4 lines
-Changed paths:
- M /branches/dev/samtools/bamtk.c
-
- * samtools-0.1.0-42
- * a lot happened to RAZF, although samtools itself is untouched. Better
- also update the version number anyway to avoid confusion
-
-------------------------------------------------------------------------
-r13 | lh3lh3 | 2008-11-26 17:03:48 +0000 (Wed, 26 Nov 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/razf.c
-
-a change from Jue, but I think it should not matter
-
-------------------------------------------------------------------------
-r12 | lh3lh3 | 2008-11-26 16:48:14 +0000 (Wed, 26 Nov 2008) | 3 lines
-Changed paths:
- M /branches/dev/samtools/razf.c
-
-fixed a potential bug in razf. However, it seems still buggy, just
-rarely happens, very rarely.
-
-------------------------------------------------------------------------
-r11 | lh3lh3 | 2008-11-26 14:02:56 +0000 (Wed, 26 Nov 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/razf.c
-
-fixed a bug in razf, with the help of Jue
-
-------------------------------------------------------------------------
-r10 | lh3lh3 | 2008-11-26 11:55:32 +0000 (Wed, 26 Nov 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/bam_index.c
-
-remove a comment
-
-------------------------------------------------------------------------
-r9 | lh3lh3 | 2008-11-26 11:37:05 +0000 (Wed, 26 Nov 2008) | 2 lines
-Changed paths:
- M /branches/dev/samtools/Makefile
- M /branches/dev/samtools/bam.h
- M /branches/dev/samtools/razf.c
- M /branches/dev/samtools/razf.h
-
- * Jue has updated razf to realize Bob's scheme
-
-------------------------------------------------------------------------
-r7 | lh3lh3 | 2008-11-25 20:37:37 +0000 (Tue, 25 Nov 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools/samtools.1
-
-the manual page
-
-------------------------------------------------------------------------
-r6 | lh3lh3 | 2008-11-25 20:37:16 +0000 (Tue, 25 Nov 2008) | 3 lines
-Changed paths:
- A /branches/dev/samtools/ChangeLog
- A /branches/dev/samtools/Makefile
- A /branches/dev/samtools/bam.c
- A /branches/dev/samtools/bam.h
- A /branches/dev/samtools/bam_aux.c
- A /branches/dev/samtools/bam_endian.h
- A /branches/dev/samtools/bam_import.c
- A /branches/dev/samtools/bam_index.c
- A /branches/dev/samtools/bam_lpileup.c
- A /branches/dev/samtools/bam_pileup.c
- A /branches/dev/samtools/bam_sort.c
- A /branches/dev/samtools/bam_tview.c
- A /branches/dev/samtools/bamtk.c
- A /branches/dev/samtools/bgzf.c
- A /branches/dev/samtools/bgzf.h
- A /branches/dev/samtools/bgzip.c
- A /branches/dev/samtools/faidx.c
- A /branches/dev/samtools/faidx.h
- A /branches/dev/samtools/khash.h
- A /branches/dev/samtools/ksort.h
- A /branches/dev/samtools/kstream.h
- A /branches/dev/samtools/misc
- A /branches/dev/samtools/misc/Makefile
- A /branches/dev/samtools/misc/all2sam.pl
- A /branches/dev/samtools/misc/maq2sam.c
- A /branches/dev/samtools/razf.c
- A /branches/dev/samtools/razf.h
- A /branches/dev/samtools/razip.c
- A /branches/dev/samtools/zutil.h
-
-The initial version of samtools, replicated from my local SVN repository.
-The current version is: 0.1.0-42. All future development will happen here.
-
-------------------------------------------------------------------------
-r5 | lh3lh3 | 2008-11-25 20:30:49 +0000 (Tue, 25 Nov 2008) | 2 lines
-Changed paths:
- A /branches/dev/samtools
-
-samtools (C version)
-
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-r703 | lh3 | 2008-11-25 20:20:02 +0000 (Tue, 25 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/samtools.1
-
-rename bamtk to samtools
-
-------------------------------------------------------------------------
-r702 | lh3 | 2008-11-25 20:15:09 +0000 (Tue, 25 Nov 2008) | 2 lines
-Changed paths:
- D /branches/prog/bam/bamtk.1
- A /branches/prog/bam/samtools.1 (from /branches/prog/bam/bamtk.1:679)
-
-rename bamtk.1 to samtools.1
-
-------------------------------------------------------------------------
-r701 | lh3 | 2008-11-25 13:29:10 +0000 (Tue, 25 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/misc/Makefile
-
- * samtools-0.1.0-41
- * small (but a bit dangerous) changes to meet the latest specification
-
-------------------------------------------------------------------------
-r700 | lh3 | 2008-11-25 13:15:11 +0000 (Tue, 25 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/misc/all2sam.pl (from /branches/prog/bam/misc/all2tam.pl:649)
- D /branches/prog/bam/misc/all2tam.pl
- A /branches/prog/bam/misc/maq2sam.c (from /branches/prog/bam/misc/maq2tam.c:699)
- D /branches/prog/bam/misc/maq2tam.c
-
-rename tam to sam
-
-------------------------------------------------------------------------
-r699 | lh3 | 2008-11-25 13:14:49 +0000 (Tue, 25 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/misc/maq2tam.c
-
-change for the new specification
-
-------------------------------------------------------------------------
-r698 | lh3 | 2008-11-24 13:15:20 +0000 (Mon, 24 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/razf.c
- M /branches/prog/bam/razf.h
-
- * add a fake BGZF mode to razf. It is fake in that it loads razf index into
- memory but gives BGZF like virtual offset
-
-------------------------------------------------------------------------
-r697 | lh3 | 2008-11-24 09:53:44 +0000 (Mon, 24 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/ChangeLog
-
-change log
-
-------------------------------------------------------------------------
-r696 | lh3 | 2008-11-24 09:53:23 +0000 (Mon, 24 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bgzf.c
-
-updated bgzf, on behalf of Bob
-
-------------------------------------------------------------------------
-r695 | lh3 | 2008-11-23 11:40:31 +0000 (Sun, 23 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/razf.c
-
-fixed a bug in razf
-
-------------------------------------------------------------------------
-r694 | lh3 | 2008-11-22 16:23:52 +0000 (Sat, 22 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_lpileup.c
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bam-0.1.0-40
- * fixed two small memory leaks
- * fixed a memory problem when seek outside the length of the sequence
-
-------------------------------------------------------------------------
-r693 | lh3 | 2008-11-22 16:10:04 +0000 (Sat, 22 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bam-0.1.0-39
- * fixed an uninitialized warning. This does not matter in fact
-
-------------------------------------------------------------------------
-r692 | lh3 | 2008-11-22 15:44:05 +0000 (Sat, 22 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/razf.c
- M /branches/prog/bam/razf.h
-
-Jue's new razf
-
-------------------------------------------------------------------------
-r691 | lh3 | 2008-11-21 21:30:39 +0000 (Fri, 21 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/bgzip.c
-
- * bam-0.1.0-38
- * get rid of some warings in bgzip.c
- * potentially improve performance in indexing for BGZF
-
-------------------------------------------------------------------------
-r690 | lh3 | 2008-11-21 21:15:51 +0000 (Fri, 21 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bgzf.c
-
-I think I have fixed the bug in bgzf
-
-------------------------------------------------------------------------
-r689 | lh3 | 2008-11-21 20:48:56 +0000 (Fri, 21 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bgzf.c
-
-bug fix by Bob
-
-------------------------------------------------------------------------
-r688 | lh3 | 2008-11-21 20:37:27 +0000 (Fri, 21 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
-
-fixed a bug due to the name change in _IOLIB
-
-------------------------------------------------------------------------
-r687 | lh3 | 2008-11-21 14:42:56 +0000 (Fri, 21 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bgzf.c
-
-fix small things
-
-------------------------------------------------------------------------
-r686 | lh3 | 2008-11-21 14:37:59 +0000 (Fri, 21 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/bgzf.c
- A /branches/prog/bam/bgzf.h
- A /branches/prog/bam/bgzip.c
-
-Bob's BGZF format, although currently buggy
-
-------------------------------------------------------------------------
-r685 | lh3 | 2008-11-21 09:48:20 +0000 (Fri, 21 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bam-0.1.0-37
- * improve interface a little bit
-
-------------------------------------------------------------------------
-r684 | lh3 | 2008-11-21 09:30:18 +0000 (Fri, 21 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bam-0.1.0-36
- * improve the interface of tview, a little bit
-
-------------------------------------------------------------------------
-r683 | lh3 | 2008-11-20 22:33:54 +0000 (Thu, 20 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam_tview.c
-
-a little better viewer
-
-------------------------------------------------------------------------
-r682 | lh3 | 2008-11-20 22:27:01 +0000 (Thu, 20 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-35
- * better viewer
-
-------------------------------------------------------------------------
-r681 | lh3 | 2008-11-20 20:51:16 +0000 (Thu, 20 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-34
- * tview is now a component of bamtk
-
-------------------------------------------------------------------------
-r680 | lh3 | 2008-11-20 19:17:30 +0000 (Thu, 20 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/bam_tview.c
-
-text alignment viewer
-
-------------------------------------------------------------------------
-r679 | lh3 | 2008-11-20 19:17:15 +0000 (Thu, 20 Nov 2008) | 5 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_lpileup.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.1
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/faidx.c
-
- * bamtk-0.1.0-33
- * added routines to reset pileup bufferes
- * fixed a bug in faidx
- * add text alignment viewer
-
-------------------------------------------------------------------------
-r678 | lh3 | 2008-11-20 11:05:02 +0000 (Thu, 20 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- A /branches/prog/bam/bam_lpileup.c (from /branches/prog/bam/bam_tview.c:668)
- D /branches/prog/bam/bam_tview.c
-
-rename tview as lpileup
-
-------------------------------------------------------------------------
-r677 | lh3 | 2008-11-20 10:08:52 +0000 (Thu, 20 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/razf.c
-
-fixed a bug in razf
-
-------------------------------------------------------------------------
-r676 | lh3 | 2008-11-19 22:52:20 +0000 (Wed, 19 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/faidx.h
-
-add documentations
-
-------------------------------------------------------------------------
-r674 | lh3 | 2008-11-19 21:39:17 +0000 (Wed, 19 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bamtk.1
- M /branches/prog/bam/faidx.h
-
-update documentation
-
-------------------------------------------------------------------------
-r673 | lh3 | 2008-11-19 21:19:03 +0000 (Wed, 19 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/bamtk.1
-
-add manual page
-
-------------------------------------------------------------------------
-r672 | lh3 | 2008-11-19 16:40:49 +0000 (Wed, 19 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/faidx.c
-
- * bamtk-0.1.0-32
- * make faidx more error resistant
-
-------------------------------------------------------------------------
-r671 | lh3 | 2008-11-19 16:09:55 +0000 (Wed, 19 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/faidx.h
-
-add index
-
-------------------------------------------------------------------------
-r670 | lh3 | 2008-11-19 16:02:39 +0000 (Wed, 19 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/faidx.c
-
- * bamtk-0.1.0-31
- * show reference sequence in pileup -v (not in the default pileup)
-
-------------------------------------------------------------------------
-r669 | lh3 | 2008-11-19 14:51:17 +0000 (Wed, 19 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/faidx.c
-
- * bamtk-0.1.0-30
- * put faidx in bamtk and remove faidx_main.c
-
-------------------------------------------------------------------------
-r668 | lh3 | 2008-11-19 14:15:05 +0000 (Wed, 19 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
- A /branches/prog/bam/faidx.c
- A /branches/prog/bam/faidx.h
- M /branches/prog/bam/razf.c
-
- * bamtk-0.1.0-29
- * fixed a bug in tview.c
- * prepare to add faidx
-
-------------------------------------------------------------------------
-r667 | lh3 | 2008-11-19 10:20:45 +0000 (Wed, 19 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/razf.c
- M /branches/prog/bam/razf.h
-
-gzip-compatible razf
-
-------------------------------------------------------------------------
-r664 | lh3 | 2008-11-18 12:50:23 +0000 (Tue, 18 Nov 2008) | 5 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-28
- * fetch: fixed a bug at an array boundary
- * fetch: fixed a bug when the whole chromosome is retrieved
- * add linear index
-
-------------------------------------------------------------------------
-r663 | lh3 | 2008-11-17 21:29:22 +0000 (Mon, 17 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-27
- * put l_qseq into core and move l_aux to bam1_t
-
-------------------------------------------------------------------------
-r662 | lh3 | 2008-11-17 20:55:16 +0000 (Mon, 17 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-26
- * save seq and qual separately
-
-------------------------------------------------------------------------
-r661 | lh3 | 2008-11-17 13:09:37 +0000 (Mon, 17 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
-
-little
-
-------------------------------------------------------------------------
-r660 | lh3 | 2008-11-17 13:06:14 +0000 (Mon, 17 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
-
-more documentations
-
-------------------------------------------------------------------------
-r659 | lh3 | 2008-11-17 12:55:08 +0000 (Mon, 17 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-25
- * make tview work for TAM
-
-------------------------------------------------------------------------
-r658 | lh3 | 2008-11-17 12:50:21 +0000 (Mon, 17 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-24
- * make tview as an independent module
-
-------------------------------------------------------------------------
-r657 | lh3 | 2008-11-17 11:26:06 +0000 (Mon, 17 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_pileup.c
-
-change little
-
-------------------------------------------------------------------------
-r656 | lh3 | 2008-11-16 21:33:19 +0000 (Sun, 16 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-23
- * also add tview for TAM
-
-------------------------------------------------------------------------
-r655 | lh3 | 2008-11-16 21:29:46 +0000 (Sun, 16 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-22
- * make tview more efficient for deep depth
-
-------------------------------------------------------------------------
-r654 | lh3 | 2008-11-16 20:52:19 +0000 (Sun, 16 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_pileup.c
- A /branches/prog/bam/bam_tview.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-21
- * fixed bug in the TAM parser: lowercase not recognized
- * unfinished function to leveled pileup (tview)
-
-------------------------------------------------------------------------
-r653 | lh3 | 2008-11-15 12:58:36 +0000 (Sat, 15 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-20
- * pileup now display deleted bases as '*'
-
-------------------------------------------------------------------------
-r652 | lh3 | 2008-11-15 09:58:39 +0000 (Sat, 15 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-19
- * fixed a bug in fetch()
- * reduce memory in indexing
-
-------------------------------------------------------------------------
-r651 | lh3 | 2008-11-14 21:56:05 +0000 (Fri, 14 Nov 2008) | 5 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-18
- * important changes are made to index: the index size is increased, but
- now we have no limit on file sizes and the new method potentially
- works with BGZF, Bob's new compression format.
-
-------------------------------------------------------------------------
-r650 | lh3 | 2008-11-14 16:03:22 +0000 (Fri, 14 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-17
- * more comments in bam.h
- * fixed a bug in bam_index.c
-
-------------------------------------------------------------------------
-r649 | lh3 | 2008-11-13 16:04:18 +0000 (Thu, 13 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bam_sort.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-16
- * use macros to retrieve pointers from bam1_t and thus reduce the size
- of bam1_t struct.
-
-------------------------------------------------------------------------
-r648 | lh3 | 2008-11-13 13:21:39 +0000 (Thu, 13 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_sort.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-15
- * make more things work over pipe
-
-------------------------------------------------------------------------
-r647 | lh3 | 2008-11-13 12:49:28 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/misc/maq2tam.c
-
-fixed a bug in maq2tam
-
-------------------------------------------------------------------------
-r646 | lh3 | 2008-11-13 11:46:59 +0000 (Thu, 13 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/Makefile
- M /branches/prog/bam/misc/Makefile
- M /branches/prog/bam/misc/maq2tam.c
-
- * bug fix in maq2tam.c
- * improve Makefile
-
-------------------------------------------------------------------------
-r645 | lh3 | 2008-11-13 11:39:46 +0000 (Thu, 13 Nov 2008) | 3 lines
-Changed paths:
- A /branches/prog/bam/misc/Makefile
- M /branches/prog/bam/misc/maq2tam.c
-
- * corrected maq2tam
- * add Makefile
-
-------------------------------------------------------------------------
-r644 | lh3 | 2008-11-13 11:25:45 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/razf.c
-
-fixed the bug in buffered write (on behalf of Jue)
-
-------------------------------------------------------------------------
-r643 | lh3 | 2008-11-13 10:53:42 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- D /branches/prog/bam/all2tam.pl
- A /branches/prog/bam/misc/all2tam.pl (from /branches/prog/bam/all2tam.pl:642)
-
-move to misc
-
-------------------------------------------------------------------------
-r642 | lh3 | 2008-11-13 10:53:23 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/all2tam.pl
-
-change tag
-
-------------------------------------------------------------------------
-r641 | lh3 | 2008-11-13 10:53:12 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- D /branches/prog/bam/utils
-
-has been renamed
-
-------------------------------------------------------------------------
-r640 | lh3 | 2008-11-13 10:52:50 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/misc (from /branches/prog/bam/utils:639)
-
-rename
-
-------------------------------------------------------------------------
-r639 | lh3 | 2008-11-13 10:52:35 +0000 (Thu, 13 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam/utils
- A /branches/prog/bam/utils/maq2tam.c
-
-utilities (converters and so on)
-
-------------------------------------------------------------------------
-r638 | lh3 | 2008-11-12 22:24:22 +0000 (Wed, 12 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-14
- * copy the text header to BAM
- * add BAM1 header flag
-
-------------------------------------------------------------------------
-r637 | lh3 | 2008-11-12 14:56:08 +0000 (Wed, 12 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/razf.c
-
- * bamtk-0.1.0-13
- * fixed a bug in razf
- * improved and fixed potential bugs in index
-
-------------------------------------------------------------------------
-r636 | lh3 | 2008-11-12 11:57:13 +0000 (Wed, 12 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
-update documentation in the HeaderDOC format
-
-------------------------------------------------------------------------
-r635 | lh3 | 2008-11-12 10:08:38 +0000 (Wed, 12 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-12
- * more documentations
- * rename baf1_core_t as bam1_core_t
-
-------------------------------------------------------------------------
-r634 | lh3 | 2008-11-11 23:00:35 +0000 (Tue, 11 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_pileup.c
-
-documentation
-
-------------------------------------------------------------------------
-r633 | lh3 | 2008-11-11 21:23:49 +0000 (Tue, 11 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-11
- * give up regional pileup. We can now use pipe to mimic that.
- * for index file, change suffix .idx to .bmi
-
-------------------------------------------------------------------------
-r632 | lh3 | 2008-11-11 21:00:11 +0000 (Tue, 11 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/razf.c
-
- * bamtk-0.1.0-10
- * make pileup work on TAM
-
-------------------------------------------------------------------------
-r631 | lh3 | 2008-11-11 09:20:29 +0000 (Tue, 11 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/razf.c
- M /branches/prog/bam/razf.h
- M /branches/prog/bam/razip.c
-
- * bamtk-0.1.0-9
- * razf now supports streaming
- * prepare to improve pileup (have not yet)
-
-------------------------------------------------------------------------
-r630 | lh3 | 2008-11-10 18:34:40 +0000 (Mon, 10 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-8
- * improve the interface of TAM parser
-
-------------------------------------------------------------------------
-r629 | lh3 | 2008-11-10 13:06:13 +0000 (Mon, 10 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-7
- * almost nothing
-
-------------------------------------------------------------------------
-r628 | lh3 | 2008-11-10 12:56:36 +0000 (Mon, 10 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-6
- * fixed a bug in bam_pileup.c
-
-------------------------------------------------------------------------
-r627 | lh3 | 2008-11-10 11:32:46 +0000 (Mon, 10 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bamtk.c
- M /branches/prog/bam/razf.c
-
- * bamtk-0.1.0-5
- * fixed a bug in razf.c, caused by my modifications
- * improve the interface of pileup. Now it will be slower but more flexible
-
-------------------------------------------------------------------------
-r626 | lh3 | 2008-11-09 20:51:04 +0000 (Sun, 09 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-4
- * view: dumping binary output
-
-------------------------------------------------------------------------
-r625 | lh3 | 2008-11-09 20:31:54 +0000 (Sun, 09 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bam_pileup.c
- M /branches/prog/bam/bam_sort.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-3
- * rename functions
-
-------------------------------------------------------------------------
-r624 | lh3 | 2008-11-09 15:07:32 +0000 (Sun, 09 Nov 2008) | 2 lines
-Changed paths:
- M /branches/prog/bam/bam.h
-
-add comments
-
-------------------------------------------------------------------------
-r623 | lh3 | 2008-11-08 22:32:49 +0000 (Sat, 08 Nov 2008) | 4 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-2
- * improve indexing for a mixture of long and short reads, although currently
- I do not know whether it really works...
-
-------------------------------------------------------------------------
-r622 | lh3 | 2008-11-08 22:13:58 +0000 (Sat, 08 Nov 2008) | 3 lines
-Changed paths:
- M /branches/prog/bam/bam_index.c
- M /branches/prog/bam/bamtk.c
-
- * bamtk-0.1.0-1
- * prepare for improving indexing algorithm
-
-------------------------------------------------------------------------
-r621 | lh3 | 2008-11-08 20:28:09 +0000 (Sat, 08 Nov 2008) | 4 lines
-Changed paths:
- A /branches/prog/bam/all2tam.pl
- M /branches/prog/bam/bam.c
- M /branches/prog/bam/bam.h
- M /branches/prog/bam/bam_import.c
- M /branches/prog/bam/bamtk.c
- D /branches/prog/bam/tam_utils.pl
-
- * bamtk-0.1.0
- * smarter integers
- * rename tam_utils.pl to all2tam.pl
-
-------------------------------------------------------------------------
-r620 | lh3 | 2008-11-08 17:17:22 +0000 (Sat, 08 Nov 2008) | 2 lines
-Changed paths:
- A /branches/prog/bam
- A /branches/prog/bam/Makefile
- A /branches/prog/bam/bam.c
- A /branches/prog/bam/bam.h
- A /branches/prog/bam/bam_endian.h
- A /branches/prog/bam/bam_import.c
- A /branches/prog/bam/bam_index.c
- A /branches/prog/bam/bam_pileup.c
- A /branches/prog/bam/bam_sort.c
- A /branches/prog/bam/bamtk.c
- A /branches/prog/bam/khash.h
- A /branches/prog/bam/ksort.h
- A /branches/prog/bam/kstream.h
- A /branches/prog/bam/razf.c
- A /branches/prog/bam/razf.h
- A /branches/prog/bam/razip.c
- A /branches/prog/bam/tam_utils.pl
- A /branches/prog/bam/zutil.h
-
-The Binary Alignment/Mapping format.
-
-------------------------------------------------------------------------
diff --git a/samtools-0.1.19/INSTALL b/samtools-0.1.19/INSTALL
deleted file mode 100644
index 37d84a9..0000000
--- a/samtools-0.1.19/INSTALL
+++ /dev/null
@@ -1,30 +0,0 @@
-System Requirements
-===================
-
-SAMtools depends on the zlib library <http://www.zlib.net>. Version 1.2.3+ is
-preferred and with 1.2.3+ you can compile razip and use it to compress a FASTA
-file. SAMtools' faidx is able to index a razip-compressed FASTA file to save
-diskspace. Older zlib also works with SAMtools, but razip cannot be compiled.
-
-The text-based viewer (tview) requires the GNU ncurses library
-<http://www.gnu.org/software/ncurses/>, which comes with Mac OS X and most of
-the modern Linux/Unix distributions. If you do not have this library installed,
-you can still compile the rest of SAMtools by manually changing:
-`-D_CURSES_LIB=1' to `-D_CURSES_LIB=0' at the line starting with `DFLAGS=', and
-comment out the line starting with `LIBCURSES='.
-
-
-Compilation
-===========
-
-Type `make' to compile samtools. If you have zlib >= 1.2.2.1, you can compile
-razip with `make razip'.
-
-
-Installation
-============
-
-Copy `samtools', `bcftools/bcftools' and other executables/scripts in `misc' to
-a location you want (e.g. a directory in your $PATH). You may also copy
-`samtools.1' and `bcftools/bcftools.1' to a directory in your $MANPATH such
-that the `man' command may find the manual.
diff --git a/samtools-0.1.19/Makefile b/samtools-0.1.19/Makefile
deleted file mode 100644
index 2f51bfc..0000000
--- a/samtools-0.1.19/Makefile
+++ /dev/null
@@ -1,101 +0,0 @@
-CC= gcc
-CFLAGS= -g -Wall -O2
-#LDFLAGS= -Wl,-rpath,\$$ORIGIN/../lib
-DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_USE_KNETFILE -D_CURSES_LIB=1
-KNETFILE_O= knetfile.o
-LOBJS= bgzf.o kstring.o bam_aux.o bam.o bam_import.o sam.o bam_index.o \
- bam_pileup.o bam_lpileup.o bam_md.o razf.o faidx.o bedidx.o \
- $(KNETFILE_O) bam_sort.o sam_header.o bam_reheader.o kprobaln.o bam_cat.o
-AOBJS= bam_tview.o bam_plcmd.o sam_view.o \
- bam_rmdup.o bam_rmdupse.o bam_mate.o bam_stat.o bam_color.o \
- bamtk.o kaln.o bam2bcf.o bam2bcf_indel.o errmod.o sample.o \
- cut_target.o phase.o bam2depth.o padding.o bedcov.o bamshuf.o \
- bam_tview_curses.o bam_tview_html.o
-PROG= samtools
-INCLUDES= -I.
-SUBDIRS= . bcftools misc
-LIBPATH=
-LIBCURSES= -lcurses # -lXCurses
-
-.SUFFIXES:.c .o
-.PHONY: all lib
-
-.c.o:
- $(CC) -c $(CFLAGS) $(DFLAGS) $(INCLUDES) $< -o $@
-
-all-recur lib-recur clean-recur cleanlocal-recur install-recur:
- @target=`echo $@ | sed s/-recur//`; \
- wdir=`pwd`; \
- list='$(SUBDIRS)'; for subdir in $$list; do \
- cd $$subdir; \
- $(MAKE) CC="$(CC)" DFLAGS="$(DFLAGS)" CFLAGS="$(CFLAGS)" \
- INCLUDES="$(INCLUDES)" LIBPATH="$(LIBPATH)" $$target || exit 1; \
- cd $$wdir; \
- done;
-
-all:$(PROG)
-
-.PHONY:all lib clean cleanlocal
-.PHONY:all-recur lib-recur clean-recur cleanlocal-recur install-recur
-
-lib:libbam.a
-
-libbam.a:$(LOBJS)
- $(AR) -csru $@ $(LOBJS)
-
-samtools:lib-recur $(AOBJS)
- $(CC) $(CFLAGS) -o $@ $(AOBJS) $(LDFLAGS) libbam.a -Lbcftools -lbcf $(LIBPATH) $(LIBCURSES) -lm -lz -lpthread
-
-razip:razip.o razf.o $(KNETFILE_O)
- $(CC) $(CFLAGS) -o $@ $^ -lz
-
-bgzip:bgzip.o bgzf.o $(KNETFILE_O)
- $(CC) $(CFLAGS) -o $@ $^ -lz -lpthread
-
-bgzf.o:bgzf.c bgzf.h
- $(CC) -c $(CFLAGS) $(DFLAGS) -DBGZF_CACHE $(INCLUDES) bgzf.c -o $@
-
-razip.o:razf.h
-bam.o:bam.h razf.h bam_endian.h kstring.h sam_header.h
-sam.o:sam.h bam.h
-bam_import.o:bam.h kseq.h khash.h razf.h
-bam_pileup.o:bam.h razf.h ksort.h
-bam_plcmd.o:bam.h faidx.h bcftools/bcf.h bam2bcf.h
-bam_index.o:bam.h khash.h ksort.h razf.h bam_endian.h
-bam_lpileup.o:bam.h ksort.h
-bam_tview.o:bam.h faidx.h bam_tview.h
-bam_tview_curses.o:bam.h faidx.h bam_tview.h
-bam_tview_html.o:bam.h faidx.h bam_tview.h
-bam_sort.o:bam.h ksort.h razf.h
-bam_md.o:bam.h faidx.h
-sam_header.o:sam_header.h khash.h
-bcf.o:bcftools/bcf.h
-bam2bcf.o:bam2bcf.h errmod.h bcftools/bcf.h
-bam2bcf_indel.o:bam2bcf.h
-errmod.o:errmod.h
-phase.o:bam.h khash.h ksort.h
-bamtk.o:bam.h
-
-faidx.o:faidx.h razf.h khash.h
-faidx_main.o:faidx.h razf.h
-
-
-libbam.1.dylib-local:$(LOBJS)
- libtool -dynamic $(LOBJS) -o libbam.1.dylib -lc -lz
-
-libbam.so.1-local:$(LOBJS)
- $(CC) -shared -Wl,-soname,libbam.so -o libbam.so.1 $(LOBJS) -lc -lz
-
-dylib:
- @$(MAKE) cleanlocal; \
- case `uname` in \
- Linux) $(MAKE) CFLAGS="$(CFLAGS) -fPIC" libbam.so.1-local;; \
- Darwin) $(MAKE) CFLAGS="$(CFLAGS) -fPIC" libbam.1.dylib-local;; \
- *) echo 'Unknown OS';; \
- esac
-
-
-cleanlocal:
- rm -fr gmon.out *.o a.out *.exe *.dSYM razip bgzip $(PROG) *~ *.a *.so.* *.so *.dylib
-
-clean:cleanlocal-recur
diff --git a/samtools-0.1.19/Makefile.mingw b/samtools-0.1.19/Makefile.mingw
deleted file mode 100644
index 7a57ffc..0000000
--- a/samtools-0.1.19/Makefile.mingw
+++ /dev/null
@@ -1,63 +0,0 @@
-CC= gcc.exe
-AR= ar.exe
-CFLAGS= -g -Wall -O2
-DFLAGS= -D_USE_KNETFILE -D_CURSES_LIB=2
-KNETFILE_O= knetfile.o
-LOBJS= bgzf.o kstring.o bam_aux.o bam.o bam_import.o sam.o bam_index.o \
- bam_pileup.o bam_lpileup.o bam_md.o razf.o faidx.o \
- $(KNETFILE_O) bam_sort.o sam_header.o bam_reheader.o kprobaln.o bedidx.o
-AOBJS= bam_tview.o bam_plcmd.o sam_view.o \
- bam_rmdup.o bam_rmdupse.o bam_mate.o bam_stat.o bam_color.o \
- bamtk.o kaln.o bam2bcf.o bam2bcf_indel.o errmod.o sample.o \
- cut_target.o phase.o bam_cat.o bam2depth.o
-BCFOBJS= bcftools/bcf.o bcftools/fet.o bcftools/bcf2qcall.o bcftools/bcfutils.o \
- bcftools/call1.o bcftools/index.o bcftools/kfunc.o bcftools/em.o \
- bcftools/kmin.o bcftools/prob1.o bcftools/vcf.o bcftools/mut.o
-PROG= samtools.exe bcftools.exe
-INCLUDES= -I. -Iwin32
-SUBDIRS= .
-LIBPATH=
-
-.SUFFIXES:.c .o
-
-.c.o:
- $(CC) -c $(CFLAGS) $(DFLAGS) $(INCLUDES) $< -o $@
-
-all:$(PROG)
-
-.PHONY:all lib clean cleanlocal
-.PHONY:all-recur lib-recur clean-recur cleanlocal-recur install-recur
-
-lib:libbam.a
-
-libbam.a:$(LOBJS)
- $(AR) -cru $@ $(LOBJS)
-
-samtools.exe:$(AOBJS) libbam.a $(BCFOBJS)
- $(CC) $(CFLAGS) -o $@ $(AOBJS) $(BCFOBJS) $(LIBPATH) -lm -L. -lbam -Lwin32 -lz -lcurses -lws2_32
-
-bcftools.exe:$(BCFOBJS) bcftools/main.o kstring.o bgzf.o knetfile.o bedidx.o
- $(CC) $(CFLAGS) -o $@ $(BCFOBJS) bcftools/main.o kstring.o bgzf.o knetfile.o bedidx.o -lm -Lwin32 -lz -lws2_32
-
-razip.o:razf.h
-bam.o:bam.h razf.h bam_endian.h kstring.h sam_header.h
-sam.o:sam.h bam.h
-bam_import.o:bam.h kseq.h khash.h razf.h
-bam_pileup.o:bam.h razf.h ksort.h
-bam_plcmd.o:bam.h faidx.h bcftools/bcf.h bam2bcf.h
-bam_index.o:bam.h khash.h ksort.h razf.h bam_endian.h
-bam_lpileup.o:bam.h ksort.h
-bam_tview.o:bam.h faidx.h
-bam_sort.o:bam.h ksort.h razf.h
-bam_md.o:bam.h faidx.h
-sam_header.o:sam_header.h khash.h
-bcf.o:bcftools/bcf.h
-bam2bcf.o:bam2bcf.h errmod.h bcftools/bcf.h
-bam2bcf_indel.o:bam2bcf.h
-errmod.o:errmod.h
-
-faidx.o:faidx.h razf.h khash.h
-faidx_main.o:faidx.h razf.h
-
-clean:
- rm -fr gmon.out *.o a.out *.exe *.dSYM razip bgzip $(PROG) *~ *.a *.so.* *.so *.dylib
diff --git a/samtools-0.1.19/NEWS b/samtools-0.1.19/NEWS
deleted file mode 100644
index 121485e..0000000
--- a/samtools-0.1.19/NEWS
+++ /dev/null
@@ -1,836 +0,0 @@
-Beta Release 0.1.19 (15 March, 2013)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes in samtools and bcftools:
-
- * The latest source code and development moved to github,
- http://github.com/samtools/samtools
-
- * Many important bugfixes and contributions by many people. Thanks to all!
-
- * Performance improvements (multi-threading)
-
- * Important changes in calling, see
- - samtools mpileup -p
- - bcftools view -m
-
- * New annotations useful for filtering (RPB, HWE, QBD, MDV)
-
- * New tools, bamcheck and plot-bamcheck
-
- * New features in samtools tview
-
- * And much more..
-
-For a detailed list of commits, please see
-http://github.com/samtools/samtools/commits/master
-
-(0.1.19: 15 March 2013, commit 96b5f2294ac0054230e88913c4983d548069ea4e)
-
-
-Beta Release 0.1.18 (2 September, 2011)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes in samtools:
-
- * Support the new =/X CIGAR operators (by Peter Cock).
-
- * Allow to subsample BAM while keeping the pairing intact (view -s).
-
- * Implemented variant distance bias as a new filter (by Petr Danecek).
-
- * Bugfix: huge memory usage during indexing
-
- * Bugfix: use of uninitialized variable in mpileup (rare)
-
- * Bugfix: wrong BAQ probability (rare)
-
-Notable changes in bcftools:
-
- * Support indel in the contrast caller.
-
- * Bugfix: LRT2=nan in rare cases
-
-(0.1.18: 2 September 2011, r982:295)
-
-
-
-Beta Release 0.1.17 (6 July, 2011)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-With the maturity of `mpileup' and the lack of update in the `pileup' command,
-the `pileup' command is now formally dropped. Most of the pileup functionality,
-such as outputting mapping quality and read positions, have been added
-`mpileup'.
-
-Since this release, `bcftools view' is able to perform contrast SNP calling
-(option -T) for discovering de novo and/or somatic mutations between a pair of
-samples or in a family trio. Potential mutations are scored by a log likelihood
-ratio, which is very simple in math, but should be comparable to more
-sophisticated methods. Note that getting the score is only the very first step.
-A lot more need to be done to reduce systematical errors due to mapping and
-reference errors and structural variations.
-
-Other notable changes in samtools:
-
- * Improved sorting order checking during indexing.
-
- * Improved region parsing. Colons in reference sequence names are parsed
- properly.
-
- * Fixed an issue where mpileup does not apply BAQ for the first few reads when
- a region is specified.
-
- * Fixed an issue where `faidx' does not work with FASTA files with long lines.
-
- * Bugfix: wrong SP genotype information in the BCF output.
-
-Other notable changes in bcftools:
-
- * Output the ML esitmate of the allele count.
-
- * Added the HWE plus F<0 filter to varFilter. For multiple samples, it
- effectively filters false heterozygous calls around centromeres.
-
- * For association mapping, perform both 1-degree and 2-degree test. The
- 2-degree test is conservative but more robust to HWE violation.
-
-(0.1.17: 6 July 2011, r973:277)
-
-
-
-Beta Release 0.1.16 (21 April, 2011)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes in samtools:
-
- * Support the new SAM/BAM type `B' in the latest SAM spec v1.4.
-
- * When the output file of `samtools merge' exists, do not overwrite it unless
- a new command-line option `-f' is applied.
-
- * Bugfix: BED support is not working when the input BED is not sorted.
-
- * Bugfix: some reads without coordinates but given on the reverse strand are
- lost in merging.
-
-Notable changes in bcftools:
-
- * Code cleanup: separated max-likelihood inference and Bayesian inference.
-
- * Test Hardy-Weinberg equilibrium with a likelihood-ratio test.
-
- * Provided another association test P-value by likelihood-ratio test.
-
- * Use Brent's method to estimate the site allele frequency when EM converges
- slowly. The resulting ML estimate of allele frequnecy is more accurate.
-
- * Added the `ldpair' command, which computes r^2 between SNP pairs given in
- an input file.
-
-Also, the `pileup' command, which has been deprecated by `mpileup' since
-version 0.1.10, will be dropped in the next release. The old `pileup' command
-is substandard and causing a lot of confusion.
-
-(0.1.16: 21 April 2011, r963:234)
-
-
-
-Beta Release 0.1.15 (10 April, 2011)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Noteable changes:
-
- * Allow to perform variant calling or to extract information in multiple
- regions specified by a BED file (`samtools mpileup -l', `samtools view -L'
- and `bcftools view -l').
-
- * Added the `depth' command to samtools to compute the per-base depth with a
- simpler interface. File `bam2depth.c', which implements this command, is the
- recommended example on how to use the mpileup APIs.
-
- * Estimate genotype frequencies with ML; perform chi^2 based Hardy-Weinberg
- test using this estimate.
-
- * For `samtools view', when `-R' is specified, drop read groups in the header
- that are not contained in the specified file.
-
- * For `samtools flagstat', separate QC-pass and QC-fail reads.
-
- * Improved the command line help of `samtools mpileup' and `bcftools view'.
-
- * Use a global variable to control the verbose level of samtools stderr
- output. Nonetheless, it has not been full utilized.
-
- * Fixed an issue in association test which may report false associations,
- possibly due to floating point underflow.
-
-(0.1.15: 10 April 2011, r949:203)
-
-
-
-Beta release 0.1.14 (21 March, 2011)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This release implements a method for testing associations for case-control
-data. The method does not call genotypes but instead sums over all genotype
-configurations to compute a chi^2 based test statistics. It can be potentially
-applied to comparing a pair of samples (e.g. a tumor-normal pair), but this
-has not been evaluated on real data.
-
-Another new feature is to make X chromosome variant calls when female and male
-samples are both present. The user needs to provide a file indicating the
-ploidy of each sample (see also manual bcftools/bcftools.1).
-
-Other notable changes:
-
- * Added `bcftools view -F' to parse BCF files generated by samtools r921 or
- older which encodes PL in a different way.
-
- * Changed the behavior of `bcftools view -s'. Now when a list of samples is
- provided, the samples in the output will be reordered to match the ordering
- in the sample list. This change is mainly designed for association test.
-
- * Sped up `bcftools view -v' for target sequencing given thousands of samples.
- Also added a new option `view -d' to skip loci where only a few samples are
- covered by reads.
-
- * Dropped HWE test. This feature has never been implemented properly. An EM
- should be much better. To be implemented in future.
-
- * Added the `cat' command to samtools. This command concatenate BAMs with
- identical sequence dictionaries in an efficient way. Modified from bam_cat.c
- written by Chris Saunders.
-
- * Added `samtools view -1' to write BAMs at a low compression level but twice
- faster to create. The `sort' command generates temporary files at a low
- compression level as well.
-
- * Added `samtools mpileup -6' to accept "BAM" with Illumina 1.3+ quality
- strings (strictly speaking, such a file is not BAM).
-
- * Added `samtools mpileup -L' to skip INDEL calling in regions with
- excessively high coverage. Such regions dramatically slow down mpileup.
-
- * Updated `misc/export2sam.pl', provided by Chris Saunders from Illumina Inc.
-
-(0.1.14: 21 March 2011, r933:170)
-
-
-
-Beta release 0.1.13 (1 March, 2011)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The most important though largely invisible modification is the change of the
-order of genotypes in the PL VCF/BCF tag. This is to conform the upcoming VCF
-spec v4.1. The change means that 0.1.13 is not backward compatible with VCF/BCF
-generated by samtools older than r921 inclusive. VCF/BCF generated by the new
-samtools will contain a line `##fileformat=VCFv4.1' as well as the samtools
-version number.
-
-Single Individual Haplotyping (SIH) is added as an experimental feature. It
-originally aims to produce haploid consensus from fosmid pool sequencing, but
-also works with short-read data. For short reads, phased blocks are usually too
-short to be useful in many applications, but they can help to rule out part of
-SNPs close to INDELs or between copies of CNVs.
-
-
-Other notable changes in samtools:
-
- * Construct per-sample consensus to reduce the effect of nearby SNPs in INDEL
- calling. This reduces the power but improves specificity.
-
- * Improved sorting order checking in indexing. Now indexing is the preferred way
- to check if a BAM is sorted.
-
- * Added a switch `-E' to mpileup and calmd. This option uses an alternative way
- to apply BAQ, which increases sensistivity, especially to MNPs, at the cost of
- a little loss in specificity.
-
- * Added `mpileup -A' to allow to use reads in anomalous pairs in SNP calling.
-
- * Added `mpileup -m' to allow fine control of the collection of INDEL candidates.
-
- * Added `mpileup -S' to compute per-sample strand bias P-value.
-
- * Added `mpileup -G' to exclude read groups in variant calling.
-
- * Fixed segfault in indel calling related to unmapped and refskip reads.
-
- * Fixed an integer overflow in INDEL calling. This bug produces wrong INDEL
- genotypes for longer short INDELs, typically over 10bp.
-
- * Fixed a bug in tview on big-endian machines.
-
- * Fixed a very rare memory issue in bam_md.c
-
- * Fixed an out-of-boundary bug in mpileup when the read base is `N'.
-
- * Fixed a compiling error when the knetfile library is not used. Fixed a
- library compiling error due to the lack of bam_nt16_nt4_table[] table.
- Suppress a compiling warning related to the latest zlib.
-
-
-Other notable changes in bcftools:
-
- * Updated the BCF spec.
-
- * Added the `FQ' VCF INFO field, which gives the phred-scaled probability
- of all samples being the same (identical to the reference or all homozygous
- variants). Option `view -f' has been dropped.
-
- * Implementated of "vcfutils.pl vcf2fq" to generate a consensus sequence
- similar to "samtools.pl pileup2fq".
-
- * Make sure the GT FORMAT field is always the first FORMAT to conform the VCF
- spec. Drop bcf-fix.pl.
-
- * Output bcftools specific INFO and FORMAT in the VCF header.
-
- * Added `view -s' to call variants from a subset of samples.
-
- * Properly convert VCF to BCF with a user provided sequence dictionary. Nonetheless,
- custom fields are still unparsed and will be stored as a missing value.
-
- * Fixed a minor bug in Fisher's exact test; the results are rarely changed.
-
-
-(0.1.13: 1 March 2011, r926:134)
-
-
-
-Beta release 0.1.12a (2 December, 2010)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This is another bug fix release:
-
- * Fixed a memory violation in mpileup, which causes segfault. Release
- 0.1.9 and above are affected.
-
- * Fixed a memory violation in the indel caller, which does not causes
- segfault, but may potentially affect deletion calls in an unexpected
- way. Release 0.1.10 and above are affected.
-
- * Fixed a bug in computing r-square in bcftools. Few are using this
- functionality and it only has minor effect.
-
- * Fixed a memory leak in bam_fetch().
-
- * Fixed a bug in writing meta information to the BAM index for the last
- sequence. This bug is invisible to most users, but it is a bug anyway.
-
- * Fixed a bug in bcftools which causes false "DP4=0,0,0,0" annotations.
-
-(0.1.12: 2 December 2010, r862)
-
-
-
-Beta release 0.1.11 (21 November, 2010)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This is mainly a bug fix release:
-
- * Fixed a bug in random retrieval (since 0.1.8). It occurs when reads
- are retrieved from a small region containing no reads.
-
- * Fixed a bug in pileup (since 0.1.9). The bug causes an assertion
- failure when the first CIGAR operation is a deletion.
-
- * Improved fault tolerence in remote access.
-
-One minor feature has been implemented in bcftools:
-
- * Added a reference-free variant calling mode. In this mode, a site is
- regarded as a variat iff the sample(s) contains two or more alleles;
- the meaning of the QUAL field in the VCF output is changed
- accordingly. Effectively, the reference allele is irrelevant to the
- result in the new mode, although the reference sequence has to be
- used in realignment when SAMtools computes genotype likelihoods.
-
-In addition, since 0.1.10, the `pileup' command has been deprecated by
-`mpileup' which is more powerful and more accurate. The `pileup' command
-will not be removed in the next few releases, but new features will not
-be added.
-
-(0.1.11: 21 November 2010, r851)
-
-
-
-Beta Release 0.1.10 (16 November, 2010)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This release is featured as the first major improvement to the indel
-caller. The method is similar to the old one implemented in the pileup
-command, but the details are handled more carefully both in theory and
-in practice. As a result, the new indel caller usually gives more
-accurate indel calls, though at the cost of sensitivity. The caller is
-implemented in the mpileup command and is invoked by default. It works
-with multiple samples.
-
-Other notable changes:
-
- * With the -r option, the calmd command writes the difference between
- the original base quality and the BAQ capped base quality at the BQ
- tag but does not modify the base quality. Please use -Ar to overwrite
- the original base quality (the 0.1.9 behavior).
-
- * Allow to set a maximum per-sample read depth to reduce memory. In
- 0.1.9, most of memory is wasted for the ultra high read depth in some
- regions (e.g. the chr1 centromere).
-
- * Optionally write per-sample read depth and per-sample strand bias
- P-value.
-
- * Compute equal-tail (Bayesian) credible interval of site allele
- frequency at the CI95 VCF annotation.
-
- * Merged the vcfutils.pl varFilter and filter4vcf for better SNP/indel
- filtering.
-
-(0.1.10: 16 November 2010, r829)
-
-
-
-Beta Release 0.1.9 (27 October, 2010)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This release is featured as the first major improvement to the samtools'
-SNP caller. It comes with a revised MAQ error model, the support of
-multi-sample SNP calling and the computation of base alignment quality
-(BAQ).
-
-The revised MAQ error model is based on the original model. It solves an
-issue of miscalling SNPs in repetitive regions. Althought such SNPs can
-usually be filtered at a later step, they mess up unfiltered calls. This
-is a theoretical flaw in the original model. The revised MAQ model
-deprecates the orginal MAQ model and the simplified SOAPsnp model.
-
-Multi-sample SNP calling is separated in two steps. The first is done by
-samtools mpileup and the second by a new program, bcftools, which is
-included in the samtools source code tree. Multi-sample SNP calling also
-works for single sample and has the advantage of enabling more powerful
-filtration. It is likely to deprecate pileup in future once a proper
-indel calling method is implemented.
-
-BAQ is the Phred-scaled probability of a read base being wrongly
-aligned. Capping base quality by BAQ has been shown to be very effective
-in suppressing false SNPs caused by misalignments around indels or in
-low-complexity regions with acceptable compromise on computation
-time. This strategy is highly recommended and can be used with other SNP
-callers as well.
-
-In addition to the three major improvements, other notable changes are:
-
- * Changes to the pileup format. A reference skip (the N CIGAR operator)
- is shown as '<' or '>' depending on the strand. Tview is also changed
- accordingly.
-
- * Accelerated pileup. The plain pileup is about 50% faster.
-
- * Regional merge. The merge command now accepts a new option to merge
- files in a specified region.
-
- * Fixed a bug in bgzip and razip which causes source files to be
- deleted even if option -c is applied.
-
- * In APIs, propogate errors to downstream callers and make samtools
- return non-zero values once errors occur.
-
-(0.1.9: 27 October 2010, r783)
-
-
-
-Beta Release 0.1.8 (11 July, 2010)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable functional changes:
-
- * Added the `reheader' command which replaces a BAM header with a new
- header. This command is much faster than replacing header by
- BAM->SAM->BAM conversions.
-
- * Added the `mpileup' command which computes the pileup of multiple
- alignments.
-
- * The `index' command now stores the number of mapped and unmapped
- reads in the index file. This information can be retrieved quickly by
- the new `idxstats' command.
-
- * By default, pileup used the SOAPsnp model for SNP calling. This
- avoids the floating overflow in the MAQ model which leads to spurious
- calls in repetitive regions, although these calls will be immediately
- filtered by varFilter.
-
- * The `tview' command now correctly handles CIGARs like 7I10M and
- 10M1P1I10M which cause assertion failure in earlier versions.
-
- * Tview accepts a region like `=10,000' where `=' stands for the
- current sequence name. This saves typing for long sequence names.
-
- * Added the `-d' option to `pileup' which avoids slow indel calling
- in ultradeep regions by subsampling reads locally.
-
- * Added the `-R' option to `view' which retrieves alignments in read
- groups listed in the specified file.
-
-Performance improvements:
-
- * The BAM->SAM conversion is up to twice faster, depending on the
- characteristic of the input.
-
- * Parsing SAM headers with a lot of reference sequences is now much
- faster.
-
- * The number of lseek() calls per query is reduced when the query
- region contains no read alignments.
-
-Bug fixes:
-
- * Fixed an issue in the indel caller that leads to miscall of indels.
- Note that this solution may not work well when the sequencing indel
- error rate is higher than the rate of SNPs.
-
- * Fixed another issue in the indel caller which may lead to incorrect
- genotype.
-
- * Fixed a bug in `sort' when option `-o' is applied.
-
- * Fixed a bug in `view -r'.
-
-APIs and other changes:
-
- * Added iterator interfaces to random access and pileup. The callback
- interfaces directly call the iterator interfaces.
-
- * The BGZF blocks holding the BAM header are indepedent of alignment
- BGZF blocks. Alignment records shorter than 64kB is guaranteed to be
- fully contained in one BGZF block. This change is fully compatible
- with the old version of samtools/picard.
-
-Changes in other utilities:
-
- * Updated export2sam.pl by Chris Saunders.
-
- * Improved the sam2vcf.pl script.
-
- * Added a Python version of varfilter.py by Aylwyn Scally.
-
-(0.1.8: 11 July 2010, r613)
-
-
-
-Beta Release 0.1.7 (10 November, 2009)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes:
-
- * Improved the indel caller in complex scenariors, in particular for
- long reads. The indel caller is now able to make reasonable indel
- calls from Craig Venter capillary reads.
-
- * Rewrote single-end duplicate removal with improved
- performance. Paired-end reads are not touched.
-
- * Duplicate removal is now library aware. Samtools remove potential
- PCR/optical dupliates inside a library rather than across libraries.
-
- * SAM header is now fully parsed, although this functionality is not
- used in merging and so on.
-
- * In samtools merge, optionally take the input file name as RG-ID and
- attach the RG tag to each alignment.
-
- * Added FTP support in the RAZF library. RAZF-compressed reference
- sequence can be retrieved remotely.
-
- * Improved network support for Win32.
-
- * Samtools sort and merge are now stable.
-
-Changes in other utilities:
-
- * Implemented sam2vcf.pl that converts the pileup format to the VCF
- format.
-
- * This release of samtools is known to work with the latest
- Bio-Samtools Perl module.
-
-(0.1.7: 10 November 2009, r510)
-
-
-
-Beta Release 0.1.6 (2 September, 2009)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes:
-
- * In tview, do not show a blank screen when no reads mapped to the
- corresponding region.
-
- * Implemented native HTTP support in the BGZF library. Samtools is now
- able to directly open a BAM file on HTTP. HTTP proxy is also
- supported via the "http_proxy" environmental variable.
-
- * Samtools is now compitable with the MinGW (win32) compiler and the
- PDCurses library.
-
- * The calmd (or fillmd) command now calculates the NM tag and replaces
- MD tags if they are wrong.
-
- * The view command now recognizes and optionally prints FLAG in HEXs or
- strings to make a SAM file more friendly to human eyes. This is a
- samtools-C extension, not implemented in Picard for the time
- being. Please type `samtools view -?' for more information.
-
- * BAM files now have an end-of-file (EOF) marker to facilitate
- truncation detection. A warning will be given if an on-disk BAM file
- does not have this marker. The warning will be seen on BAM files
- generated by an older version of samtools. It does NO harm.
-
- * New key bindings in tview: `r' to show read names and `s' to show
- reference skip (N operation) as deletions.
-
- * Fixed a bug in `samtools merge -n'.
-
- * Samtools merge now optionally copies the header of a user specified
- SAM file to the resultant BAM output.
-
- * Samtools pileup/tview works with a CIGAR with the first or the last
- operation is an indel.
-
- * Fixed a bug in bam_aux_get().
-
-
-Changes in other utilies:
-
- * Fixed wrong FLAG in maq2sam.
-
-
-(0.1.6: 2 September 2009, r453)
-
-
-
-Beta Release 0.1.5 (7 July, 2009)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes:
-
- * Support opening a BAM alignment on FTP. Users can now use "tview" to
- view alignments at the NCBI ftp site. Please read manual for more
- information.
-
- * In library, propagate errors rather than exit or complain assertion
- failure.
-
- * Simplified the building system and fixed compiling errors caused by
- zlib<1.2.2.1.
-
- * Fixed an issue about lost header information when a SAM is imported
- with "view -t".
-
- * Implemented "samtool.pl varFilter" which filters both SNPs and short
- indels. This command replaces "indelFilter".
-
- * Implemented "samtools.pl pileup2fq" to generate FASTQ consensus from
- pileup output.
-
- * In pileup, cap mapping quality at 60. This helps filtering when
- different aligners are in use.
-
- * In pileup, allow to output variant sites only.
-
- * Made pileup generate correct calls in repetitive region. At the same
- time, I am considering to implement a simplified model in SOAPsnp,
- although this has not happened yet.
-
- * In view, added '-u' option to output BAM without compression. This
- option is preferred when the output is piped to other commands.
-
- * In view, added '-l' and '-r' to get the alignments for one library or
- read group. The "@RG" header lines are now partially parsed.
-
- * Do not include command line utilities to libbam.a.
-
- * Fixed memory leaks in pileup and bam_view1().
-
- * Made faidx more tolerant to empty lines right before or after FASTA >
- lines.
-
-
-Changes in other utilities:
-
- * Updated novo2sam.pl by Colin Hercus, the key developer of novoalign.
-
-
-This release involves several modifications to the key code base which
-may potentially introduce new bugs even though we have tried to minimize
-this by testing on several examples. Please let us know if you catch
-bugs.
-
-(0.1.5: 7 July 2009, r373)
-
-
-
-Beta Release 0.1.4 (21 May, 2009)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes:
-
- * Added the 'rmdupse' command: removing duplicates for SE reads.
-
- * Fixed a critical bug in the indel caller: clipped alignments are not
- processed correctly.
-
- * Fixed a bug in the tview: gapped alignment may be incorrectly
- displayed.
-
- * Unified the interface to BAM and SAM I/O. This is done by
- implementing a wrapper on top of the old APIs and therefore old APIs
- are still valid. The new I/O APIs also recognize the @SQ header
- lines.
-
- * Generate the MD tag.
-
- * Generate "=" bases. However, the indel caller will not work when "="
- bases are present.
-
- * Enhanced support of color-read display (by Nils Homer).
-
- * Implemented the GNU building system. However, currently the building
- system does not generate libbam.a. We will improve this later. For
- the time being, `make -f Makefile.generic' is preferred.
-
- * Fixed a minor bug in pileup: the first read in a chromosome may be
- skipped.
-
- * Fixed bugs in bam_aux.c. These bugs do not affect other components as
- they were not used previously.
-
- * Output the 'SM' tag from maq2sam.
-
-(0.1.4: 21 May 2009, r297)
-
-
-
-Beta Release 0.1.3 (15 April, 2009)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes in SAMtools:
-
- * SAMtools is more consistent with the specification: a) '*' in the
- QUAL field is allowed; b) the field separator is TAB only and SPACE
- is treated as a character in a field; c) empty header is allowed.
-
- * Implemented GLFv3 support in pileup.
-
- * Fixed a severe bug in fixmate: strand information is wrongly
- overwritten.
-
- * Fixed a bug in alignment retrieval: alignments bridging n*16384bp are
- not correctly retrieved sometimes.
-
- * Fixed a bug in rmdup: segfault if unmapped reads are present.
-
- * Move indel_filter.pl to samtools.pl and improved the filtering by
- checking the actual number of alignments containing indels. The indel
- pileup line is also changed a little to make this filtration easier.
-
- * Fixed a minor bug in indexing: the bin number of an unmapped read is
- wrongly calculated.
-
- * Added `flagstat' command to show statistics on the FLAG field.
-
- * Improved indel caller by setting the maximum window size in local
- realignment.
-
-Changes in other utilities:
-
- * Fixed a bug in maq2sam: a tag name is obsolete.
-
- * Improvement to wgsim: a) added support for SOLiD read simulation; b)
- show the number of substitutions/indels/errors in read name; c)
- considerable code clean up.
-
- * Various converters: improved functionality in general.
-
- * Updated the example SAM due to the previous bug in fixmate.
-
-(0.1.3: 15 April 2009, r227)
-
-
-
-Beta Release 0.1.2 (28 January, 2008)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Notable changes in SAMtools:
-
- * Implemented a Bayesian indel caller. The new caller generate scores
- and genotype and is potentially more accurate than Maq's indel
- caller. The pileup format is also changed accordingly.
-
- * Implemented rmdup command: remove potential PCR duplicates. Note that
- this command ONLY works for FR orientation and requires ISIZE is
- correctly set.
-
- * Added fixmate command: fill in mate coordinates, ISIZE and mate
- related flags from a name-sorted alignment.
-
- * Fixed a bug in indexing: reads bridging 16x kbp were not retrieved.
-
- * Allow to select reads shown in the pileup output with a mask.
-
- * Generate GLFv2 from pileup.
-
- * Added two more flags for flagging PCR/optical duplicates and for QC
- failure.
-
- * Fixed a bug in sort command: name sorting for large alignment did not
- work.
-
- * Allow to completely disable RAZF (using Makefile.lite) as some people
- have problem to compile it.
-
- * Fixed a bug in import command when there are reads without
- coordinates.
-
- * Fixed a bug in tview: clipping broke the alignment viewer.
-
- * Fixed a compiling error when _NO_CURSES is applied.
-
- * Fixed a bug in merge command.
-
-Changes in other utilities:
-
- * Added wgsim, a paired-end reads simulator. Wgsim was adapted from
- maq's reads simulator. Colin Hercus further improved it to allow
- longer indels.
-
- * Added wgsim_eval.pl, a script that evaluates the accuracy of
- alignment on reads generated by wgsim.
-
- * Added soap2sam.pl, a SOAP2->SAM converter. This converter does not
- work properly when multiple hits are output.
-
- * Added bowtie2sam.pl, a Bowtie->SAM converter. Only the top hit will
- be retained when multiple hits are present.
-
- * Fixed a bug in export2sam.pl for QC reads.
-
- * Support RG tag at MAQ->SAM converter.
-
- * Added novo2sam.pl, a NovoAlign->SAM converter. Multiple hits and
- indel are not properly handled, though.
-
- * Added zoom2sam.pl, a ZOOM->SAM converter. It only works with the
- default Illumina output.
-
-(0.1.2: 28 January 2008; r116)
-
-
-
-Beta Release 0.1.1 (22 December, 2008)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The is the first public release of samtools. For more information,
-please check the manual page `samtools.1' and the samtools website
-http://samtools.sourceforge.net
diff --git a/samtools-0.1.19/bam.c b/samtools-0.1.19/bam.c
deleted file mode 100644
index b00d6a6..0000000
--- a/samtools-0.1.19/bam.c
+++ /dev/null
@@ -1,474 +0,0 @@
-#include <stdio.h>
-#include <ctype.h>
-#include <errno.h>
-#include <assert.h>
-#include "bam.h"
-#include "bam_endian.h"
-#include "kstring.h"
-#include "sam_header.h"
-
-int bam_is_be = 0, bam_verbose = 2, bam_no_B = 0;
-char *bam_flag2char_table = "pPuUrR12sfd\0\0\0\0\0";
-
-/**************************
- * CIGAR related routines *
- **************************/
-
-uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar)
-{
- int k, end = c->pos;
- for (k = 0; k < c->n_cigar; ++k) {
- int op = bam_cigar_op(cigar[k]);
- int len = bam_cigar_oplen(cigar[k]);
- if (op == BAM_CBACK) { // move backward
- int l, u, v;
- if (k == c->n_cigar - 1) break; // skip trailing 'B'
- for (l = k - 1, u = v = 0; l >= 0; --l) {
- int op1 = bam_cigar_op(cigar[l]);
- int len1 = bam_cigar_oplen(cigar[l]);
- if (bam_cigar_type(op1)&1) { // consume query
- if (u + len1 >= len) { // stop
- if (bam_cigar_type(op1)&2) v += len - u;
- break;
- } else u += len1;
- }
- if (bam_cigar_type(op1)&2) v += len1;
- }
- end = l < 0? c->pos : end - v;
- } else if (bam_cigar_type(op)&2) end += bam_cigar_oplen(cigar[k]);
- }
- return end;
-}
-
-int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar)
-{
- uint32_t k;
- int32_t l = 0;
- for (k = 0; k < c->n_cigar; ++k)
- if (bam_cigar_type(bam_cigar_op(cigar[k]))&1)
- l += bam_cigar_oplen(cigar[k]);
- return l;
-}
-
-/********************
- * BAM I/O routines *
- ********************/
-
-bam_header_t *bam_header_init()
-{
- bam_is_be = bam_is_big_endian();
- return (bam_header_t*)calloc(1, sizeof(bam_header_t));
-}
-
-void bam_header_destroy(bam_header_t *header)
-{
- int32_t i;
- extern void bam_destroy_header_hash(bam_header_t *header);
- if (header == 0) return;
- if (header->target_name) {
- for (i = 0; i < header->n_targets; ++i)
- free(header->target_name[i]);
- free(header->target_name);
- free(header->target_len);
- }
- free(header->text);
- if (header->dict) sam_header_free(header->dict);
- if (header->rg2lib) sam_tbl_destroy(header->rg2lib);
- bam_destroy_header_hash(header);
- free(header);
-}
-
-bam_header_t *bam_header_read(bamFile fp)
-{
- bam_header_t *header;
- char buf[4];
- int magic_len;
- int32_t i = 1, name_len;
- // check EOF
- i = bgzf_check_EOF(fp);
- if (i < 0) {
- // If the file is a pipe, checking the EOF marker will *always* fail
- // with ESPIPE. Suppress the error message in this case.
- if (errno != ESPIPE) perror("[bam_header_read] bgzf_check_EOF");
- }
- else if (i == 0) fprintf(stderr, "[bam_header_read] EOF marker is absent. The input is probably truncated.\n");
- // read "BAM1"
- magic_len = bam_read(fp, buf, 4);
- if (magic_len != 4 || strncmp(buf, "BAM\001", 4) != 0) {
- fprintf(stderr, "[bam_header_read] invalid BAM binary header (this is not a BAM file).\n");
- return 0;
- }
- header = bam_header_init();
- // read plain text and the number of reference sequences
- bam_read(fp, &header->l_text, 4);
- if (bam_is_be) bam_swap_endian_4p(&header->l_text);
- header->text = (char*)calloc(header->l_text + 1, 1);
- bam_read(fp, header->text, header->l_text);
- bam_read(fp, &header->n_targets, 4);
- if (bam_is_be) bam_swap_endian_4p(&header->n_targets);
- // read reference sequence names and lengths
- header->target_name = (char**)calloc(header->n_targets, sizeof(char*));
- header->target_len = (uint32_t*)calloc(header->n_targets, 4);
- for (i = 0; i != header->n_targets; ++i) {
- bam_read(fp, &name_len, 4);
- if (bam_is_be) bam_swap_endian_4p(&name_len);
- header->target_name[i] = (char*)calloc(name_len, 1);
- bam_read(fp, header->target_name[i], name_len);
- bam_read(fp, &header->target_len[i], 4);
- if (bam_is_be) bam_swap_endian_4p(&header->target_len[i]);
- }
- return header;
-}
-
-int bam_header_write(bamFile fp, const bam_header_t *header)
-{
- char buf[4];
- int32_t i, name_len, x;
- // write "BAM1"
- strncpy(buf, "BAM\001", 4);
- bam_write(fp, buf, 4);
- // write plain text and the number of reference sequences
- if (bam_is_be) {
- x = bam_swap_endian_4(header->l_text);
- bam_write(fp, &x, 4);
- if (header->l_text) bam_write(fp, header->text, header->l_text);
- x = bam_swap_endian_4(header->n_targets);
- bam_write(fp, &x, 4);
- } else {
- bam_write(fp, &header->l_text, 4);
- if (header->l_text) bam_write(fp, header->text, header->l_text);
- bam_write(fp, &header->n_targets, 4);
- }
- // write sequence names and lengths
- for (i = 0; i != header->n_targets; ++i) {
- char *p = header->target_name[i];
- name_len = strlen(p) + 1;
- if (bam_is_be) {
- x = bam_swap_endian_4(name_len);
- bam_write(fp, &x, 4);
- } else bam_write(fp, &name_len, 4);
- bam_write(fp, p, name_len);
- if (bam_is_be) {
- x = bam_swap_endian_4(header->target_len[i]);
- bam_write(fp, &x, 4);
- } else bam_write(fp, &header->target_len[i], 4);
- }
- bgzf_flush(fp);
- return 0;
-}
-
-static void swap_endian_data(const bam1_core_t *c, int data_len, uint8_t *data)
-{
- uint8_t *s;
- uint32_t i, *cigar = (uint32_t*)(data + c->l_qname);
- s = data + c->n_cigar*4 + c->l_qname + c->l_qseq + (c->l_qseq + 1)/2;
- for (i = 0; i < c->n_cigar; ++i) bam_swap_endian_4p(&cigar[i]);
- while (s < data + data_len) {
- uint8_t type;
- s += 2; // skip key
- type = toupper(*s); ++s; // skip type
- if (type == 'C' || type == 'A') ++s;
- else if (type == 'S') { bam_swap_endian_2p(s); s += 2; }
- else if (type == 'I' || type == 'F') { bam_swap_endian_4p(s); s += 4; }
- else if (type == 'D') { bam_swap_endian_8p(s); s += 8; }
- else if (type == 'Z' || type == 'H') { while (*s) ++s; ++s; }
- else if (type == 'B') {
- int32_t n, Bsize = bam_aux_type2size(*s);
- memcpy(&n, s + 1, 4);
- if (1 == Bsize) {
- } else if (2 == Bsize) {
- for (i = 0; i < n; i += 2)
- bam_swap_endian_2p(s + 5 + i);
- } else if (4 == Bsize) {
- for (i = 0; i < n; i += 4)
- bam_swap_endian_4p(s + 5 + i);
- }
- bam_swap_endian_4p(s+1);
- }
- }
-}
-
-int bam_read1(bamFile fp, bam1_t *b)
-{
- bam1_core_t *c = &b->core;
- int32_t block_len, ret, i;
- uint32_t x[8];
-
- assert(BAM_CORE_SIZE == 32);
- if ((ret = bam_read(fp, &block_len, 4)) != 4) {
- if (ret == 0) return -1; // normal end-of-file
- else return -2; // truncated
- }
- if (bam_read(fp, x, BAM_CORE_SIZE) != BAM_CORE_SIZE) return -3;
- if (bam_is_be) {
- bam_swap_endian_4p(&block_len);
- for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i);
- }
- c->tid = x[0]; c->pos = x[1];
- c->bin = x[2]>>16; c->qual = x[2]>>8&0xff; c->l_qname = x[2]&0xff;
- c->flag = x[3]>>16; c->n_cigar = x[3]&0xffff;
- c->l_qseq = x[4];
- c->mtid = x[5]; c->mpos = x[6]; c->isize = x[7];
- b->data_len = block_len - BAM_CORE_SIZE;
- if (b->m_data < b->data_len) {
- b->m_data = b->data_len;
- kroundup32(b->m_data);
- b->data = (uint8_t*)realloc(b->data, b->m_data);
- }
- if (bam_read(fp, b->data, b->data_len) != b->data_len) return -4;
- b->l_aux = b->data_len - c->n_cigar * 4 - c->l_qname - c->l_qseq - (c->l_qseq+1)/2;
- if (bam_is_be) swap_endian_data(c, b->data_len, b->data);
- if (bam_no_B) bam_remove_B(b);
- return 4 + block_len;
-}
-
-inline int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data)
-{
- uint32_t x[8], block_len = data_len + BAM_CORE_SIZE, y;
- int i;
- assert(BAM_CORE_SIZE == 32);
- x[0] = c->tid;
- x[1] = c->pos;
- x[2] = (uint32_t)c->bin<<16 | c->qual<<8 | c->l_qname;
- x[3] = (uint32_t)c->flag<<16 | c->n_cigar;
- x[4] = c->l_qseq;
- x[5] = c->mtid;
- x[6] = c->mpos;
- x[7] = c->isize;
- bgzf_flush_try(fp, 4 + block_len);
- if (bam_is_be) {
- for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i);
- y = block_len;
- bam_write(fp, bam_swap_endian_4p(&y), 4);
- swap_endian_data(c, data_len, data);
- } else bam_write(fp, &block_len, 4);
- bam_write(fp, x, BAM_CORE_SIZE);
- bam_write(fp, data, data_len);
- if (bam_is_be) swap_endian_data(c, data_len, data);
- return 4 + block_len;
-}
-
-int bam_write1(bamFile fp, const bam1_t *b)
-{
- return bam_write1_core(fp, &b->core, b->data_len, b->data);
-}
-
-char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of)
-{
- uint8_t *s = bam1_seq(b), *t = bam1_qual(b);
- int i;
- const bam1_core_t *c = &b->core;
- kstring_t str;
- str.l = str.m = 0; str.s = 0;
-
- kputsn(bam1_qname(b), c->l_qname-1, &str); kputc('\t', &str);
- if (of == BAM_OFDEC) { kputw(c->flag, &str); kputc('\t', &str); }
- else if (of == BAM_OFHEX) ksprintf(&str, "0x%x\t", c->flag);
- else { // BAM_OFSTR
- for (i = 0; i < 16; ++i)
- if ((c->flag & 1<<i) && bam_flag2char_table[i])
- kputc(bam_flag2char_table[i], &str);
- kputc('\t', &str);
- }
- if (c->tid < 0) kputsn("*\t", 2, &str);
- else {
- if (header) kputs(header->target_name[c->tid] , &str);
- else kputw(c->tid, &str);
- kputc('\t', &str);
- }
- kputw(c->pos + 1, &str); kputc('\t', &str); kputw(c->qual, &str); kputc('\t', &str);
- if (c->n_cigar == 0) kputc('*', &str);
- else {
- uint32_t *cigar = bam1_cigar(b);
- for (i = 0; i < c->n_cigar; ++i) {
- kputw(bam1_cigar(b)[i]>>BAM_CIGAR_SHIFT, &str);
- kputc(bam_cigar_opchr(cigar[i]), &str);
- }
- }
- kputc('\t', &str);
- if (c->mtid < 0) kputsn("*\t", 2, &str);
- else if (c->mtid == c->tid) kputsn("=\t", 2, &str);
- else {
- if (header) kputs(header->target_name[c->mtid], &str);
- else kputw(c->mtid, &str);
- kputc('\t', &str);
- }
- kputw(c->mpos + 1, &str); kputc('\t', &str); kputw(c->isize, &str); kputc('\t', &str);
- if (c->l_qseq) {
- for (i = 0; i < c->l_qseq; ++i) kputc(bam_nt16_rev_table[bam1_seqi(s, i)], &str);
- kputc('\t', &str);
- if (t[0] == 0xff) kputc('*', &str);
- else for (i = 0; i < c->l_qseq; ++i) kputc(t[i] + 33, &str);
- } else kputsn("*\t*", 3, &str);
- s = bam1_aux(b);
- while (s < b->data + b->data_len) {
- uint8_t type, key[2];
- key[0] = s[0]; key[1] = s[1];
- s += 2; type = *s; ++s;
- kputc('\t', &str); kputsn((char*)key, 2, &str); kputc(':', &str);
- if (type == 'A') { kputsn("A:", 2, &str); kputc(*s, &str); ++s; }
- else if (type == 'C') { kputsn("i:", 2, &str); kputw(*s, &str); ++s; }
- else if (type == 'c') { kputsn("i:", 2, &str); kputw(*(int8_t*)s, &str); ++s; }
- else if (type == 'S') { kputsn("i:", 2, &str); kputw(*(uint16_t*)s, &str); s += 2; }
- else if (type == 's') { kputsn("i:", 2, &str); kputw(*(int16_t*)s, &str); s += 2; }
- else if (type == 'I') { kputsn("i:", 2, &str); kputuw(*(uint32_t*)s, &str); s += 4; }
- else if (type == 'i') { kputsn("i:", 2, &str); kputw(*(int32_t*)s, &str); s += 4; }
- else if (type == 'f') { ksprintf(&str, "f:%g", *(float*)s); s += 4; }
- else if (type == 'd') { ksprintf(&str, "d:%lg", *(double*)s); s += 8; }
- else if (type == 'Z' || type == 'H') { kputc(type, &str); kputc(':', &str); while (*s) kputc(*s++, &str); ++s; }
- else if (type == 'B') {
- uint8_t sub_type = *(s++);
- int32_t n;
- memcpy(&n, s, 4);
- s += 4; // no point to the start of the array
- kputc(type, &str); kputc(':', &str); kputc(sub_type, &str); // write the typing
- for (i = 0; i < n; ++i) {
- kputc(',', &str);
- if ('c' == sub_type || 'c' == sub_type) { kputw(*(int8_t*)s, &str); ++s; }
- else if ('C' == sub_type) { kputw(*(uint8_t*)s, &str); ++s; }
- else if ('s' == sub_type) { kputw(*(int16_t*)s, &str); s += 2; }
- else if ('S' == sub_type) { kputw(*(uint16_t*)s, &str); s += 2; }
- else if ('i' == sub_type) { kputw(*(int32_t*)s, &str); s += 4; }
- else if ('I' == sub_type) { kputuw(*(uint32_t*)s, &str); s += 4; }
- else if ('f' == sub_type) { ksprintf(&str, "%g", *(float*)s); s += 4; }
- }
- }
- }
- return str.s;
-}
-
-char *bam_format1(const bam_header_t *header, const bam1_t *b)
-{
- return bam_format1_core(header, b, BAM_OFDEC);
-}
-
-void bam_view1(const bam_header_t *header, const bam1_t *b)
-{
- char *s = bam_format1(header, b);
- puts(s);
- free(s);
-}
-
-int bam_validate1(const bam_header_t *header, const bam1_t *b)
-{
- char *s;
-
- if (b->core.tid < -1 || b->core.mtid < -1) return 0;
- if (header && (b->core.tid >= header->n_targets || b->core.mtid >= header->n_targets)) return 0;
-
- if (b->data_len < b->core.l_qname) return 0;
- s = memchr(bam1_qname(b), '\0', b->core.l_qname);
- if (s != &bam1_qname(b)[b->core.l_qname-1]) return 0;
-
- // FIXME: Other fields could also be checked, especially the auxiliary data
-
- return 1;
-}
-
-// FIXME: we should also check the LB tag associated with each alignment
-const char *bam_get_library(bam_header_t *h, const bam1_t *b)
-{
- const uint8_t *rg;
- if (h->dict == 0) h->dict = sam_header_parse2(h->text);
- if (h->rg2lib == 0) h->rg2lib = sam_header2tbl(h->dict, "RG", "ID", "LB");
- rg = bam_aux_get(b, "RG");
- return (rg == 0)? 0 : sam_tbl_get(h->rg2lib, (const char*)(rg + 1));
-}
-
-/************
- * Remove B *
- ************/
-
-int bam_remove_B(bam1_t *b)
-{
- int i, j, end_j, k, l, no_qual;
- uint32_t *cigar, *new_cigar;
- uint8_t *seq, *qual, *p;
- // test if removal is necessary
- if (b->core.flag & BAM_FUNMAP) return 0; // unmapped; do nothing
- cigar = bam1_cigar(b);
- for (k = 0; k < b->core.n_cigar; ++k)
- if (bam_cigar_op(cigar[k]) == BAM_CBACK) break;
- if (k == b->core.n_cigar) return 0; // no 'B'
- if (bam_cigar_op(cigar[0]) == BAM_CBACK) goto rmB_err; // cannot be removed
- // allocate memory for the new CIGAR
- if (b->data_len + (b->core.n_cigar + 1) * 4 > b->m_data) { // not enough memory
- b->m_data = b->data_len + b->core.n_cigar * 4;
- kroundup32(b->m_data);
- b->data = (uint8_t*)realloc(b->data, b->m_data);
- cigar = bam1_cigar(b); // after realloc, cigar may be changed
- }
- new_cigar = (uint32_t*)(b->data + (b->m_data - b->core.n_cigar * 4)); // from the end of b->data
- // the core loop
- seq = bam1_seq(b); qual = bam1_qual(b);
- no_qual = (qual[0] == 0xff); // test whether base quality is available
- i = j = 0; end_j = -1;
- for (k = l = 0; k < b->core.n_cigar; ++k) {
- int op = bam_cigar_op(cigar[k]);
- int len = bam_cigar_oplen(cigar[k]);
- if (op == BAM_CBACK) { // the backward operation
- int t, u;
- if (k == b->core.n_cigar - 1) break; // ignore 'B' at the end of CIGAR
- if (len > j) goto rmB_err; // an excessively long backward
- for (t = l - 1, u = 0; t >= 0; --t) { // look back
- int op1 = bam_cigar_op(new_cigar[t]);
- int len1 = bam_cigar_oplen(new_cigar[t]);
- if (bam_cigar_type(op1)&1) { // consume the query
- if (u + len1 >= len) { // stop
- new_cigar[t] -= (len - u) << BAM_CIGAR_SHIFT;
- break;
- } else u += len1;
- }
- }
- if (bam_cigar_oplen(new_cigar[t]) == 0) --t; // squeeze out the zero-length operation
- l = t + 1;
- end_j = j; j -= len;
- } else { // other CIGAR operations
- new_cigar[l++] = cigar[k];
- if (bam_cigar_type(op)&1) { // consume the query
- if (i != j) { // no need to copy if i == j
- int u, c, c0;
- for (u = 0; u < len; ++u) { // construct the consensus
- c = bam1_seqi(seq, i+u);
- if (j + u < end_j) { // in an overlap
- c0 = bam1_seqi(seq, j+u);
- if (c != c0) { // a mismatch; choose the better base
- if (qual[j+u] < qual[i+u]) { // the base in the 2nd segment is better
- bam1_seq_seti(seq, j+u, c);
- qual[j+u] = qual[i+u] - qual[j+u];
- } else qual[j+u] -= qual[i+u]; // the 1st is better; reduce base quality
- } else qual[j+u] = qual[j+u] > qual[i+u]? qual[j+u] : qual[i+u];
- } else { // not in an overlap; copy over
- bam1_seq_seti(seq, j+u, c);
- qual[j+u] = qual[i+u];
- }
- }
- }
- i += len, j += len;
- }
- }
- }
- if (no_qual) qual[0] = 0xff; // in very rare cases, this may be modified
- // merge adjacent operations if possible
- for (k = 1; k < l; ++k)
- if (bam_cigar_op(new_cigar[k]) == bam_cigar_op(new_cigar[k-1]))
- new_cigar[k] += new_cigar[k-1] >> BAM_CIGAR_SHIFT << BAM_CIGAR_SHIFT, new_cigar[k-1] &= 0xf;
- // kill zero length operations
- for (k = i = 0; k < l; ++k)
- if (new_cigar[k] >> BAM_CIGAR_SHIFT)
- new_cigar[i++] = new_cigar[k];
- l = i;
- // update b
- memcpy(cigar, new_cigar, l * 4); // set CIGAR
- p = b->data + b->core.l_qname + l * 4;
- memmove(p, seq, (j+1)>>1); p += (j+1)>>1; // set SEQ
- memmove(p, qual, j); p += j; // set QUAL
- memmove(p, bam1_aux(b), b->l_aux); p += b->l_aux; // set optional fields
- b->core.n_cigar = l, b->core.l_qseq = j; // update CIGAR length and query length
- b->data_len = p - b->data; // update record length
- return 0;
-
-rmB_err:
- b->core.flag |= BAM_FUNMAP;
- return -1;
-}
diff --git a/samtools-0.1.19/bam.h b/samtools-0.1.19/bam.h
deleted file mode 100644
index 6add34b..0000000
--- a/samtools-0.1.19/bam.h
+++ /dev/null
@@ -1,793 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008-2010 Genome Research Ltd (GRL).
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Contact: Heng Li <lh3 at sanger.ac.uk> */
-
-#ifndef BAM_BAM_H
-#define BAM_BAM_H
-
-/*!
- @header
-
- BAM library provides I/O and various operations on manipulating files
- in the BAM (Binary Alignment/Mapping) or SAM (Sequence Alignment/Map)
- format. It now supports importing from or exporting to SAM, sorting,
- merging, generating pileup, and quickly retrieval of reads overlapped
- with a specified region.
-
- @copyright Genome Research Ltd.
- */
-
-#define BAM_VERSION "0.1.19-96b5f2294a"
-
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-#ifndef BAM_LITE
-#define BAM_VIRTUAL_OFFSET16
-#include "bgzf.h"
-/*! @abstract BAM file handler */
-typedef BGZF *bamFile;
-#define bam_open(fn, mode) bgzf_open(fn, mode)
-#define bam_dopen(fd, mode) bgzf_fdopen(fd, mode)
-#define bam_close(fp) bgzf_close(fp)
-#define bam_read(fp, buf, size) bgzf_read(fp, buf, size)
-#define bam_write(fp, buf, size) bgzf_write(fp, buf, size)
-#define bam_tell(fp) bgzf_tell(fp)
-#define bam_seek(fp, pos, dir) bgzf_seek(fp, pos, dir)
-#else
-#define BAM_TRUE_OFFSET
-#include <zlib.h>
-typedef gzFile bamFile;
-#define bam_open(fn, mode) gzopen(fn, mode)
-#define bam_dopen(fd, mode) gzdopen(fd, mode)
-#define bam_close(fp) gzclose(fp)
-#define bam_read(fp, buf, size) gzread(fp, buf, size)
-/* no bam_write/bam_tell/bam_seek() here */
-#endif
-
-/*! @typedef
- @abstract Structure for the alignment header.
- @field n_targets number of reference sequences
- @field target_name names of the reference sequences
- @field target_len lengths of the referene sequences
- @field dict header dictionary
- @field hash hash table for fast name lookup
- @field rg2lib hash table for @RG-ID -> LB lookup
- @field l_text length of the plain text in the header
- @field text plain text
-
- @discussion Field hash points to null by default. It is a private
- member.
- */
-typedef struct {
- int32_t n_targets;
- char **target_name;
- uint32_t *target_len;
- void *dict, *hash, *rg2lib;
- uint32_t l_text, n_text;
- char *text;
-} bam_header_t;
-
-/*! @abstract the read is paired in sequencing, no matter whether it is mapped in a pair */
-#define BAM_FPAIRED 1
-/*! @abstract the read is mapped in a proper pair */
-#define BAM_FPROPER_PAIR 2
-/*! @abstract the read itself is unmapped; conflictive with BAM_FPROPER_PAIR */
-#define BAM_FUNMAP 4
-/*! @abstract the mate is unmapped */
-#define BAM_FMUNMAP 8
-/*! @abstract the read is mapped to the reverse strand */
-#define BAM_FREVERSE 16
-/*! @abstract the mate is mapped to the reverse strand */
-#define BAM_FMREVERSE 32
-/*! @abstract this is read1 */
-#define BAM_FREAD1 64
-/*! @abstract this is read2 */
-#define BAM_FREAD2 128
-/*! @abstract not primary alignment */
-#define BAM_FSECONDARY 256
-/*! @abstract QC failure */
-#define BAM_FQCFAIL 512
-/*! @abstract optical or PCR duplicate */
-#define BAM_FDUP 1024
-
-#define BAM_OFDEC 0
-#define BAM_OFHEX 1
-#define BAM_OFSTR 2
-
-/*! @abstract defautl mask for pileup */
-#define BAM_DEF_MASK (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP)
-
-#define BAM_CORE_SIZE sizeof(bam1_core_t)
-
-/**
- * Describing how CIGAR operation/length is packed in a 32-bit integer.
- */
-#define BAM_CIGAR_SHIFT 4
-#define BAM_CIGAR_MASK ((1 << BAM_CIGAR_SHIFT) - 1)
-
-/*
- CIGAR operations.
- */
-/*! @abstract CIGAR: M = match or mismatch*/
-#define BAM_CMATCH 0
-/*! @abstract CIGAR: I = insertion to the reference */
-#define BAM_CINS 1
-/*! @abstract CIGAR: D = deletion from the reference */
-#define BAM_CDEL 2
-/*! @abstract CIGAR: N = skip on the reference (e.g. spliced alignment) */
-#define BAM_CREF_SKIP 3
-/*! @abstract CIGAR: S = clip on the read with clipped sequence
- present in qseq */
-#define BAM_CSOFT_CLIP 4
-/*! @abstract CIGAR: H = clip on the read with clipped sequence trimmed off */
-#define BAM_CHARD_CLIP 5
-/*! @abstract CIGAR: P = padding */
-#define BAM_CPAD 6
-/*! @abstract CIGAR: equals = match */
-#define BAM_CEQUAL 7
-/*! @abstract CIGAR: X = mismatch */
-#define BAM_CDIFF 8
-#define BAM_CBACK 9
-
-#define BAM_CIGAR_STR "MIDNSHP=XB"
-#define BAM_CIGAR_TYPE 0x3C1A7
-
-#define bam_cigar_op(c) ((c)&BAM_CIGAR_MASK)
-#define bam_cigar_oplen(c) ((c)>>BAM_CIGAR_SHIFT)
-#define bam_cigar_opchr(c) (BAM_CIGAR_STR[bam_cigar_op(c)])
-#define bam_cigar_gen(l, o) ((l)<<BAM_CIGAR_SHIFT|(o))
-#define bam_cigar_type(o) (BAM_CIGAR_TYPE>>((o)<<1)&3) // bit 1: consume query; bit 2: consume reference
-
-/*! @typedef
- @abstract Structure for core alignment information.
- @field tid chromosome ID, defined by bam_header_t
- @field pos 0-based leftmost coordinate
- @field bin bin calculated by bam_reg2bin()
- @field qual mapping quality
- @field l_qname length of the query name
- @field flag bitwise flag
- @field n_cigar number of CIGAR operations
- @field l_qseq length of the query sequence (read)
- */
-typedef struct {
- int32_t tid;
- int32_t pos;
- uint32_t bin:16, qual:8, l_qname:8;
- uint32_t flag:16, n_cigar:16;
- int32_t l_qseq;
- int32_t mtid;
- int32_t mpos;
- int32_t isize;
-} bam1_core_t;
-
-/*! @typedef
- @abstract Structure for one alignment.
- @field core core information about the alignment
- @field l_aux length of auxiliary data
- @field data_len current length of bam1_t::data
- @field m_data maximum length of bam1_t::data
- @field data all variable-length data, concatenated; structure: qname-cigar-seq-qual-aux
-
- @discussion Notes:
-
- 1. qname is zero tailing and core.l_qname includes the tailing '\0'.
- 2. l_qseq is calculated from the total length of an alignment block
- on reading or from CIGAR.
- 3. cigar data is encoded 4 bytes per CIGAR operation.
- 4. seq is nybble-encoded according to bam_nt16_table.
- */
-typedef struct {
- bam1_core_t core;
- int l_aux, data_len, m_data;
- uint8_t *data;
-} bam1_t;
-
-typedef struct __bam_iter_t *bam_iter_t;
-
-#define bam1_strand(b) (((b)->core.flag&BAM_FREVERSE) != 0)
-#define bam1_mstrand(b) (((b)->core.flag&BAM_FMREVERSE) != 0)
-
-/*! @function
- @abstract Get the CIGAR array
- @param b pointer to an alignment
- @return pointer to the CIGAR array
-
- @discussion In the CIGAR array, each element is a 32-bit integer. The
- lower 4 bits gives a CIGAR operation and the higher 28 bits keep the
- length of a CIGAR.
- */
-#define bam1_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname))
-
-/*! @function
- @abstract Get the name of the query
- @param b pointer to an alignment
- @return pointer to the name string, null terminated
- */
-#define bam1_qname(b) ((char*)((b)->data))
-
-/*! @function
- @abstract Get query sequence
- @param b pointer to an alignment
- @return pointer to sequence
-
- @discussion Each base is encoded in 4 bits: 1 for A, 2 for C, 4 for G,
- 8 for T and 15 for N. Two bases are packed in one byte with the base
- at the higher 4 bits having smaller coordinate on the read. It is
- recommended to use bam1_seqi() macro to get the base.
- */
-#define bam1_seq(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname)
-
-/*! @function
- @abstract Get query quality
- @param b pointer to an alignment
- @return pointer to quality string
- */
-#define bam1_qual(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1))
-
-/*! @function
- @abstract Get a base on read
- @param s Query sequence returned by bam1_seq()
- @param i The i-th position, 0-based
- @return 4-bit integer representing the base.
- */
-//#define bam1_seqi(s, i) ((s)[(i)/2] >> 4*(1-(i)%2) & 0xf)
-#define bam1_seqi(s, i) ((s)[(i)>>1] >> ((~(i)&1)<<2) & 0xf)
-
-#define bam1_seq_seti(s, i, c) ( (s)[(i)>>1] = ((s)[(i)>>1] & 0xf<<(((i)&1)<<2)) | (c)<<((~(i)&1)<<2) )
-
-/*! @function
- @abstract Get query sequence and quality
- @param b pointer to an alignment
- @return pointer to the concatenated auxiliary data
- */
-#define bam1_aux(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (b)->core.l_qseq + ((b)->core.l_qseq + 1)/2)
-
-#ifndef kroundup32
-/*! @function
- @abstract Round an integer to the next closest power-2 integer.
- @param x integer to be rounded (in place)
- @discussion x will be modified.
- */
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
-/*!
- @abstract Whether the machine is big-endian; modified only in
- bam_header_init().
- */
-extern int bam_is_be;
-
-/*!
- @abstract Verbose level between 0 and 3; 0 is supposed to disable all
- debugging information, though this may not have been implemented.
- */
-extern int bam_verbose;
-
-extern int bam_no_B;
-
-/*! @abstract Table for converting a nucleotide character to the 4-bit encoding. */
-extern unsigned char bam_nt16_table[256];
-
-/*! @abstract Table for converting a 4-bit encoded nucleotide to a letter. */
-extern char *bam_nt16_rev_table;
-
-extern char bam_nt16_nt4_table[];
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- /*********************
- * Low-level SAM I/O *
- *********************/
-
- /*! @abstract TAM file handler */
- typedef struct __tamFile_t *tamFile;
-
- /*!
- @abstract Open a SAM file for reading, either uncompressed or compressed by gzip/zlib.
- @param fn SAM file name
- @return SAM file handler
- */
- tamFile sam_open(const char *fn);
-
- /*!
- @abstract Close a SAM file handler
- @param fp SAM file handler
- */
- void sam_close(tamFile fp);
-
- /*!
- @abstract Read one alignment from a SAM file handler
- @param fp SAM file handler
- @param header header information (ordered names of chromosomes)
- @param b read alignment; all members in b will be updated
- @return 0 if successful; otherwise negative
- */
- int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b);
-
- /*!
- @abstract Read header information from a TAB-delimited list file.
- @param fn_list file name for the list
- @return a pointer to the header structure
-
- @discussion Each line in this file consists of chromosome name and
- the length of chromosome.
- */
- bam_header_t *sam_header_read2(const char *fn_list);
-
- /*!
- @abstract Read header from a SAM file (if present)
- @param fp SAM file handler
- @return pointer to header struct; 0 if no @SQ lines available
- */
- bam_header_t *sam_header_read(tamFile fp);
-
- /*!
- @abstract Parse @SQ lines a update a header struct
- @param h pointer to the header struct to be updated
- @return number of target sequences
-
- @discussion bam_header_t::{n_targets,target_len,target_name} will
- be destroyed in the first place.
- */
- int sam_header_parse(bam_header_t *h);
- int32_t bam_get_tid(const bam_header_t *header, const char *seq_name);
-
- /*!
- @abstract Parse @RG lines a update a header struct
- @param h pointer to the header struct to be updated
- @return number of @RG lines
-
- @discussion bam_header_t::rg2lib will be destroyed in the first
- place.
- */
- int sam_header_parse_rg(bam_header_t *h);
-
-#define sam_write1(header, b) bam_view1(header, b)
-
-
- /********************************
- * APIs for string dictionaries *
- ********************************/
-
- int bam_strmap_put(void *strmap, const char *rg, const char *lib);
- const char *bam_strmap_get(const void *strmap, const char *rg);
- void *bam_strmap_dup(const void*);
- void *bam_strmap_init();
- void bam_strmap_destroy(void *strmap);
-
-
- /*********************
- * Low-level BAM I/O *
- *********************/
-
- /*!
- @abstract Initialize a header structure.
- @return the pointer to the header structure
-
- @discussion This function also modifies the global variable
- bam_is_be.
- */
- bam_header_t *bam_header_init();
-
- /*!
- @abstract Destroy a header structure.
- @param header pointer to the header
- */
- void bam_header_destroy(bam_header_t *header);
-
- /*!
- @abstract Read a header structure from BAM.
- @param fp BAM file handler, opened by bam_open()
- @return pointer to the header structure
-
- @discussion The file position indicator must be placed at the
- beginning of the file. Upon success, the position indicator will
- be set at the start of the first alignment.
- */
- bam_header_t *bam_header_read(bamFile fp);
-
- /*!
- @abstract Write a header structure to BAM.
- @param fp BAM file handler
- @param header pointer to the header structure
- @return always 0 currently
- */
- int bam_header_write(bamFile fp, const bam_header_t *header);
-
- /*!
- @abstract Read an alignment from BAM.
- @param fp BAM file handler
- @param b read alignment; all members are updated.
- @return number of bytes read from the file
-
- @discussion The file position indicator must be
- placed right before an alignment. Upon success, this function
- will set the position indicator to the start of the next
- alignment. This function is not affected by the machine
- endianness.
- */
- int bam_read1(bamFile fp, bam1_t *b);
-
- int bam_remove_B(bam1_t *b);
-
- /*!
- @abstract Write an alignment to BAM.
- @param fp BAM file handler
- @param c pointer to the bam1_core_t structure
- @param data_len total length of variable size data related to
- the alignment
- @param data pointer to the concatenated data
- @return number of bytes written to the file
-
- @discussion This function is not affected by the machine
- endianness.
- */
- int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data);
-
- /*!
- @abstract Write an alignment to BAM.
- @param fp BAM file handler
- @param b alignment to write
- @return number of bytes written to the file
-
- @abstract It is equivalent to:
- bam_write1_core(fp, &b->core, b->data_len, b->data)
- */
- int bam_write1(bamFile fp, const bam1_t *b);
-
- /*! @function
- @abstract Initiate a pointer to bam1_t struct
- */
-#define bam_init1() ((bam1_t*)calloc(1, sizeof(bam1_t)))
-
- /*! @function
- @abstract Free the memory allocated for an alignment.
- @param b pointer to an alignment
- */
-#define bam_destroy1(b) do { \
- if (b) { free((b)->data); free(b); } \
- } while (0)
-
- /*!
- @abstract Format a BAM record in the SAM format
- @param header pointer to the header structure
- @param b alignment to print
- @return a pointer to the SAM string
- */
- char *bam_format1(const bam_header_t *header, const bam1_t *b);
-
- char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of);
-
- /*!
- @abstract Check whether a BAM record is plausibly valid
- @param header associated header structure, or NULL if unavailable
- @param b alignment to validate
- @return 0 if the alignment is invalid; non-zero otherwise
-
- @discussion Simple consistency check of some of the fields of the
- alignment record. If the header is provided, several additional checks
- are made. Not all fields are checked, so a non-zero result is not a
- guarantee that the record is valid. However it is usually good enough
- to detect when bam_seek() has been called with a virtual file offset
- that is not the offset of an alignment record.
- */
- int bam_validate1(const bam_header_t *header, const bam1_t *b);
-
- const char *bam_get_library(bam_header_t *header, const bam1_t *b);
-
-
- /***************
- * pileup APIs *
- ***************/
-
- /*! @typedef
- @abstract Structure for one alignment covering the pileup position.
- @field b pointer to the alignment
- @field qpos position of the read base at the pileup site, 0-based
- @field indel indel length; 0 for no indel, positive for ins and negative for del
- @field is_del 1 iff the base on the padded read is a deletion
- @field level the level of the read in the "viewer" mode
-
- @discussion See also bam_plbuf_push() and bam_lplbuf_push(). The
- difference between the two functions is that the former does not
- set bam_pileup1_t::level, while the later does. Level helps the
- implementation of alignment viewers, but calculating this has some
- overhead.
- */
- typedef struct {
- bam1_t *b;
- int32_t qpos;
- int indel, level;
- uint32_t is_del:1, is_head:1, is_tail:1, is_refskip:1, aux:28;
- } bam_pileup1_t;
-
- typedef int (*bam_plp_auto_f)(void *data, bam1_t *b);
-
- struct __bam_plp_t;
- typedef struct __bam_plp_t *bam_plp_t;
-
- bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data);
- int bam_plp_push(bam_plp_t iter, const bam1_t *b);
- const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);
- const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);
- void bam_plp_set_mask(bam_plp_t iter, int mask);
- void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt);
- void bam_plp_reset(bam_plp_t iter);
- void bam_plp_destroy(bam_plp_t iter);
-
- struct __bam_mplp_t;
- typedef struct __bam_mplp_t *bam_mplp_t;
-
- bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data);
- void bam_mplp_destroy(bam_mplp_t iter);
- void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt);
- int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp);
-
- /*! @typedef
- @abstract Type of function to be called by bam_plbuf_push().
- @param tid chromosome ID as is defined in the header
- @param pos start coordinate of the alignment, 0-based
- @param n number of elements in pl array
- @param pl array of alignments
- @param data user provided data
- @discussion See also bam_plbuf_push(), bam_plbuf_init() and bam_pileup1_t.
- */
- typedef int (*bam_pileup_f)(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data);
-
- typedef struct {
- bam_plp_t iter;
- bam_pileup_f func;
- void *data;
- } bam_plbuf_t;
-
- void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask);
- void bam_plbuf_reset(bam_plbuf_t *buf);
- bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data);
- void bam_plbuf_destroy(bam_plbuf_t *buf);
- int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf);
-
- int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data);
-
- struct __bam_lplbuf_t;
- typedef struct __bam_lplbuf_t bam_lplbuf_t;
-
- void bam_lplbuf_reset(bam_lplbuf_t *buf);
-
- /*! @abstract bam_plbuf_init() equivalent with level calculated. */
- bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data);
-
- /*! @abstract bam_plbuf_destroy() equivalent with level calculated. */
- void bam_lplbuf_destroy(bam_lplbuf_t *tv);
-
- /*! @abstract bam_plbuf_push() equivalent with level calculated. */
- int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *buf);
-
-
- /*********************
- * BAM indexing APIs *
- *********************/
-
- struct __bam_index_t;
- typedef struct __bam_index_t bam_index_t;
-
- /*!
- @abstract Build index for a BAM file.
- @discussion Index file "fn.bai" will be created.
- @param fn name of the BAM file
- @return always 0 currently
- */
- int bam_index_build(const char *fn);
-
- /*!
- @abstract Load index from file "fn.bai".
- @param fn name of the BAM file (NOT the index file)
- @return pointer to the index structure
- */
- bam_index_t *bam_index_load(const char *fn);
-
- /*!
- @abstract Destroy an index structure.
- @param idx pointer to the index structure
- */
- void bam_index_destroy(bam_index_t *idx);
-
- /*! @typedef
- @abstract Type of function to be called by bam_fetch().
- @param b the alignment
- @param data user provided data
- */
- typedef int (*bam_fetch_f)(const bam1_t *b, void *data);
-
- /*!
- @abstract Retrieve the alignments that are overlapped with the
- specified region.
-
- @discussion A user defined function will be called for each
- retrieved alignment ordered by its start position.
-
- @param fp BAM file handler
- @param idx pointer to the alignment index
- @param tid chromosome ID as is defined in the header
- @param beg start coordinate, 0-based
- @param end end coordinate, 0-based
- @param data user provided data (will be transferred to func)
- @param func user defined function
- */
- int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func);
-
- bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end);
- int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b);
- void bam_iter_destroy(bam_iter_t iter);
-
- /*!
- @abstract Parse a region in the format: "chr2:100,000-200,000".
- @discussion bam_header_t::hash will be initialized if empty.
- @param header pointer to the header structure
- @param str string to be parsed
- @param ref_id the returned chromosome ID
- @param begin the returned start coordinate
- @param end the returned end coordinate
- @return 0 on success; -1 on failure
- */
- int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *begin, int *end);
-
-
- /**************************
- * APIs for optional tags *
- **************************/
-
- /*!
- @abstract Retrieve data of a tag
- @param b pointer to an alignment struct
- @param tag two-character tag to be retrieved
-
- @return pointer to the type and data. The first character is the
- type that can be 'iIsScCdfAZH'.
-
- @discussion Use bam_aux2?() series to convert the returned data to
- the corresponding type.
- */
- uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]);
-
- int32_t bam_aux2i(const uint8_t *s);
- float bam_aux2f(const uint8_t *s);
- double bam_aux2d(const uint8_t *s);
- char bam_aux2A(const uint8_t *s);
- char *bam_aux2Z(const uint8_t *s);
-
- int bam_aux_del(bam1_t *b, uint8_t *s);
- void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data);
- uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2]); // an alias of bam_aux_get()
-
-
- /*****************
- * Miscellaneous *
- *****************/
-
- /*!
- @abstract Calculate the rightmost coordinate of an alignment on the
- reference genome.
-
- @param c pointer to the bam1_core_t structure
- @param cigar the corresponding CIGAR array (from bam1_t::cigar)
- @return the rightmost coordinate, 0-based
- */
- uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar);
-
- /*!
- @abstract Calculate the length of the query sequence from CIGAR.
- @param c pointer to the bam1_core_t structure
- @param cigar the corresponding CIGAR array (from bam1_t::cigar)
- @return length of the query sequence
- */
- int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar);
-
-#ifdef __cplusplus
-}
-#endif
-
-/*!
- @abstract Calculate the minimum bin that contains a region [beg,end).
- @param beg start of the region, 0-based
- @param end end of the region, 0-based
- @return bin
- */
-static inline int bam_reg2bin(uint32_t beg, uint32_t end)
-{
- --end;
- if (beg>>14 == end>>14) return 4681 + (beg>>14);
- if (beg>>17 == end>>17) return 585 + (beg>>17);
- if (beg>>20 == end>>20) return 73 + (beg>>20);
- if (beg>>23 == end>>23) return 9 + (beg>>23);
- if (beg>>26 == end>>26) return 1 + (beg>>26);
- return 0;
-}
-
-/*!
- @abstract Copy an alignment
- @param bdst destination alignment struct
- @param bsrc source alignment struct
- @return pointer to the destination alignment struct
- */
-static inline bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc)
-{
- uint8_t *data = bdst->data;
- int m_data = bdst->m_data; // backup data and m_data
- if (m_data < bsrc->data_len) { // double the capacity
- m_data = bsrc->data_len; kroundup32(m_data);
- data = (uint8_t*)realloc(data, m_data);
- }
- memcpy(data, bsrc->data, bsrc->data_len); // copy var-len data
- *bdst = *bsrc; // copy the rest
- // restore the backup
- bdst->m_data = m_data;
- bdst->data = data;
- return bdst;
-}
-
-/*!
- @abstract Duplicate an alignment
- @param src source alignment struct
- @return pointer to the destination alignment struct
- */
-static inline bam1_t *bam_dup1(const bam1_t *src)
-{
- bam1_t *b;
- b = bam_init1();
- *b = *src;
- b->m_data = b->data_len;
- b->data = (uint8_t*)calloc(b->data_len, 1);
- memcpy(b->data, src->data, b->data_len);
- return b;
-}
-
-static inline int bam_aux_type2size(int x)
-{
- if (x == 'C' || x == 'c' || x == 'A') return 1;
- else if (x == 'S' || x == 's') return 2;
- else if (x == 'I' || x == 'i' || x == 'f' || x == 'F') return 4;
- else return 0;
-}
-
-/*********************************
- *** Compatibility with htslib ***
- *********************************/
-
-typedef bam_header_t bam_hdr_t;
-
-#define bam_get_qname(b) bam1_qname(b)
-#define bam_get_cigar(b) bam1_cigar(b)
-
-#define bam_hdr_read(fp) bam_header_read(fp)
-#define bam_hdr_write(fp, h) bam_header_write(fp, h)
-#define bam_hdr_destroy(fp) bam_header_destroy(fp)
-
-#endif
diff --git a/samtools-0.1.19/bam2bcf.c b/samtools-0.1.19/bam2bcf.c
deleted file mode 100644
index 340b10b..0000000
--- a/samtools-0.1.19/bam2bcf.c
+++ /dev/null
@@ -1,467 +0,0 @@
-#include <math.h>
-#include <stdint.h>
-#include <assert.h>
-#include "bam.h"
-#include "kstring.h"
-#include "bam2bcf.h"
-#include "errmod.h"
-#include "bcftools/bcf.h"
-
-extern void ks_introsort_uint32_t(size_t n, uint32_t a[]);
-
-#define CALL_ETA 0.03f
-#define CALL_MAX 256
-#define CALL_DEFTHETA 0.83f
-#define DEF_MAPQ 20
-
-#define CAP_DIST 25
-
-bcf_callaux_t *bcf_call_init(double theta, int min_baseQ)
-{
- bcf_callaux_t *bca;
- if (theta <= 0.) theta = CALL_DEFTHETA;
- bca = calloc(1, sizeof(bcf_callaux_t));
- bca->capQ = 60;
- bca->openQ = 40; bca->extQ = 20; bca->tandemQ = 100;
- bca->min_baseQ = min_baseQ;
- bca->e = errmod_init(1. - theta);
- bca->min_frac = 0.002;
- bca->min_support = 1;
- bca->per_sample_flt = 0;
- bca->npos = 100;
- bca->ref_pos = calloc(bca->npos, sizeof(int));
- bca->alt_pos = calloc(bca->npos, sizeof(int));
- return bca;
-}
-
-
-static int get_position(const bam_pileup1_t *p, int *len)
-{
- int icig, n_tot_bases = 0, iread = 0, edist = p->qpos + 1;
- for (icig=0; icig<p->b->core.n_cigar; icig++)
- {
- // Conversion from uint32_t to MIDNSHP
- // 0123456
- // MIDNSHP
- int cig = bam1_cigar(p->b)[icig] & BAM_CIGAR_MASK;
- int ncig = bam1_cigar(p->b)[icig] >> BAM_CIGAR_SHIFT;
- if ( cig==0 )
- {
- n_tot_bases += ncig;
- iread += ncig;
- }
- else if ( cig==1 )
- {
- n_tot_bases += ncig;
- iread += ncig;
- }
- else if ( cig==4 )
- {
- iread += ncig;
- if ( iread<=p->qpos ) edist -= ncig;
- }
- }
- *len = n_tot_bases;
- return edist;
-}
-
-void bcf_call_destroy(bcf_callaux_t *bca)
-{
- if (bca == 0) return;
- errmod_destroy(bca->e);
- if (bca->npos) { free(bca->ref_pos); free(bca->alt_pos); bca->npos = 0; }
- free(bca->bases); free(bca->inscns); free(bca);
-}
-/* ref_base is the 4-bit representation of the reference base. It is
- * negative if we are looking at an indel. */
-int bcf_call_glfgen(int _n, const bam_pileup1_t *pl, int ref_base, bcf_callaux_t *bca, bcf_callret1_t *r)
-{
- int i, n, ref4, is_indel, ori_depth = 0;
- memset(r, 0, sizeof(bcf_callret1_t));
- if (ref_base >= 0) {
- ref4 = bam_nt16_nt4_table[ref_base];
- is_indel = 0;
- } else ref4 = 4, is_indel = 1;
- if (_n == 0) return -1;
- // enlarge the bases array if necessary
- if (bca->max_bases < _n) {
- bca->max_bases = _n;
- kroundup32(bca->max_bases);
- bca->bases = (uint16_t*)realloc(bca->bases, 2 * bca->max_bases);
- }
- // fill the bases array
- for (i = n = r->n_supp = 0; i < _n; ++i) {
- const bam_pileup1_t *p = pl + i;
- int q, b, mapQ, baseQ, is_diff, min_dist, seqQ;
- // set base
- if (p->is_del || p->is_refskip || (p->b->core.flag&BAM_FUNMAP)) continue;
- ++ori_depth;
- baseQ = q = is_indel? p->aux&0xff : (int)bam1_qual(p->b)[p->qpos]; // base/indel quality
- seqQ = is_indel? (p->aux>>8&0xff) : 99;
- if (q < bca->min_baseQ) continue;
- if (q > seqQ) q = seqQ;
- mapQ = p->b->core.qual < 255? p->b->core.qual : DEF_MAPQ; // special case for mapQ==255
- mapQ = mapQ < bca->capQ? mapQ : bca->capQ;
- if (q > mapQ) q = mapQ;
- if (q > 63) q = 63;
- if (q < 4) q = 4;
- if (!is_indel) {
- b = bam1_seqi(bam1_seq(p->b), p->qpos); // base
- b = bam_nt16_nt4_table[b? b : ref_base]; // b is the 2-bit base
- is_diff = (ref4 < 4 && b == ref4)? 0 : 1;
- } else {
- b = p->aux>>16&0x3f;
- is_diff = (b != 0);
- }
- if (is_diff) ++r->n_supp;
- bca->bases[n++] = q<<5 | (int)bam1_strand(p->b)<<4 | b;
- // collect annotations
- if (b < 4) r->qsum[b] += q;
- ++r->anno[0<<2|is_diff<<1|bam1_strand(p->b)];
- min_dist = p->b->core.l_qseq - 1 - p->qpos;
- if (min_dist > p->qpos) min_dist = p->qpos;
- if (min_dist > CAP_DIST) min_dist = CAP_DIST;
- r->anno[1<<2|is_diff<<1|0] += baseQ;
- r->anno[1<<2|is_diff<<1|1] += baseQ * baseQ;
- r->anno[2<<2|is_diff<<1|0] += mapQ;
- r->anno[2<<2|is_diff<<1|1] += mapQ * mapQ;
- r->anno[3<<2|is_diff<<1|0] += min_dist;
- r->anno[3<<2|is_diff<<1|1] += min_dist * min_dist;
-
- // collect read positions for ReadPosBias
- int len, pos = get_position(p, &len);
- int epos = (double)pos/(len+1) * bca->npos;
- if ( bam1_seqi(bam1_seq(p->b),p->qpos) == ref_base )
- bca->ref_pos[epos]++;
- else
- bca->alt_pos[epos]++;
- }
- r->depth = n; r->ori_depth = ori_depth;
- // glfgen
- errmod_cal(bca->e, n, 5, bca->bases, r->p);
- return r->depth;
-}
-
-double mann_whitney_1947(int n, int m, int U)
-{
- if (U<0) return 0;
- if (n==0||m==0) return U==0 ? 1 : 0;
- return (double)n/(n+m)*mann_whitney_1947(n-1,m,U-m) + (double)m/(n+m)*mann_whitney_1947(n,m-1,U);
-}
-
-void calc_ReadPosBias(bcf_callaux_t *bca, bcf_call_t *call)
-{
- int i, nref = 0, nalt = 0;
- unsigned long int U = 0;
- for (i=0; i<bca->npos; i++)
- {
- nref += bca->ref_pos[i];
- nalt += bca->alt_pos[i];
- U += nref*bca->alt_pos[i];
- bca->ref_pos[i] = 0;
- bca->alt_pos[i] = 0;
- }
-#if 0
-//todo
- double var = 0, avg = (double)(nref+nalt)/bca->npos;
- for (i=0; i<bca->npos; i++)
- {
- double ediff = bca->ref_pos[i] + bca->alt_pos[i] - avg;
- var += ediff*ediff;
- bca->ref_pos[i] = 0;
- bca->alt_pos[i] = 0;
- }
- call->read_pos.avg = avg;
- call->read_pos.var = sqrt(var/bca->npos);
- call->read_pos.dp = nref+nalt;
-#endif
- if ( !nref || !nalt )
- {
- call->read_pos_bias = -1;
- return;
- }
-
- if ( nref>=8 || nalt>=8 )
- {
- // normal approximation
- double mean = ((double)nref*nalt+1.0)/2.0;
- double var2 = (double)nref*nalt*(nref+nalt+1.0)/12.0;
- double z = (U-mean)/sqrt(var2);
- call->read_pos_bias = z;
- //fprintf(stderr,"nref=%d nalt=%d U=%ld mean=%e var=%e zval=%e\n", nref,nalt,U,mean,sqrt(var2),call->read_pos_bias);
- }
- else
- {
- double p = mann_whitney_1947(nalt,nref,U);
- // biased form claimed by GATK to behave better empirically
- // double var2 = (1.0+1.0/(nref+nalt+1.0))*(double)nref*nalt*(nref+nalt+1.0)/12.0;
- double var2 = (double)nref*nalt*(nref+nalt+1.0)/12.0;
- double z;
- if ( p >= 1./sqrt(var2*2*M_PI) ) z = 0; // equal to mean
- else
- {
- if ( U >= nref*nalt/2. ) z = sqrt(-2*log(sqrt(var2*2*M_PI)*p));
- else z = -sqrt(-2*log(sqrt(var2*2*M_PI)*p));
- }
- call->read_pos_bias = z;
- //fprintf(stderr,"nref=%d nalt=%d U=%ld p=%e var2=%e zval=%e\n", nref,nalt,U, p,var2,call->read_pos_bias);
- }
-}
-
-float mean_diff_to_prob(float mdiff, int dp, int readlen)
-{
- if ( dp==2 )
- {
- if ( mdiff==0 )
- return (2.0*readlen + 4.0*(readlen-1.0))/((float)readlen*readlen);
- else
- return 8.0*(readlen - 4.0*mdiff)/((float)readlen*readlen);
- }
-
- // This is crude empirical approximation and is not very accurate for
- // shorter read lengths (<100bp). There certainly is a room for
- // improvement.
- const float mv[24][2] = { {0,0}, {0,0}, {0,0},
- { 9.108, 4.934}, { 9.999, 3.991}, {10.273, 3.485}, {10.579, 3.160},
- {10.828, 2.889}, {11.014, 2.703}, {11.028, 2.546}, {11.244, 2.391},
- {11.231, 2.320}, {11.323, 2.138}, {11.403, 2.123}, {11.394, 1.994},
- {11.451, 1.928}, {11.445, 1.862}, {11.516, 1.815}, {11.560, 1.761},
- {11.544, 1.728}, {11.605, 1.674}, {11.592, 1.652}, {11.674, 1.613},
- {11.641, 1.570} };
-
- float m, v;
- if ( dp>=24 )
- {
- m = readlen/8.;
- if (dp>100) dp = 100;
- v = 1.476/(0.182*pow(dp,0.514));
- v = v*(readlen/100.);
- }
- else
- {
- m = mv[dp][0];
- v = mv[dp][1];
- m = m*readlen/100.;
- v = v*readlen/100.;
- v *= 1.2; // allow more variability
- }
- return 1.0/(v*sqrt(2*M_PI)) * exp(-0.5*((mdiff-m)/v)*((mdiff-m)/v));
-}
-
-void calc_vdb(bcf_callaux_t *bca, bcf_call_t *call)
-{
- int i, dp = 0;
- float mean_pos = 0, mean_diff = 0;
- for (i=0; i<bca->npos; i++)
- {
- if ( !bca->alt_pos[i] ) continue;
- dp += bca->alt_pos[i];
- int j = i<bca->npos/2 ? i : bca->npos - i;
- mean_pos += bca->alt_pos[i]*j;
- }
- if ( dp<2 )
- {
- call->vdb = -1;
- return;
- }
- mean_pos /= dp;
- for (i=0; i<bca->npos; i++)
- {
- if ( !bca->alt_pos[i] ) continue;
- int j = i<bca->npos/2 ? i : bca->npos - i;
- mean_diff += bca->alt_pos[i] * fabs(j - mean_pos);
- }
- mean_diff /= dp;
- call->vdb = mean_diff_to_prob(mean_diff, dp, bca->npos);
-}
-
-/**
- * bcf_call_combine() - sets the PL array and VDB, RPB annotations, finds the top two alleles
- * @n: number of samples
- * @calls: each sample's calls
- * @bca: auxiliary data structure for holding temporary values
- * @ref_base: the reference base
- * @call: filled with the annotations
- */
-int bcf_call_combine(int n, const bcf_callret1_t *calls, bcf_callaux_t *bca, int ref_base /*4-bit*/, bcf_call_t *call)
-{
- int ref4, i, j, qsum[4];
- int64_t tmp;
- if (ref_base >= 0) {
- call->ori_ref = ref4 = bam_nt16_nt4_table[ref_base];
- if (ref4 > 4) ref4 = 4;
- } else call->ori_ref = -1, ref4 = 0;
- // calculate qsum
- memset(qsum, 0, 4 * sizeof(int));
- for (i = 0; i < n; ++i)
- for (j = 0; j < 4; ++j)
- qsum[j] += calls[i].qsum[j];
- int qsum_tot=0;
- for (j=0; j<4; j++) { qsum_tot += qsum[j]; call->qsum[j] = 0; }
- for (j = 0; j < 4; ++j) qsum[j] = qsum[j] << 2 | j;
- // find the top 2 alleles
- for (i = 1; i < 4; ++i) // insertion sort
- for (j = i; j > 0 && qsum[j] < qsum[j-1]; --j)
- tmp = qsum[j], qsum[j] = qsum[j-1], qsum[j-1] = tmp;
- // set the reference allele and alternative allele(s)
- for (i = 0; i < 5; ++i) call->a[i] = -1;
- call->unseen = -1;
- call->a[0] = ref4;
- for (i = 3, j = 1; i >= 0; --i) {
- if ((qsum[i]&3) != ref4) {
- if (qsum[i]>>2 != 0)
- {
- if ( j<4 ) call->qsum[j] = (float)(qsum[i]>>2)/qsum_tot; // ref N can make j>=4
- call->a[j++] = qsum[i]&3;
- }
- else break;
- }
- else
- call->qsum[0] = (float)(qsum[i]>>2)/qsum_tot;
- }
- if (ref_base >= 0) { // for SNPs, find the "unseen" base
- if (((ref4 < 4 && j < 4) || (ref4 == 4 && j < 5)) && i >= 0)
- call->unseen = j, call->a[j++] = qsum[i]&3;
- call->n_alleles = j;
- } else {
- call->n_alleles = j;
- if (call->n_alleles == 1) return -1; // no reliable supporting read. stop doing anything
- }
- // set the PL array
- if (call->n < n) {
- call->n = n;
- call->PL = realloc(call->PL, 15 * n);
- }
- {
- int x, g[15], z;
- double sum_min = 0.;
- x = call->n_alleles * (call->n_alleles + 1) / 2;
- // get the possible genotypes
- for (i = z = 0; i < call->n_alleles; ++i)
- for (j = 0; j <= i; ++j)
- g[z++] = call->a[j] * 5 + call->a[i];
- for (i = 0; i < n; ++i) {
- uint8_t *PL = call->PL + x * i;
- const bcf_callret1_t *r = calls + i;
- float min = 1e37;
- for (j = 0; j < x; ++j)
- if (min > r->p[g[j]]) min = r->p[g[j]];
- sum_min += min;
- for (j = 0; j < x; ++j) {
- int y;
- y = (int)(r->p[g[j]] - min + .499);
- if (y > 255) y = 255;
- PL[j] = y;
- }
- }
-// if (ref_base < 0) fprintf(stderr, "%d,%d,%f,%d\n", call->n_alleles, x, sum_min, call->unseen);
- call->shift = (int)(sum_min + .499);
- }
- // combine annotations
- memset(call->anno, 0, 16 * sizeof(int));
- for (i = call->depth = call->ori_depth = 0, tmp = 0; i < n; ++i) {
- call->depth += calls[i].depth;
- call->ori_depth += calls[i].ori_depth;
- for (j = 0; j < 16; ++j) call->anno[j] += calls[i].anno[j];
- }
-
- calc_vdb(bca, call);
- calc_ReadPosBias(bca, call);
-
- return 0;
-}
-
-int bcf_call2bcf(int tid, int pos, bcf_call_t *bc, bcf1_t *b, bcf_callret1_t *bcr, int fmt_flag,
- const bcf_callaux_t *bca, const char *ref)
-{
- extern double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two);
- kstring_t s;
- int i, j;
- b->n_smpl = bc->n;
- b->tid = tid; b->pos = pos; b->qual = 0;
- s.s = b->str; s.m = b->m_str; s.l = 0;
- kputc('\0', &s);
- if (bc->ori_ref < 0) { // an indel
- // write REF
- kputc(ref[pos], &s);
- for (j = 0; j < bca->indelreg; ++j) kputc(ref[pos+1+j], &s);
- kputc('\0', &s);
- // write ALT
- kputc(ref[pos], &s);
- for (i = 1; i < 4; ++i) {
- if (bc->a[i] < 0) break;
- if (i > 1) {
- kputc(',', &s); kputc(ref[pos], &s);
- }
- if (bca->indel_types[bc->a[i]] < 0) { // deletion
- for (j = -bca->indel_types[bc->a[i]]; j < bca->indelreg; ++j)
- kputc(ref[pos+1+j], &s);
- } else { // insertion; cannot be a reference unless a bug
- char *inscns = &bca->inscns[bc->a[i] * bca->maxins];
- for (j = 0; j < bca->indel_types[bc->a[i]]; ++j)
- kputc("ACGTN"[(int)inscns[j]], &s);
- for (j = 0; j < bca->indelreg; ++j) kputc(ref[pos+1+j], &s);
- }
- }
- kputc('\0', &s);
- } else { // a SNP
- kputc("ACGTN"[bc->ori_ref], &s); kputc('\0', &s);
- for (i = 1; i < 5; ++i) {
- if (bc->a[i] < 0) break;
- if (i > 1) kputc(',', &s);
- kputc(bc->unseen == i? 'X' : "ACGT"[bc->a[i]], &s);
- }
- kputc('\0', &s);
- }
- kputc('\0', &s);
- // INFO
- if (bc->ori_ref < 0) ksprintf(&s,"INDEL;IS=%d,%f;", bca->max_support, bca->max_frac);
- kputs("DP=", &s); kputw(bc->ori_depth, &s); kputs(";I16=", &s);
- for (i = 0; i < 16; ++i) {
- if (i) kputc(',', &s);
- kputw(bc->anno[i], &s);
- }
- //ksprintf(&s,";RPS=%d,%f,%f", bc->read_pos.dp,bc->read_pos.avg,bc->read_pos.var);
- ksprintf(&s,";QS=%f,%f,%f,%f", bc->qsum[0],bc->qsum[1],bc->qsum[2],bc->qsum[3]);
- if (bc->vdb != -1)
- ksprintf(&s, ";VDB=%e", bc->vdb);
- if (bc->read_pos_bias != -1 )
- ksprintf(&s, ";RPB=%e", bc->read_pos_bias);
- kputc('\0', &s);
- // FMT
- kputs("PL", &s);
- if (bcr && fmt_flag) {
- if (fmt_flag & B2B_FMT_DP) kputs(":DP", &s);
- if (fmt_flag & B2B_FMT_DV) kputs(":DV", &s);
- if (fmt_flag & B2B_FMT_SP) kputs(":SP", &s);
- }
- kputc('\0', &s);
- b->m_str = s.m; b->str = s.s; b->l_str = s.l;
- bcf_sync(b);
- memcpy(b->gi[0].data, bc->PL, b->gi[0].len * bc->n);
- if (bcr && fmt_flag) {
- uint16_t *dp = (fmt_flag & B2B_FMT_DP)? b->gi[1].data : 0;
- uint16_t *dv = (fmt_flag & B2B_FMT_DV)? b->gi[1 + ((fmt_flag & B2B_FMT_DP) != 0)].data : 0;
- int32_t *sp = (fmt_flag & B2B_FMT_SP)? b->gi[1 + ((fmt_flag & B2B_FMT_DP) != 0) + ((fmt_flag & B2B_FMT_DV) != 0)].data : 0;
- for (i = 0; i < bc->n; ++i) {
- bcf_callret1_t *p = bcr + i;
- if (dp) dp[i] = p->depth < 0xffff? p->depth : 0xffff;
- if (dv) dv[i] = p->n_supp < 0xffff? p->n_supp : 0xffff;
- if (sp) {
- if (p->anno[0] + p->anno[1] < 2 || p->anno[2] + p->anno[3] < 2
- || p->anno[0] + p->anno[2] < 2 || p->anno[1] + p->anno[3] < 2)
- {
- sp[i] = 0;
- } else {
- double left, right, two;
- int x;
- kt_fisher_exact(p->anno[0], p->anno[1], p->anno[2], p->anno[3], &left, &right, &two);
- x = (int)(-4.343 * log(two) + .499);
- if (x > 255) x = 255;
- sp[i] = x;
- }
- }
- }
- }
- return 0;
-}
diff --git a/samtools-0.1.19/bam2bcf.h b/samtools-0.1.19/bam2bcf.h
deleted file mode 100644
index b2b1825..0000000
--- a/samtools-0.1.19/bam2bcf.h
+++ /dev/null
@@ -1,67 +0,0 @@
-#ifndef BAM2BCF_H
-#define BAM2BCF_H
-
-#include <stdint.h>
-#include "errmod.h"
-#include "bcftools/bcf.h"
-
-#define B2B_INDEL_NULL 10000
-
-#define B2B_FMT_DP 0x1
-#define B2B_FMT_SP 0x2
-#define B2B_FMT_DV 0x4
-
-typedef struct __bcf_callaux_t {
- int capQ, min_baseQ;
- int openQ, extQ, tandemQ; // for indels
- int min_support, max_support; // for collecting indel candidates
- double min_frac, max_frac; // for collecting indel candidates
- int per_sample_flt; // indel filtering strategy
- int *ref_pos, *alt_pos, npos; // for ReadPosBias
- // for internal uses
- int max_bases;
- int indel_types[4];
- int maxins, indelreg;
- int read_len;
- char *inscns;
- uint16_t *bases;
- errmod_t *e;
- void *rghash;
-} bcf_callaux_t;
-
-typedef struct {
- int depth, n_supp, ori_depth, qsum[4];
- unsigned int anno[16];
- float p[25];
-} bcf_callret1_t;
-
-typedef struct {
- int a[5]; // alleles: ref, alt, alt2, alt3
- float qsum[4];
- int n, n_alleles, shift, ori_ref, unseen;
- int n_supp; // number of supporting non-reference reads
- unsigned int anno[16], depth, ori_depth;
- uint8_t *PL;
- float vdb; // variant distance bias
- float read_pos_bias;
- struct { float avg, var; int dp; } read_pos;
-} bcf_call_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- bcf_callaux_t *bcf_call_init(double theta, int min_baseQ);
- void bcf_call_destroy(bcf_callaux_t *bca);
- int bcf_call_glfgen(int _n, const bam_pileup1_t *pl, int ref_base, bcf_callaux_t *bca, bcf_callret1_t *r);
- int bcf_call_combine(int n, const bcf_callret1_t *calls, bcf_callaux_t *bca, int ref_base /*4-bit*/, bcf_call_t *call);
- int bcf_call2bcf(int tid, int pos, bcf_call_t *bc, bcf1_t *b, bcf_callret1_t *bcr, int fmt_flag,
- const bcf_callaux_t *bca, const char *ref);
- int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref,
- const void *rghash);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/bam2bcf_indel.c b/samtools-0.1.19/bam2bcf_indel.c
deleted file mode 100644
index 30b3f46..0000000
--- a/samtools-0.1.19/bam2bcf_indel.c
+++ /dev/null
@@ -1,498 +0,0 @@
-#include <assert.h>
-#include <ctype.h>
-#include <string.h>
-#include "bam.h"
-#include "bam2bcf.h"
-#include "kaln.h"
-#include "kprobaln.h"
-#include "khash.h"
-KHASH_SET_INIT_STR(rg)
-
-#include "ksort.h"
-KSORT_INIT_GENERIC(uint32_t)
-
-#define MINUS_CONST 0x10000000
-#define INDEL_WINDOW_SIZE 50
-
-void *bcf_call_add_rg(void *_hash, const char *hdtext, const char *list)
-{
- const char *s, *p, *q, *r, *t;
- khash_t(rg) *hash;
- if (list == 0 || hdtext == 0) return _hash;
- if (_hash == 0) _hash = kh_init(rg);
- hash = (khash_t(rg)*)_hash;
- if ((s = strstr(hdtext, "@RG\t")) == 0) return hash;
- do {
- t = strstr(s + 4, "@RG\t"); // the next @RG
- if ((p = strstr(s, "\tID:")) != 0) p += 4;
- if ((q = strstr(s, "\tPL:")) != 0) q += 4;
- if (p && q && (t == 0 || (p < t && q < t))) { // ID and PL are both present
- int lp, lq;
- char *x;
- for (r = p; *r && *r != '\t' && *r != '\n'; ++r); lp = r - p;
- for (r = q; *r && *r != '\t' && *r != '\n'; ++r); lq = r - q;
- x = calloc((lp > lq? lp : lq) + 1, 1);
- for (r = q; *r && *r != '\t' && *r != '\n'; ++r) x[r-q] = *r;
- if (strstr(list, x)) { // insert ID to the hash table
- khint_t k;
- int ret;
- for (r = p; *r && *r != '\t' && *r != '\n'; ++r) x[r-p] = *r;
- x[r-p] = 0;
- k = kh_get(rg, hash, x);
- if (k == kh_end(hash)) k = kh_put(rg, hash, x, &ret);
- else free(x);
- } else free(x);
- }
- s = t;
- } while (s);
- return hash;
-}
-
-void bcf_call_del_rghash(void *_hash)
-{
- khint_t k;
- khash_t(rg) *hash = (khash_t(rg)*)_hash;
- if (hash == 0) return;
- for (k = kh_begin(hash); k < kh_end(hash); ++k)
- if (kh_exist(hash, k))
- free((char*)kh_key(hash, k));
- kh_destroy(rg, hash);
-}
-
-static int tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, int32_t tpos, int is_left, int32_t *_tpos)
-{
- int k, x = c->pos, y = 0, last_y = 0;
- *_tpos = c->pos;
- for (k = 0; k < c->n_cigar; ++k) {
- int op = cigar[k] & BAM_CIGAR_MASK;
- int l = cigar[k] >> BAM_CIGAR_SHIFT;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- if (c->pos > tpos) return y;
- if (x + l > tpos) {
- *_tpos = tpos;
- return y + (tpos - x);
- }
- x += l; y += l;
- last_y = y;
- } else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l;
- else if (op == BAM_CDEL || op == BAM_CREF_SKIP) {
- if (x + l > tpos) {
- *_tpos = is_left? x : x + l;
- return y;
- }
- x += l;
- }
- }
- *_tpos = x;
- return last_y;
-}
-// FIXME: check if the inserted sequence is consistent with the homopolymer run
-// l is the relative gap length and l_run is the length of the homopolymer on the reference
-static inline int est_seqQ(const bcf_callaux_t *bca, int l, int l_run)
-{
- int q, qh;
- q = bca->openQ + bca->extQ * (abs(l) - 1);
- qh = l_run >= 3? (int)(bca->tandemQ * (double)abs(l) / l_run + .499) : 1000;
- return q < qh? q : qh;
-}
-
-static inline int est_indelreg(int pos, const char *ref, int l, char *ins4)
-{
- int i, j, max = 0, max_i = pos, score = 0;
- l = abs(l);
- for (i = pos + 1, j = 0; ref[i]; ++i, ++j) {
- if (ins4) score += (toupper(ref[i]) != "ACGTN"[(int)ins4[j%l]])? -10 : 1;
- else score += (toupper(ref[i]) != toupper(ref[pos+1+j%l]))? -10 : 1;
- if (score < 0) break;
- if (max < score) max = score, max_i = i;
- }
- return max_i - pos;
-}
-
-/*
- * @n: number of samples
- */
-int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref,
- const void *rghash)
-{
- int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score1, *score2, max_ref2;
- int N, K, l_run, ref_type, n_alt;
- char *inscns = 0, *ref2, *query, **ref_sample;
- khash_t(rg) *hash = (khash_t(rg)*)rghash;
- if (ref == 0 || bca == 0) return -1;
- // mark filtered reads
- if (rghash) {
- N = 0;
- for (s = N = 0; s < n; ++s) {
- for (i = 0; i < n_plp[s]; ++i) {
- bam_pileup1_t *p = plp[s] + i;
- const uint8_t *rg = bam_aux_get(p->b, "RG");
- p->aux = 1; // filtered by default
- if (rg) {
- khint_t k = kh_get(rg, hash, (const char*)(rg + 1));
- if (k != kh_end(hash)) p->aux = 0, ++N; // not filtered
- }
- }
- }
- if (N == 0) return -1; // no reads left
- }
- // determine if there is a gap
- for (s = N = 0; s < n; ++s) {
- for (i = 0; i < n_plp[s]; ++i)
- if (plp[s][i].indel != 0) break;
- if (i < n_plp[s]) break;
- }
- if (s == n) return -1; // there is no indel at this position.
- for (s = N = 0; s < n; ++s) N += n_plp[s]; // N is the total number of reads
- { // find out how many types of indels are present
- bca->max_support = bca->max_frac = 0;
- int m, n_alt = 0, n_tot = 0, indel_support_ok = 0;
- uint32_t *aux;
- aux = calloc(N + 1, 4);
- m = max_rd_len = 0;
- aux[m++] = MINUS_CONST; // zero indel is always a type
- for (s = 0; s < n; ++s) {
- int na = 0, nt = 0;
- for (i = 0; i < n_plp[s]; ++i) {
- const bam_pileup1_t *p = plp[s] + i;
- if (rghash == 0 || p->aux == 0) {
- ++nt;
- if (p->indel != 0) {
- ++na;
- aux[m++] = MINUS_CONST + p->indel;
- }
- }
- j = bam_cigar2qlen(&p->b->core, bam1_cigar(p->b));
- if (j > max_rd_len) max_rd_len = j;
- }
- float frac = (float)na/nt;
- if ( !indel_support_ok && na >= bca->min_support && frac >= bca->min_frac )
- indel_support_ok = 1;
- if ( na > bca->max_support && frac > 0 ) bca->max_support = na, bca->max_frac = frac;
- n_alt += na;
- n_tot += nt;
- }
- // To prevent long stretches of N's to be mistaken for indels (sometimes thousands of bases),
- // check the number of N's in the sequence and skip places where half or more reference bases are Ns.
- int nN=0; for (i=pos; i-pos<max_rd_len && ref[i]; i++) if ( ref[i]=='N' ) nN++;
- if ( nN*2>i ) { free(aux); return -1; }
-
- ks_introsort(uint32_t, m, aux);
- // squeeze out identical types
- for (i = 1, n_types = 1; i < m; ++i)
- if (aux[i] != aux[i-1]) ++n_types;
- // Taking totals makes it hard to call rare indels
- if ( !bca->per_sample_flt )
- indel_support_ok = ( (float)n_alt / n_tot < bca->min_frac || n_alt < bca->min_support ) ? 0 : 1;
- if ( n_types == 1 || !indel_support_ok ) { // then skip
- free(aux); return -1;
- }
- if (n_types >= 64) {
- free(aux);
- if (bam_verbose >= 2)
- fprintf(stderr, "[%s] excessive INDEL alleles at position %d. Skip the position.\n", __func__, pos + 1);
- return -1;
- }
- types = (int*)calloc(n_types, sizeof(int));
- t = 0;
- types[t++] = aux[0] - MINUS_CONST;
- for (i = 1; i < m; ++i)
- if (aux[i] != aux[i-1])
- types[t++] = aux[i] - MINUS_CONST;
- free(aux);
- for (t = 0; t < n_types; ++t)
- if (types[t] == 0) break;
- ref_type = t; // the index of the reference type (0)
- }
- { // calculate left and right boundary
- left = pos > INDEL_WINDOW_SIZE? pos - INDEL_WINDOW_SIZE : 0;
- right = pos + INDEL_WINDOW_SIZE;
- if (types[0] < 0) right -= types[0];
- // in case the alignments stand out the reference
- for (i = pos; i < right; ++i)
- if (ref[i] == 0) break;
- right = i;
- }
- /* The following block fixes a long-existing flaw in the INDEL
- * calling model: the interference of nearby SNPs. However, it also
- * reduces the power because sometimes, substitutions caused by
- * indels are not distinguishable from true mutations. Multiple
- * sequence realignment helps to increase the power.
- *
- * Masks mismatches present in at least 70% of the reads with 'N'.
- */
- { // construct per-sample consensus
- int L = right - left + 1, max_i, max2_i;
- uint32_t *cns, max, max2;
- char *ref0, *r;
- ref_sample = calloc(n, sizeof(void*));
- cns = calloc(L, 4);
- ref0 = calloc(L, 1);
- for (i = 0; i < right - left; ++i)
- ref0[i] = bam_nt16_table[(int)ref[i+left]];
- for (s = 0; s < n; ++s) {
- r = ref_sample[s] = calloc(L, 1);
- memset(cns, 0, sizeof(int) * L);
- // collect ref and non-ref counts
- for (i = 0; i < n_plp[s]; ++i) {
- bam_pileup1_t *p = plp[s] + i;
- bam1_t *b = p->b;
- uint32_t *cigar = bam1_cigar(b);
- uint8_t *seq = bam1_seq(b);
- int x = b->core.pos, y = 0;
- for (k = 0; k < b->core.n_cigar; ++k) {
- int op = cigar[k]&0xf;
- int j, l = cigar[k]>>4;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (j = 0; j < l; ++j)
- if (x + j >= left && x + j < right)
- cns[x+j-left] += (bam1_seqi(seq, y+j) == ref0[x+j-left])? 1 : 0x10000;
- x += l; y += l;
- } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) x += l;
- else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l;
- }
- }
- // determine the consensus
- for (i = 0; i < right - left; ++i) r[i] = ref0[i];
- max = max2 = 0; max_i = max2_i = -1;
- for (i = 0; i < right - left; ++i) {
- if (cns[i]>>16 >= max>>16) max2 = max, max2_i = max_i, max = cns[i], max_i = i;
- else if (cns[i]>>16 >= max2>>16) max2 = cns[i], max2_i = i;
- }
- if ((double)(max&0xffff) / ((max&0xffff) + (max>>16)) >= 0.7) max_i = -1;
- if ((double)(max2&0xffff) / ((max2&0xffff) + (max2>>16)) >= 0.7) max2_i = -1;
- if (max_i >= 0) r[max_i] = 15;
- if (max2_i >= 0) r[max2_i] = 15;
- //for (i = 0; i < right - left; ++i) fputc("=ACMGRSVTWYHKDBN"[(int)r[i]], stderr); fputc('\n', stderr);
- }
- free(ref0); free(cns);
- }
- { // the length of the homopolymer run around the current position
- int c = bam_nt16_table[(int)ref[pos + 1]];
- if (c == 15) l_run = 1;
- else {
- for (i = pos + 2; ref[i]; ++i)
- if (bam_nt16_table[(int)ref[i]] != c) break;
- l_run = i;
- for (i = pos; i >= 0; --i)
- if (bam_nt16_table[(int)ref[i]] != c) break;
- l_run -= i + 1;
- }
- }
- // construct the consensus sequence
- max_ins = types[n_types - 1]; // max_ins is at least 0
- if (max_ins > 0) {
- int *inscns_aux = calloc(5 * n_types * max_ins, sizeof(int));
- // count the number of occurrences of each base at each position for each type of insertion
- for (t = 0; t < n_types; ++t) {
- if (types[t] > 0) {
- for (s = 0; s < n; ++s) {
- for (i = 0; i < n_plp[s]; ++i) {
- bam_pileup1_t *p = plp[s] + i;
- if (p->indel == types[t]) {
- uint8_t *seq = bam1_seq(p->b);
- for (k = 1; k <= p->indel; ++k) {
- int c = bam_nt16_nt4_table[bam1_seqi(seq, p->qpos + k)];
- assert(c<5);
- ++inscns_aux[(t*max_ins+(k-1))*5 + c];
- }
- }
- }
- }
- }
- }
- // use the majority rule to construct the consensus
- inscns = calloc(n_types * max_ins, 1);
- for (t = 0; t < n_types; ++t) {
- for (j = 0; j < types[t]; ++j) {
- int max = 0, max_k = -1, *ia = &inscns_aux[(t*max_ins+j)*5];
- for (k = 0; k < 5; ++k)
- if (ia[k] > max)
- max = ia[k], max_k = k;
- inscns[t*max_ins + j] = max? max_k : 4;
- if ( max_k==4 ) { types[t] = 0; break; } // discard insertions which contain N's
- }
- }
- free(inscns_aux);
- }
- // compute the likelihood given each type of indel for each read
- max_ref2 = right - left + 2 + 2 * (max_ins > -types[0]? max_ins : -types[0]);
- ref2 = calloc(max_ref2, 1);
- query = calloc(right - left + max_rd_len + max_ins + 2, 1);
- score1 = calloc(N * n_types, sizeof(int));
- score2 = calloc(N * n_types, sizeof(int));
- bca->indelreg = 0;
- for (t = 0; t < n_types; ++t) {
- int l, ir;
- kpa_par_t apf1 = { 1e-4, 1e-2, 10 }, apf2 = { 1e-6, 1e-3, 10 };
- apf1.bw = apf2.bw = abs(types[t]) + 3;
- // compute indelreg
- if (types[t] == 0) ir = 0;
- else if (types[t] > 0) ir = est_indelreg(pos, ref, types[t], &inscns[t*max_ins]);
- else ir = est_indelreg(pos, ref, -types[t], 0);
- if (ir > bca->indelreg) bca->indelreg = ir;
-// fprintf(stderr, "%d, %d, %d\n", pos, types[t], ir);
- // realignment
- for (s = K = 0; s < n; ++s) {
- // write ref2
- for (k = 0, j = left; j <= pos; ++j)
- ref2[k++] = bam_nt16_nt4_table[(int)ref_sample[s][j-left]];
- if (types[t] <= 0) j += -types[t];
- else for (l = 0; l < types[t]; ++l)
- ref2[k++] = inscns[t*max_ins + l];
- for (; j < right && ref[j]; ++j)
- ref2[k++] = bam_nt16_nt4_table[(int)ref_sample[s][j-left]];
- for (; k < max_ref2; ++k) ref2[k] = 4;
- if (j < right) right = j;
- // align each read to ref2
- for (i = 0; i < n_plp[s]; ++i, ++K) {
- bam_pileup1_t *p = plp[s] + i;
- int qbeg, qend, tbeg, tend, sc, kk;
- uint8_t *seq = bam1_seq(p->b);
- uint32_t *cigar = bam1_cigar(p->b);
- if (p->b->core.flag&4) continue; // unmapped reads
- // FIXME: the following loop should be better moved outside; nonetheless, realignment should be much slower anyway.
- for (kk = 0; kk < p->b->core.n_cigar; ++kk)
- if ((cigar[kk]&BAM_CIGAR_MASK) == BAM_CREF_SKIP) break;
- if (kk < p->b->core.n_cigar) continue;
- // FIXME: the following skips soft clips, but using them may be more sensitive.
- // determine the start and end of sequences for alignment
- qbeg = tpos2qpos(&p->b->core, bam1_cigar(p->b), left, 0, &tbeg);
- qend = tpos2qpos(&p->b->core, bam1_cigar(p->b), right, 1, &tend);
- if (types[t] < 0) {
- int l = -types[t];
- tbeg = tbeg - l > left? tbeg - l : left;
- }
- // write the query sequence
- for (l = qbeg; l < qend; ++l)
- query[l - qbeg] = bam_nt16_nt4_table[bam1_seqi(seq, l)];
- { // do realignment; this is the bottleneck
- const uint8_t *qual = bam1_qual(p->b), *bq;
- uint8_t *qq;
- qq = calloc(qend - qbeg, 1);
- bq = (uint8_t*)bam_aux_get(p->b, "ZQ");
- if (bq) ++bq; // skip type
- for (l = qbeg; l < qend; ++l) {
- qq[l - qbeg] = bq? qual[l] + (bq[l] - 64) : qual[l];
- if (qq[l - qbeg] > 30) qq[l - qbeg] = 30;
- if (qq[l - qbeg] < 7) qq[l - qbeg] = 7;
- }
- sc = kpa_glocal((uint8_t*)ref2 + tbeg - left, tend - tbeg + abs(types[t]),
- (uint8_t*)query, qend - qbeg, qq, &apf1, 0, 0);
- l = (int)(100. * sc / (qend - qbeg) + .499); // used for adjusting indelQ below
- if (l > 255) l = 255;
- score1[K*n_types + t] = score2[K*n_types + t] = sc<<8 | l;
- if (sc > 5) {
- sc = kpa_glocal((uint8_t*)ref2 + tbeg - left, tend - tbeg + abs(types[t]),
- (uint8_t*)query, qend - qbeg, qq, &apf2, 0, 0);
- l = (int)(100. * sc / (qend - qbeg) + .499);
- if (l > 255) l = 255;
- score2[K*n_types + t] = sc<<8 | l;
- }
- free(qq);
- }
-/*
- for (l = 0; l < tend - tbeg + abs(types[t]); ++l)
- fputc("ACGTN"[(int)ref2[tbeg-left+l]], stderr);
- fputc('\n', stderr);
- for (l = 0; l < qend - qbeg; ++l) fputc("ACGTN"[(int)query[l]], stderr);
- fputc('\n', stderr);
- fprintf(stderr, "pos=%d type=%d read=%d:%d name=%s qbeg=%d tbeg=%d score=%d\n", pos, types[t], s, i, bam1_qname(p->b), qbeg, tbeg, sc);
-*/
- }
- }
- }
- free(ref2); free(query);
- { // compute indelQ
- int *sc, tmp, *sumq;
- sc = alloca(n_types * sizeof(int));
- sumq = alloca(n_types * sizeof(int));
- memset(sumq, 0, sizeof(int) * n_types);
- for (s = K = 0; s < n; ++s) {
- for (i = 0; i < n_plp[s]; ++i, ++K) {
- bam_pileup1_t *p = plp[s] + i;
- int *sct = &score1[K*n_types], indelQ1, indelQ2, seqQ, indelQ;
- for (t = 0; t < n_types; ++t) sc[t] = sct[t]<<6 | t;
- for (t = 1; t < n_types; ++t) // insertion sort
- for (j = t; j > 0 && sc[j] < sc[j-1]; --j)
- tmp = sc[j], sc[j] = sc[j-1], sc[j-1] = tmp;
- /* errmod_cal() assumes that if the call is wrong, the
- * likelihoods of other events are equal. This is about
- * right for substitutions, but is not desired for
- * indels. To reuse errmod_cal(), I have to make
- * compromise for multi-allelic indels.
- */
- if ((sc[0]&0x3f) == ref_type) {
- indelQ1 = (sc[1]>>14) - (sc[0]>>14);
- seqQ = est_seqQ(bca, types[sc[1]&0x3f], l_run);
- } else {
- for (t = 0; t < n_types; ++t) // look for the reference type
- if ((sc[t]&0x3f) == ref_type) break;
- indelQ1 = (sc[t]>>14) - (sc[0]>>14);
- seqQ = est_seqQ(bca, types[sc[0]&0x3f], l_run);
- }
- tmp = sc[0]>>6 & 0xff;
- indelQ1 = tmp > 111? 0 : (int)((1. - tmp/111.) * indelQ1 + .499); // reduce indelQ
- sct = &score2[K*n_types];
- for (t = 0; t < n_types; ++t) sc[t] = sct[t]<<6 | t;
- for (t = 1; t < n_types; ++t) // insertion sort
- for (j = t; j > 0 && sc[j] < sc[j-1]; --j)
- tmp = sc[j], sc[j] = sc[j-1], sc[j-1] = tmp;
- if ((sc[0]&0x3f) == ref_type) {
- indelQ2 = (sc[1]>>14) - (sc[0]>>14);
- } else {
- for (t = 0; t < n_types; ++t) // look for the reference type
- if ((sc[t]&0x3f) == ref_type) break;
- indelQ2 = (sc[t]>>14) - (sc[0]>>14);
- }
- tmp = sc[0]>>6 & 0xff;
- indelQ2 = tmp > 111? 0 : (int)((1. - tmp/111.) * indelQ2 + .499);
- // pick the smaller between indelQ1 and indelQ2
- indelQ = indelQ1 < indelQ2? indelQ1 : indelQ2;
- if (indelQ > 255) indelQ = 255;
- if (seqQ > 255) seqQ = 255;
- p->aux = (sc[0]&0x3f)<<16 | seqQ<<8 | indelQ; // use 22 bits in total
- sumq[sc[0]&0x3f] += indelQ < seqQ? indelQ : seqQ;
-// fprintf(stderr, "pos=%d read=%d:%d name=%s call=%d indelQ=%d seqQ=%d\n", pos, s, i, bam1_qname(p->b), types[sc[0]&0x3f], indelQ, seqQ);
- }
- }
- // determine bca->indel_types[] and bca->inscns
- bca->maxins = max_ins;
- bca->inscns = realloc(bca->inscns, bca->maxins * 4);
- for (t = 0; t < n_types; ++t)
- sumq[t] = sumq[t]<<6 | t;
- for (t = 1; t < n_types; ++t) // insertion sort
- for (j = t; j > 0 && sumq[j] > sumq[j-1]; --j)
- tmp = sumq[j], sumq[j] = sumq[j-1], sumq[j-1] = tmp;
- for (t = 0; t < n_types; ++t) // look for the reference type
- if ((sumq[t]&0x3f) == ref_type) break;
- if (t) { // then move the reference type to the first
- tmp = sumq[t];
- for (; t > 0; --t) sumq[t] = sumq[t-1];
- sumq[0] = tmp;
- }
- for (t = 0; t < 4; ++t) bca->indel_types[t] = B2B_INDEL_NULL;
- for (t = 0; t < 4 && t < n_types; ++t) {
- bca->indel_types[t] = types[sumq[t]&0x3f];
- memcpy(&bca->inscns[t * bca->maxins], &inscns[(sumq[t]&0x3f) * max_ins], bca->maxins);
- }
- // update p->aux
- for (s = n_alt = 0; s < n; ++s) {
- for (i = 0; i < n_plp[s]; ++i) {
- bam_pileup1_t *p = plp[s] + i;
- int x = types[p->aux>>16&0x3f];
- for (j = 0; j < 4; ++j)
- if (x == bca->indel_types[j]) break;
- p->aux = j<<16 | (j == 4? 0 : (p->aux&0xffff));
- if ((p->aux>>16&0x3f) > 0) ++n_alt;
-// fprintf(stderr, "X pos=%d read=%d:%d name=%s call=%d type=%d q=%d seqQ=%d\n", pos, s, i, bam1_qname(p->b), p->aux>>16&63, bca->indel_types[p->aux>>16&63], p->aux&0xff, p->aux>>8&0xff);
- }
- }
- }
- free(score1); free(score2);
- // free
- for (i = 0; i < n; ++i) free(ref_sample[i]);
- free(ref_sample);
- free(types); free(inscns);
- return n_alt > 0? 0 : -1;
-}
diff --git a/samtools-0.1.19/bam2depth.c b/samtools-0.1.19/bam2depth.c
deleted file mode 100644
index 02311ef..0000000
--- a/samtools-0.1.19/bam2depth.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/* This program demonstrates how to generate pileup from multiple BAMs
- * simutaneously, to achieve random access and to use the BED interface.
- * To compile this program separately, you may:
- *
- * gcc -g -O2 -Wall -o bam2depth -D_MAIN_BAM2DEPTH bam2depth.c -L. -lbam -lz
- */
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <unistd.h>
-#include "bam.h"
-
-typedef struct { // auxiliary data structure
- bamFile fp; // the file handler
- bam_iter_t iter; // NULL if a region not specified
- int min_mapQ, min_len; // mapQ filter; length filter
-} aux_t;
-
-void *bed_read(const char *fn); // read a BED or position list file
-void bed_destroy(void *_h); // destroy the BED data structure
-int bed_overlap(const void *_h, const char *chr, int beg, int end); // test if chr:beg-end overlaps
-
-// This function reads a BAM alignment from one BAM file.
-static int read_bam(void *data, bam1_t *b) // read level filters better go here to avoid pileup
-{
- aux_t *aux = (aux_t*)data; // data in fact is a pointer to an auxiliary structure
- int ret = aux->iter? bam_iter_read(aux->fp, aux->iter, b) : bam_read1(aux->fp, b);
- if (!(b->core.flag&BAM_FUNMAP)) {
- if ((int)b->core.qual < aux->min_mapQ) b->core.flag |= BAM_FUNMAP;
- else if (aux->min_len && bam_cigar2qlen(&b->core, bam1_cigar(b)) < aux->min_len) b->core.flag |= BAM_FUNMAP;
- }
- return ret;
-}
-
-int read_file_list(const char *file_list,int *n,char **argv[]);
-
-#ifdef _MAIN_BAM2DEPTH
-int main(int argc, char *argv[])
-#else
-int main_depth(int argc, char *argv[])
-#endif
-{
- int i, n, tid, beg, end, pos, *n_plp, baseQ = 0, mapQ = 0, min_len = 0, nfiles;
- const bam_pileup1_t **plp;
- char *reg = 0; // specified region
- void *bed = 0; // BED data structure
- char *file_list = NULL, **fn = NULL;
- bam_header_t *h = 0; // BAM header of the 1st input
- aux_t **data;
- bam_mplp_t mplp;
-
- // parse the command line
- while ((n = getopt(argc, argv, "r:b:q:Q:l:f:")) >= 0) {
- switch (n) {
- case 'l': min_len = atoi(optarg); break; // minimum query length
- case 'r': reg = strdup(optarg); break; // parsing a region requires a BAM header
- case 'b': bed = bed_read(optarg); break; // BED or position list file can be parsed now
- case 'q': baseQ = atoi(optarg); break; // base quality threshold
- case 'Q': mapQ = atoi(optarg); break; // mapping quality threshold
- case 'f': file_list = optarg; break;
- }
- }
- if (optind == argc && !file_list) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools depth [options] in1.bam [in2.bam [...]]\n");
- fprintf(stderr, "Options:\n");
- fprintf(stderr, " -b <bed> list of positions or regions\n");
- fprintf(stderr, " -f <list> list of input BAM filenames, one per line [null]\n");
- fprintf(stderr, " -l <int> minQLen\n");
- fprintf(stderr, " -q <int> base quality threshold\n");
- fprintf(stderr, " -Q <int> mapping quality threshold\n");
- fprintf(stderr, " -r <chr:from-to> region\n");
- fprintf(stderr, "\n");
- return 1;
- }
-
- // initialize the auxiliary data structures
- if (file_list)
- {
- if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
- n = nfiles;
- argv = fn;
- optind = 0;
- }
- else
- n = argc - optind; // the number of BAMs on the command line
- data = calloc(n, sizeof(void*)); // data[i] for the i-th input
- beg = 0; end = 1<<30; tid = -1; // set the default region
- for (i = 0; i < n; ++i) {
- bam_header_t *htmp;
- data[i] = calloc(1, sizeof(aux_t));
- data[i]->fp = bam_open(argv[optind+i], "r"); // open BAM
- data[i]->min_mapQ = mapQ; // set the mapQ filter
- data[i]->min_len = min_len; // set the qlen filter
- htmp = bam_header_read(data[i]->fp); // read the BAM header
- if (i == 0) {
- h = htmp; // keep the header of the 1st BAM
- if (reg) bam_parse_region(h, reg, &tid, &beg, &end); // also parse the region
- } else bam_header_destroy(htmp); // if not the 1st BAM, trash the header
- if (tid >= 0) { // if a region is specified and parsed successfully
- bam_index_t *idx = bam_index_load(argv[optind+i]); // load the index
- data[i]->iter = bam_iter_query(idx, tid, beg, end); // set the iterator
- bam_index_destroy(idx); // the index is not needed any more; phase out of the memory
- }
- }
-
- // the core multi-pileup loop
- mplp = bam_mplp_init(n, read_bam, (void**)data); // initialization
- n_plp = calloc(n, sizeof(int)); // n_plp[i] is the number of covering reads from the i-th BAM
- plp = calloc(n, sizeof(void*)); // plp[i] points to the array of covering reads (internal in mplp)
- while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0) { // come to the next covered position
- if (pos < beg || pos >= end) continue; // out of range; skip
- if (bed && bed_overlap(bed, h->target_name[tid], pos, pos + 1) == 0) continue; // not in BED; skip
- fputs(h->target_name[tid], stdout); printf("\t%d", pos+1); // a customized printf() would be faster
- for (i = 0; i < n; ++i) { // base level filters have to go here
- int j, m = 0;
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j; // DON'T modfity plp[][] unless you really know
- if (p->is_del || p->is_refskip) ++m; // having dels or refskips at tid:pos
- else if (bam1_qual(p->b)[p->qpos] < baseQ) ++m; // low base quality
- }
- printf("\t%d", n_plp[i] - m); // this the depth to output
- }
- putchar('\n');
- }
- free(n_plp); free(plp);
- bam_mplp_destroy(mplp);
-
- bam_header_destroy(h);
- for (i = 0; i < n; ++i) {
- bam_close(data[i]->fp);
- if (data[i]->iter) bam_iter_destroy(data[i]->iter);
- free(data[i]);
- }
- free(data); free(reg);
- if (bed) bed_destroy(bed);
- if ( file_list )
- {
- for (i=0; i<n; i++) free(fn[i]);
- free(fn);
- }
- return 0;
-}
diff --git a/samtools-0.1.19/bam_aux.c b/samtools-0.1.19/bam_aux.c
deleted file mode 100644
index 4bbf975..0000000
--- a/samtools-0.1.19/bam_aux.c
+++ /dev/null
@@ -1,217 +0,0 @@
-#include <ctype.h>
-#include "bam.h"
-#include "khash.h"
-typedef char *str_p;
-KHASH_MAP_INIT_STR(s, int)
-KHASH_MAP_INIT_STR(r2l, str_p)
-
-void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data)
-{
- int ori_len = b->data_len;
- b->data_len += 3 + len;
- b->l_aux += 3 + len;
- if (b->m_data < b->data_len) {
- b->m_data = b->data_len;
- kroundup32(b->m_data);
- b->data = (uint8_t*)realloc(b->data, b->m_data);
- }
- b->data[ori_len] = tag[0]; b->data[ori_len + 1] = tag[1];
- b->data[ori_len + 2] = type;
- memcpy(b->data + ori_len + 3, data, len);
-}
-
-uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2])
-{
- return bam_aux_get(b, tag);
-}
-
-#define __skip_tag(s) do { \
- int type = toupper(*(s)); \
- ++(s); \
- if (type == 'Z' || type == 'H') { while (*(s)) ++(s); ++(s); } \
- else if (type == 'B') (s) += 5 + bam_aux_type2size(*(s)) * (*(int32_t*)((s)+1)); \
- else (s) += bam_aux_type2size(type); \
- } while(0)
-
-uint8_t *bam_aux_get(const bam1_t *b, const char tag[2])
-{
- uint8_t *s;
- int y = tag[0]<<8 | tag[1];
- s = bam1_aux(b);
- while (s < b->data + b->data_len) {
- int x = (int)s[0]<<8 | s[1];
- s += 2;
- if (x == y) return s;
- __skip_tag(s);
- }
- return 0;
-}
-// s MUST BE returned by bam_aux_get()
-int bam_aux_del(bam1_t *b, uint8_t *s)
-{
- uint8_t *p, *aux;
- aux = bam1_aux(b);
- p = s - 2;
- __skip_tag(s);
- memmove(p, s, b->l_aux - (s - aux));
- b->data_len -= s - p;
- b->l_aux -= s - p;
- return 0;
-}
-
-int bam_aux_drop_other(bam1_t *b, uint8_t *s)
-{
- if (s) {
- uint8_t *p, *aux;
- aux = bam1_aux(b);
- p = s - 2;
- __skip_tag(s);
- memmove(aux, p, s - p);
- b->data_len -= b->l_aux - (s - p);
- b->l_aux = s - p;
- } else {
- b->data_len -= b->l_aux;
- b->l_aux = 0;
- }
- return 0;
-}
-
-void bam_init_header_hash(bam_header_t *header)
-{
- if (header->hash == 0) {
- int ret, i;
- khiter_t iter;
- khash_t(s) *h;
- header->hash = h = kh_init(s);
- for (i = 0; i < header->n_targets; ++i) {
- iter = kh_put(s, h, header->target_name[i], &ret);
- kh_value(h, iter) = i;
- }
- }
-}
-
-void bam_destroy_header_hash(bam_header_t *header)
-{
- if (header->hash)
- kh_destroy(s, (khash_t(s)*)header->hash);
-}
-
-int32_t bam_get_tid(const bam_header_t *header, const char *seq_name)
-{
- khint_t k;
- khash_t(s) *h = (khash_t(s)*)header->hash;
- k = kh_get(s, h, seq_name);
- return k == kh_end(h)? -1 : kh_value(h, k);
-}
-
-int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *beg, int *end)
-{
- char *s;
- int i, l, k, name_end;
- khiter_t iter;
- khash_t(s) *h;
-
- bam_init_header_hash(header);
- h = (khash_t(s)*)header->hash;
-
- *ref_id = *beg = *end = -1;
- name_end = l = strlen(str);
- s = (char*)malloc(l+1);
- // remove space
- for (i = k = 0; i < l; ++i)
- if (!isspace(str[i])) s[k++] = str[i];
- s[k] = 0; l = k;
- // determine the sequence name
- for (i = l - 1; i >= 0; --i) if (s[i] == ':') break; // look for colon from the end
- if (i >= 0) name_end = i;
- if (name_end < l) { // check if this is really the end
- int n_hyphen = 0;
- for (i = name_end + 1; i < l; ++i) {
- if (s[i] == '-') ++n_hyphen;
- else if (!isdigit(s[i]) && s[i] != ',') break;
- }
- if (i < l || n_hyphen > 1) name_end = l; // malformated region string; then take str as the name
- s[name_end] = 0;
- iter = kh_get(s, h, s);
- if (iter == kh_end(h)) { // cannot find the sequence name
- iter = kh_get(s, h, str); // try str as the name
- if (iter == kh_end(h)) {
- if (bam_verbose >= 2) fprintf(stderr, "[%s] fail to determine the sequence name.\n", __func__);
- free(s); return -1;
- } else s[name_end] = ':', name_end = l;
- }
- } else iter = kh_get(s, h, str);
- if (iter == kh_end(h)) {
- free(s);
- return -1;
- }
- *ref_id = kh_val(h, iter);
- // parse the interval
- if (name_end < l) {
- for (i = k = name_end + 1; i < l; ++i)
- if (s[i] != ',') s[k++] = s[i];
- s[k] = 0;
- *beg = atoi(s + name_end + 1);
- for (i = name_end + 1; i != k; ++i) if (s[i] == '-') break;
- *end = i < k? atoi(s + i + 1) : 1<<29;
- if (*beg > 0) --*beg;
- } else *beg = 0, *end = 1<<29;
- free(s);
- return *beg <= *end? 0 : -1;
-}
-
-int32_t bam_aux2i(const uint8_t *s)
-{
- int type;
- if (s == 0) return 0;
- type = *s++;
- if (type == 'c') return (int32_t)*(int8_t*)s;
- else if (type == 'C') return (int32_t)*(uint8_t*)s;
- else if (type == 's') return (int32_t)*(int16_t*)s;
- else if (type == 'S') return (int32_t)*(uint16_t*)s;
- else if (type == 'i' || type == 'I') return *(int32_t*)s;
- else return 0;
-}
-
-float bam_aux2f(const uint8_t *s)
-{
- int type;
- type = *s++;
- if (s == 0) return 0.0;
- if (type == 'f') return *(float*)s;
- else return 0.0;
-}
-
-double bam_aux2d(const uint8_t *s)
-{
- int type;
- type = *s++;
- if (s == 0) return 0.0;
- if (type == 'd') return *(double*)s;
- else return 0.0;
-}
-
-char bam_aux2A(const uint8_t *s)
-{
- int type;
- type = *s++;
- if (s == 0) return 0;
- if (type == 'A') return *(char*)s;
- else return 0;
-}
-
-char *bam_aux2Z(const uint8_t *s)
-{
- int type;
- type = *s++;
- if (s == 0) return 0;
- if (type == 'Z' || type == 'H') return (char*)s;
- else return 0;
-}
-
-#ifdef _WIN32
-double drand48()
-{
- return (double)rand() / RAND_MAX;
-}
-#endif
diff --git a/samtools-0.1.19/bam_cat.c b/samtools-0.1.19/bam_cat.c
deleted file mode 100644
index a7502b9..0000000
--- a/samtools-0.1.19/bam_cat.c
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
-
-bam_cat -- efficiently concatenates bam files
-
-bam_cat can be used to concatenate BAM files. Under special
-circumstances, it can be used as an alternative to 'samtools merge' to
-concatenate multiple sorted files into a single sorted file. For this
-to work each file must be sorted, and the sorted files must be given
-as command line arguments in order such that the final read in file i
-is less than or equal to the first read in file i+1.
-
-This code is derived from the bam_reheader function in samtools 0.1.8
-and modified to perform concatenation by Chris Saunders on behalf of
-Illumina.
-
-
-########## License:
-
-The MIT License
-
-Original SAMtools work copyright (c) 2008-2009 Genome Research Ltd.
-Modified SAMtools work copyright (c) 2010 Illumina, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
-*/
-
-
-/*
-makefile:
-"""
-CC=gcc
-CFLAGS+=-g -Wall -O2 -D_FILE_OFFSET_BITS=64 -D_USE_KNETFILE -I$(SAMTOOLS_DIR)
-LDFLAGS+=-L$(SAMTOOLS_DIR)
-LDLIBS+=-lbam -lz
-
-all:bam_cat
-"""
-*/
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "knetfile.h"
-#include "bgzf.h"
-#include "bam.h"
-
-#define BUF_SIZE 0x10000
-
-#define GZIPID1 31
-#define GZIPID2 139
-
-#define BGZF_EMPTY_BLOCK_SIZE 28
-
-
-int bam_cat(int nfn, char * const *fn, const bam_header_t *h, const char* outbam)
-{
- BGZF *fp;
- FILE* fp_file;
- uint8_t *buf;
- uint8_t ebuf[BGZF_EMPTY_BLOCK_SIZE];
- const int es=BGZF_EMPTY_BLOCK_SIZE;
- int i;
-
- fp = strcmp(outbam, "-")? bgzf_open(outbam, "w") : bgzf_fdopen(fileno(stdout), "w");
- if (fp == 0) {
- fprintf(stderr, "[%s] ERROR: fail to open output file '%s'.\n", __func__, outbam);
- return 1;
- }
- if (h) bam_header_write(fp, h);
-
- buf = (uint8_t*) malloc(BUF_SIZE);
- for(i = 0; i < nfn; ++i){
- BGZF *in;
- bam_header_t *old;
- int len,j;
-
- in = strcmp(fn[i], "-")? bam_open(fn[i], "r") : bam_dopen(fileno(stdin), "r");
- if (in == 0) {
- fprintf(stderr, "[%s] ERROR: fail to open file '%s'.\n", __func__, fn[i]);
- return -1;
- }
- if (in->is_write) return -1;
-
- old = bam_header_read(in);
- if (h == 0 && i == 0) bam_header_write(fp, old);
-
- if (in->block_offset < in->block_length) {
- bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset);
- bgzf_flush(fp);
- }
-
- j=0;
-#ifdef _USE_KNETFILE
- fp_file = fp->fp;
- while ((len = knet_read(in->fp, buf, BUF_SIZE)) > 0) {
-#else
- fp_file = fp->fp;
- while (!feof(in->file) && (len = fread(buf, 1, BUF_SIZE, in->file)) > 0) {
-#endif
- if(len<es){
- int diff=es-len;
- if(j==0) {
- fprintf(stderr, "[%s] ERROR: truncated file?: '%s'.\n", __func__, fn[i]);
- return -1;
- }
- fwrite(ebuf, 1, len, fp_file);
- memcpy(ebuf,ebuf+len,diff);
- memcpy(ebuf+diff,buf,len);
- } else {
- if(j!=0) fwrite(ebuf, 1, es, fp_file);
- len-= es;
- memcpy(ebuf,buf+len,es);
- fwrite(buf, 1, len, fp_file);
- }
- j=1;
- }
-
- /* check final gzip block */
- {
- const uint8_t gzip1=ebuf[0];
- const uint8_t gzip2=ebuf[1];
- const uint32_t isize=*((uint32_t*)(ebuf+es-4));
- if(((gzip1!=GZIPID1) || (gzip2!=GZIPID2)) || (isize!=0)) {
- fprintf(stderr, "[%s] WARNING: Unexpected block structure in file '%s'.", __func__, fn[i]);
- fprintf(stderr, " Possible output corruption.\n");
- fwrite(ebuf, 1, es, fp_file);
- }
- }
- bam_header_destroy(old);
- bgzf_close(in);
- }
- free(buf);
- bgzf_close(fp);
- return 0;
-}
-
-
-
-int main_cat(int argc, char *argv[])
-{
- bam_header_t *h = 0;
- char *outfn = 0;
- int c, ret;
- while ((c = getopt(argc, argv, "h:o:")) >= 0) {
- switch (c) {
- case 'h': {
- tamFile fph = sam_open(optarg);
- if (fph == 0) {
- fprintf(stderr, "[%s] ERROR: fail to read the header from '%s'.\n", __func__, argv[1]);
- return 1;
- }
- h = sam_header_read(fph);
- sam_close(fph);
- break;
- }
- case 'o': outfn = strdup(optarg); break;
- }
- }
- if (argc - optind < 2) {
- fprintf(stderr, "Usage: samtools cat [-h header.sam] [-o out.bam] <in1.bam> <in2.bam> [...]\n");
- return 1;
- }
- ret = bam_cat(argc - optind, argv + optind, h, outfn? outfn : "-");
- free(outfn);
- return ret;
-}
diff --git a/samtools-0.1.19/bam_color.c b/samtools-0.1.19/bam_color.c
deleted file mode 100644
index 8b86e2f..0000000
--- a/samtools-0.1.19/bam_color.c
+++ /dev/null
@@ -1,145 +0,0 @@
-#include <ctype.h>
-#include "bam.h"
-
-/*!
- @abstract Get the color encoding the previous and current base
- @param b pointer to an alignment
- @param i The i-th position, 0-based
- @return color
-
- @discussion Returns 0 no color information is found.
- */
-char bam_aux_getCSi(bam1_t *b, int i)
-{
- uint8_t *c = bam_aux_get(b, "CS");
- char *cs = NULL;
-
- // return the base if the tag was not found
- if(0 == c) return 0;
-
- cs = bam_aux2Z(c);
- // adjust for strandedness and leading adaptor
- if(bam1_strand(b)) {
- i = strlen(cs) - 1 - i;
- // adjust for leading hard clip
- uint32_t cigar = bam1_cigar(b)[0];
- if((cigar & BAM_CIGAR_MASK) == BAM_CHARD_CLIP) {
- i -= cigar >> BAM_CIGAR_SHIFT;
- }
- } else { i++; }
- return cs[i];
-}
-
-/*!
- @abstract Get the color quality of the color encoding the previous and current base
- @param b pointer to an alignment
- @param i The i-th position, 0-based
- @return color quality
-
- @discussion Returns 0 no color information is found.
- */
-char bam_aux_getCQi(bam1_t *b, int i)
-{
- uint8_t *c = bam_aux_get(b, "CQ");
- char *cq = NULL;
-
- // return the base if the tag was not found
- if(0 == c) return 0;
-
- cq = bam_aux2Z(c);
- // adjust for strandedness
- if(bam1_strand(b)) {
- i = strlen(cq) - 1 - i;
- // adjust for leading hard clip
- uint32_t cigar = bam1_cigar(b)[0];
- if((cigar & BAM_CIGAR_MASK) == BAM_CHARD_CLIP) {
- i -= (cigar >> BAM_CIGAR_SHIFT);
- }
- }
- return cq[i];
-}
-
-char bam_aux_nt2int(char a)
-{
- switch(toupper(a)) {
- case 'A':
- return 0;
- break;
- case 'C':
- return 1;
- break;
- case 'G':
- return 2;
- break;
- case 'T':
- return 3;
- break;
- default:
- return 4;
- break;
- }
-}
-
-char bam_aux_ntnt2cs(char a, char b)
-{
- a = bam_aux_nt2int(a);
- b = bam_aux_nt2int(b);
- if(4 == a || 4 == b) return '4';
- return "0123"[(int)(a ^ b)];
-}
-
-/*!
- @abstract Get the color error profile at the give position
- @param b pointer to an alignment
- @return the original color if the color was an error, '-' (dash) otherwise
-
- @discussion Returns 0 no color information is found.
- */
-char bam_aux_getCEi(bam1_t *b, int i)
-{
- int cs_i;
- uint8_t *c = bam_aux_get(b, "CS");
- char *cs = NULL;
- char prev_b, cur_b;
- char cur_color, cor_color;
-
- // return the base if the tag was not found
- if(0 == c) return 0;
-
- cs = bam_aux2Z(c);
-
- // adjust for strandedness and leading adaptor
- if(bam1_strand(b)) { //reverse strand
- cs_i = strlen(cs) - 1 - i;
- // adjust for leading hard clip
- uint32_t cigar = bam1_cigar(b)[0];
- if((cigar & BAM_CIGAR_MASK) == BAM_CHARD_CLIP) {
- cs_i -= cigar >> BAM_CIGAR_SHIFT;
- }
- // get current color
- cur_color = cs[cs_i];
- // get previous base. Note: must rc adaptor
- prev_b = (cs_i == 1) ? "TGCAN"[(int)bam_aux_nt2int(cs[0])] : bam_nt16_rev_table[bam1_seqi(bam1_seq(b), i+1)];
- // get current base
- cur_b = bam_nt16_rev_table[bam1_seqi(bam1_seq(b), i)];
- }
- else {
- cs_i=i+1;
- // get current color
- cur_color = cs[cs_i];
- // get previous base
- prev_b = (0 == i) ? cs[0] : bam_nt16_rev_table[bam1_seqi(bam1_seq(b), i-1)];
- // get current base
- cur_b = bam_nt16_rev_table[bam1_seqi(bam1_seq(b), i)];
- }
-
- // corrected color
- cor_color = bam_aux_ntnt2cs(prev_b, cur_b);
-
- if(cur_color == cor_color) {
- return '-';
- }
- else {
- return cur_color;
- }
-}
diff --git a/samtools-0.1.19/bam_endian.h b/samtools-0.1.19/bam_endian.h
deleted file mode 100644
index 0fc74a8..0000000
--- a/samtools-0.1.19/bam_endian.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifndef BAM_ENDIAN_H
-#define BAM_ENDIAN_H
-
-#include <stdint.h>
-
-static inline int bam_is_big_endian()
-{
- long one= 1;
- return !(*((char *)(&one)));
-}
-static inline uint16_t bam_swap_endian_2(uint16_t v)
-{
- return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8));
-}
-static inline void *bam_swap_endian_2p(void *x)
-{
- *(uint16_t*)x = bam_swap_endian_2(*(uint16_t*)x);
- return x;
-}
-static inline uint32_t bam_swap_endian_4(uint32_t v)
-{
- v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
- return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
-}
-static inline void *bam_swap_endian_4p(void *x)
-{
- *(uint32_t*)x = bam_swap_endian_4(*(uint32_t*)x);
- return x;
-}
-static inline uint64_t bam_swap_endian_8(uint64_t v)
-{
- v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
- v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
- return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
-}
-static inline void *bam_swap_endian_8p(void *x)
-{
- *(uint64_t*)x = bam_swap_endian_8(*(uint64_t*)x);
- return x;
-}
-
-#endif
diff --git a/samtools-0.1.19/bam_import.c b/samtools-0.1.19/bam_import.c
deleted file mode 100644
index da2bf94..0000000
--- a/samtools-0.1.19/bam_import.c
+++ /dev/null
@@ -1,489 +0,0 @@
-#include <zlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <assert.h>
-#ifdef _WIN32
-#include <fcntl.h>
-#endif
-#include "kstring.h"
-#include "bam.h"
-#include "sam_header.h"
-#include "kseq.h"
-#include "khash.h"
-
-KSTREAM_INIT(gzFile, gzread, 16384)
-KHASH_MAP_INIT_STR(ref, uint64_t)
-
-void bam_init_header_hash(bam_header_t *header);
-void bam_destroy_header_hash(bam_header_t *header);
-int32_t bam_get_tid(const bam_header_t *header, const char *seq_name);
-
-unsigned char bam_nt16_table[256] = {
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 1, 2, 4, 8, 15,15,15,15, 15,15,15,15, 15, 0 /*=*/,15,15,
- 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,
- 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15,
- 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,
- 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
- 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15
-};
-
-unsigned short bam_char2flag_table[256] = {
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,BAM_FREAD1,BAM_FREAD2,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- BAM_FPROPER_PAIR,0,BAM_FMREVERSE,0, 0,BAM_FMUNMAP,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, BAM_FDUP,0,BAM_FQCFAIL,0, 0,0,0,0, 0,0,0,0,
- BAM_FPAIRED,0,BAM_FREVERSE,BAM_FSECONDARY, 0,BAM_FUNMAP,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
- 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
-};
-
-char *bam_nt16_rev_table = "=ACMGRSVTWYHKDBN";
-
-struct __tamFile_t {
- gzFile fp;
- kstream_t *ks;
- kstring_t *str;
- uint64_t n_lines;
- int is_first;
-};
-
-char **__bam_get_lines(const char *fn, int *_n) // for bam_plcmd.c only
-{
- char **list = 0, *s;
- int n = 0, dret, m = 0;
- gzFile fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r");
- kstream_t *ks;
- kstring_t *str;
- str = (kstring_t*)calloc(1, sizeof(kstring_t));
- ks = ks_init(fp);
- while (ks_getuntil(ks, '\n', str, &dret) > 0) {
- if (n == m) {
- m = m? m << 1 : 16;
- list = (char**)realloc(list, m * sizeof(char*));
- }
- if (str->s[str->l-1] == '\r')
- str->s[--str->l] = '\0';
- s = list[n++] = (char*)calloc(str->l + 1, 1);
- strcpy(s, str->s);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(str->s); free(str);
- *_n = n;
- return list;
-}
-
-static bam_header_t *hash2header(const kh_ref_t *hash)
-{
- bam_header_t *header;
- khiter_t k;
- header = bam_header_init();
- header->n_targets = kh_size(hash);
- header->target_name = (char**)calloc(kh_size(hash), sizeof(char*));
- header->target_len = (uint32_t*)calloc(kh_size(hash), 4);
- for (k = kh_begin(hash); k != kh_end(hash); ++k) {
- if (kh_exist(hash, k)) {
- int i = (int)kh_value(hash, k);
- header->target_name[i] = (char*)kh_key(hash, k);
- header->target_len[i] = kh_value(hash, k)>>32;
- }
- }
- bam_init_header_hash(header);
- return header;
-}
-bam_header_t *sam_header_read2(const char *fn)
-{
- bam_header_t *header;
- int c, dret, ret, error = 0;
- gzFile fp;
- kstream_t *ks;
- kstring_t *str;
- kh_ref_t *hash;
- khiter_t k;
- if (fn == 0) return 0;
- fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r");
- if (fp == 0) return 0;
- hash = kh_init(ref);
- ks = ks_init(fp);
- str = (kstring_t*)calloc(1, sizeof(kstring_t));
- while (ks_getuntil(ks, 0, str, &dret) > 0) {
- char *s = strdup(str->s);
- int len, i;
- i = kh_size(hash);
- ks_getuntil(ks, 0, str, &dret);
- len = atoi(str->s);
- k = kh_put(ref, hash, s, &ret);
- if (ret == 0) {
- fprintf(stderr, "[sam_header_read2] duplicated sequence name: %s\n", s);
- error = 1;
- }
- kh_value(hash, k) = (uint64_t)len<<32 | i;
- if (dret != '\n')
- while ((c = ks_getc(ks)) != '\n' && c != -1);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(str->s); free(str);
- fprintf(stderr, "[sam_header_read2] %d sequences loaded.\n", kh_size(hash));
- if (error) return 0;
- header = hash2header(hash);
- kh_destroy(ref, hash);
- return header;
-}
-static inline uint8_t *alloc_data(bam1_t *b, int size)
-{
- if (b->m_data < size) {
- b->m_data = size;
- kroundup32(b->m_data);
- b->data = (uint8_t*)realloc(b->data, b->m_data);
- }
- return b->data;
-}
-static inline void parse_error(int64_t n_lines, const char * __restrict msg)
-{
- fprintf(stderr, "Parse error at line %lld: %s\n", (long long)n_lines, msg);
- abort();
-}
-static inline void append_text(bam_header_t *header, kstring_t *str)
-{
- size_t x = header->l_text, y = header->l_text + str->l + 2; // 2 = 1 byte dret + 1 byte null
- kroundup32(x); kroundup32(y);
- if (x < y)
- {
- header->n_text = y;
- header->text = (char*)realloc(header->text, y);
- if ( !header->text )
- {
- fprintf(stderr,"realloc failed to alloc %ld bytes\n", y);
- abort();
- }
- }
- // Sanity check
- if ( header->l_text+str->l+1 >= header->n_text )
- {
- fprintf(stderr,"append_text FIXME: %ld>=%ld, x=%ld,y=%ld\n", header->l_text+str->l+1,(long)header->n_text,x,y);
- abort();
- }
- strncpy(header->text + header->l_text, str->s, str->l+1); // we cannot use strcpy() here.
- header->l_text += str->l + 1;
- header->text[header->l_text] = 0;
-}
-
-int sam_header_parse(bam_header_t *h)
-{
- char **tmp;
- int i;
- free(h->target_len); free(h->target_name);
- h->n_targets = 0; h->target_len = 0; h->target_name = 0;
- if (h->l_text < 3) return 0;
- if (h->dict == 0) h->dict = sam_header_parse2(h->text);
- tmp = sam_header2list(h->dict, "SQ", "SN", &h->n_targets);
- if (h->n_targets == 0) return 0;
- h->target_name = calloc(h->n_targets, sizeof(void*));
- for (i = 0; i < h->n_targets; ++i)
- h->target_name[i] = strdup(tmp[i]);
- free(tmp);
- tmp = sam_header2list(h->dict, "SQ", "LN", &h->n_targets);
- h->target_len = calloc(h->n_targets, 4);
- for (i = 0; i < h->n_targets; ++i)
- h->target_len[i] = atoi(tmp[i]);
- free(tmp);
- return h->n_targets;
-}
-
-bam_header_t *sam_header_read(tamFile fp)
-{
- int ret, dret;
- bam_header_t *header = bam_header_init();
- kstring_t *str = fp->str;
- while ((ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret)) >= 0 && str->s[0] == '@') { // skip header
- str->s[str->l] = dret; // note that str->s is NOT null terminated!!
- append_text(header, str);
- if (dret != '\n') {
- ret = ks_getuntil(fp->ks, '\n', str, &dret);
- str->s[str->l] = '\n'; // NOT null terminated!!
- append_text(header, str);
- }
- ++fp->n_lines;
- }
- sam_header_parse(header);
- bam_init_header_hash(header);
- fp->is_first = 1;
- return header;
-}
-
-int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b)
-{
- int ret, doff, doff0, dret, z = 0;
- bam1_core_t *c = &b->core;
- kstring_t *str = fp->str;
- kstream_t *ks = fp->ks;
-
- if (fp->is_first) {
- fp->is_first = 0;
- ret = str->l;
- } else {
- do { // special consideration for empty lines
- ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret);
- if (ret >= 0) z += str->l + 1;
- } while (ret == 0);
- }
- if (ret < 0) return -1;
- ++fp->n_lines;
- doff = 0;
-
- { // name
- c->l_qname = strlen(str->s) + 1;
- memcpy(alloc_data(b, doff + c->l_qname) + doff, str->s, c->l_qname);
- doff += c->l_qname;
- }
- { // flag
- long flag;
- char *s;
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
- flag = strtol((char*)str->s, &s, 0);
- if (*s) { // not the end of the string
- flag = 0;
- for (s = str->s; *s; ++s)
- flag |= bam_char2flag_table[(int)*s];
- }
- c->flag = flag;
- }
- { // tid, pos, qual
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->tid = bam_get_tid(header, str->s);
- if (c->tid < 0 && strcmp(str->s, "*")) {
- if (header->n_targets == 0) {
- fprintf(stderr, "[sam_read1] missing header? Abort!\n");
- exit(1);
- } else fprintf(stderr, "[sam_read1] reference '%s' is recognized as '*'.\n", str->s);
- }
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->pos = isdigit(str->s[0])? atoi(str->s) - 1 : -1;
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->qual = isdigit(str->s[0])? atoi(str->s) : 0;
- if (ret < 0) return -2;
- }
- { // cigar
- char *s, *t;
- int i, op;
- long x;
- c->n_cigar = 0;
- if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -3;
- z += str->l + 1;
- if (str->s[0] != '*') {
- uint32_t *cigar;
- for (s = str->s; *s; ++s) {
- if ((isalpha(*s)) || (*s=='=')) ++c->n_cigar;
- else if (!isdigit(*s)) parse_error(fp->n_lines, "invalid CIGAR character");
- }
- b->data = alloc_data(b, doff + c->n_cigar * 4);
- cigar = bam1_cigar(b);
- for (i = 0, s = str->s; i != c->n_cigar; ++i) {
- x = strtol(s, &t, 10);
- op = toupper(*t);
- if (op == 'M') op = BAM_CMATCH;
- else if (op == 'I') op = BAM_CINS;
- else if (op == 'D') op = BAM_CDEL;
- else if (op == 'N') op = BAM_CREF_SKIP;
- else if (op == 'S') op = BAM_CSOFT_CLIP;
- else if (op == 'H') op = BAM_CHARD_CLIP;
- else if (op == 'P') op = BAM_CPAD;
- else if (op == '=') op = BAM_CEQUAL;
- else if (op == 'X') op = BAM_CDIFF;
- else if (op == 'B') op = BAM_CBACK;
- else parse_error(fp->n_lines, "invalid CIGAR operation");
- s = t + 1;
- cigar[i] = bam_cigar_gen(x, op);
- }
- if (*s) parse_error(fp->n_lines, "unmatched CIGAR operation");
- c->bin = bam_reg2bin(c->pos, bam_calend(c, cigar));
- doff += c->n_cigar * 4;
- } else {
- if (!(c->flag&BAM_FUNMAP)) {
- fprintf(stderr, "Parse warning at line %lld: mapped sequence without CIGAR\n", (long long)fp->n_lines);
- c->flag |= BAM_FUNMAP;
- }
- c->bin = bam_reg2bin(c->pos, c->pos + 1);
- }
- }
- { // mtid, mpos, isize
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
- c->mtid = strcmp(str->s, "=")? bam_get_tid(header, str->s) : c->tid;
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
- c->mpos = isdigit(str->s[0])? atoi(str->s) - 1 : -1;
- ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
- c->isize = (str->s[0] == '-' || isdigit(str->s[0]))? atoi(str->s) : 0;
- if (ret < 0) return -4;
- }
- { // seq and qual
- int i;
- uint8_t *p = 0;
- if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -5; // seq
- z += str->l + 1;
- if (strcmp(str->s, "*")) {
- c->l_qseq = strlen(str->s);
- if (c->n_cigar && c->l_qseq != (int32_t)bam_cigar2qlen(c, bam1_cigar(b))) {
- fprintf(stderr, "Line %ld, sequence length %i vs %i from CIGAR\n",
- (long)fp->n_lines, c->l_qseq, (int32_t)bam_cigar2qlen(c, bam1_cigar(b)));
- parse_error(fp->n_lines, "CIGAR and sequence length are inconsistent");
- }
- p = (uint8_t*)alloc_data(b, doff + c->l_qseq + (c->l_qseq+1)/2) + doff;
- memset(p, 0, (c->l_qseq+1)/2);
- for (i = 0; i < c->l_qseq; ++i)
- p[i/2] |= bam_nt16_table[(int)str->s[i]] << 4*(1-i%2);
- } else c->l_qseq = 0;
- if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -6; // qual
- z += str->l + 1;
- if (strcmp(str->s, "*") && c->l_qseq != strlen(str->s))
- parse_error(fp->n_lines, "sequence and quality are inconsistent");
- p += (c->l_qseq+1)/2;
- if (strcmp(str->s, "*") == 0) for (i = 0; i < c->l_qseq; ++i) p[i] = 0xff;
- else for (i = 0; i < c->l_qseq; ++i) p[i] = str->s[i] - 33;
- doff += c->l_qseq + (c->l_qseq+1)/2;
- }
- doff0 = doff;
- if (dret != '\n' && dret != '\r') { // aux
- while (ks_getuntil(ks, KS_SEP_TAB, str, &dret) >= 0) {
- uint8_t *s, type, key[2];
- z += str->l + 1;
- if (str->l < 6 || str->s[2] != ':' || str->s[4] != ':')
- parse_error(fp->n_lines, "missing colon in auxiliary data");
- key[0] = str->s[0]; key[1] = str->s[1];
- type = str->s[3];
- s = alloc_data(b, doff + 3) + doff;
- s[0] = key[0]; s[1] = key[1]; s += 2; doff += 2;
- if (type == 'A' || type == 'a' || type == 'c' || type == 'C') { // c and C for backward compatibility
- s = alloc_data(b, doff + 2) + doff;
- *s++ = 'A'; *s = str->s[5];
- doff += 2;
- } else if (type == 'I' || type == 'i') {
- long long x;
- s = alloc_data(b, doff + 5) + doff;
- x = (long long)atoll(str->s + 5);
- if (x < 0) {
- if (x >= -127) {
- *s++ = 'c'; *(int8_t*)s = (int8_t)x;
- s += 1; doff += 2;
- } else if (x >= -32767) {
- *s++ = 's'; *(int16_t*)s = (int16_t)x;
- s += 2; doff += 3;
- } else {
- *s++ = 'i'; *(int32_t*)s = (int32_t)x;
- s += 4; doff += 5;
- if (x < -2147483648ll)
- fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.",
- (long long)fp->n_lines, x);
- }
- } else {
- if (x <= 255) {
- *s++ = 'C'; *s++ = (uint8_t)x;
- doff += 2;
- } else if (x <= 65535) {
- *s++ = 'S'; *(uint16_t*)s = (uint16_t)x;
- s += 2; doff += 3;
- } else {
- *s++ = 'I'; *(uint32_t*)s = (uint32_t)x;
- s += 4; doff += 5;
- if (x > 4294967295ll)
- fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.",
- (long long)fp->n_lines, x);
- }
- }
- } else if (type == 'f') {
- s = alloc_data(b, doff + 5) + doff;
- *s++ = 'f';
- *(float*)s = (float)atof(str->s + 5);
- s += 4; doff += 5;
- } else if (type == 'd') {
- s = alloc_data(b, doff + 9) + doff;
- *s++ = 'd';
- *(float*)s = (float)atof(str->s + 9);
- s += 8; doff += 9;
- } else if (type == 'Z' || type == 'H') {
- int size = 1 + (str->l - 5) + 1;
- if (type == 'H') { // check whether the hex string is valid
- int i;
- if ((str->l - 5) % 2 == 1) parse_error(fp->n_lines, "length of the hex string not even");
- for (i = 0; i < str->l - 5; ++i) {
- int c = toupper(str->s[5 + i]);
- if (!((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F')))
- parse_error(fp->n_lines, "invalid hex character");
- }
- }
- s = alloc_data(b, doff + size) + doff;
- *s++ = type;
- memcpy(s, str->s + 5, str->l - 5);
- s[str->l - 5] = 0;
- doff += size;
- } else if (type == 'B') {
- int32_t n = 0, Bsize, k = 0, size;
- char *p;
- if (str->l < 8) parse_error(fp->n_lines, "too few values in aux type B");
- Bsize = bam_aux_type2size(str->s[5]); // the size of each element
- for (p = (char*)str->s + 6; *p; ++p) // count the number of elements in the array
- if (*p == ',') ++n;
- p = str->s + 7; // now p points to the first number in the array
- size = 6 + Bsize * n; // total number of bytes allocated to this tag
- s = alloc_data(b, doff + 6 * Bsize * n) + doff; // allocate memory
- *s++ = 'B'; *s++ = str->s[5];
- memcpy(s, &n, 4); s += 4; // write the number of elements
- if (str->s[5] == 'c') while (p < str->s + str->l) ((int8_t*)s)[k++] = (int8_t)strtol(p, &p, 0), ++p;
- else if (str->s[5] == 'C') while (p < str->s + str->l) ((uint8_t*)s)[k++] = (uint8_t)strtol(p, &p, 0), ++p;
- else if (str->s[5] == 's') while (p < str->s + str->l) ((int16_t*)s)[k++] = (int16_t)strtol(p, &p, 0), ++p; // FIXME: avoid unaligned memory
- else if (str->s[5] == 'S') while (p < str->s + str->l) ((uint16_t*)s)[k++] = (uint16_t)strtol(p, &p, 0), ++p;
- else if (str->s[5] == 'i') while (p < str->s + str->l) ((int32_t*)s)[k++] = (int32_t)strtol(p, &p, 0), ++p;
- else if (str->s[5] == 'I') while (p < str->s + str->l) ((uint32_t*)s)[k++] = (uint32_t)strtol(p, &p, 0), ++p;
- else if (str->s[5] == 'f') while (p < str->s + str->l) ((float*)s)[k++] = (float)strtod(p, &p), ++p;
- else parse_error(fp->n_lines, "unrecognized array type");
- s += Bsize * n; doff += size;
- } else parse_error(fp->n_lines, "unrecognized type");
- if (dret == '\n' || dret == '\r') break;
- }
- }
- b->l_aux = doff - doff0;
- b->data_len = doff;
- if (bam_no_B) bam_remove_B(b);
- return z;
-}
-
-tamFile sam_open(const char *fn)
-{
- tamFile fp;
- gzFile gzfp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "rb") : gzopen(fn, "rb");
- if (gzfp == 0) return 0;
- fp = (tamFile)calloc(1, sizeof(struct __tamFile_t));
- fp->str = (kstring_t*)calloc(1, sizeof(kstring_t));
- fp->fp = gzfp;
- fp->ks = ks_init(fp->fp);
- return fp;
-}
-
-void sam_close(tamFile fp)
-{
- if (fp) {
- ks_destroy(fp->ks);
- gzclose(fp->fp);
- free(fp->str->s); free(fp->str);
- free(fp);
- }
-}
diff --git a/samtools-0.1.19/bam_index.c b/samtools-0.1.19/bam_index.c
deleted file mode 100644
index f916e04..0000000
--- a/samtools-0.1.19/bam_index.c
+++ /dev/null
@@ -1,726 +0,0 @@
-#include <ctype.h>
-#include <assert.h>
-#include "bam.h"
-#include "khash.h"
-#include "ksort.h"
-#include "bam_endian.h"
-#ifdef _USE_KNETFILE
-#include "knetfile.h"
-#endif
-
-/*!
- @header
-
- Alignment indexing. Before indexing, BAM must be sorted based on the
- leftmost coordinate of alignments. In indexing, BAM uses two indices:
- a UCSC binning index and a simple linear index. The binning index is
- efficient for alignments spanning long distance, while the auxiliary
- linear index helps to reduce unnecessary seek calls especially for
- short alignments.
-
- The UCSC binning scheme was suggested by Richard Durbin and Lincoln
- Stein and is explained by Kent et al. (2002). In this scheme, each bin
- represents a contiguous genomic region which can be fully contained in
- another bin; each alignment is associated with a bin which represents
- the smallest region containing the entire alignment. The binning
- scheme is essentially another representation of R-tree. A distinct bin
- uniquely corresponds to a distinct internal node in a R-tree. Bin A is
- a child of Bin B if region A is contained in B.
-
- In BAM, each bin may span 2^29, 2^26, 2^23, 2^20, 2^17 or 2^14 bp. Bin
- 0 spans a 512Mbp region, bins 1-8 span 64Mbp, 9-72 8Mbp, 73-584 1Mbp,
- 585-4680 128Kbp and bins 4681-37449 span 16Kbp regions. If we want to
- find the alignments overlapped with a region [rbeg,rend), we need to
- calculate the list of bins that may be overlapped the region and test
- the alignments in the bins to confirm the overlaps. If the specified
- region is short, typically only a few alignments in six bins need to
- be retrieved. The overlapping alignments can be quickly fetched.
-
- */
-
-#define BAM_MIN_CHUNK_GAP 32768
-// 1<<14 is the size of minimum bin.
-#define BAM_LIDX_SHIFT 14
-
-#define BAM_MAX_BIN 37450 // =(8^6-1)/7+1
-
-typedef struct {
- uint64_t u, v;
-} pair64_t;
-
-#define pair64_lt(a,b) ((a).u < (b).u)
-KSORT_INIT(off, pair64_t, pair64_lt)
-
-typedef struct {
- uint32_t m, n;
- pair64_t *list;
-} bam_binlist_t;
-
-typedef struct {
- int32_t n, m;
- uint64_t *offset;
-} bam_lidx_t;
-
-KHASH_MAP_INIT_INT(i, bam_binlist_t)
-
-struct __bam_index_t {
- int32_t n;
- uint64_t n_no_coor; // unmapped reads without coordinate
- khash_t(i) **index;
- bam_lidx_t *index2;
-};
-
-// requirement: len <= LEN_MASK
-static inline void insert_offset(khash_t(i) *h, int bin, uint64_t beg, uint64_t end)
-{
- khint_t k;
- bam_binlist_t *l;
- int ret;
- k = kh_put(i, h, bin, &ret);
- l = &kh_value(h, k);
- if (ret) { // not present
- l->m = 1; l->n = 0;
- l->list = (pair64_t*)calloc(l->m, 16);
- }
- if (l->n == l->m) {
- l->m <<= 1;
- l->list = (pair64_t*)realloc(l->list, l->m * 16);
- }
- l->list[l->n].u = beg; l->list[l->n++].v = end;
-}
-
-static inline void insert_offset2(bam_lidx_t *index2, bam1_t *b, uint64_t offset)
-{
- int i, beg, end;
- beg = b->core.pos >> BAM_LIDX_SHIFT;
- end = (bam_calend(&b->core, bam1_cigar(b)) - 1) >> BAM_LIDX_SHIFT;
- if (index2->m < end + 1) {
- int old_m = index2->m;
- index2->m = end + 1;
- kroundup32(index2->m);
- index2->offset = (uint64_t*)realloc(index2->offset, index2->m * 8);
- memset(index2->offset + old_m, 0, 8 * (index2->m - old_m));
- }
- if (beg == end) {
- if (index2->offset[beg] == 0) index2->offset[beg] = offset;
- } else {
- for (i = beg; i <= end; ++i)
- if (index2->offset[i] == 0) index2->offset[i] = offset;
- }
- index2->n = end + 1;
-}
-
-static void merge_chunks(bam_index_t *idx)
-{
-#if defined(BAM_TRUE_OFFSET) || defined(BAM_VIRTUAL_OFFSET16)
- khash_t(i) *index;
- int i, l, m;
- khint_t k;
- for (i = 0; i < idx->n; ++i) {
- index = idx->index[i];
- for (k = kh_begin(index); k != kh_end(index); ++k) {
- bam_binlist_t *p;
- if (!kh_exist(index, k) || kh_key(index, k) == BAM_MAX_BIN) continue;
- p = &kh_value(index, k);
- m = 0;
- for (l = 1; l < p->n; ++l) {
-#ifdef BAM_TRUE_OFFSET
- if (p->list[m].v + BAM_MIN_CHUNK_GAP > p->list[l].u) p->list[m].v = p->list[l].v;
-#else
- if (p->list[m].v>>16 == p->list[l].u>>16) p->list[m].v = p->list[l].v;
-#endif
- else p->list[++m] = p->list[l];
- } // ~for(l)
- p->n = m + 1;
- } // ~for(k)
- } // ~for(i)
-#endif // defined(BAM_TRUE_OFFSET) || defined(BAM_BGZF)
-}
-
-static void fill_missing(bam_index_t *idx)
-{
- int i, j;
- for (i = 0; i < idx->n; ++i) {
- bam_lidx_t *idx2 = &idx->index2[i];
- for (j = 1; j < idx2->n; ++j)
- if (idx2->offset[j] == 0)
- idx2->offset[j] = idx2->offset[j-1];
- }
-}
-
-bam_index_t *bam_index_core(bamFile fp)
-{
- bam1_t *b;
- bam_header_t *h;
- int i, ret;
- bam_index_t *idx;
- uint32_t last_bin, save_bin;
- int32_t last_coor, last_tid, save_tid;
- bam1_core_t *c;
- uint64_t save_off, last_off, n_mapped, n_unmapped, off_beg, off_end, n_no_coor;
-
- h = bam_header_read(fp);
- if(h == 0) {
- fprintf(stderr, "[bam_index_core] Invalid BAM header.");
- return NULL;
- }
-
- idx = (bam_index_t*)calloc(1, sizeof(bam_index_t));
- b = (bam1_t*)calloc(1, sizeof(bam1_t));
- c = &b->core;
-
- idx->n = h->n_targets;
- bam_header_destroy(h);
- idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*));
- for (i = 0; i < idx->n; ++i) idx->index[i] = kh_init(i);
- idx->index2 = (bam_lidx_t*)calloc(idx->n, sizeof(bam_lidx_t));
-
- save_bin = save_tid = last_tid = last_bin = 0xffffffffu;
- save_off = last_off = bam_tell(fp); last_coor = 0xffffffffu;
- n_mapped = n_unmapped = n_no_coor = off_end = 0;
- off_beg = off_end = bam_tell(fp);
- while ((ret = bam_read1(fp, b)) >= 0) {
- if (c->tid < 0) ++n_no_coor;
- if (last_tid < c->tid || (last_tid >= 0 && c->tid < 0)) { // change of chromosomes
- last_tid = c->tid;
- last_bin = 0xffffffffu;
- } else if ((uint32_t)last_tid > (uint32_t)c->tid) {
- fprintf(stderr, "[bam_index_core] the alignment is not sorted (%s): %d-th chr > %d-th chr\n",
- bam1_qname(b), last_tid+1, c->tid+1);
- return NULL;
- } else if ((int32_t)c->tid >= 0 && last_coor > c->pos) {
- fprintf(stderr, "[bam_index_core] the alignment is not sorted (%s): %u > %u in %d-th chr\n",
- bam1_qname(b), last_coor, c->pos, c->tid+1);
- return NULL;
- }
- if (c->tid >= 0 && !(c->flag & BAM_FUNMAP)) insert_offset2(&idx->index2[b->core.tid], b, last_off);
- if (c->bin != last_bin) { // then possibly write the binning index
- if (save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record
- insert_offset(idx->index[save_tid], save_bin, save_off, last_off);
- if (last_bin == 0xffffffffu && save_tid != 0xffffffffu) { // write the meta element
- off_end = last_off;
- insert_offset(idx->index[save_tid], BAM_MAX_BIN, off_beg, off_end);
- insert_offset(idx->index[save_tid], BAM_MAX_BIN, n_mapped, n_unmapped);
- n_mapped = n_unmapped = 0;
- off_beg = off_end;
- }
- save_off = last_off;
- save_bin = last_bin = c->bin;
- save_tid = c->tid;
- if (save_tid < 0) break;
- }
- if (bam_tell(fp) <= last_off) {
- fprintf(stderr, "[bam_index_core] bug in BGZF/RAZF: %llx < %llx\n",
- (unsigned long long)bam_tell(fp), (unsigned long long)last_off);
- return NULL;
- }
- if (c->flag & BAM_FUNMAP) ++n_unmapped;
- else ++n_mapped;
- last_off = bam_tell(fp);
- last_coor = b->core.pos;
- }
- if (save_tid >= 0) {
- insert_offset(idx->index[save_tid], save_bin, save_off, bam_tell(fp));
- insert_offset(idx->index[save_tid], BAM_MAX_BIN, off_beg, bam_tell(fp));
- insert_offset(idx->index[save_tid], BAM_MAX_BIN, n_mapped, n_unmapped);
- }
- merge_chunks(idx);
- fill_missing(idx);
- if (ret >= 0) {
- while ((ret = bam_read1(fp, b)) >= 0) {
- ++n_no_coor;
- if (c->tid >= 0 && n_no_coor) {
- fprintf(stderr, "[bam_index_core] the alignment is not sorted: reads without coordinates prior to reads with coordinates.\n");
- return NULL;
- }
- }
- }
- if (ret < -1) fprintf(stderr, "[bam_index_core] truncated file? Continue anyway. (%d)\n", ret);
- free(b->data); free(b);
- idx->n_no_coor = n_no_coor;
- return idx;
-}
-
-void bam_index_destroy(bam_index_t *idx)
-{
- khint_t k;
- int i;
- if (idx == 0) return;
- for (i = 0; i < idx->n; ++i) {
- khash_t(i) *index = idx->index[i];
- bam_lidx_t *index2 = idx->index2 + i;
- for (k = kh_begin(index); k != kh_end(index); ++k) {
- if (kh_exist(index, k))
- free(kh_value(index, k).list);
- }
- kh_destroy(i, index);
- free(index2->offset);
- }
- free(idx->index); free(idx->index2);
- free(idx);
-}
-
-void bam_index_save(const bam_index_t *idx, FILE *fp)
-{
- int32_t i, size;
- khint_t k;
- fwrite("BAI\1", 1, 4, fp);
- if (bam_is_be) {
- uint32_t x = idx->n;
- fwrite(bam_swap_endian_4p(&x), 4, 1, fp);
- } else fwrite(&idx->n, 4, 1, fp);
- for (i = 0; i < idx->n; ++i) {
- khash_t(i) *index = idx->index[i];
- bam_lidx_t *index2 = idx->index2 + i;
- // write binning index
- size = kh_size(index);
- if (bam_is_be) { // big endian
- uint32_t x = size;
- fwrite(bam_swap_endian_4p(&x), 4, 1, fp);
- } else fwrite(&size, 4, 1, fp);
- for (k = kh_begin(index); k != kh_end(index); ++k) {
- if (kh_exist(index, k)) {
- bam_binlist_t *p = &kh_value(index, k);
- if (bam_is_be) { // big endian
- uint32_t x;
- x = kh_key(index, k); fwrite(bam_swap_endian_4p(&x), 4, 1, fp);
- x = p->n; fwrite(bam_swap_endian_4p(&x), 4, 1, fp);
- for (x = 0; (int)x < p->n; ++x) {
- bam_swap_endian_8p(&p->list[x].u);
- bam_swap_endian_8p(&p->list[x].v);
- }
- fwrite(p->list, 16, p->n, fp);
- for (x = 0; (int)x < p->n; ++x) {
- bam_swap_endian_8p(&p->list[x].u);
- bam_swap_endian_8p(&p->list[x].v);
- }
- } else {
- fwrite(&kh_key(index, k), 4, 1, fp);
- fwrite(&p->n, 4, 1, fp);
- fwrite(p->list, 16, p->n, fp);
- }
- }
- }
- // write linear index (index2)
- if (bam_is_be) {
- int x = index2->n;
- fwrite(bam_swap_endian_4p(&x), 4, 1, fp);
- } else fwrite(&index2->n, 4, 1, fp);
- if (bam_is_be) { // big endian
- int x;
- for (x = 0; (int)x < index2->n; ++x)
- bam_swap_endian_8p(&index2->offset[x]);
- fwrite(index2->offset, 8, index2->n, fp);
- for (x = 0; (int)x < index2->n; ++x)
- bam_swap_endian_8p(&index2->offset[x]);
- } else fwrite(index2->offset, 8, index2->n, fp);
- }
- { // write the number of reads coor-less records.
- uint64_t x = idx->n_no_coor;
- if (bam_is_be) bam_swap_endian_8p(&x);
- fwrite(&x, 8, 1, fp);
- }
- fflush(fp);
-}
-
-static bam_index_t *bam_index_load_core(FILE *fp)
-{
- int i;
- char magic[4];
- bam_index_t *idx;
- if (fp == 0) {
- fprintf(stderr, "[bam_index_load_core] fail to load index.\n");
- return 0;
- }
- fread(magic, 1, 4, fp);
- if (strncmp(magic, "BAI\1", 4)) {
- fprintf(stderr, "[bam_index_load] wrong magic number.\n");
- fclose(fp);
- return 0;
- }
- idx = (bam_index_t*)calloc(1, sizeof(bam_index_t));
- fread(&idx->n, 4, 1, fp);
- if (bam_is_be) bam_swap_endian_4p(&idx->n);
- idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*));
- idx->index2 = (bam_lidx_t*)calloc(idx->n, sizeof(bam_lidx_t));
- for (i = 0; i < idx->n; ++i) {
- khash_t(i) *index;
- bam_lidx_t *index2 = idx->index2 + i;
- uint32_t key, size;
- khint_t k;
- int j, ret;
- bam_binlist_t *p;
- index = idx->index[i] = kh_init(i);
- // load binning index
- fread(&size, 4, 1, fp);
- if (bam_is_be) bam_swap_endian_4p(&size);
- for (j = 0; j < (int)size; ++j) {
- fread(&key, 4, 1, fp);
- if (bam_is_be) bam_swap_endian_4p(&key);
- k = kh_put(i, index, key, &ret);
- p = &kh_value(index, k);
- fread(&p->n, 4, 1, fp);
- if (bam_is_be) bam_swap_endian_4p(&p->n);
- p->m = p->n;
- p->list = (pair64_t*)malloc(p->m * 16);
- fread(p->list, 16, p->n, fp);
- if (bam_is_be) {
- int x;
- for (x = 0; x < p->n; ++x) {
- bam_swap_endian_8p(&p->list[x].u);
- bam_swap_endian_8p(&p->list[x].v);
- }
- }
- }
- // load linear index
- fread(&index2->n, 4, 1, fp);
- if (bam_is_be) bam_swap_endian_4p(&index2->n);
- index2->m = index2->n;
- index2->offset = (uint64_t*)calloc(index2->m, 8);
- fread(index2->offset, index2->n, 8, fp);
- if (bam_is_be)
- for (j = 0; j < index2->n; ++j) bam_swap_endian_8p(&index2->offset[j]);
- }
- if (fread(&idx->n_no_coor, 8, 1, fp) == 0) idx->n_no_coor = 0;
- if (bam_is_be) bam_swap_endian_8p(&idx->n_no_coor);
- return idx;
-}
-
-bam_index_t *bam_index_load_local(const char *_fn)
-{
- FILE *fp;
- char *fnidx, *fn;
-
- if (strstr(_fn, "ftp://") == _fn || strstr(_fn, "http://") == _fn) {
- const char *p;
- int l = strlen(_fn);
- for (p = _fn + l - 1; p >= _fn; --p)
- if (*p == '/') break;
- fn = strdup(p + 1);
- } else fn = strdup(_fn);
- fnidx = (char*)calloc(strlen(fn) + 5, 1);
- strcpy(fnidx, fn); strcat(fnidx, ".bai");
- fp = fopen(fnidx, "rb");
- if (fp == 0) { // try "{base}.bai"
- char *s = strstr(fn, "bam");
- if (s == fn + strlen(fn) - 3) {
- strcpy(fnidx, fn);
- fnidx[strlen(fn)-1] = 'i';
- fp = fopen(fnidx, "rb");
- }
- }
- free(fnidx); free(fn);
- if (fp) {
- bam_index_t *idx = bam_index_load_core(fp);
- fclose(fp);
- return idx;
- } else return 0;
-}
-
-#ifdef _USE_KNETFILE
-static void download_from_remote(const char *url)
-{
- const int buf_size = 1 * 1024 * 1024;
- char *fn;
- FILE *fp;
- uint8_t *buf;
- knetFile *fp_remote;
- int l;
- if (strstr(url, "ftp://") != url && strstr(url, "http://") != url) return;
- l = strlen(url);
- for (fn = (char*)url + l - 1; fn >= url; --fn)
- if (*fn == '/') break;
- ++fn; // fn now points to the file name
- fp_remote = knet_open(url, "r");
- if (fp_remote == 0) {
- fprintf(stderr, "[download_from_remote] fail to open remote file.\n");
- return;
- }
- if ((fp = fopen(fn, "wb")) == 0) {
- fprintf(stderr, "[download_from_remote] fail to create file in the working directory.\n");
- knet_close(fp_remote);
- return;
- }
- buf = (uint8_t*)calloc(buf_size, 1);
- while ((l = knet_read(fp_remote, buf, buf_size)) != 0)
- fwrite(buf, 1, l, fp);
- free(buf);
- fclose(fp);
- knet_close(fp_remote);
-}
-#else
-static void download_from_remote(const char *url)
-{
- return;
-}
-#endif
-
-bam_index_t *bam_index_load(const char *fn)
-{
- bam_index_t *idx;
- idx = bam_index_load_local(fn);
- if (idx == 0 && (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn)) {
- char *fnidx = calloc(strlen(fn) + 5, 1);
- strcat(strcpy(fnidx, fn), ".bai");
- fprintf(stderr, "[bam_index_load] attempting to download the remote index file.\n");
- download_from_remote(fnidx);
- free(fnidx);
- idx = bam_index_load_local(fn);
- }
- if (idx == 0) fprintf(stderr, "[bam_index_load] fail to load BAM index.\n");
- return idx;
-}
-
-int bam_index_build2(const char *fn, const char *_fnidx)
-{
- char *fnidx;
- FILE *fpidx;
- bamFile fp;
- bam_index_t *idx;
- if ((fp = bam_open(fn, "r")) == 0) {
- fprintf(stderr, "[bam_index_build2] fail to open the BAM file.\n");
- return -1;
- }
- idx = bam_index_core(fp);
- bam_close(fp);
- if(idx == 0) {
- fprintf(stderr, "[bam_index_build2] fail to index the BAM file.\n");
- return -1;
- }
- if (_fnidx == 0) {
- fnidx = (char*)calloc(strlen(fn) + 5, 1);
- strcpy(fnidx, fn); strcat(fnidx, ".bai");
- } else fnidx = strdup(_fnidx);
- fpidx = fopen(fnidx, "wb");
- if (fpidx == 0) {
- fprintf(stderr, "[bam_index_build2] fail to create the index file.\n");
- free(fnidx);
- bam_index_destroy(idx);
- return -1;
- }
- bam_index_save(idx, fpidx);
- bam_index_destroy(idx);
- fclose(fpidx);
- free(fnidx);
- return 0;
-}
-
-int bam_index_build(const char *fn)
-{
- return bam_index_build2(fn, 0);
-}
-
-int bam_index(int argc, char *argv[])
-{
- if (argc < 2) {
- fprintf(stderr, "Usage: samtools index <in.bam> [out.index]\n");
- return 1;
- }
- if (argc >= 3) bam_index_build2(argv[1], argv[2]);
- else bam_index_build(argv[1]);
- return 0;
-}
-
-int bam_idxstats(int argc, char *argv[])
-{
- bam_index_t *idx;
- bam_header_t *header;
- bamFile fp;
- int i;
- if (argc < 2) {
- fprintf(stderr, "Usage: samtools idxstats <in.bam>\n");
- return 1;
- }
- fp = bam_open(argv[1], "r");
- if (fp == 0) { fprintf(stderr, "[%s] fail to open BAM.\n", __func__); return 1; }
- header = bam_header_read(fp);
- bam_close(fp);
- idx = bam_index_load(argv[1]);
- if (idx == 0) { fprintf(stderr, "[%s] fail to load the index.\n", __func__); return 1; }
- for (i = 0; i < idx->n; ++i) {
- khint_t k;
- khash_t(i) *h = idx->index[i];
- printf("%s\t%d", header->target_name[i], header->target_len[i]);
- k = kh_get(i, h, BAM_MAX_BIN);
- if (k != kh_end(h))
- printf("\t%llu\t%llu", (long long)kh_val(h, k).list[1].u, (long long)kh_val(h, k).list[1].v);
- else printf("\t0\t0");
- putchar('\n');
- }
- printf("*\t0\t0\t%llu\n", (long long)idx->n_no_coor);
- bam_header_destroy(header);
- bam_index_destroy(idx);
- return 0;
-}
-
-static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[BAM_MAX_BIN])
-{
- int i = 0, k;
- if (beg >= end) return 0;
- if (end >= 1u<<29) end = 1u<<29;
- --end;
- list[i++] = 0;
- for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list[i++] = k;
- for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list[i++] = k;
- for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list[i++] = k;
- for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list[i++] = k;
- for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list[i++] = k;
- return i;
-}
-
-static inline int is_overlap(uint32_t beg, uint32_t end, const bam1_t *b)
-{
- uint32_t rbeg = b->core.pos;
- uint32_t rend = b->core.n_cigar? bam_calend(&b->core, bam1_cigar(b)) : b->core.pos + 1;
- return (rend > beg && rbeg < end);
-}
-
-struct __bam_iter_t {
- int from_first; // read from the first record; no random access
- int tid, beg, end, n_off, i, finished;
- uint64_t curr_off;
- pair64_t *off;
-};
-
-// bam_fetch helper function retrieves
-bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end)
-{
- uint16_t *bins;
- int i, n_bins, n_off;
- pair64_t *off;
- khint_t k;
- khash_t(i) *index;
- uint64_t min_off;
- bam_iter_t iter = 0;
-
- if (beg < 0) beg = 0;
- if (end < beg) return 0;
- // initialize iter
- iter = calloc(1, sizeof(struct __bam_iter_t));
- iter->tid = tid, iter->beg = beg, iter->end = end; iter->i = -1;
- //
- bins = (uint16_t*)calloc(BAM_MAX_BIN, 2);
- n_bins = reg2bins(beg, end, bins);
- index = idx->index[tid];
- if (idx->index2[tid].n > 0) {
- min_off = (beg>>BAM_LIDX_SHIFT >= idx->index2[tid].n)? idx->index2[tid].offset[idx->index2[tid].n-1]
- : idx->index2[tid].offset[beg>>BAM_LIDX_SHIFT];
- if (min_off == 0) { // improvement for index files built by tabix prior to 0.1.4
- int n = beg>>BAM_LIDX_SHIFT;
- if (n > idx->index2[tid].n) n = idx->index2[tid].n;
- for (i = n - 1; i >= 0; --i)
- if (idx->index2[tid].offset[i] != 0) break;
- if (i >= 0) min_off = idx->index2[tid].offset[i];
- }
- } else min_off = 0; // tabix 0.1.2 may produce such index files
- for (i = n_off = 0; i < n_bins; ++i) {
- if ((k = kh_get(i, index, bins[i])) != kh_end(index))
- n_off += kh_value(index, k).n;
- }
- if (n_off == 0) {
- free(bins); return iter;
- }
- off = (pair64_t*)calloc(n_off, 16);
- for (i = n_off = 0; i < n_bins; ++i) {
- if ((k = kh_get(i, index, bins[i])) != kh_end(index)) {
- int j;
- bam_binlist_t *p = &kh_value(index, k);
- for (j = 0; j < p->n; ++j)
- if (p->list[j].v > min_off) off[n_off++] = p->list[j];
- }
- }
- free(bins);
- if (n_off == 0) {
- free(off); return iter;
- }
- {
- bam1_t *b = (bam1_t*)calloc(1, sizeof(bam1_t));
- int l;
- ks_introsort(off, n_off, off);
- // resolve completely contained adjacent blocks
- for (i = 1, l = 0; i < n_off; ++i)
- if (off[l].v < off[i].v)
- off[++l] = off[i];
- n_off = l + 1;
- // resolve overlaps between adjacent blocks; this may happen due to the merge in indexing
- for (i = 1; i < n_off; ++i)
- if (off[i-1].v >= off[i].u) off[i-1].v = off[i].u;
- { // merge adjacent blocks
-#if defined(BAM_TRUE_OFFSET) || defined(BAM_VIRTUAL_OFFSET16)
- for (i = 1, l = 0; i < n_off; ++i) {
-#ifdef BAM_TRUE_OFFSET
- if (off[l].v + BAM_MIN_CHUNK_GAP > off[i].u) off[l].v = off[i].v;
-#else
- if (off[l].v>>16 == off[i].u>>16) off[l].v = off[i].v;
-#endif
- else off[++l] = off[i];
- }
- n_off = l + 1;
-#endif
- }
- bam_destroy1(b);
- }
- iter->n_off = n_off; iter->off = off;
- return iter;
-}
-
-pair64_t *get_chunk_coordinates(const bam_index_t *idx, int tid, int beg, int end, int *cnt_off)
-{ // for pysam compatibility
- bam_iter_t iter;
- pair64_t *off;
- iter = bam_iter_query(idx, tid, beg, end);
- off = iter->off; *cnt_off = iter->n_off;
- free(iter);
- return off;
-}
-
-void bam_iter_destroy(bam_iter_t iter)
-{
- if (iter) { free(iter->off); free(iter); }
-}
-
-int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b)
-{
- int ret;
- if (iter && iter->finished) return -1;
- if (iter == 0 || iter->from_first) {
- ret = bam_read1(fp, b);
- if (ret < 0 && iter) iter->finished = 1;
- return ret;
- }
- if (iter->off == 0) return -1;
- for (;;) {
- if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk
- if (iter->i == iter->n_off - 1) { ret = -1; break; } // no more chunks
- if (iter->i >= 0) assert(iter->curr_off == iter->off[iter->i].v); // otherwise bug
- if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek
- bam_seek(fp, iter->off[iter->i+1].u, SEEK_SET);
- iter->curr_off = bam_tell(fp);
- }
- ++iter->i;
- }
- if ((ret = bam_read1(fp, b)) >= 0) {
- iter->curr_off = bam_tell(fp);
- if (b->core.tid != iter->tid || b->core.pos >= iter->end) { // no need to proceed
- ret = bam_validate1(NULL, b)? -1 : -5; // determine whether end of region or error
- break;
- }
- else if (is_overlap(iter->beg, iter->end, b)) return ret;
- } else break; // end of file or error
- }
- iter->finished = 1;
- return ret;
-}
-
-int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func)
-{
- int ret;
- bam_iter_t iter;
- bam1_t *b;
- b = bam_init1();
- iter = bam_iter_query(idx, tid, beg, end);
- while ((ret = bam_iter_read(fp, iter, b)) >= 0) func(b, data);
- bam_iter_destroy(iter);
- bam_destroy1(b);
- return (ret == -1)? 0 : ret;
-}
diff --git a/samtools-0.1.19/bam_lpileup.c b/samtools-0.1.19/bam_lpileup.c
deleted file mode 100644
index d4dd63b..0000000
--- a/samtools-0.1.19/bam_lpileup.c
+++ /dev/null
@@ -1,198 +0,0 @@
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include "bam.h"
-#include "ksort.h"
-
-#define TV_GAP 2
-
-typedef struct __freenode_t {
- uint32_t level:28, cnt:4;
- struct __freenode_t *next;
-} freenode_t, *freenode_p;
-
-#define freenode_lt(a,b) ((a)->cnt < (b)->cnt || ((a)->cnt == (b)->cnt && (a)->level < (b)->level))
-KSORT_INIT(node, freenode_p, freenode_lt)
-
-/* Memory pool, similar to the one in bam_pileup.c */
-typedef struct {
- int cnt, n, max;
- freenode_t **buf;
-} mempool_t;
-
-static mempool_t *mp_init()
-{
- return (mempool_t*)calloc(1, sizeof(mempool_t));
-}
-static void mp_destroy(mempool_t *mp)
-{
- int k;
- for (k = 0; k < mp->n; ++k) free(mp->buf[k]);
- free(mp->buf); free(mp);
-}
-static inline freenode_t *mp_alloc(mempool_t *mp)
-{
- ++mp->cnt;
- if (mp->n == 0) return (freenode_t*)calloc(1, sizeof(freenode_t));
- else return mp->buf[--mp->n];
-}
-static inline void mp_free(mempool_t *mp, freenode_t *p)
-{
- --mp->cnt; p->next = 0; p->cnt = TV_GAP;
- if (mp->n == mp->max) {
- mp->max = mp->max? mp->max<<1 : 256;
- mp->buf = (freenode_t**)realloc(mp->buf, sizeof(freenode_t*) * mp->max);
- }
- mp->buf[mp->n++] = p;
-}
-
-/* core part */
-struct __bam_lplbuf_t {
- int max, n_cur, n_pre;
- int max_level, *cur_level, *pre_level;
- mempool_t *mp;
- freenode_t **aux, *head, *tail;
- int n_nodes, m_aux;
- bam_pileup_f func;
- void *user_data;
- bam_plbuf_t *plbuf;
-};
-
-void bam_lplbuf_reset(bam_lplbuf_t *buf)
-{
- freenode_t *p, *q;
- bam_plbuf_reset(buf->plbuf);
- for (p = buf->head; p->next;) {
- q = p->next;
- mp_free(buf->mp, p);
- p = q;
- }
- buf->head = buf->tail;
- buf->max_level = 0;
- buf->n_cur = buf->n_pre = 0;
- buf->n_nodes = 0;
-}
-
-static int tview_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data)
-{
- bam_lplbuf_t *tv = (bam_lplbuf_t*)data;
- freenode_t *p;
- int i, l, max_level;
- // allocate memory if necessary
- if (tv->max < n) { // enlarge
- tv->max = n;
- kroundup32(tv->max);
- tv->cur_level = (int*)realloc(tv->cur_level, sizeof(int) * tv->max);
- tv->pre_level = (int*)realloc(tv->pre_level, sizeof(int) * tv->max);
- }
- tv->n_cur = n;
- // update cnt
- for (p = tv->head; p->next; p = p->next)
- if (p->cnt > 0) --p->cnt;
- // calculate cur_level[]
- max_level = 0;
- for (i = l = 0; i < n; ++i) {
- const bam_pileup1_t *p = pl + i;
- if (p->is_head) {
- if (tv->head->next && tv->head->cnt == 0) { // then take a free slot
- freenode_t *p = tv->head->next;
- tv->cur_level[i] = tv->head->level;
- mp_free(tv->mp, tv->head);
- tv->head = p;
- --tv->n_nodes;
- } else tv->cur_level[i] = ++tv->max_level;
- } else {
- tv->cur_level[i] = tv->pre_level[l++];
- if (p->is_tail) { // then return a free slot
- tv->tail->level = tv->cur_level[i];
- tv->tail->next = mp_alloc(tv->mp);
- tv->tail = tv->tail->next;
- ++tv->n_nodes;
- }
- }
- if (tv->cur_level[i] > max_level) max_level = tv->cur_level[i];
- ((bam_pileup1_t*)p)->level = tv->cur_level[i];
- }
- assert(l == tv->n_pre);
- tv->func(tid, pos, n, pl, tv->user_data);
- // sort the linked list
- if (tv->n_nodes) {
- freenode_t *q;
- if (tv->n_nodes + 1 > tv->m_aux) { // enlarge
- tv->m_aux = tv->n_nodes + 1;
- kroundup32(tv->m_aux);
- tv->aux = (freenode_t**)realloc(tv->aux, sizeof(void*) * tv->m_aux);
- }
- for (p = tv->head, i = l = 0; p->next;) {
- if (p->level > max_level) { // then discard this entry
- q = p->next;
- mp_free(tv->mp, p);
- p = q;
- } else {
- tv->aux[i++] = p;
- p = p->next;
- }
- }
- tv->aux[i] = tv->tail; // add a proper tail for the loop below
- tv->n_nodes = i;
- if (tv->n_nodes) {
- ks_introsort(node, tv->n_nodes, tv->aux);
- for (i = 0; i < tv->n_nodes; ++i) tv->aux[i]->next = tv->aux[i+1];
- tv->head = tv->aux[0];
- } else tv->head = tv->tail;
- }
- // clean up
- tv->max_level = max_level;
- memcpy(tv->pre_level, tv->cur_level, tv->n_cur * 4);
- // squeeze out terminated levels
- for (i = l = 0; i < n; ++i) {
- const bam_pileup1_t *p = pl + i;
- if (!p->is_tail)
- tv->pre_level[l++] = tv->pre_level[i];
- }
- tv->n_pre = l;
-/*
- fprintf(stderr, "%d\t", pos+1);
- for (i = 0; i < n; ++i) {
- const bam_pileup1_t *p = pl + i;
- if (p->is_head) fprintf(stderr, "^");
- if (p->is_tail) fprintf(stderr, "$");
- fprintf(stderr, "%d,", p->level);
- }
- fprintf(stderr, "\n");
-*/
- return 0;
-}
-
-bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data)
-{
- bam_lplbuf_t *tv;
- tv = (bam_lplbuf_t*)calloc(1, sizeof(bam_lplbuf_t));
- tv->mp = mp_init();
- tv->head = tv->tail = mp_alloc(tv->mp);
- tv->func = func;
- tv->user_data = data;
- tv->plbuf = bam_plbuf_init(tview_func, tv);
- return (bam_lplbuf_t*)tv;
-}
-
-void bam_lplbuf_destroy(bam_lplbuf_t *tv)
-{
- freenode_t *p, *q;
- free(tv->cur_level); free(tv->pre_level);
- bam_plbuf_destroy(tv->plbuf);
- free(tv->aux);
- for (p = tv->head; p->next;) {
- q = p->next;
- mp_free(tv->mp, p); p = q;
- }
- mp_free(tv->mp, p);
- assert(tv->mp->cnt == 0);
- mp_destroy(tv->mp);
- free(tv);
-}
-
-int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *tv)
-{
- return bam_plbuf_push(b, tv->plbuf);
-}
diff --git a/samtools-0.1.19/bam_mate.c b/samtools-0.1.19/bam_mate.c
deleted file mode 100644
index b947c9d..0000000
--- a/samtools-0.1.19/bam_mate.c
+++ /dev/null
@@ -1,128 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "kstring.h"
-#include "bam.h"
-
-void bam_template_cigar(bam1_t *b1, bam1_t *b2, kstring_t *str)
-{
- bam1_t *swap;
- int i, end;
- uint32_t *cigar;
- str->l = 0;
- if (b1->core.tid != b2->core.tid || b1->core.tid < 0) return; // coordinateless or not on the same chr; skip
- if (b1->core.pos > b2->core.pos) swap = b1, b1 = b2, b2 = swap; // make sure b1 has a smaller coordinate
- kputc((b1->core.flag & BAM_FREAD1)? '1' : '2', str); // segment index
- kputc((b1->core.flag & BAM_FREVERSE)? 'R' : 'F', str); // strand
- for (i = 0, cigar = bam1_cigar(b1); i < b1->core.n_cigar; ++i) {
- kputw(bam_cigar_oplen(cigar[i]), str);
- kputc(bam_cigar_opchr(cigar[i]), str);
- }
- end = bam_calend(&b1->core, cigar);
- kputw(b2->core.pos - end, str);
- kputc('T', str);
- kputc((b2->core.flag & BAM_FREAD1)? '1' : '2', str); // segment index
- kputc((b2->core.flag & BAM_FREVERSE)? 'R' : 'F', str); // strand
- for (i = 0, cigar = bam1_cigar(b2); i < b2->core.n_cigar; ++i) {
- kputw(bam_cigar_oplen(cigar[i]), str);
- kputc(bam_cigar_opchr(cigar[i]), str);
- }
- bam_aux_append(b1, "CT", 'Z', str->l+1, (uint8_t*)str->s);
-}
-
-// currently, this function ONLY works if each read has one hit
-void bam_mating_core(bamFile in, bamFile out, int remove_reads)
-{
- bam_header_t *header;
- bam1_t *b[2];
- int curr, has_prev, pre_end = 0, cur_end;
- kstring_t str;
-
- str.l = str.m = 0; str.s = 0;
- header = bam_header_read(in);
- bam_header_write(out, header);
-
- b[0] = bam_init1();
- b[1] = bam_init1();
- curr = 0; has_prev = 0;
- while (bam_read1(in, b[curr]) >= 0) {
- bam1_t *cur = b[curr], *pre = b[1-curr];
- if (cur->core.tid < 0)
- {
- if ( !remove_reads ) bam_write1(out, cur);
- continue;
- }
- cur_end = bam_calend(&cur->core, bam1_cigar(cur));
- if (cur_end > (int)header->target_len[cur->core.tid]) cur->core.flag |= BAM_FUNMAP;
- if (cur->core.flag & BAM_FSECONDARY)
- {
- if ( !remove_reads ) bam_write1(out, cur);
- continue; // skip secondary alignments
- }
- if (has_prev) {
- if (strcmp(bam1_qname(cur), bam1_qname(pre)) == 0) { // identical pair name
- cur->core.mtid = pre->core.tid; cur->core.mpos = pre->core.pos;
- pre->core.mtid = cur->core.tid; pre->core.mpos = cur->core.pos;
- if (pre->core.tid == cur->core.tid && !(cur->core.flag&(BAM_FUNMAP|BAM_FMUNMAP))
- && !(pre->core.flag&(BAM_FUNMAP|BAM_FMUNMAP))) // set TLEN/ISIZE
- {
- uint32_t cur5, pre5;
- cur5 = (cur->core.flag&BAM_FREVERSE)? cur_end : cur->core.pos;
- pre5 = (pre->core.flag&BAM_FREVERSE)? pre_end : pre->core.pos;
- cur->core.isize = pre5 - cur5; pre->core.isize = cur5 - pre5;
- } else cur->core.isize = pre->core.isize = 0;
- if (pre->core.flag&BAM_FREVERSE) cur->core.flag |= BAM_FMREVERSE;
- else cur->core.flag &= ~BAM_FMREVERSE;
- if (cur->core.flag&BAM_FREVERSE) pre->core.flag |= BAM_FMREVERSE;
- else pre->core.flag &= ~BAM_FMREVERSE;
- if (cur->core.flag & BAM_FUNMAP) { pre->core.flag |= BAM_FMUNMAP; pre->core.flag &= ~BAM_FPROPER_PAIR; }
- if (pre->core.flag & BAM_FUNMAP) { cur->core.flag |= BAM_FMUNMAP; cur->core.flag &= ~BAM_FPROPER_PAIR; }
- bam_template_cigar(pre, cur, &str);
- bam_write1(out, pre);
- bam_write1(out, cur);
- has_prev = 0;
- } else { // unpaired or singleton
- pre->core.mtid = -1; pre->core.mpos = -1; pre->core.isize = 0;
- if (pre->core.flag & BAM_FPAIRED) {
- pre->core.flag |= BAM_FMUNMAP;
- pre->core.flag &= ~BAM_FMREVERSE & ~BAM_FPROPER_PAIR;
- }
- bam_write1(out, pre);
- }
- } else has_prev = 1;
- curr = 1 - curr;
- pre_end = cur_end;
- }
- if (has_prev) bam_write1(out, b[1-curr]);
- bam_header_destroy(header);
- bam_destroy1(b[0]);
- bam_destroy1(b[1]);
- free(str.s);
-}
-
-void usage()
-{
- fprintf(stderr,"Usage: samtools fixmate <in.nameSrt.bam> <out.nameSrt.bam>\n");
- fprintf(stderr,"Options:\n");
- fprintf(stderr," -r remove unmapped reads and secondary alignments\n");
- exit(1);
-}
-
-int bam_mating(int argc, char *argv[])
-{
- bamFile in, out;
- int c, remove_reads=0;
- while ((c = getopt(argc, argv, "r")) >= 0) {
- switch (c) {
- case 'r': remove_reads=1; break;
- }
- }
- if (optind+1 >= argc) usage();
- in = (strcmp(argv[optind], "-") == 0)? bam_dopen(fileno(stdin), "r") : bam_open(argv[optind], "r");
- out = (strcmp(argv[optind+1], "-") == 0)? bam_dopen(fileno(stdout), "w") : bam_open(argv[optind+1], "w");
- bam_mating_core(in, out, remove_reads);
- bam_close(in); bam_close(out);
- return 0;
-}
-
-
diff --git a/samtools-0.1.19/bam_md.c b/samtools-0.1.19/bam_md.c
deleted file mode 100644
index ce40a12..0000000
--- a/samtools-0.1.19/bam_md.c
+++ /dev/null
@@ -1,389 +0,0 @@
-#include <unistd.h>
-#include <assert.h>
-#include <string.h>
-#include <ctype.h>
-#include <math.h>
-#include "faidx.h"
-#include "sam.h"
-#include "kstring.h"
-#include "kaln.h"
-#include "kprobaln.h"
-
-#define USE_EQUAL 1
-#define DROP_TAG 2
-#define BIN_QUAL 4
-#define UPDATE_NM 8
-#define UPDATE_MD 16
-#define HASH_QNM 32
-
-char bam_nt16_nt4_table[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 };
-
-int bam_aux_drop_other(bam1_t *b, uint8_t *s);
-
-void bam_fillmd1_core(bam1_t *b, char *ref, int flag, int max_nm)
-{
- uint8_t *seq = bam1_seq(b);
- uint32_t *cigar = bam1_cigar(b);
- bam1_core_t *c = &b->core;
- int i, x, y, u = 0;
- kstring_t *str;
- int32_t old_nm_i = -1, nm = 0;
-
- str = (kstring_t*)calloc(1, sizeof(kstring_t));
- for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) {
- int j, l = cigar[i]>>4, op = cigar[i]&0xf;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (j = 0; j < l; ++j) {
- int z = y + j;
- int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]];
- if (ref[x+j] == 0) break; // out of boundary
- if ((c1 == c2 && c1 != 15 && c2 != 15) || c1 == 0) { // a match
- if (flag&USE_EQUAL) seq[z/2] &= (z&1)? 0xf0 : 0x0f;
- ++u;
- } else {
- kputw(u, str); kputc(ref[x+j], str);
- u = 0; ++nm;
- }
- }
- if (j < l) break;
- x += l; y += l;
- } else if (op == BAM_CDEL) {
- kputw(u, str); kputc('^', str);
- for (j = 0; j < l; ++j) {
- if (ref[x+j] == 0) break;
- kputc(ref[x+j], str);
- }
- u = 0;
- if (j < l) break;
- x += l; nm += l;
- } else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) {
- y += l;
- if (op == BAM_CINS) nm += l;
- } else if (op == BAM_CREF_SKIP) {
- x += l;
- }
- }
- kputw(u, str);
- // apply max_nm
- if (max_nm > 0 && nm >= max_nm) {
- for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) {
- int j, l = cigar[i]>>4, op = cigar[i]&0xf;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (j = 0; j < l; ++j) {
- int z = y + j;
- int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]];
- if (ref[x+j] == 0) break; // out of boundary
- if ((c1 == c2 && c1 != 15 && c2 != 15) || c1 == 0) { // a match
- seq[z/2] |= (z&1)? 0x0f : 0xf0;
- bam1_qual(b)[z] = 0;
- }
- }
- if (j < l) break;
- x += l; y += l;
- } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) x += l;
- else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) y += l;
- }
- }
- // update NM
- if (flag & UPDATE_NM) {
- uint8_t *old_nm = bam_aux_get(b, "NM");
- if (c->flag & BAM_FUNMAP) return;
- if (old_nm) old_nm_i = bam_aux2i(old_nm);
- if (!old_nm) bam_aux_append(b, "NM", 'i', 4, (uint8_t*)&nm);
- else if (nm != old_nm_i) {
- fprintf(stderr, "[bam_fillmd1] different NM for read '%s': %d -> %d\n", bam1_qname(b), old_nm_i, nm);
- bam_aux_del(b, old_nm);
- bam_aux_append(b, "NM", 'i', 4, (uint8_t*)&nm);
- }
- }
- // update MD
- if (flag & UPDATE_MD) {
- uint8_t *old_md = bam_aux_get(b, "MD");
- if (c->flag & BAM_FUNMAP) return;
- if (!old_md) bam_aux_append(b, "MD", 'Z', str->l + 1, (uint8_t*)str->s);
- else {
- int is_diff = 0;
- if (strlen((char*)old_md+1) == str->l) {
- for (i = 0; i < str->l; ++i)
- if (toupper(old_md[i+1]) != toupper(str->s[i]))
- break;
- if (i < str->l) is_diff = 1;
- } else is_diff = 1;
- if (is_diff) {
- fprintf(stderr, "[bam_fillmd1] different MD for read '%s': '%s' -> '%s'\n", bam1_qname(b), old_md+1, str->s);
- bam_aux_del(b, old_md);
- bam_aux_append(b, "MD", 'Z', str->l + 1, (uint8_t*)str->s);
- }
- }
- }
- // drop all tags but RG
- if (flag&DROP_TAG) {
- uint8_t *q = bam_aux_get(b, "RG");
- bam_aux_drop_other(b, q);
- }
- // reduce the resolution of base quality
- if (flag&BIN_QUAL) {
- uint8_t *qual = bam1_qual(b);
- for (i = 0; i < b->core.l_qseq; ++i)
- if (qual[i] >= 3) qual[i] = qual[i]/10*10 + 7;
- }
- free(str->s); free(str);
-}
-
-void bam_fillmd1(bam1_t *b, char *ref, int flag)
-{
- bam_fillmd1_core(b, ref, flag, 0);
-}
-
-int bam_cap_mapQ(bam1_t *b, char *ref, int thres)
-{
- uint8_t *seq = bam1_seq(b), *qual = bam1_qual(b);
- uint32_t *cigar = bam1_cigar(b);
- bam1_core_t *c = &b->core;
- int i, x, y, mm, q, len, clip_l, clip_q;
- double t;
- if (thres < 0) thres = 40; // set the default
- mm = q = len = clip_l = clip_q = 0;
- for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) {
- int j, l = cigar[i]>>4, op = cigar[i]&0xf;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (j = 0; j < l; ++j) {
- int z = y + j;
- int c1 = bam1_seqi(seq, z), c2 = bam_nt16_table[(int)ref[x+j]];
- if (ref[x+j] == 0) break; // out of boundary
- if (c2 != 15 && c1 != 15 && qual[z] >= 13) { // not ambiguous
- ++len;
- if (c1 && c1 != c2 && qual[z] >= 13) { // mismatch
- ++mm;
- q += qual[z] > 33? 33 : qual[z];
- }
- }
- }
- if (j < l) break;
- x += l; y += l; len += l;
- } else if (op == BAM_CDEL) {
- for (j = 0; j < l; ++j)
- if (ref[x+j] == 0) break;
- if (j < l) break;
- x += l;
- } else if (op == BAM_CSOFT_CLIP) {
- for (j = 0; j < l; ++j) clip_q += qual[y+j];
- clip_l += l;
- y += l;
- } else if (op == BAM_CHARD_CLIP) {
- clip_q += 13 * l;
- clip_l += l;
- } else if (op == BAM_CINS) y += l;
- else if (op == BAM_CREF_SKIP) x += l;
- }
- for (i = 0, t = 1; i < mm; ++i)
- t *= (double)len / (i+1);
- t = q - 4.343 * log(t) + clip_q / 5.;
- if (t > thres) return -1;
- if (t < 0) t = 0;
- t = sqrt((thres - t) / thres) * thres;
-// fprintf(stderr, "%s %lf %d\n", bam1_qname(b), t, q);
- return (int)(t + .499);
-}
-
-int bam_prob_realn_core(bam1_t *b, const char *ref, int flag)
-{
- int k, i, bw, x, y, yb, ye, xb, xe, apply_baq = flag&1, extend_baq = flag>>1&1, redo_baq = flag&4;
- uint32_t *cigar = bam1_cigar(b);
- bam1_core_t *c = &b->core;
- kpa_par_t conf = kpa_par_def;
- uint8_t *bq = 0, *zq = 0, *qual = bam1_qual(b);
- if ((c->flag & BAM_FUNMAP) || b->core.l_qseq == 0) return -1; // do nothing
- // test if BQ or ZQ is present
- if ((bq = bam_aux_get(b, "BQ")) != 0) ++bq;
- if ((zq = bam_aux_get(b, "ZQ")) != 0 && *zq == 'Z') ++zq;
- if (bq && redo_baq)
- {
- bam_aux_del(b, bq-1);
- bq = 0;
- }
- if (bq && zq) { // remove the ZQ tag
- bam_aux_del(b, zq-1);
- zq = 0;
- }
- if (bq || zq) {
- if ((apply_baq && zq) || (!apply_baq && bq)) return -3; // in both cases, do nothing
- if (bq && apply_baq) { // then convert BQ to ZQ
- for (i = 0; i < c->l_qseq; ++i)
- qual[i] = qual[i] + 64 < bq[i]? 0 : qual[i] - ((int)bq[i] - 64);
- *(bq - 3) = 'Z';
- } else if (zq && !apply_baq) { // then convert ZQ to BQ
- for (i = 0; i < c->l_qseq; ++i)
- qual[i] += (int)zq[i] - 64;
- *(zq - 3) = 'B';
- }
- return 0;
- }
- // find the start and end of the alignment
- x = c->pos, y = 0, yb = ye = xb = xe = -1;
- for (k = 0; k < c->n_cigar; ++k) {
- int op, l;
- op = cigar[k]&0xf; l = cigar[k]>>4;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- if (yb < 0) yb = y;
- if (xb < 0) xb = x;
- ye = y + l; xe = x + l;
- x += l; y += l;
- } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l;
- else if (op == BAM_CDEL) x += l;
- else if (op == BAM_CREF_SKIP) return -1; // do nothing if there is a reference skip
- }
- // set bandwidth and the start and the end
- bw = 7;
- if (abs((xe - xb) - (ye - yb)) > bw)
- bw = abs((xe - xb) - (ye - yb)) + 3;
- conf.bw = bw;
- xb -= yb + bw/2; if (xb < 0) xb = 0;
- xe += c->l_qseq - ye + bw/2;
- if (xe - xb - c->l_qseq > bw)
- xb += (xe - xb - c->l_qseq - bw) / 2, xe -= (xe - xb - c->l_qseq - bw) / 2;
- { // glocal
- uint8_t *s, *r, *q, *seq = bam1_seq(b), *bq;
- int *state;
- bq = calloc(c->l_qseq + 1, 1);
- memcpy(bq, qual, c->l_qseq);
- s = calloc(c->l_qseq, 1);
- for (i = 0; i < c->l_qseq; ++i) s[i] = bam_nt16_nt4_table[bam1_seqi(seq, i)];
- r = calloc(xe - xb, 1);
- for (i = xb; i < xe; ++i) {
- if (ref[i] == 0) { xe = i; break; }
- r[i-xb] = bam_nt16_nt4_table[bam_nt16_table[(int)ref[i]]];
- }
- state = calloc(c->l_qseq, sizeof(int));
- q = calloc(c->l_qseq, 1);
- kpa_glocal(r, xe-xb, s, c->l_qseq, qual, &conf, state, q);
- if (!extend_baq) { // in this block, bq[] is capped by base quality qual[]
- for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) {
- int op = cigar[k]&0xf, l = cigar[k]>>4;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (i = y; i < y + l; ++i) {
- if ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y)) bq[i] = 0;
- else bq[i] = bq[i] < q[i]? bq[i] : q[i];
- }
- x += l; y += l;
- } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l;
- else if (op == BAM_CDEL) x += l;
- }
- for (i = 0; i < c->l_qseq; ++i) bq[i] = qual[i] - bq[i] + 64; // finalize BQ
- } else { // in this block, bq[] is BAQ that can be larger than qual[] (different from the above!)
- uint8_t *left, *rght;
- left = calloc(c->l_qseq, 1); rght = calloc(c->l_qseq, 1);
- for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) {
- int op = cigar[k]&0xf, l = cigar[k]>>4;
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (i = y; i < y + l; ++i)
- bq[i] = ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y))? 0 : q[i];
- for (left[y] = bq[y], i = y + 1; i < y + l; ++i)
- left[i] = bq[i] > left[i-1]? bq[i] : left[i-1];
- for (rght[y+l-1] = bq[y+l-1], i = y + l - 2; i >= y; --i)
- rght[i] = bq[i] > rght[i+1]? bq[i] : rght[i+1];
- for (i = y; i < y + l; ++i)
- bq[i] = left[i] < rght[i]? left[i] : rght[i];
- x += l; y += l;
- } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l;
- else if (op == BAM_CDEL) x += l;
- }
- for (i = 0; i < c->l_qseq; ++i) bq[i] = 64 + (qual[i] <= bq[i]? 0 : qual[i] - bq[i]); // finalize BQ
- free(left); free(rght);
- }
- if (apply_baq) {
- for (i = 0; i < c->l_qseq; ++i) qual[i] -= bq[i] - 64; // modify qual
- bam_aux_append(b, "ZQ", 'Z', c->l_qseq + 1, bq);
- } else bam_aux_append(b, "BQ", 'Z', c->l_qseq + 1, bq);
- free(bq); free(s); free(r); free(q); free(state);
- }
- return 0;
-}
-
-int bam_prob_realn(bam1_t *b, const char *ref)
-{
- return bam_prob_realn_core(b, ref, 1);
-}
-
-int bam_fillmd(int argc, char *argv[])
-{
- int c, flt_flag, tid = -2, ret, len, is_bam_out, is_sam_in, is_uncompressed, max_nm, is_realn, capQ, baq_flag;
- samfile_t *fp, *fpout = 0;
- faidx_t *fai;
- char *ref = 0, mode_w[8], mode_r[8];
- bam1_t *b;
-
- flt_flag = UPDATE_NM | UPDATE_MD;
- is_bam_out = is_sam_in = is_uncompressed = is_realn = max_nm = capQ = baq_flag = 0;
- mode_w[0] = mode_r[0] = 0;
- strcpy(mode_r, "r"); strcpy(mode_w, "w");
- while ((c = getopt(argc, argv, "EqreuNhbSC:n:Ad")) >= 0) {
- switch (c) {
- case 'r': is_realn = 1; break;
- case 'e': flt_flag |= USE_EQUAL; break;
- case 'd': flt_flag |= DROP_TAG; break;
- case 'q': flt_flag |= BIN_QUAL; break;
- case 'h': flt_flag |= HASH_QNM; break;
- case 'N': flt_flag &= ~(UPDATE_MD|UPDATE_NM); break;
- case 'b': is_bam_out = 1; break;
- case 'u': is_uncompressed = is_bam_out = 1; break;
- case 'S': is_sam_in = 1; break;
- case 'n': max_nm = atoi(optarg); break;
- case 'C': capQ = atoi(optarg); break;
- case 'A': baq_flag |= 1; break;
- case 'E': baq_flag |= 2; break;
- default: fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n", c); return 1;
- }
- }
- if (!is_sam_in) strcat(mode_r, "b");
- if (is_bam_out) strcat(mode_w, "b");
- else strcat(mode_w, "h");
- if (is_uncompressed) strcat(mode_w, "u");
- if (optind + 1 >= argc) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools fillmd [-eubrS] <aln.bam> <ref.fasta>\n\n");
- fprintf(stderr, "Options: -e change identical bases to '='\n");
- fprintf(stderr, " -u uncompressed BAM output (for piping)\n");
- fprintf(stderr, " -b compressed BAM output\n");
- fprintf(stderr, " -S the input is SAM with header\n");
- fprintf(stderr, " -A modify the quality string\n");
- fprintf(stderr, " -r compute the BQ tag (without -A) or cap baseQ by BAQ (with -A)\n");
- fprintf(stderr, " -E extended BAQ for better sensitivity but lower specificity\n\n");
- return 1;
- }
- fp = samopen(argv[optind], mode_r, 0);
- if (fp == 0) return 1;
- if (is_sam_in && (fp->header == 0 || fp->header->n_targets == 0)) {
- fprintf(stderr, "[bam_fillmd] input SAM does not have header. Abort!\n");
- return 1;
- }
- fpout = samopen("-", mode_w, fp->header);
- fai = fai_load(argv[optind+1]);
-
- b = bam_init1();
- while ((ret = samread(fp, b)) >= 0) {
- if (b->core.tid >= 0) {
- if (tid != b->core.tid) {
- free(ref);
- ref = fai_fetch(fai, fp->header->target_name[b->core.tid], &len);
- tid = b->core.tid;
- if (ref == 0)
- fprintf(stderr, "[bam_fillmd] fail to find sequence '%s' in the reference.\n",
- fp->header->target_name[tid]);
- }
- if (is_realn) bam_prob_realn_core(b, ref, baq_flag);
- if (capQ > 10) {
- int q = bam_cap_mapQ(b, ref, capQ);
- if (b->core.qual > q) b->core.qual = q;
- }
- if (ref) bam_fillmd1_core(b, ref, flt_flag, max_nm);
- }
- samwrite(fpout, b);
- }
- bam_destroy1(b);
-
- free(ref);
- fai_destroy(fai);
- samclose(fp); samclose(fpout);
- return 0;
-}
diff --git a/samtools-0.1.19/bam_pileup.c b/samtools-0.1.19/bam_pileup.c
deleted file mode 100644
index 57434e0..0000000
--- a/samtools-0.1.19/bam_pileup.c
+++ /dev/null
@@ -1,437 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <assert.h>
-#include "sam.h"
-
-typedef struct {
- int k, x, y, end;
-} cstate_t;
-
-static cstate_t g_cstate_null = { -1, 0, 0, 0 };
-
-typedef struct __linkbuf_t {
- bam1_t b;
- uint32_t beg, end;
- cstate_t s;
- struct __linkbuf_t *next;
-} lbnode_t;
-
-/* --- BEGIN: Memory pool */
-
-typedef struct {
- int cnt, n, max;
- lbnode_t **buf;
-} mempool_t;
-
-static mempool_t *mp_init()
-{
- mempool_t *mp;
- mp = (mempool_t*)calloc(1, sizeof(mempool_t));
- return mp;
-}
-static void mp_destroy(mempool_t *mp)
-{
- int k;
- for (k = 0; k < mp->n; ++k) {
- free(mp->buf[k]->b.data);
- free(mp->buf[k]);
- }
- free(mp->buf);
- free(mp);
-}
-static inline lbnode_t *mp_alloc(mempool_t *mp)
-{
- ++mp->cnt;
- if (mp->n == 0) return (lbnode_t*)calloc(1, sizeof(lbnode_t));
- else return mp->buf[--mp->n];
-}
-static inline void mp_free(mempool_t *mp, lbnode_t *p)
-{
- --mp->cnt; p->next = 0; // clear lbnode_t::next here
- if (mp->n == mp->max) {
- mp->max = mp->max? mp->max<<1 : 256;
- mp->buf = (lbnode_t**)realloc(mp->buf, sizeof(lbnode_t*) * mp->max);
- }
- mp->buf[mp->n++] = p;
-}
-
-/* --- END: Memory pool */
-
-/* --- BEGIN: Auxiliary functions */
-
-/* s->k: the index of the CIGAR operator that has just been processed.
- s->x: the reference coordinate of the start of s->k
- s->y: the query coordiante of the start of s->k
- */
-static inline int resolve_cigar2(bam_pileup1_t *p, uint32_t pos, cstate_t *s)
-{
-#define _cop(c) ((c)&BAM_CIGAR_MASK)
-#define _cln(c) ((c)>>BAM_CIGAR_SHIFT)
-
- bam1_t *b = p->b;
- bam1_core_t *c = &b->core;
- uint32_t *cigar = bam1_cigar(b);
- int k, is_head = 0;
- // determine the current CIGAR operation
-// fprintf(stderr, "%s\tpos=%d\tend=%d\t(%d,%d,%d)\n", bam1_qname(b), pos, s->end, s->k, s->x, s->y);
- if (s->k == -1) { // never processed
- is_head = 1;
- if (c->n_cigar == 1) { // just one operation, save a loop
- if (_cop(cigar[0]) == BAM_CMATCH || _cop(cigar[0]) == BAM_CEQUAL || _cop(cigar[0]) == BAM_CDIFF) s->k = 0, s->x = c->pos, s->y = 0;
- } else { // find the first match or deletion
- for (k = 0, s->x = c->pos, s->y = 0; k < c->n_cigar; ++k) {
- int op = _cop(cigar[k]);
- int l = _cln(cigar[k]);
- if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CEQUAL || op == BAM_CDIFF) break;
- else if (op == BAM_CREF_SKIP) s->x += l;
- else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l;
- }
- assert(k < c->n_cigar);
- s->k = k;
- }
- } else { // the read has been processed before
- int op, l = _cln(cigar[s->k]);
- if (pos - s->x >= l) { // jump to the next operation
- assert(s->k < c->n_cigar); // otherwise a bug: this function should not be called in this case
- op = _cop(cigar[s->k+1]);
- if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) { // jump to the next without a loop
- if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l;
- s->x += l;
- ++s->k;
- } else { // find the next M/D/N/=/X
- if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l;
- s->x += l;
- for (k = s->k + 1; k < c->n_cigar; ++k) {
- op = _cop(cigar[k]), l = _cln(cigar[k]);
- if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) break;
- else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l;
- }
- s->k = k;
- }
- assert(s->k < c->n_cigar); // otherwise a bug
- } // else, do nothing
- }
- { // collect pileup information
- int op, l;
- op = _cop(cigar[s->k]); l = _cln(cigar[s->k]);
- p->is_del = p->indel = p->is_refskip = 0;
- if (s->x + l - 1 == pos && s->k + 1 < c->n_cigar) { // peek the next operation
- int op2 = _cop(cigar[s->k+1]);
- int l2 = _cln(cigar[s->k+1]);
- if (op2 == BAM_CDEL) p->indel = -(int)l2;
- else if (op2 == BAM_CINS) p->indel = l2;
- else if (op2 == BAM_CPAD && s->k + 2 < c->n_cigar) { // no working for adjacent padding
- int l3 = 0;
- for (k = s->k + 2; k < c->n_cigar; ++k) {
- op2 = _cop(cigar[k]); l2 = _cln(cigar[k]);
- if (op2 == BAM_CINS) l3 += l2;
- else if (op2 == BAM_CDEL || op2 == BAM_CMATCH || op2 == BAM_CREF_SKIP || op2 == BAM_CEQUAL || op2 == BAM_CDIFF) break;
- }
- if (l3 > 0) p->indel = l3;
- }
- }
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- p->qpos = s->y + (pos - s->x);
- } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) {
- p->is_del = 1; p->qpos = s->y; // FIXME: distinguish D and N!!!!!
- p->is_refskip = (op == BAM_CREF_SKIP);
- } // cannot be other operations; otherwise a bug
- p->is_head = (pos == c->pos); p->is_tail = (pos == s->end);
- }
- return 1;
-}
-
-/* --- END: Auxiliary functions */
-
-/*******************
- * pileup iterator *
- *******************/
-
-struct __bam_plp_t {
- mempool_t *mp;
- lbnode_t *head, *tail, *dummy;
- int32_t tid, pos, max_tid, max_pos;
- int is_eof, flag_mask, max_plp, error, maxcnt;
- bam_pileup1_t *plp;
- // for the "auto" interface only
- bam1_t *b;
- bam_plp_auto_f func;
- void *data;
-};
-
-bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data)
-{
- bam_plp_t iter;
- iter = calloc(1, sizeof(struct __bam_plp_t));
- iter->mp = mp_init();
- iter->head = iter->tail = mp_alloc(iter->mp);
- iter->dummy = mp_alloc(iter->mp);
- iter->max_tid = iter->max_pos = -1;
- iter->flag_mask = BAM_DEF_MASK;
- iter->maxcnt = 8000;
- if (func) {
- iter->func = func;
- iter->data = data;
- iter->b = bam_init1();
- }
- return iter;
-}
-
-void bam_plp_destroy(bam_plp_t iter)
-{
- mp_free(iter->mp, iter->dummy);
- mp_free(iter->mp, iter->head);
- if (iter->mp->cnt != 0)
- fprintf(stderr, "[bam_plp_destroy] memory leak: %d. Continue anyway.\n", iter->mp->cnt);
- mp_destroy(iter->mp);
- if (iter->b) bam_destroy1(iter->b);
- free(iter->plp);
- free(iter);
-}
-
-const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp)
-{
- if (iter->error) { *_n_plp = -1; return 0; }
- *_n_plp = 0;
- if (iter->is_eof && iter->head->next == 0) return 0;
- while (iter->is_eof || iter->max_tid > iter->tid || (iter->max_tid == iter->tid && iter->max_pos > iter->pos)) {
- int n_plp = 0;
- lbnode_t *p, *q;
- // write iter->plp at iter->pos
- iter->dummy->next = iter->head;
- for (p = iter->head, q = iter->dummy; p->next; q = p, p = p->next) {
- if (p->b.core.tid < iter->tid || (p->b.core.tid == iter->tid && p->end <= iter->pos)) { // then remove
- q->next = p->next; mp_free(iter->mp, p); p = q;
- } else if (p->b.core.tid == iter->tid && p->beg <= iter->pos) { // here: p->end > pos; then add to pileup
- if (n_plp == iter->max_plp) { // then double the capacity
- iter->max_plp = iter->max_plp? iter->max_plp<<1 : 256;
- iter->plp = (bam_pileup1_t*)realloc(iter->plp, sizeof(bam_pileup1_t) * iter->max_plp);
- }
- iter->plp[n_plp].b = &p->b;
- if (resolve_cigar2(iter->plp + n_plp, iter->pos, &p->s)) ++n_plp; // actually always true...
- }
- }
- iter->head = iter->dummy->next; // dummy->next may be changed
- *_n_plp = n_plp; *_tid = iter->tid; *_pos = iter->pos;
- // update iter->tid and iter->pos
- if (iter->head->next) {
- if (iter->tid > iter->head->b.core.tid) {
- fprintf(stderr, "[%s] unsorted input. Pileup aborts.\n", __func__);
- iter->error = 1;
- *_n_plp = -1;
- return 0;
- }
- }
- if (iter->tid < iter->head->b.core.tid) { // come to a new reference sequence
- iter->tid = iter->head->b.core.tid; iter->pos = iter->head->beg; // jump to the next reference
- } else if (iter->pos < iter->head->beg) { // here: tid == head->b.core.tid
- iter->pos = iter->head->beg; // jump to the next position
- } else ++iter->pos; // scan contiguously
- // return
- if (n_plp) return iter->plp;
- if (iter->is_eof && iter->head->next == 0) break;
- }
- return 0;
-}
-
-int bam_plp_push(bam_plp_t iter, const bam1_t *b)
-{
- if (iter->error) return -1;
- if (b) {
- if (b->core.tid < 0) return 0;
- if (b->core.flag & iter->flag_mask) return 0;
- if (iter->tid == b->core.tid && iter->pos == b->core.pos && iter->mp->cnt > iter->maxcnt) return 0;
- bam_copy1(&iter->tail->b, b);
- iter->tail->beg = b->core.pos; iter->tail->end = bam_calend(&b->core, bam1_cigar(b));
- iter->tail->s = g_cstate_null; iter->tail->s.end = iter->tail->end - 1; // initialize cstate_t
- if (b->core.tid < iter->max_tid) {
- fprintf(stderr, "[bam_pileup_core] the input is not sorted (chromosomes out of order)\n");
- iter->error = 1;
- return -1;
- }
- if ((b->core.tid == iter->max_tid) && (iter->tail->beg < iter->max_pos)) {
- fprintf(stderr, "[bam_pileup_core] the input is not sorted (reads out of order)\n");
- iter->error = 1;
- return -1;
- }
- iter->max_tid = b->core.tid; iter->max_pos = iter->tail->beg;
- if (iter->tail->end > iter->pos || iter->tail->b.core.tid > iter->tid) {
- iter->tail->next = mp_alloc(iter->mp);
- iter->tail = iter->tail->next;
- }
- } else iter->is_eof = 1;
- return 0;
-}
-
-const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp)
-{
- const bam_pileup1_t *plp;
- if (iter->func == 0 || iter->error) { *_n_plp = -1; return 0; }
- if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp;
- else { // no pileup line can be obtained; read alignments
- *_n_plp = 0;
- if (iter->is_eof) return 0;
- while (iter->func(iter->data, iter->b) >= 0) {
- if (bam_plp_push(iter, iter->b) < 0) {
- *_n_plp = -1;
- return 0;
- }
- if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp;
- // otherwise no pileup line can be returned; read the next alignment.
- }
- bam_plp_push(iter, 0);
- if ((plp = bam_plp_next(iter, _tid, _pos, _n_plp)) != 0) return plp;
- return 0;
- }
-}
-
-void bam_plp_reset(bam_plp_t iter)
-{
- lbnode_t *p, *q;
- iter->max_tid = iter->max_pos = -1;
- iter->tid = iter->pos = 0;
- iter->is_eof = 0;
- for (p = iter->head; p->next;) {
- q = p->next;
- mp_free(iter->mp, p);
- p = q;
- }
- iter->head = iter->tail;
-}
-
-void bam_plp_set_mask(bam_plp_t iter, int mask)
-{
- iter->flag_mask = mask < 0? BAM_DEF_MASK : (BAM_FUNMAP | mask);
-}
-
-void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt)
-{
- iter->maxcnt = maxcnt;
-}
-
-/*****************
- * callback APIs *
- *****************/
-
-int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data)
-{
- bam_plbuf_t *buf;
- int ret;
- bam1_t *b;
- b = bam_init1();
- buf = bam_plbuf_init(func, func_data);
- bam_plbuf_set_mask(buf, mask);
- while ((ret = bam_read1(fp, b)) >= 0)
- bam_plbuf_push(b, buf);
- bam_plbuf_push(0, buf);
- bam_plbuf_destroy(buf);
- bam_destroy1(b);
- return 0;
-}
-
-void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask)
-{
- bam_plp_set_mask(buf->iter, mask);
-}
-
-void bam_plbuf_reset(bam_plbuf_t *buf)
-{
- bam_plp_reset(buf->iter);
-}
-
-bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data)
-{
- bam_plbuf_t *buf;
- buf = calloc(1, sizeof(bam_plbuf_t));
- buf->iter = bam_plp_init(0, 0);
- buf->func = func;
- buf->data = data;
- return buf;
-}
-
-void bam_plbuf_destroy(bam_plbuf_t *buf)
-{
- bam_plp_destroy(buf->iter);
- free(buf);
-}
-
-int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf)
-{
- int ret, n_plp, tid, pos;
- const bam_pileup1_t *plp;
- ret = bam_plp_push(buf->iter, b);
- if (ret < 0) return ret;
- while ((plp = bam_plp_next(buf->iter, &tid, &pos, &n_plp)) != 0)
- buf->func(tid, pos, n_plp, plp, buf->data);
- return 0;
-}
-
-/***********
- * mpileup *
- ***********/
-
-struct __bam_mplp_t {
- int n;
- uint64_t min, *pos;
- bam_plp_t *iter;
- int *n_plp;
- const bam_pileup1_t **plp;
-};
-
-bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data)
-{
- int i;
- bam_mplp_t iter;
- iter = calloc(1, sizeof(struct __bam_mplp_t));
- iter->pos = calloc(n, 8);
- iter->n_plp = calloc(n, sizeof(int));
- iter->plp = calloc(n, sizeof(void*));
- iter->iter = calloc(n, sizeof(void*));
- iter->n = n;
- iter->min = (uint64_t)-1;
- for (i = 0; i < n; ++i) {
- iter->iter[i] = bam_plp_init(func, data[i]);
- iter->pos[i] = iter->min;
- }
- return iter;
-}
-
-void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt)
-{
- int i;
- for (i = 0; i < iter->n; ++i)
- iter->iter[i]->maxcnt = maxcnt;
-}
-
-void bam_mplp_destroy(bam_mplp_t iter)
-{
- int i;
- for (i = 0; i < iter->n; ++i) bam_plp_destroy(iter->iter[i]);
- free(iter->iter); free(iter->pos); free(iter->n_plp); free(iter->plp);
- free(iter);
-}
-
-int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp)
-{
- int i, ret = 0;
- uint64_t new_min = (uint64_t)-1;
- for (i = 0; i < iter->n; ++i) {
- if (iter->pos[i] == iter->min) {
- int tid, pos;
- iter->plp[i] = bam_plp_auto(iter->iter[i], &tid, &pos, &iter->n_plp[i]);
- iter->pos[i] = (uint64_t)tid<<32 | pos;
- }
- if (iter->plp[i] && iter->pos[i] < new_min) new_min = iter->pos[i];
- }
- iter->min = new_min;
- if (new_min == (uint64_t)-1) return 0;
- *_tid = new_min>>32; *_pos = (uint32_t)new_min;
- for (i = 0; i < iter->n; ++i) {
- if (iter->pos[i] == iter->min) { // FIXME: valgrind reports "uninitialised value(s) at this line"
- n_plp[i] = iter->n_plp[i], plp[i] = iter->plp[i];
- ++ret;
- } else n_plp[i] = 0, plp[i] = 0;
- }
- return ret;
-}
diff --git a/samtools-0.1.19/bam_plcmd.c b/samtools-0.1.19/bam_plcmd.c
deleted file mode 100644
index 54a4597..0000000
--- a/samtools-0.1.19/bam_plcmd.c
+++ /dev/null
@@ -1,606 +0,0 @@
-#include <math.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <ctype.h>
-#include <string.h>
-#include <errno.h>
-#include <sys/stat.h>
-#include <getopt.h>
-#include "sam.h"
-#include "faidx.h"
-#include "kstring.h"
-#include "sam_header.h"
-
-static inline int printw(int c, FILE *fp)
-{
- char buf[16];
- int l, x;
- if (c == 0) return fputc('0', fp);
- for (l = 0, x = c < 0? -c : c; x > 0; x /= 10) buf[l++] = x%10 + '0';
- if (c < 0) buf[l++] = '-';
- buf[l] = 0;
- for (x = 0; x < l/2; ++x) {
- int y = buf[x]; buf[x] = buf[l-1-x]; buf[l-1-x] = y;
- }
- fputs(buf, fp);
- return 0;
-}
-
-static inline void pileup_seq(const bam_pileup1_t *p, int pos, int ref_len, const char *ref)
-{
- int j;
- if (p->is_head) {
- putchar('^');
- putchar(p->b->core.qual > 93? 126 : p->b->core.qual + 33);
- }
- if (!p->is_del) {
- int c = bam_nt16_rev_table[bam1_seqi(bam1_seq(p->b), p->qpos)];
- if (ref) {
- int rb = pos < ref_len? ref[pos] : 'N';
- if (c == '=' || bam_nt16_table[c] == bam_nt16_table[rb]) c = bam1_strand(p->b)? ',' : '.';
- else c = bam1_strand(p->b)? tolower(c) : toupper(c);
- } else {
- if (c == '=') c = bam1_strand(p->b)? ',' : '.';
- else c = bam1_strand(p->b)? tolower(c) : toupper(c);
- }
- putchar(c);
- } else putchar(p->is_refskip? (bam1_strand(p->b)? '<' : '>') : '*');
- if (p->indel > 0) {
- putchar('+'); printw(p->indel, stdout);
- for (j = 1; j <= p->indel; ++j) {
- int c = bam_nt16_rev_table[bam1_seqi(bam1_seq(p->b), p->qpos + j)];
- putchar(bam1_strand(p->b)? tolower(c) : toupper(c));
- }
- } else if (p->indel < 0) {
- printw(p->indel, stdout);
- for (j = 1; j <= -p->indel; ++j) {
- int c = (ref && (int)pos+j < ref_len)? ref[pos+j] : 'N';
- putchar(bam1_strand(p->b)? tolower(c) : toupper(c));
- }
- }
- if (p->is_tail) putchar('$');
-}
-
-#include <assert.h>
-#include "bam2bcf.h"
-#include "sample.h"
-
-#define MPLP_GLF 0x10
-#define MPLP_NO_COMP 0x20
-#define MPLP_NO_ORPHAN 0x40
-#define MPLP_REALN 0x80
-#define MPLP_NO_INDEL 0x400
-#define MPLP_REDO_BAQ 0x800
-#define MPLP_ILLUMINA13 0x1000
-#define MPLP_IGNORE_RG 0x2000
-#define MPLP_PRINT_POS 0x4000
-#define MPLP_PRINT_MAPQ 0x8000
-#define MPLP_PER_SAMPLE 0x10000
-
-void *bed_read(const char *fn);
-void bed_destroy(void *_h);
-int bed_overlap(const void *_h, const char *chr, int beg, int end);
-
-typedef struct {
- int max_mq, min_mq, flag, min_baseQ, capQ_thres, max_depth, max_indel_depth, fmt_flag;
- int rflag_require, rflag_filter;
- int openQ, extQ, tandemQ, min_support; // for indels
- double min_frac; // for indels
- char *reg, *pl_list, *fai_fname;
- faidx_t *fai;
- void *bed, *rghash;
-} mplp_conf_t;
-
-typedef struct {
- bamFile fp;
- bam_iter_t iter;
- bam_header_t *h;
- int ref_id;
- char *ref;
- const mplp_conf_t *conf;
-} mplp_aux_t;
-
-typedef struct {
- int n;
- int *n_plp, *m_plp;
- bam_pileup1_t **plp;
-} mplp_pileup_t;
-
-static int mplp_func(void *data, bam1_t *b)
-{
- extern int bam_realn(bam1_t *b, const char *ref);
- extern int bam_prob_realn_core(bam1_t *b, const char *ref, int);
- extern int bam_cap_mapQ(bam1_t *b, char *ref, int thres);
- mplp_aux_t *ma = (mplp_aux_t*)data;
- int ret, skip = 0;
- do {
- int has_ref;
- ret = ma->iter? bam_iter_read(ma->fp, ma->iter, b) : bam_read1(ma->fp, b);
- if (ret < 0) break;
- if (b->core.tid < 0 || (b->core.flag&BAM_FUNMAP)) { // exclude unmapped reads
- skip = 1;
- continue;
- }
- if (ma->conf->rflag_require && !(ma->conf->rflag_require&b->core.flag)) { skip = 1; continue; }
- if (ma->conf->rflag_filter && ma->conf->rflag_filter&b->core.flag) { skip = 1; continue; }
- if (ma->conf->bed) { // test overlap
- skip = !bed_overlap(ma->conf->bed, ma->h->target_name[b->core.tid], b->core.pos, bam_calend(&b->core, bam1_cigar(b)));
- if (skip) continue;
- }
- if (ma->conf->rghash) { // exclude read groups
- uint8_t *rg = bam_aux_get(b, "RG");
- skip = (rg && bcf_str2id(ma->conf->rghash, (const char*)(rg+1)) >= 0);
- if (skip) continue;
- }
- if (ma->conf->flag & MPLP_ILLUMINA13) {
- int i;
- uint8_t *qual = bam1_qual(b);
- for (i = 0; i < b->core.l_qseq; ++i)
- qual[i] = qual[i] > 31? qual[i] - 31 : 0;
- }
- has_ref = (ma->ref && ma->ref_id == b->core.tid)? 1 : 0;
- skip = 0;
- if (has_ref && (ma->conf->flag&MPLP_REALN)) bam_prob_realn_core(b, ma->ref, (ma->conf->flag & MPLP_REDO_BAQ)? 7 : 3);
- if (has_ref && ma->conf->capQ_thres > 10) {
- int q = bam_cap_mapQ(b, ma->ref, ma->conf->capQ_thres);
- if (q < 0) skip = 1;
- else if (b->core.qual > q) b->core.qual = q;
- }
- else if (b->core.qual < ma->conf->min_mq) skip = 1;
- else if ((ma->conf->flag&MPLP_NO_ORPHAN) && (b->core.flag&1) && !(b->core.flag&2)) skip = 1;
- } while (skip);
- return ret;
-}
-
-static void group_smpl(mplp_pileup_t *m, bam_sample_t *sm, kstring_t *buf,
- int n, char *const*fn, int *n_plp, const bam_pileup1_t **plp, int ignore_rg)
-{
- int i, j;
- memset(m->n_plp, 0, m->n * sizeof(int));
- for (i = 0; i < n; ++i) {
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j;
- uint8_t *q;
- int id = -1;
- q = ignore_rg? 0 : bam_aux_get(p->b, "RG");
- if (q) id = bam_smpl_rg2smid(sm, fn[i], (char*)q+1, buf);
- if (id < 0) id = bam_smpl_rg2smid(sm, fn[i], 0, buf);
- if (id < 0 || id >= m->n) {
- assert(q); // otherwise a bug
- fprintf(stderr, "[%s] Read group %s used in file %s but absent from the header or an alignment missing read group.\n", __func__, (char*)q+1, fn[i]);
- exit(1);
- }
- if (m->n_plp[id] == m->m_plp[id]) {
- m->m_plp[id] = m->m_plp[id]? m->m_plp[id]<<1 : 8;
- m->plp[id] = realloc(m->plp[id], sizeof(bam_pileup1_t) * m->m_plp[id]);
- }
- m->plp[id][m->n_plp[id]++] = *p;
- }
- }
-}
-
-static int mpileup(mplp_conf_t *conf, int n, char **fn)
-{
- extern void *bcf_call_add_rg(void *rghash, const char *hdtext, const char *list);
- extern void bcf_call_del_rghash(void *rghash);
- mplp_aux_t **data;
- int i, tid, pos, *n_plp, tid0 = -1, beg0 = 0, end0 = 1u<<29, ref_len, ref_tid = -1, max_depth, max_indel_depth;
- const bam_pileup1_t **plp;
- bam_mplp_t iter;
- bam_header_t *h = 0;
- char *ref;
- void *rghash = 0;
-
- bcf_callaux_t *bca = 0;
- bcf_callret1_t *bcr = 0;
- bcf_call_t bc;
- bcf_t *bp = 0;
- bcf_hdr_t *bh = 0;
-
- bam_sample_t *sm = 0;
- kstring_t buf;
- mplp_pileup_t gplp;
-
- memset(&gplp, 0, sizeof(mplp_pileup_t));
- memset(&buf, 0, sizeof(kstring_t));
- memset(&bc, 0, sizeof(bcf_call_t));
- data = calloc(n, sizeof(void*));
- plp = calloc(n, sizeof(void*));
- n_plp = calloc(n, sizeof(int*));
- sm = bam_smpl_init();
-
- // read the header and initialize data
- for (i = 0; i < n; ++i) {
- bam_header_t *h_tmp;
- data[i] = calloc(1, sizeof(mplp_aux_t));
- data[i]->fp = strcmp(fn[i], "-") == 0? bam_dopen(fileno(stdin), "r") : bam_open(fn[i], "r");
- if ( !data[i]->fp )
- {
- fprintf(stderr, "[%s] failed to open %s: %s\n", __func__, fn[i], strerror(errno));
- exit(1);
- }
- data[i]->conf = conf;
- h_tmp = bam_header_read(data[i]->fp);
- if ( !h_tmp ) {
- fprintf(stderr,"[%s] fail to read the header of %s\n", __func__, fn[i]);
- exit(1);
- }
- data[i]->h = i? h : h_tmp; // for i==0, "h" has not been set yet
- bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : h_tmp->text);
- rghash = bcf_call_add_rg(rghash, h_tmp->text, conf->pl_list);
- if (conf->reg) {
- int beg, end;
- bam_index_t *idx;
- idx = bam_index_load(fn[i]);
- if (idx == 0) {
- fprintf(stderr, "[%s] fail to load index for %s\n", __func__, fn[i]);
- exit(1);
- }
- if (bam_parse_region(h_tmp, conf->reg, &tid, &beg, &end) < 0) {
- fprintf(stderr, "[%s] malformatted region or wrong seqname for %s\n", __func__, fn[i]);
- exit(1);
- }
- if (i == 0) tid0 = tid, beg0 = beg, end0 = end;
- data[i]->iter = bam_iter_query(idx, tid, beg, end);
- bam_index_destroy(idx);
- }
- if (i == 0) h = h_tmp;
- else {
- // FIXME: to check consistency
- bam_header_destroy(h_tmp);
- }
- }
- gplp.n = sm->n;
- gplp.n_plp = calloc(sm->n, sizeof(int));
- gplp.m_plp = calloc(sm->n, sizeof(int));
- gplp.plp = calloc(sm->n, sizeof(void*));
-
- fprintf(stderr, "[%s] %d samples in %d input files\n", __func__, sm->n, n);
- // write the VCF header
- if (conf->flag & MPLP_GLF) {
- kstring_t s;
- bh = calloc(1, sizeof(bcf_hdr_t));
- s.l = s.m = 0; s.s = 0;
- bp = bcf_open("-", (conf->flag&MPLP_NO_COMP)? "wu" : "w");
- for (i = 0; i < h->n_targets; ++i) {
- kputs(h->target_name[i], &s);
- kputc('\0', &s);
- }
- bh->l_nm = s.l;
- bh->name = malloc(s.l);
- memcpy(bh->name, s.s, s.l);
- s.l = 0;
- for (i = 0; i < sm->n; ++i) {
- kputs(sm->smpl[i], &s); kputc('\0', &s);
- }
- bh->l_smpl = s.l;
- bh->sname = malloc(s.l);
- memcpy(bh->sname, s.s, s.l);
- s.l = 0;
- ksprintf(&s, "##samtoolsVersion=%s\n", BAM_VERSION);
- if (conf->fai_fname) ksprintf(&s, "##reference=file://%s\n", conf->fai_fname);
- h->dict = sam_header_parse2(h->text);
- int nseq;
- const char *tags[] = {"SN","LN","UR","M5",NULL};
- char **tbl = sam_header2tbl_n(h->dict, "SQ", tags, &nseq);
- for (i=0; i<nseq; i++)
- {
- ksprintf(&s, "##contig=<ID=%s", tbl[4*i]);
- if ( tbl[4*i+1] ) ksprintf(&s, ",length=%s", tbl[4*i+1]);
- if ( tbl[4*i+2] ) ksprintf(&s, ",URL=%s", tbl[4*i+2]);
- if ( tbl[4*i+3] ) ksprintf(&s, ",md5=%s", tbl[4*i+3]);
- kputs(">\n", &s);
- }
- if (tbl) free(tbl);
- bh->txt = s.s;
- bh->l_txt = 1 + s.l;
- bcf_hdr_sync(bh);
- bcf_hdr_write(bp, bh);
- bca = bcf_call_init(-1., conf->min_baseQ);
- bcr = calloc(sm->n, sizeof(bcf_callret1_t));
- bca->rghash = rghash;
- bca->openQ = conf->openQ, bca->extQ = conf->extQ, bca->tandemQ = conf->tandemQ;
- bca->min_frac = conf->min_frac;
- bca->min_support = conf->min_support;
- bca->per_sample_flt = conf->flag & MPLP_PER_SAMPLE;
- }
- if (tid0 >= 0 && conf->fai) { // region is set
- ref = faidx_fetch_seq(conf->fai, h->target_name[tid0], 0, 0x7fffffff, &ref_len);
- ref_tid = tid0;
- for (i = 0; i < n; ++i) data[i]->ref = ref, data[i]->ref_id = tid0;
- } else ref_tid = -1, ref = 0;
- iter = bam_mplp_init(n, mplp_func, (void**)data);
- max_depth = conf->max_depth;
- if (max_depth * sm->n > 1<<20)
- fprintf(stderr, "(%s) Max depth is above 1M. Potential memory hog!\n", __func__);
- if (max_depth * sm->n < 8000) {
- max_depth = 8000 / sm->n;
- fprintf(stderr, "<%s> Set max per-file depth to %d\n", __func__, max_depth);
- }
- max_indel_depth = conf->max_indel_depth * sm->n;
- bam_mplp_set_maxcnt(iter, max_depth);
- while (bam_mplp_auto(iter, &tid, &pos, n_plp, plp) > 0) {
- if (conf->reg && (pos < beg0 || pos >= end0)) continue; // out of the region requested
- if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
- if (tid != ref_tid) {
- free(ref); ref = 0;
- if (conf->fai) ref = faidx_fetch_seq(conf->fai, h->target_name[tid], 0, 0x7fffffff, &ref_len);
- for (i = 0; i < n; ++i) data[i]->ref = ref, data[i]->ref_id = tid;
- ref_tid = tid;
- }
- if (conf->flag & MPLP_GLF) {
- int total_depth, _ref0, ref16;
- bcf1_t *b = calloc(1, sizeof(bcf1_t));
- for (i = total_depth = 0; i < n; ++i) total_depth += n_plp[i];
- group_smpl(&gplp, sm, &buf, n, fn, n_plp, plp, conf->flag & MPLP_IGNORE_RG);
- _ref0 = (ref && pos < ref_len)? ref[pos] : 'N';
- ref16 = bam_nt16_table[_ref0];
- for (i = 0; i < gplp.n; ++i)
- bcf_call_glfgen(gplp.n_plp[i], gplp.plp[i], ref16, bca, bcr + i);
- bcf_call_combine(gplp.n, bcr, bca, ref16, &bc);
- bcf_call2bcf(tid, pos, &bc, b, bcr, conf->fmt_flag, 0, 0);
- bcf_write(bp, bh, b);
- bcf_destroy(b);
- // call indels
- if (!(conf->flag&MPLP_NO_INDEL) && total_depth < max_indel_depth && bcf_call_gap_prep(gplp.n, gplp.n_plp, gplp.plp, pos, bca, ref, rghash) >= 0) {
- for (i = 0; i < gplp.n; ++i)
- bcf_call_glfgen(gplp.n_plp[i], gplp.plp[i], -1, bca, bcr + i);
- if (bcf_call_combine(gplp.n, bcr, bca, -1, &bc) >= 0) {
- b = calloc(1, sizeof(bcf1_t));
- bcf_call2bcf(tid, pos, &bc, b, bcr, conf->fmt_flag, bca, ref);
- bcf_write(bp, bh, b);
- bcf_destroy(b);
- }
- }
- } else {
- printf("%s\t%d\t%c", h->target_name[tid], pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
- for (i = 0; i < n; ++i) {
- int j, cnt;
- for (j = cnt = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j;
- if (bam1_qual(p->b)[p->qpos] >= conf->min_baseQ) ++cnt;
- }
- printf("\t%d\t", cnt);
- if (n_plp[i] == 0) {
- printf("*\t*"); // FIXME: printf() is very slow...
- if (conf->flag & MPLP_PRINT_POS) printf("\t*");
- } else {
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j;
- if (bam1_qual(p->b)[p->qpos] >= conf->min_baseQ)
- pileup_seq(plp[i] + j, pos, ref_len, ref);
- }
- putchar('\t');
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j;
- int c = bam1_qual(p->b)[p->qpos];
- if (c >= conf->min_baseQ) {
- c = c + 33 < 126? c + 33 : 126;
- putchar(c);
- }
- }
- if (conf->flag & MPLP_PRINT_MAPQ) {
- putchar('\t');
- for (j = 0; j < n_plp[i]; ++j) {
- int c = plp[i][j].b->core.qual + 33;
- if (c > 126) c = 126;
- putchar(c);
- }
- }
- if (conf->flag & MPLP_PRINT_POS) {
- putchar('\t');
- for (j = 0; j < n_plp[i]; ++j) {
- if (j > 0) putchar(',');
- printf("%d", plp[i][j].qpos + 1); // FIXME: printf() is very slow...
- }
- }
- }
- }
- putchar('\n');
- }
- }
-
- bcf_close(bp);
- bam_smpl_destroy(sm); free(buf.s);
- for (i = 0; i < gplp.n; ++i) free(gplp.plp[i]);
- free(gplp.plp); free(gplp.n_plp); free(gplp.m_plp);
- bcf_call_del_rghash(rghash);
- bcf_hdr_destroy(bh); bcf_call_destroy(bca); free(bc.PL); free(bcr);
- bam_mplp_destroy(iter);
- bam_header_destroy(h);
- for (i = 0; i < n; ++i) {
- bam_close(data[i]->fp);
- if (data[i]->iter) bam_iter_destroy(data[i]->iter);
- free(data[i]);
- }
- free(data); free(plp); free(ref); free(n_plp);
- return 0;
-}
-
-#define MAX_PATH_LEN 1024
-int read_file_list(const char *file_list,int *n,char **argv[])
-{
- char buf[MAX_PATH_LEN];
- int len, nfiles = 0;
- char **files = NULL;
- struct stat sb;
-
- *n = 0;
- *argv = NULL;
-
- FILE *fh = fopen(file_list,"r");
- if ( !fh )
- {
- fprintf(stderr,"%s: %s\n", file_list,strerror(errno));
- return 1;
- }
-
- files = calloc(nfiles,sizeof(char*));
- nfiles = 0;
- while ( fgets(buf,MAX_PATH_LEN,fh) )
- {
- // allow empty lines and trailing spaces
- len = strlen(buf);
- while ( len>0 && isspace(buf[len-1]) ) len--;
- if ( !len ) continue;
-
- // check sanity of the file list
- buf[len] = 0;
- if (stat(buf, &sb) != 0)
- {
- // no such file, check if it is safe to print its name
- int i, safe_to_print = 1;
- for (i=0; i<len; i++)
- if (!isprint(buf[i])) { safe_to_print = 0; break; }
- if ( safe_to_print )
- fprintf(stderr,"The file list \"%s\" appears broken, could not locate: %s\n", file_list,buf);
- else
- fprintf(stderr,"Does the file \"%s\" really contain a list of files and do all exist?\n", file_list);
- return 1;
- }
-
- nfiles++;
- files = realloc(files,nfiles*sizeof(char*));
- files[nfiles-1] = strdup(buf);
- }
- fclose(fh);
- if ( !nfiles )
- {
- fprintf(stderr,"No files read from %s\n", file_list);
- return 1;
- }
- *argv = files;
- *n = nfiles;
- return 0;
-}
-#undef MAX_PATH_LEN
-
-int bam_mpileup(int argc, char *argv[])
-{
- int c;
- const char *file_list = NULL;
- char **fn = NULL;
- int nfiles = 0, use_orphan = 0;
- mplp_conf_t mplp;
- memset(&mplp, 0, sizeof(mplp_conf_t));
- mplp.max_mq = 60;
- mplp.min_baseQ = 13;
- mplp.capQ_thres = 0;
- mplp.max_depth = 250; mplp.max_indel_depth = 250;
- mplp.openQ = 40; mplp.extQ = 20; mplp.tandemQ = 100;
- mplp.min_frac = 0.002; mplp.min_support = 1;
- mplp.flag = MPLP_NO_ORPHAN | MPLP_REALN;
- static struct option lopts[] =
- {
- {"rf",1,0,1}, // require flag
- {"ff",1,0,2}, // filter flag
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "Agf:r:l:M:q:Q:uaRC:BDSd:L:b:P:po:e:h:Im:F:EG:6OsV1:2:",lopts,NULL)) >= 0) {
- switch (c) {
- case 1 : mplp.rflag_require = strtol(optarg,0,0); break;
- case 2 : mplp.rflag_filter = strtol(optarg,0,0); break;
- case 'f':
- mplp.fai = fai_load(optarg);
- if (mplp.fai == 0) return 1;
- mplp.fai_fname = optarg;
- break;
- case 'd': mplp.max_depth = atoi(optarg); break;
- case 'r': mplp.reg = strdup(optarg); break;
- case 'l': mplp.bed = bed_read(optarg); break;
- case 'P': mplp.pl_list = strdup(optarg); break;
- case 'p': mplp.flag |= MPLP_PER_SAMPLE; break;
- case 'g': mplp.flag |= MPLP_GLF; break;
- case 'u': mplp.flag |= MPLP_NO_COMP | MPLP_GLF; break;
- case 'a': mplp.flag |= MPLP_NO_ORPHAN | MPLP_REALN; break;
- case 'B': mplp.flag &= ~MPLP_REALN; break;
- case 'D': mplp.fmt_flag |= B2B_FMT_DP; break;
- case 'S': mplp.fmt_flag |= B2B_FMT_SP; break;
- case 'V': mplp.fmt_flag |= B2B_FMT_DV; break;
- case 'I': mplp.flag |= MPLP_NO_INDEL; break;
- case 'E': mplp.flag |= MPLP_REDO_BAQ; break;
- case '6': mplp.flag |= MPLP_ILLUMINA13; break;
- case 'R': mplp.flag |= MPLP_IGNORE_RG; break;
- case 's': mplp.flag |= MPLP_PRINT_MAPQ; break;
- case 'O': mplp.flag |= MPLP_PRINT_POS; break;
- case 'C': mplp.capQ_thres = atoi(optarg); break;
- case 'M': mplp.max_mq = atoi(optarg); break;
- case 'q': mplp.min_mq = atoi(optarg); break;
- case 'Q': mplp.min_baseQ = atoi(optarg); break;
- case 'b': file_list = optarg; break;
- case 'o': mplp.openQ = atoi(optarg); break;
- case 'e': mplp.extQ = atoi(optarg); break;
- case 'h': mplp.tandemQ = atoi(optarg); break;
- case 'A': use_orphan = 1; break;
- case 'F': mplp.min_frac = atof(optarg); break;
- case 'm': mplp.min_support = atoi(optarg); break;
- case 'L': mplp.max_indel_depth = atoi(optarg); break;
- case 'G': {
- FILE *fp_rg;
- char buf[1024];
- mplp.rghash = bcf_str2id_init();
- if ((fp_rg = fopen(optarg, "r")) == 0)
- fprintf(stderr, "(%s) Fail to open file %s. Continue anyway.\n", __func__, optarg);
- while (!feof(fp_rg) && fscanf(fp_rg, "%s", buf) > 0) // this is not a good style, but forgive me...
- bcf_str2id_add(mplp.rghash, strdup(buf));
- fclose(fp_rg);
- }
- break;
- }
- }
- if (use_orphan) mplp.flag &= ~MPLP_NO_ORPHAN;
- if (argc == 1) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools mpileup [options] in1.bam [in2.bam [...]]\n\n");
- fprintf(stderr, "Input options:\n\n");
- fprintf(stderr, " -6 assume the quality is in the Illumina-1.3+ encoding\n");
- fprintf(stderr, " -A count anomalous read pairs\n");
- fprintf(stderr, " -B disable BAQ computation\n");
- fprintf(stderr, " -b FILE list of input BAM filenames, one per line [null]\n");
- fprintf(stderr, " -C INT parameter for adjusting mapQ; 0 to disable [0]\n");
- fprintf(stderr, " -d INT max per-BAM depth to avoid excessive memory usage [%d]\n", mplp.max_depth);
- fprintf(stderr, " -E recalculate extended BAQ on the fly thus ignoring existing BQs\n");
- fprintf(stderr, " -f FILE faidx indexed reference sequence file [null]\n");
- fprintf(stderr, " -G FILE exclude read groups listed in FILE [null]\n");
- fprintf(stderr, " -l FILE list of positions (chr pos) or regions (BED) [null]\n");
- fprintf(stderr, " -M INT cap mapping quality at INT [%d]\n", mplp.max_mq);
- fprintf(stderr, " -r STR region in which pileup is generated [null]\n");
- fprintf(stderr, " -R ignore RG tags\n");
- fprintf(stderr, " -q INT skip alignments with mapQ smaller than INT [%d]\n", mplp.min_mq);
- fprintf(stderr, " -Q INT skip bases with baseQ/BAQ smaller than INT [%d]\n", mplp.min_baseQ);
- fprintf(stderr, " --rf INT required flags: skip reads with mask bits unset []\n");
- fprintf(stderr, " --ff INT filter flags: skip reads with mask bits set []\n");
- fprintf(stderr, "\nOutput options:\n\n");
- fprintf(stderr, " -D output per-sample DP in BCF (require -g/-u)\n");
- fprintf(stderr, " -g generate BCF output (genotype likelihoods)\n");
- fprintf(stderr, " -O output base positions on reads (disabled by -g/-u)\n");
- fprintf(stderr, " -s output mapping quality (disabled by -g/-u)\n");
- fprintf(stderr, " -S output per-sample strand bias P-value in BCF (require -g/-u)\n");
- fprintf(stderr, " -u generate uncompress BCF output\n");
- fprintf(stderr, "\nSNP/INDEL genotype likelihoods options (effective with `-g' or `-u'):\n\n");
- fprintf(stderr, " -e INT Phred-scaled gap extension seq error probability [%d]\n", mplp.extQ);
- fprintf(stderr, " -F FLOAT minimum fraction of gapped reads for candidates [%g]\n", mplp.min_frac);
- fprintf(stderr, " -h INT coefficient for homopolymer errors [%d]\n", mplp.tandemQ);
- fprintf(stderr, " -I do not perform indel calling\n");
- fprintf(stderr, " -L INT max per-sample depth for INDEL calling [%d]\n", mplp.max_indel_depth);
- fprintf(stderr, " -m INT minimum gapped reads for indel candidates [%d]\n", mplp.min_support);
- fprintf(stderr, " -o INT Phred-scaled gap open sequencing error probability [%d]\n", mplp.openQ);
- fprintf(stderr, " -p apply -m and -F per-sample to increase sensitivity\n");
- fprintf(stderr, " -P STR comma separated list of platforms for indels [all]\n");
- fprintf(stderr, "\n");
- fprintf(stderr, "Notes: Assuming diploid individuals.\n\n");
- return 1;
- }
- bam_no_B = 1;
- if (file_list) {
- if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
- mpileup(&mplp,nfiles,fn);
- for (c=0; c<nfiles; c++) free(fn[c]);
- free(fn);
- } else mpileup(&mplp, argc - optind, argv + optind);
- if (mplp.rghash) bcf_str2id_thorough_destroy(mplp.rghash);
- free(mplp.reg); free(mplp.pl_list);
- if (mplp.fai) fai_destroy(mplp.fai);
- if (mplp.bed) bed_destroy(mplp.bed);
- return 0;
-}
diff --git a/samtools-0.1.19/bam_reheader.c b/samtools-0.1.19/bam_reheader.c
deleted file mode 100644
index 6619428..0000000
--- a/samtools-0.1.19/bam_reheader.c
+++ /dev/null
@@ -1,62 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include "knetfile.h"
-#include "bgzf.h"
-#include "bam.h"
-
-#define BUF_SIZE 0x10000
-
-int bam_reheader(BGZF *in, const bam_header_t *h, int fd)
-{
- BGZF *fp;
- bam_header_t *old;
- int len;
- uint8_t *buf;
- if (in->is_write) return -1;
- buf = malloc(BUF_SIZE);
- old = bam_header_read(in);
- fp = bgzf_fdopen(fd, "w");
- bam_header_write(fp, h);
- if (in->block_offset < in->block_length) {
- bgzf_write(fp, in->uncompressed_block + in->block_offset, in->block_length - in->block_offset);
- bgzf_flush(fp);
- }
-#ifdef _USE_KNETFILE
- while ((len = knet_read(in->fp, buf, BUF_SIZE)) > 0)
- fwrite(buf, 1, len, fp->fp);
-#else
- while (!feof(in->file) && (len = fread(buf, 1, BUF_SIZE, in->file)) > 0)
- fwrite(buf, 1, len, fp->file);
-#endif
- free(buf);
- fp->block_offset = in->block_offset = 0;
- bgzf_close(fp);
- return 0;
-}
-
-int main_reheader(int argc, char *argv[])
-{
- bam_header_t *h;
- BGZF *in;
- if (argc != 3) {
- fprintf(stderr, "Usage: samtools reheader <in.header.sam> <in.bam>\n");
- return 1;
- }
- { // read the header
- tamFile fph = sam_open(argv[1]);
- if (fph == 0) {
- fprintf(stderr, "[%s] fail to read the header from %s.\n", __func__, argv[1]);
- return 1;
- }
- h = sam_header_read(fph);
- sam_close(fph);
- }
- in = strcmp(argv[2], "-")? bam_open(argv[2], "r") : bam_dopen(fileno(stdin), "r");
- if (in == 0) {
- fprintf(stderr, "[%s] fail to open file %s.\n", __func__, argv[2]);
- return 1;
- }
- bam_reheader(in, h, fileno(stdout));
- bgzf_close(in);
- return 0;
-}
diff --git a/samtools-0.1.19/bam_rmdup.c b/samtools-0.1.19/bam_rmdup.c
deleted file mode 100644
index f0d2b5d..0000000
--- a/samtools-0.1.19/bam_rmdup.c
+++ /dev/null
@@ -1,206 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <zlib.h>
-#include <unistd.h>
-#include "sam.h"
-
-typedef bam1_t *bam1_p;
-
-#include "khash.h"
-KHASH_SET_INIT_STR(name)
-KHASH_MAP_INIT_INT64(pos, bam1_p)
-
-#define BUFFER_SIZE 0x40000
-
-typedef struct {
- uint64_t n_checked, n_removed;
- khash_t(pos) *best_hash;
-} lib_aux_t;
-KHASH_MAP_INIT_STR(lib, lib_aux_t)
-
-typedef struct {
- int n, max;
- bam1_t **a;
-} tmp_stack_t;
-
-static inline void stack_insert(tmp_stack_t *stack, bam1_t *b)
-{
- if (stack->n == stack->max) {
- stack->max = stack->max? stack->max<<1 : 0x10000;
- stack->a = (bam1_t**)realloc(stack->a, sizeof(bam1_t*) * stack->max);
- }
- stack->a[stack->n++] = b;
-}
-
-static inline void dump_best(tmp_stack_t *stack, samfile_t *out)
-{
- int i;
- for (i = 0; i != stack->n; ++i) {
- samwrite(out, stack->a[i]);
- bam_destroy1(stack->a[i]);
- }
- stack->n = 0;
-}
-
-static void clear_del_set(khash_t(name) *del_set)
-{
- khint_t k;
- for (k = kh_begin(del_set); k < kh_end(del_set); ++k)
- if (kh_exist(del_set, k))
- free((char*)kh_key(del_set, k));
- kh_clear(name, del_set);
-}
-
-static lib_aux_t *get_aux(khash_t(lib) *aux, const char *lib)
-{
- khint_t k = kh_get(lib, aux, lib);
- if (k == kh_end(aux)) {
- int ret;
- char *p = strdup(lib);
- lib_aux_t *q;
- k = kh_put(lib, aux, p, &ret);
- q = &kh_val(aux, k);
- q->n_checked = q->n_removed = 0;
- q->best_hash = kh_init(pos);
- return q;
- } else return &kh_val(aux, k);
-}
-
-static void clear_best(khash_t(lib) *aux, int max)
-{
- khint_t k;
- for (k = kh_begin(aux); k != kh_end(aux); ++k) {
- if (kh_exist(aux, k)) {
- lib_aux_t *q = &kh_val(aux, k);
- if (kh_size(q->best_hash) >= max)
- kh_clear(pos, q->best_hash);
- }
- }
-}
-
-static inline int sum_qual(const bam1_t *b)
-{
- int i, q;
- uint8_t *qual = bam1_qual(b);
- for (i = q = 0; i < b->core.l_qseq; ++i) q += qual[i];
- return q;
-}
-
-void bam_rmdup_core(samfile_t *in, samfile_t *out)
-{
- bam1_t *b;
- int last_tid = -1, last_pos = -1;
- tmp_stack_t stack;
- khint_t k;
- khash_t(lib) *aux;
- khash_t(name) *del_set;
-
- aux = kh_init(lib);
- del_set = kh_init(name);
- b = bam_init1();
- memset(&stack, 0, sizeof(tmp_stack_t));
-
- kh_resize(name, del_set, 4 * BUFFER_SIZE);
- while (samread(in, b) >= 0) {
- bam1_core_t *c = &b->core;
- if (c->tid != last_tid || last_pos != c->pos) {
- dump_best(&stack, out); // write the result
- clear_best(aux, BUFFER_SIZE);
- if (c->tid != last_tid) {
- clear_best(aux, 0);
- if (kh_size(del_set)) { // check
- fprintf(stderr, "[bam_rmdup_core] %llu unmatched pairs\n", (long long)kh_size(del_set));
- clear_del_set(del_set);
- }
- if ((int)c->tid == -1) { // append unmapped reads
- samwrite(out, b);
- while (samread(in, b) >= 0) samwrite(out, b);
- break;
- }
- last_tid = c->tid;
- fprintf(stderr, "[bam_rmdup_core] processing reference %s...\n", in->header->target_name[c->tid]);
- }
- }
- if (!(c->flag&BAM_FPAIRED) || (c->flag&(BAM_FUNMAP|BAM_FMUNMAP)) || (c->mtid >= 0 && c->tid != c->mtid)) {
- samwrite(out, b);
- } else if (c->isize > 0) { // paired, head
- uint64_t key = (uint64_t)c->pos<<32 | c->isize;
- const char *lib;
- lib_aux_t *q;
- int ret;
- lib = bam_get_library(in->header, b);
- q = lib? get_aux(aux, lib) : get_aux(aux, "\t");
- ++q->n_checked;
- k = kh_put(pos, q->best_hash, key, &ret);
- if (ret == 0) { // found in best_hash
- bam1_t *p = kh_val(q->best_hash, k);
- ++q->n_removed;
- if (sum_qual(p) < sum_qual(b)) { // the current alignment is better; this can be accelerated in principle
- kh_put(name, del_set, strdup(bam1_qname(p)), &ret); // p will be removed
- bam_copy1(p, b); // replaced as b
- } else kh_put(name, del_set, strdup(bam1_qname(b)), &ret); // b will be removed
- if (ret == 0)
- fprintf(stderr, "[bam_rmdup_core] inconsistent BAM file for pair '%s'. Continue anyway.\n", bam1_qname(b));
- } else { // not found in best_hash
- kh_val(q->best_hash, k) = bam_dup1(b);
- stack_insert(&stack, kh_val(q->best_hash, k));
- }
- } else { // paired, tail
- k = kh_get(name, del_set, bam1_qname(b));
- if (k != kh_end(del_set)) {
- free((char*)kh_key(del_set, k));
- kh_del(name, del_set, k);
- } else samwrite(out, b);
- }
- last_pos = c->pos;
- }
-
- for (k = kh_begin(aux); k != kh_end(aux); ++k) {
- if (kh_exist(aux, k)) {
- lib_aux_t *q = &kh_val(aux, k);
- dump_best(&stack, out);
- fprintf(stderr, "[bam_rmdup_core] %lld / %lld = %.4lf in library '%s'\n", (long long)q->n_removed,
- (long long)q->n_checked, (double)q->n_removed/q->n_checked, kh_key(aux, k));
- kh_destroy(pos, q->best_hash);
- free((char*)kh_key(aux, k));
- }
- }
- kh_destroy(lib, aux);
-
- clear_del_set(del_set);
- kh_destroy(name, del_set);
- free(stack.a);
- bam_destroy1(b);
-}
-
-void bam_rmdupse_core(samfile_t *in, samfile_t *out, int force_se);
-
-int bam_rmdup(int argc, char *argv[])
-{
- int c, is_se = 0, force_se = 0;
- samfile_t *in, *out;
- while ((c = getopt(argc, argv, "sS")) >= 0) {
- switch (c) {
- case 's': is_se = 1; break;
- case 'S': force_se = is_se = 1; break;
- }
- }
- if (optind + 2 > argc) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools rmdup [-sS] <input.srt.bam> <output.bam>\n\n");
- fprintf(stderr, "Option: -s rmdup for SE reads\n");
- fprintf(stderr, " -S treat PE reads as SE in rmdup (force -s)\n\n");
- return 1;
- }
- in = samopen(argv[optind], "rb", 0);
- out = samopen(argv[optind+1], "wb", in->header);
- if (in == 0 || out == 0) {
- fprintf(stderr, "[bam_rmdup] fail to read/write input files\n");
- return 1;
- }
- if (is_se) bam_rmdupse_core(in, out, force_se);
- else bam_rmdup_core(in, out);
- samclose(in); samclose(out);
- return 0;
-}
diff --git a/samtools-0.1.19/bam_rmdupse.c b/samtools-0.1.19/bam_rmdupse.c
deleted file mode 100644
index e7dbdc7..0000000
--- a/samtools-0.1.19/bam_rmdupse.c
+++ /dev/null
@@ -1,159 +0,0 @@
-#include <math.h>
-#include "sam.h"
-#include "khash.h"
-#include "klist.h"
-
-#define QUEUE_CLEAR_SIZE 0x100000
-#define MAX_POS 0x7fffffff
-
-typedef struct {
- int endpos;
- uint32_t score:31, discarded:1;
- bam1_t *b;
-} elem_t, *elem_p;
-#define __free_elem(p) bam_destroy1((p)->data.b)
-KLIST_INIT(q, elem_t, __free_elem)
-typedef klist_t(q) queue_t;
-
-KHASH_MAP_INIT_INT(best, elem_p)
-typedef khash_t(best) besthash_t;
-
-typedef struct {
- uint64_t n_checked, n_removed;
- besthash_t *left, *rght;
-} lib_aux_t;
-KHASH_MAP_INIT_STR(lib, lib_aux_t)
-
-static lib_aux_t *get_aux(khash_t(lib) *aux, const char *lib)
-{
- khint_t k = kh_get(lib, aux, lib);
- if (k == kh_end(aux)) {
- int ret;
- char *p = strdup(lib);
- lib_aux_t *q;
- k = kh_put(lib, aux, p, &ret);
- q = &kh_val(aux, k);
- q->left = kh_init(best);
- q->rght = kh_init(best);
- q->n_checked = q->n_removed = 0;
- return q;
- } else return &kh_val(aux, k);
-}
-
-static inline int sum_qual(const bam1_t *b)
-{
- int i, q;
- uint8_t *qual = bam1_qual(b);
- for (i = q = 0; i < b->core.l_qseq; ++i) q += qual[i];
- return q;
-}
-
-static inline elem_t *push_queue(queue_t *queue, const bam1_t *b, int endpos, int score)
-{
- elem_t *p = kl_pushp(q, queue);
- p->discarded = 0;
- p->endpos = endpos; p->score = score;
- if (p->b == 0) p->b = bam_init1();
- bam_copy1(p->b, b);
- return p;
-}
-
-static void clear_besthash(besthash_t *h, int32_t pos)
-{
- khint_t k;
- for (k = kh_begin(h); k != kh_end(h); ++k)
- if (kh_exist(h, k) && kh_val(h, k)->endpos <= pos)
- kh_del(best, h, k);
-}
-
-static void dump_alignment(samfile_t *out, queue_t *queue, int32_t pos, khash_t(lib) *h)
-{
- if (queue->size > QUEUE_CLEAR_SIZE || pos == MAX_POS) {
- khint_t k;
- while (1) {
- elem_t *q;
- if (queue->head == queue->tail) break;
- q = &kl_val(queue->head);
- if (q->discarded) {
- q->b->data_len = 0;
- kl_shift(q, queue, 0);
- continue;
- }
- if ((q->b->core.flag&BAM_FREVERSE) && q->endpos > pos) break;
- samwrite(out, q->b);
- q->b->data_len = 0;
- kl_shift(q, queue, 0);
- }
- for (k = kh_begin(h); k != kh_end(h); ++k) {
- if (kh_exist(h, k)) {
- clear_besthash(kh_val(h, k).left, pos);
- clear_besthash(kh_val(h, k).rght, pos);
- }
- }
- }
-}
-
-void bam_rmdupse_core(samfile_t *in, samfile_t *out, int force_se)
-{
- bam1_t *b;
- queue_t *queue;
- khint_t k;
- int last_tid = -2;
- khash_t(lib) *aux;
-
- aux = kh_init(lib);
- b = bam_init1();
- queue = kl_init(q);
- while (samread(in, b) >= 0) {
- bam1_core_t *c = &b->core;
- int endpos = bam_calend(c, bam1_cigar(b));
- int score = sum_qual(b);
-
- if (last_tid != c->tid) {
- if (last_tid >= 0) dump_alignment(out, queue, MAX_POS, aux);
- last_tid = c->tid;
- } else dump_alignment(out, queue, c->pos, aux);
- if ((c->flag&BAM_FUNMAP) || ((c->flag&BAM_FPAIRED) && !force_se)) {
- push_queue(queue, b, endpos, score);
- } else {
- const char *lib;
- lib_aux_t *q;
- besthash_t *h;
- uint32_t key;
- int ret;
- lib = bam_get_library(in->header, b);
- q = lib? get_aux(aux, lib) : get_aux(aux, "\t");
- ++q->n_checked;
- h = (c->flag&BAM_FREVERSE)? q->rght : q->left;
- key = (c->flag&BAM_FREVERSE)? endpos : c->pos;
- k = kh_put(best, h, key, &ret);
- if (ret == 0) { // in the hash table
- elem_t *p = kh_val(h, k);
- ++q->n_removed;
- if (p->score < score) {
- if (c->flag&BAM_FREVERSE) { // mark "discarded" and push the queue
- p->discarded = 1;
- kh_val(h, k) = push_queue(queue, b, endpos, score);
- } else { // replace
- p->score = score; p->endpos = endpos;
- bam_copy1(p->b, b);
- }
- } // otherwise, discard the alignment
- } else kh_val(h, k) = push_queue(queue, b, endpos, score);
- }
- }
- dump_alignment(out, queue, MAX_POS, aux);
-
- for (k = kh_begin(aux); k != kh_end(aux); ++k) {
- if (kh_exist(aux, k)) {
- lib_aux_t *q = &kh_val(aux, k);
- fprintf(stderr, "[bam_rmdupse_core] %lld / %lld = %.4lf in library '%s'\n", (long long)q->n_removed,
- (long long)q->n_checked, (double)q->n_removed/q->n_checked, kh_key(aux, k));
- kh_destroy(best, q->left); kh_destroy(best, q->rght);
- free((char*)kh_key(aux, k));
- }
- }
- kh_destroy(lib, aux);
- bam_destroy1(b);
- kl_destroy(q, queue);
-}
diff --git a/samtools-0.1.19/bam_sort.c b/samtools-0.1.19/bam_sort.c
deleted file mode 100644
index c46bce3..0000000
--- a/samtools-0.1.19/bam_sort.c
+++ /dev/null
@@ -1,571 +0,0 @@
-#include <stdlib.h>
-#include <ctype.h>
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-#include "bam.h"
-#include "ksort.h"
-
-static int g_is_by_qname = 0;
-
-static int strnum_cmp(const char *_a, const char *_b)
-{
- const unsigned char *a = (const unsigned char*)_a, *b = (const unsigned char*)_b;
- const unsigned char *pa = a, *pb = b;
- while (*pa && *pb) {
- if (isdigit(*pa) && isdigit(*pb)) {
- while (*pa == '0') ++pa;
- while (*pb == '0') ++pb;
- while (isdigit(*pa) && isdigit(*pb) && *pa == *pb) ++pa, ++pb;
- if (isdigit(*pa) && isdigit(*pb)) {
- int i = 0;
- while (isdigit(pa[i]) && isdigit(pb[i])) ++i;
- return isdigit(pa[i])? 1 : isdigit(pb[i])? -1 : (int)*pa - (int)*pb;
- } else if (isdigit(*pa)) return 1;
- else if (isdigit(*pb)) return -1;
- else if (pa - a != pb - b) return pa - a < pb - b? 1 : -1;
- } else {
- if (*pa != *pb) return (int)*pa - (int)*pb;
- ++pa; ++pb;
- }
- }
- return *pa? 1 : *pb? -1 : 0;
-}
-
-#define HEAP_EMPTY 0xffffffffffffffffull
-
-typedef struct {
- int i;
- uint64_t pos, idx;
- bam1_t *b;
-} heap1_t;
-
-#define __pos_cmp(a, b) ((a).pos > (b).pos || ((a).pos == (b).pos && ((a).i > (b).i || ((a).i == (b).i && (a).idx > (b).idx))))
-
-static inline int heap_lt(const heap1_t a, const heap1_t b)
-{
- if (g_is_by_qname) {
- int t;
- if (a.b == 0 || b.b == 0) return a.b == 0? 1 : 0;
- t = strnum_cmp(bam1_qname(a.b), bam1_qname(b.b));
- return (t > 0 || (t == 0 && (a.b->core.flag&0xc0) > (b.b->core.flag&0xc0)));
- } else return __pos_cmp(a, b);
-}
-
-KSORT_INIT(heap, heap1_t, heap_lt)
-
-static void swap_header_targets(bam_header_t *h1, bam_header_t *h2)
-{
- bam_header_t t;
- t.n_targets = h1->n_targets, h1->n_targets = h2->n_targets, h2->n_targets = t.n_targets;
- t.target_name = h1->target_name, h1->target_name = h2->target_name, h2->target_name = t.target_name;
- t.target_len = h1->target_len, h1->target_len = h2->target_len, h2->target_len = t.target_len;
-}
-
-static void swap_header_text(bam_header_t *h1, bam_header_t *h2)
-{
- int tempi;
- char *temps;
- tempi = h1->l_text, h1->l_text = h2->l_text, h2->l_text = tempi;
- temps = h1->text, h1->text = h2->text, h2->text = temps;
-}
-
-#define MERGE_RG 1
-#define MERGE_UNCOMP 2
-#define MERGE_LEVEL1 4
-#define MERGE_FORCE 8
-
-/*!
- @abstract Merge multiple sorted BAM.
- @param is_by_qname whether to sort by query name
- @param out output BAM file name
- @param headers name of SAM file from which to copy '@' header lines,
- or NULL to copy them from the first file to be merged
- @param n number of files to be merged
- @param fn names of files to be merged
-
- @discussion Padding information may NOT correctly maintained. This
- function is NOT thread safe.
- */
-int bam_merge_core2(int by_qname, const char *out, const char *headers, int n, char * const *fn, int flag, const char *reg, int n_threads, int level)
-{
- bamFile fpout, *fp;
- heap1_t *heap;
- bam_header_t *hout = 0;
- bam_header_t *hheaders = NULL;
- int i, j, *RG_len = 0;
- uint64_t idx = 0;
- char **RG = 0, mode[8];
- bam_iter_t *iter = 0;
-
- if (headers) {
- tamFile fpheaders = sam_open(headers);
- if (fpheaders == 0) {
- const char *message = strerror(errno);
- fprintf(stderr, "[bam_merge_core] cannot open '%s': %s\n", headers, message);
- return -1;
- }
- hheaders = sam_header_read(fpheaders);
- sam_close(fpheaders);
- }
-
- g_is_by_qname = by_qname;
- fp = (bamFile*)calloc(n, sizeof(bamFile));
- heap = (heap1_t*)calloc(n, sizeof(heap1_t));
- iter = (bam_iter_t*)calloc(n, sizeof(bam_iter_t));
- // prepare RG tag
- if (flag & MERGE_RG) {
- RG = (char**)calloc(n, sizeof(void*));
- RG_len = (int*)calloc(n, sizeof(int));
- for (i = 0; i != n; ++i) {
- int l = strlen(fn[i]);
- const char *s = fn[i];
- if (l > 4 && strcmp(s + l - 4, ".bam") == 0) l -= 4;
- for (j = l - 1; j >= 0; --j) if (s[j] == '/') break;
- ++j; l -= j;
- RG[i] = calloc(l + 1, 1);
- RG_len[i] = l;
- strncpy(RG[i], s + j, l);
- }
- }
- // read the first
- for (i = 0; i != n; ++i) {
- bam_header_t *hin;
- fp[i] = bam_open(fn[i], "r");
- if (fp[i] == 0) {
- int j;
- fprintf(stderr, "[bam_merge_core] fail to open file %s\n", fn[i]);
- for (j = 0; j < i; ++j) bam_close(fp[j]);
- free(fp); free(heap);
- // FIXME: possible memory leak
- return -1;
- }
- hin = bam_header_read(fp[i]);
- if (i == 0) { // the first BAM
- hout = hin;
- } else { // validate multiple baf
- int min_n_targets = hout->n_targets;
- if (hin->n_targets < min_n_targets) min_n_targets = hin->n_targets;
-
- for (j = 0; j < min_n_targets; ++j)
- if (strcmp(hout->target_name[j], hin->target_name[j]) != 0) {
- fprintf(stderr, "[bam_merge_core] different target sequence name: '%s' != '%s' in file '%s'\n",
- hout->target_name[j], hin->target_name[j], fn[i]);
- return -1;
- }
-
- // If this input file has additional target reference sequences,
- // add them to the headers to be output
- if (hin->n_targets > hout->n_targets) {
- swap_header_targets(hout, hin);
- // FIXME Possibly we should also create @SQ text headers
- // for the newly added reference sequences
- }
-
- bam_header_destroy(hin);
- }
- }
-
- if (hheaders) {
- // If the text headers to be swapped in include any @SQ headers,
- // check that they are consistent with the existing binary list
- // of reference information.
- if (hheaders->n_targets > 0) {
- if (hout->n_targets != hheaders->n_targets) {
- fprintf(stderr, "[bam_merge_core] number of @SQ headers in '%s' differs from number of target sequences\n", headers);
- if (!reg) return -1;
- }
- for (j = 0; j < hout->n_targets; ++j)
- if (strcmp(hout->target_name[j], hheaders->target_name[j]) != 0) {
- fprintf(stderr, "[bam_merge_core] @SQ header '%s' in '%s' differs from target sequence\n", hheaders->target_name[j], headers);
- if (!reg) return -1;
- }
- }
-
- swap_header_text(hout, hheaders);
- bam_header_destroy(hheaders);
- }
-
- if (reg) {
- int tid, beg, end;
- if (bam_parse_region(hout, reg, &tid, &beg, &end) < 0) {
- fprintf(stderr, "[%s] Malformated region string or undefined reference name\n", __func__);
- return -1;
- }
- for (i = 0; i < n; ++i) {
- bam_index_t *idx;
- idx = bam_index_load(fn[i]);
- iter[i] = bam_iter_query(idx, tid, beg, end);
- bam_index_destroy(idx);
- }
- }
-
- for (i = 0; i < n; ++i) {
- heap1_t *h = heap + i;
- h->i = i;
- h->b = (bam1_t*)calloc(1, sizeof(bam1_t));
- if (bam_iter_read(fp[i], iter[i], h->b) >= 0) {
- h->pos = ((uint64_t)h->b->core.tid<<32) | (uint32_t)((int32_t)h->b->core.pos+1)<<1 | bam1_strand(h->b);
- h->idx = idx++;
- }
- else h->pos = HEAP_EMPTY;
- }
- if (flag & MERGE_UNCOMP) level = 0;
- else if (flag & MERGE_LEVEL1) level = 1;
- strcpy(mode, "w");
- if (level >= 0) sprintf(mode + 1, "%d", level < 9? level : 9);
- if ((fpout = strcmp(out, "-")? bam_open(out, "w") : bam_dopen(fileno(stdout), "w")) == 0) {
- fprintf(stderr, "[%s] fail to create the output file.\n", __func__);
- return -1;
- }
- bam_header_write(fpout, hout);
- bam_header_destroy(hout);
- if (!(flag & MERGE_UNCOMP)) bgzf_mt(fpout, n_threads, 256);
-
- ks_heapmake(heap, n, heap);
- while (heap->pos != HEAP_EMPTY) {
- bam1_t *b = heap->b;
- if (flag & MERGE_RG) {
- uint8_t *rg = bam_aux_get(b, "RG");
- if (rg) bam_aux_del(b, rg);
- bam_aux_append(b, "RG", 'Z', RG_len[heap->i] + 1, (uint8_t*)RG[heap->i]);
- }
- bam_write1_core(fpout, &b->core, b->data_len, b->data);
- if ((j = bam_iter_read(fp[heap->i], iter[heap->i], b)) >= 0) {
- heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1)<<1 | bam1_strand(b);
- heap->idx = idx++;
- } else if (j == -1) {
- heap->pos = HEAP_EMPTY;
- free(heap->b->data); free(heap->b);
- heap->b = 0;
- } else fprintf(stderr, "[bam_merge_core] '%s' is truncated. Continue anyway.\n", fn[heap->i]);
- ks_heapadjust(heap, 0, n, heap);
- }
-
- if (flag & MERGE_RG) {
- for (i = 0; i != n; ++i) free(RG[i]);
- free(RG); free(RG_len);
- }
- for (i = 0; i != n; ++i) {
- bam_iter_destroy(iter[i]);
- bam_close(fp[i]);
- }
- bam_close(fpout);
- free(fp); free(heap); free(iter);
- return 0;
-}
-
-int bam_merge_core(int by_qname, const char *out, const char *headers, int n, char * const *fn, int flag, const char *reg)
-{
- return bam_merge_core2(by_qname, out, headers, n, fn, flag, reg, 0, -1);
-}
-
-int bam_merge(int argc, char *argv[])
-{
- int c, is_by_qname = 0, flag = 0, ret = 0, n_threads = 0, level = -1;
- char *fn_headers = NULL, *reg = 0;
-
- while ((c = getopt(argc, argv, "h:nru1R:f@:l:")) >= 0) {
- switch (c) {
- case 'r': flag |= MERGE_RG; break;
- case 'f': flag |= MERGE_FORCE; break;
- case 'h': fn_headers = strdup(optarg); break;
- case 'n': is_by_qname = 1; break;
- case '1': flag |= MERGE_LEVEL1; break;
- case 'u': flag |= MERGE_UNCOMP; break;
- case 'R': reg = strdup(optarg); break;
- case 'l': level = atoi(optarg); break;
- case '@': n_threads = atoi(optarg); break;
- }
- }
- if (optind + 2 >= argc) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools merge [-nr] [-h inh.sam] <out.bam> <in1.bam> <in2.bam> [...]\n\n");
- fprintf(stderr, "Options: -n sort by read names\n");
- fprintf(stderr, " -r attach RG tag (inferred from file names)\n");
- fprintf(stderr, " -u uncompressed BAM output\n");
- fprintf(stderr, " -f overwrite the output BAM if exist\n");
- fprintf(stderr, " -1 compress level 1\n");
- fprintf(stderr, " -l INT compression level, from 0 to 9 [-1]\n");
- fprintf(stderr, " -@ INT number of BAM compression threads [0]\n");
- fprintf(stderr, " -R STR merge file in the specified region STR [all]\n");
- fprintf(stderr, " -h FILE copy the header in FILE to <out.bam> [in1.bam]\n\n");
- fprintf(stderr, "Note: Samtools' merge does not reconstruct the @RG dictionary in the header. Users\n");
- fprintf(stderr, " must provide the correct header with -h, or uses Picard which properly maintains\n");
- fprintf(stderr, " the header dictionary in merging.\n\n");
- return 1;
- }
- if (!(flag & MERGE_FORCE) && strcmp(argv[optind], "-")) {
- FILE *fp = fopen(argv[optind], "rb");
- if (fp != NULL) {
- fclose(fp);
- fprintf(stderr, "[%s] File '%s' exists. Please apply '-f' to overwrite. Abort.\n", __func__, argv[optind]);
- return 1;
- }
- }
- if (bam_merge_core2(is_by_qname, argv[optind], fn_headers, argc - optind - 1, argv + optind + 1, flag, reg, n_threads, level) < 0) ret = 1;
- free(reg);
- free(fn_headers);
- return ret;
-}
-
-/***************
- * BAM sorting *
- ***************/
-
-#include <pthread.h>
-
-typedef bam1_t *bam1_p;
-
-static int change_SO(bam_header_t *h, const char *so)
-{
- char *p, *q, *beg = 0, *end = 0, *newtext;
- if (h->l_text > 3) {
- if (strncmp(h->text, "@HD", 3) == 0) {
- if ((p = strchr(h->text, '\n')) == 0) return -1;
- *p = '\0';
- if ((q = strstr(h->text, "\tSO:")) != 0) {
- *p = '\n'; // change back
- if (strncmp(q + 4, so, p - q - 4) != 0) {
- beg = q;
- for (q += 4; *q != '\n' && *q != '\t'; ++q);
- end = q;
- } else return 0; // no need to change
- } else beg = end = p, *p = '\n';
- }
- }
- if (beg == 0) { // no @HD
- h->l_text += strlen(so) + 15;
- newtext = malloc(h->l_text + 1);
- sprintf(newtext, "@HD\tVN:1.3\tSO:%s\n", so);
- strcat(newtext, h->text);
- } else { // has @HD but different or no SO
- h->l_text = (beg - h->text) + (4 + strlen(so)) + (h->text + h->l_text - end);
- newtext = malloc(h->l_text + 1);
- strncpy(newtext, h->text, beg - h->text);
- sprintf(newtext + (beg - h->text), "\tSO:%s", so);
- strcat(newtext, end);
- }
- free(h->text);
- h->text = newtext;
- return 0;
-}
-
-static inline int bam1_lt(const bam1_p a, const bam1_p b)
-{
- if (g_is_by_qname) {
- int t = strnum_cmp(bam1_qname(a), bam1_qname(b));
- return (t < 0 || (t == 0 && (a->core.flag&0xc0) < (b->core.flag&0xc0)));
- } else return (((uint64_t)a->core.tid<<32|(a->core.pos+1)<<1|bam1_strand(a)) < ((uint64_t)b->core.tid<<32|(b->core.pos+1)<<1|bam1_strand(b)));
-}
-KSORT_INIT(sort, bam1_p, bam1_lt)
-
-typedef struct {
- size_t buf_len;
- const char *prefix;
- bam1_p *buf;
- const bam_header_t *h;
- int index;
-} worker_t;
-
-static void write_buffer(const char *fn, const char *mode, size_t l, bam1_p *buf, const bam_header_t *h, int n_threads)
-{
- size_t i;
- bamFile fp;
- fp = strcmp(fn, "-")? bam_open(fn, mode) : bam_dopen(fileno(stdout), mode);
- if (fp == 0) return;
- bam_header_write(fp, h);
- if (n_threads > 1) bgzf_mt(fp, n_threads, 256);
- for (i = 0; i < l; ++i)
- bam_write1_core(fp, &buf[i]->core, buf[i]->data_len, buf[i]->data);
- bam_close(fp);
-}
-
-static void *worker(void *data)
-{
- worker_t *w = (worker_t*)data;
- char *name;
- ks_mergesort(sort, w->buf_len, w->buf, 0);
- name = (char*)calloc(strlen(w->prefix) + 20, 1);
- sprintf(name, "%s.%.4d.bam", w->prefix, w->index);
- write_buffer(name, "w1", w->buf_len, w->buf, w->h, 0);
- free(name);
- return 0;
-}
-
-static int sort_blocks(int n_files, size_t k, bam1_p *buf, const char *prefix, const bam_header_t *h, int n_threads)
-{
- int i;
- size_t rest;
- bam1_p *b;
- pthread_t *tid;
- pthread_attr_t attr;
- worker_t *w;
-
- if (n_threads < 1) n_threads = 1;
- if (k < n_threads * 64) n_threads = 1; // use a single thread if we only sort a small batch of records
- pthread_attr_init(&attr);
- pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
- w = calloc(n_threads, sizeof(worker_t));
- tid = calloc(n_threads, sizeof(pthread_t));
- b = buf; rest = k;
- for (i = 0; i < n_threads; ++i) {
- w[i].buf_len = rest / (n_threads - i);
- w[i].buf = b;
- w[i].prefix = prefix;
- w[i].h = h;
- w[i].index = n_files + i;
- b += w[i].buf_len; rest -= w[i].buf_len;
- pthread_create(&tid[i], &attr, worker, &w[i]);
- }
- for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
- free(tid); free(w);
- return n_files + n_threads;
-}
-
-/*!
- @abstract Sort an unsorted BAM file based on the chromosome order
- and the leftmost position of an alignment
-
- @param is_by_qname whether to sort by query name
- @param fn name of the file to be sorted
- @param prefix prefix of the output and the temporary files; upon
- sucessess, prefix.bam will be written.
- @param max_mem approxiate maximum memory (very inaccurate)
- @param full_path the given output path is the full path and not just the prefix
-
- @discussion It may create multiple temporary subalignment files
- and then merge them by calling bam_merge_core(). This function is
- NOT thread safe.
- */
-void bam_sort_core_ext(int is_by_qname, const char *fn, const char *prefix, size_t _max_mem, int is_stdout, int n_threads, int level, int full_path)
-{
- int ret, i, n_files = 0;
- size_t mem, max_k, k, max_mem;
- bam_header_t *header;
- bamFile fp;
- bam1_t *b, **buf;
- char *fnout = 0;
- char const *suffix = ".bam";
- if (full_path) suffix += 4;
-
- if (n_threads < 2) n_threads = 1;
- g_is_by_qname = is_by_qname;
- max_k = k = 0; mem = 0;
- max_mem = _max_mem * n_threads;
- buf = 0;
- fp = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r");
- if (fp == 0) {
- fprintf(stderr, "[bam_sort_core] fail to open file %s\n", fn);
- return;
- }
- header = bam_header_read(fp);
- if (is_by_qname) change_SO(header, "queryname");
- else change_SO(header, "coordinate");
- // write sub files
- for (;;) {
- if (k == max_k) {
- size_t old_max = max_k;
- max_k = max_k? max_k<<1 : 0x10000;
- buf = realloc(buf, max_k * sizeof(void*));
- memset(buf + old_max, 0, sizeof(void*) * (max_k - old_max));
- }
- if (buf[k] == 0) buf[k] = (bam1_t*)calloc(1, sizeof(bam1_t));
- b = buf[k];
- if ((ret = bam_read1(fp, b)) < 0) break;
- if (b->data_len < b->m_data>>2) { // shrink
- b->m_data = b->data_len;
- kroundup32(b->m_data);
- b->data = realloc(b->data, b->m_data);
- }
- mem += sizeof(bam1_t) + b->m_data + sizeof(void*) + sizeof(void*); // two sizeof(void*) for the data allocated to pointer arrays
- ++k;
- if (mem >= max_mem) {
- n_files = sort_blocks(n_files, k, buf, prefix, header, n_threads);
- mem = k = 0;
- }
- }
- if (ret != -1)
- fprintf(stderr, "[bam_sort_core] truncated file. Continue anyway.\n");
- // output file name
- fnout = calloc(strlen(prefix) + 20, 1);
- if (is_stdout) sprintf(fnout, "-");
- else sprintf(fnout, "%s%s", prefix, suffix);
- // write the final output
- if (n_files == 0) { // a single block
- char mode[8];
- strcpy(mode, "w");
- if (level >= 0) sprintf(mode + 1, "%d", level < 9? level : 9);
- ks_mergesort(sort, k, buf, 0);
- write_buffer(fnout, mode, k, buf, header, n_threads);
- } else { // then merge
- char **fns;
- n_files = sort_blocks(n_files, k, buf, prefix, header, n_threads);
- fprintf(stderr, "[bam_sort_core] merging from %d files...\n", n_files);
- fns = (char**)calloc(n_files, sizeof(char*));
- for (i = 0; i < n_files; ++i) {
- fns[i] = (char*)calloc(strlen(prefix) + 20, 1);
- sprintf(fns[i], "%s.%.4d%s", prefix, i, suffix);
- }
- bam_merge_core2(is_by_qname, fnout, 0, n_files, fns, 0, 0, n_threads, level);
- for (i = 0; i < n_files; ++i) {
- unlink(fns[i]);
- free(fns[i]);
- }
- free(fns);
- }
- free(fnout);
- // free
- for (k = 0; k < max_k; ++k) {
- if (!buf[k]) continue;
- free(buf[k]->data);
- free(buf[k]);
- }
- free(buf);
- bam_header_destroy(header);
- bam_close(fp);
-}
-
-void bam_sort_core(int is_by_qname, const char *fn, const char *prefix, size_t max_mem)
-{
- bam_sort_core_ext(is_by_qname, fn, prefix, max_mem, 0, 0, -1, 0);
-}
-
-int bam_sort(int argc, char *argv[])
-{
- size_t max_mem = 768<<20; // 512MB
- int c, is_by_qname = 0, is_stdout = 0, n_threads = 0, level = -1, full_path = 0;
- while ((c = getopt(argc, argv, "fnom:@:l:")) >= 0) {
- switch (c) {
- case 'f': full_path = 1; break;
- case 'o': is_stdout = 1; break;
- case 'n': is_by_qname = 1; break;
- case 'm': {
- char *q;
- max_mem = strtol(optarg, &q, 0);
- if (*q == 'k' || *q == 'K') max_mem <<= 10;
- else if (*q == 'm' || *q == 'M') max_mem <<= 20;
- else if (*q == 'g' || *q == 'G') max_mem <<= 30;
- break;
- }
- case '@': n_threads = atoi(optarg); break;
- case 'l': level = atoi(optarg); break;
- }
- }
- if (optind + 2 > argc) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools sort [options] <in.bam> <out.prefix>\n\n");
- fprintf(stderr, "Options: -n sort by read name\n");
- fprintf(stderr, " -f use <out.prefix> as full file name instead of prefix\n");
- fprintf(stderr, " -o final output to stdout\n");
- fprintf(stderr, " -l INT compression level, from 0 to 9 [-1]\n");
- fprintf(stderr, " -@ INT number of sorting and compression threads [1]\n");
- fprintf(stderr, " -m INT max memory per thread; suffix K/M/G recognized [768M]\n");
- fprintf(stderr, "\n");
- return 1;
- }
- bam_sort_core_ext(is_by_qname, argv[optind], argv[optind+1], max_mem, is_stdout, n_threads, level, full_path);
- return 0;
-}
diff --git a/samtools-0.1.19/bam_stat.c b/samtools-0.1.19/bam_stat.c
deleted file mode 100644
index f2de0f1..0000000
--- a/samtools-0.1.19/bam_stat.c
+++ /dev/null
@@ -1,77 +0,0 @@
-#include <unistd.h>
-#include <assert.h>
-#include "bam.h"
-
-typedef struct {
- long long n_reads[2], n_mapped[2], n_pair_all[2], n_pair_map[2], n_pair_good[2];
- long long n_sgltn[2], n_read1[2], n_read2[2];
- long long n_dup[2];
- long long n_diffchr[2], n_diffhigh[2];
-} bam_flagstat_t;
-
-#define flagstat_loop(s, c) do { \
- int w = ((c)->flag & BAM_FQCFAIL)? 1 : 0; \
- ++(s)->n_reads[w]; \
- if ((c)->flag & BAM_FPAIRED) { \
- ++(s)->n_pair_all[w]; \
- if ((c)->flag & BAM_FPROPER_PAIR) ++(s)->n_pair_good[w]; \
- if ((c)->flag & BAM_FREAD1) ++(s)->n_read1[w]; \
- if ((c)->flag & BAM_FREAD2) ++(s)->n_read2[w]; \
- if (((c)->flag & BAM_FMUNMAP) && !((c)->flag & BAM_FUNMAP)) ++(s)->n_sgltn[w]; \
- if (!((c)->flag & BAM_FUNMAP) && !((c)->flag & BAM_FMUNMAP)) { \
- ++(s)->n_pair_map[w]; \
- if ((c)->mtid != (c)->tid) { \
- ++(s)->n_diffchr[w]; \
- if ((c)->qual >= 5) ++(s)->n_diffhigh[w]; \
- } \
- } \
- } \
- if (!((c)->flag & BAM_FUNMAP)) ++(s)->n_mapped[w]; \
- if ((c)->flag & BAM_FDUP) ++(s)->n_dup[w]; \
- } while (0)
-
-bam_flagstat_t *bam_flagstat_core(bamFile fp)
-{
- bam_flagstat_t *s;
- bam1_t *b;
- bam1_core_t *c;
- int ret;
- s = (bam_flagstat_t*)calloc(1, sizeof(bam_flagstat_t));
- b = bam_init1();
- c = &b->core;
- while ((ret = bam_read1(fp, b)) >= 0)
- flagstat_loop(s, c);
- bam_destroy1(b);
- if (ret != -1)
- fprintf(stderr, "[bam_flagstat_core] Truncated file? Continue anyway.\n");
- return s;
-}
-int bam_flagstat(int argc, char *argv[])
-{
- bamFile fp;
- bam_header_t *header;
- bam_flagstat_t *s;
- if (argc == optind) {
- fprintf(stderr, "Usage: samtools flagstat <in.bam>\n");
- return 1;
- }
- fp = strcmp(argv[optind], "-")? bam_open(argv[optind], "r") : bam_dopen(fileno(stdin), "r");
- assert(fp);
- header = bam_header_read(fp);
- s = bam_flagstat_core(fp);
- printf("%lld + %lld in total (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
- printf("%lld + %lld duplicates\n", s->n_dup[0], s->n_dup[1]);
- printf("%lld + %lld mapped (%.2f%%:%.2f%%)\n", s->n_mapped[0], s->n_mapped[1], (float)s->n_mapped[0] / s->n_reads[0] * 100.0, (float)s->n_mapped[1] / s->n_reads[1] * 100.0);
- printf("%lld + %lld paired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
- printf("%lld + %lld read1\n", s->n_read1[0], s->n_read1[1]);
- printf("%lld + %lld read2\n", s->n_read2[0], s->n_read2[1]);
- printf("%lld + %lld properly paired (%.2f%%:%.2f%%)\n", s->n_pair_good[0], s->n_pair_good[1], (float)s->n_pair_good[0] / s->n_pair_all[0] * 100.0, (float)s->n_pair_good[1] / s->n_pair_all[1] * 100.0);
- printf("%lld + %lld with itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
- printf("%lld + %lld singletons (%.2f%%:%.2f%%)\n", s->n_sgltn[0], s->n_sgltn[1], (float)s->n_sgltn[0] / s->n_pair_all[0] * 100.0, (float)s->n_sgltn[1] / s->n_pair_all[1] * 100.0);
- printf("%lld + %lld with mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
- printf("%lld + %lld with mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
- free(s);
- bam_header_destroy(header);
- bam_close(fp);
- return 0;
-}
diff --git a/samtools-0.1.19/bam_tview.c b/samtools-0.1.19/bam_tview.c
deleted file mode 100644
index 06d5e33..0000000
--- a/samtools-0.1.19/bam_tview.c
+++ /dev/null
@@ -1,368 +0,0 @@
-#include <assert.h>
-#include "bam_tview.h"
-
-int base_tv_init(tview_t* tv,const char *fn, const char *fn_fa, const char *samples)
- {
- assert(tv!=NULL);
- assert(fn!=NULL);
- tv->mrow = 24; tv->mcol = 80;
- tv->color_for = TV_COLOR_MAPQ;
- tv->is_dot = 1;
-
- tv->fp = bam_open(fn, "r");
- if(tv->fp==0)
- {
- fprintf(stderr,"bam_open %s. %s\n", fn,fn_fa);
- exit(EXIT_FAILURE);
- }
- bgzf_set_cache_size(tv->fp, 8 * 1024 *1024);
- assert(tv->fp);
-
- tv->header = bam_header_read(tv->fp);
- if(tv->header==0)
- {
- fprintf(stderr,"Cannot read '%s'.\n", fn);
- exit(EXIT_FAILURE);
- }
- tv->idx = bam_index_load(fn);
- if (tv->idx == 0)
- {
- fprintf(stderr,"Cannot read index for '%s'.\n", fn);
- exit(EXIT_FAILURE);
- }
- tv->lplbuf = bam_lplbuf_init(tv_pl_func, tv);
- if (fn_fa) tv->fai = fai_load(fn_fa);
- tv->bca = bcf_call_init(0.83, 13);
- tv->ins = 1;
-
- if ( samples )
- {
- if ( !tv->header->dict ) tv->header->dict = sam_header_parse2(tv->header->text);
- void *iter = tv->header->dict;
- const char *key, *val;
- int n = 0;
- tv->rg_hash = kh_init(kh_rg);
- while ( (iter = sam_header2key_val(iter, "RG","ID","SM", &key, &val)) )
- {
- if ( !strcmp(samples,key) || (val && !strcmp(samples,val)) )
- {
- khiter_t k = kh_get(kh_rg, tv->rg_hash, key);
- if ( k != kh_end(tv->rg_hash) ) continue;
- int ret;
- k = kh_put(kh_rg, tv->rg_hash, key, &ret);
- kh_value(tv->rg_hash, k) = val;
- n++;
- }
- }
- if ( !n )
- {
- fprintf(stderr,"The sample or read group \"%s\" not present.\n", samples);
- exit(EXIT_FAILURE);
- }
- }
-
- return 0;
- }
-
-
-void base_tv_destroy(tview_t* tv)
- {
- bam_lplbuf_destroy(tv->lplbuf);
- bcf_call_destroy(tv->bca);
- bam_index_destroy(tv->idx);
- if (tv->fai) fai_destroy(tv->fai);
- free(tv->ref);
- bam_header_destroy(tv->header);
- bam_close(tv->fp);
- }
-
-
-int tv_pl_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data)
-{
- extern unsigned char bam_nt16_table[256];
- tview_t *tv = (tview_t*)data;
- int i, j, c, rb, attr, max_ins = 0;
- uint32_t call = 0;
- if (pos < tv->left_pos || tv->ccol > tv->mcol) return 0; // out of screen
- // print referece
- rb = (tv->ref && pos - tv->left_pos < tv->l_ref)? tv->ref[pos - tv->left_pos] : 'N';
- for (i = tv->last_pos + 1; i < pos; ++i) {
- if (i%10 == 0 && tv->mcol - tv->ccol >= 10) tv->my_mvprintw(tv,0, tv->ccol, "%-d", i+1);
- c = tv->ref? tv->ref[i - tv->left_pos] : 'N';
- tv->my_mvaddch(tv,1, tv->ccol++, c);
- }
- if (pos%10 == 0 && tv->mcol - tv->ccol >= 10) tv->my_mvprintw(tv,0, tv->ccol, "%-d", pos+1);
- { // call consensus
- bcf_callret1_t bcr;
- int qsum[4], a1, a2, tmp;
- double p[3], prior = 30;
- bcf_call_glfgen(n, pl, bam_nt16_table[rb], tv->bca, &bcr);
- for (i = 0; i < 4; ++i) qsum[i] = bcr.qsum[i]<<2 | i;
- for (i = 1; i < 4; ++i) // insertion sort
- for (j = i; j > 0 && qsum[j] > qsum[j-1]; --j)
- tmp = qsum[j], qsum[j] = qsum[j-1], qsum[j-1] = tmp;
- a1 = qsum[0]&3; a2 = qsum[1]&3;
- p[0] = bcr.p[a1*5+a1]; p[1] = bcr.p[a1*5+a2] + prior; p[2] = bcr.p[a2*5+a2];
- if ("ACGT"[a1] != toupper(rb)) p[0] += prior + 3;
- if ("ACGT"[a2] != toupper(rb)) p[2] += prior + 3;
- if (p[0] < p[1] && p[0] < p[2]) call = (1<<a1)<<16 | (int)((p[1]<p[2]?p[1]:p[2]) - p[0] + .499);
- else if (p[2] < p[1] && p[2] < p[0]) call = (1<<a2)<<16 | (int)((p[0]<p[1]?p[0]:p[1]) - p[2] + .499);
- else call = (1<<a1|1<<a2)<<16 | (int)((p[0]<p[2]?p[0]:p[2]) - p[1] + .499);
- }
- attr = tv->my_underline(tv);
- c = ",ACMGRSVTWYHKDBN"[call>>16&0xf];
- i = (call&0xffff)/10+1;
- if (i > 4) i = 4;
- attr |= tv->my_colorpair(tv,i);
- if (c == toupper(rb)) c = '.';
- tv->my_attron(tv,attr);
- tv->my_mvaddch(tv,2, tv->ccol, c);
- tv->my_attroff(tv,attr);
- if(tv->ins) {
- // calculate maximum insert
- for (i = 0; i < n; ++i) {
- const bam_pileup1_t *p = pl + i;
- if (p->indel > 0 && max_ins < p->indel) max_ins = p->indel;
- }
- }
- // core loop
- for (j = 0; j <= max_ins; ++j) {
- for (i = 0; i < n; ++i) {
- const bam_pileup1_t *p = pl + i;
- int row = TV_MIN_ALNROW + p->level - tv->row_shift;
- if (j == 0) {
- if (!p->is_del) {
- if (tv->base_for == TV_BASE_COLOR_SPACE &&
- (c = bam_aux_getCSi(p->b, p->qpos))) {
- // assume that if we found one color, we will be able to get the color error
- if (tv->is_dot && '-' == bam_aux_getCEi(p->b, p->qpos)) c = bam1_strand(p->b)? ',' : '.';
- } else {
- if (tv->show_name) {
- char *name = bam1_qname(p->b);
- c = (p->qpos + 1 >= p->b->core.l_qname)? ' ' : name[p->qpos];
- } else {
- c = bam_nt16_rev_table[bam1_seqi(bam1_seq(p->b), p->qpos)];
- if (tv->is_dot && toupper(c) == toupper(rb)) c = bam1_strand(p->b)? ',' : '.';
- }
- }
- } else c = p->is_refskip? (bam1_strand(p->b)? '<' : '>') : '*';
- } else { // padding
- if (j > p->indel) c = '*';
- else { // insertion
- if (tv->base_for == TV_BASE_NUCL) {
- if (tv->show_name) {
- char *name = bam1_qname(p->b);
- c = (p->qpos + j + 1 >= p->b->core.l_qname)? ' ' : name[p->qpos + j];
- } else {
- c = bam_nt16_rev_table[bam1_seqi(bam1_seq(p->b), p->qpos + j)];
- if (j == 0 && tv->is_dot && toupper(c) == toupper(rb)) c = bam1_strand(p->b)? ',' : '.';
- }
- } else {
- c = bam_aux_getCSi(p->b, p->qpos + j);
- if (tv->is_dot && '-' == bam_aux_getCEi(p->b, p->qpos + j)) c = bam1_strand(p->b)? ',' : '.';
- }
- }
- }
- if (row > TV_MIN_ALNROW && row < tv->mrow) {
- int x;
- attr = 0;
- if (((p->b->core.flag&BAM_FPAIRED) && !(p->b->core.flag&BAM_FPROPER_PAIR))
- || (p->b->core.flag & BAM_FSECONDARY)) attr |= tv->my_underline(tv);
- if (tv->color_for == TV_COLOR_BASEQ) {
- x = bam1_qual(p->b)[p->qpos]/10 + 1;
- if (x > 4) x = 4;
- attr |= tv->my_colorpair(tv,x);
- } else if (tv->color_for == TV_COLOR_MAPQ) {
- x = p->b->core.qual/10 + 1;
- if (x > 4) x = 4;
- attr |= tv->my_colorpair(tv,x);
- } else if (tv->color_for == TV_COLOR_NUCL) {
- x = bam_nt16_nt4_table[bam1_seqi(bam1_seq(p->b), p->qpos)] + 5;
- attr |= tv->my_colorpair(tv,x);
- } else if(tv->color_for == TV_COLOR_COL) {
- x = 0;
- switch(bam_aux_getCSi(p->b, p->qpos)) {
- case '0': x = 0; break;
- case '1': x = 1; break;
- case '2': x = 2; break;
- case '3': x = 3; break;
- case '4': x = 4; break;
- default: x = bam_nt16_nt4_table[bam1_seqi(bam1_seq(p->b), p->qpos)]; break;
- }
- x+=5;
- attr |= tv->my_colorpair(tv,x);
- } else if(tv->color_for == TV_COLOR_COLQ) {
- x = bam_aux_getCQi(p->b, p->qpos);
- if(0 == x) x = bam1_qual(p->b)[p->qpos];
- x = x/10 + 1;
- if (x > 4) x = 4;
- attr |= tv->my_colorpair(tv,x);
- }
- tv->my_attron(tv,attr);
- tv->my_mvaddch(tv,row, tv->ccol, bam1_strand(p->b)? tolower(c) : toupper(c));
- tv->my_attroff(tv,attr);
- }
- }
- c = j? '*' : rb;
- if (c == '*') {
- attr = tv->my_colorpair(tv,8);
- tv->my_attron(tv,attr);
- tv->my_mvaddch(tv,1, tv->ccol++, c);
- tv->my_attroff(tv,attr);
- } else tv->my_mvaddch(tv,1, tv->ccol++, c);
- }
- tv->last_pos = pos;
- return 0;
-}
-
-
-
-
-int tv_fetch_func(const bam1_t *b, void *data)
-{
- tview_t *tv = (tview_t*)data;
- if ( tv->rg_hash )
- {
- const uint8_t *rg = bam_aux_get(b, "RG");
- if ( !rg ) return 0;
- khiter_t k = kh_get(kh_rg, tv->rg_hash, (const char*)(rg + 1));
- if ( k == kh_end(tv->rg_hash) ) return 0;
- }
- if (tv->no_skip) {
- uint32_t *cigar = bam1_cigar(b); // this is cheating...
- int i;
- for (i = 0; i <b->core.n_cigar; ++i) {
- if ((cigar[i]&0xf) == BAM_CREF_SKIP)
- cigar[i] = cigar[i]>>4<<4 | BAM_CDEL;
- }
- }
- bam_lplbuf_push(b, tv->lplbuf);
- return 0;
-}
-
-int base_draw_aln(tview_t *tv, int tid, int pos)
- {
- assert(tv!=NULL);
- // reset
- tv->my_clear(tv);
- tv->curr_tid = tid; tv->left_pos = pos;
- tv->last_pos = tv->left_pos - 1;
- tv->ccol = 0;
- // print ref and consensus
- if (tv->fai) {
- char *str;
- if (tv->ref) free(tv->ref);
- assert(tv->curr_tid>=0);
-
- str = (char*)calloc(strlen(tv->header->target_name[tv->curr_tid]) + 30, 1);
- assert(str!=NULL);
- sprintf(str, "%s:%d-%d", tv->header->target_name[tv->curr_tid], tv->left_pos + 1, tv->left_pos + tv->mcol);
- tv->ref = fai_fetch(tv->fai, str, &tv->l_ref);
- free(str);
- }
- // draw aln
- bam_lplbuf_reset(tv->lplbuf);
- bam_fetch(tv->fp, tv->idx, tv->curr_tid, tv->left_pos, tv->left_pos + tv->mcol, tv, tv_fetch_func);
- bam_lplbuf_push(0, tv->lplbuf);
-
- while (tv->ccol < tv->mcol) {
- int pos = tv->last_pos + 1;
- if (pos%10 == 0 && tv->mcol - tv->ccol >= 10) tv->my_mvprintw(tv,0, tv->ccol, "%-d", pos+1);
- tv->my_mvaddch(tv,1, tv->ccol++, (tv->ref && pos < tv->l_ref)? tv->ref[pos - tv->left_pos] : 'N');
- ++tv->last_pos;
- }
- return 0;
-}
-
-
-
-
-static void error(const char *format, ...)
-{
- if ( !format )
- {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: bamtk tview [options] <aln.bam> [ref.fasta]\n");
- fprintf(stderr, "Options:\n");
- fprintf(stderr, " -d display output as (H)tml or (C)urses or (T)ext \n");
- fprintf(stderr, " -p chr:pos go directly to this position\n");
- fprintf(stderr, " -s STR display only reads from this sample or group\n");
- fprintf(stderr, "\n\n");
- }
- else
- {
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
- }
- exit(-1);
-}
-
-enum dipsay_mode {display_ncurses,display_html,display_text};
-extern tview_t* curses_tv_init(const char *fn, const char *fn_fa, const char *samples);
-extern tview_t* html_tv_init(const char *fn, const char *fn_fa, const char *samples);
-extern tview_t* text_tv_init(const char *fn, const char *fn_fa, const char *samples);
-
-int bam_tview_main(int argc, char *argv[])
- {
- int view_mode=display_ncurses;
- tview_t* tv=NULL;
- char *samples=NULL, *position=NULL;
- int c;
- while ((c = getopt(argc, argv, "s:p:d:")) >= 0) {
- switch (c) {
- case 's': samples=optarg; break;
- case 'p': position=optarg; break;
- case 'd':
- {
- switch(optarg[0])
- {
- case 'H': case 'h': view_mode=display_html;break;
- case 'T': case 't': view_mode=display_text;break;
- case 'C': case 'c': view_mode=display_ncurses;break;
- default: view_mode=display_ncurses;break;
- }
- break;
- }
- default: error(NULL);
- }
- }
- if (argc==optind) error(NULL);
-
- switch(view_mode)
- {
- case display_ncurses:
- {
- tv = curses_tv_init(argv[optind], (optind+1>=argc)? 0 : argv[optind+1], samples);
- break;
- }
- case display_text:
- {
- tv = text_tv_init(argv[optind], (optind+1>=argc)? 0 : argv[optind+1], samples);
- break;
- }
- case display_html:
- {
- tv = html_tv_init(argv[optind], (optind+1>=argc)? 0 : argv[optind+1], samples);
- break;
- }
- }
- if(tv==NULL)
- {
- error("cannot create view");
- return EXIT_FAILURE;
- }
-
- if ( position )
- {
- int _tid = -1, _beg, _end;
- bam_parse_region(tv->header, position, &_tid, &_beg, &_end);
- if (_tid >= 0) { tv->curr_tid = _tid; tv->left_pos = _beg; }
- }
- tv->my_drawaln(tv, tv->curr_tid, tv->left_pos);
- tv->my_loop(tv);
- tv->my_destroy(tv);
-
- return EXIT_SUCCESS;
- }
diff --git a/samtools-0.1.19/bam_tview.h b/samtools-0.1.19/bam_tview.h
deleted file mode 100644
index 80f0464..0000000
--- a/samtools-0.1.19/bam_tview.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#ifndef BAM_TVIEW_H
-#define BAM_TVIEW_H
-
-#include <ctype.h>
-#include <assert.h>
-#include <string.h>
-#include <math.h>
-#include <unistd.h>
-#include <stdarg.h>
-#include "bam.h"
-#include "faidx.h"
-#include "bam2bcf.h"
-#include "sam_header.h"
-#include "khash.h"
-
-KHASH_MAP_INIT_STR(kh_rg, const char *)
-
-typedef struct AbstractTview {
- int mrow, mcol;
-
- bam_index_t *idx;
- bam_lplbuf_t *lplbuf;
- bam_header_t *header;
- bamFile fp;
- int curr_tid, left_pos;
- faidx_t *fai;
- bcf_callaux_t *bca;
-
- int ccol, last_pos, row_shift, base_for, color_for, is_dot, l_ref, ins, no_skip, show_name;
- char *ref;
- khash_t(kh_rg) *rg_hash;
- /* callbacks */
- void (*my_destroy)(struct AbstractTview* );
- void (*my_mvprintw)(struct AbstractTview* ,int,int,const char*,...);
- void (*my_mvaddch)(struct AbstractTview*,int,int,int);
- void (*my_attron)(struct AbstractTview*,int);
- void (*my_attroff)(struct AbstractTview*,int);
- void (*my_clear)(struct AbstractTview*);
- int (*my_colorpair)(struct AbstractTview*,int);
- int (*my_drawaln)(struct AbstractTview*,int,int);
- int (*my_loop)(struct AbstractTview*);
- int (*my_underline)(struct AbstractTview*);
-} tview_t;
-
-
-char bam_aux_getCEi(bam1_t *b, int i);
-char bam_aux_getCSi(bam1_t *b, int i);
-char bam_aux_getCQi(bam1_t *b, int i);
-
-#define TV_MIN_ALNROW 2
-#define TV_MAX_GOTO 40
-#define TV_LOW_MAPQ 10
-
-#define TV_COLOR_MAPQ 0
-#define TV_COLOR_BASEQ 1
-#define TV_COLOR_NUCL 2
-#define TV_COLOR_COL 3
-#define TV_COLOR_COLQ 4
-
-#define TV_BASE_NUCL 0
-#define TV_BASE_COLOR_SPACE 1
-
-int tv_pl_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data);
-int base_tv_init(tview_t*,const char *fn, const char *fn_fa, const char *samples);
-void base_tv_destroy(tview_t*);
-int base_draw_aln(tview_t *tv, int tid, int pos);
-
-typedef struct Tixel
- {
- int ch;
- int attributes;
- }tixel_t;
-
-#endif
-
diff --git a/samtools-0.1.19/bam_tview_curses.c b/samtools-0.1.19/bam_tview_curses.c
deleted file mode 100644
index 4fdd1fb..0000000
--- a/samtools-0.1.19/bam_tview_curses.c
+++ /dev/null
@@ -1,297 +0,0 @@
-#undef _HAVE_CURSES
-
-#if _CURSES_LIB == 0
-#elif _CURSES_LIB == 1
-#include <curses.h>
-#ifndef NCURSES_VERSION
-#warning "_CURSES_LIB=1 but NCURSES_VERSION not defined; tview is NOT compiled"
-#else
-#define _HAVE_CURSES
-#endif
-#elif _CURSES_LIB == 2
-#include <xcurses.h>
-#define _HAVE_CURSES
-#else
-#warning "_CURSES_LIB is not 0, 1 or 2; tview is NOT compiled"
-#endif
-
-
-#include "bam_tview.h"
-
-#ifdef _HAVE_CURSES
-
-
-
-typedef struct CursesTview {
- tview_t view;
- WINDOW *wgoto, *whelp;
- } curses_tview_t;
-
-
-
-
-#define FROM_TV(ptr) ((curses_tview_t*)ptr)
-
-static void curses_destroy(tview_t* base)
- {
- curses_tview_t* tv=(curses_tview_t*)base;
-
-
- delwin(tv->wgoto); delwin(tv->whelp);
- endwin();
-
- base_tv_destroy(base);
-
- free(tv);
- }
-
-/*
- void (*my_mvprintw)(struct AbstractTview* ,int,int,const char*,...);
- void (*my_)(struct AbstractTview*,int,int,int);
- void (*my_attron)(struct AbstractTview*,int);
- void (*my_attroff)(struct AbstractTview*,int);
- void (*my_clear)(struct AbstractTview*);
- int (*my_colorpair)(struct AbstractTview*,int);
-*/
-
-static void curses_mvprintw(struct AbstractTview* tv,int y ,int x,const char* fmt,...)
- {
- unsigned int size=tv->mcol+2;
- char* str=malloc(size);
- if(str==0) exit(EXIT_FAILURE);
- va_list argptr;
- va_start(argptr, fmt);
- vsnprintf(str,size, fmt, argptr);
- va_end(argptr);
- mvprintw(y,x,str);
- free(str);
- }
-
-static void curses_mvaddch(struct AbstractTview* tv,int y,int x,int ch)
- {
- mvaddch(y,x,ch);
- }
-
-static void curses_attron(struct AbstractTview* tv,int flag)
- {
- attron(flag);
- }
-static void curses_attroff(struct AbstractTview* tv,int flag)
- {
- attroff(flag);
- }
-static void curses_clear(struct AbstractTview* tv)
- {
- clear();
- }
-
-static int curses_colorpair(struct AbstractTview* tv,int flag)
- {
- return COLOR_PAIR(flag);
- }
-
-static int curses_drawaln(struct AbstractTview* tv, int tid, int pos)
- {
- return base_draw_aln(tv, tid, pos);
- }
-
-
-
-static void tv_win_goto(curses_tview_t *tv, int *tid, int *pos)
- {
- char str[256], *p;
- int i, l = 0;
- tview_t *base=(tview_t*)tv;
- wborder(tv->wgoto, '|', '|', '-', '-', '+', '+', '+', '+');
- mvwprintw(tv->wgoto, 1, 2, "Goto: ");
- for (;;) {
- int c = wgetch(tv->wgoto);
- wrefresh(tv->wgoto);
- if (c == KEY_BACKSPACE || c == '\010' || c == '\177') {
- if(l > 0) --l;
- } else if (c == KEY_ENTER || c == '\012' || c == '\015') {
- int _tid = -1, _beg, _end;
- if (str[0] == '=') {
- _beg = strtol(str+1, &p, 10) - 1;
- if (_beg > 0) {
- *pos = _beg;
- return;
- }
- } else {
- bam_parse_region(base->header, str, &_tid, &_beg, &_end);
- if (_tid >= 0) {
- *tid = _tid; *pos = _beg;
- return;
- }
- }
- } else if (isgraph(c)) {
- if (l < TV_MAX_GOTO) str[l++] = c;
- } else if (c == '\027') l = 0;
- else if (c == '\033') return;
- str[l] = '\0';
- for (i = 0; i < TV_MAX_GOTO; ++i) mvwaddch(tv->wgoto, 1, 8 + i, ' ');
- mvwprintw(tv->wgoto, 1, 8, "%s", str);
- }
-}
-
-
-
-
-static void tv_win_help(curses_tview_t *tv) {
- int r = 1;
- tview_t* base=(tview_t*)base;
- WINDOW *win = tv->whelp;
- wborder(win, '|', '|', '-', '-', '+', '+', '+', '+');
- mvwprintw(win, r++, 2, " -=- Help -=- ");
- r++;
- mvwprintw(win, r++, 2, "? This window");
- mvwprintw(win, r++, 2, "Arrows Small scroll movement");
- mvwprintw(win, r++, 2, "h,j,k,l Small scroll movement");
- mvwprintw(win, r++, 2, "H,J,K,L Large scroll movement");
- mvwprintw(win, r++, 2, "ctrl-H Scroll 1k left");
- mvwprintw(win, r++, 2, "ctrl-L Scroll 1k right");
- mvwprintw(win, r++, 2, "space Scroll one screen");
- mvwprintw(win, r++, 2, "backspace Scroll back one screen");
- mvwprintw(win, r++, 2, "g Go to specific location");
- mvwprintw(win, r++, 2, "m Color for mapping qual");
- mvwprintw(win, r++, 2, "n Color for nucleotide");
- mvwprintw(win, r++, 2, "b Color for base quality");
- mvwprintw(win, r++, 2, "c Color for cs color");
- mvwprintw(win, r++, 2, "z Color for cs qual");
- mvwprintw(win, r++, 2, ". Toggle on/off dot view");
- mvwprintw(win, r++, 2, "s Toggle on/off ref skip");
- mvwprintw(win, r++, 2, "r Toggle on/off rd name");
- mvwprintw(win, r++, 2, "N Turn on nt view");
- mvwprintw(win, r++, 2, "C Turn on cs view");
- mvwprintw(win, r++, 2, "i Toggle on/off ins");
- mvwprintw(win, r++, 2, "q Exit");
- r++;
- mvwprintw(win, r++, 2, "Underline: Secondary or orphan");
- mvwprintw(win, r++, 2, "Blue: 0-9 Green: 10-19");
- mvwprintw(win, r++, 2, "Yellow: 20-29 White: >=30");
- wrefresh(win);
- wgetch(win);
-}
-
-static int curses_underline(tview_t* tv)
- {
- return A_UNDERLINE;
- }
-
-static int curses_loop(tview_t* tv)
- {
- int tid, pos;
- curses_tview_t *CTV=(curses_tview_t *)tv;
- tid = tv->curr_tid; pos = tv->left_pos;
- while (1) {
- int c = getch();
- switch (c) {
- case '?': tv_win_help(CTV); break;
- case '\033':
- case 'q': goto end_loop;
- case '/':
- case 'g': tv_win_goto(CTV, &tid, &pos); break;
- case 'm': tv->color_for = TV_COLOR_MAPQ; break;
- case 'b': tv->color_for = TV_COLOR_BASEQ; break;
- case 'n': tv->color_for = TV_COLOR_NUCL; break;
- case 'c': tv->color_for = TV_COLOR_COL; break;
- case 'z': tv->color_for = TV_COLOR_COLQ; break;
- case 's': tv->no_skip = !tv->no_skip; break;
- case 'r': tv->show_name = !tv->show_name; break;
- case KEY_LEFT:
- case 'h': --pos; break;
- case KEY_RIGHT:
- case 'l': ++pos; break;
- case KEY_SLEFT:
- case 'H': pos -= 20; break;
- case KEY_SRIGHT:
- case 'L': pos += 20; break;
- case '.': tv->is_dot = !tv->is_dot; break;
- case 'N': tv->base_for = TV_BASE_NUCL; break;
- case 'C': tv->base_for = TV_BASE_COLOR_SPACE; break;
- case 'i': tv->ins = !tv->ins; break;
- case '\010': pos -= 1000; break;
- case '\014': pos += 1000; break;
- case ' ': pos += tv->mcol; break;
- case KEY_UP:
- case 'j': --tv->row_shift; break;
- case KEY_DOWN:
- case 'k': ++tv->row_shift; break;
- case KEY_BACKSPACE:
- case '\177': pos -= tv->mcol; break;
- case KEY_RESIZE: getmaxyx(stdscr, tv->mrow, tv->mcol); break;
- default: continue;
- }
- if (pos < 0) pos = 0;
- if (tv->row_shift < 0) tv->row_shift = 0;
- tv->my_drawaln(tv, tid, pos);
- }
-end_loop:
- return 0;
-}
-
-
-
-
-tview_t* curses_tv_init(const char *fn, const char *fn_fa, const char *samples)
- {
- curses_tview_t *tv = (curses_tview_t*)calloc(1, sizeof(curses_tview_t));
- tview_t* base=(tview_t*)tv;
- if(tv==0)
- {
- fprintf(stderr,"Calloc failed\n");
- return 0;
- }
-
- base_tv_init(base,fn,fn_fa,samples);
- /* initialize callbacks */
-#define SET_CALLBACK(fun) base->my_##fun=curses_##fun;
- SET_CALLBACK(destroy);
- SET_CALLBACK(mvprintw);
- SET_CALLBACK(mvaddch);
- SET_CALLBACK(attron);
- SET_CALLBACK(attroff);
- SET_CALLBACK(clear);
- SET_CALLBACK(colorpair);
- SET_CALLBACK(drawaln);
- SET_CALLBACK(loop);
- SET_CALLBACK(underline);
-#undef SET_CALLBACK
-
- initscr();
- keypad(stdscr, TRUE);
- clear();
- noecho();
- cbreak();
-
- getmaxyx(stdscr, base->mrow, base->mcol);
- tv->wgoto = newwin(3, TV_MAX_GOTO + 10, 10, 5);
- tv->whelp = newwin(29, 40, 5, 5);
-
- start_color();
- init_pair(1, COLOR_BLUE, COLOR_BLACK);
- init_pair(2, COLOR_GREEN, COLOR_BLACK);
- init_pair(3, COLOR_YELLOW, COLOR_BLACK);
- init_pair(4, COLOR_WHITE, COLOR_BLACK);
- init_pair(5, COLOR_GREEN, COLOR_BLACK);
- init_pair(6, COLOR_CYAN, COLOR_BLACK);
- init_pair(7, COLOR_YELLOW, COLOR_BLACK);
- init_pair(8, COLOR_RED, COLOR_BLACK);
- init_pair(9, COLOR_BLUE, COLOR_BLACK);
- return base;
- }
-
-
-#else // #ifdef _HAVE_CURSES
-#include <stdio.h>
-#warning "No curses library is available; tview with curses is disabled."
-
-extern tview_t* text_tv_init(const char *fn, const char *fn_fa, const char *samples);
-
-tview_t* curses_tv_init(const char *fn, const char *fn_fa, const char *samples)
- {
- return text_tv_init(fn,fn_fa,samples);
- }
-#endif // #ifdef _HAVE_CURSES
-
-
diff --git a/samtools-0.1.19/bam_tview_html.c b/samtools-0.1.19/bam_tview_html.c
deleted file mode 100644
index f52b4c3..0000000
--- a/samtools-0.1.19/bam_tview_html.c
+++ /dev/null
@@ -1,349 +0,0 @@
-#include <unistd.h>
-#include "bam_tview.h"
-
-#define UNDERLINE_FLAG 10
-
-typedef struct HtmlTview {
- tview_t view;
- int row_count;
- tixel_t** screen;
- FILE* out;
- int attributes;/* color... */
- } html_tview_t;
-
-#define FROM_TV(ptr) ((html_tview_t*)ptr)
-
-static void html_destroy(tview_t* base)
- {
- int i;
- html_tview_t* tv=(html_tview_t*)base;
- if(tv->screen!=NULL)
- {
- for(i=0;i< tv->row_count;++i) free(tv->screen[i]);
- free(tv->screen);
- }
- base_tv_destroy(base);
- free(tv);
- }
-
-/*
- void (*my_mvprintw)(struct AbstractTview* ,int,int,const char*,...);
- void (*my_)(struct AbstractTview*,int,int,int);
- void (*my_attron)(struct AbstractTview*,int);
- void (*my_attroff)(struct AbstractTview*,int);
- void (*my_clear)(struct AbstractTview*);
- int (*my_colorpair)(struct AbstractTview*,int);
-*/
-
-static void html_mvprintw(struct AbstractTview* tv,int y ,int x,const char* fmt,...)
- {
- int i,nchars=0;
- unsigned int size=tv->mcol+2;
- char* str=malloc(size);
- if(str==0) exit(EXIT_FAILURE);
- va_list argptr;
- va_start(argptr, fmt);
- nchars=vsnprintf(str,size, fmt, argptr);
- va_end(argptr);
-
- for(i=0;i< nchars;++i)
- {
- tv->my_mvaddch(tv,y,x+i,str[i]);
- }
- free(str);
- }
-
-static void html_mvaddch(struct AbstractTview* tv,int y,int x,int ch)
- {
- tixel_t* row=NULL;
- html_tview_t* ptr=FROM_TV(tv);
- if( x >= tv->mcol ) return; //out of screen
- while(ptr->row_count<=y)
- {
- int x;
- row=(tixel_t*)calloc(tv->mcol,sizeof(tixel_t));
- if(row==0) exit(EXIT_FAILURE);
- for(x=0;x<tv->mcol;++x) {row[x].ch=' ';row[x].attributes=0;}
- ptr->screen=(tixel_t**)realloc(ptr->screen,sizeof(tixel_t*)*(ptr->row_count+1));
- ptr->screen[ptr->row_count++]=row;
- }
- row=ptr->screen[y];
- row[x].ch=ch;
- row[x].attributes=ptr->attributes;
- }
-
-static void html_attron(struct AbstractTview* tv,int flag)
- {
- html_tview_t* ptr=FROM_TV(tv);
- ptr->attributes |= flag;
-
-
- }
-
-static void html_attroff(struct AbstractTview* tv,int flag)
- {
- html_tview_t* ptr=FROM_TV(tv);
- ptr->attributes &= ~(flag);
- }
-
-static void html_clear(struct AbstractTview* tv)
- {
- html_tview_t* ptr=FROM_TV(tv);
- if(ptr->screen!=NULL)
- {
- int i;
- for(i=0;i< ptr->row_count;++i) free(ptr->screen[i]);
- free(ptr->screen);
- ptr->screen=NULL;
- }
- ptr->row_count=0;
- ptr->attributes=0;
- }
-
-static int html_colorpair(struct AbstractTview* tv,int flag)
- {
- return (1 << (flag));
- }
-
-static int html_drawaln(struct AbstractTview* tv, int tid, int pos)
- {
- int y,x;
- html_tview_t* ptr=FROM_TV(tv);
- html_clear(tv);
- base_draw_aln(tv, tid, pos);
- fputs("<html><head>",ptr->out);
- fprintf(ptr->out,"<title>%s:%d</title>",
- tv->header->target_name[tid],
- pos+1
- );
- //style
-
- fputs("<style type='text/css'>\n",ptr->out);
- fputs(".tviewbody { margin:5px; background-color:white;text-align:center;}\n",ptr->out);
- fputs(".tviewtitle {text-align:center;}\n",ptr->out);
- fputs(".tviewpre { margin:5px; background-color:white;}\n",ptr->out);
- #define CSS(id,col) fprintf(ptr->out,".tviewc%d {color:%s;}\n.tviewcu%d {color:%s;text-decoration:underline;}\n",id,col,id,col);
- CSS(0, "black");
- CSS(1, "blue");
- CSS(2, "green");
- CSS(3, "yellow");
- CSS(4, "black");
- CSS(5, "green");
- CSS(6, "cyan");
- CSS(7, "yellow");
- CSS(8, "red");
- CSS(9, "blue");
- #undef CSS
- fputs("</style>",ptr->out);
-
- fputs("</head><body>",ptr->out);
-
- fprintf(ptr->out,"<div class='tviewbody'><div class='tviewtitle'>%s:%d</div>",
- tv->header->target_name[tid],
- pos+1
- );
-
- fputs("<pre class='tviewpre'>",ptr->out);
- for(y=0;y< ptr->row_count;++y)
- {
-
- for(x=0;x< tv->mcol;++x)
- {
-
-
- if(x== 0 || ptr->screen[y][x].attributes != ptr->screen[y][x-1].attributes)
- {
- int css=0;
- fprintf(ptr->out,"<span");
- while(css<32)
- {
- //if(y>1) fprintf(stderr,"css=%d pow2=%d vs %d\n",css,(1 << (css)),ptr->screen[y][x].attributes);
- if(( (ptr->screen[y][x].attributes) & (1 << (css)))!=0)
- {
-
- fprintf(ptr->out," class='tviewc%s%d'",
- (( (ptr->screen[y][x].attributes) & (1 << (UNDERLINE_FLAG)) )!=0?"u":""),
- css);
- break;
- }
- ++css;
- }
-
-
- fputs(">",ptr->out);
- }
-
- int ch=ptr->screen[y][x].ch;
- switch(ch)
- {
- case '<': fputs("<",ptr->out);break;
- case '>': fputs(">",ptr->out);break;
- case '&': fputs("&",ptr->out);break;
- default: fputc(ch,ptr->out); break;
- }
-
-
- if(x+1 == tv->mcol || ptr->screen[y][x].attributes!=ptr->screen[y][x+1].attributes)
- {
- fputs("</span>",ptr->out);
- }
- }
- if(y+1 < ptr->row_count) fputs("<br/>",ptr->out);
- }
- fputs("</pre></div></body></html>",ptr->out);
- return 0;
- }
-
-
-#define ANSI_COLOR_RED "\x1b[31m"
-#define ANSI_COLOR_GREEN "\x1b[32m"
-#define ANSI_COLOR_YELLOW "\x1b[33m"
-#define ANSI_COLOR_BLUE "\x1b[34m"
-#define ANSI_COLOR_MAGENTA "\x1b[35m"
-#define ANSI_COLOR_CYAN "\x1b[36m"
-#define ANSI_COLOR_BLACK "\x1b[0m"
-#define ANSI_COLOR_RESET ANSI_COLOR_BLACK
-
-#define ANSI_UNDERLINE_SET "\033[4m"
-#define ANSI_UNDERLINE_UNSET "\033[0m"
-
-static int text_drawaln(struct AbstractTview* tv, int tid, int pos)
- {
- int y,x;
- html_tview_t* ptr=FROM_TV(tv);
- html_clear(tv);
- base_draw_aln(tv, tid, pos);
- int is_term= isatty(fileno(ptr->out));
-
- for(y=0;y< ptr->row_count;++y)
- {
- for(x=0;x< tv->mcol;++x)
- {
- if(is_term)
- {
- int css=0;
- while(css<32)
- {
- if(( (ptr->screen[y][x].attributes) & (1 << (css)))!=0)
- {
- break;
- }
- ++css;
- }
- switch(css)
- {
- //CSS(0, "black");
- case 1: fputs(ANSI_COLOR_BLUE,ptr->out); break;
- case 2: fputs(ANSI_COLOR_GREEN,ptr->out); break;
- case 3: fputs(ANSI_COLOR_YELLOW,ptr->out); break;
- //CSS(4, "black");
- case 5: fputs(ANSI_COLOR_GREEN,ptr->out); break;
- case 6: fputs(ANSI_COLOR_CYAN,ptr->out); break;
- case 7: fputs(ANSI_COLOR_YELLOW,ptr->out); break;
- case 8: fputs(ANSI_COLOR_RED,ptr->out); break;
- case 9: fputs(ANSI_COLOR_BLUE,ptr->out); break;
- default:break;
- }
- if(( (ptr->screen[y][x].attributes) & (1 << (UNDERLINE_FLAG)))!=0)
- {
- fputs(ANSI_UNDERLINE_SET,ptr->out);
- }
-
- }
-
-
- int ch=ptr->screen[y][x].ch;
-
- fputc(ch,ptr->out);
- if(is_term)
- {
- fputs(ANSI_COLOR_RESET,ptr->out);
- if(( (ptr->screen[y][x].attributes) & (1 << (UNDERLINE_FLAG)))!=0)
- {
- fputs(ANSI_UNDERLINE_UNSET,ptr->out);
- }
- }
- }
- fputc('\n',ptr->out);
- }
- return 0;
- }
-
-
-static int html_loop(tview_t* tv)
- {
- //tv->my_drawaln(tv, tv->curr_tid, tv->left_pos);
- return 0;
- }
-
-static int html_underline(tview_t* tv)
- {
- return (1 << UNDERLINE_FLAG);
- }
-
-/*
-static void init_pair(html_tview_t *tv,int id_ge_1, const char* pen, const char* paper)
- {
-
- }
-*/
-
-tview_t* html_tv_init(const char *fn, const char *fn_fa, const char *samples)
- {
- char* colstr=getenv("COLUMNS");
- html_tview_t *tv = (html_tview_t*)calloc(1, sizeof(html_tview_t));
- tview_t* base=(tview_t*)tv;
- if(tv==0)
- {
- fprintf(stderr,"Calloc failed\n");
- return 0;
- }
- tv->row_count=0;
- tv->screen=NULL;
- tv->out=stdout;
- tv->attributes=0;
- base_tv_init(base,fn,fn_fa,samples);
- /* initialize callbacks */
-#define SET_CALLBACK(fun) base->my_##fun=html_##fun;
- SET_CALLBACK(destroy);
- SET_CALLBACK(mvprintw);
- SET_CALLBACK(mvaddch);
- SET_CALLBACK(attron);
- SET_CALLBACK(attroff);
- SET_CALLBACK(clear);
- SET_CALLBACK(colorpair);
- SET_CALLBACK(drawaln);
- SET_CALLBACK(loop);
- SET_CALLBACK(underline);
-#undef SET_CALLBACK
-
-
- if(colstr!=0)
- {
- base->mcol=atoi(colstr);
- if(base->mcol<10) base->mcol=80;
- }
- base->mrow=99999;
-
-/*
- init_pair(tv,1, "blue", "white");
- init_pair(tv,2, "green", "white");
- init_pair(tv,3, "yellow", "white");
- init_pair(tv,4, "white", "white");
- init_pair(tv,5, "green", "white");
- init_pair(tv,6, "cyan", "white");
- init_pair(tv,7, "yellow", "white");
- init_pair(tv,8, "red", "white");
- init_pair(tv,9, "blue", "white");
- */
- return base;
- }
-
-
-tview_t* text_tv_init(const char *fn, const char *fn_fa, const char *samples)
- {
- tview_t* tv=html_tv_init(fn,fn_fa,samples);
- tv->my_drawaln=text_drawaln;
- return tv;
- }
-
diff --git a/samtools-0.1.19/bamshuf.c b/samtools-0.1.19/bamshuf.c
deleted file mode 100644
index 33a5238..0000000
--- a/samtools-0.1.19/bamshuf.c
+++ /dev/null
@@ -1,141 +0,0 @@
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include "sam.h"
-#include "ksort.h"
-
-#define DEF_CLEVEL 1
-
-static inline unsigned hash_Wang(unsigned key)
-{
- key += ~(key << 15);
- key ^= (key >> 10);
- key += (key << 3);
- key ^= (key >> 6);
- key += ~(key << 11);
- key ^= (key >> 16);
- return key;
-}
-
-static inline unsigned hash_X31_Wang(const char *s)
-{
- unsigned h = *s;
- if (h) {
- for (++s ; *s; ++s) h = (h << 5) - h + *s;
- return hash_Wang(h);
- } else return 0;
-}
-
-typedef struct {
- unsigned key;
- bam1_t *b;
-} elem_t;
-
-static inline int elem_lt(elem_t x, elem_t y)
-{
- if (x.key < y.key) return 1;
- if (x.key == y.key) {
- int t;
- t = strcmp(bam_get_qname(x.b), bam_get_qname(y.b));
- if (t < 0) return 1;
- return (t == 0 && ((x.b->core.flag>>6&3) < (y.b->core.flag>>6&3)));
- } else return 0;
-}
-
-KSORT_INIT(bamshuf, elem_t, elem_lt)
-
-static void bamshuf(const char *fn, int n_files, const char *pre, int clevel, int is_stdout)
-{
- BGZF *fp, *fpw, **fpt;
- char **fnt, modew[8];
- bam1_t *b;
- int i, l;
- bam_hdr_t *h;
- int64_t *cnt;
-
- // split
- fp = strcmp(fn, "-")? bgzf_open(fn, "r") : bgzf_dopen(fileno(stdin), "r");
- assert(fp);
- h = bam_hdr_read(fp);
- fnt = (char**)calloc(n_files, sizeof(void*));
- fpt = (BGZF**)calloc(n_files, sizeof(void*));
- cnt = (int64_t*)calloc(n_files, 8);
- l = strlen(pre);
- for (i = 0; i < n_files; ++i) {
- fnt[i] = (char*)calloc(l + 10, 1);
- sprintf(fnt[i], "%s.%.4d.bam", pre, i);
- fpt[i] = bgzf_open(fnt[i], "w1");
- bam_hdr_write(fpt[i], h);
- }
- b = bam_init1();
- while (bam_read1(fp, b) >= 0) {
- uint32_t x;
- x = hash_X31_Wang(bam_get_qname(b)) % n_files;
- bam_write1(fpt[x], b);
- ++cnt[x];
- }
- bam_destroy1(b);
- for (i = 0; i < n_files; ++i) bgzf_close(fpt[i]);
- free(fpt);
- bgzf_close(fp);
- // merge
- sprintf(modew, "w%d", (clevel >= 0 && clevel <= 9)? clevel : DEF_CLEVEL);
- if (!is_stdout) { // output to a file
- char *fnw = (char*)calloc(l + 5, 1);
- sprintf(fnw, "%s.bam", pre);
- fpw = bgzf_open(fnw, modew);
- free(fnw);
- } else fpw = bgzf_dopen(fileno(stdout), modew); // output to stdout
- bam_hdr_write(fpw, h);
- bam_hdr_destroy(h);
- for (i = 0; i < n_files; ++i) {
- int64_t j, c = cnt[i];
- elem_t *a;
- fp = bgzf_open(fnt[i], "r");
- bam_hdr_destroy(bam_hdr_read(fp));
- a = (elem_t*)calloc(c, sizeof(elem_t));
- for (j = 0; j < c; ++j) {
- a[j].b = bam_init1();
- assert(bam_read1(fp, a[j].b) >= 0);
- a[j].key = hash_X31_Wang(bam_get_qname(a[j].b));
- }
- bgzf_close(fp);
- unlink(fnt[i]);
- free(fnt[i]);
- ks_introsort(bamshuf, c, a);
- for (j = 0; j < c; ++j) {
- bam_write1(fpw, a[j].b);
- bam_destroy1(a[j].b);
- }
- free(a);
- }
- bgzf_close(fpw);
- free(fnt); free(cnt);
-}
-
-int main_bamshuf(int argc, char *argv[])
-{
- int c, n_files = 64, clevel = DEF_CLEVEL, is_stdout = 0, is_un = 0;
- while ((c = getopt(argc, argv, "n:l:uO")) >= 0) {
- switch (c) {
- case 'n': n_files = atoi(optarg); break;
- case 'l': clevel = atoi(optarg); break;
- case 'u': is_un = 1; break;
- case 'O': is_stdout = 1; break;
- }
- }
- if (is_un) clevel = 0;
- if (optind + 2 > argc) {
- fprintf(stderr, "\nUsage: bamshuf [-Ou] [-n nFiles] [-c cLevel] <in.bam> <out.prefix>\n\n");
- fprintf(stderr, "Options: -O output to stdout\n");
- fprintf(stderr, " -u uncompressed BAM output\n");
- fprintf(stderr, " -l INT compression level [%d]\n", DEF_CLEVEL);
- fprintf(stderr, " -n INT number of temporary files [%d]\n", n_files);
- fprintf(stderr, "\n");
- return 1;
- }
- bamshuf(argv[optind], n_files, argv[optind+1], clevel, is_stdout);
- return 0;
-}
diff --git a/samtools-0.1.19/bamtk.c b/samtools-0.1.19/bamtk.c
deleted file mode 100644
index 9df7c11..0000000
--- a/samtools-0.1.19/bamtk.c
+++ /dev/null
@@ -1,119 +0,0 @@
-#include <stdio.h>
-#include <unistd.h>
-#include <assert.h>
-#include <fcntl.h>
-#include "bam.h"
-
-#ifdef _USE_KNETFILE
-#include "knetfile.h"
-#endif
-
-int bam_taf2baf(int argc, char *argv[]);
-int bam_mpileup(int argc, char *argv[]);
-int bam_merge(int argc, char *argv[]);
-int bam_index(int argc, char *argv[]);
-int bam_sort(int argc, char *argv[]);
-int bam_tview_main(int argc, char *argv[]);
-int bam_mating(int argc, char *argv[]);
-int bam_rmdup(int argc, char *argv[]);
-int bam_flagstat(int argc, char *argv[]);
-int bam_fillmd(int argc, char *argv[]);
-int bam_idxstats(int argc, char *argv[]);
-int main_samview(int argc, char *argv[]);
-int main_import(int argc, char *argv[]);
-int main_reheader(int argc, char *argv[]);
-int main_cut_target(int argc, char *argv[]);
-int main_phase(int argc, char *argv[]);
-int main_cat(int argc, char *argv[]);
-int main_depth(int argc, char *argv[]);
-int main_bam2fq(int argc, char *argv[]);
-int main_pad2unpad(int argc, char *argv[]);
-int main_bedcov(int argc, char *argv[]);
-int main_bamshuf(int argc, char *argv[]);
-
-int faidx_main(int argc, char *argv[]);
-
-static int usage()
-{
- fprintf(stderr, "\n");
- fprintf(stderr, "Program: samtools (Tools for alignments in the SAM format)\n");
- fprintf(stderr, "Version: %s\n\n", BAM_VERSION);
- fprintf(stderr, "Usage: samtools <command> [options]\n\n");
- fprintf(stderr, "Command: view SAM<->BAM conversion\n");
- fprintf(stderr, " sort sort alignment file\n");
- fprintf(stderr, " mpileup multi-way pileup\n");
- fprintf(stderr, " depth compute the depth\n");
- fprintf(stderr, " faidx index/extract FASTA\n");
-#if _CURSES_LIB != 0
- fprintf(stderr, " tview text alignment viewer\n");
-#endif
- fprintf(stderr, " index index alignment\n");
- fprintf(stderr, " idxstats BAM index stats (r595 or later)\n");
- fprintf(stderr, " fixmate fix mate information\n");
- fprintf(stderr, " flagstat simple stats\n");
- fprintf(stderr, " calmd recalculate MD/NM tags and '=' bases\n");
- fprintf(stderr, " merge merge sorted alignments\n");
- fprintf(stderr, " rmdup remove PCR duplicates\n");
- fprintf(stderr, " reheader replace BAM header\n");
- fprintf(stderr, " cat concatenate BAMs\n");
- fprintf(stderr, " bedcov read depth per BED region\n");
- fprintf(stderr, " targetcut cut fosmid regions (for fosmid pool only)\n");
- fprintf(stderr, " phase phase heterozygotes\n");
- fprintf(stderr, " bamshuf shuffle and group alignments by name\n");
-// fprintf(stderr, " depad convert padded BAM to unpadded BAM\n"); // not stable
- fprintf(stderr, "\n");
-#ifdef _WIN32
- fprintf(stderr, "\
-Note: The Windows version of SAMtools is mainly designed for read-only\n\
- operations, such as viewing the alignments and generating the pileup.\n\
- Binary files generated by the Windows version may be buggy.\n\n");
-#endif
- return 1;
-}
-
-int main(int argc, char *argv[])
-{
-#ifdef _WIN32
- setmode(fileno(stdout), O_BINARY);
- setmode(fileno(stdin), O_BINARY);
-#ifdef _USE_KNETFILE
- knet_win32_init();
-#endif
-#endif
- if (argc < 2) return usage();
- if (strcmp(argv[1], "view") == 0) return main_samview(argc-1, argv+1);
- else if (strcmp(argv[1], "import") == 0) return main_import(argc-1, argv+1);
- else if (strcmp(argv[1], "mpileup") == 0) return bam_mpileup(argc-1, argv+1);
- else if (strcmp(argv[1], "merge") == 0) return bam_merge(argc-1, argv+1);
- else if (strcmp(argv[1], "sort") == 0) return bam_sort(argc-1, argv+1);
- else if (strcmp(argv[1], "index") == 0) return bam_index(argc-1, argv+1);
- else if (strcmp(argv[1], "idxstats") == 0) return bam_idxstats(argc-1, argv+1);
- else if (strcmp(argv[1], "faidx") == 0) return faidx_main(argc-1, argv+1);
- else if (strcmp(argv[1], "fixmate") == 0) return bam_mating(argc-1, argv+1);
- else if (strcmp(argv[1], "rmdup") == 0) return bam_rmdup(argc-1, argv+1);
- else if (strcmp(argv[1], "flagstat") == 0) return bam_flagstat(argc-1, argv+1);
- else if (strcmp(argv[1], "calmd") == 0) return bam_fillmd(argc-1, argv+1);
- else if (strcmp(argv[1], "fillmd") == 0) return bam_fillmd(argc-1, argv+1);
- else if (strcmp(argv[1], "reheader") == 0) return main_reheader(argc-1, argv+1);
- else if (strcmp(argv[1], "cat") == 0) return main_cat(argc-1, argv+1);
- else if (strcmp(argv[1], "targetcut") == 0) return main_cut_target(argc-1, argv+1);
- else if (strcmp(argv[1], "phase") == 0) return main_phase(argc-1, argv+1);
- else if (strcmp(argv[1], "depth") == 0) return main_depth(argc-1, argv+1);
- else if (strcmp(argv[1], "bam2fq") == 0) return main_bam2fq(argc-1, argv+1);
- else if (strcmp(argv[1], "pad2unpad") == 0) return main_pad2unpad(argc-1, argv+1);
- else if (strcmp(argv[1], "depad") == 0) return main_pad2unpad(argc-1, argv+1);
- else if (strcmp(argv[1], "bedcov") == 0) return main_bedcov(argc-1, argv+1);
- else if (strcmp(argv[1], "bamshuf") == 0) return main_bamshuf(argc-1, argv+1);
- else if (strcmp(argv[1], "pileup") == 0) {
- fprintf(stderr, "[main] The `pileup' command has been removed. Please use `mpileup' instead.\n");
- return 1;
- }
-#if _CURSES_LIB != 0
- else if (strcmp(argv[1], "tview") == 0) return bam_tview_main(argc-1, argv+1);
-#endif
- else {
- fprintf(stderr, "[main] unrecognized command '%s'\n", argv[1]);
- return 1;
- }
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/Makefile b/samtools-0.1.19/bcftools/Makefile
deleted file mode 100644
index be831de..0000000
--- a/samtools-0.1.19/bcftools/Makefile
+++ /dev/null
@@ -1,51 +0,0 @@
-CC= gcc
-CFLAGS= -g -Wall -O2 #-m64 #-arch ppc
-DFLAGS= -D_FILE_OFFSET_BITS=64 -D_USE_KNETFILE
-LOBJS= bcf.o vcf.o bcfutils.o prob1.o em.o kfunc.o kmin.o index.o fet.o mut.o bcf2qcall.o
-OMISC= ..
-AOBJS= call1.o main.o $(OMISC)/kstring.o $(OMISC)/bgzf.o $(OMISC)/knetfile.o $(OMISC)/bedidx.o
-PROG= bcftools
-INCLUDES=
-SUBDIRS= .
-
-.SUFFIXES:.c .o
-
-.c.o:
- $(CC) -c $(CFLAGS) $(DFLAGS) -I.. $(INCLUDES) $< -o $@
-
-all-recur lib-recur clean-recur cleanlocal-recur install-recur:
- @target=`echo $@ | sed s/-recur//`; \
- wdir=`pwd`; \
- list='$(SUBDIRS)'; for subdir in $$list; do \
- cd $$subdir; \
- $(MAKE) CC="$(CC)" DFLAGS="$(DFLAGS)" CFLAGS="$(CFLAGS)" \
- INCLUDES="$(INCLUDES)" LIBPATH="$(LIBPATH)" $$target || exit 1; \
- cd $$wdir; \
- done;
-
-all:$(PROG)
-
-lib:libbcf.a
-
-libbcf.a:$(LOBJS)
- $(AR) -csru $@ $(LOBJS)
-
-bcftools:lib $(AOBJS)
- $(CC) $(CFLAGS) -o $@ $(AOBJS) -L. $(LIBPATH) -lbcf -lm -lz -lpthread
-
-bcf.o:bcf.h
-vcf.o:bcf.h
-index.o:bcf.h
-bcfutils.o:bcf.h
-prob1.o:prob1.h bcf.h
-call1.o:prob1.h bcf.h
-bcf2qcall.o:bcf.h
-main.o:bcf.h
-
-bcf.pdf:bcf.tex
- pdflatex bcf
-
-cleanlocal:
- rm -fr gmon.out *.o a.out *.dSYM $(PROG) *~ *.a bcf.aux bcf.log bcf.pdf *.class libbcf.*.dylib libbcf.so*
-
-clean:cleanlocal-recur
diff --git a/samtools-0.1.19/bcftools/README b/samtools-0.1.19/bcftools/README
deleted file mode 100644
index 1d7159d..0000000
--- a/samtools-0.1.19/bcftools/README
+++ /dev/null
@@ -1,36 +0,0 @@
-The view command of bcftools calls variants, tests Hardy-Weinberg
-equilibrium (HWE), tests allele balances and estimates allele frequency.
-
-This command calls a site as a potential variant if P(ref|D,F) is below
-0.9 (controlled by the -p option), where D is data and F is the prior
-allele frequency spectrum (AFS).
-
-The view command performs two types of allele balance tests, both based
-on Fisher's exact test for 2x2 contingency tables with the row variable
-being reference allele or not. In the first table, the column variable
-is strand. Two-tail P-value is taken. We test if variant bases tend to
-come from one strand. In the second table, the column variable is
-whether a base appears in the first or the last 11bp of the read.
-One-tail P-value is taken. We test if variant bases tend to occur
-towards the end of reads, which is usually an indication of
-misalignment.
-
-Site allele frequency is estimated in two ways. In the first way, the
-frequency is esimated as \argmax_f P(D|f) under the assumption of
-HWE. Prior AFS is not used. In the second way, the frequency is
-estimated as the posterior expectation of allele counts \sum_k
-kP(k|D,F), dividied by the total number of haplotypes. HWE is not
-assumed, but the estimate depends on the prior AFS. The two estimates
-largely agree when the signal is strong, but may differ greatly on weak
-sites as in this case, the prior plays an important role.
-
-To test HWE, we calculate the posterior distribution of genotypes
-(ref-hom, het and alt-hom). Chi-square test is performed. It is worth
-noting that the model used here is prior dependent and assumes HWE,
-which is different from both models for allele frequency estimate. The
-new model actually yields a third estimate of site allele frequency.
-
-The estimate allele frequency spectrum is printed to stderr per 64k
-sites. The estimate is in fact only the first round of a EM
-procedure. The second model (not the model for HWE testing) is used to
-estimate the AFS.
\ No newline at end of file
diff --git a/samtools-0.1.19/bcftools/bcf.c b/samtools-0.1.19/bcftools/bcf.c
deleted file mode 100644
index 24728db..0000000
--- a/samtools-0.1.19/bcftools/bcf.c
+++ /dev/null
@@ -1,396 +0,0 @@
-#include <string.h>
-#include <ctype.h>
-#include <stdio.h>
-#include "kstring.h"
-#include "bcf.h"
-
-bcf_t *bcf_open(const char *fn, const char *mode)
-{
- bcf_t *b;
- b = calloc(1, sizeof(bcf_t));
- if (strchr(mode, 'w')) {
- b->fp = strcmp(fn, "-")? bgzf_open(fn, mode) : bgzf_fdopen(fileno(stdout), mode);
- } else {
- b->fp = strcmp(fn, "-")? bgzf_open(fn, mode) : bgzf_fdopen(fileno(stdin), mode);
- }
- return b;
-}
-
-int bcf_close(bcf_t *b)
-{
- int ret;
- if (b == 0) return 0;
- ret = bgzf_close(b->fp);
- free(b);
- return ret;
-}
-
-int bcf_hdr_write(bcf_t *b, const bcf_hdr_t *h)
-{
- if (b == 0 || h == 0) return -1;
- bgzf_write(b->fp, "BCF\4", 4);
- bgzf_write(b->fp, &h->l_nm, 4);
- bgzf_write(b->fp, h->name, h->l_nm);
- bgzf_write(b->fp, &h->l_smpl, 4);
- bgzf_write(b->fp, h->sname, h->l_smpl);
- bgzf_write(b->fp, &h->l_txt, 4);
- bgzf_write(b->fp, h->txt, h->l_txt);
- bgzf_flush(b->fp);
- return 16 + h->l_nm + h->l_smpl + h->l_txt;
-}
-
-bcf_hdr_t *bcf_hdr_read(bcf_t *b)
-{
- uint8_t magic[4];
- bcf_hdr_t *h;
- if (b == 0) return 0;
- h = calloc(1, sizeof(bcf_hdr_t));
- bgzf_read(b->fp, magic, 4);
- bgzf_read(b->fp, &h->l_nm, 4);
- h->name = malloc(h->l_nm);
- bgzf_read(b->fp, h->name, h->l_nm);
- bgzf_read(b->fp, &h->l_smpl, 4);
- h->sname = malloc(h->l_smpl);
- bgzf_read(b->fp, h->sname, h->l_smpl);
- bgzf_read(b->fp, &h->l_txt, 4);
- h->txt = malloc(h->l_txt);
- bgzf_read(b->fp, h->txt, h->l_txt);
- bcf_hdr_sync(h);
- return h;
-}
-
-void bcf_hdr_destroy(bcf_hdr_t *h)
-{
- if (h == 0) return;
- free(h->name); free(h->sname); free(h->txt); free(h->ns); free(h->sns);
- free(h);
-}
-
-static inline char **cnt_null(int l, char *str, int *_n)
-{
- int n = 0;
- char *p, **list;
- *_n = 0;
- if (l == 0 || str == 0) return 0;
- for (p = str; p != str + l; ++p)
- if (*p == 0) ++n;
- *_n = n;
- list = calloc(n, sizeof(void*));
- list[0] = str;
- for (p = str, n = 1; p < str + l - 1; ++p)
- if (*p == 0) list[n++] = p + 1;
- return list;
-}
-
-int bcf_hdr_sync(bcf_hdr_t *b)
-{
- if (b == 0) return -1;
- if (b->ns) free(b->ns);
- if (b->sns) free(b->sns);
- if (b->l_nm) b->ns = cnt_null(b->l_nm, b->name, &b->n_ref);
- else b->ns = 0, b->n_ref = 0;
- b->sns = cnt_null(b->l_smpl, b->sname, &b->n_smpl);
- return 0;
-}
-
-int bcf_sync(bcf1_t *b)
-{
- char *p, *tmp[5];
- int i, n, n_smpl = b->n_smpl;
- ks_tokaux_t aux;
- // set ref, alt, flt, info, fmt
- b->ref = b->alt = b->flt = b->info = b->fmt = 0;
- for (p = b->str, n = 0; p < b->str + b->l_str; ++p) {
- if (*p == 0 && p+1 != b->str + b->l_str) {
- if (n == 5) {
- ++n;
- break;
- } else tmp[n++] = p + 1;
- }
- }
- if (n != 5) {
- fprintf(stderr, "[%s] incorrect number of fields (%d != 5) at %d:%d\n", __func__, n, b->tid, b->pos);
- return -1;
- }
- b->ref = tmp[0]; b->alt = tmp[1]; b->flt = tmp[2]; b->info = tmp[3]; b->fmt = tmp[4];
- // set n_alleles
- if (*b->alt == 0) b->n_alleles = 1;
- else {
- for (p = b->alt, n = 1; *p; ++p)
- if (*p == ',') ++n;
- b->n_alleles = n + 1;
- }
- // set n_gi and gi[i].fmt
- for (p = b->fmt, n = 1; *p; ++p)
- if (*p == ':') ++n;
- if (n > b->m_gi) {
- int old_m = b->m_gi;
- b->m_gi = n;
- kroundup32(b->m_gi);
- b->gi = realloc(b->gi, b->m_gi * sizeof(bcf_ginfo_t));
- memset(b->gi + old_m, 0, (b->m_gi - old_m) * sizeof(bcf_ginfo_t));
- }
- b->n_gi = n;
- for (p = kstrtok(b->fmt, ":", &aux), n = 0; p; p = kstrtok(0, 0, &aux))
- b->gi[n++].fmt = bcf_str2int(p, aux.p - p);
- // set gi[i].len
- for (i = 0; i < b->n_gi; ++i) {
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
- b->gi[i].len = b->n_alleles * (b->n_alleles + 1) / 2;
- } else if (b->gi[i].fmt == bcf_str2int("DP", 2) || b->gi[i].fmt == bcf_str2int("HQ", 2) || b->gi[i].fmt == bcf_str2int("DV", 2)) {
- b->gi[i].len = 2;
- } else if (b->gi[i].fmt == bcf_str2int("GQ", 2) || b->gi[i].fmt == bcf_str2int("GT", 2)) {
- b->gi[i].len = 1;
- } else if (b->gi[i].fmt == bcf_str2int("SP", 2)) {
- b->gi[i].len = 4;
- } else if (b->gi[i].fmt == bcf_str2int("GL", 2)) {
- b->gi[i].len = b->n_alleles * (b->n_alleles + 1) / 2 * 4;
- }
- b->gi[i].data = realloc(b->gi[i].data, n_smpl * b->gi[i].len);
- }
- return 0;
-}
-
-int bcf_write(bcf_t *bp, const bcf_hdr_t *h, const bcf1_t *b)
-{
- int i, l = 0;
- if (b == 0) return -1;
- bgzf_write(bp->fp, &b->tid, 4);
- bgzf_write(bp->fp, &b->pos, 4);
- bgzf_write(bp->fp, &b->qual, 4);
- bgzf_write(bp->fp, &b->l_str, 4);
- bgzf_write(bp->fp, b->str, b->l_str);
- l = 12 + b->l_str;
- for (i = 0; i < b->n_gi; ++i) {
- bgzf_write(bp->fp, b->gi[i].data, b->gi[i].len * h->n_smpl);
- l += b->gi[i].len * h->n_smpl;
- }
- return l;
-}
-
-int bcf_read(bcf_t *bp, const bcf_hdr_t *h, bcf1_t *b)
-{
- int i, l = 0;
- if (b == 0) return -1;
- if (bgzf_read(bp->fp, &b->tid, 4) == 0) return -1;
- b->n_smpl = h->n_smpl;
- bgzf_read(bp->fp, &b->pos, 4);
- bgzf_read(bp->fp, &b->qual, 4);
- bgzf_read(bp->fp, &b->l_str, 4);
- if (b->l_str > b->m_str) {
- b->m_str = b->l_str;
- kroundup32(b->m_str);
- b->str = realloc(b->str, b->m_str);
- }
- bgzf_read(bp->fp, b->str, b->l_str);
- l = 12 + b->l_str;
- if (bcf_sync(b) < 0) return -2;
- for (i = 0; i < b->n_gi; ++i) {
- bgzf_read(bp->fp, b->gi[i].data, b->gi[i].len * h->n_smpl);
- l += b->gi[i].len * h->n_smpl;
- }
- return l;
-}
-
-int bcf_destroy(bcf1_t *b)
-{
- int i;
- if (b == 0) return -1;
- free(b->str);
- for (i = 0; i < b->m_gi; ++i)
- free(b->gi[i].data);
- free(b->gi);
- free(b);
- return 0;
-}
-
-static inline void fmt_str(const char *p, kstring_t *s)
-{
- if (*p == 0) kputc('.', s);
- else kputs(p, s);
-}
-
-void bcf_fmt_core(const bcf_hdr_t *h, bcf1_t *b, kstring_t *s)
-{
- int i, j, x;
- s->l = 0;
- if (h->n_ref) kputs(h->ns[b->tid], s);
- else kputw(b->tid, s);
- kputc('\t', s);
- kputw(b->pos + 1, s); kputc('\t', s);
- fmt_str(b->str, s); kputc('\t', s);
- fmt_str(b->ref, s); kputc('\t', s);
- fmt_str(b->alt, s); kputc('\t', s);
- ksprintf(s, "%.3g", b->qual); kputc('\t', s);
- fmt_str(b->flt, s); kputc('\t', s);
- fmt_str(b->info, s);
- if (b->fmt[0]) {
- kputc('\t', s);
- fmt_str(b->fmt, s);
- }
- x = b->n_alleles * (b->n_alleles + 1) / 2;
- if (b->n_gi == 0) return;
- int iPL = -1;
- if ( b->n_alleles > 2 ) {
- for (i=0; i<b->n_gi; i++) {
- if ( b->gi[i].fmt == bcf_str2int("PL", 2) ) {
- iPL = i;
- break;
- }
- }
- }
- for (j = 0; j < h->n_smpl; ++j) {
- int ploidy = b->ploidy ? b->ploidy[j] : 2;
- kputc('\t', s);
- for (i = 0; i < b->n_gi; ++i) {
- if (i) kputc(':', s);
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
- uint8_t *d = (uint8_t*)b->gi[i].data + j * x;
- int k;
- if ( ploidy==1 )
- for (k=0; k<b->n_alleles; k++)
- {
- if (k>0) kputc(',', s);
- kputw(d[(k+1)*(k+2)/2-1], s);
- }
- else
- for (k = 0; k < x; ++k) {
- if (k > 0) kputc(',', s);
- kputw(d[k], s);
- }
- } else if (b->gi[i].fmt == bcf_str2int("DP", 2) || b->gi[i].fmt == bcf_str2int("DV", 2)) {
- kputw(((uint16_t*)b->gi[i].data)[j], s);
- } else if (b->gi[i].fmt == bcf_str2int("GQ", 2)) {
- kputw(((uint8_t*)b->gi[i].data)[j], s);
- } else if (b->gi[i].fmt == bcf_str2int("SP", 2)) {
- kputw(((int32_t*)b->gi[i].data)[j], s);
- } else if (b->gi[i].fmt == bcf_str2int("GT", 2)) {
- int y = ((uint8_t*)b->gi[i].data)[j];
- if ( ploidy==1 )
- {
- if ( y>>7&1 )
- kputc('.', s);
- else
- kputc('0' + (y>>3&7), s);
- }
- else
- {
- if ( y>>7&1 )
- kputsn("./.", 3, s);
- else {
- kputc('0' + (y>>3&7), s);
- kputc("/|"[y>>6&1], s);
- kputc('0' + (y&7), s);
- }
- }
- } else if (b->gi[i].fmt == bcf_str2int("GL", 2)) {
- float *d = (float*)b->gi[i].data + j * x;
- int k;
- //printf("- %lx\n", d);
- for (k = 0; k < x; ++k) {
- if (k > 0) kputc(',', s);
- ksprintf(s, "%.2f", d[k]);
- }
- } else kputc('.', s); // custom fields
- }
- }
-}
-
-char *bcf_fmt(const bcf_hdr_t *h, bcf1_t *b)
-{
- kstring_t s;
- s.l = s.m = 0; s.s = 0;
- bcf_fmt_core(h, b, &s);
- return s.s;
-}
-
-int bcf_append_info(bcf1_t *b, const char *info, int l)
-{
- int shift = b->fmt - b->str;
- int l_fmt = b->l_str - shift;
- char *ori = b->str;
- if (b->l_str + l > b->m_str) { // enlarge if necessary
- b->m_str = b->l_str + l;
- kroundup32(b->m_str);
- b->str = realloc(b->str, b->m_str);
- }
- memmove(b->str + shift + l, b->str + shift, l_fmt); // move the FORMAT field
- memcpy(b->str + shift - 1, info, l); // append to the INFO field
- b->str[shift + l - 1] = '\0';
- b->fmt = b->str + shift + l;
- b->l_str += l;
- if (ori != b->str) bcf_sync(b); // synchronize when realloc changes the pointer
- return 0;
-}
-
-int remove_tag(char *str, const char *tag, char delim)
-{
- char *tmp = str, *p;
- int len_diff = 0, ori_len = strlen(str);
- while ( *tmp && (p = strstr(tmp,tag)) )
- {
- if ( p>str )
- {
- if ( *(p-1)!=delim ) { tmp=p+1; continue; } // shared substring
- p--;
- }
- char *q=p+1;
- while ( *q && *q!=delim ) q++;
- if ( p==str && *q ) q++; // the tag is first, don't move the delim char
- len_diff += q-p;
- if ( ! *q ) { *p = 0; break; } // the tag was last, no delim follows
- else
- memmove(p,q,ori_len-(int)(p-str)-(int)(q-p)); // *q==delim
- }
- if ( len_diff==ori_len )
- str[0]='.', str[1]=0, len_diff--;
-
- return len_diff;
-}
-
-
-void rm_info(kstring_t *s, const char *key)
-{
- char *p = s->s;
- int n = 0;
- while ( n<4 )
- {
- if ( !*p ) n++;
- p++;
- }
- char *q = p+1;
- while ( *q && q-s->s<s->l ) q++;
-
- int nrm = remove_tag(p, key, ';');
- if ( nrm )
- memmove(q-nrm, q, s->s+s->l-q+1);
- s->l -= nrm;
-}
-
-int bcf_cpy(bcf1_t *r, const bcf1_t *b)
-{
- char *t1 = r->str;
- bcf_ginfo_t *t2 = r->gi;
- int i, t3 = r->m_str, t4 = r->m_gi;
- *r = *b;
- r->str = t1; r->gi = t2; r->m_str = t3; r->m_gi = t4;
- if (r->m_str < b->m_str) {
- r->m_str = b->m_str;
- r->str = realloc(r->str, r->m_str);
- }
- memcpy(r->str, b->str, r->m_str);
- bcf_sync(r); // calling bcf_sync() is simple but inefficient
- for (i = 0; i < r->n_gi; ++i)
- memcpy(r->gi[i].data, b->gi[i].data, r->n_smpl * r->gi[i].len);
- return 0;
-}
-
-int bcf_is_indel(const bcf1_t *b)
-{
- char *p;
- if (strlen(b->ref) > 1) return 1;
- for (p = b->alt; *p; ++p)
- if (*p != ',' && p[1] != ',' && p[1] != '\0')
- return 1;
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/bcf.h b/samtools-0.1.19/bcftools/bcf.h
deleted file mode 100644
index 3315809..0000000
--- a/samtools-0.1.19/bcftools/bcf.h
+++ /dev/null
@@ -1,197 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2010 Broad Institute
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Contact: Heng Li <lh3 at live.co.uk> */
-
-#ifndef BCF_H
-#define BCF_H
-
-#define BCF_VERSION "0.1.19-96b5f2294a"
-
-#include <stdint.h>
-#include <zlib.h>
-
-#ifndef BCF_LITE
-#include "bgzf.h"
-typedef BGZF *bcfFile;
-#else
-typedef gzFile bcfFile;
-#define bgzf_open(fn, mode) gzopen(fn, mode)
-#define bgzf_fdopen(fd, mode) gzdopen(fd, mode)
-#define bgzf_close(fp) gzclose(fp)
-#define bgzf_read(fp, buf, len) gzread(fp, buf, len)
-#define bgzf_write(fp, buf, len)
-#define bgzf_flush(fp)
-#endif
-
-/*
- A member in the structs below is said to "primary" if its content
- cannot be inferred from other members in any of structs below; a
- member is said to be "derived" if its content can be derived from
- other members. For example, bcf1_t::str is primary as this comes from
- the input data, while bcf1_t::info is derived as it can always be
- correctly set if we know bcf1_t::str. Derived members are for quick
- access to the content and must be synchronized with the primary data.
- */
-
-typedef struct {
- uint32_t fmt; // format of the block, set by bcf_str2int().
- int len; // length of data for each individual
- void *data; // concatenated data
- // derived info: fmt, len (<-bcf1_t::fmt)
-} bcf_ginfo_t;
-
-typedef struct {
- int32_t tid, pos; // refID and 0-based position
- int32_t l_str, m_str; // length and the allocated size of ->str
- float qual; // SNP quality
- char *str; // concatenated string of variable length strings in VCF (from col.2 to col.7)
- char *ref, *alt, *flt, *info, *fmt; // they all point to ->str; no memory allocation
- int n_gi, m_gi; // number and the allocated size of geno fields
- bcf_ginfo_t *gi; // array of geno fields
- int n_alleles, n_smpl; // number of alleles and samples
- // derived info: ref, alt, flt, info, fmt (<-str), n_gi (<-fmt), n_alleles (<-alt), n_smpl (<-bcf_hdr_t::n_smpl)
- uint8_t *ploidy; // ploidy of all samples; if NULL, ploidy of 2 is assumed.
-} bcf1_t;
-
-typedef struct {
- int32_t n_ref, n_smpl; // number of reference sequences and samples
- int32_t l_nm; // length of concatenated sequence names; 0 padded
- int32_t l_smpl; // length of concatenated sample names; 0 padded
- int32_t l_txt; // length of header text (lines started with ##)
- char *name, *sname, *txt; // concatenated sequence names, sample names and header text
- char **ns, **sns; // array of sequence and sample names; point to name and sname, respectively
- // derived info: n_ref (<-name), n_smpl (<-sname), ns (<-name), sns (<-sname)
-} bcf_hdr_t;
-
-typedef struct {
- int is_vcf; // if the file in operation is a VCF
- void *v; // auxillary data structure for VCF
- bcfFile fp; // file handler for BCF
-} bcf_t;
-
-struct __bcf_idx_t;
-typedef struct __bcf_idx_t bcf_idx_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- // open a BCF file; for BCF file only
- bcf_t *bcf_open(const char *fn, const char *mode);
- // close file
- int bcf_close(bcf_t *b);
- // read one record from BCF; return -1 on end-of-file, and <-1 for errors
- int bcf_read(bcf_t *bp, const bcf_hdr_t *h, bcf1_t *b);
- // call this function if b->str is changed
- int bcf_sync(bcf1_t *b);
- // write a BCF record
- int bcf_write(bcf_t *bp, const bcf_hdr_t *h, const bcf1_t *b);
- // read the BCF header; BCF only
- bcf_hdr_t *bcf_hdr_read(bcf_t *b);
- // write the BCF header
- int bcf_hdr_write(bcf_t *b, const bcf_hdr_t *h);
- // set bcf_hdr_t::ns and bcf_hdr_t::sns
- int bcf_hdr_sync(bcf_hdr_t *b);
- // destroy the header
- void bcf_hdr_destroy(bcf_hdr_t *h);
- // destroy a record
- int bcf_destroy(bcf1_t *b);
- // BCF->VCF conversion
- char *bcf_fmt(const bcf_hdr_t *h, bcf1_t *b);
- // append more info
- int bcf_append_info(bcf1_t *b, const char *info, int l);
- // remove tag
- int remove_tag(char *string, const char *tag, char delim);
- // remove info tag, string is the kstring holder of bcf1_t.str
- void rm_info(kstring_t *string, const char *key);
- // copy
- int bcf_cpy(bcf1_t *r, const bcf1_t *b);
-
- // open a VCF or BCF file if "b" is set in "mode"
- bcf_t *vcf_open(const char *fn, const char *mode);
- // close a VCF/BCF file
- int vcf_close(bcf_t *bp);
- // read the VCF/BCF header
- bcf_hdr_t *vcf_hdr_read(bcf_t *bp);
- // read the sequence dictionary from a separate file; required for VCF->BCF conversion
- int vcf_dictread(bcf_t *bp, bcf_hdr_t *h, const char *fn);
- // read a VCF/BCF record; return -1 on end-of-file and <-1 for errors
- int vcf_read(bcf_t *bp, bcf_hdr_t *h, bcf1_t *b);
- // write the VCF header
- int vcf_hdr_write(bcf_t *bp, const bcf_hdr_t *h);
- // write a VCF record
- int vcf_write(bcf_t *bp, bcf_hdr_t *h, bcf1_t *b);
-
- // keep the first n alleles and discard the rest
- int bcf_shrink_alt(bcf1_t *b, int n);
- // keep the masked alleles and discard the rest
- void bcf_fit_alt(bcf1_t *b, int mask);
- // convert GL to PL
- int bcf_gl2pl(bcf1_t *b);
- // if the site is an indel
- int bcf_is_indel(const bcf1_t *b);
- bcf_hdr_t *bcf_hdr_subsam(const bcf_hdr_t *h0, int n, char *const* samples, int *list);
- int bcf_subsam(int n_smpl, int *list, bcf1_t *b);
- // move GT to the first FORMAT field
- int bcf_fix_gt(bcf1_t *b);
- // update PL generated by old samtools
- int bcf_fix_pl(bcf1_t *b);
- // convert PL to GLF-like 10-likelihood GL
- int bcf_gl10(const bcf1_t *b, uint8_t *gl);
- // convert up to 4 INDEL alleles to GLF-like 10-likelihood GL
- int bcf_gl10_indel(const bcf1_t *b, uint8_t *gl);
-
- // string hash table
- void *bcf_build_refhash(bcf_hdr_t *h);
- void bcf_str2id_destroy(void *_hash);
- void bcf_str2id_thorough_destroy(void *_hash);
- int bcf_str2id_add(void *_hash, const char *str);
- int bcf_str2id(void *_hash, const char *str);
- void *bcf_str2id_init();
-
- // indexing related functions
- int bcf_idx_build(const char *fn);
- uint64_t bcf_idx_query(const bcf_idx_t *idx, int tid, int beg);
- int bcf_parse_region(void *str2id, const char *str, int *tid, int *begin, int *end);
- bcf_idx_t *bcf_idx_load(const char *fn);
- void bcf_idx_destroy(bcf_idx_t *idx);
-
-#ifdef __cplusplus
-}
-#endif
-
-static inline uint32_t bcf_str2int(const char *str, int l)
-{
- int i;
- uint32_t x = 0;
- for (i = 0; i < l && i < 4; ++i) {
- if (str[i] == 0) return x;
- x = x<<8 | str[i];
- }
- return x;
-}
-
-#endif
diff --git a/samtools-0.1.19/bcftools/bcf.tex b/samtools-0.1.19/bcftools/bcf.tex
deleted file mode 100644
index 442fc2a..0000000
--- a/samtools-0.1.19/bcftools/bcf.tex
+++ /dev/null
@@ -1,77 +0,0 @@
-\documentclass[10pt,pdftex]{article}
-\usepackage{color}
-\definecolor{gray}{rgb}{0.7,0.7,0.7}
-
-\setlength{\topmargin}{0.0cm}
-\setlength{\textheight}{21.5cm}
-\setlength{\oddsidemargin}{0cm}
-\setlength{\textwidth}{16.5cm}
-\setlength{\columnsep}{0.6cm}
-
-\begin{document}
-
-\begin{center}
-\begin{tabular}{|l|l|l|l|l|}
-\hline
-\multicolumn{2}{|c|}{\bf Field} & \multicolumn{1}{c|}{\bf Descrption} & \multicolumn{1}{c|}{\bf Type} & \multicolumn{1}{c|}{\bf Value} \\\hline\hline
-\multicolumn{2}{|l|}{\sf magic} & Magic string & {\tt char[4]} & {\tt BCF\char92 4} \\\hline
-\multicolumn{2}{|l|}{\sf l\_seqnm} & Length of concatenated sequence names & {\tt int32\_t} & \\\hline
-\multicolumn{2}{|l|}{\sf seqnm} & Concatenated names, {\tt NULL} padded & {\tt char[{\sf l\_seqnm}]} & \\\hline
-\multicolumn{2}{|l|}{\sf l\_smpl} & Length of concatenated sample names & {\tt int32\_t} & \\\hline
-\multicolumn{2}{|l|}{\sf smpl} & Concatenated sample names & {\tt char[{\sf l\_smpl}]} & \\\hline
-\multicolumn{2}{|l|}{\sf l\_meta} & Length of the meta text (double-hash lines)& {\tt int32\_t} & \\\hline
-\multicolumn{2}{|l|}{\sf meta} & Meta text, {\tt NULL} terminated & {\tt char[{\sf l\_meta}]} & \\\hline
-\multicolumn{5}{|c|}{\it \color{gray}{List of records until the end of the file}}\\\cline{2-5}
-& {\sf seq\_id} & Reference sequence ID & {\tt int32\_t} & \\\cline{2-5}
-& {\sf pos} & Position & {\tt int32\_t} & \\\cline{2-5}
-& {\sf qual} & Variant quality & {\tt float} & \\\cline{2-5}
-& {\sf l\_str} & Length of {\sf str} & {\tt int32\_t} & \\\cline{2-5}
-& {\sf str} & {\tt ID+REF+ALT+FILTER+INFO+FORMAT}, {\tt NULL} padded & {\tt char[{\sf l\_str}]} &\\\cline{2-5}
-& \multicolumn{4}{c|}{Blocks of data; \#blocks and formats defined by {\tt FORMAT} (table below)}\\
-\hline
-\end{tabular}
-\end{center}
-
-\begin{center}
-\begin{tabular}{clp{9cm}}
-\hline
-\multicolumn{1}{l}{\bf Field} & \multicolumn{1}{l}{\bf Type} & \multicolumn{1}{l}{\bf Description} \\\hline
-{\tt DP} & {\tt uint16\_t[n]} & Read depth \\
-{\tt GL} & {\tt float[n*G]} & Log10 likelihood of data; $G=\frac{A(A+1)}{2}$, $A=\#\{alleles\}$\\
-{\tt GT} & {\tt uint8\_t[n]} & {\tt missing\char60\char60 7 | phased\char60\char60 6 | allele1\char60\char60 3 | allele2} \\
-{\tt \_GT} & {\tt uint8\_t+uint8\_t[n*P]} & {Generic GT; the first int equals the max ploidy $P$. If the highest bit is set,
- the allele is not present (e.g. due to different ploidy between samples).} \\
-{\tt GQ} & {\tt uint8\_t[n]} & {Genotype quality}\\
-{\tt HQ} & {\tt uint8\_t[n*2]} & {Haplotype quality}\\
-{\tt \_HQ} & {\tt uint8\_t+uint8\_t[n*P]} & {Generic HQ}\\
-{\tt IBD} & {\tt uint32\_t[n*2]} & {IBD}\\
-{\tt \_IBD} & {\tt uint8\_t+uint32\_t[n*P]} & {Generic IBD}\\
-{\tt PL} & {\tt uint8\_t[n*G]} & {Phred-scaled likelihood of data}\\
-{\tt PS} & {\tt uint32\_t[n]} & {Phase set}\\
-%{\tt SP} & {\tt uint8\_t[n]} & {Strand bias P-value (bcftools only)}\\
-\emph{Integer} & {\tt int32\_t[n*X]} & {Fix-sized custom Integer; $X$ defined in the header}\\
-\emph{Numeric} & {\tt double[n*X]} & {Fix-sized custom Numeric}\\
-\emph{String} & {\tt uint32\_t+char*} & {\tt NULL} padded concat. strings (int equals to the length) \\
-\hline
-\end{tabular}
-\end{center}
-
-\begin{itemize}
-\item A BCF file is in the {\tt BGZF} format.
-\item All multi-byte numbers are little-endian.
-\item In a string, a missing value `.' is an empty C string ``{\tt
- \char92 0}'' (not ``{\tt .\char92 0}'')
-\item For {\tt GL} and {\tt PL}, likelihoods of genotypes appear in the
- order of alleles in {\tt REF} and then {\tt ALT}. For example, if {\tt
- REF=C}, {\tt ALT=T,A}, likelihoods appear in the order of {\tt
- CC,CT,TT,CA,TA,AA} (NB: the ordering is different from the one in the original
- BCF proposal).
-\item Predefined {\tt FORMAT} fields can be missing from VCF headers, but custom {\tt FORMAT} fields
- are required to be explicitly defined in the headers.
-\item A {\tt FORMAT} field with its name starting with `{\tt \_}' is specific to BCF only.
- It gives an alternative binary representation of the corresponding VCF field, in case
- the default representation is unable to keep the genotype information,
- for example, when the ploidy is not 2 or there are more than 8 alleles.
-\end{itemize}
-
-\end{document}
diff --git a/samtools-0.1.19/bcftools/bcf2qcall.c b/samtools-0.1.19/bcftools/bcf2qcall.c
deleted file mode 100644
index a86bac2..0000000
--- a/samtools-0.1.19/bcftools/bcf2qcall.c
+++ /dev/null
@@ -1,91 +0,0 @@
-#include <errno.h>
-#include <math.h>
-#include <string.h>
-#include <stdlib.h>
-#include "bcf.h"
-
-static int8_t nt4_table[256] = {
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 /*'-'*/, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, -1, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, -1, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
-};
-
-static int read_I16(bcf1_t *b, int anno[16])
-{
- char *p;
- int i;
- if ((p = strstr(b->info, "I16=")) == 0) return -1;
- p += 4;
- for (i = 0; i < 16; ++i) {
- anno[i] = strtol(p, &p, 10);
- if (anno[i] == 0 && (errno == EINVAL || errno == ERANGE)) return -2;
- ++p;
- }
- return 0;
-}
-
-int bcf_2qcall(bcf_hdr_t *h, bcf1_t *b)
-{
- int a[4], k, g[10], l, map[4], k1, j, i, i0, anno[16], dp, mq, d_rest;
- char *s;
- if (b->ref[1] != 0 || b->n_alleles > 4) return -1; // ref is not a single base
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) break;
- if (i == b->n_gi) return -1; // no PL
- if (read_I16(b, anno) != 0) return -1; // no I16; FIXME: can be improved
- d_rest = dp = anno[0] + anno[1] + anno[2] + anno[3];
- if (dp == 0) return -1; // depth is zero
- mq = (int)(sqrt((double)(anno[9] + anno[11]) / dp) + .499);
- i0 = i;
- a[0] = nt4_table[(int)b->ref[0]];
- if (a[0] > 3) return -1; // ref is not A/C/G/T
- a[1] = a[2] = a[3] = -2; // -1 has a special meaning
- if (b->alt[0] == 0) return -1; // no alternate allele
- map[0] = map[1] = map[2] = map[3] = -2;
- map[a[0]] = 0;
- for (k = 0, s = b->alt, k1 = -1; k < 3 && *s; ++k, s += 2) {
- if (s[1] != ',' && s[1] != 0) return -1; // ALT is not single base
- a[k+1] = nt4_table[(int)*s];
- if (a[k+1] >= 0) map[a[k+1]] = k+1;
- else k1 = k+1;
- if (s[1] == 0) break;
- }
- for (k = 0; k < 4; ++k)
- if (map[k] < 0) map[k] = k1;
- for (i = 0; i < h->n_smpl; ++i) {
- int d;
- uint8_t *p = b->gi[i0].data + i * b->gi[i0].len;
- for (j = 0; j < b->gi[i0].len; ++j)
- if (p[j]) break;
- d = (int)((double)d_rest / (h->n_smpl - i) + .499);
- if (d == 0) d = 1;
- if (j == b->gi[i0].len) d = 0;
- d_rest -= d;
- for (k = j = 0; k < 4; ++k) {
- for (l = k; l < 4; ++l) {
- int t, x = map[k], y = map[l];
- if (x > y) t = x, x = y, y = t; // swap
- g[j++] = p[y * (y+1) / 2 + x];
- }
- }
- printf("%s\t%d\t%c", h->ns[b->tid], b->pos+1, *b->ref);
- printf("\t%d\t%d\t0", d, mq);
- for (j = 0; j < 10; ++j)
- printf("\t%d", g[j]);
- printf("\t%s\n", h->sns[i]);
- }
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/bcfutils.c b/samtools-0.1.19/bcftools/bcfutils.c
deleted file mode 100644
index 7638085..0000000
--- a/samtools-0.1.19/bcftools/bcfutils.c
+++ /dev/null
@@ -1,504 +0,0 @@
-#include <string.h>
-#include <math.h>
-#include <assert.h>
-#include "bcf.h"
-#include "kstring.h"
-#include "khash.h"
-KHASH_MAP_INIT_STR(str2id, int)
-
-#ifdef _WIN32
-#define srand48(x) srand(x)
-#define drand48() ((double)rand() / RAND_MAX)
-#endif
-
-// FIXME: valgrind report a memory leak in this function. Probably it does not get deallocated...
-void *bcf_build_refhash(bcf_hdr_t *h)
-{
- khash_t(str2id) *hash;
- int i, ret;
- hash = kh_init(str2id);
- for (i = 0; i < h->n_ref; ++i) {
- khint_t k;
- k = kh_put(str2id, hash, h->ns[i], &ret); // FIXME: check ret
- kh_val(hash, k) = i;
- }
- return hash;
-}
-
-void *bcf_str2id_init()
-{
- return kh_init(str2id);
-}
-
-void bcf_str2id_destroy(void *_hash)
-{
- khash_t(str2id) *hash = (khash_t(str2id)*)_hash;
- if (hash) kh_destroy(str2id, hash); // Note that strings are not freed.
-}
-
-void bcf_str2id_thorough_destroy(void *_hash)
-{
- khash_t(str2id) *hash = (khash_t(str2id)*)_hash;
- khint_t k;
- if (hash == 0) return;
- for (k = 0; k < kh_end(hash); ++k)
- if (kh_exist(hash, k)) free((char*)kh_key(hash, k));
- kh_destroy(str2id, hash);
-}
-
-int bcf_str2id(void *_hash, const char *str)
-{
- khash_t(str2id) *hash = (khash_t(str2id)*)_hash;
- khint_t k;
- if (!hash) return -1;
- k = kh_get(str2id, hash, str);
- return k == kh_end(hash)? -1 : kh_val(hash, k);
-}
-
-int bcf_str2id_add(void *_hash, const char *str)
-{
- khint_t k;
- int ret;
- khash_t(str2id) *hash = (khash_t(str2id)*)_hash;
- if (!hash) return -1;
- k = kh_put(str2id, hash, str, &ret);
- if (ret == 0) return kh_val(hash, k);
- kh_val(hash, k) = kh_size(hash) - 1;
- return kh_val(hash, k);
-}
-
-void bcf_fit_alt(bcf1_t *b, int mask)
-{
- mask |= 1; // REF must be always present
-
- int i,j,nals=0;
- for (i=0; i<sizeof(int); i++)
- if ( mask&1<<i) nals++;
- if ( b->n_alleles <= nals ) return;
-
- // update ALT, in principle any of the alleles can be removed
- char *p;
- if ( nals>1 )
- {
- char *dst, *src;
- int n=0, nalts=nals-1;
- for (src=dst=p=b->alt, i=1; *p; p++)
- {
- if ( *p!=',' ) continue;
-
- if ( mask&1<<i )
- {
- n++;
- if ( src!=dst )
- {
- memmove(dst,src,p-src);
- dst += p-src;
- }
- else dst = p;
- if ( n<nalts ) { *dst=','; dst++; }
- }
- i++;
-
- if ( n>=nalts ) { *dst=0; break; }
- src = p+1;
- }
- if ( n<nalts )
- {
- memmove(dst,src,p-src);
- dst += p-src;
- *dst = 0;
- }
- p = dst;
- }
- else p = b->alt, *p = '\0';
- p++;
- memmove(p, b->flt, b->str + b->l_str - b->flt);
- b->l_str -= b->flt - p;
-
- // update PL and GT
- int ipl=-1, igt=-1;
- for (i = 0; i < b->n_gi; ++i)
- {
- bcf_ginfo_t *g = b->gi + i;
- if (g->fmt == bcf_str2int("PL", 2)) ipl = i;
- if (g->fmt == bcf_str2int("GT", 2)) igt = i;
- }
-
- // .. create mapping between old and new indexes
- int npl = nals * (nals+1) / 2;
- int *map = malloc(sizeof(int)*(npl>b->n_alleles ? npl : b->n_alleles));
- int kori=0,knew=0;
- for (i=0; i<b->n_alleles; i++)
- {
- for (j=0; j<=i; j++)
- {
- int skip=0;
- if ( i && !(mask&1<<i) ) skip=1;
- if ( j && !(mask&1<<j) ) skip=1;
- if ( !skip ) { map[knew++] = kori; }
- kori++;
- }
- }
- // .. apply to all samples
- int n_smpl = b->n_smpl;
- for (i = 0; i < b->n_gi; ++i)
- {
- bcf_ginfo_t *g = b->gi + i;
- if (g->fmt == bcf_str2int("PL", 2))
- {
- g->len = npl;
- uint8_t *d = (uint8_t*)g->data;
- int ismpl, npl_ori = b->n_alleles * (b->n_alleles + 1) / 2;
- for (knew=ismpl=0; ismpl<n_smpl; ismpl++)
- {
- uint8_t *dl = d + ismpl * npl_ori;
- for (j=0; j<npl; j++) d[knew++] = dl[map[j]];
- }
- } // FIXME: to add GL
- }
- // update GTs
- map[0] = 0;
- for (i=1, knew=0; i<b->n_alleles; i++)
- map[i] = mask&1<<i ? ++knew : -1;
- for (i=0; i<n_smpl; i++)
- {
- uint8_t gt = ((uint8_t*)b->gi[igt].data)[i];
- int a1 = (gt>>3)&7;
- int a2 = gt&7;
- assert( map[a1]>=0 && map[a2]>=0 );
- ((uint8_t*)b->gi[igt].data)[i] = ((1<<7|1<<6)>) | map[a1]<<3 | map[a2];
- }
- free(map);
- b->n_alleles = nals;
- bcf_sync(b);
-}
-
-int bcf_shrink_alt(bcf1_t *b, int n)
-{
- char *p;
- int i, j, k, n_smpl = b->n_smpl;
- if (b->n_alleles <= n) return -1;
- // update ALT
- if (n > 1) {
- for (p = b->alt, k = 1; *p; ++p)
- if (*p == ',' && ++k == n) break;
- *p = '\0';
- } else p = b->alt, *p = '\0';
- ++p;
- memmove(p, b->flt, b->str + b->l_str - b->flt);
- b->l_str -= b->flt - p;
- // update PL
- for (i = 0; i < b->n_gi; ++i) {
- bcf_ginfo_t *g = b->gi + i;
- if (g->fmt == bcf_str2int("PL", 2)) {
- int l, x = b->n_alleles * (b->n_alleles + 1) / 2;
- uint8_t *d = (uint8_t*)g->data;
- g->len = n * (n + 1) / 2;
- for (l = k = 0; l < n_smpl; ++l) {
- uint8_t *dl = d + l * x;
- for (j = 0; j < g->len; ++j) d[k++] = dl[j];
- }
- } // FIXME: to add GL
- }
- b->n_alleles = n;
- bcf_sync(b);
- return 0;
-}
-
-int bcf_gl2pl(bcf1_t *b)
-{
- char *p;
- int i, n_smpl = b->n_smpl;
- bcf_ginfo_t *g;
- float *d0;
- uint8_t *d1;
- if (strstr(b->fmt, "PL")) return -1;
- if ((p = strstr(b->fmt, "GL")) == 0) return -1;
- *p = 'P';
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("GL", 2))
- break;
- g = b->gi + i;
- g->fmt = bcf_str2int("PL", 2);
- g->len /= 4; // 4 == sizeof(float)
- d0 = (float*)g->data; d1 = (uint8_t*)g->data;
- for (i = 0; i < n_smpl * g->len; ++i) {
- int x = (int)(-10. * d0[i] + .499);
- if (x > 255) x = 255;
- if (x < 0) x = 0;
- d1[i] = x;
- }
- return 0;
-}
-/* FIXME: this function will fail given AB:GTX:GT. BCFtools never
- * produces such FMT, but others may do. */
-int bcf_fix_gt(bcf1_t *b)
-{
- char *s;
- int i;
- uint32_t tmp;
- bcf_ginfo_t gt;
- // check the presence of the GT FMT
- if ((s = strstr(b->fmt, ":GT")) == 0) return 0; // no GT or GT is already the first
- assert(s[3] == '\0' || s[3] == ':'); // :GTX in fact
- tmp = bcf_str2int("GT", 2);
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == tmp) break;
- if (i == b->n_gi) return 0; // no GT in b->gi; probably a bug...
- gt = b->gi[i];
- // move GT to the first
- for (; i > 0; --i) b->gi[i] = b->gi[i-1];
- b->gi[0] = gt;
- if ( s[3]==0 )
- memmove(b->fmt + 3, b->fmt, s - b->fmt); // :GT
- else
- memmove(b->fmt + 3, b->fmt, s - b->fmt + 1); // :GT:
- b->fmt[0] = 'G'; b->fmt[1] = 'T'; b->fmt[2] = ':';
- return 0;
-}
-
-int bcf_fix_pl(bcf1_t *b)
-{
- int i;
- uint32_t tmp;
- uint8_t *PL, *swap;
- bcf_ginfo_t *gi;
- // pinpoint PL
- tmp = bcf_str2int("PL", 2);
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == tmp) break;
- if (i == b->n_gi) return 0;
- // prepare
- gi = b->gi + i;
- PL = (uint8_t*)gi->data;
- swap = alloca(gi->len);
- // loop through individuals
- for (i = 0; i < b->n_smpl; ++i) {
- int k, l, x;
- uint8_t *PLi = PL + i * gi->len;
- memcpy(swap, PLi, gi->len);
- for (k = x = 0; k < b->n_alleles; ++k)
- for (l = k; l < b->n_alleles; ++l)
- PLi[l*(l+1)/2 + k] = swap[x++];
- }
- return 0;
-}
-
-int bcf_smpl_covered(const bcf1_t *b)
-{
- int i, j, n = 0;
- uint32_t tmp;
- bcf_ginfo_t *gi;
- // pinpoint PL
- tmp = bcf_str2int("PL", 2);
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == tmp) break;
- if (i == b->n_gi) return 0;
- // count how many samples having PL!=[0..0]
- gi = b->gi + i;
- for (i = 0; i < b->n_smpl; ++i) {
- uint8_t *PLi = ((uint8_t*)gi->data) + i * gi->len;
- for (j = 0; j < gi->len; ++j)
- if (PLi[j]) break;
- if (j < gi->len) ++n;
- }
- return n;
-}
-
-static void *locate_field(const bcf1_t *b, const char *fmt, int l)
-{
- int i;
- uint32_t tmp;
- tmp = bcf_str2int(fmt, l);
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == tmp) break;
- return i == b->n_gi? 0 : b->gi[i].data;
-}
-
-int bcf_anno_max(bcf1_t *b)
-{
- int k, max_gq, max_sp, n_het;
- kstring_t str;
- uint8_t *gt, *gq;
- int32_t *sp;
- max_gq = max_sp = n_het = 0;
- gt = locate_field(b, "GT", 2);
- if (gt == 0) return -1;
- gq = locate_field(b, "GQ", 2);
- sp = locate_field(b, "SP", 2);
- if (sp)
- for (k = 0; k < b->n_smpl; ++k)
- if (gt[k]&0x3f)
- max_sp = max_sp > (int)sp[k]? max_sp : sp[k];
- if (gq)
- for (k = 0; k < b->n_smpl; ++k)
- if (gt[k]&0x3f)
- max_gq = max_gq > (int)gq[k]? max_gq : gq[k];
- for (k = 0; k < b->n_smpl; ++k) {
- int a1, a2;
- a1 = gt[k]&7; a2 = gt[k]>>3&7;
- if ((!a1 && a2) || (!a2 && a1)) { // a het
- if (gq == 0) ++n_het;
- else if (gq[k] >= 20) ++n_het;
- }
- }
- if (n_het) max_sp -= (int)(4.343 * log(n_het) + .499);
- if (max_sp < 0) max_sp = 0;
- memset(&str, 0, sizeof(kstring_t));
- if (*b->info) kputc(';', &str);
- ksprintf(&str, "MXSP=%d;MXGQ=%d", max_sp, max_gq);
- bcf_append_info(b, str.s, str.l);
- free(str.s);
- return 0;
-}
-
-// FIXME: only data are shuffled; the header is NOT
-int bcf_shuffle(bcf1_t *b, int seed)
-{
- int i, j, *a;
- if (seed > 0) srand48(seed);
- a = malloc(b->n_smpl * sizeof(int));
- for (i = 0; i < b->n_smpl; ++i) a[i] = i;
- for (i = b->n_smpl; i > 1; --i) {
- int tmp;
- j = (int)(drand48() * i);
- tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;
- }
- for (j = 0; j < b->n_gi; ++j) {
- bcf_ginfo_t *gi = b->gi + j;
- uint8_t *swap, *data = (uint8_t*)gi->data;
- swap = malloc(gi->len * b->n_smpl);
- for (i = 0; i < b->n_smpl; ++i)
- memcpy(swap + gi->len * a[i], data + gi->len * i, gi->len);
- free(gi->data);
- gi->data = swap;
- }
- free(a);
- return 0;
-}
-
-bcf_hdr_t *bcf_hdr_subsam(const bcf_hdr_t *h0, int n, char *const* samples, int *list)
-{
- int i, ret, j;
- khint_t k;
- bcf_hdr_t *h;
- khash_t(str2id) *hash;
- kstring_t s;
- s.l = s.m = 0; s.s = 0;
- hash = kh_init(str2id);
- for (i = 0; i < h0->n_smpl; ++i) {
- k = kh_put(str2id, hash, h0->sns[i], &ret);
- kh_val(hash, k) = i;
- }
- for (i = j = 0; i < n; ++i) {
- k = kh_get(str2id, hash, samples[i]);
- if (k != kh_end(hash)) {
- list[j++] = kh_val(hash, k);
- kputs(samples[i], &s); kputc('\0', &s);
- }
- }
- if (j < n)
- {
- fprintf(stderr, "<%s> %d samples in the list but not in BCF.", __func__, n - j);
- exit(1);
- }
- kh_destroy(str2id, hash);
- h = calloc(1, sizeof(bcf_hdr_t));
- *h = *h0;
- h->ns = 0; h->sns = 0;
- h->name = malloc(h->l_nm); memcpy(h->name, h0->name, h->l_nm);
- h->txt = calloc(1, h->l_txt + 1); memcpy(h->txt, h0->txt, h->l_txt);
- h->l_smpl = s.l; h->sname = s.s;
- bcf_hdr_sync(h);
- return h;
-}
-
-int bcf_subsam(int n_smpl, int *list, bcf1_t *b)
-{
- int i, j;
- for (j = 0; j < b->n_gi; ++j) {
- bcf_ginfo_t *gi = b->gi + j;
- uint8_t *swap;
- swap = malloc(gi->len * b->n_smpl);
- for (i = 0; i < n_smpl; ++i)
- memcpy(swap + i * gi->len, (uint8_t*)gi->data + list[i] * gi->len, gi->len);
- free(gi->data);
- gi->data = swap;
- }
- b->n_smpl = n_smpl;
- return 0;
-}
-
-static int8_t nt4_table[128] = {
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 /*'-'*/, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, -1, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, -1, 4, 4, 4, 4, 4, 4, 4
-};
-
-int bcf_gl10(const bcf1_t *b, uint8_t *gl)
-{
- int a[4], k, l, map[4], k1, j, i;
- const bcf_ginfo_t *PL;
- char *s;
- if (b->ref[1] != 0 || b->n_alleles > 4) return -1; // ref is not a single base or >4 alleles
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) break;
- if (i == b->n_gi) return -1; // no PL
- PL = b->gi + i;
- a[0] = nt4_table[(int)b->ref[0]];
- if (a[0] > 3 || a[0] < 0) return -1; // ref is not A/C/G/T
- a[1] = a[2] = a[3] = -2; // -1 has a special meaning
- if (b->alt[0] == 0) return -1; // no alternate allele
- map[0] = map[1] = map[2] = map[3] = -2;
- map[a[0]] = 0;
- for (k = 0, s = b->alt, k1 = -1; k < 3 && *s; ++k, s += 2) {
- if (s[1] != ',' && s[1] != 0) return -1; // ALT is not single base
- a[k+1] = nt4_table[(int)*s];
- if (a[k+1] >= 0) map[a[k+1]] = k+1;
- else k1 = k + 1;
- if (s[1] == 0) break; // the end of the ALT string
- }
- for (k = 0; k < 4; ++k)
- if (map[k] < 0) map[k] = k1;
- for (i = 0; i < b->n_smpl; ++i) {
- const uint8_t *p = PL->data + i * PL->len; // the PL for the i-th individual
- uint8_t *g = gl + 10 * i;
- for (k = j = 0; k < 4; ++k) {
- for (l = k; l < 4; ++l) {
- int t, x = map[k], y = map[l];
- if (x > y) t = x, x = y, y = t; // make sure x is the smaller
- g[j++] = p[y * (y+1) / 2 + x];
- }
- }
- }
- return 0;
-}
-
-int bcf_gl10_indel(const bcf1_t *b, uint8_t *gl)
-{
- int k, l, j, i;
- const bcf_ginfo_t *PL;
- if (b->alt[0] == 0) return -1; // no alternate allele
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) break;
- if (i == b->n_gi) return -1; // no PL
- PL = b->gi + i;
- for (i = 0; i < b->n_smpl; ++i) {
- const uint8_t *p = PL->data + i * PL->len; // the PL for the i-th individual
- uint8_t *g = gl + 10 * i;
- for (k = j = 0; k < 4; ++k) {
- for (l = k; l < 4; ++l) {
- int t, x = k, y = l;
- if (x > y) t = x, x = y, y = t; // make sure x is the smaller
- x = y * (y+1) / 2 + x;
- g[j++] = x < PL->len? p[x] : 255;
- }
- }
- }
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/call1.c b/samtools-0.1.19/bcftools/call1.c
deleted file mode 100644
index e6373d3..0000000
--- a/samtools-0.1.19/bcftools/call1.c
+++ /dev/null
@@ -1,633 +0,0 @@
-#include <unistd.h>
-#include <stdlib.h>
-#include <math.h>
-#include <zlib.h>
-#include <errno.h>
-#include "bcf.h"
-#include "prob1.h"
-#include "kstring.h"
-#include "time.h"
-
-#ifdef _WIN32
-#define srand48(x) srand(x)
-#define lrand48() rand()
-#endif
-
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-#define VC_NO_GENO 2
-#define VC_BCFOUT 4
-#define VC_CALL 8
-#define VC_VARONLY 16
-#define VC_VCFIN 32
-#define VC_UNCOMP 64
-#define VC_KEEPALT 256
-#define VC_ACGT_ONLY 512
-#define VC_QCALL 1024
-#define VC_CALL_GT 2048
-#define VC_ADJLD 4096
-#define VC_NO_INDEL 8192
-#define VC_ANNO_MAX 16384
-#define VC_FIX_PL 32768
-#define VC_EM 0x10000
-#define VC_PAIRCALL 0x20000
-#define VC_QCNT 0x40000
-#define VC_INDEL_ONLY 0x80000
-
-typedef struct {
- int flag, prior_type, n1, n_sub, *sublist, n_perm;
- uint32_t *trio_aux;
- char *prior_file, **subsam, *fn_dict;
- uint8_t *ploidy;
- double theta, pref, indel_frac, min_perm_p, min_smpl_frac, min_lrt, min_ma_lrt;
- void *bed;
-} viewconf_t;
-
-void *bed_read(const char *fn);
-void bed_destroy(void *_h);
-int bed_overlap(const void *_h, const char *chr, int beg, int end);
-
-static double ttest(int n1, int n2, int a[4])
-{
- extern double kf_betai(double a, double b, double x);
- double t, v, u1, u2;
- if (n1 == 0 || n2 == 0 || n1 + n2 < 3) return 1.0;
- u1 = (double)a[0] / n1; u2 = (double)a[2] / n2;
- if (u1 <= u2) return 1.;
- t = (u1 - u2) / sqrt(((a[1] - n1 * u1 * u1) + (a[3] - n2 * u2 * u2)) / (n1 + n2 - 2) * (1./n1 + 1./n2));
- v = n1 + n2 - 2;
-// printf("%d,%d,%d,%d,%lf,%lf,%lf\n", a[0], a[1], a[2], a[3], t, u1, u2);
- return t < 0.? 1. : .5 * kf_betai(.5*v, .5, v/(v+t*t));
-}
-
-static int test16_core(int anno[16], anno16_t *a)
-{
- extern double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two);
- double left, right;
- int i;
- a->p[0] = a->p[1] = a->p[2] = a->p[3] = 1.;
- memcpy(a->d, anno, 4 * sizeof(int));
- a->depth = anno[0] + anno[1] + anno[2] + anno[3];
- a->is_tested = (anno[0] + anno[1] > 0 && anno[2] + anno[3] > 0);
- if (a->depth == 0) return -1;
- a->mq = (int)(sqrt((anno[9] + anno[11]) / a->depth) + .499);
- kt_fisher_exact(anno[0], anno[1], anno[2], anno[3], &left, &right, &a->p[0]);
- for (i = 1; i < 4; ++i)
- a->p[i] = ttest(anno[0] + anno[1], anno[2] + anno[3], anno+4*i);
- return 0;
-}
-
-int test16(bcf1_t *b, anno16_t *a)
-{
- char *p;
- int i, anno[16];
- a->p[0] = a->p[1] = a->p[2] = a->p[3] = 1.;
- a->d[0] = a->d[1] = a->d[2] = a->d[3] = 0.;
- a->mq = a->depth = a->is_tested = 0;
- if ((p = strstr(b->info, "I16=")) == 0) return -1;
- p += 4;
- for (i = 0; i < 16; ++i) {
- errno = 0; anno[i] = strtol(p, &p, 10);
- if (anno[i] == 0 && (errno == EINVAL || errno == ERANGE)) return -2;
- ++p;
- }
- return test16_core(anno, a);
-}
-
-static int update_bcf1(bcf1_t *b, const bcf_p1aux_t *pa, const bcf_p1rst_t *pr, double pref, int flag, double em[10], int cons_llr, int64_t cons_gt)
-{
- kstring_t s;
- int has_I16, is_var;
- double fq, r;
- anno16_t a;
-
- has_I16 = test16(b, &a) >= 0? 1 : 0;
- //rm_info(b, "I16="); // FIXME: probably this function has a bug. If I move it below, I16 will not be removed!
-
- memset(&s, 0, sizeof(kstring_t));
- kputc('\0', &s); kputs(b->ref, &s); kputc('\0', &s);
- kputs(b->alt, &s); kputc('\0', &s); kputc('\0', &s);
- kputs(b->info, &s);
- if (b->info[0]) kputc(';', &s);
- { // print EM
- if (em[0] >= 0) ksprintf(&s, "AF1=%.4g", 1 - em[0]);
- if (em[4] >= 0 && em[4] <= 0.05) ksprintf(&s, ";G3=%.4g,%.4g,%.4g;HWE=%.3g", em[3], em[2], em[1], em[4]);
- if (em[5] >= 0 && em[6] >= 0) ksprintf(&s, ";AF2=%.4g,%.4g", 1 - em[5], 1 - em[6]);
- if (em[7] >= 0) ksprintf(&s, ";LRT=%.3g", em[7]);
- if (em[8] >= 0) ksprintf(&s, ";LRT2=%.3g", em[8]);
- }
- if (cons_llr > 0) {
- ksprintf(&s, ";CLR=%d", cons_llr);
- if (cons_gt > 0)
- ksprintf(&s, ";UGT=%c%c%c;CGT=%c%c%c", cons_gt&0xff, cons_gt>>8&0xff, cons_gt>>16&0xff,
- cons_gt>>32&0xff, cons_gt>>40&0xff, cons_gt>>48&0xff);
- }
- if (pr == 0) { // if pr is unset, return
- kputc('\0', &s); kputs(b->fmt, &s); kputc('\0', &s);
- free(b->str);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- bcf_sync(b);
- return 1;
- }
-
- is_var = (pr->p_ref < pref);
- r = is_var? pr->p_ref : pr->p_var;
-
-// ksprintf(&s, ";CI95=%.4g,%.4g", pr->cil, pr->cih); // FIXME: when EM is not used, ";" should be omitted!
- ksprintf(&s, ";AC1=%d", pr->ac);
- if (has_I16) ksprintf(&s, ";DP4=%d,%d,%d,%d;MQ=%d", a.d[0], a.d[1], a.d[2], a.d[3], a.mq);
- fq = pr->p_ref_folded < 0.5? -4.343 * log(pr->p_ref_folded) : 4.343 * log(pr->p_var_folded);
- if (fq < -999) fq = -999;
- if (fq > 999) fq = 999;
- ksprintf(&s, ";FQ=%.3g", fq);
- if (pr->cmp[0] >= 0.) { // two sample groups
- int i, q[3];
- for (i = 1; i < 3; ++i) {
- double x = pr->cmp[i] + pr->cmp[0]/2.;
- q[i] = x == 0? 255 : (int)(-4.343 * log(x) + .499);
- if (q[i] > 255) q[i] = 255;
- }
- if (pr->perm_rank >= 0) ksprintf(&s, ";PR=%d", pr->perm_rank);
- // ksprintf(&s, ";LRT3=%.3g", pr->lrt);
- ksprintf(&s, ";PCHI2=%.3g;PC2=%d,%d", q[1], q[2], pr->p_chi2);
- }
- if (has_I16 && a.is_tested) ksprintf(&s, ";PV4=%.2g,%.2g,%.2g,%.2g", a.p[0], a.p[1], a.p[2], a.p[3]);
- kputc('\0', &s);
- rm_info(&s, "QS=");
- rm_info(&s, "I16=");
- kputs(b->fmt, &s); kputc('\0', &s);
- free(b->str);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- b->qual = r < 1e-100? 999 : -4.343 * log(r);
- if (b->qual > 999) b->qual = 999;
- bcf_sync(b);
- if (!is_var) bcf_shrink_alt(b, 1);
- else if (!(flag&VC_KEEPALT))
- bcf_shrink_alt(b, pr->rank0 < 2? 2 : pr->rank0+1);
- if (is_var && (flag&VC_CALL_GT)) { // call individual genotype
- int i, x, old_n_gi = b->n_gi;
- s.m = b->m_str; s.l = b->l_str - 1; s.s = b->str;
- kputs(":GT:GQ", &s); kputc('\0', &s);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- bcf_sync(b);
- for (i = 0; i < b->n_smpl; ++i) {
- x = bcf_p1_call_gt(pa, pr->f_exp, i);
- ((uint8_t*)b->gi[old_n_gi].data)[i] = (x&3) == 0? 1<<3|1 : (x&3) == 1? 1 : 0;
- ((uint8_t*)b->gi[old_n_gi+1].data)[i] = x>>2;
- }
- }
- return is_var;
-}
-
-static char **read_samples(const char *fn, int *_n)
-{
- gzFile fp;
- kstream_t *ks;
- kstring_t s;
- int dret, n = 0, max = 0;
- char **sam = 0;
- *_n = 0;
- s.l = s.m = 0; s.s = 0;
- fp = gzopen(fn, "r");
- if (fp == 0)
- {
- // interpret as sample names, not as a file name
- const char *t = fn, *p = t;
- while (*t)
- {
- t++;
- if ( *t==',' || !*t )
- {
- sam = realloc(sam, sizeof(void*)*(n+1));
- sam[n] = (char*) malloc(sizeof(char)*(t-p+2));
- memcpy(sam[n], p, t-p);
- sam[n][t-p] = 0;
- sam[n][t-p+1] = 2; // assume diploid
- p = t+1;
- n++;
- }
- }
- *_n = n;
- return sam; // fail to open file
- }
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, &s, &dret) >= 0) {
- int l;
- if (max == n) {
- max = max? max<<1 : 4;
- sam = realloc(sam, sizeof(void*)*max);
- }
- l = s.l;
- sam[n] = malloc(s.l + 2);
- strcpy(sam[n], s.s);
- sam[n][l+1] = 2; // by default, diploid
- if (dret != '\n') {
- if (ks_getuntil(ks, 0, &s, &dret) >= 0) { // read ploidy, 1 or 2
- int x = (int)s.s[0] - '0';
- if (x == 1 || x == 2) sam[n][l+1] = x;
- else fprintf(stderr, "(%s) ploidy can only be 1 or 2; assume diploid\n", __func__);
- }
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- }
- ++n;
- }
- ks_destroy(ks);
- gzclose(fp);
- free(s.s);
- *_n = n;
- return sam;
-}
-
-static void write_header(bcf_hdr_t *h)
-{
- kstring_t str;
- str.l = h->l_txt? h->l_txt - 1 : 0;
- str.m = str.l + 1; str.s = h->txt;
- if (!strstr(str.s, "##INFO=<ID=DP,"))
- kputs("##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Raw read depth\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=DP4,"))
- kputs("##INFO=<ID=DP4,Number=4,Type=Integer,Description=\"# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=MQ,"))
- kputs("##INFO=<ID=MQ,Number=1,Type=Integer,Description=\"Root-mean-square mapping quality of covering reads\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=FQ,"))
- kputs("##INFO=<ID=FQ,Number=1,Type=Float,Description=\"Phred probability of all samples being the same\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=AF1,"))
- kputs("##INFO=<ID=AF1,Number=1,Type=Float,Description=\"Max-likelihood estimate of the first ALT allele frequency (assuming HWE)\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=AC1,"))
- kputs("##INFO=<ID=AC1,Number=1,Type=Float,Description=\"Max-likelihood estimate of the first ALT allele count (no HWE assumption)\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=AN,"))
- kputs("##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=IS,"))
- kputs("##INFO=<ID=IS,Number=2,Type=Float,Description=\"Maximum number of reads supporting an indel and fraction of indel reads\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=AC,"))
- kputs("##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes for each ALT allele, in the same order as listed\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=G3,"))
- kputs("##INFO=<ID=G3,Number=3,Type=Float,Description=\"ML estimate of genotype frequencies\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=HWE,"))
- kputs("##INFO=<ID=HWE,Number=1,Type=Float,Description=\"Chi^2 based HWE test P-value based on G3\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=CLR,"))
- kputs("##INFO=<ID=CLR,Number=1,Type=Integer,Description=\"Log ratio of genotype likelihoods with and without the constraint\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=UGT,"))
- kputs("##INFO=<ID=UGT,Number=1,Type=String,Description=\"The most probable unconstrained genotype configuration in the trio\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=CGT,"))
- kputs("##INFO=<ID=CGT,Number=1,Type=String,Description=\"The most probable constrained genotype configuration in the trio\">\n", &str);
-// if (!strstr(str.s, "##INFO=<ID=CI95,"))
-// kputs("##INFO=<ID=CI95,Number=2,Type=Float,Description=\"Equal-tail Bayesian credible interval of the site allele frequency at the 95% level\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=PV4,"))
- kputs("##INFO=<ID=PV4,Number=4,Type=Float,Description=\"P-values for strand bias, baseQ bias, mapQ bias and tail distance bias\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=INDEL,"))
- kputs("##INFO=<ID=INDEL,Number=0,Type=Flag,Description=\"Indicates that the variant is an INDEL.\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=PC2,"))
- kputs("##INFO=<ID=PC2,Number=2,Type=Integer,Description=\"Phred probability of the nonRef allele frequency in group1 samples being larger (,smaller) than in group2.\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=PCHI2,"))
- kputs("##INFO=<ID=PCHI2,Number=1,Type=Float,Description=\"Posterior weighted chi^2 P-value for testing the association between group1 and group2 samples.\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=QCHI2,"))
- kputs("##INFO=<ID=QCHI2,Number=1,Type=Integer,Description=\"Phred scaled PCHI2.\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=RP,"))
- kputs("##INFO=<ID=PR,Number=1,Type=Integer,Description=\"# permutations yielding a smaller PCHI2.\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=QBD,"))
- kputs("##INFO=<ID=QBD,Number=1,Type=Float,Description=\"Quality by Depth: QUAL/#reads\">\n", &str);
- //if (!strstr(str.s, "##INFO=<ID=RPS,"))
- // kputs("##INFO=<ID=RPS,Number=3,Type=Float,Description=\"Read Position Stats: depth, average, stddev\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=RPB,"))
- kputs("##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Read Position Bias\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=MDV,"))
- kputs("##INFO=<ID=MDV,Number=1,Type=Integer,Description=\"Maximum number of high-quality nonRef reads in samples\">\n", &str);
- if (!strstr(str.s, "##INFO=<ID=VDB,"))
- kputs("##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias (v2) for filtering splice-site artefacts in RNA-seq data. Note: this version may be broken.\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=GT,"))
- kputs("##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=GQ,"))
- kputs("##FORMAT=<ID=GQ,Number=1,Type=Integer,Description=\"Genotype Quality\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=GL,"))
- kputs("##FORMAT=<ID=GL,Number=3,Type=Float,Description=\"Likelihoods for RR,RA,AA genotypes (R=ref,A=alt)\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=DP,"))
- kputs("##FORMAT=<ID=DP,Number=1,Type=Integer,Description=\"# high-quality bases\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=DV,"))
- kputs("##FORMAT=<ID=DV,Number=1,Type=Integer,Description=\"# high-quality non-reference bases\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=SP,"))
- kputs("##FORMAT=<ID=SP,Number=1,Type=Integer,Description=\"Phred-scaled strand bias P-value\">\n", &str);
- if (!strstr(str.s, "##FORMAT=<ID=PL,"))
- kputs("##FORMAT=<ID=PL,Number=G,Type=Integer,Description=\"List of Phred-scaled genotype likelihoods\">\n", &str);
- h->l_txt = str.l + 1; h->txt = str.s;
-}
-
-double bcf_pair_freq(const bcf1_t *b0, const bcf1_t *b1, double f[4]);
-
-int bcfview(int argc, char *argv[])
-{
- extern int bcf_2qcall(bcf_hdr_t *h, bcf1_t *b);
- extern void bcf_p1_indel_prior(bcf_p1aux_t *ma, double x);
- extern int bcf_fix_gt(bcf1_t *b);
- extern int bcf_anno_max(bcf1_t *b);
- extern int bcf_shuffle(bcf1_t *b, int seed);
- extern uint32_t *bcf_trio_prep(int is_x, int is_son);
- extern int bcf_trio_call(uint32_t *prep, const bcf1_t *b, int *llr, int64_t *gt);
- extern int bcf_pair_call(const bcf1_t *b);
- extern int bcf_min_diff(const bcf1_t *b);
- extern int bcf_p1_get_M(bcf_p1aux_t *b);
-
- extern gzFile bcf_p1_fp_lk;
-
- bcf_t *bp, *bout = 0;
- bcf1_t *b, *blast;
- int c, *seeds = 0;
- uint64_t n_processed = 0, qcnt[256];
- viewconf_t vc;
- bcf_p1aux_t *p1 = 0;
- bcf_hdr_t *hin, *hout;
- int tid, begin, end;
- char moder[4], modew[4];
-
- tid = begin = end = -1;
- memset(&vc, 0, sizeof(viewconf_t));
- vc.prior_type = vc.n1 = -1; vc.theta = 1e-3; vc.pref = 0.5; vc.indel_frac = -1.; vc.n_perm = 0; vc.min_perm_p = 0.01; vc.min_smpl_frac = 0; vc.min_lrt = 1; vc.min_ma_lrt = -1;
- memset(qcnt, 0, 8 * 256);
- while ((c = getopt(argc, argv, "FN1:l:cC:eHAGvbSuP:t:p:QgLi:IMs:D:U:X:d:T:Ywm:K:")) >= 0) {
- switch (c) {
- case '1': vc.n1 = atoi(optarg); break;
- case 'l': vc.bed = bed_read(optarg); if (!vc.bed) { fprintf(stderr,"Could not read \"%s\"\n", optarg); return 1; } break;
- case 'D': vc.fn_dict = strdup(optarg); break;
- case 'F': vc.flag |= VC_FIX_PL; break;
- case 'N': vc.flag |= VC_ACGT_ONLY; break;
- case 'G': vc.flag |= VC_NO_GENO; break;
- case 'A': vc.flag |= VC_KEEPALT; break;
- case 'b': vc.flag |= VC_BCFOUT; break;
- case 'S': vc.flag |= VC_VCFIN; break;
- case 'c': vc.flag |= VC_CALL; break;
- case 'e': vc.flag |= VC_EM; break;
- case 'v': vc.flag |= VC_VARONLY | VC_CALL; break;
- case 'u': vc.flag |= VC_UNCOMP | VC_BCFOUT; break;
- case 'g': vc.flag |= VC_CALL_GT | VC_CALL; break;
- case 'I': vc.flag |= VC_NO_INDEL; break;
- case 'w': vc.flag |= VC_INDEL_ONLY; break;
- case 'M': vc.flag |= VC_ANNO_MAX; break;
- case 'Y': vc.flag |= VC_QCNT; break;
- case 'm': vc.min_ma_lrt = atof(optarg); break;
- case 't': vc.theta = atof(optarg); break;
- case 'p': vc.pref = atof(optarg); break;
- case 'i': vc.indel_frac = atof(optarg); break;
- case 'Q': vc.flag |= VC_QCALL; break;
- case 'L': vc.flag |= VC_ADJLD; break;
- case 'U': vc.n_perm = atoi(optarg); break;
- case 'C': vc.min_lrt = atof(optarg); break;
- case 'X': vc.min_perm_p = atof(optarg); break;
- case 'd': vc.min_smpl_frac = atof(optarg); break;
- case 'K': bcf_p1_fp_lk = gzopen(optarg, "w"); break;
- case 's': vc.subsam = read_samples(optarg, &vc.n_sub);
- vc.ploidy = calloc(vc.n_sub + 1, 1);
- for (tid = 0; tid < vc.n_sub; ++tid) vc.ploidy[tid] = vc.subsam[tid][strlen(vc.subsam[tid]) + 1];
- tid = -1;
- break;
- case 'T':
- if (strcmp(optarg, "trioauto") == 0) vc.trio_aux = bcf_trio_prep(0, 0);
- else if (strcmp(optarg, "trioxd") == 0) vc.trio_aux = bcf_trio_prep(1, 0);
- else if (strcmp(optarg, "trioxs") == 0) vc.trio_aux = bcf_trio_prep(1, 1);
- else if (strcmp(optarg, "pair") == 0) vc.flag |= VC_PAIRCALL;
- else {
- fprintf(stderr, "[%s] Option '-T' can only take value trioauto, trioxd or trioxs.\n", __func__);
- return 1;
- }
- break;
- case 'P':
- if (strcmp(optarg, "full") == 0) vc.prior_type = MC_PTYPE_FULL;
- else if (strcmp(optarg, "cond2") == 0) vc.prior_type = MC_PTYPE_COND2;
- else if (strcmp(optarg, "flat") == 0) vc.prior_type = MC_PTYPE_FLAT;
- else vc.prior_file = strdup(optarg);
- break;
- }
- }
- if (argc == optind) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: bcftools view [options] <in.bcf> [reg]\n\n");
- fprintf(stderr, "Input/output options:\n\n");
- fprintf(stderr, " -A keep all possible alternate alleles at variant sites\n");
- fprintf(stderr, " -b output BCF instead of VCF\n");
- fprintf(stderr, " -D FILE sequence dictionary for VCF->BCF conversion [null]\n");
- fprintf(stderr, " -F PL generated by r921 or before (which generate old ordering)\n");
- fprintf(stderr, " -G suppress all individual genotype information\n");
- fprintf(stderr, " -l FILE list of sites (chr pos) or regions (BED) to output [all sites]\n");
- fprintf(stderr, " -L calculate LD for adjacent sites\n");
- fprintf(stderr, " -N skip sites where REF is not A/C/G/T\n");
- fprintf(stderr, " -Q output the QCALL likelihood format\n");
- fprintf(stderr, " -s FILE list of samples to use [all samples]\n");
- fprintf(stderr, " -S input is VCF\n");
- fprintf(stderr, " -u uncompressed BCF output (force -b)\n");
- fprintf(stderr, "\nConsensus/variant calling options:\n\n");
- fprintf(stderr, " -c SNP calling (force -e)\n");
- fprintf(stderr, " -d FLOAT skip loci where less than FLOAT fraction of samples covered [0]\n");
- fprintf(stderr, " -e likelihood based analyses\n");
- fprintf(stderr, " -g call genotypes at variant sites (force -c)\n");
- fprintf(stderr, " -i FLOAT indel-to-substitution ratio [%.4g]\n", vc.indel_frac);
- fprintf(stderr, " -I skip indels\n");
- fprintf(stderr, " -m FLOAT alternative model for multiallelic and rare-variant calling, include if P(chi^2)>=FLOAT\n");
- fprintf(stderr, " -p FLOAT variant if P(ref|D)<FLOAT [%.3g]\n", vc.pref);
- fprintf(stderr, " -P STR type of prior: full, cond2, flat [full]\n");
- fprintf(stderr, " -t FLOAT scaled substitution mutation rate [%.4g]\n", vc.theta);
- fprintf(stderr, " -T STR constrained calling; STR can be: pair, trioauto, trioxd and trioxs (see manual) [null]\n");
- fprintf(stderr, " -v output potential variant sites only (force -c)\n");
- fprintf(stderr, "\nContrast calling and association test options:\n\n");
- fprintf(stderr, " -1 INT number of group-1 samples [0]\n");
- fprintf(stderr, " -C FLOAT posterior constrast for LRT<FLOAT and P(ref|D)<0.5 [%g]\n", vc.min_lrt);
- fprintf(stderr, " -U INT number of permutations for association testing (effective with -1) [0]\n");
- fprintf(stderr, " -X FLOAT only perform permutations for P(chi^2)<FLOAT [%g]\n", vc.min_perm_p);
- fprintf(stderr, "\n");
- return 1;
- }
-
- if (vc.flag & VC_CALL) vc.flag |= VC_EM;
- if ((vc.flag & VC_VCFIN) && (vc.flag & VC_BCFOUT) && vc.fn_dict == 0) {
- fprintf(stderr, "[%s] For VCF->BCF conversion please specify the sequence dictionary with -D\n", __func__);
- return 1;
- }
- if (vc.n1 <= 0) vc.n_perm = 0; // TODO: give a warning here!
- if (vc.n_perm > 0) {
- seeds = malloc(vc.n_perm * sizeof(int));
- srand48(time(0));
- for (c = 0; c < vc.n_perm; ++c) seeds[c] = lrand48();
- }
- b = calloc(1, sizeof(bcf1_t));
- blast = calloc(1, sizeof(bcf1_t));
- strcpy(moder, "r");
- if (!(vc.flag & VC_VCFIN)) strcat(moder, "b");
- strcpy(modew, "w");
- if (vc.flag & VC_BCFOUT) strcat(modew, "b");
- if (vc.flag & VC_UNCOMP) strcat(modew, "u");
- bp = vcf_open(argv[optind], moder);
- hin = hout = vcf_hdr_read(bp);
- if (vc.fn_dict && (vc.flag & VC_VCFIN))
- vcf_dictread(bp, hin, vc.fn_dict);
- bout = vcf_open("-", modew);
- if (!(vc.flag & VC_QCALL)) {
- if (vc.n_sub) {
- vc.sublist = calloc(vc.n_sub, sizeof(int));
- hout = bcf_hdr_subsam(hin, vc.n_sub, vc.subsam, vc.sublist);
- }
- write_header(hout); // always print the header
- vcf_hdr_write(bout, hout);
- }
- if (vc.flag & VC_CALL) {
- p1 = bcf_p1_init(hout->n_smpl, vc.ploidy);
- if (vc.prior_file) {
- if (bcf_p1_read_prior(p1, vc.prior_file) < 0) {
- fprintf(stderr, "[%s] fail to read the prior AFS.\n", __func__);
- return 1;
- }
- } else bcf_p1_init_prior(p1, vc.prior_type, vc.theta);
- if (vc.n1 > 0 && vc.min_lrt > 0.) { // set n1
- bcf_p1_set_n1(p1, vc.n1);
- bcf_p1_init_subprior(p1, vc.prior_type, vc.theta);
- }
- if (vc.indel_frac > 0.) bcf_p1_indel_prior(p1, vc.indel_frac); // otherwise use the default indel_frac
- }
- if (optind + 1 < argc && !(vc.flag&VC_VCFIN)) {
- void *str2id = bcf_build_refhash(hout);
- if (bcf_parse_region(str2id, argv[optind+1], &tid, &begin, &end) >= 0) {
- bcf_idx_t *idx;
- idx = bcf_idx_load(argv[optind]);
- if (idx) {
- uint64_t off;
- off = bcf_idx_query(idx, tid, begin);
- if (off == 0) {
- fprintf(stderr, "[%s] no records in the query region.\n", __func__);
- return 1; // FIXME: a lot of memory leaks...
- }
- bgzf_seek(bp->fp, off, SEEK_SET);
- bcf_idx_destroy(idx);
- }
- }
- }
- if (bcf_p1_fp_lk && p1) {
- int32_t M = bcf_p1_get_M(p1);
- gzwrite(bcf_p1_fp_lk, &M, 4);
- }
- while (vcf_read(bp, hin, b) > 0) {
- int is_indel, cons_llr = -1;
- int64_t cons_gt = -1;
- double em[10];
- if ((vc.flag & VC_VARONLY) && strcmp(b->alt, "X") == 0) continue;
- if ((vc.flag & VC_VARONLY) && vc.min_smpl_frac > 0.) {
- extern int bcf_smpl_covered(const bcf1_t *b);
- int n = bcf_smpl_covered(b);
- if ((double)n / b->n_smpl < vc.min_smpl_frac) continue;
- }
- if (vc.n_sub) bcf_subsam(vc.n_sub, vc.sublist, b);
- if (vc.flag & VC_FIX_PL) bcf_fix_pl(b);
- is_indel = bcf_is_indel(b);
- if ((vc.flag & VC_NO_INDEL) && is_indel) continue;
- if ((vc.flag & VC_INDEL_ONLY) && !is_indel) continue;
- if ((vc.flag & VC_ACGT_ONLY) && !is_indel) {
- int x;
- if (b->ref[0] == 0 || b->ref[1] != 0) continue;
- x = toupper(b->ref[0]);
- if (x != 'A' && x != 'C' && x != 'G' && x != 'T') continue;
- }
- if (vc.bed && !bed_overlap(vc.bed, hin->ns[b->tid], b->pos, b->pos + strlen(b->ref))) continue;
- if (tid >= 0) {
- int l = strlen(b->ref);
- l = b->pos + (l > 0? l : 1);
- if (b->tid != tid || b->pos >= end) break;
- if (!(l > begin && end > b->pos)) continue;
- }
- ++n_processed;
- if ((vc.flag & VC_QCNT) && !is_indel) { // summarize the difference
- int x = bcf_min_diff(b);
- if (x > 255) x = 255;
- if (x >= 0) ++qcnt[x];
- }
- if (vc.flag & VC_QCALL) { // output QCALL format; STOP here
- bcf_2qcall(hout, b);
- continue;
- }
- if (vc.trio_aux) // do trio calling
- bcf_trio_call(vc.trio_aux, b, &cons_llr, &cons_gt);
- else if (vc.flag & VC_PAIRCALL)
- cons_llr = bcf_pair_call(b);
- if (vc.flag & (VC_CALL|VC_ADJLD|VC_EM)) bcf_gl2pl(b);
- if (vc.flag & VC_EM) bcf_em1(b, vc.n1, 0x1ff, em);
- else {
- int i;
- for (i = 0; i < 9; ++i) em[i] = -1.;
- }
- if ( !(vc.flag&VC_KEEPALT) && (vc.flag&VC_CALL) && vc.min_ma_lrt>=0 )
- {
- bcf_p1_set_ploidy(b, p1); // could be improved: do this per site to allow pseudo-autosomal regions
- int gts = call_multiallelic_gt(b, p1, vc.min_ma_lrt, vc.flag&VC_VARONLY);
- if ( gts<=1 && vc.flag & VC_VARONLY ) continue;
- }
- else if (vc.flag & VC_CALL) { // call variants
- bcf_p1rst_t pr;
- int calret;
- gzwrite(bcf_p1_fp_lk, &b->tid, 4);
- gzwrite(bcf_p1_fp_lk, &b->pos, 4);
- gzwrite(bcf_p1_fp_lk, &em[0], sizeof(double));
- calret = bcf_p1_cal(b, (em[7] >= 0 && em[7] < vc.min_lrt), p1, &pr);
- if (n_processed % 100000 == 0) {
- fprintf(stderr, "[%s] %ld sites processed.\n", __func__, (long)n_processed);
- bcf_p1_dump_afs(p1);
- }
- if (pr.p_ref >= vc.pref && (vc.flag & VC_VARONLY)) continue;
- if (vc.n_perm && vc.n1 > 0 && pr.p_chi2 < vc.min_perm_p) { // permutation test
- bcf_p1rst_t r;
- int i, n = 0;
- for (i = 0; i < vc.n_perm; ++i) {
-#ifdef BCF_PERM_LRT // LRT based permutation is much faster but less robust to artifacts
- double x[10];
- bcf_shuffle(b, seeds[i]);
- bcf_em1(b, vc.n1, 1<<7, x);
- if (x[7] < em[7]) ++n;
-#else
- bcf_shuffle(b, seeds[i]);
- bcf_p1_cal(b, 1, p1, &r);
- if (pr.p_chi2 >= r.p_chi2) ++n;
-#endif
- }
- pr.perm_rank = n;
- }
- if (calret >= 0) update_bcf1(b, p1, &pr, vc.pref, vc.flag, em, cons_llr, cons_gt);
- } else if (vc.flag & VC_EM) update_bcf1(b, 0, 0, 0, vc.flag, em, cons_llr, cons_gt);
- if (vc.flag & VC_ADJLD) { // compute LD
- double f[4], r2;
- if ((r2 = bcf_pair_freq(blast, b, f)) >= 0) {
- kstring_t s;
- s.m = s.l = 0; s.s = 0;
- if (*b->info) kputc(';', &s);
- ksprintf(&s, "NEIR=%.3f;NEIF4=%.3f,%.3f,%.3f,%.3f", r2, f[0], f[1], f[2], f[3]);
- bcf_append_info(b, s.s, s.l);
- free(s.s);
- }
- bcf_cpy(blast, b);
- }
- if (vc.flag & VC_ANNO_MAX) bcf_anno_max(b);
- if (vc.flag & VC_NO_GENO) { // do not output GENO fields
- b->n_gi = 0;
- b->fmt[0] = '\0';
- b->l_str = b->fmt - b->str + 1;
- } else bcf_fix_gt(b);
- vcf_write(bout, hout, b);
- }
-
- if (bcf_p1_fp_lk) gzclose(bcf_p1_fp_lk);
- if (vc.prior_file) free(vc.prior_file);
- if (vc.flag & VC_CALL) bcf_p1_dump_afs(p1);
- if (hin != hout) bcf_hdr_destroy(hout);
- bcf_hdr_destroy(hin);
- bcf_destroy(b); bcf_destroy(blast);
- vcf_close(bp); vcf_close(bout);
- if (vc.fn_dict) free(vc.fn_dict);
- if (vc.ploidy) free(vc.ploidy);
- if (vc.trio_aux) free(vc.trio_aux);
- if (vc.n_sub) {
- int i;
- for (i = 0; i < vc.n_sub; ++i) free(vc.subsam[i]);
- free(vc.subsam); free(vc.sublist);
- }
- if (vc.bed) bed_destroy(vc.bed);
- if (vc.flag & VC_QCNT)
- for (c = 0; c < 256; ++c)
- fprintf(stderr, "QT\t%d\t%lld\n", c, (long long)qcnt[c]);
- if (seeds) free(seeds);
- if (p1) bcf_p1_destroy(p1);
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/em.c b/samtools-0.1.19/bcftools/em.c
deleted file mode 100644
index b7dfe1a..0000000
--- a/samtools-0.1.19/bcftools/em.c
+++ /dev/null
@@ -1,310 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include "bcf.h"
-#include "kmin.h"
-
-static double g_q2p[256];
-
-#define ITER_MAX 50
-#define ITER_TRY 10
-#define EPS 1e-5
-
-extern double kf_gammaq(double, double);
-
-/*
- Generic routines
- */
-// get the 3 genotype likelihoods
-static double *get_pdg3(const bcf1_t *b)
-{
- double *pdg;
- const uint8_t *PL = 0;
- int i, PL_len = 0;
- // initialize g_q2p if necessary
- if (g_q2p[0] == 0.)
- for (i = 0; i < 256; ++i)
- g_q2p[i] = pow(10., -i / 10.);
- // set PL and PL_len
- for (i = 0; i < b->n_gi; ++i) {
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
- PL = (const uint8_t*)b->gi[i].data;
- PL_len = b->gi[i].len;
- break;
- }
- }
- if (i == b->n_gi) return 0; // no PL
- // fill pdg
- pdg = malloc(3 * b->n_smpl * sizeof(double));
- for (i = 0; i < b->n_smpl; ++i) {
- const uint8_t *pi = PL + i * PL_len;
- double *p = pdg + i * 3;
- p[0] = g_q2p[pi[2]]; p[1] = g_q2p[pi[1]]; p[2] = g_q2p[pi[0]];
- }
- return pdg;
-}
-
-// estimate site allele frequency in a very naive and inaccurate way
-static double est_freq(int n, const double *pdg)
-{
- int i, gcnt[3], tmp1;
- // get a rough estimate of the genotype frequency
- gcnt[0] = gcnt[1] = gcnt[2] = 0;
- for (i = 0; i < n; ++i) {
- const double *p = pdg + i * 3;
- if (p[0] != 1. || p[1] != 1. || p[2] != 1.) {
- int which = p[0] > p[1]? 0 : 1;
- which = p[which] > p[2]? which : 2;
- ++gcnt[which];
- }
- }
- tmp1 = gcnt[0] + gcnt[1] + gcnt[2];
- return (tmp1 == 0)? -1.0 : (.5 * gcnt[1] + gcnt[2]) / tmp1;
-}
-
-/*
- Single-locus EM
- */
-
-typedef struct {
- int beg, end;
- const double *pdg;
-} minaux1_t;
-
-static double prob1(double f, void *data)
-{
- minaux1_t *a = (minaux1_t*)data;
- double p = 1., l = 0., f3[3];
- int i;
-// printf("brent %lg\n", f);
- if (f < 0 || f > 1) return 1e300;
- f3[0] = (1.-f)*(1.-f); f3[1] = 2.*f*(1.-f); f3[2] = f*f;
- for (i = a->beg; i < a->end; ++i) {
- const double *pdg = a->pdg + i * 3;
- p *= pdg[0] * f3[0] + pdg[1] * f3[1] + pdg[2] * f3[2];
- if (p < 1e-200) l -= log(p), p = 1.;
- }
- return l - log(p);
-}
-
-// one EM iteration for allele frequency estimate
-static double freq_iter(double *f, const double *_pdg, int beg, int end)
-{
- double f0 = *f, f3[3], err;
- int i;
-// printf("em %lg\n", *f);
- f3[0] = (1.-f0)*(1.-f0); f3[1] = 2.*f0*(1.-f0); f3[2] = f0*f0;
- for (i = beg, f0 = 0.; i < end; ++i) {
- const double *pdg = _pdg + i * 3;
- f0 += (pdg[1] * f3[1] + 2. * pdg[2] * f3[2])
- / (pdg[0] * f3[0] + pdg[1] * f3[1] + pdg[2] * f3[2]);
- }
- f0 /= (end - beg) * 2;
- err = fabs(f0 - *f);
- *f = f0;
- return err;
-}
-
-/* The following function combines EM and Brent's method. When the signal from
- * the data is strong, EM is faster but sometimes, EM may converge very slowly.
- * When this happens, we switch to Brent's method. The idea is learned from
- * Rasmus Nielsen.
- */
-static double freqml(double f0, int beg, int end, const double *pdg)
-{
- int i;
- double f;
- for (i = 0, f = f0; i < ITER_TRY; ++i)
- if (freq_iter(&f, pdg, beg, end) < EPS) break;
- if (i == ITER_TRY) { // haven't converged yet; try Brent's method
- minaux1_t a;
- a.beg = beg; a.end = end; a.pdg = pdg;
- kmin_brent(prob1, f0 == f? .5*f0 : f0, f, (void*)&a, EPS, &f);
- }
- return f;
-}
-
-// one EM iteration for genotype frequency estimate
-static double g3_iter(double g[3], const double *_pdg, int beg, int end)
-{
- double err, gg[3];
- int i;
- gg[0] = gg[1] = gg[2] = 0.;
-// printf("%lg,%lg,%lg\n", g[0], g[1], g[2]);
- for (i = beg; i < end; ++i) {
- double sum, tmp[3];
- const double *pdg = _pdg + i * 3;
- tmp[0] = pdg[0] * g[0]; tmp[1] = pdg[1] * g[1]; tmp[2] = pdg[2] * g[2];
- sum = (tmp[0] + tmp[1] + tmp[2]) * (end - beg);
- gg[0] += tmp[0] / sum; gg[1] += tmp[1] / sum; gg[2] += tmp[2] / sum;
- }
- err = fabs(gg[0] - g[0]) > fabs(gg[1] - g[1])? fabs(gg[0] - g[0]) : fabs(gg[1] - g[1]);
- err = err > fabs(gg[2] - g[2])? err : fabs(gg[2] - g[2]);
- g[0] = gg[0]; g[1] = gg[1]; g[2] = gg[2];
- return err;
-}
-
-// perform likelihood ratio test
-static double lk_ratio_test(int n, int n1, const double *pdg, double f3[3][3])
-{
- double r;
- int i;
- for (i = 0, r = 1.; i < n1; ++i) {
- const double *p = pdg + i * 3;
- r *= (p[0] * f3[1][0] + p[1] * f3[1][1] + p[2] * f3[1][2])
- / (p[0] * f3[0][0] + p[1] * f3[0][1] + p[2] * f3[0][2]);
- }
- for (; i < n; ++i) {
- const double *p = pdg + i * 3;
- r *= (p[0] * f3[2][0] + p[1] * f3[2][1] + p[2] * f3[2][2])
- / (p[0] * f3[0][0] + p[1] * f3[0][1] + p[2] * f3[0][2]);
- }
- return r;
-}
-
-// x[0]: ref frequency
-// x[1..3]: alt-alt, alt-ref, ref-ref frequenc
-// x[4]: HWE P-value
-// x[5..6]: group1 freq, group2 freq
-// x[7]: 1-degree P-value
-// x[8]: 2-degree P-value
-int bcf_em1(const bcf1_t *b, int n1, int flag, double x[10])
-{
- double *pdg;
- int i, n, n2;
- if (b->n_alleles < 2) return -1; // one allele only
- // initialization
- if (n1 < 0 || n1 > b->n_smpl) n1 = 0;
- if (flag & 1<<7) flag |= 7<<5; // compute group freq if LRT is required
- if (flag & 0xf<<1) flag |= 0xf<<1;
- n = b->n_smpl; n2 = n - n1;
- pdg = get_pdg3(b);
- if (pdg == 0) return -1;
- for (i = 0; i < 10; ++i) x[i] = -1.; // set to negative
- {
- if ((x[0] = est_freq(n, pdg)) < 0.) {
- free(pdg);
- return -1; // no data
- }
- x[0] = freqml(x[0], 0, n, pdg);
- }
- if (flag & (0xf<<1|3<<8)) { // estimate the genotype frequency and test HWE
- double *g = x + 1, f3[3], r;
- f3[0] = g[0] = (1 - x[0]) * (1 - x[0]);
- f3[1] = g[1] = 2 * x[0] * (1 - x[0]);
- f3[2] = g[2] = x[0] * x[0];
- for (i = 0; i < ITER_MAX; ++i)
- if (g3_iter(g, pdg, 0, n) < EPS) break;
- // Hardy-Weinberg equilibrium (HWE)
- for (i = 0, r = 1.; i < n; ++i) {
- double *p = pdg + i * 3;
- r *= (p[0] * g[0] + p[1] * g[1] + p[2] * g[2]) / (p[0] * f3[0] + p[1] * f3[1] + p[2] * f3[2]);
- }
- x[4] = kf_gammaq(.5, log(r));
- }
- if ((flag & 7<<5) && n1 > 0 && n1 < n) { // group frequency
- x[5] = freqml(x[0], 0, n1, pdg);
- x[6] = freqml(x[0], n1, n, pdg);
- }
- if ((flag & 1<<7) && n1 > 0 && n1 < n) { // 1-degree P-value
- double f[3], f3[3][3], tmp;
- f[0] = x[0]; f[1] = x[5]; f[2] = x[6];
- for (i = 0; i < 3; ++i)
- f3[i][0] = (1-f[i])*(1-f[i]), f3[i][1] = 2*f[i]*(1-f[i]), f3[i][2] = f[i]*f[i];
- tmp = log(lk_ratio_test(n, n1, pdg, f3));
- if (tmp < 0) tmp = 0;
- x[7] = kf_gammaq(.5, tmp);
- }
- if ((flag & 3<<8) && n1 > 0 && n1 < n) { // 2-degree P-value
- double g[3][3], tmp;
- for (i = 0; i < 3; ++i) memcpy(g[i], x + 1, 3 * sizeof(double));
- for (i = 0; i < ITER_MAX; ++i)
- if (g3_iter(g[1], pdg, 0, n1) < EPS) break;
- for (i = 0; i < ITER_MAX; ++i)
- if (g3_iter(g[2], pdg, n1, n) < EPS) break;
- tmp = log(lk_ratio_test(n, n1, pdg, g));
- if (tmp < 0) tmp = 0;
- x[8] = kf_gammaq(1., tmp);
- }
- // free
- free(pdg);
- return 0;
-}
-
-/*
- Two-locus EM (LD)
- */
-
-#define _G1(h, k) ((h>>1&1) + (k>>1&1))
-#define _G2(h, k) ((h&1) + (k&1))
-
-// 0: the previous site; 1: the current site
-static int pair_freq_iter(int n, double *pdg[2], double f[4])
-{
- double ff[4];
- int i, k, h;
-// printf("%lf,%lf,%lf,%lf\n", f[0], f[1], f[2], f[3]);
- memset(ff, 0, 4 * sizeof(double));
- for (i = 0; i < n; ++i) {
- double *p[2], sum, tmp;
- p[0] = pdg[0] + i * 3; p[1] = pdg[1] + i * 3;
- for (k = 0, sum = 0.; k < 4; ++k)
- for (h = 0; h < 4; ++h)
- sum += f[k] * f[h] * p[0][_G1(k,h)] * p[1][_G2(k,h)];
- for (k = 0; k < 4; ++k) {
- tmp = f[0] * (p[0][_G1(0,k)] * p[1][_G2(0,k)] + p[0][_G1(k,0)] * p[1][_G2(k,0)])
- + f[1] * (p[0][_G1(1,k)] * p[1][_G2(1,k)] + p[0][_G1(k,1)] * p[1][_G2(k,1)])
- + f[2] * (p[0][_G1(2,k)] * p[1][_G2(2,k)] + p[0][_G1(k,2)] * p[1][_G2(k,2)])
- + f[3] * (p[0][_G1(3,k)] * p[1][_G2(3,k)] + p[0][_G1(k,3)] * p[1][_G2(k,3)]);
- ff[k] += f[k] * tmp / sum;
- }
- }
- for (k = 0; k < 4; ++k) f[k] = ff[k] / (2 * n);
- return 0;
-}
-
-double bcf_pair_freq(const bcf1_t *b0, const bcf1_t *b1, double f[4])
-{
- const bcf1_t *b[2];
- int i, j, n_smpl;
- double *pdg[2], flast[4], r, f0[2];
- // initialize others
- if (b0->n_smpl != b1->n_smpl) return -1; // different number of samples
- n_smpl = b0->n_smpl;
- b[0] = b0; b[1] = b1;
- f[0] = f[1] = f[2] = f[3] = -1.;
- if (b[0]->n_alleles < 2 || b[1]->n_alleles < 2) return -1; // one allele only
- pdg[0] = get_pdg3(b0); pdg[1] = get_pdg3(b1);
- if (pdg[0] == 0 || pdg[1] == 0) {
- free(pdg[0]); free(pdg[1]);
- return -1;
- }
- // set the initial value
- f0[0] = est_freq(n_smpl, pdg[0]);
- f0[1] = est_freq(n_smpl, pdg[1]);
- f[0] = (1 - f0[0]) * (1 - f0[1]); f[3] = f0[0] * f0[1];
- f[1] = (1 - f0[0]) * f0[1]; f[2] = f0[0] * (1 - f0[1]);
- // iteration
- for (j = 0; j < ITER_MAX; ++j) {
- double eps = 0;
- memcpy(flast, f, 4 * sizeof(double));
- pair_freq_iter(n_smpl, pdg, f);
- for (i = 0; i < 4; ++i) {
- double x = fabs(f[i] - flast[i]);
- if (x > eps) eps = x;
- }
- if (eps < EPS) break;
- }
- // free
- free(pdg[0]); free(pdg[1]);
- { // calculate r^2
- double p[2], q[2], D;
- p[0] = f[0] + f[1]; q[0] = 1 - p[0];
- p[1] = f[0] + f[2]; q[1] = 1 - p[1];
- D = f[0] * f[3] - f[1] * f[2];
- r = sqrt(D * D / (p[0] * p[1] * q[0] * q[1]));
-// printf("R(%lf,%lf,%lf,%lf)=%lf\n", f[0], f[1], f[2], f[3], r);
- if (isnan(r)) r = -1.;
- }
- return r;
-}
diff --git a/samtools-0.1.19/bcftools/fet.c b/samtools-0.1.19/bcftools/fet.c
deleted file mode 100644
index 5812517..0000000
--- a/samtools-0.1.19/bcftools/fet.c
+++ /dev/null
@@ -1,112 +0,0 @@
-#include <math.h>
-#include <stdlib.h>
-
-/* This program is implemented with ideas from this web page:
- *
- * http://www.langsrud.com/fisher.htm
- */
-
-// log\binom{n}{k}
-static double lbinom(int n, int k)
-{
- if (k == 0 || n == k) return 0;
- return lgamma(n+1) - lgamma(k+1) - lgamma(n-k+1);
-}
-
-// n11 n12 | n1_
-// n21 n22 | n2_
-//-----------+----
-// n_1 n_2 | n
-
-// hypergeometric distribution
-static double hypergeo(int n11, int n1_, int n_1, int n)
-{
- return exp(lbinom(n1_, n11) + lbinom(n-n1_, n_1-n11) - lbinom(n, n_1));
-}
-
-typedef struct {
- int n11, n1_, n_1, n;
- double p;
-} hgacc_t;
-
-// incremental version of hypergenometric distribution
-static double hypergeo_acc(int n11, int n1_, int n_1, int n, hgacc_t *aux)
-{
- if (n1_ || n_1 || n) {
- aux->n11 = n11; aux->n1_ = n1_; aux->n_1 = n_1; aux->n = n;
- } else { // then only n11 changed; the rest fixed
- if (n11%11 && n11 + aux->n - aux->n1_ - aux->n_1) {
- if (n11 == aux->n11 + 1) { // incremental
- aux->p *= (double)(aux->n1_ - aux->n11) / n11
- * (aux->n_1 - aux->n11) / (n11 + aux->n - aux->n1_ - aux->n_1);
- aux->n11 = n11;
- return aux->p;
- }
- if (n11 == aux->n11 - 1) { // incremental
- aux->p *= (double)aux->n11 / (aux->n1_ - n11)
- * (aux->n11 + aux->n - aux->n1_ - aux->n_1) / (aux->n_1 - n11);
- aux->n11 = n11;
- return aux->p;
- }
- }
- aux->n11 = n11;
- }
- aux->p = hypergeo(aux->n11, aux->n1_, aux->n_1, aux->n);
- return aux->p;
-}
-
-double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two)
-{
- int i, j, max, min;
- double p, q, left, right;
- hgacc_t aux;
- int n1_, n_1, n;
-
- n1_ = n11 + n12; n_1 = n11 + n21; n = n11 + n12 + n21 + n22; // calculate n1_, n_1 and n
- max = (n_1 < n1_) ? n_1 : n1_; // max n11, for right tail
- min = n1_ + n_1 - n;
- if (min < 0) min = 0; // min n11, for left tail
- *two = *_left = *_right = 1.;
- if (min == max) return 1.; // no need to do test
- q = hypergeo_acc(n11, n1_, n_1, n, &aux); // the probability of the current table
- // left tail
- p = hypergeo_acc(min, 0, 0, 0, &aux);
- for (left = 0., i = min + 1; p < 0.99999999 * q; ++i) // loop until underflow
- left += p, p = hypergeo_acc(i, 0, 0, 0, &aux);
- --i;
- if (p < 1.00000001 * q) left += p;
- else --i;
- // right tail
- p = hypergeo_acc(max, 0, 0, 0, &aux);
- for (right = 0., j = max - 1; p < 0.99999999 * q; --j) // loop until underflow
- right += p, p = hypergeo_acc(j, 0, 0, 0, &aux);
- ++j;
- if (p < 1.00000001 * q) right += p;
- else ++j;
- // two-tail
- *two = left + right;
- if (*two > 1.) *two = 1.;
- // adjust left and right
- if (abs(i - n11) < abs(j - n11)) right = 1. - left + q;
- else left = 1.0 - right + q;
- *_left = left; *_right = right;
- return q;
-}
-
-#ifdef FET_MAIN
-#include <stdio.h>
-
-int main(int argc, char *argv[])
-{
- char id[1024];
- int n11, n12, n21, n22;
- double left, right, twotail, prob;
-
- while (scanf("%s%d%d%d%d", id, &n11, &n12, &n21, &n22) == 5) {
- prob = kt_fisher_exact(n11, n12, n21, n22, &left, &right, &twotail);
- printf("%s\t%d\t%d\t%d\t%d\t%.6g\t%.6g\t%.6g\t%.6g\n", id, n11, n12, n21, n22,
- prob, left, right, twotail);
- }
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/bcftools/index.c b/samtools-0.1.19/bcftools/index.c
deleted file mode 100644
index a7db24f..0000000
--- a/samtools-0.1.19/bcftools/index.c
+++ /dev/null
@@ -1,336 +0,0 @@
-#include <assert.h>
-#include <ctype.h>
-#include <sys/stat.h>
-#include "bam_endian.h"
-#include "kstring.h"
-#include "bcf.h"
-#ifdef _USE_KNETFILE
-#include "knetfile.h"
-#endif
-
-#define TAD_LIDX_SHIFT 13
-
-typedef struct {
- int32_t n, m;
- uint64_t *offset;
-} bcf_lidx_t;
-
-struct __bcf_idx_t {
- int32_t n;
- bcf_lidx_t *index2;
-};
-
-/************
- * indexing *
- ************/
-
-static inline void insert_offset2(bcf_lidx_t *index2, int _beg, int _end, uint64_t offset)
-{
- int i, beg, end;
- beg = _beg >> TAD_LIDX_SHIFT;
- end = (_end - 1) >> TAD_LIDX_SHIFT;
- if (index2->m < end + 1) {
- int old_m = index2->m;
- index2->m = end + 1;
- kroundup32(index2->m);
- index2->offset = (uint64_t*)realloc(index2->offset, index2->m * 8);
- memset(index2->offset + old_m, 0, 8 * (index2->m - old_m));
- }
- if (beg == end) {
- if (index2->offset[beg] == 0) index2->offset[beg] = offset;
- } else {
- for (i = beg; i <= end; ++i)
- if (index2->offset[i] == 0) index2->offset[i] = offset;
- }
- if (index2->n < end + 1) index2->n = end + 1;
-}
-
-bcf_idx_t *bcf_idx_core(bcf_t *bp, bcf_hdr_t *h)
-{
- bcf_idx_t *idx;
- int32_t last_coor, last_tid;
- uint64_t last_off;
- kstring_t *str;
- BGZF *fp = bp->fp;
- bcf1_t *b;
- int ret;
-
- b = calloc(1, sizeof(bcf1_t));
- str = calloc(1, sizeof(kstring_t));
- idx = (bcf_idx_t*)calloc(1, sizeof(bcf_idx_t));
- idx->n = h->n_ref;
- idx->index2 = calloc(h->n_ref, sizeof(bcf_lidx_t));
-
- last_tid = 0xffffffffu;
- last_off = bgzf_tell(fp); last_coor = 0xffffffffu;
- while ((ret = bcf_read(bp, h, b)) > 0) {
- int end, tmp;
- if (last_tid != b->tid) { // change of chromosomes
- last_tid = b->tid;
- } else if (last_coor > b->pos) {
- fprintf(stderr, "[bcf_idx_core] the input is out of order\n");
- free(str->s); free(str); free(idx); bcf_destroy(b);
- return 0;
- }
- tmp = strlen(b->ref);
- end = b->pos + (tmp > 0? tmp : 1);
- insert_offset2(&idx->index2[b->tid], b->pos, end, last_off);
- last_off = bgzf_tell(fp);
- last_coor = b->pos;
- }
- free(str->s); free(str); bcf_destroy(b);
- return idx;
-}
-
-void bcf_idx_destroy(bcf_idx_t *idx)
-{
- int i;
- if (idx == 0) return;
- for (i = 0; i < idx->n; ++i) free(idx->index2[i].offset);
- free(idx->index2);
- free(idx);
-}
-
-/******************
- * index file I/O *
- ******************/
-
-void bcf_idx_save(const bcf_idx_t *idx, BGZF *fp)
-{
- int32_t i, ti_is_be;
- ti_is_be = bam_is_big_endian();
- bgzf_write(fp, "BCI\4", 4);
- if (ti_is_be) {
- uint32_t x = idx->n;
- bgzf_write(fp, bam_swap_endian_4p(&x), 4);
- } else bgzf_write(fp, &idx->n, 4);
- for (i = 0; i < idx->n; ++i) {
- bcf_lidx_t *index2 = idx->index2 + i;
- // write linear index (index2)
- if (ti_is_be) {
- int x = index2->n;
- bgzf_write(fp, bam_swap_endian_4p(&x), 4);
- } else bgzf_write(fp, &index2->n, 4);
- if (ti_is_be) { // big endian
- int x;
- for (x = 0; (int)x < index2->n; ++x)
- bam_swap_endian_8p(&index2->offset[x]);
- bgzf_write(fp, index2->offset, 8 * index2->n);
- for (x = 0; (int)x < index2->n; ++x)
- bam_swap_endian_8p(&index2->offset[x]);
- } else bgzf_write(fp, index2->offset, 8 * index2->n);
- }
-}
-
-static bcf_idx_t *bcf_idx_load_core(BGZF *fp)
-{
- int i, ti_is_be;
- char magic[4];
- bcf_idx_t *idx;
- ti_is_be = bam_is_big_endian();
- if (fp == 0) {
- fprintf(stderr, "[%s] fail to load index.\n", __func__);
- return 0;
- }
- bgzf_read(fp, magic, 4);
- if (strncmp(magic, "BCI\4", 4)) {
- fprintf(stderr, "[%s] wrong magic number.\n", __func__);
- return 0;
- }
- idx = (bcf_idx_t*)calloc(1, sizeof(bcf_idx_t));
- bgzf_read(fp, &idx->n, 4);
- if (ti_is_be) bam_swap_endian_4p(&idx->n);
- idx->index2 = (bcf_lidx_t*)calloc(idx->n, sizeof(bcf_lidx_t));
- for (i = 0; i < idx->n; ++i) {
- bcf_lidx_t *index2 = idx->index2 + i;
- int j;
- bgzf_read(fp, &index2->n, 4);
- if (ti_is_be) bam_swap_endian_4p(&index2->n);
- index2->m = index2->n;
- index2->offset = (uint64_t*)calloc(index2->m, 8);
- bgzf_read(fp, index2->offset, index2->n * 8);
- if (ti_is_be)
- for (j = 0; j < index2->n; ++j) bam_swap_endian_8p(&index2->offset[j]);
- }
- return idx;
-}
-
-bcf_idx_t *bcf_idx_load_local(const char *fnidx)
-{
- BGZF *fp;
- fp = bgzf_open(fnidx, "r");
- if (fp) {
- bcf_idx_t *idx = bcf_idx_load_core(fp);
- bgzf_close(fp);
- return idx;
- } else return 0;
-}
-
-#ifdef _USE_KNETFILE
-static void download_from_remote(const char *url)
-{
- const int buf_size = 1 * 1024 * 1024;
- char *fn;
- FILE *fp;
- uint8_t *buf;
- knetFile *fp_remote;
- int l;
- if (strstr(url, "ftp://") != url && strstr(url, "http://") != url) return;
- l = strlen(url);
- for (fn = (char*)url + l - 1; fn >= url; --fn)
- if (*fn == '/') break;
- ++fn; // fn now points to the file name
- fp_remote = knet_open(url, "r");
- if (fp_remote == 0) {
- fprintf(stderr, "[download_from_remote] fail to open remote file.\n");
- return;
- }
- if ((fp = fopen(fn, "w")) == 0) {
- fprintf(stderr, "[download_from_remote] fail to create file in the working directory.\n");
- knet_close(fp_remote);
- return;
- }
- buf = (uint8_t*)calloc(buf_size, 1);
- while ((l = knet_read(fp_remote, buf, buf_size)) != 0)
- fwrite(buf, 1, l, fp);
- free(buf);
- fclose(fp);
- knet_close(fp_remote);
-}
-#else
-static void download_from_remote(const char *url)
-{
- return;
-}
-#endif
-
-static char *get_local_version(const char *fn)
-{
- struct stat sbuf;
- char *fnidx = (char*)calloc(strlen(fn) + 5, 1);
- strcat(strcpy(fnidx, fn), ".bci");
- if ((strstr(fnidx, "ftp://") == fnidx || strstr(fnidx, "http://") == fnidx)) {
- char *p, *url;
- int l = strlen(fnidx);
- for (p = fnidx + l - 1; p >= fnidx; --p)
- if (*p == '/') break;
- url = fnidx; fnidx = strdup(p + 1);
- if (stat(fnidx, &sbuf) == 0) {
- free(url);
- return fnidx;
- }
- fprintf(stderr, "[%s] downloading the index file...\n", __func__);
- download_from_remote(url);
- free(url);
- }
- if (stat(fnidx, &sbuf) == 0) return fnidx;
- free(fnidx); return 0;
-}
-
-bcf_idx_t *bcf_idx_load(const char *fn)
-{
- bcf_idx_t *idx;
- char *fname = get_local_version(fn);
- if (fname == 0) return 0;
- idx = bcf_idx_load_local(fname);
- free(fname);
- return idx;
-}
-
-int bcf_idx_build2(const char *fn, const char *_fnidx)
-{
- char *fnidx;
- BGZF *fpidx;
- bcf_t *bp;
- bcf_idx_t *idx;
- bcf_hdr_t *h;
- if ((bp = bcf_open(fn, "r")) == 0) {
- fprintf(stderr, "[bcf_idx_build2] fail to open the BAM file.\n");
- return -1;
- }
- h = bcf_hdr_read(bp);
- idx = bcf_idx_core(bp, h);
- bcf_close(bp);
- if (_fnidx == 0) {
- fnidx = (char*)calloc(strlen(fn) + 5, 1);
- strcpy(fnidx, fn); strcat(fnidx, ".bci");
- } else fnidx = strdup(_fnidx);
- fpidx = bgzf_open(fnidx, "w");
- if (fpidx == 0) {
- fprintf(stderr, "[bcf_idx_build2] fail to create the index file.\n");
- free(fnidx);
- bcf_idx_destroy(idx);
- return -1;
- }
- bcf_idx_save(idx, fpidx);
- bcf_idx_destroy(idx);
- bgzf_close(fpidx);
- free(fnidx);
- return 0;
-}
-
-int bcf_idx_build(const char *fn)
-{
- return bcf_idx_build2(fn, 0);
-}
-
-/********************************************
- * parse a region in the format chr:beg-end *
- ********************************************/
-
-int bcf_parse_region(void *str2id, const char *str, int *tid, int *begin, int *end)
-{
- char *s, *p;
- int i, l, k;
- l = strlen(str);
- p = s = (char*)malloc(l+1);
- /* squeeze out "," */
- for (i = k = 0; i != l; ++i)
- if (str[i] != ',' && !isspace(str[i])) s[k++] = str[i];
- s[k] = 0;
- for (i = 0; i != k; ++i) if (s[i] == ':') break;
- s[i] = 0;
- if ((*tid = bcf_str2id(str2id, s)) < 0) {
- free(s);
- return -1;
- }
- if (i == k) { /* dump the whole sequence */
- *begin = 0; *end = 1<<29; free(s);
- return 0;
- }
- for (p = s + i + 1; i != k; ++i) if (s[i] == '-') break;
- *begin = atoi(p);
- if (i < k) {
- p = s + i + 1;
- *end = atoi(p);
- } else *end = 1<<29;
- if (*begin > 0) --*begin;
- free(s);
- if (*begin > *end) return -1;
- return 0;
-}
-
-/*******************************
- * retrieve a specified region *
- *******************************/
-
-uint64_t bcf_idx_query(const bcf_idx_t *idx, int tid, int beg)
-{
- uint64_t min_off, *offset;
- int i;
- if (beg < 0) beg = 0;
- offset = idx->index2[tid].offset;
- for (i = beg>>TAD_LIDX_SHIFT; i < idx->index2[tid].n && offset[i] == 0; ++i);
- min_off = (i == idx->index2[tid].n)? offset[idx->index2[tid].n-1] : offset[i];
- return min_off;
-}
-
-int bcf_main_index(int argc, char *argv[])
-{
- if (argc == 1) {
- fprintf(stderr, "Usage: bcftools index <in.bcf>\n");
- return 1;
- }
- bcf_idx_build(argv[1]);
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/kfunc.c b/samtools-0.1.19/bcftools/kfunc.c
deleted file mode 100644
index a637b6c..0000000
--- a/samtools-0.1.19/bcftools/kfunc.c
+++ /dev/null
@@ -1,162 +0,0 @@
-#include <math.h>
-
-
-/* Log gamma function
- * \log{\Gamma(z)}
- * AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245
- */
-double kf_lgamma(double z)
-{
- double x = 0;
- x += 0.1659470187408462e-06 / (z+7);
- x += 0.9934937113930748e-05 / (z+6);
- x -= 0.1385710331296526 / (z+5);
- x += 12.50734324009056 / (z+4);
- x -= 176.6150291498386 / (z+3);
- x += 771.3234287757674 / (z+2);
- x -= 1259.139216722289 / (z+1);
- x += 676.5203681218835 / z;
- x += 0.9999999999995183;
- return log(x) - 5.58106146679532777 - z + (z-0.5) * log(z+6.5);
-}
-
-/* complementary error function
- * \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} dt
- * AS66, 2nd algorithm, http://lib.stat.cmu.edu/apstat/66
- */
-double kf_erfc(double x)
-{
- const double p0 = 220.2068679123761;
- const double p1 = 221.2135961699311;
- const double p2 = 112.0792914978709;
- const double p3 = 33.912866078383;
- const double p4 = 6.37396220353165;
- const double p5 = .7003830644436881;
- const double p6 = .03526249659989109;
- const double q0 = 440.4137358247522;
- const double q1 = 793.8265125199484;
- const double q2 = 637.3336333788311;
- const double q3 = 296.5642487796737;
- const double q4 = 86.78073220294608;
- const double q5 = 16.06417757920695;
- const double q6 = 1.755667163182642;
- const double q7 = .08838834764831844;
- double expntl, z, p;
- z = fabs(x) * M_SQRT2;
- if (z > 37.) return x > 0.? 0. : 2.;
- expntl = exp(z * z * - .5);
- if (z < 10. / M_SQRT2) // for small z
- p = expntl * ((((((p6 * z + p5) * z + p4) * z + p3) * z + p2) * z + p1) * z + p0)
- / (((((((q7 * z + q6) * z + q5) * z + q4) * z + q3) * z + q2) * z + q1) * z + q0);
- else p = expntl / 2.506628274631001 / (z + 1. / (z + 2. / (z + 3. / (z + 4. / (z + .65)))));
- return x > 0.? 2. * p : 2. * (1. - p);
-}
-
-/* The following computes regularized incomplete gamma functions.
- * Formulas are taken from Wiki, with additional input from Numerical
- * Recipes in C (for modified Lentz's algorithm) and AS245
- * (http://lib.stat.cmu.edu/apstat/245).
- *
- * A good online calculator is available at:
- *
- * http://www.danielsoper.com/statcalc/calc23.aspx
- *
- * It calculates upper incomplete gamma function, which equals
- * kf_gammaq(s,z)*tgamma(s).
- */
-
-#define KF_GAMMA_EPS 1e-14
-#define KF_TINY 1e-290
-
-// regularized lower incomplete gamma function, by series expansion
-static double _kf_gammap(double s, double z)
-{
- double sum, x;
- int k;
- for (k = 1, sum = x = 1.; k < 100; ++k) {
- sum += (x *= z / (s + k));
- if (x / sum < KF_GAMMA_EPS) break;
- }
- return exp(s * log(z) - z - kf_lgamma(s + 1.) + log(sum));
-}
-// regularized upper incomplete gamma function, by continued fraction
-static double _kf_gammaq(double s, double z)
-{
- int j;
- double C, D, f;
- f = 1. + z - s; C = f; D = 0.;
- // Modified Lentz's algorithm for computing continued fraction
- // See Numerical Recipes in C, 2nd edition, section 5.2
- for (j = 1; j < 100; ++j) {
- double a = j * (s - j), b = (j<<1) + 1 + z - s, d;
- D = b + a * D;
- if (D < KF_TINY) D = KF_TINY;
- C = b + a / C;
- if (C < KF_TINY) C = KF_TINY;
- D = 1. / D;
- d = C * D;
- f *= d;
- if (fabs(d - 1.) < KF_GAMMA_EPS) break;
- }
- return exp(s * log(z) - z - kf_lgamma(s) - log(f));
-}
-
-double kf_gammap(double s, double z)
-{
- return z <= 1. || z < s? _kf_gammap(s, z) : 1. - _kf_gammaq(s, z);
-}
-
-double kf_gammaq(double s, double z)
-{
- return z <= 1. || z < s? 1. - _kf_gammap(s, z) : _kf_gammaq(s, z);
-}
-
-/* Regularized incomplete beta function. The method is taken from
- * Numerical Recipe in C, 2nd edition, section 6.4. The following web
- * page calculates the incomplete beta function, which equals
- * kf_betai(a,b,x) * gamma(a) * gamma(b) / gamma(a+b):
- *
- * http://www.danielsoper.com/statcalc/calc36.aspx
- */
-static double kf_betai_aux(double a, double b, double x)
-{
- double C, D, f;
- int j;
- if (x == 0.) return 0.;
- if (x == 1.) return 1.;
- f = 1.; C = f; D = 0.;
- // Modified Lentz's algorithm for computing continued fraction
- for (j = 1; j < 200; ++j) {
- double aa, d;
- int m = j>>1;
- aa = (j&1)? -(a + m) * (a + b + m) * x / ((a + 2*m) * (a + 2*m + 1))
- : m * (b - m) * x / ((a + 2*m - 1) * (a + 2*m));
- D = 1. + aa * D;
- if (D < KF_TINY) D = KF_TINY;
- C = 1. + aa / C;
- if (C < KF_TINY) C = KF_TINY;
- D = 1. / D;
- d = C * D;
- f *= d;
- if (fabs(d - 1.) < KF_GAMMA_EPS) break;
- }
- return exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b) + a * log(x) + b * log(1.-x)) / a / f;
-}
-double kf_betai(double a, double b, double x)
-{
- return x < (a + 1.) / (a + b + 2.)? kf_betai_aux(a, b, x) : 1. - kf_betai_aux(b, a, 1. - x);
-}
-
-#ifdef KF_MAIN
-#include <stdio.h>
-int main(int argc, char *argv[])
-{
- double x = 5.5, y = 3;
- double a, b;
- printf("erfc(%lg): %lg, %lg\n", x, erfc(x), kf_erfc(x));
- printf("upper-gamma(%lg,%lg): %lg\n", x, y, kf_gammaq(y, x)*tgamma(y));
- a = 2; b = 2; x = 0.5;
- printf("incomplete-beta(%lg,%lg,%lg): %lg\n", a, b, x, kf_betai(a, b, x) / exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b)));
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/bcftools/kmin.c b/samtools-0.1.19/bcftools/kmin.c
deleted file mode 100644
index 5b8193b..0000000
--- a/samtools-0.1.19/bcftools/kmin.c
+++ /dev/null
@@ -1,209 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008, 2010 by Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Hooke-Jeeves algorithm for nonlinear minimization
-
- Based on the pseudocodes by Bell and Pike (CACM 9(9):684-685), and
- the revision by Tomlin and Smith (CACM 12(11):637-638). Both of the
- papers are comments on Kaupe's Algorithm 178 "Direct Search" (ACM
- 6(6):313-314). The original algorithm was designed by Hooke and
- Jeeves (ACM 8:212-229). This program is further revised according to
- Johnson's implementation at Netlib (opt/hooke.c).
-
- Hooke-Jeeves algorithm is very simple and it works quite well on a
- few examples. However, it might fail to converge due to its heuristic
- nature. A possible improvement, as is suggested by Johnson, may be to
- choose a small r at the beginning to quickly approach to the minimum
- and a large r at later step to hit the minimum.
- */
-
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include "kmin.h"
-
-static double __kmin_hj_aux(kmin_f func, int n, double *x1, void *data, double fx1, double *dx, int *n_calls)
-{
- int k, j = *n_calls;
- double ftmp;
- for (k = 0; k != n; ++k) {
- x1[k] += dx[k];
- ftmp = func(n, x1, data); ++j;
- if (ftmp < fx1) fx1 = ftmp;
- else { /* search the opposite direction */
- dx[k] = 0.0 - dx[k];
- x1[k] += dx[k] + dx[k];
- ftmp = func(n, x1, data); ++j;
- if (ftmp < fx1) fx1 = ftmp;
- else x1[k] -= dx[k]; /* back to the original x[k] */
- }
- }
- *n_calls = j;
- return fx1; /* here: fx1=f(n,x1) */
-}
-
-double kmin_hj(kmin_f func, int n, double *x, void *data, double r, double eps, int max_calls)
-{
- double fx, fx1, *x1, *dx, radius;
- int k, n_calls = 0;
- x1 = (double*)calloc(n, sizeof(double));
- dx = (double*)calloc(n, sizeof(double));
- for (k = 0; k != n; ++k) { /* initial directions, based on MGJ */
- dx[k] = fabs(x[k]) * r;
- if (dx[k] == 0) dx[k] = r;
- }
- radius = r;
- fx1 = fx = func(n, x, data); ++n_calls;
- for (;;) {
- memcpy(x1, x, n * sizeof(double)); /* x1 = x */
- fx1 = __kmin_hj_aux(func, n, x1, data, fx, dx, &n_calls);
- while (fx1 < fx) {
- for (k = 0; k != n; ++k) {
- double t = x[k];
- dx[k] = x1[k] > x[k]? fabs(dx[k]) : 0.0 - fabs(dx[k]);
- x[k] = x1[k];
- x1[k] = x1[k] + x1[k] - t;
- }
- fx = fx1;
- if (n_calls >= max_calls) break;
- fx1 = func(n, x1, data); ++n_calls;
- fx1 = __kmin_hj_aux(func, n, x1, data, fx1, dx, &n_calls);
- if (fx1 >= fx) break;
- for (k = 0; k != n; ++k)
- if (fabs(x1[k] - x[k]) > .5 * fabs(dx[k])) break;
- if (k == n) break;
- }
- if (radius >= eps) {
- if (n_calls >= max_calls) break;
- radius *= r;
- for (k = 0; k != n; ++k) dx[k] *= r;
- } else break; /* converge */
- }
- free(x1); free(dx);
- return fx1;
-}
-
-// I copied this function somewhere several years ago with some of my modifications, but I forgot the source.
-double kmin_brent(kmin1_f func, double a, double b, void *data, double tol, double *xmin)
-{
- double bound, u, r, q, fu, tmp, fa, fb, fc, c;
- const double gold1 = 1.6180339887;
- const double gold2 = 0.3819660113;
- const double tiny = 1e-20;
- const int max_iter = 100;
-
- double e, d, w, v, mid, tol1, tol2, p, eold, fv, fw;
- int iter;
-
- fa = func(a, data); fb = func(b, data);
- if (fb > fa) { // swap, such that f(a) > f(b)
- tmp = a; a = b; b = tmp;
- tmp = fa; fa = fb; fb = tmp;
- }
- c = b + gold1 * (b - a), fc = func(c, data); // golden section extrapolation
- while (fb > fc) {
- bound = b + 100.0 * (c - b); // the farthest point where we want to go
- r = (b - a) * (fb - fc);
- q = (b - c) * (fb - fa);
- if (fabs(q - r) < tiny) { // avoid 0 denominator
- tmp = q > r? tiny : 0.0 - tiny;
- } else tmp = q - r;
- u = b - ((b - c) * q - (b - a) * r) / (2.0 * tmp); // u is the parabolic extrapolation point
- if ((b > u && u > c) || (b < u && u < c)) { // u lies between b and c
- fu = func(u, data);
- if (fu < fc) { // (b,u,c) bracket the minimum
- a = b; b = u; fa = fb; fb = fu;
- break;
- } else if (fu > fb) { // (a,b,u) bracket the minimum
- c = u; fc = fu;
- break;
- }
- u = c + gold1 * (c - b); fu = func(u, data); // golden section extrapolation
- } else if ((c > u && u > bound) || (c < u && u < bound)) { // u lies between c and bound
- fu = func(u, data);
- if (fu < fc) { // fb > fc > fu
- b = c; c = u; u = c + gold1 * (c - b);
- fb = fc; fc = fu; fu = func(u, data);
- } else { // (b,c,u) bracket the minimum
- a = b; b = c; c = u;
- fa = fb; fb = fc; fc = fu;
- break;
- }
- } else if ((u > bound && bound > c) || (u < bound && bound < c)) { // u goes beyond the bound
- u = bound; fu = func(u, data);
- } else { // u goes the other way around, use golden section extrapolation
- u = c + gold1 * (c - b); fu = func(u, data);
- }
- a = b; b = c; c = u;
- fa = fb; fb = fc; fc = fu;
- }
- if (a > c) u = a, a = c, c = u; // swap
-
- // now, a<b<c, fa>fb and fb<fc, move on to Brent's algorithm
- e = d = 0.0;
- w = v = b; fv = fw = fb;
- for (iter = 0; iter != max_iter; ++iter) {
- mid = 0.5 * (a + c);
- tol2 = 2.0 * (tol1 = tol * fabs(b) + tiny);
- if (fabs(b - mid) <= (tol2 - 0.5 * (c - a))) {
- *xmin = b; return fb; // found
- }
- if (fabs(e) > tol1) {
- // related to parabolic interpolation
- r = (b - w) * (fb - fv);
- q = (b - v) * (fb - fw);
- p = (b - v) * q - (b - w) * r;
- q = 2.0 * (q - r);
- if (q > 0.0) p = 0.0 - p;
- else q = 0.0 - q;
- eold = e; e = d;
- if (fabs(p) >= fabs(0.5 * q * eold) || p <= q * (a - b) || p >= q * (c - b)) {
- d = gold2 * (e = (b >= mid ? a - b : c - b));
- } else {
- d = p / q; u = b + d; // actual parabolic interpolation happens here
- if (u - a < tol2 || c - u < tol2)
- d = (mid > b)? tol1 : 0.0 - tol1;
- }
- } else d = gold2 * (e = (b >= mid ? a - b : c - b)); // golden section interpolation
- u = fabs(d) >= tol1 ? b + d : b + (d > 0.0? tol1 : -tol1);
- fu = func(u, data);
- if (fu <= fb) { // u is the minimum point so far
- if (u >= b) a = b;
- else c = b;
- v = w; w = b; b = u; fv = fw; fw = fb; fb = fu;
- } else { // adjust (a,c) and (u,v,w)
- if (u < b) a = u;
- else c = u;
- if (fu <= fw || w == b) {
- v = w; w = u;
- fv = fw; fw = fu;
- } else if (fu <= fv || v == b || v == w) {
- v = u; fv = fu;
- }
- }
- }
- *xmin = b;
- return fb;
-}
diff --git a/samtools-0.1.19/bcftools/kmin.h b/samtools-0.1.19/bcftools/kmin.h
deleted file mode 100644
index 6feba45..0000000
--- a/samtools-0.1.19/bcftools/kmin.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- Copyright (c) 2008, 2010 by Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#ifndef KMIN_H
-#define KMIN_H
-
-#define KMIN_RADIUS 0.5
-#define KMIN_EPS 1e-7
-#define KMIN_MAXCALL 50000
-
-typedef double (*kmin_f)(int, double*, void*);
-typedef double (*kmin1_f)(double, void*);
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- double kmin_hj(kmin_f func, int n, double *x, void *data, double r, double eps, int max_calls);
- double kmin_brent(kmin1_f func, double a, double b, void *data, double tol, double *xmin);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/bcftools/main.c b/samtools-0.1.19/bcftools/main.c
deleted file mode 100644
index eda6217..0000000
--- a/samtools-0.1.19/bcftools/main.c
+++ /dev/null
@@ -1,191 +0,0 @@
-#include <string.h>
-#include <stdlib.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include "knetfile.h"
-#include "bcf.h"
-
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 0x10000)
-
-int bcfview(int argc, char *argv[]);
-int bcf_main_index(int argc, char *argv[]);
-
-#define BUF_SIZE 0x10000
-
-int bcf_cat(int n, char * const *fn)
-{
- int i;
- bcf_t *out;
- uint8_t *buf;
- buf = malloc(BUF_SIZE);
- out = bcf_open("-", "w");
- for (i = 0; i < n; ++i) {
- bcf_t *in;
- bcf_hdr_t *h;
- off_t end;
- struct stat s;
- in = bcf_open(fn[i], "r");
- h = bcf_hdr_read(in);
- if (i == 0) bcf_hdr_write(out, h);
- bcf_hdr_destroy(h);
-#ifdef _USE_KNETFILE
- fstat(knet_fileno((knetFile*)in->fp->fp), &s);
- end = s.st_size - 28;
- while (knet_tell((knetFile*)in->fp->fp) < end) {
- int size = knet_tell((knetFile*)in->fp->fp) + BUF_SIZE < end? BUF_SIZE : end - knet_tell((knetFile*)in->fp->fp);
- knet_read(in->fp->fp, buf, size);
- fwrite(buf, 1, size, out->fp->fp);
- }
-#else
- abort(); // FIXME: not implemented
-#endif
- bcf_close(in);
- }
- bcf_close(out);
- free(buf);
- return 0;
-}
-
-extern double bcf_pair_freq(const bcf1_t *b0, const bcf1_t *b1, double f[4]);
-
-int bcf_main_ldpair(int argc, char *argv[])
-{
- bcf_t *fp;
- bcf_hdr_t *h;
- bcf1_t *b0, *b1;
- bcf_idx_t *idx;
- kstring_t str;
- void *str2id;
- gzFile fplist;
- kstream_t *ks;
- int dret, lineno = 0;
- if (argc < 3) {
- fprintf(stderr, "Usage: bcftools ldpair <in.bcf> <in.list>\n");
- return 1;
- }
- fplist = gzopen(argv[2], "rb");
- ks = ks_init(fplist);
- memset(&str, 0, sizeof(kstring_t));
- fp = bcf_open(argv[1], "rb");
- h = bcf_hdr_read(fp);
- str2id = bcf_build_refhash(h);
- idx = bcf_idx_load(argv[1]);
- if (idx == 0) {
- fprintf(stderr, "[%s] No bcf index is found. Abort!\n", __func__);
- return 1;
- }
- b0 = calloc(1, sizeof(bcf1_t));
- b1 = calloc(1, sizeof(bcf1_t));
- while (ks_getuntil(ks, '\n', &str, &dret) >= 0) {
- char *p, *q;
- int k;
- int tid0 = -1, tid1 = -1, pos0 = -1, pos1 = -1;
- ++lineno;
- for (p = q = str.s, k = 0; *p; ++p) {
- if (*p == ' ' || *p == '\t') {
- *p = '\0';
- if (k == 0) tid0 = bcf_str2id(str2id, q);
- else if (k == 1) pos0 = atoi(q) - 1;
- else if (k == 2) tid1 = strcmp(q, "=")? bcf_str2id(str2id, q) : tid0;
- else if (k == 3) pos1 = atoi(q) - 1;
- q = p + 1;
- ++k;
- }
- }
- if (k == 3) pos1 = atoi(q) - 1;
- if (tid0 >= 0 && tid1 >= 0 && pos0 >= 0 && pos1 >= 0) {
- uint64_t off;
- double r, f[4];
- off = bcf_idx_query(idx, tid0, pos0);
- bgzf_seek(fp->fp, off, SEEK_SET);
- while (bcf_read(fp, h, b0) >= 0 && b0->pos != pos0);
- off = bcf_idx_query(idx, tid1, pos1);
- bgzf_seek(fp->fp, off, SEEK_SET);
- while (bcf_read(fp, h, b1) >= 0 && b1->pos != pos1);
- r = bcf_pair_freq(b0, b1, f);
- r *= r;
- printf("%s\t%d\t%s\t%d\t%.4g\t%.4g\t%.4g\t%.4g\t%.4g\n", h->ns[tid0], pos0+1, h->ns[tid1], pos1+1,
- r, f[0], f[1], f[2], f[3]);
- } //else fprintf(stderr, "[%s] Parse error at line %d.\n", __func__, lineno);
- }
- bcf_destroy(b0); bcf_destroy(b1);
- bcf_idx_destroy(idx);
- bcf_str2id_destroy(str2id);
- bcf_hdr_destroy(h);
- bcf_close(fp);
- free(str.s);
- ks_destroy(ks);
- gzclose(fplist);
- return 0;
-}
-
-int bcf_main_ld(int argc, char *argv[])
-{
- bcf_t *fp;
- bcf_hdr_t *h;
- bcf1_t **b, *b0;
- int i, j, m, n;
- double f[4];
- if (argc == 1) {
- fprintf(stderr, "Usage: bcftools ld <in.bcf>\n");
- return 1;
- }
- fp = bcf_open(argv[1], "rb");
- h = bcf_hdr_read(fp);
- // read the entire BCF
- m = n = 0; b = 0;
- b0 = calloc(1, sizeof(bcf1_t));
- while (bcf_read(fp, h, b0) >= 0) {
- if (m == n) {
- m = m? m<<1 : 16;
- b = realloc(b, sizeof(void*) * m);
- }
- b[n] = calloc(1, sizeof(bcf1_t));
- bcf_cpy(b[n++], b0);
- }
- bcf_destroy(b0);
- // compute pair-wise r^2
- printf("%d\n", n); // the number of loci
- for (i = 0; i < n; ++i) {
- printf("%s:%d", h->ns[b[i]->tid], b[i]->pos + 1);
- for (j = 0; j < i; ++j) {
- double r = bcf_pair_freq(b[i], b[j], f);
- printf("\t%.3f", r*r);
- }
- printf("\t1.000\n");
- }
- // free
- for (i = 0; i < n; ++i) bcf_destroy(b[i]);
- free(b);
- bcf_hdr_destroy(h);
- bcf_close(fp);
- return 0;
-}
-
-int main(int argc, char *argv[])
-{
- if (argc == 1) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Program: bcftools (Tools for data in the VCF/BCF formats)\n");
- fprintf(stderr, "Version: %s\n\n", BCF_VERSION);
- fprintf(stderr, "Usage: bcftools <command> <arguments>\n\n");
- fprintf(stderr, "Command: view print, extract, convert and call SNPs from BCF\n");
- fprintf(stderr, " index index BCF\n");
- fprintf(stderr, " cat concatenate BCFs\n");
- fprintf(stderr, " ld compute all-pair r^2\n");
- fprintf(stderr, " ldpair compute r^2 between requested pairs\n");
- fprintf(stderr, "\n");
- return 1;
- }
- if (strcmp(argv[1], "view") == 0) return bcfview(argc-1, argv+1);
- else if (strcmp(argv[1], "index") == 0) return bcf_main_index(argc-1, argv+1);
- else if (strcmp(argv[1], "ld") == 0) return bcf_main_ld(argc-1, argv+1);
- else if (strcmp(argv[1], "ldpair") == 0) return bcf_main_ldpair(argc-1, argv+1);
- else if (strcmp(argv[1], "cat") == 0) return bcf_cat(argc-2, argv+2); // cat is different ...
- else {
- fprintf(stderr, "[main] Unrecognized command.\n");
- return 1;
- }
- return 0;
-}
diff --git a/samtools-0.1.19/bcftools/mut.c b/samtools-0.1.19/bcftools/mut.c
deleted file mode 100644
index 15ef265..0000000
--- a/samtools-0.1.19/bcftools/mut.c
+++ /dev/null
@@ -1,127 +0,0 @@
-#include <stdlib.h>
-#include <stdint.h>
-#include "bcf.h"
-
-#define MAX_GENO 359
-
-int8_t seq_bitcnt[] = { 4, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 };
-char *seq_nt16rev = "XACMGRSVTWYHKDBN";
-
-uint32_t *bcf_trio_prep(int is_x, int is_son)
-{
- int i, j, k, n, map[10];
- uint32_t *ret;
- ret = calloc(MAX_GENO, 4);
- for (i = 0, k = 0; i < 4; ++i)
- for (j = i; j < 4; ++j)
- map[k++] = 1<<i|1<<j;
- for (i = 0, n = 1; i < 10; ++i) { // father
- if (is_x && seq_bitcnt[map[i]] != 1) continue;
- if (is_x && is_son) {
- for (j = 0; j < 10; ++j) // mother
- for (k = 0; k < 10; ++k) // child
- if (seq_bitcnt[map[k]] == 1 && (map[j]&map[k]))
- ret[n++] = j<<16 | i<<8 | k;
- } else {
- for (j = 0; j < 10; ++j) // mother
- for (k = 0; k < 10; ++k) // child
- if ((map[i]&map[k]) && (map[j]&map[k]) && ((map[i]|map[j])&map[k]) == map[k])
- ret[n++] = j<<16 | i<<8 | k;
- }
- }
- ret[0] = n - 1;
- return ret;
-}
-
-
-int bcf_trio_call(const uint32_t *prep, const bcf1_t *b, int *llr, int64_t *gt)
-{
- int i, j, k;
- const bcf_ginfo_t *PL;
- uint8_t *gl10;
- int map[10];
- if (b->n_smpl != 3) return -1; // not a trio
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) break;
- if (i == b->n_gi) return -1; // no PL
- gl10 = alloca(10 * b->n_smpl);
- if (bcf_gl10(b, gl10) < 0) {
- if (bcf_gl10_indel(b, gl10) < 0) return -1;
- }
- PL = b->gi + i;
- for (i = 0, k = 0; i < 4; ++i)
- for (j = i; j < 4; ++j)
- map[k++] = seq_nt16rev[1<<i|1<<j];
- for (j = 0; j < 3; ++j) // check if ref hom is the most probable in all members
- if (((uint8_t*)PL->data)[j * PL->len] != 0) break;
- if (j < 3) { // we need to go through the complex procedure
- uint8_t *g[3];
- int minc = 1<<30, minc_j = -1, minf = 0, gtf = 0, gtc = 0;
- g[0] = gl10;
- g[1] = gl10 + 10;
- g[2] = gl10 + 20;
- for (j = 1; j <= (int)prep[0]; ++j) { // compute LK with constraint
- int sum = g[0][prep[j]&0xff] + g[1][prep[j]>>8&0xff] + g[2][prep[j]>>16&0xff];
- if (sum < minc) minc = sum, minc_j = j;
- }
- gtc |= map[prep[minc_j]&0xff]; gtc |= map[prep[minc_j]>>8&0xff]<<8; gtc |= map[prep[minc_j]>>16]<<16;
- for (j = 0; j < 3; ++j) { // compute LK without constraint
- int min = 1<<30, min_k = -1;
- for (k = 0; k < 10; ++k)
- if (g[j][k] < min) min = g[j][k], min_k = k;
- gtf |= map[min_k]<<(j*8);
- minf += min;
- }
- *llr = minc - minf; *gt = (int64_t)gtc<<32 | gtf;
- } else *llr = 0, *gt = -1;
- return 0;
-}
-
-int bcf_pair_call(const bcf1_t *b)
-{
- int i, j, k;
- const bcf_ginfo_t *PL;
- if (b->n_smpl != 2) return -1; // not a pair
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) break;
- if (i == b->n_gi) return -1; // no PL
- PL = b->gi + i;
- for (j = 0; j < 2; ++j) // check if ref hom is the most probable in all members
- if (((uint8_t*)PL->data)[j * PL->len] != 0) break;
- if (j < 2) { // we need to go through the complex procedure
- uint8_t *g[2];
- int minc = 1<<30, minf = 0;
- g[0] = PL->data;
- g[1] = (uint8_t*)PL->data + PL->len;
- for (j = 0; j < PL->len; ++j) // compute LK with constraint
- minc = minc < g[0][j] + g[1][j]? minc : g[0][j] + g[1][j];
- for (j = 0; j < 2; ++j) { // compute LK without constraint
- int min = 1<<30;
- for (k = 0; k < PL->len; ++k)
- min = min < g[j][k]? min : g[j][k];
- minf += min;
- }
- return minc - minf;
- } else return 0;
-}
-
-int bcf_min_diff(const bcf1_t *b)
-{
- int i, min = 1<<30;
- const bcf_ginfo_t *PL;
- for (i = 0; i < b->n_gi; ++i)
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) break;
- if (i == b->n_gi) return -1; // no PL
- PL = b->gi + i;
- for (i = 0; i < b->n_smpl; ++i) {
- int m1, m2, j;
- const uint8_t *p = (uint8_t*)PL->data;
- m1 = m2 = 1<<30;
- for (j = 0; j < PL->len; ++j) {
- if ((int)p[j] < m1) m2 = m1, m1 = p[j];
- else if ((int)p[j] < m2) m2 = p[j];
- }
- min = min < m2 - m1? min : m2 - m1;
- }
- return min;
-}
diff --git a/samtools-0.1.19/bcftools/prob1.c b/samtools-0.1.19/bcftools/prob1.c
deleted file mode 100644
index 3539ee3..0000000
--- a/samtools-0.1.19/bcftools/prob1.c
+++ /dev/null
@@ -1,988 +0,0 @@
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <errno.h>
-#include <assert.h>
-#include <limits.h>
-#include <zlib.h>
-#include "prob1.h"
-#include "kstring.h"
-
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-#define MC_MAX_EM_ITER 16
-#define MC_EM_EPS 1e-5
-#define MC_DEF_INDEL 0.15
-
-gzFile bcf_p1_fp_lk;
-
-unsigned char seq_nt4_table[256] = {
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 /*'-'*/, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
-};
-
-struct __bcf_p1aux_t {
- int n, M, n1, is_indel;
- uint8_t *ploidy; // haploid or diploid ONLY
- double *q2p, *pdg; // pdg -> P(D|g)
- double *phi, *phi_indel;
- double *z, *zswap; // aux for afs
- double *z1, *z2, *phi1, *phi2; // only calculated when n1 is set
- double **hg; // hypergeometric distribution
- double *lf; // log factorial
- double t, t1, t2;
- double *afs, *afs1; // afs: accumulative AFS; afs1: site posterior distribution
- const uint8_t *PL; // point to PL
- int PL_len;
-};
-
-void bcf_p1_indel_prior(bcf_p1aux_t *ma, double x)
-{
- int i;
- for (i = 0; i < ma->M; ++i)
- ma->phi_indel[i] = ma->phi[i] * x;
- ma->phi_indel[ma->M] = 1. - ma->phi[ma->M] * x;
-}
-
-static void init_prior(int type, double theta, int M, double *phi)
-{
- int i;
- if (type == MC_PTYPE_COND2) {
- for (i = 0; i <= M; ++i)
- phi[i] = 2. * (i + 1) / (M + 1) / (M + 2);
- } else if (type == MC_PTYPE_FLAT) {
- for (i = 0; i <= M; ++i)
- phi[i] = 1. / (M + 1);
- } else {
- double sum;
- for (i = 0, sum = 0.; i < M; ++i)
- sum += (phi[i] = theta / (M - i));
- phi[M] = 1. - sum;
- }
-}
-
-void bcf_p1_init_prior(bcf_p1aux_t *ma, int type, double theta)
-{
- init_prior(type, theta, ma->M, ma->phi);
- bcf_p1_indel_prior(ma, MC_DEF_INDEL);
-}
-
-void bcf_p1_init_subprior(bcf_p1aux_t *ma, int type, double theta)
-{
- if (ma->n1 <= 0 || ma->n1 >= ma->M) return;
- init_prior(type, theta, 2*ma->n1, ma->phi1);
- init_prior(type, theta, 2*(ma->n - ma->n1), ma->phi2);
-}
-
-int bcf_p1_read_prior(bcf_p1aux_t *ma, const char *fn)
-{
- gzFile fp;
- kstring_t s;
- kstream_t *ks;
- long double sum;
- int dret, k;
- memset(&s, 0, sizeof(kstring_t));
- fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
- ks = ks_init(fp);
- memset(ma->phi, 0, sizeof(double) * (ma->M + 1));
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0) {
- if (strstr(s.s, "[afs] ") == s.s) {
- char *p = s.s + 6;
- for (k = 0; k <= ma->M; ++k) {
- int x;
- double y;
- x = strtol(p, &p, 10);
- if (x != k && (errno == EINVAL || errno == ERANGE)) return -1;
- ++p;
- y = strtod(p, &p);
- if (y == 0. && (errno == EINVAL || errno == ERANGE)) return -1;
- ma->phi[ma->M - k] += y;
- }
- }
- }
- ks_destroy(ks);
- gzclose(fp);
- free(s.s);
- for (sum = 0., k = 0; k <= ma->M; ++k) sum += ma->phi[k];
- fprintf(stderr, "[prior]");
- for (k = 0; k <= ma->M; ++k) ma->phi[k] /= sum;
- for (k = 0; k <= ma->M; ++k) fprintf(stderr, " %d:%.3lg", k, ma->phi[ma->M - k]);
- fputc('\n', stderr);
- for (sum = 0., k = 1; k < ma->M; ++k) sum += ma->phi[ma->M - k] * (2.* k * (ma->M - k) / ma->M / (ma->M - 1));
- fprintf(stderr, "[%s] heterozygosity=%lf, ", __func__, (double)sum);
- for (sum = 0., k = 1; k <= ma->M; ++k) sum += k * ma->phi[ma->M - k] / ma->M;
- fprintf(stderr, "theta=%lf\n", (double)sum);
- bcf_p1_indel_prior(ma, MC_DEF_INDEL);
- return 0;
-}
-
-bcf_p1aux_t *bcf_p1_init(int n, uint8_t *ploidy)
-{
- bcf_p1aux_t *ma;
- int i;
- ma = calloc(1, sizeof(bcf_p1aux_t));
- ma->n1 = -1;
- ma->n = n; ma->M = 2 * n;
- if (ploidy) {
- ma->ploidy = malloc(n);
- memcpy(ma->ploidy, ploidy, n);
- for (i = 0, ma->M = 0; i < n; ++i) ma->M += ploidy[i];
- if (ma->M == 2 * n) {
- free(ma->ploidy);
- ma->ploidy = 0;
- }
- }
- ma->q2p = calloc(256, sizeof(double));
- ma->pdg = calloc(3 * ma->n, sizeof(double));
- ma->phi = calloc(ma->M + 1, sizeof(double));
- ma->phi_indel = calloc(ma->M + 1, sizeof(double));
- ma->phi1 = calloc(ma->M + 1, sizeof(double));
- ma->phi2 = calloc(ma->M + 1, sizeof(double));
- ma->z = calloc(ma->M + 1, sizeof(double));
- ma->zswap = calloc(ma->M + 1, sizeof(double));
- ma->z1 = calloc(ma->M + 1, sizeof(double)); // actually we do not need this large
- ma->z2 = calloc(ma->M + 1, sizeof(double));
- ma->afs = calloc(ma->M + 1, sizeof(double));
- ma->afs1 = calloc(ma->M + 1, sizeof(double));
- ma->lf = calloc(ma->M + 1, sizeof(double));
- for (i = 0; i < 256; ++i)
- ma->q2p[i] = pow(10., -i / 10.);
- for (i = 0; i <= ma->M; ++i) ma->lf[i] = lgamma(i + 1);
- bcf_p1_init_prior(ma, MC_PTYPE_FULL, 1e-3); // the simplest prior
- return ma;
-}
-
-int bcf_p1_get_M(bcf_p1aux_t *b) { return b->M; }
-
-int bcf_p1_set_n1(bcf_p1aux_t *b, int n1)
-{
- if (n1 == 0 || n1 >= b->n) return -1;
- if (b->M != b->n * 2) {
- fprintf(stderr, "[%s] unable to set `n1' when there are haploid samples.\n", __func__);
- return -1;
- }
- b->n1 = n1;
- return 0;
-}
-
-void bcf_p1_set_ploidy(bcf1_t *b, bcf_p1aux_t *ma)
-{
- // bcf_p1aux_t fields are not visible outside of prob1.c, hence this wrapper.
- // Ideally, this should set ploidy per site to allow pseudo-autosomal regions
- b->ploidy = ma->ploidy;
-}
-
-void bcf_p1_destroy(bcf_p1aux_t *ma)
-{
- if (ma) {
- int k;
- free(ma->lf);
- if (ma->hg && ma->n1 > 0) {
- for (k = 0; k <= 2*ma->n1; ++k) free(ma->hg[k]);
- free(ma->hg);
- }
- free(ma->ploidy); free(ma->q2p); free(ma->pdg);
- free(ma->phi); free(ma->phi_indel); free(ma->phi1); free(ma->phi2);
- free(ma->z); free(ma->zswap); free(ma->z1); free(ma->z2);
- free(ma->afs); free(ma->afs1);
- free(ma);
- }
-}
-
-extern double kf_gammap(double s, double z);
-int test16(bcf1_t *b, anno16_t *a);
-
-// Wigginton 2005, PMID: 15789306
-// written by Jan Wigginton
-double calc_hwe(int obs_hom1, int obs_hom2, int obs_hets)
-{
- if (obs_hom1 + obs_hom2 + obs_hets == 0 ) return 1;
-
- assert(obs_hom1 >= 0 && obs_hom2 >= 0 && obs_hets >= 0);
-
- int obs_homc = obs_hom1 < obs_hom2 ? obs_hom2 : obs_hom1;
- int obs_homr = obs_hom1 < obs_hom2 ? obs_hom1 : obs_hom2;
-
- int rare_copies = 2 * obs_homr + obs_hets;
- int genotypes = obs_hets + obs_homc + obs_homr;
-
- double *het_probs = (double*) calloc(rare_copies+1, sizeof(double));
-
- /* start at midpoint */
- int mid = rare_copies * (2 * genotypes - rare_copies) / (2 * genotypes);
-
- /* check to ensure that midpoint and rare alleles have same parity */
- if ((rare_copies & 1) ^ (mid & 1)) mid++;
-
- int curr_hets = mid;
- int curr_homr = (rare_copies - mid) / 2;
- int curr_homc = genotypes - curr_hets - curr_homr;
-
- het_probs[mid] = 1.0;
- double sum = het_probs[mid];
- for (curr_hets = mid; curr_hets > 1; curr_hets -= 2)
- {
- het_probs[curr_hets - 2] = het_probs[curr_hets] * curr_hets * (curr_hets - 1.0) / (4.0 * (curr_homr + 1.0) * (curr_homc + 1.0));
- sum += het_probs[curr_hets - 2];
-
- /* 2 fewer heterozygotes for next iteration -> add one rare, one common homozygote */
- curr_homr++;
- curr_homc++;
- }
-
- curr_hets = mid;
- curr_homr = (rare_copies - mid) / 2;
- curr_homc = genotypes - curr_hets - curr_homr;
- for (curr_hets = mid; curr_hets <= rare_copies - 2; curr_hets += 2)
- {
- het_probs[curr_hets + 2] = het_probs[curr_hets] * 4.0 * curr_homr * curr_homc /((curr_hets + 2.0) * (curr_hets + 1.0));
- sum += het_probs[curr_hets + 2];
-
- /* add 2 heterozygotes for next iteration -> subtract one rare, one common homozygote */
- curr_homr--;
- curr_homc--;
- }
- int i;
- for (i = 0; i <= rare_copies; i++) het_probs[i] /= sum;
-
- /* p-value calculation for p_hwe */
- double p_hwe = 0.0;
- for (i = 0; i <= rare_copies; i++)
- {
- if (het_probs[i] > het_probs[obs_hets])
- continue;
- p_hwe += het_probs[i];
- }
-
- p_hwe = p_hwe > 1.0 ? 1.0 : p_hwe;
- free(het_probs);
- return p_hwe;
-
-}
-
-
-static void _bcf1_set_ref(bcf1_t *b, int idp)
-{
- kstring_t s;
- int old_n_gi = b->n_gi;
- s.m = b->m_str; s.l = b->l_str - 1; s.s = b->str;
- kputs(":GT", &s); kputc('\0', &s);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- bcf_sync(b);
-
- // Call GTs
- int isample, an = 0;
- for (isample = 0; isample < b->n_smpl; isample++)
- {
- if ( idp>=0 && ((uint16_t*)b->gi[idp].data)[isample]==0 )
- ((uint8_t*)b->gi[old_n_gi].data)[isample] = 1<<7;
- else
- {
- ((uint8_t*)b->gi[old_n_gi].data)[isample] = 0;
- an += b->ploidy ? b->ploidy[isample] : 2;
- }
- }
- bcf_fit_alt(b,1);
- b->qual = 999;
-
- // Prepare BCF for output: ref, alt, filter, info, format
- memset(&s, 0, sizeof(kstring_t)); kputc('\0', &s);
- kputs(b->ref, &s); kputc('\0', &s);
- kputs(b->alt, &s); kputc('\0', &s); kputc('\0', &s);
- {
- ksprintf(&s, "AN=%d;", an);
- kputs(b->info, &s);
- anno16_t a;
- int has_I16 = test16(b, &a) >= 0? 1 : 0;
- if (has_I16 )
- {
- if ( a.is_tested) ksprintf(&s, ";PV4=%.2g,%.2g,%.2g,%.2g", a.p[0], a.p[1], a.p[2], a.p[3]);
- ksprintf(&s, ";DP4=%d,%d,%d,%d;MQ=%d", a.d[0], a.d[1], a.d[2], a.d[3], a.mq);
- }
- kputc('\0', &s);
- rm_info(&s, "I16=");
- rm_info(&s, "QS=");
- }
- kputs(b->fmt, &s); kputc('\0', &s);
- free(b->str);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- bcf_sync(b);
-}
-
-int call_multiallelic_gt(bcf1_t *b, bcf_p1aux_t *ma, double threshold, int var_only)
-{
- int nals = 1;
- char *p;
- for (p=b->alt; *p; p++)
- {
- if ( *p=='X' || p[0]=='.' ) break;
- if ( p[0]==',' ) nals++;
- }
- if ( b->alt[0] && !*p ) nals++;
-
- if ( nals>4 )
- {
- if ( *b->ref=='N' ) return 0;
- fprintf(stderr,"Not ready for this, more than 4 alleles at %d: %s, %s\n", b->pos+1, b->ref,b->alt);
- exit(1);
- }
-
- // find PL, DV and DP FORMAT indexes
- uint8_t *pl = NULL;
- int i, npl = 0, idp = -1, idv = -1;
- for (i = 0; i < b->n_gi; ++i)
- {
- if (b->gi[i].fmt == bcf_str2int("PL", 2))
- {
- pl = (uint8_t*)b->gi[i].data;
- npl = b->gi[i].len;
- }
- else if (b->gi[i].fmt == bcf_str2int("DP", 2)) idp=i;
- else if (b->gi[i].fmt == bcf_str2int("DV", 2)) idv=i;
- }
- if ( nals==1 )
- {
- if ( !var_only ) _bcf1_set_ref(b, idp);
- return 1;
- }
- if ( !pl ) return -1;
-
- assert(ma->q2p[0] == 1);
-
- // Init P(D|G)
- int npdg = nals*(nals+1)/2;
- double *pdg,*_pdg;
- _pdg = pdg = malloc(sizeof(double)*ma->n*npdg);
- for (i=0; i<ma->n; i++)
- {
- int j;
- double sum = 0;
- for (j=0; j<npdg; j++)
- {
- //_pdg[j] = pow(10,-0.1*pl[j]);
- _pdg[j] = ma->q2p[pl[j]];
- sum += _pdg[j];
- }
- if ( sum )
- for (j=0; j<npdg; j++) _pdg[j] /= sum;
- _pdg += npdg;
- pl += npl;
- }
-
- if ((p = strstr(b->info, "QS=")) == 0) { fprintf(stderr,"INFO/QS is required with -m, exiting\n"); exit(1); }
- double qsum[4];
- if ( sscanf(p+3,"%lf,%lf,%lf,%lf",&qsum[0],&qsum[1],&qsum[2],&qsum[3])!=4 ) { fprintf(stderr,"Could not parse %s\n",p); exit(1); }
-
-
- // Calculate the most likely combination of alleles, remembering the most and second most likely set
- int ia,ib,ic, max_als=0, max_als2=0;
- double ref_lk = 0, max_lk = INT_MIN, max_lk2 = INT_MIN, lk_sum = INT_MIN, lk_sums[3];
- for (ia=0; ia<nals; ia++)
- {
- double lk_tot = 0;
- int iaa = (ia+1)*(ia+2)/2-1;
- int isample;
- for (isample=0; isample<ma->n; isample++)
- {
- double *p = pdg + isample*npdg;
- // assert( log(p[iaa]) <= 0 );
- lk_tot += log(p[iaa]);
- }
- if ( ia==0 ) ref_lk = lk_tot;
- if ( max_lk<lk_tot ) { max_lk2 = max_lk; max_als2 = max_als; max_lk = lk_tot; max_als = 1<<ia; }
- else if ( max_lk2<lk_tot ) { max_lk2 = lk_tot; max_als2 = 1<<ia; }
- lk_sum = lk_tot>lk_sum ? lk_tot + log(1+exp(lk_sum-lk_tot)) : lk_sum + log(1+exp(lk_tot-lk_sum));
- }
- lk_sums[0] = lk_sum;
- if ( nals>1 )
- {
- for (ia=0; ia<nals; ia++)
- {
- if ( qsum[ia]==0 ) continue;
- int iaa = (ia+1)*(ia+2)/2-1;
- for (ib=0; ib<ia; ib++)
- {
- if ( qsum[ib]==0 ) continue;
- double lk_tot = 0;
- double fa = qsum[ia]/(qsum[ia]+qsum[ib]);
- double fb = qsum[ib]/(qsum[ia]+qsum[ib]);
- double fab = 2*fa*fb; fa *= fa; fb *= fb;
- int isample, ibb = (ib+1)*(ib+2)/2-1, iab = iaa - ia + ib;
- for (isample=0; isample<ma->n; isample++)
- {
- double *p = pdg + isample*npdg;
- //assert( log(fa*p[iaa] + fb*p[ibb] + fab*p[iab]) <= 0 );
- if ( b->ploidy && b->ploidy[isample]==1 )
- lk_tot += log(fa*p[iaa] + fb*p[ibb]);
- else
- lk_tot += log(fa*p[iaa] + fb*p[ibb] + fab*p[iab]);
- }
- if ( max_lk<lk_tot ) { max_lk2 = max_lk; max_als2 = max_als; max_lk = lk_tot; max_als = 1<<ia|1<<ib; }
- else if ( max_lk2<lk_tot ) { max_lk2 = lk_tot; max_als2 = 1<<ia|1<<ib; }
- lk_sum = lk_tot>lk_sum ? lk_tot + log(1+exp(lk_sum-lk_tot)) : lk_sum + log(1+exp(lk_tot-lk_sum));
- }
- }
- lk_sums[1] = lk_sum;
- }
- if ( nals>2 )
- {
- for (ia=0; ia<nals; ia++)
- {
- if ( qsum[ia]==0 ) continue;
- int iaa = (ia+1)*(ia+2)/2-1;
- for (ib=0; ib<ia; ib++)
- {
- if ( qsum[ib]==0 ) continue;
- int ibb = (ib+1)*(ib+2)/2-1;
- int iab = iaa - ia + ib;
- for (ic=0; ic<ib; ic++)
- {
- if ( qsum[ic]==0 ) continue;
- double lk_tot = 0;
- double fa = qsum[ia]/(qsum[ia]+qsum[ib]+qsum[ic]);
- double fb = qsum[ib]/(qsum[ia]+qsum[ib]+qsum[ic]);
- double fc = qsum[ic]/(qsum[ia]+qsum[ib]+qsum[ic]);
- double fab = 2*fa*fb, fac = 2*fa*fc, fbc = 2*fb*fc; fa *= fa; fb *= fb; fc *= fc;
- int isample, icc = (ic+1)*(ic+2)/2-1;
- int iac = iaa - ia + ic, ibc = ibb - ib + ic;
- for (isample=0; isample<ma->n; isample++)
- {
- double *p = pdg + isample*npdg;
- //assert( log(fa*p[iaa] + fb*p[ibb] + fc*p[icc] + fab*p[iab] + fac*p[iac] + fbc*p[ibc]) <= 0 );
- if ( b->ploidy && b->ploidy[isample]==1 )
- lk_tot += log(fa*p[iaa] + fb*p[ibb] + fc*p[icc]);
- else
- lk_tot += log(fa*p[iaa] + fb*p[ibb] + fc*p[icc] + fab*p[iab] + fac*p[iac] + fbc*p[ibc]);
- }
- if ( max_lk<lk_tot ) { max_lk2 = max_lk; max_als2 = max_als; max_lk = lk_tot; max_als = 1<<ia|1<<ib|1<<ic; }
- else if ( max_lk2<lk_tot ) { max_lk2 = lk_tot; max_als2 = 1<<ia|1<<ib|1<<ic; }
- lk_sum = lk_tot>lk_sum ? lk_tot + log(1+exp(lk_sum-lk_tot)) : lk_sum + log(1+exp(lk_tot-lk_sum));
- }
- }
- }
- lk_sums[2] = lk_sum;
- }
-
- // Should we add another allele, does it increase the likelihood significantly?
- int n1=0, n2=0;
- for (i=0; i<nals; i++) if ( max_als&1<<i) n1++;
- for (i=0; i<nals; i++) if ( max_als2&1<<i) n2++;
- if ( n2<n1 && kf_gammap(1,2.0*(max_lk-max_lk2))<threshold )
- {
- // the threshold not exceeded, use the second most likely set with fewer alleles
- max_lk = max_lk2;
- max_als = max_als2;
- n1 = n2;
- }
- lk_sum = lk_sums[n1-1];
-
- // Get the BCF record ready for GT and GQ
- kstring_t s;
- int old_n_gi = b->n_gi;
- s.m = b->m_str; s.l = b->l_str - 1; s.s = b->str;
- kputs(":GT:GQ", &s); kputc('\0', &s);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- bcf_sync(b);
-
- // Call GTs
- int isample, gts=0, ac[4] = {0,0,0,0};
- int nRR = 0, nAA = 0, nRA = 0, max_dv = 0;
- for (isample = 0; isample < b->n_smpl; isample++)
- {
- int ploidy = b->ploidy ? b->ploidy[isample] : 2;
- double *p = pdg + isample*npdg;
- int ia, als = 0;
- double lk = 0, lk_s = 0;
- for (ia=0; ia<nals; ia++)
- {
- if ( !(max_als&1<<ia) ) continue;
- int iaa = (ia+1)*(ia+2)/2-1;
- double _lk = p[iaa]*qsum[ia]*qsum[ia];
- if ( _lk > lk ) { lk = _lk; als = ia<<3 | ia; }
- lk_s += _lk;
- }
- if ( ploidy==2 )
- {
- for (ia=0; ia<nals; ia++)
- {
- if ( !(max_als&1<<ia) ) continue;
- int iaa = (ia+1)*(ia+2)/2-1;
- for (ib=0; ib<ia; ib++)
- {
- if ( !(max_als&1<<ib) ) continue;
- int iab = iaa - ia + ib;
- double _lk = 2*qsum[ia]*qsum[ib]*p[iab];
- if ( _lk > lk ) { lk = _lk; als = ib<<3 | ia; }
- lk_s += _lk;
- }
- }
- }
- lk = -log(1-lk/lk_s)/0.2302585;
- int dp = 0;
- if ( idp>=0 && (dp=((uint16_t*)b->gi[idp].data)[isample])==0 )
- {
- // no coverage
- ((uint8_t*)b->gi[old_n_gi].data)[isample] = 1<<7;
- ((uint8_t*)b->gi[old_n_gi+1].data)[isample] = 0;
- continue;
- }
- if ( lk>99 ) lk = 99;
- ((uint8_t*)b->gi[old_n_gi].data)[isample] = als;
- ((uint8_t*)b->gi[old_n_gi+1].data)[isample] = (int)lk;
-
- // For MDV annotation
- int dv;
- if ( als && idv>=0 && (dv=((uint16_t*)b->gi[idv].data)[isample]) )
- {
- if ( max_dv < dv ) max_dv = dv;
- }
-
- // For HWE annotation; multiple ALT alleles treated as one
- if ( !als ) nRR++;
- else if ( !(als>>3&7) || !(als&7) ) nRA++;
- else nAA++;
-
- gts |= 1<<(als>>3&7) | 1<<(als&7);
- ac[ als>>3&7 ]++;
- ac[ als&7 ]++;
- }
- free(pdg);
- bcf_fit_alt(b,max_als);
-
- // The VCF spec is ambiguous about QUAL: is it the probability of anything else
- // (that is QUAL(non-ref) = P(ref)+P(any non-ref other than ALT)) or is it
- // QUAL(non-ref)=P(ref) and QUAL(ref)=1-P(ref)? Assuming the latter.
- b->qual = gts>1 ? -4.343*(ref_lk - lk_sum) : -4.343*log(1-exp(ref_lk - lk_sum));
- if ( b->qual>999 ) b->qual = 999;
-
- // Prepare BCF for output: ref, alt, filter, info, format
- memset(&s, 0, sizeof(kstring_t)); kputc('\0', &s);
- kputs(b->ref, &s); kputc('\0', &s);
- kputs(b->alt, &s); kputc('\0', &s); kputc('\0', &s);
- {
- int an=0, nalts=0;
- for (i=0; i<nals; i++)
- {
- an += ac[i];
- if ( i>0 && ac[i] ) nalts++;
- }
- ksprintf(&s, "AN=%d;", an);
- if ( nalts )
- {
- kputs("AC=", &s);
- for (i=1; i<nals; i++)
- {
- if ( !(gts&1<<i) ) continue;
- nalts--;
- ksprintf(&s,"%d", ac[i]);
- if ( nalts>0 ) kputc(',', &s);
- }
- kputc(';', &s);
- }
- kputs(b->info, &s);
- anno16_t a;
- int has_I16 = test16(b, &a) >= 0? 1 : 0;
- if (has_I16 )
- {
- if ( a.is_tested) ksprintf(&s, ";PV4=%.2g,%.2g,%.2g,%.2g", a.p[0], a.p[1], a.p[2], a.p[3]);
- ksprintf(&s, ";DP4=%d,%d,%d,%d;MQ=%d", a.d[0], a.d[1], a.d[2], a.d[3], a.mq);
- ksprintf(&s, ";QBD=%e", b->qual/(a.d[0] + a.d[1] + a.d[2] + a.d[3]));
- if ( max_dv ) ksprintf(&s, ";MDV=%d", max_dv);
- }
- if ( nAA+nRA )
- {
- double hwe = calc_hwe(nAA, nRR, nRA);
- ksprintf(&s, ";HWE=%e", hwe);
- }
- kputc('\0', &s);
- rm_info(&s, "I16=");
- rm_info(&s, "QS=");
- }
- kputs(b->fmt, &s); kputc('\0', &s);
- free(b->str);
- b->m_str = s.m; b->l_str = s.l; b->str = s.s;
- bcf_sync(b);
-
- return gts;
-}
-
-static int cal_pdg(const bcf1_t *b, bcf_p1aux_t *ma)
-{
- int i, j;
- long *p, tmp;
- p = alloca(b->n_alleles * sizeof(long));
- memset(p, 0, sizeof(long) * b->n_alleles);
- for (j = 0; j < ma->n; ++j) {
- const uint8_t *pi = ma->PL + j * ma->PL_len;
- double *pdg = ma->pdg + j * 3;
- pdg[0] = ma->q2p[pi[2]]; pdg[1] = ma->q2p[pi[1]]; pdg[2] = ma->q2p[pi[0]];
- for (i = 0; i < b->n_alleles; ++i)
- p[i] += (int)pi[(i+1)*(i+2)/2-1];
- }
- for (i = 0; i < b->n_alleles; ++i) p[i] = p[i]<<4 | i;
- for (i = 1; i < b->n_alleles; ++i) // insertion sort
- for (j = i; j > 0 && p[j] < p[j-1]; --j)
- tmp = p[j], p[j] = p[j-1], p[j-1] = tmp;
- for (i = b->n_alleles - 1; i >= 0; --i)
- if ((p[i]&0xf) == 0) break;
- return i;
-}
-
-
-int bcf_p1_call_gt(const bcf_p1aux_t *ma, double f0, int k)
-{
- double sum, g[3];
- double max, f3[3], *pdg = ma->pdg + k * 3;
- int q, i, max_i, ploidy;
- ploidy = ma->ploidy? ma->ploidy[k] : 2;
- if (ploidy == 2) {
- f3[0] = (1.-f0)*(1.-f0); f3[1] = 2.*f0*(1.-f0); f3[2] = f0*f0;
- } else {
- f3[0] = 1. - f0; f3[1] = 0; f3[2] = f0;
- }
- for (i = 0, sum = 0.; i < 3; ++i)
- sum += (g[i] = pdg[i] * f3[i]);
- for (i = 0, max = -1., max_i = 0; i < 3; ++i) {
- g[i] /= sum;
- if (g[i] > max) max = g[i], max_i = i;
- }
- max = 1. - max;
- if (max < 1e-308) max = 1e-308;
- q = (int)(-4.343 * log(max) + .499);
- if (q > 99) q = 99;
- return q<<2|max_i;
-}
-
-#define TINY 1e-20
-
-static void mc_cal_y_core(bcf_p1aux_t *ma, int beg)
-{
- double *z[2], *tmp, *pdg;
- int _j, last_min, last_max;
- assert(beg == 0 || ma->M == ma->n*2);
- z[0] = ma->z;
- z[1] = ma->zswap;
- pdg = ma->pdg;
- memset(z[0], 0, sizeof(double) * (ma->M + 1));
- memset(z[1], 0, sizeof(double) * (ma->M + 1));
- z[0][0] = 1.;
- last_min = last_max = 0;
- ma->t = 0.;
- if (ma->M == ma->n * 2) {
- int M = 0;
- for (_j = beg; _j < ma->n; ++_j) {
- int k, j = _j - beg, _min = last_min, _max = last_max, M0;
- double p[3], sum;
- M0 = M; M += 2;
- pdg = ma->pdg + _j * 3;
- p[0] = pdg[0]; p[1] = 2. * pdg[1]; p[2] = pdg[2];
- for (; _min < _max && z[0][_min] < TINY; ++_min) z[0][_min] = z[1][_min] = 0.;
- for (; _max > _min && z[0][_max] < TINY; --_max) z[0][_max] = z[1][_max] = 0.;
- _max += 2;
- if (_min == 0) k = 0, z[1][k] = (M0-k+1) * (M0-k+2) * p[0] * z[0][k];
- if (_min <= 1) k = 1, z[1][k] = (M0-k+1) * (M0-k+2) * p[0] * z[0][k] + k*(M0-k+2) * p[1] * z[0][k-1];
- for (k = _min < 2? 2 : _min; k <= _max; ++k)
- z[1][k] = (M0-k+1)*(M0-k+2) * p[0] * z[0][k] + k*(M0-k+2) * p[1] * z[0][k-1] + k*(k-1)* p[2] * z[0][k-2];
- for (k = _min, sum = 0.; k <= _max; ++k) sum += z[1][k];
- ma->t += log(sum / (M * (M - 1.)));
- for (k = _min; k <= _max; ++k) z[1][k] /= sum;
- if (_min >= 1) z[1][_min-1] = 0.;
- if (_min >= 2) z[1][_min-2] = 0.;
- if (j < ma->n - 1) z[1][_max+1] = z[1][_max+2] = 0.;
- if (_j == ma->n1 - 1) { // set pop1; ma->n1==-1 when unset
- ma->t1 = ma->t;
- memcpy(ma->z1, z[1], sizeof(double) * (ma->n1 * 2 + 1));
- }
- tmp = z[0]; z[0] = z[1]; z[1] = tmp;
- last_min = _min; last_max = _max;
- }
- //for (_j = 0; _j < last_min; ++_j) z[0][_j] = 0.; // TODO: are these necessary?
- //for (_j = last_max + 1; _j < ma->M; ++_j) z[0][_j] = 0.;
- } else { // this block is very similar to the block above; these two might be merged in future
- int j, M = 0;
- for (j = 0; j < ma->n; ++j) {
- int k, M0, _min = last_min, _max = last_max;
- double p[3], sum;
- pdg = ma->pdg + j * 3;
- for (; _min < _max && z[0][_min] < TINY; ++_min) z[0][_min] = z[1][_min] = 0.;
- for (; _max > _min && z[0][_max] < TINY; --_max) z[0][_max] = z[1][_max] = 0.;
- M0 = M;
- M += ma->ploidy[j];
- if (ma->ploidy[j] == 1) {
- p[0] = pdg[0]; p[1] = pdg[2];
- _max++;
- if (_min == 0) k = 0, z[1][k] = (M0+1-k) * p[0] * z[0][k];
- for (k = _min < 1? 1 : _min; k <= _max; ++k)
- z[1][k] = (M0+1-k) * p[0] * z[0][k] + k * p[1] * z[0][k-1];
- for (k = _min, sum = 0.; k <= _max; ++k) sum += z[1][k];
- ma->t += log(sum / M);
- for (k = _min; k <= _max; ++k) z[1][k] /= sum;
- if (_min >= 1) z[1][_min-1] = 0.;
- if (j < ma->n - 1) z[1][_max+1] = 0.;
- } else if (ma->ploidy[j] == 2) {
- p[0] = pdg[0]; p[1] = 2 * pdg[1]; p[2] = pdg[2];
- _max += 2;
- if (_min == 0) k = 0, z[1][k] = (M0-k+1) * (M0-k+2) * p[0] * z[0][k];
- if (_min <= 1) k = 1, z[1][k] = (M0-k+1) * (M0-k+2) * p[0] * z[0][k] + k*(M0-k+2) * p[1] * z[0][k-1];
- for (k = _min < 2? 2 : _min; k <= _max; ++k)
- z[1][k] = (M0-k+1)*(M0-k+2) * p[0] * z[0][k] + k*(M0-k+2) * p[1] * z[0][k-1] + k*(k-1)* p[2] * z[0][k-2];
- for (k = _min, sum = 0.; k <= _max; ++k) sum += z[1][k];
- ma->t += log(sum / (M * (M - 1.)));
- for (k = _min; k <= _max; ++k) z[1][k] /= sum;
- if (_min >= 1) z[1][_min-1] = 0.;
- if (_min >= 2) z[1][_min-2] = 0.;
- if (j < ma->n - 1) z[1][_max+1] = z[1][_max+2] = 0.;
- }
- tmp = z[0]; z[0] = z[1]; z[1] = tmp;
- last_min = _min; last_max = _max;
- }
- }
- if (z[0] != ma->z) memcpy(ma->z, z[0], sizeof(double) * (ma->M + 1));
- if (bcf_p1_fp_lk)
- gzwrite(bcf_p1_fp_lk, ma->z, sizeof(double) * (ma->M + 1));
-}
-
-static void mc_cal_y(bcf_p1aux_t *ma)
-{
- if (ma->n1 > 0 && ma->n1 < ma->n && ma->M == ma->n * 2) { // NB: ma->n1 is ineffective when there are haploid samples
- int k;
- long double x;
- memset(ma->z1, 0, sizeof(double) * (2 * ma->n1 + 1));
- memset(ma->z2, 0, sizeof(double) * (2 * (ma->n - ma->n1) + 1));
- ma->t1 = ma->t2 = 0.;
- mc_cal_y_core(ma, ma->n1);
- ma->t2 = ma->t;
- memcpy(ma->z2, ma->z, sizeof(double) * (2 * (ma->n - ma->n1) + 1));
- mc_cal_y_core(ma, 0);
- // rescale z
- x = expl(ma->t - (ma->t1 + ma->t2));
- for (k = 0; k <= ma->M; ++k) ma->z[k] *= x;
- } else mc_cal_y_core(ma, 0);
-}
-
-#define CONTRAST_TINY 1e-30
-
-extern double kf_gammaq(double s, double z); // incomplete gamma function for chi^2 test
-
-static inline double chi2_test(int a, int b, int c, int d)
-{
- double x, z;
- x = (double)(a+b) * (c+d) * (b+d) * (a+c);
- if (x == 0.) return 1;
- z = a * d - b * c;
- return kf_gammaq(.5, .5 * z * z * (a+b+c+d) / x);
-}
-
-// chi2=(a+b+c+d)(ad-bc)^2/[(a+b)(c+d)(a+c)(b+d)]
-static inline double contrast2_aux(const bcf_p1aux_t *p1, double sum, int k1, int k2, double x[3])
-{
- double p = p1->phi[k1+k2] * p1->z1[k1] * p1->z2[k2] / sum * p1->hg[k1][k2];
- int n1 = p1->n1, n2 = p1->n - p1->n1;
- if (p < CONTRAST_TINY) return -1;
- if (.5*k1/n1 < .5*k2/n2) x[1] += p;
- else if (.5*k1/n1 > .5*k2/n2) x[2] += p;
- else x[0] += p;
- return p * chi2_test(k1, k2, (n1<<1) - k1, (n2<<1) - k2);
-}
-
-static double contrast2(bcf_p1aux_t *p1, double ret[3])
-{
- int k, k1, k2, k10, k20, n1, n2;
- double sum;
- // get n1 and n2
- n1 = p1->n1; n2 = p1->n - p1->n1;
- if (n1 <= 0 || n2 <= 0) return 0.;
- if (p1->hg == 0) { // initialize the hypergeometric distribution
- /* NB: the hg matrix may take a lot of memory when there are many samples. There is a way
- to avoid precomputing this matrix, but it is slower and quite intricate. The following
- computation in this block can be accelerated with a similar strategy, but perhaps this
- is not a serious concern for now. */
- double tmp = lgamma(2*(n1+n2)+1) - (lgamma(2*n1+1) + lgamma(2*n2+1));
- p1->hg = calloc(2*n1+1, sizeof(void*));
- for (k1 = 0; k1 <= 2*n1; ++k1) {
- p1->hg[k1] = calloc(2*n2+1, sizeof(double));
- for (k2 = 0; k2 <= 2*n2; ++k2)
- p1->hg[k1][k2] = exp(lgamma(k1+k2+1) + lgamma(p1->M-k1-k2+1) - (lgamma(k1+1) + lgamma(k2+1) + lgamma(2*n1-k1+1) + lgamma(2*n2-k2+1) + tmp));
- }
- }
- { // compute
- long double suml = 0;
- for (k = 0; k <= p1->M; ++k) suml += p1->phi[k] * p1->z[k];
- sum = suml;
- }
- { // get the max k1 and k2
- double max;
- int max_k;
- for (k = 0, max = 0, max_k = -1; k <= 2*n1; ++k) {
- double x = p1->phi1[k] * p1->z1[k];
- if (x > max) max = x, max_k = k;
- }
- k10 = max_k;
- for (k = 0, max = 0, max_k = -1; k <= 2*n2; ++k) {
- double x = p1->phi2[k] * p1->z2[k];
- if (x > max) max = x, max_k = k;
- }
- k20 = max_k;
- }
- { // We can do the following with one nested loop, but that is an O(N^2) thing. The following code block is much faster for large N.
- double x[3], y;
- long double z = 0., L[2];
- x[0] = x[1] = x[2] = 0; L[0] = L[1] = 0;
- for (k1 = k10; k1 >= 0; --k1) {
- for (k2 = k20; k2 >= 0; --k2) {
- if ((y = contrast2_aux(p1, sum, k1, k2, x)) < 0) break;
- else z += y;
- }
- for (k2 = k20 + 1; k2 <= 2*n2; ++k2) {
- if ((y = contrast2_aux(p1, sum, k1, k2, x)) < 0) break;
- else z += y;
- }
- }
- ret[0] = x[0]; ret[1] = x[1]; ret[2] = x[2];
- x[0] = x[1] = x[2] = 0;
- for (k1 = k10 + 1; k1 <= 2*n1; ++k1) {
- for (k2 = k20; k2 >= 0; --k2) {
- if ((y = contrast2_aux(p1, sum, k1, k2, x)) < 0) break;
- else z += y;
- }
- for (k2 = k20 + 1; k2 <= 2*n2; ++k2) {
- if ((y = contrast2_aux(p1, sum, k1, k2, x)) < 0) break;
- else z += y;
- }
- }
- ret[0] += x[0]; ret[1] += x[1]; ret[2] += x[2];
- if (ret[0] + ret[1] + ret[2] < 0.95) { // in case of bad things happened
- ret[0] = ret[1] = ret[2] = 0; L[0] = L[1] = 0;
- for (k1 = 0, z = 0.; k1 <= 2*n1; ++k1)
- for (k2 = 0; k2 <= 2*n2; ++k2)
- if ((y = contrast2_aux(p1, sum, k1, k2, ret)) >= 0) z += y;
- if (ret[0] + ret[1] + ret[2] < 0.95) // It seems that this may be caused by floating point errors. I do not really understand why...
- z = 1.0, ret[0] = ret[1] = ret[2] = 1./3;
- }
- return (double)z;
- }
-}
-
-static double mc_cal_afs(bcf_p1aux_t *ma, double *p_ref_folded, double *p_var_folded)
-{
- int k;
- long double sum = 0., sum2;
- double *phi = ma->is_indel? ma->phi_indel : ma->phi;
- memset(ma->afs1, 0, sizeof(double) * (ma->M + 1));
- mc_cal_y(ma);
- // compute AFS
- for (k = 0, sum = 0.; k <= ma->M; ++k)
- sum += (long double)phi[k] * ma->z[k];
- for (k = 0; k <= ma->M; ++k) {
- ma->afs1[k] = phi[k] * ma->z[k] / sum;
- if (isnan(ma->afs1[k]) || isinf(ma->afs1[k])) return -1.;
- }
- // compute folded variant probability
- for (k = 0, sum = 0.; k <= ma->M; ++k)
- sum += (long double)(phi[k] + phi[ma->M - k]) / 2. * ma->z[k];
- for (k = 1, sum2 = 0.; k < ma->M; ++k)
- sum2 += (long double)(phi[k] + phi[ma->M - k]) / 2. * ma->z[k];
- *p_var_folded = sum2 / sum;
- *p_ref_folded = (phi[k] + phi[ma->M - k]) / 2. * (ma->z[ma->M] + ma->z[0]) / sum;
- // the expected frequency
- for (k = 0, sum = 0.; k <= ma->M; ++k) {
- ma->afs[k] += ma->afs1[k];
- sum += k * ma->afs1[k];
- }
- return sum / ma->M;
-}
-
-int bcf_p1_cal(const bcf1_t *b, int do_contrast, bcf_p1aux_t *ma, bcf_p1rst_t *rst)
-{
- int i, k;
- long double sum = 0.;
- ma->is_indel = bcf_is_indel(b);
- rst->perm_rank = -1;
- // set PL and PL_len
- for (i = 0; i < b->n_gi; ++i) {
- if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
- ma->PL = (uint8_t*)b->gi[i].data;
- ma->PL_len = b->gi[i].len;
- break;
- }
- }
- if (i == b->n_gi) return -1; // no PL
- if (b->n_alleles < 2) return -1; // FIXME: find a better solution
- //
- rst->rank0 = cal_pdg(b, ma);
- rst->f_exp = mc_cal_afs(ma, &rst->p_ref_folded, &rst->p_var_folded);
- rst->p_ref = ma->afs1[ma->M];
- for (k = 0, sum = 0.; k < ma->M; ++k)
- sum += ma->afs1[k];
- rst->p_var = (double)sum;
- { // compute the allele count
- double max = -1;
- rst->ac = -1;
- for (k = 0; k <= ma->M; ++k)
- if (max < ma->z[k]) max = ma->z[k], rst->ac = k;
- rst->ac = ma->M - rst->ac;
- }
- // calculate f_flat and f_em
- for (k = 0, sum = 0.; k <= ma->M; ++k)
- sum += (long double)ma->z[k];
- rst->f_flat = 0.;
- for (k = 0; k <= ma->M; ++k) {
- double p = ma->z[k] / sum;
- rst->f_flat += k * p;
- }
- rst->f_flat /= ma->M;
- { // estimate equal-tail credible interval (95% level)
- int l, h;
- double p;
- for (i = 0, p = 0.; i <= ma->M; ++i)
- if (p + ma->afs1[i] > 0.025) break;
- else p += ma->afs1[i];
- l = i;
- for (i = ma->M, p = 0.; i >= 0; --i)
- if (p + ma->afs1[i] > 0.025) break;
- else p += ma->afs1[i];
- h = i;
- rst->cil = (double)(ma->M - h) / ma->M; rst->cih = (double)(ma->M - l) / ma->M;
- }
- if (ma->n1 > 0) { // compute LRT
- double max0, max1, max2;
- for (k = 0, max0 = -1; k <= ma->M; ++k)
- if (max0 < ma->z[k]) max0 = ma->z[k];
- for (k = 0, max1 = -1; k <= ma->n1 * 2; ++k)
- if (max1 < ma->z1[k]) max1 = ma->z1[k];
- for (k = 0, max2 = -1; k <= ma->M - ma->n1 * 2; ++k)
- if (max2 < ma->z2[k]) max2 = ma->z2[k];
- rst->lrt = log(max1 * max2 / max0);
- rst->lrt = rst->lrt < 0? 1 : kf_gammaq(.5, rst->lrt);
- } else rst->lrt = -1.0;
- rst->cmp[0] = rst->cmp[1] = rst->cmp[2] = rst->p_chi2 = -1.0;
- if (do_contrast && rst->p_var > 0.5) // skip contrast2() if the locus is a strong non-variant
- rst->p_chi2 = contrast2(ma, rst->cmp);
- return 0;
-}
-
-void bcf_p1_dump_afs(bcf_p1aux_t *ma)
-{
- int k;
- fprintf(stderr, "[afs]");
- for (k = 0; k <= ma->M; ++k)
- fprintf(stderr, " %d:%.3lf", k, ma->afs[ma->M - k]);
- fprintf(stderr, "\n");
- memset(ma->afs, 0, sizeof(double) * (ma->M + 1));
-}
diff --git a/samtools-0.1.19/bcftools/prob1.h b/samtools-0.1.19/bcftools/prob1.h
deleted file mode 100644
index 6f93155..0000000
--- a/samtools-0.1.19/bcftools/prob1.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#ifndef BCF_PROB1_H
-#define BCF_PROB1_H
-
-#include "bcf.h"
-
-struct __bcf_p1aux_t;
-typedef struct __bcf_p1aux_t bcf_p1aux_t;
-
-typedef struct {
- int rank0, perm_rank; // NB: perm_rank is always set to -1 by bcf_p1_cal()
- int ac; // ML alternative allele count
- double f_exp, f_flat, p_ref_folded, p_ref, p_var_folded, p_var;
- double cil, cih;
- double cmp[3], p_chi2, lrt; // used by contrast2()
-} bcf_p1rst_t;
-
-typedef struct {
- double p[4];
- int mq, depth, is_tested, d[4];
-} anno16_t;
-
-#define MC_PTYPE_FULL 1
-#define MC_PTYPE_COND2 2
-#define MC_PTYPE_FLAT 3
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- bcf_p1aux_t *bcf_p1_init(int n, uint8_t *ploidy);
- void bcf_p1_init_prior(bcf_p1aux_t *ma, int type, double theta);
- void bcf_p1_init_subprior(bcf_p1aux_t *ma, int type, double theta);
- void bcf_p1_destroy(bcf_p1aux_t *ma);
- void bcf_p1_set_ploidy(bcf1_t *b, bcf_p1aux_t *ma);
- int bcf_p1_cal(const bcf1_t *b, int do_contrast, bcf_p1aux_t *ma, bcf_p1rst_t *rst);
- int call_multiallelic_gt(bcf1_t *b, bcf_p1aux_t *ma, double threshold, int var_only);
- int bcf_p1_call_gt(const bcf_p1aux_t *ma, double f0, int k);
- void bcf_p1_dump_afs(bcf_p1aux_t *ma);
- int bcf_p1_read_prior(bcf_p1aux_t *ma, const char *fn);
- int bcf_p1_set_n1(bcf_p1aux_t *b, int n1);
- void bcf_p1_set_folded(bcf_p1aux_t *p1a); // only effective when set_n1() is not called
-
- int bcf_em1(const bcf1_t *b, int n1, int flag, double x[10]);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/bcftools/vcf.c b/samtools-0.1.19/bcftools/vcf.c
deleted file mode 100644
index e8526a3..0000000
--- a/samtools-0.1.19/bcftools/vcf.c
+++ /dev/null
@@ -1,249 +0,0 @@
-#include <zlib.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include "bcf.h"
-#include "kstring.h"
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 4096)
-
-typedef struct {
- gzFile fp;
- FILE *fpout;
- kstream_t *ks;
- void *refhash;
- kstring_t line;
- int max_ref;
-} vcf_t;
-
-bcf_hdr_t *vcf_hdr_read(bcf_t *bp)
-{
- kstring_t meta, smpl;
- int dret;
- vcf_t *v;
- bcf_hdr_t *h;
- if (!bp->is_vcf) return bcf_hdr_read(bp);
- h = calloc(1, sizeof(bcf_hdr_t));
- v = (vcf_t*)bp->v;
- v->line.l = 0;
- memset(&meta, 0, sizeof(kstring_t));
- memset(&smpl, 0, sizeof(kstring_t));
- while (ks_getuntil(v->ks, '\n', &v->line, &dret) >= 0) {
- if (v->line.l < 2) continue;
- if (v->line.s[0] != '#') {
- free(meta.s);
- free(smpl.s);
- free(h);
- return 0; // no sample line
- }
- if (v->line.s[0] == '#' && v->line.s[1] == '#') {
- kputsn(v->line.s, v->line.l, &meta); kputc('\n', &meta);
- } else if (v->line.s[0] == '#') {
- int k;
- ks_tokaux_t aux;
- char *p;
- for (p = kstrtok(v->line.s, "\t\n", &aux), k = 0; p; p = kstrtok(0, 0, &aux), ++k) {
- if (k >= 9) {
- kputsn(p, aux.p - p, &smpl);
- kputc('\0', &smpl);
- }
- }
- break;
- }
- }
- kputc('\0', &meta);
- h->name = 0;
- h->sname = smpl.s; h->l_smpl = smpl.l;
- h->txt = meta.s; h->l_txt = meta.l;
- bcf_hdr_sync(h);
- return h;
-}
-
-bcf_t *vcf_open(const char *fn, const char *mode)
-{
- bcf_t *bp;
- vcf_t *v;
- if (strchr(mode, 'b')) return bcf_open(fn, mode);
- bp = calloc(1, sizeof(bcf_t));
- v = calloc(1, sizeof(vcf_t));
- bp->is_vcf = 1;
- bp->v = v;
- v->refhash = bcf_str2id_init();
- if (strchr(mode, 'r')) {
- v->fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
- v->ks = ks_init(v->fp);
- } else if (strchr(mode, 'w'))
- v->fpout = strcmp(fn, "-")? fopen(fn, "w") : stdout;
- return bp;
-}
-
-int vcf_dictread(bcf_t *bp, bcf_hdr_t *h, const char *fn)
-{
- vcf_t *v;
- gzFile fp;
- kstream_t *ks;
- kstring_t s, rn;
- int dret;
- if (bp == 0) return -1;
- if (!bp->is_vcf) return 0;
- s.l = s.m = 0; s.s = 0;
- rn.m = rn.l = h->l_nm; rn.s = h->name;
- v = (vcf_t*)bp->v;
- fp = gzopen(fn, "r");
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, &s, &dret) >= 0) {
- bcf_str2id_add(v->refhash, strdup(s.s));
- kputs(s.s, &rn); kputc('\0', &rn);
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- }
- ks_destroy(ks);
- gzclose(fp);
- h->l_nm = rn.l; h->name = rn.s;
- bcf_hdr_sync(h);
- free(s.s);
- return 0;
-}
-
-int vcf_close(bcf_t *bp)
-{
- vcf_t *v;
- if (bp == 0) return -1;
- if (!bp->is_vcf) return bcf_close(bp);
- v = (vcf_t*)bp->v;
- if (v->fp) {
- ks_destroy(v->ks);
- gzclose(v->fp);
- }
- if (v->fpout) fclose(v->fpout);
- free(v->line.s);
- bcf_str2id_thorough_destroy(v->refhash);
- free(v);
- free(bp);
- return 0;
-}
-
-int vcf_hdr_write(bcf_t *bp, const bcf_hdr_t *h)
-{
- vcf_t *v = (vcf_t*)bp->v;
- int i, has_ver = 0;
- if (!bp->is_vcf) return bcf_hdr_write(bp, h);
- if (h->l_txt > 0) {
- if (strstr(h->txt, "##fileformat=")) has_ver = 1;
- if (has_ver == 0) fprintf(v->fpout, "##fileformat=VCFv4.1\n");
- fwrite(h->txt, 1, h->l_txt - 1, v->fpout);
- }
- if (h->l_txt == 0) fprintf(v->fpout, "##fileformat=VCFv4.1\n");
- fprintf(v->fpout, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT");
- for (i = 0; i < h->n_smpl; ++i)
- fprintf(v->fpout, "\t%s", h->sns[i]);
- fputc('\n', v->fpout);
- return 0;
-}
-
-int vcf_write(bcf_t *bp, bcf_hdr_t *h, bcf1_t *b)
-{
- vcf_t *v = (vcf_t*)bp->v;
- extern void bcf_fmt_core(const bcf_hdr_t *h, bcf1_t *b, kstring_t *s);
- if (!bp->is_vcf) return bcf_write(bp, h, b);
- bcf_fmt_core(h, b, &v->line);
- fwrite(v->line.s, 1, v->line.l, v->fpout);
- fputc('\n', v->fpout);
- return v->line.l + 1;
-}
-
-int vcf_read(bcf_t *bp, bcf_hdr_t *h, bcf1_t *b)
-{
- int dret, k, i, sync = 0;
- vcf_t *v = (vcf_t*)bp->v;
- char *p, *q;
- kstring_t str, rn;
- ks_tokaux_t aux, a2;
- if (!bp->is_vcf) return bcf_read(bp, h, b);
- v->line.l = 0;
- str.l = 0; str.m = b->m_str; str.s = b->str;
- rn.l = rn.m = h->l_nm; rn.s = h->name;
- if (ks_getuntil(v->ks, '\n', &v->line, &dret) < 0) return -1;
- b->n_smpl = h->n_smpl;
- for (p = kstrtok(v->line.s, "\t", &aux), k = 0; p; p = kstrtok(0, 0, &aux), ++k) {
- *(char*)aux.p = 0;
- if (k == 0) { // ref
- int tid = bcf_str2id(v->refhash, p);
- if (tid < 0) {
- tid = bcf_str2id_add(v->refhash, strdup(p));
- kputs(p, &rn); kputc('\0', &rn);
- sync = 1;
- }
- b->tid = tid;
- } else if (k == 1) { // pos
- b->pos = atoi(p) - 1;
- } else if (k == 5) { // qual
- b->qual = (p[0] >= '0' && p[0] <= '9')? atof(p) : 0;
- } else if (k <= 8) { // variable length strings
- kputs(p, &str); kputc('\0', &str);
- b->l_str = str.l; b->m_str = str.m; b->str = str.s;
- if (k == 8) bcf_sync(b);
- } else { // k > 9
- if (strncmp(p, "./.", 3) == 0) {
- for (i = 0; i < b->n_gi; ++i) {
- if (b->gi[i].fmt == bcf_str2int("GT", 2)) {
- ((uint8_t*)b->gi[i].data)[k-9] = 1<<7;
- } else if (b->gi[i].fmt == bcf_str2int("GQ", 2)) {
- ((uint8_t*)b->gi[i].data)[k-9] = 0;
- } else if (b->gi[i].fmt == bcf_str2int("SP", 2)) {
- ((int32_t*)b->gi[i].data)[k-9] = 0;
- } else if (b->gi[i].fmt == bcf_str2int("DP", 2) || b->gi[i].fmt == bcf_str2int("DV", 2)) {
- ((uint16_t*)b->gi[i].data)[k-9] = 0;
- } else if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
- int y = b->n_alleles * (b->n_alleles + 1) / 2;
- memset((uint8_t*)b->gi[i].data + (k - 9) * y, 0, y);
- } else if (b->gi[i].fmt == bcf_str2int("GL", 2)) {
- int y = b->n_alleles * (b->n_alleles + 1) / 2;
- memset((float*)b->gi[i].data + (k - 9) * y, 0, y * 4);
- }
- }
- goto endblock;
- }
- for (q = kstrtok(p, ":", &a2), i = 0; q && i < b->n_gi; q = kstrtok(0, 0, &a2), ++i) {
- if (b->gi[i].fmt == bcf_str2int("GT", 2)) {
- ((uint8_t*)b->gi[i].data)[k-9] = (q[0] - '0')<<3 | (q[2] - '0') | (q[1] == '/'? 0 : 1) << 6;
- } else if (b->gi[i].fmt == bcf_str2int("GQ", 2)) {
- double _x = strtod(q, &q);
- int x = (int)(_x + .499);
- if (x > 255) x = 255;
- ((uint8_t*)b->gi[i].data)[k-9] = x;
- } else if (b->gi[i].fmt == bcf_str2int("SP", 2)) {
- int x = strtol(q, &q, 10);
- if (x > 0xffff) x = 0xffff;
- ((uint32_t*)b->gi[i].data)[k-9] = x;
- } else if (b->gi[i].fmt == bcf_str2int("DP", 2) || b->gi[i].fmt == bcf_str2int("DV", 2)) {
- int x = strtol(q, &q, 10);
- if (x > 0xffff) x = 0xffff;
- ((uint16_t*)b->gi[i].data)[k-9] = x;
- } else if (b->gi[i].fmt == bcf_str2int("PL", 2)) {
- int x, y, j;
- uint8_t *data = (uint8_t*)b->gi[i].data;
- y = b->n_alleles * (b->n_alleles + 1) / 2;
- for (j = 0; j < y; ++j) {
- x = strtol(q, &q, 10);
- if (x > 255) x = 255;
- data[(k-9) * y + j] = x;
- ++q;
- }
- } else if (b->gi[i].fmt == bcf_str2int("GL", 2)) {
- int j, y;
- float x, *data = (float*)b->gi[i].data;
- y = b->n_alleles * (b->n_alleles + 1) / 2;
- for (j = 0; j < y; ++j) {
- x = strtod(q, &q);
- data[(k-9) * y + j] = x > 0? -x/10. : x;
- ++q;
- }
- }
- }
- endblock: i = i;
- }
- }
- h->l_nm = rn.l; h->name = rn.s;
- if (sync) bcf_hdr_sync(h);
- return v->line.l + 1;
-}
diff --git a/samtools-0.1.19/bcftools/vcfutils.pl b/samtools-0.1.19/bcftools/vcfutils.pl
deleted file mode 100755
index 2b7ba0b..0000000
--- a/samtools-0.1.19/bcftools/vcfutils.pl
+++ /dev/null
@@ -1,567 +0,0 @@
-#!/usr/bin/perl -w
-
-# Author: lh3
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-&main;
-exit;
-
-sub main {
- &usage if (@ARGV < 1);
- my $command = shift(@ARGV);
- my %func = (subsam=>\&subsam, listsam=>\&listsam, fillac=>\&fillac, qstats=>\&qstats, varFilter=>\&varFilter,
- hapmap2vcf=>\&hapmap2vcf, ucscsnp2vcf=>\&ucscsnp2vcf, filter4vcf=>\&varFilter, ldstats=>\&ldstats,
- gapstats=>\&gapstats, splitchr=>\&splitchr, vcf2fq=>\&vcf2fq);
- die("Unknown command \"$command\".\n") if (!defined($func{$command}));
- &{$func{$command}};
-}
-
-sub splitchr {
- my %opts = (l=>5000000);
- getopts('l:', \%opts);
- my $l = $opts{l};
- die(qq/Usage: vcfutils.pl splitchr [-l $opts{l}] <in.fa.fai>\n/) if (@ARGV == 0 && -t STDIN);
- while (<>) {
- my @t = split;
- my $last = 0;
- for (my $i = 0; $i < $t[1];) {
- my $e = ($t[1] - $i) / $l < 1.1? $t[1] : $i + $l;
- print "$t[0]:".($i+1)."-$e\n";
- $i = $e;
- }
- }
-}
-
-sub subsam {
- die(qq/Usage: vcfutils.pl subsam <in.vcf> [samples]\n/) if (@ARGV == 0);
- my ($fh, %h);
- my $fn = shift(@ARGV);
- my @col;
- open($fh, ($fn =~ /\.gz$/)? "gzip -dc $fn |" : $fn) || die;
- $h{$_} = 1 for (@ARGV);
- while (<$fh>) {
- if (/^##/) {
- print;
- } elsif (/^#/) {
- my @t = split;
- my @s = @t[0..8]; # all fixed fields + FORMAT
- for (9 .. $#t) {
- if ($h{$t[$_]}) {
- push(@s, $t[$_]);
- push(@col, $_);
- }
- }
- pop(@s) if (@s == 9); # no sample selected; remove the FORMAT field
- print join("\t", @s), "\n";
- } else {
- my @t = split;
- if (@col == 0) {
- print join("\t", @t[0..7]), "\n";
- } else {
- print join("\t", @t[0..8], map {$t[$_]} @col), "\n";
- }
- }
- }
- close($fh);
-}
-
-sub listsam {
- die(qq/Usage: vcfutils.pl listsam <in.vcf>\n/) if (@ARGV == 0 && -t STDIN);
- while (<>) {
- if (/^#/ && !/^##/) {
- my @t = split;
- print join("\n", @t[9..$#t]), "\n";
- exit;
- }
- }
-}
-
-sub fillac {
- die(qq/Usage: vcfutils.pl fillac <in.vcf>\n\nNote: The GT field MUST BE present and always appear as the first field.\n/) if (@ARGV == 0 && -t STDIN);
- while (<>) {
- if (/^#/) {
- print;
- } else {
- my @t = split;
- my @c = (0, 0);
- my $n = 0;
- my $s = -1;
- @_ = split(":", $t[8]);
- for (0 .. $#_) {
- if ($_[$_] eq 'GT') { $s = $_; last; }
- }
- if ($s < 0) {
- print join("\t", @t), "\n";
- next;
- }
- for (9 .. $#t) {
- if ($t[$_] =~ /^0,0,0/) {
- } elsif ($t[$_] =~ /^([^\s:]+:){$s}(\d+).(\d+)/) {
- ++$c[$2]; ++$c[$3];
- $n += 2;
- }
- }
- my $AC = "AC=" . join("\t", @c[1..$#c]) . ";AN=$n";
- my $info = $t[7];
- $info =~ s/(;?)AC=(\d+)//;
- $info =~ s/(;?)AN=(\d+)//;
- if ($info eq '.') {
- $info = $AC;
- } else {
- $info .= ";$AC";
- }
- $t[7] = $info;
- print join("\t", @t), "\n";
- }
- }
-}
-
-sub ldstats {
- my %opts = (t=>0.9);
- getopts('t:', \%opts);
- die("Usage: vcfutils.pl ldstats [-t $opts{t}] <in.vcf>\n") if (@ARGV == 0 && -t STDIN);
- my $cutoff = $opts{t};
- my ($last, $lastchr) = (0x7fffffff, '');
- my ($x, $y, $n) = (0, 0, 0);
- while (<>) {
- if (/^([^#\s]+)\s(\d+)/) {
- my ($chr, $pos) = ($1, $2);
- if (/NEIR=([\d\.]+)/) {
- ++$n;
- ++$y, $x += $pos - $last if ($lastchr eq $chr && $pos > $last && $1 > $cutoff);
- }
- $last = $pos; $lastchr = $chr;
- }
- }
- print "Number of SNP intervals in strong LD (r > $opts{t}): $y\n";
- print "Fraction: ", $y/$n, "\n";
- print "Length: $x\n";
-}
-
-sub qstats {
- my %opts = (r=>'', s=>0.02, v=>undef);
- getopts('r:s:v', \%opts);
- die("Usage: vcfutils.pl qstats [-r ref.vcf] <in.vcf>\n
-Note: This command discards indels. Output: QUAL #non-indel #SNPs #transitions #joint ts/tv #joint/#ref #joint/#non-indel \n") if (@ARGV == 0 && -t STDIN);
- my %ts = (AG=>1, GA=>1, CT=>1, TC=>1);
- my %h = ();
- my $is_vcf = defined($opts{v})? 1 : 0;
- if ($opts{r}) { # read the reference positions
- my $fh;
- open($fh, $opts{r}) || die;
- while (<$fh>) {
- next if (/^#/);
- if ($is_vcf) {
- my @t = split;
- $h{$t[0],$t[1]} = $t[4];
- } else {
- $h{$1,$2} = 1 if (/^(\S+)\s+(\d+)/);
- }
- }
- close($fh);
- }
- my $hsize = scalar(keys %h);
- my @a;
- while (<>) {
- next if (/^#/);
- my @t = split;
- next if (length($t[3]) != 1 || uc($t[3]) eq 'N');
- $t[3] = uc($t[3]); $t[4] = uc($t[4]);
- my @s = split(',', $t[4]);
- $t[5] = 3 if ($t[5] eq '.' || $t[5] < 0);
- next if (length($s[0]) != 1);
- my $hit;
- if ($is_vcf) {
- $hit = 0;
- my $aa = $h{$t[0],$t[1]};
- if (defined($aa)) {
- my @aaa = split(",", $aa);
- for (@aaa) {
- $hit = 1 if ($_ eq $s[0]);
- }
- }
- } else {
- $hit = defined($h{$t[0],$t[1]})? 1 : 0;
- }
- push(@a, [$t[5], ($t[4] eq '.' || $t[4] eq $t[3])? 0 : 1, $ts{$t[3].$s[0]}? 1 : 0, $hit]);
- }
- push(@a, [-1, 0, 0, 0]); # end marker
- die("[qstats] No SNP data!\n") if (@a == 0);
- @a = sort {$b->[0]<=>$a->[0]} @a;
- my $next = $opts{s};
- my $last = $a[0];
- my @c = (0, 0, 0, 0);
- my @lc;
- $lc[1] = $lc[2] = 0;
- for my $p (@a) {
- if ($p->[0] == -1 || ($p->[0] != $last && $c[0]/@a > $next)) {
- my @x;
- $x[0] = sprintf("%.4f", $c[1]-$c[2]? $c[2] / ($c[1] - $c[2]) : 100);
- $x[1] = sprintf("%.4f", $hsize? $c[3] / $hsize : 0);
- $x[2] = sprintf("%.4f", $c[3] / $c[1]);
- my $a = $c[1] - $lc[1];
- my $b = $c[2] - $lc[2];
- $x[3] = sprintf("%.4f", $a-$b? $b / ($a-$b) : 100);
- print join("\t", $last, @c, @x), "\n";
- $next = $c[0]/@a + $opts{s};
- $lc[1] = $c[1]; $lc[2] = $c[2];
- }
- ++$c[0]; $c[1] += $p->[1]; $c[2] += $p->[2]; $c[3] += $p->[3];
- $last = $p->[0];
- }
-}
-
-sub varFilter {
- my %opts = (d=>2, D=>10000000, a=>2, W=>10, Q=>10, w=>3, p=>undef, 1=>1e-4, 2=>1e-100, 3=>0, 4=>1e-4, G=>0, S=>1000, e=>1e-4);
- getopts('pd:D:W:Q:w:a:1:2:3:4:G:S:e:', \%opts);
- die(qq/
-Usage: vcfutils.pl varFilter [options] <in.vcf>
-
-Options: -Q INT minimum RMS mapping quality for SNPs [$opts{Q}]
- -d INT minimum read depth [$opts{d}]
- -D INT maximum read depth [$opts{D}]
- -a INT minimum number of alternate bases [$opts{a}]
- -w INT SNP within INT bp around a gap to be filtered [$opts{w}]
- -W INT window size for filtering adjacent gaps [$opts{W}]
- -1 FLOAT min P-value for strand bias (given PV4) [$opts{1}]
- -2 FLOAT min P-value for baseQ bias [$opts{2}]
- -3 FLOAT min P-value for mapQ bias [$opts{3}]
- -4 FLOAT min P-value for end distance bias [$opts{4}]
- -e FLOAT min P-value for HWE (plus F<0) [$opts{e}]
- -p print filtered variants
-
-Note: Some of the filters rely on annotations generated by SAMtools\/BCFtools.
-\n/) if (@ARGV == 0 && -t STDIN);
-
- # calculate the window size
- my ($ol, $ow) = ($opts{W}, $opts{w});
- my $max_dist = $ol > $ow? $ol : $ow;
- # the core loop
- my @staging; # (indel_filtering_score, flt_tag, indel_span; chr, pos, ...)
- while (<>) {
- my @t = split;
- if (/^#/) {
- print; next;
- }
- next if ($t[4] eq '.'); # skip non-var sites
- next if ($t[3] eq 'N'); # skip sites with unknown ref ('N')
- # check if the site is a SNP
- my $type = 1; # SNP
- if (length($t[3]) > 1) {
- $type = 2; # MNP
- my @s = split(',', $t[4]);
- for (@s) {
- $type = 3 if (length != length($t[3]));
- }
- } else {
- my @s = split(',', $t[4]);
- for (@s) {
- $type = 3 if (length > 1);
- }
- }
- # clear the out-of-range elements
- while (@staging) {
- # Still on the same chromosome and the first element's window still affects this position?
- last if ($staging[0][3] eq $t[0] && $staging[0][4] + $staging[0][2] + $max_dist >= $t[1]);
- varFilter_aux(shift(@staging), $opts{p}); # calling a function is a bit slower, not much
- }
- my $flt = 0;
- # parse annotations
- my ($dp, $mq, $dp_alt) = (-1, -1, -1);
- if ($t[7] =~ /DP4=(\d+),(\d+),(\d+),(\d+)/i) {
- $dp = $1 + $2 + $3 + $4;
- $dp_alt = $3 + $4;
- }
- if ($t[7] =~ /DP=(\d+)/i) {
- $dp = $1;
- }
- $mq = $1 if ($t[7] =~ /MQ=(\d+)/i);
- # the depth and mapQ filter
- if ($dp >= 0) {
- if ($dp < $opts{d}) {
- $flt = 2;
- } elsif ($dp > $opts{D}) {
- $flt = 3;
- }
- }
- $flt = 4 if ($dp_alt >= 0 && $dp_alt < $opts{a});
- $flt = 1 if ($flt == 0 && $mq >= 0 && $mq < $opts{Q});
- $flt = 7 if ($flt == 0 && /PV4=([^,]+),([^,]+),([^,]+),([^,;\t]+)/
- && ($1<$opts{1} || $2<$opts{2} || $3<$opts{3} || $4<$opts{4}));
- $flt = 8 if ($flt == 0 && ((/MXGQ=(\d+)/ && $1 < $opts{G}) || (/MXSP=(\d+)/ && $1 >= $opts{S})));
- # HWE filter
- if ($t[7] =~ /G3=([^;,]+),([^;,]+),([^;,]+).*HWE=([^;,]+)/ && $4 < $opts{e}) {
- my $p = 2*$1 + $2;
- my $f = ($p > 0 && $p < 1)? 1 - $2 / ($p * (1-$p)) : 0;
- $flt = 9 if ($f < 0);
- }
-
- my $score = $t[5] * 100 + $dp_alt;
- my $rlen = length($t[3]) - 1; # $indel_score<0 for SNPs
- if ($flt == 0) {
- if ($type == 3) { # an indel
- # filtering SNPs and MNPs
- for my $x (@staging) {
- next if (($x->[0]&3) == 3 || $x->[1] || $x->[4] + $x->[2] + $ow < $t[1]);
- $x->[1] = 5;
- }
- # check the staging list for indel filtering
- for my $x (@staging) {
- next if (($x->[0]&3) != 3 || $x->[1] || $x->[4] + $x->[2] + $ol < $t[1]);
- if ($x->[0]>>2 < $score) {
- $x->[1] = 6;
- } else {
- $flt = 6; last;
- }
- }
- } else { # SNP or MNP
- for my $x (@staging) {
- next if (($x->[0]&3) != 3 || $x->[4] + $x->[2] + $ow < $t[1]);
- if ($x->[4] + length($x->[7]) - 1 == $t[1] && substr($x->[7], -1, 1) eq substr($t[4], 0, 1)
- && length($x->[7]) - length($x->[6]) == 1) {
- $x->[1] = 5;
- } else { $flt = 5; }
- last;
- }
- # check MNP
- for my $x (@staging) {
- next if (($x->[0]&3) == 3 || $x->[4] + $x->[2] < $t[1]);
- if ($x->[0]>>2 < $score) {
- $x->[1] = 8;
- } else {
- $flt = 8; last;
- }
- }
- }
- }
- push(@staging, [$score<<2|$type, $flt, $rlen, @t]);
- }
- # output the last few elements in the staging list
- while (@staging) {
- varFilter_aux(shift @staging, $opts{p});
- }
-}
-
-sub varFilter_aux {
- my ($first, $is_print) = @_;
- if ($first->[1] == 0) {
- print join("\t", @$first[3 .. @$first-1]), "\n";
- } elsif ($is_print) {
- print STDERR join("\t", substr("UQdDaGgPMS", $first->[1], 1), @$first[3 .. @$first-1]), "\n";
- }
-}
-
-sub gapstats {
- my (@c0, @c1);
- $c0[$_] = $c1[$_] = 0 for (0 .. 10000);
- while (<>) {
- next if (/^#/);
- my @t = split;
- next if (length($t[3]) == 1 && $t[4] =~ /^[A-Za-z](,[A-Za-z])*$/); # not an indel
- my @s = split(',', $t[4]);
- for my $x (@s) {
- my $l = length($x) - length($t[3]) + 5000;
- if ($x =~ /^-/) {
- $l = -(length($x) - 1) + 5000;
- } elsif ($x =~ /^\+/) {
- $l = length($x) - 1 + 5000;
- }
- $c0[$l] += 1 / @s;
- }
- }
- for (my $i = 0; $i < 10000; ++$i) {
- next if ($c0[$i] == 0);
- $c1[0] += $c0[$i];
- $c1[1] += $c0[$i] if (($i-5000)%3 == 0);
- printf("C\t%d\t%.2f\n", ($i-5000), $c0[$i]);
- }
- printf("3\t%d\t%d\t%.3f\n", $c1[0], $c1[1], $c1[1]/$c1[0]);
-}
-
-sub ucscsnp2vcf {
- die("Usage: vcfutils.pl <in.ucsc.snp>\n") if (@ARGV == 0 && -t STDIN);
- print "##fileformat=VCFv4.0\n";
- print join("\t", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO"), "\n";
- while (<>) {
- my @t = split("\t");
- my $indel = ($t[9] =~ /^[ACGT](\/[ACGT])+$/)? 0 : 1;
- my $pos = $t[2] + 1;
- my @alt;
- push(@alt, $t[7]);
- if ($t[6] eq '-') {
- $t[9] = reverse($t[9]);
- $t[9] =~ tr/ACGTRYMKWSNacgtrymkwsn/TGCAYRKMWSNtgcayrkmwsn/;
- }
- my @a = split("/", $t[9]);
- for (@a) {
- push(@alt, $_) if ($_ ne $alt[0]);
- }
- if ($indel) {
- --$pos;
- for (0 .. $#alt) {
- $alt[$_] =~ tr/-//d;
- $alt[$_] = "N$alt[$_]";
- }
- }
- my $ref = shift(@alt);
- my $af = $t[13] > 0? ";AF=$t[13]" : '';
- my $valid = ($t[12] eq 'unknown')? '' : ";valid=$t[12]";
- my $info = "molType=$t[10];class=$t[11]$valid$af";
- print join("\t", $t[1], $pos, $t[4], $ref, join(",", @alt), 0, '.', $info), "\n";
- }
-}
-
-sub hapmap2vcf {
- die("Usage: vcfutils.pl <in.ucsc.snp> <in.hapmap>\n") if (@ARGV == 0);
- my $fn = shift(@ARGV);
- # parse UCSC SNP
- warn("Parsing UCSC SNPs...\n");
- my ($fh, %map);
- open($fh, ($fn =~ /\.gz$/)? "gzip -dc $fn |" : $fn) || die;
- while (<$fh>) {
- my @t = split;
- next if ($t[3] - $t[2] != 1); # not SNP
- @{$map{$t[4]}} = @t[1,3,7];
- }
- close($fh);
- # write VCF
- warn("Writing VCF...\n");
- print "##fileformat=VCFv4.0\n";
- while (<>) {
- my @t = split;
- if ($t[0] eq 'rs#') { # the first line
- print join("\t", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT", @t[11..$#t]), "\n";
- } else {
- next unless ($map{$t[0]});
- next if (length($t[1]) != 3); # skip non-SNPs
- my $a = \@{$map{$t[0]}};
- my $ref = $a->[2];
- my @u = split('/', $t[1]);
- if ($u[1] eq $ref) {
- $u[1] = $u[0]; $u[0] = $ref;
- } elsif ($u[0] ne $ref) { next; }
- my $alt = $u[1];
- my %w;
- $w{$u[0]} = 0; $w{$u[1]} = 1;
- my @s = (@$a[0,1], $t[0], $ref, $alt, 0, '.', '.', 'GT');
- my $is_tri = 0;
- for (@t[11..$#t]) {
- if ($_ eq 'NN') {
- push(@s, './.');
- } else {
- my @a = ($w{substr($_,0,1)}, $w{substr($_,1,1)});
- if (!defined($a[0]) || !defined($a[1])) {
- $is_tri = 1;
- last;
- }
- push(@s, "$a[0]/$a[1]");
- }
- }
- next if ($is_tri);
- print join("\t", @s), "\n";
- }
- }
-}
-
-sub vcf2fq {
- my %opts = (d=>3, D=>100000, Q=>10, l=>5);
- getopts('d:D:Q:l:', \%opts);
- die(qq/
-Usage: vcfutils.pl vcf2fq [options] <all-site.vcf>
-
-Options: -d INT minimum depth [$opts{d}]
- -D INT maximum depth [$opts{D}]
- -Q INT min RMS mapQ [$opts{Q}]
- -l INT INDEL filtering window [$opts{l}]
-\n/) if (@ARGV == 0 && -t STDIN);
-
- my ($last_chr, $seq, $qual, $last_pos, @gaps);
- my $_Q = $opts{Q};
- my $_d = $opts{d};
- my $_D = $opts{D};
-
- my %het = (AC=>'M', AG=>'R', AT=>'W', CA=>'M', CG=>'S', CT=>'Y',
- GA=>'R', GC=>'S', GT=>'K', TA=>'W', TC=>'Y', TG=>'K');
-
- $last_chr = '';
- while (<>) {
- next if (/^#/);
- my @t = split;
- if ($last_chr ne $t[0]) {
- &v2q_post_process($last_chr, \$seq, \$qual, \@gaps, $opts{l}) if ($last_chr);
- ($last_chr, $last_pos) = ($t[0], 0);
- $seq = $qual = '';
- @gaps = ();
- }
- die("[vcf2fq] unsorted input\n") if ($t[1] - $last_pos < 0);
- if ($t[1] - $last_pos > 1) {
- $seq .= 'n' x ($t[1] - $last_pos - 1);
- $qual .= '!' x ($t[1] - $last_pos - 1);
- }
- if (length($t[3]) == 1 && $t[7] !~ /INDEL/ && $t[4] =~ /^([A-Za-z.])(,[A-Za-z])*$/) { # a SNP or reference
- my ($ref, $alt) = ($t[3], $1);
- my ($b, $q);
- $q = $1 if ($t[7] =~ /FQ=(-?[\d\.]+)/);
- if ($q < 0) {
- $_ = ($t[7] =~ /AF1=([\d\.]+)/)? $1 : 0;
- $b = ($_ < .5 || $alt eq '.')? $ref : $alt;
- $q = -$q;
- } else {
- $b = $het{"$ref$alt"};
- $b ||= 'N';
- }
- $b = lc($b);
- $b = uc($b) if (($t[7] =~ /MQ=(\d+)/ && $1 >= $_Q) && ($t[7] =~ /DP=(\d+)/ && $1 >= $_d && $1 <= $_D));
- $q = int($q + 33 + .499);
- $q = chr($q <= 126? $q : 126);
- $seq .= $b;
- $qual .= $q;
- } elsif ($t[4] ne '.') { # an INDEL
- push(@gaps, [$t[1], length($t[3])]);
- }
- $last_pos = $t[1];
- }
- &v2q_post_process($last_chr, \$seq, \$qual, \@gaps, $opts{l});
-}
-
-sub v2q_post_process {
- my ($chr, $seq, $qual, $gaps, $l) = @_;
- for my $g (@$gaps) {
- my $beg = $g->[0] > $l? $g->[0] - $l : 0;
- my $end = $g->[0] + $g->[1] + $l;
- $end = length($$seq) if ($end > length($$seq));
- substr($$seq, $beg, $end - $beg) = lc(substr($$seq, $beg, $end - $beg));
- }
- print "\@$chr\n"; &v2q_print_str($seq);
- print "+\n"; &v2q_print_str($qual);
-}
-
-sub v2q_print_str {
- my ($s) = @_;
- my $l = length($$s);
- for (my $i = 0; $i < $l; $i += 60) {
- print substr($$s, $i, 60), "\n";
- }
-}
-
-sub usage {
- die(qq/
-Usage: vcfutils.pl <command> [<arguments>]\n
-Command: subsam get a subset of samples
- listsam list the samples
- fillac fill the allele count field
- qstats SNP stats stratified by QUAL
-
- hapmap2vcf convert the hapmap format to VCF
- ucscsnp2vcf convert UCSC SNP SQL dump to VCF
-
- varFilter filtering short variants (*)
- vcf2fq VCF->fastq (**)
-
-Notes: Commands with description endting with (*) may need bcftools
- specific annotations.
-\n/);
-}
diff --git a/samtools-0.1.19/bedcov.c b/samtools-0.1.19/bedcov.c
deleted file mode 100644
index 3e4b952..0000000
--- a/samtools-0.1.19/bedcov.c
+++ /dev/null
@@ -1,127 +0,0 @@
-#include <zlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "kstring.h"
-#include "bgzf.h"
-#include "bam.h"
-
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-typedef struct {
- bamFile fp;
- bam_iter_t iter;
- int min_mapQ;
-} aux_t;
-
-static int read_bam(void *data, bam1_t *b)
-{
- aux_t *aux = (aux_t*)data;
- int ret = bam_iter_read(aux->fp, aux->iter, b);
- if ((int)b->core.qual < aux->min_mapQ) b->core.flag |= BAM_FUNMAP;
- return ret;
-}
-
-int main_bedcov(int argc, char *argv[])
-{
- extern void bam_init_header_hash(bam_header_t*);
- gzFile fp;
- kstring_t str;
- kstream_t *ks;
- bam_index_t **idx;
- bam_header_t *h = 0;
- aux_t **aux;
- int *n_plp, dret, i, n, c, min_mapQ = 0;
- int64_t *cnt;
- const bam_pileup1_t **plp;
-
- while ((c = getopt(argc, argv, "Q:")) >= 0) {
- switch (c) {
- case 'Q': min_mapQ = atoi(optarg); break;
- }
- }
- if (optind + 2 > argc) {
- fprintf(stderr, "Usage: samtools bedcov <in.bed> <in1.bam> [...]\n");
- return 1;
- }
- memset(&str, 0, sizeof(kstring_t));
- n = argc - optind - 1;
- aux = calloc(n, sizeof(void*));
- idx = calloc(n, sizeof(void*));
- for (i = 0; i < n; ++i) {
- aux[i] = calloc(1, sizeof(aux_t));
- aux[i]->min_mapQ = min_mapQ;
- aux[i]->fp = bam_open(argv[i+optind+1], "r");
- idx[i] = bam_index_load(argv[i+optind+1]);
- if (aux[i]->fp == 0 || idx[i] == 0) {
- fprintf(stderr, "ERROR: fail to open index BAM file '%s'\n", argv[i+optind+1]);
- return 2;
- }
- bgzf_set_cache_size(aux[i]->fp, 20);
- if (i == 0) h = bam_header_read(aux[0]->fp);
- }
- bam_init_header_hash(h);
- cnt = calloc(n, 8);
-
- fp = gzopen(argv[optind], "rb");
- ks = ks_init(fp);
- n_plp = calloc(n, sizeof(int));
- plp = calloc(n, sizeof(void*));
- while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0) {
- char *p, *q;
- int tid, beg, end, pos;
- bam_mplp_t mplp;
-
- for (p = q = str.s; *p && *p != '\t'; ++p);
- if (*p != '\t') goto bed_error;
- *p = 0; tid = bam_get_tid(h, q); *p = '\t';
- if (tid < 0) goto bed_error;
- for (q = p = p + 1; isdigit(*p); ++p);
- if (*p != '\t') goto bed_error;
- *p = 0; beg = atoi(q); *p = '\t';
- for (q = p = p + 1; isdigit(*p); ++p);
- if (*p == '\t' || *p == 0) {
- int c = *p;
- *p = 0; end = atoi(q); *p = c;
- } else goto bed_error;
-
- for (i = 0; i < n; ++i) {
- if (aux[i]->iter) bam_iter_destroy(aux[i]->iter);
- aux[i]->iter = bam_iter_query(idx[i], tid, beg, end);
- }
- mplp = bam_mplp_init(n, read_bam, (void**)aux);
- bam_mplp_set_maxcnt(mplp, 64000);
- memset(cnt, 0, 8 * n);
- while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0)
- if (pos >= beg && pos < end)
- for (i = 0; i < n; ++i) cnt[i] += n_plp[i];
- for (i = 0; i < n; ++i) {
- kputc('\t', &str);
- kputl(cnt[i], &str);
- }
- puts(str.s);
- bam_mplp_destroy(mplp);
- continue;
-
-bed_error:
- fprintf(stderr, "Errors in BED line '%s'\n", str.s);
- }
- free(n_plp); free(plp);
- ks_destroy(ks);
- gzclose(fp);
-
- free(cnt);
- for (i = 0; i < n; ++i) {
- if (aux[i]->iter) bam_iter_destroy(aux[i]->iter);
- bam_index_destroy(idx[i]);
- bam_close(aux[i]->fp);
- free(aux[i]);
- }
- bam_header_destroy(h);
- free(aux); free(idx);
- free(str.s);
- return 0;
-}
diff --git a/samtools-0.1.19/bedidx.c b/samtools-0.1.19/bedidx.c
deleted file mode 100644
index ec75a10..0000000
--- a/samtools-0.1.19/bedidx.c
+++ /dev/null
@@ -1,162 +0,0 @@
-#include <stdlib.h>
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
-#include <zlib.h>
-
-#ifdef _WIN32
-#define drand48() ((double)rand() / RAND_MAX)
-#endif
-
-#include "ksort.h"
-KSORT_INIT_GENERIC(uint64_t)
-
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 8192)
-
-typedef struct {
- int n, m;
- uint64_t *a;
- int *idx;
-} bed_reglist_t;
-
-#include "khash.h"
-KHASH_MAP_INIT_STR(reg, bed_reglist_t)
-
-#define LIDX_SHIFT 13
-
-typedef kh_reg_t reghash_t;
-
-int *bed_index_core(int n, uint64_t *a, int *n_idx)
-{
- int i, j, m, *idx;
- m = *n_idx = 0; idx = 0;
- for (i = 0; i < n; ++i) {
- int beg, end;
- beg = a[i]>>32 >> LIDX_SHIFT; end = ((uint32_t)a[i]) >> LIDX_SHIFT;
- if (m < end + 1) {
- int oldm = m;
- m = end + 1;
- kroundup32(m);
- idx = realloc(idx, m * sizeof(int));
- for (j = oldm; j < m; ++j) idx[j] = -1;
- }
- if (beg == end) {
- if (idx[beg] < 0) idx[beg] = i;
- } else {
- for (j = beg; j <= end; ++j)
- if (idx[j] < 0) idx[j] = i;
- }
- *n_idx = end + 1;
- }
- return idx;
-}
-
-void bed_index(void *_h)
-{
- reghash_t *h = (reghash_t*)_h;
- khint_t k;
- for (k = 0; k < kh_end(h); ++k) {
- if (kh_exist(h, k)) {
- bed_reglist_t *p = &kh_val(h, k);
- if (p->idx) free(p->idx);
- ks_introsort(uint64_t, p->n, p->a);
- p->idx = bed_index_core(p->n, p->a, &p->m);
- }
- }
-}
-
-int bed_overlap_core(const bed_reglist_t *p, int beg, int end)
-{
- int i, min_off;
- if (p->n == 0) return 0;
- min_off = (beg>>LIDX_SHIFT >= p->n)? p->idx[p->n-1] : p->idx[beg>>LIDX_SHIFT];
- if (min_off < 0) { // TODO: this block can be improved, but speed should not matter too much here
- int n = beg>>LIDX_SHIFT;
- if (n > p->n) n = p->n;
- for (i = n - 1; i >= 0; --i)
- if (p->idx[i] >= 0) break;
- min_off = i >= 0? p->idx[i] : 0;
- }
- for (i = min_off; i < p->n; ++i) {
- if ((int)(p->a[i]>>32) >= end) break; // out of range; no need to proceed
- if ((int32_t)p->a[i] > beg && (int32_t)(p->a[i]>>32) < end)
- return 1; // find the overlap; return
- }
- return 0;
-}
-
-int bed_overlap(const void *_h, const char *chr, int beg, int end)
-{
- const reghash_t *h = (const reghash_t*)_h;
- khint_t k;
- if (!h) return 0;
- k = kh_get(reg, h, chr);
- if (k == kh_end(h)) return 0;
- return bed_overlap_core(&kh_val(h, k), beg, end);
-}
-
-void *bed_read(const char *fn)
-{
- reghash_t *h = kh_init(reg);
- gzFile fp;
- kstream_t *ks;
- int dret;
- kstring_t *str;
- // read the list
- fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
- if (fp == 0) return 0;
- str = calloc(1, sizeof(kstring_t));
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, str, &dret) >= 0) { // read the chr name
- int beg = -1, end = -1;
- bed_reglist_t *p;
- khint_t k = kh_get(reg, h, str->s);
- if (k == kh_end(h)) { // absent from the hash table
- int ret;
- char *s = strdup(str->s);
- k = kh_put(reg, h, s, &ret);
- memset(&kh_val(h, k), 0, sizeof(bed_reglist_t));
- }
- p = &kh_val(h, k);
- if (dret != '\n') { // if the lines has other characters
- if (ks_getuntil(ks, 0, str, &dret) > 0 && isdigit(str->s[0])) {
- beg = atoi(str->s); // begin
- if (dret != '\n') {
- if (ks_getuntil(ks, 0, str, &dret) > 0 && isdigit(str->s[0])) {
- end = atoi(str->s); // end
- if (end < beg) end = -1;
- }
- }
- }
- }
- if (dret != '\n') while ((dret = ks_getc(ks)) > 0 && dret != '\n'); // skip the rest of the line
- if (end < 0 && beg > 0) end = beg, beg = beg - 1; // if there is only one column
- if (beg >= 0 && end > beg) {
- if (p->n == p->m) {
- p->m = p->m? p->m<<1 : 4;
- p->a = realloc(p->a, p->m * 8);
- }
- p->a[p->n++] = (uint64_t)beg<<32 | end;
- }
- }
- ks_destroy(ks);
- gzclose(fp);
- free(str->s); free(str);
- bed_index(h);
- return h;
-}
-
-void bed_destroy(void *_h)
-{
- reghash_t *h = (reghash_t*)_h;
- khint_t k;
- for (k = 0; k < kh_end(h); ++k) {
- if (kh_exist(h, k)) {
- free(kh_val(h, k).a);
- free(kh_val(h, k).idx);
- free((char*)kh_key(h, k));
- }
- }
- kh_destroy(reg, h);
-}
diff --git a/samtools-0.1.19/bgzf.c b/samtools-0.1.19/bgzf.c
deleted file mode 100644
index 880d5af..0000000
--- a/samtools-0.1.19/bgzf.c
+++ /dev/null
@@ -1,694 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
- 2011 Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <assert.h>
-#include <pthread.h>
-#include <sys/types.h>
-#include "bgzf.h"
-
-#ifdef _USE_KNETFILE
-#include "knetfile.h"
-typedef knetFile *_bgzf_file_t;
-#define _bgzf_open(fn, mode) knet_open(fn, mode)
-#define _bgzf_dopen(fp, mode) knet_dopen(fp, mode)
-#define _bgzf_close(fp) knet_close(fp)
-#define _bgzf_fileno(fp) ((fp)->fd)
-#define _bgzf_tell(fp) knet_tell(fp)
-#define _bgzf_seek(fp, offset, whence) knet_seek(fp, offset, whence)
-#define _bgzf_read(fp, buf, len) knet_read(fp, buf, len)
-#define _bgzf_write(fp, buf, len) knet_write(fp, buf, len)
-#else // ~defined(_USE_KNETFILE)
-#if defined(_WIN32) || defined(_MSC_VER)
-#define ftello(fp) ftell(fp)
-#define fseeko(fp, offset, whence) fseek(fp, offset, whence)
-#else // ~defined(_WIN32)
-extern off_t ftello(FILE *stream);
-extern int fseeko(FILE *stream, off_t offset, int whence);
-#endif // ~defined(_WIN32)
-typedef FILE *_bgzf_file_t;
-#define _bgzf_open(fn, mode) fopen(fn, mode)
-#define _bgzf_dopen(fp, mode) fdopen(fp, mode)
-#define _bgzf_close(fp) fclose(fp)
-#define _bgzf_fileno(fp) fileno(fp)
-#define _bgzf_tell(fp) ftello(fp)
-#define _bgzf_seek(fp, offset, whence) fseeko(fp, offset, whence)
-#define _bgzf_read(fp, buf, len) fread(buf, 1, len, fp)
-#define _bgzf_write(fp, buf, len) fwrite(buf, 1, len, fp)
-#endif // ~define(_USE_KNETFILE)
-
-#define BLOCK_HEADER_LENGTH 18
-#define BLOCK_FOOTER_LENGTH 8
-
-
-/* BGZF/GZIP header (speciallized from RFC 1952; little endian):
- +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
- | 31|139| 8| 4| 0| 0|255| 6| 66| 67| 2|BLK_LEN|
- +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
-*/
-static const uint8_t g_magic[19] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\0\0";
-
-#ifdef BGZF_CACHE
-typedef struct {
- int size;
- uint8_t *block;
- int64_t end_offset;
-} cache_t;
-#include "khash.h"
-KHASH_MAP_INIT_INT64(cache, cache_t)
-#endif
-
-static inline void packInt16(uint8_t *buffer, uint16_t value)
-{
- buffer[0] = value;
- buffer[1] = value >> 8;
-}
-
-static inline int unpackInt16(const uint8_t *buffer)
-{
- return buffer[0] | buffer[1] << 8;
-}
-
-static inline void packInt32(uint8_t *buffer, uint32_t value)
-{
- buffer[0] = value;
- buffer[1] = value >> 8;
- buffer[2] = value >> 16;
- buffer[3] = value >> 24;
-}
-
-static BGZF *bgzf_read_init()
-{
- BGZF *fp;
- fp = calloc(1, sizeof(BGZF));
- fp->is_write = 0;
- fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
- fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
-#ifdef BGZF_CACHE
- fp->cache = kh_init(cache);
-#endif
- return fp;
-}
-
-static BGZF *bgzf_write_init(int compress_level) // compress_level==-1 for the default level
-{
- BGZF *fp;
- fp = calloc(1, sizeof(BGZF));
- fp->is_write = 1;
- fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
- fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
- fp->compress_level = compress_level < 0? Z_DEFAULT_COMPRESSION : compress_level; // Z_DEFAULT_COMPRESSION==-1
- if (fp->compress_level > 9) fp->compress_level = Z_DEFAULT_COMPRESSION;
- return fp;
-}
-// get the compress level from the mode string
-static int mode2level(const char *__restrict mode)
-{
- int i, compress_level = -1;
- for (i = 0; mode[i]; ++i)
- if (mode[i] >= '0' && mode[i] <= '9') break;
- if (mode[i]) compress_level = (int)mode[i] - '0';
- if (strchr(mode, 'u')) compress_level = 0;
- return compress_level;
-}
-
-BGZF *bgzf_open(const char *path, const char *mode)
-{
- BGZF *fp = 0;
- assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);
- if (strchr(mode, 'r') || strchr(mode, 'R')) {
- _bgzf_file_t fpr;
- if ((fpr = _bgzf_open(path, "r")) == 0) return 0;
- fp = bgzf_read_init();
- fp->fp = fpr;
- } else if (strchr(mode, 'w') || strchr(mode, 'W')) {
- FILE *fpw;
- if ((fpw = fopen(path, "w")) == 0) return 0;
- fp = bgzf_write_init(mode2level(mode));
- fp->fp = fpw;
- }
- return fp;
-}
-
-BGZF *bgzf_dopen(int fd, const char *mode)
-{
- BGZF *fp = 0;
- assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);
- if (strchr(mode, 'r') || strchr(mode, 'R')) {
- _bgzf_file_t fpr;
- if ((fpr = _bgzf_dopen(fd, "r")) == 0) return 0;
- fp = bgzf_read_init();
- fp->fp = fpr;
- } else if (strchr(mode, 'w') || strchr(mode, 'W')) {
- FILE *fpw;
- if ((fpw = fdopen(fd, "w")) == 0) return 0;
- fp = bgzf_write_init(mode2level(mode));
- fp->fp = fpw;
- }
- return fp;
-}
-
-static int bgzf_compress(void *_dst, int *dlen, void *src, int slen, int level)
-{
- uint32_t crc;
- z_stream zs;
- uint8_t *dst = (uint8_t*)_dst;
-
- // compress the body
- zs.zalloc = NULL; zs.zfree = NULL;
- zs.next_in = src;
- zs.avail_in = slen;
- zs.next_out = dst + BLOCK_HEADER_LENGTH;
- zs.avail_out = *dlen - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH;
- if (deflateInit2(&zs, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) return -1; // -15 to disable zlib header/footer
- if (deflate(&zs, Z_FINISH) != Z_STREAM_END) return -1;
- if (deflateEnd(&zs) != Z_OK) return -1;
- *dlen = zs.total_out + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
- // write the header
- memcpy(dst, g_magic, BLOCK_HEADER_LENGTH); // the last two bytes are a place holder for the length of the block
- packInt16(&dst[16], *dlen - 1); // write the compressed length; -1 to fit 2 bytes
- // write the footer
- crc = crc32(crc32(0L, NULL, 0L), src, slen);
- packInt32((uint8_t*)&dst[*dlen - 8], crc);
- packInt32((uint8_t*)&dst[*dlen - 4], slen);
- return 0;
-}
-
-// Deflate the block in fp->uncompressed_block into fp->compressed_block. Also adds an extra field that stores the compressed block length.
-static int deflate_block(BGZF *fp, int block_length)
-{
- int comp_size = BGZF_MAX_BLOCK_SIZE;
- if (bgzf_compress(fp->compressed_block, &comp_size, fp->uncompressed_block, block_length, fp->compress_level) != 0) {
- fp->errcode |= BGZF_ERR_ZLIB;
- return -1;
- }
- fp->block_offset = 0;
- return comp_size;
-}
-
-// Inflate the block in fp->compressed_block into fp->uncompressed_block
-static int inflate_block(BGZF* fp, int block_length)
-{
- z_stream zs;
- zs.zalloc = NULL;
- zs.zfree = NULL;
- zs.next_in = fp->compressed_block + 18;
- zs.avail_in = block_length - 16;
- zs.next_out = fp->uncompressed_block;
- zs.avail_out = BGZF_MAX_BLOCK_SIZE;
-
- if (inflateInit2(&zs, -15) != Z_OK) {
- fp->errcode |= BGZF_ERR_ZLIB;
- return -1;
- }
- if (inflate(&zs, Z_FINISH) != Z_STREAM_END) {
- inflateEnd(&zs);
- fp->errcode |= BGZF_ERR_ZLIB;
- return -1;
- }
- if (inflateEnd(&zs) != Z_OK) {
- fp->errcode |= BGZF_ERR_ZLIB;
- return -1;
- }
- return zs.total_out;
-}
-
-static int check_header(const uint8_t *header)
-{
- return (header[0] == 31 && header[1] == 139 && header[2] == 8 && (header[3] & 4) != 0
- && unpackInt16((uint8_t*)&header[10]) == 6
- && header[12] == 'B' && header[13] == 'C'
- && unpackInt16((uint8_t*)&header[14]) == 2);
-}
-
-#ifdef BGZF_CACHE
-static void free_cache(BGZF *fp)
-{
- khint_t k;
- khash_t(cache) *h = (khash_t(cache)*)fp->cache;
- if (fp->is_write) return;
- for (k = kh_begin(h); k < kh_end(h); ++k)
- if (kh_exist(h, k)) free(kh_val(h, k).block);
- kh_destroy(cache, h);
-}
-
-static int load_block_from_cache(BGZF *fp, int64_t block_address)
-{
- khint_t k;
- cache_t *p;
- khash_t(cache) *h = (khash_t(cache)*)fp->cache;
- k = kh_get(cache, h, block_address);
- if (k == kh_end(h)) return 0;
- p = &kh_val(h, k);
- if (fp->block_length != 0) fp->block_offset = 0;
- fp->block_address = block_address;
- fp->block_length = p->size;
- memcpy(fp->uncompressed_block, p->block, BGZF_MAX_BLOCK_SIZE);
- _bgzf_seek((_bgzf_file_t)fp->fp, p->end_offset, SEEK_SET);
- return p->size;
-}
-
-static void cache_block(BGZF *fp, int size)
-{
- int ret;
- khint_t k;
- cache_t *p;
- khash_t(cache) *h = (khash_t(cache)*)fp->cache;
- if (BGZF_MAX_BLOCK_SIZE >= fp->cache_size) return;
- if ((kh_size(h) + 1) * BGZF_MAX_BLOCK_SIZE > fp->cache_size) {
- /* A better way would be to remove the oldest block in the
- * cache, but here we remove a random one for simplicity. This
- * should not have a big impact on performance. */
- for (k = kh_begin(h); k < kh_end(h); ++k)
- if (kh_exist(h, k)) break;
- if (k < kh_end(h)) {
- free(kh_val(h, k).block);
- kh_del(cache, h, k);
- }
- }
- k = kh_put(cache, h, fp->block_address, &ret);
- if (ret == 0) return; // if this happens, a bug!
- p = &kh_val(h, k);
- p->size = fp->block_length;
- p->end_offset = fp->block_address + size;
- p->block = malloc(BGZF_MAX_BLOCK_SIZE);
- memcpy(kh_val(h, k).block, fp->uncompressed_block, BGZF_MAX_BLOCK_SIZE);
-}
-#else
-static void free_cache(BGZF *fp) {}
-static int load_block_from_cache(BGZF *fp, int64_t block_address) {return 0;}
-static void cache_block(BGZF *fp, int size) {}
-#endif
-
-int bgzf_read_block(BGZF *fp)
-{
- uint8_t header[BLOCK_HEADER_LENGTH], *compressed_block;
- int count, size = 0, block_length, remaining;
- int64_t block_address;
- block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
- if (fp->cache_size && load_block_from_cache(fp, block_address)) return 0;
- count = _bgzf_read(fp->fp, header, sizeof(header));
- if (count == 0) { // no data read
- fp->block_length = 0;
- return 0;
- }
- if (count != sizeof(header) || !check_header(header)) {
- fp->errcode |= BGZF_ERR_HEADER;
- return -1;
- }
- size = count;
- block_length = unpackInt16((uint8_t*)&header[16]) + 1; // +1 because when writing this number, we used "-1"
- compressed_block = (uint8_t*)fp->compressed_block;
- memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);
- remaining = block_length - BLOCK_HEADER_LENGTH;
- count = _bgzf_read(fp->fp, &compressed_block[BLOCK_HEADER_LENGTH], remaining);
- if (count != remaining) {
- fp->errcode |= BGZF_ERR_IO;
- return -1;
- }
- size += count;
- if ((count = inflate_block(fp, block_length)) < 0) return -1;
- if (fp->block_length != 0) fp->block_offset = 0; // Do not reset offset if this read follows a seek.
- fp->block_address = block_address;
- fp->block_length = count;
- cache_block(fp, size);
- return 0;
-}
-
-ssize_t bgzf_read(BGZF *fp, void *data, ssize_t length)
-{
- ssize_t bytes_read = 0;
- uint8_t *output = data;
- if (length <= 0) return 0;
- assert(fp->is_write == 0);
- while (bytes_read < length) {
- int copy_length, available = fp->block_length - fp->block_offset;
- uint8_t *buffer;
- if (available <= 0) {
- if (bgzf_read_block(fp) != 0) return -1;
- available = fp->block_length - fp->block_offset;
- if (available <= 0) break;
- }
- copy_length = length - bytes_read < available? length - bytes_read : available;
- buffer = fp->uncompressed_block;
- memcpy(output, buffer + fp->block_offset, copy_length);
- fp->block_offset += copy_length;
- output += copy_length;
- bytes_read += copy_length;
- }
- if (fp->block_offset == fp->block_length) {
- fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
- fp->block_offset = fp->block_length = 0;
- }
- return bytes_read;
-}
-
-/***** BEGIN: multi-threading *****/
-
-typedef struct {
- BGZF *fp;
- struct mtaux_t *mt;
- void *buf;
- int i, errcode, toproc;
-} worker_t;
-
-typedef struct mtaux_t {
- int n_threads, n_blks, curr, done;
- volatile int proc_cnt;
- void **blk;
- int *len;
- worker_t *w;
- pthread_t *tid;
- pthread_mutex_t lock;
- pthread_cond_t cv;
-} mtaux_t;
-
-static int worker_aux(worker_t *w)
-{
- int i, tmp, stop = 0;
- // wait for condition: to process or all done
- pthread_mutex_lock(&w->mt->lock);
- while (!w->toproc && !w->mt->done)
- pthread_cond_wait(&w->mt->cv, &w->mt->lock);
- if (w->mt->done) stop = 1;
- w->toproc = 0;
- pthread_mutex_unlock(&w->mt->lock);
- if (stop) return 1; // to quit the thread
- w->errcode = 0;
- for (i = w->i; i < w->mt->curr; i += w->mt->n_threads) {
- int clen = BGZF_MAX_BLOCK_SIZE;
- if (bgzf_compress(w->buf, &clen, w->mt->blk[i], w->mt->len[i], w->fp->compress_level) != 0)
- w->errcode |= BGZF_ERR_ZLIB;
- memcpy(w->mt->blk[i], w->buf, clen);
- w->mt->len[i] = clen;
- }
- tmp = __sync_fetch_and_add(&w->mt->proc_cnt, 1);
- return 0;
-}
-
-static void *mt_worker(void *data)
-{
- while (worker_aux(data) == 0);
- return 0;
-}
-
-int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks)
-{
- int i;
- mtaux_t *mt;
- pthread_attr_t attr;
- if (!fp->is_write || fp->mt || n_threads <= 1) return -1;
- mt = calloc(1, sizeof(mtaux_t));
- mt->n_threads = n_threads;
- mt->n_blks = n_threads * n_sub_blks;
- mt->len = calloc(mt->n_blks, sizeof(int));
- mt->blk = calloc(mt->n_blks, sizeof(void*));
- for (i = 0; i < mt->n_blks; ++i)
- mt->blk[i] = malloc(BGZF_MAX_BLOCK_SIZE);
- mt->tid = calloc(mt->n_threads, sizeof(pthread_t)); // tid[0] is not used, as the worker 0 is launched by the master
- mt->w = calloc(mt->n_threads, sizeof(worker_t));
- for (i = 0; i < mt->n_threads; ++i) {
- mt->w[i].i = i;
- mt->w[i].mt = mt;
- mt->w[i].fp = fp;
- mt->w[i].buf = malloc(BGZF_MAX_BLOCK_SIZE);
- }
- pthread_attr_init(&attr);
- pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
- pthread_mutex_init(&mt->lock, 0);
- pthread_cond_init(&mt->cv, 0);
- for (i = 1; i < mt->n_threads; ++i) // worker 0 is effectively launched by the master thread
- pthread_create(&mt->tid[i], &attr, mt_worker, &mt->w[i]);
- fp->mt = mt;
- return 0;
-}
-
-static void mt_destroy(mtaux_t *mt)
-{
- int i;
- // signal all workers to quit
- pthread_mutex_lock(&mt->lock);
- mt->done = 1; mt->proc_cnt = 0;
- pthread_cond_broadcast(&mt->cv);
- pthread_mutex_unlock(&mt->lock);
- for (i = 1; i < mt->n_threads; ++i) pthread_join(mt->tid[i], 0); // worker 0 is effectively launched by the master thread
- // free other data allocated on heap
- for (i = 0; i < mt->n_blks; ++i) free(mt->blk[i]);
- for (i = 0; i < mt->n_threads; ++i) free(mt->w[i].buf);
- free(mt->blk); free(mt->len); free(mt->w); free(mt->tid);
- pthread_cond_destroy(&mt->cv);
- pthread_mutex_destroy(&mt->lock);
- free(mt);
-}
-
-static void mt_queue(BGZF *fp)
-{
- mtaux_t *mt = (mtaux_t*)fp->mt;
- assert(mt->curr < mt->n_blks); // guaranteed by the caller
- memcpy(mt->blk[mt->curr], fp->uncompressed_block, fp->block_offset);
- mt->len[mt->curr] = fp->block_offset;
- fp->block_offset = 0;
- ++mt->curr;
-}
-
-static int mt_flush(BGZF *fp)
-{
- int i;
- mtaux_t *mt = (mtaux_t*)fp->mt;
- if (fp->block_offset) mt_queue(fp); // guaranteed that assertion does not fail
- // signal all the workers to compress
- pthread_mutex_lock(&mt->lock);
- for (i = 0; i < mt->n_threads; ++i) mt->w[i].toproc = 1;
- mt->proc_cnt = 0;
- pthread_cond_broadcast(&mt->cv);
- pthread_mutex_unlock(&mt->lock);
- // worker 0 is doing things here
- worker_aux(&mt->w[0]);
- // wait for all the threads to complete
- while (mt->proc_cnt < mt->n_threads);
- // dump data to disk
- for (i = 0; i < mt->n_threads; ++i) fp->errcode |= mt->w[i].errcode;
- for (i = 0; i < mt->curr; ++i)
- if (fwrite(mt->blk[i], 1, mt->len[i], fp->fp) != mt->len[i])
- fp->errcode |= BGZF_ERR_IO;
- mt->curr = 0;
- return 0;
-}
-
-static int mt_lazy_flush(BGZF *fp)
-{
- mtaux_t *mt = (mtaux_t*)fp->mt;
- if (fp->block_offset) mt_queue(fp);
- if (mt->curr == mt->n_blks)
- return mt_flush(fp);
- return -1;
-}
-
-static ssize_t mt_write(BGZF *fp, const void *data, ssize_t length)
-{
- const uint8_t *input = data;
- ssize_t rest = length;
- while (rest) {
- int copy_length = BGZF_BLOCK_SIZE - fp->block_offset < rest? BGZF_BLOCK_SIZE - fp->block_offset : rest;
- memcpy(fp->uncompressed_block + fp->block_offset, input, copy_length);
- fp->block_offset += copy_length; input += copy_length; rest -= copy_length;
- if (fp->block_offset == BGZF_BLOCK_SIZE) mt_lazy_flush(fp);
- }
- return length - rest;
-}
-
-/***** END: multi-threading *****/
-
-int bgzf_flush(BGZF *fp)
-{
- if (!fp->is_write) return 0;
- if (fp->mt) return mt_flush(fp);
- while (fp->block_offset > 0) {
- int block_length;
- block_length = deflate_block(fp, fp->block_offset);
- if (block_length < 0) return -1;
- if (fwrite(fp->compressed_block, 1, block_length, fp->fp) != block_length) {
- fp->errcode |= BGZF_ERR_IO; // possibly truncated file
- return -1;
- }
- fp->block_address += block_length;
- }
- return 0;
-}
-
-int bgzf_flush_try(BGZF *fp, ssize_t size)
-{
- if (fp->block_offset + size > BGZF_BLOCK_SIZE) {
- if (fp->mt) return mt_lazy_flush(fp);
- else return bgzf_flush(fp);
- }
- return -1;
-}
-
-ssize_t bgzf_write(BGZF *fp, const void *data, ssize_t length)
-{
- const uint8_t *input = data;
- int block_length = BGZF_BLOCK_SIZE, bytes_written = 0;
- assert(fp->is_write);
- if (fp->mt) return mt_write(fp, data, length);
- while (bytes_written < length) {
- uint8_t* buffer = fp->uncompressed_block;
- int copy_length = block_length - fp->block_offset < length - bytes_written? block_length - fp->block_offset : length - bytes_written;
- memcpy(buffer + fp->block_offset, input, copy_length);
- fp->block_offset += copy_length;
- input += copy_length;
- bytes_written += copy_length;
- if (fp->block_offset == block_length && bgzf_flush(fp)) break;
- }
- return bytes_written;
-}
-
-int bgzf_close(BGZF* fp)
-{
- int ret, count, block_length;
- if (fp == 0) return -1;
- if (fp->is_write) {
- if (bgzf_flush(fp) != 0) return -1;
- fp->compress_level = -1;
- block_length = deflate_block(fp, 0); // write an empty block
- count = fwrite(fp->compressed_block, 1, block_length, fp->fp);
- if (fflush(fp->fp) != 0) {
- fp->errcode |= BGZF_ERR_IO;
- return -1;
- }
- if (fp->mt) mt_destroy(fp->mt);
- }
- ret = fp->is_write? fclose(fp->fp) : _bgzf_close(fp->fp);
- if (ret != 0) return -1;
- free(fp->uncompressed_block);
- free(fp->compressed_block);
- free_cache(fp);
- free(fp);
- return 0;
-}
-
-void bgzf_set_cache_size(BGZF *fp, int cache_size)
-{
- if (fp) fp->cache_size = cache_size;
-}
-
-int bgzf_check_EOF(BGZF *fp)
-{
- static uint8_t magic[28] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0";
- uint8_t buf[28];
- off_t offset;
- offset = _bgzf_tell((_bgzf_file_t)fp->fp);
- if (_bgzf_seek(fp->fp, -28, SEEK_END) < 0) return 0;
- _bgzf_read(fp->fp, buf, 28);
- _bgzf_seek(fp->fp, offset, SEEK_SET);
- return (memcmp(magic, buf, 28) == 0)? 1 : 0;
-}
-
-int64_t bgzf_seek(BGZF* fp, int64_t pos, int where)
-{
- int block_offset;
- int64_t block_address;
-
- if (fp->is_write || where != SEEK_SET) {
- fp->errcode |= BGZF_ERR_MISUSE;
- return -1;
- }
- block_offset = pos & 0xFFFF;
- block_address = pos >> 16;
- if (_bgzf_seek(fp->fp, block_address, SEEK_SET) < 0) {
- fp->errcode |= BGZF_ERR_IO;
- return -1;
- }
- fp->block_length = 0; // indicates current block has not been loaded
- fp->block_address = block_address;
- fp->block_offset = block_offset;
- return 0;
-}
-
-int bgzf_is_bgzf(const char *fn)
-{
- uint8_t buf[16];
- int n;
- _bgzf_file_t fp;
- if ((fp = _bgzf_open(fn, "r")) == 0) return 0;
- n = _bgzf_read(fp, buf, 16);
- _bgzf_close(fp);
- if (n != 16) return 0;
- return memcmp(g_magic, buf, 16) == 0? 1 : 0;
-}
-
-int bgzf_getc(BGZF *fp)
-{
- int c;
- if (fp->block_offset >= fp->block_length) {
- if (bgzf_read_block(fp) != 0) return -2; /* error */
- if (fp->block_length == 0) return -1; /* end-of-file */
- }
- c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];
- if (fp->block_offset == fp->block_length) {
- fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
- fp->block_offset = 0;
- fp->block_length = 0;
- }
- return c;
-}
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
-int bgzf_getline(BGZF *fp, int delim, kstring_t *str)
-{
- int l, state = 0;
- unsigned char *buf = (unsigned char*)fp->uncompressed_block;
- str->l = 0;
- do {
- if (fp->block_offset >= fp->block_length) {
- if (bgzf_read_block(fp) != 0) { state = -2; break; }
- if (fp->block_length == 0) { state = -1; break; }
- }
- for (l = fp->block_offset; l < fp->block_length && buf[l] != delim; ++l);
- if (l < fp->block_length) state = 1;
- l -= fp->block_offset;
- if (str->l + l + 1 >= str->m) {
- str->m = str->l + l + 2;
- kroundup32(str->m);
- str->s = (char*)realloc(str->s, str->m);
- }
- memcpy(str->s + str->l, buf + fp->block_offset, l);
- str->l += l;
- fp->block_offset += l + 1;
- if (fp->block_offset >= fp->block_length) {
- fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
- fp->block_offset = 0;
- fp->block_length = 0;
- }
- } while (state == 0);
- if (str->l == 0 && state < 0) return state;
- str->s[str->l] = 0;
- return str->l;
-}
diff --git a/samtools-0.1.19/bgzf.h b/samtools-0.1.19/bgzf.h
deleted file mode 100644
index cb67681..0000000
--- a/samtools-0.1.19/bgzf.h
+++ /dev/null
@@ -1,207 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
- 2011, 2012 Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-/* The BGZF library was originally written by Bob Handsaker from the Broad
- * Institute. It was later improved by the SAMtools developers. */
-
-#ifndef __BGZF_H
-#define __BGZF_H
-
-#include <stdint.h>
-#include <stdio.h>
-#include <zlib.h>
-#include <sys/types.h>
-
-#define BGZF_BLOCK_SIZE 0xff00 // make sure compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE
-#define BGZF_MAX_BLOCK_SIZE 0x10000
-
-#define BGZF_ERR_ZLIB 1
-#define BGZF_ERR_HEADER 2
-#define BGZF_ERR_IO 4
-#define BGZF_ERR_MISUSE 8
-
-typedef struct {
- int errcode:16, is_write:2, compress_level:14;
- int cache_size;
- int block_length, block_offset;
- int64_t block_address;
- void *uncompressed_block, *compressed_block;
- void *cache; // a pointer to a hash table
- void *fp; // actual file handler; FILE* on writing; FILE* or knetFile* on reading
- void *mt; // only used for multi-threading
-} BGZF;
-
-#ifndef KSTRING_T
-#define KSTRING_T kstring_t
-typedef struct __kstring_t {
- size_t l, m;
- char *s;
-} kstring_t;
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- /******************
- * Basic routines *
- ******************/
-
- /**
- * Open an existing file descriptor for reading or writing.
- *
- * @param fd file descriptor
- * @param mode mode matching /[rwu0-9]+/: 'r' for reading, 'w' for writing and a digit specifies
- * the zlib compression level; if both 'r' and 'w' are present, 'w' is ignored.
- * @return BGZF file handler; 0 on error
- */
- BGZF* bgzf_dopen(int fd, const char *mode);
-
- #define bgzf_fdopen(fd, mode) bgzf_dopen((fd), (mode)) // for backward compatibility
-
- /**
- * Open the specified file for reading or writing.
- */
- BGZF* bgzf_open(const char* path, const char *mode);
-
- /**
- * Close the BGZF and free all associated resources.
- *
- * @param fp BGZF file handler
- * @return 0 on success and -1 on error
- */
- int bgzf_close(BGZF *fp);
-
- /**
- * Read up to _length_ bytes from the file storing into _data_.
- *
- * @param fp BGZF file handler
- * @param data data array to read into
- * @param length size of data to read
- * @return number of bytes actually read; 0 on end-of-file and -1 on error
- */
- ssize_t bgzf_read(BGZF *fp, void *data, ssize_t length);
-
- /**
- * Write _length_ bytes from _data_ to the file.
- *
- * @param fp BGZF file handler
- * @param data data array to write
- * @param length size of data to write
- * @return number of bytes actually written; -1 on error
- */
- ssize_t bgzf_write(BGZF *fp, const void *data, ssize_t length);
-
- /**
- * Write the data in the buffer to the file.
- */
- int bgzf_flush(BGZF *fp);
-
- /**
- * Return a virtual file pointer to the current location in the file.
- * No interpetation of the value should be made, other than a subsequent
- * call to bgzf_seek can be used to position the file at the same point.
- * Return value is non-negative on success.
- */
- #define bgzf_tell(fp) ((fp->block_address << 16) | (fp->block_offset & 0xFFFF))
-
- /**
- * Set the file to read from the location specified by _pos_.
- *
- * @param fp BGZF file handler
- * @param pos virtual file offset returned by bgzf_tell()
- * @param whence must be SEEK_SET
- * @return 0 on success and -1 on error
- */
- int64_t bgzf_seek(BGZF *fp, int64_t pos, int whence);
-
- /**
- * Check if the BGZF end-of-file (EOF) marker is present
- *
- * @param fp BGZF file handler opened for reading
- * @return 1 if EOF is present; 0 if not or on I/O error
- */
- int bgzf_check_EOF(BGZF *fp);
-
- /**
- * Check if a file is in the BGZF format
- *
- * @param fn file name
- * @return 1 if _fn_ is BGZF; 0 if not or on I/O error
- */
- int bgzf_is_bgzf(const char *fn);
-
- /*********************
- * Advanced routines *
- *********************/
-
- /**
- * Set the cache size. Only effective when compiled with -DBGZF_CACHE.
- *
- * @param fp BGZF file handler
- * @param size size of cache in bytes; 0 to disable caching (default)
- */
- void bgzf_set_cache_size(BGZF *fp, int size);
-
- /**
- * Flush the file if the remaining buffer size is smaller than _size_
- */
- int bgzf_flush_try(BGZF *fp, ssize_t size);
-
- /**
- * Read one byte from a BGZF file. It is faster than bgzf_read()
- * @param fp BGZF file handler
- * @return byte read; -1 on end-of-file or error
- */
- int bgzf_getc(BGZF *fp);
-
- /**
- * Read one line from a BGZF file. It is faster than bgzf_getc()
- *
- * @param fp BGZF file handler
- * @param delim delimitor
- * @param str string to write to; must be initialized
- * @return length of the string; 0 on end-of-file; negative on error
- */
- int bgzf_getline(BGZF *fp, int delim, kstring_t *str);
-
- /**
- * Read the next BGZF block.
- */
- int bgzf_read_block(BGZF *fp);
-
- /**
- * Enable multi-threading (only effective on writing)
- *
- * @param fp BGZF file handler; must be opened for writing
- * @param n_threads #threads used for writing
- * @param n_sub_blks #blocks processed by each thread; a value 64-256 is recommended
- */
- int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/bgzip.c b/samtools-0.1.19/bgzip.c
deleted file mode 100644
index ebcafa2..0000000
--- a/samtools-0.1.19/bgzip.c
+++ /dev/null
@@ -1,206 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/select.h>
-#include <sys/stat.h>
-#include "bgzf.h"
-
-static const int WINDOW_SIZE = 64 * 1024;
-
-static int bgzip_main_usage()
-{
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: bgzip [options] [file] ...\n\n");
- fprintf(stderr, "Options: -c write on standard output, keep original files unchanged\n");
- fprintf(stderr, " -d decompress\n");
- fprintf(stderr, " -f overwrite files without asking\n");
- fprintf(stderr, " -b INT decompress at virtual file pointer INT\n");
- fprintf(stderr, " -s INT decompress INT bytes in the uncompressed file\n");
- fprintf(stderr, " -h give this help\n");
- fprintf(stderr, "\n");
- return 1;
-}
-
-static int write_open(const char *fn, int is_forced)
-{
- int fd = -1;
- char c;
- if (!is_forced) {
- if ((fd = open(fn, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0666)) < 0 && errno == EEXIST) {
- fprintf(stderr, "[bgzip] %s already exists; do you wish to overwrite (y or n)? ", fn);
- scanf("%c", &c);
- if (c != 'Y' && c != 'y') {
- fprintf(stderr, "[bgzip] not overwritten\n");
- exit(1);
- }
- }
- }
- if (fd < 0) {
- if ((fd = open(fn, O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0) {
- fprintf(stderr, "[bgzip] %s: Fail to write\n", fn);
- exit(1);
- }
- }
- return fd;
-}
-
-static void fail(BGZF* fp)
-{
- fprintf(stderr, "Error: %s\n", fp->error);
- exit(1);
-}
-
-int main(int argc, char **argv)
-{
- int c, compress, pstdout, is_forced;
- BGZF *fp;
- void *buffer;
- long start, end, size;
-
- compress = 1; pstdout = 0; start = 0; size = -1; end = -1; is_forced = 0;
- while((c = getopt(argc, argv, "cdhfb:s:")) >= 0){
- switch(c){
- case 'h': return bgzip_main_usage();
- case 'd': compress = 0; break;
- case 'c': pstdout = 1; break;
- case 'b': start = atol(optarg); break;
- case 's': size = atol(optarg); break;
- case 'f': is_forced = 1; break;
- }
- }
- if (size >= 0) end = start + size;
- if (end >= 0 && end < start) {
- fprintf(stderr, "[bgzip] Illegal region: [%ld, %ld]\n", start, end);
- return 1;
- }
- if (compress == 1) {
- struct stat sbuf;
- int f_src = fileno(stdin);
- int f_dst = fileno(stdout);
-
- if ( argc>optind )
- {
- if ( stat(argv[optind],&sbuf)<0 )
- {
- fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);
- return 1;
- }
-
- if ((f_src = open(argv[optind], O_RDONLY)) < 0) {
- fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);
- return 1;
- }
-
- if (pstdout)
- f_dst = fileno(stdout);
- else
- {
- char *name = malloc(strlen(argv[optind]) + 5);
- strcpy(name, argv[optind]);
- strcat(name, ".gz");
- f_dst = write_open(name, is_forced);
- if (f_dst < 0) return 1;
- free(name);
- }
- }
- else if (!pstdout && isatty(fileno((FILE *)stdout)) )
- return bgzip_main_usage();
-
- fp = bgzf_fdopen(f_dst, "w");
- buffer = malloc(WINDOW_SIZE);
- while ((c = read(f_src, buffer, WINDOW_SIZE)) > 0)
- if (bgzf_write(fp, buffer, c) < 0) fail(fp);
- // f_dst will be closed here
- if (bgzf_close(fp) < 0) fail(fp);
- if (argc > optind && !pstdout) unlink(argv[optind]);
- free(buffer);
- close(f_src);
- return 0;
- } else {
- struct stat sbuf;
- int f_dst;
-
- if ( argc>optind )
- {
- if ( stat(argv[optind],&sbuf)<0 )
- {
- fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), argv[optind]);
- return 1;
- }
- char *name;
- int len = strlen(argv[optind]);
- if ( strcmp(argv[optind]+len-3,".gz") )
- {
- fprintf(stderr, "[bgzip] %s: unknown suffix -- ignored\n", argv[optind]);
- return 1;
- }
- fp = bgzf_open(argv[optind], "r");
- if (fp == NULL) {
- fprintf(stderr, "[bgzip] Could not open file: %s\n", argv[optind]);
- return 1;
- }
-
- if (pstdout) {
- f_dst = fileno(stdout);
- }
- else {
- name = strdup(argv[optind]);
- name[strlen(name) - 3] = '\0';
- f_dst = write_open(name, is_forced);
- free(name);
- }
- }
- else if (!pstdout && isatty(fileno((FILE *)stdin)) )
- return bgzip_main_usage();
- else
- {
- f_dst = fileno(stdout);
- fp = bgzf_fdopen(fileno(stdin), "r");
- if (fp == NULL) {
- fprintf(stderr, "[bgzip] Could not read from stdin: %s\n", strerror(errno));
- return 1;
- }
- }
- buffer = malloc(WINDOW_SIZE);
- if (bgzf_seek(fp, start, SEEK_SET) < 0) fail(fp);
- while (1) {
- if (end < 0) c = bgzf_read(fp, buffer, WINDOW_SIZE);
- else c = bgzf_read(fp, buffer, (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start));
- if (c == 0) break;
- if (c < 0) fail(fp);
- start += c;
- write(f_dst, buffer, c);
- if (end >= 0 && start >= end) break;
- }
- free(buffer);
- if (bgzf_close(fp) < 0) fail(fp);
- if (!pstdout) unlink(argv[optind]);
- return 0;
- }
-}
diff --git a/samtools-0.1.19/cut_target.c b/samtools-0.1.19/cut_target.c
deleted file mode 100644
index 26f434f..0000000
--- a/samtools-0.1.19/cut_target.c
+++ /dev/null
@@ -1,193 +0,0 @@
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include "bam.h"
-#include "errmod.h"
-#include "faidx.h"
-
-#define ERR_DEP 0.83f
-
-typedef struct {
- int e[2][3], p[2][2];
-} score_param_t;
-
-/* Note that although the two matrics have 10 parameters in total, only 4
- * (probably 3) are free. Changing the scoring matrices in a sort of symmetric
- * way will not change the result. */
-static score_param_t g_param = { {{0,0,0},{-4,1,6}}, {{0,-14000}, {0,0}} };
-
-typedef struct {
- int min_baseQ, tid, max_bases;
- uint16_t *bases;
- bamFile fp;
- bam_header_t *h;
- char *ref;
- faidx_t *fai;
- errmod_t *em;
-} ct_t;
-
-static uint16_t gencns(ct_t *g, int n, const bam_pileup1_t *plp)
-{
- int i, j, ret, tmp, k, sum[4], qual;
- float q[16];
- if (n > g->max_bases) { // enlarge g->bases
- g->max_bases = n;
- kroundup32(g->max_bases);
- g->bases = realloc(g->bases, g->max_bases * 2);
- }
- for (i = k = 0; i < n; ++i) {
- const bam_pileup1_t *p = plp + i;
- uint8_t *seq;
- int q, baseQ, b;
- if (p->is_refskip || p->is_del) continue;
- baseQ = bam1_qual(p->b)[p->qpos];
- if (baseQ < g->min_baseQ) continue;
- seq = bam1_seq(p->b);
- b = bam_nt16_nt4_table[bam1_seqi(seq, p->qpos)];
- if (b > 3) continue;
- q = baseQ < p->b->core.qual? baseQ : p->b->core.qual;
- if (q < 4) q = 4;
- if (q > 63) q = 63;
- g->bases[k++] = q<<5 | bam1_strand(p->b)<<4 | b;
- }
- if (k == 0) return 0;
- errmod_cal(g->em, k, 4, g->bases, q);
- for (i = 0; i < 4; ++i) sum[i] = (int)(q[i<<2|i] + .499) << 2 | i;
- for (i = 1; i < 4; ++i) // insertion sort
- for (j = i; j > 0 && sum[j] < sum[j-1]; --j)
- tmp = sum[j], sum[j] = sum[j-1], sum[j-1] = tmp;
- qual = (sum[1]>>2) - (sum[0]>>2);
- k = k < 256? k : 255;
- ret = (qual < 63? qual : 63) << 2 | (sum[0]&3);
- return ret<<8|k;
-}
-
-static void process_cns(bam_header_t *h, int tid, int l, uint16_t *cns)
-{
- int i, f[2][2], *prev, *curr, *swap_tmp, s;
- uint8_t *b; // backtrack array
- b = calloc(l, 1);
- f[0][0] = f[0][1] = 0;
- prev = f[0]; curr = f[1];
- // fill the backtrack matrix
- for (i = 0; i < l; ++i) {
- int c = (cns[i] == 0)? 0 : (cns[i]>>8 == 0)? 1 : 2;
- int tmp0, tmp1;
- // compute f[0]
- tmp0 = prev[0] + g_param.e[0][c] + g_param.p[0][0]; // (s[i+1],s[i])=(0,0)
- tmp1 = prev[1] + g_param.e[0][c] + g_param.p[1][0]; // (0,1)
- if (tmp0 > tmp1) curr[0] = tmp0, b[i] = 0;
- else curr[0] = tmp1, b[i] = 1;
- // compute f[1]
- tmp0 = prev[0] + g_param.e[1][c] + g_param.p[0][1]; // (s[i+1],s[i])=(1,0)
- tmp1 = prev[1] + g_param.e[1][c] + g_param.p[1][1]; // (1,1)
- if (tmp0 > tmp1) curr[1] = tmp0, b[i] |= 0<<1;
- else curr[1] = tmp1, b[i] |= 1<<1;
- // swap
- swap_tmp = prev; prev = curr; curr = swap_tmp;
- }
- // backtrack
- s = prev[0] > prev[1]? 0 : 1;
- for (i = l - 1; i > 0; --i) {
- b[i] |= s<<2;
- s = b[i]>>s&1;
- }
- // print
- for (i = 0, s = -1; i <= l; ++i) {
- if (i == l || ((b[i]>>2&3) == 0 && s >= 0)) {
- if (s >= 0) {
- int j;
- printf("%s:%d-%d\t0\t%s\t%d\t60\t%dM\t*\t0\t0\t", h->target_name[tid], s+1, i, h->target_name[tid], s+1, i-s);
- for (j = s; j < i; ++j) {
- int c = cns[j]>>8;
- if (c == 0) putchar('N');
- else putchar("ACGT"[c&3]);
- }
- putchar('\t');
- for (j = s; j < i; ++j)
- putchar(33 + (cns[j]>>8>>2));
- putchar('\n');
- }
- //if (s >= 0) printf("%s\t%d\t%d\t%d\n", h->target_name[tid], s, i, i - s);
- s = -1;
- } else if ((b[i]>>2&3) && s < 0) s = i;
- }
- free(b);
-}
-
-static int read_aln(void *data, bam1_t *b)
-{
- extern int bam_prob_realn_core(bam1_t *b, const char *ref, int flag);
- ct_t *g = (ct_t*)data;
- int ret, len;
- ret = bam_read1(g->fp, b);
- if (ret >= 0 && g->fai && b->core.tid >= 0 && (b->core.flag&4) == 0) {
- if (b->core.tid != g->tid) { // then load the sequence
- free(g->ref);
- g->ref = fai_fetch(g->fai, g->h->target_name[b->core.tid], &len);
- g->tid = b->core.tid;
- }
- bam_prob_realn_core(b, g->ref, 1<<1|1);
- }
- return ret;
-}
-
-int main_cut_target(int argc, char *argv[])
-{
- int c, tid, pos, n, lasttid = -1, lastpos = -1, l, max_l;
- const bam_pileup1_t *p;
- bam_plp_t plp;
- uint16_t *cns;
- ct_t g;
-
- memset(&g, 0, sizeof(ct_t));
- g.min_baseQ = 13; g.tid = -1;
- while ((c = getopt(argc, argv, "f:Q:i:o:0:1:2:")) >= 0) {
- switch (c) {
- case 'Q': g.min_baseQ = atoi(optarg); break; // quality cutoff
- case 'i': g_param.p[0][1] = -atoi(optarg); break; // 0->1 transition (in) PENALTY
- case '0': g_param.e[1][0] = atoi(optarg); break; // emission SCORE
- case '1': g_param.e[1][1] = atoi(optarg); break;
- case '2': g_param.e[1][2] = atoi(optarg); break;
- case 'f': g.fai = fai_load(optarg);
- if (g.fai == 0) fprintf(stderr, "[%s] fail to load the fasta index.\n", __func__);
- break;
- }
- }
- if (argc == optind) {
- fprintf(stderr, "Usage: samtools targetcut [-Q minQ] [-i inPen] [-0 em0] [-1 em1] [-2 em2] [-f ref] <in.bam>\n");
- return 1;
- }
- l = max_l = 0; cns = 0;
- g.fp = strcmp(argv[optind], "-")? bam_open(argv[optind], "r") : bam_dopen(fileno(stdin), "r");
- g.h = bam_header_read(g.fp);
- g.em = errmod_init(1 - ERR_DEP);
- plp = bam_plp_init(read_aln, &g);
- while ((p = bam_plp_auto(plp, &tid, &pos, &n)) != 0) {
- if (tid < 0) break;
- if (tid != lasttid) { // change of chromosome
- if (cns) process_cns(g.h, lasttid, l, cns);
- if (max_l < g.h->target_len[tid]) {
- max_l = g.h->target_len[tid];
- kroundup32(max_l);
- cns = realloc(cns, max_l * 2);
- }
- l = g.h->target_len[tid];
- memset(cns, 0, max_l * 2);
- lasttid = tid;
- }
- cns[pos] = gencns(&g, n, p);
- lastpos = pos;
- }
- process_cns(g.h, lasttid, l, cns);
- free(cns);
- bam_header_destroy(g.h);
- bam_plp_destroy(plp);
- bam_close(g.fp);
- if (g.fai) {
- fai_destroy(g.fai); free(g.ref);
- }
- errmod_destroy(g.em);
- free(g.bases);
- return 0;
-}
diff --git a/samtools-0.1.19/errmod.c b/samtools-0.1.19/errmod.c
deleted file mode 100644
index fba9a8d..0000000
--- a/samtools-0.1.19/errmod.c
+++ /dev/null
@@ -1,130 +0,0 @@
-#include <math.h>
-#include "errmod.h"
-#include "ksort.h"
-KSORT_INIT_GENERIC(uint16_t)
-
-typedef struct __errmod_coef_t {
- double *fk, *beta, *lhet;
-} errmod_coef_t;
-
-typedef struct {
- double fsum[16], bsum[16];
- uint32_t c[16];
-} call_aux_t;
-
-static errmod_coef_t *cal_coef(double depcorr, double eta)
-{
- int k, n, q;
- long double sum, sum1;
- double *lC;
- errmod_coef_t *ec;
-
- ec = calloc(1, sizeof(errmod_coef_t));
- // initialize ->fk
- ec->fk = (double*)calloc(256, sizeof(double));
- ec->fk[0] = 1.0;
- for (n = 1; n != 256; ++n)
- ec->fk[n] = pow(1. - depcorr, n) * (1.0 - eta) + eta;
- // initialize ->coef
- ec->beta = (double*)calloc(256 * 256 * 64, sizeof(double));
- lC = (double*)calloc(256 * 256, sizeof(double));
- for (n = 1; n != 256; ++n) {
- double lgn = lgamma(n+1);
- for (k = 1; k <= n; ++k)
- lC[n<<8|k] = lgn - lgamma(k+1) - lgamma(n-k+1);
- }
- for (q = 1; q != 64; ++q) {
- double e = pow(10.0, -q/10.0);
- double le = log(e);
- double le1 = log(1.0 - e);
- for (n = 1; n <= 255; ++n) {
- double *beta = ec->beta + (q<<16|n<<8);
- sum1 = sum = 0.0;
- for (k = n; k >= 0; --k, sum1 = sum) {
- sum = sum1 + expl(lC[n<<8|k] + k*le + (n-k)*le1);
- beta[k] = -10. / M_LN10 * logl(sum1 / sum);
- }
- }
- }
- // initialize ->lhet
- ec->lhet = (double*)calloc(256 * 256, sizeof(double));
- for (n = 0; n < 256; ++n)
- for (k = 0; k < 256; ++k)
- ec->lhet[n<<8|k] = lC[n<<8|k] - M_LN2 * n;
- free(lC);
- return ec;
-}
-
-errmod_t *errmod_init(float depcorr)
-{
- errmod_t *em;
- em = (errmod_t*)calloc(1, sizeof(errmod_t));
- em->depcorr = depcorr;
- em->coef = cal_coef(depcorr, 0.03);
- return em;
-}
-
-void errmod_destroy(errmod_t *em)
-{
- if (em == 0) return;
- free(em->coef->lhet); free(em->coef->fk); free(em->coef->beta);
- free(em->coef); free(em);
-}
-// qual:6, strand:1, base:4
-int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q)
-{
- call_aux_t aux;
- int i, j, k, w[32];
-
- if (m > m) return -1;
- memset(q, 0, m * m * sizeof(float));
- if (n == 0) return 0;
- // calculate aux.esum and aux.fsum
- if (n > 255) { // then sample 255 bases
- ks_shuffle(uint16_t, n, bases);
- n = 255;
- }
- ks_introsort(uint16_t, n, bases);
- memset(w, 0, 32 * sizeof(int));
- memset(&aux, 0, sizeof(call_aux_t));
- for (j = n - 1; j >= 0; --j) { // calculate esum and fsum
- uint16_t b = bases[j];
- int q = b>>5 < 4? 4 : b>>5;
- if (q > 63) q = 63;
- k = b&0x1f;
- aux.fsum[k&0xf] += em->coef->fk[w[k]];
- aux.bsum[k&0xf] += em->coef->fk[w[k]] * em->coef->beta[q<<16|n<<8|aux.c[k&0xf]];
- ++aux.c[k&0xf];
- ++w[k];
- }
- // generate likelihood
- for (j = 0; j != m; ++j) {
- float tmp1, tmp3;
- int tmp2, bar_e;
- // homozygous
- for (k = 0, tmp1 = tmp3 = 0.0, tmp2 = 0; k != m; ++k) {
- if (k == j) continue;
- tmp1 += aux.bsum[k]; tmp2 += aux.c[k]; tmp3 += aux.fsum[k];
- }
- if (tmp2) {
- bar_e = (int)(tmp1 / tmp3 + 0.499);
- if (bar_e > 63) bar_e = 63;
- q[j*m+j] = tmp1;
- }
- // heterozygous
- for (k = j + 1; k < m; ++k) {
- int cjk = aux.c[j] + aux.c[k];
- for (i = 0, tmp2 = 0, tmp1 = tmp3 = 0.0; i < m; ++i) {
- if (i == j || i == k) continue;
- tmp1 += aux.bsum[i]; tmp2 += aux.c[i]; tmp3 += aux.fsum[i];
- }
- if (tmp2) {
- bar_e = (int)(tmp1 / tmp3 + 0.499);
- if (bar_e > 63) bar_e = 63;
- q[j*m+k] = q[k*m+j] = -4.343 * em->coef->lhet[cjk<<8|aux.c[k]] + tmp1;
- } else q[j*m+k] = q[k*m+j] = -4.343 * em->coef->lhet[cjk<<8|aux.c[k]]; // all the bases are either j or k
- }
- for (k = 0; k != m; ++k) if (q[j*m+k] < 0.0) q[j*m+k] = 0.0;
- }
- return 0;
-}
diff --git a/samtools-0.1.19/errmod.h b/samtools-0.1.19/errmod.h
deleted file mode 100644
index 32c07b6..0000000
--- a/samtools-0.1.19/errmod.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#ifndef ERRMOD_H
-#define ERRMOD_H
-
-#include <stdint.h>
-
-struct __errmod_coef_t;
-
-typedef struct {
- double depcorr;
- struct __errmod_coef_t *coef;
-} errmod_t;
-
-errmod_t *errmod_init(float depcorr);
-void errmod_destroy(errmod_t *em);
-
-/*
- n: number of bases
- m: maximum base
- bases[i]: qual:6, strand:1, base:4
- q[i*m+j]: phred-scaled likelihood of (i,j)
- */
-int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q);
-
-#endif
diff --git a/samtools-0.1.19/examples/00README.txt b/samtools-0.1.19/examples/00README.txt
deleted file mode 100644
index dbb276f..0000000
--- a/samtools-0.1.19/examples/00README.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-File ex1.fa contains two sequences cut from the human genome
-build36. They were exatracted with command:
-
- samtools faidx human_b36.fa 2:2043966-2045540 20:67967-69550
-
-Sequence names were changed manually for simplicity. File ex1.sam.gz
-contains MAQ alignments exatracted with:
-
- (samtools view NA18507_maq.bam 2:2044001-2045500;
- samtools view NA18507_maq.bam 20:68001-69500)
-
-and processed with `samtools fixmate' to make it self-consistent as a
-standalone alignment.
-
-To try samtools, you may run the following commands:
-
- samtools faidx ex1.fa # index the reference FASTA
- samtools import ex1.fa.fai ex1.sam.gz ex1.bam # SAM->BAM
- samtools index ex1.bam # index BAM
- samtools tview ex1.bam ex1.fa # view alignment
- samtools pileup -cf ex1.fa ex1.bam # pileup and consensus
- samtools pileup -cf ex1.fa -t ex1.fa.fai ex1.sam.gz
-
diff --git a/samtools-0.1.19/examples/Makefile b/samtools-0.1.19/examples/Makefile
deleted file mode 100644
index 309399f..0000000
--- a/samtools-0.1.19/examples/Makefile
+++ /dev/null
@@ -1,50 +0,0 @@
-all:../libbam.a ../samtools ../bcftools/bcftools \
- ex1.glf ex1.pileup.gz ex1.bam.bai ex1f-rmduppe.bam ex1f-rmdupse.bam ex1.glfview.gz ex1.bcf calDepth
- @echo; echo \# You can now launch the viewer with: \'samtools tview ex1.bam ex1.fa\'; echo;
-
-ex1.fa.fai:ex1.fa
- ../samtools faidx ex1.fa
-ex1.bam:ex1.sam.gz ex1.fa.fai
- ../samtools import ex1.fa.fai ex1.sam.gz ex1.bam
-ex1.bam.bai:ex1.bam
- ../samtools index ex1.bam
-ex1.pileup.gz:ex1.bam ex1.fa
- ../samtools pileup -cf ex1.fa ex1.bam | gzip > ex1.pileup.gz
-ex1.glf:ex1.bam ex1.fa
- ../samtools pileup -gf ex1.fa ex1.bam > ex1.glf
-ex1.glfview.gz:ex1.glf
- ../samtools glfview ex1.glf | gzip > ex1.glfview.gz
-ex1a.bam:ex1.bam
- ../samtools view -h ex1.bam | awk 'BEGIN{FS=OFS="\t"}{if(/^@/)print;else{$$1=$$1"a";print}}' | ../samtools view -bS - > $@
-ex1b.bam:ex1.bam
- ../samtools view -h ex1.bam | awk 'BEGIN{FS=OFS="\t"}{if(/^@/)print;else{$$1=$$1"b";print}}' | ../samtools view -bS - > $@
-ex1f.rg:
- (echo "@RG ID:ex1 LB:ex1 SM:ex1"; echo "@RG ID:ex1a LB:ex1 SM:ex1"; echo "@RG ID:ex1b LB:ex1b SM:ex1b") > $@
-ex1f.bam:ex1.bam ex1a.bam ex1b.bam ex1f.rg
- ../samtools merge -rh ex1f.rg $@ ex1.bam ex1a.bam ex1b.bam
-ex1f-rmduppe.bam:ex1f.bam
- ../samtools rmdup ex1f.bam $@
-ex1f-rmdupse.bam:ex1f.bam
- ../samtools rmdup -S ex1f.bam $@
-
-ex1.bcf:ex1.bam ex1.fa.fai
- ../samtools mpileup -gf ex1.fa ex1.bam > $@
-
-../bcftools/bcftools:
- (cd ../bcftools; make bcftools)
-
-../samtools:
- (cd ..; make samtools)
-
-../libbam.a:
- (cd ..; make libbam.a)
-
-calDepth:../libbam.a calDepth.c
- gcc -g -Wall -O2 -I.. calDepth.c -o $@ -L.. -lbam -lm -lz
-
-clean:
- rm -fr *.bam *.bai *.glf* *.fai *.pileup* *~ calDepth *.dSYM ex1*.rg ex1.bcf
-
-# ../samtools pileup ex1.bam|perl -ape '$_=$F[4];s/(\d+)(??{".{$1}"})|\^.//g;@_=(tr/A-Z//,tr/a-z//);$_=join("\t", at F[0,1], at _)."\n"'
-
-# ../samtools pileup -cf ex1.fa ex1.bam|perl -ape '$_=$F[8];s/\^.//g;s/(\d+)(??{".{$1}"})|\^.//g;@_=(tr/A-Za-z//,tr/,.//);$_=join("\t", at F[0,1], at _)."\n"'
diff --git a/samtools-0.1.19/examples/bam2bed.c b/samtools-0.1.19/examples/bam2bed.c
deleted file mode 100644
index bb937d1..0000000
--- a/samtools-0.1.19/examples/bam2bed.c
+++ /dev/null
@@ -1,51 +0,0 @@
-#include <stdio.h>
-#include "sam.h"
-static int fetch_func(const bam1_t *b, void *data)
-{
- samfile_t *fp = (samfile_t*)data;
- uint32_t *cigar = bam1_cigar(b);
- const bam1_core_t *c = &b->core;
- int i, l;
- if (b->core.tid < 0) return 0;
- for (i = l = 0; i < c->n_cigar; ++i) {
- int op = cigar[i]&0xf;
- if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP)
- l += cigar[i]>>4;
- }
- printf("%s\t%d\t%d\t%s\t%d\t%c\n", fp->header->target_name[c->tid],
- c->pos, c->pos + l, bam1_qname(b), c->qual, (c->flag&BAM_FREVERSE)? '-' : '+');
- return 0;
-}
-int main(int argc, char *argv[])
-{
- samfile_t *fp;
- if (argc == 1) {
- fprintf(stderr, "Usage: bam2bed <in.bam> [region]\n");
- return 1;
- }
- if ((fp = samopen(argv[1], "rb", 0)) == 0) {
- fprintf(stderr, "bam2bed: Fail to open BAM file %s\n", argv[1]);
- return 1;
- }
- if (argc == 2) { /* if a region is not specified */
- bam1_t *b = bam_init1();
- while (samread(fp, b) >= 0) fetch_func(b, fp);
- bam_destroy1(b);
- } else {
- int ref, beg, end;
- bam_index_t *idx;
- if ((idx = bam_index_load(argv[1])) == 0) {
- fprintf(stderr, "bam2bed: BAM indexing file is not available.\n");
- return 1;
- }
- bam_parse_region(fp->header, argv[2], &ref, &beg, &end);
- if (ref < 0) {
- fprintf(stderr, "bam2bed: Invalid region %s\n", argv[2]);
- return 1;
- }
- bam_fetch(fp->x.bam, idx, ref, beg, end, fp, fetch_func);
- bam_index_destroy(idx);
- }
- samclose(fp);
- return 0;
-}
diff --git a/samtools-0.1.19/examples/calDepth.c b/samtools-0.1.19/examples/calDepth.c
deleted file mode 100644
index 7a3239c..0000000
--- a/samtools-0.1.19/examples/calDepth.c
+++ /dev/null
@@ -1,62 +0,0 @@
-#include <stdio.h>
-#include "sam.h"
-
-typedef struct {
- int beg, end;
- samfile_t *in;
-} tmpstruct_t;
-
-// callback for bam_fetch()
-static int fetch_func(const bam1_t *b, void *data)
-{
- bam_plbuf_t *buf = (bam_plbuf_t*)data;
- bam_plbuf_push(b, buf);
- return 0;
-}
-// callback for bam_plbuf_init()
-static int pileup_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data)
-{
- tmpstruct_t *tmp = (tmpstruct_t*)data;
- if ((int)pos >= tmp->beg && (int)pos < tmp->end)
- printf("%s\t%d\t%d\n", tmp->in->header->target_name[tid], pos + 1, n);
- return 0;
-}
-
-int main(int argc, char *argv[])
-{
- tmpstruct_t tmp;
- if (argc == 1) {
- fprintf(stderr, "Usage: calDepth <in.bam> [region]\n");
- return 1;
- }
- tmp.beg = 0; tmp.end = 0x7fffffff;
- tmp.in = samopen(argv[1], "rb", 0);
- if (tmp.in == 0) {
- fprintf(stderr, "Fail to open BAM file %s\n", argv[1]);
- return 1;
- }
- if (argc == 2) { // if a region is not specified
- sampileup(tmp.in, -1, pileup_func, &tmp);
- } else {
- int ref;
- bam_index_t *idx;
- bam_plbuf_t *buf;
- idx = bam_index_load(argv[1]); // load BAM index
- if (idx == 0) {
- fprintf(stderr, "BAM indexing file is not available.\n");
- return 1;
- }
- bam_parse_region(tmp.in->header, argv[2], &ref, &tmp.beg, &tmp.end); // parse the region
- if (ref < 0) {
- fprintf(stderr, "Invalid region %s\n", argv[2]);
- return 1;
- }
- buf = bam_plbuf_init(pileup_func, &tmp); // initialize pileup
- bam_fetch(tmp.in->x.bam, idx, ref, tmp.beg, tmp.end, buf, fetch_func);
- bam_plbuf_push(0, buf); // finalize pileup
- bam_index_destroy(idx);
- bam_plbuf_destroy(buf);
- }
- samclose(tmp.in);
- return 0;
-}
diff --git a/samtools-0.1.19/examples/chk_indel.c b/samtools-0.1.19/examples/chk_indel.c
deleted file mode 100644
index aaa77e0..0000000
--- a/samtools-0.1.19/examples/chk_indel.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/* To compile, copy this file to the samtools source code directory and compile with:
- gcc -g -O2 -Wall chk_indel_rg.c -o chk_indel_rg -Wall -I. -L. -lbam -lz
-*/
-
-#include <string.h>
-#include "bam.h"
-
-typedef struct {
- long cnt[4]; // short:ins, short:del, long:ins, long:del
-} rgcnt_t;
-
-#include "khash.h"
-KHASH_MAP_INIT_STR(rgcnt, rgcnt_t)
-
-#define MAX_LEN 127
-#define Q_THRES 10
-#define L_THRES 6 // short: <=L_THRES; otherwise long
-
-int main(int argc, char *argv[])
-{
- bamFile fp;
- bam1_t *b;
- int i, x;
- khash_t(rgcnt) *h;
- khint_t k;
-
- if (argc == 1) {
- fprintf(stderr, "Usage: chk_indel_rg <in.bam>\n\n");
- fprintf(stderr, "Output: filename, RG, #ins-in-short-homopolymer, #del-in-short, #ins-in-long, #del-in-long\n");
- return 1;
- }
-
- h = kh_init(rgcnt);
- fp = bam_open(argv[1], "r");
- bam_header_destroy(bam_header_read(fp)); // we do not need the header
- b = bam_init1();
-
- while (bam_read1(fp, b) >= 0) {
- if (b->core.n_cigar >= 3 && b->core.qual >= Q_THRES) {
- const uint8_t *seq;
- const uint32_t *cigar = bam1_cigar(b);
- char *rg;
- for (i = 0; i < b->core.n_cigar; ++i) // check if there are 1bp indels
- if (bam_cigar_oplen(cigar[i]) == 1 && (bam_cigar_op(cigar[i]) == BAM_CDEL || bam_cigar_op(cigar[i]) == BAM_CINS))
- break;
- if (i == b->core.n_cigar) continue; // no 1bp ins or del
- if ((rg = (char*)bam_aux_get(b, "RG")) == 0) continue; // no RG tag
- seq = bam1_seq(b);
- for (i = x = 0; i < b->core.n_cigar; ++i) {
- int op = bam_cigar_op(cigar[i]);
- if (bam_cigar_oplen(cigar[i]) == 1 && (op == BAM_CDEL || op == BAM_CINS)) {
- int c, j, hrun, which;
- c = bam1_seqi(seq, x);
- for (j = x + 1, hrun = 0; j < b->core.l_qseq; ++j, ++hrun) // calculate the hompolymer run length
- if (bam1_seqi(seq, j) != c) break;
- k = kh_get(rgcnt, h, rg + 1);
- if (k == kh_end(h)) { // absent
- char *key = strdup(rg + 1);
- k = kh_put(rgcnt, h, key, &c);
- memset(&kh_val(h, k), 0, sizeof(rgcnt_t));
- }
- which = (hrun <= L_THRES? 0 : 1)<<1 | (op == BAM_CINS? 0 : 1);
- ++kh_val(h, k).cnt[which];
- }
- if (bam_cigar_type(op)&1) ++x;
- }
- }
- }
-
- for (k = 0; k != kh_end(h); ++k) {
- if (!kh_exist(h, k)) continue;
- printf("%s\t%s", argv[1], kh_key(h, k));
- for (i = 0; i < 4; ++i)
- printf("\t%ld", kh_val(h, k).cnt[i]);
- putchar('\n');
- free((char*)kh_key(h, k));
- }
-
- bam_destroy1(b);
- bam_close(fp);
- kh_destroy(rgcnt, h);
- return 0;
-}
diff --git a/samtools-0.1.19/examples/ex1.fa b/samtools-0.1.19/examples/ex1.fa
deleted file mode 100644
index ef611b4..0000000
--- a/samtools-0.1.19/examples/ex1.fa
+++ /dev/null
@@ -1,56 +0,0 @@
->seq1
-CACTAGTGGCTCATTGTAAATGTGTGGTTTAACTCGTCCATGGCCCAGCATTAGGGAGCT
-GTGGACCCTGCAGCCTGGCTGTGGGGGCCGCAGTGGCTGAGGGGTGCAGAGCCGAGTCAC
-GGGGTTGCCAGCACAGGGGCTTAACCTCTGGTGACTGCCAGAGCTGCTGGCAAGCTAGAG
-TCCCATTTGGAGCCCCTCTAAGCCGTTCTATTTGTAATGAAAACTATATTTATGCTATTC
-AGTTCTAAATATAGAAATTGAAACAGCTGTGTTTAGTGCCTTTGTTCAACCCCCTTGCAA
-CAACCTTGAGAACCCCAGGGAATTTGTCAATGTCAGGGAAGGAGCATTTTGTCAGTTACC
-AAATGTGTTTATTACCAGAGGGATGGAGGGAAGAGGGACGCTGAAGAACTTTGATGCCCT
-CTTCTTCCAAAGATGAAACGCGTAACTGCGCTCTCATTCACTCCAGCTCCCTGTCACCCA
-ATGGACCTGTGATATCTGGATTCTGGGAAATTCTTCATCCTGGACCCTGAGAGATTCTGC
-AGCCCAGCTCCAGATTGCTTGTGGTCTGACAGGCTGCAACTGTGAGCCATCACAATGAAC
-AACAGGAAGAAAAGGTCTTTCAAAAGGTGATGTGTGTTCTCATCAACCTCATACACACAC
-ATGGTTTAGGGGTATAATACCTCTACATGGCTGATTATGAAAACAATGTTCCCCAGATAC
-CATCCCTGTCTTACTTCCAGCTCCCCAGAGGGAAAGCTTTCAACGCTTCTAGCCATTTCT
-TTTGGCATTTGCCTTCAGACCCTACACGAATGCGTCTCTACCACAGGGGGCTGCGCGGTT
-TCCCATCATGAAGCACTGAACTTCCACGTCTCATCTAGGGGAACAGGGAGGTGCACTAAT
-GCGCTCCACGCCCAAGCCCTTCTCACAGTTTCTGCCCCCAGCATGGTTGTACTGGGCAAT
-ACATGAGATTATTAGGAAATGCTTTACTGTCATAACTATGAAGAGACTATTGCCAGATGA
-ACCACACATTAATACTATGTTTCTTATCTGCACATTACTACCCTGCAATTAATATAATTG
-TGTCCATGTACACACGCTGTCCTATGTACTTATCATGACTCTATCCCAAATTCCCAATTA
-CGTCCTATCTTCTTCTTAGGGAAGAACAGCTTAGGTATCAATTTGGTGTTCTGTGTAAAG
-TCTCAGGGAGCCGTCCGTGTCCTCCCATCTGGCCTCGTCCACACTGGTTCTCTTGAAAGC
-TTGGGCTGTAATGATGCCCCTTGGCCATCACCCAGTCCCTGCCCCATCTCTTGTAATCTC
-TCTCCTTTTTGCTGCATCCCTGTCTTCCTCTGTCTTGATTTACTTGTTGTTGGTTTTCTG
-TTTCTTTGTTTGATTTGGTGGAAGACATAATCCCACGCTTCCTATGGAAAGGTTGTTGGG
-AGATTTTTAATGATTCCTCAATGTTAAAATGTCTATTTTTGTCTTGACACCCAACTAATA
-TTTGTCTGAGCAAAACAGTCTAGATGAGAGAGAACTTCCCTGGAGGTCTGATGGCGTTTC
-TCCCTCGTCTTCTTA
->seq2
-TTCAAATGAACTTCTGTAATTGAAAAATTCATTTAAGAAATTACAAAATATAGTTGAAAG
-CTCTAACAATAGACTAAACCAAGCAGAAGAAAGAGGTTCAGAACTTGAAGACAAGTCTCT
-TATGAATTAACCCAGTCAGACAAAAATAAAGAAAAAAATTTTAAAAATGAACAGAGCTTT
-CAAGAAGTATGAGATTATGTAAAGTAACTGAACCTATGAGTCACAGGTATTCCTGAGGAA
-AAAGAAAAAGTGAGAAGTTTGGAAAAACTATTTGAGGAAGTAATTGGGGAAAACCTCTTT
-AGTCTTGCTAGAGATTTAGACATCTAAATGAAAGAGGCTCAAAGAATGCCAGGAAGATAC
-ATTGCAAGACAGACTTCATCAAGATATGTAGTCATCAGACTATCTAAAGTCAACATGAAG
-GAAAAAAATTCTAAAATCAGCAAGAGAAAAGCATACAGTCATCTATAAAGGAAATCCCAT
-CAGAATAACAATGGGCTTCTCAGCAGAAACCTTACAAGCCAGAAGAGATTGGATCTAATT
-TTTGGACTTCTTAAAGAAAAAAAAACCTGTCAAACACGAATGTTATGCCCTGCTAAACTA
-AGCATCATAAATGAAGGGGAAATAAAGTCAAGTCTTTCCTGACAAGCAAATGCTAAGATA
-ATTCATCATCACTAAACCAGTCCTATAAGAAATGCTCAAAAGAATTGTAAAAGTCAAAAT
-TAAAGTTCAATACTCACCATCATAAATACACACAAAAGTACAAAACTCACAGGTTTTATA
-AAACAATTGAGACTACAGAGCAACTAGGTAAAAAATTAACATTACAACAGGAACAAAACC
-TCATATATCAATATTAACTTTGAATAAAAAGGGATTAAATTCCCCCACTTAAGAGATATA
-GATTGGCAGAACAGATTTAAAAACATGAACTAACTATATGCTGTTTACAAGAAACTCATT
-AATAAAGACATGAGTTCAGGTAAAGGGGTGGAAAAAGATGTTCTACGCAAACAGAAACCA
-AATGAGAGAAGGAGTAGCTATACTTATATCAGATAAAGCACACTTTAAATCAACAACAGT
-AAAATAAAACAAAGGAGGTCATCATACAATGATAAAAAGATCAATTCAGCAAGAAGATAT
-AACCATCCTACTAAATACATATGCACCTAACACAAGACTACCCAGATTCATAAAACAAAT
-ACTACTAGACCTAAGAGGGATGAGAAATTACCTAATTGGTACAATGTACAATATTCTGAT
-GATGGTTACACTAAAAGCCCATACTTTACTGCTACTCAATATATCCATGTAACAAATCTG
-CGCTTGTACTTCTAAATCTATAAAAAAATTAAAATTTAACAAAAGTAAATAAAACACATA
-GCTAAAACTAAAAAAGCAAAAACAAAAACTATGCTAAGTATTGGTAAAGATGTGGGGAAA
-AAAGTAAACTCTCAAATATTGCTAGTGGGAGTATAAATTGTTTTCCACTTTGGAAAACAA
-TTTGGTAATTTCGTTTTTTTTTTTTTCTTTTCTCTTTTTTTTTTTTTTTTTTTTGCATGC
-CAGAAAAAAATATTTACAGTAACT
diff --git a/samtools-0.1.19/examples/ex1.sam.gz b/samtools-0.1.19/examples/ex1.sam.gz
deleted file mode 100644
index 44c07ee..0000000
Binary files a/samtools-0.1.19/examples/ex1.sam.gz and /dev/null differ
diff --git a/samtools-0.1.19/examples/toy.fa b/samtools-0.1.19/examples/toy.fa
deleted file mode 100644
index afe990a..0000000
--- a/samtools-0.1.19/examples/toy.fa
+++ /dev/null
@@ -1,4 +0,0 @@
->ref
-AGCATGTTAGATAAGATAGCTGTGCTAGTAGGCAGTCAGCGCCAT
->ref2
-aggttttataaaacaattaagtctacagagcaactacgcg
diff --git a/samtools-0.1.19/examples/toy.sam b/samtools-0.1.19/examples/toy.sam
deleted file mode 100644
index 33449b1..0000000
--- a/samtools-0.1.19/examples/toy.sam
+++ /dev/null
@@ -1,14 +0,0 @@
- at SQ SN:ref LN:45
- at SQ SN:ref2 LN:40
-r001 163 ref 7 30 8M4I4M1D3M = 37 39 TTAGATAAAGAGGATACTG * XX:B:S,12561,2,20,112
-r002 0 ref 9 30 1S2I6M1P1I1P1I4M2I * 0 0 AAAAGATAAGGGATAAA *
-r003 0 ref 9 30 5H6M * 0 0 AGCTAA *
-r004 0 ref 16 30 6M14N1I5M * 0 0 ATAGCTCTCAGC *
-r003 16 ref 29 30 6H5M * 0 0 TAGGC *
-r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT *
-x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ????????????????????
-x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ?????????????????????
-x3 0 ref2 6 30 9M4I13M * 0 0 ttataaaacAAATaattaagtctaca ??????????????????????????
-x4 0 ref2 10 30 25M * 0 0 CaaaTaattaagtctacagagcaac ?????????????????????????
-x5 0 ref2 12 30 24M * 0 0 aaTaattaagtctacagagcaact ????????????????????????
-x6 0 ref2 14 30 23M * 0 0 Taattaagtctacagagcaacta ???????????????????????
diff --git a/samtools-0.1.19/faidx.c b/samtools-0.1.19/faidx.c
deleted file mode 100644
index 51c82ac..0000000
--- a/samtools-0.1.19/faidx.c
+++ /dev/null
@@ -1,437 +0,0 @@
-#include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdint.h>
-#include "faidx.h"
-#include "khash.h"
-
-typedef struct {
- int32_t line_len, line_blen;
- int64_t len;
- uint64_t offset;
-} faidx1_t;
-KHASH_MAP_INIT_STR(s, faidx1_t)
-
-#ifndef _NO_RAZF
-#include "razf.h"
-#else
-#ifdef _WIN32
-#define ftello(fp) ftell(fp)
-#define fseeko(fp, offset, whence) fseek(fp, offset, whence)
-#else
-extern off_t ftello(FILE *stream);
-extern int fseeko(FILE *stream, off_t offset, int whence);
-#endif
-#define RAZF FILE
-#define razf_read(fp, buf, size) fread(buf, 1, size, fp)
-#define razf_open(fn, mode) fopen(fn, mode)
-#define razf_close(fp) fclose(fp)
-#define razf_seek(fp, offset, whence) fseeko(fp, offset, whence)
-#define razf_tell(fp) ftello(fp)
-#endif
-#ifdef _USE_KNETFILE
-#include "knetfile.h"
-#endif
-
-struct __faidx_t {
- RAZF *rz;
- int n, m;
- char **name;
- khash_t(s) *hash;
-};
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
-static inline void fai_insert_index(faidx_t *idx, const char *name, int len, int line_len, int line_blen, uint64_t offset)
-{
- khint_t k;
- int ret;
- faidx1_t t;
- if (idx->n == idx->m) {
- idx->m = idx->m? idx->m<<1 : 16;
- idx->name = (char**)realloc(idx->name, sizeof(void*) * idx->m);
- }
- idx->name[idx->n] = strdup(name);
- k = kh_put(s, idx->hash, idx->name[idx->n], &ret);
- t.len = len; t.line_len = line_len; t.line_blen = line_blen; t.offset = offset;
- kh_value(idx->hash, k) = t;
- ++idx->n;
-}
-
-faidx_t *fai_build_core(RAZF *rz)
-{
- char c, *name;
- int l_name, m_name, ret;
- int line_len, line_blen, state;
- int l1, l2;
- faidx_t *idx;
- uint64_t offset;
- int64_t len;
-
- idx = (faidx_t*)calloc(1, sizeof(faidx_t));
- idx->hash = kh_init(s);
- name = 0; l_name = m_name = 0;
- len = line_len = line_blen = -1; state = 0; l1 = l2 = -1; offset = 0;
- while (razf_read(rz, &c, 1)) {
- if (c == '\n') { // an empty line
- if (state == 1) {
- offset = razf_tell(rz);
- continue;
- } else if ((state == 0 && len < 0) || state == 2) continue;
- }
- if (c == '>') { // fasta header
- if (len >= 0)
- fai_insert_index(idx, name, len, line_len, line_blen, offset);
- l_name = 0;
- while ((ret = razf_read(rz, &c, 1)) != 0 && !isspace(c)) {
- if (m_name < l_name + 2) {
- m_name = l_name + 2;
- kroundup32(m_name);
- name = (char*)realloc(name, m_name);
- }
- name[l_name++] = c;
- }
- name[l_name] = '\0';
- if (ret == 0) {
- fprintf(stderr, "[fai_build_core] the last entry has no sequence\n");
- free(name); fai_destroy(idx);
- return 0;
- }
- if (c != '\n') while (razf_read(rz, &c, 1) && c != '\n');
- state = 1; len = 0;
- offset = razf_tell(rz);
- } else {
- if (state == 3) {
- fprintf(stderr, "[fai_build_core] inlined empty line is not allowed in sequence '%s'.\n", name);
- free(name); fai_destroy(idx);
- return 0;
- }
- if (state == 2) state = 3;
- l1 = l2 = 0;
- do {
- ++l1;
- if (isgraph(c)) ++l2;
- } while ((ret = razf_read(rz, &c, 1)) && c != '\n');
- if (state == 3 && l2) {
- fprintf(stderr, "[fai_build_core] different line length in sequence '%s'.\n", name);
- free(name); fai_destroy(idx);
- return 0;
- }
- ++l1; len += l2;
- if (state == 1) line_len = l1, line_blen = l2, state = 0;
- else if (state == 0) {
- if (l1 != line_len || l2 != line_blen) state = 2;
- }
- }
- }
- fai_insert_index(idx, name, len, line_len, line_blen, offset);
- free(name);
- return idx;
-}
-
-void fai_save(const faidx_t *fai, FILE *fp)
-{
- khint_t k;
- int i;
- for (i = 0; i < fai->n; ++i) {
- faidx1_t x;
- k = kh_get(s, fai->hash, fai->name[i]);
- x = kh_value(fai->hash, k);
-#ifdef _WIN32
- fprintf(fp, "%s\t%d\t%ld\t%d\t%d\n", fai->name[i], (int)x.len, (long)x.offset, (int)x.line_blen, (int)x.line_len);
-#else
- fprintf(fp, "%s\t%d\t%lld\t%d\t%d\n", fai->name[i], (int)x.len, (long long)x.offset, (int)x.line_blen, (int)x.line_len);
-#endif
- }
-}
-
-faidx_t *fai_read(FILE *fp)
-{
- faidx_t *fai;
- char *buf, *p;
- int len, line_len, line_blen;
-#ifdef _WIN32
- long offset;
-#else
- long long offset;
-#endif
- fai = (faidx_t*)calloc(1, sizeof(faidx_t));
- fai->hash = kh_init(s);
- buf = (char*)calloc(0x10000, 1);
- while (!feof(fp) && fgets(buf, 0x10000, fp)) {
- for (p = buf; *p && isgraph(*p); ++p);
- *p = 0; ++p;
-#ifdef _WIN32
- sscanf(p, "%d%ld%d%d", &len, &offset, &line_blen, &line_len);
-#else
- sscanf(p, "%d%lld%d%d", &len, &offset, &line_blen, &line_len);
-#endif
- fai_insert_index(fai, buf, len, line_len, line_blen, offset);
- }
- free(buf);
- return fai;
-}
-
-void fai_destroy(faidx_t *fai)
-{
- int i;
- for (i = 0; i < fai->n; ++i) free(fai->name[i]);
- free(fai->name);
- kh_destroy(s, fai->hash);
- if (fai->rz) razf_close(fai->rz);
- free(fai);
-}
-
-int fai_build(const char *fn)
-{
- char *str;
- RAZF *rz;
- FILE *fp;
- faidx_t *fai;
- str = (char*)calloc(strlen(fn) + 5, 1);
- sprintf(str, "%s.fai", fn);
- rz = razf_open(fn, "r");
- if (rz == 0) {
- fprintf(stderr, "[fai_build] fail to open the FASTA file %s\n",fn);
- free(str);
- return -1;
- }
- fai = fai_build_core(rz);
- razf_close(rz);
- fp = fopen(str, "wb");
- if (fp == 0) {
- fprintf(stderr, "[fai_build] fail to write FASTA index %s\n",str);
- fai_destroy(fai); free(str);
- return -1;
- }
- fai_save(fai, fp);
- fclose(fp);
- free(str);
- fai_destroy(fai);
- return 0;
-}
-
-#ifdef _USE_KNETFILE
-FILE *download_and_open(const char *fn)
-{
- const int buf_size = 1 * 1024 * 1024;
- uint8_t *buf;
- FILE *fp;
- knetFile *fp_remote;
- const char *url = fn;
- const char *p;
- int l = strlen(fn);
- for (p = fn + l - 1; p >= fn; --p)
- if (*p == '/') break;
- fn = p + 1;
-
- // First try to open a local copy
- fp = fopen(fn, "r");
- if (fp)
- return fp;
-
- // If failed, download from remote and open
- fp_remote = knet_open(url, "rb");
- if (fp_remote == 0) {
- fprintf(stderr, "[download_from_remote] fail to open remote file %s\n",url);
- return NULL;
- }
- if ((fp = fopen(fn, "wb")) == 0) {
- fprintf(stderr, "[download_from_remote] fail to create file in the working directory %s\n",fn);
- knet_close(fp_remote);
- return NULL;
- }
- buf = (uint8_t*)calloc(buf_size, 1);
- while ((l = knet_read(fp_remote, buf, buf_size)) != 0)
- fwrite(buf, 1, l, fp);
- free(buf);
- fclose(fp);
- knet_close(fp_remote);
-
- return fopen(fn, "r");
-}
-#endif
-
-faidx_t *fai_load(const char *fn)
-{
- char *str;
- FILE *fp;
- faidx_t *fai;
- str = (char*)calloc(strlen(fn) + 5, 1);
- sprintf(str, "%s.fai", fn);
-
-#ifdef _USE_KNETFILE
- if (strstr(fn, "ftp://") == fn || strstr(fn, "http://") == fn)
- {
- fp = download_and_open(str);
- if ( !fp )
- {
- fprintf(stderr, "[fai_load] failed to open remote FASTA index %s\n", str);
- free(str);
- return 0;
- }
- }
- else
-#endif
- fp = fopen(str, "rb");
- if (fp == 0) {
- fprintf(stderr, "[fai_load] build FASTA index.\n");
- fai_build(fn);
- fp = fopen(str, "rb");
- if (fp == 0) {
- fprintf(stderr, "[fai_load] fail to open FASTA index.\n");
- free(str);
- return 0;
- }
- }
-
- fai = fai_read(fp);
- fclose(fp);
-
- fai->rz = razf_open(fn, "rb");
- free(str);
- if (fai->rz == 0) {
- fprintf(stderr, "[fai_load] fail to open FASTA file.\n");
- return 0;
- }
- return fai;
-}
-
-char *fai_fetch(const faidx_t *fai, const char *str, int *len)
-{
- char *s, c;
- int i, l, k, name_end;
- khiter_t iter;
- faidx1_t val;
- khash_t(s) *h;
- int beg, end;
-
- beg = end = -1;
- h = fai->hash;
- name_end = l = strlen(str);
- s = (char*)malloc(l+1);
- // remove space
- for (i = k = 0; i < l; ++i)
- if (!isspace(str[i])) s[k++] = str[i];
- s[k] = 0; l = k;
- // determine the sequence name
- for (i = l - 1; i >= 0; --i) if (s[i] == ':') break; // look for colon from the end
- if (i >= 0) name_end = i;
- if (name_end < l) { // check if this is really the end
- int n_hyphen = 0;
- for (i = name_end + 1; i < l; ++i) {
- if (s[i] == '-') ++n_hyphen;
- else if (!isdigit(s[i]) && s[i] != ',') break;
- }
- if (i < l || n_hyphen > 1) name_end = l; // malformated region string; then take str as the name
- s[name_end] = 0;
- iter = kh_get(s, h, s);
- if (iter == kh_end(h)) { // cannot find the sequence name
- iter = kh_get(s, h, str); // try str as the name
- if (iter == kh_end(h)) {
- *len = 0;
- free(s); return 0;
- } else s[name_end] = ':', name_end = l;
- }
- } else iter = kh_get(s, h, str);
- if(iter == kh_end(h)) {
- fprintf(stderr, "[fai_fetch] Warning - Reference %s not found in FASTA file, returning empty sequence\n", str);
- free(s);
- return 0;
- };
- val = kh_value(h, iter);
- // parse the interval
- if (name_end < l) {
- for (i = k = name_end + 1; i < l; ++i)
- if (s[i] != ',') s[k++] = s[i];
- s[k] = 0;
- beg = atoi(s + name_end + 1);
- for (i = name_end + 1; i != k; ++i) if (s[i] == '-') break;
- end = i < k? atoi(s + i + 1) : val.len;
- if (beg > 0) --beg;
- } else beg = 0, end = val.len;
- if (beg >= val.len) beg = val.len;
- if (end >= val.len) end = val.len;
- if (beg > end) beg = end;
- free(s);
-
- // now retrieve the sequence
- l = 0;
- s = (char*)malloc(end - beg + 2);
- razf_seek(fai->rz, val.offset + beg / val.line_blen * val.line_len + beg % val.line_blen, SEEK_SET);
- while (razf_read(fai->rz, &c, 1) == 1 && l < end - beg && !fai->rz->z_err)
- if (isgraph(c)) s[l++] = c;
- s[l] = '\0';
- *len = l;
- return s;
-}
-
-int faidx_main(int argc, char *argv[])
-{
- if (argc == 1) {
- fprintf(stderr, "Usage: faidx <in.fasta> [<reg> [...]]\n");
- return 1;
- } else {
- if (argc == 2) fai_build(argv[1]);
- else {
- int i, j, k, l;
- char *s;
- faidx_t *fai;
- fai = fai_load(argv[1]);
- if (fai == 0) return 1;
- for (i = 2; i != argc; ++i) {
- printf(">%s\n", argv[i]);
- s = fai_fetch(fai, argv[i], &l);
- for (j = 0; j < l; j += 60) {
- for (k = 0; k < 60 && k < l - j; ++k)
- putchar(s[j + k]);
- putchar('\n');
- }
- free(s);
- }
- fai_destroy(fai);
- }
- }
- return 0;
-}
-
-int faidx_fetch_nseq(const faidx_t *fai)
-{
- return fai->n;
-}
-
-char *faidx_fetch_seq(const faidx_t *fai, char *c_name, int p_beg_i, int p_end_i, int *len)
-{
- int l;
- char c;
- khiter_t iter;
- faidx1_t val;
- char *seq=NULL;
-
- // Adjust position
- iter = kh_get(s, fai->hash, c_name);
- if(iter == kh_end(fai->hash)) return 0;
- val = kh_value(fai->hash, iter);
- if(p_end_i < p_beg_i) p_beg_i = p_end_i;
- if(p_beg_i < 0) p_beg_i = 0;
- else if(val.len <= p_beg_i) p_beg_i = val.len - 1;
- if(p_end_i < 0) p_end_i = 0;
- else if(val.len <= p_end_i) p_end_i = val.len - 1;
-
- // Now retrieve the sequence
- l = 0;
- seq = (char*)malloc(p_end_i - p_beg_i + 2);
- razf_seek(fai->rz, val.offset + p_beg_i / val.line_blen * val.line_len + p_beg_i % val.line_blen, SEEK_SET);
- while (razf_read(fai->rz, &c, 1) == 1 && l < p_end_i - p_beg_i + 1)
- if (isgraph(c)) seq[l++] = c;
- seq[l] = '\0';
- *len = l;
- return seq;
-}
-
-#ifdef FAIDX_MAIN
-int main(int argc, char *argv[]) { return faidx_main(argc, argv); }
-#endif
diff --git a/samtools-0.1.19/faidx.h b/samtools-0.1.19/faidx.h
deleted file mode 100644
index 1fb1b1f..0000000
--- a/samtools-0.1.19/faidx.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 Genome Research Ltd (GRL).
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Contact: Heng Li <lh3 at sanger.ac.uk> */
-
-#ifndef FAIDX_H
-#define FAIDX_H
-
-/*!
- @header
-
- Index FASTA files and extract subsequence.
-
- @copyright The Wellcome Trust Sanger Institute.
- */
-
-struct __faidx_t;
-typedef struct __faidx_t faidx_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- /*!
- @abstract Build index for a FASTA or razip compressed FASTA file.
- @param fn FASTA file name
- @return 0 on success; or -1 on failure
- @discussion File "fn.fai" will be generated.
- */
- int fai_build(const char *fn);
-
- /*!
- @abstract Distroy a faidx_t struct.
- @param fai Pointer to the struct to be destroyed
- */
- void fai_destroy(faidx_t *fai);
-
- /*!
- @abstract Load index from "fn.fai".
- @param fn File name of the FASTA file
- */
- faidx_t *fai_load(const char *fn);
-
- /*!
- @abstract Fetch the sequence in a region.
- @param fai Pointer to the faidx_t struct
- @param reg Region in the format "chr2:20,000-30,000"
- @param len Length of the region
- @return Pointer to the sequence; null on failure
-
- @discussion The returned sequence is allocated by malloc family
- and should be destroyed by end users by calling free() on it.
- */
- char *fai_fetch(const faidx_t *fai, const char *reg, int *len);
-
- /*!
- @abstract Fetch the number of sequences.
- @param fai Pointer to the faidx_t struct
- @return The number of sequences
- */
- int faidx_fetch_nseq(const faidx_t *fai);
-
- /*!
- @abstract Fetch the sequence in a region.
- @param fai Pointer to the faidx_t struct
- @param c_name Region name
- @param p_beg_i Beginning position number (zero-based)
- @param p_end_i End position number (zero-based)
- @param len Length of the region
- @return Pointer to the sequence; null on failure
-
- @discussion The returned sequence is allocated by malloc family
- and should be destroyed by end users by calling free() on it.
- */
- char *faidx_fetch_seq(const faidx_t *fai, char *c_name, int p_beg_i, int p_end_i, int *len);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/kaln.c b/samtools-0.1.19/kaln.c
deleted file mode 100644
index 9c0bbaa..0000000
--- a/samtools-0.1.19/kaln.c
+++ /dev/null
@@ -1,486 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2003-2006, 2008, 2009, by Heng Li <lh3lh3 at gmail.com>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdint.h>
-#include <math.h>
-#include "kaln.h"
-
-#define FROM_M 0
-#define FROM_I 1
-#define FROM_D 2
-
-typedef struct {
- int i, j;
- unsigned char ctype;
-} path_t;
-
-int aln_sm_blosum62[] = {
-/* A R N D C Q E G H I L K M F P S T W Y V * X */
- 4,-1,-2,-2, 0,-1,-1, 0,-2,-1,-1,-1,-1,-2,-1, 1, 0,-3,-2, 0,-4, 0,
- -1, 5, 0,-2,-3, 1, 0,-2, 0,-3,-2, 2,-1,-3,-2,-1,-1,-3,-2,-3,-4,-1,
- -2, 0, 6, 1,-3, 0, 0, 0, 1,-3,-3, 0,-2,-3,-2, 1, 0,-4,-2,-3,-4,-1,
- -2,-2, 1, 6,-3, 0, 2,-1,-1,-3,-4,-1,-3,-3,-1, 0,-1,-4,-3,-3,-4,-1,
- 0,-3,-3,-3, 9,-3,-4,-3,-3,-1,-1,-3,-1,-2,-3,-1,-1,-2,-2,-1,-4,-2,
- -1, 1, 0, 0,-3, 5, 2,-2, 0,-3,-2, 1, 0,-3,-1, 0,-1,-2,-1,-2,-4,-1,
- -1, 0, 0, 2,-4, 2, 5,-2, 0,-3,-3, 1,-2,-3,-1, 0,-1,-3,-2,-2,-4,-1,
- 0,-2, 0,-1,-3,-2,-2, 6,-2,-4,-4,-2,-3,-3,-2, 0,-2,-2,-3,-3,-4,-1,
- -2, 0, 1,-1,-3, 0, 0,-2, 8,-3,-3,-1,-2,-1,-2,-1,-2,-2, 2,-3,-4,-1,
- -1,-3,-3,-3,-1,-3,-3,-4,-3, 4, 2,-3, 1, 0,-3,-2,-1,-3,-1, 3,-4,-1,
- -1,-2,-3,-4,-1,-2,-3,-4,-3, 2, 4,-2, 2, 0,-3,-2,-1,-2,-1, 1,-4,-1,
- -1, 2, 0,-1,-3, 1, 1,-2,-1,-3,-2, 5,-1,-3,-1, 0,-1,-3,-2,-2,-4,-1,
- -1,-1,-2,-3,-1, 0,-2,-3,-2, 1, 2,-1, 5, 0,-2,-1,-1,-1,-1, 1,-4,-1,
- -2,-3,-3,-3,-2,-3,-3,-3,-1, 0, 0,-3, 0, 6,-4,-2,-2, 1, 3,-1,-4,-1,
- -1,-2,-2,-1,-3,-1,-1,-2,-2,-3,-3,-1,-2,-4, 7,-1,-1,-4,-3,-2,-4,-2,
- 1,-1, 1, 0,-1, 0, 0, 0,-1,-2,-2, 0,-1,-2,-1, 4, 1,-3,-2,-2,-4, 0,
- 0,-1, 0,-1,-1,-1,-1,-2,-2,-1,-1,-1,-1,-2,-1, 1, 5,-2,-2, 0,-4, 0,
- -3,-3,-4,-4,-2,-2,-3,-2,-2,-3,-2,-3,-1, 1,-4,-3,-2,11, 2,-3,-4,-2,
- -2,-2,-2,-3,-2,-1,-2,-3, 2,-1,-1,-2,-1, 3,-3,-2,-2, 2, 7,-1,-4,-1,
- 0,-3,-3,-3,-1,-2,-2,-3,-3, 3, 1,-2, 1,-1,-2,-2, 0,-3,-1, 4,-4,-1,
- -4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4, 1,-4,
- 0,-1,-1,-1,-2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-2, 0, 0,-2,-1,-1,-4,-1
-};
-
-int aln_sm_blast[] = {
- 1, -3, -3, -3, -2,
- -3, 1, -3, -3, -2,
- -3, -3, 1, -3, -2,
- -3, -3, -3, 1, -2,
- -2, -2, -2, -2, -2
-};
-
-int aln_sm_qual[] = {
- 0, -23, -23, -23, 0,
- -23, 0, -23, -23, 0,
- -23, -23, 0, -23, 0,
- -23, -23, -23, 0, 0,
- 0, 0, 0, 0, 0
-};
-
-ka_param_t ka_param_blast = { 5, 2, 5, 2, aln_sm_blast, 5, 50 };
-ka_param_t ka_param_aa2aa = { 10, 2, 10, 2, aln_sm_blosum62, 22, 50 };
-
-ka_param2_t ka_param2_qual = { 37, 11, 37, 11, 37, 11, 0, 0, aln_sm_qual, 5, 50 };
-
-static uint32_t *ka_path2cigar32(const path_t *path, int path_len, int *n_cigar)
-{
- int i, n;
- uint32_t *cigar;
- unsigned char last_type;
-
- if (path_len == 0 || path == 0) {
- *n_cigar = 0;
- return 0;
- }
-
- last_type = path->ctype;
- for (i = n = 1; i < path_len; ++i) {
- if (last_type != path[i].ctype) ++n;
- last_type = path[i].ctype;
- }
- *n_cigar = n;
- cigar = (uint32_t*)calloc(*n_cigar, 4);
-
- cigar[0] = 1u << 4 | path[path_len-1].ctype;
- last_type = path[path_len-1].ctype;
- for (i = path_len - 2, n = 0; i >= 0; --i) {
- if (path[i].ctype == last_type) cigar[n] += 1u << 4;
- else {
- cigar[++n] = 1u << 4 | path[i].ctype;
- last_type = path[i].ctype;
- }
- }
-
- return cigar;
-}
-
-/***************************/
-/* START OF common_align.c */
-/***************************/
-
-#define SET_INF(s) (s).M = (s).I = (s).D = MINOR_INF;
-
-#define set_M(MM, cur, p, sc) \
-{ \
- if ((p)->M >= (p)->I) { \
- if ((p)->M >= (p)->D) { \
- (MM) = (p)->M + (sc); (cur)->Mt = FROM_M; \
- } else { \
- (MM) = (p)->D + (sc); (cur)->Mt = FROM_D; \
- } \
- } else { \
- if ((p)->I > (p)->D) { \
- (MM) = (p)->I + (sc); (cur)->Mt = FROM_I; \
- } else { \
- (MM) = (p)->D + (sc); (cur)->Mt = FROM_D; \
- } \
- } \
-}
-#define set_I(II, cur, p) \
-{ \
- if ((p)->M - gap_open > (p)->I) { \
- (cur)->It = FROM_M; \
- (II) = (p)->M - gap_open - gap_ext; \
- } else { \
- (cur)->It = FROM_I; \
- (II) = (p)->I - gap_ext; \
- } \
-}
-#define set_end_I(II, cur, p) \
-{ \
- if (gap_end_ext >= 0) { \
- if ((p)->M - gap_end_open > (p)->I) { \
- (cur)->It = FROM_M; \
- (II) = (p)->M - gap_end_open - gap_end_ext; \
- } else { \
- (cur)->It = FROM_I; \
- (II) = (p)->I - gap_end_ext; \
- } \
- } else set_I(II, cur, p); \
-}
-#define set_D(DD, cur, p) \
-{ \
- if ((p)->M - gap_open > (p)->D) { \
- (cur)->Dt = FROM_M; \
- (DD) = (p)->M - gap_open - gap_ext; \
- } else { \
- (cur)->Dt = FROM_D; \
- (DD) = (p)->D - gap_ext; \
- } \
-}
-#define set_end_D(DD, cur, p) \
-{ \
- if (gap_end_ext >= 0) { \
- if ((p)->M - gap_end_open > (p)->D) { \
- (cur)->Dt = FROM_M; \
- (DD) = (p)->M - gap_end_open - gap_end_ext; \
- } else { \
- (cur)->Dt = FROM_D; \
- (DD) = (p)->D - gap_end_ext; \
- } \
- } else set_D(DD, cur, p); \
-}
-
-typedef struct {
- uint8_t Mt:3, It:2, Dt:3;
-} dpcell_t;
-
-typedef struct {
- int M, I, D;
-} dpscore_t;
-
-/***************************
- * banded global alignment *
- ***************************/
-uint32_t *ka_global_core(uint8_t *seq1, int len1, uint8_t *seq2, int len2, const ka_param_t *ap, int *_score, int *n_cigar)
-{
- int i, j;
- dpcell_t **dpcell, *q;
- dpscore_t *curr, *last, *s;
- int b1, b2, tmp_end;
- int *mat, end, max = 0;
- uint8_t type, ctype;
- uint32_t *cigar = 0;
-
- int gap_open, gap_ext, gap_end_open, gap_end_ext, b;
- int *score_matrix, N_MATRIX_ROW;
-
- /* initialize some align-related parameters. just for compatibility */
- gap_open = ap->gap_open;
- gap_ext = ap->gap_ext;
- gap_end_open = ap->gap_end_open;
- gap_end_ext = ap->gap_end_ext;
- b = ap->band_width;
- score_matrix = ap->matrix;
- N_MATRIX_ROW = ap->row;
-
- if (n_cigar) *n_cigar = 0;
- if (len1 == 0 || len2 == 0) return 0;
-
- /* calculate b1 and b2 */
- if (len1 > len2) {
- b1 = len1 - len2 + b;
- b2 = b;
- } else {
- b1 = b;
- b2 = len2 - len1 + b;
- }
- if (b1 > len1) b1 = len1;
- if (b2 > len2) b2 = len2;
- --seq1; --seq2;
-
- /* allocate memory */
- end = (b1 + b2 <= len1)? (b1 + b2 + 1) : (len1 + 1);
- dpcell = (dpcell_t**)malloc(sizeof(dpcell_t*) * (len2 + 1));
- for (j = 0; j <= len2; ++j)
- dpcell[j] = (dpcell_t*)malloc(sizeof(dpcell_t) * end);
- for (j = b2 + 1; j <= len2; ++j)
- dpcell[j] -= j - b2;
- curr = (dpscore_t*)malloc(sizeof(dpscore_t) * (len1 + 1));
- last = (dpscore_t*)malloc(sizeof(dpscore_t) * (len1 + 1));
-
- /* set first row */
- SET_INF(*curr); curr->M = 0;
- for (i = 1, s = curr + 1; i < b1; ++i, ++s) {
- SET_INF(*s);
- set_end_D(s->D, dpcell[0] + i, s - 1);
- }
- s = curr; curr = last; last = s;
-
- /* core dynamic programming, part 1 */
- tmp_end = (b2 < len2)? b2 : len2 - 1;
- for (j = 1; j <= tmp_end; ++j) {
- q = dpcell[j]; s = curr; SET_INF(*s);
- set_end_I(s->I, q, last);
- end = (j + b1 <= len1 + 1)? (j + b1 - 1) : len1;
- mat = score_matrix + seq2[j] * N_MATRIX_ROW;
- ++s; ++q;
- for (i = 1; i != end; ++i, ++s, ++q) {
- set_M(s->M, q, last + i - 1, mat[seq1[i]]); /* this will change s->M ! */
- set_I(s->I, q, last + i);
- set_D(s->D, q, s - 1);
- }
- set_M(s->M, q, last + i - 1, mat[seq1[i]]);
- set_D(s->D, q, s - 1);
- if (j + b1 - 1 > len1) { /* bug fixed, 040227 */
- set_end_I(s->I, q, last + i);
- } else s->I = MINOR_INF;
- s = curr; curr = last; last = s;
- }
- /* last row for part 1, use set_end_D() instead of set_D() */
- if (j == len2 && b2 != len2 - 1) {
- q = dpcell[j]; s = curr; SET_INF(*s);
- set_end_I(s->I, q, last);
- end = (j + b1 <= len1 + 1)? (j + b1 - 1) : len1;
- mat = score_matrix + seq2[j] * N_MATRIX_ROW;
- ++s; ++q;
- for (i = 1; i != end; ++i, ++s, ++q) {
- set_M(s->M, q, last + i - 1, mat[seq1[i]]); /* this will change s->M ! */
- set_I(s->I, q, last + i);
- set_end_D(s->D, q, s - 1);
- }
- set_M(s->M, q, last + i - 1, mat[seq1[i]]);
- set_end_D(s->D, q, s - 1);
- if (j + b1 - 1 > len1) { /* bug fixed, 040227 */
- set_end_I(s->I, q, last + i);
- } else s->I = MINOR_INF;
- s = curr; curr = last; last = s;
- ++j;
- }
-
- /* core dynamic programming, part 2 */
- for (; j <= len2 - b2 + 1; ++j) {
- SET_INF(curr[j - b2]);
- mat = score_matrix + seq2[j] * N_MATRIX_ROW;
- end = j + b1 - 1;
- for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i != end; ++i, ++s, ++q) {
- set_M(s->M, q, last + i - 1, mat[seq1[i]]);
- set_I(s->I, q, last + i);
- set_D(s->D, q, s - 1);
- }
- set_M(s->M, q, last + i - 1, mat[seq1[i]]);
- set_D(s->D, q, s - 1);
- s->I = MINOR_INF;
- s = curr; curr = last; last = s;
- }
-
- /* core dynamic programming, part 3 */
- for (; j < len2; ++j) {
- SET_INF(curr[j - b2]);
- mat = score_matrix + seq2[j] * N_MATRIX_ROW;
- for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i < len1; ++i, ++s, ++q) {
- set_M(s->M, q, last + i - 1, mat[seq1[i]]);
- set_I(s->I, q, last + i);
- set_D(s->D, q, s - 1);
- }
- set_M(s->M, q, last + len1 - 1, mat[seq1[i]]);
- set_end_I(s->I, q, last + i);
- set_D(s->D, q, s - 1);
- s = curr; curr = last; last = s;
- }
- /* last row */
- if (j == len2) {
- SET_INF(curr[j - b2]);
- mat = score_matrix + seq2[j] * N_MATRIX_ROW;
- for (i = j - b2 + 1, q = dpcell[j] + i, s = curr + i; i < len1; ++i, ++s, ++q) {
- set_M(s->M, q, last + i - 1, mat[seq1[i]]);
- set_I(s->I, q, last + i);
- set_end_D(s->D, q, s - 1);
- }
- set_M(s->M, q, last + len1 - 1, mat[seq1[i]]);
- set_end_I(s->I, q, last + i);
- set_end_D(s->D, q, s - 1);
- s = curr; curr = last; last = s;
- }
-
- *_score = last[len1].M;
- if (n_cigar) { /* backtrace */
- path_t *p, *path = (path_t*)malloc(sizeof(path_t) * (len1 + len2 + 2));
- i = len1; j = len2;
- q = dpcell[j] + i;
- s = last + len1;
- max = s->M; type = q->Mt; ctype = FROM_M;
- if (s->I > max) { max = s->I; type = q->It; ctype = FROM_I; }
- if (s->D > max) { max = s->D; type = q->Dt; ctype = FROM_D; }
-
- p = path;
- p->ctype = ctype; p->i = i; p->j = j; /* bug fixed 040408 */
- ++p;
- do {
- switch (ctype) {
- case FROM_M: --i; --j; break;
- case FROM_I: --j; break;
- case FROM_D: --i; break;
- }
- q = dpcell[j] + i;
- ctype = type;
- switch (type) {
- case FROM_M: type = q->Mt; break;
- case FROM_I: type = q->It; break;
- case FROM_D: type = q->Dt; break;
- }
- p->ctype = ctype; p->i = i; p->j = j;
- ++p;
- } while (i || j);
- cigar = ka_path2cigar32(path, p - path - 1, n_cigar);
- free(path);
- }
-
- /* free memory */
- for (j = b2 + 1; j <= len2; ++j)
- dpcell[j] += j - b2;
- for (j = 0; j <= len2; ++j)
- free(dpcell[j]);
- free(dpcell);
- free(curr); free(last);
-
- return cigar;
-}
-
-typedef struct {
- int M, I, D;
-} score_aux_t;
-
-#define MINUS_INF -0x40000000
-
-// matrix: len2 rows and len1 columns
-int ka_global_score(const uint8_t *_seq1, int len1, const uint8_t *_seq2, int len2, const ka_param2_t *ap)
-{
-
-#define __score_aux(_p, _q0, _sc, _io, _ie, _do, _de) { \
- int t1, t2; \
- score_aux_t *_q; \
- _q = _q0; \
- _p->M = _q->M >= _q->I? _q->M : _q->I; \
- _p->M = _p->M >= _q->D? _p->M : _q->D; \
- _p->M += (_sc); \
- ++_q; t1 = _q->M - _io - _ie; t2 = _q->I - _ie; _p->I = t1 >= t2? t1 : t2; \
- _q = _p-1; t1 = _q->M - _do - _de; t2 = _q->D - _de; _p->D = t1 >= t2? t1 : t2; \
- }
-
- int i, j, bw, scmat_size = ap->row, *scmat = ap->matrix, ret;
- const uint8_t *seq1, *seq2;
- score_aux_t *curr, *last, *swap;
- bw = abs(len1 - len2) + ap->band_width;
- i = len1 > len2? len1 : len2;
- if (bw > i + 1) bw = i + 1;
- seq1 = _seq1 - 1; seq2 = _seq2 - 1;
- curr = calloc(len1 + 2, sizeof(score_aux_t));
- last = calloc(len1 + 2, sizeof(score_aux_t));
- { // the zero-th row
- int x, end = len1;
- score_aux_t *p;
- j = 0;
- x = j + bw; end = len1 < x? len1 : x; // band end
- p = curr;
- p->M = 0; p->I = p->D = MINUS_INF;
- for (i = 1, p = &curr[1]; i <= end; ++i, ++p)
- p->M = p->I = MINUS_INF, p->D = -(ap->edo + ap->ede * i);
- p->M = p->I = p->D = MINUS_INF;
- swap = curr; curr = last; last = swap;
- }
- for (j = 1; j < len2; ++j) {
- int x, beg = 0, end = len1, *scrow, col_end;
- score_aux_t *p;
- x = j - bw; beg = 0 > x? 0 : x; // band start
- x = j + bw; end = len1 < x? len1 : x; // band end
- if (beg == 0) { // from zero-th column
- p = curr;
- p->M = p->D = MINUS_INF; p->I = -(ap->eio + ap->eie * j);
- ++beg; // then beg = 1
- }
- scrow = scmat + seq2[j] * scmat_size;
- if (end == len1) col_end = 1, --end;
- else col_end = 0;
- for (i = beg, p = &curr[beg]; i <= end; ++i, ++p)
- __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->iio, ap->iie, ap->ido, ap->ide);
- if (col_end) {
- __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->eio, ap->eie, ap->ido, ap->ide);
- ++p;
- }
- p->M = p->I = p->D = MINUS_INF;
-// for (i = 0; i <= len1; ++i) printf("(%d,%d,%d) ", curr[i].M, curr[i].I, curr[i].D); putchar('\n');
- swap = curr; curr = last; last = swap;
- }
- { // the last row
- int x, beg = 0, *scrow;
- score_aux_t *p;
- j = len2;
- x = j - bw; beg = 0 > x? 0 : x; // band start
- if (beg == 0) { // from zero-th column
- p = curr;
- p->M = p->D = MINUS_INF; p->I = -(ap->eio + ap->eie * j);
- ++beg; // then beg = 1
- }
- scrow = scmat + seq2[j] * scmat_size;
- for (i = beg, p = &curr[beg]; i < len1; ++i, ++p)
- __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->iio, ap->iie, ap->edo, ap->ede);
- __score_aux(p, &last[i-1], scrow[(int)seq1[i]], ap->eio, ap->eie, ap->edo, ap->ede);
-// for (i = 0; i <= len1; ++i) printf("(%d,%d,%d) ", curr[i].M, curr[i].I, curr[i].D); putchar('\n');
- }
- ret = curr[len1].M >= curr[len1].I? curr[len1].M : curr[len1].I;
- ret = ret >= curr[len1].D? ret : curr[len1].D;
- free(curr); free(last);
- return ret;
-}
-
-#ifdef _MAIN
-int main(int argc, char *argv[])
-{
-// int len1 = 35, len2 = 35;
-// uint8_t *seq1 = (uint8_t*)"\0\0\3\3\2\0\0\0\1\0\2\1\2\1\3\2\3\3\3\0\2\3\2\1\1\3\3\3\2\3\3\1\0\0\1";
-// uint8_t *seq2 = (uint8_t*)"\0\0\3\3\2\0\0\0\1\0\2\1\2\1\3\2\3\3\3\0\2\3\2\1\1\3\3\3\2\3\3\1\0\1\0";
- int len1 = 4, len2 = 4;
- uint8_t *seq1 = (uint8_t*)"\1\0\0\1";
- uint8_t *seq2 = (uint8_t*)"\1\0\1\0";
- int sc;
-// ka_global_core(seq1, 2, seq2, 1, &ka_param_qual, &sc, 0);
- sc = ka_global_score(seq1, len1, seq2, len2, &ka_param2_qual);
- printf("%d\n", sc);
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/kaln.h b/samtools-0.1.19/kaln.h
deleted file mode 100644
index 1ece132..0000000
--- a/samtools-0.1.19/kaln.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2003-2006, 2008, 2009 by Heng Li <lh3 at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#ifndef LH3_KALN_H_
-#define LH3_KALN_H_
-
-#include <stdint.h>
-
-#define MINOR_INF -1073741823
-
-typedef struct {
- int gap_open;
- int gap_ext;
- int gap_end_open;
- int gap_end_ext;
-
- int *matrix;
- int row;
- int band_width;
-} ka_param_t;
-
-typedef struct {
- int iio, iie, ido, ide;
- int eio, eie, edo, ede;
- int *matrix;
- int row;
- int band_width;
-} ka_param2_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- uint32_t *ka_global_core(uint8_t *seq1, int len1, uint8_t *seq2, int len2, const ka_param_t *ap,
- int *_score, int *n_cigar);
- int ka_global_score(const uint8_t *_seq1, int len1, const uint8_t *_seq2, int len2, const ka_param2_t *ap);
-#ifdef __cplusplus
-}
-#endif
-
-extern ka_param_t ka_param_blast; /* = { 5, 2, 5, 2, aln_sm_blast, 5, 50 }; */
-extern ka_param_t ka_param_qual; // only use this for global alignment!!!
-extern ka_param2_t ka_param2_qual; // only use this for global alignment!!!
-
-#endif
diff --git a/samtools-0.1.19/khash.h b/samtools-0.1.19/khash.h
deleted file mode 100644
index a7e8056..0000000
--- a/samtools-0.1.19/khash.h
+++ /dev/null
@@ -1,528 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/*
- An example:
-
-#include "khash.h"
-KHASH_MAP_INIT_INT(32, char)
-int main() {
- int ret, is_missing;
- khiter_t k;
- khash_t(32) *h = kh_init(32);
- k = kh_put(32, h, 5, &ret);
- if (!ret) kh_del(32, h, k);
- kh_value(h, k) = 10;
- k = kh_get(32, h, 10);
- is_missing = (k == kh_end(h));
- k = kh_get(32, h, 5);
- kh_del(32, h, k);
- for (k = kh_begin(h); k != kh_end(h); ++k)
- if (kh_exist(h, k)) kh_value(h, k) = 1;
- kh_destroy(32, h);
- return 0;
-}
-*/
-
-/*
- 2011-02-14 (0.2.5):
-
- * Allow to declare global functions.
-
- 2009-09-26 (0.2.4):
-
- * Improve portability
-
- 2008-09-19 (0.2.3):
-
- * Corrected the example
- * Improved interfaces
-
- 2008-09-11 (0.2.2):
-
- * Improved speed a little in kh_put()
-
- 2008-09-10 (0.2.1):
-
- * Added kh_clear()
- * Fixed a compiling error
-
- 2008-09-02 (0.2.0):
-
- * Changed to token concatenation which increases flexibility.
-
- 2008-08-31 (0.1.2):
-
- * Fixed a bug in kh_get(), which has not been tested previously.
-
- 2008-08-31 (0.1.1):
-
- * Added destructor
-*/
-
-
-#ifndef __AC_KHASH_H
-#define __AC_KHASH_H
-
-/*!
- @header
-
- Generic hash table library.
-
- @copyright Heng Li
- */
-
-#define AC_VERSION_KHASH_H "0.2.5"
-
-#include <stdlib.h>
-#include <string.h>
-#include <limits.h>
-
-/* compipler specific configuration */
-
-#if UINT_MAX == 0xffffffffu
-typedef unsigned int khint32_t;
-#elif ULONG_MAX == 0xffffffffu
-typedef unsigned long khint32_t;
-#endif
-
-#if ULONG_MAX == ULLONG_MAX
-typedef unsigned long khint64_t;
-#else
-typedef unsigned long long khint64_t;
-#endif
-
-#ifdef _MSC_VER
-#define inline __inline
-#endif
-
-typedef khint32_t khint_t;
-typedef khint_t khiter_t;
-
-#define __ac_HASH_PRIME_SIZE 32
-static const khint32_t __ac_prime_list[__ac_HASH_PRIME_SIZE] =
-{
- 0ul, 3ul, 11ul, 23ul, 53ul,
- 97ul, 193ul, 389ul, 769ul, 1543ul,
- 3079ul, 6151ul, 12289ul, 24593ul, 49157ul,
- 98317ul, 196613ul, 393241ul, 786433ul, 1572869ul,
- 3145739ul, 6291469ul, 12582917ul, 25165843ul, 50331653ul,
- 100663319ul, 201326611ul, 402653189ul, 805306457ul, 1610612741ul,
- 3221225473ul, 4294967291ul
-};
-
-#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
-#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
-#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
-#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
-#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
-#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
-#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
-
-static const double __ac_HASH_UPPER = 0.77;
-
-#define KHASH_DECLARE(name, khkey_t, khval_t) \
- typedef struct { \
- khint_t n_buckets, size, n_occupied, upper_bound; \
- khint32_t *flags; \
- khkey_t *keys; \
- khval_t *vals; \
- } kh_##name##_t; \
- extern kh_##name##_t *kh_init_##name(); \
- extern void kh_destroy_##name(kh_##name##_t *h); \
- extern void kh_clear_##name(kh_##name##_t *h); \
- extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
- extern void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
- extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
- extern void kh_del_##name(kh_##name##_t *h, khint_t x);
-
-#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
- typedef struct { \
- khint_t n_buckets, size, n_occupied, upper_bound; \
- khint32_t *flags; \
- khkey_t *keys; \
- khval_t *vals; \
- } kh_##name##_t; \
- SCOPE kh_##name##_t *kh_init_##name() { \
- return (kh_##name##_t*)calloc(1, sizeof(kh_##name##_t)); \
- } \
- SCOPE void kh_destroy_##name(kh_##name##_t *h) \
- { \
- if (h) { \
- free(h->keys); free(h->flags); \
- free(h->vals); \
- free(h); \
- } \
- } \
- SCOPE void kh_clear_##name(kh_##name##_t *h) \
- { \
- if (h && h->flags) { \
- memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(khint32_t)); \
- h->size = h->n_occupied = 0; \
- } \
- } \
- SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
- { \
- if (h->n_buckets) { \
- khint_t inc, k, i, last; \
- k = __hash_func(key); i = k % h->n_buckets; \
- inc = 1 + k % (h->n_buckets - 1); last = i; \
- while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
- if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \
- else i += inc; \
- if (i == last) return h->n_buckets; \
- } \
- return __ac_iseither(h->flags, i)? h->n_buckets : i; \
- } else return 0; \
- } \
- SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
- { \
- khint32_t *new_flags = 0; \
- khint_t j = 1; \
- { \
- khint_t t = __ac_HASH_PRIME_SIZE - 1; \
- while (__ac_prime_list[t] > new_n_buckets) --t; \
- new_n_buckets = __ac_prime_list[t+1]; \
- if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; \
- else { \
- new_flags = (khint32_t*)malloc(((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \
- memset(new_flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \
- if (h->n_buckets < new_n_buckets) { \
- h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \
- if (kh_is_map) \
- h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \
- } \
- } \
- } \
- if (j) { \
- for (j = 0; j != h->n_buckets; ++j) { \
- if (__ac_iseither(h->flags, j) == 0) { \
- khkey_t key = h->keys[j]; \
- khval_t val; \
- if (kh_is_map) val = h->vals[j]; \
- __ac_set_isdel_true(h->flags, j); \
- while (1) { \
- khint_t inc, k, i; \
- k = __hash_func(key); \
- i = k % new_n_buckets; \
- inc = 1 + k % (new_n_buckets - 1); \
- while (!__ac_isempty(new_flags, i)) { \
- if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets; \
- else i += inc; \
- } \
- __ac_set_isempty_false(new_flags, i); \
- if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { \
- { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
- if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
- __ac_set_isdel_true(h->flags, i); \
- } else { \
- h->keys[i] = key; \
- if (kh_is_map) h->vals[i] = val; \
- break; \
- } \
- } \
- } \
- } \
- if (h->n_buckets > new_n_buckets) { \
- h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \
- if (kh_is_map) \
- h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \
- } \
- free(h->flags); \
- h->flags = new_flags; \
- h->n_buckets = new_n_buckets; \
- h->n_occupied = h->size; \
- h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
- } \
- } \
- SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
- { \
- khint_t x; \
- if (h->n_occupied >= h->upper_bound) { \
- if (h->n_buckets > (h->size<<1)) kh_resize_##name(h, h->n_buckets - 1); \
- else kh_resize_##name(h, h->n_buckets + 1); \
- } \
- { \
- khint_t inc, k, i, site, last; \
- x = site = h->n_buckets; k = __hash_func(key); i = k % h->n_buckets; \
- if (__ac_isempty(h->flags, i)) x = i; \
- else { \
- inc = 1 + k % (h->n_buckets - 1); last = i; \
- while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
- if (__ac_isdel(h->flags, i)) site = i; \
- if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \
- else i += inc; \
- if (i == last) { x = site; break; } \
- } \
- if (x == h->n_buckets) { \
- if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
- else x = i; \
- } \
- } \
- } \
- if (__ac_isempty(h->flags, x)) { \
- h->keys[x] = key; \
- __ac_set_isboth_false(h->flags, x); \
- ++h->size; ++h->n_occupied; \
- *ret = 1; \
- } else if (__ac_isdel(h->flags, x)) { \
- h->keys[x] = key; \
- __ac_set_isboth_false(h->flags, x); \
- ++h->size; \
- *ret = 2; \
- } else *ret = 0; \
- return x; \
- } \
- SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \
- { \
- if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
- __ac_set_isdel_true(h->flags, x); \
- --h->size; \
- } \
- }
-
-#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
- KHASH_INIT2(name, static inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
-
-/* --- BEGIN OF HASH FUNCTIONS --- */
-
-/*! @function
- @abstract Integer hash function
- @param key The integer [khint32_t]
- @return The hash value [khint_t]
- */
-#define kh_int_hash_func(key) (khint32_t)(key)
-/*! @function
- @abstract Integer comparison function
- */
-#define kh_int_hash_equal(a, b) ((a) == (b))
-/*! @function
- @abstract 64-bit integer hash function
- @param key The integer [khint64_t]
- @return The hash value [khint_t]
- */
-#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
-/*! @function
- @abstract 64-bit integer comparison function
- */
-#define kh_int64_hash_equal(a, b) ((a) == (b))
-/*! @function
- @abstract const char* hash function
- @param s Pointer to a null terminated string
- @return The hash value
- */
-static inline khint_t __ac_X31_hash_string(const char *s)
-{
- khint_t h = *s;
- if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s;
- return h;
-}
-/*! @function
- @abstract Another interface to const char* hash function
- @param key Pointer to a null terminated string [const char*]
- @return The hash value [khint_t]
- */
-#define kh_str_hash_func(key) __ac_X31_hash_string(key)
-/*! @function
- @abstract Const char* comparison function
- */
-#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
-
-/* --- END OF HASH FUNCTIONS --- */
-
-/* Other necessary macros... */
-
-/*!
- @abstract Type of the hash table.
- @param name Name of the hash table [symbol]
- */
-#define khash_t(name) kh_##name##_t
-
-/*! @function
- @abstract Initiate a hash table.
- @param name Name of the hash table [symbol]
- @return Pointer to the hash table [khash_t(name)*]
- */
-#define kh_init(name) kh_init_##name()
-
-/*! @function
- @abstract Destroy a hash table.
- @param name Name of the hash table [symbol]
- @param h Pointer to the hash table [khash_t(name)*]
- */
-#define kh_destroy(name, h) kh_destroy_##name(h)
-
-/*! @function
- @abstract Reset a hash table without deallocating memory.
- @param name Name of the hash table [symbol]
- @param h Pointer to the hash table [khash_t(name)*]
- */
-#define kh_clear(name, h) kh_clear_##name(h)
-
-/*! @function
- @abstract Resize a hash table.
- @param name Name of the hash table [symbol]
- @param h Pointer to the hash table [khash_t(name)*]
- @param s New size [khint_t]
- */
-#define kh_resize(name, h, s) kh_resize_##name(h, s)
-
-/*! @function
- @abstract Insert a key to the hash table.
- @param name Name of the hash table [symbol]
- @param h Pointer to the hash table [khash_t(name)*]
- @param k Key [type of keys]
- @param r Extra return code: 0 if the key is present in the hash table;
- 1 if the bucket is empty (never used); 2 if the element in
- the bucket has been deleted [int*]
- @return Iterator to the inserted element [khint_t]
- */
-#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
-
-/*! @function
- @abstract Retrieve a key from the hash table.
- @param name Name of the hash table [symbol]
- @param h Pointer to the hash table [khash_t(name)*]
- @param k Key [type of keys]
- @return Iterator to the found element, or kh_end(h) is the element is absent [khint_t]
- */
-#define kh_get(name, h, k) kh_get_##name(h, k)
-
-/*! @function
- @abstract Remove a key from the hash table.
- @param name Name of the hash table [symbol]
- @param h Pointer to the hash table [khash_t(name)*]
- @param k Iterator to the element to be deleted [khint_t]
- */
-#define kh_del(name, h, k) kh_del_##name(h, k)
-
-
-/*! @function
- @abstract Test whether a bucket contains data.
- @param h Pointer to the hash table [khash_t(name)*]
- @param x Iterator to the bucket [khint_t]
- @return 1 if containing data; 0 otherwise [int]
- */
-#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
-
-/*! @function
- @abstract Get key given an iterator
- @param h Pointer to the hash table [khash_t(name)*]
- @param x Iterator to the bucket [khint_t]
- @return Key [type of keys]
- */
-#define kh_key(h, x) ((h)->keys[x])
-
-/*! @function
- @abstract Get value given an iterator
- @param h Pointer to the hash table [khash_t(name)*]
- @param x Iterator to the bucket [khint_t]
- @return Value [type of values]
- @discussion For hash sets, calling this results in segfault.
- */
-#define kh_val(h, x) ((h)->vals[x])
-
-/*! @function
- @abstract Alias of kh_val()
- */
-#define kh_value(h, x) ((h)->vals[x])
-
-/*! @function
- @abstract Get the start iterator
- @param h Pointer to the hash table [khash_t(name)*]
- @return The start iterator [khint_t]
- */
-#define kh_begin(h) (khint_t)(0)
-
-/*! @function
- @abstract Get the end iterator
- @param h Pointer to the hash table [khash_t(name)*]
- @return The end iterator [khint_t]
- */
-#define kh_end(h) ((h)->n_buckets)
-
-/*! @function
- @abstract Get the number of elements in the hash table
- @param h Pointer to the hash table [khash_t(name)*]
- @return Number of elements in the hash table [khint_t]
- */
-#define kh_size(h) ((h)->size)
-
-/*! @function
- @abstract Get the number of buckets in the hash table
- @param h Pointer to the hash table [khash_t(name)*]
- @return Number of buckets in the hash table [khint_t]
- */
-#define kh_n_buckets(h) ((h)->n_buckets)
-
-/* More conenient interfaces */
-
-/*! @function
- @abstract Instantiate a hash set containing integer keys
- @param name Name of the hash table [symbol]
- */
-#define KHASH_SET_INIT_INT(name) \
- KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
-
-/*! @function
- @abstract Instantiate a hash map containing integer keys
- @param name Name of the hash table [symbol]
- @param khval_t Type of values [type]
- */
-#define KHASH_MAP_INIT_INT(name, khval_t) \
- KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
-
-/*! @function
- @abstract Instantiate a hash map containing 64-bit integer keys
- @param name Name of the hash table [symbol]
- */
-#define KHASH_SET_INIT_INT64(name) \
- KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
-
-/*! @function
- @abstract Instantiate a hash map containing 64-bit integer keys
- @param name Name of the hash table [symbol]
- @param khval_t Type of values [type]
- */
-#define KHASH_MAP_INIT_INT64(name, khval_t) \
- KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
-
-typedef const char *kh_cstr_t;
-/*! @function
- @abstract Instantiate a hash map containing const char* keys
- @param name Name of the hash table [symbol]
- */
-#define KHASH_SET_INIT_STR(name) \
- KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
-
-/*! @function
- @abstract Instantiate a hash map containing const char* keys
- @param name Name of the hash table [symbol]
- @param khval_t Type of values [type]
- */
-#define KHASH_MAP_INIT_STR(name, khval_t) \
- KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
-
-#endif /* __AC_KHASH_H */
diff --git a/samtools-0.1.19/klist.h b/samtools-0.1.19/klist.h
deleted file mode 100644
index 2f17016..0000000
--- a/samtools-0.1.19/klist.h
+++ /dev/null
@@ -1,96 +0,0 @@
-#ifndef _LH3_KLIST_H
-#define _LH3_KLIST_H
-
-#include <stdlib.h>
-
-#define KMEMPOOL_INIT(name, kmptype_t, kmpfree_f) \
- typedef struct { \
- size_t cnt, n, max; \
- kmptype_t **buf; \
- } kmp_##name##_t; \
- static inline kmp_##name##_t *kmp_init_##name() { \
- return calloc(1, sizeof(kmp_##name##_t)); \
- } \
- static inline void kmp_destroy_##name(kmp_##name##_t *mp) { \
- size_t k; \
- for (k = 0; k < mp->n; ++k) { \
- kmpfree_f(mp->buf[k]); free(mp->buf[k]); \
- } \
- free(mp->buf); free(mp); \
- } \
- static inline kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) { \
- ++mp->cnt; \
- if (mp->n == 0) return calloc(1, sizeof(kmptype_t)); \
- return mp->buf[--mp->n]; \
- } \
- static inline void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) { \
- --mp->cnt; \
- if (mp->n == mp->max) { \
- mp->max = mp->max? mp->max<<1 : 16; \
- mp->buf = realloc(mp->buf, sizeof(void*) * mp->max); \
- } \
- mp->buf[mp->n++] = p; \
- }
-
-#define kmempool_t(name) kmp_##name##_t
-#define kmp_init(name) kmp_init_##name()
-#define kmp_destroy(name, mp) kmp_destroy_##name(mp)
-#define kmp_alloc(name, mp) kmp_alloc_##name(mp)
-#define kmp_free(name, mp, p) kmp_free_##name(mp, p)
-
-#define KLIST_INIT(name, kltype_t, kmpfree_t) \
- struct __kl1_##name { \
- kltype_t data; \
- struct __kl1_##name *next; \
- }; \
- typedef struct __kl1_##name kl1_##name; \
- KMEMPOOL_INIT(name, kl1_##name, kmpfree_t) \
- typedef struct { \
- kl1_##name *head, *tail; \
- kmp_##name##_t *mp; \
- size_t size; \
- } kl_##name##_t; \
- static inline kl_##name##_t *kl_init_##name() { \
- kl_##name##_t *kl = calloc(1, sizeof(kl_##name##_t)); \
- kl->mp = kmp_init(name); \
- kl->head = kl->tail = kmp_alloc(name, kl->mp); \
- kl->head->next = 0; \
- return kl; \
- } \
- static inline void kl_destroy_##name(kl_##name##_t *kl) { \
- kl1_##name *p; \
- for (p = kl->head; p != kl->tail; p = p->next) \
- kmp_free(name, kl->mp, p); \
- kmp_free(name, kl->mp, p); \
- kmp_destroy(name, kl->mp); \
- free(kl); \
- } \
- static inline kltype_t *kl_pushp_##name(kl_##name##_t *kl) { \
- kl1_##name *q, *p = kmp_alloc(name, kl->mp); \
- q = kl->tail; p->next = 0; kl->tail->next = p; kl->tail = p; \
- ++kl->size; \
- return &q->data; \
- } \
- static inline int kl_shift_##name(kl_##name##_t *kl, kltype_t *d) { \
- kl1_##name *p; \
- if (kl->head->next == 0) return -1; \
- --kl->size; \
- p = kl->head; kl->head = kl->head->next; \
- if (d) *d = p->data; \
- kmp_free(name, kl->mp, p); \
- return 0; \
- }
-
-#define kliter_t(name) kl1_##name
-#define klist_t(name) kl_##name##_t
-#define kl_val(iter) ((iter)->data)
-#define kl_next(iter) ((iter)->next)
-#define kl_begin(kl) ((kl)->head)
-#define kl_end(kl) ((kl)->tail)
-
-#define kl_init(name) kl_init_##name()
-#define kl_destroy(name, kl) kl_destroy_##name(kl)
-#define kl_pushp(name, kl) kl_pushp_##name(kl)
-#define kl_shift(name, kl, d) kl_shift_##name(kl, d)
-
-#endif
diff --git a/samtools-0.1.19/knetfile.c b/samtools-0.1.19/knetfile.c
deleted file mode 100644
index af09146..0000000
--- a/samtools-0.1.19/knetfile.c
+++ /dev/null
@@ -1,632 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 by Genome Research Ltd (GRL).
- 2010 by Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Probably I will not do socket programming in the next few years and
- therefore I decide to heavily annotate this file, for Linux and
- Windows as well. -ac */
-
-#include <time.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <sys/types.h>
-
-#ifndef _WIN32
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <sys/socket.h>
-#endif
-
-#include "knetfile.h"
-
-/* In winsock.h, the type of a socket is SOCKET, which is: "typedef
- * u_int SOCKET". An invalid SOCKET is: "(SOCKET)(~0)", or signed
- * integer -1. In knetfile.c, I use "int" for socket type
- * throughout. This should be improved to avoid confusion.
- *
- * In Linux/Mac, recv() and read() do almost the same thing. You can see
- * in the header file that netread() is simply an alias of read(). In
- * Windows, however, they are different and using recv() is mandatory.
- */
-
-/* This function tests if the file handler is ready for reading (or
- * writing if is_read==0). */
-static int socket_wait(int fd, int is_read)
-{
- fd_set fds, *fdr = 0, *fdw = 0;
- struct timeval tv;
- int ret;
- tv.tv_sec = 5; tv.tv_usec = 0; // 5 seconds time out
- FD_ZERO(&fds);
- FD_SET(fd, &fds);
- if (is_read) fdr = &fds;
- else fdw = &fds;
- ret = select(fd+1, fdr, fdw, 0, &tv);
-#ifndef _WIN32
- if (ret == -1) perror("select");
-#else
- if (ret == 0)
- fprintf(stderr, "select time-out\n");
- else if (ret == SOCKET_ERROR)
- fprintf(stderr, "select: %d\n", WSAGetLastError());
-#endif
- return ret;
-}
-
-#ifndef _WIN32
-/* This function does not work with Windows due to the lack of
- * getaddrinfo() in winsock. It is addapted from an example in "Beej's
- * Guide to Network Programming" (http://beej.us/guide/bgnet/). */
-static int socket_connect(const char *host, const char *port)
-{
-#define __err_connect(func) do { perror(func); freeaddrinfo(res); return -1; } while (0)
-
- int on = 1, fd;
- struct linger lng = { 0, 0 };
- struct addrinfo hints, *res = 0;
- memset(&hints, 0, sizeof(struct addrinfo));
- hints.ai_family = AF_UNSPEC;
- hints.ai_socktype = SOCK_STREAM;
- /* In Unix/Mac, getaddrinfo() is the most convenient way to get
- * server information. */
- if (getaddrinfo(host, port, &hints, &res) != 0) __err_connect("getaddrinfo");
- if ((fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) __err_connect("socket");
- /* The following two setsockopt() are used by ftplib
- * (http://nbpfaus.net/~pfau/ftplib/). I am not sure if they
- * necessary. */
- if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) __err_connect("setsockopt");
- if (setsockopt(fd, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) __err_connect("setsockopt");
- if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) __err_connect("connect");
- freeaddrinfo(res);
- return fd;
-}
-#else
-/* MinGW's printf has problem with "%lld" */
-char *int64tostr(char *buf, int64_t x)
-{
- int cnt;
- int i = 0;
- do {
- buf[i++] = '0' + x % 10;
- x /= 10;
- } while (x);
- buf[i] = 0;
- for (cnt = i, i = 0; i < cnt/2; ++i) {
- int c = buf[i]; buf[i] = buf[cnt-i-1]; buf[cnt-i-1] = c;
- }
- return buf;
-}
-
-int64_t strtoint64(const char *buf)
-{
- int64_t x;
- for (x = 0; *buf != '\0'; ++buf)
- x = x * 10 + ((int64_t) *buf - 48);
- return x;
-}
-/* In windows, the first thing is to establish the TCP connection. */
-int knet_win32_init()
-{
- WSADATA wsaData;
- return WSAStartup(MAKEWORD(2, 2), &wsaData);
-}
-void knet_win32_destroy()
-{
- WSACleanup();
-}
-/* A slightly modfied version of the following function also works on
- * Mac (and presummably Linux). However, this function is not stable on
- * my Mac. It sometimes works fine but sometimes does not. Therefore for
- * non-Windows OS, I do not use this one. */
-static SOCKET socket_connect(const char *host, const char *port)
-{
-#define __err_connect(func) \
- do { \
- fprintf(stderr, "%s: %d\n", func, WSAGetLastError()); \
- return -1; \
- } while (0)
-
- int on = 1;
- SOCKET fd;
- struct linger lng = { 0, 0 };
- struct sockaddr_in server;
- struct hostent *hp = 0;
- // open socket
- if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) __err_connect("socket");
- if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)) == -1) __err_connect("setsockopt");
- if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char*)&lng, sizeof(lng)) == -1) __err_connect("setsockopt");
- // get host info
- if (isalpha(host[0])) hp = gethostbyname(host);
- else {
- struct in_addr addr;
- addr.s_addr = inet_addr(host);
- hp = gethostbyaddr((char*)&addr, 4, AF_INET);
- }
- if (hp == 0) __err_connect("gethost");
- // connect
- server.sin_addr.s_addr = *((unsigned long*)hp->h_addr);
- server.sin_family= AF_INET;
- server.sin_port = htons(atoi(port));
- if (connect(fd, (struct sockaddr*)&server, sizeof(server)) != 0) __err_connect("connect");
- // freehostent(hp); // strangely in MSDN, hp is NOT freed (memory leak?!)
- return fd;
-}
-#endif
-
-static off_t my_netread(int fd, void *buf, off_t len)
-{
- off_t rest = len, curr, l = 0;
- /* recv() and read() may not read the required length of data with
- * one call. They have to be called repeatedly. */
- while (rest) {
- if (socket_wait(fd, 1) <= 0) break; // socket is not ready for reading
- curr = netread(fd, buf + l, rest);
- /* According to the glibc manual, section 13.2, a zero returned
- * value indicates end-of-file (EOF), which should mean that
- * read() will not return zero if EOF has not been met but data
- * are not immediately available. */
- if (curr == 0) break;
- l += curr; rest -= curr;
- }
- return l;
-}
-
-/*************************
- * FTP specific routines *
- *************************/
-
-static int kftp_get_response(knetFile *ftp)
-{
-#ifndef _WIN32
- unsigned char c;
-#else
- char c;
-#endif
- int n = 0;
- char *p;
- if (socket_wait(ftp->ctrl_fd, 1) <= 0) return 0;
- while (netread(ftp->ctrl_fd, &c, 1)) { // FIXME: this is *VERY BAD* for unbuffered I/O
- //fputc(c, stderr);
- if (n >= ftp->max_response) {
- ftp->max_response = ftp->max_response? ftp->max_response<<1 : 256;
- ftp->response = realloc(ftp->response, ftp->max_response);
- }
- ftp->response[n++] = c;
- if (c == '\n') {
- if (n >= 4 && isdigit(ftp->response[0]) && isdigit(ftp->response[1]) && isdigit(ftp->response[2])
- && ftp->response[3] != '-') break;
- n = 0;
- continue;
- }
- }
- if (n < 2) return -1;
- ftp->response[n-2] = 0;
- return strtol(ftp->response, &p, 0);
-}
-
-static int kftp_send_cmd(knetFile *ftp, const char *cmd, int is_get)
-{
- if (socket_wait(ftp->ctrl_fd, 0) <= 0) return -1; // socket is not ready for writing
- netwrite(ftp->ctrl_fd, cmd, strlen(cmd));
- return is_get? kftp_get_response(ftp) : 0;
-}
-
-static int kftp_pasv_prep(knetFile *ftp)
-{
- char *p;
- int v[6];
- kftp_send_cmd(ftp, "PASV\r\n", 1);
- for (p = ftp->response; *p && *p != '('; ++p);
- if (*p != '(') return -1;
- ++p;
- sscanf(p, "%d,%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]);
- memcpy(ftp->pasv_ip, v, 4 * sizeof(int));
- ftp->pasv_port = (v[4]<<8&0xff00) + v[5];
- return 0;
-}
-
-
-static int kftp_pasv_connect(knetFile *ftp)
-{
- char host[80], port[10];
- if (ftp->pasv_port == 0) {
- fprintf(stderr, "[kftp_pasv_connect] kftp_pasv_prep() is not called before hand.\n");
- return -1;
- }
- sprintf(host, "%d.%d.%d.%d", ftp->pasv_ip[0], ftp->pasv_ip[1], ftp->pasv_ip[2], ftp->pasv_ip[3]);
- sprintf(port, "%d", ftp->pasv_port);
- ftp->fd = socket_connect(host, port);
- if (ftp->fd == -1) return -1;
- return 0;
-}
-
-int kftp_connect(knetFile *ftp)
-{
- ftp->ctrl_fd = socket_connect(ftp->host, ftp->port);
- if (ftp->ctrl_fd == -1) return -1;
- kftp_get_response(ftp);
- kftp_send_cmd(ftp, "USER anonymous\r\n", 1);
- kftp_send_cmd(ftp, "PASS kftp@\r\n", 1);
- kftp_send_cmd(ftp, "TYPE I\r\n", 1);
- return 0;
-}
-
-int kftp_reconnect(knetFile *ftp)
-{
- if (ftp->ctrl_fd != -1) {
- netclose(ftp->ctrl_fd);
- ftp->ctrl_fd = -1;
- }
- netclose(ftp->fd);
- ftp->fd = -1;
- return kftp_connect(ftp);
-}
-
-// initialize ->type, ->host, ->retr and ->size
-knetFile *kftp_parse_url(const char *fn, const char *mode)
-{
- knetFile *fp;
- char *p;
- int l;
- if (strstr(fn, "ftp://") != fn) return 0;
- for (p = (char*)fn + 6; *p && *p != '/'; ++p);
- if (*p != '/') return 0;
- l = p - fn - 6;
- fp = calloc(1, sizeof(knetFile));
- fp->type = KNF_TYPE_FTP;
- fp->fd = -1;
- /* the Linux/Mac version of socket_connect() also recognizes a port
- * like "ftp", but the Windows version does not. */
- fp->port = strdup("21");
- fp->host = calloc(l + 1, 1);
- if (strchr(mode, 'c')) fp->no_reconnect = 1;
- strncpy(fp->host, fn + 6, l);
- fp->retr = calloc(strlen(p) + 8, 1);
- sprintf(fp->retr, "RETR %s\r\n", p);
- fp->size_cmd = calloc(strlen(p) + 8, 1);
- sprintf(fp->size_cmd, "SIZE %s\r\n", p);
- fp->seek_offset = 0;
- return fp;
-}
-// place ->fd at offset off
-int kftp_connect_file(knetFile *fp)
-{
- int ret;
- long long file_size;
- if (fp->fd != -1) {
- netclose(fp->fd);
- if (fp->no_reconnect) kftp_get_response(fp);
- }
- kftp_pasv_prep(fp);
- kftp_send_cmd(fp, fp->size_cmd, 1);
-#ifndef _WIN32
- if ( sscanf(fp->response,"%*d %lld", &file_size) != 1 )
- {
- fprintf(stderr,"[kftp_connect_file] %s\n", fp->response);
- return -1;
- }
-#else
- const char *p = fp->response;
- while (*p != ' ') ++p;
- while (*p < '0' || *p > '9') ++p;
- file_size = strtoint64(p);
-#endif
- fp->file_size = file_size;
- if (fp->offset>=0) {
- char tmp[32];
-#ifndef _WIN32
- sprintf(tmp, "REST %lld\r\n", (long long)fp->offset);
-#else
- strcpy(tmp, "REST ");
- int64tostr(tmp + 5, fp->offset);
- strcat(tmp, "\r\n");
-#endif
- kftp_send_cmd(fp, tmp, 1);
- }
- kftp_send_cmd(fp, fp->retr, 0);
- kftp_pasv_connect(fp);
- ret = kftp_get_response(fp);
- if (ret != 150) {
- fprintf(stderr, "[kftp_connect_file] %s\n", fp->response);
- netclose(fp->fd);
- fp->fd = -1;
- return -1;
- }
- fp->is_ready = 1;
- return 0;
-}
-
-
-/**************************
- * HTTP specific routines *
- **************************/
-
-knetFile *khttp_parse_url(const char *fn, const char *mode)
-{
- knetFile *fp;
- char *p, *proxy, *q;
- int l;
- if (strstr(fn, "http://") != fn) return 0;
- // set ->http_host
- for (p = (char*)fn + 7; *p && *p != '/'; ++p);
- l = p - fn - 7;
- fp = calloc(1, sizeof(knetFile));
- fp->http_host = calloc(l + 1, 1);
- strncpy(fp->http_host, fn + 7, l);
- fp->http_host[l] = 0;
- for (q = fp->http_host; *q && *q != ':'; ++q);
- if (*q == ':') *q++ = 0;
- // get http_proxy
- proxy = getenv("http_proxy");
- // set ->host, ->port and ->path
- if (proxy == 0) {
- fp->host = strdup(fp->http_host); // when there is no proxy, server name is identical to http_host name.
- fp->port = strdup(*q? q : "80");
- fp->path = strdup(*p? p : "/");
- } else {
- fp->host = (strstr(proxy, "http://") == proxy)? strdup(proxy + 7) : strdup(proxy);
- for (q = fp->host; *q && *q != ':'; ++q);
- if (*q == ':') *q++ = 0;
- fp->port = strdup(*q? q : "80");
- fp->path = strdup(fn);
- }
- fp->type = KNF_TYPE_HTTP;
- fp->ctrl_fd = fp->fd = -1;
- fp->seek_offset = 0;
- return fp;
-}
-
-int khttp_connect_file(knetFile *fp)
-{
- int ret, l = 0;
- char *buf, *p;
- if (fp->fd != -1) netclose(fp->fd);
- fp->fd = socket_connect(fp->host, fp->port);
- buf = calloc(0x10000, 1); // FIXME: I am lazy... But in principle, 64KB should be large enough.
- l += sprintf(buf + l, "GET %s HTTP/1.0\r\nHost: %s\r\n", fp->path, fp->http_host);
- l += sprintf(buf + l, "Range: bytes=%lld-\r\n", (long long)fp->offset);
- l += sprintf(buf + l, "\r\n");
- netwrite(fp->fd, buf, l);
- l = 0;
- while (netread(fp->fd, buf + l, 1)) { // read HTTP header; FIXME: bad efficiency
- if (buf[l] == '\n' && l >= 3)
- if (strncmp(buf + l - 3, "\r\n\r\n", 4) == 0) break;
- ++l;
- }
- buf[l] = 0;
- if (l < 14) { // prematured header
- netclose(fp->fd);
- fp->fd = -1;
- return -1;
- }
- ret = strtol(buf + 8, &p, 0); // HTTP return code
- if (ret == 200 && fp->offset>0) { // 200 (complete result); then skip beginning of the file
- off_t rest = fp->offset;
- while (rest) {
- off_t l = rest < 0x10000? rest : 0x10000;
- rest -= my_netread(fp->fd, buf, l);
- }
- } else if (ret != 206 && ret != 200) {
- free(buf);
- fprintf(stderr, "[khttp_connect_file] fail to open file (HTTP code: %d).\n", ret);
- netclose(fp->fd);
- fp->fd = -1;
- return -1;
- }
- free(buf);
- fp->is_ready = 1;
- return 0;
-}
-
-/********************
- * Generic routines *
- ********************/
-
-knetFile *knet_open(const char *fn, const char *mode)
-{
- knetFile *fp = 0;
- if (mode[0] != 'r') {
- fprintf(stderr, "[kftp_open] only mode \"r\" is supported.\n");
- return 0;
- }
- if (strstr(fn, "ftp://") == fn) {
- fp = kftp_parse_url(fn, mode);
- if (fp == 0) return 0;
- if (kftp_connect(fp) == -1) {
- knet_close(fp);
- return 0;
- }
- kftp_connect_file(fp);
- } else if (strstr(fn, "http://") == fn) {
- fp = khttp_parse_url(fn, mode);
- if (fp == 0) return 0;
- khttp_connect_file(fp);
- } else { // local file
-#ifdef _WIN32
- /* In windows, O_BINARY is necessary. In Linux/Mac, O_BINARY may
- * be undefined on some systems, although it is defined on my
- * Mac and the Linux I have tested on. */
- int fd = open(fn, O_RDONLY | O_BINARY);
-#else
- int fd = open(fn, O_RDONLY);
-#endif
- if (fd == -1) {
- perror("open");
- return 0;
- }
- fp = (knetFile*)calloc(1, sizeof(knetFile));
- fp->type = KNF_TYPE_LOCAL;
- fp->fd = fd;
- fp->ctrl_fd = -1;
- }
- if (fp && fp->fd == -1) {
- knet_close(fp);
- return 0;
- }
- return fp;
-}
-
-knetFile *knet_dopen(int fd, const char *mode)
-{
- knetFile *fp = (knetFile*)calloc(1, sizeof(knetFile));
- fp->type = KNF_TYPE_LOCAL;
- fp->fd = fd;
- return fp;
-}
-
-off_t knet_read(knetFile *fp, void *buf, off_t len)
-{
- off_t l = 0;
- if (fp->fd == -1) return 0;
- if (fp->type == KNF_TYPE_FTP) {
- if (fp->is_ready == 0) {
- if (!fp->no_reconnect) kftp_reconnect(fp);
- kftp_connect_file(fp);
- }
- } else if (fp->type == KNF_TYPE_HTTP) {
- if (fp->is_ready == 0)
- khttp_connect_file(fp);
- }
- if (fp->type == KNF_TYPE_LOCAL) { // on Windows, the following block is necessary; not on UNIX
- off_t rest = len, curr;
- while (rest) {
- do {
- curr = read(fp->fd, buf + l, rest);
- } while (curr < 0 && EINTR == errno);
- if (curr < 0) return -1;
- if (curr == 0) break;
- l += curr; rest -= curr;
- }
- } else l = my_netread(fp->fd, buf, len);
- fp->offset += l;
- return l;
-}
-
-off_t knet_seek(knetFile *fp, int64_t off, int whence)
-{
- if (whence == SEEK_SET && off == fp->offset) return 0;
- if (fp->type == KNF_TYPE_LOCAL) {
- /* Be aware that lseek() returns the offset after seeking,
- * while fseek() returns zero on success. */
- off_t offset = lseek(fp->fd, off, whence);
- if (offset == -1) {
- // Be silent, it is OK for knet_seek to fail when the file is streamed
- // fprintf(stderr,"[knet_seek] %s\n", strerror(errno));
- return -1;
- }
- fp->offset = offset;
- return 0;
- }
- else if (fp->type == KNF_TYPE_FTP)
- {
- if (whence==SEEK_CUR)
- fp->offset += off;
- else if (whence==SEEK_SET)
- fp->offset = off;
- else if ( whence==SEEK_END)
- fp->offset = fp->file_size+off;
- fp->is_ready = 0;
- return 0;
- }
- else if (fp->type == KNF_TYPE_HTTP)
- {
- if (whence == SEEK_END) { // FIXME: can we allow SEEK_END in future?
- fprintf(stderr, "[knet_seek] SEEK_END is not supported for HTTP. Offset is unchanged.\n");
- errno = ESPIPE;
- return -1;
- }
- if (whence==SEEK_CUR)
- fp->offset += off;
- else if (whence==SEEK_SET)
- fp->offset = off;
- fp->is_ready = 0;
- return 0;
- }
- errno = EINVAL;
- fprintf(stderr,"[knet_seek] %s\n", strerror(errno));
- return -1;
-}
-
-int knet_close(knetFile *fp)
-{
- if (fp == 0) return 0;
- if (fp->ctrl_fd != -1) netclose(fp->ctrl_fd); // FTP specific
- if (fp->fd != -1) {
- /* On Linux/Mac, netclose() is an alias of close(), but on
- * Windows, it is an alias of closesocket(). */
- if (fp->type == KNF_TYPE_LOCAL) close(fp->fd);
- else netclose(fp->fd);
- }
- free(fp->host); free(fp->port);
- free(fp->response); free(fp->retr); // FTP specific
- free(fp->path); free(fp->http_host); // HTTP specific
- free(fp);
- return 0;
-}
-
-#ifdef KNETFILE_MAIN
-int main(void)
-{
- char *buf;
- knetFile *fp;
- int type = 4, l;
-#ifdef _WIN32
- knet_win32_init();
-#endif
- buf = calloc(0x100000, 1);
- if (type == 0) {
- fp = knet_open("knetfile.c", "r");
- knet_seek(fp, 1000, SEEK_SET);
- } else if (type == 1) { // NCBI FTP, large file
- fp = knet_open("ftp://ftp.ncbi.nih.gov/1000genomes/ftp/data/NA12878/alignment/NA12878.chrom6.SLX.SRP000032.2009_06.bam", "r");
- knet_seek(fp, 2500000000ll, SEEK_SET);
- l = knet_read(fp, buf, 255);
- } else if (type == 2) {
- fp = knet_open("ftp://ftp.sanger.ac.uk/pub4/treefam/tmp/index.shtml", "r");
- knet_seek(fp, 1000, SEEK_SET);
- } else if (type == 3) {
- fp = knet_open("http://www.sanger.ac.uk/Users/lh3/index.shtml", "r");
- knet_seek(fp, 1000, SEEK_SET);
- } else if (type == 4) {
- fp = knet_open("http://www.sanger.ac.uk/Users/lh3/ex1.bam", "r");
- knet_read(fp, buf, 10000);
- knet_seek(fp, 20000, SEEK_SET);
- knet_seek(fp, 10000, SEEK_SET);
- l = knet_read(fp, buf+10000, 10000000) + 10000;
- }
- if (type != 4 && type != 1) {
- knet_read(fp, buf, 255);
- buf[255] = 0;
- printf("%s\n", buf);
- } else write(fileno(stdout), buf, l);
- knet_close(fp);
- free(buf);
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/knetfile.h b/samtools-0.1.19/knetfile.h
deleted file mode 100644
index 0a0e66f..0000000
--- a/samtools-0.1.19/knetfile.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#ifndef KNETFILE_H
-#define KNETFILE_H
-
-#include <stdint.h>
-#include <fcntl.h>
-
-#ifndef _WIN32
-#define netread(fd, ptr, len) read(fd, ptr, len)
-#define netwrite(fd, ptr, len) write(fd, ptr, len)
-#define netclose(fd) close(fd)
-#else
-#include <winsock2.h>
-#define netread(fd, ptr, len) recv(fd, ptr, len, 0)
-#define netwrite(fd, ptr, len) send(fd, ptr, len, 0)
-#define netclose(fd) closesocket(fd)
-#endif
-
-// FIXME: currently I/O is unbuffered
-
-#define KNF_TYPE_LOCAL 1
-#define KNF_TYPE_FTP 2
-#define KNF_TYPE_HTTP 3
-
-typedef struct knetFile_s {
- int type, fd;
- int64_t offset;
- char *host, *port;
-
- // the following are for FTP only
- int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready;
- char *response, *retr, *size_cmd;
- int64_t seek_offset; // for lazy seek
- int64_t file_size;
-
- // the following are for HTTP only
- char *path, *http_host;
-} knetFile;
-
-#define knet_tell(fp) ((fp)->offset)
-#define knet_fileno(fp) ((fp)->fd)
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef _WIN32
- int knet_win32_init();
- void knet_win32_destroy();
-#endif
-
- knetFile *knet_open(const char *fn, const char *mode);
-
- /*
- This only works with local files.
- */
- knetFile *knet_dopen(int fd, const char *mode);
-
- /*
- If ->is_ready==0, this routine updates ->fd; otherwise, it simply
- reads from ->fd.
- */
- off_t knet_read(knetFile *fp, void *buf, off_t len);
-
- /*
- This routine only sets ->offset and ->is_ready=0. It does not
- communicate with the FTP server.
- */
- off_t knet_seek(knetFile *fp, int64_t off, int whence);
- int knet_close(knetFile *fp);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/kprobaln.c b/samtools-0.1.19/kprobaln.c
deleted file mode 100644
index 04e526a..0000000
--- a/samtools-0.1.19/kprobaln.c
+++ /dev/null
@@ -1,280 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2003-2006, 2008-2010, by Heng Li <lh3lh3 at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdint.h>
-#include <math.h>
-#include "kprobaln.h"
-
-/*****************************************
- * Probabilistic banded glocal alignment *
- *****************************************/
-
-#define EI .25
-#define EM .33333333333
-
-static float g_qual2prob[256];
-
-#define set_u(u, b, i, k) { int x=(i)-(b); x=x>0?x:0; (u)=((k)-x+1)*3; }
-
-kpa_par_t kpa_par_def = { 0.001, 0.1, 10 };
-kpa_par_t kpa_par_alt = { 0.0001, 0.01, 10 };
-
-/*
- The topology of the profile HMM:
-
- /\ /\ /\ /\
- I[1] I[k-1] I[k] I[L]
- ^ \ \ ^ \ ^ \ \ ^
- | \ \ | \ | \ \ |
- M[0] M[1] -> ... -> M[k-1] -> M[k] -> ... -> M[L] M[L+1]
- \ \/ \/ \/ /
- \ /\ /\ /\ /
- -> D[k-1] -> D[k] ->
-
- M[0] points to every {M,I}[k] and every {M,I}[k] points M[L+1].
-
- On input, _ref is the reference sequence and _query is the query
- sequence. Both are sequences of 0/1/2/3/4 where 4 stands for an
- ambiguous residue. iqual is the base quality. c sets the gap open
- probability, gap extension probability and band width.
-
- On output, state and q are arrays of length l_query. The higher 30
- bits give the reference position the query base is matched to and the
- lower two bits can be 0 (an alignment match) or 1 (an
- insertion). q[i] gives the phred scaled posterior probability of
- state[i] being wrong.
- */
-int kpa_glocal(const uint8_t *_ref, int l_ref, const uint8_t *_query, int l_query, const uint8_t *iqual,
- const kpa_par_t *c, int *state, uint8_t *q)
-{
- double **f, **b = 0, *s, m[9], sI, sM, bI, bM, pb;
- float *qual, *_qual;
- const uint8_t *ref, *query;
- int bw, bw2, i, k, is_diff = 0, is_backward = 1, Pr;
-
- if ( l_ref<=0 || l_query<=0 ) return 0; // FIXME: this may not be an ideal fix, just prevents sefgault
-
- /*** initialization ***/
- is_backward = state && q? 1 : 0;
- ref = _ref - 1; query = _query - 1; // change to 1-based coordinate
- bw = l_ref > l_query? l_ref : l_query;
- if (bw > c->bw) bw = c->bw;
- if (bw < abs(l_ref - l_query)) bw = abs(l_ref - l_query);
- bw2 = bw * 2 + 1;
- // allocate the forward and backward matrices f[][] and b[][] and the scaling array s[]
- f = calloc(l_query+1, sizeof(void*));
- if (is_backward) b = calloc(l_query+1, sizeof(void*));
- for (i = 0; i <= l_query; ++i) { // FIXME: this will lead in segfault for l_query==0
- f[i] = calloc(bw2 * 3 + 6, sizeof(double)); // FIXME: this is over-allocated for very short seqs
- if (is_backward) b[i] = calloc(bw2 * 3 + 6, sizeof(double));
- }
- s = calloc(l_query+2, sizeof(double)); // s[] is the scaling factor to avoid underflow
- // initialize qual
- _qual = calloc(l_query, sizeof(float));
- if (g_qual2prob[0] == 0)
- for (i = 0; i < 256; ++i)
- g_qual2prob[i] = pow(10, -i/10.);
- for (i = 0; i < l_query; ++i) _qual[i] = g_qual2prob[iqual? iqual[i] : 30];
- qual = _qual - 1;
- // initialize transition probability
- sM = sI = 1. / (2 * l_query + 2); // the value here seems not to affect results; FIXME: need proof
- m[0*3+0] = (1 - c->d - c->d) * (1 - sM); m[0*3+1] = m[0*3+2] = c->d * (1 - sM);
- m[1*3+0] = (1 - c->e) * (1 - sI); m[1*3+1] = c->e * (1 - sI); m[1*3+2] = 0.;
- m[2*3+0] = 1 - c->e; m[2*3+1] = 0.; m[2*3+2] = c->e;
- bM = (1 - c->d) / l_ref; bI = c->d / l_ref; // (bM+bI)*l_ref==1
- /*** forward ***/
- // f[0]
- set_u(k, bw, 0, 0);
- f[0][k] = s[0] = 1.;
- { // f[1]
- double *fi = f[1], sum;
- int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1, _beg, _end;
- for (k = beg, sum = 0.; k <= end; ++k) {
- int u;
- double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] * EM;
- set_u(u, bw, 1, k);
- fi[u+0] = e * bM; fi[u+1] = EI * bI;
- sum += fi[u] + fi[u+1];
- }
- // rescale
- s[1] = sum;
- set_u(_beg, bw, 1, beg); set_u(_end, bw, 1, end); _end += 2;
- for (k = _beg; k <= _end; ++k) fi[k] /= sum;
- }
- // f[2..l_query]
- for (i = 2; i <= l_query; ++i) {
- double *fi = f[i], *fi1 = f[i-1], sum, qli = qual[i];
- int beg = 1, end = l_ref, x, _beg, _end;
- uint8_t qyi = query[i];
- x = i - bw; beg = beg > x? beg : x; // band start
- x = i + bw; end = end < x? end : x; // band end
- for (k = beg, sum = 0.; k <= end; ++k) {
- int u, v11, v01, v10;
- double e;
- e = (ref[k] > 3 || qyi > 3)? 1. : ref[k] == qyi? 1. - qli : qli * EM;
- set_u(u, bw, i, k); set_u(v11, bw, i-1, k-1); set_u(v10, bw, i-1, k); set_u(v01, bw, i, k-1);
- fi[u+0] = e * (m[0] * fi1[v11+0] + m[3] * fi1[v11+1] + m[6] * fi1[v11+2]);
- fi[u+1] = EI * (m[1] * fi1[v10+0] + m[4] * fi1[v10+1]);
- fi[u+2] = m[2] * fi[v01+0] + m[8] * fi[v01+2];
- sum += fi[u] + fi[u+1] + fi[u+2];
-// fprintf(stderr, "F (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, fi[u], fi[u+1], fi[u+2]); // DEBUG
- }
- // rescale
- s[i] = sum;
- set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2;
- for (k = _beg, sum = 1./sum; k <= _end; ++k) fi[k] *= sum;
- }
- { // f[l_query+1]
- double sum;
- for (k = 1, sum = 0.; k <= l_ref; ++k) {
- int u;
- set_u(u, bw, l_query, k);
- if (u < 3 || u >= bw2*3+3) continue;
- sum += f[l_query][u+0] * sM + f[l_query][u+1] * sI;
- }
- s[l_query+1] = sum; // the last scaling factor
- }
- { // compute likelihood
- double p = 1., Pr1 = 0.;
- for (i = 0; i <= l_query + 1; ++i) {
- p *= s[i];
- if (p < 1e-100) Pr1 += -4.343 * log(p), p = 1.;
- }
- Pr1 += -4.343 * log(p * l_ref * l_query);
- Pr = (int)(Pr1 + .499);
- if (!is_backward) { // skip backward and MAP
- for (i = 0; i <= l_query; ++i) free(f[i]);
- free(f); free(s); free(_qual);
- return Pr;
- }
- }
- /*** backward ***/
- // b[l_query] (b[l_query+1][0]=1 and thus \tilde{b}[][]=1/s[l_query+1]; this is where s[l_query+1] comes from)
- for (k = 1; k <= l_ref; ++k) {
- int u;
- double *bi = b[l_query];
- set_u(u, bw, l_query, k);
- if (u < 3 || u >= bw2*3+3) continue;
- bi[u+0] = sM / s[l_query] / s[l_query+1]; bi[u+1] = sI / s[l_query] / s[l_query+1];
- }
- // b[l_query-1..1]
- for (i = l_query - 1; i >= 1; --i) {
- int beg = 1, end = l_ref, x, _beg, _end;
- double *bi = b[i], *bi1 = b[i+1], y = (i > 1), qli1 = qual[i+1];
- uint8_t qyi1 = query[i+1];
- x = i - bw; beg = beg > x? beg : x;
- x = i + bw; end = end < x? end : x;
- for (k = end; k >= beg; --k) {
- int u, v11, v01, v10;
- double e;
- set_u(u, bw, i, k); set_u(v11, bw, i+1, k+1); set_u(v10, bw, i+1, k); set_u(v01, bw, i, k+1);
- e = (k >= l_ref? 0 : (ref[k+1] > 3 || qyi1 > 3)? 1. : ref[k+1] == qyi1? 1. - qli1 : qli1 * EM) * bi1[v11];
- bi[u+0] = e * m[0] + EI * m[1] * bi1[v10+1] + m[2] * bi[v01+2]; // bi1[v11] has been foled into e.
- bi[u+1] = e * m[3] + EI * m[4] * bi1[v10+1];
- bi[u+2] = (e * m[6] + m[8] * bi[v01+2]) * y;
-// fprintf(stderr, "B (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, bi[u], bi[u+1], bi[u+2]); // DEBUG
- }
- // rescale
- set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2;
- for (k = _beg, y = 1./s[i]; k <= _end; ++k) bi[k] *= y;
- }
- { // b[0]
- int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1;
- double sum = 0.;
- for (k = end; k >= beg; --k) {
- int u;
- double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] * EM;
- set_u(u, bw, 1, k);
- if (u < 3 || u >= bw2*3+3) continue;
- sum += e * b[1][u+0] * bM + EI * b[1][u+1] * bI;
- }
- set_u(k, bw, 0, 0);
- pb = b[0][k] = sum / s[0]; // if everything works as is expected, pb == 1.0
- }
- is_diff = fabs(pb - 1.) > 1e-7? 1 : 0;
- /*** MAP ***/
- for (i = 1; i <= l_query; ++i) {
- double sum = 0., *fi = f[i], *bi = b[i], max = 0.;
- int beg = 1, end = l_ref, x, max_k = -1;
- x = i - bw; beg = beg > x? beg : x;
- x = i + bw; end = end < x? end : x;
- for (k = beg; k <= end; ++k) {
- int u;
- double z;
- set_u(u, bw, i, k);
- z = fi[u+0] * bi[u+0]; if (z > max) max = z, max_k = (k-1)<<2 | 0; sum += z;
- z = fi[u+1] * bi[u+1]; if (z > max) max = z, max_k = (k-1)<<2 | 1; sum += z;
- }
- max /= sum; sum *= s[i]; // if everything works as is expected, sum == 1.0
- if (state) state[i-1] = max_k;
- if (q) k = (int)(-4.343 * log(1. - max) + .499), q[i-1] = k > 100? 99 : k;
-#ifdef _MAIN
- fprintf(stderr, "(%.10lg,%.10lg) (%d,%d:%c,%c:%d) %lg\n", pb, sum, i-1, max_k>>2,
- "ACGT"[query[i]], "ACGT"[ref[(max_k>>2)+1]], max_k&3, max); // DEBUG
-#endif
- }
- /*** free ***/
- for (i = 0; i <= l_query; ++i) {
- free(f[i]); free(b[i]);
- }
- free(f); free(b); free(s); free(_qual);
- return Pr;
-}
-
-#ifdef _MAIN
-#include <unistd.h>
-int main(int argc, char *argv[])
-{
- uint8_t conv[256], *iqual, *ref, *query;
- int c, l_ref, l_query, i, q = 30, b = 10, P;
- while ((c = getopt(argc, argv, "b:q:")) >= 0) {
- switch (c) {
- case 'b': b = atoi(optarg); break;
- case 'q': q = atoi(optarg); break;
- }
- }
- if (optind + 2 > argc) {
- fprintf(stderr, "Usage: %s [-q %d] [-b %d] <ref> <query>\n", argv[0], q, b); // example: acttc attc
- return 1;
- }
- memset(conv, 4, 256);
- conv['a'] = conv['A'] = 0; conv['c'] = conv['C'] = 1;
- conv['g'] = conv['G'] = 2; conv['t'] = conv['T'] = 3;
- ref = (uint8_t*)argv[optind]; query = (uint8_t*)argv[optind+1];
- l_ref = strlen((char*)ref); l_query = strlen((char*)query);
- for (i = 0; i < l_ref; ++i) ref[i] = conv[ref[i]];
- for (i = 0; i < l_query; ++i) query[i] = conv[query[i]];
- iqual = malloc(l_query);
- memset(iqual, q, l_query);
- kpa_par_def.bw = b;
- P = kpa_glocal(ref, l_ref, query, l_query, iqual, &kpa_par_alt, 0, 0);
- fprintf(stderr, "%d\n", P);
- free(iqual);
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/kprobaln.h b/samtools-0.1.19/kprobaln.h
deleted file mode 100644
index 0357dcc..0000000
--- a/samtools-0.1.19/kprobaln.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2003-2006, 2008, 2009 by Heng Li <lh3 at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#ifndef LH3_KPROBALN_H_
-#define LH3_KPROBALN_H_
-
-#include <stdint.h>
-
-typedef struct {
- float d, e;
- int bw;
-} kpa_par_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- int kpa_glocal(const uint8_t *_ref, int l_ref, const uint8_t *_query, int l_query, const uint8_t *iqual,
- const kpa_par_t *c, int *state, uint8_t *q);
-
-#ifdef __cplusplus
-}
-#endif
-
-extern kpa_par_t kpa_par_def, kpa_par_alt;
-
-#endif
diff --git a/samtools-0.1.19/kseq.h b/samtools-0.1.19/kseq.h
deleted file mode 100644
index a5cec7c..0000000
--- a/samtools-0.1.19/kseq.h
+++ /dev/null
@@ -1,235 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008, 2009, 2011 Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Last Modified: 05MAR2012 */
-
-#ifndef AC_KSEQ_H
-#define AC_KSEQ_H
-
-#include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r
-#define KS_SEP_TAB 1 // isspace() && !' '
-#define KS_SEP_LINE 2 // line separator: "\n" (Unix) or "\r\n" (Windows)
-#define KS_SEP_MAX 2
-
-#define __KS_TYPE(type_t) \
- typedef struct __kstream_t { \
- unsigned char *buf; \
- int begin, end, is_eof; \
- type_t f; \
- } kstream_t;
-
-#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end)
-#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0)
-
-#define __KS_BASIC(type_t, __bufsize) \
- static inline kstream_t *ks_init(type_t f) \
- { \
- kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t)); \
- ks->f = f; \
- ks->buf = (unsigned char*)malloc(__bufsize); \
- return ks; \
- } \
- static inline void ks_destroy(kstream_t *ks) \
- { \
- if (ks) { \
- free(ks->buf); \
- free(ks); \
- } \
- }
-
-#define __KS_GETC(__read, __bufsize) \
- static inline int ks_getc(kstream_t *ks) \
- { \
- if (ks->is_eof && ks->begin >= ks->end) return -1; \
- if (ks->begin >= ks->end) { \
- ks->begin = 0; \
- ks->end = __read(ks->f, ks->buf, __bufsize); \
- if (ks->end < __bufsize) ks->is_eof = 1; \
- if (ks->end == 0) return -1; \
- } \
- return (int)ks->buf[ks->begin++]; \
- }
-
-#ifndef KSTRING_T
-#define KSTRING_T kstring_t
-typedef struct __kstring_t {
- size_t l, m;
- char *s;
-} kstring_t;
-#endif
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
-#define __KS_GETUNTIL(__read, __bufsize) \
- static int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append) \
- { \
- if (dret) *dret = 0; \
- str->l = append? str->l : 0; \
- if (ks->begin >= ks->end && ks->is_eof) return -1; \
- for (;;) { \
- int i; \
- if (ks->begin >= ks->end) { \
- if (!ks->is_eof) { \
- ks->begin = 0; \
- ks->end = __read(ks->f, ks->buf, __bufsize); \
- if (ks->end < __bufsize) ks->is_eof = 1; \
- if (ks->end == 0) break; \
- } else break; \
- } \
- if (delimiter == KS_SEP_LINE) { \
- for (i = ks->begin; i < ks->end; ++i) \
- if (ks->buf[i] == '\n') break; \
- } else if (delimiter > KS_SEP_MAX) { \
- for (i = ks->begin; i < ks->end; ++i) \
- if (ks->buf[i] == delimiter) break; \
- } else if (delimiter == KS_SEP_SPACE) { \
- for (i = ks->begin; i < ks->end; ++i) \
- if (isspace(ks->buf[i])) break; \
- } else if (delimiter == KS_SEP_TAB) { \
- for (i = ks->begin; i < ks->end; ++i) \
- if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break; \
- } else i = 0; /* never come to here! */ \
- if (str->m - str->l < (size_t)(i - ks->begin + 1)) { \
- str->m = str->l + (i - ks->begin) + 1; \
- kroundup32(str->m); \
- str->s = (char*)realloc(str->s, str->m); \
- } \
- memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin); \
- str->l = str->l + (i - ks->begin); \
- ks->begin = i + 1; \
- if (i < ks->end) { \
- if (dret) *dret = ks->buf[i]; \
- break; \
- } \
- } \
- if (str->s == 0) { \
- str->m = 1; \
- str->s = (char*)calloc(1, 1); \
- } else if (delimiter == KS_SEP_LINE && str->l > 1 && str->s[str->l-1] == '\r') --str->l; \
- str->s[str->l] = '\0'; \
- return str->l; \
- } \
- static inline int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \
- { return ks_getuntil2(ks, delimiter, str, dret, 0); }
-
-#define KSTREAM_INIT(type_t, __read, __bufsize) \
- __KS_TYPE(type_t) \
- __KS_BASIC(type_t, __bufsize) \
- __KS_GETC(__read, __bufsize) \
- __KS_GETUNTIL(__read, __bufsize)
-
-#define kseq_rewind(ks) ((ks)->last_char = (ks)->f->is_eof = (ks)->f->begin = (ks)->f->end = 0)
-
-#define __KSEQ_BASIC(SCOPE, type_t) \
- SCOPE kseq_t *kseq_init(type_t fd) \
- { \
- kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t)); \
- s->f = ks_init(fd); \
- return s; \
- } \
- SCOPE void kseq_destroy(kseq_t *ks) \
- { \
- if (!ks) return; \
- free(ks->name.s); free(ks->comment.s); free(ks->seq.s); free(ks->qual.s); \
- ks_destroy(ks->f); \
- free(ks); \
- }
-
-/* Return value:
- >=0 length of the sequence (normal)
- -1 end-of-file
- -2 truncated quality string
- */
-#define __KSEQ_READ(SCOPE) \
- SCOPE int kseq_read(kseq_t *seq) \
- { \
- int c; \
- kstream_t *ks = seq->f; \
- if (seq->last_char == 0) { /* then jump to the next header line */ \
- while ((c = ks_getc(ks)) != -1 && c != '>' && c != '@'); \
- if (c == -1) return -1; /* end of file */ \
- seq->last_char = c; \
- } /* else: the first header char has been read in the previous call */ \
- seq->comment.l = seq->seq.l = seq->qual.l = 0; /* reset all members */ \
- if (ks_getuntil(ks, 0, &seq->name, &c) < 0) return -1; /* normal exit: EOF */ \
- if (c != '\n') ks_getuntil(ks, KS_SEP_LINE, &seq->comment, 0); /* read FASTA/Q comment */ \
- if (seq->seq.s == 0) { /* we can do this in the loop below, but that is slower */ \
- seq->seq.m = 256; \
- seq->seq.s = (char*)malloc(seq->seq.m); \
- } \
- while ((c = ks_getc(ks)) != -1 && c != '>' && c != '+' && c != '@') { \
- if (c == '\n') continue; /* skip empty lines */ \
- seq->seq.s[seq->seq.l++] = c; /* this is safe: we always have enough space for 1 char */ \
- ks_getuntil2(ks, KS_SEP_LINE, &seq->seq, 0, 1); /* read the rest of the line */ \
- } \
- if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */ \
- if (seq->seq.l + 1 >= seq->seq.m) { /* seq->seq.s[seq->seq.l] below may be out of boundary */ \
- seq->seq.m = seq->seq.l + 2; \
- kroundup32(seq->seq.m); /* rounded to the next closest 2^k */ \
- seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \
- } \
- seq->seq.s[seq->seq.l] = 0; /* null terminated string */ \
- if (c != '+') return seq->seq.l; /* FASTA */ \
- if (seq->qual.m < seq->seq.m) { /* allocate memory for qual in case insufficient */ \
- seq->qual.m = seq->seq.m; \
- seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \
- } \
- while ((c = ks_getc(ks)) != -1 && c != '\n'); /* skip the rest of '+' line */ \
- if (c == -1) return -2; /* error: no quality string */ \
- while (ks_getuntil2(ks, KS_SEP_LINE, &seq->qual, 0, 1) >= 0 && seq->qual.l < seq->seq.l); \
- seq->last_char = 0; /* we have not come to the next header line */ \
- if (seq->seq.l != seq->qual.l) return -2; /* error: qual string is of a different length */ \
- return seq->seq.l; \
- }
-
-#define __KSEQ_TYPE(type_t) \
- typedef struct { \
- kstring_t name, comment, seq, qual; \
- int last_char; \
- kstream_t *f; \
- } kseq_t;
-
-#define KSEQ_INIT2(SCOPE, type_t, __read) \
- KSTREAM_INIT(type_t, __read, 16384) \
- __KSEQ_TYPE(type_t) \
- __KSEQ_BASIC(SCOPE, type_t) \
- __KSEQ_READ(SCOPE)
-
-#define KSEQ_INIT(type_t, __read) KSEQ_INIT2(static, type_t, __read)
-
-#define KSEQ_DECLARE(type_t) \
- __KS_TYPE(type_t) \
- __KSEQ_TYPE(type_t) \
- extern kseq_t *kseq_init(type_t fd); \
- void kseq_destroy(kseq_t *ks); \
- int kseq_read(kseq_t *seq);
-
-#endif
diff --git a/samtools-0.1.19/ksort.h b/samtools-0.1.19/ksort.h
deleted file mode 100644
index aa0bb93..0000000
--- a/samtools-0.1.19/ksort.h
+++ /dev/null
@@ -1,285 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 Genome Research Ltd (GRL).
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* Contact: Heng Li <lh3 at sanger.ac.uk> */
-
-/*
- 2012-12-11 (0.1.4):
-
- * Defined __ks_insertsort_##name as static to compile with C99.
-
- 2008-11-16 (0.1.4):
-
- * Fixed a bug in introsort() that happens in rare cases.
-
- 2008-11-05 (0.1.3):
-
- * Fixed a bug in introsort() for complex comparisons.
-
- * Fixed a bug in mergesort(). The previous version is not stable.
-
- 2008-09-15 (0.1.2):
-
- * Accelerated introsort. On my Mac (not on another Linux machine),
- my implementation is as fast as std::sort on random input.
-
- * Added combsort and in introsort, switch to combsort if the
- recursion is too deep.
-
- 2008-09-13 (0.1.1):
-
- * Added k-small algorithm
-
- 2008-09-05 (0.1.0):
-
- * Initial version
-
-*/
-
-#ifndef AC_KSORT_H
-#define AC_KSORT_H
-
-#include <stdlib.h>
-#include <string.h>
-
-typedef struct {
- void *left, *right;
- int depth;
-} ks_isort_stack_t;
-
-#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; }
-
-#define KSORT_INIT(name, type_t, __sort_lt) \
- void ks_mergesort_##name(size_t n, type_t array[], type_t temp[]) \
- { \
- type_t *a2[2], *a, *b; \
- int curr, shift; \
- \
- a2[0] = array; \
- a2[1] = temp? temp : (type_t*)malloc(sizeof(type_t) * n); \
- for (curr = 0, shift = 0; (1ul<<shift) < n; ++shift) { \
- a = a2[curr]; b = a2[1-curr]; \
- if (shift == 0) { \
- type_t *p = b, *i, *eb = a + n; \
- for (i = a; i < eb; i += 2) { \
- if (i == eb - 1) *p++ = *i; \
- else { \
- if (__sort_lt(*(i+1), *i)) { \
- *p++ = *(i+1); *p++ = *i; \
- } else { \
- *p++ = *i; *p++ = *(i+1); \
- } \
- } \
- } \
- } else { \
- size_t i, step = 1ul<<shift; \
- for (i = 0; i < n; i += step<<1) { \
- type_t *p, *j, *k, *ea, *eb; \
- if (n < i + step) { \
- ea = a + n; eb = a; \
- } else { \
- ea = a + i + step; \
- eb = a + (n < i + (step<<1)? n : i + (step<<1)); \
- } \
- j = a + i; k = a + i + step; p = b + i; \
- while (j < ea && k < eb) { \
- if (__sort_lt(*k, *j)) *p++ = *k++; \
- else *p++ = *j++; \
- } \
- while (j < ea) *p++ = *j++; \
- while (k < eb) *p++ = *k++; \
- } \
- } \
- curr = 1 - curr; \
- } \
- if (curr == 1) { \
- type_t *p = a2[0], *i = a2[1], *eb = array + n; \
- for (; p < eb; ++i) *p++ = *i; \
- } \
- if (temp == 0) free(a2[1]); \
- } \
- void ks_heapadjust_##name(size_t i, size_t n, type_t l[]) \
- { \
- size_t k = i; \
- type_t tmp = l[i]; \
- while ((k = (k << 1) + 1) < n) { \
- if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k; \
- if (__sort_lt(l[k], tmp)) break; \
- l[i] = l[k]; i = k; \
- } \
- l[i] = tmp; \
- } \
- void ks_heapmake_##name(size_t lsize, type_t l[]) \
- { \
- size_t i; \
- for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i) \
- ks_heapadjust_##name(i, lsize, l); \
- } \
- void ks_heapsort_##name(size_t lsize, type_t l[]) \
- { \
- size_t i; \
- for (i = lsize - 1; i > 0; --i) { \
- type_t tmp; \
- tmp = *l; *l = l[i]; l[i] = tmp; ks_heapadjust_##name(0, i, l); \
- } \
- } \
- static inline void __ks_insertsort_##name(type_t *s, type_t *t) \
- { \
- type_t *i, *j, swap_tmp; \
- for (i = s + 1; i < t; ++i) \
- for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) { \
- swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp; \
- } \
- } \
- void ks_combsort_##name(size_t n, type_t a[]) \
- { \
- const double shrink_factor = 1.2473309501039786540366528676643; \
- int do_swap; \
- size_t gap = n; \
- type_t tmp, *i, *j; \
- do { \
- if (gap > 2) { \
- gap = (size_t)(gap / shrink_factor); \
- if (gap == 9 || gap == 10) gap = 11; \
- } \
- do_swap = 0; \
- for (i = a; i < a + n - gap; ++i) { \
- j = i + gap; \
- if (__sort_lt(*j, *i)) { \
- tmp = *i; *i = *j; *j = tmp; \
- do_swap = 1; \
- } \
- } \
- } while (do_swap || gap > 2); \
- if (gap != 1) __ks_insertsort_##name(a, a + n); \
- } \
- void ks_introsort_##name(size_t n, type_t a[]) \
- { \
- int d; \
- ks_isort_stack_t *top, *stack; \
- type_t rp, swap_tmp; \
- type_t *s, *t, *i, *j, *k; \
- \
- if (n < 1) return; \
- else if (n == 2) { \
- if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \
- return; \
- } \
- for (d = 2; 1ul<<d < n; ++d); \
- stack = (ks_isort_stack_t*)malloc(sizeof(ks_isort_stack_t) * ((sizeof(size_t)*d)+2)); \
- top = stack; s = a; t = a + (n-1); d <<= 1; \
- while (1) { \
- if (s < t) { \
- if (--d == 0) { \
- ks_combsort_##name(t - s + 1, s); \
- t = s; \
- continue; \
- } \
- i = s; j = t; k = i + ((j-i)>>1) + 1; \
- if (__sort_lt(*k, *i)) { \
- if (__sort_lt(*k, *j)) k = j; \
- } else k = __sort_lt(*j, *i)? i : j; \
- rp = *k; \
- if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; } \
- for (;;) { \
- do ++i; while (__sort_lt(*i, rp)); \
- do --j; while (i <= j && __sort_lt(rp, *j)); \
- if (j <= i) break; \
- swap_tmp = *i; *i = *j; *j = swap_tmp; \
- } \
- swap_tmp = *i; *i = *t; *t = swap_tmp; \
- if (i-s > t-i) { \
- if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \
- s = t-i > 16? i+1 : t; \
- } else { \
- if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \
- t = i-s > 16? i-1 : s; \
- } \
- } else { \
- if (top == stack) { \
- free(stack); \
- __ks_insertsort_##name(a, a+n); \
- return; \
- } else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \
- } \
- } \
- } \
- /* This function is adapted from: http://ndevilla.free.fr/median/ */ \
- /* 0 <= kk < n */ \
- type_t ks_ksmall_##name(size_t n, type_t arr[], size_t kk) \
- { \
- type_t *low, *high, *k, *ll, *hh, *mid; \
- low = arr; high = arr + n - 1; k = arr + kk; \
- for (;;) { \
- if (high <= low) return *k; \
- if (high == low + 1) { \
- if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \
- return *k; \
- } \
- mid = low + (high - low) / 2; \
- if (__sort_lt(*high, *mid)) KSORT_SWAP(type_t, *mid, *high); \
- if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \
- if (__sort_lt(*low, *mid)) KSORT_SWAP(type_t, *mid, *low); \
- KSORT_SWAP(type_t, *mid, *(low+1)); \
- ll = low + 1; hh = high; \
- for (;;) { \
- do ++ll; while (__sort_lt(*ll, *low)); \
- do --hh; while (__sort_lt(*low, *hh)); \
- if (hh < ll) break; \
- KSORT_SWAP(type_t, *ll, *hh); \
- } \
- KSORT_SWAP(type_t, *low, *hh); \
- if (hh <= k) low = ll; \
- if (hh >= k) high = hh - 1; \
- } \
- } \
- void ks_shuffle_##name(size_t n, type_t a[]) \
- { \
- int i, j; \
- for (i = n; i > 1; --i) { \
- type_t tmp; \
- j = (int)(drand48() * i); \
- tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp; \
- } \
- }
-
-#define ks_mergesort(name, n, a, t) ks_mergesort_##name(n, a, t)
-#define ks_introsort(name, n, a) ks_introsort_##name(n, a)
-#define ks_combsort(name, n, a) ks_combsort_##name(n, a)
-#define ks_heapsort(name, n, a) ks_heapsort_##name(n, a)
-#define ks_heapmake(name, n, a) ks_heapmake_##name(n, a)
-#define ks_heapadjust(name, i, n, a) ks_heapadjust_##name(i, n, a)
-#define ks_ksmall(name, n, a, k) ks_ksmall_##name(n, a, k)
-#define ks_shuffle(name, n, a) ks_shuffle_##name(n, a)
-
-#define ks_lt_generic(a, b) ((a) < (b))
-#define ks_lt_str(a, b) (strcmp((a), (b)) < 0)
-
-typedef const char *ksstr_t;
-
-#define KSORT_INIT_GENERIC(type_t) KSORT_INIT(type_t, type_t, ks_lt_generic)
-#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str)
-
-#endif
diff --git a/samtools-0.1.19/kstring.c b/samtools-0.1.19/kstring.c
deleted file mode 100644
index b8ff45c..0000000
--- a/samtools-0.1.19/kstring.c
+++ /dev/null
@@ -1,212 +0,0 @@
-#include <stdarg.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-#include <stdint.h>
-#include "kstring.h"
-
-int ksprintf(kstring_t *s, const char *fmt, ...)
-{
- va_list ap;
- int l;
- va_start(ap, fmt);
- l = vsnprintf(s->s + s->l, s->m - s->l, fmt, ap); // This line does not work with glibc 2.0. See `man snprintf'.
- va_end(ap);
- if (l + 1 > s->m - s->l) {
- s->m = s->l + l + 2;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- va_start(ap, fmt);
- l = vsnprintf(s->s + s->l, s->m - s->l, fmt, ap);
- }
- va_end(ap);
- s->l += l;
- return l;
-}
-
-char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux)
-{
- const char *p, *start;
- if (sep) { // set up the table
- if (str == 0 && (aux->tab[0]&1)) return 0; // no need to set up if we have finished
- aux->finished = 0;
- if (sep[1]) {
- aux->sep = -1;
- aux->tab[0] = aux->tab[1] = aux->tab[2] = aux->tab[3] = 0;
- for (p = sep; *p; ++p) aux->tab[*p>>6] |= 1ull<<(*p&0x3f);
- } else aux->sep = sep[0];
- }
- if (aux->finished) return 0;
- else if (str) aux->p = str - 1, aux->finished = 0;
- if (aux->sep < 0) {
- for (p = start = aux->p + 1; *p; ++p)
- if (aux->tab[*p>>6]>>(*p&0x3f)&1) break;
- } else {
- for (p = start = aux->p + 1; *p; ++p)
- if (*p == aux->sep) break;
- }
- aux->p = p; // end of token
- if (*p == 0) aux->finished = 1; // no more tokens
- return (char*)start;
-}
-
-// s MUST BE a null terminated string; l = strlen(s)
-int ksplit_core(char *s, int delimiter, int *_max, int **_offsets)
-{
- int i, n, max, last_char, last_start, *offsets, l;
- n = 0; max = *_max; offsets = *_offsets;
- l = strlen(s);
-
-#define __ksplit_aux do { \
- if (_offsets) { \
- s[i] = 0; \
- if (n == max) { \
- max = max? max<<1 : 2; \
- offsets = (int*)realloc(offsets, sizeof(int) * max); \
- } \
- offsets[n++] = last_start; \
- } else ++n; \
- } while (0)
-
- for (i = 0, last_char = last_start = 0; i <= l; ++i) {
- if (delimiter == 0) {
- if (isspace(s[i]) || s[i] == 0) {
- if (isgraph(last_char)) __ksplit_aux; // the end of a field
- } else {
- if (isspace(last_char) || last_char == 0) last_start = i;
- }
- } else {
- if (s[i] == delimiter || s[i] == 0) {
- if (last_char != 0 && last_char != delimiter) __ksplit_aux; // the end of a field
- } else {
- if (last_char == delimiter || last_char == 0) last_start = i;
- }
- }
- last_char = s[i];
- }
- *_max = max; *_offsets = offsets;
- return n;
-}
-
-/**********************
- * Boyer-Moore search *
- **********************/
-
-typedef unsigned char ubyte_t;
-
-// reference: http://www-igm.univ-mlv.fr/~lecroq/string/node14.html
-static int *ksBM_prep(const ubyte_t *pat, int m)
-{
- int i, *suff, *prep, *bmGs, *bmBc;
- prep = (int*)calloc(m + 256, sizeof(int));
- bmGs = prep; bmBc = prep + m;
- { // preBmBc()
- for (i = 0; i < 256; ++i) bmBc[i] = m;
- for (i = 0; i < m - 1; ++i) bmBc[pat[i]] = m - i - 1;
- }
- suff = (int*)calloc(m, sizeof(int));
- { // suffixes()
- int f = 0, g;
- suff[m - 1] = m;
- g = m - 1;
- for (i = m - 2; i >= 0; --i) {
- if (i > g && suff[i + m - 1 - f] < i - g)
- suff[i] = suff[i + m - 1 - f];
- else {
- if (i < g) g = i;
- f = i;
- while (g >= 0 && pat[g] == pat[g + m - 1 - f]) --g;
- suff[i] = f - g;
- }
- }
- }
- { // preBmGs()
- int j = 0;
- for (i = 0; i < m; ++i) bmGs[i] = m;
- for (i = m - 1; i >= 0; --i)
- if (suff[i] == i + 1)
- for (; j < m - 1 - i; ++j)
- if (bmGs[j] == m)
- bmGs[j] = m - 1 - i;
- for (i = 0; i <= m - 2; ++i)
- bmGs[m - 1 - suff[i]] = m - 1 - i;
- }
- free(suff);
- return prep;
-}
-
-void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep)
-{
- int i, j, *prep = 0, *bmGs, *bmBc;
- const ubyte_t *str, *pat;
- str = (const ubyte_t*)_str; pat = (const ubyte_t*)_pat;
- prep = (_prep == 0 || *_prep == 0)? ksBM_prep(pat, m) : *_prep;
- if (_prep && *_prep == 0) *_prep = prep;
- bmGs = prep; bmBc = prep + m;
- j = 0;
- while (j <= n - m) {
- for (i = m - 1; i >= 0 && pat[i] == str[i+j]; --i);
- if (i >= 0) {
- int max = bmBc[str[i+j]] - m + 1 + i;
- if (max < bmGs[i]) max = bmGs[i];
- j += max;
- } else return (void*)(str + j);
- }
- if (_prep == 0) free(prep);
- return 0;
-}
-
-char *kstrstr(const char *str, const char *pat, int **_prep)
-{
- return (char*)kmemmem(str, strlen(str), pat, strlen(pat), _prep);
-}
-
-char *kstrnstr(const char *str, const char *pat, int n, int **_prep)
-{
- return (char*)kmemmem(str, n, pat, strlen(pat), _prep);
-}
-
-/***********************
- * The main() function *
- ***********************/
-
-#ifdef KSTRING_MAIN
-#include <stdio.h>
-int main()
-{
- kstring_t *s;
- int *fields, n, i;
- ks_tokaux_t aux;
- char *p;
- s = (kstring_t*)calloc(1, sizeof(kstring_t));
- // test ksprintf()
- ksprintf(s, " abcdefg: %d ", 100);
- printf("'%s'\n", s->s);
- // test ksplit()
- fields = ksplit(s, 0, &n);
- for (i = 0; i < n; ++i)
- printf("field[%d] = '%s'\n", i, s->s + fields[i]);
- // test kstrtok()
- s->l = 0;
- for (p = kstrtok("ab:cde:fg/hij::k", ":/", &aux); p; p = kstrtok(0, 0, &aux)) {
- kputsn(p, aux.p - p, s);
- kputc('\n', s);
- }
- printf("%s", s->s);
- // free
- free(s->s); free(s); free(fields);
-
- {
- static char *str = "abcdefgcdgcagtcakcdcd";
- static char *pat = "cd";
- char *ret, *s = str;
- int *prep = 0;
- while ((ret = kstrstr(s, pat, &prep)) != 0) {
- printf("match: %s\n", ret);
- s = ret + prep[0];
- }
- free(prep);
- }
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/kstring.h b/samtools-0.1.19/kstring.h
deleted file mode 100644
index abd8236..0000000
--- a/samtools-0.1.19/kstring.h
+++ /dev/null
@@ -1,169 +0,0 @@
-/* The MIT License
-
- Copyright (c) by Attractive Chaos <attractor at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#ifndef KSTRING_H
-#define KSTRING_H
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
-#ifndef KSTRING_T
-#define KSTRING_T kstring_t
-typedef struct __kstring_t {
- size_t l, m;
- char *s;
-} kstring_t;
-#endif
-
-typedef struct {
- uint64_t tab[4];
- int sep, finished;
- const char *p; // end of the current token
-} ks_tokaux_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- int ksprintf(kstring_t *s, const char *fmt, ...);
- int ksplit_core(char *s, int delimiter, int *_max, int **_offsets);
- char *kstrstr(const char *str, const char *pat, int **_prep);
- char *kstrnstr(const char *str, const char *pat, int n, int **_prep);
- void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep);
-
- /* kstrtok() is similar to strtok_r() except that str is not
- * modified and both str and sep can be NULL. For efficiency, it is
- * actually recommended to set both to NULL in the subsequent calls
- * if sep is not changed. */
- char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux);
-
-#ifdef __cplusplus
-}
-#endif
-
-static inline void ks_resize(kstring_t *s, size_t size)
-{
- if (s->m < size) {
- s->m = size;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- }
-}
-
-static inline int kputsn(const char *p, int l, kstring_t *s)
-{
- if (s->l + l + 1 >= s->m) {
- s->m = s->l + l + 2;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- }
- memcpy(s->s + s->l, p, l);
- s->l += l;
- s->s[s->l] = 0;
- return l;
-}
-
-static inline int kputs(const char *p, kstring_t *s)
-{
- return kputsn(p, strlen(p), s);
-}
-
-static inline int kputc(int c, kstring_t *s)
-{
- if (s->l + 1 >= s->m) {
- s->m = s->l + 2;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- }
- s->s[s->l++] = c;
- s->s[s->l] = 0;
- return c;
-}
-
-static inline int kputw(int c, kstring_t *s)
-{
- char buf[16];
- int l, x;
- if (c == 0) return kputc('0', s);
- if(c < 0) for (l = 0, x = c; x < 0; x /= 10) buf[l++] = '0' - (x%10);
- else for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0';
- if (c < 0) buf[l++] = '-';
- if (s->l + l + 1 >= s->m) {
- s->m = s->l + l + 2;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- }
- for (x = l - 1; x >= 0; --x) s->s[s->l++] = buf[x];
- s->s[s->l] = 0;
- return 0;
-}
-
-static inline int kputuw(unsigned c, kstring_t *s)
-{
- char buf[16];
- int l, i;
- unsigned x;
- if (c == 0) return kputc('0', s);
- for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0';
- if (s->l + l + 1 >= s->m) {
- s->m = s->l + l + 2;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- }
- for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
- s->s[s->l] = 0;
- return 0;
-}
-
-static inline int kputl(long c, kstring_t *s)
-{
- char buf[32];
- long l, x;
- if (c == 0) return kputc('0', s);
- for (l = 0, x = c < 0? -c : c; x > 0; x /= 10) buf[l++] = x%10 + '0';
- if (c < 0) buf[l++] = '-';
- if (s->l + l + 1 >= s->m) {
- s->m = s->l + l + 2;
- kroundup32(s->m);
- s->s = (char*)realloc(s->s, s->m);
- }
- for (x = l - 1; x >= 0; --x) s->s[s->l++] = buf[x];
- s->s[s->l] = 0;
- return 0;
-}
-
-static inline int *ksplit(kstring_t *s, int delimiter, int *n)
-{
- int max = 0, *offsets = 0;
- *n = ksplit_core(s->s, delimiter, &max, &offsets);
- return offsets;
-}
-
-#endif
diff --git a/samtools-0.1.19/misc/HmmGlocal.java b/samtools-0.1.19/misc/HmmGlocal.java
deleted file mode 100644
index 9e93b13..0000000
--- a/samtools-0.1.19/misc/HmmGlocal.java
+++ /dev/null
@@ -1,178 +0,0 @@
-import java.io.*;
-import java.lang.*;
-
-public class HmmGlocal
-{
- private double[] qual2prob;
- private double cd, ce; // gap open probility [1e-3], gap extension probability [0.1]
- private int cb; // band width [7]
-
- public HmmGlocal(final double d, final double e, final int b) {
- cd = d; ce = e; cb = b;
- qual2prob = new double[256];
- for (int i = 0; i < 256; ++i)
- qual2prob[i] = Math.pow(10, -i/10.);
- }
- private static int set_u(final int b, final int i, final int k) {
- int x = i - b;
- x = x > 0? x : 0;
- return (k + 1 - x) * 3;
- }
- public int hmm_glocal(final byte[] _ref, final byte[] _query, final byte[] _iqual, int[] state, byte[] q) {
- int i, k;
- /*** initialization ***/
- // change coordinates
- int l_ref = _ref.length;
- byte[] ref = new byte[l_ref+1];
- for (i = 0; i < l_ref; ++i) ref[i+1] = _ref[i]; // FIXME: this is silly...
- int l_query = _query.length;
- byte[] query = new byte[l_query+1];
- double[] qual = new double[l_query+1];
- for (i = 0; i < l_query; ++i) {
- query[i+1] = _query[i];
- qual[i+1] = qual2prob[_iqual[i]];
- }
- // set band width
- int bw2, bw = l_ref > l_query? l_ref : l_query;
- if (bw > cb) bw = cb;
- if (bw < Math.abs(l_ref - l_query)) bw = Math.abs(l_ref - l_query);
- bw2 = bw * 2 + 1;
- // allocate the forward and backward matrices f[][] and b[][] and the scaling array s[]
- double[][] f = new double[l_query+1][bw2*3 + 6];
- double[][] b = new double[l_query+1][bw2*3 + 6];
- double[] s = new double[l_query+2];
- // initialize transition probabilities
- double sM, sI, bM, bI;
- sM = sI = 1. / (2 * l_query + 2);
- bM = (1 - cd) / l_query; bI = cd / l_query; // (bM+bI)*l_query==1
- double[] m = new double[9];
- m[0*3+0] = (1 - cd - cd) * (1 - sM); m[0*3+1] = m[0*3+2] = cd * (1 - sM);
- m[1*3+0] = (1 - ce) * (1 - sI); m[1*3+1] = ce * (1 - sI); m[1*3+2] = 0.;
- m[2*3+0] = 1 - ce; m[2*3+1] = 0.; m[2*3+2] = ce;
- /*** forward ***/
- // f[0]
- f[0][set_u(bw, 0, 0)] = s[0] = 1.;
- { // f[1]
- double[] fi = f[1];
- double sum;
- int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1, _beg, _end;
- for (k = beg, sum = 0.; k <= end; ++k) {
- int u;
- double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] / 3.;
- u = set_u(bw, 1, k);
- fi[u+0] = e * bM; fi[u+1] = .25 * bI;
- sum += fi[u] + fi[u+1];
- }
- // rescale
- s[1] = sum;
- _beg = set_u(bw, 1, beg); _end = set_u(bw, 1, end); _end += 2;
- for (k = _beg; k <= _end; ++k) fi[k] /= sum;
- }
- // f[2..l_query]
- for (i = 2; i <= l_query; ++i) {
- double[] fi = f[i], fi1 = f[i-1];
- double sum, qli = qual[i];
- int beg = 1, end = l_ref, x, _beg, _end;
- byte qyi = query[i];
- x = i - bw; beg = beg > x? beg : x; // band start
- x = i + bw; end = end < x? end : x; // band end
- for (k = beg, sum = 0.; k <= end; ++k) {
- int u, v11, v01, v10;
- double e;
- e = (ref[k] > 3 || qyi > 3)? 1. : ref[k] == qyi? 1. - qli : qli / 3.;
- u = set_u(bw, i, k); v11 = set_u(bw, i-1, k-1); v10 = set_u(bw, i-1, k); v01 = set_u(bw, i, k-1);
- fi[u+0] = e * (m[0] * fi1[v11+0] + m[3] * fi1[v11+1] + m[6] * fi1[v11+2]);
- fi[u+1] = .25 * (m[1] * fi1[v10+0] + m[4] * fi1[v10+1]);
- fi[u+2] = m[2] * fi[v01+0] + m[8] * fi[v01+2];
- sum += fi[u] + fi[u+1] + fi[u+2];
- //System.out.println("("+i+","+k+";"+u+"): "+fi[u]+","+fi[u+1]+","+fi[u+2]);
- }
- // rescale
- s[i] = sum;
- _beg = set_u(bw, i, beg); _end = set_u(bw, i, end); _end += 2;
- for (k = _beg, sum = 1./sum; k <= _end; ++k) fi[k] *= sum;
- }
- { // f[l_query+1]
- double sum;
- for (k = 1, sum = 0.; k <= l_ref; ++k) {
- int u = set_u(bw, l_query, k);
- if (u < 3 || u >= bw2*3+3) continue;
- sum += f[l_query][u+0] * sM + f[l_query][u+1] * sI;
- }
- s[l_query+1] = sum; // the last scaling factor
- }
- /*** backward ***/
- // b[l_query] (b[l_query+1][0]=1 and thus \tilde{b}[][]=1/s[l_query+1]; this is where s[l_query+1] comes from)
- for (k = 1; k <= l_ref; ++k) {
- int u = set_u(bw, l_query, k);
- double[] bi = b[l_query];
- if (u < 3 || u >= bw2*3+3) continue;
- bi[u+0] = sM / s[l_query] / s[l_query+1]; bi[u+1] = sI / s[l_query] / s[l_query+1];
- }
- // b[l_query-1..1]
- for (i = l_query - 1; i >= 1; --i) {
- int beg = 1, end = l_ref, x, _beg, _end;
- double[] bi = b[i], bi1 = b[i+1];
- double y = (i > 1)? 1. : 0., qli1 = qual[i+1];
- byte qyi1 = query[i+1];
- x = i - bw; beg = beg > x? beg : x;
- x = i + bw; end = end < x? end : x;
- for (k = end; k >= beg; --k) {
- int u, v11, v01, v10;
- double e;
- u = set_u(bw, i, k); v11 = set_u(bw, i+1, k+1); v10 = set_u(bw, i+1, k); v01 = set_u(bw, i, k+1);
- e = (k >= l_ref? 0 : (ref[k+1] > 3 || qyi1 > 3)? 1. : ref[k+1] == qyi1? 1. - qli1 : qli1 / 3.) * bi1[v11];
- bi[u+0] = e * m[0] + .25 * m[1] * bi1[v10+1] + m[2] * bi[v01+2]; // bi1[v11] has been foled into e.
- bi[u+1] = e * m[3] + .25 * m[4] * bi1[v10+1];
- bi[u+2] = (e * m[6] + m[8] * bi[v01+2]) * y;
- }
- // rescale
- _beg = set_u(bw, i, beg); _end = set_u(bw, i, end); _end += 2;
- for (k = _beg, y = 1./s[i]; k <= _end; ++k) bi[k] *= y;
- }
- double pb;
- { // b[0]
- int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1;
- double sum = 0.;
- for (k = end; k >= beg; --k) {
- int u = set_u(bw, 1, k);
- double e = (ref[k] > 3 || query[1] > 3)? 1. : ref[k] == query[1]? 1. - qual[1] : qual[1] / 3.;
- if (u < 3 || u >= bw2*3+3) continue;
- sum += e * b[1][u+0] * bM + .25 * b[1][u+1] * bI;
- }
- pb = b[0][set_u(bw, 0, 0)] = sum / s[0]; // if everything works as is expected, pb == 1.0
- }
- int is_diff = Math.abs(pb - 1.) > 1e-7? 1 : 0;
- /*** MAP ***/
- for (i = 1; i <= l_query; ++i) {
- double sum = 0., max = 0.;
- double[] fi = f[i], bi = b[i];
- int beg = 1, end = l_ref, x, max_k = -1;
- x = i - bw; beg = beg > x? beg : x;
- x = i + bw; end = end < x? end : x;
- for (k = beg; k <= end; ++k) {
- int u = set_u(bw, i, k);
- double z;
- sum += (z = fi[u+0] * bi[u+0]); if (z > max) { max = z; max_k = (k-1)<<2 | 0; }
- sum += (z = fi[u+1] * bi[u+1]); if (z > max) { max = z; max_k = (k-1)<<2 | 1; }
- }
- max /= sum; sum *= s[i]; // if everything works as is expected, sum == 1.0
- if (state != null) state[i-1] = max_k;
- if (q != null) {
- k = (int)(-4.343 * Math.log(1. - max) + .499);
- q[i-1] = (byte)(k > 100? 99 : k);
- }
- //System.out.println("("+pb+","+sum+")"+" ("+(i-1)+","+(max_k>>2)+","+(max_k&3)+","+max+")");
- }
- return 0;
- }
-
- public static void main(String[] args) {
- byte[] ref = {'\0', '\1', '\3', '\3', '\1'};
- byte[] query = {'\0', '\3', '\3', '\1'};
- byte[] qual = new byte[4];
- qual[0] = qual[1] = qual[2] = qual[3] = (byte)20;
- HmmGlocal hg = new HmmGlocal(1e-3, 0.1, 7);
- hg.hmm_glocal(ref, query, qual, null, null);
- }
-}
\ No newline at end of file
diff --git a/samtools-0.1.19/misc/Makefile b/samtools-0.1.19/misc/Makefile
deleted file mode 100644
index d36e7ac..0000000
--- a/samtools-0.1.19/misc/Makefile
+++ /dev/null
@@ -1,69 +0,0 @@
-CC= gcc
-CXX= g++
-CFLAGS= -g -Wall -O2 #-m64 #-arch ppc
-CXXFLAGS= $(CFLAGS)
-DFLAGS= -D_FILE_OFFSET_BITS=64
-OBJS=
-PROG= md5sum-lite md5fa maq2sam-short maq2sam-long ace2sam wgsim bamcheck
-INCLUDES= -I..
-SUBDIRS= .
-
-.SUFFIXES:.c .o
-
-.c.o:
- $(CC) -c $(CFLAGS) $(DFLAGS) $(INCLUDES) $< -o $@
-
-all:$(PROG)
-
-lib-recur all-recur clean-recur cleanlocal-recur install-recur:
- @target=`echo $@ | sed s/-recur//`; \
- wdir=`pwd`; \
- list='$(SUBDIRS)'; for subdir in $$list; do \
- cd $$subdir; \
- $(MAKE) CC="$(CC)" DFLAGS="$(DFLAGS)" CFLAGS="$(CFLAGS)" \
- INCLUDES="$(INCLUDES)" $$target || exit 1; \
- cd $$wdir; \
- done;
-
-lib:
-
-bamcheck:bamcheck.o
- $(CC) $(CFLAGS) -o $@ bamcheck.o -L.. -lm -lbam -lpthread -lz
-
-bamcheck.o:bamcheck.c ../faidx.h ../khash.h ../sam.h ../razf.h
- $(CC) $(CFLAGS) -c -I.. -o $@ bamcheck.c
-
-ace2sam:ace2sam.o
- $(CC) $(CFLAGS) -o $@ ace2sam.o -lz
-
-wgsim:wgsim.o
- $(CC) $(CFLAGS) -o $@ wgsim.o -lm -lz
-
-md5fa:md5.o md5fa.o md5.h ../kseq.h
- $(CC) $(CFLAGS) -o $@ md5.o md5fa.o -lz
-
-md5sum-lite:md5sum-lite.o
- $(CC) $(CFLAGS) -o $@ md5sum-lite.o
-
-md5sum-lite.o:md5.c md5.h
- $(CC) -c $(CFLAGS) -DMD5SUM_MAIN -o $@ md5.c
-
-maq2sam-short:maq2sam.c
- $(CC) $(CFLAGS) -o $@ maq2sam.c -lz
-
-maq2sam-long:maq2sam.c
- $(CC) $(CFLAGS) -DMAQ_LONGREADS -o $@ maq2sam.c -lz
-
-md5fa.o:md5.h md5fa.c
- $(CC) $(CFLAGS) -c -I.. -o $@ md5fa.c
-
-wgsim.o:wgsim.c ../kseq.h
- $(CC) $(CFLAGS) -c -I.. -o $@ wgsim.c
-
-ace2sam.o:ace2sam.c ../kstring.h ../kseq.h
- $(CC) $(CFLAGS) -c -I.. -o $@ ace2sam.c
-
-cleanlocal:
- rm -fr gmon.out *.o a.out *.exe *.dSYM $(PROG) *~ *.a
-
-clean:cleanlocal-recur
diff --git a/samtools-0.1.19/misc/ace2sam.c b/samtools-0.1.19/misc/ace2sam.c
deleted file mode 100644
index 325133d..0000000
--- a/samtools-0.1.19/misc/ace2sam.c
+++ /dev/null
@@ -1,249 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2011 Heng Li <lh3 at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <zlib.h>
-#include "kstring.h"
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-#define N_TMPSTR 5
-#define LINE_LEN 60
-
-// append a CIGAR operation plus length
-#define write_cigar(_c, _n, _m, _v) do { \
- if (_n == _m) { \
- _m = _m? _m<<1 : 4; \
- _c = realloc(_c, _m * sizeof(unsigned)); \
- } \
- _c[_n++] = (_v); \
- } while (0)
-
-// a fatal error
-static void fatal(const char *msg)
-{
- fprintf(stderr, "E %s\n", msg);
- exit(1);
-}
-// remove pads
-static void remove_pads(const kstring_t *src, kstring_t *dst)
-{
- int i, j;
- dst->l = 0;
- kputsn(src->s, src->l, dst);
- for (i = j = 0; i < dst->l; ++i)
- if (dst->s[i] != '*') dst->s[j++] = dst->s[i];
- dst->s[j] = 0;
- dst->l = j;
-}
-
-int main(int argc, char *argv[])
-{
- gzFile fp;
- kstream_t *ks;
- kstring_t s, t[N_TMPSTR];
- int dret, i, k, af_n, af_max, af_i, c, is_padded = 0, write_cns = 0, *p2u = 0;
- long m_cigar = 0, n_cigar = 0;
- unsigned *af, *cigar = 0;
-
- while ((c = getopt(argc, argv, "pc")) >= 0) {
- switch (c) {
- case 'p': is_padded = 1; break;
- case 'c': write_cns = 1; break;
- }
- }
- if (argc == optind) {
- fprintf(stderr, "\nUsage: ace2sam [-pc] <in.ace>\n\n");
- fprintf(stderr, "Options: -p output padded SAM\n");
- fprintf(stderr, " -c write the contig sequence in SAM\n\n");
- fprintf(stderr, "Notes: 1. Fields must appear in the following order: (CO->[BQ]->(AF)->(RD->QA))\n");
- fprintf(stderr, " 2. The order of reads in AF and in RD must be identical\n");
- fprintf(stderr, " 3. Except in BQ, words and numbers must be separated by a single SPACE or TAB\n");
- fprintf(stderr, " 4. This program writes the headerless SAM to stdout and header to stderr\n\n");
- return 1;
- }
-
- s.l = s.m = 0; s.s = 0;
- af_n = af_max = af_i = 0; af = 0;
- for (i = 0; i < N_TMPSTR; ++i) t[i].l = t[i].m = 0, t[i].s = 0;
- fp = strcmp(argv[1], "-")? gzopen(argv[optind], "r") : gzdopen(fileno(stdin), "r");
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, &s, &dret) >= 0) {
- if (strcmp(s.s, "CO") == 0) { // contig sequence
- kstring_t *cns;
- t[0].l = t[1].l = t[2].l = t[3].l = t[4].l = 0; // 0: name; 1: padded ctg; 2: unpadded ctg/padded read; 3: unpadded read; 4: SAM line
- af_n = af_i = 0; // reset the af array
- ks_getuntil(ks, 0, &s, &dret); kputs(s.s, &t[0]); // contig name
- ks_getuntil(ks, '\n', &s, &dret); // read the whole line
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0 && s.l > 0) kputsn(s.s, s.l, &t[1]); // read the padded consensus sequence
- remove_pads(&t[1], &t[2]); // construct the unpadded sequence
- // compute the array for mapping padded positions to unpadded positions
- p2u = realloc(p2u, t[1].m * sizeof(int));
- for (i = k = 0; i < t[1].l; ++i) {
- p2u[i] = k;
- if (t[1].s[i] != '*') ++k;
- }
- // write out the SAM header and contig sequences
- fprintf(stderr, "H @SQ\tSN:%s\tLN:%ld\n", t[0].s, t[is_padded?1:2].l); // The SAM header line
- cns = &t[is_padded?1:2];
- fprintf(stderr, "S >%s\n", t[0].s);
- for (i = 0; i < cns->l; i += LINE_LEN) {
- fputs("S ", stderr);
- for (k = 0; k < LINE_LEN && i + k < cns->l; ++k)
- fputc(cns->s[i + k], stderr);
- fputc('\n', stderr);
- }
-
-#define __padded2cigar(sp) do { \
- int i, l_M = 0, l_D = 0; \
- for (i = 0; i < sp.l; ++i) { \
- if (sp.s[i] == '*') { \
- if (l_M) write_cigar(cigar, n_cigar, m_cigar, l_M<<4); \
- ++l_D; l_M = 0; \
- } else { \
- if (l_D) write_cigar(cigar, n_cigar, m_cigar, l_D<<4 | 2); \
- ++l_M; l_D = 0; \
- } \
- } \
- if (l_M) write_cigar(cigar, n_cigar, m_cigar, l_M<<4); \
- else write_cigar(cigar, n_cigar, m_cigar, l_D<<4 | 2); \
- } while (0)
-
- if (write_cns) { // write the consensus SAM line (dummy read)
- n_cigar = 0;
- if (is_padded) __padded2cigar(t[1]);
- else write_cigar(cigar, n_cigar, m_cigar, t[2].l<<4);
- kputsn(t[0].s, t[0].l, &t[4]); kputs("\t516\t", &t[4]); kputsn(t[0].s, t[0].l, &t[4]); kputs("\t1\t60\t", &t[4]);
- for (i = 0; i < n_cigar; ++i) {
- kputw(cigar[i]>>4, &t[4]); kputc("MIDNSHP=X"[cigar[i]&0xf], &t[4]);
- }
- kputs("\t*\t0\t0\t", &t[4]); kputsn(t[2].s, t[2].l, &t[4]); kputs("\t*", &t[4]);
- }
- } else if (strcmp(s.s, "BQ") == 0) { // contig quality
- if (t[0].l == 0) fatal("come to 'BQ' before reading 'CO'");
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret); // read the entire "BQ" line
- if (write_cns) t[4].s[--t[4].l] = 0; // remove the trailing "*"
- for (i = 0; i < t[2].l; ++i) { // read the consensus quality
- int q;
- if (ks_getuntil(ks, 0, &s, &dret) < 0) fprintf(stderr, "E truncated contig quality\n");
- if (s.l) {
- q = atoi(s.s) + 33;
- if (q > 126) q = 126;
- if (write_cns) kputc(q, &t[4]);
- } else --i;
- }
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- ks_getuntil(ks, '\n', &s, &dret); // skip the empty line
- if (write_cns) puts(t[4].s); t[4].l = 0;
- } else if (strcmp(s.s, "AF") == 0) { // padded read position
- int reversed, neg, pos;
- if (t[0].l == 0) fatal("come to 'AF' before reading 'CO'");
- if (write_cns) {
- if (t[4].l) puts(t[4].s);
- t[4].l = 0;
- }
- ks_getuntil(ks, 0, &s, &dret); // read name
- ks_getuntil(ks, 0, &s, &dret); reversed = s.s[0] == 'C'? 1 : 0; // strand
- ks_getuntil(ks, 0, &s, &dret); pos = atoi(s.s); neg = pos < 0? 1 : 0; pos = pos < 0? -pos : pos; // position
- if (af_n == af_max) { // double the af array
- af_max = af_max? af_max<<1 : 4;
- af = realloc(af, af_max * sizeof(unsigned));
- }
- af[af_n++] = pos << 2 | neg << 1 | reversed; // keep the placement information
- } else if (strcmp(s.s, "RD") == 0) { // read sequence
- if (af_i >= af_n) fatal("more 'RD' records than 'AF'");
- t[2].l = t[3].l = t[4].l = 0;
- ks_getuntil(ks, 0, &t[4], &dret); // QNAME
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret); // read the entire RD line
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0 && s.l > 0) kputs(s.s, &t[2]); // read the read sequence
- } else if (strcmp(s.s, "QA") == 0) { // clipping
- if (af_i >= af_n) fatal("more 'QA' records than 'AF'");
- int beg, end, pos, op;
- ks_getuntil(ks, 0, &s, &dret); ks_getuntil(ks, 0, &s, &dret); // skip quality clipping
- ks_getuntil(ks, 0, &s, &dret); beg = atoi(s.s) - 1; // align clipping start
- ks_getuntil(ks, 0, &s, &dret); end = atoi(s.s); // clipping end
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- // compute 1-based POS
- pos = af[af_i]>>2; // retrieve the position information
- if (af[af_i]>>1&1) pos = -pos;
- pos += beg; // now pos is the true padded position
- // generate CIGAR
- remove_pads(&t[2], &t[3]); // backup the unpadded read sequence
- n_cigar = 0;
- if (beg) write_cigar(cigar, n_cigar, m_cigar, beg<<4|4);
- if (is_padded) {
- __padded2cigar(t[2]);
- if (beg && n_cigar > 1) cigar[1] -= beg<<4; // fix the left-hand CIGAR
- if (end < t[2].l && n_cigar) cigar[n_cigar-1] -= (t[2].l - end)<<4; // fix the right-hand CIGAR
- } else {
- // generate flattened CIGAR string
- for (i = beg, k = pos - 1; i < end; ++i, ++k)
- t[2].s[i] = t[2].s[i] != '*'? (t[1].s[k] != '*'? 0 : 1) : (t[1].s[k] != '*'? 2 : 6);
- // generate the proper CIGAR
- for (i = beg + 1, k = 1, op = t[2].s[beg]; i < end; ++i) {
- if (op != t[2].s[i]) {
- write_cigar(cigar, n_cigar, m_cigar, k<<4|op);
- op = t[2].s[i]; k = 1;
- } else ++k;
- }
- write_cigar(cigar, n_cigar, m_cigar, k<<4|op);
- // remove unnecessary "P" and possibly merge adjacent operations
- for (i = 2; i < n_cigar; ++i) {
- if ((cigar[i]&0xf) != 1 && (cigar[i-1]&0xf) == 6 && (cigar[i-2]&0xf) != 1) {
- cigar[i-1] = 0;
- if ((cigar[i]&0xf) == (cigar[i-2]&0xf)) // merge operations
- cigar[i] += cigar[i-2], cigar[i-2] = 0;
- }
- }
- for (i = k = 0; i < n_cigar; ++i) // squeeze out dumb operations
- if (cigar[i]) cigar[k++] = cigar[i];
- n_cigar = k;
- }
- if (end < t[2].l) write_cigar(cigar, n_cigar, m_cigar, (t[2].l - end)<<4|4);
- // write the SAM line for the read
- kputc('\t', &t[4]); // QNAME has already been written
- kputw((af[af_i]&1)? 16 : 0, &t[4]); kputc('\t', &t[4]); // FLAG
- kputsn(t[0].s, t[0].l, &t[4]); kputc('\t', &t[4]); // RNAME
- kputw(is_padded? pos : p2u[pos-1]+1, &t[4]); // POS
- kputs("\t60\t", &t[4]); // MAPQ
- for (i = 0; i < n_cigar; ++i) { // CIGAR
- kputw(cigar[i]>>4, &t[4]); kputc("MIDNSHP=X"[cigar[i]&0xf], &t[4]);
- }
- kputs("\t*\t0\t0\t", &t[4]); // empty MRNM, MPOS and TLEN
- kputsn(t[3].s, t[3].l, &t[4]); // unpadded SEQ
- kputs("\t*", &t[4]); // QUAL
- puts(t[4].s); // print to stdout
- ++af_i;
- } else if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(af); free(s.s); free(cigar); free(p2u);
- for (i = 0; i < N_TMPSTR; ++i) free(t[i].s);
- return 0;
-}
diff --git a/samtools-0.1.19/misc/bamcheck.c b/samtools-0.1.19/misc/bamcheck.c
deleted file mode 100644
index 352db21..0000000
--- a/samtools-0.1.19/misc/bamcheck.c
+++ /dev/null
@@ -1,1521 +0,0 @@
-/*
- Author: petr.danecek at sanger
- gcc -Wall -Winline -g -O2 -I ~/git/samtools bamcheck.c -o bamcheck -lm -lz -L ~/git/samtools -lbam -lpthread
-
- Assumptions, approximations and other issues:
- - GC-depth graph does not split reads, the starting position determines which bin is incremented.
- There are small overlaps between bins (max readlen-1). However, the bins are big (20k).
- - coverage distribution ignores softclips and deletions
- - some stats require sorted BAMs
- - GC content graph can have an untidy, step-like pattern when BAM contains multiple read lengths.
- - 'bases mapped' (stats->nbases_mapped) is calculated from read lengths given by BAM (core.l_qseq)
- - With the -t option, the whole reads are used. Except for the number of mapped bases (cigar)
- counts, no splicing is done, no indels or soft clips are considered, even small overlap is
- good enough to include the read in the stats.
-
-*/
-
-#define BAMCHECK_VERSION "2012-09-04"
-
-#define _ISOC99_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <math.h>
-#include <ctype.h>
-#include <getopt.h>
-#include <errno.h>
-#include <assert.h>
-#include "faidx.h"
-#include "khash.h"
-#include "sam.h"
-#include "sam_header.h"
-#include "razf.h"
-
-#define BWA_MIN_RDLEN 35
-#define IS_PAIRED(bam) ((bam)->core.flag&BAM_FPAIRED && !((bam)->core.flag&BAM_FUNMAP) && !((bam)->core.flag&BAM_FMUNMAP))
-#define IS_UNMAPPED(bam) ((bam)->core.flag&BAM_FUNMAP)
-#define IS_REVERSE(bam) ((bam)->core.flag&BAM_FREVERSE)
-#define IS_MATE_REVERSE(bam) ((bam)->core.flag&BAM_FMREVERSE)
-#define IS_READ1(bam) ((bam)->core.flag&BAM_FREAD1)
-#define IS_READ2(bam) ((bam)->core.flag&BAM_FREAD2)
-#define IS_DUP(bam) ((bam)->core.flag&BAM_FDUP)
-
-typedef struct
-{
- int32_t line_len, line_blen;
- int64_t len;
- uint64_t offset;
-}
-faidx1_t;
-KHASH_MAP_INIT_STR(kh_faidx, faidx1_t)
-KHASH_MAP_INIT_STR(kh_bam_tid, int)
-KHASH_MAP_INIT_STR(kh_rg, const char *)
-struct __faidx_t {
- RAZF *rz;
- int n, m;
- char **name;
- khash_t(kh_faidx) *hash;
-};
-
-typedef struct
-{
- float gc;
- uint32_t depth;
-}
-gc_depth_t;
-
-// For coverage distribution, a simple pileup
-typedef struct
-{
- int64_t pos;
- int size, start;
- int *buffer;
-}
-round_buffer_t;
-
-typedef struct { uint32_t from, to; } pos_t;
-typedef struct
-{
- int npos,mpos,cpos;
- pos_t *pos;
-}
-regions_t;
-
-typedef struct
-{
- // Parameters
- int trim_qual; // bwa trim quality
-
- // Dimensions of the quality histogram holder (quals_1st,quals_2nd), GC content holder (gc_1st,gc_2nd),
- // insert size histogram holder
- int nquals; // The number of quality bins
- int nbases; // The maximum sequence length the allocated array can hold
- int nisize; // The maximum insert size that the allocated array can hold
- int ngc; // The size of gc_1st and gc_2nd
- int nindels; // The maximum indel length for indel distribution
-
- // Arrays for the histogram data
- uint64_t *quals_1st, *quals_2nd;
- uint64_t *gc_1st, *gc_2nd;
- uint64_t *isize_inward, *isize_outward, *isize_other;
- uint64_t *acgt_cycles;
- uint64_t *read_lengths;
- uint64_t *insertions, *deletions;
- uint64_t *ins_cycles_1st, *ins_cycles_2nd, *del_cycles_1st, *del_cycles_2nd;
-
- // The extremes encountered
- int max_len; // Maximum read length
- int max_qual; // Maximum quality
- float isize_main_bulk; // There are always some unrealistically big insert sizes, report only the main part
- int is_sorted;
-
- // Summary numbers
- uint64_t total_len;
- uint64_t total_len_dup;
- uint64_t nreads_1st;
- uint64_t nreads_2nd;
- uint64_t nreads_filtered;
- uint64_t nreads_dup;
- uint64_t nreads_unmapped;
- uint64_t nreads_unpaired;
- uint64_t nreads_paired;
- uint64_t nreads_anomalous;
- uint64_t nreads_mq0;
- uint64_t nbases_mapped;
- uint64_t nbases_mapped_cigar;
- uint64_t nbases_trimmed; // bwa trimmed bases
- uint64_t nmismatches;
- uint64_t nreads_QCfailed, nreads_secondary;
-
- // GC-depth related data
- uint32_t ngcd, igcd; // The maximum number of GC depth bins and index of the current bin
- gc_depth_t *gcd; // The GC-depth bins holder
- int gcd_bin_size; // The size of GC-depth bin
- uint32_t gcd_ref_size; // The approximate size of the genome
- int32_t tid, gcd_pos; // Position of the current bin
- int32_t pos; // Position of the last read
-
- // Coverage distribution related data
- int ncov; // The number of coverage bins
- uint64_t *cov; // The coverage frequencies
- int cov_min,cov_max,cov_step; // Minimum, maximum coverage and size of the coverage bins
- round_buffer_t cov_rbuf; // Pileup round buffer
-
- // Mismatches by read cycle
- uint8_t *rseq_buf; // A buffer for reference sequence to check the mismatches against
- int mrseq_buf; // The size of the buffer
- int32_t rseq_pos; // The coordinate of the first base in the buffer
- int32_t nrseq_buf; // The used part of the buffer
- uint64_t *mpc_buf; // Mismatches per cycle
-
- // Filters
- int filter_readlen;
-
- // Target regions
- int nregions, reg_from,reg_to;
- regions_t *regions;
-
- // Auxiliary data
- int flag_require, flag_filter;
- double sum_qual; // For calculating average quality value
- samfile_t *sam;
- khash_t(kh_rg) *rg_hash; // Read groups to include, the array is null-terminated
- faidx_t *fai; // Reference sequence for GC-depth graph
- int argc; // Command line arguments to be printed on the output
- char **argv;
-}
-stats_t;
-
-void error(const char *format, ...);
-void bam_init_header_hash(bam_header_t *header);
-int is_in_regions(bam1_t *bam_line, stats_t *stats);
-
-
-// Coverage distribution methods
-inline int coverage_idx(int min, int max, int n, int step, int depth)
-{
- if ( depth < min )
- return 0;
-
- if ( depth > max )
- return n-1;
-
- return 1 + (depth - min) / step;
-}
-
-inline int round_buffer_lidx2ridx(int offset, int size, int64_t refpos, int64_t pos)
-{
- return (offset + (pos-refpos) % size) % size;
-}
-
-void round_buffer_flush(stats_t *stats, int64_t pos)
-{
- int ibuf,idp;
-
- if ( pos==stats->cov_rbuf.pos )
- return;
-
- int64_t new_pos = pos;
- if ( pos==-1 || pos - stats->cov_rbuf.pos >= stats->cov_rbuf.size )
- {
- // Flush the whole buffer, but in sequential order,
- pos = stats->cov_rbuf.pos + stats->cov_rbuf.size - 1;
- }
-
- if ( pos < stats->cov_rbuf.pos )
- error("Expected coordinates in ascending order, got %ld after %ld\n", pos,stats->cov_rbuf.pos);
-
- int ifrom = stats->cov_rbuf.start;
- int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start,stats->cov_rbuf.size,stats->cov_rbuf.pos,pos-1);
- if ( ifrom>ito )
- {
- for (ibuf=ifrom; ibuf<stats->cov_rbuf.size; ibuf++)
- {
- if ( !stats->cov_rbuf.buffer[ibuf] )
- continue;
- idp = coverage_idx(stats->cov_min,stats->cov_max,stats->ncov,stats->cov_step,stats->cov_rbuf.buffer[ibuf]);
- stats->cov[idp]++;
- stats->cov_rbuf.buffer[ibuf] = 0;
- }
- ifrom = 0;
- }
- for (ibuf=ifrom; ibuf<=ito; ibuf++)
- {
- if ( !stats->cov_rbuf.buffer[ibuf] )
- continue;
- idp = coverage_idx(stats->cov_min,stats->cov_max,stats->ncov,stats->cov_step,stats->cov_rbuf.buffer[ibuf]);
- stats->cov[idp]++;
- stats->cov_rbuf.buffer[ibuf] = 0;
- }
- stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start,stats->cov_rbuf.size,stats->cov_rbuf.pos,pos);
- stats->cov_rbuf.pos = new_pos;
-}
-
-void round_buffer_insert_read(round_buffer_t *rbuf, int64_t from, int64_t to)
-{
- if ( to-from >= rbuf->size )
- error("The read length too big (%d), please increase the buffer length (currently %d)\n", to-from+1,rbuf->size);
- if ( from < rbuf->pos )
- error("The reads are not sorted (%ld comes after %ld).\n", from,rbuf->pos);
-
- int ifrom,ito,ibuf;
- ifrom = round_buffer_lidx2ridx(rbuf->start,rbuf->size,rbuf->pos,from);
- ito = round_buffer_lidx2ridx(rbuf->start,rbuf->size,rbuf->pos,to);
- if ( ifrom>ito )
- {
- for (ibuf=ifrom; ibuf<rbuf->size; ibuf++)
- rbuf->buffer[ibuf]++;
- ifrom = 0;
- }
- for (ibuf=ifrom; ibuf<=ito; ibuf++)
- rbuf->buffer[ibuf]++;
-}
-
-// Calculate the number of bases in the read trimmed by BWA
-int bwa_trim_read(int trim_qual, uint8_t *quals, int len, int reverse)
-{
- if ( len<BWA_MIN_RDLEN ) return 0;
-
- // Although the name implies that the read cannot be trimmed to more than BWA_MIN_RDLEN,
- // the calculation can in fact trim it to (BWA_MIN_RDLEN-1). (bwa_trim_read in bwa/bwaseqio.c).
- int max_trimmed = len - BWA_MIN_RDLEN + 1;
- int l, sum=0, max_sum=0, max_l=0;
-
- for (l=0; l<max_trimmed; l++)
- {
- sum += trim_qual - quals[ reverse ? l : len-1-l ];
- if ( sum<0 ) break;
- if ( sum>max_sum )
- {
- max_sum = sum;
- // This is the correct way, but bwa clips from some reason one base less
- // max_l = l+1;
- max_l = l;
- }
- }
- return max_l;
-}
-
-
-void count_indels(stats_t *stats,bam1_t *bam_line)
-{
- int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
- int is_1st = IS_READ1(bam_line) ? 1 : 0;
- int icig;
- int icycle = 0;
- int read_len = bam_line->core.l_qseq;
- for (icig=0; icig<bam_line->core.n_cigar; icig++)
- {
- // Conversion from uint32_t to MIDNSHP
- // 0123456
- // MIDNSHP
- int cig = bam1_cigar(bam_line)[icig] & BAM_CIGAR_MASK;
- int ncig = bam1_cigar(bam_line)[icig] >> BAM_CIGAR_SHIFT;
-
- if ( cig==1 )
- {
- int idx = is_fwd ? icycle : read_len-icycle-ncig;
- if ( idx<0 )
- error("FIXME: read_len=%d vs icycle=%d\n", read_len,icycle);
- if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%d %s\n", idx,stats->nbases, stats->sam->header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam1_qname(bam_line));
- if ( is_1st )
- stats->ins_cycles_1st[idx]++;
- else
- stats->ins_cycles_2nd[idx]++;
- icycle += ncig;
- if ( ncig<=stats->nindels )
- stats->insertions[ncig-1]++;
- continue;
- }
- if ( cig==2 )
- {
- int idx = is_fwd ? icycle-1 : read_len-icycle-1;
- if ( idx<0 ) continue; // discard meaningless deletions
- if ( idx >= stats->nbases ) error("FIXME: %d vs %d\n", idx,stats->nbases);
- if ( is_1st )
- stats->del_cycles_1st[idx]++;
- else
- stats->del_cycles_2nd[idx]++;
- if ( ncig<=stats->nindels )
- stats->deletions[ncig-1]++;
- continue;
- }
- if ( cig!=3 && cig!=5 )
- icycle += ncig;
- }
-}
-
-void count_mismatches_per_cycle(stats_t *stats,bam1_t *bam_line)
-{
- int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
- int icig,iread=0,icycle=0;
- int iref = bam_line->core.pos - stats->rseq_pos;
- int read_len = bam_line->core.l_qseq;
- uint8_t *read = bam1_seq(bam_line);
- uint8_t *quals = bam1_qual(bam_line);
- uint64_t *mpc_buf = stats->mpc_buf;
- for (icig=0; icig<bam_line->core.n_cigar; icig++)
- {
- // Conversion from uint32_t to MIDNSHP
- // 0123456
- // MIDNSHP
- int cig = bam1_cigar(bam_line)[icig] & BAM_CIGAR_MASK;
- int ncig = bam1_cigar(bam_line)[icig] >> BAM_CIGAR_SHIFT;
- if ( cig==1 )
- {
- iread += ncig;
- icycle += ncig;
- continue;
- }
- if ( cig==2 )
- {
- iref += ncig;
- continue;
- }
- if ( cig==4 )
- {
- icycle += ncig;
- // Soft-clips are present in the sequence, but the position of the read marks a start of non-clipped sequence
- // iref += ncig;
- iread += ncig;
- continue;
- }
- if ( cig==5 )
- {
- icycle += ncig;
- continue;
- }
- // Ignore H and N CIGARs. The letter are inserted e.g. by TopHat and often require very large
- // chunk of refseq in memory. Not very frequent and not noticable in the stats.
- if ( cig==3 || cig==5 ) continue;
- if ( cig!=0 )
- error("TODO: cigar %d, %s:%d %s\n", cig,stats->sam->header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam1_qname(bam_line));
-
- if ( ncig+iref > stats->nrseq_buf )
- error("FIXME: %d+%d > %d, %s, %s:%d\n",ncig,iref,stats->nrseq_buf, bam1_qname(bam_line),stats->sam->header->target_name[bam_line->core.tid],bam_line->core.pos+1);
-
- int im;
- for (im=0; im<ncig; im++)
- {
- uint8_t cread = bam1_seqi(read,iread);
- uint8_t cref = stats->rseq_buf[iref];
-
- // ---------------15
- // =ACMGRSVTWYHKDBN
- if ( cread==15 )
- {
- int idx = is_fwd ? icycle : read_len-icycle-1;
- if ( idx>stats->max_len )
- error("mpc: %d>%d\n",idx,stats->max_len);
- idx = idx*stats->nquals;
- if ( idx>=stats->nquals*stats->nbases )
- error("FIXME: mpc_buf overflow\n");
- mpc_buf[idx]++;
- }
- else if ( cref && cread && cref!=cread )
- {
- uint8_t qual = quals[iread] + 1;
- if ( qual>=stats->nquals )
- error("TODO: quality too high %d>=%d (%s %d %s)\n", qual,stats->nquals, stats->sam->header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam1_qname(bam_line));
-
- int idx = is_fwd ? icycle : read_len-icycle-1;
- if ( idx>stats->max_len )
- error("mpc: %d>%d\n",idx,stats->max_len);
-
- idx = idx*stats->nquals + qual;
- if ( idx>=stats->nquals*stats->nbases )
- error("FIXME: mpc_buf overflow\n");
- mpc_buf[idx]++;
- }
-
- iref++;
- iread++;
- icycle++;
- }
- }
-}
-
-void read_ref_seq(stats_t *stats,int32_t tid,int32_t pos)
-{
- khash_t(kh_faidx) *h;
- khiter_t iter;
- faidx1_t val;
- char *chr, c;
- faidx_t *fai = stats->fai;
-
- h = fai->hash;
- chr = stats->sam->header->target_name[tid];
-
- // ID of the sequence name
- iter = kh_get(kh_faidx, h, chr);
- if (iter == kh_end(h))
- error("No such reference sequence [%s]?\n", chr);
- val = kh_value(h, iter);
-
- // Check the boundaries
- if (pos >= val.len)
- error("Was the bam file mapped with the reference sequence supplied?"
- " A read mapped beyond the end of the chromosome (%s:%d, chromosome length %d).\n", chr,pos,val.len);
- int size = stats->mrseq_buf;
- // The buffer extends beyond the chromosome end. Later the rest will be filled with N's.
- if (size+pos > val.len) size = val.len-pos;
-
- // Position the razf reader
- razf_seek(fai->rz, val.offset + pos / val.line_blen * val.line_len + pos % val.line_blen, SEEK_SET);
-
- uint8_t *ptr = stats->rseq_buf;
- int nread = 0;
- while ( nread<size && razf_read(fai->rz,&c,1) && !fai->rz->z_err )
- {
- if ( !isgraph(c) )
- continue;
-
- // Conversion between uint8_t coding and ACGT
- // -12-4---8-------
- // =ACMGRSVTWYHKDBN
- if ( c=='A' || c=='a' )
- *ptr = 1;
- else if ( c=='C' || c=='c' )
- *ptr = 2;
- else if ( c=='G' || c=='g' )
- *ptr = 4;
- else if ( c=='T' || c=='t' )
- *ptr = 8;
- else
- *ptr = 0;
- ptr++;
- nread++;
- }
- if ( nread < stats->mrseq_buf )
- {
- memset(ptr,0, stats->mrseq_buf - nread);
- nread = stats->mrseq_buf;
- }
- stats->nrseq_buf = nread;
- stats->rseq_pos = pos;
- stats->tid = tid;
-}
-
-float fai_gc_content(stats_t *stats, int pos, int len)
-{
- uint32_t gc,count,c;
- int i = pos - stats->rseq_pos, ito = i + len;
- assert( i>=0 && ito<=stats->nrseq_buf );
-
- // Count GC content
- gc = count = 0;
- for (; i<ito; i++)
- {
- c = stats->rseq_buf[i];
- if ( c==2 || c==4 )
- {
- gc++;
- count++;
- }
- else if ( c==1 || c==8 )
- count++;
- }
- return count ? (float)gc/count : 0;
-}
-
-void realloc_rseq_buffer(stats_t *stats)
-{
- int n = stats->nbases*10;
- if ( stats->gcd_bin_size > n ) n = stats->gcd_bin_size;
- if ( stats->mrseq_buf<n )
- {
- stats->rseq_buf = realloc(stats->rseq_buf,sizeof(uint8_t)*n);
- stats->mrseq_buf = n;
- }
-}
-
-void realloc_gcd_buffer(stats_t *stats, int seq_len)
-{
- if ( seq_len >= stats->gcd_bin_size )
- error("The --GC-depth bin size (%d) is set too low for the read length %d\n", stats->gcd_bin_size, seq_len);
-
- int n = 1 + stats->gcd_ref_size / (stats->gcd_bin_size - seq_len);
- if ( n <= stats->igcd )
- error("The --GC-depth bin size is too small or reference genome too big; please decrease the bin size or increase the reference length\n");
-
- if ( n > stats->ngcd )
- {
- stats->gcd = realloc(stats->gcd, n*sizeof(gc_depth_t));
- if ( !stats->gcd )
- error("Could not realloc GCD buffer, too many chromosomes or the genome too long?? [%u %u]\n", stats->ngcd,n);
- memset(&(stats->gcd[stats->ngcd]),0,(n-stats->ngcd)*sizeof(gc_depth_t));
- stats->ngcd = n;
- }
-
- realloc_rseq_buffer(stats);
-}
-
-void realloc_buffers(stats_t *stats, int seq_len)
-{
- int n = 2*(1 + seq_len - stats->nbases) + stats->nbases;
-
- stats->quals_1st = realloc(stats->quals_1st, n*stats->nquals*sizeof(uint64_t));
- if ( !stats->quals_1st )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
- memset(stats->quals_1st + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
-
- stats->quals_2nd = realloc(stats->quals_2nd, n*stats->nquals*sizeof(uint64_t));
- if ( !stats->quals_2nd )
- error("Could not realloc buffers, the sequence too long: %d (2x%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
- memset(stats->quals_2nd + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
-
- if ( stats->mpc_buf )
- {
- stats->mpc_buf = realloc(stats->mpc_buf, n*stats->nquals*sizeof(uint64_t));
- if ( !stats->mpc_buf )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
- memset(stats->mpc_buf + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
- }
-
- stats->acgt_cycles = realloc(stats->acgt_cycles, n*4*sizeof(uint64_t));
- if ( !stats->acgt_cycles )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*4*sizeof(uint64_t));
- memset(stats->acgt_cycles + stats->nbases*4, 0, (n-stats->nbases)*4*sizeof(uint64_t));
-
- stats->read_lengths = realloc(stats->read_lengths, n*sizeof(uint64_t));
- if ( !stats->read_lengths )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
- memset(stats->read_lengths + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->insertions = realloc(stats->insertions, n*sizeof(uint64_t));
- if ( !stats->insertions )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
- memset(stats->insertions + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->deletions = realloc(stats->deletions, n*sizeof(uint64_t));
- if ( !stats->deletions )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
- memset(stats->deletions + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->ins_cycles_1st = realloc(stats->ins_cycles_1st, (n+1)*sizeof(uint64_t));
- if ( !stats->ins_cycles_1st )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
- memset(stats->ins_cycles_1st + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->ins_cycles_2nd = realloc(stats->ins_cycles_2nd, (n+1)*sizeof(uint64_t));
- if ( !stats->ins_cycles_2nd )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
- memset(stats->ins_cycles_2nd + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->del_cycles_1st = realloc(stats->del_cycles_1st, (n+1)*sizeof(uint64_t));
- if ( !stats->del_cycles_1st )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
- memset(stats->del_cycles_1st + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->del_cycles_2nd = realloc(stats->del_cycles_2nd, (n+1)*sizeof(uint64_t));
- if ( !stats->del_cycles_2nd )
- error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
- memset(stats->del_cycles_2nd + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
-
- stats->nbases = n;
-
- // Realloc the coverage distribution buffer
- int *rbuffer = calloc(sizeof(int),seq_len*5);
- n = stats->cov_rbuf.size-stats->cov_rbuf.start;
- memcpy(rbuffer,stats->cov_rbuf.buffer+stats->cov_rbuf.start,n);
- if ( stats->cov_rbuf.start>1 )
- memcpy(rbuffer+n,stats->cov_rbuf.buffer,stats->cov_rbuf.start);
- stats->cov_rbuf.start = 0;
- free(stats->cov_rbuf.buffer);
- stats->cov_rbuf.buffer = rbuffer;
- stats->cov_rbuf.size = seq_len*5;
-
- realloc_rseq_buffer(stats);
-}
-
-void collect_stats(bam1_t *bam_line, stats_t *stats)
-{
- if ( stats->rg_hash )
- {
- const uint8_t *rg = bam_aux_get(bam_line, "RG");
- if ( !rg ) return;
- khiter_t k = kh_get(kh_rg, stats->rg_hash, (const char*)(rg + 1));
- if ( k == kh_end(stats->rg_hash) ) return;
- }
- if ( stats->flag_require && (bam_line->core.flag & stats->flag_require)!=stats->flag_require )
- {
- stats->nreads_filtered++;
- return;
- }
- if ( stats->flag_filter && (bam_line->core.flag & stats->flag_filter) )
- {
- stats->nreads_filtered++;
- return;
- }
- if ( !is_in_regions(bam_line,stats) )
- return;
- if ( stats->filter_readlen!=-1 && bam_line->core.l_qseq!=stats->filter_readlen )
- return;
-
- if ( bam_line->core.flag & BAM_FQCFAIL ) stats->nreads_QCfailed++;
- if ( bam_line->core.flag & BAM_FSECONDARY ) stats->nreads_secondary++;
-
- int seq_len = bam_line->core.l_qseq;
- if ( !seq_len ) return;
-
- if ( seq_len >= stats->nbases )
- realloc_buffers(stats,seq_len);
- if ( stats->max_len<seq_len )
- stats->max_len = seq_len;
-
- stats->read_lengths[seq_len]++;
-
- // Count GC and ACGT per cycle
- uint8_t base, *seq = bam1_seq(bam_line);
- int gc_count = 0;
- int i;
- int reverse = IS_REVERSE(bam_line);
- for (i=0; i<seq_len; i++)
- {
- // Conversion from uint8_t coding to ACGT
- // -12-4---8-------
- // =ACMGRSVTWYHKDBN
- // 01 2 3
- base = bam1_seqi(seq,i);
- base /= 2;
- if ( base==1 || base==2 ) gc_count++;
- else if ( base>2 ) base=3;
- if ( 4*(reverse ? seq_len-i-1 : i) + base >= stats->nbases*4 )
- error("FIXME: acgt_cycles\n");
- stats->acgt_cycles[ 4*(reverse ? seq_len-i-1 : i) + base ]++;
- }
- int gc_idx_min = gc_count*(stats->ngc-1)/seq_len;
- int gc_idx_max = (gc_count+1)*(stats->ngc-1)/seq_len;
- if ( gc_idx_max >= stats->ngc ) gc_idx_max = stats->ngc - 1;
-
- // Determine which array (1st or 2nd read) will these stats go to,
- // trim low quality bases from end the same way BWA does,
- // fill GC histogram
- uint64_t *quals;
- uint8_t *bam_quals = bam1_qual(bam_line);
- if ( bam_line->core.flag&BAM_FREAD2 )
- {
- quals = stats->quals_2nd;
- stats->nreads_2nd++;
- for (i=gc_idx_min; i<gc_idx_max; i++)
- stats->gc_2nd[i]++;
- }
- else
- {
- quals = stats->quals_1st;
- stats->nreads_1st++;
- for (i=gc_idx_min; i<gc_idx_max; i++)
- stats->gc_1st[i]++;
- }
- if ( stats->trim_qual>0 )
- stats->nbases_trimmed += bwa_trim_read(stats->trim_qual, bam_quals, seq_len, reverse);
-
- // Quality histogram and average quality
- for (i=0; i<seq_len; i++)
- {
- uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
- if ( qual>=stats->nquals )
- error("TODO: quality too high %d>=%d (%s %d %s)\n", qual,stats->nquals,stats->sam->header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam1_qname(bam_line));
- if ( qual>stats->max_qual )
- stats->max_qual = qual;
-
- quals[ i*stats->nquals+qual ]++;
- stats->sum_qual += qual;
- }
-
- // Look at the flags and increment appropriate counters (mapped, paired, etc)
- if ( IS_UNMAPPED(bam_line) )
- stats->nreads_unmapped++;
- else
- {
- if ( !bam_line->core.qual )
- stats->nreads_mq0++;
-
- count_indels(stats,bam_line);
-
- if ( !IS_PAIRED(bam_line) )
- stats->nreads_unpaired++;
- else
- {
- stats->nreads_paired++;
-
- if ( bam_line->core.tid!=bam_line->core.mtid )
- stats->nreads_anomalous++;
-
- // The insert size is tricky, because for long inserts the libraries are
- // prepared differently and the pairs point in other direction. BWA does
- // not set the paired flag for them. Similar thing is true also for 454
- // reads. Mates mapped to different chromosomes have isize==0.
- int32_t isize = bam_line->core.isize;
- if ( isize<0 ) isize = -isize;
- if ( isize >= stats->nisize )
- isize = stats->nisize-1;
- if ( isize>0 || bam_line->core.tid==bam_line->core.mtid )
- {
- int pos_fst = bam_line->core.mpos - bam_line->core.pos;
- int is_fst = IS_READ1(bam_line) ? 1 : -1;
- int is_fwd = IS_REVERSE(bam_line) ? -1 : 1;
- int is_mfwd = IS_MATE_REVERSE(bam_line) ? -1 : 1;
-
- if ( is_fwd*is_mfwd>0 )
- stats->isize_other[isize]++;
- else if ( is_fst*pos_fst>0 )
- {
- if ( is_fst*is_fwd>0 )
- stats->isize_inward[isize]++;
- else
- stats->isize_outward[isize]++;
- }
- else if ( is_fst*pos_fst<0 )
- {
- if ( is_fst*is_fwd>0 )
- stats->isize_outward[isize]++;
- else
- stats->isize_inward[isize]++;
- }
- }
- }
-
- // Number of mismatches
- uint8_t *nm = bam_aux_get(bam_line,"NM");
- if (nm)
- stats->nmismatches += bam_aux2i(nm);
-
- // Number of mapped bases from cigar
- // Conversion from uint32_t to MIDNSHP
- // 012-4--
- // MIDNSHP
- if ( bam_line->core.n_cigar == 0)
- error("FIXME: mapped read with no cigar?\n");
- int readlen=seq_len;
- if ( stats->regions )
- {
- // Count only on-target bases
- int iref = bam_line->core.pos + 1;
- for (i=0; i<bam_line->core.n_cigar; i++)
- {
- int cig = bam1_cigar(bam_line)[i]&BAM_CIGAR_MASK;
- int ncig = bam1_cigar(bam_line)[i]>>BAM_CIGAR_SHIFT;
- if ( cig==2 ) readlen += ncig;
- else if ( cig==0 )
- {
- if ( iref < stats->reg_from ) ncig -= stats->reg_from-iref;
- else if ( iref+ncig-1 > stats->reg_to ) ncig -= iref+ncig-1 - stats->reg_to;
- if ( ncig<0 ) ncig = 0;
- stats->nbases_mapped_cigar += ncig;
- iref += bam1_cigar(bam_line)[i]>>BAM_CIGAR_SHIFT;
- }
- else if ( cig==1 )
- {
- iref += ncig;
- if ( iref>=stats->reg_from && iref<=stats->reg_to )
- stats->nbases_mapped_cigar += ncig;
- }
- }
- }
- else
- {
- // Count the whole read
- for (i=0; i<bam_line->core.n_cigar; i++)
- {
- if ( (bam1_cigar(bam_line)[i]&BAM_CIGAR_MASK)==0 || (bam1_cigar(bam_line)[i]&BAM_CIGAR_MASK)==1 )
- stats->nbases_mapped_cigar += bam1_cigar(bam_line)[i]>>BAM_CIGAR_SHIFT;
- if ( (bam1_cigar(bam_line)[i]&BAM_CIGAR_MASK)==2 )
- readlen += bam1_cigar(bam_line)[i]>>BAM_CIGAR_SHIFT;
- }
- }
- stats->nbases_mapped += seq_len;
-
- if ( stats->tid==bam_line->core.tid && bam_line->core.pos<stats->pos )
- stats->is_sorted = 0;
- stats->pos = bam_line->core.pos;
-
- if ( stats->is_sorted )
- {
- if ( stats->tid==-1 || stats->tid!=bam_line->core.tid )
- round_buffer_flush(stats,-1);
-
- // Mismatches per cycle and GC-depth graph. For simplicity, reads overlapping GCD bins
- // are not splitted which results in up to seq_len-1 overlaps. The default bin size is
- // 20kbp, so the effect is negligible.
- if ( stats->fai )
- {
- int inc_ref = 0, inc_gcd = 0;
- // First pass or new chromosome
- if ( stats->rseq_pos==-1 || stats->tid != bam_line->core.tid ) { inc_ref=1; inc_gcd=1; }
- // Read goes beyond the end of the rseq buffer
- else if ( stats->rseq_pos+stats->nrseq_buf < bam_line->core.pos+readlen ) { inc_ref=1; inc_gcd=1; }
- // Read overlaps the next gcd bin
- else if ( stats->gcd_pos+stats->gcd_bin_size < bam_line->core.pos+readlen )
- {
- inc_gcd = 1;
- if ( stats->rseq_pos+stats->nrseq_buf < bam_line->core.pos+stats->gcd_bin_size ) inc_ref = 1;
- }
- if ( inc_gcd )
- {
- stats->igcd++;
- if ( stats->igcd >= stats->ngcd )
- realloc_gcd_buffer(stats, readlen);
- if ( inc_ref )
- read_ref_seq(stats,bam_line->core.tid,bam_line->core.pos);
- stats->gcd_pos = bam_line->core.pos;
- stats->gcd[ stats->igcd ].gc = fai_gc_content(stats, stats->gcd_pos, stats->gcd_bin_size);
- }
-
- count_mismatches_per_cycle(stats,bam_line);
- }
- // No reference and first pass, new chromosome or sequence going beyond the end of the gcd bin
- else if ( stats->gcd_pos==-1 || stats->tid != bam_line->core.tid || bam_line->core.pos - stats->gcd_pos > stats->gcd_bin_size )
- {
- // First pass or a new chromosome
- stats->tid = bam_line->core.tid;
- stats->gcd_pos = bam_line->core.pos;
- stats->igcd++;
- if ( stats->igcd >= stats->ngcd )
- realloc_gcd_buffer(stats, readlen);
- }
- stats->gcd[ stats->igcd ].depth++;
- // When no reference sequence is given, approximate the GC from the read (much shorter window, but otherwise OK)
- if ( !stats->fai )
- stats->gcd[ stats->igcd ].gc += (float) gc_count / seq_len;
-
- // Coverage distribution graph
- round_buffer_flush(stats,bam_line->core.pos);
- round_buffer_insert_read(&(stats->cov_rbuf),bam_line->core.pos,bam_line->core.pos+seq_len-1);
- }
- }
-
- stats->total_len += seq_len;
- if ( IS_DUP(bam_line) )
- {
- stats->total_len_dup += seq_len;
- stats->nreads_dup++;
- }
-}
-
-// Sort by GC and depth
-#define GCD_t(x) ((gc_depth_t *)x)
-static int gcd_cmp(const void *a, const void *b)
-{
- if ( GCD_t(a)->gc < GCD_t(b)->gc ) return -1;
- if ( GCD_t(a)->gc > GCD_t(b)->gc ) return 1;
- if ( GCD_t(a)->depth < GCD_t(b)->depth ) return -1;
- if ( GCD_t(a)->depth > GCD_t(b)->depth ) return 1;
- return 0;
-}
-#undef GCD_t
-
-float gcd_percentile(gc_depth_t *gcd, int N, int p)
-{
- float n,d;
- int k;
-
- n = p*(N+1)/100;
- k = n;
- if ( k<=0 )
- return gcd[0].depth;
- if ( k>=N )
- return gcd[N-1].depth;
-
- d = n - k;
- return gcd[k-1].depth + d*(gcd[k].depth - gcd[k-1].depth);
-}
-
-void output_stats(stats_t *stats)
-{
- // Calculate average insert size and standard deviation (from the main bulk data only)
- int isize, ibulk=0;
- uint64_t nisize=0, nisize_inward=0, nisize_outward=0, nisize_other=0;
- for (isize=0; isize<stats->nisize; isize++)
- {
- // Each pair was counted twice
- stats->isize_inward[isize] *= 0.5;
- stats->isize_outward[isize] *= 0.5;
- stats->isize_other[isize] *= 0.5;
-
- nisize_inward += stats->isize_inward[isize];
- nisize_outward += stats->isize_outward[isize];
- nisize_other += stats->isize_other[isize];
- nisize += stats->isize_inward[isize] + stats->isize_outward[isize] + stats->isize_other[isize];
- }
-
- double bulk=0, avg_isize=0, sd_isize=0;
- for (isize=0; isize<stats->nisize; isize++)
- {
- bulk += stats->isize_inward[isize] + stats->isize_outward[isize] + stats->isize_other[isize];
- avg_isize += isize * (stats->isize_inward[isize] + stats->isize_outward[isize] + stats->isize_other[isize]);
-
- if ( bulk/nisize > stats->isize_main_bulk )
- {
- ibulk = isize+1;
- nisize = bulk;
- break;
- }
- }
- avg_isize /= nisize ? nisize : 1;
- for (isize=1; isize<ibulk; isize++)
- sd_isize += (stats->isize_inward[isize] + stats->isize_outward[isize] + stats->isize_other[isize]) * (isize-avg_isize)*(isize-avg_isize) / nisize;
- sd_isize = sqrt(sd_isize);
-
-
- printf("# This file was produced by bamcheck (%s)\n",BAMCHECK_VERSION);
- printf("# The command line was: %s",stats->argv[0]);
- int i;
- for (i=1; i<stats->argc; i++)
- printf(" %s",stats->argv[i]);
- printf("\n");
- printf("# Summary Numbers. Use `grep ^SN | cut -f 2-` to extract this part.\n");
- printf("SN\traw total sequences:\t%ld\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd));
- printf("SN\tfiltered sequences:\t%ld\n", (long)stats->nreads_filtered);
- printf("SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd));
- printf("SN\tis paired:\t%d\n", stats->nreads_1st&&stats->nreads_2nd ? 1 : 0);
- printf("SN\tis sorted:\t%d\n", stats->is_sorted ? 1 : 0);
- printf("SN\t1st fragments:\t%ld\n", (long)stats->nreads_1st);
- printf("SN\tlast fragments:\t%ld\n", (long)stats->nreads_2nd);
- printf("SN\treads mapped:\t%ld\n", (long)(stats->nreads_paired+stats->nreads_unpaired));
- printf("SN\treads unmapped:\t%ld\n", (long)stats->nreads_unmapped);
- printf("SN\treads unpaired:\t%ld\n", (long)stats->nreads_unpaired);
- printf("SN\treads paired:\t%ld\n", (long)stats->nreads_paired);
- printf("SN\treads duplicated:\t%ld\n", (long)stats->nreads_dup);
- printf("SN\treads MQ0:\t%ld\n", (long)stats->nreads_mq0);
- printf("SN\treads QC failed:\t%ld\n", (long)stats->nreads_QCfailed);
- printf("SN\tnon-primary alignments:\t%ld\n", (long)stats->nreads_secondary);
- printf("SN\ttotal length:\t%ld\n", (long)stats->total_len);
- printf("SN\tbases mapped:\t%ld\n", (long)stats->nbases_mapped);
- printf("SN\tbases mapped (cigar):\t%ld\n", (long)stats->nbases_mapped_cigar);
- printf("SN\tbases trimmed:\t%ld\n", (long)stats->nbases_trimmed);
- printf("SN\tbases duplicated:\t%ld\n", (long)stats->total_len_dup);
- printf("SN\tmismatches:\t%ld\n", (long)stats->nmismatches);
- printf("SN\terror rate:\t%e\n", (float)stats->nmismatches/stats->nbases_mapped_cigar);
- float avg_read_length = (stats->nreads_1st+stats->nreads_2nd)?stats->total_len/(stats->nreads_1st+stats->nreads_2nd):0;
- printf("SN\taverage length:\t%.0f\n", avg_read_length);
- printf("SN\tmaximum length:\t%d\n", stats->max_len);
- printf("SN\taverage quality:\t%.1f\n", stats->total_len?stats->sum_qual/stats->total_len:0);
- printf("SN\tinsert size average:\t%.1f\n", avg_isize);
- printf("SN\tinsert size standard deviation:\t%.1f\n", sd_isize);
- printf("SN\tinward oriented pairs:\t%ld\n", (long)nisize_inward);
- printf("SN\toutward oriented pairs:\t%ld\n", (long)nisize_outward);
- printf("SN\tpairs with other orientation:\t%ld\n", (long)nisize_other);
- printf("SN\tpairs on different chromosomes:\t%ld\n", (long)stats->nreads_anomalous/2);
-
- int ibase,iqual;
- if ( stats->max_len<stats->nbases ) stats->max_len++;
- if ( stats->max_qual+1<stats->nquals ) stats->max_qual++;
- printf("# First Fragment Qualitites. Use `grep ^FFQ | cut -f 2-` to extract this part.\n");
- printf("# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n");
- for (ibase=0; ibase<stats->max_len; ibase++)
- {
- printf("FFQ\t%d",ibase+1);
- for (iqual=0; iqual<=stats->max_qual; iqual++)
- {
- printf("\t%ld", (long)stats->quals_1st[ibase*stats->nquals+iqual]);
- }
- printf("\n");
- }
- printf("# Last Fragment Qualitites. Use `grep ^LFQ | cut -f 2-` to extract this part.\n");
- printf("# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n");
- for (ibase=0; ibase<stats->max_len; ibase++)
- {
- printf("LFQ\t%d",ibase+1);
- for (iqual=0; iqual<=stats->max_qual; iqual++)
- {
- printf("\t%ld", (long)stats->quals_2nd[ibase*stats->nquals+iqual]);
- }
- printf("\n");
- }
- if ( stats->mpc_buf )
- {
- printf("# Mismatches per cycle and quality. Use `grep ^MPC | cut -f 2-` to extract this part.\n");
- printf("# Columns correspond to qualities, rows to cycles. First column is the cycle number, second\n");
- printf("# is the number of N's and the rest is the number of mismatches\n");
- for (ibase=0; ibase<stats->max_len; ibase++)
- {
- printf("MPC\t%d",ibase+1);
- for (iqual=0; iqual<=stats->max_qual; iqual++)
- {
- printf("\t%ld", (long)stats->mpc_buf[ibase*stats->nquals+iqual]);
- }
- printf("\n");
- }
- }
- printf("# GC Content of first fragments. Use `grep ^GCF | cut -f 2-` to extract this part.\n");
- int ibase_prev = 0;
- for (ibase=0; ibase<stats->ngc; ibase++)
- {
- if ( stats->gc_1st[ibase]==stats->gc_1st[ibase_prev] ) continue;
- printf("GCF\t%.2f\t%ld\n", (ibase+ibase_prev)*0.5*100./(stats->ngc-1), (long)stats->gc_1st[ibase_prev]);
- ibase_prev = ibase;
- }
- printf("# GC Content of last fragments. Use `grep ^GCL | cut -f 2-` to extract this part.\n");
- ibase_prev = 0;
- for (ibase=0; ibase<stats->ngc; ibase++)
- {
- if ( stats->gc_2nd[ibase]==stats->gc_2nd[ibase_prev] ) continue;
- printf("GCL\t%.2f\t%ld\n", (ibase+ibase_prev)*0.5*100./(stats->ngc-1), (long)stats->gc_2nd[ibase_prev]);
- ibase_prev = ibase;
- }
- printf("# ACGT content per cycle. Use `grep ^GCC | cut -f 2-` to extract this part. The columns are: cycle, and A,C,G,T counts [%%]\n");
- for (ibase=0; ibase<stats->max_len; ibase++)
- {
- uint64_t *ptr = &(stats->acgt_cycles[ibase*4]);
- uint64_t sum = ptr[0]+ptr[1]+ptr[2]+ptr[3];
- if ( ! sum ) continue;
- printf("GCC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase,100.*ptr[0]/sum,100.*ptr[1]/sum,100.*ptr[2]/sum,100.*ptr[3]/sum);
- }
- printf("# Insert sizes. Use `grep ^IS | cut -f 2-` to extract this part. The columns are: pairs total, inward oriented pairs, outward oriented pairs, other pairs\n");
- for (isize=0; isize<ibulk; isize++)
- printf("IS\t%d\t%ld\t%ld\t%ld\t%ld\n", isize, (long)(stats->isize_inward[isize]+stats->isize_outward[isize]+stats->isize_other[isize]),
- (long)stats->isize_inward[isize], (long)stats->isize_outward[isize], (long)stats->isize_other[isize]);
-
- printf("# Read lengths. Use `grep ^RL | cut -f 2-` to extract this part. The columns are: read length, count\n");
- int ilen;
- for (ilen=0; ilen<stats->max_len; ilen++)
- {
- if ( stats->read_lengths[ilen]>0 )
- printf("RL\t%d\t%ld\n", ilen, (long)stats->read_lengths[ilen]);
- }
-
- printf("# Indel distribution. Use `grep ^ID | cut -f 2-` to extract this part. The columns are: length, number of insertions, number of deletions\n");
- for (ilen=0; ilen<stats->nindels; ilen++)
- {
- if ( stats->insertions[ilen]>0 || stats->deletions[ilen]>0 )
- printf("ID\t%d\t%ld\t%ld\n", ilen+1, (long)stats->insertions[ilen], (long)stats->deletions[ilen]);
- }
-
- printf("# Indels per cycle. Use `grep ^IC | cut -f 2-` to extract this part. The columns are: cycle, number of insertions (fwd), .. (rev) , number of deletions (fwd), .. (rev)\n");
- for (ilen=0; ilen<=stats->nbases; ilen++)
- {
- // For deletions we print the index of the cycle before the deleted base (1-based) and for insertions
- // the index of the cycle of the first inserted base (also 1-based)
- if ( stats->ins_cycles_1st[ilen]>0 || stats->ins_cycles_2nd[ilen]>0 || stats->del_cycles_1st[ilen]>0 || stats->del_cycles_2nd[ilen]>0 )
- printf("IC\t%d\t%ld\t%ld\t%ld\t%ld\n", ilen+1, (long)stats->ins_cycles_1st[ilen], (long)stats->ins_cycles_2nd[ilen], (long)stats->del_cycles_1st[ilen], (long)stats->del_cycles_2nd[ilen]);
- }
-
- printf("# Coverage distribution. Use `grep ^COV | cut -f 2-` to extract this part.\n");
- if ( stats->cov[0] )
- printf("COV\t[<%d]\t%d\t%ld\n",stats->cov_min,stats->cov_min-1, (long)stats->cov[0]);
- int icov;
- for (icov=1; icov<stats->ncov-1; icov++)
- if ( stats->cov[icov] )
- printf("COV\t[%d-%d]\t%d\t%ld\n",stats->cov_min + (icov-1)*stats->cov_step, stats->cov_min + icov*stats->cov_step-1,stats->cov_min + icov*stats->cov_step-1, (long)stats->cov[icov]);
- if ( stats->cov[stats->ncov-1] )
- printf("COV\t[%d<]\t%d\t%ld\n",stats->cov_min + (stats->ncov-2)*stats->cov_step-1,stats->cov_min + (stats->ncov-2)*stats->cov_step-1, (long)stats->cov[stats->ncov-1]);
-
- // Calculate average GC content, then sort by GC and depth
- printf("# GC-depth. Use `grep ^GCD | cut -f 2-` to extract this part. The columns are: GC%%, unique sequence percentiles, 10th, 25th, 50th, 75th and 90th depth percentile\n");
- uint32_t igcd;
- for (igcd=0; igcd<stats->igcd; igcd++)
- {
- if ( stats->fai )
- stats->gcd[igcd].gc = round(100. * stats->gcd[igcd].gc);
- else
- if ( stats->gcd[igcd].depth )
- stats->gcd[igcd].gc = round(100. * stats->gcd[igcd].gc / stats->gcd[igcd].depth);
- }
- qsort(stats->gcd, stats->igcd+1, sizeof(gc_depth_t), gcd_cmp);
- igcd = 0;
- while ( igcd < stats->igcd )
- {
- // Calculate percentiles (10,25,50,75,90th) for the current GC content and print
- uint32_t nbins=0, itmp=igcd;
- float gc = stats->gcd[igcd].gc;
- while ( itmp<stats->igcd && fabs(stats->gcd[itmp].gc-gc)<0.1 )
- {
- nbins++;
- itmp++;
- }
- printf("GCD\t%.1f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n", gc, (igcd+nbins+1)*100./(stats->igcd+1),
- gcd_percentile(&(stats->gcd[igcd]),nbins,10) *avg_read_length/stats->gcd_bin_size,
- gcd_percentile(&(stats->gcd[igcd]),nbins,25) *avg_read_length/stats->gcd_bin_size,
- gcd_percentile(&(stats->gcd[igcd]),nbins,50) *avg_read_length/stats->gcd_bin_size,
- gcd_percentile(&(stats->gcd[igcd]),nbins,75) *avg_read_length/stats->gcd_bin_size,
- gcd_percentile(&(stats->gcd[igcd]),nbins,90) *avg_read_length/stats->gcd_bin_size
- );
- igcd += nbins;
- }
-}
-
-size_t mygetline(char **line, size_t *n, FILE *fp)
-{
- if (line == NULL || n == NULL || fp == NULL)
- {
- errno = EINVAL;
- return -1;
- }
- if (*n==0 || !*line)
- {
- *line = NULL;
- *n = 0;
- }
-
- size_t nread=0;
- int c;
- while ((c=getc(fp))!= EOF && c!='\n')
- {
- if ( ++nread>=*n )
- {
- *n += 255;
- *line = realloc(*line, sizeof(char)*(*n));
- }
- (*line)[nread-1] = c;
- }
- if ( nread>=*n )
- {
- *n += 255;
- *line = realloc(*line, sizeof(char)*(*n));
- }
- (*line)[nread] = 0;
- return nread>0 ? nread : -1;
-
-}
-
-void init_regions(stats_t *stats, char *file)
-{
- khiter_t iter;
- khash_t(kh_bam_tid) *header_hash;
-
- bam_init_header_hash(stats->sam->header);
- header_hash = (khash_t(kh_bam_tid)*)stats->sam->header->hash;
-
- FILE *fp = fopen(file,"r");
- if ( !fp ) error("%s: %s\n",file,strerror(errno));
-
- char *line = NULL;
- size_t len = 0;
- ssize_t nread;
- int warned = 0;
- int prev_tid=-1, prev_pos=-1;
- while ((nread = mygetline(&line, &len, fp)) != -1)
- {
- if ( line[0] == '#' ) continue;
-
- int i = 0;
- while ( i<nread && !isspace(line[i]) ) i++;
- if ( i>=nread ) error("Could not parse the file: %s [%s]\n", file,line);
- line[i] = 0;
-
- iter = kh_get(kh_bam_tid, header_hash, line);
- int tid = kh_val(header_hash, iter);
- if ( iter == kh_end(header_hash) )
- {
- if ( !warned )
- fprintf(stderr,"Warning: Some sequences not present in the BAM, e.g. \"%s\". This message is printed only once.\n", line);
- warned = 1;
- continue;
- }
-
- if ( tid >= stats->nregions )
- {
- stats->regions = realloc(stats->regions,sizeof(regions_t)*(stats->nregions+100));
- int j;
- for (j=stats->nregions; j<stats->nregions+100; j++)
- {
- stats->regions[j].npos = stats->regions[j].mpos = stats->regions[j].cpos = 0;
- stats->regions[j].pos = NULL;
- }
- stats->nregions += 100;
- }
- int npos = stats->regions[tid].npos;
- if ( npos >= stats->regions[tid].mpos )
- {
- stats->regions[tid].mpos += 1000;
- stats->regions[tid].pos = realloc(stats->regions[tid].pos,sizeof(pos_t)*stats->regions[tid].mpos);
- }
-
- if ( (sscanf(line+i+1,"%d %d",&stats->regions[tid].pos[npos].from,&stats->regions[tid].pos[npos].to))!=2 ) error("Could not parse the region [%s]\n");
- if ( prev_tid==-1 || prev_tid!=tid )
- {
- prev_tid = tid;
- prev_pos = stats->regions[tid].pos[npos].from;
- }
- if ( prev_pos>stats->regions[tid].pos[npos].from )
- error("The positions are not in chromosomal order (%s:%d comes after %d)\n", line,stats->regions[tid].pos[npos].from,prev_pos);
- stats->regions[tid].npos++;
- }
- if (line) free(line);
- if ( !stats->regions ) error("Unable to map the -t sequences to the BAM sequences.\n");
- fclose(fp);
-}
-
-void destroy_regions(stats_t *stats)
-{
- int i;
- for (i=0; i<stats->nregions; i++)
- {
- if ( !stats->regions[i].mpos ) continue;
- free(stats->regions[i].pos);
- }
- if ( stats->regions ) free(stats->regions);
-}
-
-static int fetch_read(const bam1_t *bam_line, void *data)
-{
- collect_stats((bam1_t*)bam_line,(stats_t*)data);
- return 1;
-}
-
-void reset_regions(stats_t *stats)
-{
- int i;
- for (i=0; i<stats->nregions; i++)
- stats->regions[i].cpos = 0;
-}
-
-int is_in_regions(bam1_t *bam_line, stats_t *stats)
-{
- if ( !stats->regions ) return 1;
-
- if ( bam_line->core.tid >= stats->nregions || bam_line->core.tid<0 ) return 0;
- if ( !stats->is_sorted ) error("The BAM must be sorted in order for -t to work.\n");
-
- regions_t *reg = &stats->regions[bam_line->core.tid];
- if ( reg->cpos==reg->npos ) return 0; // done for this chr
-
- // Find a matching interval or skip this read. No splicing of reads is done, no indels or soft clips considered,
- // even small overlap is enough to include the read in the stats.
- int i = reg->cpos;
- while ( i<reg->npos && reg->pos[i].to<=bam_line->core.pos ) i++;
- if ( i>=reg->npos ) { reg->cpos = reg->npos; return 0; }
- if ( bam_line->core.pos + bam_line->core.l_qseq + 1 < reg->pos[i].from ) return 0;
- reg->cpos = i;
- stats->reg_from = reg->pos[i].from;
- stats->reg_to = reg->pos[i].to;
-
- return 1;
-}
-
-void init_group_id(stats_t *stats, char *id)
-{
- if ( !stats->sam->header->dict )
- stats->sam->header->dict = sam_header_parse2(stats->sam->header->text);
- void *iter = stats->sam->header->dict;
- const char *key, *val;
- int n = 0;
- stats->rg_hash = kh_init(kh_rg);
- while ( (iter = sam_header2key_val(iter, "RG","ID","SM", &key, &val)) )
- {
- if ( !strcmp(id,key) || (val && !strcmp(id,val)) )
- {
- khiter_t k = kh_get(kh_rg, stats->rg_hash, key);
- if ( k != kh_end(stats->rg_hash) )
- fprintf(stderr, "[init_group_id] The group ID not unique: \"%s\"\n", key);
- int ret;
- k = kh_put(kh_rg, stats->rg_hash, key, &ret);
- kh_value(stats->rg_hash, k) = val;
- n++;
- }
- }
- if ( !n )
- error("The sample or read group \"%s\" not present.\n", id);
-}
-
-
-void error(const char *format, ...)
-{
- if ( !format )
- {
- printf("Version: %s\n", BAMCHECK_VERSION);
- printf("About: The program collects statistics from BAM files. The output can be visualized using plot-bamcheck.\n");
- printf("Usage: bamcheck [OPTIONS] file.bam\n");
- printf(" bamcheck [OPTIONS] file.bam chr:from-to\n");
- printf("Options:\n");
- printf(" -c, --coverage <int>,<int>,<int> Coverage distribution min,max,step [1,1000,1]\n");
- printf(" -d, --remove-dups Exlude from statistics reads marked as duplicates\n");
- printf(" -f, --required-flag <int> Required flag, 0 for unset [0]\n");
- printf(" -F, --filtering-flag <int> Filtering flag, 0 for unset [0]\n");
- printf(" --GC-depth <float,float> Bin size for GC-depth graph and the maximum reference length [2e4,4.2e9]\n");
- printf(" -h, --help This help message\n");
- printf(" -i, --insert-size <int> Maximum insert size [8000]\n");
- printf(" -I, --id <string> Include only listed read group or sample name\n");
- printf(" -l, --read-length <int> Include in the statistics only reads with the given read length []\n");
- printf(" -m, --most-inserts <float> Report only the main part of inserts [0.99]\n");
- printf(" -q, --trim-quality <int> The BWA trimming parameter [0]\n");
- printf(" -r, --ref-seq <file> Reference sequence (required for GC-depth calculation).\n");
- printf(" -t, --target-regions <file> Do stats in these regions only. Tab-delimited file chr,from,to, 1-based, inclusive.\n");
- printf(" -s, --sam Input is SAM\n");
- printf("\n");
- }
- else
- {
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
- }
- exit(-1);
-}
-
-int main(int argc, char *argv[])
-{
- char *targets = NULL;
- char *bam_fname = NULL;
- char *group_id = NULL;
- samfile_t *sam = NULL;
- char in_mode[5];
-
- stats_t *stats = calloc(1,sizeof(stats_t));
- stats->ngc = 200;
- stats->nquals = 256;
- stats->nbases = 300;
- stats->nisize = 8000;
- stats->max_len = 30;
- stats->max_qual = 40;
- stats->isize_main_bulk = 0.99; // There are always outliers at the far end
- stats->gcd_bin_size = 20e3;
- stats->gcd_ref_size = 4.2e9;
- stats->rseq_pos = -1;
- stats->tid = stats->gcd_pos = -1;
- stats->igcd = 0;
- stats->is_sorted = 1;
- stats->cov_min = 1;
- stats->cov_max = 1000;
- stats->cov_step = 1;
- stats->argc = argc;
- stats->argv = argv;
- stats->filter_readlen = -1;
- stats->nindels = stats->nbases;
-
- strcpy(in_mode, "rb");
-
- static struct option loptions[] =
- {
- {"help",0,0,'h'},
- {"remove-dups",0,0,'d'},
- {"sam",0,0,'s'},
- {"ref-seq",1,0,'r'},
- {"coverage",1,0,'c'},
- {"read-length",1,0,'l'},
- {"insert-size",1,0,'i'},
- {"most-inserts",1,0,'m'},
- {"trim-quality",1,0,'q'},
- {"target-regions",0,0,'t'},
- {"required-flag",1,0,'f'},
- {"filtering-flag",0,0,'F'},
- {"id",1,0,'I'},
- {"GC-depth",1,0,1},
- {0,0,0,0}
- };
- int opt;
- while ( (opt=getopt_long(argc,argv,"?hdsr:c:l:i:t:m:q:f:F:I:1:",loptions,NULL))>0 )
- {
- switch (opt)
- {
- case 'f': stats->flag_require=strtol(optarg,0,0); break;
- case 'F': stats->flag_filter=strtol(optarg,0,0); break;
- case 'd': stats->flag_filter|=BAM_FDUP; break;
- case 's': strcpy(in_mode, "r"); break;
- case 'r': stats->fai = fai_load(optarg);
- if (stats->fai==0)
- error("Could not load faidx: %s\n", optarg);
- break;
- case 1 : {
- float flen,fbin;
- if ( sscanf(optarg,"%f,%f",&fbin,&flen)!= 2 )
- error("Unable to parse --GC-depth %s\n", optarg);
- stats->gcd_bin_size = fbin;
- stats->gcd_ref_size = flen;
- }
- break;
- case 'c': if ( sscanf(optarg,"%d,%d,%d",&stats->cov_min,&stats->cov_max,&stats->cov_step)!= 3 )
- error("Unable to parse -c %s\n", optarg);
- break;
- case 'l': stats->filter_readlen = atoi(optarg); break;
- case 'i': stats->nisize = atoi(optarg); break;
- case 'm': stats->isize_main_bulk = atof(optarg); break;
- case 'q': stats->trim_qual = atoi(optarg); break;
- case 't': targets = optarg; break;
- case 'I': group_id = optarg; break;
- case '?':
- case 'h': error(NULL);
- default: error("Unknown argument: %s\n", optarg);
- }
- }
- if ( optind<argc )
- bam_fname = argv[optind++];
-
- if ( !bam_fname )
- {
- if ( isatty(fileno((FILE *)stdin)) )
- error(NULL);
- bam_fname = "-";
- }
-
- // Init structures
- // .. coverage bins and round buffer
- if ( stats->cov_step > stats->cov_max - stats->cov_min + 1 )
- {
- stats->cov_step = stats->cov_max - stats->cov_min;
- if ( stats->cov_step <= 0 )
- stats->cov_step = 1;
- }
- stats->ncov = 3 + (stats->cov_max-stats->cov_min) / stats->cov_step;
- stats->cov_max = stats->cov_min + ((stats->cov_max-stats->cov_min)/stats->cov_step +1)*stats->cov_step - 1;
- stats->cov = calloc(sizeof(uint64_t),stats->ncov);
- stats->cov_rbuf.size = stats->nbases*5;
- stats->cov_rbuf.buffer = calloc(sizeof(int32_t),stats->cov_rbuf.size);
- // .. bam
- if ((sam = samopen(bam_fname, in_mode, NULL)) == 0)
- error("Failed to open: %s\n", bam_fname);
- stats->sam = sam;
- if ( group_id ) init_group_id(stats, group_id);
- bam1_t *bam_line = bam_init1();
- // .. arrays
- stats->quals_1st = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
- stats->quals_2nd = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
- stats->gc_1st = calloc(stats->ngc,sizeof(uint64_t));
- stats->gc_2nd = calloc(stats->ngc,sizeof(uint64_t));
- stats->isize_inward = calloc(stats->nisize,sizeof(uint64_t));
- stats->isize_outward = calloc(stats->nisize,sizeof(uint64_t));
- stats->isize_other = calloc(stats->nisize,sizeof(uint64_t));
- stats->gcd = calloc(stats->ngcd,sizeof(gc_depth_t));
- stats->mpc_buf = stats->fai ? calloc(stats->nquals*stats->nbases,sizeof(uint64_t)) : NULL;
- stats->acgt_cycles = calloc(4*stats->nbases,sizeof(uint64_t));
- stats->read_lengths = calloc(stats->nbases,sizeof(uint64_t));
- stats->insertions = calloc(stats->nbases,sizeof(uint64_t));
- stats->deletions = calloc(stats->nbases,sizeof(uint64_t));
- stats->ins_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
- stats->ins_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
- stats->del_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
- stats->del_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
- realloc_rseq_buffer(stats);
- if ( targets )
- init_regions(stats, targets);
-
- // Collect statistics
- if ( optind<argc )
- {
- // Collect stats in selected regions only
- bam_index_t *bam_idx = bam_index_load(bam_fname);
- if (bam_idx == 0)
- error("Random alignment retrieval only works for indexed BAM files.\n");
-
- int i;
- for (i=optind; i<argc; i++)
- {
- int tid, beg, end;
- bam_parse_region(stats->sam->header, argv[i], &tid, &beg, &end);
- if ( tid < 0 ) continue;
- reset_regions(stats);
- bam_fetch(stats->sam->x.bam, bam_idx, tid, beg, end, stats, fetch_read);
- }
- bam_index_destroy(bam_idx);
- }
- else
- {
- // Stream through the entire BAM ignoring off-target regions if -t is given
- while (samread(sam,bam_line) >= 0)
- collect_stats(bam_line,stats);
- }
- round_buffer_flush(stats,-1);
-
- output_stats(stats);
-
- bam_destroy1(bam_line);
- samclose(stats->sam);
- if (stats->fai) fai_destroy(stats->fai);
- free(stats->cov_rbuf.buffer); free(stats->cov);
- free(stats->quals_1st); free(stats->quals_2nd);
- free(stats->gc_1st); free(stats->gc_2nd);
- free(stats->isize_inward); free(stats->isize_outward); free(stats->isize_other);
- free(stats->gcd);
- free(stats->rseq_buf);
- free(stats->mpc_buf);
- free(stats->acgt_cycles);
- free(stats->read_lengths);
- free(stats->insertions);
- free(stats->deletions);
- free(stats->ins_cycles_1st);
- free(stats->ins_cycles_2nd);
- free(stats->del_cycles_1st);
- free(stats->del_cycles_2nd);
- destroy_regions(stats);
- free(stats);
- if ( stats->rg_hash ) kh_destroy(kh_rg, stats->rg_hash);
-
- return 0;
-}
-
-
-
diff --git a/samtools-0.1.19/misc/blast2sam.pl b/samtools-0.1.19/misc/blast2sam.pl
deleted file mode 100755
index 084f018..0000000
--- a/samtools-0.1.19/misc/blast2sam.pl
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/perl -w
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-&blast2sam;
-
-sub blast2sam {
- my %opts = ();
- getopts('s', \%opts);
- die("Usage: blast2sam.pl <in.blastn>\n") if (-t STDIN && @ARGV == 0);
- my ($qlen, $slen, $q, $s, $qbeg, $qend, @sam, @cigar, @cmaux, $show_seq);
- $show_seq = defined($opts{s});
- @sam = (); @sam[0,4,6..8,10] = ('', 255, '*', 0, 0, '*');
- while (<>) {
- if (@cigar && (/^Query=/ || /Score =.*bits.*Expect/)) { # print
- &blast_print_sam(\@sam, \@cigar, \@cmaux, $qlen - $qend);
- @cigar = ();
- }
- if (/^Query= (\S+)/) {
- $sam[0] = $1;
- } elsif (/\((\S+)\s+letters\)/) {
- $qlen = $1; $qlen =~ s/,//g;
- } elsif (/^>(\S+)/) {
- $sam[2] = $1;
- } elsif (/Length = (\d+)/) {
- $slen = $1;
- } elsif (/Score =\s+(\S+) bits.+Expect(\(\d+\))? = (\S+)/) { # the start of an alignment block
- my ($as, $ev) = (int($1 + .499), $3);
- $ev = "1$ev" if ($ev =~ /^e/);
- @sam[1,3,9,11,12] = (0, 0, '', "AS:i:$as", "EV:Z:$ev");
- @cigar = (); $qbeg = 0;
- @cmaux = (0, 0, 0, '');
- } elsif (/Strand = (\S+) \/ (\S+)/) {
- $sam[1] |= 0x10 if ($2 eq 'Minus');
- } elsif (/Query\:\s(\d+)\s*(\S+)\s(\d+)/) {
- $q = $2;
- unless ($qbeg) {
- $qbeg = $1;
- push(@cigar, ($1-1) . "H") if ($1 > 1);
- }
- $qend = $3;
- if ($show_seq) {
- my $x = $q;
- $x =~ s/-//g; $sam[9] .= $x;
- }
- } elsif (/Sbjct\:\s(\d+)\s*(\S+)\s(\d+)/) {
- $s = $2;
- if ($sam[1] & 0x10) {
- $sam[3] = $3;
- } else {
- $sam[3] = $1 unless ($sam[3]);
- }
- &aln2cm(\@cigar, \$q, \$s, \@cmaux);
- }
- }
- &blast_print_sam(\@sam, \@cigar, \@cmaux, $qlen - $qend);
-}
-
-sub blast_print_sam {
- my ($sam, $cigar, $cmaux, $qrest) = @_;
- push(@$cigar, $cmaux->[1] . substr("MDI", $cmaux->[0], 1));
- push(@$cigar, $qrest . 'H') if ($qrest);
- if ($sam->[1] & 0x10) {
- @$cigar = reverse(@$cigar);
- $sam->[9] = reverse($sam->[9]);
- $sam->[9] =~ tr/atgcrymkswATGCRYMKSW/tacgyrkmswTACGYRKMSW/;
- }
- $sam->[9] = '*' if (!$sam->[9]);
- $sam->[5] = join('', @$cigar);
- print join("\t", @$sam), "\n";
-}
-
-sub aln2cm {
- my ($cigar, $q, $s, $cmaux) = @_;
- my $l = length($$q);
- for (my $i = 0; $i < $l; ++$i) {
- my $op;
- # set $op
- if (substr($$q, $i, 1) eq '-') { $op = 2; }
- elsif (substr($$s, $i, 1) eq '-') { $op = 1; }
- else { $op = 0; }
- # for CIGAR
- if ($cmaux->[0] == $op) {
- ++$cmaux->[1];
- } else {
- push(@$cigar, $cmaux->[1] . substr("MDI", $cmaux->[0], 1));
- $cmaux->[0] = $op; $cmaux->[1] = 1;
- }
- }
-}
diff --git a/samtools-0.1.19/misc/bowtie2sam.pl b/samtools-0.1.19/misc/bowtie2sam.pl
deleted file mode 100755
index 5dff88d..0000000
--- a/samtools-0.1.19/misc/bowtie2sam.pl
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/perl -w
-
-# Contact: lh3
-# Version: 0.1.1
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-&bowtie2sam;
-exit;
-
-sub bowtie2sam {
- my %opts = ();
- die("Usage: bowtie2sam.pl <aln.bowtie>\n") if (@ARGV == 0 && -t STDIN);
- # core loop
- my (@s, $last, @staging, $k, $best_s, $subbest_s, $best_k);
- $last = '';
- while (<>) {
- my ($name, $nm) = &bowtie2sam_aux($_, \@s); # read_name, number of mismatches
- if ($name eq $last) {
- # I do not know whether the multiple hits are ordered on the
- # number of mismatches. I assume they are not and so I have to
- # keep all these multiple hits in memory.
- @{$staging[$k]} = @s;
- if ($best_s > $nm) {
- $subbest_s = $best_s;
- $best_s = $nm;
- $best_k = $k;
- } elsif ($subbest_s > $nm) {
- $subbest_s = $nm;
- }
- ++$k;
- } else {
- if ($last) {
- if ($best_s == $subbest_s) {
- $staging[$best_k][4] = 0;
- } elsif ($subbest_s - $best_s == 1) {
- $staging[$best_k][4] = 15 if ($staging[$best_k][4] > 15);
- }
- print join("\t", @{$staging[$best_k]}), "\n";
- }
- $k = 1; $best_s = $nm; $subbest_s = 1000; $best_k = 0;
- @{$staging[0]} = @s;
- $last = $name;
- }
- }
- print join("\t", @{$staging[$best_k]}), "\n" if ($best_k >= 0);
-}
-
-sub bowtie2sam_aux {
- my ($line, $s) = @_;
- chomp($line);
- my @t = split("\t", $line);
- my $ret;
- @$s = ();
- # read name
- $s->[0] = $ret = $t[0];
- $s->[0] =~ s/\/[12]$//g;
- # initial flag (will be updated later)
- $s->[1] = 0;
- # read & quality
- $s->[9] = $t[4]; $s->[10] = $t[5];
- # cigar
- $s->[5] = length($s->[9]) . "M";
- # coor
- $s->[2] = $t[2]; $s->[3] = $t[3] + 1;
- $s->[1] |= 0x10 if ($t[1] eq '-');
- # mapQ
- $s->[4] = $t[6] == 0? 25 : 0;
- # mate coordinate
- $s->[6] = '*'; $s->[7] = $s->[8] = 0;
- # aux
- my $nm = @t - 7;
- push(@$s, "NM:i:" . (@t-7));
- push(@$s, "X$nm:i:" . ($t[6]+1));
- my $md = '';
- if ($t[7]) {
- $_ = $t[7];
- my $a = 0;
- while (/(\d+):[ACGTN]>([ACGTN])/gi) {
- my ($y, $z) = ($1, $2);
- $md .= (int($y)-$a) . $z;
- $a += $y - $a + 1;
- }
- $md .= length($s->[9]) - $a;
- } else {
- $md = length($s->[9]);
- }
- push(@$s, "MD:Z:$md");
- return ($ret, $nm);
-}
diff --git a/samtools-0.1.19/misc/export2sam.pl b/samtools-0.1.19/misc/export2sam.pl
deleted file mode 100755
index ec6dacf..0000000
--- a/samtools-0.1.19/misc/export2sam.pl
+++ /dev/null
@@ -1,545 +0,0 @@
-#!/usr/bin/env perl
-#
-#
-# export2sam.pl converts GERALD export files to SAM format.
-#
-#
-#
-########## License:
-#
-# The MIT License
-#
-# Original SAMtools work copyright (c) 2008-2009 Genome Research Ltd.
-# Modified SAMtools work copyright (c) 2010 Illumina, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#
-#
-#
-########## ChangeLog:
-#
-# Version: 2.3.1 (18MAR2011)
-#
-# - Restore file '-' as stdin input.
-#
-# Version: 2.3.0 (24JAN2011)
-#
-# - Add support for export reserved chromosome name "CONTROL",
-# which is translated to optional field "XC:Z:CONTROL".
-# - Check for ".gz" file extension on export files and open
-# these as gzip pipes when the extension is found.
-#
-# Version: 2.2.0 (16NOV2010)
-#
-# - Remove any leading zeros in export fields: RUNNO,LANE,TILE,X,Y
-# - For export records with reserved chromosome name identifiers
-# "QC" and "RM", add the optional field "XC:Z:QC" or "XC:Z:RM"
-# to the SAM record, so that these cases can be distinguished
-# from other unmatched reads.
-#
-# Version: 2.1.0 (21SEP2010)
-#
-# - Additional export record error checking.
-# - Convert export records with chromomsome value of "RM" to unmapped
-# SAM records.
-#
-# Version: 2.0.0 (15FEB2010)
-#
-# Script updated by Illumina in conjunction with CASAVA 1.7.0
-# release.
-#
-# Major changes are as follows:
-# - The CIGAR string has been updated to include all gaps from
-# ELANDv2 alignments.
-# - The ELAND single read alignment score is always stored in the
-# optional "SM" field and the ELAND paired read alignment score
-# is stored in the optional "AS" field when it exists.
-# - The MAPQ value is set to the higher of the two alignment scores,
-# but no greater than 254, i.e. min(254,max(SM,AS))
-# - The SAM "proper pair" bit (0x0002) is now set for read pairs
-# meeting ELAND's expected orientation and insert size criteria.
-# - The default quality score translation is set for export files
-# which contain Phread+64 quality values. An option,
-# "--qlogodds", has been added to translate quality values from
-# the Solexa+64 format used in export files prior to Pipeline
-# 1.3
-# - The export match descriptor is now reverse-complemented when
-# necessary such that it always corresponds to the forward
-# strand of the reference, to be consistent with other
-# information in the SAM record. It is now written to the
-# optional 'XD' field (rather than 'MD') to acknowledge its
-# minor differences from the samtools match descriptor (see
-# additional detail below).
-# - An option, "--nofilter", has been added to include reads which
-# have failed primary analysis quality filtration. Such reads
-# will have the corresponding SAM flag bit (0x0200) set.
-# - Labels in the export 'contig' field are preserved by setting
-# RNAME to "$export_chromosome/$export_contig" when the contig
-# label exists.
-#
-#
-# Contact: lh3
-# Version: 0.1.2 (03JAN2009)
-#
-#
-#
-########## Known Conversion Limitations:
-#
-# - Export records for reads that map to a position < 1 (allowed
-# in export format), are converted to unmapped reads in the SAM
-# record.
-# - Export records contain the reserved chromosome names: "NM",
-# "QC","RM" and "CONTROL". "NM" indicates that the aligner could
-# not map the read to the reference sequence set. "QC" means that
-# the aligner did not attempt to map the read due to some
-# technical limitation. "RM" means that the read mapped to a set
-# of 'contaminant' sequences specified in GERALD's RNA-seq
-# workflow. "CONTROL" means that the read is a control. All of
-# these alignment types are collapsed to the single unmapped
-# alignment state in the SAM record, but the optional SAM "XC"
-# field is used to record the original reserved chromosome name of
-# the read for all but the "NM" case.
-# - The export match descriptor is slightly different than the
-# samtools match descriptor. For this reason it is stored in the
-# optional SAM field 'XD' (and not 'MD'). Note that the export
-# match descriptor differs from the samtools version in two
-# respects: (1) indels are explicitly closed with the '$'
-# character and (2) insertions must be enumerated in the match
-# descriptor. For example a 35-base read with a two-base insertion
-# is described as: 20^2$14
-#
-#
-#
-
-my $version = "2.3.1";
-
-use strict;
-use warnings;
-
-use Getopt::Long;
-use File::Spec;
-use List::Util qw(min max);
-
-
-use constant {
- EXPORT_MACHINE => 0,
- EXPORT_RUNNO => 1,
- EXPORT_LANE => 2,
- EXPORT_TILE => 3,
- EXPORT_X => 4,
- EXPORT_Y => 5,
- EXPORT_INDEX => 6,
- EXPORT_READNO => 7,
- EXPORT_READ => 8,
- EXPORT_QUAL => 9,
- EXPORT_CHROM => 10,
- EXPORT_CONTIG => 11,
- EXPORT_POS => 12,
- EXPORT_STRAND => 13,
- EXPORT_MD => 14,
- EXPORT_SEMAP => 15,
- EXPORT_PEMAP => 16,
- EXPORT_PASSFILT => 21,
- EXPORT_SIZE => 22,
-};
-
-
-use constant {
- SAM_QNAME => 0,
- SAM_FLAG => 1,
- SAM_RNAME => 2,
- SAM_POS => 3,
- SAM_MAPQ => 4,
- SAM_CIGAR => 5,
- SAM_MRNM => 6,
- SAM_MPOS => 7,
- SAM_ISIZE => 8,
- SAM_SEQ => 9,
- SAM_QUAL => 10,
-};
-
-
-# function prototypes for Richard's code
-sub match_desc_to_cigar($);
-sub match_desc_frag_length($);
-sub reverse_compl_match_descriptor($);
-sub write_header($;$;$);
-
-
-&export2sam;
-exit;
-
-
-
-
-sub export2sam {
-
- my $cmdline = $0 . " " . join(" ", at ARGV);
- my $arg_count = scalar @ARGV;
- my $progname = (File::Spec->splitpath($0))[2];
-
- my $is_logodds_qvals = 0; # if true, assume files contain logodds (i.e. "solexa") quality values
- my $is_nofilter = 0;
- my $read1file;
- my $read2file;
- my $print_version = 0;
- my $help = 0;
-
- my $result = GetOptions( "qlogodds" => \$is_logodds_qvals,
- "nofilter" => \$is_nofilter,
- "read1=s" => \$read1file,
- "read2=s" => \$read2file,
- "version" => \$print_version,
- "help" => \$help );
-
- my $usage = <<END;
-
-$progname converts GERALD export files to SAM format.
-
-Usage: $progname --read1=FILENAME [ options ] | --version | --help
-
- --read1=FILENAME read1 export file or '-' for stdin (mandatory)
- (file may be gzipped with ".gz" extension)
- --read2=FILENAME read2 export file or '-' for stdin
- (file may be gzipped with ".gz" extension)
- --nofilter include reads that failed the basecaller
- purity filter
- --qlogodds assume export file(s) use logodds quality values
- as reported by OLB (Pipeline) prior to v1.3
- (default: phred quality values)
-
-END
-
- my $version_msg = <<END;
-
-$progname version: $version
-
-END
-
- if((not $result) or $help or ($arg_count==0)) {
- die($usage);
- }
-
- if(@ARGV) {
- print STDERR "\nERROR: Unrecognized arguments: " . join(" ", at ARGV) . "\n\n";
- die($usage);
- }
-
- if($print_version) {
- die($version_msg);
- }
-
- if(not defined($read1file)) {
- print STDERR "\nERROR: read1 export file must be specified\n\n";
- die($usage);
- }
-
- unless((-f $read1file) or ($read1file eq '-')) {
- die("\nERROR: Can't find read1 export file: '$read1file'\n\n");
- }
-
- if (defined $read2file) {
- unless((-f $read2file) or ($read2file eq '-')) {
- die("\nERROR: Can't find read2 export file: '$read2file'\n\n");
- }
- if($read1file eq $read2file) {
- die("\nERROR: read1 and read2 export filenames are the same: '$read1file'\n\n");
- }
- }
-
- my ($fh1, $fh2, $is_paired);
-
- my $read1cmd="$read1file";
- $read1cmd = "gzip -dc $read1file |" if($read1file =~ /\.gz$/);
- open($fh1, $read1cmd)
- or die("\nERROR: Can't open read1 process: '$read1cmd'\n\n");
- $is_paired = defined $read2file;
- if ($is_paired) {
- my $read2cmd="$read2file";
- $read2cmd = "gzip -dc $read2file |" if($read2file =~ /\.gz$/);
- open($fh2, $read2cmd)
- or die("\nERROR: Can't open read2 process: '$read2cmd'\n\n");
- }
- # quality value conversion table
- my @conv_table;
- if($is_logodds_qvals){ # convert from solexa+64 quality values (pipeline pre-v1.3):
- for (-64..64) {
- $conv_table[$_+64] = int(33 + 10*log(1+10**($_/10.0))/log(10)+.499);
- }
- } else { # convert from phred+64 quality values (pipeline v1.3+):
- for (-64..-1) {
- $conv_table[$_+64] = undef;
- }
- for (0..64) {
- $conv_table[$_+64] = int(33 + $_);
- }
- }
- # write the header
- print write_header( $progname, $version, $cmdline );
- # core loop
- my $export_line_count = 0;
- while (<$fh1>) {
- $export_line_count++;
- my (@s1, @s2);
- &export2sam_aux($_, $export_line_count, \@s1, \@conv_table, $is_paired, 1, $is_nofilter);
- if ($is_paired) {
- my $read2line = <$fh2>;
- if(not $read2line){
- die("\nERROR: read1 and read2 export files do not contain the same number of reads.\n Extra reads observed in read1 file at line no: $export_line_count.\n\n");
- }
- &export2sam_aux($read2line, $export_line_count, \@s2, \@conv_table, $is_paired, 2, $is_nofilter);
-
- if (@s1 && @s2) { # then set mate coordinate
- if($s1[SAM_QNAME] ne $s2[SAM_QNAME]){
- die("\nERROR: Non-paired reads in export files on line: $export_line_count.\n Read1: $_ Read2: $read2line\n");
- }
-
- my $isize = 0;
- if ($s1[SAM_RNAME] ne '*' && $s1[SAM_RNAME] eq $s2[SAM_RNAME]) { # then calculate $isize
- my $x1 = ($s1[SAM_FLAG] & 0x10)? $s1[SAM_POS] + length($s1[SAM_SEQ]) : $s1[SAM_POS];
- my $x2 = ($s2[SAM_FLAG] & 0x10)? $s2[SAM_POS] + length($s2[SAM_SEQ]) : $s2[SAM_POS];
- $isize = $x2 - $x1;
- }
-
- foreach ([\@s1,\@s2,$isize],[\@s2,\@s1,-$isize]){
- my ($sa,$sb,$is) = @{$_};
- if ($sb->[SAM_RNAME] ne '*') {
- $sa->[SAM_MRNM] = ($sb->[SAM_RNAME] eq $sa->[SAM_RNAME]) ? "=" : $sb->[SAM_RNAME];
- $sa->[SAM_MPOS] = $sb->[SAM_POS];
- $sa->[SAM_ISIZE] = $is;
- $sa->[SAM_FLAG] |= 0x20 if ($sb->[SAM_FLAG] & 0x10);
- } else {
- $sa->[SAM_FLAG] |= 0x8;
- }
- }
- }
- }
- print join("\t", @s1), "\n" if (@s1);
- print join("\t", @s2), "\n" if (@s2 && $is_paired);
- }
- close($fh1);
- if($is_paired) {
- while(my $read2line = <$fh2>){
- $export_line_count++;
- die("\nERROR: read1 and read2 export files do not contain the same number of reads.\n Extra reads observed in read2 file at line no: $export_line_count.\n\n");
- }
- close($fh2);
- }
-}
-
-sub export2sam_aux {
- my ($line, $line_no, $s, $ct, $is_paired, $read_no, $is_nofilter) = @_;
- chomp($line);
- my @t = split("\t", $line);
- if(scalar(@t) < EXPORT_SIZE) {
- my $msg="\nERROR: Unexpected number of fields in export record on line $line_no of read$read_no export file. Found " . scalar(@t) . " fields but expected " . EXPORT_SIZE . ".\n";
- $msg.="\t...erroneous export record:\n" . $line . "\n\n";
- die($msg);
- }
- @$s = ();
- my $isPassFilt = ($t[EXPORT_PASSFILT] eq 'Y');
- return if(not ($isPassFilt or $is_nofilter));
- # read name
- my $samQnamePrefix = $t[EXPORT_MACHINE] . (($t[EXPORT_RUNNO] ne "") ? "_" . int($t[EXPORT_RUNNO]) : "");
- $s->[SAM_QNAME] = join(':', $samQnamePrefix, int($t[EXPORT_LANE]), int($t[EXPORT_TILE]),
- int($t[EXPORT_X]), int($t[EXPORT_Y]));
- # initial flag (will be updated later)
- $s->[SAM_FLAG] = 0;
- if($is_paired) {
- if($t[EXPORT_READNO] != $read_no){
- die("\nERROR: read$read_no export file contains record with read number: " .$t[EXPORT_READNO] . " on line: $line_no\n\n");
- }
- $s->[SAM_FLAG] |= 1 | 1<<(5 + $read_no);
- }
- $s->[SAM_FLAG] |= 0x200 if (not $isPassFilt);
-
- # read & quality
- my $is_export_rev = ($t[EXPORT_STRAND] eq 'R');
- if ($is_export_rev) { # then reverse the sequence and quality
- $s->[SAM_SEQ] = reverse($t[EXPORT_READ]);
- $s->[SAM_SEQ] =~ tr/ACGTacgt/TGCAtgca/;
- $s->[SAM_QUAL] = reverse($t[EXPORT_QUAL]);
- } else {
- $s->[SAM_SEQ] = $t[EXPORT_READ];
- $s->[SAM_QUAL] = $t[EXPORT_QUAL];
- }
- my @convqual = ();
- foreach (unpack('C*', $s->[SAM_QUAL])){
- my $val=$ct->[$_];
- if(not defined $val){
- my $msg="\nERROR: can't interpret export quality value: " . $_ . " in read$read_no export file, line: $line_no\n";
- if( $_ < 64 ) { $msg .= " Use --qlogodds flag to translate logodds (solexa) quality values.\n"; }
- die($msg . "\n");
- }
- push @convqual,$val;
- }
-
- $s->[SAM_QUAL] = pack('C*', at convqual); # change coding
-
-
- # coor
- my $has_coor = 0;
- $s->[SAM_RNAME] = "*";
- if (($t[EXPORT_CHROM] eq 'NM') or
- ($t[EXPORT_CHROM] eq 'QC') or
- ($t[EXPORT_CHROM] eq 'RM') or
- ($t[EXPORT_CHROM] eq 'CONTROL')) {
- $s->[SAM_FLAG] |= 0x4; # unmapped
- push(@$s,"XC:Z:".$t[EXPORT_CHROM]) if($t[EXPORT_CHROM] ne 'NM');
- } elsif ($t[EXPORT_CHROM] =~ /(\d+):(\d+):(\d+)/) {
- $s->[SAM_FLAG] |= 0x4; # TODO: should I set BAM_FUNMAP in this case?
- push(@$s, "H0:i:$1", "H1:i:$2", "H2:i:$3")
- } elsif ($t[EXPORT_POS] < 1) {
- $s->[SAM_FLAG] |= 0x4; # unmapped
- } else {
- $s->[SAM_RNAME] = $t[EXPORT_CHROM];
- $s->[SAM_RNAME] .= "/" . $t[EXPORT_CONTIG] if($t[EXPORT_CONTIG] ne '');
- $has_coor = 1;
- }
- $s->[SAM_POS] = $has_coor? $t[EXPORT_POS] : 0;
-
-# print STDERR "t[14] = " . $t[14] . "\n";
- my $matchDesc = '';
- $s->[SAM_CIGAR] = "*";
- if($has_coor){
- $matchDesc = ($is_export_rev) ? reverse_compl_match_descriptor($t[EXPORT_MD]) : $t[EXPORT_MD];
-
- if($matchDesc =~ /\^/){
- # construct CIGAR string using Richard's function
- $s->[SAM_CIGAR] = match_desc_to_cigar($matchDesc); # indel processing
- } else {
- $s->[SAM_CIGAR] = length($s->[SAM_SEQ]) . "M";
- }
- }
-
-# print STDERR "cigar_string = $cigar_string\n";
-
- $s->[SAM_FLAG] |= 0x10 if ($has_coor && $is_export_rev);
- if($has_coor){
- my $semap = ($t[EXPORT_SEMAP] ne '') ? $t[EXPORT_SEMAP] : 0;
- my $pemap = 0;
- if($is_paired) {
- $pemap = ($t[EXPORT_PEMAP] ne '') ? $t[EXPORT_PEMAP] : 0;
-
- # set `proper pair' bit if non-blank, non-zero PE alignment score:
- $s->[SAM_FLAG] |= 0x02 if ($pemap > 0);
- }
- $s->[SAM_MAPQ] = min(254,max($semap,$pemap));
- } else {
- $s->[SAM_MAPQ] = 0;
- }
- # mate coordinate
- $s->[SAM_MRNM] = '*';
- $s->[SAM_MPOS] = 0;
- $s->[SAM_ISIZE] = 0;
- # aux
- push(@$s, "BC:Z:$t[EXPORT_INDEX]") if ($t[EXPORT_INDEX]);
- if($has_coor){
- # The export match descriptor differs slightly from the samtools match descriptor.
- # In order for the converted SAM files to be as compliant as possible,
- # we put the export match descriptor in optional field 'XD' rather than 'MD':
- push(@$s, "XD:Z:$matchDesc");
- push(@$s, "SM:i:$t[EXPORT_SEMAP]") if ($t[EXPORT_SEMAP] ne '');
- push(@$s, "AS:i:$t[EXPORT_PEMAP]") if ($is_paired and ($t[EXPORT_PEMAP] ne ''));
- }
-}
-
-
-
-#
-# the following code is taken from Richard Shaw's sorted2sam.pl file
-#
-sub reverse_compl_match_descriptor($)
-{
-# print "\nREVERSING THE MATCH DESCRIPTOR!\n";
- my ($match_desc) = @_;
- my $rev_compl_match_desc = reverse($match_desc);
- $rev_compl_match_desc =~ tr/ACGT\^\$/TGCA\$\^/;
-
- # Unreverse the digits of numbers.
- $rev_compl_match_desc = join('',
- map {($_ =~ /\d+/)
- ? join('', reverse(split('', $_)))
- : $_} split(/(\d+)/,
- $rev_compl_match_desc));
-
- return $rev_compl_match_desc;
-}
-
-
-
-sub match_desc_to_cigar($)
-{
- my ($match_desc) = @_;
-
- my @match_desc_parts = split(/(\^.*?\$)/, $match_desc);
- my $cigar_str = '';
- my $cigar_del_ch = 'D';
- my $cigar_ins_ch = 'I';
- my $cigar_match_ch = 'M';
-
- foreach my $match_desc_part (@match_desc_parts) {
- next if (!$match_desc_part);
-
- if ($match_desc_part =~ /^\^([ACGTN]+)\$$/) {
- # Deletion
- $cigar_str .= (length($1) . $cigar_del_ch);
- } elsif ($match_desc_part =~ /^\^(\d+)\$$/) {
- # Insertion
- $cigar_str .= ($1 . $cigar_ins_ch);
- } else {
- $cigar_str .= (match_desc_frag_length($match_desc_part)
- . $cigar_match_ch);
- }
- }
-
- return $cigar_str;
-}
-
-
-#------------------------------------------------------------------------------
-
-sub match_desc_frag_length($)
- {
- my ($match_desc_str) = @_;
- my $len = 0;
-
- my @match_desc_fields = split(/([ACGTN]+)/, $match_desc_str);
-
- foreach my $match_desc_field (@match_desc_fields) {
- next if ($match_desc_field eq '');
-
- $len += (($match_desc_field =~ /(\d+)/)
- ? $1 : length($match_desc_field));
- }
-
- return $len;
-}
-
-
-# argument holds the command line
-sub write_header($;$;$)
-{
- my ($progname,$version,$cl) = @_;
- my $complete_header = "";
- $complete_header .= "\@PG\tID:$progname\tVN:$version\tCL:$cl\n";
-
- return $complete_header;
-}
diff --git a/samtools-0.1.19/misc/interpolate_sam.pl b/samtools-0.1.19/misc/interpolate_sam.pl
deleted file mode 100755
index 6cd6831..0000000
--- a/samtools-0.1.19/misc/interpolate_sam.pl
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/perl
-use strict;
-
-###Builds interpolated pileup from SAM file
-##@description counts bases between paired ends and piles up single end reads.
-##@output, uses a #header for the RNAME and then the number of reads per base
-##@author sm8 at sanger.ac.uk, Stephen B. Montgomery
-
-##@caveats
-##Requires RNAME to have format as per example
-## chromosome:NCBI36:18:1:76117153:1
-## supercontig::NT_113883:1:137703:1
-## clone::AC138827.3:1:149397:1
-##Expects simple CIGAR characters, M, I and D
-##Expects SAM file to be sorted.
-##Expects 0x0010 to mark second read in PE file (as has been the observed case from MAQ output) (important for line 77)
-
-##Verify and read in SAM file
-my $sam_file = $ARGV[0];
-if(!defined($sam_file)) { die("No sam file defined on arg 1"); }
-unless(-f $sam_file) { die("Sam file does not exist: $sam_file"); }
-open(SAM, $sam_file) || die("Cannot open sam file");
-
-##Globals
-my $current_location = ""; ##Current RNAME being processed
-my $current_size = 0; ##Size of sequence region being processed
-my $current_position = 1; ##Current base being processed
-my $open = 0; ##Number of open reads (PE reads that have not been closed)
-my %close = (); ##Hash of closing positions, when the current_position gets to this position it subtracts the
- ##contained value from those open and deletes the indexed position from the hash
-
-while (my $line = <SAM>) {
- my @tokens = split /\t/, $line;
-
- if ($current_location ne $tokens[2]) { ##Start a new sequence region
- for (my $i = $current_position; $i <= $current_size; $i++) { ##Close the previous sequence region
- if (defined($close{$i})) {
- $open = $open - $close{$i};
- delete $close{$i};
- }
- print $open . "\n";
- }
- if ($current_location ne "") {
- print "\n";
- }
-
- ##Initiate a new sequence region
- my @location_tokens = split /:/, $tokens[2];
- $current_position = 1;
- $current_location = $tokens[2];
- $current_size = $location_tokens[4];
- $open = 0;
- %close = ();
- print "#" . $tokens[2] . "\n";
-
- ##Print pileup to just before the first read (will be 0)
- for (my $current_position = 1; $current_position < $tokens[3]; $current_position++) {
- print $open . "\n";
- }
- $current_position = $tokens[3];
-
- } else { ##Sequence region already open
- if ($tokens[3] > $current_position) { ##If the new read's position is greater than the current position
- ##cycle through to catch up to the current position
- for (my $i = $current_position; $i < $tokens[3]; $i++) {
- if (defined($close{$i})) {
- $open = $open - $close{$i};
- delete $close{$i};
- }
- print $open . "\n";
- }
- $current_position = $tokens[3];
- }
- }
- $open++; ##Increment the number of open reads
-
- if (($tokens[1] & 0x0080 || $tokens[1] & 0x0040) && $tokens[1] & 0x0010 && $tokens[1] & 0x0002) { ##if second read of mate pair, add close condition
- $open--;
- my $parsed_cig = &parseCigar($tokens[5]);
- my $seq_region_end = $tokens[3] + $parsed_cig->{'M'} + $parsed_cig->{'D'} - 1;
- if (!defined($close{$seq_region_end + 1})) { $close{$seq_region_end + 1} = 0; }
- $close{$seq_region_end + 1} = $close{$seq_region_end + 1} + 1;
- } elsif (!($tokens[1] & 0x0001) || !($tokens[1] & 0x0002)) { ##if unpaired, add close condition
- my $parsed_cig = &parseCigar($tokens[5]);
- my $seq_region_end = $tokens[3] + $parsed_cig->{'M'} + $parsed_cig->{'D'} - 1;
- if (!defined($close{$seq_region_end + 1})) { $close{$seq_region_end + 1} = 0; }
- $close{$seq_region_end + 1} = $close{$seq_region_end + 1} + 1;
- } else {
- #do nothing
- }
-}
-for (my $i = $current_position; $i <= $current_size; $i++) { ##Finish up the last sequence region
- if (defined($close{$i})) {
- $open = $open - $close{$i};
- delete $close{$i};
- }
- print $open . "\n";
-}
-print "\n";
-close(SAM);
-exit(0);
-
-##reads and tokenizes simple cigarline
-sub parseCigar() {
- my $cigar_line = shift;
- $cigar_line =~ s/([0-9]*[A-Z]{1})/$1\t/g;
- my @cigar_tokens = split /\t/, $cigar_line;
- my %parsed = ('M' => 0,
- 'I' => 0,
- 'D' => 0);
- my @events = ();
- for(my $i = 0; $i < scalar(@cigar_tokens); $i++) {
- if ($cigar_tokens[$i] =~ /([0-9]+)([A-Z]{1})/g) {
- if (!defined($parsed{$2})) { $parsed{$2} = 0; }
- my $nt = $2;
- if ($nt ne "M" && $nt ne "D" && $nt ne "I") { $nt = "M"; }
- $parsed{$nt} += $1;
- my %event_el = ("t" => $nt,
- "n" => $1);
- push @events, \%event_el;
- }
- }
- $parsed{'events'} = \@events;
- return \%parsed;
-}
diff --git a/samtools-0.1.19/misc/maq2sam.c b/samtools-0.1.19/misc/maq2sam.c
deleted file mode 100644
index 2bfbe2a..0000000
--- a/samtools-0.1.19/misc/maq2sam.c
+++ /dev/null
@@ -1,173 +0,0 @@
-#include <string.h>
-#include <zlib.h>
-#include <stdio.h>
-#include <inttypes.h>
-#include <stdlib.h>
-#include <assert.h>
-
-#define PACKAGE_VERSION "r439"
-
-//#define MAQ_LONGREADS
-
-#ifdef MAQ_LONGREADS
-# define MAX_READLEN 128
-#else
-# define MAX_READLEN 64
-#endif
-
-#define MAX_NAMELEN 36
-#define MAQMAP_FORMAT_OLD 0
-#define MAQMAP_FORMAT_NEW -1
-
-#define PAIRFLAG_FF 0x01
-#define PAIRFLAG_FR 0x02
-#define PAIRFLAG_RF 0x04
-#define PAIRFLAG_RR 0x08
-#define PAIRFLAG_PAIRED 0x10
-#define PAIRFLAG_DIFFCHR 0x20
-#define PAIRFLAG_NOMATCH 0x40
-#define PAIRFLAG_SW 0x80
-
-typedef struct
-{
- uint8_t seq[MAX_READLEN]; /* the last base is the single-end mapping quality. */
- uint8_t size, map_qual, info1, info2, c[2], flag, alt_qual;
- uint32_t seqid, pos;
- int dist;
- char name[MAX_NAMELEN];
-} maqmap1_t;
-
-typedef struct
-{
- int format, n_ref;
- char **ref_name;
- uint64_t n_mapped_reads;
- maqmap1_t *mapped_reads;
-} maqmap_t;
-
-maqmap_t *maq_new_maqmap()
-{
- maqmap_t *mm = (maqmap_t*)calloc(1, sizeof(maqmap_t));
- mm->format = MAQMAP_FORMAT_NEW;
- return mm;
-}
-void maq_delete_maqmap(maqmap_t *mm)
-{
- int i;
- if (mm == 0) return;
- for (i = 0; i < mm->n_ref; ++i)
- free(mm->ref_name[i]);
- free(mm->ref_name);
- free(mm->mapped_reads);
- free(mm);
-}
-maqmap_t *maqmap_read_header(gzFile fp)
-{
- maqmap_t *mm;
- int k, len;
- mm = maq_new_maqmap();
- gzread(fp, &mm->format, sizeof(int));
- if (mm->format != MAQMAP_FORMAT_NEW) {
- if (mm->format > 0) {
- fprintf(stderr, "** Obsolete map format is detected. Please use 'mapass2maq' command to convert the format.\n");
- exit(3);
- }
- assert(mm->format == MAQMAP_FORMAT_NEW);
- }
- gzread(fp, &mm->n_ref, sizeof(int));
- mm->ref_name = (char**)calloc(mm->n_ref, sizeof(char*));
- for (k = 0; k != mm->n_ref; ++k) {
- gzread(fp, &len, sizeof(int));
- mm->ref_name[k] = (char*)malloc(len * sizeof(char));
- gzread(fp, mm->ref_name[k], len);
- }
- /* read number of mapped reads */
- gzread(fp, &mm->n_mapped_reads, sizeof(uint64_t));
- return mm;
-}
-
-void maq2tam_core(gzFile fp, const char *rg)
-{
- maqmap_t *mm;
- maqmap1_t mm1, *m1;
- int ret;
- m1 = &mm1;
- mm = maqmap_read_header(fp);
- while ((ret = gzread(fp, m1, sizeof(maqmap1_t))) == sizeof(maqmap1_t)) {
- int j, flag = 0, se_mapq = m1->seq[MAX_READLEN-1];
- if (m1->flag) flag |= 1;
- if ((m1->flag&PAIRFLAG_PAIRED) || ((m1->flag&PAIRFLAG_SW) && m1->flag != 192)) flag |= 2;
- if (m1->flag == 192) flag |= 4;
- if (m1->flag == 64) flag |= 8;
- if (m1->pos&1) flag |= 0x10;
- if ((flag&1) && m1->dist != 0) {
- int c;
- if (m1->dist > 0) {
- if (m1->flag&(PAIRFLAG_FF|PAIRFLAG_RF)) c = 0;
- else if (m1->flag&(PAIRFLAG_FR|PAIRFLAG_RR)) c = 1;
- else c = m1->pos&1;
- } else {
- if (m1->flag&(PAIRFLAG_FF|PAIRFLAG_FR)) c = 0;
- else if (m1->flag&(PAIRFLAG_RF|PAIRFLAG_RR)) c = 1;
- else c = m1->pos&1;
- }
- if (c) flag |= 0x20;
- }
- if (m1->flag) {
- int l = strlen(m1->name);
- if (m1->name[l-2] == '/') {
- flag |= (m1->name[l-1] == '1')? 0x40 : 0x80;
- m1->name[l-2] = '\0';
- }
- }
- printf("%s\t%d\t", m1->name, flag);
- printf("%s\t%d\t", mm->ref_name[m1->seqid], (m1->pos>>1)+1);
- if (m1->flag == 130) {
- int c = (int8_t)m1->seq[MAX_READLEN-1];
- printf("%d\t", m1->alt_qual);
- if (c == 0) printf("%dM\t", m1->size);
- else {
- if (c > 0) printf("%dM%dI%dM\t", m1->map_qual, c, m1->size - m1->map_qual - c);
- else printf("%dM%dD%dM\t", m1->map_qual, -c, m1->size - m1->map_qual);
- }
- se_mapq = 0; // zero SE mapQ for reads aligned by SW
- } else {
- if (flag&4) printf("0\t*\t");
- else printf("%d\t%dM\t", m1->map_qual, m1->size);
- }
- printf("*\t0\t%d\t", m1->dist);
- for (j = 0; j != m1->size; ++j) {
- if (m1->seq[j] == 0) putchar('N');
- else putchar("ACGT"[m1->seq[j]>>6&3]);
- }
- putchar('\t');
- for (j = 0; j != m1->size; ++j)
- putchar((m1->seq[j]&0x3f) + 33);
- putchar('\t');
- if (rg) printf("RG:Z:%s\t", rg);
- if (flag&4) { // unmapped
- printf("MF:i:%d\n", m1->flag);
- } else {
- printf("MF:i:%d\t", m1->flag);
- if (m1->flag) printf("AM:i:%d\tSM:i:%d\t", m1->alt_qual, se_mapq);
- printf("NM:i:%d\tUQ:i:%d\tH0:i:%d\tH1:i:%d\n", m1->info1&0xf, m1->info2, m1->c[0], m1->c[1]);
- }
- }
- if (ret > 0)
- fprintf(stderr, "Truncated! Continue anyway.\n");
- maq_delete_maqmap(mm);
-}
-
-int main(int argc, char *argv[])
-{
- gzFile fp;
- if (argc == 1) {
- fprintf(stderr, "Version: %s\n", PACKAGE_VERSION);
- fprintf(stderr, "Usage: maq2sam <in.map> [<readGroup>]\n");
- return 1;
- }
- fp = strcmp(argv[1], "-")? gzopen(argv[1], "r") : gzdopen(fileno(stdin), "r");
- maq2tam_core(fp, argc > 2? argv[2] : 0);
- gzclose(fp);
- return 0;
-}
diff --git a/samtools-0.1.19/misc/md5.c b/samtools-0.1.19/misc/md5.c
deleted file mode 100644
index 55ae181..0000000
--- a/samtools-0.1.19/misc/md5.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * This code implements the MD5 message-digest algorithm.
- * The algorithm is due to Ron Rivest. This code was
- * written by Colin Plumb in 1993, no copyright is claimed.
- * This code is in the public domain; do with it what you wish.
- *
- * Equivalent code is available from RSA Data Security, Inc.
- * This code has been tested against that, and is equivalent,
- * except that you don't need to include two pages of legalese
- * with every copy.
- *
- * To compute the message digest of a chunk of bytes, declare an
- * MD5Context structure, pass it to MD5Init, call MD5Update as
- * needed on buffers full of bytes, and then call MD5Final, which
- * will fill a supplied 16-byte array with the digest.
- */
-
-/* Brutally hacked by John Walker back from ANSI C to K&R (no
- prototypes) to maintain the tradition that Netfone will compile
- with Sun's original "cc". */
-
-#include <string.h>
-#include "md5.h"
-
-#ifndef HIGHFIRST
-#define byteReverse(buf, len) /* Nothing */
-#else
-/*
- * Note: this code is harmless on little-endian machines.
- */
-void byteReverse(buf, longs)
- unsigned char *buf; unsigned longs;
-{
- uint32_t t;
- do {
- t = (uint32_t) ((unsigned) buf[3] << 8 | buf[2]) << 16 |
- ((unsigned) buf[1] << 8 | buf[0]);
- *(uint32_t *) buf = t;
- buf += 4;
- } while (--longs);
-}
-#endif
-
-void MD5Transform(uint32_t buf[4], uint32_t in[16]);
-
-
-/*
- * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
- * initialization constants.
- */
-void MD5Init(ctx)
- struct MD5Context *ctx;
-{
- ctx->buf[0] = 0x67452301;
- ctx->buf[1] = 0xefcdab89;
- ctx->buf[2] = 0x98badcfe;
- ctx->buf[3] = 0x10325476;
-
- ctx->bits[0] = 0;
- ctx->bits[1] = 0;
-}
-
-/*
- * Update context to reflect the concatenation of another buffer full
- * of bytes.
- */
-void MD5Update(ctx, buf, len)
- struct MD5Context *ctx; unsigned char *buf; unsigned len;
-{
- uint32_t t;
-
- /* Update bitcount */
-
- t = ctx->bits[0];
- if ((ctx->bits[0] = t + ((uint32_t) len << 3)) < t)
- ctx->bits[1]++; /* Carry from low to high */
- ctx->bits[1] += len >> 29;
-
- t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
-
- /* Handle any leading odd-sized chunks */
-
- if (t) {
- unsigned char *p = (unsigned char *) ctx->in + t;
-
- t = 64 - t;
- if (len < t) {
- memcpy(p, buf, len);
- return;
- }
- memcpy(p, buf, t);
- byteReverse(ctx->in, 16);
- MD5Transform(ctx->buf, (uint32_t *) ctx->in);
- buf += t;
- len -= t;
- }
- /* Process data in 64-byte chunks */
-
- while (len >= 64) {
- memcpy(ctx->in, buf, 64);
- byteReverse(ctx->in, 16);
- MD5Transform(ctx->buf, (uint32_t *) ctx->in);
- buf += 64;
- len -= 64;
- }
-
- /* Handle any remaining bytes of data. */
-
- memcpy(ctx->in, buf, len);
-}
-
-/*
- * Final wrapup - pad to 64-byte boundary with the bit pattern
- * 1 0* (64-bit count of bits processed, MSB-first)
- */
-void MD5Final(digest, ctx)
- unsigned char digest[16]; struct MD5Context *ctx;
-{
- unsigned count;
- unsigned char *p;
-
- /* Compute number of bytes mod 64 */
- count = (ctx->bits[0] >> 3) & 0x3F;
-
- /* Set the first char of padding to 0x80. This is safe since there is
- always at least one byte free */
- p = ctx->in + count;
- *p++ = 0x80;
-
- /* Bytes of padding needed to make 64 bytes */
- count = 64 - 1 - count;
-
- /* Pad out to 56 mod 64 */
- if (count < 8) {
- /* Two lots of padding: Pad the first block to 64 bytes */
- memset(p, 0, count);
- byteReverse(ctx->in, 16);
- MD5Transform(ctx->buf, (uint32_t *) ctx->in);
-
- /* Now fill the next block with 56 bytes */
- memset(ctx->in, 0, 56);
- } else {
- /* Pad block to 56 bytes */
- memset(p, 0, count - 8);
- }
- byteReverse(ctx->in, 14);
-
- /* Append length in bits and transform */
- ((uint32_t *) ctx->in)[14] = ctx->bits[0];
- ((uint32_t *) ctx->in)[15] = ctx->bits[1];
-
- MD5Transform(ctx->buf, (uint32_t *) ctx->in);
- byteReverse((unsigned char *) ctx->buf, 4);
- memcpy(digest, ctx->buf, 16);
- memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */
-}
-
-
-/* The four core functions - F1 is optimized somewhat */
-
-/* #define F1(x, y, z) (x & y | ~x & z) */
-#define F1(x, y, z) (z ^ (x & (y ^ z)))
-#define F2(x, y, z) F1(z, x, y)
-#define F3(x, y, z) (x ^ y ^ z)
-#define F4(x, y, z) (y ^ (x | ~z))
-
-/* This is the central step in the MD5 algorithm. */
-#define MD5STEP(f, w, x, y, z, data, s) \
- ( w += f(x, y, z) + data, w = w<<s | w>>(32-s), w += x )
-
-/*
- * The core of the MD5 algorithm, this alters an existing MD5 hash to
- * reflect the addition of 16 longwords of new data. MD5Update blocks
- * the data and converts bytes into longwords for this routine.
- */
-void MD5Transform(buf, in)
- uint32_t buf[4]; uint32_t in[16];
-{
- register uint32_t a, b, c, d;
-
- a = buf[0];
- b = buf[1];
- c = buf[2];
- d = buf[3];
-
- MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
- MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
- MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
- MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);
- MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);
- MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);
- MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);
- MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);
- MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);
- MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);
- MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
- MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
- MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
- MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
- MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
- MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
-
- MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
- MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
- MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
- MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);
- MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);
- MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
- MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
- MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);
- MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);
- MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
- MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);
- MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);
- MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
- MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
- MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
- MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
-
- MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
- MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
- MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
- MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
- MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);
- MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);
- MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);
- MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
- MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
- MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);
- MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);
- MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);
- MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);
- MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
- MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
- MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);
-
- MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
- MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
- MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
- MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);
- MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
- MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);
- MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
- MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);
- MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);
- MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
- MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);
- MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
- MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);
- MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
- MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
- MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);
-
- buf[0] += a;
- buf[1] += b;
- buf[2] += c;
- buf[3] += d;
-}
-
-/* lh3: the following code is added by me */
-
-#ifdef MD5SUM_MAIN
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#define HEX_STR "0123456789abcdef"
-
-static void md5_one(const char *fn)
-{
- unsigned char buf[4096], digest[16];
- MD5_CTX md5;
- int l;
- FILE *fp;
-
- fp = strcmp(fn, "-")? fopen(fn, "r") : stdin;
- if (fp == 0) {
- fprintf(stderr, "md5sum: %s: No such file or directory\n", fn);
- exit(1);
- }
- MD5Init(&md5);
- while ((l = fread(buf, 1, 4096, fp)) > 0)
- MD5Update(&md5, buf, l);
- MD5Final(digest, &md5);
- if (fp != stdin) fclose(fp);
- for (l = 0; l < 16; ++l)
- printf("%c%c", HEX_STR[digest[l]>>4&0xf], HEX_STR[digest[l]&0xf]);
- printf(" %s\n", fn);
-}
-int main(int argc, char *argv[])
-{
- int i;
- if (argc == 1) md5_one("-");
- else for (i = 1; i < argc; ++i) md5_one(argv[i]);
- return 0;
-}
-#endif
diff --git a/samtools-0.1.19/misc/md5.h b/samtools-0.1.19/misc/md5.h
deleted file mode 100644
index 44121e4..0000000
--- a/samtools-0.1.19/misc/md5.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- This file is adapted from a program in this page:
-
- http://www.fourmilab.ch/md5/
-
- The original source code does not work on 64-bit machines due to the
- wrong typedef "uint32". I also added prototypes.
-
- -lh3
- */
-
-#ifndef MD5_H
-#define MD5_H
-
-/* The following tests optimise behaviour on little-endian
- machines, where there is no need to reverse the byte order
- of 32 bit words in the MD5 computation. By default,
- HIGHFIRST is defined, which indicates we're running on a
- big-endian (most significant byte first) machine, on which
- the byteReverse function in md5.c must be invoked. However,
- byteReverse is coded in such a way that it is an identity
- function when run on a little-endian machine, so calling it
- on such a platform causes no harm apart from wasting time.
- If the platform is known to be little-endian, we speed
- things up by undefining HIGHFIRST, which defines
- byteReverse as a null macro. Doing things in this manner
- insures we work on new platforms regardless of their byte
- order. */
-
-#define HIGHFIRST
-
-#if __LITTLE_ENDIAN__ != 0
-#undef HIGHFIRST
-#endif
-
-#include <stdint.h>
-
-struct MD5Context {
- uint32_t buf[4];
- uint32_t bits[2];
- unsigned char in[64];
-};
-
-void MD5Init(struct MD5Context *ctx);
-void MD5Update(struct MD5Context *ctx, unsigned char *buf, unsigned len);
-void MD5Final(unsigned char digest[16], struct MD5Context *ctx);
-
-/*
- * This is needed to make RSAREF happy on some MS-DOS compilers.
- */
-typedef struct MD5Context MD5_CTX;
-
-/* Define CHECK_HARDWARE_PROPERTIES to have main,c verify
- byte order and uint32_t settings. */
-#define CHECK_HARDWARE_PROPERTIES
-
-#endif /* !MD5_H */
diff --git a/samtools-0.1.19/misc/md5fa.c b/samtools-0.1.19/misc/md5fa.c
deleted file mode 100644
index 7a165bf..0000000
--- a/samtools-0.1.19/misc/md5fa.c
+++ /dev/null
@@ -1,58 +0,0 @@
-#include <stdio.h>
-#include <zlib.h>
-#include "md5.h"
-#include "kseq.h"
-
-#define HEX_STR "0123456789abcdef"
-
-KSEQ_INIT(gzFile, gzread)
-
-static void md5_one(const char *fn)
-{
- MD5_CTX md5_one, md5_all;
- int l, i, k;
- gzFile fp;
- kseq_t *seq;
- unsigned char unordered[16], digest[16];
-
- for (l = 0; l < 16; ++l) unordered[l] = 0;
- fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
- if (fp == 0) {
- fprintf(stderr, "md5fa: %s: No such file or directory\n", fn);
- exit(1);
- }
-
- MD5Init(&md5_all);
- seq = kseq_init(fp);
- while ((l = kseq_read(seq)) >= 0) {
- for (i = k = 0; i < seq->seq.l; ++i) {
- if (islower(seq->seq.s[i])) seq->seq.s[k++] = toupper(seq->seq.s[i]);
- else if (isupper(seq->seq.s[i])) seq->seq.s[k++] = seq->seq.s[i];
- }
- MD5Init(&md5_one);
- MD5Update(&md5_one, (unsigned char*)seq->seq.s, k);
- MD5Final(digest, &md5_one);
- for (l = 0; l < 16; ++l) {
- printf("%c%c", HEX_STR[digest[l]>>4&0xf], HEX_STR[digest[l]&0xf]);
- unordered[l] ^= digest[l];
- }
- printf(" %s %s\n", fn, seq->name.s);
- MD5Update(&md5_all, (unsigned char*)seq->seq.s, k);
- }
- MD5Final(digest, &md5_all);
- kseq_destroy(seq);
- for (l = 0; l < 16; ++l)
- printf("%c%c", HEX_STR[digest[l]>>4&0xf], HEX_STR[digest[l]&0xf]);
- printf(" %s >ordered\n", fn);
- for (l = 0; l < 16; ++l)
- printf("%c%c", HEX_STR[unordered[l]>>4&0xf], HEX_STR[unordered[l]&0xf]);
- printf(" %s >unordered\n", fn);
-}
-
-int main(int argc, char *argv[])
-{
- int i;
- if (argc == 1) md5_one("-");
- else for (i = 1; i < argc; ++i) md5_one(argv[i]);
- return 0;
-}
diff --git a/samtools-0.1.19/misc/novo2sam.pl b/samtools-0.1.19/misc/novo2sam.pl
deleted file mode 100755
index 8b53c9e..0000000
--- a/samtools-0.1.19/misc/novo2sam.pl
+++ /dev/null
@@ -1,281 +0,0 @@
-#!/usr/bin/perl -w
-
-# Contact: lh3
-# Version: 0.1.3
-
-#Modified by Zayed Albertyn(zayed.albertyn at gmail.com) & Colin Hercus(colin at novocraft.com)
-
-#use strict;
-#use warnings;
-use Data::Dumper;
-use Getopt::Std;
-
-&novo2sam;
-exit;
-
-sub mating {
- my ($s1, $s2) = @_;
- my $isize = 0;
- if ($s1->[2] ne '*' && $s1->[2] eq $s2->[2]) { # then calculate $isize
- my $x1 = ($s1->[1] & 0x10)? $s1->[3] + length($s1->[9]) : $s1->[3];
- my $x2 = ($s2->[1] & 0x10)? $s2->[3] + length($s2->[9]) : $s2->[3];
- $isize = $x2 - $x1;
- }
- # update mate coordinate
- if ($s2->[2] ne '*') {
- @$s1[6..8] = (($s2->[2] eq $s1->[2])? "=" : $s2->[2], $s2->[3], $isize);
- $s1->[1] |= 0x20 if ($s2->[1] & 0x10);
- } else {
- $s1->[1] |= 0x8;
- }
- if ($s1->[2] ne '*') {
- @$s2[6..8] = (($s1->[2] eq $s2->[2])? "=" : $s1->[2], $s1->[3], -$isize);
- $s2->[1] |= 0x20 if ($s1->[1] & 0x10);
- } else {
- $s2->[1] |= 0x8;
- }
-}
-
-sub novo2sam {
- my %opts = ();
- getopts("p", \%opts);
- die("Usage: novo2sam.pl [-p] <aln.novo>\n") if (@ARGV == 0);
- my $is_paired = defined($opts{p});
- # core loop
- my @s1 = ();
- my @s2 = ();
- my ($s_last, $s_curr) = (\@s1, \@s2);
- while (<>) {
- next if (/^#/);
- next if (/(QC|NM)\s*$/ || /(R\s+\d+)\s*$/);
- &novo2sam_aux($_, $s_curr, $is_paired);
- if (@$s_last != 0 && $s_last->[0] eq $s_curr->[0]) {
- &mating($s_last, $s_curr);
- print join("\t", @$s_last), "\n";
- print join("\t", @$s_curr), "\n";
- @$s_last = (); @$s_curr = ();
- } else {
- print join("\t", @$s_last), "\n" if (@$s_last != 0);
- my $s = $s_last; $s_last = $s_curr; $s_curr = $s;
- }
- }
- print join("\t", @$s_last), "\n" if (@$s_last != 0);
-}
-
-sub novo2sam_aux {
- my ($line, $s, $is_paired) = @_;
-
- chomp($line);
- my @t = split(/\s+/, $line);
- my @variations = @t[13 .. $#t];
- @$s = ();
- return if ($t[4] ne 'U');
- my $len = length($t[2]);
- # read name
- $s->[0] = substr($t[0], 1);
- $s->[0] =~ s/\/[12]$//g;
- # initial flag (will be updated later)
- $s->[1] = 0;
- $s->[1] |= 1 | 1<<($t[1] eq 'L'? 6 : 7);
- $s->[1] |= 2 if ($t[10] eq '.');
- # read & quality
- if ($t[9] eq 'R') {
- $s->[9] = reverse($t[2]);
- $s->[10] = reverse($t[3]);
- $s->[9] =~ tr/ACGTRYMKWSNacgtrymkwsn/TGCAYRKMWSNtgcayrkmwsn/;
- } else {
- $s->[9] = $t[2]; $s->[10] = $t[3];
- }
- # cigar
- my $cigarstring ="";
- if (scalar @variations ==0 ) {
- $s->[5] = $len . "M"; # IMPORTANT: this cigar is not correct for gapped alignment
- } else {
- #convert to correct CIGAR
- my $tmpstr = join" ", at variations ;
- if ( $tmpstr=~ /\+|\-/ ) {
- $cigarstring = cigar_method($line,\@variations,$len);
- $s->[5]=$cigarstring;
- } else {
- $s->[5]=$len. "M";
- }
-}
-
-# coor
- $s->[2] = substr($t[7], 1); $s->[3] = $t[8];
- $s->[1] |= 0x10 if ($t[9] eq 'R');
- # mapQ
- $s->[4] = $t[5] > $t[6]? $t[5] : $t[6];
- # mate coordinate
- $s->[6] = '*'; $s->[7] = $s->[8] = 0;
- # aux
- push(@$s, "NM:i:".(@t-13));
- my $md = '';
- $md = mdtag($md,$line,\@variations,$len);
- push(@$s, "MD:Z:$md");
-
-}
-
-sub mdtag {
- my $oldmd = shift;
- my $line = shift;
- my $ref =shift;
- my $rdlen = shift;
- my @variations = @$ref;
- my $string="";
- my $mdtag="";
- my $t=1;
- my $q=1;
- my $deleteflag=0;
- my $len =0;
- foreach $string (@variations) {
- my ($indeltype,$insert) = indeltype($string);
- if ($indeltype eq "+") {
- $len = length ($insert);
- $q+=$len;
- next;
- }
- my $pos = $1 if $string =~ /^(\d+)/;
- $len = $pos - $t;
- if ($len !=0 || ($deleteflag eq 1 && $indeltype eq ">")) {
- $mdtag.=$len;
- }
- $t+=$len;
- $q+=$len;
- if ($indeltype eq ">") {
- $mdtag.=$insert;
- $deleteflag=0;
- $t+=1;
- $q+=1;
- }
- if ($indeltype eq "-") {
- my $deletedbase = $2 if $string =~ /(\d+)\-([A-Za-z]+)/;
- if ($deleteflag == 0 ) {
- $mdtag.="^";
- }
- $mdtag.=$deletedbase;
- $deleteflag=1;
- $t+=1;
- }
- }
- $len = $rdlen - $q + 1;
- if ($len > 0) {
- $mdtag.="$len";
- }
-# print "In:$line\n";
-# print "MD: OLD => NEW\nMD: $oldmd => $mdtag\n\n";
-
- return $mdtag;
-}
-
-sub indeltype {
- my $string = shift;
- my $insert="";
- my $indeltype;
- if ($string =~ /([A-Za-z]+)\>/) {
- $indeltype=">";
- $insert=$1;
- } elsif ($string =~ /\-/) {
- $indeltype="-";
- } elsif ($string =~ /\+([A-Za-z]+)/) {
- $indeltype="+";
- $insert=$1;
- }
- return ($indeltype,$insert);
-
-}
-
-
-sub cigar_method {
- my $line = shift;
- my $ref =shift;
- my $rdlen = shift;
- my @variations = @$ref;
- my $string="";
- my $type="";
- my $t =1;
- my $q=1;
- my $indeltype="";
- my $cigar= "";
- my $insert = "";
- my $len=0;
- my @cig=();
- foreach $string (@variations) {
- next if $string =~ />/;
- my $pos = $1 if $string =~ /^(\d+)/;
-
- if ($string =~ /\+([A-Za-z]+)/) {
- $indeltype="+";
- $insert = $1;
- }elsif ($string =~ /\-([A-Za-z]+)/) {
- $indeltype="-";
- $insert = $1;
- }
-#print "$pos $indeltype $insert $t $q\n";
- $len = $pos - $t;
- if ( $len > 0) {
- $cigar.=$len."M";
- push(@cig,$len."M");
- }
- $t+=$len;
- $q+=$len;
-
- if ($indeltype eq "-") {
- $cigar.="D";
- push(@cig,"D");
- $t++;
- }
- if ($indeltype eq "+") {
- $len = length ($insert);
- if ($len == 1) {
- $cigar.="I";
- push(@cig,"I");
- }
- if ($len > 1) {
- $cigar.=$len."I";
- push(@cig,$len."I")
- }
- $q+=$len;
- }
- $insert="";
- }
- $len= $rdlen - $q + 1;
- if ($len > 0) {
- $cigar.=$len."M";
- push(@cig,$len."M");
- }
-
- $cigar = newcigar($cigar,'D');
- $cigar = newcigar($cigar,'I');
-
- #print "$line\n";
- #print "c CIGAR:\t$cigar\n\n";
- return $cigar;
-
-}
-
-
-
-sub newcigar {
- my $cigar = shift;
- my $char = shift;
- my $new = "";
- my $copy = $cigar;
-#print "$cigar\n";
- $copy =~ s/^($char+)/$1;/g;
-#print "$copy\n";
- $copy =~ s/([^0-9$char])($char+)/$1;$2;/g;
-#print "$copy\n";
- my @parts = split(/;/,$copy);
- my $el="";
- foreach $el (@parts) {
-#print "$el\n";
- if ($el =~ /^$char+$/) {
- $new.=length($el).$char;
- }else {
- $new.=$el;
- }
-
- }
- return $new;
-}
diff --git a/samtools-0.1.19/misc/plot-bamcheck b/samtools-0.1.19/misc/plot-bamcheck
deleted file mode 100755
index 1792c6f..0000000
--- a/samtools-0.1.19/misc/plot-bamcheck
+++ /dev/null
@@ -1,882 +0,0 @@
-#!/usr/bin/env perl
-#
-# Author: petr.danecek at sanger
-#
-
-use strict;
-use warnings;
-use Carp;
-
-my $opts = parse_params();
-parse_bamcheck($opts);
-plot_qualities($opts);
-plot_acgt_cycles($opts);
-plot_gc($opts);
-plot_gc_depth($opts);
-plot_isize($opts);
-plot_coverage($opts);
-plot_mismatches_per_cycle($opts);
-plot_indel_dist($opts);
-plot_indel_cycles($opts);
-
-exit;
-
-#--------------------------------
-
-sub error
-{
- my (@msg) = @_;
- if ( scalar @msg ) { confess @msg; }
- die
- "Usage: plot-bamcheck [OPTIONS] file.bam.bc\n",
- " plot-bamcheck -p outdir/ file.bam.bc\n",
- "Options:\n",
- " -k, --keep-files Do not remove temporary files.\n",
- " -p, --prefix <path> The output files prefix, add a slash to create new directory.\n",
- " -r, --ref-stats <file.fa.gc> Optional reference stats file with expected GC content (created with -s).\n",
- " -s, --do-ref-stats <file.fa> Calculate reference sequence GC for later use with -r\n",
- " -t, --targets <file.tab> Restrict -s to the listed regions (tab-delimited chr,from,to. 1-based, inclusive)\n",
- " -h, -?, --help This help message.\n",
- "\n";
-}
-
-
-sub parse_params
-{
- $0 =~ s{^.+/}{};
- my $opts = { args=>join(' ',$0, at ARGV) };
- while (defined(my $arg=shift(@ARGV)))
- {
- if ( $arg eq '-k' || $arg eq '--keep-files' ) { $$opts{keep_files}=1; next; }
- if ( $arg eq '-r' || $arg eq '--ref-stats' ) { $$opts{ref_stats}=shift(@ARGV); next; }
- if ( $arg eq '-s' || $arg eq '--do-ref-stats' ) { $$opts{do_ref_stats}=shift(@ARGV); next; }
- if ( $arg eq '-t' || $arg eq '--targets' ) { $$opts{targets}=shift(@ARGV); next; }
- if ( $arg eq '-p' || $arg eq '--prefix' ) { $$opts{prefix}=shift(@ARGV); next; }
- if ( $arg eq '-?' || $arg eq '-h' || $arg eq '--help' ) { error(); }
- if ( -e $arg ) { $$opts{bamcheck}=$arg; next; }
- error("Unknown parameter or non-existent file \"$arg\". Run -h for help.\n");
- }
- if ( exists($$opts{do_ref_stats }) ) { do_ref_stats($opts); exit; }
- if ( !exists($$opts{bamcheck}) ) { error("No bamcheck file?\n") }
- if ( !exists($$opts{prefix}) ) { error("Expected -p parameter.\n") }
- if ( $$opts{prefix}=~m{/$} ) { `mkdir -p $$opts{prefix}`; }
- elsif ( !($$opts{prefix}=~/-$/) ) { $$opts{prefix} .= '-'; }
- return $opts;
-}
-
-
-# Creates GC stats for either the whole reference or only on target regions for exome QC
-sub do_ref_stats
-{
- my ($opts) = @_;
-
-
- my %targets = ();
- if ( exists($$opts{targets}) )
- {
- my ($prev_chr,$prev_pos);
- open(my $fh,'<',$$opts{targets}) or error("$$opts{targets}: $!");
- while (my $line=<$fh>)
- {
- if ( $line=~/^#/ ) { next; }
- my ($chr,$from,$to) = split(/\s+/,$line);
- chomp($to);
- push @{$targets{$chr}}, $from,$to;
- if ( !defined $prev_chr or $chr ne $prev_chr ) { $prev_chr=$chr; $prev_pos=$from }
- if ( $prev_pos > $from ) { error("The file must be sorted: $$opts{targets}\n"); }
- $prev_pos = $from;
- }
- close($fh);
- }
-
- my $_len = 60; # for now do only standard fasta's with 60 bases per line
- my %gc_counts = ();
- my ($skip_chr,$pos,$ireg,$regions);
- open(my $fh,'<',$$opts{do_ref_stats}) or error("$$opts{do_ref_stats}: $!");
- while (my $line=<$fh>)
- {
- if ( $line=~/^>/ )
- {
- if ( !scalar %targets ) { next; }
-
- if ( !($line=~/>(\S+)/) ) { error("FIXME: could not determine chromosome name: $line"); }
- if ( !exists($targets{$1}) ) { $skip_chr=$1; next; }
- undef $skip_chr;
- $pos = 0;
- $ireg = 0;
- $regions = $targets{$1};
- }
- if ( defined $skip_chr ) { next; }
-
- # Only $_len sized lines are considered and no chopping for target regions.
- chomp($line);
- my $len = length($line);
- if ( $len ne $_len ) { next; }
-
- if ( scalar %targets )
- {
- while ( $ireg<@$regions && $$regions[$ireg+1]<=$pos ) { $ireg += 2; }
- $pos += $len;
- if ( $ireg==@$regions ) { next; }
- if ( $pos < $$regions[$ireg] ) { next; }
- }
-
- my $gc_count = 0;
- for (my $i=0; $i<$len; $i++)
- {
- my $base = substr($line,$i,1);
- if ( $base eq 'g' || $base eq 'G' || $base eq 'c' || $base eq 'C' ) { $gc_count++; }
- }
- $gc_counts{$gc_count}++;
- }
-
- print "# Generated by $$opts{args}\n";
- print "# The columns are: GC content bin, normalized frequency\n";
- my $max;
- for my $count (values %gc_counts)
- {
- if ( !defined $max or $count>$max ) { $max=$count; }
- }
- for my $gc (sort {$a<=>$b} keys %gc_counts)
- {
- if ( $gc==0 ) { next; }
- printf "%f\t%f\n", $gc*100./$_len, $gc_counts{$gc}/$max;
- }
-}
-
-sub plot
-{
- my ($cmdfile) = @_;
- my $cmd = "gnuplot $cmdfile";
- system($cmd);
- if ( $? ) { error("The command exited with non-zero status $?:\n\t$cmd\n\n"); }
-}
-
-
-sub parse_bamcheck
-{
- my ($opts) = @_;
- open(my $fh,'<',$$opts{bamcheck}) or error("$$opts{bamcheck}: $!");
- my $line = <$fh>;
- if ( !($line=~/^# This file was produced by bamcheck (\S+)/) ) { error("Sanity check failed: was this file generated by bamcheck?"); }
- $$opts{dat}{version} = $1;
- while ($line=<$fh>)
- {
- if ( $line=~/^#/ ) { next; }
- my @items = split(/\t/,$line);
- chomp($items[-1]);
- if ( $items[0] eq 'SN' )
- {
- $$opts{dat}{$items[1]} = splice(@items,2);
- next;
- }
- push @{$$opts{dat}{$items[0]}}, [splice(@items,1)];
- }
- close($fh);
-
- # Check sanity
- if ( !exists($$opts{dat}{'sequences:'}) or !$$opts{dat}{'sequences:'} )
- {
- error("Sanity check failed: no sequences found by bamcheck??\n");
- }
-}
-
-sub older_than
-{
- my ($opts,$version) = @_;
- my ($year,$month,$day) = split(/-/,$version);
- $version = $$opts{dat}{version};
- if ( !($version=~/\((\d+)-(\d+)-(\d+)\)$/) ) { return 1; }
- if ( $1<$year ) { return 1; }
- elsif ( $1>$year ) { return 0; }
- if ( $2<$month ) { return 1; }
- elsif ( $2>$month ) { return 0; }
- if ( $3<$day ) { return 1; }
- return 0;
-}
-
-sub get_defaults
-{
- my ($opts,$img_fname,%args) = @_;
-
- if ( !($img_fname=~/\.png$/i) ) { error("FIXME: currently only PNG supported. (Easy to extend.)\n"); }
-
- # Determine the gnuplot script file name
- my $gp_file = $img_fname;
- $gp_file =~ s{\.[^.]+$}{.gp};
- if ( !($gp_file=~/.gp$/) ) { $gp_file .= '.gp'; }
-
- # Determine the default title:
- # 5446_6/5446_6.bam.bc.gp -> 5446_6
- # test.aaa.png -> test.aaa
- if ( !($$opts{bamcheck}=~m{([^/]+?)(?:\.bam)?(?:\.bc)?$}i) ) { error("FIXME: Could not determine the title from [$img_fname]\n"); }
- my $title = $1;
-
- my $dir = $gp_file;
- $dir =~ s{/[^/]+$}{};
- if ( $dir && $dir ne $gp_file ) { `mkdir -p $dir`; }
-
- my $wh = exists($args{wh}) ? $args{wh} : '600,400';
-
- open(my $fh,'>',$gp_file) or error("$gp_file: $!");
- return {
- title => $title,
- gp => $gp_file,
- img => $img_fname,
- fh => $fh,
- terminal => qq[set terminal png size $wh truecolor],
- grid => 'set grid xtics ytics y2tics back lc rgb "#cccccc"',
- };
-}
-
-sub percentile
-{
- my ($p, at vals) = @_;
- my $N = 0;
- for my $val (@vals) { $N += $val; }
- my $n = $p*($N+1)/100.;
- my $k = int($n);
- my $d = $n-$k;
- if ( $k<=0 ) { return 0; }
- if ( $k>=$N ) { return scalar @vals-1; }
- my $cnt;
- for (my $i=0; $i<@vals; $i++)
- {
- $cnt += $vals[$i];
- if ( $cnt>=$k ) { return $i; }
- }
- error("FIXME: this should not happen [percentile]\n");
-}
-
-sub plot_qualities
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{FFQ}) or !@{$$opts{dat}{FFQ}} ) { return; }
-
- my $yrange = @{$$opts{dat}{FFQ}[0]} > 50 ? @{$$opts{dat}{FFQ}[0]} : 50;
- my $is_paired = $$opts{dat}{'is paired:'};
-
- # Average quality per cycle, forward and reverse reads in one plot
- my $args = get_defaults($opts,"$$opts{prefix}quals.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set ylabel "Average Quality"
- set xlabel "Cycle"
- set yrange [0:$yrange]
- set title "$$args{title}"
- plot '-' using 1:2 with lines title 'Forward reads' ] . ($is_paired ? q[, '-' using 1:2 with lines title 'Reverse reads'] : '') . q[
- ];
- my (@fp75, at fp50, at fmean);
- my (@lp75, at lp50, at lmean);
- my ($fmax,$fmax_qual,$fmax_cycle);
- my ($lmax,$lmax_qual,$lmax_cycle);
- for my $cycle (@{$$opts{dat}{FFQ}})
- {
- my $sum=0; my $n=0;
- for (my $iqual=1; $iqual<@$cycle; $iqual++)
- {
- $sum += $$cycle[$iqual]*$iqual;
- $n += $$cycle[$iqual];
- if ( !defined $fmax or $fmax<$$cycle[$iqual] ) { $fmax=$$cycle[$iqual]; $fmax_qual=$iqual; $fmax_cycle=$$cycle[0]; }
- }
- my $p25 = percentile(25,(@$cycle)[1..$#$cycle]);
- my $p50 = percentile(50,(@$cycle)[1..$#$cycle]);
- my $p75 = percentile(75,(@$cycle)[1..$#$cycle]);
- if ( !$n ) { next; }
- push @fp75, "$$cycle[0]\t$p25\t$p75\n";
- push @fp50, "$$cycle[0]\t$p50\n";
- push @fmean, sprintf "%d\t%.2f\n", $$cycle[0],$sum/$n;
- printf $fh $fmean[-1];
- }
- print $fh "end\n";
- if ( $is_paired )
- {
- for my $cycle (@{$$opts{dat}{LFQ}})
- {
- my $sum=0; my $n=0;
- for (my $iqual=1; $iqual<@$cycle; $iqual++)
- {
- $sum += $$cycle[$iqual]*$iqual;
- $n += $$cycle[$iqual];
- if ( !defined $lmax or $lmax<$$cycle[$iqual] ) { $lmax=$$cycle[$iqual]; $lmax_qual=$iqual; $lmax_cycle=$$cycle[0]; }
- }
- my $p25 = percentile(25,(@$cycle)[1..$#$cycle]);
- my $p50 = percentile(50,(@$cycle)[1..$#$cycle]);
- my $p75 = percentile(75,(@$cycle)[1..$#$cycle]);
- if ( !$n ) { next; }
- push @lp75, "$$cycle[0]\t$p25\t$p75\n";
- push @lp50, "$$cycle[0]\t$p50\n";
- push @lmean, sprintf "%d\t%.2f\n", $$cycle[0],$sum/$n;
- printf $fh $lmean[-1];
- }
- print $fh "end\n";
- }
- close($fh);
- plot($$args{gp});
-
-
-
- # Average, mean and quality percentiles per cycle, forward and reverse reads in separate plots
- $args = get_defaults($opts,"$$opts{prefix}quals2.png",wh=>'700,500');
- $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set multiplot
- set rmargin 0
- set lmargin 0
- set tmargin 0
- set bmargin 0
- set origin 0.1,0.1
- set size 0.4,0.8
- set yrange [0:$yrange]
- set ylabel "Quality"
- set xlabel "Cycle (fwd reads)"
- plot '-' using 1:2:3 with filledcurve lt 1 lc rgb "#cccccc" t '25-75th percentile' , '-' using 1:2 with lines lc rgb "#000000" t 'Median', '-' using 1:2 with lines lt 1 t 'Mean'
- ];
- print $fh join('', at fp75),"end\n";
- print $fh join('', at fp50),"end\n";
- print $fh join('', at fmean),"end\n";
- if ( $is_paired )
- {
- print $fh qq[
- set origin 0.55,0.1
- set size 0.4,0.8
- unset ytics
- set y2tics mirror
- set yrange [0:$yrange]
- unset ylabel
- set xlabel "Cycle (rev reads)"
- set label "$$args{title}" at screen 0.5,0.95 center
- plot '-' using 1:2:3 with filledcurve lt 1 lc rgb "#cccccc" t '25-75th percentile' , '-' using 1:2 with lines lc rgb "#000000" t 'Median', '-' using 1:2 with lines lt 2 t 'Mean'
- ];
- print $fh join('', at lp75),"end\n";
- print $fh join('', at lp50),"end\n";
- print $fh join('', at lmean),"end\n";
- }
- close($fh);
- plot($$args{gp});
-
-
-
- # Quality distribution per cycle, the distribution is for each cycle plotted as a separate curve
- $args = get_defaults($opts,"$$opts{prefix}quals3.png",wh=>'600,600');
- $fh = $$args{fh};
- my $nquals = @{$$opts{dat}{FFQ}[0]}-1;
- my $ncycles = @{$$opts{dat}{FFQ}};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set multiplot
- set rmargin 0
- set lmargin 0
- set tmargin 0
- set bmargin 0
- set origin 0.15,0.52
- set size 0.8,0.4
- set title "$$args{title}"
- set ylabel "Frequency (fwd reads)"
- set label "Cycle $fmax_cycle" at $fmax_qual+1,$fmax
- unset xlabel
- set xrange [0:$nquals]
- set format x ""
- ];
- my @plots;
- for (my $i=0; $i<$ncycles; $i++) { push @plots, q['-' using 1:2 with lines t ''] }
- print $fh "plot ", join(",", @plots), "\n";
- for my $cycle (@{$$opts{dat}{FFQ}})
- {
- for (my $iqual=1; $iqual<$nquals; $iqual++) { print $fh "$iqual\t$$cycle[$iqual]\n"; }
- print $fh "end\n";
- }
- if ( $is_paired )
- {
- print $fh qq[
- set origin 0.15,0.1
- set size 0.8,0.4
- unset title
- unset format
- set xtics
- set xlabel "Quality"
- unset label
- set label "Cycle $lmax_cycle" at $lmax_qual+1,$lmax
- set ylabel "Frequency (rev reads)"
- ];
- print $fh "plot ", join(",", @plots), "\n";
- for my $cycle (@{$$opts{dat}{LFQ}})
- {
- for (my $iqual=1; $iqual<$nquals; $iqual++)
- {
- print $fh "$iqual\t$$cycle[$iqual]\n";
- }
- print $fh "end\n";
- }
- }
- close($fh);
- plot($$args{gp});
-
-
- # Heatmap qualitites
- $args = get_defaults($opts,"$$opts{prefix}quals-hm.png", wh=>'600,500');
- $fh = $$args{fh};
- my $max = defined $lmax && $lmax > $fmax ? $lmax : $fmax;
- my @ytics;
- for my $cycle (@{$$opts{dat}{FFQ}}) { if ( $$cycle[0]%10==0 ) { push @ytics,qq["$$cycle[0]" $$cycle[0]]; } }
- my $ytics = join(',', @ytics);
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- unset key
- unset colorbox
- set palette defined (0 0 0 0, 1 0 0 1, 3 0 1 0, 4 1 0 0, 6 1 1 1)
- set cbrange [0:$max]
- set yrange [0:$ncycles]
- set xrange [0:$nquals]
- set view map
- set multiplot
- set rmargin 0
- set lmargin 0
- set tmargin 0
- set bmargin 0
- set origin 0,0.46
- set size 0.95,0.6
- set obj 1 rectangle behind from first 0,0 to first $nquals,$ncycles
- set obj 1 fillstyle solid 1.0 fillcolor rgbcolor "black"
- set ylabel "Cycle (fwd reads)" offset character -1,0
- unset ytics
- set ytics ($ytics)
- unset xtics
- set title "$$args{title}"
- splot '-' matrix with image
- ];
- for my $cycle (@{$$opts{dat}{FFQ}})
- {
- for (my $iqual=1; $iqual<@$cycle; $iqual++) { print $fh "\t$$cycle[$iqual]"; }
- print $fh "\n";
- }
- print $fh "end\nend\n";
- @ytics = ();
- for my $cycle (@{$$opts{dat}{LFQ}}) { if ( $$cycle[0]%10==0 ) { push @ytics,qq["$$cycle[0]" $$cycle[0]]; } }
- $ytics = join(',', @ytics);
- print $fh qq[
- set origin 0,0.03
- set size 0.95,0.6
- set ylabel "Cycle (rev reads)" offset character -1,0
- set xlabel "Base Quality"
- unset title
- unset ytics
- set ytics ($ytics)
- set xrange [0:$nquals]
- set xtics
- set colorbox vertical user origin first ($nquals+1),0 size screen 0.025,0.812
- set cblabel "Number of bases"
- splot '-' matrix with image
- ];
- for my $cycle (@{$$opts{dat}{LFQ}})
- {
- for (my $iqual=1; $iqual<@$cycle; $iqual++) { print $fh "\t$$cycle[$iqual]"; }
- print $fh "\n";
- }
- print $fh "end\nend\n";
- close($fh);
- plot($$args{gp});
-}
-
-
-sub plot_acgt_cycles
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{GCC}) or !@{$$opts{dat}{GCC}} ) { return; }
-
- my $args = get_defaults($opts,"$$opts{prefix}acgt-cycles.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set style line 1 linecolor rgb "green"
- set style line 2 linecolor rgb "red"
- set style line 3 linecolor rgb "black"
- set style line 4 linecolor rgb "blue"
- set style increment user
- set ylabel "Base content [%]"
- set xlabel "Read Cycle"
- set yrange [0:100]
- set title "$$args{title}"
- plot '-' w l ti 'A', '-' w l ti 'C', '-' w l ti 'G', '-' w l ti 'T'
- ];
- for my $base (1..4)
- {
- for my $cycle (@{$$opts{dat}{GCC}})
- {
- print $fh $$cycle[0]+1,"\t",$$cycle[$base],"\n";
- }
- print $fh "end\n";
- }
- close($fh);
- plot($$args{gp});
-}
-
-
-sub plot_gc
-{
- my ($opts) = @_;
-
- my $is_paired = $$opts{dat}{'is paired:'};
- my $args = get_defaults($opts,"$$opts{prefix}gc-content.png");
- my $fh = $$args{fh};
- my ($gcl_max,$gcf_max,$lmax,$fmax);
- for my $gc (@{$$opts{dat}{GCF}}) { if ( !defined $gcf_max or $gcf_max<$$gc[1] ) { $gcf_max=$$gc[1]; $fmax=$$gc[0]; } }
- for my $gc (@{$$opts{dat}{GCL}}) { if ( !defined $gcl_max or $gcl_max<$$gc[1] ) { $gcl_max=$$gc[1]; $lmax=$$gc[0]; } }
- my $gcmax = $is_paired && $gcl_max > $gcf_max ? $lmax : $fmax;
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set title "$$args{title}"
- set ylabel "Normalized Frequency"
- set xlabel "GC Content [%]"
- set yrange [0:1.1]
- set label sprintf("%.1f",$gcmax) at $gcmax,1 front offset 1,0
- plot ]
- . (exists($$opts{ref_stats}) ? q['-' smooth csplines with lines lt 0 title 'Reference', ] : '')
- . q['-' smooth csplines with lines lc 1 title 'First fragments' ]
- . ($is_paired ? q[, '-' smooth csplines with lines lc 2 title 'Last fragments'] : '')
- . q[
- ];
- if ( exists($$opts{ref_stats}) )
- {
- open(my $ref,'<',$$opts{ref_stats}) or error("$$opts{ref_stats}: $!");
- while (my $line=<$ref>) { print $fh $line }
- close($ref);
- print $fh "end\n";
- }
- for my $cycle (@{$$opts{dat}{GCF}}) { printf $fh "%d\t%f\n", $$cycle[0],$$cycle[1]/$gcf_max; }
- print $fh "end\n";
- if ( $is_paired )
- {
- for my $cycle (@{$$opts{dat}{GCL}}) { printf $fh "%d\t%f\n", $$cycle[0],$$cycle[1]/$gcl_max; }
- print $fh "end\n";
- }
- close($fh);
- plot($$args{gp});
-}
-
-
-sub plot_gc_depth
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{GCD}) or !@{$$opts{dat}{GCD}} ) { return; }
-
- # Find unique sequence percentiles for 30,40, and 50% GC content, just to draw x2tics.
- my @tics = ( {gc=>30},{gc=>40},{gc=>50} );
- for my $gc (@{$$opts{dat}{GCD}})
- {
- for my $tic (@tics)
- {
- my $diff = abs($$gc[0]-$$tic{gc});
- if ( !exists($$tic{pr}) or $diff<$$tic{diff} ) { $$tic{pr}=$$gc[1]; $$tic{diff}=$diff; }
- }
- }
-
- my @x2tics;
- for my $tic (@tics) { push @x2tics, qq["$$tic{gc}" $$tic{pr}]; }
- my $x2tics = join(',', at x2tics);
-
- my $args = get_defaults($opts,"$$opts{prefix}gc-depth.png", wh=>'600,500');
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set ylabel "Mapped depth"
- set xlabel "Percentile of mapped sequence ordered by GC content"
- set x2label "GC Content [%]"
- set title "$$args{title}"
- set x2tics ($x2tics)
- set xtics nomirror
- set xrange [0.1:99.9]
-
- plot '-' using 1:2:3 with filledcurve lt 1 lc rgb "#dedede" t '10-90th percentile' , \\
- '-' using 1:2:3 with filledcurve lt 1 lc rgb "#bbdeff" t '25-75th percentile' , \\
- '-' using 1:2 with lines lc rgb "#0084ff" t 'Median'
- ];
- for my $gc (@{$$opts{dat}{GCD}}) { print $fh "$$gc[1]\t$$gc[2]\t$$gc[6]\n"; } print $fh "end\n";
- for my $gc (@{$$opts{dat}{GCD}}) { print $fh "$$gc[1]\t$$gc[3]\t$$gc[5]\n"; } print $fh "end\n";
- for my $gc (@{$$opts{dat}{GCD}}) { print $fh "$$gc[1]\t$$gc[4]\n"; } print $fh "end\n";
- close($fh);
- plot($$args{gp});
-}
-
-
-sub plot_isize
-{
- my ($opts) = @_;
-
- if ( !$$opts{dat}{'is paired:'} or !exists($$opts{dat}{IS}) or !@{$$opts{dat}{IS}} ) { return; }
-
- my ($isize_max,$isize_cnt);
- for my $isize (@{$$opts{dat}{IS}})
- {
- if ( !defined $isize_max or $isize_cnt<$$isize[1] ) { $isize_cnt=$$isize[1]; $isize_max=$$isize[0]; }
- }
-
- my $args = get_defaults($opts,"$$opts{prefix}insert-size.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set rmargin 5
- set label sprintf("%d",$isize_max) at $isize_max+10,$isize_cnt
- set ylabel "Number of pairs"
- set xlabel "Insert Size"
- set title "$$args{title}"
- plot \\
- '-' with lines lc rgb 'black' title 'All pairs', \\
- '-' with lines title 'Inward', \\
- '-' with lines title 'Outward', \\
- '-' with lines title 'Other'
- ];
- for my $isize (@{$$opts{dat}{IS}}) { print $fh "$$isize[0]\t$$isize[1]\n"; } print $fh "end\n";
- for my $isize (@{$$opts{dat}{IS}}) { print $fh "$$isize[0]\t$$isize[2]\n"; } print $fh "end\n";
- for my $isize (@{$$opts{dat}{IS}}) { print $fh "$$isize[0]\t$$isize[3]\n"; } print $fh "end\n";
- for my $isize (@{$$opts{dat}{IS}}) { print $fh "$$isize[0]\t$$isize[4]\n"; } print $fh "end\n";
- close($fh);
- plot($$args{gp});
-}
-
-
-sub plot_coverage
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{COV}) or !@{$$opts{dat}{COV}} ) { return; }
-
- my @vals;
- for my $cov (@{$$opts{dat}{COV}}) { push @vals,$$cov[2]; }
- my $i = percentile(99.8, at vals);
- my $p99 = $$opts{dat}{COV}[$i][1];
-
- my $args = get_defaults($opts,"$$opts{prefix}coverage.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set ylabel "Number of mapped bases"
- set xlabel "Coverage"
- set style fill solid border -1
- set title "$$args{title}"
- set xrange [:$p99]
- plot '-' with lines notitle
- ];
- for my $cov (@{$$opts{dat}{COV}})
- {
- if ( $$cov[2]==0 ) { next; }
- print $fh "$$cov[1]\t$$cov[2]\n";
- }
- print $fh "end\n";
- close($fh);
- plot($$args{gp});
-}
-
-
-sub plot_mismatches_per_cycle
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{MPC}) or !@{$$opts{dat}{MPC}} ) { return; }
- if ( older_than($opts,'2012-02-06') ) { plot_mismatches_per_cycle_old($opts); }
-
- my $nquals = @{$$opts{dat}{MPC}[0]} - 2;
- my $ncycles = @{$$opts{dat}{MPC}};
- my ($style,$with);
- if ( $ncycles>100 ) { $style = ''; $with = 'w l'; }
- else { $style = 'set style data histogram; set style histogram rowstacked'; $with = ''; }
-
- my $args = get_defaults($opts,"$$opts{prefix}mism-per-cycle.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set style line 1 linecolor rgb "#e40000"
- set style line 2 linecolor rgb "#ff9f00"
- set style line 3 linecolor rgb "#eeee00"
- set style line 4 linecolor rgb "#4ebd68"
- set style line 5 linecolor rgb "#0061ff"
- set style increment user
- set key left top
- $style
- set ylabel "Number of mismatches"
- set xlabel "Read Cycle"
- set style fill solid border -1
- set title "$$args{title}"
- set xrange [-1:$ncycles]
- plot '-' $with ti 'Base Quality>30', \\
- '-' $with ti '30>=Q>20', \\
- '-' $with ti '20>=Q>10', \\
- '-' $with ti '10>=Q', \\
- '-' $with ti "N's"
- ];
- for my $cycle (@{$$opts{dat}{MPC}})
- {
- my $sum; for my $idx (31..$#$cycle) { $sum += $$cycle[$idx]; }
- print $fh "$sum\n";
- }
- print $fh "end\n";
- for my $cycle (@{$$opts{dat}{MPC}})
- {
- my $sum; for my $idx (22..31) { $sum += $$cycle[$idx]; }
- print $fh "$sum\n";
- }
- print $fh "end\n";
- for my $cycle (@{$$opts{dat}{MPC}})
- {
- my $sum; for my $idx (12..21) { $sum += $$cycle[$idx]; }
- print $fh "$sum\n";
- }
- print $fh "end\n";
- for my $cycle (@{$$opts{dat}{MPC}})
- {
- my $sum; for my $idx (2..11) { $sum += $$cycle[$idx]; }
- print $fh "$sum\n";
- }
- print $fh "end\n";
- for my $cycle (@{$$opts{dat}{MPC}}) { print $fh "$$cycle[1]\n"; }
- print $fh "end\n";
- close($fh);
- plot($$args{gp});
-}
-
-sub plot_indel_dist
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{ID}) or !@{$$opts{dat}{ID}} ) { return; }
-
- my $args = get_defaults($opts,"$$opts{prefix}indel-dist.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set style line 1 linetype 1 linecolor rgb "red"
- set style line 2 linetype 2 linecolor rgb "black"
- set style line 3 linetype 3 linecolor rgb "green"
- set style increment user
- set ylabel "Indel count [log]"
- set xlabel "Indel length"
- set y2label "Insertions/Deletions ratio"
- set log y
- set y2tics nomirror
- set ytics nomirror
- set title "$$args{title}"
- plot '-' w l ti 'Insertions', '-' w l ti 'Deletions', '-' axes x1y2 w l ti "Ins/Dels ratio"
- ];
- for my $len (@{$$opts{dat}{ID}}) { print $fh "$$len[0]\t$$len[1]\n"; } print $fh "end\n";
- for my $len (@{$$opts{dat}{ID}}) { print $fh "$$len[0]\t$$len[2]\n"; } print $fh "end\n";
- for my $len (@{$$opts{dat}{ID}}) { printf $fh "%d\t%f\n", $$len[0],$$len[2]?$$len[1]/$$len[2]:0; } print $fh "end\n";
- close($fh);
- plot($$args{gp});
-}
-
-sub plot_indel_cycles
-{
- my ($opts) = @_;
-
- if ( !exists($$opts{dat}{IC}) or !@{$$opts{dat}{IC}} ) { return; }
-
- my $args = get_defaults($opts,"$$opts{prefix}indel-cycles.png");
- my $fh = $$args{fh};
- print $fh qq[
- $$args{terminal}
- set output "$$args{img}"
- $$args{grid}
- set style line 1 linetype 1 linecolor rgb "red"
- set style line 2 linetype 2 linecolor rgb "black"
- set style line 3 linetype 3 linecolor rgb "green"
- set style line 4 linetype 4 linecolor rgb "blue"
- set style increment user
- set ylabel "Indel count"
- set xlabel "Read Cycle"
- set title "$$args{title}"
- plot '-' w l ti 'Insertions (fwd)', '' w l ti 'Insertions (rev)', '' w l ti 'Deletions (fwd)', '' w l ti 'Deletions (rev)'
- ];
- for my $len (@{$$opts{dat}{IC}}) { print $fh "$$len[0]\t$$len[1]\n"; } print $fh "end\n";
- for my $len (@{$$opts{dat}{IC}}) { print $fh "$$len[0]\t$$len[2]\n"; } print $fh "end\n";
- for my $len (@{$$opts{dat}{IC}}) { print $fh "$$len[0]\t$$len[3]\n"; } print $fh "end\n";
- for my $len (@{$$opts{dat}{IC}}) { print $fh "$$len[0]\t$$len[4]\n"; } print $fh "end\n";
- close($fh);
- plot($$args{gp});
-}
-
-
-
-
-
-
-
-sub has_values
-{
- my ($opts, at tags) = @_;
- for my $tag (@tags)
- {
- my (@lines) = `cat $$opts{bamcheck} | grep ^$tag | wc -l`;
- chomp($lines[0]);
- if ( $lines[0]<2 ) { return 0; }
- }
- return 1;
-}
-
-sub plot_mismatches_per_cycle_old
-{
- my ($opts) = @_;
-
- my $args = get_defaults($opts,"$$opts{prefix}mism-per-cycle.png");
- my ($nquals) = `grep ^MPC $$opts{bamcheck} | awk '\$2==1' | sed 's,\\t,\\n,g' | wc -l`;
- my ($ncycles) = `grep ^MPC $$opts{bamcheck} | wc -l`;
- chomp($nquals);
- chomp($ncycles);
- $nquals--;
- $ncycles--;
- my @gr0_15 = (2..17);
- my @gr16_30 = (18..32);
- my @gr31_n = (33..$nquals);
- my $gr0_15 = '$'. join('+$', at gr0_15);
- my $gr16_30 = '$'. join('+$', at gr16_30);
- my $gr31_n = '$'. join('+$', at gr31_n);
-
- open(my $fh,'>',$$args{gp}) or error("$$args{gp}: $!");
- print $fh q[
- set terminal png size 600,400 truecolor font "DejaVuSansMono,9"
- set output "] . $$args{img} . q["
-
- set key left top
- set style data histogram
- set style histogram rowstacked
-
- set grid back lc rgb "#aaaaaa"
- set ylabel "Number of mismatches"
- set xlabel "Read Cycle"
- set style fill solid border -1
- set title "] . $$args{title} . qq["
- set xrange [-1:$ncycles]
-
- plot '< grep ^MPC $$opts{bamcheck} | cut -f 2-' using ($gr31_n) ti 'Base Quality>30', '' using ($gr16_30) ti '30>=Q>15', '' using ($gr0_15) ti '15>=Q'
- ];
- close($fh);
-
- plot($$args{gp});
-}
-
-
diff --git a/samtools-0.1.19/misc/psl2sam.pl b/samtools-0.1.19/misc/psl2sam.pl
deleted file mode 100755
index a96a6de..0000000
--- a/samtools-0.1.19/misc/psl2sam.pl
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/perl -w
-
-# Author: lh3
-
-# This script calculates a score using the BLAST scoring
-# system. However, I am not sure how to count gap opens and gap
-# extensions. It seems to me that column 5-8 are not what I am
-# after. This script counts gaps from the last three columns. It does
-# not generate reference skip (N) in the CIGAR as it is not easy to
-# directly tell which gaps correspond to introns.
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-my %opts = (a=>1, b=>3, q=>5, r=>2);
-getopts('a:b:q:r:', \%opts);
-die("Usage: psl2sam.pl [-a $opts{a}] [-b $opts{b}] [-q $opts{q}] [-r $opts{r}] <in.psl>\n") if (@ARGV == 0 && -t STDIN);
-
-my @stack;
-my $last = '';
-my ($a, $b, $q, $r) = ($opts{a}, $opts{b}, $opts{q}, $opts{r});
-while (<>) {
- next unless (/^\d/);
- my @t = split;
- my @s;
- my $cigar = '';
- if ($t[8] eq '-') {
- my $tmp = $t[11];
- $t[11] = $t[10] - $t[12];
- $t[12] = $t[10] - $tmp;
- }
- @s[0..4] = ($t[9], (($t[8] eq '+')? 0 : 16), $t[13], $t[15]+1, 0);
- @s[6..10] = ('*', 0, 0, '*', '*');
- $cigar .= $t[11].'H' if ($t[11]); # 5'-end clipping
- my @x = split(',', $t[18]);
- my @y = split(',', $t[19]);
- my @z = split(',', $t[20]);
- my ($y0, $z0) = ($y[0], $z[0]);
- my ($gap_open, $gap_ext) = (0, 0, 0);
- for (1 .. $t[17]-1) {
- my $ly = $y[$_] - $y[$_-1] - $x[$_-1];
- my $lz = $z[$_] - $z[$_-1] - $x[$_-1];
- if ($ly < $lz) { # del: the reference gap is longer
- ++$gap_open;
- $gap_ext += $lz - $ly;
- $cigar .= ($y[$_] - $y0) . 'M';
- $cigar .= ($lz - $ly) . 'D';
- ($y0, $z0) = ($y[$_], $z[$_]);
- } elsif ($lz < $ly) { # ins: the query gap is longer
- ++$gap_open;
- $gap_ext += $ly - $lz;
- $cigar .= ($z[$_] - $z0) . 'M';
- $cigar .= ($ly - $lz) . 'I';
- ($y0, $z0) = ($y[$_], $z[$_]);
- }
- }
- $cigar .= ($t[12] - $y0) . 'M';
- $cigar .= ($t[10] - $t[12]).'H' if ($t[10] != $t[12]); # 3'-end clipping
- $s[5] = $cigar;
- my $score = $a * $t[0] - $b * $t[1] - $q * $gap_open - $r * $gap_ext;
- $score = 0 if ($score < 0);
- $s[11] = "AS:i:$score";
- print join("\t", @s), "\n";
-}
diff --git a/samtools-0.1.19/misc/r2plot.lua b/samtools-0.1.19/misc/r2plot.lua
deleted file mode 100755
index 0a1b9f1..0000000
--- a/samtools-0.1.19/misc/r2plot.lua
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env luajit
-
-function string:split(sep, n)
- local a, start = {}, 1;
- sep = sep or "%s+";
- repeat
- local b, e = self:find(sep, start);
- if b == nil then
- table.insert(a, self:sub(start));
- break
- end
- a[#a+1] = self:sub(start, b - 1);
- start = e + 1;
- if n and #a == n then
- table.insert(a, self:sub(start));
- break
- end
- until start > #self;
- return a;
-end
-
-function io.xopen(fn, mode)
- mode = mode or 'r';
- if fn == nil then return io.stdin;
- elseif fn == '-' then return (mode == 'r' and io.stdin) or io.stdout;
- elseif fn:sub(-3) == '.gz' then return (mode == 'r' and io.popen('gzip -dc ' .. fn, 'r')) or io.popen('gzip > ' .. fn, 'w');
- elseif fn:sub(-4) == '.bz2' then return (mode == 'r' and io.popen('bzip2 -dc ' .. fn, 'r')) or io.popen('bgzip2 > ' .. fn, 'w');
- else return io.open(fn, mode) end
-end
-
-local eps = {};
-
-function eps.func(fp)
- fp = fp or io.stdout
- fp:write("/C { dup 255 and 255 div exch dup -8 bitshift 255 and 255 div 3 1 roll -16 bitshift 255 and 255 div 3 1 roll setrgbcolor } bind def\n")
- fp:write("/L { 4 2 roll moveto lineto } bind def\n")
- fp:write("/LX { dup 4 -1 roll exch moveto lineto } bind def\n")
- fp:write("/LY { dup 4 -1 roll moveto exch lineto } bind def\n")
- fp:write("/LS { 3 1 roll moveto show } bind def\n")
- fp:write("/RS { dup stringwidth pop 4 -1 roll exch sub 3 -1 roll moveto show } bind def\n")
- fp:write("/B { 4 copy 3 1 roll exch 6 2 roll 8 -2 roll moveto lineto lineto lineto closepath } bind def\n")
-end
-
-function eps.font(ft, size, fp)
- fp = fp or io.stdout
- fp:write(string.format('/FS %d def\n', size));
- fp:write('/FS4 FS 4 div def\n');
- fp:write('/' .. ft .. ' findfont FS scalefont setfont\n');
-end
-
-local scale = 8;
-
-if #arg == 0 then
- print("Usage: r2plot.lua <in.txt>");
- os.exit(1)
-end
-
-local fp = io.xopen(arg[1]);
-local n = tonumber(fp:read());
-
-print('%!PS-Adobe-3.0 EPSF-3.0');
-print('%%' .. string.format('BoundingBox: -%d -%d %.3f %.3f\n', 10*scale, scale, (n+1)*scale, (n+1)*scale));
-print(string.format('%.3f setlinewidth', scale));
-print(string.format('/plot { setgray moveto 0 %d rlineto } def', scale));
-print(string.format('/plothalf { setgray moveto 0 %.2f rlineto } def', scale/2));
-eps.func();
-eps.font('Helvetica', scale-1);
-
-local i = 1;
-for l in fp:lines() do
- local t = l:split('\t');
- print(string.format("%d %d FS4 add (%s) RS", (i-1)*scale-2, (i-1)*scale, t[1]));
- for j = 2, #t do
- if tonumber(t[j]) > 0.01 then
- print(string.format('%.2f %.2f %.2f plot stroke', (i-1+.5)*scale, (j-2)*scale, 1.-t[j]));
- end
- end
- i = i + 1;
-end
-for j = 1, 21 do
- print(string.format('%.2f %.2f %.2f plothalf stroke', -8*scale, (j-1) * scale/2, 1.-(j-1)/20));
-end
-print('showpage');
diff --git a/samtools-0.1.19/misc/sam2vcf.pl b/samtools-0.1.19/misc/sam2vcf.pl
deleted file mode 100755
index afaf91e..0000000
--- a/samtools-0.1.19/misc/sam2vcf.pl
+++ /dev/null
@@ -1,270 +0,0 @@
-#!/usr/bin/perl -w
-#
-# VCF specs: http://www.1000genomes.org/wiki/doku.php?id=1000_genomes:analysis:vcf3.3
-#
-# Contact: pd3 at sanger
-# Version: 2010-04-23
-
-use strict;
-use warnings;
-use Carp;
-
-my $opts = parse_params();
-do_pileup_to_vcf($opts);
-
-exit;
-
-#---------------
-
-sub error
-{
- my (@msg) = @_;
- if ( scalar @msg ) { croak(@msg); }
- die
- "Usage: sam2vcf.pl [OPTIONS] < in.pileup > out.vcf\n",
- "Options:\n",
- " -h, -?, --help This help message.\n",
- " -i, --indels-only Ignore SNPs.\n",
- " -r, --refseq <file.fa> The reference sequence, required when indels are present.\n",
- " -R, --keep-ref Print reference alleles as well.\n",
- " -s, --snps-only Ignore indels.\n",
- " -t, --column-title <string> The column title.\n",
- "\n";
-}
-
-
-sub parse_params
-{
- my %opts = ();
-
- $opts{fh_in} = *STDIN;
- $opts{fh_out} = *STDOUT;
-
- while (my $arg=shift(@ARGV))
- {
- if ( $arg eq '-R' || $arg eq '--keep-ref' ) { $opts{keep_ref}=1; next; }
- if ( $arg eq '-r' || $arg eq '--refseq' ) { $opts{refseq}=shift(@ARGV); next; }
- if ( $arg eq '-t' || $arg eq '--column-title' ) { $opts{title}=shift(@ARGV); next; }
- if ( $arg eq '-s' || $arg eq '--snps-only' ) { $opts{snps_only}=1; next; }
- if ( $arg eq '-i' || $arg eq '--indels-only' ) { $opts{indels_only}=1; next; }
- if ( $arg eq '-?' || $arg eq '-h' || $arg eq '--help' ) { error(); }
-
- error("Unknown parameter \"$arg\". Run -h for help.\n");
- }
- return \%opts;
-}
-
-sub iupac_to_gtype
-{
- my ($ref,$base) = @_;
- my %iupac = (
- 'K' => ['G','T'],
- 'M' => ['A','C'],
- 'S' => ['C','G'],
- 'R' => ['A','G'],
- 'W' => ['A','T'],
- 'Y' => ['C','T'],
- );
- if ( !exists($iupac{$base}) )
- {
- if ( $base ne 'A' && $base ne 'C' && $base ne 'G' && $base ne 'T' ) { error("FIXME: what is this [$base]?\n"); }
- if ( $ref eq $base ) { return ('.','0/0'); }
- return ($base,'1/1');
- }
- my $gt = $iupac{$base};
- if ( $$gt[0] eq $ref ) { return ($$gt[1],'0/1'); }
- elsif ( $$gt[1] eq $ref ) { return ($$gt[0],'0/1'); }
- return ("$$gt[0],$$gt[1]",'1/2');
-}
-
-
-sub parse_indel
-{
- my ($cons) = @_;
- if ( $cons=~/^-/ )
- {
- my $len = length($');
- return "D$len";
- }
- elsif ( $cons=~/^\+/ ) { return "I$'"; }
- elsif ( $cons eq '*' ) { return undef; }
- error("FIXME: could not parse [$cons]\n");
-}
-
-
-# An example of the pileup format:
-# 1 3000011 C C 32 0 98 1 ^~, A
-# 1 3002155 * +T/+T 53 119 52 5 +T * 4 1 0
-# 1 3003094 * -TT/-TT 31 164 60 11 -TT * 5 6 0
-# 1 3073986 * */-AAAAAAAAAAAAAA 3 3 45 9 * -AAAAAAAAAAAAAA 7 2 0
-#
-sub do_pileup_to_vcf
-{
- my ($opts) = @_;
-
- my $fh_in = $$opts{fh_in};
- my $fh_out = $$opts{fh_out};
- my ($prev_chr,$prev_pos,$prev_ref);
- my $refseq;
- my $ignore_indels = $$opts{snps_only} ? 1 : 0;
- my $ignore_snps = $$opts{indels_only} ? 1 : 0;
- my $keep_ref = $$opts{keep_ref} ? 1 : 0;
- my $title = exists($$opts{title}) ? $$opts{title} : 'data';
-
- print $fh_out
- qq[##fileformat=VCFv3.3\n],
- qq[##INFO=DP,1,Integer,"Total Depth"\n],
- qq[##FORMAT=GT,1,String,"Genotype"\n],
- qq[##FORMAT=GQ,1,Integer,"Genotype Quality"\n],
- qq[##FORMAT=DP,1,Integer,"Read Depth"\n],
- qq[#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t$title\n]
- ;
-
- while (my $line=<$fh_in>)
- {
- chomp($line);
- my (@items) = split(/\t/,$line);
- if ( scalar @items<8 )
- {
- error("\nToo few columns, does not look like output of 'samtools pileup -c': $line\n");
- }
- my ($chr,$pos,$ref,$cons,$cons_qual,$snp_qual,$rms_qual,$depth,$a1,$a2) = @items;
- $ref = uc($ref);
- $cons = uc($cons);
-
- my ($alt,$gt);
- if ( $ref eq '*' )
- {
- # An indel is involved.
- if ( $ignore_indels )
- {
- $prev_ref = $ref;
- $prev_pos = $pos;
- $prev_chr = $chr;
- next;
- }
-
- if (!defined $prev_chr || $chr ne $prev_chr || $pos ne $prev_pos)
- {
- if ( !$$opts{refseq} ) { error("Cannot do indels without the reference.\n"); }
- if ( !$refseq ) { $refseq = Fasta->new(file=>$$opts{refseq}); }
- $ref = $refseq->get_base($chr,$pos);
- $ref = uc($ref);
- }
- else { $ref = $prev_ref; }
-
- # One of the alleles can be a reference and it can come in arbitrary order. In some
- # cases */* can be encountered. In such a case, look in the additional columns.
- my ($al1,$al2) = split(m{/},$cons);
- if ( $al1 eq $al2 && $al1 eq '*' ) { $al1=$a1; $al2=$a2; }
- my $alt1 = parse_indel($al1);
- my $alt2 = parse_indel($al2);
- if ( !$alt1 && !$alt2 ) { error("FIXME: could not parse indel:\n", $line); }
- if ( !$alt1 )
- {
- $alt=$alt2;
- $gt='0/1';
- }
- elsif ( !$alt2 )
- {
- $alt=$alt1;
- $gt='0/1';
- }
- elsif ( $alt1 eq $alt2 )
- {
- $alt="$alt1";
- $gt='1/1';
- }
- else
- {
- $alt="$alt1,$alt2";
- $gt='1/2';
- }
- }
- else
- {
- if ( $ignore_snps || (!$keep_ref && $ref eq $cons) )
- {
- $prev_ref = $ref;
- $prev_pos = $pos;
- $prev_chr = $chr;
- next;
- }
-
- # SNP
- ($alt,$gt) = iupac_to_gtype($ref,$cons);
- }
-
- print $fh_out "$chr\t$pos\t.\t$ref\t$alt\t$snp_qual\t0\tDP=$depth\tGT:GQ:DP\t$gt:$cons_qual:$depth\n";
-
- $prev_ref = $ref;
- $prev_pos = $pos;
- $prev_chr = $chr;
- }
-}
-
-
-#------------- Fasta --------------------
-#
-# Uses samtools to get a requested base from a fasta file. For efficiency, preloads
-# a chunk to memory. The size of the cached sequence can be controlled by the 'size'
-# parameter.
-#
-package Fasta;
-
-use strict;
-use warnings;
-use Carp;
-
-sub Fasta::new
-{
- my ($class, at args) = @_;
- my $self = {@args};
- bless $self, ref($class) || $class;
- if ( !$$self{file} ) { $self->throw(qq[Missing the parameter "file"\n]); }
- $$self{chr} = undef;
- $$self{from} = undef;
- $$self{to} = undef;
- if ( !$$self{size} ) { $$self{size}=10_000_000; }
- bless $self, ref($class) || $class;
- return $self;
-}
-
-sub read_chunk
-{
- my ($self,$chr,$pos) = @_;
- my $to = $pos + $$self{size};
- my $cmd = "samtools faidx $$self{file} $chr:$pos-$to";
- my @out = `$cmd`;
- if ( $? ) { $self->throw("$cmd: $!"); }
- my $line = shift(@out);
- if ( !($line=~/^>$chr:(\d+)-(\d+)/) ) { $self->throw("Could not parse: $line"); }
- $$self{chr} = $chr;
- $$self{from} = $1;
- $$self{to} = $2;
- my $chunk = '';
- while ($line=shift(@out))
- {
- chomp($line);
- $chunk .= $line;
- }
- $$self{chunk} = $chunk;
- return;
-}
-
-sub get_base
-{
- my ($self,$chr,$pos) = @_;
- if ( !$$self{chr} || $chr ne $$self{chr} || $pos<$$self{from} || $pos>$$self{to} )
- {
- $self->read_chunk($chr,$pos);
- }
- my $idx = $pos - $$self{from};
- return substr($$self{chunk},$idx,1);
-}
-
-sub throw
-{
- my ($self, at msg) = @_;
- croak(@msg);
-}
diff --git a/samtools-0.1.19/misc/samtools.pl b/samtools-0.1.19/misc/samtools.pl
deleted file mode 100755
index d03c1c7..0000000
--- a/samtools-0.1.19/misc/samtools.pl
+++ /dev/null
@@ -1,528 +0,0 @@
-#!/usr/bin/perl -w
-
-# Author: lh3
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-my $version = '0.3.3';
-&usage if (@ARGV < 1);
-
-my $command = shift(@ARGV);
-my %func = (showALEN=>\&showALEN, pileup2fq=>\&pileup2fq, varFilter=>\&varFilter, plp2vcf=>\&plp2vcf,
- unique=>\&unique, uniqcmp=>\&uniqcmp, sra2hdr=>\&sra2hdr, sam2fq=>\&sam2fq);
-
-die("Unknown command \"$command\".\n") if (!defined($func{$command}));
-&{$func{$command}};
-exit(0);
-
-#
-# showALEN
-#
-
-sub showALEN {
- die(qq/Usage: samtools.pl showALEN <in.sam>\n/) if (@ARGV == 0 && -t STDIN);
- while (<>) {
- my @t = split;
- next if (/^\@/ || @t < 11);
- my $l = 0;
- $_ = $t[5];
- s/(\d+)[MI]/$l+=$1/eg;
- print join("\t", @t[0..5]), "\t$l\t", join("\t", @t[6..$#t]), "\n";
- }
-}
-
-#
-# varFilter
-#
-
-#
-# Filtration code:
-#
-# d low depth
-# D high depth
-# W too many SNPs in a window (SNP only)
-# G close to a high-quality indel (SNP only)
-# Q low RMS mapping quality (SNP only)
-# g close to another indel with higher quality (indel only)
-# s low SNP quality (SNP only)
-# i low indel quality (indel only)
-
-sub varFilter {
- my %opts = (d=>3, D=>100, l=>30, Q=>25, q=>10, G=>25, s=>100, w=>10, W=>10, N=>2, p=>undef, S=>'', i=>'');
- getopts('pq:d:D:l:Q:w:W:N:G:S:i:', \%opts);
- die(qq/
-Usage: samtools.pl varFilter [options] <in.cns-pileup>
-
-Options: -Q INT minimum RMS mapping quality for SNPs [$opts{Q}]
- -q INT minimum RMS mapping quality for gaps [$opts{q}]
- -d INT minimum read depth [$opts{d}]
- -D INT maximum read depth [$opts{D}]
- -S INT minimum SNP quality [$opts{S}]
- -i INT minimum indel quality [$opts{i}]
-
- -G INT min indel score for nearby SNP filtering [$opts{G}]
- -w INT SNP within INT bp around a gap to be filtered [$opts{w}]
-
- -W INT window size for filtering dense SNPs [$opts{W}]
- -N INT max number of SNPs in a window [$opts{N}]
-
- -l INT window size for filtering adjacent gaps [$opts{l}]
-
- -p print filtered variants
-\n/) if (@ARGV == 0 && -t STDIN);
-
- # calculate the window size
- my ($ol, $ow, $oW) = ($opts{l}, $opts{w}, $opts{W});
- my $max_dist = $ol > $ow? $ol : $ow;
- $max_dist = $oW if ($max_dist < $oW);
- # the core loop
- my @staging; # (indel_filtering_score, flt_tag)
- while (<>) {
- my @t = split;
- next if (uc($t[2]) eq uc($t[3]) || $t[3] eq '*/*'); # skip non-var sites
- # clear the out-of-range elements
- while (@staging) {
- # Still on the same chromosome and the first element's window still affects this position?
- last if ($staging[0][3] eq $t[0] && $staging[0][4] + $staging[0][2] + $max_dist >= $t[1]);
- varFilter_aux(shift(@staging), $opts{p}); # calling a function is a bit slower, not much
- }
- my ($flt, $score) = (0, -1);
- # first a simple filter
- if ($t[7] < $opts{d}) {
- $flt = 2;
- } elsif ($t[7] > $opts{D}) {
- $flt = 3;
- }
- if ($t[2] eq '*') { # an indel
- if ($opts{i} && $opts{i}>$t[5]) { $flt = 8; }
- }
- elsif ($opts{S} && $opts{S}>$t[5]) { $flt = 7; } # SNP
-
- # site dependent filters
- my $len=0;
- if ($flt == 0) {
- if ($t[2] eq '*') { # an indel
- # If deletion, remember the length of the deletion
- my ($a,$b) = split(m{/},$t[3]);
- my $alen = length($a) - 1;
- my $blen = length($b) - 1;
- if ( $alen>$blen )
- {
- if ( substr($a,0,1) eq '-' ) { $len=$alen; }
- }
- elsif ( substr($b,0,1) eq '-' ) { $len=$blen; }
-
- $flt = 1 if ($t[6] < $opts{q});
- # filtering SNPs
- if ($t[5] >= $opts{G}) {
- for my $x (@staging) {
- # Is it a SNP and is it outside the SNP filter window?
- next if ($x->[0] >= 0 || $x->[4] + $x->[2] + $ow < $t[1]);
- $x->[1] = 5 if ($x->[1] == 0);
- }
- }
- # calculate the filtering score (different from indel quality)
- $score = $t[5];
- $score += $opts{s} * $t[10] if ($t[8] ne '*');
- $score += $opts{s} * $t[11] if ($t[9] ne '*');
- # check the staging list for indel filtering
- for my $x (@staging) {
- # Is it a SNP and is it outside the gap filter window
- next if ($x->[0] < 0 || $x->[4] + $x->[2] + $ol < $t[1]);
- if ($x->[0] < $score) {
- $x->[1] = 6;
- } else {
- $flt = 6; last;
- }
- }
- } else { # a SNP
- $flt = 1 if ($t[6] < $opts{Q});
- # check adjacent SNPs
- my $k = 1;
- for my $x (@staging) {
- ++$k if ($x->[0] < 0 && $x->[4] + $x->[2] + $oW >= $t[1] && ($x->[1] == 0 || $x->[1] == 4 || $x->[1] == 5));
- }
- # filtering is necessary
- if ($k > $opts{N}) {
- $flt = 4;
- for my $x (@staging) {
- $x->[1] = 4 if ($x->[0] < 0 && $x->[4] + $x->[2] + $oW >= $t[1] && $x->[1] == 0);
- }
- } else { # then check gap filter
- for my $x (@staging) {
- next if ($x->[0] < 0 || $x->[4] + $x->[2] + $ow < $t[1]);
- if ($x->[0] >= $opts{G}) {
- $flt = 5; last;
- }
- }
- }
- }
- }
- push(@staging, [$score, $flt, $len, @t]);
- }
- # output the last few elements in the staging list
- while (@staging) {
- varFilter_aux(shift @staging, $opts{p});
- }
-}
-
-sub varFilter_aux {
- my ($first, $is_print) = @_;
- if ($first->[1] == 0) {
- print join("\t", @$first[3 .. @$first-1]), "\n";
- } elsif ($is_print) {
- print STDERR join("\t", substr("UQdDWGgsiX", $first->[1], 1), @$first[3 .. @$first-1]), "\n";
- }
-}
-
-#
-# pileup2fq
-#
-
-sub pileup2fq {
- my %opts = (d=>3, D=>255, Q=>25, G=>25, l=>10);
- getopts('d:D:Q:G:l:', \%opts);
- die(qq/
-Usage: samtools.pl pileup2fq [options] <in.cns-pileup>
-
-Options: -d INT minimum depth [$opts{d}]
- -D INT maximum depth [$opts{D}]
- -Q INT min RMS mapQ [$opts{Q}]
- -G INT minimum indel score [$opts{G}]
- -l INT indel filter winsize [$opts{l}]\n
-/) if (@ARGV == 0 && -t STDIN);
-
- my ($last_chr, $seq, $qual, @gaps, $last_pos);
- my $_Q = $opts{Q};
- my $_d = $opts{d};
- my $_D = $opts{D};
-
- $last_chr = '';
- while (<>) {
- my @t = split;
- if ($last_chr ne $t[0]) {
- &p2q_post_process($last_chr, \$seq, \$qual, \@gaps, $opts{l}) if ($last_chr);
- $last_chr = $t[0];
- $last_pos = 0;
- $seq = ''; $qual = '';
- @gaps = ();
- }
- if ($t[1] - $last_pos != 1) {
- $seq .= 'n' x ($t[1] - $last_pos - 1);
- $qual .= '!' x ($t[1] - $last_pos - 1);
- }
- if ($t[2] eq '*') {
- push(@gaps, $t[1]) if ($t[5] >= $opts{G});
- } else {
- $seq .= ($t[6] >= $_Q && $t[7] >= $_d && $t[7] <= $_D)? uc($t[3]) : lc($t[3]);
- my $q = $t[4] + 33;
- $q = 126 if ($q > 126);
- $qual .= chr($q);
- }
- $last_pos = $t[1];
- }
- &p2q_post_process($last_chr, \$seq, \$qual, \@gaps, $opts{l});
-}
-
-sub p2q_post_process {
- my ($chr, $seq, $qual, $gaps, $l) = @_;
- &p2q_filter_gaps($seq, $gaps, $l);
- print "\@$chr\n"; &p2q_print_str($seq);
- print "+\n"; &p2q_print_str($qual);
-}
-
-sub p2q_filter_gaps {
- my ($seq, $gaps, $l) = @_;
- for my $g (@$gaps) {
- my $x = $g > $l? $g - $l : 0;
- substr($$seq, $x, $l + $l) = lc(substr($$seq, $x, $l + $l));
- }
-}
-
-sub p2q_print_str {
- my ($s) = @_;
- my $l = length($$s);
- for (my $i = 0; $i < $l; $i += 60) {
- print substr($$s, $i, 60), "\n";
- }
-}
-
-#
-# sam2fq
-#
-
-sub sam2fq {
- my %opts = (n=>20, p=>'');
- getopts('n:p:', \%opts);
- die("Usage: samtools.pl sam2fq [-n 20] [-p <prefix>] <inp.sam>\n") if (@ARGV == 0 && -t STDIN);
- if ($opts{p} && $opts{n} > 1) {
- my $pre = $opts{p};
- my @fh;
- for (0 .. $opts{n}-1) {
- open($fh[$_], sprintf("| gzip > $pre.%.3d.fq.gz", $_)) || die;
- }
- my $i = 0;
- while (<>) {
- next if (/^@/);
- chomp;
- my @t = split("\t");
- next if ($t[9] eq '*');
- my ($name, $seq, $qual);
- if ($t[1] & 16) { # reverse strand
- $seq = reverse($t[9]);
- $qual = reverse($t[10]);
- $seq =~ tr/ACGTacgt/TGCAtgca/;
- } else {
- ($seq, $qual) = @t[9,10];
- }
- $name = $t[0];
- $name .= "/1" if ($t[1] & 0x40);
- $name .= "/2" if ($t[1] & 0x80);
- print {$fh[$i]} "\@$name\n$seq\n";
- if ($qual ne '*') {
- print {$fh[$i]} "+\n$qual\n";
- }
- $i = 0 if (++$i == $opts{n});
- }
- close($fh[$_]) for (0 .. $opts{n}-1);
- } else {
- die("To be implemented.\n");
- }
-}
-
-#
-# sra2hdr
-#
-
-# This subroutine does not use an XML parser. It requires that the SRA
-# XML files are properly formated.
-sub sra2hdr {
- my %opts = ();
- getopts('', \%opts);
- die("Usage: samtools.pl sra2hdr <SRA.prefix>\n") if (@ARGV == 0);
- my $pre = $ARGV[0];
- my $fh;
- # read sample
- my $sample = 'UNKNOWN';
- open($fh, "$pre.sample.xml") || die;
- while (<$fh>) {
- $sample = $1 if (/<SAMPLE.*alias="([^"]+)"/i);
- }
- close($fh);
- # read experiment
- my (%exp2lib, $exp);
- open($fh, "$pre.experiment.xml") || die;
- while (<$fh>) {
- if (/<EXPERIMENT.*accession="([^\s"]+)"/i) {
- $exp = $1;
- } elsif (/<LIBRARY_NAME>\s*(\S+)\s*<\/LIBRARY_NAME>/i) {
- $exp2lib{$exp} = $1;
- }
- }
- close($fh);
- # read run
- my ($run, @fn);
- open($fh, "$pre.run.xml") || die;
- while (<$fh>) {
- if (/<RUN.*accession="([^\s"]+)"/i) {
- $run = $1; @fn = ();
- } elsif (/<EXPERIMENT_REF.*accession="([^\s"]+)"/i) {
- print "\@RG\tID:$run\tSM:$sample\tLB:$exp2lib{$1}\n";
- } elsif (/<FILE.*filename="([^\s"]+)"/i) {
- push(@fn, $1);
- } elsif (/<\/RUN>/i) {
- if (@fn == 1) {
- print STDERR "$fn[0]\t$run\n";
- } else {
- for (0 .. $#fn) {
- print STDERR "$fn[$_]\t$run", "_", $_+1, "\n";
- }
- }
- }
- }
- close($fh);
-}
-
-#
-# unique
-#
-
-sub unique {
- my %opts = (f=>250.0, q=>5, r=>2, a=>1, b=>3);
- getopts('Qf:q:r:a:b:m', \%opts);
- die("Usage: samtools.pl unique [-f $opts{f}] <in.sam>\n") if (@ARGV == 0 && -t STDIN);
- my $last = '';
- my $recal_Q = !defined($opts{Q});
- my $multi_only = defined($opts{m});
- my @a;
- while (<>) {
- my $score = -1;
- print $_ if (/^\@/);
- $score = $1 if (/AS:i:(\d+)/);
- my @t = split("\t");
- next if (@t < 11);
- if ($score < 0) { # AS tag is unavailable
- my $cigar = $t[5];
- my ($mm, $go, $ge) = (0, 0, 0);
- $cigar =~ s/(\d+)[ID]/++$go,$ge+=$1/eg;
- $cigar = $t[5];
- $cigar =~ s/(\d+)M/$mm+=$1/eg;
- $score = $mm * $opts{a} - $go * $opts{q} - $ge * $opts{r}; # no mismatches...
- }
- $score = 1 if ($score < 1);
- if ($t[0] ne $last) {
- &unique_aux(\@a, $opts{f}, $recal_Q, $multi_only) if (@a);
- $last = $t[0];
- }
- push(@a, [$score, \@t]);
- }
- &unique_aux(\@a, $opts{f}, $recal_Q, $multi_only) if (@a);
-}
-
-sub unique_aux {
- my ($a, $fac, $is_recal, $multi_only) = @_;
- my ($max, $max2, $max_i) = (0, 0, -1);
- for (my $i = 0; $i < @$a; ++$i) {
- if ($a->[$i][0] > $max) {
- $max2 = $max; $max = $a->[$i][0]; $max_i = $i;
- } elsif ($a->[$i][0] > $max2) {
- $max2 = $a->[$i][0];
- }
- }
- if ($is_recal) {
- if (!$multi_only || @$a > 1) {
- my $q = int($fac * ($max - $max2) / $max + .499);
- $q = 250 if ($q > 250);
- $a->[$max_i][1][4] = $q < 250? $q : 250;
- }
- }
- print join("\t", @{$a->[$max_i][1]});
- @$a = ();
-}
-
-#
-# uniqcmp: compare two SAM files
-#
-
-sub uniqcmp {
- my %opts = (q=>10, s=>100);
- getopts('pq:s:', \%opts);
- die("Usage: samtools.pl uniqcmp <in1.sam> <in2.sam>\n") if (@ARGV < 2);
- my ($fh, %a);
- warn("[uniqcmp] read the first file...\n");
- &uniqcmp_aux($ARGV[0], \%a, 0);
- warn("[uniqcmp] read the second file...\n");
- &uniqcmp_aux($ARGV[1], \%a, 1);
- warn("[uniqcmp] stats...\n");
- my @cnt;
- $cnt[$_] = 0 for (0..9);
- for my $x (keys %a) {
- my $p = $a{$x};
- my $z;
- if (defined($p->[0]) && defined($p->[1])) {
- $z = ($p->[0][0] == $p->[1][0] && $p->[0][1] eq $p->[1][1] && abs($p->[0][2] - $p->[1][2]) < $opts{s})? 0 : 1;
- if ($p->[0][3] >= $opts{q} && $p->[1][3] >= $opts{q}) {
- ++$cnt[$z*3+0];
- } elsif ($p->[0][3] >= $opts{q}) {
- ++$cnt[$z*3+1];
- } elsif ($p->[1][3] >= $opts{q}) {
- ++$cnt[$z*3+2];
- }
- print STDERR "$x\t$p->[0][1]:$p->[0][2]\t$p->[0][3]\t$p->[0][4]\t$p->[1][1]:$p->[1][2]\t$p->[1][3]\t$p->[1][4]\t",
- $p->[0][5]-$p->[1][5], "\n" if ($z && defined($opts{p}) && ($p->[0][3] >= $opts{q} || $p->[1][3] >= $opts{q}));
- } elsif (defined($p->[0])) {
- ++$cnt[$p->[0][3]>=$opts{q}? 6 : 7];
- print STDERR "$x\t$p->[0][1]:$p->[0][2]\t$p->[0][3]\t$p->[0][4]\t*\t0\t*\t",
- $p->[0][5], "\n" if (defined($opts{p}) && $p->[0][3] >= $opts{q});
- } else {
- print STDERR "$x\t*\t0\t*\t$p->[1][1]:$p->[1][2]\t$p->[1][3]\t$p->[1][4]\t",
- -$p->[1][5], "\n" if (defined($opts{p}) && $p->[1][3] >= $opts{q});
- ++$cnt[$p->[1][3]>=$opts{q}? 8 : 9];
- }
- }
- print "Consistent (high, high): $cnt[0]\n";
- print "Consistent (high, low ): $cnt[1]\n";
- print "Consistent (low , high): $cnt[2]\n";
- print "Inconsistent (high, high): $cnt[3]\n";
- print "Inconsistent (high, low ): $cnt[4]\n";
- print "Inconsistent (low , high): $cnt[5]\n";
- print "Second missing (high): $cnt[6]\n";
- print "Second missing (low ): $cnt[7]\n";
- print "First missing (high): $cnt[8]\n";
- print "First missing (low ): $cnt[9]\n";
-}
-
-sub uniqcmp_aux {
- my ($fn, $a, $which) = @_;
- my $fh;
- $fn = "samtools view $fn |" if ($fn =~ /\.bam/);
- open($fh, $fn) || die;
- while (<$fh>) {
- my @t = split;
- next if (@t < 11);
-# my $l = ($t[5] =~ /^(\d+)S/)? $1 : 0;
- my $l = 0;
- my ($x, $nm) = (0, 0);
- $nm = $1 if (/NM:i:(\d+)/);
- $_ = $t[5];
- s/(\d+)[MI]/$x+=$1/eg;
- @{$a->{$t[0]}[$which]} = (($t[1]&0x10)? 1 : 0, $t[2], $t[3]-$l, $t[4], "$x:$nm", $x - 4 * $nm);
- }
- close($fh);
-}
-
-sub plp2vcf {
- while (<>) {
- my @t = split;
- next if ($t[3] eq '*/*');
- if ($t[2] eq '*') { # indel
- my @s = split("/", $t[3]);
- my (@a, @b);
- my ($ref, $alt);
- for (@s) {
- next if ($_ eq '*');
- if (/^-/) {
- push(@a, 'N'.substr($_, 1));
- push(@b, 'N');
- } elsif (/^\+/) {
- push(@a, 'N');
- push(@b, 'N'.substr($_, 1));
- }
- }
- if ($a[0] && $a[1]) {
- if (length($a[0]) < length($a[1])) {
- $ref = $a[1];
- $alt = ($b[0] . ('N' x (length($a[1]) - length($a[0])))) . ",$b[1]";
- } elsif (length($a[0]) > length($a[1])) {
- $ref = $a[0];
- $alt = ($b[1] . ('N' x (length($a[0]) - length($a[1])))) . ",$b[0]";
- } else {
- $ref = $a[0];
- $alt = ($b[0] eq $b[1])? $b[0] : "$b[0],$b[1]";
- }
- } else {
- $ref = $a[0]; $alt = $b[0];
- }
- print join("\t", @t[0,1], '.', $ref, $alt, $t[5], '.', '.'), "\n";
- } else { # SNP
- }
- }
-}
-
-#
-# Usage
-#
-
-sub usage {
- die(qq/
-Program: samtools.pl (helper script for SAMtools)
-Version: $version
-Contact: Heng Li <lh3\@sanger.ac.uk>\n
-Usage: samtools.pl <command> [<arguments>]\n
-Command: varFilter filtering SNPs and short indels
- pileup2fq generate fastq from `pileup -c'
- showALEN print alignment length (ALEN) following CIGAR
-\n/);
-}
diff --git a/samtools-0.1.19/misc/soap2sam.pl b/samtools-0.1.19/misc/soap2sam.pl
deleted file mode 100755
index b37135e..0000000
--- a/samtools-0.1.19/misc/soap2sam.pl
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/perl -w
-
-# Contact: lh3
-# Version: 0.1.1
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-&soap2sam;
-exit;
-
-sub mating {
- my ($s1, $s2) = @_;
- my $isize = 0;
- if ($s1->[2] ne '*' && $s1->[2] eq $s2->[2]) { # then calculate $isize
- my $x1 = ($s1->[1] & 0x10)? $s1->[3] + length($s1->[9]) : $s1->[3];
- my $x2 = ($s2->[1] & 0x10)? $s2->[3] + length($s2->[9]) : $s2->[3];
- $isize = $x2 - $x1;
- }
- # update mate coordinate
- if ($s2->[2] ne '*') {
- @$s1[6..8] = (($s2->[2] eq $s1->[2])? "=" : $s2->[2], $s2->[3], $isize);
- $s1->[1] |= 0x20 if ($s2->[1] & 0x10);
- } else {
- $s1->[1] |= 0x8;
- }
- if ($s1->[2] ne '*') {
- @$s2[6..8] = (($s1->[2] eq $s2->[2])? "=" : $s1->[2], $s1->[3], -$isize);
- $s2->[1] |= 0x20 if ($s1->[1] & 0x10);
- } else {
- $s2->[1] |= 0x8;
- }
-}
-
-sub soap2sam {
- my %opts = ();
- getopts("p", \%opts);
- die("Usage: soap2sam.pl [-p] <aln.soap>\n") if (@ARGV == 0 && -t STDIN);
- my $is_paired = defined($opts{p});
- # core loop
- my @s1 = ();
- my @s2 = ();
- my ($s_last, $s_curr) = (\@s1, \@s2);
- while (<>) {
- s/[\177-\377]|[\000-\010]|[\012-\040]//g;
- next if (&soap2sam_aux($_, $s_curr, $is_paired) < 0);
- if (@$s_last != 0 && $s_last->[0] eq $s_curr->[0]) {
- &mating($s_last, $s_curr);
- print join("\t", @$s_last), "\n";
- print join("\t", @$s_curr), "\n";
- @$s_last = (); @$s_curr = ();
- } else {
- print join("\t", @$s_last), "\n" if (@$s_last != 0);
- my $s = $s_last; $s_last = $s_curr; $s_curr = $s;
- }
- }
- print join("\t", @$s_last), "\n" if (@$s_last != 0);
-}
-
-sub soap2sam_aux {
- my ($line, $s, $is_paired) = @_;
- chomp($line);
- my @t = split(/\s+/, $line);
- return -1 if (@t < 9 || $line =~ /^\s/ || !$t[0]);
- @$s = ();
- # fix SOAP-2.1.x bugs
- @t = @t[0..2,4..$#t] unless ($t[3] =~ /^\d+$/);
- # read name
- $s->[0] = $t[0];
- $s->[0] =~ s/\/[12]$//g;
- # initial flag (will be updated later)
- $s->[1] = 0;
- $s->[1] |= 1 | 1<<($t[4] eq 'a'? 6 : 7);
- $s->[1] |= 2 if ($is_paired);
- # read & quality
- $s->[9] = $t[1];
- $s->[10] = (length($t[2]) > length($t[1]))? substr($t[2], 0, length($t[1])) : $t[2];
- # cigar
- $s->[5] = length($s->[9]) . "M";
- # coor
- $s->[2] = $t[7]; $s->[3] = $t[8];
- $s->[1] |= 0x10 if ($t[6] eq '-');
- # mapQ
- $s->[4] = $t[3] == 1? 30 : 0;
- # mate coordinate
- $s->[6] = '*'; $s->[7] = $s->[8] = 0;
- # aux
- push(@$s, "NM:i:$t[9]");
- my $md = '';
- if ($t[9]) {
- my @x;
- for (10 .. $#t) {
- push(@x, sprintf("%.3d,$1", $2)) if ($t[$_] =~ /^([ACGT])->(\d+)/i);
- }
- @x = sort(@x);
- my $a = 0;
- for (@x) {
- my ($y, $z) = split(",");
- $md .= (int($y)-$a) . $z;
- $a += $y - $a + 1;
- }
- $md .= length($t[1]) - $a;
- } else {
- $md = length($t[1]);
- }
- push(@$s, "MD:Z:$md");
- return 0;
-}
diff --git a/samtools-0.1.19/misc/varfilter.py b/samtools-0.1.19/misc/varfilter.py
deleted file mode 100755
index 03ce395..0000000
--- a/samtools-0.1.19/misc/varfilter.py
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/software/bin/python
-
-# Author: lh3, converted to python and modified to add -C option by Aylwyn Scally
-#
-# About:
-# varfilter.py is a port of Heng's samtools.pl varFilter script into
-# python, with an additional -C INT option. This option sets a minimum
-# consensus score, above which the script will output a pileup line
-# wherever it _could have_ called a variant, even if none is actually
-# called (i.e. hom-ref positions). This is important if you want to
-# subsequently merge the calls with those for another individual to get a
-# synoptic view of calls at each site. Without this option, and in all
-# other respects, it behaves like samtools.pl varFilter.
-#
-# Aylwyn Scally as6 at sanger.ac.uk
-
-
-# Filtration code:
-#
-# C low CNS quality (hom-ref only)
-# d low depth
-# D high depth
-# W too many SNPs in a window (SNP only)
-# G close to a high-quality indel (SNP only)
-# Q low RMS mapping quality (SNP only)
-# g close to another indel with higher quality (indel only)
-# s low SNP quality (SNP only)
-# i low indel quality (indel only)
-
-
-import sys
-import getopt
-
-def usage():
- print '''usage: varfilter.py [options] [cns-pileup]
-
-Options: -Q INT minimum RMS mapping quality for SNPs
- -q INT minimum RMS mapping quality for gaps
- -d INT minimum read depth
- -D INT maximum read depth
- -S INT minimum SNP quality
- -i INT minimum indel quality
- -C INT minimum consensus quality for hom-ref sites
-
- -G INT min indel score for nearby SNP filtering
- -w INT SNP within INT bp around a gap to be filtered
-
- -W INT window size for filtering dense SNPs
- -N INT max number of SNPs in a window
-
- -l INT window size for filtering adjacent gaps
-
- -p print filtered variants'''
-
-def varFilter_aux(first, is_print):
- try:
- if first[1] == 0:
- sys.stdout.write("\t".join(first[4:]) + "\n")
- elif is_print:
- sys.stderr.write("\t".join(["UQdDWGgsiCX"[first[1]]] + first[4:]) + "\n")
- except IOError:
- sys.exit()
-
-mindepth = 3
-maxdepth = 100
-gapgapwin = 30
-minsnpmapq = 25
-mingapmapq = 10
-minindelscore = 25
-scorefactor = 100
-snpgapwin = 10
-densesnpwin = 10
-densesnps = 2
-printfilt = False
-minsnpq = 0
-minindelq = 0
-mincnsq = 0
-
-try:
- options, args = getopt.gnu_getopt(sys.argv[1:], 'pq:d:D:l:Q:w:W:N:G:S:i:C:', [])
-except getopt.GetoptError:
- usage()
- sys.exit(2)
-for (oflag, oarg) in options:
- if oflag == '-d': mindepth = int(oarg)
- if oflag == '-D': maxdepth = int(oarg)
- if oflag == '-l': gapgapwin = int(oarg)
- if oflag == '-Q': minsnpmapq = int(oarg)
- if oflag == '-q': mingapmapq = int(oarg)
- if oflag == '-G': minindelscore = int(oarg)
- if oflag == '-s': scorefactor = int(oarg)
- if oflag == '-w': snpgapwin = int(oarg)
- if oflag == '-W': densesnpwin = int(oarg)
- if oflag == '-C': mincnsq = int(oarg)
- if oflag == '-N': densesnps = int(oarg)
- if oflag == '-p': printfilt = True
- if oflag == '-S': minsnpq = int(oarg)
- if oflag == '-i': minindelq = int(oarg)
-
-if len(args) < 1:
- inp = sys.stdin
-else:
- inp = open(args[0])
-
-# calculate the window size
-max_dist = max(gapgapwin, snpgapwin, densesnpwin)
-
-staging = []
-for t in (line.strip().split() for line in inp):
- (flt, score) = (0, -1)
- # non-var sites
- if t[3] == '*/*':
- continue
- is_snp = t[2].upper() != t[3].upper()
- if not (is_snp or mincnsq):
- continue
- # clear the out-of-range elements
- while staging:
- # Still on the same chromosome and the first element's window still affects this position?
- if staging[0][4] == t[0] and int(staging[0][5]) + staging[0][2] + max_dist >= int(t[1]):
- break
- varFilter_aux(staging.pop(0), printfilt)
-
- # first a simple filter
- if int(t[7]) < mindepth:
- flt = 2
- elif int(t[7]) > maxdepth:
- flt = 3
- if t[2] == '*': # an indel
- if minindelq and minindelq > int(t[5]):
- flt = 8
- elif is_snp:
- if minsnpq and minsnpq> int(t[5]):
- flt = 7
- else:
- if mincnsq and mincnsq > int(t[4]):
- flt = 9
-
- # site dependent filters
- dlen = 0
- if flt == 0:
- if t[2] == '*': # an indel
- # If deletion, remember the length of the deletion
- (a,b) = t[3].split('/')
- alen = len(a) - 1
- blen = len(b) - 1
- if alen>blen:
- if a[0] == '-': dlen=alen
- elif b[0] == '-': dlen=blen
-
- if int(t[6]) < mingapmapq:
- flt = 1
- # filtering SNPs
- if int(t[5]) >= minindelscore:
- for x in (y for y in staging if y[3]):
- # Is it a SNP and is it outside the SNP filter window?
- if x[0] >= 0 or int(x[5]) + x[2] + snpgapwin < int(t[1]):
- continue
- if x[1] == 0:
- x[1] = 5
-
- # calculate the filtering score (different from indel quality)
- score = int(t[5])
- if t[8] != '*':
- score += scorefactor * int(t[10])
- if t[9] != '*':
- score += scorefactor * int(t[11])
- # check the staging list for indel filtering
- for x in (y for y in staging if y[3]):
- # Is it a SNP and is it outside the gap filter window
- if x[0] < 0 or int(x[5]) + x[2] + gapgapwin < int(t[1]):
- continue
- if x[0] < score:
- x[1] = 6
- else:
- flt = 6
- break
- else: # a SNP or hom-ref
- if int(t[6]) < minsnpmapq:
- flt = 1
- # check adjacent SNPs
- k = 1
- for x in (y for y in staging if y[3]):
- if x[0] < 0 and int(x[5]) + x[2] + densesnpwin >= int(t[1]) and (x[1] == 0 or x[1] == 4 or x[1] == 5):
- k += 1
-
- # filtering is necessary
- if k > densesnps:
- flt = 4
- for x in (y for y in staging if y[3]):
- if x[0] < 0 and int(x[5]) + x[2] + densesnpwin >= int(t[1]) and x[1] == 0:
- x[1] = 4
- else: # then check gap filter
- for x in (y for y in staging if y[3]):
- if x[0] < 0 or int(x[5]) + x[2] + snpgapwin < int(t[1]):
- continue
- if x[0] >= minindelscore:
- flt = 5
- break
-
- staging.append([score, flt, dlen, is_snp] + t)
-
-# output the last few elements in the staging list
-while staging:
- varFilter_aux(staging.pop(0), printfilt)
diff --git a/samtools-0.1.19/misc/vcfutils.lua b/samtools-0.1.19/misc/vcfutils.lua
deleted file mode 100755
index 51d374e..0000000
--- a/samtools-0.1.19/misc/vcfutils.lua
+++ /dev/null
@@ -1,694 +0,0 @@
-#!/usr/bin/env luajit
-
------------------------------------
--- BEGIN: routines from klib.lua --
------------------------------------
-
--- Description: getopt() translated from the BSD getopt(); compatible with the default Unix getopt()
---[[ Example:
- for o, a in os.getopt(arg, 'a:b') do
- print(o, a)
- end
-]]--
-function os.getopt(args, ostr)
- local arg, place = nil, 0;
- return function ()
- if place == 0 then -- update scanning pointer
- place = 1
- if #args == 0 or args[1]:sub(1, 1) ~= '-' then place = 0; return nil end
- if #args[1] >= 2 then
- place = place + 1
- if args[1]:sub(2, 2) == '-' then -- found "--"
- table.remove(args, 1);
- place = 0
- return nil;
- end
- end
- end
- local optopt = place <= #args[1] and args[1]:sub(place, place) or nil
- place = place + 1;
- local oli = optopt and ostr:find(optopt) or nil
- if optopt == ':' or oli == nil then -- unknown option
- if optopt == '-' then return nil end
- if place > #args[1] then
- table.remove(args, 1);
- place = 0;
- end
- return '?';
- end
- oli = oli + 1;
- if ostr:sub(oli, oli) ~= ':' then -- do not need argument
- arg = nil;
- if place > #args[1] then
- table.remove(args, 1);
- place = 0;
- end
- else -- need an argument
- if place <= #args[1] then -- no white space
- arg = args[1]:sub(place);
- else
- table.remove(args, 1);
- if #args == 0 then -- an option requiring argument is the last one
- place = 0;
- if ostr:sub(1, 1) == ':' then return ':' end
- return '?';
- else arg = args[1] end
- end
- table.remove(args, 1);
- place = 0;
- end
- return optopt, arg;
- end
-end
-
--- Description: string split
-function string:split(sep, n)
- local a, start = {}, 1;
- sep = sep or "%s+";
- repeat
- local b, e = self:find(sep, start);
- if b == nil then
- table.insert(a, self:sub(start));
- break
- end
- a[#a+1] = self:sub(start, b - 1);
- start = e + 1;
- if n and #a == n then
- table.insert(a, self:sub(start));
- break
- end
- until start > #self;
- return a;
-end
-
--- Description: smart file open
-function io.xopen(fn, mode)
- mode = mode or 'r';
- if fn == nil then return io.stdin;
- elseif fn == '-' then return (mode == 'r' and io.stdin) or io.stdout;
- elseif fn:sub(-3) == '.gz' then return (mode == 'r' and io.popen('gzip -dc ' .. fn, 'r')) or io.popen('gzip > ' .. fn, 'w');
- elseif fn:sub(-4) == '.bz2' then return (mode == 'r' and io.popen('bzip2 -dc ' .. fn, 'r')) or io.popen('bgzip2 > ' .. fn, 'w');
- else return io.open(fn, mode) end
-end
-
--- Description: log gamma function
--- Required by: math.lbinom()
--- Reference: AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245
-function math.lgamma(z)
- local x;
- x = 0.1659470187408462e-06 / (z+7);
- x = x + 0.9934937113930748e-05 / (z+6);
- x = x - 0.1385710331296526 / (z+5);
- x = x + 12.50734324009056 / (z+4);
- x = x - 176.6150291498386 / (z+3);
- x = x + 771.3234287757674 / (z+2);
- x = x - 1259.139216722289 / (z+1);
- x = x + 676.5203681218835 / z;
- x = x + 0.9999999999995183;
- return math.log(x) - 5.58106146679532777 - z + (z-0.5) * math.log(z+6.5);
-end
-
--- Description: regularized incomplete gamma function
--- Dependent on: math.lgamma()
---[[
- Formulas are taken from Wiki, with additional input from Numerical
- Recipes in C (for modified Lentz's algorithm) and AS245
- (http://lib.stat.cmu.edu/apstat/245).
-
- A good online calculator is available at:
-
- http://www.danielsoper.com/statcalc/calc23.aspx
-
- It calculates upper incomplete gamma function, which equals
- math.igamma(s,z,true)*math.exp(math.lgamma(s))
-]]--
-function math.igamma(s, z, complement)
-
- local function _kf_gammap(s, z)
- local sum, x = 1, 1;
- for k = 1, 100 do
- x = x * z / (s + k);
- sum = sum + x;
- if x / sum < 1e-14 then break end
- end
- return math.exp(s * math.log(z) - z - math.lgamma(s + 1.) + math.log(sum));
- end
-
- local function _kf_gammaq(s, z)
- local C, D, f, TINY;
- f = 1. + z - s; C = f; D = 0.; TINY = 1e-290;
- -- Modified Lentz's algorithm for computing continued fraction. See Numerical Recipes in C, 2nd edition, section 5.2
- for j = 1, 100 do
- local d;
- local a, b = j * (s - j), j*2 + 1 + z - s;
- D = b + a * D;
- if D < TINY then D = TINY end
- C = b + a / C;
- if C < TINY then C = TINY end
- D = 1. / D;
- d = C * D;
- f = f * d;
- if math.abs(d - 1) < 1e-14 then break end
- end
- return math.exp(s * math.log(z) - z - math.lgamma(s) - math.log(f));
- end
-
- if complement then
- return ((z <= 1 or z < s) and 1 - _kf_gammap(s, z)) or _kf_gammaq(s, z);
- else
- return ((z <= 1 or z < s) and _kf_gammap(s, z)) or (1 - _kf_gammaq(s, z));
- end
-end
-
-function math.brent(func, a, b, tol)
- local gold1, gold2, tiny, max_iter = 1.6180339887, 0.3819660113, 1e-20, 100
-
- local fa, fb = func(a, data), func(b, data)
- if fb > fa then -- swap, such that f(a) > f(b)
- a, b, fa, fb = b, a, fb, fa
- end
- local c = b + gold1 * (b - a)
- local fc = func(c) -- golden section extrapolation
- while fb > fc do
- local bound = b + 100.0 * (c - b) -- the farthest point where we want to go
- local r = (b - a) * (fb - fc)
- local q = (b - c) * (fb - fa)
- if math.abs(q - r) < tiny then -- avoid 0 denominator
- tmp = q > r and tiny or 0.0 - tiny
- else tmp = q - r end
- u = b - ((b - c) * q - (b - a) * r) / (2.0 * tmp) -- u is the parabolic extrapolation point
- if (b > u and u > c) or (b < u and u < c) then -- u lies between b and c
- fu = func(u)
- if fu < fc then -- (b,u,c) bracket the minimum
- a, b, fa, fb = b, u, fb, fu
- break
- elseif fu > fb then -- (a,b,u) bracket the minimum
- c, fc = u, fu
- break
- end
- u = c + gold1 * (c - b)
- fu = func(u) -- golden section extrapolation
- elseif (c > u and u > bound) or (c < u and u < bound) then -- u lies between c and bound
- fu = func(u)
- if fu < fc then -- fb > fc > fu
- b, c, u = c, u, c + gold1 * (c - b)
- fb, fc, fu = fc, fu, func(u)
- else -- (b,c,u) bracket the minimum
- a, b, c = b, c, u
- fa, fb, fc = fb, fc, fu
- break
- end
- elseif (u > bound and bound > c) or (u < bound and bound < c) then -- u goes beyond the bound
- u = bound
- fu = func(u)
- else -- u goes the other way around, use golden section extrapolation
- u = c + gold1 * (c - b)
- fu = func(u)
- end
- a, b, c = b, c, u
- fa, fb, fc = fb, fc, fu
- end
- if a > c then a, c = c, a end -- swap
-
- -- now, a<b<c, fa>fb and fb<fc, move on to Brent's algorithm
- local e, d = 0, 0
- local w, v, fw, fv
- w, v = b, b
- fw, fv = fb, fb
- for iter = 1, max_iter do
- local mid = 0.5 * (a + c)
- local tol1 = tol * math.abs(b) + tiny
- local tol2 = 2.0 * tol1
- if math.abs(b - mid) <= tol2 - 0.5 * (c - a) then return fb, b end -- found
- if math.abs(e) > tol1 then
- -- related to parabolic interpolation
- local r = (b - w) * (fb - fv)
- local q = (b - v) * (fb - fw)
- local p = (b - v) * q - (b - w) * r
- q = 2.0 * (q - r)
- if q > 0.0 then p = 0.0 - p
- else q = 0.0 - q end
- eold, e = e, d
- if math.abs(p) >= math.abs(0.5 * q * eold) or p <= q * (a - b) or p >= q * (c - b) then
- e = b >= mid and a - b or c - b
- d = gold2 * e
- else
- d, u = p / q, b + d -- actual parabolic interpolation happens here
- if u - a < tol2 or c - u < tol2 then
- d = mid > b and tol1 or 0.0 - tol1
- end
- end
- else -- golden section interpolation
- e = b >= min and a - b or c - b
- d = gold2 * e
- end
- u = fabs(d) >= tol1 and b + d or b + (d > 0.0 and tol1 or -tol1);
- fu = func(u)
- if fu <= fb then -- u is the minimum point so far
- if u >= b then a = b
- else c = b end
- v, w, b = w, b, u
- fv, fw, fb = fw, fb, fu
- else -- adjust (a,c) and (u,v,w)
- if u < b then a = u
- else c = u end
- if fu <= fw or w == b then
- v, w = w, u
- fv, fw = fw, fu
- elseif fu <= fv or v == b or v == w then
- v, fv = u, fu;
- end
- end
- end
- return fb, b
-end
-
-matrix = {}
-
--- Description: chi^2 test for contingency tables
--- Dependent on: math.igamma()
-function matrix.chi2(a)
- if #a == 2 and #a[1] == 2 then -- 2x2 table
- local x, z
- x = (a[1][1] + a[1][2]) * (a[2][1] + a[2][2]) * (a[1][1] + a[2][1]) * (a[1][2] + a[2][2])
- if x == 0 then return 0, 1, false end
- z = a[1][1] * a[2][2] - a[1][2] * a[2][1]
- z = (a[1][1] + a[1][2] + a[2][1] + a[2][2]) * z * z / x
- return z, math.igamma(.5, .5 * z, true), true
- else -- generic table
- local rs, cs, n, m, N, z = {}, {}, #a, #a[1], 0, 0
- for i = 1, n do rs[i] = 0 end
- for j = 1, m do cs[j] = 0 end
- for i = 1, n do -- compute column sum and row sum
- for j = 1, m do cs[j], rs[i] = cs[j] + a[i][j], rs[i] + a[i][j] end
- end
- for i = 1, n do N = N + rs[i] end
- for i = 1, n do -- compute the chi^2 statistics
- for j = 1, m do
- local E = rs[i] * cs[j] / N;
- z = z + (a[i][j] - E) * (a[i][j] - E) / E
- end
- end
- return z, math.igamma(.5 * (n-1) * (m-1), .5 * z, true), true;
- end
-end
-
----------------------------------
--- END: routines from klib.lua --
----------------------------------
-
-
---------------------------
--- BEGIN: misc routines --
---------------------------
-
--- precompute an array for PL->probability conversion
--- @param m maximum PL
-function algo_init_q2p(m)
- local q2p = {}
- for i = 0, m do
- q2p[i] = math.pow(10, -i / 10)
- end
- return q2p
-end
-
--- given the haplotype frequency, compute r^2
--- @param f 4 haplotype frequencies; f[] is 0-indexed.
--- @return r^2
-function algo_r2(f)
- local p = { f[0] + f[1], f[0] + f[2] }
- local D = f[0] * f[3] - f[1] * f[2]
- return (p[1] == 0 or p[2] == 0 or 1-p[1] == 0 or 1-p[2] == 0) and 0 or D * D / (p[1] * p[2] * (1 - p[1]) * (1 - p[2]))
-end
-
--- parse a VCF line to get PL
--- @param q2p is computed by algo_init_q2p()
-function text_parse_pl(t, q2p, parse_GT)
- parse_GT = parse_GT == nil and true or false
- local ht, gt, pl = {}, {}, {}
- local s, j0 = t[9]:split(':'), 0
- for j = 1, #s do
- if s[j] == 'PL' then j0 = j break end
- end
- local has_GT = (s[1] == 'GT' and parse_GT) and true or false
- for i = 10, #t do
- if j0 > 0 then
- local s = t[i]:split(':')
- local a, b = 1, s[j0]:find(',')
- pl[#pl+1] = q2p[tonumber(s[j0]:sub(a, b - 1))]
- a, b = b + 1, s[j0]:find(',', b + 1)
- pl[#pl+1] = q2p[tonumber(s[j0]:sub(a, b - 1))]
- a, b = b + 1, s[j0]:find(',', b + 1)
- pl[#pl+1] = q2p[tonumber(s[j0]:sub(a, (b and b - 1) or nil))]
- end
- if has_GT then
- if t[i]:sub(1, 1) ~= '.' then
- local g = tonumber(t[i]:sub(1, 1)) + tonumber(t[i]:sub(3, 3));
- gt[#gt+1] = 1e-6; gt[#gt+1] = 1e-6; gt[#gt+1] = 1e-6
- gt[#gt - 2 + g] = 1
- ht[#ht+1] = tonumber(t[i]:sub(1, 1)); ht[#ht+1] = tonumber(t[i]:sub(3, 3));
- else
- gt[#gt+1] = 1; gt[#gt+1] = 1; gt[#gt+1] = 1
- ht[#ht+1] = -1; ht[#ht+1] = -1;
- end
- end
--- print(t[i], pl[#pl-2], pl[#pl-1], pl[#pl], gt[#gt-2], gt[#gt-1], gt[#gt])
- end
- if #pl == 0 then pl = nil end
- local x = has_GT and { t[1], t[2], ht, gt, pl } or { t[1], t[2], nil, nil, pl }
- return x
-end
-
--- Infer haplotype frequency
--- @param pdg genotype likelihoods P(D|g) generated by text_parse_pl(). pdg[] is 1-indexed.
--- @param eps precision [1e-5]
--- @return 2-locus haplotype frequencies, 0-indexed array
-function algo_hapfreq2(pdg, eps)
- eps = eps or 1e-5
- local n, f = #pdg[1] / 3, {[0]=0.25, 0.25, 0.25, 0.25}
- for iter = 1, 100 do
- local F = {[0]=0, 0, 0, 0}
- for i = 0, n - 1 do
- local p1, p2 = {[0]=pdg[1][i*3+1], pdg[1][i*3+2], pdg[1][i*3+3]}, {[0]=pdg[2][i*3+1], pdg[2][i*3+2], pdg[2][i*3+3]}
- local u = { [0]=
- f[0] * (f[0] * p1[0] * p2[0] + f[1] * p1[0] * p2[1] + f[2] * p1[1] * p2[0] + f[3] * p1[1] * p2[1]),
- f[1] * (f[0] * p1[0] * p2[1] + f[1] * p1[0] * p2[2] + f[2] * p1[1] * p2[1] + f[3] * p1[1] * p2[2]),
- f[2] * (f[0] * p1[1] * p2[0] + f[1] * p1[1] * p2[1] + f[2] * p1[2] * p2[0] + f[3] * p1[2] * p2[1]),
- f[3] * (f[0] * p1[1] * p2[1] + f[1] * p1[1] * p2[2] + f[2] * p1[2] * p2[1] + f[3] * p1[2] * p2[2])
- }
- local s = u[0] + u[1] + u[2] + u[3]
- s = 1 / (s * n)
- F[0] = F[0] + u[0] * s
- F[1] = F[1] + u[1] * s
- F[2] = F[2] + u[2] * s
- F[3] = F[3] + u[3] * s
- end
- local e = 0
- for k = 0, 3 do
- e = math.abs(f[k] - F[k]) > e and math.abs(f[k] - F[k]) or e
- end
- for k = 0, 3 do f[k] = F[k] end
- if e < eps then break end
--- print(f[0], f[1], f[2], f[3])
- end
- return f
-end
-
-------------------------
--- END: misc routines --
-------------------------
-
-
----------------------
--- BEGIN: commands --
----------------------
-
--- CMD vcf2bgl: convert PL tagged VCF to Beagle input --
-function cmd_vcf2bgl()
- if #arg == 0 then
- print("\nUsage: vcf2bgl.lua <in.vcf>")
- print("\nNB: This command finds PL by matching /(\\d+),(\\d+),(\\d+)/.\n");
- os.exit(1)
- end
-
- local lookup = {}
- for i = 0, 10000 do lookup[i] = string.format("%.4f", math.pow(10, -i/10)) end
-
- local fp = io.xopen(arg[1])
- for l in fp:lines() do
- if l:sub(1, 2) == '##' then -- meta lines; do nothing
- elseif l:sub(1, 1) == '#' then -- sample lines
- local t, s = l:split('\t'), {}
- for i = 10, #t do s[#s+1] = t[i]; s[#s+1] = t[i]; s[#s+1] = t[i] end
- print('marker', 'alleleA', 'alleleB', table.concat(s, '\t'))
- else -- data line
- local t = l:split('\t');
- if t[5] ~= '.' and t[5]:find(",") == nil and #t[5] == 1 and #t[4] == 1 then -- biallic SNP
- local x, z = -1, {};
- if t[9]:find('PL') then
- for i = 10, #t do
- local AA, Aa, aa = t[i]:match('(%d+),(%d+),(%d+)')
- AA = tonumber(AA); Aa = tonumber(Aa); aa = tonumber(aa);
- if AA ~= nil then
- z[#z+1] = lookup[AA]; z[#z+1] = lookup[Aa]; z[#z+1] = lookup[aa];
- else z[#z+1] = 1; z[#z+1] = 1; z[#z+1] = 1; end
- end
- print(t[1]..':'..t[2], t[4], t[5], table.concat(z, '\t'))
- elseif t[9]:find('GL') then
- print('Error: not implemented')
- os.exit(1)
- end
- end
- end
- end
- fp:close()
-end
-
--- CMD bgl2vcf: convert Beagle output to VCF
-function cmd_bgl2vcf()
- if #arg < 2 then
- print('Usage: bgl2vcf.lua <in.phased> <in.gprobs>')
- os.exit(1)
- end
-
- local fpp = io.xopen(arg[1]);
- local fpg = io.xopen(arg[2]);
- for lg in fpg:lines() do
- local tp, tg, a = fpp:read():split('%s'), lg:split('%s', 4), {}
- if tp[1] == 'I' then
- for i = 3, #tp, 2 do a[#a+1] = tp[i] end
- print('#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT', table.concat(a, '\t'))
- else
- local chr, pos = tg[1]:match('(%S+):(%d+)$')
- a = {chr, pos, '.', tg[2], tg[3], 30, '.', '.', 'GT'}
- for i = 3, #tp, 2 do
- a[#a+1] = ((tp[i] == tg[2] and 0) or 1) .. '|' .. ((tp[i+1] == tg[2] and 0) or 1)
- end
- print(table.concat(a, '\t'))
- end
- end
- fpg:close(); fpp:close();
-end
-
--- CMD freq: count alleles in each population
-function cmd_freq()
- -- parse the command line
- local site_only = true; -- print site allele frequency or not
- for c in os.getopt(arg, 's') do
- if c == 's' then site_only = false end
- end
- if #arg == 0 then
- print("\nUsage: vcfutils.lua freq [-s] <in.vcf> [samples.txt]\n")
- print("NB: 1) This command only considers biallelic variants.")
- print(" 2) Apply '-s' to get the allele frequency spectrum.")
- print(" 3) 'samples.txt' is TAB-delimited with each line consisting of sample and population.")
- print("")
- os.exit(1)
- end
-
- -- read the sample-population pairs
- local pop, sample = {}, {}
- if #arg > 1 then
- local fp = io.xopen(arg[2]);
- for l in fp:lines() do
- local s, p = l:match("^(%S+)%s+(%S+)"); -- sample, population pair
- sample[s] = p; -- FIXME: check duplications
- if pop[p] then table.insert(pop[p], s)
- else pop[p] = {s} end
- end
- fp:close();
- end
- pop['NA'] = {}
-
- -- parse VCF
- fp = (#arg >= 2 and io.xopen(arg[1])) or io.stdin;
- local col, cnt = {}, {};
- for k in pairs(pop) do
- col[k], cnt[k] = {}, {[0]=0};
- end
- for l in fp:lines() do
- if l:sub(1, 2) == '##' then -- meta lines; do nothing
- elseif l:sub(1, 1) == '#' then -- the sample line
- local t, del_NA = l:split('\t'), true;
- for i = 10, #t do
- local k = sample[t[i]]
- if k == nil then
- k, del_NA = 'NA', false
- table.insert(pop[k], t[i])
- end
- table.insert(col[k], i);
- table.insert(cnt[k], 0);
- table.insert(cnt[k], 0);
- end
- if del_NA then pop['NA'], col['NA'], cnt['NA'] = nil, nil, nil end
- else -- data lines
- local t = l:split('\t');
- if t[5] ~= '.' and t[5]:find(",") == nil then -- biallic
- if site_only == true then io.write(t[1], '\t', t[2], '\t', t[4], '\t', t[5]) end
- for k, v in pairs(col) do
- local ac, an = 0, 0;
- for i = 1, #v do
- local a1, a2 = t[v[i]]:match("^(%d).(%d)");
- if a1 ~= nil then ac, an = ac + a1 + a2, an + 2 end
- end
- if site_only == true then io.write('\t', k, ':', an, ':', ac) end
- if an == #cnt[k] then cnt[k][ac] = cnt[k][ac] + 1 end
- end
- if site_only == true then io.write('\n') end
- end
- end
- end
- fp:close();
-
- -- print
- if site_only == false then
- for k, v in pairs(cnt) do
- io.write(k .. "\t" .. #v);
- for i = 0, #v do io.write("\t" .. v[i]) end
- io.write('\n');
- end
- end
-end
-
-function cmd_vcf2chi2()
- if #arg < 3 then
- print("Usage: vcfutils.lua vcf2chi2 <in.vcf> <group1.list> <group2.list>");
- os.exit(1)
- end
-
- local g = {};
-
- -- read the list of groups
- local fp = io.xopen(arg[2]);
- for l in fp:lines() do local x = l:match("^(%S+)"); g[x] = 1 end -- FIXME: check duplicate
- fp:close()
- fp = io.xopen(arg[3]);
- for l in fp:lines() do local x = l:match("^(%S+)"); g[x] = 2 end
- fp:close()
-
- -- process VCF
- fp = io.xopen(arg[1])
- local h = {{}, {}}
- for l in fp:lines() do
- if l:sub(1, 2) == '##' then print(l) -- meta lines; do nothing
- elseif l:sub(1, 1) == '#' then -- sample lines
- local t = l:split('\t');
- for i = 10, #t do
- if g[t[i]] == 1 then table.insert(h[1], i)
- elseif g[t[i]] == 2 then table.insert(h[2], i) end
- end
- while #t > 8 do table.remove(t) end
- print(table.concat(t, "\t"))
- else -- data line
- local t = l:split('\t');
- if t[5] ~= '.' and t[5]:find(",") == nil then -- biallic
- local a = {{0, 0}, {0, 0}}
- for i = 1, 2 do
- for _, k in pairs(h[i]) do
- if t[k]:find("^0.0") then a[i][1] = a[i][1] + 2
- elseif t[k]:find("^1.1") then a[i][2] = a[i][2] + 2
- elseif t[k]:find("^0.1") or t[k]:find("^1.0") then
- a[i][1], a[i][2] = a[i][1] + 1, a[i][2] + 1
- end
- end
- end
- local chi2, p, succ = matrix.chi2(a);
- while #t > 8 do table.remove(t) end
- --print(a[1][1], a[1][2], a[2][1], a[2][2], chi2, p);
- if succ then print(table.concat(t, "\t") .. ";PCHI2=" .. string.format("%.3g", p)
- .. string.format(';AF1=%.4g;AF2=%.4g,%.4g', (a[1][2]+a[2][2]) / (a[1][1]+a[1][2]+a[2][1]+a[2][2]),
- a[1][2]/(a[1][1]+a[1][2]), a[2][2]/(a[2][1]+a[2][2])))
- else print(table.concat(t, "\t")) end
- end
- end
- end
- fp:close()
-end
-
--- CMD: compute r^2
-function cmd_r2()
- local w, is_ht, is_gt = 1, false, false
- for o, a in os.getopt(arg, 'w:hg') do
- if o == 'w' then w = tonumber(a)
- elseif o == 'h' then is_ht, is_gt = true, true
- elseif o == 'g' then is_gt = true
- end
- end
- if #arg == 0 then
- print("Usage: vcfutils.lua r2 [-hg] [-w 1] <in.vcf>")
- os.exit(1)
- end
- local stack, fp, q2p = {}, io.xopen(arg[1]), algo_init_q2p(1023)
- for l in fp:lines() do
- if l:sub(1, 1) ~= '#' then
- local t = l:split('\t')
- local x = text_parse_pl(t, q2p)
- if #t[5] == 1 and t[5] ~= '.' then -- biallelic
- local r2 = {}
- for k = 1, w do
- if is_gt == false then -- use PL
- if stack[k] then
- local pdg = { stack[k][5], x[5] }
- r2[#r2+1] = algo_r2(algo_hapfreq2(pdg))
- else r2[#r2+1] = 0 end
- elseif is_ht == false then -- use unphased GT
- if stack[k] then
- local pdg = { stack[k][4], x[4] }
- r2[#r2+1] = algo_r2(algo_hapfreq2(pdg))
- else r2[#r2+1] = 0 end
- else -- use phased GT
- if stack[k] then
- local f, ht = { [0]=0, 0, 0, 0 }, { stack[k][3], x[3] }
- for i = 1, #ht[1] do
- local j = ht[1][i] * 2 + ht[2][i]
- f[j] = f[j] + 1
- end
- local sum = f[0] + f[1] + f[2] + f[3]
- for k = 0, 3 do f[k] = f[k] / sum end
- r2[#r2+1] = algo_r2(f)
- else r2[#r2+1] = 0 end
- end
- end
- for k = 1, #r2 do
- r2[k] = string.format('%.3f', r2[k])
- end
- print(x[1], x[2], table.concat(r2, '\t'))
- if #stack == w then table.remove(stack, 1) end
- stack[#stack+1] = x
- end
- end
- end
- fp:close()
-end
-
--------------------
--- END: commands --
--------------------
-
-
--------------------
--- MAIN FUNCTION --
--------------------
-
-if #arg == 0 then
- print("\nUsage: vcfutils.lua <command> <arguments>\n")
- print("Command: freq count biallelic alleles in each population")
- print(" r2 compute r^2")
- print(" vcf2chi2 compute 1-degree chi-square between two groups of samples")
- print(" vcf2bgl convert PL annotated VCF to Beagle input")
- print(" bgl2vcf convert Beagle input to VCF")
- print("")
- os.exit(1)
-end
-
-local cmd = arg[1]
-table.remove(arg, 1)
-if cmd == 'vcf2bgl' then cmd_vcf2bgl()
-elseif cmd == 'bgl2vcf' then cmd_bgl2vcf()
-elseif cmd == 'freq' then cmd_freq()
-elseif cmd == 'r2' then cmd_r2()
-elseif cmd == 'vcf2chi2' then cmd_vcf2chi2()
-else
- print('ERROR: unknown command "' .. cmd .. '"')
- os.exit(1)
-end
diff --git a/samtools-0.1.19/misc/wgsim.c b/samtools-0.1.19/misc/wgsim.c
deleted file mode 100644
index b9c513c..0000000
--- a/samtools-0.1.19/misc/wgsim.c
+++ /dev/null
@@ -1,419 +0,0 @@
-/* The MIT License
-
- Copyright (c) 2008 Genome Research Ltd (GRL).
- 2011 Heng Li <lh3 at live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-/* This program is separated from maq's read simulator with Colin
- * Hercus' modification to allow longer indels. */
-
-#include <stdlib.h>
-#include <math.h>
-#include <time.h>
-#include <assert.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <ctype.h>
-#include <string.h>
-#include <zlib.h>
-#include "kseq.h"
-KSEQ_INIT(gzFile, gzread)
-
-#define PACKAGE_VERSION "0.3.0"
-
-const uint8_t nst_nt4_table[256] = {
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5 /*'-'*/, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
-};
-
-/* Simple normal random number generator, copied from genran.c */
-
-double ran_normal()
-{
- static int iset = 0;
- static double gset;
- double fac, rsq, v1, v2;
- if (iset == 0) {
- do {
- v1 = 2.0 * drand48() - 1.0;
- v2 = 2.0 * drand48() - 1.0;
- rsq = v1 * v1 + v2 * v2;
- } while (rsq >= 1.0 || rsq == 0.0);
- fac = sqrt(-2.0 * log(rsq) / rsq);
- gset = v1 * fac;
- iset = 1;
- return v2 * fac;
- } else {
- iset = 0;
- return gset;
- }
-}
-
-/* wgsim */
-
-enum muttype_t {NOCHANGE = 0, INSERT = 0x1000, SUBSTITUTE = 0xe000, DELETE = 0xf000};
-typedef unsigned short mut_t;
-static mut_t mutmsk = (mut_t)0xf000;
-
-typedef struct {
- int l, m; /* length and maximum buffer size */
- mut_t *s; /* sequence */
-} mutseq_t;
-
-static double ERR_RATE = 0.02;
-static double MUT_RATE = 0.001;
-static double INDEL_FRAC = 0.15;
-static double INDEL_EXTEND = 0.3;
-static double MAX_N_RATIO = 0.1;
-
-void wgsim_mut_diref(const kseq_t *ks, int is_hap, mutseq_t *hap1, mutseq_t *hap2)
-{
- int i, deleting = 0;
- mutseq_t *ret[2];
-
- ret[0] = hap1; ret[1] = hap2;
- ret[0]->l = ks->seq.l; ret[1]->l = ks->seq.l;
- ret[0]->m = ks->seq.m; ret[1]->m = ks->seq.m;
- ret[0]->s = (mut_t *)calloc(ks->seq.m, sizeof(mut_t));
- ret[1]->s = (mut_t *)calloc(ks->seq.m, sizeof(mut_t));
- for (i = 0; i != ks->seq.l; ++i) {
- int c;
- c = ret[0]->s[i] = ret[1]->s[i] = (mut_t)nst_nt4_table[(int)ks->seq.s[i]];
- if (deleting) {
- if (drand48() < INDEL_EXTEND) {
- if (deleting & 1) ret[0]->s[i] |= DELETE;
- if (deleting & 2) ret[1]->s[i] |= DELETE;
- continue;
- } else deleting = 0;
- }
- if (c < 4 && drand48() < MUT_RATE) { // mutation
- if (drand48() >= INDEL_FRAC) { // substitution
- double r = drand48();
- c = (c + (int)(r * 3.0 + 1)) & 3;
- if (is_hap || drand48() < 0.333333) { // hom
- ret[0]->s[i] = ret[1]->s[i] = SUBSTITUTE|c;
- } else { // het
- ret[drand48()<0.5?0:1]->s[i] = SUBSTITUTE|c;
- }
- } else { // indel
- if (drand48() < 0.5) { // deletion
- if (is_hap || drand48() < 0.333333) { // hom-del
- ret[0]->s[i] = ret[1]->s[i] = DELETE;
- deleting = 3;
- } else { // het-del
- deleting = drand48()<0.5?1:2;
- ret[deleting-1]->s[i] = DELETE;
- }
- } else { // insertion
- int num_ins = 0, ins = 0;
- do {
- num_ins++;
- ins = (ins << 2) | (int)(drand48() * 4.0);
- } while (num_ins < 4 && drand48() < INDEL_EXTEND);
-
- if (is_hap || drand48() < 0.333333) { // hom-ins
- ret[0]->s[i] = ret[1]->s[i] = (num_ins << 12) | (ins << 4) | c;
- } else { // het-ins
- ret[drand48()<0.5?0:1]->s[i] = (num_ins << 12) | (ins << 4) | c;
- }
- }
- }
- }
- }
-}
-void wgsim_print_mutref(const char *name, const kseq_t *ks, mutseq_t *hap1, mutseq_t *hap2)
-{
- int i;
- for (i = 0; i != ks->seq.l; ++i) {
- int c[3];
- c[0] = nst_nt4_table[(int)ks->seq.s[i]];
- c[1] = hap1->s[i]; c[2] = hap2->s[i];
- if (c[0] >= 4) continue;
- if ((c[1] & mutmsk) != NOCHANGE || (c[2] & mutmsk) != NOCHANGE) {
- printf("%s\t%d\t", name, i+1);
- if (c[1] == c[2]) { // hom
- if ((c[1]&mutmsk) == SUBSTITUTE) { // substitution
- printf("%c\t%c\t-\n", "ACGTN"[c[0]], "ACGTN"[c[1]&0xf]);
- } else if ((c[1]&mutmsk) == DELETE) { // del
- printf("%c\t-\t-\n", "ACGTN"[c[0]]);
- } else if (((c[1] & mutmsk) >> 12) <= 5) { // ins
- printf("-\t");
- int n = (c[1]&mutmsk) >> 12, ins = c[1] >> 4;
- while (n > 0) {
- putchar("ACGTN"[ins & 0x3]);
- ins >>= 2;
- n--;
- }
- printf("\t-\n");
- } else assert(0);
- } else { // het
- if ((c[1]&mutmsk) == SUBSTITUTE || (c[2]&mutmsk) == SUBSTITUTE) { // substitution
- printf("%c\t%c\t+\n", "ACGTN"[c[0]], "XACMGRSVTWYHKDBN"[1<<(c[1]&0x3)|1<<(c[2]&0x3)]);
- } else if ((c[1]&mutmsk) == DELETE) {
- printf("%c\t-\t+\n", "ACGTN"[c[0]]);
- } else if ((c[2]&mutmsk) == DELETE) {
- printf("%c\t-\t+\n", "ACGTN"[c[0]]);
- } else if (((c[1] & mutmsk) >> 12) <= 4) { // ins1
- printf("-\t");
- int n = (c[1]&mutmsk) >> 12, ins = c[1] >> 4;
- while (n > 0) {
- putchar("ACGTN"[ins & 0x3]);
- ins >>= 2;
- n--;
- }
- printf("\t+\n");
- } else if (((c[2] & mutmsk) >> 12) <= 5) { // ins2
- printf("-\t");
- int n = (c[2]&mutmsk) >> 12, ins = c[2] >> 4;
- while (n > 0) {
- putchar("ACGTN"[ins & 0x3]);
- ins >>= 2;
- n--;
- }
- printf("\t+\n");
- } else assert(0);
- }
- }
- }
-}
-
-void wgsim_core(FILE *fpout1, FILE *fpout2, const char *fn, int is_hap, uint64_t N, int dist, int std_dev, int size_l, int size_r)
-{
- kseq_t *ks;
- mutseq_t rseq[2];
- gzFile fp_fa;
- uint64_t tot_len, ii;
- int i, l, n_ref;
- char *qstr;
- int size[2], Q, max_size;
- uint8_t *tmp_seq[2];
- mut_t *target;
-
- l = size_l > size_r? size_l : size_r;
- qstr = (char*)calloc(l+1, 1);
- tmp_seq[0] = (uint8_t*)calloc(l+2, 1);
- tmp_seq[1] = (uint8_t*)calloc(l+2, 1);
- size[0] = size_l; size[1] = size_r;
- max_size = size_l > size_r? size_l : size_r;
-
- Q = (ERR_RATE == 0.0)? 'I' : (int)(-10.0 * log(ERR_RATE) / log(10.0) + 0.499) + 33;
-
- fp_fa = gzopen(fn, "r");
- ks = kseq_init(fp_fa);
- tot_len = n_ref = 0;
- fprintf(stderr, "[%s] calculating the total length of the reference sequence...\n", __func__);
- while ((l = kseq_read(ks)) >= 0) {
- tot_len += l;
- ++n_ref;
- }
- fprintf(stderr, "[%s] %d sequences, total length: %llu\n", __func__, n_ref, (long long)tot_len);
- kseq_destroy(ks);
- gzclose(fp_fa);
-
- fp_fa = gzopen(fn, "r");
- ks = kseq_init(fp_fa);
- while ((l = kseq_read(ks)) >= 0) {
- uint64_t n_pairs = (uint64_t)((long double)l / tot_len * N + 0.5);
- if (l < dist + 3 * std_dev) {
- fprintf(stderr, "[%s] skip sequence '%s' as it is shorter than %d!\n", __func__, ks->name.s, dist + 3 * std_dev);
- continue;
- }
-
- // generate mutations and print them out
- wgsim_mut_diref(ks, is_hap, rseq, rseq+1);
- wgsim_print_mutref(ks->name.s, ks, rseq, rseq+1);
-
- for (ii = 0; ii != n_pairs; ++ii) { // the core loop
- double ran;
- int d, pos, s[2], is_flip = 0;
- int n_sub[2], n_indel[2], n_err[2], ext_coor[2], j, k;
- FILE *fpo[2];
-
- do { // avoid boundary failure
- ran = ran_normal();
- ran = ran * std_dev + dist;
- d = (int)(ran + 0.5);
- d = d > max_size? d : max_size;
- pos = (int)((l - d + 1) * drand48());
- } while (pos < 0 || pos >= ks->seq.l || pos + d - 1 >= ks->seq.l);
-
- // flip or not
- if (drand48() < 0.5) {
- fpo[0] = fpout1; fpo[1] = fpout2;
- s[0] = size[0]; s[1] = size[1];
- } else {
- fpo[1] = fpout1; fpo[0] = fpout2;
- s[1] = size[0]; s[0] = size[1];
- is_flip = 1;
- }
-
- // generate the read sequences
- target = rseq[drand48()<0.5?0:1].s; // haplotype from which the reads are generated
- n_sub[0] = n_sub[1] = n_indel[0] = n_indel[1] = n_err[0] = n_err[1] = 0;
-
-#define __gen_read(x, start, iter) do { \
- for (i = (start), k = 0, ext_coor[x] = -10; i >= 0 && i < ks->seq.l && k < s[x]; iter) { \
- int c = target[i], mut_type = c & mutmsk; \
- if (ext_coor[x] < 0) { \
- if (mut_type != NOCHANGE && mut_type != SUBSTITUTE) continue; \
- ext_coor[x] = i; \
- } \
- if (mut_type == DELETE) ++n_indel[x]; \
- else if (mut_type == NOCHANGE || mut_type == SUBSTITUTE) { \
- tmp_seq[x][k++] = c & 0xf; \
- if (mut_type == SUBSTITUTE) ++n_sub[x]; \
- } else { \
- int n, ins; \
- ++n_indel[x]; \
- tmp_seq[x][k++] = c & 0xf; \
- for (n = mut_type>>12, ins = c>>4; n > 0 && k < s[x]; --n, ins >>= 2) \
- tmp_seq[x][k++] = ins & 0x3; \
- } \
- } \
- if (k != s[x]) ext_coor[x] = -10; \
- } while (0)
-
- __gen_read(0, pos, ++i);
- __gen_read(1, pos + d - 1, --i);
- for (k = 0; k < s[1]; ++k) tmp_seq[1][k] = tmp_seq[1][k] < 4? 3 - tmp_seq[1][k] : 4; // complement
- if (ext_coor[0] < 0 || ext_coor[1] < 0) { // fail to generate the read(s)
- --ii;
- continue;
- }
-
- // generate sequencing errors
- for (j = 0; j < 2; ++j) {
- int n_n = 0;
- for (i = 0; i < s[j]; ++i) {
- int c = tmp_seq[j][i];
- if (c >= 4) { // actually c should be never larger than 4 if everything is correct
- c = 4;
- ++n_n;
- } else if (drand48() < ERR_RATE) {
- // c = (c + (int)(drand48() * 3.0 + 1)) & 3; // random sequencing errors
- c = (c + 1) & 3; // recurrent sequencing errors
- ++n_err[j];
- }
- tmp_seq[j][i] = c;
- }
- if ((double)n_n / s[j] > MAX_N_RATIO) break;
- }
- if (j < 2) { // too many ambiguous bases on one of the reads
- --ii;
- continue;
- }
-
- // print
- for (j = 0; j < 2; ++j) {
- for (i = 0; i < s[j]; ++i) qstr[i] = Q;
- qstr[i] = 0;
- fprintf(fpo[j], "@%s_%u_%u_%d:%d:%d_%d:%d:%d_%llx/%d\n", ks->name.s, ext_coor[0]+1, ext_coor[1]+1,
- n_err[0], n_sub[0], n_indel[0], n_err[1], n_sub[1], n_indel[1],
- (long long)ii, j==0? is_flip+1 : 2-is_flip);
- for (i = 0; i < s[j]; ++i)
- fputc("ACGTN"[(int)tmp_seq[j][i]], fpo[j]);
- fprintf(fpo[j], "\n+\n%s\n", qstr);
- }
- }
- free(rseq[0].s); free(rseq[1].s);
- }
- kseq_destroy(ks);
- gzclose(fp_fa);
- free(qstr);
- free(tmp_seq[0]); free(tmp_seq[1]);
-}
-
-static int simu_usage()
-{
- fprintf(stderr, "\n");
- fprintf(stderr, "Program: wgsim (short read simulator)\n");
- fprintf(stderr, "Version: %s\n", PACKAGE_VERSION);
- fprintf(stderr, "Contact: Heng Li <lh3 at sanger.ac.uk>\n\n");
- fprintf(stderr, "Usage: wgsim [options] <in.ref.fa> <out.read1.fq> <out.read2.fq>\n\n");
- fprintf(stderr, "Options: -e FLOAT base error rate [%.3f]\n", ERR_RATE);
- fprintf(stderr, " -d INT outer distance between the two ends [500]\n");
- fprintf(stderr, " -s INT standard deviation [50]\n");
- fprintf(stderr, " -N INT number of read pairs [1000000]\n");
- fprintf(stderr, " -1 INT length of the first read [70]\n");
- fprintf(stderr, " -2 INT length of the second read [70]\n");
- fprintf(stderr, " -r FLOAT rate of mutations [%.4f]\n", MUT_RATE);
- fprintf(stderr, " -R FLOAT fraction of indels [%.2f]\n", INDEL_FRAC);
- fprintf(stderr, " -X FLOAT probability an indel is extended [%.2f]\n", INDEL_EXTEND);
- fprintf(stderr, " -S INT seed for random generator [-1]\n");
- fprintf(stderr, " -h haplotype mode\n");
- fprintf(stderr, "\n");
- return 1;
-}
-
-int main(int argc, char *argv[])
-{
- int64_t N;
- int dist, std_dev, c, size_l, size_r, is_hap = 0;
- FILE *fpout1, *fpout2;
- int seed = -1;
-
- N = 1000000; dist = 500; std_dev = 50;
- size_l = size_r = 70;
- while ((c = getopt(argc, argv, "e:d:s:N:1:2:r:R:hX:S:")) >= 0) {
- switch (c) {
- case 'd': dist = atoi(optarg); break;
- case 's': std_dev = atoi(optarg); break;
- case 'N': N = atoi(optarg); break;
- case '1': size_l = atoi(optarg); break;
- case '2': size_r = atoi(optarg); break;
- case 'e': ERR_RATE = atof(optarg); break;
- case 'r': MUT_RATE = atof(optarg); break;
- case 'R': INDEL_FRAC = atof(optarg); break;
- case 'X': INDEL_EXTEND = atof(optarg); break;
- case 'S': seed = atoi(optarg); break;
- case 'h': is_hap = 1; break;
- }
- }
- if (argc - optind < 3) return simu_usage();
- fpout1 = fopen(argv[optind+1], "w");
- fpout2 = fopen(argv[optind+2], "w");
- if (!fpout1 || !fpout2) {
- fprintf(stderr, "[wgsim] file open error\n");
- return 1;
- }
- srand48(seed > 0? seed : time(0));
- wgsim_core(fpout1, fpout2, argv[optind], is_hap, N, dist, std_dev, size_l, size_r);
-
- fclose(fpout1); fclose(fpout2);
- return 0;
-}
diff --git a/samtools-0.1.19/misc/wgsim_eval.pl b/samtools-0.1.19/misc/wgsim_eval.pl
deleted file mode 100755
index f919a06..0000000
--- a/samtools-0.1.19/misc/wgsim_eval.pl
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/perl -w
-
-# Contact: lh3
-# Version: 0.1.5
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-&wgsim_eval;
-exit;
-
-sub wgsim_eval {
- my %opts = (g=>5);
- getopts('pcag:', \%opts);
- die("Usage: wgsim_eval.pl [-pca] [-g $opts{g}] <in.sam>\n") if (@ARGV == 0 && -t STDIN);
- my (@c0, @c1, %fnfp);
- my ($max_q, $flag) = (0, 0);
- my $gap = $opts{g};
- $flag |= 1 if (defined $opts{p});
- $flag |= 2 if (defined $opts{c});
- while (<>) {
- next if (/^\@/);
- my @t = split("\t");
- next if (@t < 11);
- my $line = $_;
- my ($q, $is_correct, $chr, $left, $rght) = (int($t[4]/10), 1, $t[2], $t[3], $t[3]);
- $max_q = $q if ($q > $max_q);
- # right coordinate
- $_ = $t[5]; s/(\d+)[MDN]/$rght+=$1,'x'/eg;
- --$rght;
- # correct for soft clipping
- my ($left0, $rght0) = ($left, $rght);
- $left -= $1 if (/^(\d+)[SH]/);
- $rght += $1 if (/(\d+)[SH]$/);
- $left0 -= $1 if (/(\d+)[SH]$/);
- $rght0 += $1 if (/^(\d+)[SH]/);
- # skip unmapped reads
- next if (($t[1]&0x4) || $chr eq '*');
- # parse read name and check
- if ($t[0] =~ /^(\S+)_(\d+)_(\d+)_/) {
- if ($1 ne $chr) { # different chr
- $is_correct = 0;
- } else {
- if ($flag & 2) {
- if (($t[1]&0x40) && !($t[1]&0x10)) { # F3, forward
- $is_correct = 0 if (abs($2 - $left) > $gap && abs($2 - $left0) > $gap);
- } elsif (($t[1]&0x40) && ($t[1]&0x10)) { # F3, reverse
- $is_correct = 0 if (abs($3 - $rght) > $gap && abs($3 - $rght0) > $gap);
- } elsif (($t[1]&0x80) && !($t[1]&0x10)) { # R3, forward
- $is_correct = 0 if (abs($3 - $left) > $gap && abs($3 - $left0) > $gap);
- } else { # R3, reverse
- $is_correct = 0 if (abs($2 - $rght) > $gap && abs($3 - $rght0) > $gap);
- }
- } else {
- if ($t[1] & 0x10) { # reverse
- $is_correct = 0 if (abs($3 - $rght) > $gap && abs($3 - $rght0) > $gap); # in case of indels that are close to the end of a reads
- } else {
- $is_correct = 0 if (abs($2 - $left) > $gap && abs($2 - $left0) > $gap);
- }
- }
- }
- } else {
- warn("[wgsim_eval] read '$t[0]' was not generated by wgsim?\n");
- next;
- }
- ++$c0[$q];
- ++$c1[$q] unless ($is_correct);
- @{$fnfp{$t[4]}} = (0, 0) unless (defined $fnfp{$t[4]});
- ++$fnfp{$t[4]}[0];
- ++$fnfp{$t[4]}[1] unless ($is_correct);
- print STDERR $line if (($flag&1) && !$is_correct && $q > 0);
- }
- # print
- my ($cc0, $cc1) = (0, 0);
- if (!defined($opts{a})) {
- for (my $i = $max_q; $i >= 0; --$i) {
- $c0[$i] = 0 unless (defined $c0[$i]);
- $c1[$i] = 0 unless (defined $c1[$i]);
- $cc0 += $c0[$i]; $cc1 += $c1[$i];
- printf("%.2dx %12d / %-12d %12d %.3e\n", $i, $c1[$i], $c0[$i], $cc0, $cc1/$cc0) if ($cc0);
- }
- } else {
- for (reverse(sort {$a<=>$b} (keys %fnfp))) {
- next if ($_ == 0);
- $cc0 += $fnfp{$_}[0];
- $cc1 += $fnfp{$_}[1];
- print join("\t", $_, $cc0, $cc1), "\n";
- }
- }
-}
diff --git a/samtools-0.1.19/misc/zoom2sam.pl b/samtools-0.1.19/misc/zoom2sam.pl
deleted file mode 100755
index 5306bfa..0000000
--- a/samtools-0.1.19/misc/zoom2sam.pl
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/perl -w
-
-# Contact: lh3
-# Version: 0.1.0
-
-use strict;
-use warnings;
-use Getopt::Std;
-
-&zoom2sam;
-exit;
-
-sub mating {
- my ($s1, $s2) = @_;
- my $isize = 0;
- if ($s1->[2] ne '*' && $s1->[2] eq $s2->[2]) { # then calculate $isize
- my $x1 = ($s1->[1] & 0x10)? $s1->[3] + length($s1->[9]) : $s1->[3];
- my $x2 = ($s2->[1] & 0x10)? $s2->[3] + length($s2->[9]) : $s2->[3];
- $isize = $x2 - $x1;
- }
- # update mate coordinate
- if ($s2->[2] ne '*') {
- @$s1[6..8] = (($s2->[2] eq $s1->[2])? "=" : $s2->[2], $s2->[3], $isize);
- $s1->[1] |= 0x20 if ($s2->[1] & 0x10);
- } else {
- $s1->[1] |= 0x8;
- }
- if ($s1->[2] ne '*') {
- @$s2[6..8] = (($s1->[2] eq $s2->[2])? "=" : $s1->[2], $s1->[3], -$isize);
- $s2->[1] |= 0x20 if ($s1->[1] & 0x10);
- } else {
- $s2->[1] |= 0x8;
- }
-}
-
-sub zoom2sam {
- my %opts = ();
- getopts("p", \%opts);
- die("Usage: zoom2sam.pl [-p] <readLen> <aln.zoom>
-Warnings: This script only supports the default Illumina outputs.\n") if (@ARGV < 2);
- my $is_paired = defined($opts{p});
- my $len = shift(@ARGV);
- # core loop
- my @s1 = ();
- my @s2 = ();
- my ($s_last, $s_curr) = (\@s1, \@s2);
- while (<>) {
- &zoom2sam_aux($_, $s_curr, $is_paired, $len);
- if (@$s_last != 0 && $s_last->[0] eq $s_curr->[0]) {
- &mating($s_last, $s_curr);
- print join("\t", @$s_last), "\n";
- print join("\t", @$s_curr), "\n";
- @$s_last = (); @$s_curr = ();
- } else {
- print join("\t", @$s_last), "\n" if (@$s_last != 0);
- my $s = $s_last; $s_last = $s_curr; $s_curr = $s;
- }
- }
- print join("\t", @$s_last), "\n" if (@$s_last != 0);
-}
-
-sub zoom2sam_aux {
- my ($line, $s, $is_paired, $len) = @_;
- chomp($line);
- my @t = split("\t", $line);
- @$s = ();
- # read name
- $s->[0] = $t[0];
- # initial flag (will be updated later)
- $s->[1] = 0;
- $s->[1] |= 1 | 1<<6 if ($s->[0] =~ /_F$/);
- $s->[1] |= 1 | 1<<7 if ($s->[0] =~ /_R$/);
- $s->[1] |= 2 if ($is_paired);
- # read & quality
- $s->[9] = "*"; $s->[10] = "*";
- # cigar
- $s->[5] = $len . "M";
- # coor
- my @s = split(/\s+/, $t[1]);
- $s->[2] = $s[0];
- $t[1] =~ /:(\d+)$/;
- $s->[3] = $1 + 1;
- if ($s->[0] =~ /_[FR]$/) {
- my $u = ($s->[0] =~ /_F$/)? 1 : 0;
- my $w = ($t[2] eq '+')? 1 : 0;
- $s->[1] |= 0x10 if ($u ^ $w);
- $s->[0] =~ s/_[FR]$//;
- } else {
- $s->[1] |= 0x10 if ($t[2] eq '-');
- }
- # mapQ
- $s->[4] = 30;
- # mate coordinate
- $s->[6] = '*'; $s->[7] = $s->[8] = 0;
- # aux
- push(@$s, "NM:i:$t[3]");
-}
diff --git a/samtools-0.1.19/padding.c b/samtools-0.1.19/padding.c
deleted file mode 100644
index a8da562..0000000
--- a/samtools-0.1.19/padding.c
+++ /dev/null
@@ -1,479 +0,0 @@
-#include <string.h>
-#include <assert.h>
-#include <unistd.h>
-#include "kstring.h"
-#include "sam_header.h"
-#include "sam.h"
-#include "bam.h"
-#include "faidx.h"
-
-bam_header_t *bam_header_dup(const bam_header_t *h0); /*in sam.c*/
-
-static void replace_cigar(bam1_t *b, int n, uint32_t *cigar)
-{
- if (n != b->core.n_cigar) {
- int o = b->core.l_qname + b->core.n_cigar * 4;
- if (b->data_len + (n - b->core.n_cigar) * 4 > b->m_data) {
- b->m_data = b->data_len + (n - b->core.n_cigar) * 4;
- kroundup32(b->m_data);
- b->data = (uint8_t*)realloc(b->data, b->m_data);
- }
- memmove(b->data + b->core.l_qname + n * 4, b->data + o, b->data_len - o);
- memcpy(b->data + b->core.l_qname, cigar, n * 4);
- b->data_len += (n - b->core.n_cigar) * 4;
- b->core.n_cigar = n;
- } else memcpy(b->data + b->core.l_qname, cigar, n * 4);
-}
-
-#define write_cigar(_c, _n, _m, _v) do { \
- if (_n == _m) { \
- _m = _m? _m<<1 : 4; \
- _c = (uint32_t*)realloc(_c, _m * 4); \
- } \
- _c[_n++] = (_v); \
- } while (0)
-
-static void unpad_seq(bam1_t *b, kstring_t *s)
-{
- int k, j, i;
- int length;
- uint32_t *cigar = bam1_cigar(b);
- uint8_t *seq = bam1_seq(b);
- // b->core.l_qseq gives length of the SEQ entry (including soft clips, S)
- // We need the padded length after alignment from the CIGAR (excluding
- // soft clips S, but including pads from CIGAR D operations)
- length = 0;
- for (k = 0; k < b->core.n_cigar; ++k) {
- int op, ol;
- op= bam_cigar_op(cigar[k]);
- ol = bam_cigar_oplen(cigar[k]);
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF || op == BAM_CDEL)
- length += ol;
- }
- ks_resize(s, length);
- for (k = 0, s->l = 0, j = 0; k < b->core.n_cigar; ++k) {
- int op, ol;
- op = bam_cigar_op(cigar[k]);
- ol = bam_cigar_oplen(cigar[k]);
- if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
- for (i = 0; i < ol; ++i, ++j) s->s[s->l++] = bam1_seqi(seq, j);
- } else if (op == BAM_CSOFT_CLIP) {
- j += ol;
- } else if (op == BAM_CHARD_CLIP) {
- /* do nothing */
- } else if (op == BAM_CDEL) {
- for (i = 0; i < ol; ++i) s->s[s->l++] = 0;
- } else {
- fprintf(stderr, "[depad] ERROR: Didn't expect CIGAR op %c in read %s\n", BAM_CIGAR_STR[op], bam1_qname(b));
- assert(-1);
- }
- }
- assert(length == s->l);
-}
-
-int load_unpadded_ref(faidx_t *fai, char *ref_name, int ref_len, kstring_t *seq)
-{
- char base;
- char *fai_ref = 0;
- int fai_ref_len = 0, k;
-
- fai_ref = fai_fetch(fai, ref_name, &fai_ref_len);
- if (fai_ref_len != ref_len) {
- fprintf(stderr, "[depad] ERROR: FASTA sequence %s length %i, expected %i\n", ref_name, fai_ref_len, ref_len);
- free(fai_ref);
- return -1;
- }
- ks_resize(seq, ref_len);
- seq->l = 0;
- for (k = 0; k < ref_len; ++k) {
- base = fai_ref[k];
- if (base == '-' || base == '*') {
- // Map gaps to null to match unpad_seq function
- seq->s[seq->l++] = 0;
- } else {
- int i = bam_nt16_table[(int)base];
- if (i == 0 || i==16) { // Equals maps to 0, anything unexpected to 16
- fprintf(stderr, "[depad] ERROR: Invalid character %c (ASCII %i) in FASTA sequence %s\n", base, (int)base, ref_name);
- free(fai_ref);
- return -1;
- }
- seq->s[seq->l++] = i;
- }
- }
- assert(ref_len == seq->l);
- free(fai_ref);
- return 0;
-}
-
-int get_unpadded_len(faidx_t *fai, char *ref_name, int padded_len)
-{
- char base;
- char *fai_ref = 0;
- int fai_ref_len = 0, k;
- int bases=0, gaps=0;
-
- fai_ref = fai_fetch(fai, ref_name, &fai_ref_len);
- if (fai_ref_len != padded_len) {
- fprintf(stderr, "[depad] ERROR: FASTA sequence '%s' length %i, expected %i\n", ref_name, fai_ref_len, padded_len);
- free(fai_ref);
- return -1;
- }
- for (k = 0; k < padded_len; ++k) {
- //fprintf(stderr, "[depad] checking base %i of %i or %i\n", k+1, ref_len, strlen(fai_ref));
- base = fai_ref[k];
- if (base == '-' || base == '*') {
- gaps += 1;
- } else {
- int i = bam_nt16_table[(int)base];
- if (i == 0 || i==16) { // Equals maps to 0, anything unexpected to 16
- fprintf(stderr, "[depad] ERROR: Invalid character %c (ASCII %i) in FASTA sequence '%s'\n", base, (int)base, ref_name);
- free(fai_ref);
- return -1;
- }
- bases += 1;
- }
- }
- free(fai_ref);
- assert (padded_len == bases + gaps);
- return bases;
-}
-
-inline int * update_posmap(int *posmap, kstring_t ref)
-{
- int i, k;
- posmap = realloc(posmap, ref.m * sizeof(int));
- for (i = k = 0; i < ref.l; ++i) {
- posmap[i] = k;
- if (ref.s[i]) ++k;
- }
- return posmap;
-}
-
-int bam_pad2unpad(samfile_t *in, samfile_t *out, faidx_t *fai)
-{
- bam_header_t *h = 0;
- bam1_t *b = 0;
- kstring_t r, q;
- int r_tid = -1;
- uint32_t *cigar2 = 0;
- int ret = 0, n2 = 0, m2 = 0, *posmap = 0;
-
- b = bam_init1();
- r.l = r.m = q.l = q.m = 0; r.s = q.s = 0;
- int read_ret;
- h = in->header;
- while ((read_ret = samread(in, b)) >= 0) { // read one alignment from `in'
- uint32_t *cigar = bam1_cigar(b);
- n2 = 0;
- if (b->core.pos == 0 && b->core.tid >= 0 && strcmp(bam1_qname(b), h->target_name[b->core.tid]) == 0) {
- // fprintf(stderr, "[depad] Found embedded reference '%s'\n", bam1_qname(b));
- r_tid = b->core.tid;
- unpad_seq(b, &r);
- if (h->target_len[r_tid] != r.l) {
- fprintf(stderr, "[depad] ERROR: (Padded) length of '%s' is %d in BAM header, but %ld in embedded reference\n", bam1_qname(b), h->target_len[r_tid], r.l);
- return -1;
- }
- if (fai) {
- // Check the embedded reference matches the FASTA file
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &q)) {
- fprintf(stderr, "[depad] ERROR: Failed to load embedded reference '%s' from FASTA\n", h->target_name[b->core.tid]);
- return -1;
- }
- assert(r.l == q.l);
- int i;
- for (i = 0; i < r.l; ++i) {
- if (r.s[i] != q.s[i]) {
- // Show gaps as ASCII 45
- fprintf(stderr, "[depad] ERROR: Embedded sequence and reference FASTA don't match for %s base %i, '%c' vs '%c'\n",
- h->target_name[b->core.tid], i+1,
- r.s[i] ? bam_nt16_rev_table[(int)r.s[i]] : 45,
- q.s[i] ? bam_nt16_rev_table[(int)q.s[i]] : 45);
- return -1;
- }
- }
- }
- write_cigar(cigar2, n2, m2, bam_cigar_gen(b->core.l_qseq, BAM_CMATCH));
- replace_cigar(b, n2, cigar2);
- posmap = update_posmap(posmap, r);
- } else if (b->core.n_cigar > 0) {
- int i, k, op;
- if (b->core.tid < 0) {
- fprintf(stderr, "[depad] ERROR: Read '%s' has CIGAR but no RNAME\n", bam1_qname(b));
- return -1;
- } else if (b->core.tid == r_tid) {
- ; // good case, reference available
- //fprintf(stderr, "[depad] Have ref '%s' for read '%s'\n", h->target_name[b->core.tid], bam1_qname(b));
- } else if (fai) {
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &r)) {
- fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.tid]);
- return -1;
- }
- posmap = update_posmap(posmap, r);
- r_tid = b->core.tid;
- // fprintf(stderr, "[depad] Loaded %s from FASTA file\n", h->target_name[b->core.tid]);
- } else {
- fprintf(stderr, "[depad] ERROR: Missing %s embedded reference sequence (and no FASTA file)\n", h->target_name[b->core.tid]);
- return -1;
- }
- unpad_seq(b, &q);
- if (bam_cigar_op(cigar[0]) == BAM_CSOFT_CLIP) {
- write_cigar(cigar2, n2, m2, cigar[0]);
- } else if (bam_cigar_op(cigar[0]) == BAM_CHARD_CLIP) {
- write_cigar(cigar2, n2, m2, cigar[0]);
- if (b->core.n_cigar > 2 && bam_cigar_op(cigar[1]) == BAM_CSOFT_CLIP) {
- write_cigar(cigar2, n2, m2, cigar[1]);
- }
- }
- /* Determine CIGAR operator for each base in the aligned read */
- for (i = 0, k = b->core.pos; i < q.l; ++i, ++k)
- q.s[i] = q.s[i]? (r.s[k]? BAM_CMATCH : BAM_CINS) : (r.s[k]? BAM_CDEL : BAM_CPAD);
- /* Include any pads if starts with an insert */
- if (q.s[0] == BAM_CINS) {
- for (k = 0; k+1 < b->core.pos && !r.s[b->core.pos - k - 1]; ++k);
- if (k) write_cigar(cigar2, n2, m2, bam_cigar_gen(k, BAM_CPAD));
- }
- /* Count consecutive CIGAR operators to turn into a CIGAR string */
- for (i = k = 1, op = q.s[0]; i < q.l; ++i) {
- if (op != q.s[i]) {
- write_cigar(cigar2, n2, m2, bam_cigar_gen(k, op));
- op = q.s[i]; k = 1;
- } else ++k;
- }
- write_cigar(cigar2, n2, m2, bam_cigar_gen(k, op));
- if (bam_cigar_op(cigar[b->core.n_cigar-1]) == BAM_CSOFT_CLIP) {
- write_cigar(cigar2, n2, m2, cigar[b->core.n_cigar-1]);
- } else if (bam_cigar_op(cigar[b->core.n_cigar-1]) == BAM_CHARD_CLIP) {
- if (b->core.n_cigar > 2 && bam_cigar_op(cigar[b->core.n_cigar-2]) == BAM_CSOFT_CLIP) {
- write_cigar(cigar2, n2, m2, cigar[b->core.n_cigar-2]);
- }
- write_cigar(cigar2, n2, m2, cigar[b->core.n_cigar-1]);
- }
- /* Remove redundant P operators between M/X/=/D operators, e.g. 5M2P10M -> 15M */
- int pre_op, post_op;
- for (i = 2; i < n2; ++i)
- if (bam_cigar_op(cigar2[i-1]) == BAM_CPAD) {
- pre_op = bam_cigar_op(cigar2[i-2]);
- post_op = bam_cigar_op(cigar2[i]);
- /* Note don't need to check for X/= as code above will use M only */
- if ((pre_op == BAM_CMATCH || pre_op == BAM_CDEL) && (post_op == BAM_CMATCH || post_op == BAM_CDEL)) {
- /* This is a redundant P operator */
- cigar2[i-1] = 0; // i.e. 0M
- /* If had same operator either side, combine them in post_op */
- if (pre_op == post_op) {
- /* If CIGAR M, could treat as simple integers since BAM_CMATCH is zero*/
- cigar2[i] = bam_cigar_gen(bam_cigar_oplen(cigar2[i-2]) + bam_cigar_oplen(cigar2[i]), post_op);
- cigar2[i-2] = 0; // i.e. 0M
- }
- }
- }
- /* Remove the zero'd operators (0M) */
- for (i = k = 0; i < n2; ++i)
- if (cigar2[i]) cigar2[k++] = cigar2[i];
- n2 = k;
- replace_cigar(b, n2, cigar2);
- b->core.pos = posmap[b->core.pos];
- if (b->core.mtid < 0 || b->core.mpos < 0) {
- /* Nice case, no mate to worry about*/
- // fprintf(stderr, "[depad] Read '%s' mate not mapped\n", bam1_qname(b));
- /* TODO - Warning if FLAG says mate should be mapped? */
- /* Clean up funny input where mate position is given but mate reference is missing: */
- b->core.mtid = -1;
- b->core.mpos = -1;
- } else if (b->core.mtid == b->core.tid) {
- /* Nice case, same reference */
- // fprintf(stderr, "[depad] Read '%s' mate mapped to same ref\n", bam1_qname(b));
- b->core.mpos = posmap[b->core.mpos];
- } else {
- /* Nasty case, Must load alternative posmap */
- // fprintf(stderr, "[depad] Loading reference '%s' temporarily\n", h->target_name[b->core.mtid]);
- if (!fai) {
- fprintf(stderr, "[depad] ERROR: Needed reference %s sequence for mate (and no FASTA file)\n", h->target_name[b->core.mtid]);
- return -1;
- }
- /* Temporarily load the other reference sequence */
- if (load_unpadded_ref(fai, h->target_name[b->core.mtid], h->target_len[b->core.mtid], &r)) {
- fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.mtid]);
- return -1;
- }
- posmap = update_posmap(posmap, r);
- b->core.mpos = posmap[b->core.mpos];
- /* Restore the reference and posmap*/
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &r)) {
- fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.tid]);
- return -1;
- }
- posmap = update_posmap(posmap, r);
- }
- }
- samwrite(out, b);
- }
- if (read_ret < -1) {
- fprintf(stderr, "[depad] truncated file.\n");
- ret = 1;
- }
- free(r.s); free(q.s); free(posmap);
- bam_destroy1(b);
- return ret;
-}
-
-bam_header_t * fix_header(bam_header_t *old, faidx_t *fai)
-{
- int i = 0, unpadded_len = 0;
- bam_header_t *header = 0 ;
-
- header = bam_header_dup(old);
- for (i = 0; i < old->n_targets; ++i) {
- unpadded_len = get_unpadded_len(fai, old->target_name[i], old->target_len[i]);
- if (unpadded_len < 0) {
- fprintf(stderr, "[depad] ERROR getting unpadded length of '%s', padded length %i\n", old->target_name[i], old->target_len[i]);
- } else {
- header->target_len[i] = unpadded_len;
- //fprintf(stderr, "[depad] Recalculating '%s' length %i -> %i\n", old->target_name[i], old->target_len[i], header->target_len[i]);
- }
- }
- /* Duplicating the header allocated new buffer for header string */
- /* After modifying the @SQ lines it will only get smaller, since */
- /* the LN entries will be the same or shorter, and we'll remove */
- /* any MD entries (MD5 checksums). */
- assert(strlen(old->text) == strlen(header->text));
- assert (0==strcmp(old->text, header->text));
- const char *text;
- text = old->text;
- header->text[0] = '\0'; /* Resuse the allocated buffer */
- char * newtext = header->text;
- char * end=NULL;
- while (text[0]=='@') {
- end = strchr(text, '\n');
- assert(end != 0);
- if (text[1]=='S' && text[2]=='Q' && text[3]=='\t') {
- /* TODO - edit the @SQ line here to remove MD and fix LN. */
- /* For now just remove the @SQ line, and samtools will */
- /* automatically generate a minimal replacement with LN. */
- /* However, that discards any other tags like AS, SP, UR. */
- //fprintf(stderr, "[depad] Removing @SQ line\n");
- } else {
- /* Copy this line to the new header */
- strncat(newtext, text, end - text + 1);
- }
- text = end + 1;
- }
- assert (text[0]=='\0');
- /* Check we didn't overflow the buffer */
- assert (strlen(header->text) <= strlen(old->text));
- if (strlen(header->text) < header->l_text) {
- //fprintf(stderr, "[depad] Reallocating header buffer\n");
- assert (newtext == header->text);
- newtext = malloc(strlen(header->text) + 1);
- strcpy(newtext, header->text);
- free(header->text);
- header->text = newtext;
- header->l_text = strlen(newtext);
- }
- //fprintf(stderr, "[depad] Here is the new header (pending @SQ lines),\n\n%s\n(end)\n", header->text);
- return header;
-}
-
-static int usage(int is_long_help);
-
-int main_pad2unpad(int argc, char *argv[])
-{
- samfile_t *in = 0, *out = 0;
- bam_header_t *h = 0;
- faidx_t *fai = 0;
- int c, is_bamin = 1, compress_level = -1, is_bamout = 1, is_long_help = 0;
- char in_mode[5], out_mode[5], *fn_out = 0, *fn_list = 0, *fn_ref = 0;
- int ret=0;
-
- /* parse command-line options */
- strcpy(in_mode, "r"); strcpy(out_mode, "w");
- while ((c = getopt(argc, argv, "Sso:u1T:?")) >= 0) {
- switch (c) {
- case 'S': is_bamin = 0; break;
- case 's': assert(compress_level == -1); is_bamout = 0; break;
- case 'o': fn_out = strdup(optarg); break;
- case 'u': assert(is_bamout == 1); compress_level = 0; break;
- case '1': assert(is_bamout == 1); compress_level = 1; break;
- case 'T': fn_ref = strdup(optarg); break;
- case '?': is_long_help = 1; break;
- default: return usage(is_long_help);
- }
- }
- if (argc == optind) return usage(is_long_help);
-
- if (is_bamin) strcat(in_mode, "b");
- if (is_bamout) strcat(out_mode, "b");
- strcat(out_mode, "h");
- if (compress_level >= 0) {
- char tmp[2];
- tmp[0] = compress_level + '0'; tmp[1] = '\0';
- strcat(out_mode, tmp);
- }
-
- // Load FASTA reference (also needed for SAM -> BAM if missing header)
- if (fn_ref) {
- fn_list = samfaipath(fn_ref);
- fai = fai_load(fn_ref);
- }
- // open file handlers
- if ((in = samopen(argv[optind], in_mode, fn_list)) == 0) {
- fprintf(stderr, "[depad] failed to open \"%s\" for reading.\n", argv[optind]);
- ret = 1;
- goto depad_end;
- }
- if (in->header == 0) {
- fprintf(stderr, "[depad] failed to read the header from \"%s\".\n", argv[optind]);
- ret = 1;
- goto depad_end;
- }
- if (in->header->text == 0 || in->header->l_text == 0) {
- fprintf(stderr, "[depad] Warning - failed to read any header text from \"%s\".\n", argv[optind]);
- assert (0 == in->header->l_text);
- assert (0 == in->header->text);
- }
- if (fn_ref) {
- h = fix_header(in->header, fai);
- } else {
- fprintf(stderr, "[depad] Warning - reference lengths will not be corrected without FASTA reference\n");
- h = in->header;
- }
- if ((out = samopen(fn_out? fn_out : "-", out_mode, h)) == 0) {
- fprintf(stderr, "[depad] failed to open \"%s\" for writing.\n", fn_out? fn_out : "standard output");
- ret = 1;
- goto depad_end;
- }
-
- // Do the depad
- ret = bam_pad2unpad(in, out, fai);
-
-depad_end:
- // close files, free and return
- if (fai) fai_destroy(fai);
- if (h != in->header) bam_header_destroy(h);
- samclose(in);
- samclose(out);
- free(fn_list); free(fn_out);
- return ret;
-}
-
-static int usage(int is_long_help)
-{
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools depad <in.bam>\n\n");
- fprintf(stderr, "Options: -s output is SAM (default is BAM)\n");
- fprintf(stderr, " -S input is SAM (default is BAM)\n");
- fprintf(stderr, " -u uncompressed BAM output (can't use with -s)\n");
- fprintf(stderr, " -1 fast compression BAM output (can't use with -s)\n");
- fprintf(stderr, " -T FILE reference sequence file [null]\n");
- fprintf(stderr, " -o FILE output file name [stdout]\n");
- fprintf(stderr, " -? longer help\n");
- fprintf(stderr, "\n");
- if (is_long_help)
- fprintf(stderr, "Notes:\n\
-\n\
- 1. Requires embedded reference sequences (before the reads for that reference),\n\
- with the future aim to also support a FASTA padded reference sequence file.\n\
-\n\
- 2. The input padded alignment read's CIGAR strings must not use P or I operators.\n\
-\n");
- return 1;
-}
diff --git a/samtools-0.1.19/phase.c b/samtools-0.1.19/phase.c
deleted file mode 100644
index ef4eff9..0000000
--- a/samtools-0.1.19/phase.c
+++ /dev/null
@@ -1,687 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <math.h>
-#include <zlib.h>
-#include "bam.h"
-#include "errmod.h"
-
-#include "kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-#define MAX_VARS 256
-#define FLIP_PENALTY 2
-#define FLIP_THRES 4
-#define MASK_THRES 3
-
-#define FLAG_FIX_CHIMERA 0x1
-#define FLAG_LIST_EXCL 0x4
-#define FLAG_DROP_AMBI 0x8
-
-typedef struct {
- // configurations, initialized in the main function
- int flag, k, min_baseQ, min_varLOD, max_depth;
- // other global variables
- int vpos_shift;
- bamFile fp;
- char *pre;
- bamFile out[3];
- // alignment queue
- int n, m;
- bam1_t **b;
-} phaseg_t;
-
-typedef struct {
- int8_t seq[MAX_VARS]; // TODO: change to dynamic memory allocation!
- int vpos, beg, end;
- uint32_t vlen:16, single:1, flip:1, phase:1, phased:1, ambig:1;
- uint32_t in:16, out:16; // in-phase and out-phase
-} frag_t, *frag_p;
-
-#define rseq_lt(a,b) ((a)->vpos < (b)->vpos)
-
-#include "khash.h"
-KHASH_SET_INIT_INT64(set64)
-KHASH_MAP_INIT_INT64(64, frag_t)
-
-typedef khash_t(64) nseq_t;
-
-#include "ksort.h"
-KSORT_INIT(rseq, frag_p, rseq_lt)
-
-static char nt16_nt4_table[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 };
-
-static inline uint64_t X31_hash_string(const char *s)
-{
- uint64_t h = *s;
- if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s;
- return h;
-}
-
-static void count1(int l, const uint8_t *seq, int *cnt)
-{
- int i, j, n_ambi;
- uint32_t z, x;
- if (seq[l-1] == 0) return; // do nothing is the last base is ambiguous
- for (i = n_ambi = 0; i < l; ++i) // collect ambiguous bases
- if (seq[i] == 0) ++n_ambi;
- if (l - n_ambi <= 1) return; // only one SNP
- for (x = 0; x < 1u<<n_ambi; ++x) { // count
- for (i = j = 0, z = 0; i < l; ++i) {
- int c;
- if (seq[i]) c = seq[i] - 1;
- else {
- c = x>>j&1;
- ++j;
- }
- z = z<<1 | c;
- }
- ++cnt[z];
- }
-}
-
-static int **count_all(int l, int vpos, nseq_t *hash)
-{
- khint_t k;
- int i, j, **cnt;
- uint8_t *seq;
- seq = calloc(l, 1);
- cnt = calloc(vpos, sizeof(void*));
- for (i = 0; i < vpos; ++i) cnt[i] = calloc(1<<l, sizeof(int));
- for (k = 0; k < kh_end(hash); ++k) {
- if (kh_exist(hash, k)) {
- frag_t *f = &kh_val(hash, k);
- if (f->vpos >= vpos || f->single) continue; // out of region; or singleton
- if (f->vlen == 1) { // such reads should be flagged as deleted previously if everything is right
- f->single = 1;
- continue;
- }
- for (j = 1; j < f->vlen; ++j) {
- for (i = 0; i < l; ++i)
- seq[i] = j < l - 1 - i? 0 : f->seq[j - (l - 1 - i)];
- count1(l, seq, cnt[f->vpos + j]);
- }
- }
- }
- free(seq);
- return cnt;
-}
-
-// phasing
-static int8_t *dynaprog(int l, int vpos, int **w)
-{
- int *f[2], *curr, *prev, max, i;
- int8_t **b, *h = 0;
- uint32_t x, z = 1u<<(l-1), mask = (1u<<l) - 1;
- f[0] = calloc(z, sizeof(int));
- f[1] = calloc(z, sizeof(int));
- b = calloc(vpos, sizeof(void*));
- prev = f[0]; curr = f[1];
- // fill the backtrack matrix
- for (i = 0; i < vpos; ++i) {
- int *wi = w[i], *tmp;
- int8_t *bi;
- bi = b[i] = calloc(z, 1);
- /* In the following, x is the current state, which is the
- * lexicographically smaller local haplotype. xc is the complement of
- * x, or the larger local haplotype; y0 and y1 are the two predecessors
- * of x. */
- for (x = 0; x < z; ++x) { // x0 is the smaller
- uint32_t y0, y1, xc;
- int c0, c1;
- xc = ~x&mask; y0 = x>>1; y1 = xc>>1;
- c0 = prev[y0] + wi[x] + wi[xc];
- c1 = prev[y1] + wi[x] + wi[xc];
- if (c0 > c1) bi[x] = 0, curr[x] = c0;
- else bi[x] = 1, curr[x] = c1;
- }
- tmp = prev; prev = curr; curr = tmp; // swap
- }
- { // backtrack
- uint32_t max_x = 0;
- int which = 0;
- h = calloc(vpos, 1);
- for (x = 0, max = 0, max_x = 0; x < z; ++x)
- if (prev[x] > max) max = prev[x], max_x = x;
- for (i = vpos - 1, x = max_x; i >= 0; --i) {
- h[i] = which? (~x&1) : (x&1);
- which = b[i][x]? !which : which;
- x = b[i][x]? (~x&mask)>>1 : x>>1;
- }
- }
- // free
- for (i = 0; i < vpos; ++i) free(b[i]);
- free(f[0]); free(f[1]); free(b);
- return h;
-}
-
-// phase each fragment
-static uint64_t *fragphase(int vpos, const int8_t *path, nseq_t *hash, int flip)
-{
- khint_t k;
- uint64_t *pcnt;
- uint32_t *left, *rght, max;
- left = rght = 0; max = 0;
- pcnt = calloc(vpos, 8);
- for (k = 0; k < kh_end(hash); ++k) {
- if (kh_exist(hash, k)) {
- int i, c[2];
- frag_t *f = &kh_val(hash, k);
- if (f->vpos >= vpos) continue;
- // get the phase
- c[0] = c[1] = 0;
- for (i = 0; i < f->vlen; ++i) {
- if (f->seq[i] == 0) continue;
- ++c[f->seq[i] == path[f->vpos + i] + 1? 0 : 1];
- }
- f->phase = c[0] > c[1]? 0 : 1;
- f->in = c[f->phase]; f->out = c[1 - f->phase];
- f->phased = f->in == f->out? 0 : 1;
- f->ambig = (f->in && f->out && f->out < 3 && f->in <= f->out + 1)? 1 : 0;
- // fix chimera
- f->flip = 0;
- if (flip && c[0] >= 3 && c[1] >= 3) {
- int sum[2], m, mi, md;
- if (f->vlen > max) { // enlarge the array
- max = f->vlen;
- kroundup32(max);
- left = realloc(left, max * 4);
- rght = realloc(rght, max * 4);
- }
- for (i = 0, sum[0] = sum[1] = 0; i < f->vlen; ++i) { // get left counts
- if (f->seq[i]) {
- int c = f->phase? 2 - f->seq[i] : f->seq[i] - 1;
- ++sum[c == path[f->vpos + i]? 0 : 1];
- }
- left[i] = sum[1]<<16 | sum[0];
- }
- for (i = f->vlen - 1, sum[0] = sum[1] = 0; i >= 0; --i) { // get right counts
- if (f->seq[i]) {
- int c = f->phase? 2 - f->seq[i] : f->seq[i] - 1;
- ++sum[c == path[f->vpos + i]? 0 : 1];
- }
- rght[i] = sum[1]<<16 | sum[0];
- }
- // find the best flip point
- for (i = m = 0, mi = -1, md = -1; i < f->vlen - 1; ++i) {
- int a[2];
- a[0] = (left[i]&0xffff) + (rght[i+1]>>16&0xffff) - (rght[i+1]&0xffff) * FLIP_PENALTY;
- a[1] = (left[i]>>16&0xffff) + (rght[i+1]&0xffff) - (rght[i+1]>>16&0xffff) * FLIP_PENALTY;
- if (a[0] > a[1]) {
- if (a[0] > m) m = a[0], md = 0, mi = i;
- } else {
- if (a[1] > m) m = a[1], md = 1, mi = i;
- }
- }
- if (m - c[0] >= FLIP_THRES && m - c[1] >= FLIP_THRES) { // then flip
- f->flip = 1;
- if (md == 0) { // flip the tail
- for (i = mi + 1; i < f->vlen; ++i)
- if (f->seq[i] == 1) f->seq[i] = 2;
- else if (f->seq[i] == 2) f->seq[i] = 1;
- } else { // flip the head
- for (i = 0; i <= mi; ++i)
- if (f->seq[i] == 1) f->seq[i] = 2;
- else if (f->seq[i] == 2) f->seq[i] = 1;
- }
- }
- }
- // update pcnt[]
- if (!f->single) {
- for (i = 0; i < f->vlen; ++i) {
- int c;
- if (f->seq[i] == 0) continue;
- c = f->phase? 2 - f->seq[i] : f->seq[i] - 1;
- if (c == path[f->vpos + i]) {
- if (f->phase == 0) ++pcnt[f->vpos + i];
- else pcnt[f->vpos + i] += 1ull<<32;
- } else {
- if (f->phase == 0) pcnt[f->vpos + i] += 1<<16;
- else pcnt[f->vpos + i] += 1ull<<48;
- }
- }
- }
- }
- }
- free(left); free(rght);
- return pcnt;
-}
-
-static uint64_t *genmask(int vpos, const uint64_t *pcnt, int *_n)
-{
- int i, max = 0, max_i = -1, m = 0, n = 0, beg = 0, score = 0;
- uint64_t *list = 0;
- for (i = 0; i < vpos; ++i) {
- uint64_t x = pcnt[i];
- int c[4], pre = score, s;
- c[0] = x&0xffff; c[1] = x>>16&0xffff; c[2] = x>>32&0xffff; c[3] = x>>48&0xffff;
- s = (c[1] + c[3] == 0)? -(c[0] + c[2]) : (c[1] + c[3] - 1);
- if (c[3] > c[2]) s += c[3] - c[2];
- if (c[1] > c[0]) s += c[1] - c[0];
- score += s;
- if (score < 0) score = 0;
- if (pre == 0 && score > 0) beg = i; // change from zero to non-zero
- if ((i == vpos - 1 || score == 0) && max >= MASK_THRES) {
- if (n == m) {
- m = m? m<<1 : 4;
- list = realloc(list, m * 8);
- }
- list[n++] = (uint64_t)beg<<32 | max_i;
- i = max_i; // reset i to max_i
- score = 0;
- } else if (score > max) max = score, max_i = i;
- if (score == 0) max = 0;
- }
- *_n = n;
- return list;
-}
-
-// trim heading and tailing ambiguous bases; mark deleted and remove sequence
-static int clean_seqs(int vpos, nseq_t *hash)
-{
- khint_t k;
- int ret = 0;
- for (k = 0; k < kh_end(hash); ++k) {
- if (kh_exist(hash, k)) {
- frag_t *f = &kh_val(hash, k);
- int beg, end, i;
- if (f->vpos >= vpos) {
- ret = 1;
- continue;
- }
- for (i = 0; i < f->vlen; ++i)
- if (f->seq[i] != 0) break;
- beg = i;
- for (i = f->vlen - 1; i >= 0; --i)
- if (f->seq[i] != 0) break;
- end = i + 1;
- if (end - beg <= 0) kh_del(64, hash, k);
- else {
- if (beg != 0) memmove(f->seq, f->seq + beg, end - beg);
- f->vpos += beg; f->vlen = end - beg;
- f->single = f->vlen == 1? 1 : 0;
- }
- }
- }
- return ret;
-}
-
-static void dump_aln(phaseg_t *g, int min_pos, const nseq_t *hash)
-{
- int i, is_flip, drop_ambi;
- drop_ambi = g->flag & FLAG_DROP_AMBI;
- is_flip = (drand48() < 0.5);
- for (i = 0; i < g->n; ++i) {
- int end, which;
- uint64_t key;
- khint_t k;
- bam1_t *b = g->b[i];
- key = X31_hash_string(bam1_qname(b));
- end = bam_calend(&b->core, bam1_cigar(b));
- if (end > min_pos) break;
- k = kh_get(64, hash, key);
- if (k == kh_end(hash)) which = 3;
- else {
- frag_t *f = &kh_val(hash, k);
- if (f->ambig) which = drop_ambi? 2 : 3;
- else if (f->phased && f->flip) which = 2;
- else if (f->phased == 0) which = 3;
- else { // phased and not flipped
- char c = 'Y';
- which = f->phase;
- bam_aux_append(b, "ZP", 'A', 1, (uint8_t*)&c);
- }
- if (which < 2 && is_flip) which = 1 - which; // increase the randomness
- }
- if (which == 3) which = (drand48() < 0.5);
- bam_write1(g->out[which], b);
- bam_destroy1(b);
- g->b[i] = 0;
- }
- memmove(g->b, g->b + i, (g->n - i) * sizeof(void*));
- g->n -= i;
-}
-
-static int phase(phaseg_t *g, const char *chr, int vpos, uint64_t *cns, nseq_t *hash)
-{
- int i, j, n_seqs = kh_size(hash), n_masked = 0, min_pos;
- khint_t k;
- frag_t **seqs;
- int8_t *path, *sitemask;
- uint64_t *pcnt, *regmask;
-
- if (vpos == 0) return 0;
- i = clean_seqs(vpos, hash); // i is true if hash has an element with its vpos >= vpos
- min_pos = i? cns[vpos]>>32 : 0x7fffffff;
- if (vpos == 1) {
- printf("PS\t%s\t%d\t%d\n", chr, (int)(cns[0]>>32) + 1, (int)(cns[0]>>32) + 1);
- printf("M0\t%s\t%d\t%d\t%c\t%c\t%d\t0\t0\t0\t0\n//\n", chr, (int)(cns[0]>>32) + 1, (int)(cns[0]>>32) + 1,
- "ACGTX"[cns[0]&3], "ACGTX"[cns[0]>>16&3], g->vpos_shift + 1);
- for (k = 0; k < kh_end(hash); ++k) {
- if (kh_exist(hash, k)) {
- frag_t *f = &kh_val(hash, k);
- if (f->vpos) continue;
- f->flip = 0;
- if (f->seq[0] == 0) f->phased = 0;
- else f->phased = 1, f->phase = f->seq[0] - 1;
- }
- }
- dump_aln(g, min_pos, hash);
- ++g->vpos_shift;
- return 1;
- }
- { // phase
- int **cnt;
- uint64_t *mask;
- printf("PS\t%s\t%d\t%d\n", chr, (int)(cns[0]>>32) + 1, (int)(cns[vpos-1]>>32) + 1);
- sitemask = calloc(vpos, 1);
- cnt = count_all(g->k, vpos, hash);
- path = dynaprog(g->k, vpos, cnt);
- for (i = 0; i < vpos; ++i) free(cnt[i]);
- free(cnt);
- pcnt = fragphase(vpos, path, hash, 0); // do not fix chimeras when masking
- mask = genmask(vpos, pcnt, &n_masked);
- regmask = calloc(n_masked, 8);
- for (i = 0; i < n_masked; ++i) {
- regmask[i] = cns[mask[i]>>32]>>32<<32 | cns[(uint32_t)mask[i]]>>32;
- for (j = mask[i]>>32; j <= (int32_t)mask[i]; ++j)
- sitemask[j] = 1;
- }
- free(mask);
- if (g->flag & FLAG_FIX_CHIMERA) {
- free(pcnt);
- pcnt = fragphase(vpos, path, hash, 1);
- }
- }
- for (i = 0; i < n_masked; ++i)
- printf("FL\t%s\t%d\t%d\n", chr, (int)(regmask[i]>>32) + 1, (int)regmask[i] + 1);
- for (i = 0; i < vpos; ++i) {
- uint64_t x = pcnt[i];
- int8_t c[2];
- c[0] = (cns[i]&0xffff)>>2 == 0? 4 : (cns[i]&3);
- c[1] = (cns[i]>>16&0xffff)>>2 == 0? 4 : (cns[i]>>16&3);
- printf("M%d\t%s\t%d\t%d\t%c\t%c\t%d\t%d\t%d\t%d\t%d\n", sitemask[i]+1, chr, (int)(cns[0]>>32) + 1, (int)(cns[i]>>32) + 1, "ACGTX"[c[path[i]]], "ACGTX"[c[1-path[i]]],
- i + g->vpos_shift + 1, (int)(x&0xffff), (int)(x>>16&0xffff), (int)(x>>32&0xffff), (int)(x>>48&0xffff));
- }
- free(path); free(pcnt); free(regmask); free(sitemask);
- seqs = calloc(n_seqs, sizeof(void*));
- for (k = 0, i = 0; k < kh_end(hash); ++k)
- if (kh_exist(hash, k) && kh_val(hash, k).vpos < vpos && !kh_val(hash, k).single)
- seqs[i++] = &kh_val(hash, k);
- n_seqs = i;
- ks_introsort_rseq(n_seqs, seqs);
- for (i = 0; i < n_seqs; ++i) {
- frag_t *f = seqs[i];
- printf("EV\t0\t%s\t%d\t40\t%dM\t*\t0\t0\t", chr, f->vpos + 1 + g->vpos_shift, f->vlen);
- for (j = 0; j < f->vlen; ++j) {
- uint32_t c = cns[f->vpos + j];
- if (f->seq[j] == 0) putchar('N');
- else putchar("ACGT"[f->seq[j] == 1? (c&3) : (c>>16&3)]);
- }
- printf("\t*\tYP:i:%d\tYF:i:%d\tYI:i:%d\tYO:i:%d\tYS:i:%d\n", f->phase, f->flip, f->in, f->out, f->beg+1);
- }
- free(seqs);
- printf("//\n");
- fflush(stdout);
- g->vpos_shift += vpos;
- dump_aln(g, min_pos, hash);
- return vpos;
-}
-
-static void update_vpos(int vpos, nseq_t *hash)
-{
- khint_t k;
- for (k = 0; k < kh_end(hash); ++k) {
- if (kh_exist(hash, k)) {
- frag_t *f = &kh_val(hash, k);
- if (f->vpos < vpos) kh_del(64, hash, k); // TODO: if frag_t::seq is allocated dynamically, free it
- else f->vpos -= vpos;
- }
- }
-}
-
-static nseq_t *shrink_hash(nseq_t *hash) // TODO: to implement
-{
- return hash;
-}
-
-static int readaln(void *data, bam1_t *b)
-{
- phaseg_t *g = (phaseg_t*)data;
- int ret;
- ret = bam_read1(g->fp, b);
- if (ret < 0) return ret;
- if (!(b->core.flag & (BAM_FUNMAP|BAM_FSECONDARY|BAM_FQCFAIL|BAM_FDUP)) && g->pre) {
- if (g->n == g->m) {
- g->m = g->m? g->m<<1 : 16;
- g->b = realloc(g->b, g->m * sizeof(void*));
- }
- g->b[g->n++] = bam_dup1(b);
- }
- return ret;
-}
-
-static khash_t(set64) *loadpos(const char *fn, bam_header_t *h)
-{
- gzFile fp;
- kstream_t *ks;
- int ret, dret;
- kstring_t *str;
- khash_t(set64) *hash;
-
- hash = kh_init(set64);
- str = calloc(1, sizeof(kstring_t));
- fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, str, &dret) >= 0) {
- int tid = bam_get_tid(h, str->s);
- if (tid >= 0 && dret != '\n') {
- if (ks_getuntil(ks, 0, str, &dret) >= 0) {
- uint64_t x = (uint64_t)tid<<32 | (atoi(str->s) - 1);
- kh_put(set64, hash, x, &ret);
- } else break;
- }
- if (dret != '\n') while ((dret = ks_getc(ks)) > 0 && dret != '\n');
- if (dret < 0) break;
- }
- ks_destroy(ks);
- gzclose(fp);
- free(str->s); free(str);
- return hash;
-}
-
-static int gl2cns(float q[16])
-{
- int i, j, min_ij;
- float min, min2;
- min = min2 = 1e30; min_ij = -1;
- for (i = 0; i < 4; ++i) {
- for (j = i; j < 4; ++j) {
- if (q[i<<2|j] < min) min_ij = i<<2|j, min2 = min, min = q[i<<2|j];
- else if (q[i<<2|j] < min2) min2 = q[i<<2|j];
- }
- }
- return (min_ij>>2&3) == (min_ij&3)? 0 : 1<<18 | (min_ij>>2&3)<<16 | (min_ij&3) | (int)(min2 - min + .499) << 2;
-}
-
-int main_phase(int argc, char *argv[])
-{
- extern void bam_init_header_hash(bam_header_t *header);
- int c, tid, pos, vpos = 0, n, lasttid = -1, max_vpos = 0;
- const bam_pileup1_t *plp;
- bam_plp_t iter;
- bam_header_t *h;
- nseq_t *seqs;
- uint64_t *cns = 0;
- phaseg_t g;
- char *fn_list = 0;
- khash_t(set64) *set = 0;
- errmod_t *em;
- uint16_t *bases;
-
- memset(&g, 0, sizeof(phaseg_t));
- g.flag = FLAG_FIX_CHIMERA;
- g.min_varLOD = 37; g.k = 13; g.min_baseQ = 13; g.max_depth = 256;
- while ((c = getopt(argc, argv, "Q:eFq:k:b:l:D:A:")) >= 0) {
- switch (c) {
- case 'D': g.max_depth = atoi(optarg); break;
- case 'q': g.min_varLOD = atoi(optarg); break;
- case 'Q': g.min_baseQ = atoi(optarg); break;
- case 'k': g.k = atoi(optarg); break;
- case 'F': g.flag &= ~FLAG_FIX_CHIMERA; break;
- case 'e': g.flag |= FLAG_LIST_EXCL; break;
- case 'A': g.flag |= FLAG_DROP_AMBI; break;
- case 'b': g.pre = strdup(optarg); break;
- case 'l': fn_list = strdup(optarg); break;
- }
- }
- if (argc == optind) {
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools phase [options] <in.bam>\n\n");
- fprintf(stderr, "Options: -k INT block length [%d]\n", g.k);
- fprintf(stderr, " -b STR prefix of BAMs to output [null]\n");
- fprintf(stderr, " -q INT min het phred-LOD [%d]\n", g.min_varLOD);
- fprintf(stderr, " -Q INT min base quality in het calling [%d]\n", g.min_baseQ);
- fprintf(stderr, " -D INT max read depth [%d]\n", g.max_depth);
-// fprintf(stderr, " -l FILE list of sites to phase [null]\n");
- fprintf(stderr, " -F do not attempt to fix chimeras\n");
- fprintf(stderr, " -A drop reads with ambiguous phase\n");
-// fprintf(stderr, " -e do not discover SNPs (effective with -l)\n");
- fprintf(stderr, "\n");
- return 1;
- }
- g.fp = strcmp(argv[optind], "-")? bam_open(argv[optind], "r") : bam_dopen(fileno(stdin), "r");
- h = bam_header_read(g.fp);
- if (fn_list) { // read the list of sites to phase
- bam_init_header_hash(h);
- set = loadpos(fn_list, h);
- free(fn_list);
- } else g.flag &= ~FLAG_LIST_EXCL;
- if (g.pre) { // open BAMs to write
- char *s = malloc(strlen(g.pre) + 20);
- strcpy(s, g.pre); strcat(s, ".0.bam"); g.out[0] = bam_open(s, "w");
- strcpy(s, g.pre); strcat(s, ".1.bam"); g.out[1] = bam_open(s, "w");
- strcpy(s, g.pre); strcat(s, ".chimera.bam"); g.out[2] = bam_open(s, "w");
- for (c = 0; c <= 2; ++c) bam_header_write(g.out[c], h);
- free(s);
- }
-
- iter = bam_plp_init(readaln, &g);
- g.vpos_shift = 0;
- seqs = kh_init(64);
- em = errmod_init(1. - 0.83);
- bases = calloc(g.max_depth, 2);
- printf("CC\n");
- printf("CC\tDescriptions:\nCC\n");
- printf("CC\t CC comments\n");
- printf("CC\t PS start of a phase set\n");
- printf("CC\t FL filtered region\n");
- printf("CC\t M[012] markers; 0 for singletons, 1 for phased and 2 for filtered\n");
- printf("CC\t EV supporting reads; SAM format\n");
- printf("CC\t // end of a phase set\nCC\n");
- printf("CC\tFormats of PS, FL and M[012] lines (1-based coordinates):\nCC\n");
- printf("CC\t PS chr phaseSetStart phaseSetEnd\n");
- printf("CC\t FL chr filterStart filterEnd\n");
- printf("CC\t M? chr PS pos allele0 allele1 hetIndex #supports0 #errors0 #supp1 #err1\n");
- printf("CC\nCC\n");
- fflush(stdout);
- while ((plp = bam_plp_auto(iter, &tid, &pos, &n)) != 0) {
- int i, k, c, tmp, dophase = 1, in_set = 0;
- float q[16];
- if (tid < 0) break;
- if (tid != lasttid) { // change of chromosome
- g.vpos_shift = 0;
- if (lasttid >= 0) {
- seqs = shrink_hash(seqs);
- phase(&g, h->target_name[lasttid], vpos, cns, seqs);
- update_vpos(0x7fffffff, seqs);
- }
- lasttid = tid;
- vpos = 0;
- }
- if (set && kh_get(set64, set, (uint64_t)tid<<32 | pos) != kh_end(set)) in_set = 1;
- if (n > g.max_depth) continue; // do not proceed if the depth is too high
- // fill the bases array and check if there is a variant
- for (i = k = 0; i < n; ++i) {
- const bam_pileup1_t *p = plp + i;
- uint8_t *seq;
- int q, baseQ, b;
- if (p->is_del || p->is_refskip) continue;
- baseQ = bam1_qual(p->b)[p->qpos];
- if (baseQ < g.min_baseQ) continue;
- seq = bam1_seq(p->b);
- b = bam_nt16_nt4_table[bam1_seqi(seq, p->qpos)];
- if (b > 3) continue;
- q = baseQ < p->b->core.qual? baseQ : p->b->core.qual;
- if (q < 4) q = 4;
- if (q > 63) q = 63;
- bases[k++] = q<<5 | (int)bam1_strand(p->b)<<4 | b;
- }
- if (k == 0) continue;
- errmod_cal(em, k, 4, bases, q); // compute genotype likelihood
- c = gl2cns(q); // get the consensus
- // tell if to proceed
- if (set && (g.flag&FLAG_LIST_EXCL) && !in_set) continue; // not in the list
- if (!in_set && (c&0xffff)>>2 < g.min_varLOD) continue; // not a variant
- // add the variant
- if (vpos == max_vpos) {
- max_vpos = max_vpos? max_vpos<<1 : 128;
- cns = realloc(cns, max_vpos * 8);
- }
- cns[vpos] = (uint64_t)pos<<32 | c;
- for (i = 0; i < n; ++i) {
- const bam_pileup1_t *p = plp + i;
- uint64_t key;
- khint_t k;
- uint8_t *seq = bam1_seq(p->b);
- frag_t *f;
- if (p->is_del || p->is_refskip) continue;
- if (p->b->core.qual == 0) continue;
- // get the base code
- c = nt16_nt4_table[(int)bam1_seqi(seq, p->qpos)];
- if (c == (cns[vpos]&3)) c = 1;
- else if (c == (cns[vpos]>>16&3)) c = 2;
- else c = 0;
- // write to seqs
- key = X31_hash_string(bam1_qname(p->b));
- k = kh_put(64, seqs, key, &tmp);
- f = &kh_val(seqs, k);
- if (tmp == 0) { // present in the hash table
- if (vpos - f->vpos + 1 < MAX_VARS) {
- f->vlen = vpos - f->vpos + 1;
- f->seq[f->vlen-1] = c;
- f->end = bam_calend(&p->b->core, bam1_cigar(p->b));
- }
- dophase = 0;
- } else { // absent
- memset(f->seq, 0, MAX_VARS);
- f->beg = p->b->core.pos;
- f->end = bam_calend(&p->b->core, bam1_cigar(p->b));
- f->vpos = vpos, f->vlen = 1, f->seq[0] = c, f->single = f->phased = f->flip = f->ambig = 0;
- }
- }
- if (dophase) {
- seqs = shrink_hash(seqs);
- phase(&g, h->target_name[tid], vpos, cns, seqs);
- update_vpos(vpos, seqs);
- cns[0] = cns[vpos];
- vpos = 0;
- }
- ++vpos;
- }
- if (tid >= 0) phase(&g, h->target_name[tid], vpos, cns, seqs);
- bam_header_destroy(h);
- bam_plp_destroy(iter);
- bam_close(g.fp);
- kh_destroy(64, seqs);
- kh_destroy(set64, set);
- free(cns);
- errmod_destroy(em);
- free(bases);
- if (g.pre) {
- for (c = 0; c <= 2; ++c) bam_close(g.out[c]);
- free(g.pre); free(g.b);
- }
- return 0;
-}
diff --git a/samtools-0.1.19/razf.c b/samtools-0.1.19/razf.c
deleted file mode 100644
index e7499f9..0000000
--- a/samtools-0.1.19/razf.c
+++ /dev/null
@@ -1,853 +0,0 @@
-/*
- * RAZF : Random Access compressed(Z) File
- * Version: 1.0
- * Release Date: 2008-10-27
- *
- * Copyright 2008, Jue Ruan <ruanjue at gmail.com>, Heng Li <lh3 at sanger.ac.uk>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef _NO_RAZF
-
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "razf.h"
-
-
-#if ZLIB_VERNUM < 0x1221
-struct _gz_header_s {
- int text;
- uLong time;
- int xflags;
- int os;
- Bytef *extra;
- uInt extra_len;
- uInt extra_max;
- Bytef *name;
- uInt name_max;
- Bytef *comment;
- uInt comm_max;
- int hcrc;
- int done;
-};
-#warning "zlib < 1.2.2.1; RAZF writing is disabled."
-#endif
-
-#define DEF_MEM_LEVEL 8
-
-static inline uint32_t byte_swap_4(uint32_t v){
- v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
- return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
-}
-
-static inline uint64_t byte_swap_8(uint64_t v){
- v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
- v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
- return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
-}
-
-static inline int is_big_endian(){
- int x = 0x01;
- char *c = (char*)&x;
- return (c[0] != 0x01);
-}
-
-#ifndef _RZ_READONLY
-static void add_zindex(RAZF *rz, int64_t in, int64_t out){
- if(rz->index->size == rz->index->cap){
- rz->index->cap = rz->index->cap * 1.5 + 2;
- rz->index->cell_offsets = realloc(rz->index->cell_offsets, sizeof(int) * rz->index->cap);
- rz->index->bin_offsets = realloc(rz->index->bin_offsets, sizeof(int64_t) * (rz->index->cap/RZ_BIN_SIZE + 1));
- }
- if(rz->index->size % RZ_BIN_SIZE == 0) rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE] = out;
- rz->index->cell_offsets[rz->index->size] = out - rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE];
- rz->index->size ++;
-}
-
-static void save_zindex(RAZF *rz, int fd){
- int32_t i, v32;
- int is_be;
- is_be = is_big_endian();
- if(is_be) write(fd, &rz->index->size, sizeof(int));
- else {
- v32 = byte_swap_4((uint32_t)rz->index->size);
- write(fd, &v32, sizeof(uint32_t));
- }
- v32 = rz->index->size / RZ_BIN_SIZE + 1;
- if(!is_be){
- for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
- for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
- }
- write(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
- write(fd, rz->index->cell_offsets, sizeof(int32_t) * rz->index->size);
-}
-#endif
-
-#ifdef _USE_KNETFILE
-static void load_zindex(RAZF *rz, knetFile *fp){
-#else
-static void load_zindex(RAZF *rz, int fd){
-#endif
- int32_t i, v32;
- int is_be;
- if(!rz->load_index) return;
- if(rz->index == NULL) rz->index = malloc(sizeof(ZBlockIndex));
- is_be = is_big_endian();
-#ifdef _USE_KNETFILE
- knet_read(fp, &rz->index->size, sizeof(int));
-#else
- read(fd, &rz->index->size, sizeof(int));
-#endif
- if(!is_be) rz->index->size = byte_swap_4((uint32_t)rz->index->size);
- rz->index->cap = rz->index->size;
- v32 = rz->index->size / RZ_BIN_SIZE + 1;
- rz->index->bin_offsets = malloc(sizeof(int64_t) * v32);
-#ifdef _USE_KNETFILE
- knet_read(fp, rz->index->bin_offsets, sizeof(int64_t) * v32);
-#else
- read(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
-#endif
- rz->index->cell_offsets = malloc(sizeof(int) * rz->index->size);
-#ifdef _USE_KNETFILE
- knet_read(fp, rz->index->cell_offsets, sizeof(int) * rz->index->size);
-#else
- read(fd, rz->index->cell_offsets, sizeof(int) * rz->index->size);
-#endif
- if(!is_be){
- for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
- for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
- }
-}
-
-#ifdef _RZ_READONLY
-static RAZF* razf_open_w(int fd)
-{
- fprintf(stderr, "[razf_open_w] Writing is not available with zlib ver < 1.2.2.1\n");
- return 0;
-}
-#else
-static RAZF* razf_open_w(int fd){
- RAZF *rz;
-#ifdef _WIN32
- setmode(fd, O_BINARY);
-#endif
- rz = calloc(1, sizeof(RAZF));
- rz->mode = 'w';
-#ifdef _USE_KNETFILE
- rz->x.fpw = fd;
-#else
- rz->filedes = fd;
-#endif
- rz->stream = calloc(sizeof(z_stream), 1);
- rz->inbuf = malloc(RZ_BUFFER_SIZE);
- rz->outbuf = malloc(RZ_BUFFER_SIZE);
- rz->index = calloc(sizeof(ZBlockIndex), 1);
- deflateInit2(rz->stream, RZ_COMPRESS_LEVEL, Z_DEFLATED, WINDOW_BITS + 16, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
- rz->stream->avail_out = RZ_BUFFER_SIZE;
- rz->stream->next_out = rz->outbuf;
- rz->header = calloc(sizeof(gz_header), 1);
- rz->header->os = 0x03; //Unix
- rz->header->text = 0;
- rz->header->time = 0;
- rz->header->extra = malloc(7);
- strncpy((char*)rz->header->extra, "RAZF", 4);
- rz->header->extra[4] = 1; // obsolete field
- // block size = RZ_BLOCK_SIZE, Big-Endian
- rz->header->extra[5] = RZ_BLOCK_SIZE >> 8;
- rz->header->extra[6] = RZ_BLOCK_SIZE & 0xFF;
- rz->header->extra_len = 7;
- rz->header->name = rz->header->comment = 0;
- rz->header->hcrc = 0;
- deflateSetHeader(rz->stream, rz->header);
- rz->block_pos = rz->block_off = 0;
- return rz;
-}
-
-static void _razf_write(RAZF* rz, const void *data, int size){
- int tout;
- rz->stream->avail_in = size;
- rz->stream->next_in = (void*)data;
- while(1){
- tout = rz->stream->avail_out;
- deflate(rz->stream, Z_NO_FLUSH);
- rz->out += tout - rz->stream->avail_out;
- if(rz->stream->avail_out) break;
-#ifdef _USE_KNETFILE
- write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#else
- write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#endif
- rz->stream->avail_out = RZ_BUFFER_SIZE;
- rz->stream->next_out = rz->outbuf;
- if(rz->stream->avail_in == 0) break;
- };
- rz->in += size - rz->stream->avail_in;
- rz->block_off += size - rz->stream->avail_in;
-}
-
-static void razf_flush(RAZF *rz){
- uint32_t tout;
- if(rz->buf_len){
- _razf_write(rz, rz->inbuf, rz->buf_len);
- rz->buf_off = rz->buf_len = 0;
- }
- if(rz->stream->avail_out){
-#ifdef _USE_KNETFILE
- write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#else
- write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#endif
- rz->stream->avail_out = RZ_BUFFER_SIZE;
- rz->stream->next_out = rz->outbuf;
- }
- while(1){
- tout = rz->stream->avail_out;
- deflate(rz->stream, Z_FULL_FLUSH);
- rz->out += tout - rz->stream->avail_out;
- if(rz->stream->avail_out == 0){
-#ifdef _USE_KNETFILE
- write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#else
- write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#endif
- rz->stream->avail_out = RZ_BUFFER_SIZE;
- rz->stream->next_out = rz->outbuf;
- } else break;
- }
- rz->block_pos = rz->out;
- rz->block_off = 0;
-}
-
-static void razf_end_flush(RAZF *rz){
- uint32_t tout;
- if(rz->buf_len){
- _razf_write(rz, rz->inbuf, rz->buf_len);
- rz->buf_off = rz->buf_len = 0;
- }
- while(1){
- tout = rz->stream->avail_out;
- deflate(rz->stream, Z_FINISH);
- rz->out += tout - rz->stream->avail_out;
- if(rz->stream->avail_out < RZ_BUFFER_SIZE){
-#ifdef _USE_KNETFILE
- write(rz->x.fpw, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#else
- write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
-#endif
- rz->stream->avail_out = RZ_BUFFER_SIZE;
- rz->stream->next_out = rz->outbuf;
- } else break;
- }
-}
-
-static void _razf_buffered_write(RAZF *rz, const void *data, int size){
- int i, n;
- while(1){
- if(rz->buf_len == RZ_BUFFER_SIZE){
- _razf_write(rz, rz->inbuf, rz->buf_len);
- rz->buf_len = 0;
- }
- if(size + rz->buf_len < RZ_BUFFER_SIZE){
- for(i=0;i<size;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
- rz->buf_len += size;
- return;
- } else {
- n = RZ_BUFFER_SIZE - rz->buf_len;
- for(i=0;i<n;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
- size -= n;
- data += n;
- rz->buf_len += n;
- }
- }
-}
-
-int razf_write(RAZF* rz, const void *data, int size){
- int ori_size, n;
- int64_t next_block;
- ori_size = size;
- next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
- while(rz->in + rz->buf_len + size >= next_block){
- n = next_block - rz->in - rz->buf_len;
- _razf_buffered_write(rz, data, n);
- data += n;
- size -= n;
- razf_flush(rz);
- add_zindex(rz, rz->in, rz->out);
- next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
- }
- _razf_buffered_write(rz, data, size);
- return ori_size;
-}
-#endif
-
-/* gzip flag byte */
-#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
-#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
-#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME 0x08 /* bit 3 set: original file name present */
-#define COMMENT 0x10 /* bit 4 set: file comment present */
-#define RESERVED 0xE0 /* bits 5..7: reserved */
-
-static int _read_gz_header(unsigned char *data, int size, int *extra_off, int *extra_len){
- int method, flags, n, len;
- if(size < 2) return 0;
- if(data[0] != 0x1f || data[1] != 0x8b) return 0;
- if(size < 4) return 0;
- method = data[2];
- flags = data[3];
- if(method != Z_DEFLATED || (flags & RESERVED)) return 0;
- n = 4 + 6; // Skip 6 bytes
- *extra_off = n + 2;
- *extra_len = 0;
- if(flags & EXTRA_FIELD){
- if(size < n + 2) return 0;
- len = ((int)data[n + 1] << 8) | data[n];
- n += 2;
- *extra_off = n;
- while(len){
- if(n >= size) return 0;
- n ++;
- len --;
- }
- *extra_len = n - (*extra_off);
- }
- if(flags & ORIG_NAME) while(n < size && data[n++]);
- if(flags & COMMENT) while(n < size && data[n++]);
- if(flags & HEAD_CRC){
- if(n + 2 > size) return 0;
- n += 2;
- }
- return n;
-}
-
-#ifdef _USE_KNETFILE
-static RAZF* razf_open_r(knetFile *fp, int _load_index){
-#else
-static RAZF* razf_open_r(int fd, int _load_index){
-#endif
- RAZF *rz;
- int ext_off, ext_len;
- int n, is_be, ret;
- int64_t end;
- unsigned char c[] = "RAZF";
- rz = calloc(1, sizeof(RAZF));
- rz->mode = 'r';
-#ifdef _USE_KNETFILE
- rz->x.fpr = fp;
-#else
-#ifdef _WIN32
- setmode(fd, O_BINARY);
-#endif
- rz->filedes = fd;
-#endif
- rz->stream = calloc(sizeof(z_stream), 1);
- rz->inbuf = malloc(RZ_BUFFER_SIZE);
- rz->outbuf = malloc(RZ_BUFFER_SIZE);
- rz->end = rz->src_end = 0x7FFFFFFFFFFFFFFFLL;
-#ifdef _USE_KNETFILE
- n = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE);
-#else
- n = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
-#endif
- ret = _read_gz_header(rz->inbuf, n, &ext_off, &ext_len);
- if(ret == 0){
- PLAIN_FILE:
- rz->in = n;
- rz->file_type = FILE_TYPE_PLAIN;
- memcpy(rz->outbuf, rz->inbuf, n);
- rz->buf_len = n;
- free(rz->stream);
- rz->stream = NULL;
- return rz;
- }
- rz->header_size = ret;
- ret = inflateInit2(rz->stream, -WINDOW_BITS);
- if(ret != Z_OK){ inflateEnd(rz->stream); goto PLAIN_FILE;}
- rz->stream->avail_in = n - rz->header_size;
- rz->stream->next_in = rz->inbuf + rz->header_size;
- rz->stream->avail_out = RZ_BUFFER_SIZE;
- rz->stream->next_out = rz->outbuf;
- rz->file_type = FILE_TYPE_GZ;
- rz->in = rz->header_size;
- rz->block_pos = rz->header_size;
- rz->next_block_pos = rz->header_size;
- rz->block_off = 0;
- if(ext_len < 7 || memcmp(rz->inbuf + ext_off, c, 4) != 0) return rz;
- if(((((unsigned char*)rz->inbuf)[ext_off + 5] << 8) | ((unsigned char*)rz->inbuf)[ext_off + 6]) != RZ_BLOCK_SIZE){
- fprintf(stderr, " -- WARNING: RZ_BLOCK_SIZE is not %d, treat source as gz file. in %s -- %s:%d --\n", RZ_BLOCK_SIZE, __FUNCTION__, __FILE__, __LINE__);
- return rz;
- }
- rz->load_index = _load_index;
- rz->file_type = FILE_TYPE_RZ;
-#ifdef _USE_KNETFILE
- if(knet_seek(fp, -16, SEEK_END) == -1){
-#else
- if(lseek(fd, -16, SEEK_END) == -1){
-#endif
- UNSEEKABLE:
- rz->seekable = 0;
- rz->index = NULL;
- rz->src_end = rz->end = 0x7FFFFFFFFFFFFFFFLL;
- } else {
- is_be = is_big_endian();
- rz->seekable = 1;
-#ifdef _USE_KNETFILE
- knet_read(fp, &end, sizeof(int64_t));
-#else
- read(fd, &end, sizeof(int64_t));
-#endif
- if(!is_be) rz->src_end = (int64_t)byte_swap_8((uint64_t)end);
- else rz->src_end = end;
-
-#ifdef _USE_KNETFILE
- knet_read(fp, &end, sizeof(int64_t));
-#else
- read(fd, &end, sizeof(int64_t));
-#endif
- if(!is_be) rz->end = (int64_t)byte_swap_8((uint64_t)end);
- else rz->end = end;
- if(n > rz->end){
- rz->stream->avail_in -= n - rz->end;
- n = rz->end;
- }
- if(rz->end > rz->src_end){
-#ifdef _USE_KNETFILE
- knet_seek(fp, rz->in, SEEK_SET);
-#else
- lseek(fd, rz->in, SEEK_SET);
-#endif
- goto UNSEEKABLE;
- }
-#ifdef _USE_KNETFILE
- knet_seek(fp, rz->end, SEEK_SET);
- if(knet_tell(fp) != rz->end){
- knet_seek(fp, rz->in, SEEK_SET);
-#else
- if(lseek(fd, rz->end, SEEK_SET) != rz->end){
- lseek(fd, rz->in, SEEK_SET);
-#endif
- goto UNSEEKABLE;
- }
-#ifdef _USE_KNETFILE
- load_zindex(rz, fp);
- knet_seek(fp, n, SEEK_SET);
-#else
- load_zindex(rz, fd);
- lseek(fd, n, SEEK_SET);
-#endif
- }
- return rz;
-}
-
-#ifdef _USE_KNETFILE
-RAZF* razf_dopen(int fd, const char *mode){
- if (strstr(mode, "r")) fprintf(stderr,"[razf_dopen] implement me\n");
- else if(strstr(mode, "w")) return razf_open_w(fd);
- return NULL;
-}
-
-RAZF* razf_dopen2(int fd, const char *mode)
-{
- fprintf(stderr,"[razf_dopen2] implement me\n");
- return NULL;
-}
-#else
-RAZF* razf_dopen(int fd, const char *mode){
- if(strstr(mode, "r")) return razf_open_r(fd, 1);
- else if(strstr(mode, "w")) return razf_open_w(fd);
- else return NULL;
-}
-
-RAZF* razf_dopen2(int fd, const char *mode)
-{
- if(strstr(mode, "r")) return razf_open_r(fd, 0);
- else if(strstr(mode, "w")) return razf_open_w(fd);
- else return NULL;
-}
-#endif
-
-static inline RAZF* _razf_open(const char *filename, const char *mode, int _load_index){
- int fd;
- RAZF *rz;
- if(strstr(mode, "r")){
-#ifdef _USE_KNETFILE
- knetFile *fd = knet_open(filename, "r");
- if (fd == 0) {
- fprintf(stderr, "[_razf_open] fail to open %s\n", filename);
- return NULL;
- }
-#else
-#ifdef _WIN32
- fd = open(filename, O_RDONLY | O_BINARY);
-#else
- fd = open(filename, O_RDONLY);
-#endif
-#endif
- if(fd < 0) return NULL;
- rz = razf_open_r(fd, _load_index);
- } else if(strstr(mode, "w")){
-#ifdef _WIN32
- fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
-#else
- fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666);
-#endif
- if(fd < 0) return NULL;
- rz = razf_open_w(fd);
- } else return NULL;
- return rz;
-}
-
-RAZF* razf_open(const char *filename, const char *mode){
- return _razf_open(filename, mode, 1);
-}
-
-RAZF* razf_open2(const char *filename, const char *mode){
- return _razf_open(filename, mode, 0);
-}
-
-int razf_get_data_size(RAZF *rz, int64_t *u_size, int64_t *c_size){
- int64_t n;
- if(rz->mode != 'r' && rz->mode != 'R') return 0;
- switch(rz->file_type){
- case FILE_TYPE_PLAIN:
- if(rz->end == 0x7fffffffffffffffLL){
-#ifdef _USE_KNETFILE
- if(knet_seek(rz->x.fpr, 0, SEEK_CUR) == -1) return 0;
- n = knet_tell(rz->x.fpr);
- knet_seek(rz->x.fpr, 0, SEEK_END);
- rz->end = knet_tell(rz->x.fpr);
- knet_seek(rz->x.fpr, n, SEEK_SET);
-#else
- if((n = lseek(rz->filedes, 0, SEEK_CUR)) == -1) return 0;
- rz->end = lseek(rz->filedes, 0, SEEK_END);
- lseek(rz->filedes, n, SEEK_SET);
-#endif
- }
- *u_size = *c_size = rz->end;
- return 1;
- case FILE_TYPE_GZ:
- return 0;
- case FILE_TYPE_RZ:
- if(rz->src_end == rz->end) return 0;
- *u_size = rz->src_end;
- *c_size = rz->end;
- return 1;
- default:
- return 0;
- }
-}
-
-static int _razf_read(RAZF* rz, void *data, int size){
- int ret, tin;
- if(rz->z_eof || rz->z_err) return 0;
- if (rz->file_type == FILE_TYPE_PLAIN) {
-#ifdef _USE_KNETFILE
- ret = knet_read(rz->x.fpr, data, size);
-#else
- ret = read(rz->filedes, data, size);
-#endif
- if (ret == 0) rz->z_eof = 1;
- return ret;
- }
- rz->stream->avail_out = size;
- rz->stream->next_out = data;
- while(rz->stream->avail_out){
- if(rz->stream->avail_in == 0){
- if(rz->in >= rz->end){ rz->z_eof = 1; break; }
- if(rz->end - rz->in < RZ_BUFFER_SIZE){
-#ifdef _USE_KNETFILE
- rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, rz->end -rz->in);
-#else
- rz->stream->avail_in = read(rz->filedes, rz->inbuf, rz->end -rz->in);
-#endif
- } else {
-#ifdef _USE_KNETFILE
- rz->stream->avail_in = knet_read(rz->x.fpr, rz->inbuf, RZ_BUFFER_SIZE);
-#else
- rz->stream->avail_in = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
-#endif
- }
- if(rz->stream->avail_in == 0){
- rz->z_eof = 1;
- break;
- }
- rz->stream->next_in = rz->inbuf;
- }
- tin = rz->stream->avail_in;
- ret = inflate(rz->stream, Z_BLOCK);
- rz->in += tin - rz->stream->avail_in;
- if(ret == Z_NEED_DICT || ret == Z_MEM_ERROR || ret == Z_DATA_ERROR){
- fprintf(stderr, "[_razf_read] inflate error: %d %s (at %s:%d)\n", ret, rz->stream->msg ? rz->stream->msg : "", __FILE__, __LINE__);
- rz->z_err = 1;
- break;
- }
- if(ret == Z_STREAM_END){
- rz->z_eof = 1;
- break;
- }
- if ((rz->stream->data_type&128) && !(rz->stream->data_type&64)){
- rz->buf_flush = 1;
- rz->next_block_pos = rz->in;
- break;
- }
- }
- return size - rz->stream->avail_out;
-}
-
-int razf_read(RAZF *rz, void *data, int size){
- int ori_size, i;
- ori_size = size;
- while(size > 0){
- if(rz->buf_len){
- if(size < rz->buf_len){
- for(i=0;i<size;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
- rz->buf_off += size;
- rz->buf_len -= size;
- data += size;
- rz->block_off += size;
- size = 0;
- break;
- } else {
- for(i=0;i<rz->buf_len;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
- data += rz->buf_len;
- size -= rz->buf_len;
- rz->block_off += rz->buf_len;
- rz->buf_off = 0;
- rz->buf_len = 0;
- if(rz->buf_flush){
- rz->block_pos = rz->next_block_pos;
- rz->block_off = 0;
- rz->buf_flush = 0;
- }
- }
- } else if(rz->buf_flush){
- rz->block_pos = rz->next_block_pos;
- rz->block_off = 0;
- rz->buf_flush = 0;
- }
- if(rz->buf_flush) continue;
- rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
- if(rz->z_eof && rz->buf_len == 0) break;
- }
- rz->out += ori_size - size;
- return ori_size - size;
-}
-
-int razf_skip(RAZF* rz, int size){
- int ori_size;
- ori_size = size;
- while(size > 0){
- if(rz->buf_len){
- if(size < rz->buf_len){
- rz->buf_off += size;
- rz->buf_len -= size;
- rz->block_off += size;
- size = 0;
- break;
- } else {
- size -= rz->buf_len;
- rz->buf_off = 0;
- rz->buf_len = 0;
- rz->block_off += rz->buf_len;
- if(rz->buf_flush){
- rz->block_pos = rz->next_block_pos;
- rz->block_off = 0;
- rz->buf_flush = 0;
- }
- }
- } else if(rz->buf_flush){
- rz->block_pos = rz->next_block_pos;
- rz->block_off = 0;
- rz->buf_flush = 0;
- }
- if(rz->buf_flush) continue;
- rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
- if(rz->z_eof || rz->z_err) break;
- }
- rz->out += ori_size - size;
- return ori_size - size;
-}
-
-static void _razf_reset_read(RAZF *rz, int64_t in, int64_t out){
-#ifdef _USE_KNETFILE
- knet_seek(rz->x.fpr, in, SEEK_SET);
-#else
- lseek(rz->filedes, in, SEEK_SET);
-#endif
- rz->in = in;
- rz->out = out;
- rz->block_pos = in;
- rz->next_block_pos = in;
- rz->block_off = 0;
- rz->buf_flush = 0;
- rz->z_eof = rz->z_err = 0;
- inflateReset(rz->stream);
- rz->stream->avail_in = 0;
- rz->buf_off = rz->buf_len = 0;
-}
-
-int64_t razf_jump(RAZF *rz, int64_t block_start, int block_offset){
- int64_t pos;
- rz->z_eof = 0;
- if(rz->file_type == FILE_TYPE_PLAIN){
- rz->buf_off = rz->buf_len = 0;
- pos = block_start + block_offset;
-#ifdef _USE_KNETFILE
- knet_seek(rz->x.fpr, pos, SEEK_SET);
- pos = knet_tell(rz->x.fpr);
-#else
- pos = lseek(rz->filedes, pos, SEEK_SET);
-#endif
- rz->out = rz->in = pos;
- return pos;
- }
- if(block_start == rz->block_pos && block_offset >= rz->block_off) {
- block_offset -= rz->block_off;
- goto SKIP; // Needn't reset inflate
- }
- if(block_start == 0) block_start = rz->header_size; // Automaticly revist wrong block_start
- _razf_reset_read(rz, block_start, 0);
- SKIP:
- if(block_offset) razf_skip(rz, block_offset);
- return rz->block_off;
-}
-
-int64_t razf_seek(RAZF* rz, int64_t pos, int where){
- int64_t idx;
- int64_t seek_pos, new_out;
- rz->z_eof = 0;
- if (where == SEEK_CUR) pos += rz->out;
- else if (where == SEEK_END) pos += rz->src_end;
- if(rz->file_type == FILE_TYPE_PLAIN){
-#ifdef _USE_KNETFILE
- knet_seek(rz->x.fpr, pos, SEEK_SET);
- seek_pos = knet_tell(rz->x.fpr);
-#else
- seek_pos = lseek(rz->filedes, pos, SEEK_SET);
-#endif
- rz->buf_off = rz->buf_len = 0;
- rz->out = rz->in = seek_pos;
- return seek_pos;
- } else if(rz->file_type == FILE_TYPE_GZ){
- if(pos >= rz->out) goto SKIP;
- return rz->out;
- }
- if(pos == rz->out) return pos;
- if(pos > rz->src_end) return rz->out;
- if(!rz->seekable || !rz->load_index){
- if(pos >= rz->out) goto SKIP;
- }
- idx = pos / RZ_BLOCK_SIZE - 1;
- seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
- new_out = (idx + 1) * RZ_BLOCK_SIZE;
- if(pos > rz->out && new_out <= rz->out) goto SKIP;
- _razf_reset_read(rz, seek_pos, new_out);
- SKIP:
- razf_skip(rz, (int)(pos - rz->out));
- return rz->out;
-}
-
-uint64_t razf_tell2(RAZF *rz)
-{
- /*
- if (rz->load_index) {
- int64_t idx, seek_pos;
- idx = rz->out / RZ_BLOCK_SIZE - 1;
- seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
- if (seek_pos != rz->block_pos || rz->out%RZ_BLOCK_SIZE != rz->block_off)
- fprintf(stderr, "[razf_tell2] inconsistent block offset: (%lld, %lld) != (%lld, %lld)\n",
- (long long)seek_pos, (long long)rz->out%RZ_BLOCK_SIZE, (long long)rz->block_pos, (long long) rz->block_off);
- }
- */
- return (uint64_t)rz->block_pos<<16 | (rz->block_off&0xffff);
-}
-
-int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where)
-{
- if (where != SEEK_SET) return -1;
- return razf_jump(rz, voffset>>16, voffset&0xffff);
-}
-
-void razf_close(RAZF *rz){
- if(rz->mode == 'w'){
-#ifndef _RZ_READONLY
- razf_end_flush(rz);
- deflateEnd(rz->stream);
-#ifdef _USE_KNETFILE
- save_zindex(rz, rz->x.fpw);
- if(is_big_endian()){
- write(rz->x.fpw, &rz->in, sizeof(int64_t));
- write(rz->x.fpw, &rz->out, sizeof(int64_t));
- } else {
- uint64_t v64 = byte_swap_8((uint64_t)rz->in);
- write(rz->x.fpw, &v64, sizeof(int64_t));
- v64 = byte_swap_8((uint64_t)rz->out);
- write(rz->x.fpw, &v64, sizeof(int64_t));
- }
-#else
- save_zindex(rz, rz->filedes);
- if(is_big_endian()){
- write(rz->filedes, &rz->in, sizeof(int64_t));
- write(rz->filedes, &rz->out, sizeof(int64_t));
- } else {
- uint64_t v64 = byte_swap_8((uint64_t)rz->in);
- write(rz->filedes, &v64, sizeof(int64_t));
- v64 = byte_swap_8((uint64_t)rz->out);
- write(rz->filedes, &v64, sizeof(int64_t));
- }
-#endif
-#endif
- } else if(rz->mode == 'r'){
- if(rz->stream) inflateEnd(rz->stream);
- }
- if(rz->inbuf) free(rz->inbuf);
- if(rz->outbuf) free(rz->outbuf);
- if(rz->header){
- free(rz->header->extra);
- free(rz->header->name);
- free(rz->header->comment);
- free(rz->header);
- }
- if(rz->index){
- free(rz->index->bin_offsets);
- free(rz->index->cell_offsets);
- free(rz->index);
- }
- free(rz->stream);
-#ifdef _USE_KNETFILE
- if (rz->mode == 'r')
- knet_close(rz->x.fpr);
- if (rz->mode == 'w')
- close(rz->x.fpw);
-#else
- close(rz->filedes);
-#endif
- free(rz);
-}
-
-#endif
diff --git a/samtools-0.1.19/razf.h b/samtools-0.1.19/razf.h
deleted file mode 100644
index 60a0c96..0000000
--- a/samtools-0.1.19/razf.h
+++ /dev/null
@@ -1,134 +0,0 @@
- /*-
- * RAZF : Random Access compressed(Z) File
- * Version: 1.0
- * Release Date: 2008-10-27
- *
- * Copyright 2008, Jue Ruan <ruanjue at gmail.com>, Heng Li <lh3 at sanger.ac.uk>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-
-#ifndef __RAZF_RJ_H
-#define __RAZF_RJ_H
-
-#include <stdint.h>
-#include <stdio.h>
-#include "zlib.h"
-
-#ifdef _USE_KNETFILE
-#include "knetfile.h"
-#endif
-
-#if ZLIB_VERNUM < 0x1221
-#define _RZ_READONLY
-struct _gz_header_s;
-typedef struct _gz_header_s _gz_header;
-#define gz_header _gz_header
-#endif
-
-#define WINDOW_BITS 15
-
-#ifndef RZ_BLOCK_SIZE
-#define RZ_BLOCK_SIZE (1<<WINDOW_BITS)
-#endif
-
-#ifndef RZ_BUFFER_SIZE
-#define RZ_BUFFER_SIZE 4096
-#endif
-
-#ifndef RZ_COMPRESS_LEVEL
-#define RZ_COMPRESS_LEVEL 6
-#endif
-
-#define RZ_BIN_SIZE ((1LLU << 32) / RZ_BLOCK_SIZE)
-
-typedef struct {
- uint32_t *cell_offsets; // i
- int64_t *bin_offsets; // i / BIN_SIZE
- int size;
- int cap;
-} ZBlockIndex;
-/* When storing index, output bytes in Big-Endian everywhere */
-
-#define FILE_TYPE_RZ 1
-#define FILE_TYPE_PLAIN 2
-#define FILE_TYPE_GZ 3
-
-typedef struct RandomAccessZFile {
- char mode; /* 'w' : write mode; 'r' : read mode */
- int file_type;
- /* plain file or rz file, razf_read support plain file as input too, in this case, razf_read work as buffered fread */
-#ifdef _USE_KNETFILE
- union {
- knetFile *fpr;
- int fpw;
- } x;
-#else
- int filedes; /* the file descriptor */
-#endif
- z_stream *stream;
- ZBlockIndex *index;
- int64_t in, out, end, src_end;
- /* in: n bytes total in; out: n bytes total out; */
- /* end: the end of all data blocks, while the start of index; src_end: the true end position in uncompressed file */
- int buf_flush; // buffer should be flush, suspend inflate util buffer is empty
- int64_t block_pos, block_off, next_block_pos;
- /* block_pos: the start postiion of current block in compressed file */
- /* block_off: tell how many bytes have been read from current block */
- void *inbuf, *outbuf;
- int header_size;
- gz_header *header;
- /* header is used to transfer inflate_state->mode from HEAD to TYPE after call inflateReset */
- int buf_off, buf_len;
- int z_err, z_eof;
- int seekable;
- /* Indice where the source is seekable */
- int load_index;
- /* set has_index to 0 in mode 'w', then index will be discarded */
-} RAZF;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- RAZF* razf_dopen(int data_fd, const char *mode);
- RAZF *razf_open(const char *fn, const char *mode);
- int razf_write(RAZF* rz, const void *data, int size);
- int razf_read(RAZF* rz, void *data, int size);
- int64_t razf_seek(RAZF* rz, int64_t pos, int where);
- void razf_close(RAZF* rz);
-
-#define razf_tell(rz) ((rz)->out)
-
- RAZF* razf_open2(const char *filename, const char *mode);
- RAZF* razf_dopen2(int fd, const char *mode);
- uint64_t razf_tell2(RAZF *rz);
- int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/razip.c b/samtools-0.1.19/razip.c
deleted file mode 100644
index 825e732..0000000
--- a/samtools-0.1.19/razip.c
+++ /dev/null
@@ -1,141 +0,0 @@
-#include <stdio.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <string.h>
-#include "razf.h"
-
-#define WINDOW_SIZE 4096
-
-static int razf_main_usage()
-{
- printf("\n");
- printf("Usage: razip [options] [file] ...\n\n");
- printf("Options: -c write on standard output, keep original files unchanged\n");
- printf(" -d decompress\n");
- printf(" -l list compressed file contents\n");
- printf(" -b INT decompress at INT position in the uncompressed file\n");
- printf(" -s INT decompress INT bytes in the uncompressed file\n");
- printf(" -h give this help\n");
- printf("\n");
- return 0;
-}
-
-static int write_open(const char *fn, int is_forced)
-{
- int fd = -1;
- char c;
- if (!is_forced) {
- if ((fd = open(fn, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0666)) < 0 && errno == EEXIST) {
- printf("razip: %s already exists; do you wish to overwrite (y or n)? ", fn);
- scanf("%c", &c);
- if (c != 'Y' && c != 'y') {
- printf("razip: not overwritten\n");
- exit(1);
- }
- }
- }
- if (fd < 0) {
- if ((fd = open(fn, O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0) {
- fprintf(stderr, "razip: %s: Fail to write\n", fn);
- exit(1);
- }
- }
- return fd;
-}
-
-int main(int argc, char **argv)
-{
- int c, compress, pstdout, is_forced;
- RAZF *rz;
- void *buffer;
- long start, end, size;
-
- compress = 1; pstdout = 0; start = 0; size = -1; end = -1; is_forced = 0;
- while((c = getopt(argc, argv, "cdlhfb:s:")) >= 0){
- switch(c){
- case 'h': return razf_main_usage();
- case 'd': compress = 0; break;
- case 'c': pstdout = 1; break;
- case 'l': compress = 2; break;
- case 'b': start = atol(optarg); break;
- case 's': size = atol(optarg); break;
- case 'f': is_forced = 1; break;
- }
- }
- if (size >= 0) end = start + size;
- if(end >= 0 && end < start){
- fprintf(stderr, " -- Illegal region: [%ld, %ld] --\n", start, end);
- return 1;
- }
- if(compress == 1){
- int f_src, f_dst = -1;
- if(argc > optind){
- if((f_src = open(argv[optind], O_RDONLY)) < 0){
- fprintf(stderr, " -- Cannot open file: %s --\n", argv[optind]);
- return 1;
- }
- if(pstdout){
- f_dst = fileno(stdout);
- } else {
- char *name = malloc(sizeof(strlen(argv[optind]) + 5));
- strcpy(name, argv[optind]);
- strcat(name, ".rz");
- f_dst = write_open(name, is_forced);
- if (f_dst < 0) return 1;
- free(name);
- }
- } else if(pstdout){
- f_src = fileno(stdin);
- f_dst = fileno(stdout);
- } else return razf_main_usage();
- rz = razf_dopen(f_dst, "w");
- buffer = malloc(WINDOW_SIZE);
- while((c = read(f_src, buffer, WINDOW_SIZE)) > 0) razf_write(rz, buffer, c);
- razf_close(rz); // f_dst will be closed here
- if (argc > optind && !pstdout) unlink(argv[optind]);
- free(buffer);
- close(f_src);
- return 0;
- } else {
- if(argc <= optind) return razf_main_usage();
- if(compress == 2){
- rz = razf_open(argv[optind], "r");
- if(rz->file_type == FILE_TYPE_RZ) {
- printf("%20s%20s%7s %s\n", "compressed", "uncompressed", "ratio", "name");
- printf("%20lld%20lld%6.1f%% %s\n", (long long)rz->end, (long long)rz->src_end, rz->end * 100.0f / rz->src_end,
- argv[optind]);
- } else fprintf(stdout, "%s is not a regular rz file\n", argv[optind]);
- } else {
- int f_dst;
- if (argc > optind && !pstdout) {
- char *name;
- if (strstr(argv[optind], ".rz") - argv[optind] != strlen(argv[optind]) - 3) {
- printf("razip: %s: unknown suffix -- ignored\n", argv[optind]);
- return 1;
- }
- name = strdup(argv[optind]);
- name[strlen(name) - 3] = '\0';
- f_dst = write_open(name, is_forced);
- free(name);
- } else f_dst = fileno(stdout);
- rz = razf_open(argv[optind], "r");
- buffer = malloc(WINDOW_SIZE);
- razf_seek(rz, start, SEEK_SET);
- while(1){
- if(end < 0) c = razf_read(rz, buffer, WINDOW_SIZE);
- else c = razf_read(rz, buffer, (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start));
- if(c <= 0) break;
- start += c;
- write(f_dst, buffer, c);
- if(end >= 0 && start >= end) break;
- }
- free(buffer);
- if (!pstdout) unlink(argv[optind]);
- }
- razf_close(rz);
- return 0;
- }
-}
-
diff --git a/samtools-0.1.19/sam.c b/samtools-0.1.19/sam.c
deleted file mode 100644
index fa11df6..0000000
--- a/samtools-0.1.19/sam.c
+++ /dev/null
@@ -1,186 +0,0 @@
-#include <string.h>
-#include <unistd.h>
-#include "faidx.h"
-#include "sam.h"
-
-#define TYPE_BAM 1
-#define TYPE_READ 2
-
-bam_header_t *bam_header_dup(const bam_header_t *h0)
-{
- bam_header_t *h;
- int i;
- h = bam_header_init();
- *h = *h0;
- h->hash = h->dict = h->rg2lib = 0;
- h->text = (char*)calloc(h->l_text + 1, 1);
- memcpy(h->text, h0->text, h->l_text);
- h->target_len = (uint32_t*)calloc(h->n_targets, 4);
- h->target_name = (char**)calloc(h->n_targets, sizeof(void*));
- for (i = 0; i < h->n_targets; ++i) {
- h->target_len[i] = h0->target_len[i];
- h->target_name[i] = strdup(h0->target_name[i]);
- }
- return h;
-}
-static void append_header_text(bam_header_t *header, char* text, int len)
-{
- int x = header->l_text + 1;
- int y = header->l_text + len + 1; // 1 byte null
- if (text == 0) return;
- kroundup32(x);
- kroundup32(y);
- if (x < y) header->text = (char*)realloc(header->text, y);
- strncpy(header->text + header->l_text, text, len); // we cannot use strcpy() here.
- header->l_text += len;
- header->text[header->l_text] = 0;
-}
-
-int samthreads(samfile_t *fp, int n_threads, int n_sub_blks)
-{
- if (!(fp->type&1) || (fp->type&2)) return -1;
- bgzf_mt(fp->x.bam, n_threads, n_sub_blks);
- return 0;
-}
-
-samfile_t *samopen(const char *fn, const char *mode, const void *aux)
-{
- samfile_t *fp;
- fp = (samfile_t*)calloc(1, sizeof(samfile_t));
- if (strchr(mode, 'r')) { // read
- fp->type |= TYPE_READ;
- if (strchr(mode, 'b')) { // binary
- fp->type |= TYPE_BAM;
- fp->x.bam = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r");
- if (fp->x.bam == 0) goto open_err_ret;
- fp->header = bam_header_read(fp->x.bam);
- } else { // text
- fp->x.tamr = sam_open(fn);
- if (fp->x.tamr == 0) goto open_err_ret;
- fp->header = sam_header_read(fp->x.tamr);
- if (fp->header->n_targets == 0) { // no @SQ fields
- if (aux) { // check if aux is present
- bam_header_t *textheader = fp->header;
- fp->header = sam_header_read2((const char*)aux);
- if (fp->header == 0) goto open_err_ret;
- append_header_text(fp->header, textheader->text, textheader->l_text);
- bam_header_destroy(textheader);
- }
- if (fp->header->n_targets == 0 && bam_verbose >= 1)
- fprintf(stderr, "[samopen] no @SQ lines in the header.\n");
- } else if (bam_verbose >= 2) fprintf(stderr, "[samopen] SAM header is present: %d sequences.\n", fp->header->n_targets);
- }
- } else if (strchr(mode, 'w')) { // write
- fp->header = bam_header_dup((const bam_header_t*)aux);
- if (strchr(mode, 'b')) { // binary
- char bmode[3];
- int i, compress_level = -1;
- for (i = 0; mode[i]; ++i) if (mode[i] >= '0' && mode[i] <= '9') break;
- if (mode[i]) compress_level = mode[i] - '0';
- if (strchr(mode, 'u')) compress_level = 0;
- bmode[0] = 'w'; bmode[1] = compress_level < 0? 0 : compress_level + '0'; bmode[2] = 0;
- fp->type |= TYPE_BAM;
- fp->x.bam = strcmp(fn, "-")? bam_open(fn, bmode) : bam_dopen(fileno(stdout), bmode);
- if (fp->x.bam == 0) goto open_err_ret;
- bam_header_write(fp->x.bam, fp->header);
- } else { // text
- // open file
- fp->x.tamw = strcmp(fn, "-")? fopen(fn, "w") : stdout;
- if (fp->x.tamw == 0) goto open_err_ret;
- if (strchr(mode, 'X')) fp->type |= BAM_OFSTR<<2;
- else if (strchr(mode, 'x')) fp->type |= BAM_OFHEX<<2;
- else fp->type |= BAM_OFDEC<<2;
- // write header
- if (strchr(mode, 'h')) {
- int i;
- bam_header_t *alt;
- // parse the header text
- alt = bam_header_init();
- alt->l_text = fp->header->l_text; alt->text = fp->header->text;
- sam_header_parse(alt);
- alt->l_text = 0; alt->text = 0;
- // check if there are @SQ lines in the header
- fwrite(fp->header->text, 1, fp->header->l_text, fp->x.tamw); // FIXME: better to skip the trailing NULL
- if (alt->n_targets) { // then write the header text without dumping ->target_{name,len}
- if (alt->n_targets != fp->header->n_targets && bam_verbose >= 1)
- fprintf(stderr, "[samopen] inconsistent number of target sequences. Output the text header.\n");
- } else { // then dump ->target_{name,len}
- for (i = 0; i < fp->header->n_targets; ++i)
- fprintf(fp->x.tamw, "@SQ\tSN:%s\tLN:%d\n", fp->header->target_name[i], fp->header->target_len[i]);
- }
- bam_header_destroy(alt);
- }
- }
- }
- return fp;
-
-open_err_ret:
- free(fp);
- return 0;
-}
-
-void samclose(samfile_t *fp)
-{
- if (fp == 0) return;
- if (fp->header) bam_header_destroy(fp->header);
- if (fp->type & TYPE_BAM) bam_close(fp->x.bam);
- else if (fp->type & TYPE_READ) sam_close(fp->x.tamr);
- else fclose(fp->x.tamw);
- free(fp);
-}
-
-int samread(samfile_t *fp, bam1_t *b)
-{
- if (fp == 0 || !(fp->type & TYPE_READ)) return -1; // not open for reading
- if (fp->type & TYPE_BAM) return bam_read1(fp->x.bam, b);
- else return sam_read1(fp->x.tamr, fp->header, b);
-}
-
-int samwrite(samfile_t *fp, const bam1_t *b)
-{
- if (fp == 0 || (fp->type & TYPE_READ)) return -1; // not open for writing
- if (fp->type & TYPE_BAM) return bam_write1(fp->x.bam, b);
- else {
- char *s = bam_format1_core(fp->header, b, fp->type>>2&3);
- int l = strlen(s);
- fputs(s, fp->x.tamw); fputc('\n', fp->x.tamw);
- free(s);
- return l + 1;
- }
-}
-
-int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *func_data)
-{
- bam_plbuf_t *buf;
- int ret;
- bam1_t *b;
- b = bam_init1();
- buf = bam_plbuf_init(func, func_data);
- bam_plbuf_set_mask(buf, mask);
- while ((ret = samread(fp, b)) >= 0)
- bam_plbuf_push(b, buf);
- bam_plbuf_push(0, buf);
- bam_plbuf_destroy(buf);
- bam_destroy1(b);
- return 0;
-}
-
-char *samfaipath(const char *fn_ref)
-{
- char *fn_list = 0;
- if (fn_ref == 0) return 0;
- fn_list = calloc(strlen(fn_ref) + 5, 1);
- strcat(strcpy(fn_list, fn_ref), ".fai");
- if (access(fn_list, R_OK) == -1) { // fn_list is unreadable
- if (access(fn_ref, R_OK) == -1) {
- fprintf(stderr, "[samfaipath] fail to read file %s.\n", fn_ref);
- } else {
- if (bam_verbose >= 3) fprintf(stderr, "[samfaipath] build FASTA index...\n");
- if (fai_build(fn_ref) == -1) {
- fprintf(stderr, "[samfaipath] fail to build FASTA index.\n");
- free(fn_list); fn_list = 0;
- }
- }
- }
- return fn_list;
-}
diff --git a/samtools-0.1.19/sam.h b/samtools-0.1.19/sam.h
deleted file mode 100644
index 0495501..0000000
--- a/samtools-0.1.19/sam.h
+++ /dev/null
@@ -1,99 +0,0 @@
-#ifndef BAM_SAM_H
-#define BAM_SAM_H
-
-#include "bam.h"
-
-/*!
- @header
-
- This file provides higher level of I/O routines and unifies the APIs
- for SAM and BAM formats. These APIs are more convenient and
- recommended.
-
- @copyright Genome Research Ltd.
- */
-
-/*! @typedef
- @abstract SAM/BAM file handler
- @field type type of the handler; bit 1 for BAM, 2 for reading and bit 3-4 for flag format
- @field bam BAM file handler; valid if (type&1) == 1
- @field tamr SAM file handler for reading; valid if type == 2
- @field tamw SAM file handler for writing; valid if type == 0
- @field header header struct
- */
-typedef struct {
- int type;
- union {
- tamFile tamr;
- bamFile bam;
- FILE *tamw;
- } x;
- bam_header_t *header;
-} samfile_t;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- /*!
- @abstract Open a SAM/BAM file
-
- @param fn SAM/BAM file name; "-" is recognized as stdin (for
- reading) or stdout (for writing).
-
- @param mode open mode /[rw](b?)(u?)(h?)([xX]?)/: 'r' for reading,
- 'w' for writing, 'b' for BAM I/O, 'u' for uncompressed BAM output,
- 'h' for outputing header in SAM, 'x' for HEX flag and 'X' for
- string flag. If 'b' present, it must immediately follow 'r' or
- 'w'. Valid modes are "r", "w", "wh", "wx", "whx", "wX", "whX",
- "rb", "wb" and "wbu" exclusively.
-
- @param aux auxiliary data; if mode[0]=='w', aux points to
- bam_header_t; if strcmp(mode, "rb")!=0 and @SQ header lines in SAM
- are absent, aux points the file name of the list of the reference;
- aux is not used otherwise. If @SQ header lines are present in SAM,
- aux is not used, either.
-
- @return SAM/BAM file handler
- */
- samfile_t *samopen(const char *fn, const char *mode, const void *aux);
-
- /*!
- @abstract Close a SAM/BAM handler
- @param fp file handler to be closed
- */
- void samclose(samfile_t *fp);
-
- /*!
- @abstract Read one alignment
- @param fp file handler
- @param b alignment
- @return bytes read
- */
- int samread(samfile_t *fp, bam1_t *b);
-
- /*!
- @abstract Write one alignment
- @param fp file handler
- @param b alignment
- @return bytes written
- */
- int samwrite(samfile_t *fp, const bam1_t *b);
-
- /*!
- @abstract Get the pileup for a whole alignment file
- @param fp file handler
- @param mask mask transferred to bam_plbuf_set_mask()
- @param func user defined function called in the pileup process
- #param data user provided data for func()
- */
- int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *data);
-
- char *samfaipath(const char *fn_ref);
- int samthreads(samfile_t *fp, int n_threads, int n_sub_blks);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/sam_header.c b/samtools-0.1.19/sam_header.c
deleted file mode 100644
index 88b6a1c..0000000
--- a/samtools-0.1.19/sam_header.c
+++ /dev/null
@@ -1,810 +0,0 @@
-#include "sam_header.h"
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdarg.h>
-
-#include "khash.h"
-KHASH_MAP_INIT_STR(str, const char *)
-
-struct _HeaderList
-{
- struct _HeaderList *last; // Hack: Used and maintained only by list_append_to_end. Maintained in the root node only.
- struct _HeaderList *next;
- void *data;
-};
-typedef struct _HeaderList list_t;
-typedef list_t HeaderDict;
-
-typedef struct
-{
- char key[2];
- char *value;
-}
-HeaderTag;
-
-typedef struct
-{
- char type[2];
- list_t *tags;
-}
-HeaderLine;
-
-const char *o_hd_tags[] = {"SO","GO",NULL};
-const char *r_hd_tags[] = {"VN",NULL};
-
-const char *o_sq_tags[] = {"AS","M5","UR","SP",NULL};
-const char *r_sq_tags[] = {"SN","LN",NULL};
-const char *u_sq_tags[] = {"SN",NULL};
-
-const char *o_rg_tags[] = {"CN","DS","DT","FO","KS","LB","PG","PI","PL","PU","SM",NULL};
-const char *r_rg_tags[] = {"ID",NULL};
-const char *u_rg_tags[] = {"ID",NULL};
-
-const char *o_pg_tags[] = {"VN","CL",NULL};
-const char *r_pg_tags[] = {"ID",NULL};
-
-const char *types[] = {"HD","SQ","RG","PG","CO",NULL};
-const char **optional_tags[] = {o_hd_tags,o_sq_tags,o_rg_tags,o_pg_tags,NULL,NULL};
-const char **required_tags[] = {r_hd_tags,r_sq_tags,r_rg_tags,r_pg_tags,NULL,NULL};
-const char **unique_tags[] = {NULL, u_sq_tags,u_rg_tags,NULL,NULL,NULL};
-
-
-static void debug(const char *format, ...)
-{
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
-}
-
-#if 0
-// Replaced by list_append_to_end
-static list_t *list_prepend(list_t *root, void *data)
-{
- list_t *l = malloc(sizeof(list_t));
- l->next = root;
- l->data = data;
- return l;
-}
-#endif
-
-// Relies on the root->last being correct. Do not use with the other list_*
-// routines unless they are fixed to modify root->last as well.
-static list_t *list_append_to_end(list_t *root, void *data)
-{
- list_t *l = malloc(sizeof(list_t));
- l->last = l;
- l->next = NULL;
- l->data = data;
-
- if ( !root )
- return l;
-
- root->last->next = l;
- root->last = l;
- return root;
-}
-
-static list_t *list_append(list_t *root, void *data)
-{
- list_t *l = root;
- while (l && l->next)
- l = l->next;
- if ( l )
- {
- l->next = malloc(sizeof(list_t));
- l = l->next;
- }
- else
- {
- l = malloc(sizeof(list_t));
- root = l;
- }
- l->data = data;
- l->next = NULL;
- return root;
-}
-
-static void list_free(list_t *root)
-{
- list_t *l = root;
- while (root)
- {
- l = root;
- root = root->next;
- free(l);
- }
-}
-
-
-
-// Look for a tag "XY" in a predefined const char *[] array.
-static int tag_exists(const char *tag, const char **tags)
-{
- int itag=0;
- if ( !tags ) return -1;
- while ( tags[itag] )
- {
- if ( tags[itag][0]==tag[0] && tags[itag][1]==tag[1] ) return itag;
- itag++;
- }
- return -1;
-}
-
-
-
-// Mimics the behaviour of getline, except it returns pointer to the next chunk of the text
-// or NULL if everything has been read. The lineptr should be freed by the caller. The
-// newline character is stripped.
-static const char *nextline(char **lineptr, size_t *n, const char *text)
-{
- int len;
- const char *to = text;
-
- if ( !*to ) return NULL;
-
- while ( *to && *to!='\n' && *to!='\r' ) to++;
- len = to - text + 1;
-
- if ( *to )
- {
- // Advance the pointer for the next call
- if ( *to=='\n' ) to++;
- else if ( *to=='\r' && *(to+1)=='\n' ) to+=2;
- }
- if ( !len )
- return to;
-
- if ( !*lineptr )
- {
- *lineptr = malloc(len);
- *n = len;
- }
- else if ( *n<len )
- {
- *lineptr = realloc(*lineptr, len);
- *n = len;
- }
- if ( !*lineptr ) {
- debug("[nextline] Insufficient memory!\n");
- return 0;
- }
-
- memcpy(*lineptr,text,len);
- (*lineptr)[len-1] = 0;
-
- return to;
-}
-
-// name points to "XY", value_from points to the first character of the value string and
-// value_to points to the last character of the value string.
-static HeaderTag *new_tag(const char *name, const char *value_from, const char *value_to)
-{
- HeaderTag *tag = malloc(sizeof(HeaderTag));
- int len = value_to-value_from+1;
-
- tag->key[0] = name[0];
- tag->key[1] = name[1];
- tag->value = malloc(len+1);
- memcpy(tag->value,value_from,len+1);
- tag->value[len] = 0;
- return tag;
-}
-
-static HeaderTag *header_line_has_tag(HeaderLine *hline, const char *key)
-{
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- if ( tag->key[0]==key[0] && tag->key[1]==key[1] ) return tag;
- tags = tags->next;
- }
- return NULL;
-}
-
-
-// Return codes:
-// 0 .. different types or unique tags differ or conflicting tags, cannot be merged
-// 1 .. all tags identical -> no need to merge, drop one
-// 2 .. the unique tags match and there are some conflicting tags (same tag, different value) -> error, cannot be merged nor duplicated
-// 3 .. there are some missing complementary tags and no unique conflict -> can be merged into a single line
-static int sam_header_compare_lines(HeaderLine *hline1, HeaderLine *hline2)
-{
- HeaderTag *t1, *t2;
-
- if ( hline1->type[0]!=hline2->type[0] || hline1->type[1]!=hline2->type[1] )
- return 0;
-
- int itype = tag_exists(hline1->type,types);
- if ( itype==-1 ) {
- debug("[sam_header_compare_lines] Unknown type [%c%c]\n", hline1->type[0],hline1->type[1]);
- return -1; // FIXME (lh3): error; I do not know how this will be handled in Petr's code
- }
-
- if ( unique_tags[itype] )
- {
- t1 = header_line_has_tag(hline1,unique_tags[itype][0]);
- t2 = header_line_has_tag(hline2,unique_tags[itype][0]);
- if ( !t1 || !t2 ) // this should never happen, the unique tags are required
- return 2;
-
- if ( strcmp(t1->value,t2->value) )
- return 0; // the unique tags differ, cannot be merged
- }
- if ( !required_tags[itype] && !optional_tags[itype] )
- {
- t1 = hline1->tags->data;
- t2 = hline2->tags->data;
- if ( !strcmp(t1->value,t2->value) ) return 1; // identical comments
- return 0;
- }
-
- int missing=0, itag=0;
- while ( required_tags[itype] && required_tags[itype][itag] )
- {
- t1 = header_line_has_tag(hline1,required_tags[itype][itag]);
- t2 = header_line_has_tag(hline2,required_tags[itype][itag]);
- if ( !t1 && !t2 )
- return 2; // this should never happen
- else if ( !t1 || !t2 )
- missing = 1; // there is some tag missing in one of the hlines
- else if ( strcmp(t1->value,t2->value) )
- {
- if ( unique_tags[itype] )
- return 2; // the lines have a matching unique tag but have a conflicting tag
-
- return 0; // the lines contain conflicting tags, cannot be merged
- }
- itag++;
- }
- itag = 0;
- while ( optional_tags[itype] && optional_tags[itype][itag] )
- {
- t1 = header_line_has_tag(hline1,optional_tags[itype][itag]);
- t2 = header_line_has_tag(hline2,optional_tags[itype][itag]);
- if ( !t1 && !t2 )
- {
- itag++;
- continue;
- }
- if ( !t1 || !t2 )
- missing = 1; // there is some tag missing in one of the hlines
- else if ( strcmp(t1->value,t2->value) )
- {
- if ( unique_tags[itype] )
- return 2; // the lines have a matching unique tag but have a conflicting tag
-
- return 0; // the lines contain conflicting tags, cannot be merged
- }
- itag++;
- }
- if ( missing ) return 3; // there are some missing complementary tags with no conflicts, can be merged
- return 1;
-}
-
-
-static HeaderLine *sam_header_line_clone(const HeaderLine *hline)
-{
- list_t *tags;
- HeaderLine *out = malloc(sizeof(HeaderLine));
- out->type[0] = hline->type[0];
- out->type[1] = hline->type[1];
- out->tags = NULL;
-
- tags = hline->tags;
- while (tags)
- {
- HeaderTag *old = tags->data;
-
- HeaderTag *new = malloc(sizeof(HeaderTag));
- new->key[0] = old->key[0];
- new->key[1] = old->key[1];
- new->value = strdup(old->value);
- out->tags = list_append(out->tags, new);
-
- tags = tags->next;
- }
- return out;
-}
-
-static int sam_header_line_merge_with(HeaderLine *out_hline, const HeaderLine *tmpl_hline)
-{
- list_t *tmpl_tags;
-
- if ( out_hline->type[0]!=tmpl_hline->type[0] || out_hline->type[1]!=tmpl_hline->type[1] )
- return 0;
-
- tmpl_tags = tmpl_hline->tags;
- while (tmpl_tags)
- {
- HeaderTag *tmpl_tag = tmpl_tags->data;
- HeaderTag *out_tag = header_line_has_tag(out_hline, tmpl_tag->key);
- if ( !out_tag )
- {
- HeaderTag *tag = malloc(sizeof(HeaderTag));
- tag->key[0] = tmpl_tag->key[0];
- tag->key[1] = tmpl_tag->key[1];
- tag->value = strdup(tmpl_tag->value);
- out_hline->tags = list_append(out_hline->tags,tag);
- }
- tmpl_tags = tmpl_tags->next;
- }
- return 1;
-}
-
-
-static HeaderLine *sam_header_line_parse(const char *headerLine)
-{
- HeaderLine *hline;
- HeaderTag *tag;
- const char *from, *to;
- from = headerLine;
-
- if ( *from != '@' ) {
- debug("[sam_header_line_parse] expected '@', got [%s]\n", headerLine);
- return 0;
- }
- to = ++from;
-
- while (*to && *to!='\t') to++;
- if ( to-from != 2 ) {
- debug("[sam_header_line_parse] expected '@XY', got [%s]\nHint: The header tags must be tab-separated.\n", headerLine);
- return 0;
- }
-
- hline = malloc(sizeof(HeaderLine));
- hline->type[0] = from[0];
- hline->type[1] = from[1];
- hline->tags = NULL;
-
- int itype = tag_exists(hline->type, types);
-
- from = to;
- while (*to && *to=='\t') to++;
- if ( to-from != 1 ) {
- debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
- free(hline);
- return 0;
- }
- from = to;
- while (*from)
- {
- while (*to && *to!='\t') to++;
-
- if ( !required_tags[itype] && !optional_tags[itype] )
- {
- // CO is a special case, it can contain anything, including tabs
- if ( *to ) { to++; continue; }
- tag = new_tag(" ",from,to-1);
- }
- else
- tag = new_tag(from,from+3,to-1);
-
- if ( header_line_has_tag(hline,tag->key) )
- debug("The tag '%c%c' present (at least) twice on line [%s]\n", tag->key[0],tag->key[1], headerLine);
- hline->tags = list_append(hline->tags, tag);
-
- from = to;
- while (*to && *to=='\t') to++;
- if ( *to && to-from != 1 ) {
- debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
- return 0;
- }
-
- from = to;
- }
- return hline;
-}
-
-
-// Must be of an existing type, all tags must be recognised and all required tags must be present
-static int sam_header_line_validate(HeaderLine *hline)
-{
- list_t *tags;
- HeaderTag *tag;
- int itype, itag;
-
- // Is the type correct?
- itype = tag_exists(hline->type, types);
- if ( itype==-1 )
- {
- debug("The type [%c%c] not recognised.\n", hline->type[0],hline->type[1]);
- return 0;
- }
-
- // Has all required tags?
- itag = 0;
- while ( required_tags[itype] && required_tags[itype][itag] )
- {
- if ( !header_line_has_tag(hline,required_tags[itype][itag]) )
- {
- debug("The tag [%c%c] required for [%c%c] not present.\n", required_tags[itype][itag][0],required_tags[itype][itag][1],
- hline->type[0],hline->type[1]);
- return 0;
- }
- itag++;
- }
-
- // Are all tags recognised?
- tags = hline->tags;
- while ( tags )
- {
- tag = tags->data;
- if ( !tag_exists(tag->key,required_tags[itype]) && !tag_exists(tag->key,optional_tags[itype]) )
- {
- // Lower case tags are user-defined values.
- if( !(islower(tag->key[0]) || islower(tag->key[1])) )
- {
- // Neither is lower case, but tag was not recognized.
- debug("Unknown tag [%c%c] for [%c%c].\n", tag->key[0],tag->key[1], hline->type[0],hline->type[1]);
- // return 0; // Even unknown tags are allowed - for forward compatibility with new attributes
- }
- // else - allow user defined tag
- }
- tags = tags->next;
- }
-
- return 1;
-}
-
-
-static void print_header_line(FILE *fp, HeaderLine *hline)
-{
- list_t *tags = hline->tags;
- HeaderTag *tag;
-
- fprintf(fp, "@%c%c", hline->type[0],hline->type[1]);
- while (tags)
- {
- tag = tags->data;
-
- fprintf(fp, "\t");
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- fprintf(fp, "%c%c:", tag->key[0],tag->key[1]);
- fprintf(fp, "%s", tag->value);
-
- tags = tags->next;
- }
- fprintf(fp,"\n");
-}
-
-
-static void sam_header_line_free(HeaderLine *hline)
-{
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- free(tag->value);
- free(tag);
- tags = tags->next;
- }
- list_free(hline->tags);
- free(hline);
-}
-
-void sam_header_free(void *_header)
-{
- HeaderDict *header = (HeaderDict*)_header;
- list_t *hlines = header;
- while (hlines)
- {
- sam_header_line_free(hlines->data);
- hlines = hlines->next;
- }
- list_free(header);
-}
-
-HeaderDict *sam_header_clone(const HeaderDict *dict)
-{
- HeaderDict *out = NULL;
- while (dict)
- {
- HeaderLine *hline = dict->data;
- out = list_append(out, sam_header_line_clone(hline));
- dict = dict->next;
- }
- return out;
-}
-
-// Returns a newly allocated string
-char *sam_header_write(const void *_header)
-{
- const HeaderDict *header = (const HeaderDict*)_header;
- char *out = NULL;
- int len=0, nout=0;
- const list_t *hlines;
-
- // Calculate the length of the string to allocate
- hlines = header;
- while (hlines)
- {
- len += 4; // @XY and \n
-
- HeaderLine *hline = hlines->data;
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- len += strlen(tag->value) + 1; // \t
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- len += strlen(tag->value) + 3; // XY:
- tags = tags->next;
- }
- hlines = hlines->next;
- }
-
- nout = 0;
- out = malloc(len+1);
- hlines = header;
- while (hlines)
- {
- HeaderLine *hline = hlines->data;
-
- nout += sprintf(out+nout,"@%c%c",hline->type[0],hline->type[1]);
-
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- nout += sprintf(out+nout,"\t");
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- nout += sprintf(out+nout,"%c%c:", tag->key[0],tag->key[1]);
- nout += sprintf(out+nout,"%s", tag->value);
- tags = tags->next;
- }
- hlines = hlines->next;
- nout += sprintf(out+nout,"\n");
- }
- out[len] = 0;
- return out;
-}
-
-void *sam_header_parse2(const char *headerText)
-{
- list_t *hlines = NULL;
- HeaderLine *hline;
- const char *text;
- char *buf=NULL;
- size_t nbuf = 0;
- int tovalidate = 0;
-
- if ( !headerText )
- return 0;
-
- text = headerText;
- while ( (text=nextline(&buf, &nbuf, text)) )
- {
- hline = sam_header_line_parse(buf);
- if ( hline && (!tovalidate || sam_header_line_validate(hline)) )
- // With too many (~250,000) reference sequences the header parsing was too slow with list_append.
- hlines = list_append_to_end(hlines, hline);
- else
- {
- if (hline) sam_header_line_free(hline);
- sam_header_free(hlines);
- if ( buf ) free(buf);
- return NULL;
- }
- }
- if ( buf ) free(buf);
-
- return hlines;
-}
-
-void *sam_header2tbl(const void *_dict, char type[2], char key_tag[2], char value_tag[2])
-{
- const HeaderDict *dict = (const HeaderDict*)_dict;
- const list_t *l = dict;
- khash_t(str) *tbl = kh_init(str);
- khiter_t k;
- int ret;
-
- if (_dict == 0) return tbl; // return an empty (not null) hash table
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key, *value;
- key = header_line_has_tag(hline,key_tag);
- value = header_line_has_tag(hline,value_tag);
- if ( !key || !value )
- {
- l = l->next;
- continue;
- }
-
- k = kh_get(str, tbl, key->value);
- if ( k != kh_end(tbl) )
- debug("[sam_header_lookup_table] They key %s not unique.\n", key->value);
- k = kh_put(str, tbl, key->value, &ret);
- kh_value(tbl, k) = value->value;
-
- l = l->next;
- }
- return tbl;
-}
-
-char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n)
-{
- const HeaderDict *dict = (const HeaderDict*)_dict;
- const list_t *l = dict;
- int max, n;
- char **ret;
-
- ret = 0; *_n = max = n = 0;
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key;
- key = header_line_has_tag(hline,key_tag);
- if ( !key )
- {
- l = l->next;
- continue;
- }
-
- if (n == max) {
- max = max? max<<1 : 4;
- ret = realloc(ret, max * sizeof(void*));
- }
- ret[n++] = key->value;
-
- l = l->next;
- }
- *_n = n;
- return ret;
-}
-
-void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **_key, const char **_value)
-{
- list_t *l = iter;
- if ( !l ) return NULL;
-
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key, *value;
- key = header_line_has_tag(hline,key_tag);
- value = header_line_has_tag(hline,value_tag);
- if ( !key && !value )
- {
- l = l->next;
- continue;
- }
-
- *_key = key->value;
- *_value = value->value;
- return l->next;
- }
- return l;
-}
-
-const char *sam_tbl_get(void *h, const char *key)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- khint_t k;
- k = kh_get(str, tbl, key);
- return k == kh_end(tbl)? 0 : kh_val(tbl, k);
-}
-
-int sam_tbl_size(void *h)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- return h? kh_size(tbl) : 0;
-}
-
-void sam_tbl_destroy(void *h)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- kh_destroy(str, tbl);
-}
-
-void *sam_header_merge(int n, const void **_dicts)
-{
- const HeaderDict **dicts = (const HeaderDict**)_dicts;
- HeaderDict *out_dict;
- int idict, status;
-
- if ( n<2 ) return NULL;
-
- out_dict = sam_header_clone(dicts[0]);
-
- for (idict=1; idict<n; idict++)
- {
- const list_t *tmpl_hlines = dicts[idict];
-
- while ( tmpl_hlines )
- {
- list_t *out_hlines = out_dict;
- int inserted = 0;
- while ( out_hlines )
- {
- status = sam_header_compare_lines(tmpl_hlines->data, out_hlines->data);
- if ( status==0 )
- {
- out_hlines = out_hlines->next;
- continue;
- }
-
- if ( status==2 )
- {
- print_header_line(stderr,tmpl_hlines->data);
- print_header_line(stderr,out_hlines->data);
- debug("Conflicting lines, cannot merge the headers.\n");
- return 0;
- }
- if ( status==3 )
- sam_header_line_merge_with(out_hlines->data, tmpl_hlines->data);
-
- inserted = 1;
- break;
- }
- if ( !inserted )
- out_dict = list_append(out_dict, sam_header_line_clone(tmpl_hlines->data));
-
- tmpl_hlines = tmpl_hlines->next;
- }
- }
-
- return out_dict;
-}
-
-char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n)
-{
- int nout = 0;
- char **out = NULL;
-
- *n = 0;
- list_t *l = (list_t *)dict;
- if ( !l ) return NULL;
-
- int i, ntags = 0;
- while ( tags[ntags] ) ntags++;
-
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
- out = (char**) realloc(out, sizeof(char*)*(nout+1)*ntags);
- for (i=0; i<ntags; i++)
- {
- HeaderTag *key = header_line_has_tag(hline, tags[i]);
- if ( !key )
- {
- out[nout*ntags+i] = NULL;
- continue;
- }
- out[nout*ntags+i] = key->value;
- }
- nout++;
- l = l->next;
- }
- *n = nout;
- return out;
-}
-
diff --git a/samtools-0.1.19/sam_header.h b/samtools-0.1.19/sam_header.h
deleted file mode 100644
index 4b0cb03..0000000
--- a/samtools-0.1.19/sam_header.h
+++ /dev/null
@@ -1,48 +0,0 @@
-#ifndef __SAM_HEADER_H__
-#define __SAM_HEADER_H__
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- void *sam_header_parse2(const char *headerText);
- void *sam_header_merge(int n, const void **dicts);
- void sam_header_free(void *header);
- char *sam_header_write(const void *headerDict); // returns a newly allocated string
-
- /*
- // Usage example
- const char *key, *val;
- void *iter = sam_header_parse2(bam->header->text);
- while ( iter = sam_header_key_val(iter, "RG","ID","SM" &key,&val) ) printf("%s\t%s\n", key,val);
- */
- void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **key, const char **value);
- char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n);
-
- /*
- // Usage example
- int i, j, n;
- const char *tags[] = {"SN","LN","UR","M5",NULL};
- void *dict = sam_header_parse2(bam->header->text);
- char **tbl = sam_header2tbl_n(h->dict, "SQ", tags, &n);
- for (i=0; i<n; i++)
- {
- for (j=0; j<4; j++)
- if ( tbl[4*i+j] ) printf("\t%s", tbl[4*i+j]);
- else printf("-");
- printf("\n");
- }
- if (tbl) free(tbl);
- */
- char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n);
-
- void *sam_header2tbl(const void *dict, char type[2], char key_tag[2], char value_tag[2]);
- const char *sam_tbl_get(void *h, const char *key);
- int sam_tbl_size(void *h);
- void sam_tbl_destroy(void *h);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/samtools-0.1.19/sam_view.c b/samtools-0.1.19/sam_view.c
deleted file mode 100644
index 7f3fdab..0000000
--- a/samtools-0.1.19/sam_view.c
+++ /dev/null
@@ -1,441 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <math.h>
-#include <inttypes.h>
-#include "sam_header.h"
-#include "sam.h"
-#include "faidx.h"
-#include "kstring.h"
-#include "khash.h"
-KHASH_SET_INIT_STR(rg)
-
-// When counting records instead of printing them,
-// data passed to the bam_fetch callback is encapsulated in this struct.
-typedef struct {
- bam_header_t *header;
- int64_t *count; // int does overflow for very big BAMs
-} count_func_data_t;
-
-typedef khash_t(rg) *rghash_t;
-
-// FIXME: we'd better use no global variables...
-static rghash_t g_rghash = 0;
-static int g_min_mapQ = 0, g_flag_on = 0, g_flag_off = 0, g_qual_scale = 0, g_min_qlen = 0;
-static uint32_t g_subsam_seed = 0;
-static double g_subsam_frac = -1.;
-static char *g_library, *g_rg;
-static void *g_bed;
-
-void *bed_read(const char *fn);
-void bed_destroy(void *_h);
-int bed_overlap(const void *_h, const char *chr, int beg, int end);
-
-static int process_aln(const bam_header_t *h, bam1_t *b)
-{
- if (g_qual_scale > 1) {
- int i;
- uint8_t *qual = bam1_qual(b);
- for (i = 0; i < b->core.l_qseq; ++i) {
- int c = qual[i] * g_qual_scale;
- qual[i] = c < 93? c : 93;
- }
- }
- if (g_min_qlen > 0) {
- int k, qlen = 0;
- uint32_t *cigar = bam1_cigar(b);
- for (k = 0; k < b->core.n_cigar; ++k)
- if ((bam_cigar_type(bam_cigar_op(cigar[k]))&1) || bam_cigar_op(cigar[k]) == BAM_CHARD_CLIP)
- qlen += bam_cigar_oplen(cigar[k]);
- if (qlen < g_min_qlen) return 1;
- }
- if (b->core.qual < g_min_mapQ || ((b->core.flag & g_flag_on) != g_flag_on) || (b->core.flag & g_flag_off))
- return 1;
- if (g_bed && b->core.tid >= 0 && !bed_overlap(g_bed, h->target_name[b->core.tid], b->core.pos, bam_calend(&b->core, bam1_cigar(b))))
- return 1;
- if (g_subsam_frac > 0.) {
- uint32_t k = __ac_X31_hash_string(bam1_qname(b)) + g_subsam_seed;
- if ((double)(k&0xffffff) / 0x1000000 >= g_subsam_frac) return 1;
- }
- if (g_rg || g_rghash) {
- uint8_t *s = bam_aux_get(b, "RG");
- if (s) {
- if (g_rg) return (strcmp(g_rg, (char*)(s + 1)) == 0)? 0 : 1;
- if (g_rghash) {
- khint_t k = kh_get(rg, g_rghash, (char*)(s + 1));
- return (k != kh_end(g_rghash))? 0 : 1;
- }
- }
- }
- if (g_library) {
- const char *p = bam_get_library((bam_header_t*)h, b);
- return (p && strcmp(p, g_library) == 0)? 0 : 1;
- }
- return 0;
-}
-
-static char *drop_rg(char *hdtxt, rghash_t h, int *len)
-{
- char *p = hdtxt, *q, *r, *s;
- kstring_t str;
- memset(&str, 0, sizeof(kstring_t));
- while (1) {
- int toprint = 0;
- q = strchr(p, '\n');
- if (q == 0) q = p + strlen(p);
- if (q - p < 3) break; // the line is too short; then stop
- if (strncmp(p, "@RG\t", 4) == 0) {
- int c;
- khint_t k;
- if ((r = strstr(p, "\tID:")) != 0) {
- r += 4;
- for (s = r; *s != '\0' && *s != '\n' && *s != '\t'; ++s);
- c = *s; *s = '\0';
- k = kh_get(rg, h, r);
- *s = c;
- if (k != kh_end(h)) toprint = 1;
- }
- } else toprint = 1;
- if (toprint) {
- kputsn(p, q - p, &str); kputc('\n', &str);
- }
- p = q + 1;
- }
- *len = str.l;
- return str.s;
-}
-
-// callback function for bam_fetch() that prints nonskipped records
-static int view_func(const bam1_t *b, void *data)
-{
- if (!process_aln(((samfile_t*)data)->header, (bam1_t*)b))
- samwrite((samfile_t*)data, b);
- return 0;
-}
-
-// callback function for bam_fetch() that counts nonskipped records
-static int count_func(const bam1_t *b, void *data)
-{
- if (!process_aln(((count_func_data_t*)data)->header, (bam1_t*)b)) {
- (*((count_func_data_t*)data)->count)++;
- }
- return 0;
-}
-
-static int usage(int is_long_help);
-
-int main_samview(int argc, char *argv[])
-{
- int c, is_header = 0, is_header_only = 0, is_bamin = 1, ret = 0, compress_level = -1, is_bamout = 0, is_count = 0;
- int of_type = BAM_OFDEC, is_long_help = 0, n_threads = 0;
- int64_t count = 0;
- samfile_t *in = 0, *out = 0;
- char in_mode[5], out_mode[5], *fn_out = 0, *fn_list = 0, *fn_ref = 0, *fn_rg = 0, *q;
-
- /* parse command-line options */
- strcpy(in_mode, "r"); strcpy(out_mode, "w");
- while ((c = getopt(argc, argv, "SbBct:h1Ho:q:f:F:ul:r:xX?T:R:L:s:Q:@:m:")) >= 0) {
- switch (c) {
- case 's':
- if ((g_subsam_seed = strtol(optarg, &q, 10)) != 0) {
- srand(g_subsam_seed);
- g_subsam_seed = rand();
- }
- g_subsam_frac = strtod(q, &q);
- break;
- case 'm': g_min_qlen = atoi(optarg); break;
- case 'c': is_count = 1; break;
- case 'S': is_bamin = 0; break;
- case 'b': is_bamout = 1; break;
- case 't': fn_list = strdup(optarg); is_bamin = 0; break;
- case 'h': is_header = 1; break;
- case 'H': is_header_only = 1; break;
- case 'o': fn_out = strdup(optarg); break;
- case 'f': g_flag_on = strtol(optarg, 0, 0); break;
- case 'F': g_flag_off = strtol(optarg, 0, 0); break;
- case 'q': g_min_mapQ = atoi(optarg); break;
- case 'u': compress_level = 0; break;
- case '1': compress_level = 1; break;
- case 'l': g_library = strdup(optarg); break;
- case 'L': g_bed = bed_read(optarg); break;
- case 'r': g_rg = strdup(optarg); break;
- case 'R': fn_rg = strdup(optarg); break;
- case 'x': of_type = BAM_OFHEX; break;
- case 'X': of_type = BAM_OFSTR; break;
- case '?': is_long_help = 1; break;
- case 'T': fn_ref = strdup(optarg); is_bamin = 0; break;
- case 'B': bam_no_B = 1; break;
- case 'Q': g_qual_scale = atoi(optarg); break;
- case '@': n_threads = strtol(optarg, 0, 0); break;
- default: return usage(is_long_help);
- }
- }
- if (compress_level >= 0) is_bamout = 1;
- if (is_header_only) is_header = 1;
- if (is_bamout) strcat(out_mode, "b");
- else {
- if (of_type == BAM_OFHEX) strcat(out_mode, "x");
- else if (of_type == BAM_OFSTR) strcat(out_mode, "X");
- }
- if (is_bamin) strcat(in_mode, "b");
- if (is_header) strcat(out_mode, "h");
- if (compress_level >= 0) {
- char tmp[2];
- tmp[0] = compress_level + '0'; tmp[1] = '\0';
- strcat(out_mode, tmp);
- }
- if (argc == optind) return usage(is_long_help); // potential memory leak...
-
- // read the list of read groups
- if (fn_rg) {
- FILE *fp_rg;
- char buf[1024];
- int ret;
- g_rghash = kh_init(rg);
- fp_rg = fopen(fn_rg, "r");
- while (!feof(fp_rg) && fscanf(fp_rg, "%s", buf) > 0) // this is not a good style, but bear me...
- kh_put(rg, g_rghash, strdup(buf), &ret); // we'd better check duplicates...
- fclose(fp_rg);
- }
-
- // generate the fn_list if necessary
- if (fn_list == 0 && fn_ref) fn_list = samfaipath(fn_ref);
- // open file handlers
- if ((in = samopen(argv[optind], in_mode, fn_list)) == 0) {
- fprintf(stderr, "[main_samview] fail to open \"%s\" for reading.\n", argv[optind]);
- ret = 1;
- goto view_end;
- }
- if (in->header == 0) {
- fprintf(stderr, "[main_samview] fail to read the header from \"%s\".\n", argv[optind]);
- ret = 1;
- goto view_end;
- }
- if (g_rghash) { // FIXME: I do not know what "bam_header_t::n_text" is for...
- char *tmp;
- int l;
- tmp = drop_rg(in->header->text, g_rghash, &l);
- free(in->header->text);
- in->header->text = tmp;
- in->header->l_text = l;
- }
- if (!is_count && (out = samopen(fn_out? fn_out : "-", out_mode, in->header)) == 0) {
- fprintf(stderr, "[main_samview] fail to open \"%s\" for writing.\n", fn_out? fn_out : "standard output");
- ret = 1;
- goto view_end;
- }
- if (n_threads > 1) samthreads(out, n_threads, 256);
- if (is_header_only) goto view_end; // no need to print alignments
-
- if (argc == optind + 1) { // convert/print the entire file
- bam1_t *b = bam_init1();
- int r;
- while ((r = samread(in, b)) >= 0) { // read one alignment from `in'
- if (!process_aln(in->header, b)) {
- if (!is_count) samwrite(out, b); // write the alignment to `out'
- count++;
- }
- }
- if (r < -1) {
- fprintf(stderr, "[main_samview] truncated file.\n");
- ret = 1;
- }
- bam_destroy1(b);
- } else { // retrieve alignments in specified regions
- int i;
- bam_index_t *idx = 0;
- if (is_bamin) idx = bam_index_load(argv[optind]); // load BAM index
- if (idx == 0) { // index is unavailable
- fprintf(stderr, "[main_samview] random alignment retrieval only works for indexed BAM files.\n");
- ret = 1;
- goto view_end;
- }
- for (i = optind + 1; i < argc; ++i) {
- int tid, beg, end, result;
- bam_parse_region(in->header, argv[i], &tid, &beg, &end); // parse a region in the format like `chr2:100-200'
- if (tid < 0) { // reference name is not found
- fprintf(stderr, "[main_samview] region \"%s\" specifies an unknown reference name. Continue anyway.\n", argv[i]);
- continue;
- }
- // fetch alignments
- if (is_count) {
- count_func_data_t count_data = { in->header, &count };
- result = bam_fetch(in->x.bam, idx, tid, beg, end, &count_data, count_func);
- } else
- result = bam_fetch(in->x.bam, idx, tid, beg, end, out, view_func);
- if (result < 0) {
- fprintf(stderr, "[main_samview] retrieval of region \"%s\" failed due to truncated file or corrupt BAM index file\n", argv[i]);
- ret = 1;
- break;
- }
- }
- bam_index_destroy(idx); // destroy the BAM index
- }
-
-view_end:
- if (is_count && ret == 0)
- printf("%" PRId64 "\n", count);
-
- // close files, free and return
- free(fn_list); free(fn_ref); free(fn_out); free(g_library); free(g_rg); free(fn_rg);
- if (g_bed) bed_destroy(g_bed);
- if (g_rghash) {
- khint_t k;
- for (k = 0; k < kh_end(g_rghash); ++k)
- if (kh_exist(g_rghash, k)) free((char*)kh_key(g_rghash, k));
- kh_destroy(rg, g_rghash);
- }
- samclose(in);
- if (!is_count)
- samclose(out);
- return ret;
-}
-
-static int usage(int is_long_help)
-{
- fprintf(stderr, "\n");
- fprintf(stderr, "Usage: samtools view [options] <in.bam>|<in.sam> [region1 [...]]\n\n");
- fprintf(stderr, "Options: -b output BAM\n");
- fprintf(stderr, " -h print header for the SAM output\n");
- fprintf(stderr, " -H print header only (no alignments)\n");
- fprintf(stderr, " -S input is SAM\n");
- fprintf(stderr, " -u uncompressed BAM output (force -b)\n");
- fprintf(stderr, " -1 fast compression (force -b)\n");
- fprintf(stderr, " -x output FLAG in HEX (samtools-C specific)\n");
- fprintf(stderr, " -X output FLAG in string (samtools-C specific)\n");
- fprintf(stderr, " -c print only the count of matching records\n");
- fprintf(stderr, " -B collapse the backward CIGAR operation\n");
- fprintf(stderr, " -@ INT number of BAM compression threads [0]\n");
- fprintf(stderr, " -L FILE output alignments overlapping the input BED FILE [null]\n");
- fprintf(stderr, " -t FILE list of reference names and lengths (force -S) [null]\n");
- fprintf(stderr, " -T FILE reference sequence file (force -S) [null]\n");
- fprintf(stderr, " -o FILE output file name [stdout]\n");
- fprintf(stderr, " -R FILE list of read groups to be outputted [null]\n");
- fprintf(stderr, " -f INT required flag, 0 for unset [0]\n");
- fprintf(stderr, " -F INT filtering flag, 0 for unset [0]\n");
- fprintf(stderr, " -q INT minimum mapping quality [0]\n");
- fprintf(stderr, " -l STR only output reads in library STR [null]\n");
- fprintf(stderr, " -r STR only output reads in read group STR [null]\n");
- fprintf(stderr, " -s FLOAT fraction of templates to subsample; integer part as seed [-1]\n");
- fprintf(stderr, " -? longer help\n");
- fprintf(stderr, "\n");
- if (is_long_help)
- fprintf(stderr, "Notes:\n\
-\n\
- 1. By default, this command assumes the file on the command line is in\n\
- the BAM format and it prints the alignments in SAM. If `-t' is\n\
- applied, the input file is assumed to be in the SAM format. The\n\
- file supplied with `-t' is SPACE/TAB delimited with the first two\n\
- fields of each line consisting of the reference name and the\n\
- corresponding sequence length. The `.fai' file generated by `faidx'\n\
- can be used here. This file may be empty if reads are unaligned.\n\
-\n\
- 2. SAM->BAM conversion: `samtools view -bT ref.fa in.sam.gz'.\n\
-\n\
- 3. BAM->SAM conversion: `samtools view in.bam'.\n\
-\n\
- 4. A region should be presented in one of the following formats:\n\
- `chr1', `chr2:1,000' and `chr3:1000-2,000'. When a region is\n\
- specified, the input alignment file must be an indexed BAM file.\n\
-\n\
- 5. Option `-u' is preferred over `-b' when the output is piped to\n\
- another samtools command.\n\
-\n\
- 6. In a string FLAG, each character represents one bit with\n\
- p=0x1 (paired), P=0x2 (properly paired), u=0x4 (unmapped),\n\
- U=0x8 (mate unmapped), r=0x10 (reverse), R=0x20 (mate reverse)\n\
- 1=0x40 (first), 2=0x80 (second), s=0x100 (not primary), \n\
- f=0x200 (failure) and d=0x400 (duplicate). Note that `-x' and\n\
- `-X' are samtools-C specific. Picard and older samtools do not\n\
- support HEX or string flags.\n\
-\n");
- return 1;
-}
-
-int main_import(int argc, char *argv[])
-{
- int argc2, ret;
- char **argv2;
- if (argc != 4) {
- fprintf(stderr, "Usage: bamtk import <in.ref_list> <in.sam> <out.bam>\n");
- return 1;
- }
- argc2 = 6;
- argv2 = calloc(6, sizeof(char*));
- argv2[0] = "import", argv2[1] = "-o", argv2[2] = argv[3], argv2[3] = "-bt", argv2[4] = argv[1], argv2[5] = argv[2];
- ret = main_samview(argc2, argv2);
- free(argv2);
- return ret;
-}
-
-int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 9, 14, 1, 6, 5, 13, 3, 11, 7, 15 };
-
-int main_bam2fq(int argc, char *argv[])
-{
- bamFile fp;
- bam_header_t *h;
- bam1_t *b;
- int8_t *buf;
- int max_buf, c, no12 = 0;
- while ((c = getopt(argc, argv, "n")) > 0)
- if (c == 'n') no12 = 1;
- if (argc == 1) {
- fprintf(stderr, "Usage: samtools bam2fq <in.bam>\n");
- return 1;
- }
- fp = strcmp(argv[optind], "-")? bam_open(argv[optind], "r") : bam_dopen(fileno(stdin), "r");
- if (fp == 0) return 1;
- h = bam_header_read(fp);
- b = bam_init1();
- buf = 0;
- max_buf = 0;
- while (bam_read1(fp, b) >= 0) {
- int i, qlen = b->core.l_qseq;
- uint8_t *seq;
- putchar('@'); fputs(bam1_qname(b), stdout);
- if (no12) putchar('\n');
- else {
- if ((b->core.flag & 0x40) && !(b->core.flag & 0x80)) puts("/1");
- else if ((b->core.flag & 0x80) && !(b->core.flag & 0x40)) puts("/2");
- else putchar('\n');
- }
- if (max_buf < qlen + 1) {
- max_buf = qlen + 1;
- kroundup32(max_buf);
- buf = realloc(buf, max_buf);
- }
- buf[qlen] = 0;
- seq = bam1_seq(b);
- for (i = 0; i < qlen; ++i)
- buf[i] = bam1_seqi(seq, i);
- if (b->core.flag & 16) { // reverse complement
- for (i = 0; i < qlen>>1; ++i) {
- int8_t t = seq_comp_table[buf[qlen - 1 - i]];
- buf[qlen - 1 - i] = seq_comp_table[buf[i]];
- buf[i] = t;
- }
- if (qlen&1) buf[i] = seq_comp_table[buf[i]];
- }
- for (i = 0; i < qlen; ++i)
- buf[i] = bam_nt16_rev_table[buf[i]];
- puts((char*)buf);
- puts("+");
- seq = bam1_qual(b);
- for (i = 0; i < qlen; ++i)
- buf[i] = 33 + seq[i];
- if (b->core.flag & 16) { // reverse
- for (i = 0; i < qlen>>1; ++i) {
- int8_t t = buf[qlen - 1 - i];
- buf[qlen - 1 - i] = buf[i];
- buf[i] = t;
- }
- }
- puts((char*)buf);
- }
- free(buf);
- bam_destroy1(b);
- bam_header_destroy(h);
- bam_close(fp);
- return 0;
-}
diff --git a/samtools-0.1.19/sample.c b/samtools-0.1.19/sample.c
deleted file mode 100644
index 830b9d1..0000000
--- a/samtools-0.1.19/sample.c
+++ /dev/null
@@ -1,107 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include "sample.h"
-#include "khash.h"
-KHASH_MAP_INIT_STR(sm, int)
-
-bam_sample_t *bam_smpl_init(void)
-{
- bam_sample_t *s;
- s = calloc(1, sizeof(bam_sample_t));
- s->rg2smid = kh_init(sm);
- s->sm2id = kh_init(sm);
- return s;
-}
-
-void bam_smpl_destroy(bam_sample_t *sm)
-{
- int i;
- khint_t k;
- khash_t(sm) *rg2smid = (khash_t(sm)*)sm->rg2smid;
- if (sm == 0) return;
- for (i = 0; i < sm->n; ++i) free(sm->smpl[i]);
- free(sm->smpl);
- for (k = kh_begin(rg2smid); k != kh_end(rg2smid); ++k)
- if (kh_exist(rg2smid, k)) free((char*)kh_key(rg2smid, k));
- kh_destroy(sm, sm->rg2smid);
- kh_destroy(sm, sm->sm2id);
- free(sm);
-}
-
-static void add_pair(bam_sample_t *sm, khash_t(sm) *sm2id, const char *key, const char *val)
-{
- khint_t k_rg, k_sm;
- int ret;
- khash_t(sm) *rg2smid = (khash_t(sm)*)sm->rg2smid;
- k_rg = kh_get(sm, rg2smid, key);
- if (k_rg != kh_end(rg2smid)) return; // duplicated @RG-ID
- k_rg = kh_put(sm, rg2smid, strdup(key), &ret);
- k_sm = kh_get(sm, sm2id, val);
- if (k_sm == kh_end(sm2id)) { // absent
- if (sm->n == sm->m) {
- sm->m = sm->m? sm->m<<1 : 1;
- sm->smpl = realloc(sm->smpl, sizeof(void*) * sm->m);
- }
- sm->smpl[sm->n] = strdup(val);
- k_sm = kh_put(sm, sm2id, sm->smpl[sm->n], &ret);
- kh_val(sm2id, k_sm) = sm->n++;
- }
- kh_val(rg2smid, k_rg) = kh_val(sm2id, k_sm);
-}
-
-int bam_smpl_add(bam_sample_t *sm, const char *fn, const char *txt)
-{
- const char *p = txt, *q, *r;
- kstring_t buf, first_sm;
- int n = 0;
- khash_t(sm) *sm2id = (khash_t(sm)*)sm->sm2id;
- if (txt == 0) {
- add_pair(sm, sm2id, fn, fn);
- return 0;
- }
- memset(&buf, 0, sizeof(kstring_t));
- memset(&first_sm, 0, sizeof(kstring_t));
- while ((q = strstr(p, "@RG")) != 0) {
- p = q + 3;
- r = q = 0;
- if ((q = strstr(p, "\tID:")) != 0) q += 4;
- if ((r = strstr(p, "\tSM:")) != 0) r += 4;
- if (r && q) {
- char *u, *v;
- int oq, or;
- for (u = (char*)q; *u && *u != '\t' && *u != '\n'; ++u);
- for (v = (char*)r; *v && *v != '\t' && *v != '\n'; ++v);
- oq = *u; or = *v; *u = *v = '\0';
- buf.l = 0; kputs(fn, &buf); kputc('/', &buf); kputs(q, &buf);
- add_pair(sm, sm2id, buf.s, r);
- if ( !first_sm.s )
- kputs(r,&first_sm);
- *u = oq; *v = or;
- } else break;
- p = q > r? q : r;
- ++n;
- }
- if (n == 0) add_pair(sm, sm2id, fn, fn);
- // If there is only one RG tag present in the header and reads are not annotated, don't refuse to work but
- // use the tag instead.
- else if ( n==1 && first_sm.s )
- add_pair(sm,sm2id,fn,first_sm.s);
- if ( first_sm.s )
- free(first_sm.s);
-
-// add_pair(sm, sm2id, fn, fn);
- free(buf.s);
- return 0;
-}
-
-int bam_smpl_rg2smid(const bam_sample_t *sm, const char *fn, const char *rg, kstring_t *str)
-{
- khint_t k;
- khash_t(sm) *rg2smid = (khash_t(sm)*)sm->rg2smid;
- if (rg) {
- str->l = 0;
- kputs(fn, str); kputc('/', str); kputs(rg, str);
- k = kh_get(sm, rg2smid, str->s);
- } else k = kh_get(sm, rg2smid, fn);
- return k == kh_end(rg2smid)? -1 : kh_val(rg2smid, k);
-}
diff --git a/samtools-0.1.19/sample.h b/samtools-0.1.19/sample.h
deleted file mode 100644
index 85fe499..0000000
--- a/samtools-0.1.19/sample.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef BAM_SAMPLE_H
-#define BAM_SAMPLE_H
-
-#include "kstring.h"
-
-typedef struct {
- int n, m;
- char **smpl;
- void *rg2smid, *sm2id;
-} bam_sample_t;
-
-bam_sample_t *bam_smpl_init(void);
-int bam_smpl_add(bam_sample_t *sm, const char *abs, const char *txt);
-int bam_smpl_rg2smid(const bam_sample_t *sm, const char *fn, const char *rg, kstring_t *str);
-void bam_smpl_destroy(bam_sample_t *sm);
-
-#endif
diff --git a/samtools-0.1.19/samtools.1 b/samtools-0.1.19/samtools.1
deleted file mode 100644
index 5923abd..0000000
--- a/samtools-0.1.19/samtools.1
+++ /dev/null
@@ -1,1066 +0,0 @@
-.TH samtools 1 "15 March 2013" "samtools-0.1.19" "Bioinformatics tools"
-.SH NAME
-.PP
-samtools - Utilities for the Sequence Alignment/Map (SAM) format
-
-bcftools - Utilities for the Binary Call Format (BCF) and VCF
-.SH SYNOPSIS
-.PP
-samtools view -bt ref_list.txt -o aln.bam aln.sam.gz
-.PP
-samtools sort aln.bam aln.sorted
-.PP
-samtools index aln.sorted.bam
-.PP
-samtools idxstats aln.sorted.bam
-.PP
-samtools view aln.sorted.bam chr2:20,100,000-20,200,000
-.PP
-samtools merge out.bam in1.bam in2.bam in3.bam
-.PP
-samtools faidx ref.fasta
-.PP
-samtools pileup -vcf ref.fasta aln.sorted.bam
-.PP
-samtools mpileup -C50 -gf ref.fasta -r chr3:1,000-2,000 in1.bam in2.bam
-.PP
-samtools tview aln.sorted.bam ref.fasta
-.PP
-bcftools index in.bcf
-.PP
-bcftools view in.bcf chr2:100-200 > out.vcf
-.PP
-bcftools view -Nvm0.99 in.bcf > out.vcf 2> out.afs
-
-.SH DESCRIPTION
-.PP
-Samtools is a set of utilities that manipulate alignments in the BAM
-format. It imports from and exports to the SAM (Sequence Alignment/Map)
-format, does sorting, merging and indexing, and allows to retrieve reads
-in any regions swiftly.
-
-Samtools is designed to work on a stream. It regards an input file `-'
-as the standard input (stdin) and an output file `-' as the standard
-output (stdout). Several commands can thus be combined with Unix
-pipes. Samtools always output warning and error messages to the standard
-error output (stderr).
-
-Samtools is also able to open a BAM (not SAM) file on a remote FTP or
-HTTP server if the BAM file name starts with `ftp://' or `http://'.
-Samtools checks the current working directory for the index file and
-will download the index upon absence. Samtools does not retrieve the
-entire alignment file unless it is asked to do so.
-
-.SH SAMTOOLS COMMANDS AND OPTIONS
-
-.TP 10
-.B view
-samtools view [-bchuHS] [-t in.refList] [-o output] [-f reqFlag] [-F
-skipFlag] [-q minMapQ] [-l library] [-r readGroup] [-R rgFile] <in.bam>|<in.sam> [region1 [...]]
-
-Extract/print all or sub alignments in SAM or BAM format. If no region
-is specified, all the alignments will be printed; otherwise only
-alignments overlapping the specified regions will be output. An
-alignment may be given multiple times if it is overlapping several
-regions. A region can be presented, for example, in the following
-format: `chr2' (the whole chr2), `chr2:1000000' (region starting from
-1,000,000bp) or `chr2:1,000,000-2,000,000' (region between 1,000,000 and
-2,000,000bp including the end points). The coordinate is 1-based.
-
-.B OPTIONS:
-.RS
-.TP 10
-.B -b
-Output in the BAM format.
-.TP
-.BI -f \ INT
-Only output alignments with all bits in INT present in the FLAG
-field. INT can be in hex in the format of /^0x[0-9A-F]+/ [0]
-.TP
-.BI -F \ INT
-Skip alignments with bits present in INT [0]
-.TP
-.B -h
-Include the header in the output.
-.TP
-.B -H
-Output the header only.
-.TP
-.BI -l \ STR
-Only output reads in library STR [null]
-.TP
-.BI -o \ FILE
-Output file [stdout]
-.TP
-.BI -q \ INT
-Skip alignments with MAPQ smaller than INT [0]
-.TP
-.BI -r \ STR
-Only output reads in read group STR [null]
-.TP
-.BI -R \ FILE
-Output reads in read groups listed in
-.I FILE
-[null]
-.TP
-.BI -s \ FLOAT
-Fraction of templates/pairs to subsample; the integer part is treated as the
-seed for the random number generator [-1]
-.TP
-.B -S
-Input is in SAM. If @SQ header lines are absent, the
-.B `-t'
-option is required.
-.TP
-.B -c
-Instead of printing the alignments, only count them and print the
-total number. All filter options, such as
-.B `-f',
-.B `-F'
-and
-.B `-q'
-, are taken into account.
-.TP
-.BI -t \ FILE
-This file is TAB-delimited. Each line must contain the reference name
-and the length of the reference, one line for each distinct reference;
-additional fields are ignored. This file also defines the order of the
-reference sequences in sorting. If you run `samtools faidx <ref.fa>',
-the resultant index file
-.I <ref.fa>.fai
-can be used as this
-.I <in.ref_list>
-file.
-.TP
-.B -u
-Output uncompressed BAM. This option saves time spent on
-compression/decomprssion and is thus preferred when the output is piped
-to another samtools command.
-.RE
-
-.TP
-.B tview
-samtools tview
-.RB [ \-p
-.IR chr:pos ]
-.RB [ \-s
-.IR STR ]
-.RB [ \-d
-.IR display ]
-.RI <in.sorted.bam>
-.RI [ref.fasta]
-
-Text alignment viewer (based on the ncurses library). In the viewer,
-press `?' for help and press `g' to check the alignment start from a
-region in the format like `chr10:10,000,000' or `=10,000,000' when
-viewing the same reference sequence.
-
-.B Options:
-.RS
-.TP 14
-.BI -d \ display
-Output as (H)tml or (C)urses or (T)ext
-.TP
-.BI -p \ chr:pos
-Go directly to this position
-.TP
-.BI -s \ STR
-Display only reads from this sample or read group
-.RE
-
-.TP
-.B mpileup
-samtools mpileup
-.RB [ \-EBugp ]
-.RB [ \-C
-.IR capQcoef ]
-.RB [ \-r
-.IR reg ]
-.RB [ \-f
-.IR in.fa ]
-.RB [ \-l
-.IR list ]
-.RB [ \-M
-.IR capMapQ ]
-.RB [ \-Q
-.IR minBaseQ ]
-.RB [ \-q
-.IR minMapQ ]
-.I in.bam
-.RI [ in2.bam
-.RI [ ... ]]
-
-Generate BCF or pileup for one or multiple BAM files. Alignment records
-are grouped by sample identifiers in @RG header lines. If sample
-identifiers are absent, each input file is regarded as one sample.
-
-In the pileup format (without
-.BR -u or -g ),
-each
-line represents a genomic position, consisting of chromosome name,
-coordinate, reference base, read bases, read qualities and alignment
-mapping qualities. Information on match, mismatch, indel, strand,
-mapping quality and start and end of a read are all encoded at the read
-base column. At this column, a dot stands for a match to the reference
-base on the forward strand, a comma for a match on the reverse strand,
-a '>' or '<' for a reference skip, `ACGTN' for a mismatch on the forward
-strand and `acgtn' for a mismatch on the reverse strand. A pattern
-`\\+[0-9]+[ACGTNacgtn]+' indicates there is an insertion between this
-reference position and the next reference position. The length of the
-insertion is given by the integer in the pattern, followed by the
-inserted sequence. Similarly, a pattern `-[0-9]+[ACGTNacgtn]+'
-represents a deletion from the reference. The deleted bases will be
-presented as `*' in the following lines. Also at the read base column, a
-symbol `^' marks the start of a read. The ASCII of the character
-following `^' minus 33 gives the mapping quality. A symbol `$' marks the
-end of a read segment.
-
-.B Input Options:
-.RS
-.TP 10
-.B -6
-Assume the quality is in the Illumina 1.3+ encoding.
-.B -A
-Do not skip anomalous read pairs in variant calling.
-.TP
-.B -B
-Disable probabilistic realignment for the computation of base alignment
-quality (BAQ). BAQ is the Phred-scaled probability of a read base being
-misaligned. Applying this option greatly helps to reduce false SNPs
-caused by misalignments.
-.TP
-.BI -b \ FILE
-List of input BAM files, one file per line [null]
-.TP
-.BI -C \ INT
-Coefficient for downgrading mapping quality for reads containing
-excessive mismatches. Given a read with a phred-scaled probability q of
-being generated from the mapped position, the new mapping quality is
-about sqrt((INT-q)/INT)*INT. A zero value disables this
-functionality; if enabled, the recommended value for BWA is 50. [0]
-.TP
-.BI -d \ INT
-At a position, read maximally
-.I INT
-reads per input BAM. [250]
-.TP
-.B -E
-Extended BAQ computation. This option helps sensitivity especially for MNPs, but may hurt
-specificity a little bit.
-.TP
-.BI -f \ FILE
-The
-.BR faidx -indexed
-reference file in the FASTA format. The file can be optionally compressed by
-.BR razip .
-[null]
-.TP
-.BI -l \ FILE
-BED or position list file containing a list of regions or sites where pileup or BCF should be generated [null]
-.TP
-.BI -q \ INT
-Minimum mapping quality for an alignment to be used [0]
-.TP
-.BI -Q \ INT
-Minimum base quality for a base to be considered [13]
-.TP
-.BI -r \ STR
-Only generate pileup in region
-.I STR
-[all sites]
-.TP
-.B Output Options:
-
-.TP
-.B -D
-Output per-sample read depth
-.TP
-.B -g
-Compute genotype likelihoods and output them in the binary call format (BCF).
-.TP
-.B -S
-Output per-sample Phred-scaled strand bias P-value
-.TP
-.B -u
-Similar to
-.B -g
-except that the output is uncompressed BCF, which is preferred for piping.
-
-.TP
-.B Options for Genotype Likelihood Computation (for -g or -u):
-
-.TP
-.BI -e \ INT
-Phred-scaled gap extension sequencing error probability. Reducing
-.I INT
-leads to longer indels. [20]
-.TP
-.BI -h \ INT
-Coefficient for modeling homopolymer errors. Given an
-.IR l -long
-homopolymer
-run, the sequencing error of an indel of size
-.I s
-is modeled as
-.IR INT * s / l .
-[100]
-.TP
-.B -I
-Do not perform INDEL calling
-.TP
-.BI -L \ INT
-Skip INDEL calling if the average per-sample depth is above
-.IR INT .
-[250]
-.TP
-.BI -o \ INT
-Phred-scaled gap open sequencing error probability. Reducing
-.I INT
-leads to more indel calls. [40]
-.TP
-.BI -p
-Apply -m and -F thresholds per sample to increase sensitivity of calling.
-By default both options are applied to reads pooled from all samples.
-.TP
-.BI -P \ STR
-Comma dilimited list of platforms (determined by
-.BR @RG-PL )
-from which indel candidates are obtained. It is recommended to collect
-indel candidates from sequencing technologies that have low indel error
-rate such as ILLUMINA. [all]
-.RE
-
-.TP
-.B reheader
-samtools reheader <in.header.sam> <in.bam>
-
-Replace the header in
-.I in.bam
-with the header in
-.I in.header.sam.
-This command is much faster than replacing the header with a
-BAM->SAM->BAM conversion.
-
-.TP
-.B cat
-samtools cat [-h header.sam] [-o out.bam] <in1.bam> <in2.bam> [ ... ]
-
-Concatenate BAMs. The sequence dictionary of each input BAM must be identical,
-although this command does not check this. This command uses a similar trick
-to
-.B reheader
-which enables fast BAM concatenation.
-
-.TP
-.B sort
-samtools sort [-nof] [-m maxMem] <in.bam> <out.prefix>
-
-Sort alignments by leftmost coordinates. File
-.I <out.prefix>.bam
-will be created. This command may also create temporary files
-.I <out.prefix>.%d.bam
-when the whole alignment cannot be fitted into memory (controlled by
-option -m).
-
-.B OPTIONS:
-.RS
-.TP 8
-.B -o
-Output the final alignment to the standard output.
-.TP
-.B -n
-Sort by read names rather than by chromosomal coordinates
-.TP
-.B -f
-Use
-.I <out.prefix>
-as the full output path and do not append
-.I .bam
-suffix.
-.TP
-.BI -m \ INT
-Approximately the maximum required memory. [500000000]
-.RE
-
-.TP
-.B merge
-samtools merge [-nur1f] [-h inh.sam] [-R reg] <out.bam> <in1.bam> <in2.bam> [...]
-
-Merge multiple sorted alignments.
-The header reference lists of all the input BAM files, and the @SQ headers of
-.IR inh.sam ,
-if any, must all refer to the same set of reference sequences.
-The header reference list and (unless overridden by
-.BR -h )
-`@' headers of
-.I in1.bam
-will be copied to
-.IR out.bam ,
-and the headers of other files will be ignored.
-
-.B OPTIONS:
-.RS
-.TP 8
-.B -1
-Use zlib compression level 1 to comrpess the output
-.TP
-.B -f
-Force to overwrite the output file if present.
-.TP 8
-.BI -h \ FILE
-Use the lines of
-.I FILE
-as `@' headers to be copied to
-.IR out.bam ,
-replacing any header lines that would otherwise be copied from
-.IR in1.bam .
-.RI ( FILE
-is actually in SAM format, though any alignment records it may contain
-are ignored.)
-.TP
-.B -n
-The input alignments are sorted by read names rather than by chromosomal
-coordinates
-.TP
-.BI -R \ STR
-Merge files in the specified region indicated by
-.I STR
-[null]
-.TP
-.B -r
-Attach an RG tag to each alignment. The tag value is inferred from file names.
-.TP
-.B -u
-Uncompressed BAM output
-.RE
-
-.TP
-.B index
-samtools index <aln.bam>
-
-Index sorted alignment for fast random access. Index file
-.I <aln.bam>.bai
-will be created.
-
-.TP
-.B idxstats
-samtools idxstats <aln.bam>
-
-Retrieve and print stats in the index file. The output is TAB delimited
-with each line consisting of reference sequence name, sequence length, #
-mapped reads and # unmapped reads.
-
-.TP
-.B faidx
-samtools faidx <ref.fasta> [region1 [...]]
-
-Index reference sequence in the FASTA format or extract subsequence from
-indexed reference sequence. If no region is specified,
-.B faidx
-will index the file and create
-.I <ref.fasta>.fai
-on the disk. If regions are speficified, the subsequences will be
-retrieved and printed to stdout in the FASTA format. The input file can
-be compressed in the
-.B RAZF
-format.
-
-.TP
-.B fixmate
-samtools fixmate <in.nameSrt.bam> <out.bam>
-
-Fill in mate coordinates, ISIZE and mate related flags from a
-name-sorted alignment.
-
-.TP
-.B rmdup
-samtools rmdup [-sS] <input.srt.bam> <out.bam>
-
-Remove potential PCR duplicates: if multiple read pairs have identical
-external coordinates, only retain the pair with highest mapping quality.
-In the paired-end mode, this command
-.B ONLY
-works with FR orientation and requires ISIZE is correctly set. It does
-not work for unpaired reads (e.g. two ends mapped to different
-chromosomes or orphan reads).
-
-.B OPTIONS:
-.RS
-.TP 8
-.B -s
-Remove duplicate for single-end reads. By default, the command works for
-paired-end reads only.
-.TP 8
-.B -S
-Treat paired-end reads and single-end reads.
-.RE
-
-.TP
-.B calmd
-samtools calmd [-EeubSr] [-C capQcoef] <aln.bam> <ref.fasta>
-
-Generate the MD tag. If the MD tag is already present, this command will
-give a warning if the MD tag generated is different from the existing
-tag. Output SAM by default.
-
-.B OPTIONS:
-.RS
-.TP 8
-.B -A
-When used jointly with
-.B -r
-this option overwrites the original base quality.
-.TP 8
-.B -e
-Convert a the read base to = if it is identical to the aligned reference
-base. Indel caller does not support the = bases at the moment.
-.TP
-.B -u
-Output uncompressed BAM
-.TP
-.B -b
-Output compressed BAM
-.TP
-.B -S
-The input is SAM with header lines
-.TP
-.BI -C \ INT
-Coefficient to cap mapping quality of poorly mapped reads. See the
-.B pileup
-command for details. [0]
-.TP
-.B -r
-Compute the BQ tag (without -A) or cap base quality by BAQ (with -A).
-.TP
-.B -E
-Extended BAQ calculation. This option trades specificity for sensitivity, though the
-effect is minor.
-.RE
-
-.TP
-.B targetcut
-samtools targetcut [-Q minBaseQ] [-i inPenalty] [-0 em0] [-1 em1] [-2 em2] [-f ref] <in.bam>
-
-This command identifies target regions by examining the continuity of read depth, computes
-haploid consensus sequences of targets and outputs a SAM with each sequence corresponding
-to a target. When option
-.B -f
-is in use, BAQ will be applied. This command is
-.B only
-designed for cutting fosmid clones from fosmid pool sequencing [Ref. Kitzman et al. (2010)].
-.RE
-
-.TP
-.B phase
-samtools phase [-AF] [-k len] [-b prefix] [-q minLOD] [-Q minBaseQ] <in.bam>
-
-Call and phase heterozygous SNPs.
-.B OPTIONS:
-.RS
-.TP 8
-.B -A
-Drop reads with ambiguous phase.
-.TP 8
-.BI -b \ STR
-Prefix of BAM output. When this option is in use, phase-0 reads will be saved in file
-.BR STR .0.bam
-and phase-1 reads in
-.BR STR .1.bam.
-Phase unknown reads will be randomly allocated to one of the two files. Chimeric reads
-with switch errors will be saved in
-.BR STR .chimeric.bam.
-[null]
-.TP
-.B -F
-Do not attempt to fix chimeric reads.
-.TP
-.BI -k \ INT
-Maximum length for local phasing. [13]
-.TP
-.BI -q \ INT
-Minimum Phred-scaled LOD to call a heterozygote. [40]
-.TP
-.BI -Q \ INT
-Minimum base quality to be used in het calling. [13]
-.RE
-
-.SH BCFTOOLS COMMANDS AND OPTIONS
-
-.TP 10
-.B view
-.B bcftools view
-.RB [ \-AbFGNQSucgv ]
-.RB [ \-D
-.IR seqDict ]
-.RB [ \-l
-.IR listLoci ]
-.RB [ \-s
-.IR listSample ]
-.RB [ \-i
-.IR gapSNPratio ]
-.RB [ \-t
-.IR mutRate ]
-.RB [ \-p
-.IR varThres ]
-.RB [ \-m
-.IR varThres ]
-.RB [ \-P
-.IR prior ]
-.RB [ \-1
-.IR nGroup1 ]
-.RB [ \-d
-.IR minFrac ]
-.RB [ \-U
-.IR nPerm ]
-.RB [ \-X
-.IR permThres ]
-.RB [ \-T
-.IR trioType ]
-.I in.bcf
-.RI [ region ]
-
-Convert between BCF and VCF, call variant candidates and estimate allele
-frequencies.
-
-.RS
-.TP
-.B Input/Output Options:
-.TP 10
-.B -A
-Retain all possible alternate alleles at variant sites. By default, the view
-command discards unlikely alleles.
-.TP 10
-.B -b
-Output in the BCF format. The default is VCF.
-.TP
-.BI -D \ FILE
-Sequence dictionary (list of chromosome names) for VCF->BCF conversion [null]
-.TP
-.B -F
-Indicate PL is generated by r921 or before (ordering is different).
-.TP
-.B -G
-Suppress all individual genotype information.
-.TP
-.BI -l \ FILE
-List of sites at which information are outputted [all sites]
-.TP
-.B -N
-Skip sites where the REF field is not A/C/G/T
-.TP
-.B -Q
-Output the QCALL likelihood format
-.TP
-.BI -s \ FILE
-List of samples to use. The first column in the input gives the sample names
-and the second gives the ploidy, which can only be 1 or 2. When the 2nd column
-is absent, the sample ploidy is assumed to be 2. In the output, the ordering of
-samples will be identical to the one in
-.IR FILE .
-[null]
-.TP
-.B -S
-The input is VCF instead of BCF.
-.TP
-.B -u
-Uncompressed BCF output (force -b).
-.TP
-.B Consensus/Variant Calling Options:
-.TP 10
-.B -c
-Call variants using Bayesian inference. This option automatically invokes option
-.BR -e .
-.TP
-.BI -d \ FLOAT
-When
-.B -v
-is in use, skip loci where the fraction of samples covered by reads is below FLOAT. [0]
-.TP
-.B -e
-Perform max-likelihood inference only, including estimating the site allele frequency,
-testing Hardy-Weinberg equlibrium and testing associations with LRT.
-.TP
-.B -g
-Call per-sample genotypes at variant sites (force -c)
-.TP
-.BI -i \ FLOAT
-Ratio of INDEL-to-SNP mutation rate [0.15]
-.TP
-.BI -m \ FLOAT
-New model for improved multiallelic and rare-variant calling. Another
-ALT allele is accepted if P(chi^2) of LRT exceeds the FLOAT threshold. The
-parameter seems robust and the actual value usually does not affect the results
-much; a good value to use is 0.99. This is the recommended calling method. [0]
-.TP
-.BI -p \ FLOAT
-A site is considered to be a variant if P(ref|D)<FLOAT [0.5]
-.TP
-.BI -P \ STR
-Prior or initial allele frequency spectrum. If STR can be
-.IR full ,
-.IR cond2 ,
-.I flat
-or the file consisting of error output from a previous variant calling
-run.
-.TP
-.BI -t \ FLOAT
-Scaled muttion rate for variant calling [0.001]
-.TP
-.BI -T \ STR
-Enable pair/trio calling. For trio calling, option
-.B -s
-is usually needed to be applied to configure the trio members and their ordering.
-In the file supplied to the option
-.BR -s ,
-the first sample must be the child, the second the father and the third the mother.
-The valid values of
-.I STR
-are `pair', `trioauto', `trioxd' and `trioxs', where `pair' calls differences between two input samples, and `trioxd' (`trioxs') specifies that the input
-is from the X chromosome non-PAR regions and the child is a female (male). [null]
-.TP
-.B -v
-Output variant sites only (force -c)
-.TP
-.B Contrast Calling and Association Test Options:
-.TP
-.BI -1 \ INT
-Number of group-1 samples. This option is used for dividing the samples into
-two groups for contrast SNP calling or association test.
-When this option is in use, the following VCF INFO will be outputted:
-PC2, PCHI2 and QCHI2. [0]
-.TP
-.BI -U \ INT
-Number of permutations for association test (effective only with
-.BR -1 )
-[0]
-.TP
-.BI -X \ FLOAT
-Only perform permutations for P(chi^2)<FLOAT (effective only with
-.BR -U )
-[0.01]
-.RE
-
-.TP
-.B index
-.B bcftools index
-.I in.bcf
-
-Index sorted BCF for random access.
-.RE
-
-.TP
-.B cat
-.B bcftools cat
-.I in1.bcf
-.RI [ "in2.bcf " [ ... "]]]"
-
-Concatenate BCF files. The input files are required to be sorted and
-have identical samples appearing in the same order.
-.RE
-.SH SAM FORMAT
-
-Sequence Alignment/Map (SAM) format is TAB-delimited. Apart from the header lines, which are started
-with the `@' symbol, each alignment line consists of:
-
-.TS
-center box;
-cb | cb | cb
-n | l | l .
-Col Field Description
-_
-1 QNAME Query template/pair NAME
-2 FLAG bitwise FLAG
-3 RNAME Reference sequence NAME
-4 POS 1-based leftmost POSition/coordinate of clipped sequence
-5 MAPQ MAPping Quality (Phred-scaled)
-6 CIAGR extended CIGAR string
-7 MRNM Mate Reference sequence NaMe (`=' if same as RNAME)
-8 MPOS 1-based Mate POSistion
-9 TLEN inferred Template LENgth (insert size)
-10 SEQ query SEQuence on the same strand as the reference
-11 QUAL query QUALity (ASCII-33 gives the Phred base quality)
-12+ OPT variable OPTional fields in the format TAG:VTYPE:VALUE
-.TE
-
-.PP
-Each bit in the FLAG field is defined as:
-
-.TS
-center box;
-cb | cb | cb
-l | c | l .
-Flag Chr Description
-_
-0x0001 p the read is paired in sequencing
-0x0002 P the read is mapped in a proper pair
-0x0004 u the query sequence itself is unmapped
-0x0008 U the mate is unmapped
-0x0010 r strand of the query (1 for reverse)
-0x0020 R strand of the mate
-0x0040 1 the read is the first read in a pair
-0x0080 2 the read is the second read in a pair
-0x0100 s the alignment is not primary
-0x0200 f the read fails platform/vendor quality checks
-0x0400 d the read is either a PCR or an optical duplicate
-.TE
-
-where the second column gives the string representation of the FLAG field.
-
-.SH VCF FORMAT
-
-The Variant Call Format (VCF) is a TAB-delimited format with each data line consists of the following fields:
-.TS
-center box;
-cb | cb | cb
-n | l | l .
-Col Field Description
-_
-1 CHROM CHROMosome name
-2 POS the left-most POSition of the variant
-3 ID unique variant IDentifier
-4 REF the REFerence allele
-5 ALT the ALTernate allele(s), separated by comma
-6 QUAL variant/reference QUALity
-7 FILTER FILTers applied
-8 INFO INFOrmation related to the variant, separated by semi-colon
-9 FORMAT FORMAT of the genotype fields, separated by colon (optional)
-10+ SAMPLE SAMPLE genotypes and per-sample information (optional)
-.TE
-
-.PP
-The following table gives the
-.B INFO
-tags used by samtools and bcftools.
-
-.TS
-center box;
-cb | cb | cb
-l | l | l .
-Tag Format Description
-_
-AF1 double Max-likelihood estimate of the site allele frequency (AF) of the first ALT allele
-DP int Raw read depth (without quality filtering)
-DP4 int[4] # high-quality reference forward bases, ref reverse, alternate for and alt rev bases
-FQ int Consensus quality. Positive: sample genotypes different; negative: otherwise
-MQ int Root-Mean-Square mapping quality of covering reads
-PC2 int[2] Phred probability of AF in group1 samples being larger (,smaller) than in group2
-PCHI2 double Posterior weighted chi^2 P-value between group1 and group2 samples
-PV4 double[4] P-value for strand bias, baseQ bias, mapQ bias and tail distance bias
-QCHI2 int Phred-scaled PCHI2
-RP int # permutations yielding a smaller PCHI2
-CLR int Phred log ratio of genotype likelihoods with and without the trio/pair constraint
-UGT string Most probable genotype configuration without the trio constraint
-CGT string Most probable configuration with the trio constraint
-VDB float Tests variant positions within reads. Intended for filtering RNA-seq artifacts around splice sites
-RPB float Mann-Whitney rank-sum test for tail distance bias
-HWE float Hardy-Weinberg equilibrium test, Wigginton et al., PMID: 15789306
-.TE
-
-.SH EXAMPLES
-.IP o 2
-Import SAM to BAM when
-.B @SQ
-lines are present in the header:
-
- samtools view -bS aln.sam > aln.bam
-
-If
-.B @SQ
-lines are absent:
-
- samtools faidx ref.fa
- samtools view -bt ref.fa.fai aln.sam > aln.bam
-
-where
-.I ref.fa.fai
-is generated automatically by the
-.B faidx
-command.
-
-.IP o 2
-Attach the
-.B RG
-tag while merging sorted alignments:
-
- perl -e 'print "@RG\\tID:ga\\tSM:hs\\tLB:ga\\tPL:Illumina\\n at RG\\tID:454\\tSM:hs\\tLB:454\\tPL:454\\n"' > rg.txt
- samtools merge -rh rg.txt merged.bam ga.bam 454.bam
-
-The value in a
-.B RG
-tag is determined by the file name the read is coming from. In this
-example, in the
-.IR merged.bam ,
-reads from
-.I ga.bam
-will be attached
-.IR RG:Z:ga ,
-while reads from
-.I 454.bam
-will be attached
-.IR RG:Z:454 .
-
-.IP o 2
-Call SNPs and short INDELs for one diploid individual:
-
- samtools mpileup -ugf ref.fa aln.bam | bcftools view -bvcg - > var.raw.bcf
- bcftools view var.raw.bcf | vcfutils.pl varFilter -D 100 > var.flt.vcf
-
-The
-.B -D
-option of varFilter controls the maximum read depth, which should be
-adjusted to about twice the average read depth. One may consider to add
-.B -C50
-to
-.B mpileup
-if mapping quality is overestimated for reads containing excessive
-mismatches. Applying this option usually helps
-.B BWA-short
-but may not other mappers.
-
-.IP o 2
-Generate the consensus sequence for one diploid individual:
-
- samtools mpileup -uf ref.fa aln.bam | bcftools view -cg - | vcfutils.pl vcf2fq > cns.fq
-
-.IP o 2
-Call somatic mutations from a pair of samples:
-
- samtools mpileup -DSuf ref.fa aln.bam | bcftools view -bvcgT pair - > var.bcf
-
-In the output INFO field,
-.I CLR
-gives the Phred-log ratio between the likelihood by treating the
-two samples independently, and the likelihood by requiring the genotype to be identical.
-This
-.I CLR
-is effectively a score measuring the confidence of somatic calls. The higher the better.
-
-.IP o 2
-Call de novo and somatic mutations from a family trio:
-
- samtools mpileup -DSuf ref.fa aln.bam | bcftools view -bvcgT pair -s samples.txt - > var.bcf
-
-File
-.I samples.txt
-should consist of three lines specifying the member and order of samples (in the order of child-father-mother).
-Similarly,
-.I CLR
-gives the Phred-log likelihood ratio with and without the trio constraint.
-.I UGT
-shows the most likely genotype configuration without the trio constraint, and
-.I CGT
-gives the most likely genotype configuration satisfying the trio constraint.
-
-.IP o 2
-Phase one individual:
-
- samtools calmd -AEur aln.bam ref.fa | samtools phase -b prefix - > phase.out
-
-The
-.B calmd
-command is used to reduce false heterozygotes around INDELs.
-
-.IP o 2
-Call SNPs and short indels for multiple diploid individuals:
-
- samtools mpileup -P ILLUMINA -ugf ref.fa *.bam | bcftools view -bcvg - > var.raw.bcf
- bcftools view var.raw.bcf | vcfutils.pl varFilter -D 2000 > var.flt.vcf
-
-Individuals are identified from the
-.B SM
-tags in the
-.B @RG
-header lines. Individuals can be pooled in one alignment file; one
-individual can also be separated into multiple files. The
-.B -P
-option specifies that indel candidates should be collected only from
-read groups with the
-.B @RG-PL
-tag set to
-.IR ILLUMINA .
-Collecting indel candidates from reads sequenced by an indel-prone
-technology may affect the performance of indel calling.
-
-Note that there is a new calling model which can be invoked by
-
- bcftools view -m0.99 ...
-
-which fixes some severe limitations of the default method.
-
-For filtering, best results seem to be achieved by first applying the
-.IR SnpGap
-filter and then applying some machine learning approach
-
- vcf-annotate -f SnpGap=n
- vcf filter ...
-
-Both can be found in the
-.B vcftools
-and
-.B htslib
-package (links below).
-
-.IP o 2
-Derive the allele frequency spectrum (AFS) on a list of sites from multiple individuals:
-
- samtools mpileup -Igf ref.fa *.bam > all.bcf
- bcftools view -bl sites.list all.bcf > sites.bcf
- bcftools view -cGP cond2 sites.bcf > /dev/null 2> sites.1.afs
- bcftools view -cGP sites.1.afs sites.bcf > /dev/null 2> sites.2.afs
- bcftools view -cGP sites.2.afs sites.bcf > /dev/null 2> sites.3.afs
- ......
-
-where
-.I sites.list
-contains the list of sites with each line consisting of the reference
-sequence name and position. The following
-.B bcftools
-commands estimate AFS by EM.
-
-.IP o 2
-Dump BAQ applied alignment for other SNP callers:
-
- samtools calmd -bAr aln.bam > aln.baq.bam
-
-It adds and corrects the
-.B NM
-and
-.B MD
-tags at the same time. The
-.B calmd
-command also comes with the
-.B -C
-option, the same as the one in
-.B pileup
-and
-.BR mpileup .
-Apply if it helps.
-
-.SH LIMITATIONS
-.PP
-.IP o 2
-Unaligned words used in bam_import.c, bam_endian.h, bam.c and bam_aux.c.
-.IP o 2
-Samtools paired-end rmdup does not work for unpaired reads (e.g. orphan
-reads or ends mapped to different chromosomes). If this is a concern,
-please use Picard's MarkDuplicate which correctly handles these cases,
-although a little slower.
-
-.SH AUTHOR
-.PP
-Heng Li from the Sanger Institute wrote the C version of samtools. Bob
-Handsaker from the Broad Institute implemented the BGZF library and Jue
-Ruan from Beijing Genomics Institute wrote the RAZF library. John
-Marshall and Petr Danecek contribute to the source code and various
-people from the 1000 Genomes Project have contributed to the SAM format
-specification.
-
-.SH SEE ALSO
-.PP
-Samtools website: <http://samtools.sourceforge.net>
-.br
-Samtools latest source: <https://github.com/samtools/samtools>
-.br
-VCFtools website with stable link to VCF specification: <http://vcftools.sourceforge.net>
-.br
-HTSlib website: <https://github.com/samtools/htslib>
diff --git a/samtools-0.1.19/win32/libcurses.a b/samtools-0.1.19/win32/libcurses.a
deleted file mode 100644
index a3863b8..0000000
Binary files a/samtools-0.1.19/win32/libcurses.a and /dev/null differ
diff --git a/samtools-0.1.19/win32/libz.a b/samtools-0.1.19/win32/libz.a
deleted file mode 100644
index 23e8d60..0000000
Binary files a/samtools-0.1.19/win32/libz.a and /dev/null differ
diff --git a/samtools-0.1.19/win32/xcurses.h b/samtools-0.1.19/win32/xcurses.h
deleted file mode 100644
index 6f3ce19..0000000
--- a/samtools-0.1.19/win32/xcurses.h
+++ /dev/null
@@ -1,1377 +0,0 @@
-/* Public Domain Curses */
-
-/* $Id: curses.h,v 1.295 2008/07/15 17:13:25 wmcbrine Exp $ */
-
-/*----------------------------------------------------------------------*
- * PDCurses *
- *----------------------------------------------------------------------*/
-
-#ifndef __PDCURSES__
-#define __PDCURSES__ 1
-
-/*man-start**************************************************************
-
-PDCurses definitions list: (Only define those needed)
-
- XCURSES True if compiling for X11.
- PDC_RGB True if you want to use RGB color definitions
- (Red = 1, Green = 2, Blue = 4) instead of BGR.
- PDC_WIDE True if building wide-character support.
- PDC_DLL_BUILD True if building a Win32 DLL.
- NCURSES_MOUSE_VERSION Use the ncurses mouse API instead
- of PDCurses' traditional mouse API.
-
-PDCurses portable platform definitions list:
-
- PDC_BUILD Defines API build version.
- PDCURSES Enables access to PDCurses-only routines.
- XOPEN Always true.
- SYSVcurses True if you are compiling for SYSV portability.
- BSDcurses True if you are compiling for BSD portability.
-
-**man-end****************************************************************/
-
-#define PDC_BUILD 3401
-#define PDCURSES 1 /* PDCurses-only routines */
-#define XOPEN 1 /* X/Open Curses routines */
-#define SYSVcurses 1 /* System V Curses routines */
-#define BSDcurses 1 /* BSD Curses routines */
-#define CHTYPE_LONG 1 /* size of chtype; long */
-
-/*----------------------------------------------------------------------*/
-
-#include <stdarg.h>
-#include <stddef.h>
-#include <stdio.h> /* Required by X/Open usage below */
-
-#ifdef PDC_WIDE
-# include <wchar.h>
-#endif
-
-#if defined(__cplusplus) || defined(__cplusplus__) || defined(__CPLUSPLUS)
-extern "C"
-{
-# define bool _bool
-#endif
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Manifest Constants
- *
- */
-
-#ifndef FALSE
-# define FALSE 0
-#endif
-#ifndef TRUE
-# define TRUE 1
-#endif
-#ifndef NULL
-# define NULL (void *)0
-#endif
-#ifndef ERR
-# define ERR (-1)
-#endif
-#ifndef OK
-# define OK 0
-#endif
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Type Declarations
- *
- */
-
-typedef unsigned char bool; /* PDCurses Boolean type */
-
-#ifdef CHTYPE_LONG
-# if _LP64
-typedef unsigned int chtype;
-# else
-typedef unsigned long chtype; /* 16-bit attr + 16-bit char */
-# endif
-#else
-typedef unsigned short chtype; /* 8-bit attr + 8-bit char */
-#endif
-
-#ifdef PDC_WIDE
-typedef chtype cchar_t;
-#endif
-
-typedef chtype attr_t;
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Mouse Interface -- SYSVR4, with extensions
- *
- */
-
-typedef struct
-{
- int x; /* absolute column, 0 based, measured in characters */
- int y; /* absolute row, 0 based, measured in characters */
- short button[3]; /* state of each button */
- int changes; /* flags indicating what has changed with the mouse */
-} MOUSE_STATUS;
-
-#define BUTTON_RELEASED 0x0000
-#define BUTTON_PRESSED 0x0001
-#define BUTTON_CLICKED 0x0002
-#define BUTTON_DOUBLE_CLICKED 0x0003
-#define BUTTON_TRIPLE_CLICKED 0x0004
-#define BUTTON_MOVED 0x0005 /* PDCurses */
-#define WHEEL_SCROLLED 0x0006 /* PDCurses */
-#define BUTTON_ACTION_MASK 0x0007 /* PDCurses */
-
-#define PDC_BUTTON_SHIFT 0x0008 /* PDCurses */
-#define PDC_BUTTON_CONTROL 0x0010 /* PDCurses */
-#define PDC_BUTTON_ALT 0x0020 /* PDCurses */
-#define BUTTON_MODIFIER_MASK 0x0038 /* PDCurses */
-
-#define MOUSE_X_POS (Mouse_status.x)
-#define MOUSE_Y_POS (Mouse_status.y)
-
-/*
- * Bits associated with the .changes field:
- * 3 2 1 0
- * 210987654321098765432109876543210
- * 1 <- button 1 has changed
- * 10 <- button 2 has changed
- * 100 <- button 3 has changed
- * 1000 <- mouse has moved
- * 10000 <- mouse position report
- * 100000 <- mouse wheel up
- * 1000000 <- mouse wheel down
- */
-
-#define PDC_MOUSE_MOVED 0x0008
-#define PDC_MOUSE_POSITION 0x0010
-#define PDC_MOUSE_WHEEL_UP 0x0020
-#define PDC_MOUSE_WHEEL_DOWN 0x0040
-
-#define A_BUTTON_CHANGED (Mouse_status.changes & 7)
-#define MOUSE_MOVED (Mouse_status.changes & PDC_MOUSE_MOVED)
-#define MOUSE_POS_REPORT (Mouse_status.changes & PDC_MOUSE_POSITION)
-#define BUTTON_CHANGED(x) (Mouse_status.changes & (1 << ((x) - 1)))
-#define BUTTON_STATUS(x) (Mouse_status.button[(x) - 1])
-#define MOUSE_WHEEL_UP (Mouse_status.changes & PDC_MOUSE_WHEEL_UP)
-#define MOUSE_WHEEL_DOWN (Mouse_status.changes & PDC_MOUSE_WHEEL_DOWN)
-
-/* mouse bit-masks */
-
-#define BUTTON1_RELEASED 0x00000001L
-#define BUTTON1_PRESSED 0x00000002L
-#define BUTTON1_CLICKED 0x00000004L
-#define BUTTON1_DOUBLE_CLICKED 0x00000008L
-#define BUTTON1_TRIPLE_CLICKED 0x00000010L
-#define BUTTON1_MOVED 0x00000010L /* PDCurses */
-
-#define BUTTON2_RELEASED 0x00000020L
-#define BUTTON2_PRESSED 0x00000040L
-#define BUTTON2_CLICKED 0x00000080L
-#define BUTTON2_DOUBLE_CLICKED 0x00000100L
-#define BUTTON2_TRIPLE_CLICKED 0x00000200L
-#define BUTTON2_MOVED 0x00000200L /* PDCurses */
-
-#define BUTTON3_RELEASED 0x00000400L
-#define BUTTON3_PRESSED 0x00000800L
-#define BUTTON3_CLICKED 0x00001000L
-#define BUTTON3_DOUBLE_CLICKED 0x00002000L
-#define BUTTON3_TRIPLE_CLICKED 0x00004000L
-#define BUTTON3_MOVED 0x00004000L /* PDCurses */
-
-/* For the ncurses-compatible functions only, BUTTON4_PRESSED and
- BUTTON5_PRESSED are returned for mouse scroll wheel up and down;
- otherwise PDCurses doesn't support buttons 4 and 5 */
-
-#define BUTTON4_RELEASED 0x00008000L
-#define BUTTON4_PRESSED 0x00010000L
-#define BUTTON4_CLICKED 0x00020000L
-#define BUTTON4_DOUBLE_CLICKED 0x00040000L
-#define BUTTON4_TRIPLE_CLICKED 0x00080000L
-
-#define BUTTON5_RELEASED 0x00100000L
-#define BUTTON5_PRESSED 0x00200000L
-#define BUTTON5_CLICKED 0x00400000L
-#define BUTTON5_DOUBLE_CLICKED 0x00800000L
-#define BUTTON5_TRIPLE_CLICKED 0x01000000L
-
-#define MOUSE_WHEEL_SCROLL 0x02000000L /* PDCurses */
-#define BUTTON_MODIFIER_SHIFT 0x04000000L /* PDCurses */
-#define BUTTON_MODIFIER_CONTROL 0x08000000L /* PDCurses */
-#define BUTTON_MODIFIER_ALT 0x10000000L /* PDCurses */
-
-#define ALL_MOUSE_EVENTS 0x1fffffffL
-#define REPORT_MOUSE_POSITION 0x20000000L
-
-/* ncurses mouse interface */
-
-typedef unsigned long mmask_t;
-
-typedef struct
-{
- short id; /* unused, always 0 */
- int x, y, z; /* x, y same as MOUSE_STATUS; z unused */
- mmask_t bstate; /* equivalent to changes + button[], but
- in the same format as used for mousemask() */
-} MEVENT;
-
-#ifdef NCURSES_MOUSE_VERSION
-# define BUTTON_SHIFT BUTTON_MODIFIER_SHIFT
-# define BUTTON_CONTROL BUTTON_MODIFIER_CONTROL
-# define BUTTON_CTRL BUTTON_MODIFIER_CONTROL
-# define BUTTON_ALT BUTTON_MODIFIER_ALT
-#else
-# define BUTTON_SHIFT PDC_BUTTON_SHIFT
-# define BUTTON_CONTROL PDC_BUTTON_CONTROL
-# define BUTTON_ALT PDC_BUTTON_ALT
-#endif
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Structure Definitions
- *
- */
-
-typedef struct _win /* definition of a window */
-{
- int _cury; /* current pseudo-cursor */
- int _curx;
- int _maxy; /* max window coordinates */
- int _maxx;
- int _begy; /* origin on screen */
- int _begx;
- int _flags; /* window properties */
- chtype _attrs; /* standard attributes and colors */
- chtype _bkgd; /* background, normally blank */
- bool _clear; /* causes clear at next refresh */
- bool _leaveit; /* leaves cursor where it is */
- bool _scroll; /* allows window scrolling */
- bool _nodelay; /* input character wait flag */
- bool _immed; /* immediate update flag */
- bool _sync; /* synchronise window ancestors */
- bool _use_keypad; /* flags keypad key mode active */
- chtype **_y; /* pointer to line pointer array */
- int *_firstch; /* first changed character in line */
- int *_lastch; /* last changed character in line */
- int _tmarg; /* top of scrolling region */
- int _bmarg; /* bottom of scrolling region */
- int _delayms; /* milliseconds of delay for getch() */
- int _parx, _pary; /* coords relative to parent (0,0) */
- struct _win *_parent; /* subwin's pointer to parent win */
-} WINDOW;
-
-/* Avoid using the SCREEN struct directly -- use the corresponding
- functions if possible. This struct may eventually be made private. */
-
-typedef struct
-{
- bool alive; /* if initscr() called, and not endwin() */
- bool autocr; /* if cr -> lf */
- bool cbreak; /* if terminal unbuffered */
- bool echo; /* if terminal echo */
- bool raw_inp; /* raw input mode (v. cooked input) */
- bool raw_out; /* raw output mode (7 v. 8 bits) */
- bool audible; /* FALSE if the bell is visual */
- bool mono; /* TRUE if current screen is mono */
- bool resized; /* TRUE if TERM has been resized */
- bool orig_attr; /* TRUE if we have the original colors */
- short orig_fore; /* original screen foreground color */
- short orig_back; /* original screen foreground color */
- int cursrow; /* position of physical cursor */
- int curscol; /* position of physical cursor */
- int visibility; /* visibility of cursor */
- int orig_cursor; /* original cursor size */
- int lines; /* new value for LINES */
- int cols; /* new value for COLS */
- unsigned long _trap_mbe; /* trap these mouse button events */
- unsigned long _map_mbe_to_key; /* map mouse buttons to slk */
- int mouse_wait; /* time to wait (in ms) for a
- button release after a press, in
- order to count it as a click */
- int slklines; /* lines in use by slk_init() */
- WINDOW *slk_winptr; /* window for slk */
- int linesrippedoff; /* lines ripped off via ripoffline() */
- int linesrippedoffontop; /* lines ripped off on
- top via ripoffline() */
- int delaytenths; /* 1/10ths second to wait block
- getch() for */
- bool _preserve; /* TRUE if screen background
- to be preserved */
- int _restore; /* specifies if screen background
- to be restored, and how */
- bool save_key_modifiers; /* TRUE if each key modifiers saved
- with each key press */
- bool return_key_modifiers; /* TRUE if modifier keys are
- returned as "real" keys */
- bool key_code; /* TRUE if last key is a special key;
- used internally by get_wch() */
-#ifdef XCURSES
- int XcurscrSize; /* size of Xcurscr shared memory block */
- bool sb_on;
- int sb_viewport_y;
- int sb_viewport_x;
- int sb_total_y;
- int sb_total_x;
- int sb_cur_y;
- int sb_cur_x;
-#endif
- short line_color; /* color of line attributes - default -1 */
-} SCREEN;
-
-/*----------------------------------------------------------------------
- *
- * PDCurses External Variables
- *
- */
-
-#ifdef PDC_DLL_BUILD
-# ifdef CURSES_LIBRARY
-# define PDCEX __declspec(dllexport) extern
-# else
-# define PDCEX __declspec(dllimport)
-# endif
-#else
-# define PDCEX extern
-#endif
-
-PDCEX int LINES; /* terminal height */
-PDCEX int COLS; /* terminal width */
-PDCEX WINDOW *stdscr; /* the default screen window */
-PDCEX WINDOW *curscr; /* the current screen image */
-PDCEX SCREEN *SP; /* curses variables */
-PDCEX MOUSE_STATUS Mouse_status;
-PDCEX int COLORS;
-PDCEX int COLOR_PAIRS;
-PDCEX int TABSIZE;
-PDCEX chtype acs_map[]; /* alternate character set map */
-PDCEX char ttytype[]; /* terminal name/description */
-
-/*man-start**************************************************************
-
-PDCurses Text Attributes
-========================
-
-Originally, PDCurses used a short (16 bits) for its chtype. To include
-color, a number of things had to be sacrificed from the strict Unix and
-System V support. The main problem was fitting all character attributes
-and color into an unsigned char (all 8 bits!).
-
-Today, PDCurses by default uses a long (32 bits) for its chtype, as in
-System V. The short chtype is still available, by undefining CHTYPE_LONG
-and rebuilding the library.
-
-The following is the structure of a win->_attrs chtype:
-
-short form:
-
--------------------------------------------------
-|15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
--------------------------------------------------
- color number | attrs | character eg 'a'
-
-The available non-color attributes are bold, reverse and blink. Others
-have no effect. The high order char is an index into an array of
-physical colors (defined in color.c) -- 32 foreground/background color
-pairs (5 bits) plus 3 bits for other attributes.
-
-long form:
-
-----------------------------------------------------------------------------
-|31|30|29|28|27|26|25|24|23|22|21|20|19|18|17|16|15|14|13|12|..| 3| 2| 1| 0|
-----------------------------------------------------------------------------
- color number | modifiers | character eg 'a'
-
-The available non-color attributes are bold, underline, invisible,
-right-line, left-line, protect, reverse and blink. 256 color pairs (8
-bits), 8 bits for other attributes, and 16 bits for character data.
-
-**man-end****************************************************************/
-
-/*** Video attribute macros ***/
-
-#define A_NORMAL (chtype)0
-
-#ifdef CHTYPE_LONG
-# define A_ALTCHARSET (chtype)0x00010000
-# define A_RIGHTLINE (chtype)0x00020000
-# define A_LEFTLINE (chtype)0x00040000
-# define A_INVIS (chtype)0x00080000
-# define A_UNDERLINE (chtype)0x00100000
-# define A_REVERSE (chtype)0x00200000
-# define A_BLINK (chtype)0x00400000
-# define A_BOLD (chtype)0x00800000
-
-# define A_ATTRIBUTES (chtype)0xffff0000
-# define A_CHARTEXT (chtype)0x0000ffff
-# define A_COLOR (chtype)0xff000000
-
-# define A_ITALIC A_INVIS
-# define A_PROTECT (A_UNDERLINE | A_LEFTLINE | A_RIGHTLINE)
-
-# define PDC_ATTR_SHIFT 19
-# define PDC_COLOR_SHIFT 24
-#else
-# define A_BOLD (chtype)0x0100 /* X/Open */
-# define A_REVERSE (chtype)0x0200 /* X/Open */
-# define A_BLINK (chtype)0x0400 /* X/Open */
-
-# define A_ATTRIBUTES (chtype)0xff00 /* X/Open */
-# define A_CHARTEXT (chtype)0x00ff /* X/Open */
-# define A_COLOR (chtype)0xf800 /* System V */
-
-# define A_ALTCHARSET A_NORMAL /* X/Open */
-# define A_PROTECT A_NORMAL /* X/Open */
-# define A_UNDERLINE A_NORMAL /* X/Open */
-
-# define A_LEFTLINE A_NORMAL
-# define A_RIGHTLINE A_NORMAL
-# define A_ITALIC A_NORMAL
-# define A_INVIS A_NORMAL
-
-# define PDC_ATTR_SHIFT 8
-# define PDC_COLOR_SHIFT 11
-#endif
-
-#define A_STANDOUT (A_REVERSE | A_BOLD) /* X/Open */
-#define A_DIM A_NORMAL
-
-#define CHR_MSK A_CHARTEXT /* Obsolete */
-#define ATR_MSK A_ATTRIBUTES /* Obsolete */
-#define ATR_NRM A_NORMAL /* Obsolete */
-
-/* For use with attr_t -- X/Open says, "these shall be distinct", so
- this is a non-conforming implementation. */
-
-#define WA_ALTCHARSET A_ALTCHARSET
-#define WA_BLINK A_BLINK
-#define WA_BOLD A_BOLD
-#define WA_DIM A_DIM
-#define WA_INVIS A_INVIS
-#define WA_LEFT A_LEFTLINE
-#define WA_PROTECT A_PROTECT
-#define WA_REVERSE A_REVERSE
-#define WA_RIGHT A_RIGHTLINE
-#define WA_STANDOUT A_STANDOUT
-#define WA_UNDERLINE A_UNDERLINE
-
-#define WA_HORIZONTAL A_NORMAL
-#define WA_LOW A_NORMAL
-#define WA_TOP A_NORMAL
-#define WA_VERTICAL A_NORMAL
-
-/*** Alternate character set macros ***/
-
-/* 'w' = 32-bit chtype; acs_map[] index | A_ALTCHARSET
- 'n' = 16-bit chtype; it gets the fallback set because no bit is
- available for A_ALTCHARSET */
-
-#ifdef CHTYPE_LONG
-# define ACS_PICK(w, n) ((chtype)w | A_ALTCHARSET)
-#else
-# define ACS_PICK(w, n) ((chtype)n)
-#endif
-
-/* VT100-compatible symbols -- box chars */
-
-#define ACS_ULCORNER ACS_PICK('l', '+')
-#define ACS_LLCORNER ACS_PICK('m', '+')
-#define ACS_URCORNER ACS_PICK('k', '+')
-#define ACS_LRCORNER ACS_PICK('j', '+')
-#define ACS_RTEE ACS_PICK('u', '+')
-#define ACS_LTEE ACS_PICK('t', '+')
-#define ACS_BTEE ACS_PICK('v', '+')
-#define ACS_TTEE ACS_PICK('w', '+')
-#define ACS_HLINE ACS_PICK('q', '-')
-#define ACS_VLINE ACS_PICK('x', '|')
-#define ACS_PLUS ACS_PICK('n', '+')
-
-/* VT100-compatible symbols -- other */
-
-#define ACS_S1 ACS_PICK('o', '-')
-#define ACS_S9 ACS_PICK('s', '_')
-#define ACS_DIAMOND ACS_PICK('`', '+')
-#define ACS_CKBOARD ACS_PICK('a', ':')
-#define ACS_DEGREE ACS_PICK('f', '\'')
-#define ACS_PLMINUS ACS_PICK('g', '#')
-#define ACS_BULLET ACS_PICK('~', 'o')
-
-/* Teletype 5410v1 symbols -- these are defined in SysV curses, but
- are not well-supported by most terminals. Stick to VT100 characters
- for optimum portability. */
-
-#define ACS_LARROW ACS_PICK(',', '<')
-#define ACS_RARROW ACS_PICK('+', '>')
-#define ACS_DARROW ACS_PICK('.', 'v')
-#define ACS_UARROW ACS_PICK('-', '^')
-#define ACS_BOARD ACS_PICK('h', '#')
-#define ACS_LANTERN ACS_PICK('i', '*')
-#define ACS_BLOCK ACS_PICK('0', '#')
-
-/* That goes double for these -- undocumented SysV symbols. Don't use
- them. */
-
-#define ACS_S3 ACS_PICK('p', '-')
-#define ACS_S7 ACS_PICK('r', '-')
-#define ACS_LEQUAL ACS_PICK('y', '<')
-#define ACS_GEQUAL ACS_PICK('z', '>')
-#define ACS_PI ACS_PICK('{', 'n')
-#define ACS_NEQUAL ACS_PICK('|', '+')
-#define ACS_STERLING ACS_PICK('}', 'L')
-
-/* Box char aliases */
-
-#define ACS_BSSB ACS_ULCORNER
-#define ACS_SSBB ACS_LLCORNER
-#define ACS_BBSS ACS_URCORNER
-#define ACS_SBBS ACS_LRCORNER
-#define ACS_SBSS ACS_RTEE
-#define ACS_SSSB ACS_LTEE
-#define ACS_SSBS ACS_BTEE
-#define ACS_BSSS ACS_TTEE
-#define ACS_BSBS ACS_HLINE
-#define ACS_SBSB ACS_VLINE
-#define ACS_SSSS ACS_PLUS
-
-/* cchar_t aliases */
-
-#ifdef PDC_WIDE
-# define WACS_ULCORNER (&(acs_map['l']))
-# define WACS_LLCORNER (&(acs_map['m']))
-# define WACS_URCORNER (&(acs_map['k']))
-# define WACS_LRCORNER (&(acs_map['j']))
-# define WACS_RTEE (&(acs_map['u']))
-# define WACS_LTEE (&(acs_map['t']))
-# define WACS_BTEE (&(acs_map['v']))
-# define WACS_TTEE (&(acs_map['w']))
-# define WACS_HLINE (&(acs_map['q']))
-# define WACS_VLINE (&(acs_map['x']))
-# define WACS_PLUS (&(acs_map['n']))
-
-# define WACS_S1 (&(acs_map['o']))
-# define WACS_S9 (&(acs_map['s']))
-# define WACS_DIAMOND (&(acs_map['`']))
-# define WACS_CKBOARD (&(acs_map['a']))
-# define WACS_DEGREE (&(acs_map['f']))
-# define WACS_PLMINUS (&(acs_map['g']))
-# define WACS_BULLET (&(acs_map['~']))
-
-# define WACS_LARROW (&(acs_map[',']))
-# define WACS_RARROW (&(acs_map['+']))
-# define WACS_DARROW (&(acs_map['.']))
-# define WACS_UARROW (&(acs_map['-']))
-# define WACS_BOARD (&(acs_map['h']))
-# define WACS_LANTERN (&(acs_map['i']))
-# define WACS_BLOCK (&(acs_map['0']))
-
-# define WACS_S3 (&(acs_map['p']))
-# define WACS_S7 (&(acs_map['r']))
-# define WACS_LEQUAL (&(acs_map['y']))
-# define WACS_GEQUAL (&(acs_map['z']))
-# define WACS_PI (&(acs_map['{']))
-# define WACS_NEQUAL (&(acs_map['|']))
-# define WACS_STERLING (&(acs_map['}']))
-
-# define WACS_BSSB WACS_ULCORNER
-# define WACS_SSBB WACS_LLCORNER
-# define WACS_BBSS WACS_URCORNER
-# define WACS_SBBS WACS_LRCORNER
-# define WACS_SBSS WACS_RTEE
-# define WACS_SSSB WACS_LTEE
-# define WACS_SSBS WACS_BTEE
-# define WACS_BSSS WACS_TTEE
-# define WACS_BSBS WACS_HLINE
-# define WACS_SBSB WACS_VLINE
-# define WACS_SSSS WACS_PLUS
-#endif
-
-/*** Color macros ***/
-
-#define COLOR_BLACK 0
-
-#ifdef PDC_RGB /* RGB */
-# define COLOR_RED 1
-# define COLOR_GREEN 2
-# define COLOR_BLUE 4
-#else /* BGR */
-# define COLOR_BLUE 1
-# define COLOR_GREEN 2
-# define COLOR_RED 4
-#endif
-
-#define COLOR_CYAN (COLOR_BLUE | COLOR_GREEN)
-#define COLOR_MAGENTA (COLOR_RED | COLOR_BLUE)
-#define COLOR_YELLOW (COLOR_RED | COLOR_GREEN)
-
-#define COLOR_WHITE 7
-
-/*----------------------------------------------------------------------
- *
- * Function and Keypad Key Definitions.
- * Many are just for compatibility.
- *
- */
-
-#define KEY_CODE_YES 0x100 /* If get_wch() gives a key code */
-
-#define KEY_BREAK 0x101 /* Not on PC KBD */
-#define KEY_DOWN 0x102 /* Down arrow key */
-#define KEY_UP 0x103 /* Up arrow key */
-#define KEY_LEFT 0x104 /* Left arrow key */
-#define KEY_RIGHT 0x105 /* Right arrow key */
-#define KEY_HOME 0x106 /* home key */
-#define KEY_BACKSPACE 0x107 /* not on pc */
-#define KEY_F0 0x108 /* function keys; 64 reserved */
-
-#define KEY_DL 0x148 /* delete line */
-#define KEY_IL 0x149 /* insert line */
-#define KEY_DC 0x14a /* delete character */
-#define KEY_IC 0x14b /* insert char or enter ins mode */
-#define KEY_EIC 0x14c /* exit insert char mode */
-#define KEY_CLEAR 0x14d /* clear screen */
-#define KEY_EOS 0x14e /* clear to end of screen */
-#define KEY_EOL 0x14f /* clear to end of line */
-#define KEY_SF 0x150 /* scroll 1 line forward */
-#define KEY_SR 0x151 /* scroll 1 line back (reverse) */
-#define KEY_NPAGE 0x152 /* next page */
-#define KEY_PPAGE 0x153 /* previous page */
-#define KEY_STAB 0x154 /* set tab */
-#define KEY_CTAB 0x155 /* clear tab */
-#define KEY_CATAB 0x156 /* clear all tabs */
-#define KEY_ENTER 0x157 /* enter or send (unreliable) */
-#define KEY_SRESET 0x158 /* soft/reset (partial/unreliable) */
-#define KEY_RESET 0x159 /* reset/hard reset (unreliable) */
-#define KEY_PRINT 0x15a /* print/copy */
-#define KEY_LL 0x15b /* home down/bottom (lower left) */
-#define KEY_ABORT 0x15c /* abort/terminate key (any) */
-#define KEY_SHELP 0x15d /* short help */
-#define KEY_LHELP 0x15e /* long help */
-#define KEY_BTAB 0x15f /* Back tab key */
-#define KEY_BEG 0x160 /* beg(inning) key */
-#define KEY_CANCEL 0x161 /* cancel key */
-#define KEY_CLOSE 0x162 /* close key */
-#define KEY_COMMAND 0x163 /* cmd (command) key */
-#define KEY_COPY 0x164 /* copy key */
-#define KEY_CREATE 0x165 /* create key */
-#define KEY_END 0x166 /* end key */
-#define KEY_EXIT 0x167 /* exit key */
-#define KEY_FIND 0x168 /* find key */
-#define KEY_HELP 0x169 /* help key */
-#define KEY_MARK 0x16a /* mark key */
-#define KEY_MESSAGE 0x16b /* message key */
-#define KEY_MOVE 0x16c /* move key */
-#define KEY_NEXT 0x16d /* next object key */
-#define KEY_OPEN 0x16e /* open key */
-#define KEY_OPTIONS 0x16f /* options key */
-#define KEY_PREVIOUS 0x170 /* previous object key */
-#define KEY_REDO 0x171 /* redo key */
-#define KEY_REFERENCE 0x172 /* ref(erence) key */
-#define KEY_REFRESH 0x173 /* refresh key */
-#define KEY_REPLACE 0x174 /* replace key */
-#define KEY_RESTART 0x175 /* restart key */
-#define KEY_RESUME 0x176 /* resume key */
-#define KEY_SAVE 0x177 /* save key */
-#define KEY_SBEG 0x178 /* shifted beginning key */
-#define KEY_SCANCEL 0x179 /* shifted cancel key */
-#define KEY_SCOMMAND 0x17a /* shifted command key */
-#define KEY_SCOPY 0x17b /* shifted copy key */
-#define KEY_SCREATE 0x17c /* shifted create key */
-#define KEY_SDC 0x17d /* shifted delete char key */
-#define KEY_SDL 0x17e /* shifted delete line key */
-#define KEY_SELECT 0x17f /* select key */
-#define KEY_SEND 0x180 /* shifted end key */
-#define KEY_SEOL 0x181 /* shifted clear line key */
-#define KEY_SEXIT 0x182 /* shifted exit key */
-#define KEY_SFIND 0x183 /* shifted find key */
-#define KEY_SHOME 0x184 /* shifted home key */
-#define KEY_SIC 0x185 /* shifted input key */
-
-#define KEY_SLEFT 0x187 /* shifted left arrow key */
-#define KEY_SMESSAGE 0x188 /* shifted message key */
-#define KEY_SMOVE 0x189 /* shifted move key */
-#define KEY_SNEXT 0x18a /* shifted next key */
-#define KEY_SOPTIONS 0x18b /* shifted options key */
-#define KEY_SPREVIOUS 0x18c /* shifted prev key */
-#define KEY_SPRINT 0x18d /* shifted print key */
-#define KEY_SREDO 0x18e /* shifted redo key */
-#define KEY_SREPLACE 0x18f /* shifted replace key */
-#define KEY_SRIGHT 0x190 /* shifted right arrow */
-#define KEY_SRSUME 0x191 /* shifted resume key */
-#define KEY_SSAVE 0x192 /* shifted save key */
-#define KEY_SSUSPEND 0x193 /* shifted suspend key */
-#define KEY_SUNDO 0x194 /* shifted undo key */
-#define KEY_SUSPEND 0x195 /* suspend key */
-#define KEY_UNDO 0x196 /* undo key */
-
-/* PDCurses-specific key definitions -- PC only */
-
-#define ALT_0 0x197
-#define ALT_1 0x198
-#define ALT_2 0x199
-#define ALT_3 0x19a
-#define ALT_4 0x19b
-#define ALT_5 0x19c
-#define ALT_6 0x19d
-#define ALT_7 0x19e
-#define ALT_8 0x19f
-#define ALT_9 0x1a0
-#define ALT_A 0x1a1
-#define ALT_B 0x1a2
-#define ALT_C 0x1a3
-#define ALT_D 0x1a4
-#define ALT_E 0x1a5
-#define ALT_F 0x1a6
-#define ALT_G 0x1a7
-#define ALT_H 0x1a8
-#define ALT_I 0x1a9
-#define ALT_J 0x1aa
-#define ALT_K 0x1ab
-#define ALT_L 0x1ac
-#define ALT_M 0x1ad
-#define ALT_N 0x1ae
-#define ALT_O 0x1af
-#define ALT_P 0x1b0
-#define ALT_Q 0x1b1
-#define ALT_R 0x1b2
-#define ALT_S 0x1b3
-#define ALT_T 0x1b4
-#define ALT_U 0x1b5
-#define ALT_V 0x1b6
-#define ALT_W 0x1b7
-#define ALT_X 0x1b8
-#define ALT_Y 0x1b9
-#define ALT_Z 0x1ba
-
-#define CTL_LEFT 0x1bb /* Control-Left-Arrow */
-#define CTL_RIGHT 0x1bc
-#define CTL_PGUP 0x1bd
-#define CTL_PGDN 0x1be
-#define CTL_HOME 0x1bf
-#define CTL_END 0x1c0
-
-#define KEY_A1 0x1c1 /* upper left on Virtual keypad */
-#define KEY_A2 0x1c2 /* upper middle on Virt. keypad */
-#define KEY_A3 0x1c3 /* upper right on Vir. keypad */
-#define KEY_B1 0x1c4 /* middle left on Virt. keypad */
-#define KEY_B2 0x1c5 /* center on Virt. keypad */
-#define KEY_B3 0x1c6 /* middle right on Vir. keypad */
-#define KEY_C1 0x1c7 /* lower left on Virt. keypad */
-#define KEY_C2 0x1c8 /* lower middle on Virt. keypad */
-#define KEY_C3 0x1c9 /* lower right on Vir. keypad */
-
-#define PADSLASH 0x1ca /* slash on keypad */
-#define PADENTER 0x1cb /* enter on keypad */
-#define CTL_PADENTER 0x1cc /* ctl-enter on keypad */
-#define ALT_PADENTER 0x1cd /* alt-enter on keypad */
-#define PADSTOP 0x1ce /* stop on keypad */
-#define PADSTAR 0x1cf /* star on keypad */
-#define PADMINUS 0x1d0 /* minus on keypad */
-#define PADPLUS 0x1d1 /* plus on keypad */
-#define CTL_PADSTOP 0x1d2 /* ctl-stop on keypad */
-#define CTL_PADCENTER 0x1d3 /* ctl-enter on keypad */
-#define CTL_PADPLUS 0x1d4 /* ctl-plus on keypad */
-#define CTL_PADMINUS 0x1d5 /* ctl-minus on keypad */
-#define CTL_PADSLASH 0x1d6 /* ctl-slash on keypad */
-#define CTL_PADSTAR 0x1d7 /* ctl-star on keypad */
-#define ALT_PADPLUS 0x1d8 /* alt-plus on keypad */
-#define ALT_PADMINUS 0x1d9 /* alt-minus on keypad */
-#define ALT_PADSLASH 0x1da /* alt-slash on keypad */
-#define ALT_PADSTAR 0x1db /* alt-star on keypad */
-#define ALT_PADSTOP 0x1dc /* alt-stop on keypad */
-#define CTL_INS 0x1dd /* ctl-insert */
-#define ALT_DEL 0x1de /* alt-delete */
-#define ALT_INS 0x1df /* alt-insert */
-#define CTL_UP 0x1e0 /* ctl-up arrow */
-#define CTL_DOWN 0x1e1 /* ctl-down arrow */
-#define CTL_TAB 0x1e2 /* ctl-tab */
-#define ALT_TAB 0x1e3
-#define ALT_MINUS 0x1e4
-#define ALT_EQUAL 0x1e5
-#define ALT_HOME 0x1e6
-#define ALT_PGUP 0x1e7
-#define ALT_PGDN 0x1e8
-#define ALT_END 0x1e9
-#define ALT_UP 0x1ea /* alt-up arrow */
-#define ALT_DOWN 0x1eb /* alt-down arrow */
-#define ALT_RIGHT 0x1ec /* alt-right arrow */
-#define ALT_LEFT 0x1ed /* alt-left arrow */
-#define ALT_ENTER 0x1ee /* alt-enter */
-#define ALT_ESC 0x1ef /* alt-escape */
-#define ALT_BQUOTE 0x1f0 /* alt-back quote */
-#define ALT_LBRACKET 0x1f1 /* alt-left bracket */
-#define ALT_RBRACKET 0x1f2 /* alt-right bracket */
-#define ALT_SEMICOLON 0x1f3 /* alt-semi-colon */
-#define ALT_FQUOTE 0x1f4 /* alt-forward quote */
-#define ALT_COMMA 0x1f5 /* alt-comma */
-#define ALT_STOP 0x1f6 /* alt-stop */
-#define ALT_FSLASH 0x1f7 /* alt-forward slash */
-#define ALT_BKSP 0x1f8 /* alt-backspace */
-#define CTL_BKSP 0x1f9 /* ctl-backspace */
-#define PAD0 0x1fa /* keypad 0 */
-
-#define CTL_PAD0 0x1fb /* ctl-keypad 0 */
-#define CTL_PAD1 0x1fc
-#define CTL_PAD2 0x1fd
-#define CTL_PAD3 0x1fe
-#define CTL_PAD4 0x1ff
-#define CTL_PAD5 0x200
-#define CTL_PAD6 0x201
-#define CTL_PAD7 0x202
-#define CTL_PAD8 0x203
-#define CTL_PAD9 0x204
-
-#define ALT_PAD0 0x205 /* alt-keypad 0 */
-#define ALT_PAD1 0x206
-#define ALT_PAD2 0x207
-#define ALT_PAD3 0x208
-#define ALT_PAD4 0x209
-#define ALT_PAD5 0x20a
-#define ALT_PAD6 0x20b
-#define ALT_PAD7 0x20c
-#define ALT_PAD8 0x20d
-#define ALT_PAD9 0x20e
-
-#define CTL_DEL 0x20f /* clt-delete */
-#define ALT_BSLASH 0x210 /* alt-back slash */
-#define CTL_ENTER 0x211 /* ctl-enter */
-
-#define SHF_PADENTER 0x212 /* shift-enter on keypad */
-#define SHF_PADSLASH 0x213 /* shift-slash on keypad */
-#define SHF_PADSTAR 0x214 /* shift-star on keypad */
-#define SHF_PADPLUS 0x215 /* shift-plus on keypad */
-#define SHF_PADMINUS 0x216 /* shift-minus on keypad */
-#define SHF_UP 0x217 /* shift-up on keypad */
-#define SHF_DOWN 0x218 /* shift-down on keypad */
-#define SHF_IC 0x219 /* shift-insert on keypad */
-#define SHF_DC 0x21a /* shift-delete on keypad */
-
-#define KEY_MOUSE 0x21b /* "mouse" key */
-#define KEY_SHIFT_L 0x21c /* Left-shift */
-#define KEY_SHIFT_R 0x21d /* Right-shift */
-#define KEY_CONTROL_L 0x21e /* Left-control */
-#define KEY_CONTROL_R 0x21f /* Right-control */
-#define KEY_ALT_L 0x220 /* Left-alt */
-#define KEY_ALT_R 0x221 /* Right-alt */
-#define KEY_RESIZE 0x222 /* Window resize */
-#define KEY_SUP 0x223 /* Shifted up arrow */
-#define KEY_SDOWN 0x224 /* Shifted down arrow */
-
-#define KEY_MIN KEY_BREAK /* Minimum curses key value */
-#define KEY_MAX KEY_SDOWN /* Maximum curses key */
-
-#define KEY_F(n) (KEY_F0 + (n))
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Function Declarations
- *
- */
-
-/* Standard */
-
-int addch(const chtype);
-int addchnstr(const chtype *, int);
-int addchstr(const chtype *);
-int addnstr(const char *, int);
-int addstr(const char *);
-int attroff(chtype);
-int attron(chtype);
-int attrset(chtype);
-int attr_get(attr_t *, short *, void *);
-int attr_off(attr_t, void *);
-int attr_on(attr_t, void *);
-int attr_set(attr_t, short, void *);
-int baudrate(void);
-int beep(void);
-int bkgd(chtype);
-void bkgdset(chtype);
-int border(chtype, chtype, chtype, chtype, chtype, chtype, chtype, chtype);
-int box(WINDOW *, chtype, chtype);
-bool can_change_color(void);
-int cbreak(void);
-int chgat(int, attr_t, short, const void *);
-int clearok(WINDOW *, bool);
-int clear(void);
-int clrtobot(void);
-int clrtoeol(void);
-int color_content(short, short *, short *, short *);
-int color_set(short, void *);
-int copywin(const WINDOW *, WINDOW *, int, int, int, int, int, int, int);
-int curs_set(int);
-int def_prog_mode(void);
-int def_shell_mode(void);
-int delay_output(int);
-int delch(void);
-int deleteln(void);
-void delscreen(SCREEN *);
-int delwin(WINDOW *);
-WINDOW *derwin(WINDOW *, int, int, int, int);
-int doupdate(void);
-WINDOW *dupwin(WINDOW *);
-int echochar(const chtype);
-int echo(void);
-int endwin(void);
-char erasechar(void);
-int erase(void);
-void filter(void);
-int flash(void);
-int flushinp(void);
-chtype getbkgd(WINDOW *);
-int getnstr(char *, int);
-int getstr(char *);
-WINDOW *getwin(FILE *);
-int halfdelay(int);
-bool has_colors(void);
-bool has_ic(void);
-bool has_il(void);
-int hline(chtype, int);
-void idcok(WINDOW *, bool);
-int idlok(WINDOW *, bool);
-void immedok(WINDOW *, bool);
-int inchnstr(chtype *, int);
-int inchstr(chtype *);
-chtype inch(void);
-int init_color(short, short, short, short);
-int init_pair(short, short, short);
-WINDOW *initscr(void);
-int innstr(char *, int);
-int insch(chtype);
-int insdelln(int);
-int insertln(void);
-int insnstr(const char *, int);
-int insstr(const char *);
-int instr(char *);
-int intrflush(WINDOW *, bool);
-bool isendwin(void);
-bool is_linetouched(WINDOW *, int);
-bool is_wintouched(WINDOW *);
-char *keyname(int);
-int keypad(WINDOW *, bool);
-char killchar(void);
-int leaveok(WINDOW *, bool);
-char *longname(void);
-int meta(WINDOW *, bool);
-int move(int, int);
-int mvaddch(int, int, const chtype);
-int mvaddchnstr(int, int, const chtype *, int);
-int mvaddchstr(int, int, const chtype *);
-int mvaddnstr(int, int, const char *, int);
-int mvaddstr(int, int, const char *);
-int mvchgat(int, int, int, attr_t, short, const void *);
-int mvcur(int, int, int, int);
-int mvdelch(int, int);
-int mvderwin(WINDOW *, int, int);
-int mvgetch(int, int);
-int mvgetnstr(int, int, char *, int);
-int mvgetstr(int, int, char *);
-int mvhline(int, int, chtype, int);
-chtype mvinch(int, int);
-int mvinchnstr(int, int, chtype *, int);
-int mvinchstr(int, int, chtype *);
-int mvinnstr(int, int, char *, int);
-int mvinsch(int, int, chtype);
-int mvinsnstr(int, int, const char *, int);
-int mvinsstr(int, int, const char *);
-int mvinstr(int, int, char *);
-int mvprintw(int, int, const char *, ...);
-int mvscanw(int, int, const char *, ...);
-int mvvline(int, int, chtype, int);
-int mvwaddchnstr(WINDOW *, int, int, const chtype *, int);
-int mvwaddchstr(WINDOW *, int, int, const chtype *);
-int mvwaddch(WINDOW *, int, int, const chtype);
-int mvwaddnstr(WINDOW *, int, int, const char *, int);
-int mvwaddstr(WINDOW *, int, int, const char *);
-int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
-int mvwdelch(WINDOW *, int, int);
-int mvwgetch(WINDOW *, int, int);
-int mvwgetnstr(WINDOW *, int, int, char *, int);
-int mvwgetstr(WINDOW *, int, int, char *);
-int mvwhline(WINDOW *, int, int, chtype, int);
-int mvwinchnstr(WINDOW *, int, int, chtype *, int);
-int mvwinchstr(WINDOW *, int, int, chtype *);
-chtype mvwinch(WINDOW *, int, int);
-int mvwinnstr(WINDOW *, int, int, char *, int);
-int mvwinsch(WINDOW *, int, int, chtype);
-int mvwinsnstr(WINDOW *, int, int, const char *, int);
-int mvwinsstr(WINDOW *, int, int, const char *);
-int mvwinstr(WINDOW *, int, int, char *);
-int mvwin(WINDOW *, int, int);
-int mvwprintw(WINDOW *, int, int, const char *, ...);
-int mvwscanw(WINDOW *, int, int, const char *, ...);
-int mvwvline(WINDOW *, int, int, chtype, int);
-int napms(int);
-WINDOW *newpad(int, int);
-SCREEN *newterm(const char *, FILE *, FILE *);
-WINDOW *newwin(int, int, int, int);
-int nl(void);
-int nocbreak(void);
-int nodelay(WINDOW *, bool);
-int noecho(void);
-int nonl(void);
-void noqiflush(void);
-int noraw(void);
-int notimeout(WINDOW *, bool);
-int overlay(const WINDOW *, WINDOW *);
-int overwrite(const WINDOW *, WINDOW *);
-int pair_content(short, short *, short *);
-int pechochar(WINDOW *, chtype);
-int pnoutrefresh(WINDOW *, int, int, int, int, int, int);
-int prefresh(WINDOW *, int, int, int, int, int, int);
-int printw(const char *, ...);
-int putwin(WINDOW *, FILE *);
-void qiflush(void);
-int raw(void);
-int redrawwin(WINDOW *);
-int refresh(void);
-int reset_prog_mode(void);
-int reset_shell_mode(void);
-int resetty(void);
-int ripoffline(int, int (*)(WINDOW *, int));
-int savetty(void);
-int scanw(const char *, ...);
-int scr_dump(const char *);
-int scr_init(const char *);
-int scr_restore(const char *);
-int scr_set(const char *);
-int scrl(int);
-int scroll(WINDOW *);
-int scrollok(WINDOW *, bool);
-SCREEN *set_term(SCREEN *);
-int setscrreg(int, int);
-int slk_attroff(const chtype);
-int slk_attr_off(const attr_t, void *);
-int slk_attron(const chtype);
-int slk_attr_on(const attr_t, void *);
-int slk_attrset(const chtype);
-int slk_attr_set(const attr_t, short, void *);
-int slk_clear(void);
-int slk_color(short);
-int slk_init(int);
-char *slk_label(int);
-int slk_noutrefresh(void);
-int slk_refresh(void);
-int slk_restore(void);
-int slk_set(int, const char *, int);
-int slk_touch(void);
-int standend(void);
-int standout(void);
-int start_color(void);
-WINDOW *subpad(WINDOW *, int, int, int, int);
-WINDOW *subwin(WINDOW *, int, int, int, int);
-int syncok(WINDOW *, bool);
-chtype termattrs(void);
-attr_t term_attrs(void);
-char *termname(void);
-void timeout(int);
-int touchline(WINDOW *, int, int);
-int touchwin(WINDOW *);
-int typeahead(int);
-int untouchwin(WINDOW *);
-void use_env(bool);
-int vidattr(chtype);
-int vid_attr(attr_t, short, void *);
-int vidputs(chtype, int (*)(int));
-int vid_puts(attr_t, short, void *, int (*)(int));
-int vline(chtype, int);
-int vw_printw(WINDOW *, const char *, va_list);
-int vwprintw(WINDOW *, const char *, va_list);
-int vw_scanw(WINDOW *, const char *, va_list);
-int vwscanw(WINDOW *, const char *, va_list);
-int waddchnstr(WINDOW *, const chtype *, int);
-int waddchstr(WINDOW *, const chtype *);
-int waddch(WINDOW *, const chtype);
-int waddnstr(WINDOW *, const char *, int);
-int waddstr(WINDOW *, const char *);
-int wattroff(WINDOW *, chtype);
-int wattron(WINDOW *, chtype);
-int wattrset(WINDOW *, chtype);
-int wattr_get(WINDOW *, attr_t *, short *, void *);
-int wattr_off(WINDOW *, attr_t, void *);
-int wattr_on(WINDOW *, attr_t, void *);
-int wattr_set(WINDOW *, attr_t, short, void *);
-void wbkgdset(WINDOW *, chtype);
-int wbkgd(WINDOW *, chtype);
-int wborder(WINDOW *, chtype, chtype, chtype, chtype,
- chtype, chtype, chtype, chtype);
-int wchgat(WINDOW *, int, attr_t, short, const void *);
-int wclear(WINDOW *);
-int wclrtobot(WINDOW *);
-int wclrtoeol(WINDOW *);
-int wcolor_set(WINDOW *, short, void *);
-void wcursyncup(WINDOW *);
-int wdelch(WINDOW *);
-int wdeleteln(WINDOW *);
-int wechochar(WINDOW *, const chtype);
-int werase(WINDOW *);
-int wgetch(WINDOW *);
-int wgetnstr(WINDOW *, char *, int);
-int wgetstr(WINDOW *, char *);
-int whline(WINDOW *, chtype, int);
-int winchnstr(WINDOW *, chtype *, int);
-int winchstr(WINDOW *, chtype *);
-chtype winch(WINDOW *);
-int winnstr(WINDOW *, char *, int);
-int winsch(WINDOW *, chtype);
-int winsdelln(WINDOW *, int);
-int winsertln(WINDOW *);
-int winsnstr(WINDOW *, const char *, int);
-int winsstr(WINDOW *, const char *);
-int winstr(WINDOW *, char *);
-int wmove(WINDOW *, int, int);
-int wnoutrefresh(WINDOW *);
-int wprintw(WINDOW *, const char *, ...);
-int wredrawln(WINDOW *, int, int);
-int wrefresh(WINDOW *);
-int wscanw(WINDOW *, const char *, ...);
-int wscrl(WINDOW *, int);
-int wsetscrreg(WINDOW *, int, int);
-int wstandend(WINDOW *);
-int wstandout(WINDOW *);
-void wsyncdown(WINDOW *);
-void wsyncup(WINDOW *);
-void wtimeout(WINDOW *, int);
-int wtouchln(WINDOW *, int, int, int);
-int wvline(WINDOW *, chtype, int);
-
-/* Wide-character functions */
-
-#ifdef PDC_WIDE
-int addnwstr(const wchar_t *, int);
-int addwstr(const wchar_t *);
-int add_wch(const cchar_t *);
-int add_wchnstr(const cchar_t *, int);
-int add_wchstr(const cchar_t *);
-int border_set(const cchar_t *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *);
-int box_set(WINDOW *, const cchar_t *, const cchar_t *);
-int echo_wchar(const cchar_t *);
-int erasewchar(wchar_t *);
-int getbkgrnd(cchar_t *);
-int getcchar(const cchar_t *, wchar_t *, attr_t *, short *, void *);
-int getn_wstr(wint_t *, int);
-int get_wch(wint_t *);
-int get_wstr(wint_t *);
-int hline_set(const cchar_t *, int);
-int innwstr(wchar_t *, int);
-int ins_nwstr(const wchar_t *, int);
-int ins_wch(const cchar_t *);
-int ins_wstr(const wchar_t *);
-int inwstr(wchar_t *);
-int in_wch(cchar_t *);
-int in_wchnstr(cchar_t *, int);
-int in_wchstr(cchar_t *);
-char *key_name(wchar_t);
-int killwchar(wchar_t *);
-int mvaddnwstr(int, int, const wchar_t *, int);
-int mvaddwstr(int, int, const wchar_t *);
-int mvadd_wch(int, int, const cchar_t *);
-int mvadd_wchnstr(int, int, const cchar_t *, int);
-int mvadd_wchstr(int, int, const cchar_t *);
-int mvgetn_wstr(int, int, wint_t *, int);
-int mvget_wch(int, int, wint_t *);
-int mvget_wstr(int, int, wint_t *);
-int mvhline_set(int, int, const cchar_t *, int);
-int mvinnwstr(int, int, wchar_t *, int);
-int mvins_nwstr(int, int, const wchar_t *, int);
-int mvins_wch(int, int, const cchar_t *);
-int mvins_wstr(int, int, const wchar_t *);
-int mvinwstr(int, int, wchar_t *);
-int mvin_wch(int, int, cchar_t *);
-int mvin_wchnstr(int, int, cchar_t *, int);
-int mvin_wchstr(int, int, cchar_t *);
-int mvvline_set(int, int, const cchar_t *, int);
-int mvwaddnwstr(WINDOW *, int, int, const wchar_t *, int);
-int mvwaddwstr(WINDOW *, int, int, const wchar_t *);
-int mvwadd_wch(WINDOW *, int, int, const cchar_t *);
-int mvwadd_wchnstr(WINDOW *, int, int, const cchar_t *, int);
-int mvwadd_wchstr(WINDOW *, int, int, const cchar_t *);
-int mvwgetn_wstr(WINDOW *, int, int, wint_t *, int);
-int mvwget_wch(WINDOW *, int, int, wint_t *);
-int mvwget_wstr(WINDOW *, int, int, wint_t *);
-int mvwhline_set(WINDOW *, int, int, const cchar_t *, int);
-int mvwinnwstr(WINDOW *, int, int, wchar_t *, int);
-int mvwins_nwstr(WINDOW *, int, int, const wchar_t *, int);
-int mvwins_wch(WINDOW *, int, int, const cchar_t *);
-int mvwins_wstr(WINDOW *, int, int, const wchar_t *);
-int mvwin_wch(WINDOW *, int, int, cchar_t *);
-int mvwin_wchnstr(WINDOW *, int, int, cchar_t *, int);
-int mvwin_wchstr(WINDOW *, int, int, cchar_t *);
-int mvwinwstr(WINDOW *, int, int, wchar_t *);
-int mvwvline_set(WINDOW *, int, int, const cchar_t *, int);
-int pecho_wchar(WINDOW *, const cchar_t*);
-int setcchar(cchar_t*, const wchar_t*, const attr_t, short, const void*);
-int slk_wset(int, const wchar_t *, int);
-int unget_wch(const wchar_t);
-int vline_set(const cchar_t *, int);
-int waddnwstr(WINDOW *, const wchar_t *, int);
-int waddwstr(WINDOW *, const wchar_t *);
-int wadd_wch(WINDOW *, const cchar_t *);
-int wadd_wchnstr(WINDOW *, const cchar_t *, int);
-int wadd_wchstr(WINDOW *, const cchar_t *);
-int wbkgrnd(WINDOW *, const cchar_t *);
-void wbkgrndset(WINDOW *, const cchar_t *);
-int wborder_set(WINDOW *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *, const cchar_t *);
-int wecho_wchar(WINDOW *, const cchar_t *);
-int wgetbkgrnd(WINDOW *, cchar_t *);
-int wgetn_wstr(WINDOW *, wint_t *, int);
-int wget_wch(WINDOW *, wint_t *);
-int wget_wstr(WINDOW *, wint_t *);
-int whline_set(WINDOW *, const cchar_t *, int);
-int winnwstr(WINDOW *, wchar_t *, int);
-int wins_nwstr(WINDOW *, const wchar_t *, int);
-int wins_wch(WINDOW *, const cchar_t *);
-int wins_wstr(WINDOW *, const wchar_t *);
-int winwstr(WINDOW *, wchar_t *);
-int win_wch(WINDOW *, cchar_t *);
-int win_wchnstr(WINDOW *, cchar_t *, int);
-int win_wchstr(WINDOW *, cchar_t *);
-wchar_t *wunctrl(cchar_t *);
-int wvline_set(WINDOW *, const cchar_t *, int);
-#endif
-
-/* Quasi-standard */
-
-chtype getattrs(WINDOW *);
-int getbegx(WINDOW *);
-int getbegy(WINDOW *);
-int getmaxx(WINDOW *);
-int getmaxy(WINDOW *);
-int getparx(WINDOW *);
-int getpary(WINDOW *);
-int getcurx(WINDOW *);
-int getcury(WINDOW *);
-void traceoff(void);
-void traceon(void);
-char *unctrl(chtype);
-
-int crmode(void);
-int nocrmode(void);
-int draino(int);
-int resetterm(void);
-int fixterm(void);
-int saveterm(void);
-int setsyx(int, int);
-
-int mouse_set(unsigned long);
-int mouse_on(unsigned long);
-int mouse_off(unsigned long);
-int request_mouse_pos(void);
-int map_button(unsigned long);
-void wmouse_position(WINDOW *, int *, int *);
-unsigned long getmouse(void);
-unsigned long getbmap(void);
-
-/* ncurses */
-
-int assume_default_colors(int, int);
-const char *curses_version(void);
-bool has_key(int);
-int use_default_colors(void);
-int wresize(WINDOW *, int, int);
-
-int mouseinterval(int);
-mmask_t mousemask(mmask_t, mmask_t *);
-bool mouse_trafo(int *, int *, bool);
-int nc_getmouse(MEVENT *);
-int ungetmouse(MEVENT *);
-bool wenclose(const WINDOW *, int, int);
-bool wmouse_trafo(const WINDOW *, int *, int *, bool);
-
-/* PDCurses */
-
-int addrawch(chtype);
-int insrawch(chtype);
-bool is_termresized(void);
-int mvaddrawch(int, int, chtype);
-int mvdeleteln(int, int);
-int mvinsertln(int, int);
-int mvinsrawch(int, int, chtype);
-int mvwaddrawch(WINDOW *, int, int, chtype);
-int mvwdeleteln(WINDOW *, int, int);
-int mvwinsertln(WINDOW *, int, int);
-int mvwinsrawch(WINDOW *, int, int, chtype);
-int raw_output(bool);
-int resize_term(int, int);
-WINDOW *resize_window(WINDOW *, int, int);
-int waddrawch(WINDOW *, chtype);
-int winsrawch(WINDOW *, chtype);
-char wordchar(void);
-
-#ifdef PDC_WIDE
-wchar_t *slk_wlabel(int);
-#endif
-
-void PDC_debug(const char *, ...);
-int PDC_ungetch(int);
-int PDC_set_blink(bool);
-int PDC_set_line_color(short);
-void PDC_set_title(const char *);
-
-int PDC_clearclipboard(void);
-int PDC_freeclipboard(char *);
-int PDC_getclipboard(char **, long *);
-int PDC_setclipboard(const char *, long);
-
-unsigned long PDC_get_input_fd(void);
-unsigned long PDC_get_key_modifiers(void);
-int PDC_return_key_modifiers(bool);
-int PDC_save_key_modifiers(bool);
-
-#ifdef XCURSES
-WINDOW *Xinitscr(int, char **);
-void XCursesExit(void);
-int sb_init(void);
-int sb_set_horz(int, int, int);
-int sb_set_vert(int, int, int);
-int sb_get_horz(int *, int *, int *);
-int sb_get_vert(int *, int *, int *);
-int sb_refresh(void);
-#endif
-
-/*** Functions defined as macros ***/
-
-/* getch() and ungetch() conflict with some DOS libraries */
-
-#define getch() wgetch(stdscr)
-#define ungetch(ch) PDC_ungetch(ch)
-
-#define COLOR_PAIR(n) (((chtype)(n) << PDC_COLOR_SHIFT) & A_COLOR)
-#define PAIR_NUMBER(n) (((n) & A_COLOR) >> PDC_COLOR_SHIFT)
-
-/* These will _only_ work as macros */
-
-#define getbegyx(w, y, x) (y = getbegy(w), x = getbegx(w))
-#define getmaxyx(w, y, x) (y = getmaxy(w), x = getmaxx(w))
-#define getparyx(w, y, x) (y = getpary(w), x = getparx(w))
-#define getyx(w, y, x) (y = getcury(w), x = getcurx(w))
-
-#define getsyx(y, x) { if (curscr->_leaveit) (y)=(x)=-1; \
- else getyx(curscr,(y),(x)); }
-
-#ifdef NCURSES_MOUSE_VERSION
-# define getmouse(x) nc_getmouse(x)
-#endif
-
-/* return codes from PDC_getclipboard() and PDC_setclipboard() calls */
-
-#define PDC_CLIP_SUCCESS 0
-#define PDC_CLIP_ACCESS_ERROR 1
-#define PDC_CLIP_EMPTY 2
-#define PDC_CLIP_MEMORY_ERROR 3
-
-/* PDCurses key modifier masks */
-
-#define PDC_KEY_MODIFIER_SHIFT 1
-#define PDC_KEY_MODIFIER_CONTROL 2
-#define PDC_KEY_MODIFIER_ALT 4
-#define PDC_KEY_MODIFIER_NUMLOCK 8
-
-#if defined(__cplusplus) || defined(__cplusplus__) || defined(__CPLUSPLUS)
-# undef bool
-}
-#endif
-
-#endif /* __PDCURSES__ */
diff --git a/samtools-0.1.19/win32/zconf.h b/samtools-0.1.19/win32/zconf.h
deleted file mode 100644
index 03a9431..0000000
--- a/samtools-0.1.19/win32/zconf.h
+++ /dev/null
@@ -1,332 +0,0 @@
-/* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2005 Jean-loup Gailly.
- * For conditions of distribution and use, see copyright notice in zlib.h
- */
-
-/* @(#) $Id$ */
-
-#ifndef ZCONF_H
-#define ZCONF_H
-
-/*
- * If you *really* need a unique prefix for all types and library functions,
- * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it.
- */
-#ifdef Z_PREFIX
-# define deflateInit_ z_deflateInit_
-# define deflate z_deflate
-# define deflateEnd z_deflateEnd
-# define inflateInit_ z_inflateInit_
-# define inflate z_inflate
-# define inflateEnd z_inflateEnd
-# define deflateInit2_ z_deflateInit2_
-# define deflateSetDictionary z_deflateSetDictionary
-# define deflateCopy z_deflateCopy
-# define deflateReset z_deflateReset
-# define deflateParams z_deflateParams
-# define deflateBound z_deflateBound
-# define deflatePrime z_deflatePrime
-# define inflateInit2_ z_inflateInit2_
-# define inflateSetDictionary z_inflateSetDictionary
-# define inflateSync z_inflateSync
-# define inflateSyncPoint z_inflateSyncPoint
-# define inflateCopy z_inflateCopy
-# define inflateReset z_inflateReset
-# define inflateBack z_inflateBack
-# define inflateBackEnd z_inflateBackEnd
-# define compress z_compress
-# define compress2 z_compress2
-# define compressBound z_compressBound
-# define uncompress z_uncompress
-# define adler32 z_adler32
-# define crc32 z_crc32
-# define get_crc_table z_get_crc_table
-# define zError z_zError
-
-# define alloc_func z_alloc_func
-# define free_func z_free_func
-# define in_func z_in_func
-# define out_func z_out_func
-# define Byte z_Byte
-# define uInt z_uInt
-# define uLong z_uLong
-# define Bytef z_Bytef
-# define charf z_charf
-# define intf z_intf
-# define uIntf z_uIntf
-# define uLongf z_uLongf
-# define voidpf z_voidpf
-# define voidp z_voidp
-#endif
-
-#if defined(__MSDOS__) && !defined(MSDOS)
-# define MSDOS
-#endif
-#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2)
-# define OS2
-#endif
-#if defined(_WINDOWS) && !defined(WINDOWS)
-# define WINDOWS
-#endif
-#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__)
-# ifndef WIN32
-# define WIN32
-# endif
-#endif
-#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32)
-# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__)
-# ifndef SYS16BIT
-# define SYS16BIT
-# endif
-# endif
-#endif
-
-/*
- * Compile with -DMAXSEG_64K if the alloc function cannot allocate more
- * than 64k bytes at a time (needed on systems with 16-bit int).
- */
-#ifdef SYS16BIT
-# define MAXSEG_64K
-#endif
-#ifdef MSDOS
-# define UNALIGNED_OK
-#endif
-
-#ifdef __STDC_VERSION__
-# ifndef STDC
-# define STDC
-# endif
-# if __STDC_VERSION__ >= 199901L
-# ifndef STDC99
-# define STDC99
-# endif
-# endif
-#endif
-#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus))
-# define STDC
-#endif
-#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__))
-# define STDC
-#endif
-#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32))
-# define STDC
-#endif
-#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__))
-# define STDC
-#endif
-
-#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */
-# define STDC
-#endif
-
-#ifndef STDC
-# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */
-# define const /* note: need a more gentle solution here */
-# endif
-#endif
-
-/* Some Mac compilers merge all .h files incorrectly: */
-#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__)
-# define NO_DUMMY_DECL
-#endif
-
-/* Maximum value for memLevel in deflateInit2 */
-#ifndef MAX_MEM_LEVEL
-# ifdef MAXSEG_64K
-# define MAX_MEM_LEVEL 8
-# else
-# define MAX_MEM_LEVEL 9
-# endif
-#endif
-
-/* Maximum value for windowBits in deflateInit2 and inflateInit2.
- * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
- * created by gzip. (Files created by minigzip can still be extracted by
- * gzip.)
- */
-#ifndef MAX_WBITS
-# define MAX_WBITS 15 /* 32K LZ77 window */
-#endif
-
-/* The memory requirements for deflate are (in bytes):
- (1 << (windowBits+2)) + (1 << (memLevel+9))
- that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values)
- plus a few kilobytes for small objects. For example, if you want to reduce
- the default memory requirements from 256K to 128K, compile with
- make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
- Of course this will generally degrade compression (there's no free lunch).
-
- The memory requirements for inflate are (in bytes) 1 << windowBits
- that is, 32K for windowBits=15 (default value) plus a few kilobytes
- for small objects.
-*/
-
- /* Type declarations */
-
-#ifndef OF /* function prototypes */
-# ifdef STDC
-# define OF(args) args
-# else
-# define OF(args) ()
-# endif
-#endif
-
-/* The following definitions for FAR are needed only for MSDOS mixed
- * model programming (small or medium model with some far allocations).
- * This was tested only with MSC; for other MSDOS compilers you may have
- * to define NO_MEMCPY in zutil.h. If you don't need the mixed model,
- * just define FAR to be empty.
- */
-#ifdef SYS16BIT
-# if defined(M_I86SM) || defined(M_I86MM)
- /* MSC small or medium model */
-# define SMALL_MEDIUM
-# ifdef _MSC_VER
-# define FAR _far
-# else
-# define FAR far
-# endif
-# endif
-# if (defined(__SMALL__) || defined(__MEDIUM__))
- /* Turbo C small or medium model */
-# define SMALL_MEDIUM
-# ifdef __BORLANDC__
-# define FAR _far
-# else
-# define FAR far
-# endif
-# endif
-#endif
-
-#if defined(WINDOWS) || defined(WIN32)
- /* If building or using zlib as a DLL, define ZLIB_DLL.
- * This is not mandatory, but it offers a little performance increase.
- */
-# ifdef ZLIB_DLL
-# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500))
-# ifdef ZLIB_INTERNAL
-# define ZEXTERN extern __declspec(dllexport)
-# else
-# define ZEXTERN extern __declspec(dllimport)
-# endif
-# endif
-# endif /* ZLIB_DLL */
- /* If building or using zlib with the WINAPI/WINAPIV calling convention,
- * define ZLIB_WINAPI.
- * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI.
- */
-# ifdef ZLIB_WINAPI
-# ifdef FAR
-# undef FAR
-# endif
-# include <windows.h>
- /* No need for _export, use ZLIB.DEF instead. */
- /* For complete Windows compatibility, use WINAPI, not __stdcall. */
-# define ZEXPORT WINAPI
-# ifdef WIN32
-# define ZEXPORTVA WINAPIV
-# else
-# define ZEXPORTVA FAR CDECL
-# endif
-# endif
-#endif
-
-#if defined (__BEOS__)
-# ifdef ZLIB_DLL
-# ifdef ZLIB_INTERNAL
-# define ZEXPORT __declspec(dllexport)
-# define ZEXPORTVA __declspec(dllexport)
-# else
-# define ZEXPORT __declspec(dllimport)
-# define ZEXPORTVA __declspec(dllimport)
-# endif
-# endif
-#endif
-
-#ifndef ZEXTERN
-# define ZEXTERN extern
-#endif
-#ifndef ZEXPORT
-# define ZEXPORT
-#endif
-#ifndef ZEXPORTVA
-# define ZEXPORTVA
-#endif
-
-#ifndef FAR
-# define FAR
-#endif
-
-#if !defined(__MACTYPES__)
-typedef unsigned char Byte; /* 8 bits */
-#endif
-typedef unsigned int uInt; /* 16 bits or more */
-typedef unsigned long uLong; /* 32 bits or more */
-
-#ifdef SMALL_MEDIUM
- /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */
-# define Bytef Byte FAR
-#else
- typedef Byte FAR Bytef;
-#endif
-typedef char FAR charf;
-typedef int FAR intf;
-typedef uInt FAR uIntf;
-typedef uLong FAR uLongf;
-
-#ifdef STDC
- typedef void const *voidpc;
- typedef void FAR *voidpf;
- typedef void *voidp;
-#else
- typedef Byte const *voidpc;
- typedef Byte FAR *voidpf;
- typedef Byte *voidp;
-#endif
-
-#if 0 /* HAVE_UNISTD_H -- this line is updated by ./configure */
-# include <sys/types.h> /* for off_t */
-# include <unistd.h> /* for SEEK_* and off_t */
-# ifdef VMS
-# include <unixio.h> /* for off_t */
-# endif
-# define z_off_t off_t
-#endif
-#ifndef SEEK_SET
-# define SEEK_SET 0 /* Seek from beginning of file. */
-# define SEEK_CUR 1 /* Seek from current position. */
-# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */
-#endif
-#ifndef z_off_t
-# define z_off_t long
-#endif
-
-#if defined(__OS400__)
-# define NO_vsnprintf
-#endif
-
-#if defined(__MVS__)
-# define NO_vsnprintf
-# ifdef FAR
-# undef FAR
-# endif
-#endif
-
-/* MVS linker does not support external names larger than 8 bytes */
-#if defined(__MVS__)
-# pragma map(deflateInit_,"DEIN")
-# pragma map(deflateInit2_,"DEIN2")
-# pragma map(deflateEnd,"DEEND")
-# pragma map(deflateBound,"DEBND")
-# pragma map(inflateInit_,"ININ")
-# pragma map(inflateInit2_,"ININ2")
-# pragma map(inflateEnd,"INEND")
-# pragma map(inflateSync,"INSY")
-# pragma map(inflateSetDictionary,"INSEDI")
-# pragma map(compressBound,"CMBND")
-# pragma map(inflate_table,"INTABL")
-# pragma map(inflate_fast,"INFA")
-# pragma map(inflate_copyright,"INCOPY")
-#endif
-
-#endif /* ZCONF_H */
diff --git a/samtools-0.1.19/win32/zlib.h b/samtools-0.1.19/win32/zlib.h
deleted file mode 100644
index 0228179..0000000
--- a/samtools-0.1.19/win32/zlib.h
+++ /dev/null
@@ -1,1357 +0,0 @@
-/* zlib.h -- interface of the 'zlib' general purpose compression library
- version 1.2.3, July 18th, 2005
-
- Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- Jean-loup Gailly Mark Adler
- jloup at gzip.org madler at alumni.caltech.edu
-
-
- The data format used by the zlib library is described by RFCs (Request for
- Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
- (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
-*/
-
-#ifndef ZLIB_H
-#define ZLIB_H
-
-#include "zconf.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ZLIB_VERSION "1.2.3"
-#define ZLIB_VERNUM 0x1230
-
-/*
- The 'zlib' compression library provides in-memory compression and
- decompression functions, including integrity checks of the uncompressed
- data. This version of the library supports only one compression method
- (deflation) but other algorithms will be added later and will have the same
- stream interface.
-
- Compression can be done in a single step if the buffers are large
- enough (for example if an input file is mmap'ed), or can be done by
- repeated calls of the compression function. In the latter case, the
- application must provide more input and/or consume the output
- (providing more output space) before each call.
-
- The compressed data format used by default by the in-memory functions is
- the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped
- around a deflate stream, which is itself documented in RFC 1951.
-
- The library also supports reading and writing files in gzip (.gz) format
- with an interface similar to that of stdio using the functions that start
- with "gz". The gzip format is different from the zlib format. gzip is a
- gzip wrapper, documented in RFC 1952, wrapped around a deflate stream.
-
- This library can optionally read and write gzip streams in memory as well.
-
- The zlib format was designed to be compact and fast for use in memory
- and on communications channels. The gzip format was designed for single-
- file compression on file systems, has a larger header than zlib to maintain
- directory information, and uses a different, slower check method than zlib.
-
- The library does not install any signal handler. The decoder checks
- the consistency of the compressed data, so the library should never
- crash even in case of corrupted input.
-*/
-
-typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
-typedef void (*free_func) OF((voidpf opaque, voidpf address));
-
-struct internal_state;
-
-typedef struct z_stream_s {
- Bytef *next_in; /* next input byte */
- uInt avail_in; /* number of bytes available at next_in */
- uLong total_in; /* total nb of input bytes read so far */
-
- Bytef *next_out; /* next output byte should be put there */
- uInt avail_out; /* remaining free space at next_out */
- uLong total_out; /* total nb of bytes output so far */
-
- char *msg; /* last error message, NULL if no error */
- struct internal_state FAR *state; /* not visible by applications */
-
- alloc_func zalloc; /* used to allocate the internal state */
- free_func zfree; /* used to free the internal state */
- voidpf opaque; /* private data object passed to zalloc and zfree */
-
- int data_type; /* best guess about the data type: binary or text */
- uLong adler; /* adler32 value of the uncompressed data */
- uLong reserved; /* reserved for future use */
-} z_stream;
-
-typedef z_stream FAR *z_streamp;
-
-/*
- gzip header information passed to and from zlib routines. See RFC 1952
- for more details on the meanings of these fields.
-*/
-typedef struct gz_header_s {
- int text; /* true if compressed data believed to be text */
- uLong time; /* modification time */
- int xflags; /* extra flags (not used when writing a gzip file) */
- int os; /* operating system */
- Bytef *extra; /* pointer to extra field or Z_NULL if none */
- uInt extra_len; /* extra field length (valid if extra != Z_NULL) */
- uInt extra_max; /* space at extra (only when reading header) */
- Bytef *name; /* pointer to zero-terminated file name or Z_NULL */
- uInt name_max; /* space at name (only when reading header) */
- Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */
- uInt comm_max; /* space at comment (only when reading header) */
- int hcrc; /* true if there was or will be a header crc */
- int done; /* true when done reading gzip header (not used
- when writing a gzip file) */
-} gz_header;
-
-typedef gz_header FAR *gz_headerp;
-
-/*
- The application must update next_in and avail_in when avail_in has
- dropped to zero. It must update next_out and avail_out when avail_out
- has dropped to zero. The application must initialize zalloc, zfree and
- opaque before calling the init function. All other fields are set by the
- compression library and must not be updated by the application.
-
- The opaque value provided by the application will be passed as the first
- parameter for calls of zalloc and zfree. This can be useful for custom
- memory management. The compression library attaches no meaning to the
- opaque value.
-
- zalloc must return Z_NULL if there is not enough memory for the object.
- If zlib is used in a multi-threaded application, zalloc and zfree must be
- thread safe.
-
- On 16-bit systems, the functions zalloc and zfree must be able to allocate
- exactly 65536 bytes, but will not be required to allocate more than this
- if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS,
- pointers returned by zalloc for objects of exactly 65536 bytes *must*
- have their offset normalized to zero. The default allocation function
- provided by this library ensures this (see zutil.c). To reduce memory
- requirements and avoid any allocation of 64K objects, at the expense of
- compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h).
-
- The fields total_in and total_out can be used for statistics or
- progress reports. After compression, total_in holds the total size of
- the uncompressed data and may be saved for use in the decompressor
- (particularly if the decompressor wants to decompress everything in
- a single step).
-*/
-
- /* constants */
-
-#define Z_NO_FLUSH 0
-#define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */
-#define Z_SYNC_FLUSH 2
-#define Z_FULL_FLUSH 3
-#define Z_FINISH 4
-#define Z_BLOCK 5
-/* Allowed flush values; see deflate() and inflate() below for details */
-
-#define Z_OK 0
-#define Z_STREAM_END 1
-#define Z_NEED_DICT 2
-#define Z_ERRNO (-1)
-#define Z_STREAM_ERROR (-2)
-#define Z_DATA_ERROR (-3)
-#define Z_MEM_ERROR (-4)
-#define Z_BUF_ERROR (-5)
-#define Z_VERSION_ERROR (-6)
-/* Return codes for the compression/decompression functions. Negative
- * values are errors, positive values are used for special but normal events.
- */
-
-#define Z_NO_COMPRESSION 0
-#define Z_BEST_SPEED 1
-#define Z_BEST_COMPRESSION 9
-#define Z_DEFAULT_COMPRESSION (-1)
-/* compression levels */
-
-#define Z_FILTERED 1
-#define Z_HUFFMAN_ONLY 2
-#define Z_RLE 3
-#define Z_FIXED 4
-#define Z_DEFAULT_STRATEGY 0
-/* compression strategy; see deflateInit2() below for details */
-
-#define Z_BINARY 0
-#define Z_TEXT 1
-#define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */
-#define Z_UNKNOWN 2
-/* Possible values of the data_type field (though see inflate()) */
-
-#define Z_DEFLATED 8
-/* The deflate compression method (the only one supported in this version) */
-
-#define Z_NULL 0 /* for initializing zalloc, zfree, opaque */
-
-#define zlib_version zlibVersion()
-/* for compatibility with versions < 1.0.2 */
-
- /* basic functions */
-
-ZEXTERN const char * ZEXPORT zlibVersion OF((void));
-/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
- If the first character differs, the library code actually used is
- not compatible with the zlib.h header file used by the application.
- This check is automatically made by deflateInit and inflateInit.
- */
-
-/*
-ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
-
- Initializes the internal stream state for compression. The fields
- zalloc, zfree and opaque must be initialized before by the caller.
- If zalloc and zfree are set to Z_NULL, deflateInit updates them to
- use default allocation functions.
-
- The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
- 1 gives best speed, 9 gives best compression, 0 gives no compression at
- all (the input data is simply copied a block at a time).
- Z_DEFAULT_COMPRESSION requests a default compromise between speed and
- compression (currently equivalent to level 6).
-
- deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_STREAM_ERROR if level is not a valid compression level,
- Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
- with the version assumed by the caller (ZLIB_VERSION).
- msg is set to null if there is no error message. deflateInit does not
- perform any compression: this will be done by deflate().
-*/
-
-
-ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
-/*
- deflate compresses as much data as possible, and stops when the input
- buffer becomes empty or the output buffer becomes full. It may introduce some
- output latency (reading input without producing any output) except when
- forced to flush.
-
- The detailed semantics are as follows. deflate performs one or both of the
- following actions:
-
- - Compress more input starting at next_in and update next_in and avail_in
- accordingly. If not all input can be processed (because there is not
- enough room in the output buffer), next_in and avail_in are updated and
- processing will resume at this point for the next call of deflate().
-
- - Provide more output starting at next_out and update next_out and avail_out
- accordingly. This action is forced if the parameter flush is non zero.
- Forcing flush frequently degrades the compression ratio, so this parameter
- should be set only when necessary (in interactive applications).
- Some output may be provided even if flush is not set.
-
- Before the call of deflate(), the application should ensure that at least
- one of the actions is possible, by providing more input and/or consuming
- more output, and updating avail_in or avail_out accordingly; avail_out
- should never be zero before the call. The application can consume the
- compressed output when it wants, for example when the output buffer is full
- (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK
- and with zero avail_out, it must be called again after making room in the
- output buffer because there might be more output pending.
-
- Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to
- decide how much data to accumualte before producing output, in order to
- maximize compression.
-
- If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
- flushed to the output buffer and the output is aligned on a byte boundary, so
- that the decompressor can get all input data available so far. (In particular
- avail_in is zero after the call if enough output space has been provided
- before the call.) Flushing may degrade compression for some compression
- algorithms and so it should be used only when necessary.
-
- If flush is set to Z_FULL_FLUSH, all output is flushed as with
- Z_SYNC_FLUSH, and the compression state is reset so that decompression can
- restart from this point if previous compressed data has been damaged or if
- random access is desired. Using Z_FULL_FLUSH too often can seriously degrade
- compression.
-
- If deflate returns with avail_out == 0, this function must be called again
- with the same value of the flush parameter and more output space (updated
- avail_out), until the flush is complete (deflate returns with non-zero
- avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that
- avail_out is greater than six to avoid repeated flush markers due to
- avail_out == 0 on return.
-
- If the parameter flush is set to Z_FINISH, pending input is processed,
- pending output is flushed and deflate returns with Z_STREAM_END if there
- was enough output space; if deflate returns with Z_OK, this function must be
- called again with Z_FINISH and more output space (updated avail_out) but no
- more input data, until it returns with Z_STREAM_END or an error. After
- deflate has returned Z_STREAM_END, the only possible operations on the
- stream are deflateReset or deflateEnd.
-
- Z_FINISH can be used immediately after deflateInit if all the compression
- is to be done in a single step. In this case, avail_out must be at least
- the value returned by deflateBound (see below). If deflate does not return
- Z_STREAM_END, then it must be called again as described above.
-
- deflate() sets strm->adler to the adler32 checksum of all input read
- so far (that is, total_in bytes).
-
- deflate() may update strm->data_type if it can make a good guess about
- the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered
- binary. This field is only for information purposes and does not affect
- the compression algorithm in any manner.
-
- deflate() returns Z_OK if some progress has been made (more input
- processed or more output produced), Z_STREAM_END if all input has been
- consumed and all output has been produced (only when flush is set to
- Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
- if next_in or next_out was NULL), Z_BUF_ERROR if no progress is possible
- (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not
- fatal, and deflate() can be called again with more input and more output
- space to continue compressing.
-*/
-
-
-ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
-/*
- All dynamically allocated data structures for this stream are freed.
- This function discards any unprocessed input and does not flush any
- pending output.
-
- deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
- stream state was inconsistent, Z_DATA_ERROR if the stream was freed
- prematurely (some input or output was discarded). In the error case,
- msg may be set but then points to a static string (which must not be
- deallocated).
-*/
-
-
-/*
-ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
-
- Initializes the internal stream state for decompression. The fields
- next_in, avail_in, zalloc, zfree and opaque must be initialized before by
- the caller. If next_in is not Z_NULL and avail_in is large enough (the exact
- value depends on the compression method), inflateInit determines the
- compression method from the zlib header and allocates all data structures
- accordingly; otherwise the allocation will be deferred to the first call of
- inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to
- use default allocation functions.
-
- inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
- version assumed by the caller. msg is set to null if there is no error
- message. inflateInit does not perform any decompression apart from reading
- the zlib header if present: this will be done by inflate(). (So next_in and
- avail_in may be modified, but next_out and avail_out are unchanged.)
-*/
-
-
-ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
-/*
- inflate decompresses as much data as possible, and stops when the input
- buffer becomes empty or the output buffer becomes full. It may introduce
- some output latency (reading input without producing any output) except when
- forced to flush.
-
- The detailed semantics are as follows. inflate performs one or both of the
- following actions:
-
- - Decompress more input starting at next_in and update next_in and avail_in
- accordingly. If not all input can be processed (because there is not
- enough room in the output buffer), next_in is updated and processing
- will resume at this point for the next call of inflate().
-
- - Provide more output starting at next_out and update next_out and avail_out
- accordingly. inflate() provides as much output as possible, until there
- is no more input data or no more space in the output buffer (see below
- about the flush parameter).
-
- Before the call of inflate(), the application should ensure that at least
- one of the actions is possible, by providing more input and/or consuming
- more output, and updating the next_* and avail_* values accordingly.
- The application can consume the uncompressed output when it wants, for
- example when the output buffer is full (avail_out == 0), or after each
- call of inflate(). If inflate returns Z_OK and with zero avail_out, it
- must be called again after making room in the output buffer because there
- might be more output pending.
-
- The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH,
- Z_FINISH, or Z_BLOCK. Z_SYNC_FLUSH requests that inflate() flush as much
- output as possible to the output buffer. Z_BLOCK requests that inflate() stop
- if and when it gets to the next deflate block boundary. When decoding the
- zlib or gzip format, this will cause inflate() to return immediately after
- the header and before the first block. When doing a raw inflate, inflate()
- will go ahead and process the first block, and will return when it gets to
- the end of that block, or when it runs out of data.
-
- The Z_BLOCK option assists in appending to or combining deflate streams.
- Also to assist in this, on return inflate() will set strm->data_type to the
- number of unused bits in the last byte taken from strm->next_in, plus 64
- if inflate() is currently decoding the last block in the deflate stream,
- plus 128 if inflate() returned immediately after decoding an end-of-block
- code or decoding the complete header up to just before the first byte of the
- deflate stream. The end-of-block will not be indicated until all of the
- uncompressed data from that block has been written to strm->next_out. The
- number of unused bits may in general be greater than seven, except when
- bit 7 of data_type is set, in which case the number of unused bits will be
- less than eight.
-
- inflate() should normally be called until it returns Z_STREAM_END or an
- error. However if all decompression is to be performed in a single step
- (a single call of inflate), the parameter flush should be set to
- Z_FINISH. In this case all pending input is processed and all pending
- output is flushed; avail_out must be large enough to hold all the
- uncompressed data. (The size of the uncompressed data may have been saved
- by the compressor for this purpose.) The next operation on this stream must
- be inflateEnd to deallocate the decompression state. The use of Z_FINISH
- is never required, but can be used to inform inflate that a faster approach
- may be used for the single inflate() call.
-
- In this implementation, inflate() always flushes as much output as
- possible to the output buffer, and always uses the faster approach on the
- first call. So the only effect of the flush parameter in this implementation
- is on the return value of inflate(), as noted below, or when it returns early
- because Z_BLOCK is used.
-
- If a preset dictionary is needed after this call (see inflateSetDictionary
- below), inflate sets strm->adler to the adler32 checksum of the dictionary
- chosen by the compressor and returns Z_NEED_DICT; otherwise it sets
- strm->adler to the adler32 checksum of all output produced so far (that is,
- total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described
- below. At the end of the stream, inflate() checks that its computed adler32
- checksum is equal to that saved by the compressor and returns Z_STREAM_END
- only if the checksum is correct.
-
- inflate() will decompress and check either zlib-wrapped or gzip-wrapped
- deflate data. The header type is detected automatically. Any information
- contained in the gzip header is not retained, so applications that need that
- information should instead use raw inflate, see inflateInit2() below, or
- inflateBack() and perform their own processing of the gzip header and
- trailer.
-
- inflate() returns Z_OK if some progress has been made (more input processed
- or more output produced), Z_STREAM_END if the end of the compressed data has
- been reached and all uncompressed output has been produced, Z_NEED_DICT if a
- preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
- corrupted (input stream not conforming to the zlib format or incorrect check
- value), Z_STREAM_ERROR if the stream structure was inconsistent (for example
- if next_in or next_out was NULL), Z_MEM_ERROR if there was not enough memory,
- Z_BUF_ERROR if no progress is possible or if there was not enough room in the
- output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and
- inflate() can be called again with more input and more output space to
- continue decompressing. If Z_DATA_ERROR is returned, the application may then
- call inflateSync() to look for a good compression block if a partial recovery
- of the data is desired.
-*/
-
-
-ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
-/*
- All dynamically allocated data structures for this stream are freed.
- This function discards any unprocessed input and does not flush any
- pending output.
-
- inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
- was inconsistent. In the error case, msg may be set but then points to a
- static string (which must not be deallocated).
-*/
-
- /* Advanced functions */
-
-/*
- The following functions are needed only in some special applications.
-*/
-
-/*
-ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
- int level,
- int method,
- int windowBits,
- int memLevel,
- int strategy));
-
- This is another version of deflateInit with more compression options. The
- fields next_in, zalloc, zfree and opaque must be initialized before by
- the caller.
-
- The method parameter is the compression method. It must be Z_DEFLATED in
- this version of the library.
-
- The windowBits parameter is the base two logarithm of the window size
- (the size of the history buffer). It should be in the range 8..15 for this
- version of the library. Larger values of this parameter result in better
- compression at the expense of memory usage. The default value is 15 if
- deflateInit is used instead.
-
- windowBits can also be -8..-15 for raw deflate. In this case, -windowBits
- determines the window size. deflate() will then generate raw deflate data
- with no zlib header or trailer, and will not compute an adler32 check value.
-
- windowBits can also be greater than 15 for optional gzip encoding. Add
- 16 to windowBits to write a simple gzip header and trailer around the
- compressed data instead of a zlib wrapper. The gzip header will have no
- file name, no extra data, no comment, no modification time (set to zero),
- no header crc, and the operating system will be set to 255 (unknown). If a
- gzip stream is being written, strm->adler is a crc32 instead of an adler32.
-
- The memLevel parameter specifies how much memory should be allocated
- for the internal compression state. memLevel=1 uses minimum memory but
- is slow and reduces compression ratio; memLevel=9 uses maximum memory
- for optimal speed. The default value is 8. See zconf.h for total memory
- usage as a function of windowBits and memLevel.
-
- The strategy parameter is used to tune the compression algorithm. Use the
- value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
- filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no
- string match), or Z_RLE to limit match distances to one (run-length
- encoding). Filtered data consists mostly of small values with a somewhat
- random distribution. In this case, the compression algorithm is tuned to
- compress them better. The effect of Z_FILTERED is to force more Huffman
- coding and less string matching; it is somewhat intermediate between
- Z_DEFAULT and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as
- Z_HUFFMAN_ONLY, but give better compression for PNG image data. The strategy
- parameter only affects the compression ratio but not the correctness of the
- compressed output even if it is not set appropriately. Z_FIXED prevents the
- use of dynamic Huffman codes, allowing for a simpler decoder for special
- applications.
-
- deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_STREAM_ERROR if a parameter is invalid (such as an invalid
- method). msg is set to null if there is no error message. deflateInit2 does
- not perform any compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
- const Bytef *dictionary,
- uInt dictLength));
-/*
- Initializes the compression dictionary from the given byte sequence
- without producing any compressed output. This function must be called
- immediately after deflateInit, deflateInit2 or deflateReset, before any
- call of deflate. The compressor and decompressor must use exactly the same
- dictionary (see inflateSetDictionary).
-
- The dictionary should consist of strings (byte sequences) that are likely
- to be encountered later in the data to be compressed, with the most commonly
- used strings preferably put towards the end of the dictionary. Using a
- dictionary is most useful when the data to be compressed is short and can be
- predicted with good accuracy; the data can then be compressed better than
- with the default empty dictionary.
-
- Depending on the size of the compression data structures selected by
- deflateInit or deflateInit2, a part of the dictionary may in effect be
- discarded, for example if the dictionary is larger than the window size in
- deflate or deflate2. Thus the strings most likely to be useful should be
- put at the end of the dictionary, not at the front. In addition, the
- current implementation of deflate will use at most the window size minus
- 262 bytes of the provided dictionary.
-
- Upon return of this function, strm->adler is set to the adler32 value
- of the dictionary; the decompressor may later use this value to determine
- which dictionary has been used by the compressor. (The adler32 value
- applies to the whole dictionary even if only a subset of the dictionary is
- actually used by the compressor.) If a raw deflate was requested, then the
- adler32 value is not computed and strm->adler is not set.
-
- deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
- parameter is invalid (such as NULL dictionary) or the stream state is
- inconsistent (for example if deflate has already been called for this stream
- or if the compression method is bsort). deflateSetDictionary does not
- perform any compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
- z_streamp source));
-/*
- Sets the destination stream as a complete copy of the source stream.
-
- This function can be useful when several compression strategies will be
- tried, for example when there are several ways of pre-processing the input
- data with a filter. The streams that will be discarded should then be freed
- by calling deflateEnd. Note that deflateCopy duplicates the internal
- compression state which can be quite large, so this strategy is slow and
- can consume lots of memory.
-
- deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
- (such as zalloc being NULL). msg is left unchanged in both source and
- destination.
-*/
-
-ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
-/*
- This function is equivalent to deflateEnd followed by deflateInit,
- but does not free and reallocate all the internal compression state.
- The stream will keep the same compression level and any other attributes
- that may have been set by deflateInit2.
-
- deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent (such as zalloc or state being NULL).
-*/
-
-ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
- int level,
- int strategy));
-/*
- Dynamically update the compression level and compression strategy. The
- interpretation of level and strategy is as in deflateInit2. This can be
- used to switch between compression and straight copy of the input data, or
- to switch to a different kind of input data requiring a different
- strategy. If the compression level is changed, the input available so far
- is compressed with the old level (and may be flushed); the new level will
- take effect only at the next call of deflate().
-
- Before the call of deflateParams, the stream state must be set as for
- a call of deflate(), since the currently available input may have to
- be compressed and flushed. In particular, strm->avail_out must be non-zero.
-
- deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
- stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR
- if strm->avail_out was zero.
-*/
-
-ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
- int good_length,
- int max_lazy,
- int nice_length,
- int max_chain));
-/*
- Fine tune deflate's internal compression parameters. This should only be
- used by someone who understands the algorithm used by zlib's deflate for
- searching for the best matching string, and even then only by the most
- fanatic optimizer trying to squeeze out the last compressed bit for their
- specific input data. Read the deflate.c source code for the meaning of the
- max_lazy, good_length, nice_length, and max_chain parameters.
-
- deflateTune() can be called after deflateInit() or deflateInit2(), and
- returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream.
- */
-
-ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
- uLong sourceLen));
-/*
- deflateBound() returns an upper bound on the compressed size after
- deflation of sourceLen bytes. It must be called after deflateInit()
- or deflateInit2(). This would be used to allocate an output buffer
- for deflation in a single pass, and so would be called before deflate().
-*/
-
-ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
- int bits,
- int value));
-/*
- deflatePrime() inserts bits in the deflate output stream. The intent
- is that this function is used to start off the deflate output with the
- bits leftover from a previous deflate stream when appending to it. As such,
- this function can only be used for raw deflate, and must be used before the
- first deflate() call after a deflateInit2() or deflateReset(). bits must be
- less than or equal to 16, and that many of the least significant bits of
- value will be inserted in the output.
-
- deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
- gz_headerp head));
-/*
- deflateSetHeader() provides gzip header information for when a gzip
- stream is requested by deflateInit2(). deflateSetHeader() may be called
- after deflateInit2() or deflateReset() and before the first call of
- deflate(). The text, time, os, extra field, name, and comment information
- in the provided gz_header structure are written to the gzip header (xflag is
- ignored -- the extra flags are set according to the compression level). The
- caller must assure that, if not Z_NULL, name and comment are terminated with
- a zero byte, and that if extra is not Z_NULL, that extra_len bytes are
- available there. If hcrc is true, a gzip header crc is included. Note that
- the current versions of the command-line version of gzip (up through version
- 1.3.x) do not support header crc's, and will report that it is a "multi-part
- gzip file" and give up.
-
- If deflateSetHeader is not used, the default gzip header has text false,
- the time set to zero, and os set to 255, with no extra, name, or comment
- fields. The gzip header is returned to the default state by deflateReset().
-
- deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-/*
-ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
- int windowBits));
-
- This is another version of inflateInit with an extra parameter. The
- fields next_in, avail_in, zalloc, zfree and opaque must be initialized
- before by the caller.
-
- The windowBits parameter is the base two logarithm of the maximum window
- size (the size of the history buffer). It should be in the range 8..15 for
- this version of the library. The default value is 15 if inflateInit is used
- instead. windowBits must be greater than or equal to the windowBits value
- provided to deflateInit2() while compressing, or it must be equal to 15 if
- deflateInit2() was not used. If a compressed stream with a larger window
- size is given as input, inflate() will return with the error code
- Z_DATA_ERROR instead of trying to allocate a larger window.
-
- windowBits can also be -8..-15 for raw inflate. In this case, -windowBits
- determines the window size. inflate() will then process raw deflate data,
- not looking for a zlib or gzip header, not generating a check value, and not
- looking for any check values for comparison at the end of the stream. This
- is for use with other formats that use the deflate compressed data format
- such as zip. Those formats provide their own check values. If a custom
- format is developed using the raw deflate format for compressed data, it is
- recommended that a check value such as an adler32 or a crc32 be applied to
- the uncompressed data as is done in the zlib, gzip, and zip formats. For
- most applications, the zlib format should be used as is. Note that comments
- above on the use in deflateInit2() applies to the magnitude of windowBits.
-
- windowBits can also be greater than 15 for optional gzip decoding. Add
- 32 to windowBits to enable zlib and gzip decoding with automatic header
- detection, or add 16 to decode only the gzip format (the zlib format will
- return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is
- a crc32 instead of an adler32.
-
- inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_STREAM_ERROR if a parameter is invalid (such as a null strm). msg
- is set to null if there is no error message. inflateInit2 does not perform
- any decompression apart from reading the zlib header if present: this will
- be done by inflate(). (So next_in and avail_in may be modified, but next_out
- and avail_out are unchanged.)
-*/
-
-ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
- const Bytef *dictionary,
- uInt dictLength));
-/*
- Initializes the decompression dictionary from the given uncompressed byte
- sequence. This function must be called immediately after a call of inflate,
- if that call returned Z_NEED_DICT. The dictionary chosen by the compressor
- can be determined from the adler32 value returned by that call of inflate.
- The compressor and decompressor must use exactly the same dictionary (see
- deflateSetDictionary). For raw inflate, this function can be called
- immediately after inflateInit2() or inflateReset() and before any call of
- inflate() to set the dictionary. The application must insure that the
- dictionary that was used for compression is provided.
-
- inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
- parameter is invalid (such as NULL dictionary) or the stream state is
- inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
- expected one (incorrect adler32 value). inflateSetDictionary does not
- perform any decompression: this will be done by subsequent calls of
- inflate().
-*/
-
-ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
-/*
- Skips invalid compressed data until a full flush point (see above the
- description of deflate with Z_FULL_FLUSH) can be found, or until all
- available input is skipped. No output is provided.
-
- inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR
- if no more input was provided, Z_DATA_ERROR if no flush point has been found,
- or Z_STREAM_ERROR if the stream structure was inconsistent. In the success
- case, the application may save the current current value of total_in which
- indicates where valid compressed data was found. In the error case, the
- application may repeatedly call inflateSync, providing more input each time,
- until success or end of the input data.
-*/
-
-ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
- z_streamp source));
-/*
- Sets the destination stream as a complete copy of the source stream.
-
- This function can be useful when randomly accessing a large stream. The
- first pass through the stream can periodically record the inflate state,
- allowing restarting inflate at those points when randomly accessing the
- stream.
-
- inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
- (such as zalloc being NULL). msg is left unchanged in both source and
- destination.
-*/
-
-ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
-/*
- This function is equivalent to inflateEnd followed by inflateInit,
- but does not free and reallocate all the internal decompression state.
- The stream will keep attributes that may have been set by inflateInit2.
-
- inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent (such as zalloc or state being NULL).
-*/
-
-ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
- int bits,
- int value));
-/*
- This function inserts bits in the inflate input stream. The intent is
- that this function is used to start inflating at a bit position in the
- middle of a byte. The provided bits will be used before any bytes are used
- from next_in. This function should only be used with raw inflate, and
- should be used before the first inflate() call after inflateInit2() or
- inflateReset(). bits must be less than or equal to 16, and that many of the
- least significant bits of value will be inserted in the input.
-
- inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
- gz_headerp head));
-/*
- inflateGetHeader() requests that gzip header information be stored in the
- provided gz_header structure. inflateGetHeader() may be called after
- inflateInit2() or inflateReset(), and before the first call of inflate().
- As inflate() processes the gzip stream, head->done is zero until the header
- is completed, at which time head->done is set to one. If a zlib stream is
- being decoded, then head->done is set to -1 to indicate that there will be
- no gzip header information forthcoming. Note that Z_BLOCK can be used to
- force inflate() to return immediately after header processing is complete
- and before any actual data is decompressed.
-
- The text, time, xflags, and os fields are filled in with the gzip header
- contents. hcrc is set to true if there is a header CRC. (The header CRC
- was valid if done is set to one.) If extra is not Z_NULL, then extra_max
- contains the maximum number of bytes to write to extra. Once done is true,
- extra_len contains the actual extra field length, and extra contains the
- extra field, or that field truncated if extra_max is less than extra_len.
- If name is not Z_NULL, then up to name_max characters are written there,
- terminated with a zero unless the length is greater than name_max. If
- comment is not Z_NULL, then up to comm_max characters are written there,
- terminated with a zero unless the length is greater than comm_max. When
- any of extra, name, or comment are not Z_NULL and the respective field is
- not present in the header, then that field is set to Z_NULL to signal its
- absence. This allows the use of deflateSetHeader() with the returned
- structure to duplicate the header. However if those fields are set to
- allocated memory, then the application will need to save those pointers
- elsewhere so that they can be eventually freed.
-
- If inflateGetHeader is not used, then the header information is simply
- discarded. The header is always checked for validity, including the header
- CRC if present. inflateReset() will reset the process to discard the header
- information. The application would need to call inflateGetHeader() again to
- retrieve the header from the next gzip stream.
-
- inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-/*
-ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
- unsigned char FAR *window));
-
- Initialize the internal stream state for decompression using inflateBack()
- calls. The fields zalloc, zfree and opaque in strm must be initialized
- before the call. If zalloc and zfree are Z_NULL, then the default library-
- derived memory allocation routines are used. windowBits is the base two
- logarithm of the window size, in the range 8..15. window is a caller
- supplied buffer of that size. Except for special applications where it is
- assured that deflate was used with small window sizes, windowBits must be 15
- and a 32K byte window must be supplied to be able to decompress general
- deflate streams.
-
- See inflateBack() for the usage of these routines.
-
- inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of
- the paramaters are invalid, Z_MEM_ERROR if the internal state could not
- be allocated, or Z_VERSION_ERROR if the version of the library does not
- match the version of the header file.
-*/
-
-typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *));
-typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned));
-
-ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
- in_func in, void FAR *in_desc,
- out_func out, void FAR *out_desc));
-/*
- inflateBack() does a raw inflate with a single call using a call-back
- interface for input and output. This is more efficient than inflate() for
- file i/o applications in that it avoids copying between the output and the
- sliding window by simply making the window itself the output buffer. This
- function trusts the application to not change the output buffer passed by
- the output function, at least until inflateBack() returns.
-
- inflateBackInit() must be called first to allocate the internal state
- and to initialize the state with the user-provided window buffer.
- inflateBack() may then be used multiple times to inflate a complete, raw
- deflate stream with each call. inflateBackEnd() is then called to free
- the allocated state.
-
- A raw deflate stream is one with no zlib or gzip header or trailer.
- This routine would normally be used in a utility that reads zip or gzip
- files and writes out uncompressed files. The utility would decode the
- header and process the trailer on its own, hence this routine expects
- only the raw deflate stream to decompress. This is different from the
- normal behavior of inflate(), which expects either a zlib or gzip header and
- trailer around the deflate stream.
-
- inflateBack() uses two subroutines supplied by the caller that are then
- called by inflateBack() for input and output. inflateBack() calls those
- routines until it reads a complete deflate stream and writes out all of the
- uncompressed data, or until it encounters an error. The function's
- parameters and return types are defined above in the in_func and out_func
- typedefs. inflateBack() will call in(in_desc, &buf) which should return the
- number of bytes of provided input, and a pointer to that input in buf. If
- there is no input available, in() must return zero--buf is ignored in that
- case--and inflateBack() will return a buffer error. inflateBack() will call
- out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out()
- should return zero on success, or non-zero on failure. If out() returns
- non-zero, inflateBack() will return with an error. Neither in() nor out()
- are permitted to change the contents of the window provided to
- inflateBackInit(), which is also the buffer that out() uses to write from.
- The length written by out() will be at most the window size. Any non-zero
- amount of input may be provided by in().
-
- For convenience, inflateBack() can be provided input on the first call by
- setting strm->next_in and strm->avail_in. If that input is exhausted, then
- in() will be called. Therefore strm->next_in must be initialized before
- calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called
- immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in
- must also be initialized, and then if strm->avail_in is not zero, input will
- initially be taken from strm->next_in[0 .. strm->avail_in - 1].
-
- The in_desc and out_desc parameters of inflateBack() is passed as the
- first parameter of in() and out() respectively when they are called. These
- descriptors can be optionally used to pass any information that the caller-
- supplied in() and out() functions need to do their job.
-
- On return, inflateBack() will set strm->next_in and strm->avail_in to
- pass back any unused input that was provided by the last in() call. The
- return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR
- if in() or out() returned an error, Z_DATA_ERROR if there was a format
- error in the deflate stream (in which case strm->msg is set to indicate the
- nature of the error), or Z_STREAM_ERROR if the stream was not properly
- initialized. In the case of Z_BUF_ERROR, an input or output error can be
- distinguished using strm->next_in which will be Z_NULL only if in() returned
- an error. If strm->next is not Z_NULL, then the Z_BUF_ERROR was due to
- out() returning non-zero. (in() will always be called before out(), so
- strm->next_in is assured to be defined if out() returns non-zero.) Note
- that inflateBack() cannot return Z_OK.
-*/
-
-ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
-/*
- All memory allocated by inflateBackInit() is freed.
-
- inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream
- state was inconsistent.
-*/
-
-ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
-/* Return flags indicating compile-time options.
-
- Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other:
- 1.0: size of uInt
- 3.2: size of uLong
- 5.4: size of voidpf (pointer)
- 7.6: size of z_off_t
-
- Compiler, assembler, and debug options:
- 8: DEBUG
- 9: ASMV or ASMINF -- use ASM code
- 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention
- 11: 0 (reserved)
-
- One-time table building (smaller code, but not thread-safe if true):
- 12: BUILDFIXED -- build static block decoding tables when needed
- 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed
- 14,15: 0 (reserved)
-
- Library content (indicates missing functionality):
- 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking
- deflate code when not needed)
- 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect
- and decode gzip streams (to avoid linking crc code)
- 18-19: 0 (reserved)
-
- Operation variations (changes in library functionality):
- 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate
- 21: FASTEST -- deflate algorithm with only one, lowest compression level
- 22,23: 0 (reserved)
-
- The sprintf variant used by gzprintf (zero is best):
- 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format
- 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure!
- 26: 0 = returns value, 1 = void -- 1 means inferred string length returned
-
- Remainder:
- 27-31: 0 (reserved)
- */
-
-
- /* utility functions */
-
-/*
- The following utility functions are implemented on top of the
- basic stream-oriented functions. To simplify the interface, some
- default options are assumed (compression level and memory usage,
- standard memory allocation functions). The source code of these
- utility functions can easily be modified if you need special options.
-*/
-
-ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen,
- const Bytef *source, uLong sourceLen));
-/*
- Compresses the source buffer into the destination buffer. sourceLen is
- the byte length of the source buffer. Upon entry, destLen is the total
- size of the destination buffer, which must be at least the value returned
- by compressBound(sourceLen). Upon exit, destLen is the actual size of the
- compressed buffer.
- This function can be used to compress a whole file at once if the
- input file is mmap'ed.
- compress returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_BUF_ERROR if there was not enough room in the output
- buffer.
-*/
-
-ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen,
- const Bytef *source, uLong sourceLen,
- int level));
-/*
- Compresses the source buffer into the destination buffer. The level
- parameter has the same meaning as in deflateInit. sourceLen is the byte
- length of the source buffer. Upon entry, destLen is the total size of the
- destination buffer, which must be at least the value returned by
- compressBound(sourceLen). Upon exit, destLen is the actual size of the
- compressed buffer.
-
- compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_BUF_ERROR if there was not enough room in the output buffer,
- Z_STREAM_ERROR if the level parameter is invalid.
-*/
-
-ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen));
-/*
- compressBound() returns an upper bound on the compressed size after
- compress() or compress2() on sourceLen bytes. It would be used before
- a compress() or compress2() call to allocate the destination buffer.
-*/
-
-ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen,
- const Bytef *source, uLong sourceLen));
-/*
- Decompresses the source buffer into the destination buffer. sourceLen is
- the byte length of the source buffer. Upon entry, destLen is the total
- size of the destination buffer, which must be large enough to hold the
- entire uncompressed data. (The size of the uncompressed data must have
- been saved previously by the compressor and transmitted to the decompressor
- by some mechanism outside the scope of this compression library.)
- Upon exit, destLen is the actual size of the compressed buffer.
- This function can be used to decompress a whole file at once if the
- input file is mmap'ed.
-
- uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_BUF_ERROR if there was not enough room in the output
- buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete.
-*/
-
-
-typedef voidp gzFile;
-
-ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));
-/*
- Opens a gzip (.gz) file for reading or writing. The mode parameter
- is as in fopen ("rb" or "wb") but can also include a compression level
- ("wb9") or a strategy: 'f' for filtered data as in "wb6f", 'h' for
- Huffman only compression as in "wb1h", or 'R' for run-length encoding
- as in "wb1R". (See the description of deflateInit2 for more information
- about the strategy parameter.)
-
- gzopen can be used to read a file which is not in gzip format; in this
- case gzread will directly read from the file without decompression.
-
- gzopen returns NULL if the file could not be opened or if there was
- insufficient memory to allocate the (de)compression state; errno
- can be checked to distinguish the two cases (if errno is zero, the
- zlib error is Z_MEM_ERROR). */
-
-ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
-/*
- gzdopen() associates a gzFile with the file descriptor fd. File
- descriptors are obtained from calls like open, dup, creat, pipe or
- fileno (in the file has been previously opened with fopen).
- The mode parameter is as in gzopen.
- The next call of gzclose on the returned gzFile will also close the
- file descriptor fd, just like fclose(fdopen(fd), mode) closes the file
- descriptor fd. If you want to keep fd open, use gzdopen(dup(fd), mode).
- gzdopen returns NULL if there was insufficient memory to allocate
- the (de)compression state.
-*/
-
-ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
-/*
- Dynamically update the compression level or strategy. See the description
- of deflateInit2 for the meaning of these parameters.
- gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not
- opened for writing.
-*/
-
-ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
-/*
- Reads the given number of uncompressed bytes from the compressed file.
- If the input file was not in gzip format, gzread copies the given number
- of bytes into the buffer.
- gzread returns the number of uncompressed bytes actually read (0 for
- end of file, -1 for error). */
-
-ZEXTERN int ZEXPORT gzwrite OF((gzFile file,
- voidpc buf, unsigned len));
-/*
- Writes the given number of uncompressed bytes into the compressed file.
- gzwrite returns the number of uncompressed bytes actually written
- (0 in case of error).
-*/
-
-ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...));
-/*
- Converts, formats, and writes the args to the compressed file under
- control of the format string, as in fprintf. gzprintf returns the number of
- uncompressed bytes actually written (0 in case of error). The number of
- uncompressed bytes written is limited to 4095. The caller should assure that
- this limit is not exceeded. If it is exceeded, then gzprintf() will return
- return an error (0) with nothing written. In this case, there may also be a
- buffer overflow with unpredictable consequences, which is possible only if
- zlib was compiled with the insecure functions sprintf() or vsprintf()
- because the secure snprintf() or vsnprintf() functions were not available.
-*/
-
-ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
-/*
- Writes the given null-terminated string to the compressed file, excluding
- the terminating null character.
- gzputs returns the number of characters written, or -1 in case of error.
-*/
-
-ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
-/*
- Reads bytes from the compressed file until len-1 characters are read, or
- a newline character is read and transferred to buf, or an end-of-file
- condition is encountered. The string is then terminated with a null
- character.
- gzgets returns buf, or Z_NULL in case of error.
-*/
-
-ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c));
-/*
- Writes c, converted to an unsigned char, into the compressed file.
- gzputc returns the value that was written, or -1 in case of error.
-*/
-
-ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
-/*
- Reads one byte from the compressed file. gzgetc returns this byte
- or -1 in case of end of file or error.
-*/
-
-ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
-/*
- Push one character back onto the stream to be read again later.
- Only one character of push-back is allowed. gzungetc() returns the
- character pushed, or -1 on failure. gzungetc() will fail if a
- character has been pushed but not read yet, or if c is -1. The pushed
- character will be discarded if the stream is repositioned with gzseek()
- or gzrewind().
-*/
-
-ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
-/*
- Flushes all pending output into the compressed file. The parameter
- flush is as in the deflate() function. The return value is the zlib
- error number (see function gzerror below). gzflush returns Z_OK if
- the flush parameter is Z_FINISH and all output could be flushed.
- gzflush should be called only when strictly necessary because it can
- degrade compression.
-*/
-
-ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
- z_off_t offset, int whence));
-/*
- Sets the starting position for the next gzread or gzwrite on the
- given compressed file. The offset represents a number of bytes in the
- uncompressed data stream. The whence parameter is defined as in lseek(2);
- the value SEEK_END is not supported.
- If the file is opened for reading, this function is emulated but can be
- extremely slow. If the file is opened for writing, only forward seeks are
- supported; gzseek then compresses a sequence of zeroes up to the new
- starting position.
-
- gzseek returns the resulting offset location as measured in bytes from
- the beginning of the uncompressed stream, or -1 in case of error, in
- particular if the file is opened for writing and the new starting position
- would be before the current position.
-*/
-
-ZEXTERN int ZEXPORT gzrewind OF((gzFile file));
-/*
- Rewinds the given file. This function is supported only for reading.
-
- gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
-*/
-
-ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file));
-/*
- Returns the starting position for the next gzread or gzwrite on the
- given compressed file. This position represents a number of bytes in the
- uncompressed data stream.
-
- gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
-*/
-
-ZEXTERN int ZEXPORT gzeof OF((gzFile file));
-/*
- Returns 1 when EOF has previously been detected reading the given
- input stream, otherwise zero.
-*/
-
-ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
-/*
- Returns 1 if file is being read directly without decompression, otherwise
- zero.
-*/
-
-ZEXTERN int ZEXPORT gzclose OF((gzFile file));
-/*
- Flushes all pending output if necessary, closes the compressed file
- and deallocates all the (de)compression state. The return value is the zlib
- error number (see function gzerror below).
-*/
-
-ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
-/*
- Returns the error message for the last error which occurred on the
- given compressed file. errnum is set to zlib error number. If an
- error occurred in the file system and not in the compression library,
- errnum is set to Z_ERRNO and the application may consult errno
- to get the exact error code.
-*/
-
-ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
-/*
- Clears the error and end-of-file flags for file. This is analogous to the
- clearerr() function in stdio. This is useful for continuing to read a gzip
- file that is being written concurrently.
-*/
-
- /* checksum functions */
-
-/*
- These functions are not related to compression but are exported
- anyway because they might be useful in applications using the
- compression library.
-*/
-
-ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
-/*
- Update a running Adler-32 checksum with the bytes buf[0..len-1] and
- return the updated checksum. If buf is NULL, this function returns
- the required initial value for the checksum.
- An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
- much faster. Usage example:
-
- uLong adler = adler32(0L, Z_NULL, 0);
-
- while (read_buffer(buffer, length) != EOF) {
- adler = adler32(adler, buffer, length);
- }
- if (adler != original_adler) error();
-*/
-
-ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
- z_off_t len2));
-/*
- Combine two Adler-32 checksums into one. For two sequences of bytes, seq1
- and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for
- each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of
- seq1 and seq2 concatenated, requiring only adler1, adler2, and len2.
-*/
-
-ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len));
-/*
- Update a running CRC-32 with the bytes buf[0..len-1] and return the
- updated CRC-32. If buf is NULL, this function returns the required initial
- value for the for the crc. Pre- and post-conditioning (one's complement) is
- performed within this function so it shouldn't be done by the application.
- Usage example:
-
- uLong crc = crc32(0L, Z_NULL, 0);
-
- while (read_buffer(buffer, length) != EOF) {
- crc = crc32(crc, buffer, length);
- }
- if (crc != original_crc) error();
-*/
-
-ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
-
-/*
- Combine two CRC-32 check values into one. For two sequences of bytes,
- seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
- calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32
- check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
- len2.
-*/
-
-
- /* various hacks, don't look :) */
-
-/* deflateInit and inflateInit are macros to allow checking the zlib version
- * and the compiler's view of z_stream:
- */
-ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
- const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
- const char *version, int stream_size));
-ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method,
- int windowBits, int memLevel,
- int strategy, const char *version,
- int stream_size));
-ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits,
- const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
- unsigned char FAR *window,
- const char *version,
- int stream_size));
-#define deflateInit(strm, level) \
- deflateInit_((strm), (level), ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit(strm) \
- inflateInit_((strm), ZLIB_VERSION, sizeof(z_stream))
-#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
- deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
- (strategy), ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit2(strm, windowBits) \
- inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream))
-#define inflateBackInit(strm, windowBits, window) \
- inflateBackInit_((strm), (windowBits), (window), \
- ZLIB_VERSION, sizeof(z_stream))
-
-
-#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL)
- struct internal_state {int dummy;}; /* hack for buggy compilers */
-#endif
-
-ZEXTERN const char * ZEXPORT zError OF((int));
-ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp z));
-ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* ZLIB_H */
diff --git a/src/._AS_ALN b/src/._AS_ALN
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_ALN differ
diff --git a/src/._AS_ARD b/src/._AS_ARD
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_ARD differ
diff --git a/src/._AS_BAT b/src/._AS_BAT
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_BAT differ
diff --git a/src/._AS_BOG b/src/._AS_BOG
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_BOG differ
diff --git a/src/._AS_CGB b/src/._AS_CGB
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_CGB differ
diff --git a/src/._AS_CGW b/src/._AS_CGW
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_CGW differ
diff --git a/src/._AS_CNS b/src/._AS_CNS
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_CNS differ
diff --git a/src/._AS_ENV b/src/._AS_ENV
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_ENV differ
diff --git a/src/._AS_GKP b/src/._AS_GKP
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_GKP differ
diff --git a/src/._AS_LIN b/src/._AS_LIN
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_LIN differ
diff --git a/src/._AS_MER b/src/._AS_MER
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_MER differ
diff --git a/src/._AS_MSG b/src/._AS_MSG
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_MSG differ
diff --git a/src/._AS_OBT b/src/._AS_OBT
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_OBT differ
diff --git a/src/._AS_OVL b/src/._AS_OVL
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_OVL differ
diff --git a/src/._AS_OVM b/src/._AS_OVM
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_OVM differ
diff --git a/src/._AS_OVS b/src/._AS_OVS
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_OVS differ
diff --git a/src/._AS_PBR b/src/._AS_PBR
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_PBR differ
diff --git a/src/._AS_PER b/src/._AS_PER
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_PER differ
diff --git a/src/._AS_PL5 b/src/._AS_PL5
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_PL5 differ
diff --git a/src/._AS_REF b/src/._AS_REF
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_REF differ
diff --git a/src/._AS_REZ b/src/._AS_REZ
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_REZ differ
diff --git a/src/._AS_RUN b/src/._AS_RUN
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_RUN differ
diff --git a/src/._AS_TER b/src/._AS_TER
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_TER differ
diff --git a/src/._AS_UID b/src/._AS_UID
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_UID differ
diff --git a/src/._AS_UTL b/src/._AS_UTL
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_UTL differ
diff --git a/src/._AS_VWR b/src/._AS_VWR
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/._AS_VWR differ
diff --git a/src/._AS_global.C b/src/._AS_global.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/._AS_global.C differ
diff --git a/src/._AS_global.H b/src/._AS_global.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/._AS_global.H differ
diff --git a/src/._Makefile b/src/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/._Makefile differ
diff --git a/src/._c_make.as b/src/._c_make.as
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/._c_make.as differ
diff --git a/src/._c_make.gen b/src/._c_make.gen
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/._c_make.gen differ
diff --git a/src/._site_name.as b/src/._site_name.as
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/._site_name.as differ
diff --git a/src/AS_ALN/._AS_ALN_aligners.H b/src/AS_ALN/._AS_ALN_aligners.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_aligners.H differ
diff --git a/src/AS_ALN/._AS_ALN_bruteforcedp.C b/src/AS_ALN/._AS_ALN_bruteforcedp.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_bruteforcedp.C differ
diff --git a/src/AS_ALN/._AS_ALN_bruteforcedp.H b/src/AS_ALN/._AS_ALN_bruteforcedp.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_bruteforcedp.H differ
diff --git a/src/AS_ALN/._AS_ALN_dpaligner.C b/src/AS_ALN/._AS_ALN_dpaligner.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_dpaligner.C differ
diff --git a/src/AS_ALN/._AS_ALN_forcns.C b/src/AS_ALN/._AS_ALN_forcns.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_forcns.C differ
diff --git a/src/AS_ALN/._AS_ALN_loverlapper.C b/src/AS_ALN/._AS_ALN_loverlapper.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_loverlapper.C differ
diff --git a/src/AS_ALN/._AS_ALN_pieceOlap.C b/src/AS_ALN/._AS_ALN_pieceOlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_pieceOlap.C differ
diff --git a/src/AS_ALN/._AS_ALN_qvaligner.C b/src/AS_ALN/._AS_ALN_qvaligner.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._AS_ALN_qvaligner.C differ
diff --git a/src/AS_ALN/._CA_ALN_local.C b/src/AS_ALN/._CA_ALN_local.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._CA_ALN_local.C differ
diff --git a/src/AS_ALN/._CA_ALN_local.H b/src/AS_ALN/._CA_ALN_local.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._CA_ALN_local.H differ
diff --git a/src/AS_ALN/._CA_ALN_overlap.C b/src/AS_ALN/._CA_ALN_overlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._CA_ALN_overlap.C differ
diff --git a/src/AS_ALN/._Makefile b/src/AS_ALN/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ALN/._Makefile differ
diff --git a/src/AS_ALN/AS_ALN_bruteforcedp.C b/src/AS_ALN/AS_ALN_bruteforcedp.C
index e714576..b1d1620 100644
--- a/src/AS_ALN/AS_ALN_bruteforcedp.C
+++ b/src/AS_ALN/AS_ALN_bruteforcedp.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_ALN_bruteforcedp.C 4483 2013-12-12 18:48:41Z brianwalenz $";
+static const char *rcsid = "$Id: AS_ALN_bruteforcedp.C 4608 2015-02-19 10:15:21Z brianwalenz $";
#include "AS_global.H"
#include "AS_ALN_bruteforcedp.H"
@@ -81,23 +81,22 @@ alignLinker(char *alignA,
int32 lenA = strlen(stringA);
int32 lenB = strlen(stringB);
+ if ((lenA > AS_READ_MAX_NORMAL_LEN) || (lenB > AS_READ_MAX_NORMAL_LEN)) {
+ fprintf(stderr, "alignLinker()-- Reads too long. %d or %d > %d\n", lenA, lenB, AS_READ_MAX_NORMAL_LEN);
+ return;
+ }
+
memset(a, 0, sizeof(alignLinker_s));
dpActions ACT;
+ uint32 *SCOspace = new uint32 [4 * AS_READ_MAX_NORMAL_LEN * 2];
- uint32 SCO[4][AS_READ_MAX_NORMAL_LEN * 2];
-
- memset(SCO, 0, sizeof(uint32) * AS_READ_MAX_NORMAL_LEN * 2);
+ memset(SCOspace, 0, sizeof(uint32) * AS_READ_MAX_NORMAL_LEN * 2);
- uint32 *lastCol = SCO[0];
- uint32 *thisCol = SCO[1];
- uint32 *iFinal = SCO[2];
- uint32 *jFinal = SCO[3];
-
- if ((lenA > AS_READ_MAX_NORMAL_LEN) || (lenB > AS_READ_MAX_NORMAL_LEN)) {
- fprintf(stderr, "alignLinker()-- Reads too long. %d or %d > %d\n", lenA, lenB, AS_READ_MAX_NORMAL_LEN);
- return;
- }
+ uint32 *lastCol = SCOspace + 0 * AS_READ_MAX_NORMAL_LEN * 2;
+ uint32 *thisCol = SCOspace + 1 * AS_READ_MAX_NORMAL_LEN * 2;
+ uint32 *iFinal = SCOspace + 2 * AS_READ_MAX_NORMAL_LEN * 2;
+ uint32 *jFinal = SCOspace + 3 * AS_READ_MAX_NORMAL_LEN * 2;
// Definition of the box we want to do dynamic programming in.
int32 ibgn = 1;
@@ -284,6 +283,13 @@ alignLinker(char *alignA,
//M[endI][endJ].score = 0;
}
+ delete [] SCOspace;
+
+ lastCol = NULL;
+ thisCol = NULL;
+ iFinal = NULL;
+ jFinal = NULL;
+
//fprintf(stderr, "FINAL curI %u curJ %u\n", curI, curJ);
int32 alignLen = 0;
diff --git a/src/AS_ALN/AS_ALN_forcns.C b/src/AS_ALN/AS_ALN_forcns.C
index 85fd960..3d1ff09 100644
--- a/src/AS_ALN/AS_ALN_forcns.C
+++ b/src/AS_ALN/AS_ALN_forcns.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char const *rcsid = "$Id: AS_ALN_forcns.C 4483 2013-12-12 18:48:41Z brianwalenz $";
+static char const *rcsid = "$Id: AS_ALN_forcns.C 4589 2015-02-02 20:23:05Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -357,14 +357,18 @@ Optimal_Overlap_AS_forCNS(char *a, char *b,
double erate, double thresh, int minlen,
CompareOptions what) {
- char h_alignA[AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2] = {0};
- char h_alignB[AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2] = {0};
- int h_trace[AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2] = {0};
+ static char h_alignA[AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2];
+ static char h_alignB[AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2];
+ static int h_trace [AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2];
static ALNoverlap o;
alignLinker_s al;
+ memset(h_alignA, 0, sizeof(char) * (AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2));
+ memset(h_alignB, 0, sizeof(char) * (AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2));
+ memset(h_trace, 0, sizeof(int) * (AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 2));
+
if (VERBOSE_MULTIALIGN_OUTPUT >= 3)
fprintf(stderr, "Optimal_Overlap_AS_forCNS()-- Begins\n");
diff --git a/src/AS_ARD/._AS_ARD_database.cc b/src/AS_ARD/._AS_ARD_database.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._AS_ARD_database.cc differ
diff --git a/src/AS_ARD/._AS_ARD_database.hh b/src/AS_ARD/._AS_ARD_database.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._AS_ARD_database.hh differ
diff --git a/src/AS_ARD/._BCPOutput.cc b/src/AS_ARD/._BCPOutput.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._BCPOutput.cc differ
diff --git a/src/AS_ARD/._BCPOutput.hh b/src/AS_ARD/._BCPOutput.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._BCPOutput.hh differ
diff --git a/src/AS_ARD/._DBSchema.sql b/src/AS_ARD/._DBSchema.sql
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._DBSchema.sql differ
diff --git a/src/AS_ARD/._DBTextOutput.cc b/src/AS_ARD/._DBTextOutput.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._DBTextOutput.cc differ
diff --git a/src/AS_ARD/._DBTextOutput.hh b/src/AS_ARD/._DBTextOutput.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._DBTextOutput.hh differ
diff --git a/src/AS_ARD/._IAssemblyDB.hh b/src/AS_ARD/._IAssemblyDB.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._IAssemblyDB.hh differ
diff --git a/src/AS_ARD/._IDBConnection.hh b/src/AS_ARD/._IDBConnection.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._IDBConnection.hh differ
diff --git a/src/AS_ARD/._IDBOutput.hh b/src/AS_ARD/._IDBOutput.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._IDBOutput.hh differ
diff --git a/src/AS_ARD/._Makefile b/src/AS_ARD/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._Makefile differ
diff --git a/src/AS_ARD/._SQLOutput.cc b/src/AS_ARD/._SQLOutput.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._SQLOutput.cc differ
diff --git a/src/AS_ARD/._SQLOutput.hh b/src/AS_ARD/._SQLOutput.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._SQLOutput.hh differ
diff --git a/src/AS_ARD/._Sybase.cc b/src/AS_ARD/._Sybase.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._Sybase.cc differ
diff --git a/src/AS_ARD/._Sybase.hh b/src/AS_ARD/._Sybase.hh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._Sybase.hh differ
diff --git a/src/AS_ARD/._getAssembly.cc b/src/AS_ARD/._getAssembly.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._getAssembly.cc differ
diff --git a/src/AS_ARD/._loadAssembly.cc b/src/AS_ARD/._loadAssembly.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ARD/._loadAssembly.cc differ
diff --git a/src/AS_BAT/._AS_BAT_BestOverlapGraph.C b/src/AS_BAT/._AS_BAT_BestOverlapGraph.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_BestOverlapGraph.C differ
diff --git a/src/AS_BAT/._AS_BAT_BestOverlapGraph.H b/src/AS_BAT/._AS_BAT_BestOverlapGraph.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_BestOverlapGraph.H differ
diff --git a/src/AS_BAT/._AS_BAT_Breaking.C b/src/AS_BAT/._AS_BAT_Breaking.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Breaking.C differ
diff --git a/src/AS_BAT/._AS_BAT_Breaking.H b/src/AS_BAT/._AS_BAT_Breaking.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Breaking.H differ
diff --git a/src/AS_BAT/._AS_BAT_ChunkGraph.C b/src/AS_BAT/._AS_BAT_ChunkGraph.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_ChunkGraph.C differ
diff --git a/src/AS_BAT/._AS_BAT_ChunkGraph.H b/src/AS_BAT/._AS_BAT_ChunkGraph.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_ChunkGraph.H differ
diff --git a/src/AS_BAT/._AS_BAT_Datatypes.H b/src/AS_BAT/._AS_BAT_Datatypes.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Datatypes.H differ
diff --git a/src/AS_BAT/._AS_BAT_EvaluateMates.C b/src/AS_BAT/._AS_BAT_EvaluateMates.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_EvaluateMates.C differ
diff --git a/src/AS_BAT/._AS_BAT_EvaluateMates.H b/src/AS_BAT/._AS_BAT_EvaluateMates.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_EvaluateMates.H differ
diff --git a/src/AS_BAT/._AS_BAT_ExtendByMates.C b/src/AS_BAT/._AS_BAT_ExtendByMates.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_ExtendByMates.C differ
diff --git a/src/AS_BAT/._AS_BAT_FragmentInfo.C b/src/AS_BAT/._AS_BAT_FragmentInfo.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_FragmentInfo.C differ
diff --git a/src/AS_BAT/._AS_BAT_InsertSizes.C b/src/AS_BAT/._AS_BAT_InsertSizes.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_InsertSizes.C differ
diff --git a/src/AS_BAT/._AS_BAT_InsertSizes.H b/src/AS_BAT/._AS_BAT_InsertSizes.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_InsertSizes.H differ
diff --git a/src/AS_BAT/._AS_BAT_Instrumentation.C b/src/AS_BAT/._AS_BAT_Instrumentation.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Instrumentation.C differ
diff --git a/src/AS_BAT/._AS_BAT_Instrumentation.H b/src/AS_BAT/._AS_BAT_Instrumentation.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Instrumentation.H differ
diff --git a/src/AS_BAT/._AS_BAT_IntersectBubble.C b/src/AS_BAT/._AS_BAT_IntersectBubble.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_IntersectBubble.C differ
diff --git a/src/AS_BAT/._AS_BAT_IntersectBubble.H b/src/AS_BAT/._AS_BAT_IntersectBubble.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_IntersectBubble.H differ
diff --git a/src/AS_BAT/._AS_BAT_IntersectSplit.C b/src/AS_BAT/._AS_BAT_IntersectSplit.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_IntersectSplit.C differ
diff --git a/src/AS_BAT/._AS_BAT_IntersectSplit.H b/src/AS_BAT/._AS_BAT_IntersectSplit.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_IntersectSplit.H differ
diff --git a/src/AS_BAT/._AS_BAT_Joining.C b/src/AS_BAT/._AS_BAT_Joining.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Joining.C differ
diff --git a/src/AS_BAT/._AS_BAT_Joining.H b/src/AS_BAT/._AS_BAT_Joining.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Joining.H differ
diff --git a/src/AS_BAT/._AS_BAT_Logging.C b/src/AS_BAT/._AS_BAT_Logging.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Logging.C differ
diff --git a/src/AS_BAT/._AS_BAT_Logging.H b/src/AS_BAT/._AS_BAT_Logging.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Logging.H differ
diff --git a/src/AS_BAT/._AS_BAT_MateBubble.C b/src/AS_BAT/._AS_BAT_MateBubble.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MateBubble.C differ
diff --git a/src/AS_BAT/._AS_BAT_MateChecker.C b/src/AS_BAT/._AS_BAT_MateChecker.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MateChecker.C differ
diff --git a/src/AS_BAT/._AS_BAT_MateChecker.H b/src/AS_BAT/._AS_BAT_MateChecker.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MateChecker.H differ
diff --git a/src/AS_BAT/._AS_BAT_MateLocation.C b/src/AS_BAT/._AS_BAT_MateLocation.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MateLocation.C differ
diff --git a/src/AS_BAT/._AS_BAT_MateLocation.H b/src/AS_BAT/._AS_BAT_MateLocation.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MateLocation.H differ
diff --git a/src/AS_BAT/._AS_BAT_MergeSplitJoin.C b/src/AS_BAT/._AS_BAT_MergeSplitJoin.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MergeSplitJoin.C differ
diff --git a/src/AS_BAT/._AS_BAT_MergeSplitJoin.H b/src/AS_BAT/._AS_BAT_MergeSplitJoin.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MergeSplitJoin.H differ
diff --git a/src/AS_BAT/._AS_BAT_MoveContains.C b/src/AS_BAT/._AS_BAT_MoveContains.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MoveContains.C differ
diff --git a/src/AS_BAT/._AS_BAT_MoveContains.H b/src/AS_BAT/._AS_BAT_MoveContains.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_MoveContains.H differ
diff --git a/src/AS_BAT/._AS_BAT_Outputs.C b/src/AS_BAT/._AS_BAT_Outputs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Outputs.C differ
diff --git a/src/AS_BAT/._AS_BAT_Outputs.H b/src/AS_BAT/._AS_BAT_Outputs.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Outputs.H differ
diff --git a/src/AS_BAT/._AS_BAT_OverlapCache.C b/src/AS_BAT/._AS_BAT_OverlapCache.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_OverlapCache.C differ
diff --git a/src/AS_BAT/._AS_BAT_OverlapCache.H b/src/AS_BAT/._AS_BAT_OverlapCache.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_OverlapCache.H differ
diff --git a/src/AS_BAT/._AS_BAT_PlaceContains.C b/src/AS_BAT/._AS_BAT_PlaceContains.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PlaceContains.C differ
diff --git a/src/AS_BAT/._AS_BAT_PlaceContains.H b/src/AS_BAT/._AS_BAT_PlaceContains.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PlaceContains.H differ
diff --git a/src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.C b/src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.C differ
diff --git a/src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.H b/src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PlaceFragUsingOverlaps.H differ
diff --git a/src/AS_BAT/._AS_BAT_PlaceZombies.C b/src/AS_BAT/._AS_BAT_PlaceZombies.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PlaceZombies.C differ
diff --git a/src/AS_BAT/._AS_BAT_PlaceZombies.H b/src/AS_BAT/._AS_BAT_PlaceZombies.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PlaceZombies.H differ
diff --git a/src/AS_BAT/._AS_BAT_PopulateUnitig.C b/src/AS_BAT/._AS_BAT_PopulateUnitig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PopulateUnitig.C differ
diff --git a/src/AS_BAT/._AS_BAT_PopulateUnitig.H b/src/AS_BAT/._AS_BAT_PopulateUnitig.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PopulateUnitig.H differ
diff --git a/src/AS_BAT/._AS_BAT_PromoteToSingleton.C b/src/AS_BAT/._AS_BAT_PromoteToSingleton.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_PromoteToSingleton.C differ
diff --git a/src/AS_BAT/._AS_BAT_ReconstructRepeats.C b/src/AS_BAT/._AS_BAT_ReconstructRepeats.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_ReconstructRepeats.C differ
diff --git a/src/AS_BAT/._AS_BAT_RepeatJunctionEvidence.H b/src/AS_BAT/._AS_BAT_RepeatJunctionEvidence.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_RepeatJunctionEvidence.H differ
diff --git a/src/AS_BAT/._AS_BAT_SetParentAndHang.C b/src/AS_BAT/._AS_BAT_SetParentAndHang.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_SetParentAndHang.C differ
diff --git a/src/AS_BAT/._AS_BAT_SetParentAndHang.H b/src/AS_BAT/._AS_BAT_SetParentAndHang.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_SetParentAndHang.H differ
diff --git a/src/AS_BAT/._AS_BAT_SplitDiscontinuous.C b/src/AS_BAT/._AS_BAT_SplitDiscontinuous.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_SplitDiscontinuous.C differ
diff --git a/src/AS_BAT/._AS_BAT_SplitDiscontinuous.H b/src/AS_BAT/._AS_BAT_SplitDiscontinuous.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_SplitDiscontinuous.H differ
diff --git a/src/AS_BAT/._AS_BAT_Unitig.C b/src/AS_BAT/._AS_BAT_Unitig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Unitig.C differ
diff --git a/src/AS_BAT/._AS_BAT_Unitig.H b/src/AS_BAT/._AS_BAT_Unitig.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Unitig.H differ
diff --git a/src/AS_BAT/._AS_BAT_Unitig_AddAndPlaceFrag.C b/src/AS_BAT/._AS_BAT_Unitig_AddAndPlaceFrag.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Unitig_AddAndPlaceFrag.C differ
diff --git a/src/AS_BAT/._AS_BAT_Unitig_AddFrag.C b/src/AS_BAT/._AS_BAT_Unitig_AddFrag.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Unitig_AddFrag.C differ
diff --git a/src/AS_BAT/._AS_BAT_Unitig_PlaceFragUsingEdges.C b/src/AS_BAT/._AS_BAT_Unitig_PlaceFragUsingEdges.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_Unitig_PlaceFragUsingEdges.C differ
diff --git a/src/AS_BAT/._AS_BAT_bogusUtil.C b/src/AS_BAT/._AS_BAT_bogusUtil.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_bogusUtil.C differ
diff --git a/src/AS_BAT/._AS_BAT_bogusUtil.H b/src/AS_BAT/._AS_BAT_bogusUtil.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_bogusUtil.H differ
diff --git a/src/AS_BAT/._AS_BAT_findEdges.C b/src/AS_BAT/._AS_BAT_findEdges.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._AS_BAT_findEdges.C differ
diff --git a/src/AS_BAT/._Makefile b/src/AS_BAT/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._Makefile differ
diff --git a/src/AS_BAT/._analyze-mapped-unitigs-for-joins.pl b/src/AS_BAT/._analyze-mapped-unitigs-for-joins.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._analyze-mapped-unitigs-for-joins.pl differ
diff --git a/src/AS_BAT/._analyze-nucmer-gaps.pl b/src/AS_BAT/._analyze-nucmer-gaps.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._analyze-nucmer-gaps.pl differ
diff --git a/src/AS_BAT/._bogart-build-fasta.pl b/src/AS_BAT/._bogart-build-fasta.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogart-build-fasta.pl differ
diff --git a/src/AS_BAT/._bogart.C b/src/AS_BAT/._bogart.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogart.C differ
diff --git a/src/AS_BAT/._bogus-genome.css b/src/AS_BAT/._bogus-genome.css
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogus-genome.css differ
diff --git a/src/AS_BAT/._bogus-genome.json b/src/AS_BAT/._bogus-genome.json
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogus-genome.json differ
diff --git a/src/AS_BAT/._bogus-run.sh b/src/AS_BAT/._bogus-run.sh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogus-run.sh differ
diff --git a/src/AS_BAT/._bogus.C b/src/AS_BAT/._bogus.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogus.C differ
diff --git a/src/AS_BAT/._bogus.json.README b/src/AS_BAT/._bogus.json.README
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogus.json.README differ
diff --git a/src/AS_BAT/._bogusness-run.pl b/src/AS_BAT/._bogusness-run.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogusness-run.pl differ
diff --git a/src/AS_BAT/._bogusness.C b/src/AS_BAT/._bogusness.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._bogusness.C differ
diff --git a/src/AS_BAT/._classifyMates-BFS.C b/src/AS_BAT/._classifyMates-BFS.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-BFS.C differ
diff --git a/src/AS_BAT/._classifyMates-DFS.C b/src/AS_BAT/._classifyMates-DFS.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-DFS.C differ
diff --git a/src/AS_BAT/._classifyMates-RFS.C b/src/AS_BAT/._classifyMates-RFS.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-RFS.C differ
diff --git a/src/AS_BAT/._classifyMates-Suspicious.C b/src/AS_BAT/._classifyMates-Suspicious.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-Suspicious.C differ
diff --git a/src/AS_BAT/._classifyMates-globalData.C b/src/AS_BAT/._classifyMates-globalData.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-globalData.C differ
diff --git a/src/AS_BAT/._classifyMates-globalData.H b/src/AS_BAT/._classifyMates-globalData.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-globalData.H differ
diff --git a/src/AS_BAT/._classifyMates-runningTime.H b/src/AS_BAT/._classifyMates-runningTime.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-runningTime.H differ
diff --git a/src/AS_BAT/._classifyMates-saveDistance.H b/src/AS_BAT/._classifyMates-saveDistance.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates-saveDistance.H differ
diff --git a/src/AS_BAT/._classifyMates.C b/src/AS_BAT/._classifyMates.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates.C differ
diff --git a/src/AS_BAT/._classifyMates.H b/src/AS_BAT/._classifyMates.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMates.H differ
diff --git a/src/AS_BAT/._classifyMatesApply.C b/src/AS_BAT/._classifyMatesApply.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMatesApply.C differ
diff --git a/src/AS_BAT/._classifyMatesPairwise.C b/src/AS_BAT/._classifyMatesPairwise.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._classifyMatesPairwise.C differ
diff --git a/src/AS_BAT/._computeCoverageStat.C b/src/AS_BAT/._computeCoverageStat.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._computeCoverageStat.C differ
diff --git a/src/AS_BAT/._count-bubbles.pl b/src/AS_BAT/._count-bubbles.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._count-bubbles.pl differ
diff --git a/src/AS_BAT/._erate-estimate-plot-per-base-estimate.pl b/src/AS_BAT/._erate-estimate-plot-per-base-estimate.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._erate-estimate-plot-per-base-estimate.pl differ
diff --git a/src/AS_BAT/._erate-estimate-test-based-on-mapping.pl b/src/AS_BAT/._erate-estimate-test-based-on-mapping.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._erate-estimate-test-based-on-mapping.pl differ
diff --git a/src/AS_BAT/._erate-estimate.C b/src/AS_BAT/._erate-estimate.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._erate-estimate.C differ
diff --git a/src/AS_BAT/._examine-mapping-ideal.pl b/src/AS_BAT/._examine-mapping-ideal.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._examine-mapping-ideal.pl differ
diff --git a/src/AS_BAT/._examine-mapping.pl b/src/AS_BAT/._examine-mapping.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._examine-mapping.pl differ
diff --git a/src/AS_BAT/._locate-read-in-unitig-based-on-olaps.pl b/src/AS_BAT/._locate-read-in-unitig-based-on-olaps.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._locate-read-in-unitig-based-on-olaps.pl differ
diff --git a/src/AS_BAT/._markRepeatUnique.C b/src/AS_BAT/._markRepeatUnique.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._markRepeatUnique.C differ
diff --git a/src/AS_BAT/._memoryMappedFile.H b/src/AS_BAT/._memoryMappedFile.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._memoryMappedFile.H differ
diff --git a/src/AS_BAT/._petey.C b/src/AS_BAT/._petey.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._petey.C differ
diff --git a/src/AS_BAT/._plot-break-points.pl b/src/AS_BAT/._plot-break-points.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._plot-break-points.pl differ
diff --git a/src/AS_BAT/._rewriteCache.C b/src/AS_BAT/._rewriteCache.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._rewriteCache.C differ
diff --git a/src/AS_BAT/._show-false-best-edges-from-mapping.pl b/src/AS_BAT/._show-false-best-edges-from-mapping.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._show-false-best-edges-from-mapping.pl differ
diff --git a/src/AS_BAT/._splitUnitigs.C b/src/AS_BAT/._splitUnitigs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BAT/._splitUnitigs.C differ
diff --git a/src/AS_BAT/AS_BAT_BestOverlapGraph.C b/src/AS_BAT/AS_BAT_BestOverlapGraph.C
index d4a4aa2..2bd59e0 100644
--- a/src/AS_BAT/AS_BAT_BestOverlapGraph.C
+++ b/src/AS_BAT/AS_BAT_BestOverlapGraph.C
@@ -19,448 +19,571 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_BestOverlapGraph.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_BestOverlapGraph.C 4642 2015-04-21 00:15:15Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_BestOverlapGraph.H"
#include "AS_BAT_Unitig.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
// HACK
uint32 examineOnly = UINT32_MAX;
-BestOverlapGraph::BestOverlapGraph(double AS_UTG_ERROR_RATE,
- double AS_UTG_ERROR_LIMIT,
- const char *prefix,
- bool removeSuspicious,
- bool removeSpurs) {
+void
+BestOverlapGraph::removeSuspicious(void) {
+ uint32 fiLimit = FI->numFragments();
+ uint32 numThreads = omp_get_max_threads();
+ uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- bool examineOnlyTopN = false;
- bool removeFalseBest = false;
+ writeLog("BestOverlapGraph()-- removing suspicious reads from graph, with %d threads.\n", numThreads);
- setLogFile(prefix, "bestOverlapGraph");
+#pragma omp parallel for schedule(dynamic, blockSize)
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 no = 0;
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
- writeLog("BestOverlapGraph-- allocating best edges ("F_SIZE_T"MB) and containments ("F_SIZE_T"MB)\n",
- ((2 * sizeof(BestEdgeOverlap) * (FI->numFragments() + 1)) >> 20),
- ((1 * sizeof(BestContainment) * (FI->numFragments() + 1)) >> 20));
+ bool verified = false;
+ intervalList<int32> IL;
- assert(AS_UTG_ERROR_RATE >= 0.0);
- assert(AS_UTG_ERROR_RATE <= AS_MAX_ERROR_RATE);
+ uint32 fLen = FI->fragmentLength(fi);
- assert(AS_CNS_ERROR_RATE >= 0.0);
- assert(AS_CNS_ERROR_RATE <= AS_MAX_ERROR_RATE);
+ for (uint32 ii=0; (ii<no) && (verified == false); ii++) {
+ if (isOverlapBadQuality(ovl[ii]))
+ // Yuck. Don't want to use this crud.
+ continue;
- _bestA = new BestOverlaps [FI->numFragments() + 1];
- _scorA = new BestScores [FI->numFragments() + 1];
+ if ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang <= 0))
+ // Left side dovetail
+ IL.add(0, fLen + ovl[ii].b_hang);
- memset(_bestA, 0, sizeof(BestOverlaps) * (FI->numFragments() + 1));
- memset(_scorA, 0, sizeof(BestScores) * (FI->numFragments() + 1));
+ else if ((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang >= 0))
+ // Right side dovetail
+ IL.add(ovl[ii].a_hang, fLen - ovl[ii].a_hang);
- _restrict = NULL;
+ else if ((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang <= 0))
+ // I contain the other
+ IL.add(ovl[ii].a_hang, fLen - ovl[ii].a_hang - ovl[ii].b_hang);
- mismatchCutoff = AS_UTG_ERROR_RATE;
- mismatchLimit = AS_UTG_ERROR_LIMIT;
+ else if ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang >= 0))
+ // I am contained and thus now perfectly good!
+ verified = true;
- // Initialize parallelism.
+ else
+ // Huh? Coding error.
+ assert(0);
+ }
+
+ if (verified == false) {
+ IL.merge();
+ verified = (IL.numberOfIntervals() == 1);
+ }
+ if (verified == false) {
+ if (no > 0)
+ writeLog("BestOverlapGraph()-- frag "F_U32" is suspicious ("F_U32" overlaps).\n", fi, no);
+
+#pragma omp critical (suspInsert)
+ _suspicious.insert(fi);
+ }
+ }
+}
+
+
+void
+BestOverlapGraph::examineOnlyTopN(void) {
uint32 fiLimit = FI->numFragments();
uint32 numThreads = omp_get_max_threads();
uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- // PASS 0: Find suspicious fragments. For any found, mark as suspicious and don't allow
- // these to be best overlaps.
-
- if (removeSuspicious) {
- writeLog("BestOverlapGraph()-- removing suspicious reads from graph, with %d threads.\n", numThreads);
+ writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
+ writeLog("BestOverlapGraph()-- scoring highest quality %d overlaps.\n", examineOnly);
#pragma omp parallel for schedule(dynamic, blockSize)
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 no = 0;
- BAToverlap *ovl = OC->getOverlaps(fi, no);
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 no = 0;
+ uint32 n5 = 0;
+ uint32 n3 = 0;
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
- bool verified = false;
- intervalList IL;
+ sort(ovl, ovl + no, BAToverlap_sortByErate);
- uint32 fLen = FI->fragmentLength(fi);
+ for (uint32 ii=0; ii<no; ii++) {
+ if (((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang <= 0)) ||
+ ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang >= 0)))
+ // Don't do contains here!
+ continue;
- for (uint32 ii=0; (ii<no) && (verified == false); ii++) {
- if (isOverlapBadQuality(ovl[ii]))
- // Yuck. Don't want to use this crud.
- continue;
+ // Process the 5' overlaps.
+ if ((n5 < examineOnly) &&
+ (ovl[ii].a_hang < 0)) {
+ assert(ovl[ii].b_hang < 0);
+ n5++;
+ scoreEdge(ovl[ii]);
+ }
- if ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang <= 0))
- // Left side dovetail
- IL.add(0, fLen + ovl[ii].b_hang);
+ // Process the 3' overlaps.
+ if ((n3 < examineOnly) &&
+ (ovl[ii].a_hang > 0)) {
+ assert(ovl[ii].b_hang > 0);
+ n3++;
+ scoreEdge(ovl[ii]);
+ }
+ }
+ }
+}
- else if ((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang >= 0))
- // Right side dovetail
- IL.add(ovl[ii].a_hang, fLen - ovl[ii].a_hang);
- else if ((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang <= 0))
- // I contain the other
- IL.add(ovl[ii].a_hang, fLen - ovl[ii].a_hang - ovl[ii].b_hang);
+void
+BestOverlapGraph::removeSpurs(void) {
+ uint32 fiLimit = FI->numFragments();
+ uint32 numThreads = omp_get_max_threads();
+ uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- else if ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang >= 0))
- // I am contained and thus now perfectly good!
- verified = true;
+ writeLog("BestOverlapGraph()-- detecting spur fragments.\n");
- else
- // Huh? Coding error.
- assert(0);
- }
+ char *isSpur = new char [fiLimit + 1];
- if (verified == false) {
- IL.merge();
- verified = (IL.numberOfIntervals() == 1);
- }
+ memset(isSpur, 0, sizeof(char) * (fiLimit + 1));
- if (verified == false) {
- if (no > 0)
- writeLog("BestOverlapGraph()-- frag "F_U32" is suspicious ("F_U32" overlaps).\n", fi, no);
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ bool spur5 = (getBestEdgeOverlap(fi, false)->fragId() == 0);
+ bool spur3 = (getBestEdgeOverlap(fi, true)->fragId() == 0);
-#pragma omp critical (suspInsert)
- _suspicious.insert(fi);
- }
+ if ((spur5 == true) || (spur3 == true)) {
+ if ((spur5 == false) || (spur3 == false))
+ writeLog("BestOverlapGraph()-- frag "F_U32" is a %s spur.\n", fi, (spur5) ? "5'" : "3'");
+ isSpur[fi] = true;
}
}
- // PASS 1: Find containments.
+ // Remove best edges, so we can rebuild
- writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads);
+ memset(_bestA, 0, sizeof(BestOverlaps) * (fiLimit + 1));
+ memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1));
- if (false && AS_UTL_fileExists("best.contains", false, false)) {
- writeLog("BestOverlapGraph()-- loading best containes from cache.\n");
- assert(0); // Not quite done.
+ // Rebuild best edges, ignoring edges to spurs. We build edges out of spurs, but don't allow edges into them.
+ // This should prevent them from being incorporated into a promiscuous unitig, but still let them be popped
+ // as bubbles (but they shouldn't because they're spurs).
+
+ // PASS 3: Find containments.
+
+ writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads);
- } else {
#pragma omp parallel for schedule(dynamic, blockSize)
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 no = 0;
- BAToverlap *ovl = OC->getOverlaps(fi, no);
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 no = 0;
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
- for (uint32 ii=0; ii<no; ii++)
+ for (uint32 ii=0; ii<no; ii++)
+ if (isSpur[ovl[ii].b_iid] == false)
scoreContainment(ovl[ii]);
- }
}
- // PASS 2: Find dovetails.
+ // PASS 4: Find dovetails.
- if (false && AS_UTL_fileExists("best.edges", false, false)) {
- writeLog("BestOverlapGraph()-- loading best edges from cache.\n");
- assert(0); // Not quite done.
-
- } else {
- writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
+ writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
#pragma omp parallel for schedule(dynamic, blockSize)
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 no = 0;
- BAToverlap *ovl = OC->getOverlaps(fi, no);
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 no = 0;
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
- for (uint32 ii=0; ii<no; ii++)
+ for (uint32 ii=0; ii<no; ii++)
+ if (isSpur[ovl[ii].b_iid] == false)
scoreEdge(ovl[ii]);
- }
}
- // Find dovetails again, but only look at the highest quality overlaps for each side.
+ delete [] isSpur;
+}
- if (examineOnlyTopN) {
- writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
- writeLog("BestOverlapGraph()-- scoring highest quality %d overlaps.\n", examineOnly);
-#pragma omp parallel for schedule(dynamic, blockSize)
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 no = 0;
- uint32 n5 = 0;
- uint32 n3 = 0;
- BAToverlap *ovl = OC->getOverlaps(fi, no);
+void
+BestOverlapGraph::removeFalseBest(void) {
+ uint32 fiLimit = FI->numFragments();
+ uint32 numThreads = omp_get_max_threads();
+ uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- sort(ovl, ovl + no, BAToverlap_sortByErate);
-
- for (uint32 ii=0; ii<no; ii++) {
- if (((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang <= 0)) ||
- ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang >= 0)))
- // Don't do contains here!
- continue;
-
- // Process the 5' overlaps.
- if ((n5 < examineOnly) &&
- (ovl[ii].a_hang < 0)) {
- assert(ovl[ii].b_hang < 0);
- n5++;
- scoreEdge(ovl[ii]);
- }
-
- // Process the 3' overlaps.
- if ((n3 < examineOnly) &&
- (ovl[ii].a_hang > 0)) {
- assert(ovl[ii].b_hang > 0);
- n3++;
- scoreEdge(ovl[ii]);
- }
+ writeLog("BestOverlapGraph()-- detecting false best overlaps.\n");
+
+ uint32 *histo5 = new uint32 [AS_BAT_MAX_ERATE + 1];
+ uint32 *histo3 = new uint32 [AS_BAT_MAX_ERATE + 1];
+
+ memset(histo5, 0, sizeof(uint32) * (AS_BAT_MAX_ERATE + 1));
+ memset(histo3, 0, sizeof(uint32) * (AS_BAT_MAX_ERATE + 1));
+
+ uint32 *erate5 = new uint32 [fiLimit + 1];
+ uint32 *erate3 = new uint32 [fiLimit + 1];
+
+ memset(erate5, 0, sizeof(uint32) * (fiLimit + 1));
+ memset(erate3, 0, sizeof(uint32) * (fiLimit + 1));
+
+ char *altBest = new char [fiLimit + 1];
+ char *isBad = new char [fiLimit + 1];
+
+ memset(altBest, 0, sizeof(char) * (fiLimit + 1));
+ memset(isBad, 0, sizeof(char) * (fiLimit + 1));
+
+ // Compute a histogram of the current best edges, and save the erate of the best for each read.
+
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 olapsLen = 0;
+ BAToverlap *olaps = OC->getOverlaps(fi, olapsLen);
+
+ BestEdgeOverlap *ovl5 = getBestEdgeOverlap(fi, false);
+ BestEdgeOverlap *ovl3 = getBestEdgeOverlap(fi, true);
+
+ for (uint32 oo=0; oo<olapsLen; oo++) {
+ assert(fi == olaps[oo].a_iid);
+
+ if (ovl5->fragId() == olaps[oo].b_iid) {
+ histo5[olaps[oo].errorRaw]++;
+ erate5[fi] = olaps[oo].error * 100;
+ }
+
+ if (ovl3->fragId() == olaps[oo].b_iid) {
+ histo3[olaps[oo].errorRaw]++;
+ erate3[fi] = olaps[oo].error * 100;
}
}
}
- // Remove spurs
+ // Compute a nice threshold. Find the mean and stddev of the best edge error rates.
- if (removeSpurs) {
- writeLog("BestOverlapGraph()-- detecting spur fragments.\n");
+ double m5 = 0, s5 = 100;
+ double m3 = 0, s3 = 100;
- char *isSpur = new char [fiLimit + 1];
+ for (uint32 xx=0; xx<10; xx++) {
+ double mean5 = 0, mean3 = 0;
+ uint64 count5 = 0, count3 = 0;
+ double stddev5 = 100, stddev3 = 100;
- memset(isSpur, 0, sizeof(char) * (fiLimit + 1));
+ for (AS_IID er=0; er <= AS_BAT_MAX_ERATE; er++) {
+ double ER = OC->decodeError(er) * 100;
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- bool spur5 = (getBestEdgeOverlap(fi, false)->fragId() == 0);
- bool spur3 = (getBestEdgeOverlap(fi, true)->fragId() == 0);
+ if (ER <= 0.0)
+ continue;
+
+ if ((m5 - 3 * s5 <= ER) &
+ (ER <= m5 + 3 * s5)) {
+ mean5 += histo5[er] * ER;
+ count5 += histo5[er];
+ }
- if ((spur5 == true) || (spur3 == true)) {
- if ((spur5 == false) || (spur3 == false))
- writeLog("BestOverlapGraph()-- frag "F_U32" is a %s spur.\n", fi, (spur5) ? "5'" : "3'");
- isSpur[fi] = true;
+ if ((m3 - 3 * s3 <= ER) &
+ (ER <= m3 + 3 * s3)) {
+ mean3 += histo3[er] * ER;
+ count3 += histo3[er];
}
}
- // Remove best edges, so we can rebuild
+ mean5 /= count5;
+ mean3 /= count3;
- memset(_bestA, 0, sizeof(BestOverlaps) * (fiLimit + 1));
- memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1));
+ for (AS_IID er=0; er <= AS_BAT_MAX_ERATE; er++) {
+ double ER = OC->decodeError(er) * 100;
- // Rebuild best edges, ignoring edges to spurs. We build edges out of spurs, but don't allow edges into them.
- // This should prevent them from being incorporated into a promiscuous unitig, but still let them be popped
- // as bubbles (but they shouldn't because they're spurs).
+ if (ER <= 0.0)
+ continue;
- // PASS 3: Find containments.
+ if ((m5 - 3 * s5 <= ER) &
+ (ER <= m5 + 3 * s5)) {
+ stddev5 += histo5[er] * (ER - mean5) * (ER - mean5);
+ }
- writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads);
+ if ((m3 - 3 * s3 <= ER) &
+ (ER <= m3 + 3 * s3)) {
+ stddev3 += histo3[er] * (ER - mean3) * (ER - mean3);
+ }
+ }
-#pragma omp parallel for schedule(dynamic, blockSize)
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 no = 0;
- BAToverlap *ovl = OC->getOverlaps(fi, no);
+ stddev5 = sqrt(stddev5 / (count5 - 1));
+ stddev3 = sqrt(stddev3 / (count3 - 1));
- for (uint32 ii=0; ii<no; ii++)
- scoreContainment(ovl[ii]);
+ m5 = mean5; m3 = mean3;
+ s5 = stddev5; s3 = stddev3;
+
+ fprintf(stderr, "mean %.4f +- %.4f --- %.4f +- %.4f ",
+ mean5, stddev5,
+ mean3, stddev3);
+
+ fprintf(stderr, "set xtics ( %.4f, %.4f, %.4f, %.4f)\n",
+ mean5 - 3 * stddev5, mean5 + 3 * stddev5,
+ mean3 - 3 * stddev3, mean3 + 3 * stddev3);
+ } // xx 10 times to stabilize
+
+
+ // Output the erate histogram
+
+ {
+ char EN[FILENAME_MAX];
+
+ sprintf(EN, "best.edges.erate.histogram");
+
+ errno = 0;
+ FILE *EH = fopen(EN, "w");
+ if (errno)
+ fprintf(stderr, "BestOverlapGraph()-- failed to open '%s' for writing: %s\n", EN, strerror(errno)), exit(1);
+
+ for (AS_IID er=0; er <= AS_BAT_MAX_ERATE; er++) {
+ double ER = OC->decodeError(er) * 100;
+
+ if (ER <= 0.0)
+ continue;
+
+ fprintf(EH, "%.4f\t%u\t%u\t%f\t%f\n",
+ ER,
+ histo5[er],
+ histo3[er],
+ fabs(m5 - ER) / s5, // Grubb's test
+ fabs(m3 - ER) / s3);
}
- // PASS 4: Find dovetails.
+ fclose(EH);
+ }
- writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
+ // For any read with best edge above THRESHOLD error, see if there is an alternate best
+ // that is within the target error range.
-#pragma omp parallel for schedule(dynamic, blockSize)
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 no = 0;
- BAToverlap *ovl = OC->getOverlaps(fi, no);
+ double erate5thresh = m5 + 2 * s5; // Discard best if it is worse than 2 s.d. from mean.
+ double erate3thresh = m3 + 2 * s3;
- for (uint32 ii=0; ii<no; ii++)
- if (isSpur[ovl[ii].b_iid] == false)
- scoreEdge(ovl[ii]);
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+
+ if (erate5[fi] > erate5thresh) {
+ fprintf(stderr, "RECOMPUTE frag %u 5'\n", fi);
+ isBad[fi] = true;
}
- delete [] isSpur;
+ if (erate3[fi] > erate3thresh) {
+ fprintf(stderr, "RECOMPUTE frag %u 3'\n", fi);
+ isBad[fi] = true;
+ }
}
- // Remove probably false best overlaps based on error rate
+ fprintf(stderr, "thresholds %f %f\n", erate5thresh, erate3thresh);
- if (removeFalseBest) {
- writeLog("BestOverlapGraph()-- detecting false best overlaps.\n");
+ double erateCthresh = MIN(erate5thresh, erate3thresh);
- uint32 *histo5 = new uint32 [AS_BAT_MAX_ERATE + 1];
- uint32 *histo3 = new uint32 [AS_BAT_MAX_ERATE + 1];
+#if 0
+ // Remove best edges, so we can rebuild
- memset(histo5, 0, sizeof(uint32) * (AS_BAT_MAX_ERATE + 1));
- memset(histo3, 0, sizeof(uint32) * (AS_BAT_MAX_ERATE + 1));
+ memset(_bestA, 0, sizeof(BestOverlaps) * (fiLimit + 1));
+ memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1));
- uint32 *erate5 = new uint32 [fiLimit + 1];
- uint32 *erate3 = new uint32 [fiLimit + 1];
+ // Rebuild best edges, ignoring edges to spurs. We build edges out of spurs, but don't allow edges into them.
+ // This should prevent them from being incorporated into a promiscuous unitig, but still let them be popped
+ // as bubbles (but they shouldn't because they're spurs).
- memset(erate5, 0, sizeof(uint32) * (fiLimit + 1));
- memset(erate3, 0, sizeof(uint32) * (fiLimit + 1));
+ // PASS 3: Find containments.
- char *altBest = new char [fiLimit + 1];
- char *isBad = new char [fiLimit + 1];
+ writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads);
- memset(altBest, 0, sizeof(char) * (fiLimit + 1));
- memset(isBad, 0, sizeof(char) * (fiLimit + 1));
+#pragma omp parallel for schedule(dynamic, blockSize)
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 no = 0;
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
- uint32 olapsLen = 0;
- BAToverlap *olaps = OC->getOverlaps(fi, olapsLen);
+ for (uint32 ii=0; ii<no; ii++)
+ assert(ovl[ii].a_iid == fi);
- BestEdgeOverlap *ovl5 = getBestEdgeOverlap(fi, false);
- BestEdgeOverlap *ovl3 = getBestEdgeOverlap(fi, true);
+ for (uint32 ii=0; ii<no; ii++)
+ if (ovl[ii].error < erateCthresh)
+ scoreContainment(ovl[ii]);
+ }
- if (fi == 3359) {
- fprintf(stderr, "3359 best %u %u\n", ovl5->fragId(), ovl3->fragId());
- }
+ // PASS 4: Find dovetails.
- for (uint32 oo=0; oo<olapsLen; oo++) {
- assert(fi == olaps[oo].a_iid);
+ writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
- if (ovl5->fragId() == olaps[oo].b_iid) {
- histo5[olaps[oo].errorRaw]++;
- erate5[fi] = olaps[oo].error * 100;
- }
+#pragma omp parallel for schedule(dynamic, blockSize)
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 no = 0;
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
- if (ovl3->fragId() == olaps[oo].b_iid) {
- histo3[olaps[oo].errorRaw]++;
- erate3[fi] = olaps[oo].error * 100;
- }
- }
- }
+ if (isBad[ovl[fi].a_iid] == true)
+ continue;
- // Compute a nice threshold.
+ for (uint32 ii=0; ii<no; ii++)
+ assert(ovl[ii].a_iid == fi);
- double m5 = 0, s5 = 100;
- double m3 = 0, s3 = 100;
+ for (uint32 ii=0; ii<no; ii++)
+ if (isBad[ovl[ii].b_iid] == false)
+ scoreEdge(ovl[ii]);
+ }
+#endif
- for (uint32 xx=0; xx<10; xx++) {
- double mean5 = 0, mean3 = 0;
- uint64 count5 = 0, count3 = 0;
- double stddev5 = 100, stddev3 = 100;
+ delete [] histo5;
+ delete [] histo3;
- for (AS_IID er=0; er <= AS_BAT_MAX_ERATE; er++) {
- double ER = OC->decodeError(er) * 100;
+ delete [] erate5;
+ delete [] erate3;
+}
- if (ER <= 0.0)
- continue;
- if ((m5 - 3 * s5 <= ER) &
- (ER <= m5 + 3 * s5)) {
- mean5 += histo5[er] * ER;
- count5 += histo5[er];
- }
- if ((m3 - 3 * s3 <= ER) &
- (ER <= m3 + 3 * s3)) {
- mean3 += histo3[er] * ER;
- count3 += histo3[er];
- }
- }
- mean5 /= count5;
- mean3 /= count3;
- for (AS_IID er=0; er <= AS_BAT_MAX_ERATE; er++) {
- double ER = OC->decodeError(er) * 100;
- if (ER <= 0.0)
- continue;
- if ((m5 - 3 * s5 <= ER) &
- (ER <= m5 + 3 * s5)) {
- stddev5 += histo5[er] * (ER - mean5) * (ER - mean5);
- }
- if ((m3 - 3 * s3 <= ER) &
- (ER <= m3 + 3 * s3)) {
- stddev3 += histo3[er] * (ER - mean3) * (ER - mean3);
- }
- }
- stddev5 = sqrt(stddev5 / (count5 - 1));
- stddev3 = sqrt(stddev3 / (count3 - 1));
- m5 = mean5; m3 = mean3;
- s5 = stddev5; s3 = stddev3;
+void
+BestOverlapGraph::removeWeak(double threshold) {
+ uint32 fiLimit = FI->numFragments();
+ uint32 numThreads = omp_get_max_threads();
+ uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- fprintf(stderr, "mean %.4f +- %.4f --- %.4f +- %.4f ",
- mean5, stddev5,
- mean3, stddev3);
+ writeLog("BestOverlapGraph()-- detecting weak overlaps.\n");
- fprintf(stderr, "set xtics ( %.4f, %.4f, %.4f, %.4f)\n",
- mean5 - 3 * stddev5, mean5 + 3 * stddev5,
- mean3 - 3 * stddev3, mean3 + 3 * stddev3);
+ // For each read, mark an overlap as bad if it falls in the lower
+ // X% of overlaps sorted by identity.
- } // xx 10 times to stabilize
+ uint32 *minErate5p = new uint32 [fiLimit + 1];
+ uint32 *minErate3p = new uint32 [fiLimit + 1];
+ memset(minErate5p, 0, sizeof(uint32) * (fiLimit + 1));
+ memset(minErate3p, 0, sizeof(uint32) * (fiLimit + 1));
- // Output the erate histogram
+ uint32 eratesMax = 1048576;
+ uint32 erates5len = 0;
+ uint32 *erates5 = new uint32 [eratesMax];
+ uint32 erates3len = 0;
+ uint32 *erates3 = new uint32 [eratesMax];
- {
- char EN[FILENAME_MAX];
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 olapsLen = 0;
+ BAToverlap *olaps = OC->getOverlaps(fi, olapsLen);
- sprintf(EN, "best.edges.erate.histogram");
+ uint64 ovl5sum = 0;
+ uint32 ovl5cnt = 0;
- errno = 0;
- FILE *EH = fopen(EN, "w");
- if (errno)
- fprintf(stderr, "BestOverlapGraph()-- failed to open '%s' for writing: %s\n", EN, strerror(errno)), exit(1);
+ uint64 ovl3sum = 0;
+ uint32 ovl3cnt = 0;
- for (AS_IID er=0; er <= AS_BAT_MAX_ERATE; er++) {
- double ER = OC->decodeError(er) * 100;
+ erates5len = 0;
+ erates5[0] = 0;
- if (ER <= 0.0)
- continue;
+ erates3len = 0;
+ erates3[0] = 0;
- fprintf(EH, "%.4f\t%u\t%u\t%f\t%f\n",
- ER,
- histo5[er],
- histo3[er],
- fabs(m5 - ER) / s5, // Grubb's test
- fabs(m3 - ER) / s3);
- }
+ // Find the error rate histogram for each end.
- fclose(EH);
- }
+ for (uint32 oo=0; oo<olapsLen; oo++) {
+ assert(fi == olaps[oo].a_iid);
- double erate5thresh = m5 + 2 * s5;
- double erate3thresh = m3 + 2 * s3;
+ if ((AS_BAT_overlapAEndIs5prime(olaps[oo])) && (erates5len < eratesMax))
+ erates5[erates5len++] = olaps[oo].errorRaw;
+ if ((AS_BAT_overlapAEndIs3prime(olaps[oo])) && (erates5len < eratesMax))
+ erates3[erates3len++] = olaps[oo].errorRaw;
+ }
- // For any read with best edge above THRESHOLD error, see if there is an alternate best
- // that is within the target error range.
+ // Sort by increasing error rate.
- for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ sort(erates5, erates5 + erates5len);
+ sort(erates3, erates3 + erates3len);
- if (erate5[fi] > erate5thresh) {
- fprintf(stderr, "RECOMPUTE frag %u 5'\n", fi);
- isBad[fi] = true;
- }
+ // Pick a min erate for each end.
- if (erate3[fi] > erate3thresh) {
- fprintf(stderr, "RECOMPUTE frag %u 3'\n", fi);
- isBad[fi] = true;
- }
+ minErate5p[fi] = erates5[(int32)(erates5len - erates5len * threshold)];
+ minErate3p[fi] = erates3[(int32)(erates3len - erates3len * threshold)];
+
+ if ((fi % 1000) == 0) {
+ fprintf(stderr, "len %d %d t %f vals %d %f %d %f\n", erates5len, erates3len, threshold,
+ minErate5p[fi], OC->decodeError(minErate5p[fi]),
+ minErate3p[fi], OC->decodeError(minErate3p[fi]));
}
+ }
- fprintf(stderr, "thresholds %f %f\n", erate5thresh, erate3thresh);
+ delete [] erates5;
+ delete [] erates3;
- double erateCthresh = MIN(erate5thresh, erate3thresh);
+ // Throw this at the OverlapCache, so it can remove overlaps.
-#if 0
- // Remove best edges, so we can rebuild
+ OC->removeWeakOverlaps(minErate5p, minErate3p);
+
+ delete [] minErate5p;
+ delete [] minErate3p;
+}
+
+
+
+BestOverlapGraph::BestOverlapGraph(double utgErrorRate,
+ double utgErrorLimit,
+ const char *prefix,
+ double doRemoveWeakThreshold,
+ bool doRemoveSuspicious,
+ bool doRemoveSpurs) {
+
+ bool doExamineOnlyTopN = false;
+ bool doRemoveFalseBest = false;
- memset(_bestA, 0, sizeof(BestOverlaps) * (fiLimit + 1));
- memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1));
+ setLogFile(prefix, "bestOverlapGraph");
+
+ writeLog("BestOverlapGraph-- allocating best edges ("F_SIZE_T"MB) and containments ("F_SIZE_T"MB)\n",
+ ((2 * sizeof(BestEdgeOverlap) * (FI->numFragments() + 1)) >> 20),
+ ((1 * sizeof(BestContainment) * (FI->numFragments() + 1)) >> 20));
+
+ _bestA = new BestOverlaps [FI->numFragments() + 1];
+ _scorA = new BestScores [FI->numFragments() + 1];
+
+ memset(_bestA, 0, sizeof(BestOverlaps) * (FI->numFragments() + 1));
+ memset(_scorA, 0, sizeof(BestScores) * (FI->numFragments() + 1));
+
+ _restrict = NULL;
+ _restrictEnabled = false;
+
+ mismatchCutoff = utgErrorRate;
+ mismatchLimit = utgErrorLimit;
- // Rebuild best edges, ignoring edges to spurs. We build edges out of spurs, but don't allow edges into them.
- // This should prevent them from being incorporated into a promiscuous unitig, but still let them be popped
- // as bubbles (but they shouldn't because they're spurs).
+ assert(mismatchCutoff >= 0.0);
+ assert(mismatchCutoff <= AS_MAX_ERROR_RATE);
+
+ // Initialize parallelism.
- // PASS 3: Find containments.
+ uint32 fiLimit = FI->numFragments();
+ uint32 numThreads = omp_get_max_threads();
+ uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99;
- writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads);
+ // PASS 0: Find suspicious fragments. For any found, mark as suspicious and don't allow
+ // these to be best overlaps.
+ if (doRemoveWeakThreshold > 0.0)
+ removeWeak(doRemoveWeakThreshold);
+
+ if (doRemoveSuspicious)
+ removeSuspicious();
+
+ // PASS 1: Find containments.
+
+ writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads);
+
+ if (false && AS_UTL_fileExists("best.contains", false, false)) {
+ writeLog("BestOverlapGraph()-- loading best containes from cache.\n");
+ assert(0); // Not quite done.
+
+ } else {
#pragma omp parallel for schedule(dynamic, blockSize)
for (AS_IID fi=1; fi <= fiLimit; fi++) {
uint32 no = 0;
BAToverlap *ovl = OC->getOverlaps(fi, no);
for (uint32 ii=0; ii<no; ii++)
- assert(ovl[ii].a_iid == fi);
-
- for (uint32 ii=0; ii<no; ii++)
- if (ovl[ii].error < erateCthresh)
- scoreContainment(ovl[ii]);
+ scoreContainment(ovl[ii]);
}
+ }
- // PASS 4: Find dovetails.
+ // PASS 2: Find dovetails.
+ if (false && AS_UTL_fileExists("best.edges", false, false)) {
+ writeLog("BestOverlapGraph()-- loading best edges from cache.\n");
+ assert(0); // Not quite done.
+
+ } else {
writeLog("BestOverlapGraph()-- analyzing %d fragments for best edges, with %d threads.\n", fiLimit, numThreads);
#pragma omp parallel for schedule(dynamic, blockSize)
@@ -468,31 +591,28 @@ BestOverlapGraph::BestOverlapGraph(double AS_UTG_ERROR_RATE,
uint32 no = 0;
BAToverlap *ovl = OC->getOverlaps(fi, no);
- if (isBad[ovl[fi].a_iid] == true)
- continue;
-
- for (uint32 ii=0; ii<no; ii++)
- assert(ovl[ii].a_iid == fi);
-
for (uint32 ii=0; ii<no; ii++)
- if (isBad[ovl[ii].b_iid] == false)
- scoreEdge(ovl[ii]);
+ scoreEdge(ovl[ii]);
}
-#endif
+ }
- delete [] histo5;
- delete [] histo3;
+ // Now, several optional refinements.
- delete [] erate5;
- delete [] erate3;
- }
+ if (doExamineOnlyTopN)
+ examineOnlyTopN();
- // Remove temporary scoring data
+ if (doRemoveSpurs)
+ removeSpurs();
+
+ if (doRemoveFalseBest)
+ removeFalseBest();
+
+ // Done with the scoring data.
delete [] _scorA;
_scorA = NULL;
- // Remove dovetail overlaps for contained fragments.
+ // Finally, remove dovetail overlaps for contained fragments.
writeLog("BestOverlapGraph()-- removing best edges for contained fragments, with %d threads.\n", numThreads);
@@ -512,20 +632,129 @@ BestOverlapGraph::BestOverlapGraph(double AS_UTG_ERROR_RATE,
}
+void
+BestOverlapGraph::rebuildBestContainsWithoutSingletons(UnitigVector &unitigs,
+ double utgErrorRate,
+ double utgErrorLimit,
+ const char *prefix) {
+
+ mismatchCutoff = utgErrorRate;
+ mismatchLimit = utgErrorLimit;
+ assert(mismatchCutoff >= 0.0);
+ assert(mismatchCutoff <= AS_MAX_ERROR_RATE);
-BestOverlapGraph::BestOverlapGraph(double AS_UTG_ERROR_RATE,
- double AS_UTG_ERROR_LIMIT,
- set<AS_IID> *restrict) {
+ uint32 fiLimit = FI->numFragments();
+
+ assert(_restrict == NULL);
+ assert(_restrictEnabled == false);
+
+ // Save the current best containments for a nice log, then clear
+
+ assert(_bestA != NULL);
- assert(AS_UTG_ERROR_RATE >= 0.0);
- assert(AS_UTG_ERROR_RATE <= AS_MAX_ERROR_RATE);
+ BestContainment *bestCold = new BestContainment [fiLimit + 1];
- assert(AS_CNS_ERROR_RATE >= 0.0);
- assert(AS_CNS_ERROR_RATE <= AS_MAX_ERROR_RATE);
+ for (AS_IID fi=0; fi<=fiLimit; fi++) {
+ bestCold[fi] = _bestA[fi]._bestC;
- mismatchCutoff = AS_UTG_ERROR_RATE;
- mismatchLimit = AS_UTG_ERROR_LIMIT;
+ // Clearing this destroys unitigs??
+
+ if (bestCold[fi].isContained == false) {
+ assert(_bestA[fi]._bestC.container == 0);
+ assert(_bestA[fi]._bestC.sameOrientation == false);
+ assert(_bestA[fi]._bestC.a_hang == 0);
+ assert(_bestA[fi]._bestC.b_hang == 0);
+ }
+
+ _bestA[fi]._bestC.container = 0;
+ _bestA[fi]._bestC.sameOrientation = false;
+ _bestA[fi]._bestC.a_hang = 0;
+ _bestA[fi]._bestC.b_hang = 0;
+ }
+
+ // Allocate space for new scores
+
+ assert(_scorA == NULL);
+
+ _scorA = new BestScores [fiLimit + 1];
+
+ memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1));
+
+ // Rebuild contains ignoring singleton containers
+
+ for (AS_IID fi=1; fi<=fiLimit; fi++) {
+ uint32 no = 0;
+
+ if (bestCold[fi].isContained == false)
+ continue;
+
+ BAToverlap *ovl = OC->getOverlaps(fi, no);
+
+ for (uint32 ii=0; ii<no; ii++) {
+ AS_IID autg = Unitig::fragIn(ovl[ii].a_iid);
+ AS_IID butg = Unitig::fragIn(ovl[ii].b_iid);
+
+ assert(autg == 0); // Contained cannot be placed yet.
+
+ if ((butg != 0) &&
+ (unitigs[butg]->ufpath.size() == 1))
+ // Skip; container is a in a unitig, and that unitig is a singleton.
+ continue;
+
+ scoreContainment(ovl[ii]);
+ }
+ }
+
+ delete [] _scorA;
+ _scorA = NULL;
+
+ // Remove best edges for contains (shouldn't be any; we didn't make new ones since the last time we removed)
+
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ if (isContained(fi) == true) {
+ getBestEdgeOverlap(fi, false)->set(0, 0, 0, 0);
+ getBestEdgeOverlap(fi, true) ->set(0, 0, 0, 0);
+ }
+ }
+
+ // Log changes
+
+ for (AS_IID fi=0; fi<=fiLimit; fi++) {
+ if ((bestCold[fi].container != _bestA[fi]._bestC.container) ||
+ (bestCold[fi].sameOrientation != _bestA[fi]._bestC.sameOrientation) ||
+ (bestCold[fi].a_hang != _bestA[fi]._bestC.a_hang) ||
+ (bestCold[fi].b_hang != _bestA[fi]._bestC.b_hang))
+ writeLog("frag %u changed container from %c %u/%c/%d/%d to %c %u/%c/%d/%d\n",
+ fi,
+ bestCold[fi].isContained ? 'T' : 'F',
+ bestCold[fi].container,
+ bestCold[fi].sameOrientation ? 'N' : 'A',
+ bestCold[fi].a_hang,
+ bestCold[fi].b_hang,
+ _bestA[fi]._bestC.isContained ? 'T' : 'F',
+ _bestA[fi]._bestC.container,
+ _bestA[fi]._bestC.sameOrientation ? 'N' : 'A',
+ _bestA[fi]._bestC.a_hang,
+ _bestA[fi]._bestC.b_hang);
+ }
+
+ delete [] bestCold;
+}
+
+
+
+
+
+BestOverlapGraph::BestOverlapGraph(double utgErrorRate,
+ double utgErrorLimit,
+ set<AS_IID> *restrict) {
+
+ mismatchCutoff = utgErrorRate;
+ mismatchLimit = utgErrorLimit;
+
+ assert(mismatchCutoff >= 0.0);
+ assert(mismatchCutoff <= AS_MAX_ERROR_RATE);
_bestA = NULL;
_scorA = NULL;
@@ -533,7 +762,10 @@ BestOverlapGraph::BestOverlapGraph(double AS_UTG_ERROR_RATE,
_bestM.clear();
_scorM.clear();
- _restrict = restrict;
+ assert(restrict != NULL);
+
+ _restrict = restrict;
+ _restrictEnabled = true;
// PASS 0: Load the map (necessary?)
@@ -596,6 +828,9 @@ BestOverlapGraph::BestOverlapGraph(double AS_UTG_ERROR_RATE,
}
}
#endif
+
+ _restrict = NULL;
+ _restrictEnabled = false;
}
@@ -698,15 +933,26 @@ BestOverlapGraph::scoreContainment(const BAToverlap& olap) {
if (newScr > bestCscore(olap.a_iid)) {
BestContainment *c = getBestContainer(olap.a_iid);
+#if 0
+ writeLog("set best for %d from "F_U32" score="F_U64" to "F_U32" score="F_U64"\n",
+ olap.a_iid,
+ c->container, bestCscore(olap.a_iid),
+ olap.b_iid, newScr);
+#endif
+
c->container = olap.b_iid;
c->isContained = true;
c->sameOrientation = olap.flipped ? false : true;
c->a_hang = olap.flipped ? olap.b_hang : -olap.a_hang;
c->b_hang = olap.flipped ? olap.a_hang : -olap.b_hang;
- //writeLog("set best for %d from "F_U64" to "F_U64"\n", olap.a_iid, bestCscore(olap.a_iid), newScr);
-
bestCscore(olap.a_iid) = newScr;
+#if 0
+ } else {
+ writeLog("NOT best for %d WITH "F_U32" score="F_U64"\n",
+ olap.a_iid,
+ olap.b_iid, newScr);
+#endif
}
}
@@ -856,13 +1102,21 @@ BestOverlapGraph::isOverlapBadQuality(const BAToverlap& olap) {
bool
BestOverlapGraph::isOverlapRestricted(const BAToverlap &olap) {
- if (_restrict == NULL) {
+ if (_restrictEnabled == false)
+ return(false);
+
+ // Previous version would check if the two reads were both unplaced, if _restrict was NULL.
+#if 0
+ if (_restrictToUnplaced) {
if ((Unitig::fragIn(olap.a_iid) == 0) &&
(Unitig::fragIn(olap.b_iid) == 0))
return(false);
else
return(true);
}
+#endif
+
+ assert(_restrict != NULL);
if ((_restrict->count(olap.a_iid) != 0) &&
(_restrict->count(olap.b_iid) != 0))
diff --git a/src/AS_BAT/AS_BAT_BestOverlapGraph.H b/src/AS_BAT/AS_BAT_BestOverlapGraph.H
index 5e663e5..105ece1 100644
--- a/src/AS_BAT/AS_BAT_BestOverlapGraph.H
+++ b/src/AS_BAT/AS_BAT_BestOverlapGraph.H
@@ -22,7 +22,7 @@
#ifndef INCLUDE_AS_BAT_BESTOVERLAPGRAPH
#define INCLUDE_AS_BAT_BESTOVERLAPGRAPH
-static const char *rcsid_INCLUDE_AS_BAT_BESTOVERLAPGRAPH = "$Id: AS_BAT_BestOverlapGraph.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_INCLUDE_AS_BAT_BESTOVERLAPGRAPH = "$Id: AS_BAT_BestOverlapGraph.H 4580 2014-10-21 14:53:21Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_Unitig.H"
@@ -51,12 +51,20 @@ public:
class BestOverlapGraph {
+private:
+ void removeSuspicious(void);
+ void examineOnlyTopN(void);
+ void removeSpurs(void);
+ void removeFalseBest(void);
+ void removeWeak(double threshold);
+
public:
BestOverlapGraph(double erate,
double elimit,
const char *prefix,
- bool removeSuspicious,
- bool removeSpurs);
+ double doRemoveWeakThreshold,
+ bool doRemoveSuspicious,
+ bool doRemoveSpurs);
BestOverlapGraph(double erate,
double elimit,
@@ -105,6 +113,12 @@ public:
void reportBestEdges(void);
+public:
+ void rebuildBestContainsWithoutSingletons(UnitigVector &unitigs,
+ double erate,
+ double elimit,
+ const char *prefix);
+
private:
bool isOverlapBadQuality(const BAToverlap& olap);
bool isOverlapRestricted(const BAToverlap &olap);
@@ -116,19 +130,19 @@ private:
private:
uint64 &bestCscore(AS_IID id) {
- if (_scorA)
+ if (_restrictEnabled == false)
return(_scorA[id]._bestCscore);
return(_scorM[id]._bestCscore);
};
uint64 &best5score(AS_IID id) {
- if (_scorA)
+ if (_restrictEnabled == false)
return(_scorA[id]._best5score);
return(_scorM[id]._best5score);
};
uint64 &best3score(AS_IID id) {
- if (_scorA)
+ if (_restrictEnabled == false)
return(_scorA[id]._best3score);
return(_scorM[id]._best3score);
};
@@ -143,6 +157,7 @@ private:
map<AS_IID, BestScores> _scorM;
set<AS_IID> *_restrict;
+ bool _restrictEnabled;
public:
double mismatchCutoff;
diff --git a/src/AS_BAT/AS_BAT_Datatypes.H b/src/AS_BAT/AS_BAT_Datatypes.H
index 7e19db1..61be184 100644
--- a/src/AS_BAT/AS_BAT_Datatypes.H
+++ b/src/AS_BAT/AS_BAT_Datatypes.H
@@ -22,7 +22,7 @@
#ifndef INCLUDE_AS_BAT_DATATYPES
#define INCLUDE_AS_BAT_DATATYPES
-static const char *rcsid_INCLUDE_AS_BAT_DATATYPES = "$Id: AS_BAT_Datatypes.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_INCLUDE_AS_BAT_DATATYPES = "$Id: AS_BAT_Datatypes.H 4557 2014-08-11 12:24:27Z brianwalenz $";
#include "AS_global.H"
#include "AS_MSG_pmesg.H"
@@ -33,12 +33,11 @@ static const char *rcsid_INCLUDE_AS_BAT_DATATYPES = "$Id: AS_BAT_Datatypes.H 437
#include <set>
#include <list>
#include <vector>
-#include <iostream>
-#include <cmath>
-#include <limits>
#include <algorithm>
+#ifndef BROKEN_CLANG_OpenMP
#include <omp.h>
+#endif
using namespace std;
@@ -170,13 +169,17 @@ AS_BAT_overlapBEndIs3prime(const BAToverlap& olap) {
class BestEdgeOverlap {
public:
BestEdgeOverlap() {
+ clear();
+ };
+ ~BestEdgeOverlap() {
+ };
+
+ void clear(void) {
_id = 0;
_e3p = 0;
_ahang = 0;
_bhang = 0;
};
- ~BestEdgeOverlap() {
- };
void set(BAToverlap const &olap) {
_id = olap.b_iid;
@@ -219,14 +222,18 @@ private:
class BestContainment{
public:
BestContainment() {
+ clear();
+ };
+ ~BestContainment() {
+ };
+
+ void clear(void) {
container = 0;
isContained = false;
sameOrientation = false;
a_hang = 0;
b_hang = 0;
};
- ~BestContainment() {
- };
uint32 container;
#if AS_OVS_HNGBITS < 16
diff --git a/src/AS_BAT/AS_BAT_ExtendByMates.C b/src/AS_BAT/AS_BAT_ExtendByMates.C
index 8fe4c49..9285f5b 100644
--- a/src/AS_BAT/AS_BAT_ExtendByMates.C
+++ b/src/AS_BAT/AS_BAT_ExtendByMates.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_ExtendByMates.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_ExtendByMates.C 4496 2014-01-28 20:12:58Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_BestOverlapGraph.H"
@@ -72,7 +72,7 @@ extendByMates(UnitigVector &unitigs,
if (extraMates == 0)
continue;
- // Build a set of the fragments in this unitig plus their mates, and a set of just themates.
+ // Build a set of the fragments in this unitig plus their mates, and a set of just the mates.
set<AS_IID> frags;
set<AS_IID> mates;
diff --git a/src/AS_BAT/AS_BAT_Logging.H b/src/AS_BAT/AS_BAT_Logging.H
index c6869c3..f6a294c 100644
--- a/src/AS_BAT/AS_BAT_Logging.H
+++ b/src/AS_BAT/AS_BAT_Logging.H
@@ -22,7 +22,7 @@
#ifndef INCLUDE_AS_BAT_LOGGING
#define INCLUDE_AS_BAT_LOGGING
-static const char *rcsid_INCLUDE_AS_BAT_LOGGING = "$Id: AS_BAT_Logging.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_INCLUDE_AS_BAT_LOGGING = "$Id: AS_BAT_Logging.H 4557 2014-08-11 12:24:27Z brianwalenz $";
#include "AS_global.H"
#include "AS_UTL_fileIO.H"
@@ -36,7 +36,9 @@ static const char *rcsid_INCLUDE_AS_BAT_LOGGING = "$Id: AS_BAT_Logging.H 4371 20
#include <stdarg.h>
#include <errno.h>
+#ifndef BROKEN_CLANG_OpenMP
#include <omp.h>
+#endif
void setLogFile(char const *prefix, char const *name);
void writeLog(char const *fmt, ...);
diff --git a/src/AS_BAT/AS_BAT_MergeSplitJoin.C b/src/AS_BAT/AS_BAT_MergeSplitJoin.C
index fe6acb1..d3f8fee 100644
--- a/src/AS_BAT/AS_BAT_MergeSplitJoin.C
+++ b/src/AS_BAT/AS_BAT_MergeSplitJoin.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_MergeSplitJoin.C 4408 2013-09-09 02:39:13Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_MergeSplitJoin.C 4572 2014-10-14 23:06:05Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_BestOverlapGraph.H"
@@ -40,7 +40,7 @@ static const char *rcsid = "$Id: AS_BAT_MergeSplitJoin.C 4408 2013-09-09 02:39:1
#include "AS_BAT_RepeatJunctionEvidence.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
uint32 SPURIOUS_COVERAGE_THRESHOLD = 6; // Need to have more than this coverage in non-unitig reads aligned to call it a repeat area
@@ -60,9 +60,10 @@ mergeBubbles_findEnds(UnitigVector &unitigs,
ufNode &lFrg,
Unitig *target) {
- // Search for edges. For a bubble to exist, either the first or last non-contained fragment
- // must have an edge to the 'merge' unitig it is a bubble of. Ideally, both the first and
- // last will have edges to the same unitig, but we'll test and allow only a single edge.
+ // Search for edges. For a bubble to exist, at least one of the first or last non-contained
+ // fragment must have an edge to the 'target' unitig (by construction of the inputs to this
+ // routine). Ideally, both the first and last will have edges to the same unitig, but we'll test
+ // and allow only a single edge.
uint32 zIdx = ~(uint32)0;
uint32 fIdx = zIdx;
@@ -211,7 +212,8 @@ mergeBubbles_checkEnds(UnitigVector &unitigs,
placeFragUsingOverlaps(unitigs, target, fFrg.ident, placements);
#ifdef LOG_BUBBLE_TESTS
- writeLog("popBubbles()-- fFrg %u has %u potential placements.\n", fFrg.ident, placements.size());
+ writeLog("popBubbles()-- fFrg %u has %u potential placements in unitig %u.\n",
+ fFrg.ident, placements.size(), target->id());
#endif
for (uint32 i=0; i<placements.size(); i++) {
@@ -219,35 +221,39 @@ mergeBubbles_checkEnds(UnitigVector &unitigs,
if (placements[i].fCoverage < 0.99) {
#ifdef LOG_BUBBLE_FAILURE
- writeLog("popBubbles()-- fFrg %u low coverage %f at %u,%u\n",
- fFrg.ident,
- placements[i].fCoverage,
- placements[i].position.bgn, placements[i].position.end);
+ writeLog("popBubbles()-- fFrg %u low coverage %f at unitig %u %u,%u\n",
+ fFrg.ident,
+ placements[i].fCoverage,
+ placements[i].tigID,
+ placements[i].position.bgn, placements[i].position.end);
#endif
continue;
} else {
#ifdef LOG_BUBBLE_FAILURE
- writeLog("popBubbles()-- fFrg %u GOOD coverage %f at %u,%u\n",
- fFrg.ident,
- placements[i].fCoverage,
- placements[i].position.bgn, placements[i].position.end);
+ writeLog("popBubbles()-- fFrg %u GOOD coverage %f at unitig %u %u,%u\n",
+ fFrg.ident,
+ placements[i].fCoverage,
+ placements[i].tigID,
+ placements[i].position.bgn, placements[i].position.end);
#endif
}
if (placements[i].errors / placements[i].aligned < fFrgPlacement.errors / fFrgPlacement.aligned) {
#ifdef LOG_BUBBLE_FAILURE
- writeLog("popBubbles()-- fFrg %u GOOD identity %f at %u,%u\n",
- fFrg.ident,
- placements[i].errors / placements[i].aligned,
- placements[i].position.bgn, placements[i].position.end);
+ writeLog("popBubbles()-- fFrg %u GOOD identity %f at unitig %u %u,%u\n",
+ fFrg.ident,
+ placements[i].errors / placements[i].aligned,
+ placements[i].tigID,
+ placements[i].position.bgn, placements[i].position.end);
#endif
fFrgPlacement = placements[i];
} else {
#ifdef LOG_BUBBLE_FAILURE
- writeLog("popBubbles()-- fFrg %u low identity %f at %u,%u\n",
- fFrg.ident,
- placements[i].errors / placements[i].aligned,
- placements[i].position.bgn, placements[i].position.end);
+ writeLog("popBubbles()-- fFrg %u low identity %f at unitig %u %u,%u\n",
+ fFrg.ident,
+ placements[i].errors / placements[i].aligned,
+ placements[i].tigID,
+ placements[i].position.bgn, placements[i].position.end);
#endif
}
}
@@ -616,6 +622,19 @@ mergeBubbles(UnitigVector &unitigs, Unitig *target, intersectionList *ilist) {
assert(bubble->id() == Unitig::fragIn(isect->invadFrg));
+ // I don't like a number of reads filter - for 50x Illumina, 500 reads is only 1k of unitig,
+ // but for 10x PacBio, this is over 250k of unitig.
+
+ if ((bubble == NULL) ||
+ (bubble->getLength() > 50000)) {
+ writeLog("popBubbles()-- Skip bubble %u length %u with "F_SIZE_T" frags - edge from %d/%c' to utg %d %d/%c'\n",
+ bubble->id(), bubble->getLength(), bubble->ufpath.size(),
+ isect->invadFrg, isect->invad3p ? '3' : '5',
+ target->id(),
+ isect->isectFrg, isect->isect3p ? '3' : '5');
+ continue;
+ }
+
if (bubble->id() == target->id())
// HEY! We're not a bubble in ourself!
continue;
@@ -623,6 +642,15 @@ mergeBubbles(UnitigVector &unitigs, Unitig *target, intersectionList *ilist) {
ufNode fFrg; // First fragment in the bubble
ufNode lFrg; // Last fragment in the bubble
+ // We have no way of deciding if we've tested this bubble unitig already. Each bubble unitig
+ // should generate two intersection edges. If those edges are to the same target unitig, and
+ // the bubble fails to pop, we'll test the bubble twice.
+ //
+ // This is kind of by design. The two intersections could be to two different locations, and
+ // maybe one will work while the other doesn't. Though, I think we accept a placement only
+ // if the two end reads are consistent implying that we'd double test a bubble if the
+ // placements are different, and that we'd fail both times.
+
if (mergeBubbles_findEnds(unitigs, bubble, fFrg, lFrg, target) == false)
continue;
@@ -769,7 +797,7 @@ markRepeats_placeAndProcessOverlaps(UnitigVector &unitigs,
double meanError,
double stddevError,
set<AS_IID> &ovlFrags,
- intervalList &aligned,
+ intervalList<int32> &aligned,
vector<repeatJunctionEvidence> &evidence) {
aligned.clear();
@@ -847,7 +875,7 @@ markRepeats_placeAndProcessOverlaps(UnitigVector &unitigs,
#if 0
uint32
markRepeats_computeUnitigCoverage(Unitig *tig) {
- intervalList coverage;
+ intervalList<int32> coverage;
for (uint32 fi=0; fi<tig->ufpath.size(); fi++) {
ufNode frg = tig->ufpath[fi];
@@ -906,7 +934,7 @@ markRepeats_computeUnitigCoverage(Unitig *tig) {
//
void
markRepeats_filterIntervalsSpannedByFragment(Unitig *target,
- intervalList &aligned,
+ intervalList<int32> &aligned,
vector<repeatRegion> ®ions) {
uint32 tiglen = target->getLength();
@@ -914,7 +942,7 @@ markRepeats_filterIntervalsSpannedByFragment(Unitig *target,
uint32 filteredBases = 0;
uint32 filteredCovered = 0;
- intervalDepth depth(aligned);
+ intervalList<int32> depth(aligned);
aligned.merge(); // Just for a stupid log message
@@ -929,10 +957,10 @@ markRepeats_filterIntervalsSpannedByFragment(Unitig *target,
//
for (uint32 dd=0; dd<depth.numberOfIntervals(); dd++) {
- if (depth.de(dd) == 0)
+ if (depth.depth(dd) == 0)
continue;
- if (depth.de(dd) <= spuriousNoiseThreshold) {
+ if (depth.depth(dd) <= spuriousNoiseThreshold) {
filteredBases += depth.hi(dd) - depth.lo(dd);
continue;
}
@@ -1532,6 +1560,8 @@ markRepeats_breakUnitigs(UnitigVector &unitigs,
uint32 bid = breakID[fi];
if (ejtFrags.count(frg.ident) > 0) {
+ writeLog("markRepeats()-- EJECT unanchored frag %u from unitig %u (mate frag %u)\n",
+ frg.ident, target->id(), FI->mateIID(frg.ident));
target->removeFrag(frg.ident);
continue;
}
@@ -1655,7 +1685,7 @@ markRepeats(UnitigVector &unitigs,
double meanError = 0;
double stddevError = 0;
- intervalList aligned;
+ intervalList<int32> aligned;
vector<overlapPlacement> places;
diff --git a/src/AS_BAT/AS_BAT_Outputs.C b/src/AS_BAT/AS_BAT_Outputs.C
index 199e5e0..a06840c 100644
--- a/src/AS_BAT/AS_BAT_Outputs.C
+++ b/src/AS_BAT/AS_BAT_Outputs.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_Outputs.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_Outputs.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_Unitig.H"
@@ -40,14 +40,17 @@ unitigToMA(MultiAlignT *ma,
uint32 iumiid,
Unitig *utg) {
- ma->maID = iumiid;
- ma->data.unitig_coverage_stat = 1.0; // Default to just barely unique
- ma->data.unitig_microhet_prob = 1.0; // Default to 100% probability of unique
+ ma->maID = iumiid;
+ ma->data.unitig_coverage_stat = 1.0; // Default to just barely unique
+ ma->data.unitig_microhet_prob = 1.0; // Default to 100% probability of unique
- ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+ ma->data.unitig_status = AS_UNASSIGNED;
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
- ma->data.contig_status = AS_UNPLACED;
+ ma->data.contig_status = AS_UNPLACED;
// Add the fragments
diff --git a/src/AS_BAT/AS_BAT_OverlapCache.C b/src/AS_BAT/AS_BAT_OverlapCache.C
index e982e55..b11e853 100644
--- a/src/AS_BAT/AS_BAT_OverlapCache.C
+++ b/src/AS_BAT/AS_BAT_OverlapCache.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_OverlapCache.C 4451 2013-10-15 00:30:04Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_OverlapCache.C 4580 2014-10-21 14:53:21Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_OverlapCache.H"
@@ -103,6 +103,10 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
_memLimit = UINT64_MAX;
}
+ // Need to initialize thread data before we can account for their size.
+ _threadMax = omp_get_max_threads();
+ _thread = new OverlapCacheThreadData [_threadMax];
+
// Account for memory used by fragment data, best overlaps, and unitigs.
// The chunk graph is temporary, and should be less than the size of the unitigs.
@@ -112,8 +116,23 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
uint64 memUL = FI->numFragments() * sizeof(ufNode); // For fragment positions in unitigs
uint64 memUT = FI->numFragments() * sizeof(uint32) / 16; // For unitigs (assumes 32 frag / unitig)
uint64 memID = FI->numFragments() * sizeof(uint32) * 2; // For maps of fragment id to unitig id
+ uint64 memC1 = (FI->numFragments() + 1) * (sizeof(BAToverlapInt *) + sizeof(uint32));
+ uint64 memC2 = _ovsMax * (sizeof(OVSoverlap) + sizeof(uint64) + sizeof(uint64));
+ uint64 memC3 = _threadMax * _thread[0]._batMax * sizeof(BAToverlap);
+ uint64 memC4 = (FI->numFragments() + 1) * sizeof(uint32);
uint64 memOS = (_memLimit == getMemorySize()) ? (0.1 * getMemorySize()) : 0.0;
- uint64 memTT = memFI + memBE + memBC + memUL + memUT + memID + memOS;
+
+ uint64 memTT = memFI + memBE + memBC + memUL + memUT + memID + memC1 + memC2 + memC3 + memC4 + memOS;
+
+ if (onlySave) {
+ fprintf(stderr, "OverlapCache()-- Only saving overlaps, not computing unitigs.\n");
+ memBE = 0;
+ memBC = 0;
+ memUL = 0;
+ memUT = 0;
+ memID = 0;
+ memTT = memFI + memBE + memBC + memUL + memUT + memID + memOS + memC1 + memC2 + memC3 + memC4;
+ }
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for fragment data.\n", memFI >> 20);
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for best edges.\n", memBE >> 20);
@@ -121,6 +140,10 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for unitig layouts.\n", memUL >> 20);
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for unitigs.\n", memUT >> 20);
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for id maps.\n", memID >> 20);
+ fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for overlap cache pointers.\n", memC1 >> 20);
+ fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for overlap cache initial bucket.\n", memC2 >> 20);
+ fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for overlap cache thread data.\n", memC3 >> 20);
+ fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for number of overlaps per read.\n", memC4 >> 20);
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for other processes.\n", memOS >> 20);
fprintf(stderr, "OverlapCache()-- ---------\n");
fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for data structures (sum of above).\n", memTT >> 20);
@@ -128,7 +151,7 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
if (_memLimit <= memTT) {
int64 defecit = (int64)memTT - (int64)_memLimit;
- fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB available for overlaps.\n", defecit);
+ fprintf(stderr, "OverlapCache()-- %7"F_S64P"MB available for overlaps.\n", defecit);
fprintf(stderr, "OverlapCache()-- Out of memory before loading overlaps; increase -M.\n");
exit(1);
}
@@ -136,7 +159,7 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
_memLimit -= memTT;
_memUsed = 0;
- fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB available for overlaps.\n", _memLimit >> 20);
+ fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB available for overlaps.\n", _memLimit >> 20);
fprintf(stderr, "\n");
// Decide on the default block size. We want to use large blocks (to reduce the number of
@@ -166,8 +189,6 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
_cachePtr = new BAToverlapInt * [FI->numFragments() + 1];
_cacheLen = new uint32 [FI->numFragments() + 1];
- _memUsed += (FI->numFragments() + 1) * (sizeof(BAToverlapInt *) + sizeof(uint32));
-
memset(_cachePtr, 0, sizeof(BAToverlapInt *) * (FI->numFragments() + 1));
memset(_cacheLen, 0, sizeof(uint32) * (FI->numFragments() + 1));
@@ -178,13 +199,8 @@ OverlapCache::OverlapCache(OverlapStore *ovlStoreUniq,
_ovsSco = new uint64 [_ovsMax];
_ovsTmp = new uint64 [_ovsMax];
- _memUsed += _ovsMax * sizeof(OVSoverlap);
- _memUsed += _ovsMax * sizeof(uint64);
-
- _threadMax = omp_get_max_threads();
- _thread = new OverlapCacheThreadData [_threadMax];
-
- _memUsed += _threadMax * _thread[0]._batMax * sizeof(BAToverlap);
+ //_threadMax = omp_get_max_threads();
+ //_thread = new OverlapCacheThreadData [_threadMax];
_OVSerate = NULL;
_BATerate = NULL;
@@ -683,6 +699,77 @@ OverlapCache::getOverlaps(uint32 fragIID, uint32 &numOverlaps) {
+void
+OverlapCache::removeWeakOverlaps(uint32 *minErate5p,
+ uint32 *minErate3p) {
+
+ uint32 fiLimit = FI->numFragments();
+
+ uint64 saved = 0;
+ uint64 ignored = 0;
+ uint64 removed = 0;
+
+ for (AS_IID fi=1; fi <= fiLimit; fi++) {
+ uint32 numOverlaps = _cacheLen[fi];
+ BAToverlapInt *ptr = _cachePtr[fi];
+
+ for (uint32 pos=0; pos < numOverlaps; pos++) {
+ uint32 aiid = fi;
+ uint32 biid = ptr[pos].b_iid;
+ uint32 erate = ptr[pos].error;
+
+ // Ignore contained overlaps.
+
+ if (((ptr[pos].a_hang <= 0) && (ptr[pos].b_hang >= 0)) ||
+ ((ptr[pos].a_hang >= 0) && (ptr[pos].b_hang <= 0))) {
+ ignored++;
+ continue;
+ }
+
+ // Decide which end we need to be looking at.
+
+ uint32 ta = 0;
+ uint32 tb = 0;
+
+ if (ptr[pos].a_hang > 0)
+ ta = minErate3p[aiid];
+ else
+ ta = minErate5p[aiid];
+
+ if (ptr[pos].flipped == false) {
+ if (ptr[pos].b_hang > 0)
+ tb = minErate5p[biid];
+ else
+ tb = minErate3p[biid];
+
+ } else {
+ if (ptr[pos].b_hang > 0)
+ tb = minErate3p[biid];
+ else
+ tb = minErate5p[biid];
+ }
+
+ // If the erate is more than the threshold, 'remove' the overlap by maxing out the erate.
+
+ if ((erate > ta) ||
+ (erate > tb)) {
+ //fprintf(stdout, "OverlapCache::removeWeakOverlaps()-- remove %7d %7d at %.3f\n", aiid, biid, OC->decodeError(erate));
+ removed++;
+ ptr[pos].error = AS_BAT_MAX_ERATE;
+ } else {
+ saved++;
+ }
+ }
+ }
+
+ writeLog("OverlapCache::removeWeakOverlaps()-- removed "F_U64" weak overlaps.\n", removed);
+ writeLog("OverlapCache::removeWeakOverlaps()-- ignored "F_U64" contained overlaps.\n", ignored);
+ writeLog("OverlapCache::removeWeakOverlaps()-- retained "F_U64" strong overlaps.\n", saved);
+}
+
+
+
+
double
OverlapCache::findError(uint32 aIID, uint32 bIID) {
diff --git a/src/AS_BAT/AS_BAT_OverlapCache.H b/src/AS_BAT/AS_BAT_OverlapCache.H
index a799f2c..7e91ff1 100644
--- a/src/AS_BAT/AS_BAT_OverlapCache.H
+++ b/src/AS_BAT/AS_BAT_OverlapCache.H
@@ -22,7 +22,7 @@
#ifndef INCLUDE_AS_BAT_OVERLAPCACHE
#define INCLUDE_AS_BAT_OVERLAPCACHE
-static const char *rcsid_INCLUDE_AS_BAT_OVERLAPCACHE = "$Id: AS_BAT_OverlapCache.H 4398 2013-08-28 18:21:11Z brianwalenz $";
+static const char *rcsid_INCLUDE_AS_BAT_OVERLAPCACHE = "$Id: AS_BAT_OverlapCache.H 4580 2014-10-21 14:53:21Z brianwalenz $";
#include "memoryMappedFile.H"
@@ -138,6 +138,9 @@ public:
BAToverlap *getOverlaps(uint32 fragIID, uint32 &numOverlaps);
+ void removeWeakOverlaps(uint32 *minErate5p,
+ uint32 *minErate3p);
+
double decodeError(uint32 error) {
return(_BATerate[error]);
};
diff --git a/src/AS_BAT/AS_BAT_PlaceContains.C b/src/AS_BAT/AS_BAT_PlaceContains.C
index 8aedad5..3eee5c5 100644
--- a/src/AS_BAT/AS_BAT_PlaceContains.C
+++ b/src/AS_BAT/AS_BAT_PlaceContains.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_PlaceContains.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_PlaceContains.C 4497 2014-01-29 14:52:31Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_Unitig.H"
@@ -27,14 +27,24 @@ static const char *rcsid = "$Id: AS_BAT_PlaceContains.C 4371 2013-08-01 17:19:47
#include "AS_BAT_PlaceContains.H"
+#include "AS_BAT_PlaceFragUsingOverlaps.H"
+
void
placeContainsUsingBestOverlaps(UnitigVector &unitigs) {
uint32 fragsPlaced = 1;
uint32 fragsPending = 0;
+ uint32 *nReadsPer = new uint32 [unitigs.size()];
+
+ uint32 totalPlaced = 0;
+ uint32 totalPlacedInSingleton = 0;
+
logFileFlags &= ~LOG_PLACE_FRAG;
+ for (uint32 ii=0; ii<unitigs.size(); ii++)
+ nReadsPer[ii] = (unitigs[ii] == NULL) ? 0 : unitigs[ii]->getNumFrags();
+
while (fragsPlaced > 0) {
fragsPlaced = 0;
fragsPending = 0;
@@ -43,7 +53,6 @@ placeContainsUsingBestOverlaps(UnitigVector &unitigs) {
for (uint32 fid=1; fid<FI->numFragments()+1; fid++) {
BestContainment *bestcont = OG->getBestContainer(fid);
- Unitig *utg;
if (bestcont->isContained == false)
// Not a contained fragment.
@@ -59,7 +68,14 @@ placeContainsUsingBestOverlaps(UnitigVector &unitigs) {
continue;
}
- utg = unitigs[Unitig::fragIn(bestcont->container)];
+ uint32 utgid = Unitig::fragIn(bestcont->container);
+ Unitig *utg = unitigs[utgid];
+
+ totalPlaced++;
+
+ if (nReadsPer[utgid] == 1)
+ totalPlacedInSingleton++;
+
utg->addContainedFrag(fid, bestcont, logFileFlagSet(LOG_INITIAL_CONTAINED_PLACEMENT));
if (utg->id() != Unitig::fragIn(fid))
@@ -70,16 +86,19 @@ placeContainsUsingBestOverlaps(UnitigVector &unitigs) {
fragsPlaced++;
}
- writeLog("==> PLACING CONTAINED FRAGMENTS - placed %d fragments; still need to place %d\n",
+ writeLog("placeContainsUsingBestOverlaps()-- Placed %d fragments; still need to place %d\n",
fragsPlaced, fragsPending);
- if ((fragsPlaced == 0) && (fragsPending > 0)) {
- writeLog("Stopping contained fragment placement due to zombies.\n");
- fragsPlaced = 0;
- fragsPending = 0;
- }
+ if ((fragsPlaced == 0) && (fragsPending > 0))
+ writeLog("placeContainsUsingBestOverlaps()-- Stopping contained fragment placement due to zombies.\n");
}
+ writeLog("placeContainsUsingBestOverlaps()-- %u frags placed in unitigs (including singleton unitigs)\n", totalPlaced);
+ writeLog("placeContainsUsingBestOverlaps()-- %u frags placed in singleton unitigs\n", totalPlacedInSingleton);
+ writeLog("placeContainsUsingBestOverlaps()-- %u frags unplaced\n", fragsPending);
+
+ delete [] nReadsPer;
+
for (uint32 ti=1; ti<unitigs.size(); ti++) {
Unitig *utg = unitigs[ti];
@@ -129,20 +148,73 @@ placeContainsUsingBestOverlaps(Unitig *target, set<AS_IID> *fragments) {
void
placeContainsUsingAllOverlaps(UnitigVector &unitigs,
- bool withMatesToNonContained,
- bool withMatesToUnambiguousContain) {
+ bool withMatesToNonContained,
+ bool withMatesToUnambiguousContain) {
+
+ // UNFINISHED. This results in crashes later in the process.
-#if 0
for (uint32 fid=1; fid<FI->numFragments()+1; fid++) {
- ufNode frag;
+ ufNode frg;
+ //ufNode mat;
if (Unitig::fragIn(fid) > 0)
// Fragment placed already.
continue;
- frag.ident = fid;
+ frg.ident = fid;
+ //mat.ident = 0; //mid;
+
+ overlapPlacement frgPlacement;
+ //overlapPlacement matPlacement;
+
+ frgPlacement.errors = 4.0e9;
+ frgPlacement.aligned = 1;
+
+ //matPlacement.errors = 4.0e9;
+ //matPlacement.aligned = 1;
+
+ vector<overlapPlacement> placements;
+
+ // Place the read.
+
+ placeFragUsingOverlaps(unitigs, NULL, frg.ident, placements);
+
+ // Search the placements for the highest expect identity placement using all overlaps in the unitig.
+
+ Unitig *frgTig = NULL;
+ Unitig *matTig = NULL;
+
+ for (uint32 i=0; i<placements.size(); i++) {
+ if (placements[i].fCoverage < 0.99)
+ continue;
+
+ if (placements[i].errors / placements[i].aligned < frgPlacement.errors / frgPlacement.aligned) {
+ frgPlacement = placements[i];
+ frgTig = unitigs[placements[i].tigID];
+ }
+ }
+
+ frg.ident = frgPlacement.frgID;
+ frg.contained = 0;
+ frg.parent = 0;
+ frg.ahang = 0;
+ frg.bhang = 0;
+ frg.position = frgPlacement.position;
+ frg.containment_depth = 0;
+
+ if ((frg.position.bgn == 0) &&
+ (frg.position.end == 0))
+ // Failed to place the contained read anywhere. We should probably just make a new unitig
+ // for it right here.
+ continue;
+
+ // Place the mate
+
+ // Add the placed read to the unitig.
+
+ writeLog("placeContainsUsingAllOverlaps()-- frag %u placed in tig %u at %u-%u.\n",
+ frg.ident, frgTig->id(), frg.position.bgn, frg.position.end);
- placeFragUsingOverlaps(frag, ovlStoreUniq, ovlStoreRept);
+ frgTig->addFrag(frg, 0, false);
}
-#endif
}
diff --git a/src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C b/src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C
index 11a4be3..a9011c7 100644
--- a/src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C
+++ b/src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_PlaceFragUsingOverlaps.C 4407 2013-09-09 02:32:10Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_PlaceFragUsingOverlaps.C 4578 2014-10-21 14:51:33Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_Unitig.H"
@@ -27,7 +27,7 @@ static const char *rcsid = "$Id: AS_BAT_PlaceFragUsingOverlaps.C 4407 2013-09-09
#include "AS_BAT_PlaceFragUsingOverlaps.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
// Report LOTS of details on placement, including evidence.
#undef VERBOSE_PLACEMENT
@@ -412,25 +412,27 @@ placeFragUsingOverlaps(UnitigVector &unitigs,
// picture above), not just the overlapping fragment sets (left or right in the above picture
// above).
//
- intervalList bgnPoints;
- intervalList endPoints;
+ intervalList<int32> bgnPoints;
+ intervalList<int32> endPoints;
- uint32 windowSlop = 0.075 * FI->fragmentLength(frag.ident);
+ int32 windowSlop = 0.075 * FI->fragmentLength(frag.ident);
if (windowSlop < 5)
windowSlop = 5;
for (uint32 oo=bgn; oo<end; oo++) {
- int32 b = ovlPlace[oo].position.bgn;
- int32 e = ovlPlace[oo].position.end;
-
assert(ovlPlace[oo].tigID > 0);
+ int32 b = ovlPlace[oo].position.bgn;
+ int32 be = ovlPlace[oo].position.bgn + windowSlop;
+ int32 e = ovlPlace[oo].position.end;
+ int32 ee = ovlPlace[oo].position.end + windowSlop;
+
b = (b < windowSlop) ? 0 : b - windowSlop;
e = (e < windowSlop) ? 0 : e - windowSlop;
- bgnPoints.add(b - windowSlop, 2 * windowSlop);
- endPoints.add(e - windowSlop, 2 * windowSlop);
+ bgnPoints.add(b, be - b);
+ endPoints.add(e, ee - e);
}
bgnPoints.merge();
@@ -450,24 +452,22 @@ placeFragUsingOverlaps(UnitigVector &unitigs,
for (uint32 oo=bgn; oo<end; oo++) {
int32 b = ovlPlace[oo].position.bgn;
int32 e = ovlPlace[oo].position.end;
-
int32 c = 0;
ovlPlace[oo].clusterID = 0;
for (int32 r=0; r<numBgnPoints; r++)
if ((bgnPoints.lo(r) <= b) && (b <= bgnPoints.hi(r))) {
- assert(ovlPlace[oo].clusterID == 0);
+ assert(ovlPlace[oo].clusterID == 0); // Obvious; we just set it to zero above.
ovlPlace[oo].clusterID = c = r * numEndPoints + 1;
}
for (int32 r=0; r<numEndPoints; r++)
if ((endPoints.lo(r) <= e) && (e <= endPoints.hi(r))) {
- assert(ovlPlace[oo].clusterID == c);
+ assert(ovlPlace[oo].clusterID == c); // Otherwise, bgn point wasn't placed in a cluster!
ovlPlace[oo].clusterID += r;
}
}
-
sort(ovlPlace + bgn, ovlPlace + end, overlapPlacement_byCluster);
diff --git a/src/AS_BAT/AS_BAT_bogusUtil.C b/src/AS_BAT/AS_BAT_bogusUtil.C
index 97c3534..b774acf 100644
--- a/src/AS_BAT/AS_BAT_bogusUtil.C
+++ b/src/AS_BAT/AS_BAT_bogusUtil.C
@@ -17,7 +17,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BAT_bogusUtil.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BAT_bogusUtil.C 4492 2014-01-23 16:32:56Z brianwalenz $";
#include "AS_BAT_bogusUtil.H"
@@ -98,26 +98,38 @@ loadNucmer(char *nucmerName,
if (errno)
fprintf(stderr, "Failed to open '%s' for reading: %s\n", nucmerName, strerror(errno)), exit(1);
- // First FIVE lines is a header
- fgets(inLine, 1024, inFile);
- fgets(inLine, 1024, inFile);
- fgets(inLine, 1024, inFile);
- fgets(inLine, 1024, inFile);
- fgets(inLine, 1024, inFile);
+ fprintf(stderr, "Loading alignments from '%s'\n", nucmerName);
+
+ // First FOURE lines are header
+ fgets(inLine, 1024, inFile); // (file paths)
+ fgets(inLine, 1024, inFile); // NUCMER
+ fgets(inLine, 1024, inFile); // (blank)
+ fgets(inLine, 1024, inFile); // (header)
+
+ // Scan the header line, counting the number of columns
+ uint32 nCols = 0;
+ uint32 wIdent = 0;
+
+ for (uint32 xx=0; inLine[xx]; xx++) {
+ if ((wIdent == 0) && (inLine[xx+0] == '[') && (inLine[xx+1] == '%') && (inLine[xx+2] == ' '))
+ wIdent = nCols;
+ if (inLine[xx] == '[')
+ nCols++;
+ }
// Read the first line.
fgets(inLine, 1024, inFile);
chomp(inLine);
- // If W[2][0] == '|' use the numbers below.
- // Otherwise, assume -T tab-delimited output which removes the |'s
-
while (!feof(inFile)) {
+ for (uint32 xx=0; inLine[xx]; xx++)
+ if (inLine[xx] == '|')
+ inLine[xx] = ' ';
+
splitToWords W(inLine);
genomeAlignment A;
- bool isTab = (W[2][0] != '|');
- string fID = W[(isTab) ? 8 : 12];
- string gID = W[(isTab) ? 7 : 11];
+ string gID = W[nCols - 1]; // TAGS is the last header column,
+ string fID = W[nCols - 0]; // but read ID is in column +1 from there.
if (IIDmap.find(fID) == IIDmap.end()) {
IIDname.push_back(fID);
@@ -127,30 +139,33 @@ loadNucmer(char *nucmerName,
// Unlike snapper, these are already in base-based coords.
A.frgIID = IIDmap[fID];
- A.frgBgn = W((isTab) ? 2 : 3);
- A.frgEnd = W((isTab) ? 3 : 4);
+ A.frgBgn = W(2);
+ A.frgEnd = W(3);
A.genIID = refMap[gID];
A.genBgn = W(0);
A.genEnd = W(1);
A.chnBgn = refList[A.genIID].rschnBgn + A.genBgn;
A.chnEnd = refList[A.genIID].rschnBgn + A.genEnd;
- A.identity = atof(W[(isTab) ? 6 : 9]);
+ A.identity = atof(W[wIdent]);
A.isReverse = false;
A.isSpanned = false;
A.isRepeat = true;
if (A.frgBgn > A.frgEnd) {
- A.frgBgn = W((isTab) ? 3 : 4);
- A.frgEnd = W((isTab) ? 2 : 3);
+ A.frgBgn = W(3);
+ A.frgEnd = W(2);
A.isReverse = true;
}
if ((A.frgBgn >= A.frgEnd) ||
(A.genBgn >= A.genEnd)) {
fprintf(stderr, "ERROR: %s\n", inLine);
- fprintf(stderr, "ERROR: frgBgn,frgEnd %u,%u genBgn,genEnd (%u,%u)\n",
- A.frgBgn, A.frgEnd,
- A.genBgn, A.genEnd);
+ if (A.frgBgn >= A.frgEnd)
+ fprintf(stderr, "ERROR: frgBgn,frgEnd = %u,%u\n",
+ A.frgBgn, A.frgEnd);
+ if (A.genBgn >= A.genEnd)
+ fprintf(stderr, "ERROR: genBgn,genEnd = %u,%u\n",
+ A.genBgn, A.genEnd);
}
assert(A.frgBgn < A.frgEnd);
assert(A.genBgn < A.genEnd);
diff --git a/src/AS_BAT/AS_BAT_bogusUtil.H b/src/AS_BAT/AS_BAT_bogusUtil.H
index 5ceeae5..e128743 100644
--- a/src/AS_BAT/AS_BAT_bogusUtil.H
+++ b/src/AS_BAT/AS_BAT_bogusUtil.H
@@ -20,11 +20,12 @@
#ifndef INCLUDE_AS_BAT_BOGUSUTIL
#define INCLUDE_AS_BAT_BOGUSUTIL
-static const char *rcsid_INCLUDE_AS_BAT_BOGUSUTIL = "$Id: AS_BAT_bogusUtil.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_INCLUDE_AS_BAT_BOGUSUTIL = "$Id: AS_BAT_bogusUtil.H 4571 2014-10-09 13:18:54Z brianwalenz $";
#include "AS_global.H"
-#include "AS_UTL_splitToWords.H"
-#include "AS_UTL_intervalList.H"
+
+#include "splitToWords.H"
+#include "intervalList.H"
#include <string>
#include <vector>
diff --git a/src/AS_BAT/AS_BAT_findEdges.C b/src/AS_BAT/AS_BAT_findEdges.C
new file mode 100644
index 0000000..266988f
--- /dev/null
+++ b/src/AS_BAT/AS_BAT_findEdges.C
@@ -0,0 +1,151 @@
+
+
+// Given two fragments that share at least one edge, this will find that edge and construct a new
+// edge to make it mutual.
+//
+// For example, if there is a best edge from aFrg 3' to bFrg 5', this will return that edge in a3,
+// and also create the symmetric edge in b5.
+//
+static
+bool
+findEdges(ufNode *aFrg, BestEdgeOverlap &a5, BestEdgeOverlap &a3,
+ ufNode *bFrg, BestEdgeOverlap &b5, BestEdgeOverlap &b3) {
+
+ if (OG->isContained(aFrg->ident) ||
+ OG->isContained(bFrg->ident))
+ return(false);
+
+ // Grab what edges we have.
+
+ a5 = *OG->getBestEdgeOverlap(aFrg->ident, false);
+ a3 = *OG->getBestEdgeOverlap(aFrg->ident, true);
+ b5 = *OG->getBestEdgeOverlap(bFrg->ident, false);
+ b3 = *OG->getBestEdgeOverlap(bFrg->ident, true);
+
+ // Erase things that aren't correct
+
+ if (a5.fragId() != bFrg->ident) a5 = BestEdgeOverlap();
+ if (a3.fragId() != bFrg->ident) a3 = BestEdgeOverlap();
+ if (b5.fragId() != aFrg->ident) b5 = BestEdgeOverlap();
+ if (b3.fragId() != aFrg->ident) b3 = BestEdgeOverlap();
+
+ // If we have no edges left, there are no edges!
+
+ if ((b5.fragId() != aFrg->ident) && (b3.fragId() != aFrg->ident) &&
+ (a5.fragId() != bFrg->ident) && (a3.fragId() != bFrg->ident))
+ return(false);
+
+ // If we found TWO edges for any single fragment....that's madness! That means the fragment
+ // had best dovetail overlaps to the same other fragment off of both ends. We'll complain
+ // and return failure. Ideally, data like this will be cleaned up by OBT, or filtered from
+ // our input.
+ //
+ if (a5.fragId() == a3.fragId()) {
+ writeLog("findEdges()-- frag %d has multiple edges to frag %d - a5 %d/%d' a3 %d/%d'\n",
+ aFrg->ident, a5.fragId(),
+ a5.fragId(), a5.frag3p() ? 3 : 5,
+ a5.fragId(), a5.frag3p() ? 3 : 5);
+ }
+
+ if (b5.fragId() == b3.fragId()) {
+ writeLog("findEdges()-- frag %d has multiple edges to frag %d - b5 %d/%d' b3 %d/%d'\n",
+ bFrg->ident, b5.fragId(),
+ b5.fragId(), b5.frag3p() ? 3 : 5,
+ b5.fragId(), b5.frag3p() ? 3 : 5);
+ }
+
+ if (((a5.fragId() != 0) && (a5.fragId() == a3.fragId())) ||
+ ((b5.fragId() != 0) && (b5.fragId() == b3.fragId()))) {
+ a5 = BestEdgeOverlap();
+ a3 = BestEdgeOverlap();
+ b5 = BestEdgeOverlap();
+ b3 = BestEdgeOverlap();
+ return(false);
+ }
+
+ // Now, populate the other edges using whatever we have. Best case is that we have two edges
+ // (because we're done).
+
+ assert(((a5.fragId() == bFrg->ident) +
+ (a3.fragId() == bFrg->ident) +
+ (b5.fragId() == aFrg->ident) +
+ (b3.fragId() == aFrg->ident)) <= 2);
+
+ if (((a5.fragId() == bFrg->ident) || (a3.fragId() == bFrg->ident)) &&
+ ((b5.fragId() == aFrg->ident) || (b3.fragId() == aFrg->ident)))
+ return(true);
+
+ // Otherwise, we have exactly one edge, and the other one needs to be created.
+
+ assert(((a5.fragId() == bFrg->ident) +
+ (a3.fragId() == bFrg->ident) +
+ (b5.fragId() == aFrg->ident) +
+ (b3.fragId() == aFrg->ident)) == 1);
+
+ if (a5.fragId() == bFrg->ident) {
+ //assert(a5.fragId() == 0);
+ assert(a3.fragId() == 0);
+ assert(b5.fragId() == 0);
+ assert(b3.fragId() == 0);
+
+ // Edge off of A's 5' end ('false' below)...
+ // ...to B's 3' end (so ANTI or NORMAL -- negate the hangs)
+ // ...to B's 5' end (so INNIE or OUTTIE -- swap the hangs)
+ if (a5.frag3p())
+ b3.set(aFrg->ident, false, -a5.ahang(), -a5.bhang());
+ else
+ b5.set(aFrg->ident, false, a5.bhang(), a5.ahang());
+
+ } else if (a3.fragId() == bFrg->ident) {
+ assert(a5.fragId() == 0);
+ //assert(a3.fragId() == 0);
+ assert(b5.fragId() == 0);
+ assert(b3.fragId() == 0);
+
+ // Edge off of A's 3' end ('true' below)...
+ // ...to B's 3' end (so INNIE or OUTTIE -- swap the hangs)
+ // ...to B's 5' end (so ANTI or NORMAL -- negate the hangs)
+ if (a3.frag3p())
+ b3.set(aFrg->ident, true, a3.bhang(), a3.ahang());
+ else
+ b5.set(aFrg->ident, true, -a3.ahang(), -a3.bhang());
+
+ } else if (b5.fragId() == aFrg->ident) {
+ assert(a5.fragId() == 0);
+ assert(a3.fragId() == 0);
+ //assert(b5.fragId() == 0);
+ assert(b3.fragId() == 0);
+
+ if (b5.frag3p())
+ a3.set(bFrg->ident, false, -b5.ahang(), -b5.bhang());
+ else
+ a5.set(bFrg->ident, false, b5.bhang(), b5.ahang());
+
+
+ } else if (b3.fragId() == aFrg->ident) {
+ assert(a5.fragId() == 0);
+ assert(a3.fragId() == 0);
+ assert(b5.fragId() == 0);
+ //assert(b3.fragId() == 0);
+
+ if (b3.frag3p())
+ a3.set(bFrg->ident, true, b3.bhang(), b3.ahang());
+ else
+ a5.set(bFrg->ident, true, -b3.ahang(), -b3.bhang());
+
+ } else {
+ fprintf(stderr, "findEdges()-- Logically impossible!\n");
+ assert(0);
+ }
+
+ // And now we should have exactly two edges.
+
+ assert(((a5.fragId() == bFrg->ident) +
+ (a3.fragId() == bFrg->ident) +
+ (b5.fragId() == aFrg->ident) +
+ (b3.fragId() == aFrg->ident)) == 2);
+
+ return(true);
+}
+
+
diff --git a/src/AS_BAT/Makefile b/src/AS_BAT/Makefile
index ce57c22..30ed7bb 100644
--- a/src/AS_BAT/Makefile
+++ b/src/AS_BAT/Makefile
@@ -66,52 +66,13 @@ DNC_SOURCES = classifyMates-BFS.C \
DNC_OBJECTS = $(DNC_SOURCES:.C=.o)
-SOURCES = bogart.C petey.C bogus.C bogusness.C AS_BAT_bogusUtil.C classifyMates.C classifyMatesApply.C classifyMatesPairwise.C splitUnitigs.C computeCoverageStat.C rewriteCache.C $(BAT_SOURCES) $(DNC_SOURCES)
-OBJECTS = bogart.o petey.o bogus.o bogusness.o AS_BAT_bogusUtil.o classifyMates.o classifyMatesApply.o classifyMatesPairwise.o splitUnitigs.o computeCoverageStat.o rewriteCache.o $(BAT_OBJECTS) $(DNC_OBJECTS)
+SOURCES = bogart.C bogus.C bogusness.C AS_BAT_bogusUtil.C classifyMates.C classifyMatesApply.C classifyMatesPairwise.C splitUnitigs.C computeCoverageStat.C markRepeatUnique.C rewriteCache.C erate-estimate.C $(BAT_SOURCES) $(DNC_SOURCES)
+OBJECTS = bogart.o bogus.o bogusness.o AS_BAT_bogusUtil.o classifyMates.o classifyMatesApply.o classifyMatesPairwise.o splitUnitigs.o computeCoverageStat.o markRepeatUnique.o rewriteCache.o erate-estimate.o $(BAT_OBJECTS) $(DNC_OBJECTS)
-CXX_PROGS = bogart bogus bogusness classifyMates classifyMatesApply classifyMatesPairwise splitUnitigs computeCoverageStat rewriteCache
+CXX_PROGS = bogart bogus bogusness classifyMates classifyMatesApply classifyMatesPairwise splitUnitigs computeCoverageStat markRepeatUnique rewriteCache erate-estimate
-#
-# BOILERPLATE
-#
-# We'd like to include the global include (c_make.as) to get these
-# definitions, but we cannot without defining rules.
-
-OSTYPE = $(shell echo `uname`)
-MACHINETYPE = $(shell echo `uname -m`)
-
-ifeq ($(MACHINETYPE), x86_64)
- MACHINETYPE = amd64
-endif
-ifeq ($(MACHINETYPE), Power Macintosh)
- MACHINETYPE = ppc
-endif
-
-
-# This isn't perfect; if we're building debug here, we _usually_ want
-# to use the debug kmer. But checking if kmer exists, and failing if
-# we don't find the debug kmer, is a little tricky.
-
-ifneq "$(origin KMER)" "environment"
-
-ifeq ($(shell ls -d $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug 2> /dev/null), $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug)
- KMER = $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug
-endif
-
-ifeq ($(shell ls -d $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE) 2> /dev/null), $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE))
- KMER = $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)
-endif
-
-endif
-#
-# BOILERPLATE
-#
-
-INC_IMPORT_DIRS += $(KMER)/include
-LIB_IMPORT_DIRS += $(KMER)/lib
-
# Include for AS project rules
include $(LOCAL_WORK)/src/c_make.as
@@ -119,7 +80,7 @@ all: $(OBJECTS) $(LIBRARIES) $(CXX_PROGS)
@test -n nop
bogart: bogart.o $(BAT_OBJECTS) libCA.a
-petey: petey.o $(BAT_OBJECTS) libCA.a
+#petey: petey.o $(BAT_OBJECTS) libCA.a
bogus: bogus.o libCA.a AS_BAT_bogusUtil.o
bogusness: bogusness.o libCA.a AS_BAT_bogusUtil.o
classifyMates: classifyMates.o $(DNC_OBJECTS) libCA.a libutil.a
@@ -127,4 +88,6 @@ classifyMatesApply: classifyMatesApply.o $(DNC_OBJECTS) libCA.a libutil.a
classifyMatesPairwise: classifyMatesPairwise.o $(DNC_OBJECTS) libCA.a libutil.a
splitUnitigs: splitUnitigs.o libCA.a
computeCoverageStat: computeCoverageStat.o libCA.a
-rewriteCache: rewriteCache.o AS_BAT_FragmentInfo.o AS_BAT_Logging.o libCA.a
\ No newline at end of file
+markRepeatUnique: markRepeatUnique.o libCA.a
+rewriteCache: rewriteCache.o AS_BAT_FragmentInfo.o AS_BAT_Logging.o libCA.a
+erate-estimate: erate-estimate.o libCA.a
diff --git a/src/AS_BAT/analyze-mapped-unitigs-for-joins.pl b/src/AS_BAT/analyze-mapped-unitigs-for-joins.pl
new file mode 100644
index 0000000..a6b0076
--- /dev/null
+++ b/src/AS_BAT/analyze-mapped-unitigs-for-joins.pl
@@ -0,0 +1,269 @@
+#!/usr/bin/perl
+
+# Given
+# an assembly (for the frgdeg and frgutg posmap)
+# nucmer mapping of unitigs to reference
+# blasr mapping of reads to reference ($alignedReads, probably in coords format)
+#
+# Report
+# unitigs that should be merged
+# unitigs that could be merged by a read (aligned read spans the gap)
+
+use strict;
+
+my $alignedReads = shift @ARGV;
+
+die "Can't find blasr-coords aligned reads in '$alignedReads'\n" if (! -e "$alignedReads");
+
+my $b1last = 0;
+my $e1last = 0;
+my $n1last = "";
+my $f1last = "fwd";
+
+my $b2last = 0;
+my $e2last = 0;
+my $n2last = "";
+my $f2last = "fwd";
+
+my $nBubble = 0;
+my $nSpan = 0;
+my $nGap = 0;
+my $nGapEv = 0;
+my $nAbut = 0;
+my $nOvl = 0;
+my $nOvlEv = 0;
+
+# Load the name to iid map.
+
+my %NAMtoIID;
+my %NAMtoUID;
+my %UIDtoIID;
+
+my %IIDtoDEG;
+my %IIDtoUTG;
+
+open(F, "< test.gkpStore.fastqUIDmap") or die "Failed to open 'test.gkpStore.fastqUIDmap'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ if (scalar(@v) == 3) {
+ $NAMtoIID{$v[2]} = $v[1];
+ $NAMtoUID{$v[2]} = $v[0];
+ $UIDtoIID{$v[0]} = $v[1];
+ } else {
+ $NAMtoIID{$v[2]} = $v[1];
+ $NAMtoUID{$v[2]} = $v[0];
+ $UIDtoIID{$v[0]} = $v[1];
+
+ $NAMtoIID{$v[5]} = $v[4];
+ $NAMtoUID{$v[5]} = $v[3];
+ $UIDtoIID{$v[3]} = $v[4];
+ }
+}
+close(F);
+
+print STDERR "Loaded ", scalar(keys %NAMtoIID), " read names.\n";
+
+open(F, "< 9-terminator/test.posmap.frgdeg") or die "Failed to open '9-terminator/test.posmap.frgdeg'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+ my $i = $UIDtoIID{$v[0]};
+
+ die "$_" if (!defined($i));
+
+ $IIDtoDEG{$i} = $v[1];
+}
+close(F);
+
+print STDERR "Loaded ", scalar(keys %IIDtoDEG), " degenerate unitig reads.\n";
+
+open(F, "< 9-terminator/test.posmap.frgutg") or die "Failed to open '9-terminator/test.posmap.frgutg'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+ my $i = $UIDtoIID{$v[0]};
+
+ die "$_" if (!defined($i));
+
+ $IIDtoUTG{$i} = $v[1];
+}
+close(F);
+
+print STDERR "Loaded ", scalar(keys %IIDtoUTG), " unitig reads.\n";
+
+
+# UTG.coords is standard nucmer output.
+
+open(F, "< 9-terminator/UTG.coords") or die "Failed to open '9-terminator/UTG.coords'\n";
+
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ my $b1 = $v[0]; # Assembly coords
+ my $e1 = $v[1];
+
+ my $b2 = $v[3]; # Read coords
+ my $e2 = $v[4];
+
+ my $f1 = ($b1 < $e1) ? "fwd" : "rev";
+ my $f2 = ($b2 < $e2) ? "fwd" : "rev";
+
+ ($b1,$e1) = ($e1,$b1) if ($b1 > $e1); # Doesn't seem to occur.
+ ($b2,$e2) = ($e2,$b2) if ($b2 > $e2); # Definitely does occur.
+
+ #my $l1 = $v[6];
+ #my $l2 = $v[7];
+
+ my $id = $v[9];
+
+ my $n1 = $v[11];
+ my $n2 = $v[12];
+
+ if (($b1last <= $b1) && ($e1 <= $e1last)) {
+ $nBubble++;
+ #print STDERR "BUBBLE $n2 ($b1,$e1) in $n2last ($b1last,$e1last)\n";
+ next;
+ }
+
+ $nSpan++;
+
+ goto bail if ($n1last ne $n1);
+
+ #print STDERR "last $b1last,$e1last curr $b1,$e1\n";
+
+ die "coords not sorted by increasing assembly start!\n" if ($b1last > $b1);
+
+ $nGap++ if ($e1last < $b1);
+ $nAbut++ if ($e1last == $b1);
+ $nOvl++ if ($e1last > $b1);
+
+
+ my $minOvlSpan = 150;
+
+
+ if ($e1last < $b1) {
+ my $ge = $b1;
+ my $gb = $e1last;
+ my $gap = $ge - $gb;
+ my $span = 0;
+
+ open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n";
+ while (<C>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ next if ($v[9] ne $n1);
+
+ # STOP changing the read name, blasr!
+
+ if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) {
+ $v[10] = $1;
+ }
+
+ my $iid = $NAMtoIID{$v[10]};
+ my $utg = $IIDtoUTG{$iid};
+ my $deg = $IIDtoDEG{$iid};
+ my $ann;
+
+ #next if (defined($utg) || defined($deg));
+
+ if (defined($utg)) {
+ $ann = "utg $utg";
+ } elsif (defined($deg)) {
+ $ann = "deg $deg";
+ } else {
+ $ann = "singleton ";
+ }
+
+ if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) {
+ #print "$ann iid $iid -- $_\n";
+ $span++;
+ }
+ }
+ close(C);
+
+ if ($span > 0) {
+ $nGapEv++;
+ print "GAP $n1 $n2last ($b1last,$e1last,$f2last) -- $gap -- $n2 ($b1,$e1,$f2) -- SPAN $span\n";
+ print "\n";
+ print "\n";
+ }
+ }
+
+
+ if ($e1last == $b1) {
+ }
+
+
+ if ($e1last > $b1) {
+ my $ge = $e1last;
+ my $gb = $b1;
+ my $ovl = $ge - $gb;
+ my $span = 0;
+
+ open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n";
+ while (<C>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ next if ($v[9] ne $n1);
+
+ if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) {
+ $v[10] = $1;
+ }
+
+ my $iid = $NAMtoIID{$v[10]};
+ my $utg = $IIDtoUTG{$iid};
+ my $deg = $IIDtoDEG{$iid};
+ my $ann;
+
+ #next if (defined($utg) || defined($deg));
+
+ if (defined($utg)) {
+ $ann = "utg $utg";
+ } elsif (defined($deg)) {
+ $ann = "deg $deg";
+ } else {
+ $ann = "singleton ";
+ }
+
+ if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) {
+ print "$ann iid $iid -- $_\n";
+ $span++;
+ }
+ }
+ close(C);
+
+ if ($span > 0) {
+ $nOvlEv++;
+ print "OVL $n1 $n2last ($b1last,$e1last,$f2last) -- $ovl -- $n2 ($b1,$e1,$f2) -- SPAN $span\n";
+ print "\n";
+ print "\n";
+ }
+ }
+
+
+ bail:
+ ($b1last,$e1last,$n1last,$f1last) = ($b1,$e1,$n1,$f1);
+ ($b2last,$e2last,$n2last,$f2last) = ($b2,$e2,$n2,$f2);
+}
+close(F);
+
+print STDERR "nBubble $nBubble\n";
+print STDERR "nSpan $nSpan\n";
+
+print STDERR "nGap $nGap\n";
+print STDERR "nGap $nGapEv with evidence\n";
+print STDERR "nAbut $nAbut\n";
+print STDERR "nOvl $nOvl\n";
+print STDERR "nOvl $nOvlEv with evidence\n";
diff --git a/src/AS_BAT/analyze-nucmer-gaps.pl b/src/AS_BAT/analyze-nucmer-gaps.pl
new file mode 100644
index 0000000..1758664
--- /dev/null
+++ b/src/AS_BAT/analyze-nucmer-gaps.pl
@@ -0,0 +1,260 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $alignedReads = shift @ARGV;
+
+die "Can't find blasr-coords aligned reads in '$alignedReads'\n" if (! -e "$alignedReads");
+
+my $b1last = 0;
+my $e1last = 0;
+my $n1last = "";
+my $f1last = "fwd";
+
+my $b2last = 0;
+my $e2last = 0;
+my $n2last = "";
+my $f2last = "fwd";
+
+my $nBubble = 0;
+my $nSpan = 0;
+my $nGap = 0;
+my $nGapEv = 0;
+my $nAbut = 0;
+my $nOvl = 0;
+my $nOvlEv = 0;
+
+# Load the name to iid map.
+
+my %NAMtoIID;
+my %NAMtoUID;
+my %UIDtoIID;
+
+my %IIDtoDEG;
+my %IIDtoUTG;
+
+open(F, "< test.gkpStore.fastqUIDmap") or die "Failed to open 'test.gkpStore.fastqUIDmap'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ if (scalar(@v) == 3) {
+ $NAMtoIID{$v[2]} = $v[1];
+ $NAMtoUID{$v[2]} = $v[0];
+ $UIDtoIID{$v[0]} = $v[1];
+ } else {
+ $NAMtoIID{$v[2]} = $v[1];
+ $NAMtoUID{$v[2]} = $v[0];
+ $UIDtoIID{$v[0]} = $v[1];
+
+ $NAMtoIID{$v[5]} = $v[4];
+ $NAMtoUID{$v[5]} = $v[3];
+ $UIDtoIID{$v[3]} = $v[4];
+ }
+}
+close(F);
+
+print STDERR "Loaded ", scalar(keys %NAMtoIID), " read names.\n";
+
+open(F, "< 9-terminator/test.posmap.frgdeg") or die "Failed to open '9-terminator/test.posmap.frgdeg'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+ my $i = $UIDtoIID{$v[0]};
+
+ die "$_" if (!defined($i));
+
+ $IIDtoDEG{$i} = $v[1];
+}
+close(F);
+
+print STDERR "Loaded ", scalar(keys %IIDtoDEG), " degenerate unitig reads.\n";
+
+open(F, "< 9-terminator/test.posmap.frgutg") or die "Failed to open '9-terminator/test.posmap.frgdeg'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+ my $i = $UIDtoIID{$v[0]};
+
+ die "$_" if (!defined($i));
+
+ $IIDtoUTG{$i} = $v[1];
+}
+close(F);
+
+print STDERR "Loaded ", scalar(keys %IIDtoUTG), " unitig reads.\n";
+
+
+# UTG.coords is standard nucmer output.
+
+open(F, "< 9-terminator/UTG.coords") or die "Failed to open '9-terminator/UTG.coords'\n";
+
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ my $b1 = $v[0]; # Assembly coords
+ my $e1 = $v[1];
+
+ my $b2 = $v[3]; # Read coords
+ my $e2 = $v[4];
+
+ my $f1 = ($b1 < $e1) ? "fwd" : "rev";
+ my $f2 = ($b2 < $e2) ? "fwd" : "rev";
+
+ ($b1,$e1) = ($e1,$b1) if ($b1 > $e1); # Doesn't seem to occur.
+ ($b2,$e2) = ($e2,$b2) if ($b2 > $e2); # Definitely does occur.
+
+ #my $l1 = $v[6];
+ #my $l2 = $v[7];
+
+ my $id = $v[9];
+
+ my $n1 = $v[11];
+ my $n2 = $v[12];
+
+ if (($b1last <= $b1) && ($e1 <= $e1last)) {
+ $nBubble++;
+ #print STDERR "BUBBLE $n2 ($b1,$e1) in $n2last ($b1last,$e1last)\n";
+ next;
+ }
+
+ $nSpan++;
+
+ goto bail if ($n1last ne $n1);
+
+ #print STDERR "last $b1last,$e1last curr $b1,$e1\n";
+
+ die "coords not sorted by increasing assembly start!\n" if ($b1last > $b1);
+
+ $nGap++ if ($e1last < $b1);
+ $nAbut++ if ($e1last == $b1);
+ $nOvl++ if ($e1last > $b1);
+
+
+ my $minOvlSpan = 150;
+
+
+ if ($e1last < $b1) {
+ my $ge = $b1;
+ my $gb = $e1last;
+ my $gap = $ge - $gb;
+ my $span = 0;
+
+ open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n";
+ while (<C>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ next if ($v[9] ne $n1);
+
+ # STOP changing the read name, blasr!
+
+ if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) {
+ $v[10] = $1;
+ }
+
+ my $iid = $NAMtoIID{$v[10]};
+ my $utg = $IIDtoUTG{$iid};
+ my $deg = $IIDtoDEG{$iid};
+ my $ann;
+
+ #next if (defined($utg) || defined($deg));
+
+ if (defined($utg)) {
+ $ann = "utg $utg";
+ } elsif (defined($deg)) {
+ $ann = "deg $deg";
+ } else {
+ $ann = "singleton ";
+ }
+
+ if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) {
+ #print "$ann iid $iid -- $_\n";
+ $span++;
+ }
+ }
+ close(C);
+
+ if ($span > 0) {
+ $nGapEv++;
+ print "GAP $n1 $n2last ($b1last,$e1last,$f2last) -- $gap -- $n2 ($b1,$e1,$f2) -- SPAN $span\n";
+ print "\n";
+ print "\n";
+ }
+ }
+
+
+ if ($e1last == $b1) {
+ }
+
+
+ if ($e1last > $b1) {
+ my $ge = $e1last;
+ my $gb = $b1;
+ my $ovl = $ge - $gb;
+ my $span = 0;
+
+ open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n";
+ while (<C>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ next if ($v[9] ne $n1);
+
+ if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) {
+ $v[10] = $1;
+ }
+
+ my $iid = $NAMtoIID{$v[10]};
+ my $utg = $IIDtoUTG{$iid};
+ my $deg = $IIDtoDEG{$iid};
+ my $ann;
+
+ #next if (defined($utg) || defined($deg));
+
+ if (defined($utg)) {
+ $ann = "utg $utg";
+ } elsif (defined($deg)) {
+ $ann = "deg $deg";
+ } else {
+ $ann = "singleton ";
+ }
+
+ if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) {
+ print "$ann iid $iid -- $_\n";
+ $span++;
+ }
+ }
+ close(C);
+
+ if ($span > 0) {
+ $nOvlEv++;
+ print "OVL $n1 $n2last ($b1last,$e1last,$f2last) -- $ovl -- $n2 ($b1,$e1,$f2) -- SPAN $span\n";
+ print "\n";
+ print "\n";
+ }
+ }
+
+
+ bail:
+ ($b1last,$e1last,$n1last,$f1last) = ($b1,$e1,$n1,$f1);
+ ($b2last,$e2last,$n2last,$f2last) = ($b2,$e2,$n2,$f2);
+}
+close(F);
+
+print STDERR "nBubble $nBubble\n";
+print STDERR "nSpan $nSpan\n";
+
+print STDERR "nGap $nGap\n";
+print STDERR "nGap $nGapEv with evidence\n";
+print STDERR "nAbut $nAbut\n";
+print STDERR "nOvl $nOvl\n";
+print STDERR "nOvl $nOvlEv with evidence\n";
diff --git a/src/AS_BAT/bogart.C b/src/AS_BAT/bogart.C
index 075dd88..cd7d005 100644
--- a/src/AS_BAT/bogart.C
+++ b/src/AS_BAT/bogart.C
@@ -17,7 +17,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: bogart.C 4450 2013-10-14 22:32:40Z brianwalenz $";
+const char *mainid = "$Id: bogart.C 4580 2014-10-21 14:53:21Z brianwalenz $";
#include "AS_BAT_Datatypes.H"
#include "AS_BAT_BestOverlapGraph.H"
@@ -79,9 +79,13 @@ main (int argc, char * argv []) {
char *output_prefix = NULL;
bool removeSpur = false;
+ double removeWeak = 0.0;
bool removeSuspicious = false;
+ bool noContainsInSingletons = false;
bool enableJoining = false;
+ bool placeContainsUsingBest = true;
+
bool enableShatterRepeats = false;
bool enableExtendByMates = false;
bool enableReconstructRepeats = false;
@@ -114,9 +118,15 @@ main (int argc, char * argv []) {
} else if (strcmp(argv[arg], "-RS") == 0) {
removeSpur = true;
+ } else if (strcmp(argv[arg], "-RW") == 0) {
+ removeWeak = atof(argv[++arg]);
+
} else if (strcmp(argv[arg], "-NS") == 0) {
removeSuspicious = true;
+ } else if (strcmp(argv[arg], "-CS") == 0) {
+ noContainsInSingletons = true;
+
} else if (strcmp(argv[arg], "-J") == 0) {
enableJoining = true;
@@ -267,6 +277,8 @@ main (int argc, char * argv []) {
fprintf(stderr, "\n");
fprintf(stderr, " -RS Remove edges to spur reads from best overlap graph.\n");
fprintf(stderr, " -NS Don't seed promiscuous unitigs with suspicious reads.\n");
+ fprintf(stderr, " -CS Don't place contained reads in singleton unitigs.\n");
+ fprintf(stderr, " -RW t Remove weak overlaps, those in the lower t fraction of erates per overlap end.\n");
fprintf(stderr, " -J Join promiscuous unitigs using unused best edges.\n");
fprintf(stderr, " -SR Shatter repeats. Enabled with -R and -E; if neither are supplied,\n");
fprintf(stderr, " repeat fragments are promoted to singleton unitigs (unless -DP).\n");
@@ -383,7 +395,7 @@ main (int argc, char * argv []) {
Unitig::resetFragUnitigMap(FI->numFragments());
OC = new OverlapCache(ovlStoreUniq, ovlStoreRept, output_prefix, MAX(erateGraph, erateMerge), MAX(elimitGraph, elimitMerge), ovlCacheMemory, ovlCacheLimit, onlySave, doSave);
- OG = new BestOverlapGraph(erateGraph, elimitGraph, output_prefix, removeSuspicious, removeSpur);
+ OG = new BestOverlapGraph(erateGraph, elimitGraph, output_prefix, removeWeak, removeSuspicious, removeSpur);
CG = new ChunkGraph(output_prefix);
IS = NULL;
@@ -431,9 +443,19 @@ main (int argc, char * argv []) {
evaluateMates(unitigs, output_prefix, "joining");
}
- placeContainsUsingBestOverlaps(unitigs);
- //placeContainsUsingAllOverlaps(bool withMatesToNonContained,
- // bool withMatesToUnambiguousContain);
+ if (noContainsInSingletons)
+ OG->rebuildBestContainsWithoutSingletons(unitigs, erateGraph, elimitGraph, output_prefix);
+
+ if (placeContainsUsingBest) {
+ placeContainsUsingBestOverlaps(unitigs);
+
+ } else {
+ bool withMatesToNonContained = false; // Resolve ambiguous contained placements using mates to dovetail reads
+ bool withMatesToUnambiguousContain = false; // Resolve ambiguous contained placements using mates
+
+ assert(0); // Doesn't work
+ placeContainsUsingAllOverlaps(unitigs, withMatesToNonContained, withMatesToUnambiguousContain);
+ }
setLogFile(output_prefix, "placeZombies");
@@ -475,7 +497,17 @@ main (int argc, char * argv []) {
setLogFile(output_prefix, "cleanup");
splitDiscontinuousUnitigs(unitigs); // Clean up splitting problems.
- placeContainsUsingBestOverlaps(unitigs);
+
+ if (placeContainsUsingBest) {
+ placeContainsUsingBestOverlaps(unitigs);
+
+ } else {
+ bool withMatesToNonContained = false; // Resolve ambiguous contained placements using mates to dovetail reads
+ bool withMatesToUnambiguousContain = false; // Resolve ambiguous contained placements using mates
+
+ assert(0); // Doesn't work
+ placeContainsUsingAllOverlaps(unitigs, withMatesToNonContained, withMatesToUnambiguousContain);
+ }
promoteToSingleton(unitigs, enablePromoteToSingleton);
diff --git a/src/AS_BAT/bogus.C b/src/AS_BAT/bogus.C
index a2e0061..ca9e80a 100644
--- a/src/AS_BAT/bogus.C
+++ b/src/AS_BAT/bogus.C
@@ -17,7 +17,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: bogus.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: bogus.C 4572 2014-10-14 23:06:05Z brianwalenz $";
#include "AS_BAT_bogusUtil.H"
@@ -51,8 +51,8 @@ map<string, int32> IIDmap; // Maps an ID string to an IID.
vector<string> IIDname; // Maps an IID to an ID string.
vector<uint32> IIDcount; // Maps an IID to the number of alignments
-intervalList REPT;
-intervalList UNIQ;
+intervalList<int32> REPT;
+intervalList<int32> UNIQ;
bool *REPTvalid = NULL;
bool *UNIQvalid = NULL;
@@ -404,8 +404,8 @@ markWeak(void) {
if ((REPT.lo(ir) <= UNIQ.lo(iu)) &&
(UNIQ.hi(iu) <= REPT.hi(ir))) {
//fprintf(stderr, "EXCEPTION: UNIQ %ld,%ld len=%ld ct=%d contained in REPT %ld,%ld len=%ld ct=%d\n",
- // UNIQ.lo(iu), UNIQ.hi(iu), UNIQ.hi(iu) - UNIQ.lo(iu), UNIQ.ct(iu),
- // REPT.lo(ir), REPT.hi(ir), REPT.hi(ir) - REPT.lo(ir), REPT.ct(ir));
+ // UNIQ.lo(iu), UNIQ.hi(iu), UNIQ.hi(iu) - UNIQ.lo(iu), UNIQ.count(iu),
+ // REPT.lo(ir), REPT.hi(ir), REPT.hi(ir) - REPT.lo(ir), REPT.count(ir));
UNIQvalid[iu] = false;
UNIQvalidParent[iu] = ir;
UNIQexceptions++;
@@ -413,8 +413,8 @@ markWeak(void) {
if ((UNIQ.lo(iu) <= REPT.lo(ir)) &&
(REPT.hi(ir) <= UNIQ.hi(iu))) {
//fprintf(stderr, "EXCEPTION: REPT %ld,%ld len=%ld ct=%d contained in UNIQ %ld,%ld len=%ld ct=%d\n",
- // REPT.lo(ir), REPT.hi(ir), REPT.hi(ir) - REPT.lo(ir), REPT.ct(ir),
- // UNIQ.lo(iu), UNIQ.hi(iu), UNIQ.hi(iu) - UNIQ.lo(iu), UNIQ.ct(iu));
+ // REPT.lo(ir), REPT.hi(ir), REPT.hi(ir) - REPT.lo(ir), REPT.count(ir),
+ // UNIQ.lo(iu), UNIQ.hi(iu), UNIQ.hi(iu) - UNIQ.lo(iu), UNIQ.count(iu));
REPTvalid[ir] = false;
REPTvalidParent[ir] = iu;
REPTexceptions++;
@@ -612,7 +612,7 @@ main(int argc, char **argv) {
refhdr = refList[rr].rsrefName;
refbgn = REPT.lo(ir) - refList[rr].rschnBgn;
refend = REPT.hi(ir) - refList[rr].rschnBgn;
- refcnt = REPT.ct(ir);
+ refcnt = REPT.count(ir);
}
if (refcnt == 0) {
@@ -648,7 +648,7 @@ main(int argc, char **argv) {
refhdr = refList[rr].rsrefName;
refbgn = UNIQ.lo(iu) - fragTrim - refList[rr].rschnBgn;
refend = UNIQ.hi(iu) + fragTrim - refList[rr].rschnBgn;
- refcnt = UNIQ.ct(iu);
+ refcnt = UNIQ.count(iu);
}
// Not sure why some data sets (long pacbio for example) trigger this.
diff --git a/src/AS_BAT/classifyMates.C b/src/AS_BAT/classifyMates.C
index 4a23ba8..69b97cd 100644
--- a/src/AS_BAT/classifyMates.C
+++ b/src/AS_BAT/classifyMates.C
@@ -17,7 +17,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: classifyMates.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: classifyMates.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include "AS_global.H"
#include "AS_UTL_decodeRange.H"
@@ -368,7 +368,7 @@ main(int argc, char **argv) {
ss->setNumberOfWorkers(numThreads);
- for (u32bit w=0; w<numThreads; w++)
+ for (uint32 w=0; w<numThreads; w++)
ss->setThreadData(w, new cmThreadData()); // these leak
ss->run(g, beVerbose);
diff --git a/src/AS_BAT/count-bubbles.pl b/src/AS_BAT/count-bubbles.pl
new file mode 100644
index 0000000..aea0ade
--- /dev/null
+++ b/src/AS_BAT/count-bubbles.pl
@@ -0,0 +1,204 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $FILE = "test.004.buildUnitigs";
+$FILE = shift @ARGV if (scalar(@ARGV) > 0);
+
+if ((! -e "$FILE.tigStore") ||
+ (! -e "$FILE.fasta")) {
+ die "Missing tigStore or fasta. Run build-fasta.pl\n";
+}
+
+########################################
+
+print STDERR "Aligning unitigs to unitigs\n";
+if (! -e "$FILE.allVall.sim4db" ) {
+ my $cmd;
+
+ $cmd .= "/work/kmer/FreeBSD-amd64/bin/snapper2 ";
+ $cmd .= " -queries $FILE.fasta ";
+ $cmd .= " -genomic $FILE.fasta ";
+ $cmd .= " -mersize 16 ";
+ $cmd .= " -minmatchidentity 94 ";
+ $cmd .= " -minmatchcoverage 94 ";
+ $cmd .= " -aligns ";
+ $cmd .= " -verbose > $FILE.allVall.sim4db";
+
+ system($cmd);
+}
+
+print STDERR "Filtering alignments\n";
+if (! -e "$FILE.bubbles.sim4db") {
+ my $cmd;
+
+ $cmd .= "/work/kmer/FreeBSD-amd64/bin/filterPolishes ";
+ $cmd .= " -selfhits ";
+ $cmd .= " -D ";
+ $cmd .= " < $FILE.allVall.sim4db ";
+ $cmd .= " > $FILE.bubbles.sim4db";
+
+ system($cmd);
+}
+
+########################################
+
+my %detected1;
+my %detected2;
+
+my $utg;
+my $ref;
+my $ide;
+my $cov;
+
+print STDERR "Reading bubble mapping\n";
+open(F, "< $FILE.bubbles.sim4db") or die;
+while (<F>) {
+ chomp;
+
+ if (m/^edef=utg(\d+)/) {
+ $utg = $1;
+ }
+ if (m/^ddef=utg(\d+)/) {
+ $ref = $1;
+ }
+ if (m/^\d+\[(\d+)-\d+-\d+\]\s\d+\[\d+-\d+\]\s+<(\d+)-\d+-(\d+)-\w+-\w+>$/) {
+ $ide = $3;
+ $cov = $2;
+ }
+ if (m/^(\d+)-(\d+)\s\((\d+)-(\d+)\)\s/) {
+ $cov = 100 * $cov / ($2 - $1);
+ }
+ if (m/^sim4end$/) {
+ die if (!defined($utg));
+ die if (!defined($ref));
+
+ if (($ide >= 96) && ($cov >= 98)) {
+ $detected1{"$utg"}++;
+ $detected2{"$utg-$ref"}++;
+ }
+
+ #if ($cov < 100) {
+ # print STDERR "LOW COV $utg - $ref -- $cov\n";
+ #}
+
+ undef $utg;
+ undef $ref;
+ }
+}
+close(F);
+
+print STDERR "Found ", scalar(keys %detected2), " actual bubble instances, from ", scalar(keys %detected1), " unitigs.\n";
+
+my $numUnique = 0;
+
+foreach my $k (keys %detected1) {
+ if ($detected1{$k} == 1) {
+ #print STDERR "UNIQUE $k\n";
+ $numUnique++;
+ }
+}
+
+print STDERR "Found $numUnique uniquely placeable bubbles.\n";
+
+#
+# Scan the log. Discover if we successfully merged a bubble. Report bubbles that we merged that
+# were not verified by mapping. Remove bubbles we are successful on from the list of alignments
+# (so we can next report what we failed on).
+#
+
+my $numMerged = 0;
+my $numMergedUnique = 0;
+my $numMergedMultiple = 0;
+my $numMergedExtra = 0;
+my $numMergedExtraMultiple = 0;
+
+open(F, "< test.005.bubblePopping.log") or die;
+while (<F>) {
+ chomp;
+
+ if (m/merged\sbubble\sunitig\s(\d+)\sinto\sunitig\s(\d+)$/) {
+ $numMerged++;
+
+ if (!exists($detected1{"$1"})) {
+ #print STDERR "EXTRA $_ (no mapping at all)\n";
+ $numMergedExtra++;
+
+ } elsif (!exists($detected2{"$1-$2"})) {
+ #print STDERR "EXTRA $_ (no mapping for this pair out of $detected1{\"$1\"} alignments - possibly incorrectly placed!)\n";
+ $numMergedExtraMultiple++;
+
+ } elsif ($detected1{"$1"} == 1) {
+ $numMergedUnique++;
+
+ } elsif ($detected1{"$1"} > 1) {
+ #print STDERR "MULTI $_ ($detected1{\"$1\"} alignments - possibly incorrectly placed!)\n";
+ $numMergedMultiple++;
+
+ } else {
+ die;
+ }
+
+ delete $detected2{"$1-$2"};
+ delete $detected1{"$1"};
+ }
+
+}
+close(F);
+
+print STDERR "Merged $numMerged bubbles:\n";
+print STDERR " $numMergedUnique were uniquely alignable.\n";
+print STDERR " $numMergedMultiple were multiply alignable.\n";
+print STDERR " $numMergedExtra were not alignable at all.\n";
+print STDERR " $numMergedExtraMultiple were not alignable at the spot it was merged.\n";
+
+#
+# Scan the layouts. For each un-popped bubble (BOG didn't pop, but it aligned to a unitig) count
+# the number of fragments in the unitig, and length of the tig. Report that and the number of
+# places the bubble mapped.
+#
+
+my $numMergedMissed = 0;
+my $numMergedMissedUnique = 0;
+
+my $id = 0; # Unitig ID we're looking at
+my $nf = 0; # Number of fragments
+my $ln = 0; # Length of the unitig
+
+open(F, "< $FILE.layout");
+open(O, "> $FILE.bubblesMISSED");
+while (<F>) {
+ chomp;
+
+ if (m/^unitig\s+(\d+)$/) {
+ if ($nf > 0) {
+ $numMergedMissed++ if ($detected1{$id} > 0);
+ $numMergedMissedUnique++ if ($detected1{$id} == 1);
+ print O "Missed merge unitig $id of length $ln with $nf frags -- mapped to $detected1{$id} places.\n";
+ }
+
+ $id = $1;
+ $nf = 0;
+ $ln = 0;
+ }
+
+ if (($detected1{"$id"} > 0) &&
+ (m/^FRG.*position\s+(\d+)\s+(\d+)$/)) {
+ $nf++;
+
+ $ln = $1 if ($ln < $1);
+ $ln = $2 if ($ln < $2);
+ }
+}
+
+close(F);
+
+if ($nf > 0) {
+ $numMergedMissed++ if ($detected1{$id} > 0);
+ $numMergedMissedUnique++ if ($detected1{$id} == 1);
+ print O "Missed merge unitig $id of length $ln with $nf frags -- mapped to $detected1{$id} places.\n";
+}
+close(O);
+
+print STDERR "Failed to merge $numMergedMissed unitigs.\n";
+print STDERR " $numMergedMissedUnique were uniquely placeable.\n";
diff --git a/src/AS_BAT/erate-estimate-plot-per-base-estimate.pl b/src/AS_BAT/erate-estimate-plot-per-base-estimate.pl
new file mode 100644
index 0000000..1c046ec
--- /dev/null
+++ b/src/AS_BAT/erate-estimate-plot-per-base-estimate.pl
@@ -0,0 +1,113 @@
+#!/usr/bin/perl
+
+# Reads
+# overlapStore picture dump
+#
+# Generates a plot of min/ave/max overlap error rate per base in the read.
+
+use strict;
+
+# 279 280
+# 281 284 285 interesting
+
+my @iid; #= ( 281, 284, 285, 3360, 4206, 4944, 5118 );
+
+push @iid, 2276; # normal
+push @iid, 281;
+push @iid, 284;
+push @iid, 285;
+push @iid, 3360;
+push @iid, 4206;
+push @iid, 4944;
+push @iid, 5118;
+
+foreach my $iid (@iid) {
+ my $prefix = "iid$iid";
+ my @erates;
+
+ open(F, "overlapStore -p $iid test.ovlStore test.gkpStore CLR |");
+
+ open(D, "> $prefix.stats.dat");
+ open(E, "> $prefix.erate.dat");
+
+ # Ignore the header.
+ $_ = <F>;
+
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ #my @v = split '\s+', $_;
+
+ #my $biid = $v[0];
+ #my $abgn = $v[2];
+ #my $aend = $v[3];
+
+ my ($biid, $abgn, $aend, $erate);
+
+ if (m/(\d+)\s+A:\s+(\d+)\s+(\d+)\s+\(\s*\d+\)\s+B:\s+\d+\s+\d+\s+\(\s*\d+\)\s+(\d+.\d+)%/) {
+ $biid = $1;
+ $abgn = $2;
+ $aend = $3;
+ $erate = $4;
+ } else {
+ print "NOPE $_\n";
+ }
+
+ for (my $ii=$abgn; $ii<=$aend; $ii++) {
+ if (!defined($erates[$ii])) {
+ $erates[$ii] = "$erate";
+ } else {
+ $erates[$ii] .= ":$erate";
+ }
+ }
+
+ #print E "$abgn\t$aend\t$erate\n";
+ }
+
+ close(F);
+
+
+ my $maxPoint = scalar(@erates);
+
+ for (my $pp=0; $pp<$maxPoint; $pp++) {
+ my @vals = split ':', $erates[$pp];
+ my $depth = scalar (@vals);
+
+ @vals = sort {$a <=> $b } @vals;
+
+ my $min = $vals[0];
+ my $max = $vals[$depth-1];
+
+ my $sum = 0;
+ foreach my $v (@vals) {
+ $sum += $v;
+ }
+
+ my $ave = $sum / $depth;
+
+ print D "$pp\t$min\t$ave\t$max\t$depth\n";
+
+ foreach my $v (@vals) {
+ print E "$pp\t$v\n";
+ }
+ }
+
+ close(D);
+ close(E);
+
+ system("overlapStore -p $iid test.ovlStore test.gkpStore CLR > $prefix.ovlPicture");
+
+ open(F, "| gnuplot");
+ print F "set terminal 'png' size 1280,800\n";
+ print F "set output '$prefix.png'\n";
+ print F "plot [][0:30]";
+ print F " '$prefix.stats.dat' using 1:2 with lines title 'min % identity',";
+ print F " '$prefix.stats.dat' using 1:3 with lines title 'ave % identity',";
+ print F " '$prefix.stats.dat' using 1:4 with lines title 'max % identity',";
+ print F " '$prefix.stats.dat' using 1:5 with lines title 'depth',";
+ #print F " '$prefix.erate.dat' using 1:2 with points ps 0.2 title 'erate'\n";
+ print F " '$prefix.erate.dat' using 1:3 with points ps 0.3 title 'bgnerates',";
+ print F " '$prefix.erate.dat' using 2:3 with points ps 0.3 title 'enderates'\n";
+ close(F);
+}
diff --git a/src/AS_BAT/erate-estimate-test-based-on-mapping.pl b/src/AS_BAT/erate-estimate-test-based-on-mapping.pl
new file mode 100644
index 0000000..40f18ec
--- /dev/null
+++ b/src/AS_BAT/erate-estimate-test-based-on-mapping.pl
@@ -0,0 +1,225 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $erateFile = "BO/test.blasr.sam.coords.longestErate";
+my $uidMapFile = "CAordered/test.gkpStore.fastqUIDmap";
+my $olapStore = "CAordered/test.ovlStore";
+
+my %erate5; # Either the mapped global erate, or the lowest erate for each end
+my %erate3;
+
+
+# Load the map from IID to UID and name.
+
+my %UIDtoIID;
+my %UIDtoNAM;
+
+my %IIDtoUID;
+my %IIDtoNAM;
+
+my %NAMtoUID;
+my %NAMtoIID;
+
+
+if (defined($uidMapFile)) {
+ print STDERR "Read names from '$uidMapFile'\n";
+
+ open(F, "< $uidMapFile");
+ while (<F>) {
+ my @v = split '\s+', $_;
+
+ # UID IID NAM UID IID NAM
+
+ $UIDtoIID{$v[0]} = $v[1];
+ $UIDtoNAM{$v[0]} = $v[2];
+
+ $IIDtoUID{$v[1]} = $v[0];
+ $IIDtoNAM{$v[1]} = $v[2];
+
+ $NAMtoUID{$v[2]} = $v[0];
+ $NAMtoIID{$v[2]} = $v[1];
+
+ if (scalar(@v) == 6) {
+ $UIDtoIID{$v[3]} = $v[4];
+ $UIDtoNAM{$v[3]} = $v[5];
+
+ $IIDtoUID{$v[4]} = $v[3];
+ $IIDtoNAM{$v[4]} = $v[5];
+
+ $NAMtoUID{$v[5]} = $v[3];
+ $NAMtoIID{$v[5]} = $v[4];
+ }
+ }
+ close(F);
+}
+
+
+
+if (0) {
+print STDERR "Load global read erates from '$erateFile'\n";
+
+my $found = 0;
+my $lost = 0;
+
+if (! -e "$erateFile") {
+ die "Run erate-coords.pl in BO directory.\n";
+}
+
+open(F, "< $erateFile") or die;
+while(<F>) {
+ my @v = split '\s+', $_;
+
+ # This should be read NAME, not assmelby UID.
+ my $iid = $NAMtoIID{$v[0]};
+
+ if (defined($iid)) {
+ $found++;
+ $erate5{$iid} = 100 - $v[1];
+ $erate3{$iid} = 100 - $v[1];
+ } else {
+ $lost++;
+ #print STDERR "Didn't find name '$v[0]' - dropped from read set?\n";
+ }
+}
+close(F);
+
+print STDERR "Found $found erates, lost $lost.\n";
+}
+
+
+
+
+if (1) {
+print STDERR "Load overlap erates from '$olapStore'\n";
+
+open(F, "overlapStore -d $olapStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ #print "$_\n";
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $erate, $crate) = split '\s+', $_;
+
+ if (($aHang <= 0) && ($bHang >= 0)) {
+ # A contained in B
+ next;
+ }
+
+ elsif (($aHang >= 0) && ($bHang <= 0)) {
+ # B contained in A
+ next;
+ }
+
+ elsif (($aHang <= 0) && ($bHang <= 0)) {
+ # Dovetail off the 5' end
+ $erate5{$aIID} = $erate if ((!exists($erate5{$aIID})) || ($erate < $erate5{$aIID}));
+ }
+
+ elsif (($aHang >= 0) && ($bHang >= 0)) {
+ # Dovetail off the 3' end
+ $erate3{$aIID} = $erate if ((!exists($erate3{$aIID})) || ($erate < $erate3{$aIID}));
+ }
+
+ else {
+ die;
+ }
+}
+}
+
+
+
+
+
+# Scan overlaps, try to decide true from false.
+
+print STDERR "Read overlas from '$olapStore'\n";
+
+my @ratioTrue;
+my @ratioFalse;
+
+open(F, "overlapStore -d $olapStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ #print "$_\n";
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $erate, $crate) = split '\s+', $_;
+
+ my $aErate = 0;
+ my $bErate = 0;
+
+ if (($aHang <= 0) && ($bHang >= 0)) {
+ # A contained in B
+ next;
+ }
+
+ elsif (($aHang >= 0) && ($bHang <= 0)) {
+ # B contained in A
+ next;
+ }
+
+
+ elsif (($aHang <= 0) && ($bHang <= 0) && ($orient eq "N")) {
+ # Dovetail off the 5' end of A to the 3' end of B
+ $aErate = $erate5{$aIID};
+ $bErate = $erate3{$bIID};
+ }
+
+ elsif (($aHang <= 0) && ($bHang <= 0) && ($orient eq "I")) {
+ # Dovetail off the 5' end of A to the 5' end of B
+ $aErate = $erate5{$aIID};
+ $bErate = $erate5{$bIID};
+ }
+
+ elsif (($aHang >= 0) && ($bHang >= 0) && ($orient eq "N")) {
+ # Dovetail off the 3' end of A to the 5' end of B
+ $aErate = $erate3{$aIID};
+ $bErate = $erate5{$bIID};
+ }
+
+ elsif (($aHang >= 0) && ($bHang >= 0) && ($orient eq "I")) {
+ # Dovetail off the 3' end of A to the 3' end of B
+ $aErate = $erate3{$aIID};
+ $bErate = $erate3{$bIID};
+ }
+
+ else {
+ die;
+ }
+
+
+ print STDERR "Didn't find erate for IID $aIID.\n $_\n" if (!defined($aErate));
+ print STDERR "Didn't find erate for IID $bIID.\n $_\n" if (!defined($bErate));
+
+ next if (!defined($aErate));
+ next if (!defined($bErate));
+
+ my $ratio = int(100 * ($erate) / ($aErate + $bErate));
+
+ my $dist = ($aIID < $bIID) ? ($bIID - $aIID) : ($aIID - $bIID);
+
+ die if ($dist < 0);
+
+ if ($dist < 50) {
+ $ratioTrue[$ratio]++;
+ } else {
+ $ratioFalse[$ratio]++;
+ }
+}
+close(F);
+
+open(F, "> r");
+for (my $x=0; $x<1000; $x++) {
+ next if (!defined($ratioTrue[$x]) && (!defined($ratioFalse[$x])));
+
+ $ratioTrue[$x] += 0;
+ $ratioFalse[$x] += 0;
+
+ my $v = $x / 100;
+
+ print F "$v\t$ratioTrue[$x]\t$ratioFalse[$x]\n";
+}
+close(F);
diff --git a/src/AS_BAT/erate-estimate.C b/src/AS_BAT/erate-estimate.C
new file mode 100644
index 0000000..143005c
--- /dev/null
+++ b/src/AS_BAT/erate-estimate.C
@@ -0,0 +1,688 @@
+/**************************************************************************
+ * Copyright (C) 2014
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received (LICENSE.txt) a copy of the GNU General Public
+ * License along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *************************************************************************/
+
+const char *mainid = "$Id: $";
+
+#include "AS_global.H"
+#include "AS_MSG_pmesg.H"
+#include "AS_PER_gkpStore.H"
+
+#include "AS_OVS_overlap.H"
+#include "AS_OVS_overlapFile.H"
+#include "AS_OVS_overlapStore.H"
+
+
+#include "intervalList.H"
+
+#include <map>
+#include <set>
+#include <list>
+#include <vector>
+#include <algorithm>
+
+#ifndef BROKEN_CLANG_OpenMP
+#include <omp.h>
+#endif
+
+#include "../AS_OBT/AS_OBT_overlaps.C"
+
+using namespace std;
+
+
+#define ERATE_TOLERANCE 0.03
+
+
+class readErrorEstimate {
+public:
+ readErrorEstimate() {
+ clrBgn = 0;
+ clrEnd = 0;
+ clrLen = 0;
+
+ errorMean = NULL;
+ //errorStdDev = NULL;
+ //errorConfidence = NULL;
+ errorMeanU = NULL;
+ };
+
+ ~readErrorEstimate() {
+ delete [] errorMean;
+ //delete [] errorStdDev;
+ //delete [] errorConfidence;
+ delete [] errorMeanU;
+ };
+
+ void initialize(gkFragment &fr) {
+ //uint32 lid = fr.gkFragment_getLibraryIID();
+ //gkLibrary *lb = gkpStore->gkStore_getLibrary(lid);
+
+ clrBgn = 0;
+ clrEnd = 0;
+ clrLen = 0;
+
+ if (fr.gkFragment_getIsDeleted() == 1)
+ return;
+
+ clrBgn = fr.gkFragment_getClearRegionBegin(AS_READ_CLEAR_LATEST);
+ clrEnd = fr.gkFragment_getClearRegionEnd (AS_READ_CLEAR_LATEST);
+ clrLen = clrEnd - clrBgn;
+
+ errorMean = new double [clrLen + 1];
+ //errorStdDev = new double [clrLen + 1];
+ //errorConfidence = new double [clrLen + 1];
+ errorMeanU = new double [clrLen + 1];
+ };
+
+ uint32 clrBgn;
+ uint32 clrEnd;
+ uint32 clrLen;
+
+ double *errorMean;
+ //double *errorStdDev;
+ //double *errorConfidence;
+ double *errorMeanU;
+};
+
+
+
+
+uint32
+loadOverlaps(uint32 iid,
+ OVSoverlap *&ovl,
+ uint32 &ovlLen,
+ uint32 &ovlMax,
+ OverlapStore *ovlPrimary,
+ OverlapStore *ovlSecondary);
+
+
+
+
+void
+saveProfile(uint32 iid,
+ uint32 iter,
+ readErrorEstimate *readProfile,
+ uint32 *erateCnt) {
+ char N[FILENAME_MAX];
+ sprintf(N, "erate-%08u-%02u.dat", iid, iter);
+
+ FILE *F = fopen(N, "w");
+
+ if (erateCnt)
+ for (uint32 pp=0; pp<readProfile[iid].clrLen; pp++)
+ fprintf(F, "%u %.4f %u\n", pp, readProfile[iid].errorMean[pp], erateCnt[pp]);
+ else
+ for (uint32 pp=0; pp<readProfile[iid].clrLen; pp++)
+ fprintf(F, "%u %.4f\n", pp, readProfile[iid].errorMean[pp]);
+
+ fclose(F);
+
+ FILE *P = popen("gnuplot ", "w");
+ fprintf(P, "set terminal png\n");
+ fprintf(P, "set output 'erate-%08d-%02u.png'\n", iid, iter);
+ fprintf(P, "plot [] [0.00:0.25] 'erate-%08d-%02u.dat' using 1:2 with lines\n", iid, iter);
+ pclose(P);
+}
+
+
+
+
+void
+computeInitialErrorProfile(gkStore *gkpStore,
+ uint32 iidMin,
+ uint32 iidMax,
+ OverlapStore *ovlPrimary,
+ OverlapStore *ovlSecondary,
+ readErrorEstimate *readProfile) {
+
+ // Allocate space for the compute.
+
+ uint32 ovlLen = 0;
+ uint32 ovlMax = 64 * 1024;
+ OVSoverlap *ovl = new OVSoverlap [ovlMax];
+
+ memset(ovl, 0, sizeof(OVSoverlap) * ovlMax);
+
+#ifdef SLOW
+ double *erateSum = new double [AS_READ_MAX_NORMAL_LEN];
+ uint32 *erateCnt = new uint32 [AS_READ_MAX_NORMAL_LEN];
+#else
+ uint32 *erateCnt = NULL;
+#endif
+
+ AS_OVS_resetRangeOverlapStore(ovlPrimary);
+ AS_OVS_resetRangeOverlapStore(ovlSecondary);
+
+ fprintf(stderr, "Processing from IID "F_U32" to "F_U32" out of "F_U32" reads.\n",
+ iidMin,
+ iidMax,
+ gkpStore->gkStore_getNumFragments());
+
+ // PASS 1; compute an initial error rate estimate for each read
+
+ for (uint32 iid=iidMin; iid<=iidMax; iid++) {
+ if (readProfile[iid].clrLen == 0)
+ // Deleted read.
+ continue;
+
+ if (loadOverlaps(iid, ovl, ovlLen, ovlMax, ovlPrimary, ovlSecondary) == 0)
+ continue;
+
+#ifdef SLOW
+ for (uint32 pp=0; pp<=readProfile[iid].clrLen; pp++) {
+ erateSum[pp] = 0;
+ erateCnt[pp] = 0;
+ }
+#endif
+
+ intervalList<uint32,double> eRateList;
+
+ for (uint32 oo=0; oo<ovlLen; oo++) {
+ assert(ovl[oo].a_iid == iid);
+ AS_OVS_convertOVLoverlapToOBToverlap(ovl[oo],
+ readProfile[ovl[oo].a_iid].clrLen,
+ readProfile[ovl[oo].b_iid].clrLen);
+ assert(ovl[oo].a_iid == iid);
+
+ double erate = AS_OVS_decodeQuality(ovl[oo].dat.obt.erate) / 2;
+
+#ifdef SLOW
+ // Simple slow version. Can we use intervalList for this?
+ for (uint32 xx=ovl[oo].dat.obt.a_beg; xx <= ovl[oo].dat.obt.a_end; xx++) {
+ erateSum[xx] += erate;
+ erateCnt[xx] += 1;
+ }
+#else
+ eRateList.add(ovl[oo].dat.obt.a_beg, ovl[oo].dat.obt.a_end - ovl[oo].dat.obt.a_beg, erate);
+#endif
+ }
+
+
+#ifdef SLOW
+ for (uint32 pp=0; pp <= readProfile[iid].clrLen; pp++) {
+ readProfile[iid].errorMean[pp] = 0;
+ //readProfile[iid].errorStdDev[pp] = 0;
+ //readProfile[iid].errorConfidence[pp] = 0;
+
+ if (erateCnt[pp] > 0)
+ readProfile[iid].errorMean[pp] = erateSum[pp] / erateCnt[pp];
+ }
+#else
+ intervalList<uint32,double> eRateMap(eRateList);
+
+
+ for (uint32 ii=0; ii<eRateMap.numberOfIntervals(); ii++) {
+ double eVal = (eRateMap.depth(ii) > 0) ? (eRateMap.value(ii) / eRateMap.depth(ii)) : 0;
+
+ for (uint32 pp=eRateMap.lo(ii); pp < eRateMap.hi(ii); pp++)
+ readProfile[iid].errorMean[pp] = eVal;
+ }
+#endif
+
+#if 0
+ if ((iid == 9) || (iid == 295))
+ saveProfile(iid, 0, readProfile, erateCnt);
+#endif
+
+ //if ((iid % 1000) == 0)
+ // fprintf(stderr, "IID %u OVLs %u\r", iid, ovlLen);
+
+ //if (ovlLen > 100)
+ // fprintf(stderr, "IID %u OVLs %u\n", iid, ovlLen);
+ }
+
+ fprintf(stderr, "\n");
+
+ delete [] ovl;
+#ifdef SLOW
+ delete [] erateSum;
+ delete [] erateCnt;
+#endif
+}
+
+
+
+
+
+
+
+
+
+
+double
+computeEstimatedErate(OVSoverlap *ovl, readErrorEstimate *readProfile) {
+ double estError = 0;
+ int32 olapLen = 0;
+
+ uint32 ab = ovl->dat.obt.a_beg;
+ uint32 ae = ovl->dat.obt.a_end;
+
+ assert(ab < ae);
+
+ for (uint32 xx=ab; xx <= ae; xx++)
+ estError += readProfile[ovl->a_iid].errorMean[xx];
+
+ uint32 bb = ovl->dat.obt.b_beg;
+ uint32 be = ovl->dat.obt.b_end_hi * 512 + ovl->dat.obt.b_end_lo;
+
+ if (be < bb) {
+ uint32 tt = bb;
+ bb = be;
+ be = tt;
+ }
+
+ assert(bb < be);
+
+ for (uint32 xx=bb; xx <= be; xx++)
+ estError += readProfile[ovl->b_iid].errorMean[xx];
+
+ estError = estError / (((ae - ab) + (be - bb)) / 2);
+
+ return(estError);
+}
+
+
+
+
+void
+recomputeErrorProfile(gkStore *gkpStore,
+ uint32 iidMin,
+ uint32 iidMax,
+ OverlapStore *ovlPrimary,
+ OverlapStore *ovlSecondary,
+ readErrorEstimate *readProfile,
+ uint32 iter) {
+
+ // Allocate space for the compute.
+
+ uint32 ovlLen = 0;
+ uint32 ovlMax = 64 * 1024;
+ OVSoverlap *ovl = new OVSoverlap [ovlMax];
+
+ memset(ovl, 0, sizeof(OVSoverlap) * ovlMax);
+
+#ifdef SLOW
+ double *erateSum = new double [AS_READ_MAX_NORMAL_LEN];
+ uint32 *erateCnt = new uint32 [AS_READ_MAX_NORMAL_LEN];
+#else
+ uint32 *erateCnt = NULL;
+#endif
+
+ uint32 nDiscards = 0;
+ uint32 nRemain = 0;
+
+ AS_OVS_resetRangeOverlapStore(ovlPrimary);
+ AS_OVS_resetRangeOverlapStore(ovlSecondary);
+
+ fprintf(stderr, "Processing from IID "F_U32" to "F_U32" out of "F_U32" reads, iteration %u.\n",
+ iidMin,
+ iidMax,
+ gkpStore->gkStore_getNumFragments(),
+ iter);
+
+ for (uint32 iid=iidMin; iid<=iidMax; iid++) {
+ if (readProfile[iid].clrLen == 0)
+ // Deleted read.
+ continue;
+
+ if (loadOverlaps(iid, ovl, ovlLen, ovlMax, ovlPrimary, ovlSecondary) == 0)
+ continue;
+
+#ifdef SLOW
+ for (uint32 pp=0; pp<=readProfile[iid].clrLen; pp++) {
+ erateSum[pp] = 0;
+ erateCnt[pp] = 0;
+ }
+#endif
+
+ intervalList<uint32,double> eRateList;
+
+ for (uint32 oo=0; oo<ovlLen; oo++) {
+ assert(ovl[oo].a_iid == iid);
+ AS_OVS_convertOVLoverlapToOBToverlap(ovl[oo],
+ readProfile[ovl[oo].a_iid].clrLen,
+ readProfile[ovl[oo].b_iid].clrLen);
+ assert(ovl[oo].a_iid == iid);
+
+ // Compute the expected erate for this overlap based on our estimated error in both reads
+
+ double estError = computeEstimatedErate(ovl + oo, readProfile);
+ double erate = AS_OVS_decodeQuality(ovl[oo].dat.obt.erate);
+
+#ifndef NODEBUG
+ if (((ovl[oo].a_iid == 1) && (ovl[oo].b_iid == 6836)) ||
+ ((ovl[oo].b_iid == 1) && (ovl[oo].a_iid == 6836))) {
+ char ovlstr[1024];
+
+ fprintf(stderr, "%s\n", AS_OVS_toString(ovlstr, ovl[oo]));
+ //fprintf(stderr, "%s\n", AS_OVS_toString(ovlstr, obt[oo]));
+
+ fprintf(stderr, " estError = %f erate = %f\n", estError, erate);
+ }
+#endif
+
+#if 0
+ if ((iid == 9) || (iid == 295))
+ fprintf(stdout, "%8u %8u %5u %5u %5u %5u len %5u %5u est %7.5f tru %7.5f %s\n",
+ ovl[oo].a_iid, ovl[oo].b_iid,
+ ovl[oo].dat.obt.a_beg,
+ ovl[oo].dat.obt.a_end,
+ ovl[oo].dat.obt.b_beg,
+ ovl[oo].dat.obt.b_end_hi * 512 + ovl[oo].dat.obt.b_end_lo,
+ readProfile[ovl[oo].a_iid].clrLen,
+ readProfile[ovl[oo].b_iid].clrLen,
+ estError,
+ erate,
+ (estError + ERATE_TOLERANCE < erate) ? "DISCARD" : "SAVE");
+#endif
+
+ if (estError + ERATE_TOLERANCE < erate) {
+ nDiscards++;
+ continue;
+ }
+
+ nRemain++;
+
+ erate /= 2;
+
+#ifdef SLOW
+ // Simple slow version. Can we use intervalList for this?
+ for (uint32 xx=ovl[oo].dat.obt.a_beg; xx <= ovl[oo].dat.obt.a_end; xx++) {
+ erateSum[xx] += erate;
+ erateCnt[xx] += 1;
+ }
+#else
+ eRateList.add(ovl[oo].dat.obt.a_beg, ovl[oo].dat.obt.a_end - ovl[oo].dat.obt.a_beg, erate);
+#endif
+
+ }
+
+#ifdef SLOW
+ for (uint32 pp=0; pp <= readProfile[iid].clrLen; pp++) {
+ readProfile[iid].errorMeanU[pp] = 0;
+ //readProfile[iid].errorStdDev[pp] = 0;
+ //readProfile[iid].errorConfidence[pp] = 0;
+
+ if (erateCnt[pp] > 0)
+ readProfile[iid].errorMeanU[pp] = erateSum[pp] / erateCnt[pp];
+ }
+#else
+ intervalList<uint32,double> eRateMap(eRateList);
+
+ // Can have zero if there isn't overlap coverage!
+ for (uint32 ii=0; ii<eRateMap.numberOfIntervals(); ii++) {
+ double eVal = (eRateMap.depth(ii) > 0) ? (eRateMap.value(ii) / eRateMap.depth(ii)) : 0;
+
+ for (uint32 pp=eRateMap.lo(ii); pp < eRateMap.hi(ii); pp++)
+ readProfile[iid].errorMeanU[pp] = eVal;
+ }
+#endif
+
+#if 0
+ if ((iid == 9) || (iid == 295))
+ saveProfile(iid, iter, readProfile, erateCnt);
+#endif
+
+ //if ((iid % 1000) == 0)
+ // fprintf(stderr, "IID %u OVLs %u\r", iid, ovlLen);
+ }
+
+ delete [] ovl;
+#ifdef SLOW
+ delete [] erateSum;
+ delete [] erateCnt;
+#endif
+
+ // Update all the means
+
+ for (uint32 iid=iidMin; iid<=iidMax; iid++) {
+ double *t = readProfile[iid].errorMean;
+
+ readProfile[iid].errorMean = readProfile[iid].errorMeanU;
+ readProfile[iid].errorMeanU = t;
+ }
+
+ // Report stats.
+
+ fprintf(stderr, "\n");
+ fprintf(stderr, "nDiscards %u\n", nDiscards);
+ fprintf(stderr, "nRemain %u\n", nRemain);
+}
+
+
+
+
+
+
+
+void
+outputOverlaps(gkStore *gkpStore,
+ uint32 iidMin,
+ uint32 iidMax,
+ OverlapStore *ovlPrimary,
+ OverlapStore *ovlSecondary,
+ readErrorEstimate *readProfile,
+ char *outputName) {
+
+ // Allocate space for the compute.
+
+ uint32 ovlLen = 0;
+ uint32 ovlMax = 64 * 1024;
+ OVSoverlap *ovl = new OVSoverlap [ovlMax];
+ OVSoverlap *obt = new OVSoverlap [ovlMax];
+
+ memset(ovl, 0, sizeof(OVSoverlap) * ovlMax);
+
+ uint32 nDiscards = 0;
+ uint32 nRemain = 0;
+
+ AS_OVS_resetRangeOverlapStore(ovlPrimary);
+ AS_OVS_resetRangeOverlapStore(ovlSecondary);
+
+ // Open an output store
+
+ OverlapStore *storeFile = AS_OVS_createOverlapStore(outputName, TRUE);
+
+ fprintf(stderr, "Processing from IID "F_U32" to "F_U32" out of "F_U32" reads.\n",
+ iidMin,
+ iidMax,
+ gkpStore->gkStore_getNumFragments());
+
+ for (uint32 iid=iidMin; iid<=iidMax; iid++) {
+ if (readProfile[iid].clrLen == 0)
+ // Deleted read.
+ continue;
+
+ if (loadOverlaps(iid, ovl, ovlLen, ovlMax, ovlPrimary, ovlSecondary) == 0)
+ continue;
+
+ memcpy(obt, ovl, sizeof(OVSoverlap) * ovlLen);
+
+ for (uint32 oo=0; oo<ovlLen; oo++) {
+ assert(ovl[oo].a_iid == iid);
+ AS_OVS_convertOVLoverlapToOBToverlap(obt[oo],
+ readProfile[obt[oo].a_iid].clrLen,
+ readProfile[obt[oo].b_iid].clrLen);
+ assert(ovl[oo].a_iid == iid);
+
+ double estError = computeEstimatedErate(obt + oo, readProfile);
+ double erate = AS_OVS_decodeQuality(obt[oo].dat.obt.erate);
+
+#ifndef NODEBUG
+ if (((ovl[oo].a_iid == 1) && (ovl[oo].b_iid == 6836)) ||
+ ((ovl[oo].b_iid == 1) && (ovl[oo].a_iid == 6836))) {
+ char ovlstr[1024];
+
+ fprintf(stderr, "%s\n", AS_OVS_toString(ovlstr, ovl[oo]));
+ fprintf(stderr, "%s\n", AS_OVS_toString(ovlstr, obt[oo]));
+
+ fprintf(stderr, " estError = %f erate = %f\n", estError, erate);
+ }
+#endif
+
+ if (estError + ERATE_TOLERANCE < erate) {
+ nDiscards++;
+ continue;
+ }
+
+ nRemain++;
+
+ // Emit the overlap
+
+ AS_OVS_writeOverlapToStore(storeFile, ovl + oo);
+ }
+
+ //if ((iid % 1000) == 0)
+ // fprintf(stderr, "IID %u OVLs %u\r", iid, ovlLen);
+ }
+
+ AS_OVS_closeOverlapStore(storeFile);
+
+ fprintf(stderr, "\n");
+ fprintf(stderr, "nDiscards %u\n", nDiscards);
+ fprintf(stderr, "nRemain %u\n", nRemain);
+
+ delete [] ovl;
+ delete [] obt;
+}
+
+
+
+
+
+
+
+
+int
+main(int argc, char **argv) {
+ char *gkpName = 0L;
+ gkStore *gkpStore = 0L;
+ char *ovlPrimaryName = 0L;
+ OverlapStore *ovlPrimary = 0L;
+ char *ovlSecondaryName = 0L;
+ OverlapStore *ovlSecondary = 0L;
+
+ uint32 errorRate = AS_OVS_encodeQuality(0.015);
+ double errorLimit = 2.5;
+
+ char *outputPrefix = NULL;
+ char logName[FILENAME_MAX] = {0};
+ char sumName[FILENAME_MAX] = {0};
+ FILE *logFile = 0L;
+ FILE *sumFile = 0L;
+
+ bool doModify = true; // Make this false for testing
+
+ uint32 iidMin = 1;
+ uint32 iidMax = UINT32_MAX;
+
+ argc = AS_configure(argc, argv);
+
+ uint32 minEvidenceOverlap = MIN(500, AS_OVERLAP_MIN_LEN);
+ uint32 minEvidenceCoverage = 1;
+
+ int arg=1;
+ int err=0;
+
+ while (arg < argc) {
+ if (strcmp(argv[arg], "-G") == 0) {
+ gkpName = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-O") == 0) {
+ if (ovlPrimaryName == NULL)
+ ovlPrimaryName = argv[++arg];
+ else if (ovlSecondary == NULL)
+ ovlSecondaryName = argv[++arg];
+ else {
+ fprintf(stderr, "ERROR: Only two obtStores (-O) allowed.\n");
+ err++;
+ }
+
+ } else {
+ fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]);
+ err++;
+ }
+
+ arg++;
+ }
+ if (err) {
+ exit(1);
+ }
+
+ // Open gatekeeper store
+
+ gkpStore = new gkStore(gkpName, FALSE, doModify);
+
+ gkpStore->gkStore_enableClearRange(AS_READ_CLEAR_OBTMERGE);
+ gkpStore->gkStore_metadataCaching(true);
+
+ if (iidMin < 1)
+ iidMin = 1;
+ if (iidMax > gkpStore->gkStore_getNumFragments())
+ iidMax = gkpStore->gkStore_getNumFragments();
+
+ // Open overlap stores
+
+ if (ovlPrimaryName != NULL)
+ ovlPrimary = AS_OVS_openOverlapStore(ovlPrimaryName);
+
+ if (ovlSecondary != NULL)
+ ovlSecondary = AS_OVS_openOverlapStore(ovlSecondaryName);
+
+ // Load read metadata, clear ranges, read lengths, and deleted status.
+
+ readErrorEstimate *readProfile = new readErrorEstimate [iidMax + 1];
+
+ for (uint32 iid=iidMin; iid<=iidMax; iid++) {
+ gkFragment fr;
+
+ gkpStore->gkStore_getFragment(iid, &fr, GKFRAGMENT_INF);
+
+ readProfile[iid].initialize(fr);
+ }
+
+ // Allocate space for the result.
+
+ double *erate5 = new double [iidMax];
+ double *erate3 = new double [iidMax];
+
+
+ // Compute the initial read profile.
+
+ computeInitialErrorProfile(gkpStore, iidMin, iidMax,
+ ovlPrimary,
+ ovlSecondary,
+ readProfile);
+
+ // Recompute, using the existing profile to weed out probably false overlaps.
+
+ for (uint32 ii=1; ii<4; ii++)
+ recomputeErrorProfile(gkpStore, iidMin, iidMax,
+ ovlPrimary,
+ ovlSecondary,
+ readProfile,
+ ii);
+
+
+ outputOverlaps(gkpStore, iidMin, iidMax,
+ ovlPrimary,
+ ovlSecondary,
+ readProfile,
+ "TEST.ovlStore");
+
+ exit(0);
+}
diff --git a/src/AS_BAT/examine-mapping-ideal.pl b/src/AS_BAT/examine-mapping-ideal.pl
new file mode 100644
index 0000000..dd345c8
--- /dev/null
+++ b/src/AS_BAT/examine-mapping-ideal.pl
@@ -0,0 +1,365 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $TOOSHORT = 500;
+
+my $FILE = "test.004.buildUnitigs";
+$FILE = shift @ARGV if (scalar(@ARGV) > 0);
+
+my $IDEAL = "../ideal/porphyromonas_gingivalis_w83.flx.3200bp.0900bp.FJRUAFO0.ideal.intervals";
+
+if ((! -e "$FILE.tigStore") ||
+ (! -e "$FILE.fasta")) {
+ die "Missing tigStore or fasta. Run build-fasta.pl\n";
+}
+
+########################################
+
+if (! -e "$FILE.coords") {
+ my $cmd;
+
+ $cmd = "nucmer";
+ $cmd .= " --maxmatch --coords -p $FILE";
+ $cmd .= " /work/FRAGS/porphyromonas_gingivalis_w83/reference/AE015924.fasta";
+ $cmd .= " $FILE.fasta";
+
+ system($cmd);
+}
+
+########################################
+# Load the lengh of each sequence.
+
+my %length;
+
+if (-e "$FILE.fasta") {
+ open(F, "< $FILE.fasta") or die;
+ while (<F>) {
+ if (m/^>(utg\d+)\s+len=(\d+)$/) {
+ $length{$1} = $2;
+ }
+ }
+ close(F);
+} else {
+ print STDERR "No $FILE.fasta file found, cannot .....\n";
+}
+
+########################################
+# Pass one, load the coords. Analyze anything with one match, counting
+# the amount of coverage.
+
+my %nucmer;
+
+if (-e "$FILE.coords") {
+ open(F, "< $FILE.coords") or die;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ while (<F>) {
+ s/^\s+//;
+ s/\s$+//;
+ my @v = split '\s+', $_;
+
+ my $utgBgn = ($v[3] < $v[4]) ? $v[3] : $v[4];
+ my $utgEnd = ($v[3] < $v[4]) ? $v[4] : $v[3];
+ my $genBgn = ($v[0] < $v[1]) ? $v[0] : $v[1];
+ my $genEnd = ($v[0] < $v[1]) ? $v[1] : $v[0];
+
+ # Rearrange the coords so that bgn is ALWAYS less than end (we lose orientation).
+ my $str = "$utgBgn\t$utgEnd\t$genBgn\t$genEnd";
+
+ if (exists($nucmer{$v[12]})) {
+ $nucmer{$v[12]} .= "\n" . $str;
+ } else {
+ $nucmer{$v[12]} = $str;
+ }
+ }
+ close(F);
+} else {
+ die "No $FILE.coords??\n";
+}
+
+########################################
+# For things with one match, report spurs.
+
+my $spur = 0;
+my $spurShort = 0;
+my $confirmed = 0;
+my $confirmedBP = 0;
+
+foreach my $utg (keys %nucmer) {
+ my @m = split '\n', $nucmer{$utg};
+
+ if (scalar(@m) > 1) {
+ @m = sort {$a <=> $b} @m;
+ $nucmer{$utg} = join "\n", @m;
+ next;
+ }
+
+ my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $nucmer{$utg};
+
+ my $len = $utgEnd - $utgBgn;
+ die if ($len <= 0);
+
+ my $per = $len / $length{$utg};
+
+ if ($per < 0.95) {
+ if ($length{$utg} < $TOOSHORT) {
+ $spurShort++;
+ } else {
+ print STDERR "SPUR $utg len $length{$utg} mapped $len ($per%) coords $nucmer{$utg}\n";
+ $spur++;
+ }
+ } else {
+ $confirmed++;
+ $confirmedBP += $length{$utg};
+ }
+
+ delete $nucmer{$utg};
+}
+
+print STDERR "FOUND ", scalar(keys %nucmer), " MATCHES, of which $spur are SPURs (and $spurShort are SHORT SPURS)\n";
+print STDERR "FOUND $confirmed confirmed unitigs of total length $confirmedBP bp.\n";
+
+########################################
+# Things with multiple matches are candidates for chimera.
+
+my $classFalseMateSplit = 0; my $lenFalseMateSplit = 0;
+my $classShort = 0; my $lenShort = 0;
+my $classDisconnected = 0; my $lenDisconnected = 0;
+my $classRepeat = 0; my $lenRepeat = 0;
+my $classRepeatEndWithRepeat = 0; my $lenRepeatEndWithRepeat = 0;
+my $classRepeat1Unique = 0; my $lenRepeat1Unique = 0;
+my $classRepeat2Unique = 0; my $lenRepeat2Unique = 0;
+my $classMultipleBlock = 0; my $lenMultipleBlock = 0;
+my $classUnknown = 0; my $lenUnknown = 0;
+
+
+foreach my $utg (keys %nucmer) {
+ my @m = split '\n', $nucmer{$utg};
+
+ my $numBlocks = 0;
+ my @bgn;
+ my @end;
+
+ my $min = 999999999;
+ my $max = 0;
+
+ #print STDOUT "$utg\n";
+
+ # Examine the matches, decide if the multiple matches are due to
+ # unitig is a repeat
+ # short repeats interior to the unitig
+ # long repeats at the end
+ # chimer
+
+ my $numMatches = 0;
+ my $isDisconnected = 0;
+ my $verboseBuild = 0;
+
+ print STDOUT "--------------------------------------------------------------------------------\n" if ($verboseBuild);
+ print STDOUT "UNITIG $utg\n" if ($verboseBuild);
+
+ foreach my $m (@m) {
+ my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m;
+
+ $numMatches++;
+ print STDOUT " $m" if ($verboseBuild);
+
+ # Are we an exact match to the largest thing so far?
+
+ if (($utgBgn == $min) && ($utgEnd == $max)) {
+ print STDOUT " -- is exact\n" if ($verboseBuild);
+ next;
+ }
+
+ # Search for things comtained or overlapping previous matches. This assumes that matches
+ # are sorted by begin coord. We need to handle:
+ #
+ # ---------- and ----------- not -------- or -----
+ # ---- --------- ------ ----------
+ #
+ # and, the ones we're more-or-less really looking for:
+ #
+ # -------- OR --------
+ # -------- -------
+
+ # This isolates out the first case above. We don't want to save it as a region,
+ # because it is completely contained in a previous region.
+ if ($utgEnd <= $max) {
+ print STDOUT " -- is contained\n" if ($verboseBuild);
+ next;
+ }
+
+ # If we aren't even intersecting, we've found the third case.
+ $isDisconnected++ if (($max > 0) && ($max < $utgBgn));
+
+ # Finally, update the min/max extents. This must be last.
+ if ($max == 0) {
+ $min = $utgBgn;
+ $max = $utgEnd;
+ }
+ if (($utgBgn - 5 <= $min) && ($max < $utgEnd)) { # max STRICTLY LESS than utgEnd!
+ pop @end;
+ push @end, $utgEnd;
+ $max = $utgEnd;
+ print STDOUT " -- extends ", $bgn[scalar(@bgn)-1], " ", $end[scalar(@end)-1], " ", scalar(@bgn), " ", scalar(@end), " $utgBgn $utgEnd\n" if ($verboseBuild);
+ next;
+ }
+ if ($utgBgn < $min) { $min = $utgBgn; }
+ if ($max < $utgEnd) { $max = $utgEnd; }
+
+ push @bgn, $utgBgn;
+ push @end, $utgEnd;
+
+ $numBlocks++;
+
+ print STDOUT "\n" if ($verboseBuild);
+ }
+
+ # One more pass through to count repeats. Above we just found the min/max extent of the
+ # alignments on the unitig. Here we can count if alignments are interior to the extent, at the
+ # end, or completely spanning it.
+ #
+ # This fails by counting interior disconnected chimer (e.g., -- -- --) as 'isInteriorRepeat'
+
+ my $isMaxExact = 0;
+ my $isInteriorRepeat = 0;
+ my $isTerminalRepeat = 0;
+
+
+ foreach my $m (@m) {
+ my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m;
+
+ # New match covers the whole region.
+ if (($utgBgn - 5 <= $min) &&
+ ($max <= $utgEnd + 5)) {
+ $isMaxExact++;
+ next;
+ }
+
+ # New match completely contained, near the extent.
+ if (($utgBgn - 5 <= $min) ||
+ ($max <= $utgEnd + 5)) {
+ $isTerminalRepeat++;
+ next;
+ }
+
+ # Otherwise, the match must be interior.
+ $isInteriorRepeat++;
+ }
+
+
+
+ print STDERR "UNDEF LENGTH $utg\n" if (!defined($length{$utg}));
+ print STDERR "ZERO LENGTH $utg\n" if ($length{$utg} == 0);
+ print STDERR "SHORT LENGTH $utg\n" if ($length{$utg} < 64);
+
+ if (($numMatches == 2) &&
+ ($isDisconnected > 0) &&
+ ($isTerminalRepeat == 2)) {
+ $classFalseMateSplit++;
+ $lenFalseMateSplit += $length{$utg};
+ #print "$utg (len $length{$utg}) is a FALSE MATE SPLIT.\n";
+ next;
+ }
+
+ if ($isDisconnected > 0) {
+ $classDisconnected++;
+ $lenDisconnected += $length{$utg};
+ #print "$utg (len $length{$utg}) is DISCONNECTED.\n";
+ next;
+ }
+
+ if ($isMaxExact >= 2) {
+ $classRepeat++;
+ $lenRepeat += $length{$utg};
+ #print "$utg (len $length{$utg}) is a REPEAT.\n";
+ next;
+ }
+
+ if ($max < $TOOSHORT) {
+ $classShort++;
+ $lenShort += $length{$utg};
+ next;
+ }
+
+ if (($isMaxExact == 1) &&
+ ($isTerminalRepeat > 0) &&
+ ($isInteriorRepeat > 0)) {
+ $classRepeatEndWithRepeat++;
+ $lenRepeatEndWithRepeat += $length{$utg};
+ if (0) {
+ print "----------\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+ for (my $iii=0; $iii<scalar(@bgn); $iii++) {
+ print "$iii -- $bgn[$iii] $end[$iii]\n";
+ }
+ print "$utg (len $length{$utg}) is REPEAT on END OF UNIQUE with a repeat inside.\n";
+ }
+ #die if ($numBlocks > 1);
+ next;
+ }
+
+ if (($isMaxExact == 1) &&
+ ($isTerminalRepeat == $numMatches - 1)) {
+ $classRepeat1Unique++;
+ $lenRepeat1Unique += $length{$utg};
+ if (0) {
+ print "----------\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+ for (my $iii=0; $iii<scalar(@bgn); $iii++) {
+ print "$iii -- $bgn[$iii] $end[$iii]\n";
+ }
+ print "$utg (len $length{$utg}) is REPEAT WITH A BIT OF UNIQUE ON ONE END.\n";
+ }
+ next;
+ }
+
+ if (($isMaxExact == 1) &&
+ ($isTerminalRepeat == 0) &&
+ ($isInteriorRepeat == $numMatches - 1)) {
+ $classRepeat2Unique++;
+ $lenRepeat2Unique += $length{$utg};
+ #print "$utg (len $length{$utg}) is REPEAT WITH A BIT OF UNIQUE ON BOTH ENDS.\n";
+ next;
+ }
+
+ if ($numBlocks > 1) {
+ $classMultipleBlock++;
+ $lenMultipleBlock += $length{$utg};
+ if (0) {
+ print "----------\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+ for (my $iii=0; $iii<scalar(@bgn); $iii++) {
+ print "$iii -- $bgn[$iii] $end[$iii]\n";
+ }
+ print "$utg (len $length{$utg}) is MULTIPLE BLCOKS.\n";
+ }
+ next;
+ }
+
+ $classUnknown++;
+ $lenUnknown += $length{$utg};
+
+ print "$utg (len $length{$utg}) is UNCLASSIFIED.\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+
+ # Try to make some sense out of it. ....
+}
+
+print STDERR "classFalseMateSplit $classFalseMateSplit ($lenFalseMateSplit bp)\n";
+print STDERR "classDisconnected $classDisconnected ($lenDisconnected bp)\n";
+print STDERR "classRepeat $classRepeat ($lenRepeat bp)\n";
+print STDERR "classShort $classShort ($lenShort bp)\n";
+print STDERR "classRepeatEndWithRepeat $classRepeatEndWithRepeat ($lenRepeatEndWithRepeat bp)\n";
+print STDERR "classRepeat1Unique $classRepeat1Unique ($lenRepeat1Unique bp)\n";
+print STDERR "classRepeat2Unique $classRepeat2Unique ($lenRepeat2Unique bp)\n";
+print STDERR "classMultipleBlock $classMultipleBlock ($lenMultipleBlock bp)\n";
+print STDERR "classUnknown $classUnknown ($lenUnknown bp)\n";
diff --git a/src/AS_BAT/examine-mapping.pl b/src/AS_BAT/examine-mapping.pl
new file mode 100644
index 0000000..0193ab9
--- /dev/null
+++ b/src/AS_BAT/examine-mapping.pl
@@ -0,0 +1,368 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $TOOSHORT = 500;
+
+my $FILE = "test.004.buildUnitigs";
+$FILE = shift @ARGV if (scalar(@ARGV) > 0);
+
+if ((! -e "$FILE.tigStore") ||
+ (! -e "$FILE.fasta")) {
+ die "Missing tigStore or fasta. Run build-fasta.pl\n";
+}
+
+########################################
+
+if (! -e "$FILE.coords") {
+ my $cmd;
+
+ $cmd = "nucmer";
+ $cmd .= " --maxmatch --coords -p $FILE";
+ $cmd .= " /work/FRAGS/porphyromonas_gingivalis_w83/reference/AE015924.fasta";
+ $cmd .= " $FILE.fasta";
+
+ system($cmd);
+}
+
+########################################
+# Load the lengh of each sequence.
+
+my %length;
+
+if (-e "$FILE.fasta") {
+ open(F, "< $FILE.fasta") or die;
+ while (<F>) {
+ if (m/^>(utg\d+)\s+len=(\d+)$/) {
+ $length{$1} = $2;
+ }
+ }
+ close(F);
+} else {
+ print STDERR "No $FILE.fasta file found, cannot .....\n";
+}
+
+########################################
+# Pass one, load the coords. Analyze anything with one match, counting
+# the amount of coverage.
+
+my %nucmer;
+
+if (-e "$FILE.coords") {
+ open(F, "< $FILE.coords") or die;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ while (<F>) {
+ s/^\s+//;
+ s/\s$+//;
+ my @v = split '\s+', $_;
+
+ my $utgBgn = ($v[3] < $v[4]) ? $v[3] : $v[4];
+ my $utgEnd = ($v[3] < $v[4]) ? $v[4] : $v[3];
+ my $genBgn = ($v[0] < $v[1]) ? $v[0] : $v[1];
+ my $genEnd = ($v[0] < $v[1]) ? $v[1] : $v[0];
+
+ # Rearrange the coords so that bgn is ALWAYS less than end (we lose orientation).
+ my $str = "$utgBgn\t$utgEnd\t$genBgn\t$genEnd";
+
+ if (exists($nucmer{$v[12]})) {
+ $nucmer{$v[12]} .= "\n" . $str;
+ } else {
+ $nucmer{$v[12]} = $str;
+ }
+ }
+ close(F);
+} else {
+ die "No $FILE.coords??\n";
+}
+
+########################################
+# For things with one match, report spurs.
+
+my $spur = 0;
+my $spurShort = 0;
+my $confirmed = 0;
+my $confirmedBP = 0;
+
+foreach my $utg (keys %nucmer) {
+ my @m = split '\n', $nucmer{$utg};
+
+ if (scalar(@m) > 1) {
+ @m = sort {$a <=> $b} @m;
+ $nucmer{$utg} = join "\n", @m;
+ next;
+ }
+
+ my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $nucmer{$utg};
+
+ my $len = $utgEnd - $utgBgn;
+ die if ($len <= 0);
+
+ my $per = $len / $length{$utg};
+
+ if ($per < 0.95) {
+ if ($length{$utg} < $TOOSHORT) {
+ $spurShort++;
+ } else {
+ print STDERR "SPUR $utg len $length{$utg} mapped $len ($per%) coords $nucmer{$utg}\n";
+ $spur++;
+ }
+ } else {
+ $confirmed++;
+ $confirmedBP += $length{$utg};
+ }
+
+ delete $nucmer{$utg};
+}
+
+print STDERR "FOUND ", scalar(keys %nucmer), " MATCHES, of which $spur are SPURs (and $spurShort are SHORT SPURS)\n";
+print STDERR "FOUND $confirmed confirmed unitigs of total length $confirmedBP bp.\n";
+
+########################################
+# Things with multiple matches are candidates for chimera.
+
+my $classFalseMateSplit = 0; my $lenFalseMateSplit = 0;
+my $classShort = 0; my $lenShort = 0;
+my $classDisconnected = 0; my $lenDisconnected = 0;
+my $classRepeat = 0; my $lenRepeat = 0;
+my $classRepeatEndWithRepeat = 0; my $lenRepeatEndWithRepeat = 0;
+my $classRepeat1Unique = 0; my $lenRepeat1Unique = 0;
+my $classRepeat2Unique = 0; my $lenRepeat2Unique = 0;
+my $classMultipleBlock = 0; my $lenMultipleBlock = 0;
+my $classUnknown = 0; my $lenUnknown = 0;
+
+
+foreach my $utg (keys %nucmer) {
+ my @m = split '\n', $nucmer{$utg};
+
+ my $numBlocks = 0;
+ my @bgn;
+ my @end;
+
+ my $min = 999999999;
+ my $max = 0;
+
+ #print STDOUT "$utg\n";
+
+ # Examine the matches, decide if the multiple matches are due to
+ # unitig is a repeat
+ # short repeats interior to the unitig
+ # long repeats at the end
+ # chimer
+
+ my $numMatches = 0;
+ my $isDisconnected = 0;
+ my $verboseBuild = 1;
+
+ print STDOUT "--------------------------------------------------------------------------------\n" if ($verboseBuild);
+ print STDOUT "UNITIG $utg\n" if ($verboseBuild);
+
+ foreach my $m (@m) {
+ my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m;
+
+ $numMatches++;
+ print STDOUT " $m" if ($verboseBuild);
+
+ # Are we an exact match to the largest thing so far?
+
+ if (($utgBgn == $min) && ($utgEnd == $max)) {
+ print STDOUT " -- is exact\n" if ($verboseBuild);
+ next;
+ }
+
+ # Search for things comtained or overlapping previous matches. This assumes that matches
+ # are sorted by begin coord. We need to handle:
+ #
+ # ---------- and ----------- not -------- or -----
+ # ---- --------- ------ ----------
+ #
+ # and, the ones we're more-or-less really looking for:
+ #
+ # -------- OR --------
+ # -------- -------
+
+ # This isolates out the first case above. We don't want to save it as a region,
+ # because it is completely contained in a previous region.
+ if ($utgEnd <= $max) {
+ print STDOUT " -- is contained in a region\n" if ($verboseBuild);
+ next;
+ }
+
+ # If we aren't even intersecting, we've found the third case.
+ my $dlabel = "";
+ if (($max > 0) && ($max < $utgBgn)) {
+ my $dist = $utgBgn - $max;
+ $dlabel = " DISCONNECT $dist";
+ $isDisconnected++
+ }
+
+ # Finally, update the min/max extents. This must be last.
+ if ($max == 0) {
+ $min = $utgBgn;
+ $max = $utgEnd;
+ }
+ if (($utgBgn - 5 <= $min) && ($max < $utgEnd)) { # max STRICTLY LESS than utgEnd!
+ pop @end;
+ push @end, $utgEnd;
+ $max = $utgEnd;
+ print STDOUT " -- extends a region\n" if ($verboseBuild);
+ next;
+ }
+ if ($utgBgn < $min) { $min = $utgBgn; }
+ if ($max < $utgEnd) { $max = $utgEnd; }
+
+ push @bgn, $utgBgn;
+ push @end, $utgEnd;
+
+ $numBlocks++;
+
+ print STDOUT " -- makes a new region$dlabel\n" if ($verboseBuild);
+ }
+
+ # One more pass through to count repeats. Above we just found the min/max extent of the
+ # alignments on the unitig. Here we can count if alignments are interior to the extent, at the
+ # end, or completely spanning it.
+ #
+ # This fails by counting interior disconnected chimer (e.g., -- -- --) as 'isInteriorRepeat'
+
+ my $isMaxExact = 0;
+ my $isInteriorRepeat = 0;
+ my $isTerminalRepeat = 0;
+
+
+ foreach my $m (@m) {
+ my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m;
+
+ # New match covers the whole region.
+ if (($utgBgn - 5 <= $min) &&
+ ($max <= $utgEnd + 5)) {
+ $isMaxExact++;
+ next;
+ }
+
+ # New match completely contained, near the extent.
+ if (($utgBgn - 5 <= $min) ||
+ ($max <= $utgEnd + 5)) {
+ $isTerminalRepeat++;
+ next;
+ }
+
+ # Otherwise, the match must be interior.
+ $isInteriorRepeat++;
+ }
+
+
+
+ print STDERR "UNDEF LENGTH $utg\n" if (!defined($length{$utg}));
+ print STDERR "ZERO LENGTH $utg\n" if ($length{$utg} == 0);
+ print STDERR "SHORT LENGTH $utg\n" if ($length{$utg} < 64);
+
+ if (($numMatches == 2) &&
+ ($isDisconnected > 0) &&
+ ($isTerminalRepeat == 2)) {
+ $classFalseMateSplit++;
+ $lenFalseMateSplit += $length{$utg};
+ print "$utg (len $length{$utg}) is a FALSE MATE SPLIT.\n" if ($verboseBuild);
+ next;
+ }
+
+ if ($isDisconnected > 0) {
+ $classDisconnected++;
+ $lenDisconnected += $length{$utg};
+ print "$utg (len $length{$utg}) is DISCONNECTED.\n" if ($verboseBuild);
+ next;
+ }
+
+ if ($isMaxExact >= 2) {
+ $classRepeat++;
+ $lenRepeat += $length{$utg};
+ print "$utg (len $length{$utg}) is a REPEAT.\n" if ($verboseBuild);
+ next;
+ }
+
+ if ($max < $TOOSHORT) {
+ $classShort++;
+ $lenShort += $length{$utg};
+ next;
+ }
+
+ if (($isMaxExact == 1) &&
+ ($isTerminalRepeat > 0) &&
+ ($isInteriorRepeat > 0)) {
+ $classRepeatEndWithRepeat++;
+ $lenRepeatEndWithRepeat += $length{$utg};
+ if (0) {
+ print "----------\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+ for (my $iii=0; $iii<scalar(@bgn); $iii++) {
+ print "$iii -- $bgn[$iii] $end[$iii]\n";
+ }
+ print "$utg (len $length{$utg}) is REPEAT on END OF UNIQUE with a repeat inside.\n";
+ }
+ #die if ($numBlocks > 1);
+ next;
+ }
+
+ if (($isMaxExact == 1) &&
+ ($isTerminalRepeat == $numMatches - 1)) {
+ $classRepeat1Unique++;
+ $lenRepeat1Unique += $length{$utg};
+ if (0) {
+ print "----------\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+ for (my $iii=0; $iii<scalar(@bgn); $iii++) {
+ print "$iii -- $bgn[$iii] $end[$iii]\n";
+ }
+ print "$utg (len $length{$utg}) is REPEAT WITH A BIT OF UNIQUE ON ONE END.\n";
+ }
+ next;
+ }
+
+ if (($isMaxExact == 1) &&
+ ($isTerminalRepeat == 0) &&
+ ($isInteriorRepeat == $numMatches - 1)) {
+ $classRepeat2Unique++;
+ $lenRepeat2Unique += $length{$utg};
+ #print "$utg (len $length{$utg}) is REPEAT WITH A BIT OF UNIQUE ON BOTH ENDS.\n";
+ next;
+ }
+
+ if ($numBlocks > 1) {
+ $classMultipleBlock++;
+ $lenMultipleBlock += $length{$utg};
+ if (0) {
+ print "----------\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+ for (my $iii=0; $iii<scalar(@bgn); $iii++) {
+ print "$iii -- $bgn[$iii] $end[$iii]\n";
+ }
+ print "$utg (len $length{$utg}) is MULTIPLE BLCOKS.\n";
+ }
+ next;
+ }
+
+ $classUnknown++;
+ $lenUnknown += $length{$utg};
+
+ print "$utg (len $length{$utg}) is UNCLASSIFIED.\n";
+ print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n";
+ print "$nucmer{$utg}\n";
+
+ # Try to make some sense out of it. ....
+}
+
+print STDERR "classFalseMateSplit $classFalseMateSplit ($lenFalseMateSplit bp)\n";
+print STDERR "classDisconnected $classDisconnected ($lenDisconnected bp)\n";
+print STDERR "classRepeat $classRepeat ($lenRepeat bp)\n";
+print STDERR "classShort $classShort ($lenShort bp)\n";
+print STDERR "classRepeatEndWithRepeat $classRepeatEndWithRepeat ($lenRepeatEndWithRepeat bp)\n";
+print STDERR "classRepeat1Unique $classRepeat1Unique ($lenRepeat1Unique bp)\n";
+print STDERR "classRepeat2Unique $classRepeat2Unique ($lenRepeat2Unique bp)\n";
+print STDERR "classMultipleBlock $classMultipleBlock ($lenMultipleBlock bp)\n";
+print STDERR "classUnknown $classUnknown ($lenUnknown bp)\n";
diff --git a/src/AS_BAT/locate-read-in-unitig-based-on-olaps.pl b/src/AS_BAT/locate-read-in-unitig-based-on-olaps.pl
new file mode 100644
index 0000000..0cafb46
--- /dev/null
+++ b/src/AS_BAT/locate-read-in-unitig-based-on-olaps.pl
@@ -0,0 +1,199 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $prefix = shift @ARGV;
+my @reads;
+
+my @tigStore;
+
+# If no read supplied, scan the tigStore for all unitigs that have a single read spanning
+# This read should be first.
+
+print STDERR "Searching in assembly prefix '$prefix'.\n";
+
+print STDERR "Loading tigStore.\n";
+open(F, "tigStore -g $prefix.gkpStore -t $prefix.tigStore 1 -U -d layout |");
+while (!eof(F)) {
+ my $utg = <F>;
+ my $len = <F>;
+ my $cns = <F>;
+ my $qlt = <F>;
+ my $d1 = <F>; # cov stat
+ my $d2 = <F>; # microhet
+ my $d3 = <F>; # status
+ my $d4 = <F>; # unique_rept
+ my $d5 = <F>; # status
+ my $nfrg = <F>; # num frags
+ my $d7 = <F>; # num unitigs
+
+ my $maxPos = 0;
+
+ if ($utg =~ m/unitig\s+(\d+)$/) {
+ $utg = $1;
+ } else {
+ die "Out of sync on '$utg'\n";
+ }
+
+ if ($nfrg =~ m/data.num_frags\s+(\d+)$/) {
+ $nfrg = $1;
+ } else {
+ die "Out of sync on '$nfrg'\n";
+ }
+
+ my @frgs;
+
+ for (my $nnnn=0; $nnnn < $nfrg; $nnnn++) {
+ $_ = <F>;
+ chomp;
+
+ if (m/^FRG\stype\sR\sident\s+(\d+)\s+.*position\s+(\d+)\s+(\d+)$/) {
+ push @frgs, "FRG\t$1\t$2\t$3";
+
+ $maxPos = ($maxPos < $2) ? $2 : $maxPos;
+ $maxPos = ($maxPos < $3) ? $3 : $maxPos;
+ } else {
+ die "Nope: '$_'\n";
+ }
+ }
+
+ push @tigStore, "UTG\t$utg\t$nfrg\t$maxPos";
+ push @tigStore, @frgs;
+}
+close(F);
+
+
+
+
+if (defined($ARGV[0])) {
+ push @reads, $ARGV[0];
+
+} else {
+ print STDERR "SCANNING tigStore for reads that span a unitig.\n";
+
+ my $chk;
+ my $utg;
+ my $nfrg = 0;
+ my $tfrg = 0;
+ my $maxPos;
+
+ foreach my $l (@tigStore) {
+ if ($nfrg == 0) {
+ ($chk, $utg, $nfrg, $maxPos) = split '\s+', $l;
+ die if ($chk ne "UTG");
+
+ next;
+ }
+
+ $nfrg--;
+ $tfrg++;
+
+ my ($chk, $iid, $bgn, $end) = split '\s+', $l;
+ die if ($chk ne "FRG");
+
+ if ($bgn > $end) {
+ ($bgn, $end) = ($end, $bgn);
+ }
+
+ if (($bgn == 0) && ($end == $maxPos)) {
+ push @reads, $iid;
+ }
+ }
+
+ print STDERR "Searching for placements for ", scalar(@reads), " reads.\n";
+}
+
+
+
+foreach my $read (@reads) {
+ my %ovlRead;
+ my %ovlType;
+
+ #print STDERR "Searching for placements for read $read in unitigs.\n";
+ print "\n";
+
+ # Find overlapping reads.
+
+ open(F, "overlapStore -b $read -e $read -d $prefix.ovlStore |");
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ $ovlRead{$v[1]} = 1;
+ $ovlType{$v[1]} = '';
+
+ if (($v[3] < 0) && ($v[4] < 0)) {
+ $ovlType{$v[1]} = '5';
+ } elsif (($v[3] > 0) && ($v[4] > 0)) {
+ $ovlType{$v[1]} = '3';
+ } elsif (($v[3] < 0) && ($v[4] > 0)) {
+ $ovlType{$v[1]} = 'c'; # Contained
+ } else {
+ $ovlType{$v[1]} = 'C'; # Container
+ }
+ }
+ close(F);
+
+ #print STDERR "Searching for placements for read $read in unitigs, using ", scalar(keys %ovlRead), " overlapping reads.\n";
+
+ # Scan unitigs. For any unitig with more than zero overlapping reads present, report.
+
+ my $chk;
+ my $utg;
+ my $nfrg = 0;
+ my $cfrg = 0;
+ my $maxPos;
+
+ my $nmat;
+ my %ntyp;
+ my $self;
+
+ foreach my $l (@tigStore) {
+ if ($cfrg == 0) {
+ ($chk, $utg, $nfrg, $maxPos) = split '\s+', $l;
+ die if ($chk ne "UTG");
+
+ $cfrg = $nfrg;
+
+ $nmat = 0;
+ $ntyp{'5'} = 0;
+ $ntyp{'3'} = 0;
+ $ntyp{'C'} = 0;
+ $ntyp{'c'} = 0;
+ $self = 0;
+
+ next;
+ }
+
+ $cfrg--;
+
+ my ($chk, $iid, $bgn, $end) = split '\s+', $l;
+ die if ($chk ne "FRG");
+
+ if ($bgn > $end) {
+ ($bgn, $end) = ($end, $bgn);
+ }
+
+ if (exists($ovlRead{$iid})) {
+ $nmat++;
+ $ntyp{$ovlType{$iid}}++;
+
+ #print "FRG\t$iid\tat\t$bgn\t$end\t$ovlType{$iid}\n";
+ }
+
+ if ($iid == $read) {
+ $self++;
+ }
+
+ if (($cfrg == 0) && ($self + $nmat > 0)) {
+ if ($self) {
+ print "read $read\tUNITIG $utg\tof length $maxPos with 5'=$ntyp{'5'} 3'=$ntyp{'3'} contained=$ntyp{'c'} container=$ntyp{'C'} overlapping reads out of $nfrg.\n";
+ } else {
+ print "read $read\tunitig $utg\tof length $maxPos with 5'=$ntyp{'5'} 3'=$ntyp{'3'} contained=$ntyp{'c'} container=$ntyp{'C'} overlapping reads out of $nfrg.\n";
+ }
+ }
+ }
+ close(F);
+}
diff --git a/src/AS_BAT/markRepeatUnique.C b/src/AS_BAT/markRepeatUnique.C
new file mode 100644
index 0000000..ea4eff3
--- /dev/null
+++ b/src/AS_BAT/markRepeatUnique.C
@@ -0,0 +1,601 @@
+
+/**************************************************************************
+ * Copyright (C) 2014, J Craig Venter Institute. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received (LICENSE.txt) a copy of the GNU General Public
+ * License along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *************************************************************************/
+
+const char *mainid = "$Id: $";
+
+#include "AS_global.H"
+#include "AS_PER_gkpStore.H"
+#include "MultiAlign.H"
+#include "MultiAlignStore.H"
+
+#include "AS_CGB_histo.H"
+
+#include "intervalList.H"
+
+#include <algorithm>
+
+using namespace std;
+
+
+
+// Stats on repeat labeling of input unitigs.
+//
+class ruLabelStat {
+public:
+ ruLabelStat() {
+ num = 0;
+ len = 0;
+ };
+
+ void operator+=(uint64 len_) {
+ num++;
+ len += len_;
+ };
+
+ uint32 num;
+ uint64 len;
+};
+
+ruLabelStat repeat_LowReads;
+ruLabelStat repeat_LowCovStat;
+ruLabelStat repeat_Short;
+ruLabelStat repeat_MicroHet;
+ruLabelStat repeat_SingleSpan;
+ruLabelStat repeat_LowCov;
+
+ruLabelStat repeat_IsSingleton;
+ruLabelStat repeat_IsUnique;
+ruLabelStat repeat_IsRepeat;
+
+
+
+
+
+intervalList<int32> *
+computeCoverage(MultiAlignT *ma) {
+ uint32 maNum = GetNumIntMultiPoss(ma->f_list);
+ intervalList<int32> IL;
+
+ for (uint32 ii=0; ii<maNum; ii++) {
+ IntMultiPos *imp = GetIntMultiPos(ma->f_list, ii);
+
+ int32 bgn = MIN(imp->position.bgn, imp->position.end);
+ int32 end = MAX(imp->position.bgn, imp->position.end);
+ int32 len = end - bgn;
+
+ IL.add(bgn, len);
+ }
+
+ return(new intervalList<int32>(IL));
+}
+
+
+
+
+
+
+
+int
+main(int argc, char **argv) {
+ char *gkpName = NULL;
+ char *tigName = NULL;
+ int32 tigVers = -1;
+
+ int64 genomeSize = 0;
+
+ char outName[FILENAME_MAX];
+ char *outPrefix = NULL;
+
+ FILE *outLOG = NULL;
+ FILE *outSTA = NULL;
+
+ // MicroHet probability is actually the probability of the sequence being UNIQUE, based on
+ // microhet considerations. Falling below threshhold makes something a repeat.
+ double cgbApplyMicrohetCutoff = -1; // Basically turns it off, unless enabled
+ double cgbMicrohetProb = 1.e-5; // Scores less than this are considered repeats
+
+ double cgbUniqueCutoff = CGB_UNIQUE_CUTOFF;
+ double cgbDefinitelyUniqueCutoff = CGB_UNIQUE_CUTOFF;
+ double singleReadMaxCoverage = 1.0; // Reads covering more than this will demote the unitig
+ uint32 lowCovDepth = 2;
+ double lowCovFractionAllowed = 1.0;
+ uint32 tooLong = UINT32_MAX;
+ uint32 tooShort = 1000;
+ uint32 minReads = 2;
+ double minPopulous = 0;
+
+ AS_IID bgnID = 0;
+ AS_IID endID = 0;
+ AS_IID maxID = 0;
+
+ MultiAlignStore *tigStore = NULL;
+ gkStore *gkpStore = NULL;
+
+ bool doUpdate = true;
+ const bool isUnitig = true; // Required for tigStore accessor
+
+ argc = AS_configure(argc, argv);
+
+ int err = 0;
+ int arg = 1;
+ while (arg < argc) {
+ if (strcmp(argv[arg], "-g") == 0) {
+ gkpName = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-t") == 0) {
+ tigName = argv[++arg];
+ tigVers = atoi(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-o") == 0) {
+ outPrefix = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-n") == 0) {
+ doUpdate = false;
+
+
+ } else if (strcmp(argv[arg], "-e") == 0) {
+ cgbMicrohetProb = atof(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-i") == 0) {
+ cgbApplyMicrohetCutoff = atof(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-j") == 0) {
+ cgbUniqueCutoff = atof(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-k") == 0) {
+ cgbDefinitelyUniqueCutoff = atof(argv[++arg]);
+
+
+ } else if (strcmp(argv[arg], "-span") == 0) {
+ singleReadMaxCoverage = atof(argv[++arg]); // If a single read spans more than this fraction of unitig, it is demoted
+
+ } else if (strcmp(argv[arg], "-lowcov") == 0) {
+ lowCovDepth = atoi(argv[++arg]); // Coverage below this is too low
+ lowCovFractionAllowed = atof(argv[++arg]); // If unitig has more than this fraction low coverage, it is demoted
+
+ } else if (strcmp(argv[arg], "-reads") == 0) {
+ arg++;
+ minReads = atoi(argv[arg]); // If unitig has fewer than this number of reads it is demoted
+ minPopulous = atof(argv[arg]);
+
+ if (minPopulous > 1.0)
+ minPopulous = 0;
+
+ } else if (strcmp(argv[arg], "-long") == 0) {
+ tooLong = atoi(argv[++arg]); // Unitigs longer than this cannot be demoted
+
+ } else if (strcmp(argv[arg], "-short") == 0) {
+ tooShort = atoi(argv[++arg]); // Unitigs shorter than this are demoted
+
+
+ } else {
+ err++;
+ }
+
+ arg++;
+ }
+
+ if (gkpName == NULL)
+ err++;
+ if (tigName == NULL)
+ err++;
+
+ if (err) {
+ fprintf(stderr, "usage: %s -g gkpStore -t tigStore version\n", argv[0]);
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -g <G> Mandatory, path G to a gkpStore directory.\n");
+ fprintf(stderr, " -t <T> <v> Mandatory, path T to a tigStore, and version V.\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -e P Microhet probability (default 1e-5)\n");
+ fprintf(stderr, " -i C Microhet cutoff (default -1)\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -j J Unitig is not unique if astat is below J (cgbUniqueCutoff)\n");
+ fprintf(stderr, " -k K (unused) (cgbDefinitelyUniqueCutoff)\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -span F Unitig is not unique if a single read spans more than fraction F (default 1.0) of unitig\n");
+ fprintf(stderr, " -lowcov D F Unitig is not unique if fraction F (default 1.0) of unitig is below read depth D (default 2)\n");
+ fprintf(stderr, " -reads R Unitig is not unique if unitig has fewer than R (default 2) reads\n");
+ fprintf(stderr, " If R is fractional, the least populous unitigs containing fraction R of reads are marked as repeat\n");
+ fprintf(stderr, " Example: unitigs with 9 or fewer reads contain 10%% of the reads. -reads 0.10 would mark these are repeat.\n");
+ fprintf(stderr, " -long L Unitig is unique if unitig is at least L (default unlimited) bases long\n");
+ fprintf(stderr, " -short S Unitig is not unique if unitig is shorter than S (default 1000) bases long\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -o <name> Prefix for output files.\n");
+ fprintf(stderr, " -n Do not update the tigStore.\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Algorithm: The first rule to trigger will mark the unitig.\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " 1) A unitig with a single read is NOT unique.\n");
+ fprintf(stderr, " 2) A unitig with fewer than R (-reads) reads is NOT unique.\n");
+ fprintf(stderr, " 3) A unitig with a single read spanning fraction F (-span) of the unitig is NOT unique.\n");
+ fprintf(stderr, " 4) A unitig longer than L (-length) bases IS unique.\n");
+ fprintf(stderr, " 5) A unitig with astat less than J (-j) is NOT unique.\n");
+ fprintf(stderr, " 6) A unitig with microhet probability P (-e) and astat below C (-i) is NOT unique.\n");
+ fprintf(stderr, " 7) A unitig with fraction F below coverage D (-lowcov) is NOT unique.\n");
+ fprintf(stderr, " 8) A unitig shorter than S (-short) bases long is NOT unique.\n");
+ fprintf(stderr, " 9) Otherwise, the unitig IS unique.\n");
+
+ if (gkpName == NULL)
+ fprintf(stderr, "No gatekeeper store (-g option) supplied.\n");
+
+ if (tigName == NULL)
+ fprintf(stderr, "No input tigStore (-t option) supplied.\n");
+
+ if (outPrefix == NULL)
+ fprintf(stderr, "No output prefix (-o option) supplied.\n");
+
+ exit(1);
+ }
+
+ gkpStore = new gkStore(gkpName, false, false);
+ tigStore = new MultiAlignStore(tigName, tigVers, 0, 0, doUpdate, doUpdate, false);
+
+ if (endID == 0)
+ endID = tigStore->numUnitigs();
+
+ maxID = tigStore->numUnitigs();
+
+ errno = 0;
+
+ sprintf(outName, "%s.log", outPrefix);
+
+ outLOG = fopen(outName, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1);
+
+ fprintf(outLOG, "tigID\trho\tcovStat\tarrDist\n");
+
+ sprintf(outName, "%s.stats", outPrefix);
+
+ outSTA = fopen(outName, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1);
+
+ fprintf(stderr, "Command Line options:\n");
+ fprintf(stderr, " singleReadMaxCoverage %f\n", singleReadMaxCoverage);
+ fprintf(stderr, " lowCoverage %u coverage %f fraction\n", lowCovDepth, lowCovFractionAllowed);
+ if (minPopulous > 0)
+ fprintf(stderr, " minReads recomputed based on least populous fraction %.4f\n", minPopulous);
+ else
+ fprintf(stderr, " minReads %u\n", minReads);
+ fprintf(stderr, " tooLong %u\n", tooLong);
+ fprintf(stderr, " tooShort %u\n", tooShort);
+
+ //
+ // Load fragment data
+ //
+
+ fprintf(stderr, "Loading fragment data.\n");
+
+ double globalRate = 0;
+
+ bool *isNonRandom = new bool [gkpStore->gkStore_getNumFragments() + 1];
+ uint32 *fragLength = new uint32 [gkpStore->gkStore_getNumFragments() + 1];
+
+ gkStream *fs = new gkStream(gkpStore, 0, 0, GKFRAGMENT_INF);
+ gkFragment fr;
+
+ while(fs->next(&fr)) {
+ uint32 iid = fr.gkFragment_getReadIID();
+
+ isNonRandom[iid] = fr.gkFragment_getIsNonRandom();
+ fragLength[iid] = fr.gkFragment_getClearRegionLength(AS_READ_CLEAR_OBTCHIMERA);
+
+ if ((iid % 10000000) == 0)
+ fprintf(stderr, "Loading fragment information %9d out of %9d\n", iid, gkpStore->gkStore_getNumFragments());
+ }
+
+ delete fs;
+
+ //
+ // Compute:
+ // global coverage histogram
+ // global single-read fraction covered - number of bases in unitigs with specific fraction covered
+ // global number of reads per unitig
+ //
+
+ fprintf(stderr, "Generating statistics.\n");
+
+ uint32 covHistogramMax = 1048576;
+ uint32 *covHistogram = new uint32 [covHistogramMax];
+ uint32 **utgCovHistogram = new uint32 * [maxID];
+ uint32 *utgCovData = new uint32 [maxID * lowCovDepth];
+
+ memset(covHistogram, 0, sizeof(uint32) * covHistogramMax);
+ memset(utgCovHistogram, 0, sizeof(uint32) * maxID);
+ memset(utgCovData, 0, sizeof(uint32) * maxID * lowCovDepth);
+
+ uint32 *singleReadCoverageHistogram = new uint32 [1001];
+ double *singleReadCoverage = new double [maxID];
+
+ memset(singleReadCoverageHistogram, 0, sizeof(uint32) * 1001);
+ memset(singleReadCoverage, 0, sizeof(double) * maxID);
+
+ uint32 numReadsPerUnitigMax = maxID + 1;
+ uint32 *numReadsPerUnitig = new uint32 [numReadsPerUnitigMax];
+
+ memset(numReadsPerUnitig, 0, sizeof(uint32) * numReadsPerUnitigMax);
+
+ for (uint32 uu=0; uu<maxID; uu++) {
+ MultiAlignT *ma = tigStore->loadMultiAlign(uu, isUnitig);
+
+ if (ma == NULL)
+ continue;
+
+ utgCovHistogram[ma->maID] = utgCovData + ma->maID * lowCovDepth;
+
+ if (GetNumIntMultiPoss(ma->f_list) == 1)
+ continue;
+
+ // This MUST use gapped lengths, otherwise we'd need to translate all the read coords from
+ // gapped to ungapped.
+
+ uint32 maLen = GetMultiAlignLength(ma);
+ uint32 maNum = GetNumIntMultiPoss(ma->f_list);
+
+ assert(ma->data.num_frags == GetNumIntMultiPoss(ma->f_list));
+
+ // Global coverage histogram.
+
+ intervalList<int32> *ID = computeCoverage(ma);
+
+ for (uint32 ii=0; ii<ID->numberOfIntervals(); ii++) {
+ if (ID->depth(ii) < lowCovDepth)
+ utgCovHistogram[ma->maID][ID->depth(ii)] += ID->hi(ii) - ID->lo(ii) + 1;
+
+ if (ID->depth(ii) < covHistogramMax)
+ covHistogram[ID->depth(ii)] += ID->hi(ii) - ID->lo(ii) + 1;
+ }
+
+ // Single read max fraction covered.
+
+ uint32 covMax = 0;
+ uint32 cov;
+
+ for (uint32 ff=0; ff<maNum; ff++) {
+ IntMultiPos *frg = GetIntMultiPos(ma->f_list, ff);
+
+ if (frg->position.bgn < frg->position.end)
+ cov = 1000 * (frg->position.end - frg->position.bgn) / maLen;
+ else
+ cov = 1000 * (frg->position.bgn - frg->position.end) / maLen;
+
+ if (covMax < cov)
+ covMax = cov;
+ }
+
+ singleReadCoverageHistogram[covMax]++;
+
+ singleReadCoverage[ma->maID] = covMax / 1000.0;
+
+ // Number of reads per unitig
+
+ numReadsPerUnitig[uu] = maNum;
+
+ //fprintf(stderr, "unitig %u covMax %f\n", ma->maID, covMax / 1000.0);
+ }
+
+ //
+ // Analyze our collected data, decide on some thresholds.
+ //
+
+ fprintf(stderr, "Analyzing statistics.\n");
+
+ if ((minPopulous > 0.0) &&
+ (minPopulous < 1.0)) {
+ uint32 maxReadsPerUnitig = 0;
+
+ for (uint32 uu=0; uu<maxID; uu++)
+ maxReadsPerUnitig = MAX(maxReadsPerUnitig, numReadsPerUnitig[uu] + 1);
+
+ uint32 *totReadsPerNumReads = new uint32 [maxReadsPerUnitig];
+ uint32 totReads = 0;
+
+ memset(totReadsPerNumReads, 0, sizeof(uint32) * maxReadsPerUnitig + 1);
+
+ for (uint32 uu=0; uu<maxID; uu++) {
+ totReadsPerNumReads[numReadsPerUnitig[uu]] += numReadsPerUnitig[uu];
+ totReads += numReadsPerUnitig[uu];
+ }
+
+ double xx = 0.0;
+
+ for (minReads=0; xx < minPopulous; minReads++)
+ xx += (double)totReadsPerNumReads[minReads] / totReads;
+
+ minReads--;
+ xx -= (double)totReadsPerNumReads[minReads] / totReads;
+
+ fprintf(stderr, " minReads %u based on least populous unitigs with fraction %.4f of reads (wanted fraction %.4f)\n",
+ minReads, xx, minPopulous);
+
+ uint32 hist[101] = {0};
+ double vals[101] = {0};
+
+ xx = 0.0;
+
+ for (uint32 ff=0; ff<maxReadsPerUnitig; ff++) {
+ xx += (double)totReadsPerNumReads[ff] / totReads;
+ assert(xx <= 1.0 + 1e9 * ff); // For rounding issues
+ hist[(int32)(100 * xx)] = ff;
+ vals[(int32)(100 * xx)] = xx;
+ }
+ for (uint32 xx=0; xx<101; xx++)
+ if (hist[xx] > 0)
+ fprintf(outSTA, "minReads %u with fraction %.04f of reads\n", hist[xx], vals[xx]);
+
+ delete [] totReadsPerNumReads;
+ }
+
+
+ //
+ // Apply the thresholds to unitigs. The first half of these are the historical CGW rules.
+ //
+
+ fprintf(stderr, "Processing unitigs.\n");
+
+ for (uint32 uu=bgnID; uu<endID; uu++) {
+ MultiAlignT *ma = tigStore->loadMultiAlign(uu, isUnitig);
+
+ if (ma == NULL) {
+ fprintf(outLOG, "unitig %d not present\n", uu);
+ continue;
+ }
+
+ // This uses UNGAPPED lengths, because they make more sense to humans.
+
+ uint32 maLen = GetMultiAlignUngappedLength(ma);
+ uint32 maNum = GetNumIntMultiPoss(ma->f_list);
+
+ uint32 lowCovBases = 0;
+ for (uint32 ll=0; ll<lowCovDepth; ll++)
+ lowCovBases += utgCovHistogram[ma->maID][ll];
+
+ bool isUnique = true;
+ bool isSingleton = false;
+
+
+ if (maNum == 1) {
+ fprintf(outLOG, "unitig %d not unique -- singleton\n",
+ ma->maID);
+ isUnique = false;
+ isSingleton = true;
+ }
+
+ else if (maNum < minReads) {
+ fprintf(outLOG, "unitig %d not unique -- %u reads, need at least %d\n",
+ ma->maID, maNum, minReads);
+ repeat_LowReads += maLen;
+ isUnique = false;
+ }
+
+ else if (singleReadCoverage[ma->maID] > singleReadMaxCoverage) {
+ fprintf(outLOG, "unitig %d not unique -- single read spans fraction %f of unitig (>= %f)\n",
+ ma->maID,
+ singleReadCoverage[ma->maID],
+ singleReadMaxCoverage);
+ repeat_SingleSpan += maLen;
+ isUnique = false;
+ }
+
+ else if (maLen >= tooLong) {
+ fprintf(outLOG, "unitig %d IS unique -- too long to be repeat, %u > allowed %u\n",
+ ma->maID,
+ maLen, tooLong);
+ isUnique = true;
+ }
+
+ else if (tigStore->getUnitigCoverageStat(ma->maID) < cgbUniqueCutoff) {
+ fprintf(outLOG, "unitig %d not unique -- coverage stat %d, needs to be at least %f\n",
+ ma->maID, tigStore->getUnitigCoverageStat(ma->maID), cgbUniqueCutoff);
+ repeat_LowCovStat += maLen;
+ isUnique = false;
+ }
+
+ else if ((tigStore->getUnitigMicroHetProb(ma->maID) < cgbMicrohetProb) &&
+ (tigStore->getUnitigCoverageStat(ma->maID) < cgbApplyMicrohetCutoff)) {
+ fprintf(outLOG, "unitig %d not unique -- low microhetprob %f (< %f) and low coverage stat %d (< %f)\n",
+ ma->maID,
+ tigStore->getUnitigMicroHetProb(ma->maID), cgbMicrohetProb,
+ tigStore->getUnitigCoverageStat(ma->maID), cgbApplyMicrohetCutoff);
+ repeat_MicroHet += maLen;
+ isUnique = false;
+ }
+
+ else if ((double)lowCovBases / maLen > lowCovFractionAllowed) {
+ fprintf(outLOG, "unitig %d not unique -- too many low coverage bases, %u out of %u bases, fraction %f > allowed %f\n",
+ ma->maID,
+ lowCovBases, maLen,
+ (double)lowCovBases / maLen, lowCovFractionAllowed);
+ repeat_LowCov += maLen;
+ isUnique = false;
+ }
+
+ // This was an attempt to not blindly call all short unitigs as non-unique. It didn't work so
+ // well in initial limited testing. The threshold is arbitrary; older versions used
+ // cgbDefinitelyUniqueCutoff. If used, be sure to disable the real check after this!
+#if 0
+ else if ((tigStore->getUnitigCoverageStat(ma->maID) < cgbUniqueCutoff * 10) &&
+ (maLen < CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH)) {
+ fprintf(outLOG, "unitig %d not unique -- length %d too short, need to be at least %d AND coverage stat %d must be larger than %d\n",
+ ma->maID, maLen, CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH,
+ tigStore->getUnitigCoverageStat(ma->maID), cgbUniqueCutoff * 10);
+ repeat_Short += maLen;
+ isUnique = false;
+ }
+#endif
+
+ else if (maLen < tooShort) {
+ fprintf(outLOG, "unitig %d not unique -- length %d too short, need to be at least %d\n",
+ ma->maID, maLen, tooShort);
+ repeat_Short += maLen;
+ isUnique = false;
+ }
+
+ else {
+ fprintf(outLOG, "unitig %d not repeat -- no test failed\n", ma->maID);
+ }
+
+ //
+ // Allow flag to override the rules and force it to be unique or repeat. AKA, toggling.
+ //
+
+ if (isUnique) {
+ repeat_IsUnique += maLen;
+ tigStore->setUnitigSuggestUnique(ma->maID);
+ tigStore->setUnitigSuggestRepeat(ma->maID, false);
+
+ } else if (isSingleton) {
+ repeat_IsSingleton += maLen;
+ tigStore->setUnitigSuggestUnique(ma->maID, false);
+ tigStore->setUnitigSuggestRepeat(ma->maID);
+
+ } else {
+ repeat_IsRepeat += maLen;
+ tigStore->setUnitigSuggestUnique(ma->maID, false);
+ tigStore->setUnitigSuggestRepeat(ma->maID);
+ }
+ }
+
+
+ fprintf(stderr, "\n");
+ //fprintf(stderr, "Processed %d unitigs with %d fragments.\n", 0, 0);
+ fprintf(stderr, "\n");
+ fprintf(stderr, "classification number of unitigs total length\n");
+ fprintf(stderr, " unique: %17"F_U32P" %14"F_U64P"\n", repeat_IsUnique.num, repeat_IsUnique.len);
+ fprintf(stderr, " singleton: %17"F_U32P" %14"F_U64P"\n", repeat_IsSingleton.num, repeat_IsSingleton.len);
+ fprintf(stderr, " repeat: %17"F_U32P" %14"F_U64P"\n", repeat_IsRepeat.num, repeat_IsRepeat.len);
+ fprintf(stderr, " too few reads: %17"F_U32P" %14"F_U64P"\n", repeat_LowReads.num, repeat_LowReads.len);
+ fprintf(stderr, " low cov stat: %17"F_U32P" %14"F_U64P"\n", repeat_LowCovStat.num, repeat_LowCovStat.len);
+ fprintf(stderr, " too short: %17"F_U32P" %14"F_U64P"\n", repeat_Short.num, repeat_Short.len);
+ fprintf(stderr, " microhet: %17"F_U32P" %14"F_U64P"\n", repeat_MicroHet.num, repeat_MicroHet.len);
+ fprintf(stderr, " spanning read: %17"F_U32P" %14"F_U64P"\n", repeat_SingleSpan.num, repeat_SingleSpan.len);
+ fprintf(stderr, " low coverage: %17"F_U32P" %14"F_U64P"\n", repeat_LowCov.num, repeat_LowCov.len);
+
+
+
+ fclose(outLOG);
+ fclose(outSTA);
+
+ delete [] isNonRandom;
+ delete [] fragLength;
+
+ delete gkpStore;
+ delete tigStore;
+}
diff --git a/src/AS_BAT/plot-break-points.pl b/src/AS_BAT/plot-break-points.pl
new file mode 100644
index 0000000..f2a1cdb
--- /dev/null
+++ b/src/AS_BAT/plot-break-points.pl
@@ -0,0 +1,162 @@
+#!/usr/bin/perl
+
+use strict;
+
+my %nucmer;
+my %length;
+
+if (! -e "tig.fasta") {
+}
+
+if (! -e "tig.coords") {
+}
+
+if (-e "tig.fasta") {
+ open(F, "< tig.fasta") or die;
+ while (<F>) {
+ if (m/^>(utg\d+)\s+len=(\d+)$/) {
+ $length{$1} = $2;
+ }
+ }
+ close(F);
+} else {
+ print STDERR "No tig.fasta file found, cannot .....\n";
+}
+
+if (-e "tig.coords") {
+ open(F, "< tig.coords") or die;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ $_ = <F>;
+ while (<F>) {
+ s/^\s+//;
+ s/\s$+//;
+ my @v = split '\s+', $_;
+ my $str = "$v[3]\t$v[4]";
+ if (exists($nucmer{$v[12]})) {
+ $nucmer{$v[12]} .= "\t" . $str;
+ } else {
+ $nucmer{$v[12]} = $str;
+ #print "$v[12] -- $str\n";
+ }
+ }
+ close(F);
+} else {
+ print STDERR "No tig.coords file found, not plotting true break points.\n";
+}
+
+
+open(F, "find coverageplot -name '*.badCoverage' -print | sort |");
+my @files = <F>;
+chomp @files;
+close(F);
+
+
+
+#undef @files;
+#push @files, "utg000002841.badCoverage";
+#push @files, "utg000008002.badCoverage";
+#push @files, "utg000008093.badCoverage";
+
+foreach my $file (@files) {
+ my $num = 0;
+
+ if ($file =~ m/utg0+(\d+).badCoverage/) {
+ $num = $1;
+ } else {
+ print STDERR "No match '$file'\n";
+ exit(1);
+ }
+
+ open(GP, "> $file.gnuplot");
+
+ print GP "set terminal png size 1280,800\n";
+ print GP "set output \"$file.png\"\n";
+
+ my $cnt = 1;
+
+ open(F, "cat unitigger.*.intersectionBreaking.log | ") or die;
+ while (<F>) {
+ if (m/BREAK unitig (\d+) at position (\d+),(\d+) from inSize (\d+) inFrags (\d+)./) {
+ if ($1 == $num) {
+ my $s = int(-$4 / 100); # Negative: length / 100
+ my $f = int( $5 / 10); # Positive: num frags / 10
+ #print GP "set object rect from $2,$s to $3,$f fc lt 2\n";
+ print GP "set object rect from $2,$s to $3,$f fc lt $cnt\n";
+ $cnt++;
+ }
+ }
+ if (m/BREAK unitig (\d+) at position (\d+),(\d+) from MATES./) {
+ if ($1 == $num) {
+ my $s = -0.5;
+ my $f = 0.5;
+ print GP "set object rect from $2,$s to $3,$f fc lt 0\n";
+ $cnt++;
+ }
+ }
+ }
+ close(F);
+
+ if (exists($nucmer{"utg$1"})) {
+ my $positionB = 5;
+ my $positionT = 5.5;
+ my $nucmer = $nucmer{"utg$1"};
+ my @L = split '\t', $nucmer;
+
+ #print "utg$1 -- $nucmer\n";
+
+ delete $nucmer{"utg$1"};
+
+ while (scalar(@L) > 1) {
+ my $b = shift @L;
+ my $e = shift @L;
+
+ print GP "set object rect from $b,$positionB to $e,$positionT fc lt 0\n";
+ $positionB++;
+ $positionT++;
+ }
+ }
+
+
+ print GP "plot \\\n";
+ #print GP " \"$file\" using 1:3 with lines title \"fwdBad\", \\\n";
+ #print GP " \"$file\" using 1:4 with lines title \"revBad\", \\\n";
+ print GP " \"$file\" using 1:5 with lines title \"badExtFwd\", \\\n";
+ print GP " \"$file\" using 1:6 with lines title \"badExtRev\", \\\n";
+ print GP " \"$file\" using 1:7 with lines title \"badCompressed\", \\\n";
+ print GP " \"$file\" using 1:8 with lines title \"badStretched\", \\\n";
+ print GP " \"$file\" using 1:9 with lines title \"badNormal\", \\\n";
+ print GP " \"$file\" using 1:10 with lines title \"badAnti\", \\\n";
+ print GP " \"$file\" using 1:11 with lines title \"badOuttie\", \\\n";
+ print GP " \"$file\" using 1:2 with lines title \"Good\"\n";
+ print GP "\n";
+
+ close(GP);
+
+ print STDERR "$file\n";
+ system("gnuplot < $file.gnuplot > /dev/null 2>&1");
+}
+
+
+foreach my $k (keys %nucmer) {
+ my $nucmer = $nucmer{$k};
+ my @L = split '\t', $nucmer;
+
+ my $c = 0;
+ my $t = 0;
+
+ while (scalar(@L) > 0) {
+ my $b = shift @L;
+ my $e = shift @L;
+ my $l = ($b < $e) ? ($e - $b) : ($b - $e);
+
+ $t++;
+ $c++ if (($l < $length{$k} - 10) && ($l > 150));
+ }
+
+ if ($c > 1) {
+ print STDERR "$k with $c partial out of $t total regions. length $length{$k}.\n";
+ }
+}
diff --git a/src/AS_BAT/show-false-best-edges-from-mapping.pl b/src/AS_BAT/show-false-best-edges-from-mapping.pl
new file mode 100644
index 0000000..acd7512
--- /dev/null
+++ b/src/AS_BAT/show-false-best-edges-from-mapping.pl
@@ -0,0 +1,87 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $asm = shift @ARGV;
+
+# Reads:
+# $asm//best.edges
+# filtered-overlaps.true.ova
+# filtered-overlaps.false.ova
+#
+# Reports which best.edges are false
+
+my %true;
+my %false;
+
+open(F, "< filtered-overlaps.true.ova") or die;
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ $true{"$v[0]-$v[1]"}++;
+ $true{"$v[1]-$v[0]"}++;
+}
+close(F);
+
+
+open(F, "< filtered-overlaps.false.ova") or die;
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ $false{"$v[0]-$v[1]"}++;
+ $false{"$v[1]-$v[0]"}++;
+}
+close(F);
+
+
+ my $true5 = 0;
+ my $true3 = 0;
+ my $false5 = 0;
+ my $false3 = 0;
+ my $novel5 = 0;
+ my $novel3 = 0;
+
+open(F, "< $asm/best.edges") or die;
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ my $p5 = "$v[0]-$v[2]";
+ my $p3 = "$v[0]-$v[4]";
+
+ if (exists($true{$p5})) {
+ $true5++;
+ } elsif (exists($false{$p5})) {
+ print STDERR "FALSE 5 $_\n";
+ $false5++;
+ } else {
+ print STDERR "NOVEL 5 $_\n";
+ $novel5++;
+ }
+
+ if (exists($true{$p3})) {
+ $true3++;
+ } elsif (exists($false{$p3})) {
+ print STDERR "FALSE 3 $_\n";
+ $false3++;
+ } else {
+ print STDERR "NOVEL 3 $_\n";
+ $novel3++;
+ }
+}
+close(F);
+
+print "true5 $true5\n";
+print "true3 $true3\n";
+print "false5 $false5\n";
+print "false3 $false3\n";
+print "novel5 $novel5\n";
+print "novel3 $novel3\n";
diff --git a/src/AS_BOG/._AS_BOG_BestOverlapGraph.C b/src/AS_BOG/._AS_BOG_BestOverlapGraph.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_BestOverlapGraph.C differ
diff --git a/src/AS_BOG/._AS_BOG_BestOverlapGraph.H b/src/AS_BOG/._AS_BOG_BestOverlapGraph.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_BestOverlapGraph.H differ
diff --git a/src/AS_BOG/._AS_BOG_Breaking.C b/src/AS_BOG/._AS_BOG_Breaking.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Breaking.C differ
diff --git a/src/AS_BOG/._AS_BOG_ChunkGraph.C b/src/AS_BOG/._AS_BOG_ChunkGraph.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_ChunkGraph.C differ
diff --git a/src/AS_BOG/._AS_BOG_ChunkGraph.H b/src/AS_BOG/._AS_BOG_ChunkGraph.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_ChunkGraph.H differ
diff --git a/src/AS_BOG/._AS_BOG_Datatypes.H b/src/AS_BOG/._AS_BOG_Datatypes.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Datatypes.H differ
diff --git a/src/AS_BOG/._AS_BOG_EvaluateMates.C b/src/AS_BOG/._AS_BOG_EvaluateMates.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_EvaluateMates.C differ
diff --git a/src/AS_BOG/._AS_BOG_FragmentInfo.C b/src/AS_BOG/._AS_BOG_FragmentInfo.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_FragmentInfo.C differ
diff --git a/src/AS_BOG/._AS_BOG_InsertSizes.C b/src/AS_BOG/._AS_BOG_InsertSizes.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_InsertSizes.C differ
diff --git a/src/AS_BOG/._AS_BOG_InsertSizes.H b/src/AS_BOG/._AS_BOG_InsertSizes.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_InsertSizes.H differ
diff --git a/src/AS_BOG/._AS_BOG_Instrumentation.C b/src/AS_BOG/._AS_BOG_Instrumentation.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Instrumentation.C differ
diff --git a/src/AS_BOG/._AS_BOG_IntersectBubble.C b/src/AS_BOG/._AS_BOG_IntersectBubble.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_IntersectBubble.C differ
diff --git a/src/AS_BOG/._AS_BOG_IntersectSplit.C b/src/AS_BOG/._AS_BOG_IntersectSplit.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_IntersectSplit.C differ
diff --git a/src/AS_BOG/._AS_BOG_Joining.C b/src/AS_BOG/._AS_BOG_Joining.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Joining.C differ
diff --git a/src/AS_BOG/._AS_BOG_MateBubble.C b/src/AS_BOG/._AS_BOG_MateBubble.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_MateBubble.C differ
diff --git a/src/AS_BOG/._AS_BOG_MateChecker.C b/src/AS_BOG/._AS_BOG_MateChecker.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_MateChecker.C differ
diff --git a/src/AS_BOG/._AS_BOG_MateLocation.C b/src/AS_BOG/._AS_BOG_MateLocation.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_MateLocation.C differ
diff --git a/src/AS_BOG/._AS_BOG_MateLocation.H b/src/AS_BOG/._AS_BOG_MateLocation.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_MateLocation.H differ
diff --git a/src/AS_BOG/._AS_BOG_MoveContains.C b/src/AS_BOG/._AS_BOG_MoveContains.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_MoveContains.C differ
diff --git a/src/AS_BOG/._AS_BOG_Outputs.C b/src/AS_BOG/._AS_BOG_Outputs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Outputs.C differ
diff --git a/src/AS_BOG/._AS_BOG_OverlapBubble.C b/src/AS_BOG/._AS_BOG_OverlapBubble.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_OverlapBubble.C differ
diff --git a/src/AS_BOG/._AS_BOG_PlaceContains.C b/src/AS_BOG/._AS_BOG_PlaceContains.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_PlaceContains.C differ
diff --git a/src/AS_BOG/._AS_BOG_PlaceFragUsingOverlaps.C b/src/AS_BOG/._AS_BOG_PlaceFragUsingOverlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_PlaceFragUsingOverlaps.C differ
diff --git a/src/AS_BOG/._AS_BOG_PlaceZombies.C b/src/AS_BOG/._AS_BOG_PlaceZombies.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_PlaceZombies.C differ
diff --git a/src/AS_BOG/._AS_BOG_PopulateUnitig.C b/src/AS_BOG/._AS_BOG_PopulateUnitig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_PopulateUnitig.C differ
diff --git a/src/AS_BOG/._AS_BOG_SetParentAndHang.C b/src/AS_BOG/._AS_BOG_SetParentAndHang.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_SetParentAndHang.C differ
diff --git a/src/AS_BOG/._AS_BOG_SplitDiscontinuous.C b/src/AS_BOG/._AS_BOG_SplitDiscontinuous.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_SplitDiscontinuous.C differ
diff --git a/src/AS_BOG/._AS_BOG_Unitig.C b/src/AS_BOG/._AS_BOG_Unitig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Unitig.C differ
diff --git a/src/AS_BOG/._AS_BOG_Unitig.H b/src/AS_BOG/._AS_BOG_Unitig.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Unitig.H differ
diff --git a/src/AS_BOG/._AS_BOG_UnitigGraph.C b/src/AS_BOG/._AS_BOG_UnitigGraph.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_UnitigGraph.C differ
diff --git a/src/AS_BOG/._AS_BOG_UnitigGraph.H b/src/AS_BOG/._AS_BOG_UnitigGraph.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_UnitigGraph.H differ
diff --git a/src/AS_BOG/._AS_BOG_Unitig_AddAndPlaceFrag.C b/src/AS_BOG/._AS_BOG_Unitig_AddAndPlaceFrag.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Unitig_AddAndPlaceFrag.C differ
diff --git a/src/AS_BOG/._AS_BOG_Unitig_AddFrag.C b/src/AS_BOG/._AS_BOG_Unitig_AddFrag.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Unitig_AddFrag.C differ
diff --git a/src/AS_BOG/._AS_BOG_Unitig_PlaceFragUsingEdges.C b/src/AS_BOG/._AS_BOG_Unitig_PlaceFragUsingEdges.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._AS_BOG_Unitig_PlaceFragUsingEdges.C differ
diff --git a/src/AS_BOG/._BuildUnitigs.C b/src/AS_BOG/._BuildUnitigs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._BuildUnitigs.C differ
diff --git a/src/AS_BOG/._Makefile b/src/AS_BOG/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._Makefile differ
diff --git a/src/AS_BOG/._analyze-unitig-mates-from-tigStore.pl b/src/AS_BOG/._analyze-unitig-mates-from-tigStore.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._analyze-unitig-mates-from-tigStore.pl differ
diff --git a/src/AS_BOG/._analyzeBest.C b/src/AS_BOG/._analyzeBest.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._analyzeBest.C differ
diff --git a/src/AS_BOG/._find-breaks.pl b/src/AS_BOG/._find-breaks.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._find-breaks.pl differ
diff --git a/src/AS_BOG/._fixUnitigs.C b/src/AS_BOG/._fixUnitigs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._fixUnitigs.C differ
diff --git a/src/AS_BOG/._fixUnitigs.cases b/src/AS_BOG/._fixUnitigs.cases
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._fixUnitigs.cases differ
diff --git a/src/AS_BOG/._mate-based-splitting b/src/AS_BOG/._mate-based-splitting
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._mate-based-splitting differ
diff --git a/src/AS_BOG/._plot-break-points.pl b/src/AS_BOG/._plot-break-points.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._plot-break-points.pl differ
diff --git a/src/AS_BOG/._plot-log-length.pl b/src/AS_BOG/._plot-log-length.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_BOG/._plot-log-length.pl differ
diff --git a/src/AS_BOG/AS_BOG_Breaking.C b/src/AS_BOG/AS_BOG_Breaking.C
index c626ee6..3d70af7 100644
--- a/src/AS_BOG/AS_BOG_Breaking.C
+++ b/src/AS_BOG/AS_BOG_Breaking.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BOG_Breaking.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BOG_Breaking.C 4560 2014-08-22 21:58:53Z brianwalenz $";
#include "AS_BOG_Datatypes.H"
#include "AS_BOG_UnitigGraph.H"
@@ -108,7 +108,7 @@ void UnitigGraph::filterBreakPoints(ContainerMap &cMap,
// Check for sentinal at end, not added by MateChecker
UnitigBreakPoint fakeEnd = breaks.back();
- if (fakeEnd.inSize == std::numeric_limits<int>::max())
+ if (fakeEnd.inSize == INT32_MAX)
breaks.pop_back();
UnitigBreakPoints smallBPs;
diff --git a/src/AS_BOG/AS_BOG_Datatypes.H b/src/AS_BOG/AS_BOG_Datatypes.H
index 5e9df85..f6eff20 100644
--- a/src/AS_BOG/AS_BOG_Datatypes.H
+++ b/src/AS_BOG/AS_BOG_Datatypes.H
@@ -22,7 +22,7 @@
#ifndef INCLUDE_AS_BOG_DATATYPES
#define INCLUDE_AS_BOG_DATATYPES
-static const char *rcsid_INCLUDE_AS_BOG_DATATYPES = "$Id: AS_BOG_Datatypes.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_INCLUDE_AS_BOG_DATATYPES = "$Id: AS_BOG_Datatypes.H 4522 2014-04-11 19:52:56Z brianwalenz $";
#include "AS_global.H"
#include "AS_MSG_pmesg.H"
@@ -33,9 +33,6 @@ static const char *rcsid_INCLUDE_AS_BOG_DATATYPES = "$Id: AS_BOG_Datatypes.H 437
#include <set>
#include <list>
#include <vector>
-#include <iostream>
-#include <cmath>
-#include <limits>
#include <algorithm>
using namespace std;
diff --git a/src/AS_BOG/AS_BOG_IntersectSplit.C b/src/AS_BOG/AS_BOG_IntersectSplit.C
index 22d19e9..8e985ca 100644
--- a/src/AS_BOG/AS_BOG_IntersectSplit.C
+++ b/src/AS_BOG/AS_BOG_IntersectSplit.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BOG_IntersectSplit.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BOG_IntersectSplit.C 4560 2014-08-22 21:58:53Z brianwalenz $";
#include "AS_BOG_Datatypes.H"
#include "AS_BOG_UnitigGraph.H"
@@ -259,7 +259,7 @@ void UnitigGraph::breakUnitigs(ContainerMap &cMap, char *output_prefix, bool ena
breakPoint.fragPos = f->position;
breakPoint.fragsBefore = fragCount + 1;
breakPoint.fragsAfter = 0;
- breakPoint.inSize = std::numeric_limits<int>::max();
+ breakPoint.inSize = INT32_MAX;
breakPoint.inFrags = 0;
breaks.push_back(breakPoint);
diff --git a/src/AS_BOG/AS_BOG_Outputs.C b/src/AS_BOG/AS_BOG_Outputs.C
index e35c9e7..3329ba6 100644
--- a/src/AS_BOG/AS_BOG_Outputs.C
+++ b/src/AS_BOG/AS_BOG_Outputs.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BOG_Outputs.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BOG_Outputs.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_BOG_Datatypes.H"
#include "AS_BOG_UnitigGraph.H"
@@ -38,14 +38,17 @@ UnitigGraph::unitigToMA(MultiAlignT *ma,
uint32 iumiid,
Unitig *utg) {
- ma->maID = iumiid;
- ma->data.unitig_coverage_stat = 1.0; // Default to just barely unique
- ma->data.unitig_microhet_prob = 1.0; // Default to 100% probability of unique
+ ma->maID = iumiid;
+ ma->data.unitig_coverage_stat = 1.0; // Default to just barely unique
+ ma->data.unitig_microhet_prob = 1.0; // Default to 100% probability of unique
- ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+ ma->data.unitig_status = AS_UNASSIGNED;
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
- ma->data.contig_status = AS_UNPLACED;
+ ma->data.contig_status = AS_UNPLACED;
// Add the fragments
diff --git a/src/AS_BOG/AS_BOG_PlaceFragUsingOverlaps.C b/src/AS_BOG/AS_BOG_PlaceFragUsingOverlaps.C
index 4717584..909d04f 100644
--- a/src/AS_BOG/AS_BOG_PlaceFragUsingOverlaps.C
+++ b/src/AS_BOG/AS_BOG_PlaceFragUsingOverlaps.C
@@ -19,14 +19,14 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_BOG_PlaceFragUsingOverlaps.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_BOG_PlaceFragUsingOverlaps.C 4572 2014-10-14 23:06:05Z brianwalenz $";
#include "AS_BOG_Datatypes.H"
#include "AS_BOG_BestOverlapGraph.H"
#include "AS_BOG_UnitigGraph.H"
#include "AS_BOG_Unitig.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
#define MAX_OVERLAPS_PER_FRAG (16 * 1024 * 1024)
@@ -231,8 +231,8 @@ UnitigGraph::placeFragUsingOverlaps(ufNode frag,
uint32 bgn = 0;
uint32 end = 1;
- intervalList UP; // Unitig position
- intervalList FC; // Fragment coverage
+ intervalList<int32> UP; // Unitig position
+ intervalList<int32> FC; // Fragment coverage
while (bgn < ovlLen) {
UP.clear();
@@ -269,12 +269,12 @@ UnitigGraph::placeFragUsingOverlaps(ufNode frag,
if (logFileFlagSet(LOG_PLACE_FRAG))
for (uint32 i=0; i<UP.numberOfIntervals(); i++)
- fprintf(logFile, "placeFragUsingOverlaps()-- frag %d in unitig %d (len %d nfrags "F_SIZE_T") at "F_S64","F_S64" from "F_U32" overlaps\n",
+ fprintf(logFile, "placeFragUsingOverlaps()-- frag %d in unitig %d (len %d nfrags "F_SIZE_T") at "F_S32","F_S32" from "F_U32" overlaps\n",
frag.ident,
ovlPlace[bgn].tigID, unitigs[ovlPlace[bgn].tigID]->getLength(), unitigs[ovlPlace[bgn].tigID]->ufpath.size(),
UP.lo(i),
UP.hi(i),
- UP.ct(i));
+ UP.count(i));
bgn = end;
end = end + 1;
diff --git a/src/AS_BOG/analyzeBest.C b/src/AS_BOG/analyzeBest.C
index 7a40dc3..ae1bb89 100644
--- a/src/AS_BOG/analyzeBest.C
+++ b/src/AS_BOG/analyzeBest.C
@@ -19,11 +19,11 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: analyzeBest.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: analyzeBest.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
// Read the 'best.edges' and 'best.contains' outputs from BOG, compute
diff --git a/src/AS_CGB/._AS_CGB_Bubble.C b/src/AS_CGB/._AS_CGB_Bubble.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble.C differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble.H b/src/AS_CGB/._AS_CGB_Bubble.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble.H differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_Graph.C b/src/AS_CGB/._AS_CGB_Bubble_Graph.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_Graph.C differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_Graph.H b/src/AS_CGB/._AS_CGB_Bubble_Graph.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_Graph.H differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_Popper.C b/src/AS_CGB/._AS_CGB_Bubble_Popper.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_Popper.C differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_Popper.H b/src/AS_CGB/._AS_CGB_Bubble_Popper.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_Popper.H differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_PopperMethods.C b/src/AS_CGB/._AS_CGB_Bubble_PopperMethods.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_PopperMethods.C differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_PopperMethods.H b/src/AS_CGB/._AS_CGB_Bubble_PopperMethods.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_PopperMethods.H differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_VertexSet.C b/src/AS_CGB/._AS_CGB_Bubble_VertexSet.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_VertexSet.C differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_VertexSet.H b/src/AS_CGB/._AS_CGB_Bubble_VertexSet.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_VertexSet.H differ
diff --git a/src/AS_CGB/._AS_CGB_Bubble_dfs.C b/src/AS_CGB/._AS_CGB_Bubble_dfs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_Bubble_dfs.C differ
diff --git a/src/AS_CGB/._AS_CGB_all.H b/src/AS_CGB/._AS_CGB_all.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_all.H differ
diff --git a/src/AS_CGB/._AS_CGB_cga.C b/src/AS_CGB/._AS_CGB_cga.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_cga.C differ
diff --git a/src/AS_CGB/._AS_CGB_cgb.C b/src/AS_CGB/._AS_CGB_cgb.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_cgb.C differ
diff --git a/src/AS_CGB/._AS_CGB_chimeras.C b/src/AS_CGB/._AS_CGB_chimeras.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_chimeras.C differ
diff --git a/src/AS_CGB/._AS_CGB_classify.C b/src/AS_CGB/._AS_CGB_classify.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_classify.C differ
diff --git a/src/AS_CGB/._AS_CGB_edgemate.C b/src/AS_CGB/._AS_CGB_edgemate.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_edgemate.C differ
diff --git a/src/AS_CGB/._AS_CGB_fga.C b/src/AS_CGB/._AS_CGB_fga.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_fga.C differ
diff --git a/src/AS_CGB/._AS_CGB_fgb.C b/src/AS_CGB/._AS_CGB_fgb.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_fgb.C differ
diff --git a/src/AS_CGB/._AS_CGB_histo.C b/src/AS_CGB/._AS_CGB_histo.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_histo.C differ
diff --git a/src/AS_CGB/._AS_CGB_histo.H b/src/AS_CGB/._AS_CGB_histo.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_histo.H differ
diff --git a/src/AS_CGB/._AS_CGB_main.C b/src/AS_CGB/._AS_CGB_main.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_main.C differ
diff --git a/src/AS_CGB/._AS_CGB_methods.H b/src/AS_CGB/._AS_CGB_methods.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_methods.H differ
diff --git a/src/AS_CGB/._AS_CGB_traversal.C b/src/AS_CGB/._AS_CGB_traversal.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_traversal.C differ
diff --git a/src/AS_CGB/._AS_CGB_unitigger.C b/src/AS_CGB/._AS_CGB_unitigger.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_unitigger.C differ
diff --git a/src/AS_CGB/._AS_CGB_unitigger_globals.H b/src/AS_CGB/._AS_CGB_unitigger_globals.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_unitigger_globals.H differ
diff --git a/src/AS_CGB/._AS_CGB_walk.C b/src/AS_CGB/._AS_CGB_walk.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_CGB_walk.C differ
diff --git a/src/AS_CGB/._AS_FGB_contained.C b/src/AS_CGB/._AS_FGB_contained.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_FGB_contained.C differ
diff --git a/src/AS_CGB/._AS_FGB_hanging_fragment.C b/src/AS_CGB/._AS_FGB_hanging_fragment.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_FGB_hanging_fragment.C differ
diff --git a/src/AS_CGB/._AS_FGB_io.C b/src/AS_CGB/._AS_FGB_io.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_FGB_io.C differ
diff --git a/src/AS_CGB/._AS_FGB_main.C b/src/AS_CGB/._AS_FGB_main.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._AS_FGB_main.C differ
diff --git a/src/AS_CGB/._Makefile b/src/AS_CGB/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGB/._Makefile differ
diff --git a/src/AS_CGB/AS_CGB_unitigger.C b/src/AS_CGB/AS_CGB_unitigger.C
index 7b92d84..08acee1 100644
--- a/src/AS_CGB/AS_CGB_unitigger.C
+++ b/src/AS_CGB/AS_CGB_unitigger.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: AS_CGB_unitigger.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: AS_CGB_unitigger.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_CGB_all.H"
#include "AS_CGB_Bubble.H"
@@ -132,10 +132,13 @@ output_the_chunks(Tfragment *frags,
global_fragment_arrival_rate);
ma->data.unitig_microhet_prob = 1.0; // Default to 100% probability of unique
- ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+ ma->data.unitig_status = AS_UNASSIGNED;
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
- ma->data.contig_status = AS_UNPLACED;
+ ma->data.contig_status = AS_UNPLACED;
// Add the fragments
diff --git a/src/AS_CGW/._AS_CGW_dataTypes.H b/src/AS_CGW/._AS_CGW_dataTypes.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._AS_CGW_dataTypes.H differ
diff --git a/src/AS_CGW/._AS_CGW_histo.C b/src/AS_CGW/._AS_CGW_histo.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._AS_CGW_histo.C differ
diff --git a/src/AS_CGW/._AS_CGW_histo.H b/src/AS_CGW/._AS_CGW_histo.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._AS_CGW_histo.H differ
diff --git a/src/AS_CGW/._AS_CGW_main.C b/src/AS_CGW/._AS_CGW_main.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._AS_CGW_main.C differ
diff --git a/src/AS_CGW/._CIEdgeT_CGW.C b/src/AS_CGW/._CIEdgeT_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIEdgeT_CGW.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_Analysis.C b/src/AS_CGW/._CIScaffoldT_Analysis.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Analysis.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_Analysis.H b/src/AS_CGW/._CIScaffoldT_Analysis.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Analysis.H differ
diff --git a/src/AS_CGW/._CIScaffoldT_Biconnected_CGW.C b/src/AS_CGW/._CIScaffoldT_Biconnected_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Biconnected_CGW.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_CGW.C b/src/AS_CGW/._CIScaffoldT_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_CGW.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_Cleanup_CGW.C b/src/AS_CGW/._CIScaffoldT_Cleanup_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Cleanup_CGW.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_MergeScaffolds.C b/src/AS_CGW/._CIScaffoldT_MergeScaffolds.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_MergeScaffolds.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_MergeScaffolds.H b/src/AS_CGW/._CIScaffoldT_MergeScaffolds.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_MergeScaffolds.H differ
diff --git a/src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.C b/src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.H b/src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Merge_AlignScaffold.H differ
diff --git a/src/AS_CGW/._CIScaffoldT_Merge_CGW.C b/src/AS_CGW/._CIScaffoldT_Merge_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Merge_CGW.C differ
diff --git a/src/AS_CGW/._CIScaffoldT_Merge_CGW.H b/src/AS_CGW/._CIScaffoldT_Merge_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Merge_CGW.H differ
diff --git a/src/AS_CGW/._CIScaffoldT_Merge_Interleaved.C b/src/AS_CGW/._CIScaffoldT_Merge_Interleaved.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._CIScaffoldT_Merge_Interleaved.C differ
diff --git a/src/AS_CGW/._Celamy_CGW.C b/src/AS_CGW/._Celamy_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Celamy_CGW.C differ
diff --git a/src/AS_CGW/._ChiSquareTest_CGW.H b/src/AS_CGW/._ChiSquareTest_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ChiSquareTest_CGW.H differ
diff --git a/src/AS_CGW/._ChunkOverlap_CGW.C b/src/AS_CGW/._ChunkOverlap_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ChunkOverlap_CGW.C differ
diff --git a/src/AS_CGW/._ChunkOverlap_CGW.H b/src/AS_CGW/._ChunkOverlap_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ChunkOverlap_CGW.H differ
diff --git a/src/AS_CGW/._ContigT_CGW.C b/src/AS_CGW/._ContigT_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ContigT_CGW.C differ
diff --git a/src/AS_CGW/._DemoteUnitigsWithRBP_CGW.C b/src/AS_CGW/._DemoteUnitigsWithRBP_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._DemoteUnitigsWithRBP_CGW.C differ
diff --git a/src/AS_CGW/._Globals_CGW.C b/src/AS_CGW/._Globals_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Globals_CGW.C differ
diff --git a/src/AS_CGW/._Globals_CGW.H b/src/AS_CGW/._Globals_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Globals_CGW.H differ
diff --git a/src/AS_CGW/._GraphCGW_T.C b/src/AS_CGW/._GraphCGW_T.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._GraphCGW_T.C differ
diff --git a/src/AS_CGW/._GraphCGW_T.H b/src/AS_CGW/._GraphCGW_T.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._GraphCGW_T.H differ
diff --git a/src/AS_CGW/._GraphEdgeIterator.C b/src/AS_CGW/._GraphEdgeIterator.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._GraphEdgeIterator.C differ
diff --git a/src/AS_CGW/._GraphEdgeIterator.H b/src/AS_CGW/._GraphEdgeIterator.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._GraphEdgeIterator.H differ
diff --git a/src/AS_CGW/._InputDataTypes_CGW.H b/src/AS_CGW/._InputDataTypes_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._InputDataTypes_CGW.H differ
diff --git a/src/AS_CGW/._Input_CGW.C b/src/AS_CGW/._Input_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Input_CGW.C differ
diff --git a/src/AS_CGW/._Input_CGW.H b/src/AS_CGW/._Input_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Input_CGW.H differ
diff --git a/src/AS_CGW/._Instrument_CGW.C b/src/AS_CGW/._Instrument_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Instrument_CGW.C differ
diff --git a/src/AS_CGW/._Instrument_CGW.H b/src/AS_CGW/._Instrument_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Instrument_CGW.H differ
diff --git a/src/AS_CGW/._InterleavedMerging.C b/src/AS_CGW/._InterleavedMerging.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._InterleavedMerging.C differ
diff --git a/src/AS_CGW/._InterleavedMerging.H b/src/AS_CGW/._InterleavedMerging.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._InterleavedMerging.H differ
diff --git a/src/AS_CGW/._LeastSquaresGaps_CGW.C b/src/AS_CGW/._LeastSquaresGaps_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._LeastSquaresGaps_CGW.C differ
diff --git a/src/AS_CGW/._Makefile b/src/AS_CGW/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Makefile differ
diff --git a/src/AS_CGW/._MarkInternalEdgeStatus.C b/src/AS_CGW/._MarkInternalEdgeStatus.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._MarkInternalEdgeStatus.C differ
diff --git a/src/AS_CGW/._MergeEdges_CGW.C b/src/AS_CGW/._MergeEdges_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._MergeEdges_CGW.C differ
diff --git a/src/AS_CGW/._Output_CGW.C b/src/AS_CGW/._Output_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Output_CGW.C differ
diff --git a/src/AS_CGW/._Output_CGW.H b/src/AS_CGW/._Output_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Output_CGW.H differ
diff --git a/src/AS_CGW/._SEdgeT_CGW.C b/src/AS_CGW/._SEdgeT_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._SEdgeT_CGW.C differ
diff --git a/src/AS_CGW/._ScaffoldGraphIterator_CGW.H b/src/AS_CGW/._ScaffoldGraphIterator_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ScaffoldGraphIterator_CGW.H differ
diff --git a/src/AS_CGW/._ScaffoldGraph_CGW.C b/src/AS_CGW/._ScaffoldGraph_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ScaffoldGraph_CGW.C differ
diff --git a/src/AS_CGW/._ScaffoldGraph_CGW.H b/src/AS_CGW/._ScaffoldGraph_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ScaffoldGraph_CGW.H differ
diff --git a/src/AS_CGW/._ShatterScaffolds_CGW.C b/src/AS_CGW/._ShatterScaffolds_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._ShatterScaffolds_CGW.C differ
diff --git a/src/AS_CGW/._SplitScaffolds_CGW.C b/src/AS_CGW/._SplitScaffolds_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._SplitScaffolds_CGW.C differ
diff --git a/src/AS_CGW/._Stats_CGW.C b/src/AS_CGW/._Stats_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Stats_CGW.C differ
diff --git a/src/AS_CGW/._Stats_CGW.H b/src/AS_CGW/._Stats_CGW.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._Stats_CGW.H differ
diff --git a/src/AS_CGW/._TransitiveReduction_CGW.C b/src/AS_CGW/._TransitiveReduction_CGW.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._TransitiveReduction_CGW.C differ
diff --git a/src/AS_CGW/._analyzeScaffolds.C b/src/AS_CGW/._analyzeScaffolds.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._analyzeScaffolds.C differ
diff --git a/src/AS_CGW/._cgwDump.C b/src/AS_CGW/._cgwDump.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._cgwDump.C differ
diff --git a/src/AS_CGW/._dumpCloneMiddles.C b/src/AS_CGW/._dumpCloneMiddles.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._dumpCloneMiddles.C differ
diff --git a/src/AS_CGW/._dumpSingletons.C b/src/AS_CGW/._dumpSingletons.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._dumpSingletons.C differ
diff --git a/src/AS_CGW/._eCR-diagnostic.C b/src/AS_CGW/._eCR-diagnostic.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._eCR-diagnostic.C differ
diff --git a/src/AS_CGW/._eCR-examineGap.C b/src/AS_CGW/._eCR-examineGap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._eCR-examineGap.C differ
diff --git a/src/AS_CGW/._eCR-partition.C b/src/AS_CGW/._eCR-partition.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._eCR-partition.C differ
diff --git a/src/AS_CGW/._eCR.C b/src/AS_CGW/._eCR.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._eCR.C differ
diff --git a/src/AS_CGW/._eCR.H b/src/AS_CGW/._eCR.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._eCR.H differ
diff --git a/src/AS_CGW/._find-big-gaps-in-cgw.out.pl b/src/AS_CGW/._find-big-gaps-in-cgw.out.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._find-big-gaps-in-cgw.out.pl differ
diff --git a/src/AS_CGW/._find-large-gaps-in-cgw-out.pl b/src/AS_CGW/._find-large-gaps-in-cgw-out.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._find-large-gaps-in-cgw-out.pl differ
diff --git a/src/AS_CGW/._fixZLFContigs.H b/src/AS_CGW/._fixZLFContigs.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._fixZLFContigs.H differ
diff --git a/src/AS_CGW/._fragmentPlacement.C b/src/AS_CGW/._fragmentPlacement.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._fragmentPlacement.C differ
diff --git a/src/AS_CGW/._fragmentPlacement.H b/src/AS_CGW/._fragmentPlacement.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._fragmentPlacement.H differ
diff --git a/src/AS_CGW/._frgs2clones.C b/src/AS_CGW/._frgs2clones.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._frgs2clones.C differ
diff --git a/src/AS_CGW/._parse-cgw-out-for-merges.pl b/src/AS_CGW/._parse-cgw-out-for-merges.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._parse-cgw-out-for-merges.pl differ
diff --git a/src/AS_CGW/._resolveSurrogates.C b/src/AS_CGW/._resolveSurrogates.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CGW/._resolveSurrogates.C differ
diff --git a/src/AS_CGW/AS_CGW_main.C b/src/AS_CGW/AS_CGW_main.C
index e47412a..00f4051 100644
--- a/src/AS_CGW/AS_CGW_main.C
+++ b/src/AS_CGW/AS_CGW_main.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: AS_CGW_main.C 4430 2013-09-24 22:12:41Z brianwalenz $";
+const char *mainid = "$Id: AS_CGW_main.C 4557 2014-08-11 12:24:27Z brianwalenz $";
#undef CHECK_CONTIG_ORDERS
#undef CHECK_CONTIG_ORDERS_INCREMENTAL
@@ -50,7 +50,9 @@ const char *mainid = "$Id: AS_CGW_main.C 4430 2013-09-24 22:12:41Z brianwalenz $
#include "CIScaffoldT_Analysis.H" // For checking mates on load
+#ifndef BROKEN_CLANG_OpenMP
#include <omp.h>
+#endif
// Defines the logical checkpoints
@@ -189,9 +191,6 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-E") == 0) {
GlobalData->outputOverlapOnlyContigEdges = 1;
- } else if (strcmp(argv[arg], "-e") == 0) {
- GlobalData->cgbMicrohetProb = atof(argv[++arg]);
-
} else if (strcmp(argv[arg], "-F") == 0) {
GlobalData->allowDemoteMarkedUnitigs = FALSE;
@@ -210,9 +209,6 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-I") == 0) {
GlobalData->ignoreChaffUnitigs = 1;
- } else if (strcmp(argv[arg], "-i") == 0) {
- GlobalData->cgbApplyMicrohetCutoff = atof(argv[++arg]);
-
} else if (strcmp(argv[arg], "-j") == 0) {
GlobalData->cgbUniqueCutoff = atof(argv[++arg]);
diff --git a/src/AS_CGW/CIScaffoldT_CGW.C b/src/AS_CGW/CIScaffoldT_CGW.C
index b2c32ad..eeaf489 100644
--- a/src/AS_CGW/CIScaffoldT_CGW.C
+++ b/src/AS_CGW/CIScaffoldT_CGW.C
@@ -18,7 +18,7 @@
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid = "$Id: CIScaffoldT_CGW.C 4446 2013-10-11 17:26:37Z brianwalenz $";
+static char *rcsid = "$Id: CIScaffoldT_CGW.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#undef DEBUG_INSERT
#undef DEBUG_DIAG
@@ -1198,12 +1198,12 @@ DemoteSmallSingletonScaffolds(void) {
numSingletonScaffolds++;
// if we are forced marked unique and we are not allowed to be demoted, continue
- if (ScaffoldGraph->tigStore->getUnitigFUR(CI->id) == AS_FORCED_UNIQUE &&
+ if (ScaffoldGraph->tigStore->getUnitigForceUnique(CI->id) == true &&
GlobalData->allowDemoteMarkedUnitigs == FALSE) {
continue;
}
- if ((ScaffoldGraph->tigStore->getUnitigFUR(CI->id) != AS_FORCED_REPEAT &&
+ if ((ScaffoldGraph->tigStore->getUnitigForceRepeat(CI->id) == false &&
ScaffoldGraph->tigStore->getUnitigCoverageStat(CI->id) > GlobalData->cgbDefinitelyUniqueCutoff) ||
(CI->bpLength.mean > 2000.0))
continue;
diff --git a/src/AS_CGW/ContigT_CGW.C b/src/AS_CGW/ContigT_CGW.C
index a7506a8..112ce91 100644
--- a/src/AS_CGW/ContigT_CGW.C
+++ b/src/AS_CGW/ContigT_CGW.C
@@ -18,7 +18,7 @@
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid = "$Id: ContigT_CGW.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static char *rcsid = "$Id: ContigT_CGW.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#undef DEBUG_CONTIG
@@ -709,12 +709,12 @@ int IsDefinitelyUniqueContig(ContigT *contig){
*/
// when the flag says we are unique, we always return true
- if (ScaffoldGraph->tigStore->getUnitigFUR(ci->id) == AS_FORCED_UNIQUE) {
+ if (ScaffoldGraph->tigStore->getUnitigForceUnique(ci->id) == true) {
return TRUE;
}
// when the flag says we are repeat, we always return false
- if (ScaffoldGraph->tigStore->getUnitigFUR(ci->id) == AS_FORCED_REPEAT) {
+ if (ScaffoldGraph->tigStore->getUnitigForceRepeat(ci->id) == true) {
return FALSE;
}
diff --git a/src/AS_CGW/DemoteUnitigsWithRBP_CGW.C b/src/AS_CGW/DemoteUnitigsWithRBP_CGW.C
index 707683c..11229ca 100644
--- a/src/AS_CGW/DemoteUnitigsWithRBP_CGW.C
+++ b/src/AS_CGW/DemoteUnitigsWithRBP_CGW.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: DemoteUnitigsWithRBP_CGW.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: DemoteUnitigsWithRBP_CGW.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -70,13 +70,13 @@ void DemoteUnitigsWithRBP(FILE *stream, GraphCGW_T *graph){
}
// if the node was force-marked unique and we dont allow demotion, skip it and dont demote
- if (ScaffoldGraph->tigStore->getUnitigFUR(node->id) == AS_FORCED_UNIQUE && GlobalData->allowDemoteMarkedUnitigs == FALSE) {
+ if (ScaffoldGraph->tigStore->getUnitigForceUnique(node->id) == true && GlobalData->allowDemoteMarkedUnitigs == FALSE) {
continue;
}
/* fprintf(stream, "Unitig %d: branchA %d branchB %d\n", node->id, numAEndConfirmOverlap,
numBEndConfirmOverlap); */
- if((ScaffoldGraph->tigStore->getUnitigFUR(node->id) == AS_FORCED_REPEAT) || ((numAEndConfirmOverlap > 1) && (numBEndConfirmOverlap > 1))){
+ if((ScaffoldGraph->tigStore->getUnitigForceRepeat(node->id) == true) || ((numAEndConfirmOverlap > 1) && (numBEndConfirmOverlap > 1))){
fprintf(stream, "DemoteUnitigsWithRBP(): Demote unitig %d, len=%.0f, #overlaps=(%d,%d)\n",
node->id,node->bpLength.mean,numAEndConfirmOverlap,numBEndConfirmOverlap);
node->flags.bits.isUnique = FALSE;
diff --git a/src/AS_CGW/Globals_CGW.C b/src/AS_CGW/Globals_CGW.C
index 1450c7d..dff26bd 100644
--- a/src/AS_CGW/Globals_CGW.C
+++ b/src/AS_CGW/Globals_CGW.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid = "$Id: Globals_CGW.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static char *rcsid = "$Id: Globals_CGW.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_global.H"
#include "Globals_CGW.H"
@@ -51,8 +51,6 @@ Globals_CGW::Globals_CGW() {
cgbUniqueCutoff = CGB_UNIQUE_CUTOFF;
cgbDefinitelyUniqueCutoff = CGB_UNIQUE_CUTOFF;
- cgbApplyMicrohetCutoff = -1; // This basically turns it off, unless enabled
- cgbMicrohetProb = 1.e-05; // scores less than this are considered repeats
allowDemoteMarkedUnitigs = TRUE; // allow toggled unitigs to be demoted to be repeat if they were marked unique
diff --git a/src/AS_CGW/Globals_CGW.H b/src/AS_CGW/Globals_CGW.H
index 4db1597..619610a 100644
--- a/src/AS_CGW/Globals_CGW.H
+++ b/src/AS_CGW/Globals_CGW.H
@@ -22,7 +22,7 @@
#ifndef GLOBALS_CGW_H
#define GLOBALS_CGW_H
-static const char *rcsid_GLOBALS_CGW_H = "$Id: Globals_CGW.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_GLOBALS_CGW_H = "$Id: Globals_CGW.H 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_global.H"
@@ -58,8 +58,6 @@ public:
int cgbUniqueCutoff;
int cgbDefinitelyUniqueCutoff;
- int cgbApplyMicrohetCutoff;
- double cgbMicrohetProb;
int allowDemoteMarkedUnitigs;
diff --git a/src/AS_CGW/GraphCGW_T.C b/src/AS_CGW/GraphCGW_T.C
index 98e5ae1..d57ee1e 100644
--- a/src/AS_CGW/GraphCGW_T.C
+++ b/src/AS_CGW/GraphCGW_T.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid = "$Id: GraphCGW_T.C 4390 2013-08-24 05:56:55Z brianwalenz $";
+static char *rcsid = "$Id: GraphCGW_T.C 4557 2014-08-11 12:24:27Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -42,7 +42,9 @@ static char *rcsid = "$Id: GraphCGW_T.C 4390 2013-08-24 05:56:55Z brianwalenz $"
#include "Input_CGW.H"
+#ifndef BROKEN_CLANG_OpenMP
#include <omp.h>
+#endif
#include <vector>
#include <algorithm>
diff --git a/src/AS_CGW/GraphCGW_T.H b/src/AS_CGW/GraphCGW_T.H
index 6c3e80a..bdf2e8f 100644
--- a/src/AS_CGW/GraphCGW_T.H
+++ b/src/AS_CGW/GraphCGW_T.H
@@ -22,7 +22,7 @@
#ifndef GRAPH_CGW_H
#define GRAPH_CGW_H
-static const char *rcsid_GRAPH_CGW_H = "$Id: GraphCGW_T.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_GRAPH_CGW_H = "$Id: GraphCGW_T.H 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_UTL_Var.H"
#include "AS_CGW_dataTypes.H"
@@ -267,8 +267,6 @@ typedef enum {
SCRATCH_SCAFFOLD // a temporary scaffold
} ChunkInstanceType;
-typedef UnitigFUR ChunkFUR;
-
/* This enum is used to encode the presence/absence of tandem repeat overlaps
on the end of the ChunkInstance (only meaningful if isCI = TRUE)
diff --git a/src/AS_CGW/Input_CGW.C b/src/AS_CGW/Input_CGW.C
index b464336..f08ab69 100644
--- a/src/AS_CGW/Input_CGW.C
+++ b/src/AS_CGW/Input_CGW.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid = "$Id: Input_CGW.C 4429 2013-09-24 22:07:01Z brianwalenz $";
+static char *rcsid = "$Id: Input_CGW.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -40,45 +40,6 @@ static char *rcsid = "$Id: Input_CGW.C 4429 2013-09-24 22:07:01Z brianwalenz $";
#include "Input_CGW.H"
-#define CGW_MIN_READS_IN_UNIQUE 2
-
-// Due to the FBAC fragments, we get some pathologically short U-Unitigs
-// Set the following threshhold to eliminate the really short ones
-//
-#define CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH 1000
-
-
-// For each unitig, write logging on the repeat/unique decision.
-#undef LOG_REPEAT_UNIQUE_LABELING
-
-
-// Stats on repeat labeling of input unitigs.
-//
-class ruLabelStat {
-public:
- ruLabelStat() {
- num = 0;
- len = 0;
- };
-
- void operator+=(uint64 len_) {
- num++;
- len += len_;
- };
-
- uint32 num;
- uint64 len;
-};
-
-ruLabelStat repeat_LowReads;
-ruLabelStat repeat_LowCovStat;
-ruLabelStat repeat_Short;
-ruLabelStat repeat_MicroHet;
-
-ruLabelStat repeat_IsUnique;
-ruLabelStat repeat_IsRepeat;
-
-
static
void
@@ -121,101 +82,17 @@ ProcessInputUnitig(MultiAlignT *uma) {
CI.offsetAEnd.variance = 0.0;
CI.offsetBEnd = CI.bpLength;
- int isUnique = TRUE;
-
-
- //fprintf(stderr, "unitig %d "F_SIZE_T" reads %d length\n",
- // uma->maID, GetNumIntMultiPoss(uma->f_list), length);
-
-
- if (GetNumIntMultiPoss(uma->f_list) < CGW_MIN_READS_IN_UNIQUE) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d not unique -- "F_SIZE_T" reads, need at least %d\n",
- uma->maID, GetNumIntMultiPoss(uma->f_list), CGW_MIN_READS_IN_UNIQUE);
-#endif
- repeat_LowReads += length;
- isUnique = FALSE;
- }
-
- if (ScaffoldGraph->tigStore->getUnitigCoverageStat(uma->maID) < GlobalData->cgbUniqueCutoff) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d not unique -- coverage stat %d, needs to be at least %d\n",
- uma->maID, ScaffoldGraph->tigStore->getUnitigCoverageStat(uma->maID), GlobalData->cgbUniqueCutoff);
-#endif
- repeat_LowCovStat += length;
- isUnique = FALSE;
- }
-
-#ifdef SHORT_HIGH_ASTAT_ARE_UNIQUE
- // This is an attempt to not blindly call all short unitigs as non-unique. It didn't work so
- // well in initial limited testing. The threshold is arbitrary; older versions used
- // cgbDefinitelyUniqueCutoff.
- if ((ScaffoldGraph->tigStore->getUnitigCoverageStat(uma->maID) < GlobalData->cgbUniqueCutoff * 10) &&
- (length < CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH)) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d not unique -- length %d too short, need to be at least %d AND coverage stat %d must be larger than %d\n",
- uma->maID, length, CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH,
- ScaffoldGraph->tigStore->getUnitigCoverageStat(uma->maID), GlobalData->cgbUniqueCutoff * 10);
-#endif
- repeat_Short += length;
- isUnique = FALSE;
- }
-#else
- if (length < CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d not unique -- length %d too short, need to be at least %d\n",
- uma->maID, length, CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH);
-#endif
- repeat_Short += length;
- isUnique = FALSE;
- }
-#endif
-
- // MicroHet probability is actually the probability of the sequence being UNIQUE, based on
- // microhet considerations. Falling below threshhold makes something a repeat.
- // Note that this is off by default (see options -e, -i)
- // Defaults cgbMicrohetProb = 1.0 e-5
- // cgbApplyMicrohetCutoff = -1
- if ((ScaffoldGraph->tigStore->getUnitigMicroHetProb(uma->maID) < GlobalData->cgbMicrohetProb) &&
- (ScaffoldGraph->tigStore->getUnitigCoverageStat(uma->maID) < GlobalData->cgbApplyMicrohetCutoff)) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d not unique -- low microhetprob %f (< %f) and low coverage stat %d (< %d)\n",
- uma->maID,
- ScaffoldGraph->tigStore->getUnitigMicroHetProb(uma->maID), GlobalData->cgbMicrohetProb,
- ScaffoldGraph->tigStore->getUnitigCoverageStat(uma->maID), GlobalData->cgbApplyMicrohetCutoff);
-#endif
- repeat_MicroHet += length;
- isUnique = FALSE;
- }
-
- // allow flag to overwrite what the default behavior for a chunk and force it to be unique or repeat
-
- if (ScaffoldGraph->tigStore->getUnitigFUR(CI.id) == AS_FORCED_UNIQUE) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d forced unique\n",
- uma->maID);
-#endif
- isUnique = TRUE;
- }
-
- if (ScaffoldGraph->tigStore->getUnitigFUR(CI.id) == AS_FORCED_REPEAT) {
-#ifdef LOG_REPEAT_UNIQUE_LABELING
- fprintf(stderr, "unitig %d forced repeat\n",
- uma->maID);
-#endif
- isUnique = FALSE;
- }
-
- if (isUnique) {
- CI.flags.bits.isUnique = 1;
- CI.type = DISCRIMINATORUNIQUECHUNK_CGW;
+ // The unitig is repeat if we 'suggest' it so (and we don't force it unique), or if it is forced
+ // repeat (regardless of if it is forced unique).
- repeat_IsUnique += length;
- } else {
+ if (((ScaffoldGraph->tigStore->getUnitigSuggestRepeat(CI.id) == true) &&
+ (ScaffoldGraph->tigStore->getUnitigForceUnique(CI.id) == false)) ||
+ ((ScaffoldGraph->tigStore->getUnitigForceRepeat(CI.id) == true))) {
CI.flags.bits.isUnique = 0;
CI.type = UNRESOLVEDCHUNK_CGW;
-
- repeat_IsRepeat += length;
+ } else {
+ CI.flags.bits.isUnique = 1;
+ CI.type = DISCRIMINATORUNIQUECHUNK_CGW;
}
CI.flags.bits.smoothSeenAlready = FALSE;
@@ -523,7 +400,7 @@ ProcessInput(int optind, int argc, char *argv[]){
fprintf(stderr, "Checking sanity of loaded fragments.\n");
- uint32 numErrors = 0;
+ uint32 numErrors = 0;
for (int32 i=1, s=GetNumCIFragTs(ScaffoldGraph->CIFrags); i<s; i++) {
CIFragT *cifrag = GetCIFragT(ScaffoldGraph->CIFrags, i);
@@ -537,21 +414,16 @@ ProcessInput(int optind, int argc, char *argv[]){
if ((cifrag->cid == NULLINDEX) || (cifrag->CIid == NULLINDEX)) {
fprintf(stderr, "ERROR: Frag %d has null cid or CIid. Fragment is not in an input unitig!\n", i);
+
+ cifrag->flags.bits.isDeleted = 1;
numErrors++;
}
}
- fprintf(stderr, "Processed %d unitigs with %d fragments.\n", numUTG, numFRG);
- fprintf(stderr, " unique: "F_U32" unitigs with total length "F_U64"\n", repeat_IsUnique.num, repeat_IsUnique.len);
- fprintf(stderr, " repeat: "F_U32" unitigs with total length "F_U64"\n", repeat_IsRepeat.num, repeat_IsRepeat.len);
- fprintf(stderr, " few reads: "F_U32" unitigs with total length "F_U64"\n", repeat_LowReads.num, repeat_LowReads.len);
- fprintf(stderr, " low cov stat: "F_U32" unitigs with total length "F_U64"\n", repeat_LowCovStat.num, repeat_LowCovStat.len);
- fprintf(stderr, " too short: "F_U32" unitigs with total length "F_U64"\n", repeat_Short.num, repeat_Short.len);
- fprintf(stderr, " microhet: "F_U32" unitigs with total length "F_U64"\n", repeat_MicroHet.num, repeat_MicroHet.len);
if (numErrors > 0)
fprintf(stderr, "ERROR: %u fragments are not in unitigs.\n", numErrors);
- assert(numErrors == 0);
+ assert(numErrors == 0); // Can be safely disabled if you're OK losing these reads.
ScaffoldGraph->numLiveCIs = GetNumGraphNodes(ScaffoldGraph->CIGraph);
ScaffoldGraph->numOriginalCIs = GetNumGraphNodes(ScaffoldGraph->CIGraph);
diff --git a/src/AS_CGW/TransitiveReduction_CGW.C b/src/AS_CGW/TransitiveReduction_CGW.C
index 15883db..0099203 100644
--- a/src/AS_CGW/TransitiveReduction_CGW.C
+++ b/src/AS_CGW/TransitiveReduction_CGW.C
@@ -18,7 +18,7 @@
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid = "$Id: TransitiveReduction_CGW.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static char *rcsid = "$Id: TransitiveReduction_CGW.C 4557 2014-08-11 12:24:27Z brianwalenz $";
//#define INSTRUMENT_CGW
//#define INSTRUMENT_SMOOTHED
@@ -38,7 +38,9 @@ static char *rcsid = "$Id: TransitiveReduction_CGW.C 4371 2013-08-01 17:19:47Z b
#include "DataTypesREZ.H" // Target_Info_t
+#ifndef BROKEN_CLANG_OpenMP
#include <omp.h>
+#endif
static VA_TYPE(CIEdgeT) *graphCIEdges;
diff --git a/src/AS_CGW/find-big-gaps-in-cgw.out.pl b/src/AS_CGW/find-big-gaps-in-cgw.out.pl
new file mode 100644
index 0000000..11c89c1
--- /dev/null
+++ b/src/AS_CGW/find-big-gaps-in-cgw.out.pl
@@ -0,0 +1,32 @@
+#!/usr/bin/perl
+
+#InsertScaffoldContentsIntoScaffold()-- Insert CI 76908 1228bp fwd 0 +- 0 1228 +- 32 was 0 +- 0 1228 +- 32
+#InsertScaffoldContentsIntoScaffold()-- Insert CI 320456 10316bp fwd 2699 +- 53691 13015 +- 53959 was 2699 +- 53691 13015 +- 53959
+
+my $last = 0;
+my $max = 0;
+
+while (<STDIN>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ my $gs = $v[6] - $last;
+
+ $last = $v[6];
+
+ if ($gs > 500000) {
+ print "$gs -- $_\n";
+ }
+
+ $gapSize[$gs]++;
+
+ $max = ($gs < $max) ? $max : $gs;
+}
+
+
+for (my $i=0; $i<=$max; $i++) {
+ if ($gapSize[$i] > 0) {
+ print "$i\t$gapSize[$i]\n";
+ }
+}
diff --git a/src/AS_CGW/find-large-gaps-in-cgw-out.pl b/src/AS_CGW/find-large-gaps-in-cgw-out.pl
new file mode 100755
index 0000000..a0599fa
--- /dev/null
+++ b/src/AS_CGW/find-large-gaps-in-cgw-out.pl
@@ -0,0 +1,56 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Checks cgw.out for large gaps. Only checks the two scaffolds that are merged. Does not check the newly formed scaffold.
+
+my $lasbgn = 0;
+my $lasend = 0;
+my $lasctg = 0;
+my $dist = 0;
+my $first = undef;
+
+my $scf = 0;
+
+my $lastline = undef;
+my $thisline = undef;
+
+while (<STDIN>) {
+ $lastline = $thisline;
+ $thisline = $_;
+
+ if (m/InsertScaffoldContentsIntoScaffold\(\)--\s+Insert\s+scaffold\s(\d+)/) {
+ $lasbgn = 0;
+ $lasend = 0;
+ $lasctg = 0;
+ $dist = 0;
+ $first = undef;
+ $scf = $1;
+ }
+
+ if (m/InsertScaffoldContentsIntoScaffold\(\)--\s+Insert\s+CI\s+(\d+)\s+(\d+bp)\s+(.*)\s+(\d+)\s+..\s+(\d+)\s+(\d+)\s+..\s+(\d+)\s+was\s+(\d+)\s+..\s+(\d+)\s+(\d+)\s+..\s+(\d+)\s+$/) {
+ my $thsctg = $1;
+ my $thsbgn = ($4 < $6) ? $4 : $6;
+ my $thsend = ($4 < $6) ? $6 : $4;
+
+ # coords increasing coords decreasing
+ $dist = ($lasbgn < $thsbgn) ? ($thsbgn - $lasend) : ($lasbgn - $thsend);
+
+ if (defined($first)) {
+ if ($dist > 150000) {
+ print "GAP:\t$dist\tCI\t$lasctg\tpos\t$lasbgn\t$lasend\tto\tCI\t$thsctg\tpos\t$thsbgn\t$thsend\tscf\t$scf\n";
+ print $lastline;
+ print $thisline;
+
+ } else {
+ print "GAP:\t$dist\tCI\t$lasctg\tpos\t$lasbgn\t$lasend\tto\tCI\t$thsctg\tpos\t$thsbgn\t$thsend\tscf\t$scf\n";
+ }
+ }
+
+ $lasbgn = $thsbgn;
+ $lasend = $thsend;
+ $lasctg = $thsctg;
+ $first .= $_;
+ } else {
+ }
+}
diff --git a/src/AS_CGW/frgs2clones.C b/src/AS_CGW/frgs2clones.C
index 8e1e8c5..b2bc7ef 100644
--- a/src/AS_CGW/frgs2clones.C
+++ b/src/AS_CGW/frgs2clones.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: frgs2clones.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: frgs2clones.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
@@ -199,14 +199,17 @@ main(int argc, char **argv) {
// Build the MultiAlign
- ma->maID = 1;
- ma->data.unitig_coverage_stat = 0.0;
- ma->data.unitig_microhet_prob = 1.0;
+ ma->maID = 1;
+ ma->data.unitig_coverage_stat = 0.0;
+ ma->data.unitig_microhet_prob = 1.0;
- ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+ ma->data.unitig_status = AS_UNASSIGNED;
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
- ma->data.contig_status = AS_UNPLACED;
+ ma->data.contig_status = AS_UNPLACED;
// Add the fragments
diff --git a/src/AS_CGW/parse-cgw-out-for-merges.pl b/src/AS_CGW/parse-cgw-out-for-merges.pl
new file mode 100644
index 0000000..c551b11
--- /dev/null
+++ b/src/AS_CGW/parse-cgw-out-for-merges.pl
@@ -0,0 +1,195 @@
+#!/usr/bin/perl
+
+use strict;
+
+# tail -f cgw.out | egrep Examine\|isQual.\*gap.\*weight
+
+my @p; # previous numbers
+my @r; # current numbers
+
+$p[0] = $r[0] = 0;
+$p[1] = $r[1] = 0;
+$p[2] = $r[2] = 0;
+$p[3] = $r[3] = 0;
+$p[4] = $r[4] = 0;
+$p[5] = $r[5] = 0;
+$p[6] = $r[6] = 0;
+$p[7] = $r[7] = 0;
+$p[8] = $r[8] = 0;
+$p[9] = $r[9] = 0;
+$p[10] = $r[10] = 0;
+$p[11] = $r[11] = 0;
+$p[12] = $r[12] = 0;
+$p[13] = $r[13] = 0;
+$p[14] = $r[14] = 0;
+$p[15] = $r[15] = 0;
+
+my $nTests = 0;
+my $nInterleaves = 0;
+my $nMerged = 0;
+
+my $nContigs = 0;
+
+my $iter = 0;
+
+open(C, "> cgw.out.cumulative.stats");
+open(D, "> cgw.out.difference.stats");
+
+while (!eof(STDIN)) {
+
+ # scf1 len1 scf2 len2 gap weight orient
+ while ($_ !~ m/^isQualityScaffoldMergingEdge\(\)--\sMerge\sscaffolds\s(\d+)\s\((.*bp)\)\sand\s(\d+)\s\((.*bp)\)\:\sgap\s(.*bp.*bp)\sweight\s(\d+)\s(.*)\sedge/) {
+
+ if (m/Insert\sscaffold/) {
+ $nMerged++;
+ }
+
+ if (m/^MergeScaffoldsAggressive.*iter\s(\d+)\s/) {
+ $iter = $1;
+
+ for (my $i=0; $i<15; $i++) {
+ $p[$i] = $r[$i] - $p[$i];
+ }
+
+ print C "$iter\t$nTests\t$nInterleaves\t$nMerged\t$nContigs\t$r[0]\t$r[1]\t$r[2]\t$r[3]\t$r[4]\t$r[5]\t$r[6]\t$r[7]\t$r[8]\t$r[9]\t$r[10]\t$r[11]\t$r[12]\t$r[13]\t$r[14]\n";
+ print D "$iter\t$nTests\t$nInterleaves\t$nMerged\t$nContigs\t$p[0]\t$p[1]\t$p[2]\t$p[3]\t$p[4]\t$p[5]\t$p[6]\t$p[7]\t$p[8]\t$p[9]\t$p[10]\t$p[11]\t$p[12]\t$p[13]\t$p[14]\n";
+
+ @p = @r;
+ }
+
+ if (m/^CreateAContigInScaffold.*new\scontig/) {
+ $nContigs++;
+ }
+
+ $_ = <STDIN>; chomp $_;
+ }
+
+ $nTests++;
+
+ my ($scf1, $len1, $scf2, $len2, $gapSize, $weight, $orient);
+
+ #print "$_\n";
+ if ($_ =~ m/^isQualityScaffoldMergingEdge\(\)--\sMerge\sscaffolds\s(\d+)\s\((.*bp)\)\sand\s(\d+)\s\((.*bp)\)\:\sgap\s(.*bp.*bp)\sweight\s(\d+)\s(.*)\sedge/) {
+ $scf1 = $1;
+ $len1 = $2;
+ $scf2 = $3;
+ $len2 = $4;
+ $gapSize = $5;
+ $weight = $6;
+ $orient = $7;
+ } else {
+ die "Failed on '$_'\n";
+ }
+
+
+ my ($happy1, $sad1);
+
+ $_ = <STDIN>; chomp $_; # First scaffold instrument
+
+ while (m/^instrument/) {
+ $_ = <STDIN>; chomp $_; # Ignore lines about setting up the instrumenter
+ }
+
+ #print "$_\n";
+ if ($_ =~ m/isQualityScaffoldMergingEdge\(\)--\s+scaffold\s(\d+)\sinstrumenter\shappy\s(.*)\sgap\s(.*)\smisorient\sclose\s(.*)\scorrect\s(.*)\sfar\s(.*)\soriented\sclose\s(.*)\sfar\s(.*)\smissing\s(.*)\sexternal\s(.*)/) {
+ $happy1 = $1;
+ $sad1 = $3 + $4 + $5 + $6 + $7 + $8;
+ } else {
+ die "Failed on '$_'\n";
+ }
+
+
+ my ($happy2, $sad2);
+
+ #print "$_\n";
+ $_ = <STDIN>; chomp $_; # Second scaffold instrument
+ if ($_ =~ m/isQualityScaffoldMergingEdge\(\)--\s+scaffold\s(\d+)\sinstrumenter\shappy\s(.*)\sgap\s(.*)\smisorient\sclose\s(.*)\scorrect\s(.*)\sfar\s(.*)\soriented\sclose\s(.*)\sfar\s(.*)\smissing\s(.*)\sexternal\s(.*)/) {
+ $happy2 = $1;
+ $sad2 = $3 + $4 + $5 + $6 + $7 + $8;
+ } else {
+ die "Failed on '$_'\n";
+ }
+
+
+ my ($happy, $gap, $misclose, $misfar, $mis, $close, $far, $missing, $external);
+ my ($happyN, $sadN);
+
+ #print "$_\n";
+ $_ = <STDIN>; chomp $_; # Merged scaffold instrument
+ if ($_ =~ m/isQualityScaffoldMergingEdge\(\)--\s+scaffold\s\(new\)\sinstrumenter\shappy\s(.*)\sgap\s(.*)\smisorient\sclose\s(.*)\scorrect\s(.*)\sfar\s(.*)\soriented\sclose\s(.*)\sfar\s(.*)\smissing\s(.*)\sexternal\s(.*)/) {
+ $happy = $1;
+ $gap = $2;
+ $misclose = $3;
+ $misfar = $4;
+ $mis = $5;
+ $close = $6;
+ $far = $7;
+ $missing = $8;
+ $external = $9;
+
+ $happyN = $1;
+ $sadN = $3 + $4 + $5 + $6 + $7 + $8;
+ } else {
+ die "Failed on '$_'\n";
+ }
+
+
+ my $happyB = $happy1 + $happy2;
+ my $sadB = $sad1 + $sad2;
+
+ my $interleave;
+
+ $_ = <STDIN>; chomp $_; # Before/After mates
+ $_ = <STDIN>; chomp $_; # Happy enough to merge?
+ $_ = <STDIN>; chomp $_; # pass/fail stats
+ $_ = <STDIN>; chomp $_; # Interleave result (optional)
+
+ if ($_ =~ m/ExamineSEdgeForUsability_Interleaved\(\)--\s(.*)/) {
+ $nInterleaves++;
+
+ $interleave = $1;
+
+ #print "$scf1 ($len1) $scf2 ($len2) $gapSize $weight -- + $happy - $misclose $mis $misfar - $close $far - $missing -- $interleave\n";
+ #print "$scf1 ($len1) $scf2 ($len2) gap $gapSize weight $weight -- $happyB / $sadB -- $happyN / $sadN -- $interleave\n";
+ } else {
+ #print "$scf1 ($len1) $scf2 ($len2) $gapSize $weight -- + $happy - $misclose $mis $misfar - $close $far - $missing\n";
+ #print "$scf1 ($len1) $scf2 ($len2) gap $gapSize weight $weight -- $happyB / $sadB -- $happyN / $sadN\n";
+ }
+
+ # The classifications, straight from CIScaffoldT_Merge_Interleaved.c
+
+ if ($interleave eq undef) {
+ $r[0]++;
+ } elsif($interleave =~ m/Didn't expect end contigs to overlap, didn't find it, and will join with existing edge/) {
+ $r[1]++;
+ } elsif ($interleave =~ m/Expected end contigs to overlap, found overlap, will merge/) {
+ $r[2]++;
+ } elsif ($interleave =~ m/Expected end contigs to overlap, didn't find it, but will abut/) {
+ $r[3]++;
+ } elsif ($interleave =~ m/Expected end contigs to overlap, didn't find it, and abutting not allowed, will not merge/) {
+ $r[4]++;
+ } elsif ($interleave =~ m/Expected end contigs to overlap, didn't find it, will not merge/) {
+ $r[5]++;
+ } elsif ($interleave =~ m/Interleaving succeeded with contig overlaps, but failed to make adjustments; will not merge/) {
+ $r[6]++;
+ } elsif ($interleave =~ m/Interleaving succeeded with contig overlaps; will merge/) {
+ $r[7]++;
+ } elsif ($interleave =~ m/Interleaving succeeded without contig overlaps, but failed to make adjustments; will not merge/) {
+ $r[8]++;
+ } elsif ($interleave =~ m/Interleaving succeeded without contig overlaps; will merge/) {
+ $r[9]++;
+ } elsif ($interleave =~ m/Interleaving failed, abut not allowed, will not merge/) {
+ $r[10]++;
+ } elsif ($interleave =~ m/Interleaving failed, will abut/) {
+ $r[11]++;
+ } elsif ($interleave =~ m/Interleaving failed, didn't expect overlap, will join with existing edge/) {
+ $r[12]++;
+ } elsif ($interleave =~ m/Interleaving failed, will not merge/) {
+ $r[13]++;
+ } else {
+ $r[14]++;
+ }
+}
+
+close(C);
+close(D);
diff --git a/src/AS_CNS/._AbacusRefine.C b/src/AS_CNS/._AbacusRefine.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._AbacusRefine.C differ
diff --git a/src/AS_CNS/._ApplyAlignment.C b/src/AS_CNS/._ApplyAlignment.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._ApplyAlignment.C differ
diff --git a/src/AS_CNS/._BaseCall.C b/src/AS_CNS/._BaseCall.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._BaseCall.C differ
diff --git a/src/AS_CNS/._GetAlignmentTrace.C b/src/AS_CNS/._GetAlignmentTrace.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._GetAlignmentTrace.C differ
diff --git a/src/AS_CNS/._Makefile b/src/AS_CNS/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._Makefile differ
diff --git a/src/AS_CNS/._MergeMultiAligns.C b/src/AS_CNS/._MergeMultiAligns.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MergeMultiAligns.C differ
diff --git a/src/AS_CNS/._MergeRefine.C b/src/AS_CNS/._MergeRefine.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MergeRefine.C differ
diff --git a/src/AS_CNS/._MultiAlign.C b/src/AS_CNS/._MultiAlign.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlign.C differ
diff --git a/src/AS_CNS/._MultiAlign.H b/src/AS_CNS/._MultiAlign.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlign.H differ
diff --git a/src/AS_CNS/._MultiAlignContig.C b/src/AS_CNS/._MultiAlignContig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignContig.C differ
diff --git a/src/AS_CNS/._MultiAlignMatePairAnalysis.C b/src/AS_CNS/._MultiAlignMatePairAnalysis.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignMatePairAnalysis.C differ
diff --git a/src/AS_CNS/._MultiAlignMatePairAnalysis.H b/src/AS_CNS/._MultiAlignMatePairAnalysis.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignMatePairAnalysis.H differ
diff --git a/src/AS_CNS/._MultiAlignPrint.C b/src/AS_CNS/._MultiAlignPrint.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignPrint.C differ
diff --git a/src/AS_CNS/._MultiAlignSizeAnalysis.C b/src/AS_CNS/._MultiAlignSizeAnalysis.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignSizeAnalysis.C differ
diff --git a/src/AS_CNS/._MultiAlignSizeAnalysis.H b/src/AS_CNS/._MultiAlignSizeAnalysis.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignSizeAnalysis.H differ
diff --git a/src/AS_CNS/._MultiAlignStore.C b/src/AS_CNS/._MultiAlignStore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignStore.C differ
diff --git a/src/AS_CNS/._MultiAlignStore.H b/src/AS_CNS/._MultiAlignStore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignStore.H differ
diff --git a/src/AS_CNS/._MultiAlignUnitig.C b/src/AS_CNS/._MultiAlignUnitig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignUnitig.C differ
diff --git a/src/AS_CNS/._MultiAlignment_CNS.C b/src/AS_CNS/._MultiAlignment_CNS.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignment_CNS.C differ
diff --git a/src/AS_CNS/._MultiAlignment_CNS.H b/src/AS_CNS/._MultiAlignment_CNS.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignment_CNS.H differ
diff --git a/src/AS_CNS/._MultiAlignment_CNS_private.H b/src/AS_CNS/._MultiAlignment_CNS_private.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._MultiAlignment_CNS_private.H differ
diff --git a/src/AS_CNS/._PrintAlignment.C b/src/AS_CNS/._PrintAlignment.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._PrintAlignment.C differ
diff --git a/src/AS_CNS/._RefreshMANode.C b/src/AS_CNS/._RefreshMANode.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._RefreshMANode.C differ
diff --git a/src/AS_CNS/._ReplaceEndUnitigInContig.C b/src/AS_CNS/._ReplaceEndUnitigInContig.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._ReplaceEndUnitigInContig.C differ
diff --git a/src/AS_CNS/._SeqAn_CNS.C b/src/AS_CNS/._SeqAn_CNS.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._SeqAn_CNS.C differ
diff --git a/src/AS_CNS/._addReadsToUnitigs.C b/src/AS_CNS/._addReadsToUnitigs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._addReadsToUnitigs.C differ
diff --git a/src/AS_CNS/._ctgcns.C b/src/AS_CNS/._ctgcns.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._ctgcns.C differ
diff --git a/src/AS_CNS/._submit-parallel.pl b/src/AS_CNS/._submit-parallel.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._submit-parallel.pl differ
diff --git a/src/AS_CNS/._tigStore.C b/src/AS_CNS/._tigStore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._tigStore.C differ
diff --git a/src/AS_CNS/._tigStoreDelete.pl b/src/AS_CNS/._tigStoreDelete.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._tigStoreDelete.pl differ
diff --git a/src/AS_CNS/._utgcns.C b/src/AS_CNS/._utgcns.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._utgcns.C differ
diff --git a/src/AS_CNS/._utgcnsfix.C b/src/AS_CNS/._utgcnsfix.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_CNS/._utgcnsfix.C differ
diff --git a/src/AS_CNS/MultiAlign.C b/src/AS_CNS/MultiAlign.C
index 4544b78..8aa608a 100644
--- a/src/AS_CNS/MultiAlign.C
+++ b/src/AS_CNS/MultiAlign.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: MultiAlign.C 4412 2013-09-17 02:40:25Z brianwalenz $";
+static const char *rcsid = "$Id: MultiAlign.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include <assert.h>
#include <stdio.h>
@@ -31,7 +31,7 @@ static const char *rcsid = "$Id: MultiAlign.C 4412 2013-09-17 02:40:25Z brianwal
#include "MultiAlignment_CNS.H"
#include "MultiAlignment_CNS_private.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
#undef DEBUG_CREATE
#undef DEBUG_FILES
@@ -46,7 +46,11 @@ CreateMultiAlignT(void) {
ma->data.unitig_microhet_prob = 1.0;
ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
ma->data.contig_status = AS_UNPLACED;
@@ -79,7 +83,11 @@ CreateEmptyMultiAlignT(void) {
ma->data.unitig_microhet_prob = 1.0;
ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
ma->data.contig_status = AS_UNPLACED;
@@ -111,7 +119,11 @@ ClearMultiAlignT(MultiAlignT *ma) {
ma->data.unitig_microhet_prob = 1.0;
ma->data.unitig_status = AS_UNASSIGNED;
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+
+ ma->data.unitig_suggest_repeat = false;
+ ma->data.unitig_suggest_unique = false;
+ ma->data.unitig_force_repeat = false;
+ ma->data.unitig_force_unique = false;
ma->data.contig_status = AS_UNPLACED;
@@ -687,13 +699,16 @@ DumpMultiAlignForHuman(FILE *out, MultiAlignT *ma, bool isUnitig) {
fprintf(out, "len %d\n", (cns) ? (int)strlen(cns) : 0);
fprintf(out, "cns %s\n", (cns) ? cns : "");
fprintf(out, "qlt %s\n", (qlt) ? qlt : "");
- fprintf(out, "data.unitig_coverage_stat %f\n", ma->data.unitig_coverage_stat);
- fprintf(out, "data.unitig_microhet_prob %f\n", ma->data.unitig_microhet_prob);
- fprintf(out, "data.unitig_status %c\n", ma->data.unitig_status);
- fprintf(out, "data.unitig_unique_rept %c\n", ma->data.unitig_unique_rept);
- fprintf(out, "data.contig_status %c\n", ma->data.contig_status);
- fprintf(out, "data.num_frags %u\n", ma->data.num_frags);
- fprintf(out, "data.num_unitigs %u\n", ma->data.num_unitigs);
+ fprintf(out, "data.unitig_coverage_stat %f\n", ma->data.unitig_coverage_stat);
+ fprintf(out, "data.unitig_microhet_prob %f\n", ma->data.unitig_microhet_prob);
+ fprintf(out, "data.unitig_status %c\n", ma->data.unitig_status);
+ fprintf(out, "data.unitig_suggest_repeat %c\n", ma->data.unitig_suggest_repeat ? 'T' : 'F');
+ fprintf(out, "data.unitig_suggest_unique %c\n", ma->data.unitig_suggest_unique ? 'T' : 'F');
+ fprintf(out, "data.unitig_force_repeat %c\n", ma->data.unitig_force_repeat ? 'T' : 'F');
+ fprintf(out, "data.unitig_force_unique %c\n", ma->data.unitig_force_unique ? 'T' : 'F');
+ fprintf(out, "data.contig_status %c\n", ma->data.contig_status);
+ fprintf(out, "data.num_frags %u\n", ma->data.num_frags);
+ fprintf(out, "data.num_unitigs %u\n", ma->data.num_unitigs);
for (uint32 i=0; i<GetNumIntMultiPoss(ma->f_list); i++) {
IntMultiPos *imp = GetIntMultiPos(ma->f_list, i);
@@ -872,30 +887,78 @@ LoadMultiAlignFromHuman(MultiAlignT *ma, bool &isUnitig, FILE *in) {
exit(1);
}
+
+
if (LoadMultiAlignFromHumanGetLine(in, LINElen, LINEmax, LINE, W) == 0)
- fprintf(stderr, "MultiAlign %d not loaded: Expecting 'data.unitig_unique_rept' line, got empty line.\n", ma->maID), exit(1);
+ fprintf(stderr, "MultiAlign %d not loaded: Expecting 'data.unitig_suggest_repeat' line, got empty line.\n", ma->maID), exit(1);
- if (strcmp(W[0], "data.unitig_unique_rept") == 0) {
+ if (strcmp(W[0], "data.unitig_suggest_repeat") == 0) {
switch (W[1][0]) {
- case AS_FORCED_NONE:
- ma->data.unitig_unique_rept = AS_FORCED_NONE;
+ case 'T': ma->data.unitig_suggest_repeat = true; break;
+ case 'F': ma->data.unitig_suggest_repeat = false; break;
+ default:
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_suggest_repeat in '%s'\n", ma->maID, LINE);
+ exit(1);
break;
- case AS_FORCED_UNIQUE:
- ma->data.unitig_unique_rept = AS_FORCED_UNIQUE;
+ }
+ } else {
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_suggest_repeat in '%s'\n", ma->maID, LINE);
+ exit(1);
+ }
+
+ if (LoadMultiAlignFromHumanGetLine(in, LINElen, LINEmax, LINE, W) == 0)
+ fprintf(stderr, "MultiAlign %d not loaded: Expecting 'data.unitig_suggest_unique' line, got empty line.\n", ma->maID), exit(1);
+
+ if (strcmp(W[0], "data.unitig_suggest_unique") == 0) {
+ switch (W[1][0]) {
+ case 'T': ma->data.unitig_suggest_unique = true; break;
+ case 'F': ma->data.unitig_suggest_unique = false; break;
+ default:
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_suggest_unique in '%s'\n", ma->maID, LINE);
+ exit(1);
break;
- case AS_FORCED_REPEAT:
- ma->data.unitig_unique_rept = AS_FORCED_REPEAT;
+ }
+ } else {
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_suggest_unique in '%s'\n", ma->maID, LINE);
+ exit(1);
+ }
+
+ if (LoadMultiAlignFromHumanGetLine(in, LINElen, LINEmax, LINE, W) == 0)
+ fprintf(stderr, "MultiAlign %d not loaded: Expecting 'data.unitig_force_repeat' line, got empty line.\n", ma->maID), exit(1);
+
+ if (strcmp(W[0], "data.unitig_force_repeat") == 0) {
+ switch (W[1][0]) {
+ case 'T': ma->data.unitig_force_repeat = true; break;
+ case 'F': ma->data.unitig_force_repeat = false; break;
+ default:
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_force_repeat in '%s'\n", ma->maID, LINE);
+ exit(1);
break;
+ }
+ } else {
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_force_repeat in '%s'\n", ma->maID, LINE);
+ exit(1);
+ }
+
+ if (LoadMultiAlignFromHumanGetLine(in, LINElen, LINEmax, LINE, W) == 0)
+ fprintf(stderr, "MultiAlign %d not loaded: Expecting 'data.unitig_force_unique' line, got empty line.\n", ma->maID), exit(1);
+
+ if (strcmp(W[0], "data.unitig_force_unique") == 0) {
+ switch (W[1][0]) {
+ case 'T': ma->data.unitig_force_unique = true; break;
+ case 'F': ma->data.unitig_force_unique = false; break;
default:
- fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_unique_rept in '%s'\n", ma->maID, LINE);
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_force_unique in '%s'\n", ma->maID, LINE);
exit(1);
break;
}
} else {
- fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_unique_rept in '%s'\n", ma->maID, LINE);
+ fprintf(stderr, "MultiAlign %d not loaded: Unknown data.unitig_force_unique in '%s'\n", ma->maID, LINE);
exit(1);
}
+
+
if (LoadMultiAlignFromHumanGetLine(in, LINElen, LINEmax, LINE, W) == 0)
fprintf(stderr, "MultiAlign %d not loaded: Expecting 'data.contig_status' line, got empty line.\n", ma->maID), exit(1);
diff --git a/src/AS_CNS/MultiAlign.H b/src/AS_CNS/MultiAlign.H
index a0ca6f1..c8b91ad 100644
--- a/src/AS_CNS/MultiAlign.H
+++ b/src/AS_CNS/MultiAlign.H
@@ -22,7 +22,7 @@
#ifndef MULTIALIGN_H
#define MULTIALIGN_H
-static const char *rcsid_MULTIALIGN_H = "$Id: MultiAlign.H 4390 2013-08-24 05:56:55Z brianwalenz $";
+static const char *rcsid_MULTIALIGN_H = "$Id: MultiAlign.H 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_MSG_pmesg.H"
#include "AS_UTL_Var.H"
@@ -40,7 +40,12 @@ typedef struct {
double unitig_microhet_prob;
UnitigStatus unitig_status;
- UnitigFUR unitig_unique_rept;
+
+ uint32 unitig_suggest_repeat:1;
+ uint32 unitig_suggest_unique:1;
+ uint32 unitig_force_repeat:1;
+ uint32 unitig_force_unique:1;
+ uint32 spare:28;
ContigStatus contig_status;
diff --git a/src/AS_CNS/MultiAlignMatePairAnalysis.C b/src/AS_CNS/MultiAlignMatePairAnalysis.C
index 1df3584..aff1492 100644
--- a/src/AS_CNS/MultiAlignMatePairAnalysis.C
+++ b/src/AS_CNS/MultiAlignMatePairAnalysis.C
@@ -17,7 +17,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: MultiAlignMatePairAnalysis.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: MultiAlignMatePairAnalysis.C 4603 2015-02-06 09:34:23Z brianwalenz $";
#include "MultiAlignMatePairAnalysis.H"
@@ -402,6 +402,7 @@ matePairAnalysis::writeUpdate(char *prefix) {
void
matePairAnalysis::drawPlots(char *prefix) {
char datName[FILENAME_MAX];
+ char libName[FILENAME_MAX];
if (prefix[strlen(prefix)-1] == '/')
prefix[strlen(prefix)-1] = 0;
@@ -422,8 +423,15 @@ matePairAnalysis::drawPlots(char *prefix) {
if (data->dist.size() == 0)
continue;
+ strcpy(libName, ld->library->libraryName);
+
+ for (uint32 ii=0; libName[ii]; ii++)
+ if (libName[ii] == '/')
+ libName[ii] = '_';
+
sprintf(datName, "%s.%03d.%s.%s.dat",
- prefix, li, AS_READ_ORIENT_NAMES[oi], ld->library->libraryName);
+ prefix, li, AS_READ_ORIENT_NAMES[oi], libName);
+
errno = 0;
FILE *dat = fopen(datName, "w");
if (errno)
diff --git a/src/AS_CNS/MultiAlignStore.C b/src/AS_CNS/MultiAlignStore.C
index ed7b977..6a1c335 100644
--- a/src/AS_CNS/MultiAlignStore.C
+++ b/src/AS_CNS/MultiAlignStore.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: MultiAlignStore.C 4455 2013-10-18 21:34:54Z brianwalenz $";
+static const char *rcsid = "$Id: MultiAlignStore.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_global.H"
#include "AS_UTL_fileIO.H"
@@ -358,7 +358,6 @@ void
MultiAlignStore::insertMultiAlign(MultiAlignT *ma, bool isUnitig, bool keepInCache) {
assert(ma->data.unitig_status != 0);
- assert(ma->data.unitig_unique_rept != 0);
assert(ma->data.contig_status != 0);
// Check that the components do not exceed the bound. This is an inlined version of a singe test
diff --git a/src/AS_CNS/MultiAlignStore.H b/src/AS_CNS/MultiAlignStore.H
index fad5ce0..df9dd08 100644
--- a/src/AS_CNS/MultiAlignStore.H
+++ b/src/AS_CNS/MultiAlignStore.H
@@ -22,7 +22,7 @@
#ifndef MULTIALIGNSTORE_H
#define MULTIALIGNSTORE_H
-static const char *rcsid_MULTIALIGNSTORE_H = "$Id: MultiAlignStore.H 4421 2013-09-18 04:59:32Z brianwalenz $";
+static const char *rcsid_MULTIALIGNSTORE_H = "$Id: MultiAlignStore.H 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_global.H"
#include "MultiAlign.H"
@@ -143,7 +143,12 @@ public:
int32 getUnitigCoverageStat(int32 maID);
double getUnitigMicroHetProb(int32 maID);
UnitigStatus getUnitigStatus(int32 maID);
- UnitigFUR getUnitigFUR(int32 maID);
+
+ bool getUnitigSuggestRepeat(int32 maID);
+ bool getUnitigSuggestUnique(int32 maID);
+ bool getUnitigForceRepeat(int32 maID);
+ bool getUnitigForceUnique(int32 maID);
+
ContigStatus getContigStatus(int32 maID);
uint32 getNumFrags(int32 maID, bool isUnitig);
@@ -152,7 +157,11 @@ public:
void setUnitigCoverageStat(int32 maID, double cs);
void setUnitigMicroHetProb(int32 maID, double mp);
void setUnitigStatus(int32 maID, UnitigStatus status);
- void setUnitigFUR(int32 maID, UnitigFUR fur);
+
+ void setUnitigSuggestRepeat(int32 maID, bool enable=true);
+ void setUnitigSuggestUnique(int32 maID, bool enable=true);
+ void setUnitigForceRepeat(int32 maID, bool enable=true);
+ void setUnitigForceUnique(int32 maID, bool enable=true);
void setContigStatus(int32 maID, ContigStatus status);
@@ -281,11 +290,35 @@ MultiAlignStore::getUnitigStatus(int32 maID) {
}
inline
-UnitigFUR
-MultiAlignStore::getUnitigFUR(int32 maID) {
+bool
+MultiAlignStore::getUnitigSuggestRepeat(int32 maID) {
+ assert(maID >= 0);
+ assert(maID < (int32)utgLen);
+ return(utgRecord[maID].mad.unitig_suggest_repeat);
+}
+
+inline
+bool
+MultiAlignStore::getUnitigSuggestUnique(int32 maID) {
+ assert(maID >= 0);
+ assert(maID < (int32)utgLen);
+ return(utgRecord[maID].mad.unitig_suggest_unique);
+}
+
+inline
+bool
+MultiAlignStore::getUnitigForceRepeat(int32 maID) {
+ assert(maID >= 0);
+ assert(maID < (int32)utgLen);
+ return(utgRecord[maID].mad.unitig_force_repeat);
+}
+
+inline
+bool
+MultiAlignStore::getUnitigForceUnique(int32 maID) {
assert(maID >= 0);
assert(maID < (int32)utgLen);
- return(utgRecord[maID].mad.unitig_unique_rept);
+ return(utgRecord[maID].mad.unitig_force_unique);
}
inline
@@ -342,12 +375,42 @@ MultiAlignStore::setUnitigStatus(int32 maID, UnitigStatus status) {
inline
void
-MultiAlignStore::setUnitigFUR(int32 maID, UnitigFUR fur) {
+MultiAlignStore::setUnitigSuggestRepeat(int32 maID, bool enable) {
+ assert(maID >= 0);
+ assert(maID < (int32)utgLen);
+ utgRecord[maID].mad.unitig_suggest_repeat = enable;
+ if (utgCache[maID])
+ utgCache[maID]->data.unitig_suggest_repeat = enable;
+}
+
+inline
+void
+MultiAlignStore::setUnitigSuggestUnique(int32 maID, bool enable) {
+ assert(maID >= 0);
+ assert(maID < (int32)utgLen);
+ utgRecord[maID].mad.unitig_suggest_unique = enable;
+ if (utgCache[maID])
+ utgCache[maID]->data.unitig_suggest_unique = enable;
+}
+
+inline
+void
+MultiAlignStore::setUnitigForceRepeat(int32 maID, bool enable) {
+ assert(maID >= 0);
+ assert(maID < (int32)utgLen);
+ utgRecord[maID].mad.unitig_force_repeat = enable;
+ if (utgCache[maID])
+ utgCache[maID]->data.unitig_force_repeat = enable;
+}
+
+inline
+void
+MultiAlignStore::setUnitigForceUnique(int32 maID, bool enable) {
assert(maID >= 0);
assert(maID < (int32)utgLen);
- utgRecord[maID].mad.unitig_unique_rept = fur;
+ utgRecord[maID].mad.unitig_force_unique = enable;
if (utgCache[maID])
- utgCache[maID]->data.unitig_unique_rept = fur;
+ utgCache[maID]->data.unitig_force_unique = enable;
}
inline
diff --git a/src/AS_CNS/addReadsToUnitigs.C b/src/AS_CNS/addReadsToUnitigs.C
index dba6660..6334599 100644
--- a/src/AS_CNS/addReadsToUnitigs.C
+++ b/src/AS_CNS/addReadsToUnitigs.C
@@ -23,7 +23,7 @@ const char *mainid = "$Id: $";
#include "AS_global.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
#include "MultiAlign.H"
#include "MultiAlignStore.H"
@@ -31,34 +31,36 @@ const char *mainid = "$Id: $";
#include "MultiAlignment_CNS_private.H"
#include <vector>
+#include <string>
+#include <map>
using namespace std;
class readMap {
public:
readMap() {
- rIID = UINT32_MAX;
+ good = false;
+ proc = false;
+
rFWD = false;
+ rIID = UINT32_MAX;
rCNT = 0;
tIID = UINT32_MAX;
tBGN = 0;
tEND = 0;
-
- good = false;
- proc = false;
};
- uint32 rIID;
+ bool good;
+ bool proc;
+
bool rFWD;
+ uint32 rIID;
uint32 rCNT;
uint32 tIID;
uint32 tBGN;
uint32 tEND;
-
- bool good;
- bool proc;
};
@@ -81,15 +83,25 @@ main(int argc, char **argv) {
bool doConsensus = false;
bool doModify = true;
+#ifdef UNFINISHED_ADD_TO_SINGLETON
+ bool doPlaceUnmapped = true;
+ bool doDeleteUnmapped = false;
+#endif
+
int32 numFailures = 0;
int32 numSkipped = 0;
bool showResult = false;
- bool ignoreContains = false;
- double ignoreContainT = 0.75;
+ char *lookupFile = NULL;
+ map<string,uint32> lookupIID;
+
+#ifdef UNFINISHED_ADD_TO_SINGLETON
+ vector<bool> iidInTig; // true if the read is already in a unitig
+ vector<uint32> iidInTigByLib; // count of reads in tigs, by library
+ vector<uint32> iidInLib; // count of reads, by library
+#endif
- bool inplace = false;
bool loadall = false;
CNS_Options options = { CNS_OPTIONS_SPLIT_ALLELES_DEFAULT,
@@ -120,7 +132,11 @@ main(int argc, char **argv) {
fprintf(stderr, "invalid tigStore version (-t store version) '-t %s %s'.\n", argv[arg-1], argv[arg]), exit(1);
} else if (strcmp(argv[arg], "-m") == 0) {
- alignMapNames.push_back(argv[++arg]);
+ while ((argv[arg+1] != NULL) && (AS_UTL_fileExists(argv[arg+1], false, false) == true))
+ alignMapNames.push_back(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-lookup") == 0) {
+ lookupFile = argv[++arg];
} else if (strcmp(argv[arg], "-r") == 0) {
doConsensus = true;
@@ -131,19 +147,12 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-V") == 0) {
VERBOSE_MULTIALIGN_OUTPUT++;
- } else if (strcmp(argv[arg], "-n") == 0) {
- doModify = false;
-
- } else if (strcmp(argv[arg], "-nocontains") == 0) {
- ignoreContains = true;
- ignoreContainT = atof(argv[++arg]);
-
- } else if (strcmp(argv[arg], "-inplace") == 0) {
- inplace = true;
-
} else if (strcmp(argv[arg], "-loadall") == 0) {
loadall = true;
+ } else if (strcmp(argv[arg], "-n") == 0) {
+ doModify = false;
+
} else {
err++;
}
@@ -156,21 +165,130 @@ main(int argc, char **argv) {
err++;
if (alignMapNames.size() == 0)
err++;
+ if (lookupFile == NULL)
+ err++;
if (err) {
+ fprintf(stderr, "usage: %s -g gkpStore -t tigStore version -m coords\n", argv[0]);
+ fprintf(stderr, " -g gkpStore gatekeeper store\n");
+ fprintf(stderr, " -t tigStore version tigStore and version to modify\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -m map-file input map coords\n");
+ fprintf(stderr, " -M fastqUIDmap gatekeeper output fastqUIDmap for read name to IID translation\n");
+ fprintf(stderr, "\n");
+#if 0
+ fprintf(stderr, "unmapped reads: default is to promote to singleton unitigs\n");
+ fprintf(stderr, " -U leave unmapped reads alone (will crash CGW)\n");
+ fprintf(stderr, " -D delete unmapped reads from gkpStore\n");
+#else
+ fprintf(stderr, "unmapped reads: all reads that are mapped and eligible for addition must be\n");
+ fprintf(stderr, "marked as deleted before running this program. reads that are added will be\n");
+ fprintf(stderr, "undeleted. reads that are not added will remain deleted.\n");
+#endif
+ fprintf(stderr, "\n");
+ fprintf(stderr, "consensus: default is to not rebuild consensus\n");
+ fprintf(stderr, " -r rebuild consensus including the new reads\n");
+ fprintf(stderr, " -v show result\n");
+ fprintf(stderr, " -V verbose\n");
+ fprintf(stderr, " -loadall load all reads in gkpStore into memory (faster consensus)\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -n do all the work, but discard the result\n");
+
+ if (gkpName == NULL)
+ fprintf(stderr, "ERROR: no gkpStore (-g) supplied.\n");
+ if (tigName == NULL)
+ fprintf(stderr, "ERROR: no tigStore (-t) supplied.\n");
+ if (alignMapNames.size() == 0)
+ fprintf(stderr, "ERROR: no map-file (-m) inputs supplied.\n");
+ if (lookupFile == NULL)
+ fprintf(stderr, "ERROR: no fasqUIDmap (-M) supplied.\n");
+
exit(1);
}
+
+ {
+ fprintf(stderr, "Loading Name to IID map from '%s'\n", lookupFile);
+
+ errno = 0;
+ FILE *LF = fopen(lookupFile, "r");
+ if (errno)
+ fprintf(stderr, "Failed to open fastqUIDmap '%s'\n", lookupFile);
+
+ char LL[1024];
+ fgets(LL, 1024, LF);
+
+ while (!feof(LF)) {
+ chomp(LL); // Shouldn't be necessary, but splitToWords isn't doing it!
+
+ splitToWords SW(LL);
+
+ if (SW.numWords() == 3) {
+ lookupIID[string(SW[2])] = SW(1);
+
+ } else if (SW.numWords() == 6) {
+ lookupIID[string(SW[2])] = SW(1);
+ lookupIID[string(SW[5])] = SW(4);
+ //fprintf(stderr, "'%s' - %u -- '%s' - %u\n",
+ // SW[2], SW(1), SW[5], SW(4));
+
+ } else {
+ }
+
+ fgets(LL, 1024, LF);
+ }
+
+ fprintf(stderr, "Loaded "F_SIZE_T" name to IIDs\n", lookupIID.size());
+ }
+
+
+#ifdef UNFINISHED_ADD_TO_SINGLETON
+ {
+ fprintf(stderr, "Loading tig placement from tigStore '%s'\n", tigName);
+
+ tigStore = new MultiAlignStore(tigName, tigVers, 0, 0, false, false, false); // Read only
+
+ for (uint32 ti=0; ti<tigStore->numUnitigs(); ti++) {
+ if (tigStore->isDeleted(ti, true))
+ continue;
+
+ MultiAlignT *ma = tigStore->loadMultiAlign(ti, true);
+
+ if (ma == NULL)
+ continue;
+
+ uint32 fiMax = GetNumIntMultiPoss(ma->f_list);
+
+ for (uint32 fi=0; fi<fiMax; fi++) {
+ IntMultiPos *imp = GetIntMultiPos(ma->f_list, fi);
+
+ iidInTig[imp->ident] = true;
+ }
+
+ tigStore->unloadMultiAlign(ti, true);
+ }
+
+ delete tigStore;
+ }
+#endif
+
+
//
// Load the alignment map. The challenge here is to parse the unitig and read names
// into correct IIDs. We assume that:
// Reads were dumped with -dumpfasta and have names ">UID,IID"
// Unitigs were dumped with tigStore -d consensus and have names "utgIID"
+ // Alignments are in the convertToExtent -extended format
//
+
uint32 totAligns = 0;
+ uint32 totUnmap = 0;
uint32 totUnique = 0;
uint32 totDups = 0;
+ uint32 lowID = UINT32_MAX;
+ uint32 maxID = 0;
+
for (uint32 an=0; an<alignMapNames.size(); an++) {
uint32 numAligns = 0;
uint32 numUnique = 0;
@@ -189,28 +307,48 @@ main(int argc, char **argv) {
while (!feof(M)) {
numAligns++;
- for (uint32 xx=0; !isspace(L[xx]); xx++)
+ // Some gatekeeper dumps contain NAME,IID. Strip out the ,IID, BEFORE converting to words.
+ bool cF = false;
+ for (uint32 xx=0; !isspace(L[xx]); xx++) {
if (L[xx] == ',')
+ cF = true;
+ if (cF == true)
L[xx] = ' ';
+ }
splitToWords S(L);
readMap rm;
rm.rIID = S(1);
- rm.rFWD = (S(5) < S(6));
+ rm.rFWD = (S(4) < S(5));
rm.rCNT = 1;
- rm.tIID = atoi(S[7] + 3);
- rm.tBGN = S(9);
- rm.tEND = S(10);
+ assert(S[6][0] == 'u'); // Unitig must be from a tigStore dump, and will be
+ assert(S[6][1] == 't'); // named 'utg#######'.
+ assert(S[6][2] == 'g');
+
+ rm.tIID = atoi(S[6] + 3);
+ rm.tBGN = S(8);
+ rm.tEND = S(9);
+
+ // We no longer use the encoded IID, but always lookup the read name in the map.
+ rm.rIID = lookupIID[string(S[0])];
+
+ if (rm.rIID == 0)
+ fprintf(stderr, "FAILED to find IID for read '%s'\n", S[0]);
+
+ //fprintf(stderr, "%d %d %d -- %d %d %d\n", rm.rIID, rm.rFWD, rm.rCNT, rm.tIID, rm.tBGN, rm.tEND);
- if (RM.size() < rm.rIID)
+ if (RM.size() <= rm.rIID)
RM.resize(2 * rm.rIID);
rm.rCNT = RM[rm.rIID].rCNT + 1;
RM[rm.rIID] = rm;
+ lowID = (lowID < rm.rIID) ? lowID : rm.rIID;
+ maxID = (maxID < rm.rIID) ? rm.rIID : maxID;
+
fgets(L, 1024, M);
}
@@ -221,34 +359,47 @@ main(int argc, char **argv) {
totAligns += numAligns;
}
- for (uint32 xx=0; xx<RM.size(); xx++) {
+ for (uint32 xx=lowID; xx<=maxID; xx++) {
+ if (RM[xx].rCNT == 0)
+ totUnmap++;
if (RM[xx].rCNT == 1)
totUnique++;
if (RM[xx].rCNT > 1)
totDups++;
}
- fprintf(stderr, "Loaded %u aligns, %u reads had a unique alignment and %u reads had multiple aligns.\n",
- totAligns, totUnique, totDups);
+ fprintf(stderr, "Loaded %u aligns from ID %u to %u, %u reads unmapped, %u reads had a unique alignment and %u reads had multiple aligns.\n",
+ totAligns, lowID, maxID, totUnmap, totUnique, totDups);
//
// Update deletion status in gkpStore. This processes every read, regardless.
//
fprintf(stderr, "Processing mate pairs, updating gkpStore.\n");
+ gkpStore = new gkStore(gkpName, false, true); // last arg - TRUE - writable
+ uint32 unpaired = 0;
+ uint32 multiple = 0;
uint32 pairsToSame = 0;
uint32 pairsToDiff = 0;
- gkpStore = new gkStore(gkpName, FALSE, TRUE); // last arg - TRUE - writable
+#ifdef UNFINISHED_ADD_TO_SINGLETON
+ // Need to process all reads, since we don't know where the first/last unmapped read is!
+ // We could instead process from the first to last deleted read in gkpStore, or ask which
+ // libraries were being added.
+ lowID = 1;
+ maxID = gkpStore->gkStore_getNumFragments();
+#endif
- for (uint32 ff=0, mm=0; ff<RM.size(); ff++) {
+ for (uint32 ff=lowID, mm=lowID; ff<=maxID; ff++) {
gkFragment read;
gkFragment mate;
- if (RM[ff].rCNT != 1)
- // Not mapped, mapped too much
- continue;
+ // I think this is just a short circuit of the two checks of 'one or both reads has too
+ // few/many mappings' below.
+ //if (RM[ff].rCNT != 1)
+ // // Not mapped, mapped too much
+ // continue;
gkpStore->gkStore_getFragment(ff, &read, GKFRAGMENT_INF);
@@ -258,11 +409,25 @@ main(int argc, char **argv) {
// No mate, pacbio read?
continue;
+ if (mm < ff)
+ // Already processed.
+ continue;
+
gkpStore->gkStore_getFragment(mm, &mate, GKFRAGMENT_INF);
- if ((RM[ff].rCNT != 1) ||
- (RM[mm].rCNT != 1)) {
- // One or both reads has too few or too many mappings. Don't use the pair.
+ if ((RM[ff].rCNT == 0) ||
+ (RM[mm].rCNT == 0)) {
+ // One or both reads has too few mappings.
+ unpaired++;
+ RM[ff].good = false;
+ RM[mm].good = false;
+ continue;
+ }
+
+ if ((RM[ff].rCNT > 1) ||
+ (RM[mm].rCNT > 1)) {
+ // One or both reads has too many mappings.
+ multiple++;
RM[ff].good = false;
RM[mm].good = false;
continue;
@@ -285,15 +450,17 @@ main(int argc, char **argv) {
delete gkpStore;
+ fprintf(stderr, "Will NOT add %u pairs - one read failed to map.\n", unpaired);
+ fprintf(stderr, "Will NOT add %u pairs - multiple mappings.\n", multiple);
fprintf(stderr, "Will add %u pairs in the same unitig\n", pairsToSame);
fprintf(stderr, "Will add %u pairs in different unitigs\n", pairsToDiff);
//
- // Open stores. gkpStore cannot be opened fr writing, because then we can't loadall.
+ // Open stores. gkpStore cannot be opened for writing, because then we can't loadall.
//
- gkpStore = new gkStore(gkpName, FALSE, FALSE); // last arg - FALSE - not writable
- tigStore = new MultiAlignStore(tigName, tigVers, 0, 0, TRUE, TRUE, FALSE); // Write back to the same version
+ gkpStore = new gkStore(gkpName, false, false); // last arg - false - not writable
+ tigStore = new MultiAlignStore(tigName, tigVers, 0, 0, true, true, false); // Write back to the same version
if (loadall) {
fprintf(stderr, "Loading all reads into memory.\n");
@@ -313,7 +480,7 @@ main(int argc, char **argv) {
(RM[bb].proc == true)) // already added
continue;
- MultiAlignT *ma = tigStore->loadMultiAlign(RM[bb].tIID, TRUE);
+ MultiAlignT *ma = tigStore->loadMultiAlign(RM[bb].tIID, true);
uint32 ungapLength = GetMultiAlignUngappedLength(ma);
uint32 gapLength = GetMultiAlignLength(ma);
@@ -322,7 +489,7 @@ main(int argc, char **argv) {
GetMultiAlignUngapToGap(ma, ungapToGap);
- fprintf(stderr, "Loaded UTG %u offset size %lu\n", ma->maID, ungapToGap.size());
+ //fprintf(stderr, "Loaded UTG %u offset size %lu\n", ma->maID, ungapToGap.size());
//for (uint32 xx=0; xx<ungapToGap.size(); xx++)
// fprintf(stderr, "ungap %u -> gap %u\n", xx, ungapToGap[xx]);
@@ -339,7 +506,7 @@ main(int argc, char **argv) {
readsAdded++;
- fprintf(stderr, "bb=%u ee=%u ADD frag %u to unitig %u at %u,%u (from %u,%u)\n",
+ fprintf(stdout, "bb=%u ee=%u ADD frag %u to unitig %u at %u,%u (from ungapped %u,%u)\n",
bb, ee,
RM[ee].rIID, RM[ee].tIID, bgn, end, RM[ee].tBGN, RM[ee].tEND);
@@ -384,7 +551,7 @@ main(int argc, char **argv) {
if (doModify) {
fprintf(stderr, "Updating unitig %u\n", ma->maID);
- tigStore->insertMultiAlign(ma, TRUE, FALSE);
+ tigStore->insertMultiAlign(ma, true, false);
}
}
diff --git a/src/AS_CNS/tigStore.C b/src/AS_CNS/tigStore.C
index e9b494b..d0b83fa 100644
--- a/src/AS_CNS/tigStore.C
+++ b/src/AS_CNS/tigStore.C
@@ -19,11 +19,11 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: tigStore.C 4423 2013-09-18 06:21:48Z brianwalenz $";
+const char *mainid = "$Id: tigStore.C 4572 2014-10-14 23:06:05Z brianwalenz $";
#include "AS_global.H"
#include "AS_UTL_decodeRange.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
#include "MultiAlign.H"
#include "MultiAlignStore.H"
@@ -115,17 +115,49 @@ changeProperties(MultiAlignStore *tigStore,
}
tigStore->setUnitigStatus(tid, st);
- } else if (strncmp(op, "unitig_unique_rept", 18) == 0) {
- UnitigFUR ur = tigStore->getUnitigFUR(tid);
+ } else if (strncmp(op, "unitig_suggest_repeat", 21) == 0) {
+ bool ur = tigStore->getUnitigSuggestRepeat(tid);
switch (*vp) {
- case AS_FORCED_NONE: ur = AS_FORCED_NONE; break;
- case AS_FORCED_UNIQUE: ur = AS_FORCED_UNIQUE; break;
- case AS_FORCED_REPEAT: ur = AS_FORCED_REPEAT; break;
+ case 'T': ur = true; break;
+ case 'F': ur = false; break;
default:
- fprintf(stderr, "unknown unitig_unique_rept in '%s'\n", editLine);
+ fprintf(stderr, "unknown unitig_suggest_repeat in '%s'\n", editLine);
break;
}
- tigStore->setUnitigFUR(tid, ur);
+ tigStore->setUnitigSuggestRepeat(tid, ur);
+
+ } else if (strncmp(op, "unitig_suggest_unique", 21) == 0) {
+ bool ur = tigStore->getUnitigSuggestUnique(tid);
+ switch (*vp) {
+ case 'T': ur = true; break;
+ case 'F': ur = false; break;
+ default:
+ fprintf(stderr, "unknown unitig_suggest_unique in '%s'\n", editLine);
+ break;
+ }
+ tigStore->setUnitigSuggestUnique(tid, ur);
+
+ } else if (strncmp(op, "unitig_force_repeat", 21) == 0) {
+ bool ur = tigStore->getUnitigForceRepeat(tid);
+ switch (*vp) {
+ case 'T': ur = true; break;
+ case 'F': ur = false; break;
+ default:
+ fprintf(stderr, "unknown unitig_force_repeat in '%s'\n", editLine);
+ break;
+ }
+ tigStore->setUnitigForceRepeat(tid, ur);
+
+ } else if (strncmp(op, "unitig_force_unique", 21) == 0) {
+ bool ur = tigStore->getUnitigForceUnique(tid);
+ switch (*vp) {
+ case 'T': ur = true; break;
+ case 'F': ur = false; break;
+ default:
+ fprintf(stderr, "unknown unitig_force_unique in '%s'\n", editLine);
+ break;
+ }
+ tigStore->setUnitigForceUnique(tid, ur);
} else if (strncmp(op, "contig_status", 13) == 0) {
ContigStatus st = tigStore->getContigStatus(tid);
@@ -158,7 +190,10 @@ dumpProperties(MultiAlignStore *tigStore,
fprintf(stdout, "unitigCoverageStat %f\n", ma->data.unitig_coverage_stat);
fprintf(stdout, "unitigMicrohetProb %f\n", ma->data.unitig_microhet_prob);
fprintf(stdout, "unitigStatus %c/%d\n", ma->data.unitig_status, ma->data.unitig_status);
- fprintf(stdout, "unitigFUR %c/%d\n", ma->data.unitig_unique_rept, ma->data.unitig_unique_rept);
+ fprintf(stdout, "unitigSuggestRepeat %c/%d\n", ma->data.unitig_suggest_repeat, ma->data.unitig_suggest_repeat);
+ fprintf(stdout, "unitigSuggestUnique %c/%d\n", ma->data.unitig_suggest_unique, ma->data.unitig_suggest_unique);
+ fprintf(stdout, "unitigForceRepeat %c/%d\n", ma->data.unitig_force_repeat, ma->data.unitig_force_repeat);
+ fprintf(stdout, "unitigForceUnique %c/%d\n", ma->data.unitig_force_unique, ma->data.unitig_force_unique);
fprintf(stdout, "contigStatus %c/%d\n", ma->data.contig_status, ma->data.contig_status);
#if GCCONTENT
@@ -231,19 +266,48 @@ dumpUnitigs(MultiAlignStore *tigStore,
void
dumpConsensus(MultiAlignStore *tigStore,
- int32 tigID,
- int32 tigIsUnitig,
- MultiAlignT *ma,
- bool withGaps) {
+ int32 tigID,
+ int32 tigIsUnitig,
+ MultiAlignT *ma,
+ bool withGaps,
+ uint32 minCoverage) {
if (ma->consensus == NULL)
return;
- char *cns = Getchar(ma->consensus, 0);
+ char *cns = Getchar(ma->consensus, 0);
if ((cns == NULL) || (cns[0] == 0))
return;
+ // If a minCoverage is specified, convert the low coverage bases to underscores, which will be
+ // filtered later.
+
+ if (minCoverage > 0) {
+ intervalList<int32> allL;
+
+ for (uint32 i=0; i<GetNumIntMultiPoss(ma->f_list); i++) {
+ IntMultiPos *imp = GetIntMultiPos(ma->f_list, i);
+
+ int32 bgn = MIN(imp->position.bgn, imp->position.end);
+ int32 end = MAX(imp->position.bgn, imp->position.end);
+
+ allL.add(bgn, end - bgn);
+ }
+
+ intervalList<int32> ID(allL);
+
+ for (uint32 ii=0; ii<ID.numberOfIntervals(); ii++) {
+ if (ID.depth(ii) >= minCoverage)
+ continue;
+
+ for (uint32 pp=ID.lo(ii); pp<ID.hi(ii); pp++)
+ cns[pp] = '_';
+ }
+ }
+
+ // Now filter out gaps in the consensus.
+
if (withGaps == false) {
char *o = cns;
char *n = cns;
@@ -257,21 +321,58 @@ dumpConsensus(MultiAlignStore *tigStore,
*o = 0;
}
- if (tigIsUnitig)
- fprintf(stdout, ">utg%d len="F_U64" reads="F_U32" status=%c microHet=%.2f covStat=%.2f\n%s\n",
- ma->maID, GetNumchars(ma->consensus) - 1, ma->data.num_frags,
- ma->data.unitig_status,
- ma->data.unitig_microhet_prob,
- ma->data.unitig_coverage_stat,
- cns);
- else
- fprintf(stdout, ">ctg%d len="F_U64" reads="F_U32" unitigs="F_U32" status=%c\n%s\n",
- ma->maID, GetNumchars(ma->consensus) - 1, ma->data.num_frags,
- ma->data.num_unitigs,
- ma->data.contig_status,
- cns);
-}
+ // If no min coverage, we can just dump the consensus and be done. Plus we output a few bits of
+ // useful info with the sequence.
+
+ if (minCoverage == 0) {
+ if (tigIsUnitig)
+ fprintf(stdout, ">utg%d len="F_U64" reads="F_U32" status=%c microHet=%.2f covStat=%.2f\n%s\n",
+ ma->maID, GetNumchars(ma->consensus) - 1, ma->data.num_frags,
+ ma->data.unitig_status,
+ ma->data.unitig_microhet_prob,
+ ma->data.unitig_coverage_stat,
+ cns);
+ else
+ fprintf(stdout, ">ctg%d len="F_U64" reads="F_U32" unitigs="F_U32" status=%c\n%s\n",
+ ma->maID, GetNumchars(ma->consensus) - 1, ma->data.num_frags,
+ ma->data.num_unitigs,
+ ma->data.contig_status,
+ cns);
+ return;
+ }
+
+ // Otherwise, we need to find subsequences in the consensus. The useful bits of info aren't
+ // valid anymore.
+
+ uint32 cnsLen = strlen(cns);
+ uint32 part = 0;
+
+ for (uint32 bgn=0; bgn<cnsLen; bgn++)
+ if (cns[bgn] == '_')
+ cns[bgn] = 0;
+
+ for (uint32 bgn=0; bgn<cnsLen; bgn++) {
+ while ((cns[bgn] == 0) && (bgn < cnsLen))
+ bgn++;
+ if (bgn >= cnsLen)
+ break;
+
+ uint32 end = bgn + 1;
+
+ while ((cns[end] != 0) && (end < cnsLen))
+ end++;
+
+ fprintf(stdout, ">%s%d.%u bgn=%u end=%u len=%u\n%s\n",
+ (tigIsUnitig) ? "utg" : "cns",
+ ma->maID, part, bgn, end, end-bgn,
+ cns + bgn);
+
+ bgn = end + 1;
+
+ part++;
+ }
+}
@@ -281,8 +382,11 @@ dumpCoverage(MultiAlignStore *tigStore,
int32 tigIsUnitig,
MultiAlignT *ma,
uint32 minCoverage,
- uint32 maxCoverage) {
- intervalList allL;
+ uint32 maxCoverage,
+ uint64 *coverageHistogram,
+ uint32 coverageHistogramLen,
+ char *outPrefix) {
+ intervalList<int32> allL;
uint32 maxPos = 0;
@@ -291,7 +395,7 @@ dumpCoverage(MultiAlignStore *tigStore,
int32 bgn = MIN(imp->position.bgn, imp->position.end);
int32 end = MAX(imp->position.bgn, imp->position.end);
- int32 len = end - bgn + 1;
+ int32 len = end - bgn;
if (maxPos < end)
maxPos = end;
@@ -299,29 +403,63 @@ dumpCoverage(MultiAlignStore *tigStore,
allL.add(bgn, len);
}
- intervalDepth ID(allL);
+ maxPos++; // Now the C-style maxPos.
+
+ intervalList<int32> ID(allL);
- intervalList minL;
- intervalList maxL;
+ intervalList<int32> minL;
+ intervalList<int32> maxL;
+ uint32 maxDepth = 0;
+ double aveDepth = 0;
+ double sdeDepth = 0;
+
+#if 0
for (uint32 ii=0; ii<ID.numberOfIntervals(); ii++) {
- if ((ID.de(ii) < minCoverage) && (ID.lo(ii) != 0) && (ID.hi(ii) != maxPos)) {
+ if ((ID.depth(ii) < minCoverage) && (ID.lo(ii) != 0) && (ID.hi(ii) != maxPos)) {
fprintf(stderr, "%s %d low coverage interval %ld %ld max %u coverage %u\n",
- (tigIsUnitig) ? "unitig" : "contig", tigID, ID.lo(ii), ID.hi(ii), maxPos, ID.de(ii));
+ (tigIsUnitig) ? "unitig" : "contig", tigID, ID.lo(ii), ID.hi(ii), maxPos, ID.depth(ii));
minL.add(ID.lo(ii), ID.hi(ii) - ID.lo(ii) + 1);
}
- if (maxCoverage <= ID.de(ii)) {
+ if (maxCoverage <= ID.depth(ii)) {
fprintf(stderr, "%s %d high coverage interval %ld %ld max %u coverage %u\n",
- (tigIsUnitig) ? "unitig" : "contig", tigID, ID.lo(ii), ID.hi(ii), maxPos, ID.de(ii));
+ (tigIsUnitig) ? "unitig" : "contig", tigID, ID.lo(ii), ID.hi(ii), maxPos, ID.depth(ii));
maxL.add(ID.lo(ii), ID.hi(ii) - ID.lo(ii) + 1);
}
}
+#endif
+
+ for (uint32 ii=0; ii<ID.numberOfIntervals(); ii++) {
+ if (ID.depth(ii) > maxDepth)
+ maxDepth = ID.depth(ii);
+
+ aveDepth += (ID.hi(ii) - ID.lo(ii) + 1) * ID.depth(ii);
+
+ if (ID.depth(ii) < coverageHistogramLen)
+ coverageHistogram[ID.depth(ii)] += ID.hi(ii) - ID.lo(ii) + 1;
+ else
+ fprintf(stderr, "deep coverage %d\n", ID.depth(ii));
+ }
+
+ aveDepth /= maxPos;
+
+ for (uint32 ii=0; ii<ID.numberOfIntervals(); ii++) {
+ sdeDepth += (ID.hi(ii) - ID.lo(ii) + 1) * (ID.depth(ii) - aveDepth) * (ID.depth(ii) - aveDepth);
+ }
+
+ sdeDepth = sqrt(sdeDepth / maxPos);
+
+ if (maxDepth > 1000)
+ fprintf(stderr, "DEEP unitig %u of length %u with maxDepth %u\n",
+ tigID, maxPos, maxDepth);
allL.merge();
minL.merge();
maxL.merge();
+
+#if 0
if ((minL.numberOfIntervals() > 0) && (maxL.numberOfIntervals() > 0))
fprintf(stderr, "%s %d has %u intervals, %u regions below %u coverage and %u regions at or above %u coverage\n",
(tigIsUnitig) ? "unitig" : "contig", tigID,
@@ -342,6 +480,44 @@ dumpCoverage(MultiAlignStore *tigStore,
fprintf(stderr, "%s %d has %u intervals\n",
(tigIsUnitig) ? "unitig" : "contig", tigID,
allL.numberOfIntervals());
+#endif
+
+ if (outPrefix) {
+ char outName[FILENAME_MAX];
+
+ sprintf(outName, "%s.%s%08u.depth", outPrefix, (tigIsUnitig) ? "utg" : "ctg", tigID);
+
+ FILE *outFile = fopen(outName, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1);
+
+ for (uint32 ii=0; ii<ID.numberOfIntervals(); ii++) {
+ fprintf(outFile, "%d\t%u\n", ID.lo(ii), ID.depth(ii));
+ fprintf(outFile, "%d\t%u\n", ID.hi(ii) - 1, ID.depth(ii));
+ }
+
+ fclose(outFile);
+
+ FILE *gnuPlot = popen("gnuplot > /dev/null 2>&1", "w");
+
+ if (gnuPlot) {
+ fprintf(gnuPlot, "set terminal 'png'\n");
+ fprintf(gnuPlot, "set output '%s.%s%08u.png'\n", outPrefix, (tigIsUnitig) ? "utg" : "ctg", tigID);
+ fprintf(gnuPlot, "set xlabel 'position'\n");
+ fprintf(gnuPlot, "set ylabel 'coverage'\n");
+ fprintf(gnuPlot, "set terminal 'png'\n");
+ fprintf(gnuPlot, "plot '%s.%s%08u.depth' using 1:2 with lines title '%s %u length %u', \\\n",
+ outPrefix,
+ (tigIsUnitig) ? "utg" : "ctg",
+ tigID,
+ (tigIsUnitig) ? "unitig" : "contig", tigID, maxPos);
+ fprintf(gnuPlot, " %f title 'mean %.2f +- %.2f', \\\n", aveDepth, aveDepth, sdeDepth);
+ fprintf(gnuPlot, " %f title '' lt 0 lc 2, \\\n", aveDepth - sdeDepth);
+ fprintf(gnuPlot, " %f title '' lt 0 lc 2\n", aveDepth + sdeDepth);
+
+ fclose(gnuPlot);
+ }
+ }
}
@@ -351,8 +527,8 @@ dumpThinOverlap(MultiAlignStore *tigStore,
int32 tigIsUnitig,
MultiAlignT *ma,
uint32 minOverlap) {
- intervalList allL;
- intervalList ovlL;
+ intervalList<int32> allL;
+ intervalList<int32> ovlL;
uint32 maxPos = 0;
@@ -382,12 +558,12 @@ dumpThinOverlap(MultiAlignStore *tigStore,
#if 1
for (uint32 i=0; i<ovlL.numberOfIntervals(); i++)
- fprintf(stderr, "%s %u IL "F_U64" "F_U64"\n",
+ fprintf(stderr, "%s %u IL %d %d\n",
(tigIsUnitig) ? "unitig" : "contig", tigID,
ovlL.lo(i), ovlL.hi(i));
#endif
- intervalList badL;
+ intervalList<int32> badL;
for (uint32 i=1; i<ovlL.numberOfIntervals(); i++) {
assert(ovlL.lo(i) < ovlL.hi(i-1));
@@ -398,7 +574,7 @@ dumpThinOverlap(MultiAlignStore *tigStore,
#if 1
for (uint32 i=0; i<badL.numberOfIntervals(); i++)
- fprintf(stderr, "%s %u BAD "F_U64" "F_U64"\n",
+ fprintf(stderr, "%s %u BAD %d %d\n",
(tigIsUnitig) ? "unitig" : "contig", tigID,
badL.lo(i), badL.hi(i));
#endif
@@ -661,22 +837,28 @@ main (int argc, char **argv) {
char *editName = NULL;
char *replaceName = NULL;
bool sameVersion = true;
+ bool append = false;
char *buildName = NULL;
uint32 minNreads = 0;
uint32 maxNreads = UINT32_MAX;
+ uint32 minCoverage = 0;
+
MultiAlignT *ma = NULL;
int showQV = 0;
int showDots = 1;
matePairAnalysis *mpa = NULL;
- char *mpaPrefix = NULL;
sizeAnalysis *siz = NULL;
- char *sizPrefix = NULL;
uint64 sizSize = 0;
+ uint64 *cov = NULL;
+ uint64 covMax = 0;
+
+ char *outPrefix = NULL;
+
argc = AS_configure(argc, argv);
int arg=1;
@@ -728,9 +910,13 @@ main (int argc, char **argv) {
else if (strcmp(argv[arg], "unitigs") == 0)
dumpFlags = DUMP_UNITIGS;
- else if (strcmp(argv[arg], "consensus") == 0)
+ else if (strcmp(argv[arg], "consensus") == 0) {
dumpFlags = DUMP_CONSENSUS;
+ if ((arg+1 < argc) && (isdigit(argv[arg+1][0])))
+ minCoverage = atoi(argv[++arg]);
+ }
+
else if (strcmp(argv[arg], "consensusgapped") == 0)
dumpFlags = DUMP_CONSENSUSGAPPED;
@@ -789,6 +975,9 @@ main (int argc, char **argv) {
} else if (strcmp(argv[arg], "-N") == 0) {
sameVersion = false;
+ } else if (strcmp(argv[arg], "-A") == 0) {
+ append = true;
+
} else if (strcmp(argv[arg], "-compress") == 0) {
opType = OPERATION_COMPRESS;
@@ -804,7 +993,7 @@ main (int argc, char **argv) {
sizSize = atoll(argv[arg]);
} else if (strcmp(argv[arg], "-o") == 0) {
- mpaPrefix = argv[++arg];
+ outPrefix = argv[++arg];
} else {
fprintf(stderr, "%s: Unknown option '%s'\n", argv[0], argv[arg]);
@@ -839,7 +1028,8 @@ main (int argc, char **argv) {
fprintf(stderr, " properties ...properties\n");
fprintf(stderr, " frags ...a list of fragments\n");
fprintf(stderr, " unitigs ...a list of unitigs\n");
- fprintf(stderr, " consensus ...the consensus sequence\n");
+ fprintf(stderr, " consensus [C] ...the consensus sequence\n");
+ fprintf(stderr, " if C supplied, only consensus with coverage >= C is output\n");
fprintf(stderr, " consensusgapped ...the consensus sequence, with gaps as indicated in the multialignment\n");
fprintf(stderr, " layout ...the layout\n");
fprintf(stderr, " multialign ...the full multialignment\n");
@@ -871,7 +1061,7 @@ main (int argc, char **argv) {
fprintf(stderr, " -w width Width of the page.\n");
fprintf(stderr, " -s spacing Spacing between reads on the same line.\n");
fprintf(stderr, "\n");
- fprintf(stderr, " For '-d matepair':\n");
+ fprintf(stderr, " For '-d matepair' and '-d coverage':\n");
fprintf(stderr, " -o prefix Output files will be written to 'prefix.*' in the current directory.\n");
fprintf(stderr, " (defaults to 'tigStore' (the -t option) if not set.)\n");
fprintf(stderr, "\n");
@@ -912,6 +1102,9 @@ main (int argc, char **argv) {
gkpStore = new gkStore(gkpName, FALSE, FALSE);
tigStore = new MultiAlignStore(tigName, tigVers, tigPartU, tigPartC, FALSE, FALSE, FALSE);
+ if (outPrefix == NULL)
+ outPrefix = tigName;
+
if ((opType == OPERATION_EDIT) && (editName != NULL)) {
delete tigStore;
@@ -938,7 +1131,7 @@ main (int argc, char **argv) {
if (sameVersion)
tigStore = new MultiAlignStore(tigName, tigVers, tigPartU, tigPartC, TRUE, true, false); // default
else
- tigStore = new MultiAlignStore(tigName, tigVers, tigPartU, tigPartC, TRUE, false, false);
+ tigStore = new MultiAlignStore(tigName, tigVers, tigPartU, tigPartC, TRUE, false, append);
MultiAlignT *ma = CreateEmptyMultiAlignT();
bool isUnitig = false;
@@ -976,10 +1169,13 @@ main (int argc, char **argv) {
if (opType == OPERATION_PROPERTIES) {
for (uint32 i=0; i<tigStore->numUnitigs(); i++) {
if (tigStore->isDeleted(i, TRUE) == false) {
- fprintf(stdout, "unitig_coverage_stat %8u %d\n", i, tigStore->getUnitigCoverageStat(i));
- fprintf(stdout, "unitig_microhet_prob %8u %f\n", i, tigStore->getUnitigMicroHetProb(i));
- fprintf(stdout, "unitig_status %8u %c\n", i, tigStore->getUnitigStatus(i));
- fprintf(stdout, "unitig_unique_rept %8u %c\n", i, tigStore->getUnitigFUR(i));
+ fprintf(stdout, "unitig_coverage_stat %8u %d\n", i, tigStore->getUnitigCoverageStat(i));
+ fprintf(stdout, "unitig_microhet_prob %8u %f\n", i, tigStore->getUnitigMicroHetProb(i));
+ fprintf(stdout, "unitig_status %8u %c\n", i, tigStore->getUnitigStatus(i));
+ fprintf(stdout, "unitig_suggest_repeat %8u %c\n", i, tigStore->getUnitigSuggestRepeat(i) ? 'T' : 'F');
+ fprintf(stdout, "unitig_suggest_unique %8u %c\n", i, tigStore->getUnitigSuggestUnique(i) ? 'T' : 'F');
+ fprintf(stdout, "unitig_force_repeat %8u %c\n", i, tigStore->getUnitigForceRepeat(i) ? 'T' : 'F');
+ fprintf(stdout, "unitig_force_unique %8u %c\n", i, tigStore->getUnitigForceUnique(i) ? 'T' : 'F');
}
}
@@ -1015,17 +1211,17 @@ main (int argc, char **argv) {
if (nTigs <= tigIDend)
tigIDend = nTigs - 1;
-
- if (dumpFlags == DUMP_MATEPAIR) {
+ if (dumpFlags == DUMP_MATEPAIR)
mpa = new matePairAnalysis(gkpName);
- if (mpaPrefix == NULL)
- mpaPrefix = tigName;
- }
- if (dumpFlags == DUMP_SIZES) {
+ if (dumpFlags == DUMP_SIZES)
siz = new sizeAnalysis(sizSize);
- if (sizPrefix == NULL)
- sizPrefix = tigName;
+
+ if (dumpFlags == DUMP_COVERAGE) {
+ covMax = 1048576;
+ cov = new uint64 [covMax];
+
+ memset(cov, 0, sizeof(uint64) * covMax);
}
for (uint32 ti=tigIDbgn; ti<=tigIDend; ti++) {
@@ -1050,10 +1246,10 @@ main (int argc, char **argv) {
dumpUnitigs(tigStore, ti, tigIsUnitig, ma);
if (dumpFlags == DUMP_CONSENSUS)
- dumpConsensus(tigStore, ti, tigIsUnitig, ma, false);
+ dumpConsensus(tigStore, ti, tigIsUnitig, ma, false, minCoverage);
if (dumpFlags == DUMP_CONSENSUSGAPPED)
- dumpConsensus(tigStore, ti, tigIsUnitig, ma, true);
+ dumpConsensus(tigStore, ti, tigIsUnitig, ma, true, minCoverage);
if (dumpFlags == DUMP_LAYOUT)
DumpMultiAlignForHuman(stdout, ma, tigIsUnitig);
@@ -1068,7 +1264,7 @@ main (int argc, char **argv) {
siz->evaluateTig(ma, tigIsUnitig);
if (dumpFlags == DUMP_COVERAGE)
- dumpCoverage(tigStore, ti, tigIsUnitig, ma, 2, UINT32_MAX);
+ dumpCoverage(tigStore, ti, tigIsUnitig, ma, 2, UINT32_MAX, cov, covMax, outPrefix);
if (dumpFlags == DUMP_THINOVERLAP)
dumpThinOverlap(tigStore, ti, tigIsUnitig, ma, sizSize);
@@ -1083,8 +1279,8 @@ main (int argc, char **argv) {
if (mpa) {
mpa->finalize();
mpa->printSummary(stdout);
- mpa->writeUpdate(mpaPrefix);
- mpa->drawPlots(mpaPrefix);
+ mpa->writeUpdate(outPrefix);
+ mpa->drawPlots(outPrefix);
delete mpa;
}
@@ -1094,6 +1290,41 @@ main (int argc, char **argv) {
delete siz;
}
+ if (cov) {
+ char N[FILENAME_MAX];
+ FILE *F;
+
+ sprintf(N, "%s.depthHistogram", outPrefix);
+
+ errno = 0;
+ F = fopen(N, "w");
+
+ uint32 hMax = covMax - 1;
+ while ((hMax > 0) && (cov[hMax] == 0))
+ hMax--;
+
+ for (uint32 i=0; i<=hMax; i++)
+ fprintf(F, F_U32"\t"F_U64"\n", i, cov[i]);
+
+ fclose(F);
+
+ delete [] cov;
+
+ FILE *gnuPlot = popen("gnuplot > /dev/null 2>&1", "w");
+
+ if (gnuPlot) {
+ fprintf(gnuPlot, "set terminal 'png'\n");
+ fprintf(gnuPlot, "set output '%s.depthHistogram.png'\n", outPrefix);
+ fprintf(gnuPlot, "set xlabel 'depth'\n");
+ fprintf(gnuPlot, "set ylabel 'number of bases'\n");
+ fprintf(gnuPlot, "set terminal 'png'\n");
+ fprintf(gnuPlot, "plot '%s.depthHistogram' using 1:2 with lines title '%s %s %u-%u depthHistogram', \\\n",
+ outPrefix, outPrefix, (tigIsUnitig) ? "unitigs" : "contigs", tigIDbgn, tigIDend);
+
+ fclose(gnuPlot);
+ }
+ }
+
delete gkpStore;
delete tigStore;
diff --git a/src/AS_CNS/utgcns.C b/src/AS_CNS/utgcns.C
index 9a016f0..6ce6271 100644
--- a/src/AS_CNS/utgcns.C
+++ b/src/AS_CNS/utgcns.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: utgcns.C 4465 2013-11-04 19:40:29Z brianwalenz $";
+const char *mainid = "$Id: utgcns.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include "AS_global.H"
#include "MultiAlign.H"
@@ -54,20 +54,36 @@ IntMultiPos_PositionCompare(IntMultiPos const &a, IntMultiPos const &b) {
// Create a new f_list for the ma that has no contained reads.
// The original f_list is returned.
//
+
+ class readLength {
+ public:
+ AS_IID idx;
+ int32 len;
+
+ bool operator<(const readLength &that) const {
+ return(len < that.len);
+ };
+ };
+
VA_TYPE(IntMultiPos) *
stashContains(MultiAlignT *ma,
- double minCov,
- double minDove) {
+ double maxCov) {
VA_TYPE(IntMultiPos) *fl = ma->f_list;
int32 nOrig = GetNumIntMultiPoss(fl);
int32 nDove = 0;
int32 nCont = 0;
+ int32 nSave = 0;
int64 nBase = 0;
int64 nBaseDove = 0;
int64 nBaseCont = 0;
+ int64 nBaseSave = 0;
+
+ if (ma->data.num_frags == 1)
+ return(NULL);
- int32 *isDove = new int32 [nOrig];
+ int32 *isDove = new int32 [nOrig];
+ readLength *posLen = new readLength [nOrig];
IntMultiPos *imp = GetIntMultiPos(fl, 0);
std::sort(imp, imp+nOrig, IntMultiPos_PositionCompare);
@@ -75,10 +91,12 @@ stashContains(MultiAlignT *ma,
int32 loEnd = MIN(imp->position.bgn, imp->position.end);
int32 hiEnd = MAX(imp->position.bgn, imp->position.end);
- isDove[0] = 1;
- nDove = 1;
- nBaseDove += hiEnd - loEnd;
- nBase += hiEnd - loEnd;
+ isDove[0] = 1;
+ nDove = 1;
+ posLen[0].idx = 0;
+ posLen[0].len = hiEnd - loEnd;
+ nBaseDove += posLen[0].len;
+ nBase += posLen[0].len;
for (uint32 fi=1; fi<nOrig; fi++) {
imp = GetIntMultiPos(fl, fi);
@@ -86,16 +104,18 @@ stashContains(MultiAlignT *ma,
int32 lo = MIN(imp->position.bgn, imp->position.end);
int32 hi = MAX(imp->position.bgn, imp->position.end);
- nBase += hi - lo;
+ posLen[fi].idx = fi;
+ posLen[fi].len = hi - lo;
+ nBase += posLen[fi].len;
if (hi <= hiEnd) {
isDove[fi] = 0;
nCont++;
- nBaseCont += hi - lo;
+ nBaseCont += posLen[fi].len;
} else {
isDove[fi] = 1;
nDove++;
- nBaseDove += hi - lo;
+ nBaseDove += posLen[fi].len;
}
hiEnd = MAX(hi, hiEnd);
@@ -110,10 +130,28 @@ stashContains(MultiAlignT *ma,
nCont, (double)nBaseCont / hiEnd, percCont,
nDove, (double)nBaseDove / hiEnd, percDove);
- if ((totlCov >= minCov) &&
- (percDove >= minDove)) {
- fprintf(stderr, " unitig %d removing "F_S32" contains; processing only "F_S32" reads\n",
- ma->maID, nOrig - nDove, nDove);
+ if ((totlCov >= maxCov) &&
+ (maxCov > 0)) {
+ std::sort(posLen, posLen + nOrig);
+
+ nBaseSave = 0.0;
+
+ for (uint32 ii=0; ((ii < nOrig) && ((double)(nBaseSave + nBaseDove) / hiEnd < maxCov)); ii++) {
+ if (isDove[posLen[ii].idx])
+ continue;
+
+ isDove[posLen[ii].idx] = 1;
+
+ nSave++;
+ nBaseSave += posLen[ii].len;
+ }
+
+ fprintf(stderr, " unitig %d removing "F_S32" (%.2fx) contained reads; processing only "F_S32" contained (%.2fx) and "F_S32" dovetail (%.2fx) reads\n",
+ ma->maID,
+ nOrig - nDove - nSave,
+ (double)(nBaseCont - nBaseSave) / hiEnd,
+ nSave, (double)nBaseSave / hiEnd,
+ nDove, (double)nBaseDove / hiEnd);
ma->f_list = CreateVA_IntMultiPos(0);
@@ -131,6 +169,7 @@ stashContains(MultiAlignT *ma,
}
delete [] isDove;
+ delete [] posLen;
return(fl);
}
@@ -139,7 +178,8 @@ stashContains(MultiAlignT *ma,
// Restores the f_list, and updates the position of non-contained reads.
//
void
-unstashContains(MultiAlignT *ma, VA_TYPE(IntMultiPos) *fl) {
+unstashContains(MultiAlignT *ma,
+ VA_TYPE(IntMultiPos) *fl) {
if (fl == NULL)
return;
@@ -234,9 +274,8 @@ main (int argc, char **argv) {
bool showResult = false;
- bool reduceCoverage = true;
- double reduceCoverageMinCov = 100.0; // 100x coverage
- double reduceCoverageMinDove = 5.0; // 5% bases in dovetail reads
+ double maxCov = 0.0;
+ uint32 maxLen = UINT32_MAX;
bool inplace = false;
bool loadall = false;
@@ -282,12 +321,11 @@ main (int argc, char **argv) {
} else if (strcmp(argv[arg], "-V") == 0) {
VERBOSE_MULTIALIGN_OUTPUT++;
- } else if (strcmp(argv[arg], "-noreduce") == 0) {
- reduceCoverage = false;
+ } else if (strcmp(argv[arg], "-maxcoverage") == 0) {
+ maxCov = atof(argv[++arg]);
- } else if (strcmp(argv[arg], "-reduce") == 0) {
- reduceCoverageMinCov = atof(argv[++arg]);
- reduceCoverageMinDove = atof(argv[++arg]);
+ } else if (strcmp(argv[arg], "-maxlength") == 0) {
+ maxLen = atof(argv[++arg]);
} else if (strcmp(argv[arg], "-inplace") == 0) {
inplace = true;
@@ -326,19 +364,16 @@ main (int argc, char **argv) {
fprintf(stderr, "\n");
fprintf(stderr, " -n Do not update the store after computing consensus.\n");
fprintf(stderr, "\n");
- fprintf(stderr, " -noreduce Use all reads for consensus generation. By default, unitigs with\n");
- fprintf(stderr, " average depth at least 100x and more than 5%% of the read bases in\n");
- fprintf(stderr, " non-contained reads will use only the non-contained reads for consensus.\n");
+ fprintf(stderr, " -maxcoverage c Use non-contained reads and the longest contained reads, up to\n");
+ fprintf(stderr, " C coverage, for consensus generation. The default is 0, and will\n");
+ fprintf(stderr, " use all reads.\n");
fprintf(stderr, "\n");
- fprintf(stderr, " -reduce c f Use only non-contained reads for consensus generation if the unitig\n");
- fprintf(stderr, " has more than 'c' coverage and more than 'f' percent of the read bases\n");
- fprintf(stderr, " are in non-contained reads. The default is 100x (c=100) coverage and\n");
- fprintf(stderr, " 5%% (f=5) non-contained bases.\n");
+ fprintf(stderr, " -maxlength l Do not compute consensus for unitigs longer than l bases.\n");
fprintf(stderr, "\n");
fprintf(stderr, " -inplace Write the updated unitig to the same version it was read from.\n");
fprintf(stderr, "\n");
fprintf(stderr, " -t S V P If 'partition' is '.', use an unpartitioned tigStore/gkpStore.\n");
- fprintf(stderr, " -loadall If not partitioned, load ALL reads into memory.\n");
+ fprintf(stderr, " -loadall Load ALL reads into memory. Ignores partition if it exists.\n");
if (gkpName == NULL)
fprintf(stderr, "ERROR: No gkpStore (-g) supplied.\n");
@@ -387,11 +422,9 @@ main (int argc, char **argv) {
tigStore = new MultiAlignStore(tigName, tigVers, tigPart, 0, FALSE, FALSE, FALSE);
- if (tigPart == 0) {
- if (loadall) {
- fprintf(stderr, "Loading all reads into memory.\n");
- gkpStore->gkStore_load(0, 0, GKFRAGMENT_QLT);
- }
+ if (loadall) {
+ fprintf(stderr, "Loading all reads into memory.\n");
+ gkpStore->gkStore_load(0, 0, GKFRAGMENT_QLT);
} else {
gkpStore->gkStore_loadPartition(tigPart);
}
@@ -419,7 +452,6 @@ main (int argc, char **argv) {
for (uint32 i=b; i<e; i++) {
MultiAlignT *ma = tigStore->loadMultiAlign(i, true);
- VA_TYPE(IntMultiPos) *fl = NULL;
if (ma == NULL) {
// Not in our partition, or deleted.
@@ -431,29 +463,33 @@ main (int argc, char **argv) {
if ((forceCompute == false) && (exists == true)) {
// Already finished unitig consensus.
if (ma->data.num_frags > 1)
- fprintf(stderr, "Working on unitig %d (%d unitigs and %d fragments) - already computed, skipped\n",
- ma->maID, ma->data.num_unitigs, ma->data.num_frags);
+ fprintf(stderr, "Working on unitig %d of length %d (%d unitigs %d fragments) - already computed, skipped\n",
+ ma->maID, GetMultiAlignLength(ma), ma->data.num_unitigs, ma->data.num_frags);
numSkipped++;
continue;
}
+ if (GetMultiAlignLength(ma) > maxLen) {
+ fprintf(stderr, "SKIP unitig %d of length %d (%d unitigs %d fragments) - too long, skipped\n",
+ ma->maID, GetMultiAlignLength(ma), ma->data.num_unitigs, ma->data.num_frags);
+ continue;
+ }
+
if (ma->data.num_frags > 1)
- fprintf(stderr, "Working on unitig %d (%d unitigs and %d fragments)%s\n",
- ma->maID, ma->data.num_unitigs, ma->data.num_frags,
+ fprintf(stderr, "Working on unitig %d of length %d (%d unitigs %d fragments)%s\n",
+ ma->maID, GetMultiAlignLength(ma), ma->data.num_unitigs, ma->data.num_frags,
(exists) ? " - already computed, recomputing" : "");
// Build a new ma if we're ignoring contains. We'll need to put back the reads we remove
// before we add it to the store.
- if ((reduceCoverage) && ((ma->data.num_frags > 1)))
- fl = stashContains(ma, reduceCoverageMinCov, reduceCoverageMinDove);
+ VA_TYPE(IntMultiPos) *fl = stashContains(ma, maxCov);
if (MultiAlignUnitig(ma, gkpStore, &options, NULL)) {
if (showResult)
PrintMultiAlignT(stdout, ma, gkpStore, false, false, AS_READ_CLEAR_LATEST);
- if (reduceCoverage)
- unstashContains(ma, fl);
+ unstashContains(ma, fl);
if (doUpdate) {
tigStore->insertMultiAlign(ma, true, true);
diff --git a/src/AS_ENV/._Makefile b/src/AS_ENV/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._Makefile differ
diff --git a/src/AS_ENV/._chimChe.C b/src/AS_ENV/._chimChe.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._chimChe.C differ
diff --git a/src/AS_ENV/._greedyFragmentTiling.C b/src/AS_ENV/._greedyFragmentTiling.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._greedyFragmentTiling.C differ
diff --git a/src/AS_ENV/._greedy_layout_to_IUM.pl b/src/AS_ENV/._greedy_layout_to_IUM.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._greedy_layout_to_IUM.pl differ
diff --git a/src/AS_ENV/._kmer_utils.H b/src/AS_ENV/._kmer_utils.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._kmer_utils.H differ
diff --git a/src/AS_ENV/._metagenomics_ovl_analyses.C b/src/AS_ENV/._metagenomics_ovl_analyses.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._metagenomics_ovl_analyses.C differ
diff --git a/src/AS_ENV/._run_greedy.csh b/src/AS_ENV/._run_greedy.csh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._run_greedy.csh differ
diff --git a/src/AS_ENV/._utg2fasta.pl b/src/AS_ENV/._utg2fasta.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_ENV/._utg2fasta.pl differ
diff --git a/src/AS_GKP/._AS_GKP_bench.C b/src/AS_GKP/._AS_GKP_bench.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_bench.C differ
diff --git a/src/AS_GKP/._AS_GKP_buildPartition.C b/src/AS_GKP/._AS_GKP_buildPartition.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_buildPartition.C differ
diff --git a/src/AS_GKP/._AS_GKP_checkFrag.C b/src/AS_GKP/._AS_GKP_checkFrag.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_checkFrag.C differ
diff --git a/src/AS_GKP/._AS_GKP_checkLibrary.C b/src/AS_GKP/._AS_GKP_checkLibrary.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_checkLibrary.C differ
diff --git a/src/AS_GKP/._AS_GKP_checkLink.C b/src/AS_GKP/._AS_GKP_checkLink.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_checkLink.C differ
diff --git a/src/AS_GKP/._AS_GKP_checkPlace.C b/src/AS_GKP/._AS_GKP_checkPlace.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_checkPlace.C differ
diff --git a/src/AS_GKP/._AS_GKP_dump.C b/src/AS_GKP/._AS_GKP_dump.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_dump.C differ
diff --git a/src/AS_GKP/._AS_GKP_edit.C b/src/AS_GKP/._AS_GKP_edit.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_edit.C differ
diff --git a/src/AS_GKP/._AS_GKP_errors.C b/src/AS_GKP/._AS_GKP_errors.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_errors.C differ
diff --git a/src/AS_GKP/._AS_GKP_illumina.C b/src/AS_GKP/._AS_GKP_illumina.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_illumina.C differ
diff --git a/src/AS_GKP/._AS_GKP_include.H b/src/AS_GKP/._AS_GKP_include.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_include.H differ
diff --git a/src/AS_GKP/._AS_GKP_main.C b/src/AS_GKP/._AS_GKP_main.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._AS_GKP_main.C differ
diff --git a/src/AS_GKP/._Makefile b/src/AS_GKP/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._Makefile differ
diff --git a/src/AS_GKP/._build-ordered-reads.sh b/src/AS_GKP/._build-ordered-reads.sh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._build-ordered-reads.sh differ
diff --git a/src/AS_GKP/._clearRangeHistogram.C b/src/AS_GKP/._clearRangeHistogram.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._clearRangeHistogram.C differ
diff --git a/src/AS_GKP/._fastqAnalyze.C b/src/AS_GKP/._fastqAnalyze.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqAnalyze.C differ
diff --git a/src/AS_GKP/._fastqSample.C b/src/AS_GKP/._fastqSample.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqSample.C differ
diff --git a/src/AS_GKP/._fastqSimulate-checkCoverage.pl b/src/AS_GKP/._fastqSimulate-checkCoverage.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqSimulate-checkCoverage.pl differ
diff --git a/src/AS_GKP/._fastqSimulate-perfectSep.pl b/src/AS_GKP/._fastqSimulate-perfectSep.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqSimulate-perfectSep.pl differ
diff --git a/src/AS_GKP/._fastqSimulate-sort.C b/src/AS_GKP/._fastqSimulate-sort.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqSimulate-sort.C differ
diff --git a/src/AS_GKP/._fastqSimulate.C b/src/AS_GKP/._fastqSimulate.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqSimulate.C differ
diff --git a/src/AS_GKP/._fastqToCA.C b/src/AS_GKP/._fastqToCA.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._fastqToCA.C differ
diff --git a/src/AS_GKP/._frg-to-fastq.pl b/src/AS_GKP/._frg-to-fastq.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._frg-to-fastq.pl differ
diff --git a/src/AS_GKP/._gkpStoreCreate.C b/src/AS_GKP/._gkpStoreCreate.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._gkpStoreCreate.C differ
diff --git a/src/AS_GKP/._gkpStoreDumpFASTQ.C b/src/AS_GKP/._gkpStoreDumpFASTQ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._gkpStoreDumpFASTQ.C differ
diff --git a/src/AS_GKP/._sffToCA-evaluateLinker.pl b/src/AS_GKP/._sffToCA-evaluateLinker.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._sffToCA-evaluateLinker.pl differ
diff --git a/src/AS_GKP/._sffToCA.C b/src/AS_GKP/._sffToCA.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._sffToCA.C differ
diff --git a/src/AS_GKP/._upgrade-v6-to-v7.C b/src/AS_GKP/._upgrade-v6-to-v7.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._upgrade-v6-to-v7.C differ
diff --git a/src/AS_GKP/._upgrade-v7-to-v8.C b/src/AS_GKP/._upgrade-v7-to-v8.C
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._upgrade-v7-to-v8.C differ
diff --git a/src/AS_GKP/._upgrade-v8-to-v9.C b/src/AS_GKP/._upgrade-v8-to-v9.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._upgrade-v8-to-v9.C differ
diff --git a/src/AS_GKP/._upgrade-v9-to-v10.C b/src/AS_GKP/._upgrade-v9-to-v10.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_GKP/._upgrade-v9-to-v10.C differ
diff --git a/src/AS_GKP/AS_GKP_dump.C b/src/AS_GKP/AS_GKP_dump.C
index 5309bf2..6290aa4 100644
--- a/src/AS_GKP/AS_GKP_dump.C
+++ b/src/AS_GKP/AS_GKP_dump.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char const *rcsid = "$Id: AS_GKP_dump.C 4415 2013-09-17 23:32:16Z brianwalenz $";
+static char const *rcsid = "$Id: AS_GKP_dump.C 4595 2015-02-03 02:52:44Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -30,6 +30,96 @@ static char const *rcsid = "$Id: AS_GKP_dump.C 4415 2013-09-17 23:32:16Z brianwa
#include "AS_UTL_reverseComplement.H"
#include "AS_PER_gkpStore.H"
+#include <vector>
+using namespace std;
+
+
+// Container for all files of a specific type (fasta, qual, fastq, etc).
+// Ask for the file for libIID, type.
+// Special case for 'monolithic' where libIID is ignored.
+
+class dumpFileFile {
+public:
+ dumpFileFile() {
+ memset(_N, 0, sizeof(char) * 5 * FILENAME_MAX);
+ memset(_F, 0, sizeof(FILE *) * 5);
+ };
+
+ char _N[5][FILENAME_MAX];
+ FILE *_F[5];
+};
+
+
+class dumpFile {
+public:
+ dumpFile(gkStore *gkp, char *prefix, char *extension) {
+ char libName[FILENAME_MAX];
+
+ sprintf(_mono._N[0], "%s.%s", prefix, extension);
+ sprintf(_mono._N[1], "%s.1.%s", prefix, extension);
+ sprintf(_mono._N[2], "%s.2.%s", prefix, extension);
+ sprintf(_mono._N[3], "%s.paired.%s", prefix, extension);
+ sprintf(_mono._N[4], "%s.unmated.%s", prefix, extension);
+
+ _lib.resize(gkp->gkStore_getNumLibraries() + 1);
+
+ for (uint32 ll=0; ll<=gkp->gkStore_getNumLibraries(); ll++) {
+ strcpy(libName, gkp->gkStore_getLibrary(ll)->libraryName);
+
+ for (uint32 ii=0; libName[ii]; ii++)
+ if (libName[ii] == '/')
+ libName[ii] = '_';
+
+ sprintf(_lib[ll]._N[0], "%s.%03u.%s.%s", prefix, ll, libName, extension);
+ sprintf(_lib[ll]._N[1], "%s.%03u.%s.1.%s", prefix, ll, libName, extension);
+ sprintf(_lib[ll]._N[2], "%s.%03u.%s.2.%s", prefix, ll, libName, extension);
+ sprintf(_lib[ll]._N[3], "%s.%03u.%s.paired.%s", prefix, ll, libName, extension);
+ sprintf(_lib[ll]._N[4], "%s.%03u.%s.unmated.%s", prefix, ll, libName, extension);
+ }
+ };
+
+ ~dumpFile() {
+ for (uint32 ii=0; ii<5; ii++)
+ if (_mono._F[ii]) fclose(_mono._F[ii]);
+
+ for (uint32 ll=0; ll<_lib.size(); ll++)
+ for (uint32 ii=0; ii<5; ii++)
+ if (_lib[ll]._F[ii]) fclose(_lib[ll]._F[ii]);
+ };
+
+ FILE *getFile(bool withLibrary, uint32 libIID, char type) {
+ uint32 tt = 0;
+
+ switch (type) {
+ case 'f': tt=0; break;
+ case '1': tt=1; break;
+ case '2': tt=2; break;
+ case 'p': tt=3; break;
+ case 'u': tt=4; break;
+ default:
+ assert(0);
+ break;
+ }
+
+ dumpFileFile *FF = (withLibrary == false) ? &_mono : &_lib[libIID];
+
+ if (FF->_F[tt] == NULL) {
+ errno = 0;
+ FF->_F[tt] = fopen(FF->_N[tt], "w");
+
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", FF->_N[tt], strerror(errno)), exit(1);
+ }
+
+ return(FF->_F[tt]);
+ }
+
+public:
+ dumpFileFile _mono;
+ vector<dumpFileFile> _lib;
+};
+
+
void
dumpGateKeeperInfo(char *gkpStoreName,
@@ -349,6 +439,7 @@ adjustBeginEndAddMates(gkStore *gkp,
void
dumpGateKeeperAsFasta(char *gkpStoreName,
char *prefix,
+ int withLibName,
AS_IID bgnIID,
AS_IID endIID,
char *iidToDump,
@@ -364,22 +455,9 @@ dumpGateKeeperAsFasta(char *gkpStoreName,
adjustBeginEndAddMates(gkp, bgnIID, endIID, libToDump, iidToDump, frgUID, doNotFixMates, dumpAllReads, withoutUIDs);
- char fname[FILENAME_MAX]; sprintf(fname, "%s.fasta", prefix);
- char qname[FILENAME_MAX]; sprintf(qname, "%s.fasta.qv", prefix);
- char Qname[FILENAME_MAX]; sprintf(Qname, "%s.fasta.qual", prefix);
-
- errno = 0;
- FILE *f = fopen(fname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", fname, strerror(errno)), exit(1);
-
- FILE *q = fopen(qname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", qname, strerror(errno)), exit(1);
-
- FILE *Q = fopen(Qname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", Qname, strerror(errno)), exit(1);
+ dumpFile *fasta = new dumpFile(gkp, prefix, "fasta");
+ dumpFile *qvals = new dumpFile(gkp, prefix, "fasta.qv");
+ dumpFile *quals = new dumpFile(gkp, prefix, "fasta.qual");
// Dump fragments -- as soon as both reads in a mate are defined,
// we dump the mate relationship.
@@ -463,7 +541,7 @@ dumpGateKeeperAsFasta(char *gkpStoreName,
seq[i] = tolower(seq[i]);
}
- AS_UTL_writeFastA(f, seq, len, 0,
+ AS_UTL_writeFastA(fasta->getFile(withLibName, libIID, 'f'), seq, len, 0,
">%s,"F_IID" mate=%s,"F_IID" lib=%s,"F_IID" clr=%s,%d,%d deleted=%d\n",
AS_UID_toString(fUID), id1,
AS_UID_toString(mUID), id2,
@@ -471,7 +549,7 @@ dumpGateKeeperAsFasta(char *gkpStoreName,
AS_READ_CLEAR_NAMES[dumpClear], lclr, rclr,
fr.gkFragment_getIsDeleted());
- AS_UTL_writeFastA(q, qlt, len, 0,
+ AS_UTL_writeFastA(qvals->getFile(withLibName, libIID, 'f'), qlt, len, 0,
">%s,"F_IID" mate=%s,"F_IID" lib=%s,"F_IID" clr=%s,%d,%d deleted=%d\n",
AS_UID_toString(fUID), id1,
AS_UID_toString(mUID), id2,
@@ -479,7 +557,7 @@ dumpGateKeeperAsFasta(char *gkpStoreName,
AS_READ_CLEAR_NAMES[dumpClear], lclr, rclr,
fr.gkFragment_getIsDeleted());
- AS_UTL_writeQVFastA(Q, qlt, len, 0,
+ AS_UTL_writeQVFastA(quals->getFile(withLibName, libIID, 'f'), qlt, len, 0,
">%s,"F_IID" mate=%s,"F_IID" lib=%s,"F_IID" clr=%s,%d,%d deleted=%d\n",
AS_UID_toString(fUID), id1,
AS_UID_toString(mUID), id2,
@@ -488,6 +566,14 @@ dumpGateKeeperAsFasta(char *gkpStoreName,
fr.gkFragment_getIsDeleted());
}
+ safe_free(iidToDump);
+ safe_free(libToDump);
+ safe_free(frgUID);
+
+ delete fasta;
+ delete qvals;
+ delete quals;
+
delete fs;
delete gkp;
}
@@ -779,6 +865,7 @@ dumpGateKeeperAsFRG(char *gkpStoreName,
void
dumpGateKeeperAsNewbler(char *gkpStoreName,
char *prefix,
+ int withLibName,
AS_IID bgnIID,
AS_IID endIID,
char *iidToDump,
@@ -794,17 +881,8 @@ dumpGateKeeperAsNewbler(char *gkpStoreName,
adjustBeginEndAddMates(gkp, bgnIID, endIID, libToDump, iidToDump, frgUID, doNotFixMates, dumpAllReads, withoutUIDs);
- char fname[FILENAME_MAX]; sprintf(fname, "%s.fna", prefix);
- char qname[FILENAME_MAX]; sprintf(qname, "%s.fna.qual", prefix);
-
- errno = 0;
- FILE *f = fopen(fname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", fname, strerror(errno)), exit(1);
-
- FILE *q = fopen(qname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", qname, strerror(errno)), exit(1);
+ dumpFile *fna = new dumpFile(gkp, prefix, "fna");
+ dumpFile *qna = new dumpFile(gkp, prefix, "fna.qual");
// Dump fragments -- as soon as both reads in a mate are defined,
// we dump the mate relationship.
@@ -856,17 +934,24 @@ dumpGateKeeperAsNewbler(char *gkpStoreName,
lclr,
rclr);
- AS_UTL_writeFastA(f,
+ AS_UTL_writeFastA(fna->getFile(withLibName, libIID, 'f'),
fr.gkFragment_getSequence(),
fr.gkFragment_getSequenceLength(), 0,
dl, NULL);
- AS_UTL_writeQVFastA(q,
+ AS_UTL_writeQVFastA(qna->getFile(withLibName, libIID, 'f'),
fr.gkFragment_getQuality(),
fr.gkFragment_getSequenceLength(), 0,
dl, NULL);
}
+ safe_free(iidToDump);
+ safe_free(libToDump);
+ safe_free(frgUID);
+
+ delete fna;
+ delete qna;
+
delete fs;
delete gkp;
}
@@ -876,6 +961,7 @@ dumpGateKeeperAsNewbler(char *gkpStoreName,
void
dumpGateKeeperAsFastQ(char *gkpStoreName,
char *prefix,
+ int withLibName,
AS_IID bgnIID,
AS_IID endIID,
char *iidToDump,
@@ -891,29 +977,7 @@ dumpGateKeeperAsFastQ(char *gkpStoreName,
adjustBeginEndAddMates(gkp, bgnIID, endIID, libToDump, iidToDump, frgUID, doNotFixMates, dumpAllReads, withoutUIDs);
- char aname[FILENAME_MAX]; sprintf(aname, "%s.1.fastq", prefix);
- char bname[FILENAME_MAX]; sprintf(bname, "%s.2.fastq", prefix);
- char pname[FILENAME_MAX]; sprintf(pname, "%s.paired.fastq", prefix);
- char uname[FILENAME_MAX]; sprintf(uname, "%s.unmated.fastq", prefix);
-
- errno = 0;
- FILE *a = fopen(aname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", aname, strerror(errno)), exit(1);
-
- errno = 0;
- FILE *b = fopen(bname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", bname, strerror(errno)), exit(1);
-
- errno = 0;
- FILE *p = fopen(pname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", pname, strerror(errno)), exit(1);
-
- FILE *u = fopen(uname, "w");
- if (errno)
- fprintf(stderr, "Failed to open output file '%s': %s\n", uname, strerror(errno)), exit(1);
+ dumpFile *fastq = new dumpFile(gkp, prefix, "fastq");
// Dump fragments -- as soon as both reads in a mate are defined,
// we dump the mate relationship.
@@ -960,7 +1024,7 @@ dumpGateKeeperAsFastQ(char *gkpStoreName,
if ((id2 != 0) && (id2 < id1))
// Mated, and the mate is the first frag. We've already reported this one.
- continue;
+ continue;
char *seq = fr.gkFragment_getSequence() + ((dumpAllBases == false) ? fr.gkFragment_getClearRegionBegin(dumpClear) : 0);
char *qlt = fr.gkFragment_getQuality() + ((dumpAllBases == false) ? fr.gkFragment_getClearRegionBegin(dumpClear) : 0);
@@ -1046,7 +1110,7 @@ dumpGateKeeperAsFastQ(char *gkpStoreName,
if (id2 == 0) {
// Unmated read, dump to the unmated reads file.
//
- AS_UTL_writeFastQ(u, seq, len, qlt, len,
+ AS_UTL_writeFastQ(fastq->getFile(withLibName, libIID, 'u'), seq, len, qlt, len,
"@%s,%u clr="F_U32","F_U32" clv="F_U32","F_U32" max="F_U32","F_U32" tnt="F_U32","F_U32" rnd=%c\n",
AS_UID_toString(fr.gkFragment_getReadUID()), fr.gkFragment_getReadIID(),
clrBgn, clrEnd, vecBgn, vecEnd, maxBgn, maxEnd, tntBgn, tntEnd,
@@ -1060,13 +1124,13 @@ dumpGateKeeperAsFastQ(char *gkpStoreName,
// Write the first fragment (twice).
- AS_UTL_writeFastQ(a, seq, len, qlt, len,
+ AS_UTL_writeFastQ(fastq->getFile(withLibName, libIID, '1'), seq, len, qlt, len,
"@%s,%u clr="F_U32","F_U32" clv="F_U32","F_U32" max="F_U32","F_U32" tnt="F_U32","F_U32" rnd=%c\n",
AS_UID_toString(fr.gkFragment_getReadUID()), fr.gkFragment_getReadIID(),
clrBgn, clrEnd, vecBgn, vecEnd, maxBgn, maxEnd, tntBgn, tntEnd,
fr.gkFragment_getIsNonRandom() ? 'f' : 't');
- AS_UTL_writeFastQ(p, seq, len, qlt, len,
+ AS_UTL_writeFastQ(fastq->getFile(withLibName, libIID, 'p'), seq, len, qlt, len,
"@%s,%u clr="F_U32","F_U32" clv="F_U32","F_U32" max="F_U32","F_U32" tnt="F_U32","F_U32" rnd=%c\n",
AS_UID_toString(fr.gkFragment_getReadUID()), fr.gkFragment_getReadIID(),
clrBgn, clrEnd, vecBgn, vecEnd, maxBgn, maxEnd, tntBgn, tntEnd,
@@ -1160,13 +1224,13 @@ dumpGateKeeperAsFastQ(char *gkpStoreName,
}
// Write the second fragment (twice).
- AS_UTL_writeFastQ(b, seq, len, qlt, len,
+ AS_UTL_writeFastQ(fastq->getFile(withLibName, libIID, '2'), seq, len, qlt, len,
"@%s,%u clr="F_U32","F_U32" clv="F_U32","F_U32" max="F_U32","F_U32" tnt="F_U32","F_U32" rnd=%c\n",
AS_UID_toString(fr.gkFragment_getReadUID()), fr.gkFragment_getReadIID(),
clrBgn, clrEnd, vecBgn, vecEnd, maxBgn, maxEnd, tntBgn, tntEnd,
fr.gkFragment_getIsNonRandom() ? 'f' : 't');
- AS_UTL_writeFastQ(p, seq, len, qlt, len,
+ AS_UTL_writeFastQ(fastq->getFile(withLibName, libIID, 'p'), seq, len, qlt, len,
"@%s,%u clr="F_U32","F_U32" clv="F_U32","F_U32" max="F_U32","F_U32" tnt="F_U32","F_U32" rnd=%c\n",
AS_UID_toString(fr.gkFragment_getReadUID()), fr.gkFragment_getReadIID(),
clrBgn, clrEnd, vecBgn, vecEnd, maxBgn, maxEnd, tntBgn, tntEnd,
@@ -1178,6 +1242,12 @@ dumpGateKeeperAsFastQ(char *gkpStoreName,
iidToDump[id2] = 0;
}
+ safe_free(iidToDump);
+ safe_free(libToDump);
+ safe_free(frgUID);
+
+ delete fastq;
+
delete fs;
delete gkp;
}
diff --git a/src/AS_GKP/AS_GKP_illumina.C b/src/AS_GKP/AS_GKP_illumina.C
index b217ca3..c4cb2aa 100644
--- a/src/AS_GKP/AS_GKP_illumina.C
+++ b/src/AS_GKP/AS_GKP_illumina.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char const *rcsid = "$Id: AS_GKP_illumina.C 4426 2013-09-23 02:59:41Z brianwalenz $";
+static char const *rcsid = "$Id: AS_GKP_illumina.C 4641 2015-04-21 00:13:49Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -115,6 +115,7 @@ processSeq(char *N,
// Complicated, but fast, parsing of the 'snam' to find clear ranges.
char *tok = strtok(fr->snam, " \t");
+ char *tol = tok;
uint32 clrL=0, clrR=slen; // Defined range, whole read
uint32 clvL=1, clvR=0; // Undefined range
@@ -125,43 +126,62 @@ processSeq(char *N,
while (tok != NULL) {
if (((tok[0] == 'c') || (tok[0] == 'C')) &&
((tok[1] == 'l') || (tok[1] == 'L')) &&
- ((tok[2] == 'r') || (tok[2] == 'R'))) {
- clrL = atoi(tok + 4);
+ ((tok[2] == 'r') || (tok[2] == 'R')) &&
+ ((tok[3] == '='))) {
while ((*tok) && (*tok != ','))
tok++;
- clrR = atoi(tok + 1);
+ if ((*tok) && (*(tok+1))) {
+ clrL = atoi(tol + 4);
+ clrR = atoi(tok + 1);
+ //fprintf(stderr, "%s -- clr %d,%d\n", fr->snam, clrL, clrR);
+ }
}
if (((tok[0] == 'c') || (tok[0] == 'C')) &&
((tok[1] == 'l') || (tok[1] == 'L')) &&
- ((tok[2] == 'v') || (tok[2] == 'V'))) {
- clvL = atoi(tok + 4);
+ ((tok[2] == 'v') || (tok[2] == 'V')) &&
+ ((tok[3] == '='))) {
while ((*tok) && (*tok != ','))
tok++;
- clvR = atoi(tok + 1);
+ if ((*tok) && (*(tok+1))) {
+ clvL = atoi(tol + 4);
+ clvR = atoi(tok + 1);
+ //fprintf(stderr, "%s -- clv %d,%d\n", fr->snam, clvL, clvR);
+ }
}
if (((tok[0] == 'm') || (tok[0] == 'M')) &&
((tok[1] == 'a') || (tok[1] == 'A')) &&
- ((tok[2] == 'x') || (tok[2] == 'X'))) {
- clmL = atoi(tok + 4);
+ ((tok[2] == 'x') || (tok[2] == 'X')) &&
+ ((tok[3] == '='))) {
while ((*tok) && (*tok != ','))
tok++;
- clmR = atoi(tok + 1);
+ if ((*tok) && (*(tok+1))) {
+ clmL = atoi(tol + 4);
+ clmR = atoi(tok + 1);
+ //fprintf(stderr, "%s -- clm %d,%d\n", fr->snam, clmL, clmR);
+ }
}
if (((tok[0] == 't') || (tok[0] == 'T')) &&
((tok[1] == 'n') || (tok[1] == 'N')) &&
- ((tok[2] == 't') || (tok[2] == 'T'))) {
- tntL = atoi(tok + 4);
+ ((tok[2] == 't') || (tok[2] == 'T')) &&
+ ((tok[3] == '='))) {
while ((*tok) && (*tok != ','))
tok++;
- tntR = atoi(tok + 1);
+ if ((*tok) && (*(tok+1))) {
+ tntL = atoi(tol + 4);
+ tntR = atoi(tok + 1);
+ //fprintf(stderr, "%s -- tnt %d,%d\n", fr->snam, tntL, tntR);
+ }
}
if (((tok[0] == 'r') || (tok[0] == 'R')) &&
((tok[1] == 'n') || (tok[1] == 'N')) &&
- ((tok[2] == 'd') || (tok[2] == 'D'))) {
+ ((tok[2] == 'd') || (tok[2] == 'D')) &&
+ ((tok[3] == '='))) {
rnd = tok[4];
+ //fprintf(stderr, "%s -- rnd %d\n", fr->snam, rnd);
}
tok = strtok(NULL, " \t");
+ tol = tok;
}
if (clrR > slen) clrR = slen; if (clrL > clrR) clrL = clrR;
@@ -431,6 +451,11 @@ readSeq(FILE *F,
fr->qnam[0] = 0;
fr->qstr[0] = 0;
+ fr->snam[NAME_MAX_LEN - 2] = 0;
+ fr->sstr[BASE_MAX_LEN - 2] = 0;
+ fr->qnam[NAME_MAX_LEN - 2] = 0;
+ fr->qstr[BASE_MAX_LEN - 2] = 0;
+
fgets(fr->snam, NAME_MAX_LEN, F);
fgets(fr->sstr, BASE_MAX_LEN, F);
fgets(fr->qnam, NAME_MAX_LEN, F);
@@ -441,6 +466,18 @@ readSeq(FILE *F,
chomp(fr->qnam);
chomp(fr->qstr);
+ if (fr->snam[NAME_MAX_LEN - 2] != 0)
+ fprintf(stderr, "FASTQ sequence name line too long in read '%s'\n", fr->snam), exit(1);
+
+ if (fr->sstr[BASE_MAX_LEN - 2] != 0)
+ fprintf(stderr, "FASTQ sequence line too long in read '%s'\n", fr->sstr), exit(1);
+
+ if (fr->qnam[NAME_MAX_LEN - 2] != 0)
+ fprintf(stderr, "FASTQ quality name line too long in read '%s'\n", fr->qnam), exit(1);
+
+ if (fr->qstr[BASE_MAX_LEN - 2] != 0)
+ fprintf(stderr, "FASTQ quality line too long in read '%s'\n", fr->qstr), exit(1);
+
if (feof(F))
return(0);
@@ -534,7 +571,34 @@ loadFastQReads(char *lname,
} else {
// Both deleted, do nothing.
}
- }
+
+#if 0
+ // Add a bunch of deleted fragments to push us close to 4 billion reads.
+ // guido
+ gkFragment jfr;
+
+ jfr.gkFragment_enableGatekeeperMode(gkpStore);
+
+ jfr.gkFragment_setType(GKFRAGMENT_PACKED);
+ jfr.gkFragment_setIsDeleted(1);
+
+ jfr.gkFragment_setMateIID(0);
+ jfr.gkFragment_setLibraryIID(0);
+
+ jfr.gkFragment_getSequence()[0] = 0;
+ jfr.gkFragment_getQuality() [0] = 0;
+
+ for (uint32 i=0; i<1725; i++) {
+ uint64 readUID = (99 * 10 + 1) * 10000000000LLU + gkpStore->gkStore_getNumFragments() + 1;
+
+ jfr.gkFragment_setReadUID(AS_UID_fromInteger(readUID));
+
+ gkpStore->gkStore_addFragment(&jfr);
+ }
+
+ fprintf(stderr, "Added junk reads, now at "F_U32"\n", gkpStore->gkStore_getNumFragments());
+#endif
+}
delete lfrg;
delete rfrg;
@@ -587,7 +651,7 @@ loadFastQReads(char *uname,
fprintf(fastqUIDmap, F_U64"\t"F_U32"\t%s\n",
uUID, nfrg + 1, ufrg->snam+1);
-
+
} else {
// Junk read, do nothing.
}
diff --git a/src/AS_GKP/AS_GKP_include.H b/src/AS_GKP/AS_GKP_include.H
index db9e904..fe5eb49 100644
--- a/src/AS_GKP/AS_GKP_include.H
+++ b/src/AS_GKP/AS_GKP_include.H
@@ -19,12 +19,12 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-/* $Id: AS_GKP_include.H 4371 2013-08-01 17:19:47Z brianwalenz $ */
+/* $Id: AS_GKP_include.H 4499 2014-01-29 16:28:07Z brianwalenz $ */
#ifndef AS_GKP_INCLUDE_H
#define AS_GKP_INCLUDE_H
-static const char *rcsid_AS_GKP_INCLUDE_H = "$Id: AS_GKP_include.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_AS_GKP_INCLUDE_H = "$Id: AS_GKP_include.H 4499 2014-01-29 16:28:07Z brianwalenz $";
#include <stdio.h>
#include <errno.h>
@@ -99,6 +99,7 @@ dumpGateKeeperFragments(char *gkpStoreName,
void
dumpGateKeeperAsFasta(char *gkpStoreName,
char *prefix,
+ int withLibName,
AS_IID begIID,
AS_IID endIID,
char *iidToDump,
@@ -122,6 +123,7 @@ dumpGateKeeperAsFRG(char *gkpStoreName,
void
dumpGateKeeperAsNewbler(char *gkpStoreName,
char *prefix,
+ int withLibName,
AS_IID begIID,
AS_IID endIID,
char *iidToDump,
@@ -134,6 +136,7 @@ dumpGateKeeperAsNewbler(char *gkpStoreName,
void
dumpGateKeeperAsFastQ(char *gkpStoreName,
char *prefix,
+ int withLibName,
AS_IID begIID,
AS_IID endIID,
char *iidToDump,
diff --git a/src/AS_GKP/AS_GKP_main.C b/src/AS_GKP/AS_GKP_main.C
index 327f9e4..7d7ffd3 100644
--- a/src/AS_GKP/AS_GKP_main.C
+++ b/src/AS_GKP/AS_GKP_main.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: AS_GKP_main.C 4426 2013-09-23 02:59:41Z brianwalenz $";
+const char *mainid = "$Id: AS_GKP_main.C 4640 2015-04-21 00:12:47Z brianwalenz $";
#include "AS_global.H"
@@ -40,6 +40,7 @@ const char *mainid = "$Id: AS_GKP_main.C 4426 2013-09-23 02:59:41Z brianwalenz $
#include "AS_GKP_include.H"
char *progName = NULL;
+long srandSeed = 0;
gkStore *gkpStore = NULL;
gkFragment *gkFrag1 = NULL;
@@ -101,6 +102,7 @@ usage(char *filename, int longhelp) {
fprintf(stdout, " so that the untrimmed length is close to l bp\n");
fprintf(stdout, " -longestovermin <lib> <n> pick all reads longer than n bp from library lib\n");
fprintf(stdout, " -longestlength <lib> <n> pick longest reads from library lib to add up to n total bp\n");
+ fprintf(stdout, " -deterministic use a constant seed for random subset dumps\n");
fprintf(stdout, "\n");
fprintf(stdout, " ---------\n");
fprintf(stdout, " [options]\n");
@@ -142,7 +144,8 @@ usage(char *filename, int longhelp) {
fprintf(stdout, " are dumped into a single file. This will create files 'prefix.paired.fastq',\n");
fprintf(stdout, " 'prefix.1.fastq', 'prefix.2.fastq' and 'prefix.unmated.fastq' for unmated\n");
fprintf(stdout, " reads. Options -donotfixmates and -clear also apply.\n");
-
+ fprintf(stdout, " -withlibname For -dumpfasta, -dumpnewbler and -dumpfastq, embed the libraryname in the\n");
+ fprintf(stdout, " created files, e.g., prefix.libname.1.fastq for fastq files.\n");
fprintf(stdout, "\n");
if (longhelp == 0) {
@@ -349,7 +352,7 @@ constructIIDdump(char *gkpStoreName,
// Now pick N reads from our list of candidates, and let the dump
// routines fill in the missing mates
- srand48(time(NULL));
+ srand48(srandSeed == 0 ? time(NULL) : srandSeed);
if (dumpRandLength > 0) {
double a = (double)lenFrag / (numSingle + numMated);
@@ -453,6 +456,7 @@ constructIIDdumpLongest(char *gkpStoreName,
lenFrag += iter->first;
if (lenFrag >= dumpLongestLength) {
+ fprintf(stderr, "Longest picked cutoff: "F_U32"\n", iter->first);
break;
}
}
@@ -511,6 +515,7 @@ main(int argc, char **argv) {
int dumpInvert = 0;
int dumpFormat = 2;
char *dumpPrefix = NULL;
+ int dumpWithLibName = 0;
uint32 dumpRandLib = 0; // 0 means "from any library"
uint32 dumpRandMateNum = 0;
uint32 dumpRandSingNum = 0; // Not a command line option
@@ -574,6 +579,9 @@ main(int argc, char **argv) {
fprintf(stderr, "%s: -randommated told to dump 0 mates; exit.\n", argv[0]);
exit(0);
}
+ } else if (strcmp(argv[arg], "-deterministic") == 0) {
+ srandSeed = 1;
+
} else if (strcmp(argv[arg], "-randomsubset") == 0) {
dumpRandLib = atoi(argv[++arg]);
dumpRandFraction = atof(argv[++arg]);
@@ -611,6 +619,8 @@ main(int argc, char **argv) {
dump = DUMP_FRAGMENTS;
} else if (strcmp(argv[arg], "-withsequence") == 0) {
dumpWithSequence = 1;
+ } else if (strcmp(argv[arg], "-withlibname") == 0) {
+ dumpWithLibName = 1;
} else if (strcmp(argv[arg], "-clear") == 0) {
dumpClear = gkStore_decodeClearRegionLabel(argv[++arg]);
if (dumpClear == AS_READ_CLEAR_ERROR) {
@@ -683,6 +693,9 @@ main(int argc, char **argv) {
arg++;
}
+ if (begIID > endIID)
+ fprintf(stderr, "ERROR: incorrect range: -b (%d) must no larger than -e (%d).\n", begIID, endIID), exit(1);
+
if ((err) || (gkpStoreName == NULL) || (firstFileArg == 0)) {
usage(progName, hlp);
exit(1);
@@ -715,7 +728,7 @@ main(int argc, char **argv) {
dumpDoNotUseUIDs);
break;
case DUMP_FASTA:
- dumpGateKeeperAsFasta(gkpStoreName, dumpPrefix, begIID, endIID, iidToDump,
+ dumpGateKeeperAsFasta(gkpStoreName, dumpPrefix, dumpWithLibName, begIID, endIID, iidToDump,
doNotFixMates,
dumpAllReads, dumpAllBases,
dumpClear,
@@ -729,7 +742,7 @@ main(int argc, char **argv) {
dumpDoNotUseUIDs);
break;
case DUMP_NEWBLER:
- dumpGateKeeperAsNewbler(gkpStoreName, dumpPrefix, begIID, endIID, iidToDump,
+ dumpGateKeeperAsNewbler(gkpStoreName, dumpPrefix, dumpWithLibName, begIID, endIID, iidToDump,
doNotFixMates,
dumpAllReads, dumpAllBases,
dumpClear,
@@ -738,12 +751,12 @@ main(int argc, char **argv) {
case DUMP_LASTFRG:
{
gkStore *gkp = new gkStore(gkpStoreName, FALSE, FALSE);
- fprintf(stdout, "Last frag in store is iid = %d\n", gkp->gkStore_getNumFragments());
+ fprintf(stdout, "Last frag in store is iid = "F_U32"\n", gkp->gkStore_getNumFragments());
delete gkp;
}
break;
case DUMP_FASTQ:
- dumpGateKeeperAsFastQ(gkpStoreName, dumpPrefix, begIID, endIID, iidToDump,
+ dumpGateKeeperAsFastQ(gkpStoreName, dumpPrefix, dumpWithLibName, begIID, endIID, iidToDump,
doNotFixMates,
dumpAllReads, dumpAllBases,
dumpClear,
diff --git a/src/AS_GKP/build-ordered-reads.sh b/src/AS_GKP/build-ordered-reads.sh
new file mode 100644
index 0000000..0afa1be
--- /dev/null
+++ b/src/AS_GKP/build-ordered-reads.sh
@@ -0,0 +1,413 @@
+#!/bin/sh
+#
+# Given a set of trimmed reads, map them to reference, filter out any that do not
+# map completely, and generate an output set of reads that is ordered and oriented.
+#
+
+if [ -d "/work/scripts" ] ; then
+ bin=/work/wgspb/FreeBSD-amd64/bin
+ scp=/work/pacbio-overlapper/scripts
+ export PATH=${PATH}:/work/scripts
+fi
+
+if [ -d "/usr/local/projects/CELERA/bri/scripts" ] ; then
+ bin=/usr/local/projects/CELERA/bri/wgspb/Linux-amd64/bin
+ scp=/usr/local/projects/CELERA/bri/pacbio-assembler/scripts
+ export PATH=${PATH}:/usr/local/projects/CELERA/bri/scripts
+fi
+
+if [ -z $bin ] ; then
+ echo scripts not found.
+ exit
+fi
+
+if [ -z $1 ] ; then
+ echo usage: $0 work-directory reads.fastq reference.fasta
+ exit
+fi
+
+wrk=$1 # Working directory name
+pfx="test" # Prefix of the files
+inp=$2 # Path to input FASTQ reads
+ref=$3 # Path to input FASTA reference
+
+if [ ! -d $wrk ] ; then
+ mkdir $wrk
+fi
+
+if [ ! -e $inp ] ; then
+ echo "Failed to find input FASTQ reads in '$inp'"
+ exit
+fi
+
+if [ ! -e $ref ] ; then
+ echo ""
+ echo "WARNING: no reference, will not build ordered assembly."
+ echo
+fi
+
+
+if [ ! -e "$wrk/build-ordered-reads.spec" ] ; then
+ echo $wrk/build-ordered-reads.spec
+ cat > $wrk/build-ordered-reads.spec \
+<<EOF
+fakeUIDs = 1
+
+useGrid = 0
+scriptOnGrid = 0
+
+sge = -l medium
+sgeScript = -pe threaded 1 -l memory=8g
+sgeOverlap = -pe threaded 4 -l memory=2g
+sgeMerOverlapSeed = DIE
+sgeMerOverlapExtend = DIE
+sgeConsensus = -pe threaded 1 -l memory=6g
+sgeFragmentCorrection = DIE
+sgeOverlapCorrection = DIE
+
+ovlConcurrency = 3
+cnsConcurrency = 12
+
+
+ovlErrorRate = 0.30
+
+utgGraphErrorRate = 0.30
+utgMergeErrorRate = 0.30
+
+cgwErrorRate = 0.30
+cnsErrorRate = 0.30
+
+
+merylMemory = DIE
+merylThreads = 4
+
+merThreshold = 0
+
+merSize = 17
+
+doOBT = 0
+doFragmentCorrection = 0
+
+ovlThreads = 4
+ovlHashBits = 23
+ovlHashBlockLength = 10000000
+ovlRefBlockSize = 1000
+
+unitigger = bogart
+
+cnsMinFrags = 10
+cnsPartitions = 256
+EOF
+fi
+
+
+########################################
+#
+# Build a gkpStore for the reads, then dump for mapping
+#
+# Map reads to reference, build overlaps, filter
+
+if [ ! -d $wrk/BO ] ; then
+ mkdir $wrk/BO
+fi
+
+if [ ! -d "$wrk/BO/$pfx.gkpStore" ] ; then
+ echo fastqToCA
+ $bin/fastqToCA \
+ -libraryname L \
+ -technology sanger \
+ -type sanger \
+ -reads $inp \
+ > $wrk/BO/$pfx.frg
+
+ echo gatekeeper -T -F -o $wrk/BO/$pfx.gkpStore $wrk/BO/$pfx.frg
+ $bin/gatekeeper -T -F -o $wrk/BO/$pfx.gkpStore $wrk/BO/$pfx.frg
+fi
+
+if [ ! -e "$wrk/BO/$pfx.fastq" ] ; then
+ echo gatekeeper -dumpfastq $wrk/BO/$pfx $wrk/BO/$pfx.gkpStore
+ $bin/gatekeeper -dumpfastq $wrk/BO/$pfx $wrk/BO/$pfx.gkpStore
+
+ awk '{ print $1 }' < $wrk/BO/$pfx.unmated.fastq > $wrk/BO/$pfx.fastq
+
+ rm -f $wrk/BO/$pfx.unmated.fastq
+ rm -f $wrk/BO/$pfx.paired.fastq
+ rm -f $wrk/BO/$pfx.1.fastq
+ rm -f $wrk/BO/$pfx.2.fastq
+
+ echo replaceUIDwithName $wrk/BO/$pfx.gkpStore.fastqUIDmap $wrk/BO/$pfx.fastq
+ $bin/replaceUIDwithName $wrk/BO/$pfx.gkpStore.fastqUIDmap $wrk/BO/$pfx.fastq
+fi
+
+#
+# OLD blasr options
+#
+# -maxLCPLength 15
+# -nCandidates 25
+# -maxScore -500
+#
+
+if [ -e $ref ] ; then
+ if [ ! -e "$wrk/BO/$pfx.blasr.badnm.sam" ] ; then
+ echo \
+ blasr \
+ -noSplitSubreads \
+ -nproc 4 \
+ -minMatch 12 \
+ -bestn 10 \
+ -nCandidates 25 \
+ -minPctIdentity 65.0 \
+ -sam -clipping soft \
+ -out $wrk/BO/$pfx.blasr.badnm.sam \
+ $wrk/BO/$pfx.fastq \
+ $ref
+
+ blasr \
+ -noSplitSubreads \
+ -nproc 4 \
+ -minMatch 12 \
+ -bestn 10 \
+ -minPctIdentity 65.0 \
+ -sam -clipping soft \
+ -out $wrk/BO/$pfx.blasr.badnm.sam \
+ $wrk/BO/$pfx.fastq \
+ $ref
+ fi
+
+
+ if [ ! -e "$wrk/BO/$pfx.blasr.sam" ] ; then
+ echo a
+ echo \
+ samtools calmd \
+ -S $wrk/BO/$pfx.blasr.badnm.sam \
+ $ref \
+ \> $wrk/BO/$pfx.blasr.sam
+
+ samtools calmd \
+ -S $wrk/BO/$pfx.blasr.badnm.sam \
+ $ref \
+ > $wrk/BO/$pfx.blasr.sam
+ fi
+
+
+ if [ ! -e "$wrk/BO/$pfx.blasr.sam.coords" ] ; then
+ echo bowtie2-to-nucmercoords.pl $ref \< $wrk/BO/$pfx.blasr.sam \> $wrk/BO/$pfx.blasr.sam.coords
+ bowtie2-to-nucmercoords.pl $ref < $wrk/BO/$pfx.blasr.sam > $wrk/BO/$pfx.blasr.sam.coords
+ fi
+
+ if [ ! -e "$wrk/BO/$pfx.blasr.sam.coords.ova" ] ; then
+ echo infer-olaps-from-coords
+ perl $scp/infer-olaps-from-genomic-coords.pl \
+ $wrk/BO/$pfx.blasr.sam.coords \
+ $wrk/BO/$pfx.fastq \
+ $wrk/BO/$pfx.blasr.sam.coords \
+ $wrk/BO/$pfx.gkpStore.fastqUIDmap
+ fi
+fi
+
+
+
+########################################
+#
+# Start an assembly using the blasr overlaps for the perfectly mapping reads.
+#
+# We need to rebuild the overlaps to get the IIDs correct for this assembly.
+# If we were to just use the BO/gkpStore and BO/ovlStore, all the reads that
+# don't map perfectly end up as singletons.
+#
+# For this to work, it is critical that the reads have their original names, NOT UIDs.
+#
+# NOTE! Reading coords and lengths from the BL work above, and using reads from the BO work here.
+#
+
+if [ -e $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq ] ; then
+ if [ ! -d $wrk/BL ] ; then
+ mkdir $wrk/BL
+ fi
+
+ if [ ! -d "$wrk/BL/$pfx.gkpStore" ] ; then
+ $bin/fastqToCA \
+ -libraryname L \
+ -technology sanger \
+ -type sanger \
+ -reads $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq \
+ > $wrk/BL/$pfx.frg
+
+ $bin/gatekeeper -T -F \
+ -o $wrk/BL/$pfx.gkpStore \
+ $wrk/BL/$pfx.frg
+ fi
+
+ if [ ! -e "$wrk/BL/$pfx.blasr.sam.coords.ova" ] ; then
+ perl $scp/infer-olaps-from-genomic-coords.pl \
+ $wrk/BL/$pfx.blasr.sam.coords \
+ $wrk/BO/$pfx.fastq \
+ $wrk/BO/$pfx.blasr.sam.coords \
+ $wrk/BL/$pfx.gkpStore.fastqUIDmap
+ fi
+
+ if [ ! -d "$wrk/BL/$pfx.ovlStore" ] ; then
+ $bin/convertOverlap -ovl \
+ < $wrk/BL/$pfx.blasr.sam.coords.ova \
+ > $wrk/BL/$pfx.blasr.sam.coords.ovb
+
+ $bin/overlapStoreBuild \
+ -o $wrk/BL/$pfx.ovlStore \
+ -g $wrk/BL/$pfx.gkpStore \
+ -F 1 \
+ $wrk/BL/$pfx.blasr.sam.coords.ovb
+ fi
+
+ if [ ! -d "$wrk/BL/$pfx.tigStore" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/BL -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ stopAfter=unitigger \
+ $wrk/BL/$pfx.frg
+ else
+ if [ ! -e "$wrk/BL/$pfx.qc" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/BL -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ $wrk/BL/$pfx.frg
+ fi
+ fi
+fi
+
+
+
+########################################
+#
+# Assembly with all reads.
+#
+if [ ! -d $wrk/CA ] ; then
+ mkdir $wrk/CA
+fi
+
+if [ ! -d "$wrk/CA/$pfx.frg" ] ; then
+ $bin/fastqToCA \
+ -libraryname L \
+ -technology sanger \
+ -type sanger \
+ -reads $inp \
+ > $wrk/CA/$pfx.frg
+fi
+
+if [ ! -d "$wrk/CA/$pfx.tigStore" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CA -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ stopAfter=unitigger \
+ $wrk/CA/$pfx.frg
+else
+ if [ ! -e "$wrk/CA/$pfx.qc" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CA -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ $wrk/CA/$pfx.frg
+ fi
+fi
+
+
+
+########################################
+#
+# Assembly with all reads.
+#
+if [ ! -d $wrk/CAcorrected ] ; then
+ mkdir $wrk/CAcorrected
+fi
+
+if [ ! -d "$wrk/CAcorrected/$pfx.frg" ] ; then
+ $bin/fastqToCA \
+ -libraryname L \
+ -technology sanger \
+ -type sanger \
+ -reads $inp \
+ > $wrk/CAcorrected/$pfx.frg
+fi
+
+if [ ! -d "$wrk/CAcorrected/$pfx.tigStore" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CAcorrected -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ doFragmentCorrection=1 \
+ stopAfter=unitigger \
+ $wrk/CAcorrected/$pfx.frg
+else
+ if [ ! -e "$wrk/CAcorrected/$pfx.qc" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CAcorrected -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ doFragmentCorrection=1 \
+ $wrk/CAcorrected/$pfx.frg
+ fi
+fi
+
+
+
+
+
+########################################
+#
+# Assembly with the perfectly mapping reads.
+#
+
+if [ -e $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq ] ; then
+ if [ ! -d $wrk/CAordered ] ; then
+ mkdir $wrk/CAordered
+ fi
+
+ if [ ! -d "$wrk/CAordered/$pfx.frg" ] ; then
+ $bin/fastqToCA \
+ -libraryname L \
+ -technology sanger \
+ -type sanger \
+ -reads $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq \
+ > $wrk/CAordered/$pfx.frg
+ fi
+
+ if [ ! -d "$wrk/CAordered/$pfx.tigStore" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CAordered -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ stopAfter=unitigger \
+ $wrk/CAordered/$pfx.frg
+ else
+ if [ ! -e "$wrk/CAordered/$pfx.qc" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CAordered -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ $wrk/CAordered/$pfx.frg
+ fi
+ fi
+fi
+
+
+
+########################################
+#
+# Assembly with the perfectly mapping reads.
+#
+
+if [ -e $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq ] ; then
+ if [ ! -d $wrk/CAorderedCorrected ] ; then
+ mkdir $wrk/CAorderedCorrected
+ fi
+
+ if [ ! -d "$wrk/CAorderedCorrected/$pfx.frg" ] ; then
+ $bin/fastqToCA \
+ -libraryname L \
+ -technology sanger \
+ -type sanger \
+ -reads $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq \
+ > $wrk/CAorderedCorrected/$pfx.frg
+ fi
+
+ if [ ! -d "$wrk/CAorderedCorrected/$pfx.tigStore" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CAorderedCorrected -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ doFragmentCorrection=1 \
+ stopAfter=unitigger \
+ $wrk/CAorderedCorrected/$pfx.frg
+ else
+ if [ ! -e "$wrk/CAorderedCorrected/$pfx.qc" ] ; then
+ perl $bin/runCA -p $pfx -d $wrk/CAorderedCorrected -s $wrk/build-ordered-reads.spec \
+ useGrid=0 scriptOnGrid=0 \
+ doFragmentCorrection=1 \
+ $wrk/CAorderedCorrected/$pfx.frg
+ fi
+ fi
+fi
diff --git a/src/AS_GKP/fastqAnalyze.C b/src/AS_GKP/fastqAnalyze.C
index c481d63..18c420f 100644
--- a/src/AS_GKP/fastqAnalyze.C
+++ b/src/AS_GKP/fastqAnalyze.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: fastqAnalyze.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: fastqAnalyze.C 4617 2015-02-20 17:43:00Z brianwalenz $";
#include "AS_global.H"
@@ -42,7 +42,7 @@ char indexToBase[8];
#define FREQ_Z 6 // Everything else
#define FREQ_NUM 7
-#define MAX_READ_LEN 10240
+#define MAX_READ_LEN AS_READ_MAX_NORMAL_LEN+1
class nucFreq {
public:
@@ -92,10 +92,10 @@ doStats(char *inName,
vector<uint32> seqLen;
nucFreq *freq = new nucFreq;
- char A[MAX_READ_LEN];
- char B[MAX_READ_LEN];
- char C[MAX_READ_LEN];
- char D[MAX_READ_LEN];
+ char *A = new char [MAX_READ_LEN];
+ char *B = new char [MAX_READ_LEN];
+ char *C = new char [MAX_READ_LEN];
+ char *D = new char [MAX_READ_LEN];
errno = 0;
FILE *F = fopen(inName, "r");
@@ -152,6 +152,14 @@ doStats(char *inName,
fprintf(stderr, "Reading "F_U64"\r", totSeqs);
}
+ delete [] A;
+ delete [] B;
+ delete [] C;
+ delete [] D;
+
+ //fclose(O);
+ fclose(F);
+
fprintf(stderr, "Read "F_U64"\n", totSeqs);
fprintf(stdout, "%s\n", inName);
@@ -230,9 +238,6 @@ doStats(char *inName,
fprintf(stdout, "%s\t"F_U64"\t%.4f%%\n", output[ii].label, output[ii].count, output[ii].freq);
output.clear();
- //fclose(O);
- fclose(F);
-
delete freq;
}
@@ -252,10 +257,10 @@ doAnalyzeQV(char *inName,
uint32 numValid = 5;
uint32 numTrials = 100000;
- char A[MAX_READ_LEN];
- char B[MAX_READ_LEN];
- char C[MAX_READ_LEN];
- char D[MAX_READ_LEN];
+ char *A = new char [MAX_READ_LEN];
+ char *B = new char [MAX_READ_LEN];
+ char *C = new char [MAX_READ_LEN];
+ char *D = new char [MAX_READ_LEN];
errno = 0;
FILE *F = fopen(inName, "r");
@@ -315,6 +320,11 @@ doAnalyzeQV(char *inName,
fclose(F);
+ delete [] A;
+ delete [] B;
+ delete [] C;
+ delete [] D;
+
fprintf(stdout, "%s --", inName);
if (isNotSanger == false) fprintf(stdout, " SANGER");
@@ -357,10 +367,10 @@ doTransformQV(char *inName,
uint32 numValid = 0;
- char A[MAX_READ_LEN];
- char B[MAX_READ_LEN];
- char C[MAX_READ_LEN];
- char D[MAX_READ_LEN];
+ char *A = new char [MAX_READ_LEN];
+ char *B = new char [MAX_READ_LEN];
+ char *C = new char [MAX_READ_LEN];
+ char *D = new char [MAX_READ_LEN];
if (originalIsSolexa == true) numValid++;
if (originalIsIllumina == true) numValid++;
@@ -412,6 +422,11 @@ doTransformQV(char *inName,
fprintf(O, "%s%s%s%s\n", A, B, C, D);
}
+ delete [] A;
+ delete [] B;
+ delete [] C;
+ delete [] D;
+
fclose(F);
fclose(O);
}
diff --git a/src/AS_GKP/fastqSample.C b/src/AS_GKP/fastqSample.C
index c4ddec2..b29de8d 100644
--- a/src/AS_GKP/fastqSample.C
+++ b/src/AS_GKP/fastqSample.C
@@ -19,29 +19,43 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: fastqSample.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: fastqSample.C 4644 2015-05-12 13:56:47Z skoren $";
#include "AS_global.H"
+#include <vector>
+#include <algorithm>
+
+using namespace std;
+
+#define MAXLEN 1024*1024
class aRead {
public:
aRead() {
- memset(a, 0, sizeof(char) * 1024);
- memset(b, 0, sizeof(char) * 1024);
- memset(c, 0, sizeof(char) * 1024);
- memset(d, 0, sizeof(char) * 1024);
+ memset(a, 0, sizeof(char) * MAXLEN);
+ memset(b, 0, sizeof(char) * MAXLEN);
+ memset(c, 0, sizeof(char) * MAXLEN);
+ memset(d, 0, sizeof(char) * MAXLEN);
};
- ~aRead() {};
bool read(FILE *F) {
+
+ a[0] = 0;
+ b[0] = 0;
+ c[0] = 0;
+ d[0] = 0;
+
if (F == NULL)
return(false);
- fgets(a, 1024, F);
- fgets(b, 1024, F);
- fgets(c, 1024, F);
- fgets(d, 1024, F);
+ fgets(a, MAXLEN, F);
+ fgets(b, MAXLEN, F);
+ fgets(c, MAXLEN, F);
+ fgets(d, MAXLEN, F);
+
+ if (feof(F) == true)
+ return(false);
if ((a[0] != '@') || (c[0] != '+')) {
fprintf(stderr, "ERROR: Not FastQ. Read lines:\n");
@@ -51,7 +65,8 @@ public:
fprintf(stderr, " %s", d);
exit(1);
}
- return(feof(F) == false);
+
+ return(true);
};
void write(FILE *F) {
if (F == NULL)
@@ -62,201 +77,367 @@ public:
fputs(d, F);
};
+ uint32 length(void) {
+ return(strlen(b) - 1); // Newline still exists
+ };
+
private:
- char a[1024];
- char b[1024];
- char c[1024];
- char d[1024];
+ char a[MAXLEN];
+ char b[MAXLEN];
+ char c[MAXLEN];
+ char d[MAXLEN];
+};
+
+
+
+class anInput {
+public:
+ anInput() {
+ id = 0;
+ len = 0;
+ };
+ anInput(uint64 id_, uint32 len1_, uint32 len2_) {
+ id = id_;
+ len = len1_ + len2_;
+ };
+
+ uint64 id;
+ uint32 len;
+};
+
+
+inline
+bool
+anInputByLongest(const anInput &a, const anInput &b) {
+ return(a.len > b.len);
};
int
main(int argc, char **argv) {
- aRead Ar, Br;
- FILE *Ai = NULL, *Bi = NULL;
- FILE *Ao = NULL, *Bo = NULL;
+ aRead *Ar = new aRead, *Br = new aRead;
+ FILE *Ai = NULL, *Bi = NULL;
+ FILE *Ao = NULL, *Bo = NULL;
- uint32 *ids = NULL;
- char *save = NULL;
+ vector<anInput> ids;
+ vector<bool> sav;
- uint64 NUMINPUT = 0; // Number of pairs in the input
- char *NAME = NULL; // Prefix name
+ char *INPNAME = NULL;
+ char *OUTNAME = NULL;
+ bool AUTONAME = false;
+ uint64 NUMINPUT = 0; // Number of pairs in the input
+ uint64 READLENGTH = 0; // For mated reads, 2x read size
bool isMated = true;
+ uint32 MINLEN = 0;
+ uint32 LONGEST = false;
+
uint64 GENOMESIZE = 0; // Size of the genome in bp
- uint64 READLENGTH = 0; // For mated reads, 2x read size
- uint64 COVERAGE = 0; // Desired coverage in output
+ double COVERAGE = 0; // Desired coverage in output
uint64 NUMOUTPUT = 0; // Number of pairs to output
double FRACTION = 0.0; // Desired fraction of the input
+ uint64 BASES = 0; // Desired amount of sequence
+
char path1[FILENAME_MAX];
char path2[FILENAME_MAX];
+ srand48(time(NULL));
+
int arg=1;
int err=0;
while (arg < argc) {
- if (strcmp(argv[arg], "-n") == 0) {
- NAME = argv[++arg];
+ if (strcmp(argv[arg], "-I") == 0) {
+ INPNAME = argv[++arg];
+
+ } else if (strcmp(argv[arg], "-O") == 0) {
+ OUTNAME = argv[++arg];
- } else if (strcmp(argv[arg], "-t") == 0) {
- // Total reads in input
+ } else if (strcmp(argv[arg], "-A") == 0) {
+ AUTONAME = true;
+
+ } else if (strcmp(argv[arg], "-T") == 0) {
NUMINPUT = atoi(argv[++arg]);
- } else if (strcmp(argv[arg], "-u") == 0) {
+ } else if (strcmp(argv[arg], "-L") == 0) {
+ READLENGTH = atoi(argv[++arg]);
+
+ } else if (strcmp(argv[arg], "-U") == 0) {
isMated = false;
- } else if (strcmp(argv[arg], "-g") == 0) {
- // Genome size
- GENOMESIZE = atoi(argv[++arg]);
+ } else if (strcmp(argv[arg], "-m") == 0) {
+ MINLEN = atoi(argv[++arg]);
+ } else if (strcmp(argv[arg], "-max") == 0) {
+ LONGEST = true;
- } else if (strcmp(argv[arg], "-l") == 0) {
- // Length of read
- READLENGTH = atoi(argv[++arg]);
+ } else if (strcmp(argv[arg], "-g") == 0) {
+ GENOMESIZE = atol(argv[++arg]);
} else if (strcmp(argv[arg], "-c") == 0) {
- // Desired coverage
- COVERAGE = atoi(argv[++arg]);
+ COVERAGE = atof(argv[++arg]);
} else if (strcmp(argv[arg], "-p") == 0) {
- // Desired coverage
NUMOUTPUT = atoi(argv[++arg]);
} else if (strcmp(argv[arg], "-f") == 0) {
- // Desired fraction
FRACTION = atof(argv[++arg]);
+ } else if (strcmp(argv[arg], "-b") == 0) {
+ BASES = atol(argv[++arg]);
+
} else {
err++;
}
arg++;
}
- if (NAME == NULL)
+ if (INPNAME == NULL)
err++;
+ if (OUTNAME == NULL)
+ OUTNAME = INPNAME;
if ((GENOMESIZE == 0) && (COVERAGE != 0))
err++;
- if ((READLENGTH == 0) && (COVERAGE != 0))
- err++;
- if ((COVERAGE == 0) && (NUMOUTPUT == 0) && (FRACTION == 0.0))
+ if ((COVERAGE == 0) && (NUMOUTPUT == 0) && (FRACTION == 0.0) && (BASES == 0))
err++;
if (err) {
fprintf(stderr, "\n");
fprintf(stderr, "usage: %s [opts]\n", argv[0]);
- fprintf(stderr, " -n NAME name (prefix) of the reads\n");
- fprintf(stderr, " -t T total number of mate pairs in the input\n");
- fprintf(stderr, " -u reads are unmated, expected in *.u.fastq\n");
+ fprintf(stderr, " Input Specification\n");
+ fprintf(stderr, " -I NAME input name (prefix) of the reads\n");
+ fprintf(stderr, " -T T total number of mate pairs in the input (if not supplied, will be counted)\n");
+ fprintf(stderr, " -L L length of a single read (if not supplied, will be determined)\n");
+ fprintf(stderr, " -U reads are unmated, expected in *.u.fastq\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " Output Specification\n");
+ fprintf(stderr, " -O NAME output name (prefix) of the reads (default is same as -I)\n");
+ fprintf(stderr, " -A automatically include coverage or number of reads in the output name\n");
+ fprintf(stderr, " -m L ignore reads shorter than L bases\n");
+ fprintf(stderr, " -max don't sample randomly, pick the longest reads\n");
fprintf(stderr, "\n");
- fprintf(stderr, " Method 1: specify desired coverage:\n");
+ fprintf(stderr, " Method 1: specify desired output coverage:\n");
fprintf(stderr, " -g G genome size\n");
- fprintf(stderr, " -l L length of a single read\n");
- fprintf(stderr, " -c C desired coverage in the output reads (INTEGER ONLY)\n");
+ fprintf(stderr, " -c C desired coverage in the output reads\n");
fprintf(stderr, "\n");
fprintf(stderr, " Method 2: specify desired number of output pairs\n");
- fprintf(stderr, " -p N generate 2N reads, or N pairs of reads\n");
+ fprintf(stderr, " -p N for mated reads, output 2N reads, or N pairs of reads\n");
+ fprintf(stderr, " for unmated reads, output N reads\n");
fprintf(stderr, "\n");
fprintf(stderr, " Method 3: specify a desired fraction of the input:\n");
- fprintf(stderr, " -f F generate F * T pairs of reads (T as above in -t option)\n");
+ fprintf(stderr, " -f F output F * T pairs of reads (T as above in -t option)\n");
fprintf(stderr, " 0.0 < F <= 1.0\n");
fprintf(stderr, "\n");
+ fprintf(stderr, " Method 4: specify a desired total length\n");
+ fprintf(stderr, " -b B output reads/pairs until B bases is exceeded\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "\n");
fprintf(stderr, "Samples reads from paired Illumina reads NAME.1.fastq and NAME.2.fastq and outputs:\n");
fprintf(stderr, " NAME.Cx.1.fastq and N.Cx.2.fastq (for coverage based sampling)\n");
fprintf(stderr, " NAME.n=N.1.fastq and N.n=N.2.fastq (for coverage based sampling)\n");
fprintf(stderr, "\n");
- fprintf(stderr, "If -t is not supplied, the number of reads will be counted for you.\n");
+ fprintf(stderr, "If -T is not supplied, the number of reads will be counted for you.\n");
fprintf(stderr, "\n");
- if (NAME == NULL)
- fprintf(stderr, "ERROR: no name supplied with -n.\n");
+ if (INPNAME == NULL)
+ fprintf(stderr, "ERROR: no name supplied with -I.\n");
if ((GENOMESIZE == 0) && (COVERAGE != 0))
fprintf(stderr, "ERROR: no genome size supplied with -g (when using -c)\n");
- if ((READLENGTH == 0) && (COVERAGE != 0))
- fprintf(stderr, "ERROR: no read length supplied with -l (when using -c)\n");
- if ((COVERAGE == 0) && (NUMOUTPUT == 0) && (FRACTION == 0.0))
- fprintf(stderr, "ERROR: no desired coverage, number of fraction supplied with -c, -p or -f\n");
+ if ((COVERAGE == 0) && (NUMOUTPUT == 0) && (FRACTION == 0.0) && (BASES == 0))
+ fprintf(stderr, "ERROR: no method supplied with -c, -p, -f or -b\n");
fprintf(stderr, "\n");
exit(1);
}
- if (NUMINPUT == 0) {
+ //
+ // We know not enough about the reads, and are forced to scan the entire
+ // inputs.
+ //
+
+ uint64 totBasesInInput = 0;
+ uint64 totPairsInInput = 0;
+
+ if ((NUMINPUT == 0) || (READLENGTH == 0)) {
uint64 Ac = 0;
uint64 Bc = 0;
fprintf(stderr, "Counting the number of reads in the input.\n");
- sprintf(path1, "%s.%c.fastq", NAME, (isMated == true) ? '1' : 'u');
- sprintf(path2, "%s.%c.fastq", NAME, (isMated == true) ? '2' : 'u');
+ sprintf(path1, "%s.%c.fastq", INPNAME, (isMated == true) ? '1' : 'u');
+ sprintf(path2, "%s.%c.fastq", INPNAME, (isMated == true) ? '2' : 'u');
errno = 0;
Ai = fopen(path1, "r");
if (errno)
fprintf(stderr, "Failed to open '%s': %s\n", path1, strerror(errno)), exit(1);
- while (Ar.read(Ai))
- Ac++;
- fclose(Ai);
-
- fprintf(stderr, "Found "F_U64" reads in '%s'\n", Ac, path1);
if (isMated == true) {
errno = 0;
Bi = fopen(path2, "r");
if (errno)
fprintf(stderr, "Failed to open '%s': %s\n", path2, strerror(errno)), exit(1);
- while (Br.read(Bi))
- Bc++;
- fclose(Bi);
+ }
- fprintf(stderr, "Found "F_U64" reads in '%s'\n", Bc, path2);
+ bool moreA = Ar->read(Ai);
+ bool moreB = Br->read(Bi);
- if (Ac != Bc) {
- fprintf(stderr, "ERROR: Number of reads must be the same.\n");
- exit(1);
- }
+ while (moreA || moreB) {
+ uint32 lA = Ar->length();
+ uint32 lB = Br->length();
+
+ if (lA > 0) Ac++;
+ if (lB > 0) Bc++;
+
+ ids.push_back(anInput(totPairsInInput, lA, lB));
+ sav.push_back(false);
+
+ totPairsInInput += 1;
+ totBasesInInput += lA + lB;
+
+ moreA = Ar->read(Ai);
+ moreB = Br->read(Bi);
}
- NUMINPUT = Ac;
+ if (Ai) fclose(Ai);
+ if (Bi) fclose(Bi);
+
+ fprintf(stderr, "Found "F_U64" bases and "F_U64" reads in '%s'\n",
+ totBasesInInput, totPairsInInput, path1);
+
+ if (Ac != Bc) {
+ fprintf(stderr, "ERROR: Number of reads in the .1 and .2 files must be the same.\n");
+ exit(1);
+ }
}
- if (isMated)
- READLENGTH *= 2;
+ // Otherwise, both NUMINPUT and READLENGTH are defined, so we can fill out ids
+ // with defaults.
+ else {
+ if (isMated == false)
+ for (uint64 ii=0; ii<NUMINPUT; ii++) {
+ ids.push_back(anInput(ii, READLENGTH, 0));
+ sav.push_back(false);
+ }
+ else
+ for (uint64 ii=0; ii<NUMINPUT; ii++) {
+ ids.push_back(anInput(ii, READLENGTH, READLENGTH));
+ sav.push_back(false);
+ }
- if ((NUMOUTPUT == 0) && (COVERAGE > 0))
- NUMOUTPUT = COVERAGE * GENOMESIZE / READLENGTH;
+ totBasesInInput = NUMINPUT * READLENGTH;
+ totPairsInInput = NUMINPUT;
+ }
- if ((NUMOUTPUT == 0) && (FRACTION > 0))
- NUMOUTPUT = (uint64)floor(FRACTION * NUMINPUT);
+ //
+ // Decide how much data to output, either as a read or base limit.
+ //
- if ((NUMOUTPUT > NUMINPUT) && (COVERAGE > 0))
- fprintf(stderr, "ERROR: not enough reads, "F_U64" in input, "F_U64" needed, for desired coverage "F_U64".\n",
- NUMINPUT, NUMOUTPUT, COVERAGE), exit(1);
+ uint64 nBasesToOutput = 0;
+ uint64 nPairsToOutput = 0;
- ids = new uint32 [NUMINPUT];
- save = new char [NUMINPUT];
- srand48(time(NULL));
+ if (GENOMESIZE > 0) {
+ nBasesToOutput = (uint64)(COVERAGE * GENOMESIZE);
+ }
- // Make a list of IDs, and clear the array that tells us to emit a read.
- for (uint64 i=0; i<NUMINPUT; i++) {
- ids[i] = i;
- save[i] = 0;
+ if (NUMOUTPUT > 0) {
+ nPairsToOutput = NUMOUTPUT;
}
- // Randomize the ID list.
- for (uint64 i=0; i<NUMINPUT; i++) {
- uint64 p = (uint64)lrand48() % NUMINPUT;
- uint32 a = ids[p];
+ if (FRACTION > 0) {
+ nPairsToOutput = (uint64)(FRACTION * totPairsInInput);
+ }
- ids[p] = ids[i];
- ids[i] = a;
+ if (BASES > 0) {
+ nBasesToOutput = BASES;
}
- // Take the first NUMOUTPUT things from the randomized ID list, mark them as "to be output".
- for (uint64 i=0; i<NUMOUTPUT; i++)
- save[ids[i]] = 1;
+ if (totBasesInInput < nBasesToOutput)
+ fprintf(stderr, "ERROR: not enough reads, "F_U64" bp in input, "F_U64" needed for desired .....\n",
+ totBasesInInput, nBasesToOutput),
+ exit(1);
+
+ if (totPairsInInput < nPairsToOutput)
+ fprintf(stderr, "ERROR: not enough reads, "F_U64" %s in input, "F_U64" needed for desired ......\n",
+ totPairsInInput, (isMated) ? "pairs" : "reads", nPairsToOutput),
+ exit(1);
+
+ //fprintf(stderr, "OUTPUT: %lu bases\n", nBasesToOutput);
+ //fprintf(stderr, "OUTPUT: %lu pairs\n", nPairsToOutput);
+
+
+ //
+ // Randomize the ID list, or sort by length.
+ //
+
+ if (LONGEST) {
+ fprintf(stderr, "Sorting by length\n");
+ sort(ids.begin(), ids.end(), anInputByLongest);
+
+ } else {
+ fprintf(stderr, "Shuffling sequences\n");
+ for (uint64 i=0; i<totPairsInInput; i++) {
+ uint64 p = (uint64)lrand48() % totPairsInInput;
+ anInput a = ids[p];
+
+ ids[p] = ids[i];
+ ids[i] = a;
+ }
+ }
+
+
+ //
+ // Decide what to output.
+ //
+
+ if (nPairsToOutput > 0) {
+ uint64 nPairs = 0;
- sprintf(path1, "%s.%c.fastq", NAME, (isMated == true) ? '1' : 'u');
- sprintf(path2, "%s.%c.fastq", NAME, (isMated == true) ? '2' : 'u');
+ for (uint64 i=0; i<totPairsInInput; i++) {
+ if (nPairs < nPairsToOutput)
+ sav[ids[i].id] = true;
+ else
+ break;
+
+ nPairs++;
+ }
+ }
+
+ if (nBasesToOutput > 0) {
+ uint64 nBases = 0;
+
+ for (uint64 i=0; i<totPairsInInput; i++) {
+ if (nBases < nBasesToOutput)
+ sav[ids[i].id] = true;
+ else
+ break;
+
+ nBases += ids[i].len;
+ }
+ }
+
+
+ //
+ // Count.
+ //
+
+ nPairsToOutput = 0;
+ nBasesToOutput = 0;
+
+ for (uint64 i=0; i<totPairsInInput; i++) {
+ if (sav[ids[i].id] == true) {
+ nPairsToOutput += 1;
+ nBasesToOutput += ids[i].len;
+ }
+ }
+
+
+ //
+ // Do the output
+ //
+
+ sprintf(path1, "%s.%c.fastq", INPNAME, (isMated == true) ? '1' : 'u');
+ sprintf(path2, "%s.%c.fastq", INPNAME, (isMated == true) ? '2' : 'u');
errno = 0;
Ai = fopen(path1, "r");
@@ -270,12 +451,18 @@ main(int argc, char **argv) {
fprintf(stderr, "Failed to open '%s': %s\n", path2, strerror(errno)), exit(1);
}
- if (COVERAGE > 0) {
- sprintf(path1, "%s.%03"F_U64P"x.%c.fastq", NAME, COVERAGE, (isMated == true) ? '1' : 'u');
- sprintf(path2, "%s.%03"F_U64P"x.%c.fastq", NAME, COVERAGE, (isMated == true) ? '2' : 'u');
+
+ if (AUTONAME == false) {
+ sprintf(path1, "%s.%c.fastq", OUTNAME, (isMated == true) ? '1' : 'u');
+ sprintf(path2, "%s.%c.fastq", OUTNAME, (isMated == true) ? '2' : 'u');
+
+ } else if (GENOMESIZE > 0) {
+ sprintf(path1, "%s.x=%07.3f.n=%09"F_U64P".%c.fastq", OUTNAME, (double)nBasesToOutput / GENOMESIZE, nPairsToOutput, (isMated == true) ? '1' : 'u');
+ sprintf(path2, "%s.x=%07.3f.n=%09"F_U64P".%c.fastq", OUTNAME, (double)nBasesToOutput / GENOMESIZE, nPairsToOutput, (isMated == true) ? '2' : 'u');
+
} else {
- sprintf(path1, "%s.n="F_U64".%c.fastq", NAME, NUMOUTPUT, (isMated == true) ? '1' : 'u');
- sprintf(path2, "%s.n="F_U64".%c.fastq", NAME, NUMOUTPUT, (isMated == true) ? '2' : 'u');
+ sprintf(path1, "%s.x=UNKNOWN.n=%09"F_U64P".%c.fastq", OUTNAME, nPairsToOutput, (isMated == true) ? '1' : 'u');
+ sprintf(path2, "%s.x=UNKNOWN.n=%09"F_U64P".%c.fastq", OUTNAME, nPairsToOutput, (isMated == true) ? '2' : 'u');
}
errno = 0;
@@ -295,12 +482,18 @@ main(int argc, char **argv) {
if (isMated == true) {
- fprintf(stderr, "Extracting "F_U64" mate pairs into %s and %s\n",
- NUMOUTPUT, path1, path2);
- for (; Ar.read(Ai) && Br.read(Bi); i++) {
- if ((i < NUMINPUT) && (save[i])) {
- Ar.write(Ao);
- Br.write(Bo);
+
+ if (nPairsToOutput > 0)
+ fprintf(stderr, "Extracting "F_U64" mate pairs into %s and %s\n",
+ nPairsToOutput, path1, path2);
+ else
+ fprintf(stderr, "Extracting "F_U64" bases of mate pairs into %s and %s\n",
+ nBasesToOutput, path1, path2);
+
+ for (; Ar->read(Ai) && Br->read(Bi); i++) {
+ if ((i < totPairsInInput) && (sav[i])) {
+ Ar->write(Ao);
+ Br->write(Bo);
s++;
}
}
@@ -312,11 +505,17 @@ main(int argc, char **argv) {
fclose(Bo);
} else {
- fprintf(stderr, "Extracting "F_U64" reads into %s\n",
- NUMOUTPUT, path1);
- for (; Ar.read(Ai); i++) {
- if ((i < NUMINPUT) && (save[i])) {
- Ar.write(Ao);
+
+ if (nPairsToOutput > 0)
+ fprintf(stderr, "Extracting "F_U64" reads into %s\n",
+ nPairsToOutput, path1);
+ else
+ fprintf(stderr, "Extracting "F_U64" bases of reads into %s\n",
+ nBasesToOutput, path1);
+
+ for (; Ar->read(Ai); i++) {
+ if ((i < totPairsInInput) && (sav[i])) {
+ Ar->write(Ao);
s++;
}
}
@@ -325,17 +524,20 @@ main(int argc, char **argv) {
fclose(Ao);
}
- if (i > NUMINPUT) {
+ delete Ar;
+ delete Br;
+
+ if (i > totPairsInInput) {
fprintf(stderr, "WARNING: There are "F_U64" %s in the input; you claimed there are "F_U64" (-t option) %s.\n",
- i, (isMated) ? "mates" : "reads",
- NUMINPUT, (isMated) ? "mates" : "reads");
+ i, (isMated) ? "mates" : "reads",
+ totPairsInInput, (isMated) ? "mates" : "reads");
fprintf(stderr, "WARNING: Result is not a random sample of the input file.\n");
}
- if (i < NUMINPUT) {
+ if (i < totPairsInInput) {
fprintf(stderr, "WARNING: There are "F_U64" %s in the input; you claimed there are "F_U64" (-t option) %s.\n",
- i, (isMated) ? "mates" : "reads",
- NUMINPUT, (isMated) ? "mates" : "reads");
+ i, (isMated) ? "mates" : "reads",
+ totPairsInInput, (isMated) ? "mates" : "reads");
fprintf(stderr, "WARNING: Result is only %f X coverage.\n", (double)s * READLENGTH / GENOMESIZE);
}
diff --git a/src/AS_GKP/fastqSimulate-checkCoverage.pl b/src/AS_GKP/fastqSimulate-checkCoverage.pl
new file mode 100644
index 0000000..28fea6f
--- /dev/null
+++ b/src/AS_GKP/fastqSimulate-checkCoverage.pl
@@ -0,0 +1,104 @@
+#!/usr/bin/perl
+
+use strict;
+
+system("leaff -G 1 2000 2000 > base.fasta");
+system("leaff -H -G 1 500 500 | tr ACGT NNNN >> base.fasta");
+system("leaff -H -G 1 2000 2000 >> base.fasta");
+
+my (@rcov1, @ccov1);
+my (@rcov2, @ccov2);
+my (@rcov3, @ccov3);
+
+if (! -e "cov") {
+ my $max = -s "base.fasta";
+ my $maxiter = 100;
+
+ for (my $iter=0; $iter<$maxiter; $iter++) {
+ system("fastqSimulate -em 0 -ei 0 -ed 0 -f base.fasta -o t1 -l 100 -x 100 -pe 1000 0 > /dev/null 2>&1");
+ system("fastqSimulate -em 0 -ei 0 -ed 0 -f base.fasta -o t2 -l 100 -x 100 -allowgaps -pe 1000 0 > /dev/null 2>&1");
+ system("fastqSimulate -em 0 -ei 0 -ed 0 -f base.fasta -o t3 -l 100 -x 100 -allowgaps -allowns -pe 1000 0 > /dev/null 2>&1");
+
+ system("rm *.1.fastq *.2.fastq *.c.fastq");
+
+ print STDERR "Summarizing for iter $iter\n";
+
+ open(IN, "< t1.i.fastq"); summarize(\@rcov1, \@ccov1); close(IN);
+ open(IN, "< t2.i.fastq"); summarize(\@rcov2, \@ccov2); close(IN);
+ open(IN, "< t3.i.fastq"); summarize(\@rcov3, \@ccov3); close(IN);
+ }
+
+
+ open(COV, "> cov");
+
+ for (my $x=0; $x<$max; $x++) {
+ $rcov1[$x] += 0; $rcov1[$x] /= $maxiter;
+ $ccov1[$x] += 0; $ccov1[$x] /= $maxiter;
+
+ $rcov2[$x] += 0; $rcov2[$x] /= $maxiter;
+ $ccov2[$x] += 0; $ccov2[$x] /= $maxiter;
+
+ $rcov3[$x] += 0; $rcov3[$x] /= $maxiter;
+ $ccov3[$x] += 0; $ccov3[$x] /= $maxiter;
+
+ print COV "$x\t$rcov1[$x]\t$ccov1[$x]\t$rcov2[$x]\t$ccov2[$x]\t$rcov3[$x]\t$ccov3[$x]\n";
+ }
+
+ close(COV);
+}
+
+
+open(GP, "> cov.gp");
+print GP "set terminal png size 1280,800\n";
+print GP "set output 'cov.png'\n";
+print GP "plot 'cov' using 1:2 with lines lt 1 lw 1 title 'read cov, no Ns', \\\n";
+print GP " 'cov' using 1:4 with lines lt 2 lw 1 title 'read cov, clone span Ns', \\\n";
+print GP " 'cov' using 1:6 with lines lt 3 lw 1 title 'read cov, both span Ns', \\\n";
+print GP " 'cov' using 1:3 with lines lt 1 lw 2 title 'clone cov, no Ns', \\\n";
+print GP " 'cov' using 1:5 with lines lt 2 lw 2 title 'clone cov, clone span Ns', \\\n";
+print GP " 'cov' using 1:7 with lines lt 3 lw 2 title 'clone cov, both span Ns'\n";
+close(GP);
+
+system("gnuplot cov.gp");
+
+
+
+
+
+
+sub summarize(@@) {
+ my ($rcov, $ccov) = @_;
+
+ while (!eof(IN)) {
+ my $a = <IN>; chomp $a;
+ my $b = <IN>; chomp $b; my $l = length($b);
+ my $c = <IN>;
+ my $d = <IN>;
+
+ my $bgn;
+ my $end;
+ my $spn;
+
+ if ($a =~ m/^\@PE_\d+_\d+\@(\d+)-(\d+)\/1/) {
+ $bgn = $1;
+ $end = $1 + $l;
+ $spn = $2;
+ }
+
+ if ($a =~ m/^\@PE_\d+_\d+\@(\d+)-(\d+)\/2/) {
+ $bgn = $2 - $l;
+ $end = $2;
+ $spn = $bgn; # No clone span for the second read
+ }
+
+ die if (!defined($bgn));
+
+ for (my $x=$bgn; $x<$end; $x++) {
+ ${$rcov}[$x]++;
+ }
+
+ for (my $x=$bgn; $x<$spn; $x++) {
+ ${$ccov}[$x]++;
+ }
+ }
+}
diff --git a/src/AS_GKP/fastqSimulate.C b/src/AS_GKP/fastqSimulate.C
index 2c0609c..716ff53 100644
--- a/src/AS_GKP/fastqSimulate.C
+++ b/src/AS_GKP/fastqSimulate.C
@@ -17,7 +17,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: fastqSimulate.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: fastqSimulate.C 4555 2014-08-06 15:04:46Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -43,6 +43,7 @@ double readInsertRate = 0.01; // Fraction insertion error
double readDeleteRate = 0.01; // Fraction deletion error
bool allowGaps = false;
+bool allowNs = false;
const uint32 mpJunctionsNone = 0;
const uint32 mpJunctionsNormal = 1;
@@ -166,10 +167,14 @@ makeSequences(char *frag,
makeSequenceError(s1, q1, p);
+ if (s1[p] == '*') {
+ fwrite(frag, sizeof(char), fragLen, stdout);
+ fprintf(stdout, "\n");
+ }
assert(s1[p] != '*');
if (s1[p] == 0) {
- fprintf(stderr, "BREAK\n");
+ //fprintf(stderr, "BREAK\n");
break;
}
}
@@ -192,6 +197,10 @@ makeSequences(char *frag,
makeSequenceError(s2, q2, p);
+ if (s2[p] == '*') {
+ fwrite(frag, sizeof(char), fragLen, stdout);
+ fprintf(stdout, "\n");
+ }
assert(s2[p] != '*');
}
@@ -237,21 +246,26 @@ makeSE(char *seq,
// Make sure the read doesn't contain N's (redundant in this particular case)
- for (int32 i=0; i<readLen; i++)
- if (s1[i] == 'N')
- goto trySEagain;
+ if (allowNs == false)
+ for (int32 i=0; i<readLen; i++)
+ if (s1[i] == 'N')
+ goto trySEagain;
// Reverse complement?
if (drand48() < pRevComp)
reverseComplement(s1, q1, readLen);
- // Output sequence, with a descriptive ID
+ // Output sequence, with a descriptive ID. Because bowtie2 removes /1 and /2 when the
+ // mate maps concordantly, we no longer use that form.
- fprintf(outputI, "@SE_%d_%d@%d-%d/1\n", nr, idx, bgn-zer, bgn+len-zer);
+ fprintf(outputI, "@SE_%d_%d@%d-%d#1\n", nr, idx, bgn-zer, bgn+len-zer);
fprintf(outputI, "%s\n", s1);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q1);
+
+ //if ((nr % 1000) == 0)
+ // fprintf(stderr, "%9d / %9d - %5.2f%%\r", nr, numReads, 100.0 * nr / numReads);
}
delete [] s1;
@@ -301,28 +315,30 @@ makePE(char *seq,
// Make sure the reads don't contain N's
- for (int32 i=0; i<readLen; i++)
- if ((s1[i] == 'N') || (s2[i] == 'N'))
- goto tryPEagain;
+ if (allowNs == false)
+ for (int32 i=0; i<readLen; i++)
+ if ((s1[i] == 'N') || (s2[i] == 'N'))
+ goto tryPEagain;
- // Output sequences, with a descriptive ID
+ // Output sequences, with a descriptive ID. Because bowtie2 removes /1 and /2 when the
+ // mate maps concordantly, we no longer use that form.
- fprintf(outputI, "@PE%s_%d_%d@%d-%d/1\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
+ fprintf(outputI, "@PE%s_%d_%d@%d-%d#1\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
fprintf(outputI, "%s\n", s1);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q1);
- fprintf(outputI, "@PE%s_%d_%d@%d-%d/2\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
+ fprintf(outputI, "@PE%s_%d_%d@%d-%d#2\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
fprintf(outputI, "%s\n", s2);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q2);
- fprintf(output1, "@PE%s_%d_%d@%d-%d/1\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
+ fprintf(output1, "@PE%s_%d_%d@%d-%d#1\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
fprintf(output1, "%s\n", s1);
fprintf(output1, "+\n");
fprintf(output1, "%s\n", q1);
- fprintf(output2, "@PE%s_%d_%d@%d-%d/2\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
+ fprintf(output2, "@PE%s_%d_%d@%d-%d#2\n", (makeNormal) ? "normal" : "", np, idx, bgn-zer, bgn+len-zer);
fprintf(output2, "%s\n", s2);
fprintf(output2, "+\n");
fprintf(output2, "%s\n", q2);
@@ -330,15 +346,18 @@ makePE(char *seq,
reverseComplement(s1, q1, readLen);
reverseComplement(s2, q2, readLen);
- fprintf(outputC, "@PE%s_%d_%d@%d-%d/1\n", (makeNormal) ? "normal" : "", np, idx, bgn+len-zer, bgn-zer);
+ fprintf(outputC, "@PE%s_%d_%d@%d-%d#1\n", (makeNormal) ? "normal" : "", np, idx, bgn+len-zer, bgn-zer);
fprintf(outputC, "%s\n", s1);
fprintf(outputC, "+\n");
fprintf(outputC, "%s\n", q1);
- fprintf(outputC, "@PE%s_%d_%d@%d-%d/2\n", (makeNormal) ? "normal" : "", np, idx, bgn+len-zer, bgn-zer);
+ fprintf(outputC, "@PE%s_%d_%d@%d-%d#2\n", (makeNormal) ? "normal" : "", np, idx, bgn+len-zer, bgn-zer);
fprintf(outputC, "%s\n", s2);
fprintf(outputC, "+\n");
fprintf(outputC, "%s\n", q2);
+
+ //if ((np % 1000) == 0)
+ // fprintf(stderr, "%9d / %9d - %5.2f%%\r", np, numPairs, 100.0 * np / numPairs);
}
delete [] s1;
@@ -407,28 +426,30 @@ makeMP(char *seq,
// Make sure the reads don't contain N's
- for (int32 i=0; i<readLen; i++)
- if ((s1[i] == 'N') || (s2[i] == 'N'))
- goto tryMPagain;
+ if (allowNs == false)
+ for (int32 i=0; i<readLen; i++)
+ if ((s1[i] == 'N') || (s2[i] == 'N'))
+ goto tryMPagain;
- // Output sequences, with a descriptive ID
+ // Output sequences, with a descriptive ID. Because bowtie2 removes /1 and /2 when the
+ // mate maps concordantly, we no longer use that form.
- fprintf(outputI, "@fPE%s_%d_%d@%d-%d/1\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
+ fprintf(outputI, "@fPE%s_%d_%d@%d-%d#1\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
fprintf(outputI, "%s\n", s1);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q1);
- fprintf(outputI, "@fPE%s_%d_%d@%d-%d/2\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
+ fprintf(outputI, "@fPE%s_%d_%d@%d-%d#2\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
fprintf(outputI, "%s\n", s2);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q2);
- fprintf(output1, "@fPE%s_%d_%d@%d-%d/1\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
+ fprintf(output1, "@fPE%s_%d_%d@%d-%d#1\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
fprintf(output1, "%s\n", s1);
fprintf(output1, "+\n");
fprintf(output1, "%s\n", q1);
- fprintf(output2, "@fPE%s_%d_%d@%d-%d/2\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
+ fprintf(output2, "@fPE%s_%d_%d@%d-%d#2\n", (makeNormal) ? "normal" : "", np, idx, sbgn-zer, sbgn+slen-zer);
fprintf(output2, "%s\n", s2);
fprintf(output2, "+\n");
fprintf(output2, "%s\n", q2);
@@ -436,12 +457,12 @@ makeMP(char *seq,
reverseComplement(s1, q1, readLen);
reverseComplement(s2, q2, readLen);
- fprintf(outputC, "@fPE%s_%d_%d@%d-%d/1\n", (makeNormal) ? "normal" : "", np, idx, sbgn+slen-zer, sbgn-zer);
+ fprintf(outputC, "@fPE%s_%d_%d@%d-%d#1\n", (makeNormal) ? "normal" : "", np, idx, sbgn+slen-zer, sbgn-zer);
fprintf(outputC, "%s\n", s1);
fprintf(outputC, "+\n");
fprintf(outputC, "%s\n", q1);
- fprintf(outputC, "@fPE%s_%d_%d@%d-%d/2\n", (makeNormal) ? "normal" : "", np, idx, sbgn+slen-zer, sbgn-zer);
+ fprintf(outputC, "@fPE%s_%d_%d@%d-%d#2\n", (makeNormal) ? "normal" : "", np, idx, sbgn+slen-zer, sbgn-zer);
fprintf(outputC, "%s\n", s2);
fprintf(outputC, "+\n");
fprintf(outputC, "%s\n", q2);
@@ -514,9 +535,10 @@ makeMP(char *seq,
// Make sure the reads don't contain N's
- for (int32 i=0; i<readLen; i++)
- if ((s1[i] == 'N') || (s2[i] == 'N'))
- goto tryMPagain;
+ if (allowNs == false)
+ for (int32 i=0; i<readLen; i++)
+ if ((s1[i] == 'N') || (s2[i] == 'N'))
+ goto tryMPagain;
// Label the type of the read
@@ -547,22 +569,25 @@ makeMP(char *seq,
q2[shift - 0] = QV_BASE + 10;
}
- fprintf(outputI, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d/1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
+ // Output sequences, with a descriptive ID. Because bowtie2 removes /1 and /2 when the
+ // mate maps concordantly, we no longer use that form.
+
+ fprintf(outputI, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
fprintf(outputI, "%s\n", s1);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q1);
- fprintf(outputI, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d/2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
+ fprintf(outputI, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
fprintf(outputI, "%s\n", s2);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q2);
- fprintf(output1, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d/1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
+ fprintf(output1, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
fprintf(output1, "%s\n", s1);
fprintf(output1, "+\n");
fprintf(output1, "%s\n", q1);
- fprintf(output2, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d/2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
+ fprintf(output2, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift);
fprintf(output2, "%s\n", s2);
fprintf(output2, "+\n");
fprintf(output2, "%s\n", q2);
@@ -570,16 +595,19 @@ makeMP(char *seq,
reverseComplement(s1, q1, readLen);
reverseComplement(s2, q2, readLen);
- fprintf(outputC, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d/1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn+len, bgn, shift, slen, bgn+len-shift);
+ fprintf(outputC, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn+len, bgn, shift, slen, bgn+len-shift);
fprintf(outputC, "%s\n", s1);
fprintf(outputC, "+\n");
fprintf(outputC, "%s\n", q1);
- fprintf(outputC, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d/2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn+len, bgn, shift, slen, bgn+len-shift);
+ fprintf(outputC, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn+len, bgn, shift, slen, bgn+len-shift);
fprintf(outputC, "%s\n", s2);
fprintf(outputC, "+\n");
fprintf(outputC, "%s\n", q2);
}
+
+ //if ((np % 1000) == 0)
+ // fprintf(stderr, "%9d / %9d - %5.2f%%\r", np, numPairs, 100.0 * np / numPairs);
}
}
@@ -641,15 +669,17 @@ makeCC(char *seq,
if (idxf != idxr)
goto tryCCagain;
- for (int32 i=bgnf; i<bgnf+lenf; i++)
- if ((seq[i] == '>') ||
- ((allowGaps == false) && (seq[i] == 'N')))
- goto tryCCagain;
+ if (allowNs == false)
+ for (int32 i=bgnf; i<bgnf+lenf; i++)
+ if ((seq[i] == '>') ||
+ ((allowGaps == false) && (seq[i] == 'N')))
+ goto tryCCagain;
- for (int32 i=bgnr; i<bgnr+lenr; i++)
- if ((seq[i] == '>') ||
- ((allowGaps == false) && (seq[i] == 'N')))
- goto tryCCagain;
+ if (allowNs == false)
+ for (int32 i=bgnr; i<bgnr+lenr; i++)
+ if ((seq[i] == '>') ||
+ ((allowGaps == false) && (seq[i] == 'N')))
+ goto tryCCagain;
// Generate the sequence.
@@ -670,13 +700,15 @@ makeCC(char *seq,
// Make sure the read doesn't contain N's (redundant in this particular case)
- for (int32 i=0; i<readLen; i++)
- if (s1[i] == 'N')
- goto tryCCagain;
+ if (allowNs == false)
+ for (int32 i=0; i<readLen; i++)
+ if (s1[i] == 'N')
+ goto tryCCagain;
- // Output sequence, with a descriptive ID
+ // Output sequences, with a descriptive ID. Because bowtie2 removes /1 and /2 when the
+ // mate maps concordantly, we no longer use that form.
- fprintf(outputI, "@CC%c_%d_%d@%d-%d--%d@%d-%d/1\n",
+ fprintf(outputI, "@CC%c_%d_%d@%d-%d--%d@%d-%d#1\n",
(isFalse) ? 'f' : 't',
nr,
idxf, bgnf-zerf, bgnf+lenf-zerf,
@@ -684,6 +716,9 @@ makeCC(char *seq,
fprintf(outputI, "%s\n", s1);
fprintf(outputI, "+\n");
fprintf(outputI, "%s\n", q1);
+
+ //if ((nr % 1000) == 0)
+ // fprintf(stderr, "%9d / %9d - %5.2f%%\r", nr, numReads, 100.0 * nr / numReads);
}
delete [] s1;
@@ -696,14 +731,17 @@ main(int argc, char **argv) {
char *fastaName = NULL;
FILE *fastaFile = NULL;
+ int32 numSeq = 0;
+
int32 seqMax = 0;
int32 seqLen = 0;
char *seq = NULL;
- int32 readLen = 100; // Length of read to generate
- int32 numReads = 0; // Number of reads to generate, constant
- int32 numPairs = 0; // Number of pairs to generate, constant (= numReads / 2)
- double readCoverage = 0.0; // Number of pairs to generate, based on length of sequence
+ int32 readLen = 100; // Length of read to generate
+ int32 numReads = UINT32_MAX; // Number of reads to generate, constant
+ int32 numPairs = UINT32_MAX; // Number of pairs to generate, constant (= numReads / 2)
+ double readCoverage = 0.0; // Number of pairs to generate, based on length of read
+ double cloneCoverage = 0.0; // Number of pairs to generate, based on length of clone
bool seEnable = false;
@@ -750,6 +788,9 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-x") == 0) {
readCoverage = atof(argv[++arg]);
+ } else if (strcmp(argv[arg], "-X") == 0) {
+ cloneCoverage = atof(argv[++arg]);
+
} else if (strcmp(argv[arg], "-em") == 0) {
readMismatchRate = atof(argv[++arg]);
@@ -762,6 +803,10 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-allowgaps") == 0) {
allowGaps = true;
+ } else if (strcmp(argv[arg], "-allowns") == 0) {
+ allowGaps = true;
+ allowNs = true;
+
} else if (strcmp(argv[arg], "-nojunction") == 0) {
mpJunctions = mpJunctionsNone;
@@ -821,13 +866,15 @@ main(int argc, char **argv) {
((seEnable == false) &&
(peEnable == false) &&
(mpEnable == false) &&
- (ccEnable == false))) {
+ (ccEnable == false)) ||
+ ((seEnable == true) && (cloneCoverage > 0))) {
fprintf(stderr, "usage: %s -f reference.fasta -o output-prefix -l read-length ....\n", argv[0]);
fprintf(stderr, " -f ref.fasta Use sequences in ref.fasta as the genome.\n");
fprintf(stderr, " -o name Create outputs name.1.fastq and name.2.fastq (and maybe others).\n");
fprintf(stderr, " -l len Create reads of length 'len' bases.\n");
fprintf(stderr, " -n n Create 'n' reads (for -se) or 'n' pairs of reads (for -pe and -mp).\n");
- fprintf(stderr, " -x cov Set 'np' to create reads that sample the genome to 'cov' coverage.\n");
+ fprintf(stderr, " -x read-cov Set 'np' to create reads that sample the genome to 'read-cov' read coverage.\n");
+ fprintf(stderr, " -X clone-cov Set 'np' to create reads that sample the genome to 'clone-cov' clone coverage.\n");
fprintf(stderr, "\n");
fprintf(stderr, " -em err Reads will contain fraction mismatch error 'e' (0.01 == 1%% error).\n");
fprintf(stderr, " -ei err Reads will contain fraction insertion error 'e' (0.01 == 1%% error).\n");
@@ -836,6 +883,8 @@ main(int argc, char **argv) {
fprintf(stderr, " -allowgaps Allow pairs to span N regions in the reference. By default, pairs\n");
fprintf(stderr, " are not allowed to span a gap. Reads are never allowed to cover N's.\n");
fprintf(stderr, "\n");
+ fprintf(stderr, " -allowns Allow reads to contain N regions. Implies -allowgaps\n");
+ fprintf(stderr, "\n");
fprintf(stderr, " -nojunction For -mp, do not create chimeric junction reads. Create only fully PE or\n");
fprintf(stderr, " fully MP reads.\n");
fprintf(stderr, "\n");
@@ -876,6 +925,8 @@ main(int argc, char **argv) {
fprintf(stderr, "ERROR: No output prefix (-o) supplied.\n");
if ((seEnable == false) && (peEnable == false) && (mpEnable == false) && (ccEnable == false))
fprintf(stderr, "ERROR: No type (-se or -pe or -mp) selected.\n");
+ if ((seEnable == true) && (cloneCoverage > 0))
+ fprintf(stderr, "ERROR: Can't sample clone coverage with single-ended (-se) reads.\n");
exit(1);
}
@@ -977,17 +1028,20 @@ main(int argc, char **argv) {
if (errno)
fprintf(stderr, "Failed to open fasta file '%s': %s\n", fastaName, strerror(errno)), exit(1);
+ numSeq = 0;
seqMax = AS_UTL_sizeOfFile(fastaName);
seqLen = 0;
seq = new char [seqMax + 1];
memset(seq, '.', sizeof(char) * seqMax);
+ uint32 nInvalid = 0;
while (!feof(fastaFile)) {
fgets(seq + seqLen, seqMax - seqLen, fastaFile);
if (seq[seqLen] == '>') {
+ numSeq++;
seqLen++;
seqStartPositions.push_back(seqLen);
continue;
@@ -995,9 +1049,17 @@ main(int argc, char **argv) {
for (;
((seq[seqLen] != '\n') && (seq[seqLen] != '\r') && (seq[seqLen] != 0));
- seqLen++)
+ seqLen++) {
seq[seqLen] = toupper(seq[seqLen]);
+ if ((seq[seqLen] != 'N') && (validBase[seq[seqLen]] == 0)) {
+ nInvalid++;
+ //fprintf(stderr, "Replace invalid base '%c' at position %u.\n", seq[seqLen], seqLen);
+ seq[seqLen] = insertBase[randomUniform(0, 3)];
+ //q1[p] = (validBase[s1[p]]) ? QV_BASE + 8 : QV_BASE + 2;
+ }
+ }
+
assert(seqLen < seqMax);
}
@@ -1005,22 +1067,54 @@ main(int argc, char **argv) {
seq[seqLen] = 0;
- fprintf(stderr, "READ sequence of length %d\n", seqLen);
+ assert(numSeq == seqStartPositions.size());
+
+ fprintf(stderr, "Loaded %u sequences of length %d, with %u invalid bases fixed.\n",
+ numSeq, seqLen - numSeq, nInvalid);
//
// If requested, compute the number of pairs to get a desired X of coverage
//
- if (readCoverage > 0) {
- numReads = (int32)floor(readCoverage * seqLen / readLen);
- numPairs = numReads / 2;
+ {
+ uint32 cloneSize = 0;
+ uint32 cloneStdDev = 0;
+
+ uint32 readNumReads = UINT32_MAX;
+ uint32 readNumPairs = UINT32_MAX;
+
+ uint32 cloneNumReads = UINT32_MAX;
+ uint32 cloneNumPairs = UINT32_MAX;
+
+ if (peEnable) { cloneSize = peShearSize; cloneStdDev = peShearStdDev; }
+ if (mpEnable) { cloneSize = mpInsertSize; cloneStdDev = mpInsertStdDev; }
+ if (ccEnable) { cloneSize = ccJunkSize; cloneStdDev = ccJunkStdDev; }
+
+ if (readCoverage > 0) {
+ readNumReads = (uint32)floor(readCoverage * (seqLen - numSeq) / readLen);
+ readNumPairs = readNumReads / 2;
+ }
+
+ if ((cloneCoverage > 0) && (seEnable == false)) {
+ cloneNumPairs = (uint32)floor(cloneCoverage * (seqLen - numSeq) / cloneSize);
+ cloneNumReads = cloneNumPairs * 2;
+ }
+
+ numReads = MIN(numReads, readNumReads);
+ numPairs = MIN(numPairs, readNumPairs);
+
+ numReads = MIN(numReads, cloneNumReads);
+ numPairs = MIN(numPairs, cloneNumPairs);
if (seEnable)
- fprintf(stderr, "For %.2f X coverage of a %dbp genome, generate %d %dbp reads.\n",
- readCoverage, seqLen, numReads, readLen);
+ fprintf(stderr, "Generate %.2f X read coverage of a %dbp genome with %u %dbp reads.\n",
+ (double)numReads * readLen / seqLen,
+ seqLen - numSeq, numReads, readLen);
else
- fprintf(stderr, "For %.2f X coverage of a %dbp genome, generate %d pairs of %dbp reads.\n",
- readCoverage, seqLen, numPairs, readLen);
+ fprintf(stderr, "Generate %.2f X read (%.2f X clone) coverage of a %dbp genome with %u pairs of %dbp reads from a clone of %d +- %dbp.\n",
+ (double)numReads * readLen / seqLen,
+ (double)numPairs * cloneSize / seqLen,
+ seqLen - numSeq, numPairs, readLen, cloneSize, cloneStdDev);
}
//
@@ -1058,10 +1152,12 @@ main(int argc, char **argv) {
delete [] seq;
-fprintf(stderr, "nNoChange = "F_U64"\n", nNoChange);
-fprintf(stderr, "nMismatch = "F_U64"\n", nMismatch);
-fprintf(stderr, "nInsert = "F_U64"\n", nInsert);
-fprintf(stderr, "nDelete = "F_U64"\n", nDelete);
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Number of reads with:\n");
+ fprintf(stderr, " nNoChange = "F_U64"\n", nNoChange);
+ fprintf(stderr, " nMismatch = "F_U64"\n", nMismatch);
+ fprintf(stderr, " nInsert = "F_U64"\n", nInsert);
+ fprintf(stderr, " nDelete = "F_U64"\n", nDelete);
exit(0);
}
diff --git a/src/AS_GKP/gkpStoreCreate.C b/src/AS_GKP/gkpStoreCreate.C
index 608e8c9..673b860 100644
--- a/src/AS_GKP/gkpStoreCreate.C
+++ b/src/AS_GKP/gkpStoreCreate.C
@@ -19,11 +19,11 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: gkpStoreCreate.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: gkpStoreCreate.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
diff --git a/src/AS_GKP/gkpStoreDumpFASTQ.C b/src/AS_GKP/gkpStoreDumpFASTQ.C
index b3296a9..f864a42 100644
--- a/src/AS_GKP/gkpStoreDumpFASTQ.C
+++ b/src/AS_GKP/gkpStoreDumpFASTQ.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: gkpStoreDumpFASTQ.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: gkpStoreDumpFASTQ.C 4595 2015-02-03 02:52:44Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
@@ -31,7 +31,15 @@ const char *mainid = "$Id: gkpStoreDumpFASTQ.C 4371 2013-08-01 17:19:47Z brianwa
class libInfo {
public:
- libInfo(char *outPrefix, char *libName) {
+ libInfo(char *outPrefix, char *libNameIn) {
+ char libName[FILENAME_MAX];
+
+ strcpy(libName, libNameIn);
+
+ for (uint32 ii=0; libName[ii]; ii++)
+ if (libName[ii] == '/')
+ libName[ii] = '_';
+
sprintf(aname, "%s.%s.1.fastq", outPrefix, libName);
sprintf(bname, "%s.%s.2.fastq", outPrefix, libName);
sprintf(pname, "%s.%s.paired.fastq", outPrefix, libName);
@@ -149,11 +157,11 @@ main(int argc, char **argv) {
AS_IID numFrags = gkp->gkStore_getNumFragments();
AS_IID numLibs = gkp->gkStore_getNumLibraries();
- libInfo **lib = new libInfo * [numLibs];
+ libInfo **lib = new libInfo * [numLibs+1];
lib[0] = new libInfo(outPrefix, "legacy");
- for (uint32 i=1; i<numLibs; i++)
+ for (uint32 i=1; i < numLibs+1; i++)
lib[i] = new libInfo(outPrefix, gkp->gkStore_getLibrary(i)->libraryName);
if (bgnIID < 1)
diff --git a/src/AS_GKP/sffToCA.C b/src/AS_GKP/sffToCA.C
index 7b97594..370b8ee 100644
--- a/src/AS_GKP/sffToCA.C
+++ b/src/AS_GKP/sffToCA.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: sffToCA.C 4483 2013-12-12 18:48:41Z brianwalenz $";
+const char *mainid = "$Id: sffToCA.C 4513 2014-03-04 18:22:00Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -709,6 +709,21 @@ processRead(sffHeader *h,
int clf = MAX(MAX(clq, cla), MAX(cln, clp));
int crf = MIN(MIN(crq, cra), MIN(crn, crp));
+ if (clf >= crf) {
+ st.lenTooShort++;
+ st.deletedTooShort++;
+ st.notExaminedForLinker++; // because this SFF read isn't even added to the store
+
+ // Read 'FJTLYIM02PFKBO' of length 651 has bad clear ranges. clf=MAX(5,5,4,4) crf=MIN(4,655,655,655) Read deleted.
+
+ fprintf(logFile, "Read '%s' of length %d has bad clear ranges. clf=MAX(%d,%d,%d,%d) crf=MIN(%d,%d,%d,%d) Read deleted.\n",
+ r->name,
+ r->number_of_bases - h->key_length,
+ clq, cla, cln, clp,
+ crq, cra, crn, crp);
+
+ return(false);
+ }
////////////////////////////////////////
//
diff --git a/src/AS_LIN/._Makefile b/src/AS_LIN/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._Makefile differ
diff --git a/src/AS_LIN/._daxpy.c b/src/AS_LIN/._daxpy.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._daxpy.c differ
diff --git a/src/AS_LIN/._dcopy.c b/src/AS_LIN/._dcopy.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dcopy.c differ
diff --git a/src/AS_LIN/._ddot.c b/src/AS_LIN/._ddot.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._ddot.c differ
diff --git a/src/AS_LIN/._dgbtf2.c b/src/AS_LIN/._dgbtf2.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dgbtf2.c differ
diff --git a/src/AS_LIN/._dgbtrf.c b/src/AS_LIN/._dgbtrf.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dgbtrf.c differ
diff --git a/src/AS_LIN/._dgbtrs.c b/src/AS_LIN/._dgbtrs.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dgbtrs.c differ
diff --git a/src/AS_LIN/._dgemm.c b/src/AS_LIN/._dgemm.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dgemm.c differ
diff --git a/src/AS_LIN/._dgemv.c b/src/AS_LIN/._dgemv.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dgemv.c differ
diff --git a/src/AS_LIN/._dger.c b/src/AS_LIN/._dger.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dger.c differ
diff --git a/src/AS_LIN/._disnan.c b/src/AS_LIN/._disnan.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._disnan.c differ
diff --git a/src/AS_LIN/._dlaisnan.c b/src/AS_LIN/._dlaisnan.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dlaisnan.c differ
diff --git a/src/AS_LIN/._dlamch.c b/src/AS_LIN/._dlamch.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dlamch.c differ
diff --git a/src/AS_LIN/._dlaswp.c b/src/AS_LIN/._dlaswp.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dlaswp.c differ
diff --git a/src/AS_LIN/._dpbtf2.c b/src/AS_LIN/._dpbtf2.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dpbtf2.c differ
diff --git a/src/AS_LIN/._dpbtrf.c b/src/AS_LIN/._dpbtrf.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dpbtrf.c differ
diff --git a/src/AS_LIN/._dpbtrs.c b/src/AS_LIN/._dpbtrs.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dpbtrs.c differ
diff --git a/src/AS_LIN/._dpotf2.c b/src/AS_LIN/._dpotf2.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dpotf2.c differ
diff --git a/src/AS_LIN/._dptrfs.c b/src/AS_LIN/._dptrfs.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dptrfs.c differ
diff --git a/src/AS_LIN/._dpttrs.c b/src/AS_LIN/._dpttrs.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dpttrs.c differ
diff --git a/src/AS_LIN/._dscal.c b/src/AS_LIN/._dscal.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dscal.c differ
diff --git a/src/AS_LIN/._dswap.c b/src/AS_LIN/._dswap.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dswap.c differ
diff --git a/src/AS_LIN/._dsyr.c b/src/AS_LIN/._dsyr.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dsyr.c differ
diff --git a/src/AS_LIN/._dsyrk.c b/src/AS_LIN/._dsyrk.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dsyrk.c differ
diff --git a/src/AS_LIN/._dtbsv.c b/src/AS_LIN/._dtbsv.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dtbsv.c differ
diff --git a/src/AS_LIN/._dtrsm.c b/src/AS_LIN/._dtrsm.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._dtrsm.c differ
diff --git a/src/AS_LIN/._f2c.h b/src/AS_LIN/._f2c.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._f2c.h differ
diff --git a/src/AS_LIN/._idamax.c b/src/AS_LIN/._idamax.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._idamax.c differ
diff --git a/src/AS_LIN/._ieeeck.c b/src/AS_LIN/._ieeeck.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._ieeeck.c differ
diff --git a/src/AS_LIN/._ilaenv.c b/src/AS_LIN/._ilaenv.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._ilaenv.c differ
diff --git a/src/AS_LIN/._iparmq.c b/src/AS_LIN/._iparmq.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._iparmq.c differ
diff --git a/src/AS_LIN/._lsame.c b/src/AS_LIN/._lsame.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._lsame.c differ
diff --git a/src/AS_LIN/._s_cmp.c b/src/AS_LIN/._s_cmp.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._s_cmp.c differ
diff --git a/src/AS_LIN/._s_copy.c b/src/AS_LIN/._s_copy.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._s_copy.c differ
diff --git a/src/AS_LIN/._xerbla.c b/src/AS_LIN/._xerbla.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_LIN/._xerbla.c differ
diff --git a/src/AS_MER/._AS_MER_gkpStoreChain.C b/src/AS_MER/._AS_MER_gkpStoreChain.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._AS_MER_gkpStoreChain.C differ
diff --git a/src/AS_MER/._AS_MER_gkpStoreChain.H b/src/AS_MER/._AS_MER_gkpStoreChain.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._AS_MER_gkpStoreChain.H differ
diff --git a/src/AS_MER/._AS_MER_gkpStore_to_FastABase.C b/src/AS_MER/._AS_MER_gkpStore_to_FastABase.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._AS_MER_gkpStore_to_FastABase.C differ
diff --git a/src/AS_MER/._AS_MER_gkpStore_to_FastABase.H b/src/AS_MER/._AS_MER_gkpStore_to_FastABase.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._AS_MER_gkpStore_to_FastABase.H differ
diff --git a/src/AS_MER/._AS_MER_meryl.cc b/src/AS_MER/._AS_MER_meryl.cc
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._AS_MER_meryl.cc differ
diff --git a/src/AS_MER/._Makefile b/src/AS_MER/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._Makefile differ
diff --git a/src/AS_MER/._estimate-mer-threshold.C b/src/AS_MER/._estimate-mer-threshold.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._estimate-mer-threshold.C differ
diff --git a/src/AS_MER/._gkrpt.pl b/src/AS_MER/._gkrpt.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._gkrpt.pl differ
diff --git a/src/AS_MER/._merTrim-compare-logs.pl b/src/AS_MER/._merTrim-compare-logs.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._merTrim-compare-logs.pl differ
diff --git a/src/AS_MER/._merTrim.C b/src/AS_MER/._merTrim.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._merTrim.C differ
diff --git a/src/AS_MER/._merTrimAdapter.C b/src/AS_MER/._merTrimAdapter.C
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._merTrimAdapter.C differ
diff --git a/src/AS_MER/._merTrimApply.C b/src/AS_MER/._merTrimApply.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._merTrimApply.C differ
diff --git a/src/AS_MER/._merTrimResult.H b/src/AS_MER/._merTrimResult.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._merTrimResult.H differ
diff --git a/src/AS_MER/._mercy-regions.C b/src/AS_MER/._mercy-regions.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._mercy-regions.C differ
diff --git a/src/AS_MER/._mercy.C b/src/AS_MER/._mercy.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._mercy.C differ
diff --git a/src/AS_MER/._mercy.sh b/src/AS_MER/._mercy.sh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._mercy.sh differ
diff --git a/src/AS_MER/._meryl.C b/src/AS_MER/._meryl.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._meryl.C differ
diff --git a/src/AS_MER/._overmerry.C b/src/AS_MER/._overmerry.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MER/._overmerry.C differ
diff --git a/src/AS_MER/AS_MER_gkpStoreChain.C b/src/AS_MER/AS_MER_gkpStoreChain.C
index da5ce14..0a4ffaf 100644
--- a/src/AS_MER/AS_MER_gkpStoreChain.C
+++ b/src/AS_MER/AS_MER_gkpStoreChain.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_MER_gkpStoreChain.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_MER_gkpStoreChain.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include "AS_MER_gkpStoreChain.H"
@@ -38,9 +38,9 @@ gkpStoreChain::gkpStoreChain(char const *gkpName,
_numberOfSequences = 0;
- _chainIID = ~u32bitZERO;
- _chainLen = new u32bit [_maxChains];
- _chainBgnFrg = new u32bit [_maxChains];
+ _chainIID = ~uint32ZERO;
+ _chainLen = new uint32 [_maxChains];
+ _chainBgnFrg = new uint32 [_maxChains];
for (uint32 i=0; i<_maxChains; i++) {
_chainLen[i] = 0;
@@ -53,11 +53,11 @@ gkpStoreChain::gkpStoreChain(char const *gkpName,
while (stm->next(&_frg)) {
if (_frg.gkFragment_getIsDeleted() == false) {
- u32bit iid = _frg.gkFragment_getReadIID();
+ uint32 iid = _frg.gkFragment_getReadIID();
#ifdef FULLREAD
- u32bit len = _frg.gkFragment_getSequenceLength();
+ uint32 len = _frg.gkFragment_getSequenceLength();
#else
- u32bit len = _frg.gkFragment_getClearRegionLength();
+ uint32 len = _frg.gkFragment_getClearRegionLength();
#endif
if (_chainLen[_numberOfSequences] + len > _maxChainLen) {
@@ -130,9 +130,9 @@ gkpStoreChain::openFile(const char *name) {
bool
-gkpStoreChain::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
+gkpStoreChain::getSequence(uint32 iid,
+ char *&h, uint32 &hLen, uint32 &hMax,
+ char *&s, uint32 &sLen, uint32 &sMax) {
if (_chainLen[iid] == 0)
return(false);
@@ -148,7 +148,7 @@ gkpStoreChain::getSequence(u32bit iid,
h = new char [hMax];
}
- sprintf(h, "iid:"u32bitFMT"-"u32bitFMT, _chainBgnFrg[iid], _chainBgnFrg[iid+1]-1);
+ sprintf(h, "iid:"F_U32"-"F_U32, _chainBgnFrg[iid], _chainBgnFrg[iid+1]-1);
hLen = strlen(h);
sLen = 0;
@@ -175,8 +175,8 @@ gkpStoreChain::getSequence(u32bit iid,
bool
-gkpStoreChain::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
+gkpStoreChain::getSequence(uint32 iid,
+ uint32 bgn, uint32 end, char *s) {
char *t = s;
@@ -189,11 +189,11 @@ gkpStoreChain::getSequence(u32bit iid,
if ((_chainIID != iid) || (_frgLengths == 0L)) {
delete [] _frgLengths;
- _frgLengths = new u32bit [_chainBgnFrg[iid+1] - _chainBgnFrg[iid]];
+ _frgLengths = new uint32 [_chainBgnFrg[iid+1] - _chainBgnFrg[iid]];
_chainIID = iid;
- for (u32bit i=0; i<_chainBgnFrg[iid+1] - _chainBgnFrg[iid]; i++)
- _frgLengths[i] = ~u32bitZERO;
+ for (uint32 i=0; i<_chainBgnFrg[iid+1] - _chainBgnFrg[iid]; i++)
+ _frgLengths[i] = ~uint32ZERO;
_lastFrg = _chainBgnFrg[iid];
_lastPos = 0;
@@ -217,7 +217,7 @@ gkpStoreChain::getSequence(u32bit iid,
// Skip fragments that come before bgn
//
while ((frg < _chainBgnFrg[iid+1]) &&
- (_frgLengths[frg-bas] != ~u32bitZERO) &&
+ (_frgLengths[frg-bas] != ~uint32ZERO) &&
(pos + _frgLengths[frg-bas] < bgn)) {
pos += _frgLengths[frg-bas];
frg++;
@@ -234,11 +234,11 @@ gkpStoreChain::getSequence(u32bit iid,
_gkp->gkStore_getFragment(frg, &_frg, GKFRAGMENT_SEQ);
#ifdef FULLREAD
- u32bit clr = 0;
- u32bit len = _frg.gkFragment_getSequenceLength();
+ uint32 clr = 0;
+ uint32 len = _frg.gkFragment_getSequenceLength();
#else
- u32bit clr = _frg.gkFragment_getClearRegionBegin();
- u32bit len = _frg.gkFragment_getClearRegionLength();
+ uint32 clr = _frg.gkFragment_getClearRegionBegin();
+ uint32 len = _frg.gkFragment_getClearRegionLength();
#endif
_frgLengths[frg-bas] = len + 1;
diff --git a/src/AS_MER/AS_MER_gkpStoreChain.H b/src/AS_MER/AS_MER_gkpStoreChain.H
index 658a1ac..b85197d 100644
--- a/src/AS_MER/AS_MER_gkpStoreChain.H
+++ b/src/AS_MER/AS_MER_gkpStoreChain.H
@@ -53,21 +53,21 @@ protected:
seqFile *openFile(const char *filename);
public:
- u32bit find(const char *sequencename) {
+ uint32 find(const char *sequencename) {
fprintf(stderr, "gkpStoreChain::find()-- Not implemented.\n");
exit(1);
- return(~u32bitZERO);
+ return(~uint32ZERO);
};
- u32bit getSequenceLength(u32bit iid) {
+ uint32 getSequenceLength(uint32 iid) {
return(_chainLen[iid]);
};
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
+ bool getSequence(uint32 iid,
+ char *&h, uint32 &hLen, uint32 &hMax,
+ char *&s, uint32 &sLen, uint32 &sMax);
+ bool getSequence(uint32 iid,
+ uint32 bgn, uint32 end, char *s);
private:
void clear(void);
@@ -75,17 +75,17 @@ private:
gkStore *_gkp;
gkFragment _frg;
- u32bit _maxChains;
- u32bit _maxChainLen;
+ uint32 _maxChains;
+ uint32 _maxChainLen;
- u32bit _chainIID;
- u32bit *_chainLen;
- u32bit *_chainBgnFrg;
+ uint32 _chainIID;
+ uint32 *_chainLen;
+ uint32 *_chainBgnFrg;
- u32bit *_frgLengths;
+ uint32 *_frgLengths;
- u32bit _lastFrg;
- u32bit _lastPos;
+ uint32 _lastFrg;
+ uint32 _lastPos;
};
diff --git a/src/AS_MER/AS_MER_gkpStore_to_FastABase.C b/src/AS_MER/AS_MER_gkpStore_to_FastABase.C
index 3f452a4..be175e7 100644
--- a/src/AS_MER/AS_MER_gkpStore_to_FastABase.C
+++ b/src/AS_MER/AS_MER_gkpStore_to_FastABase.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_MER_gkpStore_to_FastABase.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_MER_gkpStore_to_FastABase.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include "AS_MER_gkpStore_to_FastABase.H"
@@ -126,9 +126,9 @@ gkpStoreFile::openFile(const char *name) {
bool
-gkpStoreFile::getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax) {
+gkpStoreFile::getSequence(uint32 iid,
+ char *&h, uint32 &hLen, uint32 &hMax,
+ char *&s, uint32 &sLen, uint32 &sMax) {
if (sMax == 0) {
sMax = AS_READ_MAX_NORMAL_LEN+1;
@@ -171,8 +171,8 @@ gkpStoreFile::getSequence(u32bit iid,
bool
-gkpStoreFile::getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s) {
+gkpStoreFile::getSequence(uint32 iid,
+ uint32 bgn, uint32 end, char *s) {
if (iid == 0)
fprintf(stderr, "gkpStoreFile::getSequence(part)-- someone requested iid==0?\n"), exit(1);
diff --git a/src/AS_MER/AS_MER_gkpStore_to_FastABase.H b/src/AS_MER/AS_MER_gkpStore_to_FastABase.H
index 1fb77d0..16e327a 100644
--- a/src/AS_MER/AS_MER_gkpStore_to_FastABase.H
+++ b/src/AS_MER/AS_MER_gkpStore_to_FastABase.H
@@ -67,24 +67,24 @@ protected:
seqFile *openFile(const char *filename);
public:
- u32bit find(const char *sequencename) {
+ uint32 find(const char *sequencename) {
fprintf(stderr, "gkpStoreFile::find()-- Not implemented.\n");
exit(1);
- return(~u32bitZERO);
+ return(~uint32ZERO);
};
- u32bit getSequenceLength(u32bit iid) {
+ uint32 getSequenceLength(uint32 iid) {
return(_clrEnd[iid] - _clrBeg[iid]);
};
- bool getSequence(u32bit iid,
- char *&h, u32bit &hLen, u32bit &hMax,
- char *&s, u32bit &sLen, u32bit &sMax);
- bool getSequence(u32bit iid,
- u32bit bgn, u32bit end, char *s);
+ bool getSequence(uint32 iid,
+ char *&h, uint32 &hLen, uint32 &hMax,
+ char *&s, uint32 &sLen, uint32 &sMax);
+ bool getSequence(uint32 iid,
+ uint32 bgn, uint32 end, char *s);
- uint16 clrBeg(u32bit iid) { return(_clrBeg[iid]); };
- uint16 clrEnd(u32bit iid) { return(_clrEnd[iid]); };
+ uint16 clrBeg(uint32 iid) { return(_clrBeg[iid]); };
+ uint16 clrEnd(uint32 iid) { return(_clrEnd[iid]); };
private:
void clear(void);
diff --git a/src/AS_MER/Makefile b/src/AS_MER/Makefile
index 90732a9..2522a51 100644
--- a/src/AS_MER/Makefile
+++ b/src/AS_MER/Makefile
@@ -24,86 +24,41 @@
LOCAL_WORK = $(shell cd ../..; pwd)
-# KMER should point to a configured, compiled and installed copy of
-# http://sourceforge.net/projects/kmer/
-#
-# We attempt to find it in the same place that our src/ directory is.
-
-
-# We'd like to include the global include (c_make.as) to get these
-# definitions, but we cannot without defining rules.
+MERYLSOURCE = meryl.C
+MERYLOBJECTS = meryl.o AS_MER_gkpStore_to_FastABase.o AS_MER_gkpStoreChain.o
+SOURCES = $(MERYLSOURCE) mercy.C mercy-regions.C overmerry.C merTrim.C merTrimAdapter.C merTrimApply.C estimate-mer-threshold.C AS_MER_gkpStore_to_FastABase.C AS_MER_gkpStoreChain.C
+OBJECTS = $(SOURCES:.C=.o)
+CXX_PROGS = meryl mercy overmerry merTrim merTrimApply estimate-mer-threshold
+INC_IMPORT_DIRS += $(KMER)/include
+LIB_IMPORT_DIRS += $(KMER)/lib
+KMERLIBS = libmerylguts.a libkmer.a libmeryl.a libseq.a libbio.a libutil.a
-OSTYPE = $(shell echo `uname`)
-MACHINETYPE = $(shell echo `uname -m`)
-
-ifeq ($(MACHINETYPE), x86_64)
- MACHINETYPE = amd64
-endif
-ifeq ($(MACHINETYPE), Power Macintosh)
- MACHINETYPE = ppc
-endif
-
-
-# This isn't perfect; if we're building debug here, we _usually_ want
-# to use the debug kmer. But checking if kmer exists, and failing if
-# we don't find the debug kmer, is a little tricky.
+# include again to get everything else updated
+include $(LOCAL_WORK)/src/c_make.as
-ifneq "$(origin KMER)" "environment"
-ifeq ($(shell ls -d $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug 2> /dev/null), $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug)
- KMER = $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug
+# This must come after c_make.as, grrr.
+#
+# See also AS_OVL/Makefile
+ifeq ($(OSTYPE), Linux)
+ LDFLAGS += -lpthread
+ CFLAGS += -pthread
+ CPPFLAGS += -pthread
endif
-ifeq ($(shell ls -d $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE) 2> /dev/null), $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE))
- KMER = $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)
+ifeq ($(OSTYPE), OSF1)
+ LDFLAGS += -lpthread
+ CFLAGS += -pthread
+ CPPFLAGS += -pthread
endif
+ifeq ($(OSTYPE), FreeBSD)
endif
-ifdef KMER
- MERYLSOURCE = meryl.C
- MERYLOBJECTS = meryl.o AS_MER_gkpStore_to_FastABase.o AS_MER_gkpStoreChain.o
- SOURCES = $(MERYLSOURCE) mercy.C mercy-regions.C overmerry.C merTrim.C merTrimAdapter.C merTrimApply.C estimate-mer-threshold.C AS_MER_gkpStore_to_FastABase.C AS_MER_gkpStoreChain.C
- OBJECTS = $(SOURCES:.C=.o)
- CXX_PROGS = meryl mercy overmerry merTrim merTrimApply estimate-mer-threshold
- INC_IMPORT_DIRS += $(KMER)/include
- LIB_IMPORT_DIRS += $(KMER)/lib
- KMERLIBS = libmerylguts.a libkmer.a libmeryl.a libseq.a libbio.a libutil.a
-else
- MERYLSOURCE = AS_MER_meryl.cc
- MERYLOBJECTS = AS_MER_meryl.o
- SOURCES = $(MERYLSOURCE)
- OBJECTS = $(SOURCES:.cc=.o)
- CXX_PROGS = meryl
+ifeq ($(OSTYPE), Darwin)
endif
-# include again to get everything else updated
-include $(LOCAL_WORK)/src/c_make.as
-
-
-# This must come after c_make.as, grrr.
-ifdef KMER
- # See also AS_OVL/Makefile
- ifeq ($(OSTYPE), Linux)
- LDFLAGS += -lpthread
- CFLAGS += -pthread
- CPPFLAGS += -pthread
- endif
-
- ifeq ($(OSTYPE), OSF1)
- LDFLAGS += -lpthread
- CFLAGS += -pthread
- CPPFLAGS += -pthread
- endif
-
- ifeq ($(OSTYPE), FreeBSD)
- endif
-
- ifeq ($(OSTYPE), Darwin)
- endif
-endif
-
all: $(OBJECTS) $(LIBRARIES) $(CXX_PROGS)
@test -n nop
diff --git a/src/AS_MER/estimate-mer-threshold.C b/src/AS_MER/estimate-mer-threshold.C
index 6bd2e7b..eaf1cee 100644
--- a/src/AS_MER/estimate-mer-threshold.C
+++ b/src/AS_MER/estimate-mer-threshold.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: estimate-mer-threshold.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: estimate-mer-threshold.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -39,7 +39,7 @@ main(int argc, char **argv) {
merylStreamReader *MF = 0L;
- u32bit maxCount = 0;
+ uint32 maxCount = 0;
argc = AS_configure(argc, argv);
@@ -71,9 +71,9 @@ main(int argc, char **argv) {
uint64 total = 0;
uint32 Xcoverage = 8;
- fprintf(stderr, "distinct: "u64bitFMT"\n", MF->numberOfDistinctMers());
- fprintf(stderr, "unique: "u64bitFMT"\n", MF->numberOfUniqueMers());
- fprintf(stderr, "total: "u64bitFMT"\n", MF->numberOfTotalMers());
+ fprintf(stderr, "distinct: "F_U64"\n", MF->numberOfDistinctMers());
+ fprintf(stderr, "unique: "F_U64"\n", MF->numberOfUniqueMers());
+ fprintf(stderr, "total: "F_U64"\n", MF->numberOfTotalMers());
// Pass 0: try to deduce the X coverage we have. The
// pattern we should see in mer counts is an initial spike
@@ -92,9 +92,9 @@ main(int argc, char **argv) {
uint32 i = 0;
uint32 iX = 0;
- fprintf(stderr, "distinct: "u64bitFMT"\n", MF->numberOfDistinctMers());
- fprintf(stderr, "unique: "u64bitFMT"\n", MF->numberOfUniqueMers());
- fprintf(stderr, "total: "u64bitFMT"\n", MF->numberOfTotalMers());
+ fprintf(stderr, "distinct: "F_U64"\n", MF->numberOfDistinctMers());
+ fprintf(stderr, "unique: "F_U64"\n", MF->numberOfUniqueMers());
+ fprintf(stderr, "total: "F_U64"\n", MF->numberOfTotalMers());
fprintf(stderr, "Xcoverage zero 1 0 "F_U64"\n", MF->histogram(1));
@@ -136,7 +136,7 @@ main(int argc, char **argv) {
maxCount = i;
}
- fprintf(stderr, "Set maxCount to "u32bitFMT", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
+ fprintf(stderr, "Set maxCount to "F_U32", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
i, 100.0 * distinct / totalUsefulDistinct, 100.0 * total / totalUsefulAll);
@@ -167,11 +167,11 @@ main(int argc, char **argv) {
maxCount = i;
- fprintf(stderr, "Reset maxCount to "u32bitFMT", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
+ fprintf(stderr, "Reset maxCount to "F_U32", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n",
maxCount, 100.0 * distinct / totalUsefulDistinct, 100.0 * total / totalUsefulAll);
}
- fprintf(stdout, u32bitFMT"\n", maxCount);
+ fprintf(stdout, F_U32"\n", maxCount);
return(0);
}
diff --git a/src/AS_MER/merTrim-compare-logs.pl b/src/AS_MER/merTrim-compare-logs.pl
new file mode 100644
index 0000000..329023f
--- /dev/null
+++ b/src/AS_MER/merTrim-compare-logs.pl
@@ -0,0 +1,86 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $log1 = shift @ARGV;
+my $log2 = shift @ARGV;
+
+if (!defined($log1) || !defined($log2)) {
+ die "usage: $0 run1.log run2.log\n";
+}
+
+open(L1, "< $log1") or die;
+open(L2, "< $log1") or die;
+
+my ($a1, $a2, $a3, $a4, $a5, $a6, $a7, $a8, $a9);
+my ($b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8, $b9);
+
+while (!eof(L1) && !eof(L2)) {
+
+ anotherA:
+ do {
+ $a1 = <L1>; # FINAL, or "Correct" lines
+ } while ($a1 !~ m/^FINAL/);
+ $a2 = <L1>; # ORI seq
+ $a3 = <L1>; # COR seq
+ $a4 = <L1>; # COR qlt
+ $a5 = <L1>; # COVERAGE
+ $a6 = <L1>; # CORRECTIONS
+ $a7 = <L1>; # DISCONNECTION
+ $a8 = <L1>; # ADAPTER
+ $a9 = <L1>; # RESULT
+
+ #if ($a1 =~ m/^ADAPTERSEARCH/) {
+ # goto anotherA;
+ #}
+
+ anotherB:
+ do {
+ $b1 = <L2>; # FINAL, or "Correct" lines
+ } while ($b1 !~ m/^FINAL/);
+ $b2 = <L2>; # ORI seq
+ $b3 = <L2>; # COR seq
+ $b4 = <L2>; # COR qlt
+ $b5 = <L2>; # COVERAGE
+ $b6 = <L2>; # CORRECTIONS
+ $b7 = <L2>; # DISCONNECTION
+ $b8 = <L2>; # ADAPTER
+ $b9 = <L2>; # RESULT
+
+ #if ($b1 =~ m/^ADAPTERSEARCH/) {
+ # goto anotherB;
+ #}
+
+ my ($aID, $aLen, $aBgn, $aEnd);
+ my ($bID, $bLen, $bBgn, $bEnd);
+
+ # FINAL or ADAPTERSEARCH
+ if ($a1 =~ m/^\w+\sread\s(\d+)\slen\s(\d+)\s\(trim\s(\d+)-(\d+)\)$/) {
+ $aID = $1;
+ $aLen = $2;
+ $aBgn = $3;
+ $aEnd = $4;
+ } else {
+ die "Nope a1 $a1";
+ }
+
+ if ($b1 =~ m/^\w+\sread\s(\d+)\slen\s(\d+)\s\(trim\s(\d+)-(\d+)\)$/) {
+ $bID = $1;
+ $bLen = $2;
+ $bBgn = $3;
+ $bEnd = $4;
+ } else {
+ die "Nope b1 $b1";
+ }
+
+ die "ID mismatch $aID $bID\n" if ($aID != $bID);
+
+ if (($aBgn != $bBgn) || ($aEnd != $bEnd)) {
+ print "$aID/$bID $aLen/$bLen $aBgn-$aEnd $bBgn-$bEnd\n";
+ }
+
+ if (($aID % 10000) == 0) {
+ print STDERR "$aID\n";
+ }
+}
+
diff --git a/src/AS_MER/merTrim.C b/src/AS_MER/merTrim.C
index 395806f..21ccda6 100644
--- a/src/AS_MER/merTrim.C
+++ b/src/AS_MER/merTrim.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: merTrim.C 4444 2013-10-09 17:31:38Z brianwalenz $";
+const char *mainid = "$Id: merTrim.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include "AS_global.H"
#include "AS_UTL_reverseComplement.H"
@@ -205,9 +205,9 @@ public:
uint32 i = 0;
uint32 iX = 0;
- //fprintf(stderr, "distinct: "u64bitFMT"\n", MF->numberOfDistinctMers());
- //fprintf(stderr, "unique: "u64bitFMT"\n", MF->numberOfUniqueMers());
- //fprintf(stderr, "total: "u64bitFMT"\n", MF->numberOfTotalMers());
+ //fprintf(stderr, "distinct: "F_U64"\n", MF->numberOfDistinctMers());
+ //fprintf(stderr, "unique: "F_U64"\n", MF->numberOfUniqueMers());
+ //fprintf(stderr, "total: "F_U64"\n", MF->numberOfTotalMers());
//fprintf(stderr, "Xcoverage zero 1 0 "F_U64"\n", MF->histogram(1));
@@ -868,8 +868,8 @@ mertrimComputation::analyze(void) {
memset(disconnect, 0, sizeof(uint32) * (allocLen));
while (rMS->nextMer()) {
- u32bit posBgn = rMS->thePositionInSequence();
- u32bit posEnd = rMS->thePositionInSequence() + g->merSize;
+ uint32 posBgn = rMS->thePositionInSequence();
+ uint32 posEnd = rMS->thePositionInSequence() + g->merSize;
assert(posEnd <= seqLen);
@@ -884,7 +884,7 @@ mertrimComputation::analyze(void) {
disconnect[posBgn-1] = disconnect[posBgn] = 'D';
// Add coverage for the good mer.
- for (u32bit add=posBgn; add<posEnd; add++)
+ for (uint32 add=posBgn; add<posEnd; add++)
coverage[add]++;
} // Over all mers
@@ -2395,7 +2395,7 @@ main(int argc, char **argv) {
ss->setNumberOfWorkers(g->numThreads);
- for (u32bit w=0; w<g->numThreads; w++)
+ for (uint32 w=0; w<g->numThreads; w++)
ss->setThreadData(w, new mertrimThreadData(g)); // these leak
ss->run(g, g->beVerbose); // true == verbose
diff --git a/src/AS_MER/merTrimApply.C b/src/AS_MER/merTrimApply.C
index 05daf8b..f18a37a 100644
--- a/src/AS_MER/merTrimApply.C
+++ b/src/AS_MER/merTrimApply.C
@@ -19,11 +19,11 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: merTrimApply.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: merTrimApply.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
#include "merTrimResult.H"
diff --git a/src/AS_MER/mercy-regions.C b/src/AS_MER/mercy-regions.C
index 9ef2b85..68bde17 100644
--- a/src/AS_MER/mercy-regions.C
+++ b/src/AS_MER/mercy-regions.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: mercy-regions.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: mercy-regions.C 4572 2014-10-14 23:06:05Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -27,8 +27,8 @@ const char *mainid = "$Id: mercy-regions.C 4371 2013-08-01 17:19:47Z brianwalenz
#include <errno.h>
#include "AS_global.H"
-#include "AS_UTL_splitToWords.H"
-#include "AS_UTL_intervalList.H"
+#include "splitToWords.H"
+#include "intervalList.H"
// This reads the assembly frgctg, varctg and merQC badmers, computes
// the number and location of bad-mer, bad-var regions, and their
@@ -42,9 +42,9 @@ using namespace std;
void
-readDepth(char *depthname, map<uint64,intervalDepth*> &lowCoverage) {
- char line[1024] = {0};
- map<uint64,intervalList*> ILs;
+readDepth(char *depthname, map<uint64,intervalList<uint32>*> &lowCoverage) {
+ char line[1024] = {0};
+ map<uint64,intervalList<uint32>*> ILs;
fprintf(stderr, "Reading depth from '%s'\n", depthname);
@@ -67,7 +67,7 @@ readDepth(char *depthname, map<uint64,intervalDepth*> &lowCoverage) {
fprintf(stderr, "ERROR: l="F_U32" h="F_U32"\n", beg, end);
if (ILs[uid] == 0L)
- ILs[uid] = new intervalList();
+ ILs[uid] = new intervalList<uint32>;
ILs[uid]->add(beg, end - beg);
i++;
@@ -78,11 +78,11 @@ readDepth(char *depthname, map<uint64,intervalDepth*> &lowCoverage) {
fclose(F);
fprintf(stderr, " "F_U32" lines.\n", i);
- map<uint64,intervalList*>::iterator it = ILs.begin();
- map<uint64,intervalList*>::iterator ed = ILs.end();
+ map<uint64,intervalList<uint32>*>::iterator it = ILs.begin();
+ map<uint64,intervalList<uint32>*>::iterator ed = ILs.end();
while (it != ed) {
- lowCoverage[it->first] = new intervalDepth(*it->second);
+ lowCoverage[it->first] = new intervalList<uint32>(*it->second);
delete it->second;
it->second = 0L;
it++;
@@ -91,7 +91,7 @@ readDepth(char *depthname, map<uint64,intervalDepth*> &lowCoverage) {
void
-readVariation(char *depthname, map<uint64,intervalList*> &variation) {
+readVariation(char *depthname, map<uint64,intervalList<uint32>*> &variation) {
char line[1024 * 1024] = {0};
fprintf(stderr, "Reading variation from '%s'\n", depthname);
@@ -112,7 +112,7 @@ readVariation(char *depthname, map<uint64,intervalList*> &variation) {
uint32 end = strtoul(W[3], 0L, 10);
if (variation[uid] == 0L)
- variation[uid] = new intervalList();
+ variation[uid] = new intervalList<uint32>;
variation[uid]->add(beg, end - beg);
i++;
@@ -126,7 +126,7 @@ readVariation(char *depthname, map<uint64,intervalList*> &variation) {
void
-readBadMers(char *depthname, map<uint64,intervalList*> &badMers) {
+readBadMers(char *depthname, map<uint64,intervalList<uint32>*> &badMers) {
char line[1024] = {0};
fprintf(stderr, "Reading badMers from '%s'\n", depthname);
@@ -152,7 +152,7 @@ readBadMers(char *depthname, map<uint64,intervalList*> &badMers) {
uint32 end = strtoul(W[4], 0L, 10);
if (badMers[uid] == 0L)
- badMers[uid] = new intervalList();
+ badMers[uid] = new intervalList<uint32>;
badMers[uid]->add(beg, end - beg);
i++;
@@ -168,9 +168,9 @@ readBadMers(char *depthname, map<uint64,intervalList*> &badMers) {
int
main(int argc, char **argv) {
- map<uint64,intervalList*> badMers;
- map<uint64,intervalList*> variation;
- map<uint64,intervalDepth*> lowCoverage;
+ map<uint64,intervalList<uint32>*> badMers;
+ map<uint64,intervalList<uint32>*> variation;
+ map<uint64,intervalList<uint32>*> lowCoverage;
bool showDepthIntersect = false;
bool showVariantIntersect = false;
@@ -230,15 +230,15 @@ main(int argc, char **argv) {
for (uint32 j=0; j<32; j++)
badDepth[i][j] = 0;
- map<uint64,intervalList*>::iterator it = badMers.begin();
- map<uint64,intervalList*>::iterator ed = badMers.end();
+ map<uint64,intervalList<uint32>*>::iterator it = badMers.begin();
+ map<uint64,intervalList<uint32>*>::iterator ed = badMers.end();
while (it != ed) {
uint64 uid = it->first;
- intervalList *Iv = variation[uid];
- intervalList *Ib = badMers[uid];
- intervalList *Ii = 0L;
- intervalDepth *Id = lowCoverage[uid];
+ intervalList<uint32> *Iv = variation[uid];
+ intervalList<uint32> *Ib = badMers[uid];
+ intervalList<uint32> *Ii = 0L;
+ intervalList<uint32> *Id = lowCoverage[uid];
if (Iv)
Iv->merge();
@@ -246,7 +246,7 @@ main(int argc, char **argv) {
Ib->merge();
if (Iv && Ib) {
- Ii = new intervalList();
+ Ii = new intervalList<uint32>;
Ii->intersect(*Iv, *Ib);
}
@@ -280,13 +280,13 @@ main(int argc, char **argv) {
// Low points are not allowed to be equal to high points.
if ((lo <= Ii->lo(ii)) && (Ii->lo(ii) < hi)) {
- beg = Id->de(id);
+ beg = Id->depth(id);
} else {
- fprintf(stderr, "failed to find begin "F_U64" "F_U64" -- "F_U64" "F_U64" "F_U32"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->de(id));
+ fprintf(stderr, "failed to find begin "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n",
+ Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->depth(id));
if (id > 0)
- fprintf(stderr, " "F_U64" "F_U64" -- "F_U64" "F_U64" "F_U32"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->de(id-1));
+ fprintf(stderr, " "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n",
+ Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->depth(id-1));
//exit(1);
}
}
@@ -307,13 +307,13 @@ main(int argc, char **argv) {
// High points aren't allowed to be equal to lo, but can be equal to hi.
if ((lo < Ii->hi(ii)) && (Ii->hi(ii) <= hi)) {
- end = Id->de(id);
+ end = Id->depth(id);
} else {
- fprintf(stderr, "failed to find end "F_U64" "F_U64" -- "F_U64" "F_U64" "F_U32"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->de(id));
+ fprintf(stderr, "failed to find end "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n",
+ Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->depth(id));
if (id > 0)
- fprintf(stderr, " "F_U64" "F_U64" -- "F_U64" "F_U64" "F_U32"\n",
- Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->de(id-1));
+ fprintf(stderr, " "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n",
+ Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->depth(id-1));
//exit(1);
}
}
@@ -321,7 +321,7 @@ main(int argc, char **argv) {
badBegDepth[beg]++;
badEndDepth[end]++;
- fprintf(stdout, F_U64"\t"F_U64"\t"F_U64"\tdepth="F_U32","F_U32"\n",
+ fprintf(stdout, F_U64"\t"F_U32"\t"F_U32"\tdepth="F_U32","F_U32"\n",
uid, Ii->lo(ii), Ii->hi(ii), beg, end);
if ((beg < 32) && (end < 32))
diff --git a/src/AS_MER/mercy.C b/src/AS_MER/mercy.C
index 004b37c..d4d8df4 100644
--- a/src/AS_MER/mercy.C
+++ b/src/AS_MER/mercy.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: mercy.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: mercy.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -51,14 +51,14 @@ FILE *dumpMCSF = 0L;
FILE *dumpMCMF = 0L;
char merstring[1024];
-u32bit
+uint32
findMode(char *name) {
merylStreamReader *M = new merylStreamReader(name);
- u32bit *H = new u32bit [16384];
+ uint32 *H = new uint32 [16384];
fprintf(stderr, "Finding mode of '%s'\n", name);
- for (u32bit i=0; i<16384; i++)
+ for (uint32 i=0; i<16384; i++)
H[i] = 0;
while (M->validMer()) {
@@ -67,12 +67,12 @@ findMode(char *name) {
M->nextMer();
}
- u32bit mi = 16;
- for (u32bit i=mi; i<16384; i++)
+ uint32 mi = 16;
+ for (uint32 i=mi; i<16384; i++)
if (H[i] > H[mi])
mi = i;
- fprintf(stderr, "Mode of '%s' is "u32bitFMT"\n", name, mi);
+ fprintf(stderr, "Mode of '%s' is "F_U32"\n", name, mi);
return(mi);
}
@@ -82,14 +82,14 @@ void
compare(merylStreamReader *F,
merylStreamReader *C,
kMer &minmer,
- u32bit mode,
- u32bit R[NUMCATEGORIES][NUMCATEGORIES]) {
- u32bit Ftype = 0;
- u32bit Ctype = 0;
+ uint32 mode,
+ uint32 R[NUMCATEGORIES][NUMCATEGORIES]) {
+ uint32 Ftype = 0;
+ uint32 Ctype = 0;
kMer Fmer = F->theFMer();
kMer Cmer = C->theFMer();
- u32bit Fcnt = F->theCount();
- u32bit Ccnt = C->theCount();
+ uint32 Fcnt = F->theCount();
+ uint32 Ccnt = C->theCount();
if (Fcnt == 0)
Ftype = 0;
@@ -135,9 +135,9 @@ compare(merylStreamReader *F,
if (dumpFlag)
if (Ftype == 0)
if (Ctype == 1)
- fprintf(dumpSCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpSCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
else
- fprintf(dumpMCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpMCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
}
return;
}
@@ -154,9 +154,9 @@ compare(merylStreamReader *F,
// Again, save the mer since it's in contigs, but not fragments.
if (dumpFlag)
if (Ctype == 1)
- fprintf(dumpSCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpSCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
else
- fprintf(dumpMCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpMCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
}
return;
@@ -173,25 +173,25 @@ compare(merylStreamReader *F,
if (dumpFlag) {
if (Ftype < Ctype)
if (Ctype == 2)
- fprintf(dumpMCSF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpMCSF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
else
- fprintf(dumpMCMF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpMCMF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
if ((Ftype == 0) && (Ctype == 1))
- fprintf(dumpSCZF, ">"u32bitFMT"\n%s\n", Ccnt, Cmer.merToString(merstring));
+ fprintf(dumpSCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring));
}
}
void
output(char *title,
- u32bit mode,
- u32bit R[NUMCATEGORIES][NUMCATEGORIES]) {
+ uint32 mode,
+ uint32 R[NUMCATEGORIES][NUMCATEGORIES]) {
fprintf(stdout, "\n\n%s\n", title);
fprintf(stdout, "(frags) | zero | one | <= 10 | <= 100 | <= inf | (contigs)\n");
- for (u32bit i=0; i<6; i++) {
+ for (uint32 i=0; i<6; i++) {
switch (i) {
case 0: fprintf(stdout, "zero "); break;
case 1: fprintf(stdout, "one "); break;
@@ -201,8 +201,8 @@ output(char *title,
case 5: fprintf(stdout, "<= inf "); break;
default: fprintf(stdout, "????????? "); break;
}
- for (u32bit j=0; j<5; j++)
- fprintf(stdout, u32bitFMTW(12), R[i][j]);
+ for (uint32 j=0; j<5; j++)
+ fprintf(stdout, "%12"F_U32P, R[i][j]);
fprintf(stdout, "\n");
}
}
@@ -218,8 +218,8 @@ main(int argc, char **argv) {
merylStreamReader *DC = 0L;
merylStreamReader *CO = 0L;
- u32bit AFmode = 0;
- u32bit TFmode = 0;
+ uint32 AFmode = 0;
+ uint32 TFmode = 0;
char dumpSCZFname[1024] = {0}; // single contig, zero frags
char dumpMCZFname[1024] = {0}; // low contig, zero frags
@@ -286,8 +286,8 @@ main(int argc, char **argv) {
// Check mersizes.
//
- u32bit merSize = 0;
- u32bit ms[5] = { 0 };
+ uint32 merSize = 0;
+ uint32 ms[5] = { 0 };
if (AF) merSize = ms[0] = AF->merSize();
if (TF) merSize = ms[1] = TF->merSize();
@@ -305,11 +305,11 @@ main(int argc, char **argv) {
if (differ) {
fprintf(stderr, "error: mer size differ.\n");
- fprintf(stderr, " AF - "u32bitFMT"\n", ms[0]);
- fprintf(stderr, " TF - "u32bitFMT"\n", ms[1]);
- fprintf(stderr, " AC - "u32bitFMT"\n", ms[2]);
- fprintf(stderr, " DC - "u32bitFMT"\n", ms[3]);
- fprintf(stderr, " CO - "u32bitFMT"\n", ms[4]);
+ fprintf(stderr, " AF - "F_U32"\n", ms[0]);
+ fprintf(stderr, " TF - "F_U32"\n", ms[1]);
+ fprintf(stderr, " AC - "F_U32"\n", ms[2]);
+ fprintf(stderr, " DC - "F_U32"\n", ms[3]);
+ fprintf(stderr, " CO - "F_U32"\n", ms[4]);
exit(1);
}
@@ -323,14 +323,14 @@ main(int argc, char **argv) {
fprintf(stderr, "Failed to open the dump files: %s\n", strerror(errno)), exit(1);
}
- u32bit AFvsAC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit AFvsDC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit AFvsCO[NUMCATEGORIES][NUMCATEGORIES];
- u32bit TFvsAC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit TFvsDC[NUMCATEGORIES][NUMCATEGORIES];
- u32bit TFvsCO[NUMCATEGORIES][NUMCATEGORIES];
- for (u32bit i=0; i<NUMCATEGORIES; i++)
- for (u32bit j=0; j<NUMCATEGORIES; j++) {
+ uint32 AFvsAC[NUMCATEGORIES][NUMCATEGORIES];
+ uint32 AFvsDC[NUMCATEGORIES][NUMCATEGORIES];
+ uint32 AFvsCO[NUMCATEGORIES][NUMCATEGORIES];
+ uint32 TFvsAC[NUMCATEGORIES][NUMCATEGORIES];
+ uint32 TFvsDC[NUMCATEGORIES][NUMCATEGORIES];
+ uint32 TFvsCO[NUMCATEGORIES][NUMCATEGORIES];
+ for (uint32 i=0; i<NUMCATEGORIES; i++)
+ for (uint32 j=0; j<NUMCATEGORIES; j++) {
AFvsAC[i][j] = 0;
AFvsDC[i][j] = 0;
AFvsCO[i][j] = 0;
diff --git a/src/AS_MER/overmerry.C b/src/AS_MER/overmerry.C
index 6877183..e5a1c83 100644
--- a/src/AS_MER/overmerry.C
+++ b/src/AS_MER/overmerry.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: overmerry.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: overmerry.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
@@ -47,20 +47,20 @@ const char *mainid = "$Id: overmerry.C 4371 2013-08-01 17:19:47Z brianwalenz $";
class kmerhit {
public:
union {
- u64bit num;
+ uint64 num;
struct {
- u64bit tseq:30; // sequence in the table
- u64bit tpos:AS_OVS_POSBITS; // position in that sequence
- u64bit qpos:AS_OVS_POSBITS; // position in the query sequence
- u64bit cnt:8; // count of the kmer
- u64bit pal:1; // palindromic ; 0 = nope, 1 = yup
- u64bit fwd:1; // orientation ; 0 = reverse, 1 = forward
- u64bit pad:2;
+ uint64 tseq:30; // sequence in the table
+ uint64 tpos:AS_OVS_POSBITS; // position in that sequence
+ uint64 qpos:AS_OVS_POSBITS; // position in the query sequence
+ uint64 cnt:8; // count of the kmer
+ uint64 pal:1; // palindromic ; 0 = nope, 1 = yup
+ uint64 fwd:1; // orientation ; 0 = reverse, 1 = forward
+ uint64 pad:2;
} val;
} dat;
- void setInteger(u64bit tseq, u64bit cnt, u64bit qpos, u64bit tpos, u64bit pal, u64bit fwd) {
+ void setInteger(uint64 tseq, uint64 cnt, uint64 qpos, uint64 tpos, uint64 pal, uint64 fwd) {
// Threshold cnt to the maximum allowed.
if (cnt > 255)
@@ -125,8 +125,8 @@ public:
~ovmGlobalData() {
- //fprintf(stderr, "Found "u64bitFMT" mer hits.\n", merfound);
- //fprintf(stderr, "Found "u64bitFMT" overlaps.\n", ovlfound);
+ //fprintf(stderr, "Found "F_U64" mer hits.\n", merfound);
+ //fprintf(stderr, "Found "F_U64" overlaps.\n", ovlfound);
AS_OVS_closeBinaryOverlapFile(outputFile);
@@ -325,11 +325,11 @@ public:
void
addHit(seqStream *SS,
AS_IID iid,
- u64bit qpos,
- u64bit pos,
- u64bit cnt,
- u64bit pal,
- u64bit fwd) {
+ uint64 qpos,
+ uint64 pos,
+ uint64 cnt,
+ uint64 pal,
+ uint64 fwd) {
uint32 seq = SS->sequenceNumberOfPosition(pos);
@@ -364,20 +364,20 @@ public:
kMerBuilder *qKB;
- u64bit posnFLen;
- u64bit posnFMax;
- u64bit *posnF;
+ uint64 posnFLen;
+ uint64 posnFMax;
+ uint64 *posnF;
- u64bit posnRLen;
- u64bit posnRMax;
- u64bit *posnR;
+ uint64 posnRLen;
+ uint64 posnRMax;
+ uint64 *posnR;
- u32bit hitsLen;
- u32bit hitsMax;
+ uint32 hitsLen;
+ uint32 hitsMax;
kmerhit *hits;
- u64bit merfound;
- u64bit ovlfound;
+ uint64 merfound;
+ uint64 ovlfound;
};
@@ -437,19 +437,19 @@ public:
-u32bit
+uint32
ovmWorker_analyzeReadForThreshold(ovmGlobalData *g,
ovmThreadData *t,
ovmComputation *s,
merStream *sMSTR,
uint32 *sSPAN) {
- u32bit maxCount = 0;
- u32bit cnt = 0;
- u32bit ave = 0;
+ uint32 maxCount = 0;
+ uint32 cnt = 0;
+ uint32 ave = 0;
while (sMSTR->nextMer()) {
- u32bit c = (u32bit)g->tPS->countExact(sMSTR->theFMer());
+ uint32 c = (uint32)g->tPS->countExact(sMSTR->theFMer());
if (c > 1) {
sSPAN[cnt] = c;
@@ -458,16 +458,16 @@ ovmWorker_analyzeReadForThreshold(ovmGlobalData *g,
}
}
- u32bit minc = 0; // Min count observed
- u32bit maxc = 0; // Max count observed
+ uint32 minc = 0; // Min count observed
+ uint32 maxc = 0; // Max count observed
- u32bit medi = 0; // Median count
- u32bit mean = 0; // Mean count;
+ uint32 medi = 0; // Median count
+ uint32 mean = 0; // Mean count;
- u32bit mode = 0; // Modal count
- u32bit mcnt = 0; // Times we saw the modal count
+ uint32 mode = 0; // Modal count
+ uint32 mcnt = 0; // Times we saw the modal count
- u32bit mtmp = 0; // A temporary for counting the mode
+ uint32 mtmp = 0; // A temporary for counting the mode
if (cnt > 0) {
std::sort(sSPAN, sSPAN + cnt);
@@ -482,7 +482,7 @@ ovmWorker_analyzeReadForThreshold(ovmGlobalData *g,
mtmp = 1;
- for (u32bit i=1; i<cnt; i++) {
+ for (uint32 i=1; i<cnt; i++) {
mean += sSPAN[i];
if (sSPAN[i] == sSPAN[i-1])
@@ -532,7 +532,7 @@ ovmWorker(void *G, void *T, void *S) {
false, true);
uint32 *sSPAN = new uint32 [s->end - s->beg];
- u32bit maxCountForFrag = g->maxCountGlobal;
+ uint32 maxCountForFrag = g->maxCountGlobal;
switch (g->maxCountType) {
case MAX_COUNT_GLOBAL:
@@ -557,10 +557,10 @@ ovmWorker(void *G, void *T, void *S) {
t->posnRLen = 0;
while (sMSTR->nextMer()) {
- u64bit qpos = sMSTR->thePositionInSequence();
- u64bit fcount = 0;
- u64bit rcount = 0;
- u64bit tcount = 0;
+ uint64 qpos = sMSTR->thePositionInSequence();
+ uint64 fcount = 0;
+ uint64 rcount = 0;
+ uint64 tcount = 0;
sSPAN[qpos] = sMSTR->theFMer().getMerSpan();
assert(qpos <= s->end - s->beg);
@@ -569,7 +569,7 @@ ovmWorker(void *G, void *T, void *S) {
g->tPS->getExact(sMSTR->theFMer(), t->posnF, t->posnFMax, t->posnFLen, fcount);
if (fcount < maxCountForFrag) {
- for (u32bit i=0; i<t->posnFLen; i++)
+ for (uint32 i=0; i<t->posnFLen; i++)
t->addHit(g->tSS, s->iid, qpos, t->posnF[i], fcount, 1, 0);
}
} else {
@@ -630,9 +630,9 @@ ovmWorker(void *G, void *T, void *S) {
if (tcount < maxCountForFrag) {
- for (u32bit i=0; i<t->posnFLen; i++)
+ for (uint32 i=0; i<t->posnFLen; i++)
t->addHit(g->tSS, s->iid, qpos, t->posnF[i], tcount, 0, 1);
- for (u32bit i=0; i<t->posnRLen; i++)
+ for (uint32 i=0; i<t->posnRLen; i++)
t->addHit(g->tSS, s->iid, qpos, t->posnR[i], tcount, 0, 0);
}
}
@@ -655,7 +655,7 @@ ovmWorker(void *G, void *T, void *S) {
//
std::sort(t->hits, t->hits + t->hitsLen);
- for (u32bit i=0; i<t->hitsLen; i++)
+ for (uint32 i=0; i<t->hitsLen; i++)
t->hits[i].unpackInteger();
#if 0
@@ -663,9 +663,9 @@ ovmWorker(void *G, void *T, void *S) {
// big identical overlaps will have lots and lots of mers in
// common.
//
- for (u32bit i=0; i<t->hitsLen; i++) {
+ for (uint32 i=0; i<t->hitsLen; i++) {
if (i != t->hitsLen) {
- fprintf(stderr, u32bitFMT"\t"F_U64"\t"u32bitFMT"\t"F_U64"\t%c\t"u32bitFMT"\n",
+ fprintf(stderr, F_U32"\t"F_U64"\t"F_U32"\t"F_U64"\t%c\t"F_U32"\n",
t->hits[i].dat.val.tseq, t->hits[i].dat.val.tpos,
s->iid, t->hits[i].dat.val.qpos,
t->hits[i].dat.val.pal ? 'p' : (t->hits[i].dat.val.fwd ? 'f' : 'r'),
@@ -679,8 +679,8 @@ ovmWorker(void *G, void *T, void *S) {
memset(&overlap, 0, sizeof(OVSoverlap));
- for (u32bit i=0; i<t->hitsLen; ) {
- //fprintf(stderr, "FILTER STARTS i="u32bitFMT" tseq="F_U64" tpos="F_U64" qpos="F_U64"\n",
+ for (uint32 i=0; i<t->hitsLen; ) {
+ //fprintf(stderr, "FILTER STARTS i="F_U32" tseq="F_U64" tpos="F_U64" qpos="F_U64"\n",
// i, t->hits[i].dat.val.tseq, t->hits[i].dat.val.tpos, t->hits[i].dat.val.qpos);
// By the definition of our sort, the least common mer is the
@@ -746,9 +746,9 @@ ovmWorker(void *G, void *T, void *S) {
// Now, skip ahead until we find the next pair.
//
- u64bit lastiid = t->hits[i].dat.val.tseq;
+ uint64 lastiid = t->hits[i].dat.val.tseq;
while ((i < t->hitsLen) && (t->hits[i].dat.val.tseq == lastiid)) {
- //fprintf(stderr, "FILTER OUT i="u32bitFMT" tseq="F_U64" tpos="F_U64" qpos="F_U64"\n",
+ //fprintf(stderr, "FILTER OUT i="F_U32" tseq="F_U64" tpos="F_U64" qpos="F_U64"\n",
// i, t->hits[i].dat.val.tseq, t->hits[i].dat.val.tpos, t->hits[i].dat.val.qpos);
i++;
}
@@ -889,7 +889,7 @@ main(int argc, char **argv) {
ss->setNumberOfWorkers(g->numThreads);
- for (u32bit w=0; w<g->numThreads; w++)
+ for (uint32 w=0; w<g->numThreads; w++)
ss->setThreadData(w, new ovmThreadData(g)); // these leak
ss->run(g, g->beVerbose); // true == verbose
diff --git a/src/AS_MSG/._AS_MSG_pmesg.C b/src/AS_MSG/._AS_MSG_pmesg.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_pmesg.C differ
diff --git a/src/AS_MSG/._AS_MSG_pmesg.H b/src/AS_MSG/._AS_MSG_pmesg.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_pmesg.H differ
diff --git a/src/AS_MSG/._AS_MSG_pmesg1.C b/src/AS_MSG/._AS_MSG_pmesg1.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_pmesg1.C differ
diff --git a/src/AS_MSG/._AS_MSG_pmesg2.C b/src/AS_MSG/._AS_MSG_pmesg2.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_pmesg2.C differ
diff --git a/src/AS_MSG/._AS_MSG_pmesg_documentation.txt b/src/AS_MSG/._AS_MSG_pmesg_documentation.txt
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_pmesg_documentation.txt differ
diff --git a/src/AS_MSG/._AS_MSG_pmesg_internal.H b/src/AS_MSG/._AS_MSG_pmesg_internal.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_pmesg_internal.H differ
diff --git a/src/AS_MSG/._AS_MSG_types.H b/src/AS_MSG/._AS_MSG_types.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._AS_MSG_types.H differ
diff --git a/src/AS_MSG/._ExtractMessages.C b/src/AS_MSG/._ExtractMessages.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._ExtractMessages.C differ
diff --git a/src/AS_MSG/._Makefile b/src/AS_MSG/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._Makefile differ
diff --git a/src/AS_MSG/._convert-fasta-to-v2.pl b/src/AS_MSG/._convert-fasta-to-v2.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._convert-fasta-to-v2.pl differ
diff --git a/src/AS_MSG/._convert-v1-to-v2.pl b/src/AS_MSG/._convert-v1-to-v2.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._convert-v1-to-v2.pl differ
diff --git a/src/AS_MSG/._generate-random-fragments.sh b/src/AS_MSG/._generate-random-fragments.sh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._generate-random-fragments.sh differ
diff --git a/src/AS_MSG/._remove_fragment.C b/src/AS_MSG/._remove_fragment.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._remove_fragment.C differ
diff --git a/src/AS_MSG/._tracedb-to-frg.pl b/src/AS_MSG/._tracedb-to-frg.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_MSG/._tracedb-to-frg.pl differ
diff --git a/src/AS_MSG/AS_MSG_pmesg.H b/src/AS_MSG/AS_MSG_pmesg.H
index 12e5ad4..cacc169 100644
--- a/src/AS_MSG/AS_MSG_pmesg.H
+++ b/src/AS_MSG/AS_MSG_pmesg.H
@@ -18,12 +18,12 @@
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-/* $Id: AS_MSG_pmesg.H 4371 2013-08-01 17:19:47Z brianwalenz $ */
+/* $Id: AS_MSG_pmesg.H 4518 2014-03-31 20:11:04Z brianwalenz $ */
#ifndef AS_MSG_PMESG_INCLUDE_H
#define AS_MSG_PMESG_INCLUDE_H
-static const char *rcsid_AS_MSG_PMESG_INCLUDE_H = "$Id: AS_MSG_pmesg.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_AS_MSG_PMESG_INCLUDE_H = "$Id: AS_MSG_pmesg.H 4518 2014-03-31 20:11:04Z brianwalenz $";
#include <cstdio>
#include <ctime>
@@ -261,11 +261,11 @@ typedef struct {
double coverage_stat;
double microhet_prob;
UnitigStatus status;
- UnitigFUR unique_rept;
+ char unique_rept; // NOT USED?
int32 length;
char *consensus;
char *quality;
- int32 forced;
+ int32 forced;
int32 num_frags;
IntMultiPos *f_list;
} IntUnitigMesg; // IUM
diff --git a/src/AS_MSG/AS_MSG_pmesg1.C b/src/AS_MSG/AS_MSG_pmesg1.C
index 19538f4..bf172a3 100644
--- a/src/AS_MSG/AS_MSG_pmesg1.C
+++ b/src/AS_MSG/AS_MSG_pmesg1.C
@@ -18,7 +18,7 @@
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static char *rcsid= "$Id: AS_MSG_pmesg1.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static char *rcsid= "$Id: AS_MSG_pmesg1.C 4518 2014-03-31 20:11:04Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -591,7 +591,7 @@ Read_IUM_Mesg(FILE *fin) {
mesg.status = (UnitigStatus)GetType("sta:%1[UCNSX]","status", fin);
// flag for handling unitig
- mesg.unique_rept = (UnitigFUR)GetType("fur:%1[XUR]","unique_rept", fin);
+ mesg.unique_rept = 'X'; //(UnitigFUR)GetType("fur:%1[XUR]","unique_rept", fin);
GET_FIELD(mesg.length,"len:"F_S32,"length field");
mesg.consensus = GetText("cns:",fin,TRUE);
@@ -1455,7 +1455,7 @@ static void Write_IUM_Mesg(FILE *fout, void *vmesg)
fprintf(fout,"cov:%.3f\n",mesg->coverage_stat);
fprintf(fout,"mhp:%.3f\n",mesg->microhet_prob);
fprintf(fout,"sta:%c\n",mesg->status);
- fprintf(fout,"fur:%c\n",mesg->unique_rept);
+ fprintf(fout,"fur:%c\n", 'X'); // mesg->unique_rept
fprintf(fout,"len:"F_S32"\n",mesg->length);
PutText(fout,"cns:",mesg->consensus,TRUE);
PutText(fout,"qlt:",mesg->quality,TRUE);
diff --git a/src/AS_MSG/AS_MSG_types.H b/src/AS_MSG/AS_MSG_types.H
index 61bac13..19e61f6 100644
--- a/src/AS_MSG/AS_MSG_types.H
+++ b/src/AS_MSG/AS_MSG_types.H
@@ -18,12 +18,12 @@
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-/* $Id: AS_MSG_types.H 4371 2013-08-01 17:19:47Z brianwalenz $ */
+/* $Id: AS_MSG_types.H 4518 2014-03-31 20:11:04Z brianwalenz $ */
#ifndef AS_MSG_PMESG_TYPES_H
#define AS_MSG_PMESG_TYPES_H
-static const char *rcsid_AS_MSG_PMESG_TYPES_H = "$Id: AS_MSG_types.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_AS_MSG_PMESG_TYPES_H = "$Id: AS_MSG_types.H 4518 2014-03-31 20:11:04Z brianwalenz $";
#include <cstdio>
#include <ctime>
@@ -350,12 +350,6 @@ typedef enum {
AS_UNASSIGNED = (int)'X'
} UnitigStatus;
-typedef enum {
- AS_FORCED_NONE = (int)'X',
- AS_FORCED_UNIQUE = (int)'U',
- AS_FORCED_REPEAT = (int)'R'
-} UnitigFUR;
-
/* IMP message */
typedef enum {
diff --git a/src/AS_MSG/tracedb-to-frg.pl b/src/AS_MSG/tracedb-to-frg.pl
index d22f840..6240250 100644
--- a/src/AS_MSG/tracedb-to-frg.pl
+++ b/src/AS_MSG/tracedb-to-frg.pl
@@ -215,6 +215,11 @@ sub readXML () {
}
if (m!^\s*<TRACE_END>(.*)<!i) {
$end = $1;
+
+ $end = "F" if ($end eq "FORWARD");
+ $end = "R" if ($end eq "REVERSE");
+
+ die "Invalid end=$end for TI=$xid.\n" if (($end ne "F") && ($end ne "R"));
}
if (m!^\s*<SEQ_LIB_ID>(.*)</!i) {
if ($useSLI != 0) {
@@ -1027,9 +1032,6 @@ sub runNBL ($) {
my ($sid, $seq) = readFasta(0);
my ($qid, $qlt) = readQual(0);
- $end = "F" if ($end eq "FORWARD");
- $end = "R" if ($end eq "REVERSE");
-
if (($type eq "WGS") ||
($type eq "SHOTGUN") ||
($type eq "CLONEEND") ||
diff --git a/src/AS_OBT/._AS_OBT_acceptableOverlap.C b/src/AS_OBT/._AS_OBT_acceptableOverlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._AS_OBT_acceptableOverlap.C differ
diff --git a/src/AS_OBT/._AS_OBT_acceptableOverlap.H b/src/AS_OBT/._AS_OBT_acceptableOverlap.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._AS_OBT_acceptableOverlap.H differ
diff --git a/src/AS_OBT/._AS_OBT_overlaps.C b/src/AS_OBT/._AS_OBT_overlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._AS_OBT_overlaps.C differ
diff --git a/src/AS_OBT/._AS_OBT_overlaps.H b/src/AS_OBT/._AS_OBT_overlaps.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._AS_OBT_overlaps.H differ
diff --git a/src/AS_OBT/._Makefile b/src/AS_OBT/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._Makefile differ
diff --git a/src/AS_OBT/._chimera.C b/src/AS_OBT/._chimera.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._chimera.C differ
diff --git a/src/AS_OBT/._deduplicate.C b/src/AS_OBT/._deduplicate.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._deduplicate.C differ
diff --git a/src/AS_OBT/._detect-unsplit-subreads-in-overlaps.pl b/src/AS_OBT/._detect-unsplit-subreads-in-overlaps.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._detect-unsplit-subreads-in-overlaps.pl differ
diff --git a/src/AS_OBT/._finalTrim-bestEdge.C b/src/AS_OBT/._finalTrim-bestEdge.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim-bestEdge.C differ
diff --git a/src/AS_OBT/._finalTrim-consolidate.H b/src/AS_OBT/._finalTrim-consolidate.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim-consolidate.H differ
diff --git a/src/AS_OBT/._finalTrim-evidenceBased.C b/src/AS_OBT/._finalTrim-evidenceBased.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim-evidenceBased.C differ
diff --git a/src/AS_OBT/._finalTrim-largestCovered.C b/src/AS_OBT/._finalTrim-largestCovered.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim-largestCovered.C differ
diff --git a/src/AS_OBT/._finalTrim-stats.pl b/src/AS_OBT/._finalTrim-stats.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim-stats.pl differ
diff --git a/src/AS_OBT/._finalTrim.C b/src/AS_OBT/._finalTrim.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim.C differ
diff --git a/src/AS_OBT/._finalTrim.H b/src/AS_OBT/._finalTrim.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._finalTrim.H differ
diff --git a/src/AS_OBT/._generate-random-chimeric-fragments.pl b/src/AS_OBT/._generate-random-chimeric-fragments.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._generate-random-chimeric-fragments.pl differ
diff --git a/src/AS_OBT/._initialTrim.C b/src/AS_OBT/._initialTrim.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._initialTrim.C differ
diff --git a/src/AS_OBT/._removeMateOverlap.C b/src/AS_OBT/._removeMateOverlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._removeMateOverlap.C differ
diff --git a/src/AS_OBT/._test-random-chimeric-fragments.pl b/src/AS_OBT/._test-random-chimeric-fragments.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._test-random-chimeric-fragments.pl differ
diff --git a/src/AS_OBT/._trim.C b/src/AS_OBT/._trim.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._trim.C differ
diff --git a/src/AS_OBT/._trim.H b/src/AS_OBT/._trim.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OBT/._trim.H differ
diff --git a/src/AS_OBT/AS_OBT_overlaps.C b/src/AS_OBT/AS_OBT_overlaps.C
index 8fb184e..56df215 100644
--- a/src/AS_OBT/AS_OBT_overlaps.C
+++ b/src/AS_OBT/AS_OBT_overlaps.C
@@ -23,6 +23,9 @@ static const char *rcsid = "$Id: $";
#include "AS_OBT_overlaps.H"
+// Loads overlaps for whatever read is next in the store.
+// Returns 0 if the overlaps are NOT for the iid supplied, otherwise, returns the number of overlaps.
+
uint32
loadOverlaps(uint32 iid,
OVSoverlap *&ovl,
@@ -39,9 +42,13 @@ loadOverlaps(uint32 iid,
ovl = new OVSoverlap [ovlMax];
}
- // Return if the overlaps are for the current read, or some future read.
- if (iid <= ovl[0].a_iid)
+ if (iid < ovl[0].a_iid)
+ // Overlaps loaded are for a future read.
+ return(0);
+
+ if (iid == ovl[0].a_iid)
+ // Overlaps loaded are for this read, YAY!
return(ovlLen);
// Until we load the correct overlap, repeat.
@@ -53,8 +60,10 @@ loadOverlaps(uint32 iid,
ovlLen += AS_OVS_readOverlapsFromStore(ovlSecondary, NULL, 0, AS_OVS_TYPE_ANY);
// Quit now if there are no overlaps. This simplifies the rest of the loop.
- if (ovlLen == 0)
+ if (ovlLen == 0) {
+ //fprintf(stderr, "no more overlaps.\n");
return(0);
+ }
// Allocate space for these overlaps.
while (ovlMax < ovlLen) {
@@ -71,24 +80,45 @@ loadOverlaps(uint32 iid,
//fprintf(stderr, "LOADED %d overlaps for a_iid %d\n", ovlLen, ovl[0].a_iid);
// If we read overlaps for a fragment after 'iid', we're done. The client will properly save
- // these overlaps until the iid becomes active. And just in case it doesn't, we return above
- // if the iid passed in is less than the current overlap.
+ // these overlaps until the iid becomes active.
//
- if (iid <= ovl[0].a_iid)
+ if (iid < ovl[0].a_iid) {
+ //fprintf(stderr, "looking for iid %u, read iid %u, stop, return no overlaps.\n", iid, ovl[0].a_iid);
+ return(0);
+ }
+
+ // If we've found the overlaps, we're still done.
+ //
+ if (iid == ovl[0].a_iid) {
+ //fprintf(stderr, "looking for iid %u, read iid %u, stop, return %u overlaps.\n", iid, ovl[0].a_iid, ovlLen);
return(ovlLen);
+ }
- // On the otherhand, if we read overlaps for a fragment before 'iid', we can either keep reading
- // until we find the overlaps for this fragment, or jump to the correct spot to read overlaps.
+ // On the otherhand, if we read overlaps for a fragment before 'iid', we can either keep
+ // reading until we find the overlaps for this fragment, or jump to the correct spot to read
+ // overlaps.
//
// The rule is simple. If we're within 50 of the correct IID, keep streaming. Otherwise, make
- // a jump. AS_OVS_setRangeOverlapStore() seems to ALWAYS close and open a file, which is somewhat
- // expensive, especially if the file doesn't actually change.
+ // a jump. AS_OVS_setRangeOverlapStore() seems to ALWAYS close and open a file, which is
+ // somewhat expensive, especially if the file doesn't actually change.
//
if (50 < iid - ovl[0].a_iid) {
+ //fprintf(stderr, "looking for iid %u, read iid %u, skip ahead to iid.\n", iid, ovl[0].a_iid);
AS_OVS_setRangeOverlapStore(ovlPrimary, iid, UINT32_MAX);
AS_OVS_setRangeOverlapStore(ovlSecondary, iid, UINT32_MAX);
+ //} else {
+ // fprintf(stderr, "looking for iid %u, read iid %u, skip ahead by 1.\n", iid, ovl[0].a_iid);
}
+
} while (ovl[0].a_iid < iid);
+ // Code can't get here.
+ // If we ran out of overlaps, the first return is used.
+ // If we read past the iid we're looking for, the second return is used.
+ // If we found the overlaps we're looking for, the thrd return is used.
+ // If we didn't find the overlaps, we loop.
+
+ assert(0);
+
return(ovlLen);
}
diff --git a/src/AS_OBT/chimera.C b/src/AS_OBT/chimera.C
index 4706f32..db3c9bd 100644
--- a/src/AS_OBT/chimera.C
+++ b/src/AS_OBT/chimera.C
@@ -19,12 +19,12 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: chimera.C 4482 2013-12-11 15:26:54Z brianwalenz $";
+const char *mainid = "$Id: chimera.C 4572 2014-10-14 23:06:05Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
#include "AS_UTL_decodeRange.H"
#include "AS_OBT_overlaps.H"
@@ -646,7 +646,7 @@ checkSpanningMates(const AS_IID iid,
const chimeraClear *clear,
gkStore *gkp,
vector<chimeraOvl> &olist,
- intervalList &IL) {
+ intervalList<int32> &IL) {
// Build a list of mated reads internal to this fragment. We have choice of algorithm:
// N log N -- build a set<> of the IDs for lookup
@@ -657,7 +657,7 @@ checkSpanningMates(const AS_IID iid,
// second list of <readID, mateID, position>, also sorted. The two lists are scanned, and
// whenever we find mateID (list 1) == readID (list 2) we have a mate internal to this fragment.
- intervalList CC;
+ intervalList<int32> CC;
//fprintf(reportFile, "checkSpanningMates()-- for read "F_IID"\n", iid);
@@ -753,7 +753,7 @@ checkSpanningMates(const AS_IID iid,
CC.merge();
for (uint32 i=0; i<CC.numberOfIntervals(); i++) {
- if (CC.ct(i) < 2)
+ if (CC.count(i) < 2)
continue;
#ifdef DEBUG_MATES
@@ -928,10 +928,10 @@ processChimera(const AS_IID iid,
- intervalList IL;
- uint32 hasPotentialChimera = 0;
- uint32 hasInniePair = 0;
- uint32 hasOverhang = 0;
+ intervalList<int32> IL;
+ uint32 hasPotentialChimera = 0;
+ uint32 hasInniePair = 0;
+ uint32 hasOverhang = 0;
// These types are described in Bri's VI Notebook #2 page 33.
@@ -1402,10 +1402,10 @@ processChimera(const AS_IID iid,
if (IL.numberOfIntervals() > 1) {
uint32 aveOlaps = 0;
for (uint32 interval=0; interval<IL.numberOfIntervals()-1; interval++)
- aveOlaps += IL.ct(interval);
+ aveOlaps += IL.count(interval);
aveOlaps /= (IL.numberOfIntervals() - 1);
- if (IL.ct(IL.numberOfIntervals()-1) < 0.33 * aveOlaps)
+ if (IL.count(IL.numberOfIntervals()-1) < 0.33 * aveOlaps)
ignoreLast = true;
}
@@ -1562,8 +1562,8 @@ processSubRead(const AS_IID iid,
map<AS_IID, uint32> numOlaps;
bool largePalindrome = false;
- intervalList BAD;
- intervalList BADall;
+ intervalList<int32> BAD;
+ intervalList<int32> BADall;
for (uint32 ii=0; ii<olist.size(); ii++) {
//fprintf(stderr, "COUNT OLAPS B %u %u-%u\n", olist[ii].Biid, olist[ii].Bbeg, olist[ii].Bend);
@@ -1747,9 +1747,9 @@ processSubRead(const AS_IID iid,
#if 0
if (subreadFile) {
for (uint32 bb=0; bb<BAD.numberOfIntervals(); bb++)
- fprintf(subreadFile, "BAD[%d] %ld-%ld %d hits\n", bb, BAD.lo(bb), BAD.hi(bb), BAD.ct(bb));
+ fprintf(subreadFile, "BAD[%d] %ld-%ld %d hits\n", bb, BAD.lo(bb), BAD.hi(bb), BAD.count(bb));
for (uint32 bb=0; bb<BADall.numberOfIntervals(); bb++)
- fprintf(subreadFile, "BADall[%d] %ld-%ld %d hits\n", bb, BADall.lo(bb), BADall.hi(bb), BADall.ct(bb));
+ fprintf(subreadFile, "BADall[%d] %ld-%ld %d hits\n", bb, BADall.lo(bb), BADall.hi(bb), BADall.count(bb));
}
#endif
@@ -1763,7 +1763,7 @@ processSubRead(const AS_IID iid,
for (uint32 aa=0; aa<BADall.numberOfIntervals(); aa++)
if ((BADall.lo(aa) <= BAD.lo(bb)) && (BAD.hi(bb) <= BADall.hi(aa)))
- allHits += BADall.ct(aa);
+ allHits += BADall.count(aa);
assert(allHits != 0);
@@ -1775,8 +1775,8 @@ processSubRead(const AS_IID iid,
numSpan += (clear[olist[ii].Aiid].doCheckSubRead) ? 1 : 2;
if (subreadFile)
- fprintf(subreadFile, "AcheckSub region %u ("F_U64"-"F_U64") with %u hits %u bighits - span %u largePalindrome %s\n",
- olist[0].Aiid, BAD.lo(bb), BAD.hi(bb), BAD.ct(bb), allHits,
+ fprintf(subreadFile, "AcheckSub region %u ("F_S32"-"F_S32") with %u hits %u bighits - span %u largePalindrome %s\n",
+ olist[0].Aiid, BAD.lo(bb), BAD.hi(bb), BAD.count(bb), allHits,
numSpan, largePalindrome ? "true" : "false");
if (numSpan > 9)
@@ -1784,12 +1784,12 @@ processSubRead(const AS_IID iid,
// is plenty of evidence it is true.
continue;
- if (BAD.ct(bb) + allHits / 4 + largePalindrome < 3)
+ if (BAD.count(bb) + allHits / 4 + largePalindrome < 3)
// If 2 or fewer reads claim this is a sub read junction, skip it. Evidence is weak.
continue;
if (subreadFile)
- fprintf(subreadFile, "CONFIRMED BAD REGION %ld-%ld\n", BAD.lo(bb), BAD.hi(bb));
+ fprintf(subreadFile, "CONFIRMED BAD REGION %d-%d\n", BAD.lo(bb), BAD.hi(bb));
blist.push_back(chimeraBad(olist[0].Aiid, BAD.lo(bb), BAD.hi(bb)));
}
@@ -1989,7 +1989,7 @@ main(int argc, char **argv) {
// But first, if the bad interval just touches the clear, trim it out.
if (blist.size() > 0) {
- intervalList goodRegions;
+ intervalList<int32> goodRegions;
for (uint32 bb=0; bb<blist.size(); bb++) {
if ((blist[bb].end <= res.intervalBeg) ||
@@ -2049,7 +2049,7 @@ main(int argc, char **argv) {
uint32 len = goodRegions.hi(ii) - goodRegions.lo(ii);
if (subreadFile)
- fprintf(subreadFile, "BAD iid %u trim %u %u region %lu %lu SUBREAD_REGION len %u\n",
+ fprintf(subreadFile, "BAD iid %u trim %u %u region %d %d SUBREAD_REGION len %u\n",
res.iid, res.intervalBeg, res.intervalEnd, goodRegions.lo(ii), goodRegions.hi(ii), len);
if (goodLen < len) {
diff --git a/src/AS_OBT/detect-unsplit-subreads-in-overlaps.pl b/src/AS_OBT/detect-unsplit-subreads-in-overlaps.pl
new file mode 100644
index 0000000..50e808b
--- /dev/null
+++ b/src/AS_OBT/detect-unsplit-subreads-in-overlaps.pl
@@ -0,0 +1,95 @@
+#!/usr/bin/perl
+
+use strict;
+
+my %readLen;
+
+open(F, "gatekeeper -dumpfragments -tabular test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+ $readLen{$v[1]} = $v[9];
+}
+close(F);
+
+
+my %readHash;
+my %readConf;
+my $lastRead = 0;
+my $nOvl = 0;
+
+
+my $potSub = 0;
+my $proSub = 0;
+
+open(F, "overlapStore -d test.ovlStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ if ($lastRead != $v[0]) {
+ my $mCount = scalar(keys %readConf);
+
+ if ($mCount > 0) {
+ print "$lastRead\t$mCount\n";
+ $potSub++;
+ }
+
+ if ($mCount > 5) {
+ $proSub++;
+ }
+
+ $lastRead = $v[0];
+ $nOvl = 0;
+ undef %readHash;
+ undef %readConf;
+ }
+
+ next if (($v[3] < 0) && ($v[4] > 0)); # read is contained in other
+
+ $nOvl++;
+
+ if (! exists($readHash{$v[1]})) {
+ $readHash{$v[1]} = "$v[3] $v[4]";
+ next;
+ }
+
+ my ($na, $nb) = ($v[3], $v[4]);
+ my ($oa, $ob) = split '\s+', $readHash{$v[1]};
+
+ my $len = $readLen{$v[0]};
+
+ my $nbgn = ($na > 0) ? $na : 0;
+ my $nend = ($nb < 0) ? ($len + $nb) : $len;
+
+ my $obgn = ($oa > 0) ? $oa : 0;
+ my $oend = ($ob < 0) ? ($len + $ob) : $len;
+
+ #print "$_ -- $nbgn $nend - $obgn $oend\n";
+
+ goto saveConf if (($nbgn <= $obgn) && ($nend <= $obgn)); # n before o
+ goto saveConf if (($nbgn >= $oend) && ($nend >= $oend)); # n after o
+
+ my $min = ($nbgn < $obgn) ? $obgn : $nbgn; # Largest bgn coord
+ my $max = ($nend < $oend) ? $nend : $oend; # Smallest end coord
+
+ my $nlen = $nend - $nbgn;
+ my $olen = $oend - $obgn;
+
+ my $olap = $max - $min;
+
+ #print "$_ -- $olap $nlen $olen\n";
+
+ next if (($olap / $nlen > 0.5) || ($olap / $olen > 0.5)); # reads overlap by lots
+
+ # different regions, probably sub read detected
+ saveConf:
+ $readConf{$v[1]}++;
+
+ #print "$_\n";
+}
+close(F);
+
+print STDERR "potential unsplit subreads: $potSub\n";
+print STDERR "probable unsplit subreads: $proSub\n";
diff --git a/src/AS_OBT/finalTrim-bestEdge.C b/src/AS_OBT/finalTrim-bestEdge.C
index f21379c..8af55f9 100644
--- a/src/AS_OBT/finalTrim-bestEdge.C
+++ b/src/AS_OBT/finalTrim-bestEdge.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: finalTrim-bestEdge.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: finalTrim-bestEdge.C 4604 2015-02-11 00:49:58Z brianwalenz $";
#include "finalTrim.H"
@@ -122,7 +122,7 @@ bestEdge(OVSoverlap *ovl,
assert(iid == ovl[i].a_iid);
- if ((ovl->dat.obt.erate > errorRate) &&
+ if ((ovl[i].dat.obt.erate > errorRate) &&
(tend - tbgn) * AS_OVS_decodeQuality(ovl[i].dat.obt.erate) > errorLimit)
// Overlap is crappy.
continue;
diff --git a/src/AS_OBT/finalTrim-largestCovered.C b/src/AS_OBT/finalTrim-largestCovered.C
index 7fa15a9..06ee24b 100644
--- a/src/AS_OBT/finalTrim-largestCovered.C
+++ b/src/AS_OBT/finalTrim-largestCovered.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: finalTrim-largestCovered.C 4481 2013-12-06 16:18:29Z brianwalenz $";
+static const char *rcsid = "$Id: finalTrim-largestCovered.C 4604 2015-02-11 00:49:58Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -31,7 +31,7 @@ static const char *rcsid = "$Id: finalTrim-largestCovered.C 4481 2013-12-06 16:1
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
#include "AS_OVS_overlapStore.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
bool
@@ -66,9 +66,9 @@ largestCovered(OVSoverlap *ovl,
return(false);
}
- intervalList IL;
- intervalList ID;
- int32 iid = fr.gkFragment_getReadIID();
+ intervalList<uint32> IL;
+ intervalList<uint32> ID;
+ int32 iid = fr.gkFragment_getReadIID();
for (uint32 i=0; i<ovlLen; i++) {
uint32 tbgn = ibgn + ovl[i].dat.obt.a_beg;
@@ -77,7 +77,7 @@ largestCovered(OVSoverlap *ovl,
assert(tbgn < tend);
assert(iid == ovl[i].a_iid);
- if ((ovl->dat.obt.erate > errorRate) &&
+ if ((ovl[i].dat.obt.erate > errorRate) &&
(tend - tbgn) * AS_OVS_decodeQuality(ovl[i].dat.obt.erate) > errorLimit)
// Overlap is crappy.
continue;
@@ -98,16 +98,16 @@ largestCovered(OVSoverlap *ovl,
// acceptable.
if (minCoverage > 0) {
- intervalDepth DE(IL);
+ intervalList<uint32> DE(IL);
uint32 it = 0;
uint32 ib = 0;
uint32 ie = 0;
while (it < DE.numberOfIntervals()) {
- //fprintf(stderr, "DE - %d - "F_S64" "F_S64" "F_U32"\n", fr.gkFragment_getReadIID(), DE.lo(it), DE.hi(it), DE.de(it));
+ //fprintf(stderr, "DE - %d - "F_S64" "F_S64" "F_U32"\n", fr.gkFragment_getReadIID(), DE.lo(it), DE.hi(it), DE.depth(it));
- if (DE.de(it) < minCoverage) {
+ if (DE.depth(it) < minCoverage) {
// Dropped below good coverage depth. If we have an interval, save it. Reset.
if (ie > ib) {
//fprintf(stderr, "AD1 %d-%d len %d\n", ib, ie, ie - ib);
@@ -164,7 +164,7 @@ largestCovered(OVSoverlap *ovl,
// -----------
if (minCoverage > 0) {
- intervalList FI;
+ intervalList<uint32> FI;
uint32 li = 0;
uint32 di = 0;
diff --git a/src/AS_OBT/finalTrim.C b/src/AS_OBT/finalTrim.C
index 39513b9..b3b4759 100644
--- a/src/AS_OBT/finalTrim.C
+++ b/src/AS_OBT/finalTrim.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: finalTrim.C 4481 2013-12-06 16:18:29Z brianwalenz $";
+const char *mainid = "$Id: finalTrim.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include "AS_global.H"
@@ -140,9 +140,12 @@ enforceMaximumClearRange(OVSoverlap *ovl,
int
main(int argc, char **argv) {
- gkStore *gkpStore = 0L;
- OverlapStore *ovlPrimary = 0L;
- OverlapStore *ovlSecondary = 0L;
+ char *gkpName = 0L;
+ gkStore *gkpStore = 0L;
+ char *ovlPrimaryName = 0L;
+ OverlapStore *ovlPrimary = 0L;
+ OverlapStore *ovlSecondary = 0L;
+ char *ovlSecondaryName = 0L;
uint32 errorRate = AS_OVS_encodeQuality(0.015);
double errorLimit = 2.5;
@@ -167,15 +170,13 @@ main(int argc, char **argv) {
int err=0;
while (arg < argc) {
if (strcmp(argv[arg], "-G") == 0) {
- gkpStore = new gkStore(argv[++arg], FALSE, doModify);
- gkpStore->gkStore_enableClearRange(AS_READ_CLEAR_OBTMERGE);
- gkpStore->gkStore_metadataCaching(true);
+ gkpName = argv[++arg];
} else if (strcmp(argv[arg], "-O") == 0) {
- if (ovlPrimary == NULL)
- ovlPrimary = AS_OVS_openOverlapStore(argv[++arg]);
+ if (ovlPrimaryName == NULL)
+ ovlPrimaryName = argv[++arg];
else if (ovlSecondary == NULL)
- ovlSecondary = AS_OVS_openOverlapStore(argv[++arg]);
+ ovlSecondaryName = argv[++arg];
else {
fprintf(stderr, "ERROR: Only two obtStores (-O) allowed.\n");
err++;
@@ -212,9 +213,9 @@ main(int argc, char **argv) {
arg++;
}
- if ((gkpStore == NULL) ||
- (ovlPrimary == NULL) ||
- (outputPrefix == NULL) ||
+ if ((gkpName == NULL) ||
+ (ovlPrimaryName == NULL) ||
+ (outputPrefix == NULL) ||
(err)) {
fprintf(stderr, "usage: %s -G gkpStore -O ovlStore [-O ovlStore] -o outputPrefix\n", argv[0]);
fprintf(stderr, " -e erate allow 'erate' percent error\n");
@@ -230,6 +231,18 @@ main(int argc, char **argv) {
exit(1);
}
+ gkpStore = new gkStore(gkpName, FALSE, doModify);
+
+ gkpStore->gkStore_enableClearRange(AS_READ_CLEAR_OBTMERGE);
+ gkpStore->gkStore_metadataCaching(true);
+
+ if (ovlPrimaryName != NULL)
+ ovlPrimary = AS_OVS_openOverlapStore(ovlPrimaryName);
+
+ if (ovlSecondary != NULL)
+ ovlSecondary = AS_OVS_openOverlapStore(ovlSecondaryName);
+
+
sprintf(logName, "%s.log", outputPrefix);
sprintf(sumName, "%s.summary", outputPrefix);
diff --git a/src/AS_OBT/finalTrim.H b/src/AS_OBT/finalTrim.H
index 2007028..71cde4a 100644
--- a/src/AS_OBT/finalTrim.H
+++ b/src/AS_OBT/finalTrim.H
@@ -22,7 +22,7 @@
#ifndef FINALTRIM_H
#define FINALTRIM_H
-static const char *rcsid_FINALTRIM_H = "$Id: finalTrim.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_FINALTRIM_H = "$Id: finalTrim.H 4571 2014-10-09 13:18:54Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -36,7 +36,7 @@ static const char *rcsid_FINALTRIM_H = "$Id: finalTrim.H 4371 2013-08-01 17:19:4
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
#include "AS_OVS_overlapStore.H"
-#include "AS_UTL_intervalList.H"
+#include "intervalList.H"
#define OBT_MODE_WIGGLE (5)
diff --git a/src/AS_OBT/trim.C b/src/AS_OBT/trim.C
index f8da312..8b1d05c 100644
--- a/src/AS_OBT/trim.C
+++ b/src/AS_OBT/trim.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: trim.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: trim.C 4523 2014-04-11 20:07:55Z brianwalenz $";
#include "trim.H"
@@ -123,7 +123,7 @@ findGoodQuality(double *qltD,
flen = fpos;
rlen = rpos;
- //fprintf(stderr, "qltLen = "uint32FMT" flen="uint32FMT" rlen="uint32FMT"\n", qltLen, flen, rlen);
+ //fprintf(stderr, "qltLen = "F_U32" flen="F_U32" rlen="F_U32"\n", qltLen, flen, rlen);
uint32 winningFPos = 0;
uint32 winningRPos = 0;
diff --git a/src/AS_OVL/._AS_OVL_olapstats.H b/src/AS_OVL/._AS_OVL_olapstats.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._AS_OVL_olapstats.H differ
diff --git a/src/AS_OVL/._CatCorrectsOVL.C b/src/AS_OVL/._CatCorrectsOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._CatCorrectsOVL.C differ
diff --git a/src/AS_OVL/._CatEratesOVL.C b/src/AS_OVL/._CatEratesOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._CatEratesOVL.C differ
diff --git a/src/AS_OVL/._CorrectOlapsOVL.C b/src/AS_OVL/._CorrectOlapsOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._CorrectOlapsOVL.C differ
diff --git a/src/AS_OVL/._FragCorrectOVL.C b/src/AS_OVL/._FragCorrectOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._FragCorrectOVL.C differ
diff --git a/src/AS_OVL/._FragCorrectOVL.H b/src/AS_OVL/._FragCorrectOVL.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._FragCorrectOVL.H differ
diff --git a/src/AS_OVL/._Makefile b/src/AS_OVL/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._Makefile differ
diff --git a/src/AS_OVL/._OlapFromSeedsOVL.C b/src/AS_OVL/._OlapFromSeedsOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._OlapFromSeedsOVL.C differ
diff --git a/src/AS_OVL/._OlapFromSeedsOVL.H b/src/AS_OVL/._OlapFromSeedsOVL.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._OlapFromSeedsOVL.H differ
diff --git a/src/AS_OVL/._SharedOVL.C b/src/AS_OVL/._SharedOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._SharedOVL.C differ
diff --git a/src/AS_OVL/._SharedOVL.H b/src/AS_OVL/._SharedOVL.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._SharedOVL.H differ
diff --git a/src/AS_OVL/._ShowCorrectsOVL.C b/src/AS_OVL/._ShowCorrectsOVL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._ShowCorrectsOVL.C differ
diff --git a/src/AS_OVL/._delcher.sty b/src/AS_OVL/._delcher.sty
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._delcher.sty differ
diff --git a/src/AS_OVL/._olap-seeds-doc.tex b/src/AS_OVL/._olap-seeds-doc.tex
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._olap-seeds-doc.tex differ
diff --git a/src/AS_OVL/._overlap_partition.C b/src/AS_OVL/._overlap_partition.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVL/._overlap_partition.C differ
diff --git a/src/AS_OVL/AS_OVL_delcher.C b/src/AS_OVL/AS_OVL_delcher.C
deleted file mode 100644
index b36e5e2..0000000
--- a/src/AS_OVL/AS_OVL_delcher.C
+++ /dev/null
@@ -1,188 +0,0 @@
-
-/**************************************************************************
- * This file is part of Celera Assembler, a software program that
- * assembles whole-genome shotgun reads into contigs and scaffolds.
- * Copyright (C) 1999-2004, Applera Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received (LICENSE.txt) a copy of the GNU General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *************************************************************************/
-
-static const char *rcsid = "$Id: AS_OVL_delcher.C 4371 2013-08-01 17:19:47Z brianwalenz $";
-
-/*********************************************************************
- Module: AS_OVL
- Description: Assembly Overlap Module. Computes overlaps between
- pairs of DNA strings.
- Definitions of functions declared in delcher.h
- Assumptions: Input meets specifications in the ProtoIO documents
- *********************************************************************/
-
-#include "AS_OVL_delcher.H"
-
-
-int Global_Debug_Flag = FALSE;
- // Flag for debugging
-int Verbose_Level = 0;
- // Determines amount of diagnostic printout
-
-
-
-int All_White_Space
- (const char * S)
-
-/* Return TRUE iff string S contains no characters other than
-* whitespace characters as defined by the builtin function
-* isspace () . */
-
- {
- while (* S != '\0' && isspace (* S))
- S ++;
-
- return (* S == '\0');
- }
-
-
-
-FILE * File_Open
- (const char * Filename, const char * Mode)
-
-/* Open Filename in Mode and return a pointer to its control
-* block. If fail, print a message and exit. */
-
- {
- FILE * fp;
- int retry;
-
- fp = fopen (Filename, Mode);
- for (retry = 0; fp == NULL && retry < 3; retry ++)
- {
- sleep (10);
- fp = fopen (Filename, Mode);
- }
- if (fp == NULL)
- {
- fprintf (stderr, "ERROR %d: Could not open file %s \n",
- errno, Filename);
- perror (strerror (errno));
- exit (1);
- }
-
- return fp;
- }
-
-
-
-int File_Exists (const char * Filename){
-
- /* Test for filename existence */
-
- FILE * fp;
-
- fp = fopen (Filename, "r+");
- if (fp)
- {
- fclose(fp);
- return 1;
- }
-
- return 0;
-
-}
-
-
-
-size_t Safe_fread
- (void * ptr, size_t size, size_t ct, FILE * fp)
-
-// Call fread (ptr, size, ct, fp) and check for errors
-
- {
- size_t result;
-
- result = fread (ptr, size, ct, fp);
- if (result != ct)
- {
- fprintf (stderr, "ERROR: fread failed\n");
- exit (1);
- }
-
- return result;
- }
-
-
-
-size_t Safe_fwrite
- (const void * ptr, size_t size, size_t ct, FILE * fp)
-
-// Call fwrite (ptr, size, ct, fp) and check for errors
-
- {
- size_t result;
-
- result = fwrite (ptr, size, ct, fp);
- if (result != ct)
- {
- fprintf (stderr, "ERROR: fwrite failed\n");
- exit (1);
- }
-
- return result;
- }
-
-
-int Safe_remove
- (char * filename)
-
-// Delete file named filename . Print error message and die if
-// fail.
-
- {
- int result;
-
- result = remove (filename);
-
- if (result != 0)
- {
- fprintf (stderr, "ERROR: Could not delete file \"%s\"\n",
- filename);
- exit (1);
- }
-
- return result;
- }
-
-
-
-int Safe_rename
- (char * oldname, char * newname)
-
-// Rename file oldname to newname . Print error message and die if
-// fail.
-
- {
- int result;
-
- result = rename (oldname, newname);
-
- if (result != 0)
- {
- fprintf (stderr, "ERROR: Could not rename file \"%s\" to \"%s\"\n",
- oldname, newname);
- exit (1);
- }
-
- return result;
- }
-
diff --git a/src/AS_OVL/AS_OVL_delcher.H b/src/AS_OVL/AS_OVL_delcher.H
deleted file mode 100644
index d90b3cb..0000000
--- a/src/AS_OVL/AS_OVL_delcher.H
+++ /dev/null
@@ -1,78 +0,0 @@
-
-/**************************************************************************
- * This file is part of Celera Assembler, a software program that
- * assembles whole-genome shotgun reads into contigs and scaffolds.
- * Copyright (C) 1999-2004, Applera Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received (LICENSE.txt) a copy of the GNU General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *************************************************************************/
-/*********************************************************************
- Module: AS_OVL
- Description: Assembly Overlap Module. Computes overlaps between
- pairs of DNA strings.
- Common include files and a few simple functions.
- Assumptions: Input meets specifications in the ProtoIO documents
- *********************************************************************/
-
-
-/* RCS info
- * $Id: AS_OVL_delcher.H 4371 2013-08-01 17:19:47Z brianwalenz $
- * $Revision: 1.10 $
-*/
-
-
-#ifndef __DELCHER_H_INCLUDED
-#define __DELCHER_H_INCLUDED
-
-static const char *rcsid_DELCHER_H_INCLUDED = "$Id: AS_OVL_delcher.H 4371 2013-08-01 17:19:47Z brianwalenz $";
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <math.h>
-#include <string.h>
-#include <ctype.h>
-#include <limits.h>
-#include <float.h>
-#include <time.h>
-#include <assert.h>
-#include <errno.h>
-#include <unistd.h>
-
-#include "AS_global.H"
-#include "AS_UTL_reverseComplement.H"
-
-extern int Global_Debug_Flag;
- // Flag for debugging
-extern int Verbose_Level;
- // Determines amount of diagnostic printout
-
-int All_White_Space
- (const char *);
-FILE * File_Open
- (const char *, const char *);
-int File_Exists
- (const char *);
-size_t Safe_fread
- (void * ptr, size_t size, size_t ct, FILE * fp);
-size_t Safe_fwrite
- (const void * ptr, size_t size, size_t ct, FILE * fp);
-int Safe_remove
- (char * filename);
-int Safe_rename
- (char * oldname, char * newname);
-
-
-#endif
-
diff --git a/src/AS_OVL/CorrectOlapsOVL.C b/src/AS_OVL/CorrectOlapsOVL.C
index 14bb557..29a6710 100644
--- a/src/AS_OVL/CorrectOlapsOVL.C
+++ b/src/AS_OVL/CorrectOlapsOVL.C
@@ -33,11 +33,10 @@
*
*************************************************/
-const char *mainid = "$Id: CorrectOlapsOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: CorrectOlapsOVL.C 4632 2015-03-11 14:54:45Z brianwalenz $";
#include "AS_global.H"
-#include "AS_OVL_delcher.H"
#include "AS_PER_gkpStore.H"
#include "AS_MSG_pmesg.H"
#include "AS_UTL_reverseComplement.H"
@@ -142,14 +141,14 @@ static char * Correct_File_Path;
// Name of file containing fragment corrections
static FILE * Delete_fp = NULL;
// File to which list of overlaps to delete is written if -x option is specified
-static int * Edit_Array [AS_READ_MAX_NORMAL_LEN+1];
+static int * Edit_Array_Lazy [AS_READ_MAX_NORMAL_LEN+1];
// Use for alignment calculation. Points into Edit_Space .
// (only MAX_ERRORS needed)
static int Edit_Match_Limit [AS_READ_MAX_NORMAL_LEN+1] = {0};
// This array [e] is the minimum value of Edit_Array [e] [d]
// to be worth pursuing in edit-distance computations between guides
// (only MAX_ERRORS needed)
-static int *Edit_Space = NULL;
+static int * Edit_Space_Lazy [AS_READ_MAX_NORMAL_LEN+1];
// Memory used by alignment calculation
// (only (MAX_ERRORS + 4) * MAX_ERRORS needed)
static int End_Exclude_Len = DEFAULT_END_EXCLUDE_LEN;
@@ -305,6 +304,7 @@ static int Union
static void Usage
(char * command);
+int Verbose_Level = 0;
int main
@@ -709,7 +709,10 @@ static void Correct_Frags
int correcting = FALSE;
AS_IID iid = 0;
- fp = File_Open (Correct_File_Path, "rb");
+ errno = 0;
+ fp = fopen(Correct_File_Path, "rb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Correct_File_Path, strerror(errno)), exit(1);
while (fread (& msg, sizeof (Correction_Output_t), 1, fp) == 1)
{
@@ -896,18 +899,22 @@ static void Dump_Erate_File
uint16 * erate = NULL;
int i;
- fp = File_Open (path, "wb");
+ errno = 0;
+ fp = fopen(path, "wb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", path, strerror(errno)), exit(1);
header [0] = lo_id;
header [1] = hi_id;
- Safe_fwrite (header, sizeof (int32), 2, fp);
- Safe_fwrite (&num, sizeof (uint64), 1, fp);
+
+ AS_UTL_safeWrite(fp, header, "header", sizeof (int32), 2);
+ AS_UTL_safeWrite(fp, &num, "num", sizeof (uint64), 1);
erate = (uint16 *) safe_malloc (num * sizeof(uint16));
for (i = 0; i < num; i ++)
erate [i] = olap [i] . corr_erate;
- Safe_fwrite (erate, sizeof (uint16), num, fp);
+ AS_UTL_safeWrite(fp, erate, "erate", sizeof (uint16), num);
safe_free(erate);
@@ -1162,28 +1169,6 @@ static void Initialize_Globals
// Initialize global variables used in this program
{
- int32 w = 2;
- int32 offset = 2;
- int32 del = 6;
- for (int32 i = 0; i < MAX_ERRORS; i ++) {
- offset += del;
- del += 2;
- }
-
- fprintf(stderr, "Allocating %d words for Edit_Space.\n", offset);
-
- Edit_Space = (int32 *)safe_malloc(sizeof(int32) * offset);
-
- memset(Edit_Space, 0, sizeof(int32) * offset);
-
- offset = w;
- del = 6;
- for (int32 i = 0; i < MAX_ERRORS; i ++) {
- Edit_Array [i] = Edit_Space + offset;
- offset += del;
- del += 2;
- }
-
for (int32 i = 0; i <= ERRORS_FOR_FREE; i ++)
Edit_Match_Limit [i] = 0;
@@ -1507,7 +1492,12 @@ static void Parse_Command_Line
break;
case 'o' :
- OVL_fp = File_Open (optarg, "w");
+ {
+ errno = 0;
+ OVL_fp = fopen(optarg, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", optarg, strerror(errno)), exit(1);
+ }
break;
case 'q' :
@@ -1525,7 +1515,12 @@ static void Parse_Command_Line
break;
case 'X' :
- Delete_fp = File_Open (optarg, "w");
+ {
+ errno = 0;
+ Delete_fp = fopen(optarg, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", optarg, strerror(errno)), exit(1);
+ }
break;
case '?' :
@@ -1583,6 +1578,78 @@ static void Parse_Command_Line
+
+
+// Allocate another block of 64mb for edits
+
+// Needs to be at least:
+// 52,432 to handle 40% error at 64k overlap
+// 104,860 to handle 80% error at 64k overlap
+// 209,718 to handle 40% error at 256k overlap
+// 419,434 to handle 80% error at 256k overlap
+// 3,355,446 to handle 40% error at 4m overlap
+// 6,710,890 to handle 80% error at 4m overlap
+// Bigger means we can assign more than one Edit_Array[] in one allocation.
+
+uint32 EDIT_SPACE_SIZE = 16 * 1024 * 1024;
+
+static
+void
+Allocate_More_Edit_Space(void) {
+
+ // Determine the last allocated block, and the last assigned block
+
+ int32 b = 0; // Last edit array assigned
+ int32 e = 0; // Last edit array assigned more space
+ int32 a = 0; // Last allocated block
+
+ while (Edit_Array_Lazy[b] != NULL)
+ b++;
+
+ while (Edit_Space_Lazy[a] != NULL)
+ a++;
+
+ // Fill in the edit space array. Well, not quite yet. First, decide the minimum size.
+ //
+ // Element [0] can access from [-2] to [2] = 5 elements.
+ // Element [1] can access from [-3] to [3] = 7 elements.
+ //
+ // Element [e] can access from [-2-e] to [2+e] = 5 + e * 2 elements
+ //
+ // So, our offset for this new block needs to put [e][0] at offset...
+
+ int32 Offset = 2 + b;
+ int32 Del = 6 + b * 2;
+ int32 Size = EDIT_SPACE_SIZE;
+
+ while (Size < Offset + Del)
+ Size *= 2;
+
+ // Allocate another block
+
+ Edit_Space_Lazy[a] = new int [Size];
+
+ // And, now, fill in the edit space array.
+
+ e = b;
+
+ while ((Offset + Del < Size) &&
+ (e < MAX_ERRORS)) {
+ Edit_Array_Lazy[e++] = Edit_Space_Lazy[a] + Offset;
+
+ Offset += Del;
+ Del += 2;
+ }
+
+ if (e == b)
+ fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e);
+ assert(e != b);
+}
+
+
+
+
+
static int Prefix_Edit_Dist
(char A [], int m, char T [], int n, int Error_Limit,
int * A_End, int * T_End, int * Match_To_End,
@@ -1616,7 +1683,10 @@ static int Prefix_Edit_Dist
for (Row = 0; Row < shorter && A [Row] == T [Row]; Row ++)
;
- Edit_Array [0] [0] = Row;
+ if (Edit_Array_Lazy[0] == NULL)
+ Allocate_More_Edit_Space();
+
+ Edit_Array_Lazy [0] [0] = Row;
if (Row == shorter) // Exact match
{
@@ -1632,17 +1702,21 @@ static int Prefix_Edit_Dist
{
Left = OVL_Max_int (Left - 1, -e);
Right = OVL_Min_int (Right + 1, e);
- Edit_Array [e - 1] [Left] = -2;
- Edit_Array [e - 1] [Left - 1] = -2;
- Edit_Array [e - 1] [Right] = -2;
- Edit_Array [e - 1] [Right + 1] = -2;
+
+ if (Edit_Array_Lazy[e] == NULL)
+ Allocate_More_Edit_Space();
+
+ Edit_Array_Lazy [e - 1] [Left] = -2;
+ Edit_Array_Lazy [e - 1] [Left - 1] = -2;
+ Edit_Array_Lazy [e - 1] [Right] = -2;
+ Edit_Array_Lazy [e - 1] [Right + 1] = -2;
for (d = Left; d <= Right; d ++)
{
- Row = 1 + Edit_Array [e - 1] [d];
- if ((j = Edit_Array [e - 1] [d - 1]) > Row)
+ Row = 1 + Edit_Array_Lazy [e - 1] [d];
+ if ((j = Edit_Array_Lazy [e - 1] [d - 1]) > Row)
Row = j;
- if ((j = 1 + Edit_Array [e - 1] [d + 1]) > Row)
+ if ((j = 1 + Edit_Array_Lazy [e - 1] [d + 1]) > Row)
Row = j;
while (Row < m && Row + d < n
&& A [Row] == T [Row + d])
@@ -1651,19 +1725,19 @@ static int Prefix_Edit_Dist
assert(e < MAX_ERRORS);
//assert(d < ??);
- Edit_Array [e] [d] = Row;
+ Edit_Array_Lazy [e] [d] = Row;
if (Row == m || Row + d == n)
{
#if 1
// Force last error to be mismatch rather than insertion
if (Row == m
- && 1 + Edit_Array [e - 1] [d + 1]
- == Edit_Array [e] [d]
+ && 1 + Edit_Array_Lazy [e - 1] [d + 1]
+ == Edit_Array_Lazy [e] [d]
&& d < Right)
{
d ++;
- Edit_Array [e] [d] = Edit_Array [e] [d - 1];
+ Edit_Array_Lazy [e] [d] = Edit_Array_Lazy [e] [d - 1];
}
#endif
(* A_End) = Row; // One past last align position
@@ -1674,13 +1748,13 @@ static int Prefix_Edit_Dist
for (k = e; k > 0; k --)
{
From = d;
- Max = 1 + Edit_Array [k - 1] [d];
- if ((j = Edit_Array [k - 1] [d - 1]) > Max)
+ Max = 1 + Edit_Array_Lazy [k - 1] [d];
+ if ((j = Edit_Array_Lazy [k - 1] [d - 1]) > Max)
{
From = d - 1;
Max = j;
}
- if ((j = 1 + Edit_Array [k - 1] [d + 1]) > Max)
+ if ((j = 1 + Edit_Array_Lazy [k - 1] [d + 1]) > Max)
{
From = d + 1;
Max = j;
@@ -1689,13 +1763,13 @@ static int Prefix_Edit_Dist
{
Delta_Stack [(* Delta_Len) ++] = Max - Last - 1;
d --;
- Last = Edit_Array [k - 1] [From];
+ Last = Edit_Array_Lazy [k - 1] [From];
}
else if (From == d + 1)
{
Delta_Stack [(* Delta_Len) ++] = Last - (Max - 1);
d ++;
- Last = Edit_Array [k - 1] [From];
+ Last = Edit_Array_Lazy [k - 1] [From];
}
}
Delta_Stack [(* Delta_Len) ++] = Last + 1;
@@ -1731,28 +1805,28 @@ static int Prefix_Edit_Dist
}
while (Left <= Right && Left < 0
- && Edit_Array [e] [Left] < Edit_Match_Limit [e])
+ && Edit_Array_Lazy [e] [Left] < Edit_Match_Limit [e])
Left ++;
if (Left >= 0)
while (Left <= Right
- && Edit_Array [e] [Left] + Left < Edit_Match_Limit [e])
+ && Edit_Array_Lazy [e] [Left] + Left < Edit_Match_Limit [e])
Left ++;
if (Left > Right)
break;
while (Right > 0
- && Edit_Array [e] [Right] + Right < Edit_Match_Limit [e])
+ && Edit_Array_Lazy [e] [Right] + Right < Edit_Match_Limit [e])
Right --;
if (Right <= 0)
- while (Edit_Array [e] [Right] < Edit_Match_Limit [e])
+ while (Edit_Array_Lazy [e] [Right] < Edit_Match_Limit [e])
Right --;
assert (Left <= Right);
for (d = Left; d <= Right; d ++)
- if (Edit_Array [e] [d] > Longest)
+ if (Edit_Array_Lazy [e] [d] > Longest)
{
Best_d = d;
Best_e = e;
- Longest = Edit_Array [e] [d];
+ Longest = Edit_Array_Lazy [e] [d];
}
#if 1
Score = Longest * BRANCH_PT_MATCH_VALUE - e;
@@ -2078,7 +2152,10 @@ static void Read_Olaps
& Olap, & Num_Olaps);
else
{
- fp = File_Open (Olap_Path, "r");
+ errno = 0;
+ fp = fopen(Olap_Path, "r");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Olap_Path, strerror(errno)), exit(1);
olap_size = 1000;
Olap = (Olap_Info_t *) safe_malloc
@@ -2156,7 +2233,10 @@ static void Redo_Olaps
gkpStore = new gkStore (gkpStore_Path, FALSE, FALSE);
Frag_Stream = new gkStream (gkpStore, lo_frag, hi_frag, GKFRAGMENT_SEQ);
- fp = File_Open (Correct_File_Path, "rb");
+ errno = 0;
+ fp = fopen(Correct_File_Path, "rb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Correct_File_Path, strerror(errno)), exit(1);
Correction_t *correct = new Correction_t [AS_READ_MAX_NORMAL_LEN];
Adjust_t *adjust = new Adjust_t [AS_READ_MAX_NORMAL_LEN];
@@ -2283,7 +2363,7 @@ static int Rev_Prefix_Edit_Dist
for (Row = 0; Row < shorter && A [- Row] == T [- Row]; Row ++)
;
- Edit_Array [0] [0] = Row;
+ Edit_Array_Lazy [0] [0] = Row;
if (Row == shorter) // Exact match
{
@@ -2299,34 +2379,34 @@ static int Rev_Prefix_Edit_Dist
{
Left = OVL_Max_int (Left - 1, -e);
Right = OVL_Min_int (Right + 1, e);
- Edit_Array [e - 1] [Left] = -2;
- Edit_Array [e - 1] [Left - 1] = -2;
- Edit_Array [e - 1] [Right] = -2;
- Edit_Array [e - 1] [Right + 1] = -2;
+ Edit_Array_Lazy [e - 1] [Left] = -2;
+ Edit_Array_Lazy [e - 1] [Left - 1] = -2;
+ Edit_Array_Lazy [e - 1] [Right] = -2;
+ Edit_Array_Lazy [e - 1] [Right + 1] = -2;
for (d = Left; d <= Right; d ++)
{
- Row = 1 + Edit_Array [e - 1] [d];
- if ((j = Edit_Array [e - 1] [d - 1]) > Row)
+ Row = 1 + Edit_Array_Lazy [e - 1] [d];
+ if ((j = Edit_Array_Lazy [e - 1] [d - 1]) > Row)
Row = j;
- if ((j = 1 + Edit_Array [e - 1] [d + 1]) > Row)
+ if ((j = 1 + Edit_Array_Lazy [e - 1] [d + 1]) > Row)
Row = j;
while (Row < m && Row + d < n
&& A [- Row] == T [- Row - d])
Row ++;
- Edit_Array [e] [d] = Row;
+ Edit_Array_Lazy [e] [d] = Row;
if (Row == m || Row + d == n)
{
// Force last error to be mismatch rather than insertion
if (Row == m
- && 1 + Edit_Array [e - 1] [d + 1]
- == Edit_Array [e] [d]
+ && 1 + Edit_Array_Lazy [e - 1] [d + 1]
+ == Edit_Array_Lazy [e] [d]
&& d < Right)
{
d ++;
- Edit_Array [e] [d] = Edit_Array [e] [d - 1];
+ Edit_Array_Lazy [e] [d] = Edit_Array_Lazy [e] [d - 1];
}
(* A_End) = - Row; // One past last align position
@@ -2337,13 +2417,13 @@ static int Rev_Prefix_Edit_Dist
for (k = e; k > 0; k --)
{
From = d;
- Max = 1 + Edit_Array [k - 1] [d];
- if ((j = Edit_Array [k - 1] [d - 1]) > Max)
+ Max = 1 + Edit_Array_Lazy [k - 1] [d];
+ if ((j = Edit_Array_Lazy [k - 1] [d - 1]) > Max)
{
From = d - 1;
Max = j;
}
- if ((j = 1 + Edit_Array [k - 1] [d + 1]) > Max)
+ if ((j = 1 + Edit_Array_Lazy [k - 1] [d + 1]) > Max)
{
From = d + 1;
Max = j;
@@ -2352,13 +2432,13 @@ static int Rev_Prefix_Edit_Dist
{
Delta_Stack [(* Delta_Len) ++] = Max - Last - 1;
d --;
- Last = Edit_Array [k - 1] [From];
+ Last = Edit_Array_Lazy [k - 1] [From];
}
else if (From == d + 1)
{
Delta_Stack [(* Delta_Len) ++] = Last - (Max - 1);
d ++;
- Last = Edit_Array [k - 1] [From];
+ Last = Edit_Array_Lazy [k - 1] [From];
}
}
Delta_Stack [(* Delta_Len) ++] = Last + 1;
@@ -2394,28 +2474,28 @@ static int Rev_Prefix_Edit_Dist
}
while (Left <= Right && Left < 0
- && Edit_Array [e] [Left] < Edit_Match_Limit [e])
+ && Edit_Array_Lazy [e] [Left] < Edit_Match_Limit [e])
Left ++;
if (Left >= 0)
while (Left <= Right
- && Edit_Array [e] [Left] + Left < Edit_Match_Limit [e])
+ && Edit_Array_Lazy [e] [Left] + Left < Edit_Match_Limit [e])
Left ++;
if (Left > Right)
break;
while (Right > 0
- && Edit_Array [e] [Right] + Right < Edit_Match_Limit [e])
+ && Edit_Array_Lazy [e] [Right] + Right < Edit_Match_Limit [e])
Right --;
if (Right <= 0)
- while (Edit_Array [e] [Right] < Edit_Match_Limit [e])
+ while (Edit_Array_Lazy [e] [Right] < Edit_Match_Limit [e])
Right --;
assert (Left <= Right);
for (d = Left; d <= Right; d ++)
- if (Edit_Array [e] [d] > Longest)
+ if (Edit_Array_Lazy [e] [d] > Longest)
{
Best_d = d;
Best_e = e;
- Longest = Edit_Array [e] [d];
+ Longest = Edit_Array_Lazy [e] [d];
}
#if 1
Score = Longest * BRANCH_PT_MATCH_VALUE - e;
diff --git a/src/AS_OVL/FragCorrectOVL.C b/src/AS_OVL/FragCorrectOVL.C
index 74da889..40c4dfb 100644
--- a/src/AS_OVL/FragCorrectOVL.C
+++ b/src/AS_OVL/FragCorrectOVL.C
@@ -25,10 +25,10 @@
// Programmer: A. Delcher
// Started: 4 Dec 2000
-const char *mainid = "$Id: FragCorrectOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: FragCorrectOVL.C 4632 2015-03-11 14:54:45Z brianwalenz $";
#include "AS_global.H"
-#include "AS_OVL_delcher.H"
+#include "AS_UTL_reverseComplement.H"
#include "AS_PER_gkpStore.H"
#include "FragCorrectOVL.H"
#include "AS_OVS_overlapStore.H"
@@ -95,7 +95,7 @@ const char *mainid = "$Id: FragCorrectOVL.C 4371 2013-08-01 17:19:47Z brianwalen
// a separate haplotype
#define NORMAL_DISTRIB_THOLD 3.62
// Determined by EDIT_DIST_PROB_BOUND
-#define THREAD_STACKSIZE (16 * 512 * 512)
+#define THREAD_STACKSIZE (128 * 512 * 512)
// The amount of memory to allocate for the stack of each thread
#define USE_STORE_DIRECTLY_READ
@@ -208,8 +208,8 @@ typedef struct
Frag_List_t * frag_list;
char rev_seq [AS_READ_MAX_NORMAL_LEN + 1];
int rev_id;
- int ** edit_array;
- int * edit_space;
+ int ** Edit_Array_Lazy;
+ int ** Edit_Space_Lazy;
} Thread_Work_Area_t;
@@ -220,15 +220,10 @@ static char * Correction_Filename = DEFAULT_CORRECTION_FILENAME;
// Name of file to which correction information is sent
static int Degree_Threshold = DEFAULT_DEGREE_THRESHOLD;
// Set keep flag on end of fragment if number of olaps < this value
-static int * Edit_Array [AS_READ_MAX_NORMAL_LEN+1];
- // Use for alignment calculation. Points into Edit_Space .
- // (only MAX_ERRORS needed)
static int Edit_Match_Limit [AS_READ_MAX_NORMAL_LEN+1] = {0};
// This array [e] is the minimum value of Edit_Array [e] [d]
// to be worth pursuing in edit-distance computations between guides
// (only MAX_ERRORS needed)
-static int *Edit_Space = NULL;
- // Memory used by alignment calculation
static int End_Exclude_Len = DEFAULT_END_EXCLUDE_LEN;
// Length of ends of exact-match regions not used in preventing
// sequence correction
@@ -347,6 +342,7 @@ static void Threaded_Stream_Old_Frags
static void Usage
(char * command);
+int Verbose_Level = 0;
int main
(int argc, char * argv [])
@@ -420,7 +416,10 @@ int main
}
fprintf (stderr, "Before Output_Corrections Num_Frags = %d\n", Num_Frags);
- fp = File_Open (Correction_Filename, "wb");
+ errno = 0;
+ fp = fopen(Correction_Filename, "wb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Correction_Filename, strerror(errno)), exit(1);
Output_Corrections (fp);
fclose (fp);
@@ -1192,18 +1191,6 @@ static void Initialize_Globals
int i, offset, del;
int e, start;
- // only (MAX_ERRORS + 4) * MAX_ERRORS needed
- Edit_Space = (int32 *)safe_malloc(sizeof(int32) * (AS_READ_MAX_NORMAL_LEN + 4) * AS_READ_MAX_NORMAL_LEN);
-
- offset = 2;
- del = 6;
- for (i = 0; i < MAX_ERRORS; i ++)
- {
- Edit_Array [i] = Edit_Space + offset;
- offset += del;
- del += 2;
- }
-
for (i = 0; i <= ERRORS_FOR_FREE; i ++)
Edit_Match_Limit [i] = 0;
@@ -1244,19 +1231,11 @@ static void Init_Thread_Work_Area
wa -> thread_id = id;
strcpy (wa -> rev_seq, "acgt");
- wa -> edit_array = (int **) safe_malloc(MAX_ERRORS * sizeof(int *));
- wa -> edit_space = (int *) safe_malloc((MAX_ERRORS + 4) * MAX_ERRORS * sizeof(int));
+ wa -> Edit_Array_Lazy = (int **) safe_malloc(MAX_ERRORS * sizeof(int *));
+ wa -> Edit_Space_Lazy = (int **) safe_malloc(MAX_ERRORS * sizeof(int *));
- offset = 2;
- del = 6;
- for (i = 0; i < MAX_ERRORS; i ++)
- {
- wa -> edit_array [i] = wa -> edit_space + offset;
- offset += del;
- del += 2;
- }
-
- return;
+ memset(wa->Edit_Array_Lazy, 0, sizeof(int *) * MAX_ERRORS);
+ memset(wa->Edit_Space_Lazy, 0, sizeof(int *) * MAX_ERRORS);
}
@@ -1583,6 +1562,77 @@ Parse_Command_Line(int argc, char **argv) {
+// Allocate another block of 64mb for edits
+
+// Needs to be at least:
+// 52,432 to handle 40% error at 64k overlap
+// 104,860 to handle 80% error at 64k overlap
+// 209,718 to handle 40% error at 256k overlap
+// 419,434 to handle 80% error at 256k overlap
+// 3,355,446 to handle 40% error at 4m overlap
+// 6,710,890 to handle 80% error at 4m overlap
+// Bigger means we can assign more than one Edit_Array[] in one allocation.
+
+uint32 EDIT_SPACE_SIZE = 16 * 1024 * 1024;
+
+static
+void
+Allocate_More_Edit_Space(Thread_Work_Area_t *WA) {
+
+ // Determine the last allocated block, and the last assigned block
+
+ int32 b = 0; // Last edit array assigned
+ int32 e = 0; // Last edit array assigned more space
+ int32 a = 0; // Last allocated block
+
+ while (WA->Edit_Array_Lazy[b] != NULL)
+ b++;
+
+ while (WA->Edit_Space_Lazy[a] != NULL)
+ a++;
+
+ // Fill in the edit space array. Well, not quite yet. First, decide the minimum size.
+ //
+ // Element [0] can access from [-2] to [2] = 5 elements.
+ // Element [1] can access from [-3] to [3] = 7 elements.
+ //
+ // Element [e] can access from [-2-e] to [2+e] = 5 + e * 2 elements
+ //
+ // So, our offset for this new block needs to put [e][0] at offset...
+
+ int32 Offset = 2 + b;
+ int32 Del = 6 + b * 2;
+ int32 Size = EDIT_SPACE_SIZE;
+
+ while (Size < Offset + Del)
+ Size *= 2;
+
+ // Allocate another block
+
+ WA->Edit_Space_Lazy[a] = new int [Size];
+
+ // And, now, fill in the edit space array.
+
+ e = b;
+
+ while ((Offset + Del < Size) &&
+ (e < MAX_ERRORS)) {
+ WA->Edit_Array_Lazy[e++] = WA->Edit_Space_Lazy[a] + Offset;
+
+ Offset += Del;
+ Del += 2;
+ }
+
+ if (e == b)
+ fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e);
+ assert(e != b);
+
+ //fprintf(stderr, "WorkArea %d allocates space %d of size %d for array %d through %d\n", WA->thread_id, a, Size, b, e-1);
+}
+
+
+
+
static int Prefix_Edit_Dist
(char A [], int m, char T [], int n, int Error_Limit,
int * A_End, int * T_End, int * Match_To_End,
@@ -1604,9 +1654,6 @@ static int Prefix_Edit_Dist
{
double Score, Max_Score;
int Max_Score_Len, Max_Score_Best_d, Max_Score_Best_e;
-#if 0
- int Tail_Len;
-#endif
int Best_d, Best_e, Longest, Row;
int Left, Right;
int d, e, j, shorter;
@@ -1619,7 +1666,10 @@ static int Prefix_Edit_Dist
for (Row = 0; Row < shorter && A [Row] == T [Row]; Row ++)
;
- wa -> edit_array [0] [0] = Row;
+ if (wa->Edit_Array_Lazy[0] == NULL)
+ Allocate_More_Edit_Space(wa);
+
+ wa -> Edit_Array_Lazy [0] [0] = Row;
if (Row == shorter) // Exact match
{
@@ -1635,42 +1685,46 @@ static int Prefix_Edit_Dist
{
Left = OVL_Max_int (Left - 1, -e);
Right = OVL_Min_int (Right + 1, e);
- wa -> edit_array [e - 1] [Left] = -2;
- wa -> edit_array [e - 1] [Left - 1] = -2;
- wa -> edit_array [e - 1] [Right] = -2;
- wa -> edit_array [e - 1] [Right + 1] = -2;
+
+ if (wa->Edit_Array_Lazy[e] == NULL)
+ Allocate_More_Edit_Space(wa);
+
+ wa -> Edit_Array_Lazy [e - 1] [Left] = -2;
+ wa -> Edit_Array_Lazy [e - 1] [Left - 1] = -2;
+ wa -> Edit_Array_Lazy [e - 1] [Right] = -2;
+ wa -> Edit_Array_Lazy [e - 1] [Right + 1] = -2;
for (d = Left; d <= Right; d ++)
{
- Row = 1 + wa -> edit_array [e - 1] [d];
- if ((j = wa -> edit_array [e - 1] [d - 1]) > Row)
+ Row = 1 + wa -> Edit_Array_Lazy [e - 1] [d];
+ if ((j = wa -> Edit_Array_Lazy [e - 1] [d - 1]) > Row)
Row = j;
- if ((j = 1 + wa -> edit_array [e - 1] [d + 1]) > Row)
+ if ((j = 1 + wa -> Edit_Array_Lazy [e - 1] [d + 1]) > Row)
Row = j;
while (Row < m && Row + d < n
&& A [Row] == T [Row + d])
Row ++;
- wa -> edit_array [e] [d] = Row;
+ wa -> Edit_Array_Lazy [e] [d] = Row;
if (Row == m || Row + d == n)
{
#if 1
// Force last error to be mismatch rather than insertion
if (Row == m
- && 1 + wa -> edit_array [e - 1] [d + 1]
- == wa -> edit_array [e] [d]
+ && 1 + wa -> Edit_Array_Lazy [e - 1] [d + 1]
+ == wa -> Edit_Array_Lazy [e] [d]
&& d < Right)
{
d ++;
- wa -> edit_array [e] [d] = wa -> edit_array [e] [d - 1];
+ wa -> Edit_Array_Lazy [e] [d] = wa -> Edit_Array_Lazy [e] [d - 1];
}
#endif
(* A_End) = Row; // One past last align position
(* T_End) = Row + d;
Compute_Delta
- (Delta, Delta_Len, wa -> edit_array, e, d, Row);
+ (Delta, Delta_Len, wa -> Edit_Array_Lazy, e, d, Row);
#if 0
// Check for branch point here caused by uneven
@@ -1697,28 +1751,28 @@ static int Prefix_Edit_Dist
}
while (Left <= Right && Left < 0
- && wa -> edit_array [e] [Left] < Edit_Match_Limit [e])
+ && wa -> Edit_Array_Lazy [e] [Left] < Edit_Match_Limit [e])
Left ++;
if (Left >= 0)
while (Left <= Right
- && wa -> edit_array [e] [Left] + Left < Edit_Match_Limit [e])
+ && wa -> Edit_Array_Lazy [e] [Left] + Left < Edit_Match_Limit [e])
Left ++;
if (Left > Right)
break;
while (Right > 0
- && wa -> edit_array [e] [Right] + Right < Edit_Match_Limit [e])
+ && wa -> Edit_Array_Lazy [e] [Right] + Right < Edit_Match_Limit [e])
Right --;
if (Right <= 0)
- while (wa -> edit_array [e] [Right] < Edit_Match_Limit [e])
+ while (wa -> Edit_Array_Lazy [e] [Right] < Edit_Match_Limit [e])
Right --;
assert (Left <= Right);
for (d = Left; d <= Right; d ++)
- if (wa -> edit_array [e] [d] > Longest)
+ if (wa -> Edit_Array_Lazy [e] [d] > Longest)
{
Best_d = d;
Best_e = e;
- Longest = wa -> edit_array [e] [d];
+ Longest = wa -> Edit_Array_Lazy [e] [d];
}
#if 1
Score = Longest * BRANCH_PT_MATCH_VALUE - e;
@@ -1735,7 +1789,7 @@ static int Prefix_Edit_Dist
}
Compute_Delta
- (Delta, Delta_Len, wa -> edit_array, Max_Score_Best_e,
+ (Delta, Delta_Len, wa -> Edit_Array_Lazy, Max_Score_Best_e,
Max_Score_Best_d, Max_Score_Len);
(* A_End) = Max_Score_Len;
@@ -2036,7 +2090,10 @@ static void Read_Olaps
olap_size = 1000;
Olap = (Olap_Info_t*) safe_malloc (olap_size * sizeof (Olap_Info_t));
- fp = File_Open (Olap_Path, "r");
+ errno = 0;
+ fp = fopen(Olap_Path, "r");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Olap_Path, strerror(errno)), exit(1);
while (fscanf (fp, "%d %d %d %d %s %lf",
& a_iid, & b_iid, & a_hang, & b_hang,
diff --git a/src/AS_OVL/Makefile b/src/AS_OVL/Makefile
index 65df2fd..1a41dc1 100644
--- a/src/AS_OVL/Makefile
+++ b/src/AS_OVL/Makefile
@@ -62,7 +62,7 @@ ifdef UMD_OVL
AS_SCRIPTS = runUMDOverlapper overlapPipeline dataWorkReduced
endif
-AS_OVL_CMN_SRCS = AS_OVL_delcher.C SharedOVL.C
+AS_OVL_CMN_SRCS = SharedOVL.C
AS_OVL_CMN_OBJS = $(AS_OVL_CMN_SRCS:.C=.o)
SEED_OLAP_SRCS = OlapFromSeedsOVL.C
@@ -106,17 +106,17 @@ endif
all: $(OBJECTS) $(CXX_PROGS) $(SCRIPTS)
@test -n nop
-show-corrects: ShowCorrectsOVL.o AS_OVL_delcher.o $(LIBS)
-update-erates: UpdateEratesOVL.o AS_OVL_delcher.o $(LIBS)
-force-erates: ForceEratesOVL.o AS_OVL_delcher.o $(LIBS)
-cat-corrects: CatCorrectsOVL.o AS_OVL_delcher.o $(LIBS)
-cat-erates: CatEratesOVL.o AS_OVL_delcher.o $(LIBS)
-correct-frags: FragCorrectOVL.o AS_OVL_delcher.o $(LIBS)
-correct-olaps: CorrectOlapsOVL.o AS_OVL_delcher.o $(LIBS)
+show-corrects: ShowCorrectsOVL.o $(LIBS)
+update-erates: UpdateEratesOVL.o $(LIBS)
+force-erates: ForceEratesOVL.o $(LIBS)
+cat-corrects: CatCorrectsOVL.o $(LIBS)
+cat-erates: CatEratesOVL.o $(LIBS)
+correct-frags: FragCorrectOVL.o $(LIBS)
+correct-olaps: CorrectOlapsOVL.o $(LIBS)
overlap_partition: overlap_partition.o $(LIBS)
-olap-from-seeds: OlapFromSeedsOVL.o AS_OVL_delcher.o SharedOVL.o $(LIBS)
+olap-from-seeds: OlapFromSeedsOVL.o SharedOVL.o $(LIBS)
runUMDOverlapper: $(UMD_OVL)/runUMDOverlapper.perl
diff --git a/src/AS_OVL/OlapFromSeedsOVL.C b/src/AS_OVL/OlapFromSeedsOVL.C
index 12658c8..8e08563 100644
--- a/src/AS_OVL/OlapFromSeedsOVL.C
+++ b/src/AS_OVL/OlapFromSeedsOVL.C
@@ -36,14 +36,16 @@
*************************************************/
/* RCS info
- * $Id: OlapFromSeedsOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $
+ * $Id: OlapFromSeedsOVL.C 4600 2015-02-06 05:56:41Z brianwalenz $
* $Revision: 1.45 $
*/
-const char *mainid = "$Id: OlapFromSeedsOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: OlapFromSeedsOVL.C 4600 2015-02-06 05:56:41Z brianwalenz $";
#include "OlapFromSeedsOVL.H"
+int Verbose_Level = 0; // Shared with SharedOVL.C
+
int main
(int argc, char * argv [])
@@ -99,7 +101,10 @@ int main
Show_Votes (stdout);
fprintf (stderr, "Before Output_Corrections Num_Frags = %d\n", Num_Frags);
- fp = File_Open (Correction_Filename, "wb");
+ errno = 0;
+ fp = fopen(Correction_Filename, "wb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Correction_Filename, strerror(errno)), exit(1);
Output_Corrections (fp);
fclose (fp);
}
@@ -635,9 +640,12 @@ static void Analyze_Diffs
// Eventually needs to be multi-threaded, probably in the same way that
// original votes were done, i.e., thread p does frag i iff (i % num_threads === p)
- if (Doing_Corrections)
- fp = File_Open (Correction_Filename, "wb");
-
+ if (Doing_Corrections) {
+ errno = 0;
+ fp = fopen(Correction_Filename, "wb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Correction_Filename, strerror(errno)), exit(1);
+ }
fprintf (stderr, "Start analyzing multi-alignments Num_Frags = %d\n", Num_Frags);
for (int32 i = 0; i < Num_Frags; i ++)
Analyze_Frag (fp, i);
@@ -776,21 +784,12 @@ if (0)
for (i = 0; i < n; i ++)
if (! Frag [sub] . diff_list [i] . disregard)
{
-//**ALD
-if (0)
- {
- printf ("Set_New_Homopoly_Votes for i=%d b_iid=%d\n", i,
- Frag [sub] . diff_list [i] . b_iid);
- if (Frag [sub] . diff_list [i] . b_iid == 421078)
- Global_Debug_Flag = TRUE;
- }
if (mod_dp [i] . is_homopoly_type)
// Set_Homopoly_Votes_From_Diffs (sub, Frag [sub] . diff_list + i);
Set_New_Homopoly_Votes (vote, mod_seq, mod_len, mod_dp + i);
else
// Set_Votes_From_Diffs (sub, Frag [sub] . diff_list + i);
Set_New_Standard_Votes (vote, mod_seq, mod_len, mod_dp + i);
- Global_Debug_Flag = FALSE;
}
//**ALD
@@ -2823,7 +2822,12 @@ static void Initialize_Globals
switch (OVL_Output_Type)
{
case TEXT_FILE :
- OVL_Output_fp = File_Open (OVL_Output_Path, "w");
+ {
+ errno = 0;
+ OVL_Output_fp = fopen(OVL_Output_Path, "w");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", OVL_Output_Path, strerror(errno)), exit(1);
+ }
break;
case BINARY_FILE :
Binary_OVL_Output_fp = AS_OVS_createBinaryOverlapFile (OVL_Output_Path, FALSE);
@@ -4357,7 +4361,11 @@ static void Read_Seeds
olap_size = 1000;
Olap = (Olap_Info_t *) safe_malloc (olap_size * sizeof (Olap_Info_t));
- fp = File_Open (Olap_Path, "r");
+ errno = 0;
+ fp = fopen(Olap_Path, "r");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", Olap_Path, strerror(errno)), exit(1);
+
while (fgets (line, MAX_LINE, fp) != NULL)
{
@@ -4901,11 +4909,6 @@ static void Set_New_Standard_Votes
int hp_len = 0; // length of current homopoly run
int j, k, m;
-//**ALD
-if (Global_Debug_Flag)
- printf ("a_lo/hi=%d/%d b_lo/hi=%d/%d\n", dp -> a_lo, dp -> a_hi,
- dp -> b_lo, dp -> b_hi);
-
j = dp -> a_lo;
if (dp -> diff_len > 0)
diff --git a/src/AS_OVL/OlapFromSeedsOVL.H b/src/AS_OVL/OlapFromSeedsOVL.H
index efb624f..6dadeed 100644
--- a/src/AS_OVL/OlapFromSeedsOVL.H
+++ b/src/AS_OVL/OlapFromSeedsOVL.H
@@ -33,7 +33,7 @@
*************************************************/
/* RCS info
- * $Id: OlapFromSeedsOVL.H 4371 2013-08-01 17:19:47Z brianwalenz $
+ * $Id: OlapFromSeedsOVL.H 4600 2015-02-06 05:56:41Z brianwalenz $
* $Revision: 1.27 $
*/
@@ -41,14 +41,13 @@
#ifndef __OLAPFROMSEEDS_H_INCLUDED
#define __OLAPFROMSEEDS_H_INCLUDED
-static const char *rcsid_OLAPFROMSEEDS_H_INCLUDED = "$Id: OlapFromSeedsOVL.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_OLAPFROMSEEDS_H_INCLUDED = "$Id: OlapFromSeedsOVL.H 4600 2015-02-06 05:56:41Z brianwalenz $";
//**ALD determine if use new code to analyze true multialignments
#define USE_NEW_STUFF 1
#include "AS_global.H"
-
-#include "AS_OVL_delcher.H"
+#include "AS_UTL_reverseComplement.H"
#include "AS_PER_gkpStore.H"
#include "AS_OVS_overlapStore.H"
#include "SharedOVL.H"
@@ -108,7 +107,7 @@ static const char *rcsid_OLAPFROMSEEDS_H_INCLUDED = "$Id: OlapFromSeedsOVL.H 437
// a separate haplotype
#define NORMAL_DISTRIB_THOLD 3.62
// Determined by EDIT_DIST_PROB_BOUND
-#define THREAD_STACKSIZE (16 * 512 * 512)
+#define THREAD_STACKSIZE (128 * 512 * 512)
// The amount of memory to allocate for the stack of each thread
#define USE_STORE_DIRECTLY_READ
diff --git a/src/AS_OVL/SharedOVL.C b/src/AS_OVL/SharedOVL.C
index 91895db..3cdf8d1 100644
--- a/src/AS_OVL/SharedOVL.C
+++ b/src/AS_OVL/SharedOVL.C
@@ -19,11 +19,11 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: SharedOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: SharedOVL.C 4600 2015-02-06 05:56:41Z brianwalenz $";
#include "SharedOVL.H"
-
+extern int Verbose_Level; // In OlapFromSeedsOVL.C
void Fix_Homopoly_Substitution
(const char * a_string, const char * b_string, int delta [], const HP_LV_Cell_t * cell,
@@ -1471,7 +1471,6 @@ int Rev_Homopoly_Match_Start
//**ALD
if (0 && best_end_score < MAX_HOMOPOLY_SCORE)
{
- printf ("Global_Debug_Flag=%d\n", Global_Debug_Flag);
printf ("Rev_Homopoly_Match_Start: best_end_e=%d best_end_d=%d"
" best_end_score=%d\n", best_end_e, best_end_d, best_end_score);
Show_Homopoly_Match_Array (stdout, edit_array, e);
diff --git a/src/AS_OVL/SharedOVL.H b/src/AS_OVL/SharedOVL.H
index 8c078bf..299fa06 100644
--- a/src/AS_OVL/SharedOVL.H
+++ b/src/AS_OVL/SharedOVL.H
@@ -34,7 +34,7 @@
*************************************************/
/* RCS info
- * $Id: SharedOVL.H 4371 2013-08-01 17:19:47Z brianwalenz $
+ * $Id: SharedOVL.H 4600 2015-02-06 05:56:41Z brianwalenz $
* $Revision: 1.17 $
*/
@@ -42,11 +42,9 @@
#ifndef __SHAREDOVL_H_INCLUDED
#define __SHAREDOVL_H_INCLUDED
-static const char *rcsid_SHAREDOVL_H_INCLUDED = "$Id: SharedOVL.H 4371 2013-08-01 17:19:47Z brianwalenz $";
-
-
-#include "AS_OVL_delcher.H"
+static const char *rcsid_SHAREDOVL_H_INCLUDED = "$Id: SharedOVL.H 4600 2015-02-06 05:56:41Z brianwalenz $";
+#include "AS_global.H"
// Constants
diff --git a/src/AS_OVL/ShowCorrectsOVL.C b/src/AS_OVL/ShowCorrectsOVL.C
index 17fd94c..ebdca9c 100644
--- a/src/AS_OVL/ShowCorrectsOVL.C
+++ b/src/AS_OVL/ShowCorrectsOVL.C
@@ -34,11 +34,11 @@
*************************************************/
/* RCS info
- * $Id: ShowCorrectsOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $
+ * $Id: ShowCorrectsOVL.C 4600 2015-02-06 05:56:41Z brianwalenz $
* $Revision: 1.14 $
*/
-static char *rcsid = "$Id: ShowCorrectsOVL.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static char *rcsid = "$Id: ShowCorrectsOVL.C 4600 2015-02-06 05:56:41Z brianwalenz $";
// System include files
@@ -53,7 +53,6 @@ static char *rcsid = "$Id: ShowCorrectsOVL.C 4371 2013-08-01 17:19:47Z brianwale
// Local include files
-#include "AS_OVL_delcher.H"
#include "AS_PER_gkpStore.H"
#include "AS_PER_genericStore.H"
#include "AS_MSG_pmesg.H"
@@ -85,7 +84,10 @@ int main
exit (1);
}
- fp = File_Open (argv [1], "rb");
+ errno = 0;
+ fp = fopen (argv [1], "rb");
+ if (errno)
+ fprintf(stderr, "Failed to open '%s': %s\n", argv[1], strerror(errno)), exit(1);
while (fread (& msg, sizeof (Correction_Output_t), 1, fp) == 1)
{
diff --git a/src/AS_OVM/._Makefile b/src/AS_OVM/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._Makefile differ
diff --git a/src/AS_OVM/._overlapInCore-Build_Hash_Index.C b/src/AS_OVM/._overlapInCore-Build_Hash_Index.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Build_Hash_Index.C differ
diff --git a/src/AS_OVM/._overlapInCore-Extend_Alignment.C b/src/AS_OVM/._overlapInCore-Extend_Alignment.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Extend_Alignment.C differ
diff --git a/src/AS_OVM/._overlapInCore-Find_Overlaps.C b/src/AS_OVM/._overlapInCore-Find_Overlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Find_Overlaps.C differ
diff --git a/src/AS_OVM/._overlapInCore-Output.C b/src/AS_OVM/._overlapInCore-Output.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Output.C differ
diff --git a/src/AS_OVM/._overlapInCore-Process_Overlaps.C b/src/AS_OVM/._overlapInCore-Process_Overlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Process_Overlaps.C differ
diff --git a/src/AS_OVM/._overlapInCore-Process_String_Overlaps.C b/src/AS_OVM/._overlapInCore-Process_String_Overlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Process_String_Overlaps.C differ
diff --git a/src/AS_OVM/._overlapInCore-Read_Next_Frag.C b/src/AS_OVM/._overlapInCore-Read_Next_Frag.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore-Read_Next_Frag.C differ
diff --git a/src/AS_OVM/._overlapInCore.C b/src/AS_OVM/._overlapInCore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore.C differ
diff --git a/src/AS_OVM/._overlapInCore.H b/src/AS_OVM/._overlapInCore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVM/._overlapInCore.H differ
diff --git a/src/AS_OVM/overlapInCore-Extend_Alignment.C b/src/AS_OVM/overlapInCore-Extend_Alignment.C
index 1a61b7a..bb578d1 100644
--- a/src/AS_OVM/overlapInCore-Extend_Alignment.C
+++ b/src/AS_OVM/overlapInCore-Extend_Alignment.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: overlapInCore-Extend_Alignment.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: overlapInCore-Extend_Alignment.C 4624 2015-02-21 22:49:38Z brianwalenz $";
#include "overlapInCore.H"
@@ -43,27 +43,32 @@ Set_Left_Delta (int e, int d, int * leftover, int * t_end, int t_len, Work_Area_
int from, last, max;
int j, k;
- last = WA->Edit_Array[e][d];
+ assert(WA->Edit_Array_Lazy[e] != NULL);
+
+ last = WA->Edit_Array_Lazy[e][d];
WA->Left_Delta_Len = 0;
+
for (k = e; k > 0; k--) {
+ assert(WA->Edit_Array_Lazy[k] != NULL);
+
from = d;
- max = 1 + WA->Edit_Array[k - 1][d];
- if ((j = WA->Edit_Array[k - 1][d - 1]) > max) {
+ max = 1 + WA->Edit_Array_Lazy[k - 1][d];
+ if ((j = WA->Edit_Array_Lazy[k - 1][d - 1]) > max) {
from = d - 1;
max = j;
}
- if ((j = 1 + WA->Edit_Array[k - 1][d + 1]) > max) {
+ if ((j = 1 + WA->Edit_Array_Lazy[k - 1][d + 1]) > max) {
from = d + 1;
max = j;
}
if (from == d - 1) {
WA->Left_Delta[WA->Left_Delta_Len++] = max - last - 1;
d--;
- last = WA->Edit_Array[k - 1][from];
+ last = WA->Edit_Array_Lazy[k - 1][from];
} else if (from == d + 1) {
WA->Left_Delta[WA->Left_Delta_Len++] = last - (max - 1);
d++;
- last = WA->Edit_Array[k - 1][from];
+ last = WA->Edit_Array_Lazy[k - 1][from];
}
}
(* leftover) = last;
@@ -106,27 +111,32 @@ Set_Right_Delta (int e, int d, Work_Area_t * WA) {
int from, last, max;
int i, j, k;
- last = WA->Edit_Array[e][d];
+ assert(WA->Edit_Array_Lazy[e] != NULL);
+
+ last = WA->Edit_Array_Lazy[e][d];
WA->Right_Delta_Len = 0;
+
for (k = e; k > 0; k--) {
+ assert(WA->Edit_Array_Lazy[k] != NULL);
+
from = d;
- max = 1 + WA->Edit_Array[k - 1][d];
- if ((j = WA->Edit_Array[k - 1][d - 1]) > max) {
+ max = 1 + WA->Edit_Array_Lazy[k - 1][d];
+ if ((j = WA->Edit_Array_Lazy[k - 1][d - 1]) > max) {
from = d - 1;
max = j;
}
- if ((j = 1 + WA->Edit_Array[k - 1][d + 1]) > max) {
+ if ((j = 1 + WA->Edit_Array_Lazy[k - 1][d + 1]) > max) {
from = d + 1;
max = j;
}
if (from == d - 1) {
WA->Delta_Stack[WA->Right_Delta_Len++] = max - last - 1;
d--;
- last = WA->Edit_Array[k - 1][from];
+ last = WA->Edit_Array_Lazy[k - 1][from];
} else if (from == d + 1) {
WA->Delta_Stack[WA->Right_Delta_Len++] = last - (max - 1);
d++;
- last = WA->Edit_Array[k - 1][from];
+ last = WA->Edit_Array_Lazy[k - 1][from];
}
}
WA->Delta_Stack[WA->Right_Delta_Len++] = last + 1;
@@ -144,6 +154,77 @@ Set_Right_Delta (int e, int d, Work_Area_t * WA) {
+// Allocate another block of 64mb for edits
+
+// Needs to be at least:
+// 52,432 to handle 40% error at 64k overlap
+// 104,860 to handle 80% error at 64k overlap
+// 209,718 to handle 40% error at 256k overlap
+// 419,434 to handle 80% error at 256k overlap
+// 3,355,446 to handle 40% error at 4m overlap
+// 6,710,890 to handle 80% error at 4m overlap
+// Bigger means we can assign more than one Edit_Array[] in one allocation.
+
+uint32 EDIT_SPACE_SIZE = 16 * 1024 * 1024;
+
+static
+void
+Allocate_More_Edit_Space(Work_Area_t *WA) {
+
+ // Determine the last allocated block, and the last assigned block
+
+ int32 b = 0; // Last edit array assigned
+ int32 e = 0; // Last edit array assigned more space
+ int32 a = 0; // Last allocated block
+
+ while (WA->Edit_Array_Lazy[b] != NULL)
+ b++;
+
+ while (WA->Edit_Space_Lazy[a] != NULL)
+ a++;
+
+ // Fill in the edit space array. Well, not quite yet. First, decide the minimum size.
+ //
+ // Element [0] can access from [-2] to [2] = 5 elements.
+ // Element [1] can access from [-3] to [3] = 7 elements.
+ //
+ // Element [e] can access from [-2-e] to [2+e] = 5 + e * 2 elements
+ //
+ // So, our offset for this new block needs to put [e][0] at offset...
+
+ int32 Offset = 2 + b;
+ int32 Del = 6 + b * 2;
+ int32 Size = EDIT_SPACE_SIZE;
+
+ while (Size < Offset + Del)
+ Size *= 2;
+
+ // Allocate another block
+
+ WA->Edit_Space_Lazy[a] = new int [Size];
+
+ // And, now, fill in the edit space array.
+
+ e = b;
+
+ while ((Offset + Del < Size) &&
+ (e < MAX_ERRORS)) {
+ WA->Edit_Array_Lazy[e++] = WA->Edit_Space_Lazy[a] + Offset;
+
+ Offset += Del;
+ Del += 2;
+ }
+
+ if (e == b)
+ fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e);
+ assert(e != b);
+
+ fprintf(stderr, "WorkArea %d allocates space %d of size %d for array %d through %d\n", WA->thread_id, a, Size, b, e-1);
+}
+
+
+
+
// Return the minimum number of changes (inserts, deletes, replacements)
// needed to match string A[0 .. (m-1)] with a prefix of string
// T[0 .. (n-1)] if it's not more than Error_Limit .
@@ -158,7 +239,6 @@ Set_Right_Delta (int e, int d, Work_Area_t * WA) {
// of at least one string; otherwise, set it false to indicate
// a branch point.
-
static
int
Prefix_Edit_Dist(char A[], int m, char T[], int n, int Error_Limit,
@@ -181,7 +261,10 @@ Prefix_Edit_Dist(char A[], int m, char T[], int n, int Error_Limit,
|| T[Row] == DONT_KNOW_CHAR); Row++)
;
- WA->Edit_Array[0][0] = Row;
+ if (WA->Edit_Array_Lazy[0] == NULL)
+ Allocate_More_Edit_Space(WA);
+
+ WA->Edit_Array_Lazy[0][0] = Row;
if (Row == m) {
// Exact match
@@ -193,18 +276,21 @@ Prefix_Edit_Dist(char A[], int m, char T[], int n, int Error_Limit,
Left = Right = 0;
Max_Score = 0.0;
for (e = 1; e <= Error_Limit; e++) {
+ if (WA->Edit_Array_Lazy[e] == NULL)
+ Allocate_More_Edit_Space(WA);
+
Left = MAX (Left - 1, -e);
Right = MIN (Right + 1, e);
- WA->Edit_Array[e - 1][Left] = -2;
- WA->Edit_Array[e - 1][Left - 1] = -2;
- WA->Edit_Array[e - 1][Right] = -2;
- WA->Edit_Array[e - 1][Right + 1] = -2;
+ WA->Edit_Array_Lazy[e - 1][Left] = -2;
+ WA->Edit_Array_Lazy[e - 1][Left - 1] = -2;
+ WA->Edit_Array_Lazy[e - 1][Right] = -2;
+ WA->Edit_Array_Lazy[e - 1][Right + 1] = -2;
for (d = Left; d <= Right; d++) {
- Row = 1 + WA->Edit_Array[e - 1][d];
- if ((j = WA->Edit_Array[e - 1][d - 1]) > Row)
+ Row = 1 + WA->Edit_Array_Lazy[e - 1][d];
+ if ((j = WA->Edit_Array_Lazy[e - 1][d - 1]) > Row)
Row = j;
- if ((j = 1 + WA->Edit_Array[e - 1][d + 1]) > Row)
+ if ((j = 1 + WA->Edit_Array_Lazy[e - 1][d + 1]) > Row)
Row = j;
while (Row < m && Row + d < n
&& (A[Row] == T[Row + d]
@@ -212,7 +298,7 @@ Prefix_Edit_Dist(char A[], int m, char T[], int n, int Error_Limit,
|| T[Row + d] == DONT_KNOW_CHAR))
Row++;
- WA->Edit_Array[e][d] = Row;
+ WA->Edit_Array_Lazy[e][d] = Row;
if (Row == m || Row + d == n) {
// Check for branch point here caused by uneven
@@ -233,9 +319,9 @@ Prefix_Edit_Dist(char A[], int m, char T[], int n, int Error_Limit,
}
// Force last error to be mismatch rather than insertion
- if (Row == m && 1 + WA->Edit_Array[e - 1][d + 1] == WA->Edit_Array[e][d] && d < Right) {
+ if (Row == m && 1 + WA->Edit_Array_Lazy[e - 1][d + 1] == WA->Edit_Array_Lazy[e][d] && d < Right) {
d++;
- WA->Edit_Array[e][d] = WA->Edit_Array[e][d - 1];
+ WA->Edit_Array_Lazy[e][d] = WA->Edit_Array_Lazy[e][d - 1];
}
(* A_End) = Row; // One past last align position
@@ -247,32 +333,32 @@ Prefix_Edit_Dist(char A[], int m, char T[], int n, int Error_Limit,
}
while (Left <= Right && Left < 0
- && WA->Edit_Array[e][Left] < WA->Edit_Match_Limit[e])
+ && WA->Edit_Array_Lazy[e][Left] < WA->Edit_Match_Limit[e])
Left++;
if (Left >= 0)
while (Left <= Right
- && WA->Edit_Array[e][Left] + Left < WA->Edit_Match_Limit[e])
+ && WA->Edit_Array_Lazy[e][Left] + Left < WA->Edit_Match_Limit[e])
Left++;
if (Left > Right)
break;
while (Right > 0
- && WA->Edit_Array[e][Right] + Right < WA->Edit_Match_Limit[e])
+ && WA->Edit_Array_Lazy[e][Right] + Right < WA->Edit_Match_Limit[e])
Right--;
if (Right <= 0)
- while (WA->Edit_Array[e][Right] < WA->Edit_Match_Limit[e])
+ while (WA->Edit_Array_Lazy[e][Right] < WA->Edit_Match_Limit[e])
Right--;
assert (Left <= Right);
for (d = Left; d <= Right; d++)
- if (WA->Edit_Array[e][d] > Longest) {
+ if (WA->Edit_Array_Lazy[e][d] > Longest) {
Best_d = d;
Best_e = e;
- Longest = WA->Edit_Array[e][d];
+ Longest = WA->Edit_Array_Lazy[e][d];
}
Score = Longest * Branch_Match_Value - e;
@@ -339,7 +425,10 @@ Rev_Prefix_Edit_Dist (char A[], int m, char T[], int n, int Error_Limit,
|| T[- Row] == DONT_KNOW_CHAR); Row++)
;
- WA->Edit_Array[0][0] = Row;
+ if (WA->Edit_Array_Lazy[0] == NULL)
+ Allocate_More_Edit_Space(WA);
+
+ WA->Edit_Array_Lazy[0][0] = Row;
if (Row == m) {
(* A_End) = (* T_End) = - m;
@@ -351,18 +440,21 @@ Rev_Prefix_Edit_Dist (char A[], int m, char T[], int n, int Error_Limit,
Left = Right = 0;
Max_Score = 0.0;
for (e = 1; e <= Error_Limit; e++) {
+ if (WA->Edit_Array_Lazy[e] == NULL)
+ Allocate_More_Edit_Space(WA);
+
Left = MAX (Left - 1, -e);
Right = MIN (Right + 1, e);
- WA->Edit_Array[e - 1][Left] = -2;
- WA->Edit_Array[e - 1][Left - 1] = -2;
- WA->Edit_Array[e - 1][Right] = -2;
- WA->Edit_Array[e - 1][Right + 1] = -2;
+ WA->Edit_Array_Lazy[e - 1][Left] = -2;
+ WA->Edit_Array_Lazy[e - 1][Left - 1] = -2;
+ WA->Edit_Array_Lazy[e - 1][Right] = -2;
+ WA->Edit_Array_Lazy[e - 1][Right + 1] = -2;
for (d = Left; d <= Right; d++) {
- Row = 1 + WA->Edit_Array[e - 1][d];
- if ((j = WA->Edit_Array[e - 1][d - 1]) > Row)
+ Row = 1 + WA->Edit_Array_Lazy[e - 1][d];
+ if ((j = WA->Edit_Array_Lazy[e - 1][d - 1]) > Row)
Row = j;
- if ((j = 1 + WA->Edit_Array[e - 1][d + 1]) > Row)
+ if ((j = 1 + WA->Edit_Array_Lazy[e - 1][d + 1]) > Row)
Row = j;
while (Row < m && Row + d < n
&& (A[- Row] == T[- Row - d]
@@ -370,7 +462,7 @@ Rev_Prefix_Edit_Dist (char A[], int m, char T[], int n, int Error_Limit,
|| T[- Row - d] == DONT_KNOW_CHAR))
Row++;
- WA->Edit_Array[e][d] = Row;
+ WA->Edit_Array_Lazy[e][d] = Row;
if (Row == m || Row + d == n) {
@@ -402,32 +494,32 @@ Rev_Prefix_Edit_Dist (char A[], int m, char T[], int n, int Error_Limit,
}
while (Left <= Right && Left < 0
- && WA->Edit_Array[e][Left] < WA->Edit_Match_Limit[e])
+ && WA->Edit_Array_Lazy[e][Left] < WA->Edit_Match_Limit[e])
Left++;
if (Left >= 0)
while (Left <= Right
- && WA->Edit_Array[e][Left] + Left < WA->Edit_Match_Limit[e])
+ && WA->Edit_Array_Lazy[e][Left] + Left < WA->Edit_Match_Limit[e])
Left++;
if (Left > Right)
break;
while (Right > 0
- && WA->Edit_Array[e][Right] + Right < WA->Edit_Match_Limit[e])
+ && WA->Edit_Array_Lazy[e][Right] + Right < WA->Edit_Match_Limit[e])
Right--;
if (Right <= 0)
- while (WA->Edit_Array[e][Right] < WA->Edit_Match_Limit[e])
+ while (WA->Edit_Array_Lazy[e][Right] < WA->Edit_Match_Limit[e])
Right--;
assert (Left <= Right);
for (d = Left; d <= Right; d++)
- if (WA->Edit_Array[e][d] > Longest) {
+ if (WA->Edit_Array_Lazy[e][d] > Longest) {
Best_d = d;
Best_e = e;
- Longest = WA->Edit_Array[e][d];
+ Longest = WA->Edit_Array_Lazy[e][d];
}
Score = Longest * Branch_Match_Value - e;
diff --git a/src/AS_OVM/overlapInCore.C b/src/AS_OVM/overlapInCore.C
index a6188fa..007e3b3 100644
--- a/src/AS_OVM/overlapInCore.C
+++ b/src/AS_OVM/overlapInCore.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: overlapInCore.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: overlapInCore.C 4599 2015-02-06 05:14:50Z brianwalenz $";
#include "overlapInCore.H"
#include "AS_UTL_decodeRange.H"
@@ -260,10 +260,13 @@ Initialize_Work_Area(Work_Area_t * WA, int id) {
allocated += 3 * MAX_ERRORS * sizeof(int);
- WA -> Edit_Space = (int *)safe_malloc ((MAX_ERRORS + 4) * MAX_ERRORS * sizeof (int));
- WA -> Edit_Array = (int **)safe_malloc (MAX_ERRORS * sizeof (int *));
+ WA -> Edit_Space_Lazy = (int **)safe_malloc (MAX_ERRORS * sizeof (int *));
+ WA -> Edit_Array_Lazy = (int **)safe_malloc (MAX_ERRORS * sizeof (int *));
- allocated += (MAX_ERRORS + 4) * MAX_ERRORS * sizeof (int) + MAX_ERRORS * sizeof (int *);
+ memset(WA->Edit_Space_Lazy, 0, MAX_ERRORS * sizeof(int *));
+ memset(WA->Edit_Array_Lazy, 0, MAX_ERRORS * sizeof(int *));
+
+ allocated += 2 * MAX_ERRORS * sizeof (int *);
WA -> String_Olap_Size = INIT_STRING_OLAP_SIZE;
WA -> String_Olap_Space = (String_Olap_t *) safe_malloc (WA -> String_Olap_Size * sizeof (String_Olap_t));
@@ -273,14 +276,6 @@ Initialize_Work_Area(Work_Area_t * WA, int id) {
allocated += WA -> String_Olap_Size * sizeof (String_Olap_t);
allocated += WA -> Match_Node_Size * sizeof (Match_Node_t);
- int32 Offset = 2;
- int32 Del = 6;
- for (int32 i=0; i<MAX_ERRORS; i++) {
- WA -> Edit_Array [i] = WA -> Edit_Space + Offset;
- Offset += Del;
- Del += 2;
- }
-
WA -> status = 0;
WA -> thread_id = id;
@@ -792,10 +787,10 @@ main(int argc, char **argv) {
// Adjust the number of reads to load into memory at once (for processing, not the hash table),
if (Max_Reads_Per_Batch == 0)
- Max_Reads_Per_Batch = 100000;
+ Max_Reads_Per_Batch = (Max_Hash_Strings < 100000) ? Max_Hash_Strings : 100000;
- if (Max_Hash_Strings < Max_Reads_Per_Batch)
- Max_Reads_Per_Batch = Max_Hash_Strings;
+ //if (Max_Hash_Strings < Max_Reads_Per_Batch)
+ // Max_Reads_Per_Batch = Max_Hash_Strings;
// Adjust the number of reads processed per thread. Default to having four blocks per thread,
// but make sure that (a) all threads have work to do, and (b) batches are not minuscule.
diff --git a/src/AS_OVM/overlapInCore.H b/src/AS_OVM/overlapInCore.H
index 25bc2f7..3bbfa8b 100644
--- a/src/AS_OVM/overlapInCore.H
+++ b/src/AS_OVM/overlapInCore.H
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-//static const char *rcsid = "$Id: overlapInCore.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+//static const char *rcsid = "$Id: overlapInCore.H 4599 2015-02-06 05:14:50Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -304,8 +304,10 @@ typedef struct Work_Area {
int Right_Delta_Len;
int * Right_Delta;
int * Delta_Stack;
- int * Edit_Space;
- int ** Edit_Array;
+
+ int ** Edit_Space_Lazy;
+ int ** Edit_Array_Lazy;
+
int * Edit_Match_Limit;
int * Error_Bound;
String_Olap_t * String_Olap_Space;
diff --git a/src/AS_OVS/._AS_OVS_overlap.C b/src/AS_OVS/._AS_OVS_overlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_OVS_overlap.C differ
diff --git a/src/AS_OVS/._AS_OVS_overlap.H b/src/AS_OVS/._AS_OVS_overlap.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_OVS_overlap.H differ
diff --git a/src/AS_OVS/._AS_OVS_overlapFile.C b/src/AS_OVS/._AS_OVS_overlapFile.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_OVS_overlapFile.C differ
diff --git a/src/AS_OVS/._AS_OVS_overlapFile.H b/src/AS_OVS/._AS_OVS_overlapFile.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_OVS_overlapFile.H differ
diff --git a/src/AS_OVS/._AS_OVS_overlapStore.C b/src/AS_OVS/._AS_OVS_overlapStore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_OVS_overlapStore.C differ
diff --git a/src/AS_OVS/._AS_OVS_overlapStore.H b/src/AS_OVS/._AS_OVS_overlapStore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_OVS_overlapStore.H differ
diff --git a/src/AS_OVS/._AS_UTL_histogram.H b/src/AS_OVS/._AS_UTL_histogram.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._AS_UTL_histogram.H differ
diff --git a/src/AS_OVS/._Makefile b/src/AS_OVS/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._Makefile differ
diff --git a/src/AS_OVS/._README.GES b/src/AS_OVS/._README.GES
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._README.GES differ
diff --git a/src/AS_OVS/._analyze-true-vs-test.pl b/src/AS_OVS/._analyze-true-vs-test.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._analyze-true-vs-test.pl differ
diff --git a/src/AS_OVS/._check-ordered-reads-for-missed-overlaps.pl b/src/AS_OVS/._check-ordered-reads-for-missed-overlaps.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._check-ordered-reads-for-missed-overlaps.pl differ
diff --git a/src/AS_OVS/._convertOverlap.C b/src/AS_OVS/._convertOverlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._convertOverlap.C differ
diff --git a/src/AS_OVS/._filterOverlap.C b/src/AS_OVS/._filterOverlap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._filterOverlap.C differ
diff --git a/src/AS_OVS/._filterTrue.pl b/src/AS_OVS/._filterTrue.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._filterTrue.pl differ
diff --git a/src/AS_OVS/._find-missed-true-overlaps.pl b/src/AS_OVS/._find-missed-true-overlaps.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._find-missed-true-overlaps.pl differ
diff --git a/src/AS_OVS/._infer-obt-from-genomic-blasr.pl b/src/AS_OVS/._infer-obt-from-genomic-blasr.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._infer-obt-from-genomic-blasr.pl differ
diff --git a/src/AS_OVS/._infer-olaps-from-genomic-coords.pl b/src/AS_OVS/._infer-olaps-from-genomic-coords.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._infer-olaps-from-genomic-coords.pl differ
diff --git a/src/AS_OVS/._infer-olaps-from-pairwise-blasr.pl b/src/AS_OVS/._infer-olaps-from-pairwise-blasr.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._infer-olaps-from-pairwise-blasr.pl differ
diff --git a/src/AS_OVS/._infer-olaps-from-pairwise-coords.pl b/src/AS_OVS/._infer-olaps-from-pairwise-coords.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._infer-olaps-from-pairwise-coords.pl differ
diff --git a/src/AS_OVS/._infer-ovl-from-genomic-blasr.pl b/src/AS_OVS/._infer-ovl-from-genomic-blasr.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._infer-ovl-from-genomic-blasr.pl differ
diff --git a/src/AS_OVS/._overlapStats.C b/src/AS_OVS/._overlapStats.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStats.C differ
diff --git a/src/AS_OVS/._overlapStatsBoringStuff.H b/src/AS_OVS/._overlapStatsBoringStuff.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStatsBoringStuff.H differ
diff --git a/src/AS_OVS/._overlapStore.C b/src/AS_OVS/._overlapStore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStore.C differ
diff --git a/src/AS_OVS/._overlapStore.H b/src/AS_OVS/._overlapStore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStore.H differ
diff --git a/src/AS_OVS/._overlapStoreBucketizer.C b/src/AS_OVS/._overlapStoreBucketizer.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStoreBucketizer.C differ
diff --git a/src/AS_OVS/._overlapStoreBuild.C b/src/AS_OVS/._overlapStoreBuild.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStoreBuild.C differ
diff --git a/src/AS_OVS/._overlapStoreIndexer.C b/src/AS_OVS/._overlapStoreIndexer.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStoreIndexer.C differ
diff --git a/src/AS_OVS/._overlapStoreSorter.C b/src/AS_OVS/._overlapStoreSorter.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStoreSorter.C differ
diff --git a/src/AS_OVS/._overlapStore_dump.C b/src/AS_OVS/._overlapStore_dump.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStore_dump.C differ
diff --git a/src/AS_OVS/._overlapStore_erates.C b/src/AS_OVS/._overlapStore_erates.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStore_erates.C differ
diff --git a/src/AS_OVS/._overlapStore_genomeLength.C b/src/AS_OVS/._overlapStore_genomeLength.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._overlapStore_genomeLength.C differ
diff --git a/src/AS_OVS/._rewrite-obt-dump-with-names.pl b/src/AS_OVS/._rewrite-obt-dump-with-names.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._rewrite-obt-dump-with-names.pl differ
diff --git a/src/AS_OVS/._rewrite-overlap-dump-with-names.pl b/src/AS_OVS/._rewrite-overlap-dump-with-names.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_OVS/._rewrite-overlap-dump-with-names.pl differ
diff --git a/src/AS_OVS/AS_OVS_overlap.C b/src/AS_OVS/AS_OVS_overlap.C
index 81e8f3c..53ee737 100644
--- a/src/AS_OVS/AS_OVS_overlap.C
+++ b/src/AS_OVS/AS_OVS_overlap.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: AS_OVS_overlap.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: AS_OVS_overlap.C 4576 2014-10-21 14:43:00Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -186,6 +186,46 @@ AS_OVS_convertOBTdumpToOVSoverlap(char *line, OVSoverlap *olap) {
+// NOTE! This isn't a PROPER OBT overlap, since the b coord can be
+// reversed to show fwd/rev orientation.
+//
+void
+AS_OVS_convertOVLoverlapToOBToverlap(OVSoverlap &overlap, uint32 clrLenA, uint32 clrLenB) {
+
+ if (overlap.dat.obt.type != AS_OVS_TYPE_OVL)
+ return;
+
+ int32 ahang = overlap.dat.ovl.a_hang;
+ int32 bhang = overlap.dat.ovl.b_hang;
+
+ uint32 abgn = (ahang < 0) ? (0) : (ahang);
+ uint32 aend = (bhang < 0) ? (clrLenA + bhang) : (clrLenA);
+ uint32 bbgn = (ahang < 0) ? (-ahang) : (0);
+ uint32 bend = (bhang < 0) ? (clrLenB) : (clrLenB - bhang);
+
+ bool fwd = (overlap.dat.ovl.flipped == false);
+
+ uint64 erate = overlap.dat.ovl.corr_erate;
+
+ if (overlap.dat.ovl.flipped) {
+ bbgn = clrLenB - bbgn;
+ bend = clrLenB - bend;
+ }
+
+ overlap.dat.obt.type = AS_OVS_TYPE_OBT;
+
+ overlap.dat.obt.a_beg = abgn;
+ overlap.dat.obt.a_end = aend;
+ overlap.dat.obt.b_beg = bbgn;
+ overlap.dat.obt.b_end_hi = bend >> 9;
+ overlap.dat.obt.b_end_lo = bend & 0x1ff;
+
+ overlap.dat.obt.fwd = fwd;
+
+ overlap.dat.obt.erate = erate;
+}
+
+
char *
diff --git a/src/AS_OVS/AS_OVS_overlap.H b/src/AS_OVS/AS_OVS_overlap.H
index dd88b62..160695c 100644
--- a/src/AS_OVS/AS_OVS_overlap.H
+++ b/src/AS_OVS/AS_OVS_overlap.H
@@ -22,7 +22,7 @@
#ifndef AS_OVS_OVERLAP_H
#define AS_OVS_OVERLAP_H
-static const char *rcsid_AS_OVS_OVERLAP_H = "$Id: AS_OVS_overlap.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_AS_OVS_OVERLAP_H = "$Id: AS_OVS_overlap.H 4570 2014-10-01 20:36:16Z brianwalenz $";
#include "AS_global.H"
#include "AS_MSG_pmesg.H" // pretty heavy just to get OverlapMesg.
@@ -261,6 +261,8 @@ void AS_OVS_convertOverlapMesgToOVSoverlap(OverlapMesg *omesg, OVSoverlap *ovs)
int AS_OVS_convertOVLdumpToOVSoverlap(char *line, OVSoverlap *olap);
int AS_OVS_convertOBTdumpToOVSoverlap(char *line, OVSoverlap *olap);
+void AS_OVS_convertOVLoverlapToOBToverlap(OVSoverlap &overlap, uint32 clrLenA, uint32 clrLenB);
+
char *AS_OVS_toString(char *outstr, OVSoverlap &olap);
diff --git a/src/AS_OVS/analyze-true-vs-test.pl b/src/AS_OVS/analyze-true-vs-test.pl
new file mode 100644
index 0000000..3112882
--- /dev/null
+++ b/src/AS_OVS/analyze-true-vs-test.pl
@@ -0,0 +1,552 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Reads true overlaps from the first file, separates the second file into
+# true/false based on iid. Hangs are NOT checked.
+
+my $truePrefix = "BL";
+my $testPrefix = "CAordered";
+
+my $trueOverlaps = "$truePrefix/test.ovlStore";
+my $testOverlaps = "$testPrefix/test.ovlStore";
+
+my $filtTrue = "filtered.true";
+my $filtFalse = "filtered.false";
+
+#
+# Load read lengths, make sure both assemblies are the same
+#
+
+my @readLengths;
+my %contained;
+
+open(F, "gatekeeper -dumpfragments -tabular $truePrefix/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ $readLengths[$v[1]] = $v[9];
+}
+close(F);
+
+open(F, "gatekeeper -dumpfragments -tabular $testPrefix/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ die "IID $v[1] BL $readLengths[$v[1]] != CA $v[9]\n" if ($readLengths[$v[1]] != $v[9]);
+}
+close(F);
+
+open(F, "< $testPrefix/4-unitigger/best.contains");
+while (<F>) {
+ my @v = split '\s+', $_;
+ $contained{$v[0]}++;
+}
+close(F);
+
+
+
+#
+# Load true overlaps
+#
+
+my %true; # If set, id pair is a true overlap
+my %trueComputed; # If set, id pair is a true overlap, and was computed
+
+{
+ my $last; # local storage
+ my @lenIdent; # local storage of length-vs-ident for true overlaps. ident is not known.
+
+ open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n";
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ if ($last != $aIID) {
+ open(O, "> iid$last.len-vs-ident.dat");
+ print O @lenIdent;
+ close(O);
+
+ undef @lenIdent;
+ }
+
+ my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang);
+
+ $true{"$aIID-$bIID"} = $oLen;
+ $true{"$bIID-$aIID"} = $oLen;
+
+ next if (exists($contained{$aIID}));
+ next if (exists($contained{$bIID}));
+
+ push @lenIdent, "$oLen\t$12.0\n";
+
+ $last = $aIID;
+ }
+ close(F);
+}
+
+print STDERR "true: ", scalar(keys %true) / 2, "\n";
+
+
+
+
+#
+# Process computed overlaps
+#
+
+my %trueH;
+my %falseH;
+my %keysH;
+
+my @lenIdent5T;
+my @lenIdent5F;
+
+my @lenIdent3T;
+my @lenIdent3F;
+
+my $longestF = 0;
+my $longestT = 0;
+
+
+{
+my $last = 0;
+
+open(LT, "> all.true.length-vs-ident.dat");
+open(LF, "> all.false.length-vs-ident.dat");
+
+open(TO, "> all.true.filtered-overlaps.ova");
+open(FO, "> all.false.filtered-overlaps.ova");
+
+open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$testOverlaps' for reading.\n";
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ if ($last != $aIID) {
+ if (scalar(keys %falseH) > 0) {
+
+ # error rate histogram
+ if (1) {
+ open(O, "> iid$last.eratehistogram.dat");
+ foreach my $ii (sort { $a <=> $b } keys %keysH) {
+ $trueH{$ii} += 0.0;
+ $falseH{$ii} += 0.0;
+ print O "$ii\t$trueH{$ii}\t$falseH{$ii}\n";
+ }
+ close(O);
+
+ open(O, "| gnuplot > /dev/null 2> /dev/null");
+ print O "set terminal 'png'\n";
+ print O "set output 'iid$last.eratehistogram.png'\n";
+ print O "plot 'iid$last.eratehistogram.dat' using 1:3 with lines title 'FALSE',";
+ print O " 'iid$last.eratehistogram.dat' using 1:2 with lines title 'TRUE'\n";
+ close(O);
+ }
+
+
+ # length vs identitiy
+ if (1) { #0.5 * $longestT < $longestF) {
+ #print STDERR "IID $last true: len=$longestT false: len=$longestF\n";
+
+ open(O, "> iid$last.len-vs-ident.5.true.dat");
+ print O @lenIdent5T;
+ close(O);
+
+ open(O, "> iid$last.len-vs-ident.5.false.dat");
+ print O @lenIdent5F;
+ close(O);
+
+ open(O, "> iid$last.len-vs-ident.3.true.dat");
+ print O @lenIdent3T;
+ close(O);
+
+ open(O, "> iid$last.len-vs-ident.3.false.dat");
+ print O @lenIdent3F;
+ close(O);
+
+ open(O, "| gnuplot > /dev/null 2> /dev/null");
+ print O "set terminal 'png'\n";
+ print O "set output 'iid$last.len-vs-ident.png'\n";
+ print O "plot [10:35] [0:15000]";
+ print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,";
+ print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2,";
+ print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,";
+ print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2,";
+ print O " 'iid$last.len-vs-ident.dat' using 2:1 title 'MISSED' pt 3 lc 3\n";
+
+ print O "set output 'iid$last.5.len-vs-ident.png'\n";
+ print O "plot [10:35] [0:15000]";
+ print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,";
+ print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2\n";
+
+ print O "set output 'iid$last.3.len-vs-ident.png'\n";
+ print O "plot [10:35] [0:15000]";
+ print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,";
+ print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2\n";
+ close(O);
+ }
+ }
+
+ undef %trueH;
+ undef %falseH;
+
+ undef %keysH;
+
+ undef @lenIdent5T;
+ undef @lenIdent5F;
+
+ undef @lenIdent3T;
+ undef @lenIdent3F;
+
+ $longestT = 0;
+ $longestF = 0;
+ }
+
+ if (exists($true{"$aIID-$bIID"})) {
+ print TO "$_\n";
+ } else {
+ print FO "$_\n";
+ }
+
+ $trueComputed{"$aIID-$bIID"}++;
+
+ next if (exists($contained{$aIID}));
+ next if (exists($contained{$bIID}));
+
+ my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang);
+
+ my $intErate = int($origErate);
+
+ my $is5 = ($aHang < 0);
+
+ # Computing the number of matches instead of actual length makes every overlap shorter
+ # than truth, and they all get flagged in output
+ #
+ #$oLen -= int($oLen * $origErate / 100);
+
+ if (exists($true{"$aIID-$bIID"})) {
+ $trueH{$intErate}++;
+ $keysH{$intErate}++;
+
+ print LT "$oLen\t$origErate\n";
+
+ if ($is5) {
+ push @lenIdent5T, "$oLen\t$origErate\n";
+ } else {
+ push @lenIdent3T, "$oLen\t$origErate\n";
+ }
+
+ if ($longestT < $oLen) {
+ $longestT = $oLen;
+ }
+
+ } else {
+ $falseH{$intErate}++;
+ $keysH{$intErate}++;
+
+ print LF "$oLen\t$origErate\n";
+
+ if ($is5) {
+ push @lenIdent5F, "$oLen\t$origErate\n";
+ } else {
+ push @lenIdent3F, "$oLen\t$origErate\n";
+ }
+
+ if ($longestF < $oLen) {
+ $longestF = $oLen;
+ }
+ }
+
+ $last = $aIID;
+}
+close(F);
+
+close(LT);
+close(LF);
+
+close(TO);
+close(FO);
+}
+
+
+print "convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb\n";
+system("convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb");
+
+print "convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb\n";
+system("convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb");
+
+print "rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore\n";
+system("rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore");
+
+print "overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb\n";
+system("overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb");
+
+print "overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb\n";
+system("overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb");
+
+
+
+
+
+
+
+
+my %bestTrueEdge;
+my %bestTrueContain;
+
+open(F, "< $truePrefix/4-unitigger/best.edges") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTrueEdge{"$v[0]-$v[2]"}++;
+ $bestTrueEdge{"$v[2]-$v[0]"}++;
+
+ $bestTrueEdge{"$v[0]-$v[4]"}++;
+ $bestTrueEdge{"$v[4]-$v[0]"}++;
+}
+close(F);
+
+open(F, "< $truePrefix/4-unitigger/best.contains") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTrueContain{"$v[0]-$v[3]"}++;
+}
+close(F);
+
+
+
+my %bestTestEdge;
+my %bestTestContain;
+
+open(F, "< $testPrefix/4-unitigger/best.edges") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTestEdge{"$v[0]-$v[2]"}++;
+ $bestTestEdge{"$v[2]-$v[0]"}++;
+
+ $bestTestEdge{"$v[0]-$v[4]"}++;
+ $bestTestEdge{"$v[4]-$v[0]"}++;
+}
+close(F);
+
+open(F, "< $testPrefix/4-unitigger/best.contains") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTestContain{"$v[0]-$v[3]"}++;
+}
+close(F);
+
+
+#
+# Output true overlaps that we missed
+#
+
+open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n";
+
+open(O, "> true-overlaps.missed.other.ova");
+open(E, "> true-overlaps.missed.edge.ova");
+open(C, "> true-overlaps.missed.contain.ova");
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ next if (exists($trueComputed{"$aIID-$bIID"}));
+
+ if (exists($bestTrueEdge{"$aIID-$bIID"})) {
+ print E "$_\n";
+
+ } elsif (exists($bestTrueContain{"$aIID-$bIID"})) {
+ print C "$_\n";
+
+ } else {
+ print O "$_\n";
+ }
+}
+close(O);
+close(F);
+
+
+#
+# Classify computed overlaps into true/false for each type (best edge, best contain, the others).
+#
+
+open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n";
+
+open(BET, "> computed-overlaps.bestE.true.dat");
+open(BEN, "> computed-overlaps.bestE.near.dat");
+open(BEF, "> computed-overlaps.bestE.false.dat");
+
+open(BCT, "> computed-overlaps.bestC.true.dat");
+open(BCN, "> computed-overlaps.bestC.near.dat");
+open(BCF, "> computed-overlaps.bestC.false.dat");
+
+open(OTT, "> computed-overlaps.other.true.dat");
+open(OTF, "> computed-overlaps.other.false.dat");
+
+open(BETo, "> computed-overlaps.bestE.true.ova");
+open(BENo, "> computed-overlaps.bestE.near.ova");
+open(BEFo, "> computed-overlaps.bestE.false.ova");
+
+open(BCTo, "> computed-overlaps.bestC.true.ova");
+open(BCNo, "> computed-overlaps.bestC.near.ova");
+open(BCFo, "> computed-overlaps.bestC.false.ova");
+
+open(OTTo, "> computed-overlaps.other.true.ova");
+open(OTFo, "> computed-overlaps.other.false.ova");
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ my $pair = "$aIID-$bIID";
+ my $len = computeOverlapLength($aIID, $bIID, $aHang, $bHang);
+
+ $origErate = int(10 * $origErate) / 10;
+
+ my $diff = ($aIID < $bIID) ? $bIID - $aIID : $aIID - $bIID;
+
+ die if ($diff < 0);
+
+ if (exists($bestTestEdge{$pair})) {
+ if (exists($bestTrueEdge{$pair})) {
+ print BET "$origErate\t$len\n";
+ print BETo "$_\n";
+ } elsif (exists($true{$pair})) {
+ print BEN "$origErate\t$len\n";
+ print BENo "$_\n";
+ } else {
+ print BEF "$origErate\t$len\n";
+ print BEFo "$_\n";
+ }
+
+ } elsif (exists($bestTestContain{$pair})) {
+ if (exists($bestTrueContain{$pair})) {
+ print BCT "$origErate\t$len\n";
+ print BCTo "$_\n";
+ } elsif (exists($true{$pair})) {
+ print BCN "$origErate\t$len\n";
+ print BCNo "$_\n";
+ } else {
+ print BCF "$origErate\t$len\n";
+ print BCFo "$_\n";
+ }
+
+ } else {
+ if (exists($true{$pair})) {
+ print OTT "$origErate\t$len\n";
+ print OTTo "$_\n";
+ } else {
+ print OTF "$origErate\t$len\n";
+ print OTFo "$_\n";
+ }
+ }
+}
+
+close(OTF); close(OTFo);
+close(OTT); close(OTTo);
+
+close(BCF); close(BCFo);
+close(BCN); close(BCNo);
+close(BCT); close(BCTo);
+
+close(BEF); close(BEFo);
+close(BEN); close(BENo);
+close(BET); close(BETo);
+
+system("awk '{ print \$1 }' computed-overlaps.bestE.true.dat | sort -n | uniq -c > computed-overlaps.bestE.true.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestE.near.dat | sort -n | uniq -c > computed-overlaps.bestE.near.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestE.false.dat | sort -n | uniq -c > computed-overlaps.bestE.false.eratehist");
+
+system("awk '{ print \$1 }' computed-overlaps.bestC.true.dat | sort -n | uniq -c > computed-overlaps.bestC.true.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestC.near.dat | sort -n | uniq -c > computed-overlaps.bestC.near.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestC.false.dat | sort -n | uniq -c > computed-overlaps.bestC.false.eratehist");
+
+system("awk '{ print \$1 }' computed-overlaps.other.true.dat | sort -n | uniq -c > computed-overlaps.other.true.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.other.false.dat | sort -n | uniq -c > computed-overlaps.other.false.eratehist");
+
+
+
+open(F, "> tmp.gp");
+
+print F "set terminal png\n";
+print F "\n";
+print F "set output 'computed-overlaps.bestE.eratehist.png'\n";
+print F "set title 'Computed Overlaps, best edges, error rate histogram'\n";
+print F "plot 'computed-overlaps.bestE.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n";
+print F " 'computed-overlaps.bestE.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n";
+print F " 'computed-overlaps.bestE.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, best contains, error rate histogram'\n";
+print F "set output 'computed-overlaps.bestC.eratehist.png'\n";
+print F "plot 'computed-overlaps.bestC.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n";
+print F " 'computed-overlaps.bestC.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n";
+print F " 'computed-overlaps.bestC.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, non-best, error rate histogram'\n";
+print F "set output 'computed-overlaps.other.eratehist.png'\n";
+print F "plot 'computed-overlaps.other.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n";
+print F " 'computed-overlaps.other.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n";
+print F "\n";
+print F "\n";
+print F "set title 'Computed Overlaps, best edges non-best, error-vs-length'\n";
+print F "set output 'computed-overlaps.bestE.length-vs-erate.png'\n";
+print F "plot 'computed-overlaps.bestE.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n";
+print F " 'computed-overlaps.bestE.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n";
+print F " 'computed-overlaps.bestE.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, non-best, error-vs-length'\n";
+print F "set output 'computed-overlaps.bestC.length-vs-erate.png'\n";
+print F "plot 'computed-overlaps.bestC.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n";
+print F " 'computed-overlaps.bestC.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n";
+print F " 'computed-overlaps.bestC.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, non-best, error-vs-length'\n";
+print F "set output 'computed-overlaps.other.length-vs-erate.png'\n";
+print F "plot 'computed-overlaps.other.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n";
+print F " 'computed-overlaps.other.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n";
+close(F);
+
+system("gnuplot tmp.gp && rm tmp.gp");
+
+
+
+
+
+
+
+
+sub computeOverlapLength ($$$$) {
+ my ($aIID, $bIID, $aHang, $bHang) = @_;
+
+ my $aLen = $readLengths[$aIID];
+ my $bLen = $readLengths[$bIID];
+
+ my $aOvl = 0;
+ my $bOvl = 0;
+
+ # Swiped from bogart
+
+ if ($aHang < 0) {
+ # bHang < 0 ? ---------- : ----
+ # ? ---------- : ----------
+ #
+ $aOvl = ($bHang < 0) ? ($aLen + $bHang) : ($aLen);
+ $bOvl = ($bHang < 0) ? ($bLen + $aHang) : ($bLen + $aHang - $bHang);
+ } else {
+ # bHang < 0 ? ---------- : ----------
+ # ? ---- : ----------
+ #
+ $aOvl = ($bHang < 0) ? ($aLen - $aHang + $bHang) : ($aLen - $aHang);
+ $bOvl = ($bHang < 0) ? ($bLen) : ($bLen - $bHang);
+ }
+
+ return(($aOvl + $bOvl) / 2);
+}
diff --git a/src/AS_OVS/check-ordered-reads-for-missed-overlaps.pl b/src/AS_OVS/check-ordered-reads-for-missed-overlaps.pl
new file mode 100644
index 0000000..2dbbf06
--- /dev/null
+++ b/src/AS_OVS/check-ordered-reads-for-missed-overlaps.pl
@@ -0,0 +1,132 @@
+#!/usr/bin/perl
+
+# Analyzes a single overlapStore, reports reads that have a gap in overlap. Assumes reads are ordered in
+# reference order.
+#
+# Expect a picture of:
+#
+# b ----------
+# -- < contain we correctly don't have overlap to
+# ----------
+# ----------
+# ========== < read in question
+# ----------
+# ----------
+# ----------
+# a ----------
+#
+# For some read iid N, we expect it to have overlap to iids N-b ... N+a.
+# Problem: on the b (before) side, contains can be missing
+# Problem: on both sides, we cannot detect if we're missing the thinnest overlap
+#
+
+
+use strict;
+
+my $dataset;
+
+$dataset = "BL";
+$dataset = "CAordered";
+
+my $maxB = 0;
+my $maxA = 0;
+
+sub processIIDs ($@) {
+ my $thisIID = shift @_;
+ my @iids = sort { $a <=> $b } @_;
+
+ return if (scalar(@iids) == 0);
+
+ my $minIID = $iids[0];
+ my $curIID = 0;
+ my $maxIID = $iids[scalar(@iids) - 1];
+
+ my $expectedB = 0;
+ my $expectedA = 0;
+
+ my $foundB = 0;
+ my $foundA = 0;
+
+ my $missingB = 0;
+ my $missingA = 0;
+
+ foreach my $iid (@iids) {
+ #print "$iid -- $curIID\n";
+
+ next if ($iid < $thisIID - 30); # Ignore false overlaps
+ next if ($iid > $thisIID + 30);
+
+ $expectedB = $thisIID - $iid if ($expectedB < $thisIID - $iid);
+ $expectedA = $iid - $thisIID if ($expectedA < $iid - $thisIID);
+
+ $curIID = $iid - 1 if ($curIID == 0); # If curIID==0, first time here, set to make no missing.
+
+ if ($iid < $thisIID) {
+ $foundB++;
+ if ($curIID + 1 < $iid) {
+ #print STDERR " missingB $curIID - $iid\n";
+ $missingB += $iid - 1 - $curIID;
+ }
+
+ } else {
+ $foundA++;
+ if ($curIID + 1 < $iid) {
+ print STDERR " missingA $curIID - $iid\n";
+ $missingA += $iid - 1 - $curIID;
+ }
+ }
+
+ $curIID = $iid;
+
+ $curIID++ if ($curIID + 1 == $thisIID);
+ }
+
+ my $found = $foundB + $foundA;
+ my $expected = $expectedB + $expectedA;
+ my $missing = $missingB + $missingA;
+
+ if ($missingA > 0) {
+ my $fracB = int(10000 * $missingB / $expectedB) / 100;
+ my $fracA = int(10000 * $missingA / $expectedA) / 100;
+
+ print "$thisIID range $expectedB/$expectedA found $foundB/$foundA missing $missingB/$missingA $fracB%/$fracA%\n";
+ }
+}
+
+
+
+
+
+
+my $lastIID = 0;
+my @iids;
+
+open(F, "overlapStore -d JUNKTEST3/$dataset/test.ovlStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ # v[0] = iid
+ # v[1] = iid
+ # v[2] = orient
+ # v[3] = hang
+ # v[4] = hang
+ # v[5] = ident
+ # v[6] = ident
+
+ my @v = split '\s+', $_;
+
+ if ($v[0] != $lastIID) {
+ #print STDERR "PROCESS $lastIID ", scalar(@iids), "\n";
+
+ processIIDs($lastIID, @iids);
+
+ undef @iids;
+ $lastIID = $v[0];
+ }
+
+ #print STDERR "$v[0] $v[1]\n";
+
+ push @iids, $v[1];
+}
+close(F);
diff --git a/src/AS_OVS/filterTrue.pl b/src/AS_OVS/filterTrue.pl
new file mode 100644
index 0000000..b40bf51
--- /dev/null
+++ b/src/AS_OVS/filterTrue.pl
@@ -0,0 +1,549 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Reads true overlaps from the first file, separates the second file into
+# true/false based on iid. Hangs are NOT checked.
+
+my $trueOverlaps = "BL/test.ovlStore";
+my $testOverlaps = "CAordered/test.ovlStore";
+
+my $filtTrue = "filtered.true";
+my $filtFalse = "filtered.false";
+
+#
+# Load read lengths, make sure both assemblies are the same
+#
+
+my @readLengths;
+my %contained;
+
+open(F, "gatekeeper -dumpfragments -tabular BL/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ $readLengths[$v[1]] = $v[9];
+}
+close(F);
+
+open(F, "gatekeeper -dumpfragments -tabular CAordered/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ die "IID $v[1] BL $readLengths[$v[1]] != CA $v[9]\n" if ($readLengths[$v[1]] != $v[9]);
+}
+close(F);
+
+open(F, "< CAordered/4-unitigger/best.contains");
+while (<F>) {
+ my @v = split '\s+', $_;
+ $contained{$v[0]}++;
+}
+close(F);
+
+
+
+#
+# Load true overlaps
+#
+
+my %true; # If set, id pair is a true overlap
+my %trueComputed; # If set, id pair is a true overlap, and was computed
+
+{
+ my $last; # local storage
+ my @lenIdent; # local storage of length-vs-ident for true overlaps. ident is not known.
+
+ open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n";
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ if ($last != $aIID) {
+ open(O, "> iid$last.len-vs-ident.dat");
+ print O @lenIdent;
+ close(O);
+
+ undef @lenIdent;
+ }
+
+ my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang);
+
+ $true{"$aIID-$bIID"} = $oLen;
+ $true{"$bIID-$aIID"} = $oLen;
+
+ next if (exists($contained{$aIID}));
+ next if (exists($contained{$bIID}));
+
+ push @lenIdent, "$oLen\t$12.0\n";
+
+ $last = $aIID;
+ }
+ close(F);
+}
+
+print STDERR "true: ", scalar(keys %true) / 2, "\n";
+
+
+
+
+#
+# Process computed overlaps
+#
+
+my %trueH;
+my %falseH;
+my %keysH;
+
+my @lenIdent5T;
+my @lenIdent5F;
+
+my @lenIdent3T;
+my @lenIdent3F;
+
+my $longestF = 0;
+my $longestT = 0;
+
+
+{
+my $last = 0;
+
+open(LT, "> all.true.length-vs-ident.dat");
+open(LF, "> all.false.length-vs-ident.dat");
+
+open(TO, "> all.true.filtered-overlaps.ova");
+open(FO, "> all.false.filtered-overlaps.ova");
+
+open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$testOverlaps' for reading.\n";
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ if ($last != $aIID) {
+ if (scalar(keys %falseH) > 0) {
+
+ # error rate histogram
+ if (1) {
+ open(O, "> iid$last.eratehistogram.dat");
+ foreach my $ii (sort { $a <=> $b } keys %keysH) {
+ $trueH{$ii} += 0.0;
+ $falseH{$ii} += 0.0;
+ print O "$ii\t$trueH{$ii}\t$falseH{$ii}\n";
+ }
+ close(O);
+
+ open(O, "| gnuplot > /dev/null 2> /dev/null");
+ print O "set terminal 'png'\n";
+ print O "set output 'iid$last.eratehistogram.png'\n";
+ print O "plot 'iid$last.eratehistogram.dat' using 1:3 with lines title 'FALSE',";
+ print O " 'iid$last.eratehistogram.dat' using 1:2 with lines title 'TRUE'\n";
+ close(O);
+ }
+
+
+ # length vs identitiy
+ if (1) { #0.5 * $longestT < $longestF) {
+ #print STDERR "IID $last true: len=$longestT false: len=$longestF\n";
+
+ open(O, "> iid$last.len-vs-ident.5.true.dat");
+ print O @lenIdent5T;
+ close(O);
+
+ open(O, "> iid$last.len-vs-ident.5.false.dat");
+ print O @lenIdent5F;
+ close(O);
+
+ open(O, "> iid$last.len-vs-ident.3.true.dat");
+ print O @lenIdent3T;
+ close(O);
+
+ open(O, "> iid$last.len-vs-ident.3.false.dat");
+ print O @lenIdent3F;
+ close(O);
+
+ open(O, "| gnuplot > /dev/null 2> /dev/null");
+ print O "set terminal 'png'\n";
+ print O "set output 'iid$last.len-vs-ident.png'\n";
+ print O "plot [10:35] [0:15000]";
+ print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,";
+ print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2,";
+ print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,";
+ print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2,";
+ print O " 'iid$last.len-vs-ident.dat' using 2:1 title 'MISSED' pt 3 lc 3\n";
+
+ print O "set output 'iid$last.5.len-vs-ident.png'\n";
+ print O "plot [10:35] [0:15000]";
+ print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,";
+ print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2\n";
+
+ print O "set output 'iid$last.3.len-vs-ident.png'\n";
+ print O "plot [10:35] [0:15000]";
+ print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,";
+ print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2\n";
+ close(O);
+ }
+ }
+
+ undef %trueH;
+ undef %falseH;
+
+ undef %keysH;
+
+ undef @lenIdent5T;
+ undef @lenIdent5F;
+
+ undef @lenIdent3T;
+ undef @lenIdent3F;
+
+ $longestT = 0;
+ $longestF = 0;
+ }
+
+ if (exists($true{"$aIID-$bIID"})) {
+ print TO "$_\n";
+ } else {
+ print FO "$_\n";
+ }
+
+ $trueComputed{"$aIID-$bIID"}++;
+
+ next if (exists($contained{$aIID}));
+ next if (exists($contained{$bIID}));
+
+ my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang);
+
+ my $intErate = int($origErate);
+
+ my $is5 = ($aHang < 0);
+
+ # Computing the number of matches instead of actual length makes every overlap shorter
+ # than truth, and they all get flagged in output
+ #
+ #$oLen -= int($oLen * $origErate / 100);
+
+ if (exists($true{"$aIID-$bIID"})) {
+ $trueH{$intErate}++;
+ $keysH{$intErate}++;
+
+ print LT "$oLen\t$origErate\n";
+
+ if ($is5) {
+ push @lenIdent5T, "$oLen\t$origErate\n";
+ } else {
+ push @lenIdent3T, "$oLen\t$origErate\n";
+ }
+
+ if ($longestT < $oLen) {
+ $longestT = $oLen;
+ }
+
+ } else {
+ $falseH{$intErate}++;
+ $keysH{$intErate}++;
+
+ print LF "$oLen\t$origErate\n";
+
+ if ($is5) {
+ push @lenIdent5F, "$oLen\t$origErate\n";
+ } else {
+ push @lenIdent3F, "$oLen\t$origErate\n";
+ }
+
+ if ($longestF < $oLen) {
+ $longestF = $oLen;
+ }
+ }
+
+ $last = $aIID;
+}
+close(F);
+
+close(LT);
+close(LF);
+
+close(TO);
+close(FO);
+}
+
+
+print "convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb\n";
+system("convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb");
+
+print "convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb\n";
+system("convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb");
+
+print "rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore\n";
+system("rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore");
+
+print "overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb\n";
+system("overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb");
+
+print "overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb\n";
+system("overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb");
+
+
+
+
+
+
+
+
+my %bestTrueEdge;
+my %bestTrueContain;
+
+open(F, "< BL/4-unitigger/best.edges") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTrueEdge{"$v[0]-$v[2]"}++;
+ $bestTrueEdge{"$v[2]-$v[0]"}++;
+
+ $bestTrueEdge{"$v[0]-$v[4]"}++;
+ $bestTrueEdge{"$v[4]-$v[0]"}++;
+}
+close(F);
+
+open(F, "< BL/4-unitigger/best.contains") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTrueContain{"$v[0]-$v[3]"}++;
+}
+close(F);
+
+
+
+my %bestTestEdge;
+my %bestTestContain;
+
+open(F, "< CAordered/4-unitigger/best.edges") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTestEdge{"$v[0]-$v[2]"}++;
+ $bestTestEdge{"$v[2]-$v[0]"}++;
+
+ $bestTestEdge{"$v[0]-$v[4]"}++;
+ $bestTestEdge{"$v[4]-$v[0]"}++;
+}
+close(F);
+
+open(F, "< CAordered/4-unitigger/best.contains") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $bestTestContain{"$v[0]-$v[3]"}++;
+}
+close(F);
+
+
+#
+# Output true overlaps that we missed
+#
+
+open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n";
+
+open(O, "> true-overlaps.missed.other.ova");
+open(E, "> true-overlaps.missed.edge.ova");
+open(C, "> true-overlaps.missed.contain.ova");
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ next if (exists($trueComputed{"$aIID-$bIID"}));
+
+ if (exists($bestTrueEdge{"$aIID-$bIID"})) {
+ print E "$_\n";
+
+ } elsif (exists($bestTrueContain{"$aIID-$bIID"})) {
+ print C "$_\n";
+
+ } else {
+ print O "$_\n";
+ }
+}
+close(O);
+close(F);
+
+
+#
+# Classify computed overlaps into true/false for each type (best edge, best contain, the others).
+#
+
+open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n";
+
+open(BET, "> computed-overlaps.bestE.true.dat");
+open(BEN, "> computed-overlaps.bestE.near.dat");
+open(BEF, "> computed-overlaps.bestE.false.dat");
+
+open(BCT, "> computed-overlaps.bestC.true.dat");
+open(BCN, "> computed-overlaps.bestC.near.dat");
+open(BCF, "> computed-overlaps.bestC.false.dat");
+
+open(OTT, "> computed-overlaps.other.true.dat");
+open(OTF, "> computed-overlaps.other.false.dat");
+
+open(BETo, "> computed-overlaps.bestE.true.ova");
+open(BENo, "> computed-overlaps.bestE.near.ova");
+open(BEFo, "> computed-overlaps.bestE.false.ova");
+
+open(BCTo, "> computed-overlaps.bestC.true.ova");
+open(BCNo, "> computed-overlaps.bestC.near.ova");
+open(BCFo, "> computed-overlaps.bestC.false.ova");
+
+open(OTTo, "> computed-overlaps.other.true.ova");
+open(OTFo, "> computed-overlaps.other.false.ova");
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_;
+
+ my $pair = "$aIID-$bIID";
+ my $len = computeOverlapLength($aIID, $bIID, $aHang, $bHang);
+
+ $origErate = int(10 * $origErate) / 10;
+
+ my $diff = ($aIID < $bIID) ? $bIID - $aIID : $aIID - $bIID;
+
+ die if ($diff < 0);
+
+ if (exists($bestTestEdge{$pair})) {
+ if (exists($bestTrueEdge{$pair})) {
+ print BET "$origErate\t$len\n";
+ print BETo "$_\n";
+ } elsif (exists($true{$pair})) {
+ print BEN "$origErate\t$len\n";
+ print BENo "$_\n";
+ } else {
+ print BEF "$origErate\t$len\n";
+ print BEFo "$_\n";
+ }
+
+ } elsif (exists($bestTestContain{$pair})) {
+ if (exists($bestTrueContain{$pair})) {
+ print BCT "$origErate\t$len\n";
+ print BCTo "$_\n";
+ } elsif (exists($true{$pair})) {
+ print BCN "$origErate\t$len\n";
+ print BCNo "$_\n";
+ } else {
+ print BCF "$origErate\t$len\n";
+ print BCFo "$_\n";
+ }
+
+ } else {
+ if (exists($true{$pair})) {
+ print OTT "$origErate\t$len\n";
+ print OTTo "$_\n";
+ } else {
+ print OTF "$origErate\t$len\n";
+ print OTFo "$_\n";
+ }
+ }
+}
+
+close(OTF); close(OTFo);
+close(OTT); close(OTTo);
+
+close(BCF); close(BCFo);
+close(BCN); close(BCNo);
+close(BCT); close(BCTo);
+
+close(BEF); close(BEFo);
+close(BEN); close(BENo);
+close(BET); close(BETo);
+
+system("awk '{ print \$1 }' computed-overlaps.bestE.true.dat | sort -n | uniq -c > computed-overlaps.bestE.true.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestE.near.dat | sort -n | uniq -c > computed-overlaps.bestE.near.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestE.false.dat | sort -n | uniq -c > computed-overlaps.bestE.false.eratehist");
+
+system("awk '{ print \$1 }' computed-overlaps.bestC.true.dat | sort -n | uniq -c > computed-overlaps.bestC.true.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestC.near.dat | sort -n | uniq -c > computed-overlaps.bestC.near.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.bestC.false.dat | sort -n | uniq -c > computed-overlaps.bestC.false.eratehist");
+
+system("awk '{ print \$1 }' computed-overlaps.other.true.dat | sort -n | uniq -c > computed-overlaps.other.true.eratehist");
+system("awk '{ print \$1 }' computed-overlaps.other.false.dat | sort -n | uniq -c > computed-overlaps.other.false.eratehist");
+
+
+
+open(F, "> tmp.gp");
+
+print F "set terminal png\n";
+print F "\n";
+print F "set output 'computed-overlaps.bestE.eratehist.png'\n";
+print F "set title 'Computed Overlaps, best edges, error rate histogram'\n";
+print F "plot 'computed-overlaps.bestE.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n";
+print F " 'computed-overlaps.bestE.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n";
+print F " 'computed-overlaps.bestE.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, best contains, error rate histogram'\n";
+print F "set output 'computed-overlaps.bestC.eratehist.png'\n";
+print F "plot 'computed-overlaps.bestC.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n";
+print F " 'computed-overlaps.bestC.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n";
+print F " 'computed-overlaps.bestC.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, non-best, error rate histogram'\n";
+print F "set output 'computed-overlaps.other.eratehist.png'\n";
+print F "plot 'computed-overlaps.other.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n";
+print F " 'computed-overlaps.other.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n";
+print F "\n";
+print F "\n";
+print F "set title 'Computed Overlaps, best edges non-best, error-vs-length'\n";
+print F "set output 'computed-overlaps.bestE.length-vs-erate.png'\n";
+print F "plot 'computed-overlaps.bestE.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n";
+print F " 'computed-overlaps.bestE.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n";
+print F " 'computed-overlaps.bestE.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, non-best, error-vs-length'\n";
+print F "set output 'computed-overlaps.bestC.length-vs-erate.png'\n";
+print F "plot 'computed-overlaps.bestC.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n";
+print F " 'computed-overlaps.bestC.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n";
+print F " 'computed-overlaps.bestC.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n";
+print F "\n";
+print F "set title 'Computed Overlaps, non-best, error-vs-length'\n";
+print F "set output 'computed-overlaps.other.length-vs-erate.png'\n";
+print F "plot 'computed-overlaps.other.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n";
+print F " 'computed-overlaps.other.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n";
+close(F);
+
+system("gnuplot tmp.gp && rm tmp.gp");
+
+
+
+
+
+
+
+
+sub computeOverlapLength ($$$$) {
+ my ($aIID, $bIID, $aHang, $bHang) = @_;
+
+ my $aLen = $readLengths[$aIID];
+ my $bLen = $readLengths[$bIID];
+
+ my $aOvl = 0;
+ my $bOvl = 0;
+
+ # Swiped from bogart
+
+ if ($aHang < 0) {
+ # bHang < 0 ? ---------- : ----
+ # ? ---------- : ----------
+ #
+ $aOvl = ($bHang < 0) ? ($aLen + $bHang) : ($aLen);
+ $bOvl = ($bHang < 0) ? ($bLen + $aHang) : ($bLen + $aHang - $bHang);
+ } else {
+ # bHang < 0 ? ---------- : ----------
+ # ? ---- : ----------
+ #
+ $aOvl = ($bHang < 0) ? ($aLen - $aHang + $bHang) : ($aLen - $aHang);
+ $bOvl = ($bHang < 0) ? ($bLen) : ($bLen - $bHang);
+ }
+
+ return(($aOvl + $bOvl) / 2);
+}
diff --git a/src/AS_OVS/find-missed-true-overlaps.pl b/src/AS_OVS/find-missed-true-overlaps.pl
new file mode 100644
index 0000000..7cc8b8e
--- /dev/null
+++ b/src/AS_OVS/find-missed-true-overlaps.pl
@@ -0,0 +1,91 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Given:
+# an assembly of true inferred overlaps
+# an assembly of computed overlaps
+# report the number of true overlaps that are missed in the computed overlaps
+
+my %truthOverlaps;
+my @readLengths;
+
+
+open(F, "gatekeeper -dumpfragments -tabular JUNKTEST3/BL/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ $readLengths[$v[1]] = $v[9];
+}
+close(F);
+
+open(F, "gatekeeper -dumpfragments -tabular JUNKTEST3/CAordered/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ die "IID $v[1] BL $readLengths[$v[1]] != CA $v[9]\n" if ($readLengths[$v[1]] != $v[9]);
+}
+close(F);
+
+
+
+
+open(F, "overlapStore -d JUNKTEST3/BL/test.ovlStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origID, $corrID) = split '\s+', $_;
+
+ my $aLen = $readLengths[$aIID];
+ my $bLen = $readLengths[$bIID];
+
+ my $aOvl = 0;
+ my $bOvl = 0;
+ my $oLen = 0;
+
+ # Swiped from bogart
+
+ if ($aHang < 0) {
+ # bHang < 0 ? ---------- : ----
+ # ? ---------- : ----------
+ #
+ $aOvl = ($bHang < 0) ? ($aLen + $bHang) : ($aLen);
+ $bOvl = ($bHang < 0) ? ($bLen + $aHang) : ($bLen + $aHang - $bHang);
+ } else {
+ # bHang < 0 ? ---------- : ----------
+ # ? ---- : ----------
+ #
+ $aOvl = ($bHang < 0) ? ($aLen - $aHang + $bHang) : ($aLen - $aHang);
+ $bOvl = ($bHang < 0) ? ($bLen) : ($bLen - $bHang);
+ }
+
+ $oLen = ($aOvl + $bOvl) / 2;
+
+ $truthOverlaps{"$aIID-$bIID"}++;
+}
+close(F);
+
+print STDERR "truthOverlaps: ", scalar(keys %truthOverlaps), "\n";
+
+open(F, "overlapStore -d JUNKTEST3/CAordered/test.ovlStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ # v[0] = iid
+ # v[1] = iid
+ # v[2] = orient
+ # v[3] = hang
+ # v[4] = hang
+ # v[5] = ident
+ # v[6] = ident
+
+ my ($aIID, $bIID, $orient, $aHang, $bHang, $origID, $corrID) = split '\s+', $_;
+
+
+ delete $truthOverlaps{"$aIID-$bIID"};
+}
+close(F);
+
+print STDERR "truthOverlaps missed: ", scalar(keys %truthOverlaps), "\n";
diff --git a/src/AS_OVS/infer-obt-from-genomic-blasr.pl b/src/AS_OVS/infer-obt-from-genomic-blasr.pl
new file mode 100644
index 0000000..4352555
--- /dev/null
+++ b/src/AS_OVS/infer-obt-from-genomic-blasr.pl
@@ -0,0 +1,43 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Convert BLASR default output to OBT overlaps: aIID bIID [f|r] aBgn aEnd bBgn bEnd error
+
+my %IDmap;
+
+open(F, "< $ARGV[0]") or die "Failed to open '$ARGV[0]' for reading sequence names.\n";
+while (<F>) {
+ my ($uid, $iid, $name) = split '\s+', $_;
+
+ $IDmap{$name} = $iid;
+}
+close(F);
+
+
+while (<STDIN>) {
+ my @v = split '\s+', $_;
+
+ # The first read seems to have a sub-read range appended.
+
+ if ($v[0] =~ m/^(.*)\/\d+_\d+$/) {
+ $v[0] = $1;
+ }
+
+ my $aiid = $IDmap{$v[0]};
+ my $biid = $IDmap{$v[1]};
+ my $fr = ($v[8] == 0) ? "f" : "r";
+ my $error = 100.0 - $v[3];
+
+ ($v[9], $v[10]) = ($v[10], $v[9]) if ($v[8] == 1);
+
+ die "No A iid found for '$v[0]'\n" if (!defined($aiid));
+ die "No B iid found for '$v[1]'\n" if (!defined($biid));
+
+ next if ($aiid == $biid);
+
+ # HACK!
+ #$error = 0.01;
+
+ print "$aiid\t$biid\t$fr\t$v[5]\t$v[6]\t$v[9]\t$v[10]\t$error\n";
+}
diff --git a/src/AS_OVS/infer-olaps-from-genomic-coords.pl b/src/AS_OVS/infer-olaps-from-genomic-coords.pl
new file mode 100644
index 0000000..51fa8ce
--- /dev/null
+++ b/src/AS_OVS/infer-olaps-from-genomic-coords.pl
@@ -0,0 +1,316 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $outputPrefix = $ARGV[0];
+my $readsFile = $ARGV[1];
+my $coordsFile = $ARGV[2];
+my $uidMapFile = $ARGV[3];
+
+my $orientForward = 0;
+
+#
+# Load name to IID map
+#
+
+my %NAMEtoIID;
+my %UIDtoNAME;
+
+if (defined($uidMapFile)) {
+ print STDERR "Read names from '$uidMapFile'\n";
+
+ open(F, "< $uidMapFile");
+ while (<F>) {
+ my @v = split '\s+', $_;
+
+ # UID IID NAME UID IID NAME
+
+ if (scalar(@v) == 3) {
+ $NAMEtoIID{$v[2]} = $v[1];
+ $UIDtoNAME{$v[0]} = $v[2];
+ }
+
+ if (scalar(@v) == 6) {
+ $NAMEtoIID{$v[2]} = $v[1];
+ $NAMEtoIID{$v[5]} = $v[4];
+ $UIDtoNAME{$v[0]} = $v[2];
+ $UIDtoNAME{$v[3]} = $v[5];
+ }
+ }
+ close(F);
+}
+
+#
+# Load read lengths. Needed to find reads that do not map full length, and to compute hangs.
+# Assumes reads are dumped from gkpStore and have names 'uid,iid'.
+
+my %readLength;
+my %totalAligns;
+my %goodAligns;
+
+print STDERR "Read lengths from '$readsFile'\n";
+
+open(F, "< $readsFile") or die "Couldn't read '$readsFile'\n";
+while (!eof(F)) {
+ my $a = <F>;
+ my $b = <F>;
+ my $c = <F>;
+ my $d = <F>;
+
+ if ($a =~ m/^.(\S+)$/) {
+ if (! exists($NAMEtoIID{$1})) {
+ #print STDERR "WARNING: read '$1' not in map.\n"
+ } else {
+ $readLength{$1} = length($b) - 1;
+ $totalAligns{$1} = 0;
+ $goodAligns{$1} = 0;
+ }
+ } else {
+ chomp;
+ die "Failed to parse read name from '$a'\n";
+ }
+}
+close(F);
+
+
+
+
+
+my @aligns;
+my $totalAligns = 0;
+my $goodAligns = 0;
+
+my $readOrder = 0;
+my %positions;
+
+
+open(F, "< $coordsFile");
+
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ my $b = $v[0]; # reference begin
+ my $e = $v[1]; # reference end
+
+ my $r = $v[9]; # reference name
+ my $n = $v[10]; # read name
+
+ my $f = 1;
+ my $L = $v[2]; # read begin, unused
+ my $R = $v[3]; # read end, unused
+
+ if ($R < $L) {
+ $f = 0;
+ $L = $v[3];
+ $R = $v[2];
+ }
+
+ die "L=$L R=$R\n" if ($R < $L);
+ die "b=$b e=$e\n" if ($e < $b);
+
+ next if (!exists($readLength{$n}));
+ die "Didn't find read length for read '$n'\n" if (!exists($readLength{$n}));
+
+ $totalAligns++;
+ $totalAligns{$n}++;
+
+ next if (10 < $L);
+ next if (10 < $readLength{$n} - $R);
+
+ $goodAligns++;
+ $goodAligns{$n}++;
+
+ my $scale = ($R - $L) / ($e - $b); # scale from reference coords to read coords
+
+ # The aligns line starts with reference name. This lets us group aligns per reference sequence.
+
+ push @aligns, "$r\0$b\0$e\0$f\0$scale\0$n";
+}
+
+
+# Generate olaps. We sort by reference name, then step through all aligns to that reference and
+# sort by begin coordinate.
+
+my $olaps = 0;
+
+open(OVA, "> $outputPrefix.ova");
+
+ at aligns = sort @aligns;
+
+while (scalar(@aligns) > 0) {
+ my @coords;
+
+ my @v = split '\0', $aligns[0];
+ my $r = $v[0];
+
+ # Find all coords for this reference sequence
+
+ while ($r eq $v[0]) {
+ push @coords, "$v[1]\0$v[2]\0$v[3]\0$v[4]\0$v[5]";
+
+ shift @aligns;
+ @v = split '\0', $aligns[0];
+ }
+
+ # Sort those coords by start position
+
+ @coords = sort { $a <=> $b } @coords;
+
+ # Process each one
+
+ while (scalar(@coords) > 0) {
+ my ($rb, $re, $rf, $rs, $rn) = split '\0', $coords[0];
+
+ die "Reversed $coords[0]\n" if ($re < $rb);
+
+ next if (!exists($NAMEtoIID{$rn})); # Just skip reads we don't care about.
+ die if (!exists($NAMEtoIID{$rn})); # For development, fail.
+
+ # Save this reads position
+
+ if (exists($positions{$rn})) {
+ print STDERR "WARNING: read $rn already has a position!\n";
+ }
+ $positions{$rn} = $readOrder++;
+
+ # Relabel the read name to an IID.
+
+ $rn = $NAMEtoIID{$rn};
+
+ shift @coords;
+
+ foreach my $cc (@coords) {
+ my ($cb, $ce, $cf, $cs, $cn) = split '\0', $cc;
+ my $olap = $re - $cb;
+
+ last if ($olap < 40);
+
+ my $ahang;
+ my $bhang;
+
+ if ($rf == 1) {
+ # Read is forward
+ $ahang = $cb - $rb;
+ $bhang = $ce - $re;
+ } else {
+ # Read is reverse; hangs are swapped and negative from forward case
+ $ahang = $re - $ce;
+ $bhang = $rb - $cb;
+ }
+
+ next if (!exists($NAMEtoIID{$cn}));
+ die if (!exists($NAMEtoIID{$cn}));
+
+ $cn = $NAMEtoIID{$cn};
+
+ # Scale the hangs. Hangs are computed using genomic positions, but deletions from the read
+ # cause this hang to be too large.
+
+ if ($ahang < 0) {
+ $ahang = $ahang * $cs; # hang is composed of the C read
+ } else {
+ $ahang = $ahang * $rs; # hang is composed of the R read
+ }
+
+ if ($bhang < 0) {
+ $bhang = $bhang * $rs; # hang is composed of the R read
+ } else {
+ $bhang = $bhang * $cs; # hang is composed of the C read
+ }
+
+ printf(OVA "%9s %9s %s %5d %5d %.2f %.2f\n",
+ $rn,
+ $cn,
+ ($rf == $cf) ? "N" : "I",
+ $ahang,
+ $bhang,
+ 0.0,
+ 0.0);
+
+ $olaps++;
+ }
+ }
+}
+
+close(OVA);
+
+# Output reads that aligned, didn't align.
+
+open(R, "< $readsFile");
+open(M, "> $outputPrefix.mapped.fastq");
+open(P, "> $outputPrefix.partial.fastq");
+open(F, "> $outputPrefix.failed.fastq");
+
+my $mappedReads = 0;
+my $partialReads = 0;
+my $failedReads = 0;
+
+my @mappedReads;
+my @orientReads;
+
+while (!eof(R)) {
+ my $a = <R>;
+ my $b = <R>;
+ my $c = <R>;
+ my $d = <R>;
+ my $n;
+
+ if ($a =~ m/\@(\S+)\s*/) {
+ $n = $1;
+ } else {
+ chomp;
+ die "Failed to parse read name from '$a'\n";
+ }
+
+ #die "Failed to find UIDtoNAME for UID '$n'\n" if (!exists($UIDtoNAME{$n}));
+ #
+ #$a = "\@$UIDtoNAME{$n}\n";
+
+ if ($goodAligns{$n} > 0) {
+ $mappedReads++;
+ print M "$a$b$c$d";
+
+ die "No position for '$n'\n" if (!exists($positions{$n}));
+
+ push @mappedReads, "$positions{$n}\0$a$b$c$d";
+ push @orientReads, "$positions{$n}\0$a$b$c$d";
+
+ } elsif ($totalAligns{$n} > 0) {
+ $partialReads++;
+ print P "$a$b$c$d";
+
+ } else {
+ $failedReads++;
+ print F "$a$b$c$d";
+ }
+}
+
+close(F);
+close(P);
+close(M);
+close(R);
+
+ at mappedReads = sort { $a <=> $b } @mappedReads;
+ at orientReads = sort { $a <=> $b } @orientReads;
+
+open(A, "> $outputPrefix.mapped.ordered.fastq");
+foreach my $a (@mappedReads) {
+ my ($idx,$dat) = split '\0', $a;
+ print A "$dat";
+}
+close(A);
+
+open(A, "> $outputPrefix.mapped.ordered.oriented.fastq");
+foreach my $a (@orientReads) {
+ my ($idx,$dat) = split '\0', $a;
+ print A "$dat";
+}
+close(A);
+
+
+print STDERR "RESULT $outputPrefix ", scalar(keys %readLength), " input $mappedReads mapped $partialReads partial $failedReads unaligned reads, $totalAligns aligns, $goodAligns good aligns, $olaps overlaps.\n";
diff --git a/src/AS_OVS/infer-olaps-from-pairwise-blasr.pl b/src/AS_OVS/infer-olaps-from-pairwise-blasr.pl
new file mode 100644
index 0000000..755f570
--- /dev/null
+++ b/src/AS_OVS/infer-olaps-from-pairwise-blasr.pl
@@ -0,0 +1,152 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Convert BLASR default output to OVL overlaps: aIID bIID [I|N] aHang bHang error error_corrected
+
+die "Don't use BLASR default output; use SAM format.\n"
+
+my %IDmap;
+
+# BLASR is not reporting symmetric overlaps. We need to keep track of which overlaps we have reported.
+
+my %reported;
+
+
+open(F, "< $ARGV[0]") or die "Failed to open '$ARGV[0]' for reading sequence names.\n";
+while (<F>) {
+ my ($uid, $iid, $name) = split '\s+', $_;
+
+ $IDmap{$name} = $iid;
+}
+close(F);
+
+while (<STDIN>) {
+ my @v = split '\s+', $_;
+
+ $_ = join '\t', $_;
+
+ # The first read seems to have a sub-read range appended.
+ if ($v[0] =~ m/^(.*)\/\d+_\d+$/) {
+ $v[0] = $1;
+ }
+
+ my $aiid;
+ my $biid;
+
+ # Or it might have UID,IID. If this is ALWAYS the case, we don't need the uid map, yay!
+ if ($v[0] =~ m/^\d+,(\d+)$/) {
+ $aiid = $1;
+ } else {
+ $aiid = $IDmap{$v[0]};
+ }
+
+ if ($v[1] =~ m/^\d+,(\d+)$/) {
+ $biid = $1;
+ } else {
+ $biid = $IDmap{$v[1]};
+ }
+
+ my $ni = ($v[8] == 0) ? "N" : "I";
+ my $error = 100.0 - $v[3];
+ my $ecorr = 100.0 - $v[3];
+
+ if ((exists($reported{"$aiid-$biid"}) || (exists($reported{"$biid-$aiid"})))) {
+ next;
+ }
+
+ $reported{"$aiid-$biid"} = 0;
+
+ # Argh! Do NOT flip coords if reversed.
+ #($v[9], $v[10]) = ($v[10], $v[9]) if ($v[8] == 1);
+
+ die "No A iid found for '$v[0]'\n" if (!defined($aiid));
+ die "No B iid found for '$v[1]'\n" if (!defined($biid));
+
+ next if ($aiid == $biid);
+ #next if ($aiid < $biid);
+
+ die "First read is flipped\n$_\n" if ($v[4] != 0);
+
+ my $a1 = $v[5]; # Amount unaligned on left of first
+ my $b1 = $v[7] - $v[6];
+
+ my $a2 = $v[9];
+ my $b2 = $v[11] - $v[10];
+
+ #print "'$v[7]' '$v[8]' '$v[9]' '$v[10]'\n";
+
+ #print "a1 $a1 from [5] $v[5]\n";
+ #print "b1 $b1 from [7]-[6] $v[7] - $v[6]\n";
+ #print "a2 $a2 from [9] $v[9]\n";
+ #print "b2 $b2 from [11]-[10] $v[11] - $v[10]\n";
+
+ my $fl = $v[8];
+
+ my $ahang = 0;
+ my $bhang = 0;
+ my $label = "";
+
+ # Handle A contained in B
+ if (($a1 == 0) && ($b1 == 0)) {
+ if ($fl == 0) {
+ $ahang = -$a2;
+ $bhang = $b2;
+ $label = "AinBf";
+ } else {
+ $ahang = -$b2;
+ $bhang = $a2;
+ $label = "AinBr";
+ }
+ }
+
+ # Handle B contained in A
+ elsif (($a2 == 0) && ($b2 == 0)) {
+ if ($fl == 0) {
+ $ahang = $a1;
+ $bhang = -$b1;
+ $label = "BinAf";
+ } else {
+ $ahang = $b1;
+ $bhang = -$a1;
+ $label = "BinAr";
+ }
+ }
+
+ # Handle a dovetail off the left end of A
+ elsif (($a1 == 0) && ($b2 == 0) && ($fl == 0)) {
+ $ahang = -$a2;
+ $bhang = -$b1;
+ $label = "BdoveAf";
+ }
+ elsif (($a1 == 0) && ($a2 == 0) && ($fl == 1)) {
+ $ahang = -$b2;
+ $bhang = -$b1;
+ $label = "BdoveAr";
+ }
+
+ # Handle dovetail off the right end of A
+ elsif (($b1 == 0) && ($a2 == 0) && ($fl == 0)) {
+ $ahang = $a1;
+ $bhang = $b2;
+ $label = "AdoveBf";
+ }
+ elsif (($b1 == 0) && ($b2 == 0) && ($fl == 1)) {
+ $ahang = $a1;
+ $bhang = $a2;
+ $label = "AdoveBr";
+ }
+
+ # All the rest aren't valid overlaps.
+ else {
+ next;
+ $label = "INVALID";
+ }
+
+ # HACK!
+ $error = 0.01;
+ $ecorr = 0.01;
+
+ #print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\t$label\t$_";
+ print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\n";
+}
diff --git a/src/AS_OVS/infer-olaps-from-pairwise-coords.pl b/src/AS_OVS/infer-olaps-from-pairwise-coords.pl
new file mode 100644
index 0000000..2eb269b
--- /dev/null
+++ b/src/AS_OVS/infer-olaps-from-pairwise-coords.pl
@@ -0,0 +1,184 @@
+#!/usr/bin/perl
+
+use strict;
+
+# cd /work/blasr-overlapper/JUNKTEST3/CAordered
+# time sh /work/scripts/blasr-overlaps.sh BLASROVERLAP.fasta BLASROVERLAP.fasta BLASROVERLAP
+#7043.059u 16.347s 22:09.56 530.9% 6573+816k 11+192523io 11881pf+0w
+
+#
+# NMAME to IID MAP
+#
+
+my %NAMEtoIID;
+
+open(F, "< CAordered/test.gkpStore.fastqUIDmap") or die "Failed to open 'CAordered/test.gkpStore.fastqUIDmap'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ #$IIDtoNAME{$v[1]} = $v[2];
+ $NAMEtoIID{$v[2]} = $v[1];
+}
+close(F);
+
+#
+# READ LENGTHS
+#
+
+my @readLength;
+my %contained;
+
+open(F, "gatekeeper -dumpfragments -tabular BL/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ $readLength[$v[1]] = $v[9];
+}
+close(F);
+
+open(F, "gatekeeper -dumpfragments -tabular CAordered/test.gkpStore |");
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ die "IID $v[1] BL $readLength[$v[1]] != CA $v[9]\n" if ($readLength[$v[1]] != $v[9]);
+}
+close(F);
+
+#
+#
+#
+
+my %reported;
+
+
+open(F, "< BO/PAIRS.blasr.sam.coords") or die "Failed to open 'BO/PAIRS.blasr.sam.coords'\n";
+open(O, "> BO/PAIRS.blasr.sam.ova");
+
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+$_ = <F>;
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ my $aIID = $NAMEtoIID{$v[9]};
+ my $bIID = $NAMEtoIID{$v[10]};
+
+ next if (!defined($aIID)); # Extra overlaps are not an error.
+ next if (!defined($bIID));
+
+ die "undef for a $v[9]\n" if (!defined($aIID)); # Extra overlaps are an error.
+ die "undef for b $v[10]\n" if (!defined($bIID));
+
+ next if ($aIID == $bIID);
+
+ next if (exists($reported{"$aIID-$bIID"}));
+
+ $reported{"$aIID-$bIID"}++;
+ $reported{"$bIID-$aIID"}++;
+
+
+ my $l1 = $readLength[$aIID];
+ my $a1 = $v[0] - 1; # Amount unaligned on left of first
+ my $b1 = $l1 - $v[1]; # Amount unaligned on right of first
+
+ my $ori;
+
+ my $l2 = $readLength[$bIID];
+ my $a2;
+ my $b2;
+
+ if ($v[2] < $v[3]) {
+ $ori = "N";
+ $a2 = $v[2] - 1;
+ $b2 = $l2 - $v[3];
+ } else {
+ $ori = "I";
+ $a2 = $v[3] - 1;
+ $b2 = $l2 - $v[2];
+ }
+
+ # Extend near global to be global.
+
+ my $maxTol = 10;
+
+ $a1 = 0 if ($a1 < $maxTol);
+ $b1 = $l1 if ($l1 < $b1 + $maxTol);
+
+ $a2 = 0 if ($a2 < $maxTol);
+ $b2 = $l2 if ($l2 < $b2 + $maxTol);
+
+ my $fl = ($ori eq "I");
+
+ my $ahang = 0;
+ my $bhang = 0;
+ my $label = "";
+
+ # Handle A contained in B
+ if (($a1 == 0) && ($b1 == 0)) {
+ if ($fl == 0) {
+ $ahang = -$a2;
+ $bhang = $b2;
+ $label = "AinBf";
+ } else {
+ $ahang = -$b2;
+ $bhang = $a2;
+ $label = "AinBr";
+ }
+ }
+
+ # Handle B contained in A
+ elsif (($a2 == 0) && ($b2 == 0)) {
+ if ($fl == 0) {
+ $ahang = $a1;
+ $bhang = -$b1;
+ $label = "BinAf";
+ } else {
+ $ahang = $b1;
+ $bhang = -$a1;
+ $label = "BinAr";
+ }
+ }
+
+ # Handle a dovetail off the left end of A
+ elsif (($a1 == 0) && ($b2 == 0) && ($fl == 0)) {
+ $ahang = -$a2;
+ $bhang = -$b1;
+ $label = "BdoveAf";
+ }
+ elsif (($a1 == 0) && ($a2 == 0) && ($fl == 1)) {
+ $ahang = -$b2;
+ $bhang = -$b1;
+ $label = "BdoveAr";
+ }
+
+ # Handle dovetail off the right end of A
+ elsif (($b1 == 0) && ($a2 == 0) && ($fl == 0)) {
+ $ahang = $a1;
+ $bhang = $b2;
+ $label = "AdoveBf";
+ }
+ elsif (($b1 == 0) && ($b2 == 0) && ($fl == 1)) {
+ $ahang = $a1;
+ $bhang = $a2;
+ $label = "AdoveBr";
+ }
+
+ # All the rest aren't valid overlaps.
+ else {
+ next;
+ $label = "INVALID";
+ }
+
+ my $error = 100 - $v[8];
+ my $ecorr = 100 - $v[8];
+
+ #print "$aIID\t$bIID\t$ni\t$ahang\t$bhang\t$error\t$ecorr\t$label\t$_";
+ print O "$aIID\t$bIID\t$ori\t$ahang\t$bhang\t$error\t$ecorr\n";
+}
+close(F);
+close(O);
+
+system("convertOverlap -ovl < BO/PAIRS.blasr.sam.ova > BO/PAIRS.blasr.sam.ovb");
+system("overlapStoreBuild -g CAordered/test.gkpStore -o blasr-pairs.ovlStore -F 1 BO/PAIRS.blasr.sam.ovb");
+
diff --git a/src/AS_OVS/infer-ovl-from-genomic-blasr.pl b/src/AS_OVS/infer-ovl-from-genomic-blasr.pl
new file mode 100644
index 0000000..db91c19
--- /dev/null
+++ b/src/AS_OVS/infer-ovl-from-genomic-blasr.pl
@@ -0,0 +1,150 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Convert BLASR default output to OVL overlaps: aIID bIID [I|N] aHang bHang error error_corrected
+
+my %IDmap;
+
+# BLASR is not reporting symmetric overlaps. We need to keep track of which overlaps we have reported.
+
+my %reported;
+
+
+open(F, "< $ARGV[0]") or die "Failed to open '$ARGV[0]' for reading sequence names.\n";
+while (<F>) {
+ my ($uid, $iid, $name) = split '\s+', $_;
+
+ $IDmap{$name} = $iid;
+}
+close(F);
+
+while (<STDIN>) {
+ my @v = split '\s+', $_;
+
+ $_ = join '\t', $_;
+
+ # The first read seems to have a sub-read range appended.
+ if ($v[0] =~ m/^(.*)\/\d+_\d+$/) {
+ $v[0] = $1;
+ }
+
+ my $aiid;
+ my $biid;
+
+ # Or it might have UID,IID. If this is ALWAYS the case, we don't need the uid map, yay!
+ if ($v[0] =~ m/^\d+,(\d+)$/) {
+ $aiid = $1;
+ } else {
+ $aiid = $IDmap{$v[0]};
+ }
+
+ if ($v[1] =~ m/^\d+,(\d+)$/) {
+ $biid = $1;
+ } else {
+ $biid = $IDmap{$v[1]};
+ }
+
+ my $ni = ($v[8] == 0) ? "N" : "I";
+ my $error = 100.0 - $v[3];
+ my $ecorr = 100.0 - $v[3];
+
+ if ((exists($reported{"$aiid-$biid"}) || (exists($reported{"$biid-$aiid"})))) {
+ next;
+ }
+
+ $reported{"$aiid-$biid"} = 0;
+
+ # Argh! Do NOT flip coords if reversed.
+ #($v[9], $v[10]) = ($v[10], $v[9]) if ($v[8] == 1);
+
+ die "No A iid found for '$v[0]'\n" if (!defined($aiid));
+ die "No B iid found for '$v[1]'\n" if (!defined($biid));
+
+ next if ($aiid == $biid);
+ #next if ($aiid < $biid);
+
+ die "First read is flipped\n$_\n" if ($v[4] != 0);
+
+ my $a1 = $v[5]; # Amount unaligned on left of first
+ my $b1 = $v[7] - $v[6];
+
+ my $a2 = $v[9];
+ my $b2 = $v[11] - $v[10];
+
+ #print "'$v[7]' '$v[8]' '$v[9]' '$v[10]'\n";
+
+ #print "a1 $a1 from [5] $v[5]\n";
+ #print "b1 $b1 from [7]-[6] $v[7] - $v[6]\n";
+ #print "a2 $a2 from [9] $v[9]\n";
+ #print "b2 $b2 from [11]-[10] $v[11] - $v[10]\n";
+
+ my $fl = $v[8];
+
+ my $ahang = 0;
+ my $bhang = 0;
+ my $label = "";
+
+ # Handle A contained in B
+ if (($a1 == 0) && ($b1 == 0)) {
+ if ($fl == 0) {
+ $ahang = -$a2;
+ $bhang = $b2;
+ $label = "AinBf";
+ } else {
+ $ahang = -$b2;
+ $bhang = $a2;
+ $label = "AinBr";
+ }
+ }
+
+ # Handle B contained in A
+ elsif (($a2 == 0) && ($b2 == 0)) {
+ if ($fl == 0) {
+ $ahang = $a1;
+ $bhang = -$b1;
+ $label = "BinAf";
+ } else {
+ $ahang = $b1;
+ $bhang = -$a1;
+ $label = "BinAr";
+ }
+ }
+
+ # Handle a dovetail off the left end of A
+ elsif (($a1 == 0) && ($b2 == 0) && ($fl == 0)) {
+ $ahang = -$a2;
+ $bhang = -$b1;
+ $label = "BdoveAf";
+ }
+ elsif (($a1 == 0) && ($a2 == 0) && ($fl == 1)) {
+ $ahang = -$b2;
+ $bhang = -$b1;
+ $label = "BdoveAr";
+ }
+
+ # Handle dovetail off the right end of A
+ elsif (($b1 == 0) && ($a2 == 0) && ($fl == 0)) {
+ $ahang = $a1;
+ $bhang = $b2;
+ $label = "AdoveBf";
+ }
+ elsif (($b1 == 0) && ($b2 == 0) && ($fl == 1)) {
+ $ahang = $a1;
+ $bhang = $a2;
+ $label = "AdoveBr";
+ }
+
+ # All the rest aren't valid overlaps.
+ else {
+ next;
+ $label = "INVALID";
+ }
+
+ # HACK!
+ $error = 0.01;
+ $ecorr = 0.01;
+
+ #print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\t$label\t$_";
+ print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\n";
+}
diff --git a/src/AS_OVS/overlapStore.C b/src/AS_OVS/overlapStore.C
index 48ed7b7..4f1d60b 100644
--- a/src/AS_OVS/overlapStore.C
+++ b/src/AS_OVS/overlapStore.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: overlapStore.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: overlapStore.C 4577 2014-10-21 14:44:28Z brianwalenz $";
#include "overlapStore.H"
#include "AS_OVS_overlap.H" // Just to know the sizes of structs
@@ -34,6 +34,7 @@ main(int argc, char **argv) {
uint32 dumpBinary = FALSE;
double dumpERate = 100.0;
+ uint32 dumpLength = 0;
uint32 dumpType = 0;
char *erateFile = NULL;
@@ -42,6 +43,8 @@ main(int argc, char **argv) {
uint32 endIID = UINT32_MAX;
uint32 qryIID = 0;
+ bool beVerbose = false;
+
// Genome size parameters
uint32 gs_ovlLimit = 100;
uint32 gs_winSize = 100;
@@ -92,6 +95,9 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-E") == 0) {
dumpERate = atof(argv[++arg]);
+ } else if (strcmp(argv[arg], "-L") == 0) {
+ dumpLength = atoi(argv[++arg]);
+
} else if (strcmp(argv[arg], "-d5") == 0) {
dumpType |= DUMP_5p;
@@ -113,6 +119,9 @@ main(int argc, char **argv) {
} else if (strcmp(argv[arg], "-e") == 0) {
endIID = atoi(argv[++arg]);
+ } else if (strcmp(argv[arg], "-v") == 0) {
+ beVerbose = true;
+
} else if (strcmp(argv[arg], "-q") == 0) {
if (storeName)
fprintf(stderr, "ERROR: only one of -c, -m, -d, -q, -s, -S or -u may be supplied.\n"), err++;
@@ -150,12 +159,14 @@ main(int argc, char **argv) {
fprintf(stderr, "DUMPING - report overlaps in the store\n");
fprintf(stderr, " -B Dump the store as binary, suitable for input to create a new store.\n");
fprintf(stderr, " -E erate Dump only overlaps <= erate error.\n");
+ fprintf(stderr, " -L length Dump only overlaps that are larger than L bases (only for -p picture mode).\n");
fprintf(stderr, " -d5 Dump only overlaps off the 5' end of the A frag.\n");
fprintf(stderr, " -d3 Dump only overlaps off the 3' end of the A frag.\n");
fprintf(stderr, " -dC Dump only overlaps that are contained in the A frag (B contained in A).\n");
fprintf(stderr, " -dc Dump only overlaps that are containing the A frag (A contained in B).\n");
fprintf(stderr, " -b beginIID Start dumping at 'beginIID'.\n");
fprintf(stderr, " -e endIID Stop dumping after 'endIID'.\n");
+ fprintf(stderr, " -v Report statistics (to stderr) on some dumps (-d).\n");
fprintf(stderr, "\n");
fprintf(stderr, "QUERYING - quickly ask if an overlap exists\n");
fprintf(stderr, " -q aiid biid storeName\n");
@@ -174,10 +185,10 @@ main(int argc, char **argv) {
switch (operation) {
case OP_DUMP:
- dumpStore(storeName, dumpBinary, dumpERate, dumpType, bgnIID, endIID, qryIID);
+ dumpStore(storeName, dumpBinary, dumpERate, dumpLength, dumpType, bgnIID, endIID, qryIID, beVerbose);
break;
case OP_DUMP_PICTURE:
- dumpPicture(storeName, gkpName, clearRegion, dumpERate, dumpType, qryIID);
+ dumpPicture(storeName, gkpName, clearRegion, dumpERate, dumpLength, dumpType, qryIID);
break;
case OP_GENOME_LENGTH:
estimateGenomeLength(storeName,
diff --git a/src/AS_OVS/overlapStore.H b/src/AS_OVS/overlapStore.H
index e366045..2053d74 100644
--- a/src/AS_OVS/overlapStore.H
+++ b/src/AS_OVS/overlapStore.H
@@ -22,7 +22,7 @@
#ifndef OVERLAPSTORE_H
#define OVERLAPSTORE_H
-static const char *rcsid_OVERLAPSTORE_H = "$Id: overlapStore.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_OVERLAPSTORE_H = "$Id: overlapStore.H 4577 2014-10-21 14:44:28Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -42,10 +42,24 @@ void
updateErates(char *storeName, char *eratesName);
void
-dumpStore(char *storeName, uint32 dumpBinary, double dumpERate, uint32 dumpType, uint32 bgnIID, uint32 endIID, uint32 qryIID);
+dumpStore(char *storeName,
+ uint32 dumpBinary,
+ double dumpERate,
+ uint32 dumpLength,
+ uint32 dumpType,
+ uint32 bgnIID,
+ uint32 endIID,
+ uint32 qryIID,
+ bool beVerbose);
void
-dumpPicture(char *ovlName, char *gkpName, uint32 clearRegion, double dumpERate, uint32 dumpType, uint32 qryIID);
+dumpPicture(char *ovlName,
+ char *gkpName,
+ uint32 clearRegion,
+ double dumpERate,
+ uint32 dumpLength,
+ uint32 dumpType,
+ uint32 qryIID);
void
estimateGenomeLength(char *ovlName,
diff --git a/src/AS_OVS/overlapStoreBucketizer.C b/src/AS_OVS/overlapStoreBucketizer.C
index 4131330..dc56dff 100644
--- a/src/AS_OVS/overlapStoreBucketizer.C
+++ b/src/AS_OVS/overlapStoreBucketizer.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: overlapStoreBucketizer.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: overlapStoreBucketizer.C 4559 2014-08-22 20:08:12Z brianwalenz $";
#include "AS_global.H"
@@ -74,7 +74,7 @@ writeToFile(OVSoverlap *overlap,
if (sliceFile[df] == NULL) {
char name[FILENAME_MAX];
- sprintf(name, "%s/bucket%04d/slice%03d%s", ovlName, jobIndex, df, (useGzip) ? ".gz" : "");
+ sprintf(name, "%s/create%04d/slice%03d%s", ovlName, jobIndex, df, (useGzip) ? ".gz" : "");
sliceFile[df] = AS_OVS_createBinaryOverlapFile(name, FALSE);
sliceSize[df] = 0;
}
@@ -271,14 +271,20 @@ main(int argc, char **argv) {
{
- char name[FILENAME_MAX];
-
if (AS_UTL_fileExists(ovlName, TRUE, FALSE) == false)
AS_UTL_mkdir(ovlName);
+ }
+
+
+ {
+ char name[FILENAME_MAX];
+
+ sprintf(name, "%s/create%04d", ovlName, jobIndex);
- sprintf(name, "%s/bucket%04d", ovlName, jobIndex);
if (AS_UTL_fileExists(name, TRUE, FALSE) == false)
AS_UTL_mkdir(name);
+ else
+ fprintf(stderr, "Overwriting previous result; directory '%s' exists.\n", name), exit(0);
}
@@ -287,10 +293,8 @@ main(int argc, char **argv) {
sprintf(name, "%s/bucket%04d/sliceSizes", ovlName, jobIndex);
- if (AS_UTL_fileExists(name, FALSE, FALSE) == true) {
- fprintf(stderr, "Job finished; file '%s' exists.\n", name);
- exit(0);
- }
+ if (AS_UTL_fileExists(name, FALSE, FALSE) == true)
+ fprintf(stderr, "Job finished; file '%s' exists.\n", name), exit(0);
}
@@ -476,8 +480,9 @@ main(int argc, char **argv) {
{
char name[FILENAME_MAX];
+ char finl[FILENAME_MAX];
- sprintf(name, "%s/bucket%04d/sliceSizes", ovlName, jobIndex);
+ sprintf(name, "%s/create%04d/sliceSizes", ovlName, jobIndex);
FILE *F = fopen(name, "w");
if (errno)
@@ -486,6 +491,15 @@ main(int argc, char **argv) {
AS_UTL_safeWrite(F, sliceSize, "sliceSize", sizeof(uint64), fileLimit + 1);
fclose(F);
+
+ sprintf(name, "%s/create%04d", ovlName, jobIndex);
+ sprintf(finl, "%s/bucket%04d", ovlName, jobIndex);
+
+ errno = 0;
+ rename(name, finl);
+ if (errno)
+ fprintf(stderr, "ERROR: Failed to rename '%s' to final name '%s': %s\n",
+ name, finl, strerror(errno));
}
fprintf(stderr, "overlap fate:\n");
diff --git a/src/AS_OVS/overlapStoreBuild.C b/src/AS_OVS/overlapStoreBuild.C
index a25b3ff..1cccbf3 100644
--- a/src/AS_OVS/overlapStoreBuild.C
+++ b/src/AS_OVS/overlapStoreBuild.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: overlapStoreBuild.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: overlapStoreBuild.C 4550 2014-07-31 13:56:46Z brianwalenz $";
#include "AS_global.H"
@@ -44,8 +44,8 @@ using namespace std;
#define AS_OVS_CURRENT_VERSION 2
-
-
+uint32 lastLibFirstIID = 0;
+uint32 lastLibLastIID = 0;
static
@@ -199,6 +199,22 @@ writeToDumpFile(OVSoverlap *overlap,
uint32 df = overlap->a_iid / iidPerBucket;
+ if (lastLibFirstIID > 0) {
+ uint32 firstHighDensity = lastLibFirstIID; // IID of first pacBio read
+ uint32 lastHighDensity = lastLibLastIID + 1; // IID of last pacBio read, plus 1
+ uint32 numHighDensity = lastHighDensity - firstHighDensity;
+
+ uint32 lowDensity = firstHighDensity / 64; // 64 buckets for illumina overlaps
+ uint32 highDensity = numHighDensity / 128; // 128 buckets for dense overlaps
+
+ if (overlap->a_iid < firstHighDensity)
+ df = overlap->a_iid / lowDensity;
+ else
+ df = (overlap->a_iid - firstHighDensity) / highDensity + 64; // plus 64 buckets from above
+ }
+
+ //fprintf(stderr, "IID %u DF %u\n", overlap->a_iid, df);
+
if (df >= dumpFileMax) {
char olapstring[256];
@@ -318,6 +334,9 @@ main(int argc, char **argv) {
fclose(F);
+ } else if (strcmp(argv[arg], "-big") == 0) {
+ lastLibFirstIID = atoi(argv[++arg]);
+
} else if ((argv[arg][0] == '-') && (argv[arg][1] != 0)) {
fprintf(stderr, "%s: unknown option '%s'.\n", argv[0], argv[arg]);
err++;
@@ -351,6 +370,10 @@ main(int argc, char **argv) {
fprintf(stderr, "\n");
fprintf(stderr, " -e e filter overlaps above e fraction error\n");
fprintf(stderr, " -L fileList read input filenames from 'flieList'\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " -big iid handle a large number of overlaps in the last library\n");
+ fprintf(stderr, " iid is the first read iid in the last library, from\n");
+ fprintf(stderr, " 'gatekeeper -dumpinfo *gkpStore'\n");
if (ovlName == NULL)
fprintf(stderr, "ERROR: No overlap store (-o) supplied.\n");
@@ -373,9 +396,10 @@ main(int argc, char **argv) {
gkStore *gkp = new gkStore(gkpName, FALSE, FALSE);
uint64 maxIID = gkp->gkStore_getNumFragments() + 1;
- //uint64 iidPerBucket = (uint64)ceil((double)maxIID / (double)fileLimit);
uint64 iidPerBucket = computeIIDperBucket(fileLimit, memoryLimit, maxIID, fileList);
+ lastLibLastIID = gkp->gkStore_getNumFragments();
+
uint32 dumpFileMax = sysconf(_SC_OPEN_MAX) + 1;
BinaryOverlapFile **dumpFile = new BinaryOverlapFile * [dumpFileMax];
uint64 *dumpLength = new uint64 [dumpFileMax];
diff --git a/src/AS_OVS/overlapStore_dump.C b/src/AS_OVS/overlapStore_dump.C
index 79e0c58..8df82d4 100644
--- a/src/AS_OVS/overlapStore_dump.C
+++ b/src/AS_OVS/overlapStore_dump.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-static const char *rcsid = "$Id: overlapStore_dump.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid = "$Id: overlapStore_dump.C 4581 2014-10-23 00:18:25Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -43,76 +43,110 @@ static const char *rcsid = "$Id: overlapStore_dump.C 4371 2013-08-01 17:19:47Z b
void
-dumpStore(char *ovlName, uint32 dumpBinary, double dumpERate, uint32 dumpType, uint32 bgnIID, uint32 endIID, uint32 qryIID) {
+dumpStore(char *ovlName,
+ uint32 dumpBinary,
+ double dumpERate,
+ uint32 dumpType,
+ uint32 dumpLength,
+ uint32 bgnIID,
+ uint32 endIID,
+ uint32 qryIID,
+ bool beVerbose) {
OverlapStore *ovlStore = AS_OVS_openOverlapStore(ovlName);
OVSoverlap overlap;
uint64 erate = AS_OVS_encodeQuality(dumpERate / 100.0);
char ovlString[1024];
+ uint32 ovlTooHighError = 0;
+ uint32 ovlNot5p = 0;
+ uint32 ovlNot3p = 0;
+ uint32 ovlNotContainer = 0;
+ uint32 ovlNotContainee = 0;
+ uint32 ovlDumped = 0;
+ uint32 obtTooHighError = 0;
+ uint32 obtDumped = 0;
+ uint32 merDumped = 0;
+
AS_OVS_setRangeOverlapStore(ovlStore, bgnIID, endIID);
+ // Length filtering is expensive to compute, need to load both reads to get their length.
+ //
+ //if ((dumpLength > 0) && (dumpLength < overlapLength(overlap)))
+ // continue;
+
while (AS_OVS_readOverlapFromStore(ovlStore, &overlap, AS_OVS_TYPE_ANY) == TRUE) {
+ if ((qryIID != 0) && (qryIID != overlap.b_iid))
+ continue;
+
switch (overlap.dat.ovl.type) {
case AS_OVS_TYPE_OVL:
-
- if (overlap.dat.ovl.corr_erate > erate)
- continue;
-
- if ((qryIID != 0) && (qryIID != overlap.b_iid))
+ if (overlap.dat.ovl.corr_erate > erate) {
+ ovlTooHighError++;
continue;
-
- if (((dumpType & DUMP_5p) == 0) &&
- (overlap.dat.ovl.a_hang < 0) && (overlap.dat.ovl.b_hang < 0))
+ }
+ if (((dumpType & DUMP_5p) == 1) &&
+ (overlap.dat.ovl.a_hang < 0) && (overlap.dat.ovl.b_hang < 0)) {
+ ovlNot5p++;
continue;
-
- if (((dumpType & DUMP_3p) == 0) &&
- (overlap.dat.ovl.a_hang > 0) && (overlap.dat.ovl.b_hang > 0))
+ }
+ if (((dumpType & DUMP_3p) == 1) &&
+ (overlap.dat.ovl.a_hang > 0) && (overlap.dat.ovl.b_hang > 0)) {
+ ovlNot3p++;
continue;
-
- if (((dumpType & DUMP_CONTAINS) == 0) &&
- (overlap.dat.ovl.a_hang >= 0) && (overlap.dat.ovl.b_hang <= 0))
+ }
+ if (((dumpType & DUMP_CONTAINS) == 1) &&
+ (overlap.dat.ovl.a_hang >= 0) && (overlap.dat.ovl.b_hang <= 0)) {
+ ovlNotContainer++;
continue;
-
- if (((dumpType & DUMP_CONTAINED) == 0) &&
- (overlap.dat.ovl.a_hang <= 0) && (overlap.dat.ovl.b_hang >= 0))
+ }
+ if (((dumpType & DUMP_CONTAINED) == 1) &&
+ (overlap.dat.ovl.a_hang <= 0) && (overlap.dat.ovl.b_hang >= 0)) {
+ ovlNotContainee++;
continue;
-
-
- if (dumpBinary)
- AS_UTL_safeWrite(stdout, &overlap, "dumpStore", sizeof(OVSoverlap), 1);
- else
- fprintf(stdout, "%s\n", AS_OVS_toString(ovlString, overlap));
+ }
+ ovlDumped++;
break;
case AS_OVS_TYPE_OBT:
- if (overlap.dat.obt.erate > erate)
+ if (overlap.dat.obt.erate > erate) {
+ obtTooHighError++;
continue;
-
- if ((qryIID != 0) && (qryIID != overlap.b_iid))
- continue;
-
- if (dumpBinary)
- AS_UTL_safeWrite(stdout, &overlap, "dumpStore", sizeof(OVSoverlap), 1);
- else
- fprintf(stdout, "%s\n", AS_OVS_toString(ovlString, overlap));
+ }
+ obtDumped++;
break;
case AS_OVS_TYPE_MER:
- if ((qryIID != 0) && (qryIID != overlap.b_iid))
- continue;
-
- if (dumpBinary)
- AS_UTL_safeWrite(stdout, &overlap, "dumpStore", sizeof(OVSoverlap), 1);
- else
- fprintf(stdout, "%s\n", AS_OVS_toString(ovlString, overlap));
+ merDumped++;
break;
default:
assert(0);
break;
}
+
+ // All the slow for this dump is in sprintf() within AS_OVS_toString().
+ // Without both the puts() and AS_OVS_toString(), a dump ran in 3 seconds.
+ // With both, 138 seconds.
+ // Without the puts(), 127 seconds.
+
+ if (dumpBinary)
+ AS_UTL_safeWrite(stdout, &overlap, "dumpStore", sizeof(OVSoverlap), 1);
+ else
+ //fprintf(stdout, "%s\n", AS_OVS_toString(ovlString, overlap));
+ puts(AS_OVS_toString(ovlString, overlap));
}
AS_OVS_closeOverlapStore(ovlStore);
-}
+ if (beVerbose) {
+ fprintf(stderr, "ovlTooHighError %u\n", ovlTooHighError);
+ fprintf(stderr, "ovlNot5p %u\n", ovlNot5p);
+ fprintf(stderr, "ovlNot3p %u\n", ovlNot3p);
+ fprintf(stderr, "ovlNotContainer %u\n", ovlNotContainer);
+ fprintf(stderr, "ovlNotContainee %u\n", ovlNotContainee);
+ fprintf(stderr, "ovlDumped %u\n", ovlDumped);
+ fprintf(stderr, "obtTooHighError %u\n", obtTooHighError);
+ fprintf(stderr, "obtDumped %u\n", obtDumped);
+ fprintf(stderr, "merDumped %u\n", merDumped);
+ }
+}
@@ -133,7 +167,11 @@ sortOBT(const void *a, const void *b) {
void
-dumpPicture(OVSoverlap *overlaps, uint64 novl, gkStore *gkpStore, uint32 clearRegion, uint32 qryIID) {
+dumpPicture(OVSoverlap *overlaps,
+ uint64 novl,
+ gkStore *gkpStore,
+ uint32 clearRegion,
+ uint32 qryIID) {
char ovl[256] = {0};
gkFragment A;
gkFragment B;
@@ -255,42 +293,19 @@ dumpPicture(OVSoverlap *overlaps, uint64 novl, gkStore *gkpStore, uint32 clearRe
void
-dumpPictureOBT(OVSoverlap *overlaps, uint64 novl, gkStore *gkpStore, uint32 clearRegion, uint32 qryIID) {
-
-#if 0
- for (uint32 o=0; o<novl; o++) {
- uint32 abgn = (overlaps[o].dat.obt.a_beg);
- uint32 aend = (overlaps[o].dat.obt.a_end);
-
- uint32 bbgn = (overlaps[o].dat.obt.b_beg);
- uint32 bend = (overlaps[o].dat.obt.b_end_hi << 9) | (overlaps[o].dat.obt.b_end_lo);
-
- if (overlaps[o].dat.obt.fwd) {
- // Does nothing, just replaces with the same stuff.
- overlaps[o].dat.obt.a_beg = abgn;
- overlaps[o].dat.obt.a_end = aend;
- overlaps[o].dat.obt.b_beg = bbgn;
- overlaps[o].dat.obt.b_end_hi = bend >> 9;
- overlaps[o].dat.obt.b_end_lo = bend & 0x1ff;
- } else {
- // Reverse the coords for B to indicate a reversed overlap.
- overlaps[o].dat.obt.a_beg = abgn;
- overlaps[o].dat.obt.a_end = aend;
- overlaps[o].dat.obt.b_beg = bend;
- overlaps[o].dat.obt.b_end_hi = bbgn >> 9;
- overlaps[o].dat.obt.b_end_lo = bbgn & 0x1ff;
- }
- }
-#endif
-
- dumpPicture(overlaps, novl, gkpStore, clearRegion, qryIID);
-}
-
+dumpPicture(char *ovlName,
+ char *gkpName,
+ uint32 clearRegion,
+ double dumpERate,
+ uint32 dumpLength,
+ uint32 dumpType,
+ uint32 qryIID) {
+ fprintf(stderr, "DUMPING PICTURE for ID "F_IID" in store %s (gkp %s clear %s)\n",
+ qryIID, ovlName, gkpName, AS_READ_CLEAR_NAMES[clearRegion]);
+ OverlapStore *ovlStore = AS_OVS_openOverlapStore(ovlName);
+ gkStore *gkpStore = new gkStore(gkpName, FALSE, FALSE);
-void
-dumpPictureOVL(OVSoverlap *overlaps, uint64 novl, gkStore *gkpStore, uint32 clearRegion, uint32 qryIID) {
- char ovl[256] = {0};
gkFragment A;
gkFragment B;
@@ -301,56 +316,6 @@ dumpPictureOVL(OVSoverlap *overlaps, uint64 novl, gkStore *gkpStore, uint32 clea
uint32 frgLenA = clrEndA - clrBgnA;
- // Convert from OVL to OBT overlap form; those are easier to dump....
-
- for (uint32 o=0; o<novl; o++) {
- uint32 clrBgnB, clrEndB;
-
- gkpStore->gkStore_getFragment(overlaps[o].b_iid, &B, GKFRAGMENT_INF);
- B.gkFragment_getClearRegion(clrBgnB, clrEndB, clearRegion);
-
- uint32 frgLenB = clrEndB - clrBgnB;
-
- int32 ahang = overlaps[o].dat.ovl.a_hang;
- int32 bhang = overlaps[o].dat.ovl.b_hang;
-
- uint32 abgn = (ahang < 0) ? (0) : (ahang);
- uint32 aend = (bhang < 0) ? (frgLenA + bhang) : (frgLenA);
- uint32 bbgn = (ahang < 0) ? (-ahang) : (0);
- uint32 bend = (bhang < 0) ? (frgLenB) : (frgLenB - bhang);
-
- uint64 erate = overlaps[o].dat.ovl.corr_erate;
-
- if (overlaps[o].dat.ovl.flipped) {
- bbgn = frgLenB - bbgn;
- bend = frgLenB - bend;
- }
-
- overlaps[o].dat.obt.a_beg = abgn;
- overlaps[o].dat.obt.a_end = aend;
- overlaps[o].dat.obt.b_beg = bbgn;
- overlaps[o].dat.obt.b_end_hi = bend >> 9;
- overlaps[o].dat.obt.b_end_lo = bend & 0x1ff;
-
- overlaps[o].dat.obt.erate = erate;
- }
-
- // ...and the dump function is already written.
-
- dumpPicture(overlaps, novl, gkpStore, clearRegion, qryIID);
-}
-
-
-
-
-void
-dumpPicture(char *ovlName, char *gkpName, uint32 clearRegion, double dumpERate, uint32 dumpType, uint32 qryIID) {
- fprintf(stderr, "DUMPING PICTURE for ID "F_IID" in store %s (gkp %s clear %s)\n",
- qryIID, ovlName, gkpName, AS_READ_CLEAR_NAMES[clearRegion]);
-
- OverlapStore *ovlStore = AS_OVS_openOverlapStore(ovlName);
- gkStore *gkpStore = new gkStore(gkpName, FALSE, FALSE);
-
AS_OVS_setRangeOverlapStore(ovlStore, qryIID, qryIID);
uint64 novl = 0;
@@ -362,11 +327,15 @@ dumpPicture(char *ovlName, char *gkpName, uint32 clearRegion, double dumpERate,
while (AS_OVS_readOverlapFromStore(ovlStore, &overlap, AS_OVS_TYPE_ANY) == TRUE) {
+ // For OBT, the only filter is erate (and length, at the bottom)
+
if (overlap.dat.obt.type == AS_OVS_TYPE_OBT) {
if (overlap.dat.obt.erate > erate)
continue;
}
+ // For OVL, we can also filter on overlap type.
+
if (overlap.dat.obt.type == AS_OVS_TYPE_OVL) {
if (overlap.dat.ovl.corr_erate > erate)
continue;
@@ -388,10 +357,33 @@ dumpPicture(char *ovlName, char *gkpName, uint32 clearRegion, double dumpERate,
continue;
}
+ // Convert the OVL to an OBT overlap, so we can check length. We need to do it anyway for
+ // dumpPicture(), so it isn't a horrible waste.
+
+ if (overlap.dat.obt.type == AS_OVS_TYPE_OVL) {
+ uint32 clrBgnB, clrEndB;
+
+ gkpStore->gkStore_getFragment(overlap.b_iid, &B, GKFRAGMENT_INF);
+ B.gkFragment_getClearRegion(clrBgnB, clrEndB, clearRegion);
+
+ uint32 frgLenB = clrEndB - clrBgnB;
+
+ // NOTE! These aren't proper obt overlaps, the coord shows orientation.
+ AS_OVS_convertOVLoverlapToOBToverlap(overlap, frgLenA, frgLenB);
+ }
+
+ // Finally, check the length.
+
+ if (overlap.dat.obt.type == AS_OVS_TYPE_OBT) {
+ if ((overlap.dat.obt.b_end_hi << 9 | overlap.dat.obt.b_end_lo) - overlap.dat.obt.b_beg < dumpLength)
+ continue;
+ if (overlap.dat.obt.a_end - overlap.dat.obt.a_beg < dumpLength)
+ continue;
+ }
+
overlaps[novl++] = overlap;
}
-
if (novl == 0)
fprintf(stderr, "no overlaps to show.\n");
@@ -399,8 +391,8 @@ dumpPicture(char *ovlName, char *gkpName, uint32 clearRegion, double dumpERate,
fprintf(stderr, "mer overlaps cannot be visualized.\n");
else if (overlaps[0].dat.obt.type == AS_OVS_TYPE_OBT)
- dumpPictureOBT(overlaps, novl, gkpStore, clearRegion, qryIID);
+ dumpPicture(overlaps, novl, gkpStore, clearRegion, qryIID);
else if (overlaps[0].dat.ovl.type == AS_OVS_TYPE_OVL)
- dumpPictureOVL(overlaps, novl, gkpStore, clearRegion, qryIID);
+ dumpPicture(overlaps, novl, gkpStore, clearRegion, qryIID);
}
diff --git a/src/AS_OVS/rewrite-obt-dump-with-names.pl b/src/AS_OVS/rewrite-obt-dump-with-names.pl
new file mode 100644
index 0000000..d3fae46
--- /dev/null
+++ b/src/AS_OVS/rewrite-obt-dump-with-names.pl
@@ -0,0 +1,24 @@
+#!/usr/bin/perl
+
+use strict;
+
+my @IIDtoNAME;
+
+open(F, "< test.gkpStore.fastqUIDmap") or die "failed to open test.gkpStore.fastqUIDmap\n";
+while (<F>) {
+ my ($uid, $iid, $name) = split '\s+', $_;
+
+ $IIDtoNAME[$iid] = $name;
+}
+close(F);
+
+open(F, "overlapStore -d5 -d3 -d test.obtStore |");
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ print "$IIDtoNAME[$v[0]]\t$IIDtoNAME[$v[1]]\t$v[2]\t$v[3]\t$v[4]\t$v[5]\t$v[6]\t$v[7]\n";
+}
+close(F);
diff --git a/src/AS_OVS/rewrite-overlap-dump-with-names.pl b/src/AS_OVS/rewrite-overlap-dump-with-names.pl
new file mode 100644
index 0000000..fbf0469
--- /dev/null
+++ b/src/AS_OVS/rewrite-overlap-dump-with-names.pl
@@ -0,0 +1,26 @@
+#!/usr/bin/perl
+
+use strict;
+
+my %IIDtoNAME;
+
+open(F, "< test.gkpStore.fastqUIDmap") or die "Failed to open 'test.gkpStore.fastqUIDmap'\n";
+while (<F>) {
+ my @v = split '\s+', $_;
+
+ $IIDtoNAME{$v[1]} = $v[2];
+}
+close(F);
+
+
+while (<STDIN>) {
+ $_ =~ s/^\s+//;
+ $_ =~ s/\s+$//;
+
+ my @v = split '\s+', $_;
+
+ die if (!exists($IIDtoNAME{$v[0]}));
+ die if (!exists($IIDtoNAME{$v[1]}));
+
+ print "$IIDtoNAME{$v[0]}\t$IIDtoNAME{$v[1]}\n";
+}
diff --git a/src/AS_PBR/._AS_PBR_correct.cc b/src/AS_PBR/._AS_PBR_correct.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_correct.cc differ
diff --git a/src/AS_PBR/._AS_PBR_correct.hh b/src/AS_PBR/._AS_PBR_correct.hh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_correct.hh differ
diff --git a/src/AS_PBR/._AS_PBR_dumpStore.cc b/src/AS_PBR/._AS_PBR_dumpStore.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_dumpStore.cc differ
diff --git a/src/AS_PBR/._AS_PBR_filter.cc b/src/AS_PBR/._AS_PBR_filter.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_filter.cc differ
diff --git a/src/AS_PBR/._AS_PBR_filter.hh b/src/AS_PBR/._AS_PBR_filter.hh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_filter.hh differ
diff --git a/src/AS_PBR/._AS_PBR_mates.cc b/src/AS_PBR/._AS_PBR_mates.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_mates.cc differ
diff --git a/src/AS_PBR/._AS_PBR_mates.hh b/src/AS_PBR/._AS_PBR_mates.hh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_mates.hh differ
diff --git a/src/AS_PBR/._AS_PBR_output.cc b/src/AS_PBR/._AS_PBR_output.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_output.cc differ
diff --git a/src/AS_PBR/._AS_PBR_output.hh b/src/AS_PBR/._AS_PBR_output.hh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_output.hh differ
diff --git a/src/AS_PBR/._AS_PBR_store.cc b/src/AS_PBR/._AS_PBR_store.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_store.cc differ
diff --git a/src/AS_PBR/._AS_PBR_store.hh b/src/AS_PBR/._AS_PBR_store.hh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_store.hh differ
diff --git a/src/AS_PBR/._AS_PBR_util.cc b/src/AS_PBR/._AS_PBR_util.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_util.cc differ
diff --git a/src/AS_PBR/._AS_PBR_util.hh b/src/AS_PBR/._AS_PBR_util.hh
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._AS_PBR_util.hh differ
diff --git a/src/AS_PBR/._ConvertSamToCA.cc b/src/AS_PBR/._ConvertSamToCA.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._ConvertSamToCA.cc differ
diff --git a/src/AS_PBR/._CorrectPacBio.cc b/src/AS_PBR/._CorrectPacBio.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._CorrectPacBio.cc differ
diff --git a/src/AS_PBR/._Makefile b/src/AS_PBR/._Makefile
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._Makefile differ
diff --git a/src/AS_PBR/._OutputLayout.cc b/src/AS_PBR/._OutputLayout.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._OutputLayout.cc differ
diff --git a/src/AS_PBR/._PBcR.pl b/src/AS_PBR/._PBcR.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._PBcR.pl differ
diff --git a/src/AS_PBR/._TrimFastqByQVWindow.cc b/src/AS_PBR/._TrimFastqByQVWindow.cc
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._TrimFastqByQVWindow.cc differ
diff --git a/src/AS_PBR/._addCNSToStore.pl b/src/AS_PBR/._addCNSToStore.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._addCNSToStore.pl differ
diff --git a/src/AS_PBR/._bamcat.c b/src/AS_PBR/._bamcat.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._bamcat.c differ
diff --git a/src/AS_PBR/._bamcat.h b/src/AS_PBR/._bamcat.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._bamcat.h differ
diff --git a/src/AS_PBR/._blasr.Linux-amd64.bin b/src/AS_PBR/._blasr.Linux-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._blasr.Linux-amd64.bin differ
diff --git a/src/AS_PBR/._boost b/src/AS_PBR/._boost
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/._boost differ
diff --git a/src/AS_PBR/._convertToPBCNS.pl b/src/AS_PBR/._convertToPBCNS.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._convertToPBCNS.pl differ
diff --git a/src/AS_PBR/._falcon_sense.Darwin-amd64.bin b/src/AS_PBR/._falcon_sense.Darwin-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._falcon_sense.Darwin-amd64.bin differ
diff --git a/src/AS_PBR/._falcon_sense.Linux-amd64.bin b/src/AS_PBR/._falcon_sense.Linux-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._falcon_sense.Linux-amd64.bin differ
diff --git a/src/AS_PBR/._jellyfish.Darwin-amd64.bin b/src/AS_PBR/._jellyfish.Darwin-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._jellyfish.Darwin-amd64.bin differ
diff --git a/src/AS_PBR/._jellyfish.Linux-amd64.bin b/src/AS_PBR/._jellyfish.Linux-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._jellyfish.Linux-amd64.bin differ
diff --git a/src/AS_PBR/._libgcc_s.1.dylib b/src/AS_PBR/._libgcc_s.1.dylib
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._libgcc_s.1.dylib differ
diff --git a/src/AS_PBR/._libstdc++.6.dylib b/src/AS_PBR/._libstdc++.6.dylib
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._libstdc++.6.dylib differ
diff --git a/src/AS_PBR/._mhap.tar.gz b/src/AS_PBR/._mhap.tar.gz
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._mhap.tar.gz differ
diff --git a/src/AS_PBR/._pbdagcon.Linux-amd64.bin b/src/AS_PBR/._pbdagcon.Linux-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._pbdagcon.Linux-amd64.bin differ
diff --git a/src/AS_PBR/._pbutgcns.Darwin-amd64.bin b/src/AS_PBR/._pbutgcns.Darwin-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._pbutgcns.Darwin-amd64.bin differ
diff --git a/src/AS_PBR/._pbutgcns.Linux-amd64.bin b/src/AS_PBR/._pbutgcns.Linux-amd64.bin
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/._pbutgcns.Linux-amd64.bin differ
diff --git a/src/AS_PBR/AS_PBR_correct.cc b/src/AS_PBR/AS_PBR_correct.cc
index 1405748..b380795 100755
--- a/src/AS_PBR/AS_PBR_correct.cc
+++ b/src/AS_PBR/AS_PBR_correct.cc
@@ -46,7 +46,7 @@ using namespace std;
#include <vector>
#include <algorithm>
-static const char *rcsid_AS_PBR_CORRECT_C = "$Id: AS_PBR_correct.cc 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_AS_PBR_CORRECT_C = "$Id: AS_PBR_correct.cc 4606 2015-02-17 22:27:42Z skoren $";
map<AS_IID, uint64> *globalFrgToScore;
@@ -176,6 +176,36 @@ void * correctFragments(void *ptr) {
LayRecord layout;
layout.iid = i;
+ // always output self match
+ if (waGlobal->allowLong == TRUE) {
+ SeqInterval aClr;
+ aClr.bgn = 0;
+ aClr.end = alen;
+
+ SeqInterval pos;
+ pos.bgn = aClr.bgn;
+ pos.end = aClr.end;
+
+ pos.bgn += TRIM_BACK_AMOUNT;
+ pos.end -= TRIM_BACK_AMOUNT;
+ aClr.bgn += TRIM_BACK_AMOUNT;
+ aClr.end -= TRIM_BACK_AMOUNT;
+
+ OVSoverlap olap;
+ olap.a_iid = olap.b_iid = i;
+ olap.dat.ovl.type = AS_OVS_TYPE_OVL;
+ olap.dat.ovl.flipped = 0;
+ olap.dat.ovl.corr_erate = olap.dat.ovl.orig_erate = AS_OVS_encodeQuality(0);
+ olap.dat.ovl.a_hang = olap.dat.ovl.b_hang = 0;
+
+ layout.bClrs.insert(pair<AS_IID, SeqInterval>(i, aClr));
+ layout.bOvls.insert(pair<AS_IID, OVSoverlap>(i, olap));
+ OverlapPos tileStr;
+ tileStr.position = pos;
+ tileStr.ident = i;
+ tile[i] = tileStr;
+ }
+
map<AS_IID, uint8> toSkip;
// read next batch of records from the overlap store, since this requires locking, we try to do it in batches
@@ -226,6 +256,10 @@ void * correctFragments(void *ptr) {
pos.bgn = pos.end = 0;
SeqInterval bClr;
bClr.bgn = bClr.end = 0;
+
+ if (waGlobal->allowLong != FALSE) {
+ allowCloseDovetail(olap);
+ }
convertOverlapToPosition(olap, pos, bClr, alen, blen);
// when we have an overlap that is coming from the long reads, trim it back
@@ -234,13 +268,17 @@ void * correctFragments(void *ptr) {
if (waGlobal->allowLong == FALSE) {
continue;
}
- int32 min = MIN(pos.bgn, pos.end);
- int32 max = MAX(pos.bgn, pos.end);
+ // skip self match, we always output it
+ if (bid == aid) { continue; }
+ int32 min = MAX(0,MIN(pos.bgn, pos.end));
+ int32 max = MIN(alen, MAX(pos.bgn, pos.end));
// bad overlap if it is too small to trim
if (min + TRIM_BACK_AMOUNT > alen || max < TRIM_BACK_AMOUNT || bClr.bgn + TRIM_BACK_AMOUNT > blen || bClr.end < TRIM_BACK_AMOUNT) {
+ if (waGlobal->verboseLevel >= VERBOSE_DEVELOPER) fprintf(stderr, "Fragment %d (%d) (%d-%d) versus %d (%d) (%d-%d) with error rate %f is short 1 , skipping it\n", aid, alen, min, max, bid, blen, bClr.bgn, bClr.end, AS_OVS_decodeQuality(olap.dat.ovl.corr_erate));
continue;
}
if (min + TRIM_BACK_AMOUNT < 0 || max - TRIM_BACK_AMOUNT > alen) {
+ if (waGlobal->verboseLevel >= VERBOSE_DEVELOPER) fprintf(stderr, "Fragment %d (%d) (%d-%d) versus %d (%d) (%d-%d) with error rate %f is short 2, skipping it\n", aid, alen, min, max, bid, blen, bClr.bgn, bClr.end, AS_OVS_decodeQuality(olap.dat.ovl.corr_erate));
continue;
}
if (pos.bgn < pos.end) {
@@ -263,13 +301,18 @@ void * correctFragments(void *ptr) {
// skip overlaps below our quality criteria
if (isOlapBad(olap, alen, blen, waGlobal->erate, waGlobal->elimit, waGlobal->maxErate)) {
+ if (waGlobal->verboseLevel >= VERBOSE_DEVELOPER) fprintf(stderr, "Fragment %d (%d) versus %d (%d) with error rate %f is bad, skipping it\n", aid, alen, bid, blen, AS_OVS_decodeQuality(olap.dat.ovl.corr_erate));
continue;
}
+ if (olapLength(olap, alen, blen) < waGlobal->minLength) {
+ continue;
+ }
- // non contained overlap, dont use these fragments for correction
+ // non contained overlap, dont use these fragments for correction, if close then ok
if (olap.dat.ovl.type == AS_OVS_TYPE_OVL && (olap.dat.ovl.a_hang < 0 || olap.dat.ovl.b_hang > 0)) {
// if there is a better non-contained overlap, then do not trust the contained overlap for this same fragment
if (frgToScore[bid] != 0) {
+if (waGlobal->verboseLevel >= VERBOSE_DEVELOPER) fprintf(stderr, "Fragment %d being kicked out of %d because it has a better non-contained overlap\n", bid, aid);
OVSoverlap best = frgToBest[bid];
uint32 min = MIN(tile[best.b_iid].position.bgn, tile[best.b_iid].position.end);
uint32 max = MAX(tile[best.b_iid].position.bgn, tile[best.b_iid].position.end);
diff --git a/src/AS_PBR/AS_PBR_dumpStore.cc b/src/AS_PBR/AS_PBR_dumpStore.cc
index e4940d6..709c484 100755
--- a/src/AS_PBR/AS_PBR_dumpStore.cc
+++ b/src/AS_PBR/AS_PBR_dumpStore.cc
@@ -37,7 +37,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-const char *mainid = "$Id: AS_PBR_dumpStore.cc 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: AS_PBR_dumpStore.cc 4537 2014-06-11 20:42:22Z skoren $";
using namespace std;
@@ -87,6 +87,10 @@ main(int argc, char **argv) {
for (vector<OverlapPos>::const_iterator iter = layout.mp.begin(); iter != layout.mp.end(); iter++) {
fprintf(stdout, "TLE\t"F_IID"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_S32"\t"F_S32"\n", iter->ident, iter->position.bgn, iter->position.end, layout.bClrs[iter->ident].bgn, layout.bClrs[iter->ident].end, layout.bOvls[iter->ident].dat.ovl.a_hang, layout.bOvls[iter->ident].dat.ovl.b_hang);
}
+
+ for (map<AS_IID, OVSoverlap>::const_iterator iter = layout.bOvls.begin(); iter != layout.bOvls.end(); iter++) {
+ fprintf(stderr, F_IID"\t"F_IID"\t%c\t"F_S32"\t"F_S32"\t%f\t%f\n", layout.iid, iter->first, (iter->second.dat.ovl.flipped ? 'I' : 'N'), iter->second.dat.ovl.a_hang, iter->second.dat.ovl.b_hang, AS_OVS_decodeQuality(iter->second.dat.ovl.orig_erate), AS_OVS_decodeQuality(iter->second.dat.ovl.corr_erate));
+ }
}
closeLayFile(store);
}
diff --git a/src/AS_PBR/AS_PBR_filter.cc b/src/AS_PBR/AS_PBR_filter.cc
index 339e82b..21ddf1f 100755
--- a/src/AS_PBR/AS_PBR_filter.cc
+++ b/src/AS_PBR/AS_PBR_filter.cc
@@ -47,7 +47,10 @@ using namespace std;
#include <set>
#include <map>
-static const char *rcsid_AS_PBR_FILTER_C = "$Id: AS_PBR_filter.cc 4471 2013-11-23 17:21:21Z skoren $";
+static const char *rcsid_AS_PBR_FILTER_C = "$Id: AS_PBR_filter.cc 4606 2015-02-17 22:27:42Z skoren $";
+
+static const int NUM_SD=2;
+static const bool FILTER_BY_LENGTH=false;
/**
* This function is responsible for computing the repeat threshold based on the number of PacBio sequences each short-read maps to
@@ -68,6 +71,7 @@ void computeRepeatThreshold(PBRThreadGlobals &thread_globals, AS_IID firstFrag)
for (map<AS_IID, uint8>::const_iterator iter = thread_globals.readsToPrint.begin(); iter != thread_globals.readsToPrint.end(); iter++) {
if (iter->second != 0) {
+ if (thread_globals.verboseLevel >= VERBOSE_DEVELOPER) fprintf(stderr, "Processing read %d with mappings %d\n", iter->first, iter->second);
uint32 val = iter->second;
if (val == MAX_COV) {
val = thread_globals.longReadsToPrint[iter->first];
@@ -99,6 +103,7 @@ void computeRepeatThreshold(PBRThreadGlobals &thread_globals, AS_IID firstFrag)
double IQD = 1.5 * (thirdQuartile-firstQuartile);
uint16_t minVal = IQD > firstQuartile ? 0 : (uint16_t)round(firstQuartile) - IQD;
uint16_t maxVal = (IQD + (uint16_t)round(thirdQuartile)) > MAX_COV_HIST ? MAX_COV_HIST : IQD + (uint16_t)round(thirdQuartile);
+ if (minVal == 0) { minVal++; }
fprintf(stderr, "Computed mean %f sd %f using min value %d (%f) max %d (%f)\n", mean, sd, minVal, firstQuartile, maxVal, thirdQuartile);
// now that we've got the quartile dist, run and get the updated mean just within the quartile
@@ -148,8 +153,11 @@ void computeRepeatThreshold(PBRThreadGlobals &thread_globals, AS_IID firstFrag)
fclose(histF);
}
delete[] covHist;
- thread_globals.covCutoff = MAX(thread_globals.covCutoff, MIN((int)(floor(mean*2)), (int)(floor(mean + 2*sd))));
- if (thread_globals.verboseLevel >= VERBOSE_OFF) fprintf(stderr, "Picking cutoff as %d mean would be %f +- %f (%d)\n", thread_globals.covCutoff, mean, sd, (int)(ceil(mean + 2*sd)));
+ thread_globals.covCutoff = MAX(thread_globals.covCutoff, MIN((int)(floor(mean*NUM_SD)), (int)(floor(mean + NUM_SD*sd))));
+ if (thread_globals.maxCoverage != 0 && thread_globals.covCutoff > thread_globals.maxCoverage) {
+ thread_globals.covCutoff = thread_globals.maxCoverage;
+ }
+ if (thread_globals.verboseLevel >= VERBOSE_OFF) fprintf(stderr, "Picking cutoff as %d mean would be %f +- %f (%d)\n", thread_globals.covCutoff, mean, sd, (int)(ceil(mean + NUM_SD*sd)));
}
}
@@ -227,6 +235,7 @@ void filterRepeatReads(PBRThreadGlobals &thread_globals, AS_IID firstFrag) {
// build a sorted by score map of all mapping a short-read sequence has
multimap<uint64, OverlapPos> scoreToReads;
map<AS_IID, uint64> readsToScore;
+ map<AS_IID, uint64> readsToIdy;
map<AS_IID, uint32> readsToBgn;
uint32 alen = thread_globals.frgToLen[iter->first];
@@ -239,6 +248,11 @@ void filterRepeatReads(PBRThreadGlobals &thread_globals, AS_IID firstFrag) {
if (isOlapBad(olaps[ovlPosition], alen, blen, thread_globals.erate, thread_globals.elimit, thread_globals.maxErate)) {
continue;
}
+
+ // allow close overlaps in
+ if (thread_globals.allowLong != FALSE) {
+ allowCloseDovetail(olaps[ovlPosition], true);
+ }
if (olaps[ovlPosition].dat.ovl.type == AS_OVS_TYPE_OVL && (olaps[ovlPosition].dat.ovl.a_hang > 0 || olaps[ovlPosition].dat.ovl.b_hang < 0)) {
continue;
}
@@ -251,13 +265,16 @@ void filterRepeatReads(PBRThreadGlobals &thread_globals, AS_IID firstFrag) {
// TODO: this currently assumes ties are non-randomly broken with greater positions winning which is not ideal
map<AS_IID, uint64>::iterator ovlIter = readsToScore.find(pos.ident);
if (ovlIter != readsToScore.end()) {
- if (currScore == ovlIter->second && readsToBgn[pos.ident] >= pos.position.bgn) {
- continue;
+ if (FILTER_BY_LENGTH && blen == ovlIter->second && readsToIdy[pos.ident] >= currScore) {
+ continue;
+ } else if (currScore == ovlIter->second && readsToBgn[pos.ident] >= pos.position.bgn) {
+ continue;
}
}
- readsToScore[pos.ident] = currScore;
+ readsToScore[pos.ident] = (FILTER_BY_LENGTH ? blen : currScore);
+ readsToIdy[pos.ident] = currScore;
readsToBgn[pos.ident] = pos.position.bgn;
- scoreToReads.insert(pair<uint64, OverlapPos>(currScore, pos));
+ scoreToReads.insert(pair<uint64, OverlapPos>((FILTER_BY_LENGTH ? blen : currScore), pos));
}
// now keep only the best cutoff of those and store them for easy access
diff --git a/src/AS_PBR/AS_PBR_output.cc b/src/AS_PBR/AS_PBR_output.cc
index 15dded1..cbb876f 100755
--- a/src/AS_PBR/AS_PBR_output.cc
+++ b/src/AS_PBR/AS_PBR_output.cc
@@ -51,7 +51,7 @@ using namespace std;
#include <vector>
#include <set>
-static const char *rcsid_AS_PBR_OUTPUT_C = "$Id: AS_PBR_output.cc 4484 2013-12-13 15:07:13Z skoren $";
+static const char *rcsid_AS_PBR_OUTPUT_C = "$Id: AS_PBR_output.cc 4606 2015-02-17 22:27:42Z skoren $";
static const uint32 FUDGE_BP = 5;
@@ -290,14 +290,20 @@ static void getCandidateOverlaps(PBRThreadGlobals *waGlobal, boost::dynamic_bits
/**
* Output a single AMOS layout record
*/
-static void closeRecord(PBRThreadGlobals *waGlobal, FILE *outFile, FILE *logFile, stringstream &layout, LayRecord &layRecord, uint32 lastEnd, int32 &offset, uint32 &readIID, uint32 &readSubID, bool outputAsLay) {
+static void closeRecord(PBRThreadGlobals *waGlobal, FILE *outFile, FILE *logFile, stringstream &layout, LayRecord &layRecord, uint32 lastEnd, int32 &offset, uint32 &readIID, uint32 &readSubID, OUTPUT_TYPE outputType) {
// close the layout and start a new one because we have a coverage gap
- if (outputAsLay) {
- if (lastEnd - offset >= waGlobal->minLength) {
+ if (outputType == OUT_FALCON) {
+ fprintf(outFile, "+ +\n");
+ }
+
+ if (lastEnd - offset >= waGlobal->minLength) {
+ if (outputType == OUT_AMOS) {
fprintf(outFile, "%s}\n", layout.str().c_str());
+ }
+ if ((!waGlobal->allowLong && layRecord.mp.size()) > 0 || (waGlobal->allowLong && layRecord.mp.size() > 1)) {
fprintf(logFile, "%d\t%s_%d_%d\t%d\t%d\t%d\n", layRecord.iid, waGlobal->libName, layRecord.iid, readSubID, readSubID, offset, lastEnd);
- }
+ }
}
readIID++;
readSubID++;
@@ -309,7 +315,7 @@ static void closeRecord(PBRThreadGlobals *waGlobal, FILE *outFile, FILE *logFile
/**
*output AMOS-style layout messaged based on our internal structures
*/
-void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
+void outputResults(PBRThreadGlobals *waGlobal, uint32 part, OUTPUT_TYPE outputType) {
assert(part > 0);
map<AS_IID, uint8> readsToPrint;
map<AS_IID, uint8> readsWithGaps;
@@ -326,18 +332,21 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
}
char outputName[FILENAME_MAX] = {0};
- sprintf(outputName, "%s.%d.lay", waGlobal->prefix, part);
- errno = 0;
- FILE *outFile = fopen(outputName, "w");
- if (errno) {
- fprintf(stderr, "Couldn't open '%s' for write: %s\n", outputName, strerror(errno)); exit(1);
+ FILE *outFile = stdout;
+ if (outputType != OUT_FALCON && outputType != OUT_PBDAGCON && strncmp("-", waGlobal->prefix, 1) != 0) {
+ sprintf(outputName, "%s.%d.lay", waGlobal->prefix, part);
+ errno = 0;
+ outFile = fopen(outputName, "w");
+ if (errno) {
+ fprintf(stderr, "Couldn't open '%s' for write: %s\n", outputName, strerror(errno)); exit(1);
+ }
}
char inName[FILENAME_MAX] = {0};
if (waGlobal->hasMates) {
- sprintf(inName, "%s.%d.paired.olaps", waGlobal->prefix, part);
+ sprintf(inName, "%s.%d.paired.olaps", waGlobal->inPrefix, part);
} else {
- sprintf(inName, "%s.%d.olaps", waGlobal->prefix, part);
+ sprintf(inName, "%s.%d.olaps", waGlobal->inPrefix, part);
}
errno = 0;
LayRecordStore *inFile = openLayFile(inName);
@@ -350,10 +359,10 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
ShortMapStore *inStore = NULL;
if (waGlobal->maxUncorrectedGap > 0) {
- sprintf(inRankName, "%s.%d", waGlobal->prefix, part);
+ sprintf(inRankName, "%s.%d", waGlobal->inPrefix, part);
inStore = new ShortMapStore(inRankName, false, false, true);
}
- sprintf(inRankName, "%s.%d.rank", waGlobal->prefix, part);
+ sprintf(inRankName, "%s.%d.rank", waGlobal->inPrefix, part);
errno = 0;
FILE *inRankFile = fopen(inRankName, "r");
if (errno) {
@@ -367,7 +376,6 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
readRanking[illumina].insert(corrected);
}
fclose(inRankFile);
- AS_UTL_unlink(inRankName);
uint32 readIID = 0;
uint32 readLen = 0;
@@ -383,8 +391,11 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
LayRecord layRecord;
while (readLayRecord(inFile, layRecord)) {
uint32 readSubID = 1;
- if (!outputAsLAY) {
+ if (outputType != OUT_AMOS) {
readLen = loadOneSequence(waGlobal->gkp, layRecord.iid, seq);
+ if (outputType == OUT_FALCON) {
+ fprintf(outFile, "%s_%d_%d %.*s\n", waGlobal->libName, layRecord.iid, readSubID, readLen, seq);
+ }
}
stringstream layout (stringstream::in | stringstream::out);
@@ -395,8 +406,8 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
// process record
for (vector<OverlapPos>::const_iterator iter = layRecord.mp.begin(); iter != layRecord.mp.end(); iter++) {
// skip reads over coverage
- if (waGlobal->globalRepeats == TRUE && (readRanking[iter->ident].find(layRecord.iid) == readRanking[iter->ident].end())) {
- //fprintf(stderr, "Skipping read %d to correct %d it was at cutoff %d true %d\n", iter->ident, i, waGlobal->readRanking[iter->ident][i].first, waGlobal->readRanking[iter->ident][i].second);
+ if (waGlobal->globalRepeats == TRUE && (iter->ident != layRecord.iid && readRanking[iter->ident].find(layRecord.iid) == readRanking[iter->ident].end())) {
+ //fprintf(stderr, "Skipping read %d to correct %d\n",iter->ident, layRecord.iid);
continue;
}
// if the last fragment ended before the current one starts, we have a gap
@@ -409,8 +420,12 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
uint32 gapStart = lastEnd;
uint32 gapEnd = MIN(iter->position.bgn, iter->position.end);
if (bits.size() != 0 && bits.test(inStore->getStoreIID(layRecord.iid)) == true) {
- gapStart = matchingSequencePositions[layRecord.iid].bgn;
- gapEnd = matchingSequencePositions[layRecord.iid].end;
+ if (matchingSequencePositions[layRecord.iid].bgn < 0 || matchingSequencePositions[layRecord.iid].end < 0) {
+ bits.clear();
+ } else {
+ gapStart = MIN(matchingSequencePositions[layRecord.iid].bgn, matchingSequencePositions[layRecord.iid].end);
+ gapEnd = MAX(matchingSequencePositions[layRecord.iid].bgn, matchingSequencePositions[layRecord.iid].end);
+ }
}
assert(gapEnd >= gapStart);
@@ -421,6 +436,8 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
AS_IID iid = inStore->getMappedIID(i);
if (iid != layRecord.iid) {
+ if (matchingSequencePositions[iid].bgn < 0) { matchingSequencePositions[iid].bgn = 0; }
+ if (matchingSequencePositions[iid].end < 0) { matchingSequencePositions[iid].end = 0; }
uint32 min = MIN(matchingSequencePositions[iid].bgn,matchingSequencePositions[iid].end);
uint32 max = MAX(matchingSequencePositions[iid].bgn,matchingSequencePositions[iid].end);
uint32 gapSize = (int32)(MIN(iter->position.bgn, iter->position.end) - gapStart + 1);
@@ -436,7 +453,7 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
max = (matchingSequencePositions[iid].end+(MIN_DIST_TO_RECRUIT) < waGlobal->frgToLen[iid] ? matchingSequencePositions[iid].end+(MIN_DIST_TO_RECRUIT) : waGlobal->frgToLen[iid]);
} else {
if (matchingSequencePositions[iid].bgn+MIN_DIST_TO_RECRUIT >= waGlobal->frgToLen[iid]) {
- gapPatch = MIN_DIST_TO_RECRUIT - (waGlobal->frgToLen[iid] - matchingSequencePositions[iid].bgn);
+ gapPatch = MIN(waGlobal->frgToLen[iid], MIN_DIST_TO_RECRUIT - (waGlobal->frgToLen[iid] - matchingSequencePositions[iid].bgn));
}
min = (matchingSequencePositions[iid].bgn+(MIN_DIST_TO_RECRUIT) < waGlobal->frgToLen[iid] ? matchingSequencePositions[iid].bgn+(MIN_DIST_TO_RECRUIT) : waGlobal->frgToLen[iid]);
max = (matchingSequencePositions[iid].end >= (MIN_DIST_TO_RECRUIT) ? matchingSequencePositions[iid].end - (MIN_DIST_TO_RECRUIT) : 0);
@@ -444,6 +461,9 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "Found read %d (will be called %d) sequence %d - %d (min: %d, max: %d) that could help gap %d to %d in %d (gap diff is %d)\n", iid, gapIID, matchingSequencePositions[iid].bgn, matchingSequencePositions[iid].end, min, max, gapStart, MIN(iter->position.bgn, iter->position.end), layRecord.iid, diff);
+ if (gapPatch < MIN_DIST_TO_RECRUIT) {
+ continue;
+ }
uint32 MIN_FRM_START = 50;
// ran off the end of the patchee
if (gapStart + gapPatch - tmpoff < MIN_DIST_TO_RECRUIT) {
@@ -454,7 +474,8 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
}
}
uint32 gapOffset = (gapStart+gapPatch-tmpoff >= (MIN_DIST_TO_RECRUIT) ? gapStart + gapPatch - (MIN_DIST_TO_RECRUIT) - tmpoff: MIN_FRM_START);
- if (outputAsLAY) {
+ if (gapStart + gapPatch < tmpoff) { gapOffset = MIN_FRM_START; }
+ if (outputType == OUT_AMOS) {
layout << "{TLE\nclr:"
<< 0 /*min*/
<< ","
@@ -469,14 +490,22 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
uint32 qlen = loadOneSequence(waGlobal->gkp, iid, qseq);
uint32 gaplen = (MAX(max, min) - MIN(max, min));
if (min < max) {
- fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iid, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, gapOffset, gapOffset + (max - min));
- fprintf(outFile, " %.*s %.*s\n", max - min, qseq+min, gaplen, seq+gapOffset);
+ if (outputType == OUT_PBDAGCON) {
+ fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iid, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, gapOffset, gapOffset + (max - min));
+ fprintf(outFile, " %.*s %.*s\n", max - min, qseq+min, gaplen, seq+gapOffset);
+ } else {
+ fprintf(outFile, "%d %.*s\n", iid, max-min, qseq+min);
+ }
} else {
memcpy(rseq, qseq, qlen);
rseq[qlen] = 0;
reverseComplementSequence(rseq, 0);
- fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iid, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, gapOffset, gapOffset + (min - max));
- fprintf(outFile, " %.*s %.*s\n", min - max, rseq+min, gaplen, seq + gapOffset);
+ if (outputType == OUT_PBDAGCON) {
+ fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iid, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, gapOffset, gapOffset + (min - max));
+ fprintf(outFile, " %.*s %.*s\n", min - max, rseq+max, gaplen, seq + gapOffset);
+ } else {
+ fprintf(outFile, "%d %.*s\n", iid, min - max, rseq+max);
+ }
}
}
count++;
@@ -489,11 +518,11 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
if (offset < 0) {
offset = 0;
}
- uint32 overlappingStart = (lastEnd-offset >= (MIN_DIST_TO_RECRUIT / 3) ? lastEnd - (MIN_DIST_TO_RECRUIT / 3) - offset: 0);
+ uint32 overlappingStart = (lastEnd >= ((MIN_DIST_TO_RECRUIT / 3) + offset) ? lastEnd - (MIN_DIST_TO_RECRUIT / 3) - offset: 0);
uint32 overlappingEnd = MIN(iter->position.bgn, iter->position.end) + (MIN_DIST_TO_RECRUIT / 3) - offset;
// record this gap
if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d with %d supporters had a gap from %d to %d inserting range %d from %d %d with offset %d so in original read positions are %d %d\n", layRecord.iid, count, lastEnd, MIN(iter->position.bgn, iter->position.end),gapIID, overlappingStart, overlappingEnd, offset, overlappingStart+offset, overlappingEnd+offset);
- if (outputAsLAY) {
+ if (outputType == OUT_AMOS) {
layout << "{TLE\nclr:"
<< 0
<< ","
@@ -504,71 +533,72 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
readsWithGaps[layRecord.iid] = 1;
pair<AS_IID, pair<uint32, uint32> > gapInfo(gapIID++, pair<uint32, uint32>(overlappingStart+offset, MIN(waGlobal->frgToLen[layRecord.iid], overlappingEnd+offset)));
gaps[layRecord.iid].push_back(gapInfo);
- } else {
+ } else if (outputType == OUT_PBDAGCON) {
fprintf(outFile, "%d %s_%d_%d %s %d %d %d", gapIID++, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, overlappingStart, overlappingEnd);
fprintf(outFile, " %.*s %.*s\n", overlappingEnd - overlappingStart, seq + overlappingStart, overlappingEnd - overlappingStart, seq + overlappingStart);
- }
+ } else if (outputType == OUT_FALCON) {
+ fprintf(outFile, "%d %.*s\n", gapIID++, overlappingEnd - overlappingStart, seq + overlappingStart);
+ }
} else {
- if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d had a gap from %d to %d but no one had a matching gap size it so breaking it\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end));
- closeRecord(waGlobal, outFile, reportFile, layout, layRecord, lastEnd, offset, readIID, readSubID, outputAsLAY);
+ if (waGlobal->allowLong && outputType != OUT_AMOS) {
+ // don't break, rely on the consensus to do it
+ } else {
+ if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d had a gap from %d to %d but no one had a matching gap size it so breaking it\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end));
+ closeRecord(waGlobal, outFile, reportFile, layout, layRecord, lastEnd, offset, readIID, readSubID, outputType);
+ }
}
} else { // no one could agree with this read, break it
- /*
- if (MIN(iter->position.bgn, iter->position.end) - lastEnd < 2*FUDGE_BP) {
- if (offset < 0) {
- offset = 0;
- }
- uint32 overlappingStart = (lastEnd-offset >= (MIN_DIST_TO_RECRUIT / 3) ? lastEnd - (MIN_DIST_TO_RECRUIT / 3) - offset: 0);
- uint32 overlappingEnd = MIN(iter->position.bgn, iter->position.end) + (MIN_DIST_TO_RECRUIT / 3) - offset;
- // record this gap
- if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d with 0 supporters had a gap from %d to %d inserting range %d from %d %d with offset %d so in original read positions are %d %d\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end),gapIID, overlappingStart, overlappingEnd, offset, overlappingStart+offset, overlappingEnd+offset);
- layout << "{TLE\nclr:"
- << 0
- << ","
- << overlappingEnd - overlappingStart
- << "\noff:" << overlappingStart
- << "\nsrc:" << gapIID
- << "\n}\n";
- readsWithGaps[layRecord.iid] = 1;
- pair<AS_IID, pair<uint32, uint32> > gapInfo(gapIID++, pair<uint32, uint32>(overlappingStart+offset, MIN(waGlobal->frgToLen[layRecord.iid], overlappingEnd+offset)));
- gaps[layRecord.iid].push_back(gapInfo);
- if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d had a gap from %d to %d, no one believed it but it was very small so we allowed.\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end));
+ if (waGlobal->allowLong && outputType != OUT_AMOS) {
+ // don't break, rely on the consensus to do it
} else {
- */
- if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d had a gap from %d to %d but no one believed it so breaking it\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end));
- closeRecord(waGlobal, outFile, reportFile, layout, layRecord, lastEnd, offset, readIID, readSubID, outputAsLAY);
- //}
+ if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "For fragment %d had a gap from %d to %d but no one believed it so breaking it\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end));
+ closeRecord(waGlobal, outFile, reportFile, layout, layRecord, lastEnd, offset, readIID, readSubID, outputType);
+ }
}
} else {
- // close the layout and start a new one because we have a coverage gap
- if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "Gap in sequence %d between positions %d - %d is greater than allowed %d\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end), waGlobal->maxUncorrectedGap);
- closeRecord(waGlobal, outFile, reportFile, layout, layRecord, lastEnd, offset, readIID, readSubID, outputAsLAY);
+ if (waGlobal->allowLong && outputType != OUT_AMOS) {
+ // don't break, rely on the consensus to do it
+ } else {
+ // close the layout and start a new one because we have a coverage gap
+ if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "Gap in sequence %d between positions %d - %d is greater than allowed %d\n", layRecord.iid, lastEnd, MIN(iter->position.bgn, iter->position.end), waGlobal->maxUncorrectedGap);
+ closeRecord(waGlobal, outFile, reportFile, layout, layRecord, lastEnd, offset, readIID, readSubID, outputType);
+ }
}
}
+ SeqInterval bClr = layRecord.bClrs[iter->ident];
+ if (bClr.bgn >= bClr.end) { continue; }
if (offset < 0) {
offset = MIN(iter->position.bgn, iter->position.end);
//fprintf(stderr, "Updating offset in layout "F_IID" to be "F_U32"\n", i, offset);
}
- SeqInterval bClr = layRecord.bClrs[iter->ident];
uint32 min = MIN(iter->position.bgn, iter->position.end);
uint32 max = MAX(iter->position.bgn, iter->position.end);
uint32 length = max - min;
//fprintf(stderr, "Writing layout for frg "F_IID" "F_IID" "F_U32" "F_U32" "F_U32"\n", i, iter->ident, iter->position.bgn, iter->position.end, offset);
- if (!outputAsLAY) {
+ if (outputType != OUT_AMOS) {
loadOneSequence(waGlobal->gkp, iter->ident, qseq);
uint32 qlen = bClr.end - bClr.bgn;
if (iter->position.bgn < iter->position.end) {
- fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iter->ident, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, iter->position.bgn, iter->position.bgn + qlen);
- fprintf(outFile, " %.*s %.*s\n", bClr.end - bClr.bgn, qseq+bClr.bgn, length, seq+iter->position.bgn);
+ if (outputType == OUT_PBDAGCON) {
+ fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iter->ident, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, iter->position.bgn, iter->position.bgn + qlen);
+ fprintf(outFile, " %.*s %.*s\n", bClr.end - bClr.bgn, qseq+bClr.bgn, length, seq+iter->position.bgn);
+ } else {
+if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "Output match for %d %d of length %d. It matches %d-%d and %d-%d\n", layRecord.iid, iter->ident, length, iter->position.bgn, iter->position.end, bClr.bgn, bClr.end);
+ fprintf(outFile, "%d %.*s\n", iter->ident, bClr.end - bClr.bgn, qseq+bClr.bgn);
+ }
} else {
memcpy(rseq, qseq, qlen);
rseq[qlen] = 0;
reverseComplementSequence(rseq, 0);
- fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iter->ident, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, iter->position.end, iter->position.end + qlen);
- fprintf(outFile, " %.*s %.*s\n", bClr.end - bClr.bgn, rseq+bClr.bgn, length, seq + iter->position.end);
- }
+ if (outputType == OUT_PBDAGCON) {
+ fprintf(outFile, "%d %s_%d_%d %s %d %d %d", iter->ident, waGlobal->libName, layRecord.iid, readSubID, "+", readLen, iter->position.end, iter->position.end + qlen);
+ fprintf(outFile, " %.*s %.*s\n", bClr.end - bClr.bgn, rseq+bClr.bgn, length, seq + iter->position.end);
+ } else {
+ fprintf(outFile, "%d %.*s\n", iter->ident, bClr.end - bClr.bgn, rseq+bClr.bgn);
+ }
+ }
} else {
layout << "{TLE\nclr:"
<< (iter->position.bgn < iter->position.end ? bClr.bgn : bClr.end)
@@ -579,21 +609,26 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
<< "\n}\n";
readsToPrint[iter->ident]=1;
}
- if (lastEnd < (min + length - 1)) {
+ if (layRecord.iid != iter->ident && lastEnd < (min + length - 1)) {
lastEnd = min + length - 1;
}
}
if (lastEnd - offset >= waGlobal->minLength) {
- if (outputAsLAY) {
+ if (outputType == OUT_AMOS) {
fprintf(outFile, "%s}\n", layout.str().c_str());
- }
- fprintf(reportFile, "%d\t%s_%d_%d\t%d\t%d\t%d\n", layRecord.iid, waGlobal->libName, layRecord.iid, readSubID, readSubID, offset, lastEnd);
+ }
+ if ((!waGlobal->allowLong && layRecord.mp.size()) > 0 || (waGlobal->allowLong && layRecord.mp.size() > 1)) {
+ fprintf(reportFile, "%d\t%s_%d_%d\t%d\t%d\t%d\n", layRecord.iid, waGlobal->libName, layRecord.iid, readSubID, readSubID, offset, lastEnd);
+ }
+ }
+ if (outputType == OUT_FALCON) {
+ fprintf(outFile, "+ +\n");
}
readIID++;
if (waGlobal->verboseLevel >= VERBOSE_DEBUG) fprintf(stderr, "Finished processing read %d subsegments %d\n", layRecord.iid, readSubID);
}
- if (outputAsLAY) {
+ if (outputType == OUT_AMOS) {
if (waGlobal->verboseLevel >= VERBOSE_OFF) fprintf(stderr, "Beginning output of "F_SIZE_T" reads\n", readsToPrint.size());
loadSequence(waGlobal->gkp, readsToPrint, frgToEnc);
map<AS_IID, char*> *theFrgs = &frgToEnc;
@@ -630,15 +665,10 @@ void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLAY) {
delete[] iter->second;
}
frgToEnc.clear();
+ if (outputType == OUT_FALCON) { fprintf(outFile, "- -\n"); }
fclose(outFile);
fclose(reportFile);
closeLayFile(inFile);
-
- if (waGlobal->maxUncorrectedGap > 0) {
- inStore->unlink();
- delete inStore;
- }
- AS_UTL_unlink(inName);
- fprintf(stderr, "Done output\n");
+ fprintf(stderr, "Done output\n");
}
diff --git a/src/AS_PBR/AS_PBR_output.hh b/src/AS_PBR/AS_PBR_output.hh
index c5c88ba..0ce96f8 100755
--- a/src/AS_PBR/AS_PBR_output.hh
+++ b/src/AS_PBR/AS_PBR_output.hh
@@ -42,8 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "AS_PBR_util.hh"
-static const char *rcsid_AS_PBR_OUTPUT_H = "$Id: AS_PBR_output.hh 4484 2013-12-13 15:07:13Z skoren $";
+static const char *rcsid_AS_PBR_OUTPUT_H = "$Id: AS_PBR_output.hh 4537 2014-06-11 20:42:22Z skoren $";
-extern void outputResults(PBRThreadGlobals *waGlobal, uint32 part, bool outputAsLay);
+enum OUTPUT_TYPE { OUT_AMOS, OUT_PBDAGCON, OUT_FALCON };
+extern void outputResults(PBRThreadGlobals *waGlobal, uint32 part, OUTPUT_TYPE outputType);
#endif
diff --git a/src/AS_PBR/AS_PBR_util.cc b/src/AS_PBR/AS_PBR_util.cc
index 521d653..d949053 100755
--- a/src/AS_PBR/AS_PBR_util.cc
+++ b/src/AS_PBR/AS_PBR_util.cc
@@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
using namespace std;
-static const char *rcsid_AS_PBR_UTIL_C = "$Id: AS_PBR_util.cc 4471 2013-11-23 17:21:21Z skoren $";
+static const char *rcsid_AS_PBR_UTIL_C = "$Id: AS_PBR_util.cc 4606 2015-02-17 22:27:42Z skoren $";
#include "AS_PBR_util.hh"
@@ -107,20 +107,23 @@ void convertOverlapToPosition(const OVSoverlap& olap, SeqInterval &pos, SeqInter
}
}
} else if (olap.dat.ovl.type == AS_OVS_TYPE_OBT) {
+ uint32 bend = olap.dat.obt.b_end_hi << 9 | olap.dat.obt.b_end_lo;
if (forB) {
- fprintf(stderr, "Error, OBT is not currently supported in this mode\n");
- exit(1);
+ pos.bgn = olap.dat.obt.b_beg;
+ pos.end = bend;
+ if (!olap.dat.obt.fwd) {
+ pos.bgn = bend;
+ pos.end = olap.dat.obt.b_beg;
+ }
+ bClr.bgn = olap.dat.obt.a_beg;
+ bClr.end = olap.dat.obt.a_end;
+ } else {
+ pos.bgn = olap.dat.obt.a_beg;
+ pos.end = olap.dat.obt.a_end;
+
+ bClr.bgn = MIN(olap.dat.obt.b_beg, bend);
+ bClr.end = MAX(olap.dat.obt.b_beg, bend);
}
- pos.bgn = olap.dat.obt.a_beg;
- pos.end = olap.dat.obt.a_end;
-
- if (!olap.dat.obt.fwd) {
- pos.bgn = olap.dat.obt.a_end;
- pos.end = olap.dat.obt.a_beg;
- }
- uint32 bend = olap.dat.obt.b_end_hi << 9 | olap.dat.obt.b_end_lo;
- bClr.bgn = MIN(olap.dat.obt.b_beg, bend);
- bClr.end = MAX(olap.dat.obt.b_beg, bend);
}
// update end points if necessary
diff --git a/src/AS_PBR/AS_PBR_util.hh b/src/AS_PBR/AS_PBR_util.hh
index c62bf86..297d343 100755
--- a/src/AS_PBR/AS_PBR_util.hh
+++ b/src/AS_PBR/AS_PBR_util.hh
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef AS_PBR_UTIL_H
#define AS_PBR_UTIL_H
-static const char *rcsid_AS_PBR_UTIL_H = "$Id: AS_PBR_util.hh 4471 2013-11-23 17:21:21Z skoren $";
+static const char *rcsid_AS_PBR_UTIL_H = "$Id: AS_PBR_util.hh 4606 2015-02-17 22:27:42Z skoren $";
#include "AS_global.H"
#include "AS_OVS_overlapStore.H"
@@ -71,6 +71,7 @@ const uint32 CHIMERA_MAX_SIZE = 150;
const uint32 MIN_DIST_TO_RECRUIT = 500;
const double ERATE_ADJUST = 1.5;
const uint8 TRIM_BACK_AMOUNT = 50;
+const uint8 DOVETAIL_AMOUNT = 200;
const uint8 MIN_NUM_SAMPLES = 100;
const uint8 VERBOSE_OFF = 0;
@@ -115,6 +116,7 @@ struct PBRThreadGlobals {
double repeatMultiplier;
int minLength;
char prefix[FILENAME_MAX];
+ char inPrefix[FILENAME_MAX];
char exePath[FILENAME_MAX];
double percentToEstimateInserts;
uint8 verboseLevel;
@@ -122,6 +124,7 @@ struct PBRThreadGlobals {
// read-only variables for thread
uint16 covCutoff;
+ uint16 maxCoverage;
uint16 maxUncorrectedGap;
// read-only fragment information
@@ -190,7 +193,7 @@ static uint32 olapLengthOVL(OVSoverlap ovl, uint32 alen, uint32 blen) {
static uint32 olapLengthOBT(OVSoverlap ovl, uint32 alen, uint32 blen) {
uint32 aovl = ovl.dat.obt.a_end - ovl.dat.obt.a_beg;
- uint32 bend = ovl.dat.obt.b_end_hi >> 9 | ovl.dat.obt.b_end_lo;
+ uint32 bend = ovl.dat.obt.b_end_hi << 9 | ovl.dat.obt.b_end_lo;
uint32 bbgn = MIN(ovl.dat.obt.b_beg, bend);
bend = MAX(ovl.dat.obt.b_beg, bend);
@@ -356,5 +359,27 @@ static bool rangesOverlap(const SeqInterval &first, const SeqInterval &second) {
return (end-start+1) > 0;
}
+static void allowCloseDovetail(OVSoverlap & olap, bool forB = false) {
+ if (olap.dat.ovl.type != AS_OVS_TYPE_OVL) { // do nothing
+ return;
+ }
+
+ if (forB) {
+ if (olap.dat.ovl.a_hang > 0 && olap.dat.ovl.a_hang - DOVETAIL_AMOUNT <= 0) {
+ olap.dat.ovl.a_hang = 0;
+ }
+ if (olap.dat.ovl.b_hang < 0 && olap.dat.ovl.b_hang + DOVETAIL_AMOUNT >= 0) {
+ olap.dat.ovl.b_hang = 0;
+ }
+ } else {
+ if (olap.dat.ovl.a_hang < 0 && olap.dat.ovl.a_hang + DOVETAIL_AMOUNT >= 0) {
+ olap.dat.ovl.a_hang = 0;
+ }
+ if (olap.dat.ovl.b_hang > 0 && olap.dat.ovl.b_hang - DOVETAIL_AMOUNT <= 0) {
+ olap.dat.ovl.b_hang = 0;
+ }
+ }
+}
+
extern void convertOverlapToPosition(const OVSoverlap& olap, SeqInterval &pos, SeqInterval &bClr, uint32 alen, uint32 blen, bool forB = false);
#endif
diff --git a/src/AS_PBR/ConvertSamToCA.cc b/src/AS_PBR/ConvertSamToCA.cc
index ecd33e7..cb317ff 100755
--- a/src/AS_PBR/ConvertSamToCA.cc
+++ b/src/AS_PBR/ConvertSamToCA.cc
@@ -5,7 +5,7 @@
#include <fstream>
#include <sstream>
-#include "sam.h"
+#include "bamcat.h"
using namespace std;
/**
diff --git a/src/AS_PBR/CorrectPacBio.cc b/src/AS_PBR/CorrectPacBio.cc
index 182a351..fe8ee6f 100755
--- a/src/AS_PBR/CorrectPacBio.cc
+++ b/src/AS_PBR/CorrectPacBio.cc
@@ -37,7 +37,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-const char *mainid = "$Id: CorrectPacBio.cc 4478 2013-12-02 17:42:22Z skoren $";
+const char *mainid = "$Id: CorrectPacBio.cc 4614 2015-02-19 21:25:35Z skoren $";
#include "AS_global.H"
#include "AS_MSG_pmesg.H"
@@ -143,6 +143,8 @@ main (int argc, char * argv []) {
// initialize default parameters
thread_globals.numThreads = 2;
+ thread_globals.minLength = 0;
+ thread_globals.maxCoverage = 0;
thread_globals.maxErate = 0.25;
thread_globals.erate = 0.15;
thread_globals.elimit = 4.5;
@@ -185,6 +187,9 @@ main (int argc, char * argv []) {
} else if (strcmp(argv[arg], "-L") == 0) {
thread_globals.allowLong = TRUE;
+ } else if (strcmp(argv[arg], "-l") == 0) {
+ thread_globals.minLength = atoi(argv[++arg]);
+
} else if (strcmp(argv[arg], "-e") == 0) {
thread_globals.erate = atof(argv[++arg]);
@@ -209,6 +214,12 @@ main (int argc, char * argv []) {
else { thread_globals.covCutoff = cutoff; }
fprintf(stderr, "The cutoff is set to be %d\n", thread_globals.covCutoff);
+ } else if (strcmp(argv[arg], "-CM") == 0){
+ int32 cutoff = atoi(argv[++arg]);
+ if (cutoff <= 0) { thread_globals.maxCoverage = 0; }
+ else if (cutoff >= MAX_COV_HIST) {thread_globals.maxCoverage = MAX_COV_HIST - 1; }
+ else { thread_globals.maxCoverage = cutoff; }
+
} else if (strcmp(argv[arg], "-M") == 0) {
int32 maxGap = atoi(argv[++arg]);
if (maxGap <= 0) { thread_globals.maxUncorrectedGap = 0; }
diff --git a/src/AS_PBR/Makefile b/src/AS_PBR/Makefile
index 2d8ecc2..2804463 100755
--- a/src/AS_PBR/Makefile
+++ b/src/AS_PBR/Makefile
@@ -59,17 +59,9 @@ ifeq ($(MACHINETYPE), Power Macintosh)
MACHINETYPE = ppc
endif
-ifneq "$(origin SAMTOOLS)" "environment"
- ifeq ($(shell ls -d $(LOCAL_WORK)/samtools 2> /dev/null), $(LOCAL_WORK)/samtools)
- SAMTOOLS = $(LOCAL_WORK)/samtools/
- endif
-endif
+LOCAL_NAME=$(OSTYPE)-$(MACHINETYPE)
-PBR_SOURCES = OutputLayout.cc TrimFastqByQVWindow.cc CorrectPacBio.cc AS_PBR_dumpStore.cc
-ifdef SAMTOOLS
- PBR_SOURCES += ConvertSamToCA.cc
- INC_IMPORT_DIRS += $(SAMTOOLS)
-endif
+PBR_SOURCES = OutputLayout.cc TrimFastqByQVWindow.cc CorrectPacBio.cc AS_PBR_dumpStore.cc ConvertSamToCA.cc bamcat.c
PBR_OBJECTS = $(PBR_SOURCES:.cc=.o)
@@ -87,14 +79,41 @@ OBJECTS = $(PBR_OBJECTS) $(PBR_LIB_OBJECTS)
LIBRARIES = libAS_PBR.a libCA.a
LIBS = libCA.a
-SAM_LIBS = $(LIBS) libbam.a
-CXX_PROGS = outputLayout trimFastqByQVWindow correctPacBio dumpPBRLayoutStore
-ifdef SAMTOOLS
- CXX_PROGS += convertSamToCA
+CXX_PROGS = outputLayout trimFastqByQVWindow correctPacBio dumpPBRLayoutStore convertSamToCA
+
+SCRIPTS = PBcR convertToPBCNS addCNSToStore mhap
+
+ifneq ($(wildcard falcon_sense.$(LOCAL_NAME).bin),)
+ SCRIPTS += falcon_sense
+else
+ SCRIPTS += falcon_sense_missing
+endif
+
+ifneq ($(wildcard blasr.$(LOCAL_NAME).bin),)
+ SCRIPTS += blasr
+else
+ SCRIPTS += blasr_missing
+endif
+
+ifneq ($(wildcard pbutgcns.$(LOCAL_NAME).bin),)
+ SCRIPTS += pbutgcns
+else
+ SCRIPTS += pbutgcns_missing
+endif
+
+ifneq ($(wildcard pbdagcon.$(LOCAL_NAME).bin),)
+ SCRIPTS += pbdagcon
+else
+ SCRIPTS += pbdagcon_missing
+endif
+
+ifneq ($(wildcard jellyfish.$(LOCAL_NAME).bin),)
+ SCRIPTS += jellyfish
+else
+ SCRIPTS += jellyfish_missing
endif
-SCRIPTS = pacBioToCA convertToPBCNS
# Include for AS project rules
include $(LOCAL_WORK)/src/c_make.as
@@ -112,6 +131,7 @@ ifeq ($(OSTYPE), OSF1)
endif
ifeq ($(OSTYPE), FreeBSD)
+ LDFLAGS += -lz
endif
ifeq ($(OSTYPE), Darwin)
@@ -126,25 +146,80 @@ libAS_PBR.a: $(PBR_LIB_OBJECTS)
libCA.a: $(PBR_LIB_OBJECTS)
-libbam.a:
- $(MAKE) -C $(SAMTOOLS)/
- @cp $(SAMTOOLS)/$@ $(LOCAL_LIB)/$@
-
-
outputLayout: OutputLayout.o $(LIBS)
trimFastqByQVWindow: TrimFastqByQVWindow.o $(LIBS)
-convertSamToCA: ConvertSamToCA.o $(SAM_LIBS)
+convertSamToCA: ConvertSamToCA.o bamcat.o $(LIBS)
correctPacBio: CorrectPacBio.o $(LIBS)
dumpPBRLayoutStore: AS_PBR_dumpStore.o $(LIBS)
-pacBioToCA: pacBioToCA.pl
+PBcR: PBcR.pl
@cp $< $(LOCAL_BIN)/$@
@chmod 775 $(LOCAL_BIN)/$@
+ @rm -f $(LOCAL_BIN)/pacBioToCA
+ @ln -s $@ $(LOCAL_BIN)/pacBioToCA
convertToPBCNS: convertToPBCNS.pl
@cp $< $(LOCAL_BIN)/$@
@chmod 775 $(LOCAL_BIN)/$@
+
+addCNSToStore: addCNSToStore.pl
+ @cp $< $(LOCAL_BIN)/$@
+ @chmod 775 $(LOCAL_BIN)/$@
+
+# for now we support these as binaries, we should do better and build code
+
+# This is _ALWAYS_ unpacked, regardless of up-to-dateness.
+mhap: mhap.tar.gz
+ @tar xzf $< -C $(LOCAL_LIB)/
+
+
+falcon_sense: falcon_sense.$(LOCAL_NAME).bin
+ cp $< $(LOCAL_BIN)/$@
+ chmod 775 $(LOCAL_BIN)/$@
+
+pbdagcon: pbdagcon.$(LOCAL_NAME).bin
+ cp $< $(LOCAL_BIN)/$@
+ chmod 775 $(LOCAL_BIN)/$@
+
+blasr: blasr.$(LOCAL_NAME).bin
+ cp $< $(LOCAL_BIN)/$@
+ chmod 775 $(LOCAL_BIN)/$@
+
+pbutgcns: pbutgcns.$(LOCAL_NAME).bin
+ cp $< $(LOCAL_BIN)/$@
+ chmod 775 $(LOCAL_BIN)/$@
+
+ifeq ($(OSTYPE), Darwin)
+
+jellyfish: jellyfish.$(LOCAL_NAME).bin
+ cp $< $(LOCAL_BIN)/$@
+ cp *dylib $(LOCAL_LIB)/
+ chmod 775 $(LOCAL_BIN)/$@
+
+else
+
+jellyfish: jellyfish.$(LOCAL_NAME).bin
+ cp $< $(LOCAL_BIN)/$@
+ chmod 775 $(LOCAL_BIN)/$@
+
+endif
+
+
+falcon_sense_missing:
+ @echo "!!!!!!! WARNING falcon_sense not supplied for $(LOCAL_NAME). Please add to your PATH if installed."
+
+pbdagcon_missing:
+ @echo "!!!!!!! WARNING pbdagcon not supplied for $(LOCAL_NAME). Please add to your PATH if installed."
+
+blasr_missing:
+ @echo "!!!!!!! WARNING blasr not supplied for $(LOCAL_NAME). Please add to your PATH if installed."
+
+pbutgcns_missing:
+ @echo "!!!!!!! WARNING pbutgcns not supplied for $(LOCAL_NAME). Please add to your PATH if installed."
+
+jellyfish_missing:
+ @echo "!!!!!!! WARNING jellyfish not supplied for $(LOCAL_NAME). Please add to your PATH if installed."
diff --git a/src/AS_PBR/OutputLayout.cc b/src/AS_PBR/OutputLayout.cc
index 277428e..648c351 100755
--- a/src/AS_PBR/OutputLayout.cc
+++ b/src/AS_PBR/OutputLayout.cc
@@ -37,7 +37,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-const char *mainid = "$Id: OutputLayout.cc 4484 2013-12-13 15:07:13Z skoren $";
+const char *mainid = "$Id: OutputLayout.cc 4614 2015-02-19 21:25:35Z skoren $";
#include "AS_global.H"
#include "AS_MSG_pmesg.H"
@@ -76,11 +76,12 @@ main (int argc, char * argv []) {
char *gkpStorePath = NULL;
uint32 part = 0;
- bool outputAsLay = true;
+ OUTPUT_TYPE outputType = OUT_AMOS;
// initialize default parameters
thread_globals.globalRepeats = TRUE;
thread_globals.minLength = 500;
+ thread_globals.maxCoverage = 0;
thread_globals.maxUncorrectedGap = 0;
thread_globals.hasMates = false;
thread_globals.verboseLevel = VERBOSE_OFF;
@@ -100,6 +101,9 @@ main (int argc, char * argv []) {
} else if (strcmp(argv[arg], "-o") == 0) {
strncpy(thread_globals.prefix, argv[++arg], FILENAME_MAX);
+ } else if (strcmp(argv[arg], "-i") == 0) {
+ strncpy(thread_globals.inPrefix, argv[++arg], FILENAME_MAX);
+
} else if (strcmp(argv[arg], "-l") == 0) {
thread_globals.minLength = atoi(argv[++arg]);
@@ -129,7 +133,10 @@ main (int argc, char * argv []) {
thread_globals.erate = atof(argv[++arg]);
} else if (strcmp(argv[arg], "-P") == 0) {
- outputAsLay = false;
+ outputType = OUT_PBDAGCON;
+
+ } else if (strcmp(argv[arg], "-F") == 0) {
+ outputType = OUT_FALCON;
} else {
err++;
@@ -149,6 +156,7 @@ main (int argc, char * argv []) {
fprintf(stderr, " -G Mandatory path to a gkpStore.\n");
fprintf(stderr, "\n");
fprintf(stderr, " -l %d ignore corrected fragments less than %d bp\n", thread_globals.minLength, thread_globals.minLength);
+ fprintf(stderr, " -i %s input prefix of %s\n", thread_globals.inPrefix, thread_globals.inPrefix);
fprintf(stderr, " -o %s output prefix of %s\n", thread_globals.prefix, thread_globals.prefix);
fprintf(stderr, "\n");
fprintf(stderr, " -M %d The maximum uncorrected PacBio gap that will be allowed. When there is no short-read coverage for a region, by default the pipeline will split a PacBio sequence. This option allows a number of PacBio sequences without short-read coverage to remain. For example, specifying 50, will mean 50bp can have no short-read coverage without splitting the PacBio sequence. Warning: this will allow more sequences that went through the SMRTportal to not be fixed.\n", [...]
@@ -194,7 +202,7 @@ main (int argc, char * argv []) {
gkStream *fs = new gkStream(thread_globals.gkp, 1, thread_globals.gkp->gkStore_getNumFragments(), GKFRAGMENT_INF);
uint32 numFrags = loadFragments(fs, thread_globals.libToInclude, thread_globals.frgToLen);
- outputResults(&thread_globals, part, outputAsLay);
+ outputResults(&thread_globals, part, outputType);
delete[] thread_globals.libToInclude;
delete thread_globals.gkp;
}
diff --git a/src/AS_PBR/PBcR.pl b/src/AS_PBR/PBcR.pl
new file mode 100755
index 0000000..3705876
--- /dev/null
+++ b/src/AS_PBR/PBcR.pl
@@ -0,0 +1,2992 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2011, Battelle National Biodefense Institute (BNBI);
+# all rights reserved. Authored by: Sergey Koren
+#
+# This Software was prepared for the Department of Homeland Security
+# (DHS) by the Battelle National Biodefense Institute, LLC (BNBI) as
+# part of contract HSHQDC-07-C-00020 to manage and operate the National
+# Biodefense Analysis and Countermeasures Center (NBACC), a Federally
+# Funded Research and Development Center.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of the Battelle National Biodefense Institute nor
+# the names of its contributors may be used to endorse or promote
+# products derived from this software without specific prior written
+# permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+###########################################################################
+#
+# Read in fragments from fastq-format sequence and quality files,
+# correct to pacbio fragments.
+#
+
+use strict;
+
+use Config; # for @signame
+use FindBin;
+use Cwd;
+use Carp;
+use FileHandle;
+
+use POSIX qw(ceil floor sys_wait_h);
+
+my $TAB_SIZE = 8;
+
+my %global;
+
+my @nonCAOptions = ("QV", "genomeSize", "shortReads", "longReads", "ovlMemory", "assemble", "assembleMinCoverage", "assembleCoverage","sensitive","onlyContained", "gridOptionsMhap", "mhapOvlRefBlockSize", "mhapLoadFactor", "mhapPrecompute", "mhapVersion", "mhap", "secret", "localStaging", "pbcns", "bankPath","libraryname", "specFile", "length", "coverageCutoff", "maxCoverage", "falconForce", "falconOptions", "falconcns", "maxGap", "maxUncorrectedGap", "samFOFN", "blasr", "bowtie", "threa [...]
+
+my $commandLineOptions = "";
+my @order;
+
+sub getGlobal ($) {
+ my $var = shift @_;
+ if (!exists($global{$var})) {
+ return undef;
+ }
+ return($global{$var});
+}
+
+sub setGlobal ($$) {
+ my $var = shift @_;
+ my $val = shift @_;
+
+ # If no value, set the field to undefined, the default for many of the options.
+
+ $val = undef if ($val eq "");
+
+ # CA 8.3 added support for LSF, and renamed the grid* options to 'gridEngine', and the sge*
+ # options to 'grid'.
+
+ $var = "gridEngine" if ($var eq "gridEngine"); # This one didn't change, but left here for completeness
+ $var = "gridEngineSubmitCommand" if ($var eq "gridSubmitCommand");
+ $var = "gridEngineHoldOption" if ($var eq "gridHoldOption");
+ $var = "gridEngineHoldOptionNoArray" if ($var eq "gridHoldOptionNoArray");
+ $var = "gridEngineSyncOption" if ($var eq "gridSyncOption");
+ $var = "gridEngineNameOption" if ($var eq "gridNameOption");
+ $var = "gridEngineArrayOption" if ($var eq "gridArrayOption");
+ $var = "gridEngineArrayName" if ($var eq "gridArrayName");
+ $var = "gridEngineOutputOption" if ($var eq "gridOutputOption");
+ $var = "gridEnginePropagateCommand" if ($var eq "gridPropagateCommand");
+ $var = "gridEngineNameToJobIDCommand" if ($var eq "gridNameToJobIDCommand");
+ $var = "gridEngineTaskID" if ($var eq "gridTaskID");
+ $var = "gridEngineArraySubmitID" if ($var eq "gridArraySubmitID");
+ $var = "gridEngineJobID" if ($var eq "gridJobID");
+
+ $var = "gridOptions" if ($var eq "sge");
+ $var = "gridJobName" if ($var eq "sgeName");
+ $var = "gridOptionsScript" if ($var eq "sgeScript");
+ $var = "gridOptionsMerTrim" if ($var eq "sgeMerTrim");
+ $var = "gridOptionsOverlap" if ($var eq "sgeOverlap");
+ $var = "gridOptionsMerOverlapSeed" if ($var eq "sgeMerOverlapSeed");
+ $var = "gridOptionsMerOverlapExtend" if ($var eq "sgeMerOverlapExtend");
+ $var = "gridOptionsConsensus" if ($var eq "sgeConsensus");
+ $var = "gridOptionsFragmentCorrection" if ($var eq "sgeFragmentCorrection");
+ $var = "gridOptionsOverlapCorrection" if ($var eq "sgeOverlapCorrection");
+
+ $var = "gridOptionsCorrection" if ($var eq "sgeCorrection");
+
+
+ $global{$var} = $val;
+ $val =~ s/\\\"/\"/g;
+ $val =~ s/\"/\\\"/g;
+ $val =~ s/\\\$/\$/g;
+ $val =~ s/\$/\\\$/g;
+
+ # some vars should be global like specFile
+ if ($var eq "specFile" || $var eq "fastqFile" || $var eq "samFOFN") {
+ $val = makeAbsolute($val);
+ }
+ if ($var eq "utgErrorRate") {
+ setGlobal("cgwErrorRate", $val);
+ setGlobal("cnsErrorRate", $val);
+ setGlobal("ovlErrorRate", $val);
+ }
+
+ if (($var eq "gridEngine") && ($val eq "PBS")) {
+ setGlobal("gridEngineSubmitCommand", "qsub");
+ setGlobal("gridEngineHoldOption", "-W depend=afteranyarray:\"WAIT_TAG\"");
+ setGlobal("gridEngineHoldOptionNoArray", "-W depend=afterany:\"WAIT_TAG\"");
+ setGlobal("gridEngineSyncOption", "");
+ setGlobal("gridEngineNameOption", "-d `pwd` -N");
+ setGlobal("gridEngineArrayOption", "-t ARRAY_JOBS");
+ setGlobal("gridEngineArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
+ setGlobal("gridEngineOutputOption", "-j oe -o");
+ setGlobal("gridEnginePropagateCommand", "qalter -W depend=afterany:\"WAIT_TAG\"");
+ setGlobal("gridEngineNameToJobIDCommand", "qstat -f |grep -F -B 1 WAIT_TAG | grep Id: | grep -F [] |awk '{print \$NF}'");
+ setGlobal("gridEngineNameToJobIDCommandNoArray", "qstat -f |grep -F -B 1 WAIT_TAG | grep Id: |awk '{print \$NF}'");
+ setGlobal("gridEngineTaskID", "PBS_ARRAYID");
+ setGlobal("gridEngineArraySubmitID", "\\\$PBS_ARRAYID");
+ setGlobal("gridEngineJobID", "PBS_JOBID");
+ }
+
+ if (($var eq "gridEngine") && ($val eq "SGE")) {
+ setGlobal("gridEngineSubmitCommand", "qsub");
+ setGlobal("gridEngineHoldOption", "-hold_jid \"WAIT_TAG\"");
+ setGlobal("gridEngineHoldOptionNoArray", undef);
+ setGlobal("gridEngineSyncOption", "-sync y");
+ setGlobal("gridEngineNameOption", "-cwd -N");
+ setGlobal("gridEngineArrayOption", "-t ARRAY_JOBS");
+ setGlobal("gridEngineArrayName", "ARRAY_NAME");
+ setGlobal("gridEngineOutputOption", "-j y -o");
+ setGlobal("gridEnginePropagateCommand", "qalter -hold_jid \"WAIT_TAG\"");
+ setGlobal("gridEngineNameToJobIDCommand", undef);
+ setGlobal("gridEngineNameToJobIDCommandNoArray", undef);
+ setGlobal("gridEngineTaskID", "SGE_TASK_ID");
+ setGlobal("gridEngineArraySubmitID", "\\\$TASK_ID");
+ setGlobal("gridEngineJobID", "JOB_ID");
+ }
+
+ if (($var eq "gridEngine") && ($val eq "LSF")) {
+ setGlobal("gridEngineSubmitCommand", "bsub");
+ setGlobal("gridEngineHoldOption", "-w \"numended\(\"WAIT_TAG\", \*\)\"");
+ setGlobal("gridEngineHoldOptionNoArray", "-w \"done\(\"WAIT_TAG\"\)\"");
+ setGlobal("gridEngineSyncOption", "-K");
+ setGlobal("gridEngineNameOption", "-J");
+ setGlobal("gridEngineArrayOption", "");
+ setGlobal("gridEngineArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
+ setGlobal("gridEngineOutputOption", "-o");
+ setGlobal("gridEnginePropagateCommand", "bmodify -w \"done\(\"WAIT_TAG\"\)\"");
+ setGlobal("gridEngineNameToJobIDCommand", "bjobs -A -J \"WAIT_TAG\" | grep -v JOBID");
+ setGlobal("gridEngineNameToJobIDCommandNoArray", "bjobs -J \"WAIT_TAG\" | grep -v JOBID");
+ setGlobal("gridEngineTaskID", "LSB_JOBINDEX");
+ setGlobal("gridEngineArraySubmitID", "%I");
+ setGlobal("gridEngineJobID", "LSB_JOBID");
+ }
+
+ # legacy support
+ if ($var eq "secret") {
+ setGlobal("mhap", $val);
+ }
+
+ if ($var eq "gridOptionsOverlap" & !defined(getGlobal("gridOptionsMhap"))) {
+ setGlobal("gridOptionsMhap", $val);
+ }
+
+ if ($var eq "falconcns" && ($val eq 1)) {
+ setGlobal("pbcns", 0);
+ }
+
+ if ($var eq "pbcns" && ($val eq 1)) {
+ setGlobal("falconcns", 0);
+ }
+
+ if ($var eq "threads") {
+ setGlobal("merylThreads",$val);
+ setGlobal("ovlThreads", $val);
+ setGlobal("consensusConcurrency", $val);
+ }
+
+ $commandLineOptions .= " \"$var=$val\" ";
+ # store the order so we can properly output the runCA spec file
+ push(@order, $var);
+}
+
+sub initializeLocalSystem() {
+ if (defined(getGlobal("submitToGrid")) && getGlobal("submitToGrid") == 1) {
+ if (!defined(getGlobal("threads"))) {
+ $global{"threads"} = 16;
+ }
+ if (!defined(getGlobal("ovlMemory"))) {
+ $global{"ovlMemory"} = 32;
+ }
+ if (!defined(getGlobal("merylMemory"))) {
+ $global{"merylMemory"} = 32000;
+ }
+ if (!defined(getGlobal("merylThreads"))) {
+ $global{"merylThreads"} = 16;
+ }
+ if (!defined(getGlobal("ovlStoreMemory"))) {
+ $global{"ovlStoreMemory"} = 32000;
+ }
+ if (!defined(getGlobal("ovlThreads"))) {
+ $global{"ovlThreads"} = 16;
+ }
+ if (!defined(getGlobal("ovlConcurrency"))) {
+ $global{"ovlConcurrency"} = 1;
+ }
+ if (!defined(getGlobal("consensusConcurrency"))) {
+ $global{"consensusConcurrency"} = 16;
+ }
+
+ return;
+ }
+
+ # local run, get system info
+ my $numCPU = 16;
+ my $mem = 32 * 1024 * 1024 * 1024;
+
+ if ( -e "/proc/cpuinfo" && -e "/proc/meminfo") {
+ $numCPU = `cat /proc/cpuinfo |grep processor |wc -l`;
+ chomp $numCPU;
+ $mem = `cat /proc/meminfo |grep MemTotal: |awk '{printf(\"\%d\\n\", \$2*1024)}'`;
+ chomp $mem;
+ } else {
+ my $sysInfo = `which sysctl`;
+ chomp $sysInfo;
+ if ( $sysInfo ne "") {
+ $numCPU = `$sysInfo -n hw.ncpu`;
+ chomp $numCPU;
+ $mem = `$sysInfo -n hw.memsize`;
+ chomp $mem;
+ }
+ }
+ if (!defined(getGlobal("threads"))) {
+ $global{"threads"} = $numCPU;
+ }
+ if (!defined(getGlobal("ovlMemory"))) {
+ $global{"ovlMemory"} = int $mem / 1024 / 1024 / 1024;
+ }
+ if (!defined(getGlobal("merylMemory"))) {
+ $global{"merylMemory"} = int $mem / 1024 / 1024;
+ }
+ if (!defined(getGlobal("merylThreads"))) {
+ $global{"merylThreads"} = $numCPU;
+ }
+ if (!defined(getGlobal("ovlStoreMemory"))) {
+ $global{"ovlStoreMemory"} = int $mem / 1024 / 1024;
+ }
+ if (!defined(getGlobal("ovlThreads"))) {
+ $global{"ovlThreads"} = $numCPU;
+ }
+ if (!defined(getGlobal("ovlConcurrency"))) {
+ $global{"ovlConcurrency"} = 1;
+ }
+ if (!defined(getGlobal("consensusConcurrency")) && !defined(getGlobal("cnsConcurrency"))) {
+ $global{"consensusConcurrency"} = $numCPU;
+ $global{"cnsConcurrency"} = $numCPU;
+ }
+}
+
+sub setDefaults() {
+ # grid submission options, duplicate of runCA
+ $global{"gridEngineSubmitCommand"} = "qsub";
+ $global{"gridEngineHoldOption"} = "-hold_jid \"WAIT_TAG\""; # for lsf it is -w "done("WAIT_TAG")"
+ $global{"gridEngineSyncOption"} = "-sync y"; # for lsf it is -K
+ $global{"gridEngineNameOption"} = "-cwd -N"; # for lsf it is -J
+ $global{"gridEngineArrayOption"} = "-t ARRAY_JOBS"; # for lsf, empty ("")
+ $global{"gridEngineArrayName"} = "ARRAY_NAME"; # for lsf, it is ARRAY_NAME[ARRAY_JOBS]
+ $global{"gridEngineOutputOption"} = "-j y -o"; # for lsf, it is -o
+ $global{"gridEnginePropagateCommand"} = "qalter -hold_jid \"WAIT_TAG\""; # for lsf it is bmodify -w "done(WAIT_TAG)"
+ $global{"gridEngineNameToJobIDCommand"} = undef; # for lsf it is bjobs -J "WAIT_TAG" | grep -v JOBID
+ $global{"gridEngineTaskID"} = "SGE_TASK_ID"; # for lsf it is LSB_JOBINDEX
+ $global{"gridEngineJobID"} = "JOB_ID";
+ $global{"gridEngineArraySubmitID"} = "\$TASK_ID"; # for lsf it is %I
+
+ $global{"shell"} = "/bin/bash";
+
+ $global{"utgErrorRate"} = 0.25;
+ $global{"utgErrorLimit"} = 6.5;
+ $global{"ovlErrorRate"} = 0.25;
+ $global{"cnsErrorRate"} = 0.25;
+ $global{"cgwErrorRate"} = 0.25;
+
+
+ $global{"ovlHashBlockLength"} = 1000000000;
+ $global{"ovlRefBlockLength"} = 100000000000;
+ $global{"ovlRefBlockSize"} = 0;
+ $global{"shortReads"} = undef;
+ $global{"longReads"} = undef;
+ $global{"QV"} = 54.5;
+ $global{"libraryname"} = undef;
+ $global{"falconcns"} = 1;
+ $global{"falconForce"} = undef;
+ $global{"falconOptions"} = "--max_n_read 200 --min_idt 0.70 --output_multi --local_match_count_threshold 2";
+ $global{"pbcns"} = 0;
+ $global{"bankPath"} = undef;
+ $global{"specFile"} = undef;
+ $global{"length"} = 500;
+
+ # algorithm defaults
+ $global{"frgMinLen"} = 64;
+ $global{"coverageCutoff"} = 0;
+ $global{"maxCoverage"} = 40;
+ $global{"genomeSize"} = 0;
+ $global{"maxUncorrectedGap"} = 1500;
+ $global{"repeats"} = "";
+ $global{"samFOFN"} = undef;
+ $global{"blasr"} = undef;
+ $global{"bowtie"} = undef;
+ $global{"onlyContained"} = 0;
+ $global{"sensitive"} = 0;
+ $global{"mhap"} = undef;
+ $global{"mhapVersion"} = "1.5b1";
+ $global{"mhapPrecompute"} = 1;
+ $global{"mhapLoadFactor"} = 12500;
+ $global{"merSize"} = 16;
+ $global{"doOverlapBasedTrimming"} = 0; # should this be defaulted to true or false?
+
+ # now the assembly parameters
+ $global{"assemble"} = 1;
+ $global{"batOptions"} = "-RS -NS -CS";
+ $global{"asmCns"} = "pbutgcns";
+ $global{"cnsMinFrags"} = 1000;
+ $global{"asmMerSize"} = 22;
+ $global{"assembleCoverage"} = 25;
+ $global{"assembleMinCoverage"} = 10;
+ $global{"unitigger"} = "bogart";
+ $global{"asmOvlErrorRate"} = 0.03;
+ $global{"asmUtgErrorRate"} = 0.025;
+ $global{"asmCgwErrorRate"} = 0.10;
+ $global{"asmCnsErrorRate"} = 0.10;
+ $global{"asmOBT"} = 1;
+ $global{"utgGraphErrorLimit"} = 0;
+ $global{"utgGraphErrorRate"} = 0.025;
+ $global{"utgMergeErrorLimit"} = 0;
+ $global{"utgMergeErrorRate"} = 0.025;
+ $global{"asmObtErrorRate"} = 0.03;
+ $global{"asmObtErrorLimit"} =4.5;
+ $global{"doFragmentCorrection"}=1;
+ $global{"doUnitigSplitting"}=0;
+
+ # default thread jobs/grid params based on system
+ $global{"partitions"} = 200;
+ $global{"submitToGrid"} = 0;
+ $global{"gridOptions"} = undef;
+ $global{"gridOptionsCorrection"} = "-pe threads 16 -l mem=2GB";
+ $global{"gridOptionsOverlap"} = "-pe threads 16 -l mem=2GB";
+ $global{"gridOptionsConsensus"} = "-pe threads 16";
+ $global{"gridOptionsScript"} = "-pe threads 1";
+ $global{"useGrid"} = 0;
+ $global{"scriptOnGrid"} = 0;
+
+ $global{"cleanup"} = 1;
+
+ $global{"fastqFile"} = undef;
+
+ # record the options in the order array
+ foreach my $key (keys %global) {
+ push @order, $key;
+ }
+}
+
+sub updateSpecFile($) {
+ my $outFile = shift @_;
+ open(W, "> $outFile") or die("Couldn't open '$outFile'", undef);
+
+ my $print = 1;
+ foreach my $key (@order) {
+ $print = 1;
+ foreach my $nonCaOption (@nonCAOptions) {
+ if (index($key, $nonCaOption) == 0) {
+ $print = 0;
+ last;
+ }
+ }
+ if ($print == 1) {
+ print W "$key = " . getGlobal($key) . "\n";
+ }
+ }
+ close(W);
+}
+
+sub setParametersFromFile ($@) {
+ my $specFile = shift @_;
+ my @fragFiles = @_;
+
+ # Client should be ensuring that the file exists before calling this function.
+ die "specFile '$specFile' not found.\n" if (! -e "$specFile");
+
+ print STDERR "\n";
+ print STDERR "###\n";
+ print STDERR "### Reading options from '$specFile'\n";
+ print STDERR "###\n";
+ print STDERR "\n";
+
+ open(F, "< $specFile") or die("Couldn't open '$specFile'", undef);
+
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+
+ if (-e $_) {
+ my $xx = $_;
+ $xx = "$ENV{'PWD'}/$xx" if ($xx !~ m!^/!);
+ if (-e $xx) {
+ push @fragFiles, $xx;
+ } else {
+ setGlobal("help", getGlobal("help") . "File not found '$_' after appending absolute path.\n");
+ }
+ } elsif (m/\s*(\w*)\s*=([^#]*)#*.*$/) {
+ my ($var, $val) = ($1, $2);
+ $var =~ s/^\s+//; $var =~ s/\s+$//;
+ $val =~ s/^\s+//; $val =~ s/\s+$//;
+ undef $val if ($val eq "undef");
+ if ($var eq "maxGap") {
+ $var = "maxUncorrectedGap";
+ }
+ setGlobal($var, $val);
+ } else {
+ setGlobal("help", getGlobal("help") . "File not found or unknown specFile option line '$_'.\n");
+ }
+ }
+ close(F);
+
+ return(@fragFiles);
+}
+
+sub setParametersFromCommandLine(@) {
+ my @specOpts = @_;
+
+ if (scalar(@specOpts) > 0) {
+ print STDERR "\n";
+ print STDERR "###\n";
+ print STDERR "### Reading options from the command line.\n";
+ print STDERR "###\n";
+ print STDERR "\n";
+ }
+
+ foreach my $s (@specOpts) {
+ if ($s =~ m/\s*(\w*)\s*=(.*)/) {
+ my ($var, $val) = ($1, $2);
+ $var =~ s/^\s+//; $var =~ s/\s+$//;
+ $val =~ s/^\s+//; $val =~ s/\s+$//;
+ setGlobal($var, $val);
+ } else {
+ setGlobal("help", getGlobal("help") . "Misformed command line option '$s'.\n");
+ }
+ }
+}
+
+sub makeAbsolute ($) {
+ my $val = shift @_;
+ if (defined($val) && ($val !~ m!^/!)) {
+ $val = "$ENV{'PWD'}/$val";
+ }
+ return $val;
+}
+
+sub getInstallDirectory () {
+ my @t = split '/', "$FindBin::RealBin";
+ pop @t; # bin
+ pop @t; # arch, e.g., FreeBSD-amd64
+ my $installDir = join '/', @t; # path to the assembler
+
+ return($installDir);
+}
+
+sub getBinDirectory () {
+ my $installDir = getInstallDirectory();
+
+ my $syst = `uname -s`; chomp $syst; # OS implementation
+ my $arch = `uname -m`; chomp $arch; # Hardware platform
+ my $name = `uname -n`; chomp $name; # Name of the system
+
+ $arch = "amd64" if ($arch eq "x86_64");
+ $arch = "ppc" if ($arch eq "Power Macintosh");
+
+ my $path = "$installDir/$syst-$arch/bin";
+
+ my $pathMap = getGlobal("pathMap");
+ if (defined($pathMap)) {
+ open(F, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef);
+ while (<F>) {
+ my ($n, $b) = split '\s+', $_;
+ $path = $b if ($name eq $n);
+ }
+ close(F);
+ }
+
+ return($path);
+}
+
+sub getBinDirectoryShellCode () {
+ my $installDir = getInstallDirectory();
+ my $string;
+
+ $string = "\n";
+ $string .= "syst=`uname -s`\n";
+ $string .= "arch=`uname -m`\n";
+ $string .= "name=`uname -n`\n";
+ $string .= "\n";
+ $string .= "if [ \"\$arch\" = \"x86_64\" ] ; then\n";
+ $string .= " arch=\"amd64\"\n";
+ $string .= "fi\n";
+ $string .= "if [ \"\$arch\" = \"Power Macintosh\" ] ; then\n";
+ $string .= " arch=\"ppc\"\n";
+ $string .= "fi\n";
+ $string .= "\n";
+ $string .= "bin=\"$installDir/\$syst-\$arch/bin\"\n";
+ $string .= "\n";
+
+ my $pathMap = getGlobal("pathMap");
+ if (defined($pathMap)) {
+ open(PM, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef);
+ while (<PM>) {
+ my ($n, $b) = split '\s+', $_;
+ $string .= "if [ \"\$name\" = \"$n\" ] ; then\n";
+ $string .= " bin=\"$b\"\n";
+ $string .= "fi\n";
+ }
+ close(PM);
+ $string .= "\n";
+ }
+
+ return($string);
+}
+
+sub runCommand($$) {
+ my $dir = shift;
+ my $cmd = shift;
+
+ my $t = localtime();
+ my $d = time();
+ print STDERR "----------------------------------------START $t\n$cmd\n";
+
+ my $rc = 0xffff & system("cd $dir && $cmd");
+
+ $t = localtime();
+ print STDERR "----------------------------------------END $t (", time() - $d, " seconds)\n";
+
+ return(0) if ($rc == 0);
+
+ die "Failed to execute $cmd\n";
+}
+
+################################################################################
+
+# Functions for running multiple processes at the same time.
+
+my $numberOfProcesses = 0; # Number of jobs concurrently running
+my $numberOfProcessesToWait = 0; # Number of jobs we can leave running at exit
+my @processQueue = ();
+my @processesRunning = ();
+my $printProcessCommand = 1; # Show commands as they run
+
+sub schedulerSetNumberOfProcesses {
+ $numberOfProcesses = shift @_;
+}
+
+sub schedulerSubmit {
+ chomp @_;
+ push @processQueue, @_;
+}
+
+sub schedulerForkProcess {
+ my $process = shift @_;
+ my $pid;
+
+ # From Programming Perl, page 167
+ FORK: {
+ if ($pid = fork) {
+ # Parent
+ #
+ return($pid);
+ } elsif (defined $pid) {
+ # Child
+ #
+ exec($process);
+ } elsif ($! =~ /No more processes/) {
+ # EAGIN, supposedly a recoverable fork error
+ sleep 1;
+ redo FORK;
+ } else {
+ die "Can't fork: $!\n";
+ }
+ }
+}
+
+sub schedulerReapProcess {
+ my $pid = shift @_;
+
+ if (waitpid($pid, &WNOHANG) > 0) {
+ return(1);
+ } else {
+ return(0);
+ }
+}
+
+sub schedulerRun {
+ my @newProcesses;
+
+ # Reap any processes that have finished
+ #
+ undef @newProcesses;
+ foreach my $i (@processesRunning) {
+ if (schedulerReapProcess($i) == 0) {
+ push @newProcesses, $i;
+ }
+ }
+ undef @processesRunning;
+ @processesRunning = @newProcesses;
+
+ # Run processes in any available slots
+ #
+ while ((scalar(@processesRunning) < $numberOfProcesses) &&
+ (scalar(@processQueue) > 0)) {
+ my $process = shift @processQueue;
+ print STDERR "$process\n";
+ push @processesRunning, schedulerForkProcess($process);
+ }
+}
+
+sub schedulerFinish {
+ my $child;
+ my @newProcesses;
+ my $remain;
+
+ my $t = localtime();
+ my $d = time();
+ print STDERR "----------------------------------------START CONCURRENT $t\n";
+
+ $remain = scalar(@processQueue);
+
+ # Run all submitted jobs
+ #
+ while ($remain > 0) {
+ schedulerRun();
+
+ $remain = scalar(@processQueue);
+
+ if ($remain > 0) {
+ $child = waitpid -1, 0;
+
+ undef @newProcesses;
+ foreach my $i (@processesRunning) {
+ push @newProcesses, $i if ($child != $i);
+ }
+ undef @processesRunning;
+ @processesRunning = @newProcesses;
+ }
+ }
+
+ # Wait for them to finish, if requested
+ #
+ while (scalar(@processesRunning) > $numberOfProcessesToWait) {
+ waitpid(shift @processesRunning, 0);
+ }
+
+ $t = localtime();
+ print STDERR "----------------------------------------END CONCURRENT $t (", time() - $d, " seconds)\n";
+}
+
+################################################################################
+#
+# Functions for running jobs on grid
+
+sub runningOnGrid () {
+ my $taskID = getGlobal("gridEngineJobID");
+ return(defined($ENV{$taskID}));
+}
+
+sub findNextScriptOutputFile ($$) {
+ my $wrk = shift @_;
+ my $prefix = shift @_;
+ my $idx = "00";
+ while (-e "$wrk/$prefix.$idx.sh") {
+ $idx++;
+ }
+ return("$wrk/$prefix.$idx");
+}
+
+sub buildGridArray($$$) {
+ my $name = shift @_;
+ my $maxLimit = shift @_;
+ my $globalValue = shift @_;
+
+ my $arrayJobName = getGlobal($globalValue);
+ $arrayJobName =~ s/ARRAY_NAME/$name/g;
+ $arrayJobName =~ s/ARRAY_JOBS/1-$maxLimit/g;
+
+ return $arrayJobName;
+}
+
+sub getGridArrayName($$) {
+ my $name = shift @_;
+ my $maxLimit = shift @_;
+ return buildGridArray($name, $maxLimit, "gridEngineArrayName");
+}
+
+sub getGridArrayOption($$) {
+ my $name = shift @_;
+ my $maxLimit = shift @_;
+ return buildGridArray($name, $maxLimit, "gridEngineArrayOption");
+}
+
+sub writeScriptHeader($$$$$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+ my $script = shift @_;
+ my $executable = shift @_;
+ my $options = shift @_;
+
+ open(F, "> $script") or caFailure("failed to open '$script' for writing", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "#\n";
+ print F "if [ \"x\$SGE_ROOT\" != \"x\" ]; then \n";
+ print F " # Attempt to (re)configure SGE. For reasons Bri doesn't know,\n";
+ print F " # jobs submitted to SGE, and running under SGE, fail to read his\n";
+ print F " # .tcshrc (or .bashrc, limited testing), and so they don't setup\n";
+ print F " # SGE (or ANY other paths, etc) properly. For the record,\n";
+ print F " # interactive SGE logins (qlogin, etc) DO set the environment.\n";
+ print F " \n";
+ print F " . \$SGE_ROOT/\$SGE_CELL/common/settings.sh\n";
+ print F "fi\n";
+ print F "\n";
+ print F "# On the off chance that there is a pathMap, and the host we\n";
+ print F "# eventually get scheduled on doesn't see other hosts, we decide\n";
+ print F "# at run time where the binary is.\n";
+
+ print F getBinDirectoryShellCode();
+
+ print F "/usr/bin/env perl \$bin/$executable $options\n";
+ close(F);
+
+ system("chmod +x $script");
+}
+
+sub submit($$$$$) {
+ my $wrk = shift @_;
+ my $script = shift @_;
+ my $output = shift @_;
+ my $prefix = shift @_;
+ my $waitTag = shift @_;
+
+ my $grid = getGlobal("gridOptions");
+ my $gridScript = getGlobal("gridOptionsScript");
+ my $gridPropHold = getGlobal("gridEnginePropagateHold");
+
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $holdOption = getGlobal("gridEngineHoldOption");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
+ my $holdPropagateCommand = getGlobal("gridEnginePropagateCommand");
+
+ my $jobName = $prefix;
+ if (defined($waitTag)) {
+ my $hold = $holdOption;
+ if (getGlobal("gridEngine") eq "LSF" || getGlobal("gridEngine") eq "PBS"){
+ my $tcmd = getGlobal("gridEngineNameToJobIDCommand");
+ $tcmd =~ s/WAIT_TAG/$waitTag/g;
+ my $propJobCount = `$tcmd |wc -l`;
+ chomp $propJobCount;
+ if ($propJobCount == 0) {
+ $tcmd = getGlobal("gridEngineNameToJobIDCommandNoArray");
+ $tcmd =~ s/WAIT_TAG/$waitTag/g;
+ $hold = getGlobal("gridEngineHoldOptionNoArray");
+ $propJobCount = `$tcmd |wc -l`;
+ }
+ if ($propJobCount != 1) {
+ print STDERR "Warning: multiple IDs for job $gridPropHold got $propJobCount and should have been 1.\n";
+ }
+ my $jobID = `$tcmd |tail -n 1 |awk '{print \$1}'`;
+ chomp $jobID;
+ $hold =~ s/WAIT_TAG/$jobID/g;
+ } else{
+ $hold =~ s/WAIT_TAG/$waitTag/g;
+ }
+ $waitTag = $hold;
+ }
+
+ my $qcmd = "$submitCommand $grid $gridScript $nameOption \"$jobName\" $waitTag $outputOption $output $script";
+ runCommand($wrk, $qcmd) and caFailure("Failed to submit script.\n");
+
+ if (defined($gridPropHold)) {
+ if (defined($holdPropagateCommand)) {
+ my $translateCmd = getGlobal("gridEngineNameToJobIDCommandNoArray");
+
+ # translate hold option to job id if necessar
+ if (defined($translateCmd) && $translateCmd ne "") {
+ my $tcmd = $translateCmd;
+ $tcmd =~ s/WAIT_TAG/$gridPropHold/g;
+ my $propJobCount = `$tcmd |wc -l`;
+ chomp $propJobCount;
+ if ($propJobCount != 1) {
+ print STDERR "Warning: multiple IDs for job $gridPropHold got $propJobCount and should have been 1.\n";
+ }
+ #my $jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
+ #chomp $jobID;
+ #print STDERR "Translated job ID $gridPropHold to be job $jobID\n";
+ #$gridPropHold = $jobID;
+ open(PROPS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that need to hold", undef);
+
+ # now we can get the job we are holding for
+ $tcmd = $translateCmd;
+ $tcmd =~ s/WAIT_TAG/$jobName/g;
+ my $holdJobCount = `$tcmd |wc -l`;
+ chomp $propJobCount;
+ if ($propJobCount != 1) {
+ print STDERR "Warning: multiple IDs for job $jobName got $propJobCount and should have been 1.\n";
+ }
+ #$jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
+ #chomp $jobID;
+ #print STDERR "Translated job ID $gridPropHold to be job $jobID\n";
+ #$jobName = $jobID;
+ open(HOLDS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that should be held for", undef);
+
+ # loop over all jobs and all sge hold commands to modify the jobs. We have no way to know which is the right one unfortunately
+ while (my $prop = <PROPS>) {
+ while (my $hold = <HOLDS>) {
+ chomp $hold;
+ chomp $prop;
+ my $hcmd = $holdPropagateCommand;
+ $hcmd =~ s/WAIT_TAG/$hold/g;
+ my $acmd = "$hcmd $prop";
+ print STDERR "Propagating hold to $prop to wait for job $hold\n";
+ system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$prop'.\n";
+ }
+ }
+ close(HOLDS);
+ close(PROPS);
+ } else {
+ $gridPropHold = "\"$gridPropHold\"";
+ $holdPropagateCommand =~ s/WAIT_TAG/$jobName/g;
+ my $acmd = "$holdPropagateCommand $gridPropHold";
+ system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$gridPropHold'.\n";
+ }
+ } else {
+ print STDERR "WARNING: Failed to reset hold '$gridPropHold', not supported on current grid environment.\n";
+ }
+ }
+}
+
+sub submitScript ($$$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+ my $waitTag = shift @_;
+ my $libraryname = "temp" . getGlobal("libraryname");
+
+ $wrk =~ s/$libraryname//g;
+ my $gridName = getGlobal("gridJobName");
+ $gridName = "_$gridName" if (defined($gridName));
+
+ return if (getGlobal("scriptOnGrid") == 0);
+ my $output = findNextScriptOutputFile("$wrk/$libraryname", "runPBcR.sge.out");
+ my $script = "$output.sh";
+ writeScriptHeader($wrk, $asm, $script, "PBcR", $commandLineOptions);
+ submit($wrk, $script, $output, "pBcR_$asm$gridName", $waitTag);
+
+ exit(0);
+}
+
+sub submitBatchJobs($$$$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+ my $cmd = shift @_;
+ my $tag = shift @_;
+
+ if (getGlobal("scriptOnGrid")) {
+ runCommand($wrk, $cmd) and die("Failed to submit batch jobs.");
+ submitScript($wrk, $asm, $tag);
+ } else {
+ print "Please execute:\n$cmd\n";
+ }
+}
+
+sub submitRunCAHelper($$$$$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+ my $options = shift @_;
+ my $TAG = shift @_;
+ my $submitSelf = shift @_;
+ my $holdPropagateCommand = getGlobal("gridEnginePropagateCommand");
+ my $holdOption = getGlobal("gridEngineHoldOption");
+ my $syncOption = getGlobal("gridEngineSyncOption");
+ my $grid = getGlobal("gridOptions");
+
+ if (defined($holdPropagateCommand) && $holdPropagateCommand ne "") {
+ # do nothing, we can use the hold option to propagate
+ $syncOption = "";
+ } else {
+ print "Warning: grid environment does not support hold propagate, running in sync mode\n";
+ }
+
+ if (getGlobal("scriptOnGrid")) {
+ my $output = findNextScriptOutputFile($wrk, "runPBcR.runCA.sge.out");
+ my $script = "$output.sh";
+ writeScriptHeader($wrk, $asm, $script, "runCA", $options);
+ submit($wrk, "$syncOption $script", $output, $TAG, undef);
+ if ($submitSelf) {
+ submitScript($wrk, $asm, $TAG);
+ }
+ } else {
+ print "Please execute:\nrunCA $options\n";
+ }
+}
+
+sub submitRunCA($$$$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+ my $options = shift @_;
+ my $TAG = shift @_;
+ submitRunCAHelper($wrk, $asm, $options, $TAG, 1);
+}
+
+sub submitRunPBcR($$$$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+ my $options = shift @_;
+ my $TAG = shift @_;
+ my $holdPropagateCommand = getGlobal("gridEnginePropagateCommand");
+ my $holdOption = getGlobal("gridEngineHoldOption");
+ my $syncOption = getGlobal("gridEngineSyncOption");
+ my $grid = getGlobal("gridOptions");
+
+ if (defined($holdPropagateCommand) && $holdPropagateCommand ne "") {
+ # do nothing, we can use the hold option to propagate
+ $syncOption = "";
+ } else {
+ print "Warning: grid environment does not support hold propagate, running in sync mode\n";
+ }
+
+ if (getGlobal("scriptOnGrid")) {
+ my $output = findNextScriptOutputFile($wrk, "runPBcR.sge.out");
+ my $script = "$output.sh";
+ writeScriptHeader($wrk, $asm, $script, "PBcR", $options);
+ submit($wrk, "$syncOption $script", $output, $TAG, undef);
+ submitScript($wrk, $asm, $TAG);
+ } else {
+ print "Please execute:\nPBcR $options\n";
+ }
+}
+
+################################################################################
+
+my $HISTOGRAM_CUTOFF = 0.985; #seems like updated blasr parameters map more
+
+sub pickMappingThreshold($$) {
+ my $histogram = shift @_;
+ my $conservative = shift @_;
+ my $threshold = 0;
+
+ # pick threshold as 2sd (95% of the bases)
+ my @hist;
+ my $total = 0;
+ my $sum = 0;
+ my $mean = 0;
+ my $variance = 0;
+
+ open(F, "< $histogram") or die("Couldn't open '$histogram'", undef);
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+
+ $hist[$_]++;
+ $total++;
+ my $delta = $_ - $mean;
+ $mean += $delta / $total;
+ $variance += $delta * ($_ - $mean);
+ }
+ $variance /= $total;
+ my $sd = sqrt($variance);
+ close(F);
+
+ for (my $i = 0; $i <= $#hist; $i++) {
+ $sum += $hist[$i];
+ if ($sum / $total > $HISTOGRAM_CUTOFF) {
+ $threshold = $i - 1;
+ last;
+ }
+ }
+ $threshold = ($threshold < ($mean + 3*$sd) ? floor($mean + 3*$sd) : $threshold);
+ if (defined($conservative) && $conservative == 1) {
+ $threshold = ceil($mean * 2);
+ }
+ print STDERR "Picked mapping cutoff of $threshold\n";
+
+ return $threshold;
+}
+
+my $MIN_FILES_WITHOUT_PARTITIONS = 20;
+my $REPEAT_MULTIPLIER = 10;
+my $MAX_CORRECTION_COVERAGE = 100;
+my $MAX_TO_PRECOMPUTE = 2000000000;
+my $MIN_SELF_CORRECTION = 50;
+my $MAX_SPLIT_PERCENTAGE = 0.30;
+my $FALCON_ERATE_ADJUST = 1.0;
+
+setDefaults();
+
+my @fragFiles;
+my @specOpts;
+my @cmdArgs;
+
+my $srcstr;
+
+{
+ local $, = " ";
+ $srcstr = "$0 @ARGV";
+}
+
+# process the spec file and the fragment files
+while (scalar(@ARGV) > 0) {
+ my $arg = shift @ARGV;
+ if ($arg eq "-s") {
+ setGlobal("specFile", shift @ARGV);
+
+ } elsif (($arg =~ /\.frg$|frg\.gz$|frg\.bz2$/i)) {
+ if (-e $arg) {
+ push @fragFiles, makeAbsolute($arg);
+ } else {
+ print "Invalid file $arg could not be found.\n";
+ }
+
+ } elsif ($arg =~ m/=/) {
+ push @specOpts, $arg;
+
+ } else {
+ push @cmdArgs, $arg;
+ }
+}
+
+# initialize parameters from the spec files
+my $err = 0;
+if (-e getGlobal("specFile")) {
+ @fragFiles = setParametersFromFile(getGlobal("specFile"), @fragFiles);
+} elsif (defined(getGlobal("specFile"))) {
+ print STDERR "Error: spec file " . getGlobal("specFile") . " does not exist. Double-check your paths and try again.\n";
+ $err++;
+}
+setParametersFromCommandLine(@specOpts);
+
+# finally, command-line parameters take precedence
+while (scalar(@cmdArgs) > 0) {
+ my $arg = shift @cmdArgs;
+
+ if ($arg eq "-shortReads") {
+ setGlobal("shortReads", 1);
+
+ } elsif ($arg eq "-longReads") {
+ setGlobal("longReads", 1);
+
+ } elsif ($arg eq "-coverageCutoff") {
+ setGlobal("coverageCutoff", shift @cmdArgs);
+ if (getGlobal("coverageCutoff") < 0) { setGlobal("coverageCutoff", 0); }
+
+ } elsif ($arg eq "-maxCoverage") {
+ setGlobal("maxCoverage", shift @cmdArgs);
+ if (getGlobal("maxCoverage") < 0) { setGlobal("maxCoverage", 0); }
+
+ } elsif ($arg eq "-genomeSize") {
+ setGlobal("genomeSize", shift @cmdArgs);
+ if (getGlobal("genomeSize") < 0) { setGlobal("genomeSize", 0); }
+
+ } elsif ($arg eq "-maxGap") {
+ setGlobal("maxUncorrectedGap", shift @cmdArgs);
+ if (getGlobal("maxUncorrectedGap") < 0) { setGlobal("maxUncorrectedGap", 0); }
+
+ } elsif ($arg eq "-length") {
+ setGlobal("length", shift @cmdArgs);
+
+ } elsif ($arg eq "-repeats") {
+ setGlobal("repeats", shift @cmdArgs);
+
+ } elsif ($arg eq "-fastq") {
+ setGlobal("fastqFile", shift @cmdArgs);
+
+ } elsif ($arg eq "-t" || $arg eq "-threads") {
+ my $thread = shift @cmdArgs;
+ if ($thread <= 0) { $thread = 1; }
+
+ setGlobal("threads", $thread);
+ } elsif ($arg eq "-l" || $arg eq "-libraryname") {
+ my $name = shift @cmdArgs;
+ if ($name =~ /^[0-9]/) {
+ print STDERR "Warning: library name must start with a character, not a number\n";
+ $name = "lib$name";
+ }
+ setGlobal("libraryname", $name);
+
+ } elsif ($arg eq "-partitions") {
+ setGlobal("partitions", shift @cmdArgs);
+
+ } elsif (($arg eq "-sge") || ($arg eq "-gridOptions")) {
+ setGlobal("gridOptions", shift @cmdArgs);
+
+ } elsif (($arg eq "-sgeCorrection") || ($arg eq "-gridOptionsCorrection")) {
+ setGlobal("gridOptionsCorrection", shift @cmdArgs);
+
+ } elsif ($arg eq "-sensitive") {
+ setGlobal("sensitive", 1);
+
+ } elsif ($arg eq "-noclean") {
+ setGlobal("cleanup", 0);
+
+ } elsif ($arg eq "-pbCNS") {
+ setGlobal("falconForce", 1);
+ setGlobal("falconcns", 1);
+ setGlobal("pbcns", 0);
+
+ } else {
+ print STDERR "Unknown parameter " + $err + "\n";
+ $err++;
+ }
+ }
+
+if ((scalar(@fragFiles) == 0) && (!defined(getGlobal("longReads")) || getGlobal("longReads") == 0)) {
+ print STDERR "Warning: no frag files specified, assuming self-correction of pacbio sequences.\n";
+ setGlobal("longReads", 1);
+}
+
+if (($err) || (!defined(getGlobal("fastqFile"))) || (!defined(getGlobal("specFile"))) || (!defined(getGlobal("libraryname")))) {
+ print STDERR "usage: $0 [options] -libraryname <name> -s <specfile> -fastq <fastqfile> [optional frg files]\n";
+ print STDERR " -length <int> Minimum length of PacBio sequences to correct/output.\n";
+ print STDERR " -partitions <int> Number of partitions for consensus\n";
+ print STDERR " -libraryname <string> Name of the library; freeformat text. Must be unique from any library names in the FRG files used for correction\n";
+ print STDERR " -threads <int> Number of threads to use for correction. Defaults to available cores on the local system\n";
+ print STDERR " -shortReads Use if the sequences for correction are 100bp or shorter.\n";
+
+ print STDERR " -genomeSize <int> Specify the approximate genome size. This will be used to compute the maximum number of bases to correct\n";
+ print STDERR " -maxCoverage <int> Maximum coverage of PacBio sequences to correct. Only the longest sequences adding up to this coverage will be corrected. Requires genomeSize to be specified. Defaults to 40X\n";
+ print STDERR " \"localStaging=<string>\" Specify a local path (such as /scratch) to use for caching overlap computation. Will speed up grid-based computation by avoiding disk contention. Only use when running on a grid system (SGE, PBS or LSF), not a single machine\n";
+ print STDERR "\nAdvanced options (EXPERT):\n";
+ print STDERR " -maxGap <int> The maximum uncorrected PacBio gap that will be allowed. When there is no short-read coverage for a region, by default the pipeline will split a PacBio sequence. This option will attempt to use other PacBio sequences to patch the gap and avoid splitting the read. Sequences where the gaps have no support will still be broken. For example, specifying 50, will mean any gap 50bp or smaller can have no short-read coverage (but has other PacBio [...]
+ print STDERR " -coverageCutoff Specify the pacBio coverage (integer) used to separate repeat copies instead of automatically estimating.\n";
+ print STDERR " \"blasr=<string>\" Use blasr for overlap computation instead of CA's built-in overlapper. This parameter specifies the blasr parameters to use.\n";
+ print STDERR " \"bowtie=<string>\" Use bowtie2 for overlap computation instead of CA's built-in overlapper. This parameter specifies the bowtie 2 parameters to use.\n";
+ print STDERR " samFOFN=<file> Skip overlap computation. Use the provided file of file names of SAM files as the overlaps instead. Any valid sam files should be accepted.\n";
+
+ print STDERR "\nComplete documentation at http://wgs-assembler.sourceforge.net/\n\n";
+
+ print STDERR "No fastq file specified. Please specify a fastq file containing PacBio sequences for correction using the -fastq parameter.\n" if (!defined(getGlobal("fastqFile")));
+ print STDERR "No spec file defined. Please specify a spec file using the -s option.\n" if (!defined(getGlobal("specFile")));
+ print STDERR "No library name provided. Please specify a name using the -library option.\n" if (!defined(getGlobal("libraryname")));
+ exit(1);
+}
+
+# get grid options
+my $submitCommand = getGlobal("gridEngineSubmitCommand");
+my $nameOption = getGlobal("gridEngineNameOption");
+my $outputOption = getGlobal("gridEngineOutputOption");
+
+#check for valid parameters for requested partitions and threads
+my $limit = 1024;
+my $FILES_PER_PARTITION = (getGlobal("maxUncorrectedGap") > 0 ? 3 : 1);
+my $FILES_PER_THREAD = 3;
+
+my $sysLimit = `ulimit -Sn`;
+chomp($sysLimit);
+if (defined($sysLimit)) {
+ $limit = $sysLimit;
+}
+if ($limit - $MIN_FILES_WITHOUT_PARTITIONS <= $FILES_PER_PARTITION*getGlobal("partitions") || $limit - $MIN_FILES_WITHOUT_PARTITIONS <= getGlobal("threads") * $FILES_PER_THREAD) {
+ my $maxPartitions = floor(($limit - $MIN_FILES_WITHOUT_PARTITIONS) / $FILES_PER_PARTITION);
+ my $maxThreads = (floor($limit - $MIN_FILES_WITHOUT_PARTITIONS) / $FILES_PER_THREAD);
+ setGlobal("partitions", ($maxPartitions < getGlobal("partitions") ? $maxPartitions : getGlobal("partitions")));
+ setGlobal("threads", ($maxThreads < getGlobal("threads") ? $maxThreads : getGlobal("threads")));
+ print STDERR "Warning: file handle limit of $limit prevents using requested partitions. Reset partitions to " . getGlobal("partitions") . " and " . getGlobal("threads") . " threads. If you want more partitions, reset the limit using ulimit -Sn and try again.\n";
+}
+if (getGlobal("threads") > getGlobal("partitions")) {
+ my $nThreads = getGlobal("partitions") - 1;
+ if ($nThreads <= 0) {
+ $nThreads = 1;
+ }
+ setGlobal("threads", $nThreads);
+ print STDERR "Warning: number of partitions should be > # threads. Adjusted threads to be ". getGlobal("threads") . "\n";
+}
+
+# finally configure for local system if we are not on grid
+initializeLocalSystem();
+
+print STDERR "Running with " . getGlobal("threads") . " threads and " . getGlobal("partitions") . " partitions\n";
+
+# try to find the tools we optionally support
+my $CA = getBinDirectory();
+my $JELLYFISH = getBinDirectory();
+my $AMOS = "$CA/../../../AMOS/bin/";
+my $BLASR = "$CA/../../../smrtanalysis/current/analysis/bin/";
+my $FALCON = "$CA/../../../FALCON-0.1.2/bin/";
+my $BOWTIE = "$CA/../../../bowtie2/";
+my $MHAP_VERSION = getGlobal("mhapVersion");
+my $MHAP_OVL = "$CA/../lib/java/mhap-$MHAP_VERSION.jar";
+my $JELLYFISH = "$CA/../../../jellyfish/bin/";
+my $wrk = makeAbsolute("");
+my $asm = "asm";
+my $scriptParams = getGlobal("gridOptionsScript");
+my $javaPath = "java";
+if (defined(getGlobal("javaPath"))) {
+ $javaPath = getGlobal("javaPath") . "/java";
+}
+my $pythonPath = "python";
+if (defined(getGlobal("pythonPath"))) {
+ $pythonPath = getGlobal("pythonPath") . "/python";
+}
+if (defined($scriptParams)) {
+ if (!defined(getGlobal("gridOptionsCorrection"))) {
+ setGlobal("gridOptionsCorrection", $scriptParams);
+ }
+}
+
+# check for previously existing file and do not overwrite
+if ( -e "$wrk/" . getGlobal("libraryname") . ".frg") {
+ print STDERR "Error: requested to output " . getGlobal("libraryname") . ".frg but file already exists. Will not overwrite.\n";
+goto assemble;
+}
+
+my $useGrid = getGlobal("useGrid");
+if (defined($useGrid)) {
+ setGlobal("submitToGrid", $useGrid);
+}
+elsif (defined(getGlobal("gridOptions"))) {
+ setGlobal("submitToGrid", 1);
+}
+
+my $caCNS = getGlobal("cnsConcurrency");
+if (defined($caCNS)) {
+ setGlobal("consensusConcurrency", $caCNS);
+}
+
+if ( ! -e $MHAP_OVL ) {
+ setGlobal("mhap", undef);
+}
+
+if (! -e "$JELLYFISH/jellyfish") {
+ if (-e "$CA/jellyfish") {
+ $JELLYFISH = $CA;
+ } else {
+ # try to use path
+ my $amosPath = `which jellyfish`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # blasr
+ $JELLYFISH = join '/', @t; # path to the assembler
+ }
+}
+
+if (! -e "$BLASR/blasr" || ! -e "$BLASR/sawriter") {
+ if (-e "$CA/blasr" && -e "$CA/sawriter") {
+ $BLASR = $CA;
+ } else {
+ # try to use path
+ my $amosPath = `which blasr`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # blasr
+ $BLASR = join '/', @t; # path to the assembler
+ }
+ # if we really can't find it just give up
+ if (! -e "$BLASR/blasr" || ! -e "$BLASR/sawriter") {
+ if (defined(getGlobal("blasr")) || (defined(getGlobal("longReads")) && getGlobal("longReads") == 1 && ! -e $MHAP_OVL)) {
+ my $msg = "\n";
+ $msg .= "One or both of blasr and sawriter were not found in your path.\n";
+ $msg .= "Please download from http://pacificbiosciences.github.com/DevNet/ and add to your path.\n";
+ $msg .= "PATH=$ENV{PATH}\n";
+ die $msg;
+ }
+ }
+
+ # check for consensus too
+ # make sure we have the pb consensus module available if it was requested
+ if (! -e "$BLASR/blasr" || ! -e "$BLASR/pbdagcon") {
+ if (! -e "$CA/blasr" || ! -e "$CA/pbdagcon") {
+ print STDERR "Warning: requested PBDAGON but either BLASR or pbdagcon executables were not found.\n";
+ setGlobal("pbcns", 0);
+ } else {
+ $BLASR=$CA;
+ }
+ }
+}
+
+if (! -e "$FALCON/falcon_sense") {
+ if (-e "$CA/falcon_sense") {
+ $FALCON = $CA;
+ } else {
+ # try to use path
+ my $amosPath = `which falcon_sense`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # falcon_sense
+ $FALCON = join '/', @t; # path to the assembler
+ }
+ # if we really can't find it just give up
+ if (! -e "$FALCON/falcon_sense") {
+ if (-e "$BLASR/pbdagcon" && defined(getGlobal("longReads")) && getGlobal("longReads") == 1) {
+ setGlobal("pbcns", 1);
+ } else {
+ setGlobal("pbcns", 0);
+ }
+ setGlobal("falconcns", 0);
+ }
+}
+
+my $blasrVersion = 1.2;
+if (-e "$BLASR/blasr") {
+ $blasrVersion = `$BLASR/blasr -version |tail -n 1`;
+ chomp $blasrVersion;
+}
+
+if (! -e "$AMOS/bank-transact" && (!defined(getGlobal("pbcns")) || getGlobal("pbcns") == 0)) {
+ # check ca path
+ if ( -e "$CA/bank-transact") {
+ $AMOS = $CA;
+ } else {
+ # try to use path
+ my $amosPath = `which bank-transact`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # bank-transact
+ $AMOS = join '/', @t; # path to the assembler
+ }
+ # if we really can't find it just give up
+ if (! -e "$AMOS/bank-transact") {
+ if (! -e "$BLASR/pbdagcon" && ! -e "$FALCON/falcon_sense") {
+ my $msg = "\n";
+ $msg .= "bank-transact not found in your path.\n";
+ $msg .= "Please download from http://amos.sf.net and add it to your path.\n";
+ $msg .= "PATH=$ENV{PATH}\n";
+ die $msg;
+ }
+ }
+}
+
+if (! -e "$BOWTIE/bowtie2-build") {
+ # try to use path
+ my $amosPath = `which bowtie2-build`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # bowtie2-build
+ $BOWTIE = join '/', @t; # path to the assembler
+
+ # if we really can't find it just give up
+ if (! -e "$BOWTIE/bowtie2-build") {
+ die "Bowtie2 binaries: bowtie not found in $BOWTIE\n" if defined(getGlobal("bowtie"));
+ }
+}
+
+print STDERR "********* Starting correction...\n CA: $CA\nAMOS:$AMOS\nSMRTportal:$BLASR ($blasrVersion)\nBowtie:$BOWTIE\n";
+print STDERR "******** Configuration Summary ********\n";
+foreach my $key (keys %global) {
+ if (length($key) < $TAB_SIZE) {
+ print STDERR "$key\t\t\t=";
+ } elsif (length($key) < 2*$TAB_SIZE) {
+ print STDERR "$key\t\t="
+ } else {
+ print STDERR "$key\t=";
+ }
+ print STDERR "\t" . $global{$key} . "\n";
+}
+print STDERR "****************\n";
+
+# get global variable values
+my $specFile = getGlobal("specFile");
+my $libraryname = getGlobal("libraryname");
+my $fastqFile = getGlobal("fastqFile");
+
+my $ovlErrorRate = getGlobal("utgErrorRate");
+my $ovlErrorLimit = getGlobal("utgErrorLimit");
+my $ovlMemory = getGlobal("ovlMemory");
+
+my $maxUncorrectedGap = getGlobal("maxUncorrectedGap");
+my $coverage = getGlobal("coverageCutoff");
+my $genomeSize = getGlobal("genomeSize");
+my $shortReads = getGlobal("shortReads");
+my $longReads = getGlobal("longReads");
+my $length = getGlobal("length");
+my $repeats = getGlobal("repeats");
+my $QV = getGlobal("QV");
+
+my $threads = getGlobal("threads");
+my $partitions = getGlobal("partitions");
+
+my $submitToGrid = getGlobal("submitToGrid");
+my $grid = getGlobal("gridOptions");
+my $gridOvl = getGlobal("gridOptionsOverlap");
+my $gridCorrection = getGlobal("gridOptionsCorrection");
+my $gridConsensus = getGlobal("gridOptionsConsensus");
+my $gridTaskID = getGlobal("gridEngineTaskID");
+
+my $bankPath = getGlobal("bankPath");
+
+my $consensusConcurrency = getGlobal("consensusConcurrency");
+
+my $cleanup = getGlobal("cleanup");
+
+my $cmd = "";
+
+if (defined(getGlobal("longReads")) && getGlobal("longReads") == 1) {
+ setGlobal("frgMinLen", 200);
+}
+
+if (! -e "$wrk/temp$libraryname") {
+ runCommand("$wrk", "mkdir temp$libraryname");
+ # generate the ca spec file, since we support additional options in the spec file, we need to strip those out before passing it to ca
+ updateSpecFile("$wrk/temp$libraryname/$libraryname.spec");
+}
+$specFile = "$wrk/temp$libraryname/$libraryname.spec";
+
+if (! -e "$wrk/temp$libraryname/$libraryname.frg") {
+ runCommand($wrk, "$CA/fastqToCA -libraryname $libraryname -type sanger -technology none -feature doConsensusCorrection 1 -reads " . makeAbsolute($fastqFile) . " > $wrk/temp$libraryname/$libraryname.frg");
+}
+
+# now that were ready, add the frg file info to the command line args
+my $commandLineOptionsNoFrgs = "-s $specFile $commandLineOptions";
+$commandLineOptions = "-s $specFile $commandLineOptions @fragFiles";
+
+# and we're off
+submitScript($wrk, $asm, undef) if (!runningOnGrid());
+my $gridName = (defined(getGlobal("gridJobName")) ? "_" . getGlobal("gridJobName") : "");
+if (! -d "$wrk/temp$libraryname/$asm.gkpStore") {
+ $cmd = "$CA/runCA ";
+ $cmd .= " -s $specFile -p $asm -d temp$libraryname ";
+ $cmd .= " stopAfter=initialStoreBuilding ";
+ $cmd .= " gridJobName=\"" . getGlobal("gridJobName") . "\" " if defined(getGlobal("gridJobName"));
+ $cmd .= " @fragFiles $wrk/temp$libraryname/$libraryname.frg";
+ runCommand($wrk, $cmd);
+}
+
+# make assumption that we correct using all libraries preceeding pacbio
+# figure out what number of libs we have and what lib is pacbio
+my $numLib = `$CA/gatekeeper -dumpinfo $wrk/temp$libraryname/$asm.gkpStore | grep LIB |awk '{print \$1}'`;
+chomp($numLib);
+
+my $minCorrectLib = 0;
+my $maxCorrectLib = 0;
+my $libToCorrect = 0;
+for (my $i = 1; $i <= $numLib; $i++) {
+ if (system("$CA/gatekeeper -isfeatureset $i doConsensusCorrection $wrk/temp$libraryname/$asm.gkpStore") == 0) {
+ if ($libToCorrect != 0) {
+ die("Error: only one PacBio library can be corrected. Both libraries $libToCorrect and $i are set to be corrected. Please double-check your input files and try again", undef);
+ }
+ $libToCorrect = $i;
+ } else {
+ if ($minCorrectLib == 0) { $minCorrectLib = $i; }
+ $maxCorrectLib = $i;
+ }
+}
+
+# check that we were able to find the libraries for correction as expected
+if ($libToCorrect <= $minCorrectLib) {
+ die("Error: The PacBio library $libToCorrect must be the last library loaded but it preceedes $minCorrectLib. Please double-check your input files and try again.", undef);
+}
+if (defined(getGlobal("longReads")) && getGlobal("longReads") == 1) {
+ $maxCorrectLib = $libToCorrect;
+ $minCorrectLib = $libToCorrect if ($minCorrectLib == 0);
+ $ovlErrorRate = 0.35;
+ $ovlErrorLimit = 6.5;
+} elsif (($libToCorrect == 0 || ($minCorrectLib == 0 && $maxCorrectLib == 0))) {
+ die ("Error: unable to find a library to correct. Please double-check your input files and try again.", undef);
+}
+print STDERR "Will be correcting PacBio library $libToCorrect with librarie[s] $minCorrectLib - $maxCorrectLib\n";
+my $totalBP = 0;
+my $totalNumToCorrect = 0;
+my $minCorrectID = 0;
+my $maxCorrectID = 0;
+# compute the number of bases in the gateeeker to be corrected
+open(F, "$CA/gatekeeper -dumpinfo $wrk/temp$libraryname/$asm.gkpStore |") or die("Couldn't open gatekeeper store", undef);
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @array = split '\s+';
+ if ($#array == 8 && $array[0] == $libToCorrect) {
+ $totalBP = $array[6];
+ $minCorrectID = $array[1];
+ $maxCorrectID = $array[2];
+ }
+}
+close(F);
+
+my $totalInputBP = 0;
+open(F, "$CA/gatekeeper -dumpinfo $wrk/temp$libraryname/$asm.gkpStore |") or die("Couldn't open gatekeeper store", undef);
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @array = split '\s+';
+ if ($#array == 8) {
+ if ($array[0] == $libToCorrect) {
+ $totalInputBP = $array[6];
+ }
+ }
+}
+close(F);
+
+my $seedLength = 0;
+if (! -e "$wrk/temp$libraryname/$asm.toerase.out") {
+ # here is where we filter for specified length as well as max of longest X of coverage for correction
+ # use the genome size/coverage, if available to subset the sequences
+ if ($genomeSize != 0 && getGlobal("maxCoverage") != 0) {
+ $totalBP = $genomeSize * getGlobal("maxCoverage");
+ }
+ runCommand($wrk, "$CA/gatekeeper -dumpfragments -invert -tabular -longestovermin $libToCorrect $length -longestlength $libToCorrect $totalBP $wrk/temp$libraryname/$asm.gkpStore 2> $wrk/temp$libraryname/$asm.seedlength |awk '{if (!(match(\$1, \"UID\") != 0 && length(\$1) == " . length("UID") . ")) { print \"frg uid \"\$1\" isdeleted 1\"; } }' > $wrk/temp$libraryname/$asm.toerase.uid");
+ runCommand($wrk, "$CA/gatekeeper --edit $wrk/temp$libraryname/$asm.toerase.uid $wrk/temp$libraryname/$asm.gkpStore > $wrk/temp$libraryname/$asm.toerase.out 2> $wrk/temp$libraryname/$asm.toerase.err");
+}
+open(F, "cat $wrk/temp$libraryname/$asm.seedlength |awk '{print \$NF}' |") or die ("Couldn't open seed length file", undef);
+$seedLength = <F>;
+chomp $seedLength;
+
+if ($seedLength < getGlobal("merSize")) {
+ $seedLength = getGlobal("merSize")+1;
+}
+close(F);
+
+# compute the number of bases left after our filtering gateeeker to be corrected
+my $totalCorrectingWith = 0;
+my $totalNumCorrectingWith = 0;
+open(F, "$CA/gatekeeper -dumpinfo temp$libraryname/$asm.gkpStore |") or die("Couldn't open gatekeeper store", undef);
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @array = split '\s+';
+ if ($#array == 8) {
+ if ($array[0] == $libToCorrect) {
+ $totalBP = $array[6];
+ $totalNumToCorrect = $array[3] + $array[4];
+ }
+ if ($minCorrectLib <= $array[0] && $array[0] <= $maxCorrectLib) {
+ $totalCorrectingWith += $array[6];
+ $totalNumCorrectingWith += $array[3] + $array[4];
+ }
+ }
+}
+close(F);
+
+if (defined(getGlobal("longReads")) && getGlobal("longReads") != 0) {
+ $totalCorrectingWith = $totalInputBP;
+}
+
+# since gkpStore loses the info on deleted fragments, pull it back up to make sure we got all bases for coverages
+if ( ! -e "$wrk/temp$libraryname/$asm.totalInputBP") {
+ open(F, "> $wrk/temp$libraryname/$asm.totalInputBP");
+ printf(F "%d\t%d\n", $totalCorrectingWith, $totalInputBP);
+ close(F);
+} else {
+ open(F, "< $wrk/temp$libraryname/$asm.totalInputBP");
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @array = split '\s+';
+ $totalCorrectingWith = int($array[0]);
+ $totalInputBP = int($array[1]);
+ }
+ close(F);
+}
+
+# check that we have good data
+if ($genomeSize != 0) {
+ print STDERR "Running with " . ($totalBP / $genomeSize) . "X (for genome size $genomeSize) of $libraryname sequences ($totalBP bp).\n";
+ print STDERR "Correcting with " . floor($totalCorrectingWith / $genomeSize) . "X sequences ($totalCorrectingWith bp).\n";
+} else {
+ print STDERR "Running with $totalBP bp for $libraryname.\n";
+ print STDERR "Correcting with $totalCorrectingWith bp.\n";
+}
+
+if ($totalBP == 0) {
+ print STDERR "Error: All $libraryname sequences were eliminated. Please check the length threshold of $length and your input file $fastqFile.\n";
+ runCommand("$wrk", "rm -rf temp$libraryname");
+ die;
+}
+if ($totalCorrectingWith == 0) {
+ print STDERR "Error: No high-accuracy sequences for correction. Please check your input FRG files " . join(", ", @fragFiles) . "\n";
+ runCommand("$wrk", "rm -rf temp$libraryname");
+ die;
+}
+if ($genomeSize != 0 && floor($totalCorrectingWith / $genomeSize) > $MAX_CORRECTION_COVERAGE && !(defined(getGlobal("longReads")) || getGlobal("longReads") == 0)) {
+ print STDERR "Warning: input a total of " . floor($totalCorrectingWith / $genomeSize) . " of high-accuracy coverage for correction. For best performance, at most $MAX_CORRECTION_COVERAGE is recommended.\n";
+ # could randomly subsample here
+}
+if ($genomeSize != 0 && defined(getGlobal("longReads")) && getGlobal("longReads") != 0 && ($totalCorrectingWith / $genomeSize) < $MIN_SELF_CORRECTION) {
+ print STDERR "Warning: performing self-correction with a total of " .floor($totalCorrectingWith / $genomeSize) . ". For best performance, at least $MIN_SELF_CORRECTION is recommended.\n";
+ setGlobal("sensitive", 1);
+}
+
+if (defined($longReads) && $longReads == 1) {
+ my $merSize = getGlobal("merSize");
+
+ if (defined(getGlobal("sensitive")) && getGlobal("sensitive") == 1){
+ if ( -e "$BLASR/blasr" && -e "$BLASR/pbdagcon" && (!defined(getGlobal("falconForce")) || getGlobal("falconForce") == 0)) {
+ setGlobal("falconcns", 0);
+ setGlobal("pbcns", 1);
+ }
+ if ( -e $MHAP_OVL && !defined(getGlobal("blasr"))) {
+ if (!defined(getGlobal("mhap"))) {
+ print STDERR "Warning: enabling MHAP overlapper (sensitive) to align long reads to long reads.\n";
+ setGlobal("mhap", "--weighted -k $merSize --num-hashes 768 --num-min-matches 2 --threshold 0.04 --filter-threshold 0.000005");
+ }
+ }
+ }
+ if ( -e $MHAP_OVL && !defined(getGlobal("blasr"))) {
+ if (!defined(getGlobal("mhap"))) {
+ print STDERR "Warning: enabling MHAP overlapper to align long reads to long reads.\n";
+ setGlobal("mhap", "--weighted -k $merSize --num-hashes 512 --num-min-matches 3 --threshold 0.04 --filter-threshold 0.000005");
+ }
+ # check java version and stop if not found or too old
+ if (defined(getGlobal("javaPath")) && ! -e $javaPath) {
+ die "Error: java is required to use MHAP and is not found in $javaPath specified.\n";
+ } elsif (!(defined(getGlobal("javaPath")))) {
+ my $java = `which java`;
+ chomp $java;
+ my @t = split '/', "$java";
+ pop @t; # java
+ my $jPath = join '/', @t;
+ if (! -e "$jPath/java") {
+ die "Error: java is required to use MHAP and is not found in $ENV{PATH}.\n";
+ }
+ }
+ open(F, "$javaPath -version 2>&1 |") or die("Couldn't find java in $ENV{PATH} or $javaPath. Specify javaPath=<path to java> or add it to your path.", undef);
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ if (m/java version/) {
+ my @vArray = split '\s+';
+ my $v = $vArray[2];
+ $v =~ s/\"//g;
+ @vArray = split '\.', $v;
+ if ($vArray[0] == 1 && $vArray[1] < 8) {
+ die "Error: java version 1.8 or newer is required for MHAP and found version $v. Please update and try again.\n";
+ }
+ }
+ }
+ close(F);
+ } elsif (-e "$BLASR/blasr") {
+ print STDERR "Warning: Blasr is required to align long reads to long reads. Switching blasr ON.\n";
+ setGlobal("blasr", defined(getGlobal("blasr")) ? getGlobal("blasr") . " -maxLCPLength 16" : "-minReadLength 200 -maxScore -1000 -maxLCPLength 16");
+ } else {
+ die "Aligning long reads requires either MHAP or BLASR and neither was available. Please either add SMRTportal to your path or download MHAP.";
+ }
+ if ((!defined(getGlobal("pbcns")) || getGlobal("pbcns") == 0) && -e "$FALCON/falcon_sense") {
+ setGlobal("falconcns", 1);
+ }
+} else {
+ # always default to blasr alignments when available
+ if (-e "$BLASR/blasr" && -e "$BLASR/sawriter") {
+ setGlobal("blasr", defined(getGlobal("blasr")) ? getGlobal("blasr") : "-noRefineAlign -advanceHalf -noSplitSubreads -minMatch 10 -minPctIdentity 70");
+ }
+ setGlobal("falconcns", 0);
+ if (-e "$BLASR/pbdagcon") {
+ setGlobal("pbcns", 1);
+ } else {
+ die "AMOS or PBDAGCON is required to call consensus and neither was available. Please add either SMRTportal or AMOS to your path and try again\n" if (! -e "$AMOS/bank-transact");
+ setGlobal("pbcns", 0);
+ }
+}
+
+if (-e $MHAP_OVL && defined(getGlobal("mhap"))) {
+ $MHAP_OVL="\$bin/../lib/java/mhap-$MHAP_VERSION.jar";
+}
+
+my $cutoffSpecified = 0;
+
+if (defined(getGlobal("blasr"))) {
+ if (getGlobal("blasr") =~ m/bestn/) {
+ $cutoffSpecified = 1;
+ print STDERR "Warning: cutoff manually specified, skip meryl run\n";
+ }
+}
+
+my $ignore = "";
+if (!defined(getGlobal("bowtie")) && $cutoffSpecified == 0 && !(defined(getGlobal("mhap")) && -e "$JELLYFISH/jellyfish")) {
+ # run correction up thorough meryl
+ if (! -e "$wrk/temp$libraryname/0-mercounts/$asm.nmers.ovl.fasta") {
+ $cmd = "$CA/runCA ";
+ $cmd .= "-s $specFile ";
+ $cmd .= "-p $asm -d temp$libraryname ";
+ $cmd .= "ovlHashLibrary=$libToCorrect ";
+ $cmd .= "ovlRefLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "ovlCheckLibrary=1 ";
+ $cmd .= "obtHashLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "obtRefLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "obtCheckLibrary=0 ";
+ $cmd .= "gridJobName=\"" . getGlobal("gridJobName") . "\" " if defined(getGlobal("gridJobName"));
+ $cmd .= "doOverlapBasedTrimming=0 stopAfter=meryl";
+ runCommand($wrk, $cmd);
+ }
+
+ # set the meryl threshold based on the genome size and coverage (if specified)
+ if (!defined(getGlobal("ovlMerThreshold")) || getGlobal("ovlMerThreshold") == "auto") {
+ # no threshold specified, check if the chosen one is OK
+ my $autoSetThreshold = `cat temp$libraryname/0-mercounts/*estMerThresh.out`;
+ chomp($autoSetThreshold);
+ setGlobal("ovlMerThreshold", $autoSetThreshold);
+
+ my $corrCov = (defined($genomeSize) && $genomeSize != 0 ? floor($totalCorrectingWith / $genomeSize) : 0);
+ my $pacCov = (defined($genomeSize) && $genomeSize != 0 ? floor($totalBP / $genomeSize) : 0);
+ my $maxCov = $pacCov + (defined($longReads) && $longReads == 1 ? 0 : $corrCov);
+
+ if ($autoSetThreshold < ($maxCov * $REPEAT_MULTIPLIER)) {
+ setGlobal("ovlMerThreshold", $maxCov * $REPEAT_MULTIPLIER);
+ unlink("$wrk/temp$libraryname/0-mercounts/$asm.nmers.ovl.fasta");
+ print STDERR "Resetting from auto threshold of $autoSetThreshold to be " . ($maxCov * 10) . "\n";
+ }
+ }
+}
+
+my $inMer = getGlobal("merSize");
+my $inFile = 0;
+if (glob("$wrk/*.$inMer.ignore" )) {
+ $inFile = `ls $wrk/*.$inMer.ignore |wc -l |awk '{print \$1}'`;
+}
+chomp $inFile;
+if ($inFile == 0 && !-e "$wrk/temp$libraryname/$asm.ignore" && -e "$JELLYFISH/jellyfish" && defined(getGlobal("mhap"))) {
+ my $f = makeAbsolute($fastqFile);
+ if ($f =~ /\.bz2$/i || $f =~ /\.zip$/i || $f =~ /\.gz$/i) {
+ die "Input fastq file $f is compressed. This is currently not supported, please uncompress the file and try again.";
+ }
+ $cmd = "$JELLYFISH/jellyfish count ";
+ $cmd .= " -m " . getGlobal("merSize") . " -s 120000000 -t " . getGlobal("merylThreads") . " -o $wrk/temp$libraryname/$asm.mers " . makeAbsolute($fastqFile);
+ runCommand("$wrk/temp$libraryname", $cmd);
+ if ( -e "$wrk/temp$libraryname/$asm.mers_0") {
+ $cmd = "$JELLYFISH/jellyfish merge -s 120000000 -o $wrk/temp$libraryname/$asm.mers $wrk/temp$libraryname/$asm.mers_*";
+ runCommand("$wrk/temp$libraryname", $cmd);
+ }
+
+ $cmd = "$JELLYFISH/jellyfish histo -t " . getGlobal("merylThreads") . " -f $wrk/temp$libraryname/$asm.mers > $wrk/temp$libraryname/$asm.hist";
+ runCommand("$wrk/temp$libraryname", $cmd);
+ my $total = 0;
+ my $runningSum = 0;
+ my $sum = `cat $wrk/temp$libraryname/$asm.hist |awk '{SUM+=\$NF; } END {print SUM}'`;
+ chomp $sum;
+ my $cut = `cat $wrk/temp$libraryname/$asm.hist |awk -v TOTAL=$sum '{SUM+=\$NF; if (SUM/TOTAL > 0.99) { print \$1; } }' |head -n 1`;
+ chomp $cut;
+ $cut++;
+
+ $cmd = "$JELLYFISH/jellyfish dump -c -t -L $cut $wrk/temp$libraryname/$asm.mers |awk -v TOTAL=$sum '{printf(\"\%s\\t\%0.10f\\t\%d\\t\%d\\n\", \$1, \$2/TOTAL, \$2, TOTAL)}' |sort -T . -rnk2> $wrk/temp$libraryname/$asm.ignore";
+ runCommand("$wrk/temp$libraryname", $cmd);
+ runCommand("$wrk/temp$libraryname", "rm $wrk/temp$libraryname/$asm.mers*");
+} elsif (!-e "$wrk/temp$libraryname/$asm.ignore" && defined(getGlobal("mhap"))) {
+ runCommand("$wrk/temp$libraryname", "cp $wrk/*.$inMer.ignore $wrk/temp$libraryname/$asm.ignore");
+}
+if ( -e "$wrk/temp$libraryname/$asm.ignore") {
+ $ignore = " -f $wrk/temp$libraryname/$asm.ignore";
+}
+
+if (! -d "$wrk/temp$libraryname/$asm.ovlStore") {
+ # run trimming if needed first
+ if (getGlobal("doOverlapBasedTrimming") != 0 && ! -e "$wrk/temp$libraryname/0-overlaptrim/overlaptrim.success") {
+ if (-e "$wrk/temp$libraryname/0-overlaptrim-overlap/overlap.sh") {
+ die("Overlap trimming failed. Remove the temporary directory temp$libraryname and try again.", undef);
+ }
+ $cmd = "-s $specFile ";
+ $cmd .= "-p $asm -d . ";
+ $cmd .= "ovlHashLibrary=$libToCorrect ";
+ $cmd .= "ovlRefLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "ovlCheckLibrary=1 ";
+ $cmd .= "obtHashLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "obtRefLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "obtCheckLibrary=0 ";
+ $cmd .= "gridJobName=\"" . getGlobal("gridJobName") . "\" " if defined(getGlobal("gridJobName"));
+ $cmd .= "gridEnginePropagateHold=\"pBcR_$asm$gridName\" ";
+ $cmd .= "stopAfter=overlapBasedTrimming";
+ if ($submitToGrid == 1) {
+ submitRunCA("$wrk/temp$libraryname", $asm, $cmd, "runCA_obt_$asm$gridName");
+ } else {
+ runCommand("$wrk/temp$libraryname", "$CA/runCA $cmd");
+ }
+ }
+
+ # now run the correction
+ my $ovlThreshold = getGlobal("ovlMerThreshold");
+
+ # when we were asked to not use CA's overlapper, first perform common options
+ if (defined(getGlobal("samFOFN")) || defined(getGlobal("blasr")) || defined(getGlobal("bowtie")) || defined(getGlobal("mhap"))) {
+ # check for sam tools
+ if (!-e "$CA/convertSamToCA") {
+ die("Error: request to use Blasr, Bowtie, or SAM files requires CA to be built with SAMTOOLS. Please rebuild CA and try again");
+ }
+ # create directories
+ runCommand($wrk, "mkdir $wrk/temp$libraryname/1-overlapper") if (! -d "$wrk/temp$libraryname/1-overlapper");
+
+ # first partition the data so we know how many jobs we have
+ my $cmd;
+
+ my $ovlHashBlockSize = $totalNumToCorrect;
+ my $ovlHashBlockLength = undef;
+ my $maxBatch = 1000;
+
+ if (defined(getGlobal("blasr"))) {
+ $ovlHashBlockLength = getGlobal("ovlHashBlockLength");
+ $ovlHashBlockSize = getGlobal("ovlHashBlockSize");
+ }
+ my $ovlRefBlockSize = getGlobal("ovlRefBlockSize");
+ my $ovlRefBlockLength = getGlobal("ovlRefBlockLength");
+ if (defined(getGlobal("mhap"))) {
+ setGlobal("blasr", undef);
+ setGlobal("bowtie", undef);
+ $gridOvl = getGlobal("gridOptionsMhap") if defined (getGlobal("gridOptionsMhap"));
+ $ovlRefBlockSize = getGlobal("mhapOvlRefBlockSize");
+ if (!defined($ovlRefBlockSize) || $ovlRefBlockSize == 0) {
+ my $memFactor = getGlobal("ovlMemory") / 16;
+ my @mhapSplit = split(/\s+/, getGlobal("mhap"));
+ my $numHashes = 1256;
+ for my $pos (0 .. $#mhapSplit) {
+ if ($mhapSplit[$pos] =~ m/num-hashes/) {
+ $numHashes = int($mhapSplit[$pos+1]);
+ last;
+ }
+ }
+ my $loadFactor = getGlobal("mhapLoadFactor");
+ if ($numHashes <= 786) { $loadFactor = $loadFactor * 2; }
+ $ovlRefBlockSize = floor($memFactor * $loadFactor);
+ }
+ $ovlRefBlockLength = undef;
+ $ovlHashBlockLength = undef;
+ $ovlHashBlockSize = $ovlRefBlockSize;
+ }
+
+ if (($ovlRefBlockSize > 0) && ($ovlRefBlockLength > 0)) {
+ die("can't set both ovlRefBlockSize and ovlRefBlockLength", undef);
+ }
+
+ my @job = ();
+ if (!defined(getGlobal("mhap"))) {
+ $cmd = "$CA/overlap_partition \\\n";
+ $cmd .= " -g $wrk/temp$libraryname/$asm.gkpStore \\\n";
+ $cmd .= " -bs $ovlHashBlockSize \\\n" if (defined($ovlHashBlockSize));
+ $cmd .= " -bl $ovlHashBlockLength \\\n" if (defined($ovlHashBlockLength));
+ $cmd .= " -rs $ovlRefBlockSize \\\n" if (defined($ovlRefBlockSize));
+ $cmd .= " -rl $ovlRefBlockLength \\\n" if (defined($ovlRefBlockLength));
+ $cmd .= " -H $libToCorrect \\\n";
+ $cmd .= " -R $minCorrectLib-$maxCorrectLib \\\n";
+ $cmd .= " -C \\\n";
+ $cmd .= " -o $wrk/temp$libraryname/1-overlapper";
+ runCommand($wrk, $cmd);
+ open(F, "< $wrk/temp$libraryname/1-overlapper/ovlopt") or die("failed partition for overlapper: no ovlopt file found", undef);
+ @job = <F>;
+ close(F);
+ } else {
+ open(BAT, "> $wrk/temp$libraryname/1-overlapper/ovlbat") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlbat", undef);
+ open(JOB, "> $wrk/temp$libraryname/1-overlapper/ovljob") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovljob", undef);
+ open(OPT, "> $wrk/temp$libraryname/1-overlapper/ovlopt") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlopt", undef);
+
+ my $numJobs = 1;
+ my $numInBatch = 1;
+ my $numBatch = 1;
+ my $maxRefsToCompare = floor($totalNumCorrectingWith / $ovlRefBlockSize / 2);
+ if ($maxRefsToCompare == 0) {
+ $maxRefsToCompare = floor($totalNumCorrectingWith / $ovlRefBlockSize);
+ }
+ my $maxToCompare = $maxRefsToCompare * $ovlRefBlockSize;
+ for (my $i = 1; $i <= $totalNumCorrectingWith; $i+=$ovlRefBlockSize) {
+ if ($numInBatch > $maxBatch) {
+ $numBatch++;
+ $numInBatch=1;
+ }
+ my $max = $i+$ovlHashBlockSize-1;
+ my $next = $max+1;
+ if ($max > $totalNumCorrectingWith) { $max = $totalNumCorrectingWith; }
+
+ my $limit = $next;
+ if (($totalNumCorrectingWith - $next + 1) > $maxToCompare) {
+ # split into two jobs for better load balancing
+ push(@job, "-h $i-$max -r $i-$max");
+ printf(BAT "%03d\n", $numBatch);
+ printf(JOB "%06d\n", $numJobs);
+ $numJobs++;
+ $numInBatch++;
+ $limit = $next + $maxToCompare - 1;
+ if ($next > $totalNumCorrectingWith) { $next = $totalNumCorrectingWith+1; }
+ if ($limit > $totalNumCorrectingWith) { $limit = $totalNumCorrectingWith+1; }
+ print OPT "-h $i-$max -r $next-$limit\n";
+ $limit++;
+ }
+ push(@job, "-h $i-$max -r $i-$max");
+ printf(BAT "%03d\n", $numBatch);
+ printf(JOB "%06d\n", $numJobs);
+ if ($next > $totalNumCorrectingWith) { $next = $totalNumCorrectingWith+1; }
+ if ($limit > $totalNumCorrectingWith) { $limit = $totalNumCorrectingWith+1; }
+ print OPT "-h $i-$max -r $limit-$totalNumCorrectingWith\n";
+ $numJobs++;
+ $numInBatch++;
+ }
+ close(BAT);
+ close(JOB);
+ close(OPT);
+ }
+
+ my $blasrThreshold = 0;
+ my $ovlThreads = getGlobal("ovlThreads");
+ my $numOvlJobs = 0;
+ my $numIndicies = 1;
+ my $blasrOpts = getGlobal("blasr");
+ my $bowtieOpts = getGlobal("bowtie");
+ my $suffix = "sam";
+
+ $blasrVersion="1.3.1.116174";
+ # SMRTportal 1.4 changed parameters so update our blasr options
+ if ($blasrVersion >= 1.3.1.116174) {
+ $blasrOpts =~ s/-ignoreQuality//g;
+ }
+
+ if (defined(getGlobal("samFOFN"))) {
+ my $fofn = getGlobal("samFOFN");
+ $numOvlJobs = 0;
+ } else {
+ $numOvlJobs = $#job + 1;
+ }
+
+ my $wrkDir = $wrk;
+ if (defined(getGlobal("localStaging"))) {
+ $wrkDir = getGlobal("localStaging") . "/\$USER/";
+ }
+
+ if (!-e "$wrk/temp$libraryname/1-overlapper/ovlpreplocal.sh" && defined(getGlobal("localStaging"))) {
+ open F, "> $wrk/temp$libraryname/1-overlapper/ovlpreplocal.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/ovlpreplocal.sh'");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F "jobid=\$$gridTaskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F "jobid=\$1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if test x\$jobid = x; then\n";
+ print F " echo Error: I need $gridTaskID set, or a job index on the command line\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "\n";
+ my $wrkDir = $wrkDir = getGlobal("localStaging") . "/\$USER/";
+ print F "rm -rf $wrkDir/temp$libraryname/1-overlapper/\n";
+ close(F);
+ chmod 0755, "$wrk/temp$libraryname/1-overlapper/ovlpreplocal.sh";
+
+ if ($submitToGrid == 1) {
+ my $gridName = "pBcR_ovlpreplocal_$asm$gridName";
+ my $jobName = getGridArrayName($gridName, $numOvlJobs);
+ my $arrayOpt = getGridArrayOption($gridName, $numOvlJobs);
+ submitBatchJobs($wrk, $asm, "$submitCommand $grid $gridOvl $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/1-overlapper/ovlpreplocal.sh", $jobName);
+ } else {
+ for (my $i = 1; $i <= $numOvlJobs; $i++) {
+ schedulerSubmit("$wrk/temp$libraryname/1-overlapper/ovlpreplocal.sh $i");
+ }
+ schedulerSetNumberOfProcesses(getGlobal("ovlConcurrency"));
+ schedulerFinish();
+ }
+ }
+
+ if (-e "$wrk/temp$libraryname/1-overlapper/ovlpindex") {
+ open(F, "< $wrk/temp$libraryname/1-overlapper/ovlpindex") or die("failed partition for overlapper: no ovlopt file found", undef);
+ my @indicies = <F>;
+ close(F);
+ $numIndicies += $#indicies + 1 ;
+ }
+
+ if (-e "$wrk/temp$libraryname/$asm.ovlStore.BUILDING") {
+ die "Error: overlap store building failed. See $wrk/temp$libraryname/$asm.ovlStore.err for more details";
+ }
+ if (-e "$wrk/temp$libraryname/1-overlapper/overlap.sh") {
+ goto checkforerror;
+ }
+ if (-e "$wrk/temp$libraryname/1-overlapper/ovlprep.sh") {
+ goto checkovlprep;
+ }
+
+ # dump gatekeeper store and print eid to iid, add to it the gkpStore fastq file into eidToIID and lengths
+ runCommand("$wrk/temp$libraryname", "$CA/gatekeeper -dumpfragments -tabular $asm.gkpStore |awk '{print \$1\"\\t\"\$2}' > $asm.eidToIID");
+ runCommand("$wrk/temp$libraryname", "$CA/gatekeeper -dumpfragments -tabular $asm.gkpStore |awk '{print \$2\"\\t\"\$10}' > $asm.iidToLen");
+ open F, "> $wrk/temp$libraryname/1-overlapper/ovlindex" or die ("can't open '$wrk/temp$libraryname/1-overlapper/ovlindex");
+
+
+ # now do specific steps (either import SAMs or run blasr)
+ if (defined(getGlobal("samFOFN"))) {
+ my $fofn = getGlobal("samFOFN");
+
+ # get the external UIDs from the input file rather than gatekeeper since these won't match our externally-generated SAM
+ runCommand("$wrk/temp$libraryname", "cat $asm.gkpStore.fastqUIDmap | awk '{if (NF > 3) { print \$NF\"\\t\"\$(NF-1); } print \$3\"\\t\"\$2; }' >> $asm.eidToIID");
+
+ # convert sam to ovl and put it into the overlap directory
+ my $batch = 1;
+
+ open(BAT, "> $wrk/temp$libraryname/1-overlapper/ovlbat") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlbat", undef);
+ open(JOB, "> $wrk/temp$libraryname/1-overlapper/ovljob") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovljob", undef);
+ open(OPT, "> $wrk/temp$libraryname/1-overlapper/ovlopt") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlopt", undef);
+ open(SA, "> $wrk/temp$libraryname/1-overlapper/ovlindex") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlindex", undef);
+ $numIndicies++;
+
+ open(F, "< $fofn") or die("Couldn't open '$fofn'", undef);
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+ if (! -e $_) {
+ print STDERR "Warning: could not open SAM file $_, skipping\n";
+ next;
+ } elsif ($_ =~ /\.sam$/) {
+ $suffix = "sam";
+ } elsif ($_ =~ /\.sam\.gz$/) {
+ $suffix = "sam.gz";
+ } elsif ($_ =~ /\.sam\.bz2$/) {
+ $suffix = "sam.bz2";
+ }
+
+ $numOvlJobs++;
+ if ($numOvlJobs > $maxBatch) {
+ $batch++;
+ }
+ my $batchName = sprintf("%03d", $batch);
+ my $jobName = sprintf("%06d", $numOvlJobs);
+ print BAT "$batchName\n";
+ print OPT "-r 1-$totalNumToCorrect -l 1-$totalNumToCorrect\n";
+ print JOB "$jobName\n";
+ print SA "1\n";
+ runCommand("$wrk/temp$libraryname/1-overlapper", "mkdir -p $batchName");
+ runCommand("$wrk/temp$libraryname/1-overlapper/$batchName", "unlink $wrk/temp$libraryname/1-overlapper/$batchName/$jobName.$suffix") if (-e "$wrk/temp$libraryname/1-overlapper/$batchName/$jobName.$suffix");
+ runCommand("$wrk/temp$libraryname/1-overlapper/$batchName", "cp $_ $wrk/temp$libraryname/1-overlapper/$batchName/$jobName.$suffix");
+ }
+ close(F);
+ close(BAT);
+ close(OPT);
+ close(JOB);
+ close(SA);
+ } else {
+ if (!defined(getGlobal("mhap"))) {
+ for (my $i = $minCorrectLib; $i <= $maxCorrectLib; $i++) {
+ runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfasta " . $i . "_subset " . (defined($longReads) && $longReads == 1 && $i == $libToCorrect ? " -allreads -allbases" : "") . " -randomsubset $i 0.01 $wrk/temp$libraryname/$asm.gkpStore");
+ }
+ runCommand("$wrk/temp$libraryname/1-overlapper", "cat `ls [0-9]*_subset.fasta` > correct_subset.fasta");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "rm `ls [0-9]*_subset.fasta*`");
+ }
+
+ # read in ovl options and partition data from the gatekeeper store
+ my $i = 1;
+ my $part = 1;
+ my %longIndex = {};
+ my %correctIndex = {};
+ my $numPrepJobs = 0;
+
+ open(PREP, "> $wrk/temp$libraryname/1-overlapper/ovlprep") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlprep", undef);
+ open(PINDEX, "> $wrk/temp$libraryname/1-overlapper/ovlpindex") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlpindex", undef);
+
+ foreach my $ovlJob (@job) {
+ $ovlJob =~ s/^\s+//;
+ $ovlJob =~ s/\s+$//;
+ my @values=split(/\s+/, $ovlJob);
+ my $indexIds = "";
+ my $correctIds = "";
+
+ if (!defined($longIndex{$values[1]})) {
+ # first the hash sequences
+ my @startend=split(/-/, $values[1]);
+ $indexIds = "-b $startend[0] -e $startend[1]";
+ if (!defined(getGlobal("mhap"))) {
+ print PREP "-randomsubset $libToCorrect 1 $indexIds\n";
+ print PINDEX "long_reads_part\t$numIndicies\n";
+ $numPrepJobs++;
+ }
+ $numIndicies++;
+ $longIndex{$values[1]} = 1;
+ }
+
+ if (!defined($correctIndex{$values[3]})) {
+ # now the correction sequences
+ my @startend=split(/-/, $values[3]);
+ $correctIds="-b $startend[0] -e $startend[1]";
+ my $dumpAll = 0;
+ my $start = $startend[0] > $minCorrectID ? $startend[0] : $minCorrectID;
+ my $end = $startend[1] < $maxCorrectID ? $startend[1] : $maxCorrectID;
+ if ($end - $start >= 0) {
+ $dumpAll = 1;
+ }
+ if (defined($longReads) && $longReads == 1 && $dumpAll) {
+ if (!defined(getGlobal("mhap"))) {
+ print PREP " -allreads -allbases $correctIds\n";
+ } else {
+ print PREP " -allreads -allbases $correctIds\n";
+ }
+ } else {
+ print PREP " -randomsubset 0 1 $correctIds\n";
+ }
+ print PINDEX "correct_reads_part\t$i\n";
+ $numPrepJobs++;
+ $i++;
+ $correctIndex{$values[3]} = 1;
+ }
+
+ print F $numIndicies-1 . "\n";
+ $part++;
+ }
+ close(PREP);
+ close(PINDEX);
+ close(F);
+
+ open F, "> $wrk/temp$libraryname/1-overlapper/ovlprep.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/ovlprep.sh'");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F "jobid=\$$gridTaskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F "jobid=\$1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if test x\$jobid = x; then\n";
+ print F " echo Error: I need $gridTaskID set, or a job index on the command line\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "job=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlprep | tail -n 1`\n";
+ print F "index=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlpindex |tail -n 1`\n";
+ print F "name=`echo \"\$index\" |awk '{print \$1}'`\n";
+ print F "id=`echo \"\$index\" |awk '{print \$2}'`\n";
+ print F "\n";
+ if (defined(getGlobal("localStaging"))) {
+ print F "mkdir -p $wrkDir/temp$libraryname/1-overlapper/\n";
+ }
+ print F " \$bin/gatekeeper -dumpfasta $wrkDir/temp$libraryname/1-overlapper/\$name\$id \$job $wrk/temp$libraryname/$asm.gkpStore\n";
+ if (defined(getGlobal("blasr"))) {
+ print F "if [ \$name == \"long_reads_part\" ]; then\n";
+ print F " /usr/bin/time $BLASR/sawriter $wrkDir/temp$libraryname/1-overlapper/\$name\$id.sa $wrkDir/temp$libraryname/1-overlapper/\$name\$id.fasta > $wrk/temp$libraryname/1-overlapper/\$jobid.hash.err 2>&1 \n";
+ print F "fi\n";
+ }
+ if (!defined(getGlobal("mhap"))) {
+ print F "\$bin/gatekeeper -dumpfragments -tabular \$job $wrk/temp$libraryname/$asm.gkpStore |awk '{print \$1\"\\t\"\$2}' > $wrk/temp$libraryname/1-overlapper/\$name\$id.eidToIID\n";
+ print F "\$bin/gatekeeper -dumpfragments -tabular \$job $wrk/temp$libraryname/$asm.gkpStore |awk '{print \$2\"\\t\"\$10}' > $wrk/temp$libraryname/1-overlapper/\$name\$id.iidToLen\n";
+ } elsif (!defined(getGlobal("localStaging")) && defined(getGlobal("mhapPrecompute"))) {
+ print F "/usr/bin/time $javaPath -server -Xmx" . $ovlMemory . "g -jar $MHAP_OVL " . getGlobal("mhap") . " --min-store-length " . ($seedLength-1) . " --num-threads $ovlThreads" . " $ignore -p $wrkDir/temp$libraryname/1-overlapper/\$name\$id.fasta -q $wrkDir/temp$libraryname/1-overlapper/ > $wrk/temp$libraryname/1-overlapper/\$jobid.hash.err 2>&1\n";
+ print F "rm -f $wrkDir/temp$libraryname/1-overlapper/\$name\$id.fasta\n";
+ }
+ # remove qual files they are not currently used
+ print F "rm -f $wrkDir/temp$libraryname/1-overlapper/\$name\$id.fasta.q*\n";
+
+ if (defined(getGlobal("localStaging"))) {
+ if (!defined(getGlobal("mhap"))) {
+ print F "mv $wrkDir/temp$libraryname/1-overlapper/\$name\$id.fasta $wrk/temp$libraryname/1-overlapper/\n";
+ if (defined(getGlobal("blasr"))) {
+ print F "mv $wrkDir/temp$libraryname/1-overlapper/\$name\$id.sa $wrk/temp$libraryname/1-overlapper/\n";
+ }
+ } else {
+ print F "if [ -e \"$wrkDir/temp$libraryname/1-overlapper/\$name\$id.dat\" ]; then\n";
+ print F "mv $wrkDir/temp$libraryname/1-overlapper/\$name\$id.dat $wrk/temp$libraryname/1-overlapper/\n";
+ print F "else\n";
+ print F "mv $wrkDir/temp$libraryname/1-overlapper/\$name\$id.fasta $wrk/temp$libraryname/1-overlapper/\n";
+ print F "fi\n";
+ }
+ }
+
+ close(F);
+ chmod 0755, "$wrk/temp$libraryname/1-overlapper/ovlprep.sh";
+
+ if ($submitToGrid == 1) {
+ my $gridName = "pBcR_ovlprep_$asm$gridName";
+ my $jobName = getGridArrayName($gridName, $numPrepJobs);
+ my $arrayOpt = getGridArrayOption($gridName, $numPrepJobs);
+ submitBatchJobs($wrk, $asm, "$submitCommand $grid $gridOvl $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/1-overlapper/ovlprep.sh", $jobName);
+ } else {
+ for (my $i = 1; $i <= $numPrepJobs; $i++) {
+ schedulerSubmit("$wrk/temp$libraryname/1-overlapper/ovlprep.sh $i");
+ }
+ schedulerSetNumberOfProcesses(getGlobal("ovlConcurrency"));
+ schedulerFinish();
+ }
+ checkovlprep:
+ my $failedJobs = 0;
+ my $failureMessage = "";
+ open(B, "< $wrk/temp$libraryname/1-overlapper/ovlpindex") or die("failed to open '$wrk/temp$libraryname/1-overlapper/ovlpindex'", undef);
+ while (!eof(B)) {
+ my $b = <B>; chomp $b;
+ my ($name, $id) = split '\s+', $b;
+ my $fileName = "$wrk/temp$libraryname/1-overlapper/$name$id";
+
+ if (defined(getGlobal("mhap"))) {
+ if ((! -e "$fileName.dat") && ! -e "$fileName.fasta") {
+ $failureMessage .= "ERROR: Overlap prep job $wrk/temp$libraryname/1-overlapper/$b FAILED.\n";
+ $failedJobs++;
+ }
+ } else {
+ if (defined(getGlobal("blasr")) && $fileName =~ m/long_reads/) {
+ if ((! -e "$fileName.sa")) {
+ $failureMessage .= "ERROR: Overlap prep job $wrk/temp$libraryname/1-overlapper/$b FAILED.\n";
+ $failedJobs++;
+ }
+ }
+ if (! -e "$fileName.fasta") {
+ $failureMessage .= "ERROR: Overlap prep job $wrk/temp$libraryname/1-overlapper/$b FAILED.\n";
+ $failedJobs++;
+ }
+ }
+ }
+ close(B);
+ $failureMessage .= "\n$failedJobs overlap partitioning jobs failed.";
+ if ($failedJobs > 0) {
+ die "$failureMessage\n";
+ }
+
+
+ if (defined(getGlobal("blasr")) && $cutoffSpecified == 0) {
+ runCommand("$wrk/temp$libraryname/1-overlapper", "ln -s long_reads_part1.fasta long_reads.fasta");
+ # run with best limit set to kmer with a small (0.5-1% of the data)
+ my $ovlThreads = getGlobal("ovlThreads");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "ln -s long_reads_part1.sa long.sa");
+
+ open F, "> $wrk/temp$libraryname/1-overlapper/pickNBest.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/pickNBest.sh");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+
+ if (-e "$BLASR/../../etc/setup.sh") {
+ print F "source $BLASR/../../etc/setup.sh && /usr/bin/time $BLASR/blasr -sa long.sa correct_subset.fasta long_reads.fasta -nproc $ovlThreads -bestn $ovlThreshold $blasrOpts -sam -out subset.sam";
+ } else {
+ print F "/usr/bin/time $BLASR/blasr -sa long.sa correct_subset.fasta long_reads.fasta -nproc $ovlThreads -bestn $ovlThreshold $blasrOpts -sam -out subset.sam";
+ }
+ close(F);
+ chmod 0755, "$wrk/temp$libraryname/1-overlapper/pickNBest.sh";
+ runCommand("$wrk/temp$libraryname/1-overlapper", "$wrk/temp$libraryname/1-overlapper/pickNBest.sh");
+
+ runCommand("$wrk/temp$libraryname/1-overlapper", "cat subset.sam |grep -v \"@\" | awk '{print \$1}' |sort -T . |uniq -c |awk '{print \$1}' > subset.hist");
+
+ # pick threshold as 2sd (95% of the bases)
+ $blasrThreshold = pickMappingThreshold("$wrk/temp$libraryname/1-overlapper/subset.hist", ($blasrOpts =~ m/maxLCPLength/));
+ runCommand("$wrk/temp$libraryname/1-overlapper", "unlink subset.sam");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "unlink correct_subset.fasta");
+ } elsif (defined(getGlobal("bowtie"))) {
+ # run with best limit set to all with a small (0.5-1% of the data)
+ my $ovlThreads = getGlobal("ovlThreads");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "$BOWTIE/bowtie2-build -f long_reads.fasta long.sa");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "$BOWTIE/bowtie2 -x long.sa -f correct_subset.fasta -p $ovlThreads -a $bowtieOpts -S subset.sam");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "cat subset.sam |grep -v \"@\" | awk '{print \$1}' |sort -T . |uniq -c |awk '{print \$1}' > subset.hist");
+
+ # pick threshold as 2sd (95% of the bases)
+ $blasrThreshold = pickMappingThreshold("$wrk/temp$libraryname/1-overlapper/subset.hist", 0);
+ runCommand("$wrk/temp$libraryname/1-overlapper", "unlink subset.sam");
+ runCommand("$wrk/temp$libraryname/1-overlapper", "unlink correct_subset.fasta");
+ }
+ }
+ close(F);
+
+ # now that we have our parameters, create a run job
+ open F, "> $wrk/temp$libraryname/1-overlapper/overlap.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/overlap.sh'");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F "jobid=\$$gridTaskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F "jobid=\$1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if test x\$jobid = x; then\n";
+ print F " echo Error: I need $gridTaskID set, or a job index on the command line\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "bat=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlbat | tail -n 1`\n";
+ print F "job=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovljob | tail -n 1`\n";
+ print F "opt=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlopt | tail -n 1`\n";
+ print F "sa=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlindex | tail -n 1`\n";
+ print F "\n";
+ print F "if [ ! -d $wrk/temp$libraryname/1-overlapper/\$bat ]; then\n";
+ print F " mkdir $wrk/temp$libraryname/1-overlapper/\$bat\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ -e $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovb ]; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ x\$bat = x ]; then\n";
+ print F " echo Error: Job index out of range.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "options=(\$opt)\n";
+ print F "hashParams=(\$(echo \${options[1]} | tr \"-\" \"\\n\")) \n";
+ print F "hashStart=\${hashParams[0]} \n";
+ print F "hashStart=\$((\$hashStart - 1))\n";
+ print F "hashEnd=\${hashParams[1]} \n";
+ print F "params=(\$(echo \${options[3]} | tr \"-\" \"\\n\")) \n";
+ print F "numTotalJobs=$numOvlJobs\n";
+ print F "numSA=" , ($numIndicies-1) , "\n";
+ print F "numOvlJobs=\$((\$numTotalJobs/\$numSA))\n";
+ print F "zeroJobId=\$((\$((\$jobid - 1)) % \$numOvlJobs))\n";
+ print F "oneJobId=\$((\$zeroJobId+1))\n";
+ print F "start=\${params[0]} \n";
+ print F "start=\$((\$start - 1))\n";
+ print F "end=\${params[1]} \n";
+ print F "total=\$((\$end - \$start))\n";
+ print F "refOffset=\$((\$start - \$hashEnd))\n";
+ print F "echo \"Running partition \$job with options \$opt start \$start end \$end total \$total zero job \$zeroJobId and stride \$numOvlJobs\"\n";
+ print F "\n";
+
+ my $wrkDir = $wrk;
+ my $suffix = "dat";
+ if (defined(getGlobal("mhap"))) {
+ if (defined(getGlobal("localStaging"))) {
+ $wrkDir = getGlobal("localStaging") . "/\$USER/";
+ $suffix = "fasta";
+ print F "mkdir -p $wrkDir/temp$libraryname/1-overlapper/\n";
+ } elsif (!defined(getGlobal("mhapPrecompute"))) {
+ $suffix = "fasta";
+ }
+ } else {
+ $suffix = "sam";
+ }
+ if (defined(getGlobal("blasr"))) {
+ if (-e "$BLASR/../../etc/setup.sh") {
+ print F " source $BLASR/../../etc/setup.sh \n";
+ }
+ if (defined(getGlobal("localStaging"))) {
+ print F " cp $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.sa $wrkDir\n";
+ print F " cp $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.fasta $wrkDir\n";
+ print F " cp $wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.fasta $wrkDir\n"
+ }
+ print F " /usr/bin/time $BLASR/blasr \\\n";
+ print F " -sa $wrkDir/temp$libraryname/1-overlapper/long_reads_part\$sa.sa $wrkDir/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.fasta \\\n";
+ print F " $wrkDir/temp$libraryname/1-overlapper/long_reads_part\$sa.fasta \\\n";
+ print F " -nproc $ovlThreads \\\n";
+ if ($cutoffSpecified == 0) {
+ print F " -bestn $blasrThreshold \\\n";
+ }
+ print F " $blasrOpts \\\n";
+ print F " -sam -out $wrkDir/temp$libraryname/1-overlapper/\$bat/\$job.sam \\\n";
+ print F " > $wrk/temp$libraryname/1-overlapper/\$jobid.out 2>&1 \\\n";
+ print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success\n";
+ print F " if test -e $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success ; then\n";
+ print F " echo Blasr completed.\n";
+ print F " else\n";
+ print F " echo Blasr failed.\n";
+ print F " tail $wrk/temp$libraryname/1-overlapper/\$jobid.out && exit\n";
+ print F " fi\n";
+ } elsif (defined(getGlobal("bowtie"))) {
+ print F " $BOWTIE/bowtie2 \\\n";
+ print F " -s \$start -u \$total \\\n";
+ print F " -x $wrk/temp$libraryname/1-overlapper/long.sa -f $wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.fasta \\\n";
+ print F " -p $ovlThreads -k $blasrThreshold \\\n";
+ print F " $bowtieOpts \\\n";
+ print F " -S $wrk/temp$libraryname/1-overlapper/\$bat/\$job.sam \\\n";
+ print F " > $wrk/temp$libraryname/1-overlapper/\$jobid.out 2>&1 \\\n";
+ print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success\n";
+ print F " if test -e $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success ; then\n";
+ print F " echo Bowtie completed.\n";
+ print F " else\n";
+ print F " echo Bowtie failed.\n";
+ print F " tail $wrk/temp$libraryname/1-overlapper/\$jobid.out && exit\n";
+ print F " fi\n";
+ } elsif (defined(getGlobal("mhap"))) {
+ my $javaCmd = "/usr/bin/time $javaPath -server -Xmx" . $ovlMemory . "g -jar $MHAP_OVL " . getGlobal("mhap") . " --min-store-length " . ($seedLength-1) . " --num-threads $ovlThreads" . " $ignore ";
+ my $convertCmd = "";
+
+ if ((defined(getGlobal("onlyContained")) && getGlobal("onlyContained") == 1) || !defined($longReads) || $longReads == 0) {
+ $convertCmd = "awk -v REF_OFFSET=\$refOffset -v OFFSET=\$hashStart '{if (\$5 == 0 && \$9 == 0) { ORI=\"N\"; } if (\$5 == 0 && \$9 == 1) { ORI=\"I\"; } if (\$8 <= \$12 && (\$7-\$6) / \$8 > 0.9) { if (ORI == \"N\") { print \$1+OFFSET+REF_OFFSET\"\\t\"\$2+OFFSET\"\\t\"ORI\"\\t\"(-1*\$10)\"\\t\"(\$12-\$11)\"\\t\"\$3/5\"\\t\"\$3/5; } else { print \$1+OFFSET+REF_OFFSET\"\\t\"\$2+OFFSET\"\\t\"ORI\"\\t\"(-1*(\$12-\$11))\"\\t\"\$10\"\\t\"\$3/5\"\\t\"\$3/5;} } else if (\$8 > \$12 && ( [...]
+ } else {
+ $convertCmd = "awk -v REF_OFFSET=\$refOffset -v OFFSET=\$hashStart '{if (\$5 == 0 && \$9 == 0) { print \$1+OFFSET+REF_OFFSET\"\\t\"\$2+OFFSET\"\\tf\\t\"\$6\"\\t\"\$7\"\\t\"\$10\"\\t\"\$11\"\\t\"\$3/5; } if (\$5 == 0 && \$9 == 1) { print \$1+OFFSET+REF_OFFSET\"\\t\"\$2+OFFSET\"\\tr\\t\"\$6\"\\t\"\$7\"\\t\"(\$11)\"\\t\"(\$10)\"\\t\"\$3/5; }}' | \$bin/convertOverlap -obt > $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovb";
+ }
+ print F " startIndex=`perl -w -e \"use POSIX; print floor(\$start/$ovlRefBlockSize)\"`\n";
+ print F " endIndex=`perl -w -e \"use POSIX; print ceil(\$end/$ovlRefBlockSize)\"`\n";
+ if (defined(getGlobal("localStaging"))) {
+ print F " if [ ! -e $wrkDir/temp$libraryname/1-overlapper/correct_reads_part\$sa.fasta ]; then\n";
+ print F " echo \"Copying file $wrk/temp$libraryname/1-overlapper/correct_reads_part\$sa.fasta\"\n";
+ print F " cp $wrk/temp$libraryname/1-overlapper/correct_reads_part\$sa.fasta $wrkDir/temp$libraryname/1-overlapper/correct_reads_part\$sa.fasta\n";
+ print F "fi\n";
+ }
+ print F " rm -rf $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid\n";
+ print F " mkdir -p $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid\n";
+ print F " for i in \$(seq \$((\$startIndex+1)) \$endIndex); do\n";
+ print F " leadingI=`printf %06d \$i`\n";
+ if (defined(getGlobal("localStaging"))) {
+ print F " cp $wrk/temp$libraryname/1-overlapper/correct_reads_part\$i.fasta $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid/correct_reads_part\$leadingI.fasta\n";
+ } elsif (!defined(getGlobal("mhapPrecompute"))) {
+ print F " ln -s $wrk/temp$libraryname/1-overlapper/correct_reads_part\$i.fasta $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid/correct_reads_part\$leadingI.fasta\n";
+ } else {
+ print F " ln -s $wrk/temp$libraryname/1-overlapper/correct_reads_part\$i.dat $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid/correct_reads_part\$leadingI.dat\n";
+ }
+ print F " done\n";
+ print F " if [ \$sa -eq \$numSA ]; then\n";
+ print F " $javaCmd -s $wrkDir/temp$libraryname/1-overlapper/correct_reads_part\$sa.$suffix 2> $wrk/temp$libraryname/1-overlapper/\$jobid.err | $convertCmd\n";
+ print F " else\n";
+ print F " if [ \$start -eq \$hashEnd ]; then\n";
+ print F " $javaCmd -s $wrkDir/temp$libraryname/1-overlapper/correct_reads_part\$sa.$suffix -q $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid 2> $wrk/temp$libraryname/1-overlapper/\$jobid.err | $convertCmd\n";
+ print F " else\n";
+ print F " $javaCmd --no-self -s $wrkDir/temp$libraryname/1-overlapper/correct_reads_part\$sa.$suffix -q $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid 2> $wrk/temp$libraryname/1-overlapper/\$jobid.err | $convertCmd\n";
+ print F " fi\n";
+ print F " fi\n";
+ print F " rm -rf $wrkDir/temp$libraryname/1-overlapper/stream_\$jobid\n";
+ }
+ if (!defined(getGlobal("mhap"))) {
+ print F " if test -e $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.eidToIID ; then\n";
+ print F " \$bin/convertSamToCA \\\n";
+ print F " $wrkDir/temp$libraryname/1-overlapper/\$bat/\$job.$suffix $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.eidToIID,$wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.eidToIID $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.iidToLen,$wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.iidToLen \\\n";
+ print F " > $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovls 2> $wrk/temp$libraryname/1-overlapper/\$jobid.java.err\\\n";
+ print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.java.success\n";
+ print F " else\n";
+ print F " \$bin/convertSamToCA \\\n";
+ print F " $wrkDir/temp$libraryname/1-overlapper/\$bat/\$job.$suffix $wrk/temp$libraryname/$asm.eidToIID $wrk/temp$libraryname/$asm.iidToLen \\\n";
+ print F " > $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovls 2> $wrk/temp$libraryname/1-overlapper/\$jobid.java.err\\\n";
+ print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.java.success\n";
+ print F " fi\n";
+ print F " if test -e $wrk/temp$libraryname/1-overlapper/\$jobid.java.success ; then\n";
+ print F " echo SamToCA conversion completed.\n";
+ print F " else\n";
+ print F " echo SamToCA conversion failed.\n";
+ print F " tail $wrk/temp$libraryname/1-overlapper/\$jobid.java.err && exit\n";
+ print F " fi\n";
+ print F " \$bin/convertOverlap -ovl -i $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovls -o $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovb\n";
+ print F " rm -f $wrk/temp$libraryname/1-overlapper/\$bat/\$job.sam\n";
+ }
+ close(F);
+ chmod 0755, "$wrk/temp$libraryname/1-overlapper/overlap.sh";
+
+ if ($submitToGrid == 1) {
+ my $gridName = "pBcR_ovl_$asm$gridName";
+ my $jobName = getGridArrayName($gridName, $numOvlJobs);
+ my $arrayOpt = getGridArrayOption($gridName, $numOvlJobs);
+ submitBatchJobs($wrk, $asm, "$submitCommand $grid $gridOvl $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/1-overlapper/overlap.sh", $jobName);
+ } else {
+ for (my $i = 1; $i <= $numOvlJobs; $i++) {
+ schedulerSubmit("$wrk/temp$libraryname/1-overlapper/overlap.sh $i");
+ }
+ schedulerSetNumberOfProcesses(getGlobal("ovlConcurrency"));
+ schedulerFinish();
+ }
+
+ checkforerror:
+ my $failedJobs = 0;
+ my $failureMessage = "";
+
+ open(B, "< $wrk/temp$libraryname/1-overlapper/ovlbat") or die("failed to open '$wrk/temp$libraryname/1-overlapper/ovlbat'", undef);
+ open(J, "< $wrk/temp$libraryname/1-overlapper/ovljob") or die("failed to open '$wrk/temp$libraryname/1-overlapper/ovljob'", undef);
+
+ while (!eof(B) && !eof(J)) {
+ my $b = <B>; chomp $b;
+ my $j = <J>; chomp $j;
+
+ if ((! -e "$wrk/temp$libraryname/1-overlapper/$b/$j.ovb.gz") &&
+ (! -e "$wrk/temp$libraryname/1-overlapper/$b/$j.ovb")) {
+ $failureMessage .= "ERROR: Overlap job $wrk/temp$libraryname/1-overlapper/$b/$j FAILED.\n";
+ $failedJobs++;
+ } else {
+ runCommand("$wrk/temp$libraryname/1-overlapper/$b/", "rm -f $j.ovls");
+ }
+ }
+ if (!eof(B) || !eof(J)) {
+ print STDERR "Partitioning error; '$wrk/temp$libraryname/1-overlapper/ovlbat' and '$wrk/temp$libraryname/1-overlapper/ovljob' have extra lines.\n";
+ }
+ close(B);
+ close(J);
+
+ my $errorType = "sam conversion";
+ if (defined(getGlobal("blasr"))) { $errorType = "blasr mapping"; }
+ if (defined(getGlobal("bowtie"))) { $errorType = "bowtie mapping"; }
+ $failureMessage .= "\n$failedJobs $errorType jobs failed.";
+ if ($failedJobs > 0) {
+ die "$failureMessage\n";
+ }
+ runCommand("$wrk/temp$libraryname/1-overlapper/", "rm -f *.dat");
+ # cleanup local storage if needed
+ if (!-e "$wrk/temp$libraryname/1-overlapper/ovlclean.sh" && defined(getGlobal("localStaging"))) {
+ open F, "> $wrk/temp$libraryname/1-overlapper/ovlclean.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/ovlclean.sh'");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F "jobid=\$$gridTaskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F "jobid=\$1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if test x\$jobid = x; then\n";
+ print F " echo Error: I need $gridTaskID set, or a job index on the command line\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "\n";
+ my $wrkDir = $wrkDir = getGlobal("localStaging") . "/\$USER/";
+ print F "rm -rf $wrkDir/temp$libraryname/1-overlapper/\n";
+ close(F);
+ chmod 0755, "$wrk/temp$libraryname/1-overlapper/ovlclean.sh";
+
+ if ($submitToGrid == 1) {
+ my $gridName = "pBcR_ovlclean_$asm$gridName";
+ my $jobName = getGridArrayName($gridName, $numOvlJobs);
+ my $arrayOpt = getGridArrayOption($gridName, $numOvlJobs);
+ submitBatchJobs($wrk, $asm, "$submitCommand $grid $gridOvl $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/1-overlapper/ovlclean.sh", $jobName);
+ } else {
+ for (my $i = 1; $i <= $numOvlJobs; $i++) {
+ schedulerSubmit("$wrk/temp$libraryname/1-overlapper/ovlclean.sh $i");
+ }
+ schedulerSetNumberOfProcesses(getGlobal("ovlConcurrency"));
+ schedulerFinish();
+ }
+ }
+ }
+
+ $cmd = "-s $specFile ";
+ $cmd .= "-p $asm -d . ";
+ $cmd .= "ovlMerThreshold=$ovlThreshold ";
+ $cmd .= "ovlHashLibrary=$libToCorrect ";
+ $cmd .= "ovlRefLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "ovlCheckLibrary=1 ";
+ $cmd .= "obtHashLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "obtRefLibrary=$minCorrectLib-$maxCorrectLib ";
+ $cmd .= "obtCheckLibrary=0 ";
+ $cmd .= "gridJobName=\"" . getGlobal("gridJobName") . "\" " if defined(getGlobal("gridJobName"));
+ $cmd .= "gridEnginePropagateHold=\"pBcR_$asm$gridName\" ";
+ $cmd .= "stopAfter=overlapper";
+ if ($submitToGrid == 1) {
+ submitRunCA("$wrk/temp$libraryname", $asm, $cmd, "runCA_ovl_$asm$gridName");
+ } else {
+ runCommand("$wrk/temp$libraryname", "$CA/runCA $cmd");
+ }
+}
+
+if (! -e "$wrk/temp$libraryname/$asm.layout.success") {
+ if ( -e "$wrk/temp$libraryname/$asm.layout.err") {
+ # avoid infinite loops
+ die "Error: found output from runCorrection. Stopping to avoid infinite loops. To try again, remove $asm.layout.*\n";
+ }
+ open F, "> $wrk/temp$libraryname/runCorrection.sh" or die ("can't open '$wrk/temp$libraryname/runCorrection.sh'");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F " if test -e $wrk/temp$libraryname/$asm.layout.success; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " else\n";
+ print F " \$bin/correctPacBio \\\n";
+ if (defined($longReads) && $longReads == 1) {
+ print F " -L \\\n";
+ print F " -l $length \\\n";
+ if ((!defined(getGlobal("onlyContained")) || getGlobal("onlyContained") == 0) && $genomeSize != 0 && getGlobal("maxCoverage") != 0) {
+ print F " -CM " . getGlobal("maxCoverage") . "\\\n";
+ }
+ }
+ print F " -C $coverage \\\n";
+ print F " -M $maxUncorrectedGap \\\n";
+ print F " -t $threads \\\n";
+ print F " -p $partitions \\\n";
+ print F " -o $asm \\\n";
+ print F " $repeats \\\n";
+ print F " -O $wrk/temp$libraryname/$asm.ovlStore \\\n";
+ print F " -G $wrk/temp$libraryname/$asm.gkpStore \\\n";
+ print F " -e $ovlErrorRate -c $ovlErrorRate -E $ovlErrorLimit > $wrk/temp$libraryname/$asm.layout.err 2> $wrk/temp$libraryname/$asm.layout.err && touch $wrk/temp$libraryname/$asm.layout.success\n";
+ print F " fi\n";
+ close(F);
+ chmod 0755, "$wrk/temp$libraryname/runCorrection.sh";
+
+ if ($submitToGrid == 1) {
+ my $gridName = "pBcR_correct_$asm$gridName";
+ submitBatchJobs("$wrk/temp$libraryname", $asm, "$submitCommand $grid $gridCorrection $nameOption \"$gridName\" $outputOption /dev/null $wrk/temp$libraryname/runCorrection.sh", $gridName);
+ } else {
+ runCommand("$wrk/temp$libraryname", "$wrk/temp$libraryname/runCorrection.sh");
+ }
+
+ runCommand("$wrk/temp$libraryname", "rm -rf $asm.paired.ovlStore");
+}
+
+if (! -e "$wrk/temp$libraryname/runPartition.sh") {
+ # convert QV to coverage for PBCNS
+ # we use an ad-hoc conversion where QV 60 = coverage 8 (max) and QV 50 or below = coverage 1
+ my $coverage = floor($QV - 50);
+ if ($coverage < 1) { $coverage = 1; }
+ if ($coverage > 8) { $coverage = 8; }
+
+ my $cnsType = "-consensus pbdagcon";
+ # priority is pbcns then falconcns
+ if (defined(getGlobal("pbcns")) && getGlobal("pbcns") == 1) {
+ setGlobal("falconcns", 0);
+ }
+ if (defined(getGlobal("falconcns")) && getGlobal("falconcns") == 1) {
+ setGlobal("pbcns", 0);
+ $cnsType = " -consensus falcon -pythonPath $FALCON";
+ #$coverage += 2;
+ }
+ my $falconcns = getGlobal("falconcns");
+ my $pbcns = getGlobal("pbcns");
+
+ my $threads = 1;
+ if ((defined($pbcns) && $pbcns == 1) || (defined($falconcns) && $falconcns == 1)) {
+ if ($submitToGrid == 1) {
+ $threads = $consensusConcurrency;
+ } else {
+ $threads = 8;
+ if ($consensusConcurrency < $threads) {
+ $threads = $consensusConcurrency;
+ $consensusConcurrency = 1;
+ } else {
+ $consensusConcurrency = int(int($consensusConcurrency) / int($threads));
+ }
+ }
+ }
+
+ open F, "> $wrk/temp$libraryname/runPartition.sh" or die ("can't open '$wrk/temp$libraryname/runPartition.sh'");
+ print F "#!" . getGlobal("shell") ."\n";
+ print F getBinDirectoryShellCode();
+ print F "\n";
+ print F "jobid=\$$gridTaskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F "jobid=\$1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if test x\$jobid = x; then\n";
+ print F " echo Error: I need $gridTaskID set, or a job index on the command line\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ my $wrkDir = $wrk;
+ print F "if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
+ print F " echo \"Job previously completed successfully.\"\n";
+ print F "else\n";
+ print F "if test -e $wrk/temp$libraryname/\$jobid.lay.success ; then\n";
+ print F " echo \"Layout previously generated.\"\n";
+ print F "else\n";
+ print F " \$bin/outputLayout \\\n";
+ if (defined($longReads) && $longReads == 1) {
+ print F " -L \\\n";
+ }
+ print F " -e $ovlErrorRate -M $maxUncorrectedGap \\\n";
+ print F " -i $wrk/temp$libraryname/$asm \\\n";
+ print F " -o $wrkDir/temp$libraryname/$asm \\\n";
+ print F " -p \$jobid \\\n";
+ print F " -l $length \\\n";
+ print F " $repeats \\\n";
+ if (defined($pbcns) && $pbcns == 1) {
+ print F " -P \\\n";
+ }
+ if (defined($falconcns) && $falconcns == 1) {
+ print F " -F \\\n";
+ }
+ print F " -G $wrk/temp$libraryname/$asm.gkpStore \\\n";
+ if (defined($falconcns) && $falconcns == 1) {
+ my $falconParams = getGlobal("falconOptions");
+ print F " 2> $wrk/temp$libraryname/\$jobid.lay.err | awk '{if (!/[+-]/) gsub(\"[^ACGTacgt]\", \"\", \$2); print \$0}' | $FALCON/falcon_sense $falconParams --min_cov $coverage --n_core $threads > $wrk/temp$libraryname/\$jobid.fasta 2> $wrk/temp$libraryname/\$jobid.cns.err\n";
+ print F " if [ \$? -ge 0 ]; then\n";
+ print F " touch $wrk/temp$libraryname/\$jobid.success\n";
+ print F " fi\n";
+ } elsif (defined($pbcns) && $pbcns == 1) {
+ print F " 2> $wrk/temp$libraryname/\$jobid.lay.err | \$bin/convertToPBCNS $cnsType -path $BLASR -output $wrk/temp$libraryname/\$jobid.fasta -prefix $wrkDir/temp$libraryname/\$jobid.tmp -length ";
+ if (defined($longReads) && $longReads == 1) {
+ print F "$length"; } else { print F "50";
+ }
+ print F " -coverage $coverage -threads $threads > $wrk/temp$libraryname/\$jobid.err 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
+# | $FALCON/pbdagcon -a -t 0 -m " . ((defined($longReads) && $longReads == 1) ? $length : 50) . " -j $threads -c $coverage -b 2000 > $wrk/temp$libraryname/\$jobid.fasta 2> $wrk/temp$libraryname/\$jobid.cns.err\n";
+ print F " if [ \$? -ge 0 ]; then\n";
+ print F " touch $wrk/temp$libraryname/\$jobid.success\n";
+ print F " fi\n";
+ } else {
+ print F " > $wrk/temp$libraryname/\$jobid.lay.err 2>&1 && touch $wrk/temp$libraryname/\$jobid.lay.success\n";
+ print F " if ! test -e $wrk/temp$libraryname/\$jobid.lay.success ; then\n";
+ print F " echo \"Error generating layout for job \$jobid\"\n";
+ print F " exit 1\n";
+ print F " fi\n";
+ print F " fi\n";
+ print F " \n";
+ print F " numLays=`cat $wrkDir/temp$libraryname/$asm" . ".\$jobid.lay |grep -c -m 1 \"{LAY\"`\n";
+ print F " if test \$numLays = 0 ; then\n";
+ print F " touch $wrk/temp$libraryname/\$jobid.fasta\n";
+ print F " touch $wrk/temp$libraryname/\$jobid.qual\n";
+ print F " touch $wrk/temp$libraryname/\$jobid.success\n";
+ print F " else\n";
+ if (!defined $bankPath) {
+ print F " bankPath=\"$wrk/temp$libraryname\"\n";
+ } elsif (uc($bankPath) eq "SHARED") {
+ print F " bankPath=\"/dev/shm/temp$libraryname\"\n";
+ print F " laySize=`ls -la $wrk/temp$libraryname/$asm.\$jobid.lay |awk '{print \$5}'`\n";
+ print F " tmpFSSize=`df |grep /dev/shm |awk '{print \$(NF-2)}'`\n";
+ if ($submitToGrid) {
+ print F " let requiredSize=\$laySize*2\n";
+ } else {
+ print F " let requiredSize=\$laySize*2*$consensusConcurrency\n";
+ }
+ print F " if test x\$tmpFSSize = x; then\n";
+ print F " tmpFSSize=0\n";
+ print F " fi\n";
+ print F " if [ \$requiredSize -lt \$tmpFSSize ]; then\n";
+ print F " mkdir -p /dev/shm/temp$libraryname\n";
+ print F " else\n";
+ print F " bankPath=\"$wrk/temp$libraryname\"\n";
+ print F " fi\n";
+ } else {
+ print F " bankPath=\"$bankPath/temp$libraryname\"\n";
+ print F " mkdir -p \"$bankPath/temp$libraryname\"\n";
+ }
+ print F " rm -rf \$bankPath/$asm" . ".bnk_partition\$jobid.bnk\n";
+ print F " $AMOS/bank-transact -b \$bankPath/$asm" . ".bnk_partition\$jobid.bnk -m $wrkDir/temp$libraryname/$asm.\$jobid" . ".lay -c > $wrk/temp$libraryname/bank-transact.\$jobid.err 2>&1\n";
+ if (defined($shortReads) && $shortReads == 1) {
+ print F " $AMOS/make-consensus -e 0.03 -w 5 -x $wrk/temp$libraryname/\$jobid.excluded -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success && rm $wrkDir/temp$libraryname/$asm.\$jobid.lay\n";
+ } elsif (defined($longReads) && $longReads == 1) {
+ print F " $AMOS/make-consensus -e 0.30 -w 50 -o 5 -x $wrk/temp$libraryname/\$jobid.excluded -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success && rm $wrkDir/temp$libraryname/$asm.\$jobid.lay\n";
+ } else {
+ print F " $AMOS/make-consensus -x $wrk/temp$libraryname/\$jobid.excluded -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success && rm $wrkDir/temp$libraryname/$asm.\$jobid.lay\n";
+ }
+ print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
+ print F " $AMOS/bank2fasta -e -q $wrk/temp$libraryname/\$jobid.qual -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.fasta\n";
+ print F " else\n";
+ print F " rm -rf \$bankPath/$asm" . ".bnk_partition\$jobid.bnk\n";
+ print F " $AMOS/bank-transact -b \$bankPath/$asm" . ".bnk_partition\$jobid.bnk -m $wrkDir/temp$libraryname/$asm.\$jobid" . ".lay -c > $wrk/temp$libraryname/bank-transact.\$jobid.err 2>&1\n";
+ if (defined($shortReads) && $shortReads == 1) {
+ print F " $AMOS/make-consensus -e 0.03 -w 5 -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success && rm $wrkDir/temp$libraryname/$asm.\$jobid.lay\n";
+ } elsif (defined($longReads) && $longReads == 1) {
+ print F " $AMOS/make-consensus -e 0.30 -w 50 -o 5 -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success && rm $wrkDir/temp$libraryname/$asm.\$jobid.lay\n";
+ } else {
+ print F " $AMOS/make-consensus -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success && rm $wrkDir/temp$libraryname/$asm.\$jobid.lay\n";
+ }
+ print F " $AMOS/bank2fasta -e -q $wrk/temp$libraryname/\$jobid.qual -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.fasta\n";
+ print F " fi\n";
+ print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
+ print F " $CA/trimFastqByQVWindow $wrk/temp$libraryname/\$jobid.fasta $wrk/temp$libraryname/\$jobid.qual $wrk/temp$libraryname/\$jobid.trim.fasta $wrk/temp$libraryname/\$jobid.trim.qual $QV $length > $wrk/temp$libraryname/\$jobid.trim.fastq\n";
+ print F " rm -rf \$bankPath/$asm" . ".bnk_partition\$jobid.bnk\n";
+ print F " rm $wrk/temp$libraryname/bank-transact.\$jobid.err\n";
+ print F " rm $wrk/temp$libraryname/\$jobid.excluded\n";
+ print F " rm $wrk/temp$libraryname/\$jobid.out\n";
+ print F " fi\n";
+ }
+ # cleanup intermediate files if we're successfull
+ if ($cleanup == 1) {
+ print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
+ print F " rm -f $wrk/temp$libraryname/$asm.\$jobid.shortmap*\n";
+ print F " rm -f $wrk/temp$libraryname/$asm.\$jobid.rank\n";
+ print F " rm -f $wrk/temp$libraryname/$asm.\$jobid.olaps\n";
+ print F " fi\n";
+ }
+ if ((defined($pbcns) && $pbcns == 1) || (defined($falconcns) && $falconcns == 1)) {
+ print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
+ print F " cat $wrk/temp$libraryname/\$jobid.fasta | awk '{if(\$0~/>/){print;}else{l=length(\$0);q=\"\";while(l--){q=q \" 60\"}printf(\"%s\\n\",q)}}' > $wrk/temp$libraryname/\$jobid.qual\n";
+ print F " cat $wrk/temp$libraryname/\$jobid.fasta | awk '{if(\$0~/>/){sub(/>/,\"@\",\$0);print;}else{l=length(\$0);q=\"\";while(l--){q=q \"]\"}printf(\"%s\\n+\\n%s\\n\",\$0,q)}}' > $wrk/temp$libraryname/\$jobid.trim.fastq\n";
+ print F " ln -s $wrk/temp$libraryname/\$jobid.fasta $wrk/temp$libraryname/\$jobid.trim.fasta\n";
+ print F " ln -s $wrk/temp$libraryname/\$jobid.qual $wrk/temp$libraryname/\$jobid.trim.qual\n";
+ print F " fi\n";
+ }
+ print F " fi\n";
+ print F "fi\n";
+ close(F);
+
+ chmod 0755, "$wrk/temp$libraryname/runPartition.sh";
+
+ if ($submitToGrid == 1) {
+ my $gridName = "pBcR_cns_$asm$gridName";
+ my $jobName = getGridArrayName($gridName, $partitions);
+ my $arrayOpt = getGridArrayOption($gridName, $partitions);
+ submitBatchJobs("$wrk/temp$libraryname", $asm, "$submitCommand $grid $gridConsensus $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/runPartition.sh", $jobName);
+ } else {
+ for (my $i = 1; $i <=$partitions; $i++) {
+ schedulerSubmit("$wrk/temp$libraryname/runPartition.sh $i");
+ }
+ schedulerSetNumberOfProcesses($consensusConcurrency);
+ schedulerFinish();
+ }
+}
+
+for (my $i = 1; $i <= $partitions; $i++) {
+ if (! -e "$wrk/temp$libraryname/$i.success") {
+ die "Failed to run correction job $i. Remove $wrk/temp$libraryname/runPartition.sh to try again.\n";
+ }
+}
+runCommand("$wrk/temp$libraryname", "rm -rf core.*");
+
+# combine partitions
+runCommand("$wrk/temp$libraryname", "cat `ls $asm.[0-9]*.log | sort -T . -rnk1` > corrected.log");
+
+# check how much we split the sequences up
+my %splitUIDs = ();
+runCommand("$wrk/temp$libraryname", "cat corrected.log |awk '{print \$1}' |sort -T . |uniq -c |awk '{if (\$1 > 1) print \$2}' > $asm.split.uid");
+
+# read in the split UIDs so we can process them
+open(F, "< $wrk/temp$libraryname/$asm.split.uid") or die("Couldn't open '$wrk/temp$libraryname/$asm.split.uid'", undef);
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+ chomp $_;
+
+ $splitUIDs{$_} = 1;
+}
+close(F);
+
+open (F, "< $wrk/temp$libraryname/corrected.log") or die ("Couldn't open '$wrk/temp$libraryname/corrected.log", undef);
+open (O, "> $wrk/temp$libraryname/$asm.split.allEdit") or die ("Couldn't open '$wrk/temp$libraryname/$asm.split.allEdit", undef);
+
+my $totalSplitBases = 0;
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+ chomp $_;
+
+ my ($uid, $newname, $subread, $start, $end) = split '\s+', $_;
+ my $len = $end - $start;
+ if (defined($splitUIDs{$uid}) && $splitUIDs{$uid} == 1) {
+ print O "$newname\n";
+
+ $totalSplitBases += $len;
+ }
+}
+close(F);
+close(O);
+
+print STDOUT "Total split bases is $totalSplitBases vs $totalBP so ratio is " . ($totalSplitBases/$totalBP) . "\n";
+
+#save output from temporary directory
+runCommand("$wrk/temp$libraryname", "cp $asm.layout.err $wrk/$libraryname.correction.err");
+if (-e "$wrk/temp$libraryname/$asm.layout.hist") {
+ runCommand("$wrk/temp$libraryname", "cp $asm.layout.hist $wrk/$libraryname.correction.hist");
+}
+runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.fasta |grep trim |sort -T . -rnk1` > $wrk/$libraryname.fasta");
+runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.qual |grep trim | sort -T . -rnk1` > $wrk/$libraryname.qual");
+runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.fastq |grep trim | sort -T . -rnk1` > $wrk/$libraryname.fastq");
+runCommand("$wrk", "$CA/fastqToCA -libraryname $libraryname -technology pacbio-corrected -type sanger -reads $wrk/$libraryname.fastq > $wrk/$libraryname.frg");
+
+my $numOutput = `ls $wrk/temp$libraryname/*sge.out* |wc -l`;
+chomp $numOutput;
+if ($numOutput != 0) {
+ runCommand("$wrk/temp$libraryname", "cat `ls *sge.out*` > $wrk/temp$libraryname/corrected.err");
+ runCommand("$wrk/temp$libraryname", "cp corrected.err $wrk/$libraryname.err");
+}
+
+# generate summary reports
+my %nameTranslation;
+open(F, "< $wrk/temp$libraryname/$asm.gkpStore.fastqUIDmap") or die("Couldn't open '$wrk/temp$libraryname/$asm.gkpStore.fastqUIDmap'", undef);
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+ my @splitLine=split /\s+/;
+ $nameTranslation{$splitLine[1]} = $splitLine[2];
+}
+close(F);
+
+my $maxCorrected = 0;
+my $totalCorrected = 0;
+my $totalCorrectedBP = 0;
+open(F, "< $wrk/temp$libraryname/corrected.log") or die("Couldn't open '$wrk/temp$libraryname/corrected.log'", undef);
+open(W, "> $wrk/$libraryname.log") or die("Couldn't open '$wrk/$libraryname.log'", undef);
+
+while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+ my @splitLine=split /\s+/;
+ my $len = $splitLine[4] - $splitLine[3];
+ my $name = (defined($nameTranslation{$splitLine[0]}) ? $nameTranslation{$splitLine[0]} : $splitLine[0]);
+
+ print W "INPUT_NAME\tOUTPUT_NAME\tSUBREAD\tSTART\tEND\tLENGTH\n"if ($totalCorrected == 0);
+ print W "$name\t$splitLine[1]\t$splitLine[2]\t$splitLine[3]\t$splitLine[4]\t$len\n";
+
+ $maxCorrected = $len if ($len > $maxCorrected);
+ $totalCorrected++;
+ $totalCorrectedBP += $len;
+}
+close(F);
+close(W);
+
+print STDERR "******** Correction Summary ********\n";
+printf "Total corrected bp (before consensus): $totalCorrectedBP (%.3f).\n", ($totalCorrectedBP/$totalBP)*100;
+printf "Longest sequence: $maxCorrected bp, mean: %.3f bp.\n", ($totalCorrectedBP/$totalCorrected);
+
+# finally clean up the assembly directory
+if ($cleanup == 1) {
+ runCommand("$wrk", "rm -rf temp$libraryname/[0-9]*.*");
+ runCommand("$wrk", "rm -rf temp$libraryname/$asm.[0-9]*.log");
+ runCommand("$wrk", "rm -rf temp$libraryname/$asm.gkpStore");
+ runCommand("$wrk", "rm -rf temp$libraryname/$asm.ovlStore");
+ runCommand("$wrk", "rm -rf temp$libraryname/$asm.[0-9]*.lay");
+}
+
+print STDERR "********* Finished correcting $totalBP bp (using $totalCorrectingWith bp).\n";
+
+# now assemble
+assemble:
+my $asmCoverage = getGlobal("assembleCoverage");
+
+if (defined(getGlobal("assemble")) && getGlobal("assemble") == 1) {
+ my $libraryname = getGlobal("libraryname");
+
+ if (! -d "$wrk/$libraryname" && ! -e "$wrk/$libraryname.longest$asmCoverage.frg") {
+ my $specFile = "$wrk/temp$libraryname/$libraryname.spec";
+ my $genomeSize = getGlobal("genomeSize");
+ if (!defined($genomeSize) || $genomeSize == 0) { # guestimate genome size
+ $genomeSize = 5000000;
+ }
+ print STDERR "********* Assembling corrected sequences.\n";
+
+ my $totalToAsm = int $asmCoverage * $genomeSize;
+ my $average = 0;
+ my $total = 0;
+
+ open(F, "$CA/fastqAnalyze $libraryname.fastq -stats 2>/dev/null |head |") or die ("Couldn't open corrected sequences", undef);
+ while (<F>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ my @array = split '\s+';
+ if ($array[0] eq "bases") {
+ $total = int($array[1]);
+ }
+ if ($array[0] eq "average") {
+ $average = int($array[1]);
+ }
+ }
+ close(F);
+
+ if (defined(getGlobal("falconcns")) && getGlobal("falconcns") == 1) {
+ setGlobal("asmUtgErrorRate", getGlobal("asmUtgErrorRate") * $FALCON_ERATE_ADJUST);
+ setGlobal("utgGraphErrorRate", getGlobal("utgGraphErrorRate") * $FALCON_ERATE_ADJUST);
+ }
+
+ # should use a formula, maybe 40% rounded to next 500
+ my $frgLen = 3000;
+ my $ovlLen = 100;
+ my $batOptions = getGlobal("batOptions");
+ if ($average < $frgLen) {
+ $frgLen = $average / 2;
+ $ovlLen = $average / 6;
+ if ($ovlLen < 40) { $ovlLen = 40; }
+ }
+
+ print STDERR "Assembling with average $average (min frag $frgLen) and using ovl is $ovlLen\n";
+ if ($total > $totalToAsm) {
+ runCommand("$wrk", "ln -sf $libraryname.fastq $libraryname.u.fastq");
+ runCommand("$wrk", "$CA/fastqSample -U -I $libraryname -O $libraryname.longest$asmCoverage -c $asmCoverage -g $genomeSize -max");
+ runCommand("$wrk", "mv $libraryname.longest$asmCoverage.u.fastq $libraryname.longest$asmCoverage.fastq");
+ runCommand("$wrk", "$CA/fastqToCA -libraryname $libraryname -technology pacbio-corrected -type sanger -reads $wrk/$libraryname.longest$asmCoverage.fastq > $wrk/$libraryname.longest$asmCoverage.frg");
+ } else {
+ runCommand("$wrk", "ln -sf $libraryname.frg $libraryname.longest$asmCoverage.frg");
+ runCommand("$wrk", "ln -sf $libraryname.fastq $libraryname.longest$asmCoverage.fastq");
+ }
+ my $ovlPartCmd = "";
+ my $ovlRefBlockSize = getGlobal("ovlRefBlockSize");
+ my $ovlRefBlockLength = getGlobal("ovlRefBlockLength");
+ if ($ovlRefBlockLength > 0) {
+ $ovlPartCmd = "ovlRefBlockLength=$ovlRefBlockLength ovlRefBlockSize=0";
+ } elsif ($ovlRefBlockSize > 0) {
+ $ovlPartCmd = "ovlRefBlockSize=$ovlRefBlockSize ovlRefBlockLength=0";
+ }
+ $cmd = "-s $specFile ";
+ $cmd .= "-p $asm -d $libraryname $ovlPartCmd ";
+ $cmd .= "useGrid=" .getGlobal("useGrid") . " scriptOnGrid=" . getGlobal("scriptOnGrid") . " unitigger=" . getGlobal("unitigger") . " ";
+ $cmd .= "ovlErrorRate=" . getGlobal("asmOvlErrorRate") . " utgErrorRate=" . getGlobal("asmUtgErrorRate") . " cgwErrorRate=" . getGlobal("asmCgwErrorRate") . " cnsErrorRate=" . getGlobal("asmCnsErrorRate") . " ";
+ $cmd .= "utgGraphErrorLimit=" . getGlobal("utgGraphErrorLimit") . " utgGraphErrorRate=" . getGlobal("utgGraphErrorRate") . " utgMergeErrorLimit=" . getGlobal("utgMergeErrorLimit") . " utgMergeErrorRate=" . getGlobal("utgMergeErrorRate") . " ";
+ $cmd .= "frgCorrBatchSize=100000 doOverlapBasedTrimming=" . getGlobal("asmOBT") . " obtErrorRate=" . getGlobal("asmObtErrorRate") . " obtErrorLimit=" . getGlobal("asmObtErrorLimit") . " frgMinLen=$frgLen ovlMinLen=$ovlLen \"batOptions=$batOptions\" ";
+ $cmd .= "consensus=" . getGlobal("asmCns") . " merSize=" . getGlobal("asmMerSize") . " cnsMaxCoverage=1 cnsReuseUnitigs=1 ";
+ $cmd .= "gridJobName=\"" . getGlobal("gridJobName") . "\" " if defined(getGlobal("gridJobName"));
+ $cmd .= "gridEnginePropagateHold=\"pBcR_$asm$gridName\" ";
+ $cmd .= " $libraryname.longest$asmCoverage.frg ";
+
+ # don't assemble if we don't have enough data
+ my $minToAsm = getGlobal("assembleMinCoverage");
+ if ($total < $minToAsm * $genomeSize) {
+ print STDERR "Warning: after correction only " . ($total / $genomeSize) . "X for genome $genomeSize. Not performing automated assembly. \n";
+ print STDERR "\tTo manually launch an assembly run: \n\n";
+ print STDERR "$CA/runCA $cmd";
+ print STDERR "\n\n";
+ } else {
+ if ($submitToGrid == 1) {
+ submitRunCAHelper("$wrk", $asm, $cmd, "runCA_asm_$asm$gridName", 0);
+ } else {
+ runCommand("$wrk", "$CA/runCA $cmd");
+ }
+ }
+ }
+}
diff --git a/src/AS_PBR/TrimFastqByQVWindow.cc b/src/AS_PBR/TrimFastqByQVWindow.cc
old mode 100644
new mode 100755
diff --git a/src/AS_PBR/addCNSToStore.pl b/src/AS_PBR/addCNSToStore.pl
new file mode 100755
index 0000000..dd2eaa5
--- /dev/null
+++ b/src/AS_PBR/addCNSToStore.pl
@@ -0,0 +1,345 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2011, Battelle National Biodefense Institute (BNBI);
+# all rights reserved. Authored by: Sergey Koren
+#
+
+# This Software was prepared for the Department of Homeland Security
+# (DHS) by the Battelle National Biodefense Institute, LLC (BNBI) as
+# part of contract HSHQDC-07-C-00020 to manage and operate the National
+# Biodefense Analysis and Countermeasures Center (NBACC), a Federally
+# Funded Research and Development Center.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of the Battelle National Biodefense Institute nor
+# the names of its contributors may be used to endorse or promote
+# products derived from this software without specific prior written
+# permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+###########################################################################
+#
+# Read in fragments from fastq-format sequence and quality files,
+# correct to pacbio fragments.
+#
+
+use strict;
+
+use Config; # for @signame
+use FindBin;
+use Cwd;
+use Carp;
+use FileHandle;
+
+use POSIX qw(ceil floor sys_wait_h);
+
+my %global;
+
+sub getGlobal ($) {
+ my $var = shift @_;
+ if (!exists($global{$var})) {
+ return undef;
+ }
+ return($global{$var});
+}
+
+sub setGlobal ($$) {
+ my $var = shift @_;
+ my $val = shift @_;
+
+ # If no value, set the field to undefined, the default for many of the options.
+
+ $val = undef if ($val eq "");
+
+ $global{$var} = $val;
+ $val =~ s/\\\"/\"/g;
+ $val =~ s/\"/\\\"/g;
+ $val =~ s/\\\$/\$/g;
+ $val =~ s/\$/\\\$/g;
+}
+
+sub readFasta($$$) {
+ my $file = shift @_;
+ my $prefix = shift @_;
+ my $hashref = shift @_;
+
+ my $fastaSeq = "";
+ my $id = "";
+ open(CNS, "< $file") or die("Couldn't open '$file'", undef);
+ while (<CNS>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+
+ if (m/\>/) {
+ if ($fastaSeq ne "") {
+ $fastaSeq =~ s/^\s+|\s+$//g;
+ $hashref->{$id} = $fastaSeq;
+ }
+ $fastaSeq = "";
+ $id = (split '\s+')[0];
+ $id =~ s/>//g;
+ if ($id =~ m/utg_/) {
+ $id = (split '/', $id)[0];
+ $id =~ s/utg_//g;
+ } else {
+ $id = (split ',', $id)[1];
+ }
+ $id = "$prefix.$id";
+ } else {
+ $fastaSeq = $fastaSeq . $_;
+ }
+ }
+ if ($fastaSeq ne "") {
+ $fastaSeq =~ s/^\s+|\s+$//g;
+ $hashref->{$id} = $fastaSeq;
+ }
+ close(CNS);
+}
+
+sub process($$$$) {
+ my $cns = shift @_;
+ my $raw = shift @_;
+ my $lay = shift @_;
+ my $output = shift @_;
+ my %seqs = {};
+
+ readFasta($cns, "utg", \%seqs);
+ readFasta($raw, "seq", \%seqs);
+ my $count = (scalar keys %seqs);
+ print "Loaded $count seqs\n";
+
+ my $qltChar = chr(getGlobal("quality") + ord('0'));
+ # we scan file twice, could probably be done once
+ # first scan figure out which sequences have only one read and get their cns from the read or full cns
+ my %cnsHash = {};
+ my $id = "";
+ open(LAY, "<$lay") or die ("Couldn't open '$lay'", undef);
+ while (<LAY>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+ my $type = (split '\s+')[0];
+ my $val = (split '\s+')[1];
+ if ($type eq "unitig") {
+ $id = $val;
+ } elsif ($type eq "contig") {
+ $id = $val;
+ $cnsHash{$id} = "utg.$id";
+ } elsif ($type eq "data.num_frags" && $val > 1) {
+ $cnsHash{$id} = "utg.$id";
+ } elsif ($type eq "FRG" && !defined($cnsHash{$id})) { # this is a single frg utg
+ $cnsHash{$id} = "seq." . (split '\s+')[4];
+ }
+ }
+ close(LAY);
+
+ my $fastaSeq = "";
+ my $utgID = "";
+ my $isUTG = 1;
+ open(OUT, ">$output") or die ("Couldn't open '$output'", undef);
+ open(LAY, "<$lay") or die ("Couldn't open '$lay'", undef);
+ while (<LAY>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+ my $type = (split '\s+')[0];
+ my $val = (split '\s+')[1];
+ if ($type eq "unitig") {
+ # close previous unitig
+ if ($fastaSeq ne "") {
+ print OUT "UTG type X ident\t$utgID position\t0\t" . length($fastaSeq) . " num_instances 0\n";
+ }
+ my $cnsId = $cnsHash{$val};
+ # if unknown unitig encountered due to it being too short delete it from the tigStore
+ if (!defined($seqs{$cnsId})) {
+ print STDERR "Unknown consensus for unitig $val\n";
+ $fastaSeq="";
+ print OUT "$_\n";
+ print OUT "len 0\n";
+ print OUT "cns\n";
+ print OUT "qlt\n";
+ print OUT "data.unitig_coverage_stat\t1.000000\n";
+ print OUT "data.unitig_microhet_prob\t1.000000\n";
+ print OUT "data.unitig_status\tX\n";
+ print OUT "data.unitig_suggest_repeat\tF\n";
+ print OUT "data.unitig_suggest_unique\tF\n";
+ print OUT "data.unitig_force_repeat\tF\n";
+ print OUT "data.unitig_force_unique\tF\n";
+ print OUT "data.contig_status\tU\n";
+ print OUT "data.num_frags\t0\n";
+ print OUT "data.num_unitigs\t0\n";
+ } else {
+ $fastaSeq = uc $seqs{$cnsId};
+ $utgID = $val;
+ print OUT "$_\n";
+ }
+ } elsif ($type eq "contig") {
+ # close previous contig
+ my $cnsId = $cnsHash{$val};
+ die "Unknown consensus for contig $val\n" if (!defined($seqs{$cnsId}));
+ $fastaSeq = uc $seqs{$cnsId};
+ $utgID = $val;
+ $isUTG = 0;
+ setGlobal("type", "-cp");
+ print OUT "$_\n";
+ } elsif ($fastaSeq ne "") {
+ if ($type eq "len") {
+ print OUT "len " . length($fastaSeq) . "\n";
+ } elsif ($type eq "cns") {
+ print OUT "cns $fastaSeq\n";
+ } elsif ($type eq "qlt") {
+ my $qlt = $fastaSeq;
+ $qlt =~ s/./$qltChar/g;
+ print OUT "qlt $qlt\n";
+ } elsif ($type eq "data.num_unitigs" && $val < 1) {
+ print OUT "data.num_unitigs\t1\n";
+ } else {
+ if ($type eq "FRG") {
+ my @tokenized = split '\s+';
+ my $min = ($tokenized[13] < $tokenized[14] ? $tokenized[13] : $tokenized[14]);
+ my $max = ($tokenized[13] > $tokenized[14] ? $tokenized[13] : $tokenized[14]);
+ my $fwd = ($tokenized[13] < $tokenized[14] ? "+" : "-");
+
+ if ($fastaSeq ne "" && $max > length($fastaSeq)) {
+ $max = length($fastaSeq);
+ if ($fwd eq "+") {
+ $tokenized[14] = $max;
+ } else {
+ $tokenized[13] = $max;
+ }
+ }
+ if ($fastaSeq ne "" && $min > length($fastaSeq)) {
+ $min = length($fastaSeq);
+ if ($fwd eq "+") {
+ $tokenized[13] = $min;
+ } else {
+ $tokenized[14] = $min;
+ }
+ }
+ $_ = "";
+ for (my $i = 0; $i < scalar @tokenized; $i++) {
+ $_ = $_ . "\t" . $tokenized[$i];
+ }
+
+ }
+ print OUT "$_\n";
+ }
+ }
+ }
+ if ($fastaSeq ne "" && $isUTG) {
+ print OUT "UTG type X ident\t$utgID position\t0\t" . length($fastaSeq) . " num_instances 0\n";
+ }
+
+ close(LAY);
+ close(OUT);
+}
+
+my $err = 0;
+setGlobal("prefix", "tmp");
+setGlobal("quality", 60);
+setGlobal("type", "-up");
+my $input = undef;
+my $seq = undef;
+my $output = undef;
+my $lay = undef;
+my $partition = undef;
+my $version = 1;
+
+# finally, command-line parameters take precedence
+while (scalar(@ARGV) > 0) {
+ my $arg = shift @ARGV;
+
+ if ($arg eq "-prefix") {
+ setGlobal("prefix", shift @ARGV);
+
+ } elsif ($arg eq "-partition") {
+ $partition = shift @ARGV;
+
+ } elsif ($arg eq "-path") {
+ setGlobal("path", shift @ARGV);
+
+ } elsif ($arg eq "-input") {
+ $input = shift @ARGV;
+
+ } elsif ($arg eq "-lay") {
+ $lay = shift @ARGV;
+
+ } elsif ($arg eq "-output") {
+ $output = shift @ARGV;
+
+ } elsif ($arg eq "-sequence") {
+ $seq = shift @ARGV;
+
+ } elsif ($arg eq "-quality") {
+ setGlobal("quality", shift @ARGV);
+
+ } elsif ($arg eq "-version") {
+ $version = shift @ARGV;
+
+ } else {
+ print STDERR "Unknown parameter " + $err + "\n";
+ $err++;
+ }
+ }
+
+if ($err > 0 || !defined($input) || !defined($output) || !defined($lay)) {
+ print STDERR "Error: no valid input cns specified\n" if (!defined($input));
+ print STDERR "Error: no valid output fasta specified\n" if (!defined($output));
+ print STDERR "Error: no valid input lay specified\n", if (!defined($lay));
+ print STDERR "Invalid arguments specified\n";
+
+ print STDERR "usage: $0 [options] -input <input cns> -output <output layout> -lay <input layout>\n";
+ print STDERR " -path Path to smrtportal installation. Required if smrtportal is not in path.\n";
+ print STDERR " -prefix Prefix for stores to write to\n";
+ exit(1);
+}
+# find tigStore
+my $TIG = getGlobal("path");
+if (!defined($TIG) || $TIG eq "") {
+ print STDERR "Error, could not find tigStore in path. Please update your path or specify a location using -path\n";
+ exit(1);
+}
+
+$output="$output.$partition.cnslay";
+# clear output file
+open(O, ">", $output);
+close(O);
+
+process($input, $seq, $lay, $output);
+# update store
+my $prefix = getGlobal("prefix");
+my $type = getGlobal("type");
+system("$TIG/tigStore -g $prefix.gkpStore -t $prefix.tigStore $version $type $partition -N -A -R $output");
+if ($? != 0) {
+ die "Error: tigStore could not run successfully"
+}
+# cleanup
+system("rm -f $output");
diff --git a/src/AS_PBR/bamcat.c b/src/AS_PBR/bamcat.c
new file mode 100644
index 0000000..ef63c0d
--- /dev/null
+++ b/src/AS_PBR/bamcat.c
@@ -0,0 +1,2605 @@
+#include <string.h>
+#include <unistd.h>
+
+// This is a concatenation of bits from samtools 0.1.19 needed to read sam/bam files.
+//
+// The C files are in alpha order, the header files are in (mostly) include order
+//
+// cat sam.c bam.c bam_aux.c bam_import.c bgzf.c sam_header.c > bamcat.c
+// cat bgzf.h bam.h sam.h bam_endian.h kstring.h sam_header.h khash.h kseq.h > bamcat.h
+//
+// Then replace all (#include ") with (//#include "), and include just the single bam.h.
+// A few functions were removed to cut down on the amount of crud.
+//
+// BPW - Fri Feb 20 16:14:11 EST 2015
+
+#include "bamcat.h"
+
+//#include "sam.h"
+
+#define TYPE_BAM 1
+#define TYPE_READ 2
+
+bam_header_t *bam_header_dup(const bam_header_t *h0)
+{
+ bam_header_t *h;
+ int i;
+ h = bam_header_init();
+ *h = *h0;
+ h->hash = h->dict = h->rg2lib = 0;
+ h->text = (char*)calloc(h->l_text + 1, 1);
+ memcpy(h->text, h0->text, h->l_text);
+ h->target_len = (uint32_t*)calloc(h->n_targets, 4);
+ h->target_name = (char**)calloc(h->n_targets, sizeof(void*));
+ for (i = 0; i < h->n_targets; ++i) {
+ h->target_len[i] = h0->target_len[i];
+ h->target_name[i] = strdup(h0->target_name[i]);
+ }
+ return h;
+}
+static void append_header_text(bam_header_t *header, char* text, int len)
+{
+ int x = header->l_text + 1;
+ int y = header->l_text + len + 1; // 1 byte null
+ if (text == 0) return;
+ kroundup32(x);
+ kroundup32(y);
+ if (x < y) header->text = (char*)realloc(header->text, y);
+ strncpy(header->text + header->l_text, text, len); // we cannot use strcpy() here.
+ header->l_text += len;
+ header->text[header->l_text] = 0;
+}
+
+samfile_t *samopen(const char *fn, const char *mode, const void *aux)
+{
+ samfile_t *fp;
+ fp = (samfile_t*)calloc(1, sizeof(samfile_t));
+ if (strchr(mode, 'r')) { // read
+ fp->type |= TYPE_READ;
+ if (strchr(mode, 'b')) { // binary
+ fp->type |= TYPE_BAM;
+ fp->x.bam = strcmp(fn, "-")? bam_open(fn, "r") : bam_dopen(fileno(stdin), "r");
+ if (fp->x.bam == 0) goto open_err_ret;
+ fp->header = bam_header_read(fp->x.bam);
+ } else { // text
+ fp->x.tamr = sam_open(fn);
+ if (fp->x.tamr == 0) goto open_err_ret;
+ fp->header = sam_header_read(fp->x.tamr);
+ if (fp->header->n_targets == 0) { // no @SQ fields
+ if (aux) { // check if aux is present
+ bam_header_t *textheader = fp->header;
+ fp->header = sam_header_read2((const char*)aux);
+ if (fp->header == 0) goto open_err_ret;
+ append_header_text(fp->header, textheader->text, textheader->l_text);
+ bam_header_destroy(textheader);
+ }
+ if (fp->header->n_targets == 0 && bam_verbose >= 1)
+ fprintf(stderr, "[samopen] no @SQ lines in the header.\n");
+ } else if (bam_verbose >= 2) fprintf(stderr, "[samopen] SAM header is present: %d sequences.\n", fp->header->n_targets);
+ }
+ } else if (strchr(mode, 'w')) { // write
+ fprintf(stderr, "Write not supported here.\n");
+ exit(1);
+ }
+
+ return fp;
+
+open_err_ret:
+ free(fp);
+ return 0;
+}
+
+void samclose(samfile_t *fp)
+{
+ if (fp == 0) return;
+ if (fp->header) bam_header_destroy(fp->header);
+ if (fp->type & TYPE_BAM) bam_close(fp->x.bam);
+ else if (fp->type & TYPE_READ) sam_close(fp->x.tamr);
+ else fclose(fp->x.tamw);
+ free(fp);
+}
+
+int samread(samfile_t *fp, bam1_t *b)
+{
+ if (fp == 0 || !(fp->type & TYPE_READ)) return -1; // not open for reading
+ if (fp->type & TYPE_BAM) return bam_read1(fp->x.bam, b);
+ else return sam_read1(fp->x.tamr, fp->header, b);
+}
+#include <stdio.h>
+#include <ctype.h>
+#include <errno.h>
+#include <assert.h>
+//#include "bam.h"
+//#include "bam_endian.h"
+//#include "kstring.h"
+//#include "sam_header.h"
+
+int bam_is_be = 0, bam_verbose = 2, bam_no_B = 0;
+char *bam_flag2char_table = "pPuUrR12sfd\0\0\0\0\0";
+
+
+
+
+/**************************
+ * CIGAR related routines *
+ **************************/
+
+uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar)
+{
+ int k, end = c->pos;
+ for (k = 0; k < c->n_cigar; ++k) {
+ int op = bam_cigar_op(cigar[k]);
+ int len = bam_cigar_oplen(cigar[k]);
+ if (op == BAM_CBACK) { // move backward
+ int l, u, v;
+ if (k == c->n_cigar - 1) break; // skip trailing 'B'
+ for (l = k - 1, u = v = 0; l >= 0; --l) {
+ int op1 = bam_cigar_op(cigar[l]);
+ int len1 = bam_cigar_oplen(cigar[l]);
+ if (bam_cigar_type(op1)&1) { // consume query
+ if (u + len1 >= len) { // stop
+ if (bam_cigar_type(op1)&2) v += len - u;
+ break;
+ } else u += len1;
+ }
+ if (bam_cigar_type(op1)&2) v += len1;
+ }
+ end = l < 0? c->pos : end - v;
+ } else if (bam_cigar_type(op)&2) end += bam_cigar_oplen(cigar[k]);
+ }
+ return end;
+}
+
+int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar)
+{
+ uint32_t k;
+ int32_t l = 0;
+ for (k = 0; k < c->n_cigar; ++k)
+ if (bam_cigar_type(bam_cigar_op(cigar[k]))&1)
+ l += bam_cigar_oplen(cigar[k]);
+ return l;
+}
+
+/********************
+ * BAM I/O routines *
+ ********************/
+
+bam_header_t *bam_header_init()
+{
+ bam_is_be = bam_is_big_endian();
+ return (bam_header_t*)calloc(1, sizeof(bam_header_t));
+}
+
+void bam_header_destroy(bam_header_t *header)
+{
+ int32_t i;
+ //extern void bam_destroy_header_hash(bam_header_t *header);
+ if (header == 0) return;
+ if (header->target_name) {
+ for (i = 0; i < header->n_targets; ++i)
+ free(header->target_name[i]);
+ free(header->target_name);
+ free(header->target_len);
+ }
+ free(header->text);
+ if (header->dict) sam_header_free(header->dict);
+ if (header->rg2lib) sam_tbl_destroy(header->rg2lib);
+ //bam_destroy_header_hash(header);
+ free(header);
+}
+
+bam_header_t *bam_header_read(bamFile fp)
+{
+ bam_header_t *header;
+ char buf[4];
+ int magic_len;
+ int32_t i = 1, name_len;
+ // check EOF
+ i = bgzf_check_EOF(fp);
+ if (i < 0) {
+ // If the file is a pipe, checking the EOF marker will *always* fail
+ // with ESPIPE. Suppress the error message in this case.
+ if (errno != ESPIPE) perror("[bam_header_read] bgzf_check_EOF");
+ }
+ else if (i == 0) fprintf(stderr, "[bam_header_read] EOF marker is absent. The input is probably truncated.\n");
+ // read "BAM1"
+ magic_len = bam_read(fp, buf, 4);
+ if (magic_len != 4 || strncmp(buf, "BAM\001", 4) != 0) {
+ fprintf(stderr, "[bam_header_read] invalid BAM binary header (this is not a BAM file).\n");
+ return 0;
+ }
+ header = bam_header_init();
+ // read plain text and the number of reference sequences
+ bam_read(fp, &header->l_text, 4);
+ if (bam_is_be) bam_swap_endian_4p(&header->l_text);
+ header->text = (char*)calloc(header->l_text + 1, 1);
+ bam_read(fp, header->text, header->l_text);
+ bam_read(fp, &header->n_targets, 4);
+ if (bam_is_be) bam_swap_endian_4p(&header->n_targets);
+ // read reference sequence names and lengths
+ header->target_name = (char**)calloc(header->n_targets, sizeof(char*));
+ header->target_len = (uint32_t*)calloc(header->n_targets, 4);
+ for (i = 0; i != header->n_targets; ++i) {
+ bam_read(fp, &name_len, 4);
+ if (bam_is_be) bam_swap_endian_4p(&name_len);
+ header->target_name[i] = (char*)calloc(name_len, 1);
+ bam_read(fp, header->target_name[i], name_len);
+ bam_read(fp, &header->target_len[i], 4);
+ if (bam_is_be) bam_swap_endian_4p(&header->target_len[i]);
+ }
+ return header;
+}
+
+static void swap_endian_data(const bam1_core_t *c, int data_len, uint8_t *data)
+{
+ uint8_t *s;
+ uint32_t i, *cigar = (uint32_t*)(data + c->l_qname);
+ s = data + c->n_cigar*4 + c->l_qname + c->l_qseq + (c->l_qseq + 1)/2;
+ for (i = 0; i < c->n_cigar; ++i) bam_swap_endian_4p(&cigar[i]);
+ while (s < data + data_len) {
+ uint8_t type;
+ s += 2; // skip key
+ type = toupper(*s); ++s; // skip type
+ if (type == 'C' || type == 'A') ++s;
+ else if (type == 'S') { bam_swap_endian_2p(s); s += 2; }
+ else if (type == 'I' || type == 'F') { bam_swap_endian_4p(s); s += 4; }
+ else if (type == 'D') { bam_swap_endian_8p(s); s += 8; }
+ else if (type == 'Z' || type == 'H') { while (*s) ++s; ++s; }
+ else if (type == 'B') {
+ int32_t n, Bsize = bam_aux_type2size(*s);
+ memcpy(&n, s + 1, 4);
+ if (1 == Bsize) {
+ } else if (2 == Bsize) {
+ for (i = 0; i < n; i += 2)
+ bam_swap_endian_2p(s + 5 + i);
+ } else if (4 == Bsize) {
+ for (i = 0; i < n; i += 4)
+ bam_swap_endian_4p(s + 5 + i);
+ }
+ bam_swap_endian_4p(s+1);
+ }
+ }
+}
+
+int bam_read1(bamFile fp, bam1_t *b)
+{
+ bam1_core_t *c = &b->core;
+ int32_t block_len, ret, i;
+ uint32_t x[8];
+
+ assert(BAM_CORE_SIZE == 32);
+ if ((ret = bam_read(fp, &block_len, 4)) != 4) {
+ if (ret == 0) return -1; // normal end-of-file
+ else return -2; // truncated
+ }
+ if (bam_read(fp, x, BAM_CORE_SIZE) != BAM_CORE_SIZE) return -3;
+ if (bam_is_be) {
+ bam_swap_endian_4p(&block_len);
+ for (i = 0; i < 8; ++i) bam_swap_endian_4p(x + i);
+ }
+ c->tid = x[0]; c->pos = x[1];
+ c->bin = x[2]>>16; c->qual = x[2]>>8&0xff; c->l_qname = x[2]&0xff;
+ c->flag = x[3]>>16; c->n_cigar = x[3]&0xffff;
+ c->l_qseq = x[4];
+ c->mtid = x[5]; c->mpos = x[6]; c->isize = x[7];
+ b->data_len = block_len - BAM_CORE_SIZE;
+ if (b->m_data < b->data_len) {
+ b->m_data = b->data_len;
+ kroundup32(b->m_data);
+ b->data = (uint8_t*)realloc(b->data, b->m_data);
+ }
+ if (bam_read(fp, b->data, b->data_len) != b->data_len) return -4;
+ b->l_aux = b->data_len - c->n_cigar * 4 - c->l_qname - c->l_qseq - (c->l_qseq+1)/2;
+ if (bam_is_be) swap_endian_data(c, b->data_len, b->data);
+ if (bam_no_B) bam_remove_B(b);
+ return 4 + block_len;
+}
+
+
+/************
+ * Remove B *
+ ************/
+
+int bam_remove_B(bam1_t *b)
+{
+ int i, j, end_j, k, l, no_qual;
+ uint32_t *cigar, *new_cigar;
+ uint8_t *seq, *qual, *p;
+ // test if removal is necessary
+ if (b->core.flag & BAM_FUNMAP) return 0; // unmapped; do nothing
+ cigar = bam1_cigar(b);
+ for (k = 0; k < b->core.n_cigar; ++k)
+ if (bam_cigar_op(cigar[k]) == BAM_CBACK) break;
+ if (k == b->core.n_cigar) return 0; // no 'B'
+ if (bam_cigar_op(cigar[0]) == BAM_CBACK) goto rmB_err; // cannot be removed
+ // allocate memory for the new CIGAR
+ if (b->data_len + (b->core.n_cigar + 1) * 4 > b->m_data) { // not enough memory
+ b->m_data = b->data_len + b->core.n_cigar * 4;
+ kroundup32(b->m_data);
+ b->data = (uint8_t*)realloc(b->data, b->m_data);
+ cigar = bam1_cigar(b); // after realloc, cigar may be changed
+ }
+ new_cigar = (uint32_t*)(b->data + (b->m_data - b->core.n_cigar * 4)); // from the end of b->data
+ // the core loop
+ seq = bam1_seq(b); qual = bam1_qual(b);
+ no_qual = (qual[0] == 0xff); // test whether base quality is available
+ i = j = 0; end_j = -1;
+ for (k = l = 0; k < b->core.n_cigar; ++k) {
+ int op = bam_cigar_op(cigar[k]);
+ int len = bam_cigar_oplen(cigar[k]);
+ if (op == BAM_CBACK) { // the backward operation
+ int t, u;
+ if (k == b->core.n_cigar - 1) break; // ignore 'B' at the end of CIGAR
+ if (len > j) goto rmB_err; // an excessively long backward
+ for (t = l - 1, u = 0; t >= 0; --t) { // look back
+ int op1 = bam_cigar_op(new_cigar[t]);
+ int len1 = bam_cigar_oplen(new_cigar[t]);
+ if (bam_cigar_type(op1)&1) { // consume the query
+ if (u + len1 >= len) { // stop
+ new_cigar[t] -= (len - u) << BAM_CIGAR_SHIFT;
+ break;
+ } else u += len1;
+ }
+ }
+ if (bam_cigar_oplen(new_cigar[t]) == 0) --t; // squeeze out the zero-length operation
+ l = t + 1;
+ end_j = j; j -= len;
+ } else { // other CIGAR operations
+ new_cigar[l++] = cigar[k];
+ if (bam_cigar_type(op)&1) { // consume the query
+ if (i != j) { // no need to copy if i == j
+ int u, c, c0;
+ for (u = 0; u < len; ++u) { // construct the consensus
+ c = bam1_seqi(seq, i+u);
+ if (j + u < end_j) { // in an overlap
+ c0 = bam1_seqi(seq, j+u);
+ if (c != c0) { // a mismatch; choose the better base
+ if (qual[j+u] < qual[i+u]) { // the base in the 2nd segment is better
+ bam1_seq_seti(seq, j+u, c);
+ qual[j+u] = qual[i+u] - qual[j+u];
+ } else qual[j+u] -= qual[i+u]; // the 1st is better; reduce base quality
+ } else qual[j+u] = qual[j+u] > qual[i+u]? qual[j+u] : qual[i+u];
+ } else { // not in an overlap; copy over
+ bam1_seq_seti(seq, j+u, c);
+ qual[j+u] = qual[i+u];
+ }
+ }
+ }
+ i += len, j += len;
+ }
+ }
+ }
+ if (no_qual) qual[0] = 0xff; // in very rare cases, this may be modified
+ // merge adjacent operations if possible
+ for (k = 1; k < l; ++k)
+ if (bam_cigar_op(new_cigar[k]) == bam_cigar_op(new_cigar[k-1]))
+ new_cigar[k] += new_cigar[k-1] >> BAM_CIGAR_SHIFT << BAM_CIGAR_SHIFT, new_cigar[k-1] &= 0xf;
+ // kill zero length operations
+ for (k = i = 0; k < l; ++k)
+ if (new_cigar[k] >> BAM_CIGAR_SHIFT)
+ new_cigar[i++] = new_cigar[k];
+ l = i;
+ // update b
+ memcpy(cigar, new_cigar, l * 4); // set CIGAR
+ p = b->data + b->core.l_qname + l * 4;
+ memmove(p, seq, (j+1)>>1); p += (j+1)>>1; // set SEQ
+ memmove(p, qual, j); p += j; // set QUAL
+ memmove(p, bam1_aux(b), b->l_aux); p += b->l_aux; // set optional fields
+ b->core.n_cigar = l, b->core.l_qseq = j; // update CIGAR length and query length
+ b->data_len = p - b->data; // update record length
+ return 0;
+
+rmB_err:
+ b->core.flag |= BAM_FUNMAP;
+ return -1;
+}
+#include <ctype.h>
+//#include "bam.h"
+//#include "khash.h"
+typedef char *str_p;
+KHASH_MAP_INIT_STR(s, int)
+KHASH_MAP_INIT_STR(r2l, str_p)
+
+void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data)
+{
+ int ori_len = b->data_len;
+ b->data_len += 3 + len;
+ b->l_aux += 3 + len;
+ if (b->m_data < b->data_len) {
+ b->m_data = b->data_len;
+ kroundup32(b->m_data);
+ b->data = (uint8_t*)realloc(b->data, b->m_data);
+ }
+ b->data[ori_len] = tag[0]; b->data[ori_len + 1] = tag[1];
+ b->data[ori_len + 2] = type;
+ memcpy(b->data + ori_len + 3, data, len);
+}
+
+uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2])
+{
+ return bam_aux_get(b, tag);
+}
+
+#define __skip_tag(s) do { \
+ int type = toupper(*(s)); \
+ ++(s); \
+ if (type == 'Z' || type == 'H') { while (*(s)) ++(s); ++(s); } \
+ else if (type == 'B') (s) += 5 + bam_aux_type2size(*(s)) * (*(int32_t*)((s)+1)); \
+ else (s) += bam_aux_type2size(type); \
+ } while(0)
+
+uint8_t *bam_aux_get(const bam1_t *b, const char tag[2])
+{
+ uint8_t *s;
+ int y = tag[0]<<8 | tag[1];
+ s = bam1_aux(b);
+ while (s < b->data + b->data_len) {
+ int x = (int)s[0]<<8 | s[1];
+ s += 2;
+ if (x == y) return s;
+ __skip_tag(s);
+ }
+ return 0;
+}
+// s MUST BE returned by bam_aux_get()
+int bam_aux_del(bam1_t *b, uint8_t *s)
+{
+ uint8_t *p, *aux;
+ aux = bam1_aux(b);
+ p = s - 2;
+ __skip_tag(s);
+ memmove(p, s, b->l_aux - (s - aux));
+ b->data_len -= s - p;
+ b->l_aux -= s - p;
+ return 0;
+}
+
+int bam_aux_drop_other(bam1_t *b, uint8_t *s)
+{
+ if (s) {
+ uint8_t *p, *aux;
+ aux = bam1_aux(b);
+ p = s - 2;
+ __skip_tag(s);
+ memmove(aux, p, s - p);
+ b->data_len -= b->l_aux - (s - p);
+ b->l_aux = s - p;
+ } else {
+ b->data_len -= b->l_aux;
+ b->l_aux = 0;
+ }
+ return 0;
+}
+
+void bam_init_header_hash(bam_header_t *header)
+{
+ if (header->hash == 0) {
+ int ret, i;
+ khiter_t iter;
+ khash_t(s) *h;
+ header->hash = h = kh_init(s);
+ for (i = 0; i < header->n_targets; ++i) {
+ iter = kh_put(s, h, header->target_name[i], &ret);
+ kh_value(h, iter) = i;
+ }
+ }
+}
+
+void bam_destroy_header_hash(bam_header_t *header)
+{
+ if (header->hash)
+ kh_destroy(s, (khash_t(s)*)header->hash);
+}
+
+int32_t bam_get_tid(const bam_header_t *header, const char *seq_name)
+{
+ khint_t k;
+ khash_t(s) *h = (khash_t(s)*)header->hash;
+ k = kh_get(s, h, seq_name);
+ return k == kh_end(h)? -1 : kh_value(h, k);
+}
+
+int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *beg, int *end)
+{
+ char *s;
+ int i, l, k, name_end;
+ khiter_t iter;
+ khash_t(s) *h;
+
+ bam_init_header_hash(header);
+ h = (khash_t(s)*)header->hash;
+
+ *ref_id = *beg = *end = -1;
+ name_end = l = strlen(str);
+ s = (char*)malloc(l+1);
+ // remove space
+ for (i = k = 0; i < l; ++i)
+ if (!isspace(str[i])) s[k++] = str[i];
+ s[k] = 0; l = k;
+ // determine the sequence name
+ for (i = l - 1; i >= 0; --i) if (s[i] == ':') break; // look for colon from the end
+ if (i >= 0) name_end = i;
+ if (name_end < l) { // check if this is really the end
+ int n_hyphen = 0;
+ for (i = name_end + 1; i < l; ++i) {
+ if (s[i] == '-') ++n_hyphen;
+ else if (!isdigit(s[i]) && s[i] != ',') break;
+ }
+ if (i < l || n_hyphen > 1) name_end = l; // malformated region string; then take str as the name
+ s[name_end] = 0;
+ iter = kh_get(s, h, s);
+ if (iter == kh_end(h)) { // cannot find the sequence name
+ iter = kh_get(s, h, str); // try str as the name
+ if (iter == kh_end(h)) {
+ if (bam_verbose >= 2) fprintf(stderr, "[%s] fail to determine the sequence name.\n", __func__);
+ free(s); return -1;
+ } else s[name_end] = ':', name_end = l;
+ }
+ } else iter = kh_get(s, h, str);
+ if (iter == kh_end(h)) {
+ free(s);
+ return -1;
+ }
+ *ref_id = kh_val(h, iter);
+ // parse the interval
+ if (name_end < l) {
+ for (i = k = name_end + 1; i < l; ++i)
+ if (s[i] != ',') s[k++] = s[i];
+ s[k] = 0;
+ *beg = atoi(s + name_end + 1);
+ for (i = name_end + 1; i != k; ++i) if (s[i] == '-') break;
+ *end = i < k? atoi(s + i + 1) : 1<<29;
+ if (*beg > 0) --*beg;
+ } else *beg = 0, *end = 1<<29;
+ free(s);
+ return *beg <= *end? 0 : -1;
+}
+
+int32_t bam_aux2i(const uint8_t *s)
+{
+ int type;
+ if (s == 0) return 0;
+ type = *s++;
+ if (type == 'c') return (int32_t)*(int8_t*)s;
+ else if (type == 'C') return (int32_t)*(uint8_t*)s;
+ else if (type == 's') return (int32_t)*(int16_t*)s;
+ else if (type == 'S') return (int32_t)*(uint16_t*)s;
+ else if (type == 'i' || type == 'I') return *(int32_t*)s;
+ else return 0;
+}
+
+float bam_aux2f(const uint8_t *s)
+{
+ int type;
+ type = *s++;
+ if (s == 0) return 0.0;
+ if (type == 'f') return *(float*)s;
+ else return 0.0;
+}
+
+double bam_aux2d(const uint8_t *s)
+{
+ int type;
+ type = *s++;
+ if (s == 0) return 0.0;
+ if (type == 'd') return *(double*)s;
+ else return 0.0;
+}
+
+char bam_aux2A(const uint8_t *s)
+{
+ int type;
+ type = *s++;
+ if (s == 0) return 0;
+ if (type == 'A') return *(char*)s;
+ else return 0;
+}
+
+char *bam_aux2Z(const uint8_t *s)
+{
+ int type;
+ type = *s++;
+ if (s == 0) return 0;
+ if (type == 'Z' || type == 'H') return (char*)s;
+ else return 0;
+}
+
+#ifdef _WIN32
+double drand48()
+{
+ return (double)rand() / RAND_MAX;
+}
+#endif
+#include <zlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#ifdef _WIN32
+#include <fcntl.h>
+#endif
+//#include "kstring.h"
+//#include "bam.h"
+//#include "sam_header.h"
+//#include "kseq.h"
+//#include "khash.h"
+
+KSTREAM_INIT(gzFile, gzread, 16384)
+KHASH_MAP_INIT_STR(ref, uint64_t)
+
+void bam_init_header_hash(bam_header_t *header);
+void bam_destroy_header_hash(bam_header_t *header);
+int32_t bam_get_tid(const bam_header_t *header, const char *seq_name);
+
+unsigned char bam_nt16_table[256] = {
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 1, 2, 4, 8, 15,15,15,15, 15,15,15,15, 15, 0 /*=*/,15,15,
+ 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,
+ 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15,
+ 15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,
+ 15,15, 5, 6, 8,15, 7, 9, 15,10,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+ 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15
+};
+
+unsigned short bam_char2flag_table[256] = {
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,BAM_FREAD1,BAM_FREAD2,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ BAM_FPROPER_PAIR,0,BAM_FMREVERSE,0, 0,BAM_FMUNMAP,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, BAM_FDUP,0,BAM_FQCFAIL,0, 0,0,0,0, 0,0,0,0,
+ BAM_FPAIRED,0,BAM_FREVERSE,BAM_FSECONDARY, 0,BAM_FUNMAP,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
+};
+
+char *bam_nt16_rev_table = "=ACMGRSVTWYHKDBN";
+
+struct __tamFile_t {
+ gzFile fp;
+ kstream_t *ks;
+ kstring_t *str;
+ uint64_t n_lines;
+ int is_first;
+};
+
+char **__bam_get_lines(const char *fn, int *_n) // for bam_plcmd.c only
+{
+ char **list = 0, *s;
+ int n = 0, dret, m = 0;
+ gzFile fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r");
+ kstream_t *ks;
+ kstring_t *str;
+ str = (kstring_t*)calloc(1, sizeof(kstring_t));
+ ks = ks_init(fp);
+ while (ks_getuntil(ks, '\n', str, &dret) > 0) {
+ if (n == m) {
+ m = m? m << 1 : 16;
+ list = (char**)realloc(list, m * sizeof(char*));
+ }
+ if (str->s[str->l-1] == '\r')
+ str->s[--str->l] = '\0';
+ s = list[n++] = (char*)calloc(str->l + 1, 1);
+ strcpy(s, str->s);
+ }
+ ks_destroy(ks);
+ gzclose(fp);
+ free(str->s); free(str);
+ *_n = n;
+ return list;
+}
+
+static bam_header_t *hash2header(const kh_ref_t *hash)
+{
+ bam_header_t *header;
+ khiter_t k;
+ header = bam_header_init();
+ header->n_targets = kh_size(hash);
+ header->target_name = (char**)calloc(kh_size(hash), sizeof(char*));
+ header->target_len = (uint32_t*)calloc(kh_size(hash), 4);
+ for (k = kh_begin(hash); k != kh_end(hash); ++k) {
+ if (kh_exist(hash, k)) {
+ int i = (int)kh_value(hash, k);
+ header->target_name[i] = (char*)kh_key(hash, k);
+ header->target_len[i] = kh_value(hash, k)>>32;
+ }
+ }
+ bam_init_header_hash(header);
+ return header;
+}
+bam_header_t *sam_header_read2(const char *fn)
+{
+ bam_header_t *header;
+ int c, dret, ret, error = 0;
+ gzFile fp;
+ kstream_t *ks;
+ kstring_t *str;
+ kh_ref_t *hash;
+ khiter_t k;
+ if (fn == 0) return 0;
+ fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r");
+ if (fp == 0) return 0;
+ hash = kh_init(ref);
+ ks = ks_init(fp);
+ str = (kstring_t*)calloc(1, sizeof(kstring_t));
+ while (ks_getuntil(ks, 0, str, &dret) > 0) {
+ char *s = strdup(str->s);
+ int len, i;
+ i = kh_size(hash);
+ ks_getuntil(ks, 0, str, &dret);
+ len = atoi(str->s);
+ k = kh_put(ref, hash, s, &ret);
+ if (ret == 0) {
+ fprintf(stderr, "[sam_header_read2] duplicated sequence name: %s\n", s);
+ error = 1;
+ }
+ kh_value(hash, k) = (uint64_t)len<<32 | i;
+ if (dret != '\n')
+ while ((c = ks_getc(ks)) != '\n' && c != -1);
+ }
+ ks_destroy(ks);
+ gzclose(fp);
+ free(str->s); free(str);
+ fprintf(stderr, "[sam_header_read2] %d sequences loaded.\n", kh_size(hash));
+ if (error) return 0;
+ header = hash2header(hash);
+ kh_destroy(ref, hash);
+ return header;
+}
+static inline uint8_t *alloc_data(bam1_t *b, int size)
+{
+ if (b->m_data < size) {
+ b->m_data = size;
+ kroundup32(b->m_data);
+ b->data = (uint8_t*)realloc(b->data, b->m_data);
+ }
+ return b->data;
+}
+static inline void parse_error(int64_t n_lines, const char * __restrict msg)
+{
+ fprintf(stderr, "Parse error at line %lld: %s\n", (long long)n_lines, msg);
+ abort();
+}
+static inline void append_text(bam_header_t *header, kstring_t *str)
+{
+ size_t x = header->l_text, y = header->l_text + str->l + 2; // 2 = 1 byte dret + 1 byte null
+ kroundup32(x); kroundup32(y);
+ if (x < y)
+ {
+ header->n_text = y;
+ header->text = (char*)realloc(header->text, y);
+ if ( !header->text )
+ {
+ fprintf(stderr,"realloc failed to alloc %ld bytes\n", y);
+ abort();
+ }
+ }
+ // Sanity check
+ if ( header->l_text+str->l+1 >= header->n_text )
+ {
+ fprintf(stderr,"append_text FIXME: %ld>=%ld, x=%ld,y=%ld\n", header->l_text+str->l+1,(long)header->n_text,x,y);
+ abort();
+ }
+ strncpy(header->text + header->l_text, str->s, str->l+1); // we cannot use strcpy() here.
+ header->l_text += str->l + 1;
+ header->text[header->l_text] = 0;
+}
+
+int sam_header_parse(bam_header_t *h)
+{
+ char **tmp;
+ int i;
+ free(h->target_len); free(h->target_name);
+ h->n_targets = 0; h->target_len = 0; h->target_name = 0;
+ if (h->l_text < 3) return 0;
+ if (h->dict == 0) h->dict = sam_header_parse2(h->text);
+ tmp = sam_header2list(h->dict, "SQ", "SN", &h->n_targets);
+ if (h->n_targets == 0) return 0;
+ h->target_name = calloc(h->n_targets, sizeof(void*));
+ for (i = 0; i < h->n_targets; ++i)
+ h->target_name[i] = strdup(tmp[i]);
+ free(tmp);
+ tmp = sam_header2list(h->dict, "SQ", "LN", &h->n_targets);
+ h->target_len = calloc(h->n_targets, 4);
+ for (i = 0; i < h->n_targets; ++i)
+ h->target_len[i] = atoi(tmp[i]);
+ free(tmp);
+ return h->n_targets;
+}
+
+bam_header_t *sam_header_read(tamFile fp)
+{
+ int ret, dret;
+ bam_header_t *header = bam_header_init();
+ kstring_t *str = fp->str;
+ while ((ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret)) >= 0 && str->s[0] == '@') { // skip header
+ str->s[str->l] = dret; // note that str->s is NOT null terminated!!
+ append_text(header, str);
+ if (dret != '\n') {
+ ret = ks_getuntil(fp->ks, '\n', str, &dret);
+ str->s[str->l] = '\n'; // NOT null terminated!!
+ append_text(header, str);
+ }
+ ++fp->n_lines;
+ }
+ sam_header_parse(header);
+ bam_init_header_hash(header);
+ fp->is_first = 1;
+ return header;
+}
+
+int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b)
+{
+ int ret, doff, doff0, dret, z = 0;
+ bam1_core_t *c = &b->core;
+ kstring_t *str = fp->str;
+ kstream_t *ks = fp->ks;
+
+ if (fp->is_first) {
+ fp->is_first = 0;
+ ret = str->l;
+ } else {
+ do { // special consideration for empty lines
+ ret = ks_getuntil(fp->ks, KS_SEP_TAB, str, &dret);
+ if (ret >= 0) z += str->l + 1;
+ } while (ret == 0);
+ }
+ if (ret < 0) return -1;
+ ++fp->n_lines;
+ doff = 0;
+
+ { // name
+ c->l_qname = strlen(str->s) + 1;
+ memcpy(alloc_data(b, doff + c->l_qname) + doff, str->s, c->l_qname);
+ doff += c->l_qname;
+ }
+ { // flag
+ long flag;
+ char *s;
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
+ flag = strtol((char*)str->s, &s, 0);
+ if (*s) { // not the end of the string
+ flag = 0;
+ for (s = str->s; *s; ++s)
+ flag |= bam_char2flag_table[(int)*s];
+ }
+ c->flag = flag;
+ }
+ { // tid, pos, qual
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->tid = bam_get_tid(header, str->s);
+ if (c->tid < 0 && strcmp(str->s, "*")) {
+ if (header->n_targets == 0) {
+ fprintf(stderr, "[sam_read1] missing header? Abort!\n");
+ exit(1);
+ } else fprintf(stderr, "[sam_read1] reference '%s' is recognized as '*'.\n", str->s);
+ }
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->pos = isdigit(str->s[0])? atoi(str->s) - 1 : -1;
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1; c->qual = isdigit(str->s[0])? atoi(str->s) : 0;
+ if (ret < 0) return -2;
+ }
+ { // cigar
+ char *s, *t;
+ int i, op;
+ long x;
+ c->n_cigar = 0;
+ if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -3;
+ z += str->l + 1;
+ if (str->s[0] != '*') {
+ uint32_t *cigar;
+ for (s = str->s; *s; ++s) {
+ if ((isalpha(*s)) || (*s=='=')) ++c->n_cigar;
+ else if (!isdigit(*s)) parse_error(fp->n_lines, "invalid CIGAR character");
+ }
+ b->data = alloc_data(b, doff + c->n_cigar * 4);
+ cigar = bam1_cigar(b);
+ for (i = 0, s = str->s; i != c->n_cigar; ++i) {
+ x = strtol(s, &t, 10);
+ op = toupper(*t);
+ if (op == 'M') op = BAM_CMATCH;
+ else if (op == 'I') op = BAM_CINS;
+ else if (op == 'D') op = BAM_CDEL;
+ else if (op == 'N') op = BAM_CREF_SKIP;
+ else if (op == 'S') op = BAM_CSOFT_CLIP;
+ else if (op == 'H') op = BAM_CHARD_CLIP;
+ else if (op == 'P') op = BAM_CPAD;
+ else if (op == '=') op = BAM_CEQUAL;
+ else if (op == 'X') op = BAM_CDIFF;
+ else if (op == 'B') op = BAM_CBACK;
+ else parse_error(fp->n_lines, "invalid CIGAR operation");
+ s = t + 1;
+ cigar[i] = bam_cigar_gen(x, op);
+ }
+ if (*s) parse_error(fp->n_lines, "unmatched CIGAR operation");
+ c->bin = bam_reg2bin(c->pos, bam_calend(c, cigar));
+ doff += c->n_cigar * 4;
+ } else {
+ if (!(c->flag&BAM_FUNMAP)) {
+ fprintf(stderr, "Parse warning at line %lld: mapped sequence without CIGAR\n", (long long)fp->n_lines);
+ c->flag |= BAM_FUNMAP;
+ }
+ c->bin = bam_reg2bin(c->pos, c->pos + 1);
+ }
+ }
+ { // mtid, mpos, isize
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
+ c->mtid = strcmp(str->s, "=")? bam_get_tid(header, str->s) : c->tid;
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
+ c->mpos = isdigit(str->s[0])? atoi(str->s) - 1 : -1;
+ ret = ks_getuntil(ks, KS_SEP_TAB, str, &dret); z += str->l + 1;
+ c->isize = (str->s[0] == '-' || isdigit(str->s[0]))? atoi(str->s) : 0;
+ if (ret < 0) return -4;
+ }
+ { // seq and qual
+ int i;
+ uint8_t *p = 0;
+ if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -5; // seq
+ z += str->l + 1;
+ if (strcmp(str->s, "*")) {
+ c->l_qseq = strlen(str->s);
+ if (c->n_cigar && c->l_qseq != (int32_t)bam_cigar2qlen(c, bam1_cigar(b))) {
+ fprintf(stderr, "Line %ld, sequence length %i vs %i from CIGAR\n",
+ (long)fp->n_lines, c->l_qseq, (int32_t)bam_cigar2qlen(c, bam1_cigar(b)));
+ parse_error(fp->n_lines, "CIGAR and sequence length are inconsistent");
+ }
+ p = (uint8_t*)alloc_data(b, doff + c->l_qseq + (c->l_qseq+1)/2) + doff;
+ memset(p, 0, (c->l_qseq+1)/2);
+ for (i = 0; i < c->l_qseq; ++i)
+ p[i/2] |= bam_nt16_table[(int)str->s[i]] << 4*(1-i%2);
+ } else c->l_qseq = 0;
+ if (ks_getuntil(ks, KS_SEP_TAB, str, &dret) < 0) return -6; // qual
+ z += str->l + 1;
+ if (strcmp(str->s, "*") && c->l_qseq != strlen(str->s))
+ parse_error(fp->n_lines, "sequence and quality are inconsistent");
+ p += (c->l_qseq+1)/2;
+ if (strcmp(str->s, "*") == 0) for (i = 0; i < c->l_qseq; ++i) p[i] = 0xff;
+ else for (i = 0; i < c->l_qseq; ++i) p[i] = str->s[i] - 33;
+ doff += c->l_qseq + (c->l_qseq+1)/2;
+ }
+ doff0 = doff;
+ if (dret != '\n' && dret != '\r') { // aux
+ while (ks_getuntil(ks, KS_SEP_TAB, str, &dret) >= 0) {
+ uint8_t *s, type, key[2];
+ z += str->l + 1;
+ if (str->l < 6 || str->s[2] != ':' || str->s[4] != ':')
+ parse_error(fp->n_lines, "missing colon in auxiliary data");
+ key[0] = str->s[0]; key[1] = str->s[1];
+ type = str->s[3];
+ s = alloc_data(b, doff + 3) + doff;
+ s[0] = key[0]; s[1] = key[1]; s += 2; doff += 2;
+ if (type == 'A' || type == 'a' || type == 'c' || type == 'C') { // c and C for backward compatibility
+ s = alloc_data(b, doff + 2) + doff;
+ *s++ = 'A'; *s = str->s[5];
+ doff += 2;
+ } else if (type == 'I' || type == 'i') {
+ long long x;
+ s = alloc_data(b, doff + 5) + doff;
+ x = (long long)atoll(str->s + 5);
+ if (x < 0) {
+ if (x >= -127) {
+ *s++ = 'c'; *(int8_t*)s = (int8_t)x;
+ s += 1; doff += 2;
+ } else if (x >= -32767) {
+ *s++ = 's'; *(int16_t*)s = (int16_t)x;
+ s += 2; doff += 3;
+ } else {
+ *s++ = 'i'; *(int32_t*)s = (int32_t)x;
+ s += 4; doff += 5;
+ if (x < -2147483648ll)
+ fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.",
+ (long long)fp->n_lines, x);
+ }
+ } else {
+ if (x <= 255) {
+ *s++ = 'C'; *s++ = (uint8_t)x;
+ doff += 2;
+ } else if (x <= 65535) {
+ *s++ = 'S'; *(uint16_t*)s = (uint16_t)x;
+ s += 2; doff += 3;
+ } else {
+ *s++ = 'I'; *(uint32_t*)s = (uint32_t)x;
+ s += 4; doff += 5;
+ if (x > 4294967295ll)
+ fprintf(stderr, "Parse warning at line %lld: integer %lld is out of range.",
+ (long long)fp->n_lines, x);
+ }
+ }
+ } else if (type == 'f') {
+ s = alloc_data(b, doff + 5) + doff;
+ *s++ = 'f';
+ *(float*)s = (float)atof(str->s + 5);
+ s += 4; doff += 5;
+ } else if (type == 'd') {
+ s = alloc_data(b, doff + 9) + doff;
+ *s++ = 'd';
+ *(float*)s = (float)atof(str->s + 9);
+ s += 8; doff += 9;
+ } else if (type == 'Z' || type == 'H') {
+ int size = 1 + (str->l - 5) + 1;
+ if (type == 'H') { // check whether the hex string is valid
+ int i;
+ if ((str->l - 5) % 2 == 1) parse_error(fp->n_lines, "length of the hex string not even");
+ for (i = 0; i < str->l - 5; ++i) {
+ int c = toupper(str->s[5 + i]);
+ if (!((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F')))
+ parse_error(fp->n_lines, "invalid hex character");
+ }
+ }
+ s = alloc_data(b, doff + size) + doff;
+ *s++ = type;
+ memcpy(s, str->s + 5, str->l - 5);
+ s[str->l - 5] = 0;
+ doff += size;
+ } else if (type == 'B') {
+ int32_t n = 0, Bsize, k = 0, size;
+ char *p;
+ if (str->l < 8) parse_error(fp->n_lines, "too few values in aux type B");
+ Bsize = bam_aux_type2size(str->s[5]); // the size of each element
+ for (p = (char*)str->s + 6; *p; ++p) // count the number of elements in the array
+ if (*p == ',') ++n;
+ p = str->s + 7; // now p points to the first number in the array
+ size = 6 + Bsize * n; // total number of bytes allocated to this tag
+ s = alloc_data(b, doff + 6 * Bsize * n) + doff; // allocate memory
+ *s++ = 'B'; *s++ = str->s[5];
+ memcpy(s, &n, 4); s += 4; // write the number of elements
+ if (str->s[5] == 'c') while (p < str->s + str->l) ((int8_t*)s)[k++] = (int8_t)strtol(p, &p, 0), ++p;
+ else if (str->s[5] == 'C') while (p < str->s + str->l) ((uint8_t*)s)[k++] = (uint8_t)strtol(p, &p, 0), ++p;
+ else if (str->s[5] == 's') while (p < str->s + str->l) ((int16_t*)s)[k++] = (int16_t)strtol(p, &p, 0), ++p; // FIXME: avoid unaligned memory
+ else if (str->s[5] == 'S') while (p < str->s + str->l) ((uint16_t*)s)[k++] = (uint16_t)strtol(p, &p, 0), ++p;
+ else if (str->s[5] == 'i') while (p < str->s + str->l) ((int32_t*)s)[k++] = (int32_t)strtol(p, &p, 0), ++p;
+ else if (str->s[5] == 'I') while (p < str->s + str->l) ((uint32_t*)s)[k++] = (uint32_t)strtol(p, &p, 0), ++p;
+ else if (str->s[5] == 'f') while (p < str->s + str->l) ((float*)s)[k++] = (float)strtod(p, &p), ++p;
+ else parse_error(fp->n_lines, "unrecognized array type");
+ s += Bsize * n; doff += size;
+ } else parse_error(fp->n_lines, "unrecognized type");
+ if (dret == '\n' || dret == '\r') break;
+ }
+ }
+ b->l_aux = doff - doff0;
+ b->data_len = doff;
+ if (bam_no_B) bam_remove_B(b);
+ return z;
+}
+
+tamFile sam_open(const char *fn)
+{
+ tamFile fp;
+ gzFile gzfp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "rb") : gzopen(fn, "rb");
+ if (gzfp == 0) return 0;
+ fp = (tamFile)calloc(1, sizeof(struct __tamFile_t));
+ fp->str = (kstring_t*)calloc(1, sizeof(kstring_t));
+ fp->fp = gzfp;
+ fp->ks = ks_init(fp->fp);
+ return fp;
+}
+
+void sam_close(tamFile fp)
+{
+ if (fp) {
+ ks_destroy(fp->ks);
+ gzclose(fp->fp);
+ free(fp->str->s); free(fp->str);
+ free(fp);
+ }
+}
+/* The MIT License
+
+ Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
+ 2011 Attractive Chaos <attractor at live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <pthread.h>
+#include <sys/types.h>
+//#include "bgzf.h"
+
+#ifdef _USE_KNETFILE
+//#include "knetfile.h"
+typedef knetFile *_bgzf_file_t;
+#define _bgzf_open(fn, mode) knet_open(fn, mode)
+#define _bgzf_dopen(fp, mode) knet_dopen(fp, mode)
+#define _bgzf_close(fp) knet_close(fp)
+#define _bgzf_fileno(fp) ((fp)->fd)
+#define _bgzf_tell(fp) knet_tell(fp)
+#define _bgzf_seek(fp, offset, whence) knet_seek(fp, offset, whence)
+#define _bgzf_read(fp, buf, len) knet_read(fp, buf, len)
+#define _bgzf_write(fp, buf, len) knet_write(fp, buf, len)
+#else // ~defined(_USE_KNETFILE)
+#if defined(_WIN32) || defined(_MSC_VER)
+#define ftello(fp) ftell(fp)
+#define fseeko(fp, offset, whence) fseek(fp, offset, whence)
+#else // ~defined(_WIN32)
+extern off_t ftello(FILE *stream);
+extern int fseeko(FILE *stream, off_t offset, int whence);
+#endif // ~defined(_WIN32)
+typedef FILE *_bgzf_file_t;
+#define _bgzf_open(fn, mode) fopen(fn, mode)
+#define _bgzf_dopen(fp, mode) fdopen(fp, mode)
+#define _bgzf_close(fp) fclose(fp)
+#define _bgzf_fileno(fp) fileno(fp)
+#define _bgzf_tell(fp) ftello(fp)
+#define _bgzf_seek(fp, offset, whence) fseeko(fp, offset, whence)
+#define _bgzf_read(fp, buf, len) fread(buf, 1, len, fp)
+#define _bgzf_write(fp, buf, len) fwrite(buf, 1, len, fp)
+#endif // ~define(_USE_KNETFILE)
+
+#define BLOCK_HEADER_LENGTH 18
+#define BLOCK_FOOTER_LENGTH 8
+
+
+/* BGZF/GZIP header (speciallized from RFC 1952; little endian):
+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+ | 31|139| 8| 4| 0| 0|255| 6| 66| 67| 2|BLK_LEN|
+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+*/
+static const uint8_t g_magic[19] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\0\0";
+
+#ifdef BGZF_CACHE
+typedef struct {
+ int size;
+ uint8_t *block;
+ int64_t end_offset;
+} cache_t;
+//#include "khash.h"
+KHASH_MAP_INIT_INT64(cache, cache_t)
+#endif
+
+static inline void packInt16(uint8_t *buffer, uint16_t value)
+{
+ buffer[0] = value;
+ buffer[1] = value >> 8;
+}
+
+static inline int unpackInt16(const uint8_t *buffer)
+{
+ return buffer[0] | buffer[1] << 8;
+}
+
+static inline void packInt32(uint8_t *buffer, uint32_t value)
+{
+ buffer[0] = value;
+ buffer[1] = value >> 8;
+ buffer[2] = value >> 16;
+ buffer[3] = value >> 24;
+}
+
+static BGZF *bgzf_read_init()
+{
+ BGZF *fp;
+ fp = calloc(1, sizeof(BGZF));
+ fp->is_write = 0;
+ fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+ fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+#ifdef BGZF_CACHE
+ fp->cache = kh_init(cache);
+#endif
+ return fp;
+}
+
+static BGZF *bgzf_write_init(int compress_level) // compress_level==-1 for the default level
+{
+ BGZF *fp;
+ fp = calloc(1, sizeof(BGZF));
+ fp->is_write = 1;
+ fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+ fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+ fp->compress_level = compress_level < 0? Z_DEFAULT_COMPRESSION : compress_level; // Z_DEFAULT_COMPRESSION==-1
+ if (fp->compress_level > 9) fp->compress_level = Z_DEFAULT_COMPRESSION;
+ return fp;
+}
+// get the compress level from the mode string
+static int mode2level(const char *__restrict mode)
+{
+ int i, compress_level = -1;
+ for (i = 0; mode[i]; ++i)
+ if (mode[i] >= '0' && mode[i] <= '9') break;
+ if (mode[i]) compress_level = (int)mode[i] - '0';
+ if (strchr(mode, 'u')) compress_level = 0;
+ return compress_level;
+}
+
+BGZF *bgzf_open(const char *path, const char *mode)
+{
+ BGZF *fp = 0;
+ assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);
+ if (strchr(mode, 'r') || strchr(mode, 'R')) {
+ _bgzf_file_t fpr;
+ if ((fpr = _bgzf_open(path, "r")) == 0) return 0;
+ fp = bgzf_read_init();
+ fp->fp = fpr;
+ } else if (strchr(mode, 'w') || strchr(mode, 'W')) {
+ FILE *fpw;
+ if ((fpw = fopen(path, "w")) == 0) return 0;
+ fp = bgzf_write_init(mode2level(mode));
+ fp->fp = fpw;
+ }
+ return fp;
+}
+
+BGZF *bgzf_dopen(int fd, const char *mode)
+{
+ BGZF *fp = 0;
+ assert(compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE);
+ if (strchr(mode, 'r') || strchr(mode, 'R')) {
+ _bgzf_file_t fpr;
+ if ((fpr = _bgzf_dopen(fd, "r")) == 0) return 0;
+ fp = bgzf_read_init();
+ fp->fp = fpr;
+ } else if (strchr(mode, 'w') || strchr(mode, 'W')) {
+ FILE *fpw;
+ if ((fpw = fdopen(fd, "w")) == 0) return 0;
+ fp = bgzf_write_init(mode2level(mode));
+ fp->fp = fpw;
+ }
+ return fp;
+}
+
+static int bgzf_compress(void *_dst, int *dlen, void *src, int slen, int level)
+{
+ uint32_t crc;
+ z_stream zs;
+ uint8_t *dst = (uint8_t*)_dst;
+
+ // compress the body
+ zs.zalloc = NULL; zs.zfree = NULL;
+ zs.next_in = src;
+ zs.avail_in = slen;
+ zs.next_out = dst + BLOCK_HEADER_LENGTH;
+ zs.avail_out = *dlen - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH;
+ if (deflateInit2(&zs, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) return -1; // -15 to disable zlib header/footer
+ if (deflate(&zs, Z_FINISH) != Z_STREAM_END) return -1;
+ if (deflateEnd(&zs) != Z_OK) return -1;
+ *dlen = zs.total_out + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
+ // write the header
+ memcpy(dst, g_magic, BLOCK_HEADER_LENGTH); // the last two bytes are a place holder for the length of the block
+ packInt16(&dst[16], *dlen - 1); // write the compressed length; -1 to fit 2 bytes
+ // write the footer
+ crc = crc32(crc32(0L, NULL, 0L), src, slen);
+ packInt32((uint8_t*)&dst[*dlen - 8], crc);
+ packInt32((uint8_t*)&dst[*dlen - 4], slen);
+ return 0;
+}
+
+// Deflate the block in fp->uncompressed_block into fp->compressed_block. Also adds an extra field that stores the compressed block length.
+static int deflate_block(BGZF *fp, int block_length)
+{
+ int comp_size = BGZF_MAX_BLOCK_SIZE;
+ if (bgzf_compress(fp->compressed_block, &comp_size, fp->uncompressed_block, block_length, fp->compress_level) != 0) {
+ fp->errcode |= BGZF_ERR_ZLIB;
+ return -1;
+ }
+ fp->block_offset = 0;
+ return comp_size;
+}
+
+// Inflate the block in fp->compressed_block into fp->uncompressed_block
+static int inflate_block(BGZF* fp, int block_length)
+{
+ z_stream zs;
+ zs.zalloc = NULL;
+ zs.zfree = NULL;
+ zs.next_in = fp->compressed_block + 18;
+ zs.avail_in = block_length - 16;
+ zs.next_out = fp->uncompressed_block;
+ zs.avail_out = BGZF_MAX_BLOCK_SIZE;
+
+ if (inflateInit2(&zs, -15) != Z_OK) {
+ fp->errcode |= BGZF_ERR_ZLIB;
+ return -1;
+ }
+ if (inflate(&zs, Z_FINISH) != Z_STREAM_END) {
+ inflateEnd(&zs);
+ fp->errcode |= BGZF_ERR_ZLIB;
+ return -1;
+ }
+ if (inflateEnd(&zs) != Z_OK) {
+ fp->errcode |= BGZF_ERR_ZLIB;
+ return -1;
+ }
+ return zs.total_out;
+}
+
+static int check_header(const uint8_t *header)
+{
+ return (header[0] == 31 && header[1] == 139 && header[2] == 8 && (header[3] & 4) != 0
+ && unpackInt16((uint8_t*)&header[10]) == 6
+ && header[12] == 'B' && header[13] == 'C'
+ && unpackInt16((uint8_t*)&header[14]) == 2);
+}
+
+#ifdef BGZF_CACHE
+static void free_cache(BGZF *fp)
+{
+ khint_t k;
+ khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+ if (fp->is_write) return;
+ for (k = kh_begin(h); k < kh_end(h); ++k)
+ if (kh_exist(h, k)) free(kh_val(h, k).block);
+ kh_destroy(cache, h);
+}
+
+static int load_block_from_cache(BGZF *fp, int64_t block_address)
+{
+ khint_t k;
+ cache_t *p;
+ khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+ k = kh_get(cache, h, block_address);
+ if (k == kh_end(h)) return 0;
+ p = &kh_val(h, k);
+ if (fp->block_length != 0) fp->block_offset = 0;
+ fp->block_address = block_address;
+ fp->block_length = p->size;
+ memcpy(fp->uncompressed_block, p->block, BGZF_MAX_BLOCK_SIZE);
+ _bgzf_seek((_bgzf_file_t)fp->fp, p->end_offset, SEEK_SET);
+ return p->size;
+}
+
+static void cache_block(BGZF *fp, int size)
+{
+ int ret;
+ khint_t k;
+ cache_t *p;
+ khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+ if (BGZF_MAX_BLOCK_SIZE >= fp->cache_size) return;
+ if ((kh_size(h) + 1) * BGZF_MAX_BLOCK_SIZE > fp->cache_size) {
+ /* A better way would be to remove the oldest block in the
+ * cache, but here we remove a random one for simplicity. This
+ * should not have a big impact on performance. */
+ for (k = kh_begin(h); k < kh_end(h); ++k)
+ if (kh_exist(h, k)) break;
+ if (k < kh_end(h)) {
+ free(kh_val(h, k).block);
+ kh_del(cache, h, k);
+ }
+ }
+ k = kh_put(cache, h, fp->block_address, &ret);
+ if (ret == 0) return; // if this happens, a bug!
+ p = &kh_val(h, k);
+ p->size = fp->block_length;
+ p->end_offset = fp->block_address + size;
+ p->block = malloc(BGZF_MAX_BLOCK_SIZE);
+ memcpy(kh_val(h, k).block, fp->uncompressed_block, BGZF_MAX_BLOCK_SIZE);
+}
+#else
+static void free_cache(BGZF *fp) {}
+static int load_block_from_cache(BGZF *fp, int64_t block_address) {return 0;}
+static void cache_block(BGZF *fp, int size) {}
+#endif
+
+int bgzf_read_block(BGZF *fp)
+{
+ uint8_t header[BLOCK_HEADER_LENGTH], *compressed_block;
+ int count, size = 0, block_length, remaining;
+ int64_t block_address;
+ block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+ if (fp->cache_size && load_block_from_cache(fp, block_address)) return 0;
+ count = _bgzf_read(fp->fp, header, sizeof(header));
+ if (count == 0) { // no data read
+ fp->block_length = 0;
+ return 0;
+ }
+ if (count != sizeof(header) || !check_header(header)) {
+ fp->errcode |= BGZF_ERR_HEADER;
+ return -1;
+ }
+ size = count;
+ block_length = unpackInt16((uint8_t*)&header[16]) + 1; // +1 because when writing this number, we used "-1"
+ compressed_block = (uint8_t*)fp->compressed_block;
+ memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);
+ remaining = block_length - BLOCK_HEADER_LENGTH;
+ count = _bgzf_read(fp->fp, &compressed_block[BLOCK_HEADER_LENGTH], remaining);
+ if (count != remaining) {
+ fp->errcode |= BGZF_ERR_IO;
+ return -1;
+ }
+ size += count;
+ if ((count = inflate_block(fp, block_length)) < 0) return -1;
+ if (fp->block_length != 0) fp->block_offset = 0; // Do not reset offset if this read follows a seek.
+ fp->block_address = block_address;
+ fp->block_length = count;
+ cache_block(fp, size);
+ return 0;
+}
+
+ssize_t bgzf_read(BGZF *fp, void *data, ssize_t length)
+{
+ ssize_t bytes_read = 0;
+ uint8_t *output = data;
+ if (length <= 0) return 0;
+ assert(fp->is_write == 0);
+ while (bytes_read < length) {
+ int copy_length, available = fp->block_length - fp->block_offset;
+ uint8_t *buffer;
+ if (available <= 0) {
+ if (bgzf_read_block(fp) != 0) return -1;
+ available = fp->block_length - fp->block_offset;
+ if (available <= 0) break;
+ }
+ copy_length = length - bytes_read < available? length - bytes_read : available;
+ buffer = fp->uncompressed_block;
+ memcpy(output, buffer + fp->block_offset, copy_length);
+ fp->block_offset += copy_length;
+ output += copy_length;
+ bytes_read += copy_length;
+ }
+ if (fp->block_offset == fp->block_length) {
+ fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+ fp->block_offset = fp->block_length = 0;
+ }
+ return bytes_read;
+}
+
+/***** BEGIN: multi-threading *****/
+
+typedef struct {
+ BGZF *fp;
+ struct mtaux_t *mt;
+ void *buf;
+ int i, errcode, toproc;
+} worker_t;
+
+typedef struct mtaux_t {
+ int n_threads, n_blks, curr, done;
+ volatile int proc_cnt;
+ void **blk;
+ int *len;
+ worker_t *w;
+ pthread_t *tid;
+ pthread_mutex_t lock;
+ pthread_cond_t cv;
+} mtaux_t;
+
+static int worker_aux(worker_t *w)
+{
+ int i, tmp, stop = 0;
+ // wait for condition: to process or all done
+ pthread_mutex_lock(&w->mt->lock);
+ while (!w->toproc && !w->mt->done)
+ pthread_cond_wait(&w->mt->cv, &w->mt->lock);
+ if (w->mt->done) stop = 1;
+ w->toproc = 0;
+ pthread_mutex_unlock(&w->mt->lock);
+ if (stop) return 1; // to quit the thread
+ w->errcode = 0;
+ for (i = w->i; i < w->mt->curr; i += w->mt->n_threads) {
+ int clen = BGZF_MAX_BLOCK_SIZE;
+ if (bgzf_compress(w->buf, &clen, w->mt->blk[i], w->mt->len[i], w->fp->compress_level) != 0)
+ w->errcode |= BGZF_ERR_ZLIB;
+ memcpy(w->mt->blk[i], w->buf, clen);
+ w->mt->len[i] = clen;
+ }
+ tmp = __sync_fetch_and_add(&w->mt->proc_cnt, 1);
+ return 0;
+}
+
+static void *mt_worker(void *data)
+{
+ while (worker_aux(data) == 0);
+ return 0;
+}
+
+int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks)
+{
+ int i;
+ mtaux_t *mt;
+ pthread_attr_t attr;
+ if (!fp->is_write || fp->mt || n_threads <= 1) return -1;
+ mt = calloc(1, sizeof(mtaux_t));
+ mt->n_threads = n_threads;
+ mt->n_blks = n_threads * n_sub_blks;
+ mt->len = calloc(mt->n_blks, sizeof(int));
+ mt->blk = calloc(mt->n_blks, sizeof(void*));
+ for (i = 0; i < mt->n_blks; ++i)
+ mt->blk[i] = malloc(BGZF_MAX_BLOCK_SIZE);
+ mt->tid = calloc(mt->n_threads, sizeof(pthread_t)); // tid[0] is not used, as the worker 0 is launched by the master
+ mt->w = calloc(mt->n_threads, sizeof(worker_t));
+ for (i = 0; i < mt->n_threads; ++i) {
+ mt->w[i].i = i;
+ mt->w[i].mt = mt;
+ mt->w[i].fp = fp;
+ mt->w[i].buf = malloc(BGZF_MAX_BLOCK_SIZE);
+ }
+ pthread_attr_init(&attr);
+ pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+ pthread_mutex_init(&mt->lock, 0);
+ pthread_cond_init(&mt->cv, 0);
+ for (i = 1; i < mt->n_threads; ++i) // worker 0 is effectively launched by the master thread
+ pthread_create(&mt->tid[i], &attr, mt_worker, &mt->w[i]);
+ fp->mt = mt;
+ return 0;
+}
+
+static void mt_destroy(mtaux_t *mt)
+{
+ int i;
+ // signal all workers to quit
+ pthread_mutex_lock(&mt->lock);
+ mt->done = 1; mt->proc_cnt = 0;
+ pthread_cond_broadcast(&mt->cv);
+ pthread_mutex_unlock(&mt->lock);
+ for (i = 1; i < mt->n_threads; ++i) pthread_join(mt->tid[i], 0); // worker 0 is effectively launched by the master thread
+ // free other data allocated on heap
+ for (i = 0; i < mt->n_blks; ++i) free(mt->blk[i]);
+ for (i = 0; i < mt->n_threads; ++i) free(mt->w[i].buf);
+ free(mt->blk); free(mt->len); free(mt->w); free(mt->tid);
+ pthread_cond_destroy(&mt->cv);
+ pthread_mutex_destroy(&mt->lock);
+ free(mt);
+}
+
+static void mt_queue(BGZF *fp)
+{
+ mtaux_t *mt = (mtaux_t*)fp->mt;
+ assert(mt->curr < mt->n_blks); // guaranteed by the caller
+ memcpy(mt->blk[mt->curr], fp->uncompressed_block, fp->block_offset);
+ mt->len[mt->curr] = fp->block_offset;
+ fp->block_offset = 0;
+ ++mt->curr;
+}
+
+static int mt_flush(BGZF *fp)
+{
+ int i;
+ mtaux_t *mt = (mtaux_t*)fp->mt;
+ if (fp->block_offset) mt_queue(fp); // guaranteed that assertion does not fail
+ // signal all the workers to compress
+ pthread_mutex_lock(&mt->lock);
+ for (i = 0; i < mt->n_threads; ++i) mt->w[i].toproc = 1;
+ mt->proc_cnt = 0;
+ pthread_cond_broadcast(&mt->cv);
+ pthread_mutex_unlock(&mt->lock);
+ // worker 0 is doing things here
+ worker_aux(&mt->w[0]);
+ // wait for all the threads to complete
+ while (mt->proc_cnt < mt->n_threads);
+ // dump data to disk
+ for (i = 0; i < mt->n_threads; ++i) fp->errcode |= mt->w[i].errcode;
+ for (i = 0; i < mt->curr; ++i)
+ if (fwrite(mt->blk[i], 1, mt->len[i], fp->fp) != mt->len[i])
+ fp->errcode |= BGZF_ERR_IO;
+ mt->curr = 0;
+ return 0;
+}
+
+static int mt_lazy_flush(BGZF *fp)
+{
+ mtaux_t *mt = (mtaux_t*)fp->mt;
+ if (fp->block_offset) mt_queue(fp);
+ if (mt->curr == mt->n_blks)
+ return mt_flush(fp);
+ return -1;
+}
+
+static ssize_t mt_write(BGZF *fp, const void *data, ssize_t length)
+{
+ const uint8_t *input = data;
+ ssize_t rest = length;
+ while (rest) {
+ int copy_length = BGZF_BLOCK_SIZE - fp->block_offset < rest? BGZF_BLOCK_SIZE - fp->block_offset : rest;
+ memcpy(fp->uncompressed_block + fp->block_offset, input, copy_length);
+ fp->block_offset += copy_length; input += copy_length; rest -= copy_length;
+ if (fp->block_offset == BGZF_BLOCK_SIZE) mt_lazy_flush(fp);
+ }
+ return length - rest;
+}
+
+/***** END: multi-threading *****/
+
+int bgzf_flush(BGZF *fp)
+{
+ if (!fp->is_write) return 0;
+ if (fp->mt) return mt_flush(fp);
+ while (fp->block_offset > 0) {
+ int block_length;
+ block_length = deflate_block(fp, fp->block_offset);
+ if (block_length < 0) return -1;
+ if (fwrite(fp->compressed_block, 1, block_length, fp->fp) != block_length) {
+ fp->errcode |= BGZF_ERR_IO; // possibly truncated file
+ return -1;
+ }
+ fp->block_address += block_length;
+ }
+ return 0;
+}
+
+int bgzf_flush_try(BGZF *fp, ssize_t size)
+{
+ if (fp->block_offset + size > BGZF_BLOCK_SIZE) {
+ if (fp->mt) return mt_lazy_flush(fp);
+ else return bgzf_flush(fp);
+ }
+ return -1;
+}
+
+ssize_t bgzf_write(BGZF *fp, const void *data, ssize_t length)
+{
+ const uint8_t *input = data;
+ int block_length = BGZF_BLOCK_SIZE, bytes_written = 0;
+ assert(fp->is_write);
+ if (fp->mt) return mt_write(fp, data, length);
+ while (bytes_written < length) {
+ uint8_t* buffer = fp->uncompressed_block;
+ int copy_length = block_length - fp->block_offset < length - bytes_written? block_length - fp->block_offset : length - bytes_written;
+ memcpy(buffer + fp->block_offset, input, copy_length);
+ fp->block_offset += copy_length;
+ input += copy_length;
+ bytes_written += copy_length;
+ if (fp->block_offset == block_length && bgzf_flush(fp)) break;
+ }
+ return bytes_written;
+}
+
+int bgzf_close(BGZF* fp)
+{
+ int ret, count, block_length;
+ if (fp == 0) return -1;
+ if (fp->is_write) {
+ if (bgzf_flush(fp) != 0) return -1;
+ fp->compress_level = -1;
+ block_length = deflate_block(fp, 0); // write an empty block
+ count = fwrite(fp->compressed_block, 1, block_length, fp->fp);
+ if (fflush(fp->fp) != 0) {
+ fp->errcode |= BGZF_ERR_IO;
+ return -1;
+ }
+ if (fp->mt) mt_destroy(fp->mt);
+ }
+ ret = fp->is_write? fclose(fp->fp) : _bgzf_close(fp->fp);
+ if (ret != 0) return -1;
+ free(fp->uncompressed_block);
+ free(fp->compressed_block);
+ free_cache(fp);
+ free(fp);
+ return 0;
+}
+
+void bgzf_set_cache_size(BGZF *fp, int cache_size)
+{
+ if (fp) fp->cache_size = cache_size;
+}
+
+int bgzf_check_EOF(BGZF *fp)
+{
+ static uint8_t magic[28] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0";
+ uint8_t buf[28];
+ off_t offset;
+ offset = _bgzf_tell((_bgzf_file_t)fp->fp);
+ if (_bgzf_seek(fp->fp, -28, SEEK_END) < 0) return 0;
+ _bgzf_read(fp->fp, buf, 28);
+ _bgzf_seek(fp->fp, offset, SEEK_SET);
+ return (memcmp(magic, buf, 28) == 0)? 1 : 0;
+}
+
+int64_t bgzf_seek(BGZF* fp, int64_t pos, int where)
+{
+ int block_offset;
+ int64_t block_address;
+
+ if (fp->is_write || where != SEEK_SET) {
+ fp->errcode |= BGZF_ERR_MISUSE;
+ return -1;
+ }
+ block_offset = pos & 0xFFFF;
+ block_address = pos >> 16;
+ if (_bgzf_seek(fp->fp, block_address, SEEK_SET) < 0) {
+ fp->errcode |= BGZF_ERR_IO;
+ return -1;
+ }
+ fp->block_length = 0; // indicates current block has not been loaded
+ fp->block_address = block_address;
+ fp->block_offset = block_offset;
+ return 0;
+}
+
+int bgzf_is_bgzf(const char *fn)
+{
+ uint8_t buf[16];
+ int n;
+ _bgzf_file_t fp;
+ if ((fp = _bgzf_open(fn, "r")) == 0) return 0;
+ n = _bgzf_read(fp, buf, 16);
+ _bgzf_close(fp);
+ if (n != 16) return 0;
+ return memcmp(g_magic, buf, 16) == 0? 1 : 0;
+}
+
+int bgzf_getc(BGZF *fp)
+{
+ int c;
+ if (fp->block_offset >= fp->block_length) {
+ if (bgzf_read_block(fp) != 0) return -2; /* error */
+ if (fp->block_length == 0) return -1; /* end-of-file */
+ }
+ c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];
+ if (fp->block_offset == fp->block_length) {
+ fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+ fp->block_offset = 0;
+ fp->block_length = 0;
+ }
+ return c;
+}
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+int bgzf_getline(BGZF *fp, int delim, kstring_t *str)
+{
+ int l, state = 0;
+ unsigned char *buf = (unsigned char*)fp->uncompressed_block;
+ str->l = 0;
+ do {
+ if (fp->block_offset >= fp->block_length) {
+ if (bgzf_read_block(fp) != 0) { state = -2; break; }
+ if (fp->block_length == 0) { state = -1; break; }
+ }
+ for (l = fp->block_offset; l < fp->block_length && buf[l] != delim; ++l);
+ if (l < fp->block_length) state = 1;
+ l -= fp->block_offset;
+ if (str->l + l + 1 >= str->m) {
+ str->m = str->l + l + 2;
+ kroundup32(str->m);
+ str->s = (char*)realloc(str->s, str->m);
+ }
+ memcpy(str->s + str->l, buf + fp->block_offset, l);
+ str->l += l;
+ fp->block_offset += l + 1;
+ if (fp->block_offset >= fp->block_length) {
+ fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+ fp->block_offset = 0;
+ fp->block_length = 0;
+ }
+ } while (state == 0);
+ if (str->l == 0 && state < 0) return state;
+ str->s[str->l] = 0;
+ return str->l;
+}
+//#include "sam_header.h"
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdarg.h>
+
+//#include "khash.h"
+KHASH_MAP_INIT_STR(str, const char *)
+
+struct _HeaderList
+{
+ struct _HeaderList *last; // Hack: Used and maintained only by list_append_to_end. Maintained in the root node only.
+ struct _HeaderList *next;
+ void *data;
+};
+typedef struct _HeaderList list_t;
+typedef list_t HeaderDict;
+
+typedef struct
+{
+ char key[2];
+ char *value;
+}
+HeaderTag;
+
+typedef struct
+{
+ char type[2];
+ list_t *tags;
+}
+HeaderLine;
+
+const char *o_hd_tags[] = {"SO","GO",NULL};
+const char *r_hd_tags[] = {"VN",NULL};
+
+const char *o_sq_tags[] = {"AS","M5","UR","SP",NULL};
+const char *r_sq_tags[] = {"SN","LN",NULL};
+const char *u_sq_tags[] = {"SN",NULL};
+
+const char *o_rg_tags[] = {"CN","DS","DT","FO","KS","LB","PG","PI","PL","PU","SM",NULL};
+const char *r_rg_tags[] = {"ID",NULL};
+const char *u_rg_tags[] = {"ID",NULL};
+
+const char *o_pg_tags[] = {"VN","CL",NULL};
+const char *r_pg_tags[] = {"ID",NULL};
+
+const char *types[] = {"HD","SQ","RG","PG","CO",NULL};
+const char **optional_tags[] = {o_hd_tags,o_sq_tags,o_rg_tags,o_pg_tags,NULL,NULL};
+const char **required_tags[] = {r_hd_tags,r_sq_tags,r_rg_tags,r_pg_tags,NULL,NULL};
+const char **unique_tags[] = {NULL, u_sq_tags,u_rg_tags,NULL,NULL,NULL};
+
+
+static void debug(const char *format, ...)
+{
+ va_list ap;
+ va_start(ap, format);
+ vfprintf(stderr, format, ap);
+ va_end(ap);
+}
+
+#if 0
+// Replaced by list_append_to_end
+static list_t *list_prepend(list_t *root, void *data)
+{
+ list_t *l = malloc(sizeof(list_t));
+ l->next = root;
+ l->data = data;
+ return l;
+}
+#endif
+
+// Relies on the root->last being correct. Do not use with the other list_*
+// routines unless they are fixed to modify root->last as well.
+static list_t *list_append_to_end(list_t *root, void *data)
+{
+ list_t *l = malloc(sizeof(list_t));
+ l->last = l;
+ l->next = NULL;
+ l->data = data;
+
+ if ( !root )
+ return l;
+
+ root->last->next = l;
+ root->last = l;
+ return root;
+}
+
+static list_t *list_append(list_t *root, void *data)
+{
+ list_t *l = root;
+ while (l && l->next)
+ l = l->next;
+ if ( l )
+ {
+ l->next = malloc(sizeof(list_t));
+ l = l->next;
+ }
+ else
+ {
+ l = malloc(sizeof(list_t));
+ root = l;
+ }
+ l->data = data;
+ l->next = NULL;
+ return root;
+}
+
+static void list_free(list_t *root)
+{
+ list_t *l = root;
+ while (root)
+ {
+ l = root;
+ root = root->next;
+ free(l);
+ }
+}
+
+
+
+// Look for a tag "XY" in a predefined const char *[] array.
+static int tag_exists(const char *tag, const char **tags)
+{
+ int itag=0;
+ if ( !tags ) return -1;
+ while ( tags[itag] )
+ {
+ if ( tags[itag][0]==tag[0] && tags[itag][1]==tag[1] ) return itag;
+ itag++;
+ }
+ return -1;
+}
+
+
+
+// Mimics the behaviour of getline, except it returns pointer to the next chunk of the text
+// or NULL if everything has been read. The lineptr should be freed by the caller. The
+// newline character is stripped.
+static const char *nextline(char **lineptr, size_t *n, const char *text)
+{
+ int len;
+ const char *to = text;
+
+ if ( !*to ) return NULL;
+
+ while ( *to && *to!='\n' && *to!='\r' ) to++;
+ len = to - text + 1;
+
+ if ( *to )
+ {
+ // Advance the pointer for the next call
+ if ( *to=='\n' ) to++;
+ else if ( *to=='\r' && *(to+1)=='\n' ) to+=2;
+ }
+ if ( !len )
+ return to;
+
+ if ( !*lineptr )
+ {
+ *lineptr = malloc(len);
+ *n = len;
+ }
+ else if ( *n<len )
+ {
+ *lineptr = realloc(*lineptr, len);
+ *n = len;
+ }
+ if ( !*lineptr ) {
+ debug("[nextline] Insufficient memory!\n");
+ return 0;
+ }
+
+ memcpy(*lineptr,text,len);
+ (*lineptr)[len-1] = 0;
+
+ return to;
+}
+
+// name points to "XY", value_from points to the first character of the value string and
+// value_to points to the last character of the value string.
+static HeaderTag *new_tag(const char *name, const char *value_from, const char *value_to)
+{
+ HeaderTag *tag = malloc(sizeof(HeaderTag));
+ int len = value_to-value_from+1;
+
+ tag->key[0] = name[0];
+ tag->key[1] = name[1];
+ tag->value = malloc(len+1);
+ memcpy(tag->value,value_from,len+1);
+ tag->value[len] = 0;
+ return tag;
+}
+
+static HeaderTag *header_line_has_tag(HeaderLine *hline, const char *key)
+{
+ list_t *tags = hline->tags;
+ while (tags)
+ {
+ HeaderTag *tag = tags->data;
+ if ( tag->key[0]==key[0] && tag->key[1]==key[1] ) return tag;
+ tags = tags->next;
+ }
+ return NULL;
+}
+
+
+// Return codes:
+// 0 .. different types or unique tags differ or conflicting tags, cannot be merged
+// 1 .. all tags identical -> no need to merge, drop one
+// 2 .. the unique tags match and there are some conflicting tags (same tag, different value) -> error, cannot be merged nor duplicated
+// 3 .. there are some missing complementary tags and no unique conflict -> can be merged into a single line
+static int sam_header_compare_lines(HeaderLine *hline1, HeaderLine *hline2)
+{
+ HeaderTag *t1, *t2;
+
+ if ( hline1->type[0]!=hline2->type[0] || hline1->type[1]!=hline2->type[1] )
+ return 0;
+
+ int itype = tag_exists(hline1->type,types);
+ if ( itype==-1 ) {
+ debug("[sam_header_compare_lines] Unknown type [%c%c]\n", hline1->type[0],hline1->type[1]);
+ return -1; // FIXME (lh3): error; I do not know how this will be handled in Petr's code
+ }
+
+ if ( unique_tags[itype] )
+ {
+ t1 = header_line_has_tag(hline1,unique_tags[itype][0]);
+ t2 = header_line_has_tag(hline2,unique_tags[itype][0]);
+ if ( !t1 || !t2 ) // this should never happen, the unique tags are required
+ return 2;
+
+ if ( strcmp(t1->value,t2->value) )
+ return 0; // the unique tags differ, cannot be merged
+ }
+ if ( !required_tags[itype] && !optional_tags[itype] )
+ {
+ t1 = hline1->tags->data;
+ t2 = hline2->tags->data;
+ if ( !strcmp(t1->value,t2->value) ) return 1; // identical comments
+ return 0;
+ }
+
+ int missing=0, itag=0;
+ while ( required_tags[itype] && required_tags[itype][itag] )
+ {
+ t1 = header_line_has_tag(hline1,required_tags[itype][itag]);
+ t2 = header_line_has_tag(hline2,required_tags[itype][itag]);
+ if ( !t1 && !t2 )
+ return 2; // this should never happen
+ else if ( !t1 || !t2 )
+ missing = 1; // there is some tag missing in one of the hlines
+ else if ( strcmp(t1->value,t2->value) )
+ {
+ if ( unique_tags[itype] )
+ return 2; // the lines have a matching unique tag but have a conflicting tag
+
+ return 0; // the lines contain conflicting tags, cannot be merged
+ }
+ itag++;
+ }
+ itag = 0;
+ while ( optional_tags[itype] && optional_tags[itype][itag] )
+ {
+ t1 = header_line_has_tag(hline1,optional_tags[itype][itag]);
+ t2 = header_line_has_tag(hline2,optional_tags[itype][itag]);
+ if ( !t1 && !t2 )
+ {
+ itag++;
+ continue;
+ }
+ if ( !t1 || !t2 )
+ missing = 1; // there is some tag missing in one of the hlines
+ else if ( strcmp(t1->value,t2->value) )
+ {
+ if ( unique_tags[itype] )
+ return 2; // the lines have a matching unique tag but have a conflicting tag
+
+ return 0; // the lines contain conflicting tags, cannot be merged
+ }
+ itag++;
+ }
+ if ( missing ) return 3; // there are some missing complementary tags with no conflicts, can be merged
+ return 1;
+}
+
+
+static HeaderLine *sam_header_line_clone(const HeaderLine *hline)
+{
+ list_t *tags;
+ HeaderLine *out = malloc(sizeof(HeaderLine));
+ out->type[0] = hline->type[0];
+ out->type[1] = hline->type[1];
+ out->tags = NULL;
+
+ tags = hline->tags;
+ while (tags)
+ {
+ HeaderTag *old = tags->data;
+
+ HeaderTag *new = malloc(sizeof(HeaderTag));
+ new->key[0] = old->key[0];
+ new->key[1] = old->key[1];
+ new->value = strdup(old->value);
+ out->tags = list_append(out->tags, new);
+
+ tags = tags->next;
+ }
+ return out;
+}
+
+static int sam_header_line_merge_with(HeaderLine *out_hline, const HeaderLine *tmpl_hline)
+{
+ list_t *tmpl_tags;
+
+ if ( out_hline->type[0]!=tmpl_hline->type[0] || out_hline->type[1]!=tmpl_hline->type[1] )
+ return 0;
+
+ tmpl_tags = tmpl_hline->tags;
+ while (tmpl_tags)
+ {
+ HeaderTag *tmpl_tag = tmpl_tags->data;
+ HeaderTag *out_tag = header_line_has_tag(out_hline, tmpl_tag->key);
+ if ( !out_tag )
+ {
+ HeaderTag *tag = malloc(sizeof(HeaderTag));
+ tag->key[0] = tmpl_tag->key[0];
+ tag->key[1] = tmpl_tag->key[1];
+ tag->value = strdup(tmpl_tag->value);
+ out_hline->tags = list_append(out_hline->tags,tag);
+ }
+ tmpl_tags = tmpl_tags->next;
+ }
+ return 1;
+}
+
+
+static HeaderLine *sam_header_line_parse(const char *headerLine)
+{
+ HeaderLine *hline;
+ HeaderTag *tag;
+ const char *from, *to;
+ from = headerLine;
+
+ if ( *from != '@' ) {
+ debug("[sam_header_line_parse] expected '@', got [%s]\n", headerLine);
+ return 0;
+ }
+ to = ++from;
+
+ while (*to && *to!='\t') to++;
+ if ( to-from != 2 ) {
+ debug("[sam_header_line_parse] expected '@XY', got [%s]\nHint: The header tags must be tab-separated.\n", headerLine);
+ return 0;
+ }
+
+ hline = malloc(sizeof(HeaderLine));
+ hline->type[0] = from[0];
+ hline->type[1] = from[1];
+ hline->tags = NULL;
+
+ int itype = tag_exists(hline->type, types);
+
+ from = to;
+ while (*to && *to=='\t') to++;
+ if ( to-from != 1 ) {
+ debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
+ free(hline);
+ return 0;
+ }
+ from = to;
+ while (*from)
+ {
+ while (*to && *to!='\t') to++;
+
+ if ( !required_tags[itype] && !optional_tags[itype] )
+ {
+ // CO is a special case, it can contain anything, including tabs
+ if ( *to ) { to++; continue; }
+ tag = new_tag(" ",from,to-1);
+ }
+ else
+ tag = new_tag(from,from+3,to-1);
+
+ if ( header_line_has_tag(hline,tag->key) )
+ debug("The tag '%c%c' present (at least) twice on line [%s]\n", tag->key[0],tag->key[1], headerLine);
+ hline->tags = list_append(hline->tags, tag);
+
+ from = to;
+ while (*to && *to=='\t') to++;
+ if ( *to && to-from != 1 ) {
+ debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
+ return 0;
+ }
+
+ from = to;
+ }
+ return hline;
+}
+
+
+// Must be of an existing type, all tags must be recognised and all required tags must be present
+static int sam_header_line_validate(HeaderLine *hline)
+{
+ list_t *tags;
+ HeaderTag *tag;
+ int itype, itag;
+
+ // Is the type correct?
+ itype = tag_exists(hline->type, types);
+ if ( itype==-1 )
+ {
+ debug("The type [%c%c] not recognised.\n", hline->type[0],hline->type[1]);
+ return 0;
+ }
+
+ // Has all required tags?
+ itag = 0;
+ while ( required_tags[itype] && required_tags[itype][itag] )
+ {
+ if ( !header_line_has_tag(hline,required_tags[itype][itag]) )
+ {
+ debug("The tag [%c%c] required for [%c%c] not present.\n", required_tags[itype][itag][0],required_tags[itype][itag][1],
+ hline->type[0],hline->type[1]);
+ return 0;
+ }
+ itag++;
+ }
+
+ // Are all tags recognised?
+ tags = hline->tags;
+ while ( tags )
+ {
+ tag = tags->data;
+ if ( !tag_exists(tag->key,required_tags[itype]) && !tag_exists(tag->key,optional_tags[itype]) )
+ {
+ // Lower case tags are user-defined values.
+ if( !(islower(tag->key[0]) || islower(tag->key[1])) )
+ {
+ // Neither is lower case, but tag was not recognized.
+ debug("Unknown tag [%c%c] for [%c%c].\n", tag->key[0],tag->key[1], hline->type[0],hline->type[1]);
+ // return 0; // Even unknown tags are allowed - for forward compatibility with new attributes
+ }
+ // else - allow user defined tag
+ }
+ tags = tags->next;
+ }
+
+ return 1;
+}
+
+
+static void print_header_line(FILE *fp, HeaderLine *hline)
+{
+ list_t *tags = hline->tags;
+ HeaderTag *tag;
+
+ fprintf(fp, "@%c%c", hline->type[0],hline->type[1]);
+ while (tags)
+ {
+ tag = tags->data;
+
+ fprintf(fp, "\t");
+ if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
+ fprintf(fp, "%c%c:", tag->key[0],tag->key[1]);
+ fprintf(fp, "%s", tag->value);
+
+ tags = tags->next;
+ }
+ fprintf(fp,"\n");
+}
+
+
+static void sam_header_line_free(HeaderLine *hline)
+{
+ list_t *tags = hline->tags;
+ while (tags)
+ {
+ HeaderTag *tag = tags->data;
+ free(tag->value);
+ free(tag);
+ tags = tags->next;
+ }
+ list_free(hline->tags);
+ free(hline);
+}
+
+void sam_header_free(void *_header)
+{
+ HeaderDict *header = (HeaderDict*)_header;
+ list_t *hlines = header;
+ while (hlines)
+ {
+ sam_header_line_free(hlines->data);
+ hlines = hlines->next;
+ }
+ list_free(header);
+}
+
+HeaderDict *sam_header_clone(const HeaderDict *dict)
+{
+ HeaderDict *out = NULL;
+ while (dict)
+ {
+ HeaderLine *hline = dict->data;
+ out = list_append(out, sam_header_line_clone(hline));
+ dict = dict->next;
+ }
+ return out;
+}
+
+// Returns a newly allocated string
+char *sam_header_write(const void *_header)
+{
+ const HeaderDict *header = (const HeaderDict*)_header;
+ char *out = NULL;
+ int len=0, nout=0;
+ const list_t *hlines;
+
+ // Calculate the length of the string to allocate
+ hlines = header;
+ while (hlines)
+ {
+ len += 4; // @XY and \n
+
+ HeaderLine *hline = hlines->data;
+ list_t *tags = hline->tags;
+ while (tags)
+ {
+ HeaderTag *tag = tags->data;
+ len += strlen(tag->value) + 1; // \t
+ if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
+ len += strlen(tag->value) + 3; // XY:
+ tags = tags->next;
+ }
+ hlines = hlines->next;
+ }
+
+ nout = 0;
+ out = malloc(len+1);
+ hlines = header;
+ while (hlines)
+ {
+ HeaderLine *hline = hlines->data;
+
+ nout += sprintf(out+nout,"@%c%c",hline->type[0],hline->type[1]);
+
+ list_t *tags = hline->tags;
+ while (tags)
+ {
+ HeaderTag *tag = tags->data;
+ nout += sprintf(out+nout,"\t");
+ if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
+ nout += sprintf(out+nout,"%c%c:", tag->key[0],tag->key[1]);
+ nout += sprintf(out+nout,"%s", tag->value);
+ tags = tags->next;
+ }
+ hlines = hlines->next;
+ nout += sprintf(out+nout,"\n");
+ }
+ out[len] = 0;
+ return out;
+}
+
+void *sam_header_parse2(const char *headerText)
+{
+ list_t *hlines = NULL;
+ HeaderLine *hline;
+ const char *text;
+ char *buf=NULL;
+ size_t nbuf = 0;
+ int tovalidate = 0;
+
+ if ( !headerText )
+ return 0;
+
+ text = headerText;
+ while ( (text=nextline(&buf, &nbuf, text)) )
+ {
+ hline = sam_header_line_parse(buf);
+ if ( hline && (!tovalidate || sam_header_line_validate(hline)) )
+ // With too many (~250,000) reference sequences the header parsing was too slow with list_append.
+ hlines = list_append_to_end(hlines, hline);
+ else
+ {
+ if (hline) sam_header_line_free(hline);
+ sam_header_free(hlines);
+ if ( buf ) free(buf);
+ return NULL;
+ }
+ }
+ if ( buf ) free(buf);
+
+ return hlines;
+}
+
+void *sam_header2tbl(const void *_dict, char type[2], char key_tag[2], char value_tag[2])
+{
+ const HeaderDict *dict = (const HeaderDict*)_dict;
+ const list_t *l = dict;
+ khash_t(str) *tbl = kh_init(str);
+ khiter_t k;
+ int ret;
+
+ if (_dict == 0) return tbl; // return an empty (not null) hash table
+ while (l)
+ {
+ HeaderLine *hline = l->data;
+ if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
+ {
+ l = l->next;
+ continue;
+ }
+
+ HeaderTag *key, *value;
+ key = header_line_has_tag(hline,key_tag);
+ value = header_line_has_tag(hline,value_tag);
+ if ( !key || !value )
+ {
+ l = l->next;
+ continue;
+ }
+
+ k = kh_get(str, tbl, key->value);
+ if ( k != kh_end(tbl) )
+ debug("[sam_header_lookup_table] They key %s not unique.\n", key->value);
+ k = kh_put(str, tbl, key->value, &ret);
+ kh_value(tbl, k) = value->value;
+
+ l = l->next;
+ }
+ return tbl;
+}
+
+char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n)
+{
+ const HeaderDict *dict = (const HeaderDict*)_dict;
+ const list_t *l = dict;
+ int max, n;
+ char **ret;
+
+ ret = 0; *_n = max = n = 0;
+ while (l)
+ {
+ HeaderLine *hline = l->data;
+ if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
+ {
+ l = l->next;
+ continue;
+ }
+
+ HeaderTag *key;
+ key = header_line_has_tag(hline,key_tag);
+ if ( !key )
+ {
+ l = l->next;
+ continue;
+ }
+
+ if (n == max) {
+ max = max? max<<1 : 4;
+ ret = realloc(ret, max * sizeof(void*));
+ }
+ ret[n++] = key->value;
+
+ l = l->next;
+ }
+ *_n = n;
+ return ret;
+}
+
+void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **_key, const char **_value)
+{
+ list_t *l = iter;
+ if ( !l ) return NULL;
+
+ while (l)
+ {
+ HeaderLine *hline = l->data;
+ if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
+ {
+ l = l->next;
+ continue;
+ }
+
+ HeaderTag *key, *value;
+ key = header_line_has_tag(hline,key_tag);
+ value = header_line_has_tag(hline,value_tag);
+ if ( !key && !value )
+ {
+ l = l->next;
+ continue;
+ }
+
+ *_key = key->value;
+ *_value = value->value;
+ return l->next;
+ }
+ return l;
+}
+
+const char *sam_tbl_get(void *h, const char *key)
+{
+ khash_t(str) *tbl = (khash_t(str)*)h;
+ khint_t k;
+ k = kh_get(str, tbl, key);
+ return k == kh_end(tbl)? 0 : kh_val(tbl, k);
+}
+
+int sam_tbl_size(void *h)
+{
+ khash_t(str) *tbl = (khash_t(str)*)h;
+ return h? kh_size(tbl) : 0;
+}
+
+void sam_tbl_destroy(void *h)
+{
+ khash_t(str) *tbl = (khash_t(str)*)h;
+ kh_destroy(str, tbl);
+}
+
+void *sam_header_merge(int n, const void **_dicts)
+{
+ const HeaderDict **dicts = (const HeaderDict**)_dicts;
+ HeaderDict *out_dict;
+ int idict, status;
+
+ if ( n<2 ) return NULL;
+
+ out_dict = sam_header_clone(dicts[0]);
+
+ for (idict=1; idict<n; idict++)
+ {
+ const list_t *tmpl_hlines = dicts[idict];
+
+ while ( tmpl_hlines )
+ {
+ list_t *out_hlines = out_dict;
+ int inserted = 0;
+ while ( out_hlines )
+ {
+ status = sam_header_compare_lines(tmpl_hlines->data, out_hlines->data);
+ if ( status==0 )
+ {
+ out_hlines = out_hlines->next;
+ continue;
+ }
+
+ if ( status==2 )
+ {
+ print_header_line(stderr,tmpl_hlines->data);
+ print_header_line(stderr,out_hlines->data);
+ debug("Conflicting lines, cannot merge the headers.\n");
+ return 0;
+ }
+ if ( status==3 )
+ sam_header_line_merge_with(out_hlines->data, tmpl_hlines->data);
+
+ inserted = 1;
+ break;
+ }
+ if ( !inserted )
+ out_dict = list_append(out_dict, sam_header_line_clone(tmpl_hlines->data));
+
+ tmpl_hlines = tmpl_hlines->next;
+ }
+ }
+
+ return out_dict;
+}
+
+char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n)
+{
+ int nout = 0;
+ char **out = NULL;
+
+ *n = 0;
+ list_t *l = (list_t *)dict;
+ if ( !l ) return NULL;
+
+ int i, ntags = 0;
+ while ( tags[ntags] ) ntags++;
+
+ while (l)
+ {
+ HeaderLine *hline = l->data;
+ if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
+ {
+ l = l->next;
+ continue;
+ }
+ out = (char**) realloc(out, sizeof(char*)*(nout+1)*ntags);
+ for (i=0; i<ntags; i++)
+ {
+ HeaderTag *key = header_line_has_tag(hline, tags[i]);
+ if ( !key )
+ {
+ out[nout*ntags+i] = NULL;
+ continue;
+ }
+ out[nout*ntags+i] = key->value;
+ }
+ nout++;
+ l = l->next;
+ }
+ *n = nout;
+ return out;
+}
+
diff --git a/src/AS_PBR/bamcat.h b/src/AS_PBR/bamcat.h
new file mode 100644
index 0000000..07da811
--- /dev/null
+++ b/src/AS_PBR/bamcat.h
@@ -0,0 +1,2134 @@
+/* The MIT License
+
+ Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
+ 2011, 2012 Attractive Chaos <attractor at live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+*/
+
+/* The BGZF library was originally written by Bob Handsaker from the Broad
+ * Institute. It was later improved by the SAMtools developers. */
+
+#ifndef __BGZF_H
+#define __BGZF_H
+
+#include <stdint.h>
+#include <stdio.h>
+#include <zlib.h>
+#include <sys/types.h>
+
+#define BGZF_BLOCK_SIZE 0xff00 // make sure compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE
+#define BGZF_MAX_BLOCK_SIZE 0x10000
+
+#define BGZF_ERR_ZLIB 1
+#define BGZF_ERR_HEADER 2
+#define BGZF_ERR_IO 4
+#define BGZF_ERR_MISUSE 8
+
+typedef struct {
+ int errcode:16, is_write:2, compress_level:14;
+ int cache_size;
+ int block_length, block_offset;
+ int64_t block_address;
+ void *uncompressed_block, *compressed_block;
+ void *cache; // a pointer to a hash table
+ void *fp; // actual file handler; FILE* on writing; FILE* or knetFile* on reading
+ void *mt; // only used for multi-threading
+} BGZF;
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+ size_t l, m;
+ char *s;
+} kstring_t;
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ /******************
+ * Basic routines *
+ ******************/
+
+ /**
+ * Open an existing file descriptor for reading or writing.
+ *
+ * @param fd file descriptor
+ * @param mode mode matching /[rwu0-9]+/: 'r' for reading, 'w' for writing and a digit specifies
+ * the zlib compression level; if both 'r' and 'w' are present, 'w' is ignored.
+ * @return BGZF file handler; 0 on error
+ */
+ BGZF* bgzf_dopen(int fd, const char *mode);
+
+ #define bgzf_fdopen(fd, mode) bgzf_dopen((fd), (mode)) // for backward compatibility
+
+ /**
+ * Open the specified file for reading or writing.
+ */
+ BGZF* bgzf_open(const char* path, const char *mode);
+
+ /**
+ * Close the BGZF and free all associated resources.
+ *
+ * @param fp BGZF file handler
+ * @return 0 on success and -1 on error
+ */
+ int bgzf_close(BGZF *fp);
+
+ /**
+ * Read up to _length_ bytes from the file storing into _data_.
+ *
+ * @param fp BGZF file handler
+ * @param data data array to read into
+ * @param length size of data to read
+ * @return number of bytes actually read; 0 on end-of-file and -1 on error
+ */
+ ssize_t bgzf_read(BGZF *fp, void *data, ssize_t length);
+
+ /**
+ * Write _length_ bytes from _data_ to the file.
+ *
+ * @param fp BGZF file handler
+ * @param data data array to write
+ * @param length size of data to write
+ * @return number of bytes actually written; -1 on error
+ */
+ ssize_t bgzf_write(BGZF *fp, const void *data, ssize_t length);
+
+ /**
+ * Write the data in the buffer to the file.
+ */
+ int bgzf_flush(BGZF *fp);
+
+ /**
+ * Return a virtual file pointer to the current location in the file.
+ * No interpetation of the value should be made, other than a subsequent
+ * call to bgzf_seek can be used to position the file at the same point.
+ * Return value is non-negative on success.
+ */
+ #define bgzf_tell(fp) ((fp->block_address << 16) | (fp->block_offset & 0xFFFF))
+
+ /**
+ * Set the file to read from the location specified by _pos_.
+ *
+ * @param fp BGZF file handler
+ * @param pos virtual file offset returned by bgzf_tell()
+ * @param whence must be SEEK_SET
+ * @return 0 on success and -1 on error
+ */
+ int64_t bgzf_seek(BGZF *fp, int64_t pos, int whence);
+
+ /**
+ * Check if the BGZF end-of-file (EOF) marker is present
+ *
+ * @param fp BGZF file handler opened for reading
+ * @return 1 if EOF is present; 0 if not or on I/O error
+ */
+ int bgzf_check_EOF(BGZF *fp);
+
+ /**
+ * Check if a file is in the BGZF format
+ *
+ * @param fn file name
+ * @return 1 if _fn_ is BGZF; 0 if not or on I/O error
+ */
+ int bgzf_is_bgzf(const char *fn);
+
+ /*********************
+ * Advanced routines *
+ *********************/
+
+ /**
+ * Set the cache size. Only effective when compiled with -DBGZF_CACHE.
+ *
+ * @param fp BGZF file handler
+ * @param size size of cache in bytes; 0 to disable caching (default)
+ */
+ void bgzf_set_cache_size(BGZF *fp, int size);
+
+ /**
+ * Flush the file if the remaining buffer size is smaller than _size_
+ */
+ int bgzf_flush_try(BGZF *fp, ssize_t size);
+
+ /**
+ * Read one byte from a BGZF file. It is faster than bgzf_read()
+ * @param fp BGZF file handler
+ * @return byte read; -1 on end-of-file or error
+ */
+ int bgzf_getc(BGZF *fp);
+
+ /**
+ * Read one line from a BGZF file. It is faster than bgzf_getc()
+ *
+ * @param fp BGZF file handler
+ * @param delim delimitor
+ * @param str string to write to; must be initialized
+ * @return length of the string; 0 on end-of-file; negative on error
+ */
+ int bgzf_getline(BGZF *fp, int delim, kstring_t *str);
+
+ /**
+ * Read the next BGZF block.
+ */
+ int bgzf_read_block(BGZF *fp);
+
+ /**
+ * Enable multi-threading (only effective on writing)
+ *
+ * @param fp BGZF file handler; must be opened for writing
+ * @param n_threads #threads used for writing
+ * @param n_sub_blks #blocks processed by each thread; a value 64-256 is recommended
+ */
+ int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+/* The MIT License
+
+ Copyright (c) 2008-2010 Genome Research Ltd (GRL).
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/* Contact: Heng Li <lh3 at sanger.ac.uk> */
+
+#ifndef BAM_BAM_H
+#define BAM_BAM_H
+
+/*!
+ @header
+
+ BAM library provides I/O and various operations on manipulating files
+ in the BAM (Binary Alignment/Mapping) or SAM (Sequence Alignment/Map)
+ format. It now supports importing from or exporting to SAM, sorting,
+ merging, generating pileup, and quickly retrieval of reads overlapped
+ with a specified region.
+
+ @copyright Genome Research Ltd.
+ */
+
+#define BAM_VERSION "0.1.19-96b5f2294a"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#ifndef BAM_LITE
+#define BAM_VIRTUAL_OFFSET16
+//#include "bgzf.h"
+/*! @abstract BAM file handler */
+typedef BGZF *bamFile;
+#define bam_open(fn, mode) bgzf_open(fn, mode)
+#define bam_dopen(fd, mode) bgzf_fdopen(fd, mode)
+#define bam_close(fp) bgzf_close(fp)
+#define bam_read(fp, buf, size) bgzf_read(fp, buf, size)
+#define bam_write(fp, buf, size) bgzf_write(fp, buf, size)
+#define bam_tell(fp) bgzf_tell(fp)
+#define bam_seek(fp, pos, dir) bgzf_seek(fp, pos, dir)
+#else
+#define BAM_TRUE_OFFSET
+#include <zlib.h>
+typedef gzFile bamFile;
+#define bam_open(fn, mode) gzopen(fn, mode)
+#define bam_dopen(fd, mode) gzdopen(fd, mode)
+#define bam_close(fp) gzclose(fp)
+#define bam_read(fp, buf, size) gzread(fp, buf, size)
+/* no bam_write/bam_tell/bam_seek() here */
+#endif
+
+/*! @typedef
+ @abstract Structure for the alignment header.
+ @field n_targets number of reference sequences
+ @field target_name names of the reference sequences
+ @field target_len lengths of the referene sequences
+ @field dict header dictionary
+ @field hash hash table for fast name lookup
+ @field rg2lib hash table for @RG-ID -> LB lookup
+ @field l_text length of the plain text in the header
+ @field text plain text
+
+ @discussion Field hash points to null by default. It is a private
+ member.
+ */
+typedef struct {
+ int32_t n_targets;
+ char **target_name;
+ uint32_t *target_len;
+ void *dict, *hash, *rg2lib;
+ uint32_t l_text, n_text;
+ char *text;
+} bam_header_t;
+
+/*! @abstract the read is paired in sequencing, no matter whether it is mapped in a pair */
+#define BAM_FPAIRED 1
+/*! @abstract the read is mapped in a proper pair */
+#define BAM_FPROPER_PAIR 2
+/*! @abstract the read itself is unmapped; conflictive with BAM_FPROPER_PAIR */
+#define BAM_FUNMAP 4
+/*! @abstract the mate is unmapped */
+#define BAM_FMUNMAP 8
+/*! @abstract the read is mapped to the reverse strand */
+#define BAM_FREVERSE 16
+/*! @abstract the mate is mapped to the reverse strand */
+#define BAM_FMREVERSE 32
+/*! @abstract this is read1 */
+#define BAM_FREAD1 64
+/*! @abstract this is read2 */
+#define BAM_FREAD2 128
+/*! @abstract not primary alignment */
+#define BAM_FSECONDARY 256
+/*! @abstract QC failure */
+#define BAM_FQCFAIL 512
+/*! @abstract optical or PCR duplicate */
+#define BAM_FDUP 1024
+
+#define BAM_OFDEC 0
+#define BAM_OFHEX 1
+#define BAM_OFSTR 2
+
+/*! @abstract defautl mask for pileup */
+#define BAM_DEF_MASK (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP)
+
+#define BAM_CORE_SIZE sizeof(bam1_core_t)
+
+/**
+ * Describing how CIGAR operation/length is packed in a 32-bit integer.
+ */
+#define BAM_CIGAR_SHIFT 4
+#define BAM_CIGAR_MASK ((1 << BAM_CIGAR_SHIFT) - 1)
+
+/*
+ CIGAR operations.
+ */
+/*! @abstract CIGAR: M = match or mismatch*/
+#define BAM_CMATCH 0
+/*! @abstract CIGAR: I = insertion to the reference */
+#define BAM_CINS 1
+/*! @abstract CIGAR: D = deletion from the reference */
+#define BAM_CDEL 2
+/*! @abstract CIGAR: N = skip on the reference (e.g. spliced alignment) */
+#define BAM_CREF_SKIP 3
+/*! @abstract CIGAR: S = clip on the read with clipped sequence
+ present in qseq */
+#define BAM_CSOFT_CLIP 4
+/*! @abstract CIGAR: H = clip on the read with clipped sequence trimmed off */
+#define BAM_CHARD_CLIP 5
+/*! @abstract CIGAR: P = padding */
+#define BAM_CPAD 6
+/*! @abstract CIGAR: equals = match */
+#define BAM_CEQUAL 7
+/*! @abstract CIGAR: X = mismatch */
+#define BAM_CDIFF 8
+#define BAM_CBACK 9
+
+#define BAM_CIGAR_STR "MIDNSHP=XB"
+#define BAM_CIGAR_TYPE 0x3C1A7
+
+#define bam_cigar_op(c) ((c)&BAM_CIGAR_MASK)
+#define bam_cigar_oplen(c) ((c)>>BAM_CIGAR_SHIFT)
+#define bam_cigar_opchr(c) (BAM_CIGAR_STR[bam_cigar_op(c)])
+#define bam_cigar_gen(l, o) ((l)<<BAM_CIGAR_SHIFT|(o))
+#define bam_cigar_type(o) (BAM_CIGAR_TYPE>>((o)<<1)&3) // bit 1: consume query; bit 2: consume reference
+
+/*! @typedef
+ @abstract Structure for core alignment information.
+ @field tid chromosome ID, defined by bam_header_t
+ @field pos 0-based leftmost coordinate
+ @field bin bin calculated by bam_reg2bin()
+ @field qual mapping quality
+ @field l_qname length of the query name
+ @field flag bitwise flag
+ @field n_cigar number of CIGAR operations
+ @field l_qseq length of the query sequence (read)
+ */
+typedef struct {
+ int32_t tid;
+ int32_t pos;
+ uint32_t bin:16, qual:8, l_qname:8;
+ uint32_t flag:16, n_cigar:16;
+ int32_t l_qseq;
+ int32_t mtid;
+ int32_t mpos;
+ int32_t isize;
+} bam1_core_t;
+
+/*! @typedef
+ @abstract Structure for one alignment.
+ @field core core information about the alignment
+ @field l_aux length of auxiliary data
+ @field data_len current length of bam1_t::data
+ @field m_data maximum length of bam1_t::data
+ @field data all variable-length data, concatenated; structure: qname-cigar-seq-qual-aux
+
+ @discussion Notes:
+
+ 1. qname is zero tailing and core.l_qname includes the tailing '\0'.
+ 2. l_qseq is calculated from the total length of an alignment block
+ on reading or from CIGAR.
+ 3. cigar data is encoded 4 bytes per CIGAR operation.
+ 4. seq is nybble-encoded according to bam_nt16_table.
+ */
+typedef struct {
+ bam1_core_t core;
+ int l_aux, data_len, m_data;
+ uint8_t *data;
+} bam1_t;
+
+typedef struct __bam_iter_t *bam_iter_t;
+
+#define bam1_strand(b) (((b)->core.flag&BAM_FREVERSE) != 0)
+#define bam1_mstrand(b) (((b)->core.flag&BAM_FMREVERSE) != 0)
+
+/*! @function
+ @abstract Get the CIGAR array
+ @param b pointer to an alignment
+ @return pointer to the CIGAR array
+
+ @discussion In the CIGAR array, each element is a 32-bit integer. The
+ lower 4 bits gives a CIGAR operation and the higher 28 bits keep the
+ length of a CIGAR.
+ */
+#define bam1_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname))
+
+/*! @function
+ @abstract Get the name of the query
+ @param b pointer to an alignment
+ @return pointer to the name string, null terminated
+ */
+#define bam1_qname(b) ((char*)((b)->data))
+
+/*! @function
+ @abstract Get query sequence
+ @param b pointer to an alignment
+ @return pointer to sequence
+
+ @discussion Each base is encoded in 4 bits: 1 for A, 2 for C, 4 for G,
+ 8 for T and 15 for N. Two bases are packed in one byte with the base
+ at the higher 4 bits having smaller coordinate on the read. It is
+ recommended to use bam1_seqi() macro to get the base.
+ */
+#define bam1_seq(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname)
+
+/*! @function
+ @abstract Get query quality
+ @param b pointer to an alignment
+ @return pointer to quality string
+ */
+#define bam1_qual(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1))
+
+/*! @function
+ @abstract Get a base on read
+ @param s Query sequence returned by bam1_seq()
+ @param i The i-th position, 0-based
+ @return 4-bit integer representing the base.
+ */
+//#define bam1_seqi(s, i) ((s)[(i)/2] >> 4*(1-(i)%2) & 0xf)
+#define bam1_seqi(s, i) ((s)[(i)>>1] >> ((~(i)&1)<<2) & 0xf)
+
+#define bam1_seq_seti(s, i, c) ( (s)[(i)>>1] = ((s)[(i)>>1] & 0xf<<(((i)&1)<<2)) | (c)<<((~(i)&1)<<2) )
+
+/*! @function
+ @abstract Get query sequence and quality
+ @param b pointer to an alignment
+ @return pointer to the concatenated auxiliary data
+ */
+#define bam1_aux(b) ((b)->data + (b)->core.n_cigar*4 + (b)->core.l_qname + (b)->core.l_qseq + ((b)->core.l_qseq + 1)/2)
+
+#ifndef kroundup32
+/*! @function
+ @abstract Round an integer to the next closest power-2 integer.
+ @param x integer to be rounded (in place)
+ @discussion x will be modified.
+ */
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+/*!
+ @abstract Whether the machine is big-endian; modified only in
+ bam_header_init().
+ */
+extern int bam_is_be;
+
+/*!
+ @abstract Verbose level between 0 and 3; 0 is supposed to disable all
+ debugging information, though this may not have been implemented.
+ */
+extern int bam_verbose;
+
+extern int bam_no_B;
+
+/*! @abstract Table for converting a nucleotide character to the 4-bit encoding. */
+extern unsigned char bam_nt16_table[256];
+
+/*! @abstract Table for converting a 4-bit encoded nucleotide to a letter. */
+extern char *bam_nt16_rev_table;
+
+extern char bam_nt16_nt4_table[];
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ /*********************
+ * Low-level SAM I/O *
+ *********************/
+
+ /*! @abstract TAM file handler */
+ typedef struct __tamFile_t *tamFile;
+
+ /*!
+ @abstract Open a SAM file for reading, either uncompressed or compressed by gzip/zlib.
+ @param fn SAM file name
+ @return SAM file handler
+ */
+ tamFile sam_open(const char *fn);
+
+ /*!
+ @abstract Close a SAM file handler
+ @param fp SAM file handler
+ */
+ void sam_close(tamFile fp);
+
+ /*!
+ @abstract Read one alignment from a SAM file handler
+ @param fp SAM file handler
+ @param header header information (ordered names of chromosomes)
+ @param b read alignment; all members in b will be updated
+ @return 0 if successful; otherwise negative
+ */
+ int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b);
+
+ /*!
+ @abstract Read header information from a TAB-delimited list file.
+ @param fn_list file name for the list
+ @return a pointer to the header structure
+
+ @discussion Each line in this file consists of chromosome name and
+ the length of chromosome.
+ */
+ bam_header_t *sam_header_read2(const char *fn_list);
+
+ /*!
+ @abstract Read header from a SAM file (if present)
+ @param fp SAM file handler
+ @return pointer to header struct; 0 if no @SQ lines available
+ */
+ bam_header_t *sam_header_read(tamFile fp);
+
+ /*!
+ @abstract Parse @SQ lines a update a header struct
+ @param h pointer to the header struct to be updated
+ @return number of target sequences
+
+ @discussion bam_header_t::{n_targets,target_len,target_name} will
+ be destroyed in the first place.
+ */
+ int sam_header_parse(bam_header_t *h);
+ int32_t bam_get_tid(const bam_header_t *header, const char *seq_name);
+
+ /*!
+ @abstract Parse @RG lines a update a header struct
+ @param h pointer to the header struct to be updated
+ @return number of @RG lines
+
+ @discussion bam_header_t::rg2lib will be destroyed in the first
+ place.
+ */
+ int sam_header_parse_rg(bam_header_t *h);
+
+#define sam_write1(header, b) bam_view1(header, b)
+
+
+ /********************************
+ * APIs for string dictionaries *
+ ********************************/
+
+ int bam_strmap_put(void *strmap, const char *rg, const char *lib);
+ const char *bam_strmap_get(const void *strmap, const char *rg);
+ void *bam_strmap_dup(const void*);
+ void *bam_strmap_init();
+ void bam_strmap_destroy(void *strmap);
+
+
+ /*********************
+ * Low-level BAM I/O *
+ *********************/
+
+ /*!
+ @abstract Initialize a header structure.
+ @return the pointer to the header structure
+
+ @discussion This function also modifies the global variable
+ bam_is_be.
+ */
+ bam_header_t *bam_header_init();
+
+ /*!
+ @abstract Destroy a header structure.
+ @param header pointer to the header
+ */
+ void bam_header_destroy(bam_header_t *header);
+
+ /*!
+ @abstract Read a header structure from BAM.
+ @param fp BAM file handler, opened by bam_open()
+ @return pointer to the header structure
+
+ @discussion The file position indicator must be placed at the
+ beginning of the file. Upon success, the position indicator will
+ be set at the start of the first alignment.
+ */
+ bam_header_t *bam_header_read(bamFile fp);
+
+ /*!
+ @abstract Write a header structure to BAM.
+ @param fp BAM file handler
+ @param header pointer to the header structure
+ @return always 0 currently
+ */
+ int bam_header_write(bamFile fp, const bam_header_t *header);
+
+ /*!
+ @abstract Read an alignment from BAM.
+ @param fp BAM file handler
+ @param b read alignment; all members are updated.
+ @return number of bytes read from the file
+
+ @discussion The file position indicator must be
+ placed right before an alignment. Upon success, this function
+ will set the position indicator to the start of the next
+ alignment. This function is not affected by the machine
+ endianness.
+ */
+ int bam_read1(bamFile fp, bam1_t *b);
+
+ int bam_remove_B(bam1_t *b);
+
+ /*!
+ @abstract Write an alignment to BAM.
+ @param fp BAM file handler
+ @param c pointer to the bam1_core_t structure
+ @param data_len total length of variable size data related to
+ the alignment
+ @param data pointer to the concatenated data
+ @return number of bytes written to the file
+
+ @discussion This function is not affected by the machine
+ endianness.
+ */
+ int bam_write1_core(bamFile fp, const bam1_core_t *c, int data_len, uint8_t *data);
+
+ /*!
+ @abstract Write an alignment to BAM.
+ @param fp BAM file handler
+ @param b alignment to write
+ @return number of bytes written to the file
+
+ @abstract It is equivalent to:
+ bam_write1_core(fp, &b->core, b->data_len, b->data)
+ */
+ int bam_write1(bamFile fp, const bam1_t *b);
+
+ /*! @function
+ @abstract Initiate a pointer to bam1_t struct
+ */
+#if 1
+#define bam_init1() ((bam1_t*)calloc(1, sizeof(bam1_t)))
+#else
+ bam1_t *bam_init1(void) {
+ bam1_t *b = new bam1_t;
+ memset(b, 0, sizeof(bam1_t));
+ return(b);
+ }
+#endif
+
+
+ /*! @function
+ @abstract Free the memory allocated for an alignment.
+ @param b pointer to an alignment
+ */
+#define bam_destroy1(b) do { \
+ if (b) { free((b)->data); free(b); } \
+ } while (0)
+
+ /*!
+ @abstract Format a BAM record in the SAM format
+ @param header pointer to the header structure
+ @param b alignment to print
+ @return a pointer to the SAM string
+ */
+ char *bam_format1(const bam_header_t *header, const bam1_t *b);
+
+ char *bam_format1_core(const bam_header_t *header, const bam1_t *b, int of);
+
+ /*!
+ @abstract Check whether a BAM record is plausibly valid
+ @param header associated header structure, or NULL if unavailable
+ @param b alignment to validate
+ @return 0 if the alignment is invalid; non-zero otherwise
+
+ @discussion Simple consistency check of some of the fields of the
+ alignment record. If the header is provided, several additional checks
+ are made. Not all fields are checked, so a non-zero result is not a
+ guarantee that the record is valid. However it is usually good enough
+ to detect when bam_seek() has been called with a virtual file offset
+ that is not the offset of an alignment record.
+ */
+ int bam_validate1(const bam_header_t *header, const bam1_t *b);
+
+ const char *bam_get_library(bam_header_t *header, const bam1_t *b);
+
+
+ /***************
+ * pileup APIs *
+ ***************/
+
+ /*! @typedef
+ @abstract Structure for one alignment covering the pileup position.
+ @field b pointer to the alignment
+ @field qpos position of the read base at the pileup site, 0-based
+ @field indel indel length; 0 for no indel, positive for ins and negative for del
+ @field is_del 1 iff the base on the padded read is a deletion
+ @field level the level of the read in the "viewer" mode
+
+ @discussion See also bam_plbuf_push() and bam_lplbuf_push(). The
+ difference between the two functions is that the former does not
+ set bam_pileup1_t::level, while the later does. Level helps the
+ implementation of alignment viewers, but calculating this has some
+ overhead.
+ */
+ typedef struct {
+ bam1_t *b;
+ int32_t qpos;
+ int indel, level;
+ uint32_t is_del:1, is_head:1, is_tail:1, is_refskip:1, aux:28;
+ } bam_pileup1_t;
+
+ typedef int (*bam_plp_auto_f)(void *data, bam1_t *b);
+
+ struct __bam_plp_t;
+ typedef struct __bam_plp_t *bam_plp_t;
+
+ bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data);
+ int bam_plp_push(bam_plp_t iter, const bam1_t *b);
+ const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);
+ const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);
+ void bam_plp_set_mask(bam_plp_t iter, int mask);
+ void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt);
+ void bam_plp_reset(bam_plp_t iter);
+ void bam_plp_destroy(bam_plp_t iter);
+
+ struct __bam_mplp_t;
+ typedef struct __bam_mplp_t *bam_mplp_t;
+
+ bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data);
+ void bam_mplp_destroy(bam_mplp_t iter);
+ void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt);
+ int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp);
+
+ /*! @typedef
+ @abstract Type of function to be called by bam_plbuf_push().
+ @param tid chromosome ID as is defined in the header
+ @param pos start coordinate of the alignment, 0-based
+ @param n number of elements in pl array
+ @param pl array of alignments
+ @param data user provided data
+ @discussion See also bam_plbuf_push(), bam_plbuf_init() and bam_pileup1_t.
+ */
+ typedef int (*bam_pileup_f)(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data);
+
+ typedef struct {
+ bam_plp_t iter;
+ bam_pileup_f func;
+ void *data;
+ } bam_plbuf_t;
+
+ void bam_plbuf_set_mask(bam_plbuf_t *buf, int mask);
+ void bam_plbuf_reset(bam_plbuf_t *buf);
+ bam_plbuf_t *bam_plbuf_init(bam_pileup_f func, void *data);
+ void bam_plbuf_destroy(bam_plbuf_t *buf);
+ int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf);
+
+ int bam_pileup_file(bamFile fp, int mask, bam_pileup_f func, void *func_data);
+
+ struct __bam_lplbuf_t;
+ typedef struct __bam_lplbuf_t bam_lplbuf_t;
+
+ void bam_lplbuf_reset(bam_lplbuf_t *buf);
+
+ /*! @abstract bam_plbuf_init() equivalent with level calculated. */
+ bam_lplbuf_t *bam_lplbuf_init(bam_pileup_f func, void *data);
+
+ /*! @abstract bam_plbuf_destroy() equivalent with level calculated. */
+ void bam_lplbuf_destroy(bam_lplbuf_t *tv);
+
+ /*! @abstract bam_plbuf_push() equivalent with level calculated. */
+ int bam_lplbuf_push(const bam1_t *b, bam_lplbuf_t *buf);
+
+
+ /*********************
+ * BAM indexing APIs *
+ *********************/
+
+ struct __bam_index_t;
+ typedef struct __bam_index_t bam_index_t;
+
+ /*!
+ @abstract Build index for a BAM file.
+ @discussion Index file "fn.bai" will be created.
+ @param fn name of the BAM file
+ @return always 0 currently
+ */
+ int bam_index_build(const char *fn);
+
+ /*!
+ @abstract Load index from file "fn.bai".
+ @param fn name of the BAM file (NOT the index file)
+ @return pointer to the index structure
+ */
+ bam_index_t *bam_index_load(const char *fn);
+
+ /*!
+ @abstract Destroy an index structure.
+ @param idx pointer to the index structure
+ */
+ void bam_index_destroy(bam_index_t *idx);
+
+ /*! @typedef
+ @abstract Type of function to be called by bam_fetch().
+ @param b the alignment
+ @param data user provided data
+ */
+ typedef int (*bam_fetch_f)(const bam1_t *b, void *data);
+
+ /*!
+ @abstract Retrieve the alignments that are overlapped with the
+ specified region.
+
+ @discussion A user defined function will be called for each
+ retrieved alignment ordered by its start position.
+
+ @param fp BAM file handler
+ @param idx pointer to the alignment index
+ @param tid chromosome ID as is defined in the header
+ @param beg start coordinate, 0-based
+ @param end end coordinate, 0-based
+ @param data user provided data (will be transferred to func)
+ @param func user defined function
+ */
+ int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func);
+
+ bam_iter_t bam_iter_query(const bam_index_t *idx, int tid, int beg, int end);
+ int bam_iter_read(bamFile fp, bam_iter_t iter, bam1_t *b);
+ void bam_iter_destroy(bam_iter_t iter);
+
+ /*!
+ @abstract Parse a region in the format: "chr2:100,000-200,000".
+ @discussion bam_header_t::hash will be initialized if empty.
+ @param header pointer to the header structure
+ @param str string to be parsed
+ @param ref_id the returned chromosome ID
+ @param begin the returned start coordinate
+ @param end the returned end coordinate
+ @return 0 on success; -1 on failure
+ */
+ int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *begin, int *end);
+
+
+ /**************************
+ * APIs for optional tags *
+ **************************/
+
+ /*!
+ @abstract Retrieve data of a tag
+ @param b pointer to an alignment struct
+ @param tag two-character tag to be retrieved
+
+ @return pointer to the type and data. The first character is the
+ type that can be 'iIsScCdfAZH'.
+
+ @discussion Use bam_aux2?() series to convert the returned data to
+ the corresponding type.
+ */
+ uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]);
+
+ int32_t bam_aux2i(const uint8_t *s);
+ float bam_aux2f(const uint8_t *s);
+ double bam_aux2d(const uint8_t *s);
+ char bam_aux2A(const uint8_t *s);
+ char *bam_aux2Z(const uint8_t *s);
+
+ int bam_aux_del(bam1_t *b, uint8_t *s);
+ void bam_aux_append(bam1_t *b, const char tag[2], char type, int len, uint8_t *data);
+ uint8_t *bam_aux_get_core(bam1_t *b, const char tag[2]); // an alias of bam_aux_get()
+
+
+ /*****************
+ * Miscellaneous *
+ *****************/
+
+ /*!
+ @abstract Calculate the rightmost coordinate of an alignment on the
+ reference genome.
+
+ @param c pointer to the bam1_core_t structure
+ @param cigar the corresponding CIGAR array (from bam1_t::cigar)
+ @return the rightmost coordinate, 0-based
+ */
+ uint32_t bam_calend(const bam1_core_t *c, const uint32_t *cigar);
+
+ /*!
+ @abstract Calculate the length of the query sequence from CIGAR.
+ @param c pointer to the bam1_core_t structure
+ @param cigar the corresponding CIGAR array (from bam1_t::cigar)
+ @return length of the query sequence
+ */
+ int32_t bam_cigar2qlen(const bam1_core_t *c, const uint32_t *cigar);
+
+#ifdef __cplusplus
+}
+#endif
+
+/*!
+ @abstract Calculate the minimum bin that contains a region [beg,end).
+ @param beg start of the region, 0-based
+ @param end end of the region, 0-based
+ @return bin
+ */
+static inline int bam_reg2bin(uint32_t beg, uint32_t end)
+{
+ --end;
+ if (beg>>14 == end>>14) return 4681 + (beg>>14);
+ if (beg>>17 == end>>17) return 585 + (beg>>17);
+ if (beg>>20 == end>>20) return 73 + (beg>>20);
+ if (beg>>23 == end>>23) return 9 + (beg>>23);
+ if (beg>>26 == end>>26) return 1 + (beg>>26);
+ return 0;
+}
+
+/*!
+ @abstract Copy an alignment
+ @param bdst destination alignment struct
+ @param bsrc source alignment struct
+ @return pointer to the destination alignment struct
+ */
+#if 0
+static inline bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc)
+{
+ uint8_t *data = bdst->data;
+ int m_data = bdst->m_data; // backup data and m_data
+ if (m_data < bsrc->data_len) { // double the capacity
+ m_data = bsrc->data_len; kroundup32(m_data);
+ data = (uint8_t*)realloc(data, m_data);
+ }
+ memcpy(data, bsrc->data, bsrc->data_len); // copy var-len data
+ *bdst = *bsrc; // copy the rest
+ // restore the backup
+ bdst->m_data = m_data;
+ bdst->data = data;
+ return bdst;
+}
+
+/*!
+ @abstract Duplicate an alignment
+ @param src source alignment struct
+ @return pointer to the destination alignment struct
+ */
+static inline bam1_t *bam_dup1(const bam1_t *src)
+{
+ bam1_t *b;
+ b = bam_init1();
+ *b = *src;
+ b->m_data = b->data_len;
+ b->data = (uint8_t*)calloc(b->data_len, 1);
+ memcpy(b->data, src->data, b->data_len);
+ return b;
+}
+#endif
+
+static inline int bam_aux_type2size(int x)
+{
+ if (x == 'C' || x == 'c' || x == 'A') return 1;
+ else if (x == 'S' || x == 's') return 2;
+ else if (x == 'I' || x == 'i' || x == 'f' || x == 'F') return 4;
+ else return 0;
+}
+
+/*********************************
+ *** Compatibility with htslib ***
+ *********************************/
+
+typedef bam_header_t bam_hdr_t;
+
+#define bam_get_qname(b) bam1_qname(b)
+#define bam_get_cigar(b) bam1_cigar(b)
+
+#define bam_hdr_read(fp) bam_header_read(fp)
+#define bam_hdr_write(fp, h) bam_header_write(fp, h)
+#define bam_hdr_destroy(fp) bam_header_destroy(fp)
+
+#endif
+#ifndef BAM_SAM_H
+#define BAM_SAM_H
+
+//#include "bam.h"
+
+/*!
+ @header
+
+ This file provides higher level of I/O routines and unifies the APIs
+ for SAM and BAM formats. These APIs are more convenient and
+ recommended.
+
+ @copyright Genome Research Ltd.
+ */
+
+/*! @typedef
+ @abstract SAM/BAM file handler
+ @field type type of the handler; bit 1 for BAM, 2 for reading and bit 3-4 for flag format
+ @field bam BAM file handler; valid if (type&1) == 1
+ @field tamr SAM file handler for reading; valid if type == 2
+ @field tamw SAM file handler for writing; valid if type == 0
+ @field header header struct
+ */
+typedef struct {
+ int type;
+ union {
+ tamFile tamr;
+ bamFile bam;
+ FILE *tamw;
+ } x;
+ bam_header_t *header;
+} samfile_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ /*!
+ @abstract Open a SAM/BAM file
+
+ @param fn SAM/BAM file name; "-" is recognized as stdin (for
+ reading) or stdout (for writing).
+
+ @param mode open mode /[rw](b?)(u?)(h?)([xX]?)/: 'r' for reading,
+ 'w' for writing, 'b' for BAM I/O, 'u' for uncompressed BAM output,
+ 'h' for outputing header in SAM, 'x' for HEX flag and 'X' for
+ string flag. If 'b' present, it must immediately follow 'r' or
+ 'w'. Valid modes are "r", "w", "wh", "wx", "whx", "wX", "whX",
+ "rb", "wb" and "wbu" exclusively.
+
+ @param aux auxiliary data; if mode[0]=='w', aux points to
+ bam_header_t; if strcmp(mode, "rb")!=0 and @SQ header lines in SAM
+ are absent, aux points the file name of the list of the reference;
+ aux is not used otherwise. If @SQ header lines are present in SAM,
+ aux is not used, either.
+
+ @return SAM/BAM file handler
+ */
+ samfile_t *samopen(const char *fn, const char *mode, const void *aux);
+
+ /*!
+ @abstract Close a SAM/BAM handler
+ @param fp file handler to be closed
+ */
+ void samclose(samfile_t *fp);
+
+ /*!
+ @abstract Read one alignment
+ @param fp file handler
+ @param b alignment
+ @return bytes read
+ */
+ int samread(samfile_t *fp, bam1_t *b);
+
+ /*!
+ @abstract Write one alignment
+ @param fp file handler
+ @param b alignment
+ @return bytes written
+ */
+ int samwrite(samfile_t *fp, const bam1_t *b);
+
+#if 0
+ /*!
+ @abstract Get the pileup for a whole alignment file
+ @param fp file handler
+ @param mask mask transferred to bam_plbuf_set_mask()
+ @param func user defined function called in the pileup process
+ #param data user provided data for func()
+ */
+ int sampileup(samfile_t *fp, int mask, bam_pileup_f func, void *data);
+#endif
+
+ char *samfaipath(const char *fn_ref);
+ int samthreads(samfile_t *fp, int n_threads, int n_sub_blks);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+#ifndef BAM_ENDIAN_H
+#define BAM_ENDIAN_H
+
+#include <stdint.h>
+
+static inline int bam_is_big_endian()
+{
+ long one= 1;
+ return !(*((char *)(&one)));
+}
+static inline uint16_t bam_swap_endian_2(uint16_t v)
+{
+ return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8));
+}
+static inline void *bam_swap_endian_2p(void *x)
+{
+ *(uint16_t*)x = bam_swap_endian_2(*(uint16_t*)x);
+ return x;
+}
+static inline uint32_t bam_swap_endian_4(uint32_t v)
+{
+ v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
+ return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
+}
+static inline void *bam_swap_endian_4p(void *x)
+{
+ *(uint32_t*)x = bam_swap_endian_4(*(uint32_t*)x);
+ return x;
+}
+static inline uint64_t bam_swap_endian_8(uint64_t v)
+{
+ v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
+ v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
+ return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
+}
+static inline void *bam_swap_endian_8p(void *x)
+{
+ *(uint64_t*)x = bam_swap_endian_8(*(uint64_t*)x);
+ return x;
+}
+
+#endif
+/* The MIT License
+
+ Copyright (c) by Attractive Chaos <attractor at live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+#ifndef KSTRING_H
+#define KSTRING_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+ size_t l, m;
+ char *s;
+} kstring_t;
+#endif
+
+typedef struct {
+ uint64_t tab[4];
+ int sep, finished;
+ const char *p; // end of the current token
+} ks_tokaux_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ int ksprintf(kstring_t *s, const char *fmt, ...);
+ int ksplit_core(char *s, int delimiter, int *_max, int **_offsets);
+ char *kstrstr(const char *str, const char *pat, int **_prep);
+ char *kstrnstr(const char *str, const char *pat, int n, int **_prep);
+ void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep);
+
+ /* kstrtok() is similar to strtok_r() except that str is not
+ * modified and both str and sep can be NULL. For efficiency, it is
+ * actually recommended to set both to NULL in the subsequent calls
+ * if sep is not changed. */
+ char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux);
+
+#ifdef __cplusplus
+}
+#endif
+
+static inline void ks_resize(kstring_t *s, size_t size)
+{
+ if (s->m < size) {
+ s->m = size;
+ kroundup32(s->m);
+ s->s = (char*)realloc(s->s, s->m);
+ }
+}
+
+static inline int kputsn(const char *p, int l, kstring_t *s)
+{
+ if (s->l + l + 1 >= s->m) {
+ s->m = s->l + l + 2;
+ kroundup32(s->m);
+ s->s = (char*)realloc(s->s, s->m);
+ }
+ memcpy(s->s + s->l, p, l);
+ s->l += l;
+ s->s[s->l] = 0;
+ return l;
+}
+
+static inline int kputs(const char *p, kstring_t *s)
+{
+ return kputsn(p, strlen(p), s);
+}
+
+static inline int kputc(int c, kstring_t *s)
+{
+ if (s->l + 1 >= s->m) {
+ s->m = s->l + 2;
+ kroundup32(s->m);
+ s->s = (char*)realloc(s->s, s->m);
+ }
+ s->s[s->l++] = c;
+ s->s[s->l] = 0;
+ return c;
+}
+
+static inline int kputw(int c, kstring_t *s)
+{
+ char buf[16];
+ int l, x;
+ if (c == 0) return kputc('0', s);
+ if(c < 0) for (l = 0, x = c; x < 0; x /= 10) buf[l++] = '0' - (x%10);
+ else for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0';
+ if (c < 0) buf[l++] = '-';
+ if (s->l + l + 1 >= s->m) {
+ s->m = s->l + l + 2;
+ kroundup32(s->m);
+ s->s = (char*)realloc(s->s, s->m);
+ }
+ for (x = l - 1; x >= 0; --x) s->s[s->l++] = buf[x];
+ s->s[s->l] = 0;
+ return 0;
+}
+
+static inline int kputuw(unsigned c, kstring_t *s)
+{
+ char buf[16];
+ int l, i;
+ unsigned x;
+ if (c == 0) return kputc('0', s);
+ for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0';
+ if (s->l + l + 1 >= s->m) {
+ s->m = s->l + l + 2;
+ kroundup32(s->m);
+ s->s = (char*)realloc(s->s, s->m);
+ }
+ for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+ s->s[s->l] = 0;
+ return 0;
+}
+
+static inline int kputl(long c, kstring_t *s)
+{
+ char buf[32];
+ long l, x;
+ if (c == 0) return kputc('0', s);
+ for (l = 0, x = c < 0? -c : c; x > 0; x /= 10) buf[l++] = x%10 + '0';
+ if (c < 0) buf[l++] = '-';
+ if (s->l + l + 1 >= s->m) {
+ s->m = s->l + l + 2;
+ kroundup32(s->m);
+ s->s = (char*)realloc(s->s, s->m);
+ }
+ for (x = l - 1; x >= 0; --x) s->s[s->l++] = buf[x];
+ s->s[s->l] = 0;
+ return 0;
+}
+
+static inline int *ksplit(kstring_t *s, int delimiter, int *n)
+{
+ int max = 0, *offsets = 0;
+ *n = ksplit_core(s->s, delimiter, &max, &offsets);
+ return offsets;
+}
+
+#endif
+#ifndef __SAM_HEADER_H__
+#define __SAM_HEADER_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ void *sam_header_parse2(const char *headerText);
+ void *sam_header_merge(int n, const void **dicts);
+ void sam_header_free(void *header);
+ char *sam_header_write(const void *headerDict); // returns a newly allocated string
+
+ /*
+ // Usage example
+ const char *key, *val;
+ void *iter = sam_header_parse2(bam->header->text);
+ while ( iter = sam_header_key_val(iter, "RG","ID","SM" &key,&val) ) printf("%s\t%s\n", key,val);
+ */
+ void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **key, const char **value);
+ char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n);
+
+ /*
+ // Usage example
+ int i, j, n;
+ const char *tags[] = {"SN","LN","UR","M5",NULL};
+ void *dict = sam_header_parse2(bam->header->text);
+ char **tbl = sam_header2tbl_n(h->dict, "SQ", tags, &n);
+ for (i=0; i<n; i++)
+ {
+ for (j=0; j<4; j++)
+ if ( tbl[4*i+j] ) printf("\t%s", tbl[4*i+j]);
+ else printf("-");
+ printf("\n");
+ }
+ if (tbl) free(tbl);
+ */
+ char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n);
+
+ void *sam_header2tbl(const void *dict, char type[2], char key_tag[2], char value_tag[2]);
+ const char *sam_tbl_get(void *h, const char *key);
+ int sam_tbl_size(void *h);
+ void sam_tbl_destroy(void *h);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+/* The MIT License
+
+ Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor at live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/*
+ An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+ int ret, is_missing;
+ khiter_t k;
+ khash_t(32) *h = kh_init(32);
+ k = kh_put(32, h, 5, &ret);
+ if (!ret) kh_del(32, h, k);
+ kh_value(h, k) = 10;
+ k = kh_get(32, h, 10);
+ is_missing = (k == kh_end(h));
+ k = kh_get(32, h, 5);
+ kh_del(32, h, k);
+ for (k = kh_begin(h); k != kh_end(h); ++k)
+ if (kh_exist(h, k)) kh_value(h, k) = 1;
+ kh_destroy(32, h);
+ return 0;
+}
+*/
+
+/*
+ 2011-02-14 (0.2.5):
+
+ * Allow to declare global functions.
+
+ 2009-09-26 (0.2.4):
+
+ * Improve portability
+
+ 2008-09-19 (0.2.3):
+
+ * Corrected the example
+ * Improved interfaces
+
+ 2008-09-11 (0.2.2):
+
+ * Improved speed a little in kh_put()
+
+ 2008-09-10 (0.2.1):
+
+ * Added kh_clear()
+ * Fixed a compiling error
+
+ 2008-09-02 (0.2.0):
+
+ * Changed to token concatenation which increases flexibility.
+
+ 2008-08-31 (0.1.2):
+
+ * Fixed a bug in kh_get(), which has not been tested previously.
+
+ 2008-08-31 (0.1.1):
+
+ * Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+ @header
+
+ Generic hash table library.
+
+ @copyright Heng Li
+ */
+
+#define AC_VERSION_KHASH_H "0.2.5"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+/* compipler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifdef _MSC_VER
+#define inline __inline
+#endif
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_HASH_PRIME_SIZE 32
+static const khint32_t __ac_prime_list[__ac_HASH_PRIME_SIZE] =
+{
+ 0ul, 3ul, 11ul, 23ul, 53ul,
+ 97ul, 193ul, 389ul, 769ul, 1543ul,
+ 3079ul, 6151ul, 12289ul, 24593ul, 49157ul,
+ 98317ul, 196613ul, 393241ul, 786433ul, 1572869ul,
+ 3145739ul, 6291469ul, 12582917ul, 25165843ul, 50331653ul,
+ 100663319ul, 201326611ul, 402653189ul, 805306457ul, 1610612741ul,
+ 3221225473ul, 4294967291ul
+};
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define KHASH_DECLARE(name, khkey_t, khval_t) \
+ typedef struct { \
+ khint_t n_buckets, size, n_occupied, upper_bound; \
+ khint32_t *flags; \
+ khkey_t *keys; \
+ khval_t *vals; \
+ } kh_##name##_t; \
+ extern kh_##name##_t *kh_init_##name(); \
+ extern void kh_destroy_##name(kh_##name##_t *h); \
+ extern void kh_clear_##name(kh_##name##_t *h); \
+ extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
+ extern void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+ extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+ extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ typedef struct { \
+ khint_t n_buckets, size, n_occupied, upper_bound; \
+ khint32_t *flags; \
+ khkey_t *keys; \
+ khval_t *vals; \
+ } kh_##name##_t; \
+ SCOPE kh_##name##_t *kh_init_##name() { \
+ return (kh_##name##_t*)calloc(1, sizeof(kh_##name##_t)); \
+ } \
+ SCOPE void kh_destroy_##name(kh_##name##_t *h) \
+ { \
+ if (h) { \
+ free(h->keys); free(h->flags); \
+ free(h->vals); \
+ free(h); \
+ } \
+ } \
+ SCOPE void kh_clear_##name(kh_##name##_t *h) \
+ { \
+ if (h && h->flags) { \
+ memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(khint32_t)); \
+ h->size = h->n_occupied = 0; \
+ } \
+ } \
+ SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
+ { \
+ if (h->n_buckets) { \
+ khint_t inc, k, i, last; \
+ k = __hash_func(key); i = k % h->n_buckets; \
+ inc = 1 + k % (h->n_buckets - 1); last = i; \
+ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+ if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \
+ else i += inc; \
+ if (i == last) return h->n_buckets; \
+ } \
+ return __ac_iseither(h->flags, i)? h->n_buckets : i; \
+ } else return 0; \
+ } \
+ SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+ { \
+ khint32_t *new_flags = 0; \
+ khint_t j = 1; \
+ { \
+ khint_t t = __ac_HASH_PRIME_SIZE - 1; \
+ while (__ac_prime_list[t] > new_n_buckets) --t; \
+ new_n_buckets = __ac_prime_list[t+1]; \
+ if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; \
+ else { \
+ new_flags = (khint32_t*)malloc(((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \
+ memset(new_flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(khint32_t)); \
+ if (h->n_buckets < new_n_buckets) { \
+ h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \
+ if (kh_is_map) \
+ h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \
+ } \
+ } \
+ } \
+ if (j) { \
+ for (j = 0; j != h->n_buckets; ++j) { \
+ if (__ac_iseither(h->flags, j) == 0) { \
+ khkey_t key = h->keys[j]; \
+ khval_t val; \
+ if (kh_is_map) val = h->vals[j]; \
+ __ac_set_isdel_true(h->flags, j); \
+ while (1) { \
+ khint_t inc, k, i; \
+ k = __hash_func(key); \
+ i = k % new_n_buckets; \
+ inc = 1 + k % (new_n_buckets - 1); \
+ while (!__ac_isempty(new_flags, i)) { \
+ if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets; \
+ else i += inc; \
+ } \
+ __ac_set_isempty_false(new_flags, i); \
+ if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { \
+ { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+ if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+ __ac_set_isdel_true(h->flags, i); \
+ } else { \
+ h->keys[i] = key; \
+ if (kh_is_map) h->vals[i] = val; \
+ break; \
+ } \
+ } \
+ } \
+ } \
+ if (h->n_buckets > new_n_buckets) { \
+ h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \
+ if (kh_is_map) \
+ h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \
+ } \
+ free(h->flags); \
+ h->flags = new_flags; \
+ h->n_buckets = new_n_buckets; \
+ h->n_occupied = h->size; \
+ h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+ } \
+ } \
+ SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+ { \
+ khint_t x; \
+ if (h->n_occupied >= h->upper_bound) { \
+ if (h->n_buckets > (h->size<<1)) kh_resize_##name(h, h->n_buckets - 1); \
+ else kh_resize_##name(h, h->n_buckets + 1); \
+ } \
+ { \
+ khint_t inc, k, i, site, last; \
+ x = site = h->n_buckets; k = __hash_func(key); i = k % h->n_buckets; \
+ if (__ac_isempty(h->flags, i)) x = i; \
+ else { \
+ inc = 1 + k % (h->n_buckets - 1); last = i; \
+ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+ if (__ac_isdel(h->flags, i)) site = i; \
+ if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \
+ else i += inc; \
+ if (i == last) { x = site; break; } \
+ } \
+ if (x == h->n_buckets) { \
+ if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+ else x = i; \
+ } \
+ } \
+ } \
+ if (__ac_isempty(h->flags, x)) { \
+ h->keys[x] = key; \
+ __ac_set_isboth_false(h->flags, x); \
+ ++h->size; ++h->n_occupied; \
+ *ret = 1; \
+ } else if (__ac_isdel(h->flags, x)) { \
+ h->keys[x] = key; \
+ __ac_set_isboth_false(h->flags, x); \
+ ++h->size; \
+ *ret = 2; \
+ } else *ret = 0; \
+ return x; \
+ } \
+ SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \
+ { \
+ if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
+ __ac_set_isdel_true(h->flags, x); \
+ --h->size; \
+ } \
+ }
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+ KHASH_INIT2(name, static inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+ @abstract Integer hash function
+ @param key The integer [khint32_t]
+ @return The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+ @abstract Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+ @abstract 64-bit integer hash function
+ @param key The integer [khint64_t]
+ @return The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+ @abstract 64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+/*! @function
+ @abstract const char* hash function
+ @param s Pointer to a null terminated string
+ @return The hash value
+ */
+static inline khint_t __ac_X31_hash_string(const char *s)
+{
+ khint_t h = *s;
+ if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s;
+ return h;
+}
+/*! @function
+ @abstract Another interface to const char* hash function
+ @param key Pointer to a null terminated string [const char*]
+ @return The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) __ac_X31_hash_string(key)
+/*! @function
+ @abstract Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other necessary macros... */
+
+/*!
+ @abstract Type of the hash table.
+ @param name Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+ @abstract Initiate a hash table.
+ @param name Name of the hash table [symbol]
+ @return Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+
+/*! @function
+ @abstract Destroy a hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+
+/*! @function
+ @abstract Reset a hash table without deallocating memory.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+ @abstract Resize a hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param s New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+ @abstract Insert a key to the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Key [type of keys]
+ @param r Extra return code: 0 if the key is present in the hash table;
+ 1 if the bucket is empty (never used); 2 if the element in
+ the bucket has been deleted [int*]
+ @return Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+ @abstract Retrieve a key from the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Key [type of keys]
+ @return Iterator to the found element, or kh_end(h) is the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+ @abstract Remove a key from the hash table.
+ @param name Name of the hash table [symbol]
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param k Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+
+/*! @function
+ @abstract Test whether a bucket contains data.
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return 1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+ @abstract Get key given an iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+ @abstract Get value given an iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @param x Iterator to the bucket [khint_t]
+ @return Value [type of values]
+ @discussion For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+ @abstract Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+ @abstract Get the start iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+ @abstract Get the end iterator
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+ @abstract Get the number of elements in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+ @abstract Get the number of buckets in the hash table
+ @param h Pointer to the hash table [khash_t(name)*]
+ @return Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/* More conenient interfaces */
+
+/*! @function
+ @abstract Instantiate a hash set containing integer keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name) \
+ KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing integer keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t) \
+ KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing 64-bit integer keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name) \
+ KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing 64-bit integer keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t) \
+ KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+ @abstract Instantiate a hash map containing const char* keys
+ @param name Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name) \
+ KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+ @abstract Instantiate a hash map containing const char* keys
+ @param name Name of the hash table [symbol]
+ @param khval_t Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t) \
+ KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */
+/* The MIT License
+
+ Copyright (c) 2008, 2009, 2011 Attractive Chaos <attractor at live.co.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+/* Last Modified: 05MAR2012 */
+
+#ifndef AC_KSEQ_H
+#define AC_KSEQ_H
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+
+#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r
+#define KS_SEP_TAB 1 // isspace() && !' '
+#define KS_SEP_LINE 2 // line separator: "\n" (Unix) or "\r\n" (Windows)
+#define KS_SEP_MAX 2
+
+#define __KS_TYPE(type_t) \
+ typedef struct __kstream_t { \
+ unsigned char *buf; \
+ int begin, end, is_eof; \
+ type_t f; \
+ } kstream_t;
+
+#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end)
+#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0)
+
+#define __KS_BASIC(type_t, __bufsize) \
+ static inline kstream_t *ks_init(type_t f) \
+ { \
+ kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t)); \
+ ks->f = f; \
+ ks->buf = (unsigned char*)malloc(__bufsize); \
+ return ks; \
+ } \
+ static inline void ks_destroy(kstream_t *ks) \
+ { \
+ if (ks) { \
+ free(ks->buf); \
+ free(ks); \
+ } \
+ }
+
+#define __KS_GETC(__read, __bufsize) \
+ static inline int ks_getc(kstream_t *ks) \
+ { \
+ if (ks->is_eof && ks->begin >= ks->end) return -1; \
+ if (ks->begin >= ks->end) { \
+ ks->begin = 0; \
+ ks->end = __read(ks->f, ks->buf, __bufsize); \
+ if (ks->end < __bufsize) ks->is_eof = 1; \
+ if (ks->end == 0) return -1; \
+ } \
+ return (int)ks->buf[ks->begin++]; \
+ }
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+ size_t l, m;
+ char *s;
+} kstring_t;
+#endif
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#define __KS_GETUNTIL(__read, __bufsize) \
+ static int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append) \
+ { \
+ if (dret) *dret = 0; \
+ str->l = append? str->l : 0; \
+ if (ks->begin >= ks->end && ks->is_eof) return -1; \
+ for (;;) { \
+ int i; \
+ if (ks->begin >= ks->end) { \
+ if (!ks->is_eof) { \
+ ks->begin = 0; \
+ ks->end = __read(ks->f, ks->buf, __bufsize); \
+ if (ks->end < __bufsize) ks->is_eof = 1; \
+ if (ks->end == 0) break; \
+ } else break; \
+ } \
+ if (delimiter == KS_SEP_LINE) { \
+ for (i = ks->begin; i < ks->end; ++i) \
+ if (ks->buf[i] == '\n') break; \
+ } else if (delimiter > KS_SEP_MAX) { \
+ for (i = ks->begin; i < ks->end; ++i) \
+ if (ks->buf[i] == delimiter) break; \
+ } else if (delimiter == KS_SEP_SPACE) { \
+ for (i = ks->begin; i < ks->end; ++i) \
+ if (isspace(ks->buf[i])) break; \
+ } else if (delimiter == KS_SEP_TAB) { \
+ for (i = ks->begin; i < ks->end; ++i) \
+ if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break; \
+ } else i = 0; /* never come to here! */ \
+ if (str->m - str->l < (size_t)(i - ks->begin + 1)) { \
+ str->m = str->l + (i - ks->begin) + 1; \
+ kroundup32(str->m); \
+ str->s = (char*)realloc(str->s, str->m); \
+ } \
+ memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin); \
+ str->l = str->l + (i - ks->begin); \
+ ks->begin = i + 1; \
+ if (i < ks->end) { \
+ if (dret) *dret = ks->buf[i]; \
+ break; \
+ } \
+ } \
+ if (str->s == 0) { \
+ str->m = 1; \
+ str->s = (char*)calloc(1, 1); \
+ } else if (delimiter == KS_SEP_LINE && str->l > 1 && str->s[str->l-1] == '\r') --str->l; \
+ str->s[str->l] = '\0'; \
+ return str->l; \
+ } \
+ static inline int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \
+ { return ks_getuntil2(ks, delimiter, str, dret, 0); }
+
+#define KSTREAM_INIT(type_t, __read, __bufsize) \
+ __KS_TYPE(type_t) \
+ __KS_BASIC(type_t, __bufsize) \
+ __KS_GETC(__read, __bufsize) \
+ __KS_GETUNTIL(__read, __bufsize)
+
+#define kseq_rewind(ks) ((ks)->last_char = (ks)->f->is_eof = (ks)->f->begin = (ks)->f->end = 0)
+
+#define __KSEQ_BASIC(SCOPE, type_t) \
+ SCOPE kseq_t *kseq_init(type_t fd) \
+ { \
+ kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t)); \
+ s->f = ks_init(fd); \
+ return s; \
+ } \
+ SCOPE void kseq_destroy(kseq_t *ks) \
+ { \
+ if (!ks) return; \
+ free(ks->name.s); free(ks->comment.s); free(ks->seq.s); free(ks->qual.s); \
+ ks_destroy(ks->f); \
+ free(ks); \
+ }
+
+/* Return value:
+ >=0 length of the sequence (normal)
+ -1 end-of-file
+ -2 truncated quality string
+ */
+#define __KSEQ_READ(SCOPE) \
+ SCOPE int kseq_read(kseq_t *seq) \
+ { \
+ int c; \
+ kstream_t *ks = seq->f; \
+ if (seq->last_char == 0) { /* then jump to the next header line */ \
+ while ((c = ks_getc(ks)) != -1 && c != '>' && c != '@'); \
+ if (c == -1) return -1; /* end of file */ \
+ seq->last_char = c; \
+ } /* else: the first header char has been read in the previous call */ \
+ seq->comment.l = seq->seq.l = seq->qual.l = 0; /* reset all members */ \
+ if (ks_getuntil(ks, 0, &seq->name, &c) < 0) return -1; /* normal exit: EOF */ \
+ if (c != '\n') ks_getuntil(ks, KS_SEP_LINE, &seq->comment, 0); /* read FASTA/Q comment */ \
+ if (seq->seq.s == 0) { /* we can do this in the loop below, but that is slower */ \
+ seq->seq.m = 256; \
+ seq->seq.s = (char*)malloc(seq->seq.m); \
+ } \
+ while ((c = ks_getc(ks)) != -1 && c != '>' && c != '+' && c != '@') { \
+ if (c == '\n') continue; /* skip empty lines */ \
+ seq->seq.s[seq->seq.l++] = c; /* this is safe: we always have enough space for 1 char */ \
+ ks_getuntil2(ks, KS_SEP_LINE, &seq->seq, 0, 1); /* read the rest of the line */ \
+ } \
+ if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */ \
+ if (seq->seq.l + 1 >= seq->seq.m) { /* seq->seq.s[seq->seq.l] below may be out of boundary */ \
+ seq->seq.m = seq->seq.l + 2; \
+ kroundup32(seq->seq.m); /* rounded to the next closest 2^k */ \
+ seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \
+ } \
+ seq->seq.s[seq->seq.l] = 0; /* null terminated string */ \
+ if (c != '+') return seq->seq.l; /* FASTA */ \
+ if (seq->qual.m < seq->seq.m) { /* allocate memory for qual in case insufficient */ \
+ seq->qual.m = seq->seq.m; \
+ seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \
+ } \
+ while ((c = ks_getc(ks)) != -1 && c != '\n'); /* skip the rest of '+' line */ \
+ if (c == -1) return -2; /* error: no quality string */ \
+ while (ks_getuntil2(ks, KS_SEP_LINE, &seq->qual, 0, 1) >= 0 && seq->qual.l < seq->seq.l); \
+ seq->last_char = 0; /* we have not come to the next header line */ \
+ if (seq->seq.l != seq->qual.l) return -2; /* error: qual string is of a different length */ \
+ return seq->seq.l; \
+ }
+
+#define __KSEQ_TYPE(type_t) \
+ typedef struct { \
+ kstring_t name, comment, seq, qual; \
+ int last_char; \
+ kstream_t *f; \
+ } kseq_t;
+
+#define KSEQ_INIT2(SCOPE, type_t, __read) \
+ KSTREAM_INIT(type_t, __read, 16384) \
+ __KSEQ_TYPE(type_t) \
+ __KSEQ_BASIC(SCOPE, type_t) \
+ __KSEQ_READ(SCOPE)
+
+#define KSEQ_INIT(type_t, __read) KSEQ_INIT2(static, type_t, __read)
+
+#define KSEQ_DECLARE(type_t) \
+ __KS_TYPE(type_t) \
+ __KSEQ_TYPE(type_t) \
+ extern kseq_t *kseq_init(type_t fd); \
+ void kseq_destroy(kseq_t *ks); \
+ int kseq_read(kseq_t *seq);
+
+#endif
diff --git a/src/AS_PBR/blasr.Linux-amd64.bin b/src/AS_PBR/blasr.Linux-amd64.bin
new file mode 100755
index 0000000..80489e6
Binary files /dev/null and b/src/AS_PBR/blasr.Linux-amd64.bin differ
diff --git a/src/AS_PBR/boost/._LICENSE_1_0.txt b/src/AS_PBR/boost/._LICENSE_1_0.txt
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/._LICENSE_1_0.txt differ
diff --git a/src/AS_PBR/boost/._config b/src/AS_PBR/boost/._config
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/._config differ
diff --git a/src/AS_PBR/boost/._config.hpp b/src/AS_PBR/boost/._config.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/._config.hpp differ
diff --git a/src/AS_PBR/boost/._detail b/src/AS_PBR/boost/._detail
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/._detail differ
diff --git a/src/AS_PBR/boost/._dynamic_bitset b/src/AS_PBR/boost/._dynamic_bitset
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/._dynamic_bitset differ
diff --git a/src/AS_PBR/boost/._dynamic_bitset.hpp b/src/AS_PBR/boost/._dynamic_bitset.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/._dynamic_bitset.hpp differ
diff --git a/src/AS_PBR/boost/._dynamic_bitset_fwd.hpp b/src/AS_PBR/boost/._dynamic_bitset_fwd.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/._dynamic_bitset_fwd.hpp differ
diff --git a/src/AS_PBR/boost/._limits.hpp b/src/AS_PBR/boost/._limits.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/._limits.hpp differ
diff --git a/src/AS_PBR/boost/._pending b/src/AS_PBR/boost/._pending
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/._pending differ
diff --git a/src/AS_PBR/boost/._static_assert.hpp b/src/AS_PBR/boost/._static_assert.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/._static_assert.hpp differ
diff --git a/src/AS_PBR/boost/config/._abi b/src/AS_PBR/boost/config/._abi
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/config/._abi differ
diff --git a/src/AS_PBR/boost/config/._abi_prefix.hpp b/src/AS_PBR/boost/config/._abi_prefix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._abi_prefix.hpp differ
diff --git a/src/AS_PBR/boost/config/._abi_suffix.hpp b/src/AS_PBR/boost/config/._abi_suffix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._abi_suffix.hpp differ
diff --git a/src/AS_PBR/boost/config/._auto_link.hpp b/src/AS_PBR/boost/config/._auto_link.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._auto_link.hpp differ
diff --git a/src/AS_PBR/boost/config/._compiler b/src/AS_PBR/boost/config/._compiler
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/config/._compiler differ
diff --git a/src/AS_PBR/boost/config/._no_tr1 b/src/AS_PBR/boost/config/._no_tr1
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/config/._no_tr1 differ
diff --git a/src/AS_PBR/boost/config/._platform b/src/AS_PBR/boost/config/._platform
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/config/._platform differ
diff --git a/src/AS_PBR/boost/config/._posix_features.hpp b/src/AS_PBR/boost/config/._posix_features.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._posix_features.hpp differ
diff --git a/src/AS_PBR/boost/config/._requires_threads.hpp b/src/AS_PBR/boost/config/._requires_threads.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._requires_threads.hpp differ
diff --git a/src/AS_PBR/boost/config/._select_compiler_config.hpp b/src/AS_PBR/boost/config/._select_compiler_config.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._select_compiler_config.hpp differ
diff --git a/src/AS_PBR/boost/config/._select_platform_config.hpp b/src/AS_PBR/boost/config/._select_platform_config.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._select_platform_config.hpp differ
diff --git a/src/AS_PBR/boost/config/._select_stdlib_config.hpp b/src/AS_PBR/boost/config/._select_stdlib_config.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._select_stdlib_config.hpp differ
diff --git a/src/AS_PBR/boost/config/._stdlib b/src/AS_PBR/boost/config/._stdlib
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PBR/boost/config/._stdlib differ
diff --git a/src/AS_PBR/boost/config/._suffix.hpp b/src/AS_PBR/boost/config/._suffix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._suffix.hpp differ
diff --git a/src/AS_PBR/boost/config/._user.hpp b/src/AS_PBR/boost/config/._user.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._user.hpp differ
diff --git a/src/AS_PBR/boost/config/._warning_disable.hpp b/src/AS_PBR/boost/config/._warning_disable.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/._warning_disable.hpp differ
diff --git a/src/AS_PBR/boost/config/abi/._borland_prefix.hpp b/src/AS_PBR/boost/config/abi/._borland_prefix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/abi/._borland_prefix.hpp differ
diff --git a/src/AS_PBR/boost/config/abi/._borland_suffix.hpp b/src/AS_PBR/boost/config/abi/._borland_suffix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/abi/._borland_suffix.hpp differ
diff --git a/src/AS_PBR/boost/config/abi/._msvc_prefix.hpp b/src/AS_PBR/boost/config/abi/._msvc_prefix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/abi/._msvc_prefix.hpp differ
diff --git a/src/AS_PBR/boost/config/abi/._msvc_suffix.hpp b/src/AS_PBR/boost/config/abi/._msvc_suffix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/abi/._msvc_suffix.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._borland.hpp b/src/AS_PBR/boost/config/compiler/._borland.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._borland.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._clang.hpp b/src/AS_PBR/boost/config/compiler/._clang.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._clang.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._codegear.hpp b/src/AS_PBR/boost/config/compiler/._codegear.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._codegear.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._comeau.hpp b/src/AS_PBR/boost/config/compiler/._comeau.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._comeau.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._common_edg.hpp b/src/AS_PBR/boost/config/compiler/._common_edg.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._common_edg.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._compaq_cxx.hpp b/src/AS_PBR/boost/config/compiler/._compaq_cxx.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._compaq_cxx.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._cray.hpp b/src/AS_PBR/boost/config/compiler/._cray.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._cray.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._digitalmars.hpp b/src/AS_PBR/boost/config/compiler/._digitalmars.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._digitalmars.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._gcc.hpp b/src/AS_PBR/boost/config/compiler/._gcc.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._gcc.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._gcc_xml.hpp b/src/AS_PBR/boost/config/compiler/._gcc_xml.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._gcc_xml.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._greenhills.hpp b/src/AS_PBR/boost/config/compiler/._greenhills.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._greenhills.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._hp_acc.hpp b/src/AS_PBR/boost/config/compiler/._hp_acc.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._hp_acc.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._intel.hpp b/src/AS_PBR/boost/config/compiler/._intel.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._intel.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._kai.hpp b/src/AS_PBR/boost/config/compiler/._kai.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._kai.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._metrowerks.hpp b/src/AS_PBR/boost/config/compiler/._metrowerks.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._metrowerks.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._mpw.hpp b/src/AS_PBR/boost/config/compiler/._mpw.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._mpw.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._nvcc.hpp b/src/AS_PBR/boost/config/compiler/._nvcc.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._nvcc.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._pathscale.hpp b/src/AS_PBR/boost/config/compiler/._pathscale.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._pathscale.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._pgi.hpp b/src/AS_PBR/boost/config/compiler/._pgi.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._pgi.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._sgi_mipspro.hpp b/src/AS_PBR/boost/config/compiler/._sgi_mipspro.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._sgi_mipspro.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._sunpro_cc.hpp b/src/AS_PBR/boost/config/compiler/._sunpro_cc.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._sunpro_cc.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._vacpp.hpp b/src/AS_PBR/boost/config/compiler/._vacpp.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._vacpp.hpp differ
diff --git a/src/AS_PBR/boost/config/compiler/._visualc.hpp b/src/AS_PBR/boost/config/compiler/._visualc.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/compiler/._visualc.hpp differ
diff --git a/src/AS_PBR/boost/config/no_tr1/._cmath.hpp b/src/AS_PBR/boost/config/no_tr1/._cmath.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/no_tr1/._cmath.hpp differ
diff --git a/src/AS_PBR/boost/config/no_tr1/._complex.hpp b/src/AS_PBR/boost/config/no_tr1/._complex.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/no_tr1/._complex.hpp differ
diff --git a/src/AS_PBR/boost/config/no_tr1/._functional.hpp b/src/AS_PBR/boost/config/no_tr1/._functional.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/no_tr1/._functional.hpp differ
diff --git a/src/AS_PBR/boost/config/no_tr1/._memory.hpp b/src/AS_PBR/boost/config/no_tr1/._memory.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/no_tr1/._memory.hpp differ
diff --git a/src/AS_PBR/boost/config/no_tr1/._utility.hpp b/src/AS_PBR/boost/config/no_tr1/._utility.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/no_tr1/._utility.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._aix.hpp b/src/AS_PBR/boost/config/platform/._aix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._aix.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._amigaos.hpp b/src/AS_PBR/boost/config/platform/._amigaos.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._amigaos.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._beos.hpp b/src/AS_PBR/boost/config/platform/._beos.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._beos.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._bsd.hpp b/src/AS_PBR/boost/config/platform/._bsd.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._bsd.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._cray.hpp b/src/AS_PBR/boost/config/platform/._cray.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._cray.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._cygwin.hpp b/src/AS_PBR/boost/config/platform/._cygwin.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._cygwin.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._hpux.hpp b/src/AS_PBR/boost/config/platform/._hpux.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._hpux.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._irix.hpp b/src/AS_PBR/boost/config/platform/._irix.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._irix.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._linux.hpp b/src/AS_PBR/boost/config/platform/._linux.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._linux.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._macos.hpp b/src/AS_PBR/boost/config/platform/._macos.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._macos.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._qnxnto.hpp b/src/AS_PBR/boost/config/platform/._qnxnto.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._qnxnto.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._solaris.hpp b/src/AS_PBR/boost/config/platform/._solaris.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._solaris.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._symbian.hpp b/src/AS_PBR/boost/config/platform/._symbian.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._symbian.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._vms.hpp b/src/AS_PBR/boost/config/platform/._vms.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._vms.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._vxworks.hpp b/src/AS_PBR/boost/config/platform/._vxworks.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._vxworks.hpp differ
diff --git a/src/AS_PBR/boost/config/platform/._win32.hpp b/src/AS_PBR/boost/config/platform/._win32.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/platform/._win32.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._dinkumware.hpp b/src/AS_PBR/boost/config/stdlib/._dinkumware.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._dinkumware.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._libcomo.hpp b/src/AS_PBR/boost/config/stdlib/._libcomo.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._libcomo.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._libcpp.hpp b/src/AS_PBR/boost/config/stdlib/._libcpp.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._libcpp.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._libstdcpp3.hpp b/src/AS_PBR/boost/config/stdlib/._libstdcpp3.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._libstdcpp3.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._modena.hpp b/src/AS_PBR/boost/config/stdlib/._modena.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._modena.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._msl.hpp b/src/AS_PBR/boost/config/stdlib/._msl.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._msl.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._roguewave.hpp b/src/AS_PBR/boost/config/stdlib/._roguewave.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._roguewave.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._sgi.hpp b/src/AS_PBR/boost/config/stdlib/._sgi.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._sgi.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._stlport.hpp b/src/AS_PBR/boost/config/stdlib/._stlport.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._stlport.hpp differ
diff --git a/src/AS_PBR/boost/config/stdlib/._vacpp.hpp b/src/AS_PBR/boost/config/stdlib/._vacpp.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/config/stdlib/._vacpp.hpp differ
diff --git a/src/AS_PBR/boost/detail/._dynamic_bitset.hpp b/src/AS_PBR/boost/detail/._dynamic_bitset.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/detail/._dynamic_bitset.hpp differ
diff --git a/src/AS_PBR/boost/detail/._iterator.hpp b/src/AS_PBR/boost/detail/._iterator.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/detail/._iterator.hpp differ
diff --git a/src/AS_PBR/boost/detail/._workaround.hpp b/src/AS_PBR/boost/detail/._workaround.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/detail/._workaround.hpp differ
diff --git a/src/AS_PBR/boost/dynamic_bitset/._config.hpp b/src/AS_PBR/boost/dynamic_bitset/._config.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/dynamic_bitset/._config.hpp differ
diff --git a/src/AS_PBR/boost/dynamic_bitset/._dynamic_bitset.hpp b/src/AS_PBR/boost/dynamic_bitset/._dynamic_bitset.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/dynamic_bitset/._dynamic_bitset.hpp differ
diff --git a/src/AS_PBR/boost/pending/._integer_log2.hpp b/src/AS_PBR/boost/pending/._integer_log2.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/pending/._integer_log2.hpp differ
diff --git a/src/AS_PBR/boost/pending/._lowest_bit.hpp b/src/AS_PBR/boost/pending/._lowest_bit.hpp
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PBR/boost/pending/._lowest_bit.hpp differ
diff --git a/src/AS_PBR/convertToPBCNS.pl b/src/AS_PBR/convertToPBCNS.pl
index ae01158..3e1654e 100755
--- a/src/AS_PBR/convertToPBCNS.pl
+++ b/src/AS_PBR/convertToPBCNS.pl
@@ -1,8 +1,9 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
# Copyright (C) 2011, Battelle National Biodefense Institute (BNBI);
# all rights reserved. Authored by: Sergey Koren
#
+
# This Software was prepared for the Department of Homeland Security
# (DHS) by the Battelle National Biodefense Institute, LLC (BNBI) as
# part of contract HSHQDC-07-C-00020 to manage and operate the National
@@ -54,6 +55,7 @@ use FileHandle;
use POSIX qw(ceil floor sys_wait_h);
my %global;
+my $layCounter = 0;
sub getGlobal ($) {
my $var = shift @_;
@@ -76,42 +78,217 @@ sub setGlobal ($$) {
$val =~ s/\"/\\\"/g;
$val =~ s/\\\$/\$/g;
$val =~ s/\$/\\\$/g;
+
+ if ($var eq "consensus") {
+ if ($val eq "pbdagcon") {
+ setGlobal("batch", 1000);
+ setGlobal("runBlasr", 1);
+ } elsif ($val eq "falcon" ){
+ setGlobal("batch", 1000);
+ setGlobal("runBlasr", 0);
+ } elsif ($val eq "fast") {
+ setGlobal("batch", 1);
+ setGlobal("runBlasr", 0);
+ } elsif ($val eq "pbutgcns") {
+ setGlobal("batch", 1);
+ setGlobal("runBlasr", 0);
+ } else {
+ die "Unknown consensus type $val specified.\n";
+ }
+ }
}
+sub flushFalcon($) {
+ my $output = shift @_;
+ my $consensus = getGlobal("consensus");
+
+ if ($consensus ne "falcon") { return; }
+
+ my $threads = getGlobal("threads");
+ my $cov = getGlobal("coverage");
+ my $path = getGlobal("smrtpath");
+ my $python = getGlobal("falconpath");
+ my $prefix = getGlobal("prefix");
+ my $fileName = "$prefix.cns.in";
+
+ open(CNS, ">> $fileName") or die("Couldn't open '$fileName'", undef);
+ print CNS "- -\n";
+ close(CNS);
+
+ my $libPath = "$python";
+ $libPath =~ s/bin/lib/g;
+ my $smrtPortalPath = "$path";
+ $smrtPortalPath =~ s/bin/lib/g;
+ my $pythonPath = "$path:$libPath/python2.7/site-packages:$smrtPortalPath/python2.7:\$PYTHONPATH";
+ system("export PYTHONPATH=$pythonPath && cat $fileName | $python/falcon_sense.py --trim 1 --local_match_count_threshold 3 --min_cov $cov --n_core $threads >> $output 2> $prefix.err");
-sub processLayout($$$$) {
+ if ($? != 0) {
+ die "Error: falcon_sense could not run successfully"
+ }
+}
+
+sub flushPBDAGCON($) {
+ my $output = shift @_;
+ my $consensus = getGlobal("consensus");
+
+ if ($consensus ne "pbdagcon") { return; }
+
+ my $threads = getGlobal("threads");
+ my $cov = getGlobal("coverage");
+ my $path = getGlobal("smrtpath");
+ my $length = getGlobal("length");
+ my $python = getGlobal("falconpath");
+ my $prefix = getGlobal("prefix");
+
+ if (! -e "$prefix.m5" || ! -s "$prefix.m5") { return; }
+ system("$path/pbdagcon -t 0 -m $length -j $threads -c $cov $prefix.m5 >> $output 2>/dev/null");
+ if ($? != 0) {
+ die "Error: pbdagon could not run successfully"
+ }
+}
+
+sub flushPBUTGCNS($) {
+ my $output = shift @_;
+ my $consensus = getGlobal("consensus");
+
+ if ($consensus ne "pbutgcns") { return; }
+
+ my $threads = getGlobal("threads");
+ my $cov = getGlobal("coverage");
+ my $path = getGlobal("smrtpath");
+ my $length = getGlobal("length");
+ my $python = getGlobal("falconpath");
+ my $prefix = getGlobal("prefix");
+
+ if (! -e "$prefix.cns.in" || ! -s "$prefix.cns.in") { return; }
+ system("$path/pbutgcns -j $threads $prefix.cns.in >> $output 2>/dev/null");
+ if ($? != 0) {
+ die "Error: pbutgcns could not run successfully"
+ }
+}
+
+sub processLayout($$$$$) {
my $output = shift @_;
my $id = shift @_;
my $cns = shift @_;
my $hashref = shift @_;
+ my $sensitive = shift @_;
my %hash = %$hashref;
my $prefix = getGlobal("prefix");
my $path = getGlobal("smrtpath");
my $threads = getGlobal("threads");
my $cov = getGlobal("coverage");
my $length = getGlobal("length");
+ my $consensus = getGlobal("consensus");
- open(CNS, "> $prefix.cns.fasta") or die("Couldn't open '$prefix.cns.fasta'", undef);
+ my $alnCount = scalar keys %hash;
$cns =~ s/^\s+|\s+$//g;
- print CNS ">$id\n";
- print CNS "$cns\n";
- close(CNS);
- open(ALN, "> $prefix.aln.fasta") or die ("Couldn't open '$prefix.aln.fasta'", undef);
- for (keys %hash) {
- print ALN ">$_\n";
- my $aln = $hash{$_};
- $aln =~ s/^\s+|\s+$//g;
- print ALN "$aln\n";
- }
- close(ALN);
- system("$path/blasr $prefix.aln.fasta $prefix.cns.fasta -m 5 -nproc $threads > $prefix.m5 2>/dev/null");
- if ($? != 0) {
- die "Error: blasr could not run successfully"
+ # just output our frakenconsensus
+ if ($consensus eq "fast") {
+ open(OUT, ">> $output") or die("Couldn't open '$output'", undef);
+ print OUT ">$id\n";
+ print OUT "$cns\n";
+ close(OUT);
+ return;
}
- system("$path/pbdagcon -t 0 -m $length -j $threads -c $cov $prefix.m5 >> $output 2>/dev/null");
- if ($? != 0) {
- die "Error: pbdagon could not run successfully"
+
+ # now we format our data for whatever consensus was selected
+ if ($alnCount == 0) {
+ return;
+ } elsif ($alnCount == 1) {
+ if ($cov <= 1) {
+ open(OUT, ">> $output") or die("Couldn't open '$output'", undef);
+ print OUT ">$id\n";
+ print OUT "$cns\n";
+ close(OUT);
+ } else {
+ # skip low coverage stuff
+ return;
+ }
+ } elsif (length($cns) < $length) {
+ # skip short stuff
+ return;
+ } elsif ($consensus eq "pbdagcon") {
+ open(CNS, "> $prefix.cns.fasta") or die("Couldn't open '$prefix.cns.fasta'", undef);
+ print CNS ">$id\n";
+ print CNS "$cns\n";
+ close(CNS);
+
+ open(ALN, "> $prefix.aln.fasta") or die ("Couldn't open '$prefix.aln.fasta'", undef);
+ for (keys %hash) {
+ print ALN ">$_\n";
+ my $str = $hash{$_};
+ my @tokenized = split('\s+', $str);
+ my $aln = $tokenized[2];
+ $aln =~ s/^\s+|\s+$//g;
+ print ALN "$aln\n";
+ }
+ close(ALN);
+
+ if ($sensitive) {
+ system("$path/blasr $prefix.aln.fasta $prefix.cns.fasta -sdpDel 5 -minMatch 10 -maxMatch 16 -bestn 1 -m 5 -nproc $threads >> $prefix.m5 2>/dev/null");
+ } else {
+ system("$path/blasr $prefix.aln.fasta $prefix.cns.fasta -maxMatch 16 -bestn 1 -m 5 -nproc $threads >> $prefix.m5 2>/dev/null");
+ }
+ if ($? != 0) {
+ die "Error: blasr could not run successfully"
+ }
+ $layCounter++;
+ if ($layCounter > getGlobal("batch")) {
+ flushPBDAGCON($output);
+ open(CNS, "> $prefix.m5") or die("Couldn't open '$prefix.m5'", undef);
+ close(CNS);
+ $layCounter = 0;
+ }
+ } elsif ($consensus eq "pbutgcns") {
+ if ($layCounter == 0) {
+ open(CNS, "> $prefix.cns.in") or die ("Couldn't open '$prefix.cns.in'", undef);
+ } else {
+ open(CNS, ">> $prefix.cns.in") or die ("Couldn't open '$prefix.cns.in'", undef);
+ }
+ print CNS "$id $cns\n";
+ for (keys %hash) {
+ my $str = $hash{$_};
+ my @tokenized = split('\s+', $str);
+ my $aln = $tokenized[2];
+ $aln =~ s/^\s+|\s+$//g;
+ print CNS "$_ $tokenized[0] $tokenized[1] $aln\n";
+ }
+ close(CNS);
+ $layCounter++;
+ if ($layCounter >= getGlobal("batch")) {
+ flushPBUTGCNS($output);
+ open(CNS, "> $prefix.cns.in") or die("Couldn't open '$prefix.cns.in'", undef);
+ close(CNS);
+ $layCounter = 0;
+ }
+ } elsif ($consensus eq "falcon") {
+ if ($layCounter == 0) {
+ open(CNS, "> $prefix.cns.in") or die("Couldn't open '$prefix.cns.in'", undef);
+ } else {
+ open(CNS, ">> $prefix.cns.in") or die("Couldn't open '$prefix.cns.in'", undef);
+ }
+ $id =~ s/^\s+|\s+$//g;
+ $cns =~ s/^\s+|\s+$//g;
+ print CNS "$id $cns\n";
+ for (keys %hash) {
+ my $str = $hash{$_};
+ my @tokenized = split('\s+', $str);
+ my $aln = $tokenized[2];
+ $aln =~ s/^\s+|\s+$//g;
+
+ print CNS "$_ $aln\n";
+ }
+ print CNS "+ +\n";
+ close(CNS);
+ $layCounter++;
+ if ($layCounter > getGlobal("batch")) {
+ flushFalcon($output);
+ open(CNS, "> $prefix.cns.in") or die("Couldn't open '$prefix.cns.in'", undef);
+ close(CNS);
+ $layCounter = 0;
+ }
}
}
@@ -137,11 +314,19 @@ sub processTig($$$) {
}
$fastaSeq = "";
$id = (split '\s+')[0];
- $id = (split ',', $id)[1];
+ $id =~ s/>//g;
+ if ($id =~ m/,/) {
+ $id = (split ',', $id)[1];
+ } elsif ($id =~ m/utg/) {
+ $id =~ s/utg//g;
+ }
} else {
$fastaSeq = $fastaSeq . $_;
}
}
+ if ($fastaSeq ne "") {
+ $seqs{$id} = $fastaSeq;
+ }
close(F);
my $count = (scalar keys %seqs);
print STDERR "Loaded " . $count . " sequences\n";
@@ -150,6 +335,7 @@ sub processTig($$$) {
my %lens = {};
my $utgID = 0;
my $lastEnd = 0;
+ my $isUTG = 0;
open(F, "< $input") or die("Couldn't open '$input'", undef);
while (<F>) {
s/^\s+//;
@@ -159,6 +345,13 @@ sub processTig($$$) {
next if (m/^\s*$/);
my @tokenized = split '\s+';
if ($tokenized[0] eq "unitig") {
+ $isUTG = 1;
+ if ($lastEnd != 0) {
+ $lens{$utgID} = $lastEnd;
+ }
+ $utgID = $tokenized[1];
+ } elsif ($tokenized[0] eq "contig") {
+ $isUTG = 0;
if ($lastEnd != 0) {
$lens{$utgID} = $lastEnd;
}
@@ -185,8 +378,14 @@ sub processTig($$$) {
if ($tokenized[0] eq "unitig") {
$lastEnd = $lens{$tokenized[1]};
$utgID = $tokenized[1];
+ $isUTG = 1;
$counter++;
- } elsif ($tokenized[0] eq "FRG") {
+ } elsif ($tokenized[0] eq "contig") {
+ $lastEnd = $lens{$tokenized[1]};
+ $utgID = $tokenized[1];
+ $isUTG = 0;
+ $counter++;
+ } elsif ($tokenized[0] eq "FRG" && $isUTG != 0) {
my $min = ($tokenized[13] < $tokenized[14] ? $tokenized[13] : $tokenized[14]);
my $max = ($tokenized[13] > $tokenized[14] ? $tokenized[13] : $tokenized[14]);
my $fwd = ($tokenized[13] < $tokenized[14] ? "+" : "-");
@@ -197,6 +396,17 @@ sub processTig($$$) {
$fwd = "+";
}
print OUT "$tokenized[4] utg_$utgID $fwd $lastEnd $min $max $seq $seq\n";
+ } elsif ($tokenized[0] eq "UTG" && $isUTG == 0) {
+ my $min = ($tokenized[6] < $tokenized[7] ? $tokenized[6] : $tokenized[7]);
+ my $max = ($tokenized[6] > $tokenized[7] ? $tokenized[6] : $tokenized[7]);
+ my $fwd = ($tokenized[6] < $tokenized[7] ? "+" : "-");
+ my $seq = $seqs{$tokenized[4]};
+ if ($fwd eq "-") {
+ $seq =~ tr/ATGCatgcNn/TACGtacgNn/;
+ $seq = reverse $seq;
+ $fwd = "+";
+ }
+ print OUT "$tokenized[4] utg_$utgID $fwd $lastEnd $min $max $seq $seq\n";
}
}
close(F);
@@ -204,16 +414,27 @@ sub processTig($$$) {
print STDERR "Converted $counter unitigs\n";
}
-sub processLayouts($$) {
+sub processLayouts($$$) {
my $input = shift @_;
my $output = shift @_;
+ my $sensitive = shift @_;
+ my $coverage = getGlobal("maxCoverage");
+ my $consensus = getGlobal("consensus");
my $lastEnd = 0;
+ my $lastOffset=0;
my $cns = "";
my $lastID = "";
my %toAlign = {};
- open(F, "< $input") or die("Couldn't open '$input'", undef);
- while (<F>) {
+ my %bpCov = {};
+
+ my $F;
+ if (defined($input)) {
+ open($F, "< $input") or die("Couldn't open '$input'", undef);
+ } else {
+ $F = *STDIN;
+ }
+ while (<$F>) {
s/^\s+//;
s/\s+$//;
@@ -223,34 +444,64 @@ sub processLayouts($$) {
my $hashLen = scalar keys %toAlign;
if ($lastID eq "" || $lastID ne $tokenized[1]) {
- if ($lastID ne "") { processLayout($output, $lastID, $cns, \%toAlign); }
+ if ($lastID ne "") { processLayout($output, $lastID, $cns, \%toAlign, $sensitive); }
for (keys %toAlign) { delete $toAlign{$_}; }
$cns = ' ' x ($tokenized[3] + 1);
+ for (keys %bpCov) { delete $bpCov{$_}; }
$lastEnd = 0;
+ $lastOffset = 0;
}
my $end = $tokenized[5];
my $start = $tokenized[4];
+ if ($start < 0) {
+ $lastOffset = -1*$start;
+ $cns = ' ' x ($tokenized[3] + 1 + $lastOffset);
+ }
if ($end >= $lastEnd) {
+ if ($start > $lastEnd) {
+ $start = $lastEnd;
+ }
if ($end - $start > length($tokenized[7])) {
$end = $start + length($tokenized[7]);
}
$lastEnd = $end;
- substr($cns, $start, $end-$start) = $tokenized[7];
+ substr($cns, $start+$lastOffset, $end-$start) = $tokenized[7];
+ }
+ my $belowCov = 0;
+ if ($sensitive && $consensus eq "pbdagcon") {
+ for (my $i = $start; $i < $end; $i++) {
+ if (!defined($bpCov{$i})) {
+ $bpCov{$i} = 0;
+ }
+ if ($bpCov{$i} < $coverage) {
+ $belowCov++;
+ }
+ $bpCov{$i} = $bpCov{$i} + 1;
+ }
+ }
+ if ($belowCov > length($tokenized[6])*0.2 || $consensus ne "pbdagcon" || $sensitive == 0) {
+ $toAlign{$tokenized[0]} = $start+$lastOffset . " " . $end . " " . $tokenized[6];
}
- $toAlign{$tokenized[0]} = $tokenized[6];
$lastID = $tokenized[1];
-
}
if ($lastID ne "") {
- processLayout($output, $lastID, $cns, \%toAlign);
+ processLayout($output, $lastID, $cns, \%toAlign, $sensitive);
+ }
+ flushFalcon($output);
+ flushPBDAGCON($output);
+ flushPBUTGCNS($output);
+ if (defined($input)) {
+ close($F);
}
- close(F);
}
my $err = 0;
+setGlobal("consensus", "pbdagcon");
setGlobal("prefix", "tmp");
setGlobal("coverage", 0);
setGlobal("threads", 1);
+setGlobal("maxCoverage", 5000);
+
my $input = undef;
my $output = undef;
my $inputSequences = undef;
@@ -280,6 +531,9 @@ while (scalar(@ARGV) > 0) {
} elsif ($arg eq "-path") {
setGlobal("smrtpath", shift @ARGV);
+ } elsif ($arg eq "-pythonPath") {
+ setGlobal("falconpath", shift @ARGV);
+
} elsif ($arg eq "-input") {
$input = shift @ARGV;
@@ -289,14 +543,16 @@ while (scalar(@ARGV) > 0) {
} elsif ($arg eq "-sequence") {
$inputSequences = shift @ARGV;
+ } elsif ($arg eq "-consensus") {
+ setGlobal("consensus", shift @ARGV);
+
} else {
print STDERR "Unknown parameter " + $err + "\n";
$err++;
}
}
-if ($err > 0 || !defined($input) || !defined($output)) {
- print STDERR "Error: no valid input lay specified\n" if (!defined($input));
+if ($err > 0 || !defined($output)) {
print STDERR "Error: no valid output fasta specified\n" if (!defined($output));
print STDERR "Invalid arguments specified\n";
@@ -309,21 +565,48 @@ if ($err > 0 || !defined($input) || !defined($output)) {
exit(1);
}
-# find blasr
-my $BLASR = getGlobal("smrtpath");
-if (!defined($BLASR) || $BLASR eq "") {
- # try to use path
- my $blasrPath = `which blasr`;
- chomp $blasrPath;
- my @t = split '/', "$blasrPath";
- pop @t; # blasr
- $BLASR = join '/', @t; # path to the assembler
- setGlobal("smrtpath", $BLASR);
+my $consensus = getGlobal("consensus");
+if ($consensus eq "falcon") {
+ my $falconPath = getGlobal("falconpath");
+ if (! -e "$falconPath/falcon_sense.py") {
+ print STDERR "Error: selected falcon but it was not available, using pbdagcon instead\n";
+ setGlobal("consensus", "pbdagcon");
+ }
}
-# if we really can't find it just give up
-if (! -e "$BLASR/blasr" || ! -e "$BLASR/pbdagcon") {
- print STDERR "Error, could not find blasr/smrtportal in path. Please update your path or specify a location using -path\n";
- exit(1);
+if ($consensus eq "pbutgcns") {
+ my $path = getGlobal("smrtpath");
+ if (!defined($path) || $path eq "" || ! -e "$path/pbutgcns") {
+ # try to use path
+ my $binPath = `which pbutgcns`;
+ chomp $binPath;
+ my @t = split '/', "$binPath";
+ pop @t; # pbutgncs
+ $path = join '/', @t; # path to the assembler
+ }
+ if (! -e "$path/pbutgcns") {
+ print STDERR "Error: selected pbutgcns but it was not available, using pbdagcon instead\n";
+ setGlobal("consensus", "pbdagcon");
+ } else {
+ setGlobal("smrtpath", $path);
+ }
+}
+if ($consensus eq "pbdagcon") {
+ # find blasr
+ my $BLASR = getGlobal("smrtpath");
+ if (!defined($BLASR) || $BLASR eq "") {
+ # try to use path
+ my $blasrPath = `which blasr`;
+ chomp $blasrPath;
+ my @t = split '/', "$blasrPath";
+ pop @t; # blasr
+ $BLASR = join '/', @t; # path to the assembler
+ setGlobal("smrtpath", $BLASR);
+ }
+ # if we really can't find it just give up
+ if ((! -e "$BLASR/blasr" || ! -e "$BLASR/pbdagcon")) {
+ print STDERR "Error, could not find blasr/smrtportal in path. Please update your path or specify a location using -path\n";
+ exit(1);
+ }
}
# clear output file
@@ -334,9 +617,9 @@ close(O);
if (defined($inputSequences)) {
my $out = getGlobal("prefix") . ".layout";
processTig($input, $inputSequences, $out);
- processLayouts($out, $output);
+ processLayouts($out, $output, 0);
} else {
- processLayouts($input, $output);
+ processLayouts($input, $output, 1);
}
# cleanup
system("rm -f " . getGlobal("prefix") . "*");
diff --git a/src/AS_PBR/falcon_sense.Darwin-amd64.bin b/src/AS_PBR/falcon_sense.Darwin-amd64.bin
new file mode 100755
index 0000000..71d5e04
Binary files /dev/null and b/src/AS_PBR/falcon_sense.Darwin-amd64.bin differ
diff --git a/src/AS_PBR/falcon_sense.Linux-amd64.bin b/src/AS_PBR/falcon_sense.Linux-amd64.bin
new file mode 100755
index 0000000..368d225
Binary files /dev/null and b/src/AS_PBR/falcon_sense.Linux-amd64.bin differ
diff --git a/src/AS_PBR/jellyfish.Darwin-amd64.bin b/src/AS_PBR/jellyfish.Darwin-amd64.bin
new file mode 100755
index 0000000..535c6cb
Binary files /dev/null and b/src/AS_PBR/jellyfish.Darwin-amd64.bin differ
diff --git a/src/AS_PBR/jellyfish.Linux-amd64.bin b/src/AS_PBR/jellyfish.Linux-amd64.bin
new file mode 100755
index 0000000..a7e12de
Binary files /dev/null and b/src/AS_PBR/jellyfish.Linux-amd64.bin differ
diff --git a/src/AS_PBR/libgcc_s.1.dylib b/src/AS_PBR/libgcc_s.1.dylib
new file mode 100644
index 0000000..c9d0fd0
Binary files /dev/null and b/src/AS_PBR/libgcc_s.1.dylib differ
diff --git a/src/AS_PBR/libstdc++.6.dylib b/src/AS_PBR/libstdc++.6.dylib
new file mode 100755
index 0000000..1a40525
Binary files /dev/null and b/src/AS_PBR/libstdc++.6.dylib differ
diff --git a/src/AS_PBR/mhap.tar.gz b/src/AS_PBR/mhap.tar.gz
new file mode 100644
index 0000000..0d8913b
Binary files /dev/null and b/src/AS_PBR/mhap.tar.gz differ
diff --git a/src/AS_PBR/pacBioToCA.pl b/src/AS_PBR/pacBioToCA.pl
deleted file mode 100755
index 67bcffa..0000000
--- a/src/AS_PBR/pacBioToCA.pl
+++ /dev/null
@@ -1,2056 +0,0 @@
-#!/usr/bin/perl
-
-# Copyright (C) 2011, Battelle National Biodefense Institute (BNBI);
-# all rights reserved. Authored by: Sergey Koren
-#
-# This Software was prepared for the Department of Homeland Security
-# (DHS) by the Battelle National Biodefense Institute, LLC (BNBI) as
-# part of contract HSHQDC-07-C-00020 to manage and operate the National
-# Biodefense Analysis and Countermeasures Center (NBACC), a Federally
-# Funded Research and Development Center.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# * Neither the name of the Battelle National Biodefense Institute nor
-# the names of its contributors may be used to endorse or promote
-# products derived from this software without specific prior written
-# permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-###########################################################################
-#
-# Read in fragments from fastq-format sequence and quality files,
-# correct to pacbio fragments.
-#
-
-use strict;
-
-use Config; # for @signame
-use FindBin;
-use Cwd;
-use Carp;
-use FileHandle;
-
-use POSIX qw(ceil floor sys_wait_h);
-
-my $TAB_SIZE = 8;
-
-my %global;
-
-my @nonCAOptions = ("QV", "maxCorrectionRounds", "genomeSize", "shortReads", "longReads", "pbcns", "bankPath","libraryname", "specFile", "length", "coverageCutoff", "maxCoverage", "maxGap", "maxUncorrectedGap", "samFOFN", "blasr", "bowtie", "threads", "repeats", "fastqFile", "partitions", "submitToGrid", "sgeCorrection", "consensusConcurrency", "cleanup");
-
-my $commandLineOptions = "";
-
-sub getGlobal ($) {
- my $var = shift @_;
- if (!exists($global{$var})) {
- return undef;
- }
- return($global{$var});
-}
-
-sub setGlobal ($$) {
- my $var = shift @_;
- my $val = shift @_;
-
- # If no value, set the field to undefined, the default for many of the options.
-
- $val = undef if ($val eq "");
-
- $global{$var} = $val;
- $val =~ s/\\\"/\"/g;
- $val =~ s/\"/\\\"/g;
- $val =~ s/\\\$/\$/g;
- $val =~ s/\$/\\\$/g;
-
- # some vars should be global like specFile
- if ($var eq "specFile" || $var eq "fastqFile" || $var eq "samFOFN") {
- $val = makeAbsolute($val);
- }
- if ($var eq "utgErrorRate") {
- setGlobal("cgwErrorRate", $val);
- setGlobal("cnsErrorRate", $val);
- setGlobal("ovlErrorRate", $val);
- }
-
- $commandLineOptions .= " \"$var=$val\" ";
-}
-
-sub setDefaults() {
- # grid options, duplicate of runCA
- $global{"gridSubmitCommand"} = "qsub";
- $global{"gridHoldOption"} = "-hold_jid \"WAIT_TAG\""; # for lsf it is -w "done("WAIT_TAG")"
- $global{"gridSyncOption"} = "-sync y"; # for lsf it is -K
- $global{"gridNameOption"} = "-cwd -N"; # for lsf it is -J
- $global{"gridArrayOption"} = "-t ARRAY_JOBS"; # for lsf, empty ("")
- $global{"gridArrayName"} = "ARRAY_NAME"; # for lsf, it is ARRAY_NAME[ARRAY_JOBS]
- $global{"gridOutputOption"} = "-j y -o"; # for lsf, it is -o
- $global{"gridPropagateCommand"} = "qalter -hold_jid \"WAIT_TAG\""; # for lsf it is bmodify -w "done(WAIT_TAG)"
- $global{"gridNameToJobIDCommand"} = undef; # for lsf it is bjobs -J "WAIT_TAG" | grep -v JOBID
- $global{"gridTaskID"} = "SGE_TASK_ID"; # for lsf it is LSB_JOBINDEX
- $global{"gridArraySubmitID"} = "\$TASK_ID"; # for lsf it is %I
-
- $global{"shell"} = "/bin/bash";
-
- $global{"utgErrorRate"} = 0.25;
- $global{"utgErrorLimit"} = 6.5;
- $global{"ovlErrorRate"} = 0.25;
- $global{"cnsErrorRate"} = 0.25;
- $global{"cgwErrorRate"} = 0.25;
-
- $global{"shortReads"} = undef;
- $global{"longReads"} = undef;
- $global{"QV"} = 54.5;
- $global{"libraryname"} = undef;
- $global{"pbcns"} = undef;
- $global{"bankPath"} = undef;
- $global{"specFile"} = undef;
- $global{"length"} = 500;
-
- $global{"coverageCutoff"} = 0;
- $global{"maxCoverage"} = 50;
- $global{"genomeSize"} = 0;
- $global{"maxUncorrectedGap"} = 0;
- $global{"maxCorrectionRounds"} = 0;
- $global{"repeats"} = "";
- $global{"samFOFN"} = undef;
- $global{"blasr"} = undef;
- $global{"bowtie"} = undef;
-
- $global{"threads"} = 1;
- $global{"partitions"} = 1;
- $global{"submitToGrid"} = 0;
- $global{"sge"} = undef;
- $global{"sgeCorrection"} = undef;
- $global{"consensusConcurrency"} = 8;
-
- $global{"doOverlapBasedTrimming"} = 0; # should this be defaulted to true or false?
-
- $global{"cleanup"} = 1;
-
- $global{"fastqFile"} = undef;
-}
-
-sub updateSpecFile($) {
- my $outFile = shift @_;
- open(W, "> $outFile") or die("Couldn't open '$outFile'", undef);
-
- my $print = 1;
- foreach my $key (keys %global) {
- $print = 1;
- foreach my $nonCaOption (@nonCAOptions) {
- if (index($key, $nonCaOption) == 0) {
- $print = 0;
- last;
- }
- }
- if ($print == 1) {
- print W "$key = " . getGlobal($key) . "\n";
- }
- }
- close(W);
-}
-
-sub setParametersFromFile ($@) {
- my $specFile = shift @_;
- my @fragFiles = @_;
-
- # Client should be ensuring that the file exists before calling this function.
- die "specFile '$specFile' not found.\n" if (! -e "$specFile");
-
- print STDERR "\n";
- print STDERR "###\n";
- print STDERR "### Reading options from '$specFile'\n";
- print STDERR "###\n";
- print STDERR "\n";
-
- open(F, "< $specFile") or die("Couldn't open '$specFile'", undef);
-
- while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- next if (m/^\s*\#/);
- next if (m/^\s*$/);
-
- if (-e $_) {
- my $xx = $_;
- $xx = "$ENV{'PWD'}/$xx" if ($xx !~ m!^/!);
- if (-e $xx) {
- push @fragFiles, $xx;
- } else {
- setGlobal("help", getGlobal("help") . "File not found '$_' after appending absolute path.\n");
- }
- } elsif (m/\s*(\w*)\s*=([^#]*)#*.*$/) {
- my ($var, $val) = ($1, $2);
- $var =~ s/^\s+//; $var =~ s/\s+$//;
- $val =~ s/^\s+//; $val =~ s/\s+$//;
- undef $val if ($val eq "undef");
- if ($var eq "maxGap") {
- $var = "maxUncorrectedGap";
- }
- setGlobal($var, $val);
- } else {
- setGlobal("help", getGlobal("help") . "File not found or unknown specFile option line '$_'.\n");
- }
- }
- close(F);
-
- return(@fragFiles);
-}
-
-sub setParametersFromCommandLine(@) {
- my @specOpts = @_;
-
- if (scalar(@specOpts) > 0) {
- print STDERR "\n";
- print STDERR "###\n";
- print STDERR "### Reading options from the command line.\n";
- print STDERR "###\n";
- print STDERR "\n";
- }
-
- foreach my $s (@specOpts) {
- if ($s =~ m/\s*(\w*)\s*=(.*)/) {
- my ($var, $val) = ($1, $2);
- $var =~ s/^\s+//; $var =~ s/\s+$//;
- $val =~ s/^\s+//; $val =~ s/\s+$//;
- setGlobal($var, $val);
- } else {
- setGlobal("help", getGlobal("help") . "Misformed command line option '$s'.\n");
- }
- }
-}
-
-sub makeAbsolute ($) {
- my $val = shift @_;
- if (defined($val) && ($val !~ m!^/!)) {
- $val = "$ENV{'PWD'}/$val";
- }
- return $val;
-}
-
-sub getInstallDirectory () {
- my @t = split '/', "$FindBin::RealBin";
- pop @t; # bin
- pop @t; # arch, e.g., FreeBSD-amd64
- my $installDir = join '/', @t; # path to the assembler
-
- return($installDir);
-}
-
-sub getBinDirectory () {
- my $installDir = getInstallDirectory();
-
- my $syst = `uname -s`; chomp $syst; # OS implementation
- my $arch = `uname -m`; chomp $arch; # Hardware platform
- my $name = `uname -n`; chomp $name; # Name of the system
-
- $arch = "amd64" if ($arch eq "x86_64");
- $arch = "ppc" if ($arch eq "Power Macintosh");
-
- my $path = "$installDir/$syst-$arch/bin";
-
- my $pathMap = getGlobal("pathMap");
- if (defined($pathMap)) {
- open(F, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef);
- while (<F>) {
- my ($n, $b) = split '\s+', $_;
- $path = $b if ($name eq $n);
- }
- close(F);
- }
-
- return($path);
-}
-
-sub getBinDirectoryShellCode () {
- my $installDir = getInstallDirectory();
- my $string;
-
- $string = "\n";
- $string .= "syst=`uname -s`\n";
- $string .= "arch=`uname -m`\n";
- $string .= "name=`uname -n`\n";
- $string .= "\n";
- $string .= "if [ \"\$arch\" = \"x86_64\" ] ; then\n";
- $string .= " arch=\"amd64\"\n";
- $string .= "fi\n";
- $string .= "if [ \"\$arch\" = \"Power Macintosh\" ] ; then\n";
- $string .= " arch=\"ppc\"\n";
- $string .= "fi\n";
- $string .= "\n";
- $string .= "bin=\"$installDir/\$syst-\$arch/bin\"\n";
- $string .= "\n";
-
- my $pathMap = getGlobal("pathMap");
- if (defined($pathMap)) {
- open(PM, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef);
- while (<PM>) {
- my ($n, $b) = split '\s+', $_;
- $string .= "if [ \"\$name\" = \"$n\" ] ; then\n";
- $string .= " bin=\"$b\"\n";
- $string .= "fi\n";
- }
- close(PM);
- $string .= "\n";
- }
-
- return($string);
-}
-
-sub runCommand($$) {
- my $dir = shift;
- my $cmd = shift;
-
- my $t = localtime();
- my $d = time();
- print STDERR "----------------------------------------START $t\n$cmd\n";
-
- my $rc = 0xffff & system("cd $dir && $cmd");
-
- $t = localtime();
- print STDERR "----------------------------------------END $t (", time() - $d, " seconds)\n";
-
- return(0) if ($rc == 0);
-
- die "Failed to execute $cmd\n";
-}
-
-################################################################################
-
-# Functions for running multiple processes at the same time.
-
-my $numberOfProcesses = 0; # Number of jobs concurrently running
-my $numberOfProcessesToWait = 0; # Number of jobs we can leave running at exit
-my @processQueue = ();
-my @processesRunning = ();
-my $printProcessCommand = 1; # Show commands as they run
-
-sub schedulerSetNumberOfProcesses {
- $numberOfProcesses = shift @_;
-}
-
-sub schedulerSubmit {
- chomp @_;
- push @processQueue, @_;
-}
-
-sub schedulerForkProcess {
- my $process = shift @_;
- my $pid;
-
- # From Programming Perl, page 167
- FORK: {
- if ($pid = fork) {
- # Parent
- #
- return($pid);
- } elsif (defined $pid) {
- # Child
- #
- exec($process);
- } elsif ($! =~ /No more processes/) {
- # EAGIN, supposedly a recoverable fork error
- sleep 1;
- redo FORK;
- } else {
- die "Can't fork: $!\n";
- }
- }
-}
-
-sub schedulerReapProcess {
- my $pid = shift @_;
-
- if (waitpid($pid, &WNOHANG) > 0) {
- return(1);
- } else {
- return(0);
- }
-}
-
-sub schedulerRun {
- my @newProcesses;
-
- # Reap any processes that have finished
- #
- undef @newProcesses;
- foreach my $i (@processesRunning) {
- if (schedulerReapProcess($i) == 0) {
- push @newProcesses, $i;
- }
- }
- undef @processesRunning;
- @processesRunning = @newProcesses;
-
- # Run processes in any available slots
- #
- while ((scalar(@processesRunning) < $numberOfProcesses) &&
- (scalar(@processQueue) > 0)) {
- my $process = shift @processQueue;
- print STDERR "$process\n";
- push @processesRunning, schedulerForkProcess($process);
- }
-}
-
-sub schedulerFinish {
- my $child;
- my @newProcesses;
- my $remain;
-
- my $t = localtime();
- my $d = time();
- print STDERR "----------------------------------------START CONCURRENT $t\n";
-
- $remain = scalar(@processQueue);
-
- # Run all submitted jobs
- #
- while ($remain > 0) {
- schedulerRun();
-
- $remain = scalar(@processQueue);
-
- if ($remain > 0) {
- $child = waitpid -1, 0;
-
- undef @newProcesses;
- foreach my $i (@processesRunning) {
- push @newProcesses, $i if ($child != $i);
- }
- undef @processesRunning;
- @processesRunning = @newProcesses;
- }
- }
-
- # Wait for them to finish, if requested
- #
- while (scalar(@processesRunning) > $numberOfProcessesToWait) {
- waitpid(shift @processesRunning, 0);
- }
-
- $t = localtime();
- print STDERR "----------------------------------------END CONCURRENT $t (", time() - $d, " seconds)\n";
-}
-
-################################################################################
-#
-# Functions for running jobs on grid
-
-sub runningOnGrid () {
- my $taskID = getGlobal("gridTaskID");
- return(defined($ENV{$taskID}));
-}
-
-sub findNextScriptOutputFile ($$) {
- my $wrk = shift @_;
- my $prefix = shift @_;
- my $idx = "00";
- while (-e "$wrk/$prefix.$idx.sh") {
- $idx++;
- }
- return("$wrk/$prefix.$idx");
-}
-
-sub buildGridArray($$$) {
- my $name = shift @_;
- my $maxLimit = shift @_;
- my $globalValue = shift @_;
-
- my $arrayJobName = getGlobal($globalValue);
- $arrayJobName =~ s/ARRAY_NAME/$name/g;
- $arrayJobName =~ s/ARRAY_JOBS/1-$maxLimit/g;
-
- return $arrayJobName;
-}
-
-sub getGridArrayName($$) {
- my $name = shift @_;
- my $maxLimit = shift @_;
- return buildGridArray($name, $maxLimit, "gridArrayName");
-}
-
-sub getGridArrayOption($$) {
- my $name = shift @_;
- my $maxLimit = shift @_;
- return buildGridArray($name, $maxLimit, "gridArrayOption");
-}
-
-sub writeScriptHeader($$$$$) {
- my $wrk = shift @_;
- my $asm = shift @_;
- my $script = shift @_;
- my $executable = shift @_;
- my $options = shift @_;
-
- open(F, "> $script") or caFailure("failed to open '$script' for writing", undef);
- print F "#!" . getGlobal("shell") . "\n";
- print F "#\n";
- print F "if [ \"x\$SGE_ROOT\" != \"x\" ]; then \n";
- print F " # Attempt to (re)configure SGE. For reasons Bri doesn't know,\n";
- print F " # jobs submitted to SGE, and running under SGE, fail to read his\n";
- print F " # .tcshrc (or .bashrc, limited testing), and so they don't setup\n";
- print F " # SGE (or ANY other paths, etc) properly. For the record,\n";
- print F " # interactive SGE logins (qlogin, etc) DO set the environment.\n";
- print F " \n";
- print F " . \$SGE_ROOT/\$SGE_CELL/common/settings.sh\n";
- print F "fi\n";
- print F "\n";
- print F "# On the off chance that there is a pathMap, and the host we\n";
- print F "# eventually get scheduled on doesn't see other hosts, we decide\n";
- print F "# at run time where the binary is.\n";
-
- print F getBinDirectoryShellCode();
-
- print F "/usr/bin/env perl \$bin/$executable $options\n";
- close(F);
-
- system("chmod +x $script");
-}
-
-sub submit($$$$$) {
- my $wrk = shift @_;
- my $script = shift @_;
- my $output = shift @_;
- my $prefix = shift @_;
- my $waitTag = shift @_;
-
- my $sge = getGlobal("sge");
- my $sgeScript = getGlobal("sgeScript");
- my $sgePropHold = getGlobal("sgePropagateHold");
-
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $holdOption = getGlobal("gridHoldOption");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
- my $holdPropagateCommand = getGlobal("gridPropagateCommand");
-
- my $jobName = $prefix;
- if (defined($waitTag)) {
- my $hold = $holdOption;
- $hold =~ s/WAIT_TAG/$waitTag/g;
- $waitTag = $hold;
- }
-
- my $qcmd = "$submitCommand $sge $sgeScript $nameOption \"$jobName\" $waitTag $outputOption $output $script";
- runCommand($wrk, $qcmd) and caFailure("Failed to submit script.\n");
-
- if (defined($sgePropHold)) {
- if (defined($holdPropagateCommand)) {
- my $translateCmd = getGlobal("gridNameToJobIDCommand");
-
- # translate hold option to job id if necessar
- if (defined($translateCmd) && $translateCmd ne "") {
- my $tcmd = $translateCmd;
- $tcmd =~ s/WAIT_TAG/$sgePropHold/g;
- my $propJobCount = `$tcmd |wc -l`;
- chomp $propJobCount;
- if ($propJobCount != 1) {
- print STDERR "Warning: multiple IDs for job $sgePropHold got $propJobCount and should have been 1.\n";
- }
- #my $jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
- #chomp $jobID;
- #print STDERR "Translated job ID $sgePropHold to be job $jobID\n";
- #$sgePropHold = $jobID;
- open(PROPS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that need to hold", undef);
-
- # now we can get the job we are holding for
- $tcmd = $translateCmd;
- $tcmd =~ s/WAIT_TAG/$jobName/g;
- my $holdJobCount = `$tcmd |wc -l`;
- chomp $propJobCount;
- if ($propJobCount != 1) {
- print STDERR "Warning: multiple IDs for job $jobName got $propJobCount and should have been 1.\n";
- }
- #$jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
- #chomp $jobID;
- #print STDERR "Translated job ID $sgePropHold to be job $jobID\n";
- #$jobName = $jobID;
- open(HOLDS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that should be held for", undef);
-
- # loop over all jobs and all sge hold commands to modify the jobs. We have no way to know which is the right one unfortunately
- while (my $prop = <PROPS>) {
- while (my $hold = <HOLDS>) {
- my $hcmd = $holdPropagateCommand;
- $hcmd =~ s/WAIT_TAG/$hold/g;
- my $acmd = "$hcmd $prop";
- print STDERR "Propagating hold to $prop to wait for job $hold\n";
- system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$prop'.\n";
- }
- }
- close(HOLDS);
- close(PROPS);
- } else {
- $sgePropHold = "\"$sgePropHold\"";
- $holdPropagateCommand =~ s/WAIT_TAG/$jobName/g;
- my $acmd = "$holdPropagateCommand $sgePropHold";
- system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$sgePropHold'.\n";
- }
- } else {
- print STDERR "WARNING: Failed to reset hold '$sgePropHold', not supported on current grid environment.\n";
- }
- }
-}
-
-sub submitScript ($$$) {
- my $wrk = shift @_;
- my $asm = shift @_;
- my $waitTag = shift @_;
- my $libraryname = "temp" . getGlobal("libraryname");
-
- $wrk =~ s/$libraryname//g;
- my $sgeName = getGlobal("sgeName");
- $sgeName = "_$sgeName" if (defined($sgeName));
-
- return if (getGlobal("scriptOnGrid") == 0);
- my $output = findNextScriptOutputFile("$wrk/$libraryname", "runPBcR.sge.out");
- my $script = "$output.sh";
- writeScriptHeader($wrk, $asm, $script, "pacBioToCA", $commandLineOptions);
- submit($wrk, $script, $output, "pBcR_$asm$sgeName", $waitTag);
-
- exit(0);
-}
-
-sub submitBatchJobs($$$$) {
- my $wrk = shift @_;
- my $asm = shift @_;
- my $SGE = shift @_;
- my $TAG = shift @_;
-
- if (getGlobal("scriptOnGrid")) {
- runCommand($wrk, $SGE) and die("Failed to submit batch jobs.");
- submitScript($wrk, $asm, $TAG);
- } else {
- print "Please execute:\n$SGE\n";
- }
-}
-
-sub submitRunCA($$$$) {
- my $wrk = shift @_;
- my $asm = shift @_;
- my $options = shift @_;
- my $TAG = shift @_;
- my $holdPropagateCommand = getGlobal("gridPropagateCommand");
- my $holdOption = getGlobal("gridHoldOption");
- my $syncOption = getGlobal("gridSyncOption");
- my $sge = getGlobal("sge");
-
- if (defined($holdPropagateCommand) && $holdPropagateCommand ne "") {
- # do nothing, we can use the hold option to propagate
- $syncOption = "";
- } else {
- print "Warning: grid environment does not support hold propagate, running in sync mode\n";
- }
-
- if (getGlobal("scriptOnGrid")) {
- my $output = findNextScriptOutputFile($wrk, "runPBcR.runCA.sge.out");
- my $script = "$output.sh";
- writeScriptHeader($wrk, $asm, $script, "runCA", $options);
- submit($wrk, "$syncOption $script", $output, $TAG, undef);
- submitScript($wrk, $asm, $TAG);
- } else {
- print "Please execute:\nrunCA $options\n";
- }
-}
-
-sub submitRunPBcR($$$$) {
- my $wrk = shift @_;
- my $asm = shift @_;
- my $options = shift @_;
- my $TAG = shift @_;
- my $holdPropagateCommand = getGlobal("gridPropagateCommand");
- my $holdOption = getGlobal("gridHoldOption");
- my $syncOption = getGlobal("gridSyncOption");
- my $sge = getGlobal("sge");
-
- if (defined($holdPropagateCommand) && $holdPropagateCommand ne "") {
- # do nothing, we can use the hold option to propagate
- $syncOption = "";
- } else {
- print "Warning: grid environment does not support hold propagate, running in sync mode\n";
- }
-
- if (getGlobal("scriptOnGrid")) {
- my $output = findNextScriptOutputFile($wrk, "runPBcR.sge.out");
- my $script = "$output.sh";
- writeScriptHeader($wrk, $asm, $script, "pacBioToCA", $options);
- submit($wrk, "$syncOption $script", $output, $TAG, undef);
- submitScript($wrk, $asm, $TAG);
- } else {
- print "Please execute:\npacBioToCA $options\n";
- }
-}
-
-################################################################################
-
-my $HISTOGRAM_CUTOFF = 0.985; #seems like updated blasr parameters map more
-
-sub pickMappingThreshold($$) {
- my $histogram = shift @_;
- my $conservative = shift @_;
- my $threshold = 0;
-
- # pick threshold as 2sd (95% of the bases)
- my @hist;
- my $total = 0;
- my $sum = 0;
- my $mean = 0;
- my $variance = 0;
-
- open(F, "< $histogram") or die("Couldn't open '$histogram'", undef);
- while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- next if (m/^\s*\#/);
- next if (m/^\s*$/);
-
- $hist[$_]++;
- $total++;
- my $delta = $_ - $mean;
- $mean += $delta / $total;
- $variance += $delta * ($_ - $mean);
- }
- $variance /= $total;
- my $sd = sqrt($variance);
- close(F);
-
- for (my $i = 0; $i <= $#hist; $i++) {
- $sum += $hist[$i];
- if ($sum / $total > $HISTOGRAM_CUTOFF) {
- $threshold = $i - 1;
- last;
- }
- }
- $threshold = ($threshold < ($mean + 3*$sd) ? floor($mean + 3*$sd) : $threshold);
- if (defined($conservative) && $conservative == 1) {
- $threshold = ceil($mean * 2);
- }
- print STDERR "Picked mapping cutoff of $threshold\n";
-
- return $threshold;
-}
-
-my $MIN_FILES_WITHOUT_PARTITIONS = 20;
-my $REPEAT_MULTIPLIER = 10;
-my $MAX_CORRECTION_COVERAGE = 100;
-my $MIN_SELF_CORRECTION = 50;
-my $MAX_SPLIT_PERCENTAGE = 0.30;
-
-setDefaults();
-
-my @fragFiles;
-my @specOpts;
-my @cmdArgs;
-
-my $srcstr;
-
-{
- local $, = " ";
- $srcstr = "$0 @ARGV";
-}
-
-# process the spec file and the fragment files
-while (scalar(@ARGV) > 0) {
- my $arg = shift @ARGV;
- if ($arg eq "-s") {
- setGlobal("specFile", shift @ARGV);
-
- } elsif (($arg =~ /\.frg$|frg\.gz$|frg\.bz2$/i)) {
- if (-e $arg) {
- push @fragFiles, makeAbsolute($arg);
- } else {
- print "Invalid file $arg could not be found.\n";
- }
-
- } elsif ($arg =~ m/=/) {
- push @specOpts, $arg;
-
- } else {
- push @cmdArgs, $arg;
- }
-}
-
-# initialize parameters from the spec files
-my $err = 0;
-if (-e getGlobal("specFile")) {
- @fragFiles = setParametersFromFile(getGlobal("specFile"), @fragFiles);
-} elsif (defined(getGlobal("specFile"))) {
- print STDERR "Error: spec file " . getGlobal("specFile") . " does not exist. Double-check your paths and try again.\n";
- $err++;
-}
-setParametersFromCommandLine(@specOpts);
-
-# finally, command-line parameters take precedence
-while (scalar(@cmdArgs) > 0) {
- my $arg = shift @cmdArgs;
-
- if ($arg eq "-shortReads") {
- setGlobal("shortReads", 1);
-
- } elsif ($arg eq "-longReads") {
- setGlobal("longReads", 1);
-
- } elsif ($arg eq "-pbCNS") {
- setGlobal("pbcns", 1);
-
- } elsif ($arg eq "-coverageCutoff") {
- setGlobal("coverageCutoff", shift @cmdArgs);
- if (getGlobal("coverageCutoff") < 0) { setGlobal("coverageCutoff", 0); }
-
- } elsif ($arg eq "-maxCoverage") {
- setGlobal("maxCoverage", shift @cmdArgs);
- if (getGlobal("maxCoverage") < 0) { setGlobal("maxCoverage", 0); }
-
- } elsif ($arg eq "-genomeSize") {
- setGlobal("genomeSize", shift @cmdArgs);
- if (getGlobal("genomeSize") < 0) { setGlobal("genomeSize", 0); }
-
- } elsif ($arg eq "-maxGap") {
- setGlobal("maxUncorrectedGap", shift @cmdArgs);
- if (getGlobal("maxUncorrectedGap") < 0) { setGlobal("maxUncorrectedGap", 0); }
-
- } elsif ($arg eq "-length") {
- setGlobal("length", shift @cmdArgs);
-
- } elsif ($arg eq "-repeats") {
- setGlobal("repeats", shift @cmdArgs);
-
- } elsif ($arg eq "-fastq") {
- setGlobal("fastqFile", shift @cmdArgs);
-
- } elsif ($arg eq "-t" || $arg eq "-threads") {
- setGlobal("threads", shift @cmdArgs);
- if (getGlobal("threads") <= 0) { setGlobal("threads", 1); }
-
- } elsif ($arg eq "-l" || $arg eq "-libraryname") {
- setGlobal("libraryname", shift @cmdArgs);
-
- } elsif ($arg eq "-partitions") {
- setGlobal("partitions", shift @cmdArgs);
-
- } elsif ($arg eq "-sge") {
- setGlobal("sge", shift @cmdArgs);
-
- } elsif ($arg eq "-sgeCorrection") {
- setGlobal("sgeCorrection", shift @cmdArgs);
-
- } elsif ($arg eq "-noclean") {
- setGlobal("cleanup", 0);
-
- } else {
- print STDERR "Unknown parameter " + $err + "\n";
- $err++;
- }
- }
-
-if ((scalar(@fragFiles) == 0) && (!defined(getGlobal("longReads")) || getGlobal("longReads") == 0)) {
- print STDERR "Warning: no frag files specified, assuming self-correction of pacbio sequences.\n";
- setGlobal("longReads", 1);
-}
-
-if (($err) || (!defined(getGlobal("fastqFile"))) || (!defined(getGlobal("specFile"))) || (!defined(getGlobal("libraryname")))) {
- print STDERR "usage: $0 [options] -libraryname <name> -s <specfile> -fastq <fastqfile> [optional frg files]\n";
- print STDERR " -length <int> Minimum length of PacBio sequences to correct/output.\n";
- print STDERR " -partitions <int> Number of partitions for consensus\n";
- print STDERR " -sgeCorrection <sge options> Parameters for the correction step for the grid. This should match the threads specified below, for example by using -pe threaded\n";
- print STDERR " -libraryname <string> Name of the library; freeformat text. Must be unique from any library names in the FRG files used for correction\n";
- print STDERR " -threads <int> Number of threads to use for correction.\n";
- print STDERR " -shortReads Use if the sequences for correction are 100bp or shorter.\n";
-
- print STDERR " -genomeSize <int> Specify the approximate genome size. This will be used to compute the maximum number of bases to correct and the number of mappings each sequence can have.\n";
- print STDERR " -maxCoverage <int> Maximum coverage of PacBio sequences to correct. Only the longest sequences adding up to this coverage will be corrected. Requires genomeSize or coverage parameter to be specified\n";
- print STDERR " -maxGap <int> The maximum uncorrected PacBio gap that will be allowed. When there is no short-read coverage for a region, by default the pipeline will split a PacBio sequence. This option will attempt to use other PacBio sequences to patch the gap and avoid splitting the read. Sequences where the gaps have no support will still be broken. For example, specifying 50, will mean any gap 50bp or smaller can have no short-read coverage (but has other PacBio [...]
-
- print STDERR "\nAdvanced options (EXPERT):\n";
- print STDERR " -pbCNS Use PBDAGCON for consensus instead of AMOS. Requires SMRTportal 2.1+.\n";
- print STDERR " -coverageCutoff Specify the pacBio coverage (integer) used to separate repeat copies instead of automatically estimating.\n";
- print STDERR " bankPath=<dir> Use a specified location for AMOS make-consensus instead of working directory. Specifying SHARED will use shared-memory (/dev/shm) if available. Using this option requires each concurrently running correction job to have a unique library name to avoid colissions!\n";
- print STDERR " \"blasr=<string>\" Use blasr for overlap computation instead of CA's built-in overlapper. This parameter specifies the blasr parameters to use.\n";
- print STDERR " \"bowtie=<string>\" Use bowtie2 for overlap computation instead of CA's built-in overlapper. This parameter specifies the bowtie 2 parameters to use.\n";
- print STDERR " samFOFN=<file> Skip overlap computation. Use the provided file of file names of SAM files as the overlaps instead. Any valid sam files should be accepted.\n";
-
- print STDERR "\nComplete documentation at http://wgs-assembler.sourceforge.net/\n\n";
-
- print STDERR "No fastq file specified. Please specify a fastq file containing PacBio sequences for correction using the -fastq parameter.\n" if (!defined(getGlobal("fastqFile")));
- print STDERR "No spec file defined. Please specify a spec file using the -s option.\n" if (!defined(getGlobal("specFile")));
- print STDERR "No library name provided. Please specify a name using the -library option.\n" if (!defined(getGlobal("libraryname")));
- exit(1);
-}
-
-# get grid options
-my $submitCommand = getGlobal("gridSubmitCommand");
-my $nameOption = getGlobal("gridNameOption");
-my $outputOption = getGlobal("gridOutputOption");
-
-#check for valid parameters for requested partitions and threads
-my $limit = 1024;
-my $FILES_PER_PARTITION = (getGlobal("maxUncorrectedGap") > 0 ? 3 : 1);
-my $FILES_PER_THREAD = 3;
-
-my $sysLimit = `ulimit -Sn`;
-chomp($sysLimit);
-if (defined($sysLimit)) {
- $limit = $sysLimit;
-}
-if ($limit - $MIN_FILES_WITHOUT_PARTITIONS <= $FILES_PER_PARTITION*getGlobal("partitions") || $limit - $MIN_FILES_WITHOUT_PARTITIONS <= getGlobal("threads") * $FILES_PER_THREAD) {
- my $maxPartitions = floor(($limit - $MIN_FILES_WITHOUT_PARTITIONS) / $FILES_PER_PARTITION);
- my $maxThreads = (floor($limit - $MIN_FILES_WITHOUT_PARTITIONS) / $FILES_PER_THREAD);
- setGlobal("partitions", ($maxPartitions < getGlobal("partitions") ? $maxPartitions : getGlobal("partitions")));
- setGlobal("threads", ($maxThreads < getGlobal("threads") ? $maxThreads : getGlobal("threads")));
- print STDERR "Warning: file handle limit of $limit prevents using requested partitions. Reset partitions to " . getGlobal("partitions") . " and " . getGlobal("threads") . " threads. If you want more partitions, reset the limit using ulimit -Sn and try again.\n";
-}
-if (getGlobal("threads") > getGlobal("partitions")) {
- setGlobal("threads", getGlobal("partitions") - 1);
- print STDERR "Warning: number of partitions should be > # threads. Adjusted threads to be ". getGlobal("threads") . "\n";
-}
-
-print STDERR "Running with " . getGlobal("threads") . " threads and " . getGlobal("partitions") . " partitions\n";
-
-my $CA = getBinDirectory();
-my $AMOS = "$CA/../../../AMOS/bin/";
-my $BLASR = "$CA/../../../smrtanalysis/current/analysis/bin/";
-my $BOWTIE = "$CA/../../../bowtie2/";
-my $wrk = makeAbsolute("");
-my $asm = "asm";
-my $scriptParams = getGlobal("sgeScript");
-
-if (defined($scriptParams)) {
- if (!defined(getGlobal("sgeCorrection"))) {
- setGlobal("sgeCorrection", $scriptParams);
- }
-}
-
-# check for previously existing file and do not overwrite
-if ( -e "$wrk/" . getGlobal("libraryname") . ".frg") {
- print STDERR "Error: requested to output " . getGlobal("libraryname") . ".frg but file already exists. Will not overwrite. Please remove the file and try again.\n";
- exit(1);
-}
-
-my $useGrid = getGlobal("useGrid");
-if (defined($useGrid)) {
- setGlobal("submitToGrid", $useGrid);
-}
-elsif (defined(getGlobal("sge"))) {
- setGlobal("submitToGrid", 1);
-}
-
-my $caCNS = getGlobal("cnsConcurrency");
-if (defined($caCNS)) {
- setGlobal("consensusConcurrency", $caCNS);
-}
-
-if (! -e "$AMOS/bank-transact") {
- # try to use path
- my $amosPath = `which bank-transact`;
- chomp $amosPath;
- my @t = split '/', "$amosPath";
- pop @t; # bank-transact
- $AMOS = join '/', @t; # path to the assembler
-
- # if we really can't find it just give up
- if (! -e "$AMOS/bank-transact") {
- die "AMOS binaries: bank-transact not found in $ENV{PATH}. Please download it from http://amos.sf.net and add it to your path.\n";
- }
-}
-
-if (! -e "$BLASR/blasr") {
- # try to use path
- my $amosPath = `which blasr`;
- chomp $amosPath;
- my @t = split '/', "$amosPath";
- pop @t; # blasr
- $BLASR = join '/', @t; # path to the assembler
-
- # if we really can't find it just give up
- if (! -e "$BLASR/blasr") {
- die "BLASR binaries: blasr not found in $ENV{PATH}. Please download it from http://pacificbiosciences.github.com/DevNet/ and add it to your path.\n" if (defined(getGlobal("blasr")) || (defined(getGlobal("longReads")) && getGlobal("longReads") == 1));
- }
-}
-
-my $blasrVersion = 1.2;
-if (-e "$BLASR/blasr") {
- $blasrVersion = `$BLASR/blasr -version |tail -n 1`;
- chomp $blasrVersion;
-}
-
-if (! -e "$BOWTIE/bowtie2-build") {
- # try to use path
- my $amosPath = `which bowtie2-build`;
- chomp $amosPath;
- my @t = split '/', "$amosPath";
- pop @t; # bowtie2-build
- $BOWTIE = join '/', @t; # path to the assembler
-
- # if we really can't find it just give up
- if (! -e "$BOWTIE/bowtie2-build") {
- die "Bowtie2 binaries: bowtie not found in $BOWTIE\n" if defined(getGlobal("bowtie"));
- }
-}
-
-print STDERR "********* Starting correction...\n CA: $CA\nAMOS:$AMOS\nSMRTportal:$BLASR ($blasrVersion)\nBowtie:$BOWTIE\n";
-print STDERR "******** Configuration Summary ********\n";
-foreach my $key (keys %global) {
- if (length($key) < $TAB_SIZE) {
- print STDERR "$key\t\t\t=";
- } elsif (length($key) < 2*$TAB_SIZE) {
- print STDERR "$key\t\t="
- } else {
- print STDERR "$key\t=";
- }
- print STDERR "\t" . $global{$key} . "\n";
-}
-print STDERR "****************\n";
-
-# get global variable values
-my $specFile = getGlobal("specFile");
-my $libraryname = getGlobal("libraryname");
-my $fastqFile = getGlobal("fastqFile");
-
-my $ovlErrorRate = getGlobal("utgErrorRate");
-my $ovlErrorLimit = getGlobal("utgErrorLimit");
-
-my $maxUncorrectedGap = getGlobal("maxUncorrectedGap");
-my $coverage = getGlobal("coverageCutoff");
-my $genomeSize = getGlobal("genomeSize");
-my $shortReads = getGlobal("shortReads");
-my $longReads = getGlobal("longReads");
-my $length = getGlobal("length");
-my $repeats = getGlobal("repeats");
-my $QV = getGlobal("QV");
-
-my $threads = getGlobal("threads");
-my $partitions = getGlobal("partitions");
-
-my $submitToGrid = getGlobal("submitToGrid");
-my $sge = getGlobal("sge");
-my $sgeOvl = getGlobal("sgeOverlap");
-my $sgeCorrection = getGlobal("sgeCorrection");
-my $sgeConsensus = getGlobal("sgeConsensus");
-my $sgeTaskID = getGlobal("gridTaskID");
-
-my $pbcns = getGlobal("pbcns");
-my $bankPath = getGlobal("bankPath");
-
-my $consensusConcurrency = getGlobal("consensusConcurrency");
-
-my $cleanup = getGlobal("cleanup");
-
-my $cmd = "";
-
-if (! -e "$wrk/temp$libraryname") {
- runCommand("$wrk", "mkdir temp$libraryname");
- # generate the ca spec file, since we support additional options in the spec file, we need to strip those out before passing it to ca
- updateSpecFile("$wrk/temp$libraryname/$libraryname.spec");
-}
-$specFile = "$wrk/temp$libraryname/$libraryname.spec";
-
-if (! -e "$wrk/temp$libraryname/$libraryname.frg") {
- runCommand($wrk, "$CA/fastqToCA -libraryname $libraryname -type sanger -technology none -feature doConsensusCorrection 1 -reads " . makeAbsolute($fastqFile) . " > $wrk/temp$libraryname/$libraryname.frg");
-}
-
-# now that were ready, add the frg file info to the command line args
-my $commandLineOptionsNoFrgs = "-s $specFile $commandLineOptions";
-$commandLineOptions = "-s $specFile $commandLineOptions @fragFiles";
-
-# and we're off
-submitScript($wrk, $asm, undef) if (!runningOnGrid());
-my $sgeName = (defined(getGlobal("sgeName")) ? "_" . getGlobal("sgeName") : "");
-if (! -d "$wrk/temp$libraryname/$asm.gkpStore") {
- $cmd = "$CA/runCA ";
- $cmd .= " -s $specFile -p $asm -d temp$libraryname ";
- $cmd .= " stopAfter=initialStoreBuilding ";
- $cmd .= " sgeName=\"" . getGlobal("sgeName") . "\" " if defined(getGlobal("sgeName"));
- $cmd .= " @fragFiles $wrk/temp$libraryname/$libraryname.frg";
- runCommand($wrk, $cmd);
-}
-
-# make assumption that we correct using all libraries preceeding pacbio
-# figure out what number of libs we have and what lib is pacbio
-my $numLib = `$CA/gatekeeper -dumpinfo temp$libraryname/$asm.gkpStore | grep LIB |awk '{print \$1}'`;
-chomp($numLib);
-
-my $minCorrectLib = 0;
-my $maxCorrectLib = 0;
-my $libToCorrect = 0;
-for (my $i = 1; $i <= $numLib; $i++) {
- if (system("$CA/gatekeeper -isfeatureset $i doConsensusCorrection temp$libraryname/$asm.gkpStore") == 0) {
- if ($libToCorrect != 0) {
- die("Error: only one PacBio library can be corrected. Both libraries $libToCorrect and $i are set to be corrected. Please double-check your input files and try again", undef);
- }
- $libToCorrect = $i;
- } else {
- if ($minCorrectLib == 0) { $minCorrectLib = $i; }
- $maxCorrectLib = $i;
- }
-}
-
-# check that we were able to find the libraries for correction as expected
-if ($libToCorrect <= $minCorrectLib) {
- die("Error: The PacBio library $libToCorrect must be the last library loaded but it preceedes $minCorrectLib. Please double-check your input files and try again.", undef);
-}
-if (defined(getGlobal("longReads")) && getGlobal("longReads") == 1) {
- $maxCorrectLib = $libToCorrect;
- $minCorrectLib = $libToCorrect if ($minCorrectLib == 0);
- $ovlErrorRate = 0.35;
- $ovlErrorLimit = 6.5;
-} elsif (($libToCorrect == 0 || ($minCorrectLib == 0 && $maxCorrectLib == 0))) {
- die ("Error: unable to find a library to correct. Please double-check your input files and try again.", undef);
-}
-print STDERR "Will be correcting PacBio library $libToCorrect with librarie[s] $minCorrectLib - $maxCorrectLib\n";
-my $totalBP = 0;
-my $totalNumToCorrect = 0;
-my $minCorrectID = 0;
-my $maxCorrectID = 0;
-# compute the number of bases in the gateeeker to be corrected
-open(F, "$CA/gatekeeper -dumpinfo temp$libraryname/$asm.gkpStore |") or die("Couldn't open gatekeeper store", undef);
-
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- my @array = split '\s+';
- if ($#array == 8 && $array[0] == $libToCorrect) {
- $totalBP = $array[6];
- $minCorrectID = $array[1];
- $maxCorrectID = $array[2];
- }
-}
-close(F);
-
-my $totalInputBP = 0;
-open(F, "$CA/gatekeeper -dumpinfo temp$libraryname/$asm.gkpStore |") or die("Couldn't open gatekeeper store", undef);
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- my @array = split '\s+';
- if ($#array == 8) {
- if ($array[0] == $libToCorrect) {
- $totalInputBP = $array[6];
- }
- }
-}
-close(F);
-
-if (! -e "$wrk/temp$libraryname/$asm.toerase.out") {
- # here is where we filter for specified length as well as max of longest X of coverage for correction
- # use the genome size/coverage, if available to subset the sequences
- if ($genomeSize != 0 && getGlobal("maxCoverage") != 0) {
- $totalBP = $genomeSize * getGlobal("maxCoverage");
- }
- runCommand($wrk, "$CA/gatekeeper -dumpfragments -invert -tabular -longestovermin $libToCorrect $length -longestlength $libToCorrect $totalBP temp$libraryname/$asm.gkpStore |awk '{if (!(match(\$1, \"UID\") != 0 && length(\$1) == " . length("UID") . ")) { print \"frg uid \"\$1\" isdeleted 1\"; } }' > $wrk/temp$libraryname/$asm.toerase.uid");
- runCommand($wrk, "$CA/gatekeeper --edit $wrk/temp$libraryname/$asm.toerase.uid $wrk/temp$libraryname/$asm.gkpStore > $wrk/temp$libraryname/$asm.toerase.out 2> $wrk/temp$libraryname/$asm.toerase.err");
-}
-
-# compute the number of bases left after our filtering gateeeker to be corrected
-my $totalCorrectingWith = 0;
-open(F, "$CA/gatekeeper -dumpinfo temp$libraryname/$asm.gkpStore |") or die("Couldn't open gatekeeper store", undef);
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- my @array = split '\s+';
- if ($#array == 8) {
- if ($array[0] == $libToCorrect) {
- $totalBP = $array[6];
- $totalNumToCorrect = $array[3] + $array[4];
- }
- if ($minCorrectLib <= $array[0] && $array[0] <= $maxCorrectLib) {
- $totalCorrectingWith += $array[6];
- }
- }
-}
-close(F);
-
-if (defined(getGlobal("longReads")) && getGlobal("longReads") != 0) {
- $totalCorrectingWith = $totalInputBP;
-}
-
-# check that we have good data
-if ($genomeSize != 0) {
- print STDERR "Running with " . ($totalBP / $genomeSize) . "X (for genome size $genomeSize) of $libraryname sequences ($totalBP bp).\n";
- print STDERR "Correcting with " . floor($totalCorrectingWith / $genomeSize) . "X sequences ($totalCorrectingWith bp).\n";
-} else {
- print STDERR "Running with $totalBP bp for $libraryname.\n";
- print STDERR "Correcting with $totalCorrectingWith bp.\n";
-}
-
-if ($totalBP == 0) {
- print STDERR "Error: All $libraryname sequences were eliminated. Please check the length threshold of $length and your input file $fastqFile.\n";
- runCommand("$wrk", "rm -rf temp$libraryname");
- die;
-}
-if ($totalCorrectingWith == 0) {
- print STDERR "Error: No high-accuracy sequences for correction. Please check your input FRG files " . join(", ", @fragFiles) . "\n";
- runCommand("$wrk", "rm -rf temp$libraryname");
- die;
-}
-if ($genomeSize != 0 && floor($totalCorrectingWith / $genomeSize) > $MAX_CORRECTION_COVERAGE && !(defined(getGlobal("longReads")) || getGlobal("longReads") == 0)) {
- print STDERR "Warning: input a total of " . floor($totalCorrectingWith / $genomeSize) . " of high-accuracy coverage for correction. For best performance, at most $MAX_CORRECTION_COVERAGE is recommended.\n";
- # could randomly subsample here
-}
-if ($genomeSize != 0 && defined(getGlobal("longReads")) && getGlobal("longReads") != 0 && ($totalCorrectingWith / $genomeSize) < $MIN_SELF_CORRECTION) {
- print STDERR "Warning: performing self-correction with a total of " .floor($totalCorrectingWith / $genomeSize) . ". For best performance, at least $MIN_SELF_CORRECTION is recommended.\n";
-}
-
-my $cutoffSpecified = 0;
-
-if (defined(getGlobal("blasr"))) {
- if (getGlobal("blasr") =~ m/bestn/) {
- $cutoffSpecified = 1;
- print STDERR "Warning: cutoff manually specified, skip meryl run\n";
- }
-}
-
-if (!defined(getGlobal("bowtie")) && $cutoffSpecified == 0) {
- # run correction up thorough meryl
- if (! -e "$wrk/temp$libraryname/0-mercounts/$asm.nmers.ovl.fasta") {
- $cmd = "$CA/runCA ";
- $cmd .= "-s $specFile ";
- $cmd .= "-p $asm -d temp$libraryname ";
- $cmd .= "ovlHashLibrary=$libToCorrect ";
- $cmd .= "ovlRefLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "ovlCheckLibrary=1 ";
- $cmd .= "obtHashLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "obtRefLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "obtCheckLibrary=0 ";
- $cmd .= "sgeName=\"" . getGlobal("sgeName") . "\" " if defined(getGlobal("sgeName"));
- $cmd .= "doOverlapBasedTrimming=0 stopAfter=meryl";
- runCommand($wrk, $cmd);
- }
-
- # set the meryl threshold based on the genome size and coverage (if specified)
- if (!defined(getGlobal("ovlMerThreshold")) || getGlobal("ovlMerThreshold") == "auto") {
- # no threshold specified, check if the chosen one is OK
- my $autoSetThreshold = `cat temp$libraryname/0-mercounts/*estMerThresh.out`;
- chomp($autoSetThreshold);
- setGlobal("ovlMerThreshold", $autoSetThreshold);
-
- my $corrCov = (defined($genomeSize) && $genomeSize != 0 ? floor($totalCorrectingWith / $genomeSize) : 0);
- my $pacCov = (defined($genomeSize) && $genomeSize != 0 ? floor($totalBP / $genomeSize) : 0);
- my $maxCov = $pacCov + (defined($longReads) && $longReads == 1 ? 0 : $corrCov);
-
- if ($autoSetThreshold < ($maxCov * $REPEAT_MULTIPLIER)) {
- setGlobal("ovlMerThreshold", $maxCov * $REPEAT_MULTIPLIER);
- unlink("$wrk/temp$libraryname/0-mercounts/$asm.nmers.ovl.fasta");
- print STDERR "Resetting from auto threshold of $autoSetThreshold to be " . ($maxCov * 10) . "\n";
- }
- }
-}
-
-if (! -d "$wrk/temp$libraryname/$asm.ovlStore") {
- # run trimming if needed first
- if (getGlobal("doOverlapBasedTrimming") != 0 && ! -e "$wrk/temp$libraryname/0-overlaptrim/overlaptrim.success") {
- if (-e "$wrk/temp$libraryname/0-overlaptrim-overlap/overlap.sh") {
- die("Overlap trimming failed. Remove the temporary directory temp$libraryname and try again.", undef);
- }
- $cmd = "-s $specFile ";
- $cmd .= "-p $asm -d . ";
- $cmd .= "ovlHashLibrary=$libToCorrect ";
- $cmd .= "ovlRefLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "ovlCheckLibrary=1 ";
- $cmd .= "obtHashLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "obtRefLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "obtCheckLibrary=0 ";
- $cmd .= "sgeName=\"" . getGlobal("sgeName") . "\" " if defined(getGlobal("sgeName"));
- $cmd .= "sgePropagateHold=\"pBcR_$asm$sgeName\" ";
- $cmd .= "stopAfter=overlapBasedTrimming";
- if ($submitToGrid == 1) {
- submitRunCA("$wrk/temp$libraryname", $asm, $cmd, "runCA_obt_$asm$sgeName");
- } else {
- runCommand("$wrk/temp$libraryname", "$CA/runCA $cmd");
- }
- }
-
- # now run the correction
- my $ovlThreshold = getGlobal("ovlMerThreshold");
-
- if (defined($longReads) && $longReads == 1) {
- print STDERR "Warning: Blasr is required to align long reads to long reads. Switching blasr ON.\n";
- setGlobal("blasr", defined(getGlobal("blasr")) ? getGlobal("blasr") . " -maxLCPLength 16" : "-minReadLength 200 -maxScore -1000 -maxLCPLength 16");
- }
-
- # when we were asked to not use CA's overlapper, first perform common options
- if (defined(getGlobal("samFOFN")) || defined(getGlobal("blasr")) || defined(getGlobal("bowtie"))) {
- # check for sam tools
- if (!-e "$CA/convertSamToCA") {
- die("Error: request to use Blasr, Bowtie, or SAM files requires CA to be built with SAMTOOLS. Please rebuild CA and try again");
- }
- # create directories
- runCommand($wrk, "mkdir $wrk/temp$libraryname/1-overlapper") if (! -d "$wrk/temp$libraryname/1-overlapper");
-
- # first partition the data so we know how many jobs we have
- my $cmd;
-
- my $ovlHashBlockSize = $totalNumToCorrect;
- my $ovlHashBlockLength = undef;
-
- if (defined(getGlobal("blasr"))) {
- $ovlHashBlockLength = getGlobal("ovlHashBlockLength");
- $ovlHashBlockSize = getGlobal("ovlHashBlockSize");
- }
- my $ovlRefBlockSize = getGlobal("ovlRefBlockSize");
- my $ovlRefBlockLength = getGlobal("ovlRefBlockLength");
-
- if (($ovlRefBlockSize > 0) && ($ovlRefBlockLength > 0)) {
- die("can't set both ovlRefBlockSize and ovlRefBlockLength", undef);
- }
-
- $cmd = "$CA/overlap_partition \\\n";
- $cmd .= " -g $wrk/temp$libraryname/$asm.gkpStore \\\n";
- $cmd .= " -bs $ovlHashBlockSize \\\n" if (defined($ovlHashBlockSize));
- $cmd .= " -bl $ovlHashBlockLength \\\n" if (defined($ovlHashBlockLength));
- $cmd .= " -rs $ovlRefBlockSize \\\n" if (defined($ovlRefBlockSize));
- $cmd .= " -rl $ovlRefBlockLength \\\n" if (defined($ovlRefBlockLength));
- $cmd .= " -H $libToCorrect \\\n";
- $cmd .= " -R $minCorrectLib-$maxCorrectLib \\\n";
- $cmd .= " -C \\\n";
- $cmd .= " -o $wrk/temp$libraryname/1-overlapper";
- runCommand($wrk, $cmd);
- open(F, "< $wrk/temp$libraryname/1-overlapper/ovlopt") or die("failed partition for overlapper: no ovlopt file found", undef);
- my @job = <F>;
- close(F);
-
- my $blasrThreshold = 0;
- my $ovlThreads = getGlobal("ovlThreads");
- my $numOvlJobs = 0;
- my $numIndicies = 1;
- my $blasrOpts = getGlobal("blasr");
- my $bowtieOpts = getGlobal("bowtie");
- my $suffix = "sam";
-
- $blasrVersion="1.3.1.116174";
- # SMRTportal 1.4 changed parameters so update our blasr options
- if ($blasrVersion >= 1.3.1.116174) {
- $blasrOpts =~ s/-ignoreQuality//g;
- }
-
- if (defined(getGlobal("samFOFN"))) {
- my $fofn = getGlobal("samFOFN");
- $numOvlJobs = 0;
- } else {
- $numOvlJobs = $#job + 1;
- }
- if (-e "$wrk/temp$libraryname/1-overlapper/overlap.sh") {
- goto checkforerror;
- }
-
- # dump gatekeeper store and print eid to iid, add to it the gkpStore fastq file into eidToIID and lengths
- runCommand("$wrk/temp$libraryname", "$CA/gatekeeper -dumpfragments -tabular $asm.gkpStore |awk '{print \$1\"\\t\"\$2}' > $asm.eidToIID");
- runCommand("$wrk/temp$libraryname", "$CA/gatekeeper -dumpfragments -tabular $asm.gkpStore |awk '{print \$2\"\\t\"\$10}' > $asm.iidToLen");
- open F, "> $wrk/temp$libraryname/1-overlapper/ovlindex" or die ("can't open '$wrk/temp$libraryname/1-overlapper/ovlindex");
-
-
- # now do specific steps (either import SAMs or run blasr)
- if (defined(getGlobal("samFOFN"))) {
- my $fofn = getGlobal("samFOFN");
-
- # get the external UIDs from the input file rather than gatekeeper since these won't match our externally-generated SAM
- runCommand("$wrk/temp$libraryname", "cat $asm.gkpStore.fastqUIDmap | awk '{if (NF > 3) { print \$NF\"\\t\"\$(NF-1); } print \$3\"\\t\"\$2; }' >> $asm.eidToIID");
-
- # convert sam to ovl and put it into the overlap directory
- my $batch = 1;
- my $maxBatch = 1000;
-
- open(BAT, "> $wrk/temp$libraryname/1-overlapper/ovlbat") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlbat", undef);
- open(JOB, "> $wrk/temp$libraryname/1-overlapper/ovljob") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovljob", undef);
- open(OPT, "> $wrk/temp$libraryname/1-overlapper/ovlopt") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlopt", undef);
- open(SA, "> $wrk/temp$libraryname/1-overlapper/ovlindex") or die ("Couldn't open $wrk/temp$libraryname/1-overlapper/ovlindex", undef);
- $numIndicies++;
-
- open(F, "< $fofn") or die("Couldn't open '$fofn'", undef);
- while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- next if (m/^\s*\#/);
- next if (m/^\s*$/);
- if (! -e $_) {
- print STDERR "Warning: could not open SAM file $_, skipping\n";
- next;
- } elsif ($_ =~ /\.sam$/) {
- $suffix = "sam";
- } elsif ($_ =~ /\.sam\.gz$/) {
- $suffix = "sam.gz";
- } elsif ($_ =~ /\.sam\.bz2$/) {
- $suffix = "sam.bz2";
- }
-
- $numOvlJobs++;
- if ($numOvlJobs > $maxBatch) {
- $batch++;
- }
- my $batchName = sprintf("%03d", $batch);
- my $jobName = sprintf("%06d", $numOvlJobs);
- print BAT "$batchName\n";
- print OPT "-r 1-$totalNumToCorrect -l 1-$totalNumToCorrect\n";
- print JOB "$jobName\n";
- print SA "1\n";
- runCommand("$wrk/temp$libraryname/1-overlapper", "mkdir -p $batchName");
- runCommand("$wrk/temp$libraryname/1-overlapper/$batchName", "unlink $wrk/temp$libraryname/1-overlapper/$batchName/$jobName.$suffix") if (-e "$wrk/temp$libraryname/1-overlapper/$batchName/$jobName.$suffix");
- runCommand("$wrk/temp$libraryname/1-overlapper/$batchName", "cp $_ $wrk/temp$libraryname/1-overlapper/$batchName/$jobName.$suffix");
- }
- close(F);
- close(BAT);
- close(OPT);
- close(JOB);
- close(SA);
- } else {
- for (my $i = $minCorrectLib; $i <= $maxCorrectLib; $i++) {
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfasta " . $i . "_subset " . (defined($longReads) && $longReads == 1 && $i == $libToCorrect ? " -allreads -allbases" : "") . " -randomsubset $i 0.01 $wrk/temp$libraryname/$asm.gkpStore");
- }
- runCommand("$wrk/temp$libraryname/1-overlapper", "cat `ls [0-9]*_subset.fasta` > correct_subset.fasta");
- runCommand("$wrk/temp$libraryname/1-overlapper", "rm `ls [0-9]*_subset.fasta*`");
-
- # read in ovl options and partition data from the gatekeeper store
- my $i = 1;
- my $part = 1;
- my %longIndex = {};
- my %correctIndex = {};
-
- foreach my $ovlJob (@job) {
- $ovlJob =~ s/^\s+//;
- $ovlJob =~ s/\s+$//;
- my @values=split(/\s+/, $ovlJob);
- my $indexIds = "";
- my $correctIds = "";
-
- if (!defined($longIndex{$values[1]})) {
- # first the hash sequences
- my @startend=split(/-/, $values[1]);
- $indexIds = "-b $startend[0] -e $startend[1]";
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfasta long_reads_part$numIndicies -randomsubset $libToCorrect 1 $indexIds $wrk/temp$libraryname/$asm.gkpStore");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$BLASR/sawriter long_part$numIndicies.sa long_reads_part$numIndicies.fasta");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfragments -tabular $indexIds $wrk/temp$libraryname/$asm.gkpStore |awk '{print \$1\"\\t\"\$2}' > long_reads_part$numIndicies.eidToIID");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfragments -tabular $indexIds $wrk/temp$libraryname/$asm.gkpStore |awk '{print \$2\"\\t\"\$10}' > long_reads_part$numIndicies.iidToLen");
-
- $numIndicies++;
- $longIndex{$values[1]} = 1;
- }
-
- if (!defined($correctIndex{$values[3]})) {
- # now the correction sequences
- my @startend=split(/-/, $values[3]);
- $correctIds="-b $startend[0] -e $startend[1]";
- my $dumpAll = 0;
- my $start = $startend[0] > $minCorrectID ? $startend[0] : $minCorrectID;
- my $end = $startend[1] < $maxCorrectID ? $startend[1] : $maxCorrectID;
- if ($end - $start >= 0) {
- $dumpAll = 1;
- }
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfasta correct_reads_part" . $i . (defined($longReads) && $longReads == 1 && $dumpAll ? " -allreads -allbases" : " -randomsubset 0 1") . " $correctIds $wrk/temp$libraryname/$asm.gkpStore");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfragments -tabular $correctIds $wrk/temp$libraryname/$asm.gkpStore |awk '{print \$1\"\\t\"\$2}' > correct_reads_part$i.eidToIID");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$CA/gatekeeper -dumpfragments -tabular $correctIds $wrk/temp$libraryname/$asm.gkpStore |awk '{print \$2\"\\t\"\$10}' > correct_reads_part$i.iidToLen");
- $i++;
- $correctIndex{$values[3]} = 1;
- }
-
- print F $numIndicies-1 . "\n";
- $part++;
- }
- close(F);
-
- runCommand("$wrk/temp$libraryname/1-overlapper", "ln -s long_reads_part1.fasta long_reads.fasta");
- if (defined(getGlobal("blasr")) && $cutoffSpecified == 0) {
- # run with best limit set to kmer with a small (0.5-1% of the data)
- my $ovlThreads = getGlobal("ovlThreads");
- runCommand("$wrk/temp$libraryname/1-overlapper", "ln -s long_part1.sa long.sa");
-
- open F, "> $wrk/temp$libraryname/1-overlapper/pickNBest.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/pickNBest.sh");
- print F "#!" . getGlobal("shell") ."\n";
- print F getBinDirectoryShellCode();
- print F "\n";
-
- if (-e "$BLASR/../../etc/setup.sh") {
- print F "source $BLASR/../../etc/setup.sh && $BLASR/blasr -sa long.sa correct_subset.fasta long_reads.fasta -nproc $ovlThreads -bestn $ovlThreshold $blasrOpts -sam -out subset.sam";
- } else {
- print F "$BLASR/blasr -sa long.sa correct_subset.fasta long_reads.fasta -nproc $ovlThreads -bestn $ovlThreshold $blasrOpts -sam -out subset.sam";
- }
- close(F);
- chmod 0755, "$wrk/temp$libraryname/1-overlapper/pickNBest.sh";
- runCommand("$wrk/temp$libraryname/1-overlapper", "$wrk/temp$libraryname/1-overlapper/pickNBest.sh");
-
- runCommand("$wrk/temp$libraryname/1-overlapper", "cat subset.sam |grep -v \"@\" | awk '{print \$1}' |sort -T . |uniq -c |awk '{print \$1}' > subset.hist");
-
- # pick threshold as 2sd (95% of the bases)
- $blasrThreshold = pickMappingThreshold("$wrk/temp$libraryname/1-overlapper/subset.hist", ($blasrOpts =~ m/maxLCPLength/));
- runCommand("$wrk/temp$libraryname/1-overlapper", "unlink subset.sam");
- runCommand("$wrk/temp$libraryname/1-overlapper", "unlink correct_subset.fasta");
- } elsif (defined(getGlobal("bowtie"))) {
- # run with best limit set to all with a small (0.5-1% of the data)
- my $ovlThreads = getGlobal("ovlThreads");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$BOWTIE/bowtie2-build -f long_reads.fasta long.sa");
- runCommand("$wrk/temp$libraryname/1-overlapper", "$BOWTIE/bowtie2 -x long.sa -f correct_subset.fasta -p $ovlThreads -a $bowtieOpts -S subset.sam");
- runCommand("$wrk/temp$libraryname/1-overlapper", "cat subset.sam |grep -v \"@\" | awk '{print \$1}' |sort -T . |uniq -c |awk '{print \$1}' > subset.hist");
-
- # pick threshold as 2sd (95% of the bases)
- $blasrThreshold = pickMappingThreshold("$wrk/temp$libraryname/1-overlapper/subset.hist", 0);
- runCommand("$wrk/temp$libraryname/1-overlapper", "unlink subset.sam");
- runCommand("$wrk/temp$libraryname/1-overlapper", "unlink correct_subset.fasta");
- }
- }
- close(F);
-
- # now that we have our parameters, create a run job
- open F, "> $wrk/temp$libraryname/1-overlapper/overlap.sh" or die ("can't open '$wrk/temp$libraryname/1-overlapper/overlap.sh'");
- print F "#!" . getGlobal("shell") ."\n";
- print F getBinDirectoryShellCode();
- print F "\n";
- print F "jobid=\$$sgeTaskID\n";
- print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
- print F "jobid=\$1\n";
- print F "fi\n";
- print F "\n";
- print F "if test x\$jobid = x; then\n";
- print F " echo Error: I need $sgeTaskID set, or a job index on the command line\n";
- print F " exit 1\n";
- print F "fi\n";
- print F "\n";
- print F "bat=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlbat | tail -n 1`\n";
- print F "job=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovljob | tail -n 1`\n";
- print F "opt=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlopt | tail -n 1`\n";
- print F "sa=`head -n \$jobid $wrk/temp$libraryname/1-overlapper/ovlindex | tail -n 1`\n";
- print F "\n";
- print F "if [ ! -d $wrk/temp$libraryname/1-overlapper/\$bat ]; then\n";
- print F " mkdir $wrk/temp$libraryname/1-overlapper/\$bat\n";
- print F "fi\n";
- print F "\n";
- print F "if [ -e $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovb ]; then\n";
- print F " echo Job previously completed successfully.\n";
- print F " exit\n";
- print F "fi\n";
- print F "\n";
- print F "if [ x\$bat = x ]; then\n";
- print F " echo Error: Job index out of range.\n";
- print F " exit 1\n";
- print F "fi\n";
- print F "\n";
- print F "options=(\$opt)\n";
- print F "params=(\$(echo \${options[3]} | tr \"-\" \"\\n\")) \n";
- print F "numTotalJobs=$numOvlJobs\n";
- print F "numSA=" . ($numIndicies-1) . "\n";
- print F "numOvlJobs=\$((\$numTotalJobs/\$numSA))\n";
- print F "zeroJobId=\$((\$((\$jobid - 1)) % \$numOvlJobs))\n";
- print F "oneJobId=\$((\$zeroJobId+1))\n";
- print F "start=\${params[0]} \n";
- print F "start=\$((\$start - 1))\n";
- print F "end=\${params[1]} \n";
- print F "total=\$((\$end - \$start))\n";
- print F "echo \"Running partition \$job with options \$opt start \$start end \$end total \$total zero job \$zeroJobId and stride \$numOvlJobs\"\n";
- print F "\n";
- if (defined(getGlobal("blasr"))) {
- if (-e "$BLASR/../../etc/setup.sh") {
- print F " source $BLASR/../../etc/setup.sh \n";
- }
- print F " $BLASR/blasr \\\n";
- print F " -sa $wrk/temp$libraryname/1-overlapper/long_part\$sa.sa $wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.fasta \\\n";
- print F " $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.fasta \\\n";
- print F " -nproc $ovlThreads \\\n";
- if ($cutoffSpecified == 0) {
- print F " -bestn $blasrThreshold \\\n";
- }
- print F " $blasrOpts \\\n";
- print F " -sam -out $wrk/temp$libraryname/1-overlapper/\$bat/\$job.sam \\\n";
- print F " > $wrk/temp$libraryname/1-overlapper/\$jobid.out 2>&1 \\\n";
- print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success\n";
- print F " if test -e $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success ; then\n";
- print F " echo Blasr completed.\n";
- print F " else\n";
- print F " echo Blasr failed.\n";
- print F " tail $wrk/temp$libraryname/1-overlapper/\$jobid.out && exit\n";
- print F " fi\n";
- } elsif (defined(getGlobal("bowtie"))) {
- print F " $BOWTIE/bowtie2 \\\n";
- print F " -s \$start -u \$total \\\n";
- print F " -x $wrk/temp$libraryname/1-overlapper/long.sa -f $wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.fasta \\\n";
- print F " -p $ovlThreads -k $blasrThreshold \\\n";
- print F " $bowtieOpts \\\n";
- print F " -S $wrk/temp$libraryname/1-overlapper/\$bat/\$job.sam \\\n";
- print F " > $wrk/temp$libraryname/1-overlapper/\$jobid.out 2>&1 \\\n";
- print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success\n";
- print F " if test -e $wrk/temp$libraryname/1-overlapper/\$jobid.blasr.success ; then\n";
- print F " echo Bowtie completed.\n";
- print F " else\n";
- print F " echo Bowtie failed.\n";
- print F " tail $wrk/temp$libraryname/1-overlapper/\$jobid.out && exit\n";
- print F " fi\n";
- }
- print F " if test -e $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.eidToIID ; then\n";
- print F " \$bin/convertSamToCA \\\n";
- print F " $wrk/temp$libraryname/1-overlapper/\$bat/\$job.$suffix $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.eidToIID,$wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.eidToIID $wrk/temp$libraryname/1-overlapper/long_reads_part\$sa.iidToLen,$wrk/temp$libraryname/1-overlapper/correct_reads_part\$oneJobId.iidToLen \\\n";
- print F " > $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovls 2> $wrk/temp$libraryname/1-overlapper/\$jobid.java.err\\\n";
- print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.java.success\n";
- print F " else\n";
- print F " \$bin/convertSamToCA \\\n";
- print F " $wrk/temp$libraryname/1-overlapper/\$bat/\$job.$suffix $wrk/temp$libraryname/$asm.eidToIID $wrk/temp$libraryname/$asm.iidToLen \\\n";
- print F " > $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovls 2> $wrk/temp$libraryname/1-overlapper/\$jobid.java.err\\\n";
- print F " && touch $wrk/temp$libraryname/1-overlapper/\$jobid.java.success\n";
- print F " fi\n";
- print F " if test -e $wrk/temp$libraryname/1-overlapper/\$jobid.java.success ; then\n";
- print F " echo SamToCA conversion completed.\n";
- print F " else\n";
- print F " echo SamToCA conversion failed.\n";
- print F " tail $wrk/temp$libraryname/1-overlapper/\$jobid.java.err && exit\n";
- print F " fi\n";
- print F " \$bin/convertOverlap -ovl -i $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovls -o $wrk/temp$libraryname/1-overlapper/\$bat/\$job.ovb\n";
- print F " rm $wrk/temp$libraryname/1-overlapper/\$bat/\$job.sam\n";
- close(F);
- chmod 0755, "$wrk/temp$libraryname/1-overlapper/overlap.sh";
-
- if ($submitToGrid == 1) {
- my $sgeName = "pBcR_ovl_$asm$sgeName";
- my $jobName = getGridArrayName($sgeName, $numOvlJobs);
- my $arrayOpt = getGridArrayOption($sgeName, $numOvlJobs);
- submitBatchJobs($wrk, $asm, "$submitCommand $sge $sgeOvl $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/1-overlapper/overlap.sh", $jobName);
- } else {
- for (my $i = 1; $i <= $numOvlJobs; $i++) {
- schedulerSubmit("$wrk/temp$libraryname/1-overlapper/overlap.sh $i");
- }
- schedulerSetNumberOfProcesses(getGlobal("ovlConcurrency"));
- schedulerFinish();
- }
-
- checkforerror:
- my $failedJobs = 0;
- my $failureMessage = "";
-
- open(B, "< $wrk/temp$libraryname/1-overlapper/ovlbat") or die("failed to open '$wrk/temp$libraryname/1-overlapper/ovlbat'", undef);
- open(J, "< $wrk/temp$libraryname/1-overlapper/ovljob") or die("failed to open '$wrk/temp$libraryname/1-overlapper/ovljob'", undef);
-
- while (!eof(B) && !eof(J)) {
- my $b = <B>; chomp $b;
- my $j = <J>; chomp $j;
-
- if ((! -e "$wrk/temp$libraryname/1-overlapper/$b/$j.ovb.gz") &&
- (! -e "$wrk/temp$libraryname/1-overlapper/$b/$j.ovb")) {
- $failureMessage .= "ERROR: Overlap job $wrk/temp$libraryname/1-overlapper/$b/$j FAILED.\n";
- $failedJobs++;
- } else {
- runCommand("$wrk/temp$libraryname/1-overlapper/$b/", "rm $j.ovls");
- }
- }
- if (!eof(B) || !eof(J)) {
- print STDERR "Partitioning error; '$wrk/temp$libraryname/1-overlapper/ovlbat' and '$wrk/temp$libraryname/1-overlapper/ovljob' have extra lines.\n";
- }
- close(B);
- close(J);
-
- my $errorType = "sam conversion";
- if (defined(getGlobal("blasr"))) { $errorType = "blasr mapping"; }
- if (defined(getGlobal("bowtie"))) { $errorType = "bowtie mapping"; }
- $failureMessage .= "\n$failedJobs $errorType jobs failed.";
- if ($failedJobs > 0) {
- die "$failureMessage\n";
- }
- }
-
- $cmd = "-s $specFile ";
- $cmd .= "-p $asm -d . ";
- $cmd .= "ovlMerThreshold=$ovlThreshold ";
- $cmd .= "ovlHashLibrary=$libToCorrect ";
- $cmd .= "ovlRefLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "ovlCheckLibrary=1 ";
- $cmd .= "obtHashLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "obtRefLibrary=$minCorrectLib-$maxCorrectLib ";
- $cmd .= "obtCheckLibrary=0 ";
- $cmd .= "sgeName=\"" . getGlobal("sgeName") . "\" " if defined(getGlobal("sgeName"));
- $cmd .= "sgePropagateHold=\"pBcR_$asm$sgeName\" ";
- $cmd .= "stopAfter=overlapper";
- if ($submitToGrid == 1) {
- submitRunCA("$wrk/temp$libraryname", $asm, $cmd, "runCA_ovl_$asm$sgeName");
- } else {
- runCommand("$wrk/temp$libraryname", "$CA/runCA $cmd");
- }
-}
-
-if (! -e "$wrk/temp$libraryname/$asm.layout.success") {
- if ( -e "$wrk/temp$libraryname/$asm.layout.err") {
- # avoid infinite loops
- die "Error: found output from runCorrection. Stopping to avoid infinite loops. To try again, remove $asm.layout.*\n";
- }
- open F, "> $wrk/temp$libraryname/runCorrection.sh" or die ("can't open '$wrk/temp$libraryname/runCorrection.sh'");
- print F "#!" . getGlobal("shell") ."\n";
- print F getBinDirectoryShellCode();
- print F "\n";
- print F " if test -e $wrk/temp$libraryname/$asm.layout.success; then\n";
- print F " echo Job previously completed successfully.\n";
- print F " else\n";
- print F " \$bin/correctPacBio \\\n";
- if (defined($longReads) && $longReads == 1) {
- print F " -L \\\n";
- }
- print F " -C $coverage \\\n";
- print F " -M $maxUncorrectedGap \\\n";
- print F " -t $threads \\\n";
- print F " -p $partitions \\\n";
- print F " -o $asm \\\n";
- print F " $repeats \\\n";
- print F " -O $wrk/temp$libraryname/$asm.ovlStore \\\n";
- print F " -G $wrk/temp$libraryname/$asm.gkpStore \\\n";
- print F " -e $ovlErrorRate -c $ovlErrorRate -E $ovlErrorLimit > $wrk/temp$libraryname/$asm.layout.err 2> $wrk/temp$libraryname/$asm.layout.err && touch $wrk/temp$libraryname/$asm.layout.success\n";
- print F " fi\n";
- close(F);
- chmod 0755, "$wrk/temp$libraryname/runCorrection.sh";
-
- if ($submitToGrid == 1) {
- my $sgeName = "pBcR_correct_$asm$sgeName";
- submitBatchJobs("$wrk/temp$libraryname", $asm, "$submitCommand $sge $sgeCorrection $nameOption \"$sgeName\" $outputOption /dev/null $wrk/temp$libraryname/runCorrection.sh", $sgeName);
- } else {
- runCommand("$wrk/temp$libraryname", "$wrk/temp$libraryname/runCorrection.sh");
- }
-
- runCommand("$wrk/temp$libraryname", "rm -rf $asm.paired.ovlStore");
-}
-
-if (! -e "$wrk/temp$libraryname/runPartition.sh") {
- # convert QV to coverage for PBCNS
- # we use an ad-hoc conversion where QV 60 = coverage 8 (max) and QV 50 or below = coverage 1
- my $coverage = floor($QV - 50);
- if ($coverage < 1) { $coverage = 1; }
- if ($coverage > 8) { $coverage = 8; }
-
- # make sure we have the pb consensus module available if it was requested
- if (! -e "$BLASR/blasr" || ! -e "$BLASR/pbdagcon") {
- print STDERR "Warning: requested PBDAGON but either BLASR or pbdagcon executables were not found. Defaulting to AMOS\n";
- $pbcns = 0;
- }
-
- open F, "> $wrk/temp$libraryname/runPartition.sh" or die ("can't open '$wrk/temp$libraryname/runPartition.sh'");
- print F "#!" . getGlobal("shell") ."\n";
- print F getBinDirectoryShellCode();
- print F "\n";
- print F "jobid=\$$sgeTaskID\n";
- print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
- print F "jobid=\$1\n";
- print F "fi\n";
- print F "\n";
- print F "if test x\$jobid = x; then\n";
- print F " echo Error: I need $sgeTaskID set, or a job index on the command line\n";
- print F " exit 1\n";
- print F "fi\n";
- print F "\n";
- print F "if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
- print F " echo \"Job previously completed successfully.\"\n";
- print F "else\n";
- print F "if test -e $wrk/temp$libraryname/\$jobid.lay.success ; then\n";
- print F " echo \"Layout previously generated.\"\n";
- print F "else\n";
- print F " \$bin/outputLayout \\\n";
- if (defined($longReads) && $longReads == 1) {
- print F " -L \\\n";
- }
- print F " -e $ovlErrorRate -M $maxUncorrectedGap \\\n";
- print F " -o $wrk/temp$libraryname/$asm \\\n";
- print F " -p \$jobid \\\n";
- print F " -l $length \\\n";
- print F " $repeats \\\n";
- if (defined($pbcns) && $pbcns == 1) {
- print F " -P \\\n";
- }
- print F " -G $wrk/temp$libraryname/$asm.gkpStore \\\n";
- print F " > $wrk/temp$libraryname/\$jobid.lay.err 2>&1 && touch $wrk/temp$libraryname/\$jobid.lay.success\n";
- print F " if ! test -e $wrk/temp$libraryname/\$jobid.lay.success ; then\n";
- print F " echo \"Error generating layout for job \$jobid\"\n";
- print F " exit 1\n";
- print F " fi\n";
- print F " fi\n";
- print F " \n";
- if (defined($pbcns) && $pbcns == 1) {
- print F " numLays=`cat $wrk/temp$libraryname/$asm" . ".\$jobid.lay |wc -l`\n";
- } else {
- print F " numLays=`cat $wrk/temp$libraryname/$asm" . ".\$jobid.lay |grep \"{LAY\" |wc -l`\n";
- }
- print F " if test \$numLays = 0 ; then\n";
- print F " touch $wrk/temp$libraryname/\$jobid.fasta\n";
- print F " touch $wrk/temp$libraryname/\$jobid.qual\n";
- print F " touch $wrk/temp$libraryname/\$jobid.success\n";
- print F " else\n";
- if (!defined $bankPath) {
- print F " bankPath=\"$wrk/temp$libraryname\"\n";
- } elsif (uc($bankPath) eq "SHARED") {
- print F " bankPath=\"/dev/shm/temp$libraryname\"\n";
- print F " laySize=`ls -la $wrk/temp$libraryname/$asm.\$jobid.lay |awk '{print \$5}'`\n";
- print F " tmpFSSize=`df |grep /dev/shm |awk '{print \$(NF-2)}'`\n";
- if ($submitToGrid) {
- print F " let requiredSize=\$laySize*2\n";
- } else {
- print F " let requiredSize=\$laySize*2*$consensusConcurrency\n";
- }
- print F " if test x\$tmpFSSize = x; then\n";
- print F " tmpFSSize=0\n";
- print F " fi\n";
- print F " if [ \$requiredSize -lt \$tmpFSSize ]; then\n";
- print F " mkdir -p /dev/shm/temp$libraryname\n";
- print F " else\n";
- print F " bankPath=\"$wrk/temp$libraryname\"\n";
- print F " fi\n";
- } else {
- print F " bankPath=\"$bankPath/temp$libraryname\"\n";
- print F " mkdir -p \"$bankPath/temp$libraryname\"\n";
- }
- if (defined($pbcns) && $pbcns == 1) {
- print F " \$bin/convertToPBCNS -path $BLASR -input $wrk/temp$libraryname/$asm.\$jobid.lay -output $wrk/temp$libraryname/\$jobid.fasta -prefix $wrk/temp$libraryname/\$jobid.tmp -length $length -coverage $coverage -threads 1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
- print F " cat $wrk/temp$libraryname/\$jobid.fasta | awk '{if(\$0~/>/){print;}else{l=length(\$0);q=\"\";while(l--){q=q \" 60\"}printf(\"%s\\n\",q)}}' > $wrk/temp$libraryname/\$jobid.qual\n";
- print F " cat $wrk/temp$libraryname/\$jobid.fasta | awk '{if(\$0~/>/){sub(/>/,\"@\",\$0);print;}else{l=length(\$0);q=\"\";while(l--){q=q \"9\"}printf(\"%s\\n+\\n%s\\n\",\$0,q)}}' > $wrk/temp$libraryname/\$jobid.trim.fastq\n";
- print F " ln -s $wrk/temp$libraryname/\$jobid.fasta $wrk/temp$libraryname/\$jobid.trim.fasta\n";
- print F " ln -s $wrk/temp$libraryname/\$jobid.qual $wrk/temp$libraryname/\$jobid.trim.qual\n";
- print F " fi\n";
- } else {
- print F " rm -rf \$bankPath/$asm" . ".bnk_partition\$jobid.bnk\n";
- print F " $AMOS/bank-transact -b \$bankPath/$asm" . ".bnk_partition\$jobid.bnk -m $wrk/temp$libraryname/$asm.\$jobid" . ".lay -c > $wrk/temp$libraryname/bank-transact.\$jobid.err 2>&1\n";
- if (defined($shortReads) && $shortReads == 1) {
- print F " $AMOS/make-consensus -e 0.03 -w 5 -x $wrk/temp$libraryname/\$jobid.excluded -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- } elsif (defined($longReads) && $longReads == 1) {
- print F " $AMOS/make-consensus -e 0.30 -w 50 -o 5 -x $wrk/temp$libraryname/\$jobid.excluded -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- } else {
- print F " $AMOS/make-consensus -x $wrk/temp$libraryname/\$jobid.excluded -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- }
- print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
- print F " $AMOS/bank2fasta -e -q $wrk/temp$libraryname/\$jobid.qual -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.fasta\n";
- print F " else\n";
- print F " rm -rf \$bankPath/$asm" . ".bnk_partition\$jobid.bnk\n";
- print F " $AMOS/bank-transact -b \$bankPath/$asm" . ".bnk_partition\$jobid.bnk -m $wrk/temp$libraryname/$asm.\$jobid" . ".lay -c > $wrk/temp$libraryname/bank-transact.\$jobid.err 2>&1\n";
- if (defined($shortReads) && $shortReads == 1) {
- print F " $AMOS/make-consensus -e 0.03 -w 5 -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- } elsif (defined($longReads) && $longReads == 1) {
- print F " $AMOS/make-consensus -e 0.30 -w 50 -o 5 -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- } else {
- print F " $AMOS/make-consensus -B -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.out 2>&1 && touch $wrk/temp$libraryname/\$jobid.success\n";
- }
- print F " $AMOS/bank2fasta -e -q $wrk/temp$libraryname/\$jobid.qual -b \$bankPath/" . $asm . ".bnk_partition\$jobid.bnk > $wrk/temp$libraryname/\$jobid.fasta\n";
- print F " fi\n";
- print F " if test -e $wrk/temp$libraryname/\$jobid.success ; then\n";
- print F " $CA/trimFastqByQVWindow $wrk/temp$libraryname/\$jobid.fasta $wrk/temp$libraryname/\$jobid.qual $wrk/temp$libraryname/\$jobid.trim.fasta $wrk/temp$libraryname/\$jobid.trim.qual $QV $length > $wrk/temp$libraryname/\$jobid.trim.fastq\n";
-
- print F " rm -rf \$bankPath/$asm" . ".bnk_partition\$jobid.bnk\n";
- print F " rm $wrk/temp$libraryname/bank-transact.\$jobid.err\n";
- print F " rm $wrk/temp$libraryname/\$jobid.excluded\n";
- print F " rm $wrk/temp$libraryname/\$jobid.out\n";
- print F " fi\n";
- }
- print F " fi\n";
- print F "fi\n";
- close(F);
-
- chmod 0755, "$wrk/temp$libraryname/runPartition.sh";
-
- if ($submitToGrid == 1) {
- my $sgeName = "pBcR_cns_$asm$sgeName";
- my $jobName = getGridArrayName($sgeName, $partitions);
- my $arrayOpt = getGridArrayOption($sgeName, $partitions);
- submitBatchJobs("$wrk/temp$libraryname", $asm, "$submitCommand $sge $sgeConsensus $nameOption \"$jobName\" $arrayOpt $outputOption /dev/null $wrk/temp$libraryname/runPartition.sh", $jobName);
- } else {
- for (my $i = 1; $i <=$partitions; $i++) {
- schedulerSubmit("$wrk/temp$libraryname/runPartition.sh $i");
- }
- schedulerSetNumberOfProcesses($consensusConcurrency);
- schedulerFinish();
- }
-}
-
-for (my $i = 1; $i <= $partitions; $i++) {
- if (! -e "$wrk/temp$libraryname/$i.success") {
- die "Failed to run correction job $i. Remove $wrk/temp$libraryname/runPartition.sh to try again.\n";
- }
-}
-runCommand("$wrk/temp$libraryname", "rm -rf core.*");
-
-# combine partitions
-runCommand("$wrk/temp$libraryname", "cat `ls $asm.[0-9]*.log | sort -T . -rnk1` > corrected.log");
-runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.fasta |grep -v trim |sort -T . -rnk1` > corrected.fasta");
-runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.qual |grep -v trim | sort -T . -rnk1` > corrected.qual");
-
-# check how much we split the sequences up
-my %splitUIDs = ();
-runCommand("$wrk/temp$libraryname", "cat corrected.log |awk '{print \$1}' |sort -T . |uniq -c |awk '{if (\$1 > 1) print \$2}' > $asm.split.uid");
-
-# read in the split UIDs so we can process them
-open(F, "< $wrk/temp$libraryname/$asm.split.uid") or die("Couldn't open '$wrk/temp$libraryname/$asm.split.uid'", undef);
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
- chomp $_;
-
- $splitUIDs{$_} = 1;
-}
-close(F);
-
-open (F, "< $wrk/temp$libraryname/corrected.log") or die ("Couldn't open '$wrk/temp$libraryname/corrected.log", undef);
-open (O, "> $wrk/temp$libraryname/$asm.split.allEdit") or die ("Couldn't open '$wrk/temp$libraryname/$asm.split.allEdit", undef);
-
-my $totalSplitBases = 0;
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
- chomp $_;
-
- my ($uid, $newname, $subread, $start, $end) = split '\s+', $_;
- my $len = $end - $start;
- if (defined($splitUIDs{$uid}) && $splitUIDs{$uid} == 1) {
- print O "$newname\n";
-
- $totalSplitBases += $len;
- }
-}
-close(F);
-close(O);
-
-print STDOUT "Total split bases is $totalSplitBases vs $totalBP so ratio is " . ($totalSplitBases/$totalBP) . "\n";
-if (($totalSplitBases / $totalBP) > $MAX_SPLIT_PERCENTAGE && defined(getGlobal("maxCorrectionRounds")) && getGlobal("maxCorrectionRounds") > 1) {
- if (! -e "$wrk/temp$libraryname/temproundTwo/corrected.fasta") {
- runCommand("$wrk", "$CA/gatekeeper -dumpfastq $wrk/temp$libraryname/$asm.split -iid $wrk/temp$libraryname/$asm.split.uid $wrk/temp$libraryname/$asm.gkpStore > $wrk/temp$libraryname/$asm.split.out 2> $wrk/temp$libraryname/$asm.split.err");
- runCommand("$wrk", "$CA/replaceUIDwithName $wrk/temp$libraryname/$asm.gkpStore.fastqUIDmap $wrk/temp$libraryname/$asm.split.unmated.fastq > $wrk/temp$libraryname/$asm.rename.out 2> $wrk/temp$libraryname/$asm.rename.err");
- runCommand("$wrk", "$CA/gatekeeper -T -o $wrk/temp$libraryname/$asm.corrected.gkpStore $wrk/temp$libraryname/$libraryname.frg > $wrk/temp$libraryname/$asm.corrected.out 2> $wrk/temp$libraryname/$asm.corrected.err");
- runCommand("$wrk", "$CA/gatekeeper -dumpfragments -tabular $wrk/temp$libraryname/$asm.corrected.gkpStore |grep -v UID > $wrk/temp$libraryname/corrected.uids");
-
- my %gkpSeqs = ();
- open(F, "< $wrk/temp$libraryname/corrected.uids") or die("Couldn't open '$wrk/temp$libraryname/corrected.uids'", undef);
- while (<F>) {
- s/^\s+//;
- s/\s+$//;
- chomp $_;
- my ($uid, $iid, $mate, $mateUID, $libUID, $libIID, $deleted, $random, $orient, $length, $clrBgn, $clrEnd) = split '\s+', $_;
- $gkpSeqs{$uid} = 1;
- }
- close(F);
-
- open (F, "< $wrk/temp$libraryname/$asm.split.allEdit") or die ("Couldn't open '$wrk/temp$libraryname/$asm.split.allEdit'", undef);
- open (O, "> $wrk/temp$libraryname/$asm.split.edit") or die ("Couldn't open '$wrk/temp$libraryname/$asm.split.edit", undef);
- while (<F>) {
- s/^\s+//;
- s/\s+$//;
- chomp $_;
-
- my ($uid) = split '\s+', $_;
- if (defined($gkpSeqs{$uid}) && $gkpSeqs{$uid} == 1) {
- print O "frg uid $uid isdeleted 1\n";
- }
- }
- close(F);
- close(O);
-
- runCommand("$wrk", "$CA/gatekeeper --edit $wrk/temp$libraryname/$asm.split.edit $wrk/temp$libraryname/$asm.corrected.gkpStore > $wrk/temp$libraryname/$asm.split.out 2> $wrk/temp$libraryname/$asm.split.err");
- runCommand("$wrk", "$CA/gatekeeper -dumpfrg $wrk/temp$libraryname/$asm.corrected.gkpStore > $wrk/temp$libraryname/$asm.correctedUnsplit.frg");
- runCommand("$wrk", "$CA/gatekeeper -dumpfasta $wrk/temp$libraryname/corrected.roundOne $wrk/temp$libraryname/$asm.corrected.gkpStore");
- runCommand("$wrk", "mv $wrk/temp$libraryname/corrected.roundOne.fasta.qual $wrk/temp$libraryname/corrected.roundOne.qual");
-
- # clean up intermediate files
- runCommand("$wrk", "rm -rf $wrk/temp$libraryname/$libraryname.frg");
- runCommand("$wrk", "rm -rf $wrk/temp$libraryname/asm.corrected.*");
- runCommand("$wrk", "rm -rf $wrk/temp$libraryname/$asm.split.[12].fastq");
- runCommand("$wrk", "rm -rf $wrk/temp$libraryname/$asm.split.paired.fastq");
-
- #run second round
- if ($submitToGrid == 1) {
- submitRunPBcR("$wrk/temp$libraryname", $asm, "$commandLineOptions \"fastqFile=$wrk/temp$libraryname/$asm.split.unmated.fastq\" \"maxCorrectionRounds=1\" \"libraryname=roundTwo\" sgeName=pBcR_roundTwo_$asm$sgeName sgePropagateHold=\"pBcR_$asm$sgeName\" $wrk/temp$libraryname/$asm.correctedUnsplit.frg", "pbcr_roundTwo_$asm$sgeName");
- } else {
- runCommand("$wrk/temp$libraryname", "$CA/pacBioToCA $commandLineOptions \"fastqFile=$wrk/temp$libraryname/$asm.split.unmated.fastq\" \"maxCorrectionRounds=1\" \"libraryname=roundTwo\" $wrk/temp$libraryname/$asm.correctedUnsplit.frg");
- }
- }
- runCommand("$wrk/temp$libraryname", "cat corrected.roundOne.fasta temproundTwo/corrected.fasta |awk '{if (match(\$1, \"^>\") && match(\$1, \",\")) print substr(\$1, 1, index(\$1, \",\")-1); else print \$0}' > $asm.merged.fasta");
- runCommand("$wrk/temp$libraryname", "cat corrected.roundOne.qual temproundTwo/corrected.qual |awk '{if (match(\$1, \"^>\") && match(\$1, \",\")) print substr(\$1, 1, index(\$1, \",\")-1); else print \$0}' > $asm.merged.qual");
- runCommand("$wrk/temp$libraryname", "mv $asm.merged.fasta corrected.fasta");
- runCommand("$wrk/temp$libraryname", "mv $asm.merged.qual corrected.qual");
- runCommand("$wrk/temp$libraryname", "rm [0-9]*.fasta");
- runCommand("$wrk/temp$libraryname", "rm [0=0]*.qual");
- runCommand("$CA/trimFastqByQVWindow", "$wrk/temp$libraryname/corrected.fasta $wrk/temp$libraryname/corrected.qual $wrk/temp$libraryname/corrected.trim.fasta $wrk/temp$libraryname/corrected.trim.qual $QV $length > $wrk/temp$libraryname/corrected.trim.fastq");
-
- #runCommand("$wrk/temp$libraryname", "rm -rf $wrk/temp$libraryname/temproundTwo");
-}
-
-#save output from temporary directory
-runCommand("$wrk/temp$libraryname", "cp $asm.layout.err $wrk/$libraryname.correction.err");
-if (-e "$wrk/temp$libraryname/$asm.layout.hist") {
- runCommand("$wrk/temp$libraryname", "cp $asm.layout.hist $wrk/$libraryname.correction.hist");
-}
-runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.fasta |grep trim |sort -T . -rnk1` > $wrk/$libraryname.fasta");
-runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.qual |grep trim | sort -T . -rnk1` > $wrk/$libraryname.qual");
-runCommand("$wrk/temp$libraryname", "cat `ls [0-9]*.fastq |grep trim | sort -T . -rnk1` > $wrk/$libraryname.fastq");
-runCommand("$wrk", "$CA/fastqToCA -libraryname $libraryname -technology pacbio-corrected -type sanger -reads $wrk/$libraryname.fastq > $wrk/$libraryname.frg");
-
-my $numOutput = `ls $wrk/temp$libraryname/*sge.out* |wc -l`;
-chomp $numOutput;
-if ($numOutput != 0) {
- runCommand("$wrk/temp$libraryname", "cat `ls *sge.out*` > $wrk/temp$libraryname/corrected.err");
- runCommand("$wrk/temp$libraryname", "cp corrected.err $wrk/$libraryname.err");
-}
-
-# generate summary reports
-my %nameTranslation;
-open(F, "< $wrk/temp$libraryname/$asm.gkpStore.fastqUIDmap") or die("Couldn't open '$wrk/temp$libraryname/$asm.gkpStore.fastqUIDmap'", undef);
-
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- next if (m/^\s*\#/);
- next if (m/^\s*$/);
- my @splitLine=split /\s+/;
- $nameTranslation{$splitLine[1]} = $splitLine[2];
-}
-close(F);
-
-my $maxCorrected = 0;
-my $totalCorrected = 0;
-my $totalCorrectedBP = 0;
-open(F, "< $wrk/temp$libraryname/corrected.log") or die("Couldn't open '$wrk/temp$libraryname/corrected.log'", undef);
-open(W, "> $wrk/$libraryname.log") or die("Couldn't open '$wrk/$libraryname.log'", undef);
-
-while (<F>) {
- s/^\s+//;
- s/\s+$//;
-
- next if (m/^\s*\#/);
- next if (m/^\s*$/);
- my @splitLine=split /\s+/;
- my $len = $splitLine[4] - $splitLine[3];
- my $name = (defined($nameTranslation{$splitLine[0]}) ? $nameTranslation{$splitLine[0]} : $splitLine[0]);
-
- print W "INPUT_NAME\tOUTPUT_NAME\tSUBREAD\tSTART\tEND\tLENGTH\n"if ($totalCorrected == 0);
- print W "$name\t$splitLine[1]\t$splitLine[2]\t$splitLine[3]\t$splitLine[4]\t$len\n";
-
- $maxCorrected = $len if ($len > $maxCorrected);
- $totalCorrected++;
- $totalCorrectedBP += $len;
-}
-close(F);
-close(W);
-
-print STDERR "******** Correction Summary ********\n";
-printf "Total corrected bp (before consensus): $totalCorrectedBP (%.3f).\n", ($totalCorrectedBP/$totalBP)*100;
-printf "Longest sequence: $maxCorrected bp, mean: %.3f bp.\n", ($totalCorrectedBP/$totalCorrected);
-
-# finally clean up the assembly directory
-if ($cleanup == 1) {
- runCommand("$wrk", "rm -rf temp$libraryname/[0-9]*.*");
- runCommand("$wrk", "rm -rf temp$libraryname/$asm.[0-9]*.log");
- runCommand("$wrk", "rm -rf temp$libraryname/$asm.gkpStore");
- runCommand("$wrk", "rm -rf temp$libraryname/$asm.ovlStore");
-}
-
-# trim here?
-
-print STDERR "********* Finished correcting $totalBP bp (using $totalCorrectingWith bp).\n";
diff --git a/src/AS_PBR/pbdagcon.Linux-amd64.bin b/src/AS_PBR/pbdagcon.Linux-amd64.bin
new file mode 100755
index 0000000..d06351b
Binary files /dev/null and b/src/AS_PBR/pbdagcon.Linux-amd64.bin differ
diff --git a/src/AS_PBR/pbutgcns.Darwin-amd64.bin b/src/AS_PBR/pbutgcns.Darwin-amd64.bin
new file mode 100755
index 0000000..e5554f9
Binary files /dev/null and b/src/AS_PBR/pbutgcns.Darwin-amd64.bin differ
diff --git a/src/AS_PBR/pbutgcns.Linux-amd64.bin b/src/AS_PBR/pbutgcns.Linux-amd64.bin
new file mode 100755
index 0000000..39c8af1
Binary files /dev/null and b/src/AS_PBR/pbutgcns.Linux-amd64.bin differ
diff --git a/src/AS_PER/._AS_PER_encodeSequenceQuality.C b/src/AS_PER/._AS_PER_encodeSequenceQuality.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_encodeSequenceQuality.C differ
diff --git a/src/AS_PER/._AS_PER_encodeSequenceQuality.H b/src/AS_PER/._AS_PER_encodeSequenceQuality.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_encodeSequenceQuality.H differ
diff --git a/src/AS_PER/._AS_PER_genericStore.C b/src/AS_PER/._AS_PER_genericStore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_genericStore.C differ
diff --git a/src/AS_PER/._AS_PER_genericStore.H b/src/AS_PER/._AS_PER_genericStore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_genericStore.H differ
diff --git a/src/AS_PER/._AS_PER_genericStore.doc.txt b/src/AS_PER/._AS_PER_genericStore.doc.txt
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_genericStore.doc.txt differ
diff --git a/src/AS_PER/._AS_PER_gkLibrary.C b/src/AS_PER/._AS_PER_gkLibrary.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkLibrary.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore.C b/src/AS_PER/._AS_PER_gkStore.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_IID.C b/src/AS_PER/._AS_PER_gkStore_IID.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_IID.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_PLC.C b/src/AS_PER/._AS_PER_gkStore_PLC.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_PLC.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_UID.C b/src/AS_PER/._AS_PER_gkStore_UID.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_UID.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_clearRange.C b/src/AS_PER/._AS_PER_gkStore_clearRange.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_clearRange.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_fragments.C b/src/AS_PER/._AS_PER_gkStore_fragments.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_fragments.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_load.C b/src/AS_PER/._AS_PER_gkStore_load.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_load.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_partition.C b/src/AS_PER/._AS_PER_gkStore_partition.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_partition.C differ
diff --git a/src/AS_PER/._AS_PER_gkStore_stats.C b/src/AS_PER/._AS_PER_gkStore_stats.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStore_stats.C differ
diff --git a/src/AS_PER/._AS_PER_gkStream.C b/src/AS_PER/._AS_PER_gkStream.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkStream.C differ
diff --git a/src/AS_PER/._AS_PER_gkpStore.H b/src/AS_PER/._AS_PER_gkpStore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._AS_PER_gkpStore.H differ
diff --git a/src/AS_PER/._Makefile b/src/AS_PER/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._Makefile differ
diff --git a/src/AS_PER/._gkClearRange.H b/src/AS_PER/._gkClearRange.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._gkClearRange.H differ
diff --git a/src/AS_PER/._gkFragment.H b/src/AS_PER/._gkFragment.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._gkFragment.H differ
diff --git a/src/AS_PER/._gkLibrary.H b/src/AS_PER/._gkLibrary.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._gkLibrary.H differ
diff --git a/src/AS_PER/._gkPlacement.H b/src/AS_PER/._gkPlacement.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._gkPlacement.H differ
diff --git a/src/AS_PER/._gkStore.H b/src/AS_PER/._gkStore.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._gkStore.H differ
diff --git a/src/AS_PER/._gkStream.H b/src/AS_PER/._gkStream.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._gkStream.H differ
diff --git a/src/AS_PER/._size.C b/src/AS_PER/._size.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PER/._size.C differ
diff --git a/src/AS_PL5/._AS_MSG b/src/AS_PL5/._AS_MSG
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PL5/._AS_MSG differ
diff --git a/src/AS_PL5/._AS_PER b/src/AS_PL5/._AS_PER
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_PL5/._AS_PER differ
diff --git a/src/AS_PL5/._README b/src/AS_PL5/._README
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/._README differ
diff --git a/src/AS_PL5/._caqc.pl b/src/AS_PL5/._caqc.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/._caqc.pl differ
diff --git a/src/AS_PL5/AS_MSG/._MSG.C b/src/AS_PL5/AS_MSG/._MSG.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._MSG.C differ
diff --git a/src/AS_PL5/AS_MSG/._MSG.h b/src/AS_PL5/AS_MSG/._MSG.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._MSG.h differ
diff --git a/src/AS_PL5/AS_MSG/._MSG.swig b/src/AS_PL5/AS_MSG/._MSG.swig
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._MSG.swig differ
diff --git a/src/AS_PL5/AS_MSG/._Makefile.PL b/src/AS_PL5/AS_MSG/._Makefile.PL
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._Makefile.PL differ
diff --git a/src/AS_PL5/AS_MSG/._Parser.pm b/src/AS_PL5/AS_MSG/._Parser.pm
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._Parser.pm differ
diff --git a/src/AS_PL5/AS_MSG/._acronyms.h b/src/AS_PL5/AS_MSG/._acronyms.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._acronyms.h differ
diff --git a/src/AS_PL5/AS_MSG/._parser.h b/src/AS_PL5/AS_MSG/._parser.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._parser.h differ
diff --git a/src/AS_PL5/AS_MSG/._pmesg.h b/src/AS_PL5/AS_MSG/._pmesg.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._pmesg.h differ
diff --git a/src/AS_PL5/AS_MSG/._test.pl b/src/AS_PL5/AS_MSG/._test.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_MSG/._test.pl differ
diff --git a/src/AS_PL5/AS_PER/._GateKeeperStore.pm b/src/AS_PL5/AS_PER/._GateKeeperStore.pm
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_PER/._GateKeeperStore.pm differ
diff --git a/src/AS_PL5/AS_PER/._Makefile.PL b/src/AS_PL5/AS_PER/._Makefile.PL
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_PER/._Makefile.PL differ
diff --git a/src/AS_PL5/AS_PER/._PER.swig b/src/AS_PL5/AS_PER/._PER.swig
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_PL5/AS_PER/._PER.swig differ
diff --git a/src/AS_REF/._Makefile b/src/AS_REF/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REF/._Makefile differ
diff --git a/src/AS_REF/._NOTES b/src/AS_REF/._NOTES
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REF/._NOTES differ
diff --git a/src/AS_REF/._buildRefContigs.C b/src/AS_REF/._buildRefContigs.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REF/._buildRefContigs.C differ
diff --git a/src/AS_REF/._buildRefOverlaps.C b/src/AS_REF/._buildRefOverlaps.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REF/._buildRefOverlaps.C differ
diff --git a/src/AS_REF/._frg-to-fasta.pl b/src/AS_REF/._frg-to-fasta.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REF/._frg-to-fasta.pl differ
diff --git a/src/AS_REF/._refAlignment.H b/src/AS_REF/._refAlignment.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REF/._refAlignment.H differ
diff --git a/src/AS_REF/buildRefContigs.C b/src/AS_REF/buildRefContigs.C
index c068182..6726a1f 100644
--- a/src/AS_REF/buildRefContigs.C
+++ b/src/AS_REF/buildRefContigs.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: buildRefContigs.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: buildRefContigs.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include <stdlib.h>
#include <stdio.h>
@@ -32,7 +32,9 @@ const char *mainid = "$Id: buildRefContigs.C 4371 2013-08-01 17:19:47Z brianwale
#include "AS_global.H"
#include "AS_UTL_Var.H"
#include "AS_UTL_fileIO.H"
-#include "AS_UTL_splitToWords.H"
+
+#include "splitToWords.H"
+
#include "AS_MSG_pmesg.H"
#include "AS_PER_gkpStore.H"
@@ -367,7 +369,7 @@ outputUnitigs(void) {
ium.iaccession = iumId++;
ium.coverage_stat = 10.0;
ium.status = AS_UNIQUE;
- ium.unique_rept = AS_FORCED_UNIQUE;
+ ium.unique_rept = 'U'; // AS_FORCED_UNIQUE;
ium.length = e;
ium.consensus = 0L;
ium.quality = 0L;
diff --git a/src/AS_REF/buildRefOverlaps.C b/src/AS_REF/buildRefOverlaps.C
index f3c2da2..568b73a 100644
--- a/src/AS_REF/buildRefOverlaps.C
+++ b/src/AS_REF/buildRefOverlaps.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: buildRefOverlaps.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: buildRefOverlaps.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include <stdlib.h>
#include <stdio.h>
@@ -32,7 +32,7 @@ const char *mainid = "$Id: buildRefOverlaps.C 4371 2013-08-01 17:19:47Z brianwal
#include "AS_global.H"
#include "AS_UTL_Var.H"
#include "AS_UTL_fileIO.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
#include "AS_MSG_pmesg.H"
#include "AS_PER_gkpStore.H"
diff --git a/src/AS_REZ/._CommonREZ.H b/src/AS_REZ/._CommonREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._CommonREZ.H differ
diff --git a/src/AS_REZ/._ConsistencyChecksREZ.C b/src/AS_REZ/._ConsistencyChecksREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._ConsistencyChecksREZ.C differ
diff --git a/src/AS_REZ/._ConsistencyChecksREZ.H b/src/AS_REZ/._ConsistencyChecksREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._ConsistencyChecksREZ.H differ
diff --git a/src/AS_REZ/._DataTypesREZ.H b/src/AS_REZ/._DataTypesREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._DataTypesREZ.H differ
diff --git a/src/AS_REZ/._GapFillREZ.C b/src/AS_REZ/._GapFillREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._GapFillREZ.C differ
diff --git a/src/AS_REZ/._GapWalkerREZ.C b/src/AS_REZ/._GapWalkerREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._GapWalkerREZ.C differ
diff --git a/src/AS_REZ/._GapWalkerREZ.H b/src/AS_REZ/._GapWalkerREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._GapWalkerREZ.H differ
diff --git a/src/AS_REZ/._Makefile b/src/AS_REZ/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._Makefile differ
diff --git a/src/AS_REZ/._MicroHetREZ.C b/src/AS_REZ/._MicroHetREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._MicroHetREZ.C differ
diff --git a/src/AS_REZ/._MicroHetREZ.H b/src/AS_REZ/._MicroHetREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._MicroHetREZ.H differ
diff --git a/src/AS_REZ/._PartitionsREZ.H b/src/AS_REZ/._PartitionsREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._PartitionsREZ.H differ
diff --git a/src/AS_REZ/._RepeatRez.H b/src/AS_REZ/._RepeatRez.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._RepeatRez.H differ
diff --git a/src/AS_REZ/._SUBDELREZ.H b/src/AS_REZ/._SUBDELREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._SUBDELREZ.H differ
diff --git a/src/AS_REZ/._StatisticsREZ.C b/src/AS_REZ/._StatisticsREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._StatisticsREZ.C differ
diff --git a/src/AS_REZ/._StatisticsREZ.H b/src/AS_REZ/._StatisticsREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._StatisticsREZ.H differ
diff --git a/src/AS_REZ/._UpdateREZ.C b/src/AS_REZ/._UpdateREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._UpdateREZ.C differ
diff --git a/src/AS_REZ/._UpdateREZ.H b/src/AS_REZ/._UpdateREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._UpdateREZ.H differ
diff --git a/src/AS_REZ/._UtilsREZ.C b/src/AS_REZ/._UtilsREZ.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._UtilsREZ.C differ
diff --git a/src/AS_REZ/._UtilsREZ.H b/src/AS_REZ/._UtilsREZ.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_REZ/._UtilsREZ.H differ
diff --git a/src/AS_RUN/._Makefile b/src/AS_RUN/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._Makefile differ
diff --git a/src/AS_RUN/._README b/src/AS_RUN/._README
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._README differ
diff --git a/src/AS_RUN/._asmQC b/src/AS_RUN/._asmQC
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_RUN/._asmQC differ
diff --git a/src/AS_RUN/._fragmentDepth.C b/src/AS_RUN/._fragmentDepth.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._fragmentDepth.C differ
diff --git a/src/AS_RUN/._get-commit-logs.pl b/src/AS_RUN/._get-commit-logs.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._get-commit-logs.pl differ
diff --git a/src/AS_RUN/._markUniqueUnique.C b/src/AS_RUN/._markUniqueUnique.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._markUniqueUnique.C differ
diff --git a/src/AS_RUN/._new.pl b/src/AS_RUN/._new.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._new.pl differ
diff --git a/src/AS_RUN/._replaceIIDwithName-overlapDump.pl b/src/AS_RUN/._replaceIIDwithName-overlapDump.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._replaceIIDwithName-overlapDump.pl differ
diff --git a/src/AS_RUN/._replaceUIDwithName-fastq.pl b/src/AS_RUN/._replaceUIDwithName-fastq.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._replaceUIDwithName-fastq.pl differ
diff --git a/src/AS_RUN/._replaceUIDwithName-posmap.pl b/src/AS_RUN/._replaceUIDwithName-posmap.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._replaceUIDwithName-posmap.pl differ
diff --git a/src/AS_RUN/._runCA-dedupe.pl b/src/AS_RUN/._runCA-dedupe.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._runCA-dedupe.pl differ
diff --git a/src/AS_RUN/._runCA-overlapStoreBuild.pl b/src/AS_RUN/._runCA-overlapStoreBuild.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._runCA-overlapStoreBuild.pl differ
diff --git a/src/AS_RUN/._runCA.pl b/src/AS_RUN/._runCA.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/._runCA.pl differ
diff --git a/src/AS_RUN/._sanity b/src/AS_RUN/._sanity
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_RUN/._sanity differ
diff --git a/src/AS_RUN/Makefile b/src/AS_RUN/Makefile
index afa7d7f..849fbda 100644
--- a/src/AS_RUN/Makefile
+++ b/src/AS_RUN/Makefile
@@ -37,7 +37,8 @@ SCRIPTS = runCA \
caqc_help.ini \
ca2ace.pl \
asmToAGP.pl \
- replaceUIDwithName
+ replaceUIDwithName-fastq \
+ replaceUIDwithName-posmap
# Include for AS project rules
@@ -61,7 +62,9 @@ runCA-overlapStoreBuild: runCA-overlapStoreBuild.pl
@chmod 775 $(LOCAL_BIN)/$@
TIGR: asmQC/TIGR
- @cp -r $< $(LOCAL_BIN)
+ @mkdir $(LOCAL_BIN)/TIGR
+ @cp $</AsmLib.pm $(LOCAL_BIN)/TIGR
+ @cp $</Foundation.pm $(LOCAL_BIN)/TIGR
caqc_help.ini: asmQC/caqc_help.ini
@cp $< $(LOCAL_BIN)/$@
@@ -83,7 +86,11 @@ mergeqc.pl: asmQC/mergeqc.pl
@cp $< $(LOCAL_BIN)/$@
@chmod 775 $(LOCAL_BIN)/$@
-replaceUIDwithName: replaceUIDwithName.pl
+replaceUIDwithName-fastq: replaceUIDwithName-fastq.pl
+ @cp $< $(LOCAL_BIN)/$@
+ @chmod 775 $(LOCAL_BIN)/$@
+
+replaceUIDwithName-posmap: replaceUIDwithName-posmap.pl
@cp $< $(LOCAL_BIN)/$@
@chmod 775 $(LOCAL_BIN)/$@
diff --git a/src/AS_RUN/asmQC/._TIGR b/src/AS_RUN/asmQC/._TIGR
new file mode 100755
index 0000000..bb87e14
Binary files /dev/null and b/src/AS_RUN/asmQC/._TIGR differ
diff --git a/src/AS_RUN/asmQC/._asmToAGP.pl b/src/AS_RUN/asmQC/._asmToAGP.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._asmToAGP.pl differ
diff --git a/src/AS_RUN/asmQC/._asmToFasta.pl b/src/AS_RUN/asmQC/._asmToFasta.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._asmToFasta.pl differ
diff --git a/src/AS_RUN/asmQC/._ca2ace.pl b/src/AS_RUN/asmQC/._ca2ace.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._ca2ace.pl differ
diff --git a/src/AS_RUN/asmQC/._caqc-diff.pl b/src/AS_RUN/asmQC/._caqc-diff.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._caqc-diff.pl differ
diff --git a/src/AS_RUN/asmQC/._caqc.pl b/src/AS_RUN/asmQC/._caqc.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._caqc.pl differ
diff --git a/src/AS_RUN/asmQC/._caqc_help.ini b/src/AS_RUN/asmQC/._caqc_help.ini
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._caqc_help.ini differ
diff --git a/src/AS_RUN/asmQC/._checkMates.rb b/src/AS_RUN/asmQC/._checkMates.rb
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._checkMates.rb differ
diff --git a/src/AS_RUN/asmQC/._compare-asm.sh b/src/AS_RUN/asmQC/._compare-asm.sh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._compare-asm.sh differ
diff --git a/src/AS_RUN/asmQC/._compare-qc.pl b/src/AS_RUN/asmQC/._compare-qc.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._compare-qc.pl differ
diff --git a/src/AS_RUN/asmQC/._compare-trim-against-map.pl b/src/AS_RUN/asmQC/._compare-trim-against-map.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._compare-trim-against-map.pl differ
diff --git a/src/AS_RUN/asmQC/._find-chimeric-contigs.pl b/src/AS_RUN/asmQC/._find-chimeric-contigs.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._find-chimeric-contigs.pl differ
diff --git a/src/AS_RUN/asmQC/._find-chimeric-reads.pl b/src/AS_RUN/asmQC/._find-chimeric-reads.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._find-chimeric-reads.pl differ
diff --git a/src/AS_RUN/asmQC/._mergeqc.pl b/src/AS_RUN/asmQC/._mergeqc.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._mergeqc.pl differ
diff --git a/src/AS_RUN/asmQC/._statsFromAGP.rb b/src/AS_RUN/asmQC/._statsFromAGP.rb
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._statsFromAGP.rb differ
diff --git a/src/AS_RUN/asmQC/._unitigGraph.rb b/src/AS_RUN/asmQC/._unitigGraph.rb
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._unitigGraph.rb differ
diff --git a/src/AS_RUN/asmQC/._utgCoverageGaps.rb b/src/AS_RUN/asmQC/._utgCoverageGaps.rb
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/._utgCoverageGaps.rb differ
diff --git a/src/AS_RUN/asmQC/TIGR/._AsmLib.pm b/src/AS_RUN/asmQC/TIGR/._AsmLib.pm
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/TIGR/._AsmLib.pm differ
diff --git a/src/AS_RUN/asmQC/TIGR/._Foundation.pm b/src/AS_RUN/asmQC/TIGR/._Foundation.pm
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/asmQC/TIGR/._Foundation.pm differ
diff --git a/src/AS_RUN/markUniqueUnique.C b/src/AS_RUN/markUniqueUnique.C
index 7477679..6310664 100644
--- a/src/AS_RUN/markUniqueUnique.C
+++ b/src/AS_RUN/markUniqueUnique.C
@@ -1,4 +1,4 @@
-const char *mainid = "$Id: markUniqueUnique.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: markUniqueUnique.C 4518 2014-03-31 20:11:04Z brianwalenz $";
// Assembly terminator module. It is the backend of the assembly
// pipeline and replaces internal accession numbers by external
@@ -233,8 +233,9 @@ int main (int argc, char *argv[]) {
}
if (toggled) {
- tigStore->setUnitigFUR(i, AS_FORCED_UNIQUE);
- numToggled++;
+ tigStore->setUnitigForceUnique(i, true);
+ tigStore->setUnitigForceRepeat(i, false);
+ numToggled++;
}
}
diff --git a/src/AS_RUN/new.pl b/src/AS_RUN/new.pl
new file mode 100644
index 0000000..3120954
--- /dev/null
+++ b/src/AS_RUN/new.pl
@@ -0,0 +1,6325 @@
+#!/usr/bin/env perl
+
+use strict;
+
+use Config; # for @signame
+use FindBin;
+use Cwd;
+use Carp;
+use FileHandle;
+
+use Sys::Hostname;
+
+use POSIX qw(ceil floor sys_wait_h);
+
+my $bin = undef; # Path to binaries, set once in main.
+my $cmd = undef; # Temporary string passed to system().
+my $wrk = undef; # Path to our assembly directory.
+my $asm = undef; # Name of our assembly.
+
+my $numFrags;
+my %global;
+my %synops;
+
+my $commandLineOptions = "";
+my $specLog = "";
+
+my @specFiles;
+my @specOpts;
+my @fragFiles;
+
+
+sub submitBatchJobs($$) {
+ my $SGE = shift @_;
+ my $TAG = shift @_;
+
+ if (runningOnGrid()) {
+ runCommand($wrk, $SGE) and caFailure("Failed to submit batch jobs.");
+ submitScript($TAG);
+ } else {
+ pleaseExecute($SGE);
+ }
+}
+
+
+# Decide what bin directory to use.
+#
+# When we are running on SGE, the path of this perl script is NOT
+# always the correct architecture. If the submission host is
+# FreeBSD, but the grid is Linux, the BSD box will submit
+# FreeBSD/bin/runCA.pl to the grid -- unless it knows in advance,
+# there is no way to pick the correct one. The grid host then has to
+# have enough smarts to choose the correct binaries, and that is what
+# we're doing here.
+#
+# To make it more trouble, shell scripts need to do all this by
+# themselves.
+#
+sub getInstallDirectory () {
+ my @t = split '/', "$FindBin::RealBin";
+ pop @t; # bin
+ pop @t; # arch, e.g., FreeBSD-amd64
+ my $installDir = join '/', @t; # path to the assembler
+
+ return($installDir);
+}
+
+sub getBinDirectory () {
+ my $installDir = getInstallDirectory();
+
+ my $syst = `uname -s`; chomp $syst; # OS implementation
+ my $arch = `uname -m`; chomp $arch; # Hardware platform
+ my $name = `uname -n`; chomp $name; # Name of the system
+
+ $arch = "amd64" if ($arch eq "x86_64");
+ $arch = "ppc" if ($arch eq "Power Macintosh");
+
+ my $path = "$installDir/$syst-$arch/bin";
+
+ my $pathMap = getGlobal("pathMap");
+ if (defined($pathMap)) {
+ open(F, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef);
+ while (<F>) {
+ my ($n, $b) = split '\s+', $_;
+ $path = $b if ($name eq $n);
+ }
+ close(F);
+ }
+
+ return($path);
+}
+
+sub getBinDirectoryShellCode () {
+ my $installDir = getInstallDirectory();
+ my $string;
+
+ $string = "\n";
+ $string .= "syst=`uname -s`\n";
+ $string .= "arch=`uname -m`\n";
+ $string .= "name=`uname -n`\n";
+ $string .= "\n";
+ $string .= "if [ \"\$arch\" = \"x86_64\" ] ; then\n";
+ $string .= " arch=\"amd64\"\n";
+ $string .= "fi\n";
+ $string .= "if [ \"\$arch\" = \"Power Macintosh\" ] ; then\n";
+ $string .= " arch=\"ppc\"\n";
+ $string .= "fi\n";
+ $string .= "\n";
+ $string .= "bin=\"$installDir/\$syst-\$arch/bin\"\n";
+ $string .= "\n";
+
+ my $pathMap = getGlobal("pathMap");
+ if (defined($pathMap)) {
+ open(PM, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef);
+ while (<PM>) {
+ my ($n, $b) = split '\s+', $_;
+ $string .= "if [ \"\$name\" = \"$n\" ] ; then\n";
+ $string .= " bin=\"$b\"\n";
+ $string .= "fi\n";
+ }
+ close(PM);
+ $string .= "\n";
+ }
+
+ return($string);
+}
+
+
+
+
+
+# Return the second argument, unless the first argument is found in
+# %global, in which case return that.
+#
+sub getGlobal ($) {
+ my $var = shift @_;
+ caFailure("script error -- $var has no defined value", undef) if (!exists($global{$var}));
+ return($global{$var});
+}
+
+sub setGlobal ($$) {
+ my $var = shift @_;
+ my $val = shift @_;
+
+ # If no value, set the field to undefined, the default for many of the options.
+
+ $val = undef if ($val eq "");
+
+ # Handle special cases.
+
+ if ($var eq "merSize") {
+ setGlobal("obtMerSize", $val);
+ setGlobal("ovlMerSize", $val);
+ return;
+ }
+
+ if ($var eq "merThreshold") {
+ setGlobal("obtMerThreshold", $val);
+ setGlobal("ovlMerThreshold", $val);
+ return;
+ }
+
+ if ($var eq "merDistinct") {
+ setGlobal("obtMerDistinct", $val);
+ setGlobal("ovlMerDistinct", $val);
+ return;
+ }
+
+ if ($var eq "merTotal") {
+ setGlobal("obtMerTotal", $val);
+ setGlobal("ovlMerTotal", $val);
+ return;
+ }
+
+ if ($var eq "overlapper") {
+ setGlobal("obtOverlapper", $val);
+ setGlobal("ovlOverlapper", $val);
+ return;
+ }
+
+ if (($var eq "gridEngine") && ($val eq "SGE")) {
+ setGlobal("gridSubmitCommand", "qsub");
+ setGlobal("gridHoldOption", "-hold_jid \"WAIT_TAG\"");
+ setGlobal("gridSyncOption", "-sync y");
+ setGlobal("gridNameOption", "-cwd -N");
+ setGlobal("gridArrayOption", "-t ARRAY_JOBS");
+ setGlobal("gridArrayName", "ARRAY_NAME");
+ setGlobal("gridOutputOption", "-j y -o");
+ setGlobal("gridPropagateCommand", "qalter -hold_jid \"WAIT_TAG\"");
+ setGlobal("gridNameToJobIDCommand", undef);
+ setGlobal("gridTaskID", "SGE_TASK_ID");
+ setGlobal("gridArraySubmitID", "\\\$TASK_ID");
+ setGlobal("gridJobID", "JOB_ID");
+ }
+
+ if (($var eq "gridEngine") && ($val eq "LSF")) {
+ setGlobal("gridSubmitCommand", "bsub");
+ setGlobal("gridHoldOption", "-w \"done\(\"WAIT_TAG\"\)\"");
+ setGlobal("gridSyncOption", "-K");
+ setGlobal("gridNameOption", "-J");
+ setGlobal("gridArrayOption", "");
+ setGlobal("gridArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
+ setGlobal("gridOutputOption", "-o");
+ setGlobal("gridPropagateCommand", "bmodify -w \"done\(\"WAIT_TAG\"\)\"");
+ setGlobal("gridNameToJobIDCommand", "bjobs -J \"WAIT_TAG\" | grep -v JOBID");
+ setGlobal("gridTaskID", "LSB_JOBINDEX");
+ setGlobal("gridArraySubmitID", "%I");
+ setGlobal("gridJobID", "LSB_JOBID");
+ }
+
+ # Update obsolete usage.
+
+ if ($var eq "doOverlapTrimming") {
+ print STDERR "WARNING: option doOverlapTrimming deprecated. Use doOverlapBasedTrimming in the future.\n";
+ $var = "doOverlapBasedTrimming";
+ }
+
+ if ($var eq "ovlMemory") {
+ print STDERR "ERROR: the runCA option ovlMemory is not recognized.\n";
+ print STDERR "This runCA functionality changed in CABOG version 7.\n";
+ print STDERR "Here are suggestions for new runCA option values:\n";
+ print STDERR " ovlHashBits=23\n ovlHashBlockLength= 30000000\n (replaces ovlMemory=2GB)\n";
+ print STDERR " ovlHashBits=24\n ovlHashBlockLength=110000000\n (replaces ovlMemory=4GB)\n";
+ print STDERR " ovlHashBits=25\n ovlHashBlockLength=180000000\n (replaces ovlMemory=8GB)\n";
+ exit(1);
+ }
+
+ if ($var eq "ovlHashBlockSize") {
+ print STDERR "ERROR: the runCA option ovlHashBlockSize is not recognized.\n";
+ print STDERR "This runCA functionality changed in CABOG version 7.\n";
+ print STDERR "Try using ovlHashBlockLength instead.\n";
+ exit(1);
+ }
+
+ # Update aliases.
+
+ $var = "doOverlapBasedTrimming" if ($var eq "doOBT");
+ $var = "doExtendClearRanges" if ($var eq "doECR");
+
+ # If "help" exists, we're parsing command line options, and will catch this failure in
+ # printHelp(). Otherwise, this is an internal error, and we should bomb now.
+ #
+ if (!exists($global{$var})) {
+ if (exists($global{"help"})) {
+ setGlobal("help", getGlobal("help") . "'$var' is not a valid option; see 'runCA -options' for a list of valid options.\n");
+ } else {
+ caFailure("'$var' is not a valid option Global variable.", undef);
+ }
+ }
+
+ $global{$var} = $val;
+}
+
+sub setDefaults () {
+
+ # The rules:
+ #
+ # 1) Before changing these defaults, read the (printed) documentation.
+ # 2) After changing, update the documentation.
+ # 3) Add new defaults in the correct section.
+ # 4) Keep defaults in the same order as the documentation.
+ # 5) UPDATE THE DOCUMENTATION.
+ #
+
+ $global{"overlapStoreOnGrid"} = 0;
+ $synops{"overlapStoreOnGrid"} = "Enable OverlapStore Build on Grid";
+
+
+ ##### General Configuration Options (aka miscellany)
+
+ $global{"showNext"} = undef;
+ $synops{"showNext"} = "Don't run any commands, just report what would run";
+
+ $global{"pathMap"} = undef;
+ $synops{"pathMap"} = "File with a hostname to binary directory map";
+
+ $global{"shell"} = "/bin/sh";
+ $synops{"shell"} = "Command interpreter to use; sh-compatible (e.g., bash), NOT C-shell (csh or tcsh)";
+
+ ##### Error Rates
+
+ $global{"ovlErrorRate"} = 0.06;
+ $synops{"ovlErrorRate"} = "Overlaps above this error rate are not computed";
+
+ $global{"obtErrorRate"} = undef;
+ $synops{"obtErrorRate"} = "Overlaps at or below this error rate are used for Overlap Based Trimming (OBT)";
+
+ $global{"obtErrorLimit"} = undef;
+ $synops{"obtErrorLimit"} = "Overlaps at or below this number of errors are used for Overlap Based Trimming (OBT)";
+
+ $global{"utgErrorRate"} = 0.015;
+ $synops{"utgErrorRate"} = "Overlaps at or below this error rate are used to construct unitigs (BOG and UTG)";
+
+ $global{"utgErrorLimit"} = 2.5;
+ $synops{"utgErrorLimit"} = "Overlaps at or below this number of errors are used to construct unitigs (BOG and UTG)";
+
+ $global{"utgGraphErrorRate"} = 0.030;
+ $synops{"utgGraphErrorRate"} = "Overlaps at or below this error rate are used to construct unitigs (BOGART)";
+
+ $global{"utgGraphErrorLimit"} = 3.25;
+ $synops{"utgGraphErrorLimit"} = "Overlaps at or below this number of errors are used to construct unitigs (BOGART)";
+
+ $global{"utgMergeErrorRate"} = 0.045;
+ $synops{"utgMergeErrorRate"} = "Overlaps at or below this error rate are used to construct unitigs (BOGART)";
+
+ $global{"utgMergeErrorLimit"} = 5.25;
+ $synops{"utgMergeErrorLimit"} = "Overlaps at or below this number of errors are used to construct unitigs (BOGART)";
+
+ $global{"cnsErrorRate"} = 0.06;
+ $synops{"cnsErrorRate"} = "Consensus expects alignments at about this error rate";
+
+ $global{"cgwErrorRate"} = 0.10;
+ $synops{"cgwErrorRate"} = "Unitigs/Contigs are not merged if they align above this error rate";
+
+ ##### Minimums
+
+ $global{"frgMinLen"} = 64;
+ $synops{"frgMinLen"} = "Fragments shorter than this length are not loaded into the assembler";
+
+ $global{"ovlMinLen"} = 40;
+ $synops{"ovlMinLen"} = "Overlaps shorter than this length are not computed";
+
+ ##### Stopping conditions
+
+ $global{"stopBefore"} = undef;
+ $synops{"stopBefore"} = "Tell runCA when to halt execution";
+
+ $global{"stopAfter"} = undef;
+ $synops{"stopAfter"} = "Tell runCA when to halt execution";
+
+ ##### Grid Engine configuration, how to submit jobs, etc
+
+ $global{"gridEngine"} = "SGE";
+ $global{"gridSubmitCommand"} = "qsub";
+ $global{"gridHoldOption"} = "-hold_jid \"WAIT_TAG\""; # for lsf: -w "done("WAIT_TAG")"
+ $global{"gridSyncOption"} = "-sync y"; # for lsf: -K
+ $global{"gridNameOption"} = "-cwd -N"; # for lsf: -J
+ $global{"gridArrayOption"} = "-t ARRAY_JOBS"; # for lsf: empty ("")
+ $global{"gridArrayName"} = "ARRAY_NAME"; # for lsf: ARRAY_NAME[ARRAY_JOBS]
+ $global{"gridOutputOption"} = "-j y -o"; # for lsf: -o
+ $global{"gridPropagateCommand"} = "qalter -hold_jid \"WAIT_TAG\""; # for lsf: bmodify -w "done(WAIT_TAG)"
+ $global{"gridNameToJobIDCommand"} = undef; # for lsf: bjobs -J "WAIT_TAG" | grep -v JOBID
+ $global{"gridTaskID"} = "SGE_TASK_ID"; # for lsf: LSB_JOBINDEX
+ $global{"gridArraySubmitID"} = "\\\$TASK_ID"; # for lsf: %I
+ $global{"gridJobID"} = "JOB_ID"; # for lsf: LSB_JOBID
+
+ ##### Sun Grid Engine
+
+ $global{"useGrid"} = 0;
+ $synops{"useGrid"} = "Enable SGE globally";
+
+ $global{"scriptOnGrid"} = 0;
+ $synops{"scriptOnGrid"} = "Enable SGE for runCA (and unitigger, scaffolder, other sequential phases)";
+
+ $global{"mbtOnGrid"} = 1;
+ $synops{"mbtOnGrid"} = "Enable SGE for mer-based trimming computations";
+
+ $global{"ovlOnGrid"} = 1;
+ $synops{"ovlOnGrid"} = "Enable SGE for overlap computations";
+
+ $global{"frgCorrOnGrid"} = 0;
+ $synops{"frgCorrOnGrid"} = "Enable SGE for the fragment error correction";
+
+ $global{"ovlCorrOnGrid"} = 0;
+ $synops{"ovlCorrOnGrid"} = "Enable SGE for the overlap error correction";
+
+ $global{"cnsOnGrid"} = 1;
+ $synops{"cnsOnGrid"} = "Enable SGE for consensus";
+
+ $global{"sge"} = undef;
+ $synops{"sge"} = "SGE options applied to all SGE jobs";
+
+ $global{"sgeName"} = undef;
+ $synops{"sgeName"} = "SGE jobs name suffix";
+
+ $global{"sgeScript"} = undef;
+ $synops{"sgeScript"} = "SGE options applied to runCA jobs (and unitigger, scaffolder, other sequential phases)";
+
+ $global{"sgeMerTrim"} = undef;
+ $synops{"sgeMerTrim"} = "SGE options applied to mer-based trimming jobs";
+
+ $global{"sgeOverlap"} = undef;
+ $synops{"sgeOverlap"} = "SGE options applied to overlap computation jobs";
+
+ $global{"sgeMerOverlapSeed"} = undef;
+ $synops{"sgeMerOverlapSeed"} = "SGE options applied to mer overlap seed (overmerry) jobs";
+
+ $global{"sgeMerOverlapExtend"} = undef;
+ $synops{"sgeMerOverlapExtend"} = "SGE options applied to mer overlap extend (olap-from-seeds) jobs";
+
+ $global{"sgeConsensus"} = undef;
+ $synops{"sgeConsensus"} = "SGE options applied to consensus jobs";
+
+ $global{"sgeFragmentCorrection"} = undef;
+ $synops{"sgeFragmentCorrection"} = "SGE options applied to fragment error correction jobs";
+
+ $global{"sgeOverlapCorrection"} = undef;
+ $synops{"sgeOverlapCorrection"} = "SGE options applied to overlap error correction jobs";
+
+ $global{"sgePropagateHold"} = undef;
+ $synops{"sgePropagateHold"} = undef; # Internal option
+
+ ##### Preoverlap
+
+ $global{"gkpFixInsertSizes"} = 1;
+ $synops{"gkpFixInsertSizes"} = "Update stddev to 0.10 * mean if it is too large";
+
+ $global{"gkpAllowInefficientStorage"} = 0;
+ $synops{"gkpAllowInefficientStorage"} = "Allow mis-ordered reads in gkpStore; storage is inefficient and memory consuming";
+
+ ##### Vector Trimming
+
+ $global{"vectorIntersect"} = undef;
+ $synops{"vectorIntersect"} = "File of vector clear ranges";
+
+ $global{"vectorTrimmer"} = "ca";
+ $synops{"vectorTrimmer"} = "Use the CA default vector trimmer, or figaro";
+
+ $global{"figaroFlags"} = "-T 30 -M 100 -E 500 -V f";
+ $synops{"figaroFlags"} = "Options to the figaro vector trimmer";
+
+ ##### Overlap Based Trimming
+
+ $global{"doOverlapBasedTrimming"} = 1;
+ $synops{"doOverlapBasedTrimming"} = "Enable the Overlap Based Trimming module (doOBT and doOverlapTrimming are aliases)";
+
+ $global{"doDeDuplication"} = 1;
+ $synops{"doDeDuplication"} = "Enable the OBT duplication detection and cleaning module for 454 reads, enabled automatically";
+
+ $global{"doChimeraDetection"} = "normal";
+ $synops{"doChimeraDetection"} = "Enable the OBT chimera detection and cleaning module; 'off', 'normal' or 'aggressive'";
+
+ ##### Mer Based Trimming
+
+ $global{"mbtBatchSize"} = 1000000;
+ $synops{"mbtBatchSize"} = "Process this many fragments per merTrim batch";
+
+ $global{"mbtThreads"} = 4;
+ $synops{"mbtThreads"} = "Number of threads to use when computing mer-based trimming";
+
+ $global{"mbtConcurrency"} = 1;
+ $synops{"mbtConcurrency"} = "If not SGE, number of mer-based trimming processes to run at the same time";
+
+ $global{"mbtIlluminaAdapter"} = 0;
+ $synops{"mbtIlluminaAdapter"} = "Remove Illumina adapter sequence during merTrim";
+
+ $global{"mbt454Adapter"} = 0;
+ $synops{"mbt454Adapter"} = "Remove 454 adapter sequence during merTrim";
+
+ ##### Overlapper
+
+ $global{"obtOverlapper"} = "ovl";
+ $synops{"obtOverlapper"} = "Which overlap algorithm to use for OBT overlaps";
+
+ $global{"ovlOverlapper"} = "ovl";
+ $synops{"ovlOverlapper"} = "Which overlap algorithm to use for OVL (unitigger) overlaps";
+
+ $global{"ovlStoreMemory"} = 1024;
+ $synops{"ovlStoreMemory"} = "How much memory (MB) to use when constructing overlap stores";
+
+ $global{"ovlThreads"} = 2;
+ $synops{"ovlThreads"} = "Number of threads to use when computing overlaps";
+
+ $global{"ovlConcurrency"} = 1;
+ $synops{"ovlConcurrency"} = "If not SGE, number of overlapper processes to run at the same time";
+
+ $global{"ovlHashBlockLength"} = 100000000;
+ $synops{"ovlHashBlockLength"} = "Amount of sequence (bp) to load into the overlap hash table";
+
+ $global{"ovlRefBlockSize"} = 2000000;
+ $synops{"ovlRefBlockSize"} = "Number of fragments to search against the hash table per batch";
+
+ $global{"ovlRefBlockLength"} = 0;
+ $synops{"ovlRefBlockLength"} = "Amount of sequence (bp) to search against the hash table per batch";
+
+ $global{"ovlHashBits"} = "22";
+ $synops{"ovlHashBits"} = "Width of the kmer hash. Width 22=1gb, 23=2gb, 24=4gb, 25=8gb. Plus 10b per ovlHashBlockLength";
+
+ $global{"ovlHashLoad"} = "0.75";
+ $synops{"ovlHashLoad"} = "Maximum hash table load. If set too high, table lookups are inefficent; if too low, search overhead dominates run time";
+
+ $global{"ovlMerSize"} = 22;
+ $synops{"ovlMerSize"} = "K-mer size for seeds in overlaps";
+
+ $global{"ovlMerThreshold"} = undef;
+ $synops{"ovlMerThreshold"} = "K-mer frequency threshold; mers more frequent than this count are ignored";
+
+ $global{"ovlMerDistinct"} = undef;
+ $synops{"ovlMerDistinct"} = "K-mer frequency threshold; the least frequent fraction of distinct mers can seed overlaps";
+
+ $global{"ovlMerTotal"} = undef;
+ $synops{"ovlMerTotal"} = "K-mer frequency threshold; the least frequent fraction of all mers can seed overlaps";
+
+ $global{"ovlFrequentMers"} = undef;
+ $synops{"ovlFrequentMers"} = "Do not seed overlaps with these kmers (fasta format)";
+
+ $global{"obtMerSize"} = 22;
+ $synops{"obtMerSize"} = "K-mer size";
+
+ $global{"obtMerThreshold"} = undef;
+ $synops{"obtMerThreshold"} = "K-mer frequency threshold; mers more frequent than this are ignored";
+
+ $global{"obtMerDistinct"} = undef;
+ $synops{"obtMerDistinct"} = "K-mer frequency threshold; the least frequent fraction of distinct mers can seed overlaps";
+
+ $global{"obtMerTotal"} = undef;
+ $synops{"obtMerTotal"} = "K-mer frequency threshold; the least frequent fraction of all mers can seed overlaps";
+
+ $global{"obtFrequentMers"} = undef;
+ $synops{"obtFrequentMers"} = "Do not seed overlaps with these kmers (fasta format)";
+
+ $global{"ovlHashLibrary"} = "0";
+ $synops{"ovlHashLibrary"} = "For ovl overlaps, only load hash fragments from specified lib, 0 means all";
+
+ $global{"ovlRefLibrary"} = "0";
+ $synops{"ovlRefLibrary"} = "For ovl overlaps, only load ref fragments from specified lib, 0 means all";
+
+ $global{"obtHashLibrary"} = "0";
+ $synops{"obtHashLibrary"} = "For obt overlaps, only load hash fragments from specified lib, 0 means all";
+
+ $global{"obtRefLibrary"} = "0";
+ $synops{"obtRefLibrary"} = "For obt overlaps, only load ref fragments from specified lib, 0 means all";
+
+ $global{"obtCheckLibrary"} = 1;
+ $synops{"obtCheckLibrary"} = "Check that all libraries are used during obt overlaps";
+
+ $global{"ovlCheckLibrary"} = 1;
+ $synops{"ovlCheckLibrary"} = "Check that all libraries are used during ovl overlaps";
+
+
+ $global{"merCompression"} = 1;
+ $synops{"merCompression"} = "K-mer size";
+
+ $global{"merOverlapperThreads"} = 2;
+ $synops{"merOverlapperThreads"} = "Number of threads to use for both mer overlapper seed finding and extension jobs";
+
+ $global{"merOverlapperSeedBatchSize"} = 100000;
+ $synops{"merOverlapperSeedBatchSize"} = "Number of fragments in a mer overlapper seed finding batch; directly affects memory usage";
+
+ $global{"merOverlapperExtendBatchSize"}= 75000;
+ $synops{"merOverlapperExtendBatchSize"}= "Number of fragments in a mer overlapper seed extension batch; directly affects memory usage";
+
+ $global{"merOverlapperCorrelatedDiffs"}= 0;
+ $synops{"merOverlapperCorrelatedDiffs"}= "EXPERIMENTAL!";
+
+ $global{"merOverlapperSeedConcurrency"}= 1;
+ $synops{"merOverlapperSeedConcurrency"}= "If not SGE, number of mer overlapper seed finding processes to run at the same time";
+
+ $global{"merOverlapperExtendConcurrency"}= 1;
+ $synops{"merOverlapperExtendConcurrency"}= "If not SGE, number of mer overlapper seed extension processes to run at the same time";
+
+ $global{"umdOverlapperFlags"} = "-use-uncleaned-reads -trim-error-rate 0.03 -max-minimizer-cutoff 150";
+ $synops{"umdOverlapperFlags"} = "Options for the UMD overlapper";
+
+ $global{"saveOverlaps"} = 0;
+ $synops{"saveOverlaps"} = "Save intermediate overlap files";
+
+ ##### Mers
+
+ $global{"merylMemory"} = 800;
+ $synops{"merylMemory"} = "Amount of memory, in MB, to use for mer counting";
+
+ $global{"merylThreads"} = 1;
+ $synops{"merylThreads"} = "Number of threads to use for mer counting";
+
+ ##### Fragment/Overlap Error Correction
+
+ $global{"frgCorrBatchSize"} = 200000;
+ $synops{"frgCorrBatchSize"} = "Number of fragments per fragment error detection batch, directly affects memory usage";
+
+ $global{"doFragmentCorrection"} = 1;
+ $synops{"doFragmentCorrection"} = "Do overlap error correction";
+
+ $global{"frgCorrThreads"} = 2;
+ $synops{"frgCorrThreads"} = "Number of threads to use while computing fragment errors";
+
+ $global{"frgCorrConcurrency"} = 1;
+ $synops{"frgCorrConcurrency"} = "If not SGE, number of fragment error detection processes to run at the same time";
+
+ $global{"ovlCorrBatchSize"} = 200000;
+ $synops{"ovlCorrBatchSize"} = "Number of fragments per overlap error correction batch";
+
+ $global{"ovlCorrConcurrency"} = 4;
+ $synops{"ovlCorrConcurrency"} = "If not SGE, number of overlap error correction processes to run at the same time";
+
+ ##### Illumina MP Classification
+
+ $global{"dncMPlibraries"} = undef;
+ $synops{"dncMPlibraries"} = "List of library names to run the 'de novo classifier' on";
+
+ $global{"dncBBlibraries"} = undef;
+ $synops{"dncBBlibraries"} = "List of library names to use as the bacbone in the 'de novo classifier'";
+
+ ##### Unitigger & BOG & bogart Options
+
+ $global{"unitigger"} = undef;
+ $synops{"unitigger"} = "Which unitig algorithm to use; utg (if no SFF files) or bog (Best Overlap Graph, if SFF files)";
+
+ $global{"utgGenomeSize"} = 0;
+ $synops{"utgGenomeSize"} = "An estimate of the size of the genome; decides if unitigs are unique or repeats";
+
+ $global{"utgBubblePopping"} = 1;
+ $synops{"utgBubblePopping"} = "Smooth polymorphic regions";
+
+ $global{"utgRecalibrateGAR"} = 1;
+ $synops{"utgRecalibrateGAR"} = "Use an experimental algorithm to decide unique/repeat";
+
+ $global{"bogBreakAtIntersections"} = 1;
+ $synops{"bogBreakAtIntersections"} = "EXPERT!";
+
+ $global{"bogBadMateDepth"} = 7;
+ $synops{"bogBadMateDepth"} = "EXPERT!";
+
+ $global{"batRebuildRepeats"} = 0;
+ $synops{"batRebuildRepeats"} = "Shatter repeats, rebuild more stringently";
+
+ $global{"batMateExtension"} = 0;
+ $synops{"batMateExtension"} = "Shatter repeats, extend unique unitigs with mates (leaves repeats shattered)";
+
+ $global{"batMemory"} = undef;
+ $synops{"batMemory"} = "Approximate maximum memory usage for loading overlaps, in gigabytes, default is unlimited";
+
+ $global{"batThreads"} = undef;
+ $synops{"batThreads"} = "Number of threads to use in the Merge/Split/Join phase; default is whatever OpenMP wants";
+
+ $global{"doUnitigSplitting"} = 1;
+ $synops{"doUnitigSplitting"} = "Split unitigs based on low coverage and high bad mate evidence";
+
+ ##### Scaffolder Options
+
+ $global{"cgwPurgeCheckpoints"} = 1;
+ $synops{"cgwPurgeCheckpoints"} = "Remove cgw checkpoint files when a scaffolding step finishes successfully";
+
+ $global{"cgwCompressTigStore"} = 0;
+ $synops{"cgwCompressTigStore"} = "Remove tigStore versions when a scaffolding step finishes successfully";
+
+ $global{"cgwDemoteRBP"} = 1;
+ $synops{"cgwDemoteRBP"} = "EXPERT!";
+
+ $global{"cgwUseUnitigOverlaps"} = 0;
+ $synops{"cgwUseUnitigOverlaps"} = "Use unused best overlaps (from BOG) in scaffolder (EXPERIMENTAL)";
+
+ $global{"cgwReloadMates"} = 0;
+ $synops{"cgwReloadMates"} = "Load new mate pairs from gkpStore after ckp is loaded (EXPERIMENTAL)";
+
+ $global{"astatLowBound"} = 1;
+ $synops{"astatLowBound"} = "EXPERT!";
+
+ $global{"astatHighBound"} = 5;
+ $synops{"astatHighBound"} = "EXPERT!";
+
+ $global{"stoneLevel"} = 2;
+ $synops{"stoneLevel"} = "EXPERT!";
+
+ $global{"cgwMergeFilterLevel"} = 1;
+ $synops{"cgwMergeFilterLevel"} = "Apply no (0), classic (1, default), suggested (2), or stringent (5) criteria before considering a scaffold merge";
+
+ $global{"computeInsertSize"} = undef;
+ $synops{"computeInsertSize"} = "Compute a scratch scaffolding to estimate insert sizes; default: do only if less than 1 million reads";
+
+ $global{"cgwDistanceSampleSize"} = 100;
+ $synops{"cgwDistanceSampleSize"} = "Require N mates to reestimate insert sizes";
+
+ $global{"doResolveSurrogates"} = 1;
+ $synops{"doResolveSurrogates"} = "Place fragments in surrogates in the final assembly";
+
+ $global{"doExtendClearRanges"} = 2;
+ $synops{"doExtendClearRanges"} = "Enable the clear range extension heuristic";
+
+ $global{"extendClearRangesStepSize"} = undef;
+ $synops{"extendClearRangesStepSize"} = "Batch N scaffolds per ECR run";
+
+ $global{"kickOutNonOvlContigs"} = 0;
+ $synops{"kickOutNonOvlContigs"} = "Allow kicking out a contig placed in a scaffold by mate pairs that has no overlaps to both its left and right neighbor contigs. EXPERT!\n";
+
+ $global{"doUnjiggleWhenMerging"} = 0;
+ $synops{"doUnjiggleWhenMerging"} = "After inserting rocks/stones try shifting contig positions back to their original location when computing overlaps to see if they overlap with the rock/stone and allow them to merge if they do. EXPERT!\n";
+
+ $global{"cgwContigShatterWeight"} = 0;
+ $synops{"cgwContigShatterWeight"} = "When starting from a checkpoint, for any contig connected to its scaffold by a link with less than cgwContigShatterWeight, remove it and place it into a singleton scaffold. EXPERT!\n";
+
+ $global{"cgwMergeMissingThreshold"} = 0;
+ $synops{"cgwMergeMissingThreshold"} = "When merging scaffolds, missing mates are those mates that should fall within the merged scaffold but do not. In metagenomics, this may not be the case for a conserved region within strains as the mates are missing because they are in a different strain. This is a value between 0 and 1 to specify the percentage of missing mates (relative to good mates) to ignore. A value of -1 means ignore all missing mates when merging. EXPERT!\n";
+
+ $global{"cgwMinMergeWeight"} = 2;
+ $synops{"cgwMinMergeWeight"} = "When merging scaffolds, do not use edges with weight below this.\n";
+
+ $global{"cgwPreserveConsensus"} = 0;
+ $synops{"cgwPreserveConsensus"} = "Do not remove the contig consensus sequence at the end of scaffolder; ctgcns will be skipped (faster) but quality will be lower.\n";
+
+ ##### Consensus Options
+
+ $global{"cnsPartitions"} = 128;
+ $synops{"cnsPartitions"} = "Partition consensus into N jobs";
+
+ $global{"cnsMinFrags"} = 75000;
+ $synops{"cnsMinFrags"} = "Don't make a consensus partition with fewer than N fragments";
+
+ $global{"cnsConcurrency"} = 2;
+ $synops{"cnsConcurrency"} = "If not SGE, number of consensus jobs to run at the same time";
+
+ $global{"cnsPhasing"} = 0;
+ $synops{"cnsPhasing"} = "Options for consensus phasing of SNPs\n\t0 - Do not phase SNPs to be consistent.\n\t1 - If two SNPs are joined by reads, phase them to be consistent.";
+
+ $global{"cnsReduceUnitigs"} = undef;
+ $synops{"cnsReduceUnitigs"} = "Set parameters for when and how to sample down coverage in deep unitigs.";
+
+ $global{"cnsReuseUnitigs"} = 0;
+ $synops{"cnsReuseUnitigs"} = "Do not compute single-unitig contigs again, just reuse the unitig.";
+
+ #$global{"cnsRecycleUnitigs"} = 0;
+ #$synops{"cnsRecycleUnitigs"} = "At some point, we'll come up with something for this.";
+
+ $global{"consensus"} = "cns";
+ $synops{"consensus"} = "Which consensus algorithm to use; currently only 'cns' is supported";
+
+ ##### Terminator Options
+
+ $global{"fakeUIDs"} = 0;
+ $synops{"fakeUIDs"} = "Don't query a UID server, use UIDs specific to this assembly";
+
+ $global{"uidServer"} = undef;
+ $synops{"uidServer"} = "EXPERT!";
+
+ $global{"createAGP"} = 0;
+ $synops{"createAGP"} = "Create an AGP file for the assembly";
+
+ $global{"createACE"} = 0;
+ $synops{"createACE"} = "Create an ACE file for the assembly";
+
+ $global{"createPosMap"} = 1;
+ $synops{"createPosMap"} = "Create the POSMAP files for the assembly";
+
+ $global{"merQC"} = 0;
+ $synops{"merQC"} = "Compute a mer-based QC for the assembly";
+
+ $global{"merQCmemory"} = 1024;
+ $synops{"merQCmemory"} = "Memory to use for the mer-based QC";
+
+ $global{"merQCmerSize"} = 22;
+ $synops{"merQCmerSize"} = "Mer size to use for the mer-based QC";
+
+ $global{"cleanup"} = "none";
+ $synops{"cleanup"} = "At the end of a successful assembly, remove none/some/many/all of the intermediate files";
+
+ $global{"dumpFASTQ"} = undef;
+ $synops{"dumpFASTQ"} = "At the end of a successful assembly, output final reads in FASTQ format";
+
+ ##### Options for toggling assembly.
+
+ $global{"doToggle"} = 0;
+ $synops{"doToggle"} = "At the end of a successful assembly, search for placed surrogates and toggle them to be unique unitigs. Re-run the assembly starting from scaffolder";
+
+ $global{"toggleUnitigLength"} = 2000;
+ $synops{"toggleUnitigLength"} = "Minimum length for a surrogate to be toggled.";
+
+ $global{"toggleNumInstances"} = 1;
+ $synops{"toggleNumInstances"} = "Number of instances for a surrogate to be toggled. If 0 is specified, all non-singleton unitigs are toggled to unique status.";
+
+ $global{"toggleMaxDistance"} = 1000;
+ $synops{"toggleMaxDistance"} = "Toggling will look for surrogates that appear exactly twice, both at the end of a scaffold. This parameter specifies how close to the scaffold end the surrogate must be.";
+
+ $global{"toggleDoNotDemote"} = 0;
+ $synops{"toggleDoNotDemote"} = "Do not allow CGW to demote toggled unitigs based on branching patterns.";
+
+ #### Closure Options
+
+ $global{"closureOverlaps"} = undef;
+ $synops{"closureOverlaps"} = "Option for handling overlaps involving closure reads.\n\t0 - Treat them just like regular reads, \n\t1 - Do not allow any overlaps (i.e. closure reads will stay as singletons until scaffolding), \n\t2 - allow overlaps betweeen closure reads and non-closure reads only";
+
+ $global{"closurePlacement"} = 2;
+ $synops{"closurePlacement"} = "Option for placing closure reads using the constraints.\n\t0 - Place at the first location found\n\t1 - Place at the best location (indicated by most constraints)\n\t2 - Place at multiple locations as long as the closure read/unitig in question is not unique";
+
+ ##### Ugly, command line options passed to printHelp()
+
+ $global{"help"} = "";
+ $synops{"help"} = undef;
+
+ $global{"version"} = 0;
+ $synops{"version"} = undef;
+
+ $global{"options"} = 0;
+ $synops{"options"} = undef;
+
+
+ # If this is set, it breaks the consensus.sh and overlap.sh scripts. Good grief! Why
+ # are you running runCA in a task array!?
+ #
+ if (exists($ENV{$global{"gridTaskID"}})) {
+ undef $ENV{$global{"gridTaskID"}};
+ print STDERR "ENV: $global{'gridTaskID'} needs to be unset, done.\n";
+ }
+
+
+ if (exists($ENV{'AS_OVL_ERROR_RATE'})) {
+ setGlobal("ovlErrorRate", $ENV{'AS_OVL_ERROR_RATE'});
+ print STDERR "ENV: ovlErrorRate $ENV{'AS_OVL_ERROR_RATE'} (from \$AS_OVL_ERROR_RATE)\n";
+ }
+
+ if (exists($ENV{'AS_CGW_ERROR_RATE'})) {
+ setGlobal("cgwErrorRate", $ENV{'AS_CGW_ERROR_RATE'});
+ print STDERR "ENV: cgwErrorRate $ENV{'AS_CGW_ERROR_RATE'} (from \$AS_CGW_ERROR_RATE)\n";
+ }
+
+ if (exists($ENV{'AS_CNS_ERROR_RATE'})) {
+ setGlobal("cnsErrorRate", $ENV{'AS_CNS_ERROR_RATE'});
+ print STDERR "ENV: cnsErrorRate $ENV{'AS_CNS_ERROR_RATE'} (from \$AS_CNS_ERROR_RATE)\n";
+ }
+
+ if (exists($ENV{'AS_READ_MIN_LEN'})) {
+ setGlobal("frgMinLen", $ENV{'AS_READ_MIN_LEN'});
+ print STDERR "ENV: frgMinLen $ENV{'AS_READ_MIN_LEN'} (from \$AS_READ_MIN_LEN)\n";
+ }
+
+ if (exists($ENV{'AS_OVERLAP_MIN_LEN'})) {
+ setGlobal("ovlMinLen", $ENV{'AS_OVERLAP_MIN_LEN'});
+ print STDERR "ENV: ovlMinLen $ENV{'AS_OVERLAP_MIN_LEN'} (from \$AS_OVERLAP_MIN_LEN)\n";
+ }
+}
+
+sub makeAbsolute ($) {
+ my $var = shift @_;
+ my $val = getGlobal($var);
+ if (defined($val) && ($val !~ m!^/!)) {
+ $val = "$ENV{'PWD'}/$val";
+ setGlobal($var, $val);
+ $val =~ s/\\\"/\"/g;
+ $val =~ s/\"/\\\"/g;
+ $val =~ s/\\\$/\$/g;
+ $val =~ s/\$/\\\$/g;
+ $commandLineOptions .= " \"$var=$val\" ";
+ }
+}
+
+sub fixCase ($) {
+ my $var = shift @_;
+ my $val = getGlobal($var);
+
+ if (defined($val)) {
+ $val =~ tr/A-Z/a-z/;
+ setGlobal($var, $val);
+ }
+}
+
+sub setParametersFromFile ($@) {
+ my $specFile = shift @_;
+ my @fragFiles = @_;
+
+ # Client should be ensuring that the file exists before calling this function.
+ die "specFile '$specFile' not found.\n" if (! -e "$specFile");
+
+ $specLog .= "\n";
+ $specLog .= "###\n";
+ $specLog .= "### Reading options from '$specFile'\n";
+ $specLog .= "###\n";
+ $specLog .= "\n";
+
+ open(F, "< $specFile") or caFailure("Couldn't open '$specFile'", undef);
+
+ while (<F>) {
+ $specLog .= $_;
+
+ s/^\s+//;
+ s/\s+$//;
+
+ next if (m/^\s*\#/);
+ next if (m/^\s*$/);
+
+ if (-e $_) {
+ my $xx = $_;
+ $xx = "$ENV{'PWD'}/$xx" if ($xx !~ m!^/!);
+ if (-e $xx) {
+ push @fragFiles, $xx;
+ } else {
+ setGlobal("help", getGlobal("help") . "File not found '$_' after appending absolute path.\n");
+ }
+ } elsif (m/\s*(\w*)\s*=([^#]*)#*.*$/) {
+ my ($var, $val) = ($1, $2);
+ $var =~ s/^\s+//; $var =~ s/\s+$//;
+ $val =~ s/^\s+//; $val =~ s/\s+$//;
+ undef $val if ($val eq "undef");
+ setGlobal($var, $val);
+ } else {
+ setGlobal("help", getGlobal("help") . "File not found or unknown specFile option line '$_'.\n");
+ }
+ }
+ close(F);
+
+ return(@fragFiles);
+}
+
+
+sub setParametersFromCommandLine(@) {
+ my @specOpts = @_;
+
+ if (scalar(@specOpts) > 0) {
+ $specLog .= "\n";
+ $specLog .= "###\n";
+ $specLog .= "### Reading options from the command line.\n";
+ $specLog .= "###\n";
+ $specLog .= "\n";
+ }
+
+ foreach my $s (@specOpts) {
+ $specLog .= "$s\n";
+
+ if ($s =~ m/\s*(\w*)\s*=(.*)/) {
+ my ($var, $val) = ($1, $2);
+ $var =~ s/^\s+//; $var =~ s/\s+$//;
+ $val =~ s/^\s+//; $val =~ s/\s+$//;
+ setGlobal($var, $val);
+ } else {
+ setGlobal("help", getGlobal("help") . "Misformed command line option '$s'.\n");
+ }
+ }
+}
+
+
+sub setParameters () {
+
+ # Update obsolete usages.
+ #
+ if (getGlobal("doChimeraDetection") eq "1") {
+ print STDERR "WARNING: 'doChimeraDetection=1' is obsolete; use 'doChimeraDetection=normal' in the future.\n";
+ setGlobal("doChimeraDetection", "normal");
+ }
+ if (getGlobal("doChimeraDetection") eq "0") {
+ print STDERR "WARNING: 'doChimeraDetection=0' is obsolete; use 'doChimeraDetection=off' in the future.\n";
+ setGlobal("doChimeraDetection", "off");
+ }
+
+ # Fiddle with filenames to make them absolute paths.
+ #
+ makeAbsolute("vectorIntersect");
+ makeAbsolute("pathMap");
+
+ makeAbsolute("obtFrequentMers");
+ makeAbsolute("ovlFrequentMers");
+
+ # Adjust case on some of them
+ #
+ fixCase("doChimeraDetection");
+ fixCase("obtOverlapper");
+ fixCase("ovlOverlapper");
+ fixCase("unitigger");
+ fixCase("vectorTrimmer");
+ fixCase("stopBefore");
+ fixCase("stopAfter");
+ fixCase("consensus");
+ fixCase("cleanup");
+
+ if ((getGlobal("doChimeraDetection") ne "off") && (getGlobal("doChimeraDetection") ne "normal") && (getGlobal("doChimeraDetection") ne "aggressive")) {
+ caFailure("invalid doChimeraDetection specified (" . getGlobal("doChimeraDetection") . "); must be 'off', 'normal', or 'aggressive'", undef);
+ }
+ if ((getGlobal("obtOverlapper") ne "mer") && (getGlobal("obtOverlapper") ne "ovl") && (getGlobal("obtOverlapper") ne "ovm")) {
+ caFailure("invalid obtOverlapper specified (" . getGlobal("obtOverlapper") . "); must be 'mer' or 'ovl' (or DEVEL_ONLY 'ovm')", undef);
+ }
+ if ((getGlobal("ovlOverlapper") ne "mer") && (getGlobal("ovlOverlapper") ne "ovl") && (getGlobal("ovlOverlapper") ne "ovm")) {
+ caFailure("invalid ovlOverlapper specified (" . getGlobal("ovlOverlapper") . "); must be 'mer' or 'ovl' (or DEVEL_ONLY 'ovm')", undef);
+ }
+ if (defined(getGlobal("unitigger")) && (getGlobal("unitigger") ne "utg") && (getGlobal("unitigger") ne "bog") && (getGlobal("unitigger") ne "bogart")) {
+ caFailure("invalid unitigger specified (" . getGlobal("unitigger") . "); must be 'utg' or 'bog' or 'bogart'", undef);
+ }
+ if ((getGlobal("vectorTrimmer") ne "ca") && (getGlobal("vectorTrimmer") ne "figaro")) {
+ caFailure("invalid vectorTrimmer specified (" . getGlobal("vectorTrimmer") . "); must be 'ca' or 'figaro'", undef);
+ }
+ if ((getGlobal("consensus") ne "cns") && (getGlobal("consensus") ne "seqan")) {
+ caFailure("invalid consensus specified (" . getGlobal("consensus") . "); must be 'cns' or 'seqan'", undef);
+ }
+ if ((getGlobal("cnsPhasing") ne "0") && (getGlobal("cnsPhasing") ne "1")) {
+ caFailure("invalid cnsPhasing specified (" . getGlobal("cnsPhasing") . "); must be '0' or '1'", undef);
+ }
+ if ((getGlobal("cleanup") ne "none") &&
+ (getGlobal("cleanup") ne "light") &&
+ (getGlobal("cleanup") ne "heavy") &&
+ (getGlobal("cleanup") ne "aggressive")) {
+ caFailure("invalid cleaup specified (" . getGlobal("cleanup") . "); must be 'none', 'light', 'heavy' or 'aggressive'", undef);
+ }
+
+ if (defined(getGlobal("stopBefore"))) {
+ my $ok = 0;
+ my $st = getGlobal("stopBefore");
+ $st =~ tr/A-Z/a-z/;
+
+ my $failureString = "Invalid stopBefore specified (" . getGlobal("stopBefore") . "); must be one of:\n";
+
+ my @stopBefore = ("meryl",
+ "initialTrim",
+ "deDuplication",
+ "finalTrimming",
+ "chimeraDetection",
+ "classifyMates",
+ "unitigger",
+ "scaffolder",
+ "CGW",
+ "eCR",
+ "extendClearRanges",
+ "eCRPartition",
+ "extendClearRangesPartition",
+ "terminator");
+
+ foreach my $sb (@stopBefore) {
+ $failureString .= " '$sb'\n";
+ $sb =~ tr/A-Z/a-z/;
+ if ($st eq $sb) {
+ $ok++;
+ setGlobal('stopBefore', $st);
+ }
+ }
+
+ caFailure($failureString, undef) if ($ok == 0);
+ }
+
+
+ if (defined(getGlobal("stopAfter"))) {
+ my $ok = 0;
+ my $st = getGlobal("stopAfter");
+ $st =~ tr/A-Z/a-z/;
+
+ my $failureString = "Invalid stopAfter specified (" . getGlobal("stopAfter") . "); must be one of:\n";
+
+ my @stopAfter = ("initialStoreBuilding",
+ "meryl",
+ "overlapper",
+ "OBT",
+ "overlapBasedTrimming",
+ "unitigger",
+ "classifyMates",
+ "utgcns",
+ "consensusAfterUnitigger",
+ "scaffolder",
+ "ctgcns",
+ "consensusAfterScaffolder");
+
+ foreach my $sa (@stopAfter) {
+ $failureString .= " '$sa'\n";
+ $sa =~ tr/A-Z/a-z/;
+ if ($st eq $sa) {
+ $ok++;
+ setGlobal('stopAfter', $st);
+ }
+ }
+
+ caFailure($failureString, undef) if ($ok == 0);
+ }
+
+
+ # PIck a nice looking set of binaries, and check them.
+ #
+ {
+ caFailure("can't find 'gatekeeper' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/gatekeeper");
+ caFailure("can't find 'meryl' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/meryl");
+ caFailure("can't find 'overlap' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/overlapInCore");
+ caFailure("can't find 'unitigger' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/unitigger");
+ caFailure("can't find 'cgw' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/cgw");
+ caFailure("can't find 'utgcns' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/utgcns");
+ caFailure("can't find 'ctgcns' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/ctgcns");
+ caFailure("can't find 'terminator' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/terminator");
+
+ if ((getGlobal("obtOverlapper") eq "mer") || (getGlobal("ovlOverlapper") eq "mer")) {
+ caFailure("can't find 'overmerry' program in $bin. Possibly incomplete installation", undef) if (! -x "$bin/overmerry");
+ }
+ }
+
+ # Set the globally accessible error rates. Adjust them if they
+ # look strange.
+ #
+ # We must have: ovl <= cns <= cgw
+ # We usually have: ovl == cns <= cgw
+ #
+ my $ovlER = getGlobal("ovlErrorRate");
+ my $utgER = getGlobal("utgErrorRate");
+ my $cgwER = getGlobal("cgwErrorRate");
+ my $cnsER = getGlobal("cnsErrorRate");
+
+ if (($ovlER < 0.0) || (0.40 < $ovlER)) {
+ caFailure("ovlErrorRate is $ovlER, this MUST be between 0.00 and 0.40", undef);
+ }
+ if (($utgER < 0.0) || (0.40 < $utgER)) {
+ caFailure("utgErrorRate is $utgER, this MUST be between 0.00 and 0.40", undef);
+ }
+ if (($cgwER < 0.0) || (0.40 < $cgwER)) {
+ caFailure("cgwErrorRate is $cgwER, this MUST be between 0.00 and 0.40", undef);
+ }
+ if (($cnsER < 0.0) || (0.40 < $cnsER)) {
+ caFailure("cnsErrorRate is $cnsER, this MUST be between 0.00 and 0.40", undef);
+ }
+ if ($utgER > $ovlER) {
+ caFailure("utgErrorRate is $utgER, this MUST be <= ovlErrorRate ($ovlER)", undef);
+ }
+ if ($ovlER > $cnsER) {
+ caFailure("ovlErrorRate is $ovlER, this MUST be <= cnsErrorRate ($cnsER)", undef);
+ }
+ if ($ovlER > $cgwER) {
+ caFailure("ovlErrorRate is $ovlER, this MUST be <= cgwErrorRate ($cgwER)", undef);
+ }
+ if ($cnsER > $cgwER) {
+ caFailure("cnsErrorRate is $cnsER, this MUST be <= cgwErrorRate ($cgwER)", undef);
+ }
+
+ $ENV{'AS_OVL_ERROR_RATE'} = $ovlER;
+ $ENV{'AS_CGW_ERROR_RATE'} = $cgwER;
+ $ENV{'AS_CNS_ERROR_RATE'} = $cnsER;
+
+ $ENV{'AS_READ_MIN_LEN'} = getGlobal("frgMinLen");
+ $ENV{'AS_OVERLAP_MIN_LEN'} = getGlobal("ovlMinLen");
+}
+
+sub logVersion() {
+ system("$bin/gatekeeper --version");
+ system("$bin/overlap --version");
+ system("$bin/unitigger --version");
+ system("$bin/buildUnitigs --version");
+ system("$bin/cgw --version");
+ system("$bin/consensus --version");
+ system("$bin/terminator --version");
+}
+
+sub printHelp () {
+
+ if (getGlobal("version")) {
+ logVersion();
+ exit(0);
+ }
+
+ if (getGlobal("options")) {
+ foreach my $k (sort keys %global) {
+ my $o = substr("$k ", 0, 35);
+ my $d = substr(getGlobal($k) . " ", 0, 20);
+ my $u = $synops{$k};
+
+ if (!defined(getGlobal($k))) {
+ $d = substr("<unset> ", 0, 20);
+ }
+
+ print "$o$d($u)\n";
+ }
+ exit(0);
+ }
+
+ if (getGlobal("help") ne "") {
+ print "usage: runCA -d <dir> -p <prefix> [options] <frg> ...\n";
+ print " -d <dir> Use <dir> as the working directory. Required\n";
+ print " -p <prefix> Use <prefix> as the output prefix. Required\n";
+ print "\n";
+ print " -s <specFile> Read options from the specifications file <specfile>.\n";
+ print " <specfile> can also be one of the following key words:\n";
+ print " [no]OBT - run with[out] OBT\n";
+ print " noVec - run with OBT but without Vector\n";
+ print "\n";
+ print " -version Version information\n";
+ print " -help This information\n";
+ print " -options Describe specFile options, and show default values\n";
+ print "\n";
+ print " <frg> CA formatted fragment file\n";
+ print "\n";
+ print "Complete documentation at http://wgs-assembler.sourceforge.net/\n";
+ print "\n";
+ print $global{"help"};
+ exit(0);
+ }
+
+ undef $global{"version"};
+ undef $global{"options"};
+ undef $global{"help"};
+}
+
+
+
+sub checkDirectories () {
+
+ # Check that we were supplied a work directory, and that it
+ # exists, or we can create it.
+ #
+ die "ERROR: I need a directory to run the assembly in (-d option).\n" if (!defined($wrk));
+
+ system("mkdir -p $wrk") if (! -d $wrk);
+ chmod 0755, "$wrk";
+
+ system("mkdir -p $wrk/runCA-logs") if (! -d "$wrk/runCA-logs");
+ chmod 0755, "$wrk/runCA-logs";
+
+ $ENV{'AS_RUNCA_DIRECTORY'} = $wrk;
+
+ caFailure("directory '$wrk' doesn't exist (-d option) and couldn't be created", undef) if (! -d $wrk);
+}
+
+
+sub outputSpecLog () {
+ my $time = time();
+ my $host = hostname();
+ my $pid = $$;
+
+ open(F, "> $wrk/runCA-logs/${time}_${host}_${pid}_runCA");
+ print F $specLog;
+ close(F);
+}
+
+
+
+sub findFirstCheckpoint ($) {
+ my $dir = shift @_;
+ my $firstckp = 0;
+
+ $dir = "$wrk/$dir" if (! -d $dir);
+
+ open(F, "find -L $dir -type f -name $asm.ckp.\[0-9\]\* |");
+ while (<F>) {
+ if (m/ckp.(\d+)\s*$/) {
+ $firstckp = $1 if ($1 < $firstckp);
+ }
+ }
+ close(F);
+
+ return($firstckp);
+}
+
+sub findLastCheckpoint ($) {
+ my $dir = shift @_;
+ my $lastckp = 0;
+
+ $dir = "$wrk/$dir" if (-d "$wrk/$dir");
+
+ open(F, "find -L $dir -type f -name $asm.ckp.\[0-9\]\* |");
+ while (<F>) {
+ if (m/ckp.(\d+)\s*$/) {
+ $lastckp = $1 if ($1 > $lastckp);
+ }
+ }
+ close(F);
+
+ return($lastckp);
+}
+
+sub findNumScaffoldsInCheckpoint ($$) {
+ my $dir = shift @_;
+ my $lastckp = shift @_;
+
+ open(F, "cd $wrk/$dir && $bin/getNumScaffolds ../$asm.gkpStore $asm $lastckp 2> /dev/null |");
+ my $numscaf = <F>; chomp $numscaf;
+ close(F);
+ $numscaf = int($numscaf);
+
+ return($numscaf);
+}
+
+
+sub getNumberOfFragsInStore ($$) {
+ my $wrk = shift @_;
+ my $asm = shift @_;
+
+ $numFrags = 0;
+
+ if (-e "$wrk/$asm.gkpStore/inf") {
+ open(F, "$bin/gatekeeper -lastfragiid $wrk/$asm.gkpStore 2> /dev/null |") or caFailure("failed to run gatekeeper to get the number of frags in the store", undef);
+ $_ = <F>; chomp $_;
+ close(F);
+
+ $numFrags = $1 if (m/^Last frag in store is iid = (\d+)$/);
+
+ # Check that the store is efficient
+
+ my $typeMapSize = (-s "$wrk/$asm.gkpStore/inf") / 1024 / 1024;
+
+ if (($typeMapSize > 128) && (getGlobal("gkpAllowInefficientStorage") == 0)) {
+ $typeMapSize = int($typeMapSize * 3);
+
+ my $err;
+ $err .= "reads ordered inefficiently. reorder (illumina first) or use\n";
+ $err .= "gkpAllowInefficientStorage=1. note that overlap jobs will use\n";
+ $err .= "an additional $typeMapSize mb memory per job.";
+
+ caFailure($err, "$wrk/$asm.gkpStore.err");
+ }
+ }
+
+ return($numFrags);
+}
+
+
+
+
+# Default to unitigger, unless the gkpStore says otherwise.
+#
+sub getUnitigger () {
+ my $unitigger = getGlobal("unitigger");
+
+ if (!defined($unitigger)) {
+ setGlobal("unitigger", "utg");
+
+ if (system("$bin/gatekeeper -isfeatureset 0 forceBOGunitigger $wrk/$asm.gkpStore") == 0) {
+ setGlobal("unitigger", "bog");
+ }
+ }
+
+ $unitigger = getGlobal("unitigger");
+
+ die "Unitigger not defined.\n" if (!defined($unitigger));
+
+ return($unitigger);
+}
+
+
+# Decide if we have the CA meryl or the Mighty one.
+#
+sub merylVersion () {
+ my $ver = "unknown";
+
+ open(F, "$bin/meryl -V |");
+ while (<F>) {
+ $ver = "CA" if (m/CA/);
+ $ver = "Mighty" if (m/Mighty/);
+ }
+ close(F);
+ return($ver);
+}
+
+sub stopBefore ($$) {
+ my $stopBefore = shift @_; $stopBefore =~ tr/A-Z/a-z/;
+ my $cmd = shift @_;
+ if (defined($stopBefore) &&
+ defined(getGlobal('stopBefore')) &&
+ (getGlobal('stopBefore') eq $stopBefore)) {
+ print STDERR "Stop requested before '$stopBefore'.\n";
+ print STDERR "Command:\n$cmd\n" if (defined($cmd));
+ exit(0);
+ }
+}
+
+sub stopAfter ($) {
+ my $stopAfter = shift @_; $stopAfter =~ tr/A-Z/a-z/;
+ if (defined($stopAfter) &&
+ defined(getGlobal('stopAfter')) &&
+ (getGlobal('stopAfter') eq $stopAfter)) {
+ print STDERR "Stop requested after '$stopAfter'.\n";
+ exit(0);
+ }
+}
+
+sub runningOnGrid () {
+ my $jobID = getGlobal("gridJobID");
+ return(exists($ENV{$jobID}));
+}
+
+sub findNextScriptOutputFile () {
+ my $idx = "00";
+ while (-e "$wrk/runCA.sge.out.$idx.sh") {
+ $idx++;
+ }
+ return("$wrk/runCA.sge.out.$idx");
+}
+
+sub buildGridArray($$$) {
+ my $name = shift @_;
+ my $maxLimit = shift @_;
+ my $globalValue = shift @_;
+
+ my $arrayJobName = getGlobal($globalValue);
+ $arrayJobName =~ s/ARRAY_NAME/$name/g;
+ $arrayJobName =~ s/ARRAY_JOBS/1-$maxLimit/g;
+
+ return $arrayJobName;
+}
+
+sub getGridArrayName($$) {
+ my $name = shift @_;
+ my $maxLimit = shift @_;
+ return buildGridArray($name, $maxLimit, "gridArrayName");
+}
+
+sub getGridArrayOption($$) {
+ my $name = shift @_;
+ my $maxLimit = shift @_;
+ return buildGridArray($name, $maxLimit, "gridArrayOption");
+}
+
+sub submitScript ($) {
+ my $waitTag = shift @_;
+
+ return if (getGlobal("scriptOnGrid") == 0);
+
+ my $output = findNextScriptOutputFile();
+ my $script = "$output.sh";
+
+ open(F, "> $script") or caFailure("failed to open '$script' for writing", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "#\n";
+ print F "if [ \"x\$SGE_ROOT\" != \"x\" ]; then \n";
+ print F " # Attempt to (re)configure SGE. For reasons Bri doesn't know,\n";
+ print F " # jobs submitted to SGE, and running under SGE, fail to read his\n";
+ print F " # .tcshrc (or .bashrc, limited testing), and so they don't setup\n";
+ print F " # SGE (or ANY other paths, etc) properly. For the record,\n";
+ print F " # interactive SGE logins (qlogin, etc) DO set the environment.\n";
+ print F " \n";
+ print F " . \$SGE_ROOT/\$SGE_CELL/common/settings.sh\n";
+ print F "fi\n";
+ print F "\n";
+ print F "# On the off chance that there is a pathMap, and the host we\n";
+ print F "# eventually get scheduled on doesn't see other hosts, we decide\n";
+ print F "# at run time where the binary is.\n";
+
+ print F getBinDirectoryShellCode();
+
+ print F "/usr/bin/env perl \$bin/runCA $commandLineOptions\n";
+ close(F);
+
+ system("chmod +x $script");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeScript = getGlobal("sgeScript");
+ my $sgePropHold = getGlobal("sgePropagateHold");
+
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $holdOption = getGlobal("gridHoldOption");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+ my $holdPropagateCommand = getGlobal("gridPropagateCommand");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = "rCA_$asm$sgeName";
+
+ if (defined($waitTag)) {
+ my $hold = $holdOption;
+ $hold =~ s/WAIT_TAG/$waitTag/g;
+ $waitTag = $hold;
+ }
+ my $qcmd = "$submitCommand $sge $sgeScript $nameOption \"$jobName\" $waitTag $outputOption $output $script";
+ runCommand($wrk, $qcmd) and caFailure("Failed to submit script.\n");
+
+ if (defined($sgePropHold)) {
+ if (defined($holdPropagateCommand)) {
+ my $translateCmd = getGlobal("gridNameToJobIDCommand");
+
+ # translate hold option to job id if necessary
+ if (defined($translateCmd) && $translateCmd ne "") {
+ my $tcmd = $translateCmd;
+ $tcmd =~ s/WAIT_TAG/$sgePropHold/g;
+ my $propJobCount = `$tcmd |wc -l`;
+ chomp $propJobCount;
+ if ($propJobCount != 1) {
+ print STDERR "Warning: multiple IDs for job $sgePropHold got $propJobCount and should have been 1.\n";
+ }
+ #my $jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
+ #chomp $jobID;
+ #print STDERR "Translated job ID $sgePropHold to be job $jobID\n";
+ #$sgePropHold = $jobID;
+ open(PROPS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that need to hold", undef);
+
+ # now we can get the job we are holding for
+ $tcmd = $translateCmd;
+ $tcmd =~ s/WAIT_TAG/$jobName/g;
+ my $holdJobCount = `$tcmd |wc -l`;
+ chomp $propJobCount;
+ if ($propJobCount != 1) {
+ print STDERR "Warning: multiple IDs for job $jobName got $propJobCount and should have been 1.\n";
+ }
+ #$jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
+ #chomp $jobID;
+ #print STDERR "Translated job ID $sgePropHold to be job $jobID\n";
+ #$jobName = $jobID;
+ open(HOLDS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that should be held for", undef);
+
+ # loop over all jobs and all sge hold commands to modify the jobs. We have no way to know which is the right one unfortunately
+ while (my $prop = <PROPS>) {
+ while (my $hold = <HOLDS>) {
+ chomp $hold;
+ chomp $prop;
+ my $hcmd = $holdPropagateCommand;
+ $hcmd =~ s/WAIT_TAG/$hold/g;
+ my $acmd = "$hcmd $prop";
+ print STDERR "Propagating hold to $prop to wait for job $hold\n";
+ system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$prop'.\n";
+ }
+ }
+ close(HOLDS);
+ close(PROPS);
+ } else {
+ $sgePropHold = "\"$sgePropHold\"";
+ $holdPropagateCommand =~ s/WAIT_TAG/$jobName/g;
+ my $acmd = "$holdPropagateCommand $sgePropHold";
+ system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$sgePropHold'.\n";
+ }
+ } else {
+ print STDERR "WARNING: Failed to reset hold '$sgePropHold', not supported on current grid environment.\n";
+ }
+ }
+
+ exit(0);
+}
+
+
+sub caFailure ($$) {
+ my $msg = shift @_;
+ my $log = shift @_;
+
+ print STDERR "================================================================================\n";
+ print STDERR "\n";
+ print STDERR "runCA failed.\n";
+ print STDERR "\n";
+
+ print STDERR "----------------------------------------\n";
+ print STDERR "Stack trace:\n";
+ print STDERR "\n";
+ carp;
+
+ if (-e $log) {
+ print STDERR "\n";
+ print STDERR "----------------------------------------\n";
+ print STDERR "Last few lines of the relevant log file ($log):\n";
+ print STDERR "\n";
+ system("tail -n 20 $log");
+ }
+
+ print STDERR "\n";
+ print STDERR "----------------------------------------\n";
+ print STDERR "Failure message:\n";
+ print STDERR "\n";
+ print STDERR "$msg\n";
+ print STDERR "\n";
+
+ exit(1);
+}
+
+
+
+# Bit of a wierd one here; assume path are supplied relative to $wrk.
+# Potentially gives us a bit of safety.
+#
+sub rmrf (@) {
+ foreach my $f (@_) {
+ unlink("$wrk/$f") if (-f "$wrk/$f");
+ system("rm -rf $wrk/$f") if (-d "$wrk/$f");
+ }
+}
+
+
+# Create an empty file. Much faster than system("touch ...").
+#
+sub touch ($) {
+ open(F, "> $_[0]") or caFailure("failed to touch file '$_[0]'", undef);
+ close(F);
+}
+
+
+sub pleaseExecute ($) {
+ my $file = shift @_;
+
+ print STDERR "Please execute:\n";
+ print STDERR " $file\n";
+ print STDERR "to submit jobs to the grid, then restart this script when all\n";
+ print STDERR "jobs finish. I'll make sure all jobs finished properly.\n";
+}
+
+
+# Utility to run a command and check the exit status, report time used.
+#
+sub runCommand ($$) {
+ my $dir = shift @_;
+ my $cmd = shift @_;
+
+ if (! -d $dir) {
+ caFailure("Directory '$dir' doesn't exist, can't run command.\n", "");
+ }
+
+ if (getGlobal('showNext')) {
+ print STDERR "----------------------------------------NEXT-COMMAND\n$cmd\n";
+ exit(0);
+ }
+
+ my $t = localtime();
+ my $d = time();
+ print STDERR "----------------------------------------START $t\n$cmd\n";
+
+ my $rc = 0xffff & system("cd $dir && $cmd");
+
+ $t = localtime();
+ print STDERR "----------------------------------------END $t (", time() - $d, " seconds)\n";
+
+ # Pretty much copied from Programming Perl page 230
+
+ return(0) if ($rc == 0);
+
+ # Bunch of busy work to get the names of signals. Is it really worth it?!
+ #
+ my @signame;
+ if (defined($Config{sig_name})) {
+ my $i = 0;
+ foreach my $n (split('\s+', $Config{sig_name})) {
+ $signame[$i] = $n;
+ $i++;
+ }
+ }
+
+ my $error = "ERROR: Failed with ";
+
+ if ($rc == 0xff00) {
+ $error .= "$!\n";
+ } else {
+ if ($rc & 0x80) {
+ $error .= "coredump from ";
+ }
+
+ if ($rc > 0x80) {
+ $rc >>= 8;
+ }
+ $rc &= 127;
+
+ if (defined($signame[$rc])) {
+ $error .= "signal $signame[$rc] ($rc)\n";
+ } else {
+ $error .= "signal $rc\n";
+ }
+ }
+
+ print STDERR $error;
+
+ return(1);
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+# Functions for running multiple processes at the same time.
+
+my $numberOfProcesses = 0; # Number of jobs concurrently running
+my $numberOfProcessesToWait = 0; # Number of jobs we can leave running at exit
+my @processQueue = ();
+my @processesRunning = ();
+my $printProcessCommand = 1; # Show commands as they run
+
+sub schedulerSetNumberOfProcesses {
+ $numberOfProcesses = shift @_;
+}
+
+sub schedulerSubmit {
+ chomp @_;
+ push @processQueue, @_;
+}
+
+sub schedulerForkProcess {
+ my $process = shift @_;
+ my $pid;
+
+ # From Programming Perl, page 167
+ FORK: {
+ if ($pid = fork) {
+ # Parent
+ #
+ return($pid);
+ } elsif (defined $pid) {
+ # Child
+ #
+ exec($process);
+ } elsif ($! =~ /No more processes/) {
+ # EAGIN, supposedly a recoverable fork error
+ sleep 1;
+ redo FORK;
+ } else {
+ die "Can't fork: $!\n";
+ }
+ }
+}
+
+sub schedulerReapProcess {
+ my $pid = shift @_;
+
+ if (waitpid($pid, &WNOHANG) > 0) {
+ return(1);
+ } else {
+ return(0);
+ }
+}
+
+sub schedulerRun {
+ my @newProcesses;
+
+ # Reap any processes that have finished
+ #
+ undef @newProcesses;
+ foreach my $i (@processesRunning) {
+ if (schedulerReapProcess($i) == 0) {
+ push @newProcesses, $i;
+ }
+ }
+ undef @processesRunning;
+ @processesRunning = @newProcesses;
+
+ # Run processes in any available slots
+ #
+ while ((scalar(@processesRunning) < $numberOfProcesses) &&
+ (scalar(@processQueue) > 0)) {
+ my $process = shift @processQueue;
+ print STDERR "$process\n";
+ push @processesRunning, schedulerForkProcess($process);
+ }
+}
+
+sub schedulerFinish {
+ my $child;
+ my @newProcesses;
+ my $remain;
+
+ my $t = localtime();
+ my $d = time();
+ print STDERR "----------------------------------------START CONCURRENT $t\n";
+
+ $remain = scalar(@processQueue);
+
+ # Run all submitted jobs
+ #
+ while ($remain > 0) {
+ schedulerRun();
+
+ $remain = scalar(@processQueue);
+
+ if ($remain > 0) {
+ $child = waitpid -1, 0;
+
+ undef @newProcesses;
+ foreach my $i (@processesRunning) {
+ push @newProcesses, $i if ($child != $i);
+ }
+ undef @processesRunning;
+ @processesRunning = @newProcesses;
+ }
+ }
+
+ # Wait for them to finish, if requested
+ #
+ while (scalar(@processesRunning) > $numberOfProcessesToWait) {
+ waitpid(shift @processesRunning, 0);
+ }
+
+ $t = localtime();
+ print STDERR "----------------------------------------END CONCURRENT $t (", time() - $d, " seconds)\n";
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub dumpInfo {
+ if (! -e "$wrk/$asm.gkpStore.info") {
+ $cmd = "$bin/gatekeeper ";
+ $cmd .= " -dumpinfo -tabular $wrk/$asm.gkpStore ";
+ $cmd .= "> $wrk/$asm.gkpStore.info ";
+ $cmd .= "2> $wrk/$asm.gkpStore.info.err";
+
+ if (runCommand($wrk, $cmd)) {
+ caWarn("gatekeeper -dumpinfo failed", "$wrk/$asm.gkpStore.info.err");
+ }
+ }
+}
+
+sub preoverlap {
+ my @fragFiles = @_;
+
+ $numFrags = getNumberOfFragsInStore($wrk, $asm);
+
+ # Return if there are fragments in the store, and die if there
+ # are no fragments and no source files.
+ #
+ if ($numFrags > 0) {
+ goto stopafter;
+ }
+
+ caFailure("no fragment files specified, and stores not already created", undef)
+ if (scalar(@fragFiles) == 0);
+
+ if ((! -d "$wrk/$asm.gkpStore") ||
+ (! -e "$wrk/$asm.gkpStore/inf")) {
+
+ # Make sure all the inputs are here. We also shred any
+ # supplied ace files, and convert the sff's to frg's.
+ #
+ my $failedFiles = undef;
+ my $gkpInput = "";
+ foreach my $frg (@fragFiles) {
+ if (! -e $frg) {
+ if (defined($failedFiles)) {
+ $failedFiles .= "; '$frg' not found";
+ } else {
+ $failedFiles = "'$frg' not found";
+ }
+ }
+
+ if ($frg =~ m/^(.*)\.ace$/) {
+ my @fff = split '/', $1;
+ my $ace = $frg;
+ my $nam = pop @fff;
+
+ $frg = "$wrk/$nam.shred.frg";
+
+ if (! -e "$frg") {
+ print STDERR "Shredding '$ace' -> '$frg'\n";
+ shredACE($ace, $frg);
+ }
+ }
+
+ if (($frg =~ m/^(.*)\.sff$/) ||
+ ($frg =~ m/^(.*)\.sff.gz$/) ||
+ ($frg =~ m/^(.*)\.sff.bz2$/) ||
+ ($frg =~ m/^(.*)\.sff.xz$/)) {
+ my @fff = split '/', $1;
+ my $sff = $frg;
+ my $nam = pop @fff;
+ my $log = "$wrk/$nam.sff.log";
+
+ $frg = "$wrk/$nam.sff.frg";
+
+ if (! -e "$frg") {
+ print STDERR "Converting '$sff' -> '$frg'\n";
+
+ if (runCommand($wrk, "$bin/sffToCA -libraryname $nam -linker flx -linker titanium -insertsize 3000 300 -output $frg $sff > $frg.err 2>&1")) {
+ unlink "$wrk/$frg";
+ caFailure("sffToCA failed", "$frg.err");
+ }
+ }
+ }
+
+ $gkpInput .= " $frg";
+ }
+ caFailure($failedFiles, undef) if defined($failedFiles);
+
+ $cmd = "$bin/gatekeeper ";
+ $cmd .= " -o $wrk/$asm.gkpStore.BUILDING ";
+ $cmd .= " -T " if (getGlobal("doOverlapBasedTrimming"));
+ $cmd .= " -F " if (getGlobal("gkpFixInsertSizes"));
+ $cmd .= "$gkpInput ";
+ $cmd .= "> $wrk/$asm.gkpStore.err 2>&1";
+
+ if (runCommand($wrk, $cmd)) {
+ caFailure("gatekeeper failed", "$wrk/$asm.gkpStore.err");
+ }
+
+ rename "$wrk/$asm.gkpStore.BUILDING", "$wrk/$asm.gkpStore";
+ rename "$wrk/$asm.gkpStore.BUILDING.info", "$wrk/$asm.gkpStore.info";
+ rename "$wrk/$asm.gkpStore.BUILDING.errorLog", "$wrk/$asm.gkpStore.errorLog";
+ rename "$wrk/$asm.gkpStore.BUILDING.fastqUIDmap", "$wrk/$asm.gkpStore.fastqUIDmap";
+ }
+
+ # This should never run; the info is now created by gatekeeper.
+ dumpInfo();
+
+ generateVectorTrim();
+
+ my $vi = getGlobal("vectorIntersect");
+
+ if ((defined($vi)) && (! -e "$wrk/$asm.gkpStore/$asm.vectorClearLoaded.log")) {
+ $cmd = "$bin/gatekeeper -a -v $vi -o $wrk/$asm.gkpStore ";
+ $cmd .= " > $wrk/$asm.gkpStore/$asm.vectorClearLoaded.log";
+ $cmd .= " 2> $wrk/$asm.gkpStore/$asm.vectorClearLoaded.err";
+
+ if (runCommand($wrk, $cmd)) {
+ rename "$wrk/$asm.gkpStore/$asm.vectorClearLoaded.log", "$wrk/$asm.gkpStore/$asm.vectorClearLoaded.log.FAILED";
+ caFailure("gatekeeper failed to update clear ranges", "$wrk/$asm.gkpStore/$asm.vectorClearLoaded.err");
+ }
+
+ unlink "$wrk/$asm.gkpStore/$asm.vectorClearLoaded.err";
+ }
+
+ $numFrags = getNumberOfFragsInStore($wrk, $asm);
+
+ print STDERR "numFrags = $numFrags\n";
+
+ caFailure("gatekeeper failed to add fragments", "$wrk/$asm.gkpStore.err") if ($numFrags == 0);
+
+ stopafter:
+ stopAfter("initialStoreBuilding");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+#
+# Parameters
+#
+
+my $MIN_COVERAGE = 1; # Should be 2 if there are "fake" reads in ace file
+
+my $MIN_READS = 4;
+my $MIN_CONTIG_SIZE = 600;
+
+my $SHRED_READ_LENGTH = 600;
+
+my $LOW_QUAL_DIVISOR = 4;
+my $DEFAULT_QUAL = 3;
+
+#
+# Methods for reading an ACE file.
+#
+
+sub read_AS{
+ my $fh=shift;
+
+ while(<$fh>){
+ chomp;
+ my ($id, $num_contigs, $num_reads)=split /\s+/;
+ if($id eq "AS"){
+ return ($num_contigs, $num_reads);
+ }
+ }
+ die "Could not find AS to read.\n";
+}
+
+
+sub read_CO{
+ my $fh=shift;
+
+ while(<$fh>){
+ chomp;
+ my ($id, $contig_id, $num_bases, $num_reads, $num_segments, $complementation, $sequence)=split /\s+/;
+
+ if($id eq "CO"){
+ while(<$fh>){
+ chomp;
+ if($_ eq ""){
+ last;
+ }else{
+ $sequence.=$_;
+ }
+ }
+ return($contig_id, $num_bases, $num_reads, $num_segments, $complementation, $sequence);
+ }
+ }
+ die "Could not find CO to read.\n";
+}
+
+
+sub read_BQ{
+ my $fh=shift;
+
+ my ($id, $sequence);
+
+ while(<$fh>){
+ chomp;
+ ($id)=split /\s+/;
+
+ if($id eq "BQ"){
+ while(<$fh>){
+ chomp;
+ if($_ eq ""){
+ last;
+ }else{
+ $sequence.=$_;
+ }
+ }
+ return($sequence);
+ }
+ }
+ die "Could not find BQ to read.\n";
+}
+
+
+sub read_AF{
+ my $fh=shift;
+
+ while(<$fh>){
+ chomp;
+ my ($id, $read_id, $complementation, $start)=split /\s+/;
+ if($id eq "AF"){
+ return($read_id, $complementation, $start);
+ }
+ }
+ die "Could not find AF to read.\n";
+}
+
+
+sub read_BS{
+ my $fh=shift;
+
+ while(<$fh>){
+ chomp;
+ my ($id, $start, $end, $read_id)=split /\s+/;
+ if($id eq "BS"){
+ return($start, $end, $read_id);
+ }
+ }
+ die "Could not find BS to read.\n";
+}
+
+
+sub read_RD{
+ my $fh=shift;
+
+ while(<$fh>){
+ chomp;
+ my ($id, $read_id, $num_bases, $num_read_info_items, $num_read_tags)=split /\s+/;
+ my $sequence;
+ if($id eq "RD"){
+ while(<$fh>){
+ chomp;
+ if($_ eq ""){
+ last;
+ }else{
+ $sequence.=$_;
+ }
+ }
+ return($read_id, $num_bases, $num_read_info_items, $num_read_tags, $sequence);
+ }
+ }
+ die "Could not find RD to read.\n";
+}
+
+
+sub read_QA{
+ my $fh=shift;
+
+ while(<$fh>){
+ chomp;
+ my ($id, $qual_start, $qual_end, $clip_start, $clip_end)=split /\s+/;
+ if($id eq "QA"){
+ return($qual_start, $qual_end, $clip_start, $clip_end);
+ }
+ }
+ die "Could not find QA to read.\n";
+}
+
+
+sub read_DS{
+ my $fh=shift;
+ my $id;
+ while(<$fh>){
+ chomp;
+ my ($id)=split /\s+/;
+ if($id eq "DS"){
+ return("not implemented");
+ }
+ }
+ die "Could not find DS to read.\n";
+}
+
+#
+#
+#
+
+sub emitFragment ($$$$) {
+ my $uid = shift;
+ my $lid = shift;
+ my $seq = shift;
+ my $oh = shift;
+
+ my $len = length($seq);
+
+ my $qvs = $seq;
+
+ my $q = chr($DEFAULT_QUAL + ord("0"));
+ my $l = chr($DEFAULT_QUAL/$LOW_QUAL_DIVISOR + ord("0"));
+
+ $qvs =~ s/[^ACGT]/$l/og;
+ $qvs =~ s/[ACGT]/$q/og;
+
+ print $oh "{FRG\n";
+ print $oh "act:A\n";
+ print $oh "acc:$uid\n";
+ print $oh "rnd:1\n";
+ print $oh "sta:G\n";
+ print $oh "lib:$lid\n";
+ print $oh "pla:0\n";
+ print $oh "loc:0\n";
+ print $oh "src:\n.\n";
+ print $oh "seq:\n$seq\n.\n";
+ print $oh "qlt:\n$qvs\n.\n";
+ print $oh "hps:\n.\n";
+ print $oh "clr:0,$len\n";
+ print $oh "}\n";
+}
+
+#
+#
+#
+
+sub shredContig ($$$$$) {
+ my $ctgId = shift;
+ my $avgCoverage = shift;
+ my $sequence = shift;
+ my $libId = shift;
+ my $oh = shift;
+
+ my $seq_len=length($sequence);
+
+ my @begin_shred;
+ my @end_shred;
+
+ {
+ #
+ # |*******|
+ # |###############|
+ # |-------------------------------------------------|
+ # ----------------1----------------
+ # ----------------2----------------
+ # ----------------3----------------
+ #
+ # #### represents the distance between center of read 1 and read 3
+ # [$center_range_width]
+ # **** represents the distance between centers of consective reads
+ # [$center_increments]
+ #
+
+ my $shred_len = $SHRED_READ_LENGTH;
+ $shred_len = $seq_len - 50 if $seq_len < $SHRED_READ_LENGTH;
+
+ my $num_reads=int($seq_len * $avgCoverage / $shred_len);
+ my $center_range_width = $seq_len - $shred_len;
+
+ if($num_reads==1){
+ push @begin_shred, 0;
+ push @end_shred, $shred_len;
+ }else{
+ my $center_increments = $center_range_width / ($num_reads-1);
+
+ # Cap the number of reads we will make so that we don't get
+ # redundant reads
+
+ my $i;
+ my ($prev_begin, $prev_end)=(-1,-1);
+ for($i=0; $i<$num_reads; $i++){
+ my $begin=$center_increments*$i;
+ my $end=$begin+$shred_len;
+
+ $begin=int($begin);
+ $end=int($end);
+
+ if($begin!=$prev_begin || $end!=$prev_end){
+ push @begin_shred, $begin;
+ push @end_shred, $end;
+ $prev_begin=$begin;
+ $prev_end=$end;
+ }
+ }
+ }
+
+ }
+
+ my $num_shreds = scalar(@begin_shred);
+
+ my $accomplished_coverage = $num_shreds * $SHRED_READ_LENGTH / $seq_len;
+
+ # Output sequence after it has been formatted to the specified width
+ my $shred_idx;
+ for($shred_idx=0; $shred_idx<$num_shreds; $shred_idx++){
+ my $shredded_sequence=substr($sequence,
+ $begin_shred[$shred_idx],
+ $end_shred[$shred_idx]-$begin_shred[$shred_idx]);
+
+ #"/contig=$contigID\.$shred_idx " ,
+ #"/target_coverage=$avgCoverage " ,
+ #"/accomplished_coverage=$accomplished_coverage " ,
+ #"/input_length=$seq_len " ,
+ #"/range=${$begin_shred_ref}[$shred_idx]-" ,
+ # "${$end_shred_ref}[$shred_idx]\n";
+
+ emitFragment("$libId.$ctgId.frag$shred_idx.$begin_shred[$shred_idx]-$end_shred[$shred_idx]", $libId, $shredded_sequence, $oh);
+ }
+}
+
+#
+# Main
+#
+
+sub shredACE ($$) {
+ my $aceFile = shift;
+ my $outFile = shift;
+ my $libId = $aceFile;
+
+ if ($aceFile =~ m/^.*\/(.*).ace/) {
+ $libId = $1;
+ }
+
+ my $fh = new FileHandle "< $aceFile";
+ my $oh = new FileHandle "> $outFile";
+
+ print $oh "{VER\n";
+ print $oh "ver:2\n";
+ print $oh "}\n";
+ print $oh "{LIB\n";
+ print $oh "act:A\n";
+ print $oh "acc:$libId\n";
+ print $oh "ori:U\n";
+ print $oh "mea:0.0\n";
+ print $oh "std:0.0\n";
+ print $oh "src:\n";
+ print $oh ".\n";
+ print $oh "nft:1\n";
+ print $oh "fea:\n";
+ print $oh "doNotOverlapTrim=1\n";
+ print $oh ".\n";
+ print $oh "}\n";
+
+ my ($num_contigs, $num_reads)=read_AS($fh);
+
+ my $contig_idx;
+ for($contig_idx=0; $contig_idx<$num_contigs; $contig_idx++){
+
+ my %read_position_hash;
+
+ my ($contig_id, $num_consensus_bases, $num_reads, $num_segments, $complementation, $consensus_sequence) = read_CO($fh);
+
+ my @coverage_array;
+ my $i;
+
+ # Initialize Coverage Array
+ for($i=0; $i<$num_consensus_bases; $i++){
+ $coverage_array[$i]=0;
+ }
+
+ my $quality=read_BQ($fh);
+
+ my $read_idx;
+ for($read_idx=0; $read_idx<$num_reads; $read_idx++){
+ my ($read_id, $complementation, $consensus_start_pos)=read_AF($fh);
+ $read_position_hash{$read_id}=$consensus_start_pos;
+ }
+
+ my ($base_line_start, $base_line_end, $base_line_read_id)=read_BS($fh);
+
+ for($read_idx=0; $read_idx<$num_reads; $read_idx++){
+ my ($read_id, $num_padded_bases, $num_read_info_items, $num_read_tags, $read_sequence)= read_RD($fh);
+ my ($qual_start, $qual_end, $align_start, $align_end)=read_QA($fh);
+ my $startPos = $read_position_hash{$read_id};
+
+ my $begin = $align_start + $startPos - 1;
+ my $end = $align_end + $startPos - 1;
+
+ for($i=$begin; $i<$end; $i++){
+ $coverage_array[$i]++;
+ }
+ my ($null)=read_DS($fh);
+ }
+
+
+ my $in_deep_enough=0;
+ my @sub_contig_begin_arr;
+ my @sub_contig_end_arr;
+
+ # Keep track of where we go into deep coverage region from low coverage regions
+ for($i=0; $i<$num_consensus_bases; $i++){
+ if($coverage_array[$i]>$MIN_COVERAGE && !$in_deep_enough){
+ push @sub_contig_begin_arr, $i;
+ $in_deep_enough=1;
+ }
+ if($coverage_array[$i]<=$MIN_COVERAGE && $in_deep_enough){
+ push @sub_contig_end_arr, ($i);
+ $in_deep_enough=0;
+ }
+ }
+
+ if($in_deep_enough){
+ push @sub_contig_end_arr, ($i);
+ }
+
+ for($i=0; $i<=$#sub_contig_begin_arr; $i++){
+ # Sum up coverage for each sub contig
+ my $cov_idx;
+ my $cov_sum=0;
+ for($cov_idx=$sub_contig_begin_arr[$i];
+ $cov_idx<$sub_contig_end_arr[$i];
+ $cov_idx++){
+ $cov_sum+=$coverage_array[$cov_idx];
+ }
+
+ # Compute average coverage depth
+
+ my $sub_seq_len=$sub_contig_end_arr[$i]-$sub_contig_begin_arr[$i];
+ my $avg_cov = $cov_sum / $sub_seq_len;
+
+ if($num_reads > $MIN_READS && $sub_seq_len>=$MIN_CONTIG_SIZE){
+ my $sub_contig_seq = substr($consensus_sequence,
+ $sub_contig_begin_arr[$i],
+ $sub_seq_len);
+
+ # Remove padding
+ $sub_contig_seq=~s/\*//g;
+
+ shredContig($contig_id, $avg_cov, $sub_contig_seq, $libId, $oh);
+ }
+ }
+ }
+
+ print $oh "{VER\n";
+ print $oh "ver:1\n";
+ print $oh "}\n";
+}
+
+#
+# For standalone use
+#
+
+#die "usage: $0 file.ace > file.frg\n" if (scalar(@ARGV) == 0);
+#shredACE($ARGV[0], "a.frg");
+#exit();
+
+################################################################################
+################################################################################
+################################################################################
+
+
+sub runMeryl ($$$$$$$$) {
+ my $merSize = shift @_;
+ my $merComp = shift @_;
+ my $merCanonical = shift @_;
+ my $merThresh = shift @_;
+ my $merScale = 1.0;
+ my $merDistinct = shift @_;
+ my $merTotal = shift @_;
+ my $merType = shift @_;
+ my $merDump = shift @_;
+
+ system("mkdir $wrk/0-mercounts") if (! -d "$wrk/0-mercounts");
+
+ # This is the historical default.
+ if (!defined($merThresh) && !defined($merDistinct) && !defined($merTotal)) {
+ $merThresh = "auto";
+ }
+
+ # The fasta file we should be creating.
+ my $ffile = "$wrk/0-mercounts/$asm.nmers.$merType.fasta";
+
+ if ($merThresh =~ m/auto\s*\*\s*(\S+)/) {
+ $merThresh = "auto";
+ $merScale = $1;
+ }
+
+ if ($merThresh =~ m/auto\s*\/\s*(\S+)/) {
+ $merThresh = "auto";
+ $merScale = 1.0 / $1;
+ }
+
+ # And a special case; if the threshold is zero, we can skip the rest.
+ if (($merThresh ne "auto") && ($merThresh == 0) && (!defined($merDistinct)) && (!defined($merTotal))) {
+ touch $ffile;
+ return;
+ }
+
+ if (merylVersion() eq "Mighty") {
+
+ # Use the better meryl! This is straightforward. We count,
+ # then we dump.
+
+ # Intermediate file
+ my $ofile = "$wrk/0-mercounts/$asm$merCanonical-ms$merSize-cm$merComp";
+
+ if (! -e "$ofile.mcdat") {
+ my $merylMemory = getGlobal("merylMemory");
+ my $merylThreads = getGlobal("merylThreads");
+
+ if ($merylMemory !~ m/^-/) {
+ $merylMemory = "-memory $merylMemory";
+ }
+
+ # A small optimization we could do if (a) not mer
+ # overlapper, (b) not auto threshold: only save mer
+ # counts above the smaller (of obt & ovl thresholds).
+ # It's complicated, and potentially screws up restarts
+ # (if the threshold is changed after meryl is finished,
+ # for example). It's only useful on large assemblies,
+ # which we usually assume you know what's going on
+ # anyway.
+ #
+ # N.B. the mer overlapper NEEDS all mer counts 2 and
+ # higher.
+
+ $cmd = "$bin/meryl ";
+ $cmd .= " -B $merCanonical -v -m $merSize $merylMemory -threads $merylThreads -c $merComp ";
+ $cmd .= " -L 2 ";
+ $cmd .= " -s $wrk/$asm.gkpStore:chain "; # Or 'latest' to get the original version
+ $cmd .= " -o $ofile ";
+ $cmd .= "> $wrk/0-mercounts/meryl.err 2>&1";
+
+ stopBefore("meryl", $cmd);
+
+ if (runCommand("$wrk/0-mercounts", $cmd)) {
+ caFailure("meryl failed", "$wrk/0-mercounts/meryl.err");
+ }
+ unlink "$wrk/0-mercounts/meryl.err";
+ }
+
+ if ($merThresh eq "auto") {
+ if (! -e "$ofile.estMerThresh.out") {
+ $cmd = "$bin/estimate-mer-threshold ";
+ $cmd .= " -m $ofile ";
+ $cmd .= " > $ofile.estMerThresh.out ";
+ $cmd .= "2> $ofile.estMerThresh.err";
+
+ stopBefore("meryl", $cmd);
+
+ if (runCommand("$wrk/0-mercounts", $cmd)) {
+ rename "$ofile.estMerThresh.out", "$ofile.estMerThresh.out.FAILED";
+ caFailure("estimate-mer-threshold failed", "$ofile.estMerThresh.err");
+ }
+ }
+
+ open(F, "< $ofile.estMerThresh.out") or caFailure("failed to read estimated mer threshold from '$ofile.estMerThresh.out'", undef);
+ $merThresh = <F>;
+ $merThresh = int($merThresh * $merScale);
+ close(F);
+ }
+
+ if (defined($merDistinct) || defined($merTotal)) {
+ if (! -e "$ofile.histogram") {
+ $cmd = "$bin/meryl ";
+ $cmd .= " -Dh -s $ofile ";
+ $cmd .= " > $ofile.histogram ";
+ $cmd .= "2> $ofile.histogram.err";
+
+ stopBefore("meryl", $cmd);
+
+ if (runCommand("$wrk/0-mercounts", $cmd)) {
+ rename "$ofile.histogram", "$ofile.histogram.FAILED";
+ caFailure("meryl histogram failed", "$ofile.histogram.err");
+ }
+ }
+
+ my $lastThreshold = 0;
+
+ open(F, "< $ofile.histogram") or caFailure("failed to read mer histogram from '$ofile.histogram'", undef);
+ while (<F>) {
+ my ($threshold, $num, $distinct, $total) = split '\s+', $_;
+
+ if (($merThresh > 0) && ($merThresh < $threshold)) {
+ print STDERR "Supplied merThreshold $merThresh is the smallest.\n";
+ last;
+ }
+
+ if ((defined($merDistinct)) && ($merDistinct < $distinct)) {
+ $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $lastThreshold;
+ print STDERR "Supplied merDistinct $merDistinct with threshold $lastThreshold is the smallest.\n";
+ last;
+ }
+
+ if ((defined($merTotal)) && ($merTotal < $total)) {
+ $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $lastThreshold;
+ print STDERR "Supplied merTotal $merTotal with threshold $lastThreshold is the smallest.\n";
+ last;
+ }
+
+ $lastThreshold = $threshold;
+ }
+ close(F);
+ }
+
+ #if ($merThresh == 0) {
+ # caFailure("failed to estimate a mer threshold", "$ofile.estMerThresh.err");
+ #}
+
+ # We only need the ascii dump if we're doing overlapper, mer
+ # overlapper reads meryl directly.
+ #
+ if ($merDump) {
+ if (! -e $ffile) {
+ $cmd = "$bin/meryl ";
+ $cmd .= "-Dt -n $merThresh ";
+ $cmd .= "-s $ofile ";
+ $cmd .= "> $ffile ";
+ $cmd .= "2> $ffile.err ";
+
+ if (runCommand("$wrk/0-mercounts", $cmd)) {
+ unlink $ffile;
+ caFailure("meryl failed to dump frequent mers", "$ffile.err");
+ }
+ unlink "$ffile.err";
+ }
+ }
+ } elsif (merylVersion() eq "CA") {
+
+ # Sigh. The old meryl. Not as easy. If we assume the
+ # process, in particular that the Ovl threshold is less than
+ # the Obt threshold, and that we have already computed the
+ # Ovl mers, we could filter the Ovl mers to get the Obt mers.
+ # But that's tough, especially if we allow mer compression.
+
+ my $merSkip = 10;
+
+ # Intermediate file
+ my $ofile = "$wrk/0-mercounts/$asm-ms$merSize-mt$merThresh-mk$merSkip.$merType.fasta";
+
+ if ($merComp > 0) {
+ print STDERR "ERROR! merCompression not supported without installing kmer\n";
+ print STDERR " (http://sourceforge.net/projects/kmer/).\n";
+ print STDERR "If you have installed kmer, then your build is broken, as I\n";
+ print STDERR "did not find the correct 'meryl' (meryl -V should have said Mighty).\n";
+ die;
+ }
+
+ if ($merCanonical ne "-C") {
+ print STDERR "ERROR! mer overlapper not supported without installing kmer\n";
+ print STDERR " (http://sourceforge.net/projects/kmer/).\n";
+ print STDERR "If you have installed kmer, then your build is broken, as I\n";
+ print STDERR "did not find the correct 'meryl' (meryl -V should have said Mighty).\n";
+ die;
+ }
+
+ if ($merThresh eq "auto") {
+ print STDERR "WARNING! auto picking a mer threshold not supported without installing kmer\n";
+ print STDERR " (http://sourceforge.net/projects/kmer/).\n";
+ print STDERR "Using historical defaults.\n";
+
+ if ($merType eq "obt") {
+ $merThresh = 1000;
+ } else {
+ $merThresh = 500;
+ }
+ }
+
+ if (! -e $ofile) {
+ my $mt = $merThresh / $merSkip;
+
+ $cmd = "$bin/meryl ";
+ $cmd .= "-s $wrk/$asm.gkpStore -m $merSize -n $mt -K $merSkip ";
+ $cmd .= " -o $ofile";
+ $cmd .= "> $wrk/0-mercounts/meryl.err 2>&1";
+
+ stopBefore("meryl", $cmd);
+
+ if (runCommand("$wrk/0-mercounts", $cmd)) {
+ unlink $ofile;
+ caFailure("meryl failed to dump frequent mers", "$wrk/0-mercounts/meryl.err");
+ }
+ unlink "$wrk/0-mercounts/meryl.err";
+ }
+
+ symlink($ofile, $ffile) if (! -e $ffile);
+ } else {
+ caFailure("unknown meryl version '" . merylVersion() . "'", "");
+ }
+
+ return($merThresh);
+}
+
+sub meryl {
+
+ if (getGlobal("ovlOverlapper") eq "umd") {
+ caFailure("meryl attempted to compute mer counts for the umd overlapper", undef);
+ }
+
+ my $ovlc = 0; # No compression, unless we're the mer overlapper
+ my $obtc = 0;
+
+ my $ovlC = "-C"; # Canonical, unless we're the mer overlapper
+ my $obtC = "-C"; # (except the mer overlapper now wants canonical)
+
+ my $ovlD = 1; # Dump, unless we're the mer overlapper
+ my $obtD = 1;
+
+ my $obtT = 0; # New threshold
+ my $ovlT = 0;
+
+ if (getGlobal("ovlOverlapper") eq "mer") {
+ $ovlc = getGlobal("merCompression");
+ $ovlC = "-C";
+ $ovlD = 0;
+ }
+ if (getGlobal("obtOverlapper") eq "mer") {
+ $obtc = getGlobal("merCompression");
+ $obtC = "-C";
+ $obtD = 0;
+ }
+
+
+ system("mkdir $wrk/0-mercounts") if (! -d "$wrk/0-mercounts");
+
+ if (defined(getGlobal("obtFrequentMers"))) {
+ my $ffile = "$wrk/0-mercounts/$asm.nmers.obt.fasta";
+ my $sfile = getGlobal("obtFrequentMers");
+
+ if (! -e $ffile) {
+ caFailure("obtFrequentMers '$sfile' not found", undef) if (! -e $sfile);
+ print STDERR "Using obt frequent mers in '$sfile'\n";
+ symlink $sfile, $ffile;
+ }
+ }
+
+
+ if (defined(getGlobal("ovlFrequentMers"))) {
+ my $ffile = "$wrk/0-mercounts/$asm.nmers.ovl.fasta";
+ my $sfile = getGlobal("ovlFrequentMers");
+
+ if (! -e $ffile) {
+ caFailure("ovlFrequentMers '$sfile' not found", undef) if (! -e $sfile);
+ print STDERR "Using ovl frequent mers in '$sfile'\n";
+ symlink $sfile, $ffile;
+ }
+ }
+
+ # We're only here to compute mercounts for the overlappers (all of mer, obt and ovl). If the
+ # frequent mer file(s) exist, don't bother running meryl (which runs to build mcidx/mcdat even
+ # if the fasta is there).
+
+ if ((getGlobal("doOverlapBasedTrimming")) &&
+ ((getGlobal('obtOverlapper') eq 'mer') || (! -e "$wrk/0-mercounts/$asm.nmers.obt.fasta"))) {
+ $obtT = runMeryl(getGlobal('obtMerSize'), $obtc, $obtC, getGlobal("obtMerThreshold"), getGlobal("obtMerDistinct"), getGlobal("obtMerTotal"), "obt", $obtD) if (getGlobal("doOverlapBasedTrimming"));
+ } else {
+ print STDERR "No need to run meryl for OBT (OBT is disabled).\n" if (getGlobal("doOverlapBasedTrimming") == 0);
+ print STDERR "No need to run meryl for OBT ($asm.nmers.obt.fasta exists).\n" if (-e "$wrk/0-mercounts/$asm.nmers.obt.fasta");
+ }
+
+ if ((getGlobal('ovlOverlapper') eq 'mer') || (! -e "$wrk/0-mercounts/$asm.nmers.ovl.fasta")) {
+ $ovlT = runMeryl(getGlobal('ovlMerSize'), $ovlc, $ovlC, getGlobal("ovlMerThreshold"), getGlobal("ovlMerDistinct"), getGlobal("ovlMerTotal"), "ovl", $ovlD);
+ } else {
+ print STDERR "No need to run meryl for OVL ($asm.nmers.ovl.fasta exists).\n";
+ }
+
+ if (($obtT > 0) && (getGlobal("obtMerThreshold") ne $obtT) && (getGlobal("doOverlapBasedTrimming"))) {
+ print STDERR "Reset OBT mer threshold from ", getGlobal("obtMerThreshold"), " to $obtT.\n";
+ setGlobal("obtMerThreshold", $obtT);
+ }
+
+ if (($ovlT > 0) && (getGlobal("ovlMerThreshold") ne $ovlT)) {
+ print STDERR "Reset OVL mer threshold from ", getGlobal("ovlMerThreshold"), " to $ovlT.\n";
+ setGlobal("ovlMerThreshold", $ovlT);
+ }
+
+ stopAfter("meryl");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub getUMDOverlapperClearRange ($) {
+ my $dir = shift @_;
+ my $fileName = "$asm.obtClrRange";
+
+ open(F, "ls -1 -d $wrk/$dir/*overlapperRunDir* |");
+ open(G, ">$wrk/$dir/$fileName") or caFailure("failed to write '$wrk/$dir/$fileName'", undef);
+ while (<F>) {
+ chomp;
+
+ open(T, "< $_/revisedOrigTrimsForReads.txt") or caFailure("failed to open '$_/revisedOrigTrimsForReads.txt'", undef);
+ while (<T>) {
+ my @trimData = split(/\s+/,$_);
+ my $uid = $trimData[0];
+ my $bgn = $trimData[1];
+ my $end = $trimData[2];
+
+ if ($bgn < $end) {
+ print G "frg uid $uid obt all $bgn $end\n";
+ } else {
+ print G "frg uid $uid obt all $end $bgn\n";
+ }
+ }
+ close(T);
+ }
+ close(F);
+ close(G);
+
+ return $fileName;
+}
+
+sub UMDoverlapper () {
+ goto alldone if (-d "$wrk/$asm.ovlStore");
+ goto alldone if (getGlobal("ovlOverlapper") ne "umd");
+
+ my $outDir = "1-overlapper";
+ system("mkdir $wrk/$outDir") if (! -d "$wrk/$outDir");
+
+ my $jobID = "0000001";
+ system("mkdir $wrk/$outDir/$jobID") if (! -d "$wrk/$outDir/$jobID");
+
+ my $vi = getGlobal("vectorIntersect");
+
+ #dump the frag file from gkp if it does not exist already
+ # should check if vector clear then dump vec range else dump this range
+ if (defined($vi)) {
+ if (runCommand($wrk, "$bin/gatekeeper -clear VEC -dumpfrg $wrk/$asm.gkpStore 2> $wrk/gatekeeper.err | grep -v 'No source' > $wrk/$asm.vec.frg")) {
+ caFailure("failed to dump gatekeeper store for UMD overlapper", "$wrk/gatekeeper.err");
+ }
+ }
+ elsif ( ! -s "$wrk/$asm.frg" ) {
+ if (runCommand($wrk, "$bin/gatekeeper -dumpfrg $wrk/$asm.gkpStore 2> $wrk/gatekeeper.err | grep -v 'No source' > $wrk/$asm.frg")) {
+ caFailure("failed to dump gatekeeper store for UMD overlapper", "$wrk/gatekeeper.err");
+ }
+ }
+
+ # create a job list (we have only one job for right now)
+ open(SUB, "> $wrk/$outDir/ovljobs.dat") or caFailure("failed to open '$wrk/$outDir/ovljobs.dat'", undef);
+ print SUB "$jobID "; print SUB "\n";
+ print SUB "$jobID "; print SUB "\n";
+ close(SUB);
+
+ # run frg file command
+ #
+ $cmd = "$bin/runUMDOverlapper ";
+ $cmd .= getGlobal("umdOverlapperFlags") . " ";
+
+ # when we have vector clear, pass it to the overlapper, otherwise tell the overlapper to figure it out
+ if (defined($vi)) {
+ $cmd .= "-vector-trim-file $wrk/$asm.vec.frg $wrk/$asm.vec.frg ";
+ } else {
+ $cmd .= "-calculate-trims $wrk/$asm.frg ";
+ }
+
+ $cmd .= "$wrk/$outDir/$jobID/$asm.umd.frg ";
+ $cmd .= " > $wrk/$outDir/$jobID/overlapper.out 2>$wrk/$outDir/$jobID/overlapper.err";
+
+ if (runCommand("$wrk/$outDir", $cmd)) {
+ caFailure("failed to run UMD overlapper", "$wrk/$outDir/$jobID/overlapper.err");
+ }
+
+ my $trimFile = getUMDOverlapperClearRange($outDir);
+ $cmd = "";
+ $cmd .= "$bin/gatekeeper --edit ";
+ $cmd .= "$wrk/$outDir/$trimFile $wrk/$asm.gkpStore";
+ if (runCommand("$wrk/$outDir", $cmd)) {
+ caFailure("failed to update OBT trims", "undef");
+ }
+
+ # now create the binary overlaps
+ $cmd = "";
+ $cmd .= "cat $wrk/$outDir/$jobID/$asm.umd.reliable.overlaps | ";
+ $cmd .= "awk '{print \$1\"\\t\"\$2\"\\t\"\$3\"\\t\"\$4\"\\t\"\$5\"\\t\"\$6\"\\t\"\$7}' | ";
+ $cmd .= "$bin/convertOverlap ";
+ $cmd .= "-b -ovldump ";
+ $cmd .= " -out $wrk/$outDir/$jobID/$jobID.ovb";
+ if (runCommand("$wrk/$outDir", $cmd)) {
+ caFailure("failed to create overlaps", undef);
+ }
+
+ #cleanup
+ rmrf("$asm.vec.frg");
+
+ touch("$wrk/$outDir/$jobID/$jobID.success");
+ stopAfter("overlapper");
+
+ alldone:
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub getFigaroClearRange ($) {
+ my $outDir = shift @_;
+ my $fileName = "$asm.clv";
+
+ # the figaro output is UID,IID CLR_BGN
+ # first reformat is as UID CLR_BGN
+ runCommand("$wrk/$outDir", "awk '{print substr(\$1, 1, index(\$1, \",\")-1)\" \"\$2}' $wrk/$outDir/$asm.vectorcuts > $wrk/$outDir/$asm.clrBgn");
+
+ # sort by UID and join it together with the read end to form the full vector clear range
+ runCommand("$wrk/$outDir", "sort -nk 1 -T $wrk/$outDir $wrk/$outDir/$asm.clrBgn > $wrk/$outDir/$asm.clrBgn.sorted");
+ runCommand("$wrk/$outDir", "join $wrk/$outDir/$asm.clrBgn.sorted $wrk/$asm.untrimmed -o 1.1,1.2,2.3 > $wrk/$outDir/$fileName");
+
+ # clean up
+ rmrf("$outDir/$asm.clrBgn");
+ rmrf("$outDir/$asm.clrBgn.sorted");
+
+ return $fileName;
+}
+
+sub generateFigaroTrim($) {
+ my $outDir = shift @_;
+
+ return if (-e "$wrk/$outDir/trim.success");
+
+ # run command
+ #
+ $cmd = "$bin/figaro ";
+ $cmd .= getGlobal("figaroFlags") . " ";
+ $cmd .= "-F $wrk/$asm.fasta -P $asm ";
+ $cmd .= " > $wrk/$outDir/figaro.out 2>$wrk/$outDir/figaro.err";
+
+ if (runCommand("$wrk/$outDir", $cmd)) {
+ caFailure("figaro died", "$wrk/$outDir/figaro.err");
+ }
+
+ # update the gkpStore with newly computed clear ranges
+ return getFigaroClearRange($outDir);
+}
+
+sub getUMDTrimClearRange($) {
+ my $outDir = shift @_;
+ my $fileName = "$asm.clv";
+
+ # the umd output is CLR_BGN (in the same order as the input)
+ # to join it with the UID we first number both the list of UIDs in the fasta file and the CLR_BGN
+ runCommand("$wrk/$outDir", "cat $wrk/$asm.fasta | grep \">\" | awk '{print NR\" \"substr(\$1, 2, index(\$1, \",\")-2)}' > $wrk/$outDir/$asm.numberedUids");
+ runCommand("$wrk/$outDir", "awk '{print NR\" \"\$0}' $asm.vectorcuts > $asm.numberedCuts");
+
+ # now we join them together
+ runCommand("$wrk/$outDir", "join $wrk/$outDir/$asm.numberedUids $wrk/$outDir/$asm.numberedCuts -o 1.2,2.2 > $wrk/$outDir/$asm.clrBgn");
+
+ # now we can join together the UID CLR_BGN with the read-end information for the full clear range
+ runCommand("$wrk/$outDir", "sort -nk 1 -T $wrk/$outDir $wrk/$outDir/$asm.clrBgn > $wrk/$outDir/$asm.clrBgn.sorted");
+ runCommand("$wrk/$outDir", "join $wrk/$outDir/$asm.clrBgn.sorted $wrk/$asm.untrimmed -o 1.1,1.2,2.3 > $wrk/$outDir/$fileName");
+
+ # clean up
+ rmrf("$outDir/$asm.numberedUids");
+ rmrf("$outDir/$asm.numberedCuts");
+ rmrf("$outDir/$asm.clrBgn");
+ rmrf("$outDir/$asm.clrBgn.sorted");
+ rmrf("$outDir/vectorTrimIntermediateFile001.*");
+
+ return $fileName;
+}
+
+sub generateUMDTrim($) {
+ my $outDir = shift @_;
+
+ return if (-e "$wrk/$outDir/trim.success");
+
+ # run command
+ #
+ $cmd = "$bin/dataWorkReduced/findVectorTrimPoints.perl ";
+ $cmd .= "$wrk/$asm.fasta $wrk/$outDir/$asm.vectorcuts ";
+ $cmd .= " > $wrk/$outDir/umd.out 2>$wrk/$outDir/umd.err";
+
+ if (runCommand("$wrk/$outDir", $cmd)) {
+ caFailure("UMD overlapper dataWorkReduced/findVectorTrimPoints.perl died",
+ "$wrk/$outDir/umd.err");
+ }
+
+ return getUMDTrimClearRange($outDir);
+}
+
+sub generateVectorTrim ($) {
+ my $vi = getGlobal("vectorIntersect");
+ my $trimmer = getGlobal("vectorTrimmer");
+ my $outDir = "0-preoverlap";
+ my $trimFile = undef;
+
+ # when vector insersect is specified or no external trimming is requested, do nothing
+ return if (defined($vi));
+ return if ($trimmer eq "ca");
+ return if (-e "$wrk/$outDir/trim.success");
+
+ #dump the fasta file from gkp
+ if ( ! -e "$wrk/$asm.fasta" ) {
+ if (runCommand($wrk, "$bin/gatekeeper -dumpfastaseq -clear UNTRIM $wrk/$asm.gkpStore 2> $wrk/$outDir/gatekeeper.err > $wrk/$asm.fasta")) {
+ caFailure("failed to dump gatekeeper store for figaro trimmer",
+ "$wrk/$outDir/gatekeeper.err");
+ }
+ }
+ #dump the clr range
+ if ( ! -e "$wrk/$asm.untrimmed" ) {
+ if (runCommand($wrk, "$bin/gatekeeper -dumpfragments -tabular -clear UNTRIM $wrk/$asm.gkpStore 2> $wrk/$outDir/gatekeeper.err | grep -v 'UID' |awk '{print \$1\" \"\$12\" \"\$13}' | sort -nk 1 -T $wrk/ > $wrk/$asm.untrimmed")) {
+ caFailure("failed to dump gatekeeper quality trim points for figaro trimmer",
+ "$wrk/$outDir/gatekeeper.err");
+ }
+ }
+
+ if ($trimmer eq "figaro") {
+ $trimFile = generateFigaroTrim($outDir);
+ } elsif($trimmer eq "umd") {
+ $trimFile = generateUMDTrim($outDir);
+ } else {
+ caFailure("unknown vector trimmer $trimmer", undef);
+ }
+
+ # set the global vector trim file so that the subsequent code will update the gkp for us
+ setGlobal("vectorIntersect", "$wrk/$outDir/$trimFile");
+
+ #cleanup
+ rmrf("$asm.fasta");
+ rmrf("$asm.untrimmed");
+
+ touch("$wrk/$outDir/trim.success");
+
+ return;
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub findMBTFailures ($) {
+ my $mbtJobs = shift @_;
+ my $failures = 0;
+
+ for (my $i=1; $i<=$mbtJobs; $i++) {
+ my $jobid = substr("0000" . $i, -4);
+ if (-e "$wrk/0-mertrim/$asm.$jobid.merTrim.WORKING") {
+ #print STDERR "FAILURE $wrk/0-mertrim/$asm.$jobid.merTrim.WORKING\n";
+ $failures++;
+ }
+ }
+
+ return $failures;
+}
+
+sub findMBTSuccess ($) {
+ my $mbtJobs = shift @_;
+ my $successes= 0;
+
+ for (my $i=1; $i<=$mbtJobs; $i++) {
+ my $jobid = substr("0000" . $i, -4);
+ if (-e "$wrk/0-mertrim/$asm.$jobid.merTrim") {
+ #print STDERR "SUCCESS $wrk/0-mertrim/$asm.$jobid.merTrim\n";
+ $successes++;
+ }
+ }
+
+ return($successes == $mbtJobs);
+}
+
+
+sub merTrim {
+
+ return if (getGlobal("doOverlapBasedTrimming") == 0);
+ return if (getGlobal("ovlOverlapper") eq "umd");
+
+ # Skip mer based trimming if it is done, or if the ovlStore already exists.
+ #
+ goto alldone if (-e "$wrk/0-mertrim/mertrim.success");
+ goto alldone if (-d "$wrk/$asm.ovlStore");
+
+ system("mkdir $wrk/0-mertrim") if (! -d "$wrk/0-mertrim");
+
+ # Decide if any libraries request mer based trimming. There is a simpler method to get
+ # this (see unitigger.pl), but for unity with overlapTrim.pl, we count the number
+ # of libraries that want MBT.
+ #
+ my $mbtNeeded = 0;
+
+ open(F, "$bin/gatekeeper -nouid -dumplibraries $wrk/$asm.gkpStore |");
+ while (<F>) {
+ $mbtNeeded++ if (m/doTrim_initialMerBased.*=.*1/);
+ }
+ close(F);
+
+ if ($mbtNeeded == 0) {
+ touch("$wrk/0-mertrim/mertrim.success");
+ goto alldone;
+ }
+
+ #
+ # Run mer trim on the grid.
+ #
+
+ my $mbtBatchSize = getGlobal("mbtBatchSize");
+ my $mbtJobs = int($numFrags / $mbtBatchSize) + (($numFrags % $mbtBatchSize == 0) ? 0 : 1);
+
+ my $merSize = getGlobal("obtMerSize");
+ my $merComp = 0; # getGlobal("merCompression"); -- DOES NOT WORK WITH merTrim
+
+ my $mbtThreads = getGlobal("mbtThreads");
+
+ my $taskID = getGlobal("gridTaskID");
+ my $submitTaskID = getGlobal("gridArraySubmitID");
+
+ runMeryl($merSize, $merComp, "-C", "auto", undef, undef, "mbt", 0);
+
+ if (! -e "$wrk/0-mertrim/mertrim.sh") {
+ open(F, "> $wrk/0-mertrim/mertrim.sh") or caFailure("can't open '$wrk/0-mertrim/mertrim.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F "perl='/usr/bin/env perl'\n";
+ print F "\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %04d \$jobid`\n";
+ print F "minid=`expr \$jobid \\* $mbtBatchSize - $mbtBatchSize + 1`\n";
+ print F "maxid=`expr \$jobid \\* $mbtBatchSize`\n";
+ print F "\n";
+ print F "if [ \$maxid -gt $numFrags ] ; then\n";
+ print F " maxid=$numFrags\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ \$minid -gt \$maxid ] ; then\n";
+ print F " echo Job partitioning error -- minid=\$minid maxid=\$maxid.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ -e $wrk/0-mertrim/$asm.\$jobid.merTrim ]; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+
+ print F "AS_OVL_ERROR_RATE=" , getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=" , getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=" , getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+
+ print F getBinDirectoryShellCode();
+
+ print F "\$bin/merTrim \\\n";
+ print F " -b \$minid \\\n";
+ print F " -e \$maxid \\\n";
+ print F " -g $wrk/$asm.gkpStore \\\n";
+ print F " -mCillumina \\\n" if (getGlobal("mbtIlluminaAdapter") ne "0");
+ print F " -mC454 \\\n" if (getGlobal("mbt454Adapter") ne "0");
+ print F " -t $mbtThreads \\\n";
+ print F " -m $merSize \\\n";
+ print F " -mc $wrk/0-mercounts/$asm-C-ms$merSize-cm$merComp \\\n";
+ print F " -o $wrk/0-mertrim/$asm.\$jobid.merTrim.WORKING \\\n";
+ print F " > $wrk/0-mertrim/$asm.\$jobid.err 2>&1 \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/0-mertrim/$asm.\$jobid.merTrim.WORKING $wrk/0-mertrim/$asm.\$jobid.merTrim\n";
+ print F "\n";
+ print F "exit 0\n";
+ close(F);
+
+ system("chmod +x $wrk/0-mertrim/mertrim.sh");
+ }
+
+ stopBefore("initialTrim", undef);
+
+ # Don't try to rerun failures.
+ #
+ # FAILUREHELPME
+ #
+ if (findMBTFailures($mbtJobs) > 0) {
+ caFailure("merTrim failed. See *.err in $wrk/0-mertrim", undef);
+ }
+
+ # Submit to the grid (or tell the user to do it), or just run
+ # things here
+ #
+ if (findMBTSuccess($mbtJobs) == 0) {
+ if (getGlobal("useGrid") && getGlobal("mbtOnGrid")) {
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeMerTrim = getGlobal("sgeMerTrim");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("mbt_$asm$sgeName", $mbtJobs);
+ my $arrayOpt = getGridArrayOption("mbt_$asm$sgeName", $mbtJobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeMerTrim $nameOption \"$jobName\" $arrayOpt \\\n";
+ $SGE .= " $outputOption $wrk/0-mertrim/$asm.merTrim.$submitTaskID.sge.err \\\n";
+ $SGE .= " $wrk/0-mertrim/mertrim.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ for (my $i=1; $i<=$mbtJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ schedulerSubmit("$wrk/0-mertrim/mertrim.sh $i > $wrk/0-mertrim/$asm.$out.err 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses(getGlobal("mbtConcurrency"));
+ schedulerFinish();
+ }
+ }
+
+ # Make sure everything finished ok.
+ #
+ # FAILUREHELPME
+ #
+ if (findMBTFailures($mbtJobs) > 0) {
+ caFailure("merTrim failed. See *.err in $wrk/0-mertrim", undef);
+ }
+
+
+ if (runCommand($wrk, "find -L $wrk/0-mertrim -name \\*.merTrim -print | sort > $wrk/0-mertrim/$asm.merTrim.list")) {
+ caFailure("failed to generate a list of all the merTrim results", undef);
+ }
+
+ {
+ $cmd = "$bin/merTrimApply \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -L $wrk/0-mertrim/$asm.merTrim.list \\\n";
+ $cmd .= " -l $wrk/0-mertrim/$asm.merTrim.log \\\n";
+ $cmd .= " > $wrk/0-mertrim/$asm.merTrimApply.err 2>&1";
+
+ if (runCommand($wrk, $cmd)) {
+ rename "$wrk/0-mertrim/$asm.merTrim.log", "$wrk/0-mertrim/$asm.merTrim.log.FAILED";
+ caFailure("merTrimApply failed", "$wrk/0-mertrim/$asm.merTrimApply.err");
+ }
+ }
+
+ touch("$wrk/0-mertrim/mertrim.success");
+
+ alldone:
+ #stopAfter("overlapBasedTrimming");
+ #stopAfter("OBT");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+
+sub findOvermerryFailures ($$) {
+ my $outDir = shift @_;
+ my $ovmJobs = shift @_;
+ my $failures = 0;
+
+ for (my $i=1; $i<=$ovmJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ if (-e "$wrk/$outDir/seeds/$out.ovm.WORKING.gz") {
+ $failures++;
+ }
+ }
+
+ return $failures;
+}
+
+sub findOvermerrySuccess ($$) {
+ my $outDir = shift @_;
+ my $ovmJobs = shift @_;
+ my $successes= 0;
+
+ for (my $i=1; $i<=$ovmJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ if (-e "$wrk/$outDir/seeds/$out.ovm.gz") {
+ $successes++;
+ }
+ }
+
+ return($successes == $ovmJobs);
+}
+
+sub findOlapFromSeedsFailures ($$) {
+ my $outDir = shift @_;
+ my $olpJobs = shift @_;
+ my $failures = 0;
+
+ for (my $i=1; $i<=$olpJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ if (-e "$wrk/$outDir/olaps/$out.ovb.WORKING.gz") {
+ $failures++;
+ }
+ }
+
+ return $failures;
+}
+
+sub findOlapFromSeedsSuccess ($$) {
+ my $outDir = shift @_;
+ my $olpJobs = shift @_;
+ my $successes= 0;
+
+ for (my $i=1; $i<=$olpJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ if (-e "$wrk/$outDir/olaps/$out.ovb.gz") {
+ $successes++;
+ }
+ }
+
+ return $successes == $olpJobs;
+}
+
+
+sub merOverlapper($) {
+ my $isTrim = shift @_;
+
+ return if (-d "$wrk/$asm.ovlStore");
+ return if (-d "$wrk/$asm.obtStore") && ($isTrim eq "trim");
+
+ caFailure("mer overlapper detected no fragments", undef) if ($numFrags == 0);
+ caFailure("mer overlapper doesn't know if trimming or assembling", undef) if (!defined($isTrim));
+
+ my ($outDir, $ovlOpt, $merSize, $merComp, $merType, $merylNeeded);
+
+ # Set directories and parameters for either 'trimming' or 'real'
+ # overlaps.
+
+ if ($isTrim eq "trim") {
+ $outDir = "0-overlaptrim-overlap";
+ $ovlOpt = "-G";
+ $merSize = getGlobal("obtMerSize");
+ $merComp = getGlobal("merCompression");
+ $merType = "obt";
+ $merylNeeded = (getGlobal("obtMerThreshold") =~ m/auto/) ? 1 : 0;
+ } else {
+ $outDir = "1-overlapper";
+ $ovlOpt = "";
+ $merSize = getGlobal("ovlMerSize");
+ $merComp = getGlobal("merCompression");
+ $merType = "ovl";
+ $merylNeeded = (getGlobal("ovlMerThreshold") =~ m/auto/) ? 1 : 0;
+ }
+
+ system("mkdir $wrk/$outDir") if (! -d "$wrk/$outDir");
+ system("mkdir $wrk/$outDir/seeds") if (! -d "$wrk/$outDir/seeds");
+ system("mkdir $wrk/$outDir/olaps") if (! -d "$wrk/$outDir/olaps");
+
+ # Make the directory (to hold the corrections output) and claim
+ # that fragment correction is all done. after this, the rest of
+ # the fragment/overlap correction pipeline Just Works.
+ #
+ system("mkdir $wrk/3-overlapcorrection") if ((! -d "$wrk/3-overlapcorrection") && ($isTrim ne "trim"));
+
+ my $ovmBatchSize = getGlobal("merOverlapperSeedBatchSize");
+ my $ovmJobs = int($numFrags / $ovmBatchSize) + (($numFrags % $ovmBatchSize == 0) ? 0 : 1);
+
+ my $olpBatchSize = getGlobal("merOverlapperExtendBatchSize");
+ my $olpJobs = int($numFrags / $olpBatchSize) + (($numFrags % $olpBatchSize == 0) ? 0 : 1);
+
+ # Need mer counts, unless there is only one partition.
+ meryl() if (($ovmJobs > 1) || ($merylNeeded));
+
+ my $taskID = getGlobal("gridTaskID");
+ my $submitTaskID = getGlobal("gridArraySubmitID");
+
+ # Create overmerry and olap-from-seeds jobs
+ #
+ open(F, "> $wrk/$outDir/overmerry.sh") or caFailure("can't open '$wrk/$outDir/overmerry.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %04d \$jobid`\n";
+ print F "minid=`expr \$jobid \\* $ovmBatchSize - $ovmBatchSize + 1`\n";
+ print F "maxid=`expr \$jobid \\* $ovmBatchSize`\n";
+ print F "runid=\$\$\n";
+ print F "\n";
+ print F "if [ \$maxid -gt $numFrags ] ; then\n";
+ print F " maxid=$numFrags\n";
+ print F "fi\n";
+ print F "if [ \$minid -gt \$maxid ] ; then\n";
+ print F " echo Job partitioning error -- minid=\$minid maxid=\$maxid.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=", getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=", getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=", getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+ print F "\n";
+ print F "if [ ! -d $wrk/$outDir/seeds ]; then\n";
+ print F " mkdir $wrk/$outDir/seeds\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ -e $wrk/$outDir/seeds/\$jobid.ovm.gz ]; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F getBinDirectoryShellCode();
+ print F "\$bin/overmerry \\\n";
+ print F " -g $wrk/$asm.gkpStore \\\n";
+ if ($ovmJobs > 1) {
+ print F " -mc $wrk/0-mercounts/$asm-C-ms$merSize-cm$merComp \\\n";
+ print F " -tb \$minid -te \$maxid \\\n";
+ print F " -qb \$minid \\\n";
+ }
+ print F " -m $merSize \\\n";
+ print F " -c $merComp \\\n";
+ print F " -T ", getGlobal("obtMerThreshold"), " \\\n" if ($isTrim eq "trim");
+ print F " -T ", getGlobal("ovlMerThreshold"), " \\\n" if ($isTrim ne "trim");
+ print F " -t " . getGlobal("merOverlapperThreads") . "\\\n";
+ print F " -o $wrk/$outDir/seeds/\$jobid.ovm.WORKING.gz \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/$outDir/seeds/\$jobid.ovm.WORKING.gz $wrk/$outDir/seeds/\$jobid.ovm.gz\n";
+ close(F);
+
+ system("chmod +x $wrk/$outDir/overmerry.sh");
+
+
+
+
+ open(F, "> $wrk/$outDir/olap-from-seeds.sh") or caFailure("can't open '$wrk/$outDir/olap-from-seeds.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %04d \$jobid`\n";
+ print F "minid=`expr \$jobid \\* $olpBatchSize - $olpBatchSize + 1`\n";
+ print F "maxid=`expr \$jobid \\* $olpBatchSize`\n";
+ print F "runid=\$\$\n";
+ print F "\n";
+ print F "if [ \$maxid -gt $numFrags ] ; then\n";
+ print F " maxid=$numFrags\n";
+ print F "fi\n";
+ print F "if [ \$minid -gt \$maxid ] ; then\n";
+ print F " echo Job partitioning error -- minid=\$minid maxid=\$maxid.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=", getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=", getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=", getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+ print F "\n";
+ print F "if [ ! -d $wrk/$outDir/olaps ]; then\n";
+ print F " mkdir $wrk/$outDir/olaps\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ -e $wrk/$outDir/olaps/\$jobid.ovb.gz ]; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F getBinDirectoryShellCode();
+ print F "\$bin/olap-from-seeds \\\n";
+ print F " -a -b \\\n";
+ print F " -t " . getGlobal("merOverlapperThreads") . "\\\n";
+ print F " -S $wrk/$outDir/$asm.merStore \\\n";
+ if ($isTrim eq "trim") {
+ print F " -G \\\n"; # Trim only
+ print F " -o $wrk/$outDir/olaps/\$jobid.ovb.WORKING.gz \\\n";
+ print F " $wrk/$asm.gkpStore \\\n";
+ print F " \$minid \$maxid \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/$outDir/olaps/\$jobid.ovb.WORKING.gz $wrk/$outDir/olaps/\$jobid.ovb.gz\n";
+ } else {
+ print F " -w \\\n" if (getGlobal("merOverlapperCorrelatedDiffs"));
+ print F " -c $wrk/3-overlapcorrection/\$jobid.frgcorr.WORKING \\\n";
+ print F " -o $wrk/$outDir/olaps/\$jobid.ovb.WORKING.gz \\\n";
+ print F " $wrk/$asm.gkpStore \\\n";
+ print F " \$minid \$maxid \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/$outDir/olaps/\$jobid.ovb.WORKING.gz $wrk/$outDir/olaps/\$jobid.ovb.gz \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/3-overlapcorrection/\$jobid.frgcorr.WORKING $wrk/3-overlapcorrection/\$jobid.frgcorr\n";
+ }
+ close(F);
+
+ system("chmod +x $wrk/$outDir/olap-from-seeds.sh");
+
+
+
+
+ if (! -e "$wrk/$outDir/$asm.merStore") {
+
+ # To prevent infinite loops -- stop now if the overmerry script
+ # exists. This will unfortunately make restarting from transient
+ # failures non-trivial.
+ #
+ # FAILUREHELPME
+ #
+ if (findOvermerryFailures($outDir, $ovmJobs) > 0) {
+ caFailure("overmerry failed. See *.err in $wrk/$outDir", undef);
+ }
+
+ # Submit to the grid (or tell the user to do it), or just run
+ # things here
+ #
+ if (findOvermerrySuccess($outDir, $ovmJobs) == 0) {
+ if (getGlobal("useGrid") && getGlobal("ovlOnGrid")) {
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeOverlap = getGlobal("sgeMerOverlapSeed");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("mer_$asm$sgeName", $ovmJobs);
+ my $arrayOpt = getGridArrayOption("mer_$asm$sgeName", $ovmJobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
+ $SGE .= " $outputOption $wrk/$outDir/seeds/$submitTaskID.err \\\n";
+ $SGE .= " $wrk/$outDir/overmerry.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ for (my $i=1; $i<=$ovmJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ schedulerSubmit("$wrk/$outDir/overmerry.sh $i > $wrk/$outDir/seeds/$out.err 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses(getGlobal("merOverlapperSeedConcurrency"));
+ schedulerFinish();
+ }
+ }
+
+ # Make sure everything finished ok.
+ #
+ # FAILUREHELPME
+ #
+ if (findOvermerryFailures($outDir, $ovmJobs) > 0) {
+ caFailure("overmerry failed. See *.err in $wrk/$outDir", undef);
+ }
+
+
+ if (runCommand($wrk, "find -L $wrk/$outDir/seeds \\( -name \\*ovm.gz -or -name \\*ovm \\) -print > $wrk/$outDir/$asm.merStore.list")) {
+ caFailure("failed to generate a list of all the overlap files", undef);
+ }
+
+ #$cmd = "$bin/overlapStore";
+ #$cmd .= " -c $wrk/$outDir/$asm.merStore.WORKING";
+ #$cmd .= " -g $wrk/$asm.gkpStore";
+ #$cmd .= " -M " . getGlobal("ovlStoreMemory");
+ #$cmd .= " -L $wrk/$outDir/$asm.merStore.list";
+ #$cmd .= " > $wrk/$outDir/$asm.merStore.err 2>&1";
+
+ $cmd = "$bin/overlapStoreBuild";
+ $cmd .= " -o $wrk/$outDir/$asm.merStore.WORKING";
+ $cmd .= " -g $wrk/$asm.gkpStore";
+ $cmd .= " -M " . getGlobal("ovlStoreMemory");
+ $cmd .= " -L $wrk/$outDir/$asm.merStore.list";
+ $cmd .= " > $wrk/$outDir/$asm.merStore.err 2>&1";
+
+ if (runCommand($wrk, $cmd)) {
+ caFailure("overlap store building failed", "$wrk/$outDir/$asm.merStore.err");
+ }
+
+ rename "$wrk/$outDir/$asm.merStore.WORKING", "$wrk/$outDir/$asm.merStore";
+
+ if (getGlobal("saveOverlaps") == 0) {
+ open(F, "< $wrk/$outDir/$asm.merStore.list");
+ while (<F>) {
+ chomp;
+ unlink $_;
+ }
+ close(F);
+ }
+
+ rmrf("$outDir/$asm.merStore.list");
+ rmrf("$outDir/$asm.merStore.err");
+ }
+
+
+ # To prevent infinite loops -- stop now if the overmerry script
+ # exists. This will unfortunately make restarting from transient
+ # failures non-trivial.
+ #
+ # FAILUREHELPME
+ #
+ if (findOlapFromSeedsFailures($outDir, $olpJobs) > 0) {
+ caFailure("olap-from-seeds failed. See *.err in $wrk/$outDir.", undef);
+ }
+
+ # Submit to the grid (or tell the user to do it), or just run
+ # things here
+ #
+ if (findOlapFromSeedsSuccess($outDir, $olpJobs) == 0) {
+ if (getGlobal("useGrid") && getGlobal("ovlOnGrid")) {
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeOverlap = getGlobal("sgeMerOverlapExtend");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("olp_$asm$sgeName", $olpJobs);
+ my $arrayOpt = getGridArrayOption("mer_$asm$sgeName", $ovmJobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
+ $SGE .= " $outputOption $wrk/$outDir/olaps/$submitTaskID.err \\\n";
+ $SGE .= " $wrk/$outDir/olap-from-seeds.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ for (my $i=1; $i<=$olpJobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ schedulerSubmit("$wrk/$outDir/olap-from-seeds.sh $i > $wrk/$outDir/olaps/$out.err 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses(getGlobal("merOverlapperExtendConcurrency"));
+ schedulerFinish();
+ }
+ }
+
+ # Make sure everything finished ok.
+ #
+ # FAILUREHELPME
+ #
+ if (findOlapFromSeedsFailures($outDir, $olpJobs) > 0) {
+ caFailure("olap-from-seeds failed. See *.err in $wrk/$outDir.", undef);
+ }
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+
+
+sub createOverlapJobs($) {
+ my $isTrim = shift @_;
+
+ if (-d "$wrk/$asm.ovlStore") {
+ stopAfter("meryl");
+ }
+ return if (-d "$wrk/$asm.ovlStore");
+
+ caFailure("overlapper detected no fragments", undef) if ($numFrags == 0);
+ caFailure("overlapper needs to know if trimming or assembling", undef) if (!defined($isTrim));
+
+ my $ovlThreads = getGlobal("ovlThreads");
+ my $ovlHashBits = getGlobal("ovlHashBits");
+ my $ovlHashLoad = getGlobal("ovlHashLoad");
+
+ my $outDir = "1-overlapper";
+ my $ovlOpt = "";
+ my $merSize = getGlobal("ovlMerSize");
+ my $merComp = getGlobal("merCompression");
+ my $overlap = "overlapInCore";
+
+ if ($isTrim eq "trim") {
+ $outDir = "0-overlaptrim-overlap";
+ $ovlOpt = "-G";
+ $merSize = getGlobal("obtMerSize");
+ $overlap = "overlapInCore";
+ }
+
+ system("mkdir $wrk/$outDir") if (! -d "$wrk/$outDir");
+
+ if (-e "$wrk/$outDir/overlap.sh") {
+ stopAfter("meryl");
+ }
+ return if (-e "$wrk/$outDir/overlap.sh");
+
+ # umd overlapper here
+ #
+ if (getGlobal("ovlOverlapper") eq "umd") {
+ # For Sergey:
+ #
+ # UMDoverlapper() needs to dump the gkpstore, run UMD, build
+ # the ovlStore and update gkpStore with new clear ranges.
+ # The explicit call to UMDoverlapper in main() can then go away.
+ # OBT is smart enough to disable itself if umd is enabled.
+ #
+ UMDoverlapper();
+ return;
+ }
+
+ # mer overlapper here
+ #
+ if ((($isTrim eq "trim") && (getGlobal("obtOverlapper") eq "mer")) ||
+ (($isTrim ne "trim") && (getGlobal("ovlOverlapper") eq "mer"))) {
+ merOverlapper($isTrim);
+ return;
+ }
+
+ my $hashLibrary = ($isTrim eq "trim") ? getGlobal("obtHashLibrary") : getGlobal("ovlHashLibrary");
+ my $refLibrary = ($isTrim eq "trim") ? getGlobal("obtRefLibrary") : getGlobal("ovlRefLibrary");
+ my $checkLibrary = ($isTrim eq "trim") ? getGlobal("obtCheckLibrary") : getGlobal("ovlCheckLibrary");
+
+ # To prevent infinite loops -- stop now if the overlap script
+ # exists. This will unfortunately make restarting from transient
+ # failures non-trivial.
+ #
+ # FAILUREHELPME
+ #
+ caFailure("overlapper failed\nmanual restart needed to prevent infinite loops\nremove file '$wrk/$outDir/overlap.sh'", undef) if (-e "$wrk/$outDir/overlap.sh");
+
+ meryl();
+
+ my $taskID = getGlobal("gridTaskID");
+ my $submitTaskID = getGlobal("gridArraySubmitID");
+
+ # We make a giant job array for this -- we need to know hashBeg,
+ # hashEnd, refBeg and refEnd -- from that we compute batchName
+ # and jobName.
+ #
+ # ovlopts.pl returns the batch name ($batchName), the job name
+ # ($jobName) and options to pass to overlap (-h $hashBeg-$hashEnd
+ # -r $refBeg-$refEnd). From those, we can construct the command
+ # to run.
+ #
+ open(F, "> $wrk/$outDir/overlap.sh") or caFailure("can't open '$wrk/$outDir/overlap.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F "perl='/usr/bin/env perl'\n";
+ print F "\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "bat=`head -n \$jobid $wrk/$outDir/ovlbat | tail -n 1`\n";
+ print F "job=`head -n \$jobid $wrk/$outDir/ovljob | tail -n 1`\n";
+ print F "opt=`head -n \$jobid $wrk/$outDir/ovlopt | tail -n 1`\n";
+ print F "jid=\$\$\n";
+ print F "\n";
+ print F "if [ ! -d $wrk/$outDir/\$bat ]; then\n";
+ print F " mkdir $wrk/$outDir/\$bat\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ -e $wrk/$outDir/\$bat/\$job.ovb.gz ]; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ x\$bat = x ]; then\n";
+ print F " echo Error: Job index out of range.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=" , getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=" , getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=" , getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+
+ print F getBinDirectoryShellCode();
+
+ print F "\$bin/$overlap $ovlOpt --hashbits $ovlHashBits --hashload $ovlHashLoad -t $ovlThreads \\\n";
+ print F " \$opt \\\n";
+ print F " -k $merSize \\\n";
+ print F " -k $wrk/0-mercounts/$asm.nmers.obt.fasta \\\n" if ($isTrim eq "trim");
+ print F " -k $wrk/0-mercounts/$asm.nmers.ovl.fasta \\\n" if ($isTrim ne "trim");
+ print F " -o $wrk/$outDir/\$bat/\$job.ovb.WORKING.gz \\\n";
+ print F " -H $hashLibrary \\\n" if ($hashLibrary ne "0");
+ print F " -R $refLibrary \\\n" if ($refLibrary ne "0");
+ print F " $wrk/$asm.gkpStore \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/$outDir/\$bat/\$job.ovb.WORKING.gz $wrk/$outDir/\$bat/\$job.ovb.gz\n";
+ print F "\n";
+ print F "exit 0\n";
+ close(F);
+
+ system("chmod +x $wrk/$outDir/overlap.sh");
+
+
+ my $jobs = 0;
+ my $batchName = "";
+ my $jobName = "";
+
+ {
+ my $cmd;
+
+ my $ovlHashBlockLength = getGlobal("ovlHashBlockLength");
+ my $ovlHashBlockSize = 0;
+ my $ovlRefBlockSize = getGlobal("ovlRefBlockSize");
+ my $ovlRefBlockLength = getGlobal("ovlRefBlockLength");
+
+ if (($ovlRefBlockSize > 0) && ($ovlRefBlockLength > 0)) {
+ caFailure("can't set both ovlRefBlockSize and ovlRefBlockLength", undef);
+ }
+
+ $cmd = "$bin/overlap_partition \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -bl $ovlHashBlockLength \\\n";
+ $cmd .= " -bs $ovlHashBlockSize \\\n";
+ $cmd .= " -rs $ovlRefBlockSize \\\n";
+ $cmd .= " -rl $ovlRefBlockLength \\\n";
+ $cmd .= " -H $hashLibrary \\\n" if ($hashLibrary ne "0");
+ $cmd .= " -R $refLibrary \\\n" if ($refLibrary ne "0");
+ $cmd .= " -C \\\n" if (!$checkLibrary);
+ $cmd .= " -o $wrk/$outDir \\\n";
+ $cmd .= "> $wrk/$outDir/overlap_partition.err 2>&1";
+
+ if (runCommand($wrk, $cmd)) {
+ caFailure("failed partition for overlapper", undef);
+ }
+
+ open(F, "< $wrk/$outDir/ovlbat") or caFailure("failed partition for overlapper: no ovlbat file found", undef);
+ my @bat = <F>;
+ close(F);
+
+ open(F, "< $wrk/$outDir/ovljob") or caFailure("failed partition for overlapper: no ovljob file found", undef);
+ my @job = <F>;
+ close(F);
+
+ $jobs = scalar(@job);
+ $batchName = $bat[$jobs-1]; chomp $batchName;
+ $jobName = $job[$jobs-1]; chomp $jobName;
+ }
+
+ print STDERR "Created $jobs overlap jobs. Last batch '$batchName', last job '$jobName'.\n";
+
+ # Submit to the grid (or tell the user to do it), or just run
+ # things here
+ #
+ if (getGlobal("useGrid") && getGlobal("ovlOnGrid")) {
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeOverlap = getGlobal("sgeOverlap");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("ovl_$asm$sgeName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovl_$asm$sgeName", $jobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
+ $SGE .= " $outputOption $wrk/$outDir/$submitTaskID.out \\\n";
+ $SGE .= " $wrk/$outDir/overlap.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ for (my $i=1; $i<=$jobs; $i++) {
+ my $out = substr("000000" . $i, -6);
+ schedulerSubmit("$wrk/$outDir/overlap.sh $i > $wrk/$outDir/$out.out 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses(getGlobal("ovlConcurrency"));
+ schedulerFinish();
+ }
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+# Check that the overlapper jobs properly executed. If not,
+# complain, but don't help the user fix things.
+
+
+sub checkOverlapper ($) {
+ my $isTrim = shift @_;
+
+ my $outDir = "1-overlapper";
+ my $ovlOpt = "";
+
+ if ($isTrim eq "trim") {
+ $outDir = "0-overlaptrim-overlap";
+ $ovlOpt = "-G";
+ }
+
+ my $failedJobs = 0;
+ my $failureMessage = "";
+
+ open(B, "< $wrk/$outDir/ovlbat") or caFailure("failed to open '$wrk/$outDir/ovlbat'", undef);
+ open(J, "< $wrk/$outDir/ovljob") or caFailure("failed to open '$wrk/$outDir/ovljob'", undef);
+
+ while (!eof(B) && !eof(J)) {
+ my $b = <B>; chomp $b;
+ my $j = <J>; chomp $j;
+
+ if ((! -e "$wrk/$outDir/$b/$j.ovb.gz") &&
+ (! -e "$wrk/$outDir/$b/$j.ovb")) {
+ $failureMessage .= "ERROR: Overlap job $wrk/$outDir/$b/$j FAILED.\n";
+ $failedJobs++;
+ }
+ }
+
+ if (!eof(B) || !eof(J)) {
+ print STDERR "Partitioning error; '$wrk/$outDir/ovlbat' and '$wrk/$outDir/ovljob' have extra lines.\n";
+ }
+
+ # FAILUREHELPME
+ #
+ $failureMessage .= "\n$failedJobs overlapper jobs failed";
+ caFailure($failureMessage, undef) if ($failedJobs);
+}
+
+
+sub checkMerOverlapper ($) {
+ my $isTrim = shift @_;
+
+ my $outDir = "1-overlapper";
+
+ if ($isTrim eq "trim") {
+ $outDir = "0-overlaptrim-overlap";
+ }
+
+ my $batchSize = getGlobal("merOverlapperExtendBatchSize");
+ my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
+ my $failedJobs = 0;
+
+ for (my $i=1; $i<=$jobs; $i++) {
+ my $job = substr("0000" . $i, -4);
+
+ if ((! -e "$wrk/$outDir/olaps/$job.ovb.gz") &&
+ (! -e "$wrk/$outDir/olaps/$job.ovb")) {
+ print STDERR "$wrk/$outDir/olaps/$job failed.\n";
+ $failedJobs++;
+ }
+ }
+
+ caFailure("$failedJobs overlapper jobs failed", undef) if ($failedJobs);
+}
+
+
+sub checkOverlap {
+ my $isTrim = shift @_;
+
+ caFailure("overlap checker needs to know if trimming or assembling", undef) if (!defined($isTrim));
+
+ if ($isTrim eq "trim") {
+ return if (-d "$wrk/$asm.obtStore");
+ if (getGlobal("obtOverlapper") eq "ovl") {
+ checkOverlapper($isTrim);
+ } elsif (getGlobal("obtOverlapper") eq "ovm") {
+ checkOverlapper($isTrim);
+ } elsif (getGlobal("obtOverlapper") eq "mer") {
+ checkMerOverlapper($isTrim);
+ } elsif (getGlobal("obtOverlapper") eq "umd") {
+ caFailure("checkOverlap() wanted to check umd overlapper for obt?\n", undef);
+ } else {
+ caFailure("checkOverlap() unknown obt overlapper?\n", undef);
+ }
+ } else {
+ return if (-d "$wrk/$asm.ovlStore");
+ if (getGlobal("ovlOverlapper") eq "ovl") {
+ checkOverlapper($isTrim);
+ } elsif (getGlobal("ovlOverlapper") eq "ovm") {
+ checkOverlapper($isTrim);
+ } elsif (getGlobal("ovlOverlapper") eq "mer") {
+ checkMerOverlapper($isTrim);
+ } elsif (getGlobal("ovlOverlapper") eq "umd") {
+ # Nop.
+ } else {
+ caFailure("checkOverlap() unknown ovl overlapper?\n", undef);
+ }
+ }
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub createOverlapStore {
+
+ goto alldone if (-d "$wrk/$asm.ovlStore");
+
+ if (runCommand($wrk, "find -L $wrk/1-overlapper \\( -name \\*ovb.gz -or -name \\*ovb \\) -print > $wrk/$asm.ovlStore.list")) {
+ caFailure("failed to generate a list of all the overlap files", undef);
+ }
+
+ if (getGlobal("overlapStoreOnGrid") == 1) {
+ caFailure("on grid overlapStore construction not supported yet; use runCA-overlapStoreBuild for now", undef);
+ }
+
+ $cmd = "$bin/overlapStoreBuild ";
+ $cmd .= " -o $wrk/$asm.ovlStore.BUILDING ";
+ $cmd .= " -g $wrk/$asm.gkpStore ";
+
+ if (defined(getGlobal("closureOverlaps"))){
+ $cmd .= " -i " . getGlobal("closureOverlaps");
+ }
+
+ $cmd .= " -M " . getGlobal("ovlStoreMemory");
+ $cmd .= " -L $wrk/$asm.ovlStore.list ";
+ $cmd .= " > $wrk/$asm.ovlStore.err 2>&1";
+
+ if (runCommand($wrk, $cmd)) {
+ caFailure("failed to create the overlap store", "$wrk/$asm.ovlStore.err");
+ }
+
+ rename "$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore";
+
+ if (getGlobal("saveOverlaps") == 0) {
+ open(F, "< $wrk/$asm.ovlStore.list");
+ while (<F>) {
+ chomp;
+ unlink $_;
+ }
+ close(F);
+ }
+
+ rmrf("$wrk/$asm.ovlStore.list");
+ rmrf("$wrk/$asm.ovlStore.err");
+
+ alldone:
+ stopAfter("overlapper");
+}
+
+
+
+
+
+
+################################################################################
+################################################################################
+################################################################################
+
+sub overlapTrim {
+
+ return if (getGlobal("doOverlapBasedTrimming") == 0);
+ return if (getGlobal("ovlOverlapper") eq "umd");
+
+ # Skip overlap based trimming if it is done, or if the ovlStore already exists.
+ #
+ goto alldone if (-e "$wrk/0-overlaptrim/overlaptrim.success");
+ goto alldone if (-d "$wrk/$asm.ovlStore");
+
+ system("mkdir $wrk/0-overlaptrim") if (! -d "$wrk/0-overlaptrim");
+ system("mkdir $wrk/0-overlaptrim-overlap") if (! -d "$wrk/0-overlaptrim-overlap");
+
+ # Disable dedup, unless reads request it. This avoids an expensive ovlStore build.
+ #
+ if (getGlobal("doDeDuplication") != 0) {
+ setGlobal("doDeDuplication", 0);
+
+ if (system("$bin/gatekeeper -isfeatureset 0 doRemoveDuplicateReads $wrk/$asm.gkpStore") == 0) {
+ setGlobal("doDeDuplication", 1);
+ }
+ }
+
+ #
+ # Do an initial overly-permissive quality trimming, intersected with any known vector trimming.
+ # This step also applies any clear range computed by MBT.
+ #
+
+ if ((! -e "$wrk/0-overlaptrim/$asm.initialTrim.log") &&
+ (! -e "$wrk/0-overlaptrim/$asm.initialTrim.log.bz2")) {
+ $cmd = "$bin/initialTrim \\\n";
+ $cmd .= " -log $wrk/0-overlaptrim/$asm.initialTrim.log \\\n";
+ $cmd .= " -frg $wrk/$asm.gkpStore \\\n";
+ $cmd .= " > $wrk/0-overlaptrim/$asm.initialTrim.summary \\\n";
+ $cmd .= " 2> $wrk/0-overlaptrim/$asm.initialTrim.err ";
+
+ stopBefore("initialTrim", $cmd);
+
+ if (runCommand("$wrk/0-overlaptrim", $cmd)) {
+ rename "$wrk/0-overlaptrim/$asm.initialTrim.log", "$wrk/0-overlaptrim/$asm.initialTrim.log.FAILED";
+ caFailure("initial trimming failed", "$wrk/0-overlaptrim/$asm.initialTrim.err");
+ }
+
+ unlink "0-overlaptrim/$asm.initialTrim.err";
+ }
+
+ #
+ # Decide if any libraries request overlap based trimming -- if all libraries are
+ # asking for mer based trimming, we can skip OBT.
+ #
+
+ my $obtNeeded = 0;
+
+ open(F, "$bin/gatekeeper -nouid -dumplibraries $wrk/$asm.gkpStore |");
+ while (<F>) {
+ $obtNeeded++ if (m/doRemoveDuplicateReads.*=.*1/);
+ $obtNeeded++ if (m/doTrim_finalLargestCovered.*=.*1/);
+ $obtNeeded++ if (m/doTrim_finalEvidenceBased.*=.*1/);
+ $obtNeeded++ if (m/doRemoveSpurReads.*=.*1/);
+ $obtNeeded++ if (m/doRemoveChimericReads.*=.*1/);
+ }
+ close(F);
+
+ if ($obtNeeded == 0) {
+ touch("$wrk/0-overlaptrim/overlaptrim.success");
+ goto alldone;
+ }
+
+ #
+ # Compute overlaps, if we don't have them already
+ #
+
+ if (! -e "$wrk/0-overlaptrim/$asm.obtStore") {
+ createOverlapJobs("trim");
+ checkOverlap("trim");
+
+ # Sort the overlaps -- this also duplicates each overlap so that
+ # all overlaps for a fragment A are localized.
+
+ if (runCommand("$wrk/0-overlaptrim",
+ "find -L $wrk/0-overlaptrim-overlap \\( -name \\*ovb.gz -or -name \\*ovb \\) -print > $wrk/0-overlaptrim/$asm.obtStore.list")) {
+ caFailure("failed to generate a list of all the overlap files", undef);
+ }
+
+ $cmd = "$bin/overlapStoreBuild ";
+ $cmd .= " -obt ";
+ $cmd .= " -o $wrk/0-overlaptrim/$asm.obtStore.BUILDING ";
+ $cmd .= " -g $wrk/$asm.gkpStore ";
+ $cmd .= " -M " . getGlobal('ovlStoreMemory');
+ $cmd .= " -L $wrk/0-overlaptrim/$asm.obtStore.list";
+ $cmd .= " > $wrk/0-overlaptrim/$asm.obtStore.err 2>&1";
+
+ if (runCommand("$wrk/0-overlaptrim", $cmd)) {
+ caFailure("failed to build the obt store", "$wrk/0-overlaptrim/$asm.obtStore.err");
+ }
+
+ rename "$wrk/0-overlaptrim/$asm.obtStore.BUILDING", "$wrk/0-overlaptrim/$asm.obtStore";
+
+ # Delete overlaps unless we're told to save them, or we need to dedup.
+ if ((getGlobal("saveOverlaps") == 0) && (getGlobal("doDeDuplication") == 0)) {
+ open(F, "< $wrk/0-overlaptrim/$asm.obtStore.list");
+ while (<F>) {
+ chomp;
+ unlink $_;
+ }
+ close(F);
+ }
+
+ rmrf("$wrk/0-overlaptrim/$asm.obtStore.list");
+ rmrf("$wrk/0-overlaptrim/$asm.obtStore.err");
+ }
+
+ #
+ # Deduplicate?
+ #
+
+ if ((getGlobal("doDeDuplication") != 0) &&
+ (! -e "$wrk/0-overlaptrim/$asm.deduplicate.summary")) {
+
+ if (! -e "$wrk/0-overlaptrim/$asm.dupStore") {
+ if (runCommand("$wrk/0-overlaptrim",
+ "find -L $wrk/0-overlaptrim-overlap \\( -name \\*ovb.gz -or -name \\*ovb \\) -print > $wrk/0-overlaptrim/$asm.dupStore.list")) {
+ caFailure("failed to generate a list of all the overlap files", undef);
+ }
+
+ $cmd = "$bin/overlapStoreBuild \\\n";
+ $cmd .= " -dup \\\n";
+ $cmd .= " -o $wrk/0-overlaptrim/$asm.dupStore.BUILDING \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -M \\\n" . getGlobal('ovlStoreMemory');
+ $cmd .= " -L $wrk/0-overlaptrim/$asm.dupStore.list \\\n";
+ $cmd .= " > $wrk/0-overlaptrim/$asm.dupStore.err 2>&1";
+
+ if (runCommand("$wrk/0-overlaptrim", $cmd)) {
+ caFailure("failed to build the dup store", "$wrk/0-overlaptrim/$asm.dupStore.err");
+ }
+
+ rename "$wrk/0-overlaptrim/$asm.dupStore.BUILDING", "$wrk/0-overlaptrim/$asm.dupStore";
+
+ # Delete overlaps unless we're told to save them
+ if (getGlobal("saveOverlaps") == 0) {
+ open(F, "< $wrk/0-overlaptrim/$asm.dupStore.list");
+ while (<F>) {
+ chomp;
+ unlink $_;
+ }
+ close(F);
+ }
+
+ rmrf("$asm.dupStore.list");
+ rmrf("$asm.dupStore.err");
+ }
+
+ $cmd = "$bin/deduplicate \\\n";
+ $cmd .= "-gkp $wrk/$asm.gkpStore \\\n";
+ $cmd .= "-ovs $wrk/0-overlaptrim/$asm.obtStore \\\n";
+ $cmd .= "-ovs $wrk/0-overlaptrim/$asm.dupStore \\\n";
+ $cmd .= "-report $wrk/0-overlaptrim/$asm.deduplicate.log \\\n";
+ $cmd .= "-summary $wrk/0-overlaptrim/$asm.deduplicate.summary \\\n";
+ $cmd .= "> $wrk/0-overlaptrim/$asm.deduplicate.err 2>&1";
+
+ stopBefore("deDuplication", $cmd);
+
+ if (runCommand("$wrk/0-overlaptrim", $cmd)) {
+ unlink "$wrk/0-overlaptrim/$asm.deduplicate.summary";
+ caFailure("failed to deduplicate the reads", "$wrk/0-overlaptrim/$asm.deduplicate.err");
+ }
+ }
+
+ if ((! -e "$wrk/0-overlaptrim/$asm.finalTrim.log") &&
+ (! -e "$wrk/0-overlaptrim/$asm.finalTrim.log.bz2")) {
+ my $erate = 0.03; # Used for non-Sanger 'largest covered' style
+ my $elimit = 4.5;
+
+ my $utg = getUnitigger();
+
+ if (defined(getGlobal("obtErrorRate"))) {
+ $erate = getGlobal("obtErrorRate");
+ $elimit = getGlobal("obtErrorLimit");
+
+ } elsif ($utg eq "utg") {
+ $erate = getGlobal("utgErrorRate");
+ $elimit = getGlobal("utgErrorLimit");
+
+ } elsif ($utg eq "bog") {
+ $erate = getGlobal("utgErrorRate");
+ $elimit = getGlobal("utgErrorLimit");
+
+ } elsif ($utg eq "bogart") {
+ $erate = getGlobal("utgGraphErrorRate");
+ $elimit = getGlobal("utgGraphErrorLimit");
+
+ } else {
+ caFailure("unknown unitigger '$utg' during finalTrim", undef);
+ }
+
+ $cmd = "$bin/finalTrim \\\n";
+ $cmd .= " -G $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -O $wrk/0-overlaptrim/$asm.obtStore \\\n";
+ $cmd .= " -e $erate \\\n";
+ $cmd .= " -E $elimit \\\n" if (defined($elimit));
+ $cmd .= " -o $wrk/0-overlaptrim/$asm.finalTrim \\\n";
+ $cmd .= "> $wrk/0-overlaptrim/$asm.finalTrim.err 2>&1";
+
+ stopBefore("finalTrimming", $cmd);
+
+ if (runCommand("$wrk/0-overlaptrim", $cmd)) {
+ unlink "$wrk/0-overlaptrim/$asm.finalTrim.log";
+ unlink "$wrk/0-overlaptrim/$asm.finalTrim.stats";
+ caFailure("failed to compute final trimming", "$wrk/0-overlaptrim/$asm.finalTrim.err");
+ }
+ }
+
+
+ if (getGlobal("doChimeraDetection") ne 'off') {
+ if ((! -e "$wrk/0-overlaptrim/$asm.chimera.log") &&
+ (! -e "$wrk/0-overlaptrim/$asm.chimera.log.bz2")) {
+ my $erate = getGlobal("ovlErrorRate");
+ my $elimit = getGlobal("ovlErrorRate") * 10 + 0.5;
+
+ $cmd = "$bin/chimera \\\n";
+ $cmd .= " -G $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -O $wrk/0-overlaptrim/$asm.obtStore \\\n";
+ $cmd .= " -e $erate \\\n";
+ $cmd .= " -E $elimit \\\n";
+ $cmd .= " -o $wrk/0-overlaptrim/$asm.chimera \\\n";
+ $cmd .= " -mininniepair 0 -minoverhanging 0 \\\n" if (getGlobal("doChimeraDetection") eq "aggressive");
+ $cmd .= " > $wrk/0-overlaptrim/$asm.chimera.err 2>&1";
+
+ stopBefore("chimeraDetection", $cmd);
+
+ if (runCommand("$wrk/0-overlaptrim", $cmd)) {
+ rename "$wrk/0-overlaptrim/$asm.chimera.log", "$wrk/0-overlaptrim/$asm.chimera.log.FAILED";
+ caFailure("chimera cleaning failed", "$wrk/0-overlaptrim/$asm.chimera.err");
+ }
+ }
+ }
+
+ #rmrf("$asm.obtStore");
+
+ touch("$wrk/0-overlaptrim/overlaptrim.success");
+
+ alldone:
+ stopAfter("overlapBasedTrimming");
+ stopAfter("OBT");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+
+
+sub overlapCorrection {
+ my $cleanup = 1;
+
+ return if (getGlobal("doFragmentCorrection") == 0);
+
+ return if (-e "$wrk/3-overlapcorrection/$asm.erates.updated");
+ return if (-e "$wrk/$asm.ovlStore/corrected");
+
+ system("mkdir $wrk/3-overlapcorrection") if (! -e "$wrk/3-overlapcorrection");
+
+ if (((getGlobal("ovlOverlapper") eq "ovl") &&
+ (getGlobal("ovlOverlapper") eq "ovm")) ||
+ (! -e "$wrk/3-overlapcorrection/frgcorr.sh")) {
+ my $batchSize = getGlobal("frgCorrBatchSize");
+ my $numThreads = getGlobal("frgCorrThreads");
+ my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
+
+ my $taskID = getGlobal("gridTaskID");
+ my $submitTaskID = getGlobal("gridArraySubmitID");
+
+ open(F, "> $wrk/3-overlapcorrection/frgcorr.sh") or caFailure("failed to write to '$wrk/3-overlapcorrection/frgcorr.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %04d \$jobid`\n";
+ print F "minid=`expr \$jobid \\* $batchSize - $batchSize + 1`\n";
+ print F "maxid=`expr \$jobid \\* $batchSize`\n";
+ print F "runid=\$\$\n";
+ print F "\n";
+ print F "if [ \$maxid -gt $numFrags ] ; then\n";
+ print F " maxid=$numFrags\n";
+ print F "fi\n";
+ print F "if [ \$minid -gt \$maxid ] ; then\n";
+ print F " echo Job partitioning error -- minid=\$minid maxid=\$maxid.\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=", getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=", getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=", getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+ print F "\n";
+ print F "if [ -e $wrk/3-overlapcorrection/\$jobid.frgcorr ] ; then\n";
+ print F " echo Job previously completed successfully.\n";
+ print F " exit\n";
+ print F "fi\n";
+
+ print F getBinDirectoryShellCode();
+
+ print F "\$bin/correct-frags \\\n";
+ print F " -t $numThreads \\\n";
+ print F " -S $wrk/$asm.ovlStore \\\n";
+ print F " -o $wrk/3-overlapcorrection/\$jobid.frgcorr.WORKING \\\n";
+ print F " $wrk/$asm.gkpStore \\\n";
+ print F " \$minid \$maxid \\\n";
+ print F "&& \\\n";
+ print F "mv $wrk/3-overlapcorrection/\$jobid.frgcorr.WORKING $wrk/3-overlapcorrection/\$jobid.frgcorr\n";
+
+ close(F);
+
+ chmod 0755, "$wrk/3-overlapcorrection/frgcorr.sh";
+
+ if (getGlobal("frgCorrOnGrid") && getGlobal("useGrid")) {
+ # Run the correction job on the grid.
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeFragmentCorrection = getGlobal("sgeFragmentCorrection");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("frg_$asm$sgeName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovl_$asm$sgeName", $jobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeFragmentCorrection $nameOption \"$jobName\" $arrayOpt ";
+ $SGE .= " $outputOption $wrk/3-overlapcorrection/$submitTaskID.err ";
+ $SGE .= "$wrk/3-overlapcorrection/frgcorr.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ # Run the correction job right here, right now.
+
+ for (my $i=1; $i<=$jobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ schedulerSubmit("$wrk/3-overlapcorrection/frgcorr.sh $i > $wrk/3-overlapcorrection/$out.err 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses($global{"frgCorrConcurrency"});
+ schedulerFinish();
+ }
+ }
+
+ #
+ # MERGE CORRECTION
+ #
+
+ if (! -e "$wrk/3-overlapcorrection/$asm.frgcorr") {
+ my $batchSize = (getGlobal("ovlOverlapper") eq "mer") ? getGlobal("merOverlapperExtendBatchSize") : getGlobal("frgCorrBatchSize");
+ my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
+ my $failedJobs = 0;
+
+ open(F, "> $wrk/3-overlapcorrection/cat-corrects.frgcorrlist");
+ for (my $i=1; $i<=$jobs; $i++) {
+ my $jobid = substr("0000" . $i, -4);
+
+ if (! -e "$wrk/3-overlapcorrection/$jobid.frgcorr") {
+ print STDERR "Fragment correction job $jobid failed.\n";
+ $failedJobs++;
+ }
+
+ print F "$wrk/3-overlapcorrection/$jobid.frgcorr\n";
+ }
+ close(F);
+
+ # FAILUREHELPME
+
+ if ($failedJobs) {
+ if ((getGlobal("ovlOverlapper") eq "ovl") || (getGlobal("ovlOverlapper") eq "ovm")) {
+ caFailure("$failedJobs overlap jobs failed; remove $wrk/3-overlapcorrection/frgcorr.sh to try again", undef);
+ } else {
+ caFailure("$failedJobs overlap jobs failed due to mer overlap seed extension", undef);
+ }
+ }
+
+ $cmd = "$bin/cat-corrects ";
+ $cmd .= "-L $wrk/3-overlapcorrection/cat-corrects.frgcorrlist ";
+ $cmd .= "-o $wrk/3-overlapcorrection/$asm.frgcorr ";
+ $cmd .= "> $wrk/3-overlapcorrection/cat-corrects.err 2>&1";
+
+ if (runCommand("$wrk/3-overlapcorrection", $cmd)) {
+ rename "$wrk/3-overlapcorrection/$asm.frgcorr", "$wrk/3-overlapcorrection/$asm.frgcorr.FAILED";
+ caFailure("failed to concatenate the fragment corrections", "$wrk/3-overlapcorrection/cat-corrects.err");
+ }
+
+ if ($cleanup) {
+ open(F, "< $wrk/3-overlapcorrection/cat-corrects.frgcorrlist");
+ while (<F>) {
+ if (m/^(.*)\/([0-9]*).frgcorr/) {
+ #unlink "$1/$2.frgcorr";
+ #unlink "$1/$2.err";
+ my $sge = int($2);
+ #unlink "$1/$sge.err";
+ }
+ }
+ close(F);
+ #unlink "$wrk/3-overlapcorrection/cat-corrects.frgcorrlist";
+ #unlink "$wrk/3-overlapcorrection/cat-corrects.err";
+ }
+ }
+
+ #
+ # CREATE OVERLAP CORRECTION
+ #
+
+ if (! -e "$wrk/3-overlapcorrection/ovlcorr.sh") {
+ my $batchSize = getGlobal("ovlCorrBatchSize");
+ my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
+ my $taskID = getGlobal("gridTaskID");
+ my $submitTaskID = getGlobal("gridArraySubmitID");
+
+ open(F, "> $wrk/3-overlapcorrection/ovlcorr.sh") or caFailure("failed to write '$wrk/3-overlapcorrection/ovlcorr.sh'", undef);
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ \$jobid -gt $jobs ] ; then\n";
+ print F " exit\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %04d \$jobid`\n";
+ print F "frgBeg=`expr \$jobid \\* $batchSize - $batchSize + 1`\n";
+ print F "frgEnd=`expr \$jobid \\* $batchSize`\n";
+ print F "if [ \$frgEnd -ge $numFrags ] ; then\n";
+ print F " frgEnd=$numFrags\n";
+ print F "fi\n";
+ print F "frgBeg=`printf %08d \$frgBeg`\n";
+ print F "frgEnd=`printf %08d \$frgEnd`\n";
+
+ print F getBinDirectoryShellCode();
+
+ print F "if [ ! -e $wrk/3-overlapcorrection/\$jobid.erate ] ; then\n";
+ print F " \$bin/correct-olaps \\\n";
+ print F " -S $wrk/$asm.ovlStore \\\n";
+ print F " -e $wrk/3-overlapcorrection/\$jobid.erate.WORKING \\\n";
+ print F " $wrk/$asm.gkpStore \\\n";
+ print F " $wrk/3-overlapcorrection/$asm.frgcorr \\\n";
+ print F " \$frgBeg \$frgEnd \\\n";
+ print F " && \\\n";
+ print F " mv $wrk/3-overlapcorrection/\$jobid.erate.WORKING $wrk/3-overlapcorrection/\$jobid.erate\n";
+ print F "fi\n";
+ close(F);
+
+ chmod 0755, "$wrk/3-overlapcorrection/ovlcorr.sh";
+
+ if (getGlobal("ovlCorrOnGrid") && getGlobal("useGrid")) {
+ # Run the correction job on the grid.
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeOverlapCorrection = getGlobal("sgeOverlapCorrection");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("ovc_$asm$sgeName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovc_$asm$sgeName", $jobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeOverlapCorrection $nameOption \"$jobName\" $arrayOpt ";
+ $SGE .= " $outputOption $wrk/3-overlapcorrection/$submitTaskID.err ";
+ $SGE .= "$wrk/3-overlapcorrection/ovlcorr.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ # Run the correction job right here, right now.
+
+ for (my $i=1; $i<=$jobs; $i++) {
+ my $out = substr("0000" . $i, -4);
+ schedulerSubmit("$wrk/3-overlapcorrection/ovlcorr.sh $i > $wrk/3-overlapcorrection/$out.err 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses($global{"ovlCorrConcurrency"});
+ schedulerFinish();
+ }
+ }
+
+ #
+ # APPLY OVERLAP CORRECTION
+ #
+
+ if (! -e "$wrk/3-overlapcorrection/$asm.erates.updated") {
+ my $batchSize = getGlobal("ovlCorrBatchSize");
+ my $failedJobs = 0;
+ my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
+
+ open(F, "> $wrk/3-overlapcorrection/cat-erates.eratelist");
+ for (my $i=1; $i<=$jobs; $i++) {
+ my $jobid = substr("0000" . $i, -4);
+
+ if (! -e "$wrk/3-overlapcorrection/$jobid.erate") {
+ print STDERR "Overlap correction job $i ($wrk/3-overlapcorrection/$jobid) failed.\n";
+ $failedJobs++;
+ }
+
+ print F "$wrk/3-overlapcorrection/$jobid.erate\n";
+ }
+ close(F);
+
+ # FAILUREHELPME
+
+ if ($failedJobs) {
+ caFailure("$failedJobs overlap correction jobs failed; remove $wrk/3-overlapcorrection/ovlcorr.sh (or run by hand) to try again", undef);
+ }
+
+ #unlink "$wrk/3-overlapcorrection/$asm.frgcorr" if ($cleanup);
+
+ $cmd = "$bin/cat-erates ";
+ $cmd .= "-L $wrk/3-overlapcorrection/cat-erates.eratelist ";
+ $cmd .= "-o $wrk/3-overlapcorrection/$asm.erates ";
+ $cmd .= "> $wrk/3-overlapcorrection/cat-erates.err 2>&1";
+ if (runCommand("$wrk/3-overlapcorrection", $cmd)) {
+ rename "$wrk/3-overlapcorrection/$asm.erates", "$wrk/3-overlapcorrection/$asm.erates.FAILED";
+ caFailure("failed to concatenate the overlap erate corrections", "$wrk/3-overlapcorrection/cat-erates.err");
+ }
+
+ $cmd = "$bin/overlapStore ";
+ $cmd .= " -u $wrk/$asm.ovlStore ";
+ $cmd .= " $wrk/3-overlapcorrection/$asm.erates ";
+ $cmd .= "> $wrk/3-overlapcorrection/overlapStore-update-erates.err 2>&1";
+ if (runCommand("$wrk/3-overlapcorrection", $cmd)) {
+ caFailure("failed to apply the overlap corrections", "$wrk/3-overlapcorrection/overlapStore-update-erates.err");
+ }
+
+ touch("$wrk/3-overlapcorrection/$asm.erates.updated");
+ touch("$wrk/$asm.ovlStore/corrected");
+
+ if ($cleanup) {
+ open(F, "< $wrk/3-overlapcorrection/cat-erates.eratelist");
+ while (<F>) {
+ if (m/^(.*)\/([0-9]*).erate/) {
+ #unlink "$1/$2.erate";
+ #unlink "$1/$2.err";
+ my $sge = int($2);
+ #unlink "$1/$sge.err";
+ }
+ }
+ close(F);
+
+ #unlink "$wrk/3-overlapcorrection/overlapStore-update-erates.err";
+ #unlink "$wrk/3-overlapcorrection/$asm.erates";
+
+ #unlink "$wrk/3-overlapcorrection/cat-erates.err";
+ #unlink "$wrk/3-overlapcorrection/cat-erates.eratelist";
+
+ #unlink "$wrk/3-overlapcorrection/frgcorr.sh";
+ #unlink "$wrk/3-overlapcorrection/ovlcorr.sh";
+ }
+ }
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub classifyMates () {
+
+ # This is configuration hell. For now, we just ask for the library to classify.
+ # We could later extend with parallelization, run time, memory, etc, etc.
+ #
+ # classifyIlluminaBB=<libraryname>,<libraryname>,...
+ # classifyIlluminaMP=<libraryname>,<libraryname>,...
+
+ my @libsToClassify = split ',', getGlobal("dncMPlibraries");
+ my @backboneToUse = split ',', getGlobal("dncBBlibraries");
+
+ if ((scalar(@libsToClassify) == 0) ||
+ (scalar(@backboneToUse) == 0) ||
+ (-e "$wrk/2-classifyMates/classify.success")) {
+ return;
+ }
+
+ system("mkdir $wrk/2-classifyMates") if (! -e "$wrk/2-classifyMates");
+
+ # Load a map of library name to IID.
+
+ my %libToIID;
+
+ open(F, "$bin/gatekeeper -dumplibraries -tabular $wrk/$asm.gkpStore |");
+ while (<F>) {
+ my @v = split '\s+', $_;
+ $libToIID{$v[0]} = $v[1];
+ }
+ close(F);
+
+ # Map the backbone library names to a IIDs.
+
+ my $bbIID;
+
+ foreach my $lib (@backboneToUse) {
+ if (!exists($libToIID{$lib})) {
+ caFailure("Backbone library '$lib' doesn't exist in the assembly, classifyMates failed\n", undef);
+ }
+ if (defined($bbIID)) {
+ $bbIID .= " -bl $libToIID{$lib}";
+ } else {
+ $bbIID = "$libToIID{$lib}";
+ }
+ }
+
+ # Build jobs to run. These are NOT optimal, but building optimal configurations is probably
+ # impossible. You can save memory by classifying each MP library seperately, but doing all at
+ # once is easier.
+ #
+ # The jobs are built, but they are only templates. It is up to the user to run them, and
+ # user could modify the way they are run.
+
+ my @classifyJobs;
+ my $classifiedOutputs = "";
+
+ foreach my $lib (@libsToClassify) {
+ my $mpIID = $libToIID{$lib};
+
+ if (!exists($libToIID{$lib})) {
+ caFailure("Mate pair library '$lib' doesn't exist in the assembly, classifyMates failed\n", undef);
+ }
+
+ open(F, "> $wrk/2-classifyMates/classify-$lib.sh");
+ print F "#!/bin/sh\n";
+ print F "\n";
+ print F "\n";
+ print F "if [ ! -e \"$wrk/2-classifyMates/classifyMates.sl$mpIID.O.0100.1500.bfs.100000\" ] ; then\n";
+ print F " $bin/classifyMates \\\n";
+ print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -O $wrk/$asm.ovlStore \\\n";
+ print F " -t 8 \\\n";
+ print F " -m 128 \\\n";
+ print F " -sl $mpIID \\\n";
+ print F " -bl $bbIID \\\n";
+ print F " -outtie -min 100 -max 1500 -bfs 100000 \\\n";
+ print F " -o $wrk/2-classifyMates/classifyMates.sl$mpIID.O.0100.1500.bfs.100000\n";
+ print F "fi\n";
+ print F "\n";
+ print F "# classifyMates internally appends .WORKING, then renames just before it exits.\n";
+ print F "if [ ! -e \"$wrk/2-classifyMates/classifyMates.sl$mpIID.O.0100.1500.bfs.100000\" ] ; then\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "exit 0\n";
+ close(F);
+
+ chmod 0755, "$wrk/2-classifyMates/classify-$lib.sh";
+
+ push @classifyJobs, "$wrk/2-classifyMates/classify-$lib.sh";
+ $classifiedOutputs .= " -r $wrk/2-classifyMates/classifyMates.sl$mpIID.O.0100.1500.bfs.100000 \\\n";
+ }
+
+ open(F, "> $wrk/2-classifyMates/classify-apply.sh");
+ print F "#!/bin/sh\n";
+ print F "\n";
+ print F "$bin/classifyMatesApply \\\n";
+ print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -p \\\n";
+ print F "$classifiedOutputs"; # NO INDENTATION! NO LINE CONTINUATION! Both are in the string directly.
+ print F " -o $wrk/2-classifyMates/$asm.classified.gkpStore.edit \\\n";
+ print F "> $wrk/2-classifyMates/$asm.classified.log \\\n";
+ print F "2> $wrk/2-classifyMates/$asm.classified.summary \\\n";
+ print F "&& \\\n";
+ print F "$bin/gatekeeper --edit \\\n";
+ print F " $wrk/2-classifyMates/$asm.classified.gkpStore.edit \\\n";
+ print F " $wrk/$asm.gkpStore \\\n";
+ print F "> $wrk/2-classifyMates/$asm.classified.gkpStore.edit.out \\\n";
+ print F "&& \\\n";
+ print F "touch $wrk/2-classifyMates/classify.success\n";
+ print F "\n";
+ print F "exit 0\n";
+ close(F);
+
+ chmod 0755, "$wrk/2-classifyMates/classify-apply.sh";
+
+ # Force the stop. User must run scripts by hand.
+
+ stopBefore("classifyMates", undef);
+
+ foreach my $j (@classifyJobs) {
+ if (runCommand("$wrk/2-classifyMates", $j)) {
+ caFailure("failed to run classify mates command $j", undef);
+ }
+ }
+
+ if (runCommand("$wrk/2-classifyMates", "$wrk/2-classifyMates/classify-apply.sh > $wrk/2-classifyMates/classify-apply.err 2>&1")) {
+ caFailure("failed to apply classify mates results", undef);
+ }
+ if (! -e "$wrk/2-classifyMates/classify.success") {
+ caFailure("classifyMatesApply failed.", "");
+ }
+
+ stopAfter("classifyMates");
+}
+
+
+
+
+################################################################################
+################################################################################
+################################################################################
+
+sub unitigger () {
+
+ if (0) {
+ $cmd = "$bin/removeMateOverlap -gkp $wrk/$asm.gkpStore -ovl $wrk/$asm.ovlStore";
+ if (runCommand("$wrk", $cmd)) {
+ caFailure("failed to remove mate overlaps", undef);
+ }
+ }
+
+ if (-e "$wrk/4-unitigger/unitigger.success") {
+ goto alldone;
+ }
+
+ if (-e "$wrk/$asm.tigStore") {
+ print STDERR "Skipping unitigger because tigStore exists at $wrk/$asm.tigStore\n";
+ goto alldone;
+ }
+
+ system("mkdir $wrk/4-unitigger") if (! -e "$wrk/4-unitigger");
+
+ my $e = getGlobal("utgErrorRate"); # Unitigger and BOG
+ my $E = getGlobal("utgErrorLimit");
+ my $eg = getGlobal("utgGraphErrorRate"); # BOGART
+ my $Eg = getGlobal("utgGraphErrorLimit");
+ my $em = getGlobal("utgMergeErrorRate");
+ my $Em = getGlobal("utgMergeErrorLimit");
+ my $mem = getGlobal("batMemory");
+
+ my $B = int($numFrags / getGlobal("cnsPartitions"));
+ $B = getGlobal("cnsMinFrags") if ($B < getGlobal("cnsMinFrags"));
+
+ my $u = getGlobal("utgBubblePopping");
+
+ my $unitigger = getUnitigger();
+
+ if ($unitigger eq "bogart") {
+ my $th = getGlobal("batThreads");
+
+ $cmd = "$bin/bogart ";
+ $cmd .= " -O $wrk/$asm.ovlStore ";
+ $cmd .= " -G $wrk/$asm.gkpStore ";
+ $cmd .= " -T $wrk/$asm.tigStore ";
+ $cmd .= " -B $B ";
+ $cmd .= " -eg $eg ";
+ $cmd .= " -Eg $Eg ";
+ $cmd .= " -em $em ";
+ $cmd .= " -Em $Em ";
+ $cmd .= " -threads $th " if (defined($th));
+ $cmd .= " -R " if (getGlobal("batRebuildRepeats") == 1);
+ $cmd .= " -E " if (getGlobal("batMateExtension") == 1);
+ $cmd .= " -M $mem " if (defined($mem));
+ $cmd .= " -o $wrk/4-unitigger/$asm ";
+ $cmd .= " > $wrk/4-unitigger/unitigger.err 2>&1";
+ } elsif ($unitigger eq "bog") {
+ my $bmd = getGlobal("bogBadMateDepth");
+
+ $cmd = "$bin/buildUnitigs ";
+ $cmd .= " -O $wrk/$asm.ovlStore ";
+ $cmd .= " -G $wrk/$asm.gkpStore ";
+ $cmd .= " -T $wrk/$asm.tigStore ";
+ $cmd .= " -B $B ";
+ $cmd .= " -e $e ";
+ $cmd .= " -E $E ";
+ $cmd .= " -b " if (getGlobal("bogBreakAtIntersections") == 1);
+ $cmd .= " -m $bmd " if (defined($bmd));
+ $cmd .= " -U " if ($u == 1);
+ $cmd .= " -o $wrk/4-unitigger/$asm ";
+ $cmd .= " > $wrk/4-unitigger/unitigger.err 2>&1";
+ } elsif ($unitigger eq "utg") {
+ $cmd = "$bin/unitigger ";
+ $cmd .= " -I $wrk/$asm.ovlStore ";
+ $cmd .= " -F $wrk/$asm.gkpStore ";
+ $cmd .= " -T $wrk/$asm.tigStore ";
+ $cmd .= " -B $B ";
+ $cmd .= " -e $e ";
+ $cmd .= " -k " if (getGlobal("utgRecalibrateGAR") == 1);
+ $cmd .= " -d 1 -x 1 -z 10 -j 5 -U $u ";
+ $cmd .= " -o $wrk/4-unitigger/$asm ";
+ $cmd .= " > $wrk/4-unitigger/unitigger.err 2>&1";
+ } else {
+ caFailure("unknown unitigger '$unitigger'; must be 'bog' or 'utg'", undef);
+ }
+
+ stopBefore("unitigger", $cmd);
+
+ if (runCommand("$wrk/4-unitigger", $cmd)) {
+ caFailure("failed to unitig", "$wrk/4-unitigger/unitigger.err");
+ }
+
+ touch("$wrk/4-unitigger/unitigger.success");
+
+ alldone:
+ stopAfter("unitigger");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+sub createPostUnitiggerConsensusJobs (@) {
+ my $consensusType = getGlobal("consensus");
+
+ return if (-e "$wrk/5-consensus/consensus.sh");
+
+ if (! -e "$wrk/5-consensus/$asm.partitioned") {
+ $cmd = "$bin/gatekeeper ";
+ $cmd .= " -P $wrk/4-unitigger/$asm.partitioning ";
+ $cmd .= " $wrk/$asm.gkpStore ";
+ $cmd .= "> $wrk/5-consensus/$asm.partitioned.err 2>&1";
+ if (runCommand("$wrk/5-consensus", $cmd)) {
+ caFailure("failed to partition the fragStore", "$wrk/5-consensus/$asm.partitioned.err");
+ }
+
+ touch "$wrk/5-consensus/$asm.partitioned";
+ }
+
+ my $jobs = 0;
+
+ open(F, "< $wrk/4-unitigger/$asm.partitioningInfo") or caFailure("can't open '$wrk/4-unitigger/$asm.partitioningInfo'", undef);
+ while (<F>) {
+ if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+unitigs\sand\s+(\d+)\s+fragments./) {
+ $jobs = $1;
+ }
+ }
+ close(F);
+
+ my $taskID = getGlobal("gridTaskID");
+ open(F, "> $wrk/5-consensus/consensus.sh") or caFailure("can't open '$wrk/5-consensus/consensus.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "if [ \$jobid -gt $jobs ]; then\n";
+ print F " echo Error: Only $jobs partitions, you asked for \$jobid.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %03d \$jobid`\n";
+ print F "\n";
+ print F "if [ -e $wrk/5-consensus/${asm}_\$jobid.success ] ; then\n";
+ print F " exit 0\n";
+ print F "fi\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=", getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=", getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=", getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+
+ print F getBinDirectoryShellCode();
+
+ if ($consensusType eq "cns") {
+ my $reduce = getGlobal('cnsReduceUnitigs');
+
+ print F "\$bin/utgcns \\\n";
+ print F " -g $wrk/$asm.gkpStore \\\n";
+ print F " -t $wrk/$asm.tigStore 1 \$jobid \\\n";
+ print F " -reduce $reduce \\\n" if (defined($reduce));
+ print F "> $wrk/5-consensus/${asm}_\$jobid.cns.err 2>&1 \\\n";
+ print F "&& \\\n";
+ print F "\$bin/utgcnsfix \\\n";
+ print F " -g $wrk/$asm.gkpStore \\\n";
+ print F " -t $wrk/$asm.tigStore 2 \$jobid \\\n";
+ print F " -o $wrk/5-consensus/${asm}_\$jobid.fixes \\\n";
+ print F "> $wrk/5-consensus/${asm}_\$jobid.fix.err 2>&1";
+ print F "&& \\\n";
+ print F "touch $wrk/5-consensus/${asm}_\$jobid.success\n";
+ } elsif ($consensusType eq "seqan") {
+ print F "\$bin/SeqAn_CNS \\\n";
+ print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -c \$cgbfile \\\n";
+ print F " -s \$bin/graph_consensus \\\n";
+ print F " -w $wrk/5-consensus/ \\\n";
+ print F " -o $wrk/5-consensus/${asm}_\$jobid.cgi \\\n";
+ print F " > $wrk/5-consensus/${asm}_\$jobid.cns.err 2>&1 \\\n";
+ print F "&& \\\n";
+ print F "touch $wrk/5-consensus/${asm}_\$jobid.success\n";
+ } else {
+ caFailure("unknown consensus type $consensusType; should be 'cns' or 'seqan'", undef);
+ }
+ close(F);
+
+ chmod 0755, "$wrk/5-consensus/consensus.sh";
+
+ if (getGlobal("useGrid") && getGlobal("cnsOnGrid")) {
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeConsensus = getGlobal("sgeConsensus");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("utg_$asm$sgeName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovc_$asm$sgeName", $jobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeConsensus $nameOption \"$jobName\" $arrayOpt ";
+ $SGE .= "$outputOption /dev/null ";
+ $SGE .= "$wrk/5-consensus/consensus.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ for (my $i=1; $i<=$jobs; $i++) {
+ schedulerSubmit("$wrk/5-consensus/consensus.sh $i > /dev/null 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses(getGlobal("cnsConcurrency"));
+ schedulerFinish();
+ }
+}
+
+
+
+sub postUnitiggerConsensus () {
+ my $cmd;
+
+ return if (-e "$wrk/5-consensus/consensus.success");
+
+ system("mkdir $wrk/5-consensus") if (! -d "$wrk/5-consensus");
+
+ # NOT IDEAL. We assume that jobs are finished if the script exists. We need
+ # better logic in createPostUnitiggerConsensusJobs() to tell if the job crashed.
+ #
+ if (! -e "$wrk/5-consensus/consensus.sh") {
+ createPostUnitiggerConsensusJobs();
+ }
+
+ #
+ # Check that consensus finished properly
+ #
+
+ my $failedJobs = 0;
+
+ open(F, "< $wrk/4-unitigger/$asm.partitioningInfo") or caFailure("can't open '$wrk/4-unitigger/$asm.partitioningInfo'", undef);
+ while (<F>) {
+ if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+unitigs\sand\s+(\d+)\s+fragments./) {
+ my $id = substr("000" . $1, -3);
+
+ if (! -e "$wrk/5-consensus/${asm}_$id.success") {
+ print STDERR "$wrk/5-consensus/${asm}_$id failed -- no .success.\n";
+ $failedJobs++;
+ }
+ }
+ }
+ close(F);
+
+ # FAILUREHELPME
+ #
+ caFailure("$failedJobs unitig consensus jobs failed; remove $wrk/5-consensus/consensus.sh to try again", undef) if ($failedJobs);
+
+ #
+ # Apply the utgcnsfix changes
+ #
+
+ if (! -e "$wrk/5-consensus/$asm.fixes") {
+ open(O, "> $wrk/5-consensus/$asm.fixes");
+ open(F, "< $wrk/4-unitigger/$asm.partitioningInfo") or caFailure("can't open '$wrk/4-unitigger/$asm.partitioningInfo'", undef);
+ while (<F>) {
+ if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+unitigs\sand\s+(\d+)\s+fragments./) {
+ my $id = substr("000" . $1, -3);
+
+ open(G, "< $wrk/5-consensus/${asm}_$id.fixes") or die "Failed to open '$wrk/5-consensus/${asm}_$id.fixes'\n";
+ while (<G>) {
+ print O $_;
+ }
+ close(G);
+ }
+ }
+ close(F);
+ close(O);
+
+ $cmd = "$bin/tigStore";
+ $cmd .= " -g $wrk/$asm.gkpStore";
+ $cmd .= " -t $wrk/$asm.tigStore 2";
+ $cmd .= " -N";
+ $cmd .= " -R $wrk/5-consensus/$asm.fixes";
+ $cmd .= " > $asm.fixes.err 2>&1";
+
+ if (runCommand("$wrk/5-consensus", $cmd)) {
+ caFailure("unitig utgcnsfix failed", "$wrk/5-consensus/$asm.fixes.err");
+ rename "$wrk/5-consensus/$asm.fixes", "$wrk/5-consensus/$asm.fixes.FAILED";
+ }
+ }
+
+ #
+ # Estimate insert size estimates
+ #
+ # tigStore writes output files using the tigStore as a prefix. We thus need to temporarily symlink
+ # The tigStore to the proper directory.
+ #
+
+ if (! -e "$wrk/5-consensus-insert-sizes/estimates.out") {
+ system("mkdir $wrk/5-consensus-insert-sizes") if (! -d "$wrk/5-consensus-insert-sizes");
+
+ system("ln -s ../$asm.tigStore $wrk/5-consensus-insert-sizes/$asm.tigStore");
+
+ $cmd = "$bin/tigStore \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $wrk/5-consensus-insert-sizes/$asm.tigStore 3 \\\n";
+ $cmd .= " -d matepair -U \\\n";
+ $cmd .= "> $wrk/5-consensus-insert-sizes/estimates.out 2>&1";
+
+ if (runCommand("$wrk/5-consensus-insert-sizes", $cmd)) {
+ caFailure("Insert size estimation failed", "$wrk/5-consensus-insert-sizes/estimates.out");
+ }
+
+ unlink("$wrk/5-consensus-insert-sizes/$asm.tigStore");
+ }
+
+ #
+ # Update estimates in gatekeeper
+ #
+
+ if (! -e "$wrk/5-consensus-insert-sizes/updates.err") {
+ if (! -e "$wrk/5-consensus-insert-sizes/$asm.tigStore.distupdate") {
+ rename "$wrk/5-consensus-insert-sizes/estimates.out", "$wrk/5-consensus-insert-sizes/estimates.out.FAILED";
+ caFailure("Failed to find insert size estimates", "$wrk/5-consensus-insert-sizes/estimates.out.FAILED");
+ }
+
+ $cmd = "$bin/gatekeeper \\\n";
+ $cmd .= " --edit $wrk/5-consensus-insert-sizes/$asm.tigStore.distupdate \\\n";
+ $cmd .= " $wrk/$asm.gkpStore \\\n";
+ $cmd .= "> $wrk/5-consensus-insert-sizes/updates.err 2>&1";
+
+ if (runCommand("$wrk/5-consensus-insert-sizes", $cmd)) {
+ rename "$wrk/5-consensus-insert-sizes/updates.err", "$wrk/5-consensus-insert-sizes/updates.err.FAILED";
+ caFailure("Insert size updates failed", "$wrk/5-consensus-insert-sizes/updates.err.FAILED");
+ }
+ }
+
+ #
+ # Run the chimeric unitig splitter, and then the fixer again.
+ #
+
+ if (getGlobal("doUnitigSplitting")) {
+ if (! -e "$wrk/5-consensus-split/splitUnitigs.out") {
+ system("mkdir $wrk/5-consensus-split") if (! -d "$wrk/5-consensus-split");
+
+ $cmd = "$bin/splitUnitigs \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $wrk/$asm.tigStore 3 \\\n";
+ $cmd .= "> $wrk/5-consensus-split/splitUnitigs.out 2>&1";
+
+ if (runCommand("$wrk/5-consensus-split", $cmd)) {
+ rename "$wrk/5-consensus-split/splitUnitigs.out", "$wrk/5-consensus-split/splitUnitigs.out.FAILED";
+ caFailure("Unitig splitter failed", "$wrk/5-consensus-split/splitUnitigs.out.FAILED");
+ }
+ }
+
+ if (! -e "$wrk/5-consensus-split/consensus-fix.out") {
+ $cmd = "$bin/utgcnsfix \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $wrk/$asm.tigStore 4 . \\\n";
+ $cmd .= "> $wrk/5-consensus-split/consensus-fix.out 2>&1";
+
+ if (runCommand("$wrk/5-consensus-split", $cmd)) {
+ rename "$wrk/5-consensus-split/consensus-fix.out", "$wrk/5-consensus-split/consensus-fix.out.FAILED";
+ caFailure("post split utgcnsfix failed", "$wrk/5-consensus-split/consensus-fix.out.FAILED");
+ }
+ }
+ }
+
+ #
+ # And finally, compute the coverage stat for all unitigs
+ #
+ my $l = getGlobal("utgGenomeSize");
+
+ if (! -e "$wrk/5-consensus-coverage-stat/computeCoverageStat.err") {
+ system("mkdir $wrk/5-consensus-coverage-stat") if (! -d "$wrk/5-consensus-coverage-stat");
+
+ $cmd = "$bin/computeCoverageStat \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $wrk/$asm.tigStore 5 \\\n";
+ $cmd .= " -s $l \\\n" if defined($l);
+ $cmd .= " -o $wrk/5-consensus-coverage-stat/$asm \\\n";
+ $cmd .= "> $wrk/5-consensus-coverage-stat/computeCoverageStat.err 2>&1";
+
+ if (runCommand("$wrk/5-consensus-coverage-stat", $cmd)) {
+ rename "$wrk/5-consensus-coverage-stat/computeCoverageStat.err", "$wrk/5-consensus-coverage-stat/computeCoverageStat.err.FAILED";
+ caFailure("Unitig coverage stat computation failed", "$wrk/5-consensus-coverage-stat/computeCoverageStat.err.FAILED");
+ }
+
+ if (-d "$wrk/7-0-CGW") {
+ caFailure("Unitig coverage stat updated, but there is a scaffolding already started. Remove old scaffold directories to proceed.", "");
+ }
+ }
+
+ # All jobs finished. Remove the partitioning from the gatekeeper store.
+ #
+ #system("rm -f $wrk/$asm.gkpStore/???.[0-9][0-9][0-9]");
+
+ touch("$wrk/5-consensus/consensus.success");
+
+ alldone:
+ stopAfter("utgcns");
+ stopAfter("consensusAfterUnitigger");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+
+# Don't do interleaved merging unless we are throwing stones.
+
+sub CGW ($$$$$$) {
+ my $thisDir = shift @_;
+ my $lastDir = shift @_;
+ my $tigStore = shift @_;
+ my $stoneLevel = shift @_;
+ my $logickp = shift @_;
+ my $finalRun = shift @_;
+ my $lastckp = undef;
+ my $ckp = undef;
+
+ return($thisDir) if (-e "$wrk/$thisDir/cgw.success");
+
+ if (defined($lastDir)) {
+ $lastckp = findLastCheckpoint($lastDir);
+ }
+ if (defined($lastckp) && defined($logickp)) {
+ $ckp = "-R $lastckp -N $logickp"
+ }
+
+ # If there is a timing file here, assume we are restarting. Not
+ # all restarts are possible, but we try hard to make it so.
+ #
+ if (-e "$wrk/$thisDir/$asm.timing") {
+ my $restartckp = undef;
+
+ open(F, "< $wrk/$thisDir/$asm.timing");
+ while (<F>) {
+ if (m/Writing.*ckp.(\d+)\s\(logical\s(.+)\)/) {
+ $restartckp = "-R $1 -N $2";
+ }
+ }
+ close(F);
+
+ if (!defined($restartckp)) {
+ print STDERR "Found an empty timing file, starting from the beginning: $ckp\n";
+ } else {
+ $ckp = $restartckp;
+ print STDERR "Found a timing file, restarting: $ckp\n";
+ }
+ }
+
+ system("mkdir $wrk/$thisDir") if (! -d "$wrk/$thisDir");
+
+ system("ln -s ../$lastDir/$asm.ckp.$lastckp $wrk/$thisDir/$asm.ckp.$lastckp") if (defined($lastDir));
+
+ if (-e "$wrk/$thisDir/cgw.out") {
+ my $ckp = findLastCheckpoint($thisDir);
+ my $ver = "00";
+ while (-e "$wrk/$thisDir/cgw.out.$ver.ckp.$ckp") {
+ $ver++;
+ }
+ rename "$wrk/$thisDir/cgw.out", "$wrk/$thisDir/cgw.out.$ver.ckp.$ckp"
+ }
+
+ my $sampleSize = getGlobal("cgwDistanceSampleSize");
+
+ my $astatLow = getGlobal("astatLowBound");
+ my $astatHigh = getGlobal("astatHighBound");
+
+ my $B = int($numFrags / getGlobal("cnsPartitions"));
+ $B = getGlobal("cnsMinFrags") if ($B < getGlobal("cnsMinFrags"));
+
+ my $P = getGlobal("closurePlacement");
+
+ my $shatterLevel = getGlobal("cgwContigShatterWeight");
+ my $missingMate = getGlobal("cgwMergeMissingThreshold");
+ my $minWeight = getGlobal("cgwMinMergeWeight");
+ my $filterLevel = getGlobal("cgwMergeFilterLevel");
+
+ $cmd = "$bin/cgw $ckp \\\n";
+ $cmd .= " -j $astatLow -k $astatHigh \\\n";
+ $cmd .= " -r 5 \\\n";
+ $cmd .= " -s $stoneLevel \\\n";
+ $cmd .= " -filter $filterLevel \\\n" if ($filterLevel != 0);
+ $cmd .= " -minmergeweight $minWeight \\\n";
+ $cmd .= " -S 0 \\\n" if (($finalRun == 0) || (getGlobal("doResolveSurrogates") == 0));
+ $cmd .= " -G \\\n" if ($finalRun == 0);
+ $cmd .= " -GG \\\n" if (getGlobal("cgwPreserveConsensus") == 1);
+ $cmd .= " -z \\\n" if (getGlobal("cgwDemoteRBP") == 1);
+ $cmd .= " -P $P \\\n" if (defined($P));
+ $cmd .= " -K \\\n" if (getGlobal("kickOutNonOvlContigs") != 0);
+ $cmd .= " -U \\\n" if (getGlobal("doUnjiggleWhenMerging") != 0);
+ $cmd .= " -F \\\n" if (getGlobal("toggleDoNotDemote") != 0);
+ $cmd .= " -B $B \\\n";
+ $cmd .= " -u $wrk/4-unitigger/$asm.unused.ovl \\\n" if (getGlobal("cgwUseUnitigOverlaps") != 0);
+ $cmd .= " -reloadmates \\\n" if (getGlobal("cgwReloadMates") != 0);
+ $cmd .= " -shatter $shatterLevel \\\n";
+ $cmd .= " -missingMate $missingMate \\\n";
+ $cmd .= " -m $sampleSize \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $tigStore \\\n";
+ $cmd .= " -o $wrk/$thisDir/$asm \\\n";
+ $cmd .= " > $wrk/$thisDir/cgw.out 2>&1";
+
+ stopBefore("CGW", $cmd);
+
+ if (runCommand("$wrk/$thisDir", $cmd)) {
+ caFailure("scaffolder failed", "$wrk/$thisDir/cgw.out");
+ }
+
+ if (getGlobal("cgwCompressTigStore") != 0) {
+ my $f = findFirstCheckpoint($thisDir);
+ my $l = findLastCheckpoint($thisDir);
+
+ if ($f < $l) {
+ $cmd = "$bin/tigStore \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $tigStore $l \\\n";
+ $cmd .= " -compress \\\n";
+ $cmd .= "> $wrk/$thisDir/tigStore-compress.out 2>&1";
+
+ if (runCommand("$wrk/$thisDir", $cmd)) {
+ caFailure("tigStore compression failed", "$wrk/$thisDir/tigStore-compress.out");
+ }
+ }
+ }
+
+ if (getGlobal("cgwPurgeCheckpoints") != 0) {
+ my $f = findFirstCheckpoint($thisDir);
+ my $l = findLastCheckpoint($thisDir);
+
+ while ($f < $l) {
+ #print STDERR "Purging $wrk/$thisDir/$asm.ckp.$f\n";
+ unlink "$wrk/$thisDir/$asm.ckp.$f";
+ $f++;
+ }
+ }
+
+ touch("$wrk/$thisDir/cgw.success");
+
+ return $thisDir;
+}
+
+
+sub eCR ($$$) {
+ my $thisDir = shift @_;
+ my $lastDir = shift @_;
+ my $iter = shift @_;
+
+ return $thisDir if (-e "$wrk/$thisDir/extendClearRanges.success");
+
+ my $lastckp = findLastCheckpoint($lastDir);
+
+ system("mkdir $wrk/$thisDir") if (! -d "$wrk/$thisDir");
+
+ system("ln -s ../$lastDir/$asm.ckp.$lastckp $wrk/$thisDir/$asm.ckp.$lastckp") if (! -e "$wrk/$thisDir/$asm.ckp.$lastckp");
+
+ # Partition eCR.
+
+ if (! -e "$wrk/$thisDir/extendClearRanges.partitionInfo") {
+ $cmd = "$bin/extendClearRangesPartition ";
+ $cmd .= " -g $wrk/$asm.gkpStore ";
+ $cmd .= " -t $wrk/$asm.tigStore ";
+ $cmd .= " -n $lastckp ";
+ $cmd .= " -c $asm ";
+ $cmd .= " -N 4 ";
+ $cmd .= " -p $wrk/$thisDir/extendClearRanges.partitionInfo";
+ $cmd .= " > $wrk/$thisDir/extendClearRanges.partitionInfo.err 2>&1";
+
+ stopBefore("eCRPartition", $cmd);
+ stopBefore("extendClearRangesPartition", $cmd);
+
+ if (runCommand("$wrk/$thisDir", $cmd)) {
+ caFailure("extendClearRanges partitioning failed", "$wrk/$thisDir/extendClearRanges.partitionInfo.err");
+ }
+
+ # Remove any existing eCR scripts -- possibly left behind by the user deleting
+ # the partitioinInfo and restarting.
+
+ open(F, "find -L $wrk/$thisDir -name 'extendClearRanges-scaffold.*' -print |");
+ while (<F>) {
+ chomp;
+ print STDERR "Repartitioned; remove extraneous file $_\n";
+ unlink $_;
+ }
+ close(F);
+ }
+
+ # Read the partitioning info, create jobs. No partitions? No ECR jobs.
+
+ my @jobs;
+
+ open(P, "< $wrk/$thisDir/extendClearRanges.partitionInfo") or caFailure("failed to find extendClearRanges partitioning file $wrk/$thisDir/extendClearRanges.partitionInfo", undef);
+ while (<P>) {
+ # Fields are: partitionNum BgnScf partitionNum EndScf NumFrags
+
+ my @v = split '\s+', $_;
+
+ my $curScaffold = substr("000000000$v[1]", -7);
+ my $endScaffold = substr("000000000$v[3]", -7);
+
+ my $j = "$wrk/$thisDir/extendClearRanges-scaffold.$curScaffold";
+
+ if (! -e "$j.success") {
+ if (! -e "$j.sh") {
+ open(F, "> $j.sh");
+ print F "#!" . getGlobal("shell") . "\n\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=", getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=", getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=", getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+ print F "\n";
+ print F "$bin/extendClearRanges \\\n";
+ print F " -g $wrk/$asm.gkpStore \\\n";
+ print F " -t $wrk/$asm.tigStore \\\n";
+ print F " -n $lastckp \\\n";
+ print F " -c $asm \\\n";
+ print F " -b $curScaffold -e $endScaffold \\\n";
+ print F " -i $iter \\\n";
+ print F " > $j.err 2>&1\n";
+ close(F);
+
+ system("chmod +x $j.sh");
+ }
+
+ push @jobs, "$j";
+
+ $lastckp++;
+ }
+ }
+ close(P);
+
+ # Run jobs.
+
+ stopBefore("eCR", undef);
+ stopBefore("extendClearRanges", undef);
+
+ foreach my $j (@jobs) {
+ if (runCommand("$wrk/$thisDir", "$j.sh")) {
+ caFailure("extendClearRanges failed", "$j.err");
+ }
+ touch("$j.success");
+ }
+
+ touch("$wrk/$thisDir/extendClearRanges.success");
+
+ return $thisDir;
+}
+
+
+sub updateDistanceRecords ($) {
+ my $thisDir = shift @_;
+
+ return $thisDir if (-e "$wrk/$thisDir/$asm.distupdate.success");
+
+ open(F, "> $wrk/$thisDir/$asm.distupdate");
+
+ print F "## Scaffold Based Estimates\n";
+ if (-e "$wrk/$thisDir/stat/scaffold_final.distupdate") {
+ open(A, "< $wrk/$thisDir/stat/scaffold_final.distupdate");
+ while (<A>) {
+ print F $_;
+ }
+ close(A);
+ }
+
+ print F "## Contig Based Estimates\n";
+ if (-e "$wrk/$thisDir/stat/contig_final.distupdate") {
+ open(A, "< $wrk/$thisDir/stat/contig_final.distupdate");
+ while (<A>) {
+ print F $_;
+ }
+ close(A);
+ }
+
+ close(F);
+
+ $cmd = "$bin/gatekeeper \\\n";
+ $cmd .= " --edit $wrk/$thisDir/$asm.distupdate \\\n";
+ $cmd .= " $wrk/$asm.gkpStore \\\n";
+ $cmd .= "> $wrk/$thisDir/$asm.distupdate.err 2>&1";
+
+ if (runCommand("$wrk/$thisDir", $cmd)) {
+ caFailure("Insert size updates failed", "$wrk/$thisDir/$asm.distupdate.err");
+ }
+
+ touch("$wrk/$thisDir/$asm.distupdate.success");
+
+ return $thisDir;
+}
+
+
+sub scaffolder () {
+ my $lastDir = undef;
+ my $thisDir = 0;
+ my $stoneLevel = getGlobal("stoneLevel");
+
+ stopBefore("scaffolder", undef);
+
+ goto alldone if (-e "$wrk/7-CGW/cgw.success");
+
+ # Do an initial CGW to update distances, then update the
+ # gatekeeper. This initial run shouldn't be used for later
+ # CGW'ing.
+ #
+ my $cis = getGlobal("computeInsertSize");
+ if (!defined($cis) && ($numFrags < 1000000)) {
+ $cis = 1;
+ }
+ if ($cis == 1) {
+ if (! -e "$wrk/6-clonesize/$asm.tigStore") {
+ system("mkdir -p $wrk/6-clonesize/$asm.tigStore");
+ system("cd $wrk/6-clonesize/$asm.tigStore && ln -s ../../$asm.tigStore/* .");
+ }
+ updateDistanceRecords(CGW("6-clonesize", undef, "$wrk/6-clonesize/$asm.tigStore", $stoneLevel, undef, 0));
+ }
+
+
+ # If we're not doing eCR, we just do a single scaffolder run, and
+ # get the heck outta here!
+ #
+ if (getGlobal("doExtendClearRanges") == 0) {
+ $lastDir = CGW("7-$thisDir-CGW", $lastDir, "$wrk/$asm.tigStore", $stoneLevel, undef, 1);
+ $thisDir++;
+ } else {
+
+ # Do the initial CGW, making sure to not throw stones.
+ #
+ $lastDir = updateDistanceRecords(CGW("7-$thisDir-CGW", $lastDir, "$wrk/$asm.tigStore", 0, undef, 0));
+ $thisDir++;
+
+ # Followed by at least one eCR
+ #
+ $lastDir = eCR("7-$thisDir-ECR", $lastDir, 1);
+ $thisDir++;
+
+ # Iterate eCR: do another scaffolder still without stones,
+ # then another eCR. Again, and again, until we get dizzy and
+ # fall over.
+ #
+ my $iterationMax = getGlobal("doExtendClearRanges") + 1;
+ for (my $iteration = 2; $iteration < $iterationMax; $iteration++) {
+ $lastDir = updateDistanceRecords(CGW("7-$thisDir-CGW", $lastDir, "$wrk/$asm.tigStore", 0, "ckp03-SCF-partial", 0));
+ $thisDir++;
+
+ $lastDir = eCR("7-$thisDir-ECR", $lastDir, $iteration);
+ $thisDir++;
+ }
+
+ # Then another scaffolder, chucking stones into the big holes,
+ # filling in surrogates, and writing output.
+ #
+ $lastDir = updateDistanceRecords(CGW("7-$thisDir-CGW", $lastDir, "$wrk/$asm.tigStore", $stoneLevel, "ckp03-SCF-partial", 1));
+ $thisDir++;
+ }
+
+
+ # And, finally, hold on, we're All Done! Point to the correct output directory.
+ #
+ system("ln -s $lastDir $wrk/7-CGW") if (! -d "$wrk/7-CGW");
+
+ alldone:
+ stopAfter("scaffolder");
+}
+
+
+################################################################################
+################################################################################
+################################################################################
+
+
+# Prepare for consensus on the grid
+# Partition the contigs
+# Repartition the frag store
+
+sub createPostScaffolderConsensusJobs () {
+ my $consensusType = getGlobal("consensus");
+
+ return if (-e "$wrk/8-consensus/consensus.sh");
+
+ caFailure("contig consensus didn't find '$wrk/$asm.tigStore'", undef) if (! -d "$wrk/$asm.tigStore");
+
+ my $tigVersion = findLastCheckpoint("$wrk/7-CGW");
+ caFailure("contig consensus didn't find any checkpoints in '$wrk/7-CGW'", undef) if (!defined($tigVersion));
+
+ ########################################
+ #
+ # Partition the gkpStore for consensus.
+ #
+ if (! -e "$wrk/8-consensus/$asm.partitioned") {
+ $cmd = "$bin/gatekeeper -P $wrk/7-CGW/$asm.partitioning $wrk/$asm.gkpStore ";
+ $cmd .= "> $wrk/8-consensus/$asm.partitioned.err 2>&1";
+
+ caFailure("gatekeeper partitioning failed", "$wrk/8-consensus/$asm.partitioned.err") if (runCommand("$wrk/8-consensus", $cmd));
+ touch("$wrk/8-consensus/$asm.partitioned");
+ }
+
+ ########################################
+ #
+ # Build consensus jobs for the grid -- this is very similar to that in createPostUnitiggerConsensus.pl
+ #
+ my $jobs = 0;
+
+ open(F, "< $wrk/7-CGW/$asm.partitionInfo") or caFailure("can't open '$wrk/7-CGW/$asm.partitionInfo'", undef);
+ while (<F>) {
+ if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+contigs\sand\s+(\d+)\s+fragments./) {
+ $jobs = $1;
+ }
+ }
+ close(F);
+
+ my $taskID = getGlobal("gridTaskID");
+ open(F, "> $wrk/8-consensus/consensus.sh") or caFailure("can't open '$wrk/8-consensus/consensus.sh'", undef);
+ print F "#!" . getGlobal("shell") . "\n";
+ print F "\n";
+ print F "jobid=\$$taskID\n";
+ print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n";
+ print F " jobid=\$1\n";
+ print F "fi\n";
+ print F "if [ x\$jobid = x ]; then\n";
+ print F " echo Error: I need $taskID set, or a job index on the command line.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "if [ \$jobid -gt $jobs ]; then\n";
+ print F " echo Error: Only $jobs partitions, you asked for \$jobid.\n";
+ print F " exit 1\n";
+ print F "fi\n";
+ print F "\n";
+ print F "jobid=`printf %03d \$jobid`\n";
+ print F "\n";
+ print F "if [ -e $wrk/8-consensus/${asm}_\$jobid.success ] ; then\n";
+ print F " exit 0\n";
+ print F "fi\n";
+ print F "\n";
+ print F "AS_OVL_ERROR_RATE=", getGlobal("ovlErrorRate"), "\n";
+ print F "AS_CNS_ERROR_RATE=", getGlobal("cnsErrorRate"), "\n";
+ print F "AS_CGW_ERROR_RATE=", getGlobal("cgwErrorRate"), "\n";
+ print F "AS_OVERLAP_MIN_LEN=", getGlobal("ovlMinLen"), "\n";
+ print F "AS_READ_MIN_LEN=" , getGlobal("frgMinLen"), "\n";
+ print F "export AS_OVL_ERROR_RATE AS_CNS_ERROR_RATE AS_CGW_ERROR_RATE AS_OVERLAP_MIN_LEN AS_READ_MIN_LEN\n";
+
+ print F getBinDirectoryShellCode();
+
+ if ($consensusType eq "cns") {
+ print F "\$bin/ctgcns \\\n";
+ print F " -g $wrk/$asm.gkpStore \\\n";
+ print F " -t $wrk/$asm.tigStore $tigVersion \$jobid \\\n";
+ print F " -P ", getGlobal("cnsPhasing"), " \\\n";
+ print F " -U \\\n" if (getGlobal("cnsReuseUnitigs") != 0);
+ print F " > $wrk/8-consensus/${asm}_\$jobid.err 2>&1 \\\n";
+ print F "&& \\\n";
+ print F "touch $wrk/8-consensus/${asm}_\$jobid.success\n";
+ } elsif ($consensusType eq "seqan") {
+ print F "\$bin/SeqAn_CNS \\\n";
+ print F " -G $wrk/$asm.gkpStore \\\n";
+ print F " -u $wrk/$asm.SeqStore \\\n";
+ print F " -V $tigVersion \\\n";
+ print F " -p \$jobid \\\n";
+ print F " -S \$jobid \\\n";
+ #print F " -c $cgwDir/$asm.cgw_contigs.\$jobid \\\n";
+ print F " -s \$bin/graph_consensus \\\n";
+ print F " -w $wrk/8-consensus/ \\\n";
+ print F " -o $wrk/8-consensus/$asm.cns_contigs.\$jobid \\\n";
+ print F " > $wrk/8-consensus/$asm.cns_contigs.\$jobid.err 2>&1 \\\n";
+ print F "&& \\\n";
+ print F "touch $wrk/8-consensus/$asm.cns_contigs.\$jobid.success\n";
+ } else {
+ caFailure("unknown consensus type $consensusType; must be 'cns' or 'seqan'", undef);
+ }
+ print F "exit 0\n";
+ close(F);
+
+ chmod 0755, "$wrk/8-consensus/consensus.sh";
+
+ if (getGlobal("cnsOnGrid") && getGlobal("useGrid")) {
+ my $submitCommand = getGlobal("gridSubmitCommand");
+ my $nameOption = getGlobal("gridNameOption");
+ my $outputOption = getGlobal("gridOutputOption");
+
+ my $sge = getGlobal("sge");
+ my $sgeName = getGlobal("sgeName");
+ my $sgeConsensus = getGlobal("sgeConsensus");
+
+ $sgeName = "_$sgeName" if (defined($sgeName));
+ my $jobName = getGridArrayName("ctg_$asm$sgeName", $jobs);
+ my $arrayOpt = getGridArrayOption("ctg_$asm$sgeName", $jobs);
+
+ my $SGE;
+ $SGE = "$submitCommand $sge $sgeConsensus $nameOption \"$jobName\" $arrayOpt ";
+ $SGE .= "$outputOption /dev/null ";
+ $SGE .= "$wrk/8-consensus/consensus.sh\n";
+
+ submitBatchJobs($SGE, $jobName);
+ exit(0);
+ } else {
+ for (my $i=1; $i<=$jobs; $i++) {
+ schedulerSubmit("$wrk/8-consensus/consensus.sh $i > /dev/null 2>&1");
+ }
+
+ schedulerSetNumberOfProcesses(getGlobal("cnsConcurrency"));
+ schedulerFinish();
+ }
+}
+
+
+sub postScaffolderConsensus () {
+
+ system("mkdir $wrk/8-consensus") if (! -d "$wrk/8-consensus");
+
+ goto alldone if (-e "$wrk/8-consensus/consensus.success");
+
+ createPostScaffolderConsensusJobs();
+
+ #
+ # Check that consensus finished properly
+ #
+ my $failedJobs = 0;
+
+ open(F, "< $wrk/7-CGW/$asm.partitionInfo") or caFailure("can't open '$wrk/7-CGW/$asm.partitionInfo'", undef);
+ while (<F>) {
+ if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+contigs\sand\s+(\d+)\s+fragments./) {
+ my $id = substr("000" . $1, -3);
+
+ if (! -e "$wrk/8-consensus/${asm}_$id.success") {
+ print STDERR "$wrk/8-consensus/${asm}_$id failed -- no .success.\n";
+ $failedJobs++;
+ }
+ }
+ }
+ close(F);
+
+ # FAILUREHELPME
+ #
+ caFailure("$failedJobs consensusAfterScaffolder jobs failed; remove $wrk/8-consensus/consensus.sh to try again", undef) if ($failedJobs);
+
+ # All jobs finished. Remove the partitioning from the gatekeeper store.
+ #
+ #system("rm -f $wrk/$asm.gkpStore/???.[0-9][0-9][0-9]");
+
+ touch("$wrk/8-consensus/consensus.success");
+
+ alldone:
+ stopAfter("ctgcns");
+ stopAfter("consensusAfterScaffolder");
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+
+sub summarizeConsensusStatistics ($) {
+ my $dir = shift @_;
+
+ if (! -e "$dir/consensus.stats.summary") {
+ my $NumColumnsInUnitigs = 0;
+ my $NumGapsInUnitigs = 0;
+ my $NumRunsOfGapsInUnitigReads = 0;
+ my $NumColumnsInContigs = 0;
+ my $NumGapsInContigs = 0;
+ my $NumRunsOfGapsInContigReads = 0;
+ my $NumAAMismatches = 0;
+ my $NumFAMismatches = 0;
+ my $NumVARRecords = 0;
+ my $NumVARStringsWithFlankingGaps = 0;
+ my $NumUnitigRetrySuccess = 0;
+
+ open(F, "ls $dir/$asm*.err |");
+ my @files = <F>;
+ chomp @files;
+ close(F);
+
+ foreach my $f (@files) {
+ open(F, "< $f");
+ while (<F>) {
+ $NumColumnsInUnitigs += $1 if (m/NumColumnsInUnitigs\s+=\s+(\d+)/);
+ $NumGapsInUnitigs += $1 if (m/NumGapsInUnitigs\s+=\s+(\d+)/);
+ $NumRunsOfGapsInUnitigReads += $1 if (m/NumRunsOfGapsInUnitigReads\s+=\s+(\d+)/);
+ $NumColumnsInContigs += $1 if (m/NumColumnsInContigs\s+=\s+(\d+)/);
+ $NumGapsInContigs += $1 if (m/NumGapsInContigs\s+=\s+(\d+)/);
+ $NumRunsOfGapsInContigReads += $1 if (m/NumRunsOfGapsInContigReads\s+=\s+(\d+)/);
+ $NumAAMismatches += $1 if (m/NumAAMismatches\s+=\s+(\d+)/);
+ $NumFAMismatches += $1 if (m/NumFAMismatches\s+=\s+(\d+)/);
+ $NumVARRecords += $1 if (m/NumVARRecords\s+=\s+(\d+)/);
+ $NumVARStringsWithFlankingGaps += $1 if (m/NumVARStringsWithFlankingGaps\s+=\s+(\d+)/);
+ $NumUnitigRetrySuccess += $1 if (m/NumUnitigRetrySuccess\s+=\s+(\d+)/);
+ }
+ close(F);
+ }
+
+ open(F, "> $dir/consensus.stats.summary");
+ print F "NumColumnsInUnitigs=$NumColumnsInUnitigs\n" if ($NumColumnsInUnitigs > 0);
+ print F "NumGapsInUnitigs=$NumGapsInUnitigs\n" if ($NumGapsInUnitigs > 0);
+ print F "NumRunsOfGapsInUnitigReads=$NumRunsOfGapsInUnitigReads\n" if ($NumRunsOfGapsInUnitigReads > 0);
+ print F "NumColumnsInContigs=$NumColumnsInContigs\n" if ($NumColumnsInContigs > 0);
+ print F "NumGapsInContigs=$NumGapsInContigs\n" if ($NumGapsInContigs > 0);
+ print F "NumRunsOfGapsInContigReads=$NumRunsOfGapsInContigReads\n" if ($NumRunsOfGapsInContigReads > 0);
+ print F "NumAAMismatches=$NumAAMismatches\n" if ($NumAAMismatches > 0);
+ print F "NumFAMismatches=$NumFAMismatches\n" if ($NumFAMismatches > 0);
+ print F "NumVARRecords=$NumVARRecords\n" if ($NumVARRecords > 0);
+ print F "NumVARStringsWithFlankingGaps=$NumVARStringsWithFlankingGaps\n" if ($NumVARStringsWithFlankingGaps > 0);
+ print F "NumUnitigRetrySuccess=$NumUnitigRetrySuccess\n" if ($NumUnitigRetrySuccess > 0);
+ close(F);
+ }
+}
+
+
+
+sub terminate () {
+ my $perl = "/usr/bin/env perl";
+
+ my $termDir = "$wrk/9-terminator";
+ system("mkdir $termDir") if (! -e "$termDir");
+
+ stopBefore("terminator", undef);
+
+ if (! -e "$termDir/$asm.asm") {
+ my $uidServer = getGlobal("uidServer");
+ my $fakeUIDs = getGlobal("fakeUIDs");
+
+ my $ckpVersion = findLastCheckpoint("$wrk/7-CGW");
+ my $tigVersion = $ckpVersion + 1;
+
+ caFailure("contig consensus didn't find any checkpoints in '$wrk/7-CGW'", undef) if (!defined($tigVersion));
+
+ $cmd = "$bin/terminator";
+ $cmd .= " -E $uidServer" if (defined($uidServer));
+ $cmd .= " -s $fakeUIDs" if ($fakeUIDs > 0);
+ $cmd .= " -g $wrk/$asm.gkpStore";
+ $cmd .= " -t $wrk/$asm.tigStore $tigVersion";
+ $cmd .= " -c $wrk/7-CGW/$asm $ckpVersion";
+ $cmd .= " -o $termDir/$asm";
+ $cmd .= " > $termDir/$asm.asm.err 2>&1";
+
+ if (runCommand("$termDir", $cmd)) {
+ rename "$termDir/$asm.asm", "$termDir/$asm.asm.FAILED";
+ rename "$termDir/$asm.map", "$termDir/$asm.map.FAILED";
+ caFailure("terminator failed", "$termDir/$asm.asm.err");
+ }
+ }
+
+
+ my $asmOutputFasta = "$bin/asmOutputFasta";
+ if (! -e "$termDir/$asm.scf.fasta") {
+ $cmd = "$asmOutputFasta -p $termDir/$asm $termDir/$asm.asm > $termDir/asmOutputFasta.err 2>&1";
+ if (runCommand("$termDir", $cmd)) {
+ rename "$termDir/$asm.scfcns.fasta", "$termDir/$asm.scfcns.fasta.FAILED";
+ caFailure("fasta output failed", "$termDir/asmOutputFasta.err");
+ }
+ unlink "$termDir/asmOutputFasta.err";
+ }
+
+
+ if (! -e "$termDir/$asm.singleton.fasta") {
+ my $ckpVersion = findLastCheckpoint("$wrk/7-CGW");
+ my $tigVersion = $ckpVersion + 1;
+
+ $cmd = "$bin/dumpSingletons ";
+ $cmd .= " -g $wrk/$asm.gkpStore ";
+ $cmd .= " -t $wrk/$asm.tigStore ";
+ $cmd .= " -c $wrk/7-CGW/$asm -n $ckpVersion -S ";
+ $cmd .= "> $termDir/$asm.singleton.fasta ";
+ $cmd .= "2> $termDir/dumpSingletons.err ";
+ if (runCommand("$termDir", $cmd)) {
+ print STDERR "Failed.\n";
+ rename "$termDir/$asm.singleton.fasta", "$termDir/$asm.singleton.fasta.FAILED";
+ }
+ unlink "$termDir/dumpSingletons.err";
+ }
+
+
+ ########################################
+ #
+ # Generate fragment/unitig/contig/scaffold mappings
+ #
+ ########################################
+
+
+ if (getGlobal("createPosMap") > 0) {
+ if (! -e "$termDir/$asm.posmap.frgscf") {
+ if (runCommand("$termDir", "$bin/buildPosMap -o $asm -g $wrk/$asm.gkpStore < $termDir/$asm.asm > $termDir/buildPosMap.err 2>&1")) {
+ rename "$termDir/$asm.posmap.frgscf", "$termDir/$asm.posmap.frgscf.FAILED";
+ caFailure("buildPosMap failed", "$termDir/buildPosMap.err");
+ }
+ unlink "$termDir/buildPosMap.err";
+ }
+ }
+
+ ########################################
+ #
+ # Generate a read depth histogram
+ #
+ ########################################
+ if (( -e "$termDir/$asm.posmap.frgscf") &&
+ (! -e "$termDir/$asm.qc.readdepth") &&
+ (! -e "$termDir/$asm.qc")) {
+
+ # Youch. Run five commands, do something if all are successful.
+
+ $cmd = "sort -k2n -k3n -T $termDir $termDir/$asm.posmap.frgscf > $termDir/$asm.posmap.frgscf.sorted &&";
+ $cmd .= "$bin/fragmentDepth -min 0 -max 3000 < $termDir/$asm.posmap.frgscf.sorted > $termDir/$asm.posmap.frgscf.histogram1 && ";
+ $cmd .= "$bin/fragmentDepth -min 3001 -max 10000 < $termDir/$asm.posmap.frgscf.sorted > $termDir/$asm.posmap.frgscf.histogram2 && ";
+ $cmd .= "$bin/fragmentDepth -min 10001 -max 1000000 < $termDir/$asm.posmap.frgscf.sorted > $termDir/$asm.posmap.frgscf.histogram3 && ";
+ $cmd .= "$bin/fragmentDepth -min 1000001 < $termDir/$asm.posmap.frgscf.sorted > $termDir/$asm.posmap.frgscf.histogram4 ";
+
+ if (runCommand("$termDir", $cmd) == 0) {
+ my @H1;
+ my @H2;
+ my @H3;
+ my @H4;
+ my $histMax = 0;
+
+ open(G, "< $termDir/$asm.posmap.frgscf.histogram1") or caFailure("failed to open '$termDir/$asm.posmap.frgscf.histogram1'", undef);
+ while (<G>) {
+ my ($v, $s) = split '\s+', $_;
+ $H1[$v] = $s;
+ $histMax = $v if ($histMax < $v);
+ }
+ close(G);
+
+ open(G, "< $termDir/$asm.posmap.frgscf.histogram2") or caFailure("failed to open '$termDir/$asm.posmap.frgscf.histogram2'", undef);
+ while (<G>) {
+ my ($v, $s) = split '\s+', $_;
+ $H2[$v] = $s;
+ $histMax = $v if ($histMax < $v);
+ }
+ close(G);
+
+ open(G, "< $termDir/$asm.posmap.frgscf.histogram3") or caFailure("failed to open '$termDir/$asm.posmap.frgscf.histogram3'", undef);
+ while (<G>) {
+ my ($v, $s) = split '\s+', $_;
+ $H3[$v] = $s;
+ $histMax = $v if ($histMax < $v);
+ }
+ close(G);
+
+ open(G, "< $termDir/$asm.posmap.frgscf.histogram4") or caFailure("failed to open '$termDir/$asm.posmap.frgscf.histogram4'", undef);
+ while (<G>) {
+ my ($v, $s) = split '\s+', $_;
+ $H4[$v] = $s;
+ $histMax = $v if ($histMax < $v);
+ }
+ close(G);
+
+ open(G, "> $termDir/$asm.qc.readdepth");
+ print G "\n[Read Depth Histogram]\n";
+ print G "d < 3Kbp < 10Kbp < 1Mbp < inf\n";
+ for (my $v=0; $v<=$histMax; $v++) {
+ printf(G "%-4d %-10d %-10d %-10d %-10d\n", $v, int($H1[$v]), int($H2[$v]), int($H3[$v]), int($H4[$v]));
+ }
+ }
+
+ # Remove our temporary files.
+
+ unlink "$termDir/$asm.posmap.frgscf.histogram1";
+ unlink "$termDir/$asm.posmap.frgscf.histogram2";
+ unlink "$termDir/$asm.posmap.frgscf.histogram3";
+ unlink "$termDir/$asm.posmap.frgscf.histogram4";
+ }
+
+
+ ########################################
+ #
+ # Generate statistics.
+ #
+ ########################################
+
+ if (! -e "$termDir/$asm.qc") {
+ my $qcOptions;
+
+ if ( -e "$wrk/$asm.frg" ) {
+ link "$wrk/$asm.frg", "$termDir/$asm.frg";
+ $qcOptions = "-metrics";
+ }
+ if ( -e "$wrk/$asm.catmap" && !-e "$termDir/$asm.catmap" ) {
+ link "$wrk/$asm.catmap", "$termDir/$asm.catmap";
+ }
+ if ( -e "$wrk/$asm.seq.features" && !-e "$termDir/$asm.seq.features" ) {
+ link "$wrk/$asm.seq.features", "$termDir/$asm.seq.features";
+ }
+ if (runCommand("$termDir", "$perl $bin/caqc.pl -euid $qcOptions $termDir/$asm.asm")) {
+ rename "$termDir/$asm.qc", "$termDir/$asm.qc.FAILED";
+ }
+
+ summarizeConsensusStatistics("$wrk/5-consensus");
+ summarizeConsensusStatistics("$wrk/8-consensus");
+
+ open(F, ">> $termDir/$asm.qc") or caFailure("failed to append to '$termDir/$asm.qc'", undef);
+
+ if (-e "$wrk/5-consensus/consensus.stats.summary") {
+ print F "\n[Unitig Consensus]\n";
+ open(G, "< $wrk/5-consensus/consensus.stats.summary") or caFailure("failed to open '$wrk/5-consensus/consensus.stats.summary'", undef);
+ while (<G>) {
+ print F $_;
+ }
+ close(G);
+ }
+
+ if (-e "$wrk/8-consensus/consensus.stats.summary") {
+ print F "\n[Contig Consensus]\n";
+ open(G, "< $wrk/8-consensus/consensus.stats.summary") or caFailure("failed to open '$wrk/8-consensus/consensus.stats.summary'", undef);
+ while (<G>) {
+ print F $_;
+ }
+ close(G);
+ }
+
+ if (-e "$termDir/$asm.qc.readdepth") {
+ open(G, "< $termDir/$asm.qc.readdepth") or caFailure("failed to open '$termDir/$asm.qc.readdepth'", undef);
+ while (<G>) {
+ print F $_;
+ }
+ close(G);
+ }
+
+ close(F);
+
+ unlink "$wrk/5-consensus/consensus.stats.summary";
+ unlink "$wrk/8-consensus/consensus.stats.summary";
+ unlink "$termDir/$asm.qc.readdepth";
+ }
+
+
+ ########################################
+ #
+ # Mercy merQC
+ #
+ ########################################
+
+
+ if ((getGlobal("merQC") > 0) &&
+ (! -e "$termDir/$asm.merQC") &&
+ (merylVersion() eq "Mighty")) {
+
+ system("mkdir $termDir/mercy") if (! -e "$termDir/mercy");
+
+ my $ms = getGlobal("merQCmerSize");
+ my $mem = getGlobal("merQCmemory");
+ my $verbose = "";
+
+ if (! -e "$termDir/mercy/$asm-ms$ms-frgFull.mcidx") {
+ $cmd = "$bin/meryl -B -C -m $ms -threads 4 -memory $mem $verbose ";
+ $cmd .= "-s $wrk/$asm.gkpStore:untrim ";
+ $cmd .= "-o $termDir/mercy/$asm-ms$ms-frgFull";
+ if (runCommand("$termDir/mercy", $cmd)) {
+ print STDERR "Failed.\n";
+ unlink "$termDir/mercy/$asm-ms$ms-frgFull.mcidx";
+ unlink "$termDir/mercy/$asm-ms$ms-frgFull.mcdat";
+ }
+ }
+ if (! -e "$termDir/mercy/$asm-ms$ms-frgTrim.mcidx") {
+ $cmd = "$bin/meryl -B -C -m $ms -threads 4 -memory $mem $verbose ";
+ $cmd .= "-s $wrk/$asm.gkpStore ";
+ $cmd .= "-o $termDir/mercy/$asm-ms$ms-frgTrim";
+ if (runCommand("$termDir/mercy", $cmd)) {
+ print STDERR "Failed.\n";
+ unlink "$termDir/mercy/$asm-ms$ms-frgTrim.mcidx";
+ unlink "$termDir/mercy/$asm-ms$ms-frgTrim.mcdat";
+ }
+ }
+
+ # XXX This can likely be optimized -- by feeding
+ # asmOutputcontigsFasta directly to meryl. It'd be harder
+ # (but great) if only one pass through the asm file could be
+ # made. Easier then if we write all three files at the same
+ # time.
+
+ if (! -e "$termDir/mercy/$asm.ctgNorm.fasta") {
+ link "$termDir/$asm.ctg.fasta", "$termDir/mercy/$asm.ctgNorm.fasta";
+ }
+ if (! -e "$termDir/mercy/$asm.ctgDreg.fasta") {
+ link "$termDir/$asm.deg.fasta", "$termDir/mercy/$asm.ctgDreg.fasta";
+ }
+ if (! -e "$termDir/mercy/$asm.ctgAll.fasta") {
+ system "cat $termDir/$asm.{ctg,deg}.fasta > $termDir/mercy/$asm.ctgAll.fasta";
+ }
+
+ if ((! -e "$termDir/mercy/$asm-ms$ms-ctgNorm.mcidx") &&
+ (-e "$termDir/mercy/$asm.ctgNorm.fasta")) {
+ $cmd = "$bin/meryl -B -C -m $ms -threads 4 -segments 4 $verbose ";
+ $cmd .= "-s $termDir/mercy/$asm.ctgNorm.fasta ";
+ $cmd .= "-o $termDir/mercy/$asm-ms$ms-ctgNorm";
+ if (runCommand("$termDir/mercy", $cmd)) {
+ print STDERR "Failed.\n";
+ unlink "$termDir/mercy/$asm-ms$ms-ctgNorm.mcidx";
+ unlink "$termDir/mercy/$asm-ms$ms-ctgNorm.mcdat";
+ }
+ }
+ if ((! -e "$termDir/mercy/$asm-ms$ms-ctgDreg.mcidx") &&
+ (-e "$termDir/mercy/$asm.ctgDreg.fasta")) {
+ $cmd = "$bin/meryl -B -C -m $ms -threads 4 -segments 4 $verbose ";
+ $cmd .= "-s $termDir/mercy/$asm.ctgDreg.fasta ";
+ $cmd .= "-o $termDir/mercy/$asm-ms$ms-ctgDreg";
+ if (runCommand("$termDir/mercy", $cmd)) {
+ print STDERR "Failed.\n";
+ unlink "$termDir/mercy/$asm-ms$ms-ctgDreg.mcidx";
+ unlink "$termDir/mercy/$asm-ms$ms-ctgDreg.mcdat";
+ }
+ }
+ if ((! -e "$termDir/mercy/$asm-ms$ms-ctgAll.mcidx") &&
+ (-e "$termDir/mercy/$asm.ctgAll.fasta")) {
+ $cmd = "$bin/meryl -B -C -m $ms -threads 4 -segments 4 $verbose ";
+ $cmd .= "-s $termDir/mercy/$asm.ctgAll.fasta ";
+ $cmd .= "-o $termDir/mercy/$asm-ms$ms-ctgAll";
+ if (runCommand("$termDir/mercy", $cmd)) {
+ print STDERR "Failed.\n";
+ unlink "$termDir/mercy/$asm-ms$ms-ctgAll.mcidx";
+ unlink "$termDir/mercy/$asm-ms$ms-ctgAll.mcdat";
+ }
+ }
+
+ if (! -e "$termDir/$asm-ms$ms.merQC") {
+ $cmd = "$bin/mercy ";
+ $cmd .= "-af $termDir/mercy/$asm-ms$ms-frgFull " if (-e "$termDir/mercy/$asm-ms$ms-frgFull.mcidx");
+ $cmd .= "-tf $termDir/mercy/$asm-ms$ms-frgTrim " if (-e "$termDir/mercy/$asm-ms$ms-frgTrim.mcidx");
+ $cmd .= "-co $termDir/mercy/$asm-ms$ms-ctgNorm " if (-e "$termDir/mercy/$asm-ms$ms-ctgNorm.mcidx");
+ $cmd .= "-dc $termDir/mercy/$asm-ms$ms-ctgDreg " if (-e "$termDir/mercy/$asm-ms$ms-ctgDreg.mcidx");
+ $cmd .= "-ac $termDir/mercy/$asm-ms$ms-ctgAll " if (-e "$termDir/mercy/$asm-ms$ms-ctgAll.mcidx");
+ $cmd .= "> $termDir/$asm-ms$ms.merQC";
+ if (runCommand("$termDir/mercy", $cmd)) {
+ print STDERR "Failed.\n";
+ rename "$termDir/$asm-ms$ms.merQC", "$termDir/$asm-ms$ms.merQC.FAILED";
+ }
+ }
+ }
+
+
+ ########################################
+ #
+ # AGP and ACE file generation
+ #
+ ########################################
+
+
+ if (getGlobal("createAGP") > 0) {
+ if (! -e "$termDir/$asm.agp") {
+ if (runCommand($termDir, "$perl $bin/asmToAGP.pl < $termDir/$asm.asm > $termDir/$asm.agp")) {
+ rename "$termDir/$asm.agp", "$termDir/$asm.agp.FAILED";
+ }
+ }
+ }
+
+ if (getGlobal("createACE") > 0) {
+ if (! -e "$termDir/$asm.ace.bz2") {
+ if (! -e "$termDir/$asm.frg") {
+ if (runCommand($termDir, "$bin/gatekeeper -dumpfrg -allreads $wrk/$asm.gkpStore > $termDir/$asm.frg 2> $termDir/gatekeeper.err")) {
+ caFailure("gatekeeper failed to dump fragments for ACE generation", "$termDir/gatekeeper.err");
+ }
+ unlink "$termDir/gatekeeper.err";
+ }
+ if (runCommand($termDir, "$perl $bin/ca2ace.pl $termDir/$asm.asm")) {
+ rename "$termDir/$asm.ace.bz2", "$termDir/$asm.ace.FAILED.bz2";
+ }
+ }
+ }
+
+ ########################################
+ #
+ # FASTQ dump
+ #
+ ########################################
+
+ dumpInfo();
+
+ if ((getGlobal("dumpFASTQ") > 0) &&
+ (-e "$wrk/$asm.gkpStore.info")) {
+
+ open(I, "< $wrk/$asm.gkpStore.info");
+ while (<I>) {
+ my ($libIID, $bgnIID, $endIID, $active, $deleted, $mated, $totLen, $clrLen, $libName) = split '\s+', $_;
+
+ if ($libIID > 0) {
+ my $prefix = sprintf "$wrk/9-terminator/$asm.lib.%03d.$libName", $libIID;
+
+ if (! -e "$prefix.unmated.fastq") {
+ $cmd = "$bin/gatekeeper ";
+ $cmd .= " -randomsubset $libIID 1.0 ";
+ $cmd .= " -dumpfastq $prefix ";
+ $cmd .= " $wrk/$asm.gkpStore ";
+ $cmd .= "> $prefix.err 2>&1";
+
+ runCommand($termDir, $cmd);
+ }
+ }
+ }
+ close(I);
+ }
+
+
+ ########################################
+ #
+ # Link into the work directory
+ #
+ ########################################
+
+ unlink "$wrk/$asm.asm";
+ unlink "$wrk/$asm.qc";
+
+ link "$termDir/$asm.asm", "$wrk/$asm.asm";
+ link "$termDir/$asm.qc", "$wrk/$asm.qc";
+
+ return(0);
+}
+
+################################################################################
+################################################################################
+################################################################################
+
+
+# Assembly all done, toggle the unitigs and re-run CGW and subsequent steps of the assembly.
+
+sub toggler () {
+ my $toggledDir = "10-toggledAsm";
+ my $ecrEdits = "";
+
+ return if (-d "$wrk/$toggledDir/$asm.asm");
+ return if (getGlobal("doToggle") == 0);
+
+ my $minLength = getGlobal("toggleUnitigLength");
+ my $numInstances = getGlobal("toggleNumInstances");
+ my $maxDistance = getGlobal("toggleMaxDistance");
+
+ system("mkdir $wrk/$toggledDir") if (! -d "$wrk/$toggledDir");
+
+ # A simple link to the ovlStore suffices.
+ #
+ if (! -e "$wrk/$toggledDir/$asm.ovlStore") {
+ system("ln -s ../$asm.ovlStore $wrk/$toggledDir/$asm.ovlStore");
+ }
+
+ # The gatekeeper store must be paritally copied, so that we can first undo
+ # clear range changes made by any previous ECR, and allow clear range changes
+ # to be made by future ECR.
+ #
+ if (! -e "$wrk/$toggledDir/$asm.gkpStore") {
+ system("mkdir $wrk/$toggledDir/$asm.gkpStore");
+
+ system("cd $wrk/$toggledDir/$asm.gkpStore && ln -s ../..//$asm.gkpStore/[qsu]?? .");
+ system("cp $wrk/$asm.gkpStore/[filp]?? $wrk/$toggledDir/$asm.gkpStore/");
+ system("cp $wrk/$asm.gkpStore/clr-* $wrk/$toggledDir/$asm.gkpStore/");
+
+ $cmd = "$bin/gatekeeper";
+ $cmd .= " --revertclear OBTCHIMERA $wrk/$toggledDir/$asm.gkpStore";
+ $cmd .= " > $wrk/$toggledDir/$asm.gkpStore.resetClearRange.err 2>&1";
+
+ if (runCommand("$wrk/$toggledDir", $cmd)) {
+ caFailure("failed to get pre-ECR clear-ranges for toggling", "$wrk/$toggledDir/$asm.gkpStore.resetClearRange.err");
+ }
+ }
+
+ # The tigStore needs only a partial copy, and links suffice.
+ #
+ if (! -e "$wrk/$toggledDir/$asm.tigStore") {
+ system("mkdir $wrk/$toggledDir/$asm.tigStore") ;
+
+ system("cd $wrk/$toggledDir/$asm.tigStore && ln -s ../../$asm.tigStore/*v00[12345]* .");
+ }
+
+ system("ln -s ../4-unitigger $wrk/$toggledDir") if (! -e "$wrk/$toggledDir/4-unitigger");
+ system("ln -s ../5-consensus $wrk/$toggledDir") if (! -e "$wrk/$toggledDir/5-consensus");
+
+ # Update the tigStore, flipping repeat untigs to unique unitigs.
+ #
+ if (! -e "$wrk/$toggledDir/toggled.success") {
+ $cmd = "$bin/markUniqueUnique ";
+ $cmd .= " -a $wrk/9-terminator/$asm.asm ";
+ $cmd .= " -t $wrk/$toggledDir/$asm.tigStore 5 ";
+ $cmd .= " -l $minLength ";
+ $cmd .= " -n $numInstances ";
+ $cmd .= " -d $maxDistance ";
+ $cmd .= " > $wrk/$toggledDir/toggle.err 2>&1";
+
+ if (runCommand("$wrk/$toggledDir", $cmd)) {
+ caFailure("failed to toggle unitigs ", "$wrk/$toggledDir/toggle.err");
+ }
+
+ touch("$wrk/$toggledDir/toggled.success");
+ }
+
+ my $numToggles = `tail -n 1 $wrk/$toggledDir/toggle.err | awk '{print \$2}'`;
+
+ if ($numToggles == 0) {
+ print "No toggling occured. Finished.\n";
+ return;
+ }
+
+ my $oldwrk = $wrk;
+
+ $wrk = "$wrk/$toggledDir";
+
+ scaffolder();
+ postScaffolderConsensus();
+ terminate();
+ cleaner();
+
+ $wrk = $oldwrk;
+}
+################################################################################
+################################################################################
+################################################################################
+
+
+# Assembly all done, remove some of the crud.
+
+sub cleaner () {
+ my $cleanType = getGlobal("cleanup");
+ my $cleanValu = 0;
+
+ print STDERR "The Cleaner has arrived. Doing '$cleanType'.\n";
+
+ $cleanValu = 0 if ($cleanType =~ m/none/);
+ $cleanValu = 1 if ($cleanType =~ m/light/);
+ $cleanValu = 2 if ($cleanType =~ m/heavy/);
+ $cleanValu = 3 if ($cleanType =~ m/aggressive/);
+
+
+ if ($cleanValu >= 1) {
+ #
+ # Remove some of the more useless output files,
+ # and many of the stores and whatnot that can be recreated.
+ #
+ rmrf("$asm.obtStore");
+ rmrf("0-mercounts/*blocks", "0-mercounts/*sequence");
+ rmrf("0-overlaptrim-overlap/overlap*out");
+ rmrf("1-overlapper/overlap*out");
+ rmrf("4-unitigger/$asm.fge", "4-unitigger/$asm.fgv");
+ rmrf("7*/rezlog");
+ }
+
+
+ if ($cleanValu >= 2) {
+ #
+ #
+ #
+ }
+
+
+ if ($cleanValu >= 3) {
+ #
+ # Nuke everything except 9-terminator. Be paranoid about doing it.
+ #
+ rmrf("0-mercounts");
+ rmrf("0-overlaptrim");
+ rmrf("0-overlaptrim-overlap");
+ rmrf("1-overlapper");
+ rmrf("2-frgcorr");
+ rmrf("3-ovlcorr");
+ rmrf("4-unitigger");
+ rmrf("5-consensus");
+ rmrf("7-[0-9]-CGW");
+ rmrf("7-[0-9]-ECR");
+ rmrf("7-CGW");
+ rmrf("8-consensus");
+ rmrf("$asm.SeqStore");
+ rmrf("$asm.asm");
+ rmrf("$asm.frg");
+ rmrf("$asm.gkpStore");
+ rmrf("$asm.obtStore");
+ rmrf("$asm.ovlStore");
+ rmrf("$asm.qc");
+ }
+
+
+ if ($cleanType =~ m/compress/) {
+ # Compress *.err (usually tiny)
+ # Compress overlaps (*ovb)
+ # Compress checkpoints (*ckp.*[0-9])
+ }
+}
+################################################################################
+################################################################################
+################################################################################
+
+
+setDefaults();
+
+# Check for the presence of a -options switch BEFORE we do any work.
+# This lets us print the default values of options.
+
+foreach my $arg (@ARGV) {
+ if ($arg eq "-options") {
+ setGlobal("options", 1);
+ printHelp();
+ }
+}
+
+# At some pain, we stash the original options for later use. We need
+# to use these when we resubmit ourself to SGE. We can't simply dump
+# all of @ARGV into here, because we need to fix up relative paths.
+
+while (scalar(@ARGV)) {
+ my $arg = shift @ARGV;
+
+ if ($arg =~ m/^-d/) {
+ $wrk = shift @ARGV;
+ $wrk = "$ENV{'PWD'}/$wrk" if ($wrk !~ m!^/!);
+ $commandLineOptions .= " -d \"$wrk\"";
+
+ } elsif ($arg eq "-p") {
+ $asm = shift @ARGV;
+ $commandLineOptions .= " -p \"$asm\"";
+
+ } elsif ($arg eq "-s") {
+ push @specFiles, shift @ARGV;
+
+ } elsif ($arg eq "-version") {
+ setGlobal("version", 1);
+
+ } elsif ($arg eq "-options") {
+ # Do nothing. Handled above, but we still need to process it here.
+ #setGlobal("options", 1);
+
+ } elsif (($arg =~ /\.frg$|frg\.gz$|frg\.bz2$|frg\.xz$/i) && (-e $arg)) {
+ $arg = "$ENV{'PWD'}/$arg" if ($arg !~ m!^/!);
+ push @fragFiles, $arg;
+ $commandLineOptions .= " \"$arg\"";
+
+ } elsif (($arg =~ /\.sff$|sff\.gz$|sff\.bz2$|sff\.xz$/i) && (-e $arg)) {
+ $arg = "$ENV{'PWD'}/$arg" if ($arg !~ m!^/!);
+ push @fragFiles, $arg;
+ $commandLineOptions .= " \"$arg\"";
+
+ } elsif (($arg =~ /\.ace$/i) && (-e $arg)) {
+ $arg = "$ENV{'PWD'}/$arg" if ($arg !~ m!^/!);
+ push @fragFiles, $arg;
+ $commandLineOptions .= " \"$arg\"";
+
+ } elsif ($arg =~ m/=/) {
+ push @specOpts, $arg;
+ $commandLineOptions .= " \"$arg\"";
+
+ } else {
+ setGlobal("help",
+ getGlobal("help") . "File not found or invalid command line option '$arg'\n");
+ }
+}
+
+
+setGlobal("help", getGlobal("help") . "Assembly name prefix not supplied with -p.\n") if (!defined($asm));
+setGlobal("help", getGlobal("help") . "Directory not supplied with -d.\n") if (!defined($wrk));
+
+
+$bin = getBinDirectory();
+
+ at fragFiles = setParametersFromFile("$bin/spec/runCA.default.specFile", @fragFiles) if (-e "$bin/spec/runCA.default.specFile");
+ at fragFiles = setParametersFromFile("$ENV{'HOME'}/.runCA", @fragFiles) if (-e "$ENV{'HOME'}/.runCA");
+
+
+# For each of the specfiles on the command line, find the actual file and make it an absolute path.
+# These can be in the current directory (e.g., 'my.spec'), or in the installed directory ('$bin/spec').
+#
+foreach my $specFile (@specFiles) {
+
+ if ((-e "$specFile") && (! -d "$specFile")) {
+ $specFile = "$ENV{'PWD'}/$specFile" if ($specFile !~ m!^/!);
+
+ } elsif ((-e "$bin/spec/$specFile") && (! -d "$bin/spec/$specFile")) {
+ $specFile = "$bin/spec/$specFile";
+
+ } elsif ((-e "$bin/spec/$specFile.specFile") && (! -d "$bin/spec/$specFile.specFile")) {
+ $specFile = "$bin/spec/$specFile.specFile";
+
+ } else {
+ die "specFile '$specFile' not found.\n";
+ }
+
+ $commandLineOptions .= " -s \"$specFile\"";
+
+ @fragFiles = setParametersFromFile($specFile, @fragFiles);
+}
+
+setParametersFromCommandLine(@specOpts);
+
+setParameters();
+
+printHelp();
+
+# Fail immediately if we run the script on the grid, and the gkpStore
+# directory doesn't exist and we have no input files. Without this
+# check we'd fail only after being scheduled on the grid.
+#
+if ((getGlobal("scriptOnGrid") == 1) &&
+ (! -d "$wrk/$asm.gkpStore") &&
+ (scalar(@fragFiles) == 0)) {
+ caFailure("no fragment files specified, and stores not already created", undef);
+}
+
+checkDirectories();
+outputSpecLog();
+
+# If not already on the grid, see if we should be on the grid.
+# N.B. the arg MUST BE undef.
+#
+submitScript(undef) if (!runningOnGrid());
+
+# Begin
+
+preoverlap(@fragFiles);
+merTrim(); # merTrim() MUST be before overlapTrim().
+overlapTrim();
+createOverlapJobs("normal");
+checkOverlap("normal");
+createOverlapStore();
+overlapCorrection();
+classifyMates();
+unitigger();
+postUnitiggerConsensus();
+scaffolder();
+postScaffolderConsensus();
+terminate();
+toggler();
+cleaner();
+
+exit(0);
diff --git a/src/AS_RUN/replaceUIDwithName-fastq.pl b/src/AS_RUN/replaceUIDwithName-fastq.pl
new file mode 100644
index 0000000..cd2e597
--- /dev/null
+++ b/src/AS_RUN/replaceUIDwithName-fastq.pl
@@ -0,0 +1,117 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Usage
+# fastqUIDmap fastq fastq fastq
+
+my $fastqUIDmap = shift @ARGV;
+
+my $inFile;
+my $otFile;
+
+my %UIDtoNAME;
+
+my $namesLoaded = 0;
+my $readsRenamed = 0;
+
+sub loadMoreNames {
+ my $lim = 100000;
+
+ undef %UIDtoNAME;
+
+ while (<N>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ if (scalar(@v) == 3) {
+ $UIDtoNAME{$v[0]} = $v[2];
+ $namesLoaded++;
+
+ } elsif (scalar(@v) == 6) {
+ $UIDtoNAME{$v[0]} = $v[2];
+ $UIDtoNAME{$v[3]} = $v[5];
+ $namesLoaded++;
+ $namesLoaded++;
+
+ } else {
+ die "unknown format '$_'\n";
+ }
+
+ if (--$lim == 0) {
+ return;
+ }
+ }
+}
+
+
+
+while (scalar(@ARGV)) {
+ $inFile = shift @ARGV;
+ $otFile = $inFile;
+
+ if ($inFile =~ m/(.*).fastq/) {
+ $otFile = "$1.fastq.RENAMING";
+ } else {
+ die "Failed to generate output file name.\n";
+ }
+
+ open(F, "< $inFile") or die "Failed to open '$inFile' for reading\n";
+ open(O, "> $otFile") or die "Failed to open '$otFile' for writing\n";
+ open(N, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n";
+
+ $namesLoaded = 0;
+ $readsRenamed = 0;
+
+ print STDERR "Renaming '$inFile' to '$otFile'.\n";
+
+ while (!eof(F)) {
+ my $a = <F>; chomp $a;
+ my $b = <F>;
+ my $c = <F>;
+ my $d = <F>;
+
+ if ($a =~ m/\@(\w+),\w+\s*/) {
+ # UID,IID
+ $a = $1;
+
+ } elsif ($a =~ m/\@(\w+)\s*/) {
+ # UID
+ $a = $1;
+
+ } else {
+ die "Nope '$a'\n";
+ }
+
+ while (!exists($UIDtoNAME{$a})) {
+ loadMoreNames();
+
+ #if ((!exists($UIDtoNAME{$a})) && ($readsRenamed > 0)) {
+ # print STDERR "WARNING: Looping to load more names; out of sync?\n";
+ #}
+ }
+
+ die "Didn't find UID '$a'\n" if (!exists($UIDtoNAME{$a}));
+ $a = "\@$UIDtoNAME{$a}\n";
+
+ print O "$a$b$c$d";
+
+ $readsRenamed++;
+
+ if (($readsRenamed % 10000) == 0) {
+ print STDERR "Renamed $readsRenamed reads using $namesLoaded names.\r";
+ }
+ }
+
+ print STDERR "Renamed $readsRenamed reads using $namesLoaded names.\n";
+
+ close(F);
+ close(O);
+ close(N);
+
+ if ($inFile =~ m/(.*).fastq/) {
+ rename "$1.fastq", "$1.CA_UIDs.fastq";
+ rename "$1.fastq.RENAMING", "$1.fastq";
+ }
+}
diff --git a/src/AS_RUN/replaceUIDwithName-posmap.pl b/src/AS_RUN/replaceUIDwithName-posmap.pl
new file mode 100644
index 0000000..02bc050
--- /dev/null
+++ b/src/AS_RUN/replaceUIDwithName-posmap.pl
@@ -0,0 +1,62 @@
+#!/usr/bin/perl
+
+use strict;
+
+my %UIDtoNAME;
+
+# Usage
+# fastqUIDmap posmnap posmap posmap
+
+my $fastqUIDmap = shift @ARGV;
+
+print STDERR "Loading UID map from '$fastqUIDmap'.\n";
+
+open (F, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n";
+while (<F>) {
+ chomp;
+
+ my @v = split '\s+', $_;
+
+ if (scalar(@v) == 3) {
+ $UIDtoNAME{$v[0]} = $v[2];
+
+ } elsif (scalar(@v) == 6) {
+ $UIDtoNAME{$v[0]} = $v[2];
+ $UIDtoNAME{$v[3]} = $v[5];
+
+ } else {
+ die "unknown format '$_'\n";
+ }
+}
+close(F);
+
+my $inFile;
+my $otFile;
+
+while (scalar(@ARGV)) {
+ $inFile = shift @ARGV;
+
+ print STDERR "Renaming '$inFile' to '$inFile.UID'.\n";
+
+ rename "$inFile", "$inFile.UID";
+
+ open(F, "< $inFile.UID") or die "Failed to open '$inFile.UID' for reading\n";
+ open(O, "> $inFile") or die "Failed to open '$inFile' for writing\n";
+
+ while (!eof(F)) {
+ my $a = <F>;
+ my @a = split '\s+', $a;
+
+ foreach my $a (@a) {
+ if (exists($UIDtoNAME{$a})) {
+ $a = $UIDtoNAME{$a};
+ }
+ }
+
+ print O join "\t", @a;
+ print O "\n";
+ }
+
+ close(F);
+ close(O);
+}
diff --git a/src/AS_RUN/replaceUIDwithName-simple.pl b/src/AS_RUN/replaceUIDwithName-simple.pl
deleted file mode 100644
index 12f13c3..0000000
--- a/src/AS_RUN/replaceUIDwithName-simple.pl
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-
-my %UIDtoNAME;
-
-# Usage
-# fastqUIDmap fastq fastq fastq
-
-my $fastqUIDmap = shift @ARGV;
-
-print STDERR "Loading UID map from '$fastqUIDmap'.\n";
-
-open (F, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n";
-while (<F>) {
- chomp;
-
- my @v = split '\s+', $_;
-
- if (scalar(@v) == 3) {
- $UIDtoNAME{$v[0]} = $v[2];
-
- } elsif (scalar(@v) == 6) {
- $UIDtoNAME{$v[0]} = $v[2];
- $UIDtoNAME{$v[3]} = $v[5];
-
- } else {
- die "unknown format '$_'\n";
- }
-}
-close(F);
-
-my $inFile;
-my $otFile;
-
-while (scalar(@ARGV)) {
- $inFile = shift @ARGV;
- $otFile = $inFile;
-
- if ($otFile =~ m/(.*).fastq/) {
- $otFile = "$1.nameFix.fastq";
- }
-
- open(F, "< $inFile") or die "Failed to open '$inFile' for reading\n";
- open(O, "> $otFile") or die "Failed to open '$otFile' for writing\n";
-
- print STDERR "Renaming '$inFile' to '$otFile'.\n";
-
- while (!eof(F)) {
- my $a = <F>; chomp $a;
- my $b = <F>;
- my $c = <F>;
- my $d = <F>;
-
- if ($a =~ m/\@(\w+)\s/) {
- die "Didn't find UID '$1'\n" if (!exists($UIDtoNAME{$1}));
- $a = "\@$UIDtoNAME{$1}\n";
- } else {
- print "Nope '$a'\n";
- }
-
- print O "$a$b$c$d";
- }
-
- close(F);
- close(O);
-}
diff --git a/src/AS_RUN/replaceUIDwithName.pl b/src/AS_RUN/replaceUIDwithName.pl
deleted file mode 100644
index eed9616..0000000
--- a/src/AS_RUN/replaceUIDwithName.pl
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-
-# Usage
-# fastqUIDmap fastq fastq fastq
-
-my $fastqUIDmap = shift @ARGV;
-
-my $inFile;
-my $otFile;
-
-my %UIDtoNAME;
-
-my $namesLoaded = 0;
-my $readsRenamed = 0;
-
-sub loadMoreNames {
- my $lim = 100000;
-
- undef %UIDtoNAME;
-
- while (<N>) {
- chomp;
-
- my @v = split '\s+', $_;
-
- if (scalar(@v) == 3) {
- $UIDtoNAME{$v[0]} = $v[2];
- $namesLoaded++;
-
- } elsif (scalar(@v) == 6) {
- $UIDtoNAME{$v[0]} = $v[2];
- $UIDtoNAME{$v[3]} = $v[5];
- $namesLoaded++;
- $namesLoaded++;
-
- } else {
- die "unknown format '$_'\n";
- }
-
- if (--$lim == 0) {
- return;
- }
- }
-}
-
-
-
-while (scalar(@ARGV)) {
- $inFile = shift @ARGV;
- $otFile = $inFile;
-
- if ($inFile =~ m/(.*).fastq/) {
- $otFile = "$1.fastq.RENAMING";
- } else {
- die "Failed to generate output file name.\n";
- }
-
- open(F, "< $inFile") or die "Failed to open '$inFile' for reading\n";
- open(O, "> $otFile") or die "Failed to open '$otFile' for writing\n";
- open(N, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n";
-
- $namesLoaded = 0;
- $readsRenamed = 0;
-
- print STDERR "Renaming '$inFile' to '$otFile'.\n";
-
- while (!eof(F)) {
- my $a = <F>; chomp $a;
- my $b = <F>;
- my $c = <F>;
- my $d = <F>;
-
- if ($a =~ m/\@(\w+),\w+\s*/) {
- # UID,IID
- $a = $1;
-
- } elsif ($a =~ m/\@(\w+)\s*/) {
- # UID
- $a = $1;
-
- } else {
- die "Nope '$a'\n";
- }
-
- while (!exists($UIDtoNAME{$a})) {
- loadMoreNames();
-
- #if ((!exists($UIDtoNAME{$a})) && ($readsRenamed > 0)) {
- # print STDERR "WARNING: Looping to load more names; out of sync?\n";
- #}
- }
-
- die "Didn't find UID '$a'\n" if (!exists($UIDtoNAME{$a}));
- $a = "\@$UIDtoNAME{$a}\n";
-
- print O "$a$b$c$d";
-
- $readsRenamed++;
-
- if (($readsRenamed % 10000) == 0) {
- print STDERR "Renamed $readsRenamed reads using $namesLoaded names.\r";
- }
- }
-
- print STDERR "Renamed $readsRenamed reads using $namesLoaded names.\n";
-
- close(F);
- close(O);
- close(N);
-
- if ($inFile =~ m/(.*).fastq/) {
- rename "$1.fastq", "$1.CA_UIDs.fastq";
- rename "$1.fastq.RENAMING", "$1.fastq";
- }
-}
diff --git a/src/AS_RUN/runCA-overlapStoreBuild.pl b/src/AS_RUN/runCA-overlapStoreBuild.pl
index be738c2..1c783db 100644
--- a/src/AS_RUN/runCA-overlapStoreBuild.pl
+++ b/src/AS_RUN/runCA-overlapStoreBuild.pl
@@ -172,11 +172,11 @@ my @jobArray;
my $numJobs = 0;
-open(F, "find $inp -type f -name \*.ovb.gz -print | sort |");
+open(F, "find $inp -type f -print |");
while (<F>) {
chomp;
- if (m/\d\d\d\/(\d\d\d\d\d\d).ovb.gz$/) {
+ if (m/\d\d\d\/(\d\d\d\d\d\d).ovb.[gx]z$/) {
$firstIdx = $1 if ($1 < $firstIdx);
$lastIdx = $1 if ($lastIdx < $1);
@@ -194,22 +194,56 @@ print STDERR "Found $numJobs jobs from index $firstIdx to $lastIdx.\n";
my $firstNum = int($firstIdx);
my $lastNum = int($lastIdx);
+my $missing = $lastNum - $firstNum + 1 - $numJobs;
die "First index ($firstIdx) not 000001.\n" if ($firstIdx ne "000001");
-die "Potential missing jobs.\n" if ($numJobs != $lastNum - $firstNum + 1);
-die "No jobs.\n" if ($numJobs == 0);
+if ($missing != 0) {
+ print STDERR "Missing $missing jobs.\n";
+
+ open(F, "> $inp/redo-overlapper.sh") or die;
+
+ my $cc = $firstNum;
+ my $ii = 0;
+
+ while ($cc < $lastNum) {
+ if ($jobArray[$ii] =~ m/(\d+).ovb/) {
+ if ($cc == $1) {
+ $ii++;
+ } else {
+ printf STDERR " %06d.ovb\n", $cc;
+ printf F "qsub -cwd -j y -o redo-\\\$TASK_ID -pe threads 2 -l mem=2g -t $cc overlap.sh\n";
+ }
+ } else {
+ print "Malformed job '$jobArray[$ii]'\n";
+ $ii++;
+ }
+
+ $cc++;
+ }
-# Create an output directory, and populate it with scripts
+ close(F);
-if (! -d "$wrk/$asm.${typ}Store") {
- system("mkdir -p $wrk/$asm.${typ}Store");
+ print STDERR "Submission commands written to '$inp/redo-overlapper.sh'\n";
+
+ exit(1);
}
+die "No jobs.\n" if ($numJobs == 0);
+
+# Create an output directory, and populate it with more directories and scripts
+
+system("mkdir -p $wrk/$asm.${typ}Store") if (! -d "$wrk/$asm.${typ}Store");
+system("mkdir -p $wrk/$asm.${typ}Store/scripts") if (! -d "$wrk/$asm.${typ}Store/scripts");
+system("mkdir -p $wrk/$asm.${typ}Store/logs") if (! -d "$wrk/$asm.${typ}Store/logs");
+system("mkdir -p $wrk/$asm.${typ}Store/logs/1-bucketize") if (! -d "$wrk/$asm.${typ}Store/logs/bucketize");
+system("mkdir -p $wrk/$asm.${typ}Store/logs/2-sort") if (! -d "$wrk/$asm.${typ}Store/logs/sort");
+system("mkdir -p $wrk/$asm.${typ}Store/logs/3-index") if (! -d "$wrk/$asm.${typ}Store/logs/index");
+
# Submit parallel jobs for bucketizing. This should really be part
# of overlap computation itself.
-open(F, "> $wrk/$asm.${typ}Store/1-bucketize.sh") or die;
+open(F, "> $wrk/$asm.${typ}Store/scripts/1-bucketize.sh") or die;
print F "#!/bin/sh\n";
print F "\n";
print F "jobid=\$SGE_TASK_ID\n";
@@ -237,29 +271,17 @@ print F " echo \"Job out of range.\"\n";
print F " exit\n";
print F "fi\n";
print F "\n";
-print F "if [ ! -e \"$wrk/$asm.${typ}Store/bucket\$bn\" -a -e \"\$jn.success\" ] ; then\n";
-print F " echo job \$jn claims success, but bucket $wrk/$asm.${typ}Store/bucket\$bn not found. rerunning.\n";
-print F " rm -f \$jn.success\n";
-print F "fi\n";
-print F "\n";
-print F "if [ -e \"\$jn.success\" ] ; then\n";
-print F " echo \"Input \$jn finished successfully.\"\n";
-print F " exit\n";
-print F "fi\n";
-print F "\n";
-print F "if [ -e \"\$jn.bucketizing\" ] ; then\n";
-print F " echo \"Input \$jn is in progress. Remove \$jn.bucketizing to try again.\"\n";
+print F "if [ -e \"$wrk/$asm.${typ}Store/bucket\$bn/sliceSizes\" ] ; then\n";
+print F " echo \"Bucket $wrk/$asm.${typ}Store/bucket\$bn finished successfully.\"\n";
print F " exit\n";
print F "fi\n";
print F "\n";
-print F "if [ -e \"\$jn.FAILED\" ] ; then\n";
-print F " echo \"Input \$jn failed previously, cleaning up bucket $wrk/$asm.${typ}Store/bucket\$bn and rerunning.\"\n";
-print F " rm -f \$jn.FAILED\n";
-print F " rm -rf \"$wrk/$asm.${typ}Store/bucket\$bn\"\n";
+print F "if [ -e \"$wrk/$asm.${typ}Store/create\$bn\" ] ; then\n";
+print F " echo \"Incomplete bucket $wrk/$asm.${typ}Store/create\$bn exists; you must remove to try again.\"\n";
+#print F " rm -rf \"$wrk/$asm.${typ}Store/create\$bn\"\n";
+print F " exit 1\n";
print F "fi\n";
print F "\n";
-print F "touch \"\$jn.bucketizing\"\n";
-print F "\n";
print F "$sbn\n";
print F "\n";
print F "\$bin/overlapStoreBucketizer \\\n";
@@ -271,18 +293,10 @@ print F " -obt \\\n" if ($typ eq "obt");
print F " -dup \\\n" if ($typ eq "dup");
print F " -job \$jobid \\\n";
print F " -i \$jn\n";
-print F "\n";
-print F "if [ \$? = 0 ] ; then\n";
-print F " mv -f \$jn.bucketizing \$jn.success\n";
-print F "fi\n";
-print F "\n";
-print F "if [ -e \"\$jn.bucketizing\" ] ; then\n";
-print F " mv -f \$jn.bucketizing \$jn.FAILED\n";
-print F "fi\n";
close(F);
-open(F, "> $wrk/$asm.${typ}Store/2-sort.sh") or die;
+open(F, "> $wrk/$asm.${typ}Store/scripts/2-sort.sh") or die;
print F "#!/bin/sh\n";
print F "\n";
print F "jobid=\$SGE_TASK_ID\n";
@@ -312,7 +326,7 @@ print F "fi\n";
close(F);
-open(F, "> $wrk/$asm.${typ}Store/3-index.sh") or die;
+open(F, "> $wrk/$asm.${typ}Store/scripts/3-index.sh") or die;
print F "#!/bin/sh\n";
print F "\n";
print F "$sbn\n";
@@ -339,8 +353,8 @@ $qsub1 .= " -N ovs1$asm \\\n";
$qsub1 .= " -t 1-$numJobs \\\n";
$qsub1 .= " -l memory=1g \\\n";
$qsub1 .= " -j y \\\n";
-$qsub1 .= " -o $wrk/$asm.${typ}Store/1-bucketize-\\\$TASK_ID.err \\\n";
-$qsub1 .= " $wrk/$asm.${typ}Store/1-bucketize.sh";
+$qsub1 .= " -o $wrk/$asm.${typ}Store/logs/1-bucketize/bucket-\\\$TASK_ID.err \\\n";
+$qsub1 .= " $wrk/$asm.${typ}Store/scripts/1-bucketize.sh";
$qsub2 .= "qsub -cwd -b n \\\n";
$qsub2 .= " -N ovs2$asm \\\n";
@@ -348,16 +362,16 @@ $qsub2 .= " -hold_jid ovs1$asm \\\n";
$qsub2 .= " -t 1-$jobs \\\n";
$qsub2 .= " -l memory=${maxMemory}g \\\n";
$qsub2 .= " -j y \\\n";
-$qsub2 .= " -o $wrk/$asm.${typ}Store/2-sort-\\\$TASK_ID.err \\\n";
-$qsub2 .= " $wrk/$asm.${typ}Store/2-sort.sh";
+$qsub2 .= " -o $wrk/$asm.${typ}Store/logs/2-sort/sort-\\\$TASK_ID.err \\\n";
+$qsub2 .= " $wrk/$asm.${typ}Store/scripts/2-sort.sh";
$qsub3 .= "qsub -cwd -b n \\\n";
$qsub3 .= " -N ovs3$asm \\\n";
$qsub3 .= " -hold_jid ovs2$asm \\\n";
$qsub3 .= " -l memory=1g \\\n";
$qsub3 .= " -j y \\\n";
-$qsub3 .= " -o $wrk/$asm.${typ}Store/3-index.err \\\n";
-$qsub3 .= " $wrk/$asm.${typ}Store/3-index.sh";
+$qsub3 .= " -o $wrk/$asm.${typ}Store/logs/3-index/index.err \\\n";
+$qsub3 .= " $wrk/$asm.${typ}Store/scripts/3-index.sh";
print "$qsub1\n";
system($qsub1) if ($submit == 1);
diff --git a/src/AS_RUN/runCA.pl b/src/AS_RUN/runCA.pl
index 6ab7381..c89a0c7 100644
--- a/src/AS_RUN/runCA.pl
+++ b/src/AS_RUN/runCA.pl
@@ -29,15 +29,65 @@ my @specOpts;
my @fragFiles;
+sub findBlasr($) {
+ my $cns = shift @_;
+
+ if ($cns ne "pbdagcon" && $cns ne "pbutgcns") {
+ return undef;
+ }
+
+ my $CA = getBinDirectory();
+ my $BLASR = "$CA/../../../smrtanalysis/current/analysis/bin/";
+
+ if (! -e "$BLASR/blasr" && $cns eq "pbdagcon") {
+ if (-e "$CA/blasr") {
+ $BLASR = $CA;
+ } else {
+ # try to use path
+ my $amosPath = `which blasr`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # blasr
+ $BLASR = join '/', @t; # path to the assembler
+ }
+ # if we really can't find it just give up
+ if (! -e "$BLASR/blasr") {
+ return undef;
+ }
+
+ # check for consensus too
+ # make sure we have the pb consensus module available if it was requested
+ if (! -e "$BLASR/blasr" || ! -e "$BLASR/pbdagcon") {
+ return undef;
+ }
+ } elsif (! -e "$BLASR/pbutgcns" && $cns eq "pbutgcns") {
+ if (-e "$CA/pbutgcns") {
+ $BLASR = $CA;
+ } else {
+ # try to use path
+ my $amosPath = `which pbutgcns`;
+ chomp $amosPath;
+ my @t = split '/', "$amosPath";
+ pop @t; # pbutgcns
+ $BLASR = join '/', @t; # path to the assembler
+ }
+ # if we really can't find it just give up
+ if (! -e "$BLASR/pbutgcns") {
+ return undef;
+ }
+ }
+ return $BLASR;
+}
+
sub submitBatchJobs($$) {
- my $SGE = shift @_;
- my $TAG = shift @_;
+ my $cmd = shift @_;
+ my $tag = shift @_;
if (runningOnGrid()) {
- runCommand($wrk, $SGE) and caFailure("Failed to submit batch jobs.");
- submitScript($TAG);
+ runCommand($wrk, $cmd) and caFailure("Failed to submit batch jobs.");
+ submitScript($tag);
} else {
- pleaseExecute($SGE);
+ pleaseExecute($cmd);
}
}
@@ -145,6 +195,35 @@ sub setGlobal ($$) {
$val = undef if ($val eq "");
+ # CA 8.3 added support for LSF, and renamed the grid* options to 'gridEngine', and the sge*
+ # options to 'grid'.
+
+ $var = "gridEngine" if ($var eq "gridEngine"); # This one didn't change, but left here for completeness
+ $var = "gridEngineSubmitCommand" if ($var eq "gridSubmitCommand");
+ $var = "gridEngineHoldOption" if ($var eq "gridHoldOption");
+ $var = "gridEngineHoldOptionNoArray" if ($var eq "gridHoldOptionNoArray");
+ $var = "gridEngineSyncOption" if ($var eq "gridSyncOption");
+ $var = "gridEngineNameOption" if ($var eq "gridNameOption");
+ $var = "gridEngineArrayOption" if ($var eq "gridArrayOption");
+ $var = "gridEngineArrayName" if ($var eq "gridArrayName");
+ $var = "gridEngineOutputOption" if ($var eq "gridOutputOption");
+ $var = "gridEnginePropagateCommand" if ($var eq "gridPropagateCommand");
+ $var = "gridEngineNameToJobIDCommand" if ($var eq "gridNameToJobIDCommand");
+ $var = "gridEngineTaskID" if ($var eq "gridTaskID");
+ $var = "gridEngineArraySubmitID" if ($var eq "gridArraySubmitID");
+ $var = "gridEngineJobID" if ($var eq "gridJobID");
+
+ $var = "gridOptions" if ($var eq "sge");
+ $var = "gridJobName" if ($var eq "sgeName");
+ $var = "gridOptionsScript" if ($var eq "sgeScript");
+ $var = "gridOptionsMerTrim" if ($var eq "sgeMerTrim");
+ $var = "gridOptionsOverlap" if ($var eq "sgeOverlap");
+ $var = "gridOptionsMerOverlapSeed" if ($var eq "sgeMerOverlapSeed");
+ $var = "gridOptionsMerOverlapExtend" if ($var eq "sgeMerOverlapExtend");
+ $var = "gridOptionsConsensus" if ($var eq "sgeConsensus");
+ $var = "gridOptionsFragmentCorrection" if ($var eq "sgeFragmentCorrection");
+ $var = "gridOptionsOverlapCorrection" if ($var eq "sgeOverlapCorrection");
+
# Handle special cases.
if ($var eq "merSize") {
@@ -177,34 +256,55 @@ sub setGlobal ($$) {
return;
}
+ if (($var eq "gridEngine") && ($val eq "PBS")) {
+ setGlobal("gridEngineSubmitCommand", "qsub");
+ setGlobal("gridEngineHoldOption", "-W depend=afteranyarray:\"WAIT_TAG\"");
+ setGlobal("gridEngineHoldOptionNoArray", "-W depend=afterany:\"WAIT_TAG\"");
+ setGlobal("gridEngineSyncOption", "");
+ setGlobal("gridEngineNameOption", "-d `pwd` -N");
+ setGlobal("gridEngineArrayOption", "-t ARRAY_JOBS");
+ setGlobal("gridEngineArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
+ setGlobal("gridEngineOutputOption", "-j oe -o");
+ setGlobal("gridEnginePropagateCommand", "qalter -W depend=afterany:\"WAIT_TAG\"");
+ setGlobal("gridEngineNameToJobIDCommand", "qstat -f |grep -F -B 1 WAIT_TAG | grep Id: | grep -F [] |awk '{print \$NF}'");
+ setGlobal("gridEngineNameToJobIDCommandNoArray", "qstat -f |grep -F -B 1 WAIT_TAG | grep Id: |awk '{print \$NF}'");
+ setGlobal("gridEngineTaskID", "PBS_ARRAYID");
+ setGlobal("gridEngineArraySubmitID", "\\\$PBS_ARRAYID");
+ setGlobal("gridEngineJobID", "PBS_JOBID");
+ }
+
if (($var eq "gridEngine") && ($val eq "SGE")) {
- setGlobal("gridSubmitCommand", "qsub");
- setGlobal("gridHoldOption", "-hold_jid \"WAIT_TAG\"");
- setGlobal("gridSyncOption", "-sync y");
- setGlobal("gridNameOption", "-cwd -N");
- setGlobal("gridArrayOption", "-t ARRAY_JOBS");
- setGlobal("gridArrayName", "ARRAY_NAME");
- setGlobal("gridOutputOption", "-j y -o");
- setGlobal("gridPropagateCommand", "qalter -hold_jid \"WAIT_TAG\"");
- setGlobal("gridNameToJobIDCommand", undef);
- setGlobal("gridTaskID", "SGE_TASK_ID");
- setGlobal("gridArraySubmitID", "\\\$TASK_ID");
- setGlobal("gridJobID", "JOB_ID");
+ setGlobal("gridEngineSubmitCommand", "qsub");
+ setGlobal("gridEngineHoldOption", "-hold_jid \"WAIT_TAG\"");
+ setGlobal("gridEngineHoldOptionNoArray", undef);
+ setGlobal("gridEngineSyncOption", "-sync y");
+ setGlobal("gridEngineNameOption", "-cwd -N");
+ setGlobal("gridEngineArrayOption", "-t ARRAY_JOBS");
+ setGlobal("gridEngineArrayName", "ARRAY_NAME");
+ setGlobal("gridEngineOutputOption", "-j y -o");
+ setGlobal("gridEnginePropagateCommand", "qalter -hold_jid \"WAIT_TAG\"");
+ setGlobal("gridEngineNameToJobIDCommand", undef);
+ setGlobal("gridEngineNameToJobIDCommandNoArray", undef);
+ setGlobal("gridEngineTaskID", "SGE_TASK_ID");
+ setGlobal("gridEngineArraySubmitID", "\\\$TASK_ID");
+ setGlobal("gridEngineJobID", "JOB_ID");
}
if (($var eq "gridEngine") && ($val eq "LSF")) {
- setGlobal("gridSubmitCommand", "qsub");
- setGlobal("gridHoldOption", "-w done\(\"WAIT_TAG\"\)");
- setGlobal("gridSyncOption", "-K");
- setGlobal("gridNameOption", "-J");
- setGlobal("gridArrayOption", "");
- setGlobal("gridArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
- setGlobal("gridOutputOption", "-o");
- setGlobal("gridPropagateCommand", "bmodify -w done\(\"WAIT_TAG\"\)");
- setGlobal("gridNameToJobIDCommand", "bjobs -J \"WAIT_TAG\" | grep -v JOBID");
- setGlobal("gridTaskID", "LSB_JOBINDEX");
- setGlobal("gridArraySubmitID", "%I");
- setGlobal("gridJobID", "LSB_JOBID");
+ setGlobal("gridEngineSubmitCommand", "bsub");
+ setGlobal("gridEngineHoldOption", "-w \"numended\(\"WAIT_TAG\", \*\)\"");
+ setGlobal("gridEngineHoldOptionNoArray", "-w \"done\(\"WAIT_TAG\"\)\"");
+ setGlobal("gridEngineSyncOption", "-K");
+ setGlobal("gridEngineNameOption", "-J");
+ setGlobal("gridEngineArrayOption", "");
+ setGlobal("gridEngineArrayName", "ARRAY_NAME\[ARRAY_JOBS\]");
+ setGlobal("gridEngineOutputOption", "-o");
+ setGlobal("gridEnginePropagateCommand", "bmodify -w \"done\(\"WAIT_TAG\"\)\"");
+ setGlobal("gridEngineNameToJobIDCommand", "bjobs -A -J \"WAIT_TAG\" | grep -v JOBID");
+ setGlobal("gridEngineNameToJobIDCommandNoArray", "bjobs -J \"WAIT_TAG\" | grep -v JOBID");
+ setGlobal("gridEngineTaskID", "LSB_JOBINDEX");
+ setGlobal("gridEngineArraySubmitID", "%I");
+ setGlobal("gridEngineJobID", "LSB_JOBID");
}
# Update obsolete usage.
@@ -329,74 +429,77 @@ sub setDefaults () {
##### Grid Engine configuration, how to submit jobs, etc
- $global{"gridSubmitCommand"} = "qsub";
- $global{"gridHoldOption"} = "-hold_jid \"WAIT_TAG\""; # for lsf: -w "done("WAIT_TAG")"
- $global{"gridSyncOption"} = "-sync y"; # for lsf: -K
- $global{"gridNameOption"} = "-cwd -N"; # for lsf: -J
- $global{"gridArrayOption"} = "-t ARRAY_JOBS"; # for lsf: empty ("")
- $global{"gridArrayName"} = "ARRAY_NAME"; # for lsf: ARRAY_NAME[ARRAY_JOBS]
- $global{"gridOutputOption"} = "-j y -o"; # for lsf: -o
- $global{"gridPropagateCommand"} = "qalter -hold_jid \"WAIT_TAG\""; # for lsf: bmodify -w "done(WAIT_TAG)"
- $global{"gridNameToJobIDCommand"} = undef; # for lsf: bjobs -J "WAIT_TAG" | grep -v JOBID
- $global{"gridTaskID"} = "SGE_TASK_ID"; # for lsf: LSB_JOBINDEX
- $global{"gridArraySubmitID"} = "\\\$TASK_ID"; # for lsf: %I
- $global{"gridJobID"} = "JOB_ID"; # for lsf: LSB_JOBID
+ $global{"gridEngine"} = "SGE";
+ $global{"gridEngineSubmitCommand"} = "qsub";
+ $global{"gridEngineHoldOption"} = "-hold_jid \"WAIT_TAG\""; # for lsf: -w "done("WAIT_TAG")"
+ $global{"gridEngineHoldOptionNoArray"} = undef;
+ $global{"gridEngineSyncOption"} = "-sync y"; # for lsf: -K
+ $global{"gridEngineNameOption"} = "-cwd -N"; # for lsf: -J
+ $global{"gridEngineArrayOption"} = "-t ARRAY_JOBS"; # for lsf: empty ("")
+ $global{"gridEngineArrayName"} = "ARRAY_NAME"; # for lsf: ARRAY_NAME[ARRAY_JOBS]
+ $global{"gridEngineOutputOption"} = "-j y -o"; # for lsf: -o
+ $global{"gridEnginePropagateCommand"} = "qalter -hold_jid \"WAIT_TAG\""; # for lsf: bmodify -w "done(WAIT_TAG)"
+ $global{"gridEngineNameToJobIDCommand"} = undef; # for lsf: bjobs -J "WAIT_TAG" | grep -v JOBID
+ $global{"gridEngineNameToJobIDCommandNoArray"} = undef;
+ $global{"gridEngineTaskID"} = "SGE_TASK_ID"; # for lsf: LSB_JOBINDEX
+ $global{"gridEngineArraySubmitID"} = "\\\$TASK_ID"; # for lsf: %I
+ $global{"gridEngineJobID"} = "JOB_ID"; # for lsf: LSB_JOBID
##### Sun Grid Engine
- $global{"useGrid"} = 0;
- $synops{"useGrid"} = "Enable SGE globally";
+ $global{"useGrid"} = 0;
+ $synops{"useGrid"} = "Enable grid globally";
- $global{"scriptOnGrid"} = 0;
- $synops{"scriptOnGrid"} = "Enable SGE for runCA (and unitigger, scaffolder, other sequential phases)";
+ $global{"scriptOnGrid"} = 0;
+ $synops{"scriptOnGrid"} = "Enable grid for runCA (and unitigger, scaffolder, other sequential phases)";
- $global{"mbtOnGrid"} = 1;
- $synops{"mbtOnGrid"} = "Enable SGE for mer-based trimming computations";
+ $global{"mbtOnGrid"} = 1;
+ $synops{"mbtOnGrid"} = "Enable grid for mer-based trimming computations";
- $global{"ovlOnGrid"} = 1;
- $synops{"ovlOnGrid"} = "Enable SGE for overlap computations";
+ $global{"ovlOnGrid"} = 1;
+ $synops{"ovlOnGrid"} = "Enable grid for overlap computations";
- $global{"frgCorrOnGrid"} = 0;
- $synops{"frgCorrOnGrid"} = "Enable SGE for the fragment error correction";
+ $global{"frgCorrOnGrid"} = 0;
+ $synops{"frgCorrOnGrid"} = "Enable grid for the fragment error correction";
- $global{"ovlCorrOnGrid"} = 0;
- $synops{"ovlCorrOnGrid"} = "Enable SGE for the overlap error correction";
+ $global{"ovlCorrOnGrid"} = 0;
+ $synops{"ovlCorrOnGrid"} = "Enable grid for the overlap error correction";
- $global{"cnsOnGrid"} = 1;
- $synops{"cnsOnGrid"} = "Enable SGE for consensus";
+ $global{"cnsOnGrid"} = 1;
+ $synops{"cnsOnGrid"} = "Enable grid for consensus";
- $global{"sge"} = undef;
- $synops{"sge"} = "SGE options applied to all SGE jobs";
+ $global{"gridOptions"} = undef;
+ $synops{"gridOptions"} = "SGE, LSF or PBS options applied to all grid jobs";
- $global{"sgeName"} = undef;
- $synops{"sgeName"} = "SGE jobs name suffix";
+ $global{"gridJobName"} = undef;
+ $synops{"gridJobName"} = "SGE, LSF or PBS job name suffix";
- $global{"sgeScript"} = undef;
- $synops{"sgeScript"} = "SGE options applied to runCA jobs (and unitigger, scaffolder, other sequential phases)";
+ $global{"gridOptionsScript"} = undef;
+ $synops{"gridOptionsScript"} = "SGE, LSF or PBS otpions applied to runCA jobs (and unitigger, scaffolder, other sequential phases)";
- $global{"sgeMerTrim"} = undef;
- $synops{"sgeMerTrim"} = "SGE options applied to mer-based trimming jobs";
+ $global{"gridOptionsMerTrim"} = undef;
+ $synops{"gridOptionsMerTrim"} = "SGE, LSF or PBS otpions applied to mer-based trimming jobs";
- $global{"sgeOverlap"} = undef;
- $synops{"sgeOverlap"} = "SGE options applied to overlap computation jobs";
+ $global{"gridOptionsOverlap"} = undef;
+ $synops{"gridOptionsOverlap"} = "SGE, LSF or PBS otpions applied to overlap computation jobs";
- $global{"sgeMerOverlapSeed"} = undef;
- $synops{"sgeMerOverlapSeed"} = "SGE options applied to mer overlap seed (overmerry) jobs";
+ $global{"gridOptionsMerOverlapSeed"} = undef;
+ $synops{"gridOptionsMerOverlapSeed"} = "SGE, LSF or PBS otpions applied to mer overlap seed (overmerry) jobs";
- $global{"sgeMerOverlapExtend"} = undef;
- $synops{"sgeMerOverlapExtend"} = "SGE options applied to mer overlap extend (olap-from-seeds) jobs";
+ $global{"gridOptionsMerOverlapExtend"} = undef;
+ $synops{"gridOptionsMerOverlapExtend"} = "SGE, LSF or PBS otpions applied to mer overlap extend (olap-from-seeds) jobs";
- $global{"sgeConsensus"} = undef;
- $synops{"sgeConsensus"} = "SGE options applied to consensus jobs";
+ $global{"gridOptionsConsensus"} = undef;
+ $synops{"gridOptionsConsensus"} = "SGE, LSF or PBS otpions applied to consensus jobs";
- $global{"sgeFragmentCorrection"} = undef;
- $synops{"sgeFragmentCorrection"} = "SGE options applied to fragment error correction jobs";
+ $global{"gridOptionsFragmentCorrection"} = undef;
+ $synops{"gridOptionsFragmentCorrection"} = "SGE, LSF or PBS otpions applied to fragment error correction jobs";
- $global{"sgeOverlapCorrection"} = undef;
- $synops{"sgeOverlapCorrection"} = "SGE options applied to overlap error correction jobs";
+ $global{"gridOptionsOverlapCorrection"} = undef;
+ $synops{"gridOptionsOverlapCorrection"} = "SGE, LSF or PBS otpions applied to overlap error correction jobs";
- $global{"sgePropagateHold"} = undef;
- $synops{"sgePropagateHold"} = undef; # Internal option
+ $global{"gridEnginePropagateHold"} = undef;
+ $synops{"gridEnginePropagateHold"} = undef; # Internal option
##### Preoverlap
@@ -411,12 +514,6 @@ sub setDefaults () {
$global{"vectorIntersect"} = undef;
$synops{"vectorIntersect"} = "File of vector clear ranges";
- $global{"vectorTrimmer"} = "ca";
- $synops{"vectorTrimmer"} = "Use the CA default vector trimmer, or figaro";
-
- $global{"figaroFlags"} = "-T 30 -M 100 -E 500 -V f";
- $synops{"figaroFlags"} = "Options to the figaro vector trimmer";
-
##### Overlap Based Trimming
$global{"doOverlapBasedTrimming"} = 1;
@@ -547,9 +644,6 @@ sub setDefaults () {
$global{"merOverlapperExtendConcurrency"}= 1;
$synops{"merOverlapperExtendConcurrency"}= "If not SGE, number of mer overlapper seed extension processes to run at the same time";
- $global{"umdOverlapperFlags"} = "-use-uncleaned-reads -trim-error-rate 0.03 -max-minimizer-cutoff 150";
- $synops{"umdOverlapperFlags"} = "Options for the UMD overlapper";
-
$global{"saveOverlaps"} = 0;
$synops{"saveOverlaps"} = "Save intermediate overlap files";
@@ -609,11 +703,8 @@ sub setDefaults () {
$global{"bogBadMateDepth"} = 7;
$synops{"bogBadMateDepth"} = "EXPERT!";
- $global{"batRebuildRepeats"} = 0;
- $synops{"batRebuildRepeats"} = "Shatter repeats, rebuild more stringently";
-
- $global{"batMateExtension"} = 0;
- $synops{"batMateExtension"} = "Shatter repeats, extend unique unitigs with mates (leaves repeats shattered)";
+ $global{"batOptions"} = undef;
+ $synops{"batOptions"} = "Advanced options to bogart";
$global{"batMemory"} = undef;
$synops{"batMemory"} = "Approximate maximum memory usage for loading overlaps, in gigabytes, default is unlimited";
@@ -624,6 +715,29 @@ sub setDefaults () {
$global{"doUnitigSplitting"} = 1;
$synops{"doUnitigSplitting"} = "Split unitigs based on low coverage and high bad mate evidence";
+ ##### Unitig Repeat/Unique Options (formerly in scaffolder)
+
+ $global{"astatLowBound"} = 1;
+ $synops{"astatLowBound"} = "EXPERT!";
+
+ $global{"astatHighBound"} = 5;
+ $synops{"astatHighBound"} = "EXPERT!";
+
+ $global{"maxSingleReadSpan"} = undef;
+ $synops{"maxSingleReadSpan"} = "Unitigs with a single read spanning more than this fraction of the unitig are never labeled unique";
+
+ $global{"lowCoverageDepth"} = undef;
+ $synops{"lowCoverageDepth"} = "See lowCoverageAllowed";
+
+ $global{"lowCoverageAllowed"} = undef;
+ $synops{"lowCoverageAllowed"} = "Unitigs with more than this fraction lowCoverageDepth bases are never labeled unique";
+
+ $global{"minReadsUnique"} = undef;
+ $synops{"minReadsUnique"} = "Unitigs with fewer reads that this are never labeled unique";
+
+ $global{"maxRepeatLength"} = undef;
+ $synops{"maxRepeatLength"} = "Unitigs longer than this are always labeled unique";
+
##### Scaffolder Options
$global{"cgwPurgeCheckpoints"} = 1;
@@ -641,12 +755,6 @@ sub setDefaults () {
$global{"cgwReloadMates"} = 0;
$synops{"cgwReloadMates"} = "Load new mate pairs from gkpStore after ckp is loaded (EXPERIMENTAL)";
- $global{"astatLowBound"} = 1;
- $synops{"astatLowBound"} = "EXPERT!";
-
- $global{"astatHighBound"} = 5;
- $synops{"astatHighBound"} = "EXPERT!";
-
$global{"stoneLevel"} = 2;
$synops{"stoneLevel"} = "EXPERT!";
@@ -700,8 +808,8 @@ sub setDefaults () {
$global{"cnsPhasing"} = 0;
$synops{"cnsPhasing"} = "Options for consensus phasing of SNPs\n\t0 - Do not phase SNPs to be consistent.\n\t1 - If two SNPs are joined by reads, phase them to be consistent.";
- $global{"cnsReduceUnitigs"} = undef;
- $synops{"cnsReduceUnitigs"} = "Set parameters for when and how to sample down coverage in deep unitigs.";
+ $global{"cnsMaxCoverage"} = 0;
+ $synops{"cnsMaxCoverage"} = "Limit unitig consensus to to at most this coverage";
$global{"cnsReuseUnitigs"} = 0;
$synops{"cnsReuseUnitigs"} = "Do not compute single-unitig contigs again, just reuse the unitig.";
@@ -761,14 +869,6 @@ sub setDefaults () {
$global{"toggleDoNotDemote"} = 0;
$synops{"toggleDoNotDemote"} = "Do not allow CGW to demote toggled unitigs based on branching patterns.";
- #### Closure Options
-
- $global{"closureOverlaps"} = undef;
- $synops{"closureOverlaps"} = "Option for handling overlaps involving closure reads.\n\t0 - Treat them just like regular reads, \n\t1 - Do not allow any overlaps (i.e. closure reads will stay as singletons until scaffolding), \n\t2 - allow overlaps betweeen closure reads and non-closure reads only";
-
- $global{"closurePlacement"} = 2;
- $synops{"closurePlacement"} = "Option for placing closure reads using the constraints.\n\t0 - Place at the first location found\n\t1 - Place at the best location (indicated by most constraints)\n\t2 - Place at multiple locations as long as the closure read/unitig in question is not unique";
-
##### Ugly, command line options passed to printHelp()
$global{"help"} = "";
@@ -784,9 +884,9 @@ sub setDefaults () {
# If this is set, it breaks the consensus.sh and overlap.sh scripts. Good grief! Why
# are you running runCA in a task array!?
#
- if (exists($ENV{$global{"gridTaskID"}})) {
- undef $ENV{$global{"gridTaskID"}};
- print STDERR "ENV: $global{'gridTaskID'} needs to be unset, done.\n";
+ if (exists($ENV{$global{"gridEngineTaskID"}})) {
+ undef $ENV{$global{"gridEngineTaskID"}};
+ print STDERR "ENV: $global{'gridEngineTaskID'} needs to be unset, done.\n";
}
@@ -941,7 +1041,6 @@ sub setParameters () {
fixCase("obtOverlapper");
fixCase("ovlOverlapper");
fixCase("unitigger");
- fixCase("vectorTrimmer");
fixCase("stopBefore");
fixCase("stopAfter");
fixCase("consensus");
@@ -959,11 +1058,8 @@ sub setParameters () {
if (defined(getGlobal("unitigger")) && (getGlobal("unitigger") ne "utg") && (getGlobal("unitigger") ne "bog") && (getGlobal("unitigger") ne "bogart")) {
caFailure("invalid unitigger specified (" . getGlobal("unitigger") . "); must be 'utg' or 'bog' or 'bogart'", undef);
}
- if ((getGlobal("vectorTrimmer") ne "ca") && (getGlobal("vectorTrimmer") ne "figaro")) {
- caFailure("invalid vectorTrimmer specified (" . getGlobal("vectorTrimmer") . "); must be 'ca' or 'figaro'", undef);
- }
- if ((getGlobal("consensus") ne "cns") && (getGlobal("consensus") ne "seqan")) {
- caFailure("invalid consensus specified (" . getGlobal("consensus") . "); must be 'cns' or 'seqan'", undef);
+ if ((getGlobal("consensus") ne "cns") && (getGlobal("consensus") ne "seqan") && (getGlobal("consensus") ne "pbdagcon") && (getGlobal("consensus") ne "pbutgcns")) {
+ caFailure("invalid consensus specified (" . getGlobal("consensus") . "); must be 'cns' or 'seqan' or 'pbdagcon' or 'pbutgcns'", undef);
}
if ((getGlobal("cnsPhasing") ne "0") && (getGlobal("cnsPhasing") ne "1")) {
caFailure("invalid cnsPhasing specified (" . getGlobal("cnsPhasing") . "); must be '0' or '1'", undef);
@@ -1100,8 +1196,18 @@ sub setParameters () {
$ENV{'AS_CGW_ERROR_RATE'} = $cgwER;
$ENV{'AS_CNS_ERROR_RATE'} = $cnsER;
+ # Various minimums and maximums.
+
$ENV{'AS_READ_MIN_LEN'} = getGlobal("frgMinLen");
$ENV{'AS_OVERLAP_MIN_LEN'} = getGlobal("ovlMinLen");
+
+ if ((getGlobal("obtMerSize") > 31) ||
+ (getGlobal("ovlMerSize") > 31)) {
+ my $obt = getGlobal("obtMerSize");
+ my $ovl = getGlobal("ovlMerSize");
+
+ caFailure("obtMerSize (set to $obt) or ovlMerSize (set to $ovl) cannot exceed 31 (note that 'merSize' sets both)", undef);
+ }
}
sub logVersion() {
@@ -1336,7 +1442,7 @@ sub stopAfter ($) {
}
sub runningOnGrid () {
- my $jobID = getGlobal("gridJobID");
+ my $jobID = getGlobal("gridEngineJobID");
return(exists($ENV{$jobID}));
}
@@ -1363,13 +1469,13 @@ sub buildGridArray($$$) {
sub getGridArrayName($$) {
my $name = shift @_;
my $maxLimit = shift @_;
- return buildGridArray($name, $maxLimit, "gridArrayName");
+ return buildGridArray($name, $maxLimit, "gridEngineArrayName");
}
sub getGridArrayOption($$) {
my $name = shift @_;
my $maxLimit = shift @_;
- return buildGridArray($name, $maxLimit, "gridArrayOption");
+ return buildGridArray($name, $maxLimit, "gridEngineArrayOption");
}
sub submitScript ($) {
@@ -1404,45 +1510,64 @@ sub submitScript ($) {
system("chmod +x $script");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeScript = getGlobal("sgeScript");
- my $sgePropHold = getGlobal("sgePropagateHold");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridScript = getGlobal("gridOptionsScript");
+ my $gridPropHold = getGlobal("gridEnginePropagateHold");
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $holdOption = getGlobal("gridHoldOption");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
- my $holdPropagateCommand = getGlobal("gridPropagateCommand");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $holdOption = getGlobal("gridEngineHoldOption");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
+ my $holdPropagateCommand = getGlobal("gridEnginePropagateCommand");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = "rCA_$asm$sgeName";
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = "rCA_$asm$gridName";
if (defined($waitTag)) {
my $hold = $holdOption;
- $hold =~ s/WAIT_TAG/$waitTag/g;
+ if (getGlobal("gridEngine") eq "LSF" || getGlobal("gridEngine") eq "PBS"){
+ my $tcmd = getGlobal("gridEngineNameToJobIDCommand");
+ $tcmd =~ s/WAIT_TAG/$waitTag/g;
+ my $propJobCount = `$tcmd |wc -l`;
+ chomp $propJobCount;
+ if ($propJobCount == 0) {
+ $tcmd = getGlobal("gridEngineNameToJobIDCommandNoArray");
+ $tcmd =~ s/WAIT_TAG/$waitTag/g;
+ $hold = getGlobal("gridEngineHoldOptionNoArray");
+ $propJobCount = `$tcmd |wc -l`;
+ }
+ if ($propJobCount != 1) {
+ print STDERR "Warning: multiple IDs for job $gridPropHold got $propJobCount and should have been 1.\n";
+ }
+ my $jobID = `$tcmd |tail -n 1 |awk '{print \$1}'`;
+ chomp $jobID;
+ $hold =~ s/WAIT_TAG/$jobID/g;
+ } else{
+ $hold =~ s/WAIT_TAG/$waitTag/g;
+ }
$waitTag = $hold;
}
- my $qcmd = "$submitCommand $sge $sgeScript $nameOption \"$jobName\" $waitTag $outputOption $output $script";
+ my $qcmd = "$submitCommand $grid $gridScript $nameOption \"$jobName\" $waitTag $outputOption $output $script";
runCommand($wrk, $qcmd) and caFailure("Failed to submit script.\n");
- if (defined($sgePropHold)) {
+ if (defined($gridPropHold)) {
if (defined($holdPropagateCommand)) {
- my $translateCmd = getGlobal("gridNameToJobIDCommand");
+ my $translateCmd = getGlobal("gridEngineNameToJobIDCommandNoArray");
# translate hold option to job id if necessary
if (defined($translateCmd) && $translateCmd ne "") {
my $tcmd = $translateCmd;
- $tcmd =~ s/WAIT_TAG/$sgePropHold/g;
+ $tcmd =~ s/WAIT_TAG/$gridPropHold/g;
my $propJobCount = `$tcmd |wc -l`;
chomp $propJobCount;
if ($propJobCount != 1) {
- print STDERR "Warning: multiple IDs for job $sgePropHold got $propJobCount and should have been 1.\n";
+ print STDERR "Warning: multiple IDs for job $gridPropHold got $propJobCount and should have been 1.\n";
}
#my $jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
#chomp $jobID;
- #print STDERR "Translated job ID $sgePropHold to be job $jobID\n";
- #$sgePropHold = $jobID;
+ #print STDERR "Translated job ID $gridPropHold to be job $jobID\n";
+ #$gridPropHold = $jobID;
open(PROPS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that need to hold", undef);
# now we can get the job we are holding for
@@ -1455,13 +1580,15 @@ sub submitScript ($) {
}
#$jobID = `$tcmd |head -n 1 |awk '{print \$1}'`;
#chomp $jobID;
- #print STDERR "Translated job ID $sgePropHold to be job $jobID\n";
+ #print STDERR "Translated job ID $gridPropHold to be job $jobID\n";
#$jobName = $jobID;
open(HOLDS, "$tcmd |awk '{print \$1}' | ") or die("Couldn't get list of jobs that should be held for", undef);
# loop over all jobs and all sge hold commands to modify the jobs. We have no way to know which is the right one unfortunately
while (my $prop = <PROPS>) {
while (my $hold = <HOLDS>) {
+ chomp $hold;
+ chomp $prop;
my $hcmd = $holdPropagateCommand;
$hcmd =~ s/WAIT_TAG/$hold/g;
my $acmd = "$hcmd $prop";
@@ -1472,13 +1599,13 @@ sub submitScript ($) {
close(HOLDS);
close(PROPS);
} else {
- $sgePropHold = "\"$sgePropHold\"";
+ $gridPropHold = "\"$gridPropHold\"";
$holdPropagateCommand =~ s/WAIT_TAG/$jobName/g;
- my $acmd = "$holdPropagateCommand $sgePropHold";
- system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$sgePropHold'.\n";
+ my $acmd = "$holdPropagateCommand $gridPropHold";
+ system($acmd) and print STDERR "WARNING: Failed to reset hold_jid trigger on '$gridPropHold'.\n";
}
} else {
- print STDERR "WARNING: Failed to reset hold '$sgePropHold', not supported on current grid environment.\n";
+ print STDERR "WARNING: Failed to reset hold '$gridPropHold', not supported on current grid environment.\n";
}
}
@@ -1505,7 +1632,7 @@ sub caFailure ($$) {
print STDERR "----------------------------------------\n";
print STDERR "Last few lines of the relevant log file ($log):\n";
print STDERR "\n";
- system("tail -n 20 $log");
+ system("tail -n 50 $log");
}
print STDERR "\n";
@@ -1840,8 +1967,6 @@ sub preoverlap {
# This should never run; the info is now created by gatekeeper.
dumpInfo();
- generateVectorTrim();
-
my $vi = getGlobal("vectorIntersect");
if ((defined($vi)) && (! -e "$wrk/$asm.gkpStore/$asm.vectorClearLoaded.log")) {
@@ -2410,8 +2535,6 @@ sub runMeryl ($$$$$$$$) {
}
}
- my $lastThreshold = 0;
-
open(F, "< $ofile.histogram") or caFailure("failed to read mer histogram from '$ofile.histogram'", undef);
while (<F>) {
my ($threshold, $num, $distinct, $total) = split '\s+', $_;
@@ -2421,19 +2544,17 @@ sub runMeryl ($$$$$$$$) {
last;
}
- if ((defined($merDistinct)) && ($merDistinct < $distinct)) {
- $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $lastThreshold;
- print STDERR "Supplied merDistinct $merDistinct with threshold $lastThreshold is the smallest.\n";
+ if ((defined($merDistinct)) && ($merDistinct <= $distinct)) {
+ $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $threshold;
+ print STDERR "Supplied merDistinct $merDistinct with threshold $threshold is the smallest.\n";
last;
}
- if ((defined($merTotal)) && ($merTotal < $total)) {
- $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $lastThreshold;
- print STDERR "Supplied merTotal $merTotal with threshold $lastThreshold is the smallest.\n";
+ if ((defined($merTotal)) && ($merTotal <= $total)) {
+ $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $threshold;
+ print STDERR "Supplied merTotal $merTotal with threshold $threshold is the smallest.\n";
last;
}
-
- $lastThreshold = $threshold;
}
close(F);
}
@@ -2527,11 +2648,6 @@ sub runMeryl ($$$$$$$$) {
}
sub meryl {
-
- if (getGlobal("ovlOverlapper") eq "umd") {
- caFailure("meryl attempted to compute mer counts for the umd overlapper", undef);
- }
-
my $ovlc = 0; # No compression, unless we're the mer overlapper
my $obtc = 0;
@@ -2616,252 +2732,6 @@ sub meryl {
################################################################################
################################################################################
-sub getUMDOverlapperClearRange ($) {
- my $dir = shift @_;
- my $fileName = "$asm.obtClrRange";
-
- open(F, "ls -1 -d $wrk/$dir/*overlapperRunDir* |");
- open(G, ">$wrk/$dir/$fileName") or caFailure("failed to write '$wrk/$dir/$fileName'", undef);
- while (<F>) {
- chomp;
-
- open(T, "< $_/revisedOrigTrimsForReads.txt") or caFailure("failed to open '$_/revisedOrigTrimsForReads.txt'", undef);
- while (<T>) {
- my @trimData = split(/\s+/,$_);
- my $uid = $trimData[0];
- my $bgn = $trimData[1];
- my $end = $trimData[2];
-
- if ($bgn < $end) {
- print G "frg uid $uid obt all $bgn $end\n";
- } else {
- print G "frg uid $uid obt all $end $bgn\n";
- }
- }
- close(T);
- }
- close(F);
- close(G);
-
- return $fileName;
-}
-
-sub UMDoverlapper () {
- goto alldone if (-d "$wrk/$asm.ovlStore");
- goto alldone if (getGlobal("ovlOverlapper") ne "umd");
-
- my $outDir = "1-overlapper";
- system("mkdir $wrk/$outDir") if (! -d "$wrk/$outDir");
-
- my $jobID = "0000001";
- system("mkdir $wrk/$outDir/$jobID") if (! -d "$wrk/$outDir/$jobID");
-
- my $vi = getGlobal("vectorIntersect");
-
- #dump the frag file from gkp if it does not exist already
- # should check if vector clear then dump vec range else dump this range
- if (defined($vi)) {
- if (runCommand($wrk, "$bin/gatekeeper -clear VEC -dumpfrg $wrk/$asm.gkpStore 2> $wrk/gatekeeper.err | grep -v 'No source' > $wrk/$asm.vec.frg")) {
- caFailure("failed to dump gatekeeper store for UMD overlapper", "$wrk/gatekeeper.err");
- }
- }
- elsif ( ! -s "$wrk/$asm.frg" ) {
- if (runCommand($wrk, "$bin/gatekeeper -dumpfrg $wrk/$asm.gkpStore 2> $wrk/gatekeeper.err | grep -v 'No source' > $wrk/$asm.frg")) {
- caFailure("failed to dump gatekeeper store for UMD overlapper", "$wrk/gatekeeper.err");
- }
- }
-
- # create a job list (we have only one job for right now)
- open(SUB, "> $wrk/$outDir/ovljobs.dat") or caFailure("failed to open '$wrk/$outDir/ovljobs.dat'", undef);
- print SUB "$jobID "; print SUB "\n";
- print SUB "$jobID "; print SUB "\n";
- close(SUB);
-
- # run frg file command
- #
- $cmd = "$bin/runUMDOverlapper ";
- $cmd .= getGlobal("umdOverlapperFlags") . " ";
-
- # when we have vector clear, pass it to the overlapper, otherwise tell the overlapper to figure it out
- if (defined($vi)) {
- $cmd .= "-vector-trim-file $wrk/$asm.vec.frg $wrk/$asm.vec.frg ";
- } else {
- $cmd .= "-calculate-trims $wrk/$asm.frg ";
- }
-
- $cmd .= "$wrk/$outDir/$jobID/$asm.umd.frg ";
- $cmd .= " > $wrk/$outDir/$jobID/overlapper.out 2>$wrk/$outDir/$jobID/overlapper.err";
-
- if (runCommand("$wrk/$outDir", $cmd)) {
- caFailure("failed to run UMD overlapper", "$wrk/$outDir/$jobID/overlapper.err");
- }
-
- my $trimFile = getUMDOverlapperClearRange($outDir);
- $cmd = "";
- $cmd .= "$bin/gatekeeper --edit ";
- $cmd .= "$wrk/$outDir/$trimFile $wrk/$asm.gkpStore";
- if (runCommand("$wrk/$outDir", $cmd)) {
- caFailure("failed to update OBT trims", "undef");
- }
-
- # now create the binary overlaps
- $cmd = "";
- $cmd .= "cat $wrk/$outDir/$jobID/$asm.umd.reliable.overlaps | ";
- $cmd .= "awk '{print \$1\"\\t\"\$2\"\\t\"\$3\"\\t\"\$4\"\\t\"\$5\"\\t\"\$6\"\\t\"\$7}' | ";
- $cmd .= "$bin/convertOverlap ";
- $cmd .= "-b -ovldump ";
- $cmd .= " > $wrk/$outDir/$jobID/$jobID.ovb";
- if (runCommand("$wrk/$outDir", $cmd)) {
- caFailure("failed to create overlaps", undef);
- }
-
- #cleanup
- rmrf("$asm.vec.frg");
-
- touch("$wrk/$outDir/$jobID/$jobID.success");
- stopAfter("overlapper");
-
- alldone:
-}
-
-################################################################################
-################################################################################
-################################################################################
-
-sub getFigaroClearRange ($) {
- my $outDir = shift @_;
- my $fileName = "$asm.clv";
-
- # the figaro output is UID,IID CLR_BGN
- # first reformat is as UID CLR_BGN
- runCommand("$wrk/$outDir", "awk '{print substr(\$1, 1, index(\$1, \",\")-1)\" \"\$2}' $wrk/$outDir/$asm.vectorcuts > $wrk/$outDir/$asm.clrBgn");
-
- # sort by UID and join it together with the read end to form the full vector clear range
- runCommand("$wrk/$outDir", "sort -nk 1 -T $wrk/$outDir $wrk/$outDir/$asm.clrBgn > $wrk/$outDir/$asm.clrBgn.sorted");
- runCommand("$wrk/$outDir", "join $wrk/$outDir/$asm.clrBgn.sorted $wrk/$asm.untrimmed -o 1.1,1.2,2.3 > $wrk/$outDir/$fileName");
-
- # clean up
- rmrf("$outDir/$asm.clrBgn");
- rmrf("$outDir/$asm.clrBgn.sorted");
-
- return $fileName;
-}
-
-sub generateFigaroTrim($) {
- my $outDir = shift @_;
-
- return if (-e "$wrk/$outDir/trim.success");
-
- # run command
- #
- $cmd = "$bin/figaro ";
- $cmd .= getGlobal("figaroFlags") . " ";
- $cmd .= "-F $wrk/$asm.fasta -P $asm ";
- $cmd .= " > $wrk/$outDir/figaro.out 2>$wrk/$outDir/figaro.err";
-
- if (runCommand("$wrk/$outDir", $cmd)) {
- caFailure("figaro died", "$wrk/$outDir/figaro.err");
- }
-
- # update the gkpStore with newly computed clear ranges
- return getFigaroClearRange($outDir);
-}
-
-sub getUMDTrimClearRange($) {
- my $outDir = shift @_;
- my $fileName = "$asm.clv";
-
- # the umd output is CLR_BGN (in the same order as the input)
- # to join it with the UID we first number both the list of UIDs in the fasta file and the CLR_BGN
- runCommand("$wrk/$outDir", "cat $wrk/$asm.fasta | grep \">\" | awk '{print NR\" \"substr(\$1, 2, index(\$1, \",\")-2)}' > $wrk/$outDir/$asm.numberedUids");
- runCommand("$wrk/$outDir", "awk '{print NR\" \"\$0}' $asm.vectorcuts > $asm.numberedCuts");
-
- # now we join them together
- runCommand("$wrk/$outDir", "join $wrk/$outDir/$asm.numberedUids $wrk/$outDir/$asm.numberedCuts -o 1.2,2.2 > $wrk/$outDir/$asm.clrBgn");
-
- # now we can join together the UID CLR_BGN with the read-end information for the full clear range
- runCommand("$wrk/$outDir", "sort -nk 1 -T $wrk/$outDir $wrk/$outDir/$asm.clrBgn > $wrk/$outDir/$asm.clrBgn.sorted");
- runCommand("$wrk/$outDir", "join $wrk/$outDir/$asm.clrBgn.sorted $wrk/$asm.untrimmed -o 1.1,1.2,2.3 > $wrk/$outDir/$fileName");
-
- # clean up
- rmrf("$outDir/$asm.numberedUids");
- rmrf("$outDir/$asm.numberedCuts");
- rmrf("$outDir/$asm.clrBgn");
- rmrf("$outDir/$asm.clrBgn.sorted");
- rmrf("$outDir/vectorTrimIntermediateFile001.*");
-
- return $fileName;
-}
-
-sub generateUMDTrim($) {
- my $outDir = shift @_;
-
- return if (-e "$wrk/$outDir/trim.success");
-
- # run command
- #
- $cmd = "$bin/dataWorkReduced/findVectorTrimPoints.perl ";
- $cmd .= "$wrk/$asm.fasta $wrk/$outDir/$asm.vectorcuts ";
- $cmd .= " > $wrk/$outDir/umd.out 2>$wrk/$outDir/umd.err";
-
- if (runCommand("$wrk/$outDir", $cmd)) {
- caFailure("UMD overlapper dataWorkReduced/findVectorTrimPoints.perl died",
- "$wrk/$outDir/umd.err");
- }
-
- return getUMDTrimClearRange($outDir);
-}
-
-sub generateVectorTrim ($) {
- my $vi = getGlobal("vectorIntersect");
- my $trimmer = getGlobal("vectorTrimmer");
- my $outDir = "0-preoverlap";
- my $trimFile = undef;
-
- # when vector insersect is specified or no external trimming is requested, do nothing
- return if (defined($vi));
- return if ($trimmer eq "ca");
- return if (-e "$wrk/$outDir/trim.success");
-
- #dump the fasta file from gkp
- if ( ! -e "$wrk/$asm.fasta" ) {
- if (runCommand($wrk, "$bin/gatekeeper -dumpfastaseq -clear UNTRIM $wrk/$asm.gkpStore 2> $wrk/$outDir/gatekeeper.err > $wrk/$asm.fasta")) {
- caFailure("failed to dump gatekeeper store for figaro trimmer",
- "$wrk/$outDir/gatekeeper.err");
- }
- }
- #dump the clr range
- if ( ! -e "$wrk/$asm.untrimmed" ) {
- if (runCommand($wrk, "$bin/gatekeeper -dumpfragments -tabular -clear UNTRIM $wrk/$asm.gkpStore 2> $wrk/$outDir/gatekeeper.err | grep -v 'UID' |awk '{print \$1\" \"\$12\" \"\$13}' | sort -nk 1 -T $wrk/ > $wrk/$asm.untrimmed")) {
- caFailure("failed to dump gatekeeper quality trim points for figaro trimmer",
- "$wrk/$outDir/gatekeeper.err");
- }
- }
-
- if ($trimmer eq "figaro") {
- $trimFile = generateFigaroTrim($outDir);
- } elsif($trimmer eq "umd") {
- $trimFile = generateUMDTrim($outDir);
- } else {
- caFailure("unknown vector trimmer $trimmer", undef);
- }
-
- # set the global vector trim file so that the subsequent code will update the gkp for us
- setGlobal("vectorIntersect", "$wrk/$outDir/$trimFile");
-
- #cleanup
- rmrf("$asm.fasta");
- rmrf("$asm.untrimmed");
-
- touch("$wrk/$outDir/trim.success");
-
- return;
-}
-
-################################################################################
-################################################################################
-################################################################################
-
sub findMBTFailures ($) {
my $mbtJobs = shift @_;
my $failures = 0;
@@ -2896,7 +2766,6 @@ sub findMBTSuccess ($) {
sub merTrim {
return if (getGlobal("doOverlapBasedTrimming") == 0);
- return if (getGlobal("ovlOverlapper") eq "umd");
# Skip mer based trimming if it is done, or if the ovlStore already exists.
#
@@ -2934,8 +2803,8 @@ sub merTrim {
my $mbtThreads = getGlobal("mbtThreads");
- my $taskID = getGlobal("gridTaskID");
- my $submitTaskID = getGlobal("gridArraySubmitID");
+ my $taskID = getGlobal("gridEngineTaskID");
+ my $submitTaskID = getGlobal("gridEngineArraySubmitID");
runMeryl($merSize, $merComp, "-C", "auto", undef, undef, "mbt", 0);
@@ -3017,24 +2886,24 @@ sub merTrim {
#
if (findMBTSuccess($mbtJobs) == 0) {
if (getGlobal("useGrid") && getGlobal("mbtOnGrid")) {
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeMerTrim = getGlobal("sgeMerTrim");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridMerTrim = getGlobal("gridOptionsMerTrim");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("mbt_$asm$sgeName", $mbtJobs);
- my $arrayOpt = getGridArrayOption("mbt_$asm$sgeName", $mbtJobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("mbt_$asm$gridName", $mbtJobs);
+ my $arrayOpt = getGridArrayOption("mbt_$asm$gridName", $mbtJobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeMerTrim $nameOption \"$jobName\" $arrayOpt \\\n";
- $SGE .= " $outputOption $wrk/0-mertrim/$asm.merTrim.$submitTaskID.sge.err \\\n";
- $SGE .= " $wrk/0-mertrim/mertrim.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridMerTrim $nameOption \"$jobName\" $arrayOpt \\\n";
+ $cmd .= " $outputOption $wrk/0-mertrim/$asm.merTrim.$submitTaskID.sge.err \\\n";
+ $cmd .= " $wrk/0-mertrim/mertrim.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
for (my $i=1; $i<=$mbtJobs; $i++) {
@@ -3151,6 +3020,7 @@ sub merOverlapper($) {
return if (-d "$wrk/$asm.ovlStore");
return if (-d "$wrk/$asm.obtStore") && ($isTrim eq "trim");
+ return if (-d "$wrk/$asm.tigStore");
caFailure("mer overlapper detected no fragments", undef) if ($numFrags == 0);
caFailure("mer overlapper doesn't know if trimming or assembling", undef) if (!defined($isTrim));
@@ -3195,8 +3065,8 @@ sub merOverlapper($) {
# Need mer counts, unless there is only one partition.
meryl() if (($ovmJobs > 1) || ($merylNeeded));
- my $taskID = getGlobal("gridTaskID");
- my $submitTaskID = getGlobal("gridArraySubmitID");
+ my $taskID = getGlobal("gridEngineTaskID");
+ my $submitTaskID = getGlobal("gridEngineArraySubmitID");
# Create overmerry and olap-from-seeds jobs
#
@@ -3350,24 +3220,24 @@ sub merOverlapper($) {
#
if (findOvermerrySuccess($outDir, $ovmJobs) == 0) {
if (getGlobal("useGrid") && getGlobal("ovlOnGrid")) {
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeOverlap = getGlobal("sgeMerOverlapSeed");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridOverlap = getGlobal("gridOptionsMerOverlapSeed");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("mer_$asm$sgeName", $ovmJobs);
- my $arrayOpt = getGridArrayOption("mer_$asm$sgeName", $ovmJobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("mer_$asm$gridName", $ovmJobs);
+ my $arrayOpt = getGridArrayOption("mer_$asm$gridName", $ovmJobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
- $SGE .= " $outputOption $wrk/$outDir/seeds/$submitTaskID.err \\\n";
- $SGE .= " $wrk/$outDir/overmerry.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
+ $cmd .= " $outputOption $wrk/$outDir/seeds/$submitTaskID.err \\\n";
+ $cmd .= " $wrk/$outDir/overmerry.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
for (my $i=1; $i<=$ovmJobs; $i++) {
@@ -3442,24 +3312,24 @@ sub merOverlapper($) {
#
if (findOlapFromSeedsSuccess($outDir, $olpJobs) == 0) {
if (getGlobal("useGrid") && getGlobal("ovlOnGrid")) {
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeOverlap = getGlobal("sgeMerOverlapExtend");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridOverlap = getGlobal("gridOptionsMerOverlapExtend");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("olp_$asm$sgeName", $olpJobs);
- my $arrayOpt = getGridArrayOption("mer_$asm$sgeName", $ovmJobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("olp_$asm$gridName", $olpJobs);
+ my $arrayOpt = getGridArrayOption("mer_$asm$gridName", $ovmJobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
- $SGE .= " $outputOption $wrk/$outDir/olaps/$submitTaskID.err \\\n";
- $SGE .= " $wrk/$outDir/olap-from-seeds.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
+ $cmd .= " $outputOption $wrk/$outDir/olaps/$submitTaskID.err \\\n";
+ $cmd .= " $wrk/$outDir/olap-from-seeds.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
for (my $i=1; $i<=$olpJobs; $i++) {
@@ -3490,10 +3360,9 @@ sub merOverlapper($) {
sub createOverlapJobs($) {
my $isTrim = shift @_;
- if (-d "$wrk/$asm.ovlStore") {
- stopAfter("meryl");
- }
- return if (-d "$wrk/$asm.ovlStore");
+ stopAfter("meryl") if (-d "$wrk/$asm.ovlStore");
+ return if (-d "$wrk/$asm.ovlStore");
+ return if (-d "$wrk/$asm.tigStore");
caFailure("overlapper detected no fragments", undef) if ($numFrags == 0);
caFailure("overlapper needs to know if trimming or assembling", undef) if (!defined($isTrim));
@@ -3522,20 +3391,6 @@ sub createOverlapJobs($) {
}
return if (-e "$wrk/$outDir/overlap.sh");
- # umd overlapper here
- #
- if (getGlobal("ovlOverlapper") eq "umd") {
- # For Sergey:
- #
- # UMDoverlapper() needs to dump the gkpstore, run UMD, build
- # the ovlStore and update gkpStore with new clear ranges.
- # The explicit call to UMDoverlapper in main() can then go away.
- # OBT is smart enough to disable itself if umd is enabled.
- #
- UMDoverlapper();
- return;
- }
-
# mer overlapper here
#
if ((($isTrim eq "trim") && (getGlobal("obtOverlapper") eq "mer")) ||
@@ -3558,8 +3413,8 @@ sub createOverlapJobs($) {
meryl();
- my $taskID = getGlobal("gridTaskID");
- my $submitTaskID = getGlobal("gridArraySubmitID");
+ my $taskID = getGlobal("gridEngineTaskID");
+ my $submitTaskID = getGlobal("gridEngineArraySubmitID");
# We make a giant job array for this -- we need to know hashBeg,
# hashEnd, refBeg and refEnd -- from that we compute batchName
@@ -3681,24 +3536,24 @@ sub createOverlapJobs($) {
# things here
#
if (getGlobal("useGrid") && getGlobal("ovlOnGrid")) {
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeOverlap = getGlobal("sgeOverlap");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridOverlap = getGlobal("gridOptionsOverlap");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("ovl_$asm$sgeName", $jobs);
- my $arrayOpt = getGridArrayOption("ovl_$asm$sgeName", $jobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("ovl_$asm$gridName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovl_$asm$gridName", $jobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
- $SGE .= " $outputOption $wrk/$outDir/$submitTaskID.out \\\n";
- $SGE .= " $wrk/$outDir/overlap.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridOverlap $nameOption \"$jobName\" $arrayOpt \\\n";
+ $cmd .= " $outputOption $wrk/$outDir/$submitTaskID.out \\\n";
+ $cmd .= " $wrk/$outDir/overlap.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
for (my $i=1; $i<=$jobs; $i++) {
@@ -3722,13 +3577,11 @@ sub createOverlapJobs($) {
sub checkOverlapper ($) {
my $isTrim = shift @_;
- my $outDir = "1-overlapper";
- my $ovlOpt = "";
+ return if (-d "$wrk/$asm.tigStore");
+ return if (($isTrim eq "trim") && (-d "$wrk/0-overlaptrim/$asm.obtStore"));
+ return if (($isTrim ne "trim") && (-d "$wrk/$asm.ovlStore"));
- if ($isTrim eq "trim") {
- $outDir = "0-overlaptrim-overlap";
- $ovlOpt = "-G";
- }
+ my $outDir = ($isTrim ne "trim") ? "1-overlapper" : "0-overlaptrim-overlap";
my $failedJobs = 0;
my $failureMessage = "";
@@ -3798,8 +3651,6 @@ sub checkOverlap {
checkOverlapper($isTrim);
} elsif (getGlobal("obtOverlapper") eq "mer") {
checkMerOverlapper($isTrim);
- } elsif (getGlobal("obtOverlapper") eq "umd") {
- caFailure("checkOverlap() wanted to check umd overlapper for obt?\n", undef);
} else {
caFailure("checkOverlap() unknown obt overlapper?\n", undef);
}
@@ -3811,8 +3662,6 @@ sub checkOverlap {
checkOverlapper($isTrim);
} elsif (getGlobal("ovlOverlapper") eq "mer") {
checkMerOverlapper($isTrim);
- } elsif (getGlobal("ovlOverlapper") eq "umd") {
- # Nop.
} else {
caFailure("checkOverlap() unknown ovl overlapper?\n", undef);
}
@@ -3826,6 +3675,7 @@ sub checkOverlap {
sub createOverlapStore {
goto alldone if (-d "$wrk/$asm.ovlStore");
+ goto alldone if (-d "$wrk/$asm.tigStore");
if (runCommand($wrk, "find -L $wrk/1-overlapper \\( -name \\*ovb.gz -or -name \\*ovb \\) -print > $wrk/$asm.ovlStore.list")) {
caFailure("failed to generate a list of all the overlap files", undef);
@@ -3839,10 +3689,6 @@ sub createOverlapStore {
$cmd .= " -o $wrk/$asm.ovlStore.BUILDING ";
$cmd .= " -g $wrk/$asm.gkpStore ";
- if (defined(getGlobal("closureOverlaps"))){
- $cmd .= " -i " . getGlobal("closureOverlaps");
- }
-
$cmd .= " -M " . getGlobal("ovlStoreMemory");
$cmd .= " -L $wrk/$asm.ovlStore.list ";
$cmd .= " > $wrk/$asm.ovlStore.err 2>&1";
@@ -3881,12 +3727,12 @@ sub createOverlapStore {
sub overlapTrim {
return if (getGlobal("doOverlapBasedTrimming") == 0);
- return if (getGlobal("ovlOverlapper") eq "umd");
# Skip overlap based trimming if it is done, or if the ovlStore already exists.
#
goto alldone if (-e "$wrk/0-overlaptrim/overlaptrim.success");
goto alldone if (-d "$wrk/$asm.ovlStore");
+ goto alldone if (-d "$wrk/$asm.tigStore");
system("mkdir $wrk/0-overlaptrim") if (! -d "$wrk/0-overlaptrim");
system("mkdir $wrk/0-overlaptrim-overlap") if (! -d "$wrk/0-overlaptrim-overlap");
@@ -4138,6 +3984,7 @@ sub overlapCorrection {
return if (-e "$wrk/3-overlapcorrection/$asm.erates.updated");
return if (-e "$wrk/$asm.ovlStore/corrected");
+ return if (-d "$wrk/$asm.tigStore");
system("mkdir $wrk/3-overlapcorrection") if (! -e "$wrk/3-overlapcorrection");
@@ -4148,8 +3995,8 @@ sub overlapCorrection {
my $numThreads = getGlobal("frgCorrThreads");
my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
- my $taskID = getGlobal("gridTaskID");
- my $submitTaskID = getGlobal("gridArraySubmitID");
+ my $taskID = getGlobal("gridEngineTaskID");
+ my $submitTaskID = getGlobal("gridEngineArraySubmitID");
open(F, "> $wrk/3-overlapcorrection/frgcorr.sh") or caFailure("failed to write to '$wrk/3-overlapcorrection/frgcorr.sh'", undef);
print F "#!" . getGlobal("shell") . "\n\n";
@@ -4204,24 +4051,24 @@ sub overlapCorrection {
if (getGlobal("frgCorrOnGrid") && getGlobal("useGrid")) {
# Run the correction job on the grid.
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeFragmentCorrection = getGlobal("sgeFragmentCorrection");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridFragmentCorrection = getGlobal("gridOptionsFragmentCorrection");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("frg_$asm$sgeName", $jobs);
- my $arrayOpt = getGridArrayOption("ovl_$asm$sgeName", $jobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("frg_$asm$gridName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovl_$asm$gridName", $jobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeFragmentCorrection $nameOption \"$jobName\" $arrayOpt ";
- $SGE .= " $outputOption $wrk/3-overlapcorrection/$submitTaskID.err ";
- $SGE .= "$wrk/3-overlapcorrection/frgcorr.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridFragmentCorrection $nameOption \"$jobName\" $arrayOpt ";
+ $cmd .= " $outputOption $wrk/3-overlapcorrection/$submitTaskID.err ";
+ $cmd .= "$wrk/3-overlapcorrection/frgcorr.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
# Run the correction job right here, right now.
@@ -4284,8 +4131,8 @@ sub overlapCorrection {
if (m/^(.*)\/([0-9]*).frgcorr/) {
#unlink "$1/$2.frgcorr";
#unlink "$1/$2.err";
- my $sge = int($2);
- #unlink "$1/$sge.err";
+ my $grid = int($2);
+ #unlink "$1/$grid.err";
}
}
close(F);
@@ -4301,8 +4148,8 @@ sub overlapCorrection {
if (! -e "$wrk/3-overlapcorrection/ovlcorr.sh") {
my $batchSize = getGlobal("ovlCorrBatchSize");
my $jobs = int($numFrags / $batchSize) + (($numFrags % $batchSize == 0) ? 0 : 1);
- my $taskID = getGlobal("gridTaskID");
- my $submitTaskID = getGlobal("gridArraySubmitID");
+ my $taskID = getGlobal("gridEngineTaskID");
+ my $submitTaskID = getGlobal("gridEngineArraySubmitID");
open(F, "> $wrk/3-overlapcorrection/ovlcorr.sh") or caFailure("failed to write '$wrk/3-overlapcorrection/ovlcorr.sh'", undef);
print F "jobid=\$$taskID\n";
@@ -4330,6 +4177,7 @@ sub overlapCorrection {
print F getBinDirectoryShellCode();
print F "if [ ! -e $wrk/3-overlapcorrection/\$jobid.erate ] ; then\n";
+
print F " \$bin/correct-olaps \\\n";
print F " -S $wrk/$asm.ovlStore \\\n";
print F " -e $wrk/3-overlapcorrection/\$jobid.erate.WORKING \\\n";
@@ -4345,24 +4193,24 @@ sub overlapCorrection {
if (getGlobal("ovlCorrOnGrid") && getGlobal("useGrid")) {
# Run the correction job on the grid.
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeOverlapCorrection = getGlobal("sgeOverlapCorrection");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridOverlapCorrection = getGlobal("gridOptionsOverlapCorrection");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("ovc_$asm$sgeName", $jobs);
- my $arrayOpt = getGridArrayOption("ovc_$asm$sgeName", $jobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("ovc_$asm$gridName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovc_$asm$gridName", $jobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeOverlapCorrection $nameOption \"$jobName\" $arrayOpt ";
- $SGE .= " $outputOption $wrk/3-overlapcorrection/$submitTaskID.err ";
- $SGE .= "$wrk/3-overlapcorrection/ovlcorr.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridOverlapCorrection $nameOption \"$jobName\" $arrayOpt ";
+ $cmd .= " $outputOption $wrk/3-overlapcorrection/$submitTaskID.err ";
+ $cmd .= "$wrk/3-overlapcorrection/ovlcorr.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
# Run the correction job right here, right now.
@@ -4433,8 +4281,8 @@ sub overlapCorrection {
if (m/^(.*)\/([0-9]*).erate/) {
#unlink "$1/$2.erate";
#unlink "$1/$2.err";
- my $sge = int($2);
- #unlink "$1/$sge.err";
+ my $grid = int($2);
+ #unlink "$1/$grid.err";
}
}
close(F);
@@ -4617,13 +4465,12 @@ sub unitigger () {
system("mkdir $wrk/4-unitigger") if (! -e "$wrk/4-unitigger");
- my $e = getGlobal("utgErrorRate"); # Unitigger and BOG
- my $E = getGlobal("utgErrorLimit");
- my $eg = getGlobal("utgGraphErrorRate"); # BOGART
- my $Eg = getGlobal("utgGraphErrorLimit");
- my $em = getGlobal("utgMergeErrorRate");
- my $Em = getGlobal("utgMergeErrorLimit");
- my $mem = getGlobal("batMemory");
+ my $e = getGlobal("utgErrorRate"); # Unitigger and BOG
+ my $E = getGlobal("utgErrorLimit");
+ my $eg = getGlobal("utgGraphErrorRate"); # BOGART
+ my $Eg = getGlobal("utgGraphErrorLimit");
+ my $em = getGlobal("utgMergeErrorRate");
+ my $Em = getGlobal("utgMergeErrorLimit");
my $B = int($numFrags / getGlobal("cnsPartitions"));
$B = getGlobal("cnsMinFrags") if ($B < getGlobal("cnsMinFrags"));
@@ -4633,7 +4480,9 @@ sub unitigger () {
my $unitigger = getUnitigger();
if ($unitigger eq "bogart") {
- my $th = getGlobal("batThreads");
+ my $th = getGlobal("batThreads");
+ my $mem = getGlobal("batMemory");
+ my $opts = getGlobal("batOptions");
$cmd = "$bin/bogart ";
$cmd .= " -O $wrk/$asm.ovlStore ";
@@ -4645,9 +4494,8 @@ sub unitigger () {
$cmd .= " -em $em ";
$cmd .= " -Em $Em ";
$cmd .= " -threads $th " if (defined($th));
- $cmd .= " -R " if (getGlobal("batRebuildRepeats") == 1);
- $cmd .= " -E " if (getGlobal("batMateExtension") == 1);
- $cmd .= " -M $mem " if (defined($mem));
+ $cmd .= " -M $mem " if (defined($mem));
+ $cmd .= " $opts " if (defined($opts));
$cmd .= " -o $wrk/4-unitigger/$asm ";
$cmd .= " > $wrk/4-unitigger/unitigger.err 2>&1";
} elsif ($unitigger eq "bog") {
@@ -4697,6 +4545,8 @@ sub unitigger () {
################################################################################
sub createPostUnitiggerConsensusJobs (@) {
+ my $blasr = findBlasr(getGlobal("consensus"));
+ if (!defined($blasr)) { setGlobal("consensus", "cns"); }
my $consensusType = getGlobal("consensus");
return if (-e "$wrk/5-consensus/consensus.sh");
@@ -4723,7 +4573,7 @@ sub createPostUnitiggerConsensusJobs (@) {
}
close(F);
- my $taskID = getGlobal("gridTaskID");
+ my $taskID = getGlobal("gridEngineTaskID");
open(F, "> $wrk/5-consensus/consensus.sh") or caFailure("can't open '$wrk/5-consensus/consensus.sh'", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
@@ -4757,12 +4607,12 @@ sub createPostUnitiggerConsensusJobs (@) {
print F getBinDirectoryShellCode();
if ($consensusType eq "cns") {
- my $reduce = getGlobal('cnsReduceUnitigs');
+ my $maxCov = getGlobal('cnsMaxCoverage');
print F "\$bin/utgcns \\\n";
print F " -g $wrk/$asm.gkpStore \\\n";
print F " -t $wrk/$asm.tigStore 1 \$jobid \\\n";
- print F " -reduce $reduce \\\n" if (defined($reduce));
+ print F " -maxcoverage $maxCov \\\n";
print F "> $wrk/5-consensus/${asm}_\$jobid.cns.err 2>&1 \\\n";
print F "&& \\\n";
print F "\$bin/utgcnsfix \\\n";
@@ -4782,6 +4632,19 @@ sub createPostUnitiggerConsensusJobs (@) {
print F " > $wrk/5-consensus/${asm}_\$jobid.cns.err 2>&1 \\\n";
print F "&& \\\n";
print F "touch $wrk/5-consensus/${asm}_\$jobid.success\n";
+ } elsif ($consensusType eq "pbdagcon" || $consensusType eq "pbutgcns") {
+ print F "cat $wrk/4-unitigger/$asm.partitioning |awk -v JOB=\$jobid '{if (\$1 == JOB) print \$NF}' > $wrk/5-consensus/$asm.\$jobid.iid\n";
+ print F "$bin/gatekeeper -dumpfasta $wrk/5-consensus/$asm.\$jobid -iid $wrk/5-consensus/$asm.\$jobid.iid $wrk/$asm.gkpStore\n";
+ print F "rm -f $wrk/5-consensus/$asm.\$jobid.q*\n";
+ print F "\$bin/tigStore -d layout -U -t $wrk/$asm.tigStore 1 -up \$jobid -g $wrk/$asm.gkpStore > $wrk/5-consensus/$asm.\$jobid.lay\n";
+ print F "\$bin/convertToPBCNS -path $blasr -consensus $consensusType -coverage 1 -threads " . getGlobal("cnsConcurrency") . " -prefix $wrk/5-consensus/$asm.\$jobid.tmp -length 500 -sequence $wrk/5-consensus/$asm.\$jobid.fasta -input $wrk/5-consensus/$asm.\$jobid.lay -output $wrk/5-consensus/$asm.\$jobid.fa > $wrk/5-consensus/${asm}_\$jobid.cns.err 2>&1\n";
+ print F "\$bin/addCNSToStore -path \$bin -input $wrk/5-consensus/$asm.\$jobid.fa -lay $wrk/5-consensus/$asm.\$jobid.lay -output $wrk/5-consensus/$asm.\$jobid.cns -prefix $wrk/$asm -sequence $wrk/5-consensus/$asm.\$jobid.fasta -partition \$jobid && \$bin/utgcnsfix -g $wrk/$asm.gkpStore -t $wrk/$asm.tigStore 2 \$jobid -o $wrk/5-consensus/${asm}_\$jobid.fixes > $wrk/5-consensus/${asm}_\$jobid.fix.err 2>&1 && touch $wrk/5-consensus/${asm}_\$jobid.success\n";
+ print F "if [ -e $wrk/5-consensus/${asm}_\$jobid.success ]; then\n";
+ print F " rm -f $wrk/5-consensus/${asm}.\$jobid.fasta*\n";
+ print F " rm -f $wrk/5-consensus/${asm}.\$jobid.lay\n";
+ print F "fi\n";
+ setGlobal("cnsConcurrency", 1);
+
} else {
caFailure("unknown consensus type $consensusType; should be 'cns' or 'seqan'", undef);
}
@@ -4790,24 +4653,24 @@ sub createPostUnitiggerConsensusJobs (@) {
chmod 0755, "$wrk/5-consensus/consensus.sh";
if (getGlobal("useGrid") && getGlobal("cnsOnGrid")) {
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeConsensus = getGlobal("sgeConsensus");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridConsensus = getGlobal("gridOptionsConsensus");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("utg_$asm$sgeName", $jobs);
- my $arrayOpt = getGridArrayOption("ovc_$asm$sgeName", $jobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("utg_$asm$gridName", $jobs);
+ my $arrayOpt = getGridArrayOption("ovc_$asm$gridName", $jobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeConsensus $nameOption \"$jobName\" $arrayOpt ";
- $SGE .= "$outputOption /dev/null ";
- $SGE .= "$wrk/5-consensus/consensus.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridConsensus $nameOption \"$jobName\" $arrayOpt ";
+ $cmd .= "$outputOption /dev/null ";
+ $cmd .= "$wrk/5-consensus/consensus.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
for (my $i=1; $i<=$jobs; $i++) {
@@ -4826,6 +4689,11 @@ sub postUnitiggerConsensus () {
return if (-e "$wrk/5-consensus/consensus.success");
+ if (-e "$wrk/$asm.tigStore/seqDB.v005.dat") {
+ print STDERR "Skipping consensus because tigStore version 5 exists at $wrk/$asm.tigStore\n";
+ return;
+ }
+
system("mkdir $wrk/5-consensus") if (! -d "$wrk/5-consensus");
# NOT IDEAL. We assume that jobs are finished if the script exists. We need
@@ -4859,6 +4727,39 @@ sub postUnitiggerConsensus () {
caFailure("$failedJobs unitig consensus jobs failed; remove $wrk/5-consensus/consensus.sh to try again", undef) if ($failedJobs);
#
+ # Summarize the sampling done by reading logs
+ #
+
+ if (! -e "$wrk/5-consensus/$asm.sampling") {
+ open(O, "> $wrk/5-consensus/$asm.sampling");
+ open(D, "> $wrk/5-consensus/$asm.sampling.dat");
+
+ print D "utgIID\t#_removed\tcov_removed\t#_saved\tcov_saved\t#_kept\tcov_kept\n";
+
+ open(F, "< $wrk/4-unitigger/$asm.partitioningInfo") or caFailure("can't open '$wrk/4-unitigger/$asm.partitioningInfo'", undef);
+ while (<F>) {
+ if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+unitigs\sand\s+(\d+)\s+fragments./) {
+ my $id = substr("000" . $1, -3);
+
+ open(G, "< $wrk/5-consensus/${asm}_$id.cns.err") or warn "Failed to open '$wrk/5-consensus/${asm}_$id.cns.err'\n";
+ while (<G>) {
+ s/^\s+//;
+ s/\s+$//;
+
+ if (m/unitig\s+(\d+)\s+removing\s+(\d+)\s+\((\d+.\d+)x\)\s+contained\s+reads;\s+processing\s+only\s+(\d+)\s+contained\s+\((\d+.\d+)x\)\s+and\s+(\d+)\s+dovetail\s+\((\d+.\d+)x\)\s+reads/) {
+ print O "$_\n";
+ print D "$1\t$2\t$3\t$4\t$5\t$6\t$7\n";
+ }
+ }
+ close(G);
+ }
+ }
+ close(F);
+ close(D);
+ close(O);
+ }
+
+ #
# Apply the utgcnsfix changes
#
@@ -4995,6 +4896,40 @@ sub postUnitiggerConsensus () {
}
}
+ if (! -e "$wrk/5-consensus-coverage-stat/markRepeatUnique.err") {
+ my $astatLow = getGlobal("astatLowBound");
+ my $astatHigh = getGlobal("astatHighBound");
+
+ my $maxSingleSpan = getGlobal("maxSingleReadSpan");
+ my $lowCovDepth = getGlobal("lowCoverageDepth");
+ my $lowCovAllowed = getGlobal("lowCoverageAllowed");
+ my $minReadsUnique = getGlobal("minReadsUnique");
+ my $maxRepeatLen = getGlobal("maxRepeatLength");
+
+ system("mkdir $wrk/5-consensus-coverage-stat") if (! -d "$wrk/5-consensus-coverage-stat");
+
+ $cmd = "$bin/markRepeatUnique \\\n";
+ $cmd .= " -g $wrk/$asm.gkpStore \\\n";
+ $cmd .= " -t $wrk/$asm.tigStore 5 \\\n";
+ $cmd .= " -j $astatLow \\\n";
+ $cmd .= " -k $astatHigh \\\n";
+ $cmd .= " -span $maxSingleSpan \\\n" if (defined($maxSingleSpan));
+ $cmd .= " -lowcov $lowCovDepth $lowCovAllowed \\\n" if (defined($lowCovAllowed));
+ $cmd .= " -reads $minReadsUnique \\\n" if (defined($minReadsUnique));
+ $cmd .= " -length $maxRepeatLen \\\n" if (defined($maxRepeatLen));
+ $cmd .= " -o $wrk/5-consensus-coverage-stat/$asm.markRepeatUnique \\\n";
+ $cmd .= "> $wrk/5-consensus-coverage-stat/markRepeatUnique.err 2>&1";
+
+ if (runCommand("$wrk/5-consensus-coverage-stat", $cmd)) {
+ rename "$wrk/5-consensus-coverage-stat/markRepeatUnique.err", "$wrk/5-consensus-coverage-stat/markRepeatUnique.err.FAILED";
+ caFailure("Unitig repeat/unique marking failed", "$wrk/5-consensus-coverage-stat/markRepeatUnique.err.FAILED");
+ }
+
+ if (-d "$wrk/7-0-CGW") {
+ caFailure("Unitig repeat/unique markings updated, but there is a scaffolding already started. Remove old scaffold directories to proceed.", "");
+ }
+ }
+
# All jobs finished. Remove the partitioning from the gatekeeper store.
#
#system("rm -f $wrk/$asm.gkpStore/???.[0-9][0-9][0-9]");
@@ -5075,8 +5010,6 @@ sub CGW ($$$$$$) {
my $B = int($numFrags / getGlobal("cnsPartitions"));
$B = getGlobal("cnsMinFrags") if ($B < getGlobal("cnsMinFrags"));
- my $P = getGlobal("closurePlacement");
-
my $shatterLevel = getGlobal("cgwContigShatterWeight");
my $missingMate = getGlobal("cgwMergeMissingThreshold");
my $minWeight = getGlobal("cgwMinMergeWeight");
@@ -5092,7 +5025,6 @@ sub CGW ($$$$$$) {
$cmd .= " -G \\\n" if ($finalRun == 0);
$cmd .= " -GG \\\n" if (getGlobal("cgwPreserveConsensus") == 1);
$cmd .= " -z \\\n" if (getGlobal("cgwDemoteRBP") == 1);
- $cmd .= " -P $P \\\n" if (defined($P));
$cmd .= " -K \\\n" if (getGlobal("kickOutNonOvlContigs") != 0);
$cmd .= " -U \\\n" if (getGlobal("doUnjiggleWhenMerging") != 0);
$cmd .= " -F \\\n" if (getGlobal("toggleDoNotDemote") != 0);
@@ -5382,6 +5314,8 @@ sub scaffolder () {
# Repartition the frag store
sub createPostScaffolderConsensusJobs () {
+ my $blasr = findBlasr(getGlobal("consensus"));
+ if (!defined($blasr)) { setGlobal("consensus", "cns"); }
my $consensusType = getGlobal("consensus");
return if (-e "$wrk/8-consensus/consensus.sh");
@@ -5417,7 +5351,7 @@ sub createPostScaffolderConsensusJobs () {
}
close(F);
- my $taskID = getGlobal("gridTaskID");
+ my $taskID = getGlobal("gridEngineTaskID");
open(F, "> $wrk/8-consensus/consensus.sh") or caFailure("can't open '$wrk/8-consensus/consensus.sh'", undef);
print F "#!" . getGlobal("shell") . "\n";
print F "\n";
@@ -5472,6 +5406,16 @@ sub createPostScaffolderConsensusJobs () {
print F " > $wrk/8-consensus/$asm.cns_contigs.\$jobid.err 2>&1 \\\n";
print F "&& \\\n";
print F "touch $wrk/8-consensus/$asm.cns_contigs.\$jobid.success\n";
+ } elsif ($consensusType eq "pbdagcon" || $consensusType eq "pbutgcns") {
+ print F "\$bin/tigStore -d layout -C -t $wrk/$asm.tigStore $tigVersion -cp \$jobid -g $wrk/$asm.gkpStore > $wrk/8-consensus/$asm.\$jobid.lay\n";
+ print F "\$bin/tigStore -d consensus -U -t $wrk/$asm.tigStore 2 -cp \$jobid -g $wrk/$asm.gkpStore > $wrk/8-consensus/$asm.\$jobid.fasta\n";
+ print F "\$bin/convertToPBCNS -path $blasr -consensus $consensusType -coverage 1 -threads " . getGlobal("cnsConcurrency") . " -prefix $wrk/8-consensus/$asm.\$jobid.tmp -length 500 -sequence $wrk/8-consensus/$asm.\$jobid.fasta -input $wrk/8-consensus/$asm.\$jobid.lay -output $wrk/8-consensus/$asm.\$jobid.fa > $wrk/8-consensus/${asm}_\$jobid.err 2>&1\n";
+ print F "\$bin/addCNSToStore -path \$bin -version $tigVersion -input $wrk/8-consensus/$asm.\$jobid.fa -lay $wrk/8-consensus/$asm.\$jobid.lay -output $wrk/8-consensus/$asm.\$jobid.cns -prefix $wrk/$asm -sequence $wrk/8-consensus/$asm.\$jobid.fasta -partition \$jobid && touch $wrk/8-consensus/${asm}_\$jobid.success\n";
+ print F "if [ -e $wrk/8-consensus/${asm}_\$jobid.success ]; then\n";
+ print F " rm -f $wrk/8-consensus/${asm}.\$jobid.fasta*\n";
+ print F " rm -f $wrk/8-consensus/${asm}.\$jobid.lay\n";
+ print F "fi\n";
+ setGlobal("cnsConcurrency", 1);
} else {
caFailure("unknown consensus type $consensusType; must be 'cns' or 'seqan'", undef);
}
@@ -5481,24 +5425,24 @@ sub createPostScaffolderConsensusJobs () {
chmod 0755, "$wrk/8-consensus/consensus.sh";
if (getGlobal("cnsOnGrid") && getGlobal("useGrid")) {
- my $submitCommand = getGlobal("gridSubmitCommand");
- my $nameOption = getGlobal("gridNameOption");
- my $outputOption = getGlobal("gridOutputOption");
+ my $submitCommand = getGlobal("gridEngineSubmitCommand");
+ my $nameOption = getGlobal("gridEngineNameOption");
+ my $outputOption = getGlobal("gridEngineOutputOption");
- my $sge = getGlobal("sge");
- my $sgeName = getGlobal("sgeName");
- my $sgeConsensus = getGlobal("sgeConsensus");
+ my $grid = getGlobal("gridOptions");
+ my $gridName = getGlobal("gridJobName");
+ my $gridConsensus = getGlobal("gridOptionsConsensus");
- $sgeName = "_$sgeName" if (defined($sgeName));
- my $jobName = getGridArrayName("ctg_$asm$sgeName", $jobs);
- my $arrayOpt = getGridArrayOption("ctg_$asm$sgeName", $jobs);
+ $gridName = "_$gridName" if (defined($gridName));
+ my $jobName = getGridArrayName("ctg_$asm$gridName", $jobs);
+ my $arrayOpt = getGridArrayOption("ctg_$asm$gridName", $jobs);
- my $SGE;
- $SGE = "$submitCommand $sge $sgeConsensus $nameOption \"$jobName\" $arrayOpt ";
- $SGE .= "$outputOption /dev/null ";
- $SGE .= "$wrk/8-consensus/consensus.sh\n";
+ my $cmd;
+ $cmd = "$submitCommand $grid $gridConsensus $nameOption \"$jobName\" $arrayOpt ";
+ $cmd .= "$outputOption /dev/null ";
+ $cmd .= "$wrk/8-consensus/consensus.sh\n";
- submitBatchJobs($SGE, $jobName);
+ submitBatchJobs($cmd, $jobName);
exit(0);
} else {
for (my $i=1; $i<=$jobs; $i++) {
diff --git a/src/AS_RUN/sanity/._build-all-kmer-revisions.pl b/src/AS_RUN/sanity/._build-all-kmer-revisions.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._build-all-kmer-revisions.pl differ
diff --git a/src/AS_RUN/sanity/._build-all-wgs-revisions.pl b/src/AS_RUN/sanity/._build-all-wgs-revisions.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._build-all-wgs-revisions.pl differ
diff --git a/src/AS_RUN/sanity/._compile-all-wgs-revisions.pl b/src/AS_RUN/sanity/._compile-all-wgs-revisions.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._compile-all-wgs-revisions.pl differ
diff --git a/src/AS_RUN/sanity/._sanity-all-done.pl b/src/AS_RUN/sanity/._sanity-all-done.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity-all-done.pl differ
diff --git a/src/AS_RUN/sanity/._sanity-asm-done.pl b/src/AS_RUN/sanity/._sanity-asm-done.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity-asm-done.pl differ
diff --git a/src/AS_RUN/sanity/._sanity-get-next-date.pl b/src/AS_RUN/sanity/._sanity-get-next-date.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity-get-next-date.pl differ
diff --git a/src/AS_RUN/sanity/._sanity-merge-qc.pl b/src/AS_RUN/sanity/._sanity-merge-qc.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity-merge-qc.pl differ
diff --git a/src/AS_RUN/sanity/._sanity-purge-old.pl b/src/AS_RUN/sanity/._sanity-purge-old.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity-purge-old.pl differ
diff --git a/src/AS_RUN/sanity/._sanity-update-reference.pl b/src/AS_RUN/sanity/._sanity-update-reference.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity-update-reference.pl differ
diff --git a/src/AS_RUN/sanity/._sanity.pl b/src/AS_RUN/sanity/._sanity.pl
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity.pl differ
diff --git a/src/AS_RUN/sanity/._sanity.sh b/src/AS_RUN/sanity/._sanity.sh
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_RUN/sanity/._sanity.sh differ
diff --git a/src/AS_RUN/sanity/build-all-kmer-revisions.pl b/src/AS_RUN/sanity/build-all-kmer-revisions.pl
new file mode 100644
index 0000000..e1058b8
--- /dev/null
+++ b/src/AS_RUN/sanity/build-all-kmer-revisions.pl
@@ -0,0 +1,26 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Given a local rsync'd copy of the repository, check out all versions and compile.
+
+my $kmersvn = "/work/NIGHTLY/kmer-svn"
+
+open(F, "< $kmersvn/db/current");
+my $latest = <F>;
+chomp $latest;
+close(F);
+
+
+for (my $i=1917; $i<=$latest; $i++) {
+ if (! -d "kmer$i") {
+ print "Check out r$i\n";
+ system("mkdir kmer$i");
+ system("cd kmer$i && svn co -r $i file://$kmersvn/trunk . > kmer-checkout.err 2>&1");
+ }
+
+ if (! -d "kmer$i/FreeBSD-amd64") {
+ print "Compile r$i\n";
+ system("cd kmer$i && gmake install > kmer-build.err 2>& 1 &");
+ }
+}
diff --git a/src/AS_RUN/sanity/build-all-wgs-revisions.pl b/src/AS_RUN/sanity/build-all-wgs-revisions.pl
new file mode 100644
index 0000000..79c28d8
--- /dev/null
+++ b/src/AS_RUN/sanity/build-all-wgs-revisions.pl
@@ -0,0 +1,153 @@
+#!/usr/bin/perl
+#
+# Checkout and tar up src for each commit.
+#
+# The first half comes from get-commit-logs.pl.
+#
+#--------------------------------------------------------------------------------
+
+use strict;
+
+my $wgssvn = "/work/NIGHTLY/SVN-wgs";
+my $kmersvn = "/work/NIGHTLY/SVN-kmer";
+
+my $wgsbyd = "/work/NIGHTLY/bydate";
+
+system("mkdir $wgsbyd") if (! -d "$wgsbyd");
+
+my %revToDate;
+
+if (-d "$wgsbyd/TIP") {
+ print STDERR "Updating to latest. DON'T FORGET TO sanity.pl rsync FIRST!\n";
+ system("cd $wgsbyd/TIP/src ; svn update > update.err");
+} else {
+ print STDERR "Checking out latest.\n";
+ system("mkdir $wgsbyd/TIP");
+ system("cd $wgsbyd/TIP ; svn co file://$wgssvn/trunk/src src > checkout.err");
+}
+
+print STDERR "Reading CA commit logs.\n";
+
+open(F, "cd $wgsbyd/TIP/src ; svn log |");
+while (<F>) {
+ #print STDERR $_;
+
+ # CVS relic
+ #if (m/^date:\s+(....)\/(..)\/(..)\s+(..):(..):(..);\s+author:\s+(.*);\s+state/) {
+ # my $fildate = "$1-$2-$3-$4$5";
+ # my $cvsdate = "$1/$2/$3 $4:$5:$6 GMT";
+ # my $svndate = "{$1$2$3T$4$5Z}";
+ # $logdate{"$fildate\0$cvsdate\0$svndate"}++;
+ #}
+
+ if (m/^(r\d+)\s+\|\s+(.+)\s+\|\s+(\d\d\d\d)-(\d\d)-(\d\d)\s(\d\d):(\d\d):(\d\d)\s(.\d\d\d\d)\s/) {
+ my $fildate = "$3-$4-$5-$6$7";
+ my $svndate = "{$3$4$5T$6$7Z}";
+
+ $revToDate{$1} = "$fildate\0$svndate\0$2";
+ }
+}
+close(F);
+
+
+
+my @keys = sort keys %revToDate;
+
+print STDERR "Checking out versions.\n";
+
+foreach my $wgsrev (@keys) {
+ my ($dirdate, $svndate, $author) = split '\0', $revToDate{$wgsrev};
+
+ #next if (-e "$wgsbyd/$dirdate");
+ #next if (-e "$wgsbyd/$dirdate.tar");
+ #next if (-e "$wgsbyd/$dirdate.tar.gz");
+ #next if (-e "$wgsbyd/$dirdate.tar.bz2");
+ #next if (-e "$wgsbyd/$dirdate.tar.xz");
+
+ next if ($dirdate eq "2004-04-14-1349"); # These are all one
+ next if ($dirdate eq "2004-04-14-1350"); # slow commit, which
+ next if ($dirdate eq "2004-04-14-1351"); # we capture in
+ next if ($dirdate eq "2004-04-14-1352"); # 2004-04-14-1354
+ next if ($dirdate eq "2004-04-14-1353"); #
+
+ next if ($dirdate le "2010-00");
+
+ next if ($dirdate eq "2011-09-06-1707"); # Doesn't compile
+
+ next if ($dirdate eq "2012-02-01-2010"); # Broken GWS overlapStore
+ next if ($dirdate eq "2012-02-01-2012"); # Broken GWS overlapStore
+ next if ($dirdate eq "2012-02-01-2015"); # Broken GWS overlapStore
+
+ next if ($dirdate le "2014-00");
+
+ system("mkdir -p $wgsbyd/$dirdate") if (! -e "$wgsbyd/$dirdate");
+ system("ln -s $dirdate $wgsbyd/$wgsrev") if (! -e "$wgsbyd/$wgsrev");
+
+ # Checkout the assembler
+
+ if (! -e "$wgsbyd/$dirdate/src") {
+ print "$dirdate -- $wgsrev CHECKOUT\n";
+
+ system("cd $wgsbyd/$dirdate ; svn co -r $wgsrev file://$wgssvn/trunk/src src > src.$wgsrev.err");
+ } else {
+ print "$dirdate -- $wgsrev UPDATE\n";
+
+ system("cd $wgsbyd/$dirdate/src ; svn update -r $wgsrev");
+ }
+
+ # Checkout kmer, or, more probably, just link to one that exists already
+
+ my $kmerev = "0000";
+
+ open(L, "svn info -r \"$svndate\" file://$kmersvn/trunk |") or die "Failed to get info: $!\n";
+ while (<L>) {
+ $kmerev = $1 if (m/^Revision:\s+(\d+)/);
+ }
+ close(L);
+
+ die "Didn't find a kmer revision in $wgsbyd/$dirdate/kmer/kmer.err.\n" if ($kmerev eq "0000");
+
+ if (! -e "$wgsbyd/kmer$kmerev") {
+ print "$dirdate -- kmer$kmerev CHECKOUT\n";
+
+ system("cd $wgsbyd/$dirdate && svn co -r \"$svndate\" file://$kmersvn/trunk $wgsbyd/kmer$kmerev > $wgsbyd/kmer$kmerev.err 2>&1");
+ }
+
+ system("ln -s ../kmer$kmerev $wgsbyd/$dirdate/kmer") if (! -e "$wgsbyd/$dirdate/kmer");
+
+ # Compile
+
+ if (! -e "$wgsbyd/kmer$kmerev/build.err") {
+ print "$dirdate -- kmer$kmerev COMPILE\n";
+
+ if (! -e "$wgsbyd/kmer$kmerev/Makefile") {
+ system("cd $wgsbyd/kmer$kmerev && sh configure.sh > configure.err 2>&1");
+ }
+
+ open(F, "> $wgsbyd/kmer$kmerev/build.err");
+ print F "Waiting to compile.\n";
+ close(F);
+
+ system("qsub -N kmer$kmerev -cwd -j y -o $wgsbyd/kmer$kmerev/build.err -q vomit.q -wd $wgsbyd/kmer$kmerev -b y gmake install");
+ #system("cd $wgsbyd/kmer$kmerev && gmake install > build.err 2>&1 &");
+ }
+
+ if (! -e "$wgsbyd/$dirdate/src/build.err") {
+ print "$dirdate -- $wgsrev COMPILE\n";
+
+ open(F, "> $wgsbyd/$dirdate/src/build.err");
+ print F "Waiting to compile.\n";
+ close(F);
+
+ system("qsub -N wgs$wgsrev -hold_jid kmer$kmerev -cwd -j y -o $wgsbyd/$dirdate/src/build.err -q vomit.q -wd $wgsbyd/$dirdate/src -b y gmake");
+ #system("cd $wgsbyd/$dirdate/src && gmake > build.err 2>&1");
+ }
+
+ # Archive
+
+ #if (! -e "$wgsbyd/$dirdate.tar.bz2") {
+ # Use sge, hold on both $kmerev and $wgsrev
+ # system("cd $wgsbyd && tar -cf - $dirdate | xz -9vc > $dirdate.tar.bz2 && mv $wgsbyd/$dirdate $wgsbyd/$dirdate.DELETE &");
+ #}
+}
+
diff --git a/src/AS_RUN/sanity/compile-all-wgs-revisions.pl b/src/AS_RUN/sanity/compile-all-wgs-revisions.pl
new file mode 100644
index 0000000..1f5067d
--- /dev/null
+++ b/src/AS_RUN/sanity/compile-all-wgs-revisions.pl
@@ -0,0 +1,177 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $bgn = shift @ARGV;
+my $end = shift @ARGV;
+
+my @dates;
+
+open(F, "ls -r *bz2 |");
+while (<F>) {
+ chomp;
+
+ next if (defined($bgn) && ($_ lt $bgn));
+ next if (defined($end) && ($end lt $_));
+
+ push @dates, $_;
+}
+
+$bgn = "START_OF_TIME" if (!defined($bgn));
+$end = "END_OF_TIME" if (!defined($end));
+
+print STDERR "Building ", scalar(@dates), " revisions from $bgn to $end.\n";
+
+while (scalar(@dates) > 0) {
+ my $bz2 = shift @dates;
+ my $dir;
+
+ if ($bz2 =~ m/(\d\d\d\d-\d\d-\d\d-\d\d\d\d).tar.bz2/) {
+ $dir = $1;
+ } else {
+ die "Nope bz2='$bz2'\n";
+ }
+
+ if (! -e "$dir") {
+ print STDERR "Uncompressing $bz2\n";
+ system("bzip2 -dc $bz2 | tar -xf -") and die "Failed to uncompress '$bz2'\n";
+ next;
+ }
+
+ #if (! -e "$dir/kmer/FreeBSD-amd64") {
+ # print STDERR "Building $dir/kmer\n";
+ # if ($dir le "2000-00-00-0000") {
+ # die;
+ # } else {
+ # system("cd $dir/kmer && gmake install > errs 2>&1") and die "Failed to build '$dir/kmer'\n";
+ # }
+ #}
+
+ next if ($dir le "2008-10-10-0304"); # ALL the early assemblers fail to build; no kmer
+
+ next if ($dir eq "2008-10-29-1053"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-10-29-1619"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-10-29-1649"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-10-29-1651"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-10-29-1719"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-10-29-1720"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-10-30-0449"); # buildRefUnitigs is not committed
+ next if ($dir eq "2008-11-02-0628");
+ next if ($dir eq "2009-01-16-1639");
+ next if ($dir eq "2009-01-16-1643");
+ next if ($dir eq "2009-01-16-1646");
+ next if ($dir eq "2009-01-16-1647");
+ next if ($dir eq "2009-01-16-1657");
+ next if ($dir eq "2009-03-06-2012");
+ next if ($dir eq "2009-03-31-2004");
+ next if ($dir eq "2009-03-31-2032");
+ next if ($dir eq "2009-04-08-1725");
+ next if ($dir eq "2009-04-09-1401");
+ next if ($dir eq "2009-06-10-1805");
+ next if ($dir eq "2009-06-24-1205");
+ next if ($dir eq "2009-07-06-2003");
+ next if ($dir eq "2009-07-27-0806");
+ next if ($dir eq "2009-07-28-1230");
+ next if ($dir eq "2009-08-04-1103");
+ next if ($dir eq "2009-08-04-1105");
+ next if ($dir eq "2009-08-06-1136");
+ next if ($dir eq "2009-08-14-1103"); # BREAKS WITH rename-to-c++
+ next if ($dir eq "2009-08-28-17597");
+ next if ($dir eq "2009-09-04-2024");
+ next if ($dir eq "2009-09-04-2025");
+ next if ($dir eq "2009-09-07-0740");
+ next if ($dir eq "2009-09-09-0340");
+ next if ($dir eq "2009-09-09-0745");
+ next if ($dir eq "2009-10-12-0619");
+ next if ($dir eq "2009-10-27-1244");
+ next if ($dir eq "2009-11-19-1501");
+ next if ($dir eq "2009-12-03-0119");
+ next if ($dir eq "2009-12-19-0536");
+ next if ($dir eq "2010-01-14-0044");
+ next if ($dir eq "2010-01-26-0351"); # AS_BOG_BestOverlapGraph() prototype mismatch
+ next if ($dir eq "2010-02-04-2156"); # BOG fails to build
+ next if ($dir eq "2010-02-09-1812"); # markUniqueUnique() bad header file
+ next if ($dir eq "2010-02-17-0132"); # pointer/object confusion
+ next if ($dir eq "2010-03-22-2008"); # AS_UTL_writeFastQ() not defined
+ next if ($dir eq "2010-09-28-1055"); # Makefile refers to missing overlapStore_genomeLength.c
+ next if ($dir eq "2010-09-28-1720"); # Makefile refers to missing overlapStore_genomeLength.c
+ next if ($dir eq "2010-10-01-1343");
+ next if ($dir eq "2010-10-04-0852");
+ next if ($dir eq "2010-10-15-0246"); # const char * vs char * in AS_BOG
+ next if ($dir eq "2010-12-08-1242"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c
+ next if ($dir eq "2010-12-08-1243"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c
+ next if ($dir eq "2010-12-08-1245"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c
+ next if ($dir eq "2010-12-09-0405"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c
+ next if ($dir eq "2010-12-10-1328"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c
+ next if ($dir eq "2011-01-25-1356");
+ next if ($dir eq "2011-02-11-0548");
+ next if ($dir eq "2011-03-08-2118"); # Undefined constant in AS_BOG
+ next if ($dir eq "2011-06-17-1303");
+ next if ($dir eq "2011-08-01-1835");
+ next if ($dir eq "2011-08-01-2033");
+ next if ($dir eq "2011-08-01-2100");
+ next if ($dir eq "2011-08-01-2159");
+ next if ($dir eq "2011-08-01-2233");
+ next if ($dir eq "2011-08-02-0221");
+ next if ($dir eq "2011-08-02-0223");
+ next if ($dir eq "2011-08-02-0225");
+ next if ($dir eq "2011-08-02-0318");
+ next if ($dir eq "2011-08-04-1708");
+ next if ($dir eq "2011-08-24-2114");
+ next if ($dir eq "2011-08-25-0240");
+ next if ($dir eq "2011-08-30-1332");
+ next if ($dir eq "2011-08-30-1812");
+ next if ($dir eq "2011-08-30-1813");
+ next if ($dir eq "2011-08-30-2309");
+ next if ($dir eq "2011-09-02-1714"); # ouch, broken through 2011-09-06-1707
+ next if ($dir eq "2011-09-02-1825"); # ouch.
+ next if ($dir eq "2011-09-02-2204"); # ouch.
+ next if ($dir eq "2011-09-03-0129"); # ouch.
+ next if ($dir eq "2011-09-03-0408"); # ouch.
+ next if ($dir eq "2011-09-03-0736"); # ouch.
+ next if ($dir eq "2011-09-03-0813"); # ouch.
+ next if ($dir eq "2011-09-04-0101"); # ouch.
+ next if ($dir eq "2011-09-05-1649"); # ouch.
+ next if ($dir eq "2011-09-05-2123"); # ouch.
+ next if ($dir eq "2011-09-06-0111"); # ouch.
+ next if ($dir eq "2011-09-06-0215"); # ouch.
+ next if ($dir eq "2011-09-06-0947"); # ouch.
+ next if ($dir eq "2011-09-06-1415"); # ouch.
+ next if ($dir eq "2011-09-06-1422"); # ouch.
+ next if ($dir eq "2011-09-06-1506"); # ouch.
+ next if ($dir eq "2011-09-06-1524"); # ouch.
+ next if ($dir eq "2011-09-06-1527"); # ouch.
+ next if ($dir eq "2011-09-06-1601"); # ouch.
+ next if ($dir eq "2011-09-06-1641"); # ouch.
+ next if ($dir eq "2011-09-06-1707"); # ouch.
+ next if ($dir eq "2011-12-09-2302"); # Assigning 64-bit allocation to 32-bit pointer in AS_BAT
+
+
+ if (! -e "$dir/kmer/FreeBSD-amd64") {
+ die "Didn't find kmer in $dir\n";
+ }
+
+ if (! -e "$dir/FreeBSD-amd64/bin/gatekeeper") {
+ print STDERR "Building $dir/src\n";
+
+ if ($dir le "2009-02-03-2104") {
+ # These old builds REQUIRE that SITE_NAME be set explicitly.
+ $ENV{'SITE_NAME'} = "LOCAL";
+ system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n";
+ undef $ENV{'SITE_NAME'};
+
+ } elsif ($dir lt "2009-06-12-1741") {
+ # These builds are before the C++ hack
+ system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n";
+
+ } elsif ($dir le "2009-08-05-2205") {
+ # These builds need to rename C to C++ to build.
+ # The rename script doesn't go away until (after) 2009-09-30-1832
+ system("cd $dir/src && sh rename-to-c++.sh") and die "Failed to rename '$dir/src'\n";
+ system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n";
+
+ } else {
+ system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n";
+ }
+ }
+}
diff --git a/src/AS_RUN/sanity/sanity.pl b/src/AS_RUN/sanity/sanity.pl
index bead2c6..6e3aba3 100755
--- a/src/AS_RUN/sanity/sanity.pl
+++ b/src/AS_RUN/sanity/sanity.pl
@@ -7,23 +7,24 @@ use Config; # for @signame
#
my $site = undef;
my $wrkdir = undef;
-my $wgscvs = undef;
+my $wgssvn = undef;
my $kmersvn = undef;
if (-d "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY") {
$site = "JCVI";
$wrkdir = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY";
- $wgscvs = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY/wgs-assembler-cvs";
- $kmersvn = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY/kmer-svn";
+ $wgssvn = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY/SVN-wgs";
+ $kmersvn = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY/SVN-kmer";
} elsif (-d "/work/NIGHTLY/") {
$site = "BPWI";
$wrkdir = "/work/NIGHTLY";
- $wgscvs = "/work/NIGHTLY/wgs-assembler-cvs";
- $kmersvn = "/work/NIGHTLY/kmer-svn";
+ $wgssvn = "/work/NIGHTLY/SVN-wgs";
+ $kmersvn = "/work/NIGHTLY/SVN-kmer";
} else {
die "Unknown site configuration.\n";
}
+
# Command line options are
#
# 'oper' -- thing to do.
@@ -60,15 +61,16 @@ if (-d "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY") {
my ($thisdate, $lastdate) = parseDate($ddir);
if ($oper eq "rsync") {
- system("mkdir $wgscvs") if (! -d "$wgscvs");
- system("mkdir $kmersvn") if (! -d "$kmersvn");
+ system("mkdir -p $wgssvn") if (! -d "$wgssvn");
+ system("mkdir -p $kmersvn") if (! -d "$kmersvn");
- system("cd $wgscvs && rsync -av rsync://wgs-assembler.cvs.sourceforge.net/cvsroot/wgs-assembler/\* . > rsync.out 2>&1");
- system("cd $kmersvn && rsync -av kmer.svn.sourceforge.net::svn/kmer/\* . > rsync.out 2>&1");
+ #system("cd $wgscvs && rsync -av rsync://wgs-assembler.cvs.sourceforge.net/cvsroot/wgs-assembler/\* . > rsync.out 2>&1");
+ system("cd $wgssvn && rsync -av svn.code.sf.net::p/wgs-assembler/svn/ . > rsync.out 2>&1");
+ system("cd $kmersvn && rsync -av svn.code.sf.net::p/kmer/code/ . > rsync.out 2>&1");
} elsif ($oper eq "checkout") {
- checkoutAndLogKmer($thisdate, $lastdate);
- checkoutAndLogCA($thisdate, $lastdate);
+ checkoutAndLog($thisdate, $lastdate, "file://$kmersvn/trunk", "kmer", "kmer");
+ checkoutAndLog($thisdate, $lastdate, "file://$wgssvn/trunk/src", "src", "src");
} elsif ($oper eq "build") {
buildKmer($thisdate);
@@ -130,19 +132,21 @@ sub parseDate ($) {
}
-sub checkoutAndLogKmer ($$) {
+sub checkoutAndLog ($$) {
my $thisdate = shift @_;
my $lastdate = shift @_;
+ my $repo = shift @_; # Path to local repository
+ my $target = shift @_; # Thing to checkout
+ my $path = shift @_; # Where to put it (in wgs/)
- print STDERR "Checking out $wrkdir/$thisdate (KMER)\n";
+ print STDERR "Checking out $wrkdir/$thisdate (repo=$repo path=$path)\n";
- if (-d "$wrkdir/$thisdate/wgs/kmer") {
- print STDERR "$wrkdir/$thisdate/wgs/kmer already exists. Please remove to rerun.\n";
+ if (-d "$wrkdir/$thisdate/wgs/$path") {
+ print STDERR "$wrkdir/$thisdate/wgs/$path already exists. Please remove to rerun.\n";
return;
}
- system("mkdir $wrkdir/$thisdate") if (! -d "$wrkdir/$thisdate");
- system("mkdir $wrkdir/$thisdate/wgs") if (! -d "$wrkdir/$thisdate/wgs");
+ system("mkdir -p $wrkdir/$thisdate/wgs") if (! -d "$wrkdir/$thisdate/wgs");
# Convert time-names to dates that svn can use.
@@ -163,8 +167,8 @@ sub checkoutAndLogKmer ($$) {
} else {
}
}
-
- system("cd $wrkdir/$thisdate/wgs && svn co -r \"{$thisdatesvn}\" file://$kmersvn/trunk kmer > kmer.checkout.err 2>&1");
+
+ system("cd $wrkdir/$thisdate/wgs && svn co -r \"{$thisdatesvn}\" $repo $target > $path.checkout.err 2>&1");
# This is annoying. SVN log will report changes inclusive to revisions. -r 5:9 will report
# changes made in revisions 5 through 9. When you give it a date, it finds the revision that
@@ -178,8 +182,8 @@ sub checkoutAndLogKmer ($$) {
my $loRev;
my $hiRev;
- print "svn log -v file://$kmersvn/trunk -r \"{$lastdatesvn}:{$thisdatesvn}\"\n";
- open(F, "cd $wrkdir/$thisdate/wgs && svn log -v file://$kmersvn/trunk -r \"{$lastdatesvn}:{$thisdatesvn}\" |");
+ print "svn log -v $repo -r \"{$lastdatesvn}:{$thisdatesvn}\"\n";
+ open(F, "cd $wrkdir/$thisdate/wgs && svn log -v $repo -r \"{$lastdatesvn}:{$thisdatesvn}\" |");
while (<F>) {
if (m/^r(\d+)\s+\|\s+/) {
$loRev = $1 if ((!defined($loRev)) || ($1 < $loRev));
@@ -193,127 +197,14 @@ sub checkoutAndLogKmer ($$) {
print STDERR "loRev='$loRev' hiRev='$hiRev'\n";
if (defined($loRev) && defined($hiRev) && ($loRev < $hiRev)) {
- print "svn log -v file://$kmersvn/trunk -r $loRev:$hiRev\n";
- system("cd $wrkdir/$thisdate/wgs && svn log -v file://$kmersvn/trunk -r $loRev:$hiRev > kmer.updates");
+ print "svn log -v $repo -r $loRev:$hiRev\n";
+ system("cd $wrkdir/$thisdate/wgs && svn log -v $repo -r $loRev:$hiRev > $path.updates");
}
}
print STDERR "$thisdate checked out!\n";
}
-sub checkoutAndLogCA ($$) {
- my $thisdate = shift @_;
- my $lastdate = shift @_;
- my $tag = undef; #"-r VERSION-5_40-BRANCH";
-
- print STDERR "Checking out $wrkdir/$thisdate (CA) ", (defined($tag)) ? "TAG=$tag" : "", "\n";
-
- if (-d "$wrkdir/$thisdate/wgs/src") {
- print STDERR "$wrkdir/$thisdate/wgs/src already exists. Please remove to rerun.\n";
- return;
- }
-
- system("mkdir $wrkdir/$thisdate") if (! -d "$wrkdir/$thisdate");
- system("mkdir $wrkdir/$thisdate/wgs") if (! -d "$wrkdir/$thisdate/wgs");
-
- # Convert time-names to dates that cvs can use.
-
- my $thisdatecvs;
- my $lastdatecvs;
-
- my $tz = `date +%z`; chomp $tz;
-
- if ($thisdate =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) {
- $thisdatecvs = "$1-$2-$3 $4:$5 $tz";
- } else {
- }
-
- if (defined($lastdate)) {
- if ($lastdate =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) {
- $lastdatecvs = "$1-$2-$3 $4:$5 $tz";
- } else {
- }
- }
-
- # Add -R to cvs options, for read-only repository; breaks on jcvi
- # cvs; this seems to be a BSD extension?
-
- #print STDERR "cd $wrkdir/$thisdate/wgs && cvs -r -d $wgscvs -z3 co -N -D '$thisdatecvs' $tag src\n";
- system("cd $wrkdir/$thisdate/wgs && cvs -r -d $wgscvs -z3 co -N -D '$thisdatecvs' $tag src > src.checkout.err 2>&1");
-
- if ($lastdate ne "") {
- #print STDERR "cd $wrkdir/$thisdate/wgs && cvs -r -d $wgscvs -z3 log -N -S -d '$lastdatecvs<$thisdatecvs' src\n";
- system("cd $wrkdir/$thisdate/wgs && cvs -r -d $wgscvs -z3 log -N -S -d '$lastdatecvs<$thisdatecvs' src > src.updates.raw");
-
- my $log;
- my $revs;
- my $date;
- my $auth;
- my $chng;
- my $file;
- my $inlog;
- my %logs;
- my %logdate;
-
- open(F, "< $wrkdir/$thisdate/wgs/src.updates.raw");
- while (<F>) {
- if (m/^==========.*========$/) {
- $inlog = 0;
- if (!defined($logs{$log})) {
- $logs{$log} = "$date\t$revs\t$chng\t$auth\t$file\n";
- } else {
- $logs{$log} .= "$date\t$revs\t$chng\t$auth\t$file\n";
- }
- $logdate{"$date\0$log"}++;
- undef $log;
- }
- if (m/^----------------------------$/) {
- $inlog = 0;
- if (!defined($logs{$log})) {
- $logs{$log} = "$date\t$revs\t$chng\t$auth\t$file\n";
- } else {
- $logs{$log} .= "$date\t$revs\t$chng\t$auth\t$file\n";
- }
- $logdate{"$date\0$log"}++;
- undef $log;
- }
- if ($inlog) {
- $log .= $_;
- }
- if (m/^RCS\s+file:\s+(.*),v/) {
- $file = $1;
- $file =~ s!$wgscvs/src/!!g;
- }
- if (m/^revision\s+(\d+.\d+)/) {
- $revs = $1;
- }
- if (m/^date:\s+(.*);\s+author:\s+(.*);\s+state.*lines:\s+(.*)\s*$/) {
- $date = $1;
- $auth = $2;
- $chng = $3;
- $inlog = 1;
- }
- }
- close(F);
-
-
- open(F, "> $wrkdir/$thisdate/wgs/src.updates");
- my @keys = sort keys %logdate;
- foreach my $l (@keys) {
- my ($d, $l) = split '\0', $l;
-
- if ((defined($logs{$l})) && (length($logs{$l} > 0))) {
- print F "----------------------------------------\n";
- print F "$logs{$l}\n";
- print F "$l\n";
- undef $logs{$l};
- }
- }
- close(F);
- }
-
- print STDERR "$thisdate checked out!\n";
-}
sub buildKmer ($) {
diff --git a/src/AS_TER/._Makefile b/src/AS_TER/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._Makefile differ
diff --git a/src/AS_TER/._analyzePosMap-gapFillProbability.C b/src/AS_TER/._analyzePosMap-gapFillProbability.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._analyzePosMap-gapFillProbability.C differ
diff --git a/src/AS_TER/._analyzePosMap-libraryFate.C b/src/AS_TER/._analyzePosMap-libraryFate.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._analyzePosMap-libraryFate.C differ
diff --git a/src/AS_TER/._analyzePosMap-load.C b/src/AS_TER/._analyzePosMap-load.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._analyzePosMap-load.C differ
diff --git a/src/AS_TER/._analyzePosMap.C b/src/AS_TER/._analyzePosMap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._analyzePosMap.C differ
diff --git a/src/AS_TER/._analyzePosMap.H b/src/AS_TER/._analyzePosMap.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._analyzePosMap.H differ
diff --git a/src/AS_TER/._asmOutputFasta.C b/src/AS_TER/._asmOutputFasta.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._asmOutputFasta.C differ
diff --git a/src/AS_TER/._asmOutputStatistics.C b/src/AS_TER/._asmOutputStatistics.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._asmOutputStatistics.C differ
diff --git a/src/AS_TER/._buildPosMap.C b/src/AS_TER/._buildPosMap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._buildPosMap.C differ
diff --git a/src/AS_TER/._demotePosMap.C b/src/AS_TER/._demotePosMap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._demotePosMap.C differ
diff --git a/src/AS_TER/._fragsInVars.C b/src/AS_TER/._fragsInVars.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._fragsInVars.C differ
diff --git a/src/AS_TER/._posmap-to-gff3.pl b/src/AS_TER/._posmap-to-gff3.pl
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._posmap-to-gff3.pl differ
diff --git a/src/AS_TER/._terminator.C b/src/AS_TER/._terminator.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_TER/._terminator.C differ
diff --git a/src/AS_TER/analyzePosMap.H b/src/AS_TER/analyzePosMap.H
index 7565f4c..9fbf4eb 100644
--- a/src/AS_TER/analyzePosMap.H
+++ b/src/AS_TER/analyzePosMap.H
@@ -23,7 +23,7 @@
#ifndef ANALYZEPOSMAP_H
#define ANALYZEPOSMAP_H
-static const char *rcsid_ANALYZEPOSMAP_H = "$Id: analyzePosMap.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_ANALYZEPOSMAP_H = "$Id: analyzePosMap.H 4571 2014-10-09 13:18:54Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -34,8 +34,8 @@ static const char *rcsid_ANALYZEPOSMAP_H = "$Id: analyzePosMap.H 4371 2013-08-01
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_splitToWords.H"
-#include "AS_UTL_intervalList.H"
+#include "splitToWords.H"
+#include "intervalList.H"
#include <map>
#include <vector>
diff --git a/src/AS_TER/buildPosMap.C b/src/AS_TER/buildPosMap.C
index 1e675bd..6a70850 100644
--- a/src/AS_TER/buildPosMap.C
+++ b/src/AS_TER/buildPosMap.C
@@ -20,7 +20,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: buildPosMap.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: buildPosMap.C 4595 2015-02-03 02:52:44Z brianwalenz $";
#include <stdio.h>
#include <stdlib.h>
@@ -31,7 +31,7 @@ const char *mainid = "$Id: buildPosMap.C 4371 2013-08-01 17:19:47Z brianwalenz $
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
#include <vector>
#include <set>
@@ -156,9 +156,16 @@ processMDI(SnapMateDistMesg *mdi, char *prefix) {
double bgn = mdi->min;
double width = (double)(mdi->max - mdi->min) / mdi->num_buckets;
+ char S[FILENAME_MAX];
char N[FILENAME_MAX];
- sprintf(N, "%s.posmap.libraries.%s", prefix, AS_UID_toString(mdi->erefines));
+ sprintf(S, "posmap.libraries.%s", AS_UID_toString(mdi->erefines));
+
+ for (uint32 ii=0; S[ii]; ii++)
+ if (S[ii] == '/')
+ S[ii] = '_';
+
+ sprintf(N, "%s.%s", prefix, S);
errno = 0;
FILE *F = fopen(N, "w");
diff --git a/src/AS_TER/demotePosMap.C b/src/AS_TER/demotePosMap.C
index 9dc4940..6632567 100644
--- a/src/AS_TER/demotePosMap.C
+++ b/src/AS_TER/demotePosMap.C
@@ -20,12 +20,12 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-const char *mainid = "$Id: demotePosMap.C 4371 2013-08-01 17:19:47Z brianwalenz $";
+const char *mainid = "$Id: demotePosMap.C 4571 2014-10-09 13:18:54Z brianwalenz $";
#include "AS_global.H"
#include "AS_PER_gkpStore.H"
-#include "AS_UTL_splitToWords.H"
+#include "splitToWords.H"
#include <string>
#include <vector>
diff --git a/src/AS_TER/posmap-to-gff3.pl b/src/AS_TER/posmap-to-gff3.pl
new file mode 100644
index 0000000..3b97d05
--- /dev/null
+++ b/src/AS_TER/posmap-to-gff3.pl
@@ -0,0 +1,109 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $dir = "asm/9-terminator";
+my $asm = "asm";
+
+my %scfOfInterest;
+my %ctgOfInterest;
+my %utgOfInterest;
+
+$scfOfInterest{"scf7180002220303"}++;
+$scfOfInterest{"scf7180002224952"}++;
+$scfOfInterest{"scf7180002258291"}++;
+$scfOfInterest{"scf7180002259533"}++;
+$scfOfInterest{"scf7180002280184"}++;
+$scfOfInterest{"scf7180002290117"}++;
+$scfOfInterest{"scf7180002310267"}++;
+$scfOfInterest{"scf7180002311635"}++;
+
+print "#gff-version 3\n";
+
+open(F, "< $dir/$asm.posmap.scflen") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ if ((scalar(keys %scfOfInterest) > 0) && (exists($scfOfInterest{"scf$v[0]"}))) {
+ printf "scf$v[0]\t$asm\tca_scaffold\t1\t$v[1]\t.\t.\t.\tID=scf$v[0];Name=scf$v[0]\n";
+ }
+}
+close(F);
+
+
+open(F, "< $dir/$asm.posmap.ctgscf") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ if ((scalar(keys %scfOfInterest) > 0) && (exists($scfOfInterest{"scf$v[1]"}))) {
+ $v[2]++;
+ $v[4] = "+" if ($v[4] eq "f");
+ $v[4] = "-" if ($v[4] eq "r");
+
+ print "scf$v[1]\t$asm\tca_contig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tID=ctg$v[0];Name=ctg$v[0];Parent=scf$v[1]\n";
+
+ $ctgOfInterest{"ctg$v[0]"}++;
+ }
+}
+close(F);
+
+# LOAD UNITIG MARKINGS. This is a temporary hack until posmap is updated.
+my %utgStatus;
+open(F, "< $dir/$asm.utgstatus") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ $utgStatus{$v[1]} = $v[8];
+}
+close(F);
+
+
+open(F, "< $dir/$asm.posmap.utgscf") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ if ((scalar(keys %scfOfInterest) > 0) && (exists($scfOfInterest{"scf$v[1]"}))) {
+ $v[2]++;
+ $v[4] = "+" if ($v[4] eq "f");
+ $v[4] = "-" if ($v[4] eq "r");
+
+ #print "scf$v[1]\t$asm\tca_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tID=utg$v[0];Name=utg$v[0]\n";
+
+ if ($utgStatus{$v[0]} eq "N") {
+ # Unplaced, degenerate
+ #print "scf$v[1]\t$asm\tca_degenerate_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tParent=utg$v[0]\n";
+ print "scf$v[1]\t$asm\tca_degenerate_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tID=deg$v[0]\n";
+ } elsif ($utgStatus{$v[0]} eq "U") {
+ # Placed, unique
+ #print "scf$v[1]\t$asm\tca_unique_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tParent=utg$v[0]\n";
+ print "scf$v[1]\t$asm\tca_unique_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tID=deg$v[0]\n";
+ } elsif ($utgStatus{$v[0]} eq "S") {
+ # Placed, surrogate
+ #print "scf$v[1]\t$asm\tca_surrogate_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tParent=utg$v[0]\n";
+ print "scf$v[1]\t$asm\tca_surrogate_unitig\t$v[2]\t$v[3]\t.\t$v[4]\t.\tID=deg$v[0]\n";
+ } else {
+ die "Unknown unitig status $_\n";
+ }
+ }
+}
+close(F);
+
+
+open(F, "< $dir/$asm.posmap.frgscf") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ if ((scalar(keys %scfOfInterest) > 0) && (exists($scfOfInterest{"scf$v[1]"}))) {
+ $v[2]++;
+ $v[4] = "+" if ($v[4] eq "f");
+ $v[4] = "-" if ($v[4] eq "r");
+
+ print "scf$v[1]\t$asm\tca_fragment\t$v[2]\t$v[3]\t.\t$v[4]\t.\tID=frg$v[0];Name=frg$v[0]\n";
+ }
+}
+close(F);
+
+
+open(F, "< $dir/$asm.posmap.varscf") or die;
+while (<F>) {
+ my @v = split '\s+', $_;
+ if ((scalar(keys %scfOfInterest) > 0) && (exists($scfOfInterest{"scf$v[1]"}))) {
+ print "scf$v[1]\t$asm\tca_variant\t$v[2]\t$v[3]\t.\t.\t.\n";
+ }
+}
+close(F);
diff --git a/src/AS_UID/._Makefile b/src/AS_UID/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._Makefile differ
diff --git a/src/AS_UID/._SYS_UIDclient.H b/src/AS_UID/._SYS_UIDclient.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._SYS_UIDclient.H differ
diff --git a/src/AS_UID/._SYS_UIDclient_JTC.C b/src/AS_UID/._SYS_UIDclient_JTC.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._SYS_UIDclient_JTC.C differ
diff --git a/src/AS_UID/._SYS_UIDclient_LOCAL.C b/src/AS_UID/._SYS_UIDclient_LOCAL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._SYS_UIDclient_LOCAL.C differ
diff --git a/src/AS_UID/._SYS_UIDclient_SERVER.C b/src/AS_UID/._SYS_UIDclient_SERVER.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._SYS_UIDclient_SERVER.C differ
diff --git a/src/AS_UID/._uidclient.C b/src/AS_UID/._uidclient.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._uidclient.C differ
diff --git a/src/AS_UID/._uidserver.C b/src/AS_UID/._uidserver.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._uidserver.C differ
diff --git a/src/AS_UID/._uidserver_common.H b/src/AS_UID/._uidserver_common.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UID/._uidserver_common.H differ
diff --git a/src/AS_UTL/._AS_UTL_GPL.C b/src/AS_UTL/._AS_UTL_GPL.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_GPL.C differ
diff --git a/src/AS_UTL/._AS_UTL_GPL.H b/src/AS_UTL/._AS_UTL_GPL.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_GPL.H differ
diff --git a/src/AS_UTL/._AS_UTL_Hash.C b/src/AS_UTL/._AS_UTL_Hash.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_Hash.C differ
diff --git a/src/AS_UTL/._AS_UTL_Hash.H b/src/AS_UTL/._AS_UTL_Hash.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_Hash.H differ
diff --git a/src/AS_UTL/._AS_UTL_IID.H b/src/AS_UTL/._AS_UTL_IID.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_IID.H differ
diff --git a/src/AS_UTL/._AS_UTL_UID.C b/src/AS_UTL/._AS_UTL_UID.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_UID.C differ
diff --git a/src/AS_UTL/._AS_UTL_UID.H b/src/AS_UTL/._AS_UTL_UID.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_UID.H differ
diff --git a/src/AS_UTL/._AS_UTL_Var.C b/src/AS_UTL/._AS_UTL_Var.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_Var.C differ
diff --git a/src/AS_UTL/._AS_UTL_Var.H b/src/AS_UTL/._AS_UTL_Var.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_Var.H differ
diff --git a/src/AS_UTL/._AS_UTL_alloc.C b/src/AS_UTL/._AS_UTL_alloc.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_alloc.C differ
diff --git a/src/AS_UTL/._AS_UTL_alloc.H b/src/AS_UTL/._AS_UTL_alloc.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_alloc.H differ
diff --git a/src/AS_UTL/._AS_UTL_decodeRange.C b/src/AS_UTL/._AS_UTL_decodeRange.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_decodeRange.C differ
diff --git a/src/AS_UTL/._AS_UTL_decodeRange.H b/src/AS_UTL/._AS_UTL_decodeRange.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_decodeRange.H differ
diff --git a/src/AS_UTL/._AS_UTL_extractMSG b/src/AS_UTL/._AS_UTL_extractMSG
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_extractMSG differ
diff --git a/src/AS_UTL/._AS_UTL_extractMSG.awk b/src/AS_UTL/._AS_UTL_extractMSG.awk
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_extractMSG.awk differ
diff --git a/src/AS_UTL/._AS_UTL_fasta.C b/src/AS_UTL/._AS_UTL_fasta.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_fasta.C differ
diff --git a/src/AS_UTL/._AS_UTL_fasta.H b/src/AS_UTL/._AS_UTL_fasta.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_fasta.H differ
diff --git a/src/AS_UTL/._AS_UTL_fileIO.C b/src/AS_UTL/._AS_UTL_fileIO.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_fileIO.C differ
diff --git a/src/AS_UTL/._AS_UTL_fileIO.H b/src/AS_UTL/._AS_UTL_fileIO.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_fileIO.H differ
diff --git a/src/AS_UTL/._AS_UTL_heap.C b/src/AS_UTL/._AS_UTL_heap.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_heap.C differ
diff --git a/src/AS_UTL/._AS_UTL_heap.H b/src/AS_UTL/._AS_UTL_heap.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_heap.H differ
diff --git a/src/AS_UTL/._AS_UTL_histo.C b/src/AS_UTL/._AS_UTL_histo.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_histo.C differ
diff --git a/src/AS_UTL/._AS_UTL_histo.H b/src/AS_UTL/._AS_UTL_histo.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_histo.H differ
diff --git a/src/AS_UTL/._AS_UTL_interval.C b/src/AS_UTL/._AS_UTL_interval.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_interval.C differ
diff --git a/src/AS_UTL/._AS_UTL_interval.H b/src/AS_UTL/._AS_UTL_interval.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_interval.H differ
diff --git a/src/AS_UTL/._AS_UTL_param_proc.C b/src/AS_UTL/._AS_UTL_param_proc.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_param_proc.C differ
diff --git a/src/AS_UTL/._AS_UTL_param_proc.H b/src/AS_UTL/._AS_UTL_param_proc.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_param_proc.H differ
diff --git a/src/AS_UTL/._AS_UTL_rand.C b/src/AS_UTL/._AS_UTL_rand.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_rand.C differ
diff --git a/src/AS_UTL/._AS_UTL_rand.H b/src/AS_UTL/._AS_UTL_rand.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_rand.H differ
diff --git a/src/AS_UTL/._AS_UTL_reverseComplement.C b/src/AS_UTL/._AS_UTL_reverseComplement.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_reverseComplement.C differ
diff --git a/src/AS_UTL/._AS_UTL_reverseComplement.H b/src/AS_UTL/._AS_UTL_reverseComplement.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_reverseComplement.H differ
diff --git a/src/AS_UTL/._AS_UTL_skiplist.C b/src/AS_UTL/._AS_UTL_skiplist.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_skiplist.C differ
diff --git a/src/AS_UTL/._AS_UTL_skiplist.H b/src/AS_UTL/._AS_UTL_skiplist.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_skiplist.H differ
diff --git a/src/AS_UTL/._AS_UTL_skiplist_test.C b/src/AS_UTL/._AS_UTL_skiplist_test.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_skiplist_test.C differ
diff --git a/src/AS_UTL/._AS_UTL_stackTrace.C b/src/AS_UTL/._AS_UTL_stackTrace.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_stackTrace.C differ
diff --git a/src/AS_UTL/._AS_UTL_stackTrace.H b/src/AS_UTL/._AS_UTL_stackTrace.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_stackTrace.H differ
diff --git a/src/AS_UTL/._AS_UTL_stackTraceTest.C b/src/AS_UTL/._AS_UTL_stackTraceTest.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_stackTraceTest.C differ
diff --git a/src/AS_UTL/._AS_UTL_testhisto.C b/src/AS_UTL/._AS_UTL_testhisto.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._AS_UTL_testhisto.C differ
diff --git a/src/AS_UTL/._Makefile b/src/AS_UTL/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._Makefile differ
diff --git a/src/AS_UTL/._UnionFind_AS.C b/src/AS_UTL/._UnionFind_AS.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._UnionFind_AS.C differ
diff --git a/src/AS_UTL/._UnionFind_AS.H b/src/AS_UTL/._UnionFind_AS.H
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._UnionFind_AS.H differ
diff --git a/src/AS_UTL/._extractMSG b/src/AS_UTL/._extractMSG
new file mode 100755
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._extractMSG differ
diff --git a/src/AS_UTL/._testHashTable.C b/src/AS_UTL/._testHashTable.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._testHashTable.C differ
diff --git a/src/AS_UTL/._testRand.C b/src/AS_UTL/._testRand.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._testRand.C differ
diff --git a/src/AS_UTL/._testVar.C b/src/AS_UTL/._testVar.C
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_UTL/._testVar.C differ
diff --git a/src/AS_UTL/AS_UTL_alloc.H b/src/AS_UTL/AS_UTL_alloc.H
index 4715a7f..458ecea 100644
--- a/src/AS_UTL/AS_UTL_alloc.H
+++ b/src/AS_UTL/AS_UTL_alloc.H
@@ -22,7 +22,7 @@
#ifndef AS_UTL_ALLOC_H
#define AS_UTL_ALLOC_H
-static const char *rcsid_AS_UTL_ALLOC_H = "$Id: AS_UTL_alloc.H 4371 2013-08-01 17:19:47Z brianwalenz $";
+static const char *rcsid_AS_UTL_ALLOC_H = "$Id: AS_UTL_alloc.H 4557 2014-08-11 12:24:27Z brianwalenz $";
//
// The safe_*alloc routines are the same as the normal routines,
@@ -50,6 +50,9 @@ void safe_free2(void *);
// implemented as a macro that calls free(), which then gets expanded
// into our bogus function.
//
+// And, thanks, Apple. Your iostream implementation uses malloc. (!)
+//
+#ifndef BROKEN_CLANG_OpenMP
#ifndef X86_GCC_LINUX
#define malloc(X) use_safe_malloc_instead(X)
@@ -58,6 +61,7 @@ void safe_free2(void *);
#define free(X) use_safe_free_instead(X)
#endif
+#endif
template<class T>
inline void zero(T* ptr, size_t num)
diff --git a/src/AS_UTL/AS_UTL_fileIO.C b/src/AS_UTL/AS_UTL_fileIO.C
index 96cafeb..2da5f8b 100644
--- a/src/AS_UTL/AS_UTL_fileIO.C
+++ b/src/AS_UTL/AS_UTL_fileIO.C
@@ -19,7 +19,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*************************************************************************/
-//static char *rcsid = "$Id: AS_UTL_fileIO.C 4436 2013-09-27 20:26:53Z brianwalenz $";
+//static char *rcsid = "$Id: AS_UTL_fileIO.C 4592 2015-02-02 22:07:32Z brianwalenz $";
#include "AS_UTL_fileIO.H"
@@ -64,6 +64,7 @@ AS_UTL_safeWrite(FILE *file, const void *buffer, const char *desc, size_t size,
fprintf(stderr, "safeWrite()-- Write failure on %s: %s\n", desc, strerror(errno));
fprintf(stderr, "safeWrite()-- Wanted to write "F_SIZE_T" objects (size="F_SIZE_T"), wrote "F_SIZE_T".\n",
towrite, size, written);
+ exit(1);
assert(errno == 0);
}
diff --git a/src/AS_UTL/AS_UTL_intervalList.C b/src/AS_UTL/AS_UTL_intervalList.C
deleted file mode 100644
index 45c9384..0000000
--- a/src/AS_UTL/AS_UTL_intervalList.C
+++ /dev/null
@@ -1,510 +0,0 @@
-
-/**************************************************************************
- * This file is part of Celera Assembler, a software program that
- * assembles whole-genome shotgun reads into contigs and scaffolds.
- * Copyright (C) 2005-2007, J. Craig Venter Institute.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received (LICENSE.txt) a copy of the GNU General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *************************************************************************/
-
-static const char *rcsid = "$Id: AS_UTL_intervalList.C 4397 2013-08-28 12:22:18Z brianwalenz $";
-
-#include "AS_UTL_intervalList.H"
-
-
-intervalList::intervalList(uint32 initialSize) {
- _isSorted = true;
- _isMerged = true;
- _listLen = 0;
- _listMax = initialSize;
- _list = new _intervalPair [_listMax];
-}
-
-
-intervalList::~intervalList() {
- delete [] _list;
-}
-
-
-intervalList &
-intervalList::operator=(intervalList &src) {
- _isSorted = src._isSorted;
- _isMerged = src._isMerged;
- _listLen = src._listLen;
-
- if (_listMax < src._listMax) {
- delete [] _list;
- _listMax = src._listMax;
- _list = new _intervalPair [_listMax];
- }
-
- memcpy(_list, src._list, _listLen * sizeof(_intervalPair));
-
- return(*this);
-}
-
-
-void
-intervalList::add(intervalNumber position, intervalNumber length) {
-
- if (_listLen >= _listMax) {
- _listMax *= 2;
- _intervalPair *l = new _intervalPair [_listMax];
- memcpy(l, _list, sizeof(_intervalPair) * _listLen);
- delete [] _list;
- _list = l;
- }
-
- _list[_listLen].lo = position;
- _list[_listLen].hi = position + length;
- _list[_listLen].ct = 1;
-
-#if 0
- // Aborted attempt to add a data field here. Got stuck
- // deciding how to handle merges lightweight
-
- _list[_listLen].data = 0L;
-
- if (data != ~uint64ZERO) {
- _list[_listLen].dataLen = 1;
- _list[_listLen].dataMax = 4;
- _list[_listLen].data = new uint64 [_list[_listLen].dataMax];
- _list[_listLen].data[0] = data;
- }
-#endif
-
- if ((_listLen > 0) &&
- (_list[_listLen-1].lo > _list[_listLen].lo)) {
- _isSorted = false;
- _isMerged = false;
- }
-
- _listLen++;
-}
-
-
-static
-int
-intervalList_sort_helper(const void *a, const void *b) {
- _intervalPair *A = (_intervalPair *)a;
- _intervalPair *B = (_intervalPair *)b;
-
- if (A->lo < B->lo) return(-1);
- if (A->lo > B->lo) return(1);
- if (A->hi < B->hi) return(-1);
- if (A->hi > B->hi) return(1);
- return(0);
-}
-
-
-void
-intervalList::sort(void) {
-
- if (_isSorted)
- return;
-
- if (_listLen > 1)
- qsort(_list, _listLen, sizeof(_intervalPair), intervalList_sort_helper);
-
- _isSorted = true;
-}
-
-
-void
-intervalList::merge(uint32 minOverlap) {
- uint32 thisInterval = 0;
- uint32 nextInterval = 1;
-
- if (_listLen < 2)
- return;
-
- sort();
-
- while (nextInterval < _listLen) {
-
- if ((_list[thisInterval].lo == 0) &&
- (_list[thisInterval].hi == 0)) {
-
- // Our interval is empty. Copy in the interval we are
- // examining and move to the next.
-
- // XXX This is probably useless, thisInterval should always be
- // valid.
-
- _list[thisInterval].lo = _list[nextInterval].lo;
- _list[thisInterval].hi = _list[nextInterval].hi;
- _list[thisInterval].ct = _list[nextInterval].ct;
-
- _list[nextInterval].lo = 0;
- _list[nextInterval].hi = 0;
- nextInterval++;
- } else {
-
- // This interval is valid. See if it overlaps with the next
- // interval.
-
- bool intersects = false;
-
- if ((_list[thisInterval].lo <= _list[nextInterval].lo) &&
- (_list[nextInterval].hi <= _list[thisInterval].hi))
- // next is contained in this
- intersects = true;
-
- if (_list[thisInterval].hi - minOverlap >= _list[nextInterval].lo)
- // next has thick overlap to this
- intersects = true;
-
-
- if (intersects) {
-
- // Got an intersection.
-
- // Merge nextInterval into thisInterval -- the hi range
- // is extended if the nextInterval range is larger.
- //
- if (_list[thisInterval].hi < _list[nextInterval].hi)
- _list[thisInterval].hi = _list[nextInterval].hi;
-
- _list[thisInterval].ct += _list[nextInterval].ct;
-
- // Clear the just merged nextInterval and move to the next one.
- //
- _list[nextInterval].lo = 0;
- _list[nextInterval].hi = 0;
- _list[nextInterval].ct = 0;
- nextInterval++;
- } else {
-
- // No intersection. Move along. Nothing to see here.
-
- // If there is a gap between the target and the examine (we
- // must have merged sometime in the past), copy examine to
- // the next target.
-
- thisInterval++;
-
- if (thisInterval != nextInterval) {
- _list[thisInterval].lo = _list[nextInterval].lo;
- _list[thisInterval].hi = _list[nextInterval].hi;
- _list[thisInterval].ct = _list[nextInterval].ct;
- }
-
- nextInterval++;
- }
- }
- }
-
- if (thisInterval+1 < _listLen)
- _listLen = thisInterval + 1;
-
- _isMerged = true;
-}
-
-
-void
-intervalList::invert(intervalNumber lo, intervalNumber hi) {
-
- if (!_isSorted || !_isMerged) {
- fprintf(stderr, "intervalList::invert()-- ERROR! List is not sorted or not merged!\n");
- exit(1);
- }
-
- // Create a new list to store the inversion
- //
- uint32 invLen = 0;
- uint32 invMax = _listLen + 2;
- _intervalPair *inv = new _intervalPair [invMax];
-
- // Add the first
- //
- if (lo < _list[0].lo) {
- inv[invLen].lo = lo;
- inv[invLen].hi = _list[0].lo;
- invLen++;
- }
-
- // Add the pieces
- for (uint32 i=1; i<_listLen; i++) {
- if (_list[i-1].hi < _list[i].lo) {
- inv[invLen].lo = _list[i-1].hi;
- inv[invLen].hi = _list[i].lo;
- invLen++;
- }
- }
-
- // Add the last
- if (_list[_listLen-1].hi < hi) {
- inv[invLen].lo = _list[_listLen-1].hi;
- inv[invLen].hi = hi;
- invLen++;
- }
-
- // Nuke the old list, swap in the new one
- delete [] _list;
-
- _list = inv;
- _listLen = invLen;
- _listMax = invMax;
-}
-
-
-
-uint32
-intervalList::overlapping(intervalNumber rangelo,
- intervalNumber rangehi,
- uint32 *&intervals,
- uint32 &intervalsLen,
- uint32 &intervalsMax) {
-
-
- // XXX: Naive implementation that is easy to verify (and that works
- // on an unsorted list).
-
- if (intervals == 0L) {
- intervalsMax = 256;
- intervals = new uint32 [intervalsMax];
- }
-
- intervalsLen = 0;
-
- for (uint32 i=0; i<_listLen; i++) {
- if ((rangelo <= _list[i].hi) &&
- (rangehi >= _list[i].lo)) {
- if (intervalsLen >= intervalsMax) {
- intervalsMax *= 2;
- uint32 *X = new uint32 [intervalsMax];
- memcpy(X, intervals, sizeof(uint32) * intervalsLen);
- delete [] intervals;
- intervals = X;
- }
-
- intervals[intervalsLen++] = i;
- }
- }
-
- return(intervalsLen);
-}
-
-
-void
-intervalList::merge(intervalList *IL) {
- //bool isSorted = _isSorted;
- //bool isMerged = _isMerged;
-
- for (uint32 i=0; i<IL->_listLen; i++)
- add(IL->_list[i].lo, IL->_list[i].hi - IL->_list[i].lo);
-
- //if (isSorted) sort();
- //if (isMerged) merge();
-}
-
-
-void
-intervalList::intersect(intervalList &A,
- intervalList &B) {
- A.merge();
- B.merge();
-
- uint32 ai = 0;
- uint32 bi = 0;
-
- while ((ai < A.numberOfIntervals()) &&
- (bi < B.numberOfIntervals())) {
- uint32 al = A.lo(ai);
- uint32 ah = A.hi(ai);
- uint32 bl = B.lo(bi);
- uint32 bh = B.hi(bi);
- uint32 nl = 0;
- uint32 nh = 0;
-
- // If they intersect, make a new region
- //
- if ((al <= bl) && (bl < ah)) {
- nl = bl;
- nh = (ah < bh) ? ah : bh;
- }
-
- if ((bl <= al) && (al < bh)) {
- nl = al;
- nh = (ah < bh) ? ah : bh;
- }
-
- if (nl < nh)
- add(nl, nh - nl);
-
- // Advance the list with the earlier region.
- //
- if (ah < bh) {
- // A ends before B
- ai++;
- } else if (ah > bh) {
- // B ends before A
- bi++;
- } else {
- // Exactly the same ending!
- ai++;
- bi++;
- }
- }
-}
-
-void
-intervalList::contained(intervalList &A,
- intervalList &B) {
- A.merge();
- B.merge();
-
- uint32 ai = 0;
- uint32 bi = 0;
-
- while ((ai < A.numberOfIntervals()) &&
- (bi < B.numberOfIntervals())) {
- uint32 al = A.lo(ai);
- uint32 ah = A.hi(ai);
- uint32 bl = B.lo(bi);
- uint32 bh = B.hi(bi);
-
- // If A is contained in B, make a new region.
- //
- if ((bl <= al) && (ah <= bh))
- add(bl, bh - bl);
-
-#if 0
- if ((al <= bl) && (bh <= ah))
- add(al, ah - al);
-#endif
-
- // Advance the list with the earlier region.
- //
- if (ah < bh) {
- // A ends before B
- ai++;
- } else if (ah > bh) {
- // B ends before A
- bi++;
- } else {
- // Exactly the same ending!
- ai++;
- bi++;
- }
- }
-}
-
-
-
-
-
-
-static
-int
-intervalDepth_sort_helper(const void *a, const void *b) {
- intervalDepthRegions *A = (intervalDepthRegions *)a;
- intervalDepthRegions *B = (intervalDepthRegions *)b;
-
- if (A->pos < B->pos) return(-1);
- if (A->pos > B->pos) return(1);
- return(0);
-}
-
-
-intervalDepth::intervalDepth(intervalList &IL) {
- uint32 idlen = IL.numberOfIntervals() * 2;
- intervalDepthRegions *id = new intervalDepthRegions [idlen];
-
- for (uint32 i=0; i<IL.numberOfIntervals(); i++) {
- id[2*i ].pos = IL.lo(i);
- id[2*i ].cha = 1;
- id[2*i+1].pos = IL.hi(i);
- id[2*i+1].cha = -1;
- }
-
- qsort(id, idlen, sizeof(intervalDepthRegions), intervalDepth_sort_helper);
- computeIntervals(id, idlen);
-
- delete [] id;
-}
-
-
-intervalDepth::intervalDepth(intervalDepthRegions *id, uint32 idlen) {
- qsort(id, idlen, sizeof(intervalDepthRegions), intervalDepth_sort_helper);
- computeIntervals(id, idlen);
-}
-
-
-void
-intervalDepth::computeIntervals(intervalDepthRegions *id, uint32 idlen) {
-
- // No intervals input? No intervals output.
-
- if (idlen == 0) {
- _listLen = 0;
- _listMax = 0;
- _list = NULL;
-
- return;
- }
-
- // Scan the list, counting how many times we change depth.
-
- _listMax = 1;
- for (uint32 i=1; i<idlen; i++) {
- if (id[i-1].pos != id[i].pos)
- _listMax++;
- }
-
- // Allocate the real depth of coverage intervals
-
- _listLen = 0;
- _list = new _intervalDepth [_listMax];
-
- // Build new intervals
-
- // Init first interval.
- _list[_listLen].lo = id[0].pos;
- _list[_listLen].hi = id[0].pos;
- _list[_listLen].de = id[0].cha;
-
- for (uint32 i=1; i<idlen; i++) {
- // Update the end of the current interval.
- _list[_listLen].hi = id[i].pos;
-
- //fprintf(stderr, "ID %d %d - cur %d %d-%d %d\n",
- // id[i].pos, id[i].cha,
- // _listLen, _list[_listLen].lo, _list[_listLen].hi, _list[_listLen].de);
-
- // If the position is different than the last, make a new interval
- if (id[i-1].pos != id[i].pos) {
- _listLen++;
-
- _list[_listLen].lo = id[i].pos;
- _list[_listLen].de = _list[_listLen-1].de;
- }
-
- // Finally, update the depth and end of the current interval
- _list[_listLen].hi = id[i].pos;
- _list[_listLen].de += id[i].cha;
-
- //fprintf(stderr, "ID %d %d - cur %d %d-%d %d POST\n",
- // id[i].pos, id[i].cha,
- // _listLen, _list[_listLen].lo, _list[_listLen].hi, _list[_listLen].de);
- }
-
- assert(_listLen > 0);
- assert(_listLen < _listMax);
-}
-
-intervalDepth::~intervalDepth() {
- delete [] _list;
-}
diff --git a/src/AS_UTL/AS_UTL_intervalList.H b/src/AS_UTL/AS_UTL_intervalList.H
deleted file mode 100644
index f75710d..0000000
--- a/src/AS_UTL/AS_UTL_intervalList.H
+++ /dev/null
@@ -1,173 +0,0 @@
-
-/**************************************************************************
- * This file is part of Celera Assembler, a software program that
- * assembles whole-genome shotgun reads into contigs and scaffolds.
- * Copyright (C) 2005-2007, J. Craig Venter Institute.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received (LICENSE.txt) a copy of the GNU General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *************************************************************************/
-
-#ifndef AS_UTL_INTERVALLIST_H
-#define AS_UTL_INTERVALLIST_H
-
-#include "AS_global.H"
-
-// This is kmer/libutil/intervalList.[CH], replacing u32bit with uint32.
-
-
-// I don't want this to be public, but the sort function
-// in intervalList.C needs it so.
-//
-typedef int64 intervalNumber;
-
-struct _intervalPair {
- intervalNumber lo;
- intervalNumber hi;
- uint32 ct;
-};
-
-struct _intervalDepth {
- intervalNumber lo;
- intervalNumber hi;
- uint32 de;
-};
-
-struct intervalDepthRegions {
- intervalNumber pos; // Position of the change in depth
- int32 cha; // The change in depth, e.g, +1, -1
-};
-
-
-class intervalList {
-public:
- intervalList(uint32 initialSize=32);
- ~intervalList();
-
- intervalList &operator=(intervalList &src);
-
- // Clear a list
- void clear(void) {
- _isSorted = true;
- _isMerged = true;
- _listLen = 0;
- }
-
- // Insert a new interval into the list
- void add(intervalNumber position, intervalNumber length);
-
- // Sort the set of intervals by the lo value
- void sort(void);
-
- // Merge overlapping or adjacent intervals together.
- void merge(uint32 minOverlap=0);
-
- void invert(intervalNumber lo, intervalNumber hi);
-
- // Returns the number of intervals
- uint32 numberOfIntervals(void) {
- return(_listLen);
- };
-
- // Returns the sum of the length of all intervals
- intervalNumber sumOfLengths(void) {
- intervalNumber len = 0;
- uint32 i = numberOfIntervals();
-
- if (i > 0)
- while (i--)
- len += _list[i].hi - _list[i].lo;
-
- return(len);
- };
-
-
- // Populates an array with the intervals that are within the
- // supplied interval. Return
- //
- uint32 overlapping(intervalNumber lo,
- intervalNumber hi,
- uint32 *&intervals,
- uint32 &intervalsLen,
- uint32 &intervalsMax);
-
- // Adds all the intervals in IL to this list
- //
- void merge(intervalList *IL);
-
- // Populates this intervalList with the intersection of A and B.
- // This intervalList is not cleared prior to adding new intervals.
- //
- // Both A and B call merge().
- //
- void intersect(intervalList &A,
- intervalList &B);
-
- // Populates this intervalList with regions in A that are completely
- // contained in a region in B.
- //
- // Both A and B call merge().
- //
- void contained(intervalList &A,
- intervalList &B);
-
-
- intervalNumber &lo(uint32 i) { return(_list[i].lo); };
- intervalNumber &hi(uint32 i) { return(_list[i].hi); };
- uint32 &ct(uint32 i) { return(_list[i].ct); };
-
-private:
- bool _isSorted;
- bool _isMerged;
- uint32 _listLen;
- uint32 _listMax;
- _intervalPair *_list;
-};
-
-
-
-// Takes as input an intervalList, computes the number of intervals
-// covering every position in there, stores this as a new set of
-// intervals, annotated with the depth.
-//
-// This is a static object, initialized once by the intervalList.
-//
-class intervalDepth {
-public:
- intervalDepth(intervalList &IL);
- intervalDepth(intervalDepthRegions *id, uint32 idlen);
- ~intervalDepth();
-
- // Returns the number of intervals
- uint32 numberOfIntervals(void) {
- return(_listLen);
- };
-
- intervalNumber lo(uint32 i) { return(_list[i].lo); };
- intervalNumber hi(uint32 i) { return(_list[i].hi); };
- uint32 de(uint32 i) { return(_list[i].de); };
-
-private:
- void computeIntervals(intervalDepthRegions *id, uint32 idlen);
-
- uint32 _listLen;
- uint32 _listMax;
- _intervalDepth *_list;
-};
-
-
-
-
-
-#endif // AS_UTL_INTERVALLIST_H
diff --git a/src/AS_UTL/AS_UTL_intervalListTest.C b/src/AS_UTL/AS_UTL_intervalListTest.C
deleted file mode 100644
index 4824dcd..0000000
--- a/src/AS_UTL/AS_UTL_intervalListTest.C
+++ /dev/null
@@ -1,63 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <inttypes.h>
-#include <limits.h>
-#include <unistd.h>
-#include <string.h>
-#include <ctype.h>
-
-#include <float.h>
-#include <math.h>
-
-#include <assert.h>
-#include <errno.h>
-#include <time.h>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-typedef int8_t int8;
-typedef int16_t int16;
-typedef int32_t int32;
-typedef int64_t int64;
-
-typedef uint8_t uint8;
-typedef uint16_t uint16;
-typedef uint32_t uint32;
-typedef uint64_t uint64;
-
-#include "AS_UTL_intervalList.H"
-#include "AS_UTL_intervalList.C"
-
-int
-main(int argc, char **argv) {
- intervalList L;
-
- L.add(0,10);
- L.add(1,10);
- L.add(2,10);
- L.add(3,10);
-
- L.add(20,20);
- L.add(20,20);
- L.add(20,20);
-
- L.add(60,0);
-
- L.add(60,20);
-
- for (uint32 dd=0; dd<L.numberOfIntervals(); dd++)
- fprintf(stderr, "inter[%d] %d-%d %d\n",
- dd, L.lo(dd), L.hi(dd), L.ct(dd));
-
- intervalDepth D(L);
-
- for (uint32 dd=0; dd<D.numberOfIntervals(); dd++)
- fprintf(stderr, "depth[%d] %d-%d %d\n",
- dd, D.lo(dd), D.hi(dd), D.de(dd));
-
- return(0);
-}
-
diff --git a/src/AS_UTL/AS_UTL_splitToWords.H b/src/AS_UTL/AS_UTL_splitToWords.H
deleted file mode 100644
index 9aa9249..0000000
--- a/src/AS_UTL/AS_UTL_splitToWords.H
+++ /dev/null
@@ -1,139 +0,0 @@
-
-/**************************************************************************
- * This file is part of Celera Assembler, a software program that
- * assembles whole-genome shotgun reads into contigs and scaffolds.
- * Copyright (C) 2005-2007, J. Craig Venter Institute.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received (LICENSE.txt) a copy of the GNU General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *************************************************************************/
-
-#ifndef AS_UTL_SPLITTOWORDS
-#define AS_UTL_SPLITTOWORDS
-
-#include "AS_global.H"
-
-class splitToWords {
-public:
- splitToWords() {
- _argWords = 0;
- _maxWords = 0;
- _arg = 0L;
- _maxChars = 0;
- _cmd = 0L;
- };
- splitToWords(char *cmd) {
- _argWords = 0;
- _maxWords = 0;
- _arg = 0L;
- _maxChars = 0;
- _cmd = 0L;
-
- split(cmd);
- };
- ~splitToWords() {
- delete [] _cmd;
- delete [] _arg;
- };
-
-
- void split(char *cmd) {
-
- // Step Zero:
- //
- // Count the length of the string, in words and in characters.
- // For simplicity, we overcount words, by just counting white-space.
- //
- // Then, allocate space for a temporary copy of the string, and a
- // set of pointers into the temporary copy (much like argv).
- //
- uint32 cmdChars = 1; // 1 == Space for terminating 0
- uint32 cmdWords = 2; // 2 == Space for first word and terminating 0L
-
- for (char *tmp=cmd; *tmp; tmp++) {
- cmdWords += *tmp == ' ';
- cmdWords += *tmp == '\t';
- cmdChars++;
- }
-
- if (cmdChars > _maxChars) {
- delete [] _cmd;
- _cmd = new char [cmdChars];
- _maxChars = cmdChars;
- }
- if (cmdWords > _maxWords) {
- delete [] _arg;
- _arg = new char * [cmdWords];
- _maxWords = cmdWords;
- }
-
- _argWords = 0;
-
- // Step One:
- //
- // Determine where the words are in the command string, copying the
- // string to _cmd and storing words in _arg.
- //
- bool isFirst = true;
- char *cmdI = cmd;
- char *cmdO = _cmd;
-
- while (*cmdI) {
-
- // If we are at a non-space character, we are in a word. If
- // this is the first character in the word, save the word in
- // the args list.
- //
- // Otherwise we are at a space and thus not in a word. Make
- // all spaces be string terminators, and declare that we are
- // at the start of a word.
- //
- if ((*cmdI != ' ') && (*cmdI != '\t')) {
- *cmdO = *cmdI;
-
- if (isFirst) {
- _arg[_argWords++] = cmdO;
- isFirst = false;
- }
- } else {
- *cmdO = 0;
- isFirst = true;
- }
-
- cmdI++;
- cmdO++;
- }
-
- // Finish off the list by terminating the last arg, and
- // terminating the list of args.
- //
- *cmdO = 0;
- _arg[_argWords] = 0L;
- };
-
-
- uint32 numWords(void) { return(_argWords); };
- char *getWord(uint32 i) { return(_arg[i]); };
- char *operator[](uint32 i) { return(_arg[i]); };
- int64 operator()(uint32 i) { return(strtoull(_arg[i], NULL, 10)); };
-private:
- uint32 _argWords;
- uint32 _maxWords;
- char **_arg;
- uint32 _maxChars;
- char *_cmd;
-};
-
-
-#endif // AS_UTL_SPLITTOWORDS
diff --git a/src/AS_UTL/Makefile b/src/AS_UTL/Makefile
index 96fa18a..68ce2ba 100644
--- a/src/AS_UTL/Makefile
+++ b/src/AS_UTL/Makefile
@@ -36,7 +36,6 @@ LIB_SOURCES = AS_UTL_Hash.C \
AS_UTL_fasta.C \
AS_UTL_UID.C \
AS_UTL_reverseComplement.C \
- AS_UTL_intervalList.C \
AS_UTL_decodeRange.C \
AS_UTL_stackTrace.C
diff --git a/src/AS_VWR/._Makefile b/src/AS_VWR/._Makefile
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._Makefile differ
diff --git a/src/AS_VWR/._agrep.c b/src/AS_VWR/._agrep.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._agrep.c differ
diff --git a/src/AS_VWR/._agrep.h b/src/AS_VWR/._agrep.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._agrep.h differ
diff --git a/src/AS_VWR/._button.c b/src/AS_VWR/._button.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._button.c differ
diff --git a/src/AS_VWR/._button.h b/src/AS_VWR/._button.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._button.h differ
diff --git a/src/AS_VWR/._celagram.c b/src/AS_VWR/._celagram.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._celagram.c differ
diff --git a/src/AS_VWR/._celamy.c b/src/AS_VWR/._celamy.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._celamy.c differ
diff --git a/src/AS_VWR/._gendata.c b/src/AS_VWR/._gendata.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._gendata.c differ
diff --git a/src/AS_VWR/._hash.c b/src/AS_VWR/._hash.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._hash.c differ
diff --git a/src/AS_VWR/._hash.h b/src/AS_VWR/._hash.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._hash.h differ
diff --git a/src/AS_VWR/._layout.c b/src/AS_VWR/._layout.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._layout.c differ
diff --git a/src/AS_VWR/._layout.h b/src/AS_VWR/._layout.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._layout.h differ
diff --git a/src/AS_VWR/._layout_uint.c b/src/AS_VWR/._layout_uint.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._layout_uint.c differ
diff --git a/src/AS_VWR/._nata1 b/src/AS_VWR/._nata1
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._nata1 differ
diff --git a/src/AS_VWR/._nata2.5 b/src/AS_VWR/._nata2.5
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._nata2.5 differ
diff --git a/src/AS_VWR/._nata2.7 b/src/AS_VWR/._nata2.7
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._nata2.7 differ
diff --git a/src/AS_VWR/._nata3 b/src/AS_VWR/._nata3
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._nata3 differ
diff --git a/src/AS_VWR/._nata4 b/src/AS_VWR/._nata4
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._nata4 differ
diff --git a/src/AS_VWR/._query_celamy.c b/src/AS_VWR/._query_celamy.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._query_celamy.c differ
diff --git a/src/AS_VWR/._wpack.c b/src/AS_VWR/._wpack.c
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._wpack.c differ
diff --git a/src/AS_VWR/._wpack.h b/src/AS_VWR/._wpack.h
new file mode 100644
index 0000000..313e184
Binary files /dev/null and b/src/AS_VWR/._wpack.h differ
diff --git a/src/AS_global.C b/src/AS_global.C
index 89e97c7..055d121 100644
--- a/src/AS_global.C
+++ b/src/AS_global.C
@@ -53,7 +53,7 @@ uint32 AS_OVERLAP_MIN_LEN = 40;
extern
const char *mainid;
-const char *releaseid = "CA 8.1";
+const char *releaseid = "8.3rc2";
// We take argc and argv, so, maybe, eventually, we'll want to parse
// something out of there. We return argc in case what we parse we
diff --git a/src/AS_global.H b/src/AS_global.H
index 2b4e687..76af4cb 100644
--- a/src/AS_global.H
+++ b/src/AS_global.H
@@ -25,7 +25,7 @@
#ifndef AS_GLOBAL_H
#define AS_GLOBAL_H
-static const char *rcsid_AS_GLOBAL_H = "$Id: AS_global.H 4466 2013-11-06 17:02:19Z brianwalenz $";
+static const char *rcsid_AS_GLOBAL_H = "$Id: AS_global.H 4605 2015-02-17 22:25:36Z skoren $";
// ISO C99 says that to get INT32_MAX et al, these must be defined. (7.18.2, 7.18.4, 7.8.1)
#ifndef __STDC_CONSTANT_MACROS
@@ -148,9 +148,9 @@ typedef void* PtrT;
#define F_F64I "%*lf"
// Standard typedefs
-#define F_SIZE_T "%zd"
-#define F_SIZE_TP "zd"
-#define F_SIZE_TI "%*zd"
+#define F_SIZE_T "%zu"
+#define F_SIZE_TP "zu"
+#define F_SIZE_TI "%*zu"
#define F_OFF_T F_S64
#define F_OFF_TP F_S64P
@@ -218,7 +218,7 @@ typedef uintptr_t INTPTR;
// amounts of memory too.
//
#define AS_READ_MAX_PACKED_LEN_BITS 8
-#define AS_READ_MAX_NORMAL_LEN_BITS 16
+#define AS_READ_MAX_NORMAL_LEN_BITS 18
// Do not define this.
#undef OVS_DANGEROUSLY_OVERSIZE
@@ -260,4 +260,58 @@ int AS_configure(int argc, char **argv);
// without bothering lint
#define AssertPtr(ptr) (assert((ptr) != NULL))
+
+
+// Stubs for ignoring OpenMP on clang.
+#ifdef BROKEN_CLANG_OpenMP
+
+static
+int
+omp_get_max_threads(void) {
+ return(1);
+}
+
+static
+void
+omp_set_num_threads(int x) {
+}
+
+static
+int
+omp_get_num_threads(void) {
+ return(1);
+}
+
+static
+int
+omp_get_thread_num(void) {
+ return(0);
+}
+
+typedef int omp_lock_t;
+
+static
+void
+omp_init_lock(omp_lock_t *a) {
+}
+
+static
+void
+omp_set_lock(omp_lock_t *a) {
+}
+
+static
+void
+omp_unset_lock(omp_lock_t *a) {
+}
+
+static
+void
+omp_destroy_lock(omp_lock_t *a) {
+}
+
+#endif
+
+
+
#endif
diff --git a/src/c_make.as b/src/c_make.as
index a499721..e706fcc 100644
--- a/src/c_make.as
+++ b/src/c_make.as
@@ -113,7 +113,7 @@ ifeq ($(OSTYPE), FreeBSD)
ifeq ($(BUILDPROFILE), 1)
ARCH_CFLAGS += -fopenmp
- ARCH_LDFLAGS += -fopenmp -rpath /usr/local/lib/gcc46
+ ARCH_LDFLAGS += -fopenmp -rpath /usr/local/lib/gcc46 -pg
else
ARCH_CFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
ARCH_LDFLAGS += -D_GLIBCXX_PARALLEL -fopenmp -rpath /usr/local/lib/gcc46
@@ -123,9 +123,6 @@ ifeq ($(OSTYPE), FreeBSD)
ARCH_LDFLAGS += -L/usr/local/lib -lunwind -lunwind-x86_64
ifeq ($(BUILDDEBUG), 1)
- # Inconveniently fix a problem where gcc46 doesn't work with gdb.
- CC = gcc44
- CXX = g++44
ARCH_CFLAGS += -g
else
ifeq ($(BUILDPROFILE), 1)
@@ -142,20 +139,36 @@ endif
ifeq ($(OSTYPE), Darwin)
- CC = gcc-4.2
- CXX = g++-4.2
ARCH_CFLAGS = -D_THREAD_SAFE
+ CLANG = $(shell echo `${CXX} --version 2>&1 | grep -c clang`)
+ #$(info CXX $(CXX) CLANG $(CLANG))
+
+ # For gnu gcc/g++
+ #
+ ifeq ($(CLANG), 0)
+ ARCH_CFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
+ ARCH_LDFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
+ endif
+
+ # For clang/clang++
+ #
+ # Bogart, CGW and possibly others, will not compile with clang because OpenMP is missing.
+ # If you must use clang, add the symbol below.
+ #
+ ifeq ($(CLANG), 1)
+ CC = clang
+ CXX = clang++
+ ARCH_CFLAGS += -DBROKEN_CLANG_OpenMP
+ endif
+
ifeq ($(BUILDDEBUG), 1)
ARCH_CFLAGS += -g -Wall
else
- ARCH_CFLAGS += -fast
+ ARCH_CFLAGS += -O3
endif
- ARCH_CFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
- ARCH_LDFLAGS += -D_GLIBCXX_PARALLEL -fopenmp
-
- ARCH_CFLAGS += -fPIC -m64 -fmessage-length=0 -Wimplicit -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare
+ ARCH_CFLAGS += -fPIC -m64 -fmessage-length=0 -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare
# ARCH_CFLAGS += -Wshorten-64-to-32 # Wow, tough
# ARCH_CFLAGS += -Wextra
# ARCH_CFLAGS += -pedantic (see above about pedantic)
@@ -199,25 +212,60 @@ ifeq ($(BUILDPROFILE), 1)
ARCH_LDFLAGS += -pg
endif
+
+
# One can argue that CXXFLAGS should be separate. For now, we only
# add to the flags.
CFLAGS += $(ARCH_CFLAGS)
CXXFLAGS += $(ARCH_CFLAGS) $(ARCH_CXXFLAGS)
-LDFLAGS += $(ARCH_LDFLAGS) -rdynamic
+
+
+# -rdynamic is needed for the crash reporting, iirc.
+ifneq ($(OSTYPE), Darwin)
+ LDFLAGS += $(ARCH_LDFLAGS) -rdynamic
+else
+ LDFLAGS += $(ARCH_LDFLAGS)
+endif
+
+
+
+# This isn't perfect; if we're building debug here, we _usually_ want
+# to use the debug kmer. But checking if kmer exists, and failing if
+# we don't find the debug kmer, is a little tricky.
+
+ifneq "$(origin KMER)" "environment"
+
+ifeq ($(shell ls -d $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug 2> /dev/null), $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug)
+ KMER = $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)-debug
+endif
+
+ifeq ($(shell ls -d $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE) 2> /dev/null), $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE))
+ KMER = $(LOCAL_WORK)/kmer/$(OSTYPE)-$(MACHINETYPE)
+endif
+
+endif
+
+
INC_IMPORT_DIRS += $(LOCAL_WORK)/src $(patsubst %, $(LOCAL_WORK)/src/%, $(strip $(SUBDIRS)))
+INC_IMPORT_DIRS += $(KMER)/include
INC_IMPORT_DIRS += $(ARCH_INC)
LIB_IMPORT_DIRS += $(LOCAL_LIB)
+LIB_IMPORT_DIRS += $(KMER)/lib
LIB_IMPORT_DIRS += $(ARCH_LIB)
OBJ_SEARCH_PATH = $(LOCAL_OBJ)
+
ifeq ($(SITE_NAME), JCVI)
LDFLAGS += -lcurl
endif
+
+
+
# The order of compilation here is very carefully chosen to be the
# same as the order used in running an assembly. It is extremely
# useful if you happen to be making changes to, say, the persistent
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/wgs-assembler.git
More information about the debian-med-commit
mailing list