[med-svn] [sra-sdk] 01/04: Imported Upstream version 2.5.7
Andreas Tille
tille at debian.org
Wed Dec 30 07:52:53 UTC 2015
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository sra-sdk.
commit 82d7c33105259c88c0d9a104b5b0ab7f4d2fb6ab
Author: Andreas Tille <tille at debian.org>
Date: Wed Dec 30 08:41:16 2015 +0100
Imported Upstream version 2.5.7
---
.gitignore | 16 +-
GNU-license.txt | 674 ---
LICENSE | 66 +
Makefile | 13 +-
README | 53 +-
README-WINDOWS.txt | 144 -
README-build | 237 -
README-config.txt | 19 -
README-vdb-config.txt => README-vdb-config | 4 -
README-wget.txt | 35 -
README.md | 8 +-
USAGE | 162 -
build/MSVC/2010/abi-dump.vcxproj | 2 -
...d-filter-redact.vcxproj => align-cache.vcxproj} | 76 +-
build/MSVC/2010/align-info.vcxproj | 2 -
build/MSVC/2010/cache-mgr.vcxproj | 4 +-
...q-dump-ngs.vcxproj => dbgap-mount-demo.vcxproj} | 13 +-
build/MSVC/2010/dll-project.props | 59 -
build/MSVC/2010/exe-project.props | 70 +-
build/MSVC/2010/fastq-dump-ngs.vcxproj | 5 +-
build/MSVC/2010/fastq-dump.vcxproj | 2 -
build/MSVC/2010/fastq-loader.vcxproj | 6 +-
.../{kdbmeta.vcxproj => general-loader.vcxproj} | 16 +-
...llumina-diump.vcxproj => illumina-dump.vcxproj} | 0
build/MSVC/2010/kar.vcxproj | 2 -
build/MSVC/2010/kdbmeta.vcxproj | 2 -
build/MSVC/2010/lib-project.props | 68 +-
build/MSVC/2010/md5cp.vcxproj | 4 +-
build/MSVC/2010/prefetch.vcxproj | 5 +-
build/MSVC/2010/qual-recalib-stat.vcxproj | 4 -
build/MSVC/2010/rcexplain.vcxproj | 2 -
build/MSVC/2010/read-filter-redact.vcxproj | 2 -
...astq-dump-ngs.vcxproj => ref-variation.vcxproj} | 84 +-
build/MSVC/2010/sam-dump.vcxproj | 4 +-
build/MSVC/2010/schema-replace.vcxproj | 2 -
build/MSVC/2010/sff-dump.vcxproj | 2 -
build/MSVC/2010/sra-kar.vcxproj | 2 -
build/MSVC/2010/sra-pileup.vcxproj | 4 +-
build/MSVC/2010/sra-stat.vcxproj | 4 +-
build/MSVC/2010/sra-tools.sln | 70 +-
build/MSVC/2010/srapath.vcxproj | 2 -
.../{md5cp.vcxproj => test-align-info.vcxproj} | 23 +-
...tq-loader.vcxproj => test-fastq-loader.vcxproj} | 17 +-
...kdbmeta.vcxproj => test-general-loader.vcxproj} | 22 +-
build/MSVC/2010/test-sra-tools.sln | 66 +
build/MSVC/2010/test-sra.vcxproj | 2 -
...ema-replace.vcxproj => test-vcf-loader.vcxproj} | 21 +-
.../{fastq-dump-ngs.vcxproj => var-expand.vcxproj} | 84 +-
build/MSVC/2010/vdb-common.props | 92 +-
build/MSVC/2010/vdb-config.vcxproj | 9 +-
build/MSVC/2010/vdb-copy.vcxproj | 3 -
build/MSVC/2010/vdb-decrypt.vcxproj | 2 -
.../{vdb-validate.vcxproj => vdb-diff.vcxproj} | 9 +-
build/MSVC/2010/vdb-dump.vcxproj | 3 -
build/MSVC/2010/vdb-encrypt.vcxproj | 2 -
build/MSVC/2010/vdb-lock.vcxproj | 2 -
build/MSVC/2010/vdb-passwd.vcxproj | 2 -
build/MSVC/2010/vdb-unlock.vcxproj | 2 -
build/MSVC/2010/vdb-validate.vcxproj | 2 -
...{fastq-loader.vcxproj => wb-test-fastq.vcxproj} | 17 +-
build/Makefile.clang | 16 +-
build/Makefile.env | 29 +-
build/Makefile.gcc | 16 +-
build/Makefile.install | 56 +-
build/Makefile.linux | 3 +
build/Makefile.mac | 3 +
build/ld.linux.dlib.sh | 4 +-
build/ld.linux.exe.sh | 18 +-
build/ld.linux.gcc.sh | 5 +-
build/ld.mac.clang.sh | 7 +-
build/ld.mac.gcc.sh | 7 +-
build/ld.mac.slib.sh | 35 +-
configure | 6 +-
konfigure.perl | 1204 -----
os-arch.perl | 35 -
configure => setup/install | 10 +-
setup/install.perl | 1329 ++++++
setup/install.prl | 38 +
setup/konfigure.perl | 1717 +++++++
setup/os-arch.perl | 22 +
os-arch.pm => setup/os-arch.prl | 12 +-
package.pm => setup/package.prl | 48 +-
configure => setup/s/install | 12 +-
setup/s/os.prl | 2 +
test/Makefile | 38 +-
test/{ => align-info}/Makefile | 30 +-
test/fastq-loader/.gitignore | 1 +
test/fastq-loader/Makefile | 207 +
test/fastq-loader/expected/1.1.stdout | 31 +
test/fastq-loader/expected/1.2.stdout | 31 +
test/fastq-loader/expected/1.3.stderr | 4 +
test/fastq-loader/expected/1.3.stdout | 27 +
test/fastq-loader/expected/1.4.stdout | 90 +
test/fastq-loader/expected/10.0.stdout | 31 +
test/fastq-loader/expected/11.0.stdout | 31 +
test/fastq-loader/expected/11.1.stdout | 31 +
test/fastq-loader/expected/12.0.stdout | 60 +
test/fastq-loader/expected/12.1.stdout | 60 +
test/fastq-loader/expected/12.2.stdout | 31 +
test/fastq-loader/expected/2.1.1.stdout | 44 +
test/fastq-loader/expected/2.1.stdout | 60 +
test/fastq-loader/expected/2.5.stdout | 60 +
test/fastq-loader/expected/2.6.stdout | 31 +
test/fastq-loader/expected/2.7.stdout | 31 +
test/fastq-loader/expected/2.8.1.stdout | 60 +
test/fastq-loader/expected/2.8.stdout | 60 +
test/fastq-loader/expected/2.9.1.stderr | 4 +
test/fastq-loader/expected/2.9.stderr | 4 +
test/fastq-loader/expected/3.1.stdout | 120 +
test/fastq-loader/expected/4.2.stdout | 31 +
test/fastq-loader/expected/4.3.1.stderr | 14 +
test/fastq-loader/expected/4.4.stdout | 66 +
test/fastq-loader/expected/4.5.1.stderr | 8 +
test/fastq-loader/expected/4.5.stderr | 9 +
test/fastq-loader/expected/4.6.1.stderr | 3 +
test/fastq-loader/expected/4.6.stdout | 31 +
test/fastq-loader/expected/4.7.stderr | 2 +
test/fastq-loader/expected/5.0.stdout | 31 +
test/fastq-loader/expected/6.0.stdout | 390 ++
test/fastq-loader/expected/7.1.stdout | 60 +
test/fastq-loader/expected/7.2.stdout | 60 +
test/fastq-loader/expected/7.3.stdout | 60 +
test/fastq-loader/expected/8.0.stdout | 31 +
test/fastq-loader/expected/8.1.stderr | 5 +
test/fastq-loader/expected/9.0.stdout | 60 +
test/fastq-loader/input/1.1.fastq | 5 +
test/fastq-loader/input/1.2.fastq | 5 +
test/fastq-loader/input/1.3.fastq | 5 +
test/fastq-loader/input/1.4.fastq | 24 +
test/fastq-loader/input/10.0.fastq | 4 +
test/fastq-loader/input/11.0.1.fasta | 2 +
test/fastq-loader/input/11.0.2.fasta | 2 +
test/fastq-loader/input/11.1.1.fastq | 4 +
test/fastq-loader/input/11.1.2.fastq | 4 +
test/fastq-loader/input/12.0.fastq | 8 +
test/fastq-loader/input/12.1.fastq | 8 +
test/fastq-loader/input/12.2.fastq | 4 +
test/fastq-loader/input/2.1.1.fastq | 5 +
test/fastq-loader/input/2.1.fastq | 4 +
test/fastq-loader/input/2.2.fastq | 4 +
test/fastq-loader/input/2.3.fastq | 8 +
test/fastq-loader/input/2.4.fastq | 8 +
test/fastq-loader/input/2.5.fastq | 9 +
test/fastq-loader/input/2.6.fastq | 9 +
test/fastq-loader/input/2.7.fastq | 8 +
test/fastq-loader/input/2.8.1.fastq | 4 +
test/fastq-loader/input/2.8.fastq | 4 +
test/fastq-loader/input/2.9.1.fastq | 4 +
test/fastq-loader/input/2.9.fastq | 4 +
test/fastq-loader/input/3.1a.fastq | 8 +
test/fastq-loader/input/3.1b.fastq | 9 +
test/fastq-loader/input/3.2a.fastq | 8 +
test/fastq-loader/input/3.2b.fastq | 8 +
test/fastq-loader/input/3.3a.fastq | 8 +
test/fastq-loader/input/3.3b.fastq | 8 +
test/fastq-loader/input/4.4.fastq | 13 +
test/fastq-loader/input/4.5.1.fastq | 10 +
test/fastq-loader/input/4.5.fastq | 10 +
test/fastq-loader/input/4.fastq | 5 +
test/fastq-loader/input/5.0.fastq.gz | Bin 0 -> 139 bytes
test/fastq-loader/input/6.0.fastq | 52 +
test/fastq-loader/input/7.1.fastq | 9 +
test/fastq-loader/input/7.2.fastq | 8 +
test/fastq-loader/input/7.3.fastq | 8 +
test/fastq-loader/input/8.0.fastq | 9 +
test/fastq-loader/input/8.1.fastq | 17 +
test/fastq-loader/input/9.0.fastq | 16 +
test/fastq-loader/runtestcase.sh | 106 +
test/fastq-loader/test-fastq-loader.cpp | 215 +
test/fastq-loader/wb-test-fastq.cpp | 1376 ++++++
test/general-loader/.gitignore | 3 +
test/general-loader/Makefile | 194 +
test/general-loader/column01 | 3 +
test/general-loader/column02 | 3 +
test/general-loader/expected/1.stdout | 75 +
test/general-loader/expected/1override.stdout | 75 +
test/general-loader/expected/1packed.stdout | 75 +
test/general-loader/expected/2.stderr | 2 +
test/general-loader/expected/2packed.stderr | 2 +
test/general-loader/expected/3.stdout | 16 +
test/general-loader/expected/3packed.stdout | 16 +
test/general-loader/expected/4.stdout | 160 +
test/general-loader/expected/4packed.stdout | 160 +
test/general-loader/expected/5packed.stdout | 16 +
test/general-loader/expected/6.stdout | 1 +
test/general-loader/expected/6packed.stdout | 1 +
test/general-loader/makeinputs.cpp | 226 +
test/general-loader/runtestcase.sh | 108 +
test/general-loader/schema/.gitignore | 4 +
test/general-loader/test-general-loader.cpp | 1698 +++++++
test/general-loader/test-general-writer.cpp | 341 ++
.../general-loader/test-gw-dumper.cpp | 85 +-
test/general-loader/testsource.cpp | 758 +++
test/general-loader/testsource.hpp | 177 +
test/kget/.gitignore | 3 +
configure => test/kget/Makefile | 53 +-
{tools/cg-dump => test/network-test}/Makefile | 48 +-
test/network-test/network-test.c | 105 +
test/network-test/network-test.vers | 1 +
test/network-test/network-testing.c | 436 ++
.../network-test/network-testing.h | 12 +-
test/ngs-pileup/Makefile | 140 +
test/ngs-pileup/runtestcase.sh | 90 +
test/ngs-pileup/test-ngs-pileup.cpp | 177 +
test/ngs-pileup/valgrind.suppress | 6 +
test/only-remote-repository.kfg | 2 +
test/ref-variation/Makefile | 57 +
test/ref-variation/valgrind.suppress | 6 +
{tools/sra-dbcc => test/samline}/Makefile | 34 +-
test/samline/alig-gen.c | 604 +++
test/samline/cigar.c | 446 ++
.../line_token_iter.h => test/samline/cigar.h | 37 +-
test/samline/example.sh | 64 +
test/samline/refbases.c | 145 +
.../vtblcp-priv.h => test/samline/refbases.h | 9 +-
test/samline/samline.vers | 1 +
test/tarballs/build.xml | 27 +
test/tarballs/test-tarballs.bat | 40 +
test/tarballs/test-tarballs.sh | 97 +
configure => test/vcf-loader/Makefile | 70 +-
test/vcf-loader/test-vcf-loader.cpp | 864 ++++
test/{ => vdb-dump}/Makefile | 29 +-
test/vdb-dump/SRR056386 | Bin 0 -> 14505 bytes
test/vdb-dump/expected/1.0.stdout | 1 +
test/vdb-dump/expected/1.1.stdout | 1 +
test/vschema/Makefile | 115 +
.../vschema/bio-end/bio-end-excl.vschema | 31 +-
.../vschema/bio-end/bio-end-incl.vschema | 31 +-
test/vschema/bio-end/expected/excl-1.stdout | 5 +
test/vschema/bio-end/expected/excl-2.stdout | 5 +
test/vschema/bio-end/expected/excl-3.stdout | 5 +
test/vschema/bio-end/expected/excl-4.stdout | 5 +
test/vschema/bio-end/expected/incl-1.stdout | 5 +
test/vschema/bio-end/makeinputs.cpp | 129 +
test/vschema/idx-text/checklookup.cpp | 323 ++
test/vschema/idx-text/expected/ci-1.stdout | 15 +
test/vschema/idx-text/expected/ci-2.stdout | 18 +
test/vschema/idx-text/expected/ci-3.stdout | 12 +
test/vschema/idx-text/expected/ci-4.stdout | 12 +
test/vschema/idx-text/expected/cs-1.stdout | 18 +
test/vschema/idx-text/expected/cs-2.stdout | 18 +
test/vschema/idx-text/idx-text.vschema | 105 +
test/vschema/idx-text/makeinputs.cpp | 148 +
test/vschema/runtestcase.sh | 126 +
tools/.gitignore | 3 +
tools/Makefile | 66 +-
tools/agp-load/AGP.py | 668 +++
tools/agp-load/agp-load.py | 173 +
tools/agp-load/agp.vschema | 45 +
tools/agp-load/eutils.py | 112 +
tools/{vtblcp => align-cache}/Makefile | 34 +-
tools/align-cache/align-cache.cpp | 557 +++
tools/align-cache/align-cache.vers | 1 +
tools/align-cache/align-cache.vers.h | 1 +
tools/align-cache/helper.cpp | 923 ++++
tools/align-cache/helper.h | 390 ++
tools/align-info/align-info.c | 20 +-
tools/align-info/align-info.vers | 2 +-
tools/align-info/align-info.vers.h | 2 +-
tools/bam-loader/Globals.h | 1 +
tools/bam-loader/Makefile | 32 +-
tools/bam-loader/bam-load.3.vers | 1 -
tools/bam-loader/bam-load.vers | 2 +-
tools/bam-loader/bam-loader.c | 63 +-
tools/bam-loader/bam-loader3.c | 1028 -----
tools/bam-loader/bam.c | 4085 ++++++++++++++++
tools/bam-loader/bam.h | 632 +++
tools/bam-loader/config-file.cpp | 195 +
.../key_value.h => bam-loader/config-file.hpp} | 73 +-
tools/bam-loader/fasta-file.cpp | 257 ++
.../idx-mapping.h => bam-loader/fasta-file.hpp} | 68 +-
tools/bam-loader/loader-imp.c | 1426 +++---
tools/bam-loader/loader-imp3.c | 369 --
tools/bam-loader/mem-bank.cpp | 477 ++
.../fastq-reader.h => bam-loader/mem-bank.h} | 29 +-
tools/bam-loader/reference-writer.c | 214 +-
tools/bam-loader/reference-writer.h | 12 +-
tools/bam-loader/sequence-writer.c | 24 +-
tools/cache-mgr/cache-mgr.c | 64 +-
tools/cache-mgr/cache-mgr.vers | 2 +-
tools/cache-mgr/cache-mgr.vers.h | 2 +-
tools/ccextract/Makefile | 25 +-
tools/ccextract/ccextract.c | 16 +-
tools/ccextract/ccextract.vers | 2 +-
tools/cg-dump/cg-dump.c | 1563 -------
tools/cg-dump/cg-dump.vers | 1 -
tools/cg-dump/last_rowid.c | 353 --
tools/cg-dump/line_token_iter.c | 148 -
tools/cg-dump/num-gen.c | 848 ----
tools/cg-dump/num-gen.h | 225 -
tools/cg-dump/progressbar.c | 140 -
tools/cg-dump/progressbar.h | 70 -
tools/cg-dump/sg_lookup.c | 277 --
tools/cg-load/Makefile | 5 +-
tools/cg-load/cg-load.c | 281 +-
tools/cg-load/cg-load.vers | 2 +-
tools/cg-load/defs.h | 41 +-
tools/cg-load/f1_3.c | 1 +
tools/cg-load/f1_5.c | 2 +-
tools/cg-load/{f1_5.c => f1_7.c} | 39 +-
tools/cg-load/f2_0.c | 18 +-
tools/cg-load/f2_2.c | 4 +-
tools/cg-load/f2_5.c | 98 +
tools/cg-load/factory-evidence-dnbs.c | 290 +-
tools/cg-load/factory-evidence-dnbs.h | 3 +
tools/cg-load/factory-evidence-intervals.c | 100 +-
tools/cg-load/factory-evidence-intervals.h | 3 +
tools/cg-load/factory-file.h | 4 +
tools/cg-load/factory-mappings.c | 213 +-
tools/cg-load/factory-mappings.h | 3 +
tools/cg-load/factory-reads.c | 246 +-
tools/cg-load/factory-reads.h | 3 +
tools/cg-load/factory-tag-lfr.c | 183 +-
tools/cg-load/factory-tag-lfr.h | 6 +-
tools/cg-load/file-version-factory.c | 7 +-
tools/cg-load/file.c | 149 +-
tools/cg-load/file.h | 1 +
tools/{vdb-dump/vdb_info.h => cg-load/formats.c} | 37 +-
tools/{vtblcp/vtblcp-priv.h => cg-load/formats.h} | 19 +-
tools/cg-load/writer-algn.c | 128 +-
tools/cg-load/writer-algn.h | 3 +-
tools/cg-load/writer-evidence-dnbs.c | 56 +-
tools/cg-load/writer-evidence-dnbs.h | 19 +-
tools/cg-load/writer-evidence-intervals.c | 25 +-
tools/cg-load/writer-seq.c | 36 +-
tools/cg-load/writer-seq.h | 12 +-
tools/check-blob-size/check-blob-size.c | 332 --
tools/check-blob-size/check-blob-size.vers | 1 -
tools/copycat/Makefile | 28 +-
tools/copycat/cccat.c | 12 +-
tools/copycat/ccsra.c | 2 +-
tools/copycat/cctree.c | 6 +-
tools/copycat/copycat.c | 14 +-
tools/copycat/copycat.vers | 2 +-
tools/{workspace => dbgap-mount}/Makefile | 82 +-
tools/dbgap-mount/demo.c | 360 ++
tools/dbgap-mount/demo.mkfg | 47 +
tools/dbgap-mount/demo.vers | 1 +
tools/dbgap-mount/demo.vers.h | 1 +
tools/deseq/csra-trim.vers | 1 -
tools/deseq/deseq.c | 521 ---
tools/fastq-dump/Makefile | 18 +-
tools/fastq-dump/args.cpp | 12 +-
tools/fastq-dump/args.hpp | 2 +-
tools/fastq-dump/fastq-dump.cpp | 146 +-
tools/fastq-dump/fastq-dump.vers | 2 +-
tools/fastq-dump/koutstream | 73 +-
tools/fastq-loader/Makefile | 13 +-
tools/fastq-loader/fastq-grammar.c | 714 +--
tools/fastq-loader/fastq-grammar.y | 224 +-
tools/fastq-loader/fastq-lex.c | 749 ++-
tools/fastq-loader/fastq-lex.l | 20 +-
tools/fastq-loader/fastq-loader.c | 80 +-
tools/fastq-loader/fastq-parse.h | 50 +-
tools/fastq-loader/fastq-reader.c | 45 +-
tools/fastq-loader/fastq-reader.h | 9 +-
tools/fastq-loader/latf-load.vers | 2 +-
tools/fastq-loader/latf-load.vers.h | 2 +-
tools/fastq-loader/loader-imp.c | 19 +-
tools/fastq-loader/parsing_rules.txt | 93 +
tools/fix-run-stat/fix-run-stat.c | 355 --
tools/fix-run-stat/fix-run-stat.vers | 1 -
tools/fuse/Makefile | 37 +
tools/fuse/directory.c | 8 +-
tools/fuse/remote-cache.c | 12 +-
tools/fuse/remote-directory.c | 4 +-
tools/fuse/remote-fuser.vers | 2 +-
tools/fuse/sra-fuser-test.vers | 2 +-
tools/fuse/sra-fuser-test.vers.h | 2 +-
tools/fuse/sra-fuser.vers | 2 +-
tools/fuse/sra-list.c | 6 +-
tools/fuse/sra-makeidx.c | 6 +-
tools/fuse/sra-makeidx.vers | 2 +-
tools/fuse/tar-list.c | 4 +-
tools/fuse/unix/sra-fuser-sys.c | 37 +-
tools/{meta-sync => general-loader}/Makefile | 100 +-
tools/general-loader/database-loader.cpp | 895 ++++
tools/general-loader/general-loader.cpp | 223 +
tools/general-loader/general-loader.hpp | 243 +
tools/general-loader/general-loader.vers | 1 +
tools/general-loader/general-loader.vers.h | 1 +
tools/general-loader/general-writer.cpp | 1172 +++++
tools/general-loader/general-writer.h | 1081 +++++
tools/general-loader/general-writer.hpp | 210 +
tools/general-loader/gw-dumper.cpp | 1785 +++++++
tools/general-loader/libgeneral-writer.vers | 1 +
tools/general-loader/main.cpp | 246 +
tools/general-loader/protocol-parser.cpp | 1048 +++++
tools/general-loader/utf8-like-int-codec.c | 687 +++
.../utf8-like-int-codec.h} | 63 +-
tools/kar/kar.c | 21 +-
tools/kar/kar.vers | 2 +-
tools/kar/kar.vers.h | 2 +-
tools/kar/sra-kar.c | 10 +-
tools/kar/sra-kar.vers | 2 +-
tools/kar/sra-kar.vers.h | 2 +-
tools/keyring-srv/keyring-data.c | 8 +-
tools/keyring-srv/keyring-srv.vers | 2 +-
tools/{sra-dbcc => kget}/Makefile | 31 +-
tools/kget/kget.c | 688 +++
tools/kget/kget.vers | 1 +
tools/kqsh/kqsh.c | 2 +-
tools/kqsh/kqsh.vers | 2 +-
tools/meta-sync/meta-sync.c | 1517 ------
tools/meta-sync/meta-sync.pl | 41 -
tools/meta-sync/meta-sync.vers | 1 -
tools/nenctool/nenctest.c | 10 +-
tools/nenctool/nenctest.vers | 2 +-
tools/nenctool/nenctool.c | 12 +-
tools/nenctool/nenctool.vers | 2 +-
tools/nencvalid/nencvalid.c | 9 +-
tools/nencvalid/nencvalid.vers | 2 +-
tools/{fix-run-stat => ngs-pileup}/Makefile | 46 +-
tools/ngs-pileup/main.cpp | 164 +
tools/ngs-pileup/ngs-pileup.cpp | 206 +
tools/ngs-pileup/ngs-pileup.hpp | 97 +
tools/ngs-pileup/ngs-pileup.vers | 1 +
tools/ngs-pileup/ngs-pileup.vers.h | 1 +
tools/pacbio-load/Makefile | 8 +-
tools/pacbio-load/pacbio-load.vers | 2 +-
tools/pacbio-load/pl-consensus.c | 227 +-
tools/pacbio-load/pl-context.c | 4 +-
tools/pacbio-load/pl-regions.c | 4 +-
tools/pacbio-load/pl-tools.c | 22 +-
tools/{deseq => pileup-stats}/Makefile | 47 +-
tools/pileup-stats/pileup-stats.cpp | 719 +++
tools/pileup-stats/pileup-stats.vers | 1 +
tools/pore-load/GeneralWriter.py | 312 ++
tools/pore-load/README.md | 10 +
tools/pore-load/pore-load.py | 406 ++
tools/prefetch/Makefile | 3 +-
tools/prefetch/kfile-no-q.c | 215 +
.../loader-imp3.h => prefetch/kfile-no-q.h} | 26 +-
tools/prefetch/prefetch.c | 1020 +++-
tools/prefetch/prefetch.vers | 2 +-
tools/prefetch/prefetch.vers.h | 2 +-
tools/qual-recal/Makefile | 101 -
tools/qual-recal/make_histo_data.sh | 12 -
tools/qual-recal/qual-recal.c | 654 ---
tools/qual-recalib-stat/Makefile | 2 -
tools/qual-recalib-stat/context.c | 6 +-
tools/qual-recalib-stat/context.h | 4 +-
tools/qual-recalib-stat/num-gen.c | 848 ----
tools/qual-recalib-stat/num-gen.h | 225 -
tools/qual-recalib-stat/progressbar.c | 182 -
tools/qual-recalib-stat/progressbar.h | 75 -
tools/qual-recalib-stat/qual-recalib-stat.c | 39 +-
tools/qual-recalib-stat/qual-recalib-stat.vers | 2 +-
tools/qual-recalib-stat/qual-recalib-stat.vers.h | 2 +-
tools/qual-recalib-stat/ref_exclude.c | 8 +-
tools/qual-recalib-stat/stat_mod.c | 4 +-
tools/qual-recalib-stat/stat_mod_1.c | 4 +-
tools/qual-recalib-stat/stat_mod_2.c | 4 +-
tools/qual-recalib-stat/writer.c | 58 +-
tools/qual-recalib-stat/writer.h | 1 -
tools/rd-filter-redact/rd-filter-redact.c | 6 +-
tools/rd-filter-redact/rd-filter-redact.vers | 2 +-
tools/rd-filter-redact/read-filter-redact.c | 6 +-
tools/rd-filter-redact/read-filter-redact.vers | 2 +-
tools/rd-filter-redact/read-filter-redact.vers.h | 2 +-
tools/rd-filter-redact/test-read-write-cursor.vers | 2 +-
tools/{check-blob-size => ref-variation}/Makefile | 73 +-
tools/ref-variation/common.cpp | 70 +
.../vdb_info.h => ref-variation/common.h} | 31 +-
tools/ref-variation/helper.cpp | 1404 ++++++
tools/ref-variation/helper.h | 605 +++
tools/ref-variation/ref-variation.cpp | 2102 +++++++++
tools/ref-variation/ref-variation.vers | 1 +
tools/ref-variation/ref-variation.vers.h | 1 +
tools/ref-variation/var-expand.cpp | 448 ++
tools/ref-variation/var-expand.vers | 1 +
tools/ref-variation/var-expand.vers.h | 1 +
tools/refseq-load/refseq-load.vers | 1 +
tools/run-stat/.gitignore | 6 -
tools/run-stat/Makefile | 120 -
tools/run-stat/chart.c | 465 --
tools/run-stat/chart.h | 109 -
tools/run-stat/context.c | 439 --
tools/run-stat/context.h | 98 -
tools/run-stat/helper.c | 715 ---
tools/run-stat/helper.h | 126 -
tools/run-stat/key_value.c | 549 ---
tools/run-stat/mod_cmn.c | 329 --
tools/run-stat/mod_cmn.h | 149 -
tools/run-stat/mod_reads.c | 747 ---
tools/run-stat/mod_reads_defs.h | 128 -
tools/run-stat/mod_reads_helper.c | 575 ---
tools/run-stat/mod_reads_helper.h | 61 -
tools/run-stat/num-gen.c | 497 --
tools/run-stat/progressbar.c | 191 -
tools/run-stat/progressbar.h | 76 -
tools/run-stat/rd_filter.c | 243 -
tools/run-stat/rd_filter.h | 98 -
tools/run-stat/run-stat.c | 703 ---
tools/run-stat/run-stat.vers | 1 -
tools/run-stat/svg.c | 753 ---
tools/run-stat/svg.h | 117 -
tools/sam-dump/debug.h | 34 -
tools/sam-dump/sam-dump.c | 4870 --------------------
tools/sam-dump/sam-dump.vers | 1 -
tools/sra-dbcc/sra-dbcc.c | 1577 -------
tools/sra-dbcc/sra-dbcc.vers | 1 -
tools/sra-dump/abi-dump.vers | 2 +-
tools/sra-dump/abi-dump.vers.h | 2 +-
tools/sra-dump/abi.c | 4 +-
tools/sra-dump/core.c | 9 +-
tools/sra-dump/factory.c | 4 +-
tools/sra-dump/fasta_dump.c | 6 +-
tools/sra-dump/fastq-dump.vers | 2 +-
tools/sra-dump/fastq-dump.vers.h | 2 +-
tools/sra-dump/fastq.c | 4 +-
tools/sra-dump/illumina-dump.vers | 2 +-
tools/sra-dump/illumina-dump.vers.h | 2 +-
tools/sra-dump/sff-dump.vers | 2 +-
tools/sra-dump/sff-dump.vers.h | 2 +-
tools/sra-dump/sra-dump.c | 20 +-
tools/sra-dump/sra-dump.vers | 2 +-
tools/sra-load/Makefile | 19 +-
tools/sra-load/abi-load.vers | 2 +-
tools/sra-load/common-xml.c | 11 +-
tools/sra-load/experiment-xml.c | 6 +-
tools/sra-load/fastq-load.vers | 2 +-
tools/sra-load/helicos-load.vers | 2 +-
tools/sra-load/illumina-load.vers | 2 +-
tools/sra-load/loader.c | 28 +-
tools/sra-load/pacbio-loadxml.vers | 2 +-
tools/sra-load/sff-load.vers | 2 +-
tools/sra-load/srf-absolid.c | 15 +-
tools/sra-load/srf-load.vers | 2 +-
tools/sra-load/writer-illumina.c | 2 +-
tools/sra-load/ztr-huffman.c | 15 +-
tools/sra-pileup/Makefile | 27 +-
tools/sra-pileup/cg_tools.c | 2 +-
tools/sra-pileup/cmdline_cmn.c | 6 +-
tools/sra-pileup/dyn_string.c | 20 +
tools/sra-pileup/dyn_string.h | 1 +
tools/sra-pileup/inputfiles.c | 1 +
tools/sra-pileup/md_flag.c | 268 ++
.../{cg-dump/last_rowid.h => sra-pileup/md_flag.h} | 14 +-
tools/sra-pileup/pileup_counters.c | 4 +-
.../{pileup_varcount.c => pileup_indels.c} | 71 +-
.../vtblcp-priv.h => sra-pileup/pileup_indels.h} | 8 +-
tools/sra-pileup/pileup_options.h | 1 +
tools/sra-pileup/pileup_v2.c | 8 +-
tools/sra-pileup/pileup_varcount.c | 10 +-
tools/sra-pileup/ref_regions.c | 74 +-
tools/sra-pileup/ref_walker_0.c | 20 +-
tools/sra-pileup/report_deletes.c | 2 +-
tools/sra-pileup/reref.c | 208 +-
tools/sra-pileup/sam-aligned.c | 44 +-
tools/sra-pileup/sam-dump-opts.c | 171 +-
tools/sra-pileup/sam-dump-opts.h | 3 +
tools/sra-pileup/sam-dump.c | 14 +-
tools/sra-pileup/sam-dump.vers | 2 +-
tools/sra-pileup/sam-dump.vers.h | 2 +-
tools/sra-pileup/sam-dump2.vers | 2 +-
tools/sra-pileup/sam-dump3.c | 12 +-
tools/sra-pileup/sam-hdr.c | 4 +-
tools/sra-pileup/sam-hdr1.c | 853 ++++
.../last_rowid.h => sra-pileup/sam-hdr1.h} | 18 +-
tools/sra-pileup/sra-pileup.c | 214 +-
tools/sra-pileup/sra-pileup.vers | 2 +-
tools/sra-pileup/sra-pileup.vers.h | 2 +-
tools/sra-pileup/walk_debug.c | 75 +-
tools/{sam-dump => sra-seq-count}/Makefile | 53 +-
tools/sra-seq-count/main_fkt.c | 211 +
.../last_rowid.h => sra-seq-count/options.h} | 24 +-
tools/sra-seq-count/range.hpp | 307 ++
tools/sra-seq-count/sra-seq-count.cpp | 688 +++
tools/sra-seq-count/sra-seq-count.vers | 1 +
tools/sra-sort/col-pair.c | 2 +-
tools/sra-sort/idx-mapping.c | 12 +-
tools/sra-sort/idx-mapping.h | 4 +-
tools/sra-sort/meta-pair.c | 12 +-
tools/sra-sort/poslen-col-pair.c | 2 +-
tools/sra-sort/ref-alignid-col.c | 13 +-
tools/sra-sort/sra-sort.c | 10 +-
tools/sra-sort/sra-sort.vers | 2 +-
tools/sra-stat/Makefile | 8 +-
tools/sra-stat/sra-stat.c | 1301 ++++--
.../{run-stat/mod_reads.h => sra-stat/sra-stat.h} | 26 +-
tools/sra-stat/sra-stat.vers | 2 +-
tools/sra-stat/sra-stat.vers.h | 2 +-
tools/sra-stat/sra.c | 446 ++
tools/sra/.cvsignore | 1 +
tools/sra/Makefile | 9 +-
tools/sra/table-vers.c | 4 +-
tools/sra/table-vers.vers | 2 +-
tools/srapath/srapath.c | 51 +-
tools/srapath/srapath.vers | 2 +-
tools/srapath/srapath.vers.h | 2 +-
tools/update-schema/.cvsignore | 1 +
tools/update-schema/Makefile | 4 +-
tools/update-schema/update-schema.c | 14 +-
tools/update-schema/vdb-update-schema.vers | 2 +-
tools/util/Makefile | 39 +-
tools/util/crc32sum.c | 2 +-
tools/util/kdb-index.c | 264 ++
tools/util/kdb-index.vers | 1 +
tools/util/kdbmeta.c | 10 +-
tools/util/kdbmeta.vers | 2 +-
tools/util/kdbmeta.vers.h | 2 +-
tools/util/md5cp.c | 8 +-
tools/util/md5cp.vers | 2 +-
tools/util/md5cp.vers.h | 2 +-
tools/util/pacbio-correct.c | 2 +-
tools/util/pacbio-correct.vers | 2 +-
tools/util/rcexplain.c | 2 +-
tools/util/rcexplain.vers | 2 +-
tools/util/rcexplain.vers.h | 2 +-
tools/util/rowwritetest.c | 4 +-
tools/util/rowwritetest.vers | 2 +-
tools/{sra-sort/idx-mapping.h => util/samview.c} | 74 +-
tools/util/schema-replace.c | 4 +-
tools/util/schema-replace.vers | 2 +-
tools/util/schema-replace.vers.h | 2 +-
tools/util/sortreadtest.c | 6 +-
tools/util/test-sra.c | 1129 ++++-
tools/util/test-sra.vers | 2 +-
tools/util/test-sra.vers.h | 2 +-
tools/util/testld.c | 4 +-
tools/util/testld.vers | 2 +-
tools/util/txt2kdb.c | 8 +-
tools/util/txt2kdb.vers | 2 +-
tools/util/vdb-lock.c | 2 +-
tools/util/vdb-lock.vers | 2 +-
tools/util/vdb-lock.vers.h | 2 +-
tools/util/vdb-passwd.c | 46 +-
tools/util/vdb-passwd.vers | 2 +-
tools/util/vdb-passwd.vers.h | 2 +-
tools/util/vdb-unlock.c | 2 +-
tools/util/vdb-unlock.vers | 2 +-
tools/util/vdb-unlock.vers.h | 2 +-
tools/vcf-loader/.gitignore | 1 +
tools/vcf-loader/Makefile | 9 +-
tools/vcf-loader/vcf-database.c | 5 +-
tools/vcf-loader/vcf-grammar.c | 2 +-
tools/vcf-loader/vcf-grammar.y | 2 +-
tools/vcf-loader/vcf-lex.c | 50 +-
tools/vcf-loader/vcf-lex.l | 12 +-
tools/vcf-loader/vcf-load.vers | 2 +-
tools/vcf-loader/vcf-parse.h | 5 +-
tools/vdb-config/configure.cpp | 56 +-
tools/vdb-config/interactive.cpp | 64 +-
tools/vdb-config/util.cpp | 93 +-
tools/vdb-config/util.hpp | 7 +-
tools/vdb-config/vdb-config-model.cpp | 16 +-
tools/vdb-config/vdb-config-model.hpp | 75 +-
tools/vdb-config/vdb-config.c | 921 ++--
tools/vdb-config/vdb-config.vers | 2 +-
tools/vdb-config/vdb-config.vers.h | 2 +-
tools/vdb-copy/Makefile | 2 -
tools/vdb-copy/context.c | 17 +-
tools/vdb-copy/context.h | 5 +-
tools/vdb-copy/definitions.h | 4 +
tools/vdb-copy/num-gen.c | 4 +-
tools/vdb-copy/progressbar.c | 137 -
tools/vdb-copy/progressbar.h | 70 -
tools/vdb-copy/type_matcher.c | 12 +-
tools/vdb-copy/vdb-copy.c | 50 +-
tools/vdb-copy/vdb-copy.vers | 2 +-
tools/vdb-copy/vdb-copy.vers.h | 2 +-
tools/vdb-decrypt/shared.c | 17 +-
tools/vdb-decrypt/vdb-decrypt.vers | 2 +-
tools/vdb-decrypt/vdb-decrypt.vers.h | 2 +-
tools/vdb-decrypt/vdb-encrypt.vers | 2 +-
tools/vdb-decrypt/vdb-encrypt.vers.h | 2 +-
tools/{sra-dbcc => vdb-diff}/Makefile | 33 +-
tools/vdb-diff/coldefs.c | 211 +
.../vdb-dump-num-gen.h => vdb-diff/coldefs.h} | 87 +-
tools/vdb-diff/namelist_tools.c | 299 ++
.../sg_lookup.h => vdb-diff/namelist_tools.h} | 40 +-
tools/vdb-diff/vdb-diff.c | 1017 ++++
tools/vdb-diff/vdb-diff.vers | 1 +
tools/vdb-diff/vdb-diff.vers.h | 1 +
tools/vdb-dump/Makefile | 1 -
tools/vdb-dump/help.txt | 194 +-
tools/vdb-dump/vdb-boot.vers | 2 +-
tools/vdb-dump/vdb-dump-bin.c | 397 +-
tools/vdb-dump/vdb-dump-bin.h | 2 +
tools/vdb-dump/vdb-dump-coldefs.c | 180 +-
tools/vdb-dump/vdb-dump-coldefs.h | 5 +
tools/vdb-dump/vdb-dump-context.c | 60 +-
tools/vdb-dump/vdb-dump-context.h | 26 +-
tools/vdb-dump/vdb-dump-fastq.c | 958 ++--
tools/vdb-dump/vdb-dump-formats.c | 58 +-
tools/vdb-dump/vdb-dump-helper.c | 118 +-
tools/vdb-dump/vdb-dump-helper.h | 5 +-
tools/vdb-dump/vdb-dump-num-gen.c | 477 --
tools/vdb-dump/vdb-dump-row-context.h | 2 +-
tools/vdb-dump/vdb-dump-str.c | 160 +
tools/vdb-dump/vdb-dump-str.h | 5 +
tools/vdb-dump/vdb-dump-tools.c | 7 +-
tools/vdb-dump/vdb-dump-tools.h | 2 +-
tools/vdb-dump/vdb-dump.c | 778 +++-
tools/vdb-dump/vdb-dump.vers | 2 +-
tools/vdb-dump/vdb-dump.vers.h | 2 +-
tools/vdb-dump/vdb_info.c | 462 +-
tools/vdb-dump/vdb_info.h | 2 +-
tools/vdb-validate/vdb-validate.c | 92 +-
tools/vdb-validate/vdb-validate.vers | 2 +-
tools/vdb-validate/vdb-validate.vers.h | 2 +-
tools/vdbview/b_first.png | Bin 528 -> 0 bytes
tools/vdbview/b_goto.png | Bin 475 -> 0 bytes
tools/vdbview/b_last.png | Bin 519 -> 0 bytes
tools/vdbview/b_next.png | Bin 428 -> 0 bytes
tools/vdbview/b_nextpage.png | Bin 639 -> 0 bytes
tools/vdbview/b_prev.png | Bin 426 -> 0 bytes
tools/vdbview/b_prevpage.png | Bin 636 -> 0 bytes
tools/vdbview/b_search.png | Bin 614 -> 0 bytes
tools/vdbview/background.pas | 277 --
tools/vdbview/buttons.svg | 268 --
tools/vdbview/columnsel.lfm | 90 -
tools/vdbview/columnsel.pas | 115 -
tools/vdbview/dataproxy.pas | 602 ---
tools/vdbview/gotoform.lfm | 56 -
tools/vdbview/gotoform.pas | 47 -
tools/vdbview/jobdef.pas | 27 -
tools/vdbview/linkedlist.pas | 186 -
tools/vdbview/logo_128.png | Bin 1752 -> 0 bytes
tools/vdbview/logo_16.png | Bin 368 -> 0 bytes
tools/vdbview/logo_256.png | Bin 3524 -> 0 bytes
tools/vdbview/logo_32.png | Bin 541 -> 0 bytes
tools/vdbview/logo_64.png | Bin 910 -> 0 bytes
tools/vdbview/mainwindow.lfm | 973 ----
tools/vdbview/mainwindow.pas | 793 ----
tools/vdbview/searchform.lfm | 69 -
tools/vdbview/searchform.pas | 79 -
tools/vdbview/selector_types.pas | 20 -
tools/vdbview/support_unit.pas | 217 -
tools/vdbview/svdb.pas | 676 ---
tools/vdbview/svdb_obj.pas | 339 --
tools/vdbview/vdbview.ico | Bin 25890 -> 0 bytes
tools/vdbview/vdbview.lpi | 343 --
tools/vdbview/vdbview.lpr | 24 -
tools/vdbview/vdbview.res | Bin 27156 -> 0 bytes
tools/vdbview/vdbview1.ico | Bin 25890 -> 0 bytes
tools/vdbview/version.lfm | 40 -
tools/vdbview/version.pas | 33 -
tools/vtblcp/vtblcp.c | 979 ----
tools/vtblcp/vtblcp.vers | 1 -
tools/workspace/.gitignore | 3 -
tools/workspace/path-to-kar-file.cpp | 117 -
tools/workspace/prefs-lex.l | 181 -
tools/workspace/prefs-yacc.y | 189 -
tools/workspace/sra.cpp | 1514 ------
tools/workspace/sra.vers | 1 -
tools/workspace/sratoolkit-exception.cpp | 126 -
tools/workspace/sratoolkit-exception.hpp | 111 -
tools/workspace/token.c | 228 -
tools/workspace/token.h | 99 -
tools/workspace/workspace.cpp | 1116 -----
win-configure.bat | 1 -
753 files changed, 58090 insertions(+), 42735 deletions(-)
diff --git a/.gitignore b/.gitignore
index a76dbe0..400eae0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,18 @@
-/win
+.DS_Store
+._*
+*.d
+*.db
+*.o
+*.pyc
+/bin64
+/build/BUILD.*
+/build/COMP.mac
/centos
+/ilib64
+/lib64
/mac
/Makefile.config.*
+/reconfigure
+/schema
+/test-bin64
+/win
diff --git a/GNU-license.txt b/GNU-license.txt
deleted file mode 100644
index 818433e..0000000
--- a/GNU-license.txt
+++ /dev/null
@@ -1,674 +0,0 @@
- GNU GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
- Preamble
-
- The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users. We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors. You can apply it to
-your programs, too.
-
- When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
- To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights. Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
- For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received. You must make sure that they, too, receive
-or can get the source code. And you must show them these terms so they
-know their rights.
-
- Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
- For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software. For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
- Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so. This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software. The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable. Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products. If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
- Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary. To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
- The precise terms and conditions for copying, distribution and
-modification follow.
-
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Use with the GNU Affero General Public License.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time. Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
-
- If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
- To do so, attach the following notices to the program. It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
- <one line to give the program's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
- If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
- <program> Copyright (C) <year> <name of author>
- This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
- This is free software, and you are welcome to redistribute it
- under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License. Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
- You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
- The GNU General Public License does not permit incorporating your program
-into proprietary programs. If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library. If this is what you want to do, use the GNU Lesser General
-Public License instead of this License. But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..8987076
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,66 @@
+CONTENTS
+
+ Public Domain Notice
+ Exceptions (for bundled 3rd-party code)
+ Copyright F.A.Q.
+
+
+==============================================================
+ PUBLIC DOMAIN NOTICE
+ National Center for Biotechnology Information
+
+With the exception of certain third-party files summarized below, this
+software is a "United States Government Work" under the terms of the
+United States Copyright Act. It was written as part of the authors'
+official duties as United States Government employees and thus cannot
+be copyrighted. This software is freely available to the public for
+use. The National Library of Medicine and the U.S. Government have not
+placed any restriction on its use or reproduction.
+
+Although all reasonable efforts have been taken to ensure the accuracy
+and reliability of the software and data, the NLM and the U.S.
+Government do not and cannot warrant the performance or results that
+may be obtained by using this software or data. The NLM and the U.S.
+Government disclaim all warranties, express or implied, including
+warranties of performance, merchantability or fitness for any
+particular purpose.
+
+Please cite the authors in any work or product based on this material.
+
+
+==============================================================
+EXCEPTIONS (in all cases excluding NCBI-written makefiles):
+
+See LICENSE from https://github.com/ncbi/ncbi-vdb
+
+==============================================================
+Copyright F.A.Q.
+
+
+--------------------------------------------------------------
+Q. Our product makes use of the NCBI source code, and we made changes
+ and additions to that version of the NCBI code to better fit it to
+ our needs. Can we copyright the code, and how?
+
+A. You can copyright only the *changes* or the *additions* you made to the
+ NCBI source code. You should identify unambiguously those sections of
+ the code that were modified, e.g. by commenting any changes you made
+ in the code you distribute. Therefore, your license has to make clear
+ to users that your product is a combination of code that is public domain
+ within the U.S. (but may be subject to copyright by the U.S. in foreign
+ countries) and code that has been created or modified by you.
+
+--------------------------------------------------------------
+Q. Can we (re)license all or part of the NCBI source code?
+
+A. No, you cannot license or relicense the source code written by NCBI
+ since you cannot claim any copyright in the software that was developed
+ at NCBI as a 'government work' and consequently is in the public domain
+ within the U.S.
+
+--------------------------------------------------------------
+Q. What if these copyright guidelines are not clear enough or are not
+ applicable to my particular case?
+
+A. Contact us. Send your questions to 'sra-tools at ncbi.nlm.nih.gov'.
+
diff --git a/Makefile b/Makefile
index 124aa79..3410087 100644
--- a/Makefile
+++ b/Makefile
@@ -61,10 +61,15 @@ $(SUBDIRS_STD):
#-------------------------------------------------------------------------------
# install
#
-install: std
- $(MAKE) -s TOP=$(CURDIR) -f build/Makefile.install install
+install:
+ @ echo "Checking make status of tools..."
+ @ $(MAKE) -s --no-print-directory TOP=$(CURDIR) std
+ @ $(MAKE) -s TOP=$(CURDIR) -f build/Makefile.install install
-.PHONY: install
+uninstall:
+ @ $(MAKE) -s TOP=$(CURDIR) -f build/Makefile.install uninstall
+
+.PHONY: install uninstall
#-------------------------------------------------------------------------------
# clean
@@ -128,7 +133,7 @@ $(REPORTS):
# configuration help
#
help configure:
- @ echo "Before initial build, run 'make OUTDIR=<dir> out' from"
+ @ echo "Before initial build, run './configure --build-prefix=<out>' from"
@ echo "the project root to set the output directory of your builds."
@ echo
@ echo "To select a compiler, run 'make <comp>' where"
diff --git a/README b/README
index 6abcab1..ac593f7 100644
--- a/README
+++ b/README
@@ -27,36 +27,29 @@ The NCBI SRA ( Sequence Read Archive )
Contact: sra-tools at ncbi.nlm.nih.gov
-http://trace.ncbi.nlm.nih.gov/Traces/sra/std
-
-The SRA Toolkit and SDK from NCBI is a collection of tools and
-libraries for using data in the INSDC Sequence Read Archives.
-
-With this release, NCBI has implemented Compression by Reference, a
-sequence alignment compression process for storing sequence data.
-Currently BAM, Complete Genomics and Illumina export.txt formats
-contain alignment information. Compression by Reference only stores
-the difference in base pairs between sequence data and the segments it
-aligns to. The decompression process to restore original data such as
-fastq-dump would require fast access to the actual sequences of the
-references. NCBI recommends that SRA users dedicate local disk space
-to store local references downloaded from the NCBI SRA site. Linked
-references should be in a location accessible by the SRA Reader
-software.
-
-Older files in the NCBI system may not have been compressed using
-Compression by Reference. For more information on how to use
-Reference-based compressed files, download local references, and use
-related tools please refer to Compression by Reference file on the
-NCBI SRA website:
-
- "http://trace.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=softwareReadme"
-
-
-For additional information on using and building the toolkit,
-please visit our web site at:
-
- "http://trace.ncbi.nlm.nih.gov/Traces/sra/std"
+SRA Tools web site: http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc
+Download page: https://github.com/ncbi/sra-tools/wiki/Downloads
+
+The SRA Toolkit and SDK from NCBI is a collection of tools and libraries for
+using data in the INSDC Sequence Read Archives.
+
+Much of the data submitted these days contain alignment information, for example
+in BAM, Illumina export.txt, and Complete Genomics formats. With aligned data,
+NCBI uses Compression by Reference, which only stores the differences in base
+pairs between sequence data and the segment it aligns to. The process to
+restore original data, for example as FastQ, requires fast access to the
+reference sequences that the original data was aligned to. NCBI recommends that
+SRA users dedicate local disk space to store references downloaded from the NCBI
+SRA site. As of February 2015, the complete collection of these reference sequences
+is 98 GB. While it isn't usually necessary to download the entirety of the
+reference sequences, this should give you an idea of the scale of the storage
+requirement. By default, the Toolkit will download missing reference sequences
+on demand and cache them in the user's home directory. The location of this
+cache is configurable, as is whether the download is automatic or manual.
+
+For additional information on using, configuring, and building the toolkit,
+please visit our wiki: https://github.com/ncbi/sra-tools/wiki
+or our web site at NCBI: http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc
SRA Toolkit Development Team
diff --git a/README-WINDOWS.txt b/README-WINDOWS.txt
deleted file mode 100644
index fc27bf0..0000000
--- a/README-WINDOWS.txt
+++ /dev/null
@@ -1,144 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-The NCBI SRA ( Sequence Read Archive ) SDK ( Software Development Kit )
-
-
-Contact: sra-tools at ncbi.nlm.nih.gov
-
-
-NOTICE:
-
- Effective as of release 2.1.8, NCBI is no longer supporting public sources for
- Windows builds. The sources will still contain everything that we use to build
- the binaries, but may not build in your environment.
-
- The reason for this change has to do with dependencies upon third party
- libraries, which are commonly installed on other platforms, but could be
- described as "hit and miss" on Windows. When the libraries DO exist, it is
- difficult to know if they have a correct or compatible calling convention.
-
- We will continue to distribute pre-built binaries for Windows. You are welcome
- to try your luck at compiling the sources yourself under Cygwin.
-
-
-ENVIRONMENT:
-
- The Windows build uses the same makefiles as Linux and Mac, and has
- been tested under Cygwin. You need to execute Cygwin AFTER sourcing
- the Microsoft batch file from Visual Studio. We edit the
- "cygwin.bat" file to first source "vsvars32.bat":
-
- @echo off
- call "%VS80COMNTOOLS%\vsvars32.bat"
- C:
- chdir C:\cygwin\bin
- bash --login -i
-
- By including vsvars32.bat before launching Cygwin, your bash shell
- will have all of the Microsoft tools in its path, and the Microsoft
- tools will know how to find includes and libraries.
-
- There is a conflict between the Cygwin and Microsoft link tools. The
- GNU version (that "avoids the bells and whistles of the more
- commonly-used `ln' command" but otherwise duplicates it) is not used
- or needed by the build, while the Microsoft version IS needed. To
- address this issue, our linker scripts (build/ld.win.*.sh) reorder
- the PATH directories in an attempt to ensure that the correct tool
- is located. If this proves ineffective in your environment, try
- renaming the GNU tool, e.g.:
-
- $ mv /usr/bin/link.exe /usr/bin/gnu-link.exe
-
-
-CYGWIN:
-
- While we are using Cygwin as a build environment, the binaries are
- NOT Cygwin binaries and do not link against any of their
- libraries. As a result, there is no dependency upon their runtime
- being present, but neither is there any of their emulation.
-
- In particular, paths behave very differently. Our SDK is based upon
- POSIX path conventions. Our Windows tools accept relative and
- absolute paths well, in DOS, Cygwin or MinGW POSIX-style form. But
- since we do not use Cygwin libraries, our tools have no idea of how
- to interpret "full" paths within the Cygwin Unix root, e.g.:
-
- $ cygpath -w /home
- C:\cygwin\home
-
- They CAN, however, interpret fully drive-letter qualified Cygwin
- paths:
-
- $ fastq-dump /cygdrive/C/cygwin/home/myname/SRRxxxxxx.sra
-
- Internally, this Cygwin path will be interpreted as
-
- C:\cygwin\home\myname\SRRxxxxxx.sra
-
-
-WINDOWS PATHS IN GENERAL:
-
- Windows is the only supported platform that does not present a
- single-root unified file system, such as POSIX. In the POSIX model,
- navigation from any one point in the file system to any other is a
- matter of entering and exiting directories by name. Under Windows,
- each drive has its own file system, and network paths form yet
- another file system.
-
- Like Cygwin and MinGW and probably countless other projects, we have
- artificially bridged the drive letter file systems by introducing a
- virtual root node whose immediate sub-directories are the
- single-letter drive names that are currently mounted, e.g.:
-
- C:, D: are real drives, while Z: is a mounted network drive
- "/C" and "/D" and "/Z" are all subdirectories of "/"
-
- navigation from C:\cygwin\home to D:\data
- /C/cygwin/home/../../../D/data
-
- We have NOT bridged drive letters and network paths at this
- point. There is no way for us to get from C:\cygwin\home to
- \\server\sradata for example.
-
- Network paths are represented using POSIX-style slashes, but
- otherwise look much like their Windows counterparts:
-
- \\server\data
- //server/data
-
- Because network and drive paths cannot be connected, we recommend
- that you execute your tools completely within one space. If you are
- running your tools from local or network mounted drive letters, you
- should access data from local or network mounted drives. If you are
- using network paths during operation, you should run your binaries
- from a network path.
-
- Note that the Windows tool "cmd.exe" does not support cd'ing to a
- network path for similar reasons.
-
- Your tools will execute from "cmd.exe" or from bash under Cygwin
- since they are general Windows command line utilities.
-
diff --git a/README-build b/README-build
deleted file mode 100644
index fe1d722..0000000
--- a/README-build
+++ /dev/null
@@ -1,237 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-The NCBI SRA ( Sequence Read Archive ) SDK ( Software Development Kit )
-
-
-Contact: sra-tools at ncbi.nlm.nih.gov
-http://trace.ncbi.nlm.nih.gov/Traces/sra/std
-
-
-This version of the NCBI SRA SDK generates loading and dumping tools with
-their respective libraries for building new and accessing existing runs.
-It may be built with GCC, ICC or Microsoft VC++.
-
-
-REQUIREMENTS:
-
-This software release was designed to run under Linux, MacOSX and Windows
-operating systems on Intel x86-compatible 32 and 64 bit architectures.
-
- ar # tested with version 2.15.90
- bash # certain scripts require bash
- make # GNU make version 3.80 or later
- gcc, g++ # tested with 4.1.2, but should work with others
- libz # version 1
- libbz2 # version 1
- libxml2 # tested with version 2.6.7 [Linux and Mac only]
-
-If your system does NOT have libz or libbz2, or if the build fails due to
-missing one of the expected libraries, try running "make all" which will
-attempt to download the sources to libz and libbz2 and build them.
-
-OPTIONS:
-
-Specific versions of ICC are supported as an alternate compiler.
-
- icc, icpc # tested with 11.0 (64-bit) and 10.1 (32-bit)
- # 32-bit 11.0 does not work
-
-WINDOWS BUILD:
-
-The Windows build uses the same makefiles as Linux and Mac, and has been tested
-under Cygwin. You need to execute Cygwin AFTER sourcing the Microsoft batch file
-from Visual Studio.
-
-
-CONTENTS:
-
- CHANGES # describes changes at pertinent levels
- Makefile # drives configuration and sub-target builds
- README
- README-WINDOWS.txt
- USAGE
- build # holds special makefiles and configuration
- interfaces # contains module interfaces, schema, plus
- compiler and platform specific includes
- libs # sdk library code
- tools # toolkit code
- test # testing code
-
-
-CONFIGURATION:
-
-There are three configurable parameters:
- 1) BUILD = 'debug', 'release' etc.
- 2) COMP = 'GCC' etc.
- 3) OUTDIR = <path-to-binaries-libs-objfiles>
-
-The target architecture is chosen to match your build host. At this
-time, only the Macintosh build will support cross-compilation. In the
-instructions below, x86_64 is the assumed architecture. If your host
-is i386 (32-bit), then you would substitute 32 for paths that contain
-64.
-
-
-BUILD INSTRUCTIONS:
-
-## create output directories and symlinks for first time
-
- $ OUTDIR=<path-to-output>
- $ make OUTDIR="$OUTDIR" out
-
-The path in OUTDIR MUST be a full path - relative paths may fail.
-
-## decide upon STATIC or DYNAMIC builds
-# VDB.2 was designed to make use of dynamic libraries, but
-# in many environments static builds are more convenient or may even be
-# required, due to installation restrictions.
-#
-# THE BUILD DOES NOT CURRENTLY SUPPORT PARALLEL DYNAMIC AND STATIC MODES
-# if you switch between them, you should perform a "make clean" first.
-
- $ make static
--OR_
- $ make dynamic
-
-## if you are using a dynamic build, update LD_LIBRARY_PATH - probably want to put
-# in shell startup ( ensure that libz, libbz2 and libxml2 can be found in your path )
-
- $ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OUTDIR/lib64"
-
-## the build uses "TOP" as an internal variable
-# if you have it defined in your shell environment, unset it before making
-
-## build the libraries and binaries
-
- $ make GCC debug # optionally set compiler and/or build
- $ make # builds libraries and binary executables
-
-The standard make will cause default libraries and tools to be built. Binary
-executables and both shared and static libraries can be found in their
-respective output directories.
-
-If the build fails due to not finding libz or libbz2, try:
-
- $ make all # see note above on libraries
-
-
-STATIC BUILD RESULTS:
-
-<OUTDIR>/bin64/ # (bin32 for 32-bit builds)
- abi-dump # dump to ABI-native format
- abi-load # load from ABI-native format
- align-info # retrieve information from cSRA object
- fastq-dump # dump to FASTQ format
- fastq-load # load from FASTQ format
- helicos-load # load from Helicos native format
- illumina-dump # dump to Illumina-native format
- illumina-load # load from Illumina-native format
- kar # single file archive utility
- kdbmeta # access db, table or column metadata
- rcexplain # return code display utility
- sam-dump # dump to SAM format from cSRA
- sff-dump # dumps 454 runs in SFF format
- sff-load # load from SFF format
- sra-dbcc # consistency checker
- sra-kar # sra-specific kar tool
- sra-stat # gather run statistics and print to stdout
- srf-load # load from SRF format
- vdb-copy # tool to copy tables
- vdb-dump # dump rows in a textual format
- vdb-lock # locks an object against modification
- vdb-unlock # unlocks an object
-
-<OUTDIR>/bin64/ncbi # (bin32 for 32-bit builds)
- config.kfg # simple text file for configuring file system
-
-<OUTDIR>/lib64/ # (lib32 for 32-bit builds)
- libkdb # static physical layer reading library
- libkfg # static configuration library
- libkfs # static file system library
- libklib # static support library
- libkproc # static process synchronization library
- libkq # static cross-thread queue library
- libksproc # static single-threaded emulation library
- libksrch # static search algorithm library
- libkxfs # static XML to filesystem library
- libkxml # static XML container support library
- libload # static loader tool utility library
- libsradb # static API for accessing sra
- libsrareader # static reader library
- libsraschema # static version of SRA schema
- libvdb # static virtual layer reading library
- libwkdb # static physical layer update library
- libwsradb # static update API for SRA
- libwvdb # static virtual layer update library
-
-<OUTDIR>/mod64/ # (mod32 for 32-bit builds)
- -- empty --
-
-<OUTDIR>/wmod64/ # (wmod32 for 32-bit builds)
- -- empty --
-
-
-
-DYNAMIC BUILD RESULTS:
-
-<OUTDIR>/bin64/ # (bin32 for 32-bit builds)
- -- same as static --
-
-<OUTDIR>/bin64/ncbi # (bin32 for 32-bit builds)
- -- empty --
-
-<OUTDIR>/lib64/ # (lib32 for 32-bit builds)
- libkdb # physical layer reading library
- libkfg # configuration library
- libkfs # file system library
- libklib # support library
- libkproc # process synchronization library
- libkq # cross-thread queue library
- libksproc # single-threaded emulation library
- libksrch # search algorithm library
- libkxfs # XML to filesystem library
- libkxml # XML container support library
- libload # loader tool utility library
- libsra-schema # weak-linked sra schema
- libsradb # API for accessing sra
- libsrapath # weak-link stubs for srapath
- libsrareader # reader library
- libvdb # virtual layer reading library
- libwkdb # physical layer update library
- libwsradb # update API for SRA
- libwvdb # virtual layer update library
-
-<OUTDIR>/lib64/ncbi # (lib32 for 32-bit builds)
- config.kfg # simple text file for configuring file system
-
-<OUTDIR>/mod64/ # (mod32 for 32-bit builds)
- libaxf # cSRA-specific VDB external functions
- libsraxf # sra-specific VDB external functions
- libvxf # generic VDB external functions
-
-<OUTDIR>/wmod64/ # (wmod32 for 32-bit builds)
- libwsraxf # sra-specific VDB external functions for update
- libwvxf # update VDB external functions
diff --git a/README-config.txt b/README-config.txt
deleted file mode 100644
index 648f1f7..0000000
--- a/README-config.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-About configuration-assistant:
-configuration-assistant.perl will help you to configure the SRA tools to be able to access
-the local reference repository and to download the correct references for a given cSRA file.
-
-
-To run configuration-assistant:
-- open Command Prompt (Start->Run->cmd.exe),
-- change directory to the folder where you extracted the toolkit (cd ...\sratoolkit.2.1.7-win32),
-- type 'perl configuration-assistant.perl' (without the quotation marks).
-
-
-If the following message shows up:
-
- "'perl' is not recognized as an internal or external command,
- operable program or batch file."
-
-that means perl is not installed or not installed properly.
-
-Download Perl at ActiveState.com (http://www.activestate.com/activeperl/downloads)
diff --git a/README-vdb-config.txt b/README-vdb-config
similarity index 97%
rename from README-vdb-config.txt
rename to README-vdb-config
index b08ed96..46d1686 100644
--- a/README-vdb-config.txt
+++ b/README-vdb-config
@@ -26,10 +26,6 @@
The tool 'vdb-config' can be used to inspect or change the configuration
of the sra-toolkit.
-DISCLAIMER: This is pre-release (alpha) software. Some features are not
-yet fully functional in version 2.4.0 - this will be fixed in the next
-release.
-
When called without any parameters the tool reports the current configuration
in xml-format. No changes are made.
diff --git a/README-wget.txt b/README-wget.txt
deleted file mode 100644
index ddc8fa9..0000000
--- a/README-wget.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-MAINTAINER: Micah Cowan <micah at cowan.name>
-
-Wget was originally written and mainained by Hrvoje Niksic. Please see
-the file AUTHORS for a list of major contributors, and the ChangeLogs
-for a detailed listing of all contributions.
-
-
-Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004
-2005, 2006, 2007, 2008 Free Software Foundation, Inc.
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
-USA.
-
-Additional permission under GNU GPL version 3 section 7
-
-If you modify this program, or any covered work, by linking or
-combining it with the OpenSSL project's OpenSSL library (or a
-modified version of that library), containing parts covered by the
-terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
-grants you additional permission to convey the resulting work.
-Corresponding Source for a non-source form of such a combination
-shall include the source code for the parts of OpenSSL used as well
-as that of the covered work.
diff --git a/README.md b/README.md
index 9ce9409..fbb026a 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
### Contact:
email: sra-tools at ncbi.nlm.nih.gov
-or visit the [SRA Tools web site](http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc)
+or visit the [SRA Tools web site](http://ncbi.github.io/sra-tools) - UPDATED
### Download
Visit our [download page](https://github.com/ncbi/sra-tools/wiki/Downloads) for pre-built binaries.
@@ -18,8 +18,8 @@ pairs between sequence data and the segment it aligns to. The process to
restore original data, for example as FastQ, requires fast access to the
reference sequences that the original data was aligned to. NCBI recommends that
SRA users dedicate local disk space to store references downloaded from the NCBI
-SRA site. As of Fall 2014, the complete collection of these reference sequences
-is 66 GB. While it isn't usually necessary to download the entirety of the
+SRA site. As of February 2015, the complete collection of these reference sequences
+is 98 GB. While it isn't usually necessary to download the entirety of the
reference sequences, this should give you an idea of the scale of the storage
requirement. By default, the Toolkit will download missing reference sequences
on demand and cache them in the user's home directory. The location of this
@@ -27,7 +27,7 @@ cache is configurable, as is whether the download is automatic or manual.
For additional information on using, configuring, and building the toolkit,
please visit our [wiki](https://github.com/ncbi/sra-tools/wiki)
-or our web site at [NCBI](http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc&f=std)
+or our web site at [NCBI](http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc)
SRA Toolkit Development Team
diff --git a/USAGE b/USAGE
deleted file mode 100644
index 148122e..0000000
--- a/USAGE
+++ /dev/null
@@ -1,162 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-The NCBI SRA ( Sequence Read Archive ) Toolkit
-
-
-Contact: sra-tools at ncbi.nlm.nih.gov
-http://trace.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=software
-
-
-For preliminary documentation on tool usage, consult the files within
-the "help" directory.
-
-
-CHANGES:
-
-SRA Toolkit includes new features in sam-dump tool and vdb-dump tools.
-
-Sam-dump now supports slicing across multiple sequences, and dumping cSRA files to fasta and fastq formats. In addition, sam-dump has three new parameters
--=|--hide-identical Output '=' if base is identical to reference
---gzip Compress output using gzip
---bzip2 Compress output using bzip2
-
-vdb-dump has two new parameters
--o|--column_enum_short enumerates columns in short form
--b|--boolean defines how boolean's are printed (1,T)
-
-We have combined the functionality of two scripts, config-assistant.perl and reference-assistant.perl into a single script, configuration-assistant.perl that helps users download the correct references for a given cSRA file and configure the user environment for the SRA Toolkit.
-
-DESCRIPTION:
-
- This release includes tools for reading the SRA archive, generally
- by converting individual runs into some commonly used format such as
- fastq. Support for NCBI's Compression by Reference is also included.
-
- "Linux" binaries have been created on CentOS and SuSE Linux
- distributions. They are not guaranteed to work on other
- distributions. In particular, the version of libc.so should be
- compatible. They are specific to the x86-family architectures.
-
- "Windows" binaries have been created using MSVC's (not Cygwin)
- tools. This release includes Win32 binaries. The "*-load" tools
- are not released for "Windows".
-
- "Mac OS X" binaries are available for the x86-family
- architectures. They will NOT run on PPC Macs. The 64-bit binaries
- will run only on OS 10.6. The 32-bit binaries should run on either
- 10.5 or later.
-
-
-CONTENTS:
-
- "abi-dump" - dump ABI color-space runs into their native format
- "abi-load" - load ABI color-space runs
- "fastq-dump" - dump any run of any platform in FASTQ format
- "fastq-load" - load FASTQ runs
- "helicos-load" - load HELICOS runs
- "illumina-dump" - dump Illumina runs into their native format
- "illumina-load" - load Illumina native runs
- "kar" - an archive extraction tool for .sra files
- "kdbmeta" - display the contents of one or more metadata stores
- "rcexplain" - print out error string for return codes (RC)
- "refseq-load" - ( NEW for Linux and Mac ) download a reference
- sequence object
- "sff-dump" - dump 454 runs into SFF format
- "sff-load" - load 454 SFF runs
- "sra-dbcc" - check SRA runs
- "sra-dump" - dump any run in a textual format [see note below]
- "sra-kar" - creates a single file archive from an SRA run
- "sra-stat" - display run column statistics
- "srf-load" - load SRF runs
- "sam-dump" - dump a cSRA into the SAM-format
- "vdb-copy" - copy SRA objects
- "vdb-dump" - display SRA objects in a textual format [see note below]
- "vdb-lock" - lock an object against modification
- "vdb-unlock" - unlock an object
- "align-info" - displays which references a csra-archive uses
- "vdb-config" - displays the configuration
-
-
-NOTES:
-
- The "sam-dump" tool only works on cSRA's. ( See README-csra )
- The textual dumpers "sra-dump" and "vdb-dump" are provided in this
- release as an aid in visual inspection. It is likely that their
- actual output formatting will be changed in the near future to a
- stricter, more formalized representation[s].
-
- The "help" information will be improved in near future releases, and
- the tool options will become standardized across the set. We will
- also be providing documentation on our web site.
-
- Tool options may change in the next release. Version 1 tool options
- will remain supported wherever possible in order to preserve
- operation of any existing scripts.
-
-
-CAVEATS:
-
- SRA tools are designed to handle very large amounts of data, and are
- not currently oriented toward desktop use. They work well within any
- Unix-like environment, such as Linux or MacOS's BSD shell.
-
- Windows operation presents a few challenges. We have tested our
- binaries under the MS "cmd.exe" shell and Cygwin's bash. Our tools
- know how to accept paths in Windows, Cygwin, POSIX and MinGW
- formats. Internally, all paths are treated as MinGW-style POSIX
- paths, so any information appearing in output will reflect this
- conversion:
-
- # simple file names
- SRR012345.sra => [NO CHANGE]
-
- # relative paths
- win\SRR012345.sra => win/SRR012345.sra
-
- # full or drive-relative paths
- C:\sra\win\SRR012345.sra => /C/sra/win/SRR012345.sra
-
- # network paths
- \\server\sra\SRR012345.sra => //server/sra/SRR012345.sra
-
- # POSIX paths
- /sra/posix/SRR012345.sra => [NO CHANGE]
-
- # Cygwin full paths
- /cygdrive/C/sra/SRR012345.sra => /C/sra/SRR012345.sra
-
- There are some situations where the software may behave unexpectedly
- due to path conversions. In particular, since Windows does not
- conform to POSIX path conventions, there may be difficulties
- combining network paths with non-network paths, explicitly or
- implicitly. IF YOU HAVE PROBLEMS, we recommend mounting file servers
- as network drives.
-
- If you run the software under Cygwin, you should take care to use
- either relative paths (those that do not begin with '/') or full
- paths that start with "/cygdrive/". The reason is that Cygwin
- provides its own path manipulation, but our tools are not aware of
- which shell they are running under.
diff --git a/build/MSVC/2010/abi-dump.vcxproj b/build/MSVC/2010/abi-dump.vcxproj
index 772a068..0532f80 100644
--- a/build/MSVC/2010/abi-dump.vcxproj
+++ b/build/MSVC/2010/abi-dump.vcxproj
@@ -22,8 +22,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
diff --git a/build/MSVC/2010/read-filter-redact.vcxproj b/build/MSVC/2010/align-cache.vcxproj
old mode 100644
new mode 100755
similarity index 74%
copy from build/MSVC/2010/read-filter-redact.vcxproj
copy to build/MSVC/2010/align-cache.vcxproj
index e1d65d8..5c05aed
--- a/build/MSVC/2010/read-filter-redact.vcxproj
+++ b/build/MSVC/2010/align-cache.vcxproj
@@ -1,40 +1,36 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Debug|x64">
- <Configuration>Debug</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|x64">
- <Configuration>Release</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- </ItemGroup>
-
- <Import Project=".\exe-project.props" />
-
- <ItemDefinitionGroup>
- <Link>
- <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- </ItemDefinitionGroup>
-
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\rd-filter-redact\read-filter-redact.c" />
- </ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{B7C166D7-2B45-4E77-A608-DDBD0B588082}</ProjectGuid>
- </PropertyGroup>
-
-</Project>
\ No newline at end of file
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <Import Project=".\exe-project.props" />
+ <ItemDefinitionGroup>
+ <Link>
+ <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\tools\align-cache\align-cache.cpp" />
+ <ClCompile Include="..\..\..\tools\align-cache\helper.cpp" />
+ <ClInclude Include="..\..\..\tools\align-cache\helper.h" />
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{3C8CB988-94DC-4B07-8865-B91FC52395F8}</ProjectGuid>
+ </PropertyGroup>
+</Project>
diff --git a/build/MSVC/2010/align-info.vcxproj b/build/MSVC/2010/align-info.vcxproj
index e8465f5..31af6c6 100644
--- a/build/MSVC/2010/align-info.vcxproj
+++ b/build/MSVC/2010/align-info.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/cache-mgr.vcxproj b/build/MSVC/2010/cache-mgr.vcxproj
index 672f2c2..1a3207f 100644
--- a/build/MSVC/2010/cache-mgr.vcxproj
+++ b/build/MSVC/2010/cache-mgr.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -37,4 +35,4 @@
<ProjectGuid>{9BFEB5F1-0101-48A6-82CD-CF5744E1EDF3}</ProjectGuid>
</PropertyGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/build/MSVC/2010/fastq-dump-ngs.vcxproj b/build/MSVC/2010/dbgap-mount-demo.vcxproj
similarity index 71%
copy from build/MSVC/2010/fastq-dump-ngs.vcxproj
copy to build/MSVC/2010/dbgap-mount-demo.vcxproj
index 35ed060..47b4294 100644
--- a/build/MSVC/2010/fastq-dump-ngs.vcxproj
+++ b/build/MSVC/2010/dbgap-mount-demo.vcxproj
@@ -25,20 +25,19 @@
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\ngs-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(NGS_TARGET)bin\libngs-sdk.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\kapp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\libxfs.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>c:\Program Files (x86)\Dokan\DokanLibrary\dokan.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>Kernel32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
- <ClCompile Include="..\..\..\tools\fastq-dump\args.cpp" />
- <ClCompile Include="..\..\..\tools\fastq-dump\fastq-dump.cpp" />
- <ClCompile Include="..\..\..\tools\fastq-dump\filters.cpp" />
+ <ClCompile Include="..\..\..\tools\dbgap-mount\demo.c" />
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{8DA91267-49B1-4696-AF9D-2A98B697167E}</ProjectGuid>
</PropertyGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/build/MSVC/2010/dll-project.props b/build/MSVC/2010/dll-project.props
index 8150a3b..eb92be1 100644
--- a/build/MSVC/2010/dll-project.props
+++ b/build/MSVC/2010/dll-project.props
@@ -4,71 +4,12 @@
<PropertyGroup Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
- <CharacterSet>Unicode</CharacterSet>
<TargetExt>.dll</TargetExt>
<OutDir>$(VDB_TARGET)\bin\</OutDir>
- <IntDir>$(VDB_TARGET)\obj\$(ProjectName)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <LinkIncremental>false</LinkIncremental>
- </PropertyGroup>
-
- <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <UseDebugLibraries>false</UseDebugLibraries>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <UseDebugLibraries>false</UseDebugLibraries>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
-
-
- <ItemDefinitionGroup>
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <CompileAsManaged>false</CompileAsManaged>
- <MinimalRebuild>true</MinimalRebuild>
- <DisableLanguageExtensions>false</DisableLanguageExtensions>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- <DisableSpecificWarnings>4996;4101</DisableSpecificWarnings>
- <AdditionalIncludeDirectories>$(NGS_ROOT);$(VDB_ROOT)libs/$(ProjectName)/win;$(VDB_ROOT)libs/$(ProjectName);$(VDB_ROOT)interfaces;$(VDB_ROOT)interfaces/os/win;$(VDB_ROOT)interfaces/cc/vc++;$(VDB_ROOT)interfaces/ext;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>NO_KRSRC;WINDOWS;_WIN32_WINNT=0x0502;_WINDOWS;_LIBRARY;__mod_name__=libs/$(ProjectName);__file_name__=%(Filename);__file_ext__=c;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- </ItemDefinitionGroup>
-
- <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
- <PreprocessorDefinitions>_DEBUGGING;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
- <ClCompile>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
- <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>false</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
-
<ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
<ClCompile>
<AdditionalIncludeDirectories>$(VDB_ROOT)interfaces/cc/vc++/i386;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
diff --git a/build/MSVC/2010/exe-project.props b/build/MSVC/2010/exe-project.props
index 6a6c6cb..b6b271a 100644
--- a/build/MSVC/2010/exe-project.props
+++ b/build/MSVC/2010/exe-project.props
@@ -2,56 +2,21 @@
<Import Project="vdb-common.props" />
- <ItemDefinitionGroup>
- <Link>
- <AdditionalDependencies>$(VDB_TARGET)lib\kapp.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- </ItemDefinitionGroup>
-
<PropertyGroup Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <UseDebugLibraries>false</UseDebugLibraries>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <UseDebugLibraries>false</UseDebugLibraries>
- </PropertyGroup>
-
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
-
- <PropertyGroup>
<TargetExt>.exe</TargetExt>
<OutDir>$(VDB_TARGET)\bin\</OutDir>
- <IntDir>$(VDB_TARGET)\obj\$(ProjectName)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <LinkIncremental>false</LinkIncremental>
</PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+
<ItemDefinitionGroup>
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <CompileAsManaged>false</CompileAsManaged>
- <MinimalRebuild>true</MinimalRebuild>
- <DisableLanguageExtensions>false</DisableLanguageExtensions>
- <AdditionalIncludeDirectories>$(NGS_ROOT);$(VDB_ROOT)libs/$(ProjectName)/win;$(VDB_ROOT)libs/$(ProjectName);$(VDB_ROOT)interfaces;$(VDB_ROOT)interfaces/os/win;$(VDB_ROOT)interfaces/cc/vc++;$(VDB_ROOT)interfaces/ext;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WINDOWS;__mod_name__=test/$(ProjectName);__file_name__=%(Filename);__file_ext__=c;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- <DisableSpecificWarnings>4996;4101</DisableSpecificWarnings>
- </ClCompile>
<Link>
<EntryPointSymbol>wmainCRTStartup</EntryPointSymbol>
<AdditionalDependencies>Ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\kapp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<PostBuildEvent>
<Command>del $(TargetDir)$(TargetName).lib
@@ -59,31 +24,6 @@ del $(TargetDir)$(TargetName).exp</Command>
</PostBuildEvent>
</ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
- <PreprocessorDefinitions>_DEBUGGING;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
- <ClCompile>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
- <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>false</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
-
<ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
<ClCompile>
<AdditionalIncludeDirectories>$(VDB_ROOT)interfaces/cc/vc++/i386;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
diff --git a/build/MSVC/2010/fastq-dump-ngs.vcxproj b/build/MSVC/2010/fastq-dump-ngs.vcxproj
index 35ed060..90677df 100644
--- a/build/MSVC/2010/fastq-dump-ngs.vcxproj
+++ b/build/MSVC/2010/fastq-dump-ngs.vcxproj
@@ -24,10 +24,9 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\ngs-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(NGS_TARGET)bin\libngs-sdk.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(NGS_TARGET)lib\libngs-bind-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(NGS_TARGET)lib\libngs-disp.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/fastq-dump.vcxproj b/build/MSVC/2010/fastq-dump.vcxproj
index 0326e61..2fd716a 100644
--- a/build/MSVC/2010/fastq-dump.vcxproj
+++ b/build/MSVC/2010/fastq-dump.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
diff --git a/build/MSVC/2010/fastq-loader.vcxproj b/build/MSVC/2010/fastq-loader.vcxproj
index c9fb8e9..0c58b66 100644
--- a/build/MSVC/2010/fastq-loader.vcxproj
+++ b/build/MSVC/2010/fastq-loader.vcxproj
@@ -19,13 +19,15 @@
</ProjectConfiguration>
</ItemGroup>
+ <PropertyGroup>
+ <TargetName>latf-load</TargetName>
+ </PropertyGroup>
+
<Import Project=".\exe-project.props" />
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\loader.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
diff --git a/build/MSVC/2010/kdbmeta.vcxproj b/build/MSVC/2010/general-loader.vcxproj
similarity index 68%
copy from build/MSVC/2010/kdbmeta.vcxproj
copy to build/MSVC/2010/general-loader.vcxproj
index b31f4b0..9e7afa4 100644
--- a/build/MSVC/2010/kdbmeta.vcxproj
+++ b/build/MSVC/2010/general-loader.vcxproj
@@ -18,23 +18,23 @@
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
-
<Import Project=".\exe-project.props" />
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\util\kdbmeta.c" />
+ <ItemGroup>
+ <ClCompile Include="..\..\..\tools\general-loader\general-loader.cpp" />
+ <ClCompile Include="..\..\..\tools\general-loader\database-loader.cpp" />
+ <ClCompile Include="..\..\..\tools\general-loader\protocol-parser.cpp" />
+ <ClCompile Include="..\..\..\tools\general-loader\utf8-like-int-codec.c" />
+ <ClCompile Include="..\..\..\tools\general-loader\main.cpp" />
</ItemGroup>
-
<PropertyGroup Label="Globals">
- <ProjectGuid>{D2B898E3-DDBB-4704-A932-AED218DABE18}</ProjectGuid>
+ <ProjectGuid>{11356540-C8F0-44B9-B6BA-01C87443A1CF}</ProjectGuid>
</PropertyGroup>
-
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/illumina-diump.vcxproj b/build/MSVC/2010/illumina-dump.vcxproj
similarity index 100%
rename from build/MSVC/2010/illumina-diump.vcxproj
rename to build/MSVC/2010/illumina-dump.vcxproj
diff --git a/build/MSVC/2010/kar.vcxproj b/build/MSVC/2010/kar.vcxproj
index a102266..705a16e 100644
--- a/build/MSVC/2010/kar.vcxproj
+++ b/build/MSVC/2010/kar.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/kdbmeta.vcxproj b/build/MSVC/2010/kdbmeta.vcxproj
index b31f4b0..20ee02c 100644
--- a/build/MSVC/2010/kdbmeta.vcxproj
+++ b/build/MSVC/2010/kdbmeta.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/lib-project.props b/build/MSVC/2010/lib-project.props
index 3b2d729..edd9e1e 100644
--- a/build/MSVC/2010/lib-project.props
+++ b/build/MSVC/2010/lib-project.props
@@ -4,77 +4,11 @@
<PropertyGroup Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
- <CharacterSet>Unicode</CharacterSet>
- <TargetExt>.lib</TargetExt>
- </PropertyGroup>
-
- <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
- <WholeProgramOptimization>false</WholeProgramOptimization>
- <UseDebugLibraries>true</UseDebugLibraries>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <UseDebugLibraries>false</UseDebugLibraries>
- </PropertyGroup>
-
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-
- <ImportGroup Label="ExtensionSettings" />
-
- <ImportGroup Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
-
- <PropertyGroup Label="UserMacros" />
-
- <PropertyGroup>
<TargetExt>.lib</TargetExt>
<OutDir>$(VDB_TARGET)\lib\</OutDir>
- <IntDir>$(VDB_TARGET)\obj\$(ProjectName)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <LinkIncremental>false</LinkIncremental>
</PropertyGroup>
- <ItemDefinitionGroup>
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <CompileAsManaged>false</CompileAsManaged>
- <MinimalRebuild>true</MinimalRebuild>
- <DisableLanguageExtensions>false</DisableLanguageExtensions>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- <DisableSpecificWarnings>4996;4101</DisableSpecificWarnings>
- <AdditionalIncludeDirectories>$(NGS_ROOT);$(VDB_ROOT)libs/$(ProjectName)/win;$(VDB_ROOT)libs/$(ProjectName);$(VDB_ROOT)interfaces;$(VDB_ROOT)interfaces/os/win;$(VDB_ROOT)interfaces/cc/vc++;$(VDB_ROOT)interfaces/ext;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>NO_KRSRC;_WIN32_WINNT=0x0502;WINDOWS;_WINDOWS;_LIBRARY;__mod_name__=libs/$(ProjectName);__file_name__=%(Filename);__file_ext__=c;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <EntryPointSymbol>wmainCRTStartup</EntryPointSymbol>
- </Link>
- </ItemDefinitionGroup>
-
- <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
- <PreprocessorDefinitions>_DEBUGGING;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
- <ClCompile>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
- <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>false</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
<ClCompile>
diff --git a/build/MSVC/2010/md5cp.vcxproj b/build/MSVC/2010/md5cp.vcxproj
index 34c2cc9..e44b81f 100644
--- a/build/MSVC/2010/md5cp.vcxproj
+++ b/build/MSVC/2010/md5cp.vcxproj
@@ -23,8 +23,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -36,4 +34,4 @@
<ProjectGuid>{F14519B5-5698-4363-A734-229F2A01153D}</ProjectGuid>
</PropertyGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/build/MSVC/2010/prefetch.vcxproj b/build/MSVC/2010/prefetch.vcxproj
index 6f5696f..b491b77 100644
--- a/build/MSVC/2010/prefetch.vcxproj
+++ b/build/MSVC/2010/prefetch.vcxproj
@@ -24,17 +24,16 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\tools\prefetch\prefetch.c" />
+ <ClCompile Include="..\..\..\tools\prefetch\kfile-no-q.c" />
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{2CF4BF04-4C4D-442E-A7AD-0AE5EF87C733}</ProjectGuid>
</PropertyGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/build/MSVC/2010/qual-recalib-stat.vcxproj b/build/MSVC/2010/qual-recalib-stat.vcxproj
index 64ef153..d3b56c8 100644
--- a/build/MSVC/2010/qual-recalib-stat.vcxproj
+++ b/build/MSVC/2010/qual-recalib-stat.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -33,8 +31,6 @@
<ClCompile Include="..\..\..\tools\qual-recalib-stat\columns.c" />
<ClCompile Include="..\..\..\tools\qual-recalib-stat\context.c" />
<ClCompile Include="..\..\..\tools\qual-recalib-stat\namelist_tools.c" />
- <ClCompile Include="..\..\..\tools\qual-recalib-stat\num-gen.c" />
- <ClCompile Include="..\..\..\tools\qual-recalib-stat\progressbar.c" />
<ClCompile Include="..\..\..\tools\qual-recalib-stat\qual-recalib-stat.c" />
<ClCompile Include="..\..\..\tools\qual-recalib-stat\reader.c" />
<ClCompile Include="..\..\..\tools\qual-recalib-stat\ref_exclude.c" />
diff --git a/build/MSVC/2010/rcexplain.vcxproj b/build/MSVC/2010/rcexplain.vcxproj
index 0d1c3c3..7cada4e 100644
--- a/build/MSVC/2010/rcexplain.vcxproj
+++ b/build/MSVC/2010/rcexplain.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/read-filter-redact.vcxproj b/build/MSVC/2010/read-filter-redact.vcxproj
index e1d65d8..8b821ba 100644
--- a/build/MSVC/2010/read-filter-redact.vcxproj
+++ b/build/MSVC/2010/read-filter-redact.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/fastq-dump-ngs.vcxproj b/build/MSVC/2010/ref-variation.vcxproj
old mode 100644
new mode 100755
similarity index 59%
copy from build/MSVC/2010/fastq-dump-ngs.vcxproj
copy to build/MSVC/2010/ref-variation.vcxproj
index 35ed060..d64b3a1
--- a/build/MSVC/2010/fastq-dump-ngs.vcxproj
+++ b/build/MSVC/2010/ref-variation.vcxproj
@@ -1,44 +1,42 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Debug|x64">
- <Configuration>Debug</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|x64">
- <Configuration>Release</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- </ItemGroup>
-
- <Import Project=".\exe-project.props" />
-
- <ItemDefinitionGroup>
- <Link>
- <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\ngs-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(NGS_TARGET)bin\libngs-sdk.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- </ItemDefinitionGroup>
-
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\fastq-dump\args.cpp" />
- <ClCompile Include="..\..\..\tools\fastq-dump\fastq-dump.cpp" />
- <ClCompile Include="..\..\..\tools\fastq-dump\filters.cpp" />
- </ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{8DA91267-49B1-4696-AF9D-2A98B697167E}</ProjectGuid>
- </PropertyGroup>
-
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <Import Project=".\exe-project.props" />
+ <ItemDefinitionGroup>
+ <Link>
+ <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\ngs-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(NGS_TARGET)lib\libngs-bind-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(NGS_TARGET)lib\libngs-disp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\tools\ref-variation\ref-variation.cpp" />
+ <ClCompile Include="..\..\..\tools\ref-variation\helper.cpp" />
+ <ClCompile Include="..\..\..\tools\ref-variation\common.cpp" />
+ <ClInclude Include="..\..\..\tools\ref-variation\helper.h" />
+ <ClInclude Include="..\..\..\tools\ref-variation\common.h" />
+ <ClInclude Include="..\..\..\tools\ref-variation\ref-variation.vers.h" />
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{17513747-C074-4DA8-AA32-41D0C5FDC230}</ProjectGuid>
+ </PropertyGroup>
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/sam-dump.vcxproj b/build/MSVC/2010/sam-dump.vcxproj
index 7063499..17cae13 100644
--- a/build/MSVC/2010/sam-dump.vcxproj
+++ b/build/MSVC/2010/sam-dump.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -36,12 +34,14 @@
<ClCompile Include="..\..\..\tools\sra-pileup\out_redir.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\perf_log.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\read_fkt.c" />
+ <ClCompile Include="..\..\..\tools\sra-pileup\md_flag.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\rna_splice_log.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sam-aligned.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sam-dump-opts.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sam-dump.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sam-dump3.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sam-hdr.c" />
+ <ClCompile Include="..\..\..\tools\sra-pileup\sam-hdr1.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sam-unaligned.c" />
</ItemGroup>
diff --git a/build/MSVC/2010/schema-replace.vcxproj b/build/MSVC/2010/schema-replace.vcxproj
index f665894..cc4ceb8 100644
--- a/build/MSVC/2010/schema-replace.vcxproj
+++ b/build/MSVC/2010/schema-replace.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/sff-dump.vcxproj b/build/MSVC/2010/sff-dump.vcxproj
index a6a1618..29e9041 100644
--- a/build/MSVC/2010/sff-dump.vcxproj
+++ b/build/MSVC/2010/sff-dump.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/sra-kar.vcxproj b/build/MSVC/2010/sra-kar.vcxproj
index b4c44d5..c41fb31 100644
--- a/build/MSVC/2010/sra-kar.vcxproj
+++ b/build/MSVC/2010/sra-kar.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/sra-pileup.vcxproj b/build/MSVC/2010/sra-pileup.vcxproj
index 4623dfd..5ea4f54 100644
--- a/build/MSVC/2010/sra-pileup.vcxproj
+++ b/build/MSVC/2010/sra-pileup.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -37,6 +35,7 @@
<ClCompile Include="..\..\..\tools\sra-pileup\perf_log.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\pileup_counters.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\pileup_index.c" />
+ <ClCompile Include="..\..\..\tools\sra-pileup\pileup_indels.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\pileup_stat.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\pileup_v2.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\pileup_varcount.c" />
@@ -47,6 +46,7 @@
<ClCompile Include="..\..\..\tools\sra-pileup\reref.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\sra-pileup.c" />
<ClCompile Include="..\..\..\tools\sra-pileup\walk_debug.c" />
+ <ClCompile Include="..\..\..\tools\sra-pileup\out_redir.c" />
</ItemGroup>
<PropertyGroup Label="Globals">
diff --git a/build/MSVC/2010/sra-stat.vcxproj b/build/MSVC/2010/sra-stat.vcxproj
index 6a3fa84..52a0317 100644
--- a/build/MSVC/2010/sra-stat.vcxproj
+++ b/build/MSVC/2010/sra-stat.vcxproj
@@ -24,12 +24,12 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
+ <ClCompile Include="..\..\..\tools\sra-stat\sra.c" />
<ClCompile Include="..\..\..\tools\sra-stat\sra-stat.c" />
</ItemGroup>
diff --git a/build/MSVC/2010/sra-tools.sln b/build/MSVC/2010/sra-tools.sln
index 9320790..62eb1b8 100644
--- a/build/MSVC/2010/sra-tools.sln
+++ b/build/MSVC/2010/sra-tools.sln
@@ -19,8 +19,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fastq-dump", "fastq-dump.vc
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sff-dump", "sff-dump.vcxproj", "{11616A3B-9C1F-4760-AE77-755E06D5DFB1}"
EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "illumina-diump", "illumina-diump.vcxproj", "{0D7E1861-6A3C-4CF3-B603-39BF841C3C82}"
-EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "abi-dump", "abi-dump.vcxproj", "{9DF843FA-AA45-4EB1-BB76-84342AED550C}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fastq-dump-ngs", "fastq-dump-ngs.vcxproj", "{8DA91267-49B1-4696-AF9D-2A98B697167E}"
@@ -51,6 +49,12 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "vdb-config", "vdb-config.vc
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "align-info", "align-info.vcxproj", "{A54C75F9-A609-4BEC-91AB-23F5119B62AE}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "align-cache", "align-cache.vcxproj", "{3C8CB988-94DC-4B07-8865-B91FC52395F8}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ref-variation", "ref-variation.vcxproj", "{17513747-C074-4DA8-AA32-41D0C5FDC230}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "var-expand", "var-expand.vcxproj", "{4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}"
+EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "qual-recalib-stat", "qual-recalib-stat.vcxproj", "{4AF84403-70C6-4479-A701-D735E7003505}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "prefetch", "prefetch.vcxproj", "{2CF4BF04-4C4D-442E-A7AD-0AE5EF87C733}"
@@ -65,6 +69,12 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cache-mgr", "cache-mgr.vcxp
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "vcf-loader", "vcf-loader.vcxproj", "{DA226775-67B1-4354-A5CA-7F5406E0B1E7}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "illumina-dump", "illumina-dump.vcxproj", "{0D7E1861-6A3C-4CF3-B603-39BF841C3C82}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "vdb-diff", "vdb-diff.vcxproj", "{D505BF65-BE1A-AD57-4A26-6A8FA01267E3}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "general-loader", "general-loader.vcxproj", "{11356540-C8F0-44B9-B6BA-01C87443A1CF}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
@@ -145,14 +155,6 @@ Global
{11616A3B-9C1F-4760-AE77-755E06D5DFB1}.Release|Win32.Build.0 = Release|Win32
{11616A3B-9C1F-4760-AE77-755E06D5DFB1}.Release|x64.ActiveCfg = Release|x64
{11616A3B-9C1F-4760-AE77-755E06D5DFB1}.Release|x64.Build.0 = Release|x64
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|Win32.ActiveCfg = Debug|Win32
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|Win32.Build.0 = Debug|Win32
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|x64.ActiveCfg = Debug|x64
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|x64.Build.0 = Debug|x64
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|Win32.ActiveCfg = Release|Win32
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|Win32.Build.0 = Release|Win32
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|x64.ActiveCfg = Release|x64
- {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|x64.Build.0 = Release|x64
{9DF843FA-AA45-4EB1-BB76-84342AED550C}.Debug|Win32.ActiveCfg = Debug|Win32
{9DF843FA-AA45-4EB1-BB76-84342AED550C}.Debug|Win32.Build.0 = Debug|Win32
{9DF843FA-AA45-4EB1-BB76-84342AED550C}.Debug|x64.ActiveCfg = Debug|x64
@@ -273,6 +275,30 @@ Global
{A54C75F9-A609-4BEC-91AB-23F5119B62AE}.Release|Win32.Build.0 = Release|Win32
{A54C75F9-A609-4BEC-91AB-23F5119B62AE}.Release|x64.ActiveCfg = Release|x64
{A54C75F9-A609-4BEC-91AB-23F5119B62AE}.Release|x64.Build.0 = Release|x64
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Debug|Win32.ActiveCfg = Debug|Win32
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Debug|Win32.Build.0 = Debug|Win32
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Debug|x64.ActiveCfg = Debug|x64
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Debug|x64.Build.0 = Debug|x64
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Release|Win32.ActiveCfg = Release|Win32
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Release|Win32.Build.0 = Release|Win32
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Release|x64.ActiveCfg = Release|x64
+ {3C8CB988-94DC-4B07-8865-B91FC52395F8}.Release|x64.Build.0 = Release|x64
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Debug|Win32.ActiveCfg = Debug|Win32
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Debug|Win32.Build.0 = Debug|Win32
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Debug|x64.ActiveCfg = Debug|x64
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Debug|x64.Build.0 = Debug|x64
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Release|Win32.ActiveCfg = Release|Win32
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Release|Win32.Build.0 = Release|Win32
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Release|x64.ActiveCfg = Release|x64
+ {17513747-C074-4DA8-AA32-41D0C5FDC230}.Release|x64.Build.0 = Release|x64
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Debug|Win32.ActiveCfg = Debug|Win32
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Debug|Win32.Build.0 = Debug|Win32
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Debug|x64.ActiveCfg = Debug|x64
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Debug|x64.Build.0 = Debug|x64
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Release|Win32.ActiveCfg = Release|Win32
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Release|Win32.Build.0 = Release|Win32
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Release|x64.ActiveCfg = Release|x64
+ {4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}.Release|x64.Build.0 = Release|x64
{4AF84403-70C6-4479-A701-D735E7003505}.Debug|Win32.ActiveCfg = Debug|Win32
{4AF84403-70C6-4479-A701-D735E7003505}.Debug|Win32.Build.0 = Debug|Win32
{4AF84403-70C6-4479-A701-D735E7003505}.Debug|x64.ActiveCfg = Debug|x64
@@ -329,6 +355,30 @@ Global
{DA226775-67B1-4354-A5CA-7F5406E0B1E7}.Release|Win32.Build.0 = Release|Win32
{DA226775-67B1-4354-A5CA-7F5406E0B1E7}.Release|x64.ActiveCfg = Release|x64
{DA226775-67B1-4354-A5CA-7F5406E0B1E7}.Release|x64.Build.0 = Release|x64
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|Win32.ActiveCfg = Debug|Win32
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|Win32.Build.0 = Debug|Win32
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|x64.ActiveCfg = Debug|x64
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Debug|x64.Build.0 = Debug|x64
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|Win32.ActiveCfg = Release|Win32
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|Win32.Build.0 = Release|Win32
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|x64.ActiveCfg = Release|x64
+ {0D7E1861-6A3C-4CF3-B603-39BF841C3C82}.Release|x64.Build.0 = Release|x64
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Debug|Win32.ActiveCfg = Debug|Win32
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Debug|Win32.Build.0 = Debug|Win32
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Debug|x64.ActiveCfg = Debug|x64
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Debug|x64.Build.0 = Debug|x64
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Release|Win32.ActiveCfg = Release|Win32
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Release|Win32.Build.0 = Release|Win32
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Release|x64.ActiveCfg = Release|x64
+ {D505BF65-BE1A-AD57-4A26-6A8FA01267E3}.Release|x64.Build.0 = Release|x64
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Debug|Win32.ActiveCfg = Debug|Win32
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Debug|Win32.Build.0 = Debug|Win32
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Debug|x64.ActiveCfg = Debug|x64
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Debug|x64.Build.0 = Debug|x64
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Release|Win32.ActiveCfg = Release|Win32
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Release|Win32.Build.0 = Release|Win32
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Release|x64.ActiveCfg = Release|x64
+ {11356540-C8F0-44B9-B6BA-01C87443A1CF}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
diff --git a/build/MSVC/2010/srapath.vcxproj b/build/MSVC/2010/srapath.vcxproj
index cfa374c..4608d8e 100644
--- a/build/MSVC/2010/srapath.vcxproj
+++ b/build/MSVC/2010/srapath.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/md5cp.vcxproj b/build/MSVC/2010/test-align-info.vcxproj
similarity index 55%
copy from build/MSVC/2010/md5cp.vcxproj
copy to build/MSVC/2010/test-align-info.vcxproj
index 34c2cc9..aab94c9 100644
--- a/build/MSVC/2010/md5cp.vcxproj
+++ b/build/MSVC/2010/test-align-info.vcxproj
@@ -18,22 +18,11 @@
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
- <Import Project=".\exe-project.props" />
-
- <ItemDefinitionGroup>
- <Link>
- <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- </ItemDefinitionGroup>
-
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\util\md5cp.c" />
- </ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{F14519B5-5698-4363-A734-229F2A01153D}</ProjectGuid>
- </PropertyGroup>
+ <Import Project=".\test-project.props" />
+
+ <Target Name="AfterBuild" Condition="'$(Autorun)'=='true'">
+ <Exec Command="$(VDB_TARGET)\bin\align-info -h" WorkingDirectory="$(ProjectDir)"/>
+ </Target>
+
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/fastq-loader.vcxproj b/build/MSVC/2010/test-fastq-loader.vcxproj
similarity index 73%
copy from build/MSVC/2010/fastq-loader.vcxproj
copy to build/MSVC/2010/test-fastq-loader.vcxproj
index c9fb8e9..a2d63c2 100644
--- a/build/MSVC/2010/fastq-loader.vcxproj
+++ b/build/MSVC/2010/test-fastq-loader.vcxproj
@@ -19,28 +19,25 @@
</ProjectConfiguration>
</ItemGroup>
- <Import Project=".\exe-project.props" />
+ <Import Project=".\test-project.props" />
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\loader.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
- <ClCompile Include="..\..\..\tools\fastq-loader\fastq-grammar.c" />
<ClCompile Include="..\..\..\tools\fastq-loader\fastq-lex.c" />
- <ClCompile Include="..\..\..\tools\fastq-loader\fastq-loader.c" />
+ <ClCompile Include="..\..\..\tools\fastq-loader\fastq-grammar.c" />
<ClCompile Include="..\..\..\tools\fastq-loader\fastq-reader.c" />
- <ClCompile Include="..\..\..\tools\fastq-loader\loader-imp.c" />
+ <ClCompile Include="..\..\..\test\fastq-loader\test-fastq-loader.cpp" />
</ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{D5EDA47B-AD5A-4C2A-BDBB-EA29C61749E4}</ProjectGuid>
- </PropertyGroup>
+ <Target Name="AfterBuild" Condition="'$(Autorun)'=='true'">
+ <Exec Command="$(OutDir)$(TargetName)$(TargetExt)" WorkingDirectory="$(ProjectDir)"/>
+ </Target>
+
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/kdbmeta.vcxproj b/build/MSVC/2010/test-general-loader.vcxproj
similarity index 60%
copy from build/MSVC/2010/kdbmeta.vcxproj
copy to build/MSVC/2010/test-general-loader.vcxproj
index b31f4b0..0b208b4 100644
--- a/build/MSVC/2010/kdbmeta.vcxproj
+++ b/build/MSVC/2010/test-general-loader.vcxproj
@@ -19,22 +19,26 @@
</ProjectConfiguration>
</ItemGroup>
- <Import Project=".\exe-project.props" />
+ <Import Project=".\test-project.props" />
<ItemDefinitionGroup>
+ <ClCompile>
+ <PreprocessorDefinitions>LOCAL_SCHEMA=$(VDB_ROOT.Replace('\','\\'))interfaces</PreprocessorDefinitions>
+ </ClCompile>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\util\kdbmeta.c" />
+ <ItemGroup>
+ <ClCompile Include="..\..\..\test\general-loader\test-general-loader.cpp" />
+ <ClCompile Include="..\..\..\test\general-loader\testsource.cpp" />
</ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{D2B898E3-DDBB-4704-A932-AED218DABE18}</ProjectGuid>
- </PropertyGroup>
+ <Target Name="AfterBuild" Condition="'$(Autorun)'=='true'">
+ <MakeDir Directories="..\..\..\test\general-loader\db"/>
+ <Exec Command="$(OutDir)$(TargetName)$(TargetExt)" WorkingDirectory="..\..\..\test\general-loader\"/>
+ </Target>
+
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/test-sra-tools.sln b/build/MSVC/2010/test-sra-tools.sln
new file mode 100644
index 0000000..5ce7d11
--- /dev/null
+++ b/build/MSVC/2010/test-sra-tools.sln
@@ -0,0 +1,66 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "wb-test-fastq", "wb-test-fastq.vcxproj", "{842FD617-CF9D-E3B5-48CA-89BBD0FC2351}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test-fastq-loader", "test-fastq-loader.vcxproj", "{13935AED-6BA6-E085-69CC-1BFDDC44F3DC}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test-vcf-loader", "test-vcf-loader.vcxproj", "{D59CE75D-038E-87A0-8102-FD6C780298FC}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test-general-loader", "test-general-loader.vcxproj", "{A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test-align-info", "test-align-info.vcxproj", "{327CAB1F-34B3-B220-4439-28C0A1F06851}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
+ Debug|x64 = Debug|x64
+ Release|Win32 = Release|Win32
+ Release|x64 = Release|x64
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Debug|Win32.ActiveCfg = Debug|Win32
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Debug|Win32.Build.0 = Debug|Win32
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Debug|x64.ActiveCfg = Debug|x64
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Debug|x64.Build.0 = Debug|x64
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Release|Win32.ActiveCfg = Release|Win32
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Release|Win32.Build.0 = Release|Win32
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Release|x64.ActiveCfg = Release|x64
+ {842FD617-CF9D-E3B5-48CA-89BBD0FC2351}.Release|x64.Build.0 = Release|x64
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Debug|Win32.ActiveCfg = Debug|Win32
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Debug|Win32.Build.0 = Debug|Win32
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Debug|x64.ActiveCfg = Debug|x64
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Debug|x64.Build.0 = Debug|x64
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Release|Win32.ActiveCfg = Release|Win32
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Release|Win32.Build.0 = Release|Win32
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Release|x64.ActiveCfg = Release|x64
+ {13935AED-6BA6-E085-69CC-1BFDDC44F3DC}.Release|x64.Build.0 = Release|x64
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Debug|Win32.ActiveCfg = Debug|Win32
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Debug|Win32.Build.0 = Debug|Win32
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Debug|x64.ActiveCfg = Debug|x64
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Debug|x64.Build.0 = Debug|x64
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Release|Win32.ActiveCfg = Release|Win32
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Release|Win32.Build.0 = Release|Win32
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Release|x64.ActiveCfg = Release|x64
+ {D59CE75D-038E-87A0-8102-FD6C780298FC}.Release|x64.Build.0 = Release|x64
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Debug|Win32.ActiveCfg = Debug|Win32
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Debug|Win32.Build.0 = Debug|Win32
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Debug|x64.ActiveCfg = Debug|x64
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Debug|x64.Build.0 = Debug|x64
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Release|Win32.ActiveCfg = Release|Win32
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Release|Win32.Build.0 = Release|Win32
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Release|x64.ActiveCfg = Release|x64
+ {A4DE6CC9-77BF-25A7-5178-C5D3E9239C1A}.Release|x64.Build.0 = Release|x64
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Debug|Win32.ActiveCfg = Debug|Win32
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Debug|Win32.Build.0 = Debug|Win32
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Debug|x64.ActiveCfg = Debug|x64
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Debug|x64.Build.0 = Debug|x64
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Release|Win32.ActiveCfg = Release|Win32
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Release|Win32.Build.0 = Release|Win32
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Release|x64.ActiveCfg = Release|x64
+ {327CAB1F-34B3-B220-4439-28C0A1F06851}.Release|x64.Build.0 = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/build/MSVC/2010/test-sra.vcxproj b/build/MSVC/2010/test-sra.vcxproj
index f3b859e..68ad381 100644
--- a/build/MSVC/2010/test-sra.vcxproj
+++ b/build/MSVC/2010/test-sra.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/schema-replace.vcxproj b/build/MSVC/2010/test-vcf-loader.vcxproj
similarity index 64%
copy from build/MSVC/2010/schema-replace.vcxproj
copy to build/MSVC/2010/test-vcf-loader.vcxproj
index f665894..293b803 100644
--- a/build/MSVC/2010/schema-replace.vcxproj
+++ b/build/MSVC/2010/test-vcf-loader.vcxproj
@@ -18,23 +18,24 @@
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
-
- <Import Project=".\exe-project.props" />
+
+ <Import Project=".\test-project.props" />
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\loader.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\vcf-loader.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\util\schema-replace.c" />
+ <ItemGroup>
+ <ClCompile Include="..\..\..\test\vcf-loader\test-vcf-loader.cpp" />
</ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{75A32DEA-9C0C-40CE-AE8C-CA58BAD4ADFD}</ProjectGuid>
- </PropertyGroup>
+ <Target Name="AfterBuild" Condition="'$(Autorun)'=='true'">
+ <Exec Command="$(OutDir)$(TargetName)$(TargetExt)" WorkingDirectory="$(ProjectDir)"/>
+ </Target>
+
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/fastq-dump-ngs.vcxproj b/build/MSVC/2010/var-expand.vcxproj
old mode 100644
new mode 100755
similarity index 59%
copy from build/MSVC/2010/fastq-dump-ngs.vcxproj
copy to build/MSVC/2010/var-expand.vcxproj
index 35ed060..2cc893b
--- a/build/MSVC/2010/fastq-dump-ngs.vcxproj
+++ b/build/MSVC/2010/var-expand.vcxproj
@@ -1,44 +1,42 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Debug|x64">
- <Configuration>Debug</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|x64">
- <Configuration>Release</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- </ItemGroup>
-
- <Import Project=".\exe-project.props" />
-
- <ItemDefinitionGroup>
- <Link>
- <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\ngs-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(NGS_TARGET)bin\libngs-sdk.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- </ItemDefinitionGroup>
-
- <ItemGroup>
- <ClCompile Include="..\..\..\tools\fastq-dump\args.cpp" />
- <ClCompile Include="..\..\..\tools\fastq-dump\fastq-dump.cpp" />
- <ClCompile Include="..\..\..\tools\fastq-dump\filters.cpp" />
- </ItemGroup>
-
- <PropertyGroup Label="Globals">
- <ProjectGuid>{8DA91267-49B1-4696-AF9D-2A98B697167E}</ProjectGuid>
- </PropertyGroup>
-
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <Import Project=".\exe-project.props" />
+ <ItemDefinitionGroup>
+ <Link>
+ <AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\ngs-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(NGS_TARGET)lib\libngs-bind-c++.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(NGS_TARGET)lib\libngs-disp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\tools\ref-variation\var-expand.cpp" />
+ <ClCompile Include="..\..\..\tools\ref-variation\helper.cpp" />
+ <ClCompile Include="..\..\..\tools\ref-variation\common.cpp" />
+ <ClInclude Include="..\..\..\tools\ref-variation\helper.h" />
+ <ClInclude Include="..\..\..\tools\ref-variation\common.h" />
+ <ClInclude Include="..\..\..\tools\ref-variation\var-expand.vers.h" />
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{4F3ECB20-03D7-4DD0-B993-9AB6A8142A06}</ProjectGuid>
+ </PropertyGroup>
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/vdb-common.props b/build/MSVC/2010/vdb-common.props
index 247abb6..b38fd75 100644
--- a/build/MSVC/2010/vdb-common.props
+++ b/build/MSVC/2010/vdb-common.props
@@ -1,21 +1,89 @@
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <Import Condition="Exists('$(ProjectDir)..\..\Makefile.config.win')" Project="$(ProjectDir)..\..\Makefile.config.win" />
-
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+
<PropertyGroup Label="Globals">
- <Platform Condition="'$(Platform)' == ''">x64</Platform>
- <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
+ <Platform Condition="'$(Platform)' == ''">x64</Platform>
+ <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
- <NGS_ROOT Condition="'$(NGS_ROOT)' == ''">$(ProjectDir)..\..\..\..\ngs\ngs-sdk\</NGS_ROOT>
- <NGS_OUTDIR Condition="'$(NGS_OUTDIR)' == ''">$(USERPROFILE)\</NGS_OUTDIR>
- <NGS_TARGET Condition="'$(NGS_TARGET)' == ''">$(NGS_OUTDIR)win\cl\$(Platform)\$(Configuration)\</NGS_TARGET>
+ <NGS_ROOT Condition="'$(NGS_ROOT)' == ''">$(ProjectDir)..\..\..\..\ngs\ngs-sdk\</NGS_ROOT>
+ <NGS_OUTDIR Condition="'$(NGS_OUTDIR)' == ''">$(NGS_ROOT)..\..\OUTDIR\</NGS_OUTDIR>
+ <NGS_TARGET Condition="'$(NGS_TARGET)' == ''">$(NGS_OUTDIR)win\cl\$(Platform)\$(Configuration)\</NGS_TARGET>
- <VDB_OUTDIR Condition="'$(VDB_OUTDIR)' == ''">$(USERPROFILE)\</VDB_OUTDIR>
- <VDB_ROOT Condition="'$(VDB_ROOT)' == ''">$(ProjectDir)..\..\..\..\ncbi-vdb\</VDB_ROOT>
- <VDB_TARGET Condition="'$(VDB_TARGET)' == ''">$(VDB_OUTDIR)win\cl\$(Platform)\$(Configuration)\</VDB_TARGET>
+ <VDB_OUTDIR Condition="'$(VDB_OUTDIR)' == ''">$(NGS_OUTDIR)</VDB_OUTDIR>
+ <VDB_ROOT Condition="'$(VDB_ROOT)' == ''">$(ProjectDir)..\..\..\..\ncbi-vdb\</VDB_ROOT>
+ <VDB_TARGET Condition="'$(VDB_TARGET)' == ''">$(VDB_OUTDIR)win\cl\$(Platform)\$(Configuration)\</VDB_TARGET>
- </PropertyGroup>
+ </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Label="Configuration">
+ <CharacterSet>Unicode</CharacterSet>
+ <IntDir>$(VDB_TARGET)\obj\$(ProjectName)\</IntDir>
+ <LinkIncremental>false</LinkIncremental>
+ <WholeProgramOptimization>false</WholeProgramOptimization>
+ </PropertyGroup>
+ <PropertyGroup Label="Configuration" Condition="'$(TargetName)'==''" >
+ <TargetName>$(ProjectName)</TargetName>
+ </PropertyGroup>
+
+ <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+ <UseDebugLibraries>true</UseDebugLibraries>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+ <UseDebugLibraries>false</UseDebugLibraries>
+ </PropertyGroup>
+
+ <ItemDefinitionGroup>
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <CompileAsManaged>false</CompileAsManaged>
+ <MinimalRebuild>true</MinimalRebuild>
+ <DisableLanguageExtensions>false</DisableLanguageExtensions>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <DisableSpecificWarnings>4996;4101</DisableSpecificWarnings>
+ <StringPooling>true</StringPooling>
+
+ <AdditionalIncludeDirectories>$(VDB_ROOT)interfaces;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <AdditionalIncludeDirectories>$(VDB_ROOT)interfaces/os/win;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <AdditionalIncludeDirectories>$(VDB_ROOT)interfaces/cc/vc++;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <AdditionalIncludeDirectories>$(VDB_ROOT)interfaces/ext;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+
+ <AdditionalIncludeDirectories>$(NGS_ROOT);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>__mod_name__=$(ProjectName);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>__file_name__=%(Filename);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>__file_ext__=c;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+
+ </ClCompile>
+ </ItemDefinitionGroup>
+
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+ <ClCompile>
+ <Optimization>Disabled</Optimization>
+ <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <PreprocessorDefinitions>_DEBUGGING;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+ <ClCompile>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>false</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+
</Project>
diff --git a/build/MSVC/2010/vdb-config.vcxproj b/build/MSVC/2010/vdb-config.vcxproj
index 24eb16c..e089613 100644
--- a/build/MSVC/2010/vdb-config.vcxproj
+++ b/build/MSVC/2010/vdb-config.vcxproj
@@ -24,10 +24,9 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\tui.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\tui_cpp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\tui.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\tui_cpp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <EntryPointSymbol>wmainCRTStartupNoPathConversion</EntryPointSymbol>
</Link>
</ItemDefinitionGroup>
@@ -43,4 +42,4 @@
<ProjectGuid>{7CAB9840-19BB-4B32-94BD-C29701FDBE88}</ProjectGuid>
</PropertyGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/build/MSVC/2010/vdb-copy.vcxproj b/build/MSVC/2010/vdb-copy.vcxproj
index 9019284..7147f50 100644
--- a/build/MSVC/2010/vdb-copy.vcxproj
+++ b/build/MSVC/2010/vdb-copy.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -38,7 +36,6 @@
<ClCompile Include="..\..\..\tools\vdb-copy\helper.c" />
<ClCompile Include="..\..\..\tools\vdb-copy\namelist_tools.c" />
<ClCompile Include="..\..\..\tools\vdb-copy\num-gen.c" />
- <ClCompile Include="..\..\..\tools\vdb-copy\progressbar.c" />
<ClCompile Include="..\..\..\tools\vdb-copy\redactval.c" />
<ClCompile Include="..\..\..\tools\vdb-copy\type_matcher.c" />
<ClCompile Include="..\..\..\tools\vdb-copy\vdb-copy.c" />
diff --git a/build/MSVC/2010/vdb-decrypt.vcxproj b/build/MSVC/2010/vdb-decrypt.vcxproj
index a671293..12cc60d 100644
--- a/build/MSVC/2010/vdb-decrypt.vcxproj
+++ b/build/MSVC/2010/vdb-decrypt.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/vdb-validate.vcxproj b/build/MSVC/2010/vdb-diff.vcxproj
similarity index 80%
copy from build/MSVC/2010/vdb-validate.vcxproj
copy to build/MSVC/2010/vdb-diff.vcxproj
index 34d270f..d31e0d6 100644
--- a/build/MSVC/2010/vdb-validate.vcxproj
+++ b/build/MSVC/2010/vdb-diff.vcxproj
@@ -25,16 +25,17 @@
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
- <ClCompile Include="..\..\..\tools\vdb-validate\vdb-validate.c" />
+ <ClCompile Include="..\..\..\tools\vdb-diff\namelist_tools.c" />
+ <ClCompile Include="..\..\..\tools\vdb-diff\coldefs.c" />
+ <ClCompile Include="..\..\..\tools\vdb-diff\vdb-diff.c" />
</ItemGroup>
-
+
<PropertyGroup Label="Globals">
- <ProjectGuid>{446C2DD3-36B3-4B27-8415-951F98346E99}</ProjectGuid>
</PropertyGroup>
</Project>
\ No newline at end of file
diff --git a/build/MSVC/2010/vdb-dump.vcxproj b/build/MSVC/2010/vdb-dump.vcxproj
index 2d16ae3..c736ada 100644
--- a/build/MSVC/2010/vdb-dump.vcxproj
+++ b/build/MSVC/2010/vdb-dump.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -37,7 +35,6 @@
<ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-filter.c" />
<ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-formats.c" />
<ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-helper.c" />
- <ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-num-gen.c" />
<ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-print.c" />
<ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-redir.c" />
<ClCompile Include="..\..\..\tools\vdb-dump\vdb-dump-str.c" />
diff --git a/build/MSVC/2010/vdb-encrypt.vcxproj b/build/MSVC/2010/vdb-encrypt.vcxproj
index 5322680..9cdae45 100644
--- a/build/MSVC/2010/vdb-encrypt.vcxproj
+++ b/build/MSVC/2010/vdb-encrypt.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/vdb-lock.vcxproj b/build/MSVC/2010/vdb-lock.vcxproj
index 4a4b5b0..5239f26 100644
--- a/build/MSVC/2010/vdb-lock.vcxproj
+++ b/build/MSVC/2010/vdb-lock.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/vdb-passwd.vcxproj b/build/MSVC/2010/vdb-passwd.vcxproj
index 0a8a35e..6b70f6a 100644
--- a/build/MSVC/2010/vdb-passwd.vcxproj
+++ b/build/MSVC/2010/vdb-passwd.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/vdb-unlock.vcxproj b/build/MSVC/2010/vdb-unlock.vcxproj
index a6d6b92..038057a 100644
--- a/build/MSVC/2010/vdb-unlock.vcxproj
+++ b/build/MSVC/2010/vdb-unlock.vcxproj
@@ -23,8 +23,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/vdb-validate.vcxproj b/build/MSVC/2010/vdb-validate.vcxproj
index 34d270f..5fbb36f 100644
--- a/build/MSVC/2010/vdb-validate.vcxproj
+++ b/build/MSVC/2010/vdb-validate.vcxproj
@@ -24,8 +24,6 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-vdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
diff --git a/build/MSVC/2010/fastq-loader.vcxproj b/build/MSVC/2010/wb-test-fastq.vcxproj
similarity index 73%
copy from build/MSVC/2010/fastq-loader.vcxproj
copy to build/MSVC/2010/wb-test-fastq.vcxproj
index c9fb8e9..9220abd 100644
--- a/build/MSVC/2010/fastq-loader.vcxproj
+++ b/build/MSVC/2010/wb-test-fastq.vcxproj
@@ -19,28 +19,25 @@
</ProjectConfiguration>
</ItemGroup>
- <Import Project=".\exe-project.props" />
+ <Import Project=".\test-project.props" />
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>$(VDB_TARGET)lib\ncbi-wvdb.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <AdditionalDependencies>$(VDB_TARGET)lib\bz2.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\loader.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>$(VDB_TARGET)lib\load.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
- <ClCompile Include="..\..\..\tools\fastq-loader\fastq-grammar.c" />
<ClCompile Include="..\..\..\tools\fastq-loader\fastq-lex.c" />
- <ClCompile Include="..\..\..\tools\fastq-loader\fastq-loader.c" />
+ <ClCompile Include="..\..\..\tools\fastq-loader\fastq-grammar.c" />
<ClCompile Include="..\..\..\tools\fastq-loader\fastq-reader.c" />
- <ClCompile Include="..\..\..\tools\fastq-loader\loader-imp.c" />
+ <ClCompile Include="..\..\..\test\fastq-loader\wb-test-fastq.cpp" />
</ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{D5EDA47B-AD5A-4C2A-BDBB-EA29C61749E4}</ProjectGuid>
- </PropertyGroup>
-
+ <Target Name="AfterBuild" Condition="'$(Autorun)'=='true'">
+ <Exec Command="$(OutDir)$(TargetName)$(TargetExt)" WorkingDirectory="$(ProjectDir)"/>
+ </Target>
+
</Project>
\ No newline at end of file
diff --git a/build/Makefile.clang b/build/Makefile.clang
index 7194afe..ecc0d2c 100644
--- a/build/Makefile.clang
+++ b/build/Makefile.clang
@@ -36,12 +36,14 @@ PP = gcc -E $(CFLAGS)
LD = @ $(TOP)/build/ld.sh $(OS) $(ARCH) clang \
--build $(BUILD) --ldflags "$(LDFLAGS)" $(STATIC) \
$(STATICSYSLIBS) $(CHECKSUM) --objx $(OBJX) --shlx $(SHLX) --libx $(LIBX) \
- -MD --srcdir $(SRCDIR) --bindir $(BINDIR) -L$(LIBDIR):$(ILIBDIR)
+ -MD --srcdir $(SRCDIR) --bindir $(BINDIR) -L$(LIBDIR):$(ILIBDIR):$(NGS_LIBDIR):$(VDB_LIBDIR):$(VDB_ILIBDIR)
+
+LPFLAGS = $(LDFLAGS)
LP = @ $(TOP)/build/ld.sh $(OS) $(ARCH) clang++ \
- --build $(BUILD) --ldflags "$(LDFLAGS)" $(STATIC) \
+ --build $(BUILD) --ldflags "$(LPFLAGS)" $(STATIC) \
$(STATICSYSLIBS) $(CHECKSUM) --objx $(OBJX) --shlx $(SHLX) --libx $(LIBX) \
- -MD --srcdir $(SRCDIR) --bindir $(BINDIR) -L$(LIBDIR):$(ILIBDIR)
+ -MD --srcdir $(SRCDIR) --bindir $(BINDIR) -L$(LIBDIR):$(ILIBDIR):$(NGS_LIBDIR):$(VDB_LIBDIR):$(VDB_ILIBDIR)
# tool options
WARN = -Wall -Wno-variadic-macros -Wno-long-long # -Wconversion
@@ -51,8 +53,12 @@ endif
ifeq (64,$(BITS))
CARCH = -m64
-else
- CARCH = -m32
+else
+ifeq (32_64,$(BITS))
+ CARCH = -arch i386 -arch x86_64
+else
+ CARCH = -m32
+endif
endif
ifeq (prof, $(BUILD))
diff --git a/build/Makefile.env b/build/Makefile.env
index d56d2e2..c833a48 100644
--- a/build/Makefile.env
+++ b/build/Makefile.env
@@ -75,6 +75,11 @@ else
LOCAL_OR_REMOTE = local
endif
+# pre-built sra-tools are static
+ifeq (1,$(WANTS_STATIC))
+ STATIC = --static
+ STATICSYSLIBS = --static-system-libs
+endif
# drop all suffix rules
.SUFFIXES:
@@ -133,7 +138,6 @@ OUTDIRS = schema
# clean rules
stdclean:
@ -rm -rf $(OBJDIR)
- @ -rm -rf $(addprefix $(OUTDIR)/$(OS)/$(TOOLSET)/,dyn stat)
@ -rm -rf $(addsuffix .*,$(addprefix $(ILIBDIR)/,$(ALL_LIBS))) \
$(addsuffix .*,$(addprefix $(LIBDIR)/,$(ALL_LIBS))) \
$(addsuffix -static.*,$(addprefix $(LIBDIR)/,$(ALL_LIBS))) \
@@ -319,14 +323,31 @@ INCDIRS = \
-I.
+# linker paths
+LDPATHS =
+
+ifneq (,$(HDF5_LIBDIR))
+ LDPATHS += -L$(HDF5_LIBDIR)
+endif
+ifneq (,$(XML2_LIBDIR))
+ LDPATHS += -L$(XML2_LIBDIR)
+endif
+ifneq (,$(MAGIC_LIBDIR))
+ LDPATHS += -L$(MAGIC_LIBDIR)
+endif
+ifneq (,$(FUSE_LIBDIR))
+ LDPATHS += -L$(FUSE_LIBDIR)
+endif
+
+
# defines that describe os & architecture
DLLX ?= $(SHLX)
-ARCHDEFS = -D_ARCH_BITS=$(BITS) -DLIBPREFIX=$(LPFX) -DSHLIBEXT=$(DLLX)
+ARCHDEFS = -D_ARCH_BITS=__SIZEOF_POINTER__*__CHAR_BIT__ -DLIBPREFIX=$(LPFX) -DSHLIBEXT=$(DLLX)
# default tool parameters
CFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
CPFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
-LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT)
+LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS)
#-------------------------------------------------------------------------------
# runtests
@@ -335,7 +356,7 @@ LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT)
#
ifeq ($(RUNTESTS_OVERRIDE),)
runtests: std $(TEST_TOOLS)
- @ export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;export MallocScribble=1;\
+ @ export VDB_CONFIG=$(VDB_CONFIG);export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;export MallocScribble=1;\
for i in $(TEST_TOOLS);\
do\
echo ++++++++++++++++++++++++++++++++++++++++++++++++++++++;\
diff --git a/build/Makefile.gcc b/build/Makefile.gcc
index 49bb874..e90f4f0 100644
--- a/build/Makefile.gcc
+++ b/build/Makefile.gcc
@@ -58,7 +58,11 @@ endif
ifeq (64,$(BITS))
CARCH = -m64
else
- CARCH = -m32
+ifeq (32_64,$(BITS))
+ CARCH = -arch i386 -arch x86_64
+else
+ CARCH = -m32
+endif
endif
ifeq (prof, $(BUILD))
@@ -75,11 +79,11 @@ ifeq (dbg, $(BUILD))
PED = -std=gnu99 -pedantic # -fdiagnostics-show-option
else
-ifeq (x86_64, $(ARCH))
- OPT = -O3 -Wall -Wno-variadic-macros -fno-strict-aliasing
-else
- OPT = -O3 -Wno-variadic-macros -fno-strict-aliasing
-endif
+OPT = -O3 -Wno-variadic-macros -fno-strict-aliasing -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
+ifeq (x86_64, $(ARCH))
+ OPT += -Wall
+endif
+
endif
ifeq (mac,$(OS))
diff --git a/build/Makefile.install b/build/Makefile.install
index 5e7e4a7..9d506f2 100644
--- a/build/Makefile.install
+++ b/build/Makefile.install
@@ -28,11 +28,12 @@ include $(TOP)/build/Makefile.shell
# load build configuration
include $(CONFIG_FILE)
+# load platform-specific settings
+include $(TOP)/build/Makefile.$(OS)
+
#-------------------------------------------------------------------------------
# install
#
-ifeq (linux, $(OS))
-
TOOLS_TO_INSTALL = \
abi-dump \
abi-load \
@@ -73,38 +74,46 @@ TOOLS_TO_INSTALL = \
# blastn_vdb \
# tblastn_vdb \
-TOOLS_WITH_PREFIX = $(addprefix $(BINDIR)/, $(TOOLS_TO_INSTALL))
-TOOLS_WITH_EXEX = $(addsuffix $(EXEX), $(TOOLS_WITH_PREFIX))
-TOOLS_WITH_VERSION = $(addsuffix $(VERSION_EXEX), $(TOOLS_WITH_PREFIX))
-TOOLS_WITH_MAJVERS = $(addsuffix $(MAJVERS_EXEX), $(TOOLS_WITH_PREFIX))
-
#fake root for debugging
#uncomment this line and change the test for root ( see under install: ) to succeed:
#ROOT = ~/root
-
+
EXE_TARGET = $(INST_BINDIR)
PROFILE_FILE = $(ROOT)/etc/profile.d/sra-tools
-installexes:
- @echo "Installing executables to $(INST_BINDIR)"
- @mkdir -p $(INST_BINDIR)
- @cp -P $(TOOLS_WITH_EXEX) $(TOOLS_WITH_VERSION) $(TOOLS_WITH_MAJVERS) $(INST_BINDIR)
+installexes:
+ @ echo "Installing executables to $(INST_BINDIR)/..."
+ @ mkdir -p $(INST_BINDIR)
+ @ $(MAKE) -s --no-print-directory -f $(TOP)/build/Makefile.install $(TOOLS_TO_INSTALL)
+
+$(TOOLS_TO_INSTALL):
+ @ printf "Installing $@..."
+ @ # ignore missing tools
+ @ - ( test -f $(BINDIR)/$@$(EXEX) || ( echo "not found" && false ) ) && \
+ $(COPY) $(BINDIR)/$@$(EXEX) $(BINDIR)/$@$(VERSION_EXEX) $(BINDIR)/$@$(MAJVERS_EXEX) $(INST_BINDIR) && \
+ echo "done"
+
+ifeq (linux, $(OS))
+ ifeq (0, $(shell id -u))
+ LINUX_ROOT = true
+ endif
+endif
install: installexes
-ifeq (0, $(shell id -u))
+ifeq (true, $(LINUX_ROOT))
@ # install configuration file(s)
@ echo "Installing configuration files to $(ROOT)/etc/ncbi/"
@ mkdir -p $(ROOT)/etc/ncbi/
@ cp $(TOP)/tools/vdb-copy/vdb-copy.kfg $(ROOT)/etc/ncbi/
@ #
@ echo "Updating $(PROFILE_FILE).[c]sh"
- @ echo -e \
+ @ printf \
"#version $(VERSION)\n"\
"if ! echo \$$PATH | /bin/grep -q $(EXE_TARGET)\n"\
"then export PATH=$(EXE_TARGET):\$$PATH\n"\
"fi" \
>$(PROFILE_FILE).sh && chmod 644 $(PROFILE_FILE).sh || true;
- @ echo -e \
+ @ printf \
"#version $(VERSION)\n"\
"echo \$$PATH | /bin/grep -q $(EXE_TARGET)\n"\
"if ( \$$status ) setenv PATH $(EXE_TARGET):\$$PATH\n"\
@@ -117,9 +126,20 @@ else
@ cp $(TOP)/tools/vdb-copy/vdb-copy.kfg $(INST_BINDIR)/ncbi/
endif
-else
-install:
+#-------------------------------------------------------------------------------
+# uninstall
+#
+
+TO_UNINSTALL = $(addsuffix *,$(addprefix $(INST_BINDIR)/,$(TOOLS_TO_INSTALL)))
+TO_UNINSTALL_AS_ROOT = $(ROOT)/etc/ncbi/vdb-copy.kfg $(PROFILE_FILE).sh $(PROFILE_FILE).csh
+uninstall:
+ @ echo "Uninstalling sra-tools from $(INST_BINDIR)..."
+ rm -rf $(TO_UNINSTALL)
+ifeq (true, $(LINUX_ROOT))
+ @ echo "Uninstalling $(TO_UNINSTALL_AS_ROOT) ..."
+ @ rm -rf $(TO_UNINSTALL_AS_ROOT)
endif
+ @ echo "done."
-.PHONY: install
\ No newline at end of file
+.PHONY: install installexes uninstall
diff --git a/build/Makefile.linux b/build/Makefile.linux
index 582f257..9383a51 100644
--- a/build/Makefile.linux
+++ b/build/Makefile.linux
@@ -69,3 +69,6 @@ LEX = @ true
# build matrix
COMPILERS = GCC # ICC
+
+# copy without following symlinks
+COPY = cp -P
diff --git a/build/Makefile.mac b/build/Makefile.mac
index a246b0a..17e2b64 100644
--- a/build/Makefile.mac
+++ b/build/Makefile.mac
@@ -79,3 +79,6 @@ LEX = @ true
# build matrix
COMPILERS = CLANG GCC
+
+# copy without following symlinks
+COPY = cp -R
diff --git a/build/ld.linux.dlib.sh b/build/ld.linux.dlib.sh
index 6ed1c48..1831601 100755
--- a/build/ld.linux.dlib.sh
+++ b/build/ld.linux.dlib.sh
@@ -275,8 +275,8 @@ then
fi
# add in xml
-grep HAVE_XML2 ${BUILD_DIR}/Makefile.config.linux.$ARCH
-if [ $? -eq 0 ]
+if grep -q HAVE_XML2 ${BUILD_DIR}/Makefile.config.linux.$ARCH > /dev/null
+then
if [ $HAVE_XML -ne 0 ]
then
CMD="$CMD -lxml2"
diff --git a/build/ld.linux.exe.sh b/build/ld.linux.exe.sh
index 34f370f..81dc5f0 100755
--- a/build/ld.linux.exe.sh
+++ b/build/ld.linux.exe.sh
@@ -60,7 +60,12 @@ i*)
esac
# EXE_CMD was started in tool-specific source
-CMD="$EXE_CMD $LDFLAGS"
+if [ $STATIC -eq 1 ]
+then
+ CMD="$EXE_STATIC_CMD $LDFLAGS"
+else
+ CMD="$EXE_CMD $LDFLAGS"
+fi
# if building a static executable against dynamic libraries
# the main application will substitute for name lookup
@@ -289,19 +294,22 @@ then
fi
# add in xml
-grep HAVE_XML2 ${BUILD_DIR}/Makefile.config.linux.$ARCH
-if [ $? -eq 0 ]
+if grep -q HAVE_XML2 ${BUILD_DIR}/Makefile.config.linux.$ARCH > /dev/null
+then
+ if grep -q XML2_LIBDIR ${BUILD_DIR}/Makefile.config.linux.$ARCH > /dev/null
then
+ XMLLIBDIR="-Wl,-Bstatic -L$(grep XML2_LIBDIR ${BUILD_DIR}/Makefile.config.linux.$ARCH | perl -e'while(<>){if(/XML2_LIBDIR = (.+)/){print $1}}')"
+ fi
if [ $HAVE_XML -ne 0 ]
then
- CMD="$CMD -lxml2"
+ CMD="$CMD $XMLLIBDIR -lxml2"
fi
fi
# add in math library
if [ $HAVE_M -ne 0 ]
then
- CMD="$CMD -lm"
+ CMD="$CMD -Wl,-Bdynamic -lm"
fi
# produce shared library
diff --git a/build/ld.linux.gcc.sh b/build/ld.linux.gcc.sh
index af6aaba..cfc8ad0 100755
--- a/build/ld.linux.gcc.sh
+++ b/build/ld.linux.gcc.sh
@@ -33,7 +33,10 @@ LD_REF_SYMBOLS="-Wl,-no-whole-archive"
# build command
DLIB_CMD="$LD -shared"
-EXE_CMD="$LD"
+EXE_CMD="$LD -static-libstdc++ -static-libgcc"
+EXE_STATIC_CMD="$EXE_CMD"
+#EXE_CMD="$LD"
+#EXE_STATIC_CMD="$LD -static"
# versioned output
if [ "$VERS" = "" ]
diff --git a/build/ld.mac.clang.sh b/build/ld.mac.clang.sh
index c145942..fec96e5 100755
--- a/build/ld.mac.clang.sh
+++ b/build/ld.mac.clang.sh
@@ -32,7 +32,12 @@ LD_ALL_SYMBOLS=""
LD_REF_SYMBOLS=""
# the Mac is set up for cross-compilation
-LD="$LD -Wl,-arch,$ARCH"
+if [ "$ARCH" = "fat86" ]
+then
+ LD="$LD -Wl,-arch_multiple"
+else
+ LD="$LD -Wl,-arch,$ARCH"
+fi
# build command
DLIB_CMD="$LD -dynamiclib"
diff --git a/build/ld.mac.gcc.sh b/build/ld.mac.gcc.sh
index c145942..fec96e5 100755
--- a/build/ld.mac.gcc.sh
+++ b/build/ld.mac.gcc.sh
@@ -32,7 +32,12 @@ LD_ALL_SYMBOLS=""
LD_REF_SYMBOLS=""
# the Mac is set up for cross-compilation
-LD="$LD -Wl,-arch,$ARCH"
+if [ "$ARCH" = "fat86" ]
+then
+ LD="$LD -Wl,-arch_multiple"
+else
+ LD="$LD -Wl,-arch,$ARCH"
+fi
# build command
DLIB_CMD="$LD -dynamiclib"
diff --git a/build/ld.mac.slib.sh b/build/ld.mac.slib.sh
index d865330..01f420f 100755
--- a/build/ld.mac.slib.sh
+++ b/build/ld.mac.slib.sh
@@ -40,35 +40,7 @@ SELF_NAME="$(basename $0)"
source "${0%slib.sh}cmn.sh"
# initialize command
-CMD="ar -rc"
-
-# function to convert an archive into individual object files
-convert-static ()
-{
- # list members
- local path="$1"
- local mbrs="$(ar -t $path | grep -v '__.SYMDEF SORTED')"
-
- # unpack archive into temporary directory
- mkdir -p ld-tmp
- if ! cd ld-tmp
- then
- echo "$SELF_NAME: failed to cd to ld-tmp"
- exit 5
- fi
- ar -x "$path"
-
- # rename and add to source files list
- local m=
- for m in $mbrs
- do
- mv $m $LIBNAME-$m
- CMD="$CMD ld-tmp/$LIBNAME-$m"
- done
-
- # return to prior location
- cd - > /dev/null
-}
+CMD="libtool -static -o "
# versioned output
if [ "$VERS" = "" ]
@@ -105,9 +77,8 @@ then
# add it to dependencies
DEPS="$DEPS $LIBPATH"
- # convert to individual object files
- convert-static "$LIBPATH" || exit $?
-
+ # simply add the library to the link line - libtool handles static libraries correctly
+ CMD="$CMD $LIBPATH"
fi
;;
diff --git a/configure b/configure
index 3313aac..baa556f 100755
--- a/configure
+++ b/configure
@@ -33,13 +33,13 @@ if [ -z "$CURDIR" ]
exit 1
fi
-if [ "$FILENAME" != configure -a ! -s konfigure.perl ]
+if [ "$FILENAME" != configure -a ! -s setup/konfigure.perl ]
then
echo configure: error: configure should be run as ./configure
else
perl -v > /dev/null 2>&1 || { echo "checking for perl... no"; \
echo >&2 "configure: error: perl not found."; exit 1; }
- cd $CURDIR
- perl -w ./konfigure.perl $@
+ cd $CURDIR/setup
+ perl -w konfigure.perl "$@"
fi
diff --git a/konfigure.perl b/konfigure.perl
deleted file mode 100644
index 245bd33..0000000
--- a/konfigure.perl
+++ /dev/null
@@ -1,1204 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-use strict;
-
-sub println { print @_; print "\n" }
-
-my ($filename, $directories, $suffix) = fileparse($0);
-if ($directories ne "./") {
- println "configure: error: $filename should be run as ./$filename";
- exit 1;
-}
-
-require 'package.pm';
-require 'os-arch.pm';
-
-use Cwd qw (abs_path getcwd);
-use File::Basename 'fileparse';
-use File::Spec 'catdir';
-use FindBin qw($Bin);
-use Getopt::Long 'GetOptions';
-
-check();
-
-my %PKG = PKG();
-
-my $PACKAGE_NAME = PACKAGE_NAME();
-my $OUT_MAKEFILE = 'Makefile.config';
-
-my $PACKAGE = PACKAGE();
-
-my $HOME = $ENV{HOME} || $ENV{USERPROFILE}
- || $ENV{LOGDIR} || getcwd || (getpwuid($<))[7] || abs_path('.');
-
-$PKG{UPATH} =~ s/(\$\w+)/$1/eeg;
-
-my $OUTDIR = File::Spec->catdir($HOME, $PKG{OUT});
-
-my $package_default_prefix = $PKG{PATH};
-my $schema_default_dir = $PKG{SCHEMA_PATH} if ($PKG{SCHEMA_PATH});
-
-my @REQ = REQ();
-
-my @options = ( "arch=s",
- "build=s",
- "clean",
- "debug",
- "help",
-# "output-makefile=s",
- "prefix=s",
- "status",
- "with-debug",
- "without-debug" );
-foreach my $href (@REQ) {
- my %a = %$href;
- push @options, "$a{option}=s";
- push @options, "$a{boption}=s" if ($a{boption});
- $href->{usrpath} =~ s/(\$\w+)/$1/eeg;
-}
-push @options, "shemadir" if ($PKG{SCHEMA_PATH});
-
-my %OPT;
-die "configure: error" unless (GetOptions(\%OPT, @options));
-
-if ($OPT{'help'}) {
- help();
- exit(0);
-} elsif ($OPT{'clean'}) {
- {
- foreach (glob(CONFIG_OUT() . '/Makefile.config*'),
- File::Spec->catdir(CONFIG_OUT(), 'user.status'),
- File::Spec->catdir(CONFIG_OUT(), 'Makefile.userconfig'))
- {
- my $f = $_;
- print "removing $f... ";
- if (-e $f) {
- if (unlink $f) {
- println "ok";
- } else {
- println "failed";
- }
- } else {
- println "not found";
- }
- }
- }
- if (CONFIG_OUT() ne '.') {
- foreach
- (glob('Makefile.config*'), 'user.status', 'Makefile.userconfig')
- {
- my $f = $_;
- print "removing $f... ";
- if (-e $f) {
- if (unlink $f) {
- println "ok";
- } else {
- println "failed";
- }
- } else {
- println "not found";
- }
- }
- }
- exit(0);
-} elsif ($OPT{'status'}) {
- status(1);
- exit(0);
-}
-$OPT{'prefix'} = $package_default_prefix unless ($OPT{'prefix'});
-
-my $AUTORUN = $OPT{status};
-print "checking system type... " unless ($AUTORUN);
-my ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
-println $OSTYPE unless ($AUTORUN);
-
-{
- $OPT{'prefix'} = expand($OPT{'prefix'});
- my $prefix = $OPT{'prefix'};
- $OPT{eprefix} = $prefix unless ($OPT{eprefix} || $OS eq 'win');
- my $eprefix = $OPT{eprefix};
- unless ($OPT{bindir} || $OS eq 'win') {
- $OPT{bindir} = File::Spec->catdir($eprefix, 'bin') ;
- }
- unless ($OPT{libdir} || $OS eq 'win') {
- $OPT{libdir} = File::Spec->catdir($eprefix, 'lib');
- }
- unless ($OPT{includedir} || $OS eq 'win') {
- $OPT{includedir} = File::Spec->catdir($eprefix, 'include');
- }
- if ($PKG{LNG} eq 'PYTHON' && ! $OPT{pythondir} && $OS ne 'win') {
- $OPT{pythondir} = $eprefix;
- }
- if ($PKG{LNG} eq 'JAVA' && ! $OPT{javadir} && $OS ne 'win') {
- $OPT{javadir} = File::Spec->catdir($eprefix, 'jar');
- }
- if ($PKG{EXAMP} && ! $OPT{sharedir} && $OS ne 'win') {
- $OPT{sharedir} = File::Spec->catdir($eprefix, 'share');
- }
-}
-
-# initial values
-my $TARGDIR = File::Spec->catdir($OUTDIR, $PACKAGE);
-$TARGDIR = expand($OPT{'build'}) if ($OPT{'build'});
-
-my $BUILD = 'rel';
-
-# parse command line
-$BUILD = 'dbg' if ($OPT{'with-debug'});
-$BUILD = 'rel' if ($OPT{'without-debug'});
-
-my $BUILD_TYPE = "release";
-$BUILD_TYPE = "debug" if ( $BUILD eq "dbg" );
-
-#println unless ($AUTORUN);
-
-if ($OPT{arch}) {
- my $found;
- foreach (@ARCHITECTURES) {
- if ($_ eq $OPT{arch}) {
- ++$found;
- last;
- }
- }
- if ($found) {
- $ARCH = $MARCH = $OPT{arch};
- } else {
- delete $OPT{arch};
- }
-}
-
-$OUT_MAKEFILE .= ".$OS.$ARCH";
-$OUT_MAKEFILE = File::Spec->catdir(CONFIG_OUT(), $OUT_MAKEFILE);
-
-#my $OSTYPE = `uname -s`; chomp $OSTYPE;
-
-print "checking machine architecture... " unless ($AUTORUN);
-#my $MARCH = `uname -m`; chomp $MARCH;
-println $MARCH unless ($AUTORUN);
-
-my $TOOLS = "";
-$TOOLS = "jdk" if ($PKG{LNG} eq 'JAVA');
-
-print "checking $PACKAGE_NAME version... " unless ($AUTORUN);
-my $FULL_VERSION = VERSION();
-println $FULL_VERSION unless ($AUTORUN);
-
-# determine architecture
-
-print "checking for supported architecture... " unless ($AUTORUN);
-
-my $BITS;
-
-if ( $MARCH =~ m/x86_64/i )
-{
- $BITS = 64;
-}
-elsif ( $MARCH =~ m/i?86/i )
-{
- $BITS = 32;
-}
-else
-{
- die "unrecognized Architecture - " . $ARCH;
-}
-println "$MARCH ($BITS bits) is supported" unless ($AUTORUN);
-
-# determine OS and related norms
-my ($LPFX, $OBJX, $LOBX, $LIBX, $SHLX, $EXEX, $OSINC);
-
-print "checking for supported OS... " unless ($AUTORUN);
-if ( $OSTYPE =~ m/linux/i )
-{
- $LPFX = "lib";
- $OBJX = "o";
- $LOBX = "pic.o";
- $LIBX = "a";
- $SHLX = "so";
- $EXEX = "";
- $OSINC = "unix";
- if ( $TOOLS eq "" )
- {
- $TOOLS = "gcc";
- }
-}
-elsif ( $OSTYPE =~ m/darwin/i )
-{
- $LPFX = "lib";
- $OBJX = "o";
- $LOBX = "pic.o";
- $LIBX = "a";
- $SHLX = "dylib";
- $EXEX = "";
- $OSINC = "unix";
- if ( $TOOLS eq "" )
- {
- $TOOLS = "clang";
- }
-} elsif ($OSTYPE eq 'win') {
- $TOOLS = "vc++";
-} else
-{
- die "unrecognized OS - " . $OSTYPE;
-}
-println "$OSTYPE ($OS) is supported" unless ($AUTORUN);
-
-# tool chain
-my ($CC, $CP, $AR, $ARX, $ARLS, $LD, $LP);
-my ($JAVAC, $JAVAH, $JAR);
-my ($DBG, $OPT, $PIC, $INC, $MD);
-
-print "checking for supported tool chain... " unless ($AUTORUN);
-if ( $TOOLS =~ m/gcc/i )
-{
- $CC = "gcc -c";
- $CP = "g++ -c";
- $AR = "ar rc";
- $ARX = "ar x";
- $ARLS = "ar t";
- $LD = "gcc";
- $LP = "g++";
-
- $DBG = "-g -DDEBUG";
- $OPT = "-O3";
- $PIC = "-fPIC";
- $INC = "-I";
- $MD = "-MD";
-}
-elsif ( $TOOLS =~ m/clang/i )
-{
- $CC = "clang -c";
- $CP = "clang++ -c";
- $AR = "ar rc";
- $ARX = "ar x";
- $ARLS = "ar t";
- $LD = "clang";
- $LP = "clang++";
-
- $DBG = "-g -DDEBUG";
- $OPT = "-O3";
- $PIC = "-fPIC";
- $INC = "-I";
- $MD = "-MD";
-}
-elsif ( $TOOLS =~ m/jdk/i )
-{
- $JAVAC = "javac";
- $JAVAH = "javah";
- $JAR = "jar cf";
-
- $DBG = "-g";
-} elsif ($TOOLS eq 'vc++') {
-} else
-{
- die "unrecognized tool chain - " . $TOOLS;
-}
-println "$TOOLS tool chain is supported" unless ($AUTORUN);
-
-if ($OS ne 'win') {
- $TARGDIR = File::Spec->catdir($TARGDIR, $OS, $TOOLS, $ARCH, $BUILD);
-}
-
-my @dependencies;
-
-foreach my $href (DEPENDS()) {
- $_ = $href->{name};
- my ($I, $L) = ($href->{Include});
- if ($OPT{"with-$_-prefix"}) {
- $I = File::Spec->catdir($OPT{"with-$_-prefix"}, 'include');
- $L = File::Spec->catdir($OPT{"with-$_-prefix"}, 'lib');
- }
- my ($i, $l) = find_lib($_, $I, $L);
- if (defined $i || $l) {
- my $d = 'HAVE_' . uc($_) . ' = 1';
- push @dependencies, $d;
- println "\t\t$d" if ($OPT{'debug'});
- }
- if ($i) {
- my $d = uc($_) . "_INCDIR = $i";
- push @dependencies, $d;
- println "\t\t$d" if ($OPT{'debug'});
- }
- if ($l) {
- my $d = uc($_) . "_LIBDIR = $l";
- push @dependencies, $d;
- println "\t\t$d" if ($OPT{'debug'});
- }
-}
-
-foreach my $href (@REQ) {
- $href->{bldpath} =~ s/(\$\w+)/$1/eeg if ($href->{bldpath});
- my ($found_itf, $found_lib, $found_ilib); # found directories
- my %a = %$href;
- my $is_optional = optional($a{type});
- my $need_source = $a{type} =~ /S/;
- my $need_build = $a{type} =~ /B/;
- my $need_lib = $a{type} =~ /L/;
-
- my ($inc, $lib, $ilib) = ($a{include}, $a{lib}); # file names to check
- $lib =~ s/(\$\w+)/$1/eeg;
-
- if ($need_build) {
- $ilib = $a{ilib};
- ++$need_lib;
- }
- unless ($AUTORUN) {
- if ($need_source && $need_build) {
- println
- "checking for $a{name} package source files and build results..."
- } elsif ($need_source) {
- println "checking for $a{name} package source files...";
- } else {
- println "checking for $a{name} package...";
- }
- }
- my %has_option;
- foreach my $option ($a{option}, $a{boption}) {
- next unless ($option);
- if ($OPT{$option}) {
- my $try = expand($OPT{$option});
- my ($i, $l, $il) = ($inc, $lib, $ilib);
- if ($option =~ /-build$/) {
- undef $i;
- ++$has_option{build};
- } elsif ($option =~ /-prefix$/) {
- undef $il;
- ++$has_option{prefix};
- } elsif ($option =~ /-sources$/) {
- undef $l;
- undef $il;
- ++$has_option{sources};
- }
- my ($fi, $fl, $fil) = find_in_dir($try, $i, $l, $il);
- $found_itf = $fi if (! $found_itf && $fi);
- $found_lib = $fl if (! $found_lib && $fl);
- $found_ilib = $fil if (! $found_ilib && $fil);
- }
- }
- if (! $found_itf && ! $has_option{sources} && $a{srcpath}) {
- my $try = $a{srcpath};
- ($found_itf) = find_in_dir($try, $inc);
- }
- if (! $has_option{prefix}) {
- if (! $found_itf || ($need_lib && ! $found_lib)) {
- my $try = $a{pkgpath};
- my ($fi, $fl) = find_in_dir($try, $inc, $lib);
- $found_itf = $fi if (! $found_itf && $fi);
- $found_lib = $fl if (! $found_lib && $fl);
- }
-
- if (! $found_itf || ($need_lib && ! $found_lib)) {
- my $try = $a{usrpath};
- my ($fi, $fl) = find_in_dir($try, $inc, $lib);
- $found_itf = $fi if (! $found_itf && $fi);
- $found_lib = $fl if (! $found_lib && $fl);
- }
- }
- if (! $has_option{build}) {
- if (($need_build || ($need_lib && ! $found_lib)) && $a{bldpath}) {
- my $try = $a{bldpath};
- my (undef, $fl, $fil) = find_in_dir($try, undef, $lib, $ilib);
- $found_lib = $fl if (! $found_lib && $fl);
- $found_ilib = $fil if (! $found_ilib && $fil);
- }
- }
- if (! $found_itf || ($need_lib && ! $found_lib) || ($ilib && ! $found_ilib))
- {
- if ($is_optional) {
- println "configure: optional $a{name} package not found: skipped.";
- } else {
- if ($OPT{'debug'}) {
- $_ = "$a{name}: includes: ";
- $found_itf = '' unless $found_itf;
- $_ .= $found_itf;
- unless ($need_lib) {
- $_ .= "; libs: not needed";
- } else {
- $found_lib = '' unless $found_lib;
- $_ .= "; libs: " . $found_lib;
- }
- unless ($ilib) {
- $_ .= "; ilibs: not needed";
- } else {
- $found_ilib = '' unless $found_ilib;
- $_ .= "; ilibs: " . $found_ilib;
- }
- println "\t\t$_";
- }
- println "configure: error: required $a{name} package not found.";
- exit 1;
- }
- } else {
- $found_itf = abs_path($found_itf);
- push(@dependencies, "$a{namew}_INCDIR = $found_itf");
- if ($found_lib) {
- $found_lib = abs_path($found_lib);
- push(@dependencies, "$a{namew}_LIBDIR = $found_lib");
- }
- if ($ilib && $found_ilib) {
- $found_ilib = abs_path($found_ilib);
- push(@dependencies, "$a{namew}_ILIBDIR = $found_ilib");
- }
- }
-}
-
-if ($OS ne 'win' && ! $OPT{'status'}) {
- # create Makefile.config
- println "configure: creating '$OUT_MAKEFILE'" unless ($AUTORUN);
- open my $F, ">$OUT_MAKEFILE" or die "cannot open $OUT_MAKEFILE to write";
-
- print $F <<EndText;
-### AUTO-GENERATED FILE ###
-
-OS_ARCH = \$(shell perl \$(TOP)/os-arch.perl)
-
-# install paths
-EndText
-
- L($F, "INST_BINDIR = $OPT{bindir}" ) if ($OPT{bindir});
- L($F, "INST_LIBDIR = $OPT{libdir}" ) if ($OPT{libdir});
- L($F, "INST_INCDIR = $OPT{includedir}" ) if ($OPT{includedir});
- L($F, "INST_SCHEMADIR = $OPT{'shemadir'}" ) if ($OPT{'shemadir'});
- L($F, "INST_SHAREDIR = $OPT{'sharedir'}" ) if ($OPT{'sharedir'});
- L($F, "INST_JARDIR = $OPT{'javadir'}" ) if ($OPT{'javadir'});
- L($F, "INST_PYTHONDIR = $OPT{'pythondir'}") if ($OPT{'pythondir'});
-
- my ($VERSION_SHLX, $MAJMIN_SHLX, $MAJVERS_SHLX);
- if ($OSTYPE =~ /darwin/i) {
- $VERSION_SHLX = '$(VERSION).$(SHLX)';
- $MAJMIN_SHLX = '$(MAJMIN).$(SHLX)';
- $MAJVERS_SHLX = '$(MAJVERS).$(SHLX)';
- } else {
- $VERSION_SHLX = '$(SHLX).$(VERSION)';
- $MAJMIN_SHLX = '$(SHLX).$(MAJMIN)';
- $MAJVERS_SHLX = '$(SHLX).$(MAJVERS)';
- }
-
-
- print $F <<EndText;
-
-# build type
-BUILD = $BUILD
-
-# target OS
-OS = $OS
-OSINC = $OSINC
-
-# prefix string for system libraries
-LPFX = $LPFX
-
-# suffix strings for system libraries
-LIBX = $LIBX
-VERSION_LIBX = \$(LIBX).\$(VERSION)
-MAJMIN_LIBX = \$(LIBX).\$(MAJMIN)
-MAJVERS_LIBX = \$(LIBX).\$(MAJVERS)
-
-SHLX = $SHLX
-VERSION_SHLX = $VERSION_SHLX
-MAJMIN_SHLX = $MAJMIN_SHLX
-MAJVERS_SHLX = $MAJVERS_SHLX
-
-# suffix strings for system object files
-OBJX = $OBJX
-LOBX = $LOBX
-
-# suffix string for system executable
-EXEX = $EXEX
-VERSION_EXEX = \$(EXEX).\$(VERSION)
-MAJMIN_EXEX = \$(EXEX).\$(MAJMIN)
-MAJVERS_EXEX = \$(EXEX).\$(MAJVERS)
-
-# system architecture and wordsize
-ARCH = $ARCH
-EndText
-
- L($F, "# ARCH = $ARCH ( $MARCH )") if ($ARCH ne $MARCH);
-
- print $F <<EndText;
-BITS = $BITS
-
-# tools
-EndText
-
- L($F, "CC = $CC" ) if ($CC);
- L($F, "CP = $CP" ) if ($CP);
- L($F, "AR = $AR" ) if ($AR);
- L($F, "ARX = $ARX" ) if ($ARX);
- L($F, "ARLS = $ARLS" ) if ($ARLS);
- L($F, "LD = $LD" ) if ($LD);
- L($F, "LP = $LP" ) if ($LP);
- L($F, "JAVAC = $JAVAC") if ($JAVAC);
- L($F, "JAVAH = $JAVAH") if ($JAVAH);
- L($F, "JAR = $JAR" ) if ($JAR);
- L($F);
-
- L($F, '# tool options');
- if ($BUILD eq "dbg") {
- L($F, "DBG = $DBG");
- L($F, "OPT = ");
- } else {
- L($F, "DBG = -DNDEBUG") if ($PKG{LNG} eq 'C');
- L($F, "OPT = $OPT" ) if ($OPT);
- }
- L($F, "PIC = $PIC") if ($PIC);
- if ($PKG{LNG} eq 'C') {
- if ($TOOLS =~ /clang/i) {
- L($F, 'SONAME = -install_name ' .
- '$(INST_LIBDIR)$(BITS)/$(subst $(VERSION),$(MAJVERS),$(@F)) \\');
- L($F, ' -compatibility_version $(MAJMIN) -current_version $(VERSION) \\');
- L($F, ' -flat_namespace -undefined suppress');
- } else {
- L($F, 'SONAME = -Wl,-soname=$(subst $(VERSION),$(MAJVERS),$(@F))');
- }
- L($F, "SRCINC = $INC. $INC\$(SRCDIR)");
- } elsif ($PKG{LNG} eq 'JAVA') {
- L($F, 'SRCINC = -sourcepath $(INCPATHS)');
- }
- L($F, "INCDIRS = \$(SRCINC) $INC\$(TOP)") if ($PIC);
- if ($PKG{LNG} eq 'C') {
- L($F, "CFLAGS = \$(DBG) \$(OPT) \$(INCDIRS) $MD");
- }
-
- L($F, 'CLSPATH = -classpath $(CLSDIR)');
- L($F);
-
- # version information
-
- my ($VERSION, $MAJMIN, $MAJVERS);
-
- if ($FULL_VERSION =~ /(\d+)\.(\d+)\.(\d+)-?\w*\d*/) {
- $VERSION = "$1.$2.$3";
- $MAJMIN = "$1.$2";
- $MAJVERS = $1;
- } else {
- die $VERSION;
- }
-
- print $F <<EndText;
-# $PACKAGE_NAME and library version
-VERSION = $VERSION
-MAJMIN = $MAJMIN
-MAJVERS = $MAJVERS
-
-# output path
-TARGDIR = $TARGDIR
-
-# derived paths
-MODPATH ?= \$(subst \$(TOP)/,,\$(CURDIR))
-SRCDIR ?= \$(TOP)/\$(MODPATH)
-MAKEFILE ?= \$(abspath \$(firstword \$(MAKEFILE_LIST)))
-BINDIR = \$(TARGDIR)/bin
-EndText
-
- if ($PKG{LNG} eq 'C') {
- L($F, 'LIBDIR = $(TARGDIR)/lib');
- } elsif ($PKG{LNG} eq 'JAVA') {
- L($F, 'LIBDIR = $(TARGDIR)/jar');
- }
-
- print $F <<EndText;
-ILIBDIR = \$(TARGDIR)/ilib
-OBJDIR = \$(TARGDIR)/obj/\$(MODPATH)
-CLSDIR = \$(TARGDIR)/cls
-EndText
-
- if ($PKG{LNG} eq 'JAVA') {
- L($F,
- "INCPATHS = \$(SRCDIR):\$(SRCDIR)/itf:\$(TOP)/gov/nih/nlm/ncbi/ngs")
- }
-
- print $F <<EndText;
-
-# exports
-export TOP
-export MODPATH
-export SRCDIR
-export MAKEFILE
-
-# auto-compilation rules
-EndText
-
- if ($PKG{LNG} eq 'C') {
- L($F, '$(OBJDIR)/%.$(OBJX): %.c');
- T($F, '$(CC) -o $@ $< $(CFLAGS)');
- L($F, '$(OBJDIR)/%.$(LOBX): %.c');
- T($F, '$(CC) -o $@ $< $(PIC) $(CFLAGS)');
- }
- L($F, '$(OBJDIR)/%.$(OBJX): %.cpp');
- T($F, '$(CP) -o $@ $< $(CFLAGS)');
- L($F, '$(OBJDIR)/%.$(LOBX): %.cpp');
- T($F, '$(CP) -o $@ $< $(PIC) $(CFLAGS)');
- L($F);
-
- # this is part of Makefile
- L($F, 'VPATH = $(SRCDIR)');
- L($F);
-
- # we know how to find jni headers
- if ($PKG{LNG} eq 'JAVA' and $OPT{'with-ngs-sdk-src'}) {
- L($F, "JNIPATH = $OPT{'with-ngs-sdk-src'}/language/java");
- }
-
- L($F, '# directory rules');
- if ($PKG{LNG} eq 'C') {
- L($F, '$(BINDIR) $(LIBDIR) $(ILIBDIR) '
- . '$(OBJDIR) $(INST_LIBDIR) $(INST_LIBDIR)$(BITS):');
- T($F, 'mkdir -p $@');
- } elsif ($PKG{LNG} eq 'JAVA') {
- # test if we have jni header path
- L($F, '$(LIBDIR) $(CLSDIR) $(INST_JARDIR):');
- T($F, 'mkdir -p $@');
- }
- L($F);
-
- L($F, '# not real targets');
- L($F, '.PHONY: default clean install all std $(TARGETS)');
- L($F);
-
- L($F, '# dependencies');
- if ($PKG{LNG} eq 'C') {
- L($F, 'include $(wildcard $(OBJDIR)/*.d)');
- } elsif ($PKG{LNG} eq 'JAVA') {
- L($F, 'include $(wildcard $(CLSDIR)/*.d)');
- }
- L($F, $_) foreach (@dependencies);
- L($F);
-
- if ($OS eq 'linux' || $OS eq 'mac') {
- L($F, '# installation rules');
- L($F,
- '$(INST_LIBDIR)$(BITS)/%.$(VERSION_LIBX): $(LIBDIR)/%.$(VERSION_LIBX)');
- T($F, '@ echo -n "installing \'$(@F)\'... "');
- T($F, '@ if cp $^ $@ && chmod 644 $@; \\');
- T($F, ' then \\');
- T($F, ' rm -f $(subst $(VERSION),$(MAJVERS),$@) '
- . '$(subst $(VERSION_LIBX),$(LIBX),$@) '
- . '$(subst .$(VERSION_LIBX),-static.$(LIBX),$@); \\');
- T($F, ' ln -s $(@F) $(subst $(VERSION),$(MAJVERS),$@); \\');
- T($F, ' ln -s $(subst $(VERSION),$(MAJVERS),$(@F)) '
- . '$(subst $(VERSION_LIBX),$(LIBX),$@); \\');
- T($F, ' ln -s $(subst $(VERSION_LIBX),$(LIBX),$(@F)) ' .
- '$(INST_LIBDIR)$(BITS)/$(subst .$(VERSION_LIBX),-static.$(LIBX),$(@F));'
- . ' \\');
- T($F, ' echo success; \\');
- T($F, ' else \\');
- T($F, ' echo failure; \\');
- T($F, ' false; \\');
- T($F, ' fi');
- L($F);
-
- L($F,
- '$(INST_LIBDIR)$(BITS)/%.$(VERSION_SHLX): $(LIBDIR)/%.$(VERSION_SHLX)');
- T($F, '@ echo -n "installing \'$(@F)\'... "');
- T($F, '@ if cp $^ $@ && chmod 755 $@; \\');
- T($F, ' then \\');
- T($F, ' rm -f $(subst $(VERSION),$(MAJVERS),$@) '
- . '$(subst $(VERSION_SHLX),$(SHLX),$@); \\');
- T($F, ' ln -s $(@F) $(subst $(VERSION),$(MAJVERS),$@); \\');
- T($F, ' ln -s $(subst $(VERSION),$(MAJVERS),$(@F)) '
- . '$(subst $(VERSION_SHLX),$(SHLX),$@); \\');
- T($F, ' echo success; \\');
- T($F, ' else \\');
- T($F, ' echo failure; \\');
- T($F, ' false; \\');
- T($F, ' fi');
- L($F);
-
- L($F, '$(INST_BINDIR)/%$(VERSION_EXEX): $(BINDIR)/%$(VERSION_EXEX)');
- T($F, '@ echo -n "installing \'$(@F)\'... "');
- T($F, '@ if cp $^ $@ && chmod 755 $@; \\');
- T($F, ' then \\');
- T($F, ' rm -f $(subst $(VERSION),$(MAJVERS),$@) '
- . '$(subst $(VERSION_EXEX),$(EXEX),$@); \\');
- T($F, ' ln -s $(@F) $(subst $(VERSION),$(MAJVERS),$@); \\');
- T($F, ' ln -s $(subst $(VERSION),$(MAJVERS),$(@F)) '
- . '$(subst $(VERSION_EXEX),$(EXEX),$@); \\');
- T($F, ' echo success; \\');
- T($F, ' else \\');
- T($F, ' echo failure; \\');
- T($F, ' false; \\');
- T($F, ' fi');
- }
- close $F;
-}
-
-if (! $OPT{'status'} ) {
- if ($OS eq 'win') {
- my $OUT = File::Spec->catdir(CONFIG_OUT(), 'Makefile.config.win');
- println "configure: creating '$OUT'";
- open OUT, ">$OUT" or die "cannot open $OUT to write";
- my $name = PACKAGE_NAMW();
- my $outdir = $name . '_OUTDIR';
- my $root = $name . '_ROOT';
-
- print OUT <<EndText;
-<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup Label="Globals">
- <$outdir>$TARGDIR/\</$outdir>
-EndText
- foreach my $href (@REQ) {
- my %a = %$href;
- my $NGS_SDK_PREFIX = '';
- $NGS_SDK_PREFIX = $a{found_itf} if ($a{found_itf});
- if ($a{name} eq 'ngs-sdk') {
- my $root = "$a{namew}_ROOT";
- print OUT " <$root>$NGS_SDK_PREFIX\/</$root>\n";
- last;
- }
- }
- print OUT <<EndText;
- <$root>$Bin/\</$root>
- </PropertyGroup>
-</Project>
-EndText
- close OUT;
- } else {
- println "configure: creating 'Makefile.config'" unless ($AUTORUN);
- my $CONFIG_OUT = CONFIG_OUT();
- my $out = File::Spec->catdir($CONFIG_OUT, 'Makefile.config');
- open COUT, ">$out" or die "cannot open $out to write";
- print COUT "### AUTO-GENERATED FILE ###\n";
- print COUT "\n";
- print COUT "OS_ARCH = \$(shell perl \$(TOP)/os-arch.perl)\n";
- print COUT "include \$(TOP)/$CONFIG_OUT/Makefile.config.\$(OS_ARCH)\n";
- close COUT;
- }
-}
-
-status() if ($OS ne 'win');
-
-unlink 'a.out';
-
-sub L { $_[1] = '' unless ($_[1]); print { $_[0] } "$_[1]\n" }
-sub T { print { $_[0] } "\t$_[1]\n" }
-
-sub status {
- my ($load) = @_;
- if ($load) {
- ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
- my $MAKEFILE
- = File::Spec->catdir(CONFIG_OUT(), "$OUT_MAKEFILE.$OS.$ARCH");
- println "\t\tloading $MAKEFILE" if ($OPT{'debug'});
- unless (-e $MAKEFILE) {
- print STDERR "configure: error: run ./configure [OPTIONS] first.\n";
- exit 1;
- }
- open F, $MAKEFILE or die "cannot open $MAKEFILE";
- foreach (<F>) {
- chomp;
- if (/BUILD = (.+)/) {
- $BUILD_TYPE = $1;
- } elsif (/BUILD \?= /) {
- $BUILD_TYPE = $_ unless ($BUILD_TYPE);
- }
- elsif (/TARGDIR = /) {
- $TARGDIR = $_;
- println "\t\tgot $_" if ($OPT{'debug'});
- } elsif (/TARGDIR \?= (.+)/) {
- $TARGDIR = $1 unless ($TARGDIR);
- println "\t\tgot $_" if ($OPT{'debug'});
- }
- elsif (/INST_INCDIR = (.+)/) {
- $OPT{includedir} = $1;
- }
- elsif (/INST_BINDIR = (.+)/) {
- $OPT{bindir} = $1;
- }
- elsif (/INST_LIBDIR = (.+)/) {
- $OPT{libdir} = $1;
- }
- }
- }
-
- println "build type: $BUILD_TYPE";
- println "build output path: $TARGDIR" if ($OS ne 'win');
-
-# print "prefix: "; print $OPT{'prefix'} if ($OS ne 'win'); println;
-# print "eprefix: "; print $OPT{'eprefix'} if ($OPT{'eprefix'}); println;
-
- print "includedir: ";
- print $OPT{'includedir'} if ($OPT{'includedir'});
- println;
-
- print "bindir: ";
- print $OPT{'bindir'} if ($OPT{'bindir'});
- println;
-
- print "libdir: ";
- print $OPT{'libdir'} if ($OPT{'libdir'});
- println;
-
- println "schemadir: $OPT{'shemadir'}" if ($OPT{'shemadir'});
- println "sharedir: $OPT{'sharedir'}" if ($OPT{'sharedir'});
- println "javadir: $OPT{'javadir'}" if ($OPT{'javadir'});
- println "pythondir: $OPT{'pythondir'}" if ($OPT{'pythondir'});
-
- println;
-}
-
-sub expand {
- my ($filename) = @_;
- return unless ($filename);
- if ($filename =~ /^~/) {
- $filename =~ s{ ^ ~ ( [^/]* ) }
- { $1
- ? (getpwnam($1))[7]
- : ( $ENV{HOME} || $ENV{USERPROFILE} || $ENV{LOGDIR}
- || (getpwuid($<))[7]
- )
- }ex;
- }
- my $a = abs_path($filename);
- $filename = $a if ($a);
- $filename;
-}
-
-sub find_in_dir {
- my ($dir, $include, $lib, $ilib) = @_;
- print "\t$dir... " unless ($AUTORUN);
- unless (-d $dir) {
- println "no" unless ($AUTORUN);
- println "\t\tnot found $dir" if ($OPT{'debug'});
- return;
- }
- print "[found] " if ($OPT{'debug'});
- my ($found_inc, $found_lib, $found_ilib);
- my $nl = 1;
- if ($include) {
- print "includes... " unless ($AUTORUN);
- if (-e "$dir/$include") {
- println 'yes' unless ($AUTORUN);
- $found_inc = $dir;
- } elsif (-e "$dir/include/$include") {
- println 'yes' unless ($AUTORUN);
- $found_inc = "$dir/include";
- } elsif (-e "$dir/interfaces/$include") {
- println 'yes' unless ($AUTORUN);
- $found_inc = "$dir/interfaces";
- } else {
- println 'no' unless ($AUTORUN);
- }
- $nl = 0;
- }
- if ($lib || $ilib) {
- print "\n\t" if ($nl && !$AUTORUN);
- print "libraries... " unless ($AUTORUN);
- if ($lib) {
- my $builddir = File::Spec->catdir($dir, $OS, $TOOLS, $ARCH, $BUILD);
- my $libdir = File::Spec->catdir($builddir, 'lib');
- my $ilibdir = File::Spec->catdir($builddir, 'ilib');
- my $f = File::Spec->catdir($libdir, $lib);
- print "\n\t\tchecking $f\n\t" if ($OPT{'debug'});
- my $found;
- if (-e $f) {
- $found_lib = $libdir;
- if ($ilib) {
- my $f = File::Spec->catdir($ilibdir, $ilib);
- print "\tchecking $f\n\t" if ($OPT{'debug'});
- if (-e $f) {
- println 'yes';
- $found_ilib = $ilibdir;
- } else {
- println 'no' unless ($AUTORUN);
- return;
- }
- } else {
- println 'yes';
- }
- ++$found;
- }
- if (! $found) {
- my $libdir = File::Spec->catdir($dir, 'lib' . $BITS);
- my $f = File::Spec->catdir($libdir, $lib);
- print "\tchecking $f\n\t" if ($OPT{'debug'});
- if (-e $f) {
- println 'yes';
- $found_lib = $libdir;
- ++$found;
- }
- }
- if (! $found) {
- my $builddir = File::Spec->catdir
- ($dir, $OS, $TOOLS, $ARCH, reverse_build($BUILD));
- my $libdir = File::Spec->catdir($builddir, 'lib');
- my $ilibdir = File::Spec->catdir($builddir, 'ilib');
- my $f = File::Spec->catdir($libdir, $lib);
- print "\tchecking $f\n\t" if ($OPT{'debug'});
- if (-e $f) {
- $found_lib = $libdir;
- if ($ilib) {
- my $f = File::Spec->catdir($ilibdir, $ilib);
- print "\tchecking $f\n\t" if ($OPT{'debug'});
- if (-e $f) {
- println 'yes';
- $found_ilib = $ilibdir;
- } else {
- println 'no' unless ($AUTORUN);
- return;
- }
- } else {
- println 'yes';
- }
- ++$found;
- }
- }
- }
- if ($found_lib && $ilib && ! $found_ilib) {
- println "\n\t\tfound $found_lib but no ilib/" if ($OPT{'debug'});
- print "\t" if ($OPT{'debug'});
- println 'no' unless ($AUTORUN);
- undef $found_lib;
- }
- ++$nl;
- }
- return ($found_inc, $found_lib, $found_ilib);
-}
-
-sub reverse_build {
- ($_) = @_;
- if ($_ eq 'rel') {
- return 'dbg';
- } elsif ($_ eq 'dbg') {
- return 'rel';
- } else {
- die $_;
- }
-}
-
-################################################################################
-
-sub find_lib {
- my ($n, $i, $l) = @_;
-
- print "checking for $n library... ";
-
- while (1) {
- my ($library, $log);
-
- if ($n eq 'hdf5') {
- $library = '-lhdf5';
- $log = '#include <hdf5.h> \n main() { H5close (); }';
- } elsif ($n eq 'xml2') {
- $library = '-lxml2';
- $log = '#include <libxml/xmlreader.h>\n main() { xmlInitParser();}';
- } elsif ($n eq 'magic') {
- $library = '-lmagic';
- $log = '#include <magic.h> \n main() { magic_open (0); }';
- } else {
- println 'unknown: skipped';
- return;
- }
-
- if (($i && ! -d $i) || ($l && ! -d $l)) {
- println 'no';
- return;
- }
-
- my $cmd = $log;
- $cmd =~ s/\\n/\n/g;
-
- my $gcc = "| gcc -xc " . ($i ? "-I$i " : ' ')
- . ($l ? "-L$l " : ' ') . "- $library";
- $gcc .= ' 2> /dev/null' unless ($OPT{'debug'});
-
- open GCC, $gcc or last;
- print "\n\t\trunning echo -e '$log' $gcc\n" if ($OPT{'debug'});
- print GCC "$cmd" or last;
- my $ok = close GCC;
- print "\t" if ($OPT{'debug'});
- println $ok ? 'yes' : 'no';
-
- unlink 'a.out';
-
- return if (!$ok);
-
- return ($i, $l);
- }
-
- println 'cannot run gcc: skipped';
-}
-
-################################################################################
-
-sub check {
- die "No CONFIG_OUT" unless CONFIG_OUT();
- die "No PACKAGE" unless PACKAGE();
- die "No PACKAGE_NAME" unless PACKAGE_NAME();
- die "No PACKAGE_NAMW" unless PACKAGE_NAMW();
- die "No PACKAGE_TYPE" unless PACKAGE_TYPE();
- die "No VERSION" unless VERSION();
-
- my %PKG = PKG();
-
- die "No LNG" unless $PKG{LNG};
- die "No OUT" unless $PKG{OUT};
- die "No PATH" unless $PKG{PATH};
- die "No UPATH" unless $PKG{UPATH};
-
- foreach my $href (DEPENDS()) { die "No DEPENDS::name" unless $href->{name} }
-
- foreach my $href (REQ()) {
- die "No REQ::name" unless $href->{name};
-
- my $origin = $href->{origin};
- die "No $href->{name}:origin" unless $origin;
- die "No $href->{name}:include" unless $href->{include};
- die "No $href->{name}:lib" unless $href->{lib};
- die "No $href->{name}:option" unless $href->{option};
- die "No $href->{name}:pkgpath" unless $href->{pkgpath};
- die "No $href->{name}:type" unless $href->{type};
- die "No $href->{name}:usrpath" unless $href->{usrpath};
- if ($origin eq 'I') {
- die "No $href->{name}:bldpath" unless $href->{bldpath};
- die "No $href->{name}:ilib" unless $href->{ilib};
- die "No $href->{name}:namew" unless $href->{namew};
- die "No $href->{name}:srcpath" unless $href->{srcpath};
- }
- }
-}
-
-################################################################################
-
-sub optional { $_[0] =~ /^[LS]I$/ }
-
-sub help {
-# --prefix=PREFIX install architecture-independent files in PREFIX
- print <<EndText;
-`configure' configures $PACKAGE_NAME to adapt to many kinds of systems.
-
-Usage: ./configure [OPTION]...
-
-Defaults for the options are specified in brackets.
-
-Configuration:
- -h, --help display this help and exit
-
-EndText
-
- if ($^O ne 'MSWin32') {
- print <<EndText;
-Installation directories:
- --prefix=PREFIX install all files in PREFIX
- [$package_default_prefix]
-
-EndText
-
- my $other_prefix = $PKG{UPATH};
- if ($PACKAGE eq 'sra-tools' && 0) {
- print <<EndText;
- --shemadir=DIR install schema files in DIR
- [$schema_default_dir]
-
-EndText
- }
-
- print "By default, \`make install' will install all the files in\n";
-
- if (PACKAGE_TYPE() eq 'B') {
- print "\`$package_default_prefix/bin', ";
- } else {
- print "\`$package_default_prefix/include', ";
- }
- println "\`$package_default_prefix/lib' etc.";
-
- print <<EndText;
-You can specify an installation prefix other than \`$package_default_prefix'
-using \`--prefix', for instance \`--prefix=$other_prefix'.
-EndText
- }
-
- print <<EndText;
-
-For better control, use the options below.
-
-EndText
-
- my ($required, $optional);
- foreach my $href (@REQ) {
- if (optional($href->{type})) {
- ++$optional;
- } else {
- ++$required;
- }
- }
-
- if ($required) {
- print "Required Packages:\n";
- foreach my $href (@REQ) {
- next if (optional($href->{type}));
- my %a = %$href;
- if ($a{type} =~ /S/) {
- println " --$a{option}=DIR search for $a{name} package";
- println " source files in DIR";
- } else {
- println
- " --$a{option}=DIR search for $a{name} package in DIR"
- }
- if ($a{boption}) {
- println " --$a{boption}=DIR search for $a{name} package";
- println " build output in DIR";
- }
- println;
- }
- }
-
- if ($optional) {
- print "Optional Packages:\n";
- foreach my $href (@REQ) {
- next unless (optional($href->{type}));
- my %a = %$href;
- if ($a{option} =~ /-sources$/) {
- println " --$a{option}=DIR search for $a{name} package";
- println " source files in DIR";
- } else {
- println " --$a{option}=DIR search for $a{name} files in DIR"
- }
- }
- println;
- }
-
- print <<EndText if ($^O ne 'MSWin32');
-Build tuning:
- --with-debug
- --without-debug
- --arch=name specify the name of the target architecture
-
- --build=DIR generate build output into DIR directory
- [$OUTDIR]
-
-EndText
-
- println "Miscellaneous:";
- if ($^O ne 'MSWin32') {
- println
- " --status print current configuration information"
- }
- println " --clean remove all configuration results";
- println " --debug print lots of debugging information";
- println;
-}
-
-=pod
-################################################################################
-=cut
diff --git a/os-arch.perl b/os-arch.perl
deleted file mode 100644
index 2ebf922..0000000
--- a/os-arch.perl
+++ /dev/null
@@ -1,35 +0,0 @@
-use strict;
-
-use FindBin qw($Bin);
-require "$Bin/os-arch.pm";
-
-my ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
-
-my $res = "$OS.$ARCH";
-
-if (@ARCHITECTURES) {
- my $name = "$Bin/user.status";
- if (-e $name) {
- while (1) {
- open F, $name or last;
- foreach (<F>) {
- chomp;
- @_ = split /=/;
- if ($#_ == 1) {
- if ($_[0] eq 'arch') {
- foreach (@ARCHITECTURES) {
- if ($_ eq $_[1]) {
- $res = "$OS.$_[1]";
- last;
- }
- }
- last;
- }
- }
- }
- last;
- }
- }
-}
-
-print "$res\n";
diff --git a/configure b/setup/install
similarity index 83%
copy from configure
copy to setup/install
index 3313aac..4dcf8cf 100755
--- a/configure
+++ b/setup/install
@@ -29,17 +29,17 @@ CURDIR="`dirname $0`"
if [ -z "$CURDIR" ]
then
- echo configure: error: configure should be run as ./configure
+ echo error: install should be run as ./install
exit 1
fi
-if [ "$FILENAME" != configure -a ! -s konfigure.perl ]
+if [ "$FILENAME" != install -a ! -s install.perl ]
then
- echo configure: error: configure should be run as ./configure
+ echo error: install should be run as ./install
else
perl -v > /dev/null 2>&1 || { echo "checking for perl... no"; \
- echo >&2 "configure: error: perl not found."; exit 1; }
+ echo >&2 "error: perl not found."; exit 1; }
cd $CURDIR
- perl -w ./konfigure.perl $@
+ perl -w ./install.perl "$@"
fi
diff --git a/setup/install.perl b/setup/install.perl
new file mode 100644
index 0000000..fa2bae6
--- /dev/null
+++ b/setup/install.perl
@@ -0,0 +1,1329 @@
+################################################################################
+
+use strict;
+
+require 'install.prl';
+
+use Config;
+use Cwd "abs_path";
+use File::Copy "copy";
+use File::Copy::Recursive qw(dircopy);
+use File::Path "make_path";
+use FindBin qw($Bin);
+use Getopt::Long "GetOptions";
+
+my ($OS, $MAKING, %INSTALLED_LIBS);
+{
+ my $file = 'os.prl';
+ if (-e $file) {
+ require $file;
+ $OS = OS();
+ } else {
+ ++$MAKING;
+ }
+}
+
+my %HAVE = HAVE();
+BINS() if ($HAVE{BINS});
+if ($HAVE{LIBS}) {
+ ++$HAVE{INCLUDES};
+ LIBS();
+}
+if ($HAVE{INCLUDES} || $HAVE{USR_INCLUDES}) {
+ die "no INCLUDES" unless INCLUDES();
+}
+die "no CONFIG_OUT" unless CONFIG_OUT();
+
+my @bits;
+my @options = ( 'debug', 'examplesdir=s', 'force', 'help',
+ 'includedir=s', 'no-create', 'prefix=s', 'root=s', );
+push @options, 'oldincludedir=s' if ($HAVE{USR_INCLUDES});
+if ($HAVE{JAR}) {
+ push @options, 'jardir=s';
+ if (-e "$Bin/../jar") {
+ ++$HAVE{LIBS};
+ $_{JARDIR} = expand_path("$Bin/../jar");
+ }
+} elsif ($HAVE{PYTHON} && ! $MAKING) {
+ ++$HAVE{LIBS};
+}
+if (! $MAKING && ($HAVE{JAR} || $HAVE{PYTHON})) {
+ ++$HAVE{TWO_LIBS};
+ push @options, 'ngslibdir=s', 'vdblibdir=s';
+}
+push @options, 'bindir=s' if ($HAVE{BINS});
+push @options, 'bits=s' => \@bits, 'libdir=s' if ($HAVE{LIBS});
+
+my %OPT;
+unless (GetOptions(\%OPT, @options)) {
+ print "install: error\n";
+ exit 1;
+}
+ at bits = split(/,/,join(',', at bits));
+foreach (@bits) {
+ unless (/^32$/ || /^64$/) {
+ print "install: error: bad bits option argument value\n";
+ exit 1;
+ }
+}
+if ($#bits > 0) {
+ foreach (qw(bindir libdir ngslibdir vdblibdir)) {
+ if ($OPT{$_}) {
+ print "install: error: cannot supply multiple bits arguments "
+ . "when $_ argument is provided\n";
+ exit 1;
+ }
+ }
+}
+
+$OPT{root} = expand_path($OPT{root}) if ($OPT{root});
+
+prepare();
+
+my $LINUX_ROOT;
+++$LINUX_ROOT if (linux_root());
+my $ROOT = '';
+if ($OPT{root}) {
+ $ROOT = "$OPT{root}/root";
+ ++$LINUX_ROOT;
+ foreach ("$ROOT/usr/include", "$ROOT/etc/profile.d") {
+ unless (-e $_) {
+ print "mkdir -p $_... ";
+ eval { make_path($_) };
+ if ($@) {
+ print "failure: $@\n";
+ exit 1;
+ }
+ print "ok\n";
+ }
+ }
+}
+
+my $oldincludedir = "$ROOT/usr/include";
+
+my $EXAMPLES_DIR = "$Bin/../examples";
+
+ at _ = CONFIGURE();
+
+if ($OPT{help}) {
+ help();
+ exit 0;
+}
+
+foreach (qw(BITS INCDIR
+ INST_INCDIR INST_JARDIR INST_LIBDIR INST_NGSLIBDIR INST_SHAREDIR INST_VDBLIBDIR
+ LIBX LPFX MAJVERS MAJVERS_SHLX OS OTHER_PREFIX
+ PACKAGE_NAME PREFIX SHLX VERSION VERSION_LIBX VERSION_SHLX))
+{
+ unless ($_{$_}) {
+ next if (/^INST_JARDIR$/ && ! $HAVE{JAR});
+ next if (/^INST_NGSLIBDIR$/ && ! $HAVE{TWO_LIBS});
+ next if (/^INST_SHAREDIR$/ && ! $HAVE{EXAMPLES});
+ next if (/^INST_VDBLIBDIR$/ && ! $HAVE{TWO_LIBS});
+ fatal_config("$_ not found");
+ }
+}
+unless ($_{LIBDIR32} || $_{LIBDIR64} || ($HAVE{PYTHON} && $MAKING)) {
+ fatal_config('LIBDIR not found');
+}
+
+if ($OPT{prefix}) {
+ $OPT{prefix} = expand_path($OPT{prefix});
+ $_{INST_BINDIR } = "$OPT{prefix}/bin";
+ $_{INST_LIBDIR } = "$OPT{prefix}/lib";
+ $_{INST_NGSLIBDIR} = $_{INST_VDBLIBDIR} = $_{INST_LIBDIR};
+ $_{INST_INCDIR } = "$OPT{prefix}/include";
+ $_{INST_JARDIR } = "$OPT{prefix}/jar";
+ $_{INST_SHAREDIR} = "$OPT{prefix}/share";
+}
+$_{INST_SHAREDIR} = expand_path($OPT{examplesdir }) if ($OPT{examplesdir });
+$_{INST_INCDIR } = expand_path($OPT{includedir }) if ($OPT{includedir });
+$_{INST_JARDIR } = expand_path($OPT{jardir }) if ($OPT{jardir });
+$_{BIN_TARGET } = expand_path($OPT{bindir }) if ($OPT{bindir });
+$oldincludedir = expand_path($OPT{oldincludedir}) if ($OPT{oldincludedir});
+if ($OPT{libdir}) {
+ $_{INST_NGSLIBDIR} = $_{LIB_TARGET} = expand_path($OPT{libdir}) ;
+ $_{INST_VDBLIBDIR} = $_{LIB_TARGET};
+}
+$_{INST_NGSLIBDIR}= expand_path($OPT{ngslibdir}) if ($OPT{ngslibdir});
+$_{INST_VDBLIBDIR}= expand_path($OPT{vdblibdir}) if ($OPT{vdblibdir});
+
+if ($OPT{'no-create'} && $_{OS} eq 'linux') {
+ if ($LINUX_ROOT) {
+ print "root user\n\n";
+ } else {
+ print "non root user\n\n";
+ }
+}
+
+my $failures = 0;
+my $bFailure = 1;
+
+push @bits, $_{BITS} unless (@bits);
+foreach (@bits) {
+ $_{BITS} = $_;
+
+ print "installing $_{PACKAGE_NAME} ($_{VERSION}) package";
+ print " for $_{OS}-$_{BITS}" if ($HAVE{BINS} || $HAVE{LIBS});
+ print "...\n";
+
+ if ($HAVE{BINS}) {
+ $_{BINDIR} = $_{"BINDIR$_{BITS}"};
+ unless ($_{BINDIR}) {
+ print "install: error: $_{BITS}-bit version is not available\n\n";
+ next;
+ }
+ }
+ if ($HAVE{LIBS} || $HAVE{PYTHON}) {
+# ($_{LIBDIR} for python points where ngs-sdk and ncbi-vdb dynamic libraries
+# can be found to correctly set up LD_LIBRARY_PATH
+ $_{LIBDIR} = $_{"LIBDIR$_{BITS}"};
+ unless ($_{LIBDIR}) {
+ print "install: error: $_{BITS}-bit version is not available\n\n";
+ next;
+ }
+ }
+ if ($HAVE{JAR} && ! $_{JARDIR}) {
+ $_{JARDIR} = $_{"LIBDIR$_{BITS}"};
+ unless ($_{JARDIR}) {
+ if ($_{BITS} == 64) {
+ $_{JARDIR} = $_{LIBDIR32};
+ } else {
+ $_{JARDIR} = $_{LIBDIR64};
+ }
+ unless ($_{JARDIR}) {
+ print "install: error: jar file was not cannot found\n";
+ exit 1;
+ }
+ }
+ }
+ $bFailure = 0;
+
+ if ($OPT{'no-create'}) {
+ print "includedir : '$_{INST_INCDIR }'\n" if ($HAVE{INCLUDES });
+ print "libdir : '$_{INST_LIBDIR}$_{BITS}'\n" if ($HAVE{LIBS});
+ print "jardir : '$_{INST_JARDIR }'\n" if ($HAVE{JAR });
+ print "examplesdir: '$_{INST_SHAREDIR}'\n" if ($HAVE{EXAMPLES });;
+ if ($LINUX_ROOT) {
+ print "oldincludedir: '$oldincludedir'\n" if ($HAVE{USR_INCLUDES});
+ }
+ print "\n";
+ next;
+ }
+
+ $_{BIN_TARGET} = "$_{INST_BINDIR}$_{BITS}" unless ($OPT{bindir});
+ $_{LIB_TARGET} = "$_{INST_LIBDIR}$_{BITS}" unless ($OPT{libdir});
+
+ $File::Copy::Recursive::CPRFComp = 1;
+
+ $failures += copybins () if ($HAVE{BINS});
+ $failures += copylibs () if ($HAVE{LIBS});
+ $failures += copyincludes() if ($HAVE{INCLUDES});
+ $failures += copyjars () if ($HAVE{JAR});
+ $failures += copyconfig () if ($HAVE{CONFIG});
+
+ if ($HAVE{JAR}) {
+ $File::Copy::Recursive::CPRFComp = 0;
+ $failures += copydocs() ;
+ $File::Copy::Recursive::CPRFComp = 1;
+ }
+
+ $failures += copyexamples() if ($HAVE{EXAMPLES});
+ $failures += finishinstall() unless ($failures);
+
+ unless ($failures) {
+ print "\nsuccessfully installed $_{PACKAGE_NAME} ($_{VERSION}) package";
+ } else {
+ print "\nfailed to install $_{PACKAGE_NAME} ($_{VERSION}) package";
+ }
+ print " for $_{OS}-$_{BITS}" if ($HAVE{BINS} || $HAVE{LIBS});
+ print ".\n\n";
+}
+
+$failures = 1 if (!$failures && $bFailure);
+
+exit $failures;
+
+################################################################################
+
+sub copybins {
+ unless ($_{BIN_TARGET}) {
+ print "error: cannot install executables: no BIN_TARGET\n";
+ return 1;
+ }
+ my $s = $_{BINDIR};
+ my $d = $_{BIN_TARGET};
+ print "installing executables to $d...";
+ unless (-e $s) {
+ print " failure\n";
+ print "install: error: '$s' is not found.\n";
+ return 1;
+ }
+ print "\nchecking $d... ";
+ unless (-e $d) {
+ print "not found\n";
+ print "mkdir -p $d... ";
+ eval { make_path($d) };
+ if ($@) {
+ print "failure\ninstall: error: cannot mkdir $d\n";
+ return 1;
+ } else {
+ print "success\n";
+ }
+ } else {
+ print "exists\n";
+ }
+ print "\t\tcd $d\n" if ($OPT{debug});
+ chdir $d or die "cannot cd $d";
+ my $failures = 0;
+ foreach (BINS()) {
+ print "installing '$_'..." if ($OPT{debug});
+ my $df = "$_$_{VERSION_EXEX}";
+ my $sf = "$s/$df";
+ print "\n\t\t$sf -> $df\n\t" if ($OPT{debug});
+ unless (-e $sf) {
+ print " skipped\n" if ($OPT{debug});
+ next;
+ }
+ if ((! $OPT{force}) && (-e $df) && (-M $df < -M $sf)) {
+ print " found\n" if ($OPT{debug});
+ } else {
+ unless (copy($sf, $df)) {
+ print "failure\n";
+ print "install: error: cannot copy '$sf' '$df'.\n";
+ ++$failures;
+ next;
+ }
+ my $mode = 0755;
+ printf "\tchmod %o $df\n\t", $mode if ($OPT{debug});
+ unless (chmod($mode, $df)) {
+ print " failure\n" if ($OPT{debug});
+ print "install: error: cannot chmod '$df': $!\n";
+ ++$failures;
+ next;
+ }
+ unless (symlinks($_, $df, 'bin')) {
+ print " success\n" if ($OPT{debug});
+ } else {
+ print " failure\n" if ($OPT{debug});
+ ++$failures;
+ }
+ }
+ }
+ return $failures;
+}
+
+sub copyconfig {
+ my $d;
+ if ($LINUX_ROOT) {
+ $d = "$ROOT/etc";
+ }
+ elsif ($HAVE{BINS}) {
+ $d = $_{BIN_TARGET};
+ unless ($d) {
+ print
+ "error: cannot install configuration files: no BIN_TARGET\n";
+ return 1;
+ }
+ } else {
+ $d = $_{LIB_TARGET};
+ unless ($d) {
+ print
+ "error: cannot install configuration files: no LIB_TARGET\n";
+ return 1;
+ }
+ }
+ $d = File::Spec->catdir($d, 'ncbi');
+ my $kfg = File::Spec->catdir($Bin, '..', 'libs/kfg/default.kfg');
+ unless (-e $kfg) {
+ $kfg = File::Spec->catdir($Bin, '..', 'tools/vdb-copy/vdb-copy.kfg');
+ }
+ unless (-e $kfg) {
+ if ($_{BINDIR}) {
+ $kfg = File::Spec->catdir($_{BINDIR}, 'ncbi', 'vdb-copy.kfg');
+ } elsif ($_{LIBDIR}) {
+ $kfg = File::Spec->catdir($_{LIBDIR}, 'ncbi', 'default.kfg');
+ unless (-e $kfg) {
+ print
+ "error: cannot install configuration files: no default.kfg\n";
+ return 1;
+ }
+ }
+ }
+ print "installing configuration files to $d... ";
+ print "\nchecking $d... ";
+ unless (-e $d) {
+ print "not found\n";
+ print "mkdir -p $d... ";
+ eval { make_path($d) };
+ if ($@) {
+ print "failure\ninstall: error: cannot mkdir $d\n";
+ return 1;
+ } else {
+ print "success\n";
+ }
+ } else {
+ print "exists\n";
+ }
+ my $df = File::Spec->catdir($d, 'ncbi-vdb.kfg');
+ print "\t\t$kfg -> $df\n" if ($OPT{debug});
+ unless (copy($kfg, $df)) {
+ print "install: error: cannot copy '$kfg' '$df'.\n";
+ return 1;
+ } else {
+ print "success\n";
+ return 0;
+ }
+}
+
+sub copylibs {
+ die unless ($HAVE{LIBS});
+
+ my $s = $_{LIBDIR};
+ my $d = $_{LIB_TARGET};
+
+ print "installing libraries to $d... ";
+
+ unless (-e $s) {
+ print "\tfailure\n";
+ print "install: error: '$s' is not found.\n";
+ return 1;
+ }
+
+ if ($HAVE{TWO_LIBS}) {
+ my $ngs = $_{INST_NGSLIBDIR};
+ if ($ngs && ! ($OPT{prefix} && $OPT{libdir} && $OPT{ngslibdir})) {
+ $ngs .= $_{BITS};
+ }
+ my $vdb = $_{INST_VDBLIBDIR};
+ if ($vdb && ! ($OPT{prefix} && $OPT{libdir} && $OPT{vdblibdir})) {
+ $vdb .= $_{BITS};
+ }
+ if ($ngs || $vdb) {
+ unless ($ngs && $vdb) {
+ $ngs = $d unless ($ngs);
+ $vdb = $d unless ($vdb);
+ }
+ $INSTALLED_LIBS{'ngs-sdk' } = $ngs;
+ $INSTALLED_LIBS{'ncbi-vdb'} = $vdb;
+ }
+ }
+ $INSTALLED_LIBS{0} = $d unless (%INSTALLED_LIBS);
+
+ foreach (keys %INSTALLED_LIBS) {
+ my $d = $INSTALLED_LIBS{$_};
+ print "\nchecking $d... ";
+ unless (-e $d) {
+ print "not found\n";
+ print "mkdir -p $d... ";
+ eval { make_path($d) };
+ if ($@) {
+ print "failure\ninstall: error: cannot mkdir $d\n";
+ return 1;
+ } else {
+ print "success\n";
+ }
+ } else {
+ print "exists\n";
+ }
+ }
+
+ return $MAKING ? copybldlibs($s, $d) : copydir($s, %INSTALLED_LIBS);
+}
+
+sub copybldlibs {
+ my ($s, $d) = @_;
+
+ print "\t\tcd $d\n" if ($OPT{debug});
+ chdir $d or die "cannot cd $d";
+
+ my $failures = 0;
+
+ my %LIBRARIES_TO_INSTALL = LIBS();
+ foreach (keys %LIBRARIES_TO_INSTALL) {
+ print "installing '$_'... ";
+
+ my $nb = "$_{LPFX}$_";
+ my $nv = "$nb.";
+ my $lib = 'dll';
+ if ($LIBRARIES_TO_INSTALL{$_} eq 'SHL') {
+ $nv .= $_{VERSION_SHLX};
+ } elsif ($LIBRARIES_TO_INSTALL{$_} eq 'LIB') {
+ $nv .= $_{VERSION_LIBX};
+ $lib = 'lib';
+ } else {
+ die "bad library type";
+ }
+
+ my $sf = "$s/$nv";
+ my $df = "$d/$nv";
+
+ print "\n\t\t$sf -> $df\n\t" if ($OPT{debug});
+
+ unless (-e $sf) {
+ print "failure\n";
+ print "install: error: '$sf' is not found.\n";
+ ++$failures;
+ next;
+ }
+
+ if ((! $OPT{force}) && (-e $df) && (-M $df < -M $sf)) {
+ print "found\n";
+ } else {
+ unless (copy($sf, $df)) {
+ print "failure\n";
+ print "install: error: cannot copy '$sf' '$df'.\n";
+ ++$failures;
+ next;
+ }
+ my $mode = 0644;
+ $mode = 0755 if ($lib eq 'dll');
+ printf "\tchmod %o $df\n\t", $mode if ($OPT{debug});
+ unless (chmod($mode, $df)) {
+ print "failure\n";
+ print "install: error: cannot chmod '$df': $!\n";
+ ++$failures;
+ next;
+ }
+ unless (symlinks($nb, $nv, $lib)) {
+ print "success\n";
+ } else {
+ print "failure\n";
+ ++$failures;
+ }
+ }
+ }
+
+ return $failures;
+}
+
+sub symlinks {
+ my ($nb, $nv, $type) = @_;
+
+ my @l;
+ if ($type eq 'lib') {
+ push @l, "$nb-static.$_{LIBX}";
+ push @l, "$nb.$_{LIBX}";
+ push @l, "$nb.$_{MAJVERS_LIBX}";
+ } elsif ($type eq 'dll') {
+ push @l, "$nb.$_{SHLX}";
+ push @l, "$nb.$_{MAJVERS_SHLX}";
+ } elsif ($type eq 'bin' || $type eq 'jar') {
+ push @l, $nb;
+ push @l, "$nb.$_{MAJVERS}";
+ } else {
+ print "failure\n";
+ print "install: error: unknown symlink type '$type'\n";
+ return 1;
+ }
+
+ my $failures = 0;
+
+ for (my $i = 0; $i <= $#l; ++$i) {
+ my $file = $l[$i];
+ if (-e $file) {
+ print "\trm $file\n\t" if ($OPT{debug});
+ unless (unlink $file) {
+ print "failure\n";
+ print "install: error: cannot rm '$file': $!\n";
+ ++$failures;
+ next;
+ }
+ }
+
+ my $o = $nv;
+ $o = $l[$i + 1] if ($i < $#l);
+
+ print "\tln -s $o $file\n\t" if ($OPT{debug});
+ unless (symlink $o, $file) {
+ print "failure\n";
+ print "install: error: cannot symlink '$o' '$file': $!\n";
+ ++$failures;
+ next;
+ }
+ }
+
+ return $failures;
+}
+
+sub copydir {
+ my ($s, %d) = @_;
+
+ my $failures = 0;
+
+ foreach my $pattern(keys %d) {
+ my $d = $d{$pattern};
+ print "\t\tcd $d\n" if ($OPT{debug});
+ chdir $d or die "cannot cd $d";
+
+ opendir(D, $s) or die "cannot opendir $s: $!";
+
+ while (readdir D) {
+ next if (/^\.{1,2}$/);
+ next if ($pattern && ! /$pattern/);
+
+ my $n = "$s/$_";
+
+ if (-l $n) {
+ print "\t\t$_ (symlink)... " if ($OPT{debug});
+ my $l = readlink $n;
+ if ((-e $_) && (!unlink $_)) {
+ print "error: cannot remove $l: $!\n";
+ ++$failures;
+ next;
+ }
+ unless (symlink($l, $_)) {
+ print "error: cannot create symlink from $_ to $l: $!\n";
+ ++$failures;
+ next;
+ }
+ print "success\n" if ($OPT{debug});
+ } else {
+ print "\t\t$_... " if ($OPT{debug});
+ if ((-e $_) && (!unlink $_)) {
+ print "error: cannot remove $_: $!\n";
+ ++$failures;
+ next;
+ }
+ unless (copy($n, $_)) {
+ print "error: cannot copy '$n' to '$_': $!\n";
+ ++$failures;
+ next;
+ }
+ print "success\n" if ($OPT{debug});
+ }
+ }
+
+ closedir D;
+ }
+
+ return $failures;
+}
+
+sub includes_out {
+ my $out = '';
+ eval { $out = INCLUDES_OUT(); };
+ $out = File::Spec->catdir($_{INST_INCDIR}, $out);
+ $out;
+}
+
+sub copyincludes {
+ print "installing includes to $_{INST_INCDIR}... ";
+
+ my $s = "$_{INCDIR}/" . INCLUDES();
+ unless (-e $s) {
+ print "\tfailure\n";
+ print "install: error: '$s' is not found.\n";
+ return 1;
+ }
+
+ my $out = includes_out();
+ my $d = $out;
+ $d = $_{INST_INCDIR} unless ($d);
+
+ unless (-e $d) {
+ print "\n\t\tmkdir -p $d" if ($OPT{debug});
+ eval { make_path($d) };
+ if ($@) {
+ print "\tfailure\ninstall: error: cannot mkdir $d\n";
+ return 1;
+ }
+ }
+
+ if ($out && -f $s) {
+ print "\n\t\tcp $s $d\n\t" if ($OPT{debug});
+ unless (copy($s, $d)) {
+ print "failure\n";
+ return 1;
+ }
+ } else {
+ print "\n\t\tcp -r $s $d\n\t" if ($OPT{debug});
+ unless (dircopy($s, $d)) {
+ print "\tfailure\ninstall: error: cannot copy '$s' 'd'";
+ return 1;
+ }
+ }
+
+ print "success\n";
+ return 0;
+}
+
+sub copyjars {
+ my $s = $_{JARDIR};
+ my $d = $_{INST_JARDIR};
+
+ print "installing jar files to $d... ";
+
+ unless (-e $s) {
+ print "\tfailure\n";
+ print "install: error: '$s' is not found.\n";
+ return 1;
+ }
+
+ print "\nchecking $d... ";
+ unless (-e $d) {
+ print "not found\n";
+ print "mkdir -p $d... ";
+ eval { make_path($d) };
+ if ($@) {
+ print "failure\ninstall: error: cannot mkdir $d\n";
+ return 1;
+ } else {
+ print "success\n";
+ }
+ } else {
+ print "exists\n";
+ }
+
+ return $MAKING ? copybldjars($s, $d) : copydir($s, 0 => $d);
+}
+
+sub copybldjars {
+ my ($s, $d) = @_;
+ my $n = 'ngs-java.jar';
+ $s .= "/$n";
+
+ unless (-e $s) {
+ print "\tfailure\n";
+ print "install: error: '$s' is not found.\n";
+ return 1;
+ }
+
+ my $nd = "$n.$_{VERSION}";
+ print "installing '$n'... ";
+
+ print "\t\tcd $d\n" if ($OPT{debug});
+ chdir $d or die "cannot cd $d";
+
+ $d .= "/$nd";
+
+ print "\n\t\t$s -> $d\n\t" if ($OPT{debug});
+
+ if ((! $OPT{force}) && (-e $d) && (-M $d < -M $s)) {
+ print "found\n";
+ } else {
+ unless (copy($s, $d)) {
+ print "failure\n";
+ print "install: error: cannot copy '$s' '$d'.\n";
+ return 1;
+ }
+ my $mode = 0644;
+ printf "\tchmod %o $d\n\t", $mode if ($OPT{debug});
+ unless (chmod($mode, $d)) {
+ print "failure\n";
+ print "install: error: cannot chmod '$d': $!\n";
+ return 1;
+ }
+ unless (symlinks($n, $nd, 'jar')) {
+ print "success\n";
+ } else {
+ print "failure\n";
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+sub copydocs {
+ my $s = "$_{JARDIR}/javadoc";
+ $s = expand_path("$Bin/../doc") unless ($MAKING);
+ my $d = "$_{INST_SHAREDIR}/doc";
+
+ print "installing html documents to $d... ";
+
+ unless (-e $s) {
+ print "\tfailure\n";
+ print "install: error: '$s' is not found.\n";
+ return 1;
+ }
+
+ print "\nchecking $d... ";
+ unless (-e $d) {
+ print "not found\n";
+ print "mkdir -p $d... ";
+ eval { make_path($d) };
+ if ($@) {
+ print "failure\ninstall: error: cannot mkdir $d\n";
+ return 1;
+ } else {
+ print "success\n";
+ }
+ } else {
+ print "exists\n";
+ }
+
+ print "\t\t$s -> $d\n\t" if ($OPT{debug});
+ unless (dircopy($s, $d)) {
+ print "\tfailure\ninstall: error: cannot copy '$s' to '$d'";
+ return 1;
+ }
+
+ print "success\n";
+ return 0;
+}
+
+sub copyexamples {
+ my $failures = 0;
+ my $CPRFComp = $File::Copy::Recursive::CPRFComp;
+ my $sd = $EXAMPLES_DIR;
+ return 0 unless (-e $sd);
+
+ my $d = $_{INST_SHAREDIR};
+ unless ($d) {
+ print "install: error: cannot install examples\n";
+ return 0;
+ }
+
+ if ($HAVE{JAR}) {
+ $d .= '/examples-java';
+ } elsif ($HAVE{PYTHON}) {
+ $File::Copy::Recursive::CPRFComp = 0;
+ $d .= '/examples-python';
+ }
+
+ print "installing examples to $d... ";
+
+ my $s = $sd;
+ $s = "$sd/examples" if ($HAVE{JAR} && $MAKING);
+
+ unless (-e $s) {
+ print "\tfailure\n";
+ print "install: error: '$s' is not found.\n";
+ ++$failures;
+ }
+
+ unless ($failures) {
+ print "\nchecking $d... ";
+ unless (-e $d) {
+ print "not found\n";
+ print "mkdir -p $d... ";
+ eval { make_path($d) };
+ if ($@) {
+ print "failure\ninstall: error: cannot mkdir $d\n";
+ ++$failures;
+ } else {
+ print "success\n";
+ }
+ } else {
+ print "exists\n";
+ }
+ }
+
+ unless ($failures) {
+ print "\t\t$s -> $d\n\t" if ($OPT{debug});
+ if ($HAVE{JAR} && ! $MAKING) {
+ if (copydir($s, 0 => $d)) {
+ ++$failures;
+ }
+ } else {
+ unless (dircopy($s, $d)) {
+ print "\tfailure\ninstall: error: cannot copy '$s' to '$d'";
+ ++$failures;
+ }
+ }
+ }
+
+ unless ($failures) {
+ if ($HAVE{JAR} && $MAKING) {
+ $sd = "$sd/Makefile";
+ $d = "$d/Makefile";
+ print "\t$sd -> $d\n\t" if ($OPT{debug});
+ unless (-e $sd) {
+ print "\tfailure\n";
+ print "install: error: '$sd' is not found.\n";
+ ++$failures;
+ }
+ unless ($failures) {
+ if (-e $d) {
+ unless (unlink $d) {
+ print "failure\n";
+ print "install: error: cannot rm '$d': $!\n";
+ ++$failures;
+ }
+ }
+ unless ($failures) {
+ unless (copy($sd, $d)) {
+ print "error: cannot copy '$sd' to '$d': $!\n";
+ ++$failures;
+ }
+ }
+ }
+ }
+ }
+
+ print "success\n" unless ($failures);
+
+ $File::Copy::Recursive::CPRFComp = $CPRFComp;
+
+ return $failures;
+}
+
+sub finishinstall {
+ my $failures = 0;
+
+ $_{JAR_TARGET} = "$_{INST_JARDIR}/ngs-java.jar";
+
+ my @libs;
+ if (%INSTALLED_LIBS) {
+ my %libs;
+ ++$libs{$INSTALLED_LIBS{$_}} foreach (keys %INSTALLED_LIBS);
+ push @libs, $_ foreach (keys %libs);
+ } else {
+ push @libs, $_{LIB_TARGET};
+ }
+ my $libs;
+ foreach (@libs) {
+ $libs .= ":" if ($libs);
+ $libs .= $_;
+ }
+
+ if ($HAVE{PYTHON}) {
+ chdir "$Bin/.." or die "cannot cd '$Bin/..'";
+ my $cmd = "python setup.py install";
+ $cmd .= ' --user' unless (linux_root());
+ print `$cmd`;
+ if ($?) {
+ ++$failures;
+ } else {
+ unless ($libs) {
+ print "internal python failure\n";
+ ++$failures;
+ } elsif ($HAVE{LIBS}) {
+ print <<EndText;
+Please add $libs to your LD_LIBRARY_PATH, e.g.:
+ export LD_LIBRARY_PATH=$libs:\$LD_LIBRARY_PATH
+EndText
+ }
+ }
+ } elsif ($LINUX_ROOT) {
+ print "\t\tlinux root\n" if ($OPT{debug});
+
+ if ($HAVE{USR_INCLUDES}) {
+ unless (-e $oldincludedir) {
+ print "install: error: '$oldincludedir' does not exist\n";
+ ++$failures;
+ } else {
+ my $o = includes_out();
+ if ($o) {
+ eval { INCLUDES_OUT(); };
+ if ($@) {
+ print "install: cannot find INCLUDES_OUT\n";
+ ++$failures;
+ } else {
+ my $INCLUDE_SYMLINK
+ = "$oldincludedir/" . INCLUDES_OUT();
+ print "updating $INCLUDE_SYMLINK... ";
+ unlink $INCLUDE_SYMLINK;
+ if ($OPT{debug}) {
+ print "\n\t\tln -s $o $INCLUDE_SYMLINK... ";
+ }
+ unless (symlink $o, $INCLUDE_SYMLINK) {
+ print "failure\n";
+ print "install: error: " .
+ "cannot symlink '$o' '$INCLUDE_SYMLINK': $!\n";
+ ++$failures;
+ } else {
+ print "success\n";
+ }
+ }
+ } else {
+ my $INCLUDE_SYMLINK = "$oldincludedir/" . INCLUDES();
+ print "updating $INCLUDE_SYMLINK... ";
+ unlink $INCLUDE_SYMLINK;
+ my $o = "$_{INST_INCDIR}/" . INCLUDES();
+ unless (symlink $o, $INCLUDE_SYMLINK) {
+ print "failure\n";
+ print "install: error: "
+ . "cannot symlink '$o' '$INCLUDE_SYMLINK': $!\n";
+ ++$failures;
+ } else {
+ print "success\n";
+ }
+ }
+ }
+ }
+
+ my $NAME = PACKAGE_NAME();
+ if ($HAVE{BINS} || $HAVE{JAR}
+ || ($HAVE{LIBS}
+ && ($HAVE{DLLS} || $NAME eq 'NGS-SDK' || $NAME eq 'NGS-BAM')
+ )
+ )
+ {
+ my $profile = "$ROOT/etc/profile.d";
+ my $PROFILE_FILE = "$profile/" . lc(PACKAGE_NAME());
+ unless (-e $profile) {
+ print "install: error: '$profile' does not exist\n";
+ ++$failures;
+ } else {
+ print "updating $PROFILE_FILE.[c]sh... ";
+
+ my $f = "$PROFILE_FILE.sh";
+ if (!open F, ">$f") {
+ print "failure\n";
+ print "install: error: cannot open '$f': $!\n";
+ ++$failures;
+ } else {
+ print F "#version $_{VERSION}\n\n";
+
+ if ($HAVE{LIBS}) {
+ unless (@libs) {
+ print "internal root libraries failure\n";
+ ++$failures;
+ } else {
+ if ($HAVE{DLLS}) {
+ foreach (@libs) {
+ print F <<EndText;
+if ! echo \$LD_LIBRARY_PATH | /bin/grep -q $_
+then export LD_LIBRARY_PATH=$_:\$LD_LIBRARY_PATH
+fi
+
+EndText
+ }
+ }
+ if ($NAME eq 'NGS-SDK') {
+ print F "export NGS_LIBDIR=$_{LIB_TARGET}\n";
+ } elsif ($NAME eq 'NGS-BAM') {
+ print F
+ "\nexport NGS_BAM_LIBDIR=$_{LIB_TARGET}\n"
+ }
+ }
+ }
+ if ($HAVE{JAR}) {
+ print F <<EndText;
+if ! echo \$CLASSPATH | /bin/grep -q $_{JAR_TARGET}
+then export CLASSPATH=$_{JAR_TARGET}:\$CLASSPATH
+fi
+EndText
+ }
+ if ($HAVE{BINS}) {
+ print F <<EndText;
+if ! echo \$PATH | /bin/grep -q $_{INST_BINDIR}
+then export PATH=$_{INST_BINDIR}:\$PATH
+fi
+EndText
+ }
+ close F;
+ unless (chmod(0644, $f)) {
+ print "failure\n";
+ print "install: error: cannot chmod '$f': $!\n";
+ ++$failures;
+ }
+ }
+ }
+
+ my $f = "$PROFILE_FILE.csh";
+ if (!open F, ">$f") {
+ print "failure\n";
+ print "install: error: cannot open '$f': $!\n";
+ ++$failures;
+ } else {
+ print F "#version $_{VERSION}\n\n";
+
+ if ($HAVE{LIBS}) {
+ unless (@libs) {
+ print "internal libraries failure\n";
+ ++$failures;
+ } else {
+ if ($HAVE{DLLS}) {
+ foreach (@libs) {
+ print F <<EndText;
+echo \$LD_LIBRARY_PATH | /bin/grep -q $_
+if ( \$status ) setenv LD_LIBRARY_PATH $_:\$LD_LIBRARY_PATH
+
+EndText
+ }
+ }
+ }
+ if (PACKAGE_NAME() eq 'NGS-BAM') {
+ print F "setenv NGS_BAM_LIBDIR $_{LIB_TARGET}\n";
+ } elsif (PACKAGE_NAME() eq 'NGS-SDK') {
+ print F "setenv NGS_LIBDIR $_{LIB_TARGET}\n";
+ } elsif (PACKAGE_NAME() eq 'NCBI-VDB') {
+ print F "setenv NCBI_VDB_LIBDIR $_{LIB_TARGET}\n";
+ }
+ }
+ if ($HAVE{JAR}) {
+ print F <<EndText;
+echo \$CLASSPATH | /bin/grep -q $_{JAR_TARGET}
+if ( \$status ) setenv CLASSPATH $_{JAR_TARGET}:\$CLASSPATH
+EndText
+ }
+ if ($HAVE{BINS}) {
+ print F <<EndText;
+echo \$PATH | /bin/grep -q $_{INST_BINDIR}
+if ( \$status ) setenv PATH $_{INST_BINDIR}:\$PATH
+EndText
+ }
+ close F;
+ unless (chmod(0644, $f)) {
+ print "failure\n";
+ print "install: error: cannot chmod '$f': $!\n";
+ ++$failures;
+ }
+ }
+# @ #TODO: check version of the files above
+ print "success\n" unless ($failures);
+ }
+
+ unless ($failures) {
+ if ($HAVE{LIBS}) {
+ if (PACKAGE_NAME() eq 'NGS-BAM') {
+ print "\n";
+ print "Use \$NGS_BAM_LIBDIR in your link commands, e.g.:\n";
+ print " ld -L\$NGS_BAM_LIBDIR -lngs-bam ...\n";
+ } elsif (PACKAGE_NAME() eq 'NGS-SDK') {
+ print "\nUse \$NGS_LIBDIR in your link commands, e.g.:\n";
+ print " ld -L\$NGS_LIBDIR -lngs-sdk ...\n";
+ } elsif (PACKAGE_NAME() eq 'NCBI-VDB') {
+ print "\n"
+ . "Use \$NCBI_VDB_LIBDIR in your link commands, e.g.:\n";
+ print " ld -L\$NCBI_VDB_LIBDIR -lncbi-vdb ...\n";
+ }
+ }
+ }
+ } else {
+ print "\t\tnot linux root\n" if ($OPT{debug});
+ if ($HAVE{LIBS}) {
+ unless ($libs) {
+ print "internal libraries failure\n";
+ ++$failures;
+ } else {
+ print "\n";
+ print <<EndText if ($HAVE{DLLS});
+Please add $libs to your LD_LIBRARY_PATH, e.g.:
+ export LD_LIBRARY_PATH=$libs:\$LD_LIBRARY_PATH
+EndText
+ if (PACKAGE_NAME() eq 'NGS-SDK') {
+ print "Use $libs in your link commands, e.g.:\n"
+ . "export NGS_LIBDIR=$libs\n"
+ . "ld -L\$NGS_LIBDIR -lngs-sdk ...\n";
+ } elsif (PACKAGE_NAME() eq 'NGS-BAM') {
+ print "Use $libs in your link commands, e.g.:\n"
+ . "export NGS_BAM_LIBDIR=$libs\n"
+ . "ld -L\$NGS_BAM_LIBDIR -lngs-bam ...\n";
+ }
+ }
+ }
+ if ($HAVE{JAR}) {
+ print <<EndText;
+
+Please add $_{JAR_TARGET} to your CLASSPATH, i.e.:
+ export CLASSPATH=$_{JAR_TARGET}:\$CLASSPATH
+EndText
+ }
+ }
+
+ return $failures;
+}
+
+sub expand_path {
+ my ($filename) = @_;
+ return unless ($filename);
+
+ if ($filename =~ /^~/) {
+ if ($filename =~ m|^~([^/]*)|) {
+ if ($1 && ! getpwnam($1)) {
+ print "install: error: bad path: '$filename'\n";
+ exit 1;
+ }
+ }
+
+ $filename =~ s{ ^ ~ ( [^/]* ) }
+ { $1
+ ? (getpwnam($1))[7]
+ : ( $ENV{HOME} || $ENV{USERPROFILE} || $ENV{LOGDIR}
+ || (getpwuid($<))[7]
+ )
+ }ex;
+ }
+
+ my $a = abs_path($filename);
+ $filename = $a if ($a);
+
+ $filename;
+}
+
+sub help {
+ $_{LIB_TARGET} = "$_{INST_LIBDIR}$_{BITS}";
+
+ print <<EndText;
+'install' installs $_{PACKAGE_NAME} $_{VERSION} package.
+
+Usage: ./install [OPTION]...
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+ -h, --help display this help and exit
+ -n, --no-create do not run installation
+
+EndText
+
+ if ($HAVE{TWO_LIBS}) {
+ my $p = lc(PACKAGE_NAME());
+ print "By default, `./install' will install all the files in\n";
+ print "`/usr/local/ngs/$p/jar', " if ($HAVE{JAR});
+ print <<EndText;
+`/usr/local/ngs/$p/share',
+`/usr/local/ngs/ngs-sdk/lib$_{BITS}', `/usr/local/ncbi/ncbi-vdb/lib$_{BITS}'.
+You can spefify other installation directories using the options below.
+
+Fine tuning of the installation directories:
+EndText
+ if ($HAVE{JAR}) {
+ print
+ " --jardir=DIR jar files [/usr/local/ngs/$p/jar]\n"
+ }
+ print <<EndText;
+ --ngslibdir=DIR ngs-sdk libraries [/usr/local/ngs/ngs-sdk/lib$_{BITS}]
+ --vdblibdir=DIR ncbi-vdb libraries [/usr/local/ncbi/ncbi-vdb/lib$_{BITS}]
+ --examplesdir=DIR example files [/usr/local/ngs/$p/share]
+
+ --libdir=DIR install all libraries in the same directory
+ --prefix=DIR install files in PREFIX/lib$_{BITS}, PREFIX/share etc.
+EndText
+ } else {
+ print <<EndText;
+Installation directories:
+ --prefix=PREFIX install all files in PREFIX
+ [$_{PREFIX}]
+
+By default, `./install' will install all the files in
+EndText
+
+ if ($HAVE{INCLUDES}) {
+ print
+"`$_{PREFIX}/include', `$_{PREFIX}/lib$_{BITS}' etc. You can specify\n"
+ } elsif ($HAVE{JAR}) {
+ print "`$_{PREFIX}/jar', `$_{PREFIX}/share' etc. You can specify\n"
+ } elsif ($MAKING) {
+ print "`$_{PREFIX}/share' etc. You can specify\n"
+ } else {
+ print
+"`$_{PREFIX}/lib$_{BITS}' `$_{PREFIX}/share' etc. You can specify\n"
+ }
+
+ print <<EndText;
+an installation prefix other than `$_{PREFIX}' using `--prefix',
+for instance `--prefix=$_{OTHER_PREFIX}'.
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+EndText
+
+ if ($HAVE{BINS}) {
+ print " --bindir=DIR executables [PREFIX/bin]\n";
+ }
+ if ($HAVE{JAR}) {
+ print " --jardir=DIR jar files [PREFIX/jar]\n";
+ }
+ if ($HAVE{LIBS}) {
+ print
+" --libdir=DIR object code libraries [PREFIX/lib$_{BITS}]\n"
+ }
+ if ($HAVE{INCLUDES}) {
+ print " --includedir=DIR C header files [PREFIX/include]\n";
+ }
+ if ($HAVE{USR_INCLUDES}) {
+ print
+" --oldincludedir=DIR C header files for non-gcc [$oldincludedir]\n"
+ }
+
+ if (-e $EXAMPLES_DIR) {
+ print " --examplesdir=DIR example files [PREFIX/share]\n";
+ }
+ }
+
+ if ($HAVE{LIBS}) {
+ print <<EndText;
+
+System types:
+ --bits=[32|64] use a 32- or 64-bit data model
+EndText
+ }
+
+ print "\nReport bugs to sra-tools\@ncbi.nlm.nih.gov\n";
+}
+
+sub prepare {
+ if ($MAKING) {
+ my $os_arch = `perl -w $Bin/os-arch.perl`;
+ unless ($os_arch) {
+ print "install: error\n";
+ exit 1;
+ }
+ chomp $os_arch;
+ my $config =
+ "$Bin/../" . CONFIG_OUT() . "/Makefile.config.install.$os_arch.prl";
+ fatal_config("$config not found") unless (-e "$config");
+
+ eval { require $config; };
+ fatal_config($@) if ($@);
+ } else {
+ my $a = $Config{archname64};
+ $_ = lc PACKAGE_NAME();
+ my $root = '';
+ $root = $OPT{root} if ($OPT{root});
+ my $code =
+ 'sub CONFIGURE { ' .
+ ' $_{OS } = $OS; ' .
+ ' $_{VERSION } = "1.0.0"; ' .
+ ' $_{MAJVERS } = "1"; ' .
+ ' $_{LPFX } = "lib"; ' .
+ ' $_{LIBX } = "a"; ' .
+ ' $_{MAJVERS_LIBX } = "a.1"; ' .
+ ' $_{VERSION_LIBX } = "a.1.0.0"; ' .
+ ' $_{SHLX } = "so"; ' .
+ ' $_{OTHER_PREFIX } = \'$HOME/ngs/' . $_ . '\'; ' .
+ ' $_{PREFIX } = "' . "$root/usr/local/ngs/$_" . '"; ' .
+ ' $_{INST_INCDIR } = "$_{PREFIX}/include"; ' .
+ ' $_{INST_LIBDIR } = "$_{PREFIX}/lib"; ' .
+ ' $_{INST_JARDIR } = "$_{PREFIX}/jar"; ' .
+ ' $_{INST_SHAREDIR} = "$_{PREFIX}/share"; ' .
+ ' $_{INCDIR } = "$Bin/../include"; ' .
+ ' $_{LIBDIR64 } = "$Bin/../lib64"; ' .
+ ' $_{LIBDIR32 } = "$Bin/../lib32"; ';
+ if ($HAVE{TWO_LIBS}) {
+ $code .=
+ '$_{INST_NGSLIBDIR} = "' . "$root/usr/local/ngs/ngs-sdk/lib\";"
+ . '$_{INST_VDBLIBDIR} = "' . "$root/usr/local/ncbi/ncbi-vdb/lib\";"
+ }
+ $code .= ' $_{PACKAGE_NAME} = "' . PACKAGE_NAME() . '"; ';
+
+ if (defined $Config{archname64}) {
+ $code .= ' $_{BITS} = 64; ';
+ } else {
+ $code .= ' $_{BITS} = 32; ';
+ }
+
+ $code .=
+ ' $_{MAJVERS_SHLX } = "so.1"; ' .
+ ' $_{VERSION_SHLX } = "so.1.0.0"; ' ;
+
+ $code .=
+ ' @_ ' .
+ '}';
+
+ eval $code;
+
+ die $@ if ($@);
+ }
+}
+
+sub linux_root { $^O eq 'linux' && `id -u` == 0 }
+
+sub fatal_config {
+ if ($OPT{debug}) {
+ print "\t\t";
+ print "@_";
+ print "\n";
+ }
+
+ print "install: error: run ./configure [OPTIONS] first.\n";
+
+ exit 1;
+}
+
+################################################################################
diff --git a/setup/install.prl b/setup/install.prl
new file mode 100644
index 0000000..8b26a51
--- /dev/null
+++ b/setup/install.prl
@@ -0,0 +1,38 @@
+sub PACKAGE_NAME { 'SRA-TOOLS' }
+sub HAVE { ( BINS => 1, CONFIG => 1,) }
+sub CONFIG_OUT { 'build' }
+sub BINS { qw (abi-dump
+ abi-load
+ align-info
+ bam-load
+ cache-mgr
+ cg-load
+ fastq-dump
+ fastq-load
+ helicos-load
+ illumina-dump
+ illumina-load
+ kar
+ kdbmeta
+ latf-load
+ prefetch
+ rcexplain
+ sam-dump
+ sff-dump
+ sff-load
+ sra-pileup
+ sra-sort
+ sra-stat
+ srapath
+ srf-load
+ test-sra
+ vdb-config
+ vdb-copy
+ vdb-decrypt
+ vdb-dump
+ vdb-encrypt
+ vdb-lock
+ vdb-passwd
+ vdb-unlock
+ vdb-validate ) }
+1
diff --git a/setup/konfigure.perl b/setup/konfigure.perl
new file mode 100644
index 0000000..fe8d17c
--- /dev/null
+++ b/setup/konfigure.perl
@@ -0,0 +1,1717 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+use strict;
+
+sub println { print @_; print "\n" }
+
+my ($filename, $directories, $suffix) = fileparse($0);
+if ($directories ne "./") {
+ println "configure: error: $filename should be run as ./$filename";
+ exit 1;
+}
+
+require 'package.prl';
+require 'os-arch.prl';
+
+use Cwd qw(abs_path getcwd);
+use File::Basename 'fileparse';
+use File::Spec 'catdir';
+use FindBin qw($Bin);
+use Getopt::Long "GetOptions";
+
+chdir '..' or die "cannot cd to package root";
+
+check();
+
+my $LOCAL_BUILD_OUT
+ = -e File::Spec->catdir($ENV{HOME}, 'tmp', 'local-build-out');
+
+my ($CONFIGURED, $RECONFIGURE) = ('');
+if (@ARGV) {
+ foreach (@ARGV) {
+ $CONFIGURED .= "\t" if ($CONFIGURED);
+ $CONFIGURED .= "'$_'";
+ }
+} elsif (-f 'reconfigure') {
+ ++$RECONFIGURE unless ($LOCAL_BUILD_OUT);
+}
+
+my %PKG = PKG();
+
+my $PACKAGE_NAME = PACKAGE_NAME();
+my $OUT_MAKEFILE = 'Makefile.config';
+my $INS_MAKEFILE = 'Makefile.config.install';
+
+my $PACKAGE = PACKAGE();
+
+my $HOME = $ENV{HOME} || $ENV{USERPROFILE}
+ || $ENV{LOGDIR} || getcwd || (getpwuid($<))[7] || abs_path('.');
+
+$PKG{UPATH} = expand($PKG{UPATH});
+
+my $package_default_prefix = $PKG{PATH};
+my $schema_default_dir = $PKG{SCHEMA_PATH} if ($PKG{SCHEMA_PATH});
+
+my @REQ = REQ();
+
+my @options = ( 'build-prefix=s',
+ 'clean',
+ 'debug',
+ 'help',
+ 'prefix=s',
+ 'reconfigure',
+ 'status',
+ 'with-debug',
+ 'without-debug', );
+{
+ my ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
+ push @options, 'arch=s' if (@ARCHITECTURES);
+}
+push @options, 'source=s' if ($PKG{LNG} eq 'JAVA');
+push @options, 'enable-static' if (PACKAGE_TYPE() eq 'B');
+foreach my $href (@REQ) {
+ my %a = %$href;
+ push @options, "$a{option}=s" if ($a {option});
+ push @options, "$a{boption}=s" if ($a{boption});
+ $href->{usrpath} = '' unless ($href->{usrpath});
+ $href->{usrpath} = expand($href->{usrpath});
+}
+push @options, "shemadir" if ($PKG{SCHEMA_PATH});
+
+my %OPT;
+die "configure: error" unless (GetOptions(\%OPT, @options));
+++$OPT{'reconfigure'} if ($RECONFIGURE);
+
+if ($OPT{'reconfigure'}) {
+ unless (eval 'use Getopt::Long qw(GetOptionsFromString); 1') {
+ print <<EndText;
+configure: error: your perl does not support Getopt::Long::GetOptionsFromString
+ reconfigure option is not avaliable.
+Run "sh ./reconfigure" instead.
+EndText
+ exit 1;
+ }
+ println "reconfiguring...";
+ open F, 'reconfigure' or die 'cannot open reconfigure';
+ $_ = <F>;
+ chomp;
+ unless (m|^\./configure\s*(.*)$|) {
+ println 'configure: error: cannot reconfigure';
+ println 'run "./configure --clean" then run "./configure [OPTIONS]"';
+ exit 1;
+ }
+
+ my $ARG = $1;
+ println "running \"./configure $ARG\"...";
+ undef %OPT;
+ die "configure: error" unless (GetOptionsFromString($ARG, \%OPT, @options));
+ $CONFIGURED = $ARG if ($#ARGV == -1 && $RECONFIGURE);
+ ++$OPT{reconfigure};
+}
+
+$OPT{'local-build-out'} = $LOCAL_BUILD_OUT;
+my $OUTDIR = File::Spec->catdir($HOME, $PKG{OUT});
+if ($OPT{'local-build-out'}) {
+ my $o = expand_path(File::Spec->catdir($Bin, $PKG{LOCOUT}));
+ $OUTDIR = $o if ($o);
+}
+
+if ($OPT{'help'}) {
+ help();
+ exit 0;
+} elsif ($OPT{'clean'}) {
+ {
+ foreach ('reconfigure', glob(CONFIG_OUT() . '/Makefile.config*'),
+ File::Spec->catdir(CONFIG_OUT(), 'Makefile.userconfig'),
+ File::Spec->catdir(CONFIG_OUT(), 'user.status'))
+ {
+ my $f = $_;
+ print "removing $f... ";
+ if (-e $f) {
+ if (unlink $f) {
+ println "ok";
+ } else {
+ println "failed";
+ }
+ } else {
+ println "not found";
+ }
+ }
+ }
+ if (CONFIG_OUT() ne '.') {
+ foreach
+ (glob('Makefile.config*'), 'user.status', 'Makefile.userconfig')
+ {
+ my $f = $_;
+ print "removing $f... ";
+ if (-e $f) {
+ if (unlink $f) {
+ println "ok";
+ } else {
+ println "failed";
+ }
+ } else {
+ println "not found";
+ }
+ }
+ }
+ exit 0;
+} elsif ($OPT{'status'}) {
+ status(1);
+ exit 0;
+}
+
+foreach (@ARGV) {
+ @_ = split('=');
+ next if ($#_ != 1);
+ $OPT{$_[0]} = $_[1] if ($_[0] eq 'CXX' || $_[0] eq 'LDFLAGS');
+}
+
+println "Configuring $PACKAGE_NAME package";
+
+$OPT{'prefix'} = $package_default_prefix unless ($OPT{'prefix'});
+
+my $AUTORUN = $OPT{status};
+print "checking system type... " unless ($AUTORUN);
+my ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
+println $OSTYPE unless ($AUTORUN);
+
+unless ($OSTYPE =~ /linux/i || $OSTYPE =~ /darwin/i || $OSTYPE eq 'win') {
+ println "configure: error: unsupported system '$OSTYPE'";
+ exit 1;
+}
+
+print "checking machine architecture... " unless ($AUTORUN);
+println $MARCH unless ($AUTORUN);
+unless ($MARCH =~ /x86_64/i || $MARCH =~ /i?86/i) {
+ println "configure: error: unsupported architecture '$OSTYPE'";
+ exit 1;
+}
+
+{
+ $OPT{'prefix'} = expand_path($OPT{'prefix'});
+ my $prefix = $OPT{'prefix'};
+ $OPT{'eprefix'} = $prefix unless ($OPT{'eprefix'} || $OS eq 'win');
+ my $eprefix = $OPT{'eprefix'};
+ unless ($OPT{'bindir'} || $OS eq 'win') {
+ $OPT{'bindir'} = File::Spec->catdir($eprefix, 'bin') ;
+ }
+ unless ($OPT{'libdir'} || $OS eq 'win') {
+ $OPT{'libdir'} = File::Spec->catdir($eprefix, 'lib');
+ }
+ unless ($OPT{'includedir'} || $OS eq 'win') {
+ $OPT{'includedir'} = File::Spec->catdir($eprefix, 'include');
+ }
+ if ($PKG{LNG} eq 'PYTHON' && ! $OPT{'pythondir'} && $OS ne 'win') {
+ $OPT{'pythondir'} = $eprefix;
+ }
+ if ($PKG{LNG} eq 'JAVA' && ! $OPT{'javadir'} && $OS ne 'win') {
+ $OPT{'javadir'} = File::Spec->catdir($eprefix, 'jar');
+ }
+ if ($PKG{EXAMP} && ! $OPT{'sharedir'} && $OS ne 'win') {
+ $OPT{'sharedir'} = File::Spec->catdir($eprefix, 'share');
+ }
+}
+
+# initial values
+my $TARGDIR = File::Spec->catdir($OUTDIR, $PACKAGE);
+if ($OPT{'build-prefix'}) {
+ $TARGDIR = $OPT{'build-prefix'} = expand_path($OPT{'build-prefix'});
+ unless ($TARGDIR =~ /$PACKAGE$/) {
+ $TARGDIR = File::Spec->catdir($TARGDIR, $PACKAGE);
+ }
+}
+my $BUILD_PREFIX = $TARGDIR;
+
+my $BUILD = 'rel';
+
+# parse command line
+$BUILD = 'dbg' if ($OPT{'with-debug'});
+$BUILD = 'rel' if ($OPT{'without-debug'});
+
+my $BUILD_TYPE = "release";
+$BUILD_TYPE = "debug" if ( $BUILD eq "dbg" );
+
+$OPT{arch} = $ARCH if (@ARCHITECTURES && ! $OPT{arch});
+
+if ($OPT{arch}) {
+ my $found;
+ foreach (@ARCHITECTURES) {
+ if ($_ eq $OPT{arch}) {
+ ++$found;
+ last;
+ }
+ }
+ if ($found) {
+ $ARCH = $MARCH = $OPT{arch};
+ while (1) {
+ open F, ">Makefile.config.$OS.arch" or last;
+ print F "$ARCH\n";
+ close F;
+ last;
+ }
+ println "build architecture: $ARCH" unless ($AUTORUN);
+ } else {
+ delete $OPT{arch};
+ }
+}
+
+$OUT_MAKEFILE = File::Spec->catdir(CONFIG_OUT(), "$OUT_MAKEFILE.$OS.$ARCH");
+$INS_MAKEFILE = File::Spec->catdir(CONFIG_OUT(), "$INS_MAKEFILE.$OS.$ARCH.prl");
+
+my $TOOLS = "";
+$TOOLS = "jdk" if ($PKG{LNG} eq 'JAVA');
+
+print "checking $PACKAGE_NAME version... " unless ($AUTORUN);
+my $FULL_VERSION = VERSION();
+println $FULL_VERSION unless ($AUTORUN);
+
+# determine architecture
+
+print "checking for supported architecture... " unless ($AUTORUN);
+
+my $BITS;
+
+if ($MARCH =~ /x86_64/i) {
+ $BITS = 64;
+} elsif ($MARCH eq 'fat86') {
+ $BITS = '32_64';
+} elsif ($MARCH =~ /i?86/i) {
+ $BITS = 32;
+} else {
+ die "unrecognized Architecture '$ARCH'";
+}
+println "$MARCH ($BITS bits) is supported" unless ($AUTORUN);
+
+# determine OS and related norms
+my ($LPFX, $OBJX, $LOBX, $LIBX, $SHLX, $EXEX, $OSINC);
+
+print "checking for supported OS... " unless ($AUTORUN);
+if ($OSTYPE =~ /linux/i) {
+ $LPFX = 'lib';
+ $OBJX = 'o';
+ $LOBX = 'pic.o';
+ $LIBX = 'a';
+ $SHLX = 'so';
+ $EXEX = '';
+ $OSINC = 'unix';
+ $TOOLS = 'gcc' unless ($TOOLS);
+} elsif ($OSTYPE =~ /darwin/i) {
+ $LPFX = 'lib';
+ $OBJX = 'o';
+ $LOBX = 'pic.o';
+ $LIBX = 'a';
+ $SHLX = 'dylib';
+ $EXEX = '';
+ $OSINC = 'unix';
+ $TOOLS = 'clang' unless ($TOOLS);
+} elsif ($OSTYPE eq 'win') {
+ $TOOLS = 'vc++';
+} else {
+ die "unrecognized OS '$OSTYPE'";
+}
+
+println "$OSTYPE ($OS) is supported" unless ($AUTORUN);
+
+# tool chain
+my ($CPP, $CC, $CP, $AR, $ARX, $ARLS, $LD, $LP, $MAKE_MANIFEST);
+my ($JAVAC, $JAVAH, $JAR);
+my ($ARCH_FL, $DBG, $OPT, $PIC, $INC, $MD, $LDFLAGS) = ('');
+
+print "checking for supported tool chain... " unless ($AUTORUN);
+
+$CPP = $OPT{CXX } if ($OPT{CXX });
+$LDFLAGS = $OPT{LDFLAGS} if ($OPT{LDFLAGS});
+
+if ($TOOLS =~ /gcc$/) {
+ $CPP = 'g++' unless ($CPP);
+ $CC = "$TOOLS -c";
+ $CP = "$CPP -c";
+ $AR = 'ar rc';
+ $ARX = 'ar x';
+ $ARLS = 'ar t';
+ $LD = $TOOLS;
+ $LP = $CPP;
+
+ $DBG = '-g -DDEBUG';
+ $OPT = '-O3';
+ $PIC = '-fPIC';
+ $INC = '-I';
+ $MD = '-MD';
+} elsif ($TOOLS eq 'clang') {
+ $CPP = 'clang++' unless ($CPP);
+ $CC = 'clang -c';
+ my $versionMin = '-mmacosx-version-min=10.6';
+ $CP = "$CPP -c $versionMin";
+ if ($BITS ne '32_64') {
+ $ARCH_FL = '-arch i386' if ($BITS == 32);
+ $OPT = '-O3';
+ $AR = 'ar rc';
+ $LD = "clang $ARCH_FL";
+ $LP = "$CPP $versionMin $ARCH_FL";
+ } else {
+ $MAKE_MANIFEST = '( echo "$^" > $@/manifest )';
+ $ARCH_FL = '-arch i386 -arch x86_64';
+ $OPT = '-O3';
+ $AR = 'libtool -static -o';
+ $LD = "clang -Wl,-arch_multiple $ARCH_FL -Wl,-all_load";
+ $LP = "$CPP $versionMin -Wl,-arch_multiple $ARCH_FL -Wl,-all_load";
+ }
+ $ARX = 'ar x';
+ $ARLS = 'ar t';
+
+ $DBG = '-g -DDEBUG';
+ $PIC = '-fPIC';
+ $INC = '-I';
+ $MD = '-MD';
+} elsif ($TOOLS eq 'jdk') {
+ $JAVAC = 'javac';
+ if ($OPT{source}) {
+ $JAVAC .= ' -target ' . $OPT{source} . ' -source ' . $OPT{source};
+ }
+ $JAVAH = 'javah';
+ $JAR = 'jar cf';
+
+ $DBG = '-g';
+} elsif ($TOOLS eq 'vc++') {
+} else {
+ die "unrecognized tool chain '$TOOLS'";
+}
+println "$TOOLS tool chain is supported" unless ($AUTORUN);
+
+if ($OS ne 'win' && $PKG{LNG} ne 'JAVA') {
+ $TARGDIR = File::Spec->catdir($TARGDIR, $OS, $TOOLS, $ARCH, $BUILD);
+}
+
+if ($CPP) {
+ unless (check_tool__h($CPP)) {
+ println "configure: error: '$CPP' cannot be found";
+ exit 1;
+ }
+}
+
+if ($JAVAC) {
+ unless (check_tool_h($JAVAC)) {
+ println "configure: error: '$JAVAC' cannot be found";
+ exit 1;
+ }
+}
+
+my $NO_ARRAY_BOUNDS_WARNING = '';
+if ($TOOLS =~ /gcc$/ && check_no_array_bounds()) {
+ $NO_ARRAY_BOUNDS_WARNING = '-Wno-array-bounds';
+}
+
+my @dependencies;
+
+my %DEPEND_OPTIONS;
+foreach my $href (DEPENDS()) {
+ $_ = $href->{name};
+ my ($I, $L) = ($href->{Include});
+ my $o = "with-$_-prefix";
+ ++$DEPEND_OPTIONS{$o};
+ if ($OPT{$o}) {
+ $OPT{$o} = expand_path($OPT{$o});
+ $I = File::Spec->catdir($OPT{$o}, 'include');
+ if (/^xml2$/) {
+ my $t = File::Spec->catdir($I, 'libxml2');
+ $I = $t if (-e $t);
+ }
+ $L = File::Spec->catdir($OPT{$o}, 'lib');
+ }
+ my ($i, $l) = find_lib($_, $I, $L);
+ if (defined $i || $l) {
+ my $d = 'HAVE_' . uc($_) . ' = 1';
+ push @dependencies, $d;
+ println "\t\t$d" if ($OPT{'debug'});
+ }
+ if ($i) {
+ my $d = uc($_) . "_INCDIR = $i";
+ push @dependencies, $d;
+ println "\t\t$d" if ($OPT{'debug'});
+ }
+ if ($l) {
+ my $d = uc($_) . "_LIBDIR = $l";
+ push @dependencies, $d;
+ println "\t\t$d" if ($OPT{'debug'});
+ }
+}
+
+foreach my $href (@REQ) {
+ $href->{ bldpath} = expand($href->{ bldpath}) if ($href->{ bldpath});
+ $href->{locbldpath} = expand($href->{locbldpath}) if ($href->{locbldpath});
+
+ # found directories
+ my ($found_itf, $found_bin, $found_lib, $found_ilib, $found_jar);
+
+ my %a = %$href;
+ next if ($a{option} && $DEPEND_OPTIONS{$a{option}});
+ my $is_optional = optional($a{type});
+ my $quasi_optional = $a{type} =~ /Q/;
+ my $need_source = $a{type} =~ /S/;
+ my $need_bin = $a{type} =~ /E/;
+ my $need_build = $a{type} =~ /B/;
+ my $need_lib = $a{type} =~ /L|D/;
+ my $need_itf = ! ($a{type} =~ /D/ || $a{type} =~ /E/ || $a{type} =~ /J/);
+ my $need_jar = $a{type} =~ /J/;
+
+ my ($bin, $inc, $lib, $ilib)
+ = ($a{bin}, $a{include}, $a{lib}); # file names to check
+ $lib = '' unless ($lib);
+ $lib = expand($lib);
+
+ if ($need_build) {
+ $ilib = $a{ilib};
+ ++$need_lib;
+ }
+ unless ($AUTORUN) {
+ if ($need_source && $need_build) {
+ println
+ "checking for $a{name} package source files and build results..."
+ } elsif ($need_source) {
+ println "checking for $a{name} package source files...";
+ } else {
+ println "checking for $a{name} package...";
+ }
+ }
+ my %has_option;
+ my $tolib = $need_itf || $need_lib;
+ my $tojar = $need_jar;
+ foreach my $option ($a{option}, $a{boption}) {
+ next unless ($option);
+ if ($OPT{$option}) {
+ my $try = expand_path($OPT{$option});
+ if ($tojar && ! $found_jar && -f $try) {
+ println "\tjar... $try" unless ($AUTORUN);
+ $found_jar = $try;
+ }
+ elsif ($tolib) {
+ my ($i, $l, $il) = ($inc, $lib, $ilib);
+ if ($option =~ /-build$/) {
+ undef $i;
+ ++$has_option{build};
+ } elsif ($option =~ /-prefix$/) {
+ undef $il;
+ ++$has_option{prefix};
+ } elsif ($option =~ /-sources$/) {
+ undef $l;
+ undef $il;
+ ++$has_option{sources};
+ }
+ my ($fi, $fl, $fil) = find_in_dir($try, $i, $l, $il);
+ if ($fi || $fl || $fil) {
+ $found_itf = $fi if (! $found_itf && $fi);
+ $found_lib = $fl if (! $found_lib && $fl);
+ $found_ilib = $fil if (! $found_ilib && $fil);
+ } elsif (! ($try =~ /$a{name}$/)) {
+ $try = File::Spec->catdir($try, $a{name});
+ ($fi, $fl, $fil) = find_in_dir($try, $i, $l, $il);
+ $found_itf = $fi if (! $found_itf && $fi);
+ $found_lib = $fl if (! $found_lib && $fl);
+ $found_ilib = $fil if (! $found_ilib && $fil);
+ }
+ } elsif ($need_bin) {
+ my (undef, $fl, $fil)
+ = find_in_dir($try, undef, $lib, $ilib, undef, $bin);
+ $found_bin = $fl if ($fl);
+ }
+ }
+ }
+ if (! $found_itf && ! $has_option{sources} && $a{srcpath}) {
+ my $try = $a{srcpath};
+ ($found_itf) = find_in_dir($try, $inc);
+ }
+ if (! $has_option{prefix}) {
+ my $try = $a{pkgpath};
+ if (($need_itf && ! $found_itf) || ($need_lib && ! $found_lib)) {
+ my ($fi, $fl) = find_in_dir($try, $inc, $lib);
+ $found_itf = $fi if (! $found_itf && $fi);
+ $found_lib = $fl if (! $found_lib && $fl);
+ }
+
+ if ($need_jar && ! $found_jar) {
+ (undef, $found_jar) = find_in_dir($try, undef, undef, undef, $lib);
+ }
+
+ $try = $a{usrpath};
+ if (($need_itf && ! $found_itf) || ($need_lib && ! $found_lib)) {
+ my ($fi, $fl) = find_in_dir($try, $inc, $lib);
+ $found_itf = $fi if (! $found_itf && $fi);
+ $found_lib = $fl if (! $found_lib && $fl);
+ }
+
+ if ($need_jar && ! $found_jar) {
+ (undef, $found_jar) = find_in_dir($try, undef, undef, undef, $lib);
+ }
+ }
+ if (! $has_option{build}) {
+ if ($a{bldpath}) {
+ my $tolib = $need_build || ($need_lib && ! $found_lib);
+ my $tobin = $need_bin && ! $found_bin;
+ my $tojar = $need_jar && ! $found_jar;
+ if ($tolib || $tobin || $tojar) {
+ my ($fl, $fil, $found);
+ if ($OPT{'build-prefix'}) {
+ my $try = $OPT{'build-prefix'};
+ if ($tolib) {
+ (undef, $fl, $fil)
+ = find_in_dir($try, undef, $lib, $ilib);
+ if ($fl || $fil) {
+ $found_lib = $fl if (! $found_lib && $fl);
+ $found_ilib = $fil if (! $found_ilib && $fil);
+ ++$found;
+ }
+ }
+ if ($tojar) {
+ (undef, $found_jar)
+ = find_in_dir($try, undef, undef, undef, $lib);
+ }
+ if (! ($try =~ /$a{name}$/)) {
+ $try = File::Spec->catdir($try, $a{name});
+ if ($tolib && ! $found) {
+ (undef, $fl, $fil)
+ = find_in_dir($try, undef, $lib, $ilib);
+ if ($fl || $fil) {
+ $found_lib = $fl if (! $found_lib && $fl);
+ $found_ilib = $fil if (! $found_ilib && $fil);
+ ++$found;
+ }
+ }
+ if ($need_jar && ! $found_jar) {
+ (undef, $found_jar)
+ = find_in_dir($try, undef, undef, undef, $lib);
+ }
+ }
+ }
+ unless ($found || $fl || $fil) {
+ my $try = $a{bldpath};
+ $try = $a{locbldpath} if ($OPT{'local-build-out'});
+ if ($tolib && ! $found) {
+ (undef, $fl, $fil)
+ = find_in_dir($try, undef, $lib, $ilib);
+ my $resetLib = ! $found_lib;
+ if (! $found_ilib && $fil) {
+ $found_ilib = $fil;
+ ++$resetLib;
+ }
+ $found_lib = $fl if ($resetLib && $fl);
+ }
+ if ($tobin && ! $found) {
+ (undef, $fl, $fil) =
+ find_in_dir($try, undef, $lib, $ilib, undef, $bin);
+ $found_bin = $fl if ($fl);
+ }
+ if ($need_jar && ! $found_jar) {
+ (undef, $found_jar)
+ = find_in_dir($try, undef, undef, undef, $lib);
+ }
+ }
+ }
+ }
+ }
+ if (($need_itf && ! $found_itf) || ($need_lib && ! $found_lib) ||
+ ($need_jar && ! $found_jar) || ($ilib && ! $found_ilib) ||
+ ($need_bin && ! $found_bin))
+ {
+ if ($is_optional) {
+ println "configure: optional $a{name} package not found: skipped.";
+ } elsif ($quasi_optional && $found_itf && ($need_lib && ! $found_lib)) {
+ println "configure: $a{name} package: "
+ . "found interface files but not libraries.";
+ } else {
+ if ($OPT{'debug'}) {
+ $_ = "$a{name}: includes: ";
+ $found_itf = '' unless $found_itf;
+ $_ .= $found_itf;
+ unless ($need_lib) {
+ $_ .= "; libs: not needed";
+ } else {
+ $found_lib = '' unless $found_lib;
+ $_ .= "; libs: " . $found_lib;
+ }
+ unless ($ilib) {
+ $_ .= "; ilibs: not needed";
+ } else {
+ $found_ilib = '' unless $found_ilib;
+ $_ .= "; ilibs: " . $found_ilib;
+ }
+ println "\t\t$_";
+ }
+ println "configure: error: required $a{name} package not found.";
+ exit 1;
+ }
+ } else {
+ if ($found_itf) {
+ $found_itf = abs_path($found_itf);
+ push(@dependencies, "$a{aname}_INCDIR = $found_itf");
+ println "includes: $found_itf";
+ }
+ if ($found_lib) {
+ $found_lib = abs_path($found_lib);
+ if ($a{aname} eq 'NGS' || $a{aname} eq 'VDB') {
+ if ($OPT{PYTHON_LIB_PATH}) {
+ $OPT{PYTHON_LIB_PATH} .= ':';
+ } else {
+ $OPT{PYTHON_LIB_PATH} = '';
+ }
+ $OPT{PYTHON_LIB_PATH} .= $found_lib;
+ }
+ push(@dependencies, "$a{aname}_LIBDIR = $found_lib");
+ println "libraries: $found_lib";
+ }
+ if ($ilib && $found_ilib) {
+ $found_ilib = abs_path($found_ilib);
+ push(@dependencies, "$a{aname}_ILIBDIR = $found_ilib");
+ println "ilibraries: $found_ilib";
+ }
+ if ($found_bin) {
+ $found_bin = abs_path($found_bin);
+ push(@dependencies, "$a{aname}_BINDIR = $found_bin");
+ println "bin: $found_bin";
+ }
+ if ($found_jar) {
+ $found_jar = abs_path($found_jar);
+ push(@dependencies, "$a{aname}_JAR = $found_jar");
+ println "jar: $found_jar";
+ }
+ }
+}
+
+my ($E_BINDIR, $E_LIBDIR, $VERSION, $MAJVERS, $E_VERSION_LIBX, $E_MAJVERS_LIBX,
+ $E_VERSION_EXEX, $E_MAJVERS_EXEX)
+ = ('' , '');
+
+println unless ($AUTORUN);
+
+if ($OS ne 'win' && ! $OPT{'status'}) {
+ if ($OSTYPE =~ /darwin/i && CONFIG_OUT() ne '.') {
+ my $COMP = File::Spec->catdir(CONFIG_OUT(), 'COMP.mac');
+ println "configure: creating '$COMP' ($TOOLS)" unless ($AUTORUN);
+ open F, ">$COMP" or die "cannot open $COMP to write";
+ print F "$TOOLS\n";
+ close F;
+ }
+
+ # create Makefile.config
+ println "configure: creating '$OUT_MAKEFILE'" unless ($AUTORUN);
+ open my $F, ">$OUT_MAKEFILE" or die "cannot open $OUT_MAKEFILE to write";
+
+ print $F <<EndText;
+### AUTO-GENERATED FILE ###
+
+# configuration command
+
+CONFIGURED = $CONFIGURED
+
+OS_ARCH = \$(shell perl \$(TOP)/setup/os-arch.perl)
+
+# install paths
+EndText
+
+ L($F, "INST_BINDIR = $OPT{'bindir'}" ) if ($OPT{'bindir'});
+ L($F, "INST_LIBDIR = $OPT{'libdir'}" ) if ($OPT{'libdir'});
+ L($F, "INST_INCDIR = $OPT{'includedir'}" ) if ($OPT{'includedir'});
+ L($F, "INST_SCHEMADIR = $OPT{'shemadir'}" ) if ($OPT{'shemadir'});
+ L($F, "INST_SHAREDIR = $OPT{'sharedir'}" ) if ($OPT{'sharedir'});
+ L($F, "INST_JARDIR = $OPT{'javadir'}" ) if ($OPT{'javadir'});
+ L($F, "INST_PYTHONDIR = $OPT{'pythondir'}") if ($OPT{'pythondir'});
+
+ my ($E_VERSION_SHLX, $VERSION_SHLX,
+ $E_MAJVERS_SHLX , $MAJMIN_SHLX, $MAJVERS_SHLX);
+ if ($OSTYPE =~ /darwin/i) {
+ $E_VERSION_SHLX = '$VERSION.$SHLX';
+ $VERSION_SHLX = '$(VERSION).$(SHLX)';
+ $MAJMIN_SHLX = '$(MAJMIN).$(SHLX)';
+ $E_MAJVERS_SHLX = '$MAJVERS.$SHLX';
+ $MAJVERS_SHLX = '$(MAJVERS).$(SHLX)';
+ } else {
+ $E_VERSION_SHLX = '$SHLX.$VERSION';
+ $VERSION_SHLX = '$(SHLX).$(VERSION)';
+ $MAJMIN_SHLX = '$(SHLX).$(MAJMIN)';
+ $E_MAJVERS_SHLX = '$SHLX.$MAJVERS';
+ $MAJVERS_SHLX = '$(SHLX).$(MAJVERS)';
+ }
+
+ $E_VERSION_LIBX = '$LIBX.$VERSION';
+ $E_MAJVERS_LIBX = '$LIBX.$MAJVERS';
+
+ L($F);
+ L($F, "# build type");
+
+ if ($OPT{'enable-static'}) {
+ L($F, "WANTS_STATIC = 1");
+ }
+
+ $E_VERSION_EXEX = '$EXEX.$VERSION';
+ $E_MAJVERS_EXEX = '$LIBX.$MAJVERS';
+
+ print $F <<EndText;
+BUILD = $BUILD
+
+# target OS
+OS = $OS
+OSINC = $OSINC
+
+# prefix string for system libraries
+LPFX = $LPFX
+
+# suffix strings for system libraries
+LIBX = $LIBX
+VERSION_LIBX = \$(LIBX).\$(VERSION)
+MAJMIN_LIBX = \$(LIBX).\$(MAJMIN)
+MAJVERS_LIBX = \$(LIBX).\$(MAJVERS)
+
+SHLX = $SHLX
+VERSION_SHLX = $VERSION_SHLX
+MAJMIN_SHLX = $MAJMIN_SHLX
+MAJVERS_SHLX = $MAJVERS_SHLX
+
+# suffix strings for system object files
+OBJX = $OBJX
+LOBX = $LOBX
+
+# suffix string for system executable
+EXEX = $EXEX
+VERSION_EXEX = \$(EXEX).\$(VERSION)
+MAJMIN_EXEX = \$(EXEX).\$(MAJMIN)
+MAJVERS_EXEX = \$(EXEX).\$(MAJVERS)
+
+# system architecture and wordsize
+ARCH = $ARCH
+EndText
+
+ L($F, "# ARCH = $ARCH ( $MARCH )") if ($ARCH ne $MARCH);
+
+ print $F <<EndText;
+BITS = $BITS
+
+# tools
+EndText
+
+ L($F, "CC = $CC" ) if ($CC);
+ L($F, "CPP = $CPP" ) if ($CPP);
+ L($F, "CP = $CP" ) if ($CP);
+ L($F, "AR = $AR" ) if ($AR);
+ L($F, "ARX = $ARX" ) if ($ARX);
+ L($F, "ARLS = $ARLS" ) if ($ARLS);
+ L($F, "LD = $LD" ) if ($LD);
+ L($F, "LP = $LP" ) if ($LP);
+ L($F, "JAVAC = $JAVAC" ) if ($JAVAC);
+ L($F, "JAVAH = $JAVAH" ) if ($JAVAH);
+ L($F, "JAR = $JAR" ) if ($JAR);
+ L($F, "MAKE_MANIFEST = $MAKE_MANIFEST") if ($MAKE_MANIFEST);
+ L($F);
+
+ L($F, '# tool options');
+ if ($BUILD eq "dbg") {
+ L($F, "DBG = $DBG");
+ L($F, "OPT = ");
+ } else {
+ L($F, "DBG = -DNDEBUG") if ($PKG{LNG} eq 'C');
+ L($F, "OPT = $OPT" ) if ($OPT);
+ }
+ L($F, "PIC = $PIC") if ($PIC);
+ if ($PKG{LNG} eq 'C') {
+ if ($TOOLS =~ /clang/i) {
+ L($F, 'SONAME = -install_name ' .
+ '$(INST_LIBDIR)$(BITS)/$(subst $(VERSION),$(MAJVERS),$(@F)) \\');
+ L($F, ' -compatibility_version $(MAJMIN) -current_version $(VERSION) \\');
+ L($F, ' -flat_namespace -undefined suppress');
+ } else {
+ L($F, 'SONAME = -Wl,-soname=$(subst $(VERSION),$(MAJVERS),$(@F))');
+ }
+ L($F, "SRCINC = $INC. $INC\$(SRCDIR)");
+ } elsif ($PKG{LNG} eq 'JAVA') {
+ L($F, 'SRCINC = -sourcepath $(INCPATHS)');
+ }
+ if ($PIC) {
+ if (PACKAGE_NAMW() eq 'NGS') {
+ L($F, "INCDIRS = \$(SRCINC) $INC\$(TOP) "
+ . "$INC\$(TOP)/ngs/\$(OSINC)/\$(ARCH)")
+ } elsif (PACKAGE_NAMW() eq 'NGS_BAM') {
+ L($F, "INCDIRS = \$(SRCINC) $INC\$(TOP) "
+ . "$INC\$(NGS_INCDIR)/ngs/\$(OSINC)/\$(ARCH)")
+ } else {
+ L($F, "INCDIRS = \$(SRCINC) $INC\$(TOP)")
+ }
+ }
+ if ($PKG{LNG} eq 'C') {
+ L($F, "CFLAGS = \$(DBG) \$(OPT) \$(INCDIRS) $MD $ARCH_FL");
+ }
+ L($F, "LDFLAGS = $LDFLAGS") if ($LDFLAGS);
+
+ L($F, 'CLSPATH = -classpath $(CLSDIR)');
+ L($F, "NO_ARRAY_BOUNDS_WARNING = $NO_ARRAY_BOUNDS_WARNING");
+ L($F);
+
+ # version information
+
+ my $MAJMIN;
+
+ if ($FULL_VERSION =~ /(\d+)\.(\d+)\.(\d+)-?\w*\d*/) {
+ $VERSION = "$1.$2.$3";
+ $MAJMIN = "$1.$2";
+ $MAJVERS = $1;
+ } else {
+ die $VERSION;
+ }
+
+ print $F <<EndText;
+# $PACKAGE_NAME and library version
+VERSION = $VERSION
+MAJMIN = $MAJMIN
+MAJVERS = $MAJVERS
+
+# output path
+BUILD_PREFIX = $BUILD_PREFIX
+TARGDIR = $TARGDIR
+
+# derived paths
+MODPATH ?= \$(subst \$(TOP)/,,\$(CURDIR))
+SRCDIR ?= \$(TOP)/\$(MODPATH)
+MAKEFILE ?= \$(abspath \$(firstword \$(MAKEFILE_LIST)))
+BINDIR = \$(TARGDIR)/bin
+EndText
+
+ if ($PKG{LNG} eq 'C') {
+ $E_BINDIR = '$TARGDIR/bin';
+ $E_LIBDIR = '$TARGDIR/lib';
+ L($F, 'LIBDIR = $(TARGDIR)/lib');
+ } elsif ($PKG{LNG} eq 'JAVA') {
+ $E_LIBDIR = '$TARGDIR/jar';
+ L($F, 'LIBDIR = $(TARGDIR)/jar');
+ }
+
+ L($F, 'ILIBDIR = $(TARGDIR)/ilib');
+ if ($PKG{NOMODPATH}) {
+ L($F, 'OBJDIR = $(TARGDIR)/obj');
+ } else {
+ L($F, 'OBJDIR = $(TARGDIR)/obj/$(MODPATH)');
+ }
+ L($F, 'CLSDIR = $(TARGDIR)/cls');
+
+ if ($PKG{LNG} eq 'JAVA') {
+ L($F,
+ "INCPATHS = \$(SRCDIR):\$(SRCDIR)/itf:\$(TOP)/gov/nih/nlm/ncbi/ngs")
+ }
+
+ print $F <<EndText;
+
+# exports
+export TOP
+export MODPATH
+export SRCDIR
+export MAKEFILE
+
+# auto-compilation rules
+EndText
+
+ if ($PKG{LNG} eq 'C') {
+ L($F, '$(OBJDIR)/%.$(OBJX): %.c');
+ T($F, '$(CC) -o $@ $< $(CFLAGS)');
+ L($F, '$(OBJDIR)/%.$(LOBX): %.c');
+ T($F, '$(CC) -o $@ $< $(PIC) $(CFLAGS)');
+ }
+ L($F, '$(OBJDIR)/%.$(OBJX): %.cpp');
+ T($F, '$(CP) -o $@ $< $(CFLAGS)');
+ L($F, '$(OBJDIR)/%.$(LOBX): %.cpp');
+ T($F, '$(CP) -o $@ $< $(PIC) $(CFLAGS)');
+ L($F);
+
+ # this is part of Makefile
+ L($F, 'VPATH = $(SRCDIR)');
+ L($F);
+
+ # we know how to find jni headers
+ if ($PKG{LNG} eq 'JAVA' and $OPT{'with-ngs-sdk-src'}) {
+ L($F, "JNIPATH = $OPT{'with-ngs-sdk-src'}/language/java");
+ }
+
+ L($F, '# directory rules');
+ if ($PKG{LNG} eq 'C') {
+ L($F, '$(BINDIR) $(LIBDIR) $(ILIBDIR) '
+ . '$(OBJDIR) $(INST_LIBDIR) $(INST_LIBDIR)$(BITS):');
+ T($F, 'mkdir -p $@');
+ } elsif ($PKG{LNG} eq 'JAVA') {
+ # test if we have jni header path
+ L($F, '$(LIBDIR) $(CLSDIR) $(INST_JARDIR):');
+ T($F, 'mkdir -p $@');
+ }
+ L($F);
+
+ L($F, '# not real targets');
+ L($F, '.PHONY: default clean install all std $(TARGETS)');
+ L($F);
+
+ L($F, '# dependencies');
+ if ($PKG{LNG} eq 'C') {
+ L($F, 'include $(wildcard $(OBJDIR)/*.d)');
+ } elsif ($PKG{LNG} eq 'JAVA') {
+ L($F, 'include $(wildcard $(CLSDIR)/*.d)');
+ }
+ L($F, $_) foreach (@dependencies);
+ L($F);
+
+ if ($OS eq 'linux' || $OS eq 'mac') {
+ L($F, '# installation rules');
+ L($F,
+ '$(INST_LIBDIR)$(BITS)/%.$(VERSION_LIBX): $(LIBDIR)/%.$(VERSION_LIBX)');
+ T($F, '@ echo -n "installing \'$(@F)\'... "');
+ T($F, '@ if cp $^ $@ && chmod 644 $@; \\');
+ T($F, ' then \\');
+ T($F, ' rm -f $(patsubst %$(VERSION),%$(MAJVERS),$@) '
+ . '$(patsubst %$(VERSION_LIBX),%$(LIBX),$@) '
+ . '$(patsubst %.$(VERSION_LIBX),%-static.$(LIBX),$@); \\');
+ T($F, ' ln -s $(@F) $(patsubst %$(VERSION),%$(MAJVERS),$@); \\');
+ T($F, ' ln -s $(patsubst %$(VERSION),%$(MAJVERS),$(@F)) '
+ . '$(patsubst %$(VERSION_LIBX),%$(LIBX),$@); \\');
+ T($F, ' ln -s $(patsubst %$(VERSION_LIBX),%$(LIBX),$(@F)) ' .
+ '$(INST_LIBDIR)$(BITS)/$(patsubst %.$(VERSION_LIBX),%-static.$(LIBX),$(@F));'
+ . ' \\');
+ T($F, ' echo success; \\');
+ T($F, ' else \\');
+ T($F, ' echo failure; \\');
+ T($F, ' false; \\');
+ T($F, ' fi');
+ L($F);
+
+ L($F,
+ '$(INST_LIBDIR)$(BITS)/%.$(VERSION_SHLX): $(LIBDIR)/%.$(VERSION_SHLX)');
+ T($F, '@ echo -n "installing \'$(@F)\'... "');
+ T($F, '@ if cp $^ $@ && chmod 755 $@; \\');
+ T($F, ' then \\');
+ if ($OS ne 'mac') {
+ T($F, ' rm -f $(patsubst %$(VERSION),%$(MAJVERS),$@) '
+ . '$(patsubst %$(VERSION_SHLX),%$(SHLX),$@); \\');
+ }
+ if ($OS eq 'linux') {
+ T($F, ' ln -s $(@F) $(patsubst %$(VERSION),%$(MAJVERS),$@); \\');
+ } elsif ($OS eq 'mac') {
+ T($F, ' ln -sf $(@F) '
+ . '$(patsubst %$(VERSION_SHLX),%$(MAJVERS).$(SHLX),$@); \\');
+ } else {
+ die;
+ }
+ T($F, ' ln -sf $(patsubst %$(VERSION),%$(MAJVERS),$(@F)) '
+ . '$(patsubst %$(VERSION_SHLX),%$(SHLX),$@); \\');
+ T($F, ' echo success; \\');
+ T($F, ' else \\');
+ T($F, ' echo failure; \\');
+ T($F, ' false; \\');
+ T($F, ' fi');
+ L($F);
+
+ L($F, '$(INST_BINDIR)/%$(VERSION_EXEX): $(BINDIR)/%$(VERSION_EXEX)');
+ T($F, '@ echo -n "installing \'$(@F)\'... "');
+ T($F, '@ if cp $^ $@ && chmod 755 $@; \\');
+ T($F, ' then \\');
+ T($F, ' rm -f $(patsubst %$(VERSION),%$(MAJVERS),$@) '
+ . '$(patsubst %$(VERSION_EXEX),%$(EXEX),$@); \\');
+ T($F, ' ln -s $(@F) $(patsubst %$(VERSION),%$(MAJVERS),$@); \\');
+ T($F, ' ln -s $(patsubst %$(VERSION),%$(MAJVERS),$(@F)) '
+ . '$(patsubst %$(VERSION_EXEX),%$(EXEX),$@); \\');
+ T($F, ' echo success; \\');
+ T($F, ' else \\');
+ T($F, ' echo failure; \\');
+ T($F, ' false; \\');
+ T($F, ' fi');
+ }
+ close $F;
+
+ # create Makefile.config.install
+ println "configure: creating '$INS_MAKEFILE'" unless ($AUTORUN);
+ open $F, ">$INS_MAKEFILE" or die "cannot open $INS_MAKEFILE to write";
+
+ $OPT{'javadir' } = '' unless ($OPT{'javadir' });
+ $OPT{'sharedir'} = '' unless ($OPT{'sharedir'});
+
+ print $F "sub CONFIGURE {\n";
+ print $F " \$_{PACKAGE_NAME } = '$PACKAGE_NAME';\n";
+ print $F " \$_{VERSION } = '$VERSION';\n";
+ print $F " \$_{LNG } = '$PKG{LNG}';\n";
+ print $F " \$_{OS } = '$OS';\n";
+ print $F " \$_{BITS } = $BITS;\n";
+ print $F " \$_{MAJVERS } = $MAJVERS;\n";
+ print $F " \$_{LPFX } = '$LPFX';\n";
+ print $F " \$_{LIBX } = '$LIBX';\n";
+ print $F " \$_{MAJVERS_LIBX } = '" . expand($E_MAJVERS_LIBX) . "';\n";
+ print $F " \$_{VERSION_LIBX } = '" . expand($E_VERSION_LIBX) . "';\n";
+ print $F " \$_{SHLX } = '$SHLX';\n";
+ print $F " \$_{MAJVERS_SHLX } = '" . expand($E_MAJVERS_SHLX) . "';\n";
+ print $F " \$_{VERSION_SHLX } = '" . expand($E_VERSION_SHLX) . "';\n";
+ print $F " \$_{VERSION_EXEX } = '" . expand($E_VERSION_EXEX) . "';\n";
+ print $F " \$_{MAJVERS_EXEX } = '" . expand($E_MAJVERS_EXEX) . "';\n";
+ print $F " \$_{INCDIR } = '" . expand("$Bin/.." ) . "';\n";
+ if ($PKG{LNG} ne 'PYTHON') {
+ print $F " \$_{BINDIR$BITS} = '" . expand($E_BINDIR ) . "';\n";
+ print $F " \$_{LIBDIR$BITS} = '" . expand($E_LIBDIR ) . "';\n";
+ } elsif ($OPT{PYTHON_LIB_PATH}) {
+ print $F " \$_{LIBDIR$BITS} = '$OPT{PYTHON_LIB_PATH}';\n";
+ }
+ print $F " \$_{OTHER_PREFIX } = '$PKG{UPATH}';\n";
+ print $F " \$_{PREFIX } = '$OPT{'prefix'}';\n";
+ print $F " \$_{INST_INCDIR } = '$OPT{'includedir'}';\n";
+ print $F " \$_{INST_BINDIR } = '$OPT{'bindir'}';\n";
+ print $F " \$_{INST_LIBDIR } = '$OPT{'libdir'}';\n";
+ print $F " \$_{INST_JARDIR } = '$OPT{'javadir'}';\n";
+ print $F " \$_{INST_SHAREDIR} = '$OPT{'sharedir'}';\n";
+ print $F "\n";
+ print $F " \@_\n";
+ print $F "}\n";
+ print $F "\n";
+ print $F "1\n";
+
+ close $F;
+}
+
+if (! $OPT{'status'} ) {
+ if ($OS eq 'win') {
+ my $OUT = File::Spec->catdir(CONFIG_OUT(), 'Makefile.config.win');
+ println "configure: creating '$OUT'";
+ open OUT, ">$OUT" or die "cannot open $OUT to write";
+ my $name = PACKAGE_NAMW();
+ my $outdir = $name . '_OUTDIR';
+ my $root = $name . '_ROOT';
+
+ print OUT <<EndText;
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup Label="Globals">
+ <$outdir>$TARGDIR/\</$outdir>
+EndText
+ foreach my $href (@REQ) {
+ my %a = %$href;
+ my $NGS_SDK_PREFIX = '';
+ $NGS_SDK_PREFIX = $a{found_itf} if ($a{found_itf});
+ if ($a{name} eq 'ngs-sdk') {
+ my $root = "$a{aname}_ROOT";
+ print OUT " <$root>$NGS_SDK_PREFIX\/</$root>\n";
+ last;
+ }
+ }
+ print OUT <<EndText;
+ <$root>$Bin/\</$root>
+ </PropertyGroup>
+</Project>
+EndText
+ close OUT;
+ } else {
+ println "configure: creating 'Makefile.config'" unless ($AUTORUN);
+ my $CONFIG_OUT = CONFIG_OUT();
+ my $out = File::Spec->catdir($CONFIG_OUT, 'Makefile.config');
+ open COUT, ">$out" or die "cannot open $out to write";
+ print COUT "### AUTO-GENERATED FILE ###\n";
+ print COUT "\n";
+ print COUT "OS_ARCH = \$(shell perl \$(TOP)/setup/os-arch.perl)\n";
+ print COUT "include \$(TOP)/$CONFIG_OUT/Makefile.config.\$(OS_ARCH)\n";
+ close COUT;
+ }
+}
+
+unless ($OPT{'reconfigure'}) {
+ println "configure: creating 'reconfigure'" unless ($AUTORUN);
+ $CONFIGURED =~ s/\t/ /g;
+ open my $F, '>reconfigure' or die 'cannot open reconfigure to write';
+ print $F "./configure $CONFIGURED\n";
+ close $F;
+ # my $perm = (stat $fh)[2] & 07777;
+# print "==================================================== $perm\n";
+}
+
+status() if ($OS ne 'win');
+
+unlink 'a.out';
+
+sub L { $_[1] = '' unless ($_[1]); print { $_[0] } "$_[1]\n" }
+sub T { print { $_[0] } "\t$_[1]\n" }
+
+sub status {
+ my ($load) = @_;
+ if ($load) {
+ ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
+ my $MAKEFILE
+ = File::Spec->catdir(CONFIG_OUT(), "$OUT_MAKEFILE.$OS.$ARCH");
+ println "\t\tloading $MAKEFILE" if ($OPT{'debug'});
+ unless (-e $MAKEFILE) {
+ print STDERR "configure: error: run ./configure [OPTIONS] first.\n";
+ exit 1;
+ }
+ open F, $MAKEFILE or die "cannot open $MAKEFILE";
+ foreach (<F>) {
+ chomp;
+ if (/BUILD = (.+)/) {
+ $BUILD_TYPE = $1;
+ } elsif (/BUILD \?= /) {
+ $BUILD_TYPE = $_ unless ($BUILD_TYPE);
+ } elsif (/BUILD_PREFIX = /) {
+ $BUILD_PREFIX = $_;
+ } elsif (/^CC += (.+)/) {
+ $CC = $1;
+ } elsif (/CONFIGURED = (.*)/) {
+ $CONFIGURED = $1;
+ } elsif (/CPP += (.+)/) {
+ $CPP = $1;
+ } elsif (/LDFLAGS += (.+)/) {
+ $LDFLAGS = $1;
+ } elsif (/TARGDIR = /) {
+ $TARGDIR = $_;
+ println "\t\tgot $_" if ($OPT{'debug'});
+ } elsif (/TARGDIR \?= (.+)/) {
+ $TARGDIR = $1 unless ($TARGDIR);
+ println "\t\tgot $_" if ($OPT{'debug'});
+ }
+ elsif (/INST_INCDIR = (.+)/) {
+ $OPT{'includedir'} = $1;
+ }
+ elsif (/INST_BINDIR = (.+)/) {
+ $OPT{'bindir'} = $1;
+ }
+ elsif (/INST_LIBDIR = (.+)/) {
+ $OPT{'libdir'} = $1;
+ }
+ }
+ }
+
+ println "build type: $BUILD_TYPE";
+ println "build prefix: $BUILD_PREFIX" if ($OS ne 'win');
+ println "build output path: $TARGDIR" if ($OS ne 'win');
+
+# print "prefix: "; print $OPT{'prefix'} if ($OS ne 'win'); println;
+# print "eprefix: "; print $OPT{'eprefix'} if ($OPT{'eprefix'}); println;
+
+ print "includedir: ";
+ print $OPT{'includedir'} if ($OPT{'includedir'});
+ println;
+
+ print "bindir: ";
+ print $OPT{'bindir'} if ($OPT{'bindir'});
+ println;
+
+ print "libdir: ";
+ print $OPT{'libdir'} if ($OPT{'libdir'});
+ println;
+
+ println "schemadir: $OPT{'shemadir'}" if ($OPT{'shemadir'});
+ println "sharedir: $OPT{'sharedir'}" if ($OPT{'sharedir'});
+ println "javadir: $OPT{'javadir'}" if ($OPT{'javadir'});
+ println "pythondir: $OPT{'pythondir'}" if ($OPT{'pythondir'});
+
+ println "CC = $CC" if ($CC );
+ println "CPP = $CPP" if ($CPP);
+ println "LDFLAGS = $LDFLAGS" if ($LDFLAGS);
+
+ $CONFIGURED =~ s/\t/ /g;
+ println "configured with: \"$CONFIGURED\"";
+}
+
+sub expand { $_[0] =~ s/(\$\w+)/$1/eeg; $_[0]; }
+
+sub expand_path {
+ my ($filename) = @_;
+ return unless ($filename);
+
+ if ($filename =~ /^~/) {
+ if ($filename =~ m|^~([^/]*)|) {
+ if ($1 && ! getpwnam($1)) {
+ print "configure: error: bad path: '$filename'\n";
+ exit 1;
+ }
+ }
+
+ $filename =~ s{ ^ ~ ( [^/]* ) }
+ { $1
+ ? (getpwnam($1))[7]
+ : ( $ENV{HOME} || $ENV{USERPROFILE} || $ENV{LOGDIR}
+ || (getpwuid($<))[7]
+ )
+ }ex;
+ }
+
+ my $a = abs_path($filename);
+ $filename = $a if ($a);
+ $filename;
+}
+
+sub find_in_dir {
+ my ($dir, $include, $lib, $ilib, $jar, $bin) = @_;
+ unless (-d $dir) {
+# println "no" unless ($AUTORUN);
+ println "\t\tnot found $dir" if ($OPT{'debug'});
+ return;
+ }
+# print "\t$dir... " unless ($AUTORUN);
+# print "[found] " if ($OPT{'debug'});
+ my ($found_inc, $found_lib, $found_ilib);
+ if ($include) {
+ print "\tincludes... " unless ($AUTORUN);
+ if (-e "$dir/$include") {
+ println $dir unless ($AUTORUN);
+ $found_inc = $dir;
+ } elsif (-e "$dir/include/$include") {
+ println $dir unless ($AUTORUN);
+ $found_inc = "$dir/include";
+ } elsif (-e "$dir/interfaces/$include") {
+ println $dir unless ($AUTORUN);
+ $found_inc = "$dir/interfaces";
+ } else {
+ print "$dir: " if ($OPT{'debug'});
+ println 'no' unless ($AUTORUN);
+ }
+ }
+ if ($lib || $ilib) {
+# print "\n\t" if ($nl && !$AUTORUN);
+ print "\tlibraries... " unless ($AUTORUN);
+ if ($lib) {
+ my $builddir = File::Spec->catdir($dir, $OS, $TOOLS, $ARCH, $BUILD);
+ my $libdir = File::Spec->catdir($builddir, 'lib');
+ my $ilibdir = File::Spec->catdir($builddir, 'ilib');
+ my $f = File::Spec->catdir($libdir, $lib);
+ print "\n\t\tchecking $f\n\t" if ($OPT{'debug'});
+ my $found;
+ if (-e $f) {
+ $found_lib = $libdir;
+ if ($ilib) {
+ my $f = File::Spec->catdir($ilibdir, $ilib);
+ print "\tchecking $f\n\t" if ($OPT{'debug'});
+ if (-e $f) {
+ println $ilibdir;
+ $found_ilib = $ilibdir;
+ } else {
+ println 'no' unless ($AUTORUN);
+ return;
+ }
+ } else {
+ println $libdir;
+ }
+ ++$found;
+ }
+ if (! $found) {
+ my $libdir = File::Spec->catdir($dir, 'lib' . $BITS);
+ my $f = File::Spec->catdir($libdir, $lib);
+ print "\tchecking $f\n\t" if ($OPT{'debug'});
+ if (-e $f) {
+ println $libdir;
+ $found_lib = $libdir;
+ ++$found;
+ }
+ }
+ if (! $found) {
+ my $builddir = File::Spec->catdir
+ ($dir, $OS, $TOOLS, $ARCH, reverse_build($BUILD));
+ my $libdir = File::Spec->catdir($builddir, 'lib');
+ my $ilibdir = File::Spec->catdir($builddir, 'ilib');
+ my $f = File::Spec->catdir($libdir, $lib);
+ print "\tchecking $f\n\t" if ($OPT{'debug'});
+ if (-e $f) {
+ $found_lib = $libdir;
+ if ($ilib) {
+ my $f = File::Spec->catdir($ilibdir, $ilib);
+ print "\tchecking $f\n\t" if ($OPT{'debug'});
+ if (-e $f) {
+ println $ilibdir;
+ $found_ilib = $ilibdir;
+ } else {
+ println 'no' unless ($AUTORUN);
+ return;
+ }
+ } else {
+ println $libdir;
+ }
+ ++$found;
+ } else {
+ println 'no' unless ($AUTORUN);
+ }
+ }
+ }
+ if ($found_lib && $ilib && ! $found_ilib) {
+ println "\n\t\tfound $found_lib but no ilib/" if ($OPT{'debug'});
+ print "\t" if ($OPT{'debug'});
+ println 'no' unless ($AUTORUN);
+ undef $found_lib;
+ }
+ }
+ if ($bin) {
+ print "\t... " unless ($AUTORUN);
+ my $builddir = File::Spec->catdir($dir, $OS, $TOOLS, $ARCH, $BUILD);
+ my $bdir = File::Spec->catdir($builddir, 'bin');
+ my $f = File::Spec->catdir($bdir, $bin);
+ print "\n\t\tchecking $f\n\t" if ($OPT{'debug'});
+ if (-e $f) {
+ $found_lib = $bdir;
+ println $bdir;
+ } else {
+ println 'no' unless ($AUTORUN);
+ }
+ }
+ if ($jar) {
+ print "\tjar... " unless ($AUTORUN);
+ my $try = "$dir/jar/$jar";
+ if (-e "$try") {
+ println $try unless ($AUTORUN);
+ $found_lib = $try;
+ }
+ }
+ return ($found_inc, $found_lib, $found_ilib);
+}
+
+sub reverse_build {
+ ($_) = @_;
+ if ($_ eq 'rel') {
+ return 'dbg';
+ } elsif ($_ eq 'dbg') {
+ return 'rel';
+ } else {
+ die $_;
+ }
+}
+
+################################################################################
+
+sub check_tool_h { return check_tool(@_, '-help'); }
+sub check_tool__h { return check_tool(@_, '--help'); }
+
+sub check_tool {
+ my ($tool, $o) = @_;
+ print "checking for $tool... ";
+ my $cmd = "$tool $o";
+ print "\n\t\trunning $cmd\n\t" if ($OPT{'debug'});
+ my $out = `$cmd 2>&1`;
+ if ($? == 0) {
+ println "yes";
+ return 1;
+ } else {
+ println "no";
+ return 0;
+ }
+}
+
+sub check_no_array_bounds {
+ check_compiler('O', '-Wno-array-bounds');
+}
+
+sub find_lib {
+ check_compiler('L', @_);
+}
+
+sub check_compiler {
+ my ($t, $n, $i, $l) = @_;
+ my $tool = $TOOLS;
+
+ if ($t eq 'L') {
+ print "checking for $n library... ";
+ } elsif ($t eq 'O') {
+ if ($tool && $tool =~ /gcc$/) {
+ print "checking whether $tool accepts $n... ";
+ } else {
+ return;
+ }
+ } else {
+ die "Unknown check_compiler option: '$t'";
+ }
+
+ unless ($tool) {
+ println "warning: unknown tool";
+ return;
+ }
+
+ while (1) {
+ my ($flags, $library, $log) = ('', '');
+
+ if ($t eq 'O') {
+ $flags = $n;
+ $log = ' int main() { }\n'
+ } elsif ($n eq 'hdf5') {
+ $library = '-lhdf5';
+ $log = '#include <hdf5.h> \n int main() { H5close (); }\n'
+ } elsif ($n eq 'fuse') {
+ $flags = '-D_FILE_OFFSET_BITS=64';
+ $library = '-lfuse';
+ $log = '#include <fuse.h> \n int main() { fuse_get_context(); }\n'
+ } elsif ($n eq 'magic') {
+ $library = '-lmagic';
+ $log = '#include <magic.h> \n int main() { magic_open (0); }\n'
+ } elsif ($n eq 'xml2') {
+ $library = '-lxml2';
+ $log = '#include <libxml/xmlreader.h>\n' .
+ 'int main() { xmlInitParser ( ); }\n'
+ } else {
+ println 'unknown: skipped';
+ return;
+ }
+
+ if ($i && ! -d $i) {
+ print "'$i': " if ($OPT{'debug'});
+ println 'no';
+ return;
+ }
+ if ($l && ! -d $l) {
+ print "'$l': " if ($OPT{'debug'}); println 'no';
+ return;
+ }
+
+ my $cmd = $log;
+ $cmd =~ s/\\n/\n/g;
+
+ my $gcc = "| $tool -xc $flags " . ($i ? "-I$i " : ' ')
+ . ($l ? "-L$l " : ' ') . "- $library";
+ $gcc .= ' 2> /dev/null' unless ($OPT{'debug'});
+
+ open GCC, $gcc or last;
+ print "\n\t\trunning echo -e '$log' $gcc\n" if ($OPT{'debug'});
+ print GCC "$cmd" or last;
+ my $ok = close GCC;
+ print "\t" if ($OPT{'debug'});
+ println $ok ? 'yes' : 'no';
+
+ unlink 'a.out';
+
+ return if (!$ok);
+
+ return 1 if ($t eq 'O');
+
+ return ($i, $l);
+ }
+
+ println "cannot run $tool: skipped";
+}
+
+################################################################################
+
+sub check {
+ die "No CONFIG_OUT" unless CONFIG_OUT();
+ die "No PACKAGE" unless PACKAGE();
+ die "No PACKAGE_NAME" unless PACKAGE_NAME();
+ die "No PACKAGE_NAMW" unless PACKAGE_NAMW();
+ die "No PACKAGE_TYPE" unless PACKAGE_TYPE();
+ die "No VERSION" unless VERSION();
+
+ my %PKG = PKG();
+
+ die "No LNG" unless $PKG{LNG};
+ die "No LOCOUT" unless $PKG{LOCOUT};
+ die "No OUT" unless $PKG{OUT};
+ die "No PATH" unless $PKG{PATH};
+ die "No UPATH" unless $PKG{UPATH};
+
+ foreach my $href (DEPENDS()) { die "No DEPENDS::name" unless $href->{name} }
+
+ foreach my $href (REQ()) {
+ die "No REQ::name" unless $href->{name};
+
+ die "No $href->{name}:option" unless $href->{option}
+ || $href->{boption};
+
+ die "No $href->{name}:type" unless $href->{type};
+ unless ($href->{type} =~ /I/) {
+ unless ($href->{type} =~ /E/) {
+ die "No $href->{name}:lib" unless $href->{lib};
+ }
+ die "No $href->{name}:pkgpath" unless $href->{pkgpath};
+ die "No $href->{name}:usrpath" unless $href->{usrpath};
+ }
+
+ die "No $href->{name}:origin" unless $href->{origin};
+ if ($href->{origin} eq 'I') {
+ die "No $href->{name}:aname" unless $href->{aname};
+ unless ($href->{type} =~ /D/ || $href->{type} =~ /E/
+ || $href->{type} =~ /J/)
+ {
+ die "No $href->{name}:include" unless $href->{include};
+ die "No $href->{name}:srcpath" unless $href->{srcpath};
+ }
+ unless ($href->{type} =~ /I/) {
+ die "No $href->{name}:bldpath" unless $href->{bldpath };
+ die "No $href->{name}:locbldpath" unless $href->{locbldpath};
+ }
+ if ($href->{type} =~ /B/) {
+ die "No $href->{name}:ilib" unless $href->{ilib};
+ }
+ }
+ }
+}
+
+################################################################################
+
+sub optional { $_[0] =~ /O/ }
+
+sub help {
+# --prefix=PREFIX install architecture-independent files in PREFIX
+ print <<EndText;
+`configure' configures $PACKAGE_NAME to adapt to many kinds of systems.
+
+Usage: ./configure [OPTION]...
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+ -h, --help display this help and exit
+
+EndText
+
+ if ($^O ne 'MSWin32') {
+ print <<EndText;
+Installation directories:
+ --prefix=PREFIX install all files in PREFIX
+ [$package_default_prefix]
+
+EndText
+
+ my $other_prefix = $PKG{UPATH};
+ if ($PACKAGE eq 'sra-tools' && 0) {
+ print <<EndText;
+ --shemadir=DIR install schema files in DIR
+ [$schema_default_dir]
+
+EndText
+ }
+
+ print "By default, \`make install' will install all the files in\n";
+
+ if (PACKAGE_TYPE() eq 'B') {
+ print "\`$package_default_prefix/bin', ";
+ } elsif (PACKAGE_TYPE() eq 'L') {
+ print "\`$package_default_prefix/include', ";
+ }
+ if (PACKAGE_TYPE() eq 'P') {
+ println "\`$package_default_prefix/share' etc.";
+ } else {
+ println "\`$package_default_prefix/lib' etc.";
+ }
+
+ print <<EndText;
+You can specify an installation prefix other than \`$package_default_prefix'
+using \`--prefix', for instance \`--prefix=$other_prefix'.
+EndText
+ }
+
+ print <<EndText;
+
+For better control, use the options below.
+
+EndText
+
+ my ($required, $optional);
+ foreach my $href (@REQ) {
+ if (optional($href->{type})) {
+ ++$optional;
+ } else {
+ ++$required;
+ }
+ }
+
+ if ($required) {
+ print "Required Packages:\n";
+ foreach my $href (@REQ) {
+ next if (optional($href->{type}));
+ my %a = %$href;
+ if ($a{type} =~ /S/) {
+ println " --$a{option}=DIR search for $a{name} package";
+ println " source files in DIR";
+ } else {
+ unless ($a{type} =~ /E/) {
+ println
+ " --$a{option}=DIR search for $a{name} package in DIR"
+ }
+ }
+ if ($a{boption}) {
+ println " --$a{boption}=DIR search for $a{name} package";
+ println " build output in DIR";
+ }
+ println;
+ }
+ }
+
+ if ($optional) {
+ print "Optional Packages:\n";
+ foreach my $href (@REQ) {
+ next unless (optional($href->{type}));
+ my %a = %$href;
+ if ($a{option} =~ /-sources$/) {
+ println " --$a{option}=DIR search for $a{name} package";
+ println " source files in DIR";
+ } else {
+ println " --$a{option}=DIR search for $a{name} files in DIR"
+ }
+ }
+ println;
+ }
+
+ print <<EndText if (PACKAGE_TYPE() eq 'B');
+Optional Features:
+ --enable-static build static executable [default=no]
+
+EndText
+
+ my ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
+
+ if ($^O ne 'MSWin32') {
+ print "Build tuning:\n";
+ if ($PKG{LNG} ne 'JAVA') {
+ print <<EndText;
+ --with-debug
+ --without-debug
+EndText
+ }
+
+ if (@ARCHITECTURES) {
+ print
+" --arch=name specify the name of the target architecture\n";
+ }
+
+ if ($PKG{LNG} eq 'JAVA') {
+ print <<EndText;
+ --source=release provide source compatibility with specified release,
+ generate class files for specified VM version.
+ e.g. `--source=1.6'
+EndText
+ } else {
+ print "\n";
+ }
+
+ print <<EndText;
+ --build-prefix=DIR generate build output into DIR directory
+ [$OUTDIR]
+
+EndText
+ }
+
+ println 'Miscellaneous:';
+ println ' --reconfigure rerun `configure\'';
+ println ' using the same command-line arguments';
+ if ($^O ne 'MSWin32') {
+ println
+ ' --status print current configuration information'
+ }
+ print <<EndText;
+ --clean remove all configuration results
+ --debug print lots of debugging information
+
+If `configure' was already run running `configure' without options
+will rerun `configure' using the same command-line arguments.
+
+Report bugs to sra-tools\@ncbi.nlm.nih.gov
+EndText
+}
+
+################################################################################
diff --git a/setup/os-arch.perl b/setup/os-arch.perl
new file mode 100644
index 0000000..5f3f87a
--- /dev/null
+++ b/setup/os-arch.perl
@@ -0,0 +1,22 @@
+use strict;
+
+use FindBin qw($Bin);
+require "$Bin/os-arch.prl";
+
+my ($OS, $ARCH, $OSTYPE, $MARCH, @ARCHITECTURES) = OsArch();
+
+my $res = "$OS.$ARCH";
+
+if (@ARCHITECTURES) {
+ my $name = "$Bin/../Makefile.config.$OS.arch";
+ if (-e $name) {
+ while (1) {
+ open F, $name or last;
+ $res = "$OS." . <F>;
+ chomp $res;
+ last;
+ }
+ }
+}
+
+print "$res\n";
diff --git a/os-arch.pm b/setup/os-arch.prl
similarity index 89%
rename from os-arch.pm
rename to setup/os-arch.prl
index 6fbd606..5daa75e 100644
--- a/os-arch.pm
+++ b/setup/os-arch.prl
@@ -1,5 +1,7 @@
sub OsArch {
- my ($UNAME, $HOST_OS, $HOST_ARCH, $MARCH, @ARCHITECTURES);
+ my ($UNAME, $HOST_OS, $HOST_ARCH,
+ $MARCH, # machine hardware name returned by uname -m
+ @ARCHITECTURES);
if ($^O eq 'MSWin32') {
$UNAME = $HOST_OS = 'win';
$HOST_ARCH = $MARCH = 'x86_64';
@@ -18,10 +20,12 @@ sub OsArch {
$HOST_OS = 'win';
} elsif ($UNAME =~ /xMINGW/) {
$HOST_OS = 'win';
+ } else {
+ $HOST_OS = $UNAME;
}
if ($HOST_OS eq 'mac') {
$MARCH = $HOST_ARCH = MacArch();
- @ARCHITECTURES = qw(x86_64 i386) if ($MARCH eq 'x86_64');
+ @ARCHITECTURES = qw(x86_64 i386 fat86) if ($MARCH eq 'x86_64');
} else {
$MARCH = `uname -m`;
chomp $MARCH;
@@ -50,7 +54,9 @@ sub OsArch {
@ARCHITECTURES = qw(x86_64 i386);
} elsif ($MARCH =~ /sun4v/) {
$HOST_ARCH = 'sparc64';
- @ARCHITECTURES = qw(sparc64 sparc32);
+ @ARCHITECTURES = qw(sparc64 sparc32);
+ } else {
+ $HOST_ARCH = $MARCH;
}
}
}
diff --git a/package.pm b/setup/package.prl
similarity index 53%
rename from package.pm
rename to setup/package.prl
index 209868f..3324750 100644
--- a/package.pm
+++ b/setup/package.prl
@@ -1,20 +1,22 @@
################################################################################
-sub PACKAGE { "sra-tools" }
-sub VERSION { "2.4.2-1" }
+sub PACKAGE { 'sra-tools' }
+sub VERSION { '2.5.7' }
sub PACKAGE_TYPE { 'B' }
-sub PACKAGE_NAME { "SRA-TOOLS" }
-sub PACKAGE_NAMW { "SRATOOLS" }
+sub PACKAGE_NAME { 'SRA-TOOLS' }
+sub PACKAGE_NAMW { 'SRATOOLS' }
sub CONFIG_OUT { 'build' }
sub PKG { ( LNG => 'C',
OUT => 'ncbi-outdir',
+ LOCOUT=> '../../OUTDIR',
PATH => '/usr/local/ncbi/sra-tools',
UPATH => '$HOME/ncbi/sra-tools', ) }
-sub DEPENDS { ( { name => 'hdf5' , },
- { name => 'magic', Include => '/usr/include' , },
- { name => 'xml2' , Include => '/usr/include/libxml2', } ) }
+sub DEPENDS { ( { name => 'fuse' , Include => '/usr/include' , },
+ { name => 'hdf5' , Include => '/usr/include' , },
+ { name => 'magic', Include => '/usr/include' , },
+ { name => 'xml2' , Include => '/usr/include/libxml2', } ) }
sub REQ { (
{ name => 'ngs-sdk',
- namew => 'NGS',
+ aname => 'NGS',
option => 'with-ngs-sdk-prefix',
origin => 'I',
type => 'L',
@@ -22,12 +24,12 @@ sub REQ { (
pkgpath => '/usr/local/ngs/ngs-sdk',
usrpath => '$HOME/ngs/ngs-sdk',
bldpath => '$HOME/ncbi-outdir/ngs-sdk',
+ locbldpath=>'$Bin/../../OUTDIR/ngs-sdk',
include => 'ngs/itf/Refcount.h',
lib => 'libngs-c++.a',
- ilib => 'libngs-bind-c++.a',
},
{ name => 'ncbi-vdb',
- namew => 'VDB',
+ aname => 'VDB',
option => 'with-ncbi-vdb-sources',
boption => 'with-ncbi-vdb-build',
origin => 'I',
@@ -36,17 +38,29 @@ sub REQ { (
pkgpath => '/usr/local/ncbi/ncbi-vdb',
usrpath => '$HOME/ncbi/ncbi-vdb',
bldpath => '$HOME/ncbi-outdir/ncbi-vdb',
+ locbldpath=>'$Bin/../../OUTDIR/ncbi-vdb',
include => 'klib/rc.h',
lib => 'libncbi-vdb.a',
ilib => 'libkapp.a',
- },
- { name => 'hdf5',
+ },
+ { name => 'fuse',
+ option => 'with-fuse-prefix',
+ origin => 'E',
+ type => 'LIO',
+ },
+ { name => 'hdf5',
option => 'with-hdf5-prefix',
origin => 'E',
- type => 'LI',
- pkgpath => '/usr',
- usrpath => '$HOME',
- include => 'hdf5.h',
- lib => 'libhdf5.a',
+ type => 'LIO',
+ },
+ { name => 'magic',
+ option => 'with-magic-prefix',
+ origin => 'E',
+ type => 'LIO',
+ },
+ { name => 'xml2',
+ option => 'with-xml2-prefix',
+ origin => 'E',
+ type => 'LIO',
} ) }
1
diff --git a/configure b/setup/s/install
similarity index 82%
copy from configure
copy to setup/s/install
index 3313aac..b1e201d 100755
--- a/configure
+++ b/setup/s/install
@@ -29,17 +29,17 @@ CURDIR="`dirname $0`"
if [ -z "$CURDIR" ]
then
- echo configure: error: configure should be run as ./configure
+ echo error: install should be run as ./install
exit 1
fi
-if [ "$FILENAME" != configure -a ! -s konfigure.perl ]
+if [ "$FILENAME" != install -a ! -s setup/install.perl ]
then
- echo configure: error: configure should be run as ./configure
+ echo error: install should be run as ./install
else
perl -v > /dev/null 2>&1 || { echo "checking for perl... no"; \
- echo >&2 "configure: error: perl not found."; exit 1; }
+ echo >&2 "error: perl not found."; exit 1; }
- cd $CURDIR
- perl -w ./konfigure.perl $@
+ cd $CURDIR/setup
+ perl -w ./install.perl "$@"
fi
diff --git a/setup/s/os.prl b/setup/s/os.prl
new file mode 100644
index 0000000..5f75471
--- /dev/null
+++ b/setup/s/os.prl
@@ -0,0 +1,2 @@
+sub OS { 'linux' }
+1
diff --git a/test/Makefile b/test/Makefile
index 08bddf1..93d1a59 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -22,9 +22,35 @@
#
# ===========================================================================
-#TBD
-default all std clean runtests slowtests:
- @true
-
-.PHONY: default all std clean runtests slowtests
-
\ No newline at end of file
+by_default: runtests
+
+TOP ?= $(abspath ..)
+MODULE = test
+
+include $(TOP)/build/Makefile.shell
+
+include $(TOP)/build/Makefile.config
+
+#-------------------------------------------------------------------------------
+# default
+#
+SUBDIRS = \
+ fastq-loader \
+ vcf-loader \
+ kget \
+ general-loader \
+ vschema \
+ align-info \
+ vdb-dump \
+
+# under construction
+# ngs-pileup \
+
+# common targets for non-leaf Makefiles; must follow a definition of SUBDIRS
+include $(TOP)/build/Makefile.targets
+
+$(SUBDIRS):
+ @ $(MAKE) -C $@
+
+.PHONY: default $(SUBDIRS)
+
diff --git a/test/Makefile b/test/align-info/Makefile
similarity index 75%
copy from test/Makefile
copy to test/align-info/Makefile
index 08bddf1..3266e05 100644
--- a/test/Makefile
+++ b/test/align-info/Makefile
@@ -22,9 +22,27 @@
#
# ===========================================================================
-#TBD
-default all std clean runtests slowtests:
- @true
-
-.PHONY: default all std clean runtests slowtests
-
\ No newline at end of file
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/align-info
+
+TEST_TOOLS = \
+
+include $(TOP)/build/Makefile.env
+
+$(TEST_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+.PHONY: $(TEST_TOOLS)
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# runtests
+runtests: cmdline
+
+cmdline:
+ # make sure -h does not crash
+ $(BINDIR)/align-info -h
\ No newline at end of file
diff --git a/test/fastq-loader/.gitignore b/test/fastq-loader/.gitignore
new file mode 100644
index 0000000..508061b
--- /dev/null
+++ b/test/fastq-loader/.gitignore
@@ -0,0 +1 @@
+actual
diff --git a/test/fastq-loader/Makefile b/test/fastq-loader/Makefile
new file mode 100644
index 0000000..31020d1
--- /dev/null
+++ b/test/fastq-loader/Makefile
@@ -0,0 +1,207 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+#TODO: transfer slowtests/big_loads (see CVS)
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/fastq-loader
+
+TEST_TOOLS = \
+ wb-test-fastq \
+ test-fastq-loader
+
+include $(TOP)/build/Makefile.env
+
+$(TEST_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+.PHONY: $(TEST_TOOLS)
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# white-box test
+#
+FASTQ_TEST_SRC = \
+ wb-test-fastq
+
+FASTQ_TEST_OBJ = \
+ $(addsuffix .$(OBJX),$(FASTQ_TEST_SRC))
+
+FASTQ_TEST_LIB = \
+ -skapp \
+ -sktst \
+ -sfastqloader \
+ -sloader \
+ -sload \
+ -sncbi-wvdb \
+
+$(TEST_BINDIR)/wb-test-fastq: $(FASTQ_TEST_OBJ)
+ $(LP) --exe -o $@ $^ $(FASTQ_TEST_LIB)
+
+wb: wb-test-fastq
+ $(TEST_BINDIR)/wb-test-fastq 2>&1
+
+valgrind_wb: wb-test-fastq
+ valgrind --ncbi $(TEST_BINDIR)/wb-test-fastq
+
+#-------------------------------------------------------------------------------
+# test-fastqtest-loader
+#
+FASTQTEST_LOADER_SRC = \
+ test-fastq-loader
+
+FASTQTEST_LOADER_OBJ = \
+ $(addsuffix .$(OBJX),$(FASTQTEST_LOADER_SRC))
+
+$(TEST_BINDIR)/test-fastq-loader: $(FASTQTEST_LOADER_OBJ)
+ $(LP) --exe -o $@ $^ $(FASTQ_TEST_LIB)
+
+tfl:
+ $(MAKE) -C $(OBJDIR) -f $(SRCDIR)/Makefile test-fastq-loader
+ $(TEST_BINDIR)/test-fastq-loader
+
+valgrind_tfl:
+ $(MAKE) -C $(OBJDIR) -f $(SRCDIR)/Makefile test-fastq-loader
+ valgrind --ncbi $(TEST_BINDIR)/test-fastq-loader
+
+.PHONY: test-fastq-loader
+
+#-------------------------------------------------------------------------------
+# scripted tests
+#
+runtests: set_schema smalltests
+
+set_schema: $(BINDIR)/vdb-config
+ $(BINDIR)/vdb-config -s vdb/schema/paths="$(VDB_INCDIR)"
+
+SMALLRUN = @ $(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)
+smalltests: $(TEST_TOOLS)
+ -rm -rf $(SRCDIR)/actual
+ $(BINDIR)/vdb-config -on
+# Small tests
+# Input:
+# Quality encoding: phred/logodds, ascii/decimal
+# 1.1 phred 33
+ $(SMALLRUN) 1.1 0 $(SRCDIR)/input/1.1.fastq --quality PHRED_33
+# 1.2 phred 64
+ $(SMALLRUN) 1.2 0 $(SRCDIR)/input/1.2.fastq --quality PHRED_64
+# 1.3 phred 33, decimal qualities
+ $(SMALLRUN) 1.3 3 $(SRCDIR)/input/1.3.fastq --quality PHRED_33
+# 1.4 logodds 64
+ $(SMALLRUN) 1.4 0 $(SRCDIR)/input/1.4.fastq --quality LOGODDS
+#
+# Single input file
+# 2.1 reads only, basespace
+ $(SMALLRUN) 2.1 0 $(SRCDIR)/input/2.1.fastq --quality PHRED_33
+# 2.1.1 reads only, colorspace
+ $(SMALLRUN) 2.1.1 0 $(SRCDIR)/input/2.1.1.fastq --quality PHRED_33
+# 2.2 qualities only
+ -$(SMALLRUN) 2.2 0 $(SRCDIR)/input/2.2.fastq --quality PHRED_33
+# 2.3 reads+qualities, qualities before reads
+ -$(SMALLRUN) 2.3 0 $(SRCDIR)/input/2.3.fastq --quality PHRED_33
+# 2.4 reads+qualities, reads before qualities
+ -$(SMALLRUN) 2.4 0 $(SRCDIR)/input/2.4.fastq --quality PHRED_33
+# 2.5 reads+qualities, paired up
+ $(SMALLRUN) 2.5 0 $(SRCDIR)/input/2.5.fastq --quality PHRED_33
+# 2.6 spot assembly
+ $(SMALLRUN) 2.6 0 $(SRCDIR)/input/2.6.fastq --quality PHRED_33
+# 2.7 spot assembly, read numbers not specified
+ $(SMALLRUN) 2.7 0 $(SRCDIR)/input/2.7.fastq --quality PHRED_33
+# 2.8 handling of N in reads, ALTREAD is a static column
+# TODO; investigate intermittent failure (FIXED_SPOT_LEN == 0, not 3)
+ -$(SMALLRUN) 2.8 0 $(SRCDIR)/input/2.8.fastq --quality PHRED_33
+# 2.8.1 handling of N in reads, ALTREAD is a physical column
+ $(SMALLRUN) 2.8.1 0 $(SRCDIR)/input/2.8.1.fastq --quality PHRED_33
+# 2.9 mismatching length of sequence and quality (qualities shorter)
+ $(SMALLRUN) 2.9 3 $(SRCDIR)/input/2.9.fastq --quality PHRED_33
+# 2.9.1 mismatching length of sequence and quality (qualities longer)
+ $(SMALLRUN) 2.9.1 3 $(SRCDIR)/input/2.9.1.fastq --quality PHRED_33
+#
+# Multiple files
+# 3.1 reads+qualities together
+ $(SMALLRUN) 3.1 0 $(SRCDIR)/input/3.1a.fastq $(SRCDIR)/input/3.1b.fastq --quality PHRED_33
+# 3.2 qualities separately from reads, reads file comes first
+ -$(SMALLRUN) 3.2 0 $(SRCDIR)/input/3.2a.fastq $(SRCDIR)/input/3.2b.fastq --quality PHRED_33
+# 3.3 qualities separately from reads, qualities file comes first (reject?)
+ -$(SMALLRUN) 3.3 0 $(SRCDIR)/input/3.3a.fastq $(SRCDIR)/input/3.3b.fastq --quality PHRED_33
+# 3.4 default read number
+# $(SMALLRUN) 3.4 0 -r 1 $(SRCDIR)/input/3.4a.fastq -r 2 $(SRCDIR)/input/3.4b.fastq --quality PHRED_33
+#
+# Command line options:
+# 4.1 --tmpfs temporary files created at the specified location
+ @echo "running 4.1"; export LD_LIBRARY_PATH=$(LIBDIR);$(BINDIR)/latf-load $(SRCDIR)/input/4.fastq --tmpfs $(SRCDIR)/actual/ -v -o actual/4.1.obj --quality PHRED_33 | grep "Path for scratch files: $(SRCDIR)/actual/" >/dev/null
+# 4.2 --qual-quant quality values are quantized as specified
+ $(SMALLRUN) 4.2 0 $(SRCDIR)/input/4.fastq --qual-quant 1:10,10:20,20:30,30:- --quality PHRED_33
+# 4.3 --cache-size handled correctly
+ @echo "running 4.3"; export LD_LIBRARY_PATH=$(LIBDIR);$(BINDIR)/latf-load $(SRCDIR)/input/4.fastq --cache-size 10 -v -o actual/4.1.obj --quality PHRED_33 | grep "Cache size: 10M" >/dev/null
+# 4.3.1 --cache-size rejected if invalid
+ $(SMALLRUN) 4.3.1 3 $(SRCDIR)/input/4.fastq --cache-size BIG --quality PHRED_33
+# 4.4 --max-rec-count stops at the specified record #
+ $(SMALLRUN) 4.4 0 $(SRCDIR)/input/4.4.fastq --max-rec-count 3 --quality PHRED_33
+# 4.5 --max-err-count stops after the specified # of errors
+ $(SMALLRUN) 4.5 3 $(SRCDIR)/input/4.5.fastq --max-err-count 2 --quality PHRED_33
+# 4.5.1 --max-err-pct fails if % of error records is too high
+ $(SMALLRUN) 4.5.1 3 $(SRCDIR)/input/4.5.1.fastq --quality PHRED_33 --max-err-count 10 --max-err-pct 20
+# 4.6 --platform saves correct value in the PLATFORM column
+ $(SMALLRUN) 4.6 0 $(SRCDIR)/input/4.fastq -p LS454 --quality PHRED_33
+# 4.6.1 --platform rejects invalid values
+ $(SMALLRUN) 4.6.1 3 $(SRCDIR)/input/4.fastq -p SOMETHINGELSE --quality PHRED_33
+# 4.7 input file missing
+ $(SMALLRUN) 4.7 3 $(SRCDIR)/input/not_there --quality PHRED_33
+# Gzipped input
+ $(SMALLRUN) 5.0 0 $(SRCDIR)/input/5.0.fastq.gz --quality PHRED_33
+# Misparsed quality
+ $(SMALLRUN) 6.0 0 $(SRCDIR)/input/6.0.fastq --quality PHRED_33
+# PACBIO fastq
+ $(SMALLRUN) 7.1 0 $(SRCDIR)/input/7.1.fastq --platform=PACBIO --quality PHRED_33
+ $(SMALLRUN) 7.2 0 $(SRCDIR)/input/7.2.fastq --platform=PACBIO --quality PHRED_33
+ $(SMALLRUN) 7.3 0 $(SRCDIR)/input/7.3.fastq --platform=PACBIO --quality PHRED_33
+# secondary read numbers other than 3
+ $(SMALLRUN) 8.0 0 $(SRCDIR)/input/8.0.fastq --quality PHRED_33
+# reject inconsistent secondary read numbers
+ $(SMALLRUN) 8.1 3 $(SRCDIR)/input/8.1.fastq --quality PHRED_33
+# newest Illumina tag line variation ("_" instead of " /")
+ $(SMALLRUN) 9.0 0 $(SRCDIR)/input/9.0.fastq --quality PHRED_33
+# accepting ascii quality values up to 126
+ $(SMALLRUN) 10.0 0 $(SRCDIR)/input/10.0.fastq --quality PHRED_33
+ # 2-file fasta, tag line Run.Spot.Read
+ $(SMALLRUN) 11.0 0 $(SRCDIR)/input/11.0.1.fasta $(SRCDIR)/input/11.0.2.fasta --quality PHRED_33
+ # 2-file fastq, tag line Run.Spot.Read
+ $(SMALLRUN) 11.1 0 $(SRCDIR)/input/11.1.1.fastq $(SRCDIR)/input/11.1.2.fastq --quality PHRED_33
+ # Illumina, ignore barcodes
+ $(SMALLRUN) 12.0 0 $(SRCDIR)/input/12.0.fastq --quality PHRED_33 --ignore-illumina-tags
+ # Illumina, use barcodes as spotgroups
+ $(SMALLRUN) 12.1 0 $(SRCDIR)/input/12.1.fastq --quality PHRED_33
+ # Illumina, use barcodes as spotgroups, alternative format
+ $(SMALLRUN) 12.2 0 $(SRCDIR)/input/12.2.fastq --quality PHRED_33
+ rm -rf $(SRCDIR)/actual
+
+onetest:
+ rm -rf $(SRCDIR)/actual
+ $(SMALLRUN) 1.4 0 $(SRCDIR)/input/1.4.fastq --quality LOGODDS
+
diff --git a/test/fastq-loader/expected/1.1.stdout b/test/fastq-loader/expected/1.1.stdout
new file mode 100644
index 0000000..1e9793c
--- /dev/null
+++ b/test/fastq-loader/expected/1.1.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 36
+ BIO_BASE_COUNT: 36
+ CMP_BASE_COUNT: 36
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 112332022230202..1120022110..03..0..
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 5, 13, 27, 22, 22, 19, 22, 23, 23, 23, 24, 24, 24, 23, 22, 22, 21, 22, 22, 21, 25, 24, 24, 22, 24, 22, 19, 18, 20, 19, 22, 22, 19, 22, 22, 24
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 0
+ READ_SEG: [0, 36], [36, 0]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 36
+ TMP_KEY_ID: 0
+ TRIM_LEN: 36
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/1.2.stdout b/test/fastq-loader/expected/1.2.stdout
new file mode 100644
index 0000000..831a7c4
--- /dev/null
+++ b/test/fastq-loader/expected/1.2.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 36
+ BIO_BASE_COUNT: 36
+ CMP_BASE_COUNT: 36
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 112332022230202..1120022110..03..0..
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 16, 25, 25, 22, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 0
+ READ_SEG: [0, 36], [36, 0]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 36
+ TMP_KEY_ID: 0
+ TRIM_LEN: 36
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/1.3.stderr b/test/fastq-loader/expected/1.3.stderr
new file mode 100644
index 0000000..a40494f
--- /dev/null
+++ b/test/fastq-loader/expected/1.3.stderr
@@ -0,0 +1,4 @@
+latf-load err: file="1.3.fastq"
+latf-load err:Invalid quality value (' '=32, position 2): for Phred33, valid range is from 33 to 126.
+latf-load err: format unsupported while parsing file - accession="obj" errors="1" status="failure"
+latf-load err: format unsupported while parsing file - load failed
diff --git a/test/fastq-loader/expected/1.3.stdout b/test/fastq-loader/expected/1.3.stdout
new file mode 100644
index 0000000..896a550
--- /dev/null
+++ b/test/fastq-loader/expected/1.3.stdout
@@ -0,0 +1,27 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 36
+ BIO_BASE_COUNT: 36
+ CMP_BASE_COUNT: 36
+ CMP_READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: TT
+ CS_NATIVE: false
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 8, 9, 10, 11, 12, 13, 14, 15, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 7, 8, 9, 10, 11, 12, 13
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 0
+ READ_SEG: [0, 36], [36, 0]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD, SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TMP_KEY_ID: 0
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/1.4.stdout b/test/fastq-loader/expected/1.4.stdout
new file mode 100755
index 0000000..572f76e
--- /dev/null
+++ b/test/fastq-loader/expected/1.4.stdout
@@ -0,0 +1,90 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 216
+ BIO_BASE_COUNT: 216
+ CMP_BASE_COUNT: 216
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 131033233331013023032230132132312301331032212003012030012223002013323212
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 72
+ MAX_SPOT_ID: 3
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 34, 36, 40, 40, 40, 40, 40, 40, 5, 40, 8, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 33, 40, 40, 40, 40, 40, 40, 40, 13, 40, 40, 40, 12, 40, 40, 40, 40, 40, 1, 40, 40, 3, 40
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GCAATAGCGCGTTGCCTAATCTAACGACGATGATTGATGGCTCAGGGCCAGGCCCAGAGCCCTTGCGATCAG
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 36
+ READ_SEG: [0, 36], [36, 36]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 3
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 72
+ TRIM_LEN: 72
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 216
+ BIO_BASE_COUNT: 216
+ CMP_BASE_COUNT: 216
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 110003323031230312213200233330230230030231233302133312123321320321233203
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 72
+ MAX_SPOT_ID: 3
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 27, 40, 40, 40, 40, 40, 14, 24, 40, 40, 9, 2, 4, 40, 32, 12, 40, 13, 3, 3, 2, 40, 4, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 34, 40, 11, 40, 40, 40, 40, 40, 40, 40, 5
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTTTTATCGGCAGCCGTCTGCTTTCGCGCCTAAGCCTAAGCAGCGCCTGCGCAGTCGCTGCTTAGTCGCTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 36
+ READ_SEG: [0, 36], [36, 36]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 3
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 72
+ TRIM_LEN: 72
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 216
+ BIO_BASE_COUNT: 216
+ CMP_BASE_COUNT: 216
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 122312131322312131021100002130102223101233212132311133020101010332200312
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 72
+ MAX_SPOT_ID: 3
+ MIN_SPOT_ID: 1
+ NAME: 3
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 24, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 15, 21, 40, 40, 40, 40, 40, 40, 40, 10, 40, 27, 40
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GAGCAGTACGAGCAGTACCTGTTTTTCATTGGAGATGGTCGCTGACGATGTGCGGAACCAACCGCTCCCGTC
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 36
+ READ_SEG: [0, 36], [36, 36]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 3
+ SPOT_GROUP:
+ SPOT_ID: 3
+ SPOT_LEN: 72
+ TRIM_LEN: 72
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/10.0.stdout b/test/fastq-loader/expected/10.0.stdout
new file mode 100644
index 0000000..b0da961
--- /dev/null
+++ b/test/fastq-loader/expected/10.0.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 21
+ BIO_BASE_COUNT: 21
+ CMP_BASE_COUNT: 21
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 202311120113300010103
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 21
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 65, 65, 93, 68, 68, 68, 68, 68, 70, 70, 70, 66, 70, 72, 72, 71, 71, 66, 68, 70, 70
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CCTACACTTGTATTTTGGTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 21, 0
+ READ_SEG: [0, 21], [21, 0]
+ READ_START: 0, 21
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 21
+ TMP_KEY_ID: 0
+ TRIM_LEN: 21
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/11.0.stdout b/test/fastq-loader/expected/11.0.stdout
new file mode 100644
index 0000000..346615e
--- /dev/null
+++ b/test/fastq-loader/expected/11.0.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 152
+ BIO_BASE_COUNT: 152
+ CMP_BASE_COUNT: 152
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 12231033031122330010100000000000000000000000000203301033111000111011021301111301123113320231133000013123101102202202000000000000001300011000033112330033
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 152
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GAGCAATAATGTCTATTTGGTTTTTTTTTTTTTTTTTTTTTTTTTTTCCGCCAATACACCCCACAACAAGTAACACGCCACTACATAGGATGTATTTTTGCAGCAACAAGAAGAAGGGGGGGGGGGGGGGTAAAACAAAAATACAGCGGGCG
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 76, 76
+ READ_SEG: [0, 76], [76, 76]
+ READ_START: 0, 76
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 152
+ TMP_KEY_ID: 0
+ TRIM_LEN: 152
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/11.1.stdout b/test/fastq-loader/expected/11.1.stdout
new file mode 100644
index 0000000..01676c5
--- /dev/null
+++ b/test/fastq-loader/expected/11.1.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 202
+ BIO_BASE_COUNT: 202
+ CMP_BASE_COUNT: 202
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 0300232002130023102002201303012112133111010333330332133032020333303000312301230003311300303300333003003002320021300231020022013030121121331110103333303321330320203333030003123012300033113003033003330030
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 202
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 34, 34, 34, 37, 37, 37, 37, 37, 39, 39, 38, 39, 38, 41, 41, 41, 41, 41, 41, 41, 40, 40, 41, 41, 38, 41, 41, 40, 41, 38, 40, 39, 39, 40, 41, 37, 33, 39, 38, 40, 40, 41, 38, 37, 36, 31, 33, 39, 40, 40, 36, 34, 36, 39, 40, 40, 38, 40, 38, 40, 39, 39, 40, 40, 41, 40, 40, 40, 41, 38, 36, 36, 34, 39, 39, 37, 31, 36, 38, 39, 38, 35, 39, 38, 36, 39, 37, 36, 36, 39, 39, 37, 37, 37, 35, 37, 37, 37, 36, 36, 31, 34, 34, 34, 37, 37, 37, 37, 37, 39, 39, 38, 39, 38, 41, 41, 41, 41 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: TAAAGCTTTCATTTCGTTCCCTCCATTAACTGTCATACACCAATATATTATCATAATCCTTATATTAAAATGATTGATTTTATGTAAATTATTTATAAATTTAAAGCTTTCATTTCGTTCCCTCCATTAACTGTCATACACCAATATATTATCATAATCCTTATATTAAAATGATTGATTTTATGTAAATTATTTATAAATT
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 101, 101
+ READ_SEG: [0, 101], [101, 101]
+ READ_START: 0, 101
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 202
+ TMP_KEY_ID: 0
+ TRIM_LEN: 202
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/12.0.stdout b/test/fastq-loader/expected/12.0.stdout
new file mode 100644
index 0000000..641ab0b
--- /dev/null
+++ b/test/fastq-loader/expected/12.0.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 42
+ BIO_BASE_COUNT: 42
+ CMP_BASE_COUNT: 42
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 202311120113300010103
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 21
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 65, 65, 93, 68, 68, 68, 68, 68, 70, 70, 70, 66, 70, 72, 72, 71, 71, 66, 68, 70, 70
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CCTACACTTGTATTTTGGTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 21, 0
+ READ_SEG: [0, 21], [21, 0]
+ READ_START: 0, 21
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 21
+ TRIM_LEN: 21
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 42
+ BIO_BASE_COUNT: 42
+ CMP_BASE_COUNT: 42
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 202311120113300010103
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 21
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 65, 65, 93, 68, 68, 68, 68, 68, 70, 70, 70, 66, 70, 72, 72, 71, 71, 66, 68, 70, 70
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CCTACACTTGTATTTTGGTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 21, 0
+ READ_SEG: [0, 21], [21, 0]
+ READ_START: 0, 21
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 21
+ TRIM_LEN: 21
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/12.1.stdout b/test/fastq-loader/expected/12.1.stdout
new file mode 100644
index 0000000..d6fafcd
--- /dev/null
+++ b/test/fastq-loader/expected/12.1.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 42
+ BIO_BASE_COUNT: 42
+ CMP_BASE_COUNT: 42
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 202311120113300010103
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 21
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 65, 65, 93, 68, 68, 68, 68, 68, 70, 70, 70, 66, 70, 72, 72, 71, 71, 66, 68, 70, 70
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CCTACACTTGTATTTTGGTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 21, 0
+ READ_SEG: [0, 21], [21, 0]
+ READ_START: 0, 21
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP: SG1
+ SPOT_ID: 1
+ SPOT_LEN: 21
+ TRIM_LEN: 21
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 42
+ BIO_BASE_COUNT: 42
+ CMP_BASE_COUNT: 42
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 202311120113300010103
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 21
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 65, 65, 93, 68, 68, 68, 68, 68, 70, 70, 70, 66, 70, 72, 72, 71, 71, 66, 68, 70, 70
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CCTACACTTGTATTTTGGTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 21, 0
+ READ_SEG: [0, 21], [21, 0]
+ READ_START: 0, 21
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP: SG2
+ SPOT_ID: 2
+ SPOT_LEN: 21
+ TRIM_LEN: 21
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/12.2.stdout b/test/fastq-loader/expected/12.2.stdout
new file mode 100644
index 0000000..4b332aa
--- /dev/null
+++ b/test/fastq-loader/expected/12.2.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 42
+ BIO_BASE_COUNT: 42
+ CMP_BASE_COUNT: 42
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 202311120113300010103202311120113300010103
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 42
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CCTACACTTGTATTTTGGTTACCTACACTTGTATTTTGGTTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 21, 21
+ READ_SEG: [0, 21], [21, 21]
+ READ_START: 0, 21
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP: ATCACG
+ SPOT_ID: 1
+ SPOT_LEN: 42
+ TMP_KEY_ID: 0
+ TRIM_LEN: 42
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.1.1.stdout b/test/fastq-loader/expected/2.1.1.stdout
new file mode 100644
index 0000000..39f13f2
--- /dev/null
+++ b/test/fastq-loader/expected/2.1.1.stdout
@@ -0,0 +1,44 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 12
+ BIO_BASE_COUNT: 12
+ CMP_BASE_COUNT: 12
+ CMP_CSREAD: 123123
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: TT
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 6, 0
+ READ_START: 0, 6
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 12
+ BIO_BASE_COUNT: 12
+ CMP_BASE_COUNT: 12
+ CMP_CSREAD: 123123
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: GT
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 6, 0
+ READ_START: 0, 6
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.1.stdout b/test/fastq-loader/expected/2.1.stdout
new file mode 100644
index 0000000..fdfe9dd
--- /dev/null
+++ b/test/fastq-loader/expected/2.1.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 148
+ BIO_BASE_COUNT: 148
+ CMP_BASE_COUNT: 148
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 01020330101302022131302322022022032222311023120230220132002100031300203330
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 74
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: TGGAATAACCATTCCTCATGCCTAGAAGAAGAATCTCTACAAGCAGGATTCTTGCTTTCAAAATGCCCTTATAA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 74, 0
+ READ_SEG: [0, 74], [74, 0]
+ READ_START: 0, 74
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 74
+ TRIM_LEN: 74
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 148
+ BIO_BASE_COUNT: 148
+ CMP_BASE_COUNT: 148
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 00131012333231011230202133203233112022003112203010033321302212200030111213
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 74
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: TTGCAACTATAGCAACAGCCTTCATAGGCTATGTCCTCCCGTGAGGCCAAATATCATTCTGAGGGGCCACAGTA
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 74, 0
+ READ_SEG: [0, 74], [74, 0]
+ READ_START: 0, 74
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 74
+ TRIM_LEN: 74
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.5.stdout b/test/fastq-loader/expected/2.5.stdout
new file mode 100644
index 0000000..78d3c21
--- /dev/null
+++ b/test/fastq-loader/expected/2.5.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 66
+ BIO_BASE_COUNT: 66
+ CMP_BASE_COUNT: 66
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 301212321010213302323201331023331
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 33
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: AACTGATCAACCTGCGGATCGAACGCAAGCGCA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 33
+ READ_SEG: [0, 33]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 33
+ TRIM_LEN: 33
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 66
+ BIO_BASE_COUNT: 66
+ CMP_BASE_COUNT: 66
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 311203221313213231002230003322030
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 33
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 33
+ READ_SEG: [0, 33]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 33
+ TRIM_LEN: 33
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.6.stdout b/test/fastq-loader/expected/2.6.stdout
new file mode 100644
index 0000000..c7db9a3
--- /dev/null
+++ b/test/fastq-loader/expected/2.6.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 66
+ BIO_BASE_COUNT: 66
+ CMP_BASE_COUNT: 66
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 301212321010213302323201331023331311203221313213231002230003322030
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 66
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: AACTGATCAACCTGCGGATCGAACGCAAGCGCAACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 33, 33
+ READ_SEG: [0, 33], [33, 33]
+ READ_START: 0, 33
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 66
+ TMP_KEY_ID: 0
+ TRIM_LEN: 66
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.7.stdout b/test/fastq-loader/expected/2.7.stdout
new file mode 100644
index 0000000..32a917b
--- /dev/null
+++ b/test/fastq-loader/expected/2.7.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 33
+ BIO_BASE_COUNT: 33
+ CMP_BASE_COUNT: 33
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 301212321010213302323201331023331
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 33
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: AACTGATCAACCTGCGGATCGAACGCAAGCGCA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 33
+ READ_SEG: [0, 33]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 33
+ TMP_KEY_ID: 0
+ TRIM_LEN: 33
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.8.1.stdout b/test/fastq-loader/expected/2.8.1.stdout
new file mode 100644
index 0000000..3706e68
--- /dev/null
+++ b/test/fastq-loader/expected/2.8.1.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 6
+ BIO_BASE_COUNT: 6
+ CMP_BASE_COUNT: 6
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: ..3
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 3
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: NTA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 3
+ READ_SEG: [0, 3]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 3
+ TRIM_LEN: 3
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 6
+ BIO_BASE_COUNT: 6
+ CMP_BASE_COUNT: 6
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: ...
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 3
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: NNT
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 3
+ READ_SEG: [0, 3]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 3
+ TRIM_LEN: 3
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.8.stdout b/test/fastq-loader/expected/2.8.stdout
new file mode 100644
index 0000000..17db95c
--- /dev/null
+++ b/test/fastq-loader/expected/2.8.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 6
+ BIO_BASE_COUNT: 6
+ CMP_BASE_COUNT: 6
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: ..3
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 3
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: NTA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 3
+ READ_SEG: [0, 3]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 3
+ TRIM_LEN: 3
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 6
+ BIO_BASE_COUNT: 6
+ CMP_BASE_COUNT: 6
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: ..3
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 3
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 0, 0, 0
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: NTA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 3
+ READ_SEG: [0, 3]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 3
+ TRIM_LEN: 3
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/2.9.1.stderr b/test/fastq-loader/expected/2.9.1.stderr
new file mode 100644
index 0000000..2539282
--- /dev/null
+++ b/test/fastq-loader/expected/2.9.1.stderr
@@ -0,0 +1,4 @@
+latf-load err: data inconsistent while reading document within alignment module - Spot 'BILLIEHOLIDAY': length of original quality does not match sequence
+latf-load warn: The file contained no records that were processed.
+latf-load err: data empty while reading file within alignment module - accession="obj" errors="1" status="failure"
+latf-load err: data empty while reading file within alignment module - load failed
diff --git a/test/fastq-loader/expected/2.9.stderr b/test/fastq-loader/expected/2.9.stderr
new file mode 100644
index 0000000..2539282
--- /dev/null
+++ b/test/fastq-loader/expected/2.9.stderr
@@ -0,0 +1,4 @@
+latf-load err: data inconsistent while reading document within alignment module - Spot 'BILLIEHOLIDAY': length of original quality does not match sequence
+latf-load warn: The file contained no records that were processed.
+latf-load err: data empty while reading file within alignment module - accession="obj" errors="1" status="failure"
+latf-load err: data empty while reading file within alignment module - load failed
diff --git a/test/fastq-loader/expected/3.1.stdout b/test/fastq-loader/expected/3.1.stdout
new file mode 100644
index 0000000..701afb2
--- /dev/null
+++ b/test/fastq-loader/expected/3.1.stdout
@@ -0,0 +1,120 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 144
+ BIO_BASE_COUNT: 144
+ CMP_BASE_COUNT: 144
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 112332022230202..1120022110..03..0..
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 4
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 36
+ READ_SEG: [0, 36]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 4
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 36
+ TRIM_LEN: 36
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 144
+ BIO_BASE_COUNT: 144
+ CMP_BASE_COUNT: 144
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 122311322021001....32101321......2..
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 4
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: GAGCACGAGGACCCANNNATCAACGACNANNNGANN
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 36
+ READ_SEG: [0, 36]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 4
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 36
+ TRIM_LEN: 36
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 144
+ BIO_BASE_COUNT: 144
+ CMP_BASE_COUNT: 144
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 312102231132..2..00....12...........
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 4
+ MIN_SPOT_ID: 1
+ NAME: 3
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: ACTGGAGCACGANAGNCCCNCNACTNANANCNGNNN
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 36
+ READ_SEG: [0, 36]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 4
+ SPOT_GROUP:
+ SPOT_ID: 3
+ SPOT_LEN: 36
+ TRIM_LEN: 36
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 144
+ BIO_BASE_COUNT: 144
+ CMP_BASE_COUNT: 144
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 221022311322201....30030211......3..
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 4
+ MIN_SPOT_ID: 1
+ NAME: 4
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: CTGGAGCACGAGAACNCNGCCCGGACANANGNGCNN
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 36
+ READ_SEG: [0, 36]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 4
+ SPOT_GROUP:
+ SPOT_ID: 4
+ SPOT_LEN: 36
+ TRIM_LEN: 36
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/4.2.stdout b/test/fastq-loader/expected/4.2.stdout
new file mode 100644
index 0000000..4aa7d25
--- /dev/null
+++ b/test/fastq-loader/expected/4.2.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 36
+ BIO_BASE_COUNT: 36
+ CMP_BASE_COUNT: 36
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 112332022230202..1120022110..03..0..
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 1, 10, 20, 20, 20, 10, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 10, 10, 20, 10, 20, 20, 10, 20, 20, 20
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 0
+ READ_SEG: [0, 36], [36, 0]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 36
+ TMP_KEY_ID: 0
+ TRIM_LEN: 36
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/4.3.1.stderr b/test/fastq-loader/expected/4.3.1.stderr
new file mode 100644
index 0000000..1268b1a
--- /dev/null
+++ b/test/fastq-loader/expected/4.3.1.stderr
@@ -0,0 +1,14 @@
+Usage:
+ latf-load [options] <fastq-file> ...
+
+Summary:
+ Load FASTQ formatted data files
+
+Example:
+ latf-load -p 454 -o SRZ123456 123456-1.fastq 123456-2.fastq
+
+
+Use option --help for more information.
+
+latf-load err: param incorrect while accessing argument list within application support module - accession="obj" errors="0" status="failure"
+latf-load err: param incorrect while accessing argument list within application support module - load failed
diff --git a/test/fastq-loader/expected/4.4.stdout b/test/fastq-loader/expected/4.4.stdout
new file mode 100644
index 0000000..7d001d0
--- /dev/null
+++ b/test/fastq-loader/expected/4.4.stdout
@@ -0,0 +1,66 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 105
+ BIO_BASE_COUNT: 105
+ CMP_BASE_COUNT: 105
+ CMP_CSREAD: 12312312312312312312312312312312312
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: GT
+ MAX_SPOT_ID: 3
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 35, 0
+ READ_START: 0, 35
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SPOT_COUNT: 3
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 105
+ BIO_BASE_COUNT: 105
+ CMP_BASE_COUNT: 105
+ CMP_CSREAD: 12312312312312312312312312312312312
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: GT
+ MAX_SPOT_ID: 3
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 35, 0
+ READ_START: 0, 35
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SPOT_COUNT: 3
+ SPOT_GROUP:
+ SPOT_ID: 2
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 105
+ BIO_BASE_COUNT: 105
+ CMP_BASE_COUNT: 105
+ CMP_CSREAD: 12312312312312312312312312312312312
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: GT
+ MAX_SPOT_ID: 3
+ MIN_SPOT_ID: 1
+ NAME: 3
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 35, 0
+ READ_START: 0, 35
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SPOT_COUNT: 3
+ SPOT_GROUP:
+ SPOT_ID: 3
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/4.5.1.stderr b/test/fastq-loader/expected/4.5.1.stderr
new file mode 100644
index 0000000..0cb8caf
--- /dev/null
+++ b/test/fastq-loader/expected/4.5.1.stderr
@@ -0,0 +1,8 @@
+latf-load warn: file="4.5.1.fastq"
+latf-load warn:syntax error, unexpected fqASCQUAL, expecting fqBASESEQ or fqCOLORSEQ
+latf-load warn: file="4.5.1.fastq"
+latf-load warn:syntax error, unexpected fqASCQUAL, expecting fqBASESEQ or fqCOLORSEQ
+latf-load err: data invalid while closing table - Too many bad records: records: 5, bad records: 2, bad records percentage: 0.40, allowed percentage: 0.20
+latf-load warn: path not found while removing directory within database module - table CS_SEQUENCE failed to drop temporary column 'TMP_KEY_ID'
+latf-load err: data invalid while closing table - accession="obj" errors="2" status="failure"
+latf-load err: data invalid while closing table - load failed
diff --git a/test/fastq-loader/expected/4.5.stderr b/test/fastq-loader/expected/4.5.stderr
new file mode 100644
index 0000000..99a8e64
--- /dev/null
+++ b/test/fastq-loader/expected/4.5.stderr
@@ -0,0 +1,9 @@
+latf-load warn: file="4.5.fastq"
+latf-load warn:syntax error, unexpected fqASCQUAL, expecting fqBASESEQ or fqCOLORSEQ
+latf-load warn: file="4.5.fastq"
+latf-load warn:syntax error, unexpected fqASCQUAL, expecting fqBASESEQ or fqCOLORSEQ
+latf-load warn: file="4.5.fastq"
+latf-load warn:syntax error, unexpected fqASCQUAL, expecting fqBASESEQ or fqCOLORSEQ
+latf-load err: error excessive while reading file within alignment module - Number of errors 3 exceeds limit of 2: Exiting
+latf-load err: error excessive while reading file within alignment module - accession="obj" errors="3" status="failure"
+latf-load err: error excessive while reading file within alignment module - load failed
diff --git a/test/fastq-loader/expected/4.6.1.stderr b/test/fastq-loader/expected/4.6.1.stderr
new file mode 100644
index 0000000..1f17349
--- /dev/null
+++ b/test/fastq-loader/expected/4.6.1.stderr
@@ -0,0 +1,3 @@
+latf-load err: param incorrect while accessing argument list within application support module - Invalid platform SOMETHINGELSE
+latf-load err: param incorrect while accessing argument list within application support module - accession="obj" errors="0" status="failure"
+latf-load err: param incorrect while accessing argument list within application support module - load failed
diff --git a/test/fastq-loader/expected/4.6.stdout b/test/fastq-loader/expected/4.6.stdout
new file mode 100644
index 0000000..5b19de9
--- /dev/null
+++ b/test/fastq-loader/expected/4.6.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 36
+ BIO_BASE_COUNT: 36
+ CMP_BASE_COUNT: 36
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 112332022230202..1120022110..03..0..
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_454
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 5, 13, 27, 22, 22, 19, 22, 23, 23, 23, 24, 24, 24, 23, 22, 22, 21, 22, 22, 21, 25, 24, 24, 22, 24, 22, 19, 18, 20, 19, 22, 22, 19, 22, 22, 24
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 0
+ READ_SEG: [0, 36], [36, 0]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 36
+ TMP_KEY_ID: 0
+ TRIM_LEN: 36
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/4.7.stderr b/test/fastq-loader/expected/4.7.stderr
new file mode 100644
index 0000000..e89fee5
--- /dev/null
+++ b/test/fastq-loader/expected/4.7.stderr
@@ -0,0 +1,2 @@
+latf-load err: path not found while accessing directory within file system module - accession="obj" errors="0" status="failure"
+latf-load err: path not found while accessing directory within file system module - load failed
diff --git a/test/fastq-loader/expected/5.0.stdout b/test/fastq-loader/expected/5.0.stdout
new file mode 100644
index 0000000..1e9793c
--- /dev/null
+++ b/test/fastq-loader/expected/5.0.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 36
+ BIO_BASE_COUNT: 36
+ CMP_BASE_COUNT: 36
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 112332022230202..1120022110..03..0..
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 36
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 5, 13, 27, 22, 22, 19, 22, 23, 23, 23, 24, 24, 24, 23, 22, 22, 21, 22, 22, 21, 25, 24, 24, 22, 24, 22, 19, 18, 20, 19, 22, 22, 19, 22, 22, 24
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 36, 0
+ READ_SEG: [0, 36], [36, 0]
+ READ_START: 0, 36
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 36
+ TMP_KEY_ID: 0
+ TRIM_LEN: 36
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/6.0.stdout b/test/fastq-loader/expected/6.0.stdout
new file mode 100644
index 0000000..df6b548
--- /dev/null
+++ b/test/fastq-loader/expected/6.0.stdout
@@ -0,0 +1,390 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 33333113001122113112313102030222133122230321033330003303123012303302102200033003011212211003120000330322030230111230023000331031322331013122211000001112132020113221212022310122001303300031130003321103303120130022220212122000003122200012300330033000311110001103033003330301230033021210033003
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 23, 24, 25, 39, 39, 32, 25, 14, 16, 16, 24, 17, 35, 34, 39, 39, 40, 40, 36, 40, 40, 40, 40, 38, 38, 38, 38, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 32, 24, 20, 20, 20, 18, 32, 23, 27, 36, 39, 39, 38, 40, 40, 40, 40, 38, 36, 36, 32, 39, 35, 26, 22, 16, 16, 16, 16, 26, 14, 14, 14, 32, 32, 35, 33, 38, 40, 40, 40, 40, 36, 35, 35, 34, 34, 25, 25, 25, 25, 25, 40, 22, 22, 35, 40, 39, 39, 40, 40, 40, 35, 35, 35, 38, 40, 35, 20, 20, 21, 28 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: ATATACATTTGTCTGTACAGCATGGAATTCTCATACTCTAATCAATATAAAATAATGATTGATTATTCAAGAAAATAAATTGTCAGACAAATGAAAAATAATCTTAAGCCACAGCCCTAAAATACCGTAGATACCATGAGACAAAAAACACTGCTTCCACGAGTCAGGAGCAACTCCCATTATTTTACATTTTATCACCGCCGTCCATTTCTCTTCAGTCTTTTTTACTCTTTTGATTTATTTATTTTACACAAAACAATTATTTATAATTGATTTATTCAGTTTATTTA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 290
+ READ_SEG: [0, 290]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 290
+ TRIM_LEN: 290
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 0120230102202301112310113213110012202220220222322023022130220320202112303211020022210222022022021121310130201332031103210011302222010211021210231301020303202212303210211330322111311122200220131133120121202301320220120112212332211222121001
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 37, 37, 36, 36, 36, 37, 37, 37, 37, 37, 37, 40, 40, 40, 40, 40, 40, 40, 39, 37, 26, 26, 26, 39, 39, 39, 39, 39, 39, 31, 31, 31, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 35, 37, 35, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 35, 35, 30, 25, 25, 24, 24, 33, 20, 21, 21, 25, 28, 32, 33, 35, 35, 35, 35, 35, 35, 30, 30, 30, 35, 35, 32, 28, 28, 25, 25, 25, 25, 25, 24, 27, 28, 28, 29, 29, 28, 30, 27 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: TGAAGCCAAGAAGCCACAGCAACATCATGTTTGAGGAGAAGAAGAGCTCCTAAGACGGAGGCTTCCTGTCGGCTGTTCCCTCTGGAGAAGAAGAAGTGACGTTGCCTTGCGAATGTTAGTTTGTAAGAGAACCTGTTCAGTTCGTAACCTTAATCCTCAGCCGACCTGTATTAGACACGTGTCTCCCTCCATGTATGAACTGAAGCCATCCTCCAGGTGAGTCGCTCACTCTGACCCA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 238
+ READ_SEG: [0, 238]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 238
+ TRIM_LEN: 238
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: .
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 3
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 0
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: N
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 1
+ READ_SEG: [0, 1]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 3
+ SPOT_LEN: 1
+ TRIM_LEN: 1
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 3233002222000311131011112200120012023120111002313221303322200232101021320113203131113033111022010111033201333100123131330333103123103222323120100123231
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 4
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 39, 40, 40, 28, 28, 28, 36, 33, 39, 29, 29, 29, 29, 36, 36, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 36, 33, 33, 32, 35, 38, 38, 38, 27, 27, 29, 32, 32, 32, 32, 32, 35, 35, 36, 36, 39, 39, 39, 39, 40, 40, 40, 36, 35, 37, 37, 40, 40, 40, 36, 36 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: AGCGGGAGAGGGGCACATGGTGTGAGGGTCCCAGGATGAACACCCTACGAGTAATAGAGGGATCAACCTGCTTGTAGGCATGTGCCGCACAAGAACCACAATAGGTATACCCAGCATGCGGCGCAATGATGGCTCTAGCAGGTTTGATCGT
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 151
+ READ_SEG: [0, 151]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 4
+ SPOT_LEN: 151
+ TRIM_LEN: 151
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 20223123320211200021310032023022113211020213231200100101020333313100120232001001121212120333033312111130201212320110020211321111111011101012102302211312210301021213112121103330001222301133102212210100011002111030120223123001130013200000..0000213
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 5
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 26, 26, 26, 26, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 35, 26, 26, 26, 39, 40, 40, 40, 40, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 26, 26, 26, 26, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: CCTCGTCGCTTCACTTTTCATGGGCTTCGGAGTGCTGTTCCTGCTACTTTGGGTTGGAATATACGTTTGAAGCTTTGGGTGACTGACTTATAATATGACACATTCCAGTCGAACAAAGGACATCACACACAACACCAACTGGATTCTGTACTCAATTGGACTGCACTGACAATATTTTGAGATTGTATGGAGTCTGGTTTTGTTTCACAATTGAAGATGATTTGTAAACGAAAAAANAAAAAGTA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 245
+ READ_SEG: [0, 245]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 5
+ SPOT_LEN: 245
+ TRIM_LEN: 245
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: .
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 6
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 0
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: N
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 1
+ READ_SEG: [0, 1]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 6
+ SPOT_LEN: 1
+ TRIM_LEN: 1
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 223200321003103331210230003113330102230133303001022311100130130130131100321130133000012310133313120110010203102331310332111310031021031232200102023210011110102223120033232133321130013221231231102322000101220220002200102131111102333001221113300123201102323233323230223032203330223213323101
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 7
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 40, 40, 40, 39, 39, 39, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 34, 34, 34, 34, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 39, 21, 21, 21, 21, 21, 35, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 38, 38, 38, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: CTAGGGCTGGGCAATATGACCTAAAATGTATAACCTCGGTATAATTTGGAGCACAAACGGTAACGGTACAAATCACGGTATTTTTGATGGTATACGTCCACCCAAGGCAAGCGTACCGCTGTGCAAATGGACCGTCGAGGGTTCCTAGTTTGTGTTGGAGATGAAATAGCTGCGCTGTAAACGAGTCGTCGTGGATCTTTTGGTCTTCTTTTCTTTGGACGTGTGTTCGCGGGTCTGTGCGGGTCGAACAAGCTAGCGCTAGCCTCGGCTCCGCGGAGCTGCGATGGT
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 288
+ READ_SEG: [0, 288]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 7
+ SPOT_LEN: 288
+ TRIM_LEN: 288
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 312332130330013213131311103130100220330003132112211210312031220331223032033112121022101200020112122023001312220111111111111111111111111111232
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 8
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 39, 39, 40, 40, 40, 40, 40, 40, 40, 39, 39, 38, 37, 35, 35, 35, 35, 32, 32, 21, 21, 18, 25, 24, 32, 33, 24, 25, 25, 26, 35, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 34, 35, 35, 37, 37, 37, 37, 35, 35, 35, 37, 37, 37, 37, 37, 37, 37, 35, 33, 32, 32, 32, 32, 32, 27, 23, 16, 16, 16, 16, 19, 19, 32, 32, 32, 30, 32, 32, 33, 37, 37, 37, 35, 35, 35, 35, 35, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: ACTATCATTATTTGCTGCATGCACAATGCCAAAGAATAAAATGCTGTCTGTCAATGAATGAGGCGTCTAATCCGCACTGACCTCAACTTTTCCACTGAGGATTTGCAGAGGTGTGTGTGTGTGTGTGTGTGTGTGTGTCGA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 141
+ READ_SEG: [0, 141]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 8
+ SPOT_LEN: 141
+ TRIM_LEN: 141
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 3
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 9
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 14
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: A
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 1
+ READ_SEG: [0, 1]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 9
+ SPOT_LEN: 1
+ TRIM_LEN: 1
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 1113102220321220323011002113132101023311102023132320203003331103221213312231022232120110210122020113123200020011332122312101301202210221013202122020003213210102101021102012100330320023102103000110102300112013212333113010003103101221222211022111033003
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 10
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 21, 21, 21, 33, 37, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 35, 35, 35, 34, 37, 33, 31, 30, 29, 29, 19, 17, 17, 17, 13, 13, 13, 14, 12, 19, 18, 22, 22, 25, 24 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: GTGCAAGAGGCTGAGGCTAACAAAGTGCATCAACCTATGTGGAAGCATCGAAGGCCCGCGTGGCTCAGTATGAGCAAGAGCTGAACAAGTTGAGGAACATGATCCCCTTTGTATCAGATGACCATTGAAGACCTCAACGAAGTCTTCCCCGACGACCAAGTTGGACAAGGTCAAATAATCCCTACCTGGCCCCACCAAGCCCACTTGCTGATATGTAACCCCGTTACCAGACTCTCACCTCACAATAAAT
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 250
+ READ_SEG: [0, 250]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 10
+ SPOT_LEN: 250
+ TRIM_LEN: 250
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 33311333221321211222302103222110103211322212002132122301203222112123122333113
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 11
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 39, 34, 34, 38, 38, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 38, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: ATACATATCTGCTGACAGAGCCTGGCTCTGTTGGCTGTAGAGTCCCTGCTGAGCCAGGCTCTGTCAGCAGATATGTA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 77
+ READ_SEG: [0, 77]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 11
+ SPOT_LEN: 77
+ TRIM_LEN: 77
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 303
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 12
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 16, 16, 16
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: AAT
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 3
+ READ_SEG: [0, 3]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 12
+ SPOT_LEN: 3
+ TRIM_LEN: 3
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1992
+ BIO_BASE_COUNT: 1992
+ CMP_BASE_COUNT: 1992
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 020211001202210203122201033110123121112132313130203112333120312022111011200121221103112021111111201133110022130210220230010312320130210113221001212321320221111102202100010312200222312330200030233002201312320012211123133221111221031312122131320310320002113131112033112211322122113000202200102031111120000233
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 13
+ MIN_SPOT_ID: 1
+ NAME: 13
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 40, 40, 40, 40, 26, 21, 21, 21, 30, 30, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: TCCTGTTTGAAGACCTTACTCTTGGCGTGGTCGTCACAGTAGCATGCCTTACAGCGCAGGCAGGAGTGTTGTCCCAGTCTGTTACAGGACACACACTTGTATGTTTCTGCCTGGAGGATTTGGCAGCTTGCCTGGTGCTCAAACTGATCATCCTCACACAAGAAGTTTTGGCAGAAAGAGCAGCGGAAAATTCGCCCTCCATGATCCCAGACACTACGCTCACACTCAATGCAGTCTGCATCCGTTAGGGGACATGCACAGGCGTGAGTGCTCAGACATTTTCCTCCCAAGGCACACAGGGGGATA
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 306
+ READ_SEG: [0, 306]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 13
+ SPOT_GROUP:
+ SPOT_ID: 13
+ SPOT_LEN: 306
+ TRIM_LEN: 306
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/7.1.stdout b/test/fastq-loader/expected/7.1.stdout
new file mode 100644
index 0000000..7a31be5
--- /dev/null
+++ b/test/fastq-loader/expected/7.1.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1025
+ BIO_BASE_COUNT: 1025
+ CMP_BASE_COUNT: 1025
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 32221001232021032212221120133210330331132030111131310212320130231200330232013203300231032110333113001122130311333232301021300200321100002110112231020013122132303310001331220213313321330223331101002002230033013230020231000320033101332123232101011000130130302310102031031213302310210102212222023123212031111210200121222111301020122122023130022031231211000203330131110310000200100212313123101330333110220231213313002301111130012202000333020020223303121301310203312033023110 [...]
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_PACBIO_SMRT
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 43, 43, 71, 69, 29, 29, 29, 29, 58, 58, 39, 39, 67, 70, 70, 70, 71, 71, 71, 71, 71, 71, 71, 45, 43, 47, 47, 71, 71, 71, 38, 38, 71, 45, 45, 71, 69, 69, 71, 71, 50, 50, 22, 22, 22, 62, 62, 70, 71, 60, 60, 71, 71, 52, 52, 71, 71, 50, 50, 50, 67, 64, 38, 38, 71, 57, 57, 57, 57, 71, 71, 71, 71, 39, 39, 39, 46, 48, 71, 71, 42, 42, 42, 43, 52, 52, 71, 71, 68, 18, 54, 54, 56, 39, 39, 52, 71, 71, 71, 71, 71, 57, 57, 71, 71, 42, 41, 6, 65, 36, 36, 65, 34, 34, 61, 70, 57, 42, [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: AGAGTTTGATCCTGGCTCAGAGTGAACGCTGGCGGCGTGCTTAACACATGCAAGTCGAACGGATGAAATAAGCTTGCTTATTTCGTTAGTGGCGCACGGGTGAGTAATGTATAGCTAACCTGCCCTTTAGTGGGGGACAACAGATGGAAACGTCTGCTAATACCCCATACTCCTGCGTATCATAAGATATGTTGGGAAAGATTTATTGCTAAAGGATGGGGCTTTATGGTATCAGCTAGTTGGTGGGGTAACGGCCTACCAAGGCAATGACGCCTACCTGGTTCTGAGAGGATGATCAGGCACACTGGAAACTGAGACACGGTTCCAGACTCCTACGGGAGGCAGCAGTGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGGAGGATGACGCATTTCGGTGTGTAAACTCCTTTTATAAGGGAAGATAATGACGGTACCTTATGAATAAGCACC [...]
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 507
+ READ_SEG: [0, 507]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 507
+ TRIM_LEN: 507
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1025
+ BIO_BASE_COUNT: 1025
+ CMP_BASE_COUNT: 1025
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 32221001232021032212021320133210330133132030111131310212320132221322231020312100202300020312003220322002110332013001122130133311001301021302331020020203323022300200121002213031302033303311211202123313103200132110300133220301130133202310210033312001303232321010112223301123001101020332132310013010301221222200332013012111121020121222111301201221220231302203123121100020333013331031003101002121331231033103331122112022201000230232313002322211033010002022000002213013100102 [...]
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_PACBIO_SMRT
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 71, 68, 68, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 69, 69, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 48, 48, 71, 70, 70, 71, 71, 71, 71, 71, 71, 71, 51, 51, 71, 71, 63, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 62, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 67, 67, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 21, 21, 21, 21, 23, 71, 71, 71, 53, 53, 53, 71, 71, 71, 71, 71, 71, 53, 53, 53, 71, 71, 59, 59, 71, 71, 71, 71, 71, 28, 28, 28, 71, 71, 71 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: AGAGTTTGATCCTGGCTCAGGACGAACGCTGGCGGTATGCTTAACACATGCAAGTCGAACGAGACGAGATGGAATGACCCTTCGGGGAATGAAATCTTAGAAAGTGGCGAACGGGTGAGTAACGCGTGGGTAACCTGCCTATGGAAAGGAATAGCCTCGGGAAACTGGGAGTAATGCCTTATAATACAGTGAAGTCGCATGGCTTTGCTGTTAAACGCTCCGGTGCCATAGGATGGACCCGCGTCCCATTAGCTAGTTGGTGAGATAACAGCCCACCAAGGCGACGATGGGTAACCGGTCTGAGAGGGCGAACGGTCACACTGGAACTGAGACACGGTCCAGACTCCTACGGAGGCAGCAGTGGGGAATATTGCGCAATGGGCAACCCTGACGCAGCAATACCGCGTGAGTGAAGAGGTTTTCGGATCGTAAAGCTCTGTTATTGGGGAAGAAAAAAGACGGTACCCAAG [...]
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 518
+ READ_SEG: [0, 518]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 518
+ TRIM_LEN: 518
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/7.2.stdout b/test/fastq-loader/expected/7.2.stdout
new file mode 100644
index 0000000..2fbe683
--- /dev/null
+++ b/test/fastq-loader/expected/7.2.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1033
+ BIO_BASE_COUNT: 1033
+ CMP_BASE_COUNT: 1033
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 3320110002200110011000000000002310000002200001111110022000011110002112000000110330330022003213300111110000001001320011111100000110000013211101113301100110000012311110110110001110010123000000110000001100011000001110111
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_PACBIO_SMRT
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 6, 0, 4, 0, 4, 2, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 5, 0, 0, 1, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 2, 8, 0, 1, 0, 0, 4, 0, [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: ATCCACCCCTCCCACCCACCCCCCCCCCCCTACCCCCCCTCCCCCACACACCCTCCCCCACACCCCTGTCCCCCCCACCGCCGCCCTCCCGACGCCCACACAAAAAAACCCATCCCACACACCCCCCACCCCCCATCACAACACGCCACCCACCCCCCAGCACACCACCACCCCACAAACCAGCCCCCCCACCCCCCCACCCCACCCCCCACAACAC
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 217
+ READ_SEG: [0, 217]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 217
+ TRIM_LEN: 217
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1033
+ BIO_BASE_COUNT: 1033
+ CMP_BASE_COUNT: 1033
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 20111100001100101000113030000101011011011010001110003333000110000111100011113301100330110033011101330003213332001001100033330011010100110110003211100110000033110110000000333311033001100011100132133010122010100000000033011001230330000000110123000001100033003223003213030113333003311000123033000012300000000000001111100001010100100011000011001010001101100133011010000110000111010000000111010101000110011011010112201100003300001010003210000101100013311110111000133011000321 [...]
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_PACBIO_SMRT
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 2, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 1, 1, 2, 0, 4, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 4, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6, 0, 0, 0, 1, 0, 0, 2, 0, 0, 1, 1, 2, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 4, 0, 2, 0, 0, 3, 3, 2, 0, 3, 3, 0, 0, 0, 1, 2, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3, 5, 0, [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: CCACACCCCCACCCAACCCCACGGCCCCCAACCACCACCACCAAAACACCCCGCGCCCCACCCCCACACCCCACACGCCACCCGCCACCCGCCACAACGCCCCGACGCGAAACCCACCCCGCGCCCACCAACCCACCACCCCGACACCCACCCCCCGCACCACCCCCCCCGCGCACCGCCCACCCCACAAACGACGCCAACTCCAACCCCCCCCCCGCCACCCAGCCGCCCCCCCCACCAGCCCCCCACCCCGCCCGAGCCCGACGGCCACGCGCCCGCACCCCAGCCGCCCCCAGCCCCCCCCCCCCCCACACAAAAACCAACCCAAAACAAAAACAAACCAAAACAACAAACGCCACCAAAAACAAAAACACCAAAAAAAACACCAACCAAAACAAACAACAACCACTCCACCCCCGCCCCCAACCCCGACCCCCAACAAAACGCACACCACAAAACGCCACCCCGAC [...]
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 816
+ READ_SEG: [0, 816]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 816
+ TRIM_LEN: 816
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/7.3.stdout b/test/fastq-loader/expected/7.3.stdout
new file mode 100644
index 0000000..0674152
--- /dev/null
+++ b/test/fastq-loader/expected/7.3.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1076
+ BIO_BASE_COUNT: 1076
+ CMP_BASE_COUNT: 1076
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 1013222012202312332112220332021212330330022330222013211133333202123102303221202133123222223222022210102222002332020301202110230033301121323222230022311123012302231012212200232222323231012221211111011221332310
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_PACBIO_SMRT
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: GGTAGAGGTCTTCGTCGCTGTCTCCGCTTCAGTCGCCGCCCTCGCCTCTTGCTGTGCGCGCTTCAGCAAGCCGAGTCCTGCGTCGAGAGATCTCCTCTGGTTCTCTTTCGCTTCCGGTCCTGTTCGGGCGCCACTGCTAGAGATTTCTACACTAACTAAGATGGTCTGAGGGATCTCTAGCTACCAGAGTCACACAACAGACGCTACC
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 208
+ READ_SEG: [0, 208]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 208
+ TRIM_LEN: 208
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0
+ BASE_COUNT: 1076
+ BIO_BASE_COUNT: 1076
+ CMP_BASE_COUNT: 1076
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 10021022333303332130220223031222201202222233333322222203300211203322000303033331121333021322123333320232202222131131321312123123331222322023211331233111222311133230222233230101320032111111110130220223302220223212202211222203102222000213332203303332121120201210321221233033211132123133333333303312110333333333332110233311312220332203333303003003331231323222103032103322203231220233221230331212222320013222013022231203300101312232120101101330101303120220231200112201331312 [...]
+ CS_KEY: T
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 0
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_PACBIO_SMRT
+PRIMARY_ALIGNMENT_ID: 0
+ QUALITY: 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ: GGGACCTCGCGCCGCGACGGAGGAGCCGTCTCTTGAAGAGAGCGCGCGAGAGAGGCGGGACAGGCGAGGGGCCGGCGCGTGACGCGGACGAGTCGCGCGAAGCTCCTCTCATGTACGACGTCAGCAGCGCAGAGCTCCTAGTGCGTCGCACAGAGCACATAGCCTCTCGCTAACCATCCCGACACACACAACGGAGGAGCGGAGAAGATCAGAAGACAGAGAATGGAGAGGGGACGCGAGGCGGCGCTGACAGGAACTGGCTGAGTCGCCGCTGTGCTGATGCGCGCGCGCCGCAGTGGCGCGCGCGCGCTGTTCGCGTGCAGAGGCGAGGCGCGCCGGGCCCGCGTCGTAGCTCTGGCCGACCGCTCTTAGCAGAAGCGAGTCGGCGTCAGAGATCCCATCTCCATTCTCGTCCGCCCAACGTCTAGTCCAACAACGCCAACGGCAGGAGGATGAAACAGAACGCATGA [...]
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_LEN: 868
+ READ_SEG: [0, 868]
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP:
+ SPOT_ID: 2
+ SPOT_LEN: 868
+ TRIM_LEN: 868
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/8.0.stdout b/test/fastq-loader/expected/8.0.stdout
new file mode 100644
index 0000000..c7db9a3
--- /dev/null
+++ b/test/fastq-loader/expected/8.0.stdout
@@ -0,0 +1,31 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 66
+ BIO_BASE_COUNT: 66
+ CMP_BASE_COUNT: 66
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 301212321010213302323201331023331311203221313213231002230003322030
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 66
+ MAX_SPOT_ID: 1
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: AACTGATCAACCTGCGGATCGAACGCAAGCGCAACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 33, 33
+ READ_SEG: [0, 33], [33, 33]
+ READ_START: 0, 33
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 1
+ SPOT_GROUP:
+ SPOT_ID: 1
+ SPOT_LEN: 66
+ TMP_KEY_ID: 0
+ TRIM_LEN: 66
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/expected/8.1.stderr b/test/fastq-loader/expected/8.1.stderr
new file mode 100644
index 0000000..3a036e1
--- /dev/null
+++ b/test/fastq-loader/expected/8.1.stderr
@@ -0,0 +1,5 @@
+latf-load err: file="8.1.fastq"
+latf-load err:Inconsistent secondary read number: previously used 3, now seen 2
+latf-load warn: path not found while removing directory within database module - table SEQUENCE failed to drop temporary column 'TMP_KEY_ID'
+latf-load err: format unsupported while parsing file - accession="obj" errors="1" status="failure"
+latf-load err: format unsupported while parsing file - load failed
diff --git a/test/fastq-loader/expected/9.0.stdout b/test/fastq-loader/expected/9.0.stdout
new file mode 100644
index 0000000..4ca7504
--- /dev/null
+++ b/test/fastq-loader/expected/9.0.stdout
@@ -0,0 +1,60 @@
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 404
+ BIO_BASE_COUNT: 404
+ CMP_BASE_COUNT: 404
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 2230120003312333000101101303001013310103223001100120023010030322210223120322311200013213102011200203322132132213111122002212110220112212323122012213221002021302221333210033032200312330210133320301210321
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 202
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 1
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 30, 30, 27, 35, 26, 35, 35, 35, 37, 32, 34, 23, 11, 34, 25, 34, 37, 38, 40, 33, 38, 10, 34, 34, 8, 18, 27, 34, 37, 39, 29, 34, 16, 9, 25, 36, 37, 35, 33, 34, 30, 9, 16, 33, 38, 37, 15, 33, 37, 31, 9, 24, 35, 34, 29, 33, 8, 30, 30, 34, 34, 33, 34, 25, 35, 22, 23, 19, 31, 28, 37, 36, 36, 31, 36, 32, 25, 8, 22, 8, 8, 28, 30, 24, 30, 22, 31, 31, 33, 34, 25, 29, 26, 26, 18, 28, 31, 33, 33, 33, 33, 31, 33, 31, 37, 30, 35, 32, 36, 39, 39, 19, 34, 27, 32, 34, 25, 36, 36, 39 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: CTAACTTTTATGATATTTTGGTGGTAATTTGGTATGGTTAGATTTGTTTGAAAGCCAAATTAGAGTTCTACTTAGATGTCCCCATCATGGAACAGGGAATACTGCTGCTCATGTGTCTTTCTGACAAGAACAGACTAGCAGAACTCATCTGGGAAGTAAGAGTATAGTTTATTAGAAATGATAAGTTGCGCTTAACTGGCTG
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 101, 101
+ READ_SEG: [0, 101], [101, 101]
+ READ_START: 0, 101
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP: GATCAG
+ SPOT_ID: 1
+ SPOT_LEN: 202
+ TRIM_LEN: 202
+ TRIM_START: 0
+
+ ALIGNMENT_COUNT: 0, 0
+ BASE_COUNT: 404
+ BIO_BASE_COUNT: 404
+ CMP_BASE_COUNT: 404
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CSREAD: 1131131133323131133030331132231121330123331222003300110002133123111202120023213022003330321020211202111300303030011213303300332202121220032020312021032011103132222220220023012002233111002332013120311030
+ CS_KEY: TT
+ CS_NATIVE: false
+ FIXED_SPOT_LEN: 202
+ MAX_SPOT_ID: 2
+ MIN_SPOT_ID: 1
+ NAME: 2
+ PLATFORM: SRA_PLATFORM_UNDEFINED
+PRIMARY_ALIGNMENT_ID: 0, 0
+ QUALITY: 33, 33, 31, 35, 37, 37, 37, 37, 38, 39, 38, 39, 39, 40, 40, 39, 37, 39, 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 40, 38, 40, 41, 41, 39, 39, 40, 41, 41, 37, 36, 40, 39, 40, 37, 40, 37, 34, 35, 36, 38, 39, 40, 40, 41, 40, 41, 40, 41, 37, 38, 40, 40, 37, 40, 39, 38, 40, 40, 41, 41, 38, 40, 40, 41, 41, 40, 40, 38, 41, 40, 40, 40, 38, 38, 36, 39, 40, 41, 39, 39, 39, 37, 39, 38, 39, 37, 37, 37, 37, 37, 37, 31, 33, 34, 37, 37, 37, 37, 37, 39, 39, 39, 39, 39, 39, 40, 38, 38 [...]
+ RD_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ: GTACATGTATAGCATGTATTAATACATCTACAGTATTGATATGAGAAATAAACAAAAGTATGATGTGAAGTCCCTAGTAAGAAATATTAGTTCCTGTCCTGGTAAATTAATTTGTCATAATAAATAGAAGTCAGAAATCCTTACTTCAATCCACAATGCTCTCTCCTCCCTAACTTTCTATGTGGGATAGGTACTTACAATT
+ READ_FILTER: SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS
+ READ_LEN: 101, 101
+ READ_SEG: [0, 101], [101, 101]
+ READ_START: 0, 101
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_BIOLOGICAL
+ SIGNAL_LEN: 0
+ SPOT_COUNT: 2
+ SPOT_GROUP: GATCAG
+ SPOT_ID: 2
+ SPOT_LEN: 202
+ TRIM_LEN: 202
+ TRIM_START: 0
+
diff --git a/test/fastq-loader/input/1.1.fastq b/test/fastq-loader/input/1.1.fastq
new file mode 100644
index 0000000..069381b
--- /dev/null
+++ b/test/fastq-loader/input/1.1.fastq
@@ -0,0 +1,5 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+&.<77478889998776776:997974354774779
+
diff --git a/test/fastq-loader/input/1.2.fastq b/test/fastq-loader/input/1.2.fastq
new file mode 100644
index 0000000..9f7e6b7
--- /dev/null
+++ b/test/fastq-loader/input/1.2.fastq
@@ -0,0 +1,5 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+YYYYYYYYYYYYYYYYPYYVYYYYYYYYYYYYYYYY
+
diff --git a/test/fastq-loader/input/1.3.fastq b/test/fastq-loader/input/1.3.fastq
new file mode 100644
index 0000000..39145e2
--- /dev/null
+++ b/test/fastq-loader/input/1.3.fastq
@@ -0,0 +1,5 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+41 42 43 44 45 46 47 48 49 40 41 42 43 44 45 46 47 48 49 40 41 42 43 44 45 46 47 48 49 40 41 42 43 44 45 46
+
diff --git a/test/fastq-loader/input/1.4.fastq b/test/fastq-loader/input/1.4.fastq
new file mode 100644
index 0000000..371fc95
--- /dev/null
+++ b/test/fastq-loader/input/1.4.fastq
@@ -0,0 +1,24 @@
+ at ID57_120908_30E4FAAXX:3:1:1772:953/1
+GCAATAGCGCGTTGCCTAATCTAACGACGATGATTG
++ID57_120908_30E4FAAXX:3:1:1772:953
+hhhhhhhhhhhhhhhhhhhhhhbdhhhhhhDhGhhh
+ at ID57_120908_30E4FAAXX:3:1:1772:953/2
+ATGGCTCAGGGCCAGGCCCAGAGCCCTTGCGATCAG
++ID57_120908_30E4FAAXX:3:1:1772:953
+hhhhhhhhhhhhhahhhhhhhMhhhLhhhhh<hh at h
+ at ID57_120908_30E4FAAXX:3:1:1590:999/1
+GTTTTATCGGCAGCCGTCTGCTTTCGCGCCTAAGCC
++ID57_120908_30E4FAAXX:3:1:1590:999
+hhhhhhhhhhhhh[hhhhhNXhhH=Ah`LhM??>hB
+ at ID57_120908_30E4FAAXX:3:1:1590:999/2
+TAAGCAGCGCCTGCGCAGTCGCTGCTTAGTCGCTTA
++ID57_120908_30E4FAAXX:3:1:1590:999
+hhhhhhhhhhhhhhhhhhhhhhhhhbhKhhhhhhhC
+ at ID57_120908_30E4FAAXX:3:1:1701:982/1
+GAGCAGTACGAGCAGTACCTGTTTTTCATTGGAGAT
++ID57_120908_30E4FAAXX:3:1:1701:982
+hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhXhhh
+ at ID57_120908_30E4FAAXX:3:1:1701:982/2
+GGTCGCTGACGATGTGCGGAACCAACCGCTCCCGTC
++ID57_120908_30E4FAAXX:3:1:1701:982
+hhhhhhhhhhhhhhhhhhhhhhhOUhhhhhhhIh[h
diff --git a/test/fastq-loader/input/10.0.fastq b/test/fastq-loader/input/10.0.fastq
new file mode 100644
index 0000000..e7efa74
--- /dev/null
+++ b/test/fastq-loader/input/10.0.fastq
@@ -0,0 +1,4 @@
+ at HWI-ST1374:1:1101:1161:2060#0/1
+CCTACACTTGTATTTTGGTTA
++HWI-ST1374:1:1101:1161:2060#0/1
+bb~eeeeegggcgiihhcegg
diff --git a/test/fastq-loader/input/11.0.1.fasta b/test/fastq-loader/input/11.0.1.fasta
new file mode 100644
index 0000000..67e5df2
--- /dev/null
+++ b/test/fastq-loader/input/11.0.1.fasta
@@ -0,0 +1,2 @@
+ at SRR390728.1.1
+GAGCAATAATGTCTATTTGGTTTTTTTTTTTTTTTTTTTTTTTTTTTCCGCCAATACACCCCACAACAAGTAACAC
diff --git a/test/fastq-loader/input/11.0.2.fasta b/test/fastq-loader/input/11.0.2.fasta
new file mode 100644
index 0000000..45efbc3
--- /dev/null
+++ b/test/fastq-loader/input/11.0.2.fasta
@@ -0,0 +1,2 @@
+ at SRR390728.1.2
+GCCACTACATAGGATGTATTTTTGCAGCAACAAGAAGAAGGGGGGGGGGGGGGGTAAAACAAAAATACAGCGGGCG
diff --git a/test/fastq-loader/input/11.1.1.fastq b/test/fastq-loader/input/11.1.1.fastq
new file mode 100644
index 0000000..316d9b6
--- /dev/null
+++ b/test/fastq-loader/input/11.1.1.fastq
@@ -0,0 +1,4 @@
+ at SRR493912.15083.1 15083 length=101
+TAAAGCTTTCATTTCGTTCCCTCCATTAACTGTCATACACCAATATATTATCATAATCCTTATATTAAAATGATTGATTTTATGTAAATTATTTATAAATT
++SRR493912.15083.1 15083 length=101
+CCCFFFFFHHGHGJJJJJJJIIJJGJJIJGIHHIJFBHGIIJGFE at BHIIECEHIIGIGIHHIIJIIIJGEECHHF@EGHGDHGEHFEEHHFFFDFFFEE@
\ No newline at end of file
diff --git a/test/fastq-loader/input/11.1.2.fastq b/test/fastq-loader/input/11.1.2.fastq
new file mode 100644
index 0000000..a95e6a4
--- /dev/null
+++ b/test/fastq-loader/input/11.1.2.fastq
@@ -0,0 +1,4 @@
+ at SRR493912.15083.2 15083 length=101
+TAAAGCTTTCATTTCGTTCCCTCCATTAACTGTCATACACCAATATATTATCATAATCCTTATATTAAAATGATTGATTTTATGTAAATTATTTATAAATT
++SRR493912.15083.2 15083 length=101
+CCCFFFFFHHGHGJJJJJJJIIJJGJJIJGIHHIJFBHGIIJGFE at BHIIECEHIIGIGIHHIIJIIIJGEECHHF@EGHGDHGEHFEEHHFFFDFFFEE@
\ No newline at end of file
diff --git a/test/fastq-loader/input/12.0.fastq b/test/fastq-loader/input/12.0.fastq
new file mode 100644
index 0000000..b716dab
--- /dev/null
+++ b/test/fastq-loader/input/12.0.fastq
@@ -0,0 +1,8 @@
+ at HWI-ST1374:1:1101:1161:2060#SG1/1
+CCTACACTTGTATTTTGGTTA
++HWI-ST1374:1:1101:1161:2060#SG1/1
+bb~eeeeegggcgiihhcegg
+ at HWI-ST1375:1:1101:1161:2060#SG2/1
+CCTACACTTGTATTTTGGTTA
++HWI-ST1375:1:1101:1161:2060#SG2/1
+bb~eeeeegggcgiihhcegg
diff --git a/test/fastq-loader/input/12.1.fastq b/test/fastq-loader/input/12.1.fastq
new file mode 100644
index 0000000..b716dab
--- /dev/null
+++ b/test/fastq-loader/input/12.1.fastq
@@ -0,0 +1,8 @@
+ at HWI-ST1374:1:1101:1161:2060#SG1/1
+CCTACACTTGTATTTTGGTTA
++HWI-ST1374:1:1101:1161:2060#SG1/1
+bb~eeeeegggcgiihhcegg
+ at HWI-ST1375:1:1101:1161:2060#SG2/1
+CCTACACTTGTATTTTGGTTA
++HWI-ST1375:1:1101:1161:2060#SG2/1
+bb~eeeeegggcgiihhcegg
diff --git a/test/fastq-loader/input/12.2.fastq b/test/fastq-loader/input/12.2.fastq
new file mode 100644
index 0000000..c040978
--- /dev/null
+++ b/test/fastq-loader/input/12.2.fastq
@@ -0,0 +1,4 @@
+ at HWI-ST1234:33:D1019ACXX:2:1101:1415:2223/1 1:N:0:ATCACG
+CCTACACTTGTATTTTGGTTA
+ at HWI-ST1234:33:D1019ACXX:2:1101:1415:2223/2 2:N:0:ATCACG
+CCTACACTTGTATTTTGGTTA
diff --git a/test/fastq-loader/input/2.1.1.fastq b/test/fastq-loader/input/2.1.1.fastq
new file mode 100644
index 0000000..ba0f52e
--- /dev/null
+++ b/test/fastq-loader/input/2.1.1.fastq
@@ -0,0 +1,5 @@
+ at R16:8:1:0:875#0/1
+T123123
+ at R16:8:1:0:14#0/1
+G123123
+
diff --git a/test/fastq-loader/input/2.1.fastq b/test/fastq-loader/input/2.1.fastq
new file mode 100644
index 0000000..d3cc85b
--- /dev/null
+++ b/test/fastq-loader/input/2.1.fastq
@@ -0,0 +1,4 @@
+ at R16:8:1:0:875#0/1
+TGGAATAACCATTCCTCATGCCTAGAAGAAGAATCTCTACAAGCAGGATTCTTGCTTTCAAAATGCCCTTATAA
+ at R16:8:1:0:14#0/1
+TTGCAACTATAGCAACAGCCTTCATAGGCTATGTCCTCCCGTGAGGCCAAATATCATTCTGAGGGGCCACAGTA
diff --git a/test/fastq-loader/input/2.2.fastq b/test/fastq-loader/input/2.2.fastq
new file mode 100644
index 0000000..9f59a38
--- /dev/null
+++ b/test/fastq-loader/input/2.2.fastq
@@ -0,0 +1,4 @@
+>HWI-EAS6_4_FC2010T:1:1:80:366
+BBBBBBBBBBBBBBBBBBBB
+>HWI-EAS6_4_FC2010T:1:1:212:230
+CCCCCCCCCCCCCCCCCCCC
\ No newline at end of file
diff --git a/test/fastq-loader/input/2.3.fastq b/test/fastq-loader/input/2.3.fastq
new file mode 100644
index 0000000..878d296
--- /dev/null
+++ b/test/fastq-loader/input/2.3.fastq
@@ -0,0 +1,8 @@
+>HWI-EAS6_4_FC2010T:1:1:80:366
+DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD
+>HWI-EAS6_4_FC2010T:1:1:212:230
+EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE
+>HWI-EAS6_4_FC2010T:1:1:80:366
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
+>HWI-EAS6_4_FC2010T:1:1:212:230
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
\ No newline at end of file
diff --git a/test/fastq-loader/input/2.4.fastq b/test/fastq-loader/input/2.4.fastq
new file mode 100644
index 0000000..13816d0
--- /dev/null
+++ b/test/fastq-loader/input/2.4.fastq
@@ -0,0 +1,8 @@
+>HWI-EAS6_4_FC2010T:1:1:80:366
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
+>HWI-EAS6_4_FC2010T:1:1:212:230
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
+>HWI-EAS6_4_FC2010T:1:1:80:366
+DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD
+>HWI-EAS6_4_FC2010T:1:1:212:230
+EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE
diff --git a/test/fastq-loader/input/2.5.fastq b/test/fastq-loader/input/2.5.fastq
new file mode 100644
index 0000000..28c749a
--- /dev/null
+++ b/test/fastq-loader/input/2.5.fastq
@@ -0,0 +1,9 @@
+ at HWI-EAS6_4_FC2010T:1:1:80:366
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
++HWI-EAS6_4_FC2010T:1:1:80:366
+CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
+ at HWI-EAS6_4_FC2010T:1:1:212:230
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
++HWI-EAS6_4_FC2010T:1:1:212:230
+DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD
+
diff --git a/test/fastq-loader/input/2.6.fastq b/test/fastq-loader/input/2.6.fastq
new file mode 100644
index 0000000..ee402b2
--- /dev/null
+++ b/test/fastq-loader/input/2.6.fastq
@@ -0,0 +1,9 @@
+ at HWI-EAS6_4_FC2010T:1:1:80:366/1
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
++HWI-EAS6_4_FC2010T:1:1:80:366
+EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE
+ at HWI-EAS6_4_FC2010T:1:1:80:366/2
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
++HWI-EAS6_4_FC2010T:1:1:212:230
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+
diff --git a/test/fastq-loader/input/2.7.fastq b/test/fastq-loader/input/2.7.fastq
new file mode 100644
index 0000000..70c76bc
--- /dev/null
+++ b/test/fastq-loader/input/2.7.fastq
@@ -0,0 +1,8 @@
+ at HWI-EAS6_4_FC2010T:1:1:80:366
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
++HWI-EAS6_4_FC2010T:1:1:80:366
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+ at HWI-EAS6_4_FC2010T:1:1:80:366
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
++HWI-EAS6_4_FC2010T:1:1:212:230
+GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG
diff --git a/test/fastq-loader/input/2.8.1.fastq b/test/fastq-loader/input/2.8.1.fastq
new file mode 100644
index 0000000..2a1cc02
--- /dev/null
+++ b/test/fastq-loader/input/2.8.1.fastq
@@ -0,0 +1,4 @@
+ at GG3IVWD03HJRKK
+NTA
+ at GG3IVWD03HJRKZ
+NNT
diff --git a/test/fastq-loader/input/2.8.fastq b/test/fastq-loader/input/2.8.fastq
new file mode 100644
index 0000000..1384e06
--- /dev/null
+++ b/test/fastq-loader/input/2.8.fastq
@@ -0,0 +1,4 @@
+ at GG3IVWD03HJRKK
+NTA
+ at GG3IVWD03HJRKZ
+NTA
diff --git a/test/fastq-loader/input/2.9.1.fastq b/test/fastq-loader/input/2.9.1.fastq
new file mode 100644
index 0000000..36abd6f
--- /dev/null
+++ b/test/fastq-loader/input/2.9.1.fastq
@@ -0,0 +1,4 @@
+ at BILLIEHOLIDAY
+GTCG
++BILLIEHOLIDAY
+&.&.&.&.
diff --git a/test/fastq-loader/input/2.9.fastq b/test/fastq-loader/input/2.9.fastq
new file mode 100644
index 0000000..b4c6e95
--- /dev/null
+++ b/test/fastq-loader/input/2.9.fastq
@@ -0,0 +1,4 @@
+ at BILLIEHOLIDAY
+GTCG
++BILLIEHOLIDAY
+&.
diff --git a/test/fastq-loader/input/3.1a.fastq b/test/fastq-loader/input/3.1a.fastq
new file mode 100644
index 0000000..260fcb8
--- /dev/null
+++ b/test/fastq-loader/input/3.1a.fastq
@@ -0,0 +1,8 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:118:250
+GAGCACGAGGACCCANNNATCAACGACNANNNGANN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:118:250
+IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII
diff --git a/test/fastq-loader/input/3.1b.fastq b/test/fastq-loader/input/3.1b.fastq
new file mode 100644
index 0000000..2d0d2b9
--- /dev/null
+++ b/test/fastq-loader/input/3.1b.fastq
@@ -0,0 +1,9 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:212:211
+ACTGGAGCACGANAGNCCCNCNACTNANANCNGNNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:212:211
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:182:340
+CTGGAGCACGAGAACNCNGCCCGGACANANGNGCNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:182:340
+GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG
+
diff --git a/test/fastq-loader/input/3.2a.fastq b/test/fastq-loader/input/3.2a.fastq
new file mode 100644
index 0000000..1c6c81d
--- /dev/null
+++ b/test/fastq-loader/input/3.2a.fastq
@@ -0,0 +1,8 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:212:211
+ACTGGAGCACGANAGNCCCNCNACTNANANCNGNNN
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:182:340
+CTGGAGCACGAGAACNCNGCCCGGACANANGNGCNN
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:118:250
+GAGCACGAGGACCCANNNATCAACGACNANNNGANN
diff --git a/test/fastq-loader/input/3.2b.fastq b/test/fastq-loader/input/3.2b.fastq
new file mode 100644
index 0000000..166cc59
--- /dev/null
+++ b/test/fastq-loader/input/3.2b.fastq
@@ -0,0 +1,8 @@
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:212:211
+40 40 40 40 40 27 40 40 12 8 5 6 -5 -3 0 -5 -4 0 4 -5 7 -5 7 1 0 -5 -0 -5 -1 -5 -0 -5 7 -5 -5 -5
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:182:340
+40 40 40 40 40 40 40 23 40 40 9 22 -2 14 9 -5 7 -5 0 -2 3 9 -1 0 7 3 0 -5 8 -5 -3 -5 10 2 -5 -5
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+40 3 1 22 17 18 34 8 13 21 -3 7 5 0 -0 -5 1 -0 7 3 2 -3 -3 3 -1 -1 4 -5 5 -2 -2 -5 -0 -1 -5 -5
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:118:250
+40 40 40 40 40 40 40 22 33 40 16 29 4 -1 1 -5 -5 -5 7 -3 1 -1 -1 6 -1 14 -1 -5 -2 -5 -5 -5 6 -1 -5 -5
diff --git a/test/fastq-loader/input/3.3a.fastq b/test/fastq-loader/input/3.3a.fastq
new file mode 100644
index 0000000..166cc59
--- /dev/null
+++ b/test/fastq-loader/input/3.3a.fastq
@@ -0,0 +1,8 @@
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:212:211
+40 40 40 40 40 27 40 40 12 8 5 6 -5 -3 0 -5 -4 0 4 -5 7 -5 7 1 0 -5 -0 -5 -1 -5 -0 -5 7 -5 -5 -5
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:182:340
+40 40 40 40 40 40 40 23 40 40 9 22 -2 14 9 -5 7 -5 0 -2 3 9 -1 0 7 3 0 -5 8 -5 -3 -5 10 2 -5 -5
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+40 3 1 22 17 18 34 8 13 21 -3 7 5 0 -0 -5 1 -0 7 3 2 -3 -3 3 -1 -1 4 -5 5 -2 -2 -5 -0 -1 -5 -5
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:118:250
+40 40 40 40 40 40 40 22 33 40 16 29 4 -1 1 -5 -5 -5 7 -3 1 -1 -1 6 -1 14 -1 -5 -2 -5 -5 -5 6 -1 -5 -5
diff --git a/test/fastq-loader/input/3.3b.fastq b/test/fastq-loader/input/3.3b.fastq
new file mode 100644
index 0000000..1c6c81d
--- /dev/null
+++ b/test/fastq-loader/input/3.3b.fastq
@@ -0,0 +1,8 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:212:211
+ACTGGAGCACGANAGNCCCNCNACTNANANCNGNNN
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:182:340
+CTGGAGCACGAGAACNCNGCCCGGACANANGNGCNN
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:118:250
+GAGCACGAGGACCCANNNATCAACGACNANNNGANN
diff --git a/test/fastq-loader/input/4.4.fastq b/test/fastq-loader/input/4.4.fastq
new file mode 100644
index 0000000..f53b4ad
--- /dev/null
+++ b/test/fastq-loader/input/4.4.fastq
@@ -0,0 +1,13 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+G12312312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC21F3DAAXX:8:2:342:540/1
+G12312312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC22F3DAAXX:8:2:342:540/1
+G12312312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC23F3DAAXX:8:2:342:540/1
+G12312312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC24F3DAAXX:8:2:342:540/1
+G12312312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC25F3DAAXX:8:2:342:540/1
+G12312312312312312312312312312312312
+
diff --git a/test/fastq-loader/input/4.5.1.fastq b/test/fastq-loader/input/4.5.1.fastq
new file mode 100644
index 0000000..2c5c331
--- /dev/null
+++ b/test/fastq-loader/input/4.5.1.fastq
@@ -0,0 +1,10 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX
+G12312312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC21F3DAAXX
+G12112312312312312312 312312312312312
+ at BILLIEHOLIDAY_1_FC22F3DAAXX
+G1232231231231231231 2312312312312312
+ at BILLIEHOLIDAY_1_FC23F3DAAXX
+G12313312312312312312312312312312312
+ at BILLIEHOLIDAY_1_FC24F3DAAXX
+G12312112312312312312312312312312312
diff --git a/test/fastq-loader/input/4.5.fastq b/test/fastq-loader/input/4.5.fastq
new file mode 100644
index 0000000..3781903
--- /dev/null
+++ b/test/fastq-loader/input/4.5.fastq
@@ -0,0 +1,10 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+G1231231231231231 2312312312312312312
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+G12112312312312312 312312312312312312
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+G123223123123123123 12312312312312312
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+G1231331231231231231 2312312312312312
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+G12312112312312312312 312312312312312
diff --git a/test/fastq-loader/input/4.fastq b/test/fastq-loader/input/4.fastq
new file mode 100644
index 0000000..069381b
--- /dev/null
+++ b/test/fastq-loader/input/4.fastq
@@ -0,0 +1,5 @@
+ at BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540/1
+GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN
++BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540
+&.<77478889998776776:997974354774779
+
diff --git a/test/fastq-loader/input/5.0.fastq.gz b/test/fastq-loader/input/5.0.fastq.gz
new file mode 100644
index 0000000..7c236e5
Binary files /dev/null and b/test/fastq-loader/input/5.0.fastq.gz differ
diff --git a/test/fastq-loader/input/6.0.fastq b/test/fastq-loader/input/6.0.fastq
new file mode 100644
index 0000000..1a065e7
--- /dev/null
+++ b/test/fastq-loader/input/6.0.fastq
@@ -0,0 +1,52 @@
+ at GG3IVWD03HJRKK length=290 xy=2978_1730 region=3 run=R_2010_05_11_11_15_22_
+ATATACATTTGTCTGTACAGCATGGAATTCTCATACTCTAATCAATATAAAATAATGATTGATTATTCAAGAAAATAAATTGTCAGACAAATGAAAAATAATCTTAAGCCACAGCCCTAAAATACCGTAGATACCATGAGACAAAAAACACTGCTTCCACGAGTCAGGAGCAACTCCCATTATTTTACATTTTATCACCGCCGTCCATTTCTCTTCAGTCTTTTTTACTCTTTTGATTTATTTATTTTACACAAAACAATTATTTATAATTGATTTATTCAGTTTATTTA
++
+89:HHA:/1192DCHHIIEIIIIGGGGIIIIIIIIIIIIIIIIIIIA95553A8<EHHGIIIIGEEAHD;71111;///AADBGIIIIEDDCC:::::I77DIHHIIIDDDGID556===@@H?IIIIIIIIIIIIIIIIII669997B2BBIIIIIIIIIIIIIIIIIII;;;;;;HIIIE;;;;IDA444482DE>>@<?IGHHI>>>B>IIIGGGGG677722HI1=53117;@B=;//1=8;=:DDCD>>@@G<;>>=BBEIIIIDABB=>9;00750,,,51411
+ at GG3IVWD03GMLDS length=238 xy=2600_2350 region=3 run=R_2010_05_11_11_15_22_
+TGAAGCCAAGAAGCCACAGCAACATCATGTTTGAGGAGAAGAAGAGCTCCTAAGACGGAGGCTTCCTGTCGGCTGTTCCCTCTGGAGAAGAAGAAGTGACGTTGCCTTGCGAATGTTAGTTTGTAAGAGAACCTGTTCAGTTCGTAACCTTAATCCTCAGCCGACCTGTATTAGACACGTGTCTCCCTCCATGTATGAACTGAAGCCATCCTCCAGGTGAGTCGCTCACTCTGACCCA
++
+FFEEEFFFFFFIIIIIIIHF;;;HHHHHH@@@IIIIIIIIIIIIIIIIIDFDFFFFFFFFFFFFFFFFFFFFFFFDD?::99B566:=ABDDDDDD???DDA==:::::9<==>>=?<111187>><<<=>>899>>>900048888<<<<@<888<<<?AA8555::DAAADDDFFFDCCFFFFFC<99<?FFFFDDDACCFD???AADA???AAADFDDDDDDBBB??==772212
+ at GG3IVWD03GFQBB length=1 xy=2522_1637 region=3 run=R_2010_05_11_11_15_22_
+N
++
+!
+ at GG3IVWD03F2Q8L length=151 xy=2374_2515 region=3 run=R_2010_05_11_11_15_22_
+AGCGGGAGAGGGGCACATGGTGTGAGGGTCCCAGGATGAACACCCTACGAGTAATAGAGGGATCAACCTGCTTGTAGGCATGTGCCGCACAAGAACCACAATAGGTATACCCAGCATGCGGCGCAATGATGGCTCTAGCAGGTTTGATCGT
++
+HII===EBH>>>>EEIIIIIIIIIIHHHIHHHIIIIIIIIIIHHHIIIIIIIIIIIIIHHHIIIIIIIIIIIIIIIIIIIIIEBBADGGG<<>AAAAADDEEHHHHIIIEDFFIIIEEEEIGBB===BIIIGED==AAA?/////2:=::8
+ at GG3IVWD03G7J0C length=245 xy=2839_1402 region=3 run=R_2010_05_11_11_15_22_
+CCTCGTCGCTTCACTTTTCATGGGCTTCGGAGTGCTGTTCCTGCTACTTTGGGTTGGAATATACGTTTGAAGCTTTGGGTGACTGACTTATAATATGACACATTCCAGTCGAACAAAGGACATCACACACAACACCAACTGGATTCTGTACTCAATTGGACTGCACTGACAATATTTTGAGATTGTATGGAGTCTGGTTTTGTTTCACAATTGAAGATGATTTGTAAACGAAAAAANAAAAAGTA
++
+IIIIIIIIIIIIII;;;;IIIHHHIIIIIIIIIIIIIIIIIIIIIIIHHHHHHIIIIIIIIIIID;;;HIIIIHHHHHHIIIIIIIIIIIIIIIIIIIIIII;;;;IIIIIIIIHHHIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIICCCCIIIIIIIIIIIIIIH@@BBIIIIIIIIDC>>==FB=22---0/,--5=B==@@@EIIEDDD>>>9122229666624!/3333IH8
+ at GG3IVWD03G2V2N length=1 xy=2786_0845 region=3 run=R_2010_05_11_11_15_22_
+N
++
+!
+ at GG3IVWD03HKKWM length=288 xy=2987_2884 region=3 run=R_2010_05_11_11_15_22_
+CTAGGGCTGGGCAATATGACCTAAAATGTATAACCTCGGTATAATTTGGAGCACAAACGGTAACGGTACAAATCACGGTATTTTTGATGGTATACGTCCACCCAAGGCAAGCGTACCGCTGTGCAAATGGACCGTCGAGGGTTCCTAGTTTGTGTTGGAGATGAAATAGCTGCGCTGTAAACGAGTCGTCGTGGATCTTTTGGTCTTCTTTTCTTTGGACGTGTGTTCGCGGGTCTGTGCGGGTCGAACAAGCTAGCGCTAGCCTCGGCTCCGCGGAGCTGCGATGGT
++
+IIIHHHIIHHHIIIIIIIIIIICCCCIIIIIIIIIIIIIIIIIIHHHIIIIIIIHHHIIIIIIIIIIIIHHHIIIIIIIH66666DHIIIIIIIIIIIIIGGGIIIIIIIIIIIIIIIIIIIIIHHHIIIIIIIIIHH;;;??HHIIIHHHIIIIIIIIIIII???IIIIIIIIIIIIHHHIIIIIIIIIIIIIIID><97<95B=52////8///@@EIIIIIIIIIIIHHHIIIIIH??;;;HIIIIIIIIIIIIIIIIIIIIHHHHHHIIIIIIIIIIIIIIIII
+ at GG3IVWD03G0YS1 length=141 xy=2764_1187 region=3 run=R_2010_05_11_11_15_22_
+ACTATCATTATTTGCTGCATGCACAATGCCAAAGAATAAAATGCTGTCTGTCAATGAATGAGGCGTCTAATCCGCACTGACCTCAACTTTTCCACTGAGGATTTGCAGAGGTGTGTGTGTGTGTGTGTGTGTGTGTGTCGA
++
+FFFFFFFFFFFHHIIIIIIIHHGFDDDDAA663:9AB9::;DFFFFFFFFFFFFFFFFFFFCDDFFFFDDDFFFFFFFDBAAAAA<8111144AAA?AABFFFDDDDDFFFFFFFFFFFFFFFFFFFFFFDBA??=6643/
+ at GG3IVWD03FQOVY length=1 xy=2237_0748 region=3 run=R_2010_05_11_11_15_22_
+A
++
+/
+ at GG3IVWD03HD8KU length=250 xy=2915_1884 region=3 run=R_2010_05_11_11_15_22_
+GTGCAAGAGGCTGAGGCTAACAAAGTGCATCAACCTATGTGGAAGCATCGAAGGCCCGCGTGGCTCAGTATGAGCAAGAGCTGAACAAGTTGAGGAACATGATCCCCTTTGTATCAGATGACCATTGAAGACCTCAACGAAGTCTTCCCCGACGACCAAGTTGGACAAGGTCAAATAATCCCTACCTGGCCCCACCAAGCCCACTTGCTGATATGTAACCCCGTTACCAGACTCTCACCTCACAATAAAT
++
+FFFFFFFFFFFIIIIIIIIHH666BFHIIIIIIIIIIIIIIIIIIIIIIIIIFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFDDDCFB@?>>4222.../-4377:9777996664223266:666<:333779;222244.3322667;;../426;;44...4//<66357//366.0..415;;=66440449999:?<==AAA7766@>41//14=998=?A at A:9:<>>A;;84
+ at GG3IVWD03GXKBC length=77 xy=2725_2182 region=3 run=R_2010_05_11_11_15_22_
+ATACATATCTGCTGACAGAGCCTGGCTCTGTTGGCTGTAGAGTCCCTGCTGAGCCAGGCTCTGTCAGCAGATATGTA
++
+IIIIIIIIIIIIIIIIIIIHHHHCCGGIIIIIIIIIIIIIIIIHHGHHIIIIIIIIIIIIIIIIIIIIIIIIIIIII
+ at GG3IVWD03HIDOA length=3 xy=2962_2600 region=3 run=R_2010_05_11_11_15_22_
+AAT
++
+111
+ at GG3IVWD03HB687 length=306 xy=2892_1065 region=3 run=R_2010_05_11_11_15_22_
+TCCTGTTTGAAGACCTTACTCTTGGCGTGGTCGTCACAGTAGCATGCCTTACAGCGCAGGCAGGAGTGTTGTCCCAGTCTGTTACAGGACACACACTTGTATGTTTCTGCCTGGAGGATTTGGCAGCTTGCCTGGTGCTCAAACTGATCATCCTCACACAAGAAGTTTTGGCAGAAAGAGCAGCGGAAAATTCGCCCTCCATGATCCCAGACACTACGCTCACACTCAATGCAGTCTGCATCCGTTAGGGGACATGCACAGGCGTGAGTGCTCAGACATTTTCCTCCCAAGGCACACAGGGGGATA
++
+IIII;666??HIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIHHHIIIIIIIIIIIIIIIIIIIIIHHHIIIIIIIIIIIIIIIIIIIIIIIIIIIIHHHIIIIIIIIIIIIHHHIIIIIIIIIIIIIIIIIIIHHHIIIHHHIHHHHHIIIIIIIIIICCBBIIIII;;;E<EAGGI=;4442@@HE774=7:EEEE?455IIIIIIIIIIIIIIIIIIDCCDDIIIIIIIIIIIIIIIH??>>?DIIIIIIIC????HIIIEA?AB=;=;00--..-333....:=;;;8---//;53
diff --git a/test/fastq-loader/input/7.1.fastq b/test/fastq-loader/input/7.1.fastq
new file mode 100644
index 0000000..d79f3ec
--- /dev/null
+++ b/test/fastq-loader/input/7.1.fastq
@@ -0,0 +1,9 @@
+ at m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19/ccs
+AGAGTTTGATCCTGGCTCAGAGTGAACGCTGGCGGCGTGCTTAACACATGCAAGTCGAACGGATGAAATAAGCTTGCTTATTTCGTTAGTGGCGCACGGGTGAGTAATGTATAGCTAACCTGCCCTTTAGTGGGGGACAACAGATGGAAACGTCTGCTAATACCCCATACTCCTGCGTATCATAAGATATGTTGGGAAAGATTTATTGCTAAAGGATGGGGCTTTATGGTATCAGCTAGTTGGTGGGGTAACGGCCTACCAAGGCAATGACGCCTACCTGGTTCTGAGAGGATGATCAGGCACACTGGAAACTGAGACACGGTTCCAGACTCCTACGGGAGGCAGCAGTGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGGAGGATGACGCATTTCGGTGTGTAAACTCCTTTTATAAGGGAAGATAATGACGGTACCTTATGAATAAGCACCGGCTAACTCCGTGTCAGCAGCC [...]
++m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19/ccs
+LLhf>>>>[[HHdggghhhhhhhNLPPhhhGGhNNhffhhSS777__gh]]hhUUhhSSSdaGGhZZZZhhhhHHHOQhhKKKLUUhhe3WWYHHUhhhhhZZhhKJ'bEEbCC^gZKZdhhMMM,,,,^fVVAIV]]=<=hhhNNhXXX at hhhEE]_OOOC8988VhHhXEEE_ffQQgeZZHFHUUTThOO222222_aafdhWWW^^hhRHHHh++++hFFF[hUU??BBhcaabhOOMMLNNNNRggXTTUdeh^hbcQTOOffhLLh55bXUXbII%%Whh]]hXUP\hg99hee..cgghhh111\hhhhggG99@,,\_ddKKQPYYhhhhh=67bbgHHahCCCC>??g111hhh\\\88888DFFEQQWW??Xf]EE888[$$IbecceJJfT`hXJR>>WWWYh^gOMRE>>>?hfb>>>>>hVV&&&eehhg^^^hQOTKhUT[LhhhhKZZKKK--SR&&c=111hdEdWWhcdDEdO9+ [...]
+ at m121205_055009_42163_c100416332550000001523041801151327_s1_p0/25/ccs
+AGAGTTTGATCCTGGCTCAGGACGAACGCTGGCGGTATGCTTAACACATGCAAGTCGAACGAGACGAGATGGAATGACCCTTCGGGGAATGAAATCTTAGAAAGTGGCGAACGGGTGAGTAACGCGTGGGTAACCTGCCTATGGAAAGGAATAGCCTCGGGAAACTGGGAGTAATGCCTTATAATACAGTGAAGTCGCATGGCTTTGCTGTTAAACGCTCCGGTGCCATAGGATGGACCCGCGTCCCATTAGCTAGTTGGTGAGATAACAGCCCACCAAGGCGACGATGGGTAACCGGTCTGAGAGGGCGAACGGTCACACTGGAACTGAGACACGGTCCAGACTCCTACGGAGGCAGCAGTGGGGAATATTGCGCAATGGGCAACCCTGACGCAGCAATACCGCGTGAGTGAAGAGGTTTTCGGATCGTAAAGCTCTGTTATTGGGGAAGAAAAAAGACGGTACCCAAGAAGAAAGTCCCGGCTAACTACG [...]
++m121205_055009_42163_c100416332550000001523041801151327_s1_p0/25/ccs
+heehhhhhhhhhhffhhhhhhhhhhhhhhhQQhgghhhhhhhTThh`hhhhhhhhhhhh_hhhhhhhhhhddhhhhhhhhhhh66668hhhVVVhhhhhhVVVhh\\hhhhh===hhhhhhhhhhhhhbfhhhhhhh333hhhhhhhddhhhhhggghhhhhhhhhZZZhhhhhhh;;ffhhhhhhhhhhh+$%%hhhhhffh]]]]hhhhhhhhhZZggghhhh__ghhgghhhhhdddhZZh999hhhh^^hhh__fghhhhhhhhhhhKKKd00cc==hhhhhhhbbbhhheehehhhhhhhHHHehhhhhhhhhfhhhhUhhhhhhhhhhe??hbbchhhhhhhhh333^^ffhhhhh////gghh44hhhhhhh....ggVVUhhhhhhhYMNchhhbedhhhhhhh==a'&0CCCChhhhhehh]]]]hhhhhggghhFFFFhh^HHHHHHhhMMghhMMMNNhddhDDDhgAAAQQ``hhhhhhh [...]
+
diff --git a/test/fastq-loader/input/7.2.fastq b/test/fastq-loader/input/7.2.fastq
new file mode 100644
index 0000000..6f07408
--- /dev/null
+++ b/test/fastq-loader/input/7.2.fastq
@@ -0,0 +1,8 @@
+ at m101111_134728_richard_c000027022550000000115022502211150_s1_p0/1
+ATCCACCCCTCCCACCCACCCCCCCCCCCCTACCCCCCCTCCCCCACACACCCTCCCCCACACCCCTGTCCCCCCCACCGCCGCCCTCCCGACGCCCACACAAAAAAACCCATCCCACACACCCCCCACCCCCCATCACAACACGCCACCCACCCCCCAGCACACCACCACCCCACAAACCAGCCCCCCCACCCCCCCACCCCACCCCCCACAACAC
++
+'!%!%#!!!!!!%"!!!$"!!!!!!!!!!#""!!!!$!!!#!!!!!!!!!!!!!!$#!"!!!!!!!!!!$'!!!#"!!!!!&!!"$!!!$!!!!!%!!&!!!!!!!!$!!&!%!"!!!!!!&!!$!!#!!!!$!!!!!&!!!!&!!"!!#)!"!!%!!!!!"%!!!"!"!"!!&!)!!#!!!!"$!!!!&!!%!!"!'!!!!$"!!"$!!!!!!!!"
+ at m101111_134728_richard_c000027022550000000115022502211150_s1_p0/2
+CCACACCCCCACCCAACCCCACGGCCCCCAACCACCACCACCAAAACACCCCGCGCCCCACCCCCACACCCCACACGCCACCCGCCACCCGCCACAACGCCCCGACGCGAAACCCACCCCGCGCCCACCAACCCACCACCCCGACACCCACCCCCCGCACCACCCCCCCCGCGCACCGCCCACCCCACAAACGACGCCAACTCCAACCCCCCCCCCGCCACCCAGCCGCCCCCCCCACCAGCCCCCCACCCCGCCCGAGCCCGACGGCCACGCGCCCGCACCCCAGCCGCCCCCAGCCCCCCCCCCCCCCACACAAAAACCAACCCAAAACAAAAACAAACCAAAACAACAAACGCCACCAAAAACAAAAACACCAAAAAAAACACCAACCAAAACAAACAACAACCACTCCACCCCCGCCCCCAACCCCGACCCCCAACAAAACGCACACCACAAAACGCCACCCCGACACACCACGCGCAGCCCCACACA [...]
++
+#!!$!"!!!!!""#!%!#!!!!!$!!!!&!!!$!%$!!!#!!!!!!!"&%$!!!#"!!!!!!%#!"!!!!!!!!!!!!!!!!!!!!!!!&!!'!!!"!!#!!""##!!"!!!!!!#!!!#%!#!!$$#!$$!!!"#"!!$!!!!!!!!#!!!!!$&!!&!$"!!!(!!!!"!!!!!!!!!!"#!!(&!!!!&!'$!!!!$%!!!!!!!!!!!!!!!!!!!!!!!'!!!!!!!!"!!!!#&!!!!!!!!!!!!!!&!#!!!!!!!!!!!"!!!"!$!!#!"!'%!#(!!%!#"!$!'!!!!!"!$""#!!!!"!!!%!!!!!%%!#!!!!$#!!!$!!!"!!!!!!$!!$!!!#"!!!!!!!"!!!!!!!!!!!!!!!!!!!!!#!!!"!!!$!!#"'!"!"'%!!!!!!!#!&!"!!!!!)!!!!!"**!!#!!!!&!!#!!!!"!#!!!!!!!!!!!%%!!!"!!!!(!!!!!!!!!!!!!%!#!!#&'!& [...]
diff --git a/test/fastq-loader/input/7.3.fastq b/test/fastq-loader/input/7.3.fastq
new file mode 100644
index 0000000..2b47bed
--- /dev/null
+++ b/test/fastq-loader/input/7.3.fastq
@@ -0,0 +1,8 @@
+ at m101210_094054_00126_c000028442550000000115022402181134_s1_p0/2 ccs
+GGTAGAGGTCTTCGTCGCTGTCTCCGCTTCAGTCGCCGCCCTCGCCTCTTGCTGTGCGCGCTTCAGCAAGCCGAGTCCTGCGTCGAGAGATCTCCTCTGGTTCTCTTTCGCTTCCGGTCCTGTTCGGGCGCCACTGCTAGAGATTTCTACACTAACTAAGATGGTCTGAGGGATCTCTAGCTACCAGAGTCACACAACAGACGCTACC
++m101210_094054_00126_c000028442550000000115022402181134_s1_p0/2 ccs
+5555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555
+ at m101210_094054_00126_c000028442550000000115022402181134_s1_p0/3 ccs
+GGGACCTCGCGCCGCGACGGAGGAGCCGTCTCTTGAAGAGAGCGCGCGAGAGAGGCGGGACAGGCGAGGGGCCGGCGCGTGACGCGGACGAGTCGCGCGAAGCTCCTCTCATGTACGACGTCAGCAGCGCAGAGCTCCTAGTGCGTCGCACAGAGCACATAGCCTCTCGCTAACCATCCCGACACACACAACGGAGGAGCGGAGAAGATCAGAAGACAGAGAATGGAGAGGGGACGCGAGGCGGCGCTGACAGGAACTGGCTGAGTCGCCGCTGTGCTGATGCGCGCGCGCCGCAGTGGCGCGCGCGCGCTGTTCGCGTGCAGAGGCGAGGCGCGCCGGGCCCGCGTCGTAGCTCTGGCCGACCGCTCTTAGCAGAAGCGAGTCGGCGTCAGAGATCCCATCTCCATTCTCGTCCGCCCAACGTCTAGTCCAACAACGCCAACGGCAGGAGGATGAAACAGAACGCATGAAATGATCGCGTGATGCGGACGA [...]
++m101210_094054_00126_c000028442550000000115022402181134_s1_p0/3 ccs
+555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555 [...]
diff --git a/test/fastq-loader/input/8.0.fastq b/test/fastq-loader/input/8.0.fastq
new file mode 100644
index 0000000..2e3c64e
--- /dev/null
+++ b/test/fastq-loader/input/8.0.fastq
@@ -0,0 +1,9 @@
+ at HWI-EAS6_4_FC2010T:1:1:80:366/1
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
++HWI-EAS6_4_FC2010T:1:1:80:366
+EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE
+ at HWI-EAS6_4_FC2010T:1:1:80:366/3
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
++HWI-EAS6_4_FC2010T:1:1:212:230
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+
diff --git a/test/fastq-loader/input/8.1.fastq b/test/fastq-loader/input/8.1.fastq
new file mode 100644
index 0000000..92c872b
--- /dev/null
+++ b/test/fastq-loader/input/8.1.fastq
@@ -0,0 +1,17 @@
+ at HWI-EAS6_4_FC2010T:1:1:80:366/1
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
++HWI-EAS6_4_FC2010T:1:1:80:366
+EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE
+ at HWI-EAS6_4_FC2010T:1:1:80:366/3
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
++HWI-EAS6_4_FC2010T:1:1:212:230
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+ at HWI-EAS6_5_FC2010T:1:1:80:366/1
+AACTGATCAACCTGCGGATCGAACGCAAGCGCA
++HWI-EAS6_5_FC2010T:1:1:80:366
+EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE
+ at HWI-EAS6_5_FC2010T:1:1:80:366/2
+ACAGGCTCATGCTGCTACCCTCGGGGCGAGGCC
++HWI-EAS6_5_FC2010T:1:1:212:230
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+
diff --git a/test/fastq-loader/input/9.0.fastq b/test/fastq-loader/input/9.0.fastq
new file mode 100644
index 0000000..5e3ded4
--- /dev/null
+++ b/test/fastq-loader/input/9.0.fastq
@@ -0,0 +1,16 @@
+ at DG7PMJN1:293:D12THACXX:2:1101:1161:1968_1:N:0:GATCAG
+CTAACTTTTATGATATTTTGGTGGTAATTTGGTATGGTTAGATTTGTTTGAAAGCCAAATTAGAGTTCTACTTAGATGTCCCCATCATGGAACAGGGAATA
++
+??<D;DDDFAC8,C:CFGIBG+CC)3<CFH>C1*:EFDBC?*1BGF0BF@*9DC>B)??CCBC:D784@=FEE at EA:)7))=?9?7@@BC:>;;3=@BBBB
+ at DG7PMJN1:293:D12THACXX:2:1101:1275:1946_1:N:0:GATCAG
+GTACATGTATAGCATGTATTAATACATCTACAGTATTGATATGAGAAATAAACAAAAGTATGATGTGAAGTCCCTAGTAAGAAATATTAGTTCCTGTCCTG
++
+BB at DFFFFGHGHHIIHFHIIIJJJJJJJIGIJJHHIJJFEIHIFIFCDEGHIIJIJIJFGIIFIHGIIJJGIIJJIIGJIIIGGEHIJHHHFHGHFFFFFF
+ at DG7PMJN1:293:D12THACXX:2:1101:1161:1968_2:N:0:GATCAG
+CTGCTGCTCATGTGTCTTTCTGACAAGAACAGACTAGCAGAACTCATCTGGGAAGTAAGAGTATAGTTTATTAGAAATGATAAGTTGCGCTTAACTGGCTG
++
+ at B@F?DAEHH4C<AC:EEH>CIFHIGGG>EGGHCFEGGGGCEGHIJJGGGGEBGIHG>?FHB<E<C at EGGGEGIF>HAGGEFCHH?E3593>@ACCDCBC?
+ at DG7PMJN1:293:D12THACXX:2:1101:1275:1946_2:N:0:GATCAG
+GTAAATTAATTTGTCATAATAAATAGAAGTCAGAAATCCTTACTTCAATCCACAATGCTCTCTCCTCCCTAACTTTCTATGTGGGATAGGTACTTACAATT
++
+ at BCFFFFFHHHHHHIGGIIJJIIJJIJIJHGIHIIIJGGHIGHIIJJIJGIIJJJJJJJJJJIJGIJJIJIHJJJJJJIJIEHGHCEFDFFFEEFEEEDDD
diff --git a/test/fastq-loader/runtestcase.sh b/test/fastq-loader/runtestcase.sh
new file mode 100755
index 0000000..d5e4a98
--- /dev/null
+++ b/test/fastq-loader/runtestcase.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+#echo "$0 $*"
+
+# $1 - path to vdb tools (fastq-load.3, vdb-dump)
+# $2 - work directory (expected results under expected/, actual results and temporaries created under actual/)
+# $3 - test case ID
+# $3 - expected result code from fastq-load.3
+# $4, $5, ... - command line options for fastq-load.3
+#
+# return codes:
+# 0 - passed
+# 1 - coud not create temp dir
+# 2 - unexpected return code from fastq-load.3
+# 3 - vdb-dump failed on the output of fastq-load.3
+# 4 - outputs differ
+
+BINDIR=$1
+WORKDIR=$2
+CASEID=$3
+RC=$4
+shift 4
+CMDLINE=$*
+
+DUMP="$BINDIR/vdb-dump"
+LOAD="$BINDIR/latf-load"
+TEMPDIR=$WORKDIR/actual/$CASEID
+
+echo "running $CASEID"
+
+mkdir -p $TEMPDIR
+rm -rf $TEMPDIR/*
+if [ "$?" != "0" ] ; then
+ exit 1
+fi
+export LD_LIBRARY_PATH=$BINDIR/../lib;
+
+#CMD="$LOAD $CMDLINE -o $TEMPDIR/obj --no-user-settings 1>$TEMPDIR/load.stdout 2>$TEMPDIR/load.stderr"
+CMD="$LOAD $CMDLINE -o $TEMPDIR/obj 1>$TEMPDIR/load.stdout 2>$TEMPDIR/load.stderr"
+# echo $CMD
+eval $CMD
+rc="$?"
+if [ "$rc" != "$RC" ] ; then
+ echo "$LOAD returned $rc, expected $RC"
+ echo "command executed:"
+ echo $CMD
+ cat $TEMPDIR/load.stderr
+ exit 2
+fi
+
+if [ "$rc" == "0" ] ; then
+ CMD="$DUMP $TEMPDIR/obj 1>$TEMPDIR/dump.stdout 2>$TEMPDIR/dump.stderr"
+ eval $CMD
+ rc="$?"
+ if [ "$rc" != "0" ] ; then
+ echo "command executed:"
+ echo $CMD
+ cat $TEMPDIR/dump.stderr
+ exit 3
+ fi
+ diff $WORKDIR/expected/$CASEID.stdout $TEMPDIR/dump.stdout >$TEMPDIR/diff
+ rc="$?"
+else # load failed as expected
+ # remove timestamps
+ sed -i -e 's/^....-..-..T..:..:.. //g' $TEMPDIR/load.stderr
+ # remove pathnames
+ sed -i -e 's=/.*/==g' $TEMPDIR/load.stderr
+ # remove source locations
+ sed -i -e 's=: .*:[0-9]*:[^ ]*:=:=g' $TEMPDIR/load.stderr
+ # remove version number
+ sed -i -e 's=latf-load\(\.[0-9]*\)*=latf-load=g' $TEMPDIR/load.stderr
+ diff $WORKDIR/expected/$CASEID.stderr $TEMPDIR/load.stderr >$TEMPDIR/diff
+ rc="$?"
+fi
+if [ "$rc" != "0" ] ; then
+ cat $TEMPDIR/diff
+ echo "command executed:"
+ echo $CMD
+ exit 4
+fi
+
+#rm -rf $TEMPDIR
+
+exit 0
diff --git a/test/fastq-loader/test-fastq-loader.cpp b/test/fastq-loader/test-fastq-loader.cpp
new file mode 100644
index 0000000..4ad4534
--- /dev/null
+++ b/test/fastq-loader/test-fastq-loader.cpp
@@ -0,0 +1,215 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Long-running tests for ReaderFile-related interfaces
+*/
+
+#include <cstring>
+#include <ctime>
+
+#include <ktst/unit_test.hpp>
+
+#include <klib/out.h>
+
+#include <kapp/args.h>
+
+#include <kfs/directory.h>
+#include <kfs/impl.h>
+
+#include <vdb/manager.h>
+#include <vdb/database.h>
+#include <vdb/schema.h>
+
+extern "C" {
+#include <loader/common-reader.h>
+#include <loader/common-writer.h>
+#include <loader/sequence-writer.h>
+#include <loader/alignment-writer.h>
+#include "../../tools/fastq-loader/fastq-reader.h"
+#include "../../tools/fastq-loader/fastq-parse.h"
+}
+
+using namespace std;
+using namespace ncbi::NK;
+
+TEST_SUITE(LoaderFastqTestSuite);
+
+///////////////////////////////////////////////// tests for loading FASTQ files
+
+#include <kfs/directory.h>
+#include <kfs/file.h>
+
+class TempFileFixture
+{
+public:
+ static const string TempDir;
+ static const string SchemaPath;
+ static const string DbType;
+
+public:
+ TempFileFixture()
+ : wd(0), rf(0)
+ {
+ if ( KDirectoryNativeDir ( & wd ) != 0 )
+ FAIL("KDirectoryNativeDir failed");
+
+ if ( VDBManagerMakeUpdate ( & mgr, wd ) != 0)
+ FAIL("VDBManagerMakeUpdate failed");
+
+ if ( VDBManagerMakeSchema ( mgr, & schema ) != 0 )
+ FAIL("VDBManagerMakeSchema failed");
+ if ( VSchemaParseFile( schema, SchemaPath.c_str() ) != 0 )
+ FAIL("VSchemaParseFile failed");
+
+ if ( KDirectoryCreateDir_v1 ( wd, 0775, kcmOpen | kcmInit | kcmCreate, TempDir.c_str() ) != 0 )
+ FAIL("KDirectoryOpenDirUpdate_v1 failed");
+ }
+ ~TempFileFixture()
+ {
+ if ( rf != 0 && ReaderFileRelease( rf ) != 0)
+ FAIL("ReaderFileRelease failed");
+
+ if ( schema && VSchemaRelease(schema) != 0 )
+ FAIL("VSchemaRelease failed");
+ if ( db && VDatabaseRelease(db) != 0 )
+ FAIL("VDatabaseRelease failed");
+ if ( mgr && VDBManagerRelease(mgr) != 0 )
+ FAIL("VDBManagerRelease failed");
+
+ if ( wd && ! filename.empty() && KDirectoryRemove(wd, true, filename.c_str()) != 0 )
+ FAIL("KDirectoryRemove on input failed");
+
+ if ( wd && ! dbName.empty() )
+ { // sometimes it takes several attempts to remove a non-empty dir
+ while (KDirectoryRemove(wd, true, dbName.c_str()) != 0);
+ }
+ if ( wd )
+ { // sometimes it takes several attempts to remove a non-empty dir
+ while (KDirectoryRemove(wd, true, TempDir.c_str()) != 0);
+ }
+
+ if ( wd && KDirectoryRelease ( wd ) != 0 )
+ FAIL("KDirectoryRelease failed");
+ }
+ rc_t CreateFile(const char* p_filename, const char* contents)
+ { // create and open for read
+ KFile* file;
+ filename=p_filename;
+ rc_t rc=KDirectoryCreateFile(wd, &file, true, 0664, kcmInit, p_filename);
+ if (rc == 0)
+ {
+ size_t num_writ=0;
+ rc=KFileWrite(file, 0, contents, strlen(contents), &num_writ);
+ if (rc == 0)
+ {
+ rc=KFileRelease(file);
+ }
+ else
+ {
+ KFileRelease(file);
+ }
+ file=0;
+ }
+ return FastqReaderFileMake(&rf, wd, p_filename, FASTQphred33, 0, false);
+ }
+
+ KDirectory* wd;
+ string filename;
+ const ReaderFile* rf;
+ VDBManager* mgr;
+ VSchema *schema;
+ VDatabase* db;
+ string dbName;
+};
+const string TempFileFixture::TempDir = "./tmp";
+const string TempFileFixture::SchemaPath = "align/align.vschema";
+const string TempFileFixture::DbType = "NCBI:align:db:alignment_unsorted";
+
+///////////////////////////////////////////////// FASTQ-based tests for CommonWriter
+FIXTURE_TEST_CASE(CommonWriterOneFile, TempFileFixture)
+{ // source: SRR006565
+ CreateFile(GetName(),
+ "@G15-D_3_1_903_603_0.81\n"
+ "GATTGTAGGGAGTAGGGTACAATACAGTCTGGTCTC\n"
+ "+G15-D_3_1_903_603_0.81\n"
+ "IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII\n");
+
+ dbName = string(GetName())+".db";
+ KDirectoryRemove(wd, true, dbName.c_str());
+ REQUIRE_RC(VDBManagerCreateDB(mgr, &db, schema, DbType.c_str(), kcmInit + kcmMD5, dbName.c_str()));
+
+ CommonWriterSettings settings;
+ memset(&settings, 0, sizeof(settings));
+ settings.numfiles = 1;
+ settings.tmpfs = TempDir.c_str();
+
+ CommonWriter cw;
+ REQUIRE_RC(CommonWriterInit( &cw, mgr, db, &settings ));
+
+ REQUIRE_RC(CommonWriterArchive( &cw, rf ));
+ REQUIRE_RC(CommonWriterComplete( &cw, false, 0 ));
+
+ REQUIRE_RC(CommonWriterWhack( &cw ));
+
+ //TODO: open and validate database
+}
+
+//////////////////////////////////////////// Main
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+extern "C"
+{
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+
+const char UsageDefaultName[] = "test-fastq-loader";
+
+rc_t CC UsageSummary (const char * progname)
+{
+ return KOutMsg ( "Usage:\n" "\t%s [options] -o path\n\n", progname );
+}
+
+rc_t CC Usage( const Args* args )
+{
+ return 0;
+}
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ // need to use user settings in order to get to the current schema
+ //KConfigDisableUserSettings();
+
+ rc_t rc = LoaderFastqTestSuite(argc, argv);
+ return rc;
+}
+
+}
+
diff --git a/test/fastq-loader/wb-test-fastq.cpp b/test/fastq-loader/wb-test-fastq.cpp
new file mode 100644
index 0000000..5172bbe
--- /dev/null
+++ b/test/fastq-loader/wb-test-fastq.cpp
@@ -0,0 +1,1376 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for FASTQ loader
+*/
+#include <ktst/unit_test.hpp>
+#include <klib/rc.h>
+#include <loader/common-writer.h>
+#include <kfs/directory.h>
+#include <kfs/file.h>
+
+#include "../../tools/fastq-loader/fastq-parse.h"
+#include "../../tools/fastq-loader/fastq-tokens.h"
+#include "../../tools/fastq-loader/fastq-reader.h"
+
+#include <sysalloc.h>
+#include <stdlib.h>
+#include <cstring>
+#include <stdexcept>
+#include <list>
+
+using namespace std;
+
+TEST_SUITE(FastqLoaderWbTestSuite);
+
+//////////////////////////////////////////// tests for flex-generated scanner
+
+
+// test fixture for scanner tests
+class FastqScanFixture
+{
+public:
+ FastqScanFixture()
+ {
+ pb.self = this;
+ pb.input = Input;
+ consumed = 0;
+ }
+ ~FastqScanFixture()
+ {
+ FASTQScan_yylex_destroy(&pb);
+ if (pb.record != 0)
+ {
+ KDataBufferWhack( & pb.record->source );
+ free(pb.record);
+ }
+ }
+ void InitScan(const char* p_input, bool trace=false)
+ {
+ input = p_input;
+ FASTQScan_yylex_init(&pb, trace);
+ pb.record = (FastqRecord*)calloc(sizeof(FastqRecord), 1);
+ KDataBufferMakeBytes ( & pb.record->source, 0 );
+ FASTQ_ParseBlockInit ( &pb );
+ }
+ int Scan()
+ {
+ int tokenId=FASTQ_lex(&sym, pb.scanner);
+ if (tokenId != 0)
+ {
+ tokenText=string(TokenTextPtr(&pb, &sym), sym.tokenLength);
+ }
+ else
+ {
+ tokenText.clear();
+ }
+
+ return tokenId;
+ }
+ static size_t CC Input(FASTQParseBlock* sb, char* buf, size_t max_size)
+ {
+ FastqScanFixture* self = (FastqScanFixture*)sb->self;
+ if (self->input.size() < self->consumed)
+ return 0;
+
+ size_t to_copy = min(self->input.size() - self->consumed, max_size);
+ if (to_copy == 0)
+ return 0;
+
+ memcpy(buf, self->input.c_str(), to_copy);
+ if (to_copy < max_size && buf[to_copy-1] != '\n')
+ {
+ buf[to_copy] = '\n';
+ ++to_copy;
+ }
+ self->consumed += to_copy;
+ return to_copy;
+ }
+
+ string input;
+ size_t consumed;
+ FASTQParseBlock pb;
+ FASTQToken sym;
+ string tokenText;
+};
+
+FIXTURE_TEST_CASE(EmptyInput, FastqScanFixture)
+{
+ InitScan("");
+ REQUIRE_EQUAL(Scan(), 0);
+}
+#define REQUIRE_TOKEN(tok) REQUIRE_EQUAL((int)tok, Scan());
+#define REQUIRE_TOKEN_TEXT(tok, text) REQUIRE_TOKEN(tok); REQUIRE_EQ(tokenText, string(text));
+
+#define REQUIRE_TOKEN_COORD(tok, text, line, col) \
+ REQUIRE_TOKEN_TEXT(tok, text); \
+ REQUIRE_EQ(pb.lastToken->line_no, (size_t)line); \
+ REQUIRE_EQ(pb.lastToken->column_no, (size_t)col);
+
+FIXTURE_TEST_CASE(TagLine1, FastqScanFixture)
+{
+ InitScan("@HWUSI-EAS499_1:1:3:9:1822\n");
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN_COORD(fqALPHANUM, "HWUSI-EAS499", 1, 2);
+ REQUIRE_TOKEN('_');
+ REQUIRE_TOKEN_TEXT(fqNUMBER, "1");
+ REQUIRE_TOKEN_TEXT(fqCOORDS, ":1:3:9:1822");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN(0);
+}
+
+FIXTURE_TEST_CASE(SequenceQuality, FastqScanFixture)
+{
+ InitScan("@8\n" "GATC\n" "+8:1:46:673\n" "!**'\n");
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN_TEXT(fqNUMBER, "8");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN_COORD(fqBASESEQ, "GATC", 2, 1);
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN('+');
+ REQUIRE_TOKEN_TEXT(fqTOKEN, "8:1:46:673");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN_TEXT(fqASCQUAL, "!**'");
+ REQUIRE_TOKEN(fqENDLINE);
+}
+
+FIXTURE_TEST_CASE(QualityOnly, FastqScanFixture)
+{
+ InitScan(">8\n" "\x7F!**'\n");
+ REQUIRE_TOKEN('>');
+ REQUIRE_TOKEN_TEXT(fqNUMBER, "8");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN_TEXT(fqASCQUAL, "\x7F!**'");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN(0);
+}
+
+FIXTURE_TEST_CASE(NoEOL_InQuality, FastqScanFixture)
+{
+ InitScan("@8\n" "GATC\n" "+\n" "!**'");
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN_TEXT(fqNUMBER, "8");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN_TEXT(fqBASESEQ, "GATC");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN('+');
+ REQUIRE_TOKEN(fqENDLINE);
+
+ REQUIRE_TOKEN_TEXT(fqASCQUAL, "!**'");
+ REQUIRE_TOKEN(fqENDLINE); /* this is auto-inserted by FastqScanFixture::Input() */
+ REQUIRE_TOKEN(0);
+}
+
+FIXTURE_TEST_CASE(CRnoLF, FastqScanFixture)
+{
+ InitScan("@8\r", 0);
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN_TEXT(fqNUMBER, "8");
+ REQUIRE_TOKEN(fqENDLINE);
+}
+
+FIXTURE_TEST_CASE(CommaSeparatedQuality3, FastqScanFixture)
+{
+ InitScan("@8\n" "GATC\n" "+\n" "0047044004,046,,4000,04444000,--,6-\n");
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN_TEXT(fqNUMBER, "8");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN_TEXT(fqBASESEQ, "GATC");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN('+');
+ REQUIRE_TOKEN(fqENDLINE);
+
+ REQUIRE_TOKEN_TEXT(fqASCQUAL, "0047044004,046,,4000,04444000,--,6-");
+ REQUIRE_EQUAL(Scan(), (int)fqENDLINE);
+ REQUIRE_EQUAL(Scan(), 0);
+}
+
+FIXTURE_TEST_CASE(WsBeforeEol, FastqScanFixture)
+{
+ InitScan("@ \n");
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_EQUAL(Scan(), 0);
+}
+
+FIXTURE_TEST_CASE(SkipToEol, FastqScanFixture)
+{
+ InitScan("@ kjalkjaldkj \nGATC\n");
+ REQUIRE_TOKEN('@');
+ FASTQScan_skip_to_eol(&pb);
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_TOKEN_TEXT(fqBASESEQ, "GATC"); // back to normal
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_EQUAL(Scan(), 0);
+}
+
+FIXTURE_TEST_CASE(InlineBaseSequence, FastqScanFixture)
+{
+ InitScan("@:GATC.NNNN\n");
+ REQUIRE_TOKEN('@');
+ REQUIRE_TOKEN(':');
+ FASTQScan_inline_sequence(&pb);
+ REQUIRE_TOKEN_TEXT(fqBASESEQ, "GATC.NNNN");
+ REQUIRE_TOKEN(fqENDLINE);
+ REQUIRE_EQUAL(Scan(), 0);
+}
+
+///////////////////////////////////////////////// Fastq Parser test fixture
+class ParserFixture
+{
+public:
+ ParserFixture()
+ {
+ pb.record = 0;
+ }
+ ~ParserFixture()
+ {
+ FASTQScan_yylex_destroy(&pb);
+ if (pb.record != 0)
+ {
+ KDataBufferWhack( & pb.record->source );
+ free(pb.record);
+ }
+ }
+
+ bool Parse(bool traceLex = false)
+ {
+ pb.self = this;
+ pb.input = Input;
+ pb.qualityFormat = FASTQphred33;
+ pb.defaultReadNumber = 9;
+ pb.secondaryReadNumber = 0;
+ pb.ignoreSpotGroups = false;
+
+ if (FASTQScan_yylex_init(& pb, traceLex) != 0)
+ FAIL("ParserFixture::ParserFixture: FASTQScan_yylex_init failed");
+
+ pb.record = (FastqRecord*)calloc(1, sizeof(FastqRecord));
+ if (pb.record == 0)
+ FAIL("ParserFixture::ParserFixture: malloc failed");
+ KDataBufferMakeBytes ( & pb.record->source, 0 );
+
+ //FASTQ_debug = 1;
+ FASTQ_ParseBlockInit ( &pb );
+ return FASTQ_parse( &pb ) == 1 && pb.record->rej == 0;
+ }
+
+ void AddBuffer(const string& p_text)
+ {
+ buffers.push_back(p_text);
+ }
+
+ static size_t CC Input(struct FASTQParseBlock* sb, char* buf, size_t max_size)
+ {
+ ParserFixture* self = (ParserFixture*)sb->self;
+ if (self->buffers.empty())
+ return 0;
+
+ string s = self->buffers.front();
+ self->buffers.pop_front();
+ memcpy(buf, s.c_str(), s.size()); // ignore max_size for our short test lines
+ return s.size();
+ }
+
+ list<string> buffers;
+ FASTQParseBlock pb;
+};
+
+FIXTURE_TEST_CASE(BufferBreakInTag, ParserFixture)
+{
+ AddBuffer("@HWI-ST226:170:AB075UABXX:3:1101:10089:7031 ");
+ AddBuffer("1:N:0:GCCAAT\n"
+ "TACA\n"
+ "+\n"
+ "GEGE\n");
+ REQUIRE(Parse());
+ REQUIRE_EQ(1, (int)pb.record->seq.readnumber);
+}
+
+///////////////////////////////////////////////// FastqReader test fixture
+
+class LoaderFixture
+{
+public:
+ LoaderFixture()
+ : wd(0), rf(0),
+ record(0), seq(0), reject(0),
+ read(0), readLength(0),
+ name(0), length(0),
+ errorText(0), errorLine(0), column(0),
+ quality(0), qualityAsciiOffset(0), qualityType(-1),
+ qualityFormat(FASTQphred33), defaultReadNumber(0),
+ ignoreSpotGroups(false)
+ {
+ if ( KDirectoryNativeDir ( & wd ) != 0 )
+ FAIL("KDirectoryNativeDir failed");
+ }
+ ~LoaderFixture()
+ {
+ delete [] read;
+
+ if (seq != 0 && SequenceRelease(seq) != 0)
+ FAIL("SequenceRelease failed");
+
+ if (record != 0 && RecordRelease(record) != 0)
+ FAIL("RecordRelease failed");
+
+ if (reject != 0 && RejectedRelease(reject) != 0)
+ FAIL("RejectedRelease failed");
+
+ if ( rf != 0 && ReaderFileRelease( rf ) != 0)
+ FAIL("ReaderFileRelease failed");
+
+ if ( !filename.empty() && KDirectoryRemove(wd, true, filename.c_str()) != 0)
+ FAIL("KDirectoryRemove failed");
+
+ if ( KDirectoryRelease ( wd ) != 0 )
+ FAIL("KDirectoryRelease failed");
+
+ FASTQ_debug = 0;
+ }
+ rc_t CreateFile(const char* p_filename, const char* contents)
+ { // create and open for read
+ KFile* file;
+ filename=p_filename;
+ rc_t rc=KDirectoryCreateFile(wd, &file, true, 0664, kcmInit, p_filename);
+ if (rc == 0)
+ {
+ size_t num_writ=0;
+ rc=KFileWrite(file, 0, contents, strlen(contents), &num_writ);
+ if (rc == 0)
+ {
+ rc=KFileRelease(file);
+ }
+ else
+ {
+ KFileRelease(file);
+ }
+ file=0;
+ }
+ return FastqReaderFileMake(&rf, wd, p_filename, qualityFormat, defaultReadNumber, ignoreSpotGroups);
+ }
+ void CreateFileGetRecord(const char* fileName, const char* contents)
+ {
+ if (record != 0 && RecordRelease(record) != 0)
+ throw logic_error("CreateFileGetRecord: RecordRelease failed");
+
+ if (CreateFile(fileName, contents) != 0)
+ throw logic_error("CreateFileGetRecord: CreateFile failed");
+
+ if (ReaderFileGetRecord(rf, &record) != 0 || record == 0)
+ throw logic_error("CreateFileGetRecord: ReaderFileGetRecord failed");
+ }
+ bool GetRecord()
+ {
+ if (rf == 0)
+ return false;
+
+ if (record != 0 && RecordRelease(record) != 0)
+ return false;
+ if (reject != 0 && RejectedRelease(reject) != 0)
+ return false;
+ reject = 0;
+ if (seq != 0 && SequenceRelease(seq) != 0)
+ return false;
+ seq = 0;
+
+ return ReaderFileGetRecord(rf, &record) == 0;
+ }
+
+ bool GetRejected()
+ {
+ if (reject != 0 && RejectedRelease(reject) != 0)
+ throw logic_error("GetRejected: RejectedRelease failed");
+ if (record == 0)
+ throw logic_error("GetRejected: record == 0");
+ if (RecordGetRejected(record, &reject) != 0)
+ throw logic_error("GetRejected: RecordGetRejected failed");
+ if (reject == 0)
+ return false;
+ if (RejectedGetError(reject, &errorText, &errorLine, &column, &fatal) != 0)
+ throw logic_error("IsFatal: RejectedGetError failed");
+ return true;
+ }
+
+ bool CreateFileGetSequence(const char* name, const char* contents)
+ {
+ if (CreateFile(name, contents) != 0)
+ throw logic_error("CreateFileGetSequence: CreateFile failed");
+
+ if (!GetRecord() || record == 0)
+ throw logic_error("CreateFileGetSequence: GetRecord failed");
+
+ if (GetRejected())
+ throw logic_error(string("CreateFileGetSequence: record rejected, ")+string(errorText));
+
+ if (RecordGetSequence(record, &seq) != 0)
+ throw logic_error("CreateFileGetSequence: RecordGetSequence failed:");
+
+ return seq != 0;
+ }
+
+ bool MakeReadBuffer()
+ {
+ if (SequenceGetReadLength(seq, &readLength) != 0 || readLength == 0)
+ return false;
+
+ delete [] read;
+ read = new char[readLength];
+ return true;
+ }
+
+ void BisonDebugOn()
+ {
+ FASTQ_debug = 1;
+ }
+
+ KDirectory* wd;
+ string filename;
+ const ReaderFile* rf;
+ const Record* record;
+ const Sequence* seq;
+ const Rejected* reject;
+
+ char* read;
+ uint32_t readLength;
+
+ const char* name;
+ size_t length;
+
+ const char* errorText;
+ uint64_t errorLine;
+ uint64_t column;
+ bool fatal;
+ const void* errorData;
+
+ const int8_t* quality;
+ uint8_t qualityAsciiOffset;
+ int qualityType;
+
+ enum FASTQQualityFormat qualityFormat;
+ int8_t defaultReadNumber;
+ bool ignoreSpotGroups;
+};
+
+///////////////////////////////////////////////// FASTQ test cases
+FIXTURE_TEST_CASE(EmptyFile, LoaderFixture)
+{
+ REQUIRE_RC(CreateFile(GetName(), ""));
+ REQUIRE_EQ( string(ReaderFileGetPathname(rf)), string(GetName()) );
+ REQUIRE(GetRecord());
+ REQUIRE_NULL(record);
+}
+
+FIXTURE_TEST_CASE(EndLines, LoaderFixture)
+{
+ REQUIRE_RC(CreateFile(GetName(), "\n\n"));
+ REQUIRE_EQ( string(ReaderFileGetPathname(rf)), string(GetName()) );
+ REQUIRE(GetRecord());
+ REQUIRE_NULL(record);
+}
+
+//////////////////// syntax errors and recovery
+const string SyntaxError("syntax error");
+FIXTURE_TEST_CASE(SyntaxError1, LoaderFixture)
+{
+ string input="qqq abcd";
+
+ CreateFileGetRecord(GetName(), input.c_str());
+
+ REQUIRE(GetRejected());
+ REQUIRE(!fatal);
+ REQUIRE_NOT_NULL(errorText);
+ REQUIRE_EQ(SyntaxError, string (errorText).substr(0, SyntaxError.size()));
+ REQUIRE_EQ(errorLine, (uint64_t)1);
+ REQUIRE_EQ(column, (uint64_t)4);
+
+ const void* data;
+ REQUIRE_RC(RejectedGetData(reject, &data, &length));
+ REQUIRE_NOT_NULL(data);
+ REQUIRE_EQ(input, string((const char*)data, length));
+}
+
+FIXTURE_TEST_CASE(SyntaxError2, LoaderFixture)
+{
+ #define input "qqq abcd"
+ CreateFileGetRecord(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n" input );
+ REQUIRE(GetRecord());
+
+ REQUIRE(GetRejected());
+ REQUIRE(!fatal);
+ REQUIRE_NOT_NULL(errorText);
+ REQUIRE_EQ(SyntaxError, string (errorText).substr(0, SyntaxError.size()));
+ REQUIRE_EQ(errorLine, (uint64_t)5);
+ REQUIRE_EQ(column, (uint64_t)4);
+
+ const void* data;
+ REQUIRE_RC(RejectedGetData(reject, &data, &length));
+ REQUIRE_EQ(string(input), string((const char*)data, length));
+
+
+ #undef input
+}
+
+FIXTURE_TEST_CASE(RecoveryFromErrorAtTopLevel, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(), "qqq abcd\n" "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n");
+ REQUIRE(GetRejected());
+
+ REQUIRE(GetRecord());
+ REQUIRE_NOT_NULL(record);
+
+ REQUIRE_RC(RecordGetSequence(record, &seq));
+ REQUIRE_NOT_NULL(seq);
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("SEQ_ID1"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(RecoveryFromErrorInHeader, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(), "@SEQ_ID1^\n" "GATT\n" "+\n" "!''*\n" "@SEQ_ID2\n" "GATT\n" "+\n" "!''*\n");
+ REQUIRE(GetRejected());
+
+ REQUIRE(GetRecord());
+ REQUIRE_NOT_NULL(record);
+
+ REQUIRE_RC(RecordGetSequence(record, &seq));
+ REQUIRE_NOT_NULL(seq);
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("SEQ_ID2"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(RecoveryFromErrorInRead, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(), "@SEQ_ID1\n" "G^ATT\n" "+\n" "!''*\n" "@SEQ_ID2\n" "GATT\n" "+\n" "!''*\n");
+ REQUIRE(GetRejected());
+
+ REQUIRE(GetRecord());
+ REQUIRE_NOT_NULL(record);
+
+ REQUIRE_RC(RecordGetSequence(record, &seq));
+ REQUIRE_NOT_NULL(seq);
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("SEQ_ID2"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(RecoveryFromErrorInQuality, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "\n" "@SEQ_ID2\n" "GATT\n" "+\n" "!''*\n");
+ REQUIRE(GetRejected());
+
+ REQUIRE(GetRecord());
+ REQUIRE_NOT_NULL(record);
+
+ REQUIRE_RC(RecordGetSequence(record, &seq));
+ REQUIRE_NOT_NULL(seq);
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("SEQ_ID2"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(ErrorLineNumber, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(),
+ "@HWI-ST959:56:D0AW4ACXX:8:2108:6958:112042 1:Y:0#\n"
+ "ATT\n");
+ REQUIRE(GetRejected());
+ REQUIRE_EQ(SyntaxError, string (errorText).substr(0, SyntaxError.size()));
+ REQUIRE_EQ(errorLine, (uint64_t)1);
+ REQUIRE_EQ(column, (uint64_t)49);
+}
+
+
+//////////////////// platform specification
+TEST_CASE(PlatformValid)
+{
+ REQUIRE_EQ((uint8_t)SRA_PLATFORM_ILLUMINA, PlatformToId("ILLUMINA"));
+}
+
+TEST_CASE(PlatformInvalid)
+{
+ REQUIRE_EQ((uint8_t)SRA_PLATFORM_UNDEFINED, PlatformToId("FOO"));
+}
+
+//////////////////// tag line parsing
+#define TEST_TAGLINE(line)\
+ CreateFileGetRecord(GetName(), line "\n" "GATT\n" "+\n" "!''*\n");\
+ REQUIRE(! GetRejected()); /* no error */ \
+ REQUIRE(GetRecord()); /* parsing done */ \
+ REQUIRE_NULL(record); /* input consumed */
+
+FIXTURE_TEST_CASE(Tag1, LoaderFixture) { TEST_TAGLINE("@HWUSI-EAS499:1:3:9:1822"); }
+FIXTURE_TEST_CASE(Tag2_1, LoaderFixture) { TEST_TAGLINE("@HWUSI-EAS499:1:3:9:1822:.2"); }
+FIXTURE_TEST_CASE(Tag2_2, LoaderFixture) { TEST_TAGLINE("@HWUSI-EAS499:1:3:9:1822:1.2"); }
+FIXTURE_TEST_CASE(Tag3, LoaderFixture) { TEST_TAGLINE("@BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540"); }
+FIXTURE_TEST_CASE(Tag4, LoaderFixture) { TEST_TAGLINE("@BILLIEHOLIDAY_1_FC200TYAAXX_3_1_751_675"); }
+FIXTURE_TEST_CASE(Tag5, LoaderFixture) { TEST_TAGLINE("@HWUSI-EAS499:1:3:9:1822#0/1"); }
+FIXTURE_TEST_CASE(Tag6, LoaderFixture) { TEST_TAGLINE("@1:3:9:1822:33.44"); }
+FIXTURE_TEST_CASE(Tag7, LoaderFixture) { TEST_TAGLINE("@1:3:9:1822#1/2"); }
+FIXTURE_TEST_CASE(Tag8, LoaderFixture) { TEST_TAGLINE("@HWUSI-EAS499:1:3:9:1822#CAT/1"); }
+FIXTURE_TEST_CASE(Tag9, LoaderFixture) { TEST_TAGLINE("@ERBRDQF01EGP9U"); }
+FIXTURE_TEST_CASE(Tag10, LoaderFixture) { TEST_TAGLINE("@ID57_120908_30E4FAAXX:3:1:1772:953.1"); }
+FIXTURE_TEST_CASE(Tag11, LoaderFixture) { TEST_TAGLINE(">ID57_120908_30E4FAAXX:3:1:1772:953"); }
+FIXTURE_TEST_CASE(Tag12, LoaderFixture) { TEST_TAGLINE("@741:6:1:1204:10747"); }
+FIXTURE_TEST_CASE(Tag13, LoaderFixture) { TEST_TAGLINE("@741:6:1:1204:10747/1"); }
+FIXTURE_TEST_CASE(Tag14, LoaderFixture) { TEST_TAGLINE("@SNPSTER4_246_30GCDAAXX_PE:1:1:3:896/1 run=090102_SNPSTER4_0246_30GCDAAXX_PE"); }
+FIXTURE_TEST_CASE(Tag15, LoaderFixture) { TEST_TAGLINE("@G15-D_3_1_903_603_0.81"); }
+
+//////////////////// building Sequence objects
+FIXTURE_TEST_CASE(NotRejected, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(), "@HWUSI-EAS499:1:3:9:1822:1.2\n" "GATT\n" "+\n" "!''*\n");
+ reject = (const Rejected*)1;
+ REQUIRE_RC(RecordGetRejected(record, &reject));
+ REQUIRE_NULL(reject);
+}
+
+FIXTURE_TEST_CASE(SequenceNoAlignment, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(), "@HWUSI-EAS499:1:3:9:1822:1.2\n" "GATT\n" "+\n" "!''*\n");
+
+ const Alignment* align;
+ REQUIRE_RC(RecordGetAlignment(record, &align));
+ REQUIRE_NULL(align);
+
+ REQUIRE_RC(RecordGetSequence(record, &seq));
+ REQUIRE_NOT_NULL(seq);
+}
+
+FIXTURE_TEST_CASE(TestSequenceGetReadLength, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n" ));
+ REQUIRE_RC(SequenceGetReadLength(seq, &readLength));
+ REQUIRE_EQ(readLength, 4u);
+}
+
+FIXTURE_TEST_CASE(TestSequenceGetRead, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n" ));
+ REQUIRE(MakeReadBuffer());
+ REQUIRE_RC(SequenceGetRead(seq, read));
+ REQUIRE_EQ(string(read, readLength), string("GATT"));
+}
+
+FIXTURE_TEST_CASE(TestSequenceGetRead2, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n" ));
+ REQUIRE(MakeReadBuffer());
+
+ // normal
+ REQUIRE_RC(SequenceGetRead2(seq, read, 0, 2));
+ REQUIRE_EQ(string(read, 2), string("GA"));
+
+ // stop out of range
+ REQUIRE_RC_FAIL(SequenceGetRead2(seq, read, 2, 6));
+
+ // start out of range
+ REQUIRE_RC_FAIL(SequenceGetRead2(seq, read, 20, 1));
+}
+
+FIXTURE_TEST_CASE(SequenceGetQuality33, LoaderFixture)
+{
+ qualityFormat = FASTQphred33;
+
+ REQUIRE(CreateFileGetSequence(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n" ));
+
+ REQUIRE_RC(SequenceGetQuality(seq, &quality, &qualityAsciiOffset, &qualityType));
+ REQUIRE_NOT_NULL(quality);
+ uint32_t l;
+ REQUIRE_RC(SequenceGetReadLength(seq, &l));
+ REQUIRE_EQ(qualityAsciiOffset, (uint8_t)33);
+ REQUIRE_EQ(qualityType, (int)QT_Phred);
+ REQUIRE_EQ(quality[0], (int8_t)'!');
+}
+
+FIXTURE_TEST_CASE(SequenceGetQuality64, LoaderFixture)
+{
+ qualityFormat = FASTQphred64;
+
+ REQUIRE(CreateFileGetSequence(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" "BBCC\n" ));
+
+ REQUIRE_RC(SequenceGetQuality(seq, &quality, &qualityAsciiOffset, &qualityType));
+ REQUIRE_NOT_NULL(quality);
+ uint32_t l;
+ REQUIRE_RC(SequenceGetReadLength(seq, &l));
+ REQUIRE_EQ(qualityAsciiOffset, (uint8_t)64);
+ REQUIRE_EQ(qualityType, (int)QT_Phred);
+ REQUIRE_EQ(quality[0], (int8_t)'B');
+}
+
+FIXTURE_TEST_CASE(SequenceGetQualityLogOdds, LoaderFixture)
+{
+ qualityFormat = FASTQlogodds;
+
+ REQUIRE(CreateFileGetSequence(GetName(), "@SEQ_ID1\n" "GATT\n" "+\n" ";>@H\n" ));
+
+ REQUIRE_RC(SequenceGetQuality(seq, &quality, &qualityAsciiOffset, &qualityType));
+ REQUIRE_NOT_NULL(quality);
+ uint32_t l;
+ REQUIRE_RC(SequenceGetReadLength(seq, &l));
+ REQUIRE_EQ(qualityAsciiOffset, (uint8_t)64);
+ REQUIRE_EQ(qualityType, (int)QT_LogOdds);
+ REQUIRE_EQ(quality[0], (int8_t)';');
+}
+
+FIXTURE_TEST_CASE(SequenceBaseSpace, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@q\n" "GATC\n" "+\n" "!''*\n" ));
+ REQUIRE(!SequenceIsColorSpace(seq));
+ char k;
+ REQUIRE_RC(SequenceGetCSKey(seq, &k)); // RC is 0 but k is undefined
+}
+
+FIXTURE_TEST_CASE(SequenceColorSpace, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@q\n" "G123\n" "+\n" "!''*\n" ));
+ REQUIRE(SequenceIsColorSpace(seq));
+
+ char k;
+ REQUIRE_RC(SequenceGetCSKey(seq, &k));
+ REQUIRE_EQ(k, 'G');
+
+ uint32_t l;
+ REQUIRE_RC(SequenceGetReadLength(seq, &l));
+ REQUIRE_EQ(l, (uint32_t)0);
+ REQUIRE_RC(SequenceGetCSReadLength(seq, &l));
+ REQUIRE_EQ(l, (uint32_t)3);
+
+ delete [] read;
+ read = new char[l];
+ REQUIRE_RC(SequenceGetCSRead(seq, read));
+ REQUIRE_EQ(string(read, l), string("123"));
+
+ REQUIRE_RC(SequenceGetCSQuality(seq, &quality, &qualityAsciiOffset, &qualityType));
+ REQUIRE_NOT_NULL(quality);
+ REQUIRE_EQ(qualityType, (int)QT_Phred);
+ REQUIRE_EQ(qualityAsciiOffset, (uint8_t)33);
+ REQUIRE_EQ(quality[0], (int8_t)'\'');
+ REQUIRE_EQ(quality[1], (int8_t)'\'');
+ REQUIRE_EQ(quality[2], (int8_t)'*' );
+}
+
+FIXTURE_TEST_CASE(SequenceGetOrientationIsReverse, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@q\n" "G123\n" "+\n" "!''*\n" ));
+ REQUIRE_EQ(SequenceGetOrientationSelf(seq), (int)ReadOrientationUnknown);
+ REQUIRE_EQ(SequenceGetOrientationMate(seq), (int)ReadOrientationUnknown);
+}
+
+// Read Number
+FIXTURE_TEST_CASE(ReadNumber, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@HWUSI-EAS499:1:3:9:1822#0/1\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE(SequenceIsFirst(seq));
+ REQUIRE(!SequenceIsSecond(seq));
+}
+FIXTURE_TEST_CASE(ReadNumberMissing, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@HWUSI-EAS499:1:3:9:1822\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE(!SequenceIsFirst(seq));
+ REQUIRE(!SequenceIsSecond(seq));
+}
+FIXTURE_TEST_CASE(ReadNumberDefault, LoaderFixture)
+{
+ defaultReadNumber = 2;
+ REQUIRE(CreateFileGetSequence(GetName(), "@HWUSI-EAS499:1:3:9:1822\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE(!SequenceIsFirst(seq));
+ REQUIRE(SequenceIsSecond(seq));
+}
+FIXTURE_TEST_CASE(ReadNumberOverride, LoaderFixture)
+{
+ defaultReadNumber = 1;
+ REQUIRE(CreateFileGetSequence(GetName(), "@HWUSI-EAS499:1:3:9:1822/2\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE(!SequenceIsFirst(seq));
+ REQUIRE(SequenceIsSecond(seq));
+}
+
+FIXTURE_TEST_CASE(RunSpotRead, LoaderFixture)
+{
+ defaultReadNumber = 1;
+ REQUIRE(CreateFileGetSequence(GetName(), "@SRR390728.1.2\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE(SequenceIsSecond(seq));
+}
+
+FIXTURE_TEST_CASE(RunSpotRead_withTail, LoaderFixture)
+{
+ defaultReadNumber = 1;
+ REQUIRE(CreateFileGetSequence(GetName(), "@SRR390728.1.2 123\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE(SequenceIsSecond(seq));
+}
+
+FIXTURE_TEST_CASE(RunSpotNothing, LoaderFixture)
+{
+ defaultReadNumber = 1;
+ REQUIRE(CreateFileGetSequence(GetName(), "@SRR390728.1\n" "GATT\n" "+\n" "!''*\n"));
+}
+
+// Components of Illumina tag lines:
+// @HWUSI-EAS499:1:3:9:1822#0/1"
+// spot-name HWUSI-EAS499:1:3:9:1822 tag line up to and including coordinates
+// spot-group "0" token following '#'
+// read-number "1" 1 or 2 following '/'
+
+// not implemented for now:
+// run-group HWUSI-EAS499 tag line up to and excluding coordinates
+// coords [ 1 3 9 182 ]
+// fmt-name HWUSI-EAS499:1:3:$X:$Y
+
+FIXTURE_TEST_CASE(SequenceGetSpotNameIllumina, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@HWUSI-EAS499:1:3:9:1822#0/1\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWUSI-EAS499:1:3:9:1822"), string(name, length));
+ REQUIRE(!SequenceIsSecond(seq));
+ REQUIRE(SequenceIsFirst(seq));
+}
+
+FIXTURE_TEST_CASE(SequenceGetSpotGroupIllumina, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@1:3:9:1822#CAT/1\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string("CAT"), string(name, length));
+ REQUIRE(!SequenceIsSecond(seq));
+ REQUIRE(SequenceIsFirst(seq));
+}
+
+FIXTURE_TEST_CASE(SequenceGetSpotGroupIllumina_Ignored, LoaderFixture)
+{
+ ignoreSpotGroups = true;
+ REQUIRE(CreateFileGetSequence(GetName(), "@1:3:9:1822#CAT/1\n" "GATT\n" "+\n" "!''*\n"));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string(""), string(name, length));
+ REQUIRE(!SequenceIsSecond(seq));
+ REQUIRE(SequenceIsFirst(seq));
+}
+
+FIXTURE_TEST_CASE(SequenceGetSpotGroup_Empty, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ ">HWI-EAS6_4_FC2010T:1:1:80:366\n"
+ "GATT\n"
+ "+\n"
+ "!''*\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string(""), string(name, length));
+ REQUIRE(!SequenceIsSecond(seq));
+ REQUIRE(!SequenceIsFirst(seq));
+}
+
+FIXTURE_TEST_CASE(SequenceGetSpotGroup_Zero, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ ">HWI-EAS6_4_FC2010T:1:1:80:366#0\n"
+ "GATT\n"
+ "+\n"
+ "!''*\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string(""), string(name, length));
+ REQUIRE(!SequenceIsSecond(seq));
+ REQUIRE(!SequenceIsFirst(seq));
+}
+
+FIXTURE_TEST_CASE(SequenceGetSpotGroupBarcode, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(), "@HWI-ST1234:33:D1019ACXX:2:1101:1415:2223/1 1:N:0:ATCACG\nATCG\n"));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string("ATCACG"), string(name, length));
+ REQUIRE(!SequenceIsSecond(seq));
+ REQUIRE(SequenceIsFirst(seq));
+}
+
+
+#define TEST_PAIRED(line, paired)\
+ REQUIRE(CreateFileGetSequence(GetName(), line "\n" "GATT\n" "+\n" "!''*\n"));\
+ if (paired)\
+ REQUIRE(SequenceWasPaired(seq));\
+ else\
+ REQUIRE(!SequenceWasPaired(seq));
+
+FIXTURE_TEST_CASE(SequenceWasPaired1, LoaderFixture) { TEST_PAIRED("@HWUSI-EAS499:1:3:9:1822", false); }
+FIXTURE_TEST_CASE(SequenceWasPaired2, LoaderFixture) { TEST_PAIRED("@HWUSI-EAS499:1:3:9:1822#0/0", false); }
+FIXTURE_TEST_CASE(SequenceWasPaired3, LoaderFixture) { TEST_PAIRED("@HWUSI-EAS499:1:3:9:1822#0/1", true); }
+FIXTURE_TEST_CASE(SequenceWasPaired4, LoaderFixture) { TEST_PAIRED("@HWUSI-EAS499:1:3:9:1822#CAT/1", true); }
+FIXTURE_TEST_CASE(SequenceWasPaired5, LoaderFixture) { TEST_PAIRED("@HWUSI-EAS499:1:3:9:1822/2", true); }
+
+FIXTURE_TEST_CASE(SequenceGetSpotNameOneLine, LoaderFixture)
+{ // source: SRR014283
+ REQUIRE(CreateFileGetSequence(GetName(), "USI-EAS50_1:6:1:392:881:GCT:!!!"));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("USI-EAS50_1:6:1:392:881"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(SequenceGetNameNumeric, LoaderFixture)
+{ // source: SRR094419
+ qualityFormat = FASTQphred64;
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@741:6:1:1204:10747/1\n"
+ "GTCGTTGTCCCGCTCCTCATATTCNNNNNNNNNNNN\n"
+ "+\n"
+ "bbbbbbbbbbbbbbbbbbbb````BBBBBBBBBBBB\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("741:6:1:1204:10747"), string(name, length));
+}
+
+//////////////////// parsing Quality lines
+
+FIXTURE_TEST_CASE(GtStartsQualityOnly, LoaderFixture)
+{ // source: XXX001656
+ CreateFileGetRecord(GetName(),
+ ">HWI-EAS6_4_FC2010T:1:1:80:366\n"
+ "!!\n"
+ ">HWI-EAS6_4_FC2010T:1:1:80:366\n"
+ "!!\n"
+ );
+ REQUIRE(GetRejected());
+ REQUIRE(!fatal);
+}
+
+FIXTURE_TEST_CASE(OneLineRead, LoaderFixture)
+{
+// source: SRR016872
+ CreateFileGetRecord(GetName(), "USI-EAS50_1:6:1:392:881:GCTC:!!!!\n");
+ REQUIRE(!GetRejected());
+}
+
+FIXTURE_TEST_CASE(ForcePhredOffset, LoaderFixture)
+{ // quality line looks like it may be Phred64, but we know we are dealing with Phred33
+// source: SRR014126
+ qualityFormat = FASTQphred33;
+ CreateFileGetSequence(GetName(),
+ "@R16:8:1:19:1012#0/2\n"
+ "TTAAATGACTCTTTAAAAAACACAACATACATTGATATATTTATTCCTAGATATTTGCTTATAAGACTCTAATCA\n"
+ "+\n"
+ "BCCBBBACBBCCCCBCCCCCCCCCCBCBCCCABBBBBBCCBBCBBCCBBCBCCCABBCAAABC at CCCAB@CBACC\n"
+ );
+ REQUIRE_RC(SequenceGetQuality(seq, &quality, &qualityAsciiOffset, &qualityType));
+ REQUIRE_NOT_NULL(quality);
+ REQUIRE_EQ(qualityType, (int)QT_Phred);
+ REQUIRE_EQ(qualityAsciiOffset, (uint8_t)33);
+ REQUIRE_EQ(quality[0], (int8_t)'B');
+}
+
+// Illumina spot names
+FIXTURE_TEST_CASE(IlluminaCasava_1_8, LoaderFixture)
+{ // source: SAMN01860354.fastq
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST273:315:C0LKAACXX:7:1101:1487:2221 2:Y:0:GGCTAC\n"
+ "AACA\n+\n$.%0\n"
+ ));
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST273:315:C0LKAACXX:7:1101:1487:2221"), string(name, length));
+ REQUIRE(SequenceIsSecond(seq));
+ REQUIRE(SequenceIsLowQuality(seq));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string("GGCTAC"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(IlluminaCasava_1_8_SpotGroupNumber, LoaderFixture)
+{ // source: SAMN01860354.fastq
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST273:315:C0LKAACXX:7:1101:1487:2221 2:Y:0:1\n"
+ "AACA\n+\n$.%0\n"
+ ));
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST273:315:C0LKAACXX:7:1101:1487:2221"), string(name, length));
+ REQUIRE(SequenceIsSecond(seq));
+ REQUIRE(SequenceIsLowQuality(seq));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string("1"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(IlluminaCasava_1_8_SpotGroup_MoreMadness, LoaderFixture)
+{ // source: SRR1106612
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST808:130:H0B8YADXX:1:1101:1914:2223 1:N:0:NNNNNN.GGTCCA.AAAA\n"
+ "AACA\n+\n$.%0\n"
+ ));
+
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string("NNNNNN.GGTCCA.AAAA"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(IlluminaCasava_1_8_EmptyTag, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST959:56:D0AW4ACXX:8:1101:1233:2026 2:N:0:\n"
+ "AACA\n+\n$.%0\n"
+
+ ));
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST959:56:D0AW4ACXX:8:1101:1233:2026"), string(name, length));
+ REQUIRE(SequenceIsSecond(seq));
+ REQUIRE(! SequenceIsLowQuality(seq));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string(), string(name, length));
+}
+
+FIXTURE_TEST_CASE(Illumina_NegativeCoords, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWUSI-EAS1679-0005:4:113:4454:-51#0\n"
+ "AACA\n+\n$.%0\n"
+ ));
+
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWUSI-EAS1679-0005:4:113:4454:-51"), string(name, length));
+ REQUIRE_RC(SequenceGetSpotGroup(seq, &name, &length));
+ REQUIRE_EQ(string(""), string(name, length));
+}
+
+FIXTURE_TEST_CASE(Illumina_Underscore, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@DG7PMJN1:293:D12THACXX:2:1101:1161:1968_2:N:0:GATCAG\n"
+ "AGAGTTTGAT\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("DG7PMJN1:293:D12THACXX:2:1101:1161:1968"), string(name, length));
+ REQUIRE(SequenceIsSecond(seq));
+}
+
+FIXTURE_TEST_CASE(Illumina_IdentifierAtFront, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@QSEQ161.65 DBV2SVN1:1:1101:1474:2213#0/1\n"
+ "AGAGTTTGAT\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("DBV2SVN1:1:1101:1474:2213"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(Illumina_SpaceAndIdentifierAtFront, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@ QSEQ161 EAS139:136:FC706VJ:2:2104:15343:197393 1:Y:18:ATCACG\n"
+ "AGAGTTTGAT\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("EAS139:136:FC706VJ:2:2104:15343:197393"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(SRR1778155 , LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@2-796964 M01929:5:000000000-A46YE:1:1108:16489:18207 1:N:0:2 orig_bc=TATCGGGA new_bc=TATCGGGA bc_diffs=0\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("M01929:5:000000000-A46YE:1:1108:16489:18207"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(SRA192487, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST1234:33:D1019ACXX:2:1101:1415:2223/1 1:N:0:ATCACG\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST1234:33:D1019ACXX:2:1101:1415:2223"), string(name, length));
+}
+
+//////////////////// odd syntax cases
+FIXTURE_TEST_CASE(NoEolAtEof, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(),
+ "@SEQ_ID1\n" "GATT\n" "+\n" "!''*\n"
+ "@SEQ_ID2\n" "GATT\n" "+\n" "!''*"
+ );
+ REQUIRE_NOT_NULL(record);
+ REQUIRE(! GetRejected());
+
+ REQUIRE(GetRecord());
+ REQUIRE(! GetRejected());
+ REQUIRE_NOT_NULL(record);
+}
+
+FIXTURE_TEST_CASE(GtStartsReadOnly, LoaderFixture)
+{ // source: XXX001656
+ CreateFileGetRecord(GetName(),
+ ">q\n"
+ "AAC\n"
+ ">q\n"
+ "ACA\n"
+ );
+ REQUIRE(!GetRejected());
+}
+
+//////////////////// detecting older formats
+
+FIXTURE_TEST_CASE(Quality33TooLow, LoaderFixture)
+{ // negative qualities are not allowed for Phred33
+// source: SRR016872
+ qualityFormat = FASTQphred33;
+ CreateFileGetRecord(GetName(),
+ "@HWI-EAS102_1_30LWPAAXX:5:1:1792:566\n"
+ "GAAACCCCCTATTAGANNNNCNNNNCNATCATGTCA\n"
+ "+HWI-EAS102_1_30LWPAAXX:5:1:1792:566\n"
+ "II IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII\n"
+ );
+ REQUIRE(GetRejected());
+ REQUIRE(fatal);
+}
+
+FIXTURE_TEST_CASE(Quality64TooLow, LoaderFixture)
+{ // negative qualities are not allowed for Phred64
+// source: SRR016872
+ qualityFormat = FASTQphred64;
+ CreateFileGetRecord(GetName(),
+ "@HWI-EAS102_1_30LWPAAXX:5:1:1511:102\n"
+ "GGGGTTAGTGGCAGGGGGGGGGTCTCGGGGGGGGGG\n"
+ "+HWI-EAS102_1_30LWPAAXX:5:1:1511:102\n"
+ "IIIIIIIIIIIIIIIIII;IIIIIIIIIIIIIIIII\n"
+ );
+ REQUIRE(GetRejected());
+ REQUIRE(fatal);
+}
+
+FIXTURE_TEST_CASE(QualityLogoddsTooLow, LoaderFixture)
+{ // qualities under 59 are not allowed for LogOdds
+ qualityFormat = FASTQlogodds;
+ CreateFileGetRecord(GetName(),
+ "@HWI-EAS102_1_30LWPAAXX:5:1:1511:102\n"
+ "GGGGTTAGTGGCAGGGGGGGGGTCTCGGGGGGGGGG\n"
+ "+HWI-EAS102_1_30LWPAAXX:5:1:1511:102\n"
+ "I:IIIIIIIIIIIIIIII;IIIIIIIIIIIIIIIII\n" // ':' = 58
+ );
+ REQUIRE(GetRejected());
+ REQUIRE(fatal);
+}
+
+FIXTURE_TEST_CASE(DecimalQualityRejected, LoaderFixture)
+{
+ CreateFileGetRecord(GetName(),
+ "@BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540\n"
+ "GTCGCTTCTCGGAAGNGTGAAAGACAANAATNTTNN\n"
+ "+BILLIEHOLIDAY_1_FC20F3DAAXX:8:2:342:540\n"
+ "40 3 1 22 17 18 34 8 13 21 3 7 5 0 0 5 1 0 7 3 2 3 3 3 1 1 4 5 5 2 2 5 0 1 5 5\n"
+ );
+ REQUIRE(GetRejected());
+ REQUIRE(fatal);
+}
+
+////////////////// detecting alternative formats
+FIXTURE_TEST_CASE(PacbioRaw, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19\n"
+ "AGAGTTTGAT\n"
+ "+m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19\n"
+ "LLhf>>>>[[\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19"), string(name, length));
+}
+FIXTURE_TEST_CASE(PacbioCcs, LoaderFixture)
+{
+ defaultReadNumber = -1;
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19/ccs\n"
+ "AGAGTTTGAT\n"
+ "+m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19/ccs\n"
+ "LLhf>>>>[[\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("m121205_055009_42163_c100416332550000001523041801151327_s1_p0/19/ccs"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(PacbioNoReadNumbers, LoaderFixture)
+{
+ defaultReadNumber = -1;
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@m121205_055009_42163_c100416332550000001523041801151327_s1_p0/1\n"
+ "AGAGTTTGAT\n"
+ "+m121205_055009_42163_c100416332550000001523041801151327_s1_p0/1\n"
+ "LLhf>>>>[[\n"
+ ));
+ REQUIRE(!SequenceIsFirst(seq));
+ REQUIRE(!SequenceIsSecond(seq));
+}
+
+FIXTURE_TEST_CASE(PacbioWsCcs, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@m101210_094054_00126_c000028442550000000115022402181134_s1_p0/2 ccs\n"
+ "AGAGTTTGAT\n"
+ "+m101210_094054_00126_c000028442550000000115022402181134_s1_p0/2 ccs\n"
+ "LLhf>>>>[[\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("m101210_094054_00126_c000028442550000000115022402181134_s1_p0/2"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(PacbioError, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@m130727_021351_42150_c100538232550000001823086511101336_s1_p0/53/0_106\n"
+ "TTTTTCCAAAAAAGGAGACGTAAACATTTCTTAACTTGCCAGCACTCTAATTCCAAAATCAAGTCGCATTTCTGACATTGCGGTAAGATTGTGCAATATCATATCT\n"
+ "+\n"
+ ")*'&*'+*(*+#-,-/'+-)+,'-./+*+.()*()*,$#,)'+**%+'*/,+(/,,*,'/&,+--%.-.*),,+.,,.'./%-/,/(.%.,*(.+/-/(../.&'#\n"
+ ));
+}
+
+FIXTURE_TEST_CASE(NotPairedRead_Error, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST226:170:AB075UABXX:3:1101:10089:7031 1:N:0:GCCAAT\n"
+ "TACA\n"
+ ));
+ REQUIRE(SequenceIsFirst(seq));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST226:170:AB075UABXX:3:1101:10089:7031"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(NoFragmentInfo_Error, LoaderFixture)
+{
+ defaultReadNumber = -1;
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@m130727_021351_42150_c100538232550000001823086511101336_s1_p0/283/0_9315\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("m130727_021351_42150_c100538232550000001823086511101336_s1_p0/283/0_9315"), string(name, length));
+}
+
+FIXTURE_TEST_CASE(NoColonAtTheEnd_Error, LoaderFixture)
+{
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HET-141-007:154:C391TACXX:6:2316:3220:70828 1:N:0\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE(SequenceIsFirst(seq));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HET-141-007:154:C391TACXX:6:2316:3220:70828"), string(name, length));
+}
+
+FIXTURE_TEST_CASE ( MissingRead, LoaderFixture )
+{ // source: SRR529889
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@GG3IVWD03HIDOA length=3 xy=2962_2600 region=3 run=R_2010_05_11_11_15_22_\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("GG3IVWD03HIDOA"), string(name, length));
+}
+
+FIXTURE_TEST_CASE ( UnexpectedEOLreported, LoaderFixture )
+{ // source: SRR1915965
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST1106:381:D1CDRACXX:8:1101:10000:110594\t2\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST1106:381:D1CDRACXX:8:1101:10000:110594"), string(name, length));
+ REQUIRE(SequenceIsSecond(seq));
+}
+
+FIXTURE_TEST_CASE ( AnotherUnexpectedEOLreported, LoaderFixture )
+{ // source: SRR1686805
+ REQUIRE(CreateFileGetSequence(GetName(),
+ "@HWI-ST225:626:C2Y82ACXX:3:1304:7988:75799_2\n"
+ "AACA\n+\n$.%0\n"
+ ));
+ REQUIRE_RC(SequenceGetSpotName(seq, &name, &length));
+ REQUIRE_EQ(string("HWI-ST225:626:C2Y82ACXX:3:1304:7988:75799"), string(name, length));
+ REQUIRE(SequenceIsSecond(seq));
+}
+
+
+
+// FIXTURE_TEST_CASE(Pacbio, LoaderFixture)
+// {
+ // REQUIRE(CreateFileGetSequence(GetName(),
+ // "@m120419_100821_42161_c100329130310000001523018509161273_s1_p0/11/2511_3149\n"
+ // "CTGCTTCTCCTGCTCTTCCTACTGTCCTCTCCCTGCTGTCGCTTCGCCCC\n"
+ // "TCGGTGGAGGCCGCGTTTGAGCGGCCGGTGTCCGCTGC\n"
+ // "+\n"
+ // "+*.,+*(,$+-.)**('#%*',*.*&,.(,$',,,($.,)#.-.-%'*#%\n"
+ // ",../*+*!.%!!-!/-&(.+.!.'!!//,/)!//!*-'\n"
+ // ));
+
+ // REQUIRE_RC(SequenceGetReadLength(seq, &readLength));
+ // REQUIRE_EQ(readLength, 88u);
+ // REQUIRE(MakeReadBuffer());
+ // REQUIRE_RC(SequenceGetRead(seq, read));
+ // REQUIRE_EQ(string(read, readLength), string("CTGCTTCTCCTGCTCTTCCTACTGTCCTCTCCCTGCTGTCGCTTCGCCCCTCGGTGGAGGCCGCGTTTGAGCGGCCGGTGTCCGCTGC"));
+
+ // REQUIRE_RC(SequenceGetQuality(seq, &quality, &qualityAsciiOffset, &qualityType));
+ // REQUIRE_NOT_NULL(quality);
+ // REQUIRE_EQ(qualityType, (int)QT_Phred);
+ // REQUIRE_EQ((unsigned int)qualityAsciiOffset, (unsigned int)phredOffset);
+ // REQUIRE_EQ(quality[0], (int8_t)'+');
+ // REQUIRE_EQ(quality[87], (int8_t)'\'');
+// }
+
+// @ERBRDQF01EGP9U
+// spot-name
+// spot-group
+// read-number
+// run-group
+// coords
+// fmt-name
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+rc_t CC UsageSummary (const char * progname)
+{
+ return 0;
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ return 0;
+}
+
+const char UsageDefaultName[] = "wb-test-fastq";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ KConfigDisableUserSettings();
+ rc_t rc=FastqLoaderWbTestSuite(argc, argv);
+ return rc;
+}
+
+}
diff --git a/test/general-loader/.gitignore b/test/general-loader/.gitignore
new file mode 100644
index 0000000..6fdb9d7
--- /dev/null
+++ b/test/general-loader/.gitignore
@@ -0,0 +1,3 @@
+db
+schema
+*.gw
diff --git a/test/general-loader/Makefile b/test/general-loader/Makefile
new file mode 100644
index 0000000..6fe2389
--- /dev/null
+++ b/test/general-loader/Makefile
@@ -0,0 +1,194 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/general-loader
+
+TEST_TOOLS = \
+ test-general-loader \
+ test-general-writer \
+ test-gw-dumper
+
+ALL_TOOLS = \
+ $(TEST_TOOLS) \
+
+include $(TOP)/build/Makefile.env
+
+INCDIRS += -I$(TOP)/tools/general-loader -DLOCAL_SCHEMA=$(VDB_INCDIR)
+
+$(ALL_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+.PHONY: $(TEST_TOOLS)
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# test-general-loader
+#
+TEST_GEN_LOAD_SRC = \
+ test-general-loader \
+ testsource
+
+TEST_GEN_LOAD_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_GEN_LOAD_SRC))
+
+TEST_GEN_LOAD_LIB = \
+ -sncbi-wvdb-static \
+ -sload \
+ -skapp \
+ -sktst \
+
+$(TEST_BINDIR)/test-general-loader: $(TEST_GEN_LOAD_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_GEN_LOAD_LIB)
+
+gen_load: test-general-loader
+ $(TEST_BINDIR)/test-general-loader
+
+vg_gen_load: test-general-loader
+ valgrind --ncbi $(TEST_BINDIR)/test-general-loader
+
+#-------------------------------------------------------------------------------
+# test-general-writer
+#
+TEST_GEN_WRITE_SRC = \
+ test-general-writer
+
+TEST_GEN_WRITE_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_GEN_WRITE_SRC))
+
+TEST_GEN_WRITE_LIB = \
+ -L$(LIBDIR) \
+ -sgeneral-writer
+
+
+$(TEST_BINDIR)/test-general-writer: $(TEST_GEN_WRITE_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_GEN_WRITE_LIB)
+
+vg_gen_write: test-general-writer
+ valgrind --ncbi $(TEST_BINDIR)/test-general-writer -o gen-write-out
+ rm -f gen-write-out
+
+
+runtests:$(TEST_TOOLS) #piped-load-tests
+
+piped-load-tests: $(ALL_TOOLS)
+ -$(TEST_BINDIR)/test-general-writer | $(BINDIR)/general-loader -L=info
+ -$(BINDIR)/vdb-dump -T table1 remote_db
+ rm -rf remote_db
+
+#-------------------------------------------------------------------------------
+# test-gw-dumper
+#
+TEST_GW_DUMP_SRC = \
+ test-gw-dumper
+
+TEST_GW_DUMP_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_GW_DUMP_SRC))
+
+TEST_GW_DUMP_LIB = \
+ -sgeneral-writer \
+
+$(TEST_BINDIR)/test-gw-dumper: $(TEST_GW_DUMP_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_GW_DUMP_LIB)
+
+gw_dump: test-gw-dumper
+ $(TEST_BINDIR)/$^
+
+vg_gw_dump: test-gw-dumper
+ valgrind --ncbi $(TEST_BINDIR)/test-gw-dumper
+
+#-------------------------------------------------------------------------------
+# general-loader tool tests
+#
+runtests: setup tool-tests
+
+setup:
+ @ mkdir -p db
+
+$(TEST_BINDIR)/makeinputs: makeinputs.cpp
+ @# generate numbered input streams for tool tests
+ $(CXX) makeinputs.cpp testsource.cpp -o $(TEST_BINDIR)/makeinputs -I$(INCDIRS) -L$(VDB_LIBDIR) -lncbi-vdb-static
+
+tool-tests: $(TEST_BINDIR)/makeinputs
+ @mkdir -p input && $(TEST_BINDIR)/makeinputs && rm $(TEST_BINDIR)/makeinputs
+ @#
+ @# basic, unpacked
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 1 0 -I=$(VDB_INCDIR)
+ @#
+ @# basic, packed
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 1packed 0 -I=$(VDB_INCDIR)
+ @#
+ @# target db override
+ @# use 1.gl which specifies the target as actual/1/db, and override that to where the script expects to see the result
+ @cp input/1.gl input/1override.gl && $(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 1override 0 "-I=$(VDB_INCDIR) --target actual/1override/db"
+ @# same with the storthand option
+ @cp input/1.gl input/1override.gl && $(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 1override 0 "-I=$(VDB_INCDIR) -T actual/1override/db"
+ @#
+ @# error message event, unpacked
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 2 3 "-I=$(VDB_INCDIR) -L=err"
+ @#
+ @# error message event, packed
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 2packed 3 "-I=$(VDB_INCDIR) -L=err"
+ @#
+ @# empty default values, unpacked
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 3 0 -I=$(VDB_INCDIR)
+ @#
+ @# empty default values, packed
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 3packed 0 -I=$(VDB_INCDIR)
+ @#
+ @# move ahead event, unpacked
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 4 0 -I=$(VDB_INCDIR)
+ @#
+ @# move ahead event, packed
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 4packed 0 -I=$(VDB_INCDIR)
+ @#
+ @# integer compression (packed only)
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) vdb-dump $(SRCDIR) 5packed 0 "-L=info -I=$(VDB_INCDIR)"
+ @#
+ @# software name in the metadata, unpacked
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) kdbmeta $(SRCDIR) 6 0 "-L=info -I=$(VDB_INCDIR)" SOFTWARE/formatter
+ @#
+ @# software name in the metadata, packed
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) kdbmeta $(SRCDIR) 6packed 0 "-L=info -I=$(VDB_INCDIR)" SOFTWARE/formatter
+ @#
+ @rm -rf input/*.gl actual/*
+
+one-tool: $(TEST_BINDIR)/makeinputs
+ @mkdir -p input && $(TEST_BINDIR)/makeinputs && rm $(TEST_BINDIR)/makeinputs
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) kdbmeta $(SRCDIR) 6 0 "-L=info -I=$(VDB_INCDIR)" SOFTWARE/formatter
+
+
+#-------------------------------------------------------------------------------
+# general-loader tool tests
+#
+slowtests: #piped-dump
+
+piped-dump: $(BINDIR)/pileup-stats test-general-loader
+ #$(BINDIR)/pileup-stats SRR1063272 | $(BINDIR)/general-loader #-L=info
+ $(BINDIR)/pileup-stats SRR1232302 | $(BINDIR)/general-loader -L=info
+
diff --git a/test/general-loader/column01 b/test/general-loader/column01
new file mode 100644
index 0000000..41713c2
--- /dev/null
+++ b/test/general-loader/column01
@@ -0,0 +1,3 @@
+First test sentence
+Second sentence to test the stream
+Third
\ No newline at end of file
diff --git a/test/general-loader/column02 b/test/general-loader/column02
new file mode 100644
index 0000000..41713c2
--- /dev/null
+++ b/test/general-loader/column02
@@ -0,0 +1,3 @@
+First test sentence
+Second sentence to test the stream
+Third
\ No newline at end of file
diff --git a/test/general-loader/expected/1.stdout b/test/general-loader/expected/1.stdout
new file mode 100644
index 0000000..cd95218
--- /dev/null
+++ b/test/general-loader/expected/1.stdout
@@ -0,0 +1,75 @@
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first value
+ SPOT_ID: 1
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first default
+ SPOT_ID: 2
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first default
+ SPOT_ID: 3
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: second value
+ SPOT_ID: 4
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: second default
+ SPOT_ID: 5
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/1override.stdout b/test/general-loader/expected/1override.stdout
new file mode 100644
index 0000000..cd95218
--- /dev/null
+++ b/test/general-loader/expected/1override.stdout
@@ -0,0 +1,75 @@
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first value
+ SPOT_ID: 1
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first default
+ SPOT_ID: 2
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first default
+ SPOT_ID: 3
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: second value
+ SPOT_ID: 4
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: second default
+ SPOT_ID: 5
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/1packed.stdout b/test/general-loader/expected/1packed.stdout
new file mode 100644
index 0000000..cd95218
--- /dev/null
+++ b/test/general-loader/expected/1packed.stdout
@@ -0,0 +1,75 @@
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first value
+ SPOT_ID: 1
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first default
+ SPOT_ID: 2
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: first default
+ SPOT_ID: 3
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: second value
+ SPOT_ID: 4
+ TRIM_START: 0
+
+COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP: second default
+ SPOT_ID: 5
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/2.stderr b/test/general-loader/expected/2.stderr
new file mode 100644
index 0000000..bac063e
--- /dev/null
+++ b/test/general-loader/expected/2.stderr
@@ -0,0 +1,2 @@
+general-loader.2.5.7 err: general-loader: error "something is wrong"
+general-loader.2.5.7 err: error exists while reading file - load failed
diff --git a/test/general-loader/expected/2packed.stderr b/test/general-loader/expected/2packed.stderr
new file mode 100644
index 0000000..bac063e
--- /dev/null
+++ b/test/general-loader/expected/2packed.stderr
@@ -0,0 +1,2 @@
+general-loader.2.5.7 err: general-loader: error "something is wrong"
+general-loader.2.5.7 err: error exists while reading file - load failed
diff --git a/test/general-loader/expected/3.stdout b/test/general-loader/expected/3.stdout
new file mode 100644
index 0000000..3481de0
--- /dev/null
+++ b/test/general-loader/expected/3.stdout
@@ -0,0 +1,16 @@
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/3packed.stdout b/test/general-loader/expected/3packed.stdout
new file mode 100644
index 0000000..3481de0
--- /dev/null
+++ b/test/general-loader/expected/3packed.stdout
@@ -0,0 +1,16 @@
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/4.stdout b/test/general-loader/expected/4.stdout
new file mode 100644
index 0000000..c05dd30
--- /dev/null
+++ b/test/general-loader/expected/4.stdout
@@ -0,0 +1,160 @@
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 2
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 3
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 4
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 5
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS: 100
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 6
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 7
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 8
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 9
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 10
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/4packed.stdout b/test/general-loader/expected/4packed.stdout
new file mode 100644
index 0000000..c05dd30
--- /dev/null
+++ b/test/general-loader/expected/4packed.stdout
@@ -0,0 +1,160 @@
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 2
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 3
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 4
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 5
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS: 100
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 6
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 7
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 8
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 9
+ TRIM_START: 0
+
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS:
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 10
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/5packed.stdout b/test/general-loader/expected/5packed.stdout
new file mode 100644
index 0000000..d8acc2e
--- /dev/null
+++ b/test/general-loader/expected/5packed.stdout
@@ -0,0 +1,16 @@
+ COLOR_MATRIX: 0, 1, 2, 3, 4, 1, 0, 3, 2, 4, 2, 3, 0, 1, 4, 3, 2, 1, 0, 4, 4, 4, 4, 4, 4
+ CS_KEY: T
+ CS_NATIVE: false
+ LABEL: reference
+ LABEL_LEN: 9
+ LABEL_START: 0
+ MIN_SPOT_ID: 1
+PRIMARY_ALIGNMENT_IDS: 1085102592571150095
+ RD_FILTER: SRA_READ_FILTER_PASS
+ READ_FILTER: SRA_READ_FILTER_PASS
+ READ_START: 0
+ READ_TYPE: SRA_READ_TYPE_BIOLOGICAL|SRA_READ_TYPE_FORWARD
+ SPOT_GROUP:
+ SPOT_ID: 1
+ TRIM_START: 0
+
diff --git a/test/general-loader/expected/6.stdout b/test/general-loader/expected/6.stdout
new file mode 100644
index 0000000..b8bf71a
--- /dev/null
+++ b/test/general-loader/expected/6.stdout
@@ -0,0 +1 @@
+<formatter name="some software" vers="9.8.7654"/>
diff --git a/test/general-loader/expected/6packed.stdout b/test/general-loader/expected/6packed.stdout
new file mode 100644
index 0000000..b8bf71a
--- /dev/null
+++ b/test/general-loader/expected/6packed.stdout
@@ -0,0 +1 @@
+<formatter name="some software" vers="9.8.7654"/>
diff --git a/test/general-loader/makeinputs.cpp b/test/general-loader/makeinputs.cpp
new file mode 100644
index 0000000..c9d739a
--- /dev/null
+++ b/test/general-loader/makeinputs.cpp
@@ -0,0 +1,226 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for General Loader
+*/
+#include "testsource.hpp"
+
+#include "../../tools/general-loader/utf8-like-int-codec.c"
+
+using namespace std;
+
+const char* tableName = "REFERENCE";
+const char* asciiColumnName = "SPOT_GROUP";
+const char* i64ColumnName = "PRIMARY_ALIGNMENT_IDS";
+const char* u32ColumnName = "SPOT_ID";
+
+
+static
+string
+DatabasePath ( const string& p_caseId, bool p_packed )
+{
+ string caseId = p_caseId;
+ if ( p_packed )
+ {
+ caseId += "packed";
+ }
+ return string ( "actual/" ) + caseId + "/db";
+}
+
+static
+string
+OutputPath ( const string& p_caseId, bool p_packed )
+{
+ string caseId = p_caseId;
+ if ( p_packed )
+ {
+ caseId += "packed";
+ }
+ return string ( "input/" ) + caseId + ".gl";
+}
+
+void
+OneTableOneColumnWithDefaults( const string& p_caseId, bool p_packed )
+{ // default values changed mid-data
+ TestSource source;
+ TestSource::packed = p_packed;
+
+ source . SchemaEvent ( "align/align.vschema", "NCBI:align:db:alignment_sorted" );
+ source . DatabaseEvent ( DatabasePath ( p_caseId, p_packed ) );
+
+ source . NewTableEvent ( 1, tableName );
+ source . NewColumnEvent ( 1, 1, asciiColumnName, 8 );
+ source . OpenStreamEvent();
+
+ string value1 = "first value";
+ source . CellDataEvent( 1, value1 );
+ source . NextRowEvent ( 1 );
+
+ string default1 = "first default";
+ source . CellDefaultEvent( 1, default1 );
+ source . NextRowEvent ( 1 );
+
+ source . NextRowEvent ( 1 );
+
+ string value2 = "second value";
+ source . CellDefaultEvent( 1, value2 );
+ source . NextRowEvent ( 1 );
+
+ string default2 = "second default";
+ source . CellDefaultEvent( 1, default2 );
+ source . NextRowEvent ( 1 );
+
+ source . CloseStreamEvent();
+ source . SaveBuffer ( OutputPath ( p_caseId, p_packed ) . c_str () );
+}
+
+void
+ErrorMessage( const string& p_caseId, bool p_packed )
+{
+ TestSource source;
+ TestSource::packed = p_packed;
+
+ source . SchemaEvent ( "align/align.vschema", "NCBI:align:db:alignment_sorted" );
+ source . DatabaseEvent ( DatabasePath ( p_caseId, p_packed ) );
+
+ source . NewTableEvent ( 1, tableName );
+ source . NewColumnEvent ( 1, 1, asciiColumnName, 8 );
+ source . OpenStreamEvent();
+
+ source . ErrorMessageEvent( "something is wrong" );
+
+ source . CloseStreamEvent();
+ source . SaveBuffer ( OutputPath ( p_caseId, p_packed ) . c_str () );
+}
+
+void
+EmptyDefaultValue( const string& p_caseId, bool p_packed )
+{
+ TestSource source;
+ TestSource::packed = p_packed;
+
+ source . SchemaEvent ( "align/align.vschema", "NCBI:align:db:alignment_sorted" );
+ source . DatabaseEvent ( DatabasePath ( p_caseId, p_packed ) );
+
+ source . NewTableEvent ( 1, tableName );
+ source . NewColumnEvent ( 1, 1, i64ColumnName, 64 );
+ source . OpenStreamEvent();
+
+ source . CellEmptyDefaultEvent( 1 );
+ source . NextRowEvent ( 1 );
+
+ source . CloseStreamEvent();
+ source . SaveBuffer ( OutputPath ( p_caseId, p_packed ) . c_str () );
+}
+
+void
+MoveAhead( const string& p_caseId, bool p_packed )
+{
+ TestSource source;
+ TestSource::packed = p_packed;
+
+ source . SchemaEvent ( "align/align.vschema", "NCBI:align:db:alignment_sorted" );
+ source . DatabaseEvent ( DatabasePath ( p_caseId, p_packed ) );
+
+ source . NewTableEvent ( 1, tableName );
+ source . NewColumnEvent ( 1, 1, i64ColumnName, 64 );
+ source . OpenStreamEvent();
+
+ source . CellEmptyDefaultEvent( 1 );
+ source . MoveAheadEvent ( 1, 5 );
+
+ source . CellDataEvent( 1, (int64_t)100 );
+ source . MoveAheadEvent ( 1, 5 );
+
+ // expected: 5 empty cells, 100, 4 empty cells
+ source . CloseStreamEvent();
+ source . SaveBuffer ( OutputPath ( p_caseId, p_packed ) . c_str () );
+}
+
+void
+IntegerCompression( const string& p_caseId, bool p_packed )
+{
+ if ( !p_packed )
+ { // Integer compression is used in packed mode only
+ return;
+ }
+
+ TestSource source;
+ TestSource::packed = true;
+
+ source . SchemaEvent ( "align/align.vschema", "NCBI:align:db:alignment_sorted" );
+ source . DatabaseEvent ( DatabasePath ( p_caseId, p_packed ) );
+
+ source . NewTableEvent ( 1, tableName );
+ source . NewColumnEvent ( 1, 1, i64ColumnName, 64, true );
+ source . OpenStreamEvent();
+
+ uint8_t buf[128];
+ int bytes = encode_uint64 ( 0x0F0F0F0F0F0F0F0FLL, buf, buf + sizeof buf );
+ source . CellDataEventRaw ( 1, 1, buf, bytes );
+ source . NextRowEvent ( 1 );
+
+ source . CloseStreamEvent();
+ source . SaveBuffer ( OutputPath ( p_caseId, p_packed ) . c_str () );
+}
+
+void
+SoftwareIdString( const string& p_caseId, bool p_packed )
+{ // Software identification string in metadata
+ TestSource source;
+ TestSource::packed = true;
+
+ source . SchemaEvent ( "align/align.vschema", "NCBI:align:db:alignment_sorted" );
+ source . DatabaseEvent ( DatabasePath ( p_caseId, p_packed ) );
+
+ source . SoftwareNameEvent ( string ( "some software" ), string ( "9.8.7654" ) );
+
+ source . OpenStreamEvent();
+ source . CloseStreamEvent();
+
+ source . SaveBuffer ( OutputPath ( p_caseId, p_packed ) . c_str () );
+}
+
+int main()
+{
+ for (bool packed = false; ; packed = true )
+ {
+ OneTableOneColumnWithDefaults( "1", packed );
+ ErrorMessage( "2", packed );
+ EmptyDefaultValue( "3", packed );
+ MoveAhead( "4", packed );
+ IntegerCompression( "5", packed );
+ SoftwareIdString( "6", packed );
+
+ if ( packed )
+ {
+ break;
+ }
+ }
+
+ return 0;
+}
diff --git a/test/general-loader/runtestcase.sh b/test/general-loader/runtestcase.sh
new file mode 100755
index 0000000..776aa08
--- /dev/null
+++ b/test/general-loader/runtestcase.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+#echo "$0 $*"
+
+# $1 - path to sra-tools executables (general-loader, vdb-dump)
+# $2 - name of the dumper executable (vdb-dump, kdbmeta etc.)
+# $3 - work directory (expected results under expected/, actual results and temporaries created under actual/)
+# $4 - test case ID (expect a file input/$3.gl to exist)
+# $5 - expected return code
+# $6 - command line options for general-loader
+# $7 - command line options for dumper
+#
+# return codes:
+# 0 - passed
+# 1 - could not create temp dir
+# 2 - unexpected return code from general-loader
+# 3 - vdb-dump failed on the output of general-loader
+# 4 - outputs differ
+
+BINDIR=$1
+DUMPER=$2
+WORKDIR=$3
+CASEID=$4
+RC=$5
+LOAD_OPTIONS=$6
+DUMP_OPTIONS=$7
+
+DUMP="$BINDIR/$DUMPER"
+LOAD="$BINDIR/general-loader"
+TEMPDIR=$WORKDIR/actual/$CASEID
+
+echo "running test case $CASEID"
+
+mkdir -p $TEMPDIR
+if [ "$?" != "0" ] ; then
+ echo "cannot create "
+ exit 1
+fi
+rm -rf $TEMPDIR/*
+
+CMD="cat input/$CASEID.gl | $LOAD $LOAD_OPTIONS 1>$TEMPDIR/load.stdout 2>$TEMPDIR/load.stderr"
+#echo $CMD
+eval $CMD
+rc="$?"
+if [ "$rc" != "$RC" ] ; then
+ echo "$LOAD returned $rc, expected $RC"
+ echo "command executed:"
+ echo $CMD
+ cat $TEMPDIR/load.stderr
+ exit 2
+fi
+
+if [ "$rc" == "0" ] ; then
+ CMD="$DUMP $TEMPDIR/db $DUMP_OPTIONS 1>$TEMPDIR/dump.stdout 2>$TEMPDIR/dump.stderr"
+ #echo $CMD
+ eval $CMD || ( echo "$CMD" && exit 3 )
+
+ # remove timestamps, date from metadata
+ sed -i -e 's/<timestamp>.*<\/timestamp>/<timestamp\/>/g' $TEMPDIR/dump.stdout
+ sed -i -e 's/date=".*" name/date="" name/g' $TEMPDIR/dump.stdout
+
+ diff $WORKDIR/expected/$CASEID.stdout $TEMPDIR/dump.stdout >$TEMPDIR/diff
+ rc="$?"
+else
+ # remove timestamps
+ sed -i -e 's/^....-..-..T..:..:.. //g' $TEMPDIR/load.stderr
+ # remove pathnames
+ sed -i -e 's=/.*/==g' $TEMPDIR/load.stderr
+ # remove source locations
+ sed -i -e 's=: .*:[0-9]*:[^ ]*:=:=g' $TEMPDIR/load.stderr
+ # remove version number
+ sed -i -e 's=latf-load\(\.[0-9]*\)*=latf-load=g' $TEMPDIR/load.stderr
+ diff $WORKDIR/expected/$CASEID.stderr $TEMPDIR/load.stderr >$TEMPDIR/diff
+ rc="$?"
+fi
+
+if [ "$rc" != "0" ] ; then
+ cat $TEMPDIR/diff
+ echo "command executed:"
+ echo $CMD
+ exit 4
+fi
+
+rm -rf $TEMPDIR
+
+exit 0
diff --git a/test/general-loader/schema/.gitignore b/test/general-loader/schema/.gitignore
new file mode 100644
index 0000000..4063701
--- /dev/null
+++ b/test/general-loader/schema/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/test/general-loader/test-general-loader.cpp b/test/general-loader/test-general-loader.cpp
new file mode 100644
index 0000000..f425eda
--- /dev/null
+++ b/test/general-loader/test-general-loader.cpp
@@ -0,0 +1,1698 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for General Loader
+*/
+
+#include "../../tools/general-loader/general-loader.cpp"
+#include "../../tools/general-loader/database-loader.cpp"
+#include "../../tools/general-loader/protocol-parser.cpp"
+#include "../../tools/general-loader/utf8-like-int-codec.c"
+
+#include <ktst/unit_test.hpp>
+
+#include <sysalloc.h>
+
+#include <klib/printf.h>
+#include <klib/debug.h>
+
+#include <kns/adapt.h>
+
+#include <kfs/nullfile.h>
+#include <kfs/file.h>
+#include <kfs/ramfile.h>
+
+#include <kdb/table.h>
+
+#include <kfg/config.h> // KConfigPrint
+
+#include <vfs/path.h>
+#include <vfs/path-priv.h>
+#include <vfs/manager.h>
+
+#include <vdb/vdb-priv.h>
+
+#include <time.h>
+#include <cstring>
+#include <stdexcept>
+#include <map>
+#include <fstream>
+#include <cstdio>
+
+#include "testsource.hpp"
+
+using namespace std;
+using namespace ncbi::NK;
+
+static KLogLevel l = 4;
+static rc_t argsHandler(int argc, char* argv[]) {
+ Args* args = NULL;
+ rc_t rc = ArgsMakeAndHandle(&args, argc, argv, 0, NULL, 0);
+ ArgsWhack(args);
+ KLogLevel lv = KLogLevelGet();
+ if (lv != 4) {
+ l = lv;
+ }
+ return rc;
+}
+TEST_SUITE_WITH_ARGS_HANDLER(GeneralLoaderTestSuite, argsHandler);
+
+const string ScratchDir = "./db/";
+
+static
+void
+ClearScratchDir ()
+{
+ KDirectory* wd;
+ KDirectoryNativeDir ( & wd );
+ KDirectoryClearDir ( wd, true, ScratchDir . c_str() );
+ KDirectoryRelease ( wd );
+}
+
+
+class GeneralLoaderFixture
+{
+public:
+ static const uint32_t DefaultTableId = 100;
+
+ static const uint32_t DefaultColumnId = 1;
+ static const uint32_t Column16Id = 16;
+ static const uint32_t Column32Id = 32;
+ static const uint32_t Column64Id = 64;
+
+ static std::string argv0;
+
+public:
+ GeneralLoaderFixture()
+ : m_db ( 0 ),
+ m_cursor ( 0 ),
+ m_wd ( 0 )
+ {
+ THROW_ON_RC ( KDirectoryNativeDir ( & m_wd ) );
+
+#ifdef LOCAL_SCHEMA
+ {
+ VFSManager * vfs;
+ VFSManagerMake ( & vfs );
+ VPath* path;
+ VFSManagerMakeSysPath ( vfs, &path, stringize ( LOCAL_SCHEMA ) );
+ const String *uri = NULL;
+ VPathMakeString ( path, &uri );
+ m_schemaDir = string( uri->addr, uri->size );
+ VPathRelease ( path );
+ VFSManagerRelease ( vfs );
+ }
+#endif
+
+ }
+ ~GeneralLoaderFixture()
+ {
+ RemoveDatabase();
+ if (l == 4) {
+ KLogLevelSet ( klogFatal );
+ }
+ KDirectoryRelease ( m_wd );
+ if ( ! m_tempSchemaFile . empty() )
+ {
+ remove ( m_tempSchemaFile . c_str() );
+ }
+ }
+
+ GeneralLoader* MakeLoader ( const struct KFile * p_input )
+ {
+ struct KStream * inStream;
+ THROW_ON_RC ( KStreamFromKFilePair ( & inStream, p_input, 0 ) );
+
+ GeneralLoader* ret = new GeneralLoader ( argv0, * inStream );
+ if ( ! m_schemaDir.empty() )
+ {
+ ret -> AddSchemaIncludePath ( m_schemaDir );
+ }
+
+ THROW_ON_RC ( KStreamRelease ( inStream ) );
+ THROW_ON_RC ( KFileRelease ( p_input ) );
+
+ return ret;
+ }
+ bool RunLoader ( GeneralLoader& p_loader, rc_t p_rc )
+ {
+ rc_t rc = p_loader.Run();
+ if ( rc == p_rc )
+ {
+ return true;
+ }
+
+ char buf[1024];
+ string_printf ( buf, sizeof buf, NULL, "Expected rc='%R', actual='%R'", p_rc, rc );
+ cerr << buf << endl;
+ return false;
+ }
+
+ bool Run ( const struct KFile * p_input, rc_t p_rc )
+ {
+ struct KStream* inStream;
+ THROW_ON_RC ( KStreamFromKFilePair ( & inStream, p_input, 0 ) );
+
+ GeneralLoader gl ( argv0, *inStream );
+ if ( ! m_schemaDir.empty() )
+ {
+ gl . AddSchemaIncludePath ( m_schemaDir );
+ }
+
+ rc_t rc = gl.Run();
+ bool ret;
+ if ( GetRCObject ( rc ) == GetRCObject ( p_rc ) &&
+ GetRCState ( rc ) == GetRCState ( p_rc ) )
+ {
+ ret = true;
+ }
+ else
+ {
+ char buf[1024];
+ string_printf ( buf, sizeof buf, NULL, "Expected rc='%R', actual='%R'", p_rc, rc );
+ cerr << buf << endl;
+ ret = false;
+ }
+
+ THROW_ON_RC ( KStreamRelease ( inStream ) );
+ THROW_ON_RC ( KFileRelease ( p_input ) );
+
+ return ret;
+ }
+
+ void RemoveDatabase()
+ {
+ CloseDatabase();
+ if ( ! m_source . GetDatabaseName() . empty () )
+ {
+ KDirectoryRemove ( m_wd, true, m_source . GetDatabaseName() . c_str() );
+ }
+ }
+
+ void OpenDatabase( const char* p_dbNameOverride = 0 )
+ {
+ CloseDatabase();
+
+ VDBManager * vdb;
+ THROW_ON_RC ( VDBManagerMakeUpdate ( & vdb, NULL ) );
+
+ try
+ {
+ THROW_ON_RC ( VDBManagerOpenDBUpdate ( vdb, &m_db, NULL, p_dbNameOverride != 0 ? p_dbNameOverride : m_source . GetDatabaseName() . c_str() ) );
+ }
+ catch(...)
+ {
+ VDBManagerRelease ( vdb );
+ throw;
+ }
+
+ THROW_ON_RC ( VDBManagerRelease ( vdb ) );
+ }
+ void CloseDatabase()
+ {
+ if ( m_db != 0 )
+ {
+ VDatabaseRelease ( m_db );
+ m_db = 0;
+ }
+ if ( m_cursor != 0 )
+ {
+ VCursorRelease ( m_cursor );
+ m_cursor = 0;
+ }
+ }
+
+ void SetUpStream( const char* p_dbName, const string& p_schema = "align/align.vschema", const string& p_schemaName = "NCBI:align:db:alignment_sorted" )
+ {
+ m_source . SchemaEvent ( p_schema, p_schemaName );
+ string dbName = ScratchDir + p_dbName;
+ if ( m_source.packed )
+ {
+ dbName += "-packed";
+ }
+ m_source . DatabaseEvent ( dbName );
+ }
+ void SetUpStream_OneTable( const char* p_dbName, const char* p_tableName )
+ {
+ SetUpStream( p_dbName );
+ m_source . NewTableEvent ( DefaultTableId, p_tableName );
+ }
+ void OpenStream_OneTableOneColumn ( const char* p_dbName, const char* p_tableName, const char* p_columnName, size_t p_elemBits )
+ {
+ SetUpStream_OneTable( p_dbName, p_tableName );
+ m_source . NewColumnEvent ( DefaultColumnId, DefaultTableId, p_columnName, ( uint32_t ) p_elemBits );
+ m_source . OpenStreamEvent();
+ }
+
+ bool SetUpForIntegerCompression( const char* p_dbName )
+ {
+ if ( ! TestSource::packed )
+ return false; // integer compaction is used in packed mode only
+
+ m_tempSchemaFile = string ( p_dbName ) + ".vschema";
+ string schemaText =
+ "table table1 #1.0.0\n"
+ "{\n"
+ " column U16 column16;\n"
+ " column U32 column32;\n"
+ " column U64 column64;\n"
+ "};\n"
+ "database database1 #1\n"
+ "{\n"
+ " table table1 #1 TABLE1;\n"
+ "};\n"
+ ;
+ CreateFile ( m_tempSchemaFile, schemaText );
+ SetUpStream ( p_dbName, m_tempSchemaFile, "database1" );
+
+ m_source . NewTableEvent ( DefaultTableId, "TABLE1" );
+
+ m_source . NewColumnEvent ( Column16Id, DefaultTableId, "column16", 16, true );
+ m_source . NewColumnEvent ( Column32Id, DefaultTableId, "column32", 32, true );
+ m_source . NewColumnEvent ( Column64Id, DefaultTableId, "column64", 64, true );
+
+ return true;
+ }
+
+ void OpenCursor( const char* p_table, const char* p_column )
+ {
+ OpenDatabase();
+ const VTable * tbl;
+ THROW_ON_RC ( VDatabaseOpenTableRead ( m_db, &tbl, p_table ) );
+ THROW_ON_RC ( VTableCreateCursorRead ( tbl, & m_cursor ) );
+
+ uint32_t idx;
+ THROW_ON_RC ( VCursorAddColumn ( m_cursor, &idx, p_column ) );
+ THROW_ON_RC ( VCursorOpen ( m_cursor ) );
+ THROW_ON_RC ( VTableRelease ( tbl ) );
+ }
+
+ template < typename T > T GetValue ( const char* p_table, const char* p_column, uint64_t p_row )
+ {
+ OpenCursor( p_table, p_column );
+ THROW_ON_RC ( VCursorSetRowId ( m_cursor, p_row ) );
+ THROW_ON_RC ( VCursorOpenRow ( m_cursor ) );
+
+ T ret;
+ uint32_t num_read;
+ THROW_ON_RC ( VCursorRead ( m_cursor, 1, 8 * sizeof ( T ), &ret, 1, &num_read ) );
+ THROW_ON_RC ( VCursorCloseRow ( m_cursor ) );
+ return ret;
+ }
+
+ template < typename T > bool IsNullValue ( const char* p_table, const char* p_column, uint64_t p_row )
+ {
+ OpenCursor( p_table, p_column );
+ THROW_ON_RC ( VCursorSetRowId ( m_cursor, p_row ) );
+ THROW_ON_RC ( VCursorOpenRow ( m_cursor ) );
+
+ T ret;
+ uint32_t num_read;
+ THROW_ON_RC ( VCursorRead ( m_cursor, 1, 8 * sizeof ( T ), &ret, 1, &num_read ) );
+ THROW_ON_RC ( VCursorCloseRow ( m_cursor ) );
+ return num_read == 0;
+ }
+
+ template < typename T > T GetValueWithIndex ( const char* p_table, const char* p_column, uint64_t p_row, uint32_t p_count, size_t p_index )
+ {
+ OpenCursor( p_table, p_column );
+ THROW_ON_RC ( VCursorSetRowId ( m_cursor, p_row ) );
+ THROW_ON_RC ( VCursorOpenRow ( m_cursor ) );
+
+ assert(1024 >= p_count);
+ T ret [ 1024 ];
+
+ uint32_t num_read;
+ THROW_ON_RC ( VCursorRead ( m_cursor, 1, (uint32_t) ( 8 * sizeof ( T ) ), &ret, p_count, &num_read ) );
+ THROW_ON_RC ( VCursorCloseRow ( m_cursor ) );
+ return ret [ p_index ];
+ }
+
+ void FullLog()
+ {
+ KLogLevelSet ( klogInfo );
+ }
+
+ void CreateFile ( const string& p_name, const string& p_content )
+ {
+ ofstream out( p_name . c_str() );
+ out << p_content;
+ }
+
+ std::string GetDbMetadataAttr ( const std::string& p_node, const std::string& p_attr )
+ {
+ const KMetadata *meta;
+ THROW_ON_RC ( VDatabaseOpenMetadataRead ( m_db, &meta ) );
+
+ const KMDataNode *node;
+ THROW_ON_RC ( KMetadataOpenNodeRead ( meta, &node, p_node.c_str() ) );
+
+ size_t num_read;
+ char attr [ 256 ];
+ THROW_ON_RC ( KMDataNodeReadAttr ( node, p_attr.c_str(), attr, sizeof attr, & num_read ) );
+ THROW_ON_RC ( KMDataNodeRelease ( node ) );
+ THROW_ON_RC ( KMetadataRelease ( meta ) );
+ return string ( attr, num_read );
+ }
+
+ std::string GetMetadata ( const KMetadata* p_meta, const std::string& p_node )
+ {
+ const KMDataNode *node;
+ THROW_ON_RC ( KMetadataOpenNodeRead ( p_meta, &node, p_node.c_str() ) );
+
+ size_t num_read;
+ char buf[ 256 ];
+
+ THROW_ON_RC ( KMDataNodeReadCString ( node, buf, sizeof buf, &num_read ) );
+
+ THROW_ON_RC ( KMDataNodeRelease ( node ) );
+ return string ( buf, num_read );
+
+ }
+ std::string GetDbMetadata ( VDatabase* p_db, const std::string& p_node )
+ {
+ const KMetadata *meta;
+ THROW_ON_RC ( VDatabaseOpenMetadataRead ( p_db, &meta ) );
+ string ret = GetMetadata ( meta, p_node );
+ THROW_ON_RC ( KMetadataRelease ( meta ) );
+ return ret;
+ }
+
+ TestSource m_source;
+ VDatabase * m_db;
+ const VCursor * m_cursor;
+ KDirectory* m_wd;
+ string m_tempSchemaFile;
+ string m_schemaDir;
+};
+
+template<> std::string GeneralLoaderFixture::GetValue ( const char* p_table, const char* p_column, uint64_t p_row )
+{
+ OpenCursor( p_table, p_column );
+ THROW_ON_RC ( VCursorSetRowId ( m_cursor, p_row ) );
+ THROW_ON_RC ( VCursorOpenRow ( m_cursor ) );
+
+ char buf[1024];
+ uint32_t num_read;
+ THROW_ON_RC ( VCursorRead ( m_cursor, 1, 8, &buf, sizeof buf, &num_read ) );
+ THROW_ON_RC ( VCursorCloseRow ( m_cursor ) );
+ return string ( buf, num_read );
+}
+
+std::string GeneralLoaderFixture :: argv0;
+
+const char* tableName = "REFERENCE";
+const char* columnName = "SPOT_GROUP";
+
+FIXTURE_TEST_CASE ( EmptyInput, GeneralLoaderFixture )
+{
+ const struct KFile * input;
+ REQUIRE_RC ( KFileMakeNullRead ( & input ) );
+ REQUIRE ( Run ( input, SILENT_RC ( rcNS, rcFile, rcReading, rcTransfer, rcIncomplete ) ) );
+}
+
+FIXTURE_TEST_CASE ( ShortInput, GeneralLoaderFixture )
+{
+ char buffer[10];
+ const struct KFile * input;
+ REQUIRE_RC ( KRamFileMakeRead ( & input, buffer, sizeof buffer ) );
+
+ REQUIRE ( Run ( input, SILENT_RC ( rcNS, rcFile, rcReading, rcTransfer, rcIncomplete ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadSignature, GeneralLoaderFixture )
+{
+ TestSource ts ( "badsigna" );
+ REQUIRE ( Run ( ts . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcHeader, rcCorrupt ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadEndianness, GeneralLoaderFixture )
+{
+ TestSource ts ( GeneralLoaderSignatureString, 3 );
+ REQUIRE ( Run ( ts . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcFormat, rcInvalid) ) );
+}
+
+FIXTURE_TEST_CASE ( ReverseEndianness, GeneralLoaderFixture )
+{
+ TestSource ts ( GeneralLoaderSignatureString, GW_REVERSE_ENDIAN );
+ REQUIRE ( Run ( ts . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcFormat, rcUnsupported ) ) );
+}
+
+FIXTURE_TEST_CASE ( LaterVersion, GeneralLoaderFixture )
+{
+ TestSource ts ( GeneralLoaderSignatureString, GW_GOOD_ENDIAN, GW_CURRENT_VERSION + 1 );
+ REQUIRE ( Run ( ts . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcHeader, rcBadVersion ) ) );
+}
+
+//TODO: MakeTruncatedSource (stop in the middle of an event)
+
+
+FIXTURE_TEST_CASE ( BadSchemaFileName, GeneralLoaderFixture )
+{
+ m_source . SchemaEvent ( "this file should not exist", "someSchemaName" );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcMgr, rcCreating, rcSchema, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadSchemaFileName_Long, GeneralLoaderFixture )
+{
+ m_source . SchemaEvent ( string ( GeneralLoader :: MaxPackedString + 1, 'x' ), "someSchemaName" );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcMgr, rcCreating, rcSchema, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadSchemaName, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName(), "align/align.vschema", "bad schema name" );
+ m_source . OpenStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcMgr, rcCreating, rcSchema, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadSchemaName_Long, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName(), "align/align.vschema", string ( GeneralLoader :: MaxPackedString + 1, 'x' ) );
+ m_source . OpenStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcMgr, rcCreating, rcSchema, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadTableName, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . NewTableEvent ( 1, "nosuchtable" );
+ m_source . OpenStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcMgr, rcCreating, rcSchema, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadTableName_Long, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . NewTableEvent ( 1, string ( GeneralLoader :: MaxPackedString + 1, 'x' ) );
+ m_source . OpenStreamEvent();
+
+ // the expected return code is different here due to VDB's internal limitation on 256 characters in a table name
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcDB,rcDirectory,rcResolving,rcPath,rcExcessive ) ) );
+}
+
+FIXTURE_TEST_CASE ( DuplicateTableId, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . NewTableEvent ( 1, "REFERENCE" );
+ m_source . NewTableEvent ( 1, "SEQUENCE" ); // same Id
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcTable, rcExists ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadColumnName, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . NewTableEvent ( DefaultTableId, "REFERENCE" );
+ m_source . NewColumnEvent ( 1, DefaultTableId, "nosuchcolumn", 8 );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcCursor, rcUpdating, rcColumn, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( BadTableId, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . NewTableEvent ( 1, "REFERENCE" );
+ m_source . NewColumnEvent ( 1, 2, "SPOT_GROUP", 8 );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcTable, rcInvalid ) ) );
+}
+
+FIXTURE_TEST_CASE ( DuplicateColumnName, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), "REFERENCE" );
+ m_source . NewColumnEvent ( 1, DefaultTableId, "SPOT_GROUP", 8 );
+ m_source . NewColumnEvent ( 2, DefaultTableId, "SPOT_GROUP", 8 );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcCursor, rcUpdating, rcColumn, rcExists ) ) );
+}
+
+FIXTURE_TEST_CASE ( DuplicateColumnId, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), "REFERENCE" );
+ m_source . NewColumnEvent ( 1, DefaultTableId, "SPOT_GROUP", 8 );
+ m_source . NewColumnEvent ( 1, DefaultTableId, "NAME", 8 );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcColumn, rcExists ) ) );
+}
+
+FIXTURE_TEST_CASE ( NoOpenStreamEvent, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcNS, rcFile, rcReading, rcTransfer, rcIncomplete ) ) );
+}
+
+FIXTURE_TEST_CASE ( NoCloseStreamEvent, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . OpenStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcNS, rcFile, rcReading, rcTransfer, rcIncomplete ) ) );
+}
+
+FIXTURE_TEST_CASE ( NoColumns, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // make sure database exists and is valid
+ OpenDatabase (); // did not throw => opened successfully
+}
+
+//Testing integration of software name and version input
+FIXTURE_TEST_CASE ( SoftwareName, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ const string SoftwareName = "softwarename";
+ const string Version = "2.1.1";
+ m_source . SoftwareNameEvent ( SoftwareName, Version );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate metadata
+ OpenDatabase ();
+ REQUIRE_EQ ( SoftwareName, GetDbMetadataAttr ( "SOFTWARE/formatter", "name" ) );
+ REQUIRE_EQ ( Version, GetDbMetadataAttr ( "SOFTWARE/formatter", "vers" ) );
+
+ // extract the program name from path the same way it's done in ncbi-vdb/libs/kapp/loader-meta.c:KLoaderMeta_Write
+ {
+ const char* tool_name = strrchr(argv0.c_str(), '/');
+ const char* r = strrchr(argv0.c_str(), '\\');
+ if( tool_name != NULL && r != NULL && tool_name < r ) {
+ tool_name = r;
+ }
+ if( tool_name++ == NULL) {
+ tool_name = argv0.c_str();
+ }
+
+ REQUIRE_EQ ( string ( tool_name ), GetDbMetadataAttr ( "SOFTWARE/loader", "name" ) );
+ }
+
+ REQUIRE_EQ ( string ( __DATE__), GetDbMetadataAttr ( "SOFTWARE/loader", "date" ) );
+ {
+ char buf[265];
+ string_printf ( buf, sizeof buf, NULL, "%V", KAppVersion() ); // same format as in ncbi-vdb/libs/kapp/loader-meta.c:MakeVersion()
+ REQUIRE_EQ ( string ( buf ), GetDbMetadataAttr ( "SOFTWARE/loader", "vers" ) );
+ }
+}
+
+FIXTURE_TEST_CASE ( SoftwareName_BadVersion, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ const string SoftwareName = "softwarename";
+ const string Version = "2.1..1"; // improperly formatted
+ m_source . SoftwareNameEvent ( SoftwareName, Version );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcExe, rcDatabase, rcCreating, rcMessage, rcBadVersion ) ) );
+}
+
+FIXTURE_TEST_CASE ( DBAddDatabase, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database database0 #1 { table table1 #1 TABLE1; } ;"
+ "database root_database #1 { database database0 #1 SUBDB; } ;"
+ )
+ );
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddDatabaseEvent ( 1, 0, "SUBDB", "subdb", kcmCreate );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate database
+ OpenDatabase ();
+ VDatabase * subDb;
+ REQUIRE_RC ( VDatabaseOpenDBUpdate ( m_db, & subDb, "subdb" ) );
+ REQUIRE_RC ( VDatabaseRelease ( subDb ) );
+}
+
+FIXTURE_TEST_CASE ( DBAddSubDatabase, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database database0 #1 { table table1 #1 TABLE1; } ;"
+ "database database1 #1 { database database0 #1 SUBSUBDB; } ;"
+ "database root_database #1 { database database1 #1 SUBDB; } ;"
+ )
+ );
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddDatabaseEvent ( 1, 0, "SUBDB", "subdb", kcmCreate );
+ m_source . DBAddDatabaseEvent ( 2, 1, "SUBSUBDB", "subsubdb", kcmCreate );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate database
+ OpenDatabase ();
+ VDatabase * subDb;
+ REQUIRE_RC ( VDatabaseOpenDBUpdate ( m_db, & subDb, "subdb" ) );
+ VDatabase * subSubDb;
+ REQUIRE_RC ( VDatabaseOpenDBUpdate ( subDb, & subSubDb, "subsubdb" ) );
+ REQUIRE_RC ( VDatabaseRelease ( subSubDb ) );
+ REQUIRE_RC ( VDatabaseRelease ( subDb ) );
+}
+
+FIXTURE_TEST_CASE ( DBAddTable, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database root_database #1 { table table1 #1 TABLE1; } ;"
+ )
+ );
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddTableEvent ( 1, 0, "TABLE1", "tbl", kcmCreate );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate database
+ OpenDatabase ();
+ const VTable *tbl;
+ REQUIRE_RC ( VDatabaseOpenTableRead ( m_db, & tbl, "tbl" ) );
+ REQUIRE_RC ( VTableRelease ( tbl ) );
+}
+
+FIXTURE_TEST_CASE ( DBAddTableToSubDb, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database database0 #1 { table table1 #1 TABLE1; } ;"
+ "database root_database #1 { database database0 #1 SUBDB; } ;"
+ )
+ );
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddDatabaseEvent ( 2, 0, "SUBDB", "subdb", kcmCreate );
+ m_source . DBAddTableEvent ( 1, 2, "TABLE1", "tbl", kcmCreate );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate database
+ OpenDatabase ();
+ {
+ VDatabase * subDb;
+ REQUIRE_RC ( VDatabaseOpenDBUpdate ( m_db, & subDb, "subdb" ) );
+ //const VTable *tbl;
+ //FAIL("see VDB-1617");
+ //REQUIRE_RC ( VDatabaseOpenTableRead ( subDb, & tbl, "tbl" ) );
+ //REQUIRE_RC ( VTableRelease ( tbl ) );
+ REQUIRE_RC ( VDatabaseRelease ( subDb ) );
+ }
+}
+
+FIXTURE_TEST_CASE ( DBMetadataNode, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ const string key = "dbmetadatanode";
+ const string value = "1a2b3c4d";
+ m_source . OpenStreamEvent();
+ m_source . DBMetadataNodeEvent ( 0, key, value );
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ OpenDatabase ();
+ REQUIRE_EQ ( value, GetDbMetadata( m_db, key ) );
+}
+
+FIXTURE_TEST_CASE ( SubDBMetadataNode, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database database0 #1 { table table1 #1 TABLE1; } ;"
+ "database root_database #1 { database database0 #1 SUBDB; } ;"
+ )
+ );
+ const uint32_t subDbId = 2;
+ const string key = "subdbmetadatanode";
+ const string value = "1a2b3c4dsub";
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddDatabaseEvent ( subDbId, 0, "SUBDB", "subdb", kcmCreate );
+ m_source . DBAddTableEvent ( 1, subDbId, "TABLE1", "tbl", kcmCreate );
+ m_source . OpenStreamEvent();
+ m_source . DBMetadataNodeEvent ( subDbId, key, value );
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ OpenDatabase ();
+ {
+ VDatabase * subDb;
+ REQUIRE_RC ( VDatabaseOpenDBUpdate ( m_db, & subDb, "subdb" ) );
+ REQUIRE_EQ ( value, GetDbMetadata( subDb, key ) );
+ REQUIRE_RC ( VDatabaseRelease ( subDb ) );
+ }
+}
+
+FIXTURE_TEST_CASE ( TblMetadataNode, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database root_database #1 { table table1 #1 TABLE1; } ;"
+ )
+ );
+
+ const char* tblName = "tbl";
+ const uint32_t tblId = 2;
+ const string key = "tblmetadatanode";
+ const string value = "tbl1a2b3c4d";
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddTableEvent ( tblId, 0, "TABLE1", tblName, kcmCreate );
+
+ m_source . OpenStreamEvent();
+ m_source . TblMetadataNodeEvent ( tblId, key, value);
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate database
+ OpenDatabase ();
+ {
+ const VTable *tbl;
+ REQUIRE_RC ( VDatabaseOpenTableRead ( m_db, & tbl, tblName ) );
+
+ {
+ const KMetadata *meta;
+ REQUIRE_RC ( VTableOpenMetadataRead ( tbl, &meta ) );
+ REQUIRE_EQ ( value, GetMetadata ( meta, key) );
+ REQUIRE_RC ( KMetadataRelease ( meta ) );
+ }
+ REQUIRE_RC ( VTableRelease ( tbl ) );
+ }
+}
+
+FIXTURE_TEST_CASE ( ColMetadataNode, GeneralLoaderFixture )
+{
+ string schemaFile = ScratchDir + GetName() + ".vschema";
+ CreateFile ( schemaFile,
+ string (
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database root_database #1 { table table1 #1 TABLE1; } ;"
+ )
+ );
+
+ const char* tblName = "tbl";
+ const uint32_t tblId = 2;
+ const char* colName = "column1";
+ const uint32_t colId = 5;
+ const string NodeName = "colmetadatanode";
+ const string NodeValue = "1a2b3c4d";
+
+ SetUpStream ( GetName(), schemaFile, "root_database");
+
+ m_source . DBAddTableEvent ( tblId, 0, "TABLE1", tblName, kcmCreate | kcmMD5 );
+ m_source . NewColumnEvent ( colId, tblId, colName, 8 );
+
+ m_source . OpenStreamEvent();
+ m_source . ColMetadataNodeEvent ( colId, NodeName, NodeValue );
+ // need at least 2 rows with different value in order for the column to become physical, otherwise column's metadata will not be stored
+ m_source . CellDataEvent( colId, string("blah1") );
+ m_source . NextRowEvent ( tblId );
+ m_source . CellDataEvent( colId, string("brh2") );
+ m_source . NextRowEvent ( tblId );
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ // validate metadata
+ OpenDatabase ();
+ {
+ const VTable *tbl;
+ REQUIRE_RC ( VDatabaseOpenTableRead ( m_db, & tbl, tblName ) );
+
+ {
+ const KTable* ktbl;
+ REQUIRE_RC ( VTableOpenKTableRead ( tbl, & ktbl ) );
+ {
+ const KColumn* col;
+ REQUIRE_RC ( KTableOpenColumnRead ( ktbl, & col, colName ) );
+ {
+ const KMetadata *meta;
+ REQUIRE_RC ( KColumnOpenMetadataRead ( col, &meta ) );
+
+ REQUIRE_EQ ( NodeValue, GetMetadata ( meta, NodeName ) );
+
+ REQUIRE_RC ( KMetadataRelease ( meta ) );
+ }
+ REQUIRE_RC ( KColumnRelease ( col ) );
+ }
+ REQUIRE_RC ( KTableRelease ( ktbl ) );
+ }
+ REQUIRE_RC ( VTableRelease ( tbl ) );
+ }
+}
+
+FIXTURE_TEST_CASE ( NoData, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . NewTableEvent ( 2, tableName ); // ids do not have to be consecutive
+ m_source . NewColumnEvent ( 222, 2, columnName, 8 );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ OpenCursor( tableName, columnName );
+ uint64_t count;
+ REQUIRE_RC ( VCursorIdRange ( m_cursor, 1, NULL, &count ) );
+ REQUIRE_EQ ( (uint64_t)0, count );
+}
+
+FIXTURE_TEST_CASE ( Chunk_BadColumnId, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ m_source . CellDataEvent( /*bad*/2, string("blah") );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( WriteNoCommit, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value = "a single character string cell";
+ m_source . CellDataEvent( DefaultColumnId, value );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ OpenCursor( tableName, columnName );
+ uint64_t count;
+ REQUIRE_RC ( VCursorIdRange ( m_cursor, 1, NULL, &count ) );
+ REQUIRE_EQ ( (uint64_t)0, count );
+}
+
+FIXTURE_TEST_CASE ( CommitBadTableId, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ m_source . NextRowEvent ( /*bad*/2 );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcExe, rcFile, rcReading, rcTable, rcNotFound ) ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnOneCellOneChunk, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value = "a single character string cell";
+ m_source . CellDataEvent( DefaultColumnId, value );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnOneCellOneChunk_Long, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value ( GeneralLoader :: MaxPackedString + 1, 'x' );
+ m_source . CellDataEvent( DefaultColumnId, value );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnOneCellManyChunks, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value1 = "first";
+ m_source . CellDataEvent( DefaultColumnId, value1 );
+ string value2 = "second!";
+ m_source . CellDataEvent( DefaultColumnId, value2 );
+ string value3 = "third!!!";
+ m_source . CellDataEvent( DefaultColumnId, value3 );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value1 + value2 + value3, GetValue<string> ( tableName, columnName, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( IntegerCompression_MinimumCompression, GeneralLoaderFixture )
+{
+ if ( ! SetUpForIntegerCompression ( GetName() ) )
+ return;
+
+ // no compression ( adds 1 byte per value )
+ const uint16_t u16value = 0x3456;
+ const uint32_t u32value = 0x789abcde;
+ const uint64_t u64value = 0x0F0F0F0F0F0F0F0FLL;
+
+ m_source . OpenStreamEvent();
+
+ uint8_t buf[128];
+
+ int bytes = encode_uint16 ( u16value, buf, buf + sizeof buf );
+ REQUIRE_EQ ( 3, bytes );
+ m_source . CellDataEventRaw ( Column16Id, 1, buf, bytes );
+
+ bytes = encode_uint32 ( u32value, buf, buf + sizeof buf );
+ REQUIRE_EQ ( 5, bytes );
+ m_source . CellDataEventRaw ( Column32Id, 1, buf, bytes );
+
+ bytes = encode_uint64 ( u64value, buf, buf + sizeof buf );
+ REQUIRE_EQ ( 9, bytes );
+ m_source . CellDataEventRaw ( Column64Id, 1, buf, bytes );
+
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( u16value, GetValue<uint16_t> ( "TABLE1", "column16", 1 ) );
+ REQUIRE_EQ ( u32value, GetValue<uint32_t> ( "TABLE1", "column32", 1 ) );
+ REQUIRE_EQ ( u64value, GetValue<uint64_t> ( "TABLE1", "column64", 1 ) );
+}
+
+FIXTURE_TEST_CASE ( IntegerCompression_MaximumCompression, GeneralLoaderFixture )
+{
+ if ( ! SetUpForIntegerCompression ( GetName() ) )
+ return;
+
+ // induce maximum compression ( 1 byte per value <= 0x7F )
+ const uint16_t u16value = 0;
+ const uint32_t u32value = 2;
+ const uint64_t u64value = 0x7F;
+
+ m_source . OpenStreamEvent();
+
+ uint8_t buf[128];
+
+ int bytes = encode_uint16 ( u16value, buf, buf + sizeof buf );
+ REQUIRE_EQ ( 1, bytes );
+ m_source . CellDataEventRaw ( Column16Id, 1, buf, bytes );
+
+ bytes = encode_uint32 ( u32value, buf, buf + sizeof buf );
+ REQUIRE_EQ ( 1, bytes );
+ m_source . CellDataEventRaw ( Column32Id, 1, buf, bytes );
+
+ bytes = encode_uint64 ( u64value, buf, buf + sizeof buf );
+ REQUIRE_EQ ( 1, bytes );
+ m_source . CellDataEventRaw ( Column64Id, 1, buf, bytes );
+
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( u16value, GetValue<uint16_t> ( "TABLE1", "column16", 1 ) );
+ REQUIRE_EQ ( u32value, GetValue<uint32_t> ( "TABLE1", "column32", 1 ) );
+ REQUIRE_EQ ( u64value, GetValue<uint64_t> ( "TABLE1", "column64", 1 ) );
+}
+
+FIXTURE_TEST_CASE ( IntegerCompression_MultipleValues, GeneralLoaderFixture )
+{
+ if ( ! SetUpForIntegerCompression ( GetName() ) )
+ return;
+
+ // induce maximum compression ( 1 byte per value <= 0x7F )
+ const uint16_t u16value1 = 0x0001; const uint16_t u16value2 = 0x0002;
+ const uint32_t u32value1 = 0x00000003; const uint32_t u32value2 = 0x00000004;
+ const uint64_t u64value1 = 0x0000000000000006; const uint64_t u64value2 = 0x0000000000000007;
+
+ m_source . OpenStreamEvent();
+
+ uint8_t buf[128];
+
+ {
+ int bytesTotal = 0;
+ int bytes = encode_uint16 ( u16value1, buf, buf + sizeof buf );
+ bytesTotal += bytes;
+ bytes = encode_uint16 ( u16value2, buf + bytesTotal, buf + sizeof buf );
+ bytesTotal += bytes;
+ REQUIRE_EQ ( 2, bytesTotal );
+ m_source . CellDataEventRaw ( Column16Id, 2, buf, bytesTotal );
+ }
+
+ {
+ int bytesTotal = 0;
+ int bytes = encode_uint32 ( u32value1, buf, buf + sizeof buf );
+ bytesTotal += bytes;
+ bytes = encode_uint32 ( u32value2, buf + bytesTotal, buf + sizeof buf );
+ bytesTotal += bytes;
+ REQUIRE_EQ ( 2, bytesTotal );
+ m_source . CellDataEventRaw ( Column32Id, 2, buf, bytesTotal );
+ }
+
+ {
+ int bytesTotal = 0;
+ int bytes = encode_uint64 ( u64value1, buf, buf + sizeof buf );
+ bytesTotal += bytes;
+ bytes = encode_uint64 ( u64value2, buf + bytesTotal, buf + sizeof buf );
+ bytesTotal += bytes;
+ REQUIRE_EQ ( 2, bytesTotal );
+ m_source . CellDataEventRaw ( Column64Id, 2, buf, bytesTotal );
+ }
+
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( u16value1, GetValueWithIndex<uint16_t> ( "TABLE1", "column16", 1, 2, 0 ) );
+ REQUIRE_EQ ( u16value2, GetValueWithIndex<uint16_t> ( "TABLE1", "column16", 1, 2, 1 ) );
+ REQUIRE_EQ ( u32value1, GetValueWithIndex<uint32_t> ( "TABLE1", "column32", 1, 2, 0 ) );
+ REQUIRE_EQ ( u32value2, GetValueWithIndex<uint32_t> ( "TABLE1", "column32", 1, 2, 1 ) );
+ REQUIRE_EQ ( u64value1, GetValueWithIndex<uint64_t> ( "TABLE1", "column64", 1, 2, 0 ) );
+ REQUIRE_EQ ( u64value2, GetValueWithIndex<uint64_t> ( "TABLE1", "column64", 1, 2, 1 ) );
+}
+
+// default values
+
+FIXTURE_TEST_CASE ( OneColumnDefaultNoWrite, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value = "this be my default";
+ m_source . CellDefaultEvent( DefaultColumnId, value );
+ // no WriteEvent
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnDefaultNoWrite_Long, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value ( GeneralLoader :: MaxPackedString + 1, 'x' );
+ m_source . CellDefaultEvent( DefaultColumnId, value );
+ // no WriteEvent
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( MoveAhead, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value = "this be my default";
+ m_source . CellDefaultEvent( DefaultColumnId, value );
+ m_source . MoveAheadEvent ( DefaultTableId, 3 );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 1 ) );
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 2 ) );
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 3 ) );
+ REQUIRE_THROW ( GetValue<string> ( tableName, columnName, 4 ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnDefaultOverwite, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string valueDflt = "this be my default";
+ m_source . CellDefaultEvent( DefaultColumnId, valueDflt );
+ string value = "not the default";
+ m_source . CellDataEvent( DefaultColumnId, value );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value, GetValue<string> ( tableName, columnName, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnChangeDefault, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value1 = "this be my first default";
+ m_source . CellDefaultEvent( DefaultColumnId, value1 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ string value2 = "and this be my second default";
+ m_source . CellDefaultEvent( DefaultColumnId, value2 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value1, GetValue<string> ( tableName, columnName, 1 ) );
+ REQUIRE_EQ ( value2, GetValue<string> ( tableName, columnName, 2 ) );
+}
+
+FIXTURE_TEST_CASE ( OneColumnDataAndDefaultsMixed, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+
+ string value1 = "first value";
+ m_source . CellDataEvent( DefaultColumnId, value1 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ string default1 = "first default";
+ m_source . CellDefaultEvent( DefaultColumnId, default1 );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ string value2 = "second value";
+ m_source . CellDefaultEvent( DefaultColumnId, value2 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ string default2 = "second default";
+ m_source . CellDefaultEvent( DefaultColumnId, default2 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value1, GetValue<string> ( tableName, columnName, 1 ) );
+ REQUIRE_EQ ( default1, GetValue<string> ( tableName, columnName, 2 ) );
+ REQUIRE_EQ ( default1, GetValue<string> ( tableName, columnName, 3 ) );
+ REQUIRE_EQ ( value2, GetValue<string> ( tableName, columnName, 4 ) );
+ REQUIRE_EQ ( default2, GetValue<string> ( tableName, columnName, 5 ) );
+}
+
+FIXTURE_TEST_CASE ( TwoColumnsFullRow, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), tableName );
+
+ const char* columnName1 = "SPOT_GROUP";
+ const char* columnName2 = "MAX_SEQ_LEN";
+ m_source . NewColumnEvent ( 1, DefaultTableId, columnName1, 8 );
+ m_source . NewColumnEvent ( 2, DefaultTableId, columnName2, 32 );
+ m_source . OpenStreamEvent();
+
+ string value1 = "value1";
+ m_source . CellDataEvent( 1, value1 );
+ uint32_t value2 = 12345;
+ m_source . CellDataEvent( 2, value2 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value1, GetValue<string> ( tableName, columnName1, 1 ) );
+ REQUIRE_EQ ( value2, GetValue<uint32_t> ( tableName, columnName2, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( TwoColumnsIncompleteRow, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), tableName );
+
+ const char* columnName1 = "SPOT_GROUP";
+ const char* columnName2 = "MAX_SEQ_LEN";
+ m_source . NewColumnEvent ( 1, DefaultTableId, columnName1, 8 );
+ m_source . NewColumnEvent ( 2, DefaultTableId, columnName2, 32 );
+ m_source . OpenStreamEvent();
+
+ string value1 = "value1";
+ m_source . CellDataEvent( 1, value1 );
+ // no CellData for column 2
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), SILENT_RC ( rcVDB, rcColumn, rcClosing, rcRow, rcIncomplete) ) );
+}
+
+FIXTURE_TEST_CASE ( TwoColumnsPartialRowWithDefaults, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), tableName );
+
+ const char* columnName1 = "SPOT_GROUP";
+ const char* columnName2 = "MAX_SEQ_LEN";
+ m_source . NewColumnEvent ( 1, DefaultTableId, columnName1, 8 );
+ m_source . NewColumnEvent ( 2, DefaultTableId, columnName2, 32 );
+ m_source . OpenStreamEvent();
+
+ string value1 = "value1";
+ m_source . CellDataEvent( 1, value1 );
+ uint32_t value2 = 12345;
+ m_source . CellDefaultEvent( 2, value2 );
+ // no Data Event for column2
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value1, GetValue<string> ( tableName, columnName1, 1 ) );
+ REQUIRE_EQ ( value2, GetValue<uint32_t> ( tableName, columnName2, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( TwoColumnsPartialRowWithDefaultsAndOverride, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), tableName );
+
+ const char* columnName1 = "SPOT_GROUP";
+ const char* columnName2 = "MAX_SEQ_LEN";
+ const char* columnName3 = "CIRCULAR";
+ m_source . NewColumnEvent ( 1, DefaultTableId, columnName1, 8 );
+ m_source . NewColumnEvent ( 2, DefaultTableId, columnName2, 32 );
+ m_source . NewColumnEvent ( 3, DefaultTableId, columnName3, 8 );
+ m_source . OpenStreamEvent();
+
+ string value1 = "value1";
+ m_source . CellDataEvent( 1, value1 );
+ uint32_t value2 = 12345;
+ m_source . CellDefaultEvent( 2, value2 );
+ m_source . CellDefaultEvent( 3, false );
+ bool value3 = true;
+ m_source . CellDataEvent( 3, true ); // override default
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( value1, GetValue<string> ( tableName, columnName1, 1 ) ); // explicit
+ REQUIRE_EQ ( value2, GetValue<uint32_t> ( tableName, columnName2, 1 ) ); // default
+ REQUIRE_EQ ( value3, GetValue<bool> ( tableName, columnName3, 1 ) ); // not the default
+}
+
+FIXTURE_TEST_CASE ( EmptyDefault_String, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), tableName );
+
+ const char* columnName1 = "SPOT_GROUP";
+ m_source . NewColumnEvent ( 1, DefaultTableId, columnName1, 8 );
+ m_source . OpenStreamEvent();
+
+ m_source . CellEmptyDefaultEvent( 1 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( string(), GetValue<string> ( tableName, columnName1, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( EmptyDefault_Int, GeneralLoaderFixture )
+{
+ SetUpStream_OneTable ( GetName(), tableName );
+
+ const char* columnName1 = "MAX_SEQ_LEN";
+ m_source . NewColumnEvent ( 1, DefaultTableId, columnName1, 32 );
+ m_source . OpenStreamEvent();
+
+ m_source . CellEmptyDefaultEvent( 1 );
+ m_source . NextRowEvent ( DefaultTableId );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE ( IsNullValue<uint32_t> ( tableName, columnName1, 1 ) );
+}
+
+FIXTURE_TEST_CASE ( MultipleTables_Multiple_Columns_MultipleRows, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+
+ const char* table1 = "REFERENCE";
+ const char* table1column1 = "SPOT_GROUP"; // ascii
+ const char* table1column2 = "MAX_SEQ_LEN"; // u32
+ m_source . NewTableEvent ( 100, table1 );
+ m_source . NewColumnEvent ( 1, 100, table1column1, 8 );
+ m_source . NewColumnEvent ( 2, 100, table1column2, 32 );
+
+ const char* table2 = "SEQUENCE";
+ const char* table2column1 = "PRIMARY_ALIGNMENT_ID"; // i64
+ const char* table2column2 = "ALIGNMENT_COUNT"; // u8
+ m_source . NewTableEvent ( 200, table2 );
+ m_source . NewColumnEvent ( 3, 200, table2column1, 64 );
+ m_source . NewColumnEvent ( 4, 200, table2column2, 8 );
+
+ m_source . OpenStreamEvent();
+
+ string t1c1v1 = "t1c1v1";
+ m_source . CellDataEvent( 1, t1c1v1 );
+ uint32_t t1c2v1 = 121;
+ m_source . CellDataEvent( 2, t1c2v1 );
+ m_source . NextRowEvent ( 100 );
+
+ int64_t t2c1v1 = 211;
+ m_source . CellDataEvent( 3, t2c1v1 );
+ uint8_t t2c2v1 = 221;
+ m_source . CellDataEvent( 4, t2c2v1 );
+ m_source . NextRowEvent ( 200 );
+
+ string t1c1v2 = "t1c1v2";
+ m_source . CellDataEvent( 1, t1c1v2 );
+ uint32_t t1c2v2 = 122;
+ m_source . CellDataEvent( 2, t1c2v2 );
+ m_source . NextRowEvent ( 100 );
+
+ int64_t t2c1v2 = 212;
+ m_source . CellDataEvent( 3, t2c1v2 );
+ uint8_t t2c2v2 = 222;
+ m_source . CellDataEvent( 4, t2c2v2 );
+ m_source . NextRowEvent ( 200 );
+
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+
+ REQUIRE_EQ ( t1c1v1, GetValue<string> ( table1, table1column1, 1 ) );
+ REQUIRE_EQ ( t1c2v1, GetValue<uint32_t> ( table1, table1column2, 1 ) );
+ REQUIRE_EQ ( t2c1v1, GetValue<int64_t> ( table2, table2column1, 1 ) );
+ REQUIRE_EQ ( t2c2v1, GetValue<uint8_t> ( table2, table2column2, 1 ) );
+
+ REQUIRE_EQ ( t1c1v2, GetValue<string> ( table1, table1column1, 2 ) );
+ REQUIRE_EQ ( t1c2v2, GetValue<uint32_t> ( table1, table1column2, 2 ) );
+ REQUIRE_EQ ( t2c1v2, GetValue<int64_t> ( table2, table2column1, 2 ) );
+ REQUIRE_EQ ( t2c2v2, GetValue<uint8_t> ( table2, table2column2, 2 ) );
+}
+
+FIXTURE_TEST_CASE ( AdditionalSchemaIncludePaths_Single, GeneralLoaderFixture )
+{
+ string schemaPath = "schema";
+ string includeName = string ( GetName() ) + ".inc.vschema";
+ string schemaIncludeFile = schemaPath + "/" + includeName;
+ CreateFile ( schemaIncludeFile, "table table1 #1.0.0 { column ascii column1; };" );
+ string schemaFile = string ( GetName() ) + ".vschema";
+ CreateFile ( schemaFile, string ( "include '" ) + includeName + "'; database database1 #1 { table table1 #1 TABLE1; } ;" );
+
+ SetUpStream ( GetName(), schemaFile, "database1" );
+
+ const char* table1 = "TABLE1";
+ const char* column1 = "column1";
+ m_source . NewTableEvent ( DefaultTableId, table1 );
+ m_source . NewColumnEvent ( DefaultColumnId, DefaultTableId, column1, 8 );
+ m_source . OpenStreamEvent();
+
+ string t1c1v1 = "t1c1v1";
+ m_source . CellDataEvent( DefaultColumnId, t1c1v1 );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ gl -> AddSchemaIncludePath ( schemaPath );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( t1c1v1, GetValue<string> ( table1, column1, 1 ) );
+
+ remove ( schemaIncludeFile . c_str() );
+ remove ( schemaFile . c_str() );
+}
+
+FIXTURE_TEST_CASE ( AdditionalSchemaIncludePaths_Multiple, GeneralLoaderFixture )
+{
+ string schemaPath = "schema";
+ string includeName = string ( GetName() ) + ".inc.vschema";
+ string schemaIncludeFile = schemaPath + "/" + includeName;
+ CreateFile ( schemaIncludeFile, "table table1 #1.0.0 { column ascii column1; };" );
+ string schemaFile = string ( GetName() ) + ".vschema";
+ CreateFile ( schemaFile, string ( "include '" ) + includeName + "'; database database1 #1 { table table1 #1 TABLE1; } ;" );
+
+ SetUpStream ( GetName(), schemaFile, "database1" );
+
+ const char* table1 = "TABLE1";
+ const char* column1 = "column1";
+ m_source . NewTableEvent ( DefaultTableId, table1 );
+ m_source . NewColumnEvent ( DefaultColumnId, DefaultTableId, column1, 8 );
+ m_source . OpenStreamEvent();
+
+ string t1c1v1 = "t1c1v1";
+ m_source . CellDataEvent( DefaultColumnId, t1c1v1 );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ gl -> AddSchemaIncludePath ( "path1" );
+ gl -> AddSchemaIncludePath ( string ( "path2:" ) + schemaPath + ":path3" );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( t1c1v1, GetValue<string> ( table1, column1, 1 ) );
+
+ remove ( schemaIncludeFile . c_str() );
+ remove ( schemaFile . c_str() );
+}
+
+FIXTURE_TEST_CASE ( AdditionalSchemaFiles_Single, GeneralLoaderFixture )
+{
+ string schemaPath = "schema";
+ string schemaFile = schemaPath + "/" + GetName() + ".vschema";
+ CreateFile ( schemaFile, "table table1 #1.0.0 { column ascii column1; }; database database1 #1 { table table1 #1 TABLE1; };" );
+
+ // here we specify a schema that does not exist but it's OK as long as we add a good schema later
+ SetUpStream ( GetName(), "does not exist", "database1" );
+
+ const char* table1 = "TABLE1";
+ const char* column1 = "column1";
+ m_source . NewTableEvent ( DefaultTableId, table1 );
+ m_source . NewColumnEvent ( DefaultColumnId, DefaultTableId, column1, 8 );
+ m_source . OpenStreamEvent();
+
+ string t1c1v1 = "t1c1v1";
+ m_source . CellDataEvent( DefaultColumnId, t1c1v1 );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ gl -> AddSchemaFile ( schemaFile );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( t1c1v1, GetValue<string> ( table1, column1, 1 ) );
+
+ remove ( schemaFile . c_str() );
+}
+
+FIXTURE_TEST_CASE ( AdditionalSchemaFiles_Multiple, GeneralLoaderFixture )
+{
+ string schemaPath = "schema";
+ string schemaFile1 = schemaPath + "/" + GetName() + "1.vschema";
+ CreateFile ( schemaFile1, "table table1 #1.0.0 { column ascii column1; };" );
+ string schemaFile2 = schemaPath + "/" + GetName() + "2.vschema";
+ CreateFile ( schemaFile2, "database database1 #1 { table table1 #1 TABLE1; };" ); // this file uses table1 defined in schemaFile1
+
+ string dbFile = string ( GetName() ) + ".db";
+
+ // here we specify a schema that does not exist but it's OK as long as we add a good schema later
+ m_source . SchemaEvent ( "does not exist", "database1" );
+ m_source . DatabaseEvent ( dbFile );
+
+ const char* table1 = "TABLE1";
+ const char* column1 = "column1";
+ m_source . NewTableEvent ( DefaultTableId, table1 );
+ m_source . NewColumnEvent ( DefaultColumnId, DefaultTableId, column1, 8 );
+ m_source . OpenStreamEvent();
+
+ string t1c1v1 = "t1c1v1";
+ m_source . CellDataEvent( DefaultColumnId, t1c1v1 );
+ m_source . NextRowEvent ( DefaultTableId );
+ m_source . CloseStreamEvent();
+
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ gl -> AddSchemaFile ( schemaFile1 + ":garbage.vschema:" + schemaFile2 ); // some are good
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_EQ ( t1c1v1, GetValue<string> ( table1, column1, 1 ) );
+
+ remove ( schemaFile1 . c_str() );
+ remove ( schemaFile2 . c_str() );
+}
+
+FIXTURE_TEST_CASE ( ErrorMessage, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+ m_source . OpenStreamEvent();
+ m_source . ErrorMessageEvent ( "error message" );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), RC ( rcExe, rcFile, rcReading, rcError, rcExists ) ) );
+}
+
+FIXTURE_TEST_CASE ( ErrorMessage_Long, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+ m_source . OpenStreamEvent();
+ m_source . ErrorMessageEvent ( string ( 257, 'x' ) );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), RC ( rcExe, rcFile, rcReading, rcError, rcExists ) ) );
+}
+
+FIXTURE_TEST_CASE ( LogMessage, GeneralLoaderFixture )
+{
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+ m_source . OpenStreamEvent();
+ m_source . LogMessageEvent ( "log message" );
+ m_source . CloseStreamEvent();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+}
+
+
+FIXTURE_TEST_CASE ( ProgressMessage, GeneralLoaderFixture )
+{
+ // timestamp
+ time_t timestamp = time ( NULL );
+
+ OpenStream_OneTableOneColumn ( GetName(), tableName, columnName, 8 );
+ m_source . OpenStreamEvent ();
+ m_source . ProgMessageEvent ( 123, "progress message", timestamp, 2, 45 );
+ m_source . CloseStreamEvent ();
+
+ REQUIRE ( Run ( m_source . MakeSource (), 0 ) );
+}
+
+
+FIXTURE_TEST_CASE ( TargetOverride, GeneralLoaderFixture )
+{
+ SetUpStream ( GetName() );
+ m_source . OpenStreamEvent();
+ m_source . CloseStreamEvent();
+
+ string newTarget = string ( GetName() ) + "_override";
+ {
+ GeneralLoader* gl = MakeLoader ( m_source . MakeSource () );
+ gl -> SetTargetOverride ( newTarget );
+ REQUIRE ( RunLoader ( *gl, 0 ) );
+ delete gl;
+ } // make sure loader is destroyed (= db closed) before we reopen the database for verification
+
+ REQUIRE_THROW ( OpenDatabase() ); // the db from the instruction stream was not created
+ OpenDatabase ( newTarget . c_str() ); // did not throw => overridden target db opened successfully
+
+ // clean up
+ CloseDatabase();
+ KDirectoryRemove ( m_wd, true, newTarget . c_str() );
+}
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+rc_t CC UsageSummary (const char * progname)
+{
+ return 0;
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ return 0;
+}
+
+const char UsageDefaultName[] = "test-general-loader";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+// TestEnv::verbosity = LogLevel::e_all;
+ KConfigDisableUserSettings();
+
+ ClearScratchDir();
+
+ GeneralLoaderFixture :: argv0 = argv[0];
+
+ TestSource::packed = false;
+ cerr << "Unpacked protocol: ";
+ rc_t rc = GeneralLoaderTestSuite(argc, argv);
+ if ( rc == 0 )
+ {
+ ClearScratchDir();
+
+ TestSource::packed = true;
+ cerr << "Packed protocol: ";
+ rc = GeneralLoaderTestSuite(argc, argv);
+ }
+
+ ClearScratchDir();
+
+ return rc;
+}
+
+}
+
diff --git a/test/general-loader/test-general-writer.cpp b/test/general-loader/test-general-writer.cpp
new file mode 100644
index 0000000..ad7268b
--- /dev/null
+++ b/test/general-loader/test-general-writer.cpp
@@ -0,0 +1,341 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "../../tools/general-loader/general-writer.hpp"
+
+#include <iostream>
+#include <string>
+#include <fstream>
+#include <cstdlib>
+#include <cstring>
+#include <cstdio>
+
+namespace ncbi
+{
+
+ GeneralWriter * testCreateGw ( const char * out_path, const char * schema_path, const char * software_name, const char * version )
+ {
+ GeneralWriter * ret;
+ if ( out_path == 0 )
+ {
+ // use stdout
+ ret = new GeneralWriter ( 1 );
+ }
+ else
+ {
+ // use a file
+ ret = new GeneralWriter ( out_path );
+ }
+
+ ret -> setRemotePath ( schema_path );
+ ret -> useSchema ( schema_path, std :: string ( "general_writer:test:db" ) );
+ ret -> setSoftwareName ( software_name, version );
+
+ return ret;
+ }
+
+ int testAddTable ( GeneralWriter *gw )
+ {
+ return gw -> addTable ( std :: string ( "table1" ) );
+ }
+
+ void testAddColumn ( GeneralWriter *gw, int table_id, const char *column_names [], int column_count, int *stream_ids )
+ {
+ for ( int i = 0; i < column_count; ++ i )
+ {
+ const char *name = column_names [ i ];
+ stream_ids [ i ] = gw -> addColumn ( table_id, name, 8, 0 ); // all columns are ascii for now
+ }
+ }
+
+ void testDBAddDatabase ( GeneralWriter *gw, const char *mbr_name, const char *db_name, uint8_t create_mode )
+ {
+ gw -> dbAddDatabase ( 0, mbr_name, db_name, create_mode );
+ }
+
+ void testDBAddTable ( GeneralWriter *gw, const char *mbr_name, const char *table_name, uint8_t create_mode )
+ {
+ gw -> dbAddDatabase ( 0, mbr_name, table_name, create_mode );
+ }
+
+ void testOpen ( GeneralWriter *gw )
+ {
+ gw -> open ();
+ }
+
+ void testColumnDefault ( GeneralWriter *gw, int stream_id )
+ {
+ const char *data = "some string as data";
+ gw -> columnDefault ( stream_id, 8, data, strlen ( data ) );
+ }
+
+ void testWrite ( GeneralWriter *gw, int table_id, int *stream_ids, int column_count, const char *file_names [] )
+ {
+ FILE **columns = ( FILE ** ) calloc ( column_count, sizeof *columns );
+ if ( columns == NULL )
+ throw "Failed to allocate memory";
+
+ char buffer [ 4096 ];
+ size_t buff_size = sizeof buffer;
+
+ // Populate array of file pointers
+ for ( int i = 0; i < column_count; ++ i )
+ {
+ const char *name = file_names [ i ];
+ FILE *column = fopen ( name, "r" );
+
+ if ( column == NULL )
+ throw "Error opening file";
+
+ columns [ i ] = column;
+
+ column = 0;
+ }
+
+ // open the stream
+ gw -> open ();
+
+ // set default values for each column
+ for ( int i = 0; i < column_count; ++ i )
+ gw -> columnDefault ( stream_ids [ i ], 8, "EOF", strlen ( "EOF" ) );
+
+ while ( 1 )
+ {
+ bool validColumn = false;
+ // write one line from each column
+ for ( int i = 0; i < column_count; ++ i )
+ {
+ // Skip any NULL entries in the FILE array
+ if ( columns [ i ] == NULL )
+ continue;
+
+ FILE *column = columns [ i ];
+
+ if ( fgets ( buffer, buff_size, column ) == NULL )
+ {
+ if ( ferror ( column ) )
+ {
+ fclose ( column );
+ throw "Error reading from file";
+ }
+
+ // Found EOF while reading cell data
+ // close FILE and set array pointer to NULL
+ fclose ( column );
+ column = 0;
+ columns [ i ] = 0;
+ continue;
+ }
+
+ // found valid data, write line and go to next column
+ int elem_count = strlen ( ( const char * ) buffer );
+
+ // ensure there are no new lines at the end of the string
+ for ( int j = elem_count - 1; j >= 0; -- j )
+ {
+ if ( buffer [ j ] == '\n' )
+ -- elem_count;
+ }
+
+ gw -> write ( stream_ids [ i ], 8, buffer, elem_count );
+ validColumn = true;
+ }
+
+ if ( ! validColumn )
+ break;
+
+ // go to the next row
+ gw -> nextRow ( table_id );
+ }
+ free ( columns );
+ }
+
+ void testAddDBMetadataNode ( GeneralWriter *gw, const char * node, const char *value )
+ {
+ gw -> setDBMetadataNode ( 0, node, value );
+ }
+
+ void testAddTblMetadataNode ( GeneralWriter *gw, const char * node, const char *value )
+ {
+ gw -> setTblMetadataNode ( 1, node, value );
+ }
+
+ void testAddColMetadataNode ( GeneralWriter *gw, const char * node, const char *value )
+ {
+ gw -> setColMetadataNode ( 1, node, value );
+ }
+
+ void testProgMsg ( GeneralWriter *gw, const char *name,
+ uint32_t version, uint64_t done, uint64_t total )
+ {
+ gw -> progMsg ( name, version, done, total );
+ }
+
+ void testEndStream ( GeneralWriter *gw )
+ {
+ gw -> endStream ();
+
+ try
+ {
+ std :: cerr << "Attempting add table after endStream" << std :: endl;
+ gw -> addTable ( "table1" );
+ }
+ catch ( const char x [] )
+ {
+ std :: cerr << x << std :: endl;
+ std :: cerr << "addTable correctly failed" << std :: endl;
+ }
+ }
+
+ void runTest ( int column_count, const char * columns [], const char *outfile, const char * schema_path )
+ {
+ GeneralWriter *gw;
+ try
+ {
+ const char * column_names [ column_count ];
+
+ for ( int i = 0; i < column_count ; ++ i )
+ column_names [ i ] = columns [ i ];
+
+ gw = testCreateGw ( outfile, schema_path, "softwarename", "2" );
+ std :: cerr << "CreateGw Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ int table_id = testAddTable ( gw );
+ std :: cerr << "addTable Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ int stream_ids [ column_count ];
+ testAddColumn ( gw, table_id, column_names, column_count, stream_ids );
+ std :: cerr << "addColumn Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ testDBAddDatabase ( gw, "member_name", "db_name", 1 );
+ std :: cerr << "dbAddDatabase Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ testDBAddTable ( gw, "member_name", "table_name", 1 );
+ std :: cerr << "dbAddTable Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ testWrite ( gw, table_id, stream_ids, column_count, column_names );
+ std :: cerr << "write Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ testAddDBMetadataNode ( gw, "db_metadata_node", "01a2b3c4d" );
+ std :: cerr << "setDBMetadataNode Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+ testAddTblMetadataNode ( gw, "tbl_metadata_node", "01a2b3c4d" );
+ std :: cerr << "setTblMetadataNode Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+ testAddColMetadataNode ( gw, "col_metadata_node", "01a2b3c4d" );
+ std :: cerr << "setColMetadataNode Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ testProgMsg ( gw, "name", 1, 54768, 64000 );
+ std :: cerr << "setProgMsg Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ testEndStream ( gw );
+ std :: cerr << "endStream Success" << std :: endl;
+ std :: cerr << "---------------------------------" << std :: endl;
+
+ }
+ catch ( ... )
+ {
+ delete gw;
+ throw;
+ }
+ delete gw;
+ }
+}
+
+const char * getArg ( const char* &arg, int & i, int argc, char * argv [] )
+{
+ if ( arg [ 1 ] != 0 )
+ {
+ const char * next = arg + 1;
+ arg = "\0";
+ return next;
+ }
+
+ if ( ++ i == argc )
+ throw "Missing argument";
+
+ return argv [ i ];
+}
+
+int main ( int argc, char * argv [] )
+{
+ int status = 1;
+
+ try
+ {
+
+ const char *outfile = 0;
+ const char *schema_path = "./test-general-writer.vschema";
+ int num_columns = 0;
+
+ for ( int i = 1; i < argc; ++ i )
+ {
+ const char * arg = argv [ i ];
+ if ( arg [ 0 ] != '-' )
+ {
+ // have an input column
+ argv [ num_columns ++ ] = ( char* ) arg;
+ }
+ else do switch ( ( ++ arg ) [ 0 ] )
+ {
+ case 'o':
+ outfile = getArg ( arg, i, argc, argv );
+ break;
+ case 's':
+ schema_path = getArg ( arg, i, argc, argv );
+ default:
+ throw "Invalid argument";
+ }
+ while ( arg [ 1 ] != 0 );
+ }
+
+ if ( num_columns == 0 )
+ {
+ const char * columns [ 2 ] = { "column01", "column02" };
+ ncbi :: runTest ( 2, columns, outfile, schema_path );
+ }
+ else
+ {
+ ncbi :: runTest ( num_columns, ( const char ** ) argv, outfile, schema_path );
+ }
+
+ status = 0;
+ }
+ catch ( const char x [] )
+ {
+ std :: cerr << x << std :: endl;
+ }
+
+ return status;
+}
diff --git a/tools/run-stat/num-gen.h b/test/general-loader/test-gw-dumper.cpp
similarity index 50%
rename from tools/run-stat/num-gen.h
rename to test/general-loader/test-gw-dumper.cpp
index 03f994b..6a6347d 100644
--- a/tools/run-stat/num-gen.h
+++ b/test/general-loader/test-gw-dumper.cpp
@@ -24,58 +24,57 @@
*
*/
-#ifndef _h_num_gen_
-#define _h_num_gen_
+#ifndef _cpp_gw_dumper_test_
+#define _cpp_gw_dumper_test_
-#ifdef __cplusplus
-extern "C" {
+#ifndef _hpp_general_writer_
+#include "general-writer.hpp"
#endif
-#include <klib/vector.h>
+#include <kfs/defs.h>
-/********************************************************************************
-
- A NUMBER GENERATOR
-
-input : string, for instance "3,6,8,12,44-49"
-ouptut: sequence of integers, for instance 3,6,8,12,44,45,46,47,48,49
-
-********************************************************************************/
-typedef struct ng_node
-{
- uint64_t start;
- uint64_t count;
-} ng_node;
-typedef ng_node* p_ng_node;
-
-
-typedef struct ng
+namespace ncbi
{
- Vector nodes;
- uint32_t node_count;
- uint64_t curr_node;
- uint32_t curr_node_sub_pos;
-} ng;
-typedef ng* p_ng;
+ static
+ void run ()
+ {
+ GeneralWriter gw ( "test-gw-dumper.gw" );
+ gw . setRemotePath ( "remote-path" );
+ gw . setSoftwareName ( "gw-dumper-test", "0.9.0" );
+ gw . useSchema ( "schemafile.vschema", "db_spec.schema" );
-rc_t ng_make( ng** generator );
-rc_t ng_destroy( p_ng generator );
+ int db_id = gw . dbAddDatabase ( 0, "db1", "db1", kcmCreate );
+ gw . setDBMetadataNode ( db_id, "node/path", "value01" );
-uint32_t ng_parse( p_ng generator, const char* src );
+ int tbl_id = gw . dbAddTable ( db_id, "tbl1", "tbl1", kcmCreate );
+ gw . setTblMetadataNode ( tbl_id, "tbl/path", "value01" );
-bool ng_set_range( p_ng generator,
- const int64_t first, const uint64_t count );
+ int col_id = gw . addColumn ( tbl_id, "col1", 64, 0 );
+ gw . setColMetadataNode ( col_id, "col/path", "value01" );
-bool ng_check_range( p_ng generator,
- const int64_t first, const uint64_t count );
+ tbl_id = gw . addTable ( "tbl2" );
+ gw . setTblMetadataNode ( tbl_id, "tbl2/path", "value02" );
+ }
+}
-bool ng_start( p_ng generator );
-bool ng_next( p_ng generator, uint64_t* value );
-
-bool ng_range_defined( p_ng generator );
-
-uint64_t ng_count( p_ng generator );
-
-#endif
+int main ( int argc, char * argv [] )
+{
+ try
+ {
+ ncbi :: run ();
+ }
+ catch ( const char x [] )
+ {
+ std :: cerr << "caught exception: " << x << '\n';
+ }
+ catch ( ... )
+ {
+ std :: cerr << "caught exception\n";
+ }
+
+ return 0;
+}
+
+#endif //_cpp_gw_dumper_test_
diff --git a/test/general-loader/testsource.cpp b/test/general-loader/testsource.cpp
new file mode 100644
index 0000000..0801928
--- /dev/null
+++ b/test/general-loader/testsource.cpp
@@ -0,0 +1,758 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Test stream creation class for unit-testing General Loader
+*/
+
+#include "testsource.hpp"
+
+#include "../../tools/general-loader/general-loader.hpp"
+
+#include <kfs/ramfile.h>
+
+#include <sysalloc.h>
+
+#include <cstring>
+
+#include <stdexcept>
+
+using namespace std;
+using namespace ncbi;
+
+/////////////////////////////////////////////// TestSource::Buffer
+
+TestSource::Buffer::Buffer ( const string& p_signature, uint32_t p_endness, uint32_t p_version, bool p_packing )
+: m_buffer ( 0 ), m_bufSize ( 0 )
+{
+ init ( m_header );
+ strncpy ( m_header . dad . signature, p_signature . c_str(), sizeof m_header . dad . signature );
+ m_header . dad . endian = p_endness;
+ m_header . dad . version = p_version;
+ m_header . packing = p_packing ? 1 : 0;
+
+ Write ( & m_header, m_header . dad . hdr_size );
+}
+
+TestSource::Buffer::~Buffer ()
+{
+ free ( m_buffer );
+}
+
+void
+TestSource::Buffer::Save ( std::ostream& p_out ) const
+{
+ p_out . write ( m_buffer, m_bufSize );
+}
+
+void
+TestSource::Buffer::Write ( const void * p_data, size_t p_size )
+{
+ m_buffer = ( char * ) realloc ( m_buffer, m_bufSize + p_size );
+ if ( m_buffer == 0 )
+ throw logic_error ( "TestSource::Buffer::Write: realloc failed" );
+
+ memcpy ( m_buffer + m_bufSize, p_data, p_size );
+ m_bufSize += p_size;
+}
+
+void
+TestSource::Buffer::Pad ( size_t p_alignment )
+{
+ size_t newSize = ( m_bufSize + p_alignment - 1 ) / p_alignment;
+ newSize *= p_alignment;
+ if ( m_bufSize != newSize )
+ {
+ m_buffer = ( char * ) realloc ( m_buffer, newSize );
+ if ( m_buffer == 0 )
+ throw logic_error ( "TestSource::Buffer::Write: realloc failed" );
+ memset ( m_buffer + m_bufSize, 0, newSize - m_bufSize );
+ m_bufSize = newSize;
+ }
+}
+
+void
+TestSource::Buffer::Write ( const TestSource::Event& p_event )
+{
+ if ( m_header . packing )
+ {
+ WritePacked ( p_event );
+ }
+ else
+ {
+ WriteUnpacked ( p_event );
+ }
+}
+
+void
+TestSource::Buffer::WriteUnpacked ( const TestSource::Event& p_event )
+{
+ Pad ();
+ switch ( p_event . m_event )
+ {
+ case evt_use_schema:
+ case evt_software_name:
+ case evt_db_metadata_node:
+ case evt_tbl_metadata_node:
+ case evt_col_metadata_node:
+ {
+ gw_2string_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_size1 ( hdr, p_event . m_str1 . size() );
+ set_size2 ( hdr, p_event . m_str2 . size() );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ Write ( p_event . m_str2 . c_str(), p_event . m_str2 . size() );
+ }
+ break;
+
+ case evt_remote_path:
+ case evt_new_table:
+ case evt_logmsg:
+ case evt_errmsg:
+ {
+ gw_1string_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_size ( hdr, p_event . m_str1 . size () );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ }
+ break;
+ case evt_progmsg:
+ gw_status_evt_v1 hdr;
+ init ( hdr, p_event . m_id1 , evt_progmsg );
+ set_version ( hdr, p_event . m_uint32_2 );
+ set_timestamp ( hdr, p_event . m_uint32 );
+ set_pid ( hdr, p_event . m_id1 );
+ set_size ( hdr, p_event . m_str1 . size () );
+ set_percent ( hdr, p_event . m_uint32_3 );
+
+ Write ( &hdr . dad, sizeof hdr );
+ Write ( p_event . m_str1 . c_str (), p_event . m_str1 . size () );
+
+ break;
+
+ case evt_new_column :
+ {
+ gw_column_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_table_id ( hdr, p_event . m_id2 );
+ set_elem_bits ( hdr, p_event . m_uint32 );
+ set_name_size ( hdr, p_event . m_str1 . size () );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_str1 . c_str (), p_event . m_str1 . size () );
+ }
+ break;
+
+ case evt_add_mbr_db:
+ case evt_add_mbr_tbl:
+ {
+ gw_add_mbr_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+
+ set_db_id ( hdr, p_event . m_id2 );
+ set_size1 ( hdr, p_event . m_str1 . size() );
+ set_size2 ( hdr, p_event . m_str2 . size() );
+ set_create_mode ( hdr, p_event . m_uint8 );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_str1 . data (), p_event . m_str1 . size () );
+ Write ( p_event . m_str2 . data (), p_event . m_str2 . size () );
+
+ }
+ break;
+
+ case evt_open_stream:
+ case evt_end_stream:
+ {
+ gw_evt_hdr_v1 hdr;
+ init ( hdr, 0, p_event . m_event );
+ Write ( & hdr, sizeof hdr );
+ }
+ break;
+
+ case evt_cell_data :
+ case evt_cell_default :
+ {
+ gw_data_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_elem_count ( hdr, p_event . m_uint32 );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_val . data(), p_event . m_val . size() );
+ }
+ break;
+
+ case evt_next_row:
+ case evt_empty_default :
+ {
+ gw_evt_hdr_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ Write ( & hdr, sizeof hdr );
+ }
+ break;
+
+ case evt_move_ahead:
+ {
+ gw_move_ahead_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_nrows ( hdr, p_event . m_uint64 );
+ Write ( & hdr, sizeof hdr );
+ }
+ break;
+
+ default:
+ throw logic_error ( "TestSource::Buffer::WriteUnpacked: event not implemented" );
+ }
+}
+
+void
+TestSource::Buffer::WritePacked ( const TestSource::Event& p_event )
+{
+ switch ( p_event . m_event )
+ {
+ case evt_use_schema:
+ case evt_software_name:
+ case evt_db_metadata_node:
+ case evt_tbl_metadata_node:
+ case evt_col_metadata_node:
+ if ( p_event . m_str1 . size () <= GeneralLoader :: MaxPackedString && p_event . m_str2 . size () <= GeneralLoader :: MaxPackedString )
+ {
+ gwp_2string_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+
+ set_size1 ( hdr, p_event . m_str1 . size() );
+ set_size2 ( hdr, p_event . m_str2 . size() );
+
+ Write ( & hdr, sizeof hdr );
+ }
+ else
+ {
+ gwp_2string_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_use_schema2 );
+
+ set_size1 ( hdr, p_event . m_str1 . size() );
+ set_size2 ( hdr, p_event . m_str2 . size() );
+
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ Write ( p_event . m_str2 . c_str(), p_event . m_str2 . size() );
+ break;
+
+ //TODO: the following 3 cases are almost identical - refactor
+ case evt_remote_path:
+ if ( p_event . m_str1 . size () <= GeneralLoader :: MaxPackedString )
+ {
+ gwp_1string_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_remote_path );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ else
+ {
+ gwp_1string_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_remote_path2 );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ break;
+ case evt_new_table:
+ if ( p_event . m_str1 . size () <= GeneralLoader :: MaxPackedString )
+ {
+ gwp_1string_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_new_table );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ else
+ {
+ gwp_1string_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_new_table2 );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ break;
+ case evt_logmsg:
+ {
+ gwp_1string_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ break;
+ case evt_errmsg:
+ if ( p_event . m_str1 . size () <= GeneralLoader :: MaxPackedString )
+ {
+ gwp_1string_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_errmsg );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ else
+ {
+ gwp_1string_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_errmsg2 );
+ set_size ( hdr, p_event . m_str1 . size () );
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_str1 . c_str(), p_event . m_str1 . size() );
+ break;
+ case evt_progmsg:
+ gwp_status_evt_v1 hdr;
+ init ( hdr, p_event . m_id1 , evt_progmsg );
+ set_version ( hdr, p_event . m_uint32_2 );
+ set_timestamp ( hdr, p_event . m_uint32 );
+ set_pid ( hdr, ( uint16_t ) p_event . m_id1 );
+ set_size ( hdr, p_event . m_str1 . size () );
+ set_percent ( hdr, ( uint8_t ) p_event . m_uint32_3 );
+
+ Write ( &hdr . dad, sizeof hdr );
+ Write ( p_event . m_str1 . c_str (), p_event . m_str1 . size () );
+ break;
+ case evt_new_column :
+ {
+ gwp_column_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_table_id ( hdr, p_event . m_id2 );
+ set_elem_bits ( hdr, p_event . m_uint32 );
+ hdr . flag_bits = p_event . m_uint8;
+ set_name_size ( hdr, p_event . m_str1 . size () );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_str1 . c_str (), p_event . m_str1 . size () );
+ }
+ break;
+
+ case evt_add_mbr_db:
+ case evt_add_mbr_tbl:
+ {
+ gwp_add_mbr_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+
+ set_db_id ( hdr, p_event . m_id2 );
+ set_size1 ( hdr, p_event . m_str1 . size() );
+ set_size2 ( hdr, p_event . m_str2 . size() );
+ set_create_mode ( hdr, p_event . m_uint8 );
+
+ Write ( & hdr, sizeof hdr );
+ Write ( p_event . m_str1 . data (), p_event . m_str1 . size () );
+ Write ( p_event . m_str2 . data (), p_event . m_str2 . size () );
+
+ }
+ break;
+
+ case evt_open_stream:
+ case evt_end_stream:
+ {
+ gwp_evt_hdr_v1 hdr;
+ init ( hdr, 0, p_event . m_event );
+ Write ( & hdr, sizeof hdr );
+ }
+ break;
+
+ case evt_cell_data :
+ if ( p_event . m_val . size () <= GeneralLoader :: MaxPackedString )
+ {
+ gwp_data_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_cell_data );
+ // in the packed message, we specify the number of bytes in the cell
+ set_size ( hdr, p_event . m_val . size() );
+
+ Write ( & hdr, sizeof hdr );
+ }
+ else
+ {
+ gwp_data_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_cell_data2 );
+ // in the packed message, we specify the number of bytes in the cell
+ set_size ( hdr, p_event . m_val . size() );
+
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_val . data(), p_event . m_val . size() );
+ break;
+ case evt_cell_default :
+ if ( p_event . m_val . size () <= GeneralLoader :: MaxPackedString )
+ {
+ gwp_data_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_cell_default ); //TODO: this is the only difference from evt_cell_data - refactor
+ // in the packed message, we specify the number of bytes in the cell
+ set_size ( hdr, p_event . m_val . size() );
+
+ Write ( & hdr, sizeof hdr );
+ }
+ else
+ {
+ gwp_data_evt_U16_v1 hdr;
+ init ( hdr, p_event . m_id1, evt_cell_default2 ); //TODO: this is the only difference from evt_cell_data - refactor
+ // in the packed message, we specify the number of bytes in the cell
+ set_size ( hdr, p_event . m_val . size() );
+
+ Write ( & hdr, sizeof hdr );
+ }
+ Write ( p_event . m_val . data(), p_event . m_val . size() );
+ break;
+
+ case evt_next_row:
+ case evt_empty_default :
+ {
+ gwp_evt_hdr_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ Write ( & hdr, sizeof hdr );
+ }
+ break;
+
+ case evt_move_ahead:
+ {
+ gwp_move_ahead_evt_v1 hdr;
+ init ( hdr, p_event . m_id1, p_event . m_event );
+ set_nrows ( hdr, p_event . m_uint64 );
+ Write ( & hdr, sizeof hdr );
+ }
+ break;
+
+ default:
+ throw logic_error ( "TestSource::Buffer::WritePacked: event not implemented" );
+ }
+}
+
+/////////////////////////////////////// TestSource
+
+bool TestSource::packed = false;
+
+TestSource::TestSource ( const string& p_signature, uint32_t p_endness, uint32_t p_version )
+: m_buffer ( new Buffer ( p_signature, p_endness, p_version, packed ) )
+{
+}
+
+TestSource::~TestSource()
+{
+ delete m_buffer;
+}
+
+const struct KFile *
+TestSource::MakeSource ()
+{
+ const struct KFile * ret;
+ if ( KRamFileMakeRead ( & ret,
+ const_cast < char* > ( m_buffer -> GetData() ),
+ m_buffer -> GetSize() ) != 0
+ || ret == 0 )
+ throw logic_error ( "TestSource::MakeSource KRamFileMakeRead failed" );
+
+ return ret;
+}
+
+void
+TestSource::SchemaEvent ( const std::string& p_schemaFile, const std::string& p_schemaName )
+{
+ m_buffer -> Write ( Event ( evt_use_schema, p_schemaFile, p_schemaName ) );
+}
+
+void
+TestSource::DatabaseEvent ( const std::string& p_databaseName )
+{
+ m_buffer -> Write ( Event ( evt_remote_path, p_databaseName ) );
+ m_database = p_databaseName;
+}
+
+void
+TestSource::SoftwareNameEvent ( const std::string& p_softwareName, const std::string& p_version )
+{
+ m_buffer -> Write ( Event ( evt_software_name, p_softwareName, p_version ) );
+}
+
+void
+TestSource::DBMetadataNodeEvent ( ObjectId p_id, const std::string& p_metadataNode, const std::string& p_value )
+{
+ m_buffer -> Write ( Event ( evt_db_metadata_node, p_id, p_metadataNode, p_value ) );
+}
+
+void
+TestSource::TblMetadataNodeEvent ( TableId p_id, const std::string& p_metadataNode, const std::string& p_value )
+{
+ m_buffer -> Write ( Event ( evt_tbl_metadata_node, p_id, p_metadataNode, p_value ) );
+}
+
+void
+TestSource::ColMetadataNodeEvent ( ColumnId p_id, const std::string& p_metadataNode, const std::string& p_value )
+{
+ m_buffer -> Write ( Event ( evt_col_metadata_node, p_id, p_metadataNode, p_value ) );
+}
+
+void
+TestSource::NewTableEvent ( TableId p_id, const std::string& p_table )
+{
+ m_buffer -> Write ( Event ( evt_new_table, p_id, p_table ) );
+}
+
+void
+TestSource::NewColumnEvent ( ColumnId p_columnId, TableId p_tableId, const std::string& p_column, uint32_t p_elemBits, bool p_compresssed )
+{
+ m_buffer -> Write ( Event ( evt_new_column, p_columnId, p_tableId, p_column, p_elemBits, p_compresssed ? 1 : 0 ) );
+}
+
+void
+TestSource::DBAddDatabaseEvent ( int p_db_id, int p_parent_id, const std :: string &p_mbr_name, const std :: string &p_db_name, uint8_t p_create_mode )
+{
+ m_buffer -> Write ( Event ( evt_add_mbr_db, p_db_id, p_parent_id, p_mbr_name, p_db_name, p_create_mode ) );
+}
+
+void
+TestSource::DBAddTableEvent ( int p_tbl_id, int p_db_id, const std :: string &p_mbr_name, const std :: string &p_table_name, uint8_t p_create_mode )
+{
+ m_buffer -> Write ( Event ( evt_add_mbr_tbl, p_tbl_id, p_db_id, p_mbr_name, p_table_name, p_create_mode ) );
+}
+
+void
+TestSource::OpenStreamEvent ()
+{
+ m_buffer -> Write ( Event ( evt_open_stream ) );
+}
+
+void
+TestSource::CloseStreamEvent ()
+{
+ m_buffer -> Write ( Event ( evt_end_stream ) );
+}
+
+void
+TestSource::NextRowEvent ( TableId p_id )
+{
+ m_buffer -> Write ( Event ( evt_next_row, p_id ) );
+}
+
+void
+TestSource::MoveAheadEvent ( TableId p_id, uint64_t p_count )
+{
+ m_buffer -> Write ( Event ( evt_move_ahead, p_id, p_count ) );
+}
+
+template<> void TestSource::CellDataEvent ( ColumnId p_columnId, string p_value )
+{
+ m_buffer -> Write ( Event ( evt_cell_data, p_columnId, ( uint32_t ) p_value . size(), ( uint32_t ) p_value . size(), p_value . c_str() ) );
+}
+
+void
+TestSource::CellDefaultEvent ( ColumnId p_columnId, const string& p_value )
+{
+ m_buffer -> Write ( Event ( p_value . size() == 0 ? evt_empty_default : evt_cell_default,
+ p_columnId,
+ ( uint32_t ) p_value . size(),
+ ( uint32_t ) p_value . size(),
+ p_value . c_str() ) );
+}
+
+void
+TestSource::CellEmptyDefaultEvent ( ColumnId p_columnId )
+{
+ m_buffer -> Write ( Event ( evt_empty_default, p_columnId, 0, 0, 0 ) );
+}
+
+void
+TestSource::CellDefaultEvent ( ColumnId p_columnId, uint32_t p_value )
+{
+ m_buffer -> Write ( Event ( evt_cell_default, p_columnId, 1, sizeof p_value, (const void*)&p_value ) );
+}
+void
+TestSource::CellDefaultEvent ( ColumnId p_columnId, bool p_value )
+{
+ m_buffer -> Write ( Event ( evt_cell_default, p_columnId, 1, sizeof p_value, (const void*)&p_value ) );
+}
+
+void
+TestSource::ErrorMessageEvent ( const string& p_msg )
+{
+ m_buffer -> Write ( Event ( evt_errmsg, p_msg ) );
+}
+
+void
+TestSource::LogMessageEvent ( const string& p_msg )
+{
+ m_buffer -> Write ( Event ( evt_logmsg, p_msg ) );
+}
+
+void
+TestSource::ProgMessageEvent ( ProcessId p_pid, const std :: string& p_name, uint32_t p_timestamp, uint32_t p_version, uint32_t p_percent )
+{
+ m_buffer -> Write ( Event ( evt_progmsg, p_pid, p_name, p_timestamp, p_version, p_percent ) );
+}
+
+
+void
+TestSource::SaveBuffer ( const char* p_filename ) const
+{
+ ofstream out ( p_filename );
+ m_buffer -> Save ( out );
+}
+
+TestSource::Event::Event ( gw_evt_id p_event )
+: m_event ( p_event ),
+ m_id1 ( 0 ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id1 )
+: m_event ( p_event ),
+ m_id1 ( p_id1 ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id1, uint64_t p_uint64 )
+: m_event ( p_event ),
+ m_id1 ( p_id1 ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( p_uint64 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id1, uint32_t p_id2, const std::string& p_str, uint32_t p_uint32, uint8_t p_uint8 )
+: m_event ( p_event ),
+ m_id1 ( p_id1 ),
+ m_id2 ( p_id2 ),
+ m_uint8 ( p_uint8 ),
+ m_uint32 ( p_uint32 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id, const std::string& p_str1 )
+: m_event ( p_event ),
+ m_id1 ( p_id ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str1 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, const std::string& p_str1 )
+: m_event ( p_event ),
+ m_id1 ( 0 ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str1 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, const std::string& p_str1, const std::string& p_str2 )
+: m_event ( p_event ),
+ m_id1 ( 0 ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str1 ),
+ m_str2 ( p_str2 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id, const std::string& p_str1, const std::string& p_str2 )
+: m_event ( p_event ),
+ m_id1 ( p_id ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str1 ),
+ m_str2 ( p_str2 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id1, uint32_t p_elem_count, uint32_t p_val_bytes, const void* p_val )
+: m_event ( p_event ),
+ m_id1 ( p_id1 ),
+ m_id2 ( 0 ),
+ m_uint32 ( p_elem_count ),
+ m_uint64 ( 0 )
+{
+ const char* v = ( const char* ) p_val;
+ for ( size_t i = 0; i != p_val_bytes; ++i )
+ {
+ m_val . push_back ( v [ i ] );
+ }
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id1, const std::string& p_str1, const std::string& p_str2, uint8_t p_uint8 )
+: m_event ( p_event ),
+ m_id1 ( p_id1 ),
+ m_id2 ( 0 ),
+ m_uint8 ( p_uint8 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str1 ),
+ m_str2 ( p_str2 )
+{
+}
+
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_pid, const std :: string& p_name, uint32_t p_timestamp, uint32_t p_version, uint32_t p_percent )
+: m_event ( p_event ),
+ m_id1 ( p_pid ),
+ m_id2 ( 0 ),
+ m_uint8 ( 0 ),
+ m_uint32 ( p_timestamp ),
+ m_uint32_2 ( p_version ),
+ m_uint32_3 ( p_percent ),
+ m_str1 ( p_name )
+{
+}
+TestSource::Event::Event ( gw_evt_id p_event, uint32_t p_id1, uint32_t p_id2, const std::string& p_str1, const std::string& p_str2, uint8_t p_uint8 )
+: m_event ( p_event ),
+ m_id1 ( p_id1 ),
+ m_id2 ( p_id2 ),
+ m_uint8 ( p_uint8 ),
+ m_uint32 ( 0 ),
+ m_uint64 ( 0 ),
+ m_str1 ( p_str1 ),
+ m_str2 ( p_str2 )
+{
+}
+
+
+TestSource::Event::~Event()
+{
+}
diff --git a/test/general-loader/testsource.hpp b/test/general-loader/testsource.hpp
new file mode 100644
index 0000000..b6c3694
--- /dev/null
+++ b/test/general-loader/testsource.hpp
@@ -0,0 +1,177 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _sra_tools_test_hpp_testsource
+#define _sra_tools_test_hpp_testsource
+
+/**
+* Test stream creation class for unit-testing General Loader
+*/
+
+#include <sysalloc.h>
+
+#include <kfc/defs.h>
+
+#include <kfs/file.h>
+
+#include "../../tools/general-loader/general-writer.h"
+
+#include <string>
+#include <vector>
+#include <fstream>
+
+class TestSource
+{
+public:
+ typedef uint32_t TableId;
+ typedef uint32_t ColumnId;
+ typedef uint32_t ObjectId;
+ typedef uint32_t ProcessId;
+
+public:
+ static bool packed;
+
+public:
+ TestSource( const std:: string& p_signature = GW_SIGNATURE,
+ uint32_t p_endness = GW_GOOD_ENDIAN,
+ uint32_t p_version = GW_CURRENT_VERSION );
+
+ ~TestSource();
+
+ const std::string & GetDatabaseName() { return m_database; }
+
+ const struct KFile * MakeSource ();
+
+ void SchemaEvent ( const std::string& p_schemaFile, const std::string& p_schemaName );
+ void DatabaseEvent ( const std::string& p_databaseName );
+ void SoftwareNameEvent ( const std::string& p_softwareName, const std::string& p_version );
+
+ void DBMetadataNodeEvent ( ObjectId p_id, const std::string& p_metadataNode, const std::string& p_value );
+ void TblMetadataNodeEvent ( TableId p_id, const std::string& p_metadataNode, const std::string& p_value );
+ void ColMetadataNodeEvent ( ColumnId p_id, const std::string& p_metadataNode, const std::string& p_value );
+
+ void NewTableEvent ( TableId p_id, const std::string& p_table );
+ void NewColumnEvent ( ColumnId p_columnId, TableId p_tableId,
+ const std::string& p_column, uint32_t p_elemBits, bool p_compresssed = false );
+
+ void DBAddDatabaseEvent ( int p_db_id, int p_parent_id, const std :: string &p_mbr_name,
+ const std :: string &p_db_name, uint8_t p_create_mode );
+ void DBAddTableEvent ( int ptbl_id, int p_db_id, const std :: string &p_mbr_name,
+ const std :: string &p_table_name, uint8_t p_create_mode );
+
+ void OpenStreamEvent ();
+ void CloseStreamEvent ();
+ void NextRowEvent ( TableId p_id );
+ void MoveAheadEvent ( TableId p_id, uint64_t p_count );
+ void CellDefaultEvent ( ColumnId p_columnId, const std :: string& p_value );
+ void CellDefaultEvent ( ColumnId p_columnId, uint32_t p_value );
+ void CellDefaultEvent ( ColumnId p_columnId, bool p_value );
+ void CellEmptyDefaultEvent ( ColumnId p_columnId );
+
+ void ErrorMessageEvent ( const std :: string& p_msg );
+ void LogMessageEvent ( const std :: string& p_msg );
+ void ProgMessageEvent ( ProcessId p_pid, const std :: string& p_name, uint32_t p_timestamp, uint32_t p_version, uint32_t p_percent );
+
+ template < typename T > void CellDataEvent ( ColumnId p_columnId, T p_value )
+ {
+ m_buffer -> Write ( Event ( evt_cell_data, p_columnId, 1, sizeof p_value, (const void*)&p_value ) );
+ } // see below for specialization for std::string (follows the class definition)
+
+ // use this to capture stream contents to be used in cmdline tests
+ void SaveBuffer ( const char* p_filename ) const;
+
+ void CellDataEventRaw ( ColumnId p_columnId, uint32_t p_elemCount, const void* p_value, uint32_t p_size )
+ {
+ m_buffer -> Write ( Event ( evt_cell_data, p_columnId, p_elemCount, p_size, p_value ) );
+ }
+
+private:
+ struct Event
+ {
+ Event ( gw_evt_id p_event );
+ Event ( gw_evt_id p_event, const std::string& p_str1 );
+ Event ( gw_evt_id p_event, const std::string& p_str1, const std::string& p_str2 );
+ Event ( gw_evt_id p_event, uint32_t p_id1 );
+ Event ( gw_evt_id p_event, uint32_t p_id1, uint64_t p_uint64 );
+ Event ( gw_evt_id p_event, uint32_t p_id1, const std::string& p_str1 );
+ Event ( gw_evt_id p_event, uint32_t p_id, const std::string& p_str1, const std::string& p_str2 );
+ Event ( gw_evt_id p_event, uint32_t p_id1, uint32_t p_elem_count, uint32_t p_val_bytes, const void* p_val );
+ Event ( gw_evt_id p_event, uint32_t p_id1, const std::string& p_str1, const std::string& p_str2, uint8_t p_uint8 );
+ Event ( gw_evt_id p_event, uint32_t p_id1, uint32_t p_id2, const std::string& p_str, uint32_t p_uint32, uint8_t p_uint8 );
+ Event ( gw_evt_id p_event, uint32_t p_pid, const std :: string& p_name, uint32_t p_timestamp, uint32_t p_version, uint32_t percent );
+ Event ( gw_evt_id p_event, uint32_t p_id1, uint32_t p_id2, const std::string& p_str1, const std::string& p_str2, uint8_t p_uint8 );
+
+ ~Event();
+
+ gw_evt_id m_event;
+ uint32_t m_id1;
+ uint32_t m_id2;
+
+ uint8_t m_uint8;
+ uint32_t m_uint32;
+ uint32_t m_uint32_2;
+ uint32_t m_uint32_3;
+ uint64_t m_uint64;
+
+ std :: string m_str1;
+ std :: string m_str2;
+ std :: vector < char > m_val;
+ };
+
+ class Buffer
+ {
+ public:
+ Buffer(const std :: string& p_signature, uint32_t p_endness, uint32_t p_version, bool p_packing);
+ ~Buffer();
+
+ void Write ( const Event& );
+
+ void Save ( std::ostream& p_out ) const;
+
+ const char* GetData() { return m_buffer; }
+ size_t GetSize() const { return m_bufSize; }
+
+ private:
+ void WritePacked ( const Event& );
+ void WriteUnpacked ( const Event& );
+
+ void Write( const void * p_data, size_t p_size );
+ void Pad ( size_t p_alignment = 4 );
+
+ char* m_buffer;
+ size_t m_bufSize;
+
+ gw_header_v1 m_header;
+ };
+
+ std :: string m_database;
+
+ Buffer* m_buffer;
+};
+
+template<> void TestSource::CellDataEvent ( ColumnId p_columnId, std :: string p_value );
+
+#endif
diff --git a/test/kget/.gitignore b/test/kget/.gitignore
new file mode 100644
index 0000000..d64c8f0
--- /dev/null
+++ b/test/kget/.gitignore
@@ -0,0 +1,3 @@
+*.sra
+*.dat
+*.cachetee
\ No newline at end of file
diff --git a/configure b/test/kget/Makefile
old mode 100755
new mode 100644
similarity index 64%
copy from configure
copy to test/kget/Makefile
index 3313aac..e2cd576
--- a/configure
+++ b/test/kget/Makefile
@@ -1,5 +1,3 @@
-#!/bin/sh
-
# ===========================================================================
#
# PUBLIC DOMAIN NOTICE
@@ -24,22 +22,35 @@
#
# ===========================================================================
-FILENAME="`basename $0`"
-CURDIR="`dirname $0`"
-
-if [ -z "$CURDIR" ]
- then
- echo configure: error: configure should be run as ./configure
- exit 1
-fi
-
-if [ "$FILENAME" != configure -a ! -s konfigure.perl ]
- then
- echo configure: error: configure should be run as ./configure
-else
- perl -v > /dev/null 2>&1 || { echo "checking for perl... no"; \
- echo >&2 "configure: error: perl not found."; exit 1; }
-
- cd $CURDIR
- perl -w ./konfigure.perl $@
-fi
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/kget
+
+TEST_TOOLS = \
+
+include $(TOP)/build/Makefile.env
+
+.PHONY: $(TEST_TOOLS)
+
+#-------------------------------------------------------------------------------
+# slowtests: match output vs wget
+#
+
+slowtests: diff-vs-wget
+
+ACCESSION = SRR072810
+URL = http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByRun/sra/SRR/SRR072/SRR072810/SRR072810.sra
+
+clean:
+ rm -f $(ACCESSION)*
+
+diff-vs-wget: clean
+ $(BINDIR)/kget --reliable -c ./$(ACCESSION).cachetee $(URL) $(ACCESSION).dat --progress
+ wget $(URL)
+ diff $(ACCESSION).sra ./$(ACCESSION).dat
+ #diff $(ACCESSION).sra ./$(ACCESSION).cachetee
+ rm -f $(ACCESSION)*
+
+.PHONY:
diff --git a/tools/cg-dump/Makefile b/test/network-test/Makefile
similarity index 83%
rename from tools/cg-dump/Makefile
rename to test/network-test/Makefile
index 6740423..ec2dc9f 100644
--- a/tools/cg-dump/Makefile
+++ b/test/network-test/Makefile
@@ -26,14 +26,14 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/cg-dump
+MODULE = test/network-test
include $(TOP)/build/Makefile.env
INT_TOOLS = \
EXT_TOOLS = \
- cg-dump
+ network-test
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -81,37 +81,19 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# cg-dump
+# TOOL_SRC
#
-CG_DUMP_SRC = \
- num-gen \
- progressbar \
- line_token_iter \
- last_rowid \
- sg_lookup \
- cg-dump
-
-CG_DUMP_OBJ = \
- $(addsuffix .$(OBJX),$(CG_DUMP_SRC))
-
-CG_DUMP_LIB = \
- -lkapp \
- -lload \
- -lkq \
- -ssraschema \
- -dalign-reader \
- $(READONLY_SCHEMA_LIBS) \
- -lvdb \
- -lkdb \
- -lvfs \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lksrch \
- -lkproc \
- -lklib \
- -lm
+TOOL_SRC = \
+ network-testing \
+ network-test
+
+TOOL_OBJ = \
+ $(addsuffix .$(OBJX),$(TOOL_SRC))
-$(BINDIR)/cg-dump: $(CG_DUMP_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(CG_DUMP_LIB)
+TOOL_LIB = \
+ -skapp \
+ -sncbi-vdb \
+ -lm
+$(BINDIR)/network-test: $(TOOL_OBJ)
+ $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(TOOL_LIB)
diff --git a/test/network-test/network-test.c b/test/network-test/network-test.c
new file mode 100644
index 0000000..a6c02ce
--- /dev/null
+++ b/test/network-test/network-test.c
@@ -0,0 +1,105 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "network-test.vers.h"
+
+#include <klib/out.h>
+#include <klib/rc.h>
+
+#include <kapp/main.h>
+#include <kapp/args.h>
+
+#include <kfs/file.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <stdio.h>
+
+#include "network-testing.h"
+
+const char UsageDefaultName[] = "network-test";
+
+rc_t CC UsageSummary ( const char * progname )
+{
+ return KOutMsg( "\nUsage:\n %s [accession]\n\n", progname );
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ rc_t rc;
+
+ if ( args == NULL )
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
+ else
+ rc = ArgsProgram ( args, &fullpath, &progname );
+
+ if ( rc != 0 )
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary ( progname );
+
+ HelpOptionsStandard ();
+ HelpVersion ( fullpath, KAppVersion() );
+
+ return rc;
+}
+
+
+/* Version EXTERN
+ * return 4-part version code: 0xMMmmrrrr, where
+ * MM = major release
+ * mm = minor release
+ * rrrr = bug-fix release
+ */
+ver_t CC KAppVersion ( void )
+{
+ return NETWORK_TEST_VERS;
+}
+
+static rc_t CC write_to_FILE ( void *f, const char *buffer, size_t bytes, size_t *num_writ )
+{
+ * num_writ = fwrite ( buffer, 1, bytes, f );
+ if ( * num_writ != bytes )
+ return RC ( rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete );
+ return 0;
+}
+
+rc_t CC KMain( int argc, char *argv [] )
+{
+ rc_t rc = KOutHandlerSet ( write_to_FILE, stdout );
+ if ( rc == 0 )
+ {
+ if ( argc < 2 )
+ rc = perfrom_network_test( "SRR000001" );
+ else
+ rc = perfrom_network_test( argv[ 1 ] );
+ }
+ return rc;
+}
+
diff --git a/test/network-test/network-test.vers b/test/network-test/network-test.vers
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/test/network-test/network-test.vers
@@ -0,0 +1 @@
+1.0.0
diff --git a/test/network-test/network-testing.c b/test/network-test/network-testing.c
new file mode 100644
index 0000000..11a3766
--- /dev/null
+++ b/test/network-test/network-testing.c
@@ -0,0 +1,436 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <klib/out.h>
+#include <klib/rc.h>
+#include <klib/printf.h>
+#include <klib/log.h>
+#include <klib/text.h>
+#include <klib/data-buffer.h>
+#include <klib/time.h>
+
+#include <kns/manager.h>
+#include <kns/kns-mgr-priv.h> /* KNSManagerMakeReliableHttpFile */
+#include <kns/endpoint.h>
+#include <kns/http.h>
+#include <kns/stream.h>
+
+#include <kfs/file.h>
+
+#include <vfs/manager.h>
+#include <vfs/path.h>
+#include <vfs/resolver.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+
+static rc_t ipv4_endpoint_to_string( char * buffer, size_t buflen, KEndPoint * ep )
+{
+ uint32_t b[4];
+ b[0] = ( ep->u.ipv4.addr & 0xFF000000 ) >> 24;
+ b[1] = ( ep->u.ipv4.addr & 0xFF0000 ) >> 16;
+ b[2] = ( ep->u.ipv4.addr & 0xFF00 ) >> 8;
+ b[3] = ( ep->u.ipv4.addr & 0xFF );
+ return string_printf( buffer, buflen, NULL, "ipv4: %d.%d.%d.%d : %d",
+ b[0], b[1], b[2], b[3], ep->u.ipv4.port );
+}
+
+static rc_t ipv6_endpoint_to_string( char * buffer, size_t buflen, KEndPoint * ep )
+{
+ uint32_t b[8];
+ b[0] = ( ep->u.ipv6.addr[ 0 ] << 8 ) | ep->u.ipv6.addr[ 1 ];
+ b[1] = ( ep->u.ipv6.addr[ 2 ] << 8 ) | ep->u.ipv6.addr[ 3 ];
+ b[2] = ( ep->u.ipv6.addr[ 4 ] << 8 ) | ep->u.ipv6.addr[ 5 ];
+ b[3] = ( ep->u.ipv6.addr[ 6 ] << 8 ) | ep->u.ipv6.addr[ 7 ];
+ b[4] = ( ep->u.ipv6.addr[ 8 ] << 8 ) | ep->u.ipv6.addr[ 9 ];
+ b[5] = ( ep->u.ipv6.addr[ 10 ] << 8 ) | ep->u.ipv6.addr[ 11 ];
+ b[6] = ( ep->u.ipv6.addr[ 12 ] << 8 ) | ep->u.ipv6.addr[ 13 ];
+ b[7] = ( ep->u.ipv6.addr[ 14 ] << 8 ) | ep->u.ipv6.addr[ 15 ];
+ return string_printf( buffer, buflen, NULL, "ipv6: %.04X:%.04X:%.04X:%.04X:%.04X:%.04X:%.04X:%.04X: :%d",
+ b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], ep->u.ipv6.port );
+}
+
+static rc_t ipc_endpoint_to_string( char * buffer, size_t buflen, KEndPoint * ep )
+{
+ return string_printf( buffer, buflen, NULL, "ipc: %s", ep->u.ipc_name );
+}
+
+static rc_t endpoint_to_string( char * buffer, size_t buflen, KEndPoint * ep )
+{
+ rc_t rc;
+ switch( ep->type )
+ {
+ case epIPV4 : rc = ipv4_endpoint_to_string( buffer, buflen, ep ); break;
+ case epIPV6 : rc = ipv6_endpoint_to_string( buffer, buflen, ep ); break;
+ case epIPC : rc = ipc_endpoint_to_string( buffer, buflen, ep ); break;
+ default : rc = string_printf( buffer, buflen, NULL, "unknown endpoint-tyep %d", ep->type ); break;
+ }
+ return rc;
+}
+
+static rc_t perfrom_dns_test( KNSManager const * kns_mgr, const char * domain, uint16_t port )
+{
+ rc_t rc;
+ KEndPoint ep;
+ String s_domain;
+
+ KTimeMs_t start_time = KTimeMsStamp();
+ StringInitCString( &s_domain, domain );
+ rc = KNSManagerInitDNSEndpoint( kns_mgr, &ep, &s_domain, port );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "cannot init endpoint for $(URL):$(PORT)", "URL=%s,PORT=%d", domain, port );
+ else
+ {
+ char s_endpoint[ 1024 ];
+ rc = endpoint_to_string( s_endpoint, sizeof s_endpoint, &ep );
+ KOutMsg( "\nendpoint for %s:%d is: '%s'\n", domain, port, s_endpoint );
+ }
+ KOutMsg( "in %d milliseconds\n", KTimeMsStamp() - start_time );
+
+ return rc;
+}
+
+
+static rc_t read_stream_into_databuffer( KStream * stream, KDataBuffer * databuffer )
+{
+ rc_t rc;
+
+ size_t total = 0;
+ KDataBufferMakeBytes( databuffer, 4096 );
+ while ( 1 )
+ {
+ size_t num_read;
+ uint8_t * base;
+ uint64_t avail = databuffer->elem_count - total;
+ if ( avail < 256 )
+ {
+ rc = KDataBufferResize( databuffer, databuffer->elem_count + 4096 );
+ if ( rc != 0 )
+ {
+ LogErr( klogErr, rc, "CGI: KDataBufferResize failed" );
+ break;
+ }
+ }
+
+ base = databuffer->base;
+ rc = KStreamRead( stream, & base [ total ], ( size_t ) databuffer->elem_count - total, &num_read );
+ if ( rc != 0 )
+ {
+ /* TBD - look more closely at rc */
+ if ( num_read > 0 )
+ {
+ LogErr( klogErr, rc, "CGI: KStreamRead failed" );
+ rc = 0;
+ }
+ else
+ break;
+ }
+
+ if ( num_read == 0 )
+ break;
+
+ total += num_read;
+ }
+
+ if ( rc == 0 )
+ databuffer->elem_count = total;
+ return rc;
+}
+
+
+static rc_t call_cgi( KNSManager const * kns_mgr, const char * cgi_url, uint32_t ver_major, uint32_t ver_minor,
+ const char * protocol, const char * acc, KDataBuffer * databuffer )
+{
+ KHttpRequest * req;
+ rc_t rc = KNSManagerMakeReliableClientRequest( kns_mgr, &req, 0x01000000, NULL, cgi_url );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "CGI: cannot make ReliableClientRequest $(URL)", "URL=%s", cgi_url );
+ else
+ {
+ rc = KHttpRequestAddPostParam( req, "version=%u.%u", ver_major, ver_minor );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "CGI: KHttpRequestAddPostParam version=$(V1).$(V2) failed", "V1=%d,V2=%d", ver_major, ver_minor );
+
+ if ( rc == 0 )
+ {
+ rc = KHttpRequestAddPostParam( req, "acc=%s", acc );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "CGI: KHttpRequestAddPostParam acc=$(ACC) failed", "ACC=%s", acc );
+ }
+
+ if ( rc == 0 )
+ {
+ rc = KHttpRequestAddPostParam ( req, "accept-proto=%s", protocol );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "CGI: KHttpRequestAddPostParam accept-proto=$(PROTO) failed", "PROTO=%s", protocol );
+ }
+
+ if ( rc == 0 )
+ {
+ KHttpResult *rslt;
+ rc = KHttpRequestPOST( req, &rslt );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "CGI: KHttpRequestPOST failed" );
+ else
+ {
+ uint32_t code;
+ rc = KHttpResultStatus ( rslt, &code, NULL, 0, NULL );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "CGI: KHttpResultStatus failed" );
+ else
+ {
+ if ( code != 200 )
+ pLogErr( klogErr, rc, "CGI: unexpected result-code of $(RESCODE)", "RESCODE=%d", code );
+ else
+ {
+ KStream *response;
+
+ rc = KHttpResultGetInputStream ( rslt, &response );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "CGI: KHttpResultGetInputStream failed" );
+ else
+ {
+ rc = read_stream_into_databuffer( response, databuffer );
+ KStreamRelease ( response );
+ }
+ }
+ }
+ KHttpResultRelease ( rslt );
+ }
+ }
+ KHttpRequestRelease ( req );
+ }
+ return rc;
+}
+
+
+static rc_t perform_cgi_test( KNSManager const * kns_mgr, const char * acc )
+{
+ rc_t rc;
+ KDataBuffer databuffer;
+ KTimeMs_t start_time = KTimeMsStamp();
+
+ memset( &databuffer, 0, sizeof databuffer );
+ rc = call_cgi( kns_mgr, "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi", 1, 1, "http", acc, &databuffer );
+ if ( rc == 0 )
+ {
+ const char *start = ( const void* ) databuffer.base;
+ size_t size = KDataBufferBytes( &databuffer );
+ KOutMsg( "\nCGI: Response = %.*s\n", size, start );
+ }
+ KOutMsg( "in %d milliseconds\n\n", KTimeMsStamp() - start_time );
+
+ return rc;
+}
+
+
+static rc_t print_vpath( const char * prefix, const VPath * vpath, char * buffer, size_t buflen )
+{
+ const String * s_path;
+ rc_t rc = VPathMakeString( vpath, &s_path );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "VPathMakeString() for $(PREFIX) failed", "PREFIX=%s", prefix );
+ else
+ {
+ KOutMsg( "resolved (%s) : %S\n", prefix, s_path );
+ if ( buffer != NULL && buflen > 0 )
+ string_printf( buffer, buflen, NULL, "%S", s_path );
+ StringWhack( s_path );
+ }
+ return rc;
+}
+
+
+static rc_t perform_resolve_test( const char * acc, char * buffer, size_t buflen )
+{
+ VFSManager * vfs_mgr;
+ rc_t rc = VFSManagerMake( &vfs_mgr );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "VFSManagerMake failed" );
+ else
+ {
+ VResolver * resolver;
+ rc = VFSManagerGetResolver( vfs_mgr, &resolver );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "VFSManagerGetResolver failed" );
+ else
+ {
+ VPath * query_path;
+ rc = VFSManagerMakePath( vfs_mgr, &query_path, "ncbi-acc:%s", acc );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "VFSManagerMakePath failed" );
+ else
+ {
+ const VPath * local = NULL;
+ const VPath * remote = NULL;
+ const VPath * cache = NULL;
+
+ KTimeMs_t start_time = KTimeMsStamp();
+
+ rc = VResolverQuery( resolver, eProtocolHttp, query_path, &local, NULL, NULL );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "VResolverQuery (local) failed" );
+ else
+ {
+ rc = print_vpath( "local ", local, NULL, 0 );
+ VPathRelease ( local );
+ }
+ rc = VResolverQuery( resolver, eProtocolHttp, query_path, NULL, &remote, &cache );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "VResolverQuery (remote/cache) failed" );
+ else
+ {
+ if ( remote != NULL )
+ {
+ rc = print_vpath( "remote", remote, buffer, buflen );
+ VPathRelease ( remote );
+ }
+ if ( rc == 0 && cache != NULL )
+ {
+ rc = print_vpath( "cache ", cache, NULL, 0 );
+ VPathRelease ( cache );
+ }
+ }
+
+ KOutMsg( "in %d milliseconds\n", KTimeMsStamp() - start_time );
+ VPathRelease ( query_path );
+ }
+ VResolverRelease( resolver );
+ }
+ VFSManagerRelease ( vfs_mgr );
+ }
+ return rc;
+}
+
+
+static rc_t read_blocks( const KFile * remote_file, uint32_t block_count, uint64_t * total )
+{
+ rc_t rc = 0;
+ size_t block_size = 1024 * 1024 * 32;
+
+ *total = 0;
+ uint8_t * block = malloc( block_size );
+ if ( block == NULL )
+ {
+ rc = RC( rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted );
+ LogErr( klogErr, rc, "allocation of block failed" );
+ }
+ else
+ {
+ uint32_t i;
+ uint64_t pos = 0;
+ for ( i = 0; rc == 0 && i < block_count; ++i )
+ {
+ size_t num_read;
+ rc = KFileReadAll( remote_file, pos, block, block_size, &num_read );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "KFileReadAll() at '$(POS)' failed", "POS=%ld", pos );
+ else
+ pos += num_read;
+ }
+ *total = pos;
+ free( (void *) block );
+ }
+ return rc;
+}
+
+static rc_t perform_read_test( KNSManager const * kns_mgr, const char * remote_path )
+{
+
+ const KFile * remote_file;
+ KTimeMs_t start_time = KTimeMsStamp();
+ rc_t rc = KNSManagerMakeHttpFile ( kns_mgr, &remote_file, NULL, 0x0101, "%s", remote_path );
+ if ( rc != 0 )
+ pLogErr( klogErr, rc, "KNSManagerMakeHttpFile() '$(FILENAME)' failed", "FILENAME=%s", remote_path );
+ else
+ {
+ uint64_t total;
+ KOutMsg( "\n'%s' opened\nin %d milliseconds\n", remote_path, KTimeMsStamp() - start_time );
+ start_time = KTimeMsStamp();
+ rc = read_blocks( remote_file, 10, &total );
+ KOutMsg( "total of %ld bytes read in %d milliseconds\n\n", total, KTimeMsStamp() - start_time );
+ KFileRelease( remote_file );
+ }
+ return rc;
+}
+
+rc_t perfrom_network_test( const char * acc )
+{
+ KNSManager * kns_mgr;
+ rc_t rc = KNSManagerMake( &kns_mgr );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "cannot make KNS-Manager" );
+ else
+ {
+ if ( KNSManagerGetHTTPProxyEnabled( kns_mgr ) )
+ {
+ KOutMsg( "\nKNSManager: proxy enabled\n" );
+ {
+ const String * proxy;
+ rc = KNSManagerGetHTTPProxyPath( kns_mgr, &proxy );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "cannot request proxy-path from KNS-Manager" );
+ else
+ KOutMsg( "KNSManager: proxy at : %S\n", proxy );
+ }
+ }
+ else
+ KOutMsg( "\nKNSManager: proxy disabled\n" );
+
+ if ( rc == 0 )
+ {
+ const char * user_agent;
+ rc = KNSManagerGetUserAgent( &user_agent );
+ if ( rc != 0 )
+ LogErr( klogErr, rc, "cannot request user-agent from KNS-Manager" );
+ else
+ KOutMsg( "KNSManager: user-agent = '%s'\n", user_agent );
+ }
+
+ if ( rc == 0 )
+ rc = perfrom_dns_test( kns_mgr, "www.ncbi.nlm.nih.gov", 80 );
+
+ if ( rc == 0 )
+ rc = perform_cgi_test( kns_mgr, acc );
+
+ if ( rc == 0 )
+ {
+ char remote_path[ 4096 ];
+ remote_path[ 0 ] = 0;
+ rc = perform_resolve_test( acc, remote_path, sizeof remote_path );
+ if ( rc == 0 && remote_path[ 0 ] != 0 )
+ rc = perform_read_test( kns_mgr, remote_path );
+ }
+
+ KNSManagerRelease( kns_mgr );
+ }
+ return rc;
+}
diff --git a/tools/run-stat/mod_reads.h b/test/network-test/network-testing.h
similarity index 91%
copy from tools/run-stat/mod_reads.h
copy to test/network-test/network-testing.h
index 4dec794..00cacd8 100644
--- a/tools/run-stat/mod_reads.h
+++ b/test/network-test/network-testing.h
@@ -24,19 +24,19 @@
*
*/
-#ifndef _h_stat_mod_reads_
-#define _h_stat_mod_reads_
+#ifndef _h_network_testing_
+#define _h_network_testing_
#ifdef __cplusplus
extern "C" {
#endif
-#if 0
-}
-#endif
#include <klib/rc.h>
-rc_t mod_reads_init( p_module self, const char *param );
+rc_t perfrom_network_test( const char * acc );
+#ifdef __cplusplus
+}
+#endif
#endif
diff --git a/test/ngs-pileup/Makefile b/test/ngs-pileup/Makefile
new file mode 100644
index 0000000..3611f34
--- /dev/null
+++ b/test/ngs-pileup/Makefile
@@ -0,0 +1,140 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/ngs-pileup
+
+TEST_TOOLS = \
+ test-ngs-pileup
+
+include $(TOP)/build/Makefile.env
+
+$(TEST_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+.PHONY: $(TEST_TOOLS)
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# test-ngs-pileup
+#
+TEST_NGS_PILEUP_SRC = \
+ test-ngs-pileup
+
+TEST_NGS_PILEUP_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_NGS_PILEUP_SRC))
+
+TEST_NGS_PILEUP_LIB = \
+ -sngs-c++ \
+ -sncbi-ngs-c++ \
+ -sncbi-vdb-static \
+ -skapp \
+ -sktst \
+
+$(TEST_BINDIR)/test-ngs-pileup: $(TEST_NGS_PILEUP_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_NGS_PILEUP_LIB)
+
+valgrind_ngs_pileup: test-ngs-pileup
+ valgrind --ncbi --suppressions=$(SRCDIR)/valgrind.suppress $(TEST_BINDIR)/test-ngs-pileup
+
+#-------------------------------------------------------------------------------
+# slowtests: match output vs sra-pileup
+#
+
+slowtests: diff-vs-sra-pileup
+
+diff-vs-sra-pileup:
+ -@ ./runtestcase.sh $(BINDIR) $(SRCDIR) 1.0 SRR833251
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 2.0 SRR1166393
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 3.0 ERR334733 # ILLUMINA
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 4.0 SRR1068006
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 5.0 SRR619510 # ABSOLID REF_ROWS=36172
+ -@ ./runtestcase.sh $(BINDIR) $(SRCDIR) 6.0 SRR1251988 # SEC_ROWS=198
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 7.0 SRR1186012 # ION_TORRENT
+ -@ ./runtestcase.sh $(BINDIR) $(SRCDIR) 8.1 SRR556739 -r chrY # COMPLETE_GENOMICS
+ -@ ./runtestcase.sh $(BINDIR) $(SRCDIR) 8.2 SRR556739 -r chrM # COMPLETE_GENOMICS, circular reference
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 9.0 SRR341578 -r NC_011752.1 #:19900-20022
+
+onediff:
+ @ ./runtestcase.sh $(BINDIR) $(SRCDIR) 9.1 SRR341578 -r NC_011752.1:19900-20022
+
+#alignment selection:
+#TODO: multiple references in one accession
+#TODO: multiple accessions with overlapping alignments
+#TODO: filter by reference (-r ref) using canonic name
+#TODO: filter by reference (-r ref) using common name
+#TODO: filter by multiple references (-r ref1 -r ref2)
+#TODO: filter by position on the reference (-r ref:from-to)
+#TODO: filter by multiple positions on the same reference (-r ref:from1-to1 -r ref:from2-to2)
+#TODO: primary alignment table only (-t p)
+#TODO: secondary alignment table only (-t s)
+#TODO: evidence table only (-t e)
+#TODO: include duplicates (-d 1)
+#TODO: do not include duplicates (-d 0) NB: broken in sra-pileup
+#TODO: minimum mapq value (-q min)
+
+#output control:
+#TODO: omit qualities (-n)
+
+#output file control:
+#TODO: redirect output to a file (-o)
+#TODO: bzip2 output (--bzip2)
+#TODO: gzip output (--gzip)
+
+# hidden options:
+#TODO: output alignment information (-i)
+# any more?
+
+#not sure if these are needed:
+# -p|--spotgroups <spotgroups-modes> divide by spotgroups
+# -e|--seqname use original seq-name
+# --minmismatch min percent of mismatches used in function
+# mismatch, default is 5%
+# --merge-dist If adjacent slices are closer than this,
+# they are merged and skiplist is created.
+# a value of zero disables the feature,
+# default is 10000
+# --function ref list references
+# --function ref-ex list references coverage
+# --function count sort pileup with counters
+# --function stat strand/tlen statistic
+# --function mismatch only lines with mismatch
+# --function index list deletion counts
+# --function varcount variation counters: ref-name, ref-pos,
+# ref-base, coverage, mismatch A, mismatch C,
+# mismatch G, mismatch T, deletes, inserts, ins
+# after A, ins after C, ins after G, ins
+# after T
+# --function deletes list deletions greater then 20
+#
+#Grouping of accessions into artificial spotgroups:
+# sra-pileup SRRXXXXXX=a SRRYYYYYY=b SRRZZZZZZ=a
+
+
+
+.PHONY: diff-vs-sra-pileup
diff --git a/test/ngs-pileup/runtestcase.sh b/test/ngs-pileup/runtestcase.sh
new file mode 100755
index 0000000..b28c39b
--- /dev/null
+++ b/test/ngs-pileup/runtestcase.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+#echo "$0 $*"
+
+# $1 - path to sra tools (sra-pileup, ngs-pileup)
+# $2 - work directory (actual results and temporaries created under actual/)
+# $3 - test case ID
+# $4, $5, ... - command line options for pileup programs
+#
+# return codes:
+# 0 - passed
+# 1 - could not create temp dir
+# 2 - unexpected return code from sra-pileup
+# 3 - unexpected return code from ngs-pileup
+# 4 - outputs differ
+
+BINDIR=$1
+WORKDIR=$2
+CASEID=$3
+shift 3
+CMDLINE=$*
+
+SRA_PILEUP="$BINDIR/sra-pileup"
+NGS_PILEUP="$BINDIR/ngs-pileup"
+TEMPDIR=$WORKDIR/actual/$CASEID
+
+printf "running $CASEID: "
+
+mkdir -p $TEMPDIR
+rm -rf $TEMPDIR/*
+if [ "$?" != "0" ] ; then
+ exit 1
+fi
+
+CMD="$SRA_PILEUP $CMDLINE 1>$TEMPDIR/sra.stdout.tmp 2>$TEMPDIR/sra.stderr"
+printf "sra... "
+eval "$CMD"
+if [ "$?" != "0" ] ; then
+ echo "SRA pileup failed. Command executed:"
+ echo $CMD
+ cat $TEMPDIR/sra.stderr
+ exit 2
+fi
+cut -f 1,2,4 $TEMPDIR/sra.stdout.tmp >$TEMPDIR/sra.stdout
+
+CMD="$NGS_PILEUP $CMDLINE 1>$TEMPDIR/ngs.stdout 2>$TEMPDIR/ngs.stderr"
+printf "ngs... "
+eval "$CMD"
+if [ "$?" != "0" ] ; then
+ echo "NGS pileup failed. Command executed:"
+ echo $CMD
+ cat $TEMPDIR/ngs.stderr
+ exit 3
+fi
+
+printf "diff... "
+diff $TEMPDIR/sra.stdout $TEMPDIR/ngs.stdout >$TEMPDIR/diff
+if [ "$?" != "0" ] ; then
+ cat $TEMPDIR/diff
+ echo "command executed:"
+ echo $CMD
+ exit 4
+fi
+
+printf "done\n"
+rm -rf $TEMPDIR
+
+exit 0
diff --git a/test/ngs-pileup/test-ngs-pileup.cpp b/test/ngs-pileup/test-ngs-pileup.cpp
new file mode 100644
index 0000000..0711d84
--- /dev/null
+++ b/test/ngs-pileup/test-ngs-pileup.cpp
@@ -0,0 +1,177 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for NGS Pileup
+*/
+
+#include <ktst/unit_test.hpp>
+
+#include <sysalloc.h>
+
+#include <sstream>
+
+#include "../../tools/ngs-pileup/ngs-pileup.cpp"
+
+using namespace std;
+using namespace ncbi::NK;
+
+TEST_SUITE(NgsPileupTestSuite);
+
+class NGSPileupFixture
+{
+public:
+ NGSPileupFixture()
+ {
+ ps . output = & m_str;
+ }
+
+ string Run()
+ {
+ NGS_Pileup ( ps ) . Run ();
+ return m_str . str ();
+ }
+
+ ostringstream m_str;
+ NGS_Pileup::Settings ps;
+};
+
+FIXTURE_TEST_CASE ( NoInput, NGSPileupFixture )
+{
+ Run ();
+ REQUIRE_EQ ( string(), m_str . str() );
+}
+
+FIXTURE_TEST_CASE ( BadInput, NGSPileupFixture )
+{
+ ps . AddInput ( "blah" );
+ REQUIRE_THROW ( Run() );
+ REQUIRE_EQ ( string(), m_str . str() );
+}
+
+FIXTURE_TEST_CASE ( Basic, NGSPileupFixture )
+{
+ ps . AddInput ( "SRR833251" ); // a small accession with primary and secondary alignments
+ string expectedStart = "gi|169794206|ref|NC_010410.1|\t19376\t1\n"; //TODO: expand when pileup prints out more data
+ REQUIRE_EQ ( expectedStart, Run () . substr ( 0, expectedStart . length () ) );
+}
+
+FIXTURE_TEST_CASE ( SingleReference_ByCommonName, NGSPileupFixture )
+{
+ ps . AddInput ( "ERR247027" );
+ ps . AddReference ( "Pf3D7_13" );
+ string expectedStart = "AL844509.2\t1212494\t1"; //TODO: expand when pileup prints out more data
+ string actual = Run ();
+ REQUIRE_EQ ( expectedStart, actual . substr ( 0, expectedStart . length () ) );
+}
+
+FIXTURE_TEST_CASE ( SingleReference_ByCanonicalName, NGSPileupFixture )
+{
+ ps . AddInput ( "ERR247027" );
+ ps . AddReference ( "AL844509.2" );
+ string expectedStart = "AL844509.2\t1212494\t1"; //TODO: expand when pileup prints out more data
+ REQUIRE_EQ ( expectedStart, Run () . substr ( 0, expectedStart . length () ) );
+}
+
+FIXTURE_TEST_CASE ( SingleReference_Slice, NGSPileupFixture )
+{
+ ps . AddInput ( "ERR247027" );
+ ps . AddReferenceSlice ( "AL844509.2", 1212492, 3 );
+ string expected =
+ "AL844509.2\t1212494\t1\n" /* this position is 1-based */
+ "AL844509.2\t1212495\t1\n";
+ //TODO: expand when pileup prints out more data
+ REQUIRE_EQ ( expected, Run () );
+}
+
+#if 0
+FIXTURE_TEST_CASE ( MultipleReferences, NGSPileupFixture )
+{
+ ps . AddInput ( "SRR1068024" ); // 38 references
+ const string ref1 = "0000000.72b.NC2_17738";
+ const string ref2 = "0000000.72b.NC2_14823"; // this one should come first in the output
+ ps . AddReference ( ref1 );
+ ps . AddReference ( ref2 );
+ string expectedStart = ref2 + "\t1\t1";
+
+ string res = Run();
+
+ REQUIRE_EQ ( expectedStart, res . substr ( 0, expectedStart . length () ) );
+ REQUIRE_NE ( string :: npos, res. find ( ref2 ) );
+}
+
+FIXTURE_TEST_CASE ( MultipleInputs, NGSPileupFixture )
+{ // ERR334733 ERR334777 align against the same reference, ERR334733's position comes first so it has to
+ // be the first on the output regardless of order of accessions on the command line
+ ps . AddInput ( "ERR334777" );
+ ps . AddInput ( "ERR334733" );
+ string expectedStart = "FN433596.1\t805952\t1"; // comes from ERR334733
+ string res = Run();
+ REQUIRE_EQ ( expectedStart, res . substr ( 0, expectedStart . length () ) );
+ REQUIRE_NE ( string :: npos, res. find ( "2424446" ) ); // one of the positions from ERR334777
+}
+
+FIXTURE_TEST_CASE ( AllReferencses, NGSPileupFixture )
+{
+ ps . AddInput ( "SRR341578" );
+ string expectedStart = "NC_011748.1\t1\t1";
+ REQUIRE_EQ ( expectedStart, Run () . substr ( 0, expectedStart . length () ) );
+}
+
+//TODO: multiple input overlapping
+#endif
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+rc_t CC UsageSummary (const char * progname)
+{
+ return 0;
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ return 0;
+}
+
+const char UsageDefaultName[] = "test-ngs-pileup";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ KConfigDisableUserSettings();
+ rc_t rc=NgsPileupTestSuite(argc, argv);
+ return rc;
+}
+
+}
+
diff --git a/test/ngs-pileup/valgrind.suppress b/test/ngs-pileup/valgrind.suppress
new file mode 100644
index 0000000..002d332
--- /dev/null
+++ b/test/ngs-pileup/valgrind.suppress
@@ -0,0 +1,6 @@
+{
+ vt_cache
+ Memcheck:Leak
+ fun:calloc
+ fun:_ZN3ngs7Resolve*
+}
diff --git a/test/only-remote-repository.kfg b/test/only-remote-repository.kfg
new file mode 100644
index 0000000..2e7ef51
--- /dev/null
+++ b/test/only-remote-repository.kfg
@@ -0,0 +1,2 @@
+/repository/remote/main/CGI/resolver-cgi = "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
+/repository/user/main/public/cache-enabled = "false"
diff --git a/test/ref-variation/Makefile b/test/ref-variation/Makefile
new file mode 100644
index 0000000..2877a3f
--- /dev/null
+++ b/test/ref-variation/Makefile
@@ -0,0 +1,57 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/ref-variation
+
+TEST_TOOLS = \
+
+ALL_TOOLS = \
+ $(TEST_TOOLS) \
+
+include $(TOP)/build/Makefile.env
+
+$(ALL_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+.PHONY: $(TEST_TOOLS)
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# ref-variation tool tests
+#
+runtests:
+
+one-tool: $(BINDIR)/ref-variation
+ #$(BINDIR)/ref-variation -t 16 -r NC_000002.11 -p 73613067 --query '-' -l 3 SRR1597772 SRR1597776 SRR1597780 SRR1597784 SRR1597788 SRR1597792 SRR1597796 SRR1597800 SRR1597915 SRR1597919 SRR1597924 SRR1597928 SRR1597932 SRR1597936 SRR1597940 SRR1597944 SRR1597950 SRR1597955 SRR1598087 SRR1598091 SRR1598097 SRR1598101 SRR1598105 SRR1598111 SRR1598118 SRR945586 SRR867664 SRR867668 SRR867672 SRR867676 SRR867680 SRR867684 SRR867688 SRR867692 SRR851163 SRR792574 SRR792583 SRR792595 SRR792600 [...]
+ $(BINDIR)/ref-variation -t 16 -r NC_000002.11 -p 73613067 --query '-' -l 3 SRR1597772 SRR1597776 SRR1597780 SRR1597784 SRR1597788 SRR1597792 SRR1597796 SRR1597800 SRR1597915 SRR1597919 SRR1597924 SRR1597928 SRR1597932 SRR1597936 SRR1597940 SRR1597944 SRR1597950 SRR1597955 SRR1598087 SRR1598091 SRR1598097 SRR1598101 SRR1598105 SRR1598111 SRR1598118 SRR945586 SRR867664 SRR867668 SRR867672 SRR867676 SRR867680 SRR867684 SRR867688 SRR867692 SRR851163 SRR792574 SRR792583 SRR792595 SRR792600 S [...]
+ #$(BINDIR)/ref-variation -t 2 -r NC_000002.11 -p 73613067 --query '-' -l 3 SRR1597772 SRR1597776
+
+vg: $(BINDIR)/ref-variation
+ valgrind --ncbi --suppressions=$(SRCDIR)/valgrind.suppress $(BINDIR)/ref-variation -r NC_000002.11 -p 73613067 --query '-' -l 3 \
+ SRR1597772 SRR1597776 SRR1597780 SRR1597784 SRR1597788 SRR1597792 SRR1597796 SRR1597800 SRR1597915 SRR1597919 SRR1597924 SRR1597928 SRR1597932 SRR1597936 SRR1597940 SRR1597944 SRR1597950 SRR1597955 SRR1598087 SRR1598091 SRR1598097 SRR1598101 SRR1598105 SRR1598111 SRR1598118 SRR945586 SRR867664 SRR867668 SRR867672 SRR867676 SRR867680 SRR867684 SRR867688 SRR867692 SRR851163 SRR792574 SRR792583 SRR792595 SRR792600 SRR792609 SRR792617 SRR792634 SRR792699 SRR792723 SRR792865 SRR792873 SRR [...]
diff --git a/test/ref-variation/valgrind.suppress b/test/ref-variation/valgrind.suppress
new file mode 100644
index 0000000..002d332
--- /dev/null
+++ b/test/ref-variation/valgrind.suppress
@@ -0,0 +1,6 @@
+{
+ vt_cache
+ Memcheck:Leak
+ fun:calloc
+ fun:_ZN3ngs7Resolve*
+}
diff --git a/tools/sra-dbcc/Makefile b/test/samline/Makefile
similarity index 86%
copy from tools/sra-dbcc/Makefile
copy to test/samline/Makefile
index 4d4646e..e3a76ef 100644
--- a/tools/sra-dbcc/Makefile
+++ b/test/samline/Makefile
@@ -26,21 +26,20 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/sra-dbcc
+MODULE = test/samline
include $(TOP)/build/Makefile.env
INT_TOOLS = \
-
EXT_TOOLS = \
- sra-dbcc
+ samline
ALL_TOOLS = \
$(INT_TOOLS) \
$(EXT_TOOLS)
-#------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
# outer targets
#
all std: vers-includes
@@ -82,21 +81,20 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# sra-dbcc
+# TOOL_SRC
#
-SRA_DBCC_SRC = \
- sra-dbcc
-
-SRA_DBCC_OBJ = \
- $(addsuffix .$(OBJX),$(SRA_DBCC_SRC))
-
-SRA_DBCC_LIB = \
- -lkapp \
- -lncbi-vdb \
+TOOL_SRC = \
+ refbases \
+ cigar \
+ alig-gen
-sra-dbcc.vers.h: sra-dbcc.vers
- $(TOP)/build/make-vers-inc.sh SRA_DBCC_VERS $^ > $@
+TOOL_OBJ = \
+ $(addsuffix .$(OBJX),$(TOOL_SRC))
-$(BINDIR)/sra-dbcc: $(SRA_DBCC_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SRA_DBCC_LIB)
+TOOL_LIB = \
+ -skapp \
+ -sncbi-vdb \
+ -lm
+$(BINDIR)/samline: $(TOOL_OBJ)
+ $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(TOOL_LIB)
diff --git a/test/samline/alig-gen.c b/test/samline/alig-gen.c
new file mode 100644
index 0000000..07f1f9b
--- /dev/null
+++ b/test/samline/alig-gen.c
@@ -0,0 +1,604 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "samline.vers.h"
+
+#include <kfs/file.h>
+#include <kfs/directory.h>
+
+#include <kapp/main.h>
+#include <kapp/args.h>
+
+#include <klib/out.h>
+#include <klib/rc.h>
+#include <klib/printf.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "refbases.h"
+#include "cigar.h"
+
+#define DFLT_REFNAME "NC_011752.1"
+#define DFLT_REFPOS 10000
+#define DFLT_CIGAR "50M"
+#define DFLT_INSBASES "ACGTACGTACGT"
+#define DFLT_MAPQ 20
+#define DFLT_QNAME "1"
+#define DFLT_TLEN 0
+
+static const char * refname_usage[] = { "the ref-seq-id to use 'NC_011752.1'", NULL };
+static const char * refpos_usage[] = { "the position on the reference 0-based", NULL };
+static const char * cigar_usage[] = { "the cigar-string to use", NULL };
+static const char * insbases_usage[] = { "what bases to insert ( if needed )", NULL };
+static const char * mapq_usage[] = { "what mapq to use", NULL };
+static const char * reverse_usage[] = { "alignment is reverse", NULL };
+static const char * qname_usage[] = { "query template name", NULL };
+static const char * sec_usage[] = { "secondary alignment", NULL };
+static const char * bad_usage[] = { "did not pass quality control", NULL };
+static const char * dup_usage[] = { "is PCR or optical duplicate", NULL };
+static const char * prop_usage[] = { "each fragment is properly aligned", NULL };
+static const char * show_usage[] = { "show details of calculations", NULL };
+static const char * ref_usage[] = { "return only refbases (set cigar to 100M for len=100)", NULL };
+static const char * flags_usage[] = { "decode decimal flags-value", NULL };
+static const char * header_usage[] = { "produce header", NULL };
+static const char * config_usage[] = { "procuce config-file", NULL };
+static const char * mdtag_usage[] = { "procuce md-tag", NULL };
+
+#define OPTION_REFNAME "refname"
+#define OPTION_REFPOS "refpos"
+#define OPTION_CIGAR "cigar"
+#define OPTION_INSBASES "insbases"
+#define OPTION_MAPQ "mapq"
+#define OPTION_REVERSE "reverse"
+#define OPTION_QNAME "qname"
+#define OPTION_SEC "secondary"
+#define OPTION_BAD "bad"
+#define OPTION_DUP "duplicate"
+#define OPTION_PROP "proper"
+#define OPTION_SHOW "show"
+#define OPTION_REF "ref"
+#define OPTION_FLAGS "flags"
+#define OPTION_HEADER "header"
+#define OPTION_CONFIG "config"
+#define OPTION_MDTAG "mdtag"
+
+#define ALIAS_REFNAME "r"
+#define ALIAS_REFPOS "p"
+#define ALIAS_CIGAR "c"
+#define ALIAS_INSBASES "i"
+#define ALIAS_MAPQ "m"
+#define ALIAS_REVERSE "e"
+#define ALIAS_QNAME "q"
+#define ALIAS_SEC "2"
+#define ALIAS_BAD "a"
+#define ALIAS_DUP "u"
+#define ALIAS_PROP "o"
+#define ALIAS_SHOW "s"
+#define ALIAS_REF "f"
+#define ALIAS_FLAGS "l"
+#define ALIAS_HEADER "d"
+#define ALIAS_CONFIG "n"
+#define ALIAS_MDTAG "t"
+
+OptDef Options[] =
+{
+ { OPTION_REFNAME, ALIAS_REFNAME, NULL, refname_usage, 2, true, false },
+ { OPTION_REFPOS, ALIAS_REFPOS, NULL, refpos_usage, 2, true, false },
+ { OPTION_CIGAR, ALIAS_CIGAR, NULL, cigar_usage, 2, true, false },
+ { OPTION_INSBASES, ALIAS_INSBASES, NULL, insbases_usage, 1, true, false },
+ { OPTION_MAPQ, ALIAS_MAPQ, NULL, mapq_usage, 2, true, false },
+ { OPTION_REVERSE, ALIAS_REVERSE, NULL, reverse_usage, 1, false, false },
+ { OPTION_QNAME, ALIAS_QNAME, NULL, qname_usage, 2, true, false },
+ { OPTION_SEC, ALIAS_SEC, NULL, sec_usage, 2, false, false },
+ { OPTION_BAD, ALIAS_BAD, NULL, bad_usage, 2, false, false },
+ { OPTION_DUP, ALIAS_DUP, NULL, dup_usage, 2, false, false },
+ { OPTION_PROP, ALIAS_PROP, NULL, prop_usage, 2, false, false },
+ { OPTION_SHOW, ALIAS_SHOW, NULL, show_usage, 1, false, false },
+ { OPTION_REF, ALIAS_REF, NULL, ref_usage, 1, false, false },
+ { OPTION_FLAGS, ALIAS_FLAGS, NULL, flags_usage, 1, true, false },
+ { OPTION_HEADER, ALIAS_HEADER, NULL, header_usage, 1, false, false },
+ { OPTION_CONFIG, ALIAS_CONFIG, NULL, config_usage, 1, true, false },
+ { OPTION_MDTAG, ALIAS_MDTAG, NULL, mdtag_usage, 1, false, false }
+};
+
+const char UsageDefaultName[] = "samline";
+
+rc_t CC UsageSummary ( const char * progname )
+{
+ return KOutMsg( "\nUsage:\n %s [options]\n\n", progname );
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ int i, n_options;
+ rc_t rc;
+
+ if ( args == NULL )
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
+ else
+ rc = ArgsProgram ( args, &fullpath, &progname );
+
+ if ( rc != 0 )
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary ( progname );
+ KOutMsg ( "Options:\n" );
+
+ n_options = sizeof Options / sizeof Options[ 0 ];
+ for ( i = 0; i < n_options; ++i )
+ {
+ OptDef * o = &Options[ i ];
+ HelpOptionLine( o->aliases, o->name, NULL, o->help );
+ }
+
+ KOutMsg ( "\n" );
+ HelpOptionsStandard ();
+ HelpVersion ( fullpath, KAppVersion() );
+
+ return rc;
+}
+
+
+/* Version EXTERN
+ * return 4-part version code: 0xMMmmrrrr, where
+ * MM = major release
+ * mm = minor release
+ * rrrr = bug-fix release
+ */
+ver_t CC KAppVersion ( void )
+{
+ return SAMLINE_VERS;
+}
+
+
+static const char * get_str_option( const Args * args, const char * name, uint32_t idx, const char * dflt )
+{
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, name, &count );
+ if ( ( rc == 0 )&&( count > idx ) )
+ {
+ const char * res = NULL;
+ ArgsOptionValue( args, name, idx, (const void **)&res );
+ return res;
+ }
+ else
+ return dflt;
+}
+
+
+static uint32_t get_uint32_option( const Args * args, const char * name, uint32_t idx, const uint32_t dflt )
+{
+ const char * s = get_str_option( args, name, idx, NULL );
+ if ( s == NULL )
+ return dflt;
+ return atoi( s );
+}
+
+static uint32_t get_bool_option( const Args * args, const char * name )
+{
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, name, &count );
+ return ( rc == 0 && count > 0 );
+}
+
+typedef struct alignment
+{
+ const char * refname;
+ const char * cigar_str;
+ const char * refbases;
+ char read[ 4096 ];
+ char sam[ 4096 ];
+
+ int reverse, secondary, bad, dup, prop;
+
+ uint32_t refpos, mapq, bases_in_ref, reflen;
+
+ struct cigar_t * cigar;
+} alignment;
+
+typedef struct gen_context
+{
+ const char * qname;
+ const char * insbases;
+ const char * config;
+ uint32_t flags, header;
+ int32_t tlen;
+
+ alignment alig[ 2 ];
+} gen_context;
+
+
+static rc_t CC write_to_FILE ( void *f, const char *buffer, size_t bytes, size_t *num_writ )
+{
+ * num_writ = fwrite ( buffer, 1, bytes, f );
+ if ( * num_writ != bytes )
+ return RC ( rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete );
+ return 0;
+}
+
+
+static size_t random_string( char * buffer, size_t buflen, const char * char_set, size_t length )
+{
+ size_t res = 0;
+ if ( buffer != NULL && buflen > 0 )
+ {
+ const char dflt_charset[] = "0123456789"
+ "abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+ const char * cs = ( char_set == NULL ) ? dflt_charset : char_set;
+ size_t charset_len = strlen( cs ) - 1;
+ while ( res < length && res < ( buflen - 1 ) )
+ {
+ size_t rand_idx = ( double ) rand() / RAND_MAX * charset_len;
+ buffer[ res++ ] = cs[ rand_idx ];
+ }
+ buffer[ res ] = 0;
+ }
+ return res;
+}
+
+
+static size_t random_quality( char * buffer, size_t buflen, size_t length )
+{
+ const char qualities[] = "!\"#$%&'()*+,-./0123456789:;<=>?"
+ "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"
+ "`abcdefghijklmnopqrstuvwxyz{|}~";
+ return random_string( buffer, buflen, qualities, length );
+}
+
+static uint32_t sam_flags( const alignment * alig, const alignment * other, int first, int last )
+{
+ uint32_t res = 0;
+ if ( first || last ) res |= 0x01; /* multiple fragments */
+ if ( alig->prop != 0 ) res |= 0x02; /* each fragment properly aligned */
+ if ( alig->refpos == 0 ) res |= 0x04; /* this fragment is unmapped */
+ if ( other != NULL && other->refpos == 0 ) res |= 0x08; /* next fragment is unmapped */
+ if ( alig->reverse ) res |= 0x10; /* this fragment is reversed */
+ if ( other != NULL && other->reverse ) res |= 0x20; /* next fragment is reversed */
+ if ( first ) res |= 0x40; /* this is the first fragment */
+ if ( last ) res |= 0x80; /* this is the last fragment */
+ if ( alig->secondary != 0 ) res |= 0x100; /* this is a secondary alignment */
+ if ( alig->bad != 0 ) res |= 0x200; /* this is did not pass quality controls */
+ if ( alig->dup != 0 ) res |= 0x400; /* this is PCR or optical duplicate */
+ return res;
+}
+
+
+static size_t produce_sam( char * buffer, size_t buflen,
+ const gen_context * gctx, const alignment * alig, const alignment * other )
+{
+ size_t res = 0;
+ if ( buffer != NULL ) buffer[ 0 ] = 0;
+
+ if ( buffer != NULL && gctx != NULL && alig != NULL )
+ {
+ char merged_cigar_str[ 4096 ];
+ char quality[ 4096 ];
+ int first = 0;
+ int last = 0;
+ const char * r_next = "*";
+ uint32_t r_pos = 0;
+ struct cigar_t * merged_cigar = merge_cigar_t( alig->cigar );
+
+ random_quality( quality, sizeof quality, cigar_t_readlen( merged_cigar ) ) ;
+ cigar_t_string( merged_cigar_str, sizeof merged_cigar_str, merged_cigar );
+
+ if ( other != NULL && other->refname != NULL && other->refpos != 0 )
+ {
+ r_next = other->refname;
+ first = ( alig->refpos < other->refpos );
+ last = !first;
+ r_pos = other->refpos;
+ }
+
+ string_printf ( buffer, buflen, &res,
+ "%s\t%d\t%s\t%d\t%d\t%s\t%s\t%d\t%d\t%s\t%s",
+ gctx->qname,
+ sam_flags( alig, other, first, last ),
+ alig->refname,
+ alig->refpos,
+ alig->mapq,
+ merged_cigar_str,
+ r_next,
+ r_pos,
+ gctx->tlen,
+ alig->read,
+ quality );
+
+ free_cigar_t( merged_cigar );
+ }
+ return res;
+}
+
+static void show_alig_details( const alignment * alig )
+{
+ KOutMsg ( "REFNAME : %s\n", alig->refname );
+ KOutMsg ( "REFPOS : %d\n", alig->refpos );
+ KOutMsg ( "CIGAR : %s\n", alig->cigar_str );
+ KOutMsg ( "MAPQ : %d\n", alig->mapq );
+ KOutMsg ( "REVERSE : %s\n", alig->reverse ? "YES" : "NO" );
+ KOutMsg ( "SECONDARY: %s\n", alig->secondary ? "YES" : "NO" );
+ KOutMsg ( "BAD : %s\n", alig->bad ? "YES" : "NO" );
+ KOutMsg ( "DUPLICATE: %s\n", alig->dup ? "YES" : "NO" );
+ KOutMsg ( "PROPERLY : %s\n", alig->prop ? "YES" : "NO" );
+ KOutMsg ( "REFLEN : %d\n", alig->reflen );
+ KOutMsg ( "READLEN : %d\n", cigar_t_readlen( alig->cigar ) );
+ KOutMsg ( "INSLEN : %d\n", cigar_t_inslen( alig->cigar ) );
+ KOutMsg ( "REFBASES : %s\n", alig->refbases );
+ KOutMsg ( "READ : %s\n", alig->read );
+ KOutMsg ( "SAM : %s\n", alig->sam );
+
+}
+
+static void show_details( const gen_context * gctx )
+{
+ KOutMsg ( "QNAME : %s\n", gctx->qname );
+ KOutMsg ( "INSBASES : %s\n", gctx->insbases );
+ KOutMsg ( "TLEN : %d\n", gctx->tlen );
+ KOutMsg ( "CONFIG : %s\n", gctx->config );
+ if ( gctx->tlen != 0 )
+ {
+ KOutMsg ( "----- ALIGNMENT #1 -----\n" );
+ show_alig_details( &gctx->alig[ 0 ] );
+ KOutMsg ( "----- ALIGNMENT #2 -----\n" );
+ show_alig_details( &gctx->alig[ 1 ] );
+ }
+ else
+ show_alig_details( &gctx->alig[ 0 ] );
+}
+
+
+static void show_mdtag( const gen_context * gctx )
+{
+ struct cigar_t * cigar;
+
+ KOutMsg ( "calculating MD-TAG:\n" );
+ KOutMsg ( "READ : %s\n", gctx->alig[0].read );
+ KOutMsg ( "REFBASES : %s\n", gctx->alig[0].refbases );
+
+ cigar = make_cigar_t( gctx->alig[0].cigar_str );
+ if ( cigar != NULL )
+ {
+ struct cigar_t *merged_cigar = merge_cigar_t( cigar );
+ if ( merged_cigar != NULL )
+ {
+ char merged_cigar_str[ 4096 ];
+ char the_tag[ 4096 ];
+
+ cigar_t_string( merged_cigar_str, sizeof merged_cigar_str, merged_cigar );
+ md_tag( the_tag, sizeof the_tag, merged_cigar, gctx->alig[0].read, gctx->alig[0].refbases );
+
+ KOutMsg ( "CIGAR : %s\n", merged_cigar_str );
+ KOutMsg ( "MD-TAG : %s\n", the_tag );
+
+ free_cigar_t( merged_cigar );
+ }
+ free_cigar_t( cigar );
+ }
+}
+
+static void explain_flags( const uint32_t flags )
+{
+ if ( ( flags & 0x01 ) == 0x01 )
+ KOutMsg ( "0x001 ... multiple fragments\n" );
+ if ( ( flags & 0x02 ) == 0x02 )
+ KOutMsg ( "0x002 ... each fragment properly aligned\n" );
+ if ( ( flags & 0x04 ) == 0x04 )
+ KOutMsg ( "0x004 ... this fragment is unmapped\n" );
+ if ( ( flags & 0x08 ) == 0x08 )
+ KOutMsg ( "0x008 ... next fragment is unmapped\n" );
+ if ( ( flags & 0x10 ) == 0x10 )
+ KOutMsg ( "0x010 ... this fragment is reversed\n" );
+ if ( ( flags & 0x20 ) == 0x20 )
+ KOutMsg ( "0x020 ... next fragment is reversed\n" );
+ if ( ( flags & 0x40 ) == 0x40 )
+ KOutMsg ( "0x040 ... this is the first fragment\n" );
+ if ( ( flags & 0x80 ) == 0x80 )
+ KOutMsg ( "0x080 ... this is the last fragment\n" );
+ if ( ( flags & 0x100 ) == 0x100 )
+ KOutMsg ( "0x100 ... this is a secondary alignment\n" );
+ if ( ( flags & 0x200 ) == 0x200 )
+ KOutMsg ( "0x200 ... this is did not pass quality controls\n" );
+ if ( ( flags & 0x400 ) == 0x400 )
+ KOutMsg ( "0x400 ... this is PCR or optical duplicate\n" );
+}
+
+
+static size_t write_config_line( KFile * dst, size_t at, const char * refname0, const char * refname1 )
+{
+ size_t num_in_buffer, res = 0;
+ char buffer[ 4096 ];
+ rc_t rc = string_printf ( buffer, sizeof buffer, &num_in_buffer, "%s\t%s\n", refname0, refname1 );
+ if ( rc == 0 )
+ {
+ size_t written_to_file;
+ rc = KFileWriteAll ( dst, at, buffer, num_in_buffer, &written_to_file );
+ if ( rc == 0 )
+ res = at + written_to_file;
+ }
+ return res;
+}
+
+
+static void write_config_file( const char * filename, const char * refname0, const char * refname1 )
+{
+ KDirectory *dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+ KFile * dst;
+ rc = KDirectoryCreateFile ( dir, &dst, false, 0664, kcmInit, filename );
+ if ( rc == 0 )
+ {
+ size_t pos = write_config_line( dst, 0, refname0, refname0 );
+ if ( pos > 0 && ( strcmp( refname0, refname1 ) != 0 ) )
+ write_config_line( dst, pos, refname1, refname1 );
+ KFileRelease( dst );
+ }
+ KDirectoryRelease( dir );
+ }
+}
+
+
+static void generate_alignment( const gen_context * gctx )
+{
+ /* write reference names into config-file for bam-load */
+ if ( gctx->config != NULL )
+ write_config_file( gctx->config, gctx->alig[ 0 ].refname, gctx->alig[ 1 ].refname );
+
+ /* procude SAM-header on stdout */
+ if ( gctx->header )
+ {
+ const char * refname = gctx->alig[ 0 ].refname;
+ int bases_in_ref = gctx->alig[ 0 ].bases_in_ref;
+ KOutMsg( "@HD\tVN:1.3\n" );
+ KOutMsg( "@SQ\tSN:%s\tAS:%s\tLN:%d\n", refname, refname, bases_in_ref );
+ }
+
+ /* produces SAM-line for 1st alignment */
+ KOutMsg( "%s\n", gctx->alig[ 0 ].sam );
+
+ /* produces SAM-line for 2nd alignment ( mate ) */
+ if ( gctx->tlen != 0 )
+ KOutMsg( "%s\n", gctx->alig[ 1 ].sam );
+}
+
+
+static void read_alig_context( Args * args, gen_context * gctx, alignment * alig, uint32_t idx )
+{
+ alig->refname = get_str_option( args, OPTION_REFNAME, idx, idx == 0 ? DFLT_REFNAME : NULL );
+ alig->refpos = get_uint32_option( args, OPTION_REFPOS, idx, idx == 0 ? DFLT_REFPOS : 0 );
+ alig->cigar_str = get_str_option( args, OPTION_CIGAR, idx, DFLT_CIGAR );
+ alig->mapq = get_uint32_option( args, OPTION_MAPQ, idx, DFLT_MAPQ );
+
+ alig->reverse = get_uint32_option( args, OPTION_REVERSE, idx, 0 );
+ alig->secondary = get_uint32_option( args, OPTION_SEC, idx, 0 );
+ alig->bad = get_uint32_option( args, OPTION_BAD, idx, 0 );
+ alig->dup = get_uint32_option( args, OPTION_DUP, idx, 0 );
+ alig->prop = get_uint32_option( args, OPTION_PROP, idx, 0 );
+
+ /* precalculate values need in all functions */
+ alig->cigar = make_cigar_t( alig->cigar_str );
+ alig->reflen = cigar_t_reflen( alig->cigar );
+}
+
+static void release_alig( alignment * alig )
+{
+ if ( alig->refbases != NULL ) free( ( void* ) alig->refbases );
+ free_cigar_t( alig->cigar );
+}
+
+static void read_context( Args * args, gen_context * gctx )
+{
+ alignment *alig0, *alig1;
+
+ gctx->qname = get_str_option( args, OPTION_QNAME, 0, DFLT_QNAME );
+ gctx->insbases = get_str_option( args, OPTION_INSBASES, 0, DFLT_INSBASES );
+ gctx->flags = get_uint32_option( args, OPTION_FLAGS, 0, 0 );
+ gctx->header = get_bool_option( args, OPTION_HEADER );
+ gctx->config = get_str_option( args, OPTION_CONFIG, 0, NULL );
+ gctx->tlen = 0;
+
+ read_alig_context( args, gctx, &gctx->alig[ 0 ], 0 );
+ read_alig_context( args, gctx, &gctx->alig[ 1 ], 1 );
+
+ alig0 = &gctx->alig[ 0 ];
+ alig1 = &gctx->alig[ 1 ];
+
+ if ( gctx->alig[ 1 ].refpos > 0 )
+ {
+ if ( gctx->alig[ 1 ].refname == NULL )
+ gctx->alig[ 1 ].refname = gctx->alig[ 0 ].refname;
+
+ uint32_t end = alig1->refpos + alig1->reflen;
+ gctx->tlen = ( end - alig0->refpos );
+
+ alig0->refbases = read_refbases( alig0->refname, alig0->refpos, alig0->reflen, &alig0->bases_in_ref );
+ alig1->refbases = read_refbases( alig1->refname, alig1->refpos, alig1->reflen, &alig1->bases_in_ref );
+
+ cigar_t_2_read( alig0->read, sizeof alig0->read, alig0->cigar, alig0->refbases, gctx->insbases );
+ cigar_t_2_read( alig1->read, sizeof alig1->read, alig1->cigar, alig1->refbases, gctx->insbases );
+
+ produce_sam( alig0->sam, sizeof alig0->sam, gctx, alig0, alig1 );
+ produce_sam( alig1->sam, sizeof alig1->sam, gctx, alig1, alig0 ) ;
+ }
+ else
+ {
+ alig0->refbases = read_refbases( alig0->refname, alig0->refpos, alig0->reflen, &alig0->bases_in_ref );
+ alig1->refbases = NULL;
+ alig1->bases_in_ref = 0;
+
+ cigar_t_2_read( alig0->read, sizeof alig0->read, alig0->cigar, alig0->refbases, gctx->insbases );
+ alig1->read[ 0 ] = 0;
+
+ produce_sam( alig0->sam, sizeof alig0->sam, gctx, alig0, alig1 );
+ alig1->sam[ 0 ] = 0;
+ }
+}
+
+rc_t CC KMain( int argc, char *argv [] )
+{
+ rc_t rc = KOutHandlerSet ( write_to_FILE, stdout );
+ if ( rc == 0 )
+ {
+ Args * args;
+
+ int n_options = sizeof Options / sizeof Options[ 0 ];
+ rc = ArgsMakeAndHandle( &args, argc, argv, 1, Options, n_options );
+ if ( rc == 0 )
+ {
+ gen_context gctx;
+
+ read_context( args, &gctx );
+
+ if ( get_bool_option( args, OPTION_SHOW ) )
+ show_details( &gctx );
+ else if ( get_bool_option( args, OPTION_REF ) )
+ KOutMsg ( "%s\n", gctx.alig[ 0 ].refbases );
+ else if ( get_bool_option( args, OPTION_MDTAG ) )
+ show_mdtag( &gctx );
+ else if ( gctx.flags > 0 )
+ explain_flags( gctx.flags );
+ else
+ generate_alignment( &gctx );
+
+ release_alig( &gctx.alig[ 0 ] );
+ if ( gctx.tlen != 0 )
+ release_alig( &gctx.alig[ 1 ] );
+
+ ArgsWhack( args );
+ }
+ }
+ return rc;
+}
+
diff --git a/test/samline/cigar.c b/test/samline/cigar.c
new file mode 100644
index 0000000..2e6c179
--- /dev/null
+++ b/test/samline/cigar.c
@@ -0,0 +1,446 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <klib/printf.h>
+#include <klib/out.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+#include <ctype.h>
+#include <string.h>
+
+#include "cigar.h"
+
+struct cigar_t
+{
+ char * op;
+ int * count;
+ size_t size, length;
+};
+
+
+static void init_cigar_t( struct cigar_t * c, size_t size )
+{
+ if ( c != NULL )
+ {
+ c->size = 0;
+ c->length = 0;
+ c->op = malloc( sizeof( c->op[ 0 ] ) * size );
+ if ( c->op != NULL )
+ {
+ c->count = malloc( sizeof( c->count[ 0 ] ) * size );
+ if ( c->count != NULL )
+ c->size = size;
+ else
+ free( ( void * ) c->op );
+ }
+ }
+}
+
+
+static void resize_cigar_t( struct cigar_t * c, size_t new_size )
+{
+ if ( c != NULL )
+ {
+ if ( c->size == 0 )
+ init_cigar_t( c, new_size );
+ else if ( c->size < new_size )
+ {
+ char * temp_op = c->op;
+ c->op = realloc( c->op, sizeof( c->op[ 0 ] ) * new_size );
+ if ( c->op != NULL )
+ {
+ int * temp_count = c->count;
+ c->count = realloc( c->count, sizeof( c->count[ 0 ] ) * new_size );
+ if ( c->count != NULL )
+ c->size = new_size;
+ else
+ c->count = temp_count;
+ }
+ else
+ c->op = temp_op;
+ }
+ }
+}
+
+static void append_to_cigar_t( struct cigar_t * c, char op, int count )
+{
+ if ( c->length < c->size )
+ {
+ c->op[ c->length ] = op;
+ c->count[ c->length ++ ] = count;
+ }
+}
+
+void parse_cigar_t( struct cigar_t * c, const char * cigar_str )
+{
+ if ( c != NULL && cigar_str != NULL && cigar_str[ 0 ] != 0 )
+ {
+ resize_cigar_t( c, strlen( cigar_str ) );
+ if ( c->size > 0 )
+ {
+ int count = 0;
+ while ( *cigar_str != 0 && c->length < c->size )
+ {
+ if ( isdigit( *cigar_str ) )
+ {
+ count = ( count * 10 ) + ( *cigar_str - '0' );
+ }
+ else
+ {
+ if ( count == 0 ) count = 1;
+ append_to_cigar_t( c, *cigar_str, count );
+ count = 0;
+ }
+ cigar_str++;
+ }
+ }
+ }
+}
+
+
+struct cigar_t * make_cigar_t( const char * cigar_str )
+{
+ struct cigar_t * res = malloc( sizeof * res );
+ if ( res != NULL )
+ {
+ size_t size;
+ if ( cigar_str != NULL && cigar_str[ 0 ] != 0 )
+ size = strlen( cigar_str );
+ else
+ size = 1024;
+ init_cigar_t( res, size );
+ if ( res->size == size )
+ parse_cigar_t( res, cigar_str );
+ }
+ return res;
+}
+
+
+void free_cigar_t( struct cigar_t * c )
+{
+ if ( c != NULL )
+ {
+ if ( c->op != NULL )
+ {
+ free( ( void * ) c->op );
+ c->op = NULL;
+ }
+ if ( c->count != NULL )
+ {
+ free( ( void * ) c->count );
+ c->count = NULL;
+ }
+ free( ( void * ) c );
+ }
+}
+
+
+int cigar_t_reflen( const struct cigar_t * c )
+{
+ int res = 0;
+ if ( c != NULL )
+ {
+ int i;
+ for ( i = 0; i < c->length; ++i )
+ {
+ switch( c->op[ i ] )
+ {
+ case 'A' : res += c->count[ i ]; break;
+ case 'C' : res += c->count[ i ]; break;
+ case 'G' : res += c->count[ i ]; break;
+ case 'T' : res += c->count[ i ]; break;
+
+ case 'D' : res += c->count[ i ]; break;
+ case 'M' : res += c->count[ i ]; break;
+ }
+ }
+ }
+ return res;
+}
+
+
+int cigar_t_readlen( const struct cigar_t * c )
+{
+ int res = 0;
+ if ( c != NULL )
+ {
+ int i;
+ for ( i = 0; i < c->length; ++i )
+ {
+ if ( c->op[ i ] != 'D' )
+ res += c->count[ i ];
+ }
+ }
+ return res;
+}
+
+
+int cigar_t_inslen( const struct cigar_t * c )
+{
+ int res = 0;
+ if ( c != NULL )
+ {
+ int i;
+ for ( i = 0; i < c->length; ++i )
+ {
+ if ( c->op[ i ] == 'I' )
+ res += c->count[ i ];
+ }
+ }
+ return res;
+}
+
+
+size_t cigar_t_string( char * buffer, size_t buf_len, const struct cigar_t * c )
+{
+ size_t res = 0;
+ if ( buffer != NULL && buf_len > 0 && c != NULL && c->length > 0 )
+ {
+ int i;
+ for ( i = 0; i < c->length && res < buf_len; ++i )
+ {
+ size_t num_writ;
+ string_printf( &buffer[ res ], buf_len - res, &num_writ,
+ "%d%c", c->count[ i ], c->op[ i ] );
+ res += num_writ;
+ }
+ if ( res < buf_len )
+ buffer[ res ] = 0;
+ }
+ return res;
+}
+
+
+void debug_cigar_t( const struct cigar_t * c )
+{
+ if ( c != NULL )
+ {
+ int i;
+ for ( i = 0; i < c->length; ++i )
+ KOutMsg( "c[%d]: %d x %c\n", i, c->count[ i ], c->op[ i ] );
+ }
+}
+
+static int can_merge( char op1, char op2 )
+{
+ char mop1 = op1;
+ char mop2 = op2;
+ if ( mop1 == 'A' || mop1 == 'C' || mop1 == 'G' || mop1 == 'T' )
+ mop1 = 'M';
+ if ( mop2 == 'A' || mop2 == 'C' || mop2 == 'G' || mop2 == 'T' )
+ mop2 = 'M';
+ return ( mop1 == mop2 );
+}
+
+
+struct cigar_t * merge_cigar_t( const struct cigar_t * c )
+{
+ struct cigar_t * res = NULL;
+ if ( c != NULL && c -> length > 0 )
+ {
+ res = malloc( sizeof * res );
+ if ( res != NULL )
+ {
+ init_cigar_t( res, c->size );
+ if ( res->size == c->size )
+ {
+ int i, last;
+ append_to_cigar_t( res, c->op[ 0 ], c->count[ 0 ] );
+ for ( i = 1; i < c->length; ++i )
+ {
+ last = res->length - 1;
+ if ( can_merge( c->op[ i ], res->op[ last ] ) )
+ {
+ res->count[ last ] += c->count[ i ];
+ res->op[ last ] = 'M';
+ }
+ else
+ append_to_cigar_t( res, c->op[ i ], c->count[ i ] );
+ }
+ }
+ }
+ }
+ return res;
+}
+
+
+static void append_base( char * buffer, size_t buf_len, size_t * buf_idx, int count, char c )
+{
+ int i;
+ for ( i = 0; i < count && *buf_idx < buf_len; ++i )
+ buffer[ (*buf_idx)++ ] = c;
+}
+
+static void append_bases( char * buffer, size_t buf_len, size_t * buf_idx, int count,
+ const char * src, int src_len, int *src_idx )
+{
+ int i;
+ for ( i = 0; i < count && *buf_idx < buf_len && *src_idx < src_len; ++i )
+ buffer[ (*buf_idx)++ ] = src[ (*src_idx)++ ];
+}
+
+
+size_t cigar_t_2_read( char * buffer, size_t buf_len,
+ const struct cigar_t * c, const char * ref_bases, const char * ins_bases )
+{
+ size_t res = 0;
+ if ( buffer != NULL && buf_len > 0 && c != NULL )
+ {
+ int readlen = cigar_t_readlen( c );
+ if ( readlen > 0 )
+ {
+ int needed_ref_bases = cigar_t_reflen( c );
+ int available_ref_bases = ref_bases != NULL ? strlen( ref_bases ) : 0;
+ if ( available_ref_bases >= needed_ref_bases )
+ {
+ int needed_ins_bases = cigar_t_inslen( c );
+ int available_ins_bases = ins_bases != NULL ? strlen( ins_bases ) : 0;
+ if ( available_ins_bases >= needed_ins_bases )
+ {
+ int ref_idx = 0;
+ int ins_idx = 0;
+ int cigar_idx;
+ for ( cigar_idx = 0; cigar_idx < c->length; ++cigar_idx )
+ {
+ int count = c->count[ cigar_idx ];
+ switch ( c->op[ cigar_idx ] )
+ {
+ case 'A' : append_base( buffer, buf_len, &res, count, 'A' );
+ ref_idx += count;
+ break;
+
+ case 'C' : append_base( buffer, buf_len, &res, count, 'C' );
+ ref_idx += count;
+ break;
+
+ case 'G' : append_base( buffer, buf_len, &res, count, 'G' );
+ ref_idx += count;
+ break;
+
+ case 'T' : append_base( buffer, buf_len, &res, count, 'T' );
+ ref_idx += count;
+ break;
+
+ case 'D' : ref_idx += count;
+ break;
+
+ case 'I' : append_bases( buffer, buf_len, &res, count,
+ ins_bases, available_ins_bases, &ins_idx );
+ break;
+
+ case 'M' : append_bases( buffer, buf_len, &res, count,
+ ref_bases, available_ref_bases, &ref_idx );
+ break;
+ }
+ }
+ if ( res < buf_len )
+ buffer[ res ] = 0;
+ }
+ }
+ }
+ }
+ return res;
+}
+
+
+static void print_matchcount( char * buffer, size_t buf_len, size_t *buf_idx, int *match_count )
+{
+ size_t num_writ;
+ string_printf( &buffer[ *buf_idx ], buf_len - *buf_idx, &num_writ, "%d", *match_count );
+ *match_count = 0;
+ *buf_idx += num_writ;
+}
+
+static void md_delete( char * buffer, size_t buf_len, size_t *buf_idx, int count, int *match_count,
+ const char * reference, int *ref_idx )
+{
+ if ( *match_count > 0 )
+ print_matchcount( buffer, buf_len, buf_idx, match_count );
+
+ if ( *buf_idx + count + 1 < buf_len )
+ {
+ int i;
+ buffer[ (*buf_idx)++ ] = '^';
+ for ( i = 0; i < count; ++i )
+ buffer[ (*buf_idx)++ ] = reference[ (*ref_idx)++ ];
+ }
+}
+
+static void md_match( char * buffer, size_t buf_len, size_t *buf_idx, int count, int *match_count,
+ const char * read, int *read_idx, const char *reference, int *ref_idx )
+{
+ int i;
+ for ( i = 0; i < count; ++i )
+ {
+ if ( read[ (*read_idx)++ ] == reference[ *ref_idx ] )
+ {
+ (*match_count)++;
+ }
+ else
+ {
+ print_matchcount( buffer, buf_len, buf_idx, match_count );
+ if ( *buf_idx < buf_len )
+ buffer[ (*buf_idx)++ ] = reference[ *ref_idx ];
+ }
+ (*ref_idx)++;
+ }
+}
+
+size_t md_tag( char * buffer, size_t buf_len,
+ const struct cigar_t * c, const char * read, const char * reference )
+{
+ size_t res = 0;
+ if ( buffer != NULL && buf_len > 0 && c != NULL )
+ {
+ int read_idx = 0;
+ int ref_idx = 0;
+ int match_count = 0;
+ int cigar_idx;
+ for ( cigar_idx = 0; cigar_idx < c->length; ++cigar_idx )
+ {
+ int count = c->count[ cigar_idx ];
+ switch ( c->op[ cigar_idx ] )
+ {
+ case 'D' : md_delete( buffer, buf_len, &res, count, &match_count,
+ reference, &ref_idx );
+ break;
+
+ case 'I' : read_idx += count; break;
+
+ case 'M' : md_match( buffer, buf_len, &res, count, &match_count,
+ read, &read_idx, reference, &ref_idx );
+ break;
+ }
+ }
+ if ( match_count > 0 )
+ print_matchcount( buffer, buf_len, &res, &match_count );
+
+ if ( res < buf_len )
+ buffer[ res ] = 0;
+ }
+ return res;
+}
diff --git a/tools/cg-dump/line_token_iter.h b/test/samline/cigar.h
similarity index 63%
rename from tools/cg-dump/line_token_iter.h
rename to test/samline/cigar.h
index c5e602d..d0ec04a 100644
--- a/tools/cg-dump/line_token_iter.h
+++ b/test/samline/cigar.h
@@ -24,40 +24,33 @@
*
*/
-#ifndef _h_line_token_iter_
-#define _h_line_token_iter_
+#ifndef _h_cigar_
+#define _h_cigar_
#ifdef __cplusplus
extern "C" {
#endif
-#include <klib/rc.h>
-#include <klib/text.h>
+struct cigar_t;
+struct cigar_t * make_cigar_t( const char * cigar_str );
+void free_cigar_t( struct cigar_t * c );
-typedef struct buf_line_iter
-{
- const char * nxt;
- size_t len;
- uint32_t line_nr;
-} buf_line_iter;
+int cigar_t_reflen( const struct cigar_t * c );
+int cigar_t_readlen( const struct cigar_t * c );
+int cigar_t_inslen( const struct cigar_t * c );
-rc_t buf_line_iter_init( struct buf_line_iter * self, const char * buffer, size_t len );
-rc_t buf_line_iter_get( struct buf_line_iter * self, String * line, bool * valid, uint32_t * line_nr );
+size_t cigar_t_string( char * buffer, size_t buf_len, const struct cigar_t * c );
+struct cigar_t * merge_cigar_t( const struct cigar_t * c );
-typedef struct token_iter
-{
- String line;
- char delim;
- uint32_t token_nr;
- uint32_t idx;
-} token_iter;
-
-rc_t token_iter_init( struct token_iter * self, const String * line, char delim );
-rc_t token_iter_get( struct token_iter * self, String * token, bool * valid, uint32_t * token_nr );
+size_t md_tag( char * buffer, size_t buf_len,
+ const struct cigar_t * c, const char * read, const char * reference );
+void debug_cigar_t( const struct cigar_t * c );
+size_t cigar_t_2_read( char * buffer, size_t buf_len,
+ const struct cigar_t * c, const char * ref_bases, const char * ins_bases );
#ifdef __cplusplus
}
diff --git a/test/samline/example.sh b/test/samline/example.sh
new file mode 100755
index 0000000..9e05454
--- /dev/null
+++ b/test/samline/example.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+execute()
+{
+ echo "------------------------------------------------------"
+ echo $1
+ eval $1
+ echo "."
+}
+
+# call: produce_SAM "$CONFIG" "$SAMFILE"
+produce_SAM()
+{
+ SAMLINE_BINARY="samline"
+ REFNAME="NC_011752.1"
+ REFPOS1=1000
+ REFPOS2=3500
+ CIGAR1="30MAAA20M"
+ CIGAR2="50M2D10M"
+ execute "$SAMLINE_BINARY -r $REFNAME -p $REFPOS1 -p $REFPOS2 -c $CIGAR1 -c $CIGAR2 -n $1 -d > $2"
+}
+
+# call: convert_SAM_to_BAM "$SAMFILE" "$BAMFILE"
+convert_SAM_to_BAM()
+{
+ SAMTOOLS_BINARY="/netopt/ncbi_tools64/samtools/bin/samtools"
+ execute "$SAMTOOLS_BINARY view -bS $1 > $2"
+}
+
+# call: load_BAM_to_CSRA "$CONFIG" "$BAMFILE" "$TEMP_DIR"
+load_BAM_to_CSRA()
+{
+ BAMLOAD_BINARY="bam-load"
+ execute "$BAMLOAD_BINARY -L 3 -o $3 -k $1 -E0 -Q0 $2"
+}
+
+# call: load_SAM_to_CSRA "$CONFIG" "$SAMFILE" "$TEMP_DIR"
+load_SAM_to_CSRA()
+{
+ BAMLOAD_BINARY="bam-load"
+ execute "cat $2 | $BAMLOAD_BINARY -L 3 -o $3 -k $1 -E0 -Q0 /dev/stdin"
+}
+
+# call: kar_CSRA "$FINAL_CSRA" "$TEMP_DIR"
+kar_CSRA()
+{
+ KAR_BINARY="kar"
+ execute "$KAR_BINARY --create $1 -d $2 -f"
+}
+
+SAMFILE="temp.SAM"
+BAMFILE="temp.BAM"
+CONFIG="temp.kfg"
+TEMP_DIR="temp_csra"
+FINAL_CSRA="test.csra"
+
+produce_SAM "$CONFIG" "$SAMFILE"
+#convert_SAM_to_BAM "$SAMFILE" "$BAMFILE"
+#load_BAM_to_CSRA "$CONFIG" "$BAMFILE" "$TEMP_DIR"
+load_SAM_to_CSRA "$CONFIG" "$SAMFILE" "$TEMP_DIR"
+kar_CSRA "$FINAL_CSRA" "$TEMP_DIR"
+
+execute "rm -rf $TEMP_DIR $SAMFILE $BAMFILE $CONFIG"
+execute "vdb-dump $FINAL_CSRA --info"
diff --git a/test/samline/refbases.c b/test/samline/refbases.c
new file mode 100644
index 0000000..76476d8
--- /dev/null
+++ b/test/samline/refbases.c
@@ -0,0 +1,145 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <klib/rc.h>
+
+#include <kfs/directory.h>
+
+#include <vdb/manager.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <string.h>
+
+#include "refbases.h"
+
+static uint32_t read_uint32( const VCursor * cur, uint32_t col_idx )
+{
+ uint32_t elem_bits, boff, row_len;
+ const uint32_t * value;
+ rc_t rc = VCursorCellDataDirect ( cur, 1, col_idx, &elem_bits, (const void**)&value, &boff, &row_len );
+ if ( rc == 0 )
+ return *value;
+ return 0;
+}
+
+static uint32_t read_buffer( const VCursor * cur, char * buffer, int64_t row_id,
+ uint32_t offset, size_t buflen, uint32_t col_idx )
+{
+ uint32_t elem_bits, boff, row_len, res = 0;
+ const char * value;
+ rc_t rc = VCursorCellDataDirect ( cur, row_id, col_idx, &elem_bits, (const void**)&value, &boff, &row_len );
+ if ( rc == 0 && row_len > offset )
+ {
+ res = ( row_len - offset );
+ if ( res > buflen ) res = buflen;
+ memcpy ( buffer, &value[ offset ], res );
+ }
+ return res;
+}
+
+
+static uint32_t read_bases( const VCursor * cur, char * buffer, uint32_t ref_pos_1_based,
+ uint32_t ref_len, uint32_t col_idx, uint32_t max_seq_len )
+{
+ uint32_t res = 0, n_read = 1;
+ uint32_t row_id = ( ( ref_pos_1_based - 1 ) / max_seq_len ) + 1;
+ uint32_t offset = ( ref_pos_1_based - 1 ) - ( ( row_id - 1 ) * max_seq_len );
+ size_t buflen = ref_len;
+ while ( res < ref_len && n_read > 0 )
+ {
+ n_read = read_buffer( cur, &buffer[ res ], row_id++, offset, buflen, col_idx );
+ res += n_read;
+ buflen -= n_read;
+ offset = 0;
+ }
+ return res;
+}
+
+
+char * read_refbases( const char * refname, uint32_t ref_pos_1_based, uint32_t ref_len, uint32_t * bases_in_ref )
+{
+ char * res = NULL;
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+ const VDBManager * mgr;
+ rc = VDBManagerMakeRead ( &mgr, dir );
+ if ( rc == 0 )
+ {
+ const VTable * tab;
+ rc = VDBManagerOpenTableRead( mgr, &tab, NULL, "%s", refname );
+ if ( rc == 0 )
+ {
+ const VCursor * cur;
+ rc = VTableCreateCursorRead( tab, &cur );
+ if ( rc == 0 )
+ {
+ uint32_t base_count_idx, read_idx, max_seq_len_idx;
+ rc = VCursorAddColumn( cur, &base_count_idx, "BASE_COUNT" );
+ if ( rc == 0 )
+ {
+ rc = VCursorAddColumn( cur, &read_idx, "READ" );
+ if ( rc == 0 )
+ {
+ rc = VCursorAddColumn( cur, &max_seq_len_idx, "MAX_SEQ_LEN" );
+ if ( rc == 0 )
+ {
+ rc = VCursorOpen ( cur );
+ if ( rc == 0 )
+ {
+ uint32_t base_count = read_uint32( cur, base_count_idx );
+ if ( bases_in_ref != NULL )
+ *bases_in_ref = base_count;
+ uint32_t max_seq_len = read_uint32( cur, max_seq_len_idx );
+ if ( base_count > ( ref_pos_1_based + ref_len ) && max_seq_len > 0 )
+ {
+ res = malloc( ref_len + 1 );
+ if ( res != NULL )
+ {
+ uint32_t n_read = read_bases( cur, res, ref_pos_1_based,
+ ref_len, read_idx, max_seq_len );
+ res[ n_read ] = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ VCursorRelease( cur );
+ }
+ VTableRelease( tab );
+ }
+ VDBManagerRelease( mgr );
+ }
+ KDirectoryRelease( dir );
+ }
+ return res;
+}
diff --git a/tools/vtblcp/vtblcp-priv.h b/test/samline/refbases.h
similarity index 88%
copy from tools/vtblcp/vtblcp-priv.h
copy to test/samline/refbases.h
index b8e413b..94136fb 100644
--- a/tools/vtblcp/vtblcp-priv.h
+++ b/test/samline/refbases.h
@@ -24,17 +24,18 @@
*
*/
-#ifndef _h_vtblcp_priv_
-#define _h_vtblcp_priv_
+#ifndef _h_refbases_
+#define _h_refbases_
#ifdef __cplusplus
extern "C" {
#endif
-
+char * read_refbases( const char * refname, uint32_t ref_pos_1_based,
+ uint32_t ref_len, uint32_t * bases_in_ref );
#ifdef __cplusplus
}
#endif
-#endif /* _h_vtblcp_priv_ */
+#endif
diff --git a/test/samline/samline.vers b/test/samline/samline.vers
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/test/samline/samline.vers
@@ -0,0 +1 @@
+1.0.0
diff --git a/test/tarballs/build.xml b/test/tarballs/build.xml
new file mode 100644
index 0000000..ffe59c7
--- /dev/null
+++ b/test/tarballs/build.xml
@@ -0,0 +1,27 @@
+<project>
+ <property name="sourcename" value="sratoolkit.current-win64.zip" />
+ <property name="url" value="http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/" />
+ <property name="workdir" value=".\temp" />
+ <property name="targetname" value="current-win64" />
+
+ <target name="tarballs">
+ <delete dir="${workdir}" failonerror="false"/>
+ <mkdir dir="${workdir}"/>
+ <get src="${url}/${sourcename}" dest="${workdir}/${sourcename}" />
+ <unzip src="${workdir}/${sourcename}" dest="${workdir}" />
+ <delete dir="${workdir}/${sourcename}" />
+ <dirset id="tgt" dir="${workdir}" includes="*-win64"/>
+ <property name="target" refid="tgt"/>
+ <move file="${workdir}/${target}" tofile="${workdir}/${targetname}"/>
+
+ <echo message="Target directory is ${workdir}/${targetname}"/>
+
+ <exec executable="cmd" failonerror="true" >
+ <arg value="/c"/>
+ <arg value="test-tarballs.bat"/>
+ <arg value="${workdir}\${targetname}\bin"/>
+ </exec>
+
+ <delete dir="${workdir}" failonerror="false"/>
+ </target>
+</project>
diff --git a/test/tarballs/test-tarballs.bat b/test/tarballs/test-tarballs.bat
new file mode 100644
index 0000000..7cd7051
--- /dev/null
+++ b/test/tarballs/test-tarballs.bat
@@ -0,0 +1,40 @@
+:: ===========================================================================
+::
+:: PUBLIC DOMAIN NOTICE
+:: National Center for Biotechnology Information
+::
+:: This software/database is a "United States Government Work" under the
+:: terms of the United States Copyright Act. It was written as part of
+:: the author's official duties as a United States Government employee and
+:: thus cannot be copyrighted. This software/database is freely available
+:: to the public for use. The National Library of Medicine and the U.S.
+:: Government have not placed any restriction on its use or reproduction.
+::
+:: Although all reasonable efforts have been taken to ensure the accuracy
+:: and reliability of the software and data, the NLM and the U.S.
+:: Government do not and cannot warrant the performance or results that
+:: may be obtained by using this software or data. The NLM and the U.S.
+:: Government disclaim all warranties, express or implied, including
+:: warranties of performance, merchantability or fitness for any particular
+:: purpose.
+::
+:: Please cite the author in any work or product based on this material.
+::
+:: ===========================================================================
+
+echo on
+Setlocal
+Setlocal EnableDelayedExpansion
+
+set TOOLS=abi-dump align-info blastn_vdb cache-mgr fastq-dump illumina-dump kar kdbmeta latf-load prefetch rcexplain sam-dump sff-dump ^
+ sra-kar sra-pileup sra-stat srapath tblastn_vdb test-sra vdb-config vdb-copy vdb-decrypt vdb-dump vdb-encrypt vdb-lock ^
+ vdb-unlock vdb-validate
+
+:: vdb-passwd is obsolete but still in the package
+
+set FAILED=
+
+for %%t in ( %TOOLS% ) do ( echo | set /p=%%t & %1\%%t -h >NUL & ( if errorlevel 1 ( echo | set /p=failed! & set FAILED=!FAILED! %%t ) ) & echo. )
+
+if "%FAILED%" NEQ "" ( echo. & echo FAILED: %FAILED% & exit /b 1 )
+
diff --git a/test/tarballs/test-tarballs.sh b/test/tarballs/test-tarballs.sh
new file mode 100755
index 0000000..5818921
--- /dev/null
+++ b/test/tarballs/test-tarballs.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+#
+# Download and test SRA Toolkit tarballs (see VDB-1345)
+# Errors are reported to the specified email
+#
+# Parameters:
+# $1 - working dir (will contain a copy of the latest md5sum.txt file)
+#
+# return codes:
+# 0 - tests passed
+# 1 - wget failed
+# 2 - gunzip failed
+# 3 - tar failed
+# 4 - one of the tools failed
+
+WORKDIR=$1
+if [ "${WORKDIR}" == "" ]
+then
+ WORKDIR="./temp"
+fi
+
+TOOLS="abi-dump abi-load align-info bam-load blastn_vdb cache-mgr cg-load fastq-dump fastq-load helicos-load illumina-dump \
+illumina-load kar kdbmeta latf-load prefetch rcexplain sam-dump sff-dump sff-load sra-kar sra-pileup \
+sra-sort sra-stat srapath srf-load tblastn_vdb test-sra vdb-config vdb-copy vdb-decrypt vdb-dump vdb-encrypt vdb-lock \
+vdb-unlock vdb-validate"
+
+# vdb-passwd is obsolete but still in the package
+
+case $(uname) in
+Linux)
+ python -mplatform | grep Ubuntu && OS=ubuntu64 || OS=centos_linux64
+ TOOLS="${TOOLS} pacbio-load remote-fuser"
+ ;;
+Darwin)
+ OS=mac64
+ ;;
+esac
+
+TARBALLS_URL=http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/
+TARGET=sratoolkit.current-${OS}
+
+mkdir -p ${WORKDIR}
+cd ${WORKDIR}
+
+wget -nv ${TARBALLS_URL}${TARGET}.tar.gz || exit 1
+gunzip -f ${TARGET}.tar.gz || exit 2
+PACKAGE=$(tar tf ${TARGET}.tar | head -n 1)
+rm -rf ${PACKAGE}
+tar xvf ${TARGET}.tar || exit 3
+
+FAILED=""
+for tool in ${TOOLS}
+do
+ echo $tool
+ ${PACKAGE}/bin/$tool -h
+ if [ "$?" != "0" ]
+ then
+ echo "$(pwd)/${PACKAGE}/bin/$tool failed"
+ FAILED="${FAILED} $tool"
+ fi
+done
+
+if [ "${FAILED}" != "" ]
+then
+ echo "The following tools failed: ${FAILED}"
+ exit 4
+fi
+
+rm -rf ${PACKAGE} ${TARGET}.tar
+cd -
+
+
+
diff --git a/configure b/test/vcf-loader/Makefile
old mode 100755
new mode 100644
similarity index 58%
copy from configure
copy to test/vcf-loader/Makefile
index 3313aac..32aab45
--- a/configure
+++ b/test/vcf-loader/Makefile
@@ -1,5 +1,3 @@
-#!/bin/sh
-
# ===========================================================================
#
# PUBLIC DOMAIN NOTICE
@@ -24,22 +22,52 @@
#
# ===========================================================================
-FILENAME="`basename $0`"
-CURDIR="`dirname $0`"
-
-if [ -z "$CURDIR" ]
- then
- echo configure: error: configure should be run as ./configure
- exit 1
-fi
-
-if [ "$FILENAME" != configure -a ! -s konfigure.perl ]
- then
- echo configure: error: configure should be run as ./configure
-else
- perl -v > /dev/null 2>&1 || { echo "checking for perl... no"; \
- echo >&2 "configure: error: perl not found."; exit 1; }
-
- cd $CURDIR
- perl -w ./konfigure.perl $@
-fi
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/vcf-loader
+
+TEST_TOOLS = \
+ test-vcf-loader
+
+include $(TOP)/build/Makefile.env
+
+
+# make sure runs are not cached in the user repository when running tests
+ifeq ($(wildcard ../../../asm-trace),)
+ ifeq (,$(VDB_CONFIG))
+ VDB_CONFIG = $(shell pwd)/../only-remote-repository.kfg
+ endif
+endif
+
+
+$(TEST_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+.PHONY: $(TEST_TOOLS)
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# white-box test
+#
+INCDIRS += -I$(TOP)/tools/vcf-loader
+
+VCF_TEST_SRC = \
+ test-vcf-loader
+
+VCF_TEST_OBJ = \
+ $(addsuffix .$(OBJX),$(VCF_TEST_SRC))
+
+VCF_TEST_LIB = \
+ -skapp \
+ -sktst \
+ -sncbi-wvdb \
+ -svcfloader
+
+$(TEST_BINDIR)/test-vcf-loader: $(VCF_TEST_OBJ)
+ $(LP) --exe -o $@ $^ $(VCF_TEST_LIB)
+
+valgrind: test-vcf-loader
+ valgrind --ncbi $(TEST_BINDIR)/test-vcf-loader
diff --git a/test/vcf-loader/test-vcf-loader.cpp b/test/vcf-loader/test-vcf-loader.cpp
new file mode 100644
index 0000000..145960e
--- /dev/null
+++ b/test/vcf-loader/test-vcf-loader.cpp
@@ -0,0 +1,864 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* tests for VCF reader
+*/
+
+#include <ktst/unit_test.hpp>
+
+#include <klib/out.h>
+#include <klib/namelist.h>
+
+#include <kfs/directory.h>
+#include <kfs/file.h>
+
+#include <vdb/manager.h>
+#include <vdb/schema.h>
+#include <vdb/database.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+
+#include <align/writer-reference.h>
+
+#include <sysalloc.h>
+#include <stdexcept>
+#include <sstream>
+#include <memory.h>
+
+extern "C" {
+#include "../../tools/vcf-loader/vcf-grammar.h"
+#include "../../tools/vcf-loader/vcf-parse.h"
+#include "../../tools/vcf-loader/vcf-reader.h"
+#include "../../tools/vcf-loader/vcf-database.h"
+}
+
+using namespace std;
+using namespace ncbi::NK;
+
+TEST_SUITE(VcfLoaderTestSuite);
+
+// test fixture for scanner tests
+class VcfScanFixture
+{
+public:
+ VcfScanFixture()
+ : consumed(0)
+ {
+ pb.self = this;
+ pb.input = Input;
+ }
+ ~VcfScanFixture()
+ {
+ VCFScan_yylex_destroy(&pb);
+ }
+ void InitScan(const char* p_input, bool trace=false)
+ {
+ input = p_input;
+ VCFScan_yylex_init(&pb, trace);
+ }
+ int Scan()
+ {
+ int tokenId=VCF_lex(&sym, pb.scanner);
+
+ if (tokenId != 0)
+ tokenText=string(input.c_str() + sym.tokenStart, sym.tokenLength);
+
+ return tokenId;
+ }
+ static size_t CC Input(VCFParseBlock* sb, char* buf, size_t max_size)
+ {
+ VcfScanFixture* self = (VcfScanFixture*)sb->self;
+ if (self->input.size() < self->consumed)
+ return 0;
+
+ size_t to_copy = min(self->input.size() - self->consumed, max_size);
+ if (to_copy == 0)
+ return 0;
+
+ memcpy(buf, self->input.c_str(), to_copy);
+ if (to_copy < max_size && buf[to_copy-1] != '\n')
+ {
+ buf[to_copy] = '\n';
+ ++to_copy;
+ }
+ self->consumed += to_copy;
+ return to_copy;
+ }
+
+ string input;
+ size_t consumed;
+ VCFParseBlock pb;
+ VCFToken sym;
+ string tokenText;
+};
+
+#define REQUIRE_TOKEN(tok) REQUIRE_EQUAL(Scan(), (int)tok);
+#define REQUIRE_TOKEN_TEXT(tok, text) REQUIRE_TOKEN(tok); REQUIRE_EQ(tokenText, string(text));
+
+#define REQUIRE_TOKEN_COORD(tok, text, line, col) \
+ REQUIRE_TOKEN_TEXT(tok, text); \
+ REQUIRE_EQ(pb.lastToken->line_no, (size_t)line); \
+ REQUIRE_EQ(pb.lastToken->column_no, (size_t)col);
+
+FIXTURE_TEST_CASE(EmptyInput, VcfScanFixture)
+{
+ InitScan("");
+ REQUIRE_TOKEN(0);
+}
+FIXTURE_TEST_CASE(MetaLineSimple, VcfScanFixture)
+{
+ InitScan("##fileformat=VCFv4.1\n");
+ REQUIRE_TOKEN_TEXT(vcfMETAKEY_FORMAT, "fileformat");
+ REQUIRE_TOKEN_TEXT('=', "=");
+ REQUIRE_TOKEN_TEXT(vcfMETAVALUE, "VCFv4.1");
+ REQUIRE_TOKEN(vcfENDLINE);
+ REQUIRE_TOKEN(0);
+}
+FIXTURE_TEST_CASE(MetaLineComposite, VcfScanFixture)
+{
+ InitScan("##fileformat=<key1=val1,key2=val2>\n");
+ REQUIRE_TOKEN_TEXT(vcfMETAKEY, "fileformat");
+ REQUIRE_TOKEN_TEXT('=', "=");
+ REQUIRE_TOKEN_TEXT('<', "<");
+ REQUIRE_TOKEN_TEXT(vcfMETAKEY, "key1");
+ REQUIRE_TOKEN_TEXT('=', "=");
+ REQUIRE_TOKEN_TEXT(vcfMETAVALUE, "val1");
+ REQUIRE_TOKEN_TEXT(',', ",");
+ REQUIRE_TOKEN_TEXT(vcfMETAKEY, "key2");
+ REQUIRE_TOKEN_TEXT('=', "=");
+ REQUIRE_TOKEN_TEXT(vcfMETAVALUE, "val2");
+ REQUIRE_TOKEN_TEXT('>', ">");
+ REQUIRE_TOKEN(vcfENDLINE);
+ REQUIRE_TOKEN(0);
+}
+FIXTURE_TEST_CASE(HeaderLine, VcfScanFixture)
+{
+ InitScan("#CHROM\tPOS\n");
+ REQUIRE_TOKEN_TEXT(vcfHEADERITEM, "CHROM");
+ REQUIRE_TOKEN_TEXT(vcfHEADERITEM, "POS");
+ REQUIRE_TOKEN(vcfENDLINE);
+ REQUIRE_TOKEN(0);
+}
+FIXTURE_TEST_CASE(DataItems, VcfScanFixture)
+{
+ InitScan("20\tNS=3;DP=14;AF=0.5;DB;H2\n");
+ REQUIRE_TOKEN_TEXT(vcfDATAITEM, "20");
+ REQUIRE_TOKEN_TEXT(vcfDATAITEM, "NS=3;DP=14;AF=0.5;DB;H2");
+ REQUIRE_TOKEN(vcfENDLINE);
+ REQUIRE_TOKEN(0);
+}
+FIXTURE_TEST_CASE(DataLine_TrailingTab, VcfScanFixture)
+{
+ InitScan("20\tNS=3;DP=14;AF=0.5;DB;H2\t\n");
+ REQUIRE_TOKEN_TEXT(vcfDATAITEM, "20");
+ REQUIRE_TOKEN_TEXT(vcfDATAITEM, "NS=3;DP=14;AF=0.5;DB;H2");
+ REQUIRE_TOKEN(vcfENDLINE);
+ REQUIRE_TOKEN(0);
+}
+FIXTURE_TEST_CASE(DataLine_EOF, VcfScanFixture)
+{
+ InitScan("20\tNS=3;DP=14;AF=0.5;DB;H2");
+ REQUIRE_TOKEN_TEXT(vcfDATAITEM, "20");
+ REQUIRE_TOKEN_TEXT(vcfDATAITEM, "NS=3;DP=14;AF=0.5;DB;H2");
+ REQUIRE_TOKEN(vcfENDLINE);
+ REQUIRE_TOKEN(0);
+}
+
+// parser tests
+
+class VcfParseFixture
+{
+public:
+ VcfParseFixture()
+ : expectError(false)
+ {
+ pb.self = this;
+ pb.input = Input;
+ pb.error = Error;
+ pb.metaLine = AddMetaLine;
+ pb.openMetaLine = OpenMetaLine;
+ pb.closeMetaLine = CloseMetaLine;
+ pb.keyValue = KeyValue;
+ pb.headerItem = HeaderItem;
+ pb.openDataLine = OpenDataLine;
+ pb.dataItem = DataItem;
+ pb.closeDataLine = CloseDataLine;
+ }
+ ~VcfParseFixture()
+ {
+ VCFScan_yylex_destroy(&pb);
+ }
+
+protected:
+ static size_t Input(VCFParseBlock* pb, char* buf, size_t max_size)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ self.input.read(buf, max_size);
+ return self.input.gcount();
+ }
+ static void Error(VCFParseBlock* pb, const char* message)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ if (!self.expectError)
+ throw logic_error(string("VcfParseFixture::Error:") + message);
+ }
+
+ string TokenToString(const VCFToken& t)
+ {
+ return string(input.str().c_str() + t.tokenStart, t.tokenLength);
+ }
+
+ static void AddMetaLine(VCFParseBlock* pb, VCFToken* key, VCFToken* value)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ MetaLine::value_type p;
+ p.first = self.TokenToString(*key);
+ if (value)
+ p.second = self.TokenToString(*value);
+ MetaLine ml;
+ ml.push_back(p);
+ self.meta.push_back(ml);
+ }
+ static void OpenMetaLine(VCFParseBlock* pb, VCFToken* key)
+ {
+ AddMetaLine(pb, key, NULL);
+ }
+ static void CloseMetaLine(VCFParseBlock* pb)
+ {
+ }
+ static void KeyValue(VCFParseBlock* pb, VCFToken* key, VCFToken* value)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ MetaLine::value_type p;
+ p.first = self.TokenToString(*key);
+ p.second = self.TokenToString(*value);
+
+ self.meta.back().push_back(p);
+ }
+ static void HeaderItem(VCFParseBlock* pb, VCFToken* value)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ self.header.push_back(self.TokenToString(*value));
+ }
+ static void OpenDataLine(VCFParseBlock* pb)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ self.data.push_back(DataLine());
+ }
+ static void DataItem(VCFParseBlock* pb, VCFToken* value)
+ {
+ VcfParseFixture& self = *reinterpret_cast<VcfParseFixture*>(pb->self);
+ self.data.back().push_back(self.TokenToString(*value));
+ }
+ static void CloseDataLine(VCFParseBlock* pb)
+ {
+ }
+
+ void InitScan(const char* p_input, bool trace=false)
+ {
+ input.str(p_input);
+ VCFScan_yylex_init(&pb, trace);
+ }
+
+public:
+ VCFParseBlock pb;
+
+ istringstream input;
+
+ typedef vector<pair<string, string> > MetaLine;
+ vector<MetaLine> meta;
+
+ vector<string> header; // all tokens from the header line
+
+ typedef vector<string> DataLine;
+ vector<DataLine> data;
+
+ bool expectError;
+};
+
+// parser tests
+FIXTURE_TEST_CASE(EmptyFile, VcfParseFixture)
+{
+ InitScan("");
+ expectError = true;
+ REQUIRE_EQ( VCF_parse(&pb), 0 ); // EOF
+}
+
+// meta lines
+FIXTURE_TEST_CASE(FormatLineOnly, VcfParseFixture)
+{
+ InitScan("##fileformat=VCFv4.0\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "1\t2827693\t.\tCCGTC\tC\t.\tPASS\tSVTYPE=DEL\n");
+ REQUIRE_EQ( VCF_parse(&pb), 1 );
+ REQUIRE_EQ( meta.size(), (size_t)1 );
+ REQUIRE_EQ( meta[0].size(), (size_t)1 );
+ REQUIRE_EQ( meta[0][0].first, string("fileformat") );
+ REQUIRE_EQ( meta[0][0].second, string("VCFv4.0") );
+}
+
+FIXTURE_TEST_CASE(MetaLines, VcfParseFixture)
+{
+ InitScan("##fileformat=VCFv4.0\n"
+ "##fileDate=20090805\n"
+ "##source=myImputationProgramV3.1\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "1\t2827693\t.\tCCGTC\tC\t.\tPASS\tSVTYPE=DEL\n");
+ REQUIRE_EQ( VCF_parse(&pb), 1 ); // EOF
+ REQUIRE_EQ( meta.size(), (size_t)3 );
+
+ REQUIRE_EQ( meta[0].size(), (size_t)1 );
+ REQUIRE_EQ( meta[0][0].first, string("fileformat") );
+ REQUIRE_EQ( meta[0][0].second, string("VCFv4.0") );
+
+ REQUIRE_EQ( meta[1].size(), (size_t)1 );
+ REQUIRE_EQ( meta[1][0].first, string("fileDate") );
+ REQUIRE_EQ( meta[1][0].second, string("20090805") );
+
+ REQUIRE_EQ( meta[2].size(), (size_t)1 );
+ REQUIRE_EQ( meta[2][0].first, string("source") );
+ REQUIRE_EQ( meta[2][0].second, string("myImputationProgramV3.1") );
+}
+
+FIXTURE_TEST_CASE(Header, VcfParseFixture)
+{
+ InitScan("##fileformat=VCFv4.0\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "1\t2827693\t.\tCCGTC\tC\t.\tPASS\tSVTYPE=DEL\n");
+ REQUIRE_EQ( VCF_parse(&pb), 1 );
+ REQUIRE_EQ( header.size(), (size_t)8 );
+ REQUIRE_EQ( header[0], string("CHROM") );
+ REQUIRE_EQ( header[1], string("POS") );
+ REQUIRE_EQ( header[2], string("ID") );
+ REQUIRE_EQ( header[3], string("REF") );
+ REQUIRE_EQ( header[4], string("ALT") );
+ REQUIRE_EQ( header[5], string("QUAL") );
+ REQUIRE_EQ( header[6], string("FILTER") );
+ REQUIRE_EQ( header[7], string("INFO") );
+}
+
+FIXTURE_TEST_CASE(MetaHeaderData, VcfParseFixture)
+{
+ InitScan(
+ "##fileformat=VCFv4.0\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNA00001\n"
+ "1\t2827693\t.\tCCGTC\tC\t.\tPASS\tSVTYPE=DEL;END=2827680;BKPTID=Pindel_LCS_D1099159;HOMLEN=1;HOMSEQ=C;SVLEN=-66\tGT:GQ\t1/1:13.9\n");
+ REQUIRE_EQ( VCF_parse(&pb), 1 );
+ REQUIRE_EQ( data.size(), (size_t)1 );
+
+ REQUIRE_EQ( data[0].size(), (size_t)10 );
+ REQUIRE_EQ( data[0][0], string("1") );
+ REQUIRE_EQ( data[0][1], string("2827693") );
+ REQUIRE_EQ( data[0][2], string(".") );
+ REQUIRE_EQ( data[0][3], string("CCGTC") );
+}
+
+FIXTURE_TEST_CASE(InfoLine, VcfParseFixture)
+{
+ InitScan("##fileformat=VCFv4.0\n"
+ "##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Total Depth\">\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNA00001\n"
+ "1\t2827693\t.\tCCGTC\tC\t.\tPASS\tSVTYPE=DEL\n");
+
+ REQUIRE_EQ( VCF_parse(&pb), 1 );
+ REQUIRE_EQ( meta.size(), (size_t)2 );
+ REQUIRE_EQ( meta[1].size(), (size_t)5 );
+ MetaLine& ml = meta[1];
+ REQUIRE_EQ( ml[0].first, string("INFO") );
+ REQUIRE_EQ( ml[0].second, string() );
+ REQUIRE_EQ( ml[1].first, string("ID") );
+ REQUIRE_EQ( ml[1].second, string("DP") );
+ REQUIRE_EQ( ml[2].first, string("Number") );
+ REQUIRE_EQ( ml[2].second, string("1") );
+ REQUIRE_EQ( ml[3].first, string("Type") );
+ REQUIRE_EQ( ml[3].second, string("Integer") );
+ REQUIRE_EQ( ml[4].first, string("Description") );
+ REQUIRE_EQ( ml[4].second, string("\"Total Depth\"") );
+}
+
+
+// data lines
+FIXTURE_TEST_CASE(MultiipleDataLines, VcfParseFixture)
+{
+ InitScan(
+ "##fileformat=VCFv4.0\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNA00001\n"
+ "1\t2827693\n"
+ "2\t3\n");
+ REQUIRE_EQ( VCF_parse(&pb), 1 );
+ REQUIRE_EQ( data.size(), (size_t)2 );
+
+ REQUIRE_EQ( data[0].size(), (size_t)2 );
+ REQUIRE_EQ( data[0][0], string("1") );
+ REQUIRE_EQ( data[0][1], string("2827693") );
+
+ REQUIRE_EQ( data[1].size(), (size_t)2 );
+ REQUIRE_EQ( data[1][0], string("2") );
+ REQUIRE_EQ( data[1][1], string("3") );
+}
+
+// VcfReader
+
+class VcfReaderFixture
+{
+public:
+ VcfReaderFixture()
+ : reader(0), messages(0), messageCount(0)
+ {
+ if (KDirectoryNativeDir(&wd) != 0)
+ throw logic_error(string("VcfReaderFixture: KDirectoryNativeDir failed"));
+
+ if (VcfReaderMake((const VcfReader**)&reader) != 0)
+ throw logic_error(string("VcfReaderFixture: VcfReaderMake failed"));
+ }
+ ~VcfReaderFixture()
+ {
+ KDirectoryRemove(wd, true, filename.c_str());
+
+ if (reader != 0 && VcfReaderWhack(reader) != 0)
+ throw logic_error(string("~VcfReaderFixture: VcfReaderWhack failed"));
+
+ if (KDirectoryRelease(wd) != 0)
+ throw logic_error(string("~VcfReaderFixture: KDirectoryRelease failed"));
+ }
+
+ rc_t CreateFile(const char* p_filename, const char* contents)
+ {
+ KDirectoryRemove(wd, true, p_filename);
+
+ filename = p_filename;
+
+ KFile* file;
+ rc_t rc = KDirectoryCreateFile(wd, &file, true, 0664, kcmInit, p_filename);
+ if (rc == 0)
+ {
+ size_t num_writ=0;
+ rc = KFileWrite(file, 0, contents, strlen(contents), &num_writ);
+ if (rc == 0)
+ rc = KFileRelease(file);
+ else
+ KFileRelease(file);
+ }
+ return rc;
+ }
+
+ rc_t ParseFile(const char* p_filename)
+ {
+ messages = 0;
+ messageCount = 0;
+
+ KFile* file;
+ rc_t rc = KDirectoryOpenFileRead(wd, (const KFile**)&file, p_filename);
+ if (rc == 0)
+ {
+ rc = VcfReaderParse(reader, file, &messages);
+ if (messages != NULL)
+ {
+ rc_t rc2 = VNameListCount(messages, &messageCount);
+ if (rc == 0)
+ rc = rc2;
+ }
+
+ if (rc == 0)
+ rc = KFileRelease(file);
+ else
+ KFileRelease(file);
+ }
+ return rc;
+ }
+
+ KDirectory* wd;
+ VcfReader *reader;
+ string filename;
+ const struct VNamelist* messages;
+ uint32_t messageCount;
+};
+
+FIXTURE_TEST_CASE(VcfReader_EmptyFile, VcfReaderFixture)
+{
+ REQUIRE_RC(CreateFile(GetName(), ""));
+ REQUIRE_RC_FAIL(ParseFile(GetName()));
+ REQUIRE_NOT_NULL(messages);
+ REQUIRE_EQ(1u, messageCount);
+ const char* msg;
+ REQUIRE_RC(VNameListGet ( messages, 0, &msg ));
+ REQUIRE_EQ(string("Empty file"), string(msg));
+}
+
+static string StringToSTL(const String& str)
+{
+ return string(str.addr, str.len);
+}
+
+FIXTURE_TEST_CASE(VcfReader_Parse, VcfReaderFixture)
+{
+ REQUIRE_RC(CreateFile(GetName(),
+ // this is taken from the spec:
+ // http://www.1000genomes.org/wiki/analysis/variant-call-format/vcf-variant-call-format-version-42
+ "##fileformat=VCFv4.2\n"
+ "##fileDate=20090805\n"
+ "##source=myImputationProgramV3.1\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "20\t14370\trs6054257\tG\tA\t29\tPASS\tNS=3;DP=14;AF=0.5;DB;H2\n"
+ "20\t17330\t.\tT\tA\t3\tq10\tNS=3;DP=11;AF=0.017\n"
+ ));
+ REQUIRE_RC(ParseFile(GetName()));
+
+ uint32_t count;
+ REQUIRE_RC(VcfReaderGetDataLineCount(reader, &count));
+ REQUIRE_EQ(2u, count);
+
+ const VcfDataLine* line;
+ REQUIRE_RC(VcfReaderGetDataLine(reader, 0, &line));
+ REQUIRE_NOT_NULL(line);
+ REQUIRE_EQ(string("20"), StringToSTL(line->chromosome));
+ REQUIRE_EQ(14370u, line->position);
+ REQUIRE_EQ(string("rs6054257"), StringToSTL(line->id));
+ REQUIRE_EQ(string("G"), StringToSTL(line->refBases));
+ REQUIRE_EQ(string("A"), StringToSTL(line->altBases));
+ REQUIRE_EQ(29u, (unsigned int)line->quality);
+ REQUIRE_EQ(string("PASS"), StringToSTL(line->filter));
+ REQUIRE_EQ(string("NS=3;DP=14;AF=0.5;DB;H2"), StringToSTL(line->info));
+
+ REQUIRE_RC(VcfReaderGetDataLine(reader, 1, &line));
+ REQUIRE_NOT_NULL(line);
+ REQUIRE_EQ(string("20"), StringToSTL(line->chromosome));
+ REQUIRE_EQ(17330u, line->position);
+ REQUIRE_EQ(string("."), StringToSTL(line->id));
+ REQUIRE_EQ(string("T"), StringToSTL(line->refBases));
+ REQUIRE_EQ(string("A"), StringToSTL(line->altBases));
+ REQUIRE_EQ(3u, (unsigned int)line->quality);
+ REQUIRE_EQ(string("q10"), StringToSTL(line->filter));
+ REQUIRE_EQ(string("NS=3;DP=11;AF=0.017"), StringToSTL(line->info));
+
+ REQUIRE_RC(VcfReaderGetDataLine(reader, 2, &line));
+ REQUIRE_NULL(line);
+}
+
+FIXTURE_TEST_CASE(VcfReader_Parse_GenotypeFields, VcfReaderFixture)
+{
+ REQUIRE_RC(CreateFile(GetName(),
+ "##fileformat=VCFv4.2\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "20\t14370\trs6054257\tG\tA\t29\tPASS\tNS=3;DP=14;AF=0.5;DB;H2\tblah1\t2blah\tetc...\n"
+ ));
+ REQUIRE_RC(ParseFile(GetName()));
+
+ const VcfDataLine* line;
+ REQUIRE_RC(VcfReaderGetDataLine(reader, 0, &line));
+ REQUIRE_NOT_NULL(line);
+
+ uint32_t fieldCount;
+ REQUIRE_RC( VNameListCount(line->genotypeFields, &fieldCount) );
+ REQUIRE_EQ(3u, fieldCount);
+
+ const char* name;
+
+ REQUIRE_RC(VNameListGet ( line->genotypeFields, 0, &name ));
+ REQUIRE_NOT_NULL(name);
+ REQUIRE_EQ(string("blah1"), string(name));
+
+ REQUIRE_RC(VNameListGet ( line->genotypeFields, 1, &name ));
+ REQUIRE_NOT_NULL(name);
+ REQUIRE_EQ(string("2blah"), string(name));
+
+ REQUIRE_RC(VNameListGet ( line->genotypeFields, 2, &name ));
+ REQUIRE_NOT_NULL(name);
+ REQUIRE_EQ(string("etc..."), string(name));
+}
+
+FIXTURE_TEST_CASE(VcfReader_Parse_Errors, VcfReaderFixture)
+{
+ REQUIRE_RC(CreateFile(GetName(),
+ "##fileformat=VCFv4.2\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "20\t14370\trs6054257\tG\tA\t1000\tPASS\tNS=3;DP=14;AF=0.5;DB;H2\tblah1\t2blah\tetc...\n"
+ "20\t14370\trs6054257\tG\tA\t10\tPASS\n"
+ )); // 1. 1000 is too much for a quality
+ // 2. not all mandatory fields present
+ REQUIRE_RC_FAIL(ParseFile(GetName()));
+ REQUIRE_EQ(2u, messageCount);
+ const char* msg;
+ REQUIRE_RC(VNameListGet ( messages, 0, &msg ));
+ REQUIRE_EQ(string("line 3 column 24: invalid numeric value for 'quality'"), string(msg));
+ REQUIRE_RC(VNameListGet ( messages, 1, &msg ));
+ REQUIRE_EQ(string("line 4 column 31: one or more of the 8 mandatory columns are missing"), string(msg));
+}
+
+// VcfDatabase
+class VcfDatabaseFixture : public VcfReaderFixture
+{
+public:
+ VcfDatabaseFixture()
+ {
+ if (VDBManagerMakeUpdate(&m_vdbMgr, wd) != 0)
+ throw logic_error(string("VcfDatabaseFixture(): VDBManagerMakeUpdate failed"));
+ }
+ ~VcfDatabaseFixture()
+ {
+ if (VDBManagerRelease(m_vdbMgr) != 0)
+ throw logic_error(string("~VcfDatabaseFixture: VDBManagerRelease failed"));
+ }
+
+ void Setup(const string& p_caseName)
+ {
+ m_dbName = p_caseName+".db";
+ m_cfgName = p_caseName+".cfg";
+
+ KDirectoryRemove(wd, true, m_dbName.c_str());
+
+ if (VDBManagerMakeSchema(m_vdbMgr, &m_schema) != 0)
+ throw logic_error(string("VcfDatabaseFixture::Setup: VDBManagerMakeSchema failed"));
+
+ if (VSchemaParseText ( m_schema, "vcf_schema", schema_text, string_measure(schema_text, NULL) ) != 0)
+ throw logic_error(string("VcfDatabaseFixture::Setup: VSchemaParseText failed"));
+
+ if (VDBManagerCreateDB(m_vdbMgr, &m_db, m_schema, "vcf", kcmCreate | kcmMD5, m_dbName.c_str()) != 0)
+ throw logic_error(string("VcfDatabaseFixture::Setup: VDBManagerCreateDB failed"));
+
+ if (CreateFile(m_cfgName.c_str(), "20 CM000682.1"))
+ throw logic_error(string("VcfDatabaseFixture::Setup: CreateFile failed"));
+ }
+ void Teardown()
+ {
+ if (VDatabaseRelease(m_db) != 0)
+ throw logic_error(string("VcfDatabaseFixture::Teardown: VDatabaseRelease failed"));
+ if (VSchemaRelease(m_schema) != 0)
+ throw logic_error(string("VcfDatabaseFixture::Teardown: VSchemaRelease failed"));
+ // this remove fails on Windows for unexplained reasons ("access denied" to <dbname>/tbl/REFERENCE/col), so ignore the result:
+ KDirectoryRemove(wd, true, m_dbName.c_str());
+ if (KDirectoryRemove(wd, true, m_cfgName.c_str()) != 0)
+ throw logic_error(string("VcfDatabaseFixture::Teardown: KDirectoryRemove(m_cfgName) failed"));
+ }
+
+protected:
+ static const char schema_text[];
+ static const uint32_t basesPerRow = 5000;
+
+ VDBManager* m_vdbMgr;
+ VSchema* m_schema;
+ VDatabase* m_db;
+ string m_cfgName;
+ string m_dbName;
+};
+
+const char VcfDatabaseFixture::schema_text[] =
+"version 1; "
+
+"include 'align/align.vschema';"
+
+" table variant #1 { "
+" extern column U32 ref_id = .ref_id;"
+" physical column U32 .ref_id = ref_id;"
+
+" extern column U32 position = .position;"
+" physical column U32 .position = position;"
+
+" extern column U32 length = .length;"
+" physical column U32 .length = length;"
+
+" extern column ascii sequence = .sequence;"
+" physical column ascii .sequence = sequence;"
+
+" };"
+
+" table variant_phase #1 { "
+" extern column U32 variant_id = .variant_id;"
+" physical column U32 .variant_id = variant_id;"
+
+" extern column U32 phase = .phase;"
+" physical column U32 .phase = phase;"
+" };"
+
+" table alignment #1 { "
+" extern column U32 align_id = .align_id;"
+" physical column U32 .align_id = align_id;"
+
+" extern column U32 ref_id = .ref_id;"
+" physical column U32 .ref_id = ref_id;"
+
+" extern column U32 position = .position;"
+" physical column U32 .position = position;"
+
+" extern column U32 phase = .phase;"
+" physical column U32 .phase = phase;"
+
+" extern column ascii cigar = .cigar;"
+" physical column ascii .cigar = cigar;"
+
+" extern column ascii mismatch = .mismatch;"
+" physical column ascii .mismatch = mismatch;"
+
+" };"
+
+"database vcf #1 { "
+" table NCBI:align:tbl:reference #2 REFERENCE;"
+" table variant #1 VARIANT;"
+" table variant_phase #1 VARIANT_PHASE;"
+" table alignment #1 ALIGNMENT;"
+"};";
+
+FIXTURE_TEST_CASE(VcfDatabase_GetReference, VcfDatabaseFixture)
+{
+ // this would be better done as a unit test on the ReferenceSeq interface
+ // (there is no appropriate suite for that at this point)
+ Setup(GetName());
+ const ReferenceMgr* refMgr;
+ REQUIRE_RC(ReferenceMgr_Make(&refMgr, m_db, m_vdbMgr, 0, m_cfgName.c_str(), NULL, 0, 0, 0));
+ const ReferenceSeq* seq;
+ bool dummy1 = false;
+ bool dummy2 = false;
+ REQUIRE_RC(ReferenceMgr_GetSeq(refMgr, &seq, "20", &dummy1, false, &dummy2));
+
+ INSDC_coord_zero coord = 14370;
+ int64_t ref_id;
+ INSDC_coord_zero ref_start;
+ uint64_t global_ref_start;
+ REQUIRE_RC(ReferenceSeq_TranslateOffset_int(seq, coord, &ref_id, &ref_start, &global_ref_start));
+ REQUIRE_EQ(ref_id, (int64_t)(1 + coord / basesPerRow));
+ REQUIRE_EQ(ref_start, (INSDC_coord_zero)(coord % basesPerRow));
+ REQUIRE_EQ(global_ref_start, (uint64_t)coord);
+
+ REQUIRE_RC(ReferenceSeq_Release(seq));
+ REQUIRE_RC(ReferenceMgr_Release(refMgr, true, NULL, false, NULL));
+ Teardown();
+}
+
+FIXTURE_TEST_CASE(VcfDatabaseBasic, VcfDatabaseFixture)
+{
+ Setup(GetName());
+
+ REQUIRE_RC(CreateFile(GetName(),
+ // this is taken from the spec:
+ // http://www.1000genomes.org/wiki/analysis/variant-call-format/vcf-variant-call-format-version-42
+ "##fileformat=VCFv4.2\n"
+ "##fileDate=20090805\n"
+ "##source=myImputationProgramV3.1\n"
+ "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"
+ "20\t14370\trs6054257\tG\tCCCC\t29\tPASS\tNS=3;DP=14;AF=0.5;DB;H2\n"
+ "20\t17330\t.\tT\tA\t3\tq10\tNS=3;DP=11;AF=0.017\n"
+ ));
+ REQUIRE_RC(ParseFile(GetName()));
+
+ REQUIRE_RC(VcfDatabaseSave(reader, m_cfgName.c_str(), m_db));
+
+ // verify
+ const VTable *tbl;
+ REQUIRE_RC(VDBManagerOpenTableRead(m_vdbMgr, &tbl, m_schema, (m_dbName+"/tbl/VARIANT").c_str()));
+ VCursor *cur;
+ REQUIRE_RC(VTableCreateCursorRead( tbl, (const VCursor**)&cur ));
+
+ uint32_t ref_id_idx, position_idx, length_idx, sequence_idx;
+ REQUIRE_RC(VCursorAddColumn( cur, &ref_id_idx, "ref_id" ));
+ REQUIRE_RC(VCursorAddColumn( cur, &position_idx, "position" ));
+ REQUIRE_RC(VCursorAddColumn( cur, &length_idx, "length" ));
+ REQUIRE_RC(VCursorAddColumn( cur, &sequence_idx, "sequence" ));
+
+ REQUIRE_RC(VCursorOpen( cur ));
+
+ char buf[256];
+ uint32_t row_len;
+ const uint32_t elemBits = 8;
+
+ int64_t rowId = 1;
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 1, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(8u, row_len);
+ REQUIRE_EQ(3u, *(uint32_t*)buf);
+
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 2, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(sizeof(INSDC_coord_zero), (size_t)row_len);
+ REQUIRE_EQ((INSDC_coord_zero)(14370u % basesPerRow), *(INSDC_coord_zero*)buf);
+
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 3, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(sizeof(uint32_t), (size_t)row_len);
+ REQUIRE_EQ(4u, *(uint32_t*)buf);
+
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 4, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(4u, row_len);
+ REQUIRE_EQ(string(buf, 4), string("CCCC"));
+
+ rowId = 2;
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 1, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(8u, row_len);
+ REQUIRE_EQ(4u, *(uint32_t*)buf);
+
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 2, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(sizeof(INSDC_coord_zero), (size_t)row_len);
+ REQUIRE_EQ((INSDC_coord_zero)(17330u % basesPerRow), *(INSDC_coord_zero*)buf);
+
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 3, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(sizeof(uint32_t), (size_t)row_len);
+ REQUIRE_EQ(1u, *(uint32_t*)buf);
+
+ REQUIRE_RC(VCursorReadDirect(cur, rowId, 4, elemBits, buf, sizeof(buf), &row_len ));
+ REQUIRE_EQ(1u, row_len);
+ REQUIRE_EQ(string(buf, 1), string("A"));
+
+ rowId = 3;
+ REQUIRE_RC_FAIL(VCursorReadDirect(cur, rowId, 1, elemBits, buf, sizeof(buf), &row_len ));
+
+ REQUIRE_RC(VCursorRelease(cur));
+ REQUIRE_RC(VTableRelease(tbl));
+ Teardown();
+}
+
+//////////////////////////////////////////// Main
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+extern "C"
+{
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+
+const char UsageDefaultName[] = "vcf-loader-test";
+
+rc_t CC UsageSummary (const char * progname)
+{
+ return KOutMsg ( "Usage:\n" "\t%s [options] -o path\n\n", progname );
+}
+
+rc_t CC Usage( const Args* args )
+{
+ return 0;
+}
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ // need user settings to pick up the correct schema include directory,
+ // so do not disable them
+ //KConfigDisableUserSettings();
+
+ rc_t rc = VcfLoaderTestSuite(argc, argv);
+ return rc;
+}
+
+}
+
diff --git a/test/Makefile b/test/vdb-dump/Makefile
similarity index 72%
copy from test/Makefile
copy to test/vdb-dump/Makefile
index 08bddf1..f96eb41 100644
--- a/test/Makefile
+++ b/test/vdb-dump/Makefile
@@ -22,9 +22,26 @@
#
# ===========================================================================
-#TBD
-default all std clean runtests slowtests:
- @true
-
-.PHONY: default all std clean runtests slowtests
-
\ No newline at end of file
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/vdb-dump
+
+TEST_TOOLS =
+
+include $(TOP)/build/Makefile.env
+
+.PHONY: $(TEST_TOOLS)
+
+runtests: vdb-dump
+
+vdb-dump:
+ rm -rf actual
+ mkdir -p actual
+ $(BINDIR)/vdb-dump SRR056386 -R 1 -C READ -f tab >actual/1.0.stdout; diff expected/1.0.stdout actual/1.0.stdout
+ $(BINDIR)/vdb-dump SRR056386 -R 1 -C READ -f tab -I >actual/1.1.stdout; diff expected/1.1.stdout actual/1.1.stdout
+ rm -rf actual
+
+.PHONY: kar
+
diff --git a/test/vdb-dump/SRR056386 b/test/vdb-dump/SRR056386
new file mode 100644
index 0000000..bf8b088
Binary files /dev/null and b/test/vdb-dump/SRR056386 differ
diff --git a/test/vdb-dump/expected/1.0.stdout b/test/vdb-dump/expected/1.0.stdout
new file mode 100644
index 0000000..8645a27
--- /dev/null
+++ b/test/vdb-dump/expected/1.0.stdout
@@ -0,0 +1 @@
+TCAGAATTACTTGAGGCTGCACACCATTCAAGTTGTCATGAGACACGTCATCAAGAGGCACTTCCACGTCGACATCTGCGGCAGTCTAACCGTCTGTATTAATGTTGTAATTACTTTCCATTTACTGATTTGATGTTAAAACAAATTTAATTAATTTATAACAATTTAACTTAAATCCTTCTTAATCTCTTTAAATGAACTGAGGGGGTTTTGGGGGTTTTTGGGGTTTTTGGGGGTTTTTGGGGGTCGTGCCACGCTTACCGTACATCCGCCTTGGCCTGAGACTGCCAAAGGGCACCACAGGGGGGGGATTAAGGGGN
diff --git a/test/vdb-dump/expected/1.1.stdout b/test/vdb-dump/expected/1.1.stdout
new file mode 100644
index 0000000..77cc039
--- /dev/null
+++ b/test/vdb-dump/expected/1.1.stdout
@@ -0,0 +1 @@
+1 TCAGAATTACTTGAGGCTGCACACCATTCAAGTTGTCATGAGACACGTCATCAAGAGGCACTTCCACGTCGACATCTGCGGCAGTCTAACCGTCTGTATTAATGTTGTAATTACTTTCCATTTACTGATTTGATGTTAAAACAAATTTAATTAATTTATAACAATTTAACTTAAATCCTTCTTAATCTCTTTAAATGAACTGAGGGGGTTTTGGGGGTTTTTGGGGTTTTTGGGGGTTTTTGGGGGTCGTGCCACGCTTACCGTACATCCGCCTTGGCCTGAGACTGCCAAAGGGCACCACAGGGGGGGGATTAAGGGGN
diff --git a/test/vschema/Makefile b/test/vschema/Makefile
new file mode 100644
index 0000000..97fb28b
--- /dev/null
+++ b/test/vschema/Makefile
@@ -0,0 +1,115 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/vschema
+
+include $(TOP)/build/Makefile.env
+
+clean: stdclean
+
+#-------------------------------------------------------------------------------
+# bio-end schema tests
+#
+runtests: \
+ bio-end-tests \
+ idx-text-tests
+
+#-------------------------------------------------------------------------------
+# bio-end-tests
+#
+TEST_BIO_END_MAKEINPUTS_SRC = \
+ bio-end/makeinputs
+
+TEST_BIO_END_MAKEINPUTS_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_BIO_END_MAKEINPUTS_SRC))
+
+TEST_BIO_END_MAKEINPUTS_LIB = \
+ -L$(LIBDIR) \
+ -sgeneral-writer
+
+
+bio-end-makeinputs: $(TEST_BIO_END_MAKEINPUTS_OBJ)
+ $(LP) --exe -o ./bio-end-makeinputs $^ $(TEST_BIO_END_MAKEINPUTS_LIB)
+ @rm bio-end-makeinputs.exe.d $(TEST_BIO_END_MAKEINPUTS_OBJ)
+
+bio-end-tests: bio-end-makeinputs
+ @mkdir -p $(SRCDIR)/bio-end/input && ./bio-end-makeinputs && rm ./bio-end-makeinputs
+ @# test cases
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/bio-end bio-end excl-1 0 n
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/bio-end bio-end excl-2 0 n
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/bio-end bio-end excl-3 0 n
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/bio-end bio-end excl-4 0 n
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/bio-end bio-end incl-1 0 n
+ @rm -f $(SRCDIR)/bio-end/input/*.gl
+ @echo "bio-end tests executed successfuly"
+
+#-------------------------------------------------------------------------------
+# idx-text-tests
+#
+TEST_IDX_TEXT_MAKEINPUTS_SRC = \
+ idx-text/makeinputs
+
+TEST_IDX_TEXT_MAKEINPUTS_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_IDX_TEXT_MAKEINPUTS_SRC))
+
+TEST_IDX_TEXT_MAKEINPUTS_LIB = \
+ -L$(LIBDIR) \
+ -sgeneral-writer
+
+idx-text-makeinputs: $(TEST_IDX_TEXT_MAKEINPUTS_OBJ)
+ $(LP) --exe -o ./idx-text-makeinputs $^ $(TEST_IDX_TEXT_MAKEINPUTS_LIB)
+ @rm idx-text-makeinputs.exe.d $(TEST_IDX_TEXT_MAKEINPUTS_OBJ)
+
+
+TEST_IDX_TEXT_CHECKLOOKUP_SRC = \
+ idx-text/checklookup
+
+TEST_IDX_TEXT_CHECKLOOKUP_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_IDX_TEXT_CHECKLOOKUP_SRC))
+
+TEST_IDX_TEXT_CHECKLOOKUP_LIB = \
+ -L$(LIBDIR) \
+ -sncbi-vdb
+
+idx-text-checklookup: $(TEST_IDX_TEXT_CHECKLOOKUP_OBJ)
+ $(LP) --exe -o ./idx-text-checklookup $^ $(TEST_IDX_TEXT_CHECKLOOKUP_LIB)
+ @rm idx-text-checklookup.exe.d $(TEST_IDX_TEXT_CHECKLOOKUP_OBJ)
+
+
+idx-text-tests: idx-text-makeinputs idx-text-checklookup
+ @mkdir -p $(SRCDIR)/idx-text/input && ./idx-text-makeinputs && rm ./idx-text-makeinputs
+ @# test cases
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/idx-text idx-text ci-1 0 y ./idx-text-checklookup
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/idx-text idx-text ci-2 0 y ./idx-text-checklookup
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/idx-text idx-text ci-3 0 y ./idx-text-checklookup
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/idx-text idx-text ci-4 0 y ./idx-text-checklookup
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/idx-text idx-text cs-1 0 y ./idx-text-checklookup
+ @$(SRCDIR)/runtestcase.sh $(BINDIR) $(SRCDIR)/idx-text idx-text cs-2 0 y ./idx-text-checklookup
+ @rm -f ./idx-text-checklookup
+ @rm -f $(SRCDIR)/idx-text/input/*.gl
+ @echo "idx-text tests executed successfuly"
diff --git a/tools/vdb-dump/vdb_info.h b/test/vschema/bio-end/bio-end-excl.vschema
similarity index 71%
copy from tools/vdb-dump/vdb_info.h
copy to test/vschema/bio-end/bio-end-excl.vschema
index f0bde80..3e91875 100644
--- a/tools/vdb-dump/vdb_info.h
+++ b/test/vschema/bio-end/bio-end-excl.vschema
@@ -24,24 +24,21 @@
*
*/
-#ifndef _h_vdb_info_
-#define _h_vdb_info_
+include 'insdc/insdc.vschema';
+include 'insdc/sra.vschema';
+include 'ncbi/sra.vschema';
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include <klib/rc.h>
-#include <vdb/manager.h>
+table bio_end:test:table1 #1
+{
+ column INSDC:coord:zero out_read_start;
+ column INSDC:coord:len out_read_len;
+ column INSDC:SRA:xread_type out_read_type;
-rc_t vdb_info( Vector * schema_list, dump_format_t format, const VDBManager *mgr,
- const char * acc_or_path, num_gen * row_generator );
-
-#ifdef __cplusplus
+ readonly column INSDC:coord:zero bio_end
+ = NCBI:SRA:bio_end < false > ( out_read_start, out_read_type, out_read_len );
}
-#endif
-#endif
+database bio_end:test:database1 #1
+{
+ table bio_end:test:table1 #1 table1;
+}
diff --git a/tools/vdb-dump/vdb_info.h b/test/vschema/bio-end/bio-end-incl.vschema
similarity index 71%
copy from tools/vdb-dump/vdb_info.h
copy to test/vschema/bio-end/bio-end-incl.vschema
index f0bde80..c071196 100644
--- a/tools/vdb-dump/vdb_info.h
+++ b/test/vschema/bio-end/bio-end-incl.vschema
@@ -24,24 +24,21 @@
*
*/
-#ifndef _h_vdb_info_
-#define _h_vdb_info_
+include 'insdc/insdc.vschema';
+include 'insdc/sra.vschema';
+include 'ncbi/sra.vschema';
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include <klib/rc.h>
-#include <vdb/manager.h>
+table bio_end:test:table1 #1
+{
+ column INSDC:coord:zero out_read_start;
+ column INSDC:coord:len out_read_len;
+ column INSDC:SRA:xread_type out_read_type;
-rc_t vdb_info( Vector * schema_list, dump_format_t format, const VDBManager *mgr,
- const char * acc_or_path, num_gen * row_generator );
-
-#ifdef __cplusplus
+ readonly column INSDC:coord:zero bio_end
+ = NCBI:SRA:bio_end < true > ( out_read_start, out_read_type, out_read_len );
}
-#endif
-#endif
+database bio_end:test:database1 #1
+{
+ table bio_end:test:table1 #1 table1;
+}
diff --git a/test/vschema/bio-end/expected/excl-1.stdout b/test/vschema/bio-end/expected/excl-1.stdout
new file mode 100644
index 0000000..702409a
--- /dev/null
+++ b/test/vschema/bio-end/expected/excl-1.stdout
@@ -0,0 +1,5 @@
+ bio_end: 12
+ out_read_len: 4, 4, 4
+out_read_start: 0, 4, 8
+ out_read_type: SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL
+
diff --git a/test/vschema/bio-end/expected/excl-2.stdout b/test/vschema/bio-end/expected/excl-2.stdout
new file mode 100644
index 0000000..2667e5c
--- /dev/null
+++ b/test/vschema/bio-end/expected/excl-2.stdout
@@ -0,0 +1,5 @@
+ bio_end: 8
+ out_read_len: 4, 4, 4
+out_read_start: 0, 4, 8
+ out_read_type: SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+
diff --git a/test/vschema/bio-end/expected/excl-3.stdout b/test/vschema/bio-end/expected/excl-3.stdout
new file mode 100644
index 0000000..b428943
--- /dev/null
+++ b/test/vschema/bio-end/expected/excl-3.stdout
@@ -0,0 +1,5 @@
+ bio_end: 8
+ out_read_len: 4, 4, 4, 0, 0
+out_read_start: 0, 4, 8, 12, 12
+ out_read_type: SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+
diff --git a/test/vschema/bio-end/expected/excl-4.stdout b/test/vschema/bio-end/expected/excl-4.stdout
new file mode 100644
index 0000000..43e18a6
--- /dev/null
+++ b/test/vschema/bio-end/expected/excl-4.stdout
@@ -0,0 +1,5 @@
+ bio_end: 13
+ out_read_len: 4, 4, 4, 1, 0
+out_read_start: 0, 4, 8, 12, 13
+ out_read_type: SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+
diff --git a/test/vschema/bio-end/expected/incl-1.stdout b/test/vschema/bio-end/expected/incl-1.stdout
new file mode 100644
index 0000000..6ec1b01
--- /dev/null
+++ b/test/vschema/bio-end/expected/incl-1.stdout
@@ -0,0 +1,5 @@
+ bio_end: 7
+ out_read_len: 4, 4, 4
+out_read_start: 0, 4, 8
+ out_read_type: SRA_READ_TYPE_TECHNICAL, SRA_READ_TYPE_BIOLOGICAL, SRA_READ_TYPE_TECHNICAL
+
diff --git a/test/vschema/bio-end/makeinputs.cpp b/test/vschema/bio-end/makeinputs.cpp
new file mode 100644
index 0000000..bb34391
--- /dev/null
+++ b/test/vschema/bio-end/makeinputs.cpp
@@ -0,0 +1,129 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+/**
+ * Unit tests for Bio-end function
+ */
+#include "../../../tools/general-loader/general-writer.hpp"
+
+#define TEST_SUITE "bio-end"
+
+using namespace ncbi;
+
+const char* tableName = "table1";
+
+void run( const char * p_caseId, bool inclusive, const int32_t coord_start[], const int32_t coord_len[], const int8_t coord_type[], int segments_number )
+{
+ std::string output_path = std::string ( TEST_SUITE "/input/" ) + p_caseId + ".gl";
+ std::string db_path = std::string ( TEST_SUITE "/actual/" ) + p_caseId + "/db";
+ std::string schema_path = inclusive ? TEST_SUITE "/bio-end-incl.vschema" : TEST_SUITE "/bio-end-excl.vschema";
+
+ GeneralWriter *gw;
+ try
+ {
+ gw = new GeneralWriter ( output_path );
+
+ gw -> setRemotePath ( db_path );
+ gw -> useSchema ( schema_path, "bio_end:test:database1" );
+
+ int table_id = gw -> addTable ( tableName );
+ int column_read_start_id = gw -> addIntegerColumn ( table_id, "out_read_start", 32 );
+ int column_read_len_id = gw -> addIntegerColumn ( table_id, "out_read_len", 32 );
+ int column_read_type_id = gw -> addIntegerColumn ( table_id, "out_read_type", 8 );
+
+ gw -> open ();
+
+
+#define write(column_id, column_bits, data) gw -> write ( column_id, column_bits, &data[0], segments_number );
+ write ( column_read_start_id, 32, coord_start );
+ write ( column_read_len_id, 32, coord_len );
+ write ( column_read_type_id, 8, coord_type );
+#undef write
+
+ gw -> nextRow(table_id);
+
+ gw -> endStream ();
+
+ delete gw;
+
+ }
+ catch ( ... )
+ {
+ delete gw;
+ throw;
+ }
+}
+
+int main()
+{
+ {
+ int32_t coord_start[] = {0, 4, 8};
+ int32_t coord_len[] = {4, 4, 4};
+ int8_t coord_type[] = {1, 0, 1};
+ const char * test_case_id = "excl-1";
+ bool inclusive = false;
+
+ run( test_case_id, inclusive, coord_start, coord_len, coord_type, sizeof coord_type / sizeof coord_type[0] );
+ }
+ {
+ int32_t coord_start[] = {0, 4, 8};
+ int32_t coord_len[] = {4, 4, 4};
+ int8_t coord_type[] = {0, 1, 0};
+ const char * test_case_id = "excl-2";
+ bool inclusive = false;
+
+ run( test_case_id, inclusive, coord_start, coord_len, coord_type, sizeof coord_type / sizeof coord_type[0] );
+ }
+ {
+ int32_t coord_start[] = {0, 4, 8, 12, 12};
+ int32_t coord_len[] = {4, 4, 4, 0, 0};
+ int8_t coord_type[] = {0, 1, 0, 1, 0};
+ const char * test_case_id = "excl-3";
+ bool inclusive = false;
+
+ run( test_case_id, inclusive, coord_start, coord_len, coord_type, sizeof coord_type / sizeof coord_type[0] );
+ }
+ {
+ int32_t coord_start[] = {0, 4, 8, 12, 13};
+ int32_t coord_len[] = {4, 4, 4, 1, 0};
+ int8_t coord_type[] = {0, 1, 0, 1, 0};
+ const char * test_case_id = "excl-4";
+ bool inclusive = false;
+
+ run( test_case_id, inclusive, coord_start, coord_len, coord_type, sizeof coord_type / sizeof coord_type[0] );
+ }
+ {
+ int32_t coord_start[] = {0, 4, 8};
+ int32_t coord_len[] = {4, 4, 4};
+ int8_t coord_type[] = {0, 1, 0};
+ const char * test_case_id = "incl-1";
+ bool inclusive = true;
+
+ run( test_case_id, inclusive, coord_start, coord_len, coord_type, sizeof coord_type / sizeof coord_type[0] );
+ }
+
+ return 0;
+}
diff --git a/test/vschema/idx-text/checklookup.cpp b/test/vschema/idx-text/checklookup.cpp
new file mode 100644
index 0000000..b2977f3
--- /dev/null
+++ b/test/vschema/idx-text/checklookup.cpp
@@ -0,0 +1,323 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+/**
+ * Perform checks of function idx:text:lookup on created databases,
+ * because it requires passing parameter through cursor parameter QUERY_NAME
+ */
+
+#include <klib/log.h>
+#include <klib/writer.h>
+#include <kfs/directory.h>
+#include <vdb/manager.h>
+#include <vdb/database.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+#include <vdb/vdb-priv.h>
+
+#include <iostream>
+#include <map>
+
+#define TEST_SUITE "idx-text"
+#define TABLE_NAME "table1"
+
+struct RowRange {
+ int64_t start_id;
+ int64_t stop_id;
+};
+
+struct TestCase {
+ const char* path;
+ std::map<std::string, RowRange> key_ranges;
+};
+
+static std::map<std::string, TestCase> test_cases;
+static const char* test_case_name;
+
+rc_t runChecks(const TestCase& test_case, const VCursor * cursor, uint32_t name_idx, uint32_t name_range_idx)
+{
+ rc_t rc;
+ int64_t first_id;
+ uint64_t count_id;
+
+ rc = VCursorIdRange( cursor, name_idx, &first_id, &count_id );
+ if (rc != 0)
+ {
+ LOGERR( klogInt, rc, "VCursorIdRange() failed" );
+ return rc;
+ }
+
+ for (uint64_t row_id = first_id; row_id < first_id + count_id; ++row_id)
+ {
+ const char * name = NULL;
+ uint32_t name_len;
+ RowRange *row_range;
+
+ rc = VCursorCellDataDirect( cursor, row_id, name_idx, NULL, (void const **)&name, NULL, &name_len );
+ if ( rc != 0 )
+ return rc;
+
+ rc = VCursorParamsSet( ( struct VCursorParams const * )cursor, "QUERY_NAME", "%.*s", name_len, name );
+ if ( rc != 0 )
+ return rc;
+
+ rc = VCursorCellDataDirect( cursor, row_id, name_range_idx, NULL, (void const **)&row_range, NULL, NULL );
+ if ( rc != 0 )
+ return rc;
+
+ std::string name_str(name, name_len);
+
+ if (test_case.key_ranges.find(name_str) == test_case.key_ranges.end())
+ {
+ PLOGMSG( klogInt, (klogErr, "Unexpected name '$(NAME)' in test case '$(TC_NAME)'", "TC_NAME=%s,NAME=%s", test_case_name, name_str.c_str()) );
+ return 1;
+ }
+
+ RowRange row_range_exp = test_case.key_ranges.find(name_str)->second;
+ if (row_range->start_id != row_range_exp.start_id || row_range->stop_id != row_range_exp.stop_id)
+ {
+ PLOGMSG( klogInt, (klogErr, "Row range for name '$(NAME)' in test case '$(TC_NAME)' does not match. Expected: $(EXP_S)-$(EXP_F), actual: $(ACT_S)-$(ACT_F)",
+ "TC_NAME=%s,NAME=%s,EXP_S=%ld,EXP_F=%ld,ACT_S=%ld,ACT_F=%ld",
+ test_case_name, name_str.c_str(), row_range_exp.start_id, row_range_exp.stop_id, row_range->start_id, row_range->stop_id) );
+ return 1;
+ }
+ }
+
+ return rc;
+}
+
+rc_t run(const TestCase& test_case)
+{
+ rc_t rc;
+ KDirectory * cur_dir;
+ const VDBManager * manager;
+ const VDatabase * database;
+ const VTable * table;
+ const VCursor * cursor;
+ uint32_t name_idx;
+ uint32_t name_range_idx;
+
+ rc = KDirectoryNativeDir( &cur_dir );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "KDirectoryNativeDir() failed" );
+ else
+ {
+ rc = VDBManagerMakeRead ( &manager, cur_dir );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "VDBManagerMakeRead() failed" );
+ else
+ {
+ rc = VDBManagerOpenDBRead( manager, &database, NULL, "%s", test_case.path );
+ if (rc != 0)
+ LOGERR( klogInt, rc, "VDBManagerOpenDBRead() failed" );
+ else
+ {
+ rc = VDatabaseOpenTableRead( database, &table, "%s", TABLE_NAME );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "VDatabaseOpenTableRead() failed" );
+ else
+ {
+ rc = VTableCreateCursorRead( table, &cursor );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "VTableCreateCursorRead() failed" );
+ else
+ {
+ /* add columns to cursor */
+ rc = VCursorAddColumn( cursor, &name_idx, "(utf8)NAME" );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "VCursorAddColumn() failed" );
+ else
+ {
+ rc = VCursorAddColumn( cursor, &name_range_idx, "NAME_RANGE" );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "VCursorAddColumn() failed" );
+ else
+ {
+ rc = VCursorOpen( cursor );
+ if (rc != 0)
+ LOGERR( klogInt, rc, "VCursorOpen() failed" );
+ else
+ rc = runChecks( test_case, cursor, name_idx, name_range_idx );
+ }
+ }
+ VCursorRelease( cursor );
+ }
+ VTableRelease( table );
+ }
+ VDatabaseRelease( database );
+ }
+
+ VDBManagerRelease( manager );
+ }
+ KDirectoryRelease( cur_dir );
+ }
+ return rc;
+}
+
+/* New test cases should go here */
+void initTestCases()
+{
+ {
+ std::map<std::string, RowRange> key_ranges;
+ {
+ RowRange range = {1, 4};
+ key_ranges["a"] = range;
+ key_ranges["A"] = range;
+ }
+ {
+ RowRange range = {5, 5};
+ key_ranges["b"] = range;
+ }
+
+ TestCase tc = { TEST_SUITE "/actual/ci-1/db", key_ranges };
+ test_cases["ci-1"] = tc;
+ }
+
+ {
+ std::map<std::string, RowRange> key_ranges;
+ {
+ RowRange range = {1, 5};
+ key_ranges["abcdefghi"] = range;
+ key_ranges["abcdefGHI"] = range;
+ key_ranges["ABcdefghi"] = range;
+ }
+ {
+ RowRange range = {6, 6};
+ key_ranges["ABcdefghi1234567890"] = range;
+ }
+
+ TestCase tc = { TEST_SUITE "/actual/ci-2/db", key_ranges };
+ test_cases["ci-2"] = tc;
+ }
+
+ {
+ std::map<std::string, RowRange> key_ranges;
+ {
+ RowRange range = {1, 4};
+ key_ranges["abcdefghi"] = range;
+ key_ranges["abcdefGHI"] = range;
+ key_ranges["ABcdefghi"] = range;
+ }
+
+ TestCase tc = { TEST_SUITE "/actual/ci-3/db", key_ranges };
+ test_cases["ci-3"] = tc;
+ }
+
+ {
+ std::map<std::string, RowRange> key_ranges;
+ {
+ RowRange range = {1, 4};
+ key_ranges["abcdefghi"] = range;
+ key_ranges["abcdefGHI"] = range;
+ key_ranges["ABcdefghi"] = range;
+ }
+
+ TestCase tc = { TEST_SUITE "/actual/ci-4/db", key_ranges };
+ test_cases["ci-4"] = tc;
+ }
+
+ {
+ std::map<std::string, RowRange> key_ranges;
+ {
+ RowRange range = {1, 3};
+ key_ranges["abcdefghi"] = range;
+ }
+
+ {
+ RowRange range = {4, 4};
+ key_ranges["abcdefGHI"] = range;
+ }
+
+ {
+ RowRange range = {5, 5};
+ key_ranges["ABcdefghi"] = range;
+ }
+
+ {
+ RowRange range = {6, 6};
+ key_ranges["ABcdefghi1234567890"] = range;
+ }
+
+ TestCase tc = { TEST_SUITE "/actual/cs-1/db", key_ranges };
+ test_cases["cs-1"] = tc;
+ }
+
+ {
+ std::map<std::string, RowRange> key_ranges;
+ {
+ RowRange range = {1, 3};
+ key_ranges["abcdefghi"] = range;
+ }
+
+ {
+ RowRange range = {4, 4};
+ key_ranges["abcdefGHI"] = range;
+ }
+
+ {
+ RowRange range = {5, 5};
+ key_ranges["ABcdefghi"] = range;
+ }
+
+ {
+ RowRange range = {6, 6};
+ key_ranges["ABcdefghi1234567890"] = range;
+ }
+
+ TestCase tc = { TEST_SUITE "/actual/cs-2/db", key_ranges };
+ test_cases["cs-2"] = tc;
+ }
+}
+
+int main(int argc, const char* argv[])
+{
+ /* init logging */
+ KLogLibHandlerSetStdErr();
+ KLogHandlerSetStdErr();
+ KWrtInit("checklookup", 0);
+
+ initTestCases();
+
+ if (argc < 2)
+ {
+ std::cerr << "Please specify test case name" << std::endl;
+ return 1;
+ }
+
+ test_case_name = argv[1];
+ if (test_cases.find(test_case_name) == test_cases.end())
+ {
+ std::cerr << "Unknown test case name: " << test_case_name << std::endl;
+ return 2;
+ }
+
+ rc_t rc = run(test_cases[test_case_name]);
+ if (rc != 0)
+ return 3;
+
+ std::cout << "Success" << std::endl;
+ return 0;
+}
\ No newline at end of file
diff --git a/test/vschema/idx-text/expected/ci-1.stdout b/test/vschema/idx-text/expected/ci-1.stdout
new file mode 100644
index 0000000..85403c2
--- /dev/null
+++ b/test/vschema/idx-text/expected/ci-1.stdout
@@ -0,0 +1,15 @@
+ NAME: a
+NAME_RANGE:
+
+ NAME: a
+NAME_RANGE:
+
+ NAME: a
+NAME_RANGE:
+
+ NAME: A
+NAME_RANGE:
+
+ NAME: b
+NAME_RANGE:
+
diff --git a/test/vschema/idx-text/expected/ci-2.stdout b/test/vschema/idx-text/expected/ci-2.stdout
new file mode 100644
index 0000000..cbeb94d
--- /dev/null
+++ b/test/vschema/idx-text/expected/ci-2.stdout
@@ -0,0 +1,18 @@
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefGHI
+NAME_RANGE:
+
+ NAME: ABcdefghi
+NAME_RANGE:
+
+ NAME: ABcdefghi1234567890
+NAME_RANGE:
+
diff --git a/test/vschema/idx-text/expected/ci-3.stdout b/test/vschema/idx-text/expected/ci-3.stdout
new file mode 100644
index 0000000..fa546e0
--- /dev/null
+++ b/test/vschema/idx-text/expected/ci-3.stdout
@@ -0,0 +1,12 @@
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefGHI
+NAME_RANGE:
+
+ NAME: ABcdefghi
+NAME_RANGE:
+
diff --git a/test/vschema/idx-text/expected/ci-4.stdout b/test/vschema/idx-text/expected/ci-4.stdout
new file mode 100644
index 0000000..fa546e0
--- /dev/null
+++ b/test/vschema/idx-text/expected/ci-4.stdout
@@ -0,0 +1,12 @@
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefGHI
+NAME_RANGE:
+
+ NAME: ABcdefghi
+NAME_RANGE:
+
diff --git a/test/vschema/idx-text/expected/cs-1.stdout b/test/vschema/idx-text/expected/cs-1.stdout
new file mode 100644
index 0000000..cbeb94d
--- /dev/null
+++ b/test/vschema/idx-text/expected/cs-1.stdout
@@ -0,0 +1,18 @@
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefGHI
+NAME_RANGE:
+
+ NAME: ABcdefghi
+NAME_RANGE:
+
+ NAME: ABcdefghi1234567890
+NAME_RANGE:
+
diff --git a/test/vschema/idx-text/expected/cs-2.stdout b/test/vschema/idx-text/expected/cs-2.stdout
new file mode 100644
index 0000000..cbeb94d
--- /dev/null
+++ b/test/vschema/idx-text/expected/cs-2.stdout
@@ -0,0 +1,18 @@
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefghi
+NAME_RANGE:
+
+ NAME: abcdefGHI
+NAME_RANGE:
+
+ NAME: ABcdefghi
+NAME_RANGE:
+
+ NAME: ABcdefghi1234567890
+NAME_RANGE:
+
diff --git a/test/vschema/idx-text/idx-text.vschema b/test/vschema/idx-text/idx-text.vschema
new file mode 100644
index 0000000..7ec91c5
--- /dev/null
+++ b/test/vschema/idx-text/idx-text.vschema
@@ -0,0 +1,105 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+version 1;
+
+include 'vdb/vdb.vschema';
+
+/* ======= case insensitive - lower ======= */
+table idx_text:ci_lower:test:table #1
+{
+ extern column utf8 NAME = out_name_utf8;
+ physical utf8 .NAME = idx:text:insert < 'i_name', CASE_INSENSITIVE_LOWER > ( NAME );
+
+ utf8 out_name_utf8 = idx:text:project < 'i_name', CASE_INSENSITIVE_LOWER > ( .NAME );
+
+ // Mechanism to seach for NAME
+ readonly column vdb:row_id_range NAME_RANGE
+ = idx:text:lookup < 'i_name', 'QUERY_NAME', CASE_INSENSITIVE_LOWER > ();
+}
+
+database idx_text:ci_lower:test:database #1
+{
+ table idx_text:ci_lower:test:table #1 table1;
+}
+
+/* ======= case insensitive - upper ======= */
+table idx_text:ci_upper:test:table #1
+{
+ extern column utf8 NAME = out_name_utf8;
+ physical utf8 .NAME = idx:text:insert < 'i_name', CASE_INSENSITIVE_UPPER > ( NAME );
+
+ utf8 out_name_utf8 = idx:text:project < 'i_name', CASE_INSENSITIVE_UPPER > ( .NAME );
+
+ // Mechanism to seach for NAME
+ readonly column vdb:row_id_range NAME_RANGE
+ = idx:text:lookup < 'i_name', 'QUERY_NAME', CASE_INSENSITIVE_UPPER > ();
+}
+
+database idx_text:ci_upper:test:database #1
+{
+ table idx_text:ci_upper:test:table #1 table1;
+}
+
+/* ======= case sensitive ======= */
+table idx_text:cs:test:table #1
+{
+ extern column utf8 NAME = out_name_utf8;
+ physical utf8 .NAME = idx:text:insert < 'i_name', CASE_SENSITIVE > ( NAME );
+
+ utf8 out_name_utf8 = idx:text:project < 'i_name', CASE_SENSITIVE > ( .NAME );
+
+ ascii out_name = cast ( out_name_utf8 );
+
+ // Mechanism to seach for NAME
+ readonly column vdb:row_id_range NAME_RANGE
+ = idx:text:lookup < 'i_name', 'QUERY_NAME', CASE_SENSITIVE > ();
+}
+
+database idx_text:cs:test:database #1
+{
+ table idx_text:cs:test:table #1 table1;
+}
+
+/* ======= default: case sensitive ======= */
+table idx_text:default_cs:test:table #1
+{
+ extern column utf8 NAME = out_name_utf8;
+ physical utf8 .NAME = idx:text:insert < 'i_name' > ( NAME );
+
+ utf8 out_name_utf8 = idx:text:project < 'i_name' > ( .NAME );
+
+ ascii out_name = cast ( out_name_utf8 );
+
+ // Mechanism to seach for NAME
+ readonly column vdb:row_id_range NAME_RANGE
+ = idx:text:lookup < 'i_name', 'QUERY_NAME' > ();
+}
+
+database idx_text:default_cs:test:database #1
+{
+ table idx_text:default_cs:test:table #1 table1;
+}
\ No newline at end of file
diff --git a/test/vschema/idx-text/makeinputs.cpp b/test/vschema/idx-text/makeinputs.cpp
new file mode 100644
index 0000000..e8ecdf9
--- /dev/null
+++ b/test/vschema/idx-text/makeinputs.cpp
@@ -0,0 +1,148 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+/**
+ * Unit tests for text index functionality
+ */
+#include "../../../tools/general-loader/general-writer.hpp"
+
+#include <string>
+
+#define TEST_SUITE "idx-text"
+
+using namespace ncbi;
+
+const char* tableName = "table1";
+
+enum TestCaseType
+{
+ caseInsensitiveLower,
+ caseInsensitiveUpper,
+ caseSensitive,
+ defaultCaseSensitive
+};
+
+void run( const char * p_caseId, TestCaseType test_case_type, const std::string names[], size_t names_len )
+{
+ std::string output_path = std::string ( TEST_SUITE "/input/" ) + p_caseId + ".gl";
+ std::string db_path = std::string ( TEST_SUITE "/actual/" ) + p_caseId + "/db";
+ std::string schema_path = TEST_SUITE "/idx-text.vschema";
+
+ GeneralWriter *gw;
+ try
+ {
+ const char* db_schema;
+ switch (test_case_type) {
+ case caseInsensitiveLower:
+ db_schema = "idx_text:ci_lower:test:database";
+ break;
+ case caseInsensitiveUpper:
+ db_schema = "idx_text:ci_upper:test:database";
+ break;
+ case caseSensitive:
+ db_schema = "idx_text:cs:test:database";
+ break;
+ case defaultCaseSensitive:
+ db_schema = "idx_text:default_cs:test:database";
+ break;
+ default:
+ assert(false);
+ }
+ gw = new GeneralWriter ( output_path );
+
+ gw -> setRemotePath ( db_path );
+ gw -> useSchema ( schema_path, db_schema );
+
+ int table_id = gw -> addTable ( tableName );
+ int column_name_id = gw -> addColumn ( table_id, "NAME", 8 );
+
+ gw -> open ();
+
+ for (int i = 0; i < names_len; ++i)
+ {
+ gw -> write ( column_name_id, 8, names[i].c_str(), names[i].size() );
+
+ gw -> nextRow(table_id);
+ }
+
+ gw -> endStream ();
+
+ delete gw;
+
+ }
+ catch ( ... )
+ {
+ delete gw;
+ throw;
+ }
+}
+
+int main()
+{
+ {
+ std::string names[] = {"a", "a", "a", "A", "b"};
+ const char * test_case_id = "ci-1";
+
+ run( test_case_id, caseInsensitiveLower, names, sizeof names / sizeof names[0] );
+ }
+
+ {
+ std::string names[] = {"abcdefghi", "abcdefghi", "abcdefghi", "abcdefGHI", "ABcdefghi", "ABcdefghi1234567890"};
+ const char * test_case_id = "ci-2";
+
+ run( test_case_id, caseInsensitiveLower, names, sizeof names / sizeof names[0] );
+ }
+
+ {
+ std::string names[] = {"abcdefghi", "abcdefghi", "abcdefGHI", "ABcdefghi"};
+ const char * test_case_id = "ci-3";
+
+ run( test_case_id, caseInsensitiveLower, names, sizeof names / sizeof names[0] );
+ }
+
+ {
+ std::string names[] = {"abcdefghi", "abcdefghi", "abcdefGHI", "ABcdefghi"};
+ const char * test_case_id = "ci-4";
+
+ run( test_case_id, caseInsensitiveUpper, names, sizeof names / sizeof names[0] );
+ }
+
+ {
+ std::string names[] = {"abcdefghi", "abcdefghi", "abcdefghi", "abcdefGHI", "ABcdefghi", "ABcdefghi1234567890"};
+ const char * test_case_id = "cs-1";
+
+ run( test_case_id, caseSensitive, names, sizeof names / sizeof names[0] );
+ }
+
+ {
+ std::string names[] = {"abcdefghi", "abcdefghi", "abcdefghi", "abcdefGHI", "ABcdefghi", "ABcdefghi1234567890"};
+ const char * test_case_id = "cs-2";
+
+ run( test_case_id, defaultCaseSensitive, names, sizeof names / sizeof names[0] );
+ }
+
+ return 0;
+}
diff --git a/test/vschema/runtestcase.sh b/test/vschema/runtestcase.sh
new file mode 100755
index 0000000..33f89ce
--- /dev/null
+++ b/test/vschema/runtestcase.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+#echo "$0 $*"
+
+# $1 - path to sra-tools executables (general-loader, vdb-dump)
+# $2 - work directory (expected results under expected/, actual results and temporaries created under actual/)
+# $3 - test case ID (expect a file input/$3.gl to exist)
+# $4 - expected return coode
+# $5, $6, ... - command line options for general-loader
+#
+# return codes:
+# 0 - passed
+# 1 - could not create temp dir
+# 2 - unexpected return code from general-loader
+# 3 - vdb-dump failed on the output of general-loader
+# 4 - outputs differ
+
+BINDIR=$1
+WORKDIR=$2
+SUITEID=$3
+CASEID=$4
+RC=$5
+EXTRA_CHECKER_PRESENT=$6
+if [ "$EXTRA_CHECKER_PRESENT" == "y" ] ; then
+ EXTRA_CHECKER=$7
+ shift 7
+else
+ EXTRA_CHECKER=
+ shift 6
+fi
+CMDLINE=$*
+
+DUMP="$BINDIR/vdb-dump"
+LOAD="$BINDIR/general-loader"
+TEMPDIR=$WORKDIR/actual/$CASEID
+
+echo "running test $SUITEID:$CASEID"
+
+mkdir -p $TEMPDIR
+rm -rf $TEMPDIR/*
+if [ "$?" != "0" ] ; then
+ exit 1
+fi
+
+CMD="cat $SUITEID/input/$CASEID.gl | $LOAD $CMDLINE 1>$TEMPDIR/load.stdout 2>$TEMPDIR/load.stderr"
+#echo $CMD
+eval $CMD
+rc="$?"
+if [ "$rc" != "$RC" ] ; then
+ echo "$LOAD returned $rc, expected $RC"
+ echo "command executed:"
+ echo $CMD
+
+ cat $TEMPDIR/load.stdout
+ cat $TEMPDIR/load.stderr
+ exit 2
+fi
+
+if [ "$rc" == "0" ] ; then
+ CMD="$DUMP $TEMPDIR/db 1>$TEMPDIR/dump.stdout 2>$TEMPDIR/dump.stderr"
+ #echo $CMD
+ eval $CMD || exit 3
+ diff $WORKDIR/expected/$CASEID.stdout $TEMPDIR/dump.stdout >$TEMPDIR/diff
+ rc="$?"
+else
+ # remove timestamps
+ sed -i -e 's/^....-..-..T..:..:.. //g' $TEMPDIR/load.stderr
+ # remove pathnames
+ sed -i -e 's=/.*/==g' $TEMPDIR/load.stderr
+ # remove source locations
+ sed -i -e 's=: .*:[0-9]*:[^ ]*:=:=g' $TEMPDIR/load.stderr
+ # remove version number
+ sed -i -e 's=latf-load\(\.[0-9]*\)*=latf-load=g' $TEMPDIR/load.stderr
+ diff $WORKDIR/expected/$CASEID.stderr $TEMPDIR/load.stderr >$TEMPDIR/diff
+ rc="$?"
+fi
+
+if [ "$rc" != "0" ] ; then
+ echo "command executed:"
+ echo $CMD
+
+ cat $TEMPDIR/diff
+ exit 4
+fi
+
+if [ "$EXTRA_CHECKER_PRESENT" == "y" ] ; then
+ CMD="$EXTRA_CHECKER $CASEID 1>$TEMPDIR/extra_checker.stdout 2>$TEMPDIR/extra_checker.stderr"
+ eval $CMD
+ rc="$?"
+
+ if [ "$rc" != "0" ] ; then
+ echo "command executed:"
+ echo $CMD
+
+ cat $TEMPDIR/extra_checker.stdout
+ cat $TEMPDIR/extra_checker.stderr
+
+ exit 5
+ fi
+fi
+
+rm -rf $TEMPDIR
+
+exit 0
diff --git a/tools/.gitignore b/tools/.gitignore
index 0bff5eb..f523b36 100644
--- a/tools/.gitignore
+++ b/tools/.gitignore
@@ -1,3 +1,6 @@
pileupcomp
avg-qual
sra-rewrite
+core.*
+gmon.out
+
diff --git a/tools/Makefile b/tools/Makefile
index 13dc05b..f88c281 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -34,44 +34,44 @@ include $(TOP)/build/Makefile.config
#-------------------------------------------------------------------------------
# default
#
-SUBDIRS = \
- util \
- kar \
- kqsh \
- vdb-dump \
- sra-dump \
- sra-pileup \
- bam-loader \
- fastq-loader \
- srapath \
- sra-stat \
- vdb-copy \
- rd-filter-redact \
- vdb-config \
- align-info \
+SUBDIRS = \
+ util \
+ align-cache \
+ kar \
+ kqsh \
+ vdb-dump \
+ sra-dump \
+ sra-pileup \
+ bam-loader \
+ fastq-loader \
+ srapath \
+ sra-stat \
+ vdb-copy \
+ rd-filter-redact \
+ vdb-config \
+ align-info \
qual-recalib-stat \
- prefetch \
- vdb-decrypt \
- vdb-validate \
- sra-sort \
- cache-mgr \
- vcf-loader
+ prefetch \
+ vdb-decrypt \
+ vdb-validate \
+ sra-sort \
+ cache-mgr \
+ vcf-loader \
+ copycat \
+ sra-load \
+ ccextract \
+ pacbio-load \
+ fuse \
+ vdb-diff \
+ kget \
+ ngs-pileup \
+ general-loader \
+ ref-variation \
-ifeq (1,$(HAVE_MAGIC))
- SUBDIRS += copycat
-endif
-
-ifeq (1,$(HAVE_XML2))
- SUBDIRS += sra-load ccextract
-endif
-
-ifeq (1,$(HAVE_HDF5))
- SUBDIRS += pacbio-load
-endif
ifneq (win,$(BUILD_OS))
ifneq (rwin,$(BUILD_OS))
- SUBDIRS += cg-load
+ SUBDIRS += cg-load pileup-stats
endif
endif
diff --git a/tools/agp-load/AGP.py b/tools/agp-load/AGP.py
new file mode 100644
index 0000000..83538be
--- /dev/null
+++ b/tools/agp-load/AGP.py
@@ -0,0 +1,668 @@
+#!python
+# =============================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# =============================================================================
+
+
+import tarfile
+import gzip
+import os
+from eutils import EUtils
+
+class _loc:
+ def __init__(self, mol, start, last):
+ self.mol = mol
+ self.start = start
+ self.last = last
+
+ def dir(self):
+ if self.last < self.start:
+ return -1
+ else:
+ return 1
+
+ def length(self):
+ if self.last < self.start:
+ return (self.start - self.last) + 1
+ else:
+ return (self.last - self.start) + 1
+
+ def split(self, at):
+ if self.last < self.start:
+ left = _loc(self.mol, self.start, self.start - at + 1)
+ right = _loc(self.mol, self.start - at, self.last)
+ else:
+ left = _loc(self.mol, self.start, self.start + at - 1)
+ right = _loc(self.mol, self.start + at, self.last)
+ return left, right
+
+ def merge(self, other):
+ if self.mol != other.mol:
+ return None
+ if self.last + self.dir() != other.start:
+ return None
+ return _loc(self.mol, self.start, other.last)
+
+ def __str__(self):
+ return "\t".join([self.mol, str(self.start), str(self.last)])
+
+class map_pair:
+ def __init__(self, first, second):
+ if first.last >= first.start:
+ self.first = first
+ self.second = second
+ else:
+ self.first = _loc(first.mol, first.last, first.start)
+ self.second = _loc(second.mol, second.last, second.start)
+
+ def dir(self):
+ return self.second.dir()
+
+ def agp(self):
+ r = [self.first.mol, self.first.start, self.first.last, 0, 'O', self.second.mol, self.second.start, self.second.last, 1]
+ if r[7] < r[6]:
+ r[6], r[7], r[8] = r[7], r[6], -1
+ return r
+
+ def merge(self, other):
+ if self.dir() != other.dir(): return None
+ first = self.first.merge(other.first)
+ if first == None: return None
+ second = self.second.merge(other.second)
+ if second == None: return None
+ return map_pair(first, second)
+
+ @classmethod
+ def cmp(cls, a, b):
+ if a.first.mol < b.first.mol: return -1
+ if a.first.mol > b.first.mol: return 1
+ apos = a.first.start if a.first.start < a.first.last else a.first.last
+ bpos = b.first.start if b.first.start < b.first.last else b.first.last
+ if apos < bpos: return -1
+ if apos > bpos: return 1
+ return 0
+
+class _loc_pair:
+ def __init__(self, first):
+ self.first = first
+ self.second = []
+
+ def add(self, second):
+ self.second.append(second)
+
+ def split(self, at):
+ cut = at - self.first.start
+ l1, r1 = self.first.split(cut)
+ left = _loc_pair(l1)
+ right = _loc_pair(r1)
+ for s in self.second:
+ l2, r2 = s.split(cut)
+ assert(l1.length() == l2.length())
+ assert(r1.length() == r2.length())
+ left.add(l2)
+ right.add(r2)
+ return left, right
+
+ def __str__(self):
+ rslt = ''
+ for s in self.second:
+ rslt += "\t".join(map((lambda a: str(a)), [self.first, s]))+"\n"
+ return rslt
+
+ def agp(self, rev=False):
+ rslt = []
+ for s in self.second:
+ start, last, dir = s.start, s.last, 1
+ if last < start:
+ start, last, dir = last, start, -1
+ r = [s.mol, start, last, 0, "O", self.first.mol, self.first.start, self.first.last, dir] if rev else [self.first.mol, self.first.start, self.first.last, 0, "O", s.mol, start, last, dir]
+ rslt.append(r)
+ return rslt
+
+ @classmethod
+ def make_pair(cls, first, second):
+ rslt = _loc_pair(first)
+ rslt.add(second)
+ return rslt
+
+class _loc_set:
+ def __init__(self):
+ self.set = []
+
+ def _overlap(self, first):
+ f = 0
+ e = len(self.set)
+ while f < e:
+ m = f + (e - f)/2
+ fnd = self.set[m]
+ if fnd.first.start > first.last:
+ e = m
+ continue
+ if fnd.first.last < first.start:
+ f = m + 1
+ continue
+ return m, None
+ return None, f
+
+ def _add(self, first, second):
+ m, f = self._overlap(first)
+ if m == None:
+ self.set.insert(f, _loc_pair.make_pair(first, second))
+ return None
+ if first.start == self.set[m].first.start and first.last == self.set[m].first.last:
+ self.set[m].add(second)
+ return None
+ return m
+
+ def split(self, at):
+ """ Search the list for the record that overlaps the split point, then
+ split that record. No split occurs if the split point is not fully
+ inside any record, in particular, the start is not 'inside'
+ """
+ f = 0
+ e = len(self.set)
+ while f < e:
+ m = f + (e - f)/2
+ fnd = self.set[m]
+ if fnd.first.start > at:
+ e = m
+ continue
+ if fnd.first.last < at:
+ f = m + 1
+ continue
+ if fnd.first.start < at:
+ l, r = fnd.split(at)
+ self.set[m:m+1] = [l, r]
+ break
+
+ def add(self, first, second):
+ if len(self.set) == 0:
+ self.set.append(_loc_pair.make_pair(first, second))
+ return
+ else:
+ assert(first.mol == self.set[0].first.mol)
+ self.split(first.start)
+ self.split(first.last + 1)
+ m = self._add(first, second)
+ if m == None: return
+ if self.set[m].first.start > first.start:
+ cut = self.set[m].first.start - first.start
+ l1, r1 = first.split(cut)
+ l2, r2 = second.split(cut)
+ self._add(l1, l2)
+ self.add(r1, r2)
+ return
+ if self.set[m].first.last < first.last:
+ cut = (first.last) - (self.set[m].first.last)
+ l1, r1 = first.split(first.length() - cut)
+ l2, r2 = second.split(second.length() - cut)
+ self._add(r1, r2)
+ self.add(l1, l2)
+ return
+
+ def intersect(self, other):
+ if len(self.set) == 0 or len(other.set) == 0: return
+ assert(self.set[0].first.mol == other.set[0].first.mol)
+ for r in self.set:
+ other.split(r.first.start)
+ other.split(r.first.last + 1)
+ for r in other.set:
+ self.split(r.first.start)
+ self.split(r.first.last + 1)
+ i, j = 0, 0
+ while i < len(self.set) and j < len(other.set):
+ if self.set[i].first.start == other.set[j].first.start:
+ assert(self.set[i].first.length() == other.set[j].first.length())
+ i += 1
+ j += 1
+ continue
+ if self.set[i].first.start < other.set[j].first.start:
+ del self.set[i]
+ else:
+ del other.set[j]
+ del self.set[i:]
+ del other.set[j:]
+
+ def agp(self, rev=False):
+ rslt = []
+ for r in self.set:
+ rslt.extend(r.agp(rev))
+ if rev:
+ def cmp(a, b):
+ if a[0] < b[0]: return -1
+ if a[0] > b[0]: return 1
+ if a[1] < b[1]: return -1
+ if a[1] > b[1]: return 1
+ rslt.sort(cmp)
+ return rslt
+
+ def __str__(self):
+ return reduce((lambda a, b: a + str(b)), self.set, '')
+
+
+def _loadFile(fh, file, func):
+ def loadTarFile(fh, file, func):
+ with tarfile.open(mode='r', fileobj=fh) as tar:
+ for f in tar:
+ i = tar.extractfile(f)
+ _loadFile(i, os.path.join(file, f.name), func)
+ i.close()
+
+ def loadText(fh, file, func):
+ for i, line in enumerate(fh):
+ func(line, "{}:{}".format(file, i))
+
+ try:
+ loadTarFile(fh, file, func)
+ return
+ except tarfile.ReadError:
+ pass
+ try:
+ with gzip.GzipFile(fileobj=fh) as gz:
+ loadText(gz, file[:-3] if file.endswith(".gz") else file, func)
+ return
+ except IOError:
+ pass
+ loadText(fh, file, func)
+
+
+class AGP:
+ def loadFile(self, fname):
+ """ AGP Fields:
+ object
+ beg
+ end
+ partno # not used, zero'ed out when flattened
+ type # gaps are ignored, infer them from the gaps between subobjects
+ subobject
+ sub_beg
+ sub_end
+ strand
+ source # ADDED, <file name>:<line number>, this is just for debugging
+ """
+ recs = []
+ def cb(line, src):
+ if line[0] == '#': return
+ fld = line[:-1].split("\t")
+ if len(fld) != 9: return
+ if fld[4] == "N" or fld[4] == "U": return
+ fld[0] = fld[0]
+ fld[1] = int(fld[1])
+ fld[2] = int(fld[2])
+ fld[3] = int(fld[3])
+ fld[6] = int(fld[6])
+ fld[7] = int(fld[7])
+ fld[8] = -1 if fld[8] == '-' else 1
+ fld.append(src)
+ assert(fld[2] - fld[1] == fld[7] - fld[6]) # lengths are the same
+ recs.append(fld)
+ with open(fname, "r") as fh:
+ _loadFile(fh, fname, cb)
+ self._data.extend(recs)
+
+ def __init__(self):
+ self._data = []
+
+ def _topLevelObjects(self):
+ sub = dict.fromkeys(v[5] for v in self._data)
+ return dict.fromkeys(v[0] for v in self._data if v[0] not in sub)
+
+ def topLevelObjects(self):
+ return sorted(self._topLevelObjects().keys())
+
+ def _index(self):
+ """ Sorted by
+ object (fld[0]),
+ beg (fld[1]),
+ end (fld[2]) desc
+ """
+ def index_cmp(a, b):
+ if self._data[a][0] < self._data[b][0]:
+ return -1
+ if self._data[b][0] < self._data[a][0]:
+ return 1
+ if self._data[a][1] < self._data[b][1]:
+ return -1
+ if self._data[b][1] < self._data[a][1]:
+ return 1
+ if self._data[a][2] < self._data[b][2]:
+ return 1
+ if self._data[b][2] < self._data[a][2]:
+ return -1
+ return a - b
+ return sorted(range(len(self._data)), index_cmp)
+
+ def _clean(self):
+ """ Returns a cleansed version of the input data
+ without any redundant or duplicate records
+ """
+ rslt = []
+ for i in self._index():
+ v = self._data[i]
+ if len(rslt) == 0:
+ rslt.append(v)
+ continue
+ u = rslt[-1]
+ if v[0] != u[0]: # different object
+ rslt.append(v)
+ continue
+ if v[1] > u[2]: # starts after the previous one ended
+ rslt.append(v)
+ continue
+ if v[2] > u[2]: # it's an error if it overlaps the previous one
+ print("The record from {}".format(v[9]))
+ print("\t".join(v))
+ print("conflicts with the record from {}".format(rslt[-1][9]))
+ print("\t".join(rslt[-1]))
+ assert("The AGP data contains errors:")
+ return rslt
+
+ def _flattened(self, args):
+ """ yields list of flattened AGP """
+ index = {}
+ for v in self._clean():
+ if v[0] in index:
+ index[v[0]].append(v)
+ else:
+ index[v[0]] = [v]
+ isflat = {k:False for k in index}
+ def flatten(k):
+ flat = []
+ for v in index[k]:
+ sub = v[5]
+ if sub not in index: # has no subparts
+ flat.append(v[0:3] + [0] + v[4:-1])
+ continue
+ assert(v[8] > 0) # multipart objects aren't reversable?
+ if not isflat[sub]: flatten(sub)
+ for f in index[sub]:
+ if f[2] < v[6] or f[1] > v[7]:
+ continue
+ # parts aren't splitable
+ assert(f[1] >= v[6] and f[2] <= v[7])
+ n = [v[0], f[1]+v[1]-1, f[2]+v[1]-1, 0] + f[4:]
+ flat.append(n)
+ index[k] = flat
+ isflat[k] = True
+ for k in args:
+ flatten(k)
+ yield index[k]
+
+ def flattened(self, args=[]):
+ objects = self._topLevelObjects() if len(args) == 0 else args
+ for l in self._flattened(objects):
+ for r in l:
+ yield r
+
+ def _leaves(self):
+ rslt = {}
+ for l in self._flattened(self._topLevelObjects().keys()):
+ for r in l[:]:
+ if r[5] not in rslt:
+ rslt[r[5]] = _loc_set()
+ if r[8] < 0:
+ other = _loc(r[0], r[2], r[1])
+ else:
+ other = _loc(r[0], r[1], r[2])
+ rslt[r[5]].add(_loc(r[5], r[6], r[7]), other)
+ return rslt
+
+ def remap(self, other):
+ def getSummary(acc):
+ while True:
+ summary = EUtils.summary(acc)
+ if summary:
+ return summary
+
+ superceeds_memo = {}
+ def superceeds(a, b):
+ try: return superceeds_memo[b] == a
+ except KeyError:
+ pass
+ assemblyacc = None
+ accessionversion = None
+ replacedby = None
+ try:
+ assemblyacc = getSummary(a)['assemblyacc']
+ accessionversion = getSummary(a)['accessionversion']
+ replacedby = getSummary(b)['replacedby']
+ except KeyError:
+ pass
+ if replacedby == None:
+ superceeds_memo[b] = None
+ return False
+ if replacedby == assemblyacc or replacedby == accessionversion:
+ superceeds_memo[b] = a
+ return True
+ return False
+
+ def compact(l):
+ last = None
+ for r in sorted(l, map_pair.cmp):
+ if last == None:
+ last = r
+ continue
+ x = last.merge(r)
+ if x:
+ last = x
+ continue
+ y = last.agp()
+ last = r
+ yield y
+ if last != None: yield last.agp()
+
+ def remapper():
+ s = self._leaves()
+ o = other._leaves()
+ for k in sorted(s.keys()):
+ if k not in o: continue
+ s[k].intersect(o[k])
+ n = len(s[k].set)
+ assert(n == len(o[k].set))
+ for i in range(0, n):
+ src = s[k].set[i]
+ dst = o[k].set[i]
+ assert(src.first.start == dst.first.start)
+ assert(src.first.last == dst.first.last )
+ src = src.second
+ dst = dst.second
+ if len(src) == 1 and len(dst) == 1:
+ yield map_pair(dst[0], src[0])
+ else:
+ recs = [(r1, r2) for r1 in src for r2 in dst if superceeds(r2.mol, r1.mol)]
+ for r in recs:
+ yield map_pair(r[1], r[0])
+
+ return compact(remapper())
+
+
+if __name__ == "__main__":
+ import sys
+ import subprocess
+ import time
+
+
+ def complement(b):
+ if b == 'A': return 'T'
+ if b == 'C': return 'G'
+ if b == 'G': return 'C'
+ if b == 'T': return 'A'
+ return 'N'
+
+ def clean(b):
+ if b == 'A': return 'A'
+ if b == 'C': return 'C'
+ if b == 'G': return 'G'
+ if b == 'T': return 'T'
+ return 'N'
+
+ def fetchFasta(acc, end):
+ sleep = 0
+ while True:
+ fasta = ''
+ defline = None
+ sleep += 1
+ try:
+ for line in EUtils.fasta(acc):
+ if line.startswith(">"):
+ if defline == None:
+ defline = line
+ else:
+ break
+ else:
+ fasta += line
+ if defline != None:
+ if end <= len(fasta):
+ return (defline, fasta)
+ msg = "got truncated response for {}; retrying in a moment".format(acc)
+ else:
+ msg = "got no response for {}; retrying in a moment".format(acc)
+ except KeyboardInterrupt:
+ sys.stderr.write(" cancelled\n")
+ exit(0)
+ except:
+ msg = "error fetching {}; retrying in a moment".format(acc)
+ pass
+ if sleep > 10:
+ sys.stderr.write(" too many retries; try again later.\n")
+ exit(1)
+ sys.stderr.write(msg)
+ sys.stderr.flush()
+ time.sleep(sleep)
+ sys.stderr.write("\033[{}D\033[K".format(len(msg)))
+ sys.stderr.flush()
+
+
+ cache = {}
+ def fetchObject(acc, start=0, end=-1, rev=False):
+ global cache
+ if acc in cache:
+ fasta = cache[acc]
+ else:
+ (defline, fasta) = fetchFasta(acc, end)
+ cache[acc] = fasta
+ if end < 0: end = len(fasta)
+ assert(end <= len(fasta))
+ assert(start >= 0)
+ assert(start < end)
+ if not rev:
+ return ''.join(map(clean, list(fasta[start:end])))
+ rslt = list(fasta[start:end])
+ i = 0
+ j = len(rslt)
+ while (i < j):
+ j -= 1
+ A = complement(rslt[i])
+ B = complement(rslt[j])
+ rslt[j] = A
+ rslt[i] = B
+ i += 1
+ return ''.join(rslt)
+
+ def usage():
+ print("{} [[ test | dump ] <agp source file> | remap <from agp> <to agp>]".format(sys.argv[0]))
+ exit(1)
+
+ agps = []
+ try:
+ for file in sys.argv[2:]:
+ agp = AGP()
+ agp.loadFile(file)
+ agps.append(agp)
+ except IndexError:
+ usage()
+
+ if len(agps) == 2:
+ for r in agps[0].remap(agps[1]):
+ print(r)
+ exit(0)
+
+ def unflat(agp):
+ un = {}
+ for r in agp.flattened():
+ try: un[r[0]].append(r)
+ except KeyError:
+ un[r[0]] = [r]
+ return un
+
+ def test(agp):
+ flat = unflat(agp)
+ seqID = sorted(flat.keys())
+ for object in seqID:
+ next = 1
+ test = ''
+ sys.stderr.write("reconstructing {} ".format(object))
+ sys.stderr.flush()
+ part = 0
+ count = len(flat[object])
+ progress = ''
+ for r in flat[object]:
+ erase = len(progress)
+ progress = '.'*int((part * 100.0) / count)
+ sys.stderr.write("\x08"*erase + progress)
+ read = fetchObject(r[5], r[6] - 1, r[7], r[8] < 0)
+ test += "N"*(r[1] - next)
+ test += read
+ part += 1
+ next = r[2] + 1
+ sys.stderr.write(" ")
+ cache = {}
+ (dummy, gold) = fetchFasta(object, next)
+ gold = ''.join(map(clean, list(gold)))
+ if len(test) < len(gold):
+ test += "N"*(len(gold)-len(test))
+ if gold == test:
+ sys.stderr.write(" PASSED\n".format(object))
+ else:
+ sys.stderr.write(" FAILED\n".format(object))
+ with open("{}.reconstructed.fasta".format(object), 'w') as fh:
+ fh.write(">{}\n".format(object))
+ i = 0
+ while i < len(test):
+ fh.write("{}\n".format(test[i:i+70]))
+ i += 70
+ with open("{}.expected.fasta".format(object), 'w') as fh:
+ fh.write(">{}\n".format(object))
+ i = 0
+ while i < len(gold):
+ fh.write("{}\n".format(gold[i:i+70]))
+ i += 70
+ exit(1)
+ sys.stderr.write("All passed\n")
+
+ def dump(agp):
+ flat = unflat(agp)
+ seqID = sorted(flat.keys())
+ for object in seqID:
+ for r in flat[object]:
+ print("\t".join(map((lambda x: str(x)), r)))
+
+ if sys.argv[1] == "test":
+ test(agps[0])
+ elif sys.argv[1] == "dump":
+ dump(agps[0])
+ elif sys.argv[1] == "remap" and len(agps) == 2:
+ for r in agps[0].remap(agps[1]):
+ print(r)
+ else:
+ usage()
diff --git a/tools/agp-load/agp-load.py b/tools/agp-load/agp-load.py
new file mode 100644
index 0000000..c6b11a5
--- /dev/null
+++ b/tools/agp-load/agp-load.py
@@ -0,0 +1,173 @@
+#!python
+"""The loader requires the following arguments:
+* for source: -name=<build name> <AGP file> [<AGP file>...]
+* for result: -name=<build name> <AGP file> [<AGP file>...]
+* -output=<output directory>
+
+<AGP file> can be an AGP file or a tar containing, possibly gzip'ed, AGP files.
+<build name> is stored in the output database (see below)
+
+Example:
+ python agp-load.py -name=GRCh37 GRCh37.tar -name=GRCh38.p2 GRCh38.tar -output=37_to_38.remap
+
+The loader generates three tables containing AGP data.
+# SOURCE: the build being remapped from
+# RESULT: the build being remapped to
+# REMAP: the remapping
+
+SOURCE and RESULT will have a metadata node BuildName containing the given build
+names. This AGP data is only for components, i.e. it doesn't specify gaps, as
+they can be implied.
+
+To produce approximately valid AGP from a loaded database:
+ vdb-dump 37_to_38.remap -T REMAP -f tab -b1 -C "SEQID,START,LENGTH,COMPONENT,OFFSET,REVERSED" | awk 'BEGIN{OFS="\\t"; P[""]=0}{print $1, $2, $2+$3-1, ++P[$1], "O", $4, $5, $5+$3-1, 1-($6*2)}'
+"""
+# =============================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# =============================================================================
+
+import sys
+import traceback
+from AGP import AGP
+from GeneralWriter import GeneralWriter
+from array import array
+
+usage = "Usage: {} -help | [ -output=<name of output object> -name=<build name> <AGP files> -name=<build name> <AGP files> ]\n".format(sys.argv[0])
+build = []
+output = None
+
+for arg in sys.argv[1:]:
+ if arg[0] == '-':
+ if arg == "-help":
+ sys.stderr.write(usage)
+ sys.stderr.write(__doc__)
+ exit(0)
+ if arg[0:8] == "-output=":
+ output = arg[8:]
+ elif arg[0:6] == "-name=":
+ build.append({"name": arg[6:], "files": []})
+ else:
+ sys.stderr.write(usage)
+ exit(3)
+ else:
+ build[-1]["files"].append(arg)
+
+if output == None:
+ sys.stderr.write("output must be specified\n")
+ sys.stderr.write(usage);
+ exit(3)
+
+if len(build) != 2:
+ sys.stderr.write("Two builds must be specified\n")
+ sys.stderr.write(usage);
+ exit(3)
+
+if len(build[0]["files"]) == 0 or len(build[1]["files"]) == 0:
+ sys.stderr.write("AGP file lists must be specified\n")
+ sys.stderr.write(usage);
+ exit(3)
+
+if build[0]["name"] == build[1]["name"]:
+ sys.stderr.write("Build names must be unique\n")
+ exit(3)
+
+
+def loadFiles(list):
+ agp = AGP()
+ for f in list:
+ agp.loadFile(f)
+ return agp
+
+def assemblyTblDef():
+ return {
+ 'SEQID': {
+ 'elem_bits': 8,
+ },
+ 'START': {
+ 'elem_bits': 32,
+ },
+ 'LENGTH': {
+ 'elem_bits': 32,
+ },
+ 'COMPONENT': {
+ 'elem_bits': 8,
+ },
+ 'OFFSET': {
+ 'elem_bits': 32,
+ },
+ 'REVERSED': {
+ 'elem_bits': 8,
+ },
+ }
+
+tbl = {
+ 'SOURCE': assemblyTblDef(),
+ 'RESULT': assemblyTblDef(),
+ 'REMAP': assemblyTblDef(),
+}
+
+gw = GeneralWriter( output
+ , 'agp.vschema'
+ , 'NCBI:refseq:db:remap'
+ , 'agp-load.py'
+ , '1.0.0'
+ , tbl)
+
+def logMessage(msg):
+ sys.stderr.write("{}\n".format(msg))
+
+def writeAGP(listOfAGP, tbl):
+ for r in listOfAGP:
+ tbl['SEQID' ]['data'] = str(r[0]).encode('ascii')
+ tbl['START' ]['data'] = array('I', [int(r[1])])
+ tbl['LENGTH' ]['data'] = array('I', [int(r[2] - r[1])])
+ tbl['COMPONENT']['data'] = str(r[5]).encode('ascii')
+ tbl['OFFSET' ]['data'] = array('I', [int(r[6])])
+ tbl['REVERSED' ]['data'] = array('b', [1 if r[8] < 0 else 0])
+ gw.write(tbl)
+
+def main():
+ gw.writeTableMetadata(tbl['SOURCE'], "BuildName", build[0]["name"])
+ gw.writeTableMetadata(tbl['RESULT'], "BuildName", build[1]["name"])
+
+ logMessage("loading {} ...".format(build[0]["name"]))
+ build[0]["object"] = loadFiles(build[0]["files"])
+ writeAGP(build[0]["object"].flattened(), tbl["SOURCE"])
+ logMessage("loaded {}".format(build[0]["name"]))
+
+ logMessage("loading {} ...".format(build[1]["name"]))
+ build[1]["object"] = loadFiles(build[1]["files"])
+ writeAGP(build[1]["object"].flattened(), tbl["RESULT"])
+ logMessage("loaded {}".format(build[1]["name"]))
+
+ logMessage("generating remapping data ...")
+ writeAGP(build[0]["object"].remap(build[1]["object"]), tbl["REMAP"])
+ logMessage("loaded remapping data")
+
+def cleanup():
+ logMessage("done")
+ pass
+
+main()
+gw = None
+cleanup()
diff --git a/tools/agp-load/agp.vschema b/tools/agp-load/agp.vschema
new file mode 100644
index 0000000..8ce7fa4
--- /dev/null
+++ b/tools/agp-load/agp.vschema
@@ -0,0 +1,45 @@
+/*============================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ *============================================================================
+ */
+version 1;
+
+include 'vdb/vdb.vschema';
+
+table NCBI:refseq:tbl:assembly #1.0.0
+{
+ extern column <ascii> zip_encoding SEQID; /* the accession of the refseq being assembled */
+ extern column <U32> izip_encoding START; /* the start and */
+ extern column <U32> izip_encoding LENGTH; /* length of the region */
+ extern column <ascii> zip_encoding COMPONENT; /* the accession sourcing the sequence */
+ extern column <U32> izip_encoding OFFSET; /* the offset in the component, length is the same */
+ extern column bool_encoding REVERSED; /* is the source sequence to be reverse-complemented */
+};
+
+database NCBI:refseq:db:remap #1.0.0
+{
+ table NCBI:refseq:tbl:assembly #1.0.0 SOURCE; /* how to assemble the source from its base components */
+ table NCBI:refseq:tbl:assembly #1.0.0 RESULT; /* how to assemble the result from its base components */
+ table NCBI:refseq:tbl:assembly #1.0.0 REMAP; /* how to assemble the result from the source, */
+ /* and how to remap alignments from source to result */
+};
diff --git a/tools/agp-load/eutils.py b/tools/agp-load/eutils.py
new file mode 100644
index 0000000..8ac15a1
--- /dev/null
+++ b/tools/agp-load/eutils.py
@@ -0,0 +1,112 @@
+#!python
+# =============================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# =============================================================================
+"""
+ Utility class for getting sequence and metadata for accessions via eutils.
+"""
+
+import httplib
+import json
+
+class EUtils:
+ @classmethod
+ def _response(cls, function, *args):
+ """ Builds URL and gets it """
+ url = "/entrez/eutils/{}.fcgi?".format(function)+"&".join(args)
+ try:
+ eutils = httplib.HTTPConnection("eutils.ncbi.nlm.nih.gov")
+ eutils.request('GET', url)
+ return eutils.getresponse()
+ except:
+ return None
+
+ @classmethod
+ def _giList(cls, acc):
+ """ gets gi for accession """
+ response = EUtils._response("esearch", "db=nuccore", "retmode=json", "term={}".format(acc))
+ if response:
+ return json.load(response)['esearchresult']['idlist']
+ return None
+
+ @classmethod
+ def summary(cls, acc):
+ """
+ Retrieves the metadata for an accession
+ Returns a dictionary
+ """
+ gi = EUtils._giList(acc)
+ if gi:
+ response = EUtils._response("esummary", "db=nuccore", "retmode=json", "id={}".format(gi[0]))
+ if response:
+ return json.load(response)["result"]["{}".format(gi[0])]
+ return None
+
+ @classmethod
+ def fasta(cls, acc):
+ """
+ Retrieves the FASTA sequence for an accession
+ Returns a generator on the lines of the FASTA sequence
+ """
+ gi = EUtils._giList(acc)
+ if gi:
+ response = EUtils._response("efetch", "db=nuccore", "retmode=text", "rettype=fasta", "id={}".format(gi[0]))
+ def lines():
+ buf = response.read(4096)
+ at = 0
+ while len(buf) > 0:
+ if at >= 4096:
+ buf = buf[4096:]
+ at -= 4096
+ next = buf.find("\n", at)
+ if next < 0:
+ more = response.read(4096)
+ if more:
+ buf += more
+ continue
+ line = buf[at:]
+ buf = ""
+ at = 0
+ yield line
+ else:
+ line = buf[at:next]
+ at = next + 1
+ yield line
+ return lines()
+ return None
+
+if __name__ == "__main__":
+ import sys
+
+ if sys.argv[1] == "summary":
+ for acc in sys.argv[2:]:
+ summary = EUtils.summary(acc)
+ if summary:
+ for k in sorted(summary):
+ print("{}: {}".format(k,summary[k]))
+ else:
+ print("Nothing found for {}".format(acc))
+ elif sys.argv[1] == "fasta":
+ for acc in sys.argv[2:]:
+ for line in EUtils.fasta(acc):
+ print(line)
diff --git a/tools/vtblcp/Makefile b/tools/align-cache/Makefile
similarity index 86%
rename from tools/vtblcp/Makefile
rename to tools/align-cache/Makefile
index 024d9cc..bcf306f 100644
--- a/tools/vtblcp/Makefile
+++ b/tools/align-cache/Makefile
@@ -26,14 +26,14 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/vtblcp
+MODULE = tools/align-cache
include $(TOP)/build/Makefile.env
INT_TOOLS =
EXT_TOOLS = \
- vtblcp
+ align-cache
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -73,25 +73,21 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# vtblcp
-# vdb table copy tool
+# align-cache
#
-VTBLCP_SRC = \
- vtblcp
+ALIGN_CACHE_SRC = \
+ align-cache \
+ helper
-VTBLCP_OBJ = \
- $(addsuffix .$(OBJX),$(VTBLCP_SRC))
+ALIGN_CACHE_OBJ = \
+ $(addsuffix .$(OBJX),$(ALIGN_CACHE_SRC))
-VTBLCP_LIB = \
- -swvdb \
- -swkdb \
+ALIGN_CACHE_LIB = \
-skapp \
- -skfs \
- -sklib
-
-vtblcp.vers.h: vtblcp.vers
- $(TOP)/build/make-vers-inc.sh VTBLCP_VERS $^ > $@
-
-$(BINDIR)/vtblcp: $(VTBLCP_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(VTBLCP_LIB)
+ -sncbi-wvdb \
+ -sxml2 \
+ -sm \
+ -sload
+$(BINDIR)/align-cache: $(ALIGN_CACHE_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(ALIGN_CACHE_LIB)
diff --git a/tools/align-cache/align-cache.cpp b/tools/align-cache/align-cache.cpp
new file mode 100644
index 0000000..63b1782
--- /dev/null
+++ b/tools/align-cache/align-cache.cpp
@@ -0,0 +1,557 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "align-cache.vers.h"
+#include "helper.h"
+
+#include <stdio.h>
+#include <iostream>
+
+#include <kapp/main.h>
+#include <klib/rc.h>
+#include <klib/log.h>
+
+
+namespace AlignCache
+{
+ struct Params
+ {
+ // Command line Params:
+ char const* dbPathSrc;
+ char const* dbPathDst;
+
+ // Command line options
+ int64_t id_spread_threshold;
+ size_t cursor_cache_size;
+ size_t min_cache_count;
+
+ // Internal parameters
+ bool cache_alignment_count;
+ } g_Params =
+ {
+ // Command line Params:
+ "",
+ "",
+ // Command line options
+ 50000,
+#if _ARCH_BITS == 64
+ 16UL << 30, // 16 GB
+#else
+ 1024UL << 20, // 1 GB
+#endif
+ 100000,
+ // Internal parameters
+ true
+ };
+
+ char const OPTION_ID_SPREAD_THRESHOLD[] = "threshold";
+ char const ALIAS_ID_SPREAD_THRESHOLD[] = "t";
+ char const* USAGE_ID_SPREAD_THRESHOLD[] = { "cache PRIMARY_ALIGNMENT records with difference between values of ALIGN_ID and MATE_ALIGN_ID >= the value of 'threshold' option", NULL };
+
+ char const OPTION_CURSOR_CACHE_SIZE[] = "cursor-cache";
+ //char const ALIAS_CURSOR_CACHE_SIZE[] = "c";
+ char const* USAGE_CURSOR_CACHE_SIZE[] = { "the size of the read cursor in Megabytes", NULL };
+
+ char const OPTION_MIN_CACHE_COUNT[] = "min-cache-count";
+ //char const ALIAS_MIN_CACHE_COUNT[] = "";
+ char const* USAGE_MIN_CACHE_COUNT[] = { "if the number of primary alignment ids in the src db selected for caching is less than <min-cache-count>, the cache db will not be created at all", NULL };
+
+ ::OptDef Options[] =
+ {
+ { OPTION_ID_SPREAD_THRESHOLD, ALIAS_ID_SPREAD_THRESHOLD, NULL, USAGE_ID_SPREAD_THRESHOLD, 1, true, false },
+ { OPTION_CURSOR_CACHE_SIZE, NULL, NULL, USAGE_CURSOR_CACHE_SIZE, 1, true, false },
+ { OPTION_MIN_CACHE_COUNT, NULL, NULL, USAGE_MIN_CACHE_COUNT, 1, true, false },
+ };
+
+ struct PrimaryAlignmentData
+ {
+ uint64_t prev_key;
+ VDBObjects::CVCursor const* pCursorPA;
+ uint32_t const* pColumnIndex;
+ uint32_t const* pColumnIndexCache;
+ size_t const column_count;
+ VDBObjects::CVCursor* pCursorPACache;
+ //size_t count;
+ //size_t const total_count;
+ KApp::CProgressBar* pProgressBar;
+ };
+
+ //size_t print_percent ( size_t count, size_t total_count )
+ //{
+ // size_t const total_points = 10000;
+ // if ( total_count >= total_points && !(count % (total_count / total_points)) )
+ // {
+ // size_t points = (size_t)(total_points*count/total_count);
+ // std::cout
+ // << (100*points/total_points)
+ // << "."
+ // << ( 1000*points/total_points % 10 )
+ // << ( 10000*points/total_points % 10 )
+ // << "% ("
+ // << count
+ // << "/"
+ // << total_count
+ // << ")"
+ // << std::endl;
+ // return points;
+ // }
+ // else
+ // return 0;
+ //}
+
+ template <typename T>
+ void copy_single_int_field (
+ VDBObjects::CVCursor const& curFrom,
+ VDBObjects::CVCursor& curTo,
+ int64_t row_id,
+ uint32_t column_index_from,
+ uint32_t column_index_to
+ )
+ {
+ T val;
+ curFrom.ReadItems ( row_id, column_index_from, & val, sizeof (T) );
+ curTo.Write ( column_index_to, & val, 1 );
+ }
+
+ void copy_str_field (
+ VDBObjects::CVCursor const& curFrom,
+ VDBObjects::CVCursor& curTo,
+ int64_t row_id,
+ uint32_t column_index_from,
+ uint32_t column_index_to
+ )
+ {
+ char val[4096];
+ uint32_t item_count = curFrom.ReadItems ( row_id, column_index_from, val, sizeof (val) );
+ curTo.Write ( column_index_to, val, item_count );
+ }
+
+
+ rc_t KVectorCallbackPrimaryAlignment ( uint64_t key, bool value, void *user_data )
+ {
+ if ( ::Quitting() )
+ {
+ LOGMSG ( klogWarn, "Interrupted" );
+ return 1;
+ }
+
+ assert ( value );
+ PrimaryAlignmentData* p = (PrimaryAlignmentData*)user_data;
+ int64_t prev_row_id = (int64_t)p->prev_key;
+ int64_t row_id = (int64_t)key;
+ VDBObjects::CVCursor& cur_cache = *p->pCursorPACache;
+
+ p->pProgressBar->Process ( 1, false );
+
+ //++p->count;
+ //print_percent (p->count, p->total_count);
+
+ // Filling gaps between actually cached rows with zero-length records
+ if ( p->prev_key )
+ {
+ if ( row_id - prev_row_id > 1)
+ {
+ cur_cache.OpenRow ();
+ cur_cache.CommitRow ();
+ if (row_id - prev_row_id > 2)
+ cur_cache.RepeatRow ( row_id - prev_row_id - 2 ); // -2 due to the first zero-row has been written in the previous line
+ cur_cache.CloseRow ();
+ }
+ }
+
+ // Caching (copying) actual record from PRIMARY_ALIGNMENT table
+ {
+ // The very first visit - need to set starting row_id
+ if ( p->prev_key == 0 )
+ cur_cache.SetRowId ( row_id );
+
+ VDBObjects::CVCursor const& cur_pa = *p->pCursorPA;
+ uint32_t const* ColIndexPA = p->pColumnIndex;
+ uint32_t const* ColIndexCache = p->pColumnIndexCache;
+
+ cur_cache.OpenRow ();
+
+ // MATE_ALIGN_ID
+ size_t column_index = 0;
+ copy_single_int_field <int64_t> (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // SAM_FLAGS
+ ++ column_index;
+ copy_single_int_field <uint32_t> (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // TEMPLATE_LEN
+ ++ column_index;
+ copy_single_int_field <int32_t> (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // MATE_REF_NAME
+ ++ column_index;
+ copy_str_field (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // MATE_REF_POS
+ ++ column_index;
+ copy_single_int_field <uint32_t> (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // SAM_QUALITY
+ ++ column_index;
+ copy_str_field (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // RD_FILTER
+ ++ column_index;
+ copy_single_int_field <uint8_t> (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ // SPOT_GROUP
+ ++ column_index;
+ copy_str_field (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+
+ if ( g_Params.cache_alignment_count )
+ {
+ // ALIGNMENT_COUNT
+ ++ column_index;
+ copy_single_int_field <uint8_t> (cur_pa, cur_cache, row_id, ColIndexPA[column_index], ColIndexCache[column_index] );
+ }
+
+ cur_cache.CommitRow ();
+ cur_cache.CloseRow ();
+
+ p->prev_key = key;
+ }
+
+ return 0;
+ }
+
+ bool ProcessSequenceRow ( int64_t idRow, VDBObjects::CVCursor const& cursor, KLib::CKVector& vect, uint32_t idxCol )
+ {
+ int64_t buf[3]; // TODO: find out the real type of this array
+ uint32_t items_read_count = cursor.ReadItems ( idRow, idxCol, buf, countof(buf) );
+ if ( items_read_count == 2 )
+ {
+ int64_t id1 = buf[0];
+ int64_t id2 = buf[1];
+ int64_t diff = id1 >= id2 ? id1 - id2 : id2 - id1;
+
+ if (id1 && id2 && diff > g_Params.id_spread_threshold)
+ {
+ vect.SetBool(id1, true);
+ vect.SetBool(id2, true);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ size_t FillKVectorWithAlignIDs (VDBObjects::CVDatabase const& vdb, size_t cache_size, KLib::CKVector& vect )
+ {
+ char const* ColumnNamesSequence[] =
+ {
+ "PRIMARY_ALIGNMENT_ID"
+ };
+ uint32_t ColumnIndexSequence [ countof (ColumnNamesSequence) ];
+
+ VDBObjects::CVTable table = vdb.OpenTable("SEQUENCE");
+
+ VDBObjects::CVCursor cursor = table.CreateCursorRead ( cache_size );
+ cursor.InitColumnIndex (ColumnNamesSequence, ColumnIndexSequence, countof(ColumnNamesSequence), false);
+ cursor.Open();
+
+ int64_t idRow = 0;
+ uint64_t nRowCount = 0;
+ size_t count = 0;
+
+ cursor.GetIdRange (idRow, nRowCount);
+
+ for (; (uint64_t)idRow < nRowCount; ++idRow )
+ {
+ if ( ::Quitting() )
+ {
+ LOGMSG ( klogWarn, "Interrupted" );
+ return 0;
+ }
+
+ if ( ProcessSequenceRow (idRow, cursor, vect, ColumnIndexSequence[0]) )
+ ++count;
+ }
+
+ return count;
+ }
+
+ void CachePrimaryAlignment (VDBObjects::CVDBManager& mgr, VDBObjects::CVDatabase const& vdb, size_t cache_size, KLib::CKVector const& vect, size_t vect_size, KApp::CProgressBar& progress_bar)
+ {
+ // Defining the set of columns to be copied from PRIMARY_ALIGNMENT table
+ // to the new cache table
+ // ALIGNMENT_COUNT is the optional column, it must be at the end of this array
+#define DECLARE_PA_COLUMNS( arrName, column_suffix ) char const* arrName[] =\
+ {\
+ "MATE_ALIGN_ID" column_suffix,\
+ "SAM_FLAGS" column_suffix,\
+ "TEMPLATE_LEN" column_suffix,\
+ "MATE_REF_NAME" column_suffix,\
+ "MATE_REF_POS" column_suffix,\
+ "SAM_QUALITY" column_suffix,\
+ "RD_FILTER" column_suffix,\
+ "SPOT_GROUP" column_suffix,\
+ "ALIGNMENT_COUNT" column_suffix\
+ }
+
+ DECLARE_PA_COLUMNS (ColumnNamesPrimaryAlignment, "");
+ DECLARE_PA_COLUMNS (ColumnNamesPrimaryAlignmentCache, "_CACHE");
+#undef DECLARE_PA_COLUMNS
+
+ uint32_t ColumnIndexPrimaryAlignment [ countof (ColumnNamesPrimaryAlignment) ];
+ uint32_t ColumnIndexPrimaryAlignmentCache [ countof (ColumnNamesPrimaryAlignmentCache) ];
+
+ // Openning cursor to iterate through PRIMARY_ALIGNMENT table
+ VDBObjects::CVTable tablePA = vdb.OpenTable("PRIMARY_ALIGNMENT");
+ VDBObjects::CVCursor cursorPA = tablePA.CreateCursorRead ( cache_size );
+ cursorPA.PermitPostOpenAdd();
+ cursorPA.InitColumnIndex ( ColumnNamesPrimaryAlignment, ColumnIndexPrimaryAlignment, countof(ColumnNamesPrimaryAlignment) - 1, false );
+ cursorPA.Open();
+
+ // Check if we can read ALIGNMENT_COUNT parameter
+ try
+ {
+ cursorPA.InitColumnIndex(
+ ColumnNamesPrimaryAlignment + countof(ColumnNamesPrimaryAlignment) - 1,
+ ColumnIndexPrimaryAlignment + countof(ColumnIndexPrimaryAlignment) - 1,
+ 1, false
+ );
+ }
+ catch (Utils::CErrorMsg const& e)
+ {
+ if (e.getRC() == RC ( rcVDB, rcCursor, rcUpdating, rcColumn, rcNotFound ) )
+ g_Params.cache_alignment_count = false;
+ else
+ throw;
+ }
+
+ // Creating new cache table (with the same name - PRIMARY_ALIGNMENT but in the separate DB file)
+ char const schema_path[] = "align/mate-cache.vschema";
+
+ VDBObjects::CVSchema schema = mgr.MakeSchema ();
+ schema.VSchemaParseFile ( schema_path );
+ char szCacheDBName[256] = "";
+ string_printf (szCacheDBName, countof (szCacheDBName), NULL, "%s", g_Params.dbPathDst );
+ VDBObjects::CVDatabase dbCache = mgr.CreateDB ( schema, "NCBI:align:db:mate_cache #1", kcmParents | kcmInit | kcmMD5, szCacheDBName );
+ VDBObjects::CVTable tableCache = dbCache.CreateTable ( "PRIMARY_ALIGNMENT" );
+
+ VDBObjects::CVCursor cursorCache = tableCache.CreateCursorWrite ( kcmInsert );
+ cursorCache.InitColumnIndex ( ColumnNamesPrimaryAlignmentCache, ColumnIndexPrimaryAlignmentCache, countof (ColumnNamesPrimaryAlignmentCache) - (size_t) (!g_Params.cache_alignment_count), true );
+ cursorCache.Open ();
+
+ //PrimaryAlignmentData data = { 0, &cursorPA, ColumnIndexPrimaryAlignment, ColumnIndexPrimaryAlignmentCache, countof (ColumnNamesPrimaryAlignment), &cursorCache, 0, vect_size };
+ progress_bar.Append (vect_size);
+ PrimaryAlignmentData data =
+ {
+ 0,
+ &cursorPA,
+ ColumnIndexPrimaryAlignment,
+ ColumnIndexPrimaryAlignmentCache,
+ countof (ColumnNamesPrimaryAlignment),
+ &cursorCache,
+ &progress_bar
+ };
+
+ // process each saved primary_alignment_id
+ vect.VisitBool ( KVectorCallbackPrimaryAlignment, & data );
+ cursorCache.Commit ();
+ }
+
+ int create_cache_db_impl()
+ {
+ // Adding 0% mark at the very beginning of the program
+ KApp::CProgressBar progress_bar(1);
+ progress_bar.Process ( 0, true );
+
+ VDBObjects::CVDBManager mgr;
+ mgr.Make();
+
+ VDBObjects::CVDatabase vdb = mgr.OpenDB (g_Params.dbPathSrc);
+
+ // Scan SEQUENCE table to find mate_alignment_ids that have to be cached
+ KLib::CKVector vect;
+ size_t count = FillKVectorWithAlignIDs ( vdb, g_Params.cursor_cache_size, vect );
+
+ if ( count*2 >= g_Params.min_cache_count )
+ {
+ // For each id in vect cache the PRIMARY_ALIGNMENT record
+ CachePrimaryAlignment ( mgr, vdb, g_Params.cursor_cache_size, vect, count*2, progress_bar );
+ }
+ else
+ {
+ if ( ::Quitting() == 0 )
+ {
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "The cache db will not be created because there is not "
+ "enough records to cache: $(COUNT) is found and minimum $(MIN_COUNT) is required. "
+ "The minimum required number can be changed via $(OPTION_MIN_CACHE_COUNT) parameter.",
+ "COUNT=%zu,MIN_COUNT=%zu,OPTION_MIN_CACHE_COUNT=%s",
+ count*2, g_Params.min_cache_count, OPTION_MIN_CACHE_COUNT
+ ));
+ }
+ }
+
+ mgr.Release (); // should not be necessary - destructor should do job
+ return 0;
+ }
+
+ // interpret exception processed by Utils::HandleException:
+ // filter out some errors like invalid db - users don't want to
+ // see such errors as actual errors
+ int InterpretException ( int64_t rcCodeUtil, bool bSilent, char const* szErrDesc )
+ {
+ if ( rcCodeUtil == Utils::rcUnknown ||
+ rcCodeUtil == Utils::rcErrorStdExc ||
+ rcCodeUtil == Utils::rcInvalid)
+ {
+ if ( ! bSilent )
+ LOGMSG ( klogErr, szErrDesc );
+
+ return 3;
+ }
+ else if (rcCodeUtil == SILENT_RC( rcDB,rcMgr,rcOpening,rcDatabase,rcIncorrect ) ||
+ rcCodeUtil == SILENT_RC( rcVFS,rcTree,rcResolving,rcPath,rcNotFound ))
+ {
+ if ( ! bSilent )
+ LOGMSG ( klogWarn, szErrDesc );
+ return 0;
+ }
+ else
+ {
+ if ( ! bSilent )
+ LOGMSG ( klogErr, szErrDesc );
+ return 3;
+ }
+ }
+
+ int create_cache_db_impl_safe ()
+ {
+ try
+ {
+ return create_cache_db_impl ();
+ }
+ catch (...)
+ {
+ char szErrDesc [512];
+ int64_t rc = Utils::HandleException ( true, szErrDesc, countof(szErrDesc) );
+ return InterpretException ( rc, false, szErrDesc );
+ }
+ }
+
+ int create_cache_db (int argc, char** argv)
+ {
+ try
+ {
+ KApp::CArgs args (argc, argv, Options, countof (Options), ::XMLLogger_Args, ::XMLLogger_ArgsQty);
+ KApp::CXMLLogger xml_logger ( args );
+ uint32_t param_count = args.GetParamCount ();
+ if ( param_count != 2 )
+ {
+ MiniUsage (args.GetArgs());
+ return 0;
+ }
+
+ g_Params.dbPathSrc = args.GetParamValue (0);
+ g_Params.dbPathDst = args.GetParamValue (1);
+
+ if (args.GetOptionCount (OPTION_ID_SPREAD_THRESHOLD))
+ g_Params.id_spread_threshold = args.GetOptionValueUInt<int64_t> ( OPTION_ID_SPREAD_THRESHOLD, 0 );
+
+ if (args.GetOptionCount (OPTION_CURSOR_CACHE_SIZE))
+ g_Params.cursor_cache_size = 1024*1024 * args.GetOptionValueUInt<size_t> ( OPTION_CURSOR_CACHE_SIZE, 0 );
+
+ if (args.GetOptionCount (OPTION_MIN_CACHE_COUNT))
+ g_Params.min_cache_count = args.GetOptionValueUInt <size_t> ( OPTION_MIN_CACHE_COUNT, 0 );
+
+ return create_cache_db_impl_safe ();
+ }
+ catch (...) // here we handle only exceptions in CArgs or CXMLLogger
+ {
+ return Utils::HandleException ( false, NULL, 0 );
+ }
+ }
+}
+
+
+extern "C"
+{
+ const char UsageDefaultName[] = "align-cache";
+ ver_t CC KAppVersion()
+ {
+ return ALIGN_CACHE_VERS;
+ }
+ rc_t CC UsageSummary (const char * progname)
+ {
+ printf (
+ "Usage:\n"
+ " %s [options] <src-db-path> <new-cache-db-path>\n"
+ "\n"
+ "Summary:\n"
+ " Create a cache file for given database <src-db-path>\n"
+ " PRIMARY_ALIGNMENT table and save it into <new-cache-db-path>\n"
+ "\n", progname);
+ return 0;
+ }
+ const char* param_usage_src[] = { "Path to the database", NULL };
+ const char* param_usage_dst[] = { "Path to the new cache database to be created", NULL };
+ rc_t CC Usage (::Args const* args)
+ {
+ rc_t rc = 0;
+ const char* progname = UsageDefaultName;
+ const char* fullpath = UsageDefaultName;
+
+ if (args == NULL)
+ rc = RC(rcExe, rcArgv, rcAccessing, rcSelf, rcNull);
+ else
+ rc = ArgsProgram(args, &fullpath, &progname);
+
+ UsageSummary (progname);
+
+ printf("Parameters:\n");
+
+ HelpParamLine ("src-db-path", param_usage_src);
+ HelpParamLine ("new-cache-db-path", param_usage_dst);
+
+ printf ("\nOptions:\n");
+
+ HelpOptionLine (AlignCache::ALIAS_ID_SPREAD_THRESHOLD, AlignCache::OPTION_ID_SPREAD_THRESHOLD, "value", AlignCache::USAGE_ID_SPREAD_THRESHOLD);
+ HelpOptionLine (NULL, AlignCache::OPTION_CURSOR_CACHE_SIZE, "value in MB", AlignCache::USAGE_CURSOR_CACHE_SIZE);
+ HelpOptionLine (NULL, AlignCache::OPTION_MIN_CACHE_COUNT, "count", AlignCache::USAGE_MIN_CACHE_COUNT);
+ XMLLogger_Usage();
+
+ printf ("\n");
+
+ HelpOptionsStandard ();
+
+ HelpVersion (fullpath, KAppVersion());
+
+ return rc;
+ }
+
+ rc_t CC KMain(int argc, char* argv[])
+ {
+ return AlignCache::create_cache_db (argc, argv);
+ }
+}
diff --git a/tools/align-cache/align-cache.vers b/tools/align-cache/align-cache.vers
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/tools/align-cache/align-cache.vers
@@ -0,0 +1 @@
+1.0.0
diff --git a/tools/align-cache/align-cache.vers.h b/tools/align-cache/align-cache.vers.h
new file mode 100644
index 0000000..cc5ecea
--- /dev/null
+++ b/tools/align-cache/align-cache.vers.h
@@ -0,0 +1 @@
+#define ALIGN_CACHE_VERS 0x01000000
diff --git a/tools/align-cache/helper.cpp b/tools/align-cache/helper.cpp
new file mode 100644
index 0000000..1b02c48
--- /dev/null
+++ b/tools/align-cache/helper.cpp
@@ -0,0 +1,923 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+// helper.cpp
+
+#include "helper.h"
+
+#include <algorithm>
+#include <stdio.h>
+#include <iostream>
+
+#include <vdb/vdb-priv.h>
+#include <klib/rc.h>
+
+#ifdef _WIN32
+#pragma warning (disable:4503)
+#endif
+
+// TODO: remove printfs
+namespace KLib
+{
+ CKVector::CKVector() : m_pSelf(NULL)
+ {
+ Make();
+ }
+
+ CKVector::~CKVector()
+ {
+ Release();
+ }
+
+ void CKVector::Make()
+ {
+ if (m_pSelf)
+ throw Utils::CErrorMsg (0, "Duplicated call to KVectorMake");
+
+ rc_t rc = ::KVectorMake(&m_pSelf);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorMake");
+#if DEBUG_PRINT != 0
+ printf("Created KVector %p\n", m_pSelf);
+#endif
+ }
+
+ void CKVector::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing KVector %p\n", m_pSelf);
+#endif
+ ::KVectorRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ size_t const RECORD_SIZE_IN_BITS = 2;
+ uint64_t const BIT_SET_MASK = 0x2;
+ uint64_t const BIT_VALUE_MASK = 0x1;
+ uint64_t const BIT_RECORD_MASK = BIT_SET_MASK | BIT_VALUE_MASK;
+
+ void CKVector::SetBool(uint64_t key, bool value)
+ {
+#if USING_UINT64_BITMAP == 1
+ uint64_t stored_bits = 0;
+ uint64_t key_qword = key / 64;
+ uint64_t key_bit = key % 64;
+ rc_t rc = ::KVectorGetU64 ( m_pSelf, key_qword, &stored_bits );
+ bool first_time = rc == RC ( rcCont, rcVector, rcAccessing, rcItem, rcNotFound ); // 0x1e615458
+ if ( !first_time && rc )
+ throw Utils::CErrorMsg(rc, "KVectorGetU64");
+
+ uint64_t new_bit = (uint64_t)value << key_bit;
+ uint64_t stored_bit = (uint64_t)1 << key_bit & stored_bits;
+
+ if ( first_time || new_bit != stored_bit )
+ {
+ if ( new_bit )
+ stored_bits |= new_bit;
+ else
+ stored_bits &= ~new_bit;
+
+ rc_t rc = ::KVectorSetU64 ( m_pSelf, key_qword, stored_bits );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorSetU64");
+ }
+#elif USING_UINT64_BITMAP == 2
+ uint64_t stored_bits = 0;
+ uint64_t key_qword = key / (sizeof(stored_bits) * 8 / RECORD_SIZE_IN_BITS);
+ uint64_t bit_offset_in_qword = (key % (sizeof(stored_bits) * 8 / RECORD_SIZE_IN_BITS)) * RECORD_SIZE_IN_BITS;
+ rc_t rc = ::KVectorGetU64 ( m_pSelf, key_qword, &stored_bits );
+ bool first_time = rc == RC ( rcCont, rcVector, rcAccessing, rcItem, rcNotFound ); // 0x1e615458;
+ if ( !first_time && rc )
+ throw Utils::CErrorMsg(rc, "KVectorGetU64");
+
+ uint64_t new_bit_record = (BIT_SET_MASK | (uint64_t)value) << bit_offset_in_qword;
+ uint64_t stored_bit_record = (uint64_t)BIT_RECORD_MASK << bit_offset_in_qword & stored_bits;
+
+ if ( first_time || new_bit_record != stored_bit_record )
+ {
+ stored_bits &= ~((uint64_t)BIT_RECORD_MASK << bit_offset_in_qword); // clear stored record to assign a new value by bitwise OR
+ stored_bits |= new_bit_record;
+
+ rc_t rc = ::KVectorSetU64 ( m_pSelf, key_qword, stored_bits );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorSetU64");
+ }
+#else
+
+ rc_t rc = ::KVectorSetBool ( m_pSelf, key, value );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorSetBool");
+#endif
+ }
+
+ struct UserDataU64toBool
+ {
+ rc_t ( * f ) ( uint64_t key, bool value, void *user_data );
+ void* user_data;
+ };
+#if USING_UINT64_BITMAP == 1
+ rc_t VisitU64toBoolAdapter ( uint64_t key, uint64_t value, void *user_data )
+ {
+ rc_t ( * bool_callback ) ( uint64_t key, bool value, void *user_data );
+ bool_callback = ((UserDataU64toBool*) user_data) -> f;
+ void* original_user_data = ((UserDataU64toBool*) user_data) -> user_data;
+
+ rc_t rc = 0;
+ for ( size_t i = 0; i < sizeof (value) * 8; ++i )
+ {
+ rc = bool_callback ( key * 64 + i, (bool) ((uint64_t)1 << i & value), original_user_data );
+ if ( rc )
+ return rc;
+ }
+ return rc;
+ }
+#elif USING_UINT64_BITMAP == 2
+ rc_t VisitU64toBoolAdapter ( uint64_t key, uint64_t value, void *user_data )
+ {
+ rc_t ( * bool_callback ) ( uint64_t key, bool value, void *user_data );
+ bool_callback = ((UserDataU64toBool*) user_data) -> f;
+ void* original_user_data = ((UserDataU64toBool*) user_data) -> user_data;
+
+ rc_t rc = 0;
+ for ( size_t i = 0; i < sizeof (value) * 8 / RECORD_SIZE_IN_BITS; ++i )
+ {
+ uint64_t key_bool = key * sizeof(value) * 8 / RECORD_SIZE_IN_BITS + i;
+ uint64_t record = value >> i * RECORD_SIZE_IN_BITS & BIT_RECORD_MASK;
+ if ( record & BIT_SET_MASK )
+ {
+ rc = bool_callback ( key_bool, (bool) (record & BIT_VALUE_MASK), original_user_data );
+ if ( rc )
+ return rc;
+ }
+ }
+ return rc;
+ }
+#endif
+
+ void CKVector::VisitBool(rc_t ( * f ) ( uint64_t key, bool value, void *user_data ), void *user_data) const
+ {
+#if USING_UINT64_BITMAP == 1
+ UserDataU64toBool user_data_adapter = { f, user_data };
+ ::KVectorVisitU64 ( m_pSelf, false, VisitU64toBoolAdapter, &user_data_adapter );
+#elif USING_UINT64_BITMAP == 2
+ UserDataU64toBool user_data_adapter = { f, user_data };
+ ::KVectorVisitU64 ( m_pSelf, false, VisitU64toBoolAdapter, &user_data_adapter );
+#else
+ ::KVectorVisitBool ( m_pSelf, false, f, user_data );
+#endif
+ }
+}
+
+///////////////////////////////////////////////////////////////
+
+namespace VDBObjects
+{
+ CVCursor::CVCursor() : m_pSelf(NULL)
+ {}
+
+ CVCursor::~CVCursor()
+ {
+ Release();
+ }
+
+ CVCursor::CVCursor(CVCursor const& x)
+ {
+ Clone(x);
+ }
+
+ CVCursor& CVCursor::operator=(CVCursor const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVCursor::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VCursor %p\n", m_pSelf);
+#endif
+ ::VCursorRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVCursor::Clone(CVCursor const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VCursorAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VCursor %p\n", m_pSelf);
+#endif
+ }
+
+ void CVCursor::Open() const
+ {
+ rc_t rc = ::VCursorOpen(m_pSelf);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorOpen");
+ }
+
+ void CVCursor::PermitPostOpenAdd() const
+ {
+ rc_t rc = ::VCursorPermitPostOpenAdd ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorPermitPostOpenAdd");
+ }
+
+ void CVCursor::InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount, bool set_default)
+ {
+ for (size_t i = 0; i < nCount; ++i)
+ {
+ rc_t rc = ::VCursorAddColumn(m_pSelf, & pColumnIndex[i], ColumnNames[i] );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorAddColumn - [%s]", ColumnNames[i]);
+
+ if ( set_default )
+ {
+ VTypedecl type;
+ VTypedesc desc;
+ uint32_t idx = pColumnIndex[i];
+
+ rc = ::VCursorDatatype ( m_pSelf, idx, & type, & desc );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorDatatype (column idx=%u [%s])", idx, ColumnNames[i]);
+
+ uint32_t elem_bits = ::VTypedescSizeof ( & desc );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTypedescSizeof (column idx=%u [%s])", idx, ColumnNames[i]);
+ rc = ::VCursorDefault ( m_pSelf, idx, elem_bits, "", 0, 0 );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorDefault (column idx=%u [%s])", idx, ColumnNames[i]);
+ }
+ }
+ }
+
+ void CVCursor::GetIdRange(int64_t& idFirstRow, uint64_t& nRowCount) const
+ {
+ rc_t rc = ::VCursorIdRange(m_pSelf, 0, &idFirstRow, &nRowCount);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorIdRange");
+ }
+
+ int64_t CVCursor::GetRowId () const
+ {
+ int64_t row_id;
+ rc_t rc = ::VCursorRowId ( m_pSelf, & row_id );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorRowId");
+
+ return row_id;
+ }
+
+ void CVCursor::SetRowId (int64_t row_id) const
+ {
+ rc_t rc = ::VCursorSetRowId ( m_pSelf, row_id );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorSetRowId (%ld)", row_id);
+ }
+
+ void CVCursor::OpenRow () const
+ {
+ rc_t rc = ::VCursorOpenRow ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorOpenRow");
+ }
+
+ void CVCursor::CommitRow ()
+ {
+ rc_t rc = ::VCursorCommitRow ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCommitRow");
+ }
+
+ void CVCursor::RepeatRow ( uint64_t count )
+ {
+ rc_t rc = ::VCursorRepeatRow ( m_pSelf, count );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorRepeatRow (%lu)", count);
+ }
+
+ void CVCursor::CloseRow () const
+ {
+ rc_t rc = ::VCursorCloseRow ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCloseRow");
+ }
+
+ void CVCursor::Commit ()
+ {
+ rc_t rc = ::VCursorCommit ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCommit");
+ }
+
+///////////////////////////////////////////////////////////////////////////////////
+
+ CVTable::CVTable() : m_pSelf(NULL)
+ {
+ }
+
+ CVTable::~CVTable()
+ {
+ Release();
+ }
+
+ CVTable::CVTable(CVTable const& x)
+ {
+ Clone(x);
+ }
+
+ CVTable& CVTable::operator=(CVTable const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVTable::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VTable %p\n", m_pSelf);
+#endif
+ ::VTableRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVTable::Clone(CVTable const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VTableAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VTable %p\n", m_pSelf);
+#endif
+ }
+
+ CVCursor CVTable::CreateCursorRead ( size_t cache_size ) const
+ {
+ CVCursor cursor;
+ rc_t rc = ::VTableCreateCachedCursorRead(m_pSelf, const_cast<VCursor const**>(& cursor.m_pSelf), cache_size);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTableCreateCachedCursorRead (%zu)", cache_size);
+
+#if DEBUG_PRINT != 0
+ printf("Created cursor (rd) %p\n", cursor.m_pSelf);
+#endif
+ return cursor;
+ }
+
+ CVCursor CVTable::CreateCursorWrite (::KCreateMode mode)
+ {
+ CVCursor cursor;
+ rc_t rc = ::VTableCreateCursorWrite ( m_pSelf, & cursor.m_pSelf, mode );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTableCreateCursorWrite");
+
+#if DEBUG_PRINT != 0
+ printf("Created cursor (wr) %p\n", cursor.m_pSelf);
+#endif
+ return cursor;
+ }
+
+//////////////////////////////////////////////////////////////////////
+
+ CVDatabase::CVDatabase() : m_pSelf(NULL)
+ {}
+
+ CVDatabase::~CVDatabase()
+ {
+ Release();
+ }
+
+ CVDatabase::CVDatabase(CVDatabase const& x)
+ {
+ Clone(x);
+ }
+
+ CVDatabase& CVDatabase::operator=(CVDatabase const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVDatabase::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VDatabase %p\n", m_pSelf);
+#endif
+ ::VDatabaseRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVDatabase::Clone(CVDatabase const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VDatabaseAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VDatabase %p\n", m_pSelf);
+#endif
+ }
+
+ CVTable CVDatabase::OpenTable(char const* pszTableName) const
+ {
+ CVTable table;
+ rc_t rc = ::VDatabaseOpenTableRead(m_pSelf, const_cast<VTable const**>(& table.m_pSelf), pszTableName);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDatabaseOpenTableRead (%s)", pszTableName);
+
+#if DEBUG_PRINT != 0
+ printf("Opened table %p (%s)\n", table.m_pSelf, pszTableName);
+#endif
+ return table;
+ }
+
+ CVTable CVDatabase::CreateTable ( char const* pszTableName )
+ {
+ CVTable table;
+ //rc_t rc = ::VDatabaseCreateTableDefault ( m_pSelf, & table.m_pSelf, pszTableName, pszTableName );
+ rc_t rc = ::VDatabaseCreateTableByMask ( m_pSelf, & table.m_pSelf, pszTableName, 0, 0, pszTableName );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDatabaseCreateTableDefault (%s)", pszTableName);
+
+#if DEBUG_PRINT != 0
+ printf("Created table %p (%s)\n", table.m_pSelf, pszTableName);
+#endif
+ return table;
+ }
+
+ void CVDatabase::ColumnCreateParams ( ::KCreateMode cmode, ::KChecksum checksum, size_t pgsize )
+ {
+ rc_t rc = ::VDatabaseColumnCreateParams ( m_pSelf, cmode, checksum, pgsize );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDatabaseColumnCreateParams");
+ }
+
+//////////////////////////////////////////////////////////////////////
+
+ CVSchema::CVSchema() : m_pSelf (NULL)
+ {
+ }
+ CVSchema::~CVSchema()
+ {
+ Release();
+ }
+
+ CVSchema::CVSchema(CVSchema const& x)
+ {
+ Clone (x);
+ }
+
+ CVSchema& CVSchema::operator=(CVSchema const& x)
+ {
+ Clone (x);
+ return *this;
+ }
+
+ void CVSchema::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VSchema %p\n", m_pSelf);
+#endif
+ ::VSchemaRelease ( m_pSelf );
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVSchema::Clone ( CVSchema const& x )
+ {
+ if (false && m_pSelf)
+ {
+ assert(0);
+ Release();
+ }
+ m_pSelf = x.m_pSelf;
+ ::VSchemaAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VSchema %p\n", m_pSelf);
+#endif
+ }
+
+ void CVSchema::VSchemaParseFile ( char const* pszFilePath )
+ {
+ rc_t rc = ::VSchemaParseFile ( m_pSelf, pszFilePath );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VSchemaParseFile (%s)", pszFilePath);
+ }
+
+//////////////////////////////////////////////////////////////////////
+
+ CVDBManager::CVDBManager() : m_pSelf(NULL)
+ {}
+
+ CVDBManager::~CVDBManager()
+ {
+ Release();
+ }
+
+ void CVDBManager::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VDBManager %p\n", m_pSelf);
+#endif
+ ::VDBManagerRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+#if MANAGER_WRITABLE != 0
+ void CVDBManager::Make()
+ {
+ assert(m_pSelf == NULL);
+ if (m_pSelf)
+ throw Utils::CErrorMsg(0, "Double call to VDBManagerMakeUpdate");
+
+ rc_t rc = ::VDBManagerMakeUpdate ( & m_pSelf, NULL );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerMakeUpdate");
+
+ /*rc = VDBManagerDisablePagemapThread ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerDisablePagemapThread");*/
+
+#if DEBUG_PRINT != 0
+ printf("Created VDBManager (wr) %p\n", m_pSelf);
+#endif
+ }
+#else
+ void CVDBManager::Make()
+ {
+ assert(m_pSelf == NULL);
+ if (m_pSelf)
+ throw Utils::CErrorMsg(0, "Double call to VDBManagerMakeRead");
+
+ rc_t rc = ::VDBManagerMakeRead(const_cast<VDBManager const**>(&m_pSelf), NULL);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerMakeRead");
+
+#if DEBUG_PRINT != 0
+ printf("Created VDBManager (rd) %p\n", m_pSelf);
+#endif
+ }
+#endif
+
+ CVDatabase CVDBManager::OpenDB(char const* pszDBName) const
+ {
+ CVDatabase vdb;
+ rc_t rc = ::VDBManagerOpenDBRead(m_pSelf, const_cast<VDatabase const**>(& vdb.m_pSelf), NULL, pszDBName);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerOpenDBRead (%s)", pszDBName);
+
+#if DEBUG_PRINT != 0
+ printf("Opened database %p (%s)\n", vdb.m_pSelf, pszDBName);
+#endif
+ return vdb;
+ }
+#if MANAGER_WRITABLE != 0
+ CVDatabase CVDBManager::CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath )
+ {
+ CVDatabase vdb;
+ rc_t rc = ::VDBManagerCreateDB ( m_pSelf, & vdb.m_pSelf, schema.m_pSelf, pszTypeDesc, cmode, pszPath );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerCreateDB (%s)", pszPath);
+
+#if DEBUG_PRINT != 0
+ printf("Created database %p (%s)\n", vdb.m_pSelf, pszPath);
+#endif
+ // set creation mode of objects ( tables, columns, etc. ) to
+ // create new or re-initialize existing, plus attach md5 checksums
+ // to all files.
+ // set blob creation mode to record 32-bit CRC within blob
+ // continue to use default page size...
+ vdb.ColumnCreateParams ( kcmInit | kcmMD5, kcsCRC32, 0 );
+ return vdb;
+ }
+#endif
+
+ CVSchema CVDBManager::MakeSchema () const
+ {
+ CVSchema schema;
+ rc_t rc = ::VDBManagerMakeSchema ( m_pSelf, & schema.m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerMakeSchema");
+
+#if DEBUG_PRINT != 0
+ printf("Created Schema %p\n", schema.m_pSelf);
+#endif
+ return schema;
+ }
+}
+
+namespace KApp
+{
+ CArgs::CArgs (int argc, char** argv, ::OptDef const* pOptions, size_t option_count)
+ : m_pSelf(NULL)
+ {
+ MakeAndHandle ( argc, argv, pOptions, option_count );
+ }
+
+ CArgs::CArgs (int argc, char** argv, ::OptDef const* pOptions1, size_t option_count1, ::OptDef const* pOptions2, size_t option_count2)
+ : m_pSelf(NULL)
+ {
+ MakeAndHandle ( argc, argv, pOptions1, option_count1, pOptions2, option_count2 );
+ }
+
+ CArgs::~CArgs()
+ {
+ Release();
+ }
+
+ void CArgs::MakeAndHandle (int argc, char** argv, ::OptDef const* pOptions, size_t option_count)
+ {
+ if (m_pSelf)
+ throw Utils::CErrorMsg (0, "Duplicated call to ArgsMakeAndHandle");
+
+ rc_t rc = ::ArgsMakeAndHandle (&m_pSelf, argc, argv, 1, pOptions, option_count);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsMakeAndHandle");
+#if DEBUG_PRINT != 0
+ printf("Created Args %p\n", m_pSelf);
+#endif
+ }
+ void CArgs::MakeAndHandle ( int argc, char** argv, ::OptDef const* pOptions1, size_t option_count1, ::OptDef const* pOptions2, size_t option_count2 )
+ {
+ if (m_pSelf)
+ throw Utils::CErrorMsg (0, "Duplicated call to ArgsMakeAndHandle");
+
+ rc_t rc = ::ArgsMakeAndHandle (&m_pSelf, argc, argv, 2, pOptions1, option_count1, pOptions2, option_count2);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsMakeAndHandle(2)");
+#if DEBUG_PRINT != 0
+ printf("Created Args(2) %p\n", m_pSelf);
+#endif
+ }
+
+ void CArgs::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing Args %p\n", m_pSelf);
+#endif
+ ::ArgsRelease (m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ ::Args const* CArgs::GetArgs () const
+ {
+ return m_pSelf;
+ }
+
+ uint32_t CArgs::GetParamCount () const
+ {
+ uint32_t ret = 0;
+ rc_t rc = ::ArgsParamCount ( m_pSelf, &ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsParamCount");
+
+ return ret;
+ }
+
+ char const* CArgs::GetParamValue ( uint32_t iteration ) const
+ {
+ void const* ret = NULL;
+ rc_t rc = ::ArgsParamValue ( m_pSelf, iteration, & ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsParamValue");
+
+ return static_cast <char const*> (ret);
+ }
+
+ uint32_t CArgs::GetOptionCount ( char const* option_name ) const
+ {
+ uint32_t ret = 0;
+ rc_t rc = ::ArgsOptionCount ( m_pSelf, option_name, &ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsOptionCount (%s)", option_name);
+
+ return ret;
+ }
+
+ char const* CArgs::GetOptionValue ( char const* option_name, uint32_t iteration ) const
+ {
+ void const* ret = NULL;
+ rc_t rc = ::ArgsOptionValue ( m_pSelf, option_name, iteration, & ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsOptionValue (%s)", option_name);
+
+ return static_cast <char const*> (ret);
+ }
+
+////////////////////////////////
+
+ CProgressBar::CProgressBar ( uint64_t size )
+ {
+ Make ( size );
+ }
+
+ CProgressBar::~CProgressBar ()
+ {
+ Release ();
+ }
+
+ void CProgressBar::Make ( uint64_t size )
+ {
+ rc_t rc = ::KLoadProgressbar_Make ( &m_pSelf, size );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLoadProgressbar_Make");
+#if DEBUG_PRINT != 0
+ printf ( "Created ProgressBar %p\n", m_pSelf );
+#endif
+ }
+ void CProgressBar::Release ()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing ProgressBar %p\n", m_pSelf);
+#endif
+ ::KLoadProgressbar_Release ( m_pSelf, true );
+ m_pSelf = NULL;
+ }
+ }
+
+ void CProgressBar::Append ( uint64_t chunk )
+ {
+ rc_t rc = ::KLoadProgressbar_Append ( m_pSelf, chunk );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLoadProgressbar_Append");
+ }
+
+ void CProgressBar::Process ( uint64_t chunk, bool force_report )
+ {
+ rc_t rc = ::KLoadProgressbar_Process ( m_pSelf, chunk, force_report );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLoadProgressbar_Process");
+ }
+
+///////////////////////////////////////////
+ CXMLLogger::CXMLLogger ( CArgs const& args )
+ {
+ Make ( args );
+ }
+
+ CXMLLogger::~CXMLLogger ()
+ {
+ Release ();
+ }
+
+ void CXMLLogger::Make ( CArgs const& args )
+ {
+ rc_t rc = ::XMLLogger_Make ( &m_pSelf, NULL, args.m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "XMLLogger_Make");
+#if DEBUG_PRINT != 0
+ printf ( "Created XMLLogger %p\n", m_pSelf );
+#endif
+ }
+
+ void CXMLLogger::Release ()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing XMLLogger %p\n", m_pSelf);
+#endif
+ ::XMLLogger_Release ( m_pSelf );
+ m_pSelf = NULL;
+ }
+ }
+}
+
+namespace Utils
+{
+ CErrorMsg::CErrorMsg(rc_t rc, char const* fmt_str, ...)
+ : m_rc(rc)
+ {
+ va_list args;
+ va_start(args, fmt_str);
+ string_vprintf (m_szDescr, countof(m_szDescr), NULL, fmt_str, args);
+ va_end(args);
+ }
+
+ rc_t CErrorMsg::getRC() const
+ {
+ return m_rc;
+ }
+ char const* CErrorMsg::what() const throw()
+ {
+ return m_szDescr;
+ }
+
+ int64_t HandleException ( bool bSilent, char* pErrDesc, size_t sizeErrDesc )
+ {
+ try
+ {
+ throw;
+ }
+ catch (Utils::CErrorMsg const& e)
+ {
+ char szBufErr[512];
+ if ( pErrDesc == NULL )
+ {
+ pErrDesc = szBufErr;
+ sizeErrDesc = countof(szBufErr);
+ }
+ size_t rc = e.getRC();
+ rc_t res;
+ if (rc != 0)
+ res = string_printf(pErrDesc, sizeErrDesc, NULL, "%s failed with code 0x%08x (%u) [%R]", e.what(), rc, rc, rc);
+ else
+ res = string_printf(pErrDesc, sizeErrDesc, NULL, "%s", e.what());
+ if (res == rcBuffer || res == rcInsufficient)
+ pErrDesc [sizeErrDesc - 1] = '\0';
+
+ if ( ! bSilent )
+ LOGMSG ( klogErr, pErrDesc );
+
+ return rc;
+ }
+ catch (std::exception const& e)
+ {
+ char szBufErr[512];
+ if ( pErrDesc == NULL )
+ {
+ pErrDesc = szBufErr;
+ sizeErrDesc = countof(szBufErr);
+ }
+ rc_t res = string_printf(pErrDesc, sizeErrDesc, NULL, "std::exception: %s", e.what());
+ if (res == rcBuffer || res == rcInsufficient)
+ pErrDesc [sizeErrDesc - 1] = '\0';
+
+ if ( ! bSilent )
+ LOGMSG ( klogErr, pErrDesc );
+
+ return Utils::rcErrorStdExc;
+ }
+ catch (...)
+ {
+ char szBufErr[512];
+ if ( pErrDesc == NULL )
+ {
+ pErrDesc = szBufErr;
+ sizeErrDesc = countof(szBufErr);
+ }
+ rc_t res = string_printf(pErrDesc, sizeErrDesc, NULL, "Unexpected exception occured");
+ if (res == rcBuffer || res == rcInsufficient)
+ pErrDesc [sizeErrDesc - 1] = '\0';
+
+ if ( ! bSilent )
+ LOGMSG ( klogErr, pErrDesc );
+
+ return Utils::rcUnknown;
+ }
+
+ assert ( false );
+ return Utils::rcInvalid; // this shall never be reached
+ }
+}
diff --git a/tools/align-cache/helper.h b/tools/align-cache/helper.h
new file mode 100644
index 0000000..1fe213f
--- /dev/null
+++ b/tools/align-cache/helper.h
@@ -0,0 +1,390 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+// helper.h
+#include <exception>
+#include <string.h>
+
+#include <stdint.h>
+
+#include <vdb/manager.h>
+#include <vdb/schema.h>
+#include <vdb/database.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+#include <klib/printf.h>
+#include <klib/vector.h>
+#include <kapp/args.h>
+#include <kapp/progressbar.h>
+#include <kapp/log-xml.h>
+
+
+#ifndef countof
+#define countof(arr) (sizeof(arr)/sizeof(arr[0]))
+#endif
+
+#ifdef _WIN32
+#pragma warning (disable:4503)
+#endif
+
+#define USING_UINT64_BITMAP 0
+#define MANAGER_WRITABLE 1
+#define DEBUG_PRINT 0
+
+namespace KLib
+{
+ class CKVector
+ {
+ public:
+ CKVector();
+ CKVector (CKVector const& x);
+ CKVector& operator= (CKVector const& x);
+ ~CKVector();
+
+ void SetBool (uint64_t key, bool value);
+ void VisitBool (rc_t ( * f ) ( uint64_t key, bool value, void *user_data ), void *user_data) const;
+
+ private:
+ void Make();
+ void Release();
+
+ ::KVector* m_pSelf;
+ };
+}
+
+namespace Utils
+{
+ class CErrorMsg : public std::exception
+ {
+ public:
+ CErrorMsg(rc_t rc, char const* fmt_str, ...);
+
+ rc_t getRC() const;
+ virtual char const* what() const throw();
+
+ private:
+ char m_szDescr[256];
+ rc_t m_rc;
+ };
+
+ template <typename T> T atoi_t ( char const* str_val )
+ {
+ if ( str_val [0] == '\0' )
+ throw Utils::CErrorMsg(0, "atoi_t: invalid input string (empty)");
+
+ size_t i = 0;
+ char sign = '+';
+ if ( str_val[0] == '-' || str_val[0] == '+' )
+ {
+ ++i;
+ sign = str_val[0];
+ }
+
+ T ret = 0;
+ for (; str_val[i] != '\0'; ++i )
+ {
+ if ( str_val[i] < '0' || str_val[i] > '9' )
+ throw Utils::CErrorMsg(0, "atoi_t: invalid input string \"%s\" (invalid character: '%c' at pos=%zu)", str_val, str_val[i], i+1);
+ ret = ret*10 + str_val[i] - '0';
+ }
+
+ return sign == '-' ? -ret : ret;
+ }
+
+ template <typename T> T atou_t ( char const* str_val )
+ {
+ if ( str_val [0] == '\0' )
+ throw Utils::CErrorMsg(0, "atou_t: invalid input string (empty)");
+
+ T ret = 0;
+ for ( size_t i = 0; str_val[i] != '\0'; ++i )
+ {
+ if ( str_val[i] < '0' || str_val[i] > '9' )
+ throw Utils::CErrorMsg(0, "atoi_t: invalid input string \"%s\" (invalid character: '%c' at pos=%zu)", str_val, str_val[i], i+1);
+ ret = ret*10 + str_val[i] - '0';
+ }
+
+ return ret;
+ }
+
+ enum ErrorHandlerCode
+ {
+ rcUnknown = -2,
+ rcErrorStdExc = -1,
+ rcInvalid = -99
+ // value > 0 means rc_t returned from a VDB-function
+ // zero shall not be returned
+ };
+
+ // This function must be called inside catch block only
+ // if bSilent == true then produce no output, only return ErrorHandlerCode
+ // pErrDesc and sizeErrDesc - the buffer to write error description to (NULL - OK)
+ int64_t HandleException ( bool bSilent, char* pErrDesc, size_t sizeErrDesc );
+}
+
+namespace VDBObjects
+{
+ /* functor to remove trailing '\n' from char reads */
+ template<typename T> class CPostReadAction
+ {
+ T* m_pBuf;
+ uint32_t m_nCount;
+ public:
+ CPostReadAction(T* pBuf, uint32_t nCount) : m_pBuf(pBuf), m_nCount(nCount) {}
+ void operator()() const;
+ };
+ template<typename T> inline void CPostReadAction<T>::operator()() const {}
+ template<> inline void CPostReadAction<char>::operator()() const { m_pBuf[m_nCount] = '\0'; }
+ template<> inline void CPostReadAction<unsigned char>::operator()() const { m_pBuf[m_nCount] = '\0'; }
+
+ class CVCursor;
+ class CVTable;
+ class CVDatabase;
+ class CVSchema;
+
+/////////////////////////////////////////////////////////////////////////////////
+
+ class CVDBManager
+ {
+ public:
+ CVDBManager();
+ ~CVDBManager();
+ CVDBManager(CVDBManager const& x);
+ CVDBManager& operator=(CVDBManager const& x);
+
+ void Make();
+ void Release();
+ CVDatabase OpenDB ( char const* pszDBName ) const;
+ CVDatabase CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath );
+ CVSchema MakeSchema () const;
+
+ private:
+ ::VDBManager* m_pSelf;
+ };
+
+/////////////////////////////////////////////////////////////////////////////////
+
+ class CVSchema
+ {
+ public:
+ friend CVSchema CVDBManager::MakeSchema () const;
+ friend CVDatabase CVDBManager::CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath );
+
+ CVSchema();
+ ~CVSchema();
+ CVSchema(CVSchema const& x);
+ CVSchema& operator=(CVSchema const& x);
+
+ void Make();
+ void Release();
+ void VSchemaParseFile(char const* pszFilePath);
+
+ private:
+ void Clone ( CVSchema const& x );
+ ::VSchema* m_pSelf;
+ };
+
+/////////////////////////////////////////////////////////////////////////////////
+
+ class CVDatabase
+ {
+ public:
+ friend CVDatabase CVDBManager::OpenDB ( char const* pszDBName ) const;
+ friend CVDatabase CVDBManager::CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath );
+
+ CVDatabase();
+ ~CVDatabase();
+ CVDatabase(CVDatabase const& x);
+ CVDatabase& operator=(CVDatabase const& x);
+
+ void Release();
+ CVTable OpenTable ( char const* pszTableName ) const;
+ CVTable CreateTable ( char const* pszTableName );
+ void ColumnCreateParams ( ::KCreateMode cmode, ::KChecksum checksum, size_t pgsize );
+
+ private:
+ void Clone(CVDatabase const& x);
+ ::VDatabase* m_pSelf;
+ };
+
+//////////////////////////////////////////////////////////////
+
+ class CVTable
+ {
+ public:
+ friend CVTable CVDatabase::OpenTable(char const* pszTableName) const;
+ friend CVTable CVDatabase::CreateTable ( char const* pszTableName );
+
+ CVTable();
+ ~CVTable();
+ CVTable(CVTable const& x);
+ CVTable& operator=(CVTable const& x);
+
+ void Release();
+ CVCursor CreateCursorRead ( size_t cache_size ) const;
+ CVCursor CreateCursorWrite ( ::KCreateMode mode );
+
+ private:
+ void Clone(CVTable const& x);
+ ::VTable* m_pSelf;
+ };
+
+////////////////////////////////////////////////////////////////////////////
+
+ class CVCursor
+ {
+ public:
+ friend CVCursor CVTable::CreateCursorRead ( size_t cache_size ) const;
+ friend CVCursor CVTable::CreateCursorWrite (::KCreateMode mode);
+
+ CVCursor();
+ ~CVCursor();
+ CVCursor(CVCursor const& x);
+ CVCursor& operator=(CVCursor const& x);
+
+ void Release();
+ void PermitPostOpenAdd() const;
+#if MANAGER_WRITABLE != 0
+ void InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount, bool set_default);
+#else
+ void InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount);
+#endif
+ void Open() const;
+ void GetIdRange(int64_t& idFirstRow, uint64_t& nRowCount) const;
+
+ template <typename T> uint32_t ReadItems (int64_t idRow, uint32_t idxCol, T* pBuf, uint32_t nBufLen) const
+ {
+ uint32_t nItemsRead = 0;
+
+ rc_t rc = ::VCursorReadDirect(m_pSelf, idRow, idxCol, 8*sizeof(T), pBuf, nBufLen, &nItemsRead);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorReadDirect: row_id=%ld, idxCol=%u", idRow, idxCol);
+
+ //CPostReadAction<T>(pBuf, nItemsRead)();
+
+ return nItemsRead;
+ }
+
+ template <typename T> void Write (uint32_t idxCol, T const* pBuf, uint64_t count)
+ {
+ rc_t rc = ::VCursorWrite ( m_pSelf, idxCol, 8 * sizeof(T), pBuf, 0, count );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorWrite: idxCol=%u", idxCol);
+ }
+
+ int64_t GetRowId () const;
+ void SetRowId (int64_t row_id) const;
+ void OpenRow () const;
+ void CommitRow ();
+ void RepeatRow ( uint64_t count );
+ void CloseRow () const;
+ void Commit ();
+
+ private:
+ void Clone(CVCursor const& x);
+ ::VCursor* m_pSelf;
+ };
+}
+
+///////////////////////
+
+namespace KApp
+{
+ class CArgs;
+ class CXMLLogger
+ {
+ public:
+ CXMLLogger ( CArgs const& args );
+ CXMLLogger (CXMLLogger const& x);
+ CXMLLogger& operator= (CXMLLogger const& x);
+ ~CXMLLogger ();
+
+ void Make ( CArgs const& args );
+
+ private:
+ void Release ();
+
+ XMLLogger const* m_pSelf;
+ };
+
+/////////////////////////////////////////
+
+ class CArgs
+ {
+ public:
+ friend void CXMLLogger::Make ( CArgs const& args );
+
+ CArgs ( int argc, char** argv, ::OptDef const* pOptions, size_t option_count );
+ CArgs ( int argc, char** argv, ::OptDef const* pOptions1, size_t option_count1, ::OptDef const* pOptions2, size_t option_count2 );
+ CArgs ( CArgs const& x );
+ CArgs& operator= ( CArgs const& x );
+ ~CArgs ();
+
+ ::Args const* GetArgs () const;
+ uint32_t GetParamCount () const;
+ char const* GetParamValue ( uint32_t iteration ) const;
+ uint32_t GetOptionCount ( char const* option_name ) const;
+ char const* GetOptionValue ( char const* option_name, uint32_t iteration ) const;
+
+ template <typename T> T GetOptionValueInt ( char const* option_name, uint32_t iteration ) const
+ {
+ char const* str_val = GetOptionValue ( option_name, iteration );
+ return Utils::atoi_t <T> ( str_val );
+ }
+ template <typename T> T GetOptionValueUInt ( char const* option_name, uint32_t iteration ) const
+ {
+ char const* str_val = GetOptionValue ( option_name, iteration );
+ return Utils::atou_t <T> ( str_val );
+ }
+
+ private:
+
+ void MakeAndHandle ( int argc, char** argv, ::OptDef const* pOptions, size_t option_count );
+ // TODO: it's better to make ::ArgsMakeAndHandle be able to take va_list
+ void MakeAndHandle ( int argc, char** argv, ::OptDef const* pOptions1, size_t option_count1, ::OptDef const* pOptions2, size_t option_count2 );
+ void Release ();
+
+ ::Args* m_pSelf;
+ };
+
+ class CProgressBar
+ {
+ public:
+ CProgressBar ( uint64_t size );
+ CProgressBar ( CProgressBar const& x );
+ CProgressBar& operator= ( CProgressBar const& x );
+ ~CProgressBar ();
+
+ void Append ( uint64_t chunk );
+ void Process ( uint64_t chunk, bool force_report );
+
+ private:
+ void Make ( uint64_t size );
+ void Release ();
+
+ KLoadProgressbar const* m_pSelf;
+ };
+}
diff --git a/tools/align-info/align-info.c b/tools/align-info/align-info.c
index b939d48..9f06145 100644
--- a/tools/align-info/align-info.c
+++ b/tools/align-info/align-info.c
@@ -112,7 +112,7 @@ rc_t CC UsageSummary (const char * progname) {
"\n", progname);
}
-static const char* param_usage[] = { "Path to the database" };
+static const char* param_usage[] = { "Path to the database", NULL };
rc_t CC Usage(const Args* args) {
rc_t rc = 0 ;
@@ -183,11 +183,15 @@ static rc_t bam_header(const VDatabase* db) {
}
else {
rc = KMDataNodeReadCString(node, buffer, bsize, &size);
- if (rc == 0)
- { break; }
- else if (i == 0 && GetRCObject(rc) == rcBuffer
- && GetRCState(rc) == rcInsufficient)
- { rc = 0; }
+ if (rc == 0) {
+ break;
+ }
+ else if (i == 0
+ && GetRCObject(rc) == (enum RCObject)rcBuffer
+ && GetRCState (rc) == rcInsufficient)
+ {
+ rc = 0;
+ }
}
DISP_RC2(rc, path, "while calling KMDataNodeReadCString");
}
@@ -202,7 +206,7 @@ static rc_t bam_header(const VDatabase* db) {
return rc;
}
-static int CC sort_callback(const void* p1, const void* p2, void* data) {
+static int64_t CC sort_callback(const void* p1, const void* p2, void* data) {
int i1 = *(int*) p1;
int i2 = *(int*) p2;
return i1 - i2;
@@ -510,7 +514,7 @@ rc_t CC KMain(int argc, char* argv[]) {
LOGERR(klogErr, rc, "Too many database parameters");
break;
}
- rc = ArgsParamValue(args, 0, &prm.dbPath);
+ rc = ArgsParamValue(args, 0, (const void **)&prm.dbPath);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving database name");
break;
diff --git a/tools/align-info/align-info.vers b/tools/align-info/align-info.vers
index 8e8299d..35d16fb 100644
--- a/tools/align-info/align-info.vers
+++ b/tools/align-info/align-info.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/align-info/align-info.vers.h b/tools/align-info/align-info.vers.h
index 260db0c..088696c 100644
--- a/tools/align-info/align-info.vers.h
+++ b/tools/align-info/align-info.vers.h
@@ -1 +1 @@
-#define ALIGN_INFO_VERS 0x02040002
+#define ALIGN_INFO_VERS 0x02050007
diff --git a/tools/bam-loader/Globals.h b/tools/bam-loader/Globals.h
index 587bf81..8a4e718 100644
--- a/tools/bam-loader/Globals.h
+++ b/tools/bam-loader/Globals.h
@@ -85,6 +85,7 @@ typedef struct globals
bool noSecondary;
bool hasTI;
bool acceptHardClip;
+ bool allowMultiMapping; /* allow multiple reference names to map to the same real reference */
} Globals;
extern Globals G;
diff --git a/tools/bam-loader/Makefile b/tools/bam-loader/Makefile
index 18082f1..ab5202a 100644
--- a/tools/bam-loader/Makefile
+++ b/tools/bam-loader/Makefile
@@ -31,8 +31,7 @@ MODULE = tools/bam-loader
INT_TOOLS = \
EXT_TOOLS = \
- bam-load \
- bam-load.3
+ bam-load
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -79,14 +78,16 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# bam-load.2.*
+# bam-load
#
BAMLOAD_SRC = \
bam-loader \
+ bam \
alignment-writer \
reference-writer \
sequence-writer \
- loader-imp
+ loader-imp \
+ mem-bank
BAMLOAD_OBJ = \
$(addsuffix .$(OBJX),$(BAMLOAD_SRC))
@@ -98,26 +99,5 @@ BAMLOAD_LIB = \
-lm
$(BINDIR)/bam-load: $(BAMLOAD_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(BAMLOAD_LIB)
-
-#-------------------------------------------------------------------------------
-# bam-load.3
-#
-BAMLOAD3_SRC = \
- bam-loader3 \
- bam-reader \
- loader-imp3
-
-BAMLOAD3_OBJ = \
- $(addsuffix .$(OBJX),$(BAMLOAD3_SRC))
-
-BAMLOAD3_LIB = \
- -lkapp \
- -lloader \
- -lload \
- -sncbi-wvdb \
- -lm
-
-$(BINDIR)/bam-load.3: $(BAMLOAD3_OBJ)
- $(LD) --exe --vers $(SRCDIR)/bam-load.3.vers -o $@ $^ $(BAMLOAD3_LIB)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(BAMLOAD_LIB)
diff --git a/tools/bam-loader/bam-load.3.vers b/tools/bam-loader/bam-load.3.vers
deleted file mode 100644
index 4a36342..0000000
--- a/tools/bam-loader/bam-load.3.vers
+++ /dev/null
@@ -1 +0,0 @@
-3.0.0
diff --git a/tools/bam-loader/bam-load.vers b/tools/bam-loader/bam-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/bam-loader/bam-load.vers
+++ b/tools/bam-loader/bam-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/bam-loader/bam-loader.c b/tools/bam-loader/bam-loader.c
index 633f7a0..d5cb688 100644
--- a/tools/bam-loader/bam-loader.c
+++ b/tools/bam-loader/bam-loader.c
@@ -80,6 +80,7 @@ static char const option_ref_file[] = "ref-file";
static char const option_TI[] = "TI";
static char const option_max_warn_dup_flag[] = "max-warning-dup-flag";
static char const option_accept_hard_clip[] = "accept-hard-clip";
+static char const option_allow_multi_map[] = "allow-multi-map";
#define OPTION_INPUT option_input
#define OPTION_OUTPUT option_output
@@ -102,6 +103,7 @@ static char const option_accept_hard_clip[] = "accept-hard-clip";
#define OPTION_TI option_TI
#define OPTION_MAX_WARN_DUP_FLAG option_max_warn_dup_flag
#define OPTION_ACCEPT_HARD_CLIP option_accept_hard_clip
+#define OPTION_ALLOW_MULTI_MAP option_allow_multi_map
#define ALIAS_INPUT "i"
#define ALIAS_OUTPUT "o"
@@ -150,7 +152,7 @@ char const * config_usage[] =
static
char const * min_mapq_usage[] =
{
- "Minimum mapping quality to be considered an alignment.",
+ "Filter secondary alignments by minimum mapping quality.",
NULL
};
@@ -329,6 +331,14 @@ char const * use_accept_hard_clip[] =
NULL
};
+static
+char const * use_allow_multi_map[] =
+{
+ "allow the same reference to be mapped to multiple names in the input files",
+ "(default is disallow, old behaviors was to allow it)",
+ NULL
+};
+
OptDef Options[] =
{
/* order here is same as in param array below!!! */
@@ -361,7 +371,8 @@ OptDef Options[] =
{ OPTION_REF_FILE, ALIAS_REF_FILE, NULL, use_ref_file, 0, true, false },
{ OPTION_TI, NULL, NULL, use_TI, 1, false, false },
{ OPTION_MAX_WARN_DUP_FLAG, NULL, NULL, use_max_dup_warnings, 1, true, false },
- { OPTION_ACCEPT_HARD_CLIP, NULL, NULL, use_accept_hard_clip, 1, false, false }
+ { OPTION_ACCEPT_HARD_CLIP, NULL, NULL, use_accept_hard_clip, 1, false, false },
+ { OPTION_ALLOW_MULTI_MAP, NULL, NULL, use_allow_multi_map, 1, false, false }
};
const char* OptHelpParam[] =
@@ -396,7 +407,8 @@ const char* OptHelpParam[] =
"path-to-file", /* reference fasta file */
NULL, /* use XT->TI */
"count", /* max. duplicate warning count */
- NULL /* allow hard clipping */
+ NULL, /* allow hard clipping */
+ NULL /* allow multimapping */
};
rc_t UsageSummary (char const * progname)
@@ -626,7 +638,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
}
while(pcount-- > 0) {
- rc = ArgsOptionValue(args, OPTION_REF_FILE, pcount, &G.refFiles[pcount]);
+ rc = ArgsOptionValue(args, OPTION_REF_FILE, pcount, (const void **)&G.refFiles[pcount]);
if (rc)
break;
}
@@ -636,7 +648,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_TMPFS, 0, &G.tmpfs);
+ rc = ArgsOptionValue (args, OPTION_TMPFS, 0, (const void **)&G.tmpfs);
if (rc)
break;
}
@@ -653,7 +665,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_INPUT, 0, &G.inpath);
+ rc = ArgsOptionValue (args, OPTION_INPUT, 0, (const void **)&G.inpath);
if (rc)
break;
}
@@ -670,7 +682,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, option_ref_filter, 0, &G.refFilter);
+ rc = ArgsOptionValue (args, option_ref_filter, 0, (const void **)&G.refFilter);
if (rc)
break;
}
@@ -687,7 +699,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_CONFIG, 0, &G.refXRefPath);
+ rc = ArgsOptionValue (args, OPTION_CONFIG, 0, (const void **)&G.refXRefPath);
if (rc)
break;
}
@@ -704,7 +716,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_OUTPUT, 0, &G.outpath);
+ rc = ArgsOptionValue (args, OPTION_OUTPUT, 0, (const void **)&G.outpath);
if (rc)
break;
}
@@ -727,7 +739,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MINMAPQ, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MINMAPQ, 0, (const void **)&value);
if (rc)
break;
G.minMapQual = strtoul(value, &dummy, 0);
@@ -738,7 +750,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_QCOMP, 0, &G.QualQuantizer);
+ rc = ArgsOptionValue (args, OPTION_QCOMP, 0, (const void **)&G.QualQuantizer);
if (rc)
break;
}
@@ -748,7 +760,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, option_edit_aligned_qual, 0, &value);
+ rc = ArgsOptionValue (args, option_edit_aligned_qual, 0, (const void **)&value);
if (rc)
break;
G.alignedQualValue = strtoul(value, &dummy, 0);
@@ -766,7 +778,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_CACHE_SIZE, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_CACHE_SIZE, 0, (const void **)&value);
if (rc)
break;
G.cache_size = strtoul(value, &dummy, 0) * 1024UL * 1024UL;
@@ -783,7 +795,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MAX_WARN_DUP_FLAG, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MAX_WARN_DUP_FLAG, 0, (const void **)&value);
if (rc)
break;
G.maxWarnCount_DupConflict = strtoul(value, &dummy, 0);
@@ -804,7 +816,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MAX_REC_COUNT, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MAX_REC_COUNT, 0, (const void **)&value);
if (rc)
break;
G.maxAlignCount = strtoul(value, &dummy, 0);
@@ -815,7 +827,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MAX_ERR_COUNT, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MAX_ERR_COUNT, 0, (const void **)&value);
if (rc)
break;
G.maxErrCount = strtoul(value, &dummy, 0);
@@ -826,7 +838,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MIN_MATCH, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MIN_MATCH, 0, (const void **)&value);
if (rc)
break;
G.minMatchCount = strtoul(value, &dummy, 0);
@@ -867,6 +879,11 @@ rc_t CC KMain (int argc, char * argv[])
break;
G.acceptHardClip = pcount > 0;
+ rc = ArgsOptionCount (args, OPTION_ALLOW_MULTI_MAP, &pcount);
+ if (rc)
+ break;
+ G.allowMultiMapping = pcount > 0;
+
rc = ArgsOptionCount (args, OPTION_NOMATCH_LOG, &pcount);
if (rc)
break;
@@ -874,7 +891,7 @@ rc_t CC KMain (int argc, char * argv[])
{
KDirectory *dir;
- rc = ArgsOptionValue (args, OPTION_NOMATCH_LOG, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_NOMATCH_LOG, 0, (const void **)&value);
if (rc) break;
rc = KDirectoryNativeDir(&dir);
if (rc) break;
@@ -887,7 +904,7 @@ rc_t CC KMain (int argc, char * argv[])
if (rc)
break;
if (pcount == 1) {
- rc = ArgsOptionValue (args, OPTION_HEADER, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_HEADER, 0, (const void **)&value);
if (rc) break;
rc = LoadHeader(&G.headerText, value, G.inpath);
if (rc) break;
@@ -912,7 +929,7 @@ rc_t CC KMain (int argc, char * argv[])
unsigned i;
for (i = 0; i < pcount; ++i) {
- rc = ArgsParamValue(args, i, &value);
+ rc = ArgsParamValue(args, i, (const void **)&value);
if (rc) break;
need += strlen(value) + 1;
}
@@ -928,7 +945,7 @@ rc_t CC KMain (int argc, char * argv[])
unsigned i;
for (i = 0; i < pcount; ++i) {
- rc = ArgsOptionValue(args, OPTION_UNALIGNED, i, &value);
+ rc = ArgsOptionValue(args, OPTION_UNALIGNED, i, (const void **)&value);
if (rc) break;
need += strlen(value) + 1;
}
@@ -947,7 +964,7 @@ rc_t CC KMain (int argc, char * argv[])
unsigned i;
for (i = 0; i < pcount; ++i) {
- rc = ArgsOptionValue(args, OPTION_UNALIGNED, i, &value);
+ rc = ArgsOptionValue(args, OPTION_UNALIGNED, i, (const void **)&value);
if (rc) break;
unalgnd[n_unalgnd++] = name_buffer + next_name;
@@ -965,7 +982,7 @@ rc_t CC KMain (int argc, char * argv[])
unsigned i;
for (i = 0; i < pcount; ++i) {
- rc = ArgsParamValue(args, i, &value);
+ rc = ArgsParamValue(args, i, (const void **)&value);
if (rc) break;
aligned[n_aligned++] = name_buffer + next_name;
diff --git a/tools/bam-loader/bam-loader3.c b/tools/bam-loader/bam-loader3.c
deleted file mode 100644
index d2a62cd..0000000
--- a/tools/bam-loader/bam-loader3.c
+++ /dev/null
@@ -1,1028 +0,0 @@
-/*===========================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * ===========================================================================
- *
- */
-
-#include "bam-load.3.vers.h"
-
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <klib/text.h>
-#include <klib/log.h>
-#include <klib/out.h>
-#include <klib/status.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
-#include <kfs/directory.h>
-#include <kfs/file.h>
-#include <kapp/log-xml.h>
-#include <align/writer-refseq.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <assert.h>
-#include <limits.h>
-
-#include <loader/common-writer.h>
-#include "loader-imp3.h"
-
-/* MARK: Arguments and Usage */
-static char const option_input[] = "input";
-static char const option_output[] = "output";
-static char const option_tmpfs[] = "tmpfs";
-static char const option_config[] = "config";
-static char const option_min_mapq[] = "min-mapq";
-static char const option_qual_compress[] = "qual-quant";
-static char const option_cache_size[] = "cache-size";
-static char const option_unsorted[] = "unsorted";
-static char const option_max_err_count[] = "max-err-count";
-static char const option_max_err_pct[] = "max-err-pct";
-static char const option_max_rec_count[] = "max-rec-count";
-static char const option_no_verify[] = "no-verify";
-static char const option_only_verify[] = "only-verify";
-static char const option_use_qual[] = "use-QUAL";
-static char const option_ref_filter[] = "ref-filter";
-static char const option_ref_config[] = "ref-config";
-static char const option_edit_aligned_qual[] = "edit-aligned-qual";
-static char const option_unaligned[] = "unaligned";
-static char const option_accept_dup[] = "accept-dups";
-static char const option_accept_nomatch[] = "accept-nomatch";
-static char const option_nomatch_log[] = "nomatch-log";
-static char const option_keep_mismatch_qual[] = "keep-mismatch-qual";
-static char const option_min_match[] = "minimum-match";
-static char const option_header[] = "header";
-static char const option_no_cs[] = "no-cs";
-static char const option_no_secondary[] = "no-secondary";
-static char const option_ref_file[] = "ref-file";
-static char const option_TI[] = "TI";
-static char const option_max_warn_dup_flag[] = "max-warning-dup-flag";
-static char const option_accept_hard_clip[] = "accept-hard-clip";
-static char const option_parse_spot_name[] = "parse-spot-name";
-
-#define OPTION_INPUT option_input
-#define OPTION_OUTPUT option_output
-#define OPTION_TMPFS option_tmpfs
-#define OPTION_CONFIG option_config
-#define OPTION_MINMAPQ option_min_mapq
-#define OPTION_QCOMP option_qual_compress
-#define OPTION_CACHE_SIZE option_cache_size
-#define OPTION_MAX_ERR_COUNT option_max_err_count
-#define OPTION_MAX_ERR_PCT option_max_err_pct
-#define OPTION_MAX_REC_COUNT option_max_rec_count
-#define OPTION_UNALIGNED option_unaligned
-#define OPTION_ACCEPT_DUP option_accept_dup
-#define OPTION_ACCEPT_NOMATCH option_accept_nomatch
-#define OPTION_NOMATCH_LOG option_nomatch_log
-#define OPTION_MIN_MATCH option_min_match
-#define OPTION_HEADER option_header
-#define OPTION_NO_CS option_no_cs
-#define OPTION_NO_SECONDARY option_no_secondary
-#define OPTION_REF_FILE option_ref_file
-#define OPTION_TI option_TI
-#define OPTION_MAX_WARN_DUP_FLAG option_max_warn_dup_flag
-#define OPTION_ACCEPT_HARD_CLIP option_accept_hard_clip
-#define OPTION_PARSE_SPOT_NAME option_parse_spot_name
-
-#define ALIAS_INPUT "i"
-#define ALIAS_OUTPUT "o"
-#define ALIAS_TMPFS "t"
-#define ALIAS_CONFIG "k"
-#define ALIAS_MINMAPQ "q"
-#define ALIAS_QCOMP "Q"
-#define ALIAS_MAX_ERR_COUNT "E"
-#define ALIAS_UNALIGNED "u"
-#define ALIAS_ACCEPT_DUP "d"
-#define ALIAS_NO_SECONDARY "P"
-#define ALIAS_REF_FILE "r"
-#define ALIAS_MAX_ERR_PCT "p"
-
-static
-char const * input_usage[] =
-{
- "Path where to get fasta files from.",
- NULL
-};
-
-static
-char const * output_usage[] =
-{
- "Path and Name of the output database.",
- NULL
-};
-
-static
-char const * tmpfs_usage[] =
-{
- "Path to be used for scratch files.",
- NULL
-};
-
-static
-char const * config_usage[] =
-{
- "Path to configuration file:",
- "maps the input BAM file's reference names to the equivalent GenBank accession.",
- "It is a tab-delimited text file with unix line endings (\\n - LF) with the following fields in this order:",
- "#1 reference name as it occurs in the BAM file's SN field of @SQ header record;",
- "#2 INSDC reference ID",
- NULL
-};
-
-static
-char const * min_mapq_usage[] =
-{
- "Minimum mapping quality to be considered an alignment.",
- NULL
-};
-
-static
-char const * qcomp_usage[] =
-{
- "Quality scores quantization level, can be a number (0: none, 1: 2bit, 2: 1bit), or a string like '1:10,10:20,20:30,30:-' (which is equivalent to 1).",
- NULL
-};
-
-static
-char const * unsorted_usage[] =
-{
- "Tell the loader to expect unsorted input (requires more memory)",
- NULL
-};
-
-static
-char const * cache_size_usage[] =
-{
- "Set the cache size in MB for the temporary indices",
- NULL
-};
-
-static
-char const * mrc_usage[] =
-{
- "Set the maximum number of records to process from the BAM file",
- NULL
-};
-
-static
-char const * mec_usage[] =
-{
- "Set the maximum number of errors to ignore from the BAM file",
- NULL
-};
-
-static
-char const * no_verify_usage[] =
-{
- "Skip verify existence of references from the BAM file",
- NULL
-};
-
-static
-char const * only_verify_usage[] =
-{
- "Exit after verifying existence of references from the BAM file",
- NULL
-};
-
-static
-char const * use_QUAL_usage[] =
-{
- "use QUAL column for quality values (default is to use OQ if it is available)",
- NULL
-};
-
-static
-char const * use_ref_filter[] =
-{
- "Only process alignments to the given reference",
- NULL
-};
-
-static
-char const * use_ref_config[] =
-{
- "Only process alignments to references in the config file",
- NULL
-};
-
-static
-char const * use_edit_aligned_qual[] =
-{
- "Convert quality at aligned positions to this value",
- NULL
-};
-
-static
-char const * use_keep_mismatch_qual[] =
-{
- "Don't quantized quality at mismatched positions",
- NULL
-};
-
-static
-char const * use_unaligned[] =
-{
- "Specify file without aligned reads",
- NULL
-};
-
-static
-char const * use_accept_dups[] =
-{
- "Accept spots inconsistent PCR duplicate flags",
- NULL
-};
-
-static
-char const * use_accept_nomatch[] =
-{
- "Accept alignments with no matching bases",
- NULL
-};
-
-static
-char const * use_nomatch_log[] =
-{
- "Where to write info for alignments with no matching bases",
- NULL
-};
-
-static
-char const * use_min_match[] =
-{
- "minimum number of matches for an alignment",
- NULL
-};
-
-static
-char const * use_header[] =
-{
- "path to a file containing the SAM header to store in the resulting cSRA, recommended in case of multiple input BAMs",
- NULL
-};
-
-static
-char const * use_no_cs[] =
-{
- "turn off awareness of colorspace",
- NULL
-};
-
-static
-char const * use_no_secondary[] =
-{
- "ignore alignments marked as secondary",
- NULL
-};
-
-static
-char const * use_ref_file[] =
-{
- "path to a fasta file with references",
- NULL
-};
-
-static
-char const * use_TI[] =
-{
- "for trace alignments",
- NULL
-};
-
-static
-char const * use_max_dup_warnings[] =
-{
- "set limit for number of duplicate flag mismatch warnings",
- NULL
-};
-
-static
-char const * use_accept_hard_clip[] =
-{
- "accept hard clipping in CIGAR",
- NULL
-};
-
-static
-char const * use_parse_spot_name[] =
-{
- "parse spot name to remove extra characters",
- NULL
-};
-
-static
-char const * use_max_err_pct[] =
-{
- "acceptable percentage of spots creation errors, default is 5",
- NULL
-};
-
-OptDef Options[] =
-{
- /* order here is same as in param array below!!! */
- { OPTION_INPUT, ALIAS_INPUT, NULL, input_usage, 1, true, false },
- { OPTION_OUTPUT, ALIAS_OUTPUT, NULL, output_usage, 1, true, true },
- { OPTION_CONFIG, ALIAS_CONFIG, NULL, config_usage, 1, true, false },
- { OPTION_HEADER, NULL, NULL, use_header, 1, true, false },
- { OPTION_TMPFS, ALIAS_TMPFS, NULL, tmpfs_usage, 1, true, false },
- { OPTION_UNALIGNED, ALIAS_UNALIGNED, NULL, use_unaligned, 256, true, false },
- { OPTION_ACCEPT_DUP, ALIAS_ACCEPT_DUP, NULL, use_accept_dups, 1, false, false },
- { OPTION_ACCEPT_NOMATCH, NULL, NULL, use_accept_nomatch, 1, false, false },
- { OPTION_NOMATCH_LOG, NULL, NULL, use_nomatch_log, 1, true, false },
- { OPTION_QCOMP, ALIAS_QCOMP, NULL, qcomp_usage, 1, true, false },
- { OPTION_MINMAPQ, ALIAS_MINMAPQ, NULL, min_mapq_usage, 1, true, false },
- { OPTION_CACHE_SIZE, NULL, NULL, cache_size_usage, 1, true, false },
- { OPTION_NO_CS, NULL, NULL, use_no_cs, 1, false, false },
- { OPTION_MIN_MATCH, NULL, NULL, use_min_match, 1, true, false },
- { OPTION_NO_SECONDARY, ALIAS_NO_SECONDARY, NULL, use_no_secondary, 1, false, false },
- { option_unsorted, NULL, NULL, unsorted_usage, 1, false, false },
- { option_no_verify, NULL, NULL, no_verify_usage, 1, false, false },
- { option_only_verify, NULL, NULL, only_verify_usage, 1, false, false },
- { option_use_qual, NULL, NULL, use_QUAL_usage, 1, false, false },
- { option_ref_config, NULL, NULL, use_ref_config, 1, false, false },
- { option_ref_filter, NULL, NULL, use_ref_filter, 1, true, false },
- { option_edit_aligned_qual, NULL, NULL, use_edit_aligned_qual, 1, true, false },
- { option_keep_mismatch_qual, NULL, NULL, use_keep_mismatch_qual, 1, false, false },
- { OPTION_MAX_REC_COUNT, NULL, NULL, mrc_usage, 1, true, false },
- { OPTION_MAX_ERR_COUNT, ALIAS_MAX_ERR_COUNT, NULL, mec_usage, 1, true, false },
- { OPTION_REF_FILE, ALIAS_REF_FILE, NULL, use_ref_file, 0, true, false },
- { OPTION_TI, NULL, NULL, use_TI, 1, false, false },
- { OPTION_MAX_WARN_DUP_FLAG, NULL, NULL, use_max_dup_warnings, 1, true, false },
- { OPTION_ACCEPT_HARD_CLIP, NULL, NULL, use_accept_hard_clip, 1, false, false },
- { OPTION_PARSE_SPOT_NAME, NULL, NULL, use_parse_spot_name, 1, false, false },
- { OPTION_MAX_ERR_PCT, ALIAS_MAX_ERR_PCT, NULL, use_max_err_pct, 1, true, false },
-};
-
-const char* OptHelpParam[] =
-{
- /* order here is same as in OptDef array above!!! */
- "path",
- "path",
- "path-to-file",
- "path-to-file",
- "path",
- "path-to-file",
- NULL,
- NULL,
- NULL,
- "level",
- "phred-score",
- "mbytes",
- NULL,
- NULL,
- "count",
- NULL,
- NULL,
- NULL,
- NULL,
- NULL,
- NULL,
- "name",
- "new-value",
- NULL,
- NULL,
- NULL,
- "path-to-file",
- NULL,
- "count",
- NULL,
- NULL,
- NULL
-};
-
-rc_t UsageSummary (char const * progname)
-{
- return KOutMsg (
- "Usage:\n"
- "\t%s [options] <bam-file>\n"
- "\n"
- "Summary:\n"
- "\tLoad a BAM formatted data file\n"
- "\n"
- "Example:\n"
- "\t%s -o /tmp/SRZ123456 -k analysis.bam.cfg 123456.bam\n"
- "\n"
- ,progname, progname);
-}
-
-char const UsageDefaultName[] = "bam-load";
-
-rc_t CC Usage (const Args * args)
-{
- rc_t rc;
- int i;
- const char * progname = UsageDefaultName;
- const char * fullpath = UsageDefaultName;
- const size_t argsQty = sizeof(Options) / sizeof(Options[0]);
-
- if (args == NULL)
- rc = RC (rcApp, rcArgv, rcAccessing, rcSelf, rcNull);
- else
- rc = ArgsProgram (args, &fullpath, &progname);
- if (rc)
- progname = fullpath = UsageDefaultName;
-
- UsageSummary (progname);
-
- for(i = 0; i < argsQty; i++ ) {
- if( Options[i].required && Options[i].help[0] != NULL ) {
- HelpOptionLine(Options[i].aliases, Options[i].name, OptHelpParam[i], Options[i].help);
- }
- }
- OUTMSG(("\nOptions:\n"));
- for(i = 0; i < argsQty; i++ ) {
- if( !Options[i].required && Options[i].help[0] != NULL ) {
- HelpOptionLine(Options[i].aliases, Options[i].name, OptHelpParam[i], Options[i].help);
- }
- }
- XMLLogger_Usage();
- OUTMSG(("\n"));
- HelpOptionsStandard ();
- HelpVersion (fullpath, KAppVersion());
- return rc;
-}
-
-/* MARK: Definitions and Globals */
-
-#define SCHEMAFILE "align/align.vschema"
-
-CommonWriterSettings G;
-
-uint32_t CC KAppVersion (void)
-{
- return BAM_LOAD_3_VERS;
-}
-
-#ifdef _WIN32
-#include <process.h>
-#else
-#include <unistd.h>
-#endif
-static void set_pid(void)
-{
- G.pid = getpid();
-}
-
-static rc_t PathWithBasePath(char rslt[], size_t sz, char const path[], char const base[])
-{
- size_t const plen = strlen(path);
- bool const hasBase = base && base[0];
- bool const isBareName = strchr(path, '/') == NULL;
-
- if (isBareName && hasBase) {
- if (string_printf(rslt, sz, NULL, "%s/%s", base, path) == 0)
- return 0;
- }
- else if (plen < sz) {
- strcpy(rslt, path);
- return 0;
- }
- {
- rc_t const rc = RC(rcApp, rcArgv, rcAccessing, rcBuffer, rcInsufficient);
- (void)LOGERR(klogErr, rc, "The path to the file is too long");
- return rc;
- }
-}
-
-static rc_t OpenFile(KFile const **kf, char const path[], char const base[])
-{
- char fname[4096];
- rc_t rc = PathWithBasePath(fname, sizeof(fname), path, base);
-
- if (rc == 0) {
- KDirectory *dir;
-
- rc = KDirectoryNativeDir(&dir);
- if (rc == 0) {
- rc = KDirectoryOpenFileRead(dir, kf, "%s", fname);
- KDirectoryRelease(dir);
- }
- }
- return rc;
-}
-
-static rc_t LoadHeader(char const **rslt, char const path[], char const base[])
-{
- KFile const *kf;
- rc_t rc = OpenFile(&kf, path, base);
-
- *rslt = NULL;
- if (rc == 0) {
- uint64_t fsize;
- rc = KFileSize(kf, &fsize);
- if (rc == 0) {
- char *fdata = malloc(fsize+1);
-
- if (fdata) {
- size_t nread;
- rc = KFileRead(kf, 0, fdata, fsize, &nread);
- if (rc == 0) {
- if (nread) {
- fdata[nread] = '\0';
- *rslt = fdata;
- }
- else {
- free(fdata);
- rc = RC(rcApp, rcArgv, rcAccessing, rcFile, rcEmpty);
- (void)PLOGERR(klogErr, (klogErr, rc, "File '$(file)' is empty", "file=%s", path));
- }
- }
- else {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to read file '$(file)'", "file=%s", path));
- }
- }
- else {
- rc = RC(rcApp, rcArgv, rcAccessing, rcMemory, rcExhausted);
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to read file '$(file)'", "file=%s", path));
- }
- }
- KFileRelease(kf);
- }
- else {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open file '$(file)'", "file=%s", path));
- }
- return rc;
-}
-
-rc_t CC KMain (int argc, char * argv[])
-{
- Args * args;
- rc_t rc;
- unsigned n_aligned = 0;
- unsigned n_unalgnd = 0;
- char *aligned[256];
- char *unalgnd[256];
- char *name_buffer = NULL;
- unsigned next_name = 0;
- unsigned nbsz = 0;
- char const *value;
- char *dummy;
- const XMLLogger* xml_logger = NULL;
-
- memset(&G, 0, sizeof(G));
-
- G.mode = mode_Archive;
- G.maxSeqLen = TableWriterRefSeq_MAX_SEQ_LEN;
- G.schemaPath = SCHEMAFILE;
- G.omit_aligned_reads = true;
- G.omit_reference_reads = true;
- G.minMapQual = 0; /* accept all */
- G.tmpfs = "/tmp";
-#if _ARCH_BITS == 32
- G.cache_size = ( size_t ) 1 << 30;
-#else
- G.cache_size = ( size_t ) 10 << 30;
-#endif
- G.maxErrCount = 1000;
- G.maxErrPct = 5;
- G.minMatchCount = 10;
-
- set_pid();
-
- rc = ArgsMakeAndHandle (&args, argc, argv, 2, Options,
- sizeof Options / sizeof (OptDef), XMLLogger_Args, XMLLogger_ArgsQty);
-
- while (rc == 0) {
- uint32_t pcount;
-
- if( (rc = XMLLogger_Make(&xml_logger, NULL, args)) != 0 ) {
- break;
- }
- rc = ArgsOptionCount(args, option_only_verify, &pcount);
- if (rc)
- break;
- G.onlyVerifyReferences = (pcount > 0);
-
- rc = ArgsOptionCount(args, option_no_verify, &pcount);
- if (rc)
- break;
- G.noVerifyReferences = (pcount > 0);
-
- rc = ArgsOptionCount(args, option_use_qual, &pcount);
- if (rc)
- break;
- G.useQUAL = (pcount > 0);
-
- rc = ArgsOptionCount(args, option_ref_config, &pcount);
- if (rc)
- break;
- G.limit2config = (pcount > 0);
-
- rc = ArgsOptionCount(args, OPTION_REF_FILE, &pcount);
- if (rc)
- break;
- G.refFiles = calloc(pcount + 1, sizeof(*(G.refFiles)));
- if( !G.refFiles ) {
- rc = RC(rcApp, rcArgv, rcAccessing, rcMemory, rcExhausted);
- break;
- }
- while(pcount-- > 0) {
- rc = ArgsOptionValue(args, OPTION_REF_FILE, pcount, &G.refFiles[pcount]);
- if (rc)
- break;
- }
-
- rc = ArgsOptionCount (args, OPTION_TMPFS, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_TMPFS, 0, &G.tmpfs);
- if (rc)
- break;
- }
- else if (pcount > 1)
- {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
- OUTMSG (("Single parameter required\n"));
- MiniUsage (args);
- break;
- }
-
- rc = ArgsOptionCount (args, OPTION_INPUT, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_INPUT, 0, &G.inpath);
- if (rc)
- break;
- }
- else if (pcount > 1)
- {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
- OUTMSG (("Single input parameter required\n"));
- MiniUsage (args);
- break;
- }
-
- rc = ArgsOptionCount (args, option_ref_filter, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, option_ref_filter, 0, &G.refFilter);
- if (rc)
- break;
- }
- else if (pcount > 1)
- {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
- OUTMSG (("Single parameter required\n"));
- MiniUsage (args);
- break;
- }
-
- rc = ArgsOptionCount (args, OPTION_CONFIG, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_CONFIG, 0, &G.refXRefPath);
- if (rc)
- break;
- }
- else if (pcount > 1)
- {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
- OUTMSG (("Single input parameter required\n"));
- MiniUsage (args);
- break;
- }
-
- rc = ArgsOptionCount (args, OPTION_OUTPUT, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_OUTPUT, 0, &G.outpath);
- if (rc)
- break;
- }
- else if (pcount > 1)
- {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
- OUTMSG (("Single output parameter required\n"));
- MiniUsage (args);
- break;
- }
- else if (!G.onlyVerifyReferences) {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcInsufficient);
- OUTMSG (("Output parameter required\n"));
- MiniUsage (args);
- break;
- }
-
- rc = ArgsOptionCount (args, OPTION_MINMAPQ, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_MINMAPQ, 0, &value);
- if (rc)
- break;
- G.minMapQual = strtoul(value, &dummy, 0);
- }
-
- rc = ArgsOptionCount (args, OPTION_QCOMP, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_QCOMP, 0, &G.QualQuantizer);
- if (rc)
- break;
- }
-
- rc = ArgsOptionCount (args, option_edit_aligned_qual, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, option_edit_aligned_qual, 0, &value);
- if (rc)
- break;
- G.alignedQualValue = strtoul(value, &dummy, 0);
- if (G.alignedQualValue == 0) {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcIncorrect);
- OUTMSG (("edit-aligned-qual: bad value\n"));
- MiniUsage (args);
- break;
- }
- G.editAlignedQual = true;
- }
-
- rc = ArgsOptionCount (args, OPTION_CACHE_SIZE, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_CACHE_SIZE, 0, &value);
- if (rc)
- break;
- G.cache_size = strtoul(value, &dummy, 0) * 1024UL * 1024UL;
- if (G.cache_size == 0) {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcIncorrect);
- OUTMSG (("cache-size: bad value\n"));
- MiniUsage (args);
- break;
- }
- }
-
- rc = ArgsOptionCount (args, OPTION_MAX_WARN_DUP_FLAG, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_MAX_WARN_DUP_FLAG, 0, &value);
- if (rc)
- break;
- G.maxWarnCount_DupConflict = strtoul(value, &dummy, 0);
- }
-
- rc = ArgsOptionCount (args, option_unsorted, &pcount);
- if (rc)
- break;
- G.expectUnsorted = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_MAX_REC_COUNT, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_MAX_REC_COUNT, 0, &value);
- if (rc)
- break;
- G.maxAlignCount = strtoul(value, &dummy, 0);
- }
-
- rc = ArgsOptionCount (args, OPTION_MAX_ERR_COUNT, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_MAX_ERR_COUNT, 0, &value);
- if (rc)
- break;
- G.maxErrCount = strtoul(value, &dummy, 0);
- }
-
- rc = ArgsOptionCount (args, OPTION_MAX_ERR_PCT, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_MAX_ERR_PCT, 0, &value);
- if (rc)
- break;
- G.maxErrPct = strtoul(value, &dummy, 0);
- }
-
- rc = ArgsOptionCount (args, OPTION_MIN_MATCH, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_MIN_MATCH, 0, &value);
- if (rc)
- break;
- G.minMatchCount = strtoul(value, &dummy, 0);
- }
-
- rc = ArgsOptionCount (args, OPTION_ACCEPT_DUP, &pcount);
- if (rc)
- break;
- G.acceptBadDups = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_ACCEPT_NOMATCH, &pcount);
- if (rc)
- break;
- G.acceptNoMatch = pcount > 0;
-
- rc = ArgsOptionCount (args, option_keep_mismatch_qual, &pcount);
- if (rc)
- break;
- G.keepMismatchQual = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_NO_CS, &pcount);
- if (rc)
- break;
- G.noColorSpace = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_NO_SECONDARY, &pcount);
- if (rc)
- break;
- G.noSecondary = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_TI, &pcount);
- if (rc)
- break;
- G.hasTI = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_ACCEPT_HARD_CLIP, &pcount);
- if (rc)
- break;
- G.acceptHardClip = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_PARSE_SPOT_NAME, &pcount);
- if (rc)
- break;
- G.parseSpotName = pcount > 0;
-
- rc = ArgsOptionCount (args, OPTION_NOMATCH_LOG, &pcount);
- if (rc)
- break;
- if (pcount == 1)
- {
- KDirectory *dir;
-
- rc = ArgsOptionValue (args, OPTION_NOMATCH_LOG, 0, &value);
- if (rc) break;
- rc = KDirectoryNativeDir(&dir);
- if (rc) break;
- rc = KDirectoryCreateFile(dir, &G.noMatchLog, 0, 0664, kcmInit, "%s", value);
- KDirectoryRelease(dir);
- if (rc) break;
- }
-
- rc = ArgsOptionCount (args, OPTION_HEADER, &pcount);
- if (rc)
- break;
- if (pcount == 1) {
- rc = ArgsOptionValue (args, OPTION_HEADER, 0, &value);
- if (rc) break;
- rc = LoadHeader(&G.headerText, value, G.inpath);
- if (rc) break;
- }
-
- rc = ArgsParamCount (args, &pcount);
- if (rc) break;
- if (pcount == 0)
- {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcInsufficient);
- MiniUsage (args);
- break;
- }
- else if (pcount > sizeof(aligned)/sizeof(aligned[0])) {
- rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
- (void)PLOGERR(klogErr, (klogErr, rc, "$(count) input files is too many, $(max) is the limit",
- "count=%u,max=%u", (unsigned)pcount, (unsigned)(sizeof(aligned)/sizeof(aligned[0]))));
- break;
- }
- else {
- unsigned need = G.inpath ? (strlen(G.inpath) + 1) * pcount : 0;
- unsigned i;
-
- for (i = 0; i < pcount; ++i) {
- rc = ArgsParamValue(args, i, &value);
- if (rc) break;
- need += strlen(value) + 1;
- }
- nbsz = need;
- }
-
- rc = ArgsOptionCount (args, OPTION_UNALIGNED, &pcount);
- if (rc)
- break;
- if (pcount > 0)
- {
- unsigned need = G.inpath ? (strlen(G.inpath) + 1) * pcount : 0;
- unsigned i;
-
- for (i = 0; i < pcount; ++i) {
- rc = ArgsOptionValue(args, OPTION_UNALIGNED, i, &value);
- if (rc) break;
- need += strlen(value) + 1;
- }
- if (rc) break;
- nbsz += need;
- }
-
- name_buffer = malloc(nbsz);
- if (name_buffer == NULL) {
- rc = RC(rcApp, rcArgv, rcAccessing, rcMemory, rcExhausted);
- break;
- }
-
- rc = ArgsOptionCount (args, OPTION_UNALIGNED, &pcount);
- if (rc == 0) {
- unsigned i;
-
- for (i = 0; i < pcount; ++i) {
- rc = ArgsOptionValue(args, OPTION_UNALIGNED, i, &value);
- if (rc) break;
-
- unalgnd[n_unalgnd++] = name_buffer + next_name;
- rc = PathWithBasePath(name_buffer + next_name, nbsz - next_name, value, G.inpath);
- if (rc) break;
- next_name += strlen(name_buffer + next_name) + 1;
- }
- if (rc) break;
- }
- else
- break;
-
- rc = ArgsParamCount (args, &pcount);
- if (rc == 0) {
- unsigned i;
-
- for (i = 0; i < pcount; ++i) {
- rc = ArgsParamValue(args, i, &value);
- if (rc) break;
-
- aligned[n_aligned++] = name_buffer + next_name;
- rc = PathWithBasePath(name_buffer + next_name, nbsz - next_name, value, G.inpath);
- if (rc) break;
- next_name += strlen(name_buffer + next_name) + 1;
- }
- }
- else
- break;
-
- rc = run(argv[0], &G, n_aligned, (char const **)aligned, n_unalgnd, (char const **)unalgnd);
- break;
- }
- free(name_buffer);
- free((void *)G.headerText);
- free(G.refFiles);
-
- value = G.outpath ? strrchr(G.outpath, '/') : "/???";
- if( value == NULL ) {
- value = G.outpath;
- } else {
- value++;
- }
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "load failed",
- "severity=total,status=failure,accession=%s,errors=%u", value, G.errCount));
- } else {
- (void)PLOGMSG(klogInfo, (klogInfo, "loaded",
- "severity=total,status=success,accession=%s,errors=%u", value, G.errCount));
- }
- ArgsWhack(args);
- XMLLogger_Release(xml_logger);
- return rc;
-}
diff --git a/tools/bam-loader/bam.c b/tools/bam-loader/bam.c
new file mode 100644
index 0000000..c6db6ee
--- /dev/null
+++ b/tools/bam-loader/bam.c
@@ -0,0 +1,4085 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include <klib/defs.h>
+#include <klib/debug.h>
+#include <klib/sort.h>
+#include <klib/rc.h>
+#include <kfs/file.h>
+#include <kfs/directory.h>
+#include <kfs/mmap.h>
+#include <klib/printf.h>
+#include <klib/log.h>
+#include <klib/text.h>
+#include <klib/refcount.h>
+#include <sysalloc.h>
+
+#include <atomic32.h>
+#include <strtol.h>
+
+#include "bam.h"
+
+#include <vfs/path.h>
+#include <vfs/path-priv.h>
+#include <kfs/kfs-priv.h>
+
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+#if 1
+/*_DEBUGGING*/
+#include <stdio.h>
+#include <os-native.h>
+#endif
+
+#include <endian.h>
+#include <byteswap.h>
+
+#include <zlib.h>
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static uint16_t LE2HUI16(void const *X) { uint16_t y; memcpy(&y, X, sizeof(y)); return y; }
+static uint32_t LE2HUI32(void const *X) { uint32_t y; memcpy(&y, X, sizeof(y)); return y; }
+static uint64_t LE2HUI64(void const *X) { uint64_t y; memcpy(&y, X, sizeof(y)); return y; }
+static int16_t LE2HI16(void const *X) { int16_t y; memcpy(&y, X, sizeof(y)); return y; }
+static int32_t LE2HI32(void const *X) { int32_t y; memcpy(&y, X, sizeof(y)); return y; }
+/* static int64_t LE2HI64(void const *X) { int64_t y; memcpy(&y, X, sizeof(y)); return y; } */
+#endif
+#if __BYTE_ORDER == __BIG_ENDIAN
+static uint16_t LE2HUI16(void const *X) { uint16_t y; memcpy(&y, X, sizeof(y)); return (uint16_t)bswap_16(y); }
+static uint32_t LE2HUI32(void const *X) { uint32_t y; memcpy(&y, X, sizeof(y)); return (uint32_t)bswap_32(y); }
+static uint64_t LE2HUI64(void const *X) { uint64_t y; memcpy(&y, X, sizeof(y)); return (uint64_t)bswap_64(y); }
+static int16_t LE2HI16(void const *X) { int16_t y; memcpy(&y, X, sizeof(y)); return ( int16_t)bswap_16(y); }
+static int32_t LE2HI32(void const *X) { int32_t y; memcpy(&y, X, sizeof(y)); return ( int32_t)bswap_32(y); }
+static int64_t LE2HI64(void const *X) { int64_t y; memcpy(&y, X, sizeof(y)); return ( int64_t)bswap_64(y); }
+#endif
+
+typedef struct BAMIndex BAMIndex;
+typedef struct BufferedFile BufferedFile;
+typedef struct SAMFile SAMFile;
+typedef struct BGZFile BGZFile;
+
+#define ZLIB_BLOCK_SIZE (64u * 1024u)
+#define RGLR_BUFFER_SIZE (256u * ZLIB_BLOCK_SIZE)
+#define PIPE_BUFFER_SIZE (4096u)
+
+typedef uint8_t zlib_block_t[ZLIB_BLOCK_SIZE];
+
+typedef struct RawFile_vt_s {
+ rc_t (*FileRead)(void *, zlib_block_t, unsigned *);
+ uint64_t (*FileGetPos)(void const *);
+ float (*FileProPos)(void const *);
+ uint64_t (*FileGetSize)(void const *);
+ rc_t (*FileSetPos)(void *, uint64_t);
+ void (*FileWhack)(void *);
+} RawFile_vt;
+
+/* MARK: SAMFile */
+
+struct BufferedFile {
+ KFile const *kf;
+ void *buf;
+ uint64_t fmax; /* file size if known or 0 */
+ uint64_t fpos; /* position in file of first byte in buffer */
+ size_t bpos; /* position in buffer of read head */
+ size_t bmax; /* number of valid bytes in buffer */
+ size_t size; /* maximum number of that can be read into buffer */
+};
+
+static rc_t BufferedFileRead(BufferedFile *const self)
+{
+ uint64_t const fpos = self->fpos + self->bmax;
+ rc_t const rc = KFileRead(self->kf, fpos, self->buf, self->size, &self->bmax);
+ if (rc) {
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("Error %R read %u bytes from SAM/BAM file at position %lu\n", rc, self->bmax, fpos));
+ return rc;
+ }
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("Read %u bytes from SAM/BAM file at position %lu\n", self->bmax, fpos));
+ self->fpos = fpos;
+ self->bpos = 0;
+ return 0;
+}
+
+static rc_t BufferedFileInit(BufferedFile *const self, KFile const *const kf)
+{
+ rc_t const rc = KFileSize(kf, &self->fmax);
+ if (rc || self->fmax == 0) {
+ self->fmax = 0;
+ self->size = PIPE_BUFFER_SIZE;
+ }
+ else
+ self->size = self->fmax < RGLR_BUFFER_SIZE ? self->fmax : RGLR_BUFFER_SIZE;
+
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("File size is %lu; Read buffer is %lu\n", self->fmax, self->size));
+ self->buf = malloc(self->size);
+ self->fpos = 0;
+ self->bpos = 0;
+ self->bmax = 0;
+ self->kf = kf;
+ KFileAddRef(kf);
+
+ return self->buf != NULL ? 0 : RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+}
+
+static void BufferedFileWhack(BufferedFile *const self)
+{
+ KFileRelease(self->kf);
+ free(self->buf);
+}
+
+static uint64_t BufferedFileGetPos(BufferedFile const *const self)
+{
+ return self->fpos + self->bpos;
+}
+
+static float BufferedFileProPos(BufferedFile const *const self)
+{
+ return self->fmax == 0 ? -1.0 : (BufferedFileGetPos(self) / (double)self->fmax);
+}
+
+static rc_t BufferedFileSetPos(BufferedFile *const self, uint64_t const pos)
+{
+ if (pos >= self->fpos && pos < self->fpos + self->bmax) {
+ self->bpos = (size_t)(pos - self->fpos);
+ }
+ else if (pos < self->fmax) {
+ self->fpos = pos;
+ self->bpos = 0;
+ self->bmax = 0;
+ }
+ else
+ return RC(rcAlign, rcFile, rcPositioning, rcParam, rcInvalid);
+ return 0;
+}
+
+static uint64_t BufferedFileGetSize(BufferedFile const *const self)
+{
+ return self->fmax;
+}
+
+struct SAMFile {
+ BufferedFile file;
+ int putback;
+ rc_t last;
+};
+
+static int SAMFileRead1(SAMFile *const self)
+{
+ if (self->putback < 0) {
+ if (self->file.bpos == self->file.bmax) {
+ rc_t const rc = BufferedFileRead(&self->file);
+
+ self->last = rc;
+ if (rc || self->file.bmax == 0) return -1;
+ }
+ return ((char const *)self->file.buf)[self->file.bpos++];
+ }
+ else {
+ int const x = self->putback;
+ self->putback = -1;
+ return x;
+ }
+}
+
+static rc_t SAMFileLastError(SAMFile const *const self)
+{
+ return self->last ? self->last : SILENT_RC(rcAlign, rcFile, rcReading, rcData, rcInsufficient);
+}
+
+static void SAMFilePutBack(SAMFile *const self, int ch)
+{
+ if (self->file.bpos > 0)
+ --self->file.bpos;
+ else
+ self->putback = ch;
+}
+
+static
+rc_t SAMFileInit(SAMFile *self, RawFile_vt *vt)
+{
+ static RawFile_vt const my_vt = {
+ (rc_t (*)(void *, zlib_block_t, unsigned *))NULL,
+ (uint64_t (*)(void const *))BufferedFileGetPos,
+ (float (*)(void const *))BufferedFileProPos,
+ (uint64_t (*)(void const *))BufferedFileGetSize,
+ (rc_t (*)(void *, uint64_t))BufferedFileSetPos,
+ (void (*)(void *))NULL
+ };
+
+ self->putback = -1;
+ self->last = 0;
+ *vt = my_vt;
+
+ return 0;
+}
+
+/* MARK: BGZFile *** Start *** */
+
+#define VALIDATE_BGZF_HEADER 1
+#if (ZLIB_VERNUM < 0x1230)
+#undef VALIDATE_BGZF_HEADER
+#warning "zlib too old, inflateGetHeader not available, not validating BGZF headers"
+#else
+#endif
+
+#define CG_NUM_SEGS 4
+
+struct BGZFile {
+ BufferedFile file;
+ z_stream zs;
+};
+
+static
+rc_t BGZFileGetMoreBytes(BGZFile *self)
+{
+ rc_t const rc = BufferedFileRead(&self->file);
+ if (rc)
+ return rc;
+
+ if (self->file.bmax == 0)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInsufficient);
+
+ self->zs.avail_in = (uInt)(self->file.bmax);
+ self->zs.next_in = (Bytef *)self->file.buf;
+
+ return 0;
+}
+
+static
+rc_t BGZFileRead(BGZFile *self, zlib_block_t dst, unsigned *pNumRead)
+{
+#if VALIDATE_BGZF_HEADER
+ uint8_t extra[256];
+ gz_header head;
+#endif
+ rc_t rc = 0;
+ unsigned loops;
+ int zr;
+
+ *pNumRead = 0;
+ if (self->file.bmax == 0 || self->zs.avail_in == 0) {
+ rc = BGZFileGetMoreBytes(self);
+ if (rc)
+ return rc;
+ }
+
+#if VALIDATE_BGZF_HEADER
+ memset(&head, 0, sizeof(head));
+ head.extra = extra;
+ head.extra_max = sizeof(extra);
+
+ zr = inflateGetHeader(&self->zs, &head);
+ assert(zr == Z_OK);
+#endif
+
+ self->zs.next_out = (Bytef *)dst;
+ self->zs.avail_out = sizeof(zlib_block_t);
+
+ for (loops = 0; loops != 2; ++loops) {
+ {
+ uLong const initial = self->zs.total_in;
+
+ zr = inflate(&self->zs, Z_FINISH);
+ {
+ uLong const final = self->zs.total_in;
+ uLong const len = final - initial;
+
+ self->file.bpos += len;
+ }
+ }
+ assert(self->zs.avail_in == self->file.bmax - self->file.bpos);
+
+ switch (zr) {
+ case Z_OK:
+ case Z_BUF_ERROR:
+ rc = BGZFileGetMoreBytes(self);
+ if ( rc != 0 )
+ {
+ if ( GetRCObject( rc ) == (enum RCObject)rcData && GetRCState( rc ) == rcInsufficient )
+ {
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("EOF in Zlib block after %lu bytes\n", self->file.fpos + self->file.bpos));
+ rc = RC( rcAlign, rcFile, rcReading, rcFile, rcTooShort );
+ }
+ return rc;
+ }
+ break;
+ case Z_STREAM_END:
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("Zlib block size (before/after): %u/%u\n", self->zs.total_in, self->zs.total_out));
+#if VALIDATE_BGZF_HEADER
+ if (head.done) {
+ unsigned const extra_len = head.extra_len;
+ unsigned i;
+ unsigned bsize = 0;
+
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("GZIP Header extra length: %u\n", extra_len));
+ for (i = 0; i < extra_len; ) {
+ uint8_t const si1 = extra[i + 0];
+ uint8_t const si2 = extra[i + 1];
+ unsigned const slen = LE2HUI16(&extra[i + 2]);
+
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("GZIP Header extra: %c%c(%u)\n", si1, si2, slen));
+ if (si1 == 'B' && si2 == 'C') {
+ bsize = 1 + LE2HUI16(&extra[i + 4]);
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("BGZF Header extra field BC: bsize %u\n", bsize));
+ break;
+ }
+ i += slen + 4;
+ }
+ if (bsize == 0 || bsize != self->zs.total_in) {
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("BGZF Header extra field BC not found\n"));
+ rc = RC(rcAlign, rcFile, rcReading, rcFormat, rcInvalid); /* not BGZF */
+ }
+ }
+ else {
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("GZIP Header not found\n"));
+ rc = RC(rcAlign, rcFile, rcReading, rcFile, rcCorrupt);
+ }
+#endif
+ *pNumRead = (unsigned)self->zs.total_out; /* <= 64k */
+ zr = inflateReset(&self->zs);
+ assert(zr == Z_OK);
+ return rc;
+ default:
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("Unexpected Zlib result %i: %s\n", zr, self->zs.msg ? self->zs.msg : "unknown"));
+ return RC(rcAlign, rcFile, rcReading, rcFile, rcCorrupt);
+ }
+ }
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BGZF), ("Failed reading BAM file after %lu bytes\n", self->file.fpos + self->file.bpos));
+ return RC(rcAlign, rcFile, rcReading, rcFile, rcTooShort);
+}
+
+static void BGZFileWhack(BGZFile *self)
+{
+ inflateEnd(&self->zs);
+}
+
+static rc_t BGZFileInit(BGZFile *const self, RawFile_vt *const vt)
+{
+ int i;
+ static RawFile_vt const my_vt = {
+ (rc_t (*)(void *, zlib_block_t, unsigned *))BGZFileRead,
+ (uint64_t (*)(void const *))BufferedFileGetPos,
+ (float (*)(void const *))BufferedFileProPos,
+ (uint64_t (*)(void const *))BufferedFileGetSize,
+ (rc_t (*)(void *, uint64_t))BufferedFileSetPos,
+ (void (*)(void *))BGZFileWhack
+ };
+
+ *vt = my_vt;
+
+ i = inflateInit2(&self->zs, MAX_WBITS + 16); /* max + enable gzip headers */
+ switch (i) {
+ case Z_OK:
+ break;
+ case Z_MEM_ERROR:
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ default:
+ return RC(rcAlign, rcFile, rcConstructing, rcNoObj, rcUnexpected);
+ }
+
+ return 0;
+}
+
+/* MARK: BAM_File structures */
+
+struct BAM_File {
+ union {
+ BGZFile bam;
+ SAMFile sam;
+ } file;
+ RawFile_vt vt;
+
+ BAMRefSeq *refSeq; /* pointers into headerData1 except name points into headerData2 */
+ BAMReadGroup *readGroup; /* pointers into headerData1 */
+ char const *version;
+ char const *header;
+ void *headerData1; /* gets used for refSeq and readGroup */
+ void *headerData2; /* gets used for refSeq */
+ BAM_Alignment *bufLocker;
+ BAM_Alignment *nocopy; /* used to hold current record for BAM_FileRead2 */
+
+ uint64_t fpos_first;
+ uint64_t fpos_cur;
+
+ size_t nocopy_size;
+
+ unsigned refSeqs;
+ unsigned readGroups;
+ unsigned ucfirst; /* offset of first record in uncompressed buffer */
+ unsigned bufSize; /* current size of uncompressed buffer */
+ unsigned bufCurrent; /* location in uncompressed buffer of read head */
+ bool eof;
+ bool isSAM;
+ zlib_block_t buffer; /* uncompressed buffer */
+};
+
+/* MARK: Alignment structures */
+
+struct bam_alignment_s {
+ uint8_t rID[4];
+ uint8_t pos[4];
+ uint8_t read_name_len;
+ uint8_t mapQual;
+ uint8_t bin[2];
+ uint8_t n_cigars[2];
+ uint8_t flags[2];
+ uint8_t read_len[4];
+ uint8_t mate_rID[4];
+ uint8_t mate_pos[4];
+ uint8_t ins_size[4];
+ char read_name[1 /* read_name_len */];
+/* if you change length of read_name,
+ * adjust calculation of offsets in BAM_AlignmentSetOffsets */
+/* uint32_t cigar[n_cigars];
+ * uint8_t seq[(read_len + 1) / 2];
+ * uint8_t qual[read_len];
+ * uint8_t extra[...];
+ */
+};
+
+typedef union bam_alignment_u {
+ struct bam_alignment_s cooked;
+ uint8_t raw[sizeof(struct bam_alignment_s)];
+} bam_alignment;
+
+struct offset_size_s {
+ unsigned offset;
+ unsigned size; /* this is the total length of the tag; length of data is size - 3 */
+};
+
+struct BAM_Alignment {
+ BAM_File *parent;
+ bam_alignment const *data;
+ uint8_t *storage;
+ unsigned datasize;
+
+ unsigned cigar;
+ unsigned seq;
+ unsigned qual;
+ unsigned numExtra;
+ unsigned hasColor;
+ struct offset_size_s extra[1];
+};
+
+static const char cigarChars[] = {
+ ct_Match,
+ ct_Insert,
+ ct_Delete,
+ ct_Skip,
+ ct_SoftClip,
+ ct_HardClip,
+ ct_Padded,
+ ct_Equal,
+ ct_NotEqual
+ /* ct_Overlap must not appear in actual BAM file */
+};
+
+/* MARK: Alignment accessors */
+
+static int32_t getRefSeqId(const BAM_Alignment *cself) {
+ return LE2HI32(cself->data->cooked.rID);
+}
+
+static int32_t getPosition(const BAM_Alignment *cself) {
+ return LE2HI32(cself->data->cooked.pos);
+}
+
+static uint8_t getReadNameLength(const BAM_Alignment *cself) {
+ return cself->data->cooked.read_name_len;
+}
+
+static uint16_t getBin(const BAM_Alignment *cself) {
+ return LE2HUI16(cself->data->cooked.bin);
+}
+
+static uint8_t getMapQual(const BAM_Alignment *cself) {
+ return cself->data->cooked.mapQual;
+}
+
+static uint16_t getCigarCount(const BAM_Alignment *cself) {
+ return LE2HUI16(cself->data->cooked.n_cigars);
+}
+
+static uint16_t getFlags(const BAM_Alignment *cself) {
+ return LE2HUI16(cself->data->cooked.flags);
+}
+
+static uint32_t getReadLen(const BAM_Alignment *cself) {
+ return LE2HUI32(cself->data->cooked.read_len);
+}
+
+static int32_t getMateRefSeqId(const BAM_Alignment *cself) {
+ return LE2HI32(cself->data->cooked.mate_rID);
+}
+
+static int32_t getMatePos(const BAM_Alignment *cself) {
+ return LE2HI32(cself->data->cooked.mate_pos);
+}
+
+static int32_t getInsertSize(const BAM_Alignment *cself) {
+ return LE2HI32(cself->data->cooked.ins_size);
+}
+
+static char const *getReadName(const BAM_Alignment *cself) {
+ return &cself->data->cooked.read_name[0];
+}
+
+static void const *getCigarBase(BAM_Alignment const *cself)
+{
+ return &cself->data->raw[cself->cigar];
+}
+
+static int opt_tag_cmp(char const a[2], char const b[2])
+{
+ int const d0 = (int)a[0] - (int)b[0];
+ return d0 ? d0 : ((int)a[1] - (int)b[1]);
+}
+
+static int64_t OptTag_sort(void const *A, void const *B, void *ctx)
+{
+ BAM_Alignment const *const self = ctx;
+ unsigned const a_off = ((struct offset_size_s const *)A)->offset;
+ unsigned const b_off = ((struct offset_size_s const *)B)->offset;
+ char const *const a = (char const *)&self->data->raw[a_off];
+ char const *const b = (char const *)&self->data->raw[b_off];
+ int const diff = opt_tag_cmp(a, b);
+
+ if ( diff != 0 )
+ return diff;
+ else if ( a < b )
+ return -1;
+ else
+ return a > b;
+}
+
+static unsigned tag_findfirst(BAM_Alignment const *const self, char const tag[2])
+{
+ unsigned f = 0;
+ unsigned e = self->numExtra;
+
+ while (f < e) {
+ unsigned const m = f + ((e - f) >> 1);
+ char const *const mtag = (char const *)&self->data->raw[self->extra[m].offset];
+ int const d = opt_tag_cmp(tag, mtag);
+
+ if (d > 0)
+ f = m + 1;
+ else
+ e = m;
+ }
+ return f;
+}
+
+static unsigned tag_runlength(BAM_Alignment const *const self,
+ char const tag[2],
+ unsigned const at)
+{
+ unsigned n;
+
+ for (n = 0; n + at < self->numExtra; ++n) {
+ if (opt_tag_cmp(tag, (char const *)&self->data->raw[self->extra[n + at].offset]) != 0)
+ break;
+ }
+ return n;
+}
+
+static struct offset_size_s const *tag_search(BAM_Alignment const *const self,
+ char const tag[2],
+ int const which)
+{
+ unsigned const fnd = tag_findfirst(self, tag);
+ unsigned const run = tag_runlength(self, tag, fnd);
+ unsigned const want = which < 0 ? (run + which) : which;
+
+ return run == 0 ? NULL : &self->extra[fnd + (want % run)];
+}
+
+static char const *get_RG(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "RG", 0);
+ return (char const *)(x && cself->data->raw[x->offset + 2] == 'Z' ? &cself->data->raw[x->offset + 3] : NULL);
+}
+
+static struct offset_size_s const *get_CS_info(BAM_Alignment const *cself)
+{
+ return tag_search(cself, "CS", 0);
+}
+
+static struct offset_size_s const *get_CQ_info(BAM_Alignment const *cself)
+{
+ return tag_search(cself, "CQ", 0);
+}
+
+static char const *get_CS(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = get_CS_info(cself);
+ return (char const *)(x && cself->data->raw[x->offset + 2] == 'Z' ? &cself->data->raw[x->offset + 3] : NULL);
+}
+
+static uint8_t const *get_CQ(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = get_CQ_info(cself);
+ return (uint8_t const *)(x && cself->data->raw[x->offset + 2] == 'Z' ? &cself->data->raw[x->offset + 3] : NULL);
+}
+
+static struct offset_size_s const *get_OQ_info(BAM_Alignment const *cself)
+{
+ return tag_search(cself, "OQ", 0);
+}
+
+static uint8_t const *get_OQ(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = get_OQ_info(cself);
+ return (uint8_t const *)(x && cself->data->raw[x->offset + 2] == 'Z' ? &cself->data->raw[x->offset + 3] : NULL);
+}
+
+static char const *get_XT(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "XT", 0);
+ return (char const *)(x && cself->data->raw[x->offset + 2] == 'Z' ? &cself->data->raw[x->offset + 3] : NULL);
+}
+
+static uint8_t const *get_XS(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "XS", -1); /* want last one */
+ return (uint8_t const *)(x && cself->data->raw[x->offset + 2] == 'A' ? &cself->data->raw[x->offset + 3] : NULL);
+}
+
+static struct offset_size_s const *get_CG_ZA_info(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "ZA", 0);
+ return x;
+}
+
+static struct offset_size_s const *get_CG_ZI_info(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "ZI", 0);
+ return x;
+}
+
+static struct offset_size_s const *get_CG_GC_info(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "GC", 0);
+ return x;
+}
+
+static struct offset_size_s const *get_CG_GS_info(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "GS", 0);
+ return x;
+}
+
+static struct offset_size_s const *get_CG_GQ_info(BAM_Alignment const *cself)
+{
+ struct offset_size_s const *const x = tag_search(cself, "GQ", 0);
+ return x;
+}
+
+/* MARK: BAM_File Reading functions */
+
+/* returns (rcData, rcInsufficient) if eof */
+static rc_t BAM_FileFillBuffer(BAM_File *self)
+{
+ rc_t const rc = self->vt.FileRead(&self->file, self->buffer, &self->bufSize);
+ if (rc)
+ return rc;
+ if (self->bufSize == 0 || self->bufSize <= self->bufCurrent)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInsufficient);
+ return 0;
+}
+
+static rc_t BAM_FileReadn(BAM_File *self, const unsigned len, uint8_t dst[/* len */]) {
+ rc_t rc;
+ unsigned cur;
+ unsigned n = 0;
+
+ if (len == 0)
+ return 0;
+
+ for (cur = 0; ; cur += n) {
+ if (self->bufSize > self->bufCurrent) {
+ n = self->bufSize - self->bufCurrent;
+ if (cur + n > len)
+ n = len - cur;
+ memcpy(&dst[cur], &self->buffer[self->bufCurrent], n);
+ self->bufCurrent += n;
+ }
+ if (self->bufCurrent != self->bufSize && self->bufSize != 0)
+ return 0;
+ if (self->bufSize != 0) {
+ /* a seek has not just been done so update the file position.
+ * if we didn't and a request for the position is made before the
+ * next read, we will not have the position of the next read.
+ *
+ * if a seek had just been done then
+ * self->fpos_cur == BGZFileGetPos(&self->file)
+ * is already true.
+ */
+ self->fpos_cur = self->vt.FileGetPos(&self->file);
+ self->bufCurrent = 0;
+ self->bufSize = 0;
+ if (cur + n == len)
+ return 0;
+ }
+
+ rc = BAM_FileFillBuffer(self);
+ if (rc)
+ return rc;
+ }
+}
+
+static void const *BAM_FilePeek(BAM_File const *const self, unsigned const offset)
+{
+ return &self->buffer[self->bufCurrent + offset];
+}
+
+static unsigned BAM_FileMaxPeek(BAM_File const *self)
+{
+ return self->bufSize > self->bufCurrent ? self->bufSize - self->bufCurrent : 0;
+}
+
+static int32_t BAM_FilePeekI32(BAM_File *self)
+{
+ return LE2HI32(BAM_FilePeek(self, 0));
+}
+
+static rc_t BAM_FileReadI32(BAM_File *self, int32_t *rhs)
+{
+ uint8_t buf[sizeof(int32_t)];
+ rc_t rc = BAM_FileReadn(self, sizeof(int32_t), buf);
+
+ if (rc == 0)
+ *rhs = LE2HI32(buf);
+ return rc;
+}
+
+/* MARK: BAM File header parsing functions */
+
+static unsigned ParseHD(char const **rslt, unsigned const hlen, char hdata[])
+{
+ unsigned i;
+ unsigned tag;
+ unsigned value;
+ int st = 0;
+ int ws = 1;
+
+ for (i = 0; i < hlen; ++i) {
+ char const cc = hdata[i];
+
+ if (ws && isspace(cc))
+ continue;
+ ws = 0;
+
+ switch (st) {
+ case 0:
+ tag = i;
+ ++st;
+ break;
+ case 1:
+ if (isspace(cc))
+ return 0;
+ ++st;
+ break;
+ case 2:
+ if (cc != ':')
+ return 0;
+ hdata[i] = '\0';
+ value = i + 1;
+ ++st;
+ break;
+ case 3:
+ if (cc == '\t' || cc == '\r' || cc == '\n') {
+ hdata[i] = '\0';
+
+ if (strcmp(&hdata[tag], "VN") == 0)
+ *rslt = &hdata[value];
+
+ ++st;
+ ws = 1;
+ }
+ break;
+ case 4:
+ if (cc == '@')
+ return i;
+ tag = i;
+ st = 1;
+ break;
+ }
+ }
+ return st == 4 ? i : 0;
+}
+
+static unsigned ParseSQ(BAMRefSeq *rs, unsigned const hlen, char hdata[])
+{
+ unsigned i;
+ unsigned tag;
+ unsigned value;
+ int st = 0;
+ int ws = 1;
+
+ for (i = 0; i < hlen; ++i) {
+ char const cc = hdata[i];
+
+ if (ws && isspace(cc))
+ continue;
+ ws = 0;
+
+ switch (st) {
+ case 0:
+ tag = i;
+ ++st;
+ break;
+ case 1:
+ if (isspace(cc))
+ return 0;
+ ++st;
+ break;
+ case 2:
+#define HACKAMATIC 1
+#if HACKAMATIC
+ if (cc != ':') {
+ if (i + 1 >= hlen || hdata[i+1] != ':')
+ return 0;
+ else
+ ++i;
+ }
+#else
+ if (cc != ':')
+ return 0;
+#endif
+ hdata[i] = '\0';
+ value = i + 1;
+ ++st;
+ break;
+ case 3:
+ if (cc == '\t' || cc == '\r' || cc == '\n') {
+ unsigned j;
+
+ hdata[i] = '\0';
+
+ while (value < i && isspace(hdata[value]))
+ ++value;
+ for (j = i; value < j && isspace(hdata[j - 1]); )
+ hdata[--j] = '\0';
+
+ if (strcmp(&hdata[tag], "SN") == 0)
+ rs->name = &hdata[value];
+ else if (strcmp(&hdata[tag], "LN") == 0)
+ rs->length = strtou64(&hdata[value], NULL, 10);
+ else if (strcmp(&hdata[tag], "AS") == 0)
+ rs->assemblyId = &hdata[value];
+#if HACKAMATIC
+ else if (strcmp(&hdata[tag], "M5") == 0 || strcmp(&hdata[tag], "MD5") == 0)
+#else
+ else if (strcmp(&hdata[tag], "M5") == 0)
+#endif
+#undef HACKAMATIC
+ {
+ unsigned len = j - value;
+
+ if ((hdata[value] == '\'' || hdata[value] == '"') && hdata[value + len - 1] == hdata[value]) {
+ ++value;
+ len -= 2;
+ }
+ if (len == 32) {
+ rs->checksum = &rs->checksum_array[0];
+ for (j = 0; j != 16; ++j) {
+ int const ch1 = toupper(hdata[value + j * 2 + 0]);
+ int const ch2 = toupper(hdata[value + j * 2 + 1]);
+
+ if (isxdigit(ch1) && isxdigit(ch2)) {
+ rs->checksum_array[j] =
+ ((ch1 > '9' ? (ch1 - ('A' - 10)) : (ch1 - '0')) << 4) +
+ (ch2 > '9' ? (ch2 - ('A' - 10)) : (ch2 - '0'));
+ }
+ else {
+ rs->checksum = NULL;
+ break;
+ }
+ }
+ }
+ }
+ else if (strcmp(&hdata[tag], "UR") == 0)
+ rs->uri = &hdata[value];
+ else if (strcmp(&hdata[tag], "SP") == 0)
+ rs->species = &hdata[value];
+
+ ++st;
+ ws = 1;
+ }
+ break;
+ case 4:
+ if (cc == '@')
+ return i;
+ tag = i;
+ st = 1;
+ break;
+ }
+ }
+ return st == 4 ? i : 0;
+}
+
+static unsigned ParseRG(BAMReadGroup *dst, unsigned const hlen, char hdata[])
+{
+ unsigned i;
+ unsigned tag;
+ unsigned value;
+ int st = 0;
+ int ws = 1;
+
+ for (i = 0; i < hlen; ++i) {
+ char const cc = hdata[i];
+
+ if (ws && isspace(cc))
+ continue;
+ ws = 0;
+
+ switch (st) {
+ case 0:
+ tag = i;
+ ++st;
+ break;
+ case 1:
+ if (isspace(cc))
+ return 0;
+ ++st;
+ break;
+ case 2:
+ if (cc != ':')
+ return 0;
+ hdata[i] = '\0';
+ value = i + 1;
+ ++st;
+ break;
+ case 3:
+ if (cc == '\t' || cc == '\r' || cc == '\n') {
+ unsigned j = i;
+
+ hdata[i] = '\0';
+
+ while (value < i && isspace(hdata[value]))
+ ++value;
+ while (value < j && isspace(hdata[j - 1]))
+ hdata[--j] = '\0';
+
+ if ((hdata[value] == '\"' || hdata[value] == '\'') && hdata[value] == hdata[j - 1]) {
+ ++value;
+ hdata[j - 1] = '\0';
+ }
+ if (strcmp(&hdata[tag], "ID") == 0)
+ dst->name = &hdata[value];
+ else if (strcmp(&hdata[tag], "SM") == 0)
+ dst->sample = &hdata[value];
+ else if (strcmp(&hdata[tag], "LB") == 0)
+ dst->library = &hdata[value];
+ else if (strcmp(&hdata[tag], "DS") == 0)
+ dst->description = &hdata[value];
+ else if (strcmp(&hdata[tag], "PU") == 0)
+ dst->unit = &hdata[value];
+ else if (strcmp(&hdata[tag], "PI") == 0)
+ dst->insertSize = &hdata[value];
+ else if (strcmp(&hdata[tag], "CN") == 0)
+ dst->center = &hdata[value];
+ else if (strcmp(&hdata[tag], "DT") == 0)
+ dst->runDate = &hdata[value];
+ else if (strcmp(&hdata[tag], "PL") == 0)
+ dst->platform = &hdata[value];
+
+ ++st;
+ ws = 1;
+ }
+ break;
+ case 4:
+ if (cc == '@')
+ return i;
+ tag = i;
+ st = 1;
+ break;
+ }
+ }
+ return st == 4 ? i : 0;
+}
+
+static bool ParseHeader(BAM_File *self, char hdata[], size_t hlen) {
+ unsigned rg = 0;
+ unsigned sq = 0;
+ unsigned i;
+ unsigned tag;
+ int st = 0;
+ int ws = 1;
+
+ for (i = 0; i < hlen; ++i) {
+ char const cc = hdata[i];
+
+ if (ws && isspace(cc))
+ continue;
+ ws = 0;
+
+ switch (st) {
+ case 0:
+ if (cc == '@')
+ ++st;
+ else
+ return false;
+ break;
+ case 1:
+ if (isspace(cc))
+ return false;
+ tag = i;
+ ++st;
+ break;
+ case 2:
+ if (isspace(cc)) {
+ hdata[i] = '\0';
+ if (i - tag == 2) {
+ if (strcmp(&hdata[tag], "HD") == 0) {
+ unsigned const used = ParseHD(&self->version, hlen - i - 1, &hdata[i + 1]);
+ if (used == 0) return false;
+ i += used;
+ st = 0;
+ break;
+ }
+ if (strcmp(&hdata[tag], "SQ") == 0) {
+ unsigned const used = ParseSQ(&self->refSeq[sq++], hlen - i - 1, &hdata[i + 1]);
+ if (used == 0) return false;
+ i += used;
+ st = 0;
+ break;
+ }
+ if (strcmp(&hdata[tag], "RG") == 0) {
+ unsigned const used = ParseRG(&self->readGroup[rg++], hlen - i - 1, &hdata[i + 1]);
+ if (used == 0) return false;
+ i += used;
+ st = 0;
+ break;
+ }
+ }
+ if (st == 2) {
+ ++st;
+ ws = 0;
+ }
+ }
+ else if (i - tag > 2)
+ ++st;
+ break;
+ case 3:
+ if (cc == '\r' || cc == '\n') {
+ st = 0;
+ ws = 1;
+ }
+ break;
+ }
+ }
+ return true;
+}
+
+static int64_t comp_ReadGroup(const void *A, const void *B, void *ignored) {
+ BAMReadGroup const *const a = A;
+ BAMReadGroup const *const b = B;
+
+ /* make null names sort to the bottom */
+ if (a->name == NULL || b->name == NULL)
+ return 0;
+ if (a->name == NULL)
+ return 1;
+ if (b->name == NULL)
+ return -1;
+
+ /* make empty names sort to the bottom */
+ if (a->name[0] == '\0' || b->name[0] == '\0')
+ return 0;
+ if (a->name[0] == '\0')
+ return 1;
+ if (b->name[0] == '\0')
+ return -1;
+
+ return strcmp(a->name, b->name);
+}
+
+static int64_t comp_RefSeqName(const void *A, const void *B, void *ignored) {
+ BAMRefSeq const *const a = A;
+ BAMRefSeq const *const b = B;
+
+ /* make null names sort to the bottom */
+ if (a->name == NULL || b->name == NULL)
+ return 0;
+ if (a->name == NULL)
+ return 1;
+ if (b->name == NULL)
+ return -1;
+
+ /* make empty names sort to the bottom */
+ if (a->name[0] == '\0' || b->name[0] == '\0')
+ return 0;
+ if (a->name[0] == '\0')
+ return 1;
+ if (b->name[0] == '\0')
+ return -1;
+ {
+ int const cmp = strcmp(a->name, b->name);
+ return cmp != 0 ? cmp : (int64_t)a->id - (int64_t)b->id;
+ }
+}
+
+static unsigned MeasureHeader(unsigned *RG, unsigned *SQ, char const text[])
+{
+ unsigned size;
+ unsigned cur = 0;
+
+ for (size = 0; ; ++size) {
+ int const ch = text[size];
+
+ if (ch == '\0')
+ return size;
+ if (ch == '\n') {
+ cur = 0;
+ continue;
+ }
+ if (++cur == 3) {
+ if (text[size - 1] == 'R' && ch == 'G') {
+ ++*RG;
+ continue;
+ }
+ if (text[size - 1] == 'S' && ch == 'Q') {
+ ++*SQ;
+ continue;
+ }
+ }
+ }
+}
+
+static rc_t ProcessHeaderText(BAM_File *self, char const text[], bool makeCopy)
+{
+ unsigned RG = 0;
+ unsigned SQ = 0;
+ unsigned const size = MeasureHeader(&RG, &SQ, text);
+ unsigned i;
+
+ if (SQ) {
+ self->refSeq = calloc(SQ, sizeof(self->refSeq[0]));
+ if (self->refSeq == NULL)
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ }
+ self->refSeqs = SQ;
+
+ if (RG) {
+ self->readGroup = calloc(RG, sizeof(self->readGroup[0]));
+ if (self->readGroup == NULL)
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ }
+ self->readGroups = RG;
+
+ if (makeCopy) {
+ void *const tmp = malloc(size + 1);
+ if (tmp == NULL)
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ self->header = tmp; /* a const copy of the original */
+ memcpy(tmp, text, size + 1);
+ }
+ else
+ self->header = text;
+ {
+ char *const copy = malloc(size + 1); /* an editable copy */
+ if (copy == NULL)
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ self->headerData1 = copy; /* so it's not leaked */
+ memcpy(copy, text, size + 1);
+
+ {
+ bool const parsed = ParseHeader(self, copy, size);
+ if (!parsed)
+ return RC(rcAlign, rcFile, rcParsing, rcData, rcInvalid);
+ }
+ }
+ for (i = 0; i < self->readGroups; ++i)
+ self->readGroup[i].id = i;
+
+ ksort(self->readGroup, self->readGroups, sizeof(self->readGroup[0]), comp_ReadGroup, NULL);
+
+ /* remove read groups with missing and empty names */
+ for (i = self->readGroups; i != 0; ) {
+ BAMReadGroup const *const rg = &self->readGroup[--i];
+ char const *const name = rg->name;
+ if (name == NULL || name[0] == '\0') {
+ (void)PLOGMSG(klogWarn, (klogWarn, "Read Group #$(rg) is missing ID in SAM header", "rg=%i", (int)rg->id));
+ --self->readGroups;
+ }
+ }
+
+ /* check for duplicate read groups names */
+ for (i = 1; i < self->readGroups; ++i) {
+ BAMReadGroup const *const a = &self->readGroup[i - 1];
+ BAMReadGroup const *const b = &self->readGroup[i - 0];
+
+ if (strcmp(a->name, b->name) == 0) {
+ (void)PLOGMSG(klogWarn, (klogWarn, "Read Groups #$(r1) and #$(r2) have the same ID '$(id)' in SAM header",
+ "r1=%i,r2=%i,id=%s", (int)a->id, (int)b->id, a->name));
+ }
+ }
+
+ /* these id's are temporary, for reporting only
+ * in BAM, they'll get the id from the second part of the header
+ * in SAM, they'll get reassigned in alphabetical order
+ */
+ for (i = 0; i < self->refSeqs; ++i)
+ self->refSeq[i].id = i;
+
+ ksort(self->refSeq, self->refSeqs, sizeof(self->refSeq[0]), comp_RefSeqName, NULL);
+
+ /* remove references with missing and empty names */
+ for (i = self->refSeqs; i != 0; ) {
+ BAMRefSeq const *const ref = &self->refSeq[--i];
+ char const *const name = ref->name;
+ if (name == NULL || name[0] == '\0') {
+ (void)PLOGMSG(klogWarn, (klogWarn, "Reference #$(rg) is missing name in SAM header", "rg=%i", (int)ref->id));
+ --self->refSeqs;
+ }
+ }
+
+ /* check for and remove duplicate reference names */
+ for (i = self->refSeqs; i > 1; --i) {
+ BAMRefSeq *const a = &self->refSeq[i - 2];
+ BAMRefSeq *const b = &self->refSeq[i - 1];
+
+ if (strcmp(a->name, b->name) == 0) {
+ (void)PLOGMSG(klogWarn, (klogWarn, "References #$(r1) and #$(r2) have the same name '$(id)' in SAM header",
+ "r1=%i,r2=%i,id=%s", (int)a->id, (int)b->id, a->name));
+ memmove(a, b, (self->refSeqs - i + 1) * sizeof(self->refSeq[0]));
+ --self->refSeqs;
+ }
+ }
+
+ /* check for zero-length references */
+ for (i = 0; i != self->refSeqs; ++i) {
+ BAMRefSeq const *const rs = &self->refSeq[i];
+
+ if (rs->length == 0)
+ (void)PLOGMSG(klogWarn, (klogWarn, "Reference '$(ref)' has zero length", "ref=%s", rs->name));
+ }
+
+ return 0;
+}
+
+static rc_t ReadMagic(BAM_File *self)
+{
+ uint8_t sig[4];
+ rc_t rc = BAM_FileReadn(self, 4, sig);
+
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("BAM signature: '%c%c%c' %u\n", sig[0], sig[1], sig[2], sig[3]));
+ if (rc == 0 && (sig[0] != 'B' || sig[1] != 'A' || sig[2] != 'M' || sig[3] != 1))
+ rc = RC(rcAlign, rcFile, rcReading, rcHeader, rcBadVersion);
+ return rc;
+}
+
+static rc_t ReadHeaders(BAM_File *self,
+ char **headerText, size_t *headerTextLen,
+ char **refData, unsigned *numrefs)
+{
+ unsigned hlen;
+ char *htxt = NULL;
+ unsigned nrefs;
+ char *rdat = NULL;
+ unsigned rdsz;
+ unsigned rdms;
+ unsigned i;
+ int32_t i32;
+ rc_t rc = BAM_FileReadI32(self, &i32);
+
+ if (rc) return rc;
+
+ if (i32 < 0) {
+ rc = RC(rcAlign, rcFile, rcReading, rcHeader, rcInvalid);
+ goto BAILOUT;
+ }
+ hlen = i32;
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("BAM Header text size: %u\n", hlen));
+
+ htxt = malloc(hlen + 1);
+ if (htxt == NULL) {
+ rc = RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+ goto BAILOUT;
+ }
+
+ rc = BAM_FileReadn(self, hlen, (uint8_t *)htxt); if (rc) goto BAILOUT;
+ htxt[hlen] = '\0';
+
+ rc = BAM_FileReadI32(self, &i32); if (rc) goto BAILOUT;
+ if (i32 < 0) {
+ rc = RC(rcAlign, rcFile, rcReading, rcHeader, rcInvalid);
+ goto BAILOUT;
+ }
+ nrefs = i32;
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("BAM Header reference count: %u\n", nrefs));
+ if (nrefs) {
+ rdms = nrefs * 16;
+ if (rdms < 4096)
+ rdms = 4096;
+ rdat = malloc(rdms);
+ if (rdat == NULL) {
+ rc = RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+ goto BAILOUT;
+ }
+ for (i = rdsz = 0; i < nrefs; ++i) {
+ rc = BAM_FileReadI32(self, &i32); if (rc) goto BAILOUT;
+ if (i32 <= 0) {
+ rc = RC(rcAlign, rcFile, rcReading, rcHeader, rcInvalid);
+ goto BAILOUT;
+ }
+ if (rdsz + i32 + 8 > rdms) {
+ void *tmp;
+
+ do { rdms <<= 1; } while (rdsz + i32 + 8 > rdms);
+ tmp = realloc(rdat, rdms);
+ if (tmp == NULL) {
+ rc = RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+ goto BAILOUT;
+ }
+ rdat = tmp;
+ }
+ memcpy(rdat + rdsz, &i32, 4);
+ rdsz += 4;
+ rc = BAM_FileReadn(self, i32, (uint8_t *)&rdat[rdsz]); if (rc) goto BAILOUT;
+ rdsz += i32;
+ rc = BAM_FileReadI32(self, &i32); if (rc) goto BAILOUT;
+ memcpy(rdat + rdsz, &i32, 4);
+ rdsz += 4;
+ }
+ }
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("BAM Header reference size: %u\n", rdsz));
+
+ *headerText = htxt;
+ *headerTextLen = hlen;
+ *refData = rdat;
+ *numrefs = nrefs;
+ return 0;
+
+BAILOUT:
+ if (htxt)
+ free(htxt);
+ if (rdat)
+ free(rdat);
+
+ return rc;
+}
+
+static unsigned FindRefSeqByName(char const name[], bool match, unsigned const N, BAMRefSeq const refSeq[])
+{
+ unsigned f = 0;
+ unsigned e = N;
+
+ while (f < e) {
+ unsigned const m = f + ((e - f) >> 1);
+ int const cmp = strcmp(name, refSeq[m].name);
+
+ if (cmp < 0)
+ e = m;
+ else if (cmp > 0)
+ f = m + 1;
+ else
+ return m;
+ }
+ return match ? N : f;
+}
+
+static void FindAndSetupRefSeq(BAMRefSeq *rs, unsigned const refSeqs, BAMRefSeq const refSeq[])
+{
+ unsigned const fnd = FindRefSeqByName(rs->name, true, refSeqs, refSeq);
+ if (fnd != refSeqs) {
+ rs->assemblyId = refSeq[fnd].assemblyId;
+ rs->uri = refSeq[fnd].uri;
+ rs->species = refSeq[fnd].species;
+ if (refSeq[fnd].checksum) {
+ rs->checksum = &rs->checksum_array[0];
+ memcpy(rs->checksum_array, refSeq[fnd].checksum_array, 16);
+ }
+ else
+ rs->checksum = NULL;
+ }
+}
+
+static rc_t ProcessBAMHeader(BAM_File *self, char const headerText[])
+{
+ unsigned i;
+ unsigned cp;
+ char *htxt;
+ char *rdat;
+ size_t hlen;
+ unsigned nrefs;
+ BAMRefSeq *refSeq;
+ rc_t rc = ReadMagic(self);
+
+ if (rc) return rc;
+
+ rc = ReadHeaders(self, &htxt, &hlen, &rdat, &nrefs);
+ if (rc) return rc;
+
+ self->fpos_first = self->fpos_cur;
+ self->ucfirst = self->bufCurrent;
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("BAM Data records start at: %lu+%u\n", self->ucfirst, self->fpos_first));
+
+ if (nrefs) {
+ refSeq = calloc(nrefs, sizeof(self->refSeq[0]));
+ if (refSeq == NULL)
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ }
+
+ if (headerText) {
+ free(htxt);
+ rc = ProcessHeaderText(self, headerText, true);
+ }
+ else
+ rc = ProcessHeaderText(self, htxt, false);
+
+ if (rc) return rc;
+
+ for (i = cp = 0; i < nrefs; ++i) {
+ unsigned const nlen = LE2HUI32(rdat + cp);
+ char *const name = rdat + cp + 4;
+ unsigned const rlen = LE2HUI32(rdat + cp + nlen + 4);
+
+ cp += nlen + 8;
+ name[nlen] = '\0';
+
+ refSeq[i].id = i;
+ refSeq[i].name = name;
+ refSeq[i].length = rlen;
+ FindAndSetupRefSeq(&refSeq[i], self->refSeqs, self->refSeq);
+ }
+ free(self->refSeq);
+ self->refSeq = refSeq;
+ self->refSeqs = nrefs;
+ self->headerData2 = rdat; /* so it's not leaked */
+
+ return 0;
+}
+
+static rc_t ProcessSAMHeader(BAM_File *self, char const substitute[])
+{
+ SAMFile *const file = &self->file.sam;
+ size_t headerSize = 0;
+ char *headerText = NULL;
+ rc_t rc;
+ int st = 0;
+
+ for ( ; ; ) {
+ void *const tmp = headerText;
+ int const ch = SAMFileRead1(file);
+
+ if (ch < 0)
+ return SAMFileLastError(file);
+
+ if (st == 0) {
+ if (ch != '@') {
+ SAMFilePutBack(file, ch);
+ break;
+ }
+ st = 1;
+ }
+ headerText = realloc(headerText, headerSize + 2);
+ if (headerText == NULL) {
+ free(tmp);
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+ }
+ headerText[headerSize++] = ch;
+ headerText[headerSize] = '\0';
+ if (ch == '\n')
+ st = 0;
+ }
+
+ if (substitute)
+ rc = ProcessHeaderText(self, substitute, true);
+ else if (headerText)
+ rc = ProcessHeaderText(self, headerText, false);
+ else {
+ rc = RC(rcAlign, rcFile, rcConstructing, rcHeader, rcNotFound);
+ (void)LOGERR(klogErr, rc, "SAM header required");
+ }
+ if (rc == 0) {
+ unsigned i;
+
+ for (i = 0; i < self->refSeqs; ++i)
+ self->refSeq[i].id = i;
+ }
+ return rc;
+}
+
+/* MARK: BAM File destructor */
+
+static void BAM_FileWhack(BAM_File *self) {
+ if (self->refSeq)
+ free(self->refSeq);
+ if (self->readGroup)
+ free(self->readGroup);
+ if (self->header)
+ free((void *)self->header);
+ if (self->headerData1)
+ free((void *)self->headerData1);
+ if (self->headerData2)
+ free((void *)self->headerData2);
+ if (self->nocopy)
+ free(self->nocopy);
+ if (self->vt.FileWhack)
+ self->vt.FileWhack(&self->file);
+ BufferedFileWhack(&self->file.bam.file);
+}
+
+/* MARK: BAM File constructors */
+
+/* file is retained */
+static rc_t BAM_FileMakeWithKFileAndHeader(BAM_File const **cself,
+ KFile const *file,
+ char const *headerText)
+{
+ BAM_File *self = calloc(1, sizeof(*self));
+ rc_t rc;
+
+ if (self == NULL)
+ return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
+
+ rc = BufferedFileInit(&self->file.bam.file, file);
+ if (rc) {
+ free(self);
+ return rc;
+ }
+
+ rc = BGZFileInit(&self->file.bam, &self->vt);
+ if (rc == 0) {
+ rc = ProcessBAMHeader(self, headerText);
+ if (rc == 0) {
+ *cself = self;
+ return 0;
+ }
+ }
+ BGZFileWhack(&self->file.bam);
+
+ self->file.sam.file.bpos = 0;
+ rc = SAMFileInit(&self->file.sam, &self->vt);
+ if (rc == 0) {
+ self->isSAM = true;
+ rc = ProcessSAMHeader(self, headerText);
+ if (rc == 0) {
+ *cself = self;
+ return 0;
+ }
+ }
+ BufferedFileWhack(&self->file.sam.file);
+ free(self);
+
+ return rc;
+}
+
+rc_t BAM_FileMakeWithHeader(const BAM_File **cself,
+ char const headerText[],
+ char const path[], ... )
+{
+ KDirectory *dir;
+ va_list args;
+ rc_t rc;
+ const KFile *kf;
+
+ if (cself == NULL)
+ return RC(rcAlign, rcFile, rcOpening, rcParam, rcNull);
+ *cself = NULL;
+
+ rc = KDirectoryNativeDir(&dir);
+ if (rc) return rc;
+ va_start(args, path);
+ rc = KDirectoryVOpenFileRead(dir, &kf, path, args);
+ if (rc == 0) {
+ rc = BAM_FileMakeWithKFileAndHeader(cself, kf, headerText);
+ KFileRelease(kf);
+ }
+ va_end(args);
+ KDirectoryRelease(dir);
+ return rc;
+}
+
+/* MARK: BAM File ref-counting */
+
+rc_t BAM_FileAddRef(const BAM_File *cself) {
+ return 0;
+}
+
+rc_t BAM_FileRelease(const BAM_File *cself) {
+ BAM_File *self = (BAM_File *)cself;
+
+ if (cself != NULL) {
+ BAM_FileWhack(self);
+ free(self);
+ }
+ return 0;
+}
+
+/* MARK: BAM File positioning */
+
+float BAM_FileGetProportionalPosition(const BAM_File *self)
+{
+ return self->vt.FileProPos(&self->file);
+}
+
+rc_t BAM_FileGetPosition(const BAM_File *self, BAM_FilePosition *pos) {
+ *pos = (self->fpos_cur << 16) | self->bufCurrent;
+ return 0;
+}
+
+static void BAM_FileAdvance(BAM_File *const self, unsigned distance)
+{
+ self->bufCurrent += distance;
+ if (self->bufCurrent == self->bufSize) {
+ self->fpos_cur = self->vt.FileGetPos(&self->file);
+ self->bufCurrent = 0;
+ self->bufSize = 0;
+ }
+}
+
+/* MARK: BAM Alignment contruction */
+
+static int TagTypeSize(int const type)
+{
+ switch (type) {
+ case dt_ASCII: /* A */
+ case dt_INT8: /* c */
+ case dt_UINT8: /* C */
+ return 1;
+
+ case dt_INT16: /* s */
+ case dt_UINT16: /* S */
+ return 2;
+
+ case dt_INT: /* i */
+ case dt_UINT: /* I */
+ case dt_FLOAT32: /* f */
+ return 4;
+#if 0
+ case dt_FLOAT64: /* d */
+ return 8;
+#endif
+ case dt_CSTRING: /* Z */
+ case dt_HEXSTRING: /* H */
+ return -'S';
+
+ case dt_NUM_ARRAY: /* B */
+ return -'A';
+ }
+ return 0;
+}
+
+static void ColorCheck(BAM_Alignment *const self, char const tag[2], unsigned const len)
+{
+ if (tag[0] == 'C' && len != 0) {
+ int const ch = tag[1];
+ int const flag = ch == 'Q' ? 2 : ch == 'S' ? 1 : 0;
+
+ if (flag)
+ self->hasColor ^= (len << 2) | flag;
+ }
+}
+
+static rc_t ParseOptData(BAM_Alignment *const self, size_t const maxsize,
+ size_t const xtra, size_t const datasize)
+{
+ size_t const maxExtra = (maxsize - (sizeof(*self) - sizeof(self->extra))) / sizeof(self->extra[0]);
+ char const *const base = (char const *)self->data->raw;
+ unsigned i = 0;
+ unsigned len;
+ unsigned offset;
+
+ self->numExtra = 0;
+ for (len = 0, offset = (unsigned)xtra; offset < datasize; offset += len) {
+ int const valuelen1 = TagTypeSize(base[offset + 2]);
+ unsigned valuelen;
+
+ if (valuelen1 < 0) {
+ char const *const value = &base[offset + 3];
+
+ if (-valuelen1 == 'S') {
+ valuelen = 0;
+ while (value[valuelen] != '\0') {
+ ++valuelen;
+ if (offset + valuelen >= datasize) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ return rc;
+ }
+ }
+ ColorCheck(self, base + offset, valuelen);
+ ++valuelen;
+ }
+ else {
+ int const elem_size = TagTypeSize(value[0]);
+
+ assert(-valuelen1 == 'A');
+ if (elem_size <= 0) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcUnexpected);
+ return rc;
+ }
+ else {
+ int const elem_count = LE2HI32(&value[1]);
+
+ valuelen = elem_size * elem_count + 1 + 4;
+ if (offset + valuelen >= datasize) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ return rc;
+ }
+ }
+ }
+ }
+ else if (valuelen1 == 0) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcUnexpected);
+ return rc;
+ }
+ else
+ valuelen = valuelen1;
+
+ len = valuelen + 3;
+ if (i < maxExtra) {
+ self->extra[i].offset = offset;
+ self->extra[i].size = len;
+ }
+ ++i;
+ }
+ self->numExtra = i;
+ if (2 <= i && i <= maxExtra)
+ ksort(self->extra, i, sizeof(self->extra[0]), OptTag_sort, self);
+
+ return 0;
+}
+
+static rc_t ParseOptDataLog(BAM_Alignment *const self, unsigned const maxsize,
+ unsigned const xtra, unsigned const datasize)
+{
+ unsigned const maxExtra = (maxsize - (sizeof(*self) - sizeof(self->extra))) / sizeof(self->extra[0]);
+ char const *const base = (char const *)self->data->raw;
+ unsigned i = 0;
+ unsigned len;
+ unsigned offset;
+
+ self->numExtra = 0;
+ for (len = 0, offset = (unsigned)xtra; offset < datasize; offset += len) {
+ int const type = base[offset + 2];
+ int const valuelen1 = TagTypeSize(type);
+ unsigned valuelen;
+
+ if (valuelen1 > 0)
+ valuelen = valuelen1;
+ else if (valuelen1 < 0) {
+ char const *const value = &base[offset + 3];
+
+ if (-valuelen1 == 'S') {
+ valuelen = 0;
+ while (value[valuelen] != '\0') {
+ ++valuelen;
+ if (offset + valuelen >= datasize) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ (void)LOGERR(klogErr, rc,
+ "Parsing BAM optional fields: "
+ "unterminated string");
+ return rc;
+ }
+ }
+ ColorCheck(self, base + offset, valuelen);
+ ++valuelen;
+ }
+ else {
+ int const elem_type = value[0];
+ int const elem_size = TagTypeSize(elem_type);
+
+ assert(-valuelen1 == 'A');
+ if (elem_size <= 0) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcUnexpected);
+ (void)LOGERR(klogErr, rc,
+ "Parsing BAM optional fields: "
+ "unknown array type");
+ return rc;
+ }
+ else {
+ int const elem_count = LE2HI32(&value[1]);
+
+ valuelen = elem_size * elem_count + 1 + 4;
+ if (offset + valuelen >= datasize) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ (void)LOGERR(klogErr, rc,
+ "Parsing BAM optional fields: "
+ "array too big");
+ return rc;
+ }
+ }
+ }
+ }
+ else {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcData, rcUnexpected);
+ (void)LOGERR(klogErr, rc,
+ "Parsing BAM optional fields: "
+ "unknown type");
+ return rc;
+ }
+
+ len = valuelen + 3;
+ if (i < maxExtra) {
+ self->extra[i].offset = offset;
+ self->extra[i].size = len;
+ }
+ ++i;
+ }
+ self->numExtra = i;
+ if (2 <= i && i <= maxExtra)
+ ksort(self->extra, i, sizeof(self->extra[0]), OptTag_sort, self);
+
+ return 0;
+}
+
+static unsigned BAM_AlignmentSize(unsigned const max_extra_tags)
+{
+ BAM_Alignment const *const y = NULL;
+
+ return sizeof(*y) + (max_extra_tags ? max_extra_tags - 1 : 0) * sizeof(y->extra);
+}
+
+static unsigned BAM_AlignmentSetOffsets(BAM_Alignment *const self)
+{
+ unsigned const nameLen = getReadNameLength(self);
+ unsigned const cigCnt = getCigarCount(self);
+ unsigned const readLen = getReadLen(self);
+ unsigned const cigar = (unsigned)(&((struct bam_alignment_s const *)NULL)->read_name[nameLen] - (const char *)NULL);
+ unsigned const seq = cigar + 4 * cigCnt;
+ unsigned const qual = seq + (readLen + 1) / 2;
+ unsigned const xtra = qual + readLen;
+
+ self->cigar = cigar;
+ self->seq = seq;
+ self->qual = qual;
+
+ return xtra;
+}
+
+static bool BAM_AlignmentInit(BAM_Alignment *const self, unsigned const maxsize,
+ unsigned const datasize, void const *const data)
+{
+ memset(self, 0, sizeof(*self));
+ self->data = data;
+ self->datasize = datasize;
+ {
+ unsigned const xtra = BAM_AlignmentSetOffsets(self);
+
+ if ( datasize >= xtra
+ && datasize >= self->cigar
+ && datasize >= self->seq
+ && datasize >= self->qual)
+ {
+ rc_t const rc = ParseOptData(self, maxsize, xtra, datasize);
+
+ if (rc == 0)
+ return true;
+ }
+ return false;
+ }
+}
+
+static bool BAM_AlignmentInitLog(BAM_Alignment *const self, unsigned const maxsize,
+ unsigned const datasize, void const *const data)
+{
+ memset(self, 0, sizeof(*self));
+ self->data = data;
+ self->datasize = datasize;
+ {
+ unsigned const xtra = BAM_AlignmentSetOffsets(self);
+
+ if ( datasize >= xtra
+ && datasize >= self->cigar
+ && datasize >= self->seq
+ && datasize >= self->qual)
+ {
+ rc_t const rc = ParseOptDataLog(self, maxsize, xtra, datasize);
+
+ if (rc == 0) {
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("{"
+ "\"BAM record\": "
+ "{ "
+ "\"size\": %u, "
+ "\"name length\": %u, "
+ "\"cigar count\": %u, "
+ "\"read length\": %u, "
+ "\"extra count\": %u "
+ "}"
+ "}\n",
+ (unsigned)datasize,
+ (unsigned)getReadNameLength(self),
+ (unsigned)getCigarCount(self),
+ (unsigned)getReadLen(self),
+ (unsigned)self->numExtra));
+ return true;
+ }
+ }
+ DBGMSG(DBG_ALIGN, DBG_FLAG(DBG_ALIGN_BAM), ("{"
+ "\"BAM record\": "
+ "{ "
+ "\"size\": %u, "
+ "\"name length\": %u, "
+ "\"cigar count\": %u, "
+ "\"read length\": %u "
+ "}"
+ "}\n",
+ (unsigned)datasize,
+ (unsigned)getReadNameLength(self),
+ (unsigned)getCigarCount(self),
+ (unsigned)getReadLen(self)));
+ return false;
+ }
+}
+
+static void BAM_AlignmentLogParseError(BAM_Alignment const *self)
+{
+ char const *const reason = self->cigar > self->datasize ? "BAM Record CIGAR too long"
+ : self->seq > self->datasize ? "BAM Record SEQ too long"
+ : self->qual > self->datasize ? "BAM Record QUAL too long"
+ : self->qual + getReadLen(self) > self->datasize ? "BAM Record EXTRA too long"
+ : "BAM Record EXTRA parsing failure";
+
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), reason);
+}
+
+/* MARK: BAM Alignment readers */
+
+/* returns
+ * (rcAlign, rcFile, rcReading, rcBuffer, rcNotAvailable)
+ * or
+ * (rcAlign, rcFile, rcReading, rcBuffer, rcInsufficient)
+ * if should read with copy
+ */
+static
+rc_t BAM_FileReadNoCopy(BAM_File *const self, unsigned actsize[], BAM_Alignment rhs[],
+ unsigned const maxsize)
+{
+ unsigned const maxPeek = BAM_FileMaxPeek(self);
+ bool isgood;
+
+ *actsize = 0;
+
+ if (maxPeek == 0) {
+ rc_t const rc = BAM_FileFillBuffer(self);
+
+ if (rc == 0)
+ return BAM_FileReadNoCopy(self, actsize, rhs, maxsize);
+
+ if ( GetRCObject( rc ) == (enum RCObject)rcData && GetRCState( rc ) == rcInsufficient )
+ {
+ self->eof = true;
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcNotFound);
+ }
+ return rc;
+ }
+ if (maxPeek < 4)
+ return RC(rcAlign, rcFile, rcReading, rcBuffer, rcNotAvailable);
+ else {
+ int32_t const i32 = BAM_FilePeekI32(self);
+
+ if (i32 <= 0)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+
+ if (maxPeek < i32 + 4)
+ return RC(rcAlign, rcFile, rcReading, rcBuffer, rcNotAvailable);
+
+ isgood = BAM_AlignmentInitLog(rhs, maxsize, i32, BAM_FilePeek(self, 4));
+ rhs[0].parent = self;
+ }
+ *actsize = BAM_AlignmentSize(rhs[0].numExtra);
+ if (isgood && *actsize > maxsize)
+ return RC(rcAlign, rcFile, rcReading, rcBuffer, rcInsufficient);
+
+ BAM_FileAdvance(self, 4 + rhs->datasize);
+ return isgood ? 0 : RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid);
+}
+
+static
+unsigned BAM_AlignmentSizeFromData(unsigned const datasize, void const *data)
+{
+ BAM_Alignment temp;
+
+ BAM_AlignmentInit(&temp, sizeof(temp), datasize, data);
+
+ return BAM_AlignmentSize(temp.numExtra);
+}
+
+static bool BAM_AlignmentIsEmpty(BAM_Alignment const *const self)
+{
+ if (getReadNameLength(self) == 0)
+ return true;
+ if (self->hasColor == 3)
+ return false;
+ if (getReadLen(self) != 0)
+ return false;
+ if (getCigarCount(self) != 0)
+ return false;
+ return true;
+}
+
+static
+rc_t BAM_FileReadCopy(BAM_File *const self, BAM_Alignment const *rslt[], bool const log)
+{
+ void *storage;
+ void const *data;
+ unsigned datasize;
+ rc_t rc;
+
+ rslt[0] = NULL;
+ {
+ int32_t i32;
+
+ rc = BAM_FileReadI32(self, &i32);
+ if ( rc != 0 )
+ {
+ if ( GetRCObject( rc ) == (enum RCObject)rcData && GetRCState( rc ) == rcInsufficient )
+ {
+ self->eof = true;
+ rc = RC( rcAlign, rcFile, rcReading, rcRow, rcNotFound );
+ }
+ return rc;
+ }
+ if (i32 <= 0)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+
+ datasize = i32;
+ }
+ if (BAM_FileMaxPeek(self) < datasize) {
+ data = storage = malloc(datasize);
+ if (storage == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+
+ rc = BAM_FileReadn(self, datasize, storage);
+ }
+ else {
+ storage = NULL;
+ data = (bam_alignment *)&self->buffer[self->bufCurrent];
+
+ BAM_FileAdvance(self, datasize);
+ }
+ if (rc == 0) {
+ unsigned const rsltsize = BAM_AlignmentSizeFromData(datasize, data);
+ BAM_Alignment *const y = malloc(rsltsize);
+
+ if (y) {
+ if ((log ? BAM_AlignmentInitLog : BAM_AlignmentInit)(y, rsltsize, datasize, data)) {
+ if (storage == NULL)
+ self->bufLocker = y;
+ else
+ y->storage = storage;
+
+ y->parent = self;
+ rslt[0] = y;
+
+ if (BAM_AlignmentIsEmpty(y))
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcEmpty);
+ return 0;
+ }
+ rc = RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid);
+ free(y);
+ }
+ else
+ rc = RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+ }
+ free(storage);
+
+ return rc;
+}
+
+static
+rc_t BAM_FileBreakLock(BAM_File *const self)
+{
+ if (self->bufLocker != NULL) {
+ if (self->bufLocker->storage == NULL)
+ self->bufLocker->storage = malloc(self->bufLocker->datasize);
+ if (self->bufLocker->storage == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+
+ memcpy(self->bufLocker->storage, self->bufLocker->data, self->bufLocker->datasize);
+ self->bufLocker->data = (bam_alignment *)&self->bufLocker->storage[0];
+ self->bufLocker = NULL;
+ }
+ return 0;
+}
+
+/* MARK: SAM code */
+
+static void SAM2BAM_ConvertShort(void *const Dst, int value)
+{
+ uint8_t *const dst = Dst;
+ dst[0] = (uint8_t)(value >> 0);
+ dst[1] = (uint8_t)(value >> 8);
+}
+
+static void SAM2BAM_ConvertInt(void *const Dst, int value)
+{
+ uint8_t *const dst = Dst;
+ dst[0] = (uint8_t)(value >> 0);
+ dst[1] = (uint8_t)(value >> 8);
+ dst[2] = (uint8_t)(value >> 16);
+ dst[3] = (uint8_t)(value >> 24);
+}
+
+static int SAM2BAM_CIGAR_OpCount(char const cigar[])
+{
+ unsigned i;
+ unsigned n = 0;
+ int st = 0;
+
+ if (cigar[0] == '*' && cigar[1] == '\0')
+ return 0;
+
+ for (i = 0; ; ++i) {
+ int const ch = cigar[i];
+
+ if (ch == '\0')
+ break;
+
+ if (st == 0) {
+ if (!isdigit(ch))
+ return -1;
+ st = 1;
+ }
+ else if (!isdigit(ch)) {
+ ++n;
+ st = 0;
+ }
+ }
+ return st == 0 ? n : -1;
+}
+
+static int SAM2BAM_ConvertCIGAR1(uint8_t dst[4], char const value[])
+{
+ int len = 0;
+ int code = 0;
+ unsigned i = 0;
+
+ if (value[0] == '\0')
+ return 0;
+
+ for ( ; ; ) {
+ int const ch = value[i++];
+
+ if (!isdigit(ch)) {
+ switch (ch) {
+ case 'M':
+ code = 0;
+ break;
+ case 'I':
+ code = 1;
+ break;
+ case 'D':
+ code = 2;
+ break;
+ case 'N':
+ code = 3;
+ break;
+ case 'S':
+ code = 4;
+ break;
+ case 'H':
+ code = 5;
+ break;
+ case 'P':
+ code = 6;
+ break;
+ case '=':
+ code = 7;
+ break;
+ case 'X':
+ code = 8;
+ break;
+ default:
+ return -1;
+ }
+ break;
+ }
+ len = (len * 10) + (ch - '0');
+ if (len >= (1 << 24))
+ return -2;
+ }
+ SAM2BAM_ConvertInt(dst, (len << 4) | code);
+ return i;
+}
+
+static rc_t SAM2BAM_ConvertCIGAR(unsigned const insize, void /* inout */ *const data, void const *const endp)
+{
+ char *const value = data;
+ uint8_t *const dst = (void *)(value + insize);
+ unsigned j = 0;
+ unsigned i;
+
+ for (i = 0; i < insize; ++j) {
+ if ((void const *)(dst + j * 4 + 4) >= endp)
+ return RC(rcAlign, rcFile, rcReading, rcBuffer, rcInsufficient);
+ {
+ int const k = SAM2BAM_ConvertCIGAR1(dst + j * 4, value + i);
+ if (k > 0)
+ i += k;
+ else
+ return k == 0 ? 0 : RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ }
+ }
+ memmove(data, dst, 4 * j);
+ return 0;
+}
+
+static int SAM2BAM_ConvertBase(int base)
+{
+ static char const tr[] = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,0x0, -1, -1,
+ -1,0x1,0xE,0x2,0xD, -1, -1,0x4,0xB, -1, -1,0xC, -1,0x3,0xF, -1,
+ -1, -1,0x5,0x6,0x8, -1,0x7,0x9, -1,0xA, -1, -1, -1, -1, -1, -1,
+ -1,0x1,0xE,0x2,0xD, -1, -1,0x4,0xB, -1, -1,0xC, -1,0x3,0xF, -1,
+ -1, -1,0x5,0x6,0x8, -1,0x7,0x9, -1,0xA, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ };
+ return tr[base];
+}
+
+static rc_t SAM2BAM_ConvertSEQ(unsigned const insize, void /* inout */ *const data, void const *const endp)
+{
+ char const *const value = data;
+ uint8_t *const dst = data;
+ unsigned const n = insize & ~((unsigned)1);
+ unsigned j = 0;
+ unsigned i;
+
+ for (i = 0; i < n; i += 2, ++j) {
+ int const hi = SAM2BAM_ConvertBase(value[i + 0]);
+ int const lo = SAM2BAM_ConvertBase(value[i + 1]);
+
+ if (hi < 0 || lo < 0)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+
+ dst[j] = (hi << 4) | lo;
+ }
+ if (n != insize) {
+ int const hi = SAM2BAM_ConvertBase(value[n]);
+ int const lo = 0;
+
+ if (hi < 0 || lo < 0)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+
+ dst[j] = (hi << 4) | lo;
+ }
+ return 0;
+}
+
+static rc_t SAM2BAM_ConvertQUAL(unsigned const insize, void /* inout */ *const data, void const *const endp)
+{
+ char const *const value = data;
+ uint8_t *const dst = data;
+ unsigned i;
+
+ for (i = 0; i < insize; ++i) {
+ int const ch = value[i];
+
+ if (ch < '!' || ch > '~')
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ dst[i] = ch - 33;
+ }
+ return 0;
+}
+
+static int SAM2BAM_ScanValue(void *const dst, char const *src, bool isFloat, bool isArray)
+{
+ static char const *Fmt[] = {
+ ",%f%n",
+ ",%i%n"
+ };
+ char const *const fmt = Fmt[isFloat ? 0 : 1] + (isArray ? 0 : 1);
+ union { int i; float f; } x;
+ int n = 0;
+
+ x.i = 0;
+ if (sscanf(src, fmt, &x, &n) != 1)
+ return -1;
+ SAM2BAM_ConvertInt(dst, x.i);
+ return n;
+}
+
+static int SAM2BAM_ConvertEXTRA(unsigned const insize, void /* inout */ *const data, void const *const endp)
+{
+ if (insize < 5) /* XX:T:\0 */
+ return -1;
+ {
+ char const *const src = data;
+
+ if (src[2] != ':' || src[4] != ':')
+ return -3;
+ {
+ int const type = src[3];
+ char *const dst = data;
+
+ dst[2] = type;
+
+ switch (type) {
+ case 'A':
+ dst[3] = src[5];
+ return 4;
+ case 'H':
+ case 'Z':
+ memmove(dst + 3, src + 5, insize - 5);
+ dst[insize - 2] = '\0';
+ return insize - 1;
+ case 'i':
+ case 'f': {
+ if ((void const *)&dst[7] >= endp)
+ return -2;
+ {
+ int const n = SAM2BAM_ScanValue(&dst[3], src + 5, type == 'f', false);
+ return (n < 0 || n + 5 != insize) ? -4 : 7;
+ }
+ }
+ case 'B':
+ break;
+ default:
+ return -3;
+ }
+
+ if (insize < 8) /* XX:B:T,x\0 */
+ return -1;
+
+ switch (src[5]) {
+ case 'c':
+ case 'C':
+ case 's':
+ case 'S':
+ case 'i':
+ case 'I':
+ case 'f':
+ break;
+ default:
+ return -3;
+ }
+ {
+ uint8_t *scratch = (void *)(src + insize);
+ int const subtype = src[5] == 'f' ? 'f' : 'i';
+ unsigned i;
+
+ dst[3] = subtype;
+ for (i = 6; i < insize; ) {
+ if ((void const *)&scratch[4] >= endp)
+ return -2;
+ {
+ int const n = SAM2BAM_ScanValue(scratch, src + 5, subtype == 'f', true);
+ if (n < 0)
+ return -4;
+ i += n;
+ }
+ scratch += 4;
+ }
+ {
+ unsigned const written = scratch - (uint8_t const *)(src + insize);
+ memmove(dst + 4, src + insize, written);
+ return written + 5;
+ }
+ }
+ }
+ }
+}
+
+static rc_t BAM_FileReadSAM(BAM_File *const self, BAM_Alignment const **const rslt)
+{
+ void const *const endp = self->buffer + sizeof(self->buffer);
+ struct bam_alignment_s *raw = (void *)self->buffer;
+ struct {
+ int namelen;
+ int FLAG;
+ int RNAME;
+ int POS;
+ int MAPQ;
+ int cigars;
+ int RNEXT;
+ int PNEXT;
+ int TLEN;
+
+ int readlen;
+
+ char *QNAME;
+ uint32_t *CIGAR; /* probably not aligned */
+ uint8_t *SEQ;
+ uint8_t *QUAL;
+ char *EXTRA;
+ } temp;
+ unsigned field = 1;
+ char *scratch;
+ unsigned i = 0;
+ int n = 0;
+
+ memset(raw, 0, sizeof(*raw));
+ memset(&temp, 0, sizeof(temp));
+ scratch = temp.QNAME = &raw->read_name[0];
+
+ for ( ; ; ) {
+ int const ch = SAMFileRead1(&self->file.sam);
+ if (ch < 0) {
+ rc_t const rc = SAMFileLastError(&self->file.sam);
+ return (i == 0 && field == 1 && (rc == 0 || GetRCState(rc) == rcInsufficient)) ? RC(rcAlign, rcFile, rcReading, rcRow, rcNotFound) : rc;
+ }
+ if ((void const *)&scratch[i] >= endp)
+ return RC(rcAlign, rcFile, rcReading, rcBuffer, rcInsufficient);
+
+ if (!(ch == '\t' || ch == '\n')) {
+ if (field != 0)
+ scratch[i++] = ch;
+ continue;
+ }
+ scratch[i] = '\0';
+
+ switch (field) {
+ case 0:
+ if (ch == '\n')
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ break;
+ case 1:
+ if (i == 1 && scratch[0] == '*') {
+ temp.namelen = 0;
+ }
+ else {
+ temp.namelen = i + 1; /* includes NULL terminator */
+ scratch += i + 1; /* don't want to overwrite it */
+ if (temp.namelen > 255) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error QNAME is too long");
+ field = 0;
+ }
+ }
+ temp.CIGAR = (void *)scratch;
+ break;
+ case 2:
+ if (sscanf(scratch, "%i%n", &temp.FLAG, &n) != 1 || n != i) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing FLAG");
+ field = 0;
+ }
+ break;
+ case 3:
+ if (i == 1 && scratch[0] == '*')
+ temp.RNAME = -1;
+ else {
+ unsigned const id = FindRefSeqByName(scratch, true, self->refSeqs, self->refSeq);
+ if (id < self->refSeqs)
+ temp.RNAME = id;
+ else {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record missing reference");
+ field = 0;
+ }
+ }
+ break;
+ case 4:
+ if (sscanf(scratch, "%i%n", &temp.POS, &n) != 1 || n != i) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing POS");
+ field = 0;
+ }
+ break;
+ case 5:
+ if (sscanf(scratch, "%i%n", &temp.MAPQ, &n) != 1 || n != i) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing MAPQ");
+ field = 0;
+ }
+ if (temp.MAPQ > 255) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error MAPQ > 255");
+ field = 0;
+ }
+ break;
+ case 6:
+ temp.cigars = SAM2BAM_CIGAR_OpCount(scratch);
+ if (temp.cigars < 0) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing CIGAR");
+ field = 0;
+ }
+ else if (temp.cigars > 0) {
+ scratch += 4 * temp.cigars;
+ temp.SEQ = (uint8_t *)scratch;
+ {
+ rc_t const rc = SAM2BAM_ConvertCIGAR(i, temp.CIGAR, endp);
+ if (rc) {
+ LOGERR(klogErr, rc, "SAM Record error parsing CIGAR");
+ field = 0;
+ }
+ }
+ }
+ else
+ temp.SEQ = (uint8_t *)scratch;
+ break;
+ case 7:
+ if (i == 1 && scratch[0] == '*')
+ temp.RNEXT = -1;
+ else if (i == 1 && scratch[0] == '=')
+ temp.RNEXT = temp.RNAME;
+ else {
+ unsigned const id = FindRefSeqByName(scratch, true, self->refSeqs, self->refSeq);
+ if (id < self->refSeqs)
+ temp.RNEXT = id;
+ else {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record missing reference");
+ field = 0;
+ }
+ }
+ break;
+ case 8:
+ if (sscanf(scratch, "%i%n", &temp.PNEXT, &n) != 1 || n != i) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing PNEXT");
+ field = 0;
+ }
+ break;
+ case 9:
+ if (sscanf(scratch, "%i%n", &temp.TLEN, &n) != 1 || n != i) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing TLEN");
+ field = 0;
+ }
+ break;
+ case 10:
+ if (i == 1 && scratch[0] == '*')
+ temp.readlen = 0;
+ else {
+ temp.readlen = i;
+ scratch += (i + 1) >> 1;
+ temp.QUAL = (uint8_t *)scratch;
+ {
+ rc_t const rc = SAM2BAM_ConvertSEQ(i, temp.SEQ, endp);
+ if (rc) {
+ LOGERR(klogErr, rc, "SAM Record error converting SEQ");
+ field = 0;
+ }
+ }
+ }
+ break;
+ case 11:
+ if (temp.readlen == 0)
+ break;
+ if (i == 1 && scratch[0] == '*')
+ memset(temp.QUAL, 0xFF, temp.readlen);
+ else if (i == temp.readlen) {
+ rc_t const rc = SAM2BAM_ConvertQUAL(i, temp.QUAL, endp);
+ if (rc) {
+ LOGERR(klogErr, rc, "SAM Record error converting QUAL");
+ field = 0;
+ }
+ }
+ else {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error length of SEQ != length of QUAL");
+ field = 0;
+ }
+ scratch += temp.readlen;
+ break;
+ default:
+ {
+ int const n = SAM2BAM_ConvertEXTRA(i, scratch, endp);
+ if (n < 0) {
+ LOGERR(klogErr, RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid), "SAM Record error parsing optional field");
+ field = 0;
+ }
+ else {
+ if (n == 0)
+ --field;
+ else
+ scratch += n;
+ }
+ break;
+ }
+ }
+ if (ch == '\n') {
+ if (field < 11) {
+ rc_t const rc = RC(rcAlign, rcFile, rcReading, rcRow, rcTooShort);
+ LOGERR(klogErr, rc, "SAM Record error too few fields");
+ return rc;
+ }
+ break;
+ }
+ i = 0;
+ if (field > 0)
+ ++field;
+ }
+
+ SAM2BAM_ConvertInt(raw->rID, temp.RNAME);
+ SAM2BAM_ConvertInt(raw->pos, temp.POS - 1);
+ raw->read_name_len = temp.namelen;
+ raw->mapQual = temp.MAPQ;
+ SAM2BAM_ConvertShort(raw->n_cigars, temp.cigars);
+ SAM2BAM_ConvertShort(raw->flags, temp.FLAG);
+ SAM2BAM_ConvertInt(raw->read_len, temp.readlen);
+ SAM2BAM_ConvertInt(raw->mate_rID, temp.RNEXT);
+ SAM2BAM_ConvertInt(raw->mate_pos, temp.PNEXT - 1);
+ SAM2BAM_ConvertInt(raw->ins_size, temp.TLEN);
+ {
+ unsigned const datasize = (char *)scratch - (char *)self->buffer;
+ unsigned const rsltsize = BAM_AlignmentSizeFromData(datasize, self->buffer);
+ BAM_Alignment *const y = malloc(rsltsize);
+
+ if (y == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+
+ if (BAM_AlignmentInitLog(y, rsltsize, datasize, self->buffer)) {
+ y->parent = self;
+ rslt[0] = y;
+
+ if (BAM_AlignmentIsEmpty(y))
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcEmpty);
+ return 0;
+ }
+ free(y);
+ }
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcInvalid);
+}
+
+rc_t BAM_FileRead2(const BAM_File *cself, const BAM_Alignment **rhs)
+{
+ BAM_File *const self = (BAM_File *)cself;
+ unsigned actsize = 0;
+ rc_t rc;
+
+ if (self == NULL || rhs == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcParam, rcNull);
+
+ *rhs = NULL;
+
+ if (self->bufCurrent >= self->bufSize && self->eof)
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcNotFound);
+
+ if (self->isSAM) return BAM_FileReadSAM(self, rhs);
+
+ rc = BAM_FileBreakLock(self);
+ if (rc)
+ return rc;
+
+ if (self->nocopy_size == 0) {
+ size_t const size = 4096u;
+ void *const temp = malloc(size);
+
+ if (temp == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+
+ self->nocopy = temp;
+ self->nocopy_size = size;
+ }
+
+AGAIN:
+ rc = BAM_FileReadNoCopy(self, &actsize, self->nocopy, (unsigned)self->nocopy_size);
+ if (rc == 0) {
+ *rhs = self->nocopy;
+ if (BAM_AlignmentIsEmpty(self->nocopy)) {
+ rc = RC(rcAlign, rcFile, rcReading, rcRow, rcEmpty);
+ LOGERR(klogWarn, rc, "BAM Record contains no alignment or sequence data");
+ }
+ }
+ else if ( GetRCObject( rc ) == (enum RCObject)rcBuffer && GetRCState( rc ) == rcInsufficient )
+ {
+ unsigned const size = (actsize + 4095u) & ~4095u;
+ void *const temp = realloc(self->nocopy, size);
+
+ if (temp == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+
+ self->nocopy = temp;
+ self->nocopy_size = size;
+
+ goto AGAIN;
+ }
+ else if ( GetRCObject( rc ) == (enum RCObject)rcBuffer && GetRCState( rc ) == rcNotAvailable )
+ {
+ rc = BAM_FileReadCopy( self, rhs, true );
+ }
+ else if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcInvalid) {
+ BAM_AlignmentLogParseError(self->nocopy);
+ }
+ return rc;
+}
+
+rc_t BAM_FileRead(const BAM_File *cself, const BAM_Alignment **rhs)
+{
+ BAM_File *const self = (BAM_File *)cself;
+
+ if (self == NULL || rhs == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcParam, rcNull);
+
+ *rhs = NULL;
+
+ if (self->bufCurrent >= self->bufSize && self->eof)
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcNotFound);
+ else {
+ rc_t const rc = BAM_FileBreakLock(self);
+ if (rc)
+ return rc;
+ }
+ return BAM_FileReadCopy(self, rhs, false);
+}
+
+/* MARK: BAM File header info accessor */
+
+rc_t BAM_FileGetRefSeqById(const BAM_File *cself, int32_t id, const BAMRefSeq **rhs)
+{
+ *rhs = NULL;
+ if (id >= 0 && id < cself->refSeqs)
+ *rhs = &cself->refSeq[id];
+ return 0;
+}
+
+rc_t BAM_FileGetReadGroupByName(const BAM_File *cself, const char *name, const BAMReadGroup **rhs)
+{
+ BAMReadGroup rg;
+
+ *rhs = NULL;
+
+ rg.name = name;
+ if (rg.name != NULL)
+ *rhs = kbsearch(&rg, cself->readGroup, cself->readGroups, sizeof(rg), comp_ReadGroup, NULL);
+
+ return 0;
+}
+
+rc_t BAM_FileGetRefSeqCount(const BAM_File *cself, unsigned *rhs)
+{
+ *rhs = cself->refSeqs;
+ return 0;
+}
+
+rc_t BAM_FileGetRefSeq(const BAM_File *cself, unsigned i, const BAMRefSeq **rhs)
+{
+ *rhs = NULL;
+ if (i < cself->refSeqs)
+ *rhs = &cself->refSeq[i];
+ return 0;
+}
+
+rc_t BAM_FileGetReadGroupCount(const BAM_File *cself, unsigned *rhs)
+{
+ *rhs = cself->readGroups;
+ return 0;
+}
+
+rc_t BAM_FileGetReadGroup(const BAM_File *cself, unsigned i, const BAMReadGroup **rhs)
+{
+ *rhs = NULL;
+ if (i < cself->readGroups)
+ *rhs = &cself->readGroup[i];
+ return 0;
+}
+
+rc_t BAM_FileGetHeaderText(BAM_File const *cself, char const **header, size_t *header_len)
+{
+ *header = cself->header;
+ *header_len = *header ? string_size( *header ) : 0;
+ return 0;
+}
+
+/* MARK: BAM Alignment destructor */
+
+static rc_t BAM_AlignmentWhack(BAM_Alignment *self)
+{
+ if (self->parent->bufLocker == self)
+ self->parent->bufLocker = NULL;
+ if (self != self->parent->nocopy) {
+ free(self->storage);
+ free(self);
+ }
+ return 0;
+}
+
+/* MARK: BAM Alignment ref-counting */
+
+rc_t BAM_AlignmentAddRef(const BAM_Alignment *cself)
+{
+ return 0;
+}
+
+rc_t BAM_AlignmentRelease(const BAM_Alignment *cself)
+{
+ BAM_AlignmentWhack((BAM_Alignment *)cself);
+
+ return 0;
+}
+
+#if 0
+uint16_t BAM_AlignmentIffyFields(const BAM_Alignment *self)
+{
+}
+
+uint16_t BAM_AlignmentBadFields(const BAM_Alignment *self)
+{
+}
+#endif
+
+/* MARK: BAM Alignment accessors */
+
+static uint32_t BAM_AlignmentGetCigarElement(const BAM_Alignment *self, unsigned i)
+{
+ return LE2HUI32(&((uint8_t const *)getCigarBase(self))[i * 4]);
+}
+
+rc_t BAM_AlignmentGetRefSeqId(const BAM_Alignment *cself, int32_t *rhs)
+{
+ *rhs = getRefSeqId(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetPosition(const BAM_Alignment *cself, int64_t *rhs)
+{
+ *rhs = getPosition(cself);
+ return 0;
+}
+
+bool BAM_AlignmentIsMapped(const BAM_Alignment *cself)
+{
+ if (((getFlags(cself) & BAMFlags_SelfIsUnmapped) == 0) && getRefSeqId(cself) >= 0 && getPosition(cself) >= 0)
+ return true;
+ return false;
+}
+
+/* static bool BAM_AlignmentIsMateMapped(const BAM_Alignment *cself)
+{
+ if (((getFlags(cself) & BAMFlags_MateIsUnmapped) == 0) && getMateRefSeqId(cself) >= 0 && getMatePos(cself) >= 0)
+ return true;
+ return false;
+} */
+
+rc_t BAM_AlignmentGetAlignmentDetail(
+ const BAM_Alignment *self,
+ BAM_AlignmentDetail *rslt, uint32_t count, uint32_t *actual,
+ int32_t *pfirst, int32_t *plast
+ )
+{
+ unsigned i;
+ unsigned ccnt; /* cigar count */
+ int32_t gpos; /* refSeq pos in global coordinates */
+ unsigned rpos; /* read pos (always local coordinates) */
+ uint32_t rlen; /* read length */
+ int32_t first = -1;
+ int32_t last = -1;
+
+ if (!self)
+ return RC(rcAlign, rcFile, rcReading, rcSelf, rcNull);
+
+ rlen = getReadLen(self);
+ ccnt = getCigarCount(self);
+ gpos = getPosition(self);
+
+ if (gpos < 0)
+ ccnt = 0;
+
+ if (actual)
+ *actual = ccnt;
+
+ if (pfirst)
+ *pfirst = -1;
+
+ if (plast)
+ *plast = -1;
+
+ if (ccnt == 0)
+ return 0;
+
+ if (rslt == NULL) {
+ if (actual == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcParam, rcNull);
+ count = 0;
+ }
+
+ if (count < ccnt)
+ return RC(rcAlign, rcFile, rcReading, rcBuffer, rcInsufficient);
+
+ for (rpos = 0, i = 0; i != ccnt; ++i) {
+ uint32_t len = BAM_AlignmentGetCigarElement(self, i);
+ int op = len & 0x0F;
+
+ if (op > sizeof(cigarChars))
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+
+ op = cigarChars[op];
+ len >>= 4;
+
+ rslt[i].refSeq_pos = gpos;
+ rslt[i].read_pos = rpos;
+ rslt[i].length = len;
+ rslt[i].type = (BAMCigarType)op;
+
+ switch ((BAMCigarType)op) {
+ case ct_Match:
+ case ct_Equal:
+ if (first == -1)
+ first = i;
+ last = i;
+ gpos += len;
+ rpos += len;
+ break;
+ case ct_Insert:
+ case ct_SoftClip:
+ gpos += len;
+ break;
+ case ct_Delete:
+ case ct_Skip:
+ rpos += len;
+ break;
+ case ct_HardClip:
+ case ct_Padded:
+ rslt[i].refSeq_pos = -1;
+ rslt[i].read_pos = -1;
+ break;
+ default:
+ break;
+ }
+
+ if (rslt[i].read_pos > rlen)
+ return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ }
+ if (pfirst)
+ *pfirst = first;
+
+ if (plast)
+ *plast = last;
+
+ return 0;
+}
+
+static
+unsigned ReferenceLengthFromCIGAR(const BAM_Alignment *self)
+{
+ unsigned i;
+ unsigned n = getCigarCount(self);
+ unsigned y;
+
+ for (i = 0, y = 0; i != n; ++i) {
+ uint32_t const len = BAM_AlignmentGetCigarElement(self, i);
+
+ switch (cigarChars[len & 0x0F]) {
+ case ct_Match:
+ case ct_Equal:
+ case ct_NotEqual:
+ case ct_Delete:
+ case ct_Skip:
+ y += len >> 4;
+ break;
+ default:
+ break;
+ }
+ }
+ return y;
+}
+
+static unsigned SequenceLengthFromCIGAR(const BAM_Alignment *self)
+{
+ unsigned i;
+ unsigned n = getCigarCount(self);
+ unsigned y;
+
+ for (i = 0, y = 0; i != n; ++i) {
+ uint32_t const len = BAM_AlignmentGetCigarElement(self, i);
+
+ switch (cigarChars[len & 0x0F]) {
+ case ct_Match:
+ case ct_Equal:
+ case ct_NotEqual:
+ case ct_Insert:
+ case ct_SoftClip:
+ y += len >> 4;
+ break;
+ default:
+ break;
+ }
+ }
+ return y;
+}
+
+rc_t BAM_AlignmentGetPosition2(const BAM_Alignment *cself, int64_t *rhs, uint32_t *length)
+{
+ *rhs = getPosition(cself);
+ if (*rhs >= 0)
+ *length = ReferenceLengthFromCIGAR(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetReadGroupName(const BAM_Alignment *cself, const char **rhs)
+{
+ *rhs = get_RG(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetReadName(const BAM_Alignment *cself, const char **rhs)
+{
+ *rhs = getReadName(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetReadName2(const BAM_Alignment *cself, const char **rhs, size_t *length)
+{
+ *length = getReadNameLength(cself) - 1;
+ *rhs = getReadName(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetReadName3(const BAM_Alignment *cself, const char **rhs, size_t *length)
+{
+ char const *const name = getReadName(cself);
+ size_t len = getReadNameLength(cself);
+ size_t i;
+
+ for (i = len; i; ) {
+ int const ch = name[--i];
+
+ if (ch == '/') {
+ len = i;
+ break;
+ }
+ if (!isdigit(ch))
+ break;
+ }
+ *rhs = name;
+ *length = len;
+
+ return 0;
+}
+
+rc_t BAM_AlignmentGetFlags(const BAM_Alignment *cself, uint16_t *rhs)
+{
+ *rhs = getFlags(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetMapQuality(const BAM_Alignment *cself, uint8_t *rhs)
+{
+ *rhs = getMapQual(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetCigarCount(const BAM_Alignment *cself, unsigned *rhs)
+{
+ *rhs = getCigarCount(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetRawCigar(const BAM_Alignment *cself, uint32_t const *rslt[], uint32_t *length)
+{
+ *rslt = getCigarBase(cself);
+ *length = getCigarCount(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetCigar(const BAM_Alignment *cself, uint32_t i, BAMCigarType *type, uint32_t *length)
+{
+ uint32_t x;
+
+ if (i >= getCigarCount(cself))
+ return RC(rcAlign, rcFile, rcReading, rcParam, rcInvalid);
+
+ x = BAM_AlignmentGetCigarElement(cself, i);
+ *type = (BAMCigarType)(cigarChars[x & 0x0F]);
+ *length = x >> 4;
+ return 0;
+}
+
+rc_t BAM_AlignmentGetReadLength(const BAM_Alignment *cself, uint32_t *rhs)
+{
+ *rhs = getReadLen(cself);
+ return 0;
+}
+
+static int get1Base(BAM_Alignment const *const self, unsigned const i)
+{
+/*
+ * = A C M G R S V T W Y H K D B N
+ * 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111
+ * 1111 1000 0100 1100 0010 1010 0110 1110 0001 1001 0101 1101 0011 1011 0111 0000
+ * N T G K C Y S B A W R D M H V =
+ */
+ static const char tr[16] = "=ACMGRSVTWYHKDBN";
+/* static const char ctr[16] = "=TGKCYSBAWRDMHVN"; */
+ uint8_t const *const seq = &self->data->raw[self->seq];
+ unsigned const b4na2 = seq[i >> 1];
+ unsigned const b4na = (i & 1) == 0 ? (b4na2 >> 4) : (b4na2 & 0x0F);
+
+ return tr[b4na];
+}
+
+static int get1Qual(BAM_Alignment const *const self, unsigned const i)
+{
+ uint8_t const *const src = &self->data->raw[self->qual];
+
+ return src[i];
+}
+
+rc_t BAM_AlignmentGetSequence2(const BAM_Alignment *cself, char *rhs, uint32_t start, uint32_t stop)
+{
+ unsigned const n = getReadLen(cself);
+ unsigned si, di;
+
+ if (stop == 0 || stop > n)
+ stop = n;
+
+ for (di = 0, si = start; si != stop; ++si, ++di) {
+ rhs[di] = get1Base(cself, si);
+ }
+ return 0;
+}
+
+rc_t BAM_AlignmentGetSequence(const BAM_Alignment *cself, char *rhs)
+{
+ return BAM_AlignmentGetSequence2(cself, rhs, 0, 0);
+}
+
+bool BAM_AlignmentHasColorSpace(BAM_Alignment const *cself)
+{
+ return get_CS(cself) != NULL;
+}
+
+rc_t BAM_AlignmentGetCSKey(BAM_Alignment const *cself, char rhs[1])
+{
+ char const *const vCS = get_CS(cself);
+
+ if (vCS)
+ rhs[0] = vCS[0];
+ return 0;
+}
+
+rc_t BAM_AlignmentGetCSSeqLen(BAM_Alignment const *cself, uint32_t *rhs)
+{
+ struct offset_size_s const *const vCS = get_CS_info(cself);
+
+ *rhs = vCS ? vCS->size - 5 : 0;
+ return 0;
+}
+
+rc_t BAM_AlignmentGetCSSequence(BAM_Alignment const *cself, char rhs[], uint32_t seqlen)
+{
+ char const *const vCS = get_CS(cself);
+
+ if (vCS) {
+ unsigned i;
+
+ for (i = 0;i != seqlen; ++i) {
+ char const ch = vCS[i+1];
+
+ rhs[i] = (ch == '4') ? '.' : ch;
+ }
+ }
+ return 0;
+}
+
+rc_t BAM_AlignmentGetQuality(const BAM_Alignment *cself, const uint8_t **rhs)
+{
+ *rhs = &cself->data->raw[cself->qual];
+ return 0;
+}
+
+rc_t BAM_AlignmentGetQuality2(BAM_Alignment const *cself, uint8_t const **rhs, uint8_t *offset)
+{
+ uint8_t const *const OQ = get_OQ(cself);
+
+ if (OQ) {
+ struct offset_size_s const *const oq = get_OQ_info(cself);
+
+ if (oq->size - 4 == getReadLen(cself)) {
+ *offset = 33;
+ *rhs = OQ;
+ }
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInconsistent);
+ }
+ else {
+ *offset = 0;
+ *rhs = &cself->data->raw[cself->qual];
+ }
+ return 0;
+}
+
+rc_t BAM_AlignmentGetCSQuality(BAM_Alignment const *cself, uint8_t const **rhs, uint8_t *offset)
+{
+ struct offset_size_s const *const cs = get_CS_info(cself);
+ struct offset_size_s const *const cq = get_CQ_info(cself);
+ uint8_t const *const CQ = get_CQ(cself);
+
+ if (cs && cq && CQ) {
+ if (cs->size == cq->size) {
+ *offset = 33;
+ *rhs = CQ + 1;
+ return 0;
+ }
+ if (cs->size == cq->size + 1) {
+ *offset = 33;
+ *rhs = CQ;
+ return 0;
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInconsistent);
+ }
+ *offset = 0;
+ *rhs = &cself->data->raw[cself->qual];
+ return 0;
+}
+
+rc_t BAM_AlignmentGetMateRefSeqId(const BAM_Alignment *cself, int32_t *rhs)
+{
+ *rhs = getMateRefSeqId(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetMatePosition(const BAM_Alignment *cself, int64_t *rhs)
+{
+ *rhs = getMatePos(cself);
+ return 0;
+}
+
+rc_t BAM_AlignmentGetInsertSize(const BAM_Alignment *cself, int64_t *rhs)
+{
+ *rhs = getInsertSize(cself);
+ return 0;
+}
+
+static int FormatOptData(BAM_Alignment const *const self,
+ size_t const maxsize,
+ char buffer[])
+{
+ char const *const base = (char const *)&self->data->raw[self->qual + getReadLen(self)];
+ unsigned i;
+ unsigned offset;
+ unsigned cur = 0;
+ int j;
+
+ for (i = 0, offset = 0; i < self->numExtra; ++i) {
+ int type;
+ union { float f; uint32_t i; } fi;
+
+ if (cur + 7 > maxsize)
+ return -1;
+ buffer[cur++] = '\t';
+ buffer[cur++] = base[offset++];
+ buffer[cur++] = base[offset++];
+ buffer[cur++] = ':';
+ type = base[offset++];
+
+ switch (type) {
+ case dt_ASCII: /* A */
+ buffer[cur++] = 'A';
+ buffer[cur++] = ':';
+ buffer[cur++] = base[offset++];
+ break;
+
+ case dt_INT8: /* c */
+ buffer[cur++] = 'i';
+ buffer[cur++] = ':';
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)*((int8_t const *)(base + offset)));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 1;
+ break;
+
+ case dt_UINT8: /* C */
+ buffer[cur++] = 'i';
+ buffer[cur++] = ':';
+ j = snprintf(buffer + cur, maxsize - cur, "%u", (unsigned)*((uint8_t const *)(base + offset)));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 1;
+ break;
+
+ case dt_INT16: /* s */
+ buffer[cur++] = 'i';
+ buffer[cur++] = ':';
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)LE2HI16(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 2;
+ break;
+
+ case dt_UINT16: /* S */
+ buffer[cur++] = 'i';
+ buffer[cur++] = ':';
+ j = snprintf(buffer + cur, maxsize - cur, "%u", (unsigned)LE2HUI16(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 2;
+ break;
+
+ case dt_INT: /* i */
+ buffer[cur++] = 'i';
+ buffer[cur++] = ':';
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)LE2HI32(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 4;
+ break;
+
+ case dt_UINT: /* I */
+ buffer[cur++] = 'i';
+ buffer[cur++] = ':';
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)LE2HI32(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 4;
+ break;
+
+ case dt_FLOAT32: /* f */
+ buffer[cur++] = 'f';
+ buffer[cur++] = ':';
+ fi.i = LE2HUI32(base + offset);
+ j = snprintf(buffer + cur, maxsize - cur, "%f", fi.f);
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 4;
+ break;
+
+ case dt_HEXSTRING: /* H */
+ case dt_CSTRING: /* Z */
+ buffer[cur++] = type == dt_CSTRING ? 'Z' : 'H';
+ buffer[cur++] = ':';
+ for ( ; ; ) {
+ int const ch = base[offset++];
+
+ if (ch == '\0')
+ break;
+ if (cur >= maxsize)
+ return -1;
+ buffer[cur++] = ch;
+ }
+ break;
+
+ case dt_NUM_ARRAY: /* B */
+ buffer[cur++] = 'B';
+ buffer[cur++] = ':';
+ {
+ int const elemtype = base[offset++];
+ unsigned const elemcount = LE2HUI32(base + offset);
+ unsigned k;
+
+ if (cur + 2 >= maxsize)
+ return -1;
+ buffer[cur++] = elemtype;
+ offset += 4;
+ for (k = 0; k < elemcount; ++k) {
+ buffer[cur++] = ',';
+ switch (elemtype) {
+ case dt_INT8:
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)*((int8_t const *)(base + offset)));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 1;
+ break;
+
+ case dt_UINT8:
+ j = snprintf(buffer + cur, maxsize - cur, "%u", (unsigned)*((uint8_t const *)(base + offset)));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 1;
+ break;
+
+ case dt_INT16:
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)LE2HI16(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 2;
+ break;
+
+ case dt_UINT16:
+ j = snprintf(buffer + cur, maxsize - cur, "%u", (unsigned)LE2HUI16(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 2;
+ break;
+
+ case dt_INT:
+ j = snprintf(buffer + cur, maxsize - cur, "%i", (int)LE2HI32(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 4;
+ break;
+
+ case dt_UINT:
+ j = snprintf(buffer + cur, maxsize - cur, "%u", (unsigned)LE2HUI32(base + offset));
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 4;
+ break;
+
+ case dt_FLOAT32:
+ fi.i = LE2HUI32(base + offset);
+ j = snprintf(buffer + cur, maxsize - cur, "%f", fi.f);
+ if ((cur += j) >= maxsize)
+ return -1;
+ offset += 4;
+
+ default:
+ return -1;
+ break;
+ }
+ }
+ }
+ break;
+
+ default:
+ return -1;
+ break;
+ }
+ }
+ return cur;
+}
+
+static rc_t FormatSAM(BAM_Alignment const *self,
+ size_t *const actsize,
+ size_t const maxsize,
+ char *const buffer)
+{
+ int i = 0;
+ size_t cur = 0;
+ unsigned j;
+ int const refSeqId = getRefSeqId(self);
+ int const refPos = getPosition(self);
+ unsigned const cigCount = getCigarCount(self);
+ uint32_t const *const cigar = getCigarBase(self);
+ int const mateRefSeqId = getMateRefSeqId(self);
+ int const mateRefPos = getMatePos(self);
+ unsigned const readlen = getReadLen(self);
+
+ i = snprintf(&buffer[cur], maxsize - cur,
+ "%s\t%i\t%s\t%i\t%i\t",
+ getReadName(self),
+ getFlags(self),
+ refSeqId < 0 ? "*" : self->parent->refSeq[refSeqId].name,
+ refPos < 0 ? 0 : refPos + 1,
+ getMapQual(self)
+ );
+ if ((cur += i) > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+
+ if (cigCount > 0) {
+ for (j = 0; j < cigCount; ++j) {
+ uint32_t const el = cigar[j];
+ BAMCigarType const type = (BAMCigarType)(cigarChars[el & 0x0F]);
+ unsigned const length = el >> 4;
+
+ i = snprintf(&buffer[cur], maxsize - cur, "%u%c", length, type);
+ if ((cur += i) > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ }
+ }
+ else {
+ if ((cur + 1) > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ buffer[cur++] = '*';
+ }
+ i = snprintf(&buffer[cur], maxsize - cur,
+ "\t%s\t%i\t%i\t",
+ mateRefSeqId < 0 ? "*" : mateRefSeqId == refSeqId ? "=" : self->parent->refSeq[mateRefSeqId].name,
+ mateRefPos < 0 ? 0 : mateRefPos + 1,
+ getInsertSize(self)
+ );
+ if ((cur += i) > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ if (readlen) {
+ uint8_t const *const qual = &self->data->raw[self->qual];
+
+ if (cur + 2 * readlen + 1 > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ BAM_AlignmentGetSequence(self, &buffer[cur]);
+ cur += readlen;
+ buffer[cur] = '\t';
+ ++cur;
+
+ for (j = 0; j < readlen; ++j) {
+ if (qual[j] != 0xFF)
+ goto HAS_QUAL;
+ }
+ if (1) {
+ buffer[cur++] = '*';
+ }
+ else {
+ HAS_QUAL:
+ for (j = 0; j < readlen; ++j)
+ buffer[cur++] = qual[j] + 33;
+ }
+ }
+ else {
+ i = snprintf(&buffer[cur], maxsize - cur, "*\t*");
+ if ((cur += i) > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ }
+ i = FormatOptData(self, maxsize - cur, &buffer[cur]);
+ if (i < 0)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ if ((cur += i) + 2 > maxsize)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcExcessive);
+ buffer[cur++] = '\n';
+ buffer[cur] = '\0';
+ *actsize = cur;
+
+ return 0;
+}
+
+#define FORMAT_SAM_SCRATCH_SIZE ((size_t)(64u * 1024u))
+static rc_t FormatSAMBuffer(BAM_Alignment const *self,
+ size_t actSize[],
+ size_t const maxsize,
+ char *const buffer)
+{
+ char scratch[FORMAT_SAM_SCRATCH_SIZE];
+ size_t actsize = 0;
+ rc_t const rc = FormatSAM(self, &actsize, FORMAT_SAM_SCRATCH_SIZE, scratch);
+
+ actSize[0] = actsize;
+ if (rc) return rc;
+
+ if (actsize > maxsize)
+ return RC(rcAlign, rcReading, rcRow, rcBuffer, rcInsufficient);
+
+ memcpy(buffer, scratch, actsize);
+ return 0;
+}
+
+rc_t BAM_AlignmentFormatSAM(BAM_Alignment const *self,
+ size_t *const actSize,
+ size_t const maxsize,
+ char *const buffer)
+{
+ if (self == NULL)
+ return RC(rcAlign, rcReading, rcRow, rcSelf, rcNull);
+ if (buffer == NULL)
+ return RC(rcAlign, rcReading, rcRow, rcParam, rcNull);
+ else {
+ size_t actsize = 0;
+ rc_t const rc = (maxsize < FORMAT_SAM_SCRATCH_SIZE ? FormatSAMBuffer : FormatSAM)(self, &actsize, maxsize, buffer);
+
+ if (actSize)
+ *actSize = actsize;
+ return rc;
+ }
+}
+
+typedef struct OptForEach_ctx_s {
+ BAMOptData *val;
+ BAMOptData **alloced;
+ size_t valsize;
+ rc_t rc;
+ BAMOptionalDataFunction user_f;
+ void *user_ctx;
+} OptForEach_ctx_t;
+
+static bool i_OptDataForEach(BAM_Alignment const *cself, void *Ctx, char const tag[2], BAMOptDataValueType type, unsigned count, void const *value, unsigned size)
+{
+ OptForEach_ctx_t *ctx = (OptForEach_ctx_t *)Ctx;
+ size_t const need = (size_t)&((BAMOptData const *)NULL)->u.f64[(count * size + sizeof(double) - 1)/sizeof(double)];
+
+ if (need > ctx->valsize) {
+ void *const temp = realloc(ctx->alloced, need);
+ if (temp == NULL) {
+ ctx->rc = RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+ return true;
+ }
+ *ctx->alloced = ctx->val = temp;
+ ctx->valsize = need;
+ }
+ ctx->val->type = type;
+ ctx->val->element_count = (type == dt_CSTRING || type == dt_HEXSTRING) ? size - 1 : count;
+
+ memcpy(ctx->val->u.u8, value, size * count);
+#if __BYTE_ORDER == __BIG_ENDIAN
+ {{
+ unsigned di;
+ uint32_t elem_count = ctx->val->element_count;
+
+ switch (size) {
+ case 2:
+ for (di = 0; di != elem_count; ++di)
+ ctx->val->u.u16[di] = LE2HUI16(&ctx->val->u.u16[di]);
+ break;
+ case 4:
+ for (di = 0; di != elem_count; ++di)
+ ctx->val->u.u32[di] = LE2HUI32(&ctx->val->u.u32[di]);
+ break;
+ case 8:
+ for (di = 0; di != elem_count; ++di)
+ ctx->val->u.u64[di] = LE2HUI64(&ctx->val->u.u64[di]);
+ break;
+ }
+ }}
+#endif
+ ctx->rc = ctx->user_f(ctx->user_ctx, tag, ctx->val);
+ return ctx->rc != 0;
+}
+
+rc_t BAM_AlignmentOptDataForEach(const BAM_Alignment *cself, void *user_ctx, BAMOptionalDataFunction f)
+{
+ union u {
+ BAMOptData value;
+ uint8_t storage[4096];
+ } value_auto;
+ OptForEach_ctx_t ctx;
+ rc_t rc = 0;
+ unsigned i;
+
+ ctx.val = &value_auto.value;
+ ctx.alloced = NULL;
+ ctx.valsize = sizeof(value_auto);
+ ctx.rc = 0;
+ ctx.user_f = f;
+ ctx.user_ctx = user_ctx;
+
+ for (i = 0; i != cself->numExtra; ++i) {
+ char const *const tag = (char const *)&cself->data->raw[cself->extra[i].offset];
+ uint8_t type = tag[2];
+ uint8_t const *const vp = (uint8_t const *)&tag[3];
+ unsigned len = cself->extra[i].size - 3;
+ unsigned size = cself->extra[i].size - 3;
+ unsigned count = 1;
+ unsigned offset = 0;
+
+ if (type == dt_NUM_ARRAY) {
+ unsigned elem_size = 0;
+ uint32_t elem_count = 0;
+
+ offset = len = 5;
+ switch (vp[0]) {
+ case dt_INT8:
+ case dt_UINT8:
+ elem_size = 1;
+ break;
+ case dt_INT16:
+ case dt_UINT16:
+ elem_size = 2;
+ break;
+ case dt_FLOAT32:
+ case dt_INT:
+ case dt_UINT:
+ elem_size = 4;
+ break;
+#if 0
+ case dt_FLOAT64:
+ elem_size = 8;
+ break;
+#endif
+ default:
+ rc = RC(rcAlign, rcFile, rcReading, rcData, rcUnexpected);
+ break;
+ }
+ if (rc)
+ break;
+ elem_count = LE2HUI32(&vp[1]);
+ len += elem_size * elem_count;
+ type = vp[0];
+ count = elem_count;
+ size = elem_size;
+ break;
+ }
+ if (i_OptDataForEach(cself, &ctx, tag, type, count, &vp[offset], size))
+ break;
+ }
+ rc = rc ? rc : ctx.rc;
+ if (ctx.alloced)
+ free(ctx.alloced);
+ return rc;
+}
+
+/* MARK: Complete Genomics stuff */
+
+bool BAM_AlignmentHasCGData(BAM_Alignment const *self)
+{
+ return get_CG_GC_info(self) && get_CG_GS_info(self) && get_CG_GQ_info(self);
+}
+
+rc_t BAM_AlignmentCGReadLength(BAM_Alignment const *self, uint32_t *readlen)
+{
+ struct offset_size_s const *const GCi = get_CG_GC_info(self);
+ struct offset_size_s const *const GSi = get_CG_GS_info(self);
+ struct offset_size_s const *const GQi = get_CG_GQ_info(self);
+
+ if (GCi && GSi && GQi) {
+ char const *GS = (char const *)&self->data->raw[GSi->offset + 3];
+ char const *GQ = (char const *)&self->data->raw[GQi->offset + 3];
+ char const *GC = (char const *)&self->data->raw[GCi->offset + 3];
+ unsigned oplen = 0;
+ unsigned i;
+ unsigned di = 0;
+ unsigned si = 0;
+
+ for (i = 0; ; ++i) {
+ int const ch = GC[i];
+
+ if (ch == '\0')
+ break;
+ if (isdigit(ch)) {
+ oplen = oplen * 10 + (ch - '0');
+ }
+ else if (ch != 'S' && ch != 'G')
+ return RC(rcAlign, rcRow, rcReading, rcData, rcUnexpected);
+ else {
+ unsigned const jmax = (ch == 'G') ? (oplen * 2) : oplen;
+ unsigned j;
+
+ if (ch == 'S') {
+ ;
+ }
+ else {
+ for (j = 0; j < jmax; ++j) {
+ int const base = *GS++;
+ int const qual = *GQ++;
+
+ switch (base) {
+ case 'A':
+ case 'C':
+ case 'M':
+ case 'G':
+ case 'R':
+ case 'S':
+ case 'V':
+ case 'T':
+ case 'W':
+ case 'Y':
+ case 'H':
+ case 'K':
+ case 'D':
+ case 'B':
+ case 'N':
+ break;
+ default:
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ }
+ if (qual < 33)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ }
+ }
+ si += oplen;
+ di += jmax;
+ oplen = 0;
+ }
+ }
+ if (*GS != '\0' || *GQ != '\0' || si != getReadLen(self))
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+
+ *readlen = di;
+ return 0;
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcNotFound);
+}
+
+static unsigned BAM_AlignmentParseCGTag(BAM_Alignment const *self, size_t const max_cg_segs, unsigned cg_segs[/* max_cg_segs */])
+{
+ struct offset_size_s const *const GCi = get_CG_GC_info(self);
+ char const *cur = (char const *)&self->data->raw[GCi->offset + 3];
+ unsigned i = 0;
+ int last_op = 0;
+
+ memset(cg_segs, 0, max_cg_segs * sizeof(cg_segs[0]));
+
+ while (*cur != '\0' && i < max_cg_segs) {
+ char *endp;
+ unsigned const op_len = (unsigned)strtol(cur, &endp, 10);
+ int const op = *endp;
+
+ cur = endp + 1;
+ if (op == last_op)
+ cg_segs[i - 1] += op_len;
+ else
+ cg_segs[i++] = op_len;
+ last_op = op;
+ }
+ return i;
+}
+
+static
+rc_t ExtractInt32(BAM_Alignment const *self, int32_t *result,
+ struct offset_size_s const *const tag)
+{
+ int64_t y;
+ int const type = self->data->raw[tag->offset + 2];
+ void const *const pvalue = &self->data->raw[tag->offset + 3];
+
+ switch (type) {
+ case 'c':
+ if (tag->size == 4)
+ y = *((int8_t const *)pvalue);
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ break;
+ case 'C':
+ if (tag->size == 4)
+ y = *((uint8_t const *)pvalue);
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ break;
+ case 's':
+ if (tag->size == 5)
+ y = LE2HI16(pvalue);
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ break;
+ case 'S':
+ if (tag->size == 5)
+ y = LE2HUI16(pvalue);
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ break;
+ case 'i':
+ if (tag->size == 7)
+ y = LE2HI32(pvalue);
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ break;
+ case 'I':
+ if (tag->size == 7)
+ y = LE2HUI32(pvalue);
+ else
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+ break;
+ default:
+ return RC(rcAlign, rcRow, rcReading, rcData, rcNotFound);
+ }
+ if (INT32_MIN <= y && y <= INT32_MAX) {
+ *result = (int32_t)y;
+ return 0;
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcInvalid);
+}
+
+rc_t BAM_AlignmentGetCGAlignGroup(BAM_Alignment const *self,
+ char buffer[],
+ size_t max_size,
+ size_t *act_size)
+{
+ struct offset_size_s const *const ZA = get_CG_ZA_info(self);
+ struct offset_size_s const *const ZI = get_CG_ZI_info(self);
+
+ if (ZA && ZI) {
+ rc_t rc;
+ int32_t za;
+ int32_t zi;
+
+ rc = ExtractInt32(self, &za, ZA); if (rc) return rc;
+ rc = ExtractInt32(self, &zi, ZI); if (rc) return rc;
+ return string_printf(buffer, max_size, act_size, "%i_%i", zi, za);
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcNotFound);
+}
+
+rc_t BAM_AlignmentGetCGSeqQual(BAM_Alignment const *self,
+ char sequence[],
+ uint8_t quality[])
+{
+ struct offset_size_s const *const GCi = get_CG_GC_info(self);
+ struct offset_size_s const *const GSi = get_CG_GS_info(self);
+ struct offset_size_s const *const GQi = get_CG_GQ_info(self);
+
+ if (GCi && GSi && GQi) {
+ char const *GS = (char const *)&self->data->raw[GSi->offset + 3];
+ char const *GQ = (char const *)&self->data->raw[GQi->offset + 3];
+ char const *GC = (char const *)&self->data->raw[GCi->offset + 3];
+ unsigned oplen = 0;
+ unsigned di = 0;
+ unsigned si = 0;
+
+ for ( ; ; ) {
+ int const ch = *GC++;
+
+ if (ch == '\0')
+ break;
+ if (isdigit(ch)) {
+ oplen = oplen * 10 + (ch - '0');
+ continue;
+ }
+ if (ch == 'S') {
+ unsigned i;
+
+ for (i = 0; i < oplen; ++i, ++di, ++si) {
+ unsigned const base = get1Base(self, si);
+ unsigned const qual = get1Qual(self, si);
+
+ sequence[di] = base;
+ quality [di] = qual;
+ }
+ }
+ else {
+ unsigned i;
+
+ for (i = 0; i < oplen * 2; ++i, ++di) {
+ unsigned const base = *GS++;
+ unsigned const qual = *GQ++ - 33;
+
+ sequence[di] = base;
+ quality [di] = qual;
+ }
+ si += oplen;
+ }
+ oplen = 0;
+ }
+ return 0;
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcNotFound);
+}
+
+
+static unsigned splice(uint32_t cigar[], unsigned n, unsigned at, unsigned out, unsigned in, uint32_t const new_values[/* in */])
+{
+ assert(at + out <= n);
+ memmove(&cigar[at + in], &cigar[at + out], (n - at - out) * 4);
+ if (in)
+ memcpy(&cigar[at], new_values, in * 4);
+ return n + in - out;
+}
+
+#define OPCODE_2_FIX (0xF)
+
+static unsigned insert_B(unsigned const T, unsigned const G, unsigned const n, uint32_t cigar[/* n */])
+{
+ unsigned i;
+ unsigned pos;
+
+ for (pos = i = 0; i < n; ++i) {
+ int const opcode = cigar[i] & 0xF;
+
+ switch (opcode) {
+ case 0:
+ case 1:
+ case 4:
+ case 7:
+ case 8:
+ {{
+ unsigned const len = cigar[i] >> 4;
+ unsigned const nxt = pos + len;
+
+ if (pos <= T && T <= nxt) {
+ unsigned const l = T - pos;
+ unsigned const r = len - l;
+ uint32_t op[4];
+
+ op[0] = (l << 4) | opcode;
+ op[1] = (G << 4) | 9; /* B */
+ op[2] = (G << 4) | 0; /* M this is not backwards */
+ op[3] = (r << 4) | opcode;
+
+ return splice(cigar, n, i, 1,
+ 4 - (l == 0 ? 1 : 0) - (r == 0 ? 1 : 0),
+ op + (l == 0 ? 1 : 0));
+ }
+ pos = nxt;
+ }}
+ break;
+ default:
+ break;
+ }
+ }
+ return n;
+}
+
+static unsigned canonicalize(uint32_t cigar[], unsigned n)
+{
+ unsigned i;
+
+ /* remove zero-length and P operations */
+ for (i = n; i > 0; ) {
+ --i;
+ if (cigar[i] >> 4 == 0 || (cigar[i] & 0xF) == 6)
+ n = splice(cigar, n, i, 1, 0, NULL);
+ }
+ /* merge adjacent operations of the same type */
+ for (i = 1; i < n; ) {
+ unsigned const opL = cigar[i-1] & 0xF;
+ unsigned const opI = cigar[ i ] & 0xF;
+
+ if (opI == opL) {
+ unsigned const oplen = (cigar[i] >> 4) + (cigar[i-1] >> 4);
+ uint32_t const op = (oplen << 4) | opI;
+
+ n = splice(cigar, n, i-1, 2, 1, &op);
+ }
+ else
+ ++i;
+ }
+ return n;
+}
+
+static unsigned GetCGCigar(BAM_Alignment const *self, unsigned const N, uint32_t cigar[/* N */])
+{
+ unsigned i;
+ unsigned S;
+ unsigned n = getCigarCount(self);
+ unsigned seg[64];
+ unsigned const segs = BAM_AlignmentParseCGTag(self, sizeof(seg)/sizeof(seg[0]), seg);
+ unsigned const gaps = (segs - 1) >> 1;
+
+ if (2 * gaps + 1 != segs)
+ return RC(rcAlign, rcRow, rcReading, rcData, rcUnexpected);
+
+ if (N < n + 2 * gaps)
+ return RC(rcAlign, rcRow, rcReading, rcBuffer, rcInsufficient);
+
+ memcpy(cigar, getCigarBase(self), n * 4);
+
+ if (n > 1)
+ n = canonicalize(cigar, n); /* just in case */
+
+ for (i = 0, S = 0; i < gaps; ++i) {
+ unsigned const s = seg[2 * i + 0];
+ unsigned const g = seg[2 * i + 1];
+
+ S += s + g;
+ if (g > 0)
+ n = insert_B(S, g, n, cigar);
+ S += g;
+ }
+ return n;
+}
+
+rc_t BAM_AlignmentGetCGCigar(BAM_Alignment const *self,
+ uint32_t *cigar,
+ uint32_t cig_max,
+ uint32_t *cig_act)
+{
+ struct offset_size_s const *const GCi = get_CG_GC_info(self);
+
+ *cig_act = 0;
+
+ if (GCi) {
+ *cig_act = GetCGCigar(self, cig_max, cigar);
+ return 0;
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcNotFound);
+}
+
+/* MARK: end CG stuff */
+
+rc_t BAM_AlignmentGetTI(BAM_Alignment const *self, uint64_t *ti)
+{
+ char const *const TI = get_XT(self);
+ long long unsigned temp;
+
+ if (TI && sscanf(TI, "ti|%llu", &temp) == 1) {
+ *ti = (uint64_t)temp;
+ return 0;
+ }
+ return RC(rcAlign, rcRow, rcReading, rcData, rcNotFound);
+}
+
+rc_t BAM_AlignmentGetRNAStrand(BAM_Alignment const *const self, uint8_t *const rslt)
+{
+ if (rslt) {
+ uint8_t const *const XS = get_XS(self);
+
+ *rslt = XS ? XS[0] : ' ';
+ }
+ return 0;
+}
diff --git a/tools/bam-loader/bam.h b/tools/bam-loader/bam.h
new file mode 100644
index 0000000..ae81c60
--- /dev/null
+++ b/tools/bam-loader/bam.h
@@ -0,0 +1,632 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+/*--------------------------------------------------------------------------
+ * BAM_Alignment
+ */
+typedef struct BAM_Alignment BAM_Alignment;
+
+
+/* AddRef
+ * Release
+ */
+rc_t BAM_AlignmentAddRef ( const BAM_Alignment *self );
+rc_t BAM_AlignmentRelease ( const BAM_Alignment *self );
+
+
+/* GetReadLength
+ * get the sequence length
+ * i.e. the number of elements of both sequence and quality
+ *
+ * "length" [ OUT ] - length in bases of query sequence and quality
+ */
+rc_t BAM_AlignmentGetReadLength ( const BAM_Alignment *self, uint32_t *length );
+
+
+/* GetSequence
+ * get the sequence data [0..ReadLength)
+ * caller provides buffer of ReadLength bytes
+ *
+ * "sequence" [ OUT ] - pointer to a buffer of at least ReadLength bytes
+ */
+rc_t BAM_AlignmentGetSequence ( const BAM_Alignment *self, char *sequence );
+
+/* GetSequence2
+ * get the sequence data [0..ReadLength)
+ * caller provides buffer of ReadLength bytes
+ *
+ * "sequence" [ OUT ] - pointer to a buffer of at least ReadLength bytes
+ *
+ * "start" [ IN ] and "stop" [ IN ] - zero-based coordinates, half-closed interval
+ */
+rc_t BAM_AlignmentGetSequence2 ( const BAM_Alignment *self, char *sequence, uint32_t start, uint32_t stop);
+
+
+/* GetQuality
+ * get the raw quality data [0..ReadLength)
+ * values are unsigned with 0xFF == missing
+ *
+ * "quality" [ OUT ] - return param for quality sequence
+ * held internally, validity is guaranteed for the life of the BAM_Alignment
+ */
+rc_t BAM_AlignmentGetQuality ( const BAM_Alignment *self, const uint8_t **quality );
+
+/* GetQuality2
+ * get the raw quality data [0..ReadLength) from OQ if possible else from QUAL
+ * values are unsigned with 0xFF == missing
+ *
+ * "quality" [ OUT ] - return param for quality sequence
+ * held internally, validity is guaranteed for the life of the BAM_Alignment
+ */
+rc_t BAM_AlignmentGetQuality2(const BAM_Alignment *self, const uint8_t **quality, uint8_t *offset);
+
+/* GetRefSeqId
+ * get id of reference sequence
+ * pass result into BAM_FileGetRefSeqById to get the Reference Sequence record
+ *
+ * "refSeqId" [ OUT ] - zero-based id of reference sequence
+ * returns -1 if set as invalid within BAM ( rc may be zero )
+ */
+rc_t BAM_AlignmentGetRefSeqId ( const BAM_Alignment *self, int32_t *refSeqId );
+
+/* GetMateRefSeqId
+ * get id of mate's reference sequence
+ * pass result into BAM_FileGetRefSeqById to get the Reference Sequence record
+ *
+ * "refSeqId" [ OUT ] - zero-based id of reference sequence
+ * returns -1 if invalid
+ */
+rc_t BAM_AlignmentGetMateRefSeqId ( const BAM_Alignment *self, int32_t *refSeqId );
+
+
+/* GetPosition
+ * get the aligned position on the ref. seq.
+ *
+ * "n" [ IN ] - zero-based position index for cases of multiple alignments
+ *
+ * "pos" [ OUT ] - zero-based position on reference sequence
+ * returns -1 if invalid
+ */
+rc_t BAM_AlignmentGetPosition ( const BAM_Alignment *self, int64_t *pos );
+
+/* GetPosition2
+ * get the aligned start position on the ref. seq.
+ * get the aligned length on the ref. seq.
+ *
+ * "n" [ IN ] - zero-based position index for cases of multiple alignments
+ *
+ * "pos" [ OUT ] - zero-based position on reference sequence
+ * returns -1 if invalid
+ *
+ * "length" [ OUT ] - length of alignment on reference sequence
+ * returns 0 if invalid
+ */
+rc_t BAM_AlignmentGetPosition2 ( const BAM_Alignment *self, int64_t *pos, uint32_t *length );
+
+
+/* GetMatePosition
+ * starting coordinate of mate's alignment on ref. seq.
+ *
+ * "pos" [ OUT ] - zero-based position on reference sequence
+ * returns -1 if invalid
+ */
+rc_t BAM_AlignmentGetMatePosition ( const BAM_Alignment *self, int64_t *pos );
+
+
+/* IsMapped
+ * is the alignment mapped to something
+ */
+bool BAM_AlignmentIsMapped ( const BAM_Alignment *self );
+
+
+/* GetReadGroupName
+ * get the name of the read group (i.e. accession)
+ * pass result into BAM_FileGetReadGroupByName to get the Read Group record
+ *
+ * "name" [ OUT ] - return param for NUL-terminated read group name
+ * held internally, validity is guaranteed for the life of the BAM_Alignment
+ */
+rc_t BAM_AlignmentGetReadGroupName ( const BAM_Alignment *self, const char **name );
+
+
+/* GetReadName
+ * get the read name (i.e. spot name)
+ * GetReadName2
+ * get the read name and length in bytes
+ *
+ * "name" [ OUT ] - return param for NUL-terminated read name
+ * held internally, validity is guaranteed for the life of the BAM_Alignment
+ *
+ * "length" [ OUT ] - return the number of bytes in "name"
+ * excluding terminating NUL.
+ */
+rc_t BAM_AlignmentGetReadName ( const BAM_Alignment *self, const char **name );
+rc_t BAM_AlignmentGetReadName2 ( const BAM_Alignment *self, const char **name, size_t *length );
+
+
+/* GetReadName3
+ * get the read name and length in bytes
+ * applies fixups to name
+ *
+ * "name" [ OUT ] - return param for read name
+ * held internally, validity is guaranteed for the life of the BAM_Alignment
+ *
+ * "length" [ OUT ] - return the number of bytes in "name"
+ */
+rc_t BAM_AlignmentGetReadName3 ( const BAM_Alignment *self, const char **name, size_t *length );
+
+/* HasColorSpace
+ * Does the alignment have colorspace info
+ */
+bool BAM_AlignmentHasColorSpace ( const BAM_Alignment *self );
+
+/* GetCSKey
+ * get the colorspace key
+ *
+ * "cskey" [ OUT ] - return param
+ */
+rc_t BAM_AlignmentGetCSKey ( const BAM_Alignment *self, char cskey[1] );
+
+rc_t BAM_AlignmentGetCSSeqLen ( const BAM_Alignment *self, uint32_t *seqLen );
+/* GetCSSequence
+ * get the colorspace sequence data [0..seqLen)
+ * caller provides buffer of seqLen bytes
+ *
+ * "csseq" [ OUT ] - pointer to a buffer of at least seqLen bytes
+ * "seqLen" [ IN ] - length of sequence from BAM_AlignmentGetCSSeqLen
+ */
+rc_t BAM_AlignmentGetCSSequence ( const BAM_Alignment *self, char *csseq, uint32_t seqLen );
+
+rc_t BAM_AlignmentGetCSQuality(BAM_Alignment const *cself, uint8_t const **quality, uint8_t *offset);
+
+
+/* GetFlags
+ * return the raw "flags" bitmap word
+ *
+ * "flags" [ OUT ] - return parameter for bitmap word
+ */
+enum BAMFlags
+{
+ BAMFlags_bit_WasPaired = 0, /* was paired when sequenced */
+ BAMFlags_bit_IsMappedAsPair,
+ BAMFlags_bit_SelfIsUnmapped,
+ BAMFlags_bit_MateIsUnmapped,
+ BAMFlags_bit_SelfIsReverse,
+ BAMFlags_bit_MateIsReverse,
+ BAMFlags_bit_IsFirst, /* and mate exists */
+ BAMFlags_bit_IsSecond, /* and mate exists */
+ BAMFlags_bit_IsNotPrimary, /* a read having split hits may have multiple primary alignments */
+ BAMFlags_bit_IsLowQuality, /* fails platform/vendor quality checks */
+ BAMFlags_bit_IsDuplicate, /* PCR or optical dup */
+
+ BAMFlags_WasPaired = (1 << BAMFlags_bit_WasPaired),
+ BAMFlags_IsMappedAsPair = (1 << BAMFlags_bit_IsMappedAsPair),
+ BAMFlags_SelfIsUnmapped = (1 << BAMFlags_bit_SelfIsUnmapped),
+ BAMFlags_MateIsUnmapped = (1 << BAMFlags_bit_MateIsUnmapped),
+ BAMFlags_SelfIsReverse = (1 << BAMFlags_bit_SelfIsReverse),
+ BAMFlags_MateIsReverse = (1 << BAMFlags_bit_MateIsReverse),
+ BAMFlags_IsFirst = (1 << BAMFlags_bit_IsFirst),
+ BAMFlags_IsSecond = (1 << BAMFlags_bit_IsSecond),
+ BAMFlags_IsNotPrimary = (1 << BAMFlags_bit_IsNotPrimary),
+ BAMFlags_IsLowQuality = (1 << BAMFlags_bit_IsLowQuality),
+ BAMFlags_IsDuplicate = (1 << BAMFlags_bit_IsDuplicate)
+};
+
+rc_t BAM_AlignmentGetFlags ( const BAM_Alignment *self, uint16_t *flags );
+
+
+/* GetMapQuality
+ * return the quality score of mapping
+ *
+ * "qual" [ OUT ] - return param for quality score
+ */
+rc_t BAM_AlignmentGetMapQuality ( const BAM_Alignment *self, uint8_t *qual );
+
+
+/* GetAlignmentDetail
+ * get the alignment details
+ *
+ * "rslt" [ OUT, NULL OKAY ] and "count" [ IN ] - array to hold detail records
+ *
+ * "actual" [ OUT, NULL OKAY ] - number of elements written to "rslt"
+ * required if "rslt" is not NULL
+ *
+ * "firstMatch" [ OUT, NULL OKAY ] - zero-based index into "rslt" of the first match to the refSeq
+ * or < 0 if invalid
+ *
+ * "lastMatch" [ OUT, NULL OKAY ] - zero-based index into "rslt" of the last match to the refSeq
+ * or < 0 if invalid
+ */
+typedef uint32_t BAMCigarType;
+enum BAMCigarTypes
+{
+ ct_Match = 'M', /* 0 */
+ ct_Insert = 'I', /* 1 */
+ ct_Delete = 'D', /* 2 */
+ ct_Skip = 'N', /* 3 */
+ ct_SoftClip = 'S', /* 4 */
+ ct_HardClip = 'H', /* 5 */
+ ct_Padded = 'P', /* 6 */
+ ct_Equal = '=', /* 7 */
+ ct_NotEqual = 'X', /* 8 */
+ ct_Overlap = 'B' /* Complete Genomics extension */
+};
+
+typedef struct BAM_AlignmentDetail BAM_AlignmentDetail;
+struct BAM_AlignmentDetail
+{
+ int64_t refSeq_pos; /* position on refSeq where this alignment region starts or -1 if NA */
+ int32_t read_pos; /* position on read where this alignment region starts or -1 if NA */
+ uint32_t length; /* length of alignment region */
+ BAMCigarType type; /* type of alignment */
+};
+
+rc_t BAM_AlignmentGetAlignmentDetail ( const BAM_Alignment *self,
+ BAM_AlignmentDetail *rslt, uint32_t count, uint32_t *actual,
+ int32_t *firstMatch, int32_t *lastMatch );
+
+
+/* GetCigarCount
+ * the number of CIGAR elements
+ * a CIGAR element consists of the pair of matching op code and op length
+ *
+ * "n" [ OUT ] - return param for cigar count
+ */
+rc_t BAM_AlignmentGetCigarCount ( const BAM_Alignment *self, uint32_t *n );
+
+
+rc_t BAM_AlignmentGetRawCigar(const BAM_Alignment *cself, uint32_t const **rslt, uint32_t *length);
+
+/* GetCigar
+ * get CIGAR element n [0..GetCigarCount)
+ */
+rc_t BAM_AlignmentGetCigar ( const BAM_Alignment *self,
+ uint32_t n, BAMCigarType *type, uint32_t *length );
+
+
+/* GetInsertSize
+ * distance in bases to start of mate's alignment on ref. seq.
+ *
+ * "size" [ OUT ] - >0 for first in pair, <0 for second
+ */
+rc_t BAM_AlignmentGetInsertSize ( const BAM_Alignment *self, int64_t *size );
+
+rc_t BAM_AlignmentFormatSAM(const BAM_Alignment *self,
+ size_t *actsize,
+ size_t maxsize,
+ char *buffer);
+
+/* OptDataForEach
+ * DANGER
+ * these optional fields are the weakest part of BAM.
+ *
+ * It is probably best to not use this info.
+ * You can't count on them being there.
+ * Moreover, you might need to interpret the types correctly.
+ */
+typedef uint32_t BAMOptDataValueType;
+enum BAMOptDataValueTypes
+{
+ dt_CSTRING = 'Z',
+ dt_INT8 = 'c',
+ dt_UINT8 = 'C',
+ dt_INT16 = 's',
+ dt_UINT16 = 'S',
+ dt_INT = 'i',
+ dt_UINT = 'I',
+ dt_FLOAT32 = 'f',
+#if 0
+ dt_FLOAT64 = 'd', /* removed? not in Dec 19 2013 version of SAMv1.pdf */
+#endif
+ dt_ASCII = 'A',
+ dt_HEXSTRING = 'H',
+ dt_NUM_ARRAY = 'B'
+};
+
+#define OPT_TAG_X "X?" /* end user data */
+#define OPT_TAG_Y "Y?" /* end user data */
+#define OPT_TAG_Z "Z?" /* end user data */
+
+#define OPT_TAG_ReadGroup "RG" /* Read Group; same as BAM_AlignmentGetReadGroupName */
+#define OPT_TAG_Library "LB" /* LIbrary; also BAMReadGroup */
+#define OPT_TAG_Unit "PU" /* Platform specific Unit; also BAMReadGroup */
+#define OPT_TAG_Program "PG" /* Alignment software name */
+#define OPT_TAG_AlignScore "AS" /* Alignment Score (MapQuality?) */
+#define OPT_TAG_SecQual "SQ" /* second called base:2 and quality:6; length == ReadLength? warning */
+#define OPT_TAG_MateMapQual "MQ" /* map Quality of mate */
+#define OPT_TAG_NumMismatch "NM" /* Number of Mismatches */
+#define OPT_TAG_Hits0 "H0" /* Number of perfect hits */
+#define OPT_TAG_Hits1 "H1" /* Number of off-by-one */
+#define OPT_TAG_Hits2 "H2" /* Number of off-by-two */
+#define OPT_TAG_CondQual "UQ" /* conditional Quality of read */
+#define OPT_TAG_CondQPair "PQ" /* conditional Quality of pair */
+#define OPT_TAG_ReadHits "NH" /* Number of times this read (spot) aligns */
+#define OPT_TAG_ReadHits2 "IH" /* Number of times this read (spot) aligns that are in this file */
+#define OPT_TAG_HitIndex "HI" /* n-th hit for this read in this file */
+#define OPT_TAG_Match2 "MD" /* another sort of matching string like CIGAR but different? */
+#define OPT_TAG_ColorKey "CS" /* primer and first color */
+#define OPT_TAG_ColorQual "CQ" /* quality of above */
+#define OPT_TAG_ColorMisses "CM" /* Number of color-space Mismatches */
+#define OPT_TAG_SeqOverlap "GS"
+#define OPT_TAG_QualOverlap "GQ"
+#define OPT_TAG_OverlapDesc "GC"
+#define OPT_TAG_MateSeq "R2" /* sequence of the mate */
+#define OPT_TAG_MateQual "Q2" /* quality scores of the mate */
+#define OPT_TAG_OtherQual "S2"
+#define OPT_TAG_NextHitRef "CC" /* Reference name of the next hit */
+#define OPT_TAG_NextHitPos "CP" /* coordinate of the next hit */
+#define OPT_TAG_SingleMapQ "SM" /* quality of mapping as if not paired */
+#define OPT_TAG_AM "AM"
+#define OPT_TAG_MAQFlag "MQ"
+
+
+struct BAMOptData
+{
+ BAMOptDataValueType type;
+ uint32_t element_count;
+ union {
+ int8_t i8[8];
+ uint8_t u8[8];
+ int16_t i16[4];
+ uint16_t u16[4];
+ int32_t i32[2];
+ uint32_t u32[2];
+ int64_t i64[2];
+ uint64_t u64[2];
+ float f32[2];
+ double f64[1];
+ char asciiz[8];
+ } u;
+};
+
+typedef struct BAMOptData BAMOptData;
+
+typedef rc_t ( * BAMOptionalDataFunction )
+ ( void *ctx, const char tag[2], const BAMOptData *value );
+
+rc_t BAM_AlignmentOptDataForEach
+ ( const BAM_Alignment *self, void *ctx, BAMOptionalDataFunction callback );
+
+
+bool BAM_AlignmentHasCGData(BAM_Alignment const *self);
+
+rc_t BAM_AlignmentCGReadLength(BAM_Alignment const *self, uint32_t *readlen);
+
+rc_t BAM_AlignmentGetCGSeqQual(BAM_Alignment const *self,
+ char sequence[],
+ uint8_t quality[]);
+
+rc_t BAM_AlignmentGetCGCigar(BAM_Alignment const *self,
+ uint32_t *cigar,
+ uint32_t cig_max,
+ uint32_t *cig_act);
+
+rc_t BAM_AlignmentGetTI(BAM_Alignment const *self, uint64_t *ti);
+
+/* strand = '+', '-', or ' ' */
+rc_t BAM_AlignmentGetRNAStrand(BAM_Alignment const *self, uint8_t *strand);
+
+rc_t BAM_AlignmentGetCGAlignGroup(BAM_Alignment const *self,
+ char buffer[],
+ size_t max_size,
+ size_t *act_size);
+
+
+/*--------------------------------------------------------------------------
+ * BAM_File
+ */
+typedef struct BAM_File BAM_File;
+
+typedef struct BAMRefSeq BAMRefSeq;
+struct BAMRefSeq
+{
+ uint64_t length;
+ const char *name; /* not null unique */
+ const char *assemblyId;
+ const uint8_t *checksum;
+ const char *uri;
+ const char *species;
+ uint32_t id;
+ uint8_t checksum_array[16];
+};
+
+typedef struct BAMReadGroup BAMReadGroup;
+struct BAMReadGroup
+{
+ const char *name; /* not null unique, accession e.g. SRR001138 */
+ const char *sample; /* not null */
+ const char *library;
+ const char *description;
+ const char *unit; /* platform specific identifier, e.g. BI.080214_SL-XAJ_0001_FC2044KAAXX.7 */
+ const char *insertSize;
+ const char *center; /* e.g. BI */
+ const char *runDate;
+ const char *platform; /* e.g. ILLUMINA */
+ uint32_t id;
+};
+
+
+/* 64-bit structure stored as an integer
+ * The high-order 48 bits store the position in the file at which a
+ * compressed block starts. The low-order 16 bits store the position
+ * in the decompressed block at which a record starts. This is the
+ * way that positions are represented in BAM indices.
+ */
+typedef uint64_t BAM_FilePosition;
+
+
+/* Make
+ * open the BAM file specified by path
+ *
+ * "path" [ IN ] - NUL terminated string or format
+ */
+rc_t BAM_FileMake ( const BAM_File **result, const char *path, ... );
+
+rc_t BAM_FileMakeWithHeader ( const BAM_File **result,
+ char const headerText[],
+ char const path[], ... );
+
+/* MakeWithDir
+ * open the BAM file specified by path and supplied directory
+ *
+ * "dir" [ IN ] - directory object used to open file
+ *
+ * "path" [ IN ] - NUL terminated string or format
+ */
+rc_t BAM_FileMakeWithDir ( const BAM_File **result,
+ struct KDirectory const *dir, const char *path, ... );
+rc_t BAM_FileVMakeWithDir ( const BAM_File **result,
+ struct KDirectory const *dir, const char *path, va_list args );
+
+/* Make
+ * open the BAM file specified by file
+ *
+ * "file" [ IN ] - an open KFile
+ */
+rc_t BAM_FileMakeWithKFile(const BAM_File **result,
+ struct KFile const *file);
+
+/* AddRef
+ * Release
+ */
+rc_t BAM_FileAddRef ( const BAM_File *self );
+rc_t BAM_FileRelease ( const BAM_File *self );
+
+
+/* GetPosition
+ * get the position of the about-to-be read alignment
+ * this position can be stored
+ * this position can be passed into SetPosition to seek to the same alignment
+ *
+ * "pos" [ OUT ] - return parameter for position
+ */
+rc_t BAM_FileGetPosition ( const BAM_File *self, BAM_FilePosition *pos );
+
+
+/* GetProportionalPosition
+ * get the aproximate proportional position in the input file
+ * this is intended to be useful for computing progress
+ *
+ * NB - does not return rc_t
+ */
+float BAM_FileGetProportionalPosition ( const BAM_File *self );
+
+
+/* Read
+ * read an aligment
+ *
+ * "result" [ OUT ] - return param for BAM_Alignment object
+ * must be released with BAM_AlignmentRelease
+ *
+ * returns RC(..., ..., ..., rcRow, rcNotFound) at end
+ */
+rc_t BAM_FileRead ( const BAM_File *self, const BAM_Alignment **result );
+
+
+/* Read
+ * read an aligment
+ *
+ * "result" [ OUT ] - return param for BAM_Alignment object
+ * must be released with BAM_AlignmentRelease, is invalidated or contents
+ * change on next call to BAM_FileRead2. Unlike with BAM_FileRead, no attempt is
+ * made to preserve this object.
+ *
+ * returns:
+ * RC(..., ..., ..., rcRow, rcNotFound) at end
+ * RC(..., ..., ..., rcRow, rcInvalid) and RC(..., ..., ..., rcRow, rcEmpty)
+ * are not fatal and are resumable
+ *
+ * tries to use static buffers and will log messages about parsing errors
+ */
+rc_t BAM_FileRead2 ( const BAM_File *self, const BAM_Alignment **result );
+
+
+/* Rewind
+ * reset the position back to the first aligment in the file
+ */
+rc_t BAM_FileRewind ( const BAM_File *self );
+
+
+/* SetPosition
+ * set the position to a particular alignment
+ * pass in the values from GetPosition
+ */
+rc_t BAM_FileSetPosition ( const BAM_File *self, const BAM_FilePosition *pos );
+
+
+/* GetRefSeqCount
+ * get the number of Reference Sequences refered to in the header
+ * this is not necessarily the number of Reference Sequences referenced
+ * by the alignments
+ */
+rc_t BAM_FileGetRefSeqCount ( const BAM_File *self, uint32_t *count );
+
+
+/* GetRefSeq
+ * get the n'th Ref. Seq. where n is [0..RefSeqCount)
+ * the resulting pointer is static-like; it is freed when the BAM_File is.
+ * IOW, it is good for precisely at long as the BAM_File is.
+ */
+rc_t BAM_FileGetRefSeq ( const BAM_File *self, uint32_t n, const BAMRefSeq **result );
+
+
+/* GetRefSeqById
+ * get a Ref. Seq. by its id
+ * the resulting pointer is static-like; it is freed when the BAM_File is.
+ * IOW, it is good for precisely at long as the BAM_File is.
+ */
+rc_t BAM_FileGetRefSeqById ( const BAM_File *self, int32_t id, const BAMRefSeq **result );
+
+
+/* GetReadGroupCount
+ * get the number of Read Groups (accessions, etc.) refered to in the header
+ * this is not necessarily the number of Read Groups referenced
+ * by the alignments
+ */
+rc_t BAM_FileGetReadGroupCount ( const BAM_File *self, uint32_t *count );
+
+/* GetReadGroup
+ * get the n'th Read Group where n is [0..ReadGroupCount)
+ * the resulting pointer is static-like; it is freed when the BAM_File is.
+ * IOW, it is good for precisely at long as the BAM_File is.
+ */
+rc_t BAM_FileGetReadGroup ( const BAM_File *self, unsigned n, const BAMReadGroup **result );
+
+/* GetHeaderText
+ * get the text of the BAM header file
+ * the resulting pointer is static-like; it is freed when the BAM_File is.
+ * IOW, it is good for precisely at long as the BAM_File is.
+ */
+rc_t BAM_FileGetHeaderText(BAM_File const *cself, char const **header, size_t *header_len);
+
+
+/* GetReadGroupByName
+ * get a Read Group by its name
+ * the resulting pointer is static-like; it is freed when the BAM_File is.
+ * IOW, it is good for precisely at long as the BAM_File is.
+ */
+rc_t BAM_FileGetReadGroupByName ( const BAM_File *self,
+ const char *name, const BAMReadGroup **result );
diff --git a/tools/bam-loader/config-file.cpp b/tools/bam-loader/config-file.cpp
new file mode 100644
index 0000000..9eca698
--- /dev/null
+++ b/tools/bam-loader/config-file.cpp
@@ -0,0 +1,195 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*/
+
+#include "config-file.hpp"
+#include <string>
+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <cctype>
+#include <sstream>
+
+/*
+ * Config file:
+ * The config file consists of lines containing whitespace (ASCII 9 or 32)
+ * seperated fields. The fields are:
+ * NAME (unique)
+ * SEQID
+ * extra (optional)
+ */
+
+/// reads a line with leading and trailing whitespace trimmed;
+/// ignores lines consisting entirely of whitespace;
+static std::string const getline(std::istream &is)
+{
+ std::string line("");
+ auto ws = true;
+ size_t len = 0;
+
+ for ( ; ; ) {
+ auto const ch = is.get();
+ if (ch < 0)
+ break;
+ if (ws && isspace(ch))
+ continue;
+
+ if (ch == '\n' || ch == '\r')
+ break;
+
+ ws = false;
+ line.push_back(ch);
+ if (!isspace(ch))
+ len = line.size();
+ }
+ line.erase(len);
+
+ return line;
+}
+
+template <typename T>
+struct Range {
+ T start, end;
+
+ Range(T const init) : start(init), end(init) {}
+ T const size() const {
+ return start < end ? (end - start) : 0;
+ }
+};
+
+struct Parse {
+ Range<std::string::size_type> name;
+ Range<std::string::size_type> seqid;
+ Range<std::string::size_type> extra;
+
+ Parse()
+ : name(std::string::npos)
+ , seqid(std::string::npos)
+ , extra(std::string::npos)
+ {}
+ bool good() const {
+ return name.size() > 0 && seqid.size() > 0;
+ }
+};
+
+static Parse parseLine(std::string const &in) {
+ static std::string const whitespace(" \t");
+ Parse rslt;
+
+ if (in[0] != '#') {
+ rslt.name.start = 0;
+
+ rslt.name.end = in.find_first_of(whitespace);
+ if (rslt.name.end != std::string::npos) {
+ rslt.seqid.start = in.find_first_not_of(whitespace, rslt.name.end);
+ if (rslt.seqid.start != std::string::npos) {
+ rslt.seqid.end = in.find_first_of(whitespace, rslt.seqid.start);
+ if (rslt.seqid.end == std::string::npos)
+ rslt.seqid.end = in.size();
+ else {
+ rslt.extra.start = in.find_first_not_of(whitespace, rslt.seqid.end);
+ rslt.extra.end = in.size();
+ }
+ }
+ }
+ }
+
+ return rslt;
+}
+
+static ConfigFile::Line makeLine(std::string const &in, Parse const &parse) {
+ ConfigFile::Line rslt;
+
+ rslt.NAME = in.substr(parse.name.start, parse.name.size());
+ rslt.SEQID = in.substr(parse.seqid.start, parse.seqid.size());
+ if (parse.extra.size() > 0)
+ rslt.EXTRA = in.substr(parse.extra.start, parse.extra.size());
+ else
+ rslt.EXTRA = "";
+
+ return rslt;
+}
+
+static ConfigFile::Unparsed makeUnparsed(unsigned const lineno, std::string const &line) {
+ ConfigFile::Unparsed rslt;
+
+ rslt.lineno = lineno;
+ rslt.line = line;
+
+ return rslt;
+}
+
+ConfigFile::ConfigFile(std::istream &is) {
+ unsigned lineno = 0;
+
+ while (is.good()) {
+ auto const in = getline(is);
+ if (in.size() == 0) {
+ break;
+ }
+ ++lineno;
+
+ auto const parse = parseLine(in);
+ if (parse.good())
+ lines.push_back(makeLine(in, parse));
+ else
+ unparsed.push_back(makeUnparsed(lineno, in));
+ }
+ if (is.eof())
+ msg = "no errors";
+ else
+ msg = "error reading input";
+
+ std::ostringstream oss;
+ oss << msg << "; lines read: " << lineno;
+ msg = oss.str();
+}
+
+ConfigFile ConfigFile::load(std::string const &filename) {
+ std::ifstream ifs(filename);
+
+ return ifs.is_open() ? ConfigFile::load(ifs) : ConfigFile();
+}
+
+void ConfigFile::printDescription(std::ostream &os, bool const detail) const {
+ os << "Loaded " << lines.size() << " records" << std::endl;
+ if (detail) {
+ for (auto i = lines.begin(); i != lines.end(); ++i)
+ os << i->NAME << '\t' << i->SEQID << '\t' << i->EXTRA << std::endl;
+ }
+ os << "Unparsed lines: " << unparsed.size() << std::endl;
+ if (detail) {
+ for (auto i = unparsed.begin(); i != unparsed.end(); ++i)
+ os << i->lineno << '\t' << i->line << std::endl;
+ }
+ os << msg << std::endl;
+}
+
+#ifdef TESTING
+int main(int argc, char *argv[]) {
+ auto const config = argc > 1 ? ConfigFile::load(argv[1]) : ConfigFile::load(std::cin);
+
+ config.printDescription(std::cout, true);
+}
+#endif
diff --git a/tools/run-stat/key_value.h b/tools/bam-loader/config-file.hpp
similarity index 58%
rename from tools/run-stat/key_value.h
rename to tools/bam-loader/config-file.hpp
index 35e6719..dbb15f4 100644
--- a/tools/run-stat/key_value.h
+++ b/tools/bam-loader/config-file.hpp
@@ -1,4 +1,4 @@
-/*===========================================================================
+/*==============================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
@@ -21,44 +21,45 @@
* Please cite the author in any work or product based on this material.
*
* ===========================================================================
-*
*/
-#ifndef _h_key_value_
-#define _h_key_value_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include <klib/rc.h>
-#include "helper.h"
-
-typedef struct report
-{
- VNamelist * columns;
- uint32_t col_count;
- Vector data;
- uint32_t *max_width;
-} report;
-typedef report* p_report;
-
-
-rc_t report_init( p_report * self, uint32_t prealloc );
-
-rc_t report_destroy( p_report self );
-
-rc_t report_clear( p_report self );
-
-rc_t report_set_columns( p_report self, size_t count, ... );
+#include <string>
+#include <vector>
+#include <iostream>
-rc_t report_new_data( p_report self );
+/*
+ * Config file:
+ * The config file consists of lines containing whitespace (ASCII 9 or 32)
+ * seperated fields. The fields are:
+ * NAME (unique)
+ * SEQID
+ * extra (optional)
+ */
-rc_t report_add_data( p_report self, const size_t estimated_len, const char * fmt, ... );
+class ConfigFile {
+ ConfigFile() {}
+ ConfigFile(std::istream &is);
+public:
+ struct Line {
+ std::string NAME;
+ std::string SEQID;
+ std::string EXTRA;
+ };
+ struct Unparsed {
+ unsigned lineno;
+ std::string line;
+ };
+
+ std::vector<Line const> lines;
+ std::vector<Unparsed const> unparsed;
+ std::string msg;
-rc_t report_print( p_report self, p_char_buffer dst, uint32_t mode );
+ ~ConfigFile() {}
+
+ void printDescription(std::ostream &, bool detail = false) const;
-#endif
\ No newline at end of file
+ static ConfigFile load(std::istream &is) {
+ return ConfigFile(is);
+ }
+ static ConfigFile load(std::string const &filename);
+};
diff --git a/tools/bam-loader/fasta-file.cpp b/tools/bam-loader/fasta-file.cpp
new file mode 100644
index 0000000..013cee8
--- /dev/null
+++ b/tools/bam-loader/fasta-file.cpp
@@ -0,0 +1,257 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*/
+
+#include "fasta-file.hpp"
+#include <string>
+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <cctype>
+
+/*
+ * Fasta files:
+ * Fasta file consists of one of more sequences. A sequence in a fasta file
+ * consists of a seqid line followed by lines containing the bases of the
+ * sequence. A seqid line starts with '>' and the next word (whitespace
+ * delimited) is the seqid.
+ */
+
+struct tmpSequence {
+ std::string SEQID;
+ std::string SEQID_LINE;
+ size_t data_start;
+ unsigned data_size;
+ bool hadErrors;
+};
+
+/// reads a line with leading and trailing whitespace trimmed;
+/// ignores lines consisting entirely of whitespace;
+static std::string const getline(std::istream &is)
+{
+ std::string line("");
+ auto ws = true;
+ size_t len = 0;
+
+ for ( ; ; ) {
+ auto const ch = is.get();
+ if (ch < 0)
+ break;
+ if (ws && isspace(ch))
+ continue;
+
+ if (ch == '\n' || ch == '\r')
+ break;
+
+ ws = false;
+ line.push_back(ch);
+ if (!isspace(ch))
+ len = line.size();
+ }
+ line.erase(len);
+
+ return line;
+}
+
+static bool cleanCopyFastaSequence(char *const dst,
+ char const *const src,
+ unsigned const len)
+{
+ static char const tr[] = {
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','.',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ','A','B','C','D',' ',' ','G','H',' ',' ','K',' ','M','N',' ',
+ ' ',' ','R','S','T',' ','V','W','N','Y',' ',' ',' ',' ',' ',' ',
+ ' ','A','B','C','D',' ',' ','G','H',' ',' ','K',' ','M','N',' ',
+ ' ',' ','R','S','T',' ','V','W','N','Y',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ ' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',
+ };
+
+ auto hadErrors = false;
+
+ for (unsigned i = 0; i < len; ++i) {
+ int const ch = src[i]; // .ACMGRSVTWYHKDBN and lower case
+
+ if (ch != ' ')
+ dst[i] = ch;
+ else {
+ dst[i] = 'N';
+ hadErrors = true;
+ }
+ }
+
+ return hadErrors;
+}
+
+static bool readFile(std::istream &is,
+ char *&data,
+ size_t &size,
+ size_t &limit,
+ std::vector<tmpSequence> &sequences
+ )
+{
+ static std::string const whitespace("\t ");
+ tmpSequence sequence;
+ auto st = 0;
+
+ for ( ; ; ) {
+ auto line = getline(is);
+
+ switch (st) {
+ case 1:
+ if (line[0] == '>') {
+ auto const seqidstart = line.find_first_not_of(whitespace, 1);
+
+ if (seqidstart != std::string::npos) {
+ sequence.SEQID_LINE.push_back(' ');
+ sequence.SEQID_LINE.append(line, seqidstart, std::string::npos);
+ }
+ break;
+ }
+ ++st;
+ /* fallthrough */
+ case 2:
+ if (line[0] == '>' || line.size() == 0) {
+ sequences.push_back(sequence);
+ st = 0;
+ /* fallthrough */
+ }
+ else {
+ auto const start = line.find_first_not_of(whitespace);
+ auto const len = line.size() - start;
+
+ if (size + len > limit) {
+ do { limit <<= 1; } while (size + len > limit);
+ auto const tmp = realloc(reinterpret_cast<void *>(data), limit);
+
+ if (!tmp) throw std::bad_alloc();
+ data = reinterpret_cast<char *>(tmp);
+ }
+
+ sequence.hadErrors |= cleanCopyFastaSequence(data + size, line.data() + start, len);
+ size += len;
+ sequence.data_size += len;
+ break;
+ }
+ case 0:
+ if (line.size() == 0)
+ return true;
+
+ if (line[0] == '>') {
+ auto const seqidstart = line.find_first_not_of(whitespace, 1);
+ if (seqidstart != std::string::npos) {
+ auto const seqidend = line.find_first_of(whitespace, seqidstart);
+
+ sequence.SEQID_LINE = std::string(line, seqidstart);
+ sequence.SEQID = std::string(line, seqidstart, seqidend - seqidstart);
+ sequence.data_start = size;
+ sequence.data_size = 0;
+ sequence.hadErrors = false;
+ ++st;
+ break;
+ }
+ }
+ default:
+ return false;
+ }
+ }
+}
+
+FastaFile::FastaFile(std::istream &is) : data(nullptr)
+{
+ std::vector<tmpSequence> tmp;
+ size_t limit = 1024u * 1024u;
+ size_t size = 0;
+ auto data = reinterpret_cast<char *>(malloc(limit));
+
+ if (!data)
+ throw std::bad_alloc();
+
+ if (readFile(is, data, size, limit, tmp) && is.eof()) {
+ this->data = realloc(reinterpret_cast<void *>(data), size);
+ if (!this->data) throw std::bad_alloc();
+
+ for (auto i = tmp.begin(); i != tmp.end(); ++i) {
+ Sequence seq;
+
+ seq.SEQID = i->SEQID;
+ seq.SEQID_LINE = i->SEQID_LINE;
+ seq.data = data + i->data_start;
+ seq.length = i->data_size;
+ seq.hadErrors = i->hadErrors;
+
+ sequences.push_back(seq);
+ }
+ }
+ else {
+ free(data);
+ }
+}
+
+FastaFile FastaFile::load(std::string const filename)
+{
+ std::ifstream ifs(filename);
+
+ return ifs.is_open() ? FastaFile::load(ifs) : FastaFile();
+}
+
+#ifdef TESTING
+void wait(std::string const &msg = "Waiting") {
+ std::string s;
+
+ std::cout << msg << "... [Press enter]" << std::endl;
+ std::getline(std::cin, s);
+}
+
+void test(std::string const &filename) {
+ auto const test = FastaFile::load(filename);
+
+ std::cout << "Loaded " << test.sequences.size() << " sequences" << std::endl;
+
+ size_t total = 0;
+ for (auto i = test.sequences.begin(); i != test.sequences.end(); ++i)
+ total += i->length;
+
+ std::cout << "Loaded " << total << " bases" << std::endl;
+
+// wait("Run leaks");
+}
+
+int main(int argc, char *argv[])
+{
+ if (argc > 1) {
+ test(argv[1]);
+ }
+// wait("Run leaks again");
+}
+#endif
diff --git a/tools/sra-sort/idx-mapping.h b/tools/bam-loader/fasta-file.hpp
similarity index 57%
copy from tools/sra-sort/idx-mapping.h
copy to tools/bam-loader/fasta-file.hpp
index 7691526..543c4b0 100644
--- a/tools/sra-sort/idx-mapping.h
+++ b/tools/bam-loader/fasta-file.hpp
@@ -1,4 +1,4 @@
-/*===========================================================================
+/*==============================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
@@ -21,39 +21,43 @@
* Please cite the author in any work or product based on this material.
*
* ===========================================================================
- *
*/
-#ifndef _h_sra_sort_idx_mapping_
-#define _h_sra_sort_idx_mapping_
-
-#ifndef _h_sra_sort_defs_
-#include "sort-defs.h"
-#endif
-
+#include <string>
+#include <vector>
+#include <iostream>
-/*--------------------------------------------------------------------------
- * IdxMapping
- *
+/*
+ * Fasta files:
+ * Fasta file consists of one of more sequences. A sequence in a fasta file
+ * consists of a seqid line followed by lines containing the bases of the
+ * sequence. A seqid line starts with '>' and the next word (whitespace
+ * delimited) is the seqid.
*/
-typedef struct IdxMapping IdxMapping;
-struct IdxMapping
-{
- int64_t old_id, new_id;
-};
-
-#if USE_OLD_KSORT
-/* ksort callbacks */
-int CC IdxMappingCmpOld ( const void *a, const void *b, void *data );
-int CC IdxMappingCmpNew ( const void *a, const void *b, void *data );
-
-#else
-
-/* ksort_inlines */
-void IdxMappingSortOld ( IdxMapping *self, const ctx_t *ctx, size_t count );
-void IdxMappingSortNew ( IdxMapping *self, const ctx_t *ctx, size_t count );
-
-#endif
-
-#endif /* _h_sra_sort_idx_mapping_ */
+class FastaFile {
+ FastaFile() : data(NULL) {}
+ FastaFile(std::istream &is);
+
+ void *data;
+public:
+ struct Sequence {
+ std::string SEQID;
+ std::string SEQID_LINE;
+ char const *data;
+ unsigned length;
+ bool hadErrors; // erroneous base values are replaced with N
+ };
+
+ std::vector<Sequence const> sequences;
+
+ ~FastaFile() {
+ free(data);
+ data = nullptr;
+ }
+
+ static FastaFile load(std::istream &is) {
+ return FastaFile(is);
+ }
+ static FastaFile load(std::string const filename);
+};
diff --git a/tools/bam-loader/loader-imp.c b/tools/bam-loader/loader-imp.c
index 2666bf1..3dbbd5f 100644
--- a/tools/bam-loader/loader-imp.c
+++ b/tools/bam-loader/loader-imp.c
@@ -38,9 +38,6 @@
#include <kfs/directory.h>
#include <kfs/file.h>
-#include <kfs/mmap.h>
-#include <kfs/pagefile.h>
-#include <kfs/pmem.h>
#include <kdb/btree.h>
#include <kdb/manager.h>
#include <kdb/database.h>
@@ -69,6 +66,9 @@
#include <sysalloc.h>
#include <atomic32.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/mman.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
@@ -77,33 +77,32 @@
#include <limits.h>
#include <time.h>
-#include <align/bam.h>
-
+#include "bam.h"
#include "Globals.h"
#include "sequence-writer.h"
#include "reference-writer.h"
#include "alignment-writer.h"
+#include "mem-bank.h"
#define NUM_ID_SPACES (256u)
-#define MMA_NUM_CHUNKS_BITS (24u)
+#define MMA_NUM_CHUNKS_BITS (20u)
#define MMA_NUM_SUBCHUNKS_BITS ((32u)-(MMA_NUM_CHUNKS_BITS))
#define MMA_SUBCHUNK_SIZE (1u << MMA_NUM_CHUNKS_BITS)
#define MMA_SUBCHUNK_COUNT (1u << MMA_NUM_SUBCHUNKS_BITS)
typedef struct {
- KFile *fp;
+ int fd;
size_t elemSize;
- uint64_t fsize;
+ off_t fsize;
+ uint8_t *current;
struct mma_map_s {
struct mma_submap_s {
uint8_t *base;
- KMMap *mmap;
} submap[MMA_SUBCHUNK_COUNT];
} map[NUM_ID_SPACES];
} MMArray;
-#define FRAG_CHUNK_SIZE (128)
typedef struct {
uint32_t primaryId[2];
uint32_t spotId;
@@ -141,13 +140,12 @@ typedef struct context_t {
KBTree *key2id[NUM_ID_SPACES];
char *key2id_names;
MMArray *id2value;
- KMemBank *fragsBoth; /*** mate will be there soon ***/
- KMemBank *fragsOne; /*** mate may not be found soon or even show up ***/
+ MemBank *frags;
int64_t spotId;
int64_t primaryId;
int64_t secondId;
uint64_t alignCount;
-
+
uint32_t idCount[NUM_ID_SPACES];
uint32_t key2id_hash[NUM_ID_SPACES];
@@ -155,12 +153,12 @@ typedef struct context_t {
unsigned key2id_name_max;
unsigned key2id_name_alloc;
unsigned key2id_count;
-
+
unsigned key2id_name[NUM_ID_SPACES];
/* this array is kept in name order */
/* this maps the names to key2id and idCount */
unsigned key2id_oid[NUM_ID_SPACES];
-
+
unsigned pass;
bool isColorSpace;
} context_t;
@@ -175,80 +173,125 @@ static char const *Print_ctx_value_t(ctx_value_t const *const self)
return buffer;
}
-static rc_t MMArrayMake(MMArray **rslt, KFile *fp, uint32_t elemSize)
+static rc_t MMArrayMake(MMArray **rslt, int fd, uint32_t elemSize)
{
MMArray *const self = calloc(1, sizeof(*self));
if (self == NULL)
return RC(rcExe, rcMemMap, rcConstructing, rcMemory, rcExhausted);
self->elemSize = (elemSize + 3) & ~(3u); /** align to 4 byte **/
- self->fp = fp;
- KFileAddRef(fp);
+ self->fd = fd;
*rslt = self;
return 0;
}
#define PERF 0
+#define PROT 0
static rc_t MMArrayGet(MMArray *const self, void **const value, uint64_t const element)
{
+ size_t const chunk = MMA_SUBCHUNK_SIZE * self->elemSize;
unsigned const bin_no = element >> 32;
unsigned const subbin = ((uint32_t)element) >> MMA_NUM_CHUNKS_BITS;
unsigned const in_bin = (uint32_t)element & (MMA_SUBCHUNK_SIZE - 1);
if (bin_no >= sizeof(self->map)/sizeof(self->map[0]))
return RC(rcExe, rcMemMap, rcConstructing, rcId, rcExcessive);
-
+
if (self->map[bin_no].submap[subbin].base == NULL) {
- size_t const chunk = MMA_SUBCHUNK_SIZE * self->elemSize;
- size_t const fsize = self->fsize + chunk;
- rc_t rc = KFileSetSize(self->fp, fsize);
-
- if (rc == 0) {
- KMMap *mmap;
-
- self->fsize = fsize;
- rc = KMMapMakeRgnUpdate(&mmap, self->fp, self->fsize, chunk);
- if (rc == 0) {
- void *base;
-
- rc = KMMapAddrUpdate(mmap, &base);
- if (rc == 0) {
+ off_t const cur_fsize = self->fsize;
+ off_t const new_fsize = cur_fsize + chunk;
+
+ if (ftruncate(self->fd, new_fsize) != 0)
+ return RC(rcExe, rcFile, rcResizing, rcSize, rcExcessive);
+ else {
+ void *const base = mmap(NULL, chunk, PROT_READ|PROT_WRITE,
+ MAP_FILE|MAP_SHARED, self->fd, cur_fsize);
+
+ self->fsize = new_fsize;
+ if (base == MAP_FAILED) {
+ PLOGMSG(klogErr, (klogErr, "Failed to construct map for bin $(bin), subbin $(subbin)", "bin=%u,subbin=%u", bin_no, subbin));
+ return RC(rcExe, rcMemMap, rcConstructing, rcMemory, rcExhausted);
+ }
+ else {
#if PERF
- static unsigned mapcount = 0;
+ static unsigned mapcount = 0;
- (void)PLOGMSG(klogInfo, (klogInfo, "Number of mmaps: $(cnt)", "cnt=%u", ++mapcount));
+ (void)PLOGMSG(klogInfo, (klogInfo, "Number of mmaps: $(cnt)", "cnt=%u", ++mapcount));
#endif
- self->map[bin_no].submap[subbin].mmap = mmap;
- self->map[bin_no].submap[subbin].base = base;
-
- goto GET_MAP;
- }
- KMMapRelease(mmap);
+ self->map[bin_no].submap[subbin].base = base;
}
}
- return rc;
}
-GET_MAP:
- *value = &self->map[bin_no].submap[subbin].base[(size_t)in_bin * self->elemSize];
+ uint8_t *const next = self->map[bin_no].submap[subbin].base;
+#if PROT
+ if (next != self->current) {
+ void *const current = self->current;
+
+ if (current)
+ mprotect(current, chunk, PROT_NONE);
+
+ mprotect(self->current = next, chunk, PROT_READ|PROT_WRITE);
+ }
+#endif
+ *value = &next[(size_t)in_bin * self->elemSize];
return 0;
}
+static rc_t MMArrayGetRead(MMArray *const self, void const **const value, uint64_t const element)
+{
+ unsigned const bin_no = element >> 32;
+ unsigned const subbin = ((uint32_t)element) >> MMA_NUM_CHUNKS_BITS;
+ unsigned const in_bin = (uint32_t)element & (MMA_SUBCHUNK_SIZE - 1);
+
+ if (bin_no >= sizeof(self->map)/sizeof(self->map[0]))
+ return RC(rcExe, rcMemMap, rcConstructing, rcId, rcExcessive);
+
+ if (self->map[bin_no].submap[subbin].base == NULL)
+ return RC(rcExe, rcMemMap, rcReading, rcId, rcInvalid);
+
+ uint8_t *const next = self->map[bin_no].submap[subbin].base;
+#if PROT
+ size_t const chunk = MMA_SUBCHUNK_SIZE * self->elemSize;
+ if (next != self->current) {
+ void *const current = self->current;
+
+ if (current)
+ mprotect(current, chunk, PROT_NONE);
+
+ mprotect(self->current = next, chunk, PROT_READ);
+ }
+#endif
+ *value = &next[(size_t)in_bin * self->elemSize];
+ return 0;
+}
+
+static void MMArrayLock(MMArray *const self)
+{
+#if PROT
+ size_t const chunk = MMA_SUBCHUNK_SIZE * self->elemSize;
+ void *const current = self->current;
+
+ self->current = NULL;
+ if (current)
+ mprotect(current, chunk, PROT_NONE);
+#endif
+}
+
static void MMArrayWhack(MMArray *self)
{
+ size_t const chunk = MMA_SUBCHUNK_SIZE * self->elemSize;
unsigned i;
for (i = 0; i != sizeof(self->map)/sizeof(self->map[0]); ++i) {
unsigned j;
-
+
for (j = 0; j != sizeof(self->map[0].submap)/sizeof(self->map[0].submap[0]); ++j) {
- if (self->map[i].submap[j].mmap)
- KMMapRelease(self->map[i].submap[j].mmap);
- self->map[i].submap[j].mmap = NULL;
- self->map[i].submap[j].base = NULL;
+ if (self->map[i].submap[j].base)
+ munmap(self->map[i].submap[j].base, chunk);
}
}
- KFileRelease(self->fp);
+ close(self->fd);
free(self);
}
@@ -260,11 +303,11 @@ static rc_t OpenKBTree(KBTree **const rslt, unsigned n, unsigned max)
KDirectory *dir;
char fname[4096];
rc_t rc;
-
+
rc = KDirectoryNativeDir(&dir);
if (rc)
return rc;
-
+
rc = string_printf(fname, sizeof(fname), NULL, "%s/key2id.%u.%u", G.tmpfs, G.pid, n); if (rc) return rc;
rc = KDirectoryCreateFile(dir, &file, true, 0600, kcmInit, "%s", fname);
KDirectoryRemove(dir, 0, "%s", fname);
@@ -309,7 +352,7 @@ static rc_t GetKeyIDOld(context_t *const ctx, uint64_t *const rslt, bool *const
char *hbuf = NULL;
size_t bsize = sizeof(sbuf);
size_t actsize;
-
+
if (keylen + namelen + 2 > bsize) {
hbuf = malloc(bsize = keylen + namelen + 2);
if (hbuf == NULL)
@@ -317,7 +360,7 @@ static rc_t GetKeyIDOld(context_t *const ctx, uint64_t *const rslt, bool *const
buf = hbuf;
}
rc = string_printf(buf, bsize, &actsize, "%s\t%.*s", key, (int)namelen, name);
-
+
tmpKey = ctx->idCount[0];
rc = KBTreeEntry(ctx->key2id[0], &tmpKey, wasInserted, buf, actsize);
if (hbuf)
@@ -331,8 +374,9 @@ static rc_t GetKeyIDOld(context_t *const ctx, uint64_t *const rslt, bool *const
return rc;
}
-static unsigned HashKey(void const *key, unsigned keylen)
+static unsigned HashKey(void const *const key, unsigned const keylen)
{
+#if 0
/* There is nothing special about this hash. It was randomly generated. */
static const uint8_t T1[] = {
64, 186, 39, 203, 54, 211, 98, 32, 26, 23, 219, 94, 77, 60, 56, 184,
@@ -354,13 +398,24 @@ static unsigned HashKey(void const *key, unsigned keylen)
};
unsigned h = 0x55;
unsigned i = keylen;
-
+
do { h = T1[h ^ ((uint8_t)i)]; } while ((i >>= 8) != 0);
for (i = 0; i != keylen; ++i)
h = T1[h ^ ((uint8_t const *)key)[i]];
return h;
+#else
+ /* FNV-1a hash with folding */
+ uint64_t h = 0xcbf29ce484222325;
+ unsigned i;
+
+ for (i = 0; i < keylen; ++i) {
+ uint8_t const octet = ((uint8_t const *)key)[i];
+ h = (h ^ octet) * 0x100000001b3ull;
+ }
+ return ((uint32_t)(h ^ (h >> 32))) % NUM_ID_SPACES;
+#endif
}
#define USE_ILLUMINA_NAMING_CORRECTION 1
@@ -368,44 +423,44 @@ static unsigned HashKey(void const *key, unsigned keylen)
static size_t GetFixedNameLength(char const name[], size_t const namelen)
{
#if USE_ILLUMINA_NAMING_CORRECTION
-/*** Check for possible fixes to illumina names ****/
+ /*** Check for possible fixes to illumina names ****/
size_t newlen=namelen;
/*** First get rid of possible "/1" "/2" "/3" at the end - violates SAM spec **/
if(newlen > 2 && name[newlen-2] == '/' && (name[newlen-1] == '1' || name[newlen-1] == '2' || name[newlen-1] == '3')){
- newlen -=2;
+ newlen -=2;
}
if(newlen > 2 && name[newlen-2] == '#' && (name[newlen-1] == '0')){ /*** Now, find "#0" ***/
- newlen -=2;
+ newlen -=2;
} else if(newlen>10){ /*** find #ACGT ***/
- int i=newlen;
- for(i--;i>4;i--){ /*** stopping at 4 since the rest of record should still contain :x:y ***/
- char a=toupper(name[i]);
- if(a != 'A' && a != 'C' && a !='G' && a !='T'){
- break;
- }
- }
- if(name[i]=='#'){
- switch (newlen-i){ /** allowed values for illumina barcodes :5,6,8 **/
- case 5:
- case 6:
- case 8:
- newlen=i;
- break;
- default:
- break;
- }
- }
+ int i=newlen;
+ for(i--;i>4;i--){ /*** stopping at 4 since the rest of record should still contain :x:y ***/
+ char a=toupper(name[i]);
+ if(a != 'A' && a != 'C' && a !='G' && a !='T'){
+ break;
+ }
+ }
+ if (name[i] == '#'){
+ switch (newlen-i) { /** allowed values for illumina barcodes :5,6,8 **/
+ case 5:
+ case 6:
+ case 8:
+ newlen=i;
+ break;
+ default:
+ break;
+ }
+ }
}
if(newlen < namelen){ /*** check for :x:y at the end now - to make sure it is illumina **/
- int i=newlen;
- for(i--;i>0 && isdigit(name[i]);i--){}
- if(name[i]==':'){
- for(i--;i>0 && isdigit(name[i]);i--){}
- if(name[i]==':' && newlen > 0){ /*** some name before :x:y should still exist **/
- /*** looks like illumina ***/
- return newlen;
- }
- }
+ int i=newlen;
+ for(i--;i>0 && isdigit(name[i]);i--){}
+ if(name[i]==':'){
+ for(i--;i>0 && isdigit(name[i]);i--){}
+ if(name[i]==':' && newlen > 0){ /*** some name before :x:y should still exist **/
+ /*** looks like illumina ***/
+ return newlen;
+ }
+ }
}
#endif
return namelen;
@@ -429,7 +484,7 @@ rc_t GetKeyID(context_t *const ctx,
unsigned f;
unsigned e = ctx->key2id_count;
uint64_t tmpKey;
-
+
*rslt = 0;
{{
uint32_t const bucket_value = ctx->key2id_hash[h];
@@ -437,7 +492,7 @@ rc_t GetKeyID(context_t *const ctx,
unsigned const i1 = (uint8_t)(bucket_value >> 8);
unsigned const i2 = (uint8_t)(bucket_value >> 16);
unsigned const i3 = (uint8_t)(bucket_value >> 24);
-
+
if (n > 0 && strcmp(key, ctx->key2id_names + ctx->key2id_name[i1]) == 0) {
f = i1;
/*
@@ -461,7 +516,7 @@ rc_t GetKeyID(context_t *const ctx,
unsigned const m = (f + e) / 2;
unsigned const oid = ctx->key2id_oid[m];
int const diff = strcmp(key, ctx->key2id_names + ctx->key2id_name[oid]);
-
+
if (diff < 0)
e = m;
else if (diff > 0)
@@ -475,13 +530,13 @@ rc_t GetKeyID(context_t *const ctx,
unsigned const name_max = ctx->key2id_name_max + keylen + 1;
KBTree *tree;
rc_t rc = OpenKBTree(&tree, ctx->key2id_count + 1, 1); /* ctx->key2id_max); */
-
+
if (rc) return rc;
-
+
if (ctx->key2id_name_alloc < name_max) {
unsigned alloc = ctx->key2id_name_alloc;
void *tmp;
-
+
if (alloc == 0)
alloc = 4096;
while (alloc < name_max)
@@ -506,7 +561,7 @@ rc_t GetKeyID(context_t *const ctx,
ctx->idCount[f] = 0;
if ((uint8_t)ctx->key2id_hash[h] < 3) {
unsigned const n = (uint8_t)ctx->key2id_hash[h] + 1;
-
+
ctx->key2id_hash[h] = (((ctx->key2id_hash[h] & ~(0xFFu)) | f) << 8) | n;
}
else {
@@ -519,7 +574,6 @@ rc_t GetKeyID(context_t *const ctx,
tmpKey = ctx->idCount[f];
rc = KBTreeEntry(ctx->key2id[f], &tmpKey, wasInserted, name, namelen);
if (rc == 0) {
- /* fprintf(stderr, "GetKeyID: { Key: '%s', Name: '%.*s', id: '%u:%x', new: %s }\n", key, (int)namelen, name, (unsigned)f, (unsigned)tmpKey, *wasInserted ? "true" : "false"); */
*rslt = (((uint64_t)f) << 32) | tmpKey;
if (*wasInserted)
++ctx->idCount[f];
@@ -533,43 +587,27 @@ rc_t GetKeyID(context_t *const ctx,
static rc_t OpenMMapFile(context_t *const ctx, KDirectory *const dir)
{
- KFile *file = NULL;
+ int fd;
char fname[4096];
rc_t rc = string_printf(fname, sizeof(fname), NULL, "%s/id2value.%u", G.tmpfs, G.pid);
-
+
if (rc)
return rc;
-
- rc = KDirectoryCreateFile(dir, &file, true, 0600, kcmInit, "%s", fname);
- KDirectoryRemove(dir, 0, "%s", fname);
- if (rc == 0)
- rc = MMArrayMake(&ctx->id2value, file, sizeof(ctx_value_t));
- KFileRelease(file);
- return rc;
+
+ fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, S_IRUSR|S_IWUSR);
+ if (fd < 0)
+ return RC(rcExe, rcFile, rcCreating, rcFile, rcNotFound);
+ unlink(fname);
+ return MMArrayMake(&ctx->id2value, fd, sizeof(ctx_value_t));
}
-static rc_t OpenMBankFile(context_t *const ctx, KDirectory *const dir, int which, size_t climit)
+static rc_t TmpfsDirectory(KDirectory **const rslt)
{
- KFile *file = NULL;
- char fname[4096];
- char const *const suffix = which == 1 ? "One" : "Both";
- KMemBank **const mbank = which == 1 ? &ctx->fragsOne : &ctx->fragsBoth;
- rc_t rc = string_printf(fname, sizeof(fname), NULL, "%s/frag_data%s.%u", G.tmpfs, suffix, G.pid);
-
- if (rc)
- return rc;
-
- rc = KDirectoryCreateFile(dir, &file, true, 0600, kcmInit, "%s", fname);
- KDirectoryRemove(dir, 0, "%s", fname);
+ KDirectory *dir;
+ rc_t rc = KDirectoryNativeDir(&dir);
if (rc == 0) {
- KPageFile *backing;
-
- rc = KPageFileMakeUpdate(&backing, file, climit, false);
- KFileRelease(file);
- if (rc == 0) {
- rc = KMemBankMake(mbank, FRAG_CHUNK_SIZE, 0, backing);
- KPageFileRelease(backing);
- }
+ rc = KDirectoryOpenDirUpdate(dir, rslt, false, "%s", G.tmpfs);
+ KDirectoryRelease(dir);
}
return rc;
}
@@ -579,29 +617,26 @@ static rc_t SetupContext(context_t *ctx, unsigned numfiles)
rc_t rc = 0;
memset(ctx, 0, sizeof(*ctx));
-
+
if (G.mode == mode_Archive) {
KDirectory *dir;
- size_t fragSizeBoth; /*** temporary hold for first side of mate pair with both sides aligned**/
- size_t fragSizeOne; /*** temporary hold for first side of mate pair with one side aligned**/
+ size_t fragSize[2];
- fragSizeBoth = (G.cache_size / 8);
- fragSizeOne = (G.cache_size / 2);
+ fragSize[1] = (G.cache_size / 8);
+ fragSize[0] = fragSize[1] * 4;
rc = KLoadProgressbar_Make(&ctx->progress[0], 0); if (rc) return rc;
rc = KLoadProgressbar_Make(&ctx->progress[1], 0); if (rc) return rc;
rc = KLoadProgressbar_Make(&ctx->progress[2], 0); if (rc) return rc;
rc = KLoadProgressbar_Make(&ctx->progress[3], 0); if (rc) return rc;
-
+
KLoadProgressbar_Append(ctx->progress[0], 100 * numfiles);
-
- rc = KDirectoryNativeDir(&dir);
+
+ rc = TmpfsDirectory(&dir);
if (rc == 0)
rc = OpenMMapFile(ctx, dir);
if (rc == 0)
- rc = OpenMBankFile(ctx, dir, 0, fragSizeBoth);
- if (rc == 0)
- rc = OpenMBankFile(ctx, dir, 1, fragSizeOne);
+ rc = MemBankMake(&ctx->frags, dir, G.pid, fragSize);
KDirectoryRelease(dir);
}
return rc;
@@ -609,10 +644,8 @@ static rc_t SetupContext(context_t *ctx, unsigned numfiles)
static void ContextReleaseMemBank(context_t *ctx)
{
- KMemBankRelease(ctx->fragsOne);
- ctx->fragsOne = NULL;
- KMemBankRelease(ctx->fragsBoth);
- ctx->fragsBoth = NULL;
+ MemBankRelease(ctx->frags);
+ ctx->frags = NULL;
}
static void ContextRelease(context_t *ctx)
@@ -625,12 +658,12 @@ static void ContextRelease(context_t *ctx)
}
static
-void COPY_QUAL(uint8_t D[], uint8_t const S[], unsigned const L, bool const R)
+void COPY_QUAL(uint8_t D[], uint8_t const S[], unsigned const L, bool const R)
{
if (R) {
unsigned i;
unsigned j;
-
+
for (i = 0, j = L - 1; i != L; ++i, --j)
D[i] = S[j];
}
@@ -642,43 +675,43 @@ static
void COPY_READ(INSDC_dna_text D[], INSDC_dna_text const S[], unsigned const L, bool const R)
{
static INSDC_dna_text const compl[] = {
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , '.', 0 ,
- '0', '1', '2', '3', 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 'T', 'V', 'G', 'H', 0 , 0 , 'C',
- 'D', 0 , 0 , 'M', 0 , 'K', 'N', 0 ,
- 0 , 0 , 'Y', 'S', 'A', 'A', 'B', 'W',
- 0 , 'R', 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 'T', 'V', 'G', 'H', 0 , 0 , 'C',
- 'D', 0 , 0 , 'M', 0 , 'K', 'N', 0 ,
- 0 , 0 , 'Y', 'S', 'A', 'A', 'B', 'W',
- 0 , 'R', 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
- 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , '.', 0 ,
+ '0', '1', '2', '3', 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 'T', 'V', 'G', 'H', 0 , 0 , 'C',
+ 'D', 0 , 0 , 'M', 0 , 'K', 'N', 0 ,
+ 0 , 0 , 'Y', 'S', 'A', 'A', 'B', 'W',
+ 0 , 'R', 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 'T', 'V', 'G', 'H', 0 , 0 , 'C',
+ 'D', 0 , 0 , 'M', 0 , 'K', 'N', 0 ,
+ 0 , 0 , 'Y', 'S', 'A', 'A', 'B', 'W',
+ 0 , 'R', 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0
};
if (R) {
unsigned i;
unsigned j;
-
+
for (i = 0, j = L - 1; i != L; ++i, --j)
D[i] = compl[((uint8_t const *)S)[j]];
}
@@ -686,26 +719,26 @@ void COPY_READ(INSDC_dna_text D[], INSDC_dna_text const S[], unsigned const L, b
memcpy(D, S, L);
}
-static rc_t OpenBAM(const BAMFile **bam, VDatabase *db, const char bamFile[])
+static rc_t OpenBAM(const BAM_File **bam, VDatabase *db, const char bamFile[])
{
- rc_t rc = BAMFileMakeWithHeader(bam, G.headerText, "%s", bamFile);
+ rc_t rc = BAM_FileMakeWithHeader(bam, G.headerText, "%s", bamFile);
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "Failed to open '$(file)'", "file=%s", bamFile));
}
else if (db) {
KMetadata *dbmeta;
-
+
rc = VDatabaseOpenMetadataUpdate(db, &dbmeta);
if (rc == 0) {
KMDataNode *node;
-
+
rc = KMetadataOpenNodeUpdate(dbmeta, &node, "BAM_HEADER");
KMetadataRelease(dbmeta);
if (rc == 0) {
char const *header;
size_t size;
-
- rc = BAMFileGetHeaderText(*bam, &header, &size);
+
+ rc = BAM_FileGetHeaderText(*bam, &header, &size);
if (rc == 0) {
rc = KMDataNodeWrite(node, header, size);
}
@@ -717,20 +750,20 @@ static rc_t OpenBAM(const BAMFile **bam, VDatabase *db, const char bamFile[])
return rc;
}
-static rc_t VerifyReferences(BAMFile const *bam, Reference const *ref)
+static rc_t VerifyReferences(BAM_File const *bam, Reference const *ref)
{
rc_t rc = 0;
uint32_t n;
unsigned i;
-
- BAMFileGetRefSeqCount(bam, &n);
+
+ BAM_FileGetRefSeqCount(bam, &n);
for (i = 0; i != n; ++i) {
BAMRefSeq const *refSeq;
-
- BAMFileGetRefSeq(bam, i, &refSeq);
+
+ BAM_FileGetRefSeq(bam, i, &refSeq);
if (G.refFilter && strcmp(refSeq->name, G.refFilter) != 0)
continue;
-
+
rc = ReferenceVerify(ref, refSeq->name, refSeq->length, refSeq->checksum);
if (rc) {
if (GetRCObject(rc) == rcChecksum && GetRCState(rc) == rcUnequal) {
@@ -759,34 +792,62 @@ static rc_t VerifyReferences(BAMFile const *bam, Reference const *ref)
return 0;
}
-static uint8_t GetMapQ(BAMAlignment const *rec)
+static uint8_t GetMapQ(BAM_Alignment const *rec)
{
uint8_t mapQ;
-
- BAMAlignmentGetMapQuality(rec, &mapQ);
+
+ BAM_AlignmentGetMapQuality(rec, &mapQ);
return mapQ;
}
-static void EditAlignedQualities(uint8_t qual[], bool const hasMismatch[], unsigned readlen)
+static bool EditAlignedQualities(uint8_t qual[], bool const hasMismatch[], unsigned readlen)
{
unsigned i;
-
+ bool changed = false;
+
for (i = 0; i < readlen; ++i) {
- uint8_t const q = hasMismatch[i] ? G.alignedQualValue : qual[i];
+ uint8_t const q_0 = qual[i];
+ uint8_t const q_1= hasMismatch[i] ? G.alignedQualValue : q_0;
- qual[i] = q;
+ if (q_0 != q_1) {
+ changed = true;
+ break;
+ }
+ }
+ if (!changed)
+ return false;
+ for (i = 0; i < readlen; ++i) {
+ uint8_t const q_0 = qual[i];
+ uint8_t const q_1= hasMismatch[i] ? G.alignedQualValue : q_0;
+
+ qual[i] = q_1;
}
+ return true;
}
-static void EditUnalignedQualities(uint8_t qual[], bool const hasMismatch[], unsigned readlen)
+static bool EditUnalignedQualities(uint8_t qual[], bool const hasMismatch[], unsigned readlen)
{
unsigned i;
-
+ bool changed = false;
+
for (i = 0; i < readlen; ++i) {
- uint8_t const q = (qual[i] & 0x7F) | (hasMismatch[i] ? 0x80 : 0);
+ uint8_t const q_0 = qual[i];
+ uint8_t const q_1 = (q_0 & 0x7F) | (hasMismatch[i] ? 0x80 : 0);
- qual[i] = q;
+ if (q_0 != q_1) {
+ changed = true;
+ break;
+ }
+ }
+ if (!changed)
+ return false;
+ for (i = 0; i < readlen; ++i) {
+ uint8_t const q_0 = qual[i];
+ uint8_t const q_1 = (q_0 & 0x7F) | (hasMismatch[i] ? 0x80 : 0);
+
+ qual[i] = q_1;
}
+ return true;
}
static bool platform_cmp(char const platform[], char const test[])
@@ -796,7 +857,7 @@ static bool platform_cmp(char const platform[], char const test[])
for (i = 0; ; ++i) {
int ch1 = test[i];
int ch2 = toupper(platform[i]);
-
+
if (ch1 != ch2)
break;
if (ch1 == 0)
@@ -806,18 +867,18 @@ static bool platform_cmp(char const platform[], char const test[])
}
static
-INSDC_SRA_platform_id GetINSDCPlatform(BAMFile const *bam, char const name[]) {
+INSDC_SRA_platform_id GetINSDCPlatform(BAM_File const *bam, char const name[]) {
if (name) {
BAMReadGroup const *rg;
- BAMFileGetReadGroupByName(bam, name, &rg);
+ BAM_FileGetReadGroupByName(bam, name, &rg);
if (rg && rg->platform) {
switch (toupper(rg->platform[0])) {
case 'C':
if (platform_cmp(rg->platform, "COMPLETE GENOMICS"))
return SRA_PLATFORM_COMPLETE_GENOMICS;
if (platform_cmp(rg->platform, "CAPILLARY"))
- return SRA_PLATFORM_SANGER;
+ return SRA_PLATFORM_CAPILLARY;
break;
case 'H':
if (platform_cmp(rg->platform, "HELICOS"))
@@ -833,6 +894,14 @@ INSDC_SRA_platform_id GetINSDCPlatform(BAMFile const *bam, char const name[]) {
if (platform_cmp(rg->platform, "LS454"))
return SRA_PLATFORM_454;
break;
+ case 'N':
+ if (platform_cmp(name, "NANOPORE"))
+ return SRA_PLATFORM_OXFORD_NANOPORE;
+ break;
+ case 'O':
+ if (platform_cmp(name, "OXFORD_NANOPORE"))
+ return SRA_PLATFORM_OXFORD_NANOPORE;
+ break;
case 'P':
if (platform_cmp(rg->platform, "PACBIO"))
return SRA_PLATFORM_PACBIO_SMRT;
@@ -840,6 +909,8 @@ INSDC_SRA_platform_id GetINSDCPlatform(BAMFile const *bam, char const name[]) {
case 'S':
if (platform_cmp(rg->platform, "SOLID"))
return SRA_PLATFORM_ABSOLID;
+ if (platform_cmp(name, "SANGER"))
+ return SRA_PLATFORM_CAPILLARY;
break;
default:
break;
@@ -854,7 +925,7 @@ rc_t CheckLimitAndLogError(void)
{
++G.errCount;
if (G.maxErrCount > 0 && G.errCount > G.maxErrCount) {
- (void)PLOGERR(klogErr, (klogErr, RC(rcAlign, rcFile, rcReading, rcError, rcExcessive), "Number of errors $(cnt) exceeds limit of $(max): Exiting", "cnt=%u,max=%u", G.errCount, G.maxErrCount));
+ (void)PLOGERR(klogErr, (klogErr, SILENT_RC(rcAlign, rcFile, rcReading, rcError, rcExcessive), "Number of errors $(cnt) exceeds limit of $(max): Exiting", "cnt=%u,max=%u", G.errCount, G.maxErrCount));
return RC(rcAlign, rcFile, rcReading, rcError, rcExcessive);
}
return 0;
@@ -867,7 +938,7 @@ void RecordNoMatch(char const readName[], char const refName[], uint32_t const r
static uint64_t lpos = 0;
char logbuf[256];
size_t len;
-
+
if (string_printf(logbuf, sizeof(logbuf), &len, "%s\t%s\t%u\n", readName, refName, refPos) == 0) {
KFileWrite(G.noMatchLog, lpos, logbuf, len, NULL);
lpos += len;
@@ -880,7 +951,7 @@ rc_t LogNoMatch(char const readName[], char const refName[], unsigned rpos, unsi
{
rc_t const rc = CheckLimitAndLogError();
static unsigned count = 0;
-
+
++count;
if (rc) {
(void)PLOGMSG(klogInfo, (klogInfo, "This is the last warning; this class of warning occurred $(occurred) times",
@@ -899,28 +970,210 @@ rc_t LogDupConflict(char const readName[])
{
rc_t const rc = CheckLimitAndLogError();
static unsigned count = 0;
-
+
++count;
if (rc) {
(void)PLOGMSG(klogInfo, (klogInfo, "This is the last warning; this class of warning occurred $(occurred) times",
"occurred=%u", count));
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
+ (void)PLOGERR(klogWarn, (klogWarn, SILENT_RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
"Spot '$(name)' is both a duplicate and NOT a duplicate!",
"name=%s", readName));
}
else if (G.maxWarnCount_DupConflict == 0 || count < G.maxWarnCount_DupConflict)
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
+ (void)PLOGERR(klogWarn, (klogWarn, SILENT_RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
"Spot '$(name)' is both a duplicate and NOT a duplicate!",
"name=%s", readName));
return rc;
}
+static char const *const CHANGED[] = {
+ "FLAG changed",
+ "QUAL changed",
+ "SEQ changed",
+ "record made unaligned",
+ "record made unfragmented",
+ "mate alignment lost",
+ "record discarded",
+ "reference name changed"
+};
+
+static char const *const REASONS[] = {
+/* FLAG changed */
+ "0x400 and 0x200 both set", /* 0 */
+ "conflicting PCR Dup flags", /* 1 */
+ "primary alignment already exists", /* 2 */
+ "was already recorded as unaligned", /* 3 */
+/* QUAL changed */
+ "original quality used", /* 4 */
+ "unaligned colorspace", /* 5 */
+ "aligned bases", /* 6 */
+ "unaligned bases", /* 7 */
+ "reversed", /* 8 */
+/* unaligned */
+ "low MAPQ", /* 9 */
+ "low match count", /* 10 */
+ "missing alignment info", /* 11 */
+ "missing reference position", /* 12 */
+ "invalid alignment info", /* 13 */
+ "invalid reference position", /* 14 */
+ "invalid reference", /* 15 */
+ "unaligned reference", /* 16 */
+ "unknown reference", /* 17 */
+ "hard-clipped colorspace", /* 18 */
+/* unfragmented */
+ "missing fragment info", /* 19 */
+ "too many fragments", /* 20 */
+/* mate info lost */
+ "invalid mate reference", /* 21 */
+ "missing mate alignment info", /* 22 */
+ "unknown mate reference", /* 23 */
+/* discarded */
+ "conflicting PCR duplicate", /* 24 */
+ "conflicting fragment info", /* 25 */
+ "reference is skipped" /* 26 */
+/* reference name changed */
+ "reference was named more than once" /* 27 */
+};
+
+static struct {
+ unsigned what, why;
+} const CHANGES[] = {
+ {0, 0},
+ {0, 1},
+ {0, 2},
+ {0, 3},
+ {1, 4},
+ {1, 5},
+ {1, 6},
+ {1, 7},
+ {1, 8},
+ {2, 8},
+ {3, 9},
+ {3, 10},
+ {3, 11},
+ {3, 12},
+ {3, 13},
+ {3, 14},
+ {3, 15},
+ {3, 16},
+ {3, 17},
+ {3, 18},
+ {4, 19},
+ {4, 20},
+ {5, 21},
+ {5, 22},
+ {5, 23},
+ {6, 24},
+ {6, 25},
+ {6, 26},
+ {6, 17},
+ {7, 27},
+};
+
+#define NUMBER_OF_CHANGES ((unsigned)(sizeof(CHANGES)/sizeof(CHANGES[0])))
+static unsigned change_counter[NUMBER_OF_CHANGES];
+
+static void LOG_CHANGE(unsigned const change)
+{
+ ++change_counter[change];
+}
+
+static void PrintChangeReport(void)
+{
+ unsigned i;
+
+ for (i = 0; i != NUMBER_OF_CHANGES; ++i) {
+ if (change_counter[i] > 0) {
+ char const *const what = CHANGED[CHANGES[i].what];
+ char const *const why = REASONS[CHANGES[i].why];
+
+ PLOGMSG(klogInfo, (klogInfo, "$(what) $(times) times because $(reason)", "what=%s,reason=%s,times=%u", what, why, change_counter[i]));
+ }
+ }
+}
+
+static rc_t RecordChange(KMDataNode *const node,
+ char const node_name[],
+ unsigned const node_number,
+ char const what[],
+ char const why[],
+ unsigned const count)
+{
+ KMDataNode *sub = NULL;
+ rc_t const rc_sub = KMDataNodeOpenNodeUpdate(node, &sub, "%s_%u", node_name, node_number);
+
+ if (rc_sub) return rc_sub;
+ {
+ uint32_t const count_temp = count;
+ rc_t const rc_attr1 = KMDataNodeWriteAttr(sub, "change", what);
+ rc_t const rc_attr2 = KMDataNodeWriteAttr(sub, "reason", why);
+ rc_t const rc_value = KMDataNodeWriteB32(sub, &count_temp);
+
+ KMDataNodeRelease(sub);
+ if (rc_attr1) return rc_attr1;
+ if (rc_attr2) return rc_attr2;
+ if (rc_value) return rc_value;
+
+ return 0;
+ }
+}
+
+static rc_t RecordChanges(KMDataNode *const node, char const name[])
+{
+ if (node) {
+ unsigned i;
+ unsigned j = 0;
+
+ for (i = 0; i != NUMBER_OF_CHANGES; ++i) {
+ if (change_counter[i] > 0) {
+ char const *const what = CHANGED[CHANGES[i].what];
+ char const *const why = REASONS[CHANGES[i].why];
+ rc_t const rc = RecordChange(node, name, ++j, what, why, change_counter[i]);
+
+ if (rc) return rc;
+ }
+ }
+ }
+ return 0;
+}
+
+#define FLAG_CHANGED_400_AND_200 do { LOG_CHANGE( 0); } while(0)
+#define FLAG_CHANGED_PCR_DUP do { LOG_CHANGE( 1); } while(0)
+#define FLAG_CHANGED_PRIMARY_DUP do { LOG_CHANGE( 2); } while(0)
+#define FLAG_CHANGED_WAS_UNALIGNED do { LOG_CHANGE( 3); } while(0)
+#define QUAL_CHANGED_OQ do { LOG_CHANGE( 4); } while(0)
+#define QUAL_CHANGED_UNALIGNED_CS do { LOG_CHANGE( 5); } while(0)
+#define QUAL_CHANGED_ALIGNED_EDIT do { LOG_CHANGE( 6); } while(0)
+#define QUAL_CHANGED_UNALIGN_EDIT do { LOG_CHANGE( 7); } while(0)
+#define QUAL_CHANGED_REVERSED do { LOG_CHANGE( 8); } while(0)
+#define SEQ__CHANGED_REV_COMP do { LOG_CHANGE( 9); } while(0)
+#define UNALIGNED_LOW_MAPQ do { LOG_CHANGE(10); } while(0)
+#define UNALIGNED_LOW_MATCH_COUNT do { LOG_CHANGE(11); } while(0)
+#define UNALIGNED_MISSING_INFO do { LOG_CHANGE(12); } while(0)
+#define UNALIGNED_MISSING_REF_POS do { LOG_CHANGE(13); } while(0)
+#define UNALIGNED_INVALID_INFO do { LOG_CHANGE(14); } while(0)
+#define UNALIGNED_INVALID_REF_POS do { LOG_CHANGE(15); } while(0)
+#define UNALIGNED_INVALID_REF do { LOG_CHANGE(16); } while(0)
+#define UNALIGNED_UNALIGNED_REF do { LOG_CHANGE(17); } while(0)
+#define UNALIGNED_UNKNOWN_REF do { LOG_CHANGE(18); } while(0)
+#define UNALIGNED_HARD_CLIPPED_CS do { LOG_CHANGE(19); } while(0)
+#define UNFRAGMENT_MISSING_INFO do { LOG_CHANGE(20); } while(0)
+#define UNFRAGMENT_TOO_MANY do { LOG_CHANGE(21); } while(0)
+#define MATE_INFO_LOST_INVALID do { LOG_CHANGE(22); } while(0)
+#define MATE_INFO_LOST_MISSING do { LOG_CHANGE(23); } while(0)
+#define MATE_INFO_LOST_UNKNOWN_REF do { LOG_CHANGE(24); } while(0)
+#define DISCARD_PCR_DUP do { LOG_CHANGE(25); } while(0)
+#define DISCARD_BAD_FRAGMENT_INFO do { LOG_CHANGE(26); } while(0)
+#define DISCARD_SKIP_REFERENCE do { LOG_CHANGE(27); } while(0)
+#define DISCARD_UNKNOWN_REFERENCE do { LOG_CHANGE(28); } while(0)
+#define RENAMED_REFERENCE do { LOG_CHANGE(29); } while(0)
+
static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
Reference *ref, Sequence *seq, Alignment *align,
bool *had_alignments, bool *had_sequences)
{
- const BAMFile *bam;
- const BAMAlignment *rec;
+ const BAM_File *bam;
+ const BAM_Alignment *rec;
KDataBuffer buf;
KDataBuffer fragBuf;
KDataBuffer cigBuf;
@@ -932,11 +1185,12 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
SequenceRecord srec;
char spotGroup[512];
size_t namelen;
- unsigned progress = 0;
+ float progress = 0.0;
unsigned warned = 0;
long fcountBoth=0;
long fcountOne=0;
int skipRefSeqID = -1;
+ int unmapRefSeqId = -1;
uint64_t recordsRead = 0;
uint64_t recordsProcessed = 0;
uint64_t filterFlagConflictRecords=0; /*** counts number of conflicts between flags 0x400 and 0x200 ***/
@@ -947,15 +1201,15 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
char alignGroup[32];
size_t alignGroupLen;
AlignmentRecord data;
-
+
memset(&data, 0, sizeof(data));
-
+
rc = OpenBAM(&bam, db, bamFile);
if (rc) return rc;
if (!G.noVerifyReferences && ref != NULL) {
rc = VerifyReferences(bam, ref);
if (G.onlyVerifyReferences) {
- BAMFileRelease(bam);
+ BAM_FileRelease(bam);
return rc;
}
}
@@ -963,16 +1217,16 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
uint32_t rgcount;
unsigned rgi;
- BAMFileGetReadGroupCount(bam, &rgcount);
+ BAM_FileGetReadGroupCount(bam, &rgcount);
if (rgcount > (sizeof(ctx->key2id)/sizeof(ctx->key2id[0]) - 1))
ctx->key2id_max = 1;
else
ctx->key2id_max = sizeof(ctx->key2id)/sizeof(ctx->key2id[0]);
-
+
for (rgi = 0; rgi != rgcount; ++rgi) {
BAMReadGroup const *rg;
-
- BAMFileGetReadGroup(bam, rgi, &rg);
+
+ BAM_FileGetReadGroup(bam, rgi, &rg);
if (rg && rg->platform && platform_cmp(rg->platform, "CAPILLARY")) {
G.hasTI = true;
break;
@@ -980,19 +1234,19 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
}
}
memset(&srec, 0, sizeof(srec));
-
+
rc = KDataBufferMake(&cigBuf, 32, 0);
if (rc)
return rc;
-
- rc = KDataBufferMake(&fragBuf, 8, FRAG_CHUNK_SIZE);
+
+ rc = KDataBufferMake(&fragBuf, 8, 1024);
if (rc)
return rc;
-
+
rc = KDataBufferMake(&buf, 16, 0);
if (rc)
return rc;
-
+
if (rc == 0) {
(void)PLOGMSG(klogInfo, (klogInfo, "Loading '$(file)'", "file=%s", bamFile));
}
@@ -1013,36 +1267,45 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
bool originally_aligned;
bool isPrimary;
uint32_t opCount;
- bool hasCG;
+ bool hasCG = false;
uint64_t ti = 0;
uint32_t csSeqLen = 0;
- rc = BAMFileRead2(bam, &rec);
+ rc = BAM_FileRead2(bam, &rec);
if (rc) {
- if (GetRCModule(rc) == rcAlign && GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound)
+ if (GetRCModule(rc) == rcAlign && GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound) {
+ (void)PLOGMSG(klogInfo, (klogInfo, "EOF '$(file)'; read $(read); processed $(proc)", "file=%s,read=%lu,proc=%lu", bamFile, (unsigned long)recordsRead, (unsigned long)recordsProcessed));
rc = 0;
+ }
else if (GetRCModule(rc) == rcAlign && GetRCObject(rc) == rcRow && GetRCState(rc) == rcEmpty) {
++recordsRead;
(void)PLOGERR(klogWarn, (klogWarn, rc, "File '$(file)'; record $(recno)", "file=%s,recno=%lu", bamFile, recordsRead));
rc = CheckLimitAndLogError();
goto LOOP_END;
}
+ else {
+ (void)PLOGERR(klogInfo, (klogInfo, rc, "Error '$(file)'; read $(read); processed $(proc)", "file=%s,read=%lu,proc=%lu", bamFile, (unsigned long)recordsRead, (unsigned long)recordsProcessed));
+ }
break;
}
++recordsRead;
- if ((unsigned)(BAMFileGetProportionalPosition(bam) * 100.0) > progress) {
- unsigned new_value = BAMFileGetProportionalPosition(bam) * 100.0;
- KLoadProgressbar_Process(ctx->progress[0], new_value - progress, false);
- progress = new_value;
+
+ {
+ float const new_value = BAM_FileGetProportionalPosition(bam) * 100.0;
+ float const delta = new_value - progress;
+ if (delta > 1.0) {
+ KLoadProgressbar_Process(ctx->progress[0], delta, false);
+ progress = new_value;
+ }
}
/**************************************************************/
if (!G.noColorSpace) {
- if (BAMAlignmentHasColorSpace(rec)) {/*BAM*/
+ if (BAM_AlignmentHasColorSpace(rec)) {/*BAM*/
if (isNotColorSpace) {
MIXED_BASE_AND_COLOR:
- rc = RC(rcApp, rcFile, rcReading, rcData, rcInconsistent);
+ rc = RC(rcApp, rcFile, rcReading, rcData, rcInconsistent);
(void)PLOGERR(klogErr, (klogErr, rc, "File '$(file)' contains base space and color space", "file=%s", bamFile));
goto LOOP_END;
}
@@ -1053,40 +1316,45 @@ MIXED_BASE_AND_COLOR:
else
isNotColorSpace = true;
}
- hasCG = BAMAlignmentHasCGData(rec);/*BAM*/
- if (hasCG) {
- BAMAlignmentGetCigarCount(rec, &opCount);/*BAM*/
+ rc = BAM_AlignmentCGReadLength(rec, &readlen);/*BAM*/
+ if (rc != 0 && GetRCState(rc) != rcNotFound) {
+ (void)LOGERR(klogErr, rc, "Invalid CG data");
+ goto LOOP_END;
+ }
+ if (rc == 0) {
+ hasCG = true;
+ BAM_AlignmentGetCigarCount(rec, &opCount);/*BAM*/
rc = KDataBufferResize(&cigBuf, opCount * 2 + 5);
if (rc) {
(void)LOGERR(klogErr, rc, "Failed to resize CIGAR buffer");
goto LOOP_END;
}
-
- rc = AlignmentRecordInit(&data, readlen = 35);
+
+ rc = AlignmentRecordInit(&data, readlen);
if (rc == 0)
rc = KDataBufferResize(&buf, readlen);
if (rc) {
(void)LOGERR(klogErr, rc, "Failed to resize record buffer");
goto LOOP_END;
}
-
+
seqDNA = buf.base;
qual = (uint8_t *)&seqDNA[readlen];
}
else {
uint32_t const *tmp;
-
- BAMAlignmentGetRawCigar(rec, &tmp, &opCount);/*BAM*/
+
+ BAM_AlignmentGetRawCigar(rec, &tmp, &opCount);/*BAM*/
rc = KDataBufferResize(&cigBuf, opCount);
if (rc) {
(void)LOGERR(klogErr, rc, "Failed to resize CIGAR buffer");
goto LOOP_END;
}
memcpy(cigBuf.base, tmp, opCount * sizeof(uint32_t));
-
- BAMAlignmentGetReadLength(rec, &readlen);/*BAM*/
+
+ BAM_AlignmentGetReadLength(rec, &readlen);/*BAM*/
if (isColorSpace) {
- BAMAlignmentGetCSSeqLen(rec, &csSeqLen);
+ BAM_AlignmentGetCSSeqLen(rec, &csSeqLen);
if (readlen > csSeqLen) {
rc = RC(rcAlign, rcRow, rcReading, rcData, rcInconsistent);
(void)LOGERR(klogErr, rc, "Sequence length and CS Sequence length are inconsistent");
@@ -1097,7 +1365,7 @@ MIXED_BASE_AND_COLOR:
}
else if (readlen == 0) {
}
-
+
rc = AlignmentRecordInit(&data, readlen | csSeqLen);
if (rc == 0)
rc = KDataBufferResize(&buf, readlen | csSeqLen);
@@ -1105,24 +1373,24 @@ MIXED_BASE_AND_COLOR:
(void)LOGERR(klogErr, rc, "Failed to resize record buffer");
goto LOOP_END;
}
-
+
seqDNA = buf.base;
qual = (uint8_t *)&seqDNA[readlen | csSeqLen];
}
- BAMAlignmentGetReadName2(rec, &name, &namelen);/*BAM*/
- BAMAlignmentGetSequence(rec, seqDNA);/*BAM*/
+ BAM_AlignmentGetReadName2(rec, &name, &namelen);/*BAM*/
+ BAM_AlignmentGetSequence(rec, seqDNA);/*BAM*/
if (G.useQUAL) {
uint8_t const *squal;
-
- BAMAlignmentGetQuality(rec, &squal);/*BAM*/
+
+ BAM_AlignmentGetQuality(rec, &squal);/*BAM*/
memcpy(qual, squal, readlen);
}
else {
uint8_t const *squal;
uint8_t qoffset = 0;
unsigned i;
-
- rc = BAMAlignmentGetQuality2(rec, &squal, &qoffset);/*BAM*/
+
+ rc = BAM_AlignmentGetQuality2(rec, &squal, &qoffset);/*BAM*/
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "Spot '$(name)': length of original quality does not match sequence", "name=%s", name));
goto LOOP_END;
@@ -1130,112 +1398,155 @@ MIXED_BASE_AND_COLOR:
if (qoffset) {
for (i = 0; i != readlen; ++i)
qual[i] = squal[i] - qoffset;
+ QUAL_CHANGED_OQ;
}
else
memcpy(qual, squal, readlen);
}
if (hasCG) {
- rc = BAMAlignmentGetCGSeqQual(rec, seqDNA, qual);/*BAM*/
- if (rc == 0)
- rc = BAMAlignmentGetCGCigar(rec, cigBuf.base, cigBuf.elem_count, &opCount);/*BAM*/
+ rc = BAM_AlignmentGetCGSeqQual(rec, seqDNA, qual);
+ if (rc == 0) {
+ rc = BAM_AlignmentGetCGCigar(rec, cigBuf.base, cigBuf.elem_count, &opCount);/*BAM*/
+ }
if (rc) {
(void)LOGERR(klogErr, rc, "Failed to read CG data");
goto LOOP_END;
}
}
if (G.hasTI) {
- rc = BAMAlignmentGetTI(rec, &ti);/*BAM*/
+ rc = BAM_AlignmentGetTI(rec, &ti);/*BAM*/
if (rc)
ti = 0;
rc = 0;
}
data.data.align_group.buffer = alignGroup;
- if (BAMAlignmentGetCGAlignGroup(rec, alignGroup, sizeof(alignGroup), &alignGroupLen) == 0)/*BAM*/
+ if (BAM_AlignmentGetCGAlignGroup(rec, alignGroup, sizeof(alignGroup), &alignGroupLen) == 0)/*BAM*/
data.data.align_group.elements = alignGroupLen;
else
data.data.align_group.elements = 0;
AR_MAPQ(data) = GetMapQ(rec);
- BAMAlignmentGetFlags(rec, &flags);/*BAM*/
- BAMAlignmentGetReadName2(rec, &name, &namelen);/*BAM*/
+ BAM_AlignmentGetFlags(rec, &flags);/*BAM*/
+ BAM_AlignmentGetReadName2(rec, &name, &namelen);/*BAM*/
{{
char const *rgname;
- BAMAlignmentGetReadGroupName(rec, &rgname);/*BAM*/
+ BAM_AlignmentGetReadGroupName(rec, &rgname);/*BAM*/
if (rgname)
strcpy(spotGroup, rgname);
else
spotGroup[0] = '\0';
- }}
+ }}
AR_REF_ORIENT(data) = (flags & BAMFlags_SelfIsReverse) == 0 ? false : true;/*BAM*/
isPrimary = (flags & BAMFlags_IsNotPrimary) == 0 ? true : false;/*BAM*/
if (G.noSecondary && !isPrimary)
goto LOOP_END;
originally_aligned = (flags & BAMFlags_SelfIsUnmapped) == 0;/*BAM*/
- aligned = originally_aligned && (AR_MAPQ(data) >= G.minMapQual);
+ aligned = originally_aligned;
+#if 0
+ if (originally_aligned && AR_MAPQ(data) < G.minMapQual) {
+ aligned = false;
+ UNALIGNED_LOW_MAPQ;
+ }
+#else
+ /* min-mapq now only applies to secondary alignment to match cg-load
+ * see [SRA-2778] in JIRA
+ */
+ if (!isPrimary && AR_MAPQ(data) < G.minMapQual)
+ goto LOOP_END;
+#endif
+ if (aligned && isColorSpace && readlen == 0) {
+ /* detect hard clipped colorspace */
+ /* reads and make unaligned */
+ aligned = false;
+ UNALIGNED_HARD_CLIPPED_CS;
+ }
- if (isColorSpace && readlen == 0) /* detect hard clipped colorspace */
- aligned = false; /* reads and make unaligned */
-
if (aligned && align == NULL) {
rc = RC(rcApp, rcFile, rcReading, rcData, rcInconsistent);
(void)PLOGERR(klogErr, (klogErr, rc, "File '$(file)' contains aligned records", "file=%s", bamFile));
goto LOOP_END;
}
while (aligned) {
- BAMAlignmentGetPosition(rec, &rpos);/*BAM*/
- BAMAlignmentGetRefSeqId(rec, &refSeqId);/*BAM*/
+ BAM_AlignmentGetPosition(rec, &rpos);/*BAM*/
+ BAM_AlignmentGetRefSeqId(rec, &refSeqId);/*BAM*/
if (rpos >= 0 && refSeqId >= 0) {
- if (refSeqId == skipRefSeqID)
+ if (refSeqId == skipRefSeqID) {
+ DISCARD_SKIP_REFERENCE;
goto LOOP_END;
+ }
+ if (refSeqId == unmapRefSeqId) {
+ aligned = false;
+ UNALIGNED_UNALIGNED_REF;
+ break;
+ }
+ unmapRefSeqId = -1;
if (refSeqId == lastRefSeqId)
break;
refSeq = NULL;
- BAMFileGetRefSeqById(bam, refSeqId, &refSeq);/*BAM*/
+ BAM_FileGetRefSeqById(bam, refSeqId, &refSeq);/*BAM*/
if (refSeq == NULL) {
- rc = RC(rcApp, rcFile, rcReading, rcData, rcInconsistent);
+ rc = SILENT_RC(rcApp, rcFile, rcReading, rcData, rcInconsistent);
(void)PLOGERR(klogWarn, (klogWarn, rc, "File '$(file)': Spot '$(name)' refers to an unknown Reference number $(refSeqId)", "file=%s,refSeqId=%i,name=%s", bamFile, (int)refSeqId, name));
rc = CheckLimitAndLogError();
+ DISCARD_UNKNOWN_REFERENCE;
goto LOOP_END;
}
else {
+ bool shouldUnmap = false;
+ bool wasRenamed = false;
+
if (G.refFilter && strcmp(G.refFilter, refSeq->name) != 0) {
(void)PLOGMSG(klogInfo, (klogInfo, "Skipping Reference '$(name)'", "name=%s", refSeq->name));
skipRefSeqID = refSeqId;
+ DISCARD_SKIP_REFERENCE;
goto LOOP_END;
}
-
- rc = ReferenceSetFile(ref, refSeq->name, refSeq->length, refSeq->checksum);
+
+ rc = ReferenceSetFile(ref, refSeq->name, refSeq->length, refSeq->checksum, &shouldUnmap, &wasRenamed);
if (rc == 0) {
lastRefSeqId = refSeqId;
+ if (shouldUnmap) {
+ aligned = false;
+ unmapRefSeqId = refSeqId;
+ UNALIGNED_UNALIGNED_REF;
+ }
+ if (wasRenamed) {
+ RENAMED_REFERENCE;
+ }
break;
}
if (GetRCObject(rc) == rcConstraint && GetRCState(rc) == rcViolated) {
int const level = G.limit2config ? klogWarn : klogErr;
-
+
(void)PLOGMSG(level, (level, "Could not find a Reference to match { name: '$(name)', length: $(rlen) }", "name=%s,rlen=%u", refSeq->name, (unsigned)refSeq->length));
}
else if (!G.limit2config)
(void)PLOGERR(klogErr, (klogErr, rc, "File '$(file)': Spot '$(sname)' refers to an unknown Reference '$(rname)'", "file=%s,rname=%s,sname=%s", bamFile, refSeq->name, name));
- if (G.limit2config)
+ if (G.limit2config) {
rc = 0;
+ UNALIGNED_UNKNOWN_REF;
+ }
goto LOOP_END;
}
}
else if (refSeqId < 0) {
(void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)' was marked aligned, but reference id = $(id) is invalid", "name=%.*s,id=%i", namelen, name, refSeqId));
if ((rc = CheckLimitAndLogError()) != 0) goto LOOP_END;
+ UNALIGNED_INVALID_REF;
}
else {
(void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)' was marked aligned, but reference position = $(pos) is invalid", "name=%.*s,pos=%i", namelen, name, rpos));
if ((rc = CheckLimitAndLogError()) != 0) goto LOOP_END;
+ UNALIGNED_INVALID_REF_POS;
}
aligned = false;
}
- if (!aligned && (G.refFilter != NULL || G.limit2config))
+ if (!aligned && (G.refFilter != NULL || G.limit2config)) {
+ assert("this shouldn't happen");
goto LOOP_END;
-
+ }
rc = GetKeyID(ctx, &keyId, &wasInserted, spotGroup, name, namelen);
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "KBTreeEntry: failed on key '$(key)'", "key=%.*s", namelen, name));
@@ -1246,9 +1557,9 @@ MIXED_BASE_AND_COLOR:
(void)PLOGERR(klogErr, (klogErr, rc, "MMArrayGet: failed on id '$(id)'", "id=%u", keyId));
goto LOOP_END;
}
-
+
AR_KEY(data) = keyId;
-
+
mated = false;
if (flags & BAMFlags_WasPaired) {/*BAM*/
if ((flags & BAMFlags_IsFirst) != 0)/*BAM*/
@@ -1265,18 +1576,20 @@ MIXED_BASE_AND_COLOR:
(void)LOGMSG(klogWarn, "Spots without fragment info have been encountered");
warned |= 1;
}
+ UNFRAGMENT_MISSING_INFO;
break;
case 3:
if ((warned & 2) == 0) {
(void)LOGMSG(klogWarn, "Spots with more than two fragments have been encountered");
warned |= 2;
}
+ UNFRAGMENT_TOO_MANY;
break;
}
}
if (!mated)
AR_READNO(data) = 1;
-
+
if (wasInserted) {
memset(value, 0, sizeof(*value));
value->unmated = !mated;
@@ -1284,45 +1597,60 @@ MIXED_BASE_AND_COLOR:
value->platform = GetINSDCPlatform(bam, spotGroup);
}
else {
- if (!G.acceptBadDups && value->pcr_dup != ((flags & BAMFlags_IsDuplicate) == 0 ? 0 : 1)) {/*BAM*/
+ int const o_pcr_dup = value->pcr_dup;
+ int const n_pcr_dup = (flags & BAMFlags_IsDuplicate) == 0 ? 0 : 1;
+
+ if (!G.acceptBadDups && o_pcr_dup != n_pcr_dup) {
rc = LogDupConflict(name);
- goto LOOP_END; /* TODO: is this correct? */
+ DISCARD_PCR_DUP;
+ goto LOOP_END;
+ }
+ value->pcr_dup = o_pcr_dup & n_pcr_dup;
+ if (o_pcr_dup != (o_pcr_dup & n_pcr_dup)) {
+ FLAG_CHANGED_PCR_DUP;
}
- value->pcr_dup &= (flags & BAMFlags_IsDuplicate) == 0 ? 0 : 1;/*BAM*/
if (mated && value->unmated) {
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
+ (void)PLOGERR(klogWarn, (klogWarn, SILENT_RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
"Spot '$(name)', which was first seen without mate info, now has mate info",
"name=%s", name));
rc = CheckLimitAndLogError();
+ DISCARD_BAD_FRAGMENT_INFO;
goto LOOP_END;
}
else if (!mated && !value->unmated) {
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
+ (void)PLOGERR(klogWarn, (klogWarn, SILENT_RC(rcApp, rcFile, rcReading, rcData, rcInconsistent),
"Spot '$(name)', which was first seen with mate info, now has no mate info",
"name=%s", name));
rc = CheckLimitAndLogError();
+ DISCARD_BAD_FRAGMENT_INFO;
goto LOOP_END;
}
}
-
+
++recordsProcessed;
if (isPrimary) {
switch (AR_READNO(data)) {
case 1:
- if (CTX_VALUE_GET_P_ID(*value, 0) != 0)
+ if (CTX_VALUE_GET_P_ID(*value, 0) != 0) {
isPrimary = false;
+ FLAG_CHANGED_PRIMARY_DUP;
+ }
else if (aligned && value->unaligned_1) {
(void)PLOGMSG(klogWarn, (klogWarn, "Read 1 of spot '$(name)', which was unmapped, is now being mapped at position $(pos) on reference '$(ref)'; this alignment will be considered as secondary", "name=%s,ref=%s,pos=%u", name, refSeq->name, rpos));
isPrimary = false;
+ FLAG_CHANGED_WAS_UNALIGNED;
}
break;
case 2:
- if (CTX_VALUE_GET_P_ID(*value, 1) != 0)
+ if (CTX_VALUE_GET_P_ID(*value, 1) != 0) {
isPrimary = false;
+ FLAG_CHANGED_PRIMARY_DUP;
+ }
else if (aligned && value->unaligned_2) {
(void)PLOGMSG(klogWarn, (klogWarn, "Read 2 of spot '$(name)', which was unmapped, is now being mapped at position $(pos) on reference '$(ref)'; this alignment will be considered as secondary", "name=%s,ref=%s,pos=%u", name, refSeq->name, rpos));
isPrimary = false;
+ FLAG_CHANGED_WAS_UNALIGNED;
}
break;
default:
@@ -1334,15 +1662,15 @@ MIXED_BASE_AND_COLOR:
uint32_t matches = 0;
uint8_t rna_orient = ' ';
- BAMAlignmentGetRNAStrand(rec, &rna_orient);
+ BAM_AlignmentGetRNAStrand(rec, &rna_orient);
rc = ReferenceRead(ref, &data, rpos, cigBuf.base, opCount, seqDNA, readlen,
rna_orient == '+' ? NCBI_align_ro_intron_plus :
rna_orient == '-' ? NCBI_align_ro_intron_minus :
- hasCG ? NCBI_align_ro_complete_genomics :
- NCBI_align_ro_intron_unknown, &matches);
+ hasCG ? NCBI_align_ro_complete_genomics :
+ NCBI_align_ro_intron_unknown, &matches);
if (rc) {
aligned = false;
-
+
if ( (GetRCState(rc) == rcViolated && GetRCObject(rc) == rcConstraint)
|| (GetRCState(rc) == rcExcessive && GetRCObject(rc) == rcRange))
{
@@ -1350,16 +1678,28 @@ MIXED_BASE_AND_COLOR:
}
if (GetRCState(rc) == rcViolated && GetRCObject(rc) == rcConstraint) {
rc = LogNoMatch(name, refSeq->name, (unsigned)rpos, (unsigned)matches);
+ UNALIGNED_LOW_MATCH_COUNT;
}
- else if (GetRCObject(rc) == rcData && GetRCState(rc) == rcInvalid) {
+#define DATA_INVALID_ERRORS_ARE_DEADLY 0
+#if DATA_INVALID_ERRORS_ARE_DEADLY
+ else if (((int)GetRCObject(rc)) == ((int)rcData) && GetRCState(rc) == rcInvalid) {
+ UNALIGNED_INVALID_INFO;
(void)PLOGERR(klogWarn, (klogWarn, rc, "Spot '$(name)': bad alignment to reference '$(ref)' at $(pos)", "name=%s,ref=%s,pos=%u", name, refSeq->name, rpos));
CheckLimitAndLogError();
}
- else if (GetRCObject(rc) == rcData) {
+#endif
+ else if (((int)GetRCObject(rc)) == ((int)rcData) && GetRCState(rc) == rcNotAvailable) {
+ (void)PLOGERR(klogWarn, (klogWarn, rc, "Spot '$(name)': sequence was hard clipped", "name=%s", name));
+ CheckLimitAndLogError();
+ }
+ else if (((int)GetRCObject(rc)) == ((int)rcData)) {
+ UNALIGNED_INVALID_INFO;
(void)PLOGERR(klogWarn, (klogWarn, rc, "Spot '$(name)': bad alignment to reference '$(ref)' at $(pos)", "name=%s,ref=%s,pos=%u", name, refSeq->name, rpos));
+ /* Data errors may get reset; alignment will be unmapped at any rate */
rc = CheckLimitAndLogError();
}
else {
+ UNALIGNED_INVALID_REF_POS;
(void)PLOGERR(klogWarn, (klogWarn, rc, "Spot '$(name)': error reading reference '$(ref)' at $(pos)", "name=%s,ref=%s,pos=%u", name, refSeq->name, rpos));
rc = CheckLimitAndLogError();
}
@@ -1368,20 +1708,21 @@ MIXED_BASE_AND_COLOR:
}
if (isColorSpace) {
/* must be after ReferenceRead */
- BAMAlignmentGetCSKey(rec, &cskey);/*BAM*/
- BAMAlignmentGetCSSequence(rec, seqDNA, csSeqLen);/*BAM*/
+ BAM_AlignmentGetCSKey(rec, &cskey);/*BAM*/
+ BAM_AlignmentGetCSSequence(rec, seqDNA, csSeqLen);/*BAM*/
if (!aligned && !G.useQUAL) {
uint8_t const *squal;
uint8_t qoffset = 0;
-
- rc = BAMAlignmentGetCSQuality(rec, &squal, &qoffset);/*BAM*/
+
+ rc = BAM_AlignmentGetCSQuality(rec, &squal, &qoffset);/*BAM*/
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "Spot '$(name)': length of colorspace quality does not match sequence", "name=%s", name));
goto LOOP_END;
}
if (qoffset) {
unsigned i;
-
+
+ QUAL_CHANGED_UNALIGNED_CS;
for (i = 0; i < csSeqLen; ++i)
qual[i] = squal[i] - qoffset;
}
@@ -1390,10 +1731,14 @@ MIXED_BASE_AND_COLOR:
readlen = csSeqLen;
}
}
-
+
if (aligned) {
- if (G.editAlignedQual ) EditAlignedQualities (qual, AR_HAS_MISMATCH(data), readlen);
- if (G.keepMismatchQual) EditUnalignedQualities(qual, AR_HAS_MISMATCH(data), readlen);
+ if (G.editAlignedQual && EditAlignedQualities (qual, AR_HAS_MISMATCH(data), readlen)) {
+ QUAL_CHANGED_ALIGNED_EDIT;
+ }
+ if (G.keepMismatchQual && EditUnalignedQualities(qual, AR_HAS_MISMATCH(data), readlen)) {
+ QUAL_CHANGED_UNALIGN_EDIT;
+ }
}
else if (isPrimary) {
switch (AR_READNO(data)) {
@@ -1426,182 +1771,203 @@ MIXED_BASE_AND_COLOR:
}
}
if (mated) {
- if (isPrimary || !originally_aligned) {
- if (CTX_VALUE_GET_S_ID(*value) != 0) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)' has already been assigned a spot id", "name=%.*s", namelen, name));
+ int64_t const spotId = CTX_VALUE_GET_S_ID(*value);
+ uint32_t const fragmentId = value->fragmentId;
+ bool const spotHasBeenWritten = (spotId != 0);
+ bool const spotHasFragmentInfo = (fragmentId != 0);
+ bool const spotIsFirstSeen = (spotHasBeenWritten || spotHasFragmentInfo) ? false : true;
+
+ if (spotHasBeenWritten) {
+ /* do nothing */
+ }
+ else if (spotIsFirstSeen) {
+ /* start spot assembly */
+ unsigned sz;
+ FragmentInfo fi;
+ int32_t mate_refSeqId = -1;
+ int64_t pnext = 0;
+
+ memset(&fi, 0, sizeof(fi));
+ fi.aligned = aligned;
+ fi.ti = ti;
+ fi.orientation = AR_REF_ORIENT(data);
+ fi.otherReadNo = AR_READNO(data);
+ fi.sglen = strlen(spotGroup);
+ fi.readlen = readlen;
+ fi.cskey = cskey;
+ fi.is_bad = (flags & BAMFlags_IsLowQuality) != 0;/*BAM*/
+ sz = sizeof(fi) + 2*fi.readlen + fi.sglen;
+ if (align) {
+ BAM_AlignmentGetMateRefSeqId(rec, &mate_refSeqId);/*BAM*/
+ BAM_AlignmentGetMatePosition(rec, &pnext);/*BAM*/
}
- else if (!value->has_a_read) {
- /* new mated fragment - do spot assembly */
- unsigned sz;
- uint64_t fragmentId;
- FragmentInfo fi;
- KMemBank *frags;
- int32_t mate_refSeqId = -1;
- int64_t pnext = 0;
+ if(align && mate_refSeqId == refSeqId && pnext > 0 && pnext!=rpos /*** weird case in some bams**/){
+ rc = MemBankAlloc(ctx->frags, &value->fragmentId, sz, 0, false);
+ fcountBoth++;
+ } else {
+ rc = MemBankAlloc(ctx->frags, &value->fragmentId, sz, 0, true);
+ fcountOne++;
+ }
+ if (rc) {
+ (void)LOGERR(klogErr, rc, "KMemBankAlloc failed");
+ goto LOOP_END;
+ }
+ /*printf("IN:%10d\tcnt2=%ld\tcnt1=%ld\n",value->fragmentId,fcountBoth,fcountOne);*/
+
+ rc = KDataBufferResize(&fragBuf, sz);
+ if (rc) {
+ (void)LOGERR(klogErr, rc, "Failed to resize fragment buffer");
+ goto LOOP_END;
+ }
+ {{
+ int const revcmp = (isColorSpace && !aligned) ? 0 : fi.orientation;
+ uint8_t *dst = (uint8_t*) fragBuf.base;
- memset(&fi, 0, sizeof(fi));
- fi.aligned = aligned;
- fi.ti = ti;
- fi.orientation = AR_REF_ORIENT(data);
- fi.otherReadNo = AR_READNO(data);
- fi.sglen = strlen(spotGroup);
- fi.readlen = readlen;
- fi.cskey = cskey;
- fi.is_bad = (flags & BAMFlags_IsLowQuality) != 0;/*BAM*/
- sz = sizeof(fi) + 2*fi.readlen + fi.sglen;
- if (align) {
- BAMAlignmentGetMateRefSeqId(rec, &mate_refSeqId);/*BAM*/
- BAMAlignmentGetMatePosition(rec, &pnext);/*BAM*/
- }
- if(align && mate_refSeqId == refSeqId && pnext > 0 && pnext!=rpos /*** weird case in some bams**/){
- frags = ctx->fragsBoth;
- rc = KMemBankAlloc(frags, &fragmentId, sz, 0);
- value->fragmentId = fragmentId*2;
- fcountBoth++;
- } else {
- frags = ctx->fragsOne;
- rc = KMemBankAlloc(frags, &fragmentId, sz, 0);
- value->fragmentId = fragmentId*2+1;
- fcountOne++;
+ if (revcmp) {
+ QUAL_CHANGED_REVERSED;
+ SEQ__CHANGED_REV_COMP;
}
+ memcpy(dst,&fi,sizeof(fi));
+ dst += sizeof(fi);
+ COPY_READ((char *)dst, seqDNA, fi.readlen, revcmp);
+ dst += fi.readlen;
+ COPY_QUAL(dst, qual, fi.readlen, revcmp);
+ dst += fi.readlen;
+ memcpy(dst,spotGroup,fi.sglen);
+ }}
+ rc = MemBankWrite(ctx->frags, value->fragmentId, 0, fragBuf.base, sz, &rsize);
+ if (rc) {
+ (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankWrite failed writing fragment $(id)", "id=%u", value->fragmentId));
+ goto LOOP_END;
+ }
+ value->has_a_read = 1;
+ }
+ else if (spotHasFragmentInfo) {
+ /* continue spot assembly */
+ FragmentInfo *fip;
+ {
+ size_t size1;
+ size_t size2;
+
+ rc = MemBankSize(ctx->frags, fragmentId, &size1);
if (rc) {
- (void)LOGERR(klogErr, rc, "KMemBankAlloc failed");
+ (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankSize failed on fragment $(id)", "id=%u", fragmentId));
goto LOOP_END;
}
- /*printf("IN:%10d\tcnt2=%ld\tcnt1=%ld\n",value->fragmentId,fcountBoth,fcountOne);*/
- rc = KDataBufferResize(&fragBuf, sz);
+ rc = KDataBufferResize(&fragBuf, size1);
+ fip = (FragmentInfo *)fragBuf.base;
if (rc) {
- (void)LOGERR(klogErr, rc, "Failed to resize fragment buffer");
+ (void)PLOGERR(klogErr, (klogErr, rc, "Failed to resize fragment buffer", ""));
goto LOOP_END;
}
- {{
- uint8_t *dst = (uint8_t*) fragBuf.base;
- memcpy(dst,&fi,sizeof(fi));
- dst += sizeof(fi);
- COPY_READ((char *)dst, seqDNA, fi.readlen, (isColorSpace && !aligned) ? 0 : fi.orientation);
- dst += fi.readlen;
- COPY_QUAL(dst, qual, fi.readlen, (isColorSpace && !aligned) ? 0 : fi.orientation);
- dst += fi.readlen;
- memcpy(dst,spotGroup,fi.sglen);
- }}
- rc = KMemBankWrite(frags, fragmentId, 0, fragBuf.base, sz, &rsize);
+
+ rc = MemBankRead(ctx->frags, fragmentId, 0, fragBuf.base, size1, &size2);
if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankWrite failed writing fragment $(id)", "id=%u", fragmentId));
+ (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankRead failed on fragment $(id)", "id=%u", fragmentId));
goto LOOP_END;
}
- value->has_a_read = 1;
+ assert(size1 == size2);
}
- else if (value->fragmentId != 0 ) {
- /* might be second fragment */
- uint64_t sz;
- FragmentInfo *fip;
- KMemBank *frags;
-
- if(value->fragmentId & 1) frags = ctx->fragsOne;
- else frags = ctx->fragsBoth;
+ if (AR_READNO(data) == fip->otherReadNo) {
+ /* is a repeat of the same read; do nothing */
+ }
+ else {
+ /* mate found; finish spot assembly */
+ unsigned readLen[2];
+ unsigned read1 = 0;
+ unsigned read2 = 1;
+ uint8_t *src = (uint8_t*) fip + sizeof(*fip);
- rc=KMemBankSize(frags, value->fragmentId>>1, &sz);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankSize failed on fragment $(id)", "id=%u", value->fragmentId>>1));
- goto LOOP_END;
+ if (AR_READNO(data) < fip->otherReadNo) {
+ read1 = 1;
+ read2 = 0;
}
- rc=KDataBufferResize(&fragBuf, (size_t)sz);
+ readLen[read1] = fip->readlen;
+ readLen[read2] = readlen;
+ rc = SequenceRecordInit(&srec, 2, readLen);
if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to resize fragment buffer", ""));
- goto LOOP_END;
- }
- rc=KMemBankRead(frags, value->fragmentId>>1, 0, fragBuf.base, sz, &rsize);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankRead failed on fragment $(id)", "id=%u", value->fragmentId>>1));
+ (void)PLOGERR(klogErr, (klogErr, rc, "Failed resizing sequence record buffer", ""));
goto LOOP_END;
}
+ srec.ti[read1] = fip->ti;
+ srec.aligned[read1] = fip->aligned;
+ srec.is_bad[read1] = fip->is_bad;
+ srec.orientation[read1] = fip->orientation;
+ srec.cskey[read1] = fip->cskey;
+ memcpy(srec.seq + srec.readStart[read1], src, fip->readlen);
+ src += fip->readlen;
+ memcpy(srec.qual + srec.readStart[read1], src, fip->readlen);
+ src += fip->readlen;
- assert( rsize == sz );
- fip = (FragmentInfo *) fragBuf.base;
- if(AR_READNO(data) != fip->otherReadNo) {
- /* mate found */
- unsigned readLen[2];
- unsigned read1 = 0;
- unsigned read2 = 1;
- uint8_t *src = (uint8_t*) fip + sizeof(*fip);
-
- if (AR_READNO(data) < fip->otherReadNo) {
- read1 = 1;
- read2 = 0;
- }
- readLen[read1] = fip->readlen;
- readLen[read2] = readlen;
- rc = SequenceRecordInit(&srec, 2, readLen);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed resizing sequence record buffer", ""));
- goto LOOP_END;
- }
- srec.ti[read1] = fip->ti;
- srec.aligned[read1] = fip->aligned;
- srec.is_bad[read1] = fip->is_bad;
- srec.orientation[read1] = fip->orientation;
- srec.cskey[read1] = fip->cskey;
- memcpy(srec.seq + srec.readStart[read1], src, fip->readlen);
- src += fip->readlen;
- memcpy(srec.qual + srec.readStart[read1], src, fip->readlen);
- src += fip->readlen;
-
- srec.orientation[read2] = AR_REF_ORIENT(data);
- COPY_READ(srec.seq + srec.readStart[read2], seqDNA, srec.readLen[read2], (isColorSpace && !aligned) ? 0 : srec.orientation[read2]);
- COPY_QUAL(srec.qual + srec.readStart[read2], qual, srec.readLen[read2], (isColorSpace && !aligned) ? 0 : srec.orientation[read2]);
-
- srec.keyId = keyId;
- srec.is_bad[read2] = (flags & BAMFlags_IsLowQuality) != 0;
- srec.aligned[read2] = aligned;
- srec.cskey[read2] = cskey;
- srec.ti[read2] = ti;
+ srec.orientation[read2] = AR_REF_ORIENT(data);
+ {
+ int const revcmp = (isColorSpace && !aligned) ? 0 : srec.orientation[read2];
- srec.spotGroup = spotGroup;
- srec.spotGroupLen = strlen(spotGroup);
- if (value->pcr_dup && (srec.is_bad[0] || srec.is_bad[1])) {
- filterFlagConflictRecords++;
- if(filterFlagConflictRecords < MAX_WARNINGS_FLAG_CONFLICT){
- (void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)': both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "name=%s", name));
- } else if(filterFlagConflictRecords == MAX_WARNINGS_FLAG_CONFLICT){
- (void)PLOGMSG(klogWarn, (klogWarn, "Last reported warning: Spot '$(name)': both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "name=%s", name));
- }
- }
- rc = SequenceWriteRecord(seq, &srec, isColorSpace, value->pcr_dup, value->platform);
- if (rc) {
- (void)LOGERR(klogErr, rc, "SequenceWriteRecord failed");
- goto LOOP_END;
+ if (revcmp) {
+ QUAL_CHANGED_REVERSED;
+ SEQ__CHANGED_REV_COMP;
}
- CTX_VALUE_SET_S_ID(*value, ++ctx->spotId);
- if(value->fragmentId & 1){
- fcountOne--;
- } else {
- fcountBoth--;
+ COPY_READ(srec.seq + srec.readStart[read2], seqDNA, srec.readLen[read2], revcmp);
+ COPY_QUAL(srec.qual + srec.readStart[read2], qual, srec.readLen[read2], revcmp);
+ }
+ srec.keyId = keyId;
+ srec.is_bad[read2] = (flags & BAMFlags_IsLowQuality) != 0;
+ srec.aligned[read2] = aligned;
+ srec.cskey[read2] = cskey;
+ srec.ti[read2] = ti;
+
+ srec.spotGroup = spotGroup;
+ srec.spotGroupLen = strlen(spotGroup);
+ if (value->pcr_dup && (srec.is_bad[0] || srec.is_bad[1])) {
+ FLAG_CHANGED_400_AND_200;
+ filterFlagConflictRecords++;
+ if (filterFlagConflictRecords < MAX_WARNINGS_FLAG_CONFLICT) {
+ (void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)': both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "name=%s", name));
}
- /* printf("OUT:%9d\tcnt2=%ld\tcnt1=%ld\n",value->fragmentId,fcountBoth,fcountOne);*/
- rc = KMemBankFree(frags, value->fragmentId>>1);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankFree failed on fragment $(id)", "id=%u", value->fragmentId>>1));
- goto LOOP_END;
+ else if (filterFlagConflictRecords == MAX_WARNINGS_FLAG_CONFLICT) {
+ (void)PLOGMSG(klogWarn, (klogWarn, "Last reported warning: Spot '$(name)': both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "name=%s", name));
}
- value->fragmentId = 0;
}
+ rc = SequenceWriteRecord(seq, &srec, isColorSpace, value->pcr_dup, value->platform);
+ if (rc) {
+ (void)LOGERR(klogErr, rc, "SequenceWriteRecord failed");
+ goto LOOP_END;
+ }
+ CTX_VALUE_SET_S_ID(*value, ++ctx->spotId);
+ if(fragmentId & 1){
+ fcountOne--;
+ } else {
+ fcountBoth--;
+ }
+ /* printf("OUT:%9d\tcnt2=%ld\tcnt1=%ld\n",fragmentId,fcountBoth,fcountOne);*/
+ rc = MemBankFree(ctx->frags, fragmentId);
+ if (rc) {
+ (void)PLOGERR(klogErr, (klogErr, rc, "KMemBankFree failed on fragment $(id)", "id=%u", fragmentId));
+ goto LOOP_END;
+ }
+ value->fragmentId = 0;
}
}
+ else {
+ (void)PLOGMSG(klogErr, (klogErr, "Spot '$(name)' has caused the loader to enter an illogical state", "name=%s", name));
+ assert("this should never happen");
+ }
+
if (!isPrimary && aligned) {
int32_t bam_mrid;
int64_t mpos;
- int64_t mrid;
+ int64_t mrid = 0;
int64_t tlen;
-
- BAMAlignmentGetMatePosition(rec, &mpos);/*BAM*/
- BAMAlignmentGetMateRefSeqId(rec, &bam_mrid);/*BAM*/
- BAMAlignmentGetInsertSize(rec, &tlen);/*BAM*/
-
+
+ BAM_AlignmentGetMatePosition(rec, &mpos);/*BAM*/
+ BAM_AlignmentGetMateRefSeqId(rec, &bam_mrid);/*BAM*/
+ BAM_AlignmentGetInsertSize(rec, &tlen);/*BAM*/
+
if (mpos >= 0 && bam_mrid >= 0 && tlen != 0) {
BAMRefSeq const *mref;/*BAM*/
-
- BAMFileGetRefSeq(bam, bam_mrid, &mref);/*BAM*/
+
+ BAM_FileGetRefSeq(bam, bam_mrid, &mref);/*BAM*/
if (mref) {
rc_t rc_temp = ReferenceGet1stRow(ref, &mrid, mref->name);
if (rc_temp == 0) {
@@ -1611,17 +1977,23 @@ MIXED_BASE_AND_COLOR:
}
else {
(void)PLOGERR(klogWarn, (klogWarn, rc_temp, "Failed to get refID for $(name)", "name=%s", mref->name));
- mrid = 0;
+ MATE_INFO_LOST_UNKNOWN_REF;
}
data.mate_ref_id = mrid;
}
+ else {
+ MATE_INFO_LOST_INVALID;
+ }
+ }
+ else if (mpos >= 0 || bam_mrid >= 0 || tlen != 0) {
+ MATE_INFO_LOST_MISSING;
}
}
}
else if (CTX_VALUE_GET_S_ID(*value) == 0 && (isPrimary || !originally_aligned)) {
/* new unmated fragment - no spot assembly */
unsigned readLen[1];
-
+
readLen[0] = readlen;
rc = SequenceRecordInit(&srec, 1, readLen);
if (rc) {
@@ -1633,21 +2005,30 @@ MIXED_BASE_AND_COLOR:
srec.is_bad[0] = (flags & BAMFlags_IsLowQuality) != 0;
srec.orientation[0] = AR_REF_ORIENT(data);
srec.cskey[0] = cskey;
- COPY_READ(srec.seq + srec.readStart[0], seqDNA, readlen, (isColorSpace && !aligned) ? 0 : srec.orientation[0]);
- COPY_QUAL(srec.qual + srec.readStart[0], qual, readlen, (isColorSpace && !aligned) ? 0 : srec.orientation[0]);
-
+ {
+ int const revcmp = (isColorSpace && !aligned) ? 0 : srec.orientation[0];
+
+ if (revcmp) {
+ QUAL_CHANGED_REVERSED;
+ SEQ__CHANGED_REV_COMP;
+ }
+ COPY_READ(srec.seq + srec.readStart[0], seqDNA, readlen, revcmp);
+ COPY_QUAL(srec.qual + srec.readStart[0], qual, readlen, revcmp);
+ }
+
srec.keyId = keyId;
-
+
srec.spotGroup = spotGroup;
srec.spotGroupLen = strlen(spotGroup);
if (value->pcr_dup && srec.is_bad[0]) {
+ FLAG_CHANGED_400_AND_200;
filterFlagConflictRecords++;
if (filterFlagConflictRecords < MAX_WARNINGS_FLAG_CONFLICT) {
(void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)': both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "name=%s", name));
}
else if (filterFlagConflictRecords == MAX_WARNINGS_FLAG_CONFLICT) {
(void)PLOGMSG(klogWarn, (klogWarn, "Last reported warning: Spot '$(name)': both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "name=%s", name));
- }
+ }
}
rc = SequenceWriteRecord(seq, &srec, isColorSpace, value->pcr_dup, value->platform);
if (rc) {
@@ -1657,20 +2038,20 @@ MIXED_BASE_AND_COLOR:
CTX_VALUE_SET_S_ID(*value, ++ctx->spotId);
value->fragmentId = 0;
}
-
+
if (aligned) {
if (value->alignmentCount[AR_READNO(data) - 1] < 254)
++value->alignmentCount[AR_READNO(data) - 1];
++ctx->alignCount;
-
+
assert(keyId >> 32 < ctx->key2id_count);
assert((uint32_t)keyId < ctx->idCount[keyId >> 32]);
-
+
rc = AlignmentWriteRecord(align, &data);
if (rc == 0) {
if (!isPrimary)
data.alignId = ++ctx->secondId;
-
+
rc = ReferenceAddAlignId(ref, data.alignId, isPrimary);
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "ReferenceAddAlignId failed", ""));
@@ -1684,9 +2065,9 @@ MIXED_BASE_AND_COLOR:
}
}
/**************************************************************/
-
+
LOOP_END:
- BAMAlignmentRelease(rec);
+ BAM_AlignmentRelease(rec);
++reccount;
if (G.maxAlignCount > 0 && reccount >= G.maxAlignCount)
break;
@@ -1697,12 +2078,13 @@ MIXED_BASE_AND_COLOR:
(void)PLOGMSG(klogWarn, (klogWarn, "$(cnt1) out of $(cnt2) records contained warning : both 0x400 and 0x200 flag bits set, only 0x400 will be saved", "cnt1=%lu,cnt2=%lu", filterFlagConflictRecords,recordsProcessed));
}
if (rc == 0 && recordsProcessed == 0) {
- (void)LOGMSG(klogWarn, (G.limit2config || G.refFilter != NULL) ?
+ (void)LOGMSG(klogWarn, (G.limit2config || G.refFilter != NULL) ?
"All records from the file were filtered out" :
"The file contained no records that were processed.");
rc = RC(rcAlign, rcFile, rcReading, rcData, rcEmpty);
}
- BAMFileRelease(bam);
+ BAM_FileRelease(bam);
+ MMArrayLock(ctx->id2value);
KDataBufferWhack(&buf);
KDataBufferWhack(&fragBuf);
KDataBufferWhack(&srec.storage);
@@ -1715,16 +2097,14 @@ static rc_t WriteSoloFragments(context_t *ctx, Sequence *seq)
{
uint32_t i;
unsigned j;
- uint32_t fcountOne = 0;
- uint32_t fcountBoth = 0;
uint64_t idCount = 0;
rc_t rc;
KDataBuffer fragBuf;
SequenceRecord srec;
-
+
++ctx->pass;
memset(&srec, 0, sizeof(srec));
-
+
rc = KDataBufferMake(&fragBuf, 8, 0);
if (rc) {
(void)LOGERR(klogErr, rc, "KDataBufferMake failed");
@@ -1734,37 +2114,26 @@ static rc_t WriteSoloFragments(context_t *ctx, Sequence *seq)
idCount += ctx->idCount[j];
}
KLoadProgressbar_Append(ctx->progress[ctx->pass - 1], idCount);
-
+
for (idCount = 0, j = 0; j < ctx->key2id_count; ++j) {
for (i = 0; i != ctx->idCount[j]; ++i, ++idCount) {
uint64_t const keyId = ((uint64_t)j << 32) | i;
ctx_value_t *value;
size_t rsize;
- uint64_t id;
- uint64_t sz;
+ size_t sz;
unsigned readLen[2];
unsigned read = 0;
FragmentInfo const *fip;
uint8_t const *src;
- KMemBank *frags;
-
+
rc = MMArrayGet(ctx->id2value, (void **)&value, keyId);
if (rc)
break;
KLoadProgressbar_Process(ctx->progress[ctx->pass - 1], 1, false);
if (value->fragmentId == 0)
continue;
- if (value->fragmentId & 1) {
- frags = ctx->fragsOne;
- fcountOne++;
- }
- else {
- frags = ctx->fragsBoth;
- fcountBoth++;
- }
- id = value->fragmentId >> 1;
-
- rc = KMemBankSize(frags, id, &sz);
+
+ rc = MemBankSize(ctx->frags, value->fragmentId, &sz);
if (rc) {
(void)LOGERR(klogErr, rc, "KMemBankSize failed");
break;
@@ -1774,7 +2143,7 @@ static rc_t WriteSoloFragments(context_t *ctx, Sequence *seq)
(void)LOGERR(klogErr, rc, "KDataBufferResize failed");
break;
}
- rc = KMemBankRead(frags, id, 0, fragBuf.base, sz, &rsize);
+ rc = MemBankRead(ctx->frags, value->fragmentId, 0, fragBuf.base, sz, &rsize);
if (rc) {
(void)LOGERR(klogErr, rc, "KMemBankRead failed");
break;
@@ -1782,21 +2151,21 @@ static rc_t WriteSoloFragments(context_t *ctx, Sequence *seq)
assert( rsize == sz );
fip = (FragmentInfo const *)fragBuf.base;
src = (uint8_t const *)&fip[1];
-
+
readLen[0] = readLen[1] = 0;
if (!value->unmated && ( (fip->aligned && CTX_VALUE_GET_P_ID(*value, 0) == 0)
|| (value->unaligned_2)))
{
read = 1;
}
-
+
readLen[read] = fip->readlen;
rc = SequenceRecordInit(&srec, value->unmated ? 1 : 2, readLen);
if (rc) {
(void)LOGERR(klogErr, rc, "SequenceRecordInit failed");
break;
}
-
+
srec.ti[read] = fip->ti;
srec.aligned[read] = fip->aligned;
srec.is_bad[read] = fip->is_bad;
@@ -1809,7 +2178,7 @@ static rc_t WriteSoloFragments(context_t *ctx, Sequence *seq)
srec.spotGroup = (char *)src;
srec.spotGroupLen = fip->sglen;
srec.keyId = keyId;
-
+
rc = SequenceWriteRecord(seq, &srec, ctx->isColorSpace, value->pcr_dup, value->platform);
if (rc) {
(void)LOGERR(klogErr, rc, "SequenceWriteRecord failed");
@@ -1819,7 +2188,7 @@ static rc_t WriteSoloFragments(context_t *ctx, Sequence *seq)
CTX_VALUE_SET_S_ID(*value, ++ctx->spotId);
}
}
- /*printf("DONE_SOLO:\tcnt2=%d\tcnt1=%d\n",fcountBoth,fcountOne);*/
+ MMArrayLock(ctx->id2value);
KDataBufferWhack(&fragBuf);
KDataBufferWhack(&srec.storage);
return rc;
@@ -1829,19 +2198,19 @@ static rc_t SequenceUpdateAlignInfo(context_t *ctx, Sequence *seq)
{
rc_t rc = 0;
uint64_t row;
- const ctx_value_t *value;
+ ctx_value_t const *value;
uint64_t keyId;
-
+
++ctx->pass;
KLoadProgressbar_Append(ctx->progress[ctx->pass - 1], ctx->spotId + 1);
-
+
for (row = 1; row <= ctx->spotId; ++row) {
rc = SequenceReadKey(seq, row, &keyId);
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "Failed to get key for row $(row)", "row=%u", (unsigned)row));
break;
}
- rc = MMArrayGet(ctx->id2value, (void **)&value, keyId);
+ rc = MMArrayGetRead(ctx->id2value, (void const **)&value, keyId);
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "Failed to read info for row $(row), index $(idx)", "row=%u,idx=%u", (unsigned)row, (unsigned)keyId));
break;
@@ -1853,10 +2222,10 @@ static rc_t SequenceUpdateAlignInfo(context_t *ctx, Sequence *seq)
}
{{
int64_t primaryId[2];
-
+
primaryId[0] = CTX_VALUE_GET_P_ID(*value, 0);
primaryId[1] = CTX_VALUE_GET_P_ID(*value, 1);
-
+
rc = SequenceUpdateAlignData(seq, row, value->unmated ? 1 : 2,
primaryId,
value->alignmentCount);
@@ -1867,6 +2236,7 @@ static rc_t SequenceUpdateAlignInfo(context_t *ctx, Sequence *seq)
}
KLoadProgressbar_Process(ctx->progress[ctx->pass - 1], 1, false);
}
+ MMArrayLock(ctx->id2value);
return rc;
}
@@ -1874,7 +2244,7 @@ static rc_t AlignmentUpdateSpotInfo(context_t *ctx, Alignment *align)
{
rc_t rc;
uint64_t keyId;
-
+
++ctx->pass;
KLoadProgressbar_Append(ctx->progress[ctx->pass - 1], ctx->alignCount);
@@ -1882,7 +2252,7 @@ static rc_t AlignmentUpdateSpotInfo(context_t *ctx, Alignment *align)
rc = AlignmentStartUpdatingSpotIds(align);
while (rc == 0 && (rc = Quitting()) == 0) {
ctx_value_t const *value;
-
+
rc = AlignmentGetSpotKey(align, &keyId);
if (rc) {
if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound)
@@ -1891,10 +2261,10 @@ static rc_t AlignmentUpdateSpotInfo(context_t *ctx, Alignment *align)
}
assert(keyId >> 32 < ctx->key2id_count);
assert((uint32_t)keyId < ctx->idCount[keyId >> 32]);
- rc = MMArrayGet(ctx->id2value, (void **)&value, keyId);
+ rc = MMArrayGetRead(ctx->id2value, (void const **)&value, keyId);
if (rc == 0) {
int64_t const spotId = CTX_VALUE_GET_S_ID(*value);
-
+
if (spotId == 0) {
(void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(id)' was never assigned a spot id, probably has no primary alignments", "id=%lx", keyId));
/* (void)PLOGMSG(klogWarn, (klogWarn, "Spot #$(i): { $(s) }", "i=%lu,s=%s", keyId, Print_ctx_value_t(value))); */
@@ -1903,6 +2273,7 @@ static rc_t AlignmentUpdateSpotInfo(context_t *ctx, Alignment *align)
}
KLoadProgressbar_Process(ctx->progress[ctx->pass - 1], 1, false);
}
+ MMArrayLock(ctx->id2value);
return rc;
}
@@ -1919,12 +2290,12 @@ static rc_t ArchiveBAM(VDBManager *mgr, VDatabase *db,
context_t ctx;
bool has_sequences = false;
unsigned i;
-
+
*has_alignments = false;
rc = ReferenceInit(&ref, mgr, db);
if (rc)
return rc;
-
+
if (G.onlyVerifyReferences) {
for (i = 0; i < bamFiles && rc == 0; ++i) {
rc = ProcessBAM(bamFile[i], NULL, db, &ref, NULL, NULL, NULL, NULL);
@@ -1934,16 +2305,16 @@ static rc_t ArchiveBAM(VDBManager *mgr, VDatabase *db,
}
SequenceInit(&seq, db);
align = AlignmentMake(db);
-
+
rc = SetupContext(&ctx, bamFiles + seqFiles);
if (rc)
return rc;
-
+
++ctx.pass;
for (i = 0; i < bamFiles && rc == 0; ++i) {
bool this_has_alignments = false;
bool this_has_sequences = false;
-
+
rc = ProcessBAM(bamFile[i], &ctx, db, &ref, &seq, align, &this_has_alignments, &this_has_sequences);
*has_alignments |= this_has_alignments;
has_sequences |= this_has_sequences;
@@ -1951,7 +2322,7 @@ static rc_t ArchiveBAM(VDBManager *mgr, VDatabase *db,
for (i = 0; i < seqFiles && rc == 0; ++i) {
bool this_has_alignments = false;
bool this_has_sequences = false;
-
+
rc = ProcessBAM(seqFile[i], &ctx, db, &ref, &seq, align, &this_has_alignments, &this_has_sequences);
*has_alignments |= this_has_alignments;
has_sequences |= this_has_sequences;
@@ -1980,7 +2351,7 @@ static rc_t ArchiveBAM(VDBManager *mgr, VDatabase *db,
}
}
}
-
+
if (*has_alignments && rc == 0 && (rc = Quitting()) == 0) {
(void)LOGMSG(klogInfo, "Writing alignment spot ids");
rc = AlignmentUpdateSpotInfo(&ctx, align);
@@ -1992,9 +2363,9 @@ static rc_t ArchiveBAM(VDBManager *mgr, VDatabase *db,
rc2 = ReferenceWhack(&ref, *has_alignments && rc == 0 && (rc = Quitting()) == 0);
if (rc == 0)
rc = rc2;
-
+
SequenceWhack(&seq, rc == 0);
-
+
ContextRelease(&ctx);
if (rc == 0) {
@@ -2007,7 +2378,7 @@ rc_t WriteLoaderSignature(KMetadata *meta, char const progName[])
{
KMDataNode *node;
rc_t rc = KMetadataOpenNodeUpdate(meta, &node, "/");
-
+
if (rc == 0) {
rc = KLoaderMeta_Write(node, progName, __DATE__, "BAM", KAppVersion());
KMDataNodeRelease(node);
@@ -2022,7 +2393,7 @@ rc_t OpenPath(char const path[], KDirectory **dir)
{
KDirectory *p;
rc_t rc = KDirectoryNativeDir(&p);
-
+
if (rc == 0) {
rc = KDirectoryOpenDirUpdate(p, dir, false, "%s", path);
KDirectoryRelease(p);
@@ -2035,7 +2406,7 @@ rc_t ConvertDatabaseToUnmapped(VDatabase *db)
{
VTable* tbl;
rc_t rc = VDatabaseOpenTableUpdate(db, &tbl, "SEQUENCE");
- if (rc == 0)
+ if (rc == 0)
{
VTableRenameColumn(tbl, false, "CMP_ALTREAD", "ALTREAD");
VTableRenameColumn(tbl, false, "CMP_READ", "READ");
@@ -2053,24 +2424,24 @@ rc_t run(char const progName[],
rc_t rc;
rc_t rc2;
char const *db_type = G.expectUnsorted ? "NCBI:align:db:alignment_unsorted" : "NCBI:align:db:alignment_sorted";
-
+
rc = VDBManagerMakeUpdate(&mgr, NULL);
if (rc) {
(void)LOGERR (klogErr, rc, "failed to create VDB Manager!");
}
else {
bool has_alignments = false;
-
+
rc = VDBManagerDisablePagemapThread(mgr);
if (rc == 0)
{
-
+
if (G.onlyVerifyReferences) {
rc = ArchiveBAM(mgr, NULL, bamFiles, bamFile, 0, NULL, &has_alignments);
}
else {
VSchema *schema;
-
+
rc = VDBManagerMakeSchema(mgr, &schema);
if (rc) {
(void)LOGERR (klogErr, rc, "failed to create schema");
@@ -2083,7 +2454,7 @@ rc_t run(char const progName[],
}
else {
VDatabase *db;
-
+
rc = VDBManagerCreateDB(mgr, &db, schema, db_type,
kcmInit + kcmMD5, "%s", G.outpath);
rc2 = VSchemaRelease(schema);
@@ -2093,33 +2464,46 @@ rc_t run(char const progName[],
rc = rc2;
if (rc == 0) {
rc = ArchiveBAM(mgr, db, bamFiles, bamFile, seqFiles, seqFile, &has_alignments);
+ if (rc == 0)
+ PrintChangeReport();
if (rc == 0 && !has_alignments) {
rc = ConvertDatabaseToUnmapped(db);
}
-
+
rc2 = VDatabaseRelease(db);
if (rc2)
(void)LOGERR(klogWarn, rc2, "Failed to close database");
if (rc == 0)
rc = rc2;
-
+
if (rc == 0) {
- KMetadata *meta;
- KDBManager *kmgr;
-
- rc = VDBManagerOpenKDBManagerUpdate(mgr, &kmgr);
- if (rc == 0) {
- KDatabase *kdb;
-
- rc = KDBManagerOpenDBUpdate(kmgr, &kdb, "%s", G.outpath);
+ KMetadata *meta = NULL;
+
+ {
+ KDBManager *kmgr = NULL;
+
+ rc = VDBManagerOpenKDBManagerUpdate(mgr, &kmgr);
if (rc == 0) {
- rc = KDatabaseOpenMetadataUpdate(kdb, &meta);
- KDatabaseRelease(kdb);
+ KDatabase *kdb;
+
+ rc = KDBManagerOpenDBUpdate(kmgr, &kdb, "%s", G.outpath);
+ if (rc == 0) {
+ rc = KDatabaseOpenMetadataUpdate(kdb, &meta);
+ KDatabaseRelease(kdb);
+ }
+ KDBManagerRelease(kmgr);
}
- KDBManagerRelease(kmgr);
}
if (rc == 0) {
rc = WriteLoaderSignature(meta, progName);
+ if (rc == 0) {
+ KMDataNode *changes = NULL;
+
+ rc = KMetadataOpenNodeUpdate(meta, &changes, "CHANGES");
+ if (rc == 0)
+ RecordChanges(changes, "CHANGE");
+ KMDataNodeRelease(changes);
+ }
KMetadataRelease(meta);
}
}
diff --git a/tools/bam-loader/loader-imp3.c b/tools/bam-loader/loader-imp3.c
deleted file mode 100644
index b3387ad..0000000
--- a/tools/bam-loader/loader-imp3.c
+++ /dev/null
@@ -1,369 +0,0 @@
-/*===========================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * ===========================================================================
- *
- */
-
-#include <sysalloc.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <klib/rc.h>
-#include <klib/log.h>
-
-#include <kdb/btree.h>
-#include <kdb/meta.h>
-#include <kdb/manager.h>
-#include <kdb/database.h>
-
-#include <kapp/loader-meta.h>
-#include <kapp/main.h>
-
-#include <kfs/directory.h>
-
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/database.h>
-#include <vdb/vdb-priv.h>
-
-#include <loader/common-writer.h>
-#include <loader/sequence-writer.h>
-#include <loader/reference-writer.h>
-
-#include "bam-reader.h"
-
-static rc_t OpenBAM(const ReaderFile **bam, VDatabase *db, const CommonWriterSettings* G, const char bamFile[])
-{
- rc_t rc = BamReaderFileMake(bam, G->headerText, bamFile);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open '$(file)'", "file=%s", bamFile));
- }
- else if (db) {
- KMetadata *dbmeta;
-
- rc = VDatabaseOpenMetadataUpdate(db, &dbmeta);
- if (rc == 0) {
- KMDataNode *node;
-
- rc = KMetadataOpenNodeUpdate(dbmeta, &node, "BAM_HEADER");
- KMetadataRelease(dbmeta);
- if (rc == 0) {
- char const *header;
- size_t size;
-
- rc = BAMFileGetHeaderText(ToBam(*bam), &header, &size);
- if (rc == 0) {
- rc = KMDataNodeWrite(node, header, size);
- }
- KMDataNodeRelease(node);
- }
- }
- }
-
- return rc;
-}
-
-static rc_t VerifyReferences(ReferenceInfo const *bam, const CommonWriterSettings* G, Reference const *ref)
-{
- rc_t rc = 0;
- uint32_t n;
- unsigned i;
-
- ReferenceInfoGetRefSeqCount(bam, &n);
- for (i = 0; i != n; ++i) {
- ReferenceSequence refSeq;
-
- ReferenceInfoGetRefSeq(bam, i, &refSeq);
- if (G->refFilter && strcmp(refSeq.name, G->refFilter) != 0)
- continue;
-
- rc = ReferenceVerify(ref, refSeq.name, refSeq.length, refSeq.checksum);
- if (rc) {
- if (GetRCObject(rc) == rcChecksum && GetRCState(rc) == rcUnequal) {
-#if NCBI
- (void)PLOGMSG(klogWarn, (klogWarn, "Reference: '$(name)', Length: $(len); checksums do not match", "name=%s,len=%u", refSeq.name, (unsigned)refSeq.length));
-#endif
- }
- else
- if (GetRCObject(rc) == rcSize && GetRCState(rc) == rcUnequal) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Reference: '$(name)', Length: $(len); lengths do not match", "name=%s,len=%u", refSeq.name, (unsigned)refSeq.length));
- }
- else if (GetRCObject(rc) == rcSize && GetRCState(rc) == rcEmpty) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Reference: '$(name)', Length: $(len); fasta file is empty", "name=%s,len=%u", refSeq.name, (unsigned)refSeq.length));
- }
- else if (GetRCObject(rc) == rcId && GetRCState(rc) == rcNotFound) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Reference: '$(name)', Length: $(len); no match found", "name=%s,len=%u", refSeq.name, (unsigned)refSeq.length));
- }
- else {
- (void)PLOGERR(klogWarn, (klogWarn, rc, "Reference: '$(name)', Length: $(len); error", "name=%s,len=%u", refSeq.name, (unsigned)refSeq.length));
- }
- }
- else if (G->onlyVerifyReferences) {
- (void)PLOGMSG(klogInfo, (klogInfo, "Reference: '$(name)', Length: $(len); match found", "name=%s,len=%u", refSeq.name, (unsigned)refSeq.length));
- }
- }
- return 0;
-}
-
-rc_t ProcessReferences(const CommonWriterSettings* G,
- VDBManager *mgr,
- VDatabase *db,
- unsigned bamFiles,
- char const *bamFile[])
-{
- Reference ref;
- rc_t rc = ReferenceInit(&ref, mgr, db, G->expectUnsorted, G->acceptHardClip, G->refXRefPath, G->inpath, G->maxSeqLen, G->refFiles);
- if (rc)
- return rc;
-
- if (!G->noVerifyReferences) {
- unsigned i;
- for (i = 0; i < bamFiles; ++i) {
- const ReaderFile *bam;
- const ReferenceInfo* ri;
-
- rc = OpenBAM(&bam, db, G, bamFile[i]);
- if (rc)
- break;
-
- rc = ReaderFileGetReferenceInfo(bam, &ri);
- if (rc == 0 && ri != NULL)
- {
- rc = VerifyReferences(ri, G, &ref);
- }
- ReferenceInfoRelease(ri);
-
- ReaderFileRelease(bam);
-
- if (rc)
- break;
- }
- }
-
- ReferenceWhack(&ref, false, G->maxSeqLen, Quitting);
- return rc;
-}
-
-rc_t ArchiveBAM(CommonWriterSettings* G,
- VDBManager *mgr,
- VDatabase *db,
- unsigned bamFiles,
- char const *bamFile[],
- unsigned seqFiles,
- char const *seqFile[],
- bool *has_alignments)
-{
- rc_t rc = 0;
- unsigned i;
- CommonWriter cw;
-
- rc = CommonWriterInit( &cw, mgr, db, G);
- if (rc != 0)
- return rc;
-
- for (i = 0; i < bamFiles && rc == 0; ++i) {
- const ReaderFile *reader;
- rc = OpenBAM(&reader, db, G, bamFile[i]);
- if (rc == 0)
- {
- rc = CommonWriterArchive( &cw, reader );
- if (rc != 0)
- ReaderFileRelease(reader);
- else
- rc = ReaderFileRelease(reader);
- }
- }
- for (i = 0; i < seqFiles && rc == 0; ++i) {
- const ReaderFile *reader;
- rc = OpenBAM(&reader, db, G, seqFile[i]);
- if (rc == 0)
- {
- rc = CommonWriterArchive( &cw, reader );
- if (rc != 0)
- ReaderFileRelease(reader);
- else
- rc = ReaderFileRelease(reader);
- }
- }
- if (rc == 0)
- rc = CommonWriterComplete( &cw, Quitting() != 0, G->maxMateDistance );
- else
- CommonWriterComplete( &cw, true, 0 );
-
- *has_alignments = cw.had_alignments;
- G->errCount = cw.err_count;
-
- if (rc == 0)
- rc = CommonWriterWhack( &cw );
- else
- CommonWriterWhack( &cw );
-
- if (rc == 0) {
- (void)LOGMSG(klogInfo, "Successfully loaded all files");
- }
- return rc;
-}
-
-rc_t WriteLoaderSignature(KMetadata *meta, char const progName[])
-{
- KMDataNode *node;
- rc_t rc = KMetadataOpenNodeUpdate(meta, &node, "/");
-
- if (rc == 0) {
- rc = KLoaderMeta_Write(node, progName, __DATE__, "BAM", KAppVersion());
- KMDataNodeRelease(node);
- }
- if (rc) {
- (void)LOGERR(klogErr, rc, "Cannot update loader meta");
- }
- return rc;
-}
-
-rc_t OpenPath(char const path[], KDirectory **dir)
-{
- KDirectory *p;
- rc_t rc = KDirectoryNativeDir(&p);
-
- if (rc == 0) {
- rc = KDirectoryOpenDirUpdate(p, dir, false, "%s", path);
- KDirectoryRelease(p);
- }
- return rc;
-}
-
-static
-rc_t ConvertDatabaseToUnmapped(VDatabase *db)
-{
- VTable* tbl;
- rc_t rc = VDatabaseOpenTableUpdate(db, &tbl, "SEQUENCE");
- if (rc == 0)
- {
- rc = VTableRenameColumn(tbl, false, "CMP_ALTREAD", "ALTREAD");
- if (rc == 0 || GetRCState(rc) == rcNotFound)
- rc = VTableRenameColumn(tbl, false, "CMP_READ", "READ");
-/* if (rc == 0 || GetRCState(rc) == rcNotFound)
- rc = VTableRenameColumn(tbl, false, "CMP_ALTCSREAD", "ALTCSREAD");
- if (rc == 0 || GetRCState(rc) == rcNotFound)
- rc = VTableRenameColumn(tbl, false, "CMP_CSREAD", "CSREAD");*/
- if (GetRCState(rc) == rcNotFound)
- rc = 0;
- rc = VTableRelease(tbl);
- }
- return rc;
-}
-
-rc_t run(char const progName[], CommonWriterSettings* G,
- unsigned bamFiles, char const *bamFile[],
- unsigned seqFiles, char const *seqFile[])
-{
- VDBManager *mgr;
- rc_t rc;
- rc_t rc2;
- char const *db_type = G->expectUnsorted ? "NCBI:align:db:alignment_unsorted" : "NCBI:align:db:alignment_sorted";
-
- rc = VDBManagerMakeUpdate(&mgr, NULL);
- if (rc) {
- (void)LOGERR (klogErr, rc, "failed to create VDB Manager!");
- }
- else {
- bool has_alignments = false;
-
- rc = VDBManagerDisablePagemapThread(mgr);
- if (rc == 0)
- {
- if (G->onlyVerifyReferences) {
- rc = ProcessReferences(G, mgr, NULL, bamFiles, bamFile);
- }
- else {
- VSchema *schema;
-
- rc = VDBManagerMakeSchema(mgr, &schema);
- if (rc) {
- (void)LOGERR (klogErr, rc, "failed to create schema");
- }
- else {
- (void)(rc = VSchemaAddIncludePath(schema, "%s", G->schemaIncludePath));
- rc = VSchemaParseFile(schema, "%s", G->schemaPath);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "failed to parse schema file $(file)", "file=%s", G->schemaPath));
- }
- else {
- VDatabase *db;
-
- rc = VDBManagerCreateDB(mgr, &db, schema, db_type,
- kcmInit + kcmMD5, "%s", G->outpath);
- rc2 = VSchemaRelease(schema);
- if (rc2)
- (void)LOGERR(klogWarn, rc2, "Failed to release schema");
- if (rc == 0)
- rc = rc2;
- if (rc == 0) {
- rc = ProcessReferences(G, mgr, db, bamFiles, bamFile);
- if (rc == 0)
- {
- rc = ArchiveBAM(G, mgr, db, bamFiles, bamFile, seqFiles, seqFile, &has_alignments);
- }
- }
-
- if (rc == 0 && !has_alignments) {
- rc = ConvertDatabaseToUnmapped(db);
- }
-
- rc2 = VDatabaseRelease(db);
- if (rc2)
- (void)LOGERR(klogWarn, rc2, "Failed to close database");
- if (rc == 0)
- rc = rc2;
-
- if (rc == 0) {
- KMetadata *meta;
- KDBManager *kmgr;
-
- rc = VDBManagerOpenKDBManagerUpdate(mgr, &kmgr);
- if (rc == 0) {
- KDatabase *kdb;
-
- rc = KDBManagerOpenDBUpdate(kmgr, &kdb, "%s", G->outpath);
- if (rc == 0) {
- rc = KDatabaseOpenMetadataUpdate(kdb, &meta);
- KDatabaseRelease(kdb);
- }
- KDBManagerRelease(kmgr);
- }
- if (rc == 0) {
- rc = WriteLoaderSignature(meta, progName);
- KMetadataRelease(meta);
- }
- }
- }
- }
- }
- }
- rc2 = VDBManagerRelease(mgr);
- if (rc2)
- (void)LOGERR(klogWarn, rc2, "Failed to release VDB Manager");
- if (rc == 0)
- rc = rc2;
- }
- return rc;
-}
diff --git a/tools/bam-loader/mem-bank.cpp b/tools/bam-loader/mem-bank.cpp
new file mode 100644
index 0000000..75d8eda
--- /dev/null
+++ b/tools/bam-loader/mem-bank.cpp
@@ -0,0 +1,477 @@
+/* ===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include <klib/defs.h>
+#include <klib/rc.h>
+
+extern "C" {
+#include "mem-bank.h"
+}
+
+#define USE_KMEMBANK 0
+
+#if USE_KMEMBANK
+#include <kfs/file.h>
+#include <kfs/directory.h>
+#include <kfs/pagefile.h>
+#include <kfs/pmem.h>
+#include <klib/log.h>
+#include <klib/printf.h>
+
+#define FRAG_CHUNK_SIZE (128)
+
+struct MemBank {
+ KMemBank *fragsOne;
+ KMemBank *fragsBoth;
+};
+
+static rc_t OpenMBankFile(KMemBank **const mbank, KDirectory *const dir, int const pid, char const *const suffix, size_t const climit)
+{
+ KFile *file = NULL;
+ char fname[4096];
+ rc_t rc = string_printf(fname, sizeof(fname), NULL, "frag_data%s.%u", suffix, pid);
+
+ if (rc)
+ return rc;
+
+ rc = KDirectoryCreateFile(dir, &file, true, 0600, kcmInit, "%s", fname);
+ KDirectoryRemove(dir, 0, "%s", fname);
+ if (rc == 0) {
+ KPageFile *backing;
+
+ rc = KPageFileMakeUpdate(&backing, file, climit, false);
+ KFileRelease(file);
+ if (rc == 0) {
+ rc = KMemBankMake(mbank, FRAG_CHUNK_SIZE, 0, backing);
+ KPageFileRelease(backing);
+ }
+ }
+ return rc;
+}
+
+static rc_t MemBank_Make(MemBank **rslt, KDirectory *const dir, int const pid, size_t const climits[2])
+{
+ KMemBank *fragsOne;
+
+ rc_t rc = OpenMBankFile(&fragsOne, dir, pid, "One", climits[0]);
+ if (rc == 0) {
+ KMemBank *fragsBoth;
+
+ rc = OpenMBankFile(&fragsBoth, dir, pid, "Both", climits[1]);
+
+ if (rc == 0) {
+ MemBank *const self = reinterpret_cast<MemBank *>(malloc(sizeof(self)));
+
+ if (self) {
+ self->fragsOne = fragsOne;
+ self->fragsBoth = fragsBoth;
+
+ *rslt = self;
+ return 0;
+ }
+ rc = RC(rcApp, rcFile, rcConstructing, rcMemory, rcExhausted);
+ KMemBankRelease(fragsBoth);
+ }
+ KMemBankRelease(fragsOne);
+ }
+ return rc;
+}
+
+void MemBank_Release(MemBank *const self)
+{
+ KMemBankRelease(self->fragsBoth);
+ KMemBankRelease(self->fragsOne);
+ free(self);
+}
+
+rc_t MemBank_Alloc(MemBank *const self, uint32_t *const Id, size_t const size, bool const clear, bool const longlived)
+{
+ uint64_t id = 0;
+ KMemBank *const mbank = longlived ? self->fragsOne : self->fragsBoth;
+ rc_t rc = KMemBankAlloc(mbank, &id, size, clear);
+ uint32_t const id_out = (uint32_t)((id << 1) + (longlived ? 1 : 0));
+
+ if ((uint64_t)id_out != ((id << 1) + (longlived ? 1 : 0))) {
+ PLOGMSG(klogFatal, (klogFatal, "membank '$(which)': id space overflow", longlived ? "fragsOne" : "fragsBoth"));
+ abort();
+ }
+ Id[0] = id_out;
+ return rc;
+}
+
+rc_t MemBank_Write(MemBank *const self, uint32_t const id, uint64_t const pos, void const *const buffer, size_t const size, size_t *const num_writ)
+{
+ uint64_t const myId = id >> 1;
+ bool const longlived = (id & 1) ? true : false;
+ KMemBank *const mbank = longlived ? self->fragsOne : self->fragsBoth;
+
+ return KMemBankWrite(mbank, myId, pos, buffer, size, num_writ);
+}
+
+rc_t MemBank_Size(MemBank const *const self, uint32_t const id, size_t *const rslt)
+{
+ uint64_t const myId = id >> 1;
+ bool const longlived = (id & 1) ? true : false;
+ KMemBank const *const mbank = longlived ? self->fragsOne : self->fragsBoth;
+ uint64_t size;
+ rc_t const rc = KMemBankSize(mbank, myId, &size);
+
+ *rslt = size;
+ return rc;
+}
+
+rc_t MemBank_Read(MemBank const *const self, uint32_t const id, uint64_t const pos, void *const buffer, size_t const bsize, size_t *const num_read)
+{
+ uint64_t const myId = id >> 1;
+ bool const longlived = (id & 1) ? true : false;
+ KMemBank *const mbank = longlived ? self->fragsOne : self->fragsBoth;
+
+ return KMemBankRead(mbank, myId, pos, buffer, bsize, num_read);
+}
+
+rc_t MemBank_Free(MemBank *const self, uint32_t const id)
+{
+ uint64_t const myId = id >> 1;
+ bool const longlived = (id & 1) ? true : false;
+ KMemBank *const mbank = longlived ? self->fragsOne : self->fragsBoth;
+
+ return KMemBankFree(mbank, myId);
+}
+
+#else
+
+#include <map>
+#include <set>
+#include <vector>
+#include <iostream>
+#include <stdexcept>
+#include <algorithm>
+
+class pmem
+{
+ struct allocation {
+ void *memory;
+ size_t size;
+
+ allocation(int = 0) : memory(0), size(0) {}
+ };
+ typedef std::map<uint32_t, struct allocation> my_map_t;
+ typedef std::set<uint32_t> my_set_t;
+ my_map_t in_use;
+ my_set_t no_use;
+ my_map_t::size_type max_in_use;
+ my_set_t::size_type max_no_use;
+ my_map_t::size_type total_allocs;
+ my_map_t::size_type total_frees;
+
+ void *get(uint32_t const id) const
+ {
+ my_map_t::const_iterator const i = in_use.find(id);
+
+ if (i == in_use.end())
+ throw std::runtime_error("attempt to access invalid id");
+
+ return i->second.memory;
+ }
+public:
+ pmem(int = 0)
+ : max_in_use(0)
+ , max_no_use(0)
+ , total_allocs(0)
+ , total_frees(0)
+ {}
+ ~pmem() {
+ my_map_t::iterator i;
+
+ for (i = in_use.begin(); i != in_use.end(); ++i) {
+ free(i->second.memory);
+ ++total_frees;
+ }
+#if 0
+ std::cerr << "max. used: " << max_in_use << std::endl;
+ std::cerr << "max. free: " << max_no_use << std::endl;
+ std::cerr << "num alloc: " << total_allocs << std::endl;
+ std::cerr << "num frees: " << total_frees << std::endl;
+#endif
+ }
+
+ void Write(uint32_t id, size_t const offset, size_t const size, void const *const data) const
+ {
+ if (offset + size <= this->Size(id)) {
+ char *dst = reinterpret_cast<char *>(get(id)) + offset;
+ char const *src = reinterpret_cast<char const *>(data);
+
+ std::copy(src, src + size, dst);
+ return;
+ }
+ throw std::runtime_error("attempt to write more than was allocated");
+ }
+ uint32_t Alloc(size_t const size, bool const clear = true)
+ {
+ my_map_t::key_type new_key;
+
+ if (no_use.begin() == no_use.end()) {
+ my_map_t::size_type const new_id = in_use.size() + 1;
+
+ if (max_in_use < new_id)
+ max_in_use = new_id;
+
+ new_key = (my_map_t::key_type)new_id;
+ if (new_key < new_id)
+ throw std::runtime_error("pmem overflow");
+ }
+ else {
+ my_set_t::const_iterator const j = no_use.begin();
+
+ new_key = *j;
+ no_use.erase(j);
+ }
+ my_map_t::iterator const i = in_use.insert(my_map_t::value_type(new_key, 0)).first;
+
+ void *const alloc = calloc(1, size);
+ if (alloc) {
+ i->second.memory = alloc;
+ i->second.size = size;
+
+ ++total_allocs;
+ return i->first;
+ }
+ throw std::bad_alloc();
+ }
+ void Free(uint32_t const id)
+ {
+ my_map_t::iterator const i = in_use.find(id);
+
+ if (i == in_use.end())
+ throw std::runtime_error("attempt to free invalid id");
+
+ void *const alloc = i->second.memory;
+
+ no_use.insert(id);
+ if (max_no_use < no_use.size())
+ max_no_use = no_use.size();
+
+ in_use.erase(i);
+ free(alloc);
+ ++total_frees;
+ }
+ size_t Size(uint32_t const id) const
+ {
+ my_map_t::const_iterator const i = in_use.find(id);
+
+ if (i == in_use.end())
+ throw std::runtime_error("attempt to access invalid or freed id");
+
+ return i->second.size;
+ }
+ void const *Read(uint32_t const id) const
+ {
+ return get(id);
+ }
+};
+
+rc_t MemBank_Make(MemBank **bank, struct KDirectory * = 0, int = 0, size_t const * = 0)
+{
+ try {
+ pmem *const rslt = new pmem;
+
+ *bank = reinterpret_cast<MemBank *>(rslt);
+ return 0;
+ }
+ catch (std::bad_alloc const &e) {
+ return RC(rcApp, rcFile, rcConstructing, rcMemory, rcExhausted);
+ }
+ catch (std::exception const &e) {
+ std::cout << e.what() << std::endl;
+ abort();
+ }
+ catch (...) {
+ std::cout << "this is bad!" << std::endl;
+ abort();
+ }
+}
+
+void MemBank_Release(MemBank *const self)
+{
+ delete reinterpret_cast<pmem *>(self);
+}
+
+rc_t MemBank_Alloc(MemBank *const Self, uint32_t *const id, size_t const bytes, bool const clear, bool const longlived)
+{
+ try {
+ pmem *const self = reinterpret_cast<pmem *>(Self);
+
+ *id = self->Alloc(bytes, clear);
+ return 0;
+ }
+ catch (std::bad_alloc const &e) {
+ return RC(rcApp, rcFile, rcAllocating, rcMemory, rcExhausted);
+ }
+ catch (std::exception const &e) {
+ std::cerr << e.what() << std::endl;
+ abort();
+ }
+ catch (...) {
+ std::cout << "this is bad!" << std::endl;
+ abort();
+ }
+}
+
+rc_t MemBank_Write(MemBank *const Self, uint32_t const id, uint64_t const pos, void const *const buffer, size_t const bsize, size_t *const num_writ)
+{
+ try {
+ pmem *const self = reinterpret_cast<pmem *>(Self);
+
+ *num_writ = 0;
+
+ size_t const size = self->Size((uint32_t)id);
+
+ if (pos >= size)
+ return 0;
+
+ size_t const actsize = (bsize + pos > size) ? (size - pos) : size;
+
+ *num_writ = actsize;
+ self->Write(id, pos, actsize, buffer);
+
+ return 0;
+ }
+ catch (std::exception const &e) {
+ std::cerr << e.what() << std::endl;
+ abort();
+ }
+ catch (...) {
+ std::cout << "this is bad!" << std::endl;
+ abort();
+ }
+}
+
+rc_t MemBank_Size(MemBank const *const Self, uint32_t const id, size_t *const size)
+{
+ try {
+ pmem const *const self = reinterpret_cast<pmem const *>(Self);
+
+ *size = self->Size(id);
+ return 0;
+ }
+ catch (std::exception const &e) {
+ std::cerr << e.what() << std::endl;
+ abort();
+ }
+ catch (...) {
+ std::cout << "this is bad!" << std::endl;
+ abort();
+ }
+}
+
+rc_t MemBank_Read(MemBank const *const Self, uint32_t const id, uint64_t const pos, void *const buffer, size_t const bsize, size_t *const num_read)
+{
+ try {
+ pmem const *const self = reinterpret_cast<pmem const *>(Self);
+
+ *num_read = 0;
+
+ size_t const size = self->Size(id);
+ char const *data = reinterpret_cast<char const *>(self->Read(id)) + pos;
+
+ if (pos >= size)
+ return 0;
+
+ size_t const actsize = (bsize + pos > size) ? (size - pos) : size;
+
+ *num_read = actsize;
+
+ char *dst = reinterpret_cast<char *>(buffer);
+ std::copy(data, data + actsize, dst);
+
+ return 0;
+ }
+ catch (std::exception const &e) {
+ std::cerr << e.what() << std::endl;
+ abort();
+ }
+ catch (...) {
+ std::cout << "this is bad!" << std::endl;
+ abort();
+ }
+}
+
+rc_t MemBank_Free(MemBank *const Self, uint32_t const id)
+{
+ try {
+ pmem *const self = reinterpret_cast<pmem *>(Self);
+
+ self->Free(id);
+ return 0;
+ }
+ catch (std::exception const &e) {
+ std::cerr << e.what() << std::endl;
+ abort();
+ }
+ catch (...) {
+ std::cout << "this is bad!" << std::endl;
+ abort();
+ }
+}
+#endif
+
+extern "C" {
+ rc_t MemBankMake(MemBank **bank, struct KDirectory *dir, int pid, size_t const climits[2])
+ {
+ return MemBank_Make(bank, dir, pid, climits);
+ }
+
+ void MemBankRelease(MemBank *const self)
+ {
+ MemBank_Release(self);
+ }
+
+ rc_t MemBankAlloc(MemBank *const Self, uint32_t *const id, size_t const bytes, bool const clear, bool const longlived)
+ {
+ return MemBank_Alloc(Self, id, bytes, clear, longlived);
+ }
+
+ rc_t MemBankWrite(MemBank *const Self, uint32_t const id, uint64_t const pos, void const *const buffer, size_t const bsize, size_t *const num_writ)
+ {
+ return MemBank_Write(Self, id, pos, buffer, bsize, num_writ);
+ }
+
+ rc_t MemBankSize(MemBank const *const Self, uint32_t const id, size_t *const size)
+ {
+ return MemBank_Size(Self, id, size);
+ }
+
+ rc_t MemBankRead(MemBank const *const Self, uint32_t const id, uint64_t const pos, void *const buffer, size_t const bsize, size_t *const num_read)
+ {
+ return MemBank_Read(Self, id, pos, buffer, bsize, num_read);
+ }
+
+ rc_t MemBankFree(MemBank *const Self, uint32_t const id)
+ {
+ return MemBank_Free(Self, id);
+ }
+}
+
diff --git a/tools/fastq-loader/fastq-reader.h b/tools/bam-loader/mem-bank.h
similarity index 64%
copy from tools/fastq-loader/fastq-reader.h
copy to tools/bam-loader/mem-bank.h
index 4fb1f6a..6d9257a 100644
--- a/tools/fastq-loader/fastq-reader.h
+++ b/tools/bam-loader/mem-bank.h
@@ -1,4 +1,4 @@
-/*===========================================================================
+/* ===========================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
@@ -24,27 +24,18 @@
*
*/
-#ifndef _h_fastq_reader_
-#define _h_fastq_reader_
+typedef struct MemBank MemBank;
-#ifndef _h_klib_defs_
-#include <klib/defs.h>
-#endif
+rc_t MemBankMake(MemBank **rslt, struct KDirectory *dir, int pid, size_t const climits[2]);
-#ifdef __cplusplus
-extern "C" {
-#endif
+void MemBankRelease(MemBank *self);
-/*--------------------------------------------------------------------------
- * forwards
- */
-struct KDirectory;
-struct ReaderFile;
+rc_t MemBankAlloc(MemBank *self, uint32_t *id, size_t size, bool clear, bool longlived);
+
+rc_t MemBankWrite(MemBank *self, uint32_t id, uint64_t pos, void const *buffer, size_t size, size_t *num_writ);
-rc_t CC FastqReaderFileMake( const struct ReaderFile **self, const struct KDirectory* dir, const char* file, uint8_t phredOffset, uint8_t phredMax, int8_t defaultReadNumber);
+rc_t MemBankSize(MemBank const *self, uint32_t id, size_t *rslt);
-#ifdef __cplusplus
-}
-#endif
+rc_t MemBankRead(MemBank const *self, uint32_t id, uint64_t pos, void *buffer, size_t bsize, size_t *num_read);
-#endif /* _h_fastq_reader_ */
+rc_t MemBankFree(MemBank *self, uint32_t id);
diff --git a/tools/bam-loader/reference-writer.c b/tools/bam-loader/reference-writer.c
index b57661f..fa6c7de 100644
--- a/tools/bam-loader/reference-writer.c
+++ b/tools/bam-loader/reference-writer.c
@@ -66,6 +66,12 @@ struct overlap_s {
uint32_t max; /* maximum end pos of any alignment that starts before this chunk and ends in this chunk */
};
+struct s_reference_info {
+ unsigned name; /* offset of start of name in ref_names */
+ unsigned id;
+ unsigned lastOffset;
+};
+
extern void ReferenceMgr_DumpConfig(ReferenceMgr const *const self);
rc_t ReferenceInit(Reference *self, const VDBManager *mgr, VDatabase *db)
@@ -79,6 +85,8 @@ rc_t ReferenceInit(Reference *self, const VDBManager *mgr, VDatabase *db)
self->coverage.elem_bits = self->mismatches.elem_bits = self->indels.elem_bits = 32;
self->pri_align.elem_bits = self->sec_align.elem_bits = 64;
self->pri_overlap.elem_bits = self->sec_overlap.elem_bits = sizeof(struct overlap_s) * 8;
+ self->ref_names.elem_bits = 8;
+ self->ref_info.elem_bits = 8 * sizeof(struct s_reference_info);
rc = ReferenceMgr_Make(&self->mgr, db, mgr, ewrefmgr_co_Coverage,
G.refXRefPath, G.inpath,
@@ -129,10 +137,10 @@ rc_t Unsorted(Reference *self) {
#define BAIL_ON_FAIL(STMT) do { rc_t const rc__ = (STMT); if (rc__) return rc__; } while(0)
-static rc_t FlushBuffers(Reference *self, uint64_t upto, bool full, bool final)
+static rc_t FlushBuffers(Reference *self, unsigned upto, bool full, bool final)
{
if (!self->out_of_order && upto > 0) {
- size_t offset = 0;
+ unsigned offset = 0;
unsigned *const miss = (unsigned *)self->mismatches.base;
unsigned *const indel = (unsigned *)self->indels.base;
unsigned *const cov = (unsigned *)self->coverage.base;
@@ -142,7 +150,7 @@ static rc_t FlushBuffers(Reference *self, uint64_t upto, bool full, bool final)
while ((self->curPos + offset + (full ? 0 : G.maxSeqLen)) <= upto) {
ReferenceSeqCoverage data;
- uint64_t const curPos = self->curPos + offset;
+ unsigned const curPos = self->curPos + offset;
unsigned const n = self->endPos > (curPos + G.maxSeqLen) ?
G.maxSeqLen : (self->endPos - curPos);
unsigned const m = curPos + n > upto ? upto - curPos : n;
@@ -195,7 +203,7 @@ static rc_t FlushBuffers(Reference *self, uint64_t upto, bool full, bool final)
++chunk;
}
if (!final && offset > 0) {
- unsigned const newChunkCount = self->pri_overlap.elem_count - chunk;
+ unsigned const newChunkCount = (unsigned)self->pri_overlap.elem_count - chunk;
unsigned const newBaseCount = self->endPos - self->curPos - offset;
memmove(self->pri_overlap.base, pri_overlap + chunk, newChunkCount * sizeof(pri_overlap[0]));
@@ -213,51 +221,193 @@ static rc_t FlushBuffers(Reference *self, uint64_t upto, bool full, bool final)
return 0;
}
-rc_t ReferenceSetFile(Reference *self, const char id[],
- uint64_t length, uint8_t const md5[16])
+static int str__cmp(char const A[], char const B[])
{
- const ReferenceSeq *rseq;
- unsigned n;
+ unsigned i;
- for (n = 0; ; ++n) {
- if (self->last_id[n] != id[n])
- break;
- if (self->last_id[n] == 0 && id[n] == 0)
+ for (i = 0; ; ++i) {
+ int const a = A[i];
+ int const b = B[i];
+
+ if (a != b)
+ return a - b;
+ if (a == 0)
return 0;
}
- while (id[n])
- ++n;
- if (n >= sizeof(self->last_id))
- return RC(rcApp, rcTable, rcAccessing, rcParam, rcTooLong);
+}
+
+static int str__equal(char const A[], char const B[])
+{
+ unsigned i;
- BAIL_ON_FAIL(FlushBuffers(self, self->length, true, true));
- BAIL_ON_FAIL(ReferenceMgr_GetSeq(self->mgr, &rseq, id));
+ for (i = 0; ; ++i) {
+ int const a = A[i];
+ int const b = B[i];
+
+ if (a != b)
+ return 0;
+ if (a == 0)
+ return 1;
+ }
+}
+
+static unsigned str__len(char const A[])
+{
+ unsigned i;
+
+ for (i = 0; ; ++i) {
+ int const a = A[i];
+
+ if (a == 0)
+ return i;
+ }
+}
+
+static unsigned bsearch_name(char const qry[], char const names[],
+ unsigned const count,
+ struct s_reference_info const refInfo[],
+ int found[])
+{
+ unsigned f = 0;
+ unsigned e = count;
+
+ while (f < e) {
+ unsigned const m = f + ((e - f) >> 1);
+ char const *const name = &names[refInfo[m].name];
+ int const diff = str__cmp(qry, name);
+
+ if (diff < 0)
+ e = m;
+ else if (diff > 0)
+ f = m + 1;
+ else {
+ found[0] = 1;
+ return m;
+ }
+ }
+ return f;
+}
+
+static struct s_reference_info s_reference_info_make(unsigned const name, unsigned const id)
+{
+ struct s_reference_info rslt;
+
+ rslt.name = name;
+ rslt.id = id;
+ rslt.lastOffset = 0;
+
+ return rslt;
+}
+
+static unsigned GetLastOffset(Reference const *const self)
+{
+ if (self->last_id < self->ref_info.elem_count) {
+ struct s_reference_info const *const refInfoBase = self->ref_info.base;
+ return refInfoBase[self->last_id].lastOffset;
+ }
+ return 0;
+}
+
+static void SetLastOffset(Reference *const self, unsigned const newValue)
+{
+ if (self->last_id < self->ref_info.elem_count) {
+ struct s_reference_info *const refInfoBase = self->ref_info.base;
+ refInfoBase[self->last_id].lastOffset = newValue;
+ }
+}
+
+rc_t ReferenceSetFile(Reference *const self, char const id[],
+ uint64_t const length, uint8_t const md5[16],
+ bool *const shouldUnmap,
+ bool *const wasRenamed)
+{
+ ReferenceSeq const *rseq;
+ int found = 0;
+ unsigned at = 0;
+ char const *actid = id;
+
+ if (!G.allowMultiMapping) {
+ assert(actid != NULL);
+ }
+ if (self->last_id < self->ref_info.elem_count) {
+ struct s_reference_info const *const refInfoBase = self->ref_info.base;
+ struct s_reference_info const refInfo = refInfoBase[self->last_id];
+ char const *const nameBase = self->ref_names.base;
+ char const *const last = nameBase + refInfo.id;
+
+ if (str__equal(id, last)) {
+ return 0;
+ }
+ }
+ BAIL_ON_FAIL(FlushBuffers(self, self->length, true, true));
+ BAIL_ON_FAIL(ReferenceMgr_GetSeq(self->mgr, &rseq, id, shouldUnmap, G.allowMultiMapping, wasRenamed));
+
if (self->rseq)
ReferenceSeq_Release(self->rseq);
self->rseq = rseq;
-
- memcpy(self->last_id, id, n + 1);
+
+ if (*wasRenamed)
+ ReferenceSeq_GetID(rseq, &actid);
+
+ at = bsearch_name(actid, self->ref_names.base, self->ref_info.elem_count, self->ref_info.base, &found);
+ if (!found) {
+ unsigned const len1 = str__len(actid);
+ unsigned const len2 = str__len(id);
+ unsigned const name_at = self->ref_names.elem_count;
+ unsigned const id_at = *wasRenamed ? (name_at + len1 + 1) : name_at;
+ struct s_reference_info const new_elem = s_reference_info_make(name_at, id_at);
+ rc_t const rc = KDataBufferResize(&self->ref_names, name_at + len1 + 1 + (*wasRenamed ? (len2 + 1) : 0));
+
+ if (rc)
+ return rc;
+ else {
+ unsigned const count = (unsigned)self->ref_info.elem_count;
+ rc_t const rc = KDataBufferResize(&self->ref_info, count + 1);
+ struct s_reference_info *const refInfoBase = self->ref_info.base;
+
+ if (rc)
+ return rc;
+
+ memmove(((char *)self->ref_names.base) + name_at, actid, len1 + 1);
+ if (*wasRenamed)
+ memmove(((char *)self->ref_names.base) + id_at, id, len2 + 1);
+
+ memmove(refInfoBase + at + 1, refInfoBase + at, (count - at) * sizeof(*refInfoBase));
+ refInfoBase[at] = new_elem;
+ }
+ (void)PLOGMSG(klogInfo, (klogInfo, "Processing Reference '$(id)'", "id=%s", id));
+ if (*wasRenamed)
+ (void)PLOGMSG(klogInfo, (klogInfo, "Reference '$(id)' was renamed to '$(actid)'", "id=%s,actid=%s", id, *actid));
+ }
+ else if (!self->out_of_order)
+ Unsorted(self);
+
+ self->last_id = at;
self->curPos = self->endPos = 0;
- self->length = length;
- self->lastOffset = 0;
+ self->length = (unsigned)length;
KDataBufferResize(&self->pri_overlap, 0);
KDataBufferResize(&self->sec_overlap, 0);
- if(!self->out_of_order) (void)PLOGMSG(klogInfo, (klogInfo, "Processing Reference '$(id)'", "id=%s", id));
-
return 0;
}
-rc_t ReferenceVerify(Reference const *self, char const id[], uint64_t length, uint8_t const md5[16])
+rc_t ReferenceVerify(Reference const *const self,
+ char const id[],
+ uint64_t const length,
+ uint8_t const md5[16])
{
- return ReferenceMgr_Verify(self->mgr, id, length, md5);
+ bool wasRenamed = false;
+ return ReferenceMgr_Verify(self->mgr, id, (unsigned)length, md5, G.allowMultiMapping, &wasRenamed);
}
rc_t ReferenceGet1stRow(Reference const *self, int64_t *refID, char const refName[])
{
- const ReferenceSeq* rseq;
- rc_t rc = ReferenceMgr_GetSeq(self->mgr, &rseq, refName);
+ ReferenceSeq const *rseq;
+ bool shouldUnmap = false;
+ bool wasRenamed = false;
+ rc_t rc = ReferenceMgr_GetSeq(self->mgr, &rseq, refName, &shouldUnmap, G.allowMultiMapping, &wasRenamed);
+
if( rc == 0 ) {
rc = ReferenceSeq_Get1stRow(rseq, refID);
ReferenceSeq_Release(rseq);
@@ -294,7 +444,7 @@ rc_t ReferenceAddCoverage(Reference *const self,
}
if ((refEnd - self->curPos) / G.maxSeqLen >= self->pri_overlap.elem_count) {
unsigned const chunks = (refEnd - self->curPos) / G.maxSeqLen + 1;
- unsigned const end = self->pri_overlap.elem_count;
+ unsigned const end = (unsigned)self->pri_overlap.elem_count;
BAIL_ON_FAIL(KDataBufferResize(&self->pri_overlap, chunks));
BAIL_ON_FAIL(KDataBufferResize(&self->sec_overlap, chunks));
@@ -394,7 +544,7 @@ rc_t ReferenceRead(Reference *self, AlignmentRecord *data, uint64_t const pos,
if (!G.acceptNoMatch && data->data.ref_len == 0)
return RC(rcApp, rcFile, rcReading, rcConstraint, rcViolated);
- if (!self->out_of_order && pos < self->lastOffset) {
+ if (!self->out_of_order && pos < GetLastOffset(self)) {
return Unsorted(self);
}
if (!self->out_of_order) {
@@ -402,7 +552,7 @@ rc_t ReferenceRead(Reference *self, AlignmentRecord *data, uint64_t const pos,
unsigned nmatch;
unsigned indels;
- self->lastOffset = data->data.effective_offset;
+ SetLastOffset(self, data->data.effective_offset);
GetCounts(data, seqLen, &nmatch, &nmis, &indels);
*matches = nmatch;
@@ -456,6 +606,8 @@ rc_t ReferenceWhack(Reference *self, bool commit)
KDataBufferWhack(&self->coverage);
KDataBufferWhack(&self->pri_overlap);
KDataBufferWhack(&self->sec_overlap);
+ KDataBufferWhack(&self->ref_names);
+ KDataBufferWhack(&self->ref_info);
if (self->rseq)
rc = ReferenceSeq_Release(self->rseq);
if (self->out_of_order) {
diff --git a/tools/bam-loader/reference-writer.h b/tools/bam-loader/reference-writer.h
index 64e7ade..efb61f7 100644
--- a/tools/bam-loader/reference-writer.h
+++ b/tools/bam-loader/reference-writer.h
@@ -38,10 +38,11 @@ typedef struct s_reference {
const ReferenceMgr *mgr;
const ReferenceSeq *rseq;
int64_t lastRefId;
- int32_t lastOffset;
unsigned curPos;
unsigned endPos;
unsigned length;
+ unsigned last_id; /* == ref_info.elem_count if no last id */
+
KDataBuffer coverage;
KDataBuffer mismatches;
KDataBuffer indels;
@@ -49,12 +50,17 @@ typedef struct s_reference {
KDataBuffer sec_align;
KDataBuffer pri_overlap;
KDataBuffer sec_overlap;
+ KDataBuffer ref_names;
+ KDataBuffer ref_info;
+
bool out_of_order;
- char last_id[256];
} Reference;
rc_t ReferenceInit(Reference *self, const VDBManager *mgr, VDatabase *db);
-rc_t ReferenceSetFile(Reference *self, const char *id, uint64_t length, uint8_t const md5[16]);
+rc_t ReferenceSetFile(Reference *self, char const id[],
+ uint64_t length, uint8_t const md5[16],
+ bool *shouldUnmap,
+ bool *wasRenamed);
rc_t ReferenceVerify(Reference const *self, char const id[], uint64_t length, uint8_t const md5[16]);
rc_t ReferenceGet1stRow(Reference const *self, int64_t *refID, char const refName[]);
rc_t ReferenceAddAlignId(Reference *self,
diff --git a/tools/bam-loader/sequence-writer.c b/tools/bam-loader/sequence-writer.c
index ec0f22e..5002a5b 100644
--- a/tools/bam-loader/sequence-writer.c
+++ b/tools/bam-loader/sequence-writer.c
@@ -106,15 +106,18 @@ rc_t SequenceWriteRecord(Sequence *self,
}
for (i = 0; i != nreads; ++i) {
- alcnt[i] = rec->aligned[i] ? 1 : 0;
- readLen[i] = rec->readLen[i];
- readStart[i] = rec->readStart[i];
- readType[i] = SRA_READ_TYPE_BIOLOGICAL | (rec->orientation[i] ?
- SRA_READ_TYPE_REVERSE:
- SRA_READ_TYPE_FORWARD
- );
- readFilter[i] = isDup ? SRA_READ_FILTER_CRITERIA
- : rec->is_bad[i] ? SRA_READ_FILTER_REJECT : SRA_READ_FILTER_PASS;
+ int const count = rec->aligned[i] ? 1 : 0;
+ int const len = rec->readLen[i];
+ int const start = rec->readStart[i];
+ int const type = len == 0 ? SRA_READ_TYPE_TECHNICAL : SRA_READ_TYPE_BIOLOGICAL;
+ int const dir = len == 0 ? 0 : rec->orientation[i] ? SRA_READ_TYPE_REVERSE : SRA_READ_TYPE_FORWARD;
+ int const filter = isDup ? SRA_READ_FILTER_CRITERIA : rec->is_bad[i] ? SRA_READ_FILTER_REJECT : SRA_READ_FILTER_PASS;
+
+ alcnt[i] = count;
+ readLen[i] = len;
+ readStart[i] = start;
+ readType[i] = type | dir;
+ readFilter[i] = filter;
}
memset(&data, 0, sizeof(data));
@@ -205,14 +208,13 @@ rc_t SequenceUpdateAlignData(Sequence *self, int64_t rowId, unsigned nreads,
void SequenceWhack(Sequence *self, bool commit) {
uint64_t dummyRows;
- rc_t rc;
VDatabaseRelease(self->db);
if (self->tbl == NULL)
return;
- rc = TableWriterSeq_Whack(self->tbl, commit, &dummyRows);
+ (void)TableWriterSeq_Whack(self->tbl, commit, &dummyRows);
}
/* MARK: SequenceRecord Object */
diff --git a/tools/cache-mgr/cache-mgr.c b/tools/cache-mgr/cache-mgr.c
index 671280d..fd5734f 100644
--- a/tools/cache-mgr/cache-mgr.c
+++ b/tools/cache-mgr/cache-mgr.c
@@ -63,6 +63,7 @@ static const char * clear_usage[] = { "clear cache", NULL };
static const char * enable_usage[] = { "enable repository [user/site/rem]", NULL };
static const char * disable_usage[] = { "disable repository [user/site/rem]", NULL };
static const char * detail_usage[] = { "show detailed report", NULL };
+static const char * tstzero_usage[] = { "test for zero blocks (for report function)", NULL };
static const char * urname_usage[] = { "restrict to this user-repository", NULL };
static const char * max_rem_usage[] = { "remove until reached that many bytes", NULL };
static const char * rem_dir_usage[] = { "remove directories, not only files", NULL };
@@ -97,12 +98,15 @@ static const char * rem_dir_usage[] = { "remove directories, not only files", NU
#define OPTION_REMDIR "remove-dirs"
#define ALIAS_REMDIR "i"
+#define OPTION_TSTZERO "test-zero"
+#define ALIAS_TSTZERO "z"
OptDef ToolOptions[] =
{
{ OPTION_CREPORT, ALIAS_CREPORT, NULL, report_usage, 1, false, false },
{ OPTION_RREPORT, ALIAS_RREPORT, NULL, rreport_usage, 1, false, false },
{ OPTION_DETAIL, ALIAS_DETAIL, NULL, detail_usage, 1, false, false },
+ { OPTION_TSTZERO, ALIAS_TSTZERO, NULL, tstzero_usage, 1, false, false },
{ OPTION_UNLOCK, ALIAS_UNLOCK, NULL, unlock_usage, 1, false, false },
{ OPTION_CLEAR, ALIAS_CLEAR, NULL, clear_usage, 1, false, false },
{ OPTION_MAXREM, ALIAS_MAXREM, NULL, max_rem_usage, 1, true, false },
@@ -208,6 +212,7 @@ typedef struct tool_options
KRepCategory category;
bool detailed;
+ bool tstzero;
bool remove_dirs;
} tool_options;
@@ -283,7 +288,7 @@ static KRepCategory get_repo_select( const Args * args, const char * name )
else if ( count > 0 )
{
const char * s = NULL;
- rc = ArgsOptionValue( args, name, 0, &s );
+ rc = ArgsOptionValue( args, name, 0, (const void **)&s );
if ( rc == 0 && s != NULL )
{
if ( string_cmp_1 ( s, "user" ) )
@@ -310,7 +315,7 @@ static rc_t get_user_repo_name( const Args * args, const char ** name )
else if ( count > 0 )
{
const char * s = NULL;
- rc = ArgsOptionValue( args, OPTION_URNAME, 0, &s );
+ rc = ArgsOptionValue( args, OPTION_URNAME, 0, (const void **)&s );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
@@ -335,7 +340,7 @@ static rc_t get_max_remove( const Args * args, uint64_t * max_rem )
else if ( count > 0 )
{
const char * s = NULL;
- rc = ArgsOptionValue( args, OPTION_MAXREM, 0, &s );
+ rc = ArgsOptionValue( args, OPTION_MAXREM, 0, (const void **)&s );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
@@ -380,7 +385,7 @@ static rc_t get_tool_options( Args * args, tool_options * options )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *value = NULL;
- rc = ArgsParamValue( args, idx, &value );
+ rc = ArgsParamValue( args, idx, (const void **)&value );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
@@ -393,6 +398,7 @@ static rc_t get_tool_options( Args * args, tool_options * options )
options->main_function = tf_unknown;
options->detailed = get_bool_option( args, OPTION_DETAIL );
+ options->tstzero = get_bool_option( args, OPTION_TSTZERO );
options->remove_dirs = get_bool_option( args, OPTION_REMDIR );
if ( get_bool_option( args, OPTION_CREPORT ) )
@@ -576,8 +582,12 @@ typedef struct report_data
static rc_t on_report_cache_file( visit_ctx * obj )
{
rc_t rc = 0;
- uint64_t file_size = 0, used_size = 0;
+ uint64_t file_size = 0;
+ uint64_t used_size = 0;
+ uint64_t checked_blocks = 0;
+ uint64_t empty_blocks = 0;
float completeness = 0.0;
+
bool locked = false;
report_data * data = obj->data;
@@ -601,18 +611,29 @@ static rc_t on_report_cache_file( visit_ctx * obj )
}
else
{
- rc = GetCacheCompleteness( f, &completeness );
+ rc = GetCacheCompleteness( f, &completeness, &used_size ); /* libs/kfs/cacheteefile.c */
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"GetCacheCompleteness( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
- {
- used_size = file_size * ( completeness / 100 );
- data->used_file_size += used_size;
- }
- }
+ {
+ data->used_file_size += used_size;
+ }
+
+ if ( rc == 0 && obj->options->tstzero )
+ {
+ rc = Has_Cache_Zero_Blocks( f, &checked_blocks, &empty_blocks ); /* libs/kfs/cacheteefile.c */
+ if ( rc != 0 )
+ {
+ PLOGERR( klogErr, ( klogErr, rc,
+ "Has_Cache_Zero_Blocks( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
+ }
+ }
+
+ KFileRelease( f );
+ }
}
if ( rc == 0 )
{
@@ -623,12 +644,19 @@ static rc_t on_report_cache_file( visit_ctx * obj )
if ( rc == 0 && obj->options->detailed )
{
if ( locked )
- rc = KOutMsg( "%s complete by %.02f %% [%,u of %,u]bytes locked\n",
+ rc = KOutMsg( "%s complete by %.02f %% [%,lu of %,lu]bytes locked\n",
obj->path, completeness, used_size, file_size );
else
- rc = KOutMsg( "%s complete by %.02f %% [%,u of %,u]bytes\n",
+ rc = KOutMsg( "%s complete by %.02f %% [%,lu of %,lu]bytes\n",
obj->path, completeness, used_size, file_size );
}
+
+ if ( rc == 0 && obj->options->tstzero )
+ {
+ rc = KOutMsg( "%s has %lu blocks set in bitmap where %lu are empty\n",
+ obj->path, checked_blocks, empty_blocks );
+ }
+
return rc;
}
@@ -651,7 +679,7 @@ static rc_t on_report_full_file( visit_ctx * obj )
data->file_size += file_size;
data->used_file_size += file_size;
if ( obj->options->detailed )
- rc = KOutMsg( "%s complete file of %,u bytes\n", obj->path, file_size );
+ rc = KOutMsg( "%s complete file of %,lu bytes\n", obj->path, file_size );
}
return rc;
}
@@ -700,9 +728,9 @@ static rc_t perform_report( visit_ctx * octx )
if ( rc == 0 )
rc = KOutMsg( "%,u complete file(s)\n", data.full_count );
if ( rc == 0 )
- rc = KOutMsg( "%,u bytes in cached files\n", data.file_size );
+ rc = KOutMsg( "%,lu bytes in cached files\n", data.file_size );
if ( rc == 0 )
- rc = KOutMsg( "%,u bytes used in cached files\n", data.used_file_size );
+ rc = KOutMsg( "%,lu bytes used in cached files\n", data.used_file_size );
if ( rc == 0 )
rc = KOutMsg( "%,u lock files\n", data.lock_count );
@@ -953,7 +981,7 @@ static rc_t on_clear_path( visit_ctx * obj )
else
{
if ( obj->options->detailed )
- rc = KOutMsg( "FILE: '%s' removed (%,u bytes)\n", obj->path, file_size );
+ rc = KOutMsg( "FILE: '%s' removed (%,lu bytes)\n", obj->path, file_size );
data->removed_files++;
data->removed_size += file_size;
if ( obj->options->max_remove > 0 && data->removed_size >= obj->options->max_remove )
@@ -988,7 +1016,7 @@ static rc_t perform_clear( visit_ctx * octx )
if ( rc == 0 )
rc = KOutMsg( "%,u directories removed\n", data.removed_directories );
if ( rc == 0 )
- rc = KOutMsg( "%,u bytes removed\n", data.removed_size );
+ rc = KOutMsg( "%,lu bytes removed\n", data.removed_size );
return rc;
}
diff --git a/tools/cache-mgr/cache-mgr.vers b/tools/cache-mgr/cache-mgr.vers
index 8e8299d..35d16fb 100644
--- a/tools/cache-mgr/cache-mgr.vers
+++ b/tools/cache-mgr/cache-mgr.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/cache-mgr/cache-mgr.vers.h b/tools/cache-mgr/cache-mgr.vers.h
index 04e8d7e..7b7372c 100644
--- a/tools/cache-mgr/cache-mgr.vers.h
+++ b/tools/cache-mgr/cache-mgr.vers.h
@@ -1 +1 @@
-#define CACHE_MGR_VERS 0x02040002
+#define CACHE_MGR_VERS 0x02050007
diff --git a/tools/ccextract/Makefile b/tools/ccextract/Makefile
index 2872c36..72d418c 100644
--- a/tools/ccextract/Makefile
+++ b/tools/ccextract/Makefile
@@ -32,11 +32,7 @@ include $(TOP)/build/Makefile.env
INT_TOOLS =
-EXT_TOOLS =
-
-ifeq (linux,$(OS))
- EXT_TOOLS += ccextract
-endif
+EXT_TOOLS = ccextract
ALL_TOOLS = \
@@ -46,12 +42,24 @@ ALL_TOOLS = \
#-------------------------------------------------------------------------------
# outer targets
#
+ifeq (1,$(HAVE_XML2))
+
all std: vers-includes
@ $(MAKE_CMD) $(TARGDIR)/std
$(ALL_TOOLS): vers-includes
@ $(MAKE_CMD) $(BINDIR)/$@
+else
+
+all std: $(ALL_TOOLS)
+
+$(ALL_TOOLS):
+ @ echo "NOTE - $(@F) cannot be built:" \
+ "It requires our internal library 'libkxml'" \
+ "which requires 'libxml2' and its development headers."
+endif
+
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
@@ -91,12 +99,7 @@ CCEXTRACT_LIB = \
-lkapp \
-lkxfs \
-lkxml \
- -sncbi-vdb \
-
+ -sncbi-vdb
$(BINDIR)/ccextract: $(CCEXTRACT_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(CCEXTRACT_LIB)
-
-
-
-
diff --git a/tools/ccextract/ccextract.c b/tools/ccextract/ccextract.c
index a38386c..b6fdf4a 100644
--- a/tools/ccextract/ccextract.c
+++ b/tools/ccextract/ccextract.c
@@ -111,13 +111,15 @@ static
const char * first_usage[] =
{
"The path to a archive file ",
+ NULL
};
static
const char * second_usage[] =
{
"A file by path or ID to extract",
- "If none are given all files are extracted"
+ "If none are given all files are extracted",
+ NULL
};
rc_t CC Usage (const Args * args)
@@ -288,11 +290,11 @@ void CC extnode_whack (BSTNode * n, void * data)
/* ----------------------------------------------------------------------
*/
static
-int CC extnode_sort (const BSTNode * item, const BSTNode * n)
+int64_t CC extnode_sort (const BSTNode * item, const BSTNode * n)
{
const extnode * l;
const extnode * r;
- int ii;
+ int64_t ii;
l = (const extnode *)item;
r = (const extnode *)n;
@@ -1175,7 +1177,7 @@ rc_t KMain ( int argc, char *argv [] )
}
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_XML, 0, &options.xmlstr);
+ rc = ArgsOptionValue (args, OPTION_XML, 0, (const void **)&options.xmlstr);
if (rc)
{
LOGERR (klogFatal, rc, "failed to get XML value");
@@ -1198,7 +1200,7 @@ rc_t KMain ( int argc, char *argv [] )
}
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_DIR, 0, &options.dirstr);
+ rc = ArgsOptionValue (args, OPTION_DIR, 0, (const void **)&options.dirstr);
if (rc)
{
LOGERR (klogFatal, rc, "failed to get directory value");
@@ -1235,7 +1237,7 @@ rc_t KMain ( int argc, char *argv [] )
VectorInit (&options.pathstr, 0, block);
}
- rc = ArgsParamValue (args, 0, &options.arcstr);
+ rc = ArgsParamValue (args, 0, (const void **)&options.arcstr);
if (rc)
LOGERR (klogFatal, rc, "failed to retrieve archive parameter");
else
@@ -1252,7 +1254,7 @@ rc_t KMain ( int argc, char *argv [] )
for (ix = 1; ix < pcount; ++ix)
{
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
{
PLOGERR (klogFatal,
diff --git a/tools/ccextract/ccextract.vers b/tools/ccextract/ccextract.vers
index 8e8299d..35d16fb 100644
--- a/tools/ccextract/ccextract.vers
+++ b/tools/ccextract/ccextract.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/cg-dump/cg-dump.c b/tools/cg-dump/cg-dump.c
deleted file mode 100644
index ede1b40..0000000
--- a/tools/cg-dump/cg-dump.c
+++ /dev/null
@@ -1,1563 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "cg-dump.vers.h"
-
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/table.h>
-#include <vdb/cursor.h>
-#include <vdb/database.h>
-
-#include <kdb/manager.h> /* because of path-types */
-#include <kdb/meta.h> /* because of exploring meta-data */
-
-#include <kfs/bzip.h>
-#include <kfs/gzip.h>
-#include <kfs/buffile.h>
-#include <kfs/file.h>
-#include <kfs/directory.h>
-
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <kapp/queue-file.h>
-
-#include <klib/log.h>
-#include <klib/out.h>
-#include <klib/status.h>
-#include <klib/text.h>
-#include <klib/time.h>
-#include <klib/printf.h>
-#include <klib/container.h>
-#include <klib/rc.h>
-
-#include "num-gen.h"
-#include "progressbar.h"
-#include "sg_lookup.h"
-#include "last_rowid.h"
-
-#include <os-native.h>
-#include <sysalloc.h>
-#include <strtol.h> /* strtou64 */
-#include <string.h> /* memset */
-
-#define CURSOR_CACHE_SIZE 256*1024*1024
-#define DEFAULT_CUTOFF 30000000
-
-const char UsageDefaultName[] = "cg-dump";
-
-static const char * rows_usage[] = { "rows to dump (if ommited: all rows)", NULL };
-static const char * cutoff_usage[] = { "how many spots max. per output-file", NULL };
-static const char * force_usage[] = { "force to overwrite output directory if it already exists", NULL };
-static const char * cache_usage[] = { "size of cursor-cache", NULL };
-static const char * comp_usage[] = { "output-compression, 'none', 'gzip', 'bzip'(dflt) or 'null'", NULL };
-static const char * prog_usage[] = { "show progressbar", NULL };
-static const char * lib_usage[] = { "LIBRARY-value for output-file-header", NULL };
-static const char * sample_usage[] = { "SAMPLE-value for output-file-header", NULL };
-static const char * prefix_usage[] = { "name-prefix for output-files", NULL };
-static const char * wbuf_usage[] = { "use write-buffering of this size", NULL };
-static const char * queue_usage[] = { "use background threads for writing", NULL };
-static const char * qbytes_usage[] = { "background producer limit", NULL };
-static const char * qblock_usage[] = { "background blocksize", NULL };
-static const char * qtimeout_usage[] = { "timeout for background writers", NULL };
-static const char * lrowid_usage[] = { "find the highest row-id in the out-dir", NULL };
-
-#define OPTION_ROWS "rows"
-#define OPTION_CUTOFF "cutoff"
-#define OPTION_FORCE "force"
-#define OPTION_CACHE "cache"
-#define OPTION_COMP "compress"
-#define OPTION_PROG "progress"
-#define OPTION_LIB "lib"
-#define OPTION_SAMPLE "sample"
-#define OPTION_PREFIX "prefix"
-#define OPTION_WBUF "writebuff"
-#define OPTION_QUEUE "queued"
-#define OPTION_QUEUE_BYTES "queue-bytes"
-#define OPTION_QUEUE_BLOCK "queue-block"
-#define OPTION_QUEUE_TIMEOUT "queue-timeout"
-#define OPTION_LAST_ROWID "last-rowid"
-
-#define ALIAS_ROWS "R"
-#define ALIAS_CUTOFF "C"
-#define ALIAS_FORCE "f"
-#define ALIAS_CACHE "a"
-#define ALIAS_COMP "s"
-#define ALIAS_PROG "p"
-#define ALIAS_WBUF "w"
-#define ALIAS_QUEUE "q"
-
-OptDef DumpOptions[] =
-{
- { OPTION_ROWS, ALIAS_ROWS, NULL, rows_usage, 1, true, false },
- { OPTION_CUTOFF, ALIAS_CUTOFF, NULL, cutoff_usage, 1, true, false },
- { OPTION_FORCE, ALIAS_FORCE, NULL, force_usage, 1, false, false },
- { OPTION_CACHE, ALIAS_CACHE, NULL, cache_usage, 1, true, false },
- { OPTION_COMP, ALIAS_COMP, NULL, comp_usage, 1, true, false },
- { OPTION_PROG, ALIAS_PROG, NULL, prog_usage, 1, false, false },
- { OPTION_LIB, NULL, NULL, lib_usage, 1, true, false },
- { OPTION_SAMPLE, NULL, NULL, sample_usage, 1, true, false },
- { OPTION_PREFIX, NULL, NULL, prefix_usage, 1, true, false },
- { OPTION_WBUF, ALIAS_WBUF, NULL, wbuf_usage, 1, true, false },
- { OPTION_QUEUE, ALIAS_QUEUE, NULL, queue_usage, 1, false, false },
- { OPTION_QUEUE_BYTES, NULL, NULL, qbytes_usage, 1, true, false },
- { OPTION_QUEUE_BLOCK, NULL, NULL, qblock_usage, 1, true, false },
- { OPTION_QUEUE_TIMEOUT, NULL, NULL, qtimeout_usage, 1, true, false },
- { OPTION_LAST_ROWID, NULL, NULL, lrowid_usage, 1, false, false }
-};
-
-
-rc_t CC UsageSummary ( const char * progname )
-{
- return KOutMsg ( "\nUsage:\n %s <input-path or accession> <output-path> [options]\n\n",
- progname );
-}
-
-rc_t CC Usage ( const Args * args )
-{
- const char * progname = UsageDefaultName;
- const char * fullpath = UsageDefaultName;
- uint32_t n, i;
- rc_t rc;
-
- if ( args == NULL )
- rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
- else
- rc = ArgsProgram ( args, &fullpath, &progname );
-
- if ( rc )
- progname = fullpath = UsageDefaultName;
-
- UsageSummary ( progname );
-
- KOutMsg ( "Options:\n" );
-
- n = ( sizeof( DumpOptions ) / sizeof( DumpOptions[ 0 ] ) );
- for( i = 0; i < n; i++ )
- {
- if ( DumpOptions[ i ].help != NULL )
- {
- HelpOptionLine( DumpOptions[ i ].aliases, DumpOptions[ i ].name,
- NULL, DumpOptions[ i ].help );
- }
- }
- KOutMsg( "\n" );
-
- HelpOptionsStandard ();
-
- HelpVersion ( fullpath, KAppVersion() );
-
- return rc;
-}
-
-
-/* Version EXTERN
- * return 4-part version code: 0xMMmmrrrr, where
- * MM = major release
- * mm = minor release
- * rrrr = bug-fix release
- */
-ver_t CC KAppVersion ( void )
-{
- return CG_DUMP_VERS;
-}
-
-
-enum output_compression
-{
- oc_none = 0, /* do not compress output */
- oc_gzip, /* compress output with gzip */
- oc_bzip, /* compress output with bzip2 */
- oc_null /* do not create output at all */
-};
-
-
-typedef struct cg_dump_opts
-{
- uint64_t cutoff; /* how many bytes of output before new sub-file starts */
- uint64_t cursor_cache; /* size of cursor-cache */
- uint32_t first_chunk; /* starting number of chunks */
- size_t wbuff_size; /* write-buffer size */
- size_t qbytes; /* producer limit for background threads */
- size_t qblock; /* block-size for background threads */
- uint32_t qtimeout; /* timeout for background writers */
-
- bool overwrite; /* if output dir exists, overwrite? */
- bool show_progress; /* show progressbar */
- bool use_queue; /* use queued writer */
- bool get_last_rowid; /* discover the last row-id written */
-
- enum output_compression comp; /* what type of output compression? */
-
- const char * lib; /* LIBRARY - value for output-file-header */
- const char * sample; /* SAMPLE - value for output-file-header */
- const char * prefix; /* eventual prefix for output-filename */
-} cg_dump_opts;
-
-
-typedef struct lane
-{
- BSTNode node;
-
- const String * name;
- KFile * reads;
-/* KFile * mappings; */
- uint64_t write_pos;
- uint64_t spot_count;
- uint32_t chunk;
-} lane;
-
-
-typedef struct cg_dump_ctx
-{
- num_gen * rows;
- KDirectory * dir;
- KDirectory * out_dir;
- const VCursor * seq_cur;
-/* const VCursor * prim_cur; */
- BSTree lanes;
- struct sg_lookup * lookup;
-
- progressbar * progress;
- uint8_t fract_digits;
-
- uint32_t seq_read_idx;
- uint32_t seq_qual_idx;
- uint32_t seq_sg_idx;
-/* uint32_t seq_read_len_idx;
- uint32_t seq_prim_id_idx;
-
- uint32_t prim_cigar_idx;
- uint32_t prim_refname_idx;
- uint32_t prim_refpos_idx; */
-
- const char * dst;
-} cg_dump_ctx;
-
-
-const char * s_Database = "Database";
-const char * s_Table = "Table";
-const char * s_PrereleaseTbl = "Prerelease Table";
-const char * s_Column = "Column";
-const char * s_Index = "Index";
-const char * s_NotFound = "not found";
-const char * s_BadPath = "bad path";
-const char * s_File = "File";
-const char * s_Dir = "Dir";
-const char * s_CharDev = "CharDev";
-const char * s_BlockDev = "BlockDev";
-const char * s_FIFO = "FIFO";
-const char * s_ZombieFile = "ZombieFile";
-const char * s_Dataset = "Dataset";
-const char * s_Datatype = "Datatype";
-const char * s_Unknown = "Unknown";
-
-
-const char * pathtype_2_pchar( int path_type )
-{
- const char * res = s_Unknown;
- switch ( path_type )
- {
- case kptDatabase : res = s_Database; break;
- case kptTable : res = s_Table; break;
- case kptPrereleaseTbl : res = s_PrereleaseTbl; break;
- case kptColumn : res = s_Column; break;
- case kptIndex : res = s_Index; break;
- case kptNotFound : res = s_NotFound; break;
- case kptBadPath : res = s_BadPath; break;
- case kptFile : res = s_File; break;
- case kptDir : res = s_Dir; break;
- case kptCharDev : res = s_CharDev; break;
- case kptBlockDev : res = s_BlockDev; break;
- case kptFIFO : res = s_FIFO; break;
- case kptZombieFile : res = s_ZombieFile; break;
- case kptDataset : res = s_Dataset; break;
- case kptDatatype : res = s_Datatype; break;
- }
- return res;
-}
-
-static int CC String_lane_cmp ( const void * item, const BSTNode * n )
-{
- const String * spot_group = ( const String * ) item;
- const lane * sg_lane = ( const lane * ) n;
- return StringCompare ( sg_lane->name, spot_group );
-}
-
-
-static int CC lane_lane_cmp ( const BSTNode * item, const BSTNode * n )
-{
- const lane * lane1 = ( const lane * ) item;
- const lane * lane2 = ( const lane * ) n;
- return StringCompare ( lane2->name, lane1->name );
-}
-
-
-const char * bzip_ext = ".bz2";
-const char * gzip_ext = ".gz";
-const char * none_ext = "";
-
-
-static rc_t write_header_line( lane * l, const char * line, size_t len )
-{
- size_t num_writ_file;
- rc_t rc = KFileWrite ( l->reads, l->write_pos, line, len, &num_writ_file );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot write output for header" );
- }
- else
- {
- l->write_pos += num_writ_file;
- }
- return rc;
-}
-
-
-static rc_t write_header( cg_dump_opts * opts, struct sg_lookup * lookup, lane * l )
-{
- char buffer[ 1024 ];
- size_t num_writ_buf;
- sg * entry = NULL;
-
- rc_t rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#GENERATED_BY\t%s\n", UsageDefaultName );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate GENERATED_BY for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
-
- if ( rc == 0 )
- {
- KTime now;
- KTimeLocal ( &now, KTimeStamp () );
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#GENERATED_AT\t%lT\n", &now );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate GENERATED_AT for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SOFTWARE_VERSION\t%.3V\n", KAppVersion() );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate SOFTWARE_VERSION for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#FORMAT_VERSION\t2.0\n" );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate FORMAT_VERSION for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#TYPE\tSAM_READS\n" );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate SAM_READS for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SLIDE\t%.*s\n", ( l->name->len - 4 ), l->name->addr );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate SLIDE for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LANE\t%.*s\n", 3, l->name->addr + ( ( l->name->len - 3 ) ) );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate LANE for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#BATCH_FILE_NUMBER\t%d\n", l->chunk );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate BATCH_FILE_NUMBER for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LANE\t%S\n", l->name );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate LANE(2) for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- perform_sg_lookup( lookup, l->name, &entry );
- if ( entry != NULL )
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LIBRARY\t%S\n", &entry->lib );
- else
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LIBRARY\t%s\n", opts->lib ); /* opts->lib */
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate LIBRARY for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- if ( entry != NULL )
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SAMPLE\t%S\n", &entry->sample );
- else
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SAMPLE\t%s\n", opts->sample ); /* opts->sample */
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate SAMPLE for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- if ( entry != NULL )
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#FIELD_SIZE\t%S\n", &entry->field_size );
- else
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#FIELD_SIZE\t460800\n" );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate FIELD_SIZE for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
-
- if ( rc == 0 )
- {
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "\n>readOffset\tside\tbases\tscores\n" );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot generate columns for header" );
- }
- else
- rc = write_header_line( l, buffer, num_writ_buf );
- }
- return rc;
-}
-
-
-static rc_t make_read_file( cg_dump_opts * opts, struct sg_lookup * lookup, KDirectory * dir, lane * l )
-{
- rc_t rc;
- if ( opts->comp == oc_null )
- {
- l->reads = NULL;
- rc = 0;
- }
- else
- {
- const char * ext;
- switch( opts->comp )
- {
- case oc_none : ext = none_ext; break;
- case oc_gzip : ext = gzip_ext; break;
- case oc_bzip : ext = bzip_ext; break;
- case oc_null : ext = none_ext; break;
- }
- rc = KDirectoryCreateFile ( dir, &l->reads, true, 0664, kcmCreate,
- "%s%.*s_%03d.tsv%s", opts->prefix, l->name->len, l->name->addr, l->chunk, ext );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create reads-file for lane '$(lane)' / chunk '$(chunk)'",
- "lane=%S,chunk=%d", l->name, l->chunk ) );
- }
- else if ( opts->wbuff_size > 0 )
- {
- KFile * buffered;
- rc = KBufWriteFileMakeWrite ( &buffered, l->reads, opts->wbuff_size );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create buffered reads-file for lane '$(lane)' / chunk '$(chunk)'",
- "lane=%S,chunk=%d", l->name, l->chunk ) );
- }
- else
- {
- KFileRelease( l->reads );
- l->reads = buffered;
- }
- }
-
- if ( rc == 0 && opts->comp != oc_none )
- {
-
- KFile * compressed = l->reads;
- if ( opts->comp == oc_bzip )
- {
- rc = KFileMakeBzip2ForWrite ( &compressed, l->reads );
- }
- else if ( opts->comp == oc_gzip )
- {
- rc = KFileMakeGzipForWrite ( &compressed, l->reads );
- }
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create bzip reads-file for lane '$(lane)' / chunk '$(chunk)'",
- "lane=%S,chunk=%d", l->name, l->chunk ) );
- }
- else
- {
- KFileRelease( l->reads );
- l->reads = compressed;
- }
- }
-
- if ( rc == 0 && opts->use_queue )
- {
- KFile * qf;
- rc = KQueueFileMakeWrite ( &qf, l->reads, opts->qbytes, opts->qblock, opts->qtimeout );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create background writer for lane '$(lane)' / chunk '$(chunk)'",
- "lane=%S,chunk=%d", l->name, l->chunk ) );
- }
- else
- {
- KFileRelease( l->reads );
- l->reads = qf;
- }
- }
-
- }
-
- if ( rc == 0 )
- {
- rc = write_header( opts, lookup, l );
- }
-
- return rc;
-}
-
-
-/*
-static rc_t make_mappings_file( KDirectory * dir, lane * l )
-{
- rc_t rc = KDirectoryCreateFile ( dir, &l->mappings, true, 0664, kcmCreate,
- "mappings_%.*s_%03d.tsv.gz", l->name->len, l->name->addr, l->chunk );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create mappings-file for lane '$(lane)' / chunk '$(chunk)'",
- "lane=%S,chunk=%d", l->name, l->chunk ) );
- }
- else
- {
- KFile * bz;
- rc = KFileMakeBzip2ForWrite ( &bz, l->mappings );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create bzip mappings-file for lane '$(lane)' / chunk '$(chunk)'",
- "lane=%S,chunk=%d", l->name, l->chunk ) );
- }
- else
- {
- KFileRelease( l->mappings );
- l->mappings = bz;
- }
- }
-}
-
-*/
-
-static rc_t make_lane( cg_dump_opts * opts, struct sg_lookup * lookup, KDirectory * dir, String * spot_group, lane ** sg_lane )
-{
- rc_t rc = 0;
- ( *sg_lane ) = malloc( sizeof ** sg_lane );
- if ( *sg_lane == NULL )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcMemory, rcExhausted );
- (void)LOGERR( klogErr, rc, "memory exhausted when creating new lane" );
- }
- else
- {
- (*sg_lane)->chunk = opts->first_chunk;
- (*sg_lane)->write_pos = 0;
- (*sg_lane)->spot_count = 0;
- rc = StringCopy ( &( (*sg_lane)->name ), spot_group );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot copy name for new lane" );
- free( *sg_lane );
- }
- else
- {
- rc = make_read_file( opts, lookup, dir, *sg_lane );
- if ( rc != 0 )
- {
- StringWhack ( (*sg_lane)->name );
- free( *sg_lane );
- }
-/*
- else
- {
- rc = make_mappings_file( dir, *sg_lane );
- if ( rc != 0 )
- {
- KFileRelease( (*sg_lane)->reads );
- StringWhack ( (*sg_lane)->name );
- free( *sg_lane );
- }
- }
-*/
- }
- }
- return rc;
-}
-
-
-static void whack_lane( lane * l )
-{
-/* KFileRelease( l->mappings ); */
- KFileRelease( l->reads );
- StringWhack ( l->name );
- free( l );
-}
-
-
-static rc_t cg_dump_write_spot( cg_dump_opts * opts, cg_dump_ctx * cg_ctx, uint64_t row_id, lane * l )
-{
- uint32_t elem_bits, boff, read_len;
- const char * read;
-
- rc_t rc = VCursorCellDataDirect( cg_ctx->seq_cur, row_id, cg_ctx->seq_read_idx, &elem_bits, (const void**)&read, &boff, &read_len );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot read READ in row #$(row_id)", "row_id=%lu", row_id ) );
- }
- else
- {
- uint32_t qual_len;
- const char * qual;
- rc = VCursorCellDataDirect( cg_ctx->seq_cur, row_id, cg_ctx->seq_qual_idx, &elem_bits, (const void**)&qual, &boff, &qual_len );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot read QUALITY in row #$(row_id)", "row_id=%lu", row_id ) );
- }
- else
- {
- if ( ( read_len != 70 ) && ( qual_len != 70 ) )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcInvalid );
- (void)LOGERR( klogErr, rc, "len of read/quality columns do not match cg-length of 2 x 35" );
- }
- else
- {
- char buffer[ 1024 ];
- size_t num_writ_buf;
- rc = string_printf ( buffer, sizeof buffer, &num_writ_buf,
- "%lu\t0\t%.35s\t%.35s\n%lu\t1\t%.35s\t%.35s\n",
- row_id, read, qual, row_id, &(read[35]), &(qual[35]) );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot generate output in row #$(row_id)", "row_id=%lu", row_id ) );
- }
- else
- {
- if ( opts->comp != oc_null )
- {
- if ( l->spot_count >= opts->cutoff )
- {
- KFileRelease( l->reads );
- l->chunk++;
- l->spot_count = 0;
- l->write_pos = 0;
- rc = make_read_file( opts, cg_ctx->lookup, cg_ctx->out_dir, l );
- }
- if ( rc == 0 )
- {
- size_t num_writ_file;
- rc = KFileWrite ( l->reads, l->write_pos, buffer, num_writ_buf, &num_writ_file );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot write output in row #$(row_id)", "row_id=%lu", row_id ) );
- }
- else
- {
- l->write_pos += num_writ_file;
- l->spot_count ++;
- }
- }
- }
- }
- }
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_row( cg_dump_opts * opts, cg_dump_ctx * cg_ctx, uint64_t row_id )
-{
- uint32_t elem_bits, boff, sg_len;
- const char * sg;
- rc_t rc = VCursorCellDataDirect( cg_ctx->seq_cur, row_id, cg_ctx->seq_sg_idx, &elem_bits, (const void**)&sg, &boff, &sg_len );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot read spot-group in row #$(row_id)", "row_id=%lu", row_id ) );
- }
- else
- {
- String spot_group;
- lane * sg_lane;
-
- StringInit( &spot_group, sg, sg_len, sg_len );
- sg_lane = ( lane * )BSTreeFind ( &cg_ctx->lanes, &spot_group, String_lane_cmp );
- if ( sg_lane == NULL )
- {
- /* KOutMsg( "row %lu (%S) not found, create it\n", row_id, &spot_group ); */
- rc = make_lane( opts, cg_ctx->lookup, cg_ctx->out_dir, &spot_group, &sg_lane );
- if ( rc == 0 )
- {
- rc = BSTreeInsert ( &cg_ctx->lanes, ( BSTNode * )sg_lane, lane_lane_cmp );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot insert new lane" );
- whack_lane( sg_lane );
- }
- }
- }
- else
- {
- /* KOutMsg( "row %lu (%S) found, use it\n", row_id, &spot_group ); */
- }
- if ( rc == 0 )
- {
- cg_dump_write_spot( opts, cg_ctx, row_id, sg_lane ); /* <================== */
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_loop( cg_dump_opts * opts, cg_dump_ctx * cg_ctx )
-{
- const num_gen_iter * iter;
- rc_t rc = num_gen_iterator_make( cg_ctx->rows, &iter );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot make num-gen-iterator" );
- }
- else
- {
- uint64_t row_id;
- rc_t rc1 = num_gen_iterator_next( iter, &row_id );
- while ( rc == 0 && rc1 == 0 )
- {
- rc = Quitting(); /* to be able to cancel the loop by signal */
- if ( rc == 0 )
- {
- rc = cg_dump_row( opts, cg_ctx, row_id ); /* <================== */
- if ( rc == 0 )
- {
- rc1 = num_gen_iterator_next( iter, &row_id );
- if ( rc1 == 0 )
- {
- if ( opts->show_progress )
- {
- uint32_t percent;
- rc = num_gen_iterator_percent( iter, cg_ctx->fract_digits, &percent );
- if ( rc == 0 )
- update_progressbar( cg_ctx->progress, cg_ctx->fract_digits, percent );
- }
- }
- }
- }
- }
- if ( opts->show_progress )
- KOutMsg( "\n" );
- num_gen_iterator_destroy( iter );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_add_column( const VCursor * cur, uint32_t * idx, const char * col_name, const char * cur_name )
-{
- rc_t rc = VCursorAddColumn ( cur, idx, "%s", col_name );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot add $(col_name) to $(cur_name)",
- "col_name=%s,cur_name=%s", col_name, cur_name ) );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_open_cursor( const VCursor * cur, const char * cur_name )
-{
- rc_t rc = VCursorOpen ( cur );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot open $(cur_name)", "cur_name=%s", cur_name ) );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_prepare_seq_tab( cg_dump_ctx * cg_ctx )
-{
- const VCursor * cur = cg_ctx->seq_cur;
- const char * cur_name = "SEQ-cursor";
- rc_t rc = cg_dump_add_column( cur, &cg_ctx->seq_read_idx, "(INSDC:dna:text)READ", cur_name );
-/*
- if ( rc == 0 )
- rc = cg_dump_add_column( cur, &cg_ctx->seq_read_len_idx, "(INSDC:coord:len)READ_LEN", cur_name );
-*/
- if ( rc == 0 )
- rc = cg_dump_add_column( cur, &cg_ctx->seq_qual_idx, "(INSDC:quality:text:phred_33)QUALITY", cur_name );
-/*
- if ( rc == 0 )
- rc = cg_dump_add_column( cur, &cg_ctx->seq_prim_id_idx, "(I64)PRIMARY_ALIGNMENT_ID", cur_name );
-*/
- if ( rc == 0 )
- rc = cg_dump_add_column( cur, &cg_ctx->seq_sg_idx, "(ascii)SPOT_GROUP", cur_name );
- if ( rc == 0 )
- rc = cg_dump_open_cursor( cur, cur_name );
- return rc;
-}
-
-
-/*
-static rc_t cg_dump_prepare_prim_tab( cg_dump_ctx * cg_ctx )
-{
- const VCursor * cur = cg_ctx->prim_cur;
- const char * cur_name = "PRIM-cursor";
- rc_t rc = cg_dump_add_column( cur, &cg_ctx->prim_cigar_idx, "(ascii)CIGAR_SHORT", cur_name );
- if ( rc == 0 )
- rc = cg_dump_add_column( cur, &cg_ctx->prim_refname_idx, "(ascii)REF_NAME", cur_name );
- if ( rc == 0 )
- rc = cg_dump_add_column( cur, &cg_ctx->prim_refpos_idx, "(INSDC:coord:zero)REF_POS", cur_name );
- if ( rc == 0 )
- rc = cg_dump_open_cursor( cur, cur_name );
- return rc;
-}
-*/
-
-static rc_t cg_dump_adjust_rowrange( cg_dump_ctx * cg_ctx )
-{
- int64_t first;
- uint64_t count;
- rc_t rc = VCursorIdRange( cg_ctx->seq_cur, 0, &first, &count );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot detect Id-Range for SEQ-cursor" );
- }
- else
- {
- rc = num_gen_range_check( ( num_gen * )cg_ctx->rows, first, count );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot define range of rows" );
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_setup_progressbar( cg_dump_ctx * cg_ctx )
-{
- rc_t rc = make_progressbar( &cg_ctx->progress );
- if ( rc == 0 )
- {
- uint64_t count;
- const num_gen_iter * iter;
- cg_ctx->fract_digits = 0;
- rc = num_gen_iterator_make( cg_ctx->rows, &iter );
- if ( rc == 0 )
- {
- if ( num_gen_iterator_count( iter, &count ) == 0 )
- {
- if ( count > 10000 )
- {
- if ( count > 100000 )
- cg_ctx->fract_digits = 2;
- else
- cg_ctx->fract_digits = 1;
- }
- }
- num_gen_iterator_destroy( iter );
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_create_output_dir( cg_dump_ctx * cg_ctx )
-{
- rc_t rc = KDirectoryCreateDir ( cg_ctx->dir, 0775, ( kcmCreate | kcmParents ), "%s", cg_ctx->dst );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot create directory '$(dir)'", "dir=%s", cg_ctx->dst ) );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_clear_output_dir( cg_dump_opts * opts, cg_dump_ctx * cg_ctx )
-{
- rc_t rc;
- if ( opts->overwrite )
- {
- rc = KDirectoryClearDir ( cg_ctx->dir, true, "%s", cg_ctx->dst );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot clear directory '$(dt)'",
- "dt=%s", cg_ctx->dst ) );
- }
- }
- else
- {
- rc = RC( rcExe, rcNoTarg, rcReading, rcParam, rcInvalid );
- (void)PLOGERR( klogErr, ( klogErr, rc, "output-directory exitst already '$(dt)', use force-switch",
- "dt=%s", cg_ctx->dst ) );
- KOutMsg( "output-directory exitst already '%s', use force-switch", cg_ctx->dst );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_remove_file( cg_dump_opts * opts, cg_dump_ctx * cg_ctx )
-{
- rc_t rc;
- if ( opts->overwrite )
- {
- rc = KDirectoryRemove ( cg_ctx->dir, true, "%s", cg_ctx->dst );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot remove file '$(dt)'",
- "dt=%s", cg_ctx->dst ) );
- }
- }
- else
- {
- rc = RC( rcExe, rcNoTarg, rcReading, rcParam, rcInvalid );
- (void)PLOGERR( klogErr, ( klogErr, rc, "output-directory-name exists as file '$(dt)', use force-switch",
- "dt=%s", cg_ctx->dst ) );
- KOutMsg( "output-directory-name exists as file '%s', use force-switch", cg_ctx->dst );
-
- }
- return rc;
-}
-
-
-static rc_t cg_dump_prepare_output( cg_dump_opts * opts, cg_dump_ctx * cg_ctx )
-{
- rc_t rc = 0;
-
- uint32_t pt = ( KDirectoryPathType ( cg_ctx->dir, "%s", cg_ctx->dst ) & ~ kptAlias );
- switch ( pt )
- {
- case kptNotFound : rc = cg_dump_create_output_dir( cg_ctx ); break;
-
- case kptDir : rc = cg_dump_clear_output_dir( opts, cg_ctx ); break;
-
- case kptFile : rc = cg_dump_remove_file( opts, cg_ctx );
- if ( rc == 0 )
- rc = cg_dump_create_output_dir( cg_ctx );
- break;
-
- default : rc = RC( rcExe, rcNoTarg, rcReading, rcParam, rcInvalid );
- (void)PLOGERR( klogErr, ( klogErr, rc, "invalid output-directory-type '$(dt)'",
- "dt=%s", pathtype_2_pchar( pt ) ) );
- break;
- }
- if ( rc == 0 )
- {
- rc = KDirectoryOpenDirUpdate ( cg_ctx->dir, &cg_ctx->out_dir, false, "%s", cg_ctx->dst );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot open output-directory for update" );
- }
- else
- {
- BSTreeInit( &cg_ctx->lanes );
- }
- }
- return rc;
-}
-
-
-static void CC whack_lanes_nodes( BSTNode *n, void *data )
-{
- whack_lane( ( lane * )n );
-}
-
-
-static rc_t cg_dump_src_dst_rows_cur( cg_dump_opts * opts, cg_dump_ctx * cg_ctx )
-{
- /* preparations */
- rc_t rc = cg_dump_prepare_seq_tab( cg_ctx );
-/*
- if ( rc == 0 )
- rc = cg_dump_prepare_prim_tab( cg_ctx );
-*/
- if ( rc == 0 )
- rc = cg_dump_adjust_rowrange( cg_ctx );
- if ( rc == 0 )
- rc = cg_dump_setup_progressbar( cg_ctx );
- if ( rc == 0 )
- rc = cg_dump_prepare_output( opts, cg_ctx );
-
- /* loop through the SEQUENCE-table */
- if ( rc == 0 )
- {
- rc = cg_dump_loop( opts, cg_ctx ); /* <================== */
-
- if ( cg_ctx->progress != NULL )
- destroy_progressbar( cg_ctx->progress );
- BSTreeWhack ( &cg_ctx->lanes, whack_lanes_nodes, NULL );
- KDirectoryRelease( cg_ctx->out_dir );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_src_dst_rows_vdb( cg_dump_opts * opts, const VDBManager * mgr, const char * src, cg_dump_ctx * cg_ctx )
-{
- const VDatabase * db;
- rc_t rc = VDBManagerOpenDBRead( mgr, &db, NULL, "%s", src );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot open database" );
- }
- else
- {
- rc = parse_sg_lookup( cg_ctx->lookup, db ); /* in sg_lookup.c */
- if ( rc == 0 )
- {
- const VTable * seq;
- rc = VDatabaseOpenTableRead( db, &seq, "SEQUENCE" );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot open SEQUENCE-table" );
- }
- else
- {
- const VTable * prim;
- rc = VDatabaseOpenTableRead( db, &prim, "PRIMARY_ALIGNMENT" );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot open PRIMARY-ALIGNMENT-table" );
- }
- else
- {
- if ( opts->cursor_cache > 0 )
- rc = VTableCreateCachedCursorRead( seq, &cg_ctx->seq_cur, opts->cursor_cache );
- else
- rc = VTableCreateCursorRead( seq, &cg_ctx->seq_cur );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create cursor for SEQUENCE-table" );
- }
- else
- {
- /*
- if ( opts->cursor_cache > 0 )
- rc = VTableCreateCachedCursorRead( prim, &cg_ctx->prim_cur, opts->cursor_cache );
- else
- rc = VTableCreateCursorRead( prim, &cg_ctx->prim_cur );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create cursor for PRIMARY_ALIGNMENT-table" );
- }
- */
-
- if ( rc == 0 )
- {
- rc = cg_dump_src_dst_rows_cur( opts, cg_ctx ); /* <================== */
- /* VCursorRelease( cg_ctx->prim_cur ); */
- }
- VCursorRelease( cg_ctx->seq_cur );
- }
- /* VTableRelease( prim ); */
- }
- VTableRelease( seq );
- }
- }
- VDatabaseRelease( db );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_src_dst_rows( cg_dump_opts * opts, const char * src, cg_dump_ctx * cg_ctx )
-{
- rc_t rc = KDirectoryNativeDir( &cg_ctx->dir );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create internal directory object" );
- }
- else
- {
- const VDBManager * mgr;
- rc = VDBManagerMakeRead ( &mgr, cg_ctx->dir );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create vdb-manager" );
- }
- else
- {
- int path_type = ( VDBManagerPathType ( mgr, "%s", src ) & ~ kptAlias );
- if ( path_type != kptDatabase )
- {
- rc = RC( rcExe, rcNoTarg, rcReading, rcParam, rcInvalid );
- if ( path_type == kptNotFound )
- {
- (void)LOGERR( klogErr, rc, "the source object cannot be found" );
- }
- else
- {
- (void)LOGERR( klogErr, rc, "source cannot be used" );
- KOutMsg( "it is instead: '%s'\n", pathtype_2_pchar( path_type ) );
- }
- }
- else
- {
- rc = cg_dump_src_dst_rows_vdb( opts, mgr, src, cg_ctx ); /* <================== */
- }
- VDBManagerRelease( mgr );
- }
- KDirectoryRelease( cg_ctx->dir );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_discover_last_rowid( const char * dst )
-{
- rc_t rc = KOutMsg( "\n***** discover last processed row-id *****\n" );
- if ( rc == 0 )
- {
- int64_t last_row_id = 0;
- rc = discover_last_rowid( dst, &last_row_id );
- if ( rc == 0 )
- {
- rc = KOutMsg( "last row-id: %ld\n", last_row_id );
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_get_uint64t_option( Args * args, const char * name, uint64_t * value, bool * found )
-{
- uint32_t count;
-
- rc_t rc = ArgsOptionCount( args, name, &count );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect count of $(name) option", "name=%s", name ) );
- }
- else
- {
- *found = ( count > 0 );
- }
-
- if ( rc == 0 && ( *found ) )
- {
- const char * s;
- rc = ArgsOptionValue( args, name, 0, &s );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect value of $(name) option", "name=%s", name ) );
- }
- else
- {
- char * endp;
- *value = strtou64( s, &endp, 10 );
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_get_size_option( Args * args, const char * name, size_t * value, bool * found )
-{
- uint32_t count;
-
- rc_t rc = ArgsOptionCount( args, name, &count );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect count of $(name) option", "name=%s", name ) );
- }
- else
- {
- *found = ( count > 0 );
- }
-
- if ( rc == 0 && ( *found ) )
- {
- const char * s;
- rc = ArgsOptionValue( args, name, 0, &s );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect value of $(name) option", "name=%s", name ) );
- }
- else
- {
- *value = atoi( s );
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_get_uint32t_option( Args * args, const char * name, uint32_t * value, bool * found )
-{
- uint32_t count;
-
- rc_t rc = ArgsOptionCount( args, name, &count );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect count of $(name) option", "name=%s", name ) );
- }
- else
- {
- *found = ( count > 0 );
- }
-
- if ( rc == 0 && ( *found ) )
- {
- const char * s;
- rc = ArgsOptionValue( args, name, 0, &s );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect value of $(name) option", "name=%s", name ) );
- }
- else
- {
- *value = atoi( s );
- }
- }
- return rc;
-}
-
-
-static rc_t cg_dump_get_bool_option( Args * args, const char * name, bool * value )
-{
- uint32_t count;
-
- rc_t rc = ArgsOptionCount( args, name, &count );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect count of $(name) option", "name=%s", name ) );
- }
- else
- {
- *value = ( count > 0 );
- }
- return rc;
-}
-
-
-static rc_t cg_dump_get_string_option( Args * args, const char * name, const char ** value, bool * found )
-{
- uint32_t count;
-
- rc_t rc = ArgsOptionCount( args, name, &count );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect count of $(name) option", "name=%s", name ) );
- }
- else
- {
- *found = ( count > 0 );
- }
-
- if ( rc == 0 && ( *found ) )
- {
- rc = ArgsOptionValue( args, name, 0, value );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot detect value of $(name) option", "name=%s", name ) );
- }
- }
- return rc;
-}
-
-
-const char * value_comp_none = "none";
-const char * value_comp_bzip = "bzip";
-const char * value_comp_gzip = "gzip";
-const char * value_comp_null = "null";
-
-
-static rc_t cg_dump_gather_output_compression( Args * args, cg_dump_opts * opts )
-{
- const char * value;
- bool found;
- rc_t rc = cg_dump_get_string_option( args, OPTION_COMP, &value, &found );
- if ( rc == 0 )
- {
- opts->comp = oc_bzip;
- if ( found )
- {
- if ( string_cmp ( value, string_size( value ),
- value_comp_none, string_size( value_comp_none ), string_size( value ) ) == 0 )
- {
- opts->comp = oc_none;
- }
- else if ( string_cmp ( value, string_size( value ),
- value_comp_bzip, string_size( value_comp_bzip ), string_size( value ) ) == 0 )
- {
- opts->comp = oc_bzip;
- }
- else if ( string_cmp ( value, string_size( value ),
- value_comp_gzip, string_size( value_comp_gzip ), string_size( value ) ) == 0 )
- {
- opts->comp = oc_gzip;
- }
- else if ( string_cmp ( value, string_size( value ),
- value_comp_null, string_size( value_comp_null ), string_size( value ) ) == 0 )
- {
- opts->comp = oc_null;
- }
- }
- }
- return rc;
-}
-
-
-const char * dflt_lib = "no library";
-const char * dflt_sample = "no sample";
-const char * dflt_prefix = "";
-
-static rc_t cg_dump_gather_opts( Args * args, cg_dump_opts * opts )
-{
- bool found;
-
- rc_t rc = cg_dump_get_uint64t_option( args, OPTION_CUTOFF, &opts->cutoff, &found );
- if ( rc == 0 && !found )
- opts->cutoff = DEFAULT_CUTOFF;
-
- if ( rc == 0 )
- rc = cg_dump_get_bool_option( args, OPTION_FORCE, &opts->overwrite );
-
- if ( rc == 0 )
- rc = cg_dump_get_uint64t_option( args, OPTION_CACHE, &opts->cursor_cache, &found );
- if ( rc == 0 && !found )
- opts->cursor_cache = CURSOR_CACHE_SIZE;
-
- if ( rc == 0 )
- rc = cg_dump_gather_output_compression( args, opts );
-
- if ( rc == 0 )
- rc = cg_dump_get_bool_option( args, OPTION_PROG, &opts->show_progress );
-
- opts->first_chunk = 1;
-
- if ( rc == 0 )
- rc = cg_dump_get_string_option( args, OPTION_LIB, &opts->lib, &found );
- if ( rc == 0 && !found )
- opts->lib = dflt_lib;
-
- if ( rc == 0 )
- rc = cg_dump_get_string_option( args, OPTION_SAMPLE, &opts->sample, &found );
- if ( rc == 0 && !found )
- opts->sample = dflt_sample;
-
- if ( rc == 0 )
- rc = cg_dump_get_string_option( args, OPTION_PREFIX, &opts->prefix, &found );
- if ( rc == 0 && !found )
- opts->prefix = dflt_prefix;
-
- if ( rc == 0 )
- rc = cg_dump_get_size_option( args, OPTION_WBUF, &opts->wbuff_size, &found );
-
- if ( rc == 0 )
- rc = cg_dump_get_bool_option( args, OPTION_QUEUE, &opts->use_queue );
-
- if ( rc == 0 )
- rc = cg_dump_get_size_option( args, OPTION_QUEUE_BYTES, &opts->qbytes, &found );
- if ( rc == 0 && !found )
- opts->qbytes = ( 2 * 64 * 1024 );
-
- if ( rc == 0 )
- rc = cg_dump_get_size_option( args, OPTION_QUEUE_BLOCK, &opts->qblock, &found );
- if ( rc == 0 && !found )
- opts->qblock = ( 64 * 1024 );
-
- if ( rc == 0 )
- rc = cg_dump_get_uint32t_option( args, OPTION_QUEUE_TIMEOUT, &opts->qtimeout, &found );
- if ( rc == 0 && !found )
- opts->qtimeout = 10000;
-
- if ( rc == 0 )
- rc = cg_dump_get_bool_option( args, OPTION_LAST_ROWID, &opts->get_last_rowid );
-
- return rc;
-}
-
-
-static rc_t cg_dump_src_dst( Args * args, const char * src, cg_dump_ctx * cg_ctx )
-{
- rc_t rc = num_gen_make( &cg_ctx->rows );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create internal object" );
- }
- else
- {
- uint32_t count;
- rc = ArgsOptionCount( args, OPTION_ROWS, &count );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot detect count of program option : 'rows'" );
- }
- else
- {
- if ( count > 0 )
- {
- const char * s;
- rc = ArgsOptionValue( args, OPTION_ROWS, 0, &s );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot detect value of program option : 'rows'" );
- }
- else
- {
- num_gen_parse( cg_ctx->rows, s );
- }
- }
- if ( rc == 0 )
- {
- rc = make_sg_lookup( &cg_ctx->lookup ); /* in sg_lookup.c */
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create spot-group lookup table" );
- }
- else
- {
- cg_dump_opts opts;
- memset( &opts, 0, sizeof opts );
- rc = cg_dump_gather_opts( args, &opts );
- if ( rc == 0 )
- {
- if ( opts.get_last_rowid )
- {
- rc = cg_dump_discover_last_rowid( cg_ctx->dst );
- }
- else
- {
- rc = cg_dump_src_dst_rows( &opts, src, cg_ctx ); /* <================== */
- }
- }
-
- destroy_sg_lookup( cg_ctx->lookup ); /* in sg_lookup.c */
- }
- }
- }
- num_gen_destroy( cg_ctx->rows );
- }
- return rc;
-}
-
-
-rc_t CC KMain ( int argc, char *argv [] )
-{
- Args * args;
- rc_t rc = ArgsMakeAndHandle ( &args, argc, argv, 1,
- DumpOptions, sizeof DumpOptions / sizeof DumpOptions [ 0 ] );
- if ( rc == 0 )
- {
- uint32_t count;
- rc = ArgsParamCount( args, &count );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot detect count of program arguments" );
- }
- else
- {
- if ( count != 2 )
- {
- Usage ( args );
- }
- else
- {
- cg_dump_ctx cg_ctx;
- const char * src;
-
- memset( &cg_ctx, 0, sizeof cg_ctx );
- rc = ArgsParamValue( args, 0, &src );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot detect source - argument" );
- }
- else
- {
- rc = ArgsParamValue( args, 1, &cg_ctx.dst );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot detect destination - argument" );
- }
- else
- {
- rc = cg_dump_src_dst( args, src, &cg_ctx ); /* <================== */
- }
- }
- }
- }
- ArgsWhack ( args );
- }
- return rc;
-}
diff --git a/tools/cg-dump/cg-dump.vers b/tools/cg-dump/cg-dump.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/cg-dump/cg-dump.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/cg-dump/last_rowid.c b/tools/cg-dump/last_rowid.c
deleted file mode 100644
index fbbc179..0000000
--- a/tools/cg-dump/last_rowid.c
+++ /dev/null
@@ -1,353 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <os-native.h>
-#include <sysalloc.h>
-
-#include "last_rowid.h"
-#include "line_token_iter.h"
-#include <klib/log.h>
-#include <klib/out.h>
-#include <klib/text.h>
-#include <klib/printf.h>
-#include <klib/namelist.h>
-
-#include <kfs/directory.h>
-#include <kfs/file.h>
-
-#include <stdlib.h>
-
-
-typedef struct sg_entry
-{
- BSTNode node;
-
- const String * spot_group;
- const String * full_filename;
- uint32_t batch;
-} sg_entry;
-
-
-static int CC String_entry_cmp ( const void * item, const BSTNode * n )
-{
- const String * name = ( const String * ) item;
- const sg_entry * sg = ( const sg_entry * ) n;
- return StringCompare ( sg->spot_group, name );
-}
-
-
-static int CC entry_entry_cmp ( const BSTNode * item, const BSTNode * n )
-{
- const sg_entry * sg1 = ( const sg_entry * ) item;
- const sg_entry * sg2 = ( const sg_entry * ) n;
- return StringCompare ( sg2->spot_group, sg1->spot_group );
-}
-
-
-static rc_t make_sg_entry( const String * spot_group, const String * full_filename, uint32_t batch, sg_entry ** entry )
-{
- rc_t rc = 0;
- ( *entry ) = malloc( sizeof ** entry );
- if ( *entry == NULL )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcMemory, rcExhausted );
- (void)LOGERR( klogErr, rc, "memory exhausted when creating new spotgroup-lookup-entry" );
- }
- else
- {
- rc = StringCopy ( &( ( *entry )->spot_group ), spot_group );
- if ( rc == 0 )
- rc = StringCopy ( &( (*entry)->full_filename ), full_filename );
- if ( rc == 0 )
- (*entry)->batch = batch;
- }
- return rc;
-}
-
-
-rc_t insert_file( BSTree * entries, const char * filename )
-{
- rc_t rc;
- String S;
- struct token_iter ti;
-
- StringInitCString( &S, filename );
- rc = token_iter_init( &ti, &S, '.' );
- if ( rc == 0 )
- {
- bool valid;
- String basename;
-
- rc = token_iter_get( &ti, &basename, &valid, NULL );
- if ( rc == 0 && valid )
- {
- String sg;
- const char * ptr;
- char s_batch[ 4 ];
- uint32_t batch;
- sg_entry * entry;
-
- StringInit( &sg, basename.addr, basename.size - 4, basename.size - 4 );
- ptr = ( basename.addr + sg.size + 1 );
- s_batch[ 0 ] = *ptr++;
- s_batch[ 1 ] = *ptr++;
- s_batch[ 2 ] = *ptr++;
- s_batch[ 3 ] = 0;
- batch = atoi( s_batch );
-
- entry = ( sg_entry * ) BSTreeFind ( entries, ( void * )&sg, String_entry_cmp );
- if ( entry == NULL )
- {
- rc = make_sg_entry( &sg, &S, batch, &entry );
- if ( rc == 0 )
- rc = BSTreeInsert ( entries, ( BSTNode * )entry, entry_entry_cmp );
- }
- else
- {
- if ( batch > entry->batch )
- entry->batch = batch;
- }
- }
- }
- return rc;
-}
-
-
-static void CC whack_entry( BSTNode * n, void * data )
-{
- const sg_entry * sg = ( const sg_entry * ) n;
- StringWhack ( sg->spot_group );
- StringWhack ( sg->full_filename );
- free( ( void * ) sg );
-}
-
-
-typedef struct walk_ctx
-{
- rc_t rc;
- int64_t last_row_id;
- const KDirectory *dir;
-} walk_ctx;
-
-
-enum ft
-{
- ft_none = 0, /* it is uncompressed tvs */
- ft_gzip, /* use gzip */
- ft_bzip, /* use bzip2 */
- ft_unknown /* unknown extension */
-};
-
-
-enum ft get_compression( const String * filename )
-{
- enum ft res = ft_unknown;
- const char * ptr = filename->addr + ( filename->size - 3 );
- if ( ptr[ 0 ] == 'b' )
- {
- if ( ptr[ 1 ] == 'z' && ptr[ 2 ] == '2' )
- res = ft_bzip;
- }
- else if ( ptr[ 0 ] == 'g' )
- {
- if ( ptr[ 1 ] == 'z' )
- res = ft_gzip;
- }
- else if ( ptr[ 0 ] == 't' )
- {
- if ( ptr[ 1 ] == 's' && ptr[ 2 ] == 'v' )
- res = ft_none;
- }
- return res;
-}
-
-
-static rc_t raw_extract_last_rowid( const KFile * f, int64_t * row_id )
-{
- rc_t rc = 0;
- KOutMsg( "uncompressed tsv-file\n" );
- return rc;
-}
-
-
-static rc_t gzip_extract_last_rowid( const KFile * f, int64_t * row_id )
-{
- rc_t rc = 0;
- KOutMsg( "compressed with gzip\n" );
- return rc;
-}
-
-
-static rc_t bzip_extract_last_rowid( const KFile * f, int64_t * row_id )
-{
- const KFile *bz;
- rc_t rc = KFileMakeBzip2ForRead ( &bz, f );
- if ( rc != 0 )
- {
-
- }
- else
- {
-
- }
- return rc;
-}
-
-
-static rc_t extract_last_rowid( const KDirectory *dir, const String * filename, int64_t * row_id )
-{
- const KFile * f;
- rc_t rc = KDirectoryOpenFileRead ( dir, &f, "%.*s", filename->size, filename->addr );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot open $(fn)", "fn=%.*s", filename->size, filename->addr ) );
- }
- else
- {
- enum ft comp = get_compression( filename );
- KOutMsg( "file '%S' opened :", filename );
-
- switch ( comp )
- {
- case ft_none : raw_extract_last_rowid( f, row_id ); break;
- case ft_gzip : gzip_extract_last_rowid( f, row_id ); break;
- case ft_bzip : bzip_extract_last_rowid( f, row_id ); break;
- default : KOutMsg( "unknown file-type\n" ); break;
- }
- rc = KFileRelease ( f );
- }
- return rc;
-}
-
-
-static bool CC on_file ( BSTNode *n, void *data )
-{
- walk_ctx * wctx = ( walk_ctx * )data;
- if ( wctx != NULL && wctx->rc == 0 )
- {
- const sg_entry * sg = ( const sg_entry * ) n;
- if ( sg != NULL )
- {
- size_t num_writ, l;
- l = sg->spot_group->size + 1;
- wctx->rc = string_printf ( ( char * )( sg->full_filename->addr + l ), 4, &num_writ, "%.03u", sg->batch );
- if ( wctx->rc == 0 )
- {
- int64_t row_id = 0;
- ( ( char * ) sg->full_filename->addr )[ l + 3 ] = '.';
- wctx->rc = extract_last_rowid( wctx->dir, sg->full_filename, &row_id );
- if ( wctx->rc == 0 )
- {
- if ( row_id > wctx->last_row_id )
- wctx->last_row_id = row_id;
- return false;
- }
- }
- }
- }
- return true;
-}
-
-
-rc_t discover_last_rowid( const char * src, int64_t * last_row_id )
-{
- KDirectory *dir;
- rc_t rc = KDirectoryNativeDir ( &dir );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot create native directory" );
- }
- else
- {
- const KDirectory * src_dir;
- rc = KDirectoryOpenDirRead ( dir, &src_dir, false, "%s", src );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot open directory #$(d)", "d=%s", src ) );
- }
- else
- {
- KNamelist *files;
- rc = KDirectoryList ( src_dir, &files, NULL, NULL, "." );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot list files in output directory" );
- }
- else
- {
- uint32_t count;
- rc = KNamelistCount ( files, &count );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot count the files in the output directory" );
- }
- else
- {
- rc = KOutMsg( "%u files in '%s' found\n", count, src );
- if ( rc == 0 )
- {
- uint32_t idx;
- BSTree entries;
-
- BSTreeInit( &entries );
- for ( idx = 0; idx < count && rc == 0; ++idx )
- {
- const char * filename;
- rc = KNamelistGet ( files, idx, &filename );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "cannot retrieve filename #$(idx)", "idx=%u", idx ) );
- }
- else
- rc = insert_file( &entries, filename );
- }
-
- if ( rc == 0 )
- {
- walk_ctx wctx;
-
- wctx.rc = rc;
- wctx.last_row_id = *last_row_id;
- wctx.dir = src_dir;
- BSTreeDoUntil ( &entries, false, on_file, &wctx );
- if ( wctx.rc == 0 )
- {
- *last_row_id = wctx.last_row_id;
- }
- else
- rc = wctx.rc;
- }
- BSTreeWhack ( &entries, whack_entry, NULL );
- }
- }
- KNamelistRelease ( files );
- }
- KDirectoryRelease( src_dir );
- }
- KDirectoryRelease( dir );
- }
- return rc;
-}
\ No newline at end of file
diff --git a/tools/cg-dump/line_token_iter.c b/tools/cg-dump/line_token_iter.c
deleted file mode 100644
index 61b1553..0000000
--- a/tools/cg-dump/line_token_iter.c
+++ /dev/null
@@ -1,148 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <os-native.h>
-#include <sysalloc.h>
-
-#include "line_token_iter.h"
-#include <klib/log.h>
-#include <klib/out.h>
-
-#include <stdlib.h>
-
-/* ---------------------------------------------------------------------------- */
-/* >>>>> buf line iter <<<<< */
-
-rc_t buf_line_iter_init( struct buf_line_iter * self, const char * buffer, size_t len )
-{
- rc_t rc = 0;
-
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else
- {
- /* we do accept null buffer, empty len, we just do not hand out lines later */
- self->nxt = buffer;
- self->len = len;
- self->line_nr = 0;
- }
- return rc;
-}
-
-
-rc_t buf_line_iter_get( struct buf_line_iter * self, String * line, bool * valid, uint32_t * line_nr )
-{
- rc_t rc = 0;
-
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else if ( line == NULL || valid == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
- else
- {
- *valid = ( self->nxt != NULL && self->len > 0 );
- if ( *valid )
- {
- char * nl = string_chr ( self->nxt, self->len, '\n' );
- if ( nl == NULL )
- {
- /* no newline found... */
- StringInit( line, self->nxt, self->len, self->len );
- self->len = 0; /* next call will return *valid == false */
- }
- else
- {
- /* we found a newline... */
- size_t l = ( nl - self->nxt );
- StringInit( line, self->nxt, l, l );
- self->len -= ( l + 1 );
- if ( self->len > 0 )
- self->nxt += ( l + 1 );
- }
- if ( line_nr != NULL )
- *line_nr = self->line_nr++;
- }
- }
- return rc;
-}
-
-
-/* ---------------------------------------------------------------------------- */
-/* >>>>> token iter <<<<< */
-
-rc_t token_iter_init( struct token_iter * self, const String * line, char delim )
-{
- rc_t rc = 0;
-
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else
- {
- /* we do accept null buffer, empty len, we just do not hand out lines later */
- StringInit( &self->line, line->addr, line->size, line->len );
- self->delim = delim;
- self->token_nr = 0;
- self->idx = 0;
- }
- return rc;
-}
-
-
-rc_t token_iter_get( struct token_iter * self, String * token, bool * valid, uint32_t * token_nr )
-{
- rc_t rc = 0;
-
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else if ( token == NULL || valid == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
- else
- {
- while ( ( self->idx < self->line.len ) && ( self->line.addr[ self->idx ] == self->delim ) )
- self->idx++;
- *valid = ( self->idx < self->line.len );
- if ( *valid )
- {
- char * end;
- size_t l = ( self->line.len - self->idx );
- token->addr = &( self->line.addr[ self->idx ] );
- end = string_chr ( token->addr, l, self->delim );
- if ( end == NULL )
- {
- token->size = token->len = l;
- self->idx = self->line.len;
- }
- else
- {
- token->size = token->len = ( end - token->addr );
- self->idx = ( end - self->line.addr );
- }
- if ( token_nr != NULL )
- *token_nr = self->token_nr++;
- }
- }
- return rc;
-}
diff --git a/tools/cg-dump/num-gen.c b/tools/cg-dump/num-gen.c
deleted file mode 100644
index 637fdf1..0000000
--- a/tools/cg-dump/num-gen.c
+++ /dev/null
@@ -1,848 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "num-gen.h"
-#include <klib/printf.h>
-
-#include <sysalloc.h>
-#include <strtol.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-typedef struct num_gen_node
-{
- uint64_t start;
- uint64_t end;
- uint64_t count; /* 0 ... skip, >0 ... valid */
-} num_gen_node;
-typedef struct num_gen_node * p_num_gen_node;
-
-/* **************************************************************************
-{ start:5,count: 0 } ---> [ ]
-{ start:5,count: 1 } ---> [ 5 ]
-{ start:5,count: 2 } ---> [ 5, 6 ]
- ************************************************************************** */
-
-struct num_gen
-{
- Vector nodes;
-};
-
-
-struct num_gen_iter
-{
- Vector nodes;
- uint64_t curr_node;
- uint64_t curr_node_sub_pos;
- uint64_t total;
- uint64_t progress;
-};
-
-/* forward decl. for fixing-function */
-static rc_t num_gen_fix_overlaps( num_gen* self, uint32_t *count );
-
-
-/* helper function to destroy a node*/
-static void CC num_gen_node_destroy( void *item, void *data )
-{
- free( item );
-}
-
-
-/* helper function to create a node from start/count */
-static p_num_gen_node num_gen_make_node( const uint64_t start, const uint64_t count )
-{
- p_num_gen_node p = ( p_num_gen_node )malloc( sizeof( num_gen_node ) );
- if ( p )
- {
- p->start = start;
- p->end = start + count - 1;
- p->count = count;
- }
- return p;
-}
-
-
-/* helper callback to compare 2 nodes, lets VectorInsert create a sorted vector */
-static int CC num_gen_insert_helper( const void* item1, const void* item2 )
-{
- const p_num_gen_node node1 = ( p_num_gen_node )item1;
- const p_num_gen_node node2 = ( p_num_gen_node )item2;
- int res = 0;
- if ( node1->start == node2->start )
- {
- if ( node1->count < node2->count )
- res = -1;
- else if ( node1->count > node2->count )
- res = 1;
- }
- else if ( node1->start < node2->start )
- res = -1;
- else
- res = 1;
- return res;
-}
-
-
-/* helper callback to create a deep and conditional copy of a node-vector */
-static void CC num_gen_copy_cb( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- if ( node->count > 0 )
- {
- Vector * dst = ( Vector *)data;
- p_num_gen_node new_node = num_gen_make_node( node->start, node->count );
- if ( new_node != NULL )
- VectorInsert( dst, new_node, NULL, num_gen_insert_helper );
- }
-}
-
-
-/* helper function that creates a deep and conditional copy of a node-vector */
-static void num_gen_copy_vector( const Vector * src, Vector * dst )
-{
- if ( src == NULL || dst == NULL )
- return;
- VectorForEach ( src, false, num_gen_copy_cb, dst );
-}
-
-
-/* helper callback to add up all count values in the vector*/
-static void CC num_gen_total_count_cb( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- if ( node != NULL )
- {
- uint64_t * total = ( uint64_t *)data;
- if ( total != NULL )
- *total += node->count;
- }
-}
-
-
-/* helper function that adds up all count values in the vector*/
-static uint64_t num_gen_total_count( const Vector * src )
-{
- uint64_t res = 0;
- if ( src != NULL )
- VectorForEach ( src, false, num_gen_total_count_cb, &res );
- return res;
-}
-
-
-/* helper function for the parse-function */
-static rc_t num_gen_add_node( num_gen* self, const uint64_t from,
- const uint64_t to )
-{
- p_num_gen_node node = NULL;
- int64_t count = ( to - from );
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcSelf, rcNull );
-
- if ( count >= 0 )
- node = num_gen_make_node( from, count + 1 );
- else
- node = num_gen_make_node( to, -( count + 1 ) );
- if ( node == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcMemory, rcExhausted );
- return VectorInsert( &(self->nodes), node, NULL, num_gen_insert_helper );
-}
-
-
-#define MAX_NUM_STR 12
-/* helper-structure for num_gen_parse() */
-typedef struct num_gen_parse_ctx
-{
- uint32_t num_str_idx;
- bool this_is_the_first_number;
- uint64_t num1;
- uint64_t num2;
- char num_str[ MAX_NUM_STR + 1 ];
-} num_gen_parse_ctx;
-typedef num_gen_parse_ctx* p_num_gen_parse_ctx;
-
-
-/* helper for num_gen_parse() */
-static void num_gen_convert_ctx( p_num_gen_parse_ctx ctx )
-{
- char *endp;
-
- ctx->num_str[ ctx->num_str_idx ] = 0;
- ctx->num1 = strtou64( ctx->num_str, &endp, 10 );
- ctx->this_is_the_first_number = false;
- ctx->num_str_idx = 0;
-}
-
-
-/* helper for num_gen_parse() */
-static rc_t num_gen_convert_and_add_ctx( num_gen* self, p_num_gen_parse_ctx ctx )
-{
- char *endp;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcSelf, rcNull );
- if ( ctx == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcParam, rcNull );
- if ( ctx->num_str_idx == 0 )
- return RC( rcVDB, rcNoTarg, rcInserting, rcParam, rcEmpty );
-
- /* terminate the source-string */
- ctx->num_str[ ctx->num_str_idx ] = 0;
- /* convert the string into a uint64_t */
- if ( ctx->this_is_the_first_number )
- {
- ctx->num1 = strtou64( ctx->num_str, &endp, 10 );
- ctx->num2 = ctx->num1;
- }
- else
- ctx->num2 = strtou64( ctx->num_str, &endp, 10 );
- /* empty the source-string to be reused */
- ctx->num_str_idx = 0;
-
- ctx->this_is_the_first_number = true;
- return num_gen_add_node( self, ctx->num1, ctx->num2 );
-}
-
-
-/* parse the given string and insert the found ranges
- into the number-generator, fixes eventual overlaps */
-rc_t num_gen_parse( num_gen* self, const char* src )
-{
- size_t i, n;
- num_gen_parse_ctx ctx;
- rc_t rc = 0;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcSelf, rcNull );
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcParam, rcNull );
-
- n = string_measure ( src, NULL );
- if ( n == 0 )
- return RC( rcVDB, rcNoTarg, rcParsing, rcParam, rcEmpty );
-
- ctx.num_str_idx = 0;
- ctx.this_is_the_first_number = true;
- for ( i = 0; i < n && rc == 0; ++i )
- {
- switch ( src[ i ] )
- {
- /* a dash switches from N1-mode into N2-mode */
- case '-' :
- num_gen_convert_ctx( &ctx );
- break;
-
- /* a comma ends a single number or a range */
- case ',' :
- rc = num_gen_convert_and_add_ctx( self, &ctx );
- break;
-
- /* in both mode add the char to the temp string */
- default:
- if ( ( src[i]>='0' )&&( src[i]<='9' )&&( ctx.num_str_idx < MAX_NUM_STR ) )
- ctx.num_str[ ctx.num_str_idx++ ] = src[ i ];
- break;
- }
- }
- /* dont forget to add what is left in ctx.num_str ... */
- if ( ctx.num_str_idx > 0 )
- rc = num_gen_convert_and_add_ctx( self, &ctx );
- if ( rc == 0 )
- rc = num_gen_fix_overlaps( self, NULL );
- return rc;
-}
-
-
-/* inserts the given ranges into the number-generator,
- fixes eventual overlaps */
-rc_t num_gen_add( num_gen* self, const uint64_t first, const uint64_t count )
-{
- rc_t rc;
- uint64_t num_1 = first;
- uint64_t num_2 = first;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcSelf, rcNull );
-
- /* this is necessary because virtual columns which have a
- infinite row-range, get reported with first=1,count=0 */
- if ( count > 0 )
- num_2 = ( first + count - 1 );
- rc = num_gen_add_node( self, num_1, num_2 );
- if ( rc == 0 )
- rc = num_gen_fix_overlaps( self, NULL );
- return rc;
-}
-
-
-/* helper function for range-check */
-static bool CC num_gen_check_range_start( p_num_gen_node the_node,
- const uint64_t range_start )
-{
- bool res = true;
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( the_node->start < range_start )
- {
- the_node->start = range_start;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- res = false;
- }
- }
- return res;
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_check_range_end( p_num_gen_node the_node,
- const uint64_t last_tab_row )
-{
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( last_node_row > last_tab_row )
- {
- last_node_row = last_tab_row;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- }
- }
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_check_range_callback( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- p_num_gen_node the_range = ( p_num_gen_node )data;
- uint64_t last_tab_row = ( the_range->start + the_range->count - 1 );
-
- /* ignore invalid nodes... */
- if ( the_node->start == 0 || the_node->count == 0 )
- return;
-
- /* check if the start value is not out of range... */
- if ( num_gen_check_range_start( the_node, the_range->start ) )
- num_gen_check_range_end( the_node, last_tab_row );
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_count_invalid_nodes( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- uint32_t *invalid_count = ( uint32_t * )data;
-
- if ( ( the_node->start == 0 )&&( the_node->count == 0 ) )
- ( *invalid_count )++;
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_copy_valid_nodes( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- Vector *dest = ( Vector * )data;
-
- if ( ( node->start != 0 )&&( node->count != 0 ) )
- VectorInsert ( dest, node, NULL, num_gen_insert_helper );
- else
- free ( node );
-}
-
-
-/* helper function for range-check */
-static void num_gen_remove_invalid_nodes( num_gen* self )
-{
- Vector temp_nodes;
- uint32_t count = VectorLength( &(self->nodes) );
-
- if ( count < 1 )
- return;
- /* create a temp. vector */
- VectorInit( &temp_nodes, 0, count );
-
- /* copy all valid nodes into the temp. vector */
- VectorForEach ( &(self->nodes), false,
- num_gen_copy_valid_nodes, &temp_nodes );
-
- /* clear all nodes so far...,
- DO NOT PASS num_gen_node_destroy into it */
- VectorWhack( &(self->nodes), NULL, NULL );
-
- /* initialize and copy (shallow) the valid nodes back
- into the generator */
- VectorCopy ( &temp_nodes, &(self->nodes) );
-
- /* destroy the temp-vector,
- DO NOT PASS num_gen_node_destroy into it */
- VectorWhack ( &temp_nodes, NULL, NULL );
-}
-
-
-/* helper function for trim */
-rc_t num_gen_trim( num_gen* self, const int64_t first, const uint64_t count )
-{
- num_gen_node trim_range;
- uint32_t invalid_nodes = 0;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcValidating, rcSelf, rcNull );
- if ( count == 0 )
- return RC( rcVDB, rcNoTarg, rcValidating, rcParam, rcNull );
-
- /* walk all nodes to check for boundaries... */
- trim_range.start = first;
- trim_range.count = count;
-
- VectorForEach ( &(self->nodes), false,
- num_gen_check_range_callback, &trim_range );
-
- VectorForEach ( &(self->nodes), false,
- num_gen_count_invalid_nodes, &invalid_nodes );
- if ( invalid_nodes > 0 )
- num_gen_remove_invalid_nodes( self );
-
- return 0;
-}
-
-
-rc_t num_gen_make( num_gen** self )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
-
- *self = calloc( 1, sizeof( num_gen ) );
- if ( *self == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
-
- VectorInit( &((*self)->nodes ), 0, 5 );
- return 0;
-}
-
-
-rc_t num_gen_make_from_str( num_gen** self, const char *src )
-{
- rc_t rc = num_gen_make( self );
- if ( rc == 0 )
- {
- rc = num_gen_parse( *self, src );
- if ( rc == 0 )
- rc = num_gen_fix_overlaps( *self, NULL );
- }
- return rc;
-}
-
-
-rc_t num_gen_make_from_range( num_gen** self,
- const int64_t first, const uint64_t count )
-{
- rc_t rc = num_gen_make( self );
- if ( rc != 0 )
- return rc;
- return num_gen_add( *self, first, count );
-}
-
-
-rc_t num_gen_clear( num_gen* self )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcClearing, rcSelf, rcNull );
-
- if ( VectorLength( &(self->nodes) ) > 0 )
- {
- /* clear all nodes so far... */
- VectorWhack( &(self->nodes), num_gen_node_destroy, NULL );
-
- /* re-init the vector */
- VectorInit( &(self->nodes ), 0, 5 );
- }
- return 0;
-}
-
-
-rc_t num_gen_destroy( num_gen* self )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
-
- VectorWhack( &(self->nodes), num_gen_node_destroy, NULL );
- free( self );
- return 0;
-}
-
-
-bool num_gen_empty( const num_gen* self )
-{
- if ( self == NULL )
- return true;
- return ( VectorLength( &(self->nodes) ) < 1 );
-}
-
-
-typedef struct overlap_ctx
-{
- p_num_gen_node prev;
- uint32_t overlaps;
-} overlap_ctx;
-typedef overlap_ctx* p_overlap_ctx;
-
-
-/* static bool CC num_gen_overlap_fix_cb( void *item, void *data ) */
-static bool CC num_gen_overlap_fix_cb( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- p_overlap_ctx ctx = ( p_overlap_ctx )data;
-
- /* skip invalid nodes */
- if ( node->count ==0 || node->start == 0 || node->end == 0 )
- return false;
- /* if we do not have a previous node, take this one... */
- if ( ctx->prev == NULL )
- {
- ctx->prev = node;
- return false;
- }
- /* if we do not have an overlap,
- take this node as prev-node and continue */
- if ( ctx->prev->end < node->start )
- {
- ctx->prev = node;
- return false;
- }
- /* we have a overlap, the end of the prev-node is inside
- the current-node, we fix it by expanding the prev-node
- to the end of this node, and later declaring this
- node as invalid */
- if ( ctx->prev->end < node->end )
- {
- ctx->prev->end = node->end;
- ctx->prev->count = ( ctx->prev->end - ctx->prev->start ) + 1;
- }
- /* if the prev-node ends after this node, all we have to
- do is declaring this node as invalid */
- node->count = 0;
- node->start = 0;
- node->end = 0;
- return true;
-}
-
-
-static rc_t num_gen_fix_overlaps( num_gen* self, uint32_t *count )
-{
- overlap_ctx ctx;
- bool fix_executed = false;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
-
- ctx.overlaps = 0;
- do
- {
- ctx.prev = NULL;
- fix_executed = VectorDoUntil ( &(self->nodes), false,
- num_gen_overlap_fix_cb, &ctx );
- } while ( fix_executed );
-
- if ( count )
- *count = ctx.overlaps;
- return 0;
-}
-
-
-typedef struct string_ctx
-{
- char *s;
- uint32_t len;
-} string_ctx;
-typedef string_ctx* p_string_ctx;
-
-
-static void string_ctx_add( p_string_ctx ctx, char *s )
-{
- uint32_t len = string_measure ( s, NULL );
- if ( len > 0 )
- {
- if ( ctx->len == 0 )
- ctx->s = malloc( len + 1 );
- else
- ctx->s = realloc( ctx->s, ctx->len + len );
- memcpy( &(ctx->s[ctx->len]), s, len );
- ctx->len += len;
- }
-}
-
-
-static void CC num_gen_as_string_cb( void *item, void *data )
-{
- char temp[40];
- p_num_gen_node node = ( p_num_gen_node )item;
- long unsigned int start = node->start;
- long unsigned int end = ( start + node->count - 1 );
- switch( node->count )
- {
- case 0 : temp[ 0 ] = 0;
- break;
- case 1 : string_printf ( temp, sizeof temp, NULL, "%lu,", start );
- break;
- default: string_printf ( temp, sizeof temp, NULL, "%lu-%lu,", start, end );
- break;
- }
- string_ctx_add( ( p_string_ctx )data, temp );
-}
-
-
-rc_t num_gen_as_string( const num_gen* self, char **s )
-{
- string_ctx ctx;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( s == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
-
- ctx.s = NULL;
- ctx.len = 0;
- VectorForEach ( &(self->nodes), false, num_gen_as_string_cb, &ctx );
- if ( ctx.len == 0 )
- {
- *s = NULL;
- return RC( rcVDB, rcNoTarg, rcReading, rcData, rcEmpty );
- }
- ctx.s[ ctx.len ] = 0;
- *s = ctx.s;
- return 0;
-}
-
-
-static void CC num_gen_debug_cb( void *item, void *data )
-{
- char temp[40];
- p_num_gen_node node = ( p_num_gen_node )item;
- long unsigned int start = node->start;
- long unsigned int count = node->count;
- string_printf ( temp, sizeof temp, NULL, "[s:%lu c:%lu]", start, count );
- string_ctx_add( ( p_string_ctx )data, temp );
-}
-
-
-rc_t num_gen_debug( const num_gen* self, char **s )
-{
- string_ctx ctx;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( s == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
-
- ctx.s = NULL;
- ctx.len = 0;
- VectorForEach ( &(self->nodes), false, num_gen_debug_cb, &ctx );
- if ( ctx.len == 0 )
- {
- *s = NULL;
- return RC( rcVDB, rcNoTarg, rcReading, rcData, rcEmpty );
- }
- ctx.s[ ctx.len ] = 0;
- *s = ctx.s;
- return 0;
-}
-
-
-static bool CC num_gen_contains_cb( void *item, void *data )
-{
- bool res = false;
- p_num_gen_node node = ( p_num_gen_node )item;
- if ( node->count > 0 )
- {
- uint64_t *value = ( uint64_t * )data;
- uint64_t end = node->start + node->count - 1;
- res = ( node->start <= *value && *value <= end );
- }
- return res;
-}
-
-
-rc_t num_gen_contains_value( const num_gen* self, const uint64_t value )
-{
- uint64_t temp = value;
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( VectorDoUntil ( &(self->nodes), false,
- num_gen_contains_cb, &temp ) )
- return 0;
- else
- return RC( rcVDB, rcNoTarg, rcReading, rcData, rcEmpty );
-}
-
-
-rc_t num_gen_range_check( num_gen* self,
- const int64_t first, const uint64_t count )
-{
- /* if the user did not specify a row-range, take all rows */
- if ( num_gen_empty( self ) )
- return num_gen_add( self, first, count );
- /* if the user did specify a row-range, check the boundaries */
- else
- return num_gen_trim( self, first, count );
-}
-
-
-rc_t num_gen_iterator_make( const num_gen* self, const num_gen_iter **iter )
-{
- uint32_t count;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( iter == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
-
- *iter = NULL;
- count = VectorLength( &(self->nodes) );
- if ( count < 1 )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- else
- {
- num_gen_iter *temp = calloc( 1, sizeof( num_gen_iter ) );
- if ( temp == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- VectorInit( &(temp->nodes), 0, count );
- num_gen_copy_vector( &(self->nodes), &(temp->nodes ) );
- temp->total = num_gen_total_count( &(temp->nodes ) );
- *iter = temp;
- }
- return 0;
-}
-
-rc_t num_gen_iterator_destroy( const num_gen_iter *self )
-{
- num_gen_iter *temp;
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
-
- temp = (num_gen_iter *)self;
- VectorWhack( &(temp->nodes), num_gen_node_destroy, NULL );
- free( temp );
- return 0;
-}
-
-rc_t num_gen_iterator_next( const num_gen_iter* self, uint64_t* value )
-{
- num_gen_iter* temp;
- p_num_gen_node node;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( value == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- if ( self->curr_node >= VectorLength( &(self->nodes) ) )
- return RC( rcVDB, rcNoTarg, rcReading, rcId, rcInvalid );
-
- temp = ( num_gen_iter *)self;
- *value = 0;
- node = (p_num_gen_node)VectorGet( &(temp->nodes), temp->curr_node );
- if ( node == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcItem, rcInvalid );
-
- *value = node->start;
- if ( node->count < 2 )
- /* the node is a single-number-node, next node for next time */
- temp->curr_node++;
- else
- {
- /* the node is a number range, add the sub-position */
- *value += temp->curr_node_sub_pos++;
- /* if the sub-positions are use up, switch to next node */
- if ( temp->curr_node_sub_pos >= node->count )
- {
- temp->curr_node++;
- temp->curr_node_sub_pos = 0;
- }
- }
- (temp->progress)++;
- return 0;
-}
-
-
-rc_t num_gen_iterator_count( const num_gen_iter* self, uint64_t* count )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( count == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- *count = self->total;
- return 0;
-}
-
-
-rc_t num_gen_iterator_percent( const num_gen_iter* self,
- const uint8_t fract_digits,
- uint32_t* value )
-{
- uint32_t factor = 100;
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( value == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- if ( fract_digits > 0 )
- {
- if ( fract_digits > 1 )
- factor = 10000;
- else
- factor = 1000;
- }
-
- if ( self->total > 0 )
- {
- if ( self->progress >= self->total )
- *value = factor;
- else
- {
- uint64_t temp = self->progress;
- temp *= factor;
- temp /= self->total;
- *value = (uint16_t) temp;
- }
- }
- else
- *value = 0;
- return 0;
-}
diff --git a/tools/cg-dump/num-gen.h b/tools/cg-dump/num-gen.h
deleted file mode 100644
index 5b0324f..0000000
--- a/tools/cg-dump/num-gen.h
+++ /dev/null
@@ -1,225 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_num_gen_
-#define _h_num_gen_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef _h_klib_vector_
-#include <klib/vector.h>
-#endif
-
-#ifndef _h_klib_rc_
-#include <klib/rc.h>
-#endif
-
-#ifndef _h_klib_text_
-#include <klib/text.h>
-#endif
-
-
-/*--------------------------------------------------------------------------
- * A NUMBER GENERATOR
- *
- * input : string, for instance "3,6,8,12,44-49"
- * ouptut: sequence of integers, for instance 3,6,8,12,44,45,46,47,48,49
- */
-
-
-/*--------------------------------------------------------------------------
- * opaque number-generator and it's iterator
- */
-typedef struct num_gen num_gen;
-typedef struct num_gen_iter num_gen_iter;
-
-
-/*--------------------------------------------------------------------------
- * num_gen_make
- *
- * creates a empty number-generator
- * or creates a number-generator and parses the string
- * or creates and presets it with a range
- */
-rc_t num_gen_make( num_gen** self );
-rc_t num_gen_make_from_str( num_gen** self, const char *src );
-rc_t num_gen_make_from_range( num_gen** self,
- const int64_t first, const uint64_t count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_destroy
- *
- * destroys a number-generator
- */
-rc_t num_gen_destroy( num_gen* self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_clear
- *
- * resets a number-generator, to be empty just like after num_gen_make()
- */
-rc_t num_gen_clear( num_gen* self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_parse
- *
- * parses a given string in this form: "3,6,8,12,44-49"
- * does not clear the number-generator before parsing
- * eventual overlaps with the previous content are consolidated
- */
-rc_t num_gen_parse( num_gen* self, const char* src );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_add
- *
- * inserts the given interval into the number-generator
- *
- * num_gen_add( *g, 10, 30 )
- * is equivalent to:
- * num_gen_parse( *g, "10-39" );
- *
- * eventual overlaps with the previous content are consolidated
- */
-rc_t num_gen_add( num_gen* self, const uint64_t first, const uint64_t count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_trim
- *
- * checks if the content of the number-generator is inside the given interval
- * removes or shortens internal nodes if necessary
- */
-rc_t num_gen_trim( num_gen* self, const int64_t first, const uint64_t count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_empty
- *
- * checks if the generator has no ranges defined
- */
-bool num_gen_empty( const num_gen* self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_as_string
- *
- * allocates a string that contains the generator as text
- * *s = "1-5,20,24-25"
- * caller has to free the string
- */
-rc_t num_gen_as_string( const num_gen* self, char **s );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_debug
- *
- * allocates a string that contains the internal intervals as text
- * *s = "[s:1 c:5][s:20 c:1][s:24 c:2]"
- * [s...start-value c:count]
- * caller has to free the string
- */
-rc_t num_gen_debug( const num_gen* self, char **s );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_contains_value
- *
- * checks if the generator contains the given value
- */
-rc_t num_gen_contains_value( const num_gen* self, const uint64_t value );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_range_check
- *
- * if the generator is empty --> set it to the given range
- * if it is not empty ---------> trim it to the given range
- */
-rc_t num_gen_range_check( num_gen* self,
- const int64_t first, const uint64_t count );
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_make
- *
- * creates a iterator from the number-generator
- * the iterator contains a constant copy of the number-ranges
- * after this call it is safe to destroy or change the number-generator
- * returns an error-code if the number-generator was empty,
- * and *iter will be NULL
- */
-rc_t num_gen_iterator_make( const num_gen* self, const num_gen_iter **iter );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_destroy
- *
- * destroys the iterator
- */
-rc_t num_gen_iterator_destroy( const num_gen_iter *self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_count
- *
- * returns how many values the iterator contains
- */
-rc_t num_gen_iterator_count( const num_gen_iter* self, uint64_t* count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_next
- *
- * pulls the next value out of the iterator...
- * returns an error-code if the iterator has no more values
- */
-rc_t num_gen_iterator_next( const num_gen_iter* self, uint64_t* value );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_percent
- *
- * return in value the percentage of the iterator...
- * depending on fract-digits the percentage will be:
- * fract_digits = 0 ... full percent's
- * fract_digits = 1 ... 1/10-th of a percent
- * fract_digits = 2 ... 1/100-th of a percent
- */
-rc_t num_gen_iterator_percent( const num_gen_iter* self,
- const uint8_t fract_digits,
- uint32_t* value );
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/cg-dump/progressbar.c b/tools/cg-dump/progressbar.c
deleted file mode 100644
index 973f54d..0000000
--- a/tools/cg-dump/progressbar.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/rc.h>
-#include <klib/out.h>
-
-#include <os-native.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-
-typedef struct progressbar
-{
- bool initialized;
- uint16_t percent;
-} progressbar;
-
-
-rc_t make_progressbar( progressbar ** pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- (*pb) = calloc( 1, sizeof( progressbar ) );
- if ( *pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- return 0;
-}
-
-
-rc_t destroy_progressbar( progressbar * pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
- free( pb );
- return 0;
-}
-
-static void progess_0a( const uint16_t percent )
-{
- KOutMsg( "| %2u%%", percent );
-}
-
-static void progess_0( const uint16_t percent )
-{
- if ( percent & 1 )
- KOutMsg( "\b\b\b\b- %2u%%", percent );
- else
- KOutMsg( "\b\b\b%2u%%", percent );
-}
-
-static void progess_1a( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- KOutMsg( "| %2u.%01u%%", p1, p0 );
-}
-
-static void progess_1( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- KOutMsg( "\b\b\b\b\b\b- %2u.%01u%%", p1, p0 );
- else
- KOutMsg( "\b\b\b\b\b%2u.%01u%%", p1, p0 );
-}
-
-static void progess_2a( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- KOutMsg( "| %2u.%02u%%", p1, p0 );
-}
-
-static void progess_2( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- KOutMsg( "\b\b\b\b\b\b\b- %2u.%02u%%", p1, p0 );
- else
- KOutMsg( "\b\b\b\b\b\b%2u.%02u%%", p1, p0 );
-}
-
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent )
-{
- uint8_t digits = fract_digits;
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcSelf, rcNull );
- if ( digits > 2 )
- digits = 2;
- if ( pb->initialized )
- {
- if ( pb->percent != percent )
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0( percent ); break;
- case 1 : progess_1( percent ); break;
- case 2 : progess_2( percent ); break;
- }
- }
- }
- else
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0a( percent ); break;
- case 1 : progess_1a( percent ); break;
- case 2 : progess_2a( percent ); break;
- }
- pb->initialized = true;
- }
- return 0;
-}
diff --git a/tools/cg-dump/progressbar.h b/tools/cg-dump/progressbar.h
deleted file mode 100644
index f8bef61..0000000
--- a/tools/cg-dump/progressbar.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_progressbar_
-#define _h_progressbar_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct progressbar progressbar;
-
-/*--------------------------------------------------------------------------
- * make_progressbar
- *
- * creates a progressbar with zero-values inside
- * does not output anything
- */
-rc_t make_progressbar( progressbar ** pb );
-
-
-/*--------------------------------------------------------------------------
- * destroy_progressbar
- *
- * destroy's the progressbar
- * does not output anything
- */
-rc_t destroy_progressbar( progressbar * pb );
-
-
-/*--------------------------------------------------------------------------
- * update_progressbar
- *
- * sets the progressbar to a specific percentage
- * outputs only if the percentage has changed from the last call
- * the precentage is in 1/10-th of a percent ( 21,6% = 216 )
- * expects the percents in increasing order ( does not jump back )
- * writes a growing bar made from '-'-chars every 2nd percent
- */
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/cg-dump/sg_lookup.c b/tools/cg-dump/sg_lookup.c
deleted file mode 100644
index 431bdcc..0000000
--- a/tools/cg-dump/sg_lookup.c
+++ /dev/null
@@ -1,277 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <os-native.h>
-#include <sysalloc.h>
-
-#include "sg_lookup.h"
-#include "line_token_iter.h"
-#include <klib/log.h>
-#include <klib/out.h>
-#include <kdb/meta.h>
-
-#include <stdlib.h>
-
-
-typedef struct sg_lookup
-{
- BSTree spotgroups;
- const char * buffer;
- size_t buflen;
-} sg_lookup;
-
-
-rc_t make_sg_lookup( struct sg_lookup ** self )
-{
- rc_t rc = 0;
-
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else
- {
- (*self) = calloc( 1, sizeof( sg_lookup ) );
- if ( *self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- return rc;
-}
-
-
-static int CC String_sg_cmp ( const void * item, const BSTNode * n )
-{
- const String * name = ( const String * ) item;
- const sg * sg_item = ( const sg * ) n;
- return StringCompare ( &sg_item->name, name );
-}
-
-
-static int CC sg_sg_cmp ( const BSTNode * item, const BSTNode * n )
-{
- const sg * sg1 = ( const sg * ) item;
- const sg * sg2 = ( const sg * ) n;
- return StringCompare ( &sg2->name, &sg1->name );
-}
-
-
-static sg * find_entry( struct sg_lookup * self, const String * name )
-{
- return ( sg * )BSTreeFind ( &self->spotgroups, ( void * )name, String_sg_cmp );
-}
-
-
-static rc_t make_sg( String * name, String * field_size, String * lib, String * sample, sg ** entry )
-{
- rc_t rc = 0;
- ( *entry ) = malloc( sizeof ** entry );
- if ( *entry == NULL )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcMemory, rcExhausted );
- (void)LOGERR( klogErr, rc, "memory exhausted when creating new spotgroup-lookup-entry" );
- }
- else
- {
- StringInit( ( String * )&( *entry )->name, name->addr, name->size, name->len );
- StringInit( ( String * )&( *entry )->field_size, field_size->addr, field_size->size, field_size->len );
- StringInit( ( String * )&( *entry )->lib, lib->addr, lib->size, lib->len );
- StringInit( ( String * )&( *entry )->sample, sample->addr, sample->size, sample->len );
- }
- return rc;
-}
-
-
-static rc_t parse_buffer( struct sg_lookup * self )
-{
- struct buf_line_iter bli;
- bool valid = true;
- rc_t rc = buf_line_iter_init( &bli, self->buffer, self->buflen );
- while ( ( rc == 0 ) && valid )
- {
- String line;
- uint32_t line_nr;
- rc = buf_line_iter_get( &bli, &line, &valid, &line_nr );
- if ( rc == 0 && valid )
- {
- token_iter ti;
- bool ti_valid = true;
- String name, field_size, lib, sample;
-
- StringInit( &name, NULL, 0, 0 );
- StringInit( &field_size, NULL, 0, 0 );
- StringInit( &lib, NULL, 0, 0 );
- StringInit( &sample, NULL, 0, 0 );
-
- rc = token_iter_init( &ti, &line, '\t' );
- while ( ( rc == 0 ) && ti_valid )
- {
- String token;
- uint32_t token_nr;
- rc = token_iter_get( &ti, &token, &ti_valid, &token_nr );
- if ( rc == 0 && ti_valid )
- {
- if ( token_nr == 0 && ( string_cmp( token.addr, token.len, "@RG", 3, 3 ) != 0 ) )
- ti_valid = false;
- else
- {
- token_iter sub_ti;
- String *value_dst = NULL;
- bool sub_ti_valid = true;
- rc = token_iter_init( &sub_ti, &token, ':' );
- while ( ( rc == 0 ) && sub_ti_valid )
- {
- String sub_token;
- uint32_t sub_token_nr;
- rc = token_iter_get( &sub_ti, &sub_token, &sub_ti_valid, &sub_token_nr );
- if ( rc == 0 && sub_ti_valid )
- {
- if ( sub_token_nr == 0 )
- {
- if ( string_cmp( sub_token.addr, sub_token.len, "ID", 2, 2 ) == 0 )
- value_dst = &name;
- else if ( string_cmp( sub_token.addr, sub_token.len, "DS", 2, 2 ) == 0 )
- value_dst = &field_size;
- else if ( string_cmp( sub_token.addr, sub_token.len, "LB", 2, 2 ) == 0 )
- value_dst = &lib;
- else if ( string_cmp( sub_token.addr, sub_token.len, "SM", 2, 2 ) == 0 )
- value_dst = &sample;
- }
- else if ( sub_token_nr == 1 )
- {
- if ( value_dst != NULL )
- StringInit( value_dst, sub_token.addr, sub_token.size, sub_token.len );
- }
- }
- }
- }
- }
- }
-
- if ( name.addr != NULL && field_size.addr != NULL &&
- lib.addr != NULL && sample.addr != NULL )
- {
- sg * entry = find_entry( self, &name );
- /* KOutMsg( "entry: name='%S' fs='%S' lib='%S' sample='%S'\n", &name, &field_size, &lib, &sample ); */
- if ( entry == NULL )
- {
- rc = make_sg( &name, &field_size, &lib, &sample, &entry );
- if ( rc == 0 )
- {
- rc = BSTreeInsert ( &self->spotgroups, ( BSTNode * )entry, sg_sg_cmp );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot insert new spotgroup" );
- free( entry );
- }
- }
- }
- else
- {
- /* so far we ignore it if we find the same spotgroup-name twice in the meta-data */
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t parse_sg_lookup( struct sg_lookup * self, const VDatabase * db )
-{
- rc_t rc = 0;
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else
- {
- const KMetadata *meta;
- rc_t rc = VDatabaseOpenMetadataRead ( db, &meta );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot open metdata on database" );
- }
- else
- {
- const KMDataNode *node;
- rc_t rc1 = KMetadataOpenNodeRead ( meta, &node, "BAM_HEADER" );
- if ( rc1 == 0 )
- {
- size_t num_read;
- /* explore how much data we must read... */
- rc = KMDataNodeRead ( node, 0, NULL, 0, &num_read, &self->buflen );
- if ( rc == 0 )
- {
- if ( self->buffer != NULL )
- free( (void *)self->buffer );
-
- self->buffer = malloc( self->buflen + 1 );
- if ( self->buffer != NULL )
- {
- size_t num_read2;
- rc = KMDataNodeReadCString ( node, (char *)self->buffer, self->buflen + 1, &num_read2 );
- if ( rc == 0 )
- rc = parse_buffer( self );
- }
- }
- KMDataNodeRelease ( node );
- }
- KMetadataRelease ( meta );
- }
- }
- return rc;
-}
-
-
-rc_t perform_sg_lookup( struct sg_lookup * self, const String * name, sg ** entry )
-{
- rc_t rc = 0;
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- else
- {
- ( *entry ) = find_entry( self, name );
- }
- return rc;
-}
-
-
-static void CC whack_spotgroup( BSTNode *n, void *data )
-{
- free( ( void * )n );
-}
-
-
-rc_t destroy_sg_lookup( struct sg_lookup * self )
-{
- rc_t rc = 0;
-
- if ( self == NULL )
- rc = RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
- else
- {
- BSTreeWhack ( &self->spotgroups, whack_spotgroup, NULL );
- free( (void *)self->buffer );
- free( self );
- }
- return rc;
-}
diff --git a/tools/cg-load/Makefile b/tools/cg-load/Makefile
index 7713925..e3c4e3d 100644
--- a/tools/cg-load/Makefile
+++ b/tools/cg-load/Makefile
@@ -84,14 +84,17 @@ CGLOAD_SRC = \
factory-tag-lfr \
f1_3 \
f1_5 \
+ f1_7 \
f2_0 \
f2_2 \
+ f2_5 \
file \
file-version-factory \
+ formats \
writer-algn \
writer-evidence-dnbs \
writer-evidence-intervals \
- writer-seq
+ writer-seq \
CGLOAD_OBJ = \
$(addsuffix .$(OBJX),$(CGLOAD_SRC))
diff --git a/tools/cg-load/cg-load.c b/tools/cg-load/cg-load.c
index 378a888..541f017 100644
--- a/tools/cg-load/cg-load.c
+++ b/tools/cg-load/cg-load.c
@@ -23,45 +23,33 @@
* ===========================================================================
*
*/
-#include <klib/container.h>
-#include <klib/log.h>
-#include <klib/out.h>
-#include <klib/status.h>
-#include <klib/rc.h>
-#include <klib/text.h>
-#include <klib/printf.h>
-#include <kfs/directory.h>
-#include <kfs/arc.h>
-#include <kfs/tar.h>
-#include <kfs/file.h>
-#include <kdb/manager.h>
-#include <kdb/meta.h>
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <kapp/loader-meta.h>
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/database.h>
-#include <vdb/table.h>
-#include <insdc/insdc.h>
-#include <align/writer-reference.h>
-#include <kapp/log-xml.h>
-
-#include "debug.h"
-#include "cg-load.vers.h"
-#include "file.h"
-#include "writer-seq.h"
-#include "writer-algn.h"
-#include "writer-evidence-intervals.h"
-
-#include <os-native.h>
-#include <sysalloc.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <limits.h>
+
+
+#include "cg-load.vers.h" /* CG_LOAD_VERS */
+#include "debug.h" /* DEBUG_MSG */
+#include "file.h" /* FGroupMAP_LoadReads */
+#include "formats.h" /* get_cg_reads_ngaps */
+
+#include <kapp/log-xml.h> /* XMLLogger */
+#include <kapp/loader-meta.h> /* KLoaderMeta_Write */
+#include <kapp/main.h> /* KAppVersion */
+
+#include <kdb/manager.h> /* kptDatabase */
+#include <kdb/meta.h> /* KMetadata */
+
+#include <kfs/arc.h> /* KDirectoryOpenArcDirRead */
+#include <kfs/tar.h> /* KArcParseTAR */
+
+#include <klib/out.h> /* OUTMSG */
+#include <klib/printf.h> /* string_printf */
+#include <klib/status.h> /* STSMSG */
+
+#include <vdb/schema.h> /* VDBManagerMakeSchema */
+
+#include <sysalloc.h> /* malloc */
+
#include <assert.h>
+#include <errno.h>
typedef struct SParam_struct
{
@@ -91,6 +79,9 @@ typedef struct SParam_struct
uint32_t min_mapq;
uint32_t single_mate;
uint32_t cluster_size;
+ uint32_t load_other_evidence;
+
+ uint32_t read_len;
} SParam;
typedef struct DB_Handle_struct {
@@ -139,16 +130,24 @@ rc_t DB_Init(const SParam* p, DB_Handle* h)
LOGERR(klogErr, rc, "failed to create alignment writer");
}
- else if( (rc = CGWriterSeq_Make(&h->wseq, &h->reads, h->db, (p->force_readw ? ewseq_co_SaveRead : 0) |
- (p->no_spot_group ? 0 : ewseq_co_SpotGroup), p->qual_quant)) != 0 ) {
+ else if ((rc = CGWriterSeq_Make(&h->wseq, &h->reads, h->db,
+ (p->force_readw ? ewseq_co_SaveRead : 0) |
+ (p->no_spot_group ? 0 : ewseq_co_SpotGroup), p->qual_quant,
+ p->read_len))
+ != 0)
+ {
LOGERR(klogErr, rc, "failed to create sequence writer");
} else if( p->asm_path && (rc = CGWriterEvdInt_Make(&h->wev_int, &h->ev_int, h->db, h->rmgr, 0)) != 0 ) {
LOGERR(klogErr, rc, "failed to create evidence intervals writer");
-
- } else if( p->asm_path && (rc = CGWriterEvdDnbs_Make(&h->wev_dnb, &h->ev_dnb, h->db, h->rmgr, 0)) != 0 ) {
+ }
+ else if (p->asm_path
+ && (rc = CGWriterEvdDnbs_Make
+ (&h->wev_dnb, &h->ev_dnb, h->db, h->rmgr, 0, p->read_len)) != 0)
+ {
LOGERR(klogErr, rc, "failed to create evidence dnbs writer");
- } else {
+ }
+ else {
const char** r = p->refFiles;
while( rc == 0 && *r != NULL ) {
if( (rc = ReferenceMgr_FastaPath(h->rmgr, *r++)) != 0 ) {
@@ -167,39 +166,58 @@ rc_t DB_Fini(const SParam* p, DB_Handle* h, bool drop)
/* THIS FUNCTION MAKES NO ATTEMPT TO PRESERVE INITIAL ERROR CODES
EACH SUCCESSIVE ERROR OVERWRITES THE PREVIOUS CODE */
if( h != NULL ) {
+ PLOGMSG(klogInfo, (klogInfo, "DB_Fini was called with drop=$(drop)",
+ "drop=%s", drop ? "true" : "false"));
+
PLOGMSG(klogInfo, (klogInfo, "Fini SEQUENCE", "severity=status"));
- if( (rc2 = CGWriterSeq_Whack(h->wseq, !drop, NULL)) != 0 && !drop ) {
- drop = true;
- rc = rc2;
+ if( (rc2 = CGWriterSeq_Whack(h->wseq, !drop, NULL)) != 0) {
+ LOGERR(klogErr, rc2, "Failed SEQUENCE release");
+ if (!drop) {
+ drop = true;
+ rc = rc2;
+ }
}
h->wseq = NULL;
h->reads = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini (PRI&SEC)_ALIGNMENT", "severity=status"));
- if( (rc2 = CGWriterAlgn_Whack(h->walgn, !drop, NULL, NULL)) != 0 && !drop ) {
- drop = true;
- rc = rc2;
+ if( (rc2 = CGWriterAlgn_Whack(h->walgn, !drop, NULL, NULL)) != 0) {
+ LOGERR(klogErr, rc2, "Failed (PRI&SEC)_ALIGNMENT release");
+ if (!drop) {
+ drop = true;
+ rc = rc2;
+ }
}
h->walgn = NULL;
h->mappings = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini EVIDENCE_INTERVAL", "severity=status"));
- if( (rc2 = CGWriterEvdInt_Whack(h->wev_int, !drop, NULL)) != 0 && !drop ) {
- drop = true;
- rc = rc2;
+ if( (rc2 = CGWriterEvdInt_Whack(h->wev_int, !drop, NULL)) != 0) {
+ LOGERR(klogErr, rc2, "Failed EVIDENCE_INTERVAL release");
+ if (!drop) {
+ drop = true;
+ rc = rc2;
+ }
}
h->wev_int = NULL;
h->ev_int = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini EVIDENCE_ALIGNMENT", "severity=status"));
- if( (rc2 = CGWriterEvdDnbs_Whack(h->wev_dnb, !drop, NULL)) != 0 && !drop ) {
- drop = true;
- rc = rc2;
+ if( (rc2 = CGWriterEvdDnbs_Whack(h->wev_dnb, !drop, NULL)) != 0) {
+ LOGERR(klogErr, rc2, "Failed EVIDENCE_ALIGNMENT release");
+ if (!drop) {
+ drop = true;
+ rc = rc2;
+ }
}
h->wev_dnb = NULL;
h->ev_dnb = NULL;
PLOGMSG(klogInfo, (klogInfo, "Fini calculating reference coverage", "severity=status"));
- if( (rc2 = ReferenceMgr_Release(h->rmgr, !drop, NULL, drop ? false : true, Quitting)) != 0 && !drop ) {
- drop = true;
- rc = rc2;
- LOGERR(klogErr, rc, "Failed calculating reference coverage");
+ if( (rc2 = ReferenceMgr_Release
+ (h->rmgr, !drop, NULL, drop ? false : true, Quitting)) != 0)
+ {
+ LOGERR(klogErr, rc2, "Failed calculating reference coverage");
+ if (!drop) {
+ drop = true;
+ rc = rc2;
+ }
}
h->rmgr = NULL;
if( rc == 0 )
@@ -216,17 +234,17 @@ rc_t DB_Fini(const SParam* p, DB_Handle* h, bool drop)
KMetadataRelease(meta);
}
}
- PLOGMSG(klogInfo, (klogInfo, "Fini VDatabaseRelease", "severity=status"));
+ PLOGERR(klogInfo, (klogInfo, rc, "Fini VDatabaseRelease", "severity=status"));
VDatabaseRelease(h->db);
h->db = NULL;
VSchemaRelease(h->schema);
h->schema = NULL;
if( drop || rc != 0 ) {
rc2 = VDBManagerDrop(h->mgr, kptDatabase, "%s", p->out);
- if( GetRCState(rc2) == rcNotFound ) {
- /* WHAT WOULD BE THE POINT OF RESETTING "rc" TO ZERO? */
+ /*if( GetRCState(rc2) == rcNotFound ) {
+ // WHAT WOULD BE THE POINT OF RESETTING "rc" TO ZERO?
rc = 0;
- } else if( rc2 != 0 ) {
+ } else*/ if( rc2 != 0 ) {
if ( rc == 0 )
rc = rc2;
PLOGERR(klogErr, (klogErr, rc2, "cannot drop db at '$(path)'", PLOG_S(path), p->out));
@@ -332,31 +350,33 @@ void FGroupMAP_CloseFiles(FGroupMAP *g)
{
CGLoaderFile_Close(g->seq);
CGLoaderFile_Close(g->align);
+ CGLoaderFile_Close(g->tagLfr);
}
-static
-void CC FGroupMAP_Whack( BSTNode *node, void *data )
-{
+static void CC FGroupMAP_Whack(BSTNode *node, void *data) {
FGroupMAP* n = (FGroupMAP*)node;
- CGLoaderFile_Release(n->seq, false);
- CGLoaderFile_Release(n->align, false);
+
+ CGLoaderFile_Release(n->seq , false);
+ CGLoaderFile_Release(n->align , false);
+ CGLoaderFile_Release(n->tagLfr, false);
+
free(node);
}
static
-int CC FGroupMAP_Cmp( const void *item, const BSTNode *node )
+int64_t CC FGroupMAP_Cmp( const void *item, const BSTNode *node )
{
const FGroupKey* i = (const FGroupKey*)item;
const FGroupMAP* n = (const FGroupMAP*)node;
- int r = strcmp(i->assembly_id, n->key.assembly_id);
+ int64_t r = strcmp(i->assembly_id, n->key.assembly_id);
if( r == 0 ) {
if( i->type == n->key.type ) {
switch( i->type ) {
case cg_eFileType_READS:
if( (r = strcmp(i->u.map.slide, n->key.u.map.slide)) == 0 ) {
if( (r = strcmp(i->u.map.lane, n->key.u.map.lane)) == 0 ) {
- r = *(i->u.map.batch_file_number) - *(n->key.u.map.batch_file_number);
+ r = (int64_t)(*(i->u.map.batch_file_number)) - (int64_t)(*(n->key.u.map.batch_file_number));
}
}
break;
@@ -377,7 +397,7 @@ int CC FGroupMAP_Cmp( const void *item, const BSTNode *node )
}
static
-int CC FGroupMAP_Sort( const BSTNode *item, const BSTNode *n )
+int64_t CC FGroupMAP_Sort( const BSTNode *item, const BSTNode *n )
{
return FGroupMAP_Cmp(&((const FGroupMAP*)item)->key, n);
}
@@ -498,6 +518,7 @@ typedef struct DirVisit_Data_Struct {
const SParam* param;
BSTree* tree;
const KDirectory* dir;
+ uint32_t format_version;
} DirVisit_Data;
/* you cannot addref to this dir object cause it's created on stack silently */
@@ -507,30 +528,68 @@ rc_t CC DirVisitor(const KDirectory *dir, uint32_t type, const char *name, void
rc_t rc = 0;
DirVisit_Data* d = (DirVisit_Data*)data;
+ assert(d);
+
if( (type & ~kptAlias) == kptFile ) {
if (strcmp(&name[strlen(name) - 4], ".tsv") == 0 ||
strcmp(&name[strlen(name) - 8], ".tsv.bz2") == 0 ||
strcmp(&name[strlen(name) - 7], ".tsv.gz") == 0)
{
- char buf[4096];
+ char buf[4096] = "";
const CGLoaderFile* file = NULL;
FGroupKey key;
if( (rc = KDirectoryResolvePath(dir, true, buf, sizeof(buf), "%s", name)) == 0 &&
(rc = CGLoaderFile_Make(&file, d->dir, buf, NULL, !d->param->no_read_ahead)) == 0 &&
- (rc = FGroupKey_Make(&key, file, d->param)) == 0 ) {
-
+ (rc = FGroupKey_Make(&key, file, d->param)) == 0 )
+ {
FGroupMAP* found = (FGroupMAP*)BSTreeFind(d->tree, &key, FGroupMAP_Cmp);
- DEBUG_MSG(5, ("file %s recognized\n", name));
- if( found != NULL ) {
- rc = FGroupMAP_Set(found, file);
- } else {
- FGroupMAP* x = calloc(1, sizeof(*x));
- if( x == NULL ) {
- rc = RC(rcExe, rcFile, rcInserting, rcMemory, rcExhausted);
- } else {
- memcpy(&x->key, &key, sizeof(key));
- if( (rc = FGroupMAP_Set(x, file)) == 0 ) {
- rc = BSTreeInsertUnique(d->tree, &x->dad, NULL, FGroupMAP_Sort);
+
+ assert(file && file->cg_file);
+ if (file->cg_file->type == cg_eFileType_READS ||
+ file->cg_file->type == cg_eFileType_MAPPINGS)
+ {
+ if (d->format_version == 0) {
+ d->format_version = file->cg_file->format_version;
+ }
+ else if (d->format_version != file->cg_file->format_version)
+ {
+ rc = RC(rcExe, rcFile, rcInserting,
+ rcData, rcInconsistent);
+ PLOGERR(klogErr, (klogErr, rc, "READS or MAPPINGS "
+ "format_version mismatch: $(o) vs. $(n)",
+ "o=%d,n=%d",
+ d->format_version, file->cg_file->format_version));
+ }
+ }
+
+ if (rc == 0) {
+ if (d->param != NULL && ! d->param->load_other_evidence &&
+ strstr(buf, "/EVIDENCE") != NULL &&
+ strstr(buf, "/EVIDENCE/") == NULL)
+ {
+ DEBUG_MSG(5, ("file %s recognized as %s: ignored\n",
+ name, buf));
+ rc = CGLoaderFile_Release(file, true);
+ file = NULL;
+ }
+ else {
+ DEBUG_MSG(5, ("file %s recognized as %s\n", name, buf));
+ if (found != NULL) {
+ rc = FGroupMAP_Set(found, file);
+ }
+ else {
+ FGroupMAP* x = calloc(1, sizeof(*x));
+ if (x == NULL) {
+ rc = RC(rcExe,
+ rcFile, rcInserting, rcMemory, rcExhausted);
+ }
+ else {
+ memcpy(&x->key, &key, sizeof(key));
+ if ((rc = FGroupMAP_Set(x, file)) == 0) {
+ rc = BSTreeInsertUnique(d->tree,
+ &x->dad, NULL, FGroupMAP_Sort);
+ }
+ }
}
}
}
@@ -785,6 +844,7 @@ static rc_t Load( SParam* param )
dv.param = param;
dv.tree = &slides;
dv.dir = param->map_dir;
+ dv.format_version = 0;
rc = KDirectoryVisit( param->map_dir, true, DirVisitor, &dv, NULL );
if ( rc == 0 )
{
@@ -798,6 +858,11 @@ static rc_t Load( SParam* param )
rc = KDirectoryVisit( param->asm_dir, true, DirVisitor, &dv, NULL );
}
}
+
+ if (rc == 0) {
+ param->read_len = get_cg_read_len(dv.format_version);
+ }
+
if ( rc == 0 )
{
/* SHOULD HAVE A BSTreeEmpty FUNCTION OR SOMETHING...
@@ -889,6 +954,8 @@ ver_t CC KAppVersion( void )
const char* map_usage[] = {"MAP input directory path containing files", NULL};
const char* asm_usage[] = {"ASM input directory path containing files", NULL};
+const char* load_extra_evidence_usage[]
+ = { "load extra evidence files", NULL};
const char* schema_usage[] = {"database schema file name", NULL};
const char* output_usage[] = {"output database path", NULL};
const char* force_usage[] = {"force output overwrite", NULL};
@@ -911,6 +978,7 @@ const char* library_usage[] = {"copy extra file/directory into output", NULL};
enum OptDefIndex {
eopt_MapInput = 0,
eopt_AsmInput,
+ eopt_LoadExtraEvidence,
eopt_Schema,
eopt_RefSeqConfig,
eopt_RefSeqPath,
@@ -935,6 +1003,8 @@ OptDef MainArgs[] =
/* if you change order in this array, rearrange enum above accordingly! */
{ "map", "m", NULL, map_usage, 1, true, true },
{ "asm", "a", NULL, asm_usage, 1, true, false },
+ { "load-extra-evidence", NULL, NULL,
+ load_extra_evidence_usage, 1, false, false },
{ "schema", "s", NULL, schema_usage, 1, true, false },
{ "refseq-config", "k", NULL, refseqcfg_usage, 1, true, false },
{ "refseq-path", "i", NULL, refseqpath_usage, 1, true, false },
@@ -1018,54 +1088,62 @@ rc_t CC KMain( int argc, char* argv[] )
uint32_t count;
if( (rc = ArgsParamCount (args, &count)) != 0 || count != 0 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
- ArgsParamValue(args, 0, &errmsg);
+ ArgsParamValue(args, 0, (const void **)&errmsg);
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_Output].name, &count)) != 0 || count != 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, count ? rcExcessive : rcInsufficient);
errmsg = MainArgs[eopt_Output].name;
- } else if( (rc = ArgsOptionValue(args, MainArgs[eopt_Output].name, 0, ¶ms.out)) != 0 ) {
+ } else if( (rc = ArgsOptionValue(args, MainArgs[eopt_Output].name, 0, (const void **)¶ms.out)) != 0 ) {
errmsg = MainArgs[eopt_Output].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_MapInput].name, &count)) != 0 || count != 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, count ? rcExcessive : rcInsufficient);
errmsg = MainArgs[eopt_MapInput].name;
- } else if( (rc = ArgsOptionValue(args, MainArgs[eopt_MapInput].name, 0, ¶ms.map_path)) != 0 ) {
+ } else if( (rc = ArgsOptionValue(args, MainArgs[eopt_MapInput].name, 0, (const void **)¶ms.map_path)) != 0 ) {
errmsg = MainArgs[eopt_MapInput].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_Schema].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, count ? rcExcessive : rcInsufficient);
errmsg = MainArgs[eopt_Schema].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_Schema].name, 0, ¶ms.schema)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_Schema].name, 0, (const void **)¶ms.schema)) != 0 ) {
errmsg = MainArgs[eopt_Schema].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_AsmInput].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_AsmInput].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_AsmInput].name, 0, ¶ms.asm_path)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_AsmInput].name, 0, (const void **)¶ms.asm_path)) != 0 ) {
errmsg = MainArgs[eopt_AsmInput].name;
- } else if( (rc = ArgsOptionCount(args, MainArgs[eopt_RefSeqConfig].name, &count)) != 0 || count > 1 ) {
+ }
+ else if ((rc = ArgsOptionCount(args,
+ MainArgs[eopt_LoadExtraEvidence].name,
+ ¶ms.load_other_evidence)) != 0 )
+ {
+ errmsg = MainArgs[eopt_LoadExtraEvidence].name;
+
+ }
+ else if( (rc = ArgsOptionCount(args, MainArgs[eopt_RefSeqConfig].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_RefSeqConfig].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_RefSeqConfig].name, 0, ¶ms.refseqcfg)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_RefSeqConfig].name, 0, (const void **)¶ms.refseqcfg)) != 0 ) {
errmsg = MainArgs[eopt_RefSeqConfig].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_RefSeqPath].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_RefSeqPath].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_RefSeqPath].name, 0, ¶ms.refseqpath)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_RefSeqPath].name, 0, (const void **)¶ms.refseqpath)) != 0 ) {
errmsg = MainArgs[eopt_RefSeqPath].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_Library].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_Library].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_Library].name, 0, ¶ms.library)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_Library].name, 0, (const void **)¶ms.library)) != 0 ) {
errmsg = MainArgs[eopt_Library].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_RefSeqChunk].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_RefSeqChunk].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_RefSeqChunk].name, 0, &refseq_chunk)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_RefSeqChunk].name, 0, (const void **)&refseq_chunk)) != 0 ) {
errmsg = MainArgs[eopt_RefSeqChunk].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_Force].name, ¶ms.force)) != 0 ) {
@@ -1089,18 +1167,18 @@ rc_t CC KMain( int argc, char* argv[] )
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_QualQuantization].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_QualQuantization].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_QualQuantization].name, 0, ¶ms.qual_quant)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_QualQuantization].name, 0, (const void **)¶ms.qual_quant)) != 0 ) {
errmsg = MainArgs[eopt_QualQuantization].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_MinMapQ].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_MinMapQ].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_MinMapQ].name, 0, &min_mapq)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_MinMapQ].name, 0, (const void **)&min_mapq)) != 0 ) {
errmsg = MainArgs[eopt_MinMapQ].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_ClusterSize].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_ClusterSize].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_ClusterSize].name, 0, &cluster_size)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_ClusterSize].name, 0, (const void **)&cluster_size)) != 0 ) {
errmsg = MainArgs[eopt_ClusterSize].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_SingleMate].name, ¶ms.single_mate)) != 0 ) {
errmsg = MainArgs[eopt_SingleMate].name;
@@ -1120,7 +1198,7 @@ rc_t CC KMain( int argc, char* argv[] )
break;
}
while(rc == 0 && count-- > 0) {
- rc = ArgsOptionValue(args, MainArgs[eopt_RefFile].name, count, ¶ms.refFiles[count]);
+ rc = ArgsOptionValue(args, MainArgs[eopt_RefFile].name, count, (const void **)¶ms.refFiles[count]);
}
}
@@ -1163,6 +1241,9 @@ rc_t CC KMain( int argc, char* argv[] )
}
} while( false );
KDirectoryRelease( params.input_dir );
+
+ free(params.refFiles);
+ params.refFiles = NULL;
}
}
/* find accession as last part of path for internal XML logging */
diff --git a/tools/cg-load/cg-load.vers b/tools/cg-load/cg-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/cg-load/cg-load.vers
+++ b/tools/cg-load/cg-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/cg-load/defs.h b/tools/cg-load/defs.h
index d983d9b..d470e96 100644
--- a/tools/cg-load/defs.h
+++ b/tools/cg-load/defs.h
@@ -26,6 +26,13 @@
#ifndef _tools_cg_load_defs_h_
#define _tools_cg_load_defs_h_
+
+#include <stdint.h> /* uint64_t */
+
+
+#define CG_FORMAT_2_5 0x02050000
+
+
/* buffers and sizes */
/*
@@ -36,8 +43,13 @@
#define CG_LANE (8)
#define CG_READS_NREADS (2)
-#define CG_READS_SPOT_LEN (70)
-#define CG_READS_NGAPS (3)
+
+#define CG_READS15_SPOT_LEN 70
+#define CG_READS25_SPOT_LEN 60
+
+#define CG_READS15_NGAPS 3
+#define CG_READS25_NGAPS 2
+
#define CG_MAPPINGS_MAX (2048)
#define CG_EVDNC_PLOIDY (3)
#define CG_EVDNC_INTERVALID_LEN (32)
@@ -76,6 +88,8 @@ typedef uint16_t CGFIELD_NUMBER_LEVELS_TYPE;
typedef uint16_t CGFIELD_MEAN_LEVEL_X_TYPE;
typedef uint16_t CGFIELD_WELL_ID;
+/******************************************************************************/
+
typedef enum CG_EFileType_enum {
cg_eFileType_Unknown = 0,
cg_eFileType_READS = 1,
@@ -116,18 +130,19 @@ typedef enum CG_EFileType_enum {
cg_eFileType_COVERAGE_DISTRIBUTION_CODING,
cg_eFileType_COVERAGE_BY_GC_CODING,
cg_eFileType_LIB_SMALL_GAPS_ROLLUP,
- cg_eFileType_DIPLOID_SOMATIC_CNV_DETAILSCORES,
+ cg_eFileType_DIPLOID_SOMATIC_CNV_DETAIL_SCORES,
cg_eFileType_TAG_LFR,
cg_eFileType_Last
} CG_EFileType;
-typedef enum CG_EReadsFlags_enum {
- cg_eLeftHalfDnbNoMatches = 0x01,
- cg_eLeftHalfDnbMapOverflow = 0x02,
- cg_eRightHalfDnbNoMatches = 0x04,
- cg_eRightHalfDnbMapOverflow = 0x08
-} CG_EReadsFlags;
+typedef enum {
+ cg_eLibraryType_Unknown,
+ cg_eLibraryType_PureLFR,
+ cg_eLibraryType_MixedLFR,
+ cg_eLibraryType_MixedSTD,
+ cg_eLibraryType_Standard,
+} CG_ELibraryType;
typedef enum CG_EMappingsFlags_enum {
cg_eLastDNBRecord = 0x01,
@@ -137,4 +152,12 @@ typedef enum CG_EMappingsFlags_enum {
cg_eRevDnbStrand = 0x04
} CG_EMappingsFlags;
+typedef enum CG_EReadsFlags_enum {
+ cg_eLeftHalfDnbNoMatches = 0x01,
+ cg_eLeftHalfDnbMapOverflow = 0x02,
+ cg_eRightHalfDnbNoMatches = 0x04,
+ cg_eRightHalfDnbMapOverflow = 0x08
+} CG_EReadsFlags;
+
+
#endif /* _tools_cg_load_defs_h_ */
diff --git a/tools/cg-load/f1_3.c b/tools/cg-load/f1_3.c
index 315f4e0..3b0d400 100644
--- a/tools/cg-load/f1_3.c
+++ b/tools/cg-load/f1_3.c
@@ -59,6 +59,7 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "TUMOR_CNV_SEGMENTS", cg_eFileType_TUMOR_CNV_SEGMENTS, NULL },
{ "CNV_DETAILS_SCORES", cg_eFileType_CNV_DETAILS_SCORES, NULL },
{ "CNV-DETAIL-SCORES", cg_eFileType_CNV_DETAILS_SCORES, NULL },
+ { "CNV_DETAIL_SCORES", cg_eFileType_CNV_DETAILS_SCORES, NULL }, /*VDB-1404*/
{ "TUMOR_DETAILS_SCORES", cg_eFileType_TUMOR_DETAILS_SCORES, NULL },
{ "JUNCTIONS", cg_eFileType_JUNCTIONS, NULL },
{ "JUNCTION-DNBS", cg_eFileType_JUNCTION_DNBS, NULL },
diff --git a/tools/cg-load/f1_5.c b/tools/cg-load/f1_5.c
index f766a8b..3a0dab8 100644
--- a/tools/cg-load/f1_5.c
+++ b/tools/cg-load/f1_5.c
@@ -66,7 +66,7 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
/* undocumented */
{ "VAR-OLPL", cg_eFileType_VAR_OLPL, NULL },
- { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr15_Make },
};
rc_t CGFile15_Make(const CGFileType** self,
diff --git a/tools/cg-load/f1_5.c b/tools/cg-load/f1_7.c
similarity index 77%
copy from tools/cg-load/f1_5.c
copy to tools/cg-load/f1_7.c
index f766a8b..1970d79 100644
--- a/tools/cg-load/f1_5.c
+++ b/tools/cg-load/f1_7.c
@@ -23,13 +23,21 @@
* ===========================================================================
*/
-#include "factory-evidence-dnbs.h"
-#include "factory-evidence-intervals.h"
-#include "factory-mappings.h"
-#include "factory-reads.h"
-#include "factory-tag-lfr.h"
-#include "file.h"
+#include "factory-reads.h" /* CGReads15_Make */
+#include "factory-mappings.h" /* CGMappings15_Make */
+#include "factory-evidence-intervals.h" /* CGEvidenceIntervals15_Make */
+#include "factory-evidence-dnbs.h" /* CGEvidenceDnbs15_Make */
+#include "factory-tag-lfr.h" /* CGTagLfr15_Make */
+
+
+#include "file.h" /* CGFileType */
+
+
+/* File format v1.7
+ Software v1.12
+#FORMAT_VERSION 1.7 */
+
static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "READS", cg_eFileType_READS, CGReads15_Make },
@@ -47,8 +55,6 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "EVIDENCE-DNBS", cg_eFileType_EVIDENCE_DNBS, CGEvidenceDnbs15_Make },
{ "EVIDENCE-INTERVALS",
cg_eFileType_EVIDENCE_INTERVALS, CGEvidenceIntervals15_Make },
- { "COVERAGE-DISTRIBUTION", cg_eFileType_COVERAGE_DISTRIBUTION, NULL },
- { "COVERAGE-BY-GC", cg_eFileType_COVERAGE_BY_GC, NULL },
{ "DEPTH-OF-COVERAGE", cg_eFileType_DEPTH_OF_COVERAGE, NULL },
{ "INDEL-LENGTH-CODING", cg_eFileType_INDEL_LENGTH_CODING, NULL },
{ "INDEL-LENGTH", cg_eFileType_INDEL_LENGTH, NULL },
@@ -63,13 +69,26 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "TUMOR_DETAILS_SCORES", cg_eFileType_TUMOR_DETAILS_SCORES, NULL },
{ "JUNCTIONS", cg_eFileType_JUNCTIONS, NULL },
{ "JUNCTION-DNBS", cg_eFileType_JUNCTION_DNBS, NULL },
+ { "VAR-OLPL" , cg_eFileType_VAR_OLPL, NULL },
+ { "MEI" , cg_eFileType_MEI , NULL },
+
+ { "COVERAGE-BY-GC" , cg_eFileType_COVERAGE_BY_GC , NULL },
+ { "COVERAGE-DISTRIBUTION", cg_eFileType_COVERAGE_DISTRIBUTION, NULL },
+
+ /* semi-documented:
+ mentioned in "Data File Formats.File format v1.7.Software v1.12 */
+ { "COVERAGE-BY-GC-CODING", cg_eFileType_COVERAGE_BY_GC , NULL },
+ { "COVERAGE-DISTRIBUTION-CODING",
+ cg_eFileType_COVERAGE_DISTRIBUTION, NULL },
+
/* undocumented */
{ "VAR-OLPL", cg_eFileType_VAR_OLPL, NULL },
- { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr15_Make },
};
-rc_t CGFile15_Make(const CGFileType** self,
+
+rc_t CGFile17_Make(const CGFileType** self,
const char* type, const CGLoaderFile* file)
{
return CGLoaderFileMakeCGFileType(file, type,
diff --git a/tools/cg-load/f2_0.c b/tools/cg-load/f2_0.c
index 7af4181..ca5b938 100644
--- a/tools/cg-load/f2_0.c
+++ b/tools/cg-load/f2_0.c
@@ -69,8 +69,24 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "SV-EVENTS", cg_eFileType_SV_EVENTS, NULL },
{ "VAR-OLPL", cg_eFileType_VAR_OLPL, NULL },
{ "MEI", cg_eFileType_MEI, NULL },
+ /* from the documentation */
+ { "TUMOR-CNV-DETAILS", cg_eFileType_TUMOR_CNV_DETAILS, NULL },
+ { "DIPLOID-SOMATIC-CNV-SEGMENTS",
+ cg_eFileType_DIPLOID_SOMATIC_CNV_SEGMENTS, NULL },
+ { "COVERAGE-DISTRIBUTION-CODING",
+ cg_eFileType_COVERAGE_DISTRIBUTION_CODING, NULL },
+ { "COVERAGE-BY-GC-CODING", cg_eFileType_COVERAGE_BY_GC_CODING, NULL },
+ { "LIB-SMALL-GAPS-ROLLUP", cg_eFileType_LIB_SMALL_GAPS_ROLLUP, NULL },
- { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr15_Make },
+
+ /* VDB-1383 */
+ { "NONDIPLOID-SOMATIC-CNV-SEGMENTS",
+ cg_eFileType_NONDIPLOID_SOMATIC_CNV_SEGMENTS, NULL },
+ { "NONDIPLOID-SOMATIC-CNV-DETAILS",
+ cg_eFileType_NONDIPLOID_SOMATIC_CNV_DETAILS, NULL },
+ { "DIPLOID-SOMATIC-CNV-DETAIL-SCORES",
+ cg_eFileType_DIPLOID_SOMATIC_CNV_DETAIL_SCORES, NULL },
};
rc_t CGFile20_Make(const CGFileType** self,
diff --git a/tools/cg-load/f2_2.c b/tools/cg-load/f2_2.c
index 02c84c1..774f36f 100644
--- a/tools/cg-load/f2_2.c
+++ b/tools/cg-load/f2_2.c
@@ -84,9 +84,9 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "LIB-SMALL-GAPS-ROLLUP", cg_eFileType_LIB_SMALL_GAPS_ROLLUP, NULL },
/* from Cancer Sequencing Service Data File Formats file test */
{ "DIPLOID-SOMATIC-CNV-DETAILSCORES",
- cg_eFileType_DIPLOID_SOMATIC_CNV_DETAILSCORES, NULL },
+ cg_eFileType_DIPLOID_SOMATIC_CNV_DETAIL_SCORES, NULL },
- { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr15_Make },
};
rc_t CGFile22_Make(const CGFileType** self,
diff --git a/tools/cg-load/f2_5.c b/tools/cg-load/f2_5.c
new file mode 100644
index 0000000..48c414d
--- /dev/null
+++ b/tools/cg-load/f2_5.c
@@ -0,0 +1,98 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*/
+
+#include "factory-evidence-dnbs.h"
+#include "factory-evidence-intervals.h"
+#include "factory-mappings.h"
+#include "factory-reads.h"
+#include "factory-tag-lfr.h"
+
+#include "file.h"
+
+static const CGFileTypeFactory cg_ETypeXX_names[] = {
+ { "READS" , cg_eFileType_READS, CGReads25_Make },
+ { "MAPPINGS" , cg_eFileType_MAPPINGS, CGMappings25_Make },
+ { "LIB-DNB" , cg_eFileType_LIB_DNB, NULL },
+ { "LIB-MATE-GAPS" , cg_eFileType_LIB_MATE_GAPS, NULL },
+ { "LIB-SEQDEP-GAPS" , cg_eFileType_LIB_SEQDEP_GAPS, NULL },
+ { "REFMETRICS" , cg_eFileType_REFMETRICS, NULL },
+ { "IDENTIFIER-MAPPING" , cg_eFileType_IDENTIFIER_MAPPING, NULL },
+ { "DBSNP-TO-CGI" , cg_eFileType_DBSNP_TO_CGI, NULL },
+ { "GENE-ANNOTATION" , cg_eFileType_GENE_ANNOTATION, NULL },
+ { "SUMMARY-REPORT" , cg_eFileType_SUMMARY_REPORT, NULL },
+ { "VAR-ANNOTATION" , cg_eFileType_VAR_ANNOTATION, NULL },
+ { "GENE-VAR-SUMMARY-REPORT", cg_eFileType_GENE_VAR_SUMMARY_REPORT, NULL },
+ { "EVIDENCE-CORRELATION" , cg_eFileType_EVIDENCE_CORRELATION, NULL },
+ { "EVIDENCE-DNBS", cg_eFileType_EVIDENCE_DNBS, CGEvidenceDnbs25_Make },
+ { "EVIDENCE-INTERVALS" , cg_eFileType_EVIDENCE_INTERVALS,
+ CGEvidenceIntervals25_Make },
+ { "COVERAGE-DISTRIBUTION" , cg_eFileType_COVERAGE_DISTRIBUTION, NULL },
+ { "COVERAGE-BY-GC" , cg_eFileType_COVERAGE_BY_GC, NULL },
+ { "DEPTH-OF-COVERAGE" , cg_eFileType_DEPTH_OF_COVERAGE, NULL },
+ { "INDEL-LENGTH-CODING" , cg_eFileType_INDEL_LENGTH_CODING, NULL },
+ { "INDEL-LENGTH" , cg_eFileType_INDEL_LENGTH, NULL },
+ { "SUBSTITUTION-LENGTH-CODING"
+ , cg_eFileType_SUBSTITUTION_LENGTH_CODING, NULL },
+ { "SUBSTITUTION-LENGTH" , cg_eFileType_SUBSTITUTION_LENGTH, NULL },
+ { "CNV_SEGMENTS" , cg_eFileType_CNV_SEGMENTS, NULL },
+ { "CNV-SEGMENTS" , cg_eFileType_CNV_SEGMENTS, NULL },
+ { "TUMOR_CNV_SEGMENTS" , cg_eFileType_TUMOR_CNV_SEGMENTS, NULL },
+ { "TUMOR-CNV-SEGMENTS" , cg_eFileType_TUMOR_CNV_SEGMENTS, NULL },
+ { "CNV_DETAILS_SCORES" , cg_eFileType_CNV_DETAILS_SCORES, NULL },
+ { "CNV-DETAILS-SCORES" , cg_eFileType_CNV_DETAILS_SCORES, NULL },
+ { "CNV-DETAIL-SCORES" , cg_eFileType_CNV_DETAILS_SCORES, NULL },
+ { "TUMOR_DETAILS_SCORES" , cg_eFileType_TUMOR_DETAILS_SCORES, NULL },
+ { "TUMOR-DETAILS-SCORES" , cg_eFileType_TUMOR_DETAILS_SCORES, NULL },
+ { "NONDIPLOID-SOMATIC-CNV-SEGMENTS",
+ cg_eFileType_NONDIPLOID_SOMATIC_CNV_SEGMENTS, NULL },
+ { "NONDIPLOID-SOMATIC-CNV-DETAILS",
+ cg_eFileType_NONDIPLOID_SOMATIC_CNV_DETAILS, NULL },
+ { "JUNCTIONS" , cg_eFileType_JUNCTIONS, NULL },
+ { "JUNCTION-DNBS" , cg_eFileType_JUNCTION_DNBS, NULL },
+ { "SV-EVENTS" , cg_eFileType_SV_EVENTS, NULL },
+ { "VAR-OLPL" , cg_eFileType_VAR_OLPL, NULL },
+ { "MEI" , cg_eFileType_MEI, NULL },
+ /* from Standard Sequencing Service Data File Formats file test */
+ { "TUMOR-CNV-DETAILS", cg_eFileType_TUMOR_CNV_DETAILS, NULL },
+ { "DIPLOID-SOMATIC-CNV-SEGMENTS"
+ , cg_eFileType_DIPLOID_SOMATIC_CNV_SEGMENTS, NULL },
+ { "COVERAGE-DISTRIBUTION-CODING"
+ , cg_eFileType_COVERAGE_DISTRIBUTION_CODING, NULL },
+ { "COVERAGE-BY-GC-CODING", cg_eFileType_COVERAGE_BY_GC_CODING, NULL },
+ { "LIB-SMALL-GAPS-ROLLUP", cg_eFileType_LIB_SMALL_GAPS_ROLLUP, NULL },
+ /* from Cancer Sequencing Service Data File Formats file test */
+ { "DIPLOID-SOMATIC-CNV-DETAILSCORES"
+ , cg_eFileType_DIPLOID_SOMATIC_CNV_DETAIL_SCORES, NULL },
+
+ { "TAG_LFR" , cg_eFileType_TAG_LFR, CGTagLfr25_Make },
+};
+
+rc_t CGFile25_Make(const CGFileType** self,
+ const char* type, const CGLoaderFile* file)
+{
+ return CGLoaderFileMakeCGFileType(file, type,
+ cg_ETypeXX_names, sizeof cg_ETypeXX_names / sizeof cg_ETypeXX_names[0],
+ self);
+}
diff --git a/tools/cg-load/factory-evidence-dnbs.c b/tools/cg-load/factory-evidence-dnbs.c
index c85c810..b48af38 100644
--- a/tools/cg-load/factory-evidence-dnbs.c
+++ b/tools/cg-load/factory-evidence-dnbs.c
@@ -22,20 +22,24 @@
*
* ===========================================================================
*/
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
+
+#include "debug.h"
+
+#include "factory-cmn.h"
+#include "factory-evidence-dnbs.h"
typedef struct CGEvidenceDnbs15 CGEvidenceDnbs15;
#define CGFILETYPE_IMPL CGEvidenceDnbs15
#include "file.h"
-#include "factory-cmn.h"
-#include "factory-evidence-dnbs.h"
-#include "debug.h"
+
+#include <klib/printf.h>
+#include <klib/rc.h>
+
+#include <sysalloc.h>
#include <assert.h>
-#include <string.h>
#include <stdlib.h>
+#include <string.h>
struct CGEvidenceDnbs15 {
CGFileType dad;
@@ -86,6 +90,68 @@ rc_t CC CGEvidenceDnbs15_Header(const CGEvidenceDnbs15* cself, const char* buf,
return rc;
}
+static rc_t CC CGEvidenceDnbs25_Header(
+ const CGEvidenceDnbs15* cself, const char* buf, const size_t len)
+{
+ rc_t rc = 0;
+ size_t slen;
+ CGEvidenceDnbs15* self = (CGEvidenceDnbs15*)cself;
+
+ /* from SRA-2617 files */
+ if (strncmp("APPROVAL\t", buf, slen = 9) == 0) {
+ }
+ else if (strncmp("TITLE\t", buf, slen = 6) == 0) {
+ }
+ else if (strncmp("ADDRESS\t", buf, slen = 8) == 0) {
+ }
+
+ /* From Table 1: Header Metadata Present in all Data Files */
+ else if (strncmp("CUSTOMER_SAMPLE_ID\t", buf, slen = 19) == 0) {
+ }
+ else if (strncmp("SAMPLE_SOURCE\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("REPORTED_GENDER\t", buf, slen = 16) == 0) {
+ }
+ else if (strncmp("CALLED_GENDER\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("TUMOR_STATUS\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_TYPE\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_SOURCE\t", buf, slen = 13) == 0) {
+ }
+
+ else if (strncmp("ASSEMBLY_ID\t", buf, slen = 12) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->assembly_id, sizeof(self->assembly_id));
+ }
+ else if (strncmp("CHROMOSOME\t", buf, slen = 11) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->chromosome, sizeof(self->chromosome));
+ }
+ else if (strncmp("GENERATED_AT\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_at, sizeof(self->generated_at));
+ }
+ else if (strncmp("GENERATED_BY\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_by, sizeof(self->generated_by));
+ }
+ else if (strncmp("SAMPLE\t", buf, slen = 7) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->sample, sizeof(self->sample));
+ }
+ else if (strncmp("SOFTWARE_VERSION\t", buf, slen = 17) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->software_version, sizeof(self->software_version));
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcName, rcUnrecognized);
+ }
+
+ return rc;
+}
+
static
rc_t CGEvidenceDnbs15_GetAssemblyId(const CGEvidenceDnbs15* cself, const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id)
{
@@ -116,8 +182,9 @@ rc_t CGEvidenceDnbs15_GetChromosome(const CGEvidenceDnbs15* cself, const CGFIELD
return 0;
}
-static
-rc_t CC CGEvidenceDnbs_Read(const CGEvidenceDnbs15* cself, const char* interval_id, TEvidenceDnbsData* data, int score_allele_num)
+static rc_t CC CGEvidenceDnbs_Read(
+ const CGEvidenceDnbs15* cself, const char* interval_id,
+ TEvidenceDnbsData* data, int score_allele_num)
{
rc_t rc = 0;
TEvidenceDnbsData_dnb* m = NULL;
@@ -267,6 +334,189 @@ rc_t CC CGEvidenceDnbs_Read(const CGEvidenceDnbs15* cself, const char* interval_
return rc;
}
+static rc_t CC CGEvidenceDnbs25_Read(const CGEvidenceDnbs15 *cself,
+ const char *interval_id, TEvidenceDnbsData *data)
+{
+ const int score_allele_num = 4;
+ rc_t rc = 0;
+ TEvidenceDnbsData_dnb* m = NULL;
+ static TEvidenceDnbsData_dnb next_rec;
+ static char next_interval_id[32] = "";
+
+ /* local copy of unused TEvidenceDnbsData_dnb struct elements */
+ char reference_alignment[CG_EVDNC_ALLELE_CIGAR_LEN];
+ INSDC_coord_zero mate_offset_in_reference;
+ char mate_reference_alignment[CG_EVDNC_ALLELE_CIGAR_LEN];
+ uint16_t score_allele[4] = {0, 0, 0, 0}; /* v >= 2.0 has ScoreAllele[0-3] */
+ char qual[CG_EVDNC_SPOT_LEN];
+
+ bool lfr = false;
+ assert(cself->file->cg_file);
+ lfr = cself->file->cg_file->libraryType == cg_eLibraryType_PureLFR;
+
+ strcpy(data->interval_id, interval_id);
+ data->qty = 0;
+ /* already read one rec for this interval_id */
+ if( next_interval_id[0] != '\0' ) {
+ if( strcmp(next_interval_id, interval_id) != 0 ) {
+ /* nothing todo since next interval id is different */
+ return rc;
+ }
+ m = &data->dnbs[data->qty++];
+ memcpy(m, &next_rec, sizeof(next_rec));
+ DEBUG_MSG(10, ("%3u evidenceDnbs: '%s'\t'%s'\t'%s'\t'%s'\t%u\t%lu\t%hu\t%c\t%c\t%i\t'%.*s'"
+ "\t%i\tnot_used\t0\tnot_used\t%c\t0\t0\t0\t'%.*s'\t'--'\n",
+ data->qty, next_interval_id, m->chr, m->slide, m->lane, m->file_num_in_lane,
+ m->dnb_offset_in_lane_file, m->allele_index, m->side, m->strand, m->offset_in_allele,
+ m->allele_alignment_length, m->allele_alignment, m->offset_in_reference,
+ m->mapping_quality, m->read_len, m->read));
+ }
+ do {
+ int i = 0;
+ char tmp[2];
+ CG_LINE_START(cself->file, b, len, p);
+ if( b == NULL || len == 0 ) {
+ next_interval_id[0] = '\0';
+ break; /* EOF */
+ }
+ if( data->qty >= data->max_qty ) {
+ TEvidenceDnbsData_dnb* x;
+ data->max_qty += 100;
+ x = realloc(data->dnbs, sizeof(*(data->dnbs)) * data->max_qty);
+ if( x == NULL ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcMemory, rcExhausted);
+ break;
+ }
+ data->dnbs = x;
+ }
+ m = &data->dnbs[data->qty++];
+
+ /*DEBUG_MSG(10, ("%2hu evidenceDnbs: '%.*s'\n", data->qty, len, b));*/
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, next_interval_id, sizeof(next_interval_id));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, m->chr, sizeof(m->chr));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, m->slide, sizeof(m->slide));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, m->lane, sizeof(m->lane));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2u32(b, p - b, &m->file_num_in_lane);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2u64(b, p - b, &m->dnb_offset_in_lane_file);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2u16(b, p - b, &m->allele_index);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, tmp, sizeof(tmp));
+ if( tmp[0] != 'L' && tmp[0] != 'R' ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ m->side = tmp[0];
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, tmp, sizeof(tmp));
+ if( tmp[0] != '+' && tmp[0] != '-' ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ m->strand = tmp[0];
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i32(b, p - b, &m->offset_in_allele);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, m->allele_alignment, sizeof(m->allele_alignment));
+ m->allele_alignment_length = p - b;
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i32(b, p - b, &m->offset_in_reference);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, reference_alignment, sizeof(reference_alignment));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i32(b, p - b, &mate_offset_in_reference);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, mate_reference_alignment, sizeof(mate_reference_alignment));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, tmp, sizeof(tmp));
+ if( tmp[0] < 33 || tmp[0] > 126 ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ m->mapping_quality = tmp[0];
+ for (i = 0; i < score_allele_num; ++i) {
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2u16(b, p - b, &score_allele[i]);
+ if (rc != 0) {
+ score_allele[i] = 0;
+ rc = 0;
+ }
+ }
+ CG_LINE_NEXT_FIELD(b, len, p);
+ m->read_len = p - b;
+ rc = str2buf(b, m->read_len, m->read, sizeof(m->read));
+
+ /* Scores */
+ if (lfr) {
+ CG_LINE_NEXT_FIELD(b, len, p);
+ }
+ else {
+ CG_LINE_LAST_FIELD(b, len, p);
+ }
+ if (m->read_len != p - b) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInconsistent);
+ }
+ else {
+ rc = str2buf(b, p - b, qual, sizeof(qual));
+ }
+
+ if (lfr) {
+ /* Standard Sequencing Service Data File Formats (File format v2.5)
+ states that Scores is the last column in evidenceDnbs file
+ but in 2.5 submission mentioned in SRA-2617
+ there are additionally wellId and wellScore columns.
+ #LIBRARY_TYPE is Pure LFR for this submission. */
+
+ /* wellId */
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2u16(b, p - b, &m->wellId);
+ if (rc == 0 && (m->wellId < 0 || m->wellId > 384)) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+
+ /* wellScore */
+ CG_LINE_LAST_FIELD(b, len, p);
+ }
+
+ ((CGEvidenceDnbs15*)cself)->records++;
+ if( strcmp(next_interval_id, data->interval_id) != 0 ) {
+ if (score_allele_num == 3) {
+ DEBUG_MSG(10, ("%3u evidenceDnbs: '%s'\t'%s'\t'%s'\t'%s'\t%u\t%lu\t%hu\t%c\t%c\t%i\t'%.*s'"
+ "\t%i\t'%s'\t%i\t'%s'\t%c\t%hu\t%hu\t%hu\t'%.*s'\t'%s'\n",
+ data->qty, next_interval_id, m->chr, m->slide, m->lane, m->file_num_in_lane,
+ m->dnb_offset_in_lane_file, m->allele_index, m->side, m->strand, m->offset_in_allele,
+ m->allele_alignment_length, m->allele_alignment, m->offset_in_reference,
+ reference_alignment, mate_offset_in_reference, mate_reference_alignment,
+ m->mapping_quality, score_allele[0], score_allele[1], score_allele[2], m->read_len, m->read, qual));
+ }
+ else if (score_allele_num == 4) {
+ DEBUG_MSG(10, ("%3u evidenceDnbs: '%s'\t'%s'\t'%s'\t'%s'\t%u\t%lu\t%hu\t%c\t%c\t%i\t'%.*s'"
+ "\t%i\t'%s'\t%i\t'%s'\t%c\t%hu\t%hu\t%hu\t%hu\t'%.*s'\t'%s'\n",
+ data->qty, next_interval_id, m->chr, m->slide, m->lane, m->file_num_in_lane,
+ m->dnb_offset_in_lane_file, m->allele_index, m->side, m->strand, m->offset_in_allele,
+ m->allele_alignment_length, m->allele_alignment, m->offset_in_reference,
+ reference_alignment, mate_offset_in_reference, mate_reference_alignment,
+ m->mapping_quality, score_allele[0], score_allele[1], score_allele[2], score_allele[3], m->read_len, m->read, qual));
+ }
+ else { assert(0); }
+ }
+ CG_LINE_END();
+ if( next_interval_id[0] == '\0' ) {
+ break;
+ }
+ if( strcmp(next_interval_id, data->interval_id) != 0 ) {
+ /* next record is from next interval, remeber it and stop */
+ memcpy(&next_rec, m, sizeof(next_rec));
+ data->qty--;
+ break;
+ }
+ } while( rc == 0 );
+ return rc;
+}
+
static
rc_t CC CGEvidenceDnbs15_Read(const CGEvidenceDnbs15* self, const char* interval_id, TEvidenceDnbsData* data)
{ return CGEvidenceDnbs_Read(self, interval_id, data, 3); }
@@ -311,6 +561,23 @@ static const CGFileType_vt CGEvidenceDnbs20_vt =
CGEvidenceDnbs15_Release
};
+static const CGFileType_vt CGEvidenceDnbs25_vt = {
+ CGEvidenceDnbs25_Header,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ CGEvidenceDnbs25_Read,
+ NULL, /* tag_lfr */
+ CGEvidenceDnbs15_GetAssemblyId,
+ NULL,
+ NULL,
+ NULL,
+ CGEvidenceDnbs15_GetSample,
+ CGEvidenceDnbs15_GetChromosome,
+ CGEvidenceDnbs15_Release
+};
+
static
rc_t CC CGEvidenceDnbs_Make(const CGFileType** cself, const CGLoaderFile* file,
const CGFileType_vt* vt)
@@ -351,3 +618,8 @@ rc_t CC CGEvidenceDnbs20_Make(const CGFileType** self, const CGLoaderFile* file)
rc_t CC CGEvidenceDnbs22_Make(const CGFileType** self, const CGLoaderFile* file)
{ return CGEvidenceDnbs_Make(self, file, &CGEvidenceDnbs20_vt); }
+
+rc_t CC CGEvidenceDnbs25_Make(const CGFileType** self, const CGLoaderFile* file)
+{
+ return CGEvidenceDnbs_Make(self, file, &CGEvidenceDnbs25_vt);
+}
diff --git a/tools/cg-load/factory-evidence-dnbs.h b/tools/cg-load/factory-evidence-dnbs.h
index 4ba4b4f..1fa0194 100644
--- a/tools/cg-load/factory-evidence-dnbs.h
+++ b/tools/cg-load/factory-evidence-dnbs.h
@@ -39,5 +39,8 @@ rc_t CC CGEvidenceDnbs20_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
rc_t CC CGEvidenceDnbs22_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
+rc_t CC CGEvidenceDnbs25_Make(const struct CGFileType** self,
+ const struct CGLoaderFile* file);
+
#endif /* _tools_cg_load_factory_evidence_dnbs_h_ */
diff --git a/tools/cg-load/factory-evidence-intervals.c b/tools/cg-load/factory-evidence-intervals.c
index f672084..339a02d 100644
--- a/tools/cg-load/factory-evidence-intervals.c
+++ b/tools/cg-load/factory-evidence-intervals.c
@@ -22,16 +22,19 @@
*
* ===========================================================================
*/
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
+
+#include "debug.h"
+#include "factory-cmn.h"
+#include "factory-evidence-intervals.h"
typedef struct CGEvidenceIntervals15 CGEvidenceIntervals15;
#define CGFILETYPE_IMPL CGEvidenceIntervals15
#include "file.h"
-#include "factory-cmn.h"
-#include "factory-evidence-intervals.h"
-#include "debug.h"
+
+#include <klib/printf.h>
+#include <klib/rc.h>
+
+#include <sysalloc.h>
#include <assert.h>
#include <stdlib.h>
@@ -75,6 +78,68 @@ rc_t CC CGEvidenceIntervals15_Header(const CGEvidenceIntervals15* cself, const c
return rc;
}
+static rc_t CC CGEvidenceIntervals25_Header(const CGEvidenceIntervals15* cself,
+ const char* buf, const size_t len)
+{
+ rc_t rc = 0;
+ size_t slen = 0;
+ CGEvidenceIntervals15* self = (CGEvidenceIntervals15*)cself;
+
+ /* from SRA-2617 files */
+ if (strncmp("APPROVAL\t", buf, slen = 9) == 0) {
+ }
+ else if (strncmp("TITLE\t", buf, slen = 6) == 0) {
+ }
+ else if (strncmp("ADDRESS\t", buf, slen = 8) == 0) {
+ }
+
+ /* From Table 1: Header Metadata Present in all Data Files */
+ else if (strncmp("CUSTOMER_SAMPLE_ID\t", buf, slen = 19) == 0) {
+ }
+ else if (strncmp("SAMPLE_SOURCE\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("REPORTED_GENDER\t", buf, slen = 16) == 0) {
+ }
+ else if (strncmp("CALLED_GENDER\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("TUMOR_STATUS\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_TYPE\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_SOURCE\t", buf, slen = 13) == 0) {
+ }
+
+ else if (strncmp("ASSEMBLY_ID\t", buf, slen = 12) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->assembly_id, sizeof(self->assembly_id));
+ }
+ else if (strncmp("CHROMOSOME\t", buf, slen = 11) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->chromosome, sizeof(self->chromosome));
+ }
+ else if (strncmp("GENERATED_AT\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_at, sizeof(self->generated_at));
+ }
+ else if (strncmp("GENERATED_BY\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_by, sizeof(self->generated_by));
+ }
+ else if (strncmp("SAMPLE\t", buf, slen = 7) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->sample, sizeof(self->sample));
+ }
+ else if (strncmp("SOFTWARE_VERSION\t", buf, slen = 17) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->software_version, sizeof(self->software_version));
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcName, rcUnrecognized);
+ }
+
+ return rc;
+}
+
static
rc_t CGEvidenceIntervals15_GetAssemblyId(const CGEvidenceIntervals15* cself, const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id)
{
@@ -262,6 +327,23 @@ static const CGFileType_vt CGEvidenceIntervals20_vt =
CGEvidenceIntervals15_Release
};
+static const CGFileType_vt CGEvidenceIntervals25_vt = {
+ CGEvidenceIntervals25_Header,
+ NULL,
+ NULL,
+ NULL,
+ CGEvidenceIntervals20_Read,
+ NULL,
+ NULL, /* tag_lfr */
+ CGEvidenceIntervals15_GetAssemblyId,
+ NULL,
+ NULL,
+ NULL,
+ CGEvidenceIntervals15_GetSample,
+ CGEvidenceIntervals15_GetChromosome,
+ CGEvidenceIntervals15_Release
+};
+
static
rc_t CGEvidenceIntervals_Make(const CGFileType** cself, const CGLoaderFile* file, const CGFileType_vt* vt)
{
@@ -306,3 +388,9 @@ rc_t CGEvidenceIntervals20_Make(const CGFileType** self,
rc_t CGEvidenceIntervals22_Make(const CGFileType** self,
const CGLoaderFile* file)
{ return CGEvidenceIntervals20_Make(self, file); }
+
+rc_t CGEvidenceIntervals25_Make(const CGFileType **self,
+ const CGLoaderFile *file)
+{
+ return CGEvidenceIntervals_Make(self, file, &CGEvidenceIntervals25_vt);
+}
diff --git a/tools/cg-load/factory-evidence-intervals.h b/tools/cg-load/factory-evidence-intervals.h
index 2f84fd4..4f781f1 100644
--- a/tools/cg-load/factory-evidence-intervals.h
+++ b/tools/cg-load/factory-evidence-intervals.h
@@ -39,5 +39,8 @@ rc_t CGEvidenceIntervals20_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
rc_t CGEvidenceIntervals22_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
+rc_t CGEvidenceIntervals25_Make(const struct CGFileType** self,
+ const struct CGLoaderFile* file);
+
#endif /* _tools_cg_load_factory_evidence_intervals_h_ */
diff --git a/tools/cg-load/factory-file.h b/tools/cg-load/factory-file.h
index 8931fca..c19c398 100644
--- a/tools/cg-load/factory-file.h
+++ b/tools/cg-load/factory-file.h
@@ -35,9 +35,13 @@ rc_t CGFile13_Make(const struct CGFileType** self,
const char* type, const struct CGLoaderFile* file);
rc_t CGFile15_Make(const struct CGFileType** self,
const char* type, const struct CGLoaderFile* file);
+rc_t CGFile17_Make(const struct CGFileType** self,
+ const char* type, const struct CGLoaderFile* file);
rc_t CGFile20_Make(const struct CGFileType** self,
const char* type, const struct CGLoaderFile* file);
rc_t CGFile22_Make(const struct CGFileType** self,
const char* type, const struct CGLoaderFile* file);
+rc_t CGFile25_Make(const struct CGFileType** self,
+ const char* type, const struct CGLoaderFile* file);
#endif /* _tools_cg_load_factory_file_h_ */
diff --git a/tools/cg-load/factory-mappings.c b/tools/cg-load/factory-mappings.c
index 6e00f08..8434c84 100644
--- a/tools/cg-load/factory-mappings.c
+++ b/tools/cg-load/factory-mappings.c
@@ -22,19 +22,24 @@
*
* ===========================================================================
*/
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
+
+
+#include "debug.h"
+#include "factory-cmn.h"
+#include "factory-mappings.h"
typedef struct CGMappings15 CGMappings15;
#define CGFILETYPE_IMPL CGMappings15
#include "file.h"
-#include "factory-cmn.h"
-#include "factory-mappings.h"
-#include "debug.h"
-#include <string.h>
+#include <klib/printf.h>
+#include <klib/rc.h>
+
+#include <sysalloc.h>
+
#include <stdlib.h>
+#include <string.h>
+
struct CGMappings15 {
CGFileType dad;
@@ -97,6 +102,81 @@ rc_t CC CGMappings15_Header(const CGMappings15* cself, const char* buf, const si
return rc;
}
+static rc_t CC CGMappings25_Header(const CGMappings15* cself,
+ const char* buf, const size_t len)
+{
+ rc_t rc = 0;
+ size_t slen = 0;
+ CGMappings15* self = (CGMappings15*)cself;
+
+ /* from SRA-2617 files */
+ if (strncmp("APPROVAL\t", buf, slen = 9) == 0) {
+ }
+ else if (strncmp("TITLE\t", buf, slen = 6) == 0) {
+ }
+ else if (strncmp("ADDRESS\t", buf, slen = 8) == 0) {
+ }
+
+ /* From Table 1: Header Metadata Present in all Data Files */
+ else if (strncmp("CUSTOMER_SAMPLE_ID\t", buf, slen = 19) == 0) {
+ }
+ else if (strncmp("SAMPLE_SOURCE\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("REPORTED_GENDER\t", buf, slen = 16) == 0) {
+ }
+ else if (strncmp("CALLED_GENDER\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("TUMOR_STATUS\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_TYPE\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_SOURCE\t", buf, slen = 13) == 0) {
+ }
+
+ else if (strncmp("ASSEMBLY_ID\t", buf, slen = 12) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->assembly_id, sizeof(self->assembly_id));
+ }
+ else if (strncmp("BATCH_FILE_NUMBER\t", buf, slen = 18) == 0) {
+ rc = str2u32(&buf[slen], len - slen, &self->batch_file_number);
+ if (self->batch_file_number < 1 ) {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcItem, rcOutofrange);
+ }
+ }
+ else if (strncmp("GENERATED_AT\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_at, sizeof(self->generated_at));
+ }
+ else if (strncmp("GENERATED_BY\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_by, sizeof(self->generated_by));
+ }
+ else if (strncmp("LANE\t", buf, slen = 5) == 0) {
+ rc = str2buf(&buf[slen], len - slen, self->lane, sizeof(self->lane));
+ }
+ else if (strncmp("LIBRARY\t", buf, slen = 8) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->library, sizeof(self->library));
+ }
+ else if (strncmp("SAMPLE\t", buf, slen = 7) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->sample, sizeof(self->sample));
+ }
+ else if (strncmp("SLIDE\t", buf, slen = 6) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->slide, sizeof(self->slide));
+ }
+ else if (strncmp("SOFTWARE_VERSION\t", buf, slen = 17) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->software_version, sizeof(self->software_version));
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcName, rcUnrecognized);
+ }
+
+ return rc;
+}
+
static
rc_t CGMappings15_GetAssemblyId(const CGMappings15* cself, const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id)
{
@@ -248,6 +328,103 @@ rc_t CC CGMappings22_Read(const CGMappings15* cself, TMappingsData* data)
DEBUG_MSG(10, ("mappings %4u: %u\t'%s'\t%u\t%i\t%i\t%i\t%c\t%u\t%c\n",
data->map_qty - 1, m->flags, m->chr, m->offset,
m->gap[0], m->gap[1], m->gap[2], m->weight, m->mate, armWeight));
+ armWeight += 0; /* shut up used variable warning in release build */
+ CG_LINE_END();
+ } while( rc == 0 && !(m->flags & cg_eLastDNBRecord) );
+ if (rc == 0) {
+ unsigned i;
+ unsigned const n = data->map_qty;
+
+ for (i = 0; i != n && rc == 0; ++i) {
+ unsigned const mate = data->map[i].mate;
+
+ if (mate > n)
+ data->map[i].mate = n;
+ }
+ }
+ return rc;
+}
+
+static rc_t CC CGMappings25_Read(
+ const CGMappings15 *cself, TMappingsData *data)
+{
+ rc_t rc = 0;
+ TMappingsData_map* m = NULL;
+
+ assert(cself);
+
+ data->map_qty = 0;
+ do {
+ char tmp[2];
+ char armWeight = '\0';
+ CG_LINE_START(cself->file, b, len, p);
+ if( b == NULL || len == 0 ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInsufficient);
+ break;
+ }
+ m = &data->map[data->map_qty];
+ m->saved = false;
+ /*DEBUG_MSG(10, ("mappings %4u: '%.*s'\n", data->map_qty, len, b));*/
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2u16(b, p - b, &m->flags);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, m->chr, sizeof(m->chr));
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i32(b, p - b, &m->offset);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i16(b, p - b, &m->gap[0]);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i16(b, p - b, &m->gap[1]);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2i16(b, p - b, &m->gap[2]);
+ CG_LINE_NEXT_FIELD(b, len, p);
+ rc = str2buf(b, p - b, tmp, sizeof(tmp));
+ if( tmp[0] < 33 || tmp[0] > 126 ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ m->weight = tmp[0];
+ CG_LINE_NEXT_FIELD(b, len, p);
+ if( (rc = str2u32(b, p - b, &m->mate)) != 0 ) {
+ } else if( m->flags > 7 ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ } else if( ++data->map_qty >= CG_MAPPINGS_MAX ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcBuffer, rcInsufficient);
+ }
+ CG_LINE_LAST_FIELD(b, len, p);
+ rc = str2buf(b, p - b, tmp, sizeof(tmp));
+ if (rc == 0 && (tmp[0] < 33 || tmp[0] > 126)) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ armWeight = tmp[0]; /* ignore armWeight */
+ ((CGMappings15*)cself)->records++;
+ DEBUG_MSG(10, ("mappings %4u: %u\t'%s'\t%u\t%i\t%i\t%i\t%c\t%u\t%c\n",
+ data->map_qty - 1, m->flags, m->chr, m->offset,
+ m->gap[0], m->gap[1], m->gap[2], m->weight, m->mate, armWeight));
+ armWeight += 0; /* shut up used variable warning in release build */
+
+ if (rc == 0) {
+ assert(m->gap[2] == 0);
+ if (m->gap[2] != 0) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ if (m->flags & cg_eRevDnbStrand) {
+ assert(m->gap[0] == -1);
+ if (m->gap[0] != -1) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ }
+ else {
+ assert(m->gap[1] == -1);
+ if (m->gap[1] != -1) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ }
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Bad gap value in mapping file (File format >= v2.5)");
+ }
+ }
+
CG_LINE_END();
} while( rc == 0 && !(m->flags & cg_eLastDNBRecord) );
if (rc == 0) {
@@ -300,6 +477,23 @@ static const CGFileType_vt CGMappings22_vt =
CGMappings15_Release
};
+static const CGFileType_vt CGMappings25_vt = {
+ CGMappings25_Header,
+ NULL,
+ NULL,
+ CGMappings25_Read,
+ NULL,
+ NULL,
+ NULL, /* tag_lfr */
+ CGMappings15_GetAssemblyId,
+ CGMappings15_GetSlide,
+ CGMappings15_GetLane,
+ CGMappings15_GetBatchFileNumber,
+ NULL,
+ NULL,
+ CGMappings15_Release
+};
+
static
rc_t CC CGMappings_Make(const CGFileType** cself,
const CGLoaderFile* file, const CGFileType_vt* vt)
@@ -338,3 +532,8 @@ rc_t CC CGMappings20_Make(const CGFileType** self, const CGLoaderFile* file)
rc_t CC CGMappings22_Make(const CGFileType** self, const CGLoaderFile* file)
{ return CGMappings_Make(self, file, &CGMappings22_vt); }
+
+rc_t CC CGMappings25_Make(const CGFileType **self, const CGLoaderFile *file)
+{
+ return CGMappings_Make(self, file, &CGMappings25_vt);
+}
diff --git a/tools/cg-load/factory-mappings.h b/tools/cg-load/factory-mappings.h
index cc283b7..428352b 100644
--- a/tools/cg-load/factory-mappings.h
+++ b/tools/cg-load/factory-mappings.h
@@ -39,5 +39,8 @@ rc_t CC CGMappings20_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
rc_t CC CGMappings22_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
+rc_t CC CGMappings25_Make(const struct CGFileType** self,
+ const struct CGLoaderFile* file);
+
#endif /* _tools_cg_load_factory_mappings_h_ */
diff --git a/tools/cg-load/factory-reads.c b/tools/cg-load/factory-reads.c
index e0fa92b..be194e5 100644
--- a/tools/cg-load/factory-reads.c
+++ b/tools/cg-load/factory-reads.c
@@ -22,20 +22,23 @@
*
* ===========================================================================
*/
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
+
+#include "debug.h"
+#include "factory-cmn.h"
+#include "factory-reads.h"
typedef struct CGReads15 CGReads15;
#define CGFILETYPE_IMPL CGReads15
#include "file.h"
-#include "factory-cmn.h"
-#include "factory-reads.h"
-#include "debug.h"
+
+#include <klib/printf.h>
+#include <klib/rc.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
#include <stdlib.h>
#include <string.h>
-#include <os-native.h>
struct CGReads15 {
CGFileType dad;
@@ -95,6 +98,87 @@ rc_t CC CGReads15_Header(const CGReads15* cself, const char* buf, const size_t l
return rc;
}
+static rc_t CC CGReads25_Header(const CGReads15* cself,
+ const char* buf, const size_t len)
+{
+ rc_t rc = 0;
+ size_t slen = 0;
+ CGReads15* self = (CGReads15*)cself;
+
+ /* from SRA-2617 files */
+ if (strncmp("APPROVAL\t", buf, slen = 9) == 0) {
+ }
+ else if (strncmp("TITLE\t", buf, slen = 6) == 0) {
+ }
+ else if (strncmp("ADDRESS\t", buf, slen = 8) == 0) {
+ }
+
+ /* From Table 1: Header Metadata Present in all Data Files */
+ else if (strncmp("CUSTOMER_SAMPLE_ID\t", buf, slen = 19) == 0) {
+ }
+ else if (strncmp("SAMPLE_SOURCE\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("REPORTED_GENDER\t", buf, slen = 16) == 0) {
+ }
+ else if (strncmp("CALLED_GENDER\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("TUMOR_STATUS\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_TYPE\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_SOURCE\t", buf, slen = 13) == 0) {
+ }
+
+ else if (strncmp("ASSEMBLY_ID\t", buf, slen = 12) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->assembly_id, sizeof(self->assembly_id));
+ }
+ else if (strncmp("BATCH_FILE_NUMBER\t", buf, slen = 18) == 0) {
+ rc = str2u32(&buf[slen], len - slen, &self->batch_file_number);
+ if (self->batch_file_number < 1) {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcItem, rcOutofrange);
+ }
+ }
+ else if (strncmp("BATCH_OFFSET\t", buf, slen = 13) == 0) {
+ rc = str2u64(&buf[slen], len - slen, &self->batch_offset);
+ }
+ else if (strncmp("FIELD_SIZE\t", buf, slen = 11) == 0) {
+ rc = str2u32(&buf[slen], len - slen, &self->field_size);
+ }
+ else if (strncmp("GENERATED_AT\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_at, sizeof(self->generated_at));
+ }
+ else if (strncmp("GENERATED_BY\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_by, sizeof(self->generated_by));
+ }
+ else if (strncmp("LANE\t", buf, slen = 5) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->lane, sizeof(self->lane));
+ }
+ else if (strncmp("LIBRARY\t", buf, slen = 8) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->library, sizeof(self->library));
+ }
+ else if (strncmp("SAMPLE\t", buf, slen = 7) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->sample, sizeof(self->sample));
+ }
+ else if (strncmp("SLIDE\t", buf, slen = 6) == 0) {
+ rc = str2buf(&buf[slen], len - slen, self->slide, sizeof(self->slide));
+ }
+ else if (strncmp("SOFTWARE_VERSION\t", buf, slen = 17) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->software_version, sizeof(self->software_version));
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcName, rcUnrecognized);
+ }
+
+ return rc;
+}
+
static
rc_t CGReads15_GetAssemblyId(const CGReads15* cself, const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id)
{
@@ -132,63 +216,138 @@ rc_t CGReads15_GetBatchFileNumber(const CGReads15* cself, const CGFIELD_BATCH_FI
return 0;
}
-static
-rc_t CC CGReads15_Read(const CGReads15* cself, TReadsData* data)
-{
+
+static rc_t CC CGReads15_Read(const CGReads15* cself, TReadsData* data) {
rc_t rc = 0;
- if( cself->start_rowid == 0 ) {
+ if (cself->start_rowid == 0) {
((CGReads15*)cself)->start_rowid = data->rowid;
}
CG_LINE_START(cself->file, b, len, p);
- if( b == NULL || len == 0) {
+ if (b == NULL || len == 0) {
rc = RC(rcRuntime, rcFile, rcReading, rcData, rcDone);
break;
}
/*DEBUG_MSG(10, ("reads: '%.*s'\n", len, b));*/
CG_LINE_NEXT_FIELD(b, len, p);
- if( (rc = str2u16(b, p - b, &data->flags)) != 0 ) {
- } else if( data->flags > 10 ) {
+ if ((rc = str2u16(b, p - b, &data->flags)) != 0) {
+ }
+ else if (data->flags > 10) {
rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
- } else if( (data->flags & 0x03) == 3 || (data->flags & 0x07) == 7 ) {
+ }
+ else if ((data->flags & 0x03) == 3 || (data->flags & 0x07) == 7) {
rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInvalid);
}
CG_LINE_NEXT_FIELD(b, len, p);
data->seq.sequence.elements = p - b;
- if( data->seq.sequence.elements != CG_READS_SPOT_LEN ) {
+ if (data->seq.sequence.elements != CG_READS15_SPOT_LEN) {
rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInvalid);
- } else {
- rc = str2buf(b, data->seq.sequence.elements, data->read, sizeof(data->read));
+ }
+ else {
+ rc = str2buf(b, data->seq.sequence.elements,
+ data->read, sizeof(data->read));
/* clear cache, set in algnment writer */
data->reverse[0] = '\0';
- data->reverse[CG_READS_SPOT_LEN / 2] = '\0';
+ data->reverse[CG_READS15_SPOT_LEN / 2] = '\0';
}
CG_LINE_LAST_FIELD(b, len, p);
data->seq.quality.elements = p - b;
- if( data->seq.quality.elements != CG_READS_SPOT_LEN ) {
+ if (data->seq.quality.elements != CG_READS15_SPOT_LEN) {
rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInvalid);
- } else {
- rc = str2buf(b, data->seq.quality.elements, data->qual, sizeof(data->qual));
}
- if( cself->records == 0 ) {
+ else {
+ rc = str2buf(b, data->seq.quality.elements,
+ data->qual, sizeof(data->qual));
+ }
+ data->seq.spot_len = CG_READS15_SPOT_LEN;
+
+ data->reads_format = 0x01050000;
+
+ if (cself->records == 0) {
size_t w;
#if 0
- rc = string_printf(((CGReads15*)cself)->spot_group, sizeof(cself->spot_group), &w, "%s:%s:%s:%04u",
- cself->assembly_id, cself->slide, cself->lane, cself->batch_file_number);
+ rc = string_printf(((CGReads15*)cself)->spot_group,
+ sizeof(cself->spot_group), &w, "%s:%s:%s:%04u", cself->assembly_id,
+ cself->slide, cself->lane, cself->batch_file_number);
#else
- rc = string_printf(((CGReads15*)cself)->spot_group, sizeof(cself->spot_group), &w, "%s-%s",
- cself->slide, cself->lane);
+ rc = string_printf(((CGReads15*)cself)->spot_group,
+ sizeof(cself->spot_group), &w, "%s-%s", cself->slide, cself->lane);
#endif
data->seq.spot_group.buffer = cself->spot_group;
data->seq.spot_group.elements = w;
}
((CGReads15*)cself)->records++;
- DEBUG_MSG(10, ("reads: %u\t'%s'\t'%s'\n", data->flags, data->read, data->qual));
+ DEBUG_MSG(10,
+ ("reads: %u\t'%s'\t'%s'\n", data->flags, data->read, data->qual));
+ CG_LINE_END();
+
+ return rc;
+}
+
+static rc_t CC CGReads25_Read(const CGReads15* cself, TReadsData* data) {
+ rc_t rc = 0;
+
+ if (cself->start_rowid == 0) {
+ ((CGReads15*)cself)->start_rowid = data->rowid;
+ }
+ CG_LINE_START(cself->file, b, len, p);
+ if (b == NULL || len == 0) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcDone);
+ break;
+ }
+ /*DEBUG_MSG(10, ("reads: '%.*s'\n", len, b));*/
+ CG_LINE_NEXT_FIELD(b, len, p);
+ if ((rc = str2u16(b, p - b, &data->flags)) != 0) {
+ }
+ else if (data->flags > 10) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcOutofrange);
+ }
+ else if ((data->flags & 0x03) == 3 || (data->flags & 0x07) == 7) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInvalid);
+ }
+ CG_LINE_NEXT_FIELD(b, len, p);
+ data->seq.sequence.elements = p - b;
+ if (data->seq.sequence.elements != CG_READS25_SPOT_LEN) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInvalid);
+ }
+ else {
+ rc = str2buf(b, data->seq.sequence.elements,
+ data->read, sizeof(data->read));
+ /* clear cache, set in algnment writer */
+ data->reverse[0] = '\0';
+ data->reverse[CG_READS25_SPOT_LEN / 2] = '\0';
+ }
+ CG_LINE_LAST_FIELD(b, len, p);
+ data->seq.quality.elements = p - b;
+ if (data->seq.quality.elements != CG_READS25_SPOT_LEN) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcData, rcInvalid);
+ }
+ else {
+ rc = str2buf(b, data->seq.quality.elements,
+ data->qual, sizeof(data->qual));
+ }
+ data->seq.spot_len = CG_READS25_SPOT_LEN;
+
+ data->reads_format = 0x02050000;
+
+ if (cself->records == 0) {
+ size_t w;
+
+ rc = string_printf(((CGReads15*)cself)->spot_group,
+ sizeof(cself->spot_group), &w, "%s-%s", cself->slide, cself->lane);
+ data->seq.spot_group.buffer = cself->spot_group;
+ data->seq.spot_group.elements = w;
+ }
+ ((CGReads15*)cself)->records++;
+ DEBUG_MSG(10,
+ ("reads: %u\t'%s'\t'%s'\n", data->flags, data->read, data->qual));
CG_LINE_END();
+
return rc;
}
+
static
rc_t CGReads15_GetStartRow(const CGReads15* cself, int64_t* rowid)
{
@@ -226,7 +385,25 @@ static const CGFileType_vt CGReads15_vt =
CGReads15_Release
};
-rc_t CC CGReads15_Make(const CGFileType** cself, const CGLoaderFile* file)
+static const CGFileType_vt CGReads25_vt = {
+ CGReads25_Header,
+ CGReads25_Read,
+ CGReads15_GetStartRow,
+ NULL,
+ NULL,
+ NULL,
+ NULL, /* tag_lfr */
+ CGReads15_GetAssemblyId,
+ CGReads15_GetSlide,
+ CGReads15_GetLane,
+ CGReads15_GetBatchFileNumber,
+ NULL,
+ NULL,
+ CGReads15_Release
+};
+
+static rc_t CC CGReads_Make(const CGFileType** cself,
+ const CGLoaderFile* file, const CGFileType_vt *vt)
{
rc_t rc = 0;
CGReads15* obj = NULL;
@@ -241,7 +418,7 @@ rc_t CC CGReads15_Make(const CGFileType** cself, const CGLoaderFile* file)
} else {
obj->file = file;
obj->dad.type = cg_eFileType_READS;
- obj->dad.vt = &CGReads15_vt;
+ obj->dad.vt = vt;
}
}
if( rc == 0 ) {
@@ -252,6 +429,11 @@ rc_t CC CGReads15_Make(const CGFileType** cself, const CGLoaderFile* file)
return rc;
}
+rc_t CC CGReads15_Make(const CGFileType** cself, const CGLoaderFile* file)
+{
+ return CGReads_Make(cself, file, &CGReads15_vt);
+}
+
rc_t CC CGReads13_Make(const CGFileType** self, const CGLoaderFile* file)
{ return CGReads15_Make(self, file); }
@@ -260,3 +442,7 @@ rc_t CC CGReads20_Make(const CGFileType** self, const CGLoaderFile* file)
rc_t CC CGReads22_Make(const CGFileType** self, const CGLoaderFile* file)
{ return CGReads15_Make(self, file); }
+
+rc_t CC CGReads25_Make(const CGFileType **self, const CGLoaderFile *file) {
+ return CGReads_Make(self, file, &CGReads25_vt);
+}
diff --git a/tools/cg-load/factory-reads.h b/tools/cg-load/factory-reads.h
index e0266d3..84bde21 100644
--- a/tools/cg-load/factory-reads.h
+++ b/tools/cg-load/factory-reads.h
@@ -39,5 +39,8 @@ rc_t CC CGReads20_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
rc_t CC CGReads22_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
+rc_t CC CGReads25_Make(const struct CGFileType** self,
+ const struct CGLoaderFile* file);
+
#endif /* _tools_cg_load_factory_reads_h_ */
diff --git a/tools/cg-load/factory-tag-lfr.c b/tools/cg-load/factory-tag-lfr.c
index 2586571..c986f64 100644
--- a/tools/cg-load/factory-tag-lfr.c
+++ b/tools/cg-load/factory-tag-lfr.c
@@ -22,20 +22,23 @@
*
* ===========================================================================
*/
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
+
+#include "debug.h"
+#include "factory-cmn.h"
+#include "factory-tag-lfr.h"
typedef struct CGTagLfr CGTagLfr;
#define CGFILETYPE_IMPL CGTagLfr
#include "file.h"
-#include "factory-cmn.h"
-#include "factory-tag-lfr.h"
-#include "debug.h"
+
+#include <klib/printf.h>
+#include <klib/rc.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
#include <stdlib.h>
#include <string.h>
-#include <os-native.h>
struct CGTagLfr {
CGFileType dad;
@@ -58,8 +61,9 @@ struct CGTagLfr {
CGFIELD_WELL_ID wellId;
};
-static
-rc_t CC CGTagLfr_Header(const CGTagLfr* cself, const char* buf, const size_t len)
+
+static rc_t CC CGTagLfr15_Header(const CGTagLfr* cself,
+ const char* buf, const size_t len)
{
rc_t rc = 0;
size_t slen;
@@ -96,8 +100,89 @@ rc_t CC CGTagLfr_Header(const CGTagLfr* cself, const char* buf, const size_t len
return rc;
}
-static
-rc_t CGTagLfr_GetAssemblyId(const CGTagLfr* cself, const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id)
+static rc_t CC CGTagLfr25_Header(const CGTagLfr* cself,
+ const char* buf, const size_t len)
+{
+ rc_t rc = 0;
+ size_t slen;
+ CGTagLfr* self = (CGTagLfr*)cself;
+
+ /* from SRA-2617 files */
+ if (strncmp("APPROVAL\t", buf, slen = 9) == 0) {
+ }
+ else if (strncmp("TITLE\t", buf, slen = 6) == 0) {
+ }
+ else if (strncmp("ADDRESS\t", buf, slen = 8) == 0) {
+ }
+
+ /* From Table 1: Header Metadata Present in all Data Files */
+ else if (strncmp("CUSTOMER_SAMPLE_ID\t", buf, slen = 19) == 0) {
+ }
+ else if (strncmp("SAMPLE_SOURCE\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("REPORTED_GENDER\t", buf, slen = 16) == 0) {
+ }
+ else if (strncmp("CALLED_GENDER\t", buf, slen = 14) == 0) {
+ }
+ else if (strncmp("TUMOR_STATUS\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_TYPE\t", buf, slen = 13) == 0) {
+ }
+ else if (strncmp("LIBRARY_SOURCE\t", buf, slen = 13) == 0) {
+ }
+
+ else if (strncmp("ASSEMBLY_ID\t", buf, slen = 12) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->assembly_id, sizeof(self->assembly_id));
+ }
+ else if (strncmp("BATCH_FILE_NUMBER\t", buf, slen = 18) == 0) {
+ rc = str2u32(&buf[slen], len - slen, &self->batch_file_number);
+ if (self->batch_file_number < 1) {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcItem, rcOutofrange);
+ }
+ }
+ else if (strncmp("BATCH_OFFSET\t", buf, slen = 13) == 0) {
+ rc = str2u64(&buf[slen], len - slen, &self->batch_offset);
+ }
+ else if (strncmp("FIELD_SIZE\t", buf, slen = 11) == 0) {
+ rc = str2u32(&buf[slen], len - slen, &self->field_size);
+ }
+ else if (strncmp("GENERATED_AT\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_at, sizeof(self->generated_at));
+ }
+ else if (strncmp("GENERATED_BY\t", buf, slen = 13) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->generated_by, sizeof(self->generated_by));
+ }
+ else if (strncmp("LANE\t", buf, slen = 5) == 0) {
+ rc = str2buf(&buf[slen], len - slen, self->lane, sizeof(self->lane));
+ }
+ else if (strncmp("LIBRARY\t", buf, slen = 8) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->library, sizeof(self->library));
+ }
+ else if (strncmp("SAMPLE\t", buf, slen = 7) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->sample, sizeof(self->sample));
+ }
+ else if (strncmp("SLIDE\t", buf, slen = 6) == 0) {
+ rc = str2buf(&buf[slen], len - slen, self->slide, sizeof(self->slide));
+ }
+ else if (strncmp("SOFTWARE_VERSION\t", buf, slen = 17) == 0) {
+ rc = str2buf(&buf[slen], len - slen,
+ self->software_version, sizeof(self->software_version));
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing, rcName, rcUnrecognized);
+ }
+
+ return rc;
+}
+
+
+static rc_t CGTagLfr15_GetAssemblyId(const CGTagLfr* cself,
+ const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id)
{
if( cself->assembly_id[0] == '\0' ) {
return RC(rcRuntime, rcFile, rcReading, rcFormat, rcInvalid);
@@ -106,8 +191,8 @@ rc_t CGTagLfr_GetAssemblyId(const CGTagLfr* cself, const CGFIELD_ASSEMBLY_ID_TYP
return 0;
}
-static
-rc_t CGTagLfr_GetSlide(const CGTagLfr* cself, const CGFIELD_SLIDE_TYPE** slide)
+static rc_t CGTagLfr15_GetSlide(const CGTagLfr* cself,
+ const CGFIELD_SLIDE_TYPE** slide)
{
if( cself->slide[0] == '\0' ) {
return RC(rcRuntime, rcFile, rcReading, rcFormat, rcInvalid);
@@ -116,8 +201,8 @@ rc_t CGTagLfr_GetSlide(const CGTagLfr* cself, const CGFIELD_SLIDE_TYPE** slide)
return 0;
}
-static
-rc_t CGTagLfr_GetLane(const CGTagLfr* cself, const CGFIELD_LANE_TYPE** lane)
+static rc_t CGTagLfr15_GetLane(const CGTagLfr* cself,
+ const CGFIELD_LANE_TYPE** lane)
{
if( cself->lane[0] == '\0' ) {
return RC(rcRuntime, rcFile, rcReading, rcFormat, rcInvalid);
@@ -126,22 +211,22 @@ rc_t CGTagLfr_GetLane(const CGTagLfr* cself, const CGFIELD_LANE_TYPE** lane)
return 0;
}
-static
-rc_t CGTagLfr_GetBatchFileNumber(const CGTagLfr* cself, const CGFIELD_BATCH_FILE_NUMBER_TYPE** batch_file_number)
+static rc_t CGTagLfr15_GetBatchFileNumber(const CGTagLfr* cself,
+ const CGFIELD_BATCH_FILE_NUMBER_TYPE** batch_file_number)
{
*batch_file_number = &cself->batch_file_number;
return 0;
}
-static
-rc_t CGTagLfr_GetStartRow(const CGTagLfr* cself, int64_t* rowid)
+static rc_t CGTagLfr15_GetStartRow(
+ const CGTagLfr* cself, int64_t* rowid)
{
*rowid = cself->start_rowid;
return 0;
}
-static
-rc_t CC CGTagLfr_GetTagLfr(const CGFILETYPE_IMPL* cself, TReadsData* data)
+static rc_t CC CGTagLfr15_GetTagLfr(
+ const CGFILETYPE_IMPL* cself, TReadsData* data)
{
rc_t rc = 0;
uint16_t wellScore = 0;
@@ -204,7 +289,7 @@ rc_t CC CGTagLfr_GetTagLfr(const CGFILETYPE_IMPL* cself, TReadsData* data)
}
static
-void CC CGTagLfr_Release(const CGTagLfr* cself, uint64_t* records)
+void CC CGTagLfr15_Release(const CGTagLfr* cself, uint64_t* records)
{
if( cself != NULL ) {
CGTagLfr* self = (CGTagLfr*)cself;
@@ -215,25 +300,44 @@ void CC CGTagLfr_Release(const CGTagLfr* cself, uint64_t* records)
}
}
-static const CGFileType_vt CGTagLfr_vt =
+static const CGFileType_vt CGTagLfr15_vt =
{
- CGTagLfr_Header,
+ CGTagLfr15_Header,
NULL,
- CGTagLfr_GetStartRow,
+ CGTagLfr15_GetStartRow,
NULL,
NULL,
NULL,
- CGTagLfr_GetTagLfr, /* tag_lfr */
- CGTagLfr_GetAssemblyId,
- CGTagLfr_GetSlide,
- CGTagLfr_GetLane,
- CGTagLfr_GetBatchFileNumber,
+ CGTagLfr15_GetTagLfr, /* tag_lfr */
+ CGTagLfr15_GetAssemblyId,
+ CGTagLfr15_GetSlide,
+ CGTagLfr15_GetLane,
+ CGTagLfr15_GetBatchFileNumber,
NULL,
NULL,
- CGTagLfr_Release
+ CGTagLfr15_Release
};
-rc_t CC CGTagLfr_Make(const CGFileType** cself, const CGLoaderFile* file)
+static const CGFileType_vt CGTagLfr25_vt =
+{
+ CGTagLfr25_Header,
+ NULL,
+ CGTagLfr15_GetStartRow,
+ NULL,
+ NULL,
+ NULL,
+ CGTagLfr15_GetTagLfr, /* tag_lfr */
+ CGTagLfr15_GetAssemblyId,
+ CGTagLfr15_GetSlide,
+ CGTagLfr15_GetLane,
+ CGTagLfr15_GetBatchFileNumber,
+ NULL,
+ NULL,
+ CGTagLfr15_Release
+};
+
+static rc_t CC CGTagLfr_Make(const CGFileType **cself,
+ const CGLoaderFile *file, const CGFileType_vt *vt)
{
rc_t rc = 0;
CGTagLfr* obj = NULL;
@@ -248,14 +352,25 @@ rc_t CC CGTagLfr_Make(const CGFileType** cself, const CGLoaderFile* file)
} else {
obj->file = file;
obj->dad.type = cg_eFileType_TAG_LFR;
- obj->dad.vt = &CGTagLfr_vt;
+ obj->dad.vt = vt;
}
}
if( rc == 0 ) {
*cself = &obj->dad;
} else {
- CGTagLfr_Release(obj, NULL);
+ CGTagLfr15_Release(obj, NULL);
}
return rc;
}
+rc_t CC CGTagLfr15_Make(const CGFileType **cself,
+ const CGLoaderFile *file)
+{
+ return CGTagLfr_Make(cself, file, &CGTagLfr15_vt);
+}
+
+rc_t CC CGTagLfr25_Make(const CGFileType **cself,
+ const CGLoaderFile *file)
+{
+ return CGTagLfr_Make(cself, file, &CGTagLfr25_vt);
+}
diff --git a/tools/cg-load/factory-tag-lfr.h b/tools/cg-load/factory-tag-lfr.h
index 5188f3a..95a706e 100644
--- a/tools/cg-load/factory-tag-lfr.h
+++ b/tools/cg-load/factory-tag-lfr.h
@@ -26,12 +26,16 @@
#ifndef _tools_cg_load_factory_tag_lfr_h_
#define _tools_cg_load_factory_tag_lfr_h_
+
#include <klib/defs.h>
struct CGFileType;
struct CGLoaderFile;
-rc_t CC CGTagLfr_Make(const struct CGFileType** self,
+rc_t CC CGTagLfr15_Make(const struct CGFileType** self,
+ const struct CGLoaderFile* file);
+rc_t CC CGTagLfr25_Make(const struct CGFileType** self,
const struct CGLoaderFile* file);
+
#endif /* _tools_cg_load_factory_tag_lfr_h_ */
diff --git a/tools/cg-load/file-version-factory.c b/tools/cg-load/file-version-factory.c
index 7304c0e..662e552 100644
--- a/tools/cg-load/file-version-factory.c
+++ b/tools/cg-load/file-version-factory.c
@@ -27,9 +27,10 @@
typedef struct CGIgnored CGIgnored;
#define CGFILETYPE_IMPL CGIgnored
-
#include "file.h" /* CGLoaderFile */
+#include <sysalloc.h>
+
#include <assert.h>
rc_t CGLoaderFile_CreateCGFile(CGLoaderFile* self,
@@ -43,10 +44,14 @@ rc_t CGLoaderFile_CreateCGFile(CGLoaderFile* self,
case 0x01050000:
case 0x01060000:
return CGFile15_Make(&self->cg_file, TYPE, self);
+ case 0x01070000:
+ return CGFile17_Make(&self->cg_file, TYPE, self);
case 0x02000000:
return CGFile20_Make(&self->cg_file, TYPE, self);
case 0x02020000:
return CGFile22_Make(&self->cg_file, TYPE, self);
+ case 0x02050000:
+ return CGFile25_Make(&self->cg_file, TYPE, self);
default: {
const char* name = NULL;
rc_t rc = CGLoaderFile_Filename(self, &name);
diff --git a/tools/cg-load/file.c b/tools/cg-load/file.c
index 1294701..1fcecff 100644
--- a/tools/cg-load/file.c
+++ b/tools/cg-load/file.c
@@ -22,22 +22,29 @@
*
* ===========================================================================
*/
+
+
+#include "debug.h"
+#include "defs.h"
+#include "factory-file.h"
+#include "file.h"
+
+#include <kapp/loader-file.h>
+
+#include <kfs/directory.h>
+
#include <klib/log.h>
#include <klib/rc.h>
#include <klib/status.h>
-#include <kfs/directory.h>
-#include <kapp/loader-file.h>
-#include <strtol.h>
-#include "defs.h"
-#include "file.h"
-#include "factory-file.h"
-#include "debug.h"
+#include <strtol.h>
+#include <sysalloc.h>
+#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <ctype.h>
+
rc_t CGLoaderFile_IsEof(const CGLoaderFile* cself, bool* eof)
{
@@ -60,17 +67,20 @@ rc_t CGLoaderFile_Line(const CGLoaderFile* cself, uint64_t* line)
}
-rc_t CGLoaderFile_Readline(const CGLoaderFile* cself, const void** buffer, size_t* length)
+rc_t CGLoaderFile_Readline(const CGLoaderFile* cself,
+ const void** buffer, size_t* length)
{
return KLoaderFile_Readline(cself ? cself->file : NULL, buffer, length);
}
-rc_t CGLoaderFile_Filename(const CGLoaderFile* cself, const char** name)
+rc_t CGLoaderFile_Filename(
+ const CGLoaderFile* cself, const char** name)
{
return KLoaderFile_Name(cself ? cself->file : NULL, name);
}
-rc_t CGLoaderFile_LOG(const CGLoaderFile* cself, KLogLevel lvl, rc_t rc, const char *msg, const char *fmt, ...)
+rc_t CGLoaderFile_LOG(const CGLoaderFile* cself, KLogLevel lvl,
+ rc_t rc, const char *msg, const char *fmt, ...)
{
if( cself != NULL ) {
va_list args;
@@ -103,9 +113,7 @@ rc_t CGLoaderFile_Release(const CGLoaderFile* cself, bool ignored)
return rc;
}
-static
-rc_t parse_version(const char* buf, size_t len, uint32_t* v)
-{
+static rc_t parse_version(const char* buf, size_t len, uint32_t* v) {
rc_t rc = 0;
int i = 0;
int64_t q;
@@ -132,6 +140,78 @@ rc_t parse_version(const char* buf, size_t len, uint32_t* v)
}
static
+rc_t _CGLoaderFileParseLibraryType(const CGLoaderFile *self,
+ const char *buf, size_t len, uint32_t *t)
+{
+ char libraryType[64] = "";
+ rc_t rc = str2buf(buf, len, libraryType, sizeof(libraryType));
+ if (rc != 0) {
+ return rc;
+ }
+ assert(t);
+ /* From Standard Sequencing Service Data File Formats. File format v2.5 */
+ switch (libraryType[0]) {
+ case 'P': {
+ const char n[] = "PureLFR" ; /* From File format v2.5 */
+ const char s[] = "Pure LFR"; /* From submission in SRA-2617 */
+ if (string_cmp(n, sizeof n - 1, libraryType,
+ string_measure(libraryType, NULL), sizeof n - 1) == 0
+ ||string_cmp(s, sizeof s - 1, libraryType,
+ string_measure(libraryType, NULL), sizeof s - 1) == 0)
+ {
+ *t = cg_eLibraryType_PureLFR;
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing,
+ rcData, rcUnrecognized);
+ }
+ break;
+ }
+ case 'M': {
+ const char l[] = "Mixed-LFR";
+ const char s[] = "Mixed-STD";
+ if (string_cmp(l, sizeof l - 1, libraryType,
+ string_measure(libraryType, NULL), sizeof l - 1) == 0)
+ {
+ *t = cg_eLibraryType_MixedLFR;
+ }
+ else if (string_cmp(s, sizeof s - 1, libraryType,
+ string_measure(libraryType, NULL), sizeof s - 1) == 0)
+ {
+ *t = cg_eLibraryType_MixedSTD;
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing,
+ rcData, rcUnrecognized);
+ }
+ break;
+ }
+ case 'S': {
+ const char a[] = "Standard";
+ if (string_cmp(a, sizeof a - 1, libraryType,
+ string_measure(libraryType, NULL), sizeof a - 1) == 0)
+ {
+ *t = cg_eLibraryType_Standard;
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing,
+ rcData, rcUnrecognized);
+ }
+ break;
+ }
+ default:
+ rc = RC(rcRuntime, rcFile, rcConstructing,
+ rcData, rcUnrecognized);
+ break;
+ }
+ if (rc != 0) {
+ CGLoaderFile_LOG(self, klogErr, rc,
+ "unexpected LIBRARY_TYPE value <$(t)>", "t=%s", libraryType);
+ }
+ return rc;
+}
+
+static
rc_t CGLoaderFile_header(const CGLoaderFile* cself)
{
rc_t rc = 0;
@@ -141,10 +221,13 @@ rc_t CGLoaderFile_header(const CGLoaderFile* cself)
const char* buf;
size_t len;
uint32_t fver = 0;
- char type[64];
+ char type[64] = "";
+ uint32_t libraryType = cg_eLibraryType_Unknown;
do {
- if( (rc = CGLoaderFile_Readline(self, (const void**)&buf, &len)) == 0 ) {
+ if ((rc = CGLoaderFile_Readline(self, (const void**)&buf, &len))
+ == 0)
+ {
if( buf == NULL ) {
rc = RC(rcRuntime, rcFile, rcConstructing, rcData, rcTooShort);
} else if( len == 0 ) {
@@ -152,33 +235,47 @@ rc_t CGLoaderFile_header(const CGLoaderFile* cself)
} else if( buf[0] == '>' ) {
/* start of records */
break;
- } else if( buf[0] == '#' ) {
+ }
+ else if( buf[0] == '#' ) {
len--; buf++;
- if( strncmp("FORMAT_VERSION\t", buf, 15) == 0 ) {
+ if (strncmp("FORMAT_VERSION\t", buf, 15) == 0) {
rc = parse_version(&buf[15], len - 15, &fver);
- } else if( strncmp("TYPE\t", buf, 5) == 0 ) {
+ }
+ else if (strncmp("LIBRARY_TYPE\t", buf, 13) == 0) {
+ rc = _CGLoaderFileParseLibraryType(cself,
+ &buf[13], len - 13, &libraryType);
+ }
+ else if (strncmp("TYPE\t", buf, 5) == 0) {
rc = str2buf(&buf[5], len - 5, type, sizeof(type));
}
- } else {
- rc = RC(rcRuntime, rcFile, rcConstructing, rcData, rcUnrecognized);
+ }
+ else {
+ rc = RC(rcRuntime, rcFile, rcConstructing,
+ rcData, rcUnrecognized);
}
}
- } while( rc == 0 && self->cg_file == NULL );
- if( rc == 0 ) {
+ } while (rc == 0 && self->cg_file == NULL);
+
+ if (rc == 0) {
rc = CGLoaderFile_CreateCGFile(self, fver, type);
if( rc == 0 ) {
CGFileType* f = (CGFileType*)self->cg_file;
f->format_version = fver;
- if( f->type == cg_eFileType_Unknown || !f->vt || !f->vt->header ) {
- rc = RC(rcRuntime, rcFile, rcConstructing, rcInterface, rcIncomplete);
+ f->libraryType = libraryType;
+
+ if (f->type == cg_eFileType_Unknown ||
+ !f->vt || !f->vt->header)
+ {
+ rc = RC(rcRuntime, rcFile, rcConstructing,
+ rcInterface, rcIncomplete);
}
}
if( rc == 0 ) {
/* we need to restart file for loading all header by sub class */
KLoaderFile_Reset(self->file);
}
- while( rc == 0 ) {
+ while (rc == 0) {
if( (rc = CGLoaderFile_Readline(self, (const void**)&buf, &len)) == 0 ) {
if( buf == NULL ) {
rc = RC(rcRuntime, rcFile, rcConstructing, rcData, rcTooShort);
diff --git a/tools/cg-load/file.h b/tools/cg-load/file.h
index 1e750f3..1efeed4 100644
--- a/tools/cg-load/file.h
+++ b/tools/cg-load/file.h
@@ -253,6 +253,7 @@ typedef struct CGFileType_vt_struct {
struct CGFileType {
uint32_t format_version;
CG_EFileType type;
+ CG_ELibraryType libraryType;
const CGFileType_vt* vt;
};
diff --git a/tools/vdb-dump/vdb_info.h b/tools/cg-load/formats.c
similarity index 67%
copy from tools/vdb-dump/vdb_info.h
copy to tools/cg-load/formats.c
index f0bde80..e3808cc 100644
--- a/tools/vdb-dump/vdb_info.h
+++ b/tools/cg-load/formats.c
@@ -24,24 +24,31 @@
*
*/
-#ifndef _h_vdb_info_
-#define _h_vdb_info_
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
+#include "defs.h" /* CG_FORMAT_2_5 */
+#include "formats.h" /* get_cg_reads_ngaps */
+
+#include <assert.h>
+
+
+/* CG native format files have some changes since version 2.5 */
-#include <klib/rc.h>
-#include <vdb/manager.h>
-rc_t vdb_info( Vector * schema_list, dump_format_t format, const VDBManager *mgr,
- const char * acc_or_path, num_gen * row_generator );
+uint32_t get_cg_reads_ngaps(uint32_t reads_format) {
+ assert(reads_format);
-#ifdef __cplusplus
+ return reads_format < CG_FORMAT_2_5 ? 3 : 2;
}
-#endif
-#endif
+/* IN FORMAT VERSION 2.5 SPOT LENGTH WAS CHANGES FROM 70 to 60 */
+uint32_t get_cg_read_len(uint32_t reads_format) {
+ uint32_t spot_len = CG_READS15_SPOT_LEN;
+
+ assert(reads_format);
+
+ if (reads_format >= CG_FORMAT_2_5) {
+ spot_len = CG_READS25_SPOT_LEN;
+ }
+
+ return spot_len / 2;
+}
diff --git a/tools/vtblcp/vtblcp-priv.h b/tools/cg-load/formats.h
similarity index 79%
copy from tools/vtblcp/vtblcp-priv.h
copy to tools/cg-load/formats.h
index b8e413b..d378442 100644
--- a/tools/vtblcp/vtblcp-priv.h
+++ b/tools/cg-load/formats.h
@@ -1,3 +1,6 @@
+#ifndef _tools_cg_load_formats_h_
+#define _tools_cg_load_formats_h_
+
/*===========================================================================
*
* PUBLIC DOMAIN NOTICE
@@ -24,17 +27,15 @@
*
*/
-#ifndef _h_vtblcp_priv_
-#define _h_vtblcp_priv_
-#ifdef __cplusplus
-extern "C" {
-#endif
+#include <stdint.h> /* uint32_t */
+
+
+/* CG native format files have some changes since version 2.5 */
+uint32_t get_cg_reads_ngaps(uint32_t reads_format);
+uint32_t get_cg_read_len (uint32_t reads_format);
-#ifdef __cplusplus
-}
-#endif
-#endif /* _h_vtblcp_priv_ */
+#endif /* _tools_cg_load_formats_h_ */
diff --git a/tools/cg-load/writer-algn.c b/tools/cg-load/writer-algn.c
index b2921ab..4015250 100644
--- a/tools/cg-load/writer-algn.c
+++ b/tools/cg-load/writer-algn.c
@@ -23,34 +23,36 @@
* ===========================================================================
*
*/
-#include <klib/log.h>
+
+
+#include "debug.h" /* DEBUG_MSG */
+#include "defs.h" /* cg_eRightHalfDnbMap */
+#include "formats.h" /* get_cg_reads_ngaps */
+#include "writer-algn.h" /* CGWriterAlgn */
+
+#include <align/align.h> /* NCBI_align_ro_complete_genomics */
+#include <align/dna-reverse-cmpl.h> /* DNAReverseCompliment */
+
+#include <klib/log.h> /* PLOGERR */
#include <klib/rc.h>
-#include <klib/printf.h>
-#include <klib/sort.h>
-#include <kfs/file.h>
-#include <insdc/insdc.h>
-#include <align/dna-reverse-cmpl.h>
-#include <align/align.h>
-
-#include "defs.h"
-#include "writer-algn.h"
-#include "debug.h"
-
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <math.h>
+#include <klib/sort.h> /* ksort */
+
+#include <sysalloc.h>
+
#include <assert.h>
+#include <math.h> /* pow */
+#include <string.h> /* strcmp */
+
typedef struct CGWriterAlgn_match_struct {
/* filled out by ReferenceMgr_Compress */
INSDC_coord_zero read_start;
INSDC_coord_len read_len;
- bool has_ref_offset[CG_READS_SPOT_LEN];
- int32_t ref_offset[CG_READS_SPOT_LEN];
- uint8_t ref_offset_type[CG_READS_SPOT_LEN];
- bool has_mismatch[CG_READS_SPOT_LEN];
- char mismatch[CG_READS_SPOT_LEN];
+ bool has_ref_offset[CG_READS15_SPOT_LEN];
+ int32_t ref_offset[CG_READS15_SPOT_LEN];
+ uint8_t ref_offset_type[CG_READS15_SPOT_LEN];
+ bool has_mismatch[CG_READS15_SPOT_LEN];
+ char mismatch[CG_READS15_SPOT_LEN];
int64_t ref_id;
INSDC_coord_zero ref_start;
/* fill oud here */
@@ -194,16 +196,42 @@ rc_t CGWriterAlgn_Save(CGWriterAlgn *const self,
if( !map->saved ) {
CGWriterAlgn_match *const match = &self->match[mate];
TableWriterAlgnData *const algn = &self->algn[mate];
- uint32_t* cigar, g;
- uint32_t left_cigar[] = { 5 << 4, 0, 10 << 4, 0, 10 << 4, 0, 10 << 4 };
- uint32_t right_cigar[] = { 10 << 4, 0, 10 << 4, 0, 10 << 4, 0, 5 << 4 };
- const uint32_t read_len = CG_READS_SPOT_LEN / 2;
- const char* read;
+
+ uint32_t g = 0;
+
+ uint32_t* cigar = NULL;
+ uint32_t left_cigar15 []={ 5 << 4, 0, 10 << 4, 0, 10 << 4, 0,10 << 4 };
+ uint32_t right_cigar15[]={ 10 << 4, 0, 10 << 4, 0, 10 << 4, 0, 5 << 4 };
+ uint32_t left_cigar25 []={ 10 << 4, 0, 10 << 4, 0, 10 << 4, 0, 0 };
+ uint32_t right_cigar25[]={ 10 << 4, 0, 10 << 4, 0, 10 << 4, 0, 0 };
+ uint32_t *left_cigar = NULL;
+ uint32_t *right_cigar = NULL;
+ uint32_t cg_reads_ngaps = 0;
+
+ const char *read = NULL;
+ uint32_t read_len = 0;
+
+ assert(rd);
+
+ cg_reads_ngaps = get_cg_reads_ngaps(rd->reads_format);
+
+ read_len = rd->seq.spot_len / 2;
+ if (cg_reads_ngaps == 3) {
+ left_cigar = left_cigar15;
+ right_cigar = right_cigar15;
+ }
+ else if (cg_reads_ngaps == 2) {
+ left_cigar = left_cigar25;
+ right_cigar = right_cigar25;
+ }
+ else {
+ assert(0);
+ }
if (match->seq_read_id == 2) {
read = &((const char*)(rd->seq.sequence.buffer))[read_len];
cigar = right_cigar;
- g = CG_READS_SPOT_LEN / 2;
+ g = read_len;
}
else {
read = rd->seq.sequence.buffer;
@@ -220,7 +248,7 @@ rc_t CGWriterAlgn_Save(CGWriterAlgn *const self,
read = &rd->reverse[g];
cigar = (cigar == left_cigar) ? right_cigar : left_cigar;
}
- for(g = 0; g < CG_READS_NGAPS; g++) {
+ for(g = 0; g < cg_reads_ngaps; g++) {
if( map->gap[g] > 0 ) {
cigar[g * 2 + 1] = (map->gap[g] << 4) | 3; /* 'xN' */
} else if( map->gap[g] < 0 ) {
@@ -256,6 +284,7 @@ rc_t CGWriterAlgn_Save(CGWriterAlgn *const self,
map->saved = true;
}
}
+
return rc;
}
@@ -409,16 +438,17 @@ bool check_in_cluster(TMappingsData_map const *const a, TMappingsData_map const
}
static
-int clustering_sort_cb(void const *const A, void const *const B, void *const ctx)
+int64_t clustering_sort_cb(void const *const A, void const *const B, void *const ctx)
{
TMappingsData const *const data = (TMappingsData const *)ctx;
unsigned const ia = *(unsigned const *)A;
unsigned const ib = *(unsigned const *)B;
TMappingsData_map const *const a = &data->map[ia];
TMappingsData_map const *const b = &data->map[ib];
- int res;
+ int64_t res;
+ unsigned j = 0;
- res = (int)(a->flags & cg_eRightHalfDnbMap) - (int)(b->flags & cg_eRightHalfDnbMap); /**** separate by DNP side ***/
+ res = (int64_t)(a->flags & cg_eRightHalfDnbMap) - (int64_t)(b->flags & cg_eRightHalfDnbMap); /**** separate by DNP side ***/
if (res) return res;
res = strcmp(a->chr, b->chr); /* same chromosome ? **/
@@ -428,14 +458,18 @@ int clustering_sort_cb(void const *const A, void const *const B, void *const ctx
if (res) return res;
/**cluster is defined here; now pick the winner **/
- res = (int)a->saved - (int)b->saved; /*** if already saved **/
+ res = (int64_t)a->saved - (int64_t)b->saved; /*** if already saved **/
if (res) return -res;
- res = (int)a->weight - (int)b->weight; /*** has higher score **/
+ res = (int64_t)a->weight - (int64_t)b->weight; /*** has higher score **/
if (res) return -res;
-
- res = (int)(a->gap[0] + a->gap[1] + a->gap[2])
- - (int)(b->gap[0] + b->gap[1] + b->gap[2]); /** has lower projection on the reference **/
+
+ res = 0;
+ assert(data->cg_reads_ngaps);
+ for (j = 0; j != data->cg_reads_ngaps; ++j) {
+ res += (int64_t)(a->gap[j]) - (int64_t)(b->gap[j]);
+ } /** has lower projection on the reference **/
+
return res;
}
@@ -517,26 +551,35 @@ rc_t CGWriterAlgn_Write_int(CGWriterAlgn *const self, TReadsData *const read)
unsigned i;
unsigned countLeft = 0;
unsigned countRight = 0;
-
+
+ assert(read);
+
+ data->cg_reads_ngaps = get_cg_reads_ngaps(read->reads_format);
+ assert(data->cg_reads_ngaps);
+
for (i = 0; i != N; ++i) {
char const *const refname = data->map[i].chr;
unsigned j;
- INSDC_coord_len reflen = 35;
+ INSDC_coord_len reflen = read->seq.spot_len / 2;
ReferenceSeq const *rseq;
+ bool shouldUnmap = false;
+ bool wasRenamed = false;
memset(&self->match[i], 0, sizeof(self->match[i]));
- rc = ReferenceMgr_GetSeq(self->rmgr, &rseq, refname);
+ rc = ReferenceMgr_GetSeq(self->rmgr, &rseq, refname, &shouldUnmap, true, &wasRenamed);
if (rc) {
(void)PLOGERR(klogErr, (klogErr, rc, "Failed accessing Reference '$(ref)'", "ref=%s", refname));
break;
}
+ assert(shouldUnmap == false);
rc = ReferenceSeq_Get1stRow(rseq, &self->match[i].ref_id); /* if the above worked, this is infallible */
assert(rc == 0);
ReferenceSeq_Release(rseq);
- for (j = 0; j != CG_READS_NGAPS; ++j)
+ for (j = 0; j != data->cg_reads_ngaps; ++j) {
reflen += data->map[i].gap[j];
+ }
self->match[i].seq_spot_id = read->rowid;
self->match[i].mapq = data->map[i].weight - 33;
@@ -621,7 +664,10 @@ rc_t CGWriterAlgn_Write(const CGWriterAlgn* cself, TReadsData* read)
{
assert(cself != NULL);
assert(read != NULL);
- assert(read->seq.sequence.buffer != NULL && read->seq.sequence.elements == CG_READS_SPOT_LEN);
+ assert(read->seq.sequence.buffer != NULL
+ && read->seq.sequence.elements == read->seq.spot_len
+ && (read->seq.sequence.elements == CG_READS15_SPOT_LEN ||
+ read->seq.sequence.elements == CG_READS25_SPOT_LEN));
memset(read->prim_algn_id, 0, sizeof(read->prim_algn_id));
memset(read->align_count, 0, sizeof(read->align_count));
diff --git a/tools/cg-load/writer-algn.h b/tools/cg-load/writer-algn.h
index 6ccb019..b852c4d 100644
--- a/tools/cg-load/writer-algn.h
+++ b/tools/cg-load/writer-algn.h
@@ -40,7 +40,7 @@ typedef struct TMappingsData_map_struct {
uint16_t flags;
char chr[CG_CHROMOSOME_NAME];
INSDC_coord_zero offset;
- int16_t gap[CG_READS_NGAPS];
+ int16_t gap[CG_READS15_NGAPS];
uint8_t weight;
uint32_t mate;
} TMappingsData_map;
@@ -48,6 +48,7 @@ typedef struct TMappingsData_map_struct {
typedef struct TMappingsData_struct {
uint16_t map_qty;
TMappingsData_map map[CG_MAPPINGS_MAX];
+ uint32_t cg_reads_ngaps;
} TMappingsData;
typedef struct CGWriterAlgn CGWriterAlgn;
diff --git a/tools/cg-load/writer-evidence-dnbs.c b/tools/cg-load/writer-evidence-dnbs.c
index de989d0..81d538f 100644
--- a/tools/cg-load/writer-evidence-dnbs.c
+++ b/tools/cg-load/writer-evidence-dnbs.c
@@ -23,28 +23,35 @@
* ===========================================================================
*
*/
+
+#include "debug.h"
+#include "defs.h"
+#include "writer-evidence-dnbs.h"
+#include "writer-evidence-intervals.h"
+
+#include <align/align.h>
+#include <align/dna-reverse-cmpl.h>
+#include <align/writer-alignment.h>
+
+#include <insdc/insdc.h>
+
+#include <kfs/file.h>
+
+#include <klib/container.h>
#include <klib/log.h>
#include <klib/rc.h>
-#include <klib/container.h>
-#include <kfs/file.h>
-#include <insdc/insdc.h>
+
#include <vdb/database.h>
#include <vdb/table.h>
-#include <align/writer-alignment.h>
-#include <align/dna-reverse-cmpl.h>
-#include <align/align.h>
-#include "debug.h"
-#include "defs.h"
-#include "writer-evidence-intervals.h"
-#include "writer-evidence-dnbs.h"
+#include <sysalloc.h>
-#include <stdlib.h>
+#include <assert.h>
+#include <ctype.h>
+#include <errno.h>
#include <stdio.h>
+#include <stdlib.h>
#include <string.h>
-#include <errno.h>
-#include <ctype.h>
-#include <assert.h>
typedef struct CGWriterEvdDnb_match_struct {
@@ -78,10 +85,12 @@ struct CGWriterEvdDnbs {
TEvidenceDnbsData data;
uint64_t bad_allele_index;
+ uint32_t read_len;
};
-rc_t CGWriterEvdDnbs_Make(const CGWriterEvdDnbs** cself, TEvidenceDnbsData** data,
- VDatabase* db, const ReferenceMgr* rmgr, const uint32_t options)
+rc_t CGWriterEvdDnbs_Make(const CGWriterEvdDnbs **cself,
+ TEvidenceDnbsData **data, VDatabase *db, const ReferenceMgr *rmgr,
+ const uint32_t options, uint32_t read_len)
{
rc_t rc = 0;
CGWriterEvdDnbs* self;
@@ -120,6 +129,9 @@ rc_t CGWriterEvdDnbs_Make(const CGWriterEvdDnbs** cself, TEvidenceDnbsData** dat
/* set to 1st row for evidence_interval to collect ids */
self->data.last_rowid = 1;
+
+ self->read_len = read_len;
+
*data = &self->data;
}
}
@@ -178,10 +190,14 @@ rc_t CGWriterEvdDnbs_Write(const CGWriterEvdDnbs* cself, const TEvidenceInterval
DEBUG_MSG(3, ("bad allele_index for interval %s %s[%hu]\n", cself->data.dnbs[i].chr, cself->data.interval_id, i + 1));
self->bad_allele_index++;
continue;
- } else {
- const char* read;
- char reversed[CG_READS_SPOT_LEN];
- uint32_t read_len = CG_READS_SPOT_LEN / 2;
+ }
+ else {
+ const char *read = NULL;
+ char reversed[CG_READS15_SPOT_LEN] = "";
+ uint32_t read_len = cself->read_len;
+
+ assert(read_len == CG_READS15_SPOT_LEN / 2
+ || read_len == CG_READS25_SPOT_LEN / 2);
if( cself->data.dnbs[i].side == 'L' ) {
self->match.seq_read_id = 1;
diff --git a/tools/cg-load/writer-evidence-dnbs.h b/tools/cg-load/writer-evidence-dnbs.h
index 6c9014b..f0c17af 100644
--- a/tools/cg-load/writer-evidence-dnbs.h
+++ b/tools/cg-load/writer-evidence-dnbs.h
@@ -26,13 +26,20 @@
#ifndef _tools_cg_load_writer_evidence_dnbs_h_
#define _tools_cg_load_writer_evidence_dnbs_h_
-#include <insdc/insdc.h>
-#include <vdb/database.h>
-#include <sra/sradb.h>
#include "defs.h"
#include "factory-cmn.h"
+#include <insdc/insdc.h>
+
+#include <sra/sradb.h>
+
+#include <vdb/database.h>
+
+
+struct ReferenceMgr;
+
+
typedef struct TEvidenceDnbsData_dnb_struct {
/* translated:
@@ -62,6 +69,7 @@ typedef struct TEvidenceDnbsData_dnb_struct {
size_t read_len;
/*char qual[CG_EVDNC_SPOT_LEN];
*/
+ CGFIELD_WELL_ID wellId;
} TEvidenceDnbsData_dnb;
typedef struct TEvidenceDnbsData {
@@ -76,8 +84,9 @@ typedef struct CGWriterEvdDnbs CGWriterEvdDnbs;
struct TEvidenceIntervalsData;
-rc_t CGWriterEvdDnbs_Make(const CGWriterEvdDnbs** cself, TEvidenceDnbsData** data,
- VDatabase* db, const ReferenceMgr* rmgr, const uint32_t options);
+rc_t CGWriterEvdDnbs_Make(const CGWriterEvdDnbs **cself,
+ TEvidenceDnbsData **data, VDatabase *db, const struct ReferenceMgr *rmgr,
+ const uint32_t options, uint32_t read_len);
rc_t CGWriterEvdDnbs_Whack(const CGWriterEvdDnbs* cself, bool commit, uint64_t* rows);
diff --git a/tools/cg-load/writer-evidence-intervals.c b/tools/cg-load/writer-evidence-intervals.c
index d72621a..e5efd8f 100644
--- a/tools/cg-load/writer-evidence-intervals.c
+++ b/tools/cg-load/writer-evidence-intervals.c
@@ -23,23 +23,28 @@
* ===========================================================================
*
*/
+
+#include "debug.h"
+#include "defs.h"
+#include "file.h"
+#include "writer-evidence-dnbs.h"
+#include "writer-evidence-intervals.h"
+
+#include <align/align.h>
+
+#include <kfs/file.h>
#include <klib/log.h>
-#include <klib/rc.h>
#include <klib/printf.h>
-#include <kfs/file.h>
+#include <klib/rc.h>
+
#include <vdb/table.h>
-#include <align/align.h>
-#include "defs.h"
-#include "writer-evidence-intervals.h"
-#include "writer-evidence-dnbs.h"
-#include "debug.h"
-#include "file.h"
+#include <sysalloc.h>
+#include <assert.h>
+#include <ctype.h>
#include <stdlib.h>
#include <string.h>
-#include <ctype.h>
-#include <assert.h>
typedef struct CGWriterEvdInt_match_struct {
diff --git a/tools/cg-load/writer-seq.c b/tools/cg-load/writer-seq.c
index 43b34bd..c936fbc 100644
--- a/tools/cg-load/writer-seq.c
+++ b/tools/cg-load/writer-seq.c
@@ -23,23 +23,29 @@
* ===========================================================================
*
*/
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <insdc/insdc.h>
#include "debug.h"
#include "defs.h"
#include "writer-seq.h"
-#include <stdlib.h>
+#include <insdc/insdc.h>
+
+#include <klib/log.h>
+#include <klib/rc.h>
+
+#include <sysalloc.h>
+
#include <assert.h>
+#include <stdlib.h>
const INSDC_SRA_platform_id DFTL_platform = SRA_PLATFORM_COMPLETE_GENOMICS;
const char DFTL_label[] = "LeftRight";
const INSDC_coord_zero DFTL_label_start[CG_READS_NREADS] = {0, 4};
const INSDC_coord_len DFTL_label_len[CG_READS_NREADS] = {4, 5};
-const INSDC_coord_zero DFTL_read_start[CG_READS_NREADS] = {0, 35};
-const INSDC_coord_len DFTL_read_len[CG_READS_NREADS] = {35, 35};
+
+INSDC_coord_zero DFTL_read_start[CG_READS_NREADS] = {0, 35};
+INSDC_coord_len DFTL_read_len [CG_READS_NREADS] = {35, 35};
+
const INSDC_SRA_read_filter DFTL_read_filter[CG_READS_NREADS] = {SRA_READ_FILTER_PASS, SRA_READ_FILTER_PASS};
struct CGWriterSeq {
@@ -47,11 +53,16 @@ struct CGWriterSeq {
TReadsData data;
};
-rc_t CGWriterSeq_Make(const CGWriterSeq** cself, TReadsData** data, VDatabase* db, const uint32_t options, const char* quality_quantization)
+rc_t CGWriterSeq_Make(const CGWriterSeq** cself,
+ TReadsData** data, VDatabase* db, const uint32_t options,
+ const char* quality_quantization, uint32_t read_len)
{
rc_t rc = 0;
CGWriterSeq* self;
+ assert(read_len == CG_READS15_SPOT_LEN / 2
+ || read_len == CG_READS25_SPOT_LEN / 2);
+
if( cself == NULL || db == NULL ) {
return RC(rcExe, rcFormatter, rcConstructing, rcParam, rcNull);
}
@@ -64,7 +75,16 @@ rc_t CGWriterSeq_Make(const CGWriterSeq** cself, TReadsData** data, VDatabase* d
p.elements = 1;
if( (rc = TableWriterSeq_Make(&self->base, db, options | ewseq_co_AlignData | ewseq_co_SpotGroup, quality_quantization)) != 0 ) {
LOGERR(klogErr, rc, "sequence table");
- } else if( (rc = TableWriteSeq_WriteDefault(self->base, ewseq_cn_PLATFORM, &p)) == 0 ) {
+ }
+ else if ((rc =
+ TableWriteSeq_WriteDefault(self->base, ewseq_cn_PLATFORM, &p)) == 0)
+ {
+ int i = 0;
+ for (i = 0; i < CG_READS_NREADS; ++i) {
+ DFTL_read_len[i] = read_len;
+ }
+ DFTL_read_start[1] = read_len;
+
/* attach data pointer to data */
self->data.seq.nreads = CG_READS_NREADS;
self->data.seq.alignment_count.buffer = self->data.align_count;
diff --git a/tools/cg-load/writer-seq.h b/tools/cg-load/writer-seq.h
index b9a1da7..5cc899e 100644
--- a/tools/cg-load/writer-seq.h
+++ b/tools/cg-load/writer-seq.h
@@ -32,12 +32,14 @@
#include <align/writer-sequence.h>
typedef struct TReadsData_struct {
+ uint32_t reads_format;
+
uint16_t flags;
- char read[CG_READS_SPOT_LEN + 1];
- char qual[CG_READS_SPOT_LEN + 1];
+ char read[CG_READS15_SPOT_LEN + 1];
+ char qual[CG_READS15_SPOT_LEN + 1];
/* reverse read cache by half-dnb */
- INSDC_dna_text reverse[CG_READS_SPOT_LEN];
+ INSDC_dna_text reverse[CG_READS15_SPOT_LEN];
TableWriterSeqData seq;
int64_t rowid;
@@ -49,7 +51,9 @@ typedef struct TReadsData_struct {
typedef struct CGWriterSeq CGWriterSeq;
-rc_t CGWriterSeq_Make(const CGWriterSeq** cself, TReadsData** data, VDatabase* db, const uint32_t options, const char* quality_quantization);
+rc_t CGWriterSeq_Make(const CGWriterSeq** cself,
+ TReadsData** data, VDatabase* db, const uint32_t options,
+ const char* quality_quantization, uint32_t read_len);
rc_t CGWriterSeq_Whack(const CGWriterSeq* cself, bool commit, uint64_t* rows);
diff --git a/tools/check-blob-size/check-blob-size.c b/tools/check-blob-size/check-blob-size.c
deleted file mode 100644
index 3f081fa..0000000
--- a/tools/check-blob-size/check-blob-size.c
+++ /dev/null
@@ -1,332 +0,0 @@
-/*******************************************************************************
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * =============================================================================
- */
-
-#include "check-blob-size.vers.h" /* CHECK_BLOB_SIZE_VERS */
-#include <kapp/main.h>
-#include <kdb/manager.h>
-#include <kdb/table.h>
-#include <kdb/column.h>
-#include <kdb/namelist.h> /* KTableListCol */
-#include <klib/namelist.h> /* KNamelistRelease */
-#include <klib/log.h> /* LOGERR */
-#include <klib/out.h> /* OUTMSG */
-#include <klib/debug.h> /* DBGMSG */
-#include <klib/rc.h>
-#include <assert.h>
-#include <string.h> /* memset */
-#include <stdio.h> /* printf */
-#define DISP_RC(rc, msg) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, msg))
-#define DISP_RC2(rc, name, msg) (void)((rc == 0) ? 0 : \
- PLOGERR(klogInt, (klogInt,rc, "$(msg): $(name)","msg=%s,name=%s",msg,name)))
-typedef struct CmdArgs {
- Args* args;
- uint32_t count;
-
- uint32_t i;
-} CmdArgs;
-ver_t CC KAppVersion(void) { return CHECK_BLOB_SIZE_VERS; }
-static void summary(const char* progname) {
- OUTMSG(("Usage:\n"
- " %s [Options] <table>\n", progname));
-}
-const char UsageDefaultName[] = "check-blob-size";
-static const char* def_name = UsageDefaultName;
-
-rc_t CC UsageSummary (const char * progname)
-{
- return 0;
-}
-
-rc_t CC Usage(const Args* args)
-{
- rc_t rc = 0;
- const char* progname = def_name;
- const char* fullpath = def_name;
- assert(args);
- rc = ArgsProgram(args, &fullpath, &progname);
- if (rc)
- { progname = fullpath = def_name; }
- summary(progname);
- OUTMSG(("\nOptions\n"));
- HelpOptionsStandard();
- HelpVersion(fullpath, KAppVersion());
- return rc;
-}
-/* MINIUSAGE(def_name) */
-static rc_t CmdArgsInit(int argc, char** argv, CmdArgs* cmdArgs)
-{
- rc_t rc = 0;
-
- assert(argv && cmdArgs);
-
- memset(cmdArgs, 0, sizeof *cmdArgs);
-
- rc = ArgsMakeAndHandle(&cmdArgs->args, argc, argv, 0, NULL, 0);
- DISP_RC(rc, "while calling ArgsMakeAndHandle");
-
- if (rc == 0) {
- do {
- Args* args = cmdArgs->args;
- rc = ArgsParamCount(args, &cmdArgs->count);
- if (rc) {
- DISP_RC(rc, "while calling ArgsParamCount");
- break;
- }
- if (cmdArgs->count < 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
- LOGERR(klogErr, rc, "Missing table parameter");
- break;
- }
- } while (false);
- }
-
- if (rc != 0)
- { MiniUsage(cmdArgs->args); }
-
- return rc;
-}
-static rc_t CmdArgsGetNextTable(CmdArgs* args, const char** table)
-{
- rc_t rc = 0;
- assert(args && table);
- *table = NULL;
- if (args->i >= args->count) /* no more tables to give :( */
- { return rc; }
- rc = ArgsParamValue(args->args, args->i, table);
- if (rc) {
- PLOGERR(klogInt, (klogInt, rc,
- "while calling ArgsParamValue($(i))", "i=%d", args->i));
- }
- ++args->i;
- return rc;
-}
-static rc_t CmdArgsDestroy(CmdArgs* args)
-{
- rc_t rc = 0;
- assert(args);
- rc = ArgsWhack(args->args);
- DISP_RC(rc, "while calling ArgsWhack");
- args->args = NULL;
- return rc;
-}
-static rc_t CheckBlob(const char* name, const KColumn* col, int64_t id,
- int64_t* lastId, size_t* blobSize)
-{
- rc_t rc = 0;
- int64_t first = 0;
- uint32_t count = 0;
- const KColumnBlob* blob = NULL;
- assert(col && lastId && blobSize);
- rc = KColumnOpenBlobRead(col, &blob, id);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "while calling KColumnOpenBlobRead($(col), $(id))",
- "col=%s,id=%d", name, id));
- }
- else {
- rc = KColumnBlobIdRange(blob, &first, &count);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "while calling KColumnBlobIdRange($(col), $(id))",
- "col=%s,id=%d", name, id));
- }
- }
- if (rc == 0) {
- size_t num_read = 0;
- size_t remaining = 0;
- rc = KColumnBlobRead(blob, 0, NULL, 0, &num_read, &remaining);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "while calling KColumnBlobRead($(col), $(id))",
- "col=%s,id=%d", name, id));
- }
- else {
- assert(num_read == 0 && remaining);
- *lastId = first + count - 1;
- *blobSize = remaining;
- DBGMSG(DBG_APP, DBG_COND_2,
- ("first %d, count %d, size %d\n", first, count, remaining));
- }
- }
- {
- rc_t rc2 = KColumnBlobRelease(blob);
- DISP_RC(rc2, "while calling KColumnBlobRelease");
- if (rc == 0)
- { rc = rc2; }
- blob = NULL;
- }
- return rc;
-}
-static rc_t CheckCol(const KTable* tbl,
- const char* name, size_t* maxBlobSize)
-{
- rc_t rc = 0;
- uint64_t first = 0;
- uint64_t count = 0;
- const KColumn* col = NULL;
- assert(tbl && name && maxBlobSize);
- if (rc == 0) {
- rc = KTableOpenColumnRead(tbl, &col, "%s", name);
- DISP_RC2(rc, name, "while opening column");
- }
- if (rc == 0) {
- rc = KColumnIdRange(col, &first, &count);
- DISP_RC2(rc, name, "while calling KColumnIdRange");
- }
- if (rc == 0) {
- uint64_t nBlobs = 0;
- uint64_t i = 0;
- *maxBlobSize = 0;
- DBGMSG(DBG_APP, DBG_COND_1,
- ("Column %18s: %d(%d)\n", name, first, count));
- for (i = first; i < first + count && rc == 0;) {
- int64_t lastId = 0;
- rc = Quitting();
- if (rc)
- { LOGMSG(klogWarn, "Interrupted"); }
- else {
- size_t blobSize = 0;
- rc = CheckBlob(name, col, i, &lastId, &blobSize);
- i = lastId + 1;
- ++nBlobs;
- if (rc == 0 && blobSize > *maxBlobSize)
- { *maxBlobSize = blobSize; }
- }
- }
- if (rc == 0) {
- PLOGMSG(klogInfo, (klogInfo,
- "'$(col): $(blobs) blobs (max $(max))",
- "col=%s,blobs=%d,max=%d", name, nBlobs, *maxBlobSize));
- }
- }
- {
- rc_t rc2 = KColumnRelease(col);
- DISP_RC(rc2, "while calling KColumnRelease");
- if (rc == 0)
- { rc = rc2; }
- col = NULL;
- }
- return rc;
-}
-static rc_t Check(const KTable* tbl, size_t* maxBlobSize)
-{
- rc_t rc = 0;
- uint32_t cCount = 0;
- struct KNamelist* columns = NULL;
- assert(tbl && maxBlobSize);
- rc = KTableListCol(tbl, &columns);
- DISP_RC(rc, "while calling KTableListCol");
- if (rc == 0) {
- rc = KNamelistCount(columns, &cCount);
- DISP_RC(rc, "while calling KTableColNamelistCount");
- }
- if (rc == 0) {
- uint32_t iCol = 0;
- *maxBlobSize = 0;
- for (iCol = 0; iCol < cCount && rc == 0; ++iCol) {
- const char* name = NULL;
- if (rc == 0) {
- rc = KNamelistGet(columns, iCol, &name);
- if (rc) {
- PLOGERR(klogInt, (klogInt, rc,
- "while calling KTableColNamelistGet($(i))",
- "i=%d", iCol));
- }
- }
- if (rc == 0) {
- size_t maxColBlobSize = 0;
- rc = CheckCol(tbl, name, &maxColBlobSize);
- if (rc == 0 && maxColBlobSize > *maxBlobSize)
- { *maxBlobSize = maxColBlobSize; }
- }
- }
- }
- {
- rc_t rc2 = KNamelistRelease(columns);
- DISP_RC(rc2, "while calling KNamelistRelease");
- if (rc == 0)
- { rc = rc2; }
- columns = NULL;
- }
- return rc;
-}
-rc_t CC KMain(int argc, char* argv[])
-{
- rc_t rc = 0;
-
- CmdArgs args;
- const KDBManager* mgr = NULL;
-
- assert(argc && argv);
-
- def_name = argv[0];
-
- rc = LogLevelSet("info");
- DISP_RC(rc, "while calling LogLevelSet");
-
- rc = CmdArgsInit(argc, argv, &args);
- if (rc == 0) {
- rc = KDBManagerMakeRead(&mgr, NULL);
- DISP_RC(rc, "while calling KDBManagerMakeRead");
- }
- while (rc == 0) {
- const char* table = NULL;
- const KTable* tbl = NULL;
- rc = CmdArgsGetNextTable(&args, &table);
- if (table == NULL)
- { break; }
- if (rc == 0) {
- rc = KDBManagerOpenTableRead(mgr, &tbl, "%s", table);
- DISP_RC2(rc, table, "while opening table");
- }
- if (rc == 0) {
- size_t maxBlobSize = 0;
- PLOGMSG(klogInfo, (klogInfo, "'$(table)'", "table=%s", table));
- rc = Check(tbl, &maxBlobSize);
- if (rc == 0)
- { printf("%lu\t%s\n", maxBlobSize, table); }
- }
- {
- rc_t rc2 = KTableRelease(tbl);
- DISP_RC(rc2, "while calling KTableRelease");
- if (rc == 0)
- { rc = rc2; }
- tbl = NULL;
- }
- }
- {
- rc_t rc2 = KDBManagerRelease(mgr);
- DISP_RC(rc2, "while calling KDBManagerRelease");
- if (rc == 0)
- { rc = rc2; }
- mgr = NULL;
- }
- {
- rc_t rc2 = CmdArgsDestroy(&args);
- if (rc == 0)
- { rc = rc2; }
- }
- return rc;
-}
diff --git a/tools/check-blob-size/check-blob-size.vers b/tools/check-blob-size/check-blob-size.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/check-blob-size/check-blob-size.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/copycat/Makefile b/tools/copycat/Makefile
index a3d0ca7..14c2367 100644
--- a/tools/copycat/Makefile
+++ b/tools/copycat/Makefile
@@ -32,12 +32,7 @@ include $(TOP)/build/Makefile.env
INT_TOOLS =
-EXT_TOOLS =
-
-ifeq (linux,$(OS))
- EXT_TOOLS += copycat
-endif
-
+EXT_TOOLS = copycat
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -46,12 +41,24 @@ ALL_TOOLS = \
#-------------------------------------------------------------------------------
# outer targets
#
+ifeq (1,$(HAVE_MAGIC))
+
all std: vers-includes
@ $(MAKE_CMD) $(TARGDIR)/std
$(ALL_TOOLS): vers-includes
@ $(MAKE_CMD) $(BINDIR)/$@
+else
+
+all std: $(ALL_TOOLS)
+
+$(ALL_TOOLS):
+ @ echo "NOTE - $(@F) cannot be built:" \
+ "It requires our internal library 'libkff'" \
+ "which requires 'libmagic' and its development headers."
+endif
+
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
@@ -96,14 +103,11 @@ COPYCAT_OBJ = \
$(addsuffix .$(OBJX),$(COPYCAT_SRC))
COPYCAT_LIB = \
- -lkapp \
+ -skapp \
-sncbi-vdb \
- -lkff \
- -lmagic
+ -skff \
+ -smagic-static
$(BINDIR)/copycat: $(COPYCAT_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(COPYCAT_LIB)
-
-
-
diff --git a/tools/copycat/cccat.c b/tools/copycat/cccat.c
index 115474d..f86c6bf 100644
--- a/tools/copycat/cccat.c
+++ b/tools/copycat/cccat.c
@@ -1482,30 +1482,30 @@ rc_t copycat_add_crc (const copycat_pb * ppb)
orc = KFileRelease (pb.df);
if (orc)
{
- LOGERR (klogErr, orc, "Error closing out crc calculator");
- /* an error her implies an error in the copy so report it */
+ LOGERR (klogErr, orc, "Failed to close out crc calculator");
+ /* an error here implies an error in the copy so report it */
rc = orc;
}
}
/* if there were no errors, read the CRC from formatter.
this must be done AFTER releasing the CRC file,
or nothing will ever get written */
- if ( orc == 0 )
+ if ( rc == 0 )
{
bool bin;
orc = KCRC32SumFmtFind ( fmt, ppb->name, &pb.node->crc32, &bin );
if (orc)
PLOGERR (klogWarn,
(klogWarn, orc,
- "Error in obtaining the CRC for '$(path)'",
+ "Failed to obtain the CRC for '$(path)'",
"path=%s", pb.name));
- /* an error her isn't an error in copy */
+ /* an error here isn't an error in copy */
}
}
/* dump the formatter, but not fnull */
orc = KCRC32SumFmtRelease ( fmt );
if (orc)
- LOGERR (klogWarn, orc, "Error closing off CRC storage");
+ LOGERR (klogWarn, orc, "Failed to close off CRC storage");
/* this error we do not need to track */
if (rc == 0)
rc = orc;
diff --git a/tools/copycat/ccsra.c b/tools/copycat/ccsra.c
index 8936cd3..4c3d270 100644
--- a/tools/copycat/ccsra.c
+++ b/tools/copycat/ccsra.c
@@ -881,7 +881,7 @@ void CC list_item_whack (void * item, void * data)
free (item);
}
static
-int CC list_item_cmp (const void * item, const void * n)
+int64_t CC list_item_cmp (const void * item, const void * n)
{
const list_item * l = item;
const list_item * r = n;
diff --git a/tools/copycat/cctree.c b/tools/copycat/cctree.c
index 8dd736b..f3bdaa2 100644
--- a/tools/copycat/cctree.c
+++ b/tools/copycat/cctree.c
@@ -297,7 +297,7 @@ rc_t CCNameMake ( CCName **np, KTime_t mtime, CCName *dad,
* Sort
*/
static
-int CCNameCmp ( const void *item, const BSTNode *n )
+int64_t CCNameCmp ( const void *item, const BSTNode *n )
{
const String *a = item;
const CCName *b = ( const CCName* ) n;
@@ -305,11 +305,11 @@ int CCNameCmp ( const void *item, const BSTNode *n )
}
static
-int CCNameSort ( const BSTNode *item, const BSTNode *n )
+int64_t CCNameSort ( const BSTNode *item, const BSTNode *n )
{
const CCName *a = ( const CCName* ) item;
const CCName *b = ( const CCName* ) n;
- int cmp = StringCompare ( & a -> name, & b -> name );
+ int64_t cmp = StringCompare ( & a -> name, & b -> name );
if (cmp != 0)
return cmp;
#if 0
diff --git a/tools/copycat/copycat.c b/tools/copycat/copycat.c
index 6aa06bc..e768501 100644
--- a/tools/copycat/copycat.c
+++ b/tools/copycat/copycat.c
@@ -974,7 +974,7 @@ rc_t KMain ( int argc, char *argv [] )
break;
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_CACHE, 0, &cache);
+ rc = ArgsOptionValue (args, OPTION_CACHE, 0, (const void **)&cache);
if (rc)
break;
}
@@ -986,7 +986,7 @@ rc_t KMain ( int argc, char *argv [] )
break;
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_DEST, 0, &dest);
+ rc = ArgsOptionValue (args, OPTION_DEST, 0, (const void **)&dest);
if (rc)
break;
}
@@ -1000,7 +1000,7 @@ rc_t KMain ( int argc, char *argv [] )
break;
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_EXTRACT, 0, &extract);
+ rc = ArgsOptionValue (args, OPTION_EXTRACT, 0, (const void **)&extract);
if (rc)
break;
}
@@ -1024,7 +1024,7 @@ rc_t KMain ( int argc, char *argv [] )
rc = ArgsOptionCount (args, OPTION_XMLBASE, &pcount);
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_XMLBASE, 0, &xml_base);
+ rc = ArgsOptionValue (args, OPTION_XMLBASE, 0, (const void **)&xml_base);
if (rc)
break;
@@ -1043,7 +1043,7 @@ rc_t KMain ( int argc, char *argv [] )
char * end;
uint32_t val;
- rc = ArgsOptionValue (args, OPTION_INBLOCK, 0, &start);
+ rc = ArgsOptionValue (args, OPTION_INBLOCK, 0, (const void **)&start);
if (rc)
break;
@@ -1064,7 +1064,7 @@ rc_t KMain ( int argc, char *argv [] )
char * end;
uint32_t val;
- rc = ArgsOptionValue (args, OPTION_OUTBLOCK, 0, &start);
+ rc = ArgsOptionValue (args, OPTION_OUTBLOCK, 0, (const void **)&start);
if (rc)
break;
@@ -1127,7 +1127,7 @@ rc_t KMain ( int argc, char *argv [] )
VPath * kp;
const char * pc;
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
{
LOGERR (klogFatal, rc, "unable to extract path parameter");
diff --git a/tools/copycat/copycat.vers b/tools/copycat/copycat.vers
index 8e8299d..35d16fb 100644
--- a/tools/copycat/copycat.vers
+++ b/tools/copycat/copycat.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/workspace/Makefile b/tools/dbgap-mount/Makefile
similarity index 78%
rename from tools/workspace/Makefile
rename to tools/dbgap-mount/Makefile
index 710b4ef..1e97665 100644
--- a/tools/workspace/Makefile
+++ b/tools/dbgap-mount/Makefile
@@ -26,16 +26,27 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/workspace
+MODULE = tools/dbgap-mount
include $(TOP)/build/Makefile.env
+
+ifeq (win,$(OS))
+ DOCAN_LIBS = -lDOKAN
+else
+ DOCAN_LIBS = -lfuse
+endif
+
+ifeq (mac,$(OS))
+TOP = $(shell cd ../.. && pwd )
+DOCAN_LIBS = -L/usr/local/lib -losxfuse
+endif
+
INT_TOOLS = \
- path-to-kar-file
EXT_TOOLS = \
- sra
-
+ demo
+
ALL_TOOLS = \
$(INT_TOOLS) \
$(EXT_TOOLS)
@@ -52,20 +63,20 @@ $(ALL_TOOLS): vers-includes
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
-# std
+# all
#
-$(TARGDIR)/std: \
- $(addprefix $(BINDIR)/,$(EXT_TOOLS))
+$(TARGDIR)/all: \
+ $(addprefix $(BINDIR)/,$(ALL_TOOLS))
-.PHONY: $(TARGDIR)/std
+.PHONY: $(TARGDIR)/all
#-------------------------------------------------------------------------------
-# all
+# std
#
-$(TARGDIR)/all: \
- $(addprefix $(BINDIR)/,$(ALL_TOOLS))
+$(TARGDIR)/std: \
+ $(addprefix $(BINDIR)/,$(EXT_TOOLS))
-.PHONY: $(TARGDIR)/all
+.PHONY: $(TARGDIR)/std
#-------------------------------------------------------------------------------
# vers-includes
@@ -82,44 +93,29 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# sratoolkit
+# tag
#
-.PRECIOUS = \
- prefs-yacc.c \
- prefs-lex.c
-
-TOOLKIT_SRC = \
- prefs-yacc \
- prefs-lex \
- sra \
- token \
- sratoolkit-exception \
-
+tag: \
+ $(addsuffix _tag,$(ALL_TOOLS))
-TOOLKIT_OBJ = \
- $(addsuffix .$(OBJX),$(TOOLKIT_SRC))
+.PHONY: tag $(addsuffix _tag,$(ALL_TOOLS))
-TOOLKIT_LIB = \
- -skapp \
- -sncbi-vdb
-
-$(BINDIR)/sra: $(TOOLKIT_OBJ)
- $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(TOOLKIT_LIB)
#-------------------------------------------------------------------------------
-# testing utilities
+# demo
+# test program for new XFS toy
#
-PATH2KAR_SRC = \
- path-to-kar-file \
- sratoolkit-exception
+ZDB_NFUS_SRC = \
+ demo
+
+ZDB_NFUS_OBJ = \
+ $(addsuffix .$(OBJX),$(ZDB_NFUS_SRC))
+ZDB_NFUS_LIB = -lkapp -lncbi-vdb -lxfs $(DOCAN_LIBS)
-PATH2KAR_OBJ = \
- $(addsuffix .$(OBJX),$(PATH2KAR_SRC))
+$(BINDIR)/demo: $(ZDB_NFUS_OBJ)
+ $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(ZDB_NFUS_LIB)
-PATH2KAR_LIB = \
- -skapp \
- -sncbi-vdb
+demo_tag:
+ @ $(TOP)/build/tag-module.sh $(MODULE) demo $(ZDB_NFUS_OBJ)
-$(BINDIR)/path-to-kar-file: $(PATH2KAR_OBJ)
- $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(PATH2KAR_LIB)
diff --git a/tools/dbgap-mount/demo.c b/tools/dbgap-mount/demo.c
new file mode 100644
index 0000000..4d331c4
--- /dev/null
+++ b/tools/dbgap-mount/demo.c
@@ -0,0 +1,360 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "demo.vers.h" /* VDB_PASSWD_VERS */
+
+#include <sysalloc.h>
+#include <kapp/main.h> /* KMain */
+
+#include <klib/text.h>
+#include <klib/log.h> /* LOGERR */
+#include <klib/out.h> /* OUTMSG */
+#include <klib/refcount.h>
+#include <klib/rc.h>
+
+#include <kfs/directory.h>
+#include <kfs/file.h>
+
+#include <xfs/model.h>
+#include <xfs/node.h>
+#include <xfs/tree.h>
+#include <xfs/xfs.h>
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef WINDOWS
+
+#include <windows.h> /* Sleep () */
+
+#else
+
+#include <unistd.h> /* sleep () */
+
+#endif /* WINDOWS */
+
+/******************************************************************************/
+
+
+#ifdef JOJOBA
+
+static
+void
+SLEPOY ( int Sec )
+{
+
+printf ( "Sleeping %d seconds\n", Sec );
+#ifdef WINDOWS
+ Sleep ( Sec * 1000 );
+#else
+ sleep ( Sec );
+#endif
+
+printf ( " DONE [ Sleeping %d seconds ]\n", Sec );
+
+}
+
+#endif /* JOJOBA */
+
+XFS_EXTERN rc_t CC XFS_InitAll_MHR ( const char * ConfigFile );
+XFS_EXTERN rc_t CC XFS_DisposeAll_MHR ();
+
+static
+rc_t
+MakeModel (
+ struct XFSModel ** Model,
+ const char * ProjectId,
+ bool ReadOnly
+)
+{
+ rc_t RCt;
+ struct XFSModel * Mod;
+ struct XFSModelNode * ModNod;
+
+ RCt = 0;
+ Mod = NULL;
+
+ RCt = XFSModelFromScratch ( & Mod, NULL );
+ if ( RCt == 0 ) {
+ RCt = XFSModelAddRootNode ( Mod, "gap-project" );
+ if ( RCt == 0 ) {
+ ModNod = ( struct XFSModelNode * ) XFSModelRootNode ( Mod );
+ if ( ModNod == NULL ) {
+ RCt = XFS_RC ( rcInvalid );
+ }
+ else {
+ RCt = XFSModelNodeSetProperty (
+ ModNod,
+ XFS_MODEL_MODE,
+ ( ReadOnly
+ ? XFS_MODEL_MODE_RO
+ : XFS_MODEL_MODE_RW
+ )
+ );
+ if ( RCt == 0 ) {
+ RCt = XFSModelNodeSetProperty (
+ ModNod,
+ XFS_MODEL_PROJECTID,
+ ProjectId
+ );
+ if ( RCt == 0 ) {
+ * Model = Mod;
+ }
+ }
+
+ }
+ }
+ }
+
+ return RCt;
+}
+
+
+static
+rc_t run (
+ const char * ProjectId,
+ const char * MountPoint,
+ bool ReadOnly,
+ bool Daemonize
+)
+{
+ rc_t RCt;
+ struct XFSModel * TheModel;
+ struct XFSTree * TheTree;
+ struct XFSControl * TheControl;
+
+ RCt = 0;
+ TheModel = NULL;
+ TheTree = NULL;
+ TheControl = NULL;
+
+ OUTMSG ( ( "<<--- run()\n" ) );
+
+ XFS_InitAll_MHR ( NULL );
+
+ RCt = MakeModel ( & TheModel, ProjectId, ReadOnly );
+
+ printf ( "HA(XFSModelMake)[RC=%d]\n", RCt );
+
+ if ( RCt == 0 ) {
+ RCt = XFSTreeMake ( TheModel, & TheTree );
+
+ printf ( "HA(XFSTreeMake)[RC=%d]\n", RCt );
+
+ if ( RCt == 0 ) {
+ printf ( "HA(XFSControlMake)[RC=%d]\n", RCt );
+
+ RCt = XFSControlMake ( TheTree, & TheControl );
+ printf ( "HE(XFSControlMake)[0x%p][RC=%d]\n", ( void * ) TheControl, RCt );
+
+ if ( RCt == 0 ) {
+
+ XFSControlSetMountPoint ( TheControl, MountPoint );
+ XFSControlSetLabel ( TheControl, "Olaffsen" );
+
+ if ( ! Daemonize ) {
+ XFSControlSetArg ( TheControl, "-f", "-f" );
+ }
+
+ printf ( "HA(XFSStart)\n" );
+ RCt = XFSStart ( TheControl );
+
+ printf ( "HE(XFSStart)[RC=%d]\n", RCt );
+ if ( RCt == 0 ) {
+ printf ( "HA(XFSStop)\n" );
+ RCt = XFSStop ( TheControl );
+ printf ( "HE(XFSStop)[RC=%d]\n", RCt );
+ }
+ else {
+ printf ( "Can not start FUSE\n" );
+ }
+ }
+
+ XFSControlDispose ( TheControl );
+
+ XFSTreeRelease ( TheTree );
+ }
+
+ XFSModelRelease ( TheModel );
+ }
+
+ XFS_DisposeAll_MHR ();
+
+ OUTMSG ( ( "--->> run()\n" ) );
+
+ return RCt;
+}
+
+ver_t CC KAppVersion(void) { return DEMO_VERS; }
+
+/* Here I will temporarily parce arguments, and will attach
+ * toolkit ones later ... test program :)
+ */
+char ProgramName[333];
+char ProjectId [33];
+int ProjectIdInt = 0;
+char MountPoint[333];
+bool ReadOnly = true;
+bool Daemonize = false;
+
+#define RO_TAG "ro"
+#define RW_TAG "rw"
+#define DM_TAG "-d"
+
+static
+void
+RightUsage()
+{
+ printf("\ndbGaP mount tool demo program. Will mount and show content of cart files\n");
+ printf("\nUsage: %s [%s|%s] [%s] project_id mount_point\n\n\
+Where:\n\
+ project_id - usually integer greater that zero and less than twelve\n\
+ %s - mount in read only mode\n\
+ %s - mount in read-write mode\n\
+ %s - run mounter as daemon\n\
+ mount_point - point to mount\n\
+\n\n", ProgramName, RO_TAG, RW_TAG, DM_TAG, RO_TAG, RW_TAG, DM_TAG );
+} /* RightUsage() */
+
+static
+bool
+ParseArgs ( int argc, char ** argv )
+{
+ const char * PPU;
+ const char * Arg;
+ int llp;
+
+ Arg = NULL;
+ llp = 0;
+ * ProgramName = 0;
+ * ProjectId = 0;
+ ProjectIdInt = 0;
+ * MountPoint = 0;
+
+ ReadOnly = true;
+ Daemonize = false;
+
+ /* Herer wer arer extractingr programr namer
+ */
+ PPU = strrchr ( * argv, '/' );
+ if ( PPU == NULL ) {
+ PPU = * argv;
+ }
+ else {
+ PPU ++;
+ }
+ strcpy ( ProgramName, PPU );
+
+ /* Herer shouldr ber atr leastr oner argumentr - projectr idr
+ */
+ if ( argc <= 2 ) {
+ printf ( "ERROR : too few arguments\n" );
+ return false;
+ }
+
+ llp = 1;
+ Arg = * ( argv + llp );
+
+ /* firstr paramr couldr ber "ro|rw" orr "-d"
+ */
+ if ( strcmp ( Arg, RO_TAG ) == 0 ) {
+ ReadOnly = true;
+
+ llp ++;
+ }
+ else {
+ ReadOnly = false;
+ if ( strcmp ( Arg, RW_TAG ) == 0 ) {
+
+ llp ++;
+ }
+ }
+
+ /* secondr paramr "-d" orr projectr idr
+ */
+ if ( argc <= llp ) {
+ printf ( "ERROR : too few arguments\n" );
+ return false;
+ }
+ Arg = * ( argv + llp );
+ if ( strcmp ( Arg, DM_TAG ) == 0 ) {
+ Daemonize = true;
+
+ llp ++;
+ }
+
+ /* andr nowr itr isr projectr idr
+ */
+ if ( argc <= llp ) {
+ printf ( "ERROR : too few arguments\n" );
+ return false;
+ }
+ Arg = * ( argv + llp );
+ strcpy ( ProjectId, Arg );
+ /* checkr thatr integerr
+ */
+ ProjectIdInt = atol ( ProjectId );
+ if ( ProjectIdInt <= 0 ) {
+ printf ( "ERROR : invalid project_id '%s'\n", ProjectId );
+ return false;
+ }
+ llp ++;
+
+
+ /* mountr pointr ifr existsr
+ */
+ if ( llp < argc ) {
+ Arg = * ( argv + llp );
+ strcpy ( MountPoint, Arg );
+ llp ++;
+ }
+
+ printf ( "PrI [%d]\n", ProjectIdInt );
+ printf ( "MnP [%s]\n", MountPoint );
+
+
+ return true;
+} /* ParseArgs() */
+
+
+const char UsageDefaultName[] = "Henri Fuseli";
+rc_t CC UsageSummary (const char* progname) { return 0; }
+rc_t CC Usage ( const Args * args ) { return 0; }
+
+rc_t CC KMain(int argc, char *argv[]) {
+
+ if ( ! ParseArgs ( argc, argv ) ) {
+ RightUsage();
+ return 1;
+ }
+
+ return run (
+ ProjectId,
+ MountPoint,
+ ReadOnly,
+ Daemonize
+ );
+}
diff --git a/tools/dbgap-mount/demo.mkfg b/tools/dbgap-mount/demo.mkfg
new file mode 100755
index 0000000..66ab293
--- /dev/null
+++ b/tools/dbgap-mount/demo.mkfg
@@ -0,0 +1,47 @@
+## There is description of Test model
+
+root/type="simple-container"
+root/security="r-xr-xr-x"
+root/children="project,workspace,data"
+
+project/label="Project"
+project/type="kart-collection"
+project/source="/panfs/pan1/trace_work/iskhakov/Tests/dbGaP/data/Project"
+project/security="r-xr-xr-x"
+
+workspace/label="EncryptedWorkspace"
+workspace/type="workspace"
+workspace/security="rwxrwxr-x"
+workspace/source="/panfs/pan1/trace_work/iskhakov/Tests/dbGaP/data/ENC"
+workspace/password="aaa"
+workspace/enctype="AES128"
+
+data/label="Data"
+data/type="simple-container"
+data/children="remote,dir,tar,encfile"
+
+remote/label="RemoteRepository"
+remote/type="remote-repository"
+remote/source="http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant"
+remote/security="r-xr-xr-x"
+
+dir/label="MirroredDirectory"
+dir/type="directory"
+dir/source="/home/iskhakov/HLAM/"
+dir/security="rwxr-xr-x"
+
+tar/label="TarArchive"
+tar/type="tar-archive"
+tar/source="/home/iskhakov/HLAM/mumu.tar"
+
+readme/label="README.txt"
+readme/type="readme"
+help/text="File was automatically generated"
+help/security="r-xr--r--"
+
+encfile/label="iFILE.txt"
+encfile/type="encrypted-file"
+encfile/source="/home/iskhakov/HLAM/HURI.txt.ncbi_enc"
+encfile/security="r--r--r--"
+encfile/password="aaa"
+encfile/enctype="AES128"
diff --git a/tools/dbgap-mount/demo.vers b/tools/dbgap-mount/demo.vers
new file mode 100644
index 0000000..8acdd82
--- /dev/null
+++ b/tools/dbgap-mount/demo.vers
@@ -0,0 +1 @@
+0.0.1
diff --git a/tools/dbgap-mount/demo.vers.h b/tools/dbgap-mount/demo.vers.h
new file mode 100644
index 0000000..72c639a
--- /dev/null
+++ b/tools/dbgap-mount/demo.vers.h
@@ -0,0 +1 @@
+#define DEMO_VERS 0x00000001
diff --git a/tools/deseq/csra-trim.vers b/tools/deseq/csra-trim.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/deseq/csra-trim.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/deseq/deseq.c b/tools/deseq/deseq.c
deleted file mode 100644
index b59b6bf..0000000
--- a/tools/deseq/deseq.c
+++ /dev/null
@@ -1,521 +0,0 @@
-/*==============================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- *==============================================================================
- */
-
-#include "csra-trim.vers.h"
-
-#include <vdb/manager.h> /* VDBManager */
-#include <vdb/database.h> /* VDatabase */
-#include <vdb/table.h> /* VTable */
-#include <vdb/schema.h> /* VSchema */
-#include <vdb/cursor.h> /* VCursor */
-
-#include <kdb/meta.h>
-
-#include <kapp/main.h>
-
-#include <klib/out.h> /* OUTMSG */
-#include <klib/log.h> /* (void)LOGERR */
-#include <klib/debug.h> /* DBGMSG */
-#include <klib/rc.h> /* RC */
-
-#include <assert.h>
-#include <stdlib.h> /* free */
-#include <string.h> /* strcmp */
-
-struct Params {
- char const *dbPath;
- char const *output;
- bool exclude_secondary;
-} static Params;
-
-typedef struct ColumnInfo_s {
- char const *col;
- char const *src;
- int group;
-} ColumnInfo_t;
-
-static ColumnInfo_t const priCol[] = {
- /* these columns are independent and are physical copies */
- { "SEQ_READ_ID" , NULL, 0 },
- { "GLOBAL_REF_START", NULL, 0 },
- { "REF_ORIENTATION" , NULL, 0 },
- { "MAPQ" , NULL, 0 },
-
- /* these columns are pulled in from SEQUENCE */
- { "SEQ_SPOT_ID" , NULL, 1 },
- { "SPOT_GROUP" , NULL, 1 },
- { "MATE_ALIGN_ID", NULL, 1 },
- { "READ_FILTER" , NULL, 1 },
-
- /* these columns all rely on clipping */
- { "(bool)HAS_REF_OFFSET", "(bool)CLIPPED_HAS_REF_OFFSET", 2 },
- { "(bool)HAS_MISMATCH" , "(bool)CLIPPED_HAS_MISMATCH" , 2 },
- { "REF_OFFSET" , "CLIPPED_REF_OFFSET" , 2 },
- { "MISMATCH" , "CLIPPED_MISMATCH" , 2 },
- { "CMP_QUALITY" , "CMP_QUALITY" , 2 },
-
- { NULL, NULL, 0 }
-};
-
-static ColumnInfo_t const secCol[] = {
- /* these columns are independent and are physical copies */
- { "SEQ_READ_ID" , NULL, 0 },
- { "GLOBAL_REF_START" , NULL, 0 },
- { "REF_ORIENTATION" , NULL, 0 },
- { "MAPQ" , NULL, 0 },
- { "PRIMARY_ALIGNMENT_ID", NULL, 0 },
-
- /* these columns are pulled in from SEQUENCE */
- { "SEQ_SPOT_ID" , NULL, 1 },
- { "SPOT_GROUP" , NULL, 1 },
- { "MATE_ALIGN_ID", NULL, 1 },
- { "READ_FILTER" , NULL, 1 },
-
- /* these columns all rely on clipping */
- { "(bool)HAS_REF_OFFSET", "(bool)CLIPPED_HAS_REF_OFFSET", 2 },
- { "(bool)HAS_MISMATCH" , "(bool)CLIPPED_HAS_MISMATCH" , 2 },
- { "REF_OFFSET" , "CLIPPED_REF_OFFSET" , 2 },
- { "MISMATCH" , "CLIPPED_MISMATCH" , 2 },
- { "CMP_QUALITY" , "CMP_QUALITY" , 2 },
-
- { NULL, NULL, 0 }
-};
-
-
-static ColumnInfo_t const refCol[] = {
- /* these columns are independent and are physical copies */
- { "MAX_SEQ_LEN" , NULL, 0 },
- { "NAME" , NULL, 0 },
- { "(INSDC:dna:text)CS_KEY" , NULL, 0 },
- { "(INSDC:dna:text)CMP_READ", NULL, 0 },
- { "SEQ_ID" , NULL, 0 },
- { "SEQ_START" , NULL, 0 },
- { "SEQ_LEN" , NULL, 0 },
- { "CIRCULAR" , NULL, 0 },
- { "CGRAPH_HIGH" , NULL, 0 },
- { "CGRAPH_LOW" , NULL, 0 },
- { "CGRAPH_MISMATCHES" , NULL, 0 },
- { "CGRAPH_INDELS" , NULL, 0 },
- { "PRIMARY_ALIGNMENT_IDS" , NULL, 0 },
- { "SECONDARY_ALIGNMENT_IDS" , NULL, 0 },
- { "(bool)PRESERVE_QUAL" , NULL, 0 },
- { NULL, NULL, 0 },
-};
-
-static rc_t CopyCursor(VCursor *dst, uint32_t const dcid[],
- VCursor const *src, uint32_t const scid[], unsigned n)
-{
- rc_t rc;
- bool done = false;
-
- do {
- rc_t rc2;
-
- rc = VCursorOpenRow(dst);
- while (rc == 0) {
- unsigned i;
- int64_t row;
-
- rc = VCursorRowId(dst, &row); if (rc) break;
- for (i = 0; i != n && rc == 0; ++i) {
- uint32_t elem_bits;
- void const *data;
- uint32_t offset;
- uint32_t rowlen;
-
- rc = VCursorCellDataDirect(src, row, scid[i], &elem_bits, &data, &offset, &rowlen);
- if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound && i == 0) {
- rc = 0;
- done = true;
- break;
- }
- if (rc) break;
- rc = VCursorWrite(dst, dcid[i], elem_bits, data, offset, rowlen); if (rc) break;
- }
- break;
- }
- if (rc == 0 && !done) {
- rc = VCursorCommitRow(dst);
- }
- rc2 = VCursorCloseRow(dst); if (rc == 0) rc = rc2;
- } while (!done && rc == 0 && (rc = Quitting()) == 0);
- if (rc == 0)
- rc = VCursorCommit(dst);
- return rc;
-}
-
-static rc_t CopyTable(VDatabase *dst, VDatabase const *src,
- char const tblName[], ColumnInfo_t const col[],
- bool required)
-{
- uint32_t scid[64];
- uint32_t dcid[64];
- VTable const *stbl;
- rc_t rc = VDatabaseOpenTableRead(src, &stbl, "%s", tblName);
-
- if (rc == 0) {
- VTable *dtbl;
-
- rc = VDatabaseCreateTable(dst, &dtbl, tblName, kcmCreate, "%s", tblName);
- if (rc == 0) {
- unsigned i = 0;
- bool done = false;
-
- rc = VTableColumnCreateParams(dtbl, kcmCreate, kcsCRC32, 0);
- while (rc == 0 && !done && col[i].col) {
- unsigned n = 1;
- unsigned k;
- int group = col[i].group;
- VCursor const *scurs;
-
- if (group != 0) while (col[i + n].col && col[i + n].group == group) ++n;
- rc = VTableCreateCursorRead(stbl, &scurs);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create read cursor", ""));
- done = true;
- }
- else {
- VCursor *dcurs;
-
- rc = VTableCreateCursorWrite(dtbl, &dcurs, kcmInsert);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create write cursor", ""));
- done = true;
- }
- else {
- for (k = 0; k != n; ++k) {
- char const *dcol = col[i + k].col;
- char const *scol = col[i + k].src;
-
- if (scol == NULL) scol = dcol;
- (void)PLOGMSG(klogInfo, (klogInfo, "Copying column '$(col)' of table '$(tbl)'", "col=%s,tbl=%s", dcol, tblName));
- rc = VCursorAddColumn(scurs, &scid[k], "%s", scol);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to add column to read cursor", ""));
- done = true;
- break;
- }
- rc = VCursorAddColumn(dcurs, &dcid[k], "%s", dcol);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to add column to write cursor", ""));
- done = true;
- break;
- }
- }
- if (rc == 0) {
- rc = VCursorOpen(scurs);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open read cursor", ""));
- done = true;
- }
- else {
- rc = VCursorOpen(dcurs);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open write cursor", ""));
- done = true;
- }
- else
- rc = CopyCursor(dcurs, dcid, scurs, scid, n);
- }
- }
- }
- VCursorRelease(dcurs);
- }
- VCursorRelease(scurs);
- i += n;
- }
- if (rc == 0)
- VTableReindex(dtbl);
- VTableRelease(dtbl);
- }
- VTableRelease(stbl);
- }
- else if (!required) {
- /*
- (void)PLOGERR(klogInfo, (klogInfo, rc, "Failed to open optional table '$(tbl)'", "tbl=%s", tblName));
- */
- rc = 0;
- }
- else {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open required table '$(tbl)'", "tbl=%s", tblName));
- }
-
- return rc;
-}
-
-static rc_t OpenDatabases(VDatabase **dst, VDatabase const **src)
-{
- VDBManager *vdb;
- rc_t rc = VDBManagerMakeUpdate(&vdb, NULL);
-
- if (rc == 0) {
- VSchema *schema;
-
- rc = VDBManagerMakeSchema(vdb, &schema);
- if (rc == 0) {
- rc = VSchemaParseFile(schema, "align/align.vschema");
- if (rc == 0) {
- rc = VDBManagerOpenDBRead(vdb, src, NULL, "%s", Params.dbPath);
- if (rc == 0) {
- rc = VDBManagerCreateDB(vdb, dst, schema,
- "NCBI:align:db:alignment_sorted",
- kcmInit + kcmMD5, "%s", Params.output);
- if (rc == 0)
- rc = VDatabaseColumnCreateParams(*dst, kcmInit, kcmMD5, 0);
- if (rc)
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create output '$(outname)'", "outname=%s", Params.output));
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open input '$(inname)'", "inname=%s", Params.dbPath));
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to load schema", ""));
- VSchemaRelease(schema);
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create schema", ""));
- VDBManagerRelease(vdb);
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create manager", ""));
- return rc;
-}
-
-static rc_t ReadMeta(VDatabase const *src, char const name[], void **value, size_t *size)
-{
- KMetadata const *meta;
- rc_t rc = VDatabaseOpenMetadataRead(src, &meta);
-
- *value = NULL;
- *size = 0;
- if (rc == 0) {
- KMDataNode const *node;
-
- rc = KMetadataOpenNodeRead(meta, &node, "%s", name);
- KMetadataRelease(meta);
- if (rc == 0) {
- char dummy;
- size_t remain;
- void *buf;
- size_t bsize;
-
- KMDataNodeRead(node, 0, &dummy, 0, &bsize, &remain);
- if (remain == 0) {
- (void)LOGMSG(klogErr, "Failed to read metadata");
- rc = RC(rcApp, rcMetadata, rcReading, rcData, rcNotFound);
- }
- else {
- buf = malloc(remain);
- if (buf) {
- rc = KMDataNodeRead(node, 0, buf, remain, &bsize, &remain);
- if (rc == 0) {
- *value = buf;
- *size = bsize;
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to read metadata", ""));
- }
- else {
- rc = RC(rcApp, rcMetadata, rcReading, rcMemory, rcExhausted);
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to read metadata", ""));
- }
- }
- KMDataNodeRelease(node);
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to load metadata", ""));
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to load metadata", ""));
- return rc;
-}
-
-static rc_t WriteMeta(VDatabase *dst, char const name[], void *value, size_t size)
-{
- KMetadata *meta;
- rc_t rc = VDatabaseOpenMetadataUpdate(dst, &meta);
-
- if (rc == 0) {
- KMDataNode *node;
-
- rc = KMetadataOpenNodeUpdate(meta, &node, "%s", name);
- KMetadataRelease(meta);
- if (rc == 0) {
- rc = KMDataNodeWrite(node, value, size);
- KMDataNodeRelease(node);
- }
- }
- if (rc)
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to write metadata", ""));
- return rc;
-}
-
-static rc_t CopyBAMHeader(VDatabase *dst, VDatabase const *src)
-{
- void *header;
- size_t hsize;
- rc_t rc = ReadMeta(src, "BAM_HEADER", &header, &hsize);
-
- if (rc == 0)
- rc = WriteMeta(dst, "BAM_HEADER", header, hsize);
- if (header) free(header);
- return rc;
-}
-
-static rc_t run(void)
-{
- VDatabase *dst = NULL;
- VDatabase const *src = NULL;
- rc_t rc = OpenDatabases(&dst, &src);
-
- while (rc == 0) {
- rc = CopyBAMHeader(dst, src); if (rc) break;
- rc = CopyTable(dst, src, "REFERENCE", refCol, true); if (rc) break;
- rc = CopyTable(dst, src, "PRIMARY_ALIGNMENT", priCol, true); if (rc) break;
- if (!Params.exclude_secondary)
- rc = CopyTable(dst, src, "SECONDARY_ALIGNMENT", secCol, false); break;
- break;
- }
- VDatabaseRelease(dst);
- VDatabaseRelease(src);
- return rc;
-}
-
-static const char* usage_output[] = { "path of output database", NULL };
-static const char* usage_exclude[] = { "exclude secondary alignments", NULL };
-
-static const char* param_usage[] = { "Path to the database" };
-
-OptDef Options[] =
-{
- { "output", "o", NULL, usage_output, 1, true, true },
- { "exclude-secondary", "x", NULL, usage_exclude, 1, false, false }
-};
-
-rc_t CC UsageSummary (const char * progname) {
- return KOutMsg (
-"Usage:\n"
-" %s [options] <db-path>\n"
-"\n"
-"Summary:\n"
-" Creates an analysis alignment database from an archive alignment database\n"
-" by flattening and removing sequence data.\n"
-, progname);
- }
-
-rc_t CC Usage(const Args* args) {
- rc_t rc = 0 ;
-
- const char* progname = UsageDefaultName;
- const char* fullpath = UsageDefaultName;
-
- if (args == NULL)
- { rc = RC(rcApp, rcArgv, rcAccessing, rcSelf, rcNull); }
- else
- { rc = ArgsProgram(args, &fullpath, &progname); }
-
- UsageSummary(progname);
-
- KOutMsg("Parameters:\n");
-
- HelpParamLine ("db-path", param_usage);
-
- KOutMsg ("\nOptions:\n");
-
- HelpOptionLine ("o", "output", NULL, usage_output);
-
- HelpOptionsStandard ();
-
- HelpVersion (fullpath, KAppVersion());
-
- return rc;
-}
-
-const char UsageDefaultName[] = "csra-trim";
-
-ver_t CC KAppVersion(void) { return CSRA_TRIM_VERS; }
-
-rc_t CC KMain(int argc, char* argv[]) {
- rc_t rc = 0;
- Args* args = NULL;
-
- do {
- uint32_t pcount = 0;
-
- rc = ArgsMakeAndHandle(&args, argc, argv, 1,
- Options, sizeof Options / sizeof (OptDef));
- if (rc) {
- (void)LOGERR(klogErr, rc, "While calling ArgsMakeAndHandle");
- break;
- }
- rc = ArgsParamCount(args, &pcount);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure parsing database name");
- break;
- }
- if (pcount < 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
- MiniUsage(args);
- break;
- }
- if (pcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- (void)LOGERR(klogErr, rc, "Too many database parameters");
- break;
- }
- rc = ArgsParamValue(args, 0, &Params.dbPath);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure retrieving database name");
- break;
- }
-
- rc = ArgsOptionCount (args, "output", &pcount);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure to get 'output' argument");
- break;
- }
- if (pcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- (void)LOGERR(klogErr, rc, "Too many output parameters");
- break;
- }
- rc = ArgsOptionValue(args, "output", 0, &Params.output);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure retrieving output name");
- break;
- }
-
- rc = ArgsOptionCount (args, "exclude-secondary", &pcount);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure to get 'exclude-secondary' argument");
- break;
- }
- Params.exclude_secondary = pcount > 0;
- } while (false);
-
- rc = rc ? rc : run();
- ArgsRelease(args);
- return rc;
-}
diff --git a/tools/fastq-dump/Makefile b/tools/fastq-dump/Makefile
index 1c84544..16cc5e9 100644
--- a/tools/fastq-dump/Makefile
+++ b/tools/fastq-dump/Makefile
@@ -33,7 +33,7 @@ MODULE = tools/fastq-dump
include $(TOP)/build/Makefile.env
ALL_TOOLS = \
- fastq-dump \
+ fastq-dump-new \
#-------------------------------------------------------------------------------
# outer targets
@@ -72,22 +72,10 @@ clean: stdclean
# Common dumper definitions
#
DUMP_LIBS = \
- -lngs-c++-ncbi \
- -lngs-sdk-ncbi \
- -lncbi-vdb \
- -lkapp
-
-DUMP_LIBS = \
- -L$(LIBDIR) \
- -L$(VDB_LIBDIR) \
- ~/ngs-sdk/x86_64/lib \
- -lngs-c++ \
-lncbi-ngs-c++ \
+ -lngs-c++ \
-lncbi-vdb-static \
-lkapp \
- -lpthread \
- -ldl \
- -lm
#-------------------------------------------------------------------------------
# fastq-dump
@@ -102,5 +90,5 @@ INCDIRS += -I $(TOP)/ngs/ngs-c++
FASTQ_DUMP_OBJ = \
$(addsuffix .$(OBJX),$(FASTQ_DUMP_SRC))
-$(BINDIR)/fastq-dump: $(FASTQ_DUMP_OBJ)
+$(BINDIR)/fastq-dump-new: $(FASTQ_DUMP_OBJ)
$(LP) --exe --vers $(SRCDIR) -o $@ $^ $(DUMP_LIBS)
diff --git a/tools/fastq-dump/args.cpp b/tools/fastq-dump/args.cpp
index eadbc89..d7455a4 100644
--- a/tools/fastq-dump/args.cpp
+++ b/tools/fastq-dump/args.cpp
@@ -261,7 +261,7 @@ AOptVal :: reset ( const String & Name, Args * TheArgs, bool needValue )
_M_val . resize ( count );
if ( needValue ) {
for ( uint32_t i = 0; i < count; i ++ ) {
- const char * val = NULL;
+ const void * val = NULL;
if ( ArgsOptionValue ( TheArgs, Name . c_str (), i, & val ) != 0 ) {
std :: stringstream Vsg;
Vsg << "reset: Can not get value for option \"" << Name;
@@ -269,7 +269,7 @@ AOptVal :: reset ( const String & Name, Args * TheArgs, bool needValue )
throw ErrorMsg ( Vsg . str () );
}
- _M_val [ i ] = val == NULL ? "" : val;
+ _M_val [ i ] = val == NULL ? "" : static_cast <char const*> (val);
}
}
@@ -321,14 +321,14 @@ AParVal :: reset ( const String & Name, Args * TheArgs, bool )
_M_val . resize ( count );
for ( uint32_t i = 0; i < count; i ++ ) {
- const char * val = NULL;
+ const void * val = NULL;
if ( ArgsParamValue ( TheArgs, i, & val ) != 0 ) {
std :: stringstream Vsg;
Vsg << "reset: Can not get parameter in series " << i;
throw ErrorMsg ( Vsg . str () );
}
- _M_val [ i ] = val;
+ _M_val [ i ] = static_cast <char const*> (val);
}
_M_name = Name;
@@ -453,7 +453,7 @@ AArgs :: addStdOpts ()
static
void
-__toOpt ( const struct AOptDef & In, OptDef & Out )
+__toOpt ( const AOptDef & In, OptDef & Out )
{
if ( ! In . good () ) {
throw ErrorMsg ( "__toOpt: IN is not good" );
@@ -471,7 +471,7 @@ __toOpt ( const struct AOptDef & In, OptDef & Out )
} /* __toOpt () */
void
-AArgs :: addOpt ( const struct AOptDef & Opt )
+AArgs :: addOpt ( const AOptDef & Opt )
{
if ( ! good () ) {
throw ErrorMsg ( "addOpt: Not good " );
diff --git a/tools/fastq-dump/args.hpp b/tools/fastq-dump/args.hpp
index 76723dc..8fe241e 100644
--- a/tools/fastq-dump/args.hpp
+++ b/tools/fastq-dump/args.hpp
@@ -279,7 +279,7 @@ public :
/* Options handling
*/
void addStdOpts ();
- void addOpt ( const struct AOptDef & Opt );
+ void addOpt ( const AOptDef & Opt );
inline size_t optDefCount () const
{ return _M_optDefs . size (); };
diff --git a/tools/fastq-dump/fastq-dump.cpp b/tools/fastq-dump/fastq-dump.cpp
index f9d8bce..4bd8a0b 100644
--- a/tools/fastq-dump/fastq-dump.cpp
+++ b/tools/fastq-dump/fastq-dump.cpp
@@ -8,7 +8,6 @@
#include <klib/rc.h>
#include <kfc/rc.h>
-
#include <sysalloc.h>
#include <string.h> /* strcmp () */
@@ -18,14 +17,9 @@
#include "koutstream"
-
-/* Sorry for that ... but it is just a test */
-using namespace std;
-using namespace ncbi;
-using namespace ngs;
-
namespace ngs {
+
/*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*/
/* Arguments */
/*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*_*/
@@ -39,8 +33,8 @@ public :
static const char * _sM_fastaName;
static const char * _sM_legacyReportName;
- static const uint64_t _sM_minSpotIdDefValue = 1;
- static const uint64_t _sM_maxSpotIdDefValue = ~0;
+ static const int64_t _sM_minSpotIdDefValue = 1;
+ static const int64_t _sM_maxSpotIdDefValue = 0;
public :
typedef AArgs PAPAHEN;
@@ -53,9 +47,9 @@ public :
inline const String & accession () const
{ return _M_accession; };
- inline uint64_t minSpotId () const
+ inline int64_t minSpotId () const
{ return _M_minSpotId; };
- inline uint64_t maxSpotId () const
+ inline int64_t maxSpotId () const
{ return _M_maxSpotId; };
inline uint64_t minReadLength () const
@@ -83,8 +77,8 @@ private :
/*) Full Spot Fulters
(*/
- uint64_t _M_minSpotId; /* -N | --minSpotId < rowid > */
- uint64_t _M_maxSpotId; /* -X | --maxSpotId < rowid > */
+ int64_t _M_minSpotId; /* -N | --minSpotId < rowid > */
+ int64_t _M_maxSpotId; /* -X | --maxSpotId < rowid > */
uint64_t _M_minReadLength; /* -M | --minReadLen <len> */
ReadCategory _M_category; /* -Y | --category */
uint64_t _M_fasta; /* -A | --fasta */
@@ -133,6 +127,7 @@ DumpArgs :: __customInit ()
TheOpt . setNeedValue ( true );
TheOpt . setRequired ( false );
TheOpt . setHlp ( "Minimum spot id" );
+ TheOpt . setMaxCount ( 1 );
addOpt ( TheOpt );
}
@@ -248,12 +243,12 @@ DumpArgs :: __customParse ()
_M_accession = parVal () . val ( 0 );
if ( _M_accession . empty () ) {
- cerr << "ERROR: <accession> is not defined" << endl;
+ std :: cerr << "ERROR: <accession> is not defined" << std :: endl;
throw ErrorMsg ( "__custromParse: ERROR: <accession> is not defined" );
}
AOptVal optV = optVal ( _sM_spotIdName );
- uint64_t __spotId = ~0;
+ int64_t __spotId = 0;
if ( optV . exist () ) {
if ( optVal ( _sM_minSpotIdName ) . exist () ) {
@@ -268,7 +263,7 @@ DumpArgs :: __customParse ()
throw ErrorMsg ( String ( "__custromParse: ERROR: Too many \"" ) + _sM_spotIdName + "\" values");
}
- __spotId = optV . uint64Val ();
+ __spotId = optV . int64Val ();
}
optV = optVal ( _sM_minSpotIdName );
@@ -277,7 +272,7 @@ DumpArgs :: __customParse ()
throw ErrorMsg ( String ( "__custromParse: ERROR: Too many \"" ) + _sM_minSpotIdName + "\" values");
}
- _M_minSpotId = optV . uint64Val ();
+ _M_minSpotId = optV . int64Val ();
}
optV = optVal ( _sM_maxSpotIdName );
@@ -286,24 +281,13 @@ DumpArgs :: __customParse ()
throw ErrorMsg ( String ( "__custromParse: ERROR: Too many \"" ) + _sM_maxSpotIdName + "\" values");
}
- _M_maxSpotId = optV . uint64Val ();
+ _M_maxSpotId = optV . int64Val ();
}
- if ( __spotId != ( uint64_t ) ~0 ) {
+ if ( __spotId != 0 ) {
_M_minSpotId = __spotId;
_M_maxSpotId = __spotId;
}
- else {
- if ( _M_maxSpotId < _M_minSpotId ) {
- uint64_t Si = _M_minSpotId;
- _M_minSpotId = _M_maxSpotId;
- _M_maxSpotId = Si;
- }
- }
-
- if ( _M_minSpotId == 0 ) {
- throw ErrorMsg ( String ( "__custromParse: ERROR: Invalid value '0' for \"" ) + _sM_minSpotIdName + "\"");
- }
optV = optVal ( _sM_minReadLengthName );
if ( optV . exist () ) {
@@ -361,9 +345,6 @@ DumpArgs :: __customParse ()
}
_M_fasta = optV . uint64Val ();
- if ( _M_fasta == 0 ) {
- _M_fasta = ~0;
- }
}
_M_legacyReport = optVal ( _sM_legacyReportName ) . exist ();
@@ -376,6 +357,8 @@ DumpArgs :: __customParse ()
// KMain, and all other
((*/
+using namespace ngs;
+
ver_t CC KAppVersion ( void ) { return 0; }
const char UsageDefaultName[] = "fq-d";
@@ -389,9 +372,8 @@ UsageSummary ( const char * progname )
"Usage:\n"
" %s [Options] <Accession>\n"
"\n"
- "Summary:\n"
- " Does something incredibly useful or we wouldn't have written it.\n"
- " I mean very very useful like pre-slicomg bread.\n"
+ "\n"
+ "Use option --help for more information\n"
"\n",
progname
);
@@ -468,13 +450,13 @@ KMain ( int argc, char * argv [] )
TheArgs . dispose ();
}
- catch ( exception & E ) {
- cerr << "Hi, I am exception '" << E.what () << "'" << endl;
+ catch ( std :: exception & E ) {
+ std :: cerr << "Exception handled '" << E.what () << "'" << std :: endl;
return 1;
}
catch ( ... ) {
- cerr << "Hi, I am UNKNOWN exception" << endl;
+ std :: cerr << "UNKNOWN exception handled" << std :: endl;
return 2;
}
@@ -494,7 +476,7 @@ setupFilters ( AFilters & Filters, const DumpArgs & TheArgs )
static
void
dumpFastQ (
- uint64_t SpotId,
+ int64_t SpotId,
const ngs :: String & CollectionName,
const ReadIterator & Iterator
)
@@ -509,45 +491,45 @@ dumpFastQ (
(*/
kout << "@"
<< CollectionName
- << "."
+ << '.'
<< SpotId
- << " "
+ << ' '
<< ReadName
<< " length="
<< Bases . size ()
- << "\n"
+ << '\n'
;
/*) Second is going base itsefl
(*/
kout << Bases
- << "\n"
+ << '\n'
;
/*) Third, header for qualities
(*/
- kout << "+"
+ kout << '+'
<< CollectionName
- << "."
+ << '.'
<< SpotId
- << " "
+ << ' '
<< ReadName
<< " length="
<< Qualities . size ()
- << "\n"
+ << '\n'
;
/*) Finally there are qualities
(*/
kout << Qualities
- << "\n"
+ << '\n'
;
} /* dumpFastQ () */
static
void
dumpFastA (
- uint64_t SpotId,
+ int64_t SpotId,
const ngs :: String & CollectionName,
const ReadIterator & Iterator,
uint64_t Width
@@ -560,31 +542,37 @@ dumpFastA (
uint64_t __l = Bases . size ();
- /*) First, we are doint base header
+ /*) First, we are doing base header
(*/
- kout << "<"
+ kout << '>'
<< CollectionName
- << "."
+ << '.'
<< SpotId
- << " "
+ << ' '
<< ReadName
<< " length="
<< __l
- << "\n"
+ << '\n'
;
- /*) Second is going base itsefl by width
+ /*) Second is going base itself by width
(*/
uint64_t __p = 0;
const char * __s = Bases . data ();
- while ( __p < __l ) {
- uint64_t __t = :: min ( Width, __l - __p );
- kout << string ( __s, __p, __t ) << "\n" ;
+ if ( 0 < Width ) {
+ while ( __p < __l ) {
+ uint64_t __t = std :: min ( Width, __l - __p );
- __p += __t;
+ kout << std :: string ( __s, ( std :: string :: size_type ) __p, ( std :: string :: size_type ) __t ) << "\n" ;
+
+ __p += __t;
+ }
+ }
+ else {
+ kout << __s << "\n" ;
}
} /* dumpFastA () */
@@ -597,25 +585,41 @@ void
run ( const DumpArgs & TheArgs )
{
if ( ! TheArgs . good () ) {
- throw ngs :: ErrorMsg ( "Invalid BC object" );
+ throw ErrorMsg ( "Invalid arguments" );
+ }
+
+ ngs :: String Acc ( TheArgs . accession () . c_str () );
+ ReadCollection RCol = ncbi :: NGS :: openReadCollection ( Acc );
+
+ int64_t minSpot = TheArgs . minSpotId ();
+ int64_t maxSpot = TheArgs . maxSpotId ();
+
+ if ( minSpot == 0 ) {
+ minSpot = 1;
}
- ngs :: ReadCollection RCol = NGS :: openReadCollection (
- TheArgs . accession () . c_str ()
- );
+ if ( maxSpot == 0 ) {
+ maxSpot = RCol . getReadCount ();
+ }
+
+ if ( maxSpot < minSpot ) {
+ int64_t Id = minSpot;
+ minSpot = maxSpot;
+ maxSpot = Id;
+ }
ReadIterator Iterator = RCol.getReadRange (
- TheArgs . minSpotId () ,
- TheArgs . maxSpotId () - TheArgs . minSpotId () + 1,
- TheArgs . category ()
- );
+ minSpot,
+ maxSpot - minSpot + 1,
+ TheArgs . category ()
+ );
ngs :: String ReadCollectionName = RCol.getName ();
AFilters Filters ( TheArgs . accession () );
setupFilters ( Filters, TheArgs );
- for ( uint64_t llp = TheArgs . minSpotId () ; Iterator.nextRead (); llp ++ ) {
+ for ( int64_t llp = TheArgs . minSpotId () ; Iterator.nextRead (); llp ++ ) {
if ( Filters . checkIt ( Iterator ) ) {
if ( TheArgs . fastaDump () ) {
@@ -627,7 +631,9 @@ run ( const DumpArgs & TheArgs )
}
}
- cerr << Filters . report ( TheArgs . legacyReport () );
+ kout.flush ();
+
+ std :: cerr << Filters . report ( TheArgs . legacyReport () );
} /* run () */
diff --git a/tools/fastq-dump/fastq-dump.vers b/tools/fastq-dump/fastq-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/fastq-dump/fastq-dump.vers
+++ b/tools/fastq-dump/fastq-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/fastq-dump/koutstream b/tools/fastq-dump/koutstream
index ad15dc3..5967f6f 100644
--- a/tools/fastq-dump/koutstream
+++ b/tools/fastq-dump/koutstream
@@ -36,6 +36,7 @@
#include <ostream>
#include <klib/out.h>
+#include <klib/text.h>
/*)))
/// Something weird. That class will throw exception if any
@@ -57,6 +58,14 @@ extern "C" {
}
#endif /* _DO_ALWAYS_THROW_EXCEPTION */
+/*\
+|*| That address is responsible for buffered/unbuffered basic_koutbuf
+|*| Uncomment if You does not need buffering on the basic_koutbuf level
+\*/
+
+/*\
+#define _DO_USE_UNBUFFERED_KOUTBUF
+\*/
/*))))
<<<< Lurics:
@@ -79,6 +88,7 @@ namespace ngs {
/*))
// Buffer
((*/
+
template < typename _CharT, typename _Traits >
class basic_koutbuf
: public std :: basic_streambuf < _CharT, _Traits > {
@@ -96,7 +106,14 @@ public :
protected :
+ /*\ This is not only change
+ \*/
+#ifdef _DO_USE_UNBUFFERED_KOUTBUF
+ static const std :: streamsize _cM_buffer_size = 256;
+#else /* _DO_USE_UNBUFFERED_KOUTBUF */
static const std :: streamsize _cM_buffer_size = 4096 * 32;
+#endif /* _DO_USE_UNBUFFERED_KOUTBUF */
+
char_type * _M_char;
public :
@@ -106,7 +123,7 @@ public :
: __streambuf_type ()
, _M_char ( allocator_type () . allocate ( _cM_buffer_size ) )
{
- this -> setbuf ( _M_char, sizeof ( _M_char ) - 1 );
+ this -> setbuf ( _M_char, _cM_buffer_size );
__setp ();
};
@@ -134,10 +151,20 @@ public :
_M_char = NULL;
};
+#ifdef _DO_USE_UNBUFFERED_KOUTBUF
std :: streamsize xsputn ( const char_type * __s, std :: streamsize __n )
{
- std :: streamsize __a = this -> epptr () - this -> pptr ();
+ if ( __sync () == - 1 ) {
+ return 0;
+ }
+ return __koutmsg ( __s, __n );
+ };
+#else /* _DO_USE_UNBUFFERED_KOUTBUF */
+ std :: streamsize xsputn ( const char_type * __s, std :: streamsize __n )
+ {
+ std :: streamsize __a = this -> epptr () - this -> pptr ();
+
if ( __n < __a ) {
/*) Here we are adding string to a buffer
(*/
@@ -180,25 +207,43 @@ public :
(*/
return __n;
};
+#endif /* _DO_USE_UNBUFFERED_KOUTBUF */
- virtual int sync ()
+ int sync ()
{
return __sync ();
};
protected :
- int __sync ()
+ int __sync ()
{
int __ret = 0;
- if ( this -> pbase () < this -> pptr () ) {
- /*) KOurMsg is working with null terminated
- (*/
- * ( this -> pptr () ) = 0;
+ std :: streamsize __l = this -> pptr () - this -> pbase ();
- rc_t __rc = KOutMsg ( "%s", this -> pbase () );
+ if ( 0 < __l ) {
+ __ret = __koutmsg ( this -> pbase (), __l );
+
+ /*\
+ |*| Don't really know if we shoud check result here
+ \*/
+ __setp ();
+ }
+
+ return __ret;
+ };
+
+ int __koutmsg ( const char_type * __s, std :: streamsize __n )
+ {
+ if ( 0 < __n ) {
+ :: String __t;
+
+ StringInit ( ( & __t ), __s, __n, __n );
+
+ rc_t __rc = KOutMsg ( "%S", & __t );
if ( __rc != 0 ) {
+
#ifdef _DO_ALWAYS_THROW_EXCEPTION
/* That was requested by Kurt: I shoud throw
@@ -211,22 +256,20 @@ protected :
throw ErrorMsg ( __m );
#else
- __ret = - 1;
+ __n = - 1;
#endif /* _DO_ALWAYS_THROW_EXCEPTION */
}
-
- __setp ();
}
- return __ret;
+ return __n;
};
- void __setp ()
+ void __setp ()
{
/*) the only valid method to set _M_out_cur to zero
/ also, one char reserved for a zero
(*/
- this -> setp ( _M_char, _M_char + sizeof ( _M_char ) - 1 );
+ this -> setp ( _M_char, _M_char + _cM_buffer_size );
};
}; /* class basic_koutbuf */
diff --git a/tools/fastq-loader/Makefile b/tools/fastq-loader/Makefile
index 184ab3a..e2552f7 100644
--- a/tools/fastq-loader/Makefile
+++ b/tools/fastq-loader/Makefile
@@ -102,8 +102,17 @@ FASTQ_SRC = \
fastq-grammar \
fastq-lex
-$(SRCDIR)/fastq-grammar.c $(SRCDIR)/fastq-tokens.h: $(SRCDIR)/fastq-grammar.y
- $(YACC) -o $(SRCDIR)/fastq-grammar.c --defines=$(SRCDIR)/fastq-tokens.h -v $^
+# flex/bison should only be invoked manually in an environment ensures the correct versions:
+# bison 2.5, flex 2.5.35
+bison:
+ bison --warnings=error -o $(SRCDIR)/fastq-grammar.c --defines=$(SRCDIR)/fastq-tokens.h -v --no-lines $(SRCDIR)/fastq-grammar.y
+
+ifeq (dbg,$(BUILD))
+ FLEX_DEBUG = -d
+endif
+
+flex:
+ flex $(FLEX_DEBUG) -DYY_BUF_SIZE=16777216 -t $(SRCDIR)/fastq-lex.l | grep -v '^#line' > $(SRCDIR)/fastq-lex.c
FASTQ_OBJ = \
$(addsuffix .$(LOBX),$(FASTQ_SRC))
diff --git a/tools/fastq-loader/fastq-grammar.c b/tools/fastq-loader/fastq-grammar.c
index 11cadf0..4a1fd9f 100644
--- a/tools/fastq-loader/fastq-grammar.c
+++ b/tools/fastq-loader/fastq-grammar.c
@@ -89,11 +89,15 @@
static void AddQuality(FASTQParseBlock* pb, const FASTQToken* token);
static void SetReadNumber(FASTQParseBlock* pb, const FASTQToken* token);
+ static void SetSpotGroup(FASTQParseBlock* pb, const FASTQToken* token);
+ static void SetRead(FASTQParseBlock* pb, const FASTQToken* token);
+
static void StartSpotName(FASTQParseBlock* pb, size_t offset);
static void GrowSpotName(FASTQParseBlock* pb, const FASTQToken* token);
static void StopSpotName(FASTQParseBlock* pb);
- static void SetSpotGroup(FASTQParseBlock* pb, const FASTQToken* token);
- static void SetRead(FASTQParseBlock* pb, const FASTQToken* token);
+ static void RestartSpotName(FASTQParseBlock* pb);
+ static void SaveSpotName(FASTQParseBlock* pb);
+ static void RevertSpotName(FASTQParseBlock* pb);
#define UNLEX do { if (yychar != YYEMPTY && yychar != YYEOF) FASTQ_unlex(pb, & yylval); } while (0)
@@ -370,18 +374,18 @@ union yyalloc
#endif /* !YYCOPY_NEEDED */
/* YYFINAL -- State number of the termination state. */
-#define YYFINAL 21
+#define YYFINAL 19
/* YYLAST -- Last index in YYTABLE. */
-#define YYLAST 126
+#define YYLAST 151
/* YYNTOKENS -- Number of terminals. */
#define YYNTOKENS 24
/* YYNNTS -- Number of nonterminals. */
-#define YYNNTS 54
+#define YYNNTS 52
/* YYNRULES -- Number of rules. */
-#define YYNRULES 102
+#define YYNRULES 105
/* YYNRULES -- Number of states. */
-#define YYNSTATES 147
+#define YYNSTATES 150
/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */
#define YYUNDEFTOK 2
@@ -396,13 +400,13 @@ static const yytype_uint8 yytranslate[] =
0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 20, 2, 2, 2, 2,
- 2, 2, 2, 23, 2, 19, 18, 21, 2, 2,
+ 2, 2, 2, 2, 2, 21, 2, 2, 2, 2,
+ 2, 2, 2, 23, 2, 20, 19, 22, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 14, 2,
- 2, 22, 16, 2, 15, 2, 2, 2, 2, 2,
+ 2, 17, 16, 2, 15, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 17, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 18, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -427,68 +431,70 @@ static const yytype_uint8 yytranslate[] =
YYRHS. */
static const yytype_uint16 yyprhs[] =
{
- 0, 0, 3, 6, 8, 10, 11, 12, 13, 14,
- 25, 29, 31, 33, 36, 38, 42, 47, 51, 52,
- 56, 57, 61, 63, 65, 66, 70, 71, 76, 77,
- 81, 82, 87, 89, 91, 93, 96, 98, 99, 103,
- 104, 109, 110, 116, 117, 123, 124, 130, 131, 137,
- 138, 145, 147, 149, 152, 155, 158, 161, 164, 167,
- 168, 172, 173, 177, 178, 180, 181, 185, 186, 190,
- 191, 196, 197, 201, 202, 207, 208, 209, 210, 211,
- 212, 213, 227, 229, 231, 232, 234, 237, 240, 243,
- 246, 249, 253, 257, 261, 265, 270, 274, 279, 281,
- 284, 287, 291
+ 0, 0, 3, 6, 8, 9, 10, 11, 21, 25,
+ 27, 29, 32, 34, 38, 43, 47, 50, 51, 55,
+ 56, 61, 62, 66, 68, 70, 71, 75, 76, 81,
+ 82, 86, 87, 92, 94, 96, 98, 101, 111, 116,
+ 120, 121, 126, 127, 132, 134, 137, 141, 143, 145,
+ 148, 149, 153, 156, 160, 164, 167, 170, 171, 177,
+ 178, 184, 185, 191, 192, 199, 203, 205, 207, 210,
+ 213, 216, 219, 222, 225, 226, 230, 231, 235, 237,
+ 238, 242, 243, 248, 250, 251, 252, 253, 254, 255,
+ 267, 268, 272, 273, 275, 277, 278, 282, 286, 288,
+ 292, 297, 299, 302, 305, 309
};
/* YYRHS -- A `-1'-separated list of the rules' RHS. */
static const yytype_int8 yyrhs[] =
{
- 25, 0, -1, 32, 74, -1, 32, -1, 74, -1,
- -1, -1, -1, -1, 53, 26, 12, 27, 14, 28,
- 43, 14, 29, 76, -1, 5, 1, 31, -1, 30,
- -1, 0, -1, 31, 30, -1, 7, -1, 33, 31,
- 36, -1, 33, 31, 1, 31, -1, 1, 31, 36,
- -1, -1, 15, 34, 44, -1, -1, 16, 35, 44,
- -1, 37, -1, 40, -1, -1, 8, 38, 31, -1,
- -1, 37, 8, 39, 31, -1, -1, 9, 41, 31,
- -1, -1, 40, 9, 42, 31, -1, 8, -1, 9,
- -1, 45, -1, 45, 57, -1, 73, -1, -1, 53,
- 46, 54, -1, -1, 53, 12, 47, 54, -1, -1,
- 53, 12, 17, 48, 64, -1, -1, 53, 12, 14,
- 49, 54, -1, -1, 53, 12, 14, 50, 53, -1,
- -1, 53, 12, 18, 51, 53, -1, -1, 53, 12,
- 14, 18, 52, 53, -1, 5, -1, 4, -1, 53,
- 17, -1, 53, 19, -1, 53, 18, -1, 53, 14,
- -1, 53, 5, -1, 53, 4, -1, -1, 20, 55,
- 4, -1, -1, 20, 56, 5, -1, -1, 61, -1,
- -1, 6, 58, 64, -1, -1, 6, 59, 72, -1,
- -1, 57, 6, 60, 72, -1, -1, 21, 62, 4,
- -1, -1, 61, 21, 63, 53, -1, -1, -1, -1,
- -1, -1, -1, 4, 65, 14, 66, 5, 67, 14,
- 68, 4, 69, 14, 70, 71, -1, 5, -1, 4,
- -1, -1, 5, -1, 72, 4, -1, 72, 5, -1,
- 72, 17, -1, 72, 21, -1, 72, 22, -1, 3,
- 18, 4, -1, 3, 21, 4, -1, 73, 6, 72,
- -1, 73, 6, 4, -1, 73, 6, 4, 72, -1,
- 75, 31, 76, -1, 75, 31, 1, 31, -1, 23,
- -1, 75, 10, -1, 77, 31, -1, 76, 77, 31,
- -1, 11, -1
+ 25, 0, -1, 31, 72, -1, 31, -1, -1, -1,
+ -1, 55, 12, 26, 14, 27, 43, 14, 28, 74,
+ -1, 5, 1, 30, -1, 29, -1, 0, -1, 30,
+ 29, -1, 7, -1, 32, 30, 36, -1, 32, 30,
+ 1, 30, -1, 1, 30, 36, -1, 1, 30, -1,
+ -1, 15, 33, 44, -1, -1, 15, 6, 34, 44,
+ -1, -1, 16, 35, 44, -1, 37, -1, 40, -1,
+ -1, 8, 38, 30, -1, -1, 37, 8, 39, 30,
+ -1, -1, 9, 41, 30, -1, -1, 40, 9, 42,
+ 30, -1, 8, -1, 9, -1, 47, -1, 47, 59,
+ -1, 47, 59, 6, 4, 14, 5, 14, 4, 68,
+ -1, 47, 59, 6, 5, -1, 47, 59, 6, -1,
+ -1, 47, 6, 45, 62, -1, -1, 47, 6, 46,
+ 5, -1, 71, -1, 55, 59, -1, 55, 59, 6,
+ -1, 55, -1, 50, -1, 50, 56, -1, -1, 55,
+ 48, 56, -1, 49, 50, -1, 49, 50, 56, -1,
+ 49, 5, 17, -1, 55, 6, -1, 55, 12, -1,
+ -1, 55, 12, 18, 51, 62, -1, -1, 55, 12,
+ 14, 52, 55, -1, -1, 55, 12, 19, 53, 55,
+ -1, -1, 55, 12, 14, 19, 54, 55, -1, 55,
+ 12, 14, -1, 5, -1, 4, -1, 55, 18, -1,
+ 55, 20, -1, 55, 19, -1, 55, 14, -1, 55,
+ 5, -1, 55, 4, -1, -1, 21, 57, 4, -1,
+ -1, 21, 58, 5, -1, 21, -1, -1, 22, 60,
+ 4, -1, -1, 59, 22, 61, 55, -1, 4, -1,
+ -1, -1, -1, -1, -1, 4, 63, 14, 64, 5,
+ 65, 14, 66, 4, 67, 68, -1, -1, 14, 69,
+ 70, -1, -1, 8, -1, 4, -1, -1, 3, 19,
+ 4, -1, 3, 22, 4, -1, 3, -1, 73, 30,
+ 74, -1, 73, 30, 1, 30, -1, 23, -1, 73,
+ 10, -1, 75, 30, -1, 74, 75, 30, -1, 11,
+ -1
};
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
- 0, 77, 77, 79, 81, 83, 84, 85, 87, 83,
- 90, 92, 96, 97, 101, 105, 106, 107, 111, 111,
- 112, 112, 116, 117, 121, 121, 123, 123, 128, 128,
- 130, 130, 135, 136, 141, 142, 143, 147, 147, 149,
- 149, 152, 151, 158, 158, 160, 160, 162, 162, 164,
- 164, 169, 170, 171, 172, 173, 174, 175, 176, 180,
- 180, 182, 182, 184, 188, 189, 189, 191, 191, 193,
- 193, 198, 197, 210, 209, 221, 222, 223, 224, 225,
- 226, 221, 231, 232, 233, 237, 238, 239, 240, 241,
- 242, 246, 247, 248, 249, 250, 256, 257, 261, 262,
- 266, 267, 270
+ 0, 81, 81, 83, 88, 89, 91, 87, 94, 96,
+ 100, 101, 105, 109, 110, 111, 112, 116, 116, 117,
+ 117, 118, 118, 122, 123, 127, 127, 129, 129, 134,
+ 134, 136, 136, 141, 142, 147, 148, 150, 152, 153,
+ 155, 155, 156, 156, 157, 158, 159, 160, 164, 165,
+ 166, 166, 167, 168, 169, 173, 183, 185, 184, 191,
+ 191, 192, 192, 193, 193, 194, 198, 199, 200, 201,
+ 202, 203, 204, 205, 209, 209, 211, 211, 213, 218,
+ 217, 230, 229, 241, 242, 243, 244, 245, 246, 242,
+ 251, 251, 252, 256, 257, 258, 262, 263, 264, 270,
+ 271, 275, 276, 280, 281, 284
};
#endif
@@ -499,16 +505,15 @@ static const char *const yytname[] =
{
"fqENDOFTEXT", "error", "$undefined", "fqRUNDOTSPOT", "fqNUMBER",
"fqALPHANUM", "fqWS", "fqENDLINE", "fqBASESEQ", "fqCOLORSEQ", "fqTOKEN",
- "fqASCQUAL", "fqCOORDS", "fqUNRECOGNIZED", "':'", "'@'", "'>'", "'_'",
- "'.'", "'-'", "'#'", "'/'", "'='", "'+'", "$accept", "sequence", "$@1",
- "$@2", "$@3", "$@4", "endfile", "endline", "readLines", "header", "$@5",
+ "fqASCQUAL", "fqCOORDS", "fqUNRECOGNIZED", "':'", "'@'", "'>'", "'='",
+ "'_'", "'.'", "'-'", "'#'", "'/'", "'+'", "$accept", "sequence", "$@1",
+ "$@2", "$@3", "endfile", "endline", "readLines", "header", "$@4", "$@5",
"$@6", "read", "baseRead", "$@7", "$@8", "csRead", "$@9", "$@10",
- "inlineRead", "tagLine", "nameSpotGroup", "$@11", "$@12", "$@13", "$@14",
- "$@15", "$@16", "$@17", "name", "spotGroup", "$@18", "$@19",
- "readNumberOrTail", "$@20", "$@21", "$@22", "readNumber", "$@23", "$@24",
- "casava1_8", "$@25", "$@26", "$@27", "$@28", "$@29", "$@30",
- "indexSequence", "tail", "runSpotRead", "qualityLines", "qualityHeader",
- "quality", "qualityLine", 0
+ "inlineRead", "tagLine", "$@11", "$@12", "nameSpotGroup", "$@13",
+ "nameWS", "nameWithCoords", "$@14", "$@15", "$@16", "$@17", "name",
+ "spotGroup", "$@18", "$@19", "readNumber", "$@20", "$@21", "casava1_8",
+ "$@22", "$@23", "$@24", "$@25", "$@26", "indexSequence", "$@27", "index",
+ "runSpotRead", "qualityLines", "qualityHeader", "quality", "qualityLine", 0
};
#endif
@@ -518,41 +523,41 @@ static const char *const yytname[] =
static const yytype_uint16 yytoknum[] =
{
0, 256, 257, 258, 259, 260, 261, 262, 263, 264,
- 265, 266, 267, 268, 58, 64, 62, 95, 46, 45,
- 35, 47, 61, 43
+ 265, 266, 267, 268, 58, 64, 62, 61, 95, 46,
+ 45, 35, 47, 43
};
# endif
/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
static const yytype_uint8 yyr1[] =
{
- 0, 24, 25, 25, 25, 26, 27, 28, 29, 25,
- 25, 25, 30, 30, 31, 32, 32, 32, 34, 33,
- 35, 33, 36, 36, 38, 37, 39, 37, 41, 40,
- 42, 40, 43, 43, 44, 44, 44, 46, 45, 47,
- 45, 48, 45, 49, 45, 50, 45, 51, 45, 52,
- 45, 53, 53, 53, 53, 53, 53, 53, 53, 55,
- 54, 56, 54, 54, 57, 58, 57, 59, 57, 60,
- 57, 62, 61, 63, 61, 65, 66, 67, 68, 69,
- 70, 64, 71, 71, 71, 72, 72, 72, 72, 72,
- 72, 73, 73, 73, 73, 73, 74, 74, 75, 75,
- 76, 76, 77
+ 0, 24, 25, 25, 26, 27, 28, 25, 25, 25,
+ 29, 29, 30, 31, 31, 31, 31, 33, 32, 34,
+ 32, 35, 32, 36, 36, 38, 37, 39, 37, 41,
+ 40, 42, 40, 43, 43, 44, 44, 44, 44, 44,
+ 45, 44, 46, 44, 44, 44, 44, 44, 47, 47,
+ 48, 47, 47, 47, 47, 49, 50, 51, 50, 52,
+ 50, 53, 50, 54, 50, 50, 55, 55, 55, 55,
+ 55, 55, 55, 55, 57, 56, 58, 56, 56, 60,
+ 59, 61, 59, 62, 63, 64, 65, 66, 67, 62,
+ 69, 68, 68, 70, 70, 70, 71, 71, 71, 72,
+ 72, 73, 73, 74, 74, 75
};
/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */
static const yytype_uint8 yyr2[] =
{
- 0, 2, 2, 1, 1, 0, 0, 0, 0, 10,
- 3, 1, 1, 2, 1, 3, 4, 3, 0, 3,
- 0, 3, 1, 1, 0, 3, 0, 4, 0, 3,
- 0, 4, 1, 1, 1, 2, 1, 0, 3, 0,
- 4, 0, 5, 0, 5, 0, 5, 0, 5, 0,
- 6, 1, 1, 2, 2, 2, 2, 2, 2, 0,
- 3, 0, 3, 0, 1, 0, 3, 0, 3, 0,
- 4, 0, 3, 0, 4, 0, 0, 0, 0, 0,
- 0, 13, 1, 1, 0, 1, 2, 2, 2, 2,
- 2, 3, 3, 3, 3, 4, 3, 4, 1, 2,
- 2, 3, 1
+ 0, 2, 2, 1, 0, 0, 0, 9, 3, 1,
+ 1, 2, 1, 3, 4, 3, 2, 0, 3, 0,
+ 4, 0, 3, 1, 1, 0, 3, 0, 4, 0,
+ 3, 0, 4, 1, 1, 1, 2, 9, 4, 3,
+ 0, 4, 0, 4, 1, 2, 3, 1, 1, 2,
+ 0, 3, 2, 3, 3, 2, 2, 0, 5, 0,
+ 5, 0, 5, 0, 6, 3, 1, 1, 2, 2,
+ 2, 2, 2, 2, 0, 3, 0, 3, 1, 0,
+ 3, 0, 4, 1, 0, 0, 0, 0, 0, 11,
+ 0, 3, 0, 1, 1, 0, 3, 3, 1, 3,
+ 4, 1, 2, 2, 3, 1
};
/* YYDEFACT[STATE-NAME] -- Default reduction number in state STATE-NUM.
@@ -560,130 +565,136 @@ static const yytype_uint8 yyr2[] =
means the default is an error. */
static const yytype_uint8 yydefact[] =
{
- 0, 12, 0, 52, 0, 14, 18, 20, 98, 0,
- 11, 0, 3, 0, 5, 4, 0, 0, 0, 0,
- 0, 1, 13, 2, 0, 58, 57, 56, 53, 55,
- 54, 0, 99, 0, 24, 28, 17, 22, 23, 10,
- 0, 51, 19, 34, 37, 36, 21, 0, 15, 6,
- 0, 102, 96, 0, 0, 0, 26, 30, 0, 0,
- 65, 71, 35, 64, 39, 63, 0, 16, 0, 97,
- 0, 100, 25, 29, 0, 0, 91, 92, 0, 0,
- 0, 69, 73, 43, 41, 47, 63, 59, 38, 94,
- 85, 93, 7, 101, 27, 31, 75, 66, 68, 72,
- 0, 0, 49, 63, 0, 0, 0, 40, 0, 0,
- 95, 86, 87, 88, 89, 90, 0, 0, 70, 74,
- 0, 44, 46, 42, 48, 60, 62, 32, 33, 0,
- 76, 50, 8, 0, 0, 77, 9, 0, 78, 0,
- 79, 0, 80, 84, 83, 82, 81
+ 0, 10, 0, 67, 0, 12, 17, 21, 0, 9,
+ 0, 3, 0, 0, 16, 0, 19, 0, 0, 1,
+ 11, 101, 2, 0, 0, 73, 72, 4, 71, 68,
+ 70, 69, 25, 29, 15, 23, 24, 8, 0, 98,
+ 66, 18, 35, 0, 48, 47, 44, 22, 102, 0,
+ 0, 13, 0, 0, 0, 27, 31, 20, 0, 0,
+ 40, 79, 36, 66, 52, 0, 78, 49, 55, 56,
+ 0, 45, 0, 105, 99, 0, 14, 5, 26, 30,
+ 0, 0, 96, 97, 0, 0, 0, 39, 81, 54,
+ 53, 0, 0, 65, 57, 61, 51, 46, 100, 0,
+ 103, 0, 28, 32, 83, 41, 43, 80, 0, 38,
+ 0, 75, 77, 63, 0, 0, 0, 104, 33, 34,
+ 0, 0, 0, 82, 0, 60, 58, 62, 6, 85,
+ 0, 64, 0, 0, 0, 7, 86, 92, 0, 90,
+ 37, 87, 95, 0, 94, 93, 91, 88, 92, 89
};
/* YYDEFGOTO[NTERM-NUM]. */
static const yytype_int16 yydefgoto[] =
{
- -1, 9, 31, 68, 116, 134, 10, 11, 12, 13,
- 19, 20, 36, 37, 54, 74, 38, 55, 75, 129,
- 42, 43, 65, 86, 105, 103, 104, 106, 120, 44,
- 88, 108, 109, 62, 78, 79, 100, 63, 80, 101,
- 97, 117, 133, 137, 139, 141, 143, 146, 91, 45,
- 15, 16, 52, 53
+ -1, 8, 52, 101, 132, 9, 10, 11, 12, 17,
+ 38, 18, 34, 35, 53, 80, 36, 54, 81, 120,
+ 41, 84, 85, 42, 70, 43, 44, 115, 114, 116,
+ 124, 45, 67, 91, 92, 62, 86, 110, 105, 121,
+ 133, 138, 143, 148, 140, 142, 146, 46, 22, 23,
+ 74, 75
};
/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
STATE-NUM. */
-#define YYPACT_NINF -80
-static const yytype_int8 yypact[] =
+#define YYPACT_NINF -73
+static const yytype_int16 yypact[] =
{
- 5, -80, -5, -80, 21, -80, -80, -80, -80, 16,
- -80, 4, 30, -5, 73, -80, 42, 23, -5, 76,
- 76, -80, -80, -80, 33, -80, -80, -80, -80, -80,
- -80, 15, -80, 12, -80, -80, -80, 50, 59, -80,
- 45, -80, -80, 9, 55, 64, -80, -5, -80, -80,
- -5, -80, 75, -5, -5, -5, -80, -80, 91, 92,
- 79, -80, 93, 77, 29, 82, 60, -80, 83, -80,
- -5, -80, -80, -80, -5, -5, -80, -80, 96, 98,
- 101, -80, -80, 32, -80, -80, 82, 102, -80, 98,
- -80, 40, -80, -80, -80, -80, -80, -80, 40, -80,
- 98, 78, -80, 82, 78, 96, 78, -80, 104, 105,
- 40, -80, -80, -80, -80, -80, 80, 95, 40, 73,
- 78, -80, 73, -80, 73, -80, -80, -80, -80, 97,
- -80, 73, -80, 107, 75, -80, 75, 99, -80, 110,
- -80, 103, -80, 89, -80, -80, -80
+ 40, -73, 1, -73, 12, -73, 36, -73, 18, -73,
+ 39, 25, 1, 46, -2, 1, -73, 104, 104, -73,
+ -73, -73, -73, 80, 83, -73, -73, -73, -73, -73,
+ -73, -73, -73, -73, -73, 49, 50, -73, 104, 84,
+ -73, -73, 3, 92, 51, 15, -73, -73, -73, 4,
+ 1, -73, 56, 1, 1, -73, -73, -73, 74, 98,
+ 108, -73, 6, 71, 51, 63, 100, -73, -73, 75,
+ 51, 32, 1, -73, 110, 1, -73, -73, -73, -73,
+ 1, 1, -73, -73, 111, 117, 119, 107, -73, -73,
+ -73, 121, 122, 57, -73, -73, -73, -73, -73, 1,
+ -73, 109, -73, -73, 112, -73, -73, -73, 114, -73,
+ 115, -73, -73, -73, 115, 111, 115, -73, -73, -73,
+ 116, 118, 124, 81, 115, 81, -73, 81, -73, -73,
+ 120, 81, 110, 126, 129, 110, -73, 123, 125, -73,
+ -73, -73, 65, 131, -73, -73, -73, -73, 123, -73
};
/* YYPGOTO[NTERM-NUM]. */
-static const yytype_int8 yypgoto[] =
+static const yytype_int16 yypgoto[] =
{
- -80, -80, -80, -80, -80, -80, 108, 1, -80, -80,
- -80, -80, 94, -80, -80, -80, -80, -80, -80, -80,
- 106, -80, -80, -80, -80, -80, -80, -80, -80, 0,
- -79, -80, -80, -80, -80, -80, -80, -80, -80, -80,
- 10, -80, -80, -80, -80, -80, -80, -80, -71, -80,
- 109, -80, -18, -51
+ -73, -73, -73, -73, -73, 128, -1, -73, -73, -73,
+ -73, -73, 127, -73, -73, -73, -73, -73, -73, -73,
+ -15, -73, -73, -73, -73, -73, 93, -73, -73, -73,
+ -73, 0, -60, -73, -73, 95, -73, -73, 26, -73,
+ -73, -73, -73, -73, -6, -73, -73, -73, -73, -73,
+ 11, -72
};
/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If
positive, shift that token. If negative, reduce the rule which
number is the opposite. If YYTABLE_NINF, syntax error. */
-#define YYTABLE_NINF -68
+#define YYTABLE_NINF -85
static const yytype_int16 yytable[] =
{
- 14, 70, 5, 17, 1, 1, 2, 107, 98, 3,
- 4, 5, 5, 50, 24, 60, 21, 33, 110, 39,
- 6, 7, 18, 51, 121, -51, -51, 49, 8, 118,
- 61, 34, 35, -51, 47, -51, -45, -45, -51, -51,
- -51, 34, 35, 83, 111, 112, 84, 85, 67, 5,
- 102, 69, 32, 8, 71, 72, 73, 113, 56, 25,
- 26, 114, 115, 58, 89, 90, 59, 64, 57, 27,
- 66, 93, 28, 29, 30, 94, 95, 25, 26, 40,
- 3, 41, 3, 41, -67, 70, 51, 27, 127, 128,
- 28, 29, 30, 144, 145, 76, 77, 92, 82, 81,
- 96, 119, 87, 90, 122, 99, 124, -61, 125, 130,
- 126, 132, 135, 138, 140, 123, 136, 142, 48, 22,
- 131, 23, 0, 0, 0, 0, 46
+ 13, 14, 99, 47, 90, 72, 32, 33, 5, 60,
+ 96, 24, 87, 15, 37, 73, -66, -66, 19, 25,
+ 26, 68, 49, 57, -66, 61, -66, 69, 88, 28,
+ -66, -66, -66, 29, 30, 31, -50, 61, 97, 1,
+ 1, 2, 16, 65, 3, 4, 5, 5, 21, 76,
+ 25, 26, 78, 79, 88, 6, 7, 55, 27, 56,
+ 28, -59, -59, 99, 29, 30, 31, 25, 26, 144,
+ 77, 98, 66, 145, 100, 69, 113, 28, 82, 102,
+ 103, 29, 30, 31, 50, 25, 26, 5, 89, 93,
+ 48, 32, 33, 94, 95, 28, 3, 63, 117, 29,
+ 30, 31, 83, 58, -74, -76, 59, 39, 3, 40,
+ 123, 108, 109, -42, 125, 104, 127, 118, 119, 3,
+ 40, 73, 106, 107, 131, 111, -84, 112, 122, 130,
+ 128, 136, 129, 137, 134, 147, 64, 139, 20, 141,
+ 71, 126, 149, 135, 0, 0, 0, 0, 0, 0,
+ 0, 51
};
#define yypact_value_is_default(yystate) \
- ((yystate) == (-80))
+ ((yystate) == (-73))
#define yytable_value_is_error(yytable_value) \
YYID (0)
static const yytype_int16 yycheck[] =
{
- 0, 52, 7, 2, 0, 0, 1, 86, 79, 4,
- 5, 7, 7, 1, 13, 6, 0, 16, 89, 18,
- 15, 16, 1, 11, 103, 4, 5, 12, 23, 100,
- 21, 8, 9, 12, 1, 14, 4, 5, 17, 18,
- 19, 8, 9, 14, 4, 5, 17, 18, 47, 7,
- 18, 50, 10, 23, 53, 54, 55, 17, 8, 4,
- 5, 21, 22, 18, 4, 5, 21, 12, 9, 14,
- 6, 70, 17, 18, 19, 74, 75, 4, 5, 3,
- 4, 5, 4, 5, 5, 136, 11, 14, 8, 9,
- 17, 18, 19, 4, 5, 4, 4, 14, 21, 6,
- 4, 101, 20, 5, 104, 4, 106, 5, 4, 14,
- 5, 14, 5, 14, 4, 105, 134, 14, 24, 11,
- 120, 12, -1, -1, -1, -1, 20
+ 0, 2, 74, 18, 64, 1, 8, 9, 7, 6,
+ 70, 12, 6, 1, 15, 11, 4, 5, 0, 4,
+ 5, 6, 23, 38, 12, 22, 14, 12, 22, 14,
+ 18, 19, 20, 18, 19, 20, 21, 22, 6, 0,
+ 0, 1, 6, 43, 4, 5, 7, 7, 23, 50,
+ 4, 5, 53, 54, 22, 15, 16, 8, 12, 9,
+ 14, 4, 5, 135, 18, 19, 20, 4, 5, 4,
+ 14, 72, 21, 8, 75, 12, 19, 14, 4, 80,
+ 81, 18, 19, 20, 1, 4, 5, 7, 17, 14,
+ 10, 8, 9, 18, 19, 14, 4, 5, 99, 18,
+ 19, 20, 4, 19, 4, 5, 22, 3, 4, 5,
+ 110, 4, 5, 5, 114, 4, 116, 8, 9, 4,
+ 5, 11, 5, 4, 124, 4, 14, 5, 14, 5,
+ 14, 5, 14, 4, 14, 4, 43, 14, 10, 14,
+ 45, 115, 148, 132, -1, -1, -1, -1, -1, -1,
+ -1, 24
};
/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
symbol of state STATE-NUM. */
static const yytype_uint8 yystos[] =
{
- 0, 0, 1, 4, 5, 7, 15, 16, 23, 25,
- 30, 31, 32, 33, 53, 74, 75, 31, 1, 34,
- 35, 0, 30, 74, 31, 4, 5, 14, 17, 18,
- 19, 26, 10, 31, 8, 9, 36, 37, 40, 31,
- 3, 5, 44, 45, 53, 73, 44, 1, 36, 12,
- 1, 11, 76, 77, 38, 41, 8, 9, 18, 21,
- 6, 21, 57, 61, 12, 46, 6, 31, 27, 31,
- 77, 31, 31, 31, 39, 42, 4, 4, 58, 59,
- 62, 6, 21, 14, 17, 18, 47, 20, 54, 4,
- 5, 72, 14, 31, 31, 31, 4, 64, 72, 4,
- 60, 63, 18, 49, 50, 48, 51, 54, 55, 56,
- 72, 4, 5, 17, 21, 22, 28, 65, 72, 53,
- 52, 54, 53, 64, 53, 4, 5, 8, 9, 43,
- 14, 53, 14, 66, 29, 5, 76, 67, 14, 68,
- 4, 69, 14, 70, 4, 5, 71
+ 0, 0, 1, 4, 5, 7, 15, 16, 25, 29,
+ 30, 31, 32, 55, 30, 1, 6, 33, 35, 0,
+ 29, 23, 72, 73, 30, 4, 5, 12, 14, 18,
+ 19, 20, 8, 9, 36, 37, 40, 30, 34, 3,
+ 5, 44, 47, 49, 50, 55, 71, 44, 10, 30,
+ 1, 36, 26, 38, 41, 8, 9, 44, 19, 22,
+ 6, 22, 59, 5, 50, 55, 21, 56, 6, 12,
+ 48, 59, 1, 11, 74, 75, 30, 14, 30, 30,
+ 39, 42, 4, 4, 45, 46, 60, 6, 22, 17,
+ 56, 57, 58, 14, 18, 19, 56, 6, 30, 75,
+ 30, 27, 30, 30, 4, 62, 5, 4, 4, 5,
+ 61, 4, 5, 19, 52, 51, 53, 30, 8, 9,
+ 43, 63, 14, 55, 54, 55, 62, 55, 14, 14,
+ 5, 55, 28, 64, 14, 74, 5, 4, 65, 14,
+ 68, 14, 69, 66, 4, 8, 70, 4, 67, 68
};
#define yyerrok (yyerrstatus = 0)
@@ -1534,194 +1545,185 @@ yyreduce:
case 4:
- { UNLEX; return 1; }
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); StopSpotName(pb); }
break;
case 5:
- { StartSpotName(pb, 0); }
+ { FASTQScan_inline_sequence(pb); }
break;
case 6:
- { GrowSpotName(pb, &(yyvsp[(3) - (3)])); StopSpotName(pb); }
+ { FASTQScan_inline_quality(pb); }
break;
case 7:
- { FASTQScan_inline_sequence(pb); }
+ { UNLEX; return 1; }
break;
case 8:
- { FASTQScan_inline_quality(pb); }
- break;
-
- case 9:
-
{ UNLEX; return 1; }
break;
- case 10:
+ case 9:
- { UNLEX; return 1; }
+ { return 0; }
break;
- case 11:
+ case 17:
- { return 0; }
+ { StartSpotName(pb, 1); }
break;
- case 18:
+ case 19:
- { StartSpotName(pb, 1); }
+ { StartSpotName(pb, 1 + (yyvsp[(2) - (2)]).tokenLength); }
break;
- case 20:
+ case 21:
{ StartSpotName(pb, 1); }
break;
- case 22:
+ case 23:
{ pb->record->seq.is_colorspace = false; }
break;
- case 23:
+ case 24:
{ pb->record->seq.is_colorspace = true; }
break;
- case 24:
+ case 25:
{ SetRead(pb, & (yyvsp[(1) - (1)])); }
break;
- case 26:
+ case 27:
{ SetRead(pb, & (yyvsp[(2) - (2)])); }
break;
- case 28:
+ case 29:
{ SetRead(pb, & (yyvsp[(1) - (1)])); }
break;
- case 30:
+ case 31:
{ SetRead(pb, & (yyvsp[(2) - (2)])); }
break;
- case 32:
+ case 33:
{ SetRead(pb, & (yyvsp[(1) - (1)])); pb->record->seq.is_colorspace = false; }
break;
- case 33:
+ case 34:
{ SetRead(pb, & (yyvsp[(1) - (1)])); pb->record->seq.is_colorspace = true; }
break;
- case 37:
+ case 38:
- { StopSpotName(pb); }
+ { FASTQScan_skip_to_eol(pb); }
break;
case 39:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); StopSpotName(pb); }
+ { FASTQScan_skip_to_eol(pb); }
break;
- case 41:
+ case 40:
- { /* another crazy variation by Illumina, this time "_" is used as " /" */
- GrowSpotName(pb, &(yyvsp[(2) - (3)]));
- StopSpotName(pb);
- GrowSpotName(pb, &(yyvsp[(3) - (3)]));
- }
+ { GrowSpotName(pb, &(yyvsp[(1) - (2)])); StopSpotName(pb); }
break;
- case 43:
+ case 41:
- { GrowSpotName(pb, &(yyvsp[(2) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)])); StopSpotName(pb); }
+ { FASTQScan_skip_to_eol(pb); }
break;
- case 45:
+ case 42:
- { GrowSpotName(pb, &(yyvsp[(2) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)]));}
+ { GrowSpotName(pb, &(yyvsp[(1) - (2)])); StopSpotName(pb); }
break;
- case 47:
+ case 43:
- { GrowSpotName(pb, &(yyvsp[(2) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)]));}
+ { FASTQScan_skip_to_eol(pb); }
break;
- case 49:
+ case 44:
- { GrowSpotName(pb, &(yyvsp[(2) - (4)])); GrowSpotName(pb, &(yyvsp[(3) - (4)])); GrowSpotName(pb, &(yyvsp[(3) - (4)]));}
+ { FASTQScan_skip_to_eol(pb); }
break;
- case 51:
+ case 46:
- { GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ { FASTQScan_skip_to_eol(pb); }
break;
- case 52:
+ case 50:
- { GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
- break;
-
- case 53:
-
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { StopSpotName(pb); }
break;
case 54:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { RevertSpotName(pb); FASTQScan_skip_to_eol(pb); }
break;
case 55:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { /* 'name' without coordinates attached will be ignored if followed by a name with coordinates (see the previous production).
+ however, if not followed, this will be the spot name, so we need to save the 'name's coordinates in case
+ we need to revert to them later (see call to RevertSpotName() above) */
+ SaveSpotName(pb);
+ GrowSpotName(pb, &(yyvsp[(2) - (2)])); /* need to account for white space but it is not part of the spot name */
+ RestartSpotName(pb); /* clean up for the potential nameWithCoords to start here */
+ }
break;
case 56:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); StopSpotName(pb); }
break;
case 57:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
- break;
-
- case 58:
-
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { /* another variation by Illumina, this time "_" is used as " /" */
+ GrowSpotName(pb, &(yyvsp[(2) - (3)]));
+ StopSpotName(pb);
+ GrowSpotName(pb, &(yyvsp[(3) - (3)]));
+ }
break;
case 59:
- { GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ { GrowSpotName(pb, &(yyvsp[(2) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)]));}
break;
- case 60:
+ case 61:
- { SetSpotGroup(pb, &(yyvsp[(3) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)])); }
+ { GrowSpotName(pb, &(yyvsp[(2) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)]));}
break;
- case 61:
+ case 63:
- { GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ { GrowSpotName(pb, &(yyvsp[(2) - (4)])); GrowSpotName(pb, &(yyvsp[(3) - (4)])); GrowSpotName(pb, &(yyvsp[(4) - (4)]));}
break;
- case 62:
+ case 65:
- { SetSpotGroup(pb, &(yyvsp[(3) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)])); }
+ { GrowSpotName(pb, &(yyvsp[(2) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)])); StopSpotName(pb); }
break;
- case 65:
+ case 66:
{ GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
break;
@@ -1731,124 +1733,154 @@ yyreduce:
{ GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
break;
+ case 68:
+
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ break;
+
case 69:
{ GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
break;
+ case 70:
+
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ break;
+
case 71:
- { /* in PACBIO fastq, the first '/' and the following digits are treated as a continuation of the spot name, not a read number */
- if (IS_PACBIO(pb)) pb->spotNameDone = false;
- GrowSpotName(pb, &(yyvsp[(1) - (1)]));
- }
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
break;
case 72:
- {
- if (!IS_PACBIO(pb)) SetReadNumber(pb, &(yyvsp[(3) - (3)]));
- GrowSpotName(pb, &(yyvsp[(3) - (3)]));
- StopSpotName(pb);
- }
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
break;
case 73:
- {
- if (IS_PACBIO(pb)) pb->spotNameDone = false;
- GrowSpotName(pb, &(yyvsp[(2) - (2)]));
- }
+ { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
break;
case 74:
- {
- if (IS_PACBIO(pb)) StopSpotName(pb);
- }
+ { StopSpotName(pb); }
break;
case 75:
- { SetReadNumber(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); StopSpotName(pb); }
+ { SetSpotGroup(pb, &(yyvsp[(3) - (3)])); }
break;
case 76:
- { GrowSpotName(pb, &(yyvsp[(3) - (3)])); }
+ { StopSpotName(pb); }
break;
case 77:
- { GrowSpotName(pb, &(yyvsp[(5) - (5)])); if ((yyvsp[(5) - (5)]).tokenLength == 1 && TokenTextPtr(pb, &(yyvsp[(5) - (5)]))[0] == 'Y') pb->record->seq.lowQuality = true; }
+ { SetSpotGroup(pb, &(yyvsp[(3) - (3)])); }
break;
case 78:
- { GrowSpotName(pb, &(yyvsp[(7) - (7)])); }
+ { StopSpotName(pb); }
break;
case 79:
- { GrowSpotName(pb, &(yyvsp[(9) - (9)])); }
+ { /* in PACBIO fastq, the first '/' and the following digits are treated as a continuation of the spot name, not a read number */
+ if (IS_PACBIO(pb)) pb->spotNameDone = false;
+ GrowSpotName(pb, &(yyvsp[(1) - (1)]));
+ }
break;
case 80:
- { GrowSpotName(pb, &(yyvsp[(11) - (11)])); }
+ {
+ if (!IS_PACBIO(pb)) SetReadNumber(pb, &(yyvsp[(3) - (3)]));
+ GrowSpotName(pb, &(yyvsp[(3) - (3)]));
+ StopSpotName(pb);
+ }
+ break;
+
+ case 81:
+
+ {
+ if (IS_PACBIO(pb)) pb->spotNameDone = false;
+ GrowSpotName(pb, &(yyvsp[(2) - (2)]));
+ }
break;
case 82:
- { SetSpotGroup(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ {
+ if (IS_PACBIO(pb)) StopSpotName(pb);
+ }
break;
case 83:
- { SetSpotGroup(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ { SetReadNumber(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); StopSpotName(pb); }
+ break;
+
+ case 84:
+
+ { SetReadNumber(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); StopSpotName(pb); }
break;
case 85:
- { GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ { GrowSpotName(pb, &(yyvsp[(3) - (3)])); }
break;
case 86:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { GrowSpotName(pb, &(yyvsp[(5) - (5)])); if ((yyvsp[(5) - (5)]).tokenLength == 1 && TokenTextPtr(pb, &(yyvsp[(5) - (5)]))[0] == 'Y') pb->record->seq.lowQuality = true; }
break;
case 87:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { GrowSpotName(pb, &(yyvsp[(7) - (7)])); }
break;
case 88:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { GrowSpotName(pb, &(yyvsp[(9) - (9)])); }
+ break;
+
+ case 90:
+
+ { GrowSpotName(pb, &(yyvsp[(1) - (1)])); FASTQScan_inline_sequence(pb); }
break;
- case 89:
+ case 93:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { SetSpotGroup(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
break;
- case 90:
+ case 94:
- { GrowSpotName(pb, &(yyvsp[(2) - (2)])); }
+ { SetSpotGroup(pb, &(yyvsp[(1) - (1)])); GrowSpotName(pb, &(yyvsp[(1) - (1)])); }
+ break;
+
+ case 96:
+
+ { GrowSpotName(pb, &(yyvsp[(1) - (3)])); StopSpotName(pb); SetReadNumber(pb, &(yyvsp[(3) - (3)])); }
break;
- case 91:
+ case 97:
- { SetReadNumber(pb, &(yyvsp[(3) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)])); StopSpotName(pb); }
+ { GrowSpotName(pb, &(yyvsp[(1) - (3)])); StopSpotName(pb); SetReadNumber(pb, &(yyvsp[(3) - (3)])); }
break;
- case 92:
+ case 98:
- { SetReadNumber(pb, &(yyvsp[(3) - (3)])); GrowSpotName(pb, &(yyvsp[(3) - (3)])); StopSpotName(pb); }
+ { GrowSpotName(pb, &(yyvsp[(1) - (1)])); StopSpotName(pb); }
break;
- case 102:
+ case 105:
{ AddQuality(pb, & (yyvsp[(1) - (1)])); }
break;
@@ -2086,32 +2118,83 @@ yyreturn:
+/* values used in validating quality lines */
+#define MIN_PHRED_33 33
+#define MAX_PHRED_33 126
+#define MIN_PHRED_64 64
+#define MAX_PHRED_64 127
+#define MIN_LOGODDS 59
+#define MAX_LOGODDS 126
+
void AddQuality(FASTQParseBlock* pb, const FASTQToken* token)
{
- if (pb->phredOffset != 0)
+ uint8_t floor;
+ uint8_t ceiling;
+ const char* format;
+ switch ( pb->qualityFormat )
{
- uint8_t floor = pb->phredOffset == 33 ? MIN_PHRED_33 : MIN_PHRED_64;
- uint8_t ceiling = pb->maxPhred == 0 ? (pb->phredOffset == 33 ? MAX_PHRED_33 : MAX_PHRED_64) : pb->maxPhred;
+ case FASTQphred33:
+ floor = MIN_PHRED_33;
+ ceiling = MAX_PHRED_33;
+ format = "Phred33";
+ pb -> qualityAsciiOffset = 33;
+ break;
+ case FASTQphred64:
+ floor = MIN_PHRED_64;
+ ceiling = MAX_PHRED_64;
+ format = "Phred64";
+ pb -> qualityAsciiOffset = 64;
+ break;
+ case FASTQlogodds:
+ floor = MIN_LOGODDS;
+ ceiling = MAX_LOGODDS;
+ format = "Logodds";
+ pb -> qualityAsciiOffset = 64;
+ break;
+ default:
+ /* TODO:
+ if qualityAsciiOffset is 0,
+ guess based on the raw values on the first quality line:
+ if all values are above MAX_PHRED_33, qualityAsciiOffset = 64
+ if all values are in MIN_PHRED_33..MAX_PHRED_33, qualityAsciiOffset = 33
+ if any value is below MIN_PHRED_33, abort
+ if the guess is 33 and proven wrong (a raw quality value >MAX_PHRED_33 is encountered and no values below MIN_PHRED_64 ever seen)
+ reopen the file,
+ qualityAsciiOffset = 64
+ try to parse again
+ if a value below MIN_PHRED_64 seen, abort
+ */
+ {
+ char buf[200];
+ sprintf ( buf, "Invalid quality format: %d.", pb->qualityFormat );
+ pb->fatalError = true;
+ yyerror(pb, buf);
+ return;
+ }
+ }
+
+ { /* make sure all qualities fall into the required range */
unsigned int i;
for (i=0; i < token->tokenLength; ++i)
{
- char buf[200];
- char ch = TokenTextPtr(pb, token)[i];
+ char ch = TokenTextPtr(pb, token)[i];
if (ch < floor || ch > ceiling)
{
- sprintf(buf, "Invalid quality value ('%c'=%d, position %d): for %s, valid range is from %d to %d.",
- ch,
- ch,
- i,
- pb->phredOffset == 33 ? "Phred33": "Phred64",
- floor,
- ceiling);
+ char buf[200];
+ sprintf ( buf, "Invalid quality value ('%c'=%d, position %d): for %s, valid range is from %d to %d.",
+ ch,
+ ch,
+ i,
+ format,
+ floor,
+ ceiling);
pb->fatalError = true;
yyerror(pb, buf);
return;
}
}
}
+
if (pb->qualityLength == 0)
{
pb->qualityOffset = token->tokenStart;
@@ -2173,11 +2256,33 @@ void SetReadNumber(FASTQParseBlock* pb, const FASTQToken* token)
void StartSpotName(FASTQParseBlock* pb, size_t offset)
{
pb->spotNameOffset = offset;
+ pb->spotNameLength = 0;
}
+
+void SaveSpotName(FASTQParseBlock* pb)
+{
+ pb->spotNameOffset_saved = pb->spotNameOffset;
+ pb->spotNameLength_saved = pb->spotNameLength;
+}
+
+void RestartSpotName(FASTQParseBlock* pb)
+{
+ pb->spotNameOffset += pb->spotNameLength;
+ pb->spotNameLength = 0;
+}
+
+void RevertSpotName(FASTQParseBlock* pb)
+{
+ pb->spotNameOffset = pb->spotNameOffset_saved;
+ pb->spotNameLength = pb->spotNameLength_saved;
+}
+
void GrowSpotName(FASTQParseBlock* pb, const FASTQToken* token)
{
if (!pb->spotNameDone)
+ {
pb->spotNameLength += token->tokenLength;
+ }
}
void StopSpotName(FASTQParseBlock* pb)
@@ -2187,10 +2292,13 @@ void StopSpotName(FASTQParseBlock* pb)
void SetSpotGroup(FASTQParseBlock* pb, const FASTQToken* token)
{
- if (token->tokenLength != 1 || TokenTextPtr(pb, token)[0] != '0') /* ignore spot group 0 */
+ if ( ! pb->ignoreSpotGroups )
{
- pb->spotGroupOffset = token->tokenStart;
- pb->spotGroupLength = token->tokenLength;
+ if (token->tokenLength != 1 || TokenTextPtr(pb, token)[0] != '0') /* ignore spot group 0 */
+ {
+ pb->spotGroupOffset = token->tokenStart;
+ pb->spotGroupLength = token->tokenLength;
+ }
}
}
diff --git a/tools/fastq-loader/fastq-grammar.y b/tools/fastq-loader/fastq-grammar.y
index 25559f9..509e187 100644
--- a/tools/fastq-loader/fastq-grammar.y
+++ b/tools/fastq-loader/fastq-grammar.y
@@ -40,11 +40,15 @@
static void AddQuality(FASTQParseBlock* pb, const FASTQToken* token);
static void SetReadNumber(FASTQParseBlock* pb, const FASTQToken* token);
+ static void SetSpotGroup(FASTQParseBlock* pb, const FASTQToken* token);
+ static void SetRead(FASTQParseBlock* pb, const FASTQToken* token);
+
static void StartSpotName(FASTQParseBlock* pb, size_t offset);
static void GrowSpotName(FASTQParseBlock* pb, const FASTQToken* token);
static void StopSpotName(FASTQParseBlock* pb);
- static void SetSpotGroup(FASTQParseBlock* pb, const FASTQToken* token);
- static void SetRead(FASTQParseBlock* pb, const FASTQToken* token);
+ static void RestartSpotName(FASTQParseBlock* pb);
+ static void SaveSpotName(FASTQParseBlock* pb);
+ static void RevertSpotName(FASTQParseBlock* pb);
#define UNLEX do { if (yychar != YYEMPTY && yychar != YYEOF) FASTQ_unlex(pb, & yylval); } while (0)
@@ -78,10 +82,10 @@ sequence /* have to return the lookahead symbol before returning since it belong
| readLines { UNLEX; return 1; }
- | qualityLines { UNLEX; return 1; }
+/* | qualityLines { UNLEX; return 1; } */
- | name { StartSpotName(pb, 0); }
- fqCOORDS { GrowSpotName(pb, &$3); StopSpotName(pb); }
+ | name
+ fqCOORDS { GrowSpotName(pb, &$2); StopSpotName(pb); }
':' { FASTQScan_inline_sequence(pb); }
inlineRead
':' { FASTQScan_inline_quality(pb); }
@@ -105,10 +109,12 @@ readLines
: header endline read
| header endline error endline
| error endline read
+ | error endline
;
header
: '@' { StartSpotName(pb, 1); } tagLine
+ | '@' fqWS { StartSpotName(pb, 1 + $2.tokenLength); } tagLine
| '>' { StartSpotName(pb, 1); } tagLine
;
@@ -139,30 +145,53 @@ inlineRead
/*************** tag line rules *****************/
tagLine
: nameSpotGroup
- | nameSpotGroup readNumberOrTail
- | runSpotRead
+ | nameSpotGroup readNumber
+
+ | nameSpotGroup readNumber fqWS fqNUMBER ':' fqALPHANUM ':' fqNUMBER indexSequence
+
+ | nameSpotGroup readNumber fqWS fqALPHANUM { FASTQScan_skip_to_eol(pb); }
+ | nameSpotGroup readNumber fqWS { FASTQScan_skip_to_eol(pb); }
+
+ | nameSpotGroup fqWS { GrowSpotName(pb, &$1); StopSpotName(pb); } casava1_8 { FASTQScan_skip_to_eol(pb); }
+ | nameSpotGroup fqWS { GrowSpotName(pb, &$1); StopSpotName(pb); } fqALPHANUM { FASTQScan_skip_to_eol(pb); } /* no recognizable read number */
+ | runSpotRead { FASTQScan_skip_to_eol(pb); }
+ | name readNumber
+ | name readNumber fqWS { FASTQScan_skip_to_eol(pb); }
+ | name
;
nameSpotGroup
- : name { StopSpotName(pb); }
- spotGroup
- | name fqCOORDS { GrowSpotName(pb, &$2); StopSpotName(pb); }
- spotGroup
+ : nameWithCoords
+ | nameWithCoords spotGroup
+ | name { StopSpotName(pb); } spotGroup
+ | nameWS nameWithCoords /* nameWS ignored */
+ | nameWS nameWithCoords spotGroup /* nameWS ignored */
+ | nameWS fqALPHANUM '=' { RevertSpotName(pb); FASTQScan_skip_to_eol(pb); }
+ ;
+
+nameWS
+ : name fqWS
+ { /* 'name' without coordinates attached will be ignored if followed by a name with coordinates (see the previous production).
+ however, if not followed, this will be the spot name, so we need to save the 'name's coordinates in case
+ we need to revert to them later (see call to RevertSpotName() above) */
+ SaveSpotName(pb);
+ GrowSpotName(pb, &$2); /* need to account for white space but it is not part of the spot name */
+ RestartSpotName(pb); /* clean up for the potential nameWithCoords to start here */
+ }
+
+nameWithCoords
+ : name fqCOORDS { GrowSpotName(pb, &$2); StopSpotName(pb); }
| name fqCOORDS '_'
- { /* another crazy variation by Illumina, this time "_" is used as " /" */
+ { /* another variation by Illumina, this time "_" is used as " /" */
GrowSpotName(pb, &$2);
StopSpotName(pb);
GrowSpotName(pb, &$3);
}
casava1_8
- | name fqCOORDS ':' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3); StopSpotName(pb); }
- spotGroup
- | name fqCOORDS ':' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3);}
- name
- | name fqCOORDS '.' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3);}
- name
- | name fqCOORDS ':' '.' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3); GrowSpotName(pb, &$3);}
- name
+ | name fqCOORDS ':' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3);} name
+ | name fqCOORDS '.' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3);} name
+ | name fqCOORDS ':' '.' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3); GrowSpotName(pb, &$4);} name
+ | name fqCOORDS ':' { GrowSpotName(pb, &$2); GrowSpotName(pb, &$3); StopSpotName(pb); }
;
name
@@ -177,22 +206,13 @@ name
;
spotGroup
- : '#' { GrowSpotName(pb, &$1); }
- fqNUMBER { SetSpotGroup(pb, &$3); GrowSpotName(pb, &$3); }
- | '#' { GrowSpotName(pb, &$1); }
- fqALPHANUM { SetSpotGroup(pb, &$3); GrowSpotName(pb, &$3); }
- |
+ : '#' { StopSpotName(pb); }
+ fqNUMBER { SetSpotGroup(pb, &$3); }
+ | '#' { StopSpotName(pb); }
+ fqALPHANUM { SetSpotGroup(pb, &$3); }
+ | '#' { StopSpotName(pb); }
;
-readNumberOrTail
- : readNumber
- | fqWS { GrowSpotName(pb, &$1); }
- casava1_8
- | fqWS { GrowSpotName(pb, &$1); }
- tail
- | readNumberOrTail fqWS { GrowSpotName(pb, &$2); } tail
- ;
-
readNumber
: '/'
{ /* in PACBIO fastq, the first '/' and the following digits are treated as a continuation of the spot name, not a read number */
@@ -216,38 +236,32 @@ readNumber
if (IS_PACBIO(pb)) StopSpotName(pb);
}
;
-
+
casava1_8
: fqNUMBER { SetReadNumber(pb, &$1); GrowSpotName(pb, &$1); StopSpotName(pb); }
+ | fqNUMBER { SetReadNumber(pb, &$1); GrowSpotName(pb, &$1); StopSpotName(pb); }
':' { GrowSpotName(pb, &$3); }
fqALPHANUM { GrowSpotName(pb, &$5); if ($5.tokenLength == 1 && TokenTextPtr(pb, &$5)[0] == 'Y') pb->record->seq.lowQuality = true; }
':' { GrowSpotName(pb, &$7); }
fqNUMBER { GrowSpotName(pb, &$9); }
- ':' { GrowSpotName(pb, &$11); }
indexSequence
;
-
+
indexSequence
- : fqALPHANUM { SetSpotGroup(pb, &$1); GrowSpotName(pb, &$1); }
- | fqNUMBER { SetSpotGroup(pb, &$1); GrowSpotName(pb, &$1); }
+ : ':' { GrowSpotName(pb, &$1); FASTQScan_inline_sequence(pb); } index
|
;
-tail
- : fqALPHANUM { GrowSpotName(pb, &$1); }
- | tail fqNUMBER { GrowSpotName(pb, &$2); }
- | tail fqALPHANUM { GrowSpotName(pb, &$2); }
- | tail '_' { GrowSpotName(pb, &$2); }
- | tail '/' { GrowSpotName(pb, &$2); }
- | tail '=' { GrowSpotName(pb, &$2); }
+index
+ : fqBASESEQ { SetSpotGroup(pb, &$1); GrowSpotName(pb, &$1); }
+ | fqNUMBER { SetSpotGroup(pb, &$1); GrowSpotName(pb, &$1); }
+ |
;
-
+
runSpotRead
- : fqRUNDOTSPOT '.' fqNUMBER { SetReadNumber(pb, &$3); GrowSpotName(pb, &$3); StopSpotName(pb); }
- | fqRUNDOTSPOT '/' fqNUMBER { SetReadNumber(pb, &$3); GrowSpotName(pb, &$3); StopSpotName(pb); }
- | runSpotRead fqWS tail
- | runSpotRead fqWS fqNUMBER
- | runSpotRead fqWS fqNUMBER tail
+ : fqRUNDOTSPOT '.' fqNUMBER { GrowSpotName(pb, &$1); StopSpotName(pb); SetReadNumber(pb, &$3); }
+ | fqRUNDOTSPOT '/' fqNUMBER { GrowSpotName(pb, &$1); StopSpotName(pb); SetReadNumber(pb, &$3); }
+ | fqRUNDOTSPOT { GrowSpotName(pb, &$1); StopSpotName(pb); }
;
/*************** quality rules *****************/
@@ -272,32 +286,83 @@ qualityLine
%%
+/* values used in validating quality lines */
+#define MIN_PHRED_33 33
+#define MAX_PHRED_33 126
+#define MIN_PHRED_64 64
+#define MAX_PHRED_64 127
+#define MIN_LOGODDS 59
+#define MAX_LOGODDS 126
+
void AddQuality(FASTQParseBlock* pb, const FASTQToken* token)
{
- if (pb->phredOffset != 0)
+ uint8_t floor;
+ uint8_t ceiling;
+ const char* format;
+ switch ( pb->qualityFormat )
{
- uint8_t floor = pb->phredOffset == 33 ? MIN_PHRED_33 : MIN_PHRED_64;
- uint8_t ceiling = pb->maxPhred == 0 ? (pb->phredOffset == 33 ? MAX_PHRED_33 : MAX_PHRED_64) : pb->maxPhred;
+ case FASTQphred33:
+ floor = MIN_PHRED_33;
+ ceiling = MAX_PHRED_33;
+ format = "Phred33";
+ pb -> qualityAsciiOffset = 33;
+ break;
+ case FASTQphred64:
+ floor = MIN_PHRED_64;
+ ceiling = MAX_PHRED_64;
+ format = "Phred64";
+ pb -> qualityAsciiOffset = 64;
+ break;
+ case FASTQlogodds:
+ floor = MIN_LOGODDS;
+ ceiling = MAX_LOGODDS;
+ format = "Logodds";
+ pb -> qualityAsciiOffset = 64;
+ break;
+ default:
+ /* TODO:
+ if qualityAsciiOffset is 0,
+ guess based on the raw values on the first quality line:
+ if all values are above MAX_PHRED_33, qualityAsciiOffset = 64
+ if all values are in MIN_PHRED_33..MAX_PHRED_33, qualityAsciiOffset = 33
+ if any value is below MIN_PHRED_33, abort
+ if the guess is 33 and proven wrong (a raw quality value >MAX_PHRED_33 is encountered and no values below MIN_PHRED_64 ever seen)
+ reopen the file,
+ qualityAsciiOffset = 64
+ try to parse again
+ if a value below MIN_PHRED_64 seen, abort
+ */
+ {
+ char buf[200];
+ sprintf ( buf, "Invalid quality format: %d.", pb->qualityFormat );
+ pb->fatalError = true;
+ yyerror(pb, buf);
+ return;
+ }
+ }
+
+ { /* make sure all qualities fall into the required range */
unsigned int i;
for (i=0; i < token->tokenLength; ++i)
{
- char buf[200];
- char ch = TokenTextPtr(pb, token)[i];
+ char ch = TokenTextPtr(pb, token)[i];
if (ch < floor || ch > ceiling)
{
- sprintf(buf, "Invalid quality value ('%c'=%d, position %d): for %s, valid range is from %d to %d.",
- ch,
- ch,
- i,
- pb->phredOffset == 33 ? "Phred33": "Phred64",
- floor,
- ceiling);
+ char buf[200];
+ sprintf ( buf, "Invalid quality value ('%c'=%d, position %d): for %s, valid range is from %d to %d.",
+ ch,
+ ch,
+ i,
+ format,
+ floor,
+ ceiling);
pb->fatalError = true;
yyerror(pb, buf);
return;
}
}
}
+
if (pb->qualityLength == 0)
{
pb->qualityOffset = token->tokenStart;
@@ -359,11 +424,33 @@ void SetReadNumber(FASTQParseBlock* pb, const FASTQToken* token)
void StartSpotName(FASTQParseBlock* pb, size_t offset)
{
pb->spotNameOffset = offset;
+ pb->spotNameLength = 0;
+}
+
+void SaveSpotName(FASTQParseBlock* pb)
+{
+ pb->spotNameOffset_saved = pb->spotNameOffset;
+ pb->spotNameLength_saved = pb->spotNameLength;
+}
+
+void RestartSpotName(FASTQParseBlock* pb)
+{
+ pb->spotNameOffset += pb->spotNameLength;
+ pb->spotNameLength = 0;
+}
+
+void RevertSpotName(FASTQParseBlock* pb)
+{
+ pb->spotNameOffset = pb->spotNameOffset_saved;
+ pb->spotNameLength = pb->spotNameLength_saved;
}
+
void GrowSpotName(FASTQParseBlock* pb, const FASTQToken* token)
{
if (!pb->spotNameDone)
+ {
pb->spotNameLength += token->tokenLength;
+ }
}
void StopSpotName(FASTQParseBlock* pb)
@@ -373,10 +460,13 @@ void StopSpotName(FASTQParseBlock* pb)
void SetSpotGroup(FASTQParseBlock* pb, const FASTQToken* token)
{
- if (token->tokenLength != 1 || TokenTextPtr(pb, token)[0] != '0') /* ignore spot group 0 */
+ if ( ! pb->ignoreSpotGroups )
{
- pb->spotGroupOffset = token->tokenStart;
- pb->spotGroupLength = token->tokenLength;
+ if (token->tokenLength != 1 || TokenTextPtr(pb, token)[0] != '0') /* ignore spot group 0 */
+ {
+ pb->spotGroupOffset = token->tokenStart;
+ pb->spotGroupLength = token->tokenLength;
+ }
}
}
diff --git a/tools/fastq-loader/fastq-lex.c b/tools/fastq-loader/fastq-lex.c
index d157484..d3dabdc 100644
--- a/tools/fastq-loader/fastq-lex.c
+++ b/tools/fastq-loader/fastq-lex.c
@@ -5,6 +5,14 @@
/* A lexical scanner generated by flex */
+/* %not-for-header */
+
+/* %if-c-only */
+/* %if-not-reentrant */
+/* %endif */
+/* %endif */
+/* %ok-for-header */
+
#define FLEX_SCANNER
#define YY_FLEX_MAJOR_VERSION 2
#define YY_FLEX_MINOR_VERSION 5
@@ -13,16 +21,32 @@
#define FLEX_BETA
#endif
+/* %if-c++-only */
+/* %endif */
+
+/* %if-c-only */
+
+/* %endif */
+
+/* %if-c-only */
+
+/* %endif */
+
/* First, we deal with platform-specific or compiler-specific issues. */
/* begin standard C headers. */
+/* %if-c-only */
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <stdlib.h>
+/* %endif */
+/* %if-tables-serialization */
+/* %endif */
/* end standard C headers. */
+/* %if-c-or-c++ */
/* flex integer type definitions */
#ifndef FLEXINT_H
@@ -86,6 +110,11 @@ typedef unsigned int flex_uint32_t;
#endif /* ! FLEXINT_H */
+/* %endif */
+
+/* %if-c++-only */
+/* %endif */
+
#ifdef __cplusplus
/* The "const" storage-class-modifier is valid. */
@@ -107,8 +136,13 @@ typedef unsigned int flex_uint32_t;
#define yyconst
#endif
+/* %not-for-header */
+
/* Returned upon end-of-file. */
#define YY_NULL 0
+/* %ok-for-header */
+
+/* %not-for-header */
/* Promotes a possibly negative, possibly signed char to an unsigned
* integer for use as an array index. If the signed char is negative,
@@ -116,6 +150,9 @@ typedef unsigned int flex_uint32_t;
* double cast.
*/
#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c)
+/* %ok-for-header */
+
+/* %if-reentrant */
/* An opaque pointer. */
#ifndef YY_TYPEDEF_YY_SCANNER_T
@@ -134,6 +171,11 @@ typedef void* yyscan_t;
#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
#define yy_flex_debug yyg->yy_flex_debug_r
+/* %endif */
+
+/* %if-not-reentrant */
+/* %endif */
+
/* Enter a start condition. This macro really ought to take a parameter,
* but we do it the disgusting crufty way forced on us by the ()-less
* definition of BEGIN.
@@ -169,6 +211,14 @@ typedef void* yyscan_t;
typedef struct yy_buffer_state *YY_BUFFER_STATE;
#endif
+/* %if-not-reentrant */
+/* %endif */
+
+/* %if-c-only */
+/* %if-not-reentrant */
+/* %endif */
+/* %endif */
+
#define EOB_ACT_CONTINUE_SCAN 0
#define EOB_ACT_END_OF_FILE 1
#define EOB_ACT_LAST_MATCH 2
@@ -213,7 +263,12 @@ typedef size_t yy_size_t;
#define YY_STRUCT_YY_BUFFER_STATE
struct yy_buffer_state
{
+/* %if-c-only */
FILE *yy_input_file;
+/* %endif */
+
+/* %if-c++-only */
+/* %endif */
char *yy_ch_buf; /* input buffer */
char *yy_buf_pos; /* current position in input buffer */
@@ -274,6 +329,15 @@ struct yy_buffer_state
};
#endif /* !YY_STRUCT_YY_BUFFER_STATE */
+/* %if-c-only Standard (non-C++) definition */
+/* %not-for-header */
+
+/* %if-not-reentrant */
+/* %endif */
+/* %ok-for-header */
+
+/* %endif */
+
/* We provide macros for accessing buffer states in case in the
* future we want to put the buffer states in a more general
* "scanner state".
@@ -289,6 +353,15 @@ struct yy_buffer_state
*/
#define YY_CURRENT_BUFFER_LVALUE yyg->yy_buffer_stack[yyg->yy_buffer_stack_top]
+/* %if-c-only Standard (non-C++) definition */
+
+/* %if-not-reentrant */
+/* %not-for-header */
+
+/* %ok-for-header */
+
+/* %endif */
+
void FASTQ_restart (FILE *input_file ,yyscan_t yyscanner );
void FASTQ__switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
YY_BUFFER_STATE FASTQ__create_buffer (FILE *file,int size ,yyscan_t yyscanner );
@@ -307,6 +380,8 @@ YY_BUFFER_STATE FASTQ__scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanne
YY_BUFFER_STATE FASTQ__scan_string (yyconst char *yy_str ,yyscan_t yyscanner );
YY_BUFFER_STATE FASTQ__scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
+/* %endif */
+
void *FASTQ_alloc (yy_size_t ,yyscan_t yyscanner );
void *FASTQ_realloc (void *,yy_size_t ,yyscan_t yyscanner );
void FASTQ_free (void * ,yyscan_t yyscanner );
@@ -335,34 +410,44 @@ void FASTQ_free (void * ,yyscan_t yyscanner );
#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol)
+/* %% [1.0] yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here */
/* Begin user sect3 */
#define FASTQ_wrap(n) 1
#define YY_SKIP_YYWRAP
+#define FLEX_DEBUG
+
typedef unsigned char YY_CHAR;
typedef int yy_state_type;
#define yytext_ptr yytext_r
+/* %if-c-only Standard (non-C++) definition */
+
static yy_state_type yy_get_previous_state (yyscan_t yyscanner );
static yy_state_type yy_try_NUL_trans (yy_state_type current_state ,yyscan_t yyscanner);
static int yy_get_next_buffer (yyscan_t yyscanner );
static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
+/* %endif */
+
/* Done after the current pattern has been matched and before the
* corresponding action - sets up yytext.
*/
#define YY_DO_BEFORE_ACTION \
yyg->yytext_ptr = yy_bp; \
+/* %% [2.0] code to fiddle yytext and yyleng for yymore() goes here \ */\
yyleng = (size_t) (yy_cp - yy_bp); \
yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \
+/* %% [3.0] code to copy yytext_ptr to yytext[] goes here, if %array \ */\
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 29
-#define YY_END_OF_BUFFER 30
+/* %% [4.0] data tables for the DFA and the user's section 1 definitions go here */
+#define YY_NUM_RULES 33
+#define YY_END_OF_BUFFER 34
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
@@ -370,46 +455,50 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static yyconst flex_int16_t yy_acclist[185] =
+static yyconst flex_int16_t yy_acclist[207] =
{ 0,
- 30, 11, 29, 8, 11, 29, 28, 29, 11, 28,
- 29, 7, 11, 29, 6, 7, 11, 29, 11, 29,
- 7, 11, 29, 3, 11, 29, 11, 29,16393, 2,
- 11, 29, 1, 11, 29, 7, 11, 29,16393, 7,
- 11, 29,16393, 8, 11, 29, 12, 29, 11, 12,
- 29, 29, 29, 16, 29, 29,16399, 29,16397,16399,
- 29,16397,16399, 29, 20, 29, 20, 29, 24, 29,
- 21, 24, 29, 23, 29, 23, 24, 29, 24, 29,
- 16406, 21, 24, 29,16406, 17, 29, 17, 29, 27,
- 29, 21, 27, 29, 26, 29, 26, 27, 29, 27,
-
- 29,16409, 21, 27, 29,16409, 8, 28, 7, 6,
- 7, 7, 8201, 8201,16393,16393,16394, 7,16394, 7,
- 16393, 8, 12, 12, 16, 8207, 16, 8207,16399, 8205,
- 8207, 16, 8205, 8207,16397,16399,16397,16398,16399,16398,
- 16399, 19, 19, 20, 21, 23, 8214, 8214,16406, 21,
- 16406, 17, 17, 18, 18, 26, 8217, 8217,16409, 21,
- 16409, 7, 8201, 8202, 8201, 8202,16394, 8202, 8202, 8205,
- 8206, 8207, 16, 8205, 8206, 8207, 8206, 8207, 16, 8206,
- 8207, 7, 5, 4
+ 31, 31, 34, 11, 33, 8, 11, 33, 32, 33,
+ 11, 32, 33, 7, 11, 33, 6, 7, 11, 33,
+ 11, 33, 7, 11, 33, 3, 11, 33, 11, 33,
+ 16393, 2, 11, 33, 1, 11, 33, 7, 11, 33,
+ 16393, 7, 11, 33,16393, 8, 11, 33, 12, 33,
+ 11, 12, 33, 33, 33, 16, 33, 33,16399, 33,
+ 16397,16399, 33,16397,16399, 33, 22, 33, 22, 33,
+ 26, 33, 23, 26, 33, 25, 33, 25, 26, 33,
+ 26, 33,16408, 23, 26, 33,16408, 20, 33, 20,
+ 33, 17, 33, 19, 33, 17, 33, 29, 33, 23,
+
+ 29, 33, 28, 33, 28, 29, 33, 29, 33,16411,
+ 23, 29, 33,16411, 31, 33, 30, 33, 30, 31,
+ 33, 8, 32, 7, 6, 7, 7, 8201, 8201,16393,
+ 16393,16394, 7,16394, 7,16393, 8, 12, 12, 16,
+ 8207, 16, 8207,16399, 8205, 8207, 16, 8205, 8207,16397,
+ 16399,16397,16398,16399,16398,16399, 21, 21, 22, 23,
+ 25, 8216, 8216,16408, 23,16408, 20, 17, 19, 17,
+ 18, 18, 28, 8219, 8219,16411, 23,16411, 31, 30,
+ 30, 31, 30, 7, 8201, 8202, 8201, 8202,16394, 8202,
+ 8202, 8205, 8206, 8207, 16, 8205, 8206, 8207, 8206, 8207,
+
+ 16, 8206, 8207, 7, 5, 4
} ;
-static yyconst flex_int16_t yy_accept[127] =
+static yyconst flex_int16_t yy_accept[141] =
{ 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 2, 4, 7, 9, 12, 15, 19,
- 21, 24, 27, 30, 33, 36, 40, 44, 47, 49,
- 52, 53, 54, 56, 58, 61, 64, 65, 67, 69,
- 71, 74, 76, 79, 82, 86, 88, 90, 92, 95,
- 97, 100, 103, 107, 108, 109, 110, 112, 112, 113,
- 114, 115, 116, 118, 120, 122, 123, 124, 125, 125,
- 126, 128, 129, 130, 133, 135, 137, 140, 142, 142,
- 143, 145, 146, 147, 148, 149, 150, 152, 153, 155,
-
- 156, 157, 158, 159, 160, 162, 162, 163, 165, 167,
- 168, 169, 170, 174, 177, 180, 182, 182, 183, 183,
- 183, 183, 184, 184, 185, 185
+ 1, 1, 1, 2, 3, 4, 6, 9, 11, 14,
+ 17, 21, 23, 26, 29, 32, 35, 38, 42, 46,
+ 49, 51, 54, 55, 56, 58, 60, 63, 66, 67,
+ 69, 71, 73, 76, 78, 81, 84, 88, 90, 92,
+ 94, 96, 98, 100, 103, 105, 108, 111, 115, 117,
+ 119, 122, 123, 124, 125, 127, 127, 128, 129, 130,
+ 131, 133, 135, 137, 138, 139, 140, 140, 141, 143,
+ 144, 145, 148, 150, 152, 155, 157, 157, 158, 160,
+
+ 161, 162, 163, 164, 165, 167, 168, 169, 170, 172,
+ 173, 174, 175, 176, 177, 179, 180, 181, 183, 184,
+ 184, 185, 187, 189, 190, 191, 192, 196, 199, 202,
+ 204, 204, 205, 205, 205, 205, 206, 206, 207, 207
} ;
static yyconst flex_int32_t yy_ec[256] =
@@ -427,7 +516,7 @@ static yyconst flex_int32_t yy_ec[256] =
8, 8, 15, 8, 8, 8, 8, 8, 8, 17,
8, 8, 8, 8, 8, 15, 8, 8, 8, 8,
- 8, 8, 5, 5, 5, 5, 1, 1, 1, 1,
+ 8, 8, 5, 5, 5, 5, 5, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -450,133 +539,147 @@ static yyconst flex_int32_t yy_meta[19] =
6, 7, 3, 3, 8, 4, 8, 4
} ;
-static yyconst flex_int16_t yy_base[143] =
+static yyconst flex_int16_t yy_base[159] =
{ 0,
0, 12, 28, 26, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 43, 19, 33, 59, 65, 63, 72,
- 89, 95, 194, 313, 18, 313, 190, 0, 28, 38,
- 174, 313, 47, 313, 313, 99, 44, 116, 313, 188,
- 313, 183, 313, 122, 126, 46, 182, 313, 181, 313,
- 68, 313, 180, 72, 142, 0, 73, 313, 71, 313,
- 164, 81, 147, 98, 313, 0, 0, 140, 129, 313,
- 139, 102, 151, 166, 0, 0, 313, 128, 118, 313,
- 313, 110, 176, 313, 91, 0, 178, 186, 85, 313,
- 313, 130, 313, 313, 83, 108, 0, 0, 0, 124,
-
- 313, 313, 76, 134, 0, 129, 146, 313, 61, 194,
- 313, 56, 313, 54, 313, 39, 28, 24, 148, 152,
- 16, 154, 161, 163, 313, 204, 212, 220, 228, 236,
- 244, 249, 256, 264, 272, 280, 287, 291, 298, 301,
- 303, 305
+ 0, 0, 0, 43, 19, 33, 59, 65, 69, 0,
+ 86, 92, 35, 45, 312, 335, 18, 335, 308, 0,
+ 40, 43, 259, 335, 52, 335, 335, 96, 65, 113,
+ 335, 273, 335, 271, 335, 119, 123, 67, 270, 335,
+ 267, 335, 79, 335, 265, 79, 139, 335, 164, 0,
+ 91, 99, 335, 127, 335, 162, 127, 144, 131, 335,
+ 133, 147, 335, 0, 0, 147, 126, 335, 136, 148,
+ 151, 166, 0, 0, 335, 125, 115, 335, 335, 107,
+ 176, 335, 100, 0, 178, 186, 94, 335, 335, 156,
+
+ 335, 335, 88, 153, 0, 335, 0, 153, 0, 162,
+ 335, 335, 67, 170, 0, 180, 335, 182, 335, 181,
+ 183, 335, 61, 194, 335, 56, 335, 54, 335, 39,
+ 28, 24, 189, 191, 16, 195, 197, 199, 335, 210,
+ 218, 226, 234, 242, 250, 258, 263, 270, 278, 286,
+ 294, 301, 305, 312, 320, 323, 325, 327
} ;
-static yyconst flex_int16_t yy_def[143] =
+static yyconst flex_int16_t yy_def[159] =
{ 0,
- 125, 1, 1, 3, 126, 126, 126, 126, 126, 126,
- 126, 126, 126, 127, 128, 128, 129, 130, 126, 126,
- 131, 131, 125, 125, 125, 125, 125, 132, 132, 125,
- 132, 125, 133, 125, 125, 125, 36, 125, 125, 125,
- 125, 134, 125, 135, 135, 45, 136, 125, 136, 125,
- 125, 125, 125, 137, 137, 138, 138, 125, 125, 125,
- 125, 139, 139, 125, 125, 132, 29, 140, 132, 125,
- 125, 133, 125, 132, 37, 38, 125, 125, 134, 125,
- 125, 134, 135, 125, 134, 45, 45, 83, 136, 125,
- 125, 125, 125, 125, 125, 137, 55, 138, 57, 125,
-
- 125, 125, 125, 139, 63, 125, 132, 125, 125, 125,
- 125, 125, 125, 134, 125, 134, 141, 107, 125, 125,
- 142, 125, 125, 125, 0, 125, 125, 125, 125, 125,
- 125, 125, 125, 125, 125, 125, 125, 125, 125, 125,
- 125, 125
+ 139, 1, 1, 3, 140, 140, 140, 140, 140, 140,
+ 140, 140, 140, 141, 142, 142, 143, 144, 140, 19,
+ 145, 145, 146, 146, 139, 139, 139, 139, 139, 147,
+ 147, 139, 147, 139, 148, 139, 139, 139, 38, 139,
+ 139, 139, 139, 149, 139, 150, 150, 47, 151, 139,
+ 151, 139, 139, 139, 139, 152, 152, 139, 139, 153,
+ 139, 153, 139, 139, 139, 139, 154, 154, 155, 139,
+ 155, 139, 139, 147, 31, 156, 147, 139, 139, 148,
+ 139, 147, 39, 40, 139, 139, 149, 139, 139, 149,
+ 150, 139, 149, 47, 47, 91, 151, 139, 139, 139,
+
+ 139, 139, 139, 152, 57, 139, 153, 139, 62, 139,
+ 139, 139, 139, 154, 68, 155, 139, 155, 139, 139,
+ 147, 139, 139, 139, 139, 139, 139, 149, 139, 149,
+ 157, 121, 139, 139, 158, 139, 139, 139, 0, 139,
+ 139, 139, 139, 139, 139, 139, 139, 139, 139, 139,
+ 139, 139, 139, 139, 139, 139, 139, 139
} ;
-static yyconst flex_int16_t yy_nxt[332] =
+static yyconst flex_int16_t yy_nxt[354] =
{ 0,
- 24, 25, 26, 27, 24, 25, 24, 28, 24, 29,
- 29, 30, 24, 24, 28, 31, 28, 28, 32, 64,
- 33, 48, 49, 64, 34, 35, 36, 123, 37, 38,
- 39, 40, 120, 38, 33, 48, 49, 67, 67, 119,
- 36, 115, 37, 42, 42, 43, 42, 68, 68, 70,
- 71, 45, 72, 66, 87, 88, 113, 46, 111, 45,
- 51, 52, 53, 108, 51, 50, 51, 52, 53, 92,
- 55, 56, 92, 92, 94, 95, 92, 57, 102, 56,
- 56, 99, 100, 102, 103, 94, 57, 90, 56, 58,
- 59, 60, 61, 84, 63, 58, 59, 60, 61, 64,
-
- 63, 70, 71, 64, 70, 71, 66, 73, 74, 66,
- 94, 95, 81, 75, 66, 75, 66, 76, 77, 78,
- 80, 76, 79, 79, 81, 82, 79, 79, 84, 85,
- 77, 92, 100, 100, 86, 92, 102, 103, 117, 117,
- 86, 70, 86, 92, 94, 95, 107, 97, 92, 102,
- 103, 106, 105, 108, 109, 118, 118, 121, 121, 73,
- 110, 122, 122, 122, 122, 72, 101, 72, 111, 112,
- 124, 124, 124, 124, 110, 74, 79, 79, 81, 82,
- 113, 114, 93, 91, 90, 80, 87, 88, 115, 116,
- 77, 69, 65, 125, 88, 88, 111, 112, 125, 125,
-
- 125, 125, 110, 110, 41, 41, 41, 41, 41, 41,
- 41, 41, 44, 44, 44, 44, 44, 44, 44, 44,
- 47, 47, 47, 47, 47, 47, 47, 47, 50, 50,
- 50, 50, 50, 50, 50, 50, 54, 54, 54, 54,
- 54, 54, 54, 54, 62, 62, 62, 62, 62, 62,
- 62, 62, 66, 125, 66, 125, 66, 72, 125, 125,
- 72, 125, 125, 72, 79, 79, 79, 79, 79, 79,
- 79, 79, 83, 83, 83, 83, 83, 83, 83, 83,
- 89, 89, 89, 89, 89, 89, 89, 89, 96, 96,
- 96, 96, 96, 96, 96, 98, 125, 125, 98, 104,
-
- 104, 104, 104, 104, 104, 104, 68, 68, 117, 117,
- 121, 121, 23, 125, 125, 125, 125, 125, 125, 125,
- 125, 125, 125, 125, 125, 125, 125, 125, 125, 125,
- 125
+ 26, 27, 28, 29, 26, 27, 26, 30, 26, 31,
+ 31, 32, 26, 26, 30, 33, 30, 30, 34, 72,
+ 35, 50, 51, 72, 36, 37, 38, 137, 39, 40,
+ 41, 42, 134, 40, 35, 50, 51, 70, 71, 133,
+ 38, 129, 39, 44, 44, 45, 44, 70, 71, 75,
+ 75, 47, 76, 76, 78, 79, 127, 48, 125, 47,
+ 53, 54, 55, 122, 53, 52, 53, 54, 55, 112,
+ 57, 58, 59, 80, 74, 95, 96, 60, 61, 61,
+ 100, 102, 103, 62, 100, 60, 63, 64, 65, 66,
+ 102, 68, 63, 64, 65, 66, 98, 68, 78, 79,
+
+ 108, 108, 92, 74, 81, 82, 74, 109, 110, 89,
+ 83, 74, 83, 74, 84, 85, 86, 88, 84, 87,
+ 87, 89, 90, 87, 87, 92, 93, 85, 100, 112,
+ 113, 94, 100, 117, 118, 119, 118, 94, 78, 94,
+ 100, 102, 103, 121, 105, 100, 112, 113, 72, 115,
+ 78, 79, 72, 122, 123, 102, 103, 100, 120, 81,
+ 124, 100, 108, 108, 111, 80, 106, 80, 125, 126,
+ 110, 110, 112, 113, 124, 82, 87, 87, 89, 90,
+ 127, 128, 117, 118, 119, 118, 95, 96, 129, 130,
+ 131, 131, 132, 132, 96, 96, 125, 126, 135, 135,
+
+ 136, 136, 124, 124, 136, 136, 138, 138, 138, 138,
+ 43, 43, 43, 43, 43, 43, 43, 43, 46, 46,
+ 46, 46, 46, 46, 46, 46, 49, 49, 49, 49,
+ 49, 49, 49, 49, 52, 52, 52, 52, 52, 52,
+ 52, 52, 56, 56, 56, 56, 56, 56, 56, 56,
+ 67, 67, 67, 67, 67, 67, 67, 67, 69, 69,
+ 69, 69, 69, 69, 69, 69, 74, 101, 74, 99,
+ 74, 80, 98, 88, 80, 85, 77, 80, 87, 87,
+ 87, 87, 87, 87, 87, 87, 91, 91, 91, 91,
+ 91, 91, 91, 91, 97, 97, 97, 97, 97, 97,
+
+ 97, 97, 104, 104, 104, 104, 104, 104, 104, 107,
+ 73, 139, 107, 114, 114, 114, 114, 114, 114, 114,
+ 116, 116, 116, 116, 116, 116, 116, 116, 76, 76,
+ 131, 131, 135, 135, 25, 139, 139, 139, 139, 139,
+ 139, 139, 139, 139, 139, 139, 139, 139, 139, 139,
+ 139, 139, 139
} ;
-static yyconst flex_int16_t yy_chk[332] =
+static yyconst flex_int16_t yy_chk[354] =
{ 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 2, 25,
- 2, 15, 15, 25, 2, 2, 2, 121, 2, 3,
- 3, 3, 118, 3, 4, 16, 16, 29, 29, 117,
- 4, 116, 4, 14, 14, 14, 14, 30, 30, 33,
- 33, 14, 37, 37, 46, 46, 114, 14, 112, 14,
- 17, 17, 17, 109, 17, 18, 18, 18, 18, 51,
- 18, 19, 59, 51, 54, 54, 59, 19, 103, 19,
- 20, 57, 57, 62, 62, 95, 20, 89, 20, 21,
- 21, 21, 21, 85, 21, 22, 22, 22, 22, 64,
-
- 22, 36, 36, 64, 72, 72, 36, 36, 36, 36,
- 96, 96, 82, 36, 36, 36, 36, 38, 38, 38,
- 79, 38, 44, 44, 44, 44, 45, 45, 45, 45,
- 78, 92, 100, 100, 45, 92, 104, 104, 106, 106,
- 45, 71, 45, 55, 55, 55, 69, 55, 63, 63,
- 63, 68, 63, 73, 73, 107, 107, 119, 119, 73,
- 73, 120, 120, 122, 122, 73, 61, 73, 74, 74,
- 123, 123, 124, 124, 74, 74, 83, 83, 83, 83,
- 87, 87, 53, 49, 47, 42, 87, 87, 88, 88,
- 40, 31, 27, 23, 88, 88, 110, 110, 0, 0,
-
- 0, 0, 110, 110, 126, 126, 126, 126, 126, 126,
- 126, 126, 127, 127, 127, 127, 127, 127, 127, 127,
- 128, 128, 128, 128, 128, 128, 128, 128, 129, 129,
- 129, 129, 129, 129, 129, 129, 130, 130, 130, 130,
- 130, 130, 130, 130, 131, 131, 131, 131, 131, 131,
- 131, 131, 132, 0, 132, 0, 132, 133, 0, 0,
- 133, 0, 0, 133, 134, 134, 134, 134, 134, 134,
- 134, 134, 135, 135, 135, 135, 135, 135, 135, 135,
- 136, 136, 136, 136, 136, 136, 136, 136, 137, 137,
- 137, 137, 137, 137, 137, 138, 0, 0, 138, 139,
-
- 139, 139, 139, 139, 139, 139, 140, 140, 141, 141,
- 142, 142, 125, 125, 125, 125, 125, 125, 125, 125,
- 125, 125, 125, 125, 125, 125, 125, 125, 125, 125,
- 125
+ 1, 1, 1, 1, 1, 1, 1, 1, 2, 27,
+ 2, 15, 15, 27, 2, 2, 2, 135, 2, 3,
+ 3, 3, 132, 3, 4, 16, 16, 23, 23, 131,
+ 4, 130, 4, 14, 14, 14, 14, 24, 24, 31,
+ 31, 14, 32, 32, 35, 35, 128, 14, 126, 14,
+ 17, 17, 17, 123, 17, 18, 18, 18, 18, 113,
+ 18, 19, 19, 39, 39, 48, 48, 19, 19, 19,
+ 53, 56, 56, 19, 53, 19, 21, 21, 21, 21,
+ 103, 21, 22, 22, 22, 22, 97, 22, 38, 38,
+
+ 61, 61, 93, 38, 38, 38, 38, 62, 62, 90,
+ 38, 38, 38, 38, 40, 40, 40, 87, 40, 46,
+ 46, 46, 46, 47, 47, 47, 47, 86, 64, 67,
+ 67, 47, 64, 69, 69, 71, 71, 47, 79, 47,
+ 57, 57, 57, 77, 57, 68, 68, 68, 72, 68,
+ 80, 80, 72, 81, 81, 104, 104, 100, 76, 81,
+ 81, 100, 108, 108, 66, 81, 59, 81, 82, 82,
+ 110, 110, 114, 114, 82, 82, 91, 91, 91, 91,
+ 95, 95, 116, 116, 118, 118, 95, 95, 96, 96,
+ 120, 120, 121, 121, 96, 96, 124, 124, 133, 133,
+
+ 134, 134, 124, 124, 136, 136, 137, 137, 138, 138,
+ 140, 140, 140, 140, 140, 140, 140, 140, 141, 141,
+ 141, 141, 141, 141, 141, 141, 142, 142, 142, 142,
+ 142, 142, 142, 142, 143, 143, 143, 143, 143, 143,
+ 143, 143, 144, 144, 144, 144, 144, 144, 144, 144,
+ 145, 145, 145, 145, 145, 145, 145, 145, 146, 146,
+ 146, 146, 146, 146, 146, 146, 147, 55, 147, 51,
+ 147, 148, 49, 44, 148, 42, 33, 148, 149, 149,
+ 149, 149, 149, 149, 149, 149, 150, 150, 150, 150,
+ 150, 150, 150, 150, 151, 151, 151, 151, 151, 151,
+
+ 151, 151, 152, 152, 152, 152, 152, 152, 152, 153,
+ 29, 25, 153, 154, 154, 154, 154, 154, 154, 154,
+ 155, 155, 155, 155, 155, 155, 155, 155, 156, 156,
+ 157, 157, 158, 158, 139, 139, 139, 139, 139, 139,
+ 139, 139, 139, 139, 139, 139, 139, 139, 139, 139,
+ 139, 139, 139
} ;
/* Table of booleans, true if rule could match eol. */
-static yyconst flex_int32_t yy_rule_can_match_eol[30] =
+static yyconst flex_int32_t yy_rule_can_match_eol[34] =
{ 0,
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0,
- 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, };
+ 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, };
+
+static yyconst flex_int16_t yy_rule_linenum[33] =
+ { 0,
+ 90, 91, 93, 97, 98, 99, 100, 101, 102, 103,
+ 104, 107, 111, 112, 113, 114, 119, 120, 121, 122,
+ 127, 128, 133, 138, 139, 146, 151, 152, 153, 158,
+ 159, 162
+ } ;
#define YY_TRAILING_MASK 0x2000
#define YY_TRAILING_HEAD_MASK 0x4000
@@ -649,6 +752,16 @@ goto find_rule; \
#define YY_NO_UNISTD_H 1
/*%option debug*/
+
+
+
+
+
+
+
+
+
+
/* there are more: =ACMGRSVTWYHKDBN. plus lowercase */
#define INITIAL 0
@@ -662,17 +775,25 @@ goto find_rule; \
#define IN_QUALITY 8
#define INLINE_SEQUENCE 9
#define INLINE_QUALITY 10
+#define SKIP_TO_EOL 11
#ifndef YY_NO_UNISTD_H
/* Special case for "unistd.h", since it is non-ANSI. We include it way
* down here because we want the user's section 1 to have been scanned first.
* The user has a chance to override it with an option.
*/
+/* %if-c-only */
#include <unistd.h>
+/* %endif */
+/* %if-c++-only */
+/* %endif */
#endif
#define YY_EXTRA_TYPE FASTQParseBlock*
+/* %if-c-only Reentrant structure and macros (non-C++). */
+/* %if-reentrant */
+
/* Holds the entire state of the reentrant scanner. */
struct yyguts_t
{
@@ -720,8 +841,14 @@ struct yyguts_t
}; /* end struct yyguts_t */
+/* %if-c-only */
+
static int yy_init_globals (yyscan_t yyscanner );
+/* %endif */
+
+/* %if-reentrant */
+
/* This must go here because YYSTYPE and YYLTYPE are included
* from bison output in section 1.*/
# define yylval yyg->yylval_r
@@ -730,6 +857,10 @@ int FASTQ_lex_init (yyscan_t* scanner);
int FASTQ_lex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner);
+/* %endif */
+
+/* %endif End reentrant structures and macros. */
+
/* Accessor methods to globals.
These are made visible to non-reentrant scanners for convenience. */
@@ -759,10 +890,14 @@ int FASTQ_get_lineno (yyscan_t yyscanner );
void FASTQ_set_lineno (int line_number ,yyscan_t yyscanner );
+/* %if-bison-bridge */
+
YYSTYPE * FASTQ_get_lval (yyscan_t yyscanner );
void FASTQ_set_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
+/* %endif */
+
/* Macros after this point can all be overridden by user definitions in
* section 1.
*/
@@ -775,8 +910,14 @@ extern int FASTQ_wrap (yyscan_t yyscanner );
#endif
#endif
+/* %not-for-header */
+
static void yyunput (int c,char *buf_ptr ,yyscan_t yyscanner);
+/* %ok-for-header */
+
+/* %endif */
+
#ifndef yytext_ptr
static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner);
#endif
@@ -786,21 +927,29 @@ static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
#endif
#ifndef YY_NO_INPUT
+/* %if-c-only Standard (non-C++) definition */
+/* %not-for-header */
#ifdef __cplusplus
static int yyinput (yyscan_t yyscanner );
#else
static int input (yyscan_t yyscanner );
#endif
+/* %ok-for-header */
+/* %endif */
#endif
+/* %if-c-only */
+
static void yy_push_state (int new_state ,yyscan_t yyscanner);
static void yy_pop_state (yyscan_t yyscanner );
static int yy_top_state (yyscan_t yyscanner );
+/* %endif */
+
/* Amount of stuff to slurp up with each read. */
#ifndef YY_READ_BUF_SIZE
#define YY_READ_BUF_SIZE 8192
@@ -808,10 +957,14 @@ static int input (yyscan_t yyscanner );
/* Copy whatever the last rule matched to the standard output. */
#ifndef ECHO
+/* %if-c-only Standard (non-C++) definition */
/* This used to be an fputs(), but since the string might contain NUL's,
* we now use fwrite().
*/
#define ECHO fwrite( yytext, yyleng, 1, yyout )
+/* %endif */
+/* %if-c++-only C++ definition */
+/* %endif */
#endif
/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL,
@@ -819,6 +972,7 @@ static int input (yyscan_t yyscanner );
*/
#ifndef YY_INPUT
#define YY_INPUT(buf,result,max_size) \
+/* %% [5.0] fread()/read() definition of YY_INPUT goes here unless we're doing C++ \ */\
if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
{ \
int c = '*'; \
@@ -847,6 +1001,8 @@ static int input (yyscan_t yyscanner );
} \
}\
\
+/* %if-c++-only C++ definition \ */\
+/* %endif */
#endif
@@ -865,22 +1021,41 @@ static int input (yyscan_t yyscanner );
/* Report a fatal error. */
#ifndef YY_FATAL_ERROR
+/* %if-c-only */
#define YY_FATAL_ERROR(msg) yy_fatal_error( msg , yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
#endif
+/* %if-tables-serialization structures and prototypes */
+/* %not-for-header */
+
+/* %ok-for-header */
+
+/* %not-for-header */
+
+/* %tables-yydmap generated elements */
+/* %endif */
/* end tables serialization structures and prototypes */
+/* %ok-for-header */
+
/* Default declaration of generated scanner - a define so the user can
* easily add parameters.
*/
#ifndef YY_DECL
#define YY_DECL_IS_OURS 1
+/* %if-c-only Standard (non-C++) definition */
extern int FASTQ_lex \
(YYSTYPE * yylval_param ,yyscan_t yyscanner);
#define YY_DECL int FASTQ_lex \
(YYSTYPE * yylval_param , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only C++ definition */
+/* %endif */
#endif /* !YY_DECL */
/* Code executed at the beginning of each rule, after yytext and yyleng
@@ -895,12 +1070,15 @@ extern int FASTQ_lex \
#define YY_BREAK break;
#endif
+/* %% [6.0] YY_RULE_SETUP definition goes here */
#define YY_RULE_SETUP \
if ( yyleng > 0 ) \
YY_CURRENT_BUFFER_LVALUE->yy_at_bol = \
(yytext[yyleng - 1] == '\n'); \
YY_USER_ACTION
+/* %not-for-header */
+
/** The main scanner function which does all the work.
*/
YY_DECL
@@ -910,6 +1088,10 @@ YY_DECL
register int yy_act;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+/* %% [7.0] user's declarations go here */
+
+
+
yylval = yylval_param;
if ( !yyg->yy_init )
@@ -930,10 +1112,18 @@ YY_DECL
yyg->yy_start = 1; /* first start state */
if ( ! yyin )
+/* %if-c-only */
yyin = stdin;
+/* %endif */
+/* %if-c++-only */
+/* %endif */
if ( ! yyout )
+/* %if-c-only */
yyout = stdout;
+/* %endif */
+/* %if-c++-only */
+/* %endif */
if ( ! YY_CURRENT_BUFFER ) {
FASTQ_ensure_buffer_stack (yyscanner);
@@ -946,6 +1136,7 @@ YY_DECL
while ( 1 ) /* loops until end-of-file is reached */
{
+/* %% [8.0] yymore()-related code goes here */
yy_cp = yyg->yy_c_buf_p;
/* Support of yytext. */
@@ -956,6 +1147,7 @@ YY_DECL
*/
yy_bp = yy_cp;
+/* %% [9.0] code to set up and find next match goes here */
yy_current_state = yyg->yy_start;
yy_current_state += YY_AT_BOL();
@@ -969,16 +1161,17 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 126 )
+ if ( yy_current_state >= 140 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
*yyg->yy_state_ptr++ = yy_current_state;
++yy_cp;
}
- while ( yy_current_state != 125 );
+ while ( yy_current_state != 139 );
yy_find_action:
+/* %% [10.0] code to find the action number goes here */
yy_current_state = *--yyg->yy_state_ptr;
yyg->yy_lp = yy_accept[yy_current_state];
find_rule: /* we branch to this label when backing up */
@@ -1019,6 +1212,8 @@ find_rule: /* we branch to this label when backing up */
YY_DO_BEFORE_ACTION;
+/* %% [11.0] code for yylineno update goes here */
+
if ( yy_act != YY_END_OF_BUFFER && yy_rule_can_match_eol[yy_act] )
{
int yyl;
@@ -1033,8 +1228,26 @@ find_rule: /* we branch to this label when backing up */
do_action: /* This label is used only to access EOF actions. */
+/* %% [12.0] debug code goes here */
+ if ( yy_flex_debug )
+ {
+ if ( yy_act == 0 )
+ fprintf( stderr, "--scanner backing up\n" );
+ else if ( yy_act < 33 )
+ fprintf( stderr, "--accepting rule at line %ld (\"%s\")\n",
+ (long)yy_rule_linenum[yy_act], yytext );
+ else if ( yy_act == 33 )
+ fprintf( stderr, "--accepting default rule (\"%s\")\n",
+ yytext );
+ else if ( yy_act == 34 )
+ fprintf( stderr, "--(end of buffer or a NUL)\n" );
+ else
+ fprintf( stderr, "--EOF (start condition %d)\n", YY_START );
+ }
+
switch ( yy_act )
{ /* beginning of action switch */
+/* %% [13.0] actions go here */
case 1:
YY_RULE_SETUP
{ BEGIN TAG_LINE; return yytext[0]; }
@@ -1048,6 +1261,7 @@ YY_RULE_SETUP
{ BEGIN QTAG_LINE; return yytext[0]; }
YY_BREAK
+
case 4:
YY_RULE_SETUP
{ return fqCOORDS; }
@@ -1089,6 +1303,7 @@ YY_RULE_SETUP
{ BEGIN IN_SEQUENCE; ENDLINE; }
YY_BREAK
+
case 13:
/* rule 13 can match eol */
YY_RULE_SETUP
@@ -1112,6 +1327,8 @@ YY_RULE_SETUP
{ BEGIN 0; return fqUNRECOGNIZED; }
YY_BREAK
+
+
case 17:
YY_RULE_SETUP
{ BEGIN 0; return fqBASESEQ; }
@@ -1120,32 +1337,47 @@ case 18:
YY_RULE_SETUP
{ BEGIN 0; return fqCOLORSEQ; }
YY_BREAK
-
case 19:
+YY_RULE_SETUP
+{ BEGIN 0; return fqNUMBER; }
+ YY_BREAK
+case 20:
+/* rule 20 can match eol */
+YY_RULE_SETUP
+{ BEGIN 0; ENDLINE; }
+ YY_BREAK
+
+
+
+case 21:
*yy_cp = yyg->yy_hold_char; /* undo effects of setting up yytext */
yyg->yy_c_buf_p = yy_cp -= 1;
YY_DO_BEFORE_ACTION; /* set up yytext again */
YY_RULE_SETUP
{ return fqTOKEN; /* do not bother to parse the inside of the line */ }
YY_BREAK
-case 20:
-/* rule 20 can match eol */
+case 22:
+/* rule 22 can match eol */
YY_RULE_SETUP
{ BEGIN IN_QUALITY; ENDLINE }
YY_BREAK
-case 21:
+
+
+case 23:
YY_RULE_SETUP
{ }
YY_BREAK
-case 22:
-/* rule 22 can match eol */
+
+
+case 24:
+/* rule 24 can match eol */
YY_RULE_SETUP
{ return fqASCQUAL; }
YY_BREAK
-case 23:
-/* rule 23 can match eol */
+case 25:
+/* rule 25 can match eol */
YY_RULE_SETUP
{ /* if read was split across several lines, expect the same number of lines in quality */
if (yyextra->expectedQualityLines <= 1)
@@ -1155,32 +1387,46 @@ YY_RULE_SETUP
ENDLINE;
}
YY_BREAK
-case 24:
+case 26:
YY_RULE_SETUP
{ return yytext[0]; }
YY_BREAK
-case 25:
-/* rule 25 can match eol */
+
+
+case 27:
+/* rule 27 can match eol */
YY_RULE_SETUP
{ return fqASCQUAL; }
YY_BREAK
-case 26:
-/* rule 26 can match eol */
+case 28:
+/* rule 28 can match eol */
YY_RULE_SETUP
{ BEGIN 0; ENDLINE; }
YY_BREAK
-case 27:
+case 29:
YY_RULE_SETUP
{ return yytext[0]; }
YY_BREAK
-case 28:
-/* rule 28 can match eol */
+
+
+case 30:
+/* rule 30 can match eol */
+YY_RULE_SETUP
+{ BEGIN 0; ENDLINE; }
+ YY_BREAK
+case 31:
+YY_RULE_SETUP
+{ BEGIN 0; return fqENDOFTEXT; }
+ YY_BREAK
+
+case 32:
+/* rule 32 can match eol */
YY_RULE_SETUP
{ ENDLINE; }
YY_BREAK
-case 29:
+case 33:
YY_RULE_SETUP
ECHO;
YY_BREAK
@@ -1195,6 +1441,7 @@ ECHO;
case YY_STATE_EOF(IN_QUALITY):
case YY_STATE_EOF(INLINE_SEQUENCE):
case YY_STATE_EOF(INLINE_QUALITY):
+ case YY_STATE_EOF(SKIP_TO_EOL):
yyterminate();
case YY_END_OF_BUFFER:
@@ -1260,6 +1507,7 @@ ECHO;
else
{
+/* %% [14.0] code to do back-up for compressed tables and set up yy_cp goes here */
yy_cp = yyg->yy_c_buf_p;
goto yy_find_action;
}
@@ -1325,6 +1573,14 @@ ECHO;
} /* end of action switch */
} /* end of scanning one token */
} /* end of FASTQ_lex */
+/* %ok-for-header */
+
+/* %if-c++-only */
+/* %not-for-header */
+
+/* %ok-for-header */
+
+/* %endif */
/* yy_get_next_buffer - try to read in a new buffer
*
@@ -1333,7 +1589,11 @@ ECHO;
* EOB_ACT_CONTINUE_SCAN - continue scanning from current position
* EOB_ACT_END_OF_FILE - end of file
*/
+/* %if-c-only */
static int yy_get_next_buffer (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf;
@@ -1439,12 +1699,19 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
/* yy_get_previous_state - get the state just before the EOB char was reached */
+/* %if-c-only */
+/* %not-for-header */
+
static yy_state_type yy_get_previous_state (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
register yy_state_type yy_current_state;
register char *yy_cp;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+/* %% [15.0] code to get the start state into yy_current_state goes here */
yy_current_state = yyg->yy_start;
yy_current_state += YY_AT_BOL();
@@ -1453,11 +1720,12 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
for ( yy_cp = yyg->yytext_ptr + YY_MORE_ADJ; yy_cp < yyg->yy_c_buf_p; ++yy_cp )
{
+/* %% [16.0] code to find the next state goes here */
register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1);
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 126 )
+ if ( yy_current_state >= 140 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -1472,27 +1740,37 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
* synopsis
* next_state = yy_try_NUL_trans( current_state );
*/
+/* %if-c-only */
static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
register int yy_is_jam;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */
+/* %% [17.0] code to find the next state, and perhaps do backing up, goes here */
register YY_CHAR yy_c = 1;
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 126 )
+ if ( yy_current_state >= 140 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
- yy_is_jam = (yy_current_state == 125);
+ yy_is_jam = (yy_current_state == 139);
if ( ! yy_is_jam )
*yyg->yy_state_ptr++ = yy_current_state;
return yy_is_jam ? 0 : yy_current_state;
}
+/* %if-c-only */
+
static void yyunput (int c, register char * yy_bp , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
register char *yy_cp;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -1525,6 +1803,8 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
*--yy_cp = (char) c;
+/* %% [18.0] update yylineno here */
+
if ( c == '\n' ){
--yylineno;
}
@@ -1533,7 +1813,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
yyg->yy_hold_char = *yy_cp;
yyg->yy_c_buf_p = yy_cp;
}
+/* %if-c-only */
+
+/* %endif */
+/* %if-c-only */
#ifndef YY_NO_INPUT
#ifdef __cplusplus
static int yyinput (yyscan_t yyscanner)
@@ -1541,6 +1825,9 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
static int input (yyscan_t yyscanner)
#endif
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
int c;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -1605,6 +1892,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
*yyg->yy_c_buf_p = '\0'; /* preserve yytext */
yyg->yy_hold_char = *++yyg->yy_c_buf_p;
+/* %% [19.0] update BOL and yylineno */
YY_CURRENT_BUFFER_LVALUE->yy_at_bol = (c == '\n');
if ( YY_CURRENT_BUFFER_LVALUE->yy_at_bol )
@@ -1615,14 +1903,20 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
return c;
}
+/* %if-c-only */
#endif /* ifndef YY_NO_INPUT */
+/* %endif */
/** Immediately switch to a different input stream.
* @param input_file A readable stream.
* @param yyscanner The scanner object.
* @note This function does not reset the start condition to @c INITIAL .
*/
+/* %if-c-only */
void FASTQ_restart (FILE * input_file , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -1640,7 +1934,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
* @param new_buffer The new input buffer.
* @param yyscanner The scanner object.
*/
+/* %if-c-only */
void FASTQ__switch_to_buffer (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -1672,7 +1970,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
yyg->yy_did_buffer_switch_on_eof = 1;
}
+/* %if-c-only */
static void FASTQ__load_buffer_state (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars;
@@ -1687,7 +1989,11 @@ static void FASTQ__load_buffer_state (yyscan_t yyscanner)
* @param yyscanner The scanner object.
* @return the allocated buffer state.
*/
+/* %if-c-only */
YY_BUFFER_STATE FASTQ__create_buffer (FILE * file, int size , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
YY_BUFFER_STATE b;
@@ -1715,7 +2021,11 @@ static void FASTQ__load_buffer_state (yyscan_t yyscanner)
* @param b a buffer created with FASTQ__create_buffer()
* @param yyscanner The scanner object.
*/
+/* %if-c-only */
void FASTQ__delete_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -1731,11 +2041,22 @@ static void FASTQ__load_buffer_state (yyscan_t yyscanner)
FASTQ_free((void *) b ,yyscanner );
}
+/* %if-c-only */
+
+/* %endif */
+
+/* %if-c++-only */
+/* %endif */
+
/* Initializes or reinitializes a buffer.
* This function is sometimes called more than once on the same buffer,
* such as during a FASTQ_restart() or at EOF.
*/
+/* %if-c-only */
static void FASTQ__init_buffer (YY_BUFFER_STATE b, FILE * file , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
int oerrno = errno;
@@ -1755,8 +2076,13 @@ static void FASTQ__load_buffer_state (yyscan_t yyscanner)
b->yy_bs_column = 0;
}
+/* %if-c-only */
+
b->yy_is_interactive = 0;
+/* %endif */
+/* %if-c++-only */
+/* %endif */
errno = oerrno;
}
@@ -1764,7 +2090,11 @@ static void FASTQ__load_buffer_state (yyscan_t yyscanner)
* @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER.
* @param yyscanner The scanner object.
*/
+/* %if-c-only */
void FASTQ__flush_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if ( ! b )
@@ -1788,13 +2118,18 @@ static void FASTQ__load_buffer_state (yyscan_t yyscanner)
FASTQ__load_buffer_state(yyscanner );
}
+/* %if-c-or-c++ */
/** Pushes the new state onto the stack. The new state becomes
* the current state. This function will allocate the stack
* if necessary.
* @param new_buffer The new state.
* @param yyscanner The scanner object.
*/
+/* %if-c-only */
void FASTQ_push_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if (new_buffer == NULL)
@@ -1820,12 +2155,18 @@ void FASTQ_push_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner)
FASTQ__load_buffer_state(yyscanner );
yyg->yy_did_buffer_switch_on_eof = 1;
}
+/* %endif */
+/* %if-c-or-c++ */
/** Removes and deletes the top of the stack, if present.
* The next element becomes the new top.
* @param yyscanner The scanner object.
*/
+/* %if-c-only */
void FASTQ_pop_buffer_state (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if (!YY_CURRENT_BUFFER)
@@ -1841,11 +2182,17 @@ void FASTQ_pop_buffer_state (yyscan_t yyscanner)
yyg->yy_did_buffer_switch_on_eof = 1;
}
}
+/* %endif */
+/* %if-c-or-c++ */
/* Allocates the stack if it does not exist.
* Guarantees space for at least one push.
*/
+/* %if-c-only */
static void FASTQ_ensure_buffer_stack (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
int num_to_alloc;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -1888,7 +2235,9 @@ static void FASTQ_ensure_buffer_stack (yyscan_t yyscanner)
yyg->yy_buffer_stack_max = num_to_alloc;
}
}
+/* %endif */
+/* %if-c-only */
/** Setup the input buffer state to scan directly from a user-specified character buffer.
* @param base the character buffer
* @param size the size in bytes of the character buffer
@@ -1923,7 +2272,9 @@ YY_BUFFER_STATE FASTQ__scan_buffer (char * base, yy_size_t size , yyscan_t yys
return b;
}
+/* %endif */
+/* %if-c-only */
/** Setup the input buffer state to scan a string. The next call to FASTQ_lex() will
* scan from a @e copy of @a str.
* @param yystr a NUL-terminated string to scan
@@ -1937,7 +2288,9 @@ YY_BUFFER_STATE FASTQ__scan_string (yyconst char * yystr , yyscan_t yyscanner)
return FASTQ__scan_bytes(yystr,strlen(yystr) ,yyscanner);
}
+/* %endif */
+/* %if-c-only */
/** Setup the input buffer state to scan the given bytes. The next call to FASTQ_lex() will
* scan from a @e copy of @a bytes.
* @param bytes the byte buffer to scan
@@ -1974,8 +2327,13 @@ YY_BUFFER_STATE FASTQ__scan_bytes (yyconst char * yybytes, int _yybytes_len ,
return b;
}
+/* %endif */
+/* %if-c-only */
static void yy_push_state (int new_state , yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if ( yyg->yy_start_stack_ptr >= yyg->yy_start_stack_depth )
@@ -2000,7 +2358,11 @@ YY_BUFFER_STATE FASTQ__scan_bytes (yyconst char * yybytes, int _yybytes_len ,
BEGIN(new_state);
}
+/* %if-c-only */
static void yy_pop_state (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if ( --yyg->yy_start_stack_ptr < 0 )
@@ -2009,7 +2371,11 @@ YY_BUFFER_STATE FASTQ__scan_bytes (yyconst char * yybytes, int _yybytes_len ,
BEGIN(yyg->yy_start_stack[yyg->yy_start_stack_ptr]);
}
+/* %if-c-only */
static int yy_top_state (yyscan_t yyscanner)
+/* %endif */
+/* %if-c++-only */
+/* %endif */
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
return yyg->yy_start_stack[yyg->yy_start_stack_ptr - 1];
@@ -2019,11 +2385,15 @@ YY_BUFFER_STATE FASTQ__scan_bytes (yyconst char * yybytes, int _yybytes_len ,
#define YY_EXIT_FAILURE 2
#endif
+/* %if-c-only */
static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner)
{
(void) fprintf( stderr, "%s\n", msg );
exit( YY_EXIT_FAILURE );
}
+/* %endif */
+/* %if-c++-only */
+/* %endif */
/* Redefine yyless() so it works in section 3 code. */
@@ -2044,6 +2414,9 @@ static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner)
/* Accessor methods (get/set functions) to struct members. */
+/* %if-c-only */
+/* %if-reentrant */
+
/** Get the user-defined data for this scanner.
* @param yyscanner The scanner object.
*/
@@ -2053,6 +2426,8 @@ YY_EXTRA_TYPE FASTQ_get_extra (yyscan_t yyscanner)
return yyextra;
}
+/* %endif */
+
/** Get the current line number.
* @param yyscanner The scanner object.
*/
@@ -2116,6 +2491,8 @@ char *FASTQ_get_text (yyscan_t yyscanner)
return yytext;
}
+/* %if-reentrant */
+
/** Set the user-defined data. This data is never touched by the scanner.
* @param user_defined The data to be associated with this scanner.
* @param yyscanner The scanner object.
@@ -2126,6 +2503,8 @@ void FASTQ_set_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner)
yyextra = user_defined ;
}
+/* %endif */
+
/** Set the current line number.
* @param line_number
* @param yyscanner The scanner object.
@@ -2186,8 +2565,13 @@ void FASTQ_set_debug (int bdebug , yyscan_t yyscanner)
yy_flex_debug = bdebug ;
}
+/* %endif */
+
+/* %if-reentrant */
/* Accessor methods for yylval and yylloc */
+/* %if-bison-bridge */
+
YYSTYPE * FASTQ_get_lval (yyscan_t yyscanner)
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -2200,6 +2584,8 @@ void FASTQ_set_lval (YYSTYPE * yylval_param , yyscan_t yyscanner)
yylval = yylval_param;
}
+/* %endif */
+
/* User-visible API */
/* FASTQ_lex_init is special because it creates the scanner itself, so it is
@@ -2264,6 +2650,9 @@ int FASTQ_lex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals
return yy_init_globals ( *ptr_yy_globals );
}
+/* %endif if-c-only */
+
+/* %if-c-only */
static int yy_init_globals (yyscan_t yyscanner)
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
@@ -2301,7 +2690,9 @@ static int yy_init_globals (yyscan_t yyscanner)
*/
return 0;
}
+/* %endif */
+/* %if-c-only SNIP! this currently causes conflicts with the c++ scanner */
/* FASTQ_lex_destroy is for both reentrant and non-reentrant scanners. */
int FASTQ_lex_destroy (yyscan_t yyscanner)
{
@@ -2329,11 +2720,14 @@ int FASTQ_lex_destroy (yyscan_t yyscanner)
* FASTQ_lex() is called, initialization will occur. */
yy_init_globals( yyscanner);
+/* %if-reentrant */
/* Destroy the main struct (reentrant only). */
FASTQ_free ( yyscanner , yyscanner );
yyscanner = NULL;
+/* %endif */
return 0;
}
+/* %endif */
/*
* Internal utility routines.
@@ -2381,7 +2775,15 @@ void FASTQ_free (void * ptr , yyscan_t yyscanner)
free( (char *) ptr ); /* see FASTQ_realloc() for (char *) cast */
}
+/* %if-tables-serialization definitions */
+/* %define-yytables The name for this specific scanner's tables. */
#define YYTABLES_NAME "yytables"
+/* %endif */
+
+/* %ok-for-header */
+
+
+
rc_t CC FASTQScan_yylex_init(FASTQParseBlock* sb, bool debug)
{
@@ -2425,6 +2827,15 @@ void CC FASTQScan_inline_quality(FASTQParseBlock* pb)
yy_push_state(INLINE_QUALITY, pb->scanner);
}
+void CC FASTQScan_skip_to_eol(FASTQParseBlock* pb)
+{
+ struct yyguts_t* yyg = (struct yyguts_t*)pb->scanner;
+ if ( yytext[0] != '\n' && yytext[0] != '\r' ) /* not at an EOL already */
+ {
+ yy_push_state(SKIP_TO_EOL, pb->scanner);
+ }
+}
+
void CC FASTQ_unlex(FASTQParseBlock* pb, FASTQToken* token)
{
size_t i;
diff --git a/tools/fastq-loader/fastq-lex.l b/tools/fastq-loader/fastq-lex.l
index 56b46e3..e96a37d 100644
--- a/tools/fastq-loader/fastq-lex.l
+++ b/tools/fastq-loader/fastq-lex.l
@@ -71,6 +71,7 @@
%x IN_QUALITY
%x INLINE_SEQUENCE
%x INLINE_QUALITY
+%x SKIP_TO_EOL
ws [ \t]+
digits [0-9]+
@@ -82,7 +83,7 @@ base [ACGTacgtNn.]
cskey [ACGTacgt]
color [0-3.]
-ascqual [\x0E-\x1F \x21-\x7E]+
+ascqual [\x0E-\x1F \x21-\x7F]+
%%
@@ -117,6 +118,8 @@ ascqual [\x0E-\x1F \x21-\x7E]+
{
{base}+ { BEGIN 0; return fqBASESEQ; }
{cskey}{color}+ { BEGIN 0; return fqCOLORSEQ; }
+ {digits} { BEGIN 0; return fqNUMBER; }
+ {eol} { BEGIN 0; ENDLINE; }
}
<QTAG_LINE>
@@ -150,6 +153,12 @@ ascqual [\x0E-\x1F \x21-\x7E]+
. { return yytext[0]; }
}
+<SKIP_TO_EOL>
+{
+ .*{eol} { BEGIN 0; ENDLINE; }
+ .* { BEGIN 0; return fqENDOFTEXT; }
+}
+
{eol} { ENDLINE; }
%%
@@ -196,6 +205,15 @@ void CC FASTQScan_inline_quality(FASTQParseBlock* pb)
yy_push_state(INLINE_QUALITY, pb->scanner);
}
+void CC FASTQScan_skip_to_eol(FASTQParseBlock* pb)
+{
+ struct yyguts_t* yyg = (struct yyguts_t*)pb->scanner;
+ if ( yytext[0] != '\n' && yytext[0] != '\r' ) /* not at an EOL already */
+ {
+ yy_push_state(SKIP_TO_EOL, pb->scanner);
+ }
+}
+
void CC FASTQ_unlex(FASTQParseBlock* pb, FASTQToken* token)
{
size_t i;
diff --git a/tools/fastq-loader/fastq-loader.c b/tools/fastq-loader/fastq-loader.c
index 45754b3..41f5670 100644
--- a/tools/fastq-loader/fastq-loader.c
+++ b/tools/fastq-loader/fastq-loader.c
@@ -43,12 +43,15 @@
#include <kapp/log-xml.h>
#include <align/writer-refseq.h>
+#include "fastq-parse.h"
+
extern rc_t run(char const argv0[],
struct CommonWriterSettings* G,
unsigned countReads,
const char* reads[],
- uint8_t qualityOffset,
- const int8_t defaultReadNumbers[]);
+ enum FASTQQualityFormat qualityFormat,
+ const int8_t defaultReadNumbers[],
+ bool ignoreSpotGroups);
/* MARK: Arguments and Usage */
static char const option_input[] = "input";
@@ -62,6 +65,7 @@ static char const option_platform[] = "platform";
static char const option_quality[] = "quality";
static char const option_read[] = "read";
static char const option_max_err_pct[] = "max-err-pct";
+static char const option_ignore_illumina_tags[] = "ignore-illumina-tags";
#define OPTION_INPUT option_input
#define OPTION_OUTPUT option_output
@@ -74,6 +78,7 @@ static char const option_max_err_pct[] = "max-err-pct";
#define OPTION_QUALITY option_quality
#define OPTION_READ option_read
#define OPTION_MAX_ERR_PCT option_max_err_pct
+#define OPTION_IGNORE_ILLUMINA_TAGS option_ignore_illumina_tags
#define ALIAS_INPUT "i"
#define ALIAS_OUTPUT "o"
@@ -137,7 +142,7 @@ char const * use_platform[] =
static
char const * use_quality[] =
{
- "Quality encoding (PHRED_33, PHRED_64)",
+ "Quality encoding (PHRED_33, PHRED_64, LOGODDS)",
NULL
};
@@ -155,18 +160,26 @@ char const * use_max_err_pct[] =
NULL
};
+static
+char const * use_ignore_illumina_tags[] =
+{
+ "ignore barcodes contained in Illumina-formatted names",
+ NULL
+};
+
OptDef Options[] =
{
- /* order here is same as in param array below!!! */ /* max#, needs param, required */
- { OPTION_OUTPUT, ALIAS_OUTPUT, NULL, output_usage, 1, true, true },
- { OPTION_TMPFS, ALIAS_TMPFS, NULL, tmpfs_usage, 1, true, false },
- { OPTION_QCOMP, ALIAS_QCOMP, NULL, qcomp_usage, 1, true, false },
- { OPTION_CACHE_SIZE, NULL, NULL, cache_size_usage, 1, true, false },
- { OPTION_MAX_REC_COUNT, NULL, NULL, mrc_usage, 1, true, false },
- { OPTION_MAX_ERR_COUNT, ALIAS_MAX_ERR_COUNT, NULL, mec_usage, 1, true, false },
- { OPTION_PLATFORM, ALIAS_PLATFORM, NULL, use_platform, 1, true, false },
- { OPTION_QUALITY, ALIAS_QUALITY, NULL, use_quality, 1, true, true },
- { OPTION_MAX_ERR_PCT, NULL, NULL, use_max_err_pct, 1, true, false },
+ /* order here is same as in param array below!!! */ /* max#, needs param, required */
+ { OPTION_OUTPUT, ALIAS_OUTPUT, NULL, output_usage, 1, true, true },
+ { OPTION_TMPFS, ALIAS_TMPFS, NULL, tmpfs_usage, 1, true, false },
+ { OPTION_QCOMP, ALIAS_QCOMP, NULL, qcomp_usage, 1, true, false },
+ { OPTION_CACHE_SIZE, NULL, NULL, cache_size_usage, 1, true, false },
+ { OPTION_MAX_REC_COUNT, NULL, NULL, mrc_usage, 1, true, false },
+ { OPTION_MAX_ERR_COUNT, ALIAS_MAX_ERR_COUNT, NULL, mec_usage, 1, true, false },
+ { OPTION_PLATFORM, ALIAS_PLATFORM, NULL, use_platform, 1, true, false },
+ { OPTION_QUALITY, ALIAS_QUALITY, NULL, use_quality, 1, true, true },
+ { OPTION_MAX_ERR_PCT, NULL, NULL, use_max_err_pct, 1, true, false },
+ { OPTION_IGNORE_ILLUMINA_TAGS, NULL, NULL, use_ignore_illumina_tags, 1, false, false },
/* { OPTION_READ, ALIAS_READ, NULL, use_read, 0, true, false },*/
};
@@ -182,6 +195,7 @@ const char* OptHelpParam[] =
NULL,
NULL,
NULL,
+ NULL,
};
rc_t UsageSummary (char const * progname)
@@ -290,7 +304,8 @@ rc_t CC KMain (int argc, char * argv[])
char const *value;
char *dummy;
const XMLLogger* xml_logger = NULL;
- uint8_t qualityOffset;
+ enum FASTQQualityFormat qualityFormat;
+ bool ignoreSpotGroups;
memset(&G, 0, sizeof(G));
@@ -329,7 +344,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_TMPFS, 0, &G.tmpfs);
+ rc = ArgsOptionValue (args, OPTION_TMPFS, 0, (const void **)&G.tmpfs);
if (rc)
break;
}
@@ -346,7 +361,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_OUTPUT, 0, &G.outpath);
+ rc = ArgsOptionValue (args, OPTION_OUTPUT, 0, (const void **)&G.outpath);
if (rc)
break;
}
@@ -369,7 +384,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_QCOMP, 0, &G.QualQuantizer);
+ rc = ArgsOptionValue (args, OPTION_QCOMP, 0, (const void **)&G.QualQuantizer);
if (rc)
break;
}
@@ -379,7 +394,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_CACHE_SIZE, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_CACHE_SIZE, 0, (const void **)&value);
if (rc)
break;
G.cache_size = strtoul(value, &dummy, 0) * 1024UL * 1024UL;
@@ -398,7 +413,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MAX_REC_COUNT, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MAX_REC_COUNT, 0, (const void **)&value);
if (rc)
break;
G.maxAlignCount = strtoul(value, &dummy, 0);
@@ -409,7 +424,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MAX_ERR_COUNT, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MAX_ERR_COUNT, 0, (const void **)&value);
if (rc)
break;
G.maxErrCount = strtoul(value, &dummy, 0);
@@ -420,7 +435,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_MAX_ERR_PCT, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_MAX_ERR_PCT, 0, (const void **)&value);
if (rc)
break;
G.maxErrPct = strtoul(value, &dummy, 0);
@@ -431,7 +446,7 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_PLATFORM, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_PLATFORM, 0, (const void **)&value);
if (rc)
break;
G.platform = PlatformToId(value);
@@ -451,13 +466,15 @@ rc_t CC KMain (int argc, char * argv[])
break;
if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_QUALITY, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_QUALITY, 0, (const void **)&value);
if (rc)
break;
if (strcmp(value, "PHRED_33") == 0)
- qualityOffset = 33;
+ qualityFormat = FASTQphred33;
else if (strcmp(value, "PHRED_64") == 0)
- qualityOffset = 64;
+ qualityFormat = FASTQphred64;
+ else if (strcmp(value, "LOGODDS") == 0)
+ qualityFormat = FASTQlogodds;
else
{
rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcIncorrect);
@@ -467,8 +484,13 @@ rc_t CC KMain (int argc, char * argv[])
}
}
else
- qualityOffset = 0;
+ qualityFormat = 0;
+ rc = ArgsOptionCount (args, OPTION_IGNORE_ILLUMINA_TAGS, &pcount);
+ if (rc)
+ break;
+ ignoreSpotGroups = pcount > 0;
+
rc = ArgsParamCount (args, &pcount);
if (rc) break;
if (pcount == 0)
@@ -488,7 +510,7 @@ rc_t CC KMain (int argc, char * argv[])
unsigned i;
for (i = 0; i < pcount; ++i) {
- rc = ArgsParamValue(args, i, &value);
+ rc = ArgsParamValue(args, i, (const void **)&value);
if (rc) break;
need += strlen(value) + 1;
}
@@ -506,7 +528,7 @@ rc_t CC KMain (int argc, char * argv[])
unsigned i;
for (i = 0; i < pcount; ++i) {
- rc = ArgsParamValue(args, i, &value);
+ rc = ArgsParamValue(args, i, (const void **)&value);
if (rc) break;
defaultReadNumbers[i] = 0;
@@ -519,7 +541,7 @@ rc_t CC KMain (int argc, char * argv[])
else
break;
- rc = run(argv[0], &G, pcount, (char const **)files, qualityOffset, defaultReadNumbers);
+ rc = run(argv[0], &G, pcount, (char const **)files, qualityFormat, defaultReadNumbers, ignoreSpotGroups);
break;
}
free(name_buffer);
diff --git a/tools/fastq-loader/fastq-parse.h b/tools/fastq-loader/fastq-parse.h
index 3ec93eb..79a6b76 100644
--- a/tools/fastq-loader/fastq-parse.h
+++ b/tools/fastq-loader/fastq-parse.h
@@ -36,11 +36,13 @@
extern "C" {
#endif
-/* values used in validating quality lines */
-#define MIN_PHRED_33 33
-#define MAX_PHRED_33 126
-#define MIN_PHRED_64 64
-#define MAX_PHRED_64 127
+enum FASTQQualityFormat
+{
+ FASTQunknown,
+ FASTQphred33,
+ FASTQphred64,
+ FASTQlogodds
+};
struct FastqSequence
{
@@ -66,7 +68,8 @@ struct FastqSequence
bool is_colorspace;
String quality;
- uint8_t qualityOffset;
+ uint8_t qualityFormat;
+ uint8_t qualityAsciiOffset;
bool lowQuality;
};
@@ -91,41 +94,47 @@ typedef struct FASTQToken
/* obtain a pointer to the token's text */
#define TokenTextPtr(pb, token) ((const char*)((pb)->record->source.base) + (token)->tokenStart)
-
typedef struct FASTQParseBlock
{
void* self;
size_t (CC *input)(struct FASTQParseBlock* sb, char* buf, size_t max_size);
- uint8_t phredOffset;
- uint8_t maxPhred;
+ /* inputs for the parser */
+ size_t expectedQualityLines;
+ uint8_t qualityFormat; /* see enum FASTQQualityFormat above */
+ int8_t defaultReadNumber; /* -1: never assign read numbers */
+
+ /* Secondary (>1) read number observed previously (usually 2, sometimes 3).
+ Once one is seen, do not allow any other values in the same input file.
+ 0 = has not seen one yet in this input.
+ Always record as 2 */
+ uint8_t secondaryReadNumber;
+
+ bool ignoreSpotGroups;
+
+ /* temporaries and outputs for the parser */
void* scanner;
size_t length; /* input characters consumed for the current record */
FASTQToken* lastToken;
struct FastqRecord* record;
size_t column;
- /* temporaries for bison */
/* all offsets are into record->source */
size_t spotNameOffset;
size_t spotNameLength;
+ size_t spotNameOffset_saved; /* sometimes needed to revert to older values */
+ size_t spotNameLength_saved;
bool spotNameDone;
+
size_t spotGroupOffset;
size_t spotGroupLength;
+
size_t readOffset;
size_t readLength;
+
size_t qualityOffset;
size_t qualityLength;
-
- size_t expectedQualityLines;
-
- int8_t defaultReadNumber; /* -1: never assign read numbers */
-
- /* Secondary (>1) read number observed previously (usually 2, sometimes 3).
- Once one is seen, do not allow any other values in the same input file.
- 0 = has not seen one yet in this input.
- Always record as 2 */
- uint8_t secondaryReadNumber;
+ uint8_t qualityAsciiOffset;
bool fatalError;
} FASTQParseBlock;
@@ -136,6 +145,7 @@ extern void FASTQScan_yylex_destroy(FASTQParseBlock* context);
/* explicit FLEX state control for bison*/
extern void FASTQScan_inline_sequence(FASTQParseBlock* pb);
extern void FASTQScan_inline_quality(FASTQParseBlock* pb);
+extern void FASTQScan_skip_to_eol(FASTQParseBlock* pb); /*the next token will be EOL or EOF*/
extern void FASTQ_set_lineno (int line_number, void* scanner);
diff --git a/tools/fastq-loader/fastq-reader.c b/tools/fastq-loader/fastq-reader.c
index 6b5e0f2..a90c292 100644
--- a/tools/fastq-loader/fastq-reader.c
+++ b/tools/fastq-loader/fastq-reader.c
@@ -222,7 +222,7 @@ FastqSequenceInit(FastqSequence* self)
StringInit(&self->read, 0, 0, 0);
self->is_colorspace = false;
StringInit(&self->quality, 0, 0, 0);
- self->qualityOffset = 0;
+ self->qualityAsciiOffset = 0;
self->lowQuality = false;
return 0;
@@ -321,14 +321,21 @@ static rc_t FastqSequenceGetQuality ( const FastqSequence *self, const int8_t **
return RC(rcAlign, rcRow, rcReading, rcData, rcInconsistent);
*quality = (const int8_t *)self->quality.addr;
- *offset = self->qualityOffset;
+ *offset = self->qualityAsciiOffset;
}
else
{
*quality = NULL;
*offset = 0;
}
- *qualType = QT_Phred;
+ if ( self->qualityFormat == FASTQlogodds )
+ {
+ *qualType = QT_LogOdds;
+ }
+ else
+ {
+ *qualType = QT_Phred;
+ }
return 0;
}
@@ -391,7 +398,7 @@ static rc_t FastqSequenceGetCSQuality ( const FastqSequence *self, const int8_t
if ( self->quality.size != 0 && self->is_colorspace )
{
*quality = (const int8_t *)self->quality.addr + 1;
- *offset = self->qualityOffset;
+ *offset = self->qualityAsciiOffset;
}
else
{
@@ -567,7 +574,8 @@ rc_t FastqReaderFileGetRecord ( const FastqReaderFile *f, const Record** result
StringInit( & self->pb.record->seq.spotgroup, (const char*)self->pb.record->source.base + self->pb.spotGroupOffset, self->pb.spotGroupLength, (uint32_t)self->pb.spotGroupLength);
StringInit( & self->pb.record->seq.read, (const char*)self->pb.record->source.base + self->pb.readOffset, self->pb.readLength, (uint32_t)self->pb.readLength);
StringInit( & self->pb.record->seq.quality, (const char*)self->pb.record->source.base + self->pb.qualityOffset, self->pb.qualityLength, (uint32_t)self->pb.qualityLength);
- self->pb.record->seq.qualityOffset = self->pb.phredOffset;
+ self->pb.record->seq.qualityFormat = self->pb.qualityFormat;
+ self->pb.record->seq.qualityAsciiOffset = self->pb.qualityAsciiOffset;
if (self->pb.record->seq.readnumber == 0)
self->pb.record->seq.readnumber = self->pb.defaultReadNumber;
@@ -628,7 +636,12 @@ size_t CC FASTQ_input(FASTQParseBlock* pb, char* buf, size_t max_size)
return length;
}
-rc_t CC FastqReaderFileMake( const ReaderFile **reader, const KDirectory* dir, const char* file, uint8_t phredOffset, uint8_t phredMax, int8_t defaultReadNumber)
+rc_t CC FastqReaderFileMake( const ReaderFile **reader,
+ const KDirectory* dir,
+ const char* file,
+ enum FASTQQualityFormat qualityFormat,
+ int8_t defaultReadNumber,
+ bool ignoreSpotGroups)
{
rc_t rc;
FastqReaderFile* self = (FastqReaderFile*) malloc ( sizeof * self );
@@ -654,23 +667,11 @@ rc_t CC FastqReaderFileMake( const ReaderFile **reader, const KDirectory* dir, c
if (rc == 0)
{
self->pb.self = self;
- self->pb.input = FASTQ_input;
- self->pb.phredOffset = phredOffset;
- self->pb.maxPhred = phredMax;
- /* TODO:
- if phredOffset is 0,
- guess based on the raw values on the first quality line:
- if all values are above MAX_PHRED_33, phredOffset = 64
- if all values are in MIN_PHRED_33..MAX_PHRED_33, phredOffset = 33
- if any value is below MIN_PHRED_33, abort
- if the guess is 33 and proven wrong (a raw quality value >MAX_PHRED_33 is encountered and no values below MIN_PHRED_64 ever seen)
- reopen the file,
- phredOffset = 64
- try to parse again
- if a value below MIN_PHRED_64 seen, abort
- */
+ self->pb.input = FASTQ_input;
+ self->pb.qualityFormat = qualityFormat;
self->pb.defaultReadNumber = defaultReadNumber;
self->pb.secondaryReadNumber = 0;
+ self->pb.ignoreSpotGroups = ignoreSpotGroups;
rc = FASTQScan_yylex_init(& self->pb, false);
if (rc == 0)
@@ -704,3 +705,5 @@ rc_t FastqReaderFileGetReferenceInfo ( const READERFILE_IMPL *self, const Refere
*result = NULL;
return 0;
}
+
+
diff --git a/tools/fastq-loader/fastq-reader.h b/tools/fastq-loader/fastq-reader.h
index 4fb1f6a..758cac9 100644
--- a/tools/fastq-loader/fastq-reader.h
+++ b/tools/fastq-loader/fastq-reader.h
@@ -31,6 +31,8 @@
#include <klib/defs.h>
#endif
+#include "fastq-parse.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -41,7 +43,12 @@ extern "C" {
struct KDirectory;
struct ReaderFile;
-rc_t CC FastqReaderFileMake( const struct ReaderFile **self, const struct KDirectory* dir, const char* file, uint8_t phredOffset, uint8_t phredMax, int8_t defaultReadNumber);
+rc_t CC FastqReaderFileMake( const struct ReaderFile **self,
+ const struct KDirectory* dir,
+ const char* file,
+ enum FASTQQualityFormat qualityFormat,
+ int8_t defaultReadNumber,
+ bool ignoreSpotGroups);
#ifdef __cplusplus
}
diff --git a/tools/fastq-loader/latf-load.vers b/tools/fastq-loader/latf-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/fastq-loader/latf-load.vers
+++ b/tools/fastq-loader/latf-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/fastq-loader/latf-load.vers.h b/tools/fastq-loader/latf-load.vers.h
index fa26c04..f9532ef 100644
--- a/tools/fastq-loader/latf-load.vers.h
+++ b/tools/fastq-loader/latf-load.vers.h
@@ -1 +1 @@
-#define LATF_LOAD_VERS 0x02040002
+#define LATF_LOAD_VERS 0x02050007
diff --git a/tools/fastq-loader/loader-imp.c b/tools/fastq-loader/loader-imp.c
index 8a03038..89bdd05 100644
--- a/tools/fastq-loader/loader-imp.c
+++ b/tools/fastq-loader/loader-imp.c
@@ -58,8 +58,9 @@ rc_t AcrhiveFASTQ(CommonWriterSettings* G,
VDatabase *db,
unsigned seqFiles,
char const *seqFile[],
- uint8_t qualityOffset,
- const int8_t defaultReadNumbers[])
+ enum FASTQQualityFormat qualityFormat,
+ const int8_t defaultReadNumbers[],
+ bool ignoreSpotGroups)
{
rc_t rc = 0;
unsigned i;
@@ -80,9 +81,9 @@ rc_t AcrhiveFASTQ(CommonWriterSettings* G,
for (i = 0; i < seqFiles; ++i) {
const ReaderFile *reader;
if (G->platform == SRA_PLATFORM_PACBIO_SMRT)
- rc = FastqReaderFileMake(&reader, dir, seqFile[i], 33, 33 + 93, -1);
+ rc = FastqReaderFileMake(&reader, dir, seqFile[i], FASTQphred33, -1, ignoreSpotGroups);
else
- rc = FastqReaderFileMake(&reader, dir, seqFile[i], qualityOffset, 0, defaultReadNumbers[i]);
+ rc = FastqReaderFileMake(&reader, dir, seqFile[i], qualityFormat, defaultReadNumbers[i], ignoreSpotGroups);
if (rc == 0)
{
@@ -165,7 +166,13 @@ rc_t ConvertDatabaseToUnmapped(VDatabase* db)
return rc;
}
-rc_t run(char const progName[], CommonWriterSettings* G, unsigned seqFiles, const char *seqFile[], uint8_t qualityOffset, const int8_t defaultReadNumbers[])
+rc_t run ( char const progName[],
+ CommonWriterSettings* G,
+ unsigned seqFiles,
+ const char *seqFile[],
+ uint8_t qualityOffset,
+ const int8_t defaultReadNumbers[],
+ bool ignoreSpotGroups )
{
VDBManager *mgr;
rc_t rc;
@@ -200,7 +207,7 @@ rc_t run(char const progName[], CommonWriterSettings* G, unsigned seqFiles, cons
if (rc == 0)
rc = rc2;
if (rc == 0) {
- rc = AcrhiveFASTQ(G, mgr, db, seqFiles, seqFile, qualityOffset, defaultReadNumbers);
+ rc = AcrhiveFASTQ(G, mgr, db, seqFiles, seqFile, qualityOffset, defaultReadNumbers, ignoreSpotGroups);
}
if (rc == 0) {
diff --git a/tools/fastq-loader/parsing_rules.txt b/tools/fastq-loader/parsing_rules.txt
new file mode 100755
index 0000000..9fed691
--- /dev/null
+++ b/tools/fastq-loader/parsing_rules.txt
@@ -0,0 +1,93 @@
+
+This is a summary of parsing rules for fastq files, as implemented in the latf-load tool.
+
+I. Notation
+
+The parsing rules are specified using a variant of Backus-Naur Form notation (BNF). The conventions are:
+
+ Token names are specified in capital letters. Tokens are defined in section II.
+ Names of non-terminals are specified in lowercase. Non-terminals are defined in section III.
+ { A } means "A repeated 0 or more times"
+ [ A ] means "optional A"
+ A | B means "A or B"
+ ( ) parentheses are used to group items
+
+II. Tokens
+
+The following tokens are used in the grammar description.
+
+ ALPHANUM - a sequence of (Latin) letters and decimal digits or dashes, beginning with a letter
+ NUMBER - a sequence of decimal digits
+ WS - a sequence of white space characters (spaces or tabs)
+ EOL - an end of line character (not considered a white space)
+ COORDS - a sequence of 4 decimal numbers, each preceded by ':', e.g. :8:2:342:540
+ RUNDOTSPOT - a special identifier representing a spot inside a run, in the format [SDE]RR{digits}.{digits}. An example would be "DRR000123.12"
+ BASES - a sequence of characters representing bases in text notation. Valid characters are A, C, G, T, a, c, g, t, N, n.
+ CSBASES - a sequence of characters representing bases in colorspace notation. The first character has to be one of A, C, G, T, a, c, g, t.
+ The remaining characters have to be '1', '2', '3', or '.'
+ QUAL - a sequence of characters representing qualities. The range of valid characters is determined by quality encoding
+ specified by the command line option --quality, 33-126 for PHRED_33 or 64-127 for PHRED_64, inclusive.
+
+In addition, grammar rules use literal tokens in apostrophes, e.g. ':'
+
+III. Parsing rules
+
+1. Input file
+
+ input:
+ { readLines [ qualityLines ] } |
+ { qualityLines } |
+ { name COORDS ':' ( BASES | CSBASES ) ':' QUAL EOL }
+
+Normally, an input file consists of a sequence of reads, each occupying multiple lines which identify read and contain its bases, with or without qualities.
+It is possible to have a file with only qualities specified for every read; in that case the loader must also be given a file with corresponding bases.
+
+An alternative is an "inline" format which has name, bases and qualities of each read on a single line.
+
+
+2. Read
+
+ readLines : ( '@' | '>' ) ( nameSpotGroup [ readNumberOrTail ] | runSpotRead ) EOL read
+
+ read : baseRead | csRead
+
+ baseRead : BASES EOL { BASES EOL }
+
+ csRead : CSBASES EOL { CSBASES EOL }
+
+ qualityLines: '+' { fqTOKEN } EOL quality
+
+ quality: QUAL EOL { QUAL EOL }
+
+A read in a fastq file is identified by its tag line. Tag line is followed by bases in text or colorspace format.
+
+An optional quality specification may follow. The quality specification starts with '+' and its own tag line, which is
+expected to match the tag line of the textually preceding read, although it is not enforced by latf-load.
+
+3. Tag line
+
+ nameSpotGroup : name [ COORDS [ ':' ] ] spotGroup |
+ name COORDS [ ':' ] [ '.' ] name |
+ name COORDS '_' [ casava1_8 ]
+
+ name : ( ALPHANUM | NUMBER ) { '_' | '-' | '.' | ':' | ALPHANUM | NUMBER }
+
+ spotGroup : '#' ( NUMBER | ALPHANUM )
+
+ readNumberOrTail : ( readNumber | WS [ casava1_8 ] | WS tail ) { WS tail }
+
+ readNumber : '/' NUMBER { '/' name }
+
+ casava1_8 : NUMBER ':' ALPHANUM ':' NUMBER ':' ( ALPHANUM | NUMBER )
+
+ tail : ALPHANUM { NUMBER | ALPHANUM | '_' | '/' | '=' )
+
+ runSpotRead : RUNDOTSPOT [ ( '.' | '/' ) NUMBER ] { tail | NUMBER }
+
+The purpose of the tag line is to identify a read. The components of a tag line may represent run name (possibly including coordinates), spot group, read number, and some additional information which may be of no concern to latf-load.
+Tag lines come in a multitude of variations on the basic fastq format, somewhat loosely defined in e.g. http://en.wikipedia.org/wiki/FASTQ_format
+Parsing rules for the tag line in latf-load have been designed to recognize major variations of the basic tag line format and
+evolved based on format variations encountered in real life submissions.
+
+
+
diff --git a/tools/fix-run-stat/fix-run-stat.c b/tools/fix-run-stat/fix-run-stat.c
deleted file mode 100644
index fefefbc..0000000
--- a/tools/fix-run-stat/fix-run-stat.c
+++ /dev/null
@@ -1,355 +0,0 @@
-/******************************************************************************/
-#include <kapp/main.h> /* KMain */
-#include <sra/wsradb.h> /* SRAMgr */
-#include <kdb/table.h> /* KTableRelease */
-#include <kxml/xml.h> /* KXMLMgrRelease */
-#include <kfs/file.h> /* KFileRelease */
-#include <klib/container.h> /* BSTree */
-#include <klib/log.h> /* LOGERR */
-#include <klib/rc.h> /* RC */
-#include <stdlib.h> /* calloc */
-#include <string.h> /* memset */
-#include <assert.h>
-#define DISP_RC(rc, msg) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, msg))
-#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
- if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
-#define FREE(obj) do { free(obj); obj = NULL; } while (false)
-typedef struct SValue {
- uint64_t val;
- bool found;
-} SValue;
-typedef struct Meta {
- SValue base_count;
- SValue base_count_bio;
- SValue spot_count;
- const char* member_name;
- bool found;
-} Meta;
-typedef struct MetaMember {
- BSTNode n;
- char* member_name;
- bool checked;
- Meta* meta;
-} MetaMember;
-typedef struct XmlMeta { /* XML data: Just one of root or tr is present */
- Meta root; /* sra-stat root node attributes */
- BSTree* tr; /* sra-stat spot-group nodes (Member) attributes */
-} XmlMeta;
-typedef struct SDbMeta { /* DB metadata */
- const Meta root; /* meta root */
-
- bool stats_found; /* STATS */
- const Meta* table; /* STATS/TABLE */
- const BSTree* tr; /* STATS/SPOT_GROUP */
-} SDbMeta;
-typedef struct AppPrm {
- const char* path;
-} AppPrm;
-typedef struct AppCtx {
- SRAMgr* s_mgr;
- const KXMLMgr* x_mgr;
- KDirectory* dir;
-} AppCtx;
-static rc_t AppCtxInit(rc_t rc, AppCtx* ctx) {
- assert(ctx);
- memset(ctx, 0 , sizeof *ctx);
- if (rc) {
- return rc;
- }
- if (rc == 0) {
- rc = SRAMgrMakeUpdate(&ctx->s_mgr, NULL);
- DISP_RC(rc, "while calling SRAMgrMakeUpdate");
- }
- if (rc == 0) {
- rc = KXMLMgrMakeRead(&ctx->x_mgr);
- DISP_RC(rc, "while calling KXMLMgrMakeRead");
- }
- if (rc == 0) {
- rc = KDirectoryNativeDir(&ctx->dir);
- DISP_RC(rc, "while calling KDirectoryNativeDir");
- }
- return rc;
-}
-static rc_t AppCtxDestroy(rc_t rc, AppCtx* ctx) {
- assert(ctx);
- RELEASE(KDirectory, ctx->dir);
- RELEASE(KXMLMgr, ctx->x_mgr);
- RELEASE(SRAMgr, ctx->s_mgr);
- return rc;
-}
-static int CC MetaMemberCmp(const void *item, const BSTNode *n) {
- const char *key = item;
- const MetaMember *a = ( const MetaMember* ) n;
- return strcmp(key, a->member_name);
-}
-static int CC MetaMemberSort(const BSTNode *item, const BSTNode *n) {
- const MetaMember *a = (const MetaMember*) item;
- return MetaMemberCmp(a->member_name, n);
-}
-static rc_t s_KXMLNodeReadAttr(rc_t rc, const KXMLNode* node, const char* name,
- SValue *val, bool required)
-{
- return 0;
-}
-static rc_t ParseSraMetaNode(const KXMLNode* node,
- const char* name, const char* member_name, Meta* meta)
-{
- rc_t rc = 0;
- bool required = member_name;
- rc = s_KXMLNodeReadAttr
- (rc, node, "base_count", &meta->base_count, required);
- rc = s_KXMLNodeReadAttr
- (rc, node, "base_count_bio", &meta->base_count_bio, required);
- rc = s_KXMLNodeReadAttr
- (rc, node, "spot_count", &meta->spot_count, required);
- if (rc == 0) {
- if (meta->base_count.found != meta->base_count_bio.found ||
- meta->base_count.found != meta->spot_count.found)
- {
- rc = RC(rcExe, rcMetadata, rcReading, rcNode, rcNotFound);
- if (member_name) {
- PLOGERR(klogErr, (klogErr, rc, "One of statistics attributes "
- "in '$(name)|$(attr)' is missed",
- "name=%s,attr=%s", name, member_name));
- }
- else {
- PLOGERR(klogErr, (klogErr, rc,
- "One of statistics attributes in '$(name)' is missed",
- "name=%s", name));
- }
- }
- else {
- meta->found = meta->base_count.found;
- }
- }
- return rc;
-}
-static rc_t XmlMetaInitMemser(BSTree* tr,
- const KXMLNode* node, const char* path, const char* member_name)
-{
- rc_t rc = 0;
- MetaMember* member
- = (MetaMember*)BSTreeFind(tr, member_name, MetaMemberCmp);
- if (member) {
- rc = RC(rcExe, rcMetadata, rcReading, rcAttr, rcDuplicate);
- PLOGERR(klogErr, (
- klogErr, rc, "Member/@member_name='$(name)'",
- "name=%s", member_name));
- }
- else {
- member = calloc(1, sizeof(*member));
- if (member == NULL) {
- rc = RC(rcExe,
- rcStorage, rcAllocating, rcMemory, rcExhausted);
- }
- if (rc == 0) {
- member->member_name = strdup(member_name);
- if (member == NULL) {
- rc = RC(rcExe,
- rcStorage, rcAllocating, rcMemory, rcExhausted);
- }
- }
- if (rc == 0) {
- member->meta = calloc(1, sizeof(*member->meta));
- if (member->meta == NULL) {
- rc = RC(rcExe,
- rcStorage, rcAllocating, rcMemory, rcExhausted);
- }
- }
- if (rc == 0) {
- rc = ParseSraMetaNode(node, path, member_name, member->meta);
- }
- if (rc) {
- if (member) {
- FREE(member->member_name);
- FREE(member->meta);
- FREE(member);
- }
- }
- else {
- BSTreeInsert(tr, (BSTNode*)member, MetaMemberSort);
- }
- }
- return rc;
-}
-static rc_t XmlMetaInitMembers(BSTree* tr, const KXMLNode* node) {
- rc_t rc = 0;
- const char path[] = "Member";
- uint32_t count = 0;
- uint32_t idx = 0;
- const KXMLNodeset *members = NULL;
- if (rc == 0) {
- rc = KXMLNodeOpenNodesetRead(node, &members, path);
- DISP_RC(rc, path);
- }
- if (rc == 0) {
- rc = KXMLNodesetCount(members, &count);
- DISP_RC(rc, path);
- }
- for (idx = 0; idx < count && rc == 0; ++idx) {
- const KXMLNode *node = NULL;
- char member_name[256] = "";
- if (rc == 0) {
- rc = KXMLNodesetGetNodeRead(members, &node, idx);
- DISP_RC(rc, path);
- }
- if (rc == 0) {
- size_t size = 0;
- rc = KXMLNodeReadAttrCString(node, "member_name",
- member_name, sizeof member_name, &size);
- if (rc) {
- if (GetRCState(rc) == rcInsufficient) {
- member_name[sizeof member_name - 1] = '\0';
- rc = 0;
- }
- }
- }
- if (rc == 0) {
- rc = XmlMetaInitMemser(tr, node, path, member_name);
- }
- RELEASE(KXMLNode, node);
- }
- RELEASE(KXMLNodeset, members);
- return rc;
-}
-static rc_t XmlMetaInit(rc_t rc, XmlMeta* meta,
- const AppCtx* ctx, const char* xml)
-{
- const KFile* f = NULL;
- const KXMLDoc* doc = NULL;
- const KXMLNodeset* ns = NULL;
- const KXMLNode* node = NULL;
- const char path[] = "/Run";
- memset(meta, 0, sizeof *meta);
- if (rc) {
- return rc;
- }
- if (rc == 0) {
- rc = KDirectoryOpenFileRead(ctx->dir, &f, "%s", xml);
- DISP_RC(rc, xml);
- }
- if (rc == 0) {
- rc = KXMLMgrMakeDocRead(ctx->x_mgr, &doc, f);
- DISP_RC(rc, xml);
- }
- if (rc == 0) {
- rc = KXMLDocOpenNodesetRead(doc, &ns, path);
- DISP_RC(rc, path);
- }
- if (rc == 0) {
- rc = KXMLNodesetGetNodeRead(ns, &node, 0);
- DISP_RC(rc, path);
- }
- if (rc == 0) {
- rc = ParseSraMetaNode(node, path, NULL, &meta->root);
- }
- if (rc == 0 && !meta->root.found) {
- meta->tr = calloc(1, sizeof *meta->tr);
- if (meta->tr == NULL) {
- rc = RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
- }
- else {
- BSTreeInit(meta->tr);
- rc = XmlMetaInitMembers(meta->tr, node);
- }
- }
- RELEASE(KXMLNode, node);
- RELEASE(KXMLNodeset, ns);
- RELEASE(KXMLDoc, doc);
- RELEASE(KFile, f);
- return rc;
-}
-static rc_t XmlMetaDestroy(rc_t rc, const XmlMeta* meta) {
- return rc;
-}
-static int Size(const BSTree* tr) { return 0; }
-static bool MetaEqual(const Meta* lhs, const Meta* rhs) { return false; }
-static bool Equal(const Meta* xml, const BSTree* tr, const char* key)
-{ return false; }
-static bool BSTreeEqual(const BSTree* lhs, const BSTree* rhs) { return false; }
-static rc_t CheckMeta(const XmlMeta* xml, const SDbMeta* db, bool* found) {
- rc_t rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcUnequal);
- assert(xml && db && found);
- *found = false;
- if (xml->root.found) { /* has statistics in root node: no spot-groups */
- if (Size(xml->tr) > 0) { /* should have no Member nodes */
- rc = RC(rcExe, rcXmlDoc, rcReading, rcXmlDoc, rcInvalid);
- }
- else if (db->stats_found) { /* has STATS */
- if (!MetaEqual(&xml->root, db->table)) {
- /* xml: Run != STATS/TABLE */
- rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcUnequal);
- }
- else if (!Equal(&xml->root, db->tr, "default")) {
- /* xml: Run != STATS/SPOT_GROUP/default */
- rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcUnequal);
- }
- else if (Size(db->tr) != 1) {
- /* STATS/SPOT_GROUP should have just default subnode */
- rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcInvalid);
- } else {
- *found = true;
- rc = 0;
- }
- }
- else { /* no STATS: check with meta in root */
- if (!db->root.found) { /* no root meta */
- rc = 0;
- }
- else if (!MetaEqual(&xml->root, &db->root)) {
- rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcUnequal);
- }
- else {
- *found = true;
- rc = 0;
- }
- }
- }
- else { /* sra-stat xml has spotgroups info */
- if (db->root.found) { /* unknown: could it have root meta nodes? */
- rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcInvalid);
- }
- else if (Size(db->tr) > 0) { /* exist STATS/SPOT_GROUP/... */
- if (!BSTreeEqual(xml->tr, db->tr)) {
- rc = RC(rcExe, rcMetadata, rcReading, rcMetadata, rcUnequal);
- }
- /* could add Equal(sum(xml->tr), meta->table) */
- else {
- *found = true;
- rc = 0;
- }
- }
- else { /* no statistics meta */
- rc = 0;
- }
- }
- return rc;
-}
-static rc_t fix_run_stat(const XmlMeta* xml) {
- rc_t rc = 0;
- bool found = false;
- const SRATable *tbl = NULL;
- SDbMeta db;
- memset(&db, 0, sizeof db);
- rc = CheckMeta(xml, &db, &found);
- RELEASE(SRATable, tbl);
- return rc;
-}
-const char UsageDefaultName[] = "fix-run-stat";
-rc_t CC UsageSummary (const char * prog_name) { return 0; }
-rc_t CC Usage ( const Args * args ) { return 0; }
-ver_t CC KAppVersion ( void ) { return 0x01000000; }
-rc_t KMain(int argc, char *argv []) {
- rc_t rc = 0;
- AppCtx ctx;
- XmlMeta meta;
- rc = AppCtxInit(rc, &ctx);
- rc = XmlMetaInit(rc,
- &meta, &ctx, "/home/klymenka/fix-run-stat/SRR331456/SRR331456.xml");
- if (rc == 0) {
- rc = fix_run_stat(&meta);
- }
- rc = XmlMetaDestroy(rc, &meta);
- rc = AppCtxDestroy(rc, &ctx);
- return rc;
-}
diff --git a/tools/fix-run-stat/fix-run-stat.vers b/tools/fix-run-stat/fix-run-stat.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/fix-run-stat/fix-run-stat.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/fuse/Makefile b/tools/fuse/Makefile
index 4dde500..87c0932 100644
--- a/tools/fuse/Makefile
+++ b/tools/fuse/Makefile
@@ -42,12 +42,49 @@ ALL_TOOLS = \
#-------------------------------------------------------------------------------
# outer targets
#
+ifeq (1,$(HAVE_XML2))
+
+ifeq (1,$(HAVE_FUSE))
+
+ifeq (64,$(BITS))
+
all std: vers-includes
@ $(MAKE_CMD) $(TARGDIR)/std
$(ALL_TOOLS): vers-includes
@ $(MAKE_CMD) $(BINDIR)/$@
+else
+
+all std: $(ALL_TOOLS)
+
+$(ALL_TOOLS):
+ @ echo "NOTE - $(@F) cannot be built: it requires a 64-bit system."
+
+endif
+
+else
+
+all std: $(ALL_TOOLS)
+
+$(ALL_TOOLS):
+ @ echo "NOTE - $(@F) cannot be built: " \
+ "it requires 'fuse' library and its development headers."
+
+endif
+
+else
+
+all std: $(ALL_TOOLS)
+
+$(ALL_TOOLS):
+ @ echo "NOTE - $(@F) cannot be built: " \
+ "It requires our internal library 'libkxml'" \
+ "which requires 'libxml2' and its development headers."
+
+endif
+
+
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
diff --git a/tools/fuse/directory.c b/tools/fuse/directory.c
index f309cff..e740928 100644
--- a/tools/fuse/directory.c
+++ b/tools/fuse/directory.c
@@ -53,19 +53,19 @@ typedef struct DirNodeChild_struct {
} DirNodeChild;
static
-int DirNodeChild_Sort(const BSTNode *item, const BSTNode *node)
+int64_t DirNodeChild_Sort(const BSTNode *item, const BSTNode *node)
{
return strcmp(((const DirNodeChild*)item)->child->name, ((const DirNodeChild*)node)->child->name);
}
static
-int DirNodeChild_FindByName(const void *item, const BSTNode *node)
+int64_t DirNodeChild_FindByName(const void *item, const BSTNode *node)
{
return strcmp((const char*)item, ((const DirNodeChild*)node)->child->name);
}
static
-int DirNodeChild_FindBySrc(const void *item, const BSTNode *node)
+int64_t DirNodeChild_FindBySrc(const void *item, const BSTNode *node)
{
return strcmp((const char*)item, ((const DirNodeChild*)node)->name);
}
@@ -142,7 +142,7 @@ rc_t DirectoryNode_Unlock(const DirectoryNode* cself)
static
rc_t DirectoryNode_IsChild(const DirectoryNode* cself, const char* subpath, const FSNode** node,
- int (*finder)(const void *item, const BSTNode *node))
+ int64_t (*finder)(const void *item, const BSTNode *node))
{
DirNodeChild* n = (DirNodeChild*)BSTreeFind(&cself->children, subpath, finder);
if( n != NULL ) {
diff --git a/tools/fuse/remote-cache.c b/tools/fuse/remote-cache.c
index e5bde52..a7692a9 100644
--- a/tools/fuse/remote-cache.c
+++ b/tools/fuse/remote-cache.c
@@ -874,7 +874,8 @@ _RCacheEntryMake (
/*))
// Comparator: common case
((*/
-int CC
+static
+int64_t CC
_RcUrLcMp ( const char * Url1, const char * Url2 )
{
if ( Url1 == NULL || Url2 == NULL ) {
@@ -893,7 +894,8 @@ _RcUrLcMp ( const char * Url1, const char * Url2 )
/*))
// Comparator: we suppose that 'item' is an Url
((*/
-int CC
+static
+int64_t CC
_RcEnTrYcMp ( const void * item, const BSTNode * node )
{
return _RcUrLcMp (
@@ -907,7 +909,8 @@ _RcEnTrYcMp ( const void * item, const BSTNode * node )
/*))
// Sorter for BSTreeInsert
((*/
-int CC
+static
+int64_t CC
_RcNoDeCmP ( const BSTNode * node1, const BSTNode * node2 )
{
return _RcUrLcMp (
@@ -1137,10 +1140,7 @@ RmOutMsg ( " |<-- Cache Entry [%s]\n", Path );
Directory,
& TeeFile,
HttpFile,
- NULL, /* logger */
_HttpBlockSize, /* blocksize */
- 1, /* cluster */
- false, /* report */
Path
);
if ( RCt == 0 ) {
diff --git a/tools/fuse/remote-directory.c b/tools/fuse/remote-directory.c
index aeb1f57..487fafe 100644
--- a/tools/fuse/remote-directory.c
+++ b/tools/fuse/remote-directory.c
@@ -53,7 +53,7 @@ typedef struct DirNodeChild_struct {
static
-int DirNodeChild_FindByName(const void *item, const BSTNode *node)
+int64_t DirNodeChild_FindByName(const void *item, const BSTNode *node)
{
return strcmp((const char*)item, ((const DirNodeChild*)node)->child->name);
}
@@ -119,7 +119,7 @@ rc_t DirectoryNode_Unlock(const DirectoryNode* cself)
static
rc_t DirectoryNode_IsChild(const DirectoryNode* cself, const char* subpath, const FSNode** node,
- int (*finder)(const void *item, const BSTNode *node))
+ int64_t (*finder)(const void *item, const BSTNode *node))
{
DirNodeChild* n = (DirNodeChild*)BSTreeFind(&cself->children, subpath, finder);
if( n != NULL ) {
diff --git a/tools/fuse/remote-fuser.vers b/tools/fuse/remote-fuser.vers
index 8e8299d..35d16fb 100644
--- a/tools/fuse/remote-fuser.vers
+++ b/tools/fuse/remote-fuser.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/fuse/sra-fuser-test.vers b/tools/fuse/sra-fuser-test.vers
index 227cea2..35d16fb 100644
--- a/tools/fuse/sra-fuser-test.vers
+++ b/tools/fuse/sra-fuser-test.vers
@@ -1 +1 @@
-2.0.0
+2.5.7
diff --git a/tools/fuse/sra-fuser-test.vers.h b/tools/fuse/sra-fuser-test.vers.h
index f02d05a..3b1a842 100644
--- a/tools/fuse/sra-fuser-test.vers.h
+++ b/tools/fuse/sra-fuser-test.vers.h
@@ -1 +1 @@
-#define SRA_FUSER_TEST_VERS 0x02000000
+#define SRA_FUSER_TEST_VERS 0x02050007
diff --git a/tools/fuse/sra-fuser.vers b/tools/fuse/sra-fuser.vers
index 197c4d5..35d16fb 100644
--- a/tools/fuse/sra-fuser.vers
+++ b/tools/fuse/sra-fuser.vers
@@ -1 +1 @@
-2.4.0
+2.5.7
diff --git a/tools/fuse/sra-list.c b/tools/fuse/sra-list.c
index 463af47..27c791f 100644
--- a/tools/fuse/sra-list.c
+++ b/tools/fuse/sra-list.c
@@ -275,7 +275,7 @@ typedef struct SRAListNode_FindData_struct {
} SRAListNode_FindData;
static
-int SRAListNode_Cmp(const SRAListNode_FindData* l, const SRAListNode* r)
+int64_t SRAListNode_Cmp(const SRAListNode_FindData* l, const SRAListNode* r)
{
int d = strcmp(l->accession, r->accession);
if( d == 0 && l->path != r->path ) {
@@ -291,13 +291,13 @@ int SRAListNode_Cmp(const SRAListNode_FindData* l, const SRAListNode* r)
}
static
-int SRAListNode_Find(const void* item, const BSTNode* node)
+int64_t SRAListNode_Find(const void* item, const BSTNode* node)
{
return SRAListNode_Cmp((SRAListNode_FindData*)item, (SRAListNode*)node);
}
static
-int SRAListNode_Sort(const BSTNode *item, const BSTNode *node)
+int64_t SRAListNode_Sort(const BSTNode *item, const BSTNode *node)
{
SRAListNode* i = (SRAListNode*)item;
SRAListNode_FindData data;
diff --git a/tools/fuse/sra-makeidx.c b/tools/fuse/sra-makeidx.c
index 72a0baa..5dab079 100644
--- a/tools/fuse/sra-makeidx.c
+++ b/tools/fuse/sra-makeidx.c
@@ -847,13 +847,13 @@ rc_t KMain(int argc, char *argv[])
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_BlockSize].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_BlockSize].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_BlockSize].name, 0, &blksz)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_BlockSize].name, 0, (const void **)&blksz)) != 0 ) {
errmsg = MainArgs[eopt_BlockSize].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_Accession].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = MainArgs[eopt_Accession].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_Accession].name, 0, &g_accession)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args, MainArgs[eopt_Accession].name, 0, (const void **)&g_accession)) != 0 ) {
errmsg = MainArgs[eopt_Accession].name;
} else if( (rc = ArgsOptionCount(args, MainArgs[eopt_DumpIndex].name, &dump)) != 0 ) {
@@ -880,7 +880,7 @@ rc_t KMain(int argc, char *argv[])
}
g_file_block_sz = val;
}
- if( (rc = ArgsParamValue(args, 0, &table_dir)) != 0 ) {
+ if( (rc = ArgsParamValue(args, 0, (const void **)&table_dir)) != 0 ) {
errmsg = "table";
break;
}
diff --git a/tools/fuse/sra-makeidx.vers b/tools/fuse/sra-makeidx.vers
index 197c455..35d16fb 100644
--- a/tools/fuse/sra-makeidx.vers
+++ b/tools/fuse/sra-makeidx.vers
@@ -1 +1 @@
-2.4.21
+2.5.7
diff --git a/tools/fuse/tar-list.c b/tools/fuse/tar-list.c
index 946cc28..a963f05 100644
--- a/tools/fuse/tar-list.c
+++ b/tools/fuse/tar-list.c
@@ -481,7 +481,7 @@ rc_t TarFileList_Size(const TarFileList* cself, uint64_t* file_sz)
}
static
-int TarFileList_Sort(const BSTNode *item, const BSTNode *node)
+int64_t TarFileList_Sort(const BSTNode *item, const BSTNode *node)
{
const TarEntry* i = (const TarEntry*)item;
const TarEntry* n = (const TarEntry*)node;
@@ -528,7 +528,7 @@ rc_t TarFileList_Open(const TarFileList* cself)
}
static
-int TarFileList_Find(const void *item, const BSTNode *node)
+int64_t TarFileList_Find(const void *item, const BSTNode *node)
{
uint64_t pos = *(uint64_t*)item;
const TarEntry* n = (const TarEntry*)node;
diff --git a/tools/fuse/unix/sra-fuser-sys.c b/tools/fuse/unix/sra-fuser-sys.c
index d058be6..1cf3a18 100644
--- a/tools/fuse/unix/sra-fuser-sys.c
+++ b/tools/fuse/unix/sra-fuser-sys.c
@@ -363,7 +363,7 @@ int UX_FUSE_ftruncate(const char *path, off_t o, struct fuse_file_info *fi)
}
static
-void CoreUsage(int fd, const char *progName, bool showHelp, bool showVersion, bool fail)
+void CoreUsage(int fd, const char *progName, bool showHelp, bool showVersion, bool fail, bool forceShowHelp)
{
/* used only for FUSE built-in help and version printing */
struct fuse_operations ops;
@@ -382,7 +382,7 @@ void CoreUsage(int fd, const char *progName, bool showHelp, bool showVersion, bo
p = progName;
}
UsageSummary(p);
- if( !fail ) {
+ if( !fail || forceShowHelp ) {
fuse_opt_add_arg(&args, "-ho");
KOutMsg("\n"
" -x|--xml-dir <path> XML file with virtual directory structure\n"
@@ -451,7 +451,7 @@ rc_t CC KMain(int argc, char *argv[])
int i;
rc_t rc;
- bool showHelp = argc < 2, showVersion = false, unmount = false, foreground = false;
+ bool missedArgs = argc < 2, showHelp = false, showVersion = false, unmount = false, foreground = false;
const char* mount_point = NULL, *xml_path = NULL, *log_file = NULL;
const char* sra_cache = NULL, *xml_root = ".";
char** fargs = (char**)calloc(argc, sizeof(char*));
@@ -479,7 +479,7 @@ rc_t CC KMain(int argc, char *argv[])
if( i++ == argc - 1 ) {
rc = RC(rcExe, rcArgv, rcValidating, rcParam, rcInsufficient);
LOGERR(klogErr, rc, "XML validation setting value");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
} else if( !strcmp(argv[i], "ignore") ) {
xml_validate = eXML_NoFail;
} else if( !strcmp(argv[i], "nocheck") ) {
@@ -487,7 +487,7 @@ rc_t CC KMain(int argc, char *argv[])
} else {
rc = RC(rcExe, rcArgv, rcValidating, rcParam, rcUnrecognized);
PLOGERR(klogErr, (klogErr, rc, "XML validation setting value '$(lvl)'", PLOG_S(lvl), argv[i]));
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
} else if(!strcmp(argv[i], "-ds") || !strcmp(argv[i], "--SRA-check")) {
sra_sync = AsciiToU32(argv[++i], NULL, NULL);
@@ -499,20 +499,20 @@ rc_t CC KMain(int argc, char *argv[])
if( i == argc - 1 ) {
rc = RC(rcExe, rcArgv, rcValidating, rcParam, rcInsufficient);
LOGERR(klogErr, rc, "missing log level");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
} else if( (rc = LogLevelSet(argv[++i])) != 0 ) {
PLOGERR(klogErr, (klogErr, rc, "log level $(lvl)", PLOG_S(lvl), argv[i]));
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
} else if(!strcmp(argv[i], "-+") || !strcmp (argv[i], "--debug")) {
#ifdef _DEBUGGING
if( i == argc - 1 ) {
rc = RC(rcExe, rcArgv, rcValidating, rcParam, rcInsufficient);
LOGERR(klogErr, rc, "missing debug level");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
} else if( (rc = KDbgSetString(argv[++i])) != 0 ) {
PLOGERR(klogErr, (klogErr, rc, "debug level $(lvl)", PLOG_S(lvl), argv[i]));
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
#else
i++;
@@ -537,16 +537,19 @@ rc_t CC KMain(int argc, char *argv[])
}
}
}
+ if( missedArgs ) {
+ CoreUsage(log_fd, argv[0], missedArgs, showVersion, true, true);
+ }
if( showHelp || showVersion ) {
- CoreUsage(log_fd, argv[0], showHelp, showVersion, false);
+ CoreUsage(log_fd, argv[0], showHelp, showVersion, false, false);
}
if( (rc = LogFile_Init(log_file, log_sync, foreground, &log_fd)) != 0 ) {
LOGERR(klogErr, rc, log_file ? log_file : "no log");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
if( mount_point == NULL ) {
LOGERR(klogErr, RC(rcExe, rcArgv, rcValidating, rcParam, rcInsufficient), "mountpoint");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
if( unmount ) {
fuse_unmount(mount_point);
@@ -554,15 +557,15 @@ rc_t CC KMain(int argc, char *argv[])
}
if( xml_path == NULL ) {
LOGERR(klogErr, RC(rcExe, rcArgv, rcValidating, rcParam, rcInsufficient), "virtual directory XML");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
if( i != argc ) {
LOGERR(klogErr, RC(rcExe, rcArgv, rcValidating, rcParam, rcExcessive), argv[i]);
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
if( stat(mount_point, &g_mount_point_stat) < 0 ) {
PLOGMSG(klogErr, (klogErr, "$(p): $(e)", PLOG_2(PLOG_S(p),PLOG_S(e)), mount_point, strerror(errno)));
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
g_mount_point_stat.st_dev = 0;
g_mount_point_stat.st_ino = 0;
@@ -572,7 +575,7 @@ rc_t CC KMain(int argc, char *argv[])
if( stat(xml_path, &g_dflt_file_stat) < 0 ) {
PLOGMSG(klogErr, (klogErr, "$(p): $(e)", PLOG_2(PLOG_S(p),PLOG_S(e)), xml_path, strerror(errno)));
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
g_dflt_file_stat.st_dev = 0;
g_dflt_file_stat.st_ino = 0;
@@ -585,7 +588,7 @@ rc_t CC KMain(int argc, char *argv[])
if( (rc = Initialize(sra_sync, xml_path, xml_sync, sra_cache, xml_root, xml_validate)) != 0 ) {
LOGERR(klogErr, rc, "at initialization");
- CoreUsage(log_fd, argv[0], true, false, true);
+ CoreUsage(log_fd, argv[0], true, false, true, false);
}
DEBUG_MSG(8, ("Mount point set to '%s'\n", mount_point));
diff --git a/tools/meta-sync/Makefile b/tools/general-loader/Makefile
similarity index 66%
rename from tools/meta-sync/Makefile
rename to tools/general-loader/Makefile
index 372d4b6..e1ac4de 100644
--- a/tools/meta-sync/Makefile
+++ b/tools/general-loader/Makefile
@@ -1,4 +1,4 @@
-# ==============================================================================
+# ===========================================================================
#
# PUBLIC DOMAIN NOTICE
# National Center for Biotechnology Information
@@ -26,42 +26,44 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/meta-sync
+
+MODULE = tools/general-loader
include $(TOP)/build/Makefile.env
+ALL_LIBS = \
+ general-writer
+
INT_TOOLS = \
+ gw-dumper
EXT_TOOLS = \
- meta-sync
+ general-loader
-ALL_TOOLS = \
+ALL_TOOLS = \
$(INT_TOOLS) \
$(EXT_TOOLS)
-#------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
# outer targets
#
all std: vers-includes
- @ $(MAKE_CMD) $(TARGDIR)/$@
+ @ $(MAKE_CMD) $(TARGDIR)/std
+
+$(INT_TOOLS): vers-includes
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
-$(ALL_TOOLS): vers-includes
+$(EXT_TOOLS): vers-includes
@ $(MAKE_CMD) $(BINDIR)/$@
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
-# all
-#
-$(TARGDIR)/all: \
- $(addprefix $(BINDIR)/,$(ALL_TOOLS))
-
-.PHONY: $(TARGDIR)/all
-
-#-------------------------------------------------------------------------------
# std
#
$(TARGDIR)/std: \
+ $(addprefix $(LIBDIR)/,$(ALL_LIBS)) \
+ $(addprefix $(TEST_BINDIR)/,$(INT_TOOLS)) \
$(addprefix $(BINDIR)/,$(EXT_TOOLS))
.PHONY: $(TARGDIR)/std
@@ -81,34 +83,46 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# meta-sync
+# general-writer
+#
+$(LIBDIR)/general-writer: $(LIBDIR)/$(LPFX)general-writer.$(LIBX)
+
+GEN_WRITER_SRC = \
+ general-writer \
+ utf8-like-int-codec
+
+GEN_WRITER_OBJ = \
+ $(addsuffix .$(LOBX),$(GEN_WRITER_SRC))
+
+$(LIBDIR)/$(LPFX)general-writer.$(LIBX): $(GEN_WRITER_OBJ)
+ $(LD) --slib --vers $(SRCDIR) -o $@ $^
+
+#-------------------------------------------------------------------------------
+# gw-dumper
#
-META_SYNC_SRC = \
- meta-sync \
-
-META_SYNC_OBJ = \
- $(addsuffix .$(OBJX),$(META_SYNC_SRC))
-
-META_SYNC_LIB = \
- -lkapp \
- -dvxf \
- -lsraschema \
- -lvdb \
- -lkdb \
- -ssrapath \
- -lvfs \
- -lkxml \
- -lkfg \
- -lkfs \
- -lrdbms \
- -lklib \
- -X/export/home/sybase/clients-mssql/current/lib -lct\
- -lm
-
-meta-sync.vers.h: meta-sync.vers
- $(TOP)/build/make-vers-inc.sh META_SYNC_VERS $^ > $@
-
-$(BINDIR)/meta-sync: $(META_SYNC_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(META_SYNC_LIB)
+
+$(TEST_BINDIR)/gw-dumper: $(SRCDIR)/gw-dumper.cpp $(SRCDIR)/utf8-like-int-codec.c
+ c++ $(DBG) -o $@ $^ -I$(SRCDIR) -I$(VDB_INCDIR)/os/$(OS) -I$(VDB_INCDIR)
+
#-------------------------------------------------------------------------------
+# general-loader
+#
+GEN_LOAD_SRC = \
+ general-loader \
+ protocol-parser \
+ database-loader \
+ utf8-like-int-codec \
+ main \
+
+GEN_LOAD_OBJ = \
+ $(addsuffix .$(OBJX),$(GEN_LOAD_SRC))
+
+GEN_LOAD_LIBS = \
+ -sncbi-wvdb-static \
+ -sload \
+ -skapp \
+
+$(BINDIR)/general-loader: $(GEN_LOAD_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(GEN_LOAD_LIBS)
+
diff --git a/tools/general-loader/database-loader.cpp b/tools/general-loader/database-loader.cpp
new file mode 100644
index 0000000..a1ce2ed
--- /dev/null
+++ b/tools/general-loader/database-loader.cpp
@@ -0,0 +1,895 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "general-loader.hpp"
+
+#include <klib/rc.h>
+#include <klib/log.h>
+#include <klib/time.h>
+
+#include <kdb/meta.h>
+#include <kdb/table.h>
+
+#include <vdb/manager.h>
+#include <vdb/schema.h>
+#include <vdb/database.h>
+#include <vdb/cursor.h>
+#include <vdb/table.h>
+#include <vdb/vdb-priv.h>
+
+#include <kapp/loader-meta.h>
+#include <kapp/main.h>
+
+#include <algorithm>
+
+using namespace std;
+
+///////////// GeneralLoader::DatabaseLoader
+
+GeneralLoader :: DatabaseLoader :: DatabaseLoader ( const std::string& p_programName,
+ const Paths& p_includePaths,
+ const Paths& p_schemas,
+ const std::string& p_dbNameOverride )
+: m_includePaths ( p_includePaths ),
+ m_schemas ( p_schemas ),
+ m_programName ( p_programName ),
+ m_databaseName ( p_dbNameOverride ), // if specified, overrides the database path coming in from the stream
+ m_softwareVersion ( 0 ),
+ m_mgr ( 0 ),
+ m_schema ( 0 ),
+ m_databaseNameOverridden ( ! m_databaseName.empty() )
+{
+ m_databases . insert ( Databases :: value_type ( 0, (VDatabase*)0 ) ); // reserve root database
+}
+
+GeneralLoader :: DatabaseLoader :: ~DatabaseLoader ()
+{
+ m_tables . clear();
+ m_columns . clear ();
+
+ for ( Cursors::iterator it = m_cursors . begin(); it != m_cursors . end(); ++it )
+ {
+ VCursorRelease ( *it );
+ }
+
+ for ( Databases::iterator it = m_databases . begin(); it != m_databases . end(); ++it )
+ {
+ VDatabaseRelease ( it -> second );
+ }
+
+ if ( m_schema != 0 )
+ {
+ VSchemaRelease ( m_schema );
+ m_schema = 0;
+ }
+
+ if ( m_mgr != 0 )
+ {
+ VDBManagerRelease ( m_mgr );
+ m_mgr = 0;
+ }
+}
+
+const GeneralLoader :: DatabaseLoader :: Column*
+GeneralLoader :: DatabaseLoader :: GetColumn ( uint32_t p_columnId ) const
+{
+ Columns::const_iterator curIt = m_columns . find ( p_columnId );
+ if ( curIt != m_columns . end () )
+ {
+ return & curIt -> second;
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: UseSchema ( const string& p_file, const string& p_name )
+{
+ pLogMsg ( klogDebug, "database-loader: schema file '$(s1)', name '$(s2)'", "s1=%s,s2=%s",
+ p_file . c_str (), p_name . c_str () );
+
+ rc_t rc = VDBManagerMakeUpdate ( & m_mgr, NULL );
+ if ( rc == 0 )
+ {
+ for ( Paths::const_iterator it = m_includePaths . begin(); it != m_includePaths . end(); ++it )
+ {
+ rc = VDBManagerAddSchemaIncludePath ( m_mgr, "%s", it -> c_str() );
+ if ( rc == 0 )
+ {
+ pLogMsg ( klogDebug,
+ "database-loader: Added schema include path '$(s)'",
+ "s=%s",
+ it -> c_str() );
+ }
+ else if ( GetRCObject ( rc ) == (RCObject)rcPath )
+ {
+ pLogMsg ( klogWarn,
+ "database-loader: Schema include path not found: '$(s)'",
+ "s=%s",
+ it -> c_str() );
+ rc = 0;
+ }
+ else
+ {
+ return rc;
+ }
+ }
+
+ rc = VDBManagerMakeSchema ( m_mgr, & m_schema );
+ if ( rc == 0 )
+ {
+ bool found = false;
+ if ( ! p_file . empty () )
+ {
+ rc = VSchemaParseFile ( m_schema, "%s", p_file . c_str () );
+ if ( rc == 0 )
+ {
+ pLogMsg ( klogDebug,
+ "database-loader: Added schema file '$(s)'",
+ "s=%s",
+ p_file . c_str () );
+ found = true;
+ }
+ else if ( GetRCObject ( rc ) == (RCObject)rcPath && GetRCState ( rc ) == rcNotFound )
+ {
+ pLogMsg ( klogWarn,
+ "database-loader: Schema file not found: '$(s)'",
+ "s=%s",
+ p_file . c_str () );
+ rc = 0;
+ }
+ }
+ // if p_file is empty, there should be other schema files specified externally
+ // through the command line options, in m_schemas
+
+ if ( rc == 0 )
+ {
+ for ( Paths::const_iterator it = m_schemas. begin(); it != m_schemas . end(); ++it )
+ {
+ rc = VSchemaParseFile ( m_schema, "%s", it -> c_str() );
+ if ( rc == 0 )
+ {
+ pLogMsg ( klogDebug,
+ "database-loader: Added schema file '$(s)'",
+ "s=%s",
+ it -> c_str() );
+ found = true;
+ }
+ else if ( GetRCObject ( rc ) == (RCObject)rcPath && GetRCState ( rc ) == rcNotFound )
+ {
+ pLogMsg ( klogWarn,
+ "database-loader: Schema file not found: '$(s)'",
+ "s=%s",
+ it -> c_str() );
+ rc = 0;
+ }
+ }
+ }
+
+ if ( found )
+ {
+ m_schemaName = p_name;
+ }
+ else
+ {
+ rc = RC ( rcVDB, rcMgr, rcCreating, rcSchema, rcNotFound );
+ }
+ }
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: RemotePath ( const string& p_path )
+{
+ if ( m_databaseNameOverridden )
+ {
+ pLogMsg ( klogWarn,
+ "database-loader: remote path '$(s1)' ignored, overridden to '$(s2)'",
+ "s1=%s,s2=%s",
+ p_path . c_str (), m_databaseName . c_str () );
+ }
+ else
+ {
+ pLogMsg ( klogDebug, "database-loader: remote path '$(s1)'", "s1=%s", p_path . c_str () );
+ m_databaseName = p_path;
+ }
+ return 0;
+}
+
+static
+void
+check_vers_component ( const char * vers, const char * end, long num, unsigned long max, char term )
+{
+ if ( vers == end )
+ throw "bad version";
+ if ( * end != 0 && * end != term )
+ throw "bad version";
+ if ( num < 0 || (unsigned long)num > max )
+ throw "bad version";
+}
+
+static
+ver_t
+string2ver_t ( const char * vers )
+{
+ ver_t ret = 0;
+ char * end;
+ long num = strtol ( vers, & end, 10 );
+ check_vers_component ( vers, end, num, 255, '.' );
+ if ( * end == '.' )
+ {
+ ret = num << 24;
+ vers = end + 1;
+ num = strtol ( vers, & end, 10 );
+ check_vers_component ( vers, end, num, 255, '.' );
+ if ( * end == '.' )
+ {
+ ret |= num << 16;
+ vers = end + 1;
+ num = strtol ( vers, & end, 10 );
+ check_vers_component ( vers, end, num, 0xFFFF, 0 );
+ ret |= num;
+ }
+ }
+ return ret;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: SoftwareName ( const string& p_name, const string& p_version )
+{
+ pLogMsg ( klogDebug, "database-loader: SoftwareName '$(n)', version '$(v)'", "n=%s,v=%s", p_name . c_str(), p_version . c_str() );
+ try
+ {
+ m_softwareVersion = string2ver_t ( p_version.c_str() );
+ }
+ catch (...)
+ {
+ return RC ( rcExe, rcDatabase, rcCreating, rcMessage, rcBadVersion );
+ }
+ m_softwareName = p_name;
+ return 0;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: NewTable ( uint32_t p_tableId, const string& p_tableName )
+{
+ return AddMbrTbl ( p_tableId, 0, p_tableName, p_tableName, kcmCreate | kcmMD5 );
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: NewColumn ( uint32_t p_columnId, uint32_t p_tableId, uint32_t p_elemBits, uint8_t p_flagBits, const string& p_columnName )
+{
+ pLogMsg ( klogDebug, "database-loader: adding column '$(c)'", "c=%s", p_columnName . c_str() );
+
+ rc_t rc = 0;
+ Tables::const_iterator table = m_tables . find ( p_tableId );
+ if ( table != m_tables . end() )
+ {
+ if ( m_columns . find ( p_columnId ) == m_columns . end () )
+ {
+ uint32_t cursor_idx = table -> second . cursorIdx;
+ uint32_t column_idx;
+ rc = VCursorAddColumn ( m_cursors [ cursor_idx ],
+ & column_idx,
+ "%s",
+ p_columnName . c_str() );
+ if ( rc == 0 )
+ {
+ Column col;
+ col . name = p_columnName;
+ col . tableId = p_tableId;
+ col . cursorIdx = cursor_idx;
+ col . columnIdx = column_idx;
+ col . elemBits = p_elemBits;
+ col . flagBits = p_flagBits;
+ m_columns [ p_columnId ] = col;
+ pLogMsg ( klogDebug,
+ "database-loader: tableId = $(t), added column '$(c)', columnIdx = $(i1), elemBits = $(i2), flagBits = $(i3)",
+ "t=%u,c=%s,i1=%u,i2=%u,i3=%u",
+ p_tableId, p_columnName . c_str(), col.columnIdx, col.elemBits, col.flagBits );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcExists );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcTable, rcInvalid );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: AddMbrDB ( uint32_t p_objId, uint32_t p_parentId, const std :: string &p_mbrName, const std :: string &p_dbName, uint8_t p_createMode )
+{
+ pLogMsg ( klogDebug,
+ "database-loader: adding database id=$(i) parent=$(p) mbrName='$(m)' dbName='$(n)' mode=$(d)",
+ "m=%s,n=%s,i=%u,p=%u,d=%u",
+ p_objId, p_parentId, p_mbrName . c_str(), p_dbName . c_str (), ( unsigned int ) p_createMode );
+
+ rc_t rc = MakeDatabase ( p_parentId );
+ if ( rc == 0 )
+ {
+ Databases :: const_iterator dad = m_databases . find ( p_parentId );
+ assert ( dad != m_databases . end () );
+
+ if ( m_databases . find ( p_objId ) == m_databases . end () )
+ {
+ VDatabase *db;
+ rc = VDatabaseCreateDB ( dad -> second, & db, p_mbrName . c_str(), p_createMode, "%s", p_dbName . c_str () );
+ if ( rc == 0 )
+ {
+ m_databases . insert ( Databases :: value_type ( p_objId, db ) );
+ m_dbParents . insert ( DatabaseToParent :: value_type ( p_objId, p_parentId ) );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcDatabase, rcExists );
+ }
+ }
+
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: AddMbrTbl ( uint32_t p_tblId, uint32_t p_dbId, const std :: string &p_mbrName, const std :: string &p_tblName, uint8_t p_createMode )
+{
+ pLogMsg ( klogDebug,
+ "database-loader: adding table id=$(i) parent=$(p) mbrName='$(m)' dbName='$(n)' mode=$(d)",
+ "m=%s,n=%s,i=%u,p=%u,d=%u",
+ p_mbrName . c_str(), p_tblName . c_str (), p_tblId, p_dbId, ( unsigned int ) p_createMode );
+
+ rc_t rc = 0;
+ if ( m_tables . find ( p_tblId ) == m_tables . end() )
+ {
+ VTable* table;
+ rc = MakeDatabase ( p_dbId );
+ if ( rc == 0 )
+ {
+ Databases::iterator it = m_databases . find ( p_dbId );
+ if ( it != m_databases . end() )
+ {
+ rc = VDatabaseCreateTable ( it -> second , & table, p_mbrName . c_str (), p_createMode, "%s", p_tblName . c_str ());
+ if ( rc == 0 )
+ {
+ VCursor* cursor;
+ rc = VTableCreateCursorWrite ( table, & cursor, kcmInsert );
+ if ( rc == 0 )
+ {
+ m_cursors . push_back ( cursor );
+ Table t;
+ t . name = p_tblName;
+ t . databaseId = p_dbId;
+ t . cursorIdx = ( uint32_t ) m_cursors . size() - 1;
+ m_tables [ p_tblId ] = t;
+ }
+ rc_t rc2 = VTableRelease ( table );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcDatabase, rcNotFound );
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcTable, rcExists );
+ }
+
+ return rc;
+}
+
+static
+rc_t WriteMetadata ( KMetadata* p_meta, const string& p_metadata_node, const string& p_value )
+{
+ KMDataNode* node;
+ rc_t rc = KMetadataOpenNodeUpdate ( p_meta, & node, p_metadata_node . c_str () );
+ if ( rc == 0 )
+ {
+ rc = KMDataNodeWrite ( node, p_value . c_str (), p_value . size () );
+
+ rc_t rc2 = KMDataNodeRelease ( node );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: DBMetadataNode ( uint32_t p_objId, const string& p_metadata_node, const string& p_value )
+{
+ pLogMsg ( klogDebug,
+ "database-loader: adding metadata node '$(n)=$(v)' to database $(i)",
+ "n=%s,v=%s,i=%u",
+ p_metadata_node . c_str(), p_value.c_str(), p_objId );
+
+ rc_t rc = 0;
+ Databases::iterator it = m_databases . find ( p_objId );
+ if ( it != m_databases . end() )
+ {
+ struct KMetadata* meta;
+ rc = VDatabaseOpenMetadataUpdate ( it -> second, & meta );
+ if ( rc == 0 )
+ {
+ rc = WriteMetadata ( meta,p_metadata_node, p_value );
+ rc_t rc2 = KMetadataRelease ( meta );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcDatabase, rcNotFound );
+ }
+
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: TblMetadataNode ( uint32_t p_objId, const string& p_metadata_node, const string& p_value )
+{
+ pLogMsg ( klogDebug,
+ "database-loader: adding metadata node '$(n)=$(v)' to table $(i)",
+ "n=%s,v=%s,i=%u",
+ p_metadata_node . c_str(), p_value.c_str(), p_objId );
+
+ rc_t rc = 0;
+ Tables::iterator it = m_tables . find ( p_objId );
+ if ( it != m_tables . end() )
+ {
+ struct VTable* tbl;
+ assert ( m_cursors [ it -> second . cursorIdx ] );
+ rc = VCursorOpenParentUpdate ( m_cursors [ it -> second . cursorIdx ], &tbl );
+ if ( rc == 0 )
+ {
+ struct KMetadata* meta;
+ rc = VTableOpenMetadataUpdate ( tbl, & meta );
+ if ( rc == 0 )
+ {
+ rc = WriteMetadata ( meta,p_metadata_node, p_value );
+ rc_t rc2 = KMetadataRelease ( meta );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ rc_t rc2 = VTableRelease ( tbl );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcTable, rcNotFound );
+ }
+
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: ColMetadataNode ( uint32_t p_objId, const string& p_metadata_node, const string& p_value )
+{
+ pLogMsg ( klogDebug,
+ "database-loader: adding metadata node '$(n)=$(v)' to column $(i)",
+ "n=%s,v=%s,i=%u",
+ p_metadata_node . c_str(), p_value.c_str(), p_objId );
+
+ rc_t rc = 0;
+ Columns::iterator it = m_columns . find ( p_objId );
+ if ( it != m_columns . end() )
+ {
+ it -> second . metadata [ p_metadata_node ] = p_value;
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: SaveColumnMetadata ( const Column& p_col )
+{
+ if ( p_col . metadata . size () == 0 )
+ {
+ return 0;
+ }
+
+ assert ( m_tables . find ( p_col . tableId ) != m_tables.end() );
+ const Table& t = m_tables [ p_col . tableId ];
+
+ assert ( m_databases . find ( t . databaseId ) != m_databases.end() );
+ assert ( m_databases . find ( t . databaseId ) -> second != 0 );
+
+ VTable* table;
+ rc_t rc = VDatabaseOpenTableUpdate ( m_databases [ t . databaseId ], & table, t . name . c_str () );
+ if ( rc == 0 )
+ {
+ KTable* ktbl;
+ rc = VTableOpenKTableUpdate ( table, & ktbl );
+ if ( rc == 0 )
+ {
+ KColumn* col;
+ rc = KTableOpenColumnUpdate ( ktbl, & col, p_col . name . c_str () );
+ if ( rc == 0 )
+ {
+ KMetadata *meta;
+ rc = KColumnOpenMetadataUpdate ( col, &meta );
+ if ( rc == 0 )
+ {
+ for ( Column :: Metadata :: const_iterator it = p_col . metadata . begin(); it != p_col . metadata . end(); ++ it )
+ {
+ rc = WriteMetadata ( meta, it -> first . c_str (), it -> second . c_str () );
+ if ( rc != 0 )
+ {
+ break;
+ }
+ }
+ rc_t rc2 = KMetadataRelease ( meta );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ rc_t rc2 = KColumnRelease ( col );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ rc_t rc2 = KTableRelease ( ktbl );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ rc_t rc2 = VTableRelease ( table );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: CursorWrite ( const struct Column& p_col, const void* p_data, size_t p_size )
+{
+ return VCursorWrite ( m_cursors [ p_col . cursorIdx ],
+ p_col . columnIdx,
+ p_col . elemBits,
+ p_data,
+ 0,
+ p_size );
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: CursorDefault ( const struct Column& p_col, const void* p_data, size_t p_size )
+{
+ return VCursorDefault ( m_cursors [ p_col . cursorIdx ],
+ p_col . columnIdx,
+ p_col . elemBits,
+ p_data,
+ 0,
+ p_size );
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: CellData ( uint32_t p_columnId, const void* p_data, size_t p_elemCount )
+{
+ rc_t rc = 0;
+ Columns::const_iterator curIt = m_columns . find ( p_columnId );
+ if ( curIt != m_columns . end () )
+ {
+ const Column& col = curIt -> second;
+ pLogMsg ( klogDebug,
+ "database-loader: columnIdx = $(i), elem size=$(s) bits, elem count=$(c)",
+ "i=%u,s=%u,c=%u",
+ col . columnIdx, col . elemBits, p_elemCount );
+ rc = CursorWrite ( col, p_data, p_elemCount );
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: CellDefault ( uint32_t p_columnId, const void* p_data, size_t p_elemCount )
+{ //TODO: this and Handle_CellData are almost identical - refactor
+ rc_t rc = 0;
+ Columns::const_iterator curIt = m_columns . find ( p_columnId );
+ if ( curIt != m_columns . end () )
+ {
+ const Column& col = curIt -> second;
+ pLogMsg ( klogDebug,
+ "database-loader: columnIdx = $(i), elem size=$(s) bits, elem count=$(c)",
+ "i=%u,s=%u,c=%u",
+ col . columnIdx, col . elemBits, p_elemCount );
+ rc = CursorDefault ( col, p_data, p_elemCount );
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: MakeDatabase( uint32_t p_id )
+{
+ rc_t rc = 0;
+
+ Databases::iterator it = m_databases . find ( p_id );
+ if ( it != m_databases . end() )
+ {
+ VDatabase*& db = it -> second;
+ if ( db == 0 ) // only create once
+ {
+ rc = VDBManagerCreateDB ( m_mgr,
+ & db,
+ m_schema,
+ m_schemaName . c_str (),
+ kcmInit + kcmMD5,
+ "%s",
+ m_databaseName . c_str () );
+ if ( rc == 0 && p_id == 0 )
+ { // populate the root database's metadata
+ struct KMetadata* meta;
+ rc = VDatabaseOpenMetadataUpdate ( db, &meta );
+ if ( rc == 0 )
+ {
+ KMDataNode *node;
+ rc = KMetadataOpenNodeUpdate ( meta, &node, "/" );
+
+ if (rc == 0)
+ {
+ rc = KLoaderMeta_WriteWithVersion ( node, m_programName.c_str(), __DATE__, KAppVersion(), m_softwareName.c_str(), m_softwareVersion );
+ KMDataNodeRelease(node);
+ }
+
+ rc_t rc2 = KMetadataRelease ( meta );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcDatabase, rcNotFound );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: OpenStream ()
+{
+ pLogMsg ( klogDebug,
+ "database-loader: Database created, schema spec='$(s)', database='$(d)'",
+ "s=%s,d=%s",
+ m_schemaName . c_str (), m_databaseName . c_str () );
+
+ rc_t rc = MakeDatabase ( 0 );
+ if ( rc == 0 )
+ {
+ for ( Cursors::iterator it = m_cursors . begin(); it != m_cursors . end(); ++it )
+ {
+ rc_t rc = VCursorOpen ( *it );
+ if ( rc != 0 )
+ {
+ return rc;
+ }
+ rc = VCursorOpenRow ( *it );
+ if ( rc != 0 )
+ {
+ break;
+ }
+ }
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: CloseStream ()
+{
+ rc_t rc = 0;
+ rc_t rc2 = 0;
+
+ for ( Cursors::iterator it = m_cursors . begin(); it != m_cursors . end(); ++it )
+ {
+ rc = VCursorCloseRow ( *it );
+ if ( rc == 0 )
+ {
+ rc = VCursorCommit ( *it );
+ if ( rc == 0 )
+ {
+ struct VTable* table;
+ rc = VCursorOpenParentUpdate ( *it, &table );
+ if ( rc == 0 )
+ {
+ rc = VCursorRelease ( *it );
+ if ( rc == 0 )
+ {
+ rc = VTableReindex ( table );
+ }
+ }
+ rc2 = VTableRelease ( table );
+ if ( rc == 0 )
+ {
+ rc = rc2;
+ }
+ }
+ }
+ if ( rc != 0 )
+ {
+ break;
+ }
+ }
+ m_cursors . clear ();
+
+ if ( rc == 0 )
+ { // save column-level metadata collected from ColMetadata events
+ for ( Columns::iterator it = m_columns. begin(); it != m_columns. end(); ++it )
+ {
+ rc = SaveColumnMetadata ( it -> second );
+ if ( rc != 0 )
+ {
+ break;
+ }
+ }
+ }
+
+ for ( Databases::iterator it = m_databases . begin(); it != m_databases . end(); ++it )
+ {
+ VDatabaseRelease ( it -> second );
+ }
+ m_databases . clear();
+
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: NextRow ( uint32_t p_tableId )
+{
+ rc_t rc = 0;
+ Tables::const_iterator table = m_tables . find ( p_tableId );
+ if ( table != m_tables . end() )
+ {
+ VCursor * cursor = m_cursors [ table -> second . cursorIdx ];
+ rc = VCursorCommitRow ( cursor );
+ if ( rc == 0 )
+ {
+ rc = VCursorCloseRow ( cursor );
+ if ( rc == 0 )
+ {
+ rc = VCursorOpenRow ( cursor );
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcTable, rcNotFound );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: MoveAhead ( uint32_t p_tableId, uint64_t p_count )
+{
+ rc_t rc = 0;
+ Tables::const_iterator table = m_tables . find ( p_tableId );
+ if ( table != m_tables . end() )
+ {
+ VCursor * cursor = m_cursors [ table -> second . cursorIdx ];
+ for ( uint64_t i = 0; i < p_count; ++i )
+ { // for now, simulate proper handling (this will commit the current row and insert count-1 empty rows)
+ rc = VCursorCommitRow ( cursor );
+ if ( rc != 0 )
+ {
+ break;
+ }
+ rc = VCursorCloseRow ( cursor );
+ if ( rc != 0 )
+ {
+ break;
+ }
+ rc = VCursorOpenRow ( cursor );
+ if ( rc != 0 )
+ {
+ break;
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcTable, rcNotFound );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: ErrorMessage ( const string & p_text )
+{
+ pLogMsg ( klogErr, "general-loader: error \"$(t)\"", "t=%s", p_text . c_str () );
+ return RC ( rcExe, rcFile, rcReading, rcError, rcExists );
+}
+
+rc_t
+GeneralLoader :: DatabaseLoader :: LogMessage ( const string & p_text )
+{
+#pragma message ( "need to pass the app-name from front end" )
+ pLogMsg ( klogInfo, "general-loader: log \"$(t)\"", "t=%s", p_text . c_str () );
+ return 0;
+}
+
+
+/*
+<Log>
+ <status app="sra-stat" message="processed 12%" pid="60234"
+ timestamp="2015-09-21T19:49:45" version="2.5.1" percent="12"/>
+</Log>
+ */
+rc_t
+GeneralLoader :: DatabaseLoader :: ProgressMessage ( const std :: string& p_name, uint32_t p_pid,
+ uint32_t p_timestamp, uint32_t p_version, uint32_t p_percent )
+{
+ KTime kt;
+ KTimeLocal ( &kt, ( KTime_t ) p_timestamp );
+
+ pLogMsg ( klogInfo,
+ "processed $(percent)%"
+ ,
+ "app=%s,pid=%u,timestamp=%lT,version=%V,percent=%u"
+ ,
+ p_name . c_str (),
+ p_pid,
+ & kt,
+ ( ver_t ) p_version,
+ p_percent );
+ return 0;
+}
diff --git a/tools/general-loader/general-loader.cpp b/tools/general-loader/general-loader.cpp
new file mode 100644
index 0000000..74f2512
--- /dev/null
+++ b/tools/general-loader/general-loader.cpp
@@ -0,0 +1,223 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "general-loader.hpp"
+
+#include <klib/rc.h>
+#include <klib/log.h>
+
+#include <kns/stream.h>
+
+#include <kfs/directory.h>
+
+#include "general-writer.h"
+
+using namespace std;
+
+///////////// GeneralLoader::Reader
+
+GeneralLoader::Reader::Reader( const struct KStream& p_input )
+: m_input ( p_input ),
+ m_buffer ( 0 ),
+ m_bufSize ( 0 ),
+ m_readCount ( 0 )
+{
+ KStreamAddRef ( & m_input );
+}
+
+GeneralLoader::Reader::~Reader()
+{
+ KStreamRelease ( & m_input );
+ free ( m_buffer );
+}
+
+rc_t
+GeneralLoader::Reader::Read( void * p_buffer, size_t p_size )
+{
+ pLogMsg ( klogDebug,
+ "general-loader: reading $(s) bytes, offset=$(o)",
+ "s=%u,o=%lu",
+ ( unsigned int ) p_size, m_readCount );
+
+ m_readCount += p_size;
+ return KStreamReadExactly ( & m_input, p_buffer, p_size );
+}
+
+rc_t
+GeneralLoader::Reader::Read( size_t p_size )
+{
+ if ( p_size > m_bufSize )
+ {
+ m_buffer = realloc ( m_buffer, p_size );
+ if ( m_buffer == 0 )
+ {
+ m_bufSize = 0;
+ m_readCount = 0;
+ return RC ( rcExe, rcFile, rcReading, rcMemory, rcExhausted );
+ }
+ }
+
+ pLogMsg ( klogDebug, "general-loader: reading $(s) bytes", "s=%u", ( unsigned int ) p_size );
+
+ m_readCount += p_size;
+ return KStreamReadExactly ( & m_input, m_buffer, p_size );
+}
+
+void
+GeneralLoader::Reader::Align( uint8_t p_bytes )
+{
+ if ( m_readCount % p_bytes != 0 )
+ {
+ Read ( p_bytes - m_readCount % p_bytes );
+ }
+}
+
+
+///////////// GeneralLoader
+
+GeneralLoader::GeneralLoader ( const std::string& p_programName, const struct KStream& p_input )
+: m_programName ( p_programName ),
+ m_reader ( p_input )
+{
+}
+
+GeneralLoader::~GeneralLoader ()
+{
+}
+
+void
+GeneralLoader::SetTargetOverride( const std::string& p_path )
+{
+ m_targetOverride = p_path;
+}
+
+void
+GeneralLoader::SplitAndAdd( Paths& p_paths, const string& p_path )
+{
+ size_t startPos = 0;
+ size_t colonPos = p_path . find ( ':', startPos );
+ while ( colonPos != string::npos )
+ {
+ p_paths . push_back ( p_path . substr ( startPos, colonPos - startPos ) );
+ startPos = colonPos + 1;
+ colonPos = p_path . find ( ':', startPos );
+ }
+ p_paths . push_back ( p_path . substr ( startPos ) );
+}
+
+void
+GeneralLoader::AddSchemaIncludePath( const string& p_path )
+{
+ SplitAndAdd ( m_includePaths, p_path );
+}
+
+void
+GeneralLoader::AddSchemaFile( const string& p_path )
+{
+ SplitAndAdd ( m_schemas, p_path );
+}
+
+rc_t
+GeneralLoader::Run()
+{
+ bool packed;
+ rc_t rc = ReadHeader ( packed );
+ if ( rc == 0 )
+ {
+ DatabaseLoader loader ( m_programName, m_includePaths, m_schemas, m_targetOverride );
+ if ( packed )
+ {
+ PackedProtocolParser p;
+ rc = p . ParseEvents ( m_reader, loader );
+ }
+ else
+ {
+ UnpackedProtocolParser p;
+ rc = p . ParseEvents ( m_reader, loader );
+ }
+
+ if ( rc != 0 && ! loader . GetDatabaseName() . empty () )
+ {
+ KDirectory* wd;
+ KDirectoryNativeDir ( & wd );
+ KDirectoryRemove ( wd, true, loader . GetDatabaseName() . c_str () );
+ KDirectoryRelease ( wd );
+ }
+ }
+
+ return rc;
+}
+
+rc_t
+GeneralLoader::ReadHeader ( bool& p_packed )
+{
+ struct gw_header_v1 header;
+
+ rc_t rc = m_reader . Read ( & header, sizeof header );
+ if ( rc == 0 )
+ {
+ if ( strncmp ( header . dad . signature, GeneralLoaderSignatureString, sizeof ( header . dad . signature ) ) != 0 )
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcHeader, rcCorrupt );
+ }
+ else
+ {
+ switch ( header . dad . endian )
+ {
+ case GW_GOOD_ENDIAN:
+ if ( header . dad . version > GW_CURRENT_VERSION ) /* > comparison so it can read multiple versions */
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcHeader, rcBadVersion );
+ }
+ else
+ {
+ rc = 0;
+ }
+ break;
+ case GW_REVERSE_ENDIAN:
+ LogMsg ( klogErr, "general-loader event: Detected reverse endianness (not yet supported)" );
+ rc = RC ( rcExe, rcFile, rcReading, rcFormat, rcUnsupported );
+ //TODO: apply byte order correction before checking the version number
+ break;
+ default:
+ rc = RC ( rcExe, rcFile, rcReading, rcFormat, rcInvalid );
+ break;
+ }
+ }
+ }
+
+ if ( rc == 0 && header . dad . hdr_size > sizeof header )
+ {
+ rc = m_reader . Read ( header . dad . hdr_size - sizeof header );
+ }
+
+ if ( rc == 0 )
+ {
+ p_packed = header. packing != 0;
+ }
+
+ return rc;
+}
diff --git a/tools/general-loader/general-loader.hpp b/tools/general-loader/general-loader.hpp
new file mode 100644
index 0000000..314e782
--- /dev/null
+++ b/tools/general-loader/general-loader.hpp
@@ -0,0 +1,243 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#ifndef _sra_tools_hpp_general_loader_
+#define _sra_tools_hpp_general_loader_
+
+#include <klib/defs.h>
+
+#include <string>
+#include <vector>
+#include <map>
+
+struct KStream;
+struct VCursor;
+struct VDatabase;
+struct VDBManager;
+struct VSchema;
+
+#define GeneralLoaderSignatureString GW_SIGNATURE
+
+class GeneralLoader
+{
+public:
+ static const uint32_t MaxPackedString = 256;
+
+public:
+ GeneralLoader ( const std :: string& p_programName, const struct KStream& p_input );
+ ~GeneralLoader ();
+
+ void AddSchemaIncludePath( const std::string& p_path );
+ void AddSchemaFile( const std::string& p_file );
+ void SetTargetOverride( const std::string& p_path );
+
+ rc_t Run ();
+
+private:
+
+ typedef std::vector < std::string > Paths;
+
+private:
+
+ class Reader
+ {
+ public:
+ Reader( const struct KStream& p_input );
+ ~Reader();
+
+ // read into caller's buffer
+ rc_t Read( void * p_buffer, size_t p_size );
+
+ // if rc == 0, there are p_size bytes available through GetBuffer until the next call to Read
+ rc_t Read( size_t p_size );
+
+ const void* GetBuffer() const { return m_buffer; }
+
+ void Align( uint8_t p_bytes = 4 );
+
+ uint64_t GetReadCount() { return m_readCount; }
+
+ private:
+ const struct KStream& m_input;
+ void* m_buffer;
+ size_t m_bufSize;
+ uint64_t m_readCount;
+ };
+
+ class DatabaseLoader
+ {
+ public:
+ struct Table
+ {
+ std :: string name;
+ uint32_t databaseId; // key into Databases
+ uint32_t cursorIdx;
+ };
+
+ struct Column
+ {
+ std :: string name;
+
+ uint32_t tableId; // key into Tables
+ uint32_t cursorIdx; // index into Cursors
+ uint32_t columnIdx; // index in the VCursor
+ uint32_t elemBits;
+ uint32_t flagBits;
+
+ typedef std :: map < std :: string, std :: string > Metadata;
+ Metadata metadata;
+
+ bool IsCompressed () const { return ( flagBits & 1 ) == 1; }
+ };
+
+ public:
+ DatabaseLoader ( const std :: string& p_programName, const Paths& p_includePaths, const Paths& p_schemas, const std::string& p_dbNameOverride = std::string() );
+ ~DatabaseLoader();
+
+ rc_t UseSchema ( const std :: string& p_file, const std :: string& p_name );
+ rc_t RemotePath ( const std :: string& p_path );
+ rc_t SoftwareName ( const std :: string& p_softwareName, const std :: string& p_version );
+ rc_t NewTable ( uint32_t p_tableId, const std :: string& p_tableName );
+ rc_t NewColumn ( uint32_t p_columnId,
+ uint32_t p_tableId,
+ uint32_t p_elemBits,
+ uint8_t p_flags,
+ const std :: string& p_columnName );
+
+ rc_t DBMetadataNode ( uint32_t p_objId, const std :: string& p_metadata_node, const std :: string& p_value );
+ rc_t TblMetadataNode ( uint32_t p_objId, const std :: string& p_metadata_node, const std :: string& p_value );
+ rc_t ColMetadataNode ( uint32_t p_objId, const std :: string& p_metadata_node, const std :: string& p_value );
+
+ rc_t AddMbrDB ( uint32_t p_objId, uint32_t p_parentId, const std :: string &mbr_name, const std :: string &db_name, uint8_t p_create_mode );
+ rc_t AddMbrTbl ( uint32_t p_objId, uint32_t p_parentId, const std :: string &mbr_name, const std :: string &db_name, uint8_t p_create_mode );
+
+ rc_t CellData ( uint32_t p_columnId, const void* p_data, size_t p_elemCount );
+ rc_t CellDefault ( uint32_t p_columnId, const void* p_data, size_t p_elemCount );
+ rc_t NextRow ( uint32_t p_tableId );
+ rc_t MoveAhead ( uint32_t p_tableId, uint64_t p_count );
+ rc_t ErrorMessage ( const std :: string& p_text );
+ rc_t LogMessage ( const std :: string& p_text );
+ rc_t ProgressMessage ( const std :: string& p_name, uint32_t p_pid, uint32_t p_timestamp, uint32_t p_version, uint32_t p_percent );
+ rc_t OpenStream ();
+ rc_t CloseStream ();
+
+ const std :: string& GetDatabaseName() const { return m_databaseName; }
+ const Column* GetColumn ( uint32_t p_columnId ) const;
+
+ private:
+ // Active Cursors
+ typedef std::vector < struct VCursor * > Cursors;
+
+ // from TableId to Table
+ typedef std::map < uint32_t, Table > Tables;
+
+ // from ColumnId to Column
+ typedef std::map < uint32_t, Column > Columns;
+
+ // From database id to VDatabase. id == 0 for the root database.
+ typedef std::map < uint32_t, VDatabase* > Databases;
+
+ // From database id to parent database id
+ typedef std::map < uint32_t, uint32_t > DatabaseToParent;
+
+ private:
+ rc_t MakeDatabase ( uint32_t p_id );
+ rc_t CursorWrite ( const Column& p_col, const void* p_data, size_t p_size );
+ rc_t CursorDefault ( const Column& p_col, const void* p_data, size_t p_size );
+ rc_t SaveColumnMetadata ( const Column& p_col );
+
+ private:
+ Paths m_includePaths;
+ Paths m_schemas;
+
+ std :: string m_programName;
+ std :: string m_databaseName;
+ std :: string m_schemaName;
+
+ std :: string m_softwareName;
+ ver_t m_softwareVersion;
+
+ Cursors m_cursors;
+ Tables m_tables;
+ Columns m_columns;
+ Databases m_databases;
+ DatabaseToParent m_dbParents;
+
+ struct VDBManager* m_mgr;
+ struct VSchema* m_schema;
+
+ bool m_databaseNameOverridden;
+ };
+
+ class ProtocolParser
+ {
+ public:
+ virtual rc_t ParseEvents ( Reader&, DatabaseLoader& ) = 0;
+
+ protected:
+ template <typename TEvent> rc_t ReadEvent ( Reader& p_reader, TEvent& p_event );
+ };
+
+ class UnpackedProtocolParser : public ProtocolParser
+ {
+ public:
+ virtual rc_t ParseEvents ( Reader&, DatabaseLoader& );
+ };
+
+ class PackedProtocolParser : public ProtocolParser
+ {
+ public:
+ virtual rc_t ParseEvents ( Reader&, DatabaseLoader& );
+
+ private:
+ // read p_dataSize bytes and use one of the decoder functions in utf8-like-int-codec.h to unpack a sequence of integer values,
+ // stored in m_unpackingBuf as a collection of bytes
+ template < typename T_uintXX > rc_t UncompressInt ( Reader& p_reader, uint16_t p_dataSize, int ( * p_decode ) ( uint8_t const* buf_start, uint8_t const* buf_xend, T_uintXX* ret_decoded ) );
+
+ rc_t ParseData ( Reader& p_reader, DatabaseLoader& p_dbLoader, uint32_t p_columnId, uint32_t p_dataSize );
+
+ std::vector<uint8_t> m_unpackingBuf;
+ };
+
+private:
+ GeneralLoader(const GeneralLoader&);
+ GeneralLoader& operator = ( const GeneralLoader&);
+
+ rc_t ReadHeader ( bool& p_packed );
+
+ void CleanUp ();
+
+ static void SplitAndAdd( Paths& p_paths, const std::string& p_path );
+
+private:
+ std::string m_programName;
+ Reader m_reader;
+ Paths m_includePaths;
+ Paths m_schemas;
+ std::string m_targetOverride;
+};
+
+#endif
diff --git a/tools/general-loader/general-loader.vers b/tools/general-loader/general-loader.vers
new file mode 100644
index 0000000..35d16fb
--- /dev/null
+++ b/tools/general-loader/general-loader.vers
@@ -0,0 +1 @@
+2.5.7
diff --git a/tools/general-loader/general-loader.vers.h b/tools/general-loader/general-loader.vers.h
new file mode 100644
index 0000000..9b7ad62
--- /dev/null
+++ b/tools/general-loader/general-loader.vers.h
@@ -0,0 +1 @@
+#define GENERAL_LOADER_VERS 0x02050007
diff --git a/tools/general-loader/general-writer.cpp b/tools/general-loader/general-writer.cpp
new file mode 100644
index 0000000..32b1a47
--- /dev/null
+++ b/tools/general-loader/general-writer.cpp
@@ -0,0 +1,1172 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "general-writer.hpp"
+#include "utf8-like-int-codec.h"
+
+#include <iterator>
+#include <cstdlib>
+#include <iomanip>
+
+#include <time.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+
+#define PROGRESS_EVENT 0
+
+namespace ncbi
+{
+
+#if GW_CURRENT_VERSION <= 2
+ typedef :: gwp_1string_evt_v1 gwp_1string_evt;
+ typedef :: gwp_2string_evt_v1 gwp_2string_evt;
+ typedef :: gwp_column_evt_v1 gwp_column_evt;
+ typedef :: gwp_data_evt_v1 gwp_data_evt;
+ typedef :: gwp_1string_evt_U16_v1 gwp_1string_evt_U16;
+ typedef :: gwp_2string_evt_U16_v1 gwp_2string_evt_U16;
+ typedef :: gwp_data_evt_U16_v1 gwp_data_evt_U16;
+#else
+#error "unrecognized GW version"
+#endif
+
+ // ask the general-loader to use this when naming its output
+ void GeneralWriter :: setRemotePath ( const std :: string & remote_db )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case header_written:
+ new_state = remote_name_sent;
+ break;
+ case schema_sent:
+ new_state = remote_name_and_schema_sent;
+ break;
+ case software_name_sent:
+ new_state = remote_name_and_software_name_sent;
+ break;
+ case schema_and_software_name_sent:
+ new_state = remote_name_schema_and_software_name_sent;
+ break;
+ default:
+ throw "state violation setting remote path";
+ }
+
+ size_t str_size = remote_db . size ();
+ if ( str_size > 0x10000 )
+ throw "remote path too long";
+
+ gwp_1string_evt_U16 hdr;
+ init ( hdr, 0, evt_remote_path2 );
+ set_size ( hdr, str_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( remote_db . data (), str_size );
+
+ state = new_state;
+ }
+
+ // tell the general-loader to use this pre-defined schema
+ void GeneralWriter :: useSchema ( const std :: string & schema_file_name,
+ const std :: string & schema_db_spec )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case header_written:
+ new_state = schema_sent;
+ break;
+ case remote_name_sent:
+ new_state = remote_name_and_schema_sent;
+ break;
+ case software_name_sent:
+ new_state = schema_and_software_name_sent;
+ break;
+ case remote_name_and_software_name_sent:
+ new_state = remote_name_schema_and_software_name_sent;
+ break;
+ default:
+ throw "state violation using schema";
+ }
+
+ size_t str1_size = schema_file_name . size ();
+ if ( str1_size > 0x10000 )
+ throw "schema path too long";
+
+ size_t str2_size = schema_db_spec . size ();
+ if ( str2_size > 0x10000 )
+ throw "schema spec too long";
+
+ gwp_2string_evt_U16 hdr;
+ init ( hdr, 0, evt_use_schema2 );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( schema_file_name . data (), str1_size );
+ internal_write ( schema_db_spec . data (), str2_size );
+
+ state = new_state;
+ }
+
+ void GeneralWriter :: setSoftwareName ( const std :: string & name,
+ const std :: string & version )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case header_written:
+ new_state = software_name_sent;
+ break;
+ case remote_name_sent:
+ new_state = remote_name_and_software_name_sent;
+ break;
+ case schema_sent:
+ new_state = schema_and_software_name_sent;
+ break;
+ case remote_name_and_schema_sent:
+ new_state = remote_name_schema_and_software_name_sent;
+ break;
+ default:
+ throw "state violation using schema";
+ }
+
+ size_t str1_size = name . size ();
+ if ( str1_size > 0x100 )
+ throw "name too long";
+
+ size_t str2_size = version . size ();
+ if ( str2_size > 0x100 )
+ throw "version too long";
+
+ gwp_2string_evt_v1 hdr;
+ init ( hdr, 0, evt_software_name );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( name . data (), str1_size );
+ internal_write ( version . data (), str2_size );
+
+ state = new_state;
+ }
+
+
+ int GeneralWriter :: addTable ( const std :: string &table_name )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case schema_sent:
+ case remote_name_and_schema_sent:
+ case remote_name_schema_and_software_name_sent:
+ new_state = have_table;
+ break;
+ case have_table:
+ case have_column:
+ new_state = state;
+ break;
+ default:
+ throw "state violation adding table";
+ }
+
+ // create a pair between the outer db id ( 0 ) and this table name
+ // this pair will act as an unique key of tables, whereas table_name
+ // itself can be repeated if multiple databases are in use
+ int_dbtbl tbl ( 0, table_name );
+
+ // prediction this is the index
+ int id = ( int ) tables.size() + 1;
+ if ( id > 256 )
+ throw "maximum number of tables exceeded";
+
+ // make sure we never record a table name twice under the same db
+ std :: pair < std :: map < int_dbtbl, int > :: iterator, bool > result =
+ table_name_idx.insert ( std :: pair < int_dbtbl, int > ( tbl, id ) );
+
+ // if first time
+ if ( result.second )
+ {
+ tables.push_back ( tbl );
+
+ size_t str_size = table_name . size ();
+ if ( str_size > 0x10000 )
+ throw "maximum table name length exceeded";
+
+ gwp_1string_evt_U16 hdr;
+ init ( hdr, id, evt_new_table2 );
+ set_size ( hdr, str_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( table_name.data (), str_size );
+
+ state = new_state;
+ }
+
+ // 1 based table id
+ return result.first->second;
+ }
+
+ int GeneralWriter :: addColumn ( int table_id,
+ const std :: string &column_name, uint32_t elem_bits, uint8_t flag_bits )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case have_table:
+ case have_column:
+ new_state = have_column;
+ break;
+ default:
+ throw "state violation adding column";
+ }
+
+ if ( table_id <= 0 || ( size_t ) table_id > tables.size () )
+ throw "Invalid table id";
+
+ // the thing to insert into map
+ // even if the caller wants us to use integer compaction,
+ // it must be with a size we know how to use
+ if ( ( flag_bits & 1 ) != 0 )
+ {
+ switch ( elem_bits )
+ {
+ case 16:
+ case 32:
+ case 64:
+ break;
+ default:
+ flag_bits ^= 1;
+ }
+ }
+
+ int_stream stream ( table_id, column_name, elem_bits, flag_bits );
+
+ // prediction this is the index
+ int id = ( int ) streams.size() + 1;
+ if ( id > 256 )
+ throw "maximum number of columns exceeded";
+
+ // make sure we never record a column-spec twice
+ std :: pair < std :: map < int_stream, int > :: iterator, bool > result =
+ column_name_idx.insert ( std :: pair < int_stream, int > ( stream, id ) );
+
+ // if first time
+ if ( result.second )
+ {
+ streams.push_back ( stream );
+
+ size_t str_size = column_name . size ();
+ if ( str_size > 256 )
+ throw "maximum column spec length exceeded";
+
+ // TBD - write new column stream event to stream
+ gwp_column_evt hdr;
+ init ( hdr, id, evt_new_column );
+ set_table_id ( hdr, table_id );
+ set_elem_bits ( hdr, elem_bits );
+ hdr.flag_bits = flag_bits;
+ set_name_size ( hdr, str_size );
+
+ // write header & data
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( column_name.data (), str_size );
+
+ state = new_state;
+ }
+
+ // 1 based stream id
+ return result.first->second;
+ }
+
+ int GeneralWriter :: dbAddDatabase ( int db_id, const std :: string &mbr_name,
+ const std :: string &db_name, uint8_t create_mode )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case schema_sent:
+ case remote_name_and_schema_sent:
+ case remote_name_schema_and_software_name_sent:
+ new_state = have_table;
+ break;
+ case have_table:
+ case have_column:
+ new_state = state;
+ break;
+ default:
+ throw "state violation adding db";
+ }
+
+ // zero ( 0 ) is a valid db_id
+ if ( db_id < 0 || ( size_t ) db_id > dbs.size () )
+ throw "Invalid database id";
+
+ int_dbtbl db ( db_id, db_name );
+
+ // prediction this is the index
+ int id = ( int ) dbs.size() + 1;
+ if ( id > 256 )
+ throw "maximum number of databases exceeded";
+
+ // make sure we never record a table name twice under the same db
+ std :: pair < std :: map < int_dbtbl, int > :: iterator, bool > result =
+ db_name_idx.insert ( std :: pair < int_dbtbl, int > ( db, id ) );
+
+ // if first time
+ if ( result.second )
+ {
+ dbs.push_back ( db );
+
+ size_t str_size = mbr_name . size ();
+ if ( str_size > 0x100 )
+ throw "maximum member name length exceeded";
+
+ str_size = db_name . size ();
+ if ( str_size > 0x100 )
+ throw "maximum db name length exceeded";
+
+ gwp_add_mbr_evt_v1 hdr;
+ init ( hdr, id, evt_add_mbr_db );
+ set_db_id ( hdr, db_id );
+ set_size1 ( hdr, mbr_name.size () );
+ set_size2 ( hdr, db_name.size () );
+ set_create_mode ( hdr, create_mode );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( mbr_name.data (), mbr_name.size () );
+ internal_write ( db_name.data (), db_name.size () );
+
+ state = new_state;
+ }
+
+ // 1 based db id
+ return result.first->second;
+ }
+
+ int GeneralWriter :: dbAddTable ( int db_id, const std :: string &mbr_name,
+ const std :: string &table_name, uint8_t create_mode )
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case schema_sent:
+ case remote_name_and_schema_sent:
+ case remote_name_schema_and_software_name_sent:
+ new_state = have_table;
+ break;
+ case have_table:
+ case have_column:
+ new_state = state;
+ break;
+ default:
+ throw "state violation adding db_table";
+ }
+
+ // zero ( 0 ) is a valid db_id
+ if ( db_id < 0 || ( size_t ) db_id > dbs.size () )
+ throw "Invalid database id";
+
+ // create a pair between the outer db id ( 0 ) and this table name
+ // this pair will act as an unique key of tables, whereas table_name
+ // itself can be repeated if multiple databases are in use
+ int_dbtbl tbl ( db_id, table_name );
+
+ // prediction this is the index
+ int id = ( int ) tables.size() + 1;
+ if ( id > 256 )
+ throw "maximum number of tables exceeded";
+
+ // make sure we never record a table name twice under the same db
+ std :: pair < std :: map < int_dbtbl, int > :: iterator, bool > result =
+ table_name_idx.insert ( std :: pair < int_dbtbl, int > ( tbl, id ) );
+
+ // if first time
+ if ( result.second )
+ {
+ tables.push_back ( tbl );
+
+ size_t str_size = mbr_name . size ();
+ if ( str_size > 0x100 )
+ throw "maximum member name length exceeded";
+
+ str_size = table_name . size ();
+ if ( str_size > 0x100 )
+ throw "maximum table name length exceeded";
+
+ gwp_add_mbr_evt_v1 hdr;
+ init ( hdr, id, evt_add_mbr_tbl );
+ set_db_id ( hdr, db_id );
+ set_size1 ( hdr, mbr_name.size () );
+ set_size2 ( hdr, table_name.size () );
+ set_create_mode ( hdr, create_mode );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( mbr_name.data (), mbr_name.size () );
+ internal_write ( table_name.data (), table_name.size () );
+
+ state = new_state;
+ }
+
+ // 1 based table id
+ return result.first->second;
+ }
+
+
+
+ void GeneralWriter :: open ()
+ {
+ stream_state new_state = uninitialized;
+
+ switch ( state )
+ {
+ case have_column:
+ new_state = opened;
+ break;
+ case opened:
+ return;
+ default:
+ throw "state violation opening stream";
+ }
+
+ gwp_evt_hdr hdr;
+ init ( hdr, 0, evt_open_stream );
+
+ // write header
+ write_event ( & hdr, sizeof hdr );
+
+ state = new_state;
+ }
+
+ void GeneralWriter :: setDBMetadataNode ( int obj_id,
+ const std :: string & node_path,
+ const std :: string & value )
+ {
+ // zero ( 0 ) is a valid db_id
+ if ( obj_id < 0 || ( size_t ) obj_id > dbs.size () )
+ throw "Invalid database id";
+
+ size_t str1_size = node_path . size ();
+ if ( str1_size > STRING_LIMIT_16 )
+ throw "DB_path too long";
+
+ size_t str2_size = value . size ();
+ if ( str2_size > STRING_LIMIT_16 )
+ throw "value too long";
+
+ if ( str1_size <= STRING_LIMIT_8 && str2_size <= STRING_LIMIT_8 )
+ {
+ // use 8-bit sizes
+ gwp_2string_evt_v1 hdr;
+ init ( hdr, obj_id, evt_db_metadata_node );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+ else
+ {
+ // use 16-bit sizes
+ gwp_2string_evt_U16_v1 hdr;
+ init ( hdr, obj_id, evt_db_metadata_node2 );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+
+ internal_write ( node_path . data (), str1_size );
+ internal_write ( value . data (), str2_size );
+ }
+
+ void GeneralWriter :: setTblMetadataNode ( int obj_id,
+ const std :: string & node_path,
+ const std :: string & value )
+ {
+ if ( obj_id <= 0 || ( size_t ) obj_id > tables.size () )
+ throw "Invalid table id";
+
+ size_t str1_size = node_path . size ();
+ if ( str1_size > STRING_LIMIT_16 )
+ throw "tbl_path too long";
+
+ size_t str2_size = value . size ();
+ if ( str2_size > STRING_LIMIT_16 )
+ throw "value too long";
+
+ if ( str1_size <= STRING_LIMIT_8 && str2_size <= STRING_LIMIT_8 )
+ {
+ // use 8-bit sizes
+ gwp_2string_evt_v1 hdr;
+ init ( hdr, obj_id, evt_tbl_metadata_node );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+ else
+ {
+ // use 16-bit sizes
+ gwp_2string_evt_U16_v1 hdr;
+ init ( hdr, obj_id, evt_tbl_metadata_node2 );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+
+ internal_write ( node_path . data (), str1_size );
+ internal_write ( value . data (), str2_size );
+
+ }
+ void GeneralWriter :: setColMetadataNode ( int obj_id,
+ const std :: string & node_path,
+ const std :: string & value )
+ {
+ if ( obj_id <= 0 || ( size_t ) obj_id > streams.size () )
+ throw "Invalid column id";
+
+ size_t str1_size = node_path . size ();
+ if ( str1_size > STRING_LIMIT_16 )
+ throw "tbl_path too long";
+
+ size_t str2_size = value . size ();
+ if ( str2_size > STRING_LIMIT_16 )
+ throw "value too long";
+
+ if ( str1_size <= STRING_LIMIT_8 && str2_size <= STRING_LIMIT_8 )
+ {
+ // use 8-bit sizes
+ gwp_2string_evt_v1 hdr;
+ init ( hdr, obj_id, evt_col_metadata_node );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+ else
+ {
+ // use 16-bit sizes
+ gwp_2string_evt_U16_v1 hdr;
+ init ( hdr, obj_id, evt_col_metadata_node2 );
+ set_size1 ( hdr, str1_size );
+ set_size2 ( hdr, str2_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+
+ internal_write ( node_path . data (), str1_size );
+ internal_write ( value . data (), str2_size );
+ }
+
+
+ void GeneralWriter :: columnDefault ( int stream_id, uint32_t elem_bits, const void *data, uint32_t elem_count )
+ {
+ switch ( state )
+ {
+ case opened:
+ break;
+ default:
+ throw "state violation setting column default";
+ }
+
+ if ( stream_id < 0 )
+ throw "Stream_id is not valid";
+ if ( stream_id > ( int ) streams.size () )
+ throw "Stream_id is out of bounds";
+
+ if ( elem_bits == 0 )
+ return;
+
+ if ( data == 0 && elem_count != 0 )
+ throw "Invalid data ptr";
+
+ if ( elem_bits != streams [ stream_id - 1 ] . elem_bits )
+ throw "Invalid elem_bits";
+
+ size_t num_bytes = ( ( size_t ) elem_bits * elem_count + 7 ) / 8;
+ if ( num_bytes == 0 )
+ {
+ gwp_evt_hdr_v1 eh;
+ init ( eh, stream_id, evt_empty_default );
+ write_event ( & eh, sizeof eh );
+ }
+ else
+ {
+ if ( num_bytes <= 256 )
+ {
+ gwp_data_evt chunk;
+ init ( chunk, stream_id, evt_cell_default );
+ set_size ( chunk, num_bytes );
+ write_event ( & chunk . dad, sizeof chunk );
+ }
+ else if ( num_bytes <= 0x10000 )
+ {
+ gwp_data_evt_U16 chunk;
+ init ( chunk, stream_id, evt_cell_default2 );
+ set_size ( chunk, num_bytes );
+ write_event ( & chunk . dad, sizeof chunk );
+ }
+ else
+ {
+ throw "default cell-data exceeds maximum";
+ }
+ internal_write ( data, num_bytes );
+ }
+ }
+
+ template < class T >
+ int encode_int ( T val, uint8_t * start, uint8_t * end );
+
+ template <>
+ int encode_int < uint16_t > ( uint16_t val, uint8_t * start, uint8_t * end )
+ {
+ int ret = encode_uint16 ( val, start, end );
+ if ( ret > 0 )
+ {
+ uint16_t val2;
+ int ret2 = decode_uint16 ( start, start + ret, & val2 );
+ assert ( ret == ret2 && val == val2 );
+ }
+ return ret;
+ }
+
+ template <>
+ int encode_int < uint32_t > ( uint32_t val, uint8_t * start, uint8_t * end )
+ {
+ int ret = encode_uint32 ( val, start, end );
+ if ( ret > 0 )
+ {
+ uint32_t val2;
+ int ret2 = decode_uint32 ( start, start + ret, & val2 );
+ assert ( ret == ret2 && val == val2 );
+ }
+ return ret;
+ }
+
+ template <>
+ int encode_int < uint64_t > ( uint64_t val, uint8_t * start, uint8_t * end )
+ {
+ int ret = encode_uint64 ( val, start, end );
+ if ( ret > 0 )
+ {
+ uint64_t val2;
+ int ret2 = decode_uint64 ( start, start + ret, & val2 );
+ assert ( ret == ret2 && val == val2 );
+ }
+ return ret;
+ }
+
+ struct encode_result { uint32_t num_elems, num_bytes; };
+
+ const size_t bsize = 0x10000;
+
+ template < class T > static
+ encode_result encode_buffer ( uint8_t * buffer, const void * data, uint32_t first, uint32_t elem_count )
+ {
+ uint8_t * start = buffer;
+ uint8_t * end = buffer + bsize;
+ const T * input = ( const T * ) data;
+
+ uint32_t i;
+ for ( i = first; i < elem_count; ++ i )
+ {
+ int num_writ = encode_int < T > ( input [ i ], start, end );
+ if ( num_writ <= 0 )
+ {
+ if ( num_writ < 0 )
+ throw "error encoding integer data";
+ break;
+ }
+ start += num_writ;
+ }
+
+ if ( start == buffer )
+ throw "INTERNAL ERROR: no data to encode";
+
+ encode_result rslt;
+ rslt . num_elems = i;
+ rslt . num_bytes = start - buffer;
+
+ return rslt;
+ }
+
+ void GeneralWriter :: write ( int stream_id, uint32_t elem_bits, const void *data, uint32_t elem_count )
+ {
+ switch ( state )
+ {
+ case opened:
+ break;
+ default:
+ throw "state violation writing column data";
+ }
+
+ if ( stream_id < 0 )
+ throw "Stream_id is not valid";
+ if ( stream_id > ( int ) streams.size () )
+ throw "Stream_id is out of bounds";
+
+ if ( elem_bits == 0 || elem_count == 0 )
+ return;
+
+ if ( data == 0 )
+ throw "Invalid data ptr";
+
+ const int_stream & s = streams [ stream_id - 1 ];
+
+ if ( elem_bits != s . elem_bits )
+ throw "Invalid elem_bits";
+
+ bool compact_int = ( s . flag_bits & 1 ) != 0;
+
+ const uint8_t * dp = ( const uint8_t * ) data;
+
+ if ( compact_int )
+ {
+ uint32_t elem;
+ encode_result rslt;
+ encode_result ( * encode ) ( uint8_t * buffer, const void * data, uint32_t first, uint32_t elem_count );
+
+ switch ( elem_bits )
+ {
+ case 16:
+ encode = encode_buffer < uint16_t >;
+ break;
+ case 32:
+ encode = encode_buffer < uint32_t >;
+ break;
+ case 64:
+ encode = encode_buffer < uint64_t >;
+ break;
+ default:
+ throw "INTERNAL ERROR: corrupt element bits";
+ }
+
+ for ( elem = 0; elem < elem_count; elem = rslt . num_elems )
+ {
+ rslt = ( * encode ) ( packing_buffer, data, elem, elem_count );
+ if ( rslt . num_bytes <= 256 )
+ {
+ assert ( rslt . num_bytes != 0 );
+ gwp_data_evt chunk;
+ init ( chunk, stream_id, evt_cell_data );
+ set_size ( chunk, rslt . num_bytes );
+ write_event ( & chunk . dad, sizeof chunk );
+ }
+ else
+ {
+ gwp_data_evt_U16 chunk;
+ init ( chunk, stream_id, evt_cell_data2 );
+ set_size ( chunk, rslt . num_bytes );
+ write_event ( & chunk . dad, sizeof chunk );
+ }
+ internal_write ( packing_buffer, rslt . num_bytes );
+ }
+ }
+ else
+ {
+ size_t num_bytes = ( ( size_t ) elem_bits * elem_count + 7 ) / 8;
+
+ while ( num_bytes >= 0x10000 )
+ {
+ gwp_data_evt_U16 chunk;
+ init ( chunk, stream_id, evt_cell_data2 );
+ set_size ( chunk, 0x10000 );
+ write_event ( & chunk . dad, sizeof chunk );
+ internal_write ( dp, 0x10000 );
+ num_bytes -= 0x10000;
+ dp += 0x10000;
+ }
+
+ if ( num_bytes <= 256 )
+ {
+ gwp_data_evt chunk;
+ init ( chunk, stream_id, evt_cell_data );
+ set_size ( chunk, num_bytes );
+ write_event ( & chunk . dad, sizeof chunk );
+ }
+ else
+ {
+ gwp_data_evt_U16 chunk;
+ init ( chunk, stream_id, evt_cell_data2 );
+ set_size ( chunk, num_bytes );
+ write_event ( & chunk . dad, sizeof chunk );
+ }
+
+ internal_write ( data, num_bytes );
+ }
+ }
+
+ void GeneralWriter :: nextRow ( int table_id )
+ {
+ switch ( state )
+ {
+ case opened:
+ break;
+ default:
+ throw "state violation advancing to next row";
+ }
+
+ if ( table_id < 0 || ( size_t ) table_id > tables.size () )
+ throw "Invalid table id";
+
+ gwp_evt_hdr hdr;
+ init ( hdr, table_id, evt_next_row );
+ write_event ( & hdr, sizeof hdr );
+ }
+
+
+ void GeneralWriter :: moveAhead ( int table_id, uint64_t nrows )
+ {
+ switch ( state )
+ {
+ case opened:
+ break;
+ default:
+ throw "state violation moving ahead nrows";
+ }
+
+ if ( table_id < 0 || ( size_t ) table_id > tables.size () )
+ throw "Invalid table id";
+
+ gwp_move_ahead_evt_v1 hdr;
+ init ( hdr, table_id, evt_move_ahead );
+ set_nrows ( hdr, nrows );
+ write_event ( & hdr . dad, sizeof hdr );
+ }
+
+ void GeneralWriter :: logError ( const std :: string & msg )
+ {
+ switch ( state )
+ {
+ case header_written:
+ case remote_name_sent:
+ case schema_sent:
+ case software_name_sent:
+ case remote_name_and_schema_sent:
+ case remote_name_and_software_name_sent:
+ case schema_and_software_name_sent:
+ case remote_name_schema_and_software_name_sent:
+ case have_table:
+ case have_column:
+ case opened:
+ case error:
+ break;
+ default:
+ return;
+ }
+
+ gwp_1string_evt_U16 hdr;
+ init ( hdr, 0, evt_errmsg2 );
+
+ const char * msg_data = msg . data ();
+ size_t str_size = msg . size ();
+ if ( str_size == 0 )
+ {
+ msg_data = "ERROR: (NO MSG)";
+ str_size = strlen ( msg_data );
+ }
+ else if ( str_size > 0x10000 )
+ str_size = 0x10000;
+
+ set_size ( hdr, str_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( msg_data, str_size );
+ }
+
+ void GeneralWriter :: logMsg ( const std :: string &msg )
+ {
+ switch ( state )
+ {
+ case header_written:
+ case remote_name_sent:
+ case schema_sent:
+ case software_name_sent:
+ case remote_name_and_schema_sent:
+ case remote_name_and_software_name_sent:
+ case schema_and_software_name_sent:
+ case remote_name_schema_and_software_name_sent:
+ case have_table:
+ case have_column:
+ case opened:
+ break;
+ default:
+ return;
+ }
+
+ size_t str_size = msg . size ();
+ if ( str_size == 0 )
+ return;
+
+ if ( str_size > 0x10000 )
+ str_size = 0x10000;
+
+ gwp_1string_evt_U16 hdr;
+ init ( hdr, 0, evt_logmsg );
+
+ set_size ( hdr, str_size );
+ write_event ( & hdr . dad, sizeof hdr );
+ internal_write ( msg . data (), str_size );
+ }
+
+ void GeneralWriter :: progMsg ( const std :: string & name, uint32_t version,
+ uint64_t done, uint64_t total )
+ {
+ switch ( state )
+ {
+ case opened:
+ break;
+ default:
+ return;
+ }
+
+ size_t str_size = name . size ();
+ if ( str_size == 0 )
+ throw "zero-length app-name";
+ if ( str_size > 0x100 )
+ str_size = 0x100;
+
+ // timestamp
+ time_t timestamp = time ( NULL );
+
+ if ( total == 0 )
+ throw "illegal total value: would divide by zero";
+ if ( done > total )
+ throw "illegal done value: greater than total";
+
+ // calculate percentage done
+ double fpercent = ( double ) done / total;
+ assert ( fpercent >= 0.0 && fpercent <= 100.0 );
+ uint8_t percent = ( uint8_t ) ( fpercent * 100 );
+
+ gwp_status_evt_v1 hdr;
+ init ( hdr, 0, evt_progmsg );
+ set_pid ( hdr, pid );
+ set_version ( hdr, version );
+ set_timestamp ( hdr, ( uint32_t ) timestamp );
+ set_size ( hdr, str_size );
+ set_percent ( hdr, percent );
+
+ write_event ( &hdr . dad, sizeof hdr );
+ internal_write ( name.data (), str_size );
+ }
+
+ void GeneralWriter :: endStream ()
+ {
+ switch ( state )
+ {
+ case header_written:
+ case remote_name_sent:
+ case schema_sent:
+ case software_name_sent:
+ case remote_name_and_schema_sent:
+ case remote_name_and_software_name_sent:
+ case schema_and_software_name_sent:
+ case remote_name_schema_and_software_name_sent:
+ case have_table:
+ case have_column:
+ case opened:
+ case error:
+ break;
+ default:
+ return;
+ }
+
+ gwp_evt_hdr hdr;
+ init ( hdr, 0, evt_end_stream );
+ write_event ( & hdr, sizeof hdr );
+
+ state = closed;
+
+ flush ();
+ }
+
+
+ // Constructors
+ GeneralWriter :: GeneralWriter ( const std :: string &out_path )
+ : out ( out_path.c_str(), std::ofstream::binary )
+ , evt_count ( 0 )
+ , byte_count ( 0 )
+ , pid ( getpid () )
+ , packing_buffer ( 0 )
+ , output_buffer ( 0 )
+ , output_bsize ( 0 )
+ , output_marker ( 0 )
+ , out_fd ( -1 )
+ , state ( uninitialized )
+ {
+ packing_buffer = new uint8_t [ bsize ];
+ writeHeader ();
+ }
+
+
+ // Constructors
+ GeneralWriter :: GeneralWriter ( int _out_fd, size_t buffer_size )
+ : evt_count ( 0 )
+ , byte_count ( 0 )
+ , pid ( getpid () )
+ , packing_buffer ( 0 )
+ , output_buffer ( 0 )
+ , output_bsize ( buffer_size )
+ , output_marker ( 0 )
+ , out_fd ( _out_fd )
+ , state ( uninitialized )
+ {
+ packing_buffer = new uint8_t [ bsize ];
+ output_buffer = new uint8_t [ buffer_size ];
+ writeHeader ();
+ }
+
+ GeneralWriter :: ~GeneralWriter ()
+ {
+ try
+ {
+ endStream ();
+ }
+ catch ( ... )
+ {
+ }
+
+ delete [] output_buffer;
+ delete [] packing_buffer;
+
+ output_bsize = output_marker = 0;
+ output_buffer = packing_buffer = 0;
+ }
+
+ bool GeneralWriter :: int_stream :: operator < ( const int_stream &s ) const
+ {
+ if ( table_id != s.table_id )
+ return table_id < s.table_id;
+ return column_name.compare ( s.column_name ) < 0;
+ }
+
+ GeneralWriter :: int_stream :: int_stream ( int _table_id, const std :: string &_column_name, uint32_t _elem_bits, uint8_t _flag_bits )
+ : table_id ( _table_id )
+ , column_name ( _column_name )
+ , elem_bits ( _elem_bits )
+ , flag_bits ( _flag_bits )
+ {
+ }
+
+ bool GeneralWriter :: int_dbtbl :: operator < ( const int_dbtbl &db ) const
+ {
+ if ( db_id != db.db_id )
+ return db_id < db.db_id;
+ return obj_name.compare ( db.obj_name ) < 0;
+ }
+
+ GeneralWriter :: int_dbtbl :: int_dbtbl ( int _db_id, const std :: string &_obj_name )
+ : db_id ( _db_id )
+ , obj_name ( _obj_name )
+ {
+ }
+
+ // Private methods
+
+ uint32_t GeneralWriter :: getPid ()
+ {
+ return ( uint32_t ) pid;
+ }
+
+ void GeneralWriter :: writeHeader ()
+ {
+ :: gw_header_v1 hdr;
+ init ( hdr );
+ internal_write ( & hdr, sizeof hdr );
+ state = header_written;
+
+ }
+
+ void GeneralWriter :: flush ()
+ {
+ if ( out_fd < 0 )
+ out . flush ();
+ else
+ {
+ ssize_t num_writ;
+ for ( size_t total = 0; total < output_marker; total += num_writ )
+ {
+ num_writ = :: write ( out_fd, & output_buffer [ total ], output_marker - total );
+ if ( num_writ < 0 )
+ throw "Error writing to fd";
+ if ( num_writ == 0 )
+ throw "Transfer incomplete writing to fd";
+ }
+
+ output_marker = 0;
+ }
+ }
+
+ void GeneralWriter :: internal_write ( const void * data, size_t num_bytes )
+ {
+ if ( out_fd < 0 )
+ {
+ out.write ( ( const char * ) data, num_bytes );
+ byte_count += num_bytes;
+ }
+ else
+ {
+ size_t total;
+ const uint8_t * p = ( const uint8_t * ) data;
+ for ( total = 0; total < num_bytes; )
+ {
+ size_t avail = output_bsize - output_marker;
+ if ( avail == 0 )
+ {
+ flush ();
+ avail = output_bsize - output_marker;
+ }
+
+ size_t to_write = num_bytes - total;
+ if ( to_write > avail )
+ to_write = avail;
+
+ assert ( to_write != 0 );
+ memcpy ( & output_buffer [ output_marker ], & p [ total ], to_write );
+ output_marker += to_write;
+ total += to_write;
+ }
+
+ byte_count += total;
+ }
+ }
+
+ void GeneralWriter :: write_event ( const gwp_evt_hdr * e, size_t evt_size )
+ {
+#if PROGRESS_EVENT
+ uint64_t ec = evt_count;
+ if ( ( ec % 10000 ) == 0 )
+ {
+ if ( ( ec % 500000 ) == 0 )
+ std :: cerr << "\n% [" << std :: setw ( 12 ) << byte_count << "] " << std :: setw ( 9 ) << ec + 1 << ' ';
+ std :: cerr << '.';
+ }
+#endif
+ ++ evt_count;
+
+ assert ( evt ( * e ) != evt_bad_event );
+ assert ( evt ( * e ) < evt_max_id );
+
+ internal_write ( e, evt_size );
+ }
+
+}
diff --git a/tools/general-loader/general-writer.h b/tools/general-loader/general-writer.h
new file mode 100644
index 0000000..378df5d
--- /dev/null
+++ b/tools/general-loader/general-writer.h
@@ -0,0 +1,1081 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _h_general_writer_
+#define _h_general_writer_
+
+#include <stdint.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+#include <string.h>
+#endif
+
+
+/*----------------------------------------------------------------------
+ * event codes
+ */
+
+enum gw_evt_id
+{
+ evt_bad_event,
+
+ evt_errmsg, /* convey processing error msg */
+ evt_end_stream, /* cleanly terminates a stream */
+
+ evt_remote_path, /* sets remote output path */
+ evt_use_schema, /* conveys schema usage */
+ evt_new_table, /* create a new table */
+ evt_new_column, /* create a new column in tbl */
+ evt_open_stream, /* open stream for data flow */
+
+ evt_cell_default, /* set/reset cell default val */
+ evt_cell_data, /* write/append data cell */
+ evt_next_row, /* move to next row in table */
+ evt_move_ahead, /* move ahead by N rows */
+
+ evt_errmsg2,
+ evt_remote_path2,
+ evt_use_schema2,
+ evt_new_table2,
+ evt_cell_default2, /* packed default <= 64K bytes */
+ evt_cell_data2, /* packed data <= 64K bytes */
+ evt_empty_default, /* set cell default to empty */
+
+ /* BEGIN VERSION 2 MESSAGES */
+ evt_software_name, /* sets software name */
+ evt_db_metadata_node, /* uses gw(p)_2string_evt_v1 */
+ evt_tbl_metadata_node,
+ evt_col_metadata_node,
+ evt_db_metadata_node2, /* uses gwp_2string_evt_U16 */
+ evt_tbl_metadata_node2,
+ evt_col_metadata_node2,
+
+ evt_add_mbr_db,
+ evt_add_mbr_tbl,
+
+ evt_logmsg,
+ evt_progmsg,
+
+ evt_max_id /* must be last */
+};
+
+#define GW_SIGNATURE "NCBIgnld"
+#define GW_GOOD_ENDIAN 1
+#define GW_REVERSE_ENDIAN ( 1 << 24 )
+#define GW_CURRENT_VERSION 2
+
+//These are not to change
+#define STRING_LIMIT_8 0x100
+#define STRING_LIMIT_16 0x10000
+#define ID_LOWER_LIMIT 0
+#define ID_UPPER_LIMIT 255
+
+/********************************
+ * DESCRIPTION OF STREAM EVENTS *
+ ********************************
+
+ *. BYTE ORDER
+ All binary data are in origination-host-native byte order.
+ The originating host indicates its byte order in the stream header.
+ Two byte-orders are supported: little-endian and big-endian.
+
+ *. PACKING
+ The preferred mode of operation for reduction of network bandwidth
+ is "packed". In this mode, the event structures use single-byte members
+ whenever possible, and there is no word alignment within the stream.
+ In addition, events with integer data can utilize a packing algorithm
+ to reduce the bandwidth consumed when the values are typically small
+ with regard to the stated type.
+
+ Packed mode is useful from C and C++ where there is ample support
+ from the language itself. Non-packed mode is more natural for languages
+ such as Python and Java and uses word-size members within structs and
+ does not perform any integer data packing.
+
+ NB - packed mode makes use of knowing that zero-length items
+ are invalid and should not be sent, and so stores length-1 wherever
+ a length is required, thus giving a range of 1..256 rather than
+ 0..255 for a single byte. The same is true for ids.
+
+ 1. STREAM HEADER
+ The stream starts with a header appropriate for the protocol
+ version in use. For version 1, use a "gw_header_v1".
+
+ Every stream header must have the standard "gw_header" as its
+ base definition. This portion presents an 8-character signature
+ to identify the type of stream, followed immediately by an integer
+ indicating originating-host byte order. The receiver must verify that
+ this value is either GW_GOOD_ENDIAN or GW_REVERSE_ENDIAN; any other
+ value means the stream is either not of this protocol or corrupt.
+ A value of GW_REVERSE_ENDIAN implies that the receiver must perform
+ byte-swapping on all word values within all events and their data.
+ This is likely to be a nearly untenable task by any but the
+ general-loader, where the latter has access to type information.
+
+ The value presented for version must be >= 1 and <= GW_CURRENT_VERSION.
+ A value of 0 must be rejected as not conforming to protocol while a
+ value > GW_CURRENT_VERSION cannot be processed.
+
+ The v1 value for "packing" currently allows for two values: 0 and 1,
+ where 0 means that no packing will be used and 1 indicates packed
+ events will be used. This affects how all subsequent events are to
+ be sent and received.
+
+ 2. SET REMOTE PATH [ OPTIONAL ]
+ In the not-packed case, use "gw_1string_evt".
+ GW_SET_ID_EVT ( & evt.dad, 0, evt_remote_path );
+ evt.sz = strlen ( path );
+ follow with the bytes in path
+ write ( path, strlen ( path ) );
+ end with 0..3 bytes of value 0 to realign stream to 4-byte boundary.
+
+ In the packed case where strlen ( path ) <= 256, use "gwp_1string_evt".
+ GWP_SET_ID_EVT ( & evt.dad, 0, evt_remote_path );
+ evt.sz = strlen ( path ) - 1;
+ follow with the bytes in path
+ write ( path, strlen ( path ) );
+
+ In the packed case where strlen ( path ) > 0x100 but <= 0x10000, use "gwp_1string_evt_U16"
+ GWP_SET_ID_EVT ( & evt.dad, 0, evt_remote_path2 );
+ evt.sz = strlen ( path ) - 1;
+ follow with the bytes in path
+ write ( path, strlen ( path ) );
+
+ MORE TO COME...
+
+ */
+
+
+/*======================================================================
+ * version 1
+ */
+
+/* gw_header
+ * common to all versions
+ */
+struct gw_header
+{
+ char signature [ 8 ]; /* = GW_SIGNATURE */
+ uint32_t endian; /* = GW_GOOD_ENDIAN or GW_REVERSE_ENDIAN */
+ uint32_t version; /* 0 < version <= GW_CURRENT_VERSION */
+ uint32_t hdr_size; /* the size of the entire header, including alignment */
+};
+
+/* gw_header_v1
+ * v1 header
+ */
+struct gw_header_v1
+{
+ gw_header dad;
+ uint32_t packing; /* 0 = no packing, 1 = byte packing */
+};
+
+
+/*----------------------------------------------------------------------
+ * full-size ( not packed ) events
+ */
+
+/* gw_evt_hdr
+ * common header to not-packed v1 events
+ *
+ * used as-is for events:
+ * { evt_end_stream, evt_open_stream, evt_next_row }
+ */
+struct gw_evt_hdr_v1
+{
+ uint32_t id_evt; /* bits 0..23 = id, bits 24..31 = event code */
+};
+
+/* gw_1string_evt_v1
+ * event to convey a single string of information
+ *
+ * used for events:
+ * { evt_errmsg, evt_remote_path, evt_new_table }
+ */
+struct gw_1string_evt_v1
+{
+ gw_evt_hdr_v1 dad; /* common header : id = 0 or new table id > 0 */
+ uint32_t sz; /* size of string in bytes - NO trailing NUL byte */
+ /* char str [ sz ]; * string data. *
+ char align [ 0..3 ]; * ( ( 4 - sizeof str % 4 ) % 4 ) zeros to align */
+};
+
+/* gw_2string_evt
+ * event used to send a pair of strings
+ *
+ * used for events:
+ * { evt_use_schema }
+ */
+struct gw_2string_evt_v1
+{
+ gw_evt_hdr_v1 dad; /* common header : id = 0 */
+ uint32_t sz1; /* size of string 1 in bytes, NO trailing NUL byte */
+ uint32_t sz2; /* size of string 2 in bytes, NO trailing NUL byte */
+ /* char str [ sz1+sz2 ]; * string data. *
+ char align [ 0..3 ]; * ( ( 4 - sizeof str % 4 ) % 4 ) zeros */
+};
+
+/* gw_column_evt
+ * event used to create a new column
+ *
+ * used for events:
+ * { evt_new_column }
+ */
+struct gw_column_evt_v1
+{
+ gw_evt_hdr_v1 dad; /* common header : id = new column id > 0 */
+ uint32_t table_id; /* id of column's table */
+ uint32_t elem_bits; /* the size of each element in data type */
+ uint32_t name_sz; /* the size in bytes of column "name" ( spec ) */
+ /* char name [ name_sz ]; * the column name/spec. *
+ char align [ 0..3 ]; * ( ( 4 - sizeof name % 4 ) % 4 ) zeros */
+};
+
+/* gw_data_evt
+ * event used to transfer cell data
+ *
+ * used for events:
+ * { evt_cell_default, evt_cell_data }
+ */
+struct gw_data_evt_v1
+{
+ gw_evt_hdr_v1 dad; /* common header : id = column id */
+ uint32_t elem_count; /* the number of elements in data */
+ /* uint8_t data [ x ]; * event data. actual size is: *
+ * ( ( elem_count * col . elem_bits ) + 7 ) / 8 *
+ char align [ 0..3 ]; * ( ( 4 - sizeof data % 4 ) % 4 ) zeros */
+};
+
+/* gw_move_ahead_evt_v1
+ */
+struct gw_move_ahead_evt_v1
+{
+ gw_evt_hdr_v1 dad; /* common header : id = column id */
+ uint32_t nrows [ 2 ]; /* the number of rows to move ahead */
+};
+
+struct gw_add_mbr_evt_v1
+{
+ gw_evt_hdr_v1 dad;
+ uint32_t db_id;
+ uint32_t mbr_sz;
+ uint32_t name_sz;
+ uint8_t create_mode;
+};
+
+struct gw_status_evt_v1
+{
+ gw_evt_hdr_v1 dad;
+ uint32_t version;
+ uint32_t timestamp;
+ uint32_t pid;
+ uint32_t name_sz;
+ uint32_t percent;
+};
+
+/*----------------------------------------------------------------------
+ * packed events
+ * used for C/C++ level operations
+ */
+
+/* gwp_evt_hdr
+ * common header to packed v1 events
+ *
+ * given that id 0 is illegal, this structure does not allocate a
+ * code for it, but rather stores all ids as id-1, i.e. 1..256 => 0..255.
+ *
+ * used as-is for events:
+ * { evt_end_stream, evt_open_stream, evt_next_row }
+ */
+struct gwp_evt_hdr_v1
+{
+ uint8_t _evt; /* event code from enum */
+ uint8_t _id; /* ids 1..256 represented as 0..255 */
+};
+
+/* gwp_1string_evt
+ * event to convey a single string of information
+ *
+ * used for events:
+ * { evt_errmsg, evt_remote_path, evt_new_table }
+ */
+struct gwp_1string_evt_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = 0 or new table id > 0 */
+ uint8_t sz; /* size of string - 1 in bytes - NO trailing NUL */
+ /* char str [ sz+1 ]; * string data */
+};
+
+/* gwp_2string_evt
+ * event used to send a pair of strings
+ *
+ * used for events:
+ * { evt_use_schema }
+ */
+struct gwp_2string_evt_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = 0 */
+ uint8_t sz1; /* size of string 1 - 1 in bytes, NO trailing NUL */
+ uint8_t sz2; /* size of string 2 - 1 in bytes, NO trailing NUL */
+ /* char str[ sz1+sz2+2 ]; * string data. */
+};
+
+/* gwp_column_evt
+ * event used to create a new column
+ *
+ * used for events:
+ * { evt_new_column }
+ */
+struct gwp_column_evt_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = new column id > 0 */
+ uint8_t table_id; /* id - 1 of column's table */
+ uint8_t elem_bits; /* the size - 1 of each element in data type */
+ uint8_t flag_bits; /* bit[0] = 1 means uses integer element packing */
+ uint8_t name_sz; /* the size - 1 in bytes of column "name" ( spec ) */
+ /* char name [ name_sz+1 ]; the column name/spec. */
+};
+
+/* gwp_data_evt
+ * event used to transfer cell data
+ *
+ * used for events:
+ * { evt_cell_default, evt_cell_data }
+ */
+struct gwp_data_evt_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = column id */
+ uint8_t sz; /* the size - 1 of data in bytes */
+ /* uint8_t data [ sz+1 ]; * event data. */
+};
+
+/* gwp_move_ahead_evt_v1
+ */
+struct gwp_move_ahead_evt_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = column id */
+ uint16_t nrows [ 4 ]; /* the number of rows to move ahead */
+};
+
+
+/* SPECIAL VERSIONS WITH 16-BIT SIZE FIELDS */
+
+/* gwp_1string_evt_U16
+ * event to convey a single string of information
+ *
+ * used for events:
+ * { evt_errmsg2, evt_remote_path2, evt_new_table2 }
+ *
+ * ...whenever size of string > 256
+ */
+struct gwp_1string_evt_U16_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = 0 or new table id > 0 */
+ uint16_t sz; /* size of string - 1 in bytes - NO trailing NUL */
+ /* char str [ sz+1 ]; * string data. */
+};
+
+/* gwp_2string_evt_U16
+ * event used to send a pair of strings
+ *
+ * used for events:
+ * { evt_use_schema2 }
+ *
+ * ...whenever size of any string > 256
+ */
+struct gwp_2string_evt_U16_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = 0 */
+ uint16_t sz1; /* size of string 1 - 1 in bytes, NO trailing NUL */
+ uint16_t sz2; /* size of string 2 - 1 in bytes, NO trailing NUL */
+ /* char str[ sz1+sz2+2 ]; * string data. */
+};
+
+
+/* gwp_data_evt_16
+ * event used to transfer cell data
+ *
+ * used for events:
+ * { evt_cell_default2, evt_cell_data2 }
+ */
+struct gwp_data_evt_U16_v1
+{
+ gwp_evt_hdr_v1 dad; /* common header : id = column id */
+ uint16_t sz; /* the size - 1 of data in bytes */
+ /* uint8_t data [ sz+1 ]; * event data. */
+};
+
+struct gwp_add_mbr_evt_v1
+{
+ gwp_evt_hdr_v1 dad;
+ uint8_t db_id;
+ uint8_t mbr_sz;
+ uint8_t name_sz;
+ uint8_t create_mode;
+};
+
+struct gwp_status_evt_v1
+{
+ gwp_evt_hdr_v1 dad;
+ uint32_t version;
+ uint32_t timestamp;
+ uint32_t pid;
+ uint8_t name_sz;
+ uint8_t percent;
+};
+
+#ifdef __cplusplus
+/*======================================================================
+ * support for C++
+ */
+
+#include <string.h>
+#include <assert.h>
+
+namespace ncbi
+{
+ // gw_header
+ inline void init ( :: gw_header & hdr )
+ {
+ memcpy ( hdr . signature, GW_SIGNATURE, sizeof hdr . signature );
+ hdr . endian = GW_GOOD_ENDIAN;
+ hdr . version = GW_CURRENT_VERSION;
+ hdr . hdr_size = sizeof ( :: gw_header );
+ }
+
+ inline void init ( :: gw_header & hdr, size_t hdr_size )
+ {
+ init ( hdr );
+ hdr . hdr_size = ( uint32_t ) hdr_size;
+ }
+
+ // gw_header_v1
+ inline void init ( :: gw_header_v1 & hdr )
+ { init ( hdr . dad, sizeof ( :: gw_header_v1 ) ); hdr . packing = 1; }
+
+ inline void init ( :: gw_header_v1 & hdr, const :: gw_header & dad )
+ { hdr . dad = dad; hdr . packing = 0; }
+
+
+
+ // gw_evt_hdr
+ inline void init ( :: gw_evt_hdr_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ assert ( id < 0x1000000 );
+ assert ( evt != evt_bad_event );
+ assert ( evt < evt_max_id );
+
+ hdr . id_evt = ( id & 0xFFFFFF ) | ( ( uint32_t ) evt << 24 );
+ }
+
+ inline uint32_t id ( const :: gw_evt_hdr_v1 & self )
+ { return self . id_evt & 0xFFFFFF; }
+
+ inline gw_evt_id evt ( const :: gw_evt_hdr_v1 & self )
+ { return ( gw_evt_id ) ( self . id_evt >> 24 ); }
+
+
+ // recording string size
+ inline void set_string_size ( uint32_t & sz, size_t bytes )
+ {
+ assert ( bytes != 0 );
+ assert ( sizeof bytes == 4 || ( bytes >> 32 ) == 0 );
+ sz = ( uint32_t ) bytes;
+ }
+
+ // gw_1string_evt
+ inline void init ( :: gw_1string_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ { init ( hdr . dad, id, evt ); hdr . sz = 0; }
+
+ inline void init ( :: gw_1string_evt_v1 & hdr, const :: gw_evt_hdr_v1 & dad )
+ { hdr . dad = dad; hdr . sz = 0; }
+
+ inline size_t size ( const :: gw_1string_evt_v1 & self )
+ { return self . sz; }
+
+ inline void set_size ( :: gw_1string_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz, bytes ); }
+
+
+ // gw_2string_evt
+ inline void init ( :: gw_2string_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz1 = hdr . sz2 = 0;
+ }
+
+ inline void init ( :: gw_2string_evt_v1 & hdr, const :: gw_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz1 = hdr . sz2 = 0;
+ }
+
+ inline size_t size1 ( const gw_2string_evt_v1 & self )
+ { return self . sz1; }
+
+ inline size_t size2 ( const gw_2string_evt_v1 & self )
+ { return self . sz2; }
+
+ inline void set_size1 ( :: gw_2string_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz1, bytes ); }
+
+ inline void set_size2 ( :: gw_2string_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz2, bytes ); }
+
+ // gw_column_evt
+ inline void init ( :: gw_column_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . table_id = hdr . elem_bits = hdr . name_sz = 0;
+ }
+
+ inline void init ( :: gw_column_evt_v1 & hdr, const :: gw_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . table_id = hdr . elem_bits = hdr . name_sz = 0;
+ }
+
+ inline uint32_t table_id ( const :: gw_column_evt_v1 & self )
+ { return self . table_id; }
+
+ inline uint32_t elem_bits ( const :: gw_column_evt_v1 & self )
+ { return self . elem_bits; }
+
+ inline uint8_t flag_bits ( const :: gw_column_evt_v1 & self )
+ { return 0; }
+
+ inline size_t name_size ( const :: gw_column_evt_v1 & self )
+ { return self . name_sz; }
+
+ inline void set_table_id ( :: gw_column_evt_v1 & self, uint32_t table_id )
+ {
+ assert ( table_id != 0 );
+ self . table_id = table_id;
+ }
+
+ inline void set_elem_bits ( :: gw_column_evt_v1 & self, uint32_t elem_bits )
+ {
+ assert ( elem_bits != 0 );
+ self . elem_bits = elem_bits;
+ }
+
+ inline void set_name_size ( :: gw_column_evt_v1 & self, size_t name_size )
+ { set_string_size ( self . name_sz, name_size ); }
+
+
+ // gw_data_evt
+ inline void init ( :: gw_data_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ { init ( hdr . dad, id, evt ); hdr . elem_count = 0; }
+
+ inline void init ( :: gw_data_evt_v1 & hdr, const :: gw_evt_hdr_v1 & dad )
+ { hdr . dad = dad; hdr . elem_count = 0; }
+
+ inline uint32_t elem_count ( const :: gw_data_evt_v1 & self )
+ { return self . elem_count; }
+
+ inline void set_elem_count ( :: gw_data_evt_v1 & self, uint32_t elem_count )
+ {
+ assert ( elem_count != 0 );
+ self . elem_count = elem_count;
+ }
+
+ // gw_move_ahead_evt_v1
+ inline void init ( :: gw_move_ahead_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ memset ( & hdr . nrows, 0, sizeof hdr . nrows );
+ }
+
+ inline void init ( :: gw_move_ahead_evt_v1 & hdr, const :: gw_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ memset ( & hdr . nrows, 0, sizeof hdr . nrows );
+ }
+
+ inline uint64_t get_nrows ( const :: gw_move_ahead_evt_v1 & self )
+ {
+ uint64_t nrows;
+ memcpy ( & nrows, & self . nrows, sizeof nrows );
+ return nrows;
+ }
+
+ inline void set_nrows ( :: gw_move_ahead_evt_v1 & self, uint64_t nrows )
+ {
+ memcpy ( & self . nrows, & nrows, sizeof self . nrows );
+ }
+
+ // gw_add_mbr_evt
+ inline void init ( :: gw_add_mbr_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . db_id = hdr . mbr_sz = hdr . name_sz = hdr . create_mode = 0;
+ }
+
+ inline void init ( :: gw_add_mbr_evt_v1 & hdr, const :: gw_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . db_id = hdr . mbr_sz = hdr . name_sz = hdr . create_mode = 0;
+ }
+
+ inline uint32_t db_id ( const gw_add_mbr_evt_v1 & self )
+ { return self . db_id; }
+
+ inline void set_db_id ( :: gw_add_mbr_evt_v1 & self, uint8_t db_id )
+ {
+ assert ( db_id >= 0 );
+ self . db_id = db_id;
+ }
+
+ inline size_t size1 ( const gw_add_mbr_evt_v1 & self )
+ { return ( size_t ) self . mbr_sz; }
+
+ inline size_t size2 ( const gw_add_mbr_evt_v1 & self )
+ { return ( size_t ) self . name_sz; }
+
+ inline void set_size1 ( :: gw_add_mbr_evt_v1 & self, size_t bytes )
+ {
+ set_string_size ( self . mbr_sz, bytes );
+ }
+
+ inline void set_size2 ( :: gw_add_mbr_evt_v1 & self, size_t bytes )
+ {
+ set_string_size ( self . name_sz, bytes );
+ }
+
+ inline uint8_t create_mode ( const gw_add_mbr_evt_v1 & self )
+ { return self . create_mode; }
+
+ inline void set_create_mode ( :: gw_add_mbr_evt_v1 & self, uint8_t mode )
+ {
+ self . create_mode = mode;
+ }
+
+ // gw_status_evt
+ inline void init ( :: gw_status_evt_v1 &hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . version = hdr . timestamp = hdr . pid = hdr . name_sz = hdr . percent = 0;
+ }
+
+ inline void init ( :: gw_status_evt_v1 &hdr, const :: gw_evt_hdr_v1 &dad )
+ {
+ hdr . dad = dad;
+ hdr . version = hdr . timestamp = hdr . pid = hdr . name_sz = hdr . percent = 0;
+ }
+
+ inline void set_version ( :: gw_status_evt_v1 &self, uint32_t version )
+ {
+ assert ( version > 0 );
+ self . version = version;
+ }
+
+ inline uint32_t version ( const :: gw_status_evt_v1 &self )
+ { return self . version; }
+
+ inline void set_timestamp ( :: gw_status_evt_v1 &self, uint32_t timestamp )
+ {
+ assert ( timestamp > 0 );
+ self . timestamp = timestamp;
+ }
+
+ inline uint32_t timestamp ( const :: gw_status_evt_v1 &self )
+ { return self . timestamp; }
+
+ inline void set_pid ( :: gw_status_evt_v1 &self, uint32_t pid )
+ {
+ assert ( pid > 0 );
+ self . pid = pid;
+ }
+
+ inline uint32_t pid ( const :: gw_status_evt_v1 &self )
+ { return self . pid; }
+
+ inline void set_size ( :: gw_status_evt_v1 &self, size_t bytes )
+ {
+ set_string_size ( self . name_sz, bytes );
+ }
+
+ inline size_t size ( const :: gw_status_evt_v1 &self )
+ { return ( size_t ) self . name_sz; }
+
+ inline void set_percent ( :: gw_status_evt_v1 &self, uint32_t percent )
+ {
+ self . percent = percent;
+ }
+
+ inline uint32_t percent ( const :: gw_status_evt_v1 &self )
+ { return self . percent; }
+
+ ////////// packed events //////////
+
+ // gwp_evt_hdr
+ inline void init ( :: gwp_evt_hdr_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ // allow zero, but treat as 256
+ assert ( id <= 0x100 );
+ assert ( evt != evt_bad_event );
+ assert ( evt < evt_max_id );
+
+ hdr . _evt = ( uint8_t ) evt;
+ hdr . _id = ( uint8_t ) ( id - 1 );
+ }
+
+ inline uint32_t id ( const :: gwp_evt_hdr_v1 & self )
+ { return ( uint32_t ) self . _id + 1; }
+
+ inline gw_evt_id evt ( const :: gwp_evt_hdr_v1 & self )
+ { return ( gw_evt_id ) self . _evt; }
+
+
+ // recording string size
+ inline void set_string_size ( uint8_t & sz, size_t bytes )
+ {
+ assert ( bytes != 0 );
+ assert ( bytes <= 0x100 );
+ sz = ( uint8_t ) ( bytes - 1 );
+ }
+
+ // gwp_1string_evt
+ inline void init ( :: gwp_1string_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz = 0;
+ }
+
+ inline void init ( :: gwp_1string_evt_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz = 0;
+ }
+
+ inline size_t size ( const :: gwp_1string_evt_v1 & self )
+ { return ( size_t ) self . sz + 1; }
+
+ inline void set_size ( :: gwp_1string_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz, bytes ); }
+
+
+ // gwp_2string_evt
+ inline void init ( :: gwp_2string_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz1 = hdr . sz2 = 0;
+ }
+
+ inline void init ( :: gwp_2string_evt_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz1 = hdr . sz2 = 0;
+ }
+
+ inline size_t size1 ( const gwp_2string_evt_v1 & self )
+ { return ( size_t ) self . sz1 + 1; }
+
+ inline size_t size2 ( const gwp_2string_evt_v1 & self )
+ { return ( size_t ) self . sz2 + 1; }
+
+ inline void set_size1 ( :: gwp_2string_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz1, bytes ); }
+
+ inline void set_size2 ( :: gwp_2string_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz2, bytes ); }
+
+
+ // gwp_column_evt
+ inline void init ( :: gwp_column_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . table_id = hdr . elem_bits = hdr . name_sz = 0;
+ }
+
+ inline void init ( :: gwp_column_evt_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . table_id = hdr . elem_bits = hdr . name_sz = 0;
+ }
+
+ inline uint32_t table_id ( const :: gwp_column_evt_v1 & self )
+ { return ( uint32_t ) self . table_id + 1; }
+
+ inline uint32_t elem_bits ( const :: gwp_column_evt_v1 & self )
+ { return ( uint32_t ) self . elem_bits + 1; }
+
+ inline uint8_t flag_bits ( const :: gwp_column_evt_v1 & self )
+ { return self . flag_bits; }
+
+ inline size_t name_size ( const :: gwp_column_evt_v1 & self )
+ { return ( size_t ) self . name_sz + 1; }
+
+ inline void set_table_id ( :: gwp_column_evt_v1 & self, uint32_t table_id )
+ {
+ assert ( table_id != 0 );
+ assert ( table_id <= 0x100 );
+ self . table_id = ( uint8_t ) ( table_id - 1 );
+ }
+
+ inline void set_elem_bits ( :: gwp_column_evt_v1 & self, uint32_t elem_bits )
+ {
+ assert ( elem_bits != 0 );
+ assert ( elem_bits <= 0x100 );
+ self . elem_bits = ( uint8_t ) ( elem_bits - 1 );
+ }
+
+ inline void set_name_size ( :: gwp_column_evt_v1 & self, size_t name_size )
+ { set_string_size ( self . name_sz, name_size ); }
+
+
+ // gwp_data_evt
+ inline void init ( :: gwp_data_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz = 0;
+ }
+
+ inline void init ( :: gwp_data_evt_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz = 0;
+ }
+
+ inline uint32_t size ( const :: gwp_data_evt_v1 & self )
+ { return ( uint32_t ) self . sz + 1; }
+
+ inline void set_size ( :: gwp_data_evt_v1 & self, size_t bytes )
+ { set_string_size ( self . sz, bytes ); }
+
+
+ // gwp_move_ahead_evt_v1
+ inline void init ( :: gwp_move_ahead_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ memset ( & hdr . nrows, 0, sizeof hdr . nrows );
+ }
+
+ inline void init ( :: gwp_move_ahead_evt_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ memset ( & hdr . nrows, 0, sizeof hdr . nrows );
+ }
+
+ inline uint64_t get_nrows ( const :: gwp_move_ahead_evt_v1 & self )
+ {
+ uint64_t nrows;
+ memcpy ( & nrows, & self . nrows, sizeof nrows );
+ return nrows;
+ }
+
+ inline void set_nrows ( :: gwp_move_ahead_evt_v1 & self, uint64_t nrows )
+ {
+ memcpy ( & self . nrows, & nrows, sizeof self . nrows );
+ }
+
+
+ // recording string size
+ inline void set_string_size ( uint16_t & sz, size_t bytes )
+ {
+ assert ( bytes != 0 );
+ assert ( bytes <= 0x10000 );
+ sz = ( uint16_t ) ( bytes - 1 );
+ }
+
+ // gwp_1string_evt_U16
+ inline void init ( :: gwp_1string_evt_U16_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz = 0;
+ }
+
+ inline void init ( :: gwp_1string_evt_U16_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz = 0;
+ }
+
+ inline size_t size ( const :: gwp_1string_evt_U16_v1 & self )
+ { return ( size_t ) self . sz + 1; }
+
+ inline void set_size ( :: gwp_1string_evt_U16_v1 & self, size_t bytes )
+ { set_string_size ( self . sz, bytes ); }
+
+
+ // gwp_2string_evt
+ inline void init ( :: gwp_2string_evt_U16_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz1 = hdr . sz2 = 0;
+ }
+
+ inline void init ( :: gwp_2string_evt_U16_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz1 = hdr . sz2 = 0;
+ }
+
+ inline size_t size1 ( const gwp_2string_evt_U16_v1 & self )
+ { return ( size_t ) self . sz1 + 1; }
+
+ inline size_t size2 ( const gwp_2string_evt_U16_v1 & self )
+ { return ( size_t ) self . sz2 + 1; }
+
+ inline void set_size1 ( :: gwp_2string_evt_U16_v1 & self, size_t bytes )
+ { set_string_size ( self . sz1, bytes ); }
+
+ inline void set_size2 ( :: gwp_2string_evt_U16_v1 & self, size_t bytes )
+ { set_string_size ( self . sz2, bytes ); }
+
+
+ // gwp_data_evt
+ inline void init ( :: gwp_data_evt_U16_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . sz = 0;
+ }
+
+ inline void init ( :: gwp_data_evt_U16_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . sz = 0;
+ }
+
+ inline uint32_t size ( const :: gwp_data_evt_U16_v1 & self )
+ { return ( uint32_t ) self . sz + 1; }
+
+ inline void set_size ( :: gwp_data_evt_U16_v1 & self, size_t bytes )
+ { set_string_size ( self . sz, bytes ); }
+
+
+ // gwp_add_mbr_evt
+ inline void init ( :: gwp_add_mbr_evt_v1 & hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . db_id = hdr . mbr_sz = hdr . name_sz = hdr . create_mode = 0;
+ }
+
+ inline void init ( :: gwp_add_mbr_evt_v1 & hdr, const :: gwp_evt_hdr_v1 & dad )
+ {
+ hdr . dad = dad;
+ hdr . db_id = hdr . mbr_sz = hdr . name_sz = hdr . create_mode = 0;
+ }
+
+ inline uint8_t db_id ( const gwp_add_mbr_evt_v1 & self )
+ { return self . db_id; }
+
+
+ inline void set_db_id ( :: gwp_add_mbr_evt_v1 & self, uint8_t db_id )
+ {
+ assert ( db_id >= 0 );
+ self . db_id = db_id;
+ }
+
+ inline size_t size1 ( const gwp_add_mbr_evt_v1 & self )
+ { return ( size_t ) self . mbr_sz + 1; }
+
+ inline size_t size2 ( const gwp_add_mbr_evt_v1 & self )
+ { return ( size_t ) self . name_sz + 1; }
+
+ inline void set_size1 ( :: gwp_add_mbr_evt_v1 & self, size_t bytes )
+ {
+ set_string_size ( self . mbr_sz, bytes );
+ }
+
+ inline void set_size2 ( :: gwp_add_mbr_evt_v1 & self, size_t bytes )
+ {
+ set_string_size ( self . name_sz, bytes );
+ }
+
+ inline uint8_t create_mode ( const gwp_add_mbr_evt_v1 & self )
+ { return self . create_mode; }
+
+ inline void set_create_mode ( :: gwp_add_mbr_evt_v1 & self, uint8_t mode )
+ {
+ self . create_mode = mode;
+ }
+
+ // gwp_status_evt
+ inline void init ( :: gwp_status_evt_v1 &hdr, uint32_t id, gw_evt_id evt )
+ {
+ init ( hdr . dad, id, evt );
+ hdr . version = hdr . timestamp = 0;
+ hdr . name_sz = hdr . percent = 0;
+ }
+
+ inline void init ( :: gwp_status_evt_v1 &hdr, const :: gwp_evt_hdr_v1 &dad )
+ {
+ hdr . dad = dad;
+ hdr . version = hdr . timestamp = 0;
+ hdr . name_sz = hdr . percent = 0;
+ }
+
+ inline void set_pid ( :: gwp_status_evt_v1 &self, int pid )
+ {
+ assert ( pid > 0 );
+ self . pid = ( uint32_t ) pid;
+ }
+
+ inline uint32_t pid ( const :: gwp_status_evt_v1 &self )
+ { return self . pid; }
+
+
+ inline void set_version ( :: gwp_status_evt_v1 &self, uint32_t version )
+ {
+ assert ( version != 0 );
+ self . version = version;
+ }
+
+ inline uint32_t version ( const :: gwp_status_evt_v1 &self )
+ { return self . version; }
+
+ inline void set_timestamp ( :: gwp_status_evt_v1 &self, uint32_t timestamp )
+ {
+ assert ( timestamp != 0 );
+ self . timestamp = timestamp;
+ }
+
+ inline uint32_t timestamp ( const :: gwp_status_evt_v1 &self )
+ { return self . timestamp; }
+
+ inline void set_size ( :: gwp_status_evt_v1 &self, size_t bytes )
+ {
+ set_string_size ( self . name_sz, bytes );
+ }
+
+ inline size_t size ( const :: gwp_status_evt_v1 &self )
+ { return ( size_t ) self . name_sz + 1; }
+
+ inline void set_percent ( :: gwp_status_evt_v1 &self, uint32_t percent )
+ {
+ self . percent = percent;
+ }
+
+ inline uint32_t percent ( const :: gwp_status_evt_v1 &self )
+ { return self . percent; }
+
+}
+#endif
+
+
+#endif /*_h_general_writer_*/
diff --git a/tools/general-loader/general-writer.hpp b/tools/general-loader/general-writer.hpp
new file mode 100644
index 0000000..5c98ebd
--- /dev/null
+++ b/tools/general-loader/general-writer.hpp
@@ -0,0 +1,210 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _hpp_general_writer_
+#define _hpp_general_writer_
+
+#ifndef _h_general_writer_
+#include "general-writer.h"
+#endif
+
+#include <iostream>
+#include <string>
+#include <stdint.h>
+#include <vector>
+#include <fstream>
+#include <map>
+
+#include <string.h>
+
+namespace ncbi
+{
+#if GW_CURRENT_VERSION <= 2
+ typedef :: gwp_evt_hdr_v1 gwp_evt_hdr;
+#else
+#error "unrecognized GW version"
+#endif
+
+ class GeneralWriter
+ {
+ public:
+
+ // ask the general-loader to use this when naming its output
+ void setRemotePath ( const std :: string & remote_db );
+
+ void setSoftwareName ( const std :: string & name,
+ const std :: string & version );
+
+ // tell the general-loader to use this pre-defined schema
+ void useSchema ( const std :: string & schema_file_name,
+ const std :: string & schema_db_spec );
+
+ // add a new table
+ // the table-id is returned
+ int addTable ( const std :: string &table_name );
+
+ // add a column to an existing table
+ // the column-id is returned
+ int addColumn ( int table_id, const std :: string &column_name, uint32_t elem_bits, uint8_t flags = 0 );
+
+ // when the column is known to have integer data, use this method
+ // it will utilize payload packing for reduced bandwidth
+ inline int addIntegerColumn ( int table_id, const std :: string &column_name, uint32_t elem_bits )
+ { return addColumn ( table_id, column_name, elem_bits, 1 ); }
+
+ int dbAddDatabase ( int db_id, const std :: string &mbr_name,
+ const std :: string &db_name, uint8_t create_mode );
+
+ int dbAddTable ( int db_id, const std :: string &mbr_name,
+ const std :: string &tbl_name, uint8_t create_mode );
+
+ // ensure there are atleast one table and one column
+ // set GeneralWriter to open state
+ // write out open_stream event header
+ void open ();
+
+ // add or set metadata on a specific object
+ // where obj_id == 0 => outer database, and
+ // any other positive id means the database, table or column
+ void setDBMetadataNode ( int obj_id,
+ const std :: string & node_path,
+ const std :: string & value );
+ void setTblMetadataNode ( int obj_id,
+ const std :: string & node_path,
+ const std :: string & value );
+ void setColMetadataNode ( int obj_id,
+ const std :: string & node_path,
+ const std :: string & value );
+
+ // generates a chunk of cell data
+ // MUST be entire default value in one event
+ void columnDefault ( int stream_id, uint32_t elem_bits, const void *data, uint32_t elem_count );
+
+ // generate a chunk of cell data
+ // may be repeated as often as necessary to complete a single cell's data
+ void write ( int stream_id, uint32_t elem_bits, const void *data, uint32_t elem_count );
+
+ // commit and close current row, move to next row
+ void nextRow ( int table_id );
+
+ // commit and close current row, move ahead by nrows
+ void moveAhead ( int table_id, uint64_t nrows );
+
+ // indicate some sort of exception
+ void logError ( const std :: string & msg );
+
+ // XML logging for general-loader
+ void logMsg ( const std :: string &msg );
+
+ // indicate progress
+ void progMsg ( const std :: string &name,
+ uint32_t version, uint64_t done, uint64_t total );
+
+ // generates an end event
+ // puts object into state that will reject any further transmissions
+ void endStream ();
+
+ // out_fd writes to an open file descriptor ( generally stdout )
+ // out_path initializes output stream for writing to a file
+ GeneralWriter ( int out_fd, size_t buffer_size = 32 * 1024 );
+ GeneralWriter ( const std :: string & out_path );
+
+ // output stream is flushed and closed
+ ~ GeneralWriter ();
+
+ private:
+
+ void writeHeader ();
+ void internal_write ( const void *data, size_t num_bytes );
+ void write_event ( const gwp_evt_hdr * evt, size_t evt_size );
+ void flush ();
+ uint32_t getPid ();
+
+ struct int_stream
+ {
+ bool operator < ( const int_stream &s ) const;
+ int_stream ( int table_id, const std :: string &column_name, uint32_t elem_bits, uint8_t flag_bits );
+
+ int table_id;
+ std :: string column_name;
+ uint32_t elem_bits;
+ uint8_t flag_bits;
+ };
+
+ struct int_dbtbl
+ {
+ bool operator < ( const int_dbtbl &db ) const;
+ int_dbtbl ( int db_id, const std :: string &obj_name );
+
+ int db_id;
+ std :: string obj_name;
+ };
+
+ std :: ofstream out;
+
+ std :: map < int_dbtbl, int > db_name_idx;
+ std :: map < int_dbtbl, int > table_name_idx;
+ std :: map < int_stream, int > column_name_idx;
+
+ std :: vector < int_stream > streams;
+ std :: vector < int_dbtbl > tables;
+ std :: vector < int_dbtbl > dbs;
+
+ uint64_t evt_count;
+ uint64_t byte_count;
+
+ int pid;
+
+ uint8_t * packing_buffer;
+
+ uint8_t * output_buffer;
+ size_t output_bsize;
+ size_t output_marker;
+
+ int out_fd;
+
+ enum stream_state
+ {
+ uninitialized,
+ header_written,
+ remote_name_sent,
+ schema_sent,
+ software_name_sent,
+ remote_name_and_schema_sent,
+ remote_name_and_software_name_sent,
+ schema_and_software_name_sent,
+ remote_name_schema_and_software_name_sent,
+ have_table,
+ have_column,
+ opened,
+ closed,
+ error
+ };
+ stream_state state;
+ };
+}
+
+#endif // _hpp_general_writer_
diff --git a/tools/general-loader/gw-dumper.cpp b/tools/general-loader/gw-dumper.cpp
new file mode 100644
index 0000000..9525317
--- /dev/null
+++ b/tools/general-loader/gw-dumper.cpp
@@ -0,0 +1,1785 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "general-writer.h"
+#include "utf8-like-int-codec.h"
+
+#include <kfs/defs.h>
+
+#include <iostream>
+#include <vector>
+
+#include <stdio.h>
+#include <string.h>
+#include <byteswap.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+using namespace ncbi;
+
+namespace gw_dump
+{
+ static bool display;
+ static uint32_t verbose;
+ static uint64_t event_num;
+ static uint64_t jump_event;
+ static uint64_t end_event;
+ static uint64_t foffset;
+
+ struct tbl_entry
+ {
+ tbl_entry ( const std :: string & _name )
+ : row_id ( 1 )
+ , member_name ( _name )
+ , tbl_name ( _name )
+ {
+ }
+
+ tbl_entry ( const std :: string & _member_name, const std :: string & _tbl_name )
+ : row_id ( 1 )
+ , member_name ( _member_name )
+ , tbl_name ( _tbl_name )
+ {
+ }
+
+ int64_t row_id;
+ std :: string member_name;
+ std :: string tbl_name;
+ };
+
+ static std :: vector < tbl_entry > tbl_entries;
+
+ struct db_entry
+ {
+ db_entry ( const std :: string & _member_name, const std :: string & _db_name )
+ : row_id ( 1 )
+ , member_name ( _member_name )
+ , db_name ( _db_name )
+ {
+ }
+
+ int64_t row_id;
+ std :: string member_name;
+ std :: string db_name;
+ };
+
+ static std :: vector < db_entry > db_entries;
+
+
+ struct col_entry
+ {
+ col_entry ( uint32_t _table_id, const std :: string & name, uint32_t _elem_bits, uint8_t _flag_bits = 0 )
+ : table_id ( _table_id )
+ , spec ( name )
+ , elem_bits ( _elem_bits )
+ , flag_bits ( _flag_bits )
+ {
+ }
+
+ ~ col_entry () {}
+
+ uint32_t table_id;
+ std :: string spec;
+ uint32_t elem_bits;
+ uint8_t flag_bits;
+ };
+ static std :: vector < col_entry > col_entries;
+
+ static
+ size_t readFILE ( void * buffer, size_t elem_size, size_t elem_count, FILE * in )
+ {
+ size_t num_read = fread ( buffer, elem_size, elem_count, in );
+ foffset += num_read * elem_size;
+ return num_read;
+ }
+
+ /* read_1string
+ */
+ template < class T > static
+ char * read_1string ( const T & eh, FILE * in )
+ {
+ size_t string_size = size ( eh );
+ char * string_buffer = new char [ string_size ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size, in );
+ if ( num_read != string_size )
+ {
+ delete [] string_buffer;
+ throw "failed to read string data";
+ }
+
+ return string_buffer;
+ }
+
+ template <>
+ char * read_1string < :: gw_1string_evt_v1 > ( const :: gw_1string_evt_v1 & eh, FILE * in )
+ {
+ size_t string_size_uint32 = ( size ( eh ) + 3 ) / 4;
+ uint32_t * string_buffer = new uint32_t [ string_size_uint32 ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size_uint32, in );
+ if ( num_read != string_size_uint32 )
+ {
+ delete [] string_buffer;
+ throw "failed to read string data";
+ }
+
+ return ( char * ) string_buffer;
+ }
+
+ template <>
+ char * read_1string < :: gw_status_evt_v1 > ( const :: gw_status_evt_v1 & eh, FILE * in )
+ {
+ size_t string_size_uint32 = ( size ( eh ) + 3 ) / 4;
+ uint32_t * string_buffer = new uint32_t [ string_size_uint32 ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size_uint32, in );
+ if ( num_read != string_size_uint32 )
+ {
+ delete [] string_buffer;
+ throw "failed to read string data";
+ }
+
+ return ( char * ) string_buffer;
+ }
+
+ /* whack_1string
+ */
+ template < class T > static
+ void whack_1string ( const T & eh, char * string_buffer )
+ {
+ delete [] string_buffer;
+ }
+
+ template <>
+ void whack_1string < :: gw_1string_evt_v1 > ( const :: gw_1string_evt_v1 & eh, char * string_buffer )
+ {
+ uint32_t * buffer = ( uint32_t * ) string_buffer;
+ delete [] buffer;
+ }
+
+ /* read_2string
+ */
+ template < class T > static
+ char * read_2string ( const T & eh, FILE * in )
+ {
+ size_t string_size = size1 ( eh ) + size2 ( eh );
+ char * string_buffer = new char [ string_size ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size, in );
+ if ( num_read != string_size )
+ {
+ delete [] string_buffer;
+ throw "failed to read dual string data";
+ }
+
+ return string_buffer;
+ }
+
+ template <>
+ char * read_2string < :: gw_2string_evt_v1 > ( const :: gw_2string_evt_v1 & eh, FILE * in )
+ {
+ size_t string_size_uint32 = ( size1 ( eh ) + size2 ( eh ) + 3 ) / 4;
+ uint32_t * string_buffer = new uint32_t [ string_size_uint32 ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size_uint32, in );
+ if ( num_read != string_size_uint32 )
+ {
+ delete [] string_buffer;
+ throw "failed to read dual string data";
+ }
+
+ return ( char * ) string_buffer;
+ }
+
+ /* whack_2string
+ */
+ template < class T > static
+ void whack_2string ( const T & eh, char * string_buffer )
+ {
+ delete [] string_buffer;
+ }
+
+ template <>
+ void whack_2string < :: gw_2string_evt_v1 > ( const :: gw_2string_evt_v1 & eh, char * string_buffer )
+ {
+ uint32_t * buffer = ( uint32_t * ) string_buffer;
+ delete [] buffer;
+ }
+
+ /* read_colname
+ */
+ template < class T > static
+ char * read_colname ( const T & eh, FILE * in )
+ {
+ size_t string_size = name_size ( eh );
+ char * string_buffer = new char [ string_size ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size, in );
+ if ( num_read != string_size )
+ {
+ delete [] string_buffer;
+ throw "failed to read column name";
+ }
+
+ return string_buffer;
+ }
+
+ template <>
+ char * read_colname < :: gw_column_evt_v1 > ( const :: gw_column_evt_v1 & eh, FILE * in )
+ {
+ size_t string_size_uint32 = ( name_size ( eh ) + 3 ) / 4;
+ uint32_t * string_buffer = new uint32_t [ string_size_uint32 ];
+ size_t num_read = readFILE ( string_buffer, sizeof string_buffer [ 0 ], string_size_uint32, in );
+ if ( num_read != string_size_uint32 )
+ {
+ delete [] string_buffer;
+ throw "failed to read column name";
+ }
+
+ return ( char * ) string_buffer;
+ }
+
+ /* whack_colname
+ */
+ template < class T > static
+ void whack_colname ( const T & eh, char * string_buffer )
+ {
+ delete [] string_buffer;
+ }
+
+ template <>
+ void whack_colname < :: gw_column_evt_v1 > ( const :: gw_column_evt_v1 & eh, char * string_buffer )
+ {
+ uint32_t * buffer = ( uint32_t * ) string_buffer;
+ delete [] buffer;
+ }
+
+
+ /* check_move_ahead
+ */
+ template < class T > static
+ void check_move_ahead ( const T & eh )
+ {
+ if ( id ( eh . dad ) == 0 )
+ throw "bad table id within move-ahead event (null)";
+ if ( id ( eh . dad ) > tbl_entries . size () )
+ throw "bad table id within move-ahead event";
+ }
+
+ /* dump_move_ahead
+ */
+ template < class D, class T > static
+ void dump_move_ahead ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( eh . nrows, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read move-ahead event";
+
+ check_move_ahead ( eh );
+
+ // advance row-id
+ tbl_entry & te = tbl_entries [ id ( eh . dad ) - 1 ];
+ te . row_id += get_nrows ( eh );
+
+ if ( display )
+ {
+ const std :: string & tbl_name = te . tbl_name;
+
+ std :: cout
+ << event_num << ": move-ahead\n"
+ << " table_id = " << id ( eh . dad ) << " ( \"" << tbl_name << "\" )\n"
+ << " nrows = " << get_nrows ( eh ) << '\n'
+ << " row_id = " << te . row_id << '\n'
+ ;
+ }
+ }
+
+
+ /* check_next_row
+ * all:
+ * 0 < id <= count(tbls)
+ */
+ template < class T > static
+ void check_next_row ( const T & eh )
+ {
+ if ( id ( eh ) == 0 )
+ throw "bad table id within next-row event (null)";
+ if ( id ( eh ) > tbl_entries . size () )
+ throw "bad table id within next-row event";
+ }
+
+ /* dump_next_row
+ */
+ template < class T > static
+ void dump_next_row ( FILE * in, const T & eh )
+ {
+ check_next_row ( eh );
+
+ // advance row-id
+ tbl_entry & te = tbl_entries [ id ( eh ) - 1 ];
+ ++ te . row_id;
+
+ if ( display )
+ {
+ const std :: string & tbl_name = te . tbl_name;
+
+ std :: cout
+ << event_num << ": next-row\n"
+ << " table_id = " << id ( eh ) << " ( \"" << tbl_name << "\" )\n"
+ << " row_id = " << te . row_id << '\n'
+ ;
+ }
+ }
+
+ /* check_empty_default
+ */
+ template < class T > static
+ void check_empty_default ( const T & eh )
+ {
+ if ( id ( eh ) == 0 )
+ throw "bad cell event id (null)";
+ if ( id ( eh ) > col_entries . size () )
+ throw "bad cell event id";
+ }
+
+ /* dump_empty_default
+ */
+ template < class T > static
+ void dump_empty_default ( FILE * in, const T & eh )
+ {
+ check_empty_default ( eh );
+
+ if ( display )
+ {
+ const col_entry & entry = col_entries [ id ( eh ) - 1 ];
+ const std :: string & tbl_name = tbl_entries [ entry . table_id - 1 ] . tbl_name;
+
+ std :: cout
+ << event_num << ": cell-default\n"
+ << " stream_id = " << id ( eh ) << " ( " << tbl_name << " . " << entry . spec << " )\n"
+ << " elem_bits = " << entry . elem_bits << '\n'
+ << " elem_count = 0 ( empty )\n"
+ ;
+ }
+ }
+
+
+ /* check_cell_event
+ * all:
+ * 0 < id <= count ( columns )
+ */
+ template < class T > static
+ void check_cell_event ( const T & eh )
+ {
+ if ( id ( eh . dad ) == 0 )
+ throw "bad cell event id (null)";
+ if ( id ( eh . dad ) > col_entries . size () )
+ throw "bad cell event id";
+ }
+
+ /* check_int_packing
+ * deeply check contents for adherance to protocol
+ */
+ template < class T >
+ int decode_int ( const uint8_t * start, const uint8_t * end, T * decoded );
+
+ template <>
+ int decode_int < uint16_t > ( const uint8_t * start, const uint8_t * end, uint16_t * decoded )
+ {
+ return decode_uint16 ( start, end, decoded );
+ }
+
+ template <>
+ int decode_int < uint32_t > ( const uint8_t * start, const uint8_t * end, uint32_t * decoded )
+ {
+ return decode_uint32 ( start, end, decoded );
+ }
+
+ template <>
+ int decode_int < uint64_t > ( const uint8_t * start, const uint8_t * end, uint64_t * decoded )
+ {
+ return decode_uint64 ( start, end, decoded );
+ }
+
+ template < class T > static
+ size_t check_int_packing ( const uint8_t * data_buffer, size_t data_size )
+ {
+ const uint8_t * start = data_buffer;
+ const uint8_t * end = data_buffer + data_size;
+
+ size_t unpacked_size;
+ for ( unpacked_size = 0; start < end; unpacked_size += sizeof ( T ) )
+ {
+ T decoded;
+ int num_read = decode_int < T > ( start, end, & decoded );
+ if ( num_read <= 0 )
+ {
+ switch ( num_read )
+ {
+ case CODEC_INSUFFICIENT_BUFFER:
+ throw "truncated data in packed integer buffer";
+ case CODEC_INVALID_FORMAT:
+ throw "corrupt data in packed integer buffer";
+ case CODEC_UNKNOWN_ERROR:
+ throw "unknown error in packed integer buffer";
+ default:
+ throw "INTERNAL ERROR: decode_uintXX returned invalid error code";
+ }
+ }
+ start += num_read;
+ }
+
+ return unpacked_size;
+ }
+
+
+ /* dump_cell_event
+ */
+ template < class D, class T > static
+ void dump_cell_event ( FILE * in, const D & e, const char * type )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read cell event";
+
+ check_cell_event ( eh );
+
+ size_t data_size = size ( eh );
+ uint8_t * data_buffer = new uint8_t [ data_size ];
+ num_read = readFILE ( data_buffer, sizeof data_buffer [ 0 ], data_size, in );
+ if ( num_read != data_size )
+ {
+ delete [] data_buffer;
+ throw "failed to read cell data";
+ }
+
+ bool packed_int = false;
+ size_t unpacked_size = data_size;
+ const col_entry & entry = col_entries [ id ( eh . dad ) - 1 ];
+
+ if ( ( entry . flag_bits & 1 ) != 0 )
+ {
+ size_t data_size = size ( eh );
+
+ switch ( entry . elem_bits )
+ {
+ case 16:
+ unpacked_size = check_int_packing < uint16_t > ( data_buffer, data_size );
+ break;
+ case 32:
+ unpacked_size = check_int_packing < uint32_t > ( data_buffer, data_size );
+ break;
+ case 64:
+ unpacked_size = check_int_packing < uint64_t > ( data_buffer, data_size );
+ break;
+ default:
+ throw "bad element size for packed integer";
+ }
+
+ packed_int = true;
+ }
+
+ if ( display )
+ {
+ const std :: string & tbl_name = tbl_entries [ entry . table_id - 1 ] . tbl_name;
+
+ std :: cout
+ << event_num << ": cell-" << type << '\n'
+ << " stream_id = " << id ( eh . dad ) << " ( " << tbl_name << " . " << entry . spec << " )\n"
+ << " elem_bits = " << entry . elem_bits << '\n'
+ ;
+ if ( packed_int )
+ {
+ std :: cout
+ << " elem_count = " << ( unpacked_size * 8 ) / entry . elem_bits
+ << " ( " << unpacked_size << " bytes, " << data_size << " packed )\n"
+ ;
+ }
+ else
+ {
+ std :: cout
+ << " elem_count = " << ( data_size * 8 ) / entry . elem_bits << " ( " << data_size << " bytes )\n"
+ ;
+ }
+ }
+
+ delete [] data_buffer;
+ }
+
+ template <>
+ void dump_cell_event < gw_evt_hdr_v1, gw_data_evt_v1 > ( FILE * in, const gw_evt_hdr_v1 & e, const char * type )
+ {
+ gw_data_evt_v1 eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . elem_count, sizeof eh - sizeof ( gw_evt_hdr_v1 ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read cell event";
+
+ check_cell_event ( eh );
+
+ const col_entry & entry = col_entries [ id ( eh . dad ) - 1 ];
+
+ size_t data_size_uint32 = ( ( uint64_t ) entry . elem_bits * elem_count ( eh ) + 31 ) / 32;
+ uint32_t * data_buffer = new uint32_t [ data_size_uint32 ];
+ num_read = readFILE ( data_buffer, sizeof data_buffer [ 0 ], data_size_uint32, in );
+ if ( num_read != data_size_uint32 )
+ {
+ delete [] data_buffer;
+ throw "failed to read cell data";
+ }
+
+ if ( display )
+ {
+ const std :: string & tbl_name = tbl_entries [ entry . table_id - 1 ] . tbl_name;
+
+ std :: cout
+ << event_num << ": cell-" << type << '\n'
+ << " stream_id = " << id ( eh . dad ) << " ( " << tbl_name << " . " << entry . spec << " )\n"
+ << " elem_bits = " << entry . elem_bits << '\n'
+ << " elem_count = " << elem_count ( eh ) << '\n'
+ ;
+ }
+
+ delete [] data_buffer;
+ }
+
+
+ /* check_open_stream
+ */
+ static
+ void check_open_stream ( const gw_evt_hdr_v1 & eh )
+ {
+ if ( id ( eh ) != 0 )
+ throw "non-zero id within open-stream event";
+ }
+
+ static
+ void check_open_stream ( const gwp_evt_hdr_v1 & eh )
+ {
+ }
+
+
+ /* dump_open_stream
+ */
+ template < class T > static
+ void dump_open_stream ( FILE * in, const T & eh )
+ {
+ check_open_stream ( eh );
+
+ if ( display )
+ {
+ std :: cout
+ << event_num << ": open-stream\n"
+ ;
+ }
+ }
+
+
+ /* check_new_column
+ * all:
+ * id == count ( columns ) + 1
+ * 0 < table_id <= count ( tbls )
+ * length ( name-spec ) != 0
+ * packed:
+ * flags in { 0, 1 }
+ */
+ static
+ void check_new_column ( const gw_column_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) == 0 )
+ throw "bad column/stream id";
+ if ( ( size_t ) id ( eh . dad ) <= col_entries . size () )
+ throw "column id already specified";
+ if ( ( size_t ) id ( eh . dad ) - 1 > col_entries . size () )
+ throw "column id out of order";
+ if ( table_id ( eh ) == 0 )
+ throw "bad column table-id (null)";
+ if ( table_id ( eh ) > tbl_entries . size () )
+ throw "bad column table-id";
+ if ( name_size ( eh ) == 0 )
+ throw "empty column name";
+ }
+
+ static
+ void check_new_column ( const gwp_column_evt_v1 & eh )
+ {
+ if ( ( size_t ) id ( eh . dad ) <= col_entries . size () )
+ throw "column id already specified";
+ if ( ( size_t ) id ( eh . dad ) - 1 > col_entries . size () )
+ throw "column id out of order";
+ if ( table_id ( eh ) == 0 )
+ throw "bad column table-id (null)";
+ if ( table_id ( eh ) > tbl_entries . size () )
+ throw "bad column table-id";
+ if ( name_size ( eh ) == 0 )
+ throw "empty column name";
+
+ if ( ( eh . flag_bits & 0xFE ) != 0 )
+ throw "uninitialized flag_bits";
+ }
+
+
+ /* dump_new_column
+ */
+ template < class D, class T > static
+ void dump_new_column ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . table_id, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read new-column event";
+
+ check_new_column ( eh );
+
+ char * string_buffer = read_colname ( eh, in );
+ std :: string name ( string_buffer, name_size ( eh ) );
+ col_entries . push_back ( col_entry ( table_id ( eh ), name, elem_bits ( eh ), flag_bits ( eh ) ) );
+
+ if ( display )
+ {
+ const std :: string & tbl_name = tbl_entries [ table_id ( eh ) - 1 ] . tbl_name;
+
+ std :: cout
+ << event_num << ": new-column\n"
+ << " table_id = " << table_id ( eh ) << " ( \"" << tbl_name << "\" )\n"
+ << " column_name [ " << name_size ( eh ) << " ] = \"" << name << "\"\n"
+ ;
+ }
+
+ whack_colname ( eh, string_buffer );
+ }
+
+ /* check_new_table
+ * all:
+ * id == count ( tbls ) + 1
+ * length ( name ) != 0
+ */
+ template < class T > static
+ void check_new_table ( const T & eh )
+ {
+ if ( id ( eh . dad ) == 0 )
+ throw "bad table id";
+ if ( ( size_t ) id ( eh . dad ) <= tbl_entries . size () )
+ throw "table id already specified";
+ if ( ( size_t ) id ( eh . dad ) - 1 > tbl_entries . size () )
+ throw "table id out of order";
+ if ( size ( eh ) == 0 )
+ throw "empty table name";
+ }
+
+ /* dump_new_table
+ */
+ template < class D, class T > static
+ void dump_new_table ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read new-table event";
+
+ check_new_table ( eh );
+
+ char * string_buffer = read_1string ( eh, in );
+ std :: string name ( string_buffer, size ( eh ) );
+ tbl_entries . push_back ( tbl_entry ( name ) );
+
+ if ( display )
+ {
+ std :: cout
+ << event_num << ": new-table\n"
+ << " table_name [ " << size ( eh ) << " ] = \"" << name << "\"\n"
+ ;
+ }
+
+ whack_1string ( eh, string_buffer );
+ }
+
+ /* check_use_schema
+ * non-packed:
+ * id == 0
+ * all:
+ * length ( schema-path ) != 0
+ * length ( schema-spec ) != 0
+ */
+ template < class T > static
+ void check_use_schema ( const T & eh )
+ {
+ if ( size1 ( eh ) == 0 )
+ throw "empty schema file path";
+ if ( size2 ( eh ) == 0 )
+ throw "empty schema spec";
+ }
+
+ template < >
+ void check_use_schema < gw_2string_evt_v1 > ( const gw_2string_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "non-zero table id";
+ if ( size1 ( eh ) == 0 )
+ throw "empty schema file path";
+ if ( size2 ( eh ) == 0 )
+ throw "empty schema spec";
+ }
+
+ /* dump_use_schema
+ */
+ template < class D, class T > static
+ void dump_use_schema ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz1, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read use-schema event";
+
+ check_use_schema ( eh );
+
+ char * string_buffer = read_2string ( eh, in );
+
+ if ( display )
+ {
+ std :: string schema_file_name ( string_buffer, size1 ( eh ) );
+ std :: string schema_db_spec ( & string_buffer [ size1 ( eh ) ], size2 ( eh ) );
+ std :: cout
+ << event_num << ": use-schema\n"
+ << " schema_file_name [ " << size1 ( eh ) << " ] = \"" << schema_file_name << "\"\n"
+ << " schema_db_spec [ " << size2 ( eh ) << " ] = \"" << schema_db_spec << "\"\n"
+ ;
+ }
+
+ whack_2string ( eh, string_buffer );
+ }
+
+ /* check_software_name
+ * non-packed:
+ * id == 0
+ * all:
+ * length ( schema-path ) != 0
+ * length ( schema-spec ) != 0
+ */
+
+ void check_vers_component ( const char * vers, const char * end, long num, unsigned long max, char term )
+ {
+ if ( vers == end )
+ throw "bad version";
+ if ( * end != 0 && * end != term )
+ throw "bad version";
+ if ( num < 0 || num > max )
+ throw "bad version";
+ }
+
+ void check_vers ( const char * vers )
+ {
+ char * end;
+ long num = strtol ( vers, & end, 10 );
+ check_vers_component ( vers, end, num, 255, '.' );
+ if ( * end == '.' )
+ {
+ vers = end + 1;
+ num = strtol ( vers, & end, 10 );
+ check_vers_component ( vers, end, num, 255, '.' );
+ if ( * end == '.' )
+ {
+ vers = end + 1;
+ num = strtol ( vers, & end, 10 );
+ check_vers_component ( vers, end, num, 0xFFFF, 0 );
+ }
+ }
+ }
+
+ template < class T > static
+ void check_software_name ( const T & eh )
+ {
+ if ( size1 ( eh ) == 0 )
+ throw "empty software name";
+ if ( size2 ( eh ) == 0 )
+ throw "empty version";
+ }
+
+ template < >
+ void check_software_name < gw_2string_evt_v1 > ( const gw_2string_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "non-zero table id";
+ if ( size1 ( eh ) == 0 )
+ throw "empty software name";
+ if ( size2 ( eh ) == 0 )
+ throw "empty version";
+ }
+
+ /* dump_software_name
+ */
+ template < class D, class T > static
+ void dump_software_name ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz1, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read software_name event";
+
+ check_software_name ( eh );
+
+ char * string_buffer = read_2string ( eh, in );
+ std :: string software_name ( string_buffer, size1 ( eh ) );
+ std :: string version ( & string_buffer [ size1 ( eh ) ], size2 ( eh ) );
+
+ check_vers ( version . c_str () );
+
+ if ( display )
+ {
+ std :: cout
+ << event_num << ": software-name\n"
+ << " software_name [ " << size1 ( eh ) << " ] = \"" << software_name << "\"\n"
+ << " version [ " << size2 ( eh ) << " ] = \"" << version << "\"\n"
+ ;
+ }
+
+ whack_2string ( eh, string_buffer );
+ }
+
+ /* check_metadata_node
+ *
+ */
+ template < class T > static
+ void check_metadata_node ( const T & eh )
+ {
+ if ( size1 ( eh ) == 0 )
+ throw "empty metadata node";
+ if ( size2 ( eh ) == 0 )
+ throw "empty value";
+ }
+
+ template <>
+ void check_metadata_node < gw_2string_evt_v1 > ( const gw_2string_evt_v1 & eh )
+ {
+ if ( size1 ( eh ) == 0 )
+ throw "empty metadata node";
+ if ( size2 ( eh ) == 0 )
+ throw "empty value";
+ }
+
+ /* dump_metadata_node
+ */
+ template < class D, class T > static
+ void dump_metadata_node ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz1, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read metadata_node event";
+
+ check_metadata_node ( eh );
+
+ char *string_buffer = read_2string ( eh, in );
+ std :: string node_path ( string_buffer, size1 ( eh ) );
+ std :: string value ( & string_buffer [ size1 ( eh ) ], size2 ( eh ) );
+
+ if ( display )
+ {
+ std :: cout
+ << event_num << ": metadata-node\n"
+ << " metadata_node [ " << size1 ( eh ) << " ] = \"" << node_path << "\"\n"
+ << " value [ " << size2 ( eh ) << " ] = \"" << value << "\"\n";
+ }
+
+ whack_1string ( eh, string_buffer );
+ }
+
+ /* check_add_mbr
+ *
+ */
+ template < class T > static
+ void check_add_mbr ( const T & eh )
+ {
+ if ( db_id ( eh ) < 0 || db_id ( eh ) > db_entries . size () )
+ throw "invalid database id";
+ if ( size1 ( eh ) == 0 )
+ throw "empty member node";
+ if ( size2 ( eh ) == 0 )
+ throw "empty value";
+ switch ( create_mode ( eh ) & kcmValueMask )
+ {
+ // only allowed mode values
+ // or throw an exception
+ case kcmOpen:
+ case kcmInit:
+ case kcmCreate:
+ break;
+ default:
+ throw "invalid create mode";
+ }
+ if ( ( create_mode ( eh ) & ( kcmBitMask & ~ kcmMD5 & ~ kcmParents ) ) != 0 )
+ throw "invalid create mode";
+ }
+
+ template <>
+ void check_add_mbr < gw_add_mbr_evt_v1 > ( const gw_add_mbr_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "non-zero table id";
+ if ( db_id ( eh ) < 0 || db_id ( eh ) > db_entries . size () )
+ throw "invalid database id";
+ if ( size1 ( eh ) == 0 )
+ throw "empty member name";
+ if ( size2 ( eh ) == 0 )
+ throw "empty db/tbl name";
+ switch ( create_mode ( eh ) & kcmValueMask )
+ {
+ // only allowed mode values
+ // or throw an exception
+ case kcmOpen:
+ case kcmInit:
+ case kcmCreate:
+ break;
+ default:
+ throw "invalid create mode";
+ }
+ if ( ( create_mode ( eh ) & ( kcmBitMask & ~ kcmMD5 & ~ kcmParents ) ) != 0 )
+ throw "invalid create mode";
+ }
+
+ /* dump_add_mbr
+ */
+ template < class D, class T > static
+ void dump_add_mbr ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . db_id, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read add_mbr event";
+
+ check_add_mbr ( eh );
+
+ uint32_t dbid = db_id ( eh );
+ char *string_buffer = read_2string ( eh, in );
+ std :: string member_name ( string_buffer, size1 ( eh ) );
+ std :: string db_tbl_name ( & string_buffer [ size1 ( eh ) ], size2 ( eh ) );
+ uint8_t cmode = create_mode ( eh );
+
+ switch ( evt ( eh . dad ) )
+ {
+ case evt_add_mbr_db:
+ db_entries . push_back ( db_entry ( member_name, db_tbl_name ) );
+ break;
+ case evt_add_mbr_tbl:
+ tbl_entries . push_back ( tbl_entry ( member_name, db_tbl_name ) );
+ break;
+ default:
+ throw "logic error";
+ }
+
+ if ( display )
+ {
+ std :: cout
+ << event_num << ": add-member\n"
+ << " db_id [ " << dbid << " ]\n"
+ << " add_mbr [ " << size1 ( eh ) << " ] = \"" << member_name << "\"\n"
+ << " db/tbl [ " << size2 ( eh ) << " ] = \"" << db_tbl_name << "\"\n"
+ << " create_mode [ "
+ << ( uint32_t ) cmode
+ << " ] ( ";
+ switch ( cmode & kcmValueMask )
+ {
+ case kcmOpen:
+ std :: cout << "kcmOpen";
+ break;
+ case kcmInit:
+ std :: cout << "kcmInit";
+ break;
+ case kcmCreate:
+ std :: cout << "kcmCreate";
+ break;
+ }
+ if ( ( cmode & kcmParents ) != 0 )
+ std :: cout << ", kcmParents";
+ if ( ( cmode & kcmMD5 ) != 0 )
+ std :: cout << ", kcmMD5";
+ std :: cout << " )\n";
+ }
+
+ whack_2string ( eh, string_buffer );
+ }
+
+ /* check_remote_path
+ * non-packed:
+ * id == 0
+ * all:
+ * length ( remote-path ) != 0
+ */
+ template < class T > static
+ void check_remote_path ( const T & eh )
+ {
+ if ( size ( eh ) == 0 )
+ throw "empty remote path";
+ }
+
+ template <>
+ void check_remote_path < gw_1string_evt_v1 > ( const gw_1string_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "non-zero table id";
+ if ( size ( eh ) == 0 )
+ throw "empty remote path";
+ }
+
+ /* dump_remote_path
+ */
+ template < class D, class T > static
+ void dump_remote_path ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read remote-path event";
+
+ check_remote_path ( eh );
+
+ char * string_buffer = read_1string ( eh, in );
+
+ if ( display )
+ {
+ std :: string path ( string_buffer, size ( eh ) );
+ std :: cout
+ << event_num << ": remote-path\n"
+ << " remote_db_name [ " << size ( eh ) << " ] = \"" << path << "\"\n"
+ ;
+ }
+
+ whack_1string ( eh, string_buffer );
+ }
+
+ /* check_end_stream
+ * non-packed:
+ * id == 0
+ */
+ static
+ void check_end_stream ( const gw_evt_hdr_v1 & eh )
+ {
+ if ( id ( eh ) != 0 )
+ throw "non-zero id within end-stream event";
+ }
+
+ static
+ void check_end_stream ( const gwp_evt_hdr_v1 & eh )
+ {
+ }
+
+ /* dump_end_stream
+ */
+ template < class T > static
+ bool dump_end_stream ( FILE * in, const T & eh )
+ {
+ check_end_stream ( eh );
+ if ( display )
+ {
+ std :: cout
+ << "END\n"
+ ;
+ }
+ return false;
+ }
+
+ /* check_errmsg
+ * non-packed
+ * id == 0
+ * all:
+ * length ( msg ) != 0
+ */
+ template < class T > static
+ void check_errmsg ( const T & eh )
+ {
+ if ( size ( eh ) == 0 )
+ throw "empty error message";
+ }
+
+ template <>
+ void check_errmsg < gw_1string_evt_v1 > ( const gw_1string_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "bad error-message id ( should be 0 )";
+ if ( size ( eh ) == 0 )
+ throw "empty error message";
+ }
+
+ /* dump_errmsg
+ */
+ template < class D, class T > static
+ void dump_errmsg ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read error-message event";
+
+ check_errmsg ( eh );
+
+ char * string_buffer = read_1string ( eh, in );
+
+ if ( display )
+ {
+ std :: string msg ( string_buffer, size ( eh ) );
+
+ std :: cout
+ << event_num << ": error-message\n"
+ << " msg [ " << size ( eh ) << " ] = \"" << msg << "\"\n"
+ ;
+ }
+
+ whack_1string ( eh, string_buffer );
+ }
+
+ /* check_logmsg
+ * non-packed
+ * id == 0
+ * all:
+ * length ( msg ) != 0
+ */
+ template < class T > static
+ void check_logmsg ( const T & eh )
+ {
+ if ( size ( eh ) == 0 )
+ throw "empty log message";
+ }
+
+ template <>
+ void check_logmsg < gw_1string_evt_v1 > ( const gw_1string_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "bad log-message id ( should be 0 )";
+ if ( size ( eh ) == 0 )
+ throw "empty log message";
+ }
+
+ /* dump_logmsg
+ */
+ template < class D, class T > static
+ void dump_logmsg ( FILE * in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & eh . sz, sizeof eh - sizeof ( D ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read log-message event";
+
+ check_logmsg ( eh );
+
+ char * string_buffer = read_1string ( eh, in );
+
+ if ( display )
+ {
+ std :: string msg ( string_buffer, size ( eh ) );
+
+ std :: cout
+ << event_num << ": log-message\n"
+ << " msg [ " << size ( eh ) << " ] = \"" << msg << "\"\n"
+ ;
+ }
+
+ whack_1string ( eh, string_buffer );
+ }
+
+ /* check_progmsg
+ * non-packed
+ * id == 0
+ * all:
+ * length ( msg ) != 0
+ * pid != 0
+ * timestamp > 0
+ * version > 0
+ * percent > 0 && < 100
+ */
+ template < class T > static
+ void check_progmsg ( const T & eh )
+ {
+ if ( size ( eh ) == 0 )
+ throw "empty prog message";
+ if ( pid ( eh ) == 0 )
+ throw "invalid pid";
+ if ( timestamp ( eh ) == 0 )
+ throw "empty timestamp";
+ if ( version ( eh ) == 0 )
+ throw "invalid version number";
+ if ( percent ( eh ) < 0 || percent ( eh ) > 100 )
+ throw "invalid percent";
+ }
+
+ template <>
+ void check_progmsg < gw_status_evt_v1 > ( const gw_status_evt_v1 & eh )
+ {
+ if ( id ( eh . dad ) != 0 )
+ throw "bad prog-message id ( should be 0 )";
+ if ( size ( eh ) == 0 )
+ throw "empty prog message";
+ if ( pid ( eh ) == 0 )
+ throw "invlaid pid";
+ if ( timestamp ( eh ) == 0 )
+ throw "empty timestamp";
+ if ( version ( eh ) == 0 )
+ throw "invalid version number";
+ if ( percent ( eh ) < 0 || percent ( eh ) > 100 )
+ throw "invalid percent";
+ }
+
+ /* dump-progmsg
+ */
+ template < class D, class T > static
+ void dump_progmsg ( FILE *in, const D & e )
+ {
+ T eh;
+ init ( eh, e );
+
+ size_t num_read = readFILE ( & ( & eh . dad ) [ 1 ], sizeof eh - sizeof eh . dad, 1, in );
+ if ( num_read != 1 )
+ throw "failed to read prog-message event";
+
+ check_progmsg ( eh );
+
+ char *string_buffer = read_1string ( eh, in );
+ std :: string app_name ( string_buffer, size ( eh ) );
+ uint32_t _pid = pid ( eh );
+ uint32_t _timestamp = timestamp ( eh );
+ uint32_t _version = version ( eh );
+ uint32_t _percent = percent ( eh );
+
+ if ( display )
+ {
+ time_t ts = ( time_t ) _timestamp;
+ char time_str [ 256 ];
+ asctime_r ( localtime ( & ts ), time_str );
+ size_t len = strlen ( time_str );
+ while ( len > 0 && time_str [ len - 1 ] == '\n' )
+ time_str [ -- len ] = 0;
+
+ std :: cout
+ << event_num << ": prog-msg\n"
+ << " app [ " << app_name << " ] \n"
+ << " message [ proccessed " << _percent << "% ] \n"
+ << " pid [ " << _pid << " ]\n"
+ << " timestamp [ " << time_str << " ( " << _timestamp << " ) ] \n"
+ << " version [ " << ( _version >> 24 ) << '.' << ( ( _version >> 16 ) & 0xFF ) << '.' << ( _version & 0xFFFF ) << " ] \n"
+ << " percent [ " << _percent << " ]\n ";
+ }
+
+ whack_1string ( eh, string_buffer );
+ }
+
+ /* dump_v1_event
+ * the events are not packed
+ */
+ static
+ bool dump_v1_event ( FILE * in )
+ {
+ if ( jump_event == event_num )
+ display = true;
+ else if ( end_event == event_num )
+ display = false;
+
+ gw_evt_hdr_v1 e;
+ memset ( & e, 0, sizeof e );
+
+ size_t num_read = readFILE ( & e, sizeof e, 1, in );
+ if ( num_read != 1 )
+ {
+ int ch = fgetc ( in );
+ if ( ch == EOF )
+ throw "EOF before end-stream";
+
+ throw "failed to read event";
+ }
+ switch ( evt ( e ) )
+ {
+ case evt_bad_event:
+ throw "illegal event id - possibly block of zeros";
+ case evt_errmsg:
+ dump_errmsg < gw_evt_hdr_v1, gw_1string_evt_v1 > ( in, e );
+ break;
+ case evt_end_stream:
+ return dump_end_stream ( in, e );
+ case evt_remote_path:
+ dump_remote_path < gw_evt_hdr_v1, gw_1string_evt_v1 > ( in, e );
+ break;
+ case evt_use_schema:
+ dump_use_schema < gw_evt_hdr_v1, gw_2string_evt_v1 > ( in, e );
+ break;
+ case evt_new_table:
+ dump_new_table < gw_evt_hdr_v1, gw_1string_evt_v1 > ( in, e );
+ break;
+ case evt_new_column:
+ dump_new_column < gw_evt_hdr_v1, gw_column_evt_v1 > ( in, e );
+ break;
+ case evt_open_stream:
+ dump_open_stream ( in, e );
+ break;
+ case evt_cell_default:
+ dump_cell_event < gw_evt_hdr_v1, gw_data_evt_v1 > ( in, e, "default" );
+ break;
+ case evt_cell_data:
+ dump_cell_event < gw_evt_hdr_v1, gw_data_evt_v1 > ( in, e, "data" );
+ break;
+ case evt_next_row:
+ dump_next_row ( in, e );
+ break;
+ case evt_move_ahead:
+ dump_move_ahead < gw_evt_hdr_v1, gw_move_ahead_evt_v1 > ( in, e );
+ break;
+ case evt_errmsg2:
+ case evt_remote_path2:
+ case evt_use_schema2:
+ case evt_new_table2:
+ case evt_cell_default2:
+ case evt_cell_data2:
+ throw "packed event id within non-packed stream";
+ case evt_empty_default:
+ dump_empty_default < gw_evt_hdr_v1 > ( in, e );
+ break;
+
+ // add in new message handlers for version 2
+ case evt_software_name:
+ dump_software_name < gw_evt_hdr_v1, gw_2string_evt_v1 > ( in, e );
+ break;
+ case evt_db_metadata_node:
+ dump_metadata_node < gw_evt_hdr_v1, gw_2string_evt_v1 > ( in, e );
+ break;
+ case evt_tbl_metadata_node:
+ dump_metadata_node < gw_evt_hdr_v1, gw_2string_evt_v1 > ( in, e );
+ break;
+ case evt_col_metadata_node:
+ dump_metadata_node < gw_evt_hdr_v1, gw_2string_evt_v1 > ( in, e );
+ break;
+ case evt_db_metadata_node2:
+ case evt_tbl_metadata_node2:
+ case evt_col_metadata_node2:
+ throw "packed event id within non-packed stream";
+ case evt_add_mbr_db:
+ dump_add_mbr < gw_evt_hdr_v1, gw_add_mbr_evt_v1 > ( in, e );
+ break;
+ case evt_add_mbr_tbl:
+ dump_add_mbr < gw_evt_hdr_v1, gw_add_mbr_evt_v1 > ( in, e );
+ break;
+
+ case evt_logmsg:
+ dump_logmsg < gw_evt_hdr_v1, gw_1string_evt_v1 > ( in, e );
+ break;
+ case evt_progmsg:
+ dump_progmsg < gw_evt_hdr_v1, gw_status_evt_v1 > ( in, e );
+ break;
+
+ default:
+ throw "unrecognized event id";
+ }
+ return true;
+ }
+
+ /* dump_v1_packed_event
+ * the events are all packed
+ */
+ static
+ bool dump_v1_packed_event ( FILE * in )
+ {
+ if ( jump_event == event_num )
+ display = true;
+ else if ( end_event == event_num )
+ display = false;
+
+ gwp_evt_hdr_v1 e;
+ memset ( & e, 0, sizeof e );
+
+ size_t num_read = readFILE ( & e, sizeof e, 1, in );
+ if ( num_read != 1 )
+ {
+ int ch = fgetc ( in );
+ if ( ch == EOF )
+ throw "EOF before end-stream";
+
+ throw "failed to read event";
+ }
+ switch ( evt ( e ) )
+ {
+ case evt_bad_event:
+ throw "illegal event id - possibly block of zeros";
+ case evt_errmsg:
+ dump_errmsg < gwp_evt_hdr_v1, gwp_1string_evt_v1 > ( in, e );
+ break;
+ case evt_end_stream:
+ return dump_end_stream ( in, e );
+ case evt_remote_path:
+ dump_remote_path < gwp_evt_hdr_v1, gwp_1string_evt_v1 > ( in, e );
+ break;
+ case evt_use_schema:
+ dump_use_schema < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_new_table:
+ dump_new_table < gwp_evt_hdr_v1, gwp_1string_evt_v1 > ( in, e );
+ break;
+ case evt_new_column:
+ dump_new_column < gwp_evt_hdr_v1, gwp_column_evt_v1 > ( in, e );
+ break;
+ case evt_open_stream:
+ dump_open_stream ( in, e );
+ break;
+ case evt_cell_default:
+ dump_cell_event < gwp_evt_hdr_v1, gwp_data_evt_v1 > ( in, e, "default" );
+ break;
+ case evt_cell_data:
+ dump_cell_event < gwp_evt_hdr_v1, gwp_data_evt_v1 > ( in, e, "data" );
+ break;
+ case evt_next_row:
+ dump_next_row ( in, e );
+ break;
+ case evt_move_ahead:
+ dump_move_ahead < gwp_evt_hdr_v1, gwp_move_ahead_evt_v1 > ( in, e );
+ break;
+ case evt_errmsg2:
+ dump_errmsg < gwp_evt_hdr_v1, gwp_1string_evt_U16_v1 > ( in, e );
+ break;
+ case evt_remote_path2:
+ dump_remote_path < gwp_evt_hdr_v1, gwp_1string_evt_U16_v1 > ( in, e );
+ break;
+ case evt_use_schema2:
+ dump_use_schema < gwp_evt_hdr_v1, gwp_2string_evt_U16_v1 > ( in, e );
+ break;
+ case evt_new_table2:
+ dump_new_table < gwp_evt_hdr_v1, gwp_1string_evt_U16_v1 > ( in, e );
+ break;
+ case evt_cell_default2:
+ dump_cell_event < gwp_evt_hdr_v1, gwp_data_evt_U16_v1 > ( in, e, "default" );
+ break;
+ case evt_cell_data2:
+ dump_cell_event < gwp_evt_hdr_v1, gwp_data_evt_U16_v1 > ( in, e, "data" );
+ break;
+ case evt_empty_default:
+ dump_empty_default < gwp_evt_hdr_v1 > ( in, e );
+ break;
+
+ // add in new message handlers for version 2
+ case evt_software_name:
+ dump_software_name < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_db_metadata_node:
+ dump_metadata_node < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_tbl_metadata_node:
+ dump_metadata_node < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_col_metadata_node:
+ dump_metadata_node < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_db_metadata_node2:
+ dump_metadata_node < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_tbl_metadata_node2:
+ dump_metadata_node < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_col_metadata_node2:
+ dump_metadata_node < gwp_evt_hdr_v1, gwp_2string_evt_v1 > ( in, e );
+ break;
+ case evt_add_mbr_db:
+ dump_add_mbr < gwp_evt_hdr_v1, gwp_add_mbr_evt_v1 > ( in, e );
+ break;
+ case evt_add_mbr_tbl:
+ dump_add_mbr < gwp_evt_hdr_v1, gwp_add_mbr_evt_v1 > ( in, e );
+ break;
+ case evt_logmsg:
+ dump_logmsg < gwp_evt_hdr_v1, gwp_1string_evt_U16_v1 > ( in, e );
+ break;
+ case evt_progmsg:
+ dump_progmsg < gwp_evt_hdr_v1, gwp_status_evt_v1 > ( in, e );
+ break;
+
+ default:
+ throw "unrecognized packed event id";
+ }
+ return true;
+ }
+
+ static
+ void check_v1_header ( const gw_header_v1 & hdr )
+ {
+ if ( hdr . packing > 1 )
+ throw "bad packing spec";
+ }
+
+ static
+ void dump_v1_header ( FILE * in, const gw_header & dad, bool & packed )
+ {
+ gw_header_v1 hdr;
+ init ( hdr, dad );
+
+ size_t num_read = readFILE ( & hdr . packing, sizeof hdr - sizeof ( gw_header ), 1, in );
+ if ( num_read != 1 )
+ throw "failed to read v1 header";
+
+ check_v1_header ( hdr );
+
+ if ( hdr . packing )
+ packed = true;
+
+ if ( display )
+ {
+ std :: cout
+ << "header: version " << hdr . dad . version << '\n'
+ << " hdr_size = " << hdr . dad . hdr_size << '\n'
+ << " packing = " << hdr . packing << '\n'
+ ;
+ }
+ }
+
+ static
+ void check_header ( const gw_header & hdr )
+ {
+ if ( memcmp ( hdr . signature, GW_SIGNATURE, sizeof hdr . signature ) != 0 )
+ throw "bad header signature";
+ if ( hdr . endian != GW_GOOD_ENDIAN )
+ {
+ if ( hdr . endian != GW_REVERSE_ENDIAN )
+ throw "bad header byte order";
+ throw "reversed header byte order";
+ }
+ if ( hdr . version < 1 )
+ throw "bad header version";
+ if ( hdr . version > GW_CURRENT_VERSION )
+ throw "unknown header version";
+ }
+
+ static
+ uint32_t dump_header ( FILE * in, bool & packed )
+ {
+ gw_header hdr;
+ size_t num_read = readFILE ( & hdr, sizeof hdr, 1, in );
+ if ( num_read != 1 )
+ throw "failed to read header";
+
+ check_header ( hdr );
+
+ switch ( hdr . version )
+ {
+ case 1:
+ case 2:
+ dump_v1_header ( in, hdr, packed );
+ break;
+ default:
+ throw "UNIMPLEMENTED: missing new version dumper";
+ }
+ return hdr . version;
+ }
+
+ static
+ void dump ( FILE * in )
+ {
+ foffset = 0;
+
+ bool packed = false;
+ uint32_t version = dump_header ( in, packed );
+
+ event_num = 1;
+ switch ( version )
+ {
+ case 1:
+ case 2:
+
+ if ( packed )
+ {
+ while ( dump_v1_packed_event ( in ) )
+ ++ event_num;
+ }
+ else
+ {
+ while ( dump_v1_event ( in ) )
+ ++ event_num;
+ }
+ break;
+ }
+
+ int ch = fgetc ( in );
+ if ( ch != EOF )
+ throw "excess data after end-stream";
+ }
+
+ static
+ const char * nextArg ( int & i, int argc, char * argv [] )
+ {
+ if ( ++ i >= argc )
+ throw "expected argument";
+ return argv [ i ];
+ }
+
+ static
+ const char * nextArg ( const char * & argp, int & i, int argc, char * argv [] )
+ {
+ const char * arg = argp;
+ argp = "\0";
+
+ if ( arg [ 1 ] != 0 )
+ return arg + 1;
+
+ return nextArg ( i, argc, argv );
+ }
+
+ static
+ uint64_t atoU64 ( const char * str )
+ {
+ char * end;
+ long i = strtol ( str, & end, 0 );
+ if ( end [ 0 ] != 0 )
+ throw "badly formed number";
+ if ( i < 0 )
+ throw "number out of bounds";
+ return ( uint64_t ) i;
+ }
+
+ static
+ void help ( const char * tool_path )
+ {
+ const char * tool_name = strrchr ( tool_path, '/' );
+ if ( tool_name ++ == NULL )
+ tool_name = tool_path;
+
+ std :: cout
+ << '\n'
+ << "Usage:\n"
+ << " " << tool_name << " [options] [<stream-file> ...]\n"
+ << '\n'
+ << "Summary:\n"
+ << " This is a tool to analyze and display the contents of a stream produced by\n"
+ << " the \"general-writer\" library.\n"
+ << '\n'
+ << " Input may be taken from stdin ( DEFAULT ) or from one or more stream-files.\n"
+ << '\n'
+ << "Options:\n"
+ << " -j event-num jump to numbered event before displaying.\n"
+ << " ( event numbers are 1-based, so the first event is 1. )\n"
+ << " -N event-count display a limited number of events and then go quiet.\n"
+ << " -v increase verbosity. Use multiple times for increased verbosity.\n"
+ << " ( currently this only enables or disables display. )\n"
+ << " -h display this help message\n"
+ << '\n'
+ << tool_path << '\n'
+ ;
+ }
+
+ static
+ void run ( int argc, char * argv [] )
+ {
+ uint32_t num_files = 0;
+
+ for ( int i = 1; i < argc; ++ i )
+ {
+ const char * arg = argv [ i ];
+ if ( arg [ 0 ] != '-' )
+ argv [ ++ num_files ] = ( char * ) arg;
+ else do switch ( ( ++ arg ) [ 0 ] )
+ {
+ case 'v':
+ ++ verbose;
+ break;
+ case 'j':
+ jump_event = atoU64 ( nextArg ( arg, i, argc, argv ) );
+ break;
+ case 'N':
+ end_event = atoU64 ( nextArg ( arg, i, argc, argv ) );
+ break;
+ case 'h':
+ case '?':
+ help ( argv [ 0 ] );
+ return;
+ default:
+ throw "unrecognized option";
+ }
+ while ( arg [ 1 ] != 0 );
+ }
+
+ if ( verbose && jump_event == 0 )
+ display = true;
+
+ end_event += jump_event;
+
+ if ( num_files == 0 )
+ dump ( stdin );
+ else for ( uint32_t i = 1; i <= num_files; ++ i )
+ {
+ FILE * in = fopen ( argv [ i ], "rb" );
+ if ( in == 0 )
+ std :: cerr << "WARNING: failed to open input file: \"" << argv [ i ] << '\n';
+ else
+ {
+ dump ( in );
+ fclose ( in );
+ }
+ }
+ }
+}
+
+int main ( int argc, char * argv [] )
+{
+ int status = 1;
+ try
+ {
+ gw_dump :: run ( argc, argv );
+ status = 0;
+ }
+ catch ( const char x [] )
+ {
+ std :: cerr
+ << "ERROR: offset "
+ << gw_dump :: foffset
+ << ": event "
+ << gw_dump :: event_num
+ << ": "
+ << x
+ << '\n'
+ ;
+ }
+ catch ( ... )
+ {
+ std :: cerr
+ << "ERROR: offset "
+ << gw_dump :: foffset
+ << ": event "
+ << gw_dump :: event_num
+ << ": unknown error\n"
+ ;
+ }
+
+ return status;
+}
diff --git a/tools/general-loader/libgeneral-writer.vers b/tools/general-loader/libgeneral-writer.vers
new file mode 100644
index 0000000..35d16fb
--- /dev/null
+++ b/tools/general-loader/libgeneral-writer.vers
@@ -0,0 +1 @@
+2.5.7
diff --git a/tools/general-loader/main.cpp b/tools/general-loader/main.cpp
new file mode 100644
index 0000000..8a284a2
--- /dev/null
+++ b/tools/general-loader/main.cpp
@@ -0,0 +1,246 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "general-loader.vers.h"
+
+#include "general-loader.hpp"
+
+#include <sysalloc.h>
+
+#include <string.h>
+#include <stdlib.h>
+#include <limits.h>
+
+#include <iostream>
+
+#include <kapp/main.h>
+#include <kapp/args.h>
+#include <kapp/log-xml.h>
+
+#include <klib/out.h>
+#include <klib/rc.h>
+#include <klib/log.h>
+
+#include <kns/stream.h>
+
+static char const option_include_paths[] = "include";
+#define OPTION_INCLUDE_PATHS option_include_paths
+#define ALIAS_INCLUDE_PATHS "I"
+static
+char const * include_paths_usage[] =
+{
+ "Additional directories to search for schema include files. Can specify multiple paths separated by ':'.",
+ NULL
+};
+
+static char const option_schemas[] = "schema";
+#define OPTION_SCHEMAS option_schemas
+#define ALIAS_SCHEMAS "S"
+static
+char const * schemas_usage[] =
+{
+ "Schema file to use. Can specify multiple files separated by ':'.",
+ NULL
+};
+
+static char const option_target[] = "target";
+#define OPTION_TARGET option_target
+#define ALIAS_TARGET "T"
+static
+char const * target_usage[] =
+{
+ "Database file to create. Overrides any remote path specifications coming from the input stream",
+ NULL
+};
+
+OptDef Options[] =
+{
+ /* order here is same as in param array below!!! */
+ /* max#, needs param, required */
+ { OPTION_INCLUDE_PATHS, ALIAS_INCLUDE_PATHS, NULL, include_paths_usage, 0, true, false },
+ { OPTION_SCHEMAS, ALIAS_SCHEMAS, NULL, schemas_usage, 0, true, false },
+ { OPTION_TARGET, ALIAS_TARGET, NULL, target_usage, 1, true, false },
+};
+
+const char* OptHelpParam[] =
+{
+ /* order here is same as in OptDef array above!!! */
+ "path(s)",
+ "path(s)",
+ "path",
+ "",
+};
+
+rc_t UsageSummary (char const * progname)
+{
+ return KOutMsg (
+ "Usage:\n"
+ "\t%s [options] \n"
+ "\n"
+ "Summary:\n"
+ "\tPopulate a VDB database from standard input\n"
+ "\n"
+ ,progname);
+}
+
+char const UsageDefaultName[] = "general-loader";
+
+rc_t CC Usage (const Args * args)
+{
+ rc_t rc;
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+
+ if (args == NULL)
+ rc = RC (rcApp, rcArgv, rcAccessing, rcSelf, rcNull);
+ else
+ rc = ArgsProgram (args, &fullpath, &progname);
+ if (rc)
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary (progname);
+
+ const size_t argsQty = sizeof(Options) / sizeof(Options[0]);
+ for(size_t i = 0; i < argsQty; i++ ) {
+ if( Options[i].required && Options[i].help[0] != NULL ) {
+ HelpOptionLine(Options[i].aliases, Options[i].name, OptHelpParam[i], Options[i].help);
+ }
+ }
+ OUTMSG(("\nOptions:\n"));
+ for(size_t i = 0; i < argsQty; i++ ) {
+ if( !Options[i].required && Options[i].help[0] != NULL ) {
+ HelpOptionLine(Options[i].aliases, Options[i].name, OptHelpParam[i], Options[i].help);
+ }
+ }
+ XMLLogger_Usage();
+
+ HelpOptionsStandard ();
+ HelpVersion (fullpath, KAppVersion());
+ return rc;
+}
+
+uint32_t CC KAppVersion (void)
+{
+ return GENERAL_LOADER_VERS;
+}
+
+rc_t CC KMain (int argc, char * argv[])
+{
+ Args * args;
+ uint32_t pcount;
+ const XMLLogger* xml_logger = NULL;
+ rc_t rc = ArgsMakeAndHandle (&args, argc, argv, 2
+ , Options, sizeof Options / sizeof (OptDef)
+ , XMLLogger_Args, XMLLogger_ArgsQty);
+
+ if ( rc == 0 )
+ {
+ rc = XMLLogger_Make(&xml_logger, NULL, args);
+ if (rc == 0)
+ {
+ rc = ArgsParamCount (args, &pcount);
+ if ( rc == 0 )
+ {
+ if ( pcount != 0 )
+ {
+ rc = RC(rcApp, rcArgv, rcAccessing, rcParam, rcExcessive);
+ MiniUsage (args);
+ }
+ else
+ {
+ const KStream *std_in;
+ rc = KStreamMakeStdIn ( & std_in );
+ if ( rc == 0 )
+ {
+ KStream* buffered;
+ rc = KStreamMakeBuffered ( &buffered, std_in, 0 /*input-only*/, 0 /*use default size*/ );
+ if ( rc == 0 )
+ {
+ GeneralLoader loader ( argv[0], *buffered );
+
+ rc = ArgsOptionCount (args, OPTION_INCLUDE_PATHS, &pcount);
+ if ( rc == 0 )
+ {
+ for ( uint32_t i = 0 ; i < pcount; ++i )
+ {
+ const void* value;
+ rc = ArgsOptionValue (args, OPTION_INCLUDE_PATHS, i, &value);
+ if ( rc != 0 )
+ {
+ break;
+ }
+ loader . AddSchemaIncludePath ( static_cast <char const*> (value) );
+ }
+ }
+
+ rc = ArgsOptionCount (args, OPTION_SCHEMAS, &pcount);
+ if ( rc == 0 )
+ {
+ for ( uint32_t i = 0 ; i < pcount; ++i )
+ {
+ const void* value;
+ rc = ArgsOptionValue (args, OPTION_SCHEMAS, i, &value);
+ if ( rc != 0 )
+ {
+ break;
+ }
+ loader . AddSchemaFile( static_cast <char const*> (value) );
+ }
+ }
+
+ rc = ArgsOptionCount (args, OPTION_TARGET, &pcount);
+ if ( rc == 0 && pcount == 1 )
+ {
+ const void* value;
+ rc = ArgsOptionValue (args, OPTION_TARGET, 0, &value);
+ if ( rc == 0 )
+ {
+ loader . SetTargetOverride ( static_cast <char const*> (value) );
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ rc = loader . Run();
+ }
+ KStreamRelease ( buffered );
+ }
+ KStreamRelease ( std_in );
+ }
+ }
+ }
+ }
+ }
+
+ if ( rc != 0)
+ {
+ LOGERR ( klogErr, rc, "load failed" );
+ }
+
+ ArgsWhack(args);
+ XMLLogger_Release(xml_logger);
+ return rc;
+}
diff --git a/tools/general-loader/protocol-parser.cpp b/tools/general-loader/protocol-parser.cpp
new file mode 100644
index 0000000..398e3a9
--- /dev/null
+++ b/tools/general-loader/protocol-parser.cpp
@@ -0,0 +1,1048 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "general-loader.hpp"
+
+#include <klib/rc.h>
+#include <klib/log.h>
+
+#include "general-writer.h"
+#include "utf8-like-int-codec.h"
+
+using namespace std;
+
+///////////// GeneralLoader::ProtocolParser
+
+template <typename TEvent>
+rc_t
+GeneralLoader :: ProtocolParser :: ReadEvent ( Reader& p_reader, TEvent& p_event )
+{ // read the part of p_event that is outside of p_event.dad (event header)
+ if ( sizeof p_event > sizeof p_event . dad )
+ {
+ char * start = (char*) & p_event . dad;
+ start += sizeof p_event . dad;
+ return p_reader . Read ( start, sizeof p_event - sizeof p_event . dad );
+ }
+ return 0;
+};
+
+///////////// GeneralLoader::UnpackedProtocolParser
+
+rc_t
+GeneralLoader :: UnpackedProtocolParser :: ParseEvents ( Reader& p_reader, DatabaseLoader& p_dbLoader )
+{
+ rc_t rc;
+ do
+ {
+ p_reader . Align ();
+
+ struct gw_evt_hdr_v1 evt_header;
+ rc = p_reader . Read ( & evt_header, sizeof ( evt_header ) );
+ if ( rc != 0 )
+ {
+ break;
+ }
+
+ switch ( ncbi :: evt ( evt_header ) )
+ {
+ case evt_use_schema:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Use-Schema" );
+
+ gw_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t schema_file_size = ncbi :: size1 ( evt );
+ size_t schema_name_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( schema_file_size + schema_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . UseSchema ( string ( ( const char * ) p_reader . GetBuffer (), schema_file_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + schema_file_size, schema_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_remote_path:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Remote-Path" );
+
+ gw_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t database_name_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( database_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . RemotePath ( string ( ( const char * ) p_reader . GetBuffer (), database_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_software_name:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Software-Name" );
+
+ gw_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t name_size = ncbi :: size1 ( evt );
+ size_t version_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( name_size + version_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . SoftwareName ( string ( ( const char * ) p_reader . GetBuffer (), name_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + name_size, version_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_db_metadata_node:
+ {
+ uint32_t objId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: DB-Metadata-Node, id=$(i)", "i=%u", objId );
+
+ gw_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t node_size = ncbi :: size1 ( evt );
+ size_t value_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( node_size + value_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . DBMetadataNode ( objId,
+ string ( ( const char * ) p_reader . GetBuffer (), node_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + node_size, value_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_tbl_metadata_node:
+ {
+ uint32_t objId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Tbl-Metadata-Node, id=$(i)", "i=%u", objId );
+
+ gw_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t node_size = ncbi :: size1 ( evt );
+ size_t value_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( node_size + value_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . TblMetadataNode ( objId,
+ string ( ( const char * ) p_reader . GetBuffer (), node_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + node_size, value_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_col_metadata_node:
+ {
+ uint32_t objId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Col-Metadata-Node, id=$(i)", "i=%u", objId );
+
+ gw_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t node_size = ncbi :: size1 ( evt );
+ size_t value_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( node_size + value_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ColMetadataNode ( objId,
+ string ( ( const char * ) p_reader . GetBuffer (), node_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + node_size, value_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_new_table:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: New-Table, id=$(i)", "i=%u", tableId );
+
+ gw_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t table_name_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( table_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . NewTable ( tableId, string ( ( const char * ) p_reader . GetBuffer (), table_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_new_column:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: New-Column, id=$(i)", "i=%u", columnId );
+
+ gw_column_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t col_name_size = ncbi :: name_size ( evt );
+ rc = p_reader . Read ( col_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . NewColumn ( columnId,
+ ncbi :: table_id ( evt ),
+ ncbi :: elem_bits ( evt ),
+ ncbi :: flag_bits ( evt ),
+ string ( ( const char * ) p_reader . GetBuffer (), col_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_add_mbr_db:
+ {
+ uint32_t db_id = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Add-Mbr-DB, id=$(i)", "i=%u", db_id );
+
+ gw_add_mbr_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ uint32_t parent_id = ncbi :: db_id ( evt );
+ size_t db_mbr_size = ncbi :: size1 ( evt );
+ size_t db_name_size = ncbi :: size2 ( evt );
+ uint8_t create_mode = ncbi :: create_mode ( evt );
+ rc = p_reader . Read ( db_mbr_size + db_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . AddMbrDB ( db_id,
+ parent_id,
+ string ( ( const char * ) p_reader . GetBuffer (), db_mbr_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + db_mbr_size, db_name_size ),
+ create_mode );
+ }
+ }
+ }
+ break;
+
+ case evt_add_mbr_tbl:
+ {
+ uint32_t db_tbl_id = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Add-Mbr-Table, id=$(i)", "i=%u", db_tbl_id );
+
+ gw_add_mbr_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ uint32_t parent_id = ncbi :: db_id ( evt );
+ size_t db_mbr_size = ncbi :: size1 ( evt );
+ size_t db_name_size = ncbi :: size2 ( evt );
+ uint8_t create_mode = ncbi :: create_mode ( evt );
+ rc = p_reader . Read ( db_mbr_size + db_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . AddMbrTbl ( db_tbl_id,
+ parent_id,
+ string ( ( const char * ) p_reader . GetBuffer (), db_mbr_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + db_mbr_size, db_name_size ),
+ create_mode );
+ }
+ }
+ }
+ break;
+
+ case evt_cell_data:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-Data, id=$(i)", "i=%u", columnId );
+
+ gw_data_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ const DatabaseLoader :: Column* col = p_dbLoader . GetColumn ( columnId );
+ if ( col != 0 )
+ {
+ size_t elem_count = ncbi :: elem_count ( evt );
+ rc = p_reader . Read ( ( col -> elemBits * elem_count + 7 ) / 8 );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . CellData ( columnId, p_reader. GetBuffer (), elem_count );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ }
+ }
+ break;
+
+ case evt_cell_default:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-Default, id=$(i)", "i=%u", columnId );
+
+ gw_data_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ const DatabaseLoader :: Column* col = p_dbLoader . GetColumn ( columnId );
+ if ( col != 0 )
+ {
+ size_t elem_count = ncbi :: elem_count ( evt );
+ rc = p_reader . Read ( ( col -> elemBits * elem_count + 7 ) / 8 );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . CellDefault ( columnId, p_reader. GetBuffer (), elem_count );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ }
+ }
+ break;
+
+ case evt_empty_default:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-EmptyDefault, id=$(i)", "i=%u", columnId );
+ rc = p_dbLoader . CellDefault ( columnId, 0, 0 );
+ }
+ break;
+
+ case evt_open_stream:
+ LogMsg ( klogDebug, "protocol-parser event: Open-Stream" );
+ rc = p_dbLoader . OpenStream ();
+ break;
+
+ case evt_end_stream:
+ LogMsg ( klogDebug, "protocol-parser event: End-Stream" );
+ return p_dbLoader . CloseStream ();
+
+ case evt_next_row:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Next-Row, id=$(i)", "i=%u", tableId );
+ rc = p_dbLoader . NextRow ( tableId );
+ }
+ break;
+
+ case evt_move_ahead:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Move-Ahead, id=$(i)", "i=%u", tableId );
+
+ gw_move_ahead_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . MoveAhead ( tableId, ncbi :: get_nrows ( evt ) );
+ }
+ }
+ break;
+
+ case evt_errmsg:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Error-Message" );
+
+ gw_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t message_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( message_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ErrorMessage ( string ( ( const char * ) p_reader . GetBuffer (), message_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_logmsg:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Log-Message" );
+
+ gw_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t message_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( message_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . LogMessage ( string ( ( const char * ) p_reader . GetBuffer (), message_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_progmsg:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Progress-Message" );
+
+ gw_status_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t name_sz = ncbi :: size ( evt );
+ uint32_t pid = ncbi :: pid ( evt );
+ uint32_t version = ncbi :: version ( evt );
+ uint32_t timestamp = ncbi :: timestamp ( evt );
+ uint32_t percent = ncbi :: percent ( evt );
+
+ rc = p_reader . Read ( name_sz );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ProgressMessage ( string ( ( const char * ) p_reader . GetBuffer (), name_sz ),
+ pid,
+ timestamp,
+ version,
+ percent );
+ }
+ }
+ }
+ break;
+
+ default:
+ pLogMsg ( klogErr,
+ "unexpected general-loader event at $(o): $(e)",
+ "o=%lu,e=%i",
+ ( unsigned long ) p_reader . GetReadCount(), ( int ) ncbi :: evt ( evt_header ) );
+ rc = RC ( rcExe, rcFile, rcReading, rcData, rcUnexpected );
+ break;
+ }
+ }
+ while ( rc == 0 );
+
+ return rc;
+}
+
+///////////// GeneralLoader::PackedProtocolParser
+
+template < typename T_uintXX >
+rc_t
+GeneralLoader :: PackedProtocolParser :: UncompressInt ( Reader& p_reader, uint16_t p_dataSize, int (*p_decode) ( uint8_t const* buf_start, uint8_t const* buf_xend, T_uintXX* ret_decoded ) )
+{
+ m_unpackingBuf . clear();
+ // reserve enough for the best-packed case, when each element is represented with 1 byte
+ m_unpackingBuf . reserve ( sizeof ( T_uintXX ) * p_dataSize );
+
+ const uint8_t* buf_begin = reinterpret_cast<const uint8_t*> ( p_reader . GetBuffer() );
+ const uint8_t* buf_end = buf_begin + p_dataSize;
+ while ( buf_begin < buf_end )
+ {
+ T_uintXX ret_decoded;
+ int numRead = p_decode ( buf_begin, buf_end, &ret_decoded );
+ if ( numRead <= 0 )
+ {
+ pLogMsg ( klogErr, "protocol-parser: decode_uintXX() returned $(i)", "i=%i", numRead );
+ return RC ( rcExe, rcFile, rcReading, rcData, rcCorrupt );
+ }
+
+ for ( size_t i = 0; i < sizeof ( T_uintXX ); ++i )
+ {
+ m_unpackingBuf . push_back ( reinterpret_cast<const uint8_t*> ( & ret_decoded ) [ i ] );
+ }
+
+ buf_begin += numRead;
+ }
+
+ return 0;
+}
+
+rc_t
+GeneralLoader :: PackedProtocolParser :: ParseData ( Reader& p_reader, DatabaseLoader& p_dbLoader, uint32_t p_columnId, uint32_t p_dataSize )
+{
+ rc_t rc = 0;
+ const DatabaseLoader :: Column* col = p_dbLoader . GetColumn ( p_columnId );
+ if ( col != 0 )
+ {
+ rc = p_reader . Read ( p_dataSize );
+ if ( rc == 0 )
+ {
+ if ( col -> IsCompressed () )
+ {
+ switch ( col -> elemBits )
+ {
+ case 16:
+ rc = UncompressInt ( p_reader, p_dataSize, decode_uint16 );
+ break;
+ case 32:
+ rc = UncompressInt ( p_reader, p_dataSize, decode_uint32 );
+ break;
+ case 64:
+ rc = UncompressInt ( p_reader, p_dataSize, decode_uint64 );
+ break;
+ default:
+ LogMsg ( klogErr, "protocol-parser: bad element size for packed integer" );
+ rc = RC ( rcExe, rcFile, rcReading, rcData, rcInvalid );
+ break;
+ }
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . CellData ( p_columnId, m_unpackingBuf . data(), m_unpackingBuf . size() * 8 / col -> elemBits );
+ }
+ }
+ else
+ {
+ rc = p_dbLoader . CellData ( p_columnId, p_reader. GetBuffer (), p_dataSize * 8 / col -> elemBits );
+ }
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ return rc;
+}
+
+rc_t
+GeneralLoader :: PackedProtocolParser :: ParseEvents( Reader& p_reader, DatabaseLoader& p_dbLoader )
+{
+ rc_t rc;
+ do
+ {
+ struct gwp_evt_hdr_v1 evt_header;
+ rc = p_reader . Read ( & evt_header, sizeof ( evt_header ) );
+ if ( rc != 0 )
+ {
+ break;
+ }
+
+ switch ( ncbi :: evt ( evt_header ) )
+ {
+ case evt_use_schema:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Use-Schema (packed)" );
+
+ gwp_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t schema_file_size = ncbi :: size1 ( evt );
+ size_t schema_name_size = ncbi :: size2 ( evt );
+
+ rc = p_reader . Read ( schema_file_size + schema_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . UseSchema ( string ( ( const char * ) p_reader . GetBuffer (), schema_file_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + schema_file_size, schema_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_use_schema2:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Use-Schema2" );
+
+ gwp_2string_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t schema_file_size = ncbi :: size1 ( evt );
+ size_t schema_name_size = ncbi :: size2 ( evt );
+
+ rc = p_reader . Read ( schema_file_size + schema_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . UseSchema ( string ( ( const char * ) p_reader . GetBuffer (), schema_file_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + schema_file_size, schema_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_remote_path:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Remote-Path (packed)" );
+
+ gwp_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t database_name_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( database_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . RemotePath ( string ( ( const char * ) p_reader . GetBuffer (), database_name_size ) );
+ }
+ }
+ }
+ break;
+ case evt_remote_path2:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Remote-Path2" );
+
+ gwp_1string_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t database_name_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( database_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . RemotePath ( string ( ( const char * ) p_reader . GetBuffer (), database_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_software_name:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Software-Name (packed)" );
+
+ gwp_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t name_size = ncbi :: size1 ( evt );
+ size_t version_size = ncbi :: size2 ( evt );
+
+ rc = p_reader . Read ( name_size + version_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . SoftwareName ( string ( ( const char * ) p_reader . GetBuffer (), name_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + name_size, version_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_db_metadata_node:
+ {
+ uint32_t objId = ncbi :: id ( evt_header );
+ if ( objId == 256 ) // a special case for the root database; same as 0
+ {
+ objId = 0;
+ }
+ pLogMsg ( klogDebug, "protocol-parser event: Metadata-Node (packed), id=$(i)", "i=%u", objId );
+
+ gwp_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t node_size = ncbi :: size1 ( evt );
+ size_t value_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( node_size + value_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . DBMetadataNode ( objId,
+ string ( ( const char * ) p_reader . GetBuffer (), node_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + node_size, value_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_tbl_metadata_node:
+ {
+ uint32_t objId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Tbl-Metadata-Node (packed), id=$(i)", "i=%u", objId );
+
+ gwp_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t node_size = ncbi :: size1 ( evt );
+ size_t value_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( node_size + value_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . TblMetadataNode ( objId,
+ string ( ( const char * ) p_reader . GetBuffer (), node_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + node_size, value_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_col_metadata_node:
+ {
+ uint32_t objId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Col-Metadata-Node (packed), id=$(i)", "i=%u", objId );
+
+ gwp_2string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t node_size = ncbi :: size1 ( evt );
+ size_t value_size = ncbi :: size2 ( evt );
+ rc = p_reader . Read ( node_size + value_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ColMetadataNode ( objId,
+ string ( ( const char * ) p_reader . GetBuffer (), node_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + node_size, value_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_new_table:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: New-Table (packed), id=$(i)", "i=%u", tableId );
+
+ gwp_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t table_name_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( table_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . NewTable ( tableId, string ( ( const char * ) p_reader . GetBuffer (), table_name_size ) );
+ }
+ }
+ }
+ break;
+ case evt_new_table2:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: New-Table2, id=$(i)", "i=%u", tableId );
+
+ gwp_1string_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t table_name_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( table_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . NewTable ( tableId, string ( ( const char * ) p_reader . GetBuffer (), table_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_new_column:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: New-Column (packed), id=$(i)", "i=%u", columnId );
+
+ gwp_column_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t col_name_size = ncbi :: name_size ( evt );
+ rc = p_reader . Read ( col_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . NewColumn ( columnId,
+ ncbi :: table_id ( evt ),
+ ncbi :: elem_bits ( evt ),
+ ncbi :: flag_bits ( evt ),
+ string ( ( const char * ) p_reader . GetBuffer (), col_name_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_add_mbr_db:
+ {
+ uint32_t db_id = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Add-Mbr-DB ( packed ), id=$(i)", "i=%u", db_id );
+
+ gwp_add_mbr_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ uint32_t parent_id = ncbi :: db_id ( evt );
+ size_t db_mbr_size = ncbi :: size1 ( evt );
+ size_t db_name_size = ncbi :: size2 ( evt );
+ uint8_t create_mode = ncbi :: create_mode ( evt );
+ rc = p_reader . Read ( db_mbr_size + db_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . AddMbrDB ( db_id,
+ parent_id,
+ string ( ( const char * ) p_reader . GetBuffer (), db_mbr_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + db_mbr_size, db_name_size ),
+ create_mode );
+ }
+ }
+ }
+ break;
+
+ case evt_add_mbr_tbl:
+ {
+ uint32_t db_tbl_id = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Add-Mbr-Table ( packed ), id=$(i)", "i=%u", db_tbl_id );
+
+ gwp_add_mbr_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ uint32_t parent_id = ncbi :: db_id ( evt );
+ size_t db_mbr_size = ncbi :: size1 ( evt );
+ size_t db_name_size = ncbi :: size2 ( evt );
+ uint8_t create_mode = ncbi :: create_mode ( evt );
+ rc = p_reader . Read ( db_mbr_size + db_name_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . AddMbrTbl ( db_tbl_id,
+ parent_id,
+ string ( ( const char * ) p_reader . GetBuffer (), db_mbr_size ),
+ string ( ( const char * ) p_reader . GetBuffer () + db_mbr_size, db_name_size ),
+ create_mode );
+ }
+ }
+ }
+ break;
+
+ case evt_open_stream:
+ LogMsg ( klogDebug, "protocol-parser event: Open-Stream (packed)" );
+ rc = p_dbLoader . OpenStream ();
+ break;
+
+ case evt_end_stream:
+ LogMsg ( klogDebug, "protocol-parser event: End-Stream (packed)" );
+ return p_dbLoader . CloseStream ();
+
+ case evt_cell_data:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-Data (packed), id=$(i)", "i=%u", columnId );
+
+ gwp_data_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ rc = ParseData ( p_reader, p_dbLoader, columnId, ncbi :: size ( evt ) );
+ }
+ }
+ break;
+
+ case evt_cell_data2:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-Data2, id=$(i)", "i=%u", columnId );
+
+ gwp_data_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ rc = ParseData ( p_reader, p_dbLoader, columnId, ncbi :: size ( evt ) );
+ }
+ }
+ break;
+
+ case evt_cell_default:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-Default (packed), id=$(i)", "i=%u", columnId );
+
+ gwp_data_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ const DatabaseLoader :: Column* col = p_dbLoader . GetColumn ( columnId );
+ if ( col != 0 )
+ {
+ size_t dataSize = ncbi :: size ( evt );
+ rc = p_reader . Read ( dataSize );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . CellDefault ( columnId, p_reader. GetBuffer (), dataSize * 8 / col -> elemBits );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ }
+ }
+ break;
+
+ case evt_cell_default2:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-Default2, id=$(i)", "i=%u", columnId );
+
+ gwp_data_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ { // same as above - refactor
+ const DatabaseLoader :: Column* col = p_dbLoader . GetColumn ( columnId );
+ if ( col != 0 )
+ {
+ size_t dataSize = ncbi :: size ( evt );
+ rc = p_reader . Read ( dataSize );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . CellDefault ( columnId, p_reader. GetBuffer (), dataSize * 8 / col -> elemBits );
+ }
+ }
+ else
+ {
+ rc = RC ( rcExe, rcFile, rcReading, rcColumn, rcNotFound );
+ }
+ }
+ }
+ break;
+
+ case evt_empty_default:
+ {
+ uint32_t columnId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Cell-EmptyDefault (packed), id=$(i)", "i=%u", columnId );
+ rc = p_dbLoader . CellDefault ( columnId, 0, 0 );
+ }
+ break;
+
+ case evt_next_row:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Next-Row (packed), id=$(i)", "i=%u", tableId );
+ rc = p_dbLoader . NextRow ( tableId );
+ }
+ break;
+
+ case evt_move_ahead:
+ {
+ uint32_t tableId = ncbi :: id ( evt_header );
+ pLogMsg ( klogDebug, "protocol-parser event: Move-Ahead (packed), id=$(i)", "i=%u", tableId );
+
+ gwp_move_ahead_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . MoveAhead ( tableId, ncbi :: get_nrows ( evt ) );
+ }
+ }
+ break;
+
+ case evt_errmsg:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Error-Message (packed)" );
+
+ gwp_1string_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t message_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( message_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ErrorMessage ( string ( ( const char * ) p_reader . GetBuffer (), message_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_errmsg2:
+ {
+ LogMsg ( klogDebug, "protocol-parser event: Error-Message2" );
+
+ gwp_1string_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t message_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( message_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ErrorMessage ( string ( ( const char * ) p_reader . GetBuffer (), message_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_logmsg:
+ {
+ LogMsg ( klogDebug, "protocol-parser event ( packed ): Log-Message" );
+
+ gwp_1string_evt_U16_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t message_size = ncbi :: size ( evt );
+ rc = p_reader . Read ( message_size );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . LogMessage ( string ( ( const char * ) p_reader . GetBuffer (), message_size ) );
+ }
+ }
+ }
+ break;
+
+ case evt_progmsg:
+ {
+ LogMsg ( klogDebug, "protocol-parser event ( packed ): Progress-Message" );
+
+ gwp_status_evt_v1 evt;
+ rc = ReadEvent ( p_reader, evt );
+ if ( rc == 0 )
+ {
+ size_t name_sz = ncbi :: size ( evt );
+ uint32_t pid = ncbi :: pid ( evt );
+ uint32_t version = ncbi :: version ( evt );
+ uint32_t timestamp = ncbi :: timestamp ( evt );
+ uint32_t percent = ncbi :: percent ( evt );
+
+ rc = p_reader . Read ( name_sz );
+ if ( rc == 0 )
+ {
+ rc = p_dbLoader . ProgressMessage ( string ( ( const char * ) p_reader . GetBuffer (), name_sz ),
+ pid,
+ timestamp,
+ version,
+ percent );
+ }
+ }
+ }
+ break;
+
+ default:
+ pLogMsg ( klogErr,
+ "unexpected general-loader event at $(o): $(e)",
+ "o=%lu,e=%i",
+ ( unsigned long ) p_reader . GetReadCount(), ( int ) ncbi :: evt ( evt_header ) );
+ rc = RC ( rcExe, rcFile, rcReading, rcData, rcUnexpected );
+ break;
+ }
+ }
+ while ( rc == 0 );
+
+ return rc;
+}
+
diff --git a/tools/general-loader/utf8-like-int-codec.c b/tools/general-loader/utf8-like-int-codec.c
new file mode 100644
index 0000000..e4e73b0
--- /dev/null
+++ b/tools/general-loader/utf8-like-int-codec.c
@@ -0,0 +1,687 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "utf8-like-int-codec.h"
+
+#define NOT_ENCODED_SEQUENCE 0xFFu
+
+#define MAX_VALUE_BYTE_1 0x7F
+#define MAX_VALUE_BYTE_2 0x7FF
+#define MAX_VALUE_BYTE_3 0xFFFF
+#define MAX_VALUE_BYTE_4 0x1FFFFF
+#define MAX_VALUE_BYTE_5 0x3FFFFFF
+#define MAX_VALUE_BYTE_6 0x7FFFFFFF
+#define MAX_VALUE_BYTE_7 0xFFFFFFFFF
+
+
+int encode_uint16 ( uint16_t value_to_encode, uint8_t* buf_start, uint8_t* buf_xend )
+{
+ size_t dst_buf_size = buf_xend - buf_start;
+ int ret = CODEC_UNKNOWN_ERROR;
+
+ /* optimization: process 1-byte case in the very beginning */
+ if (value_to_encode <= MAX_VALUE_BYTE_1)
+ {
+ if ( dst_buf_size < 1 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = (uint8_t)value_to_encode;
+ ret = 1;
+ }
+ else if ( value_to_encode <= MAX_VALUE_BYTE_2 )
+ {
+ /* 1-byte case is processed already, so starting from 2-byte one */
+ if ( dst_buf_size < 2 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xC0 | (uint8_t)(value_to_encode >> 6);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 2;
+ }
+ else /* encoding will take at least 3 bytes which is pointless - writing directly */
+ {
+ if (dst_buf_size < 3)
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = NOT_ENCODED_SEQUENCE;
+ buf_start[1] = (uint8_t)( value_to_encode >> 8 );
+ buf_start[2] = (uint8_t)( value_to_encode );
+
+ ret = 3;
+ }
+
+ return ret;
+}
+
+int decode_uint16 ( uint8_t const* buf_start, uint8_t const* buf_xend, uint16_t* ret_decoded )
+{
+ size_t src_buf_size = buf_xend - buf_start;
+ int ret = CODEC_INVALID_FORMAT;
+
+ if (src_buf_size < 1)
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ if ( (int8_t)buf_start[0] >= 0 )
+ {
+ *ret_decoded = buf_start[0];
+ return 1;
+ }
+
+ /*neg_ch = ~ buf_start[0];*/
+ /*
+ neg_ch | buf_start[0]
+ invalid: 01nnnnnn >= 0x40 | 10xxxxxx < 0xC0
+ valid:
+ 2-byte: 001nnnnn >= 0x20 | 110xxxxx < 0xE0
+ 3-byte: 0001nnnn >= 0x10 | 1110xxxx < 0xF0
+ 4-byte: 00001nnn >= 0x08 | 11110xxx < 0xF8
+ 5-byte: 000001nn >= 0x04 | 111110xx < 0xFC
+ 6-byte: 0000001n >= 0x02 | 1111110x < 0xFE
+ 7-byte: 00000001 >= 0x01 | 11111110 < 0xFF
+
+ So, no need to have neg_ch to achieve the filtering
+ */
+
+ /* invalid 1st byte */
+ if ( buf_start[0] < 0xC0 )
+ return CODEC_INVALID_FORMAT;
+ else if ( buf_start[0] < 0xE0 )
+ {
+ if ( src_buf_size < 2 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x1F;
+ uint8_t byte1;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ *ret_decoded = (uint16_t)byte0 << 6 |
+ (uint16_t)byte1;
+
+ ret = 2;
+ }
+ }
+ else if ( buf_start[0] == NOT_ENCODED_SEQUENCE )
+ {
+ if ( src_buf_size < 3 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ *ret_decoded = (uint16_t)buf_start[1] << 8 |
+ (uint16_t)buf_start[2];
+
+ ret = 3;
+ }
+
+ return ret;
+}
+
+
+int encode_uint32 ( uint32_t value_to_encode, uint8_t* buf_start, uint8_t* buf_xend )
+{
+ size_t dst_buf_size = buf_xend - buf_start;
+ int ret = CODEC_UNKNOWN_ERROR;
+
+ if (value_to_encode <= MAX_VALUE_BYTE_1)
+ {
+ if ( dst_buf_size < 1 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = (uint8_t)value_to_encode;
+ ret = 1;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_2)
+ {
+ if ( dst_buf_size < 2 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xC0 | (uint8_t)(value_to_encode >> 6);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 2;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_3)
+ {
+ if ( dst_buf_size < 3 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xE0 | (uint8_t)(value_to_encode >> 12);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 3;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_4)
+ {
+ if ( dst_buf_size < 4 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xF0 | (uint8_t)(value_to_encode >> 18);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 12 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[3] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 4;
+ }
+ else /* encoding will take at least 5 bytes which is pointless - writing directly */
+ {
+ if (dst_buf_size < 5)
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = NOT_ENCODED_SEQUENCE;
+ buf_start[1] = (uint8_t)( value_to_encode >> 24 );
+ buf_start[2] = (uint8_t)( value_to_encode >> 16 );
+ buf_start[3] = (uint8_t)( value_to_encode >> 8 );
+ buf_start[4] = (uint8_t)( value_to_encode );
+
+ ret = 5;
+ }
+
+ return ret;
+}
+
+int decode_uint32 ( uint8_t const* buf_start, uint8_t const* buf_xend, uint32_t* ret_decoded )
+{
+ size_t src_buf_size = buf_xend - buf_start;
+ int ret = CODEC_INVALID_FORMAT;
+
+ if (src_buf_size < 1)
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ /* 1-byte sequence */
+ if ( (int8_t)buf_start[0] >= 0 )
+ {
+ *ret_decoded = buf_start[0];
+ return 1;
+ }
+
+ /*neg_ch = ~ buf_start[0];*/
+ /*
+ neg_ch | buf_start[0]
+ invalid: 01nnnnnn >= 0x40 | 10xxxxxx < 0xC0
+ valid:
+ 2-byte: 001nnnnn >= 0x20 | 110xxxxx < 0xE0
+ 3-byte: 0001nnnn >= 0x10 | 1110xxxx < 0xF0
+ 4-byte: 00001nnn >= 0x08 | 11110xxx < 0xF8
+ 5-byte: 000001nn >= 0x04 | 111110xx < 0xFC
+ 6-byte: 0000001n >= 0x02 | 1111110x < 0xFE
+ 7-byte: 00000001 >= 0x01 | 11111110 < 0xFF
+
+ So, no need to have neg_ch to achieve the filtering
+ */
+
+ /* invalid 1st byte */
+ if ( buf_start[0] < 0xC0 )
+ return CODEC_INVALID_FORMAT;
+ /* 2-byte sequence */
+ else if ( buf_start[0] < 0xE0 )
+ {
+ if ( src_buf_size < 2 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x1F;
+ uint8_t byte1;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ *ret_decoded = (uint32_t)byte0 << 6 |
+ (uint32_t)byte1;
+
+ ret = 2;
+ }
+ }
+ /* 3-byte sequence */
+ else if ( buf_start[0] < 0xF0 )
+ {
+ if ( src_buf_size < 3 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x0F;
+ uint8_t byte1, byte2;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ *ret_decoded = (uint32_t)byte0 << 12 |
+ (uint32_t)byte1 << 6 |
+ (uint32_t)byte2;
+
+ ret = 3;
+ }
+ }
+ /* 4-byte sequence */
+ else if ( buf_start[0] < 0xF8 )
+ {
+ if ( src_buf_size < 4 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x07;
+ uint8_t byte1, byte2, byte3;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ if ( ( buf_start[3] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte3 = buf_start[3] & 0x3F;
+
+ *ret_decoded = (uint32_t)byte0 << 18 |
+ (uint32_t)byte1 << 12 |
+ (uint32_t)byte2 << 6 |
+ (uint32_t)byte3;
+
+ ret = 4;
+ }
+ }
+ else if ( buf_start[0] == NOT_ENCODED_SEQUENCE )
+ {
+ if ( src_buf_size < 5 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ *ret_decoded = (uint32_t)buf_start[1] << 24 |
+ (uint32_t)buf_start[2] << 16 |
+ (uint32_t)buf_start[3] << 8 |
+ (uint32_t)buf_start[4];
+
+ ret = 5;
+ }
+
+ return ret;
+}
+
+
+int encode_uint64 ( uint64_t value_to_encode, uint8_t* buf_start, uint8_t* buf_xend )
+{
+ size_t dst_buf_size = buf_xend - buf_start;
+ int ret = CODEC_UNKNOWN_ERROR;
+
+ if (value_to_encode <= MAX_VALUE_BYTE_1)
+ {
+ if ( dst_buf_size < 1 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = (uint8_t)value_to_encode;
+ ret = 1;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_2)
+ {
+ if ( dst_buf_size < 2 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xC0 | (uint8_t)(value_to_encode >> 6);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 2;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_3)
+ {
+ if ( dst_buf_size < 3 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xE0 | (uint8_t)(value_to_encode >> 12);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 3;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_4)
+ {
+ if ( dst_buf_size < 4 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xF0 | (uint8_t)(value_to_encode >> 18);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 12 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[3] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 4;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_5)
+ {
+ if ( dst_buf_size < 5 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xF8 | (uint8_t)(value_to_encode >> 24);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 18 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode >> 12 & 0x3F);
+ buf_start[3] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[4] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 5;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_6)
+ {
+ if ( dst_buf_size < 6 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xFC | (uint8_t)(value_to_encode >> 30);
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 24 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode >> 18 & 0x3F);
+ buf_start[3] = 0x80 | (uint8_t)(value_to_encode >> 12 & 0x3F);
+ buf_start[4] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[5] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 6;
+ }
+ else if (value_to_encode <= MAX_VALUE_BYTE_7)
+ {
+ if ( dst_buf_size < 7 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = 0xFE;/* | (uint8_t)(value_to_encode >> 36);*/ /* this shift will always give 0 */
+ buf_start[1] = 0x80 | (uint8_t)(value_to_encode >> 30 & 0x3F);
+ buf_start[2] = 0x80 | (uint8_t)(value_to_encode >> 24 & 0x3F);
+ buf_start[3] = 0x80 | (uint8_t)(value_to_encode >> 18 & 0x3F);
+ buf_start[4] = 0x80 | (uint8_t)(value_to_encode >> 12 & 0x3F);
+ buf_start[5] = 0x80 | (uint8_t)(value_to_encode >> 6 & 0x3F);
+ buf_start[6] = 0x80 | (uint8_t)(value_to_encode & 0x3F);
+
+ ret = 7;
+ }
+ else /* there is a limitation - cannot encode in more than 7 bytes, writing directly now */
+ {
+ if (dst_buf_size < 9)
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ buf_start[0] = NOT_ENCODED_SEQUENCE;
+ buf_start[1] = (uint8_t)( value_to_encode >> 56 );
+ buf_start[2] = (uint8_t)( value_to_encode >> 48 );
+ buf_start[3] = (uint8_t)( value_to_encode >> 40 );
+ buf_start[4] = (uint8_t)( value_to_encode >> 32 );
+ buf_start[5] = (uint8_t)( value_to_encode >> 24 );
+ buf_start[6] = (uint8_t)( value_to_encode >> 16 );
+ buf_start[7] = (uint8_t)( value_to_encode >> 8 );
+ buf_start[8] = (uint8_t)( value_to_encode );
+
+ ret = 9;
+ }
+
+ return ret;
+}
+
+int decode_uint64 ( uint8_t const* buf_start, uint8_t const* buf_xend, uint64_t* ret_decoded )
+{
+ size_t src_buf_size = buf_xend - buf_start;
+ int ret = CODEC_INVALID_FORMAT;
+
+ if (src_buf_size < 1)
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ /* 1-byte sequence */
+ if ( (int8_t)buf_start[0] >= 0 )
+ {
+ *ret_decoded = buf_start[0];
+ return 1;
+ }
+
+ /*neg_ch = ~ buf_start[0];*/
+ /*
+ neg_ch | buf_start[0]
+ invalid: 01nnnnnn >= 0x40 | 10xxxxxx < 0xC0
+ valid:
+ 2-byte: 001nnnnn >= 0x20 | 110xxxxx < 0xE0
+ 3-byte: 0001nnnn >= 0x10 | 1110xxxx < 0xF0
+ 4-byte: 00001nnn >= 0x08 | 11110xxx < 0xF8
+ 5-byte: 000001nn >= 0x04 | 111110xx < 0xFC
+ 6-byte: 0000001n >= 0x02 | 1111110x < 0xFE
+ 7-byte: 00000001 >= 0x01 | 11111110 < 0xFF
+
+ So, no need to have neg_ch to achieve the filtering
+ */
+
+ /* invalid 1st byte */
+ if ( buf_start[0] < 0xC0 )
+ return CODEC_INVALID_FORMAT;
+ /* 2-byte sequence */
+ else if ( buf_start[0] < 0xE0 )
+ {
+ if ( src_buf_size < 2 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x1F;
+ uint8_t byte1;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ *ret_decoded = (uint64_t)byte0 << 6 |
+ (uint64_t)byte1;
+
+ ret = 2;
+ }
+ }
+ /* 3-byte sequence */
+ else if ( buf_start[0] < 0xF0 )
+ {
+ if ( src_buf_size < 3 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x0F;
+ uint8_t byte1, byte2;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ *ret_decoded = (uint64_t)byte0 << 12 |
+ (uint64_t)byte1 << 6 |
+ (uint64_t)byte2;
+
+ ret = 3;
+ }
+ }
+ /* 4-byte sequence */
+ else if ( buf_start[0] < 0xF8 )
+ {
+ if ( src_buf_size < 4 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x07;
+ uint8_t byte1, byte2, byte3;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ if ( ( buf_start[3] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte3 = buf_start[3] & 0x3F;
+
+ *ret_decoded = (uint64_t)byte0 << 18 |
+ (uint64_t)byte1 << 12 |
+ (uint64_t)byte2 << 6 |
+ (uint64_t)byte3;
+
+ ret = 4;
+ }
+ }
+ /* 5-byte sequence */
+ else if ( buf_start[0] < 0xFC )
+ {
+ if ( src_buf_size < 5 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x03;
+ uint8_t byte1, byte2, byte3, byte4;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ if ( ( buf_start[3] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte3 = buf_start[3] & 0x3F;
+
+ if ( ( buf_start[4] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte4 = buf_start[4] & 0x3F;
+
+ *ret_decoded = (uint64_t)byte0 << 24 |
+ (uint64_t)byte1 << 18 |
+ (uint64_t)byte2 << 12 |
+ (uint64_t)byte3 << 6 |
+ (uint64_t)byte4;
+
+ ret = 5;
+ }
+ }
+ /* 6-byte sequence */
+ else if ( buf_start[0] < 0xFE )
+ {
+ if ( src_buf_size < 6 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte0 = buf_start[0] & 0x01;
+ uint8_t byte1, byte2, byte3, byte4, byte5;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ if ( ( buf_start[3] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte3 = buf_start[3] & 0x3F;
+
+ if ( ( buf_start[4] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte4 = buf_start[4] & 0x3F;
+
+ if ( ( buf_start[5] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte5 = buf_start[5] & 0x3F;
+
+ *ret_decoded = (uint64_t)byte0 << 30 |
+ (uint64_t)byte1 << 24 |
+ (uint64_t)byte2 << 18 |
+ (uint64_t)byte3 << 12 |
+ (uint64_t)byte4 << 6 |
+ (uint64_t)byte5;
+
+ ret = 6;
+ }
+ }
+ /* 6-byte sequence */
+ else if ( buf_start[0] == 0xFE )
+ {
+ if ( src_buf_size < 7 )
+ return CODEC_INSUFFICIENT_BUFFER;
+ else
+ {
+ uint8_t byte1, byte2, byte3, byte4, byte5, byte6;
+
+ if ( ( buf_start[1] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte1 = buf_start[1] & 0x3F;
+
+ if ( ( buf_start[2] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte2 = buf_start[2] & 0x3F;
+
+ if ( ( buf_start[3] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte3 = buf_start[3] & 0x3F;
+
+ if ( ( buf_start[4] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte4 = buf_start[4] & 0x3F;
+
+ if ( ( buf_start[5] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte5 = buf_start[5] & 0x3F;
+
+ if ( ( buf_start[6] & 0xC0 ) != 0x80 )
+ return CODEC_INVALID_FORMAT;
+ byte6 = buf_start[6] & 0x3F;
+
+ *ret_decoded = (uint64_t)byte1 << 30 |
+ (uint64_t)byte2 << 24 |
+ (uint64_t)byte3 << 18 |
+ (uint64_t)byte4 << 12 |
+ (uint64_t)byte5 << 6 |
+ (uint64_t)byte6;
+
+ ret = 7;
+ }
+ }
+ else /* if ( buf_start[0] == NOT_ENCODED_SEQUENCE ) */
+ {
+ if ( src_buf_size < 9 )
+ return CODEC_INSUFFICIENT_BUFFER;
+
+ *ret_decoded = (uint64_t)buf_start[1] << 56 |
+ (uint64_t)buf_start[2] << 48 |
+ (uint64_t)buf_start[3] << 40 |
+ (uint64_t)buf_start[4] << 32 |
+ (uint64_t)buf_start[5] << 24 |
+ (uint64_t)buf_start[6] << 16 |
+ (uint64_t)buf_start[7] << 8 |
+ (uint64_t)buf_start[8];
+
+ ret = 9;
+ }
+
+ return ret;
+}
diff --git a/tools/run-stat/definitions.h b/tools/general-loader/utf8-like-int-codec.h
similarity index 54%
rename from tools/run-stat/definitions.h
rename to tools/general-loader/utf8-like-int-codec.h
index af23257..0b579ac 100644
--- a/tools/run-stat/definitions.h
+++ b/tools/general-loader/utf8-like-int-codec.h
@@ -24,55 +24,46 @@
*
*/
-#ifndef _h_definitions_
-#define _h_definitions_
+#ifndef _h_utf8_like_int_codec_
+#define _h_utf8_like_int_codec_
+
+#include <stdint.h>
+#include <stdlib.h> /* size_t on linux */
#ifdef __cplusplus
extern "C" {
#endif
-#ifndef _h_klib_defs_
-#include <klib/defs.h>
-#endif
-
-#ifndef _h_klib_out_
-#include <klib/out.h>
-#endif
-
-#ifndef _h_klib_rc_
-#include <klib/rc.h>
-#endif
-
-#define DISP_RC(rc,err) (void)((rc == 0) ? 0 : LogErr( klogInt, rc, err ))
+enum
+{
+ CODEC_INSUFFICIENT_BUFFER = 0,
+ CODEC_INVALID_FORMAT = -1,
+ CODEC_UNKNOWN_ERROR = -2
+};
-#define DISP_RC2(rc,err,succ) \
- (void)((rc != 0)? 0 : (succ) ? LOGMSG( klogInfo, succ ) : LogErr( klogInt, rc, err ))
+/*
+all encode_uintXX return:
+ value <= 0: error: one of CODEC_* above
+ value > 0: success: number of bytes written to buf_start
-#define RUN_STAT_COLS "READ,QUALITY,READ_LEN,READ_START,READ_TYPE"
+all decode_uintXX return:
+ value <= 0: error: one of CODEC_* above
+ value > 0: success: number of bytes read from buf_start
+*/
-#define SRA_PF_UNDEF 0
-#define SRA_PF_454 1
-#define SRA_PF_ILLUMINA 2
-#define SRA_PF_ABSOLID 3
-#define SRA_PF_COMPLETE_GENOMICS 4
-#define SRA_PF_HELICOS 5
-#define SRA_PF_UNKNOWN 6
+int encode_uint16 ( uint16_t value_to_encode, uint8_t* buf_start, uint8_t* buf_xend );
+int decode_uint16 ( uint8_t const* buf_start, uint8_t const* buf_xend, uint16_t* ret_decoded );
-#define SRA_READ_FILTER_PASS 0
-#define SRA_READ_FILTER_REJECT 1
-#define SRA_READ_FILTER_CRITERIA 2
-#define SRA_READ_FILTER_REDACTED 3
+int encode_uint32 ( uint32_t value_to_encode, uint8_t* buf_start, uint8_t* buf_xend );
+int decode_uint32 ( uint8_t const* buf_start, uint8_t const* buf_xend, uint32_t* ret_decoded );
-/* report output formats */
-#define RT_TXT 0
-#define RT_CSV 1
-#define RT_XML 2
-#define RT_JSO 3
+int encode_uint64 ( uint64_t value_to_encode, uint8_t* buf_start, uint8_t* buf_xend );
+int decode_uint64 ( uint8_t const* buf_start, uint8_t const* buf_xend, uint64_t* ret_decoded );
-#define DEFAULT_REPORT_PREFIX "report"
#ifdef __cplusplus
}
#endif
-#endif
+#endif /* _h_utf8_like_int_codec_ */
+
diff --git a/tools/kar/kar.c b/tools/kar/kar.c
index c83a780..6ab8231 100644
--- a/tools/kar/kar.c
+++ b/tools/kar/kar.c
@@ -260,7 +260,7 @@ static rc_t pnamesNodeMake (pnamesNode ** pself, const char * path)
return 0;
}
-static int CC pnameFindCmp (const void * _item, const BSTNode *_n)
+static int64_t CC pnameFindCmp (const void * _item, const BSTNode *_n)
{
const char * item = _item;
const pnamesNode * n = (const pnamesNode *)_n;
@@ -283,7 +283,7 @@ static int CC pnameFindCmp (const void * _item, const BSTNode *_n)
return (strcmp (item, n->name));
}
-static int CC pnamesInsertCmp (const BSTNode * _item, const BSTNode *_n)
+static int64_t CC pnamesInsertCmp (const BSTNode * _item, const BSTNode *_n)
{
const pnamesNode * item = (const pnamesNode *)_item;
const pnamesNode * n = (const pnamesNode *)_n;
@@ -361,7 +361,7 @@ static KSRAFileAlignment get_alignment (const char * str)
}
static
-int CC sort_cmp (const void ** l, const void ** r, void * data)
+int64_t CC sort_cmp (const void ** l, const void ** r, void * data)
{
/* KDirectory * d; */
uint64_t lz, rz;
@@ -1361,7 +1361,7 @@ rc_t CC KMain ( int argc, char *argv [] )
break;
for (ix = 0; ix < pcount; ++ix)
{
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
break;
rc = pnamesInsert (pc);
@@ -1381,7 +1381,7 @@ rc_t CC KMain ( int argc, char *argv [] )
pc = NULL;
else
{
- rc = ArgsOptionValue (args, OPTION_ALIGN, 0, &pc);
+ rc = ArgsOptionValue (args, OPTION_ALIGN, 0, (const void **)&pc);
if (rc)
break;
}
@@ -1412,7 +1412,7 @@ rc_t CC KMain ( int argc, char *argv [] )
break;
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_DIRECTORY, 0, &directory);
+ rc = ArgsOptionValue (args, OPTION_DIRECTORY, 0, (const void **)&directory);
if (rc)
break;
@@ -1425,7 +1425,7 @@ rc_t CC KMain ( int argc, char *argv [] )
break;
if (pcount)
{
- rc = ArgsOptionValue (args, OPTION_CREATE, 0, &archive);
+ rc = ArgsOptionValue (args, OPTION_CREATE, 0, (const void **)&archive);
if (rc)
break;
@@ -1440,7 +1440,7 @@ rc_t CC KMain ( int argc, char *argv [] )
if (mode == OM_NONE)
{
mode = OM_TEST;
- rc = ArgsOptionValue (args, OPTION_TEST, 0, &archive);
+ rc = ArgsOptionValue (args, OPTION_TEST, 0, (const void **)&archive);
if (rc)
break;
}
@@ -1456,7 +1456,7 @@ rc_t CC KMain ( int argc, char *argv [] )
if (mode == OM_NONE)
{
mode = OM_TEST;
- rc = ArgsOptionValue (args, OPTION_LIST, 0, &archive);
+ rc = ArgsOptionValue (args, OPTION_LIST, 0, (const void **)&archive);
if (rc)
break;
}
@@ -1472,7 +1472,7 @@ rc_t CC KMain ( int argc, char *argv [] )
if (mode == OM_NONE)
{
mode = OM_EXTRACT;
- rc = ArgsOptionValue (args, OPTION_EXTRACT, 0, &archive);
+ rc = ArgsOptionValue (args, OPTION_EXTRACT, 0, (const void **)&archive);
if (rc)
break;
}
@@ -1484,6 +1484,7 @@ rc_t CC KMain ( int argc, char *argv [] )
if (mode == OM_NONE)
{
MiniUsage(args);
+ rc = RC(rcApp, rcArgv, rcParsing, rcParam, rcInsufficient);
break;
}
diff --git a/tools/kar/kar.vers b/tools/kar/kar.vers
index 8e8299d..35d16fb 100644
--- a/tools/kar/kar.vers
+++ b/tools/kar/kar.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/kar/kar.vers.h b/tools/kar/kar.vers.h
index 3b56111..e25cd88 100644
--- a/tools/kar/kar.vers.h
+++ b/tools/kar/kar.vers.h
@@ -1 +1 @@
-#define KAR_VERS 0x02040002
+#define KAR_VERS 0x02050007
diff --git a/tools/kar/sra-kar.c b/tools/kar/sra-kar.c
index ab6da3a..e5ef56d 100644
--- a/tools/kar/sra-kar.c
+++ b/tools/kar/sra-kar.c
@@ -339,17 +339,19 @@ rc_t CC Usage (const Args * args)
if (pcount == 0)
{
KOutMsg ("missing source table\n");
- rc = MiniUsage (args);
+ MiniUsage (args);
+ rc = RC (rcExe, rcArgv, rcParsing, rcPath, rcInsufficient);
break;
}
else if (pcount > 2)
{
KOutMsg ("Too many parameters\n");
- rc = MiniUsage (args);
+ MiniUsage (args);
+ rc = RC (rcExe, rcArgv, rcParsing, rcPath, rcExcessive);
break;
}
- rc = ArgsParamValue (args, 0, &pb.src_path);
+ rc = ArgsParamValue (args, 0, (const void **)&pb.src_path);
if (rc)
{
KOutMsg ("failure to get source path/name\n");
@@ -377,7 +379,7 @@ rc_t CC Usage (const Args * args)
}
else
{
- rc = ArgsParamValue (args, 1, &pb.dst_path);
+ rc = ArgsParamValue (args, 1, (const void **)&pb.dst_path);
if (rc)
{
LOGERR (klogInt, rc, "failure to get destination path");
diff --git a/tools/kar/sra-kar.vers b/tools/kar/sra-kar.vers
index 8e8299d..35d16fb 100644
--- a/tools/kar/sra-kar.vers
+++ b/tools/kar/sra-kar.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/kar/sra-kar.vers.h b/tools/kar/sra-kar.vers.h
index 855bafb..e1d73c2 100644
--- a/tools/kar/sra-kar.vers.h
+++ b/tools/kar/sra-kar.vers.h
@@ -1 +1 @@
-#define SRA_KAR_VERS 0x02040002
+#define SRA_KAR_VERS 0x02050007
diff --git a/tools/keyring-srv/keyring-data.c b/tools/keyring-srv/keyring-data.c
index f1920c1..58cdeac 100644
--- a/tools/keyring-srv/keyring-data.c
+++ b/tools/keyring-srv/keyring-data.c
@@ -47,11 +47,11 @@ void KeyRingDataWhack ( KeyRingData* self )
BSTreeWhack ( & self->objects, ObjectWhack, NULL );
}
-int CC FindProject ( const void *item, const BSTNode *n )
+static int64_t CC FindProject ( const void *item, const BSTNode *n )
{
return StringCompare((const String*)item, ((const Project*)n)->name);
}
-int CC SortProjects ( const BSTNode *item, const BSTNode *n )
+static int64 CC SortProjects ( const BSTNode *item, const BSTNode *n )
{
return StringCompare(((const Project*)item)->name, ((const Project*)n)->name);
}
@@ -214,11 +214,11 @@ void ObjectWhack ( BSTNode *n, void *data )
free(self);
}
-int CC FindObject( const void *item, const BSTNode *n )
+static int64_t CC FindObject( const void *item, const BSTNode *n )
{
return StringCompare((const String*)item, ((const Object*)n)->name);
}
-int CC SortObjects( const BSTNode *item, const BSTNode *n )
+static int64_t CC SortObjects( const BSTNode *item, const BSTNode *n )
{
return StringCompare(((const Project*)item)->name, ((const Object*)n)->name);
}
diff --git a/tools/keyring-srv/keyring-srv.vers b/tools/keyring-srv/keyring-srv.vers
index 8e8299d..35d16fb 100644
--- a/tools/keyring-srv/keyring-srv.vers
+++ b/tools/keyring-srv/keyring-srv.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-dbcc/Makefile b/tools/kget/Makefile
similarity index 86%
copy from tools/sra-dbcc/Makefile
copy to tools/kget/Makefile
index 4d4646e..ba4231e 100644
--- a/tools/sra-dbcc/Makefile
+++ b/tools/kget/Makefile
@@ -26,21 +26,20 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/sra-dbcc
+MODULE = tools/kget
include $(TOP)/build/Makefile.env
INT_TOOLS = \
-
EXT_TOOLS = \
- sra-dbcc
+ kget
ALL_TOOLS = \
$(INT_TOOLS) \
$(EXT_TOOLS)
-#------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
# outer targets
#
all std: vers-includes
@@ -82,21 +81,19 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# sra-dbcc
+# kget
#
-SRA_DBCC_SRC = \
- sra-dbcc
-
-SRA_DBCC_OBJ = \
- $(addsuffix .$(OBJX),$(SRA_DBCC_SRC))
+TOOL_SRC = \
+ kget
-SRA_DBCC_LIB = \
- -lkapp \
- -lncbi-vdb \
+TOOL_OBJ = \
+ $(addsuffix .$(OBJX),$(TOOL_SRC))
-sra-dbcc.vers.h: sra-dbcc.vers
- $(TOP)/build/make-vers-inc.sh SRA_DBCC_VERS $^ > $@
+TOOL_LIB = \
+ -skapp \
+ -sncbi-vdb \
+ -lm
-$(BINDIR)/sra-dbcc: $(SRA_DBCC_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SRA_DBCC_LIB)
+$(BINDIR)/kget: $(TOOL_OBJ)
+ $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(TOOL_LIB)
diff --git a/tools/kget/kget.c b/tools/kget/kget.c
new file mode 100644
index 0000000..8d5bdb3
--- /dev/null
+++ b/tools/kget/kget.c
@@ -0,0 +1,688 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "kget.vers.h"
+
+#include <kapp/main.h>
+#include <kapp/args.h>
+
+#include <klib/out.h>
+#include <klib/log.h>
+#include <klib/text.h>
+#include <klib/rc.h>
+#include <klib/time.h>
+
+#include <kfs/directory.h>
+#include <kfs/file.h>
+#include <kfs/buffile.h>
+#include <kfs/cacheteefile.h>
+//#include <kfs/big_block_reader.h>
+
+#include <kns/manager.h>
+#include <kns/kns-mgr-priv.h>
+#include <kns/http.h>
+
+#include <kproc/timeout.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+/*===========================================================================
+
+ kget is a works like a simple version of wget
+
+ =========================================================================== */
+
+const char UsageDefaultName[] = "kget";
+
+rc_t CC UsageSummary ( const char * progname )
+{
+ return KOutMsg ("\n"
+ "Usage:\n"
+ " %s <path> [<path> ...] [options]\n"
+ "\n", progname);
+}
+
+
+#define OPTION_VERB "verb"
+#define ALIAS_VERB "b"
+static const char * verb_usage[] = { "execute verbose", NULL };
+
+#define OPTION_BLOCK "block-size"
+#define ALIAS_BLOCK "s"
+static const char * block_usage[] = { "how many bytes per block", NULL };
+
+#define OPTION_SHOW "show-size"
+#define ALIAS_SHOW "w"
+static const char * show_usage[] = { "query size of remote file first", NULL };
+
+#define OPTION_CACHE "cache"
+#define ALIAS_CACHE "c"
+static const char * cache_usage[] = { "wrap the remote-file into a KCacheTeeFile", NULL };
+
+#define OPTION_RAND "random"
+#define ALIAS_RAND "r"
+static const char * random_usage[] = { "request blocks in random order", NULL };
+
+#define OPTION_REP "repeat"
+#define ALIAS_REP "e"
+static const char * repeat_usage[] = { "request blocks with repeats if in random order", NULL };
+
+#define OPTION_CREPORT "report"
+#define ALIAS_CREPORT "p"
+static const char * creport_usage[] = { "report cache usage", NULL };
+
+#define OPTION_COMPLETE "complete"
+static const char * complete_usage[] = { "check if 1st parameter is a complete", NULL };
+
+#define OPTION_TRUNC "truncate"
+static const char * truncate_usage[] = { "truncate the file ( 1st parameter ) / remove trailing cache-bitmap", NULL };
+
+#define OPTION_START "start"
+static const char * start_usage[] = { "offset where to read from", NULL };
+
+#define OPTION_COUNT "count"
+static const char * count_usage[] = { "number of bytes to read", NULL };
+
+#define OPTION_PROGRESS "progress"
+static const char * progress_usage[] = { "show progress", NULL };
+
+#define OPTION_RELIABLE "reliable"
+static const char * reliable_usage[] = { "use reliable version of http-file", NULL };
+
+#define OPTION_BUFFER "buffer"
+#define ALIAS_BUFFER "u"
+static const char * buffer_usage[] = { "wrap remote file into KBufFile with this buffer-size", NULL };
+
+#define OPTION_SLEEP "sleep"
+#define ALIAS_SLEEP "i"
+static const char * sleep_usage[] = { "sleep inbetween requests by this amount of ms", NULL };
+
+#define OPTION_TIMEOUT "timeout"
+#define ALIAS_TIMEOUT "m"
+static const char * timeout_usage[] = { "use timed read with tis amount of ms as timeout", NULL };
+
+OptDef MyOptions[] =
+{
+/* name alias fkt usage-txt, cnt, needs value, required */
+ { OPTION_VERB, ALIAS_VERB, NULL, verb_usage, 1, false, false },
+ { OPTION_BLOCK, ALIAS_BLOCK, NULL, block_usage, 1, true, false },
+ { OPTION_SHOW, ALIAS_SHOW, NULL, show_usage, 1, false, false },
+ { OPTION_CACHE, ALIAS_CACHE, NULL, cache_usage, 1, true, false },
+ { OPTION_RAND, ALIAS_RAND, NULL, random_usage, 1, false, false },
+ { OPTION_REP, ALIAS_REP, NULL, repeat_usage, 1, false, false },
+ { OPTION_CREPORT, ALIAS_CREPORT, NULL, creport_usage, 1, false, false },
+ { OPTION_BUFFER, ALIAS_BUFFER, NULL, buffer_usage, 1, true, false },
+ { OPTION_SLEEP, ALIAS_SLEEP, NULL, sleep_usage, 1, true, false },
+ { OPTION_TIMEOUT, ALIAS_TIMEOUT, NULL, timeout_usage, 1, true, false },
+ { OPTION_COMPLETE,NULL, NULL, complete_usage, 1, false, false },
+ { OPTION_TRUNC, NULL, NULL, truncate_usage, 1, false, false },
+ { OPTION_START, NULL, NULL, start_usage, 1, true, false },
+ { OPTION_COUNT, NULL, NULL, count_usage, 1, true, false },
+ { OPTION_PROGRESS,NULL, NULL, progress_usage, 1, false, false },
+ { OPTION_RELIABLE,NULL, NULL, reliable_usage, 1, false, false }
+};
+
+rc_t CC Usage ( const Args * args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ uint32_t i;
+ rc_t rc;
+
+ if ( args == NULL )
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
+ else
+ rc = ArgsProgram ( args, &fullpath, &progname );
+
+ if ( rc )
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary ( progname );
+
+ KOutMsg ( "Options:\n" );
+
+ HelpOptionsStandard ();
+ for ( i = 0; i < sizeof ( MyOptions ) / sizeof ( MyOptions[ 0 ] ); ++i )
+ HelpOptionLine ( MyOptions[i].aliases, MyOptions[i].name, NULL, MyOptions[i].help );
+
+ return rc;
+}
+
+
+/* Version EXTERN
+ * return 4-part version code: 0xMMmmrrrr, where
+ * MM = major release
+ * mm = minor release
+ * rrrr = bug-fix release
+ */
+ver_t CC KAppVersion ( void )
+{
+ return KGET_VERS;
+}
+
+
+typedef struct fetch_ctx
+{
+ const char *url;
+ const char *destination;
+ const char *cache_file;
+ size_t blocksize;
+ size_t start, count;
+ size_t buffer_size;
+ size_t sleep_time;
+ size_t timeout_time;
+ bool verbose;
+ bool show_filesize;
+ bool random;
+ bool with_repeats;
+ bool show_curl_version;
+ bool report_cache;
+ bool check_cache_complete;
+ bool truncate_cache;
+ bool local_read_only;
+ bool show_progress;
+ bool reliable;
+} fetch_ctx;
+
+
+static rc_t src_2_dst( const KFile *src, KFile *dst, char * buffer,
+ uint64_t pos, size_t * num_read, fetch_ctx * ctx )
+{
+ rc_t rc;
+ size_t n_transfer = ( ctx->count == 0 ? ctx->blocksize : ctx->count );
+
+ if ( ctx->timeout_time == 0 )
+ rc = KFileReadAll ( src, pos, buffer, n_transfer, num_read );
+ else
+ {
+ timeout_t tm;
+ rc = TimeoutInit ( &tm, ctx->timeout_time );
+ if ( rc == 0 )
+ rc = KFileTimedReadAll ( src, pos, buffer, n_transfer, num_read, &tm );
+ }
+ if ( rc == 0 && *num_read > 0 )
+ {
+ size_t num_writ;
+ rc = KFileWriteAll ( dst, pos, buffer, *num_read, &num_writ );
+ }
+ return rc;
+}
+
+
+static rc_t block_loop_in_order( const KFile *src, KFile *dst, char * buffer,
+ uint64_t * bytes_copied, fetch_ctx * ctx )
+{
+ rc_t rc = 0;
+ uint64_t pos = 0;
+ uint32_t blocks = 0;
+ size_t num_read = 1;
+ KOutMsg( "copy-mode : linear read/write\n" );
+ while ( rc == 0 && num_read > 0 )
+ {
+ rc = src_2_dst( src, dst, buffer, pos, &num_read, ctx );
+ if ( rc == 0 ) pos += num_read;
+ if ( ctx->show_progress && ( ( blocks & 0x0F ) == 0 ) ) KOutMsg( "." );
+ blocks++;
+ if ( ctx->sleep_time > 0 ) KSleepMs( ctx->sleep_time );
+ }
+ *bytes_copied = pos;
+ if ( ctx->show_progress ) KOutMsg( "\n" );
+ KOutMsg( "%d blocks a %d bytes\n", blocks, ctx->blocksize );
+ return rc;
+}
+
+
+
+static uint32_t randr( uint32_t min, uint32_t max )
+{
+ double scaled = ( ( double )rand() / RAND_MAX );
+ return ( ( max - min + 1 ) * scaled ) + min;
+}
+
+
+static rc_t block_loop_random( const KFile *src, KFile *dst, char * buffer,
+ uint64_t *bytes_copied, fetch_ctx * ctx )
+{
+ uint64_t src_size;
+ rc_t rc = KFileSize ( src, &src_size );
+ KOutMsg( "copy-mode : random blocks\n" );
+ if ( rc == 0 )
+ {
+ rc = KFileSetSize ( dst, src_size );
+ if ( rc == 0 )
+ {
+ uint32_t block_count = ( src_size / ctx->blocksize ) + 1;
+ uint32_t * block_vector = malloc( block_count * ( sizeof * block_vector ) );
+ if ( block_vector != NULL )
+ {
+ uint32_t loop;
+
+ /* fill the block_vector with ascending numbers */
+ for ( loop = 0; loop < block_count; loop++ )
+ block_vector[ loop ] = loop;
+
+ /* randomize them */
+ for ( loop = 0; loop < block_count; loop++ )
+ {
+ uint32_t src_idx = randr( 0, block_count - 1 );
+ uint32_t dst_idx = randr( 0, block_count - 1 );
+ /* swap it... */
+ uint32_t tmp = block_vector[ dst_idx ];
+ block_vector[ dst_idx ] = block_vector[ src_idx ];
+ block_vector[ src_idx ] = tmp;
+ }
+
+ for ( loop = 0; rc == 0 && loop < block_count; loop++ )
+ {
+ size_t num_read;
+ uint64_t pos = ctx->blocksize;
+ pos *= block_vector[ loop ];
+ rc = src_2_dst( src, dst, buffer, pos, &num_read, ctx );
+ if ( rc == 0 ) *bytes_copied += num_read;
+ if ( ctx->show_progress && ( ( loop & 0x0F ) == 0 ) ) KOutMsg( "." );
+ if ( ctx->sleep_time > 0 ) KSleepMs( ctx->sleep_time );
+
+ }
+ free( block_vector );
+ if ( ctx->show_progress ) KOutMsg( "\n" );
+ KOutMsg( "%d blocks a %d bytes\n", loop, ctx->blocksize );
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t copy_file( const KFile * src, KFile * dst, fetch_ctx * ctx )
+{
+ rc_t rc = 0;
+ size_t buffer_size = ( ctx->count == 0 ? ctx->blocksize : ctx->count );
+ char * buffer = malloc( buffer_size );
+ if ( buffer == NULL )
+ {
+ rc = RC( rcExe, rcFile, rcPacking, rcMemory, rcExhausted );
+ KOutMsg( "cant make buffer of size %u\n", buffer_size );
+ }
+ else
+ {
+ uint64_t bytes_copied = 0;
+ if ( ctx->count == 0 )
+ {
+ if ( ctx->random )
+ rc = block_loop_random( src, dst, buffer, &bytes_copied, ctx );
+ else
+ rc = block_loop_in_order( src, dst, buffer, &bytes_copied, ctx );
+ }
+ else
+ {
+ size_t num_read;
+ rc = src_2_dst( src, dst, buffer, ctx->start, &num_read, ctx );
+ if ( rc == 0 ) bytes_copied = num_read;
+ }
+ KOutMsg( "%lu bytes copied\n", bytes_copied );
+ free( buffer );
+ }
+ return rc;
+}
+
+
+static rc_t fetch_cached( KDirectory *dir, const KFile *src, KFile *dst, fetch_ctx *ctx )
+{
+ rc_t rc = 0;
+ const KFile *tee; /* this is the file that forks persistent_content with remote */
+ size_t cache_tee_block = 0; /* ctx->blocksize; */
+
+ KOutMsg( "persistent cache created\n" );
+
+ rc = KDirectoryMakeCacheTee ( dir, /* the KDirectory for the the sparse-file */
+ &tee, /* the newly created cache-tee-file */
+ src, /* the file that we are wrapping ( usually the remote http-file ) */
+ cache_tee_block, /* how big one block in the cache-tee-file will be */
+ ctx->cache_file ); /* the sparse-file we use write to */
+ if ( rc == 0 )
+ {
+ KOutMsg( "cache tee created\n" );
+ rc = copy_file( tee, dst, ctx );
+ KFileRelease( tee );
+ }
+ return rc;
+}
+
+
+static void extract_name( char ** dst, const char * url )
+{
+ char * last_slash = string_rchr ( url, string_size( url ), '/' );
+ if ( last_slash == NULL )
+ *dst = string_dup_measure( "out.bin", NULL );
+ else
+ *dst = string_dup_measure( last_slash + 1, NULL );
+}
+
+
+static rc_t fetch_from( KDirectory *dir, fetch_ctx *ctx, char * outfile,
+ const KFile * src )
+{
+ uint64_t file_size;
+ rc_t rc = KFileSize( src, &file_size );
+ if ( rc != 0 )
+ {
+ KOutMsg( "cannot disover src-size >%R<\n", rc );
+ }
+ else
+ {
+ KFile *dst;
+ KOutMsg( "src-size = %lu\n", file_size );
+ rc = KDirectoryCreateFile ( dir, &dst, false, 0664, kcmInit, outfile );
+ if ( rc == 0 )
+ {
+ KOutMsg( "dst >%s< created\n", outfile );
+ if ( rc == 0 )
+ {
+ if ( ctx->cache_file != NULL )
+ rc = fetch_cached( dir, src, dst, ctx );
+ else
+ rc = copy_file( src, dst, ctx );
+ }
+ KFileRelease( dst );
+ }
+ }
+ return rc;
+}
+
+
+static rc_t make_remote_file( struct KNSManager * kns_mgr, const KFile ** src, fetch_ctx * ctx )
+{
+ rc_t rc;
+
+ KNSManagerSetVerbose( kns_mgr, ctx->verbose );
+ if ( ctx->reliable )
+ rc = KNSManagerMakeReliableHttpFile( kns_mgr, src, NULL, 0x01010000, ctx->url );
+ else
+ rc = KNSManagerMakeHttpFile ( kns_mgr, src, NULL, 0x01010000, ctx->url );
+
+ if ( rc != 0 )
+ {
+ if ( ctx->reliable )
+ (void)LOGERR( klogInt, rc, "KNSManagerMakeReliableHttpFile() failed" );
+ else
+ (void)LOGERR( klogInt, rc, "KNSManagerMakeHttpFile() failed" );
+ }
+ else
+ {
+ const KFile * temp_file;
+ if ( ctx->buffer_size > 0 )
+ {
+ /* there is no cache_location! just wrap the remote file in a buffer */
+ /* rc = KBigBlockReaderMake ( &temp_file, *src, ctx->buffer_size ); */
+ rc = KBufFileMakeRead ( & temp_file, *src, ctx->buffer_size );
+ if ( rc == 0 )
+ {
+ KOutMsg( "remote-file wrapped in new big-block-reader of size %d\n", ctx->buffer_size );
+ KFileRelease ( *src );
+ *src = temp_file;
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t fetch( KDirectory *dir, fetch_ctx *ctx )
+{
+ rc_t rc = 0;
+ char * outfile;
+ const KFile * remote;
+ struct KNSManager * kns_mgr;
+
+ if ( ctx->destination == NULL )
+ extract_name( &outfile, ctx->url );
+ else
+ outfile = string_dup_measure( ctx->destination, NULL );
+
+ KOutMsg( "fetching: >%s<\n", ctx->url );
+ KOutMsg( "into : >%s<\n", outfile );
+ if ( ctx->count > 0 )
+ KOutMsg( "range : %u.%u\n", ctx->start, ctx->count );
+
+ rc = KNSManagerMake ( &kns_mgr );
+ if ( rc != 0 )
+ (void)LOGERR( klogInt, rc, "KNSManagerMake() failed" );
+ else
+ {
+ rc = make_remote_file( kns_mgr, &remote, ctx );
+ if ( rc == 0 )
+ {
+ rc = fetch_from( dir, ctx, outfile, remote );
+ KFileRelease( remote );
+ }
+ KNSManagerRelease( kns_mgr );
+ }
+
+ free( outfile );
+ return rc;
+}
+
+static rc_t show_size( KDirectory *dir, fetch_ctx *ctx )
+{
+ rc_t rc = 0;
+ const KFile * remote;
+ struct KNSManager * kns_mgr;
+
+ KOutMsg( "source: >%s<\n", ctx->url );
+ rc = KNSManagerMake ( &kns_mgr );
+ if ( rc != 0 )
+ (void)LOGERR( klogInt, rc, "KNSManagerMake() failed" );
+ else
+ {
+ rc = make_remote_file( kns_mgr, &remote, ctx );
+ if ( rc == 0 )
+ {
+ uint64_t file_size;
+ rc = KFileSize( remote, &file_size );
+ KOutMsg( "file-size = %u\n", file_size );
+ KFileRelease( remote );
+ }
+ KNSManagerRelease( kns_mgr );
+ }
+ return rc;
+}
+
+
+static rc_t check_cache_complete( KDirectory *dir, fetch_ctx *ctx )
+{
+ rc_t rc = 0;
+ const KFile *f;
+
+ KOutMsg( "checking if this cache file >%s< is complete\n", ctx->url );
+
+ rc = KDirectoryOpenFileRead( dir, &f, "%s", ctx->url );
+ if ( rc == 0 )
+ {
+ bool is_complete;
+ rc = IsCacheFileComplete( f, &is_complete );
+ if ( rc != 0 )
+ KOutMsg( "error performing IsCacheFileComplete() %R\n", rc );
+ else
+ {
+ if ( is_complete )
+ KOutMsg( "the file is complete\n" );
+ else
+ {
+ float percent = 0;
+ uint64_t bytes_cached;
+ rc = GetCacheCompleteness( f, &percent, &bytes_cached );
+ if ( rc == 0 )
+ KOutMsg( "the file is %f%% complete ( %lu bytes are cached )\n", percent, bytes_cached );
+ }
+ }
+ KFileRelease( f );
+ }
+ return rc;
+}
+
+
+static rc_t truncate_cache( KDirectory *dir, fetch_ctx *ctx )
+{
+ rc_t rc = 0;
+ KFile *f;
+
+ KOutMsg( "truncating this cache file >%s< (from it's trailing bitmap)\n", ctx->url );
+
+ rc = KDirectoryOpenFileWrite( dir, &f, true, "%s", ctx->url );
+ if ( rc == 0 )
+ {
+ rc = TruncateCacheFile( f );
+ if ( rc == 0 )
+ KOutMsg( "the file was truncated\n" );
+ else
+ KOutMsg( "the file was not truncated: %R\n", rc );
+ KFileRelease( f );
+ }
+ return rc;
+}
+
+
+rc_t get_bool( Args * args, const char *option, bool *value )
+{
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, option, &count );
+ *value = ( rc == 0 && count > 0 );
+ return rc;
+}
+
+
+rc_t get_str( Args * args, const char *option, const char ** value )
+{
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, option, &count );
+ *value = NULL;
+ if ( rc == 0 && count > 0 )
+ rc = ArgsOptionValue( args, option, 0, (const void **)value );
+ return rc;
+}
+
+
+rc_t get_size_t( Args * args, const char *option, size_t *value, size_t dflt )
+{
+ const char * s;
+ rc_t rc = get_str( args, option, &s );
+ *value = dflt;
+ if ( rc == 0 && s != NULL )
+ *value = atoi( s );
+ else
+ *value = dflt;
+ return rc;
+}
+
+
+rc_t get_fetch_ctx( Args * args, fetch_ctx * ctx )
+{
+ rc_t rc = 0;
+ uint32_t count;
+
+ ctx->url = NULL;
+ ctx->verbose = false;
+ rc = ArgsParamCount( args, &count );
+ if ( rc == 0 && count > 0 )
+ {
+ rc = ArgsParamValue( args, 0, (const void **)&ctx->url );
+ if ( rc == 0 )
+ {
+ if ( count > 1 )
+ rc = ArgsParamValue( args, 1, (const void **)&ctx->destination );
+ else
+ ctx->destination = NULL;
+ }
+ }
+
+ if ( rc == 0 ) rc = get_bool( args, OPTION_VERB, &ctx->verbose );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_SHOW, &ctx->show_filesize );
+ if ( rc == 0 ) rc = get_str( args, OPTION_CACHE, &ctx->cache_file );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_RAND, &ctx->random );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_REP, &ctx->with_repeats );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_CREPORT, &ctx->report_cache );
+ if ( rc == 0 ) rc = get_size_t( args, OPTION_BLOCK, &ctx->blocksize, ( 32 * 1024 ) );
+ if ( rc == 0 ) rc = get_size_t( args, OPTION_BUFFER, &ctx->buffer_size, 0 );
+ if ( rc == 0 ) rc = get_size_t( args, OPTION_SLEEP, &ctx->sleep_time, 0 );
+ if ( rc == 0 ) rc = get_size_t( args, OPTION_TIMEOUT, &ctx->timeout_time, 0 );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_COMPLETE, &ctx->check_cache_complete );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_TRUNC, &ctx->truncate_cache );
+ if ( rc == 0 ) rc = get_size_t( args, OPTION_START, &ctx->start, 0 );
+ if ( rc == 0 ) rc = get_size_t( args, OPTION_COUNT, &ctx->count, 0 );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_PROGRESS, &ctx->show_progress );
+ if ( rc == 0 ) rc = get_bool( args, OPTION_RELIABLE, &ctx->reliable );
+
+ return rc;
+}
+
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ Args * args;
+ rc_t rc;
+
+ rc = ArgsMakeAndHandle ( &args, argc, argv, 1,
+ MyOptions, sizeof ( MyOptions ) / sizeof ( OptDef ) );
+ if ( rc == 0 )
+ {
+ fetch_ctx ctx;
+
+ rc = get_fetch_ctx( args, &ctx );
+ if ( rc == 0 )
+ {
+ if ( ctx.url == NULL )
+ KOutMsg( "URL is missing!\n" );
+ else
+ {
+ KDirectory *dir;
+ rc = KDirectoryNativeDir ( &dir );
+ if ( rc == 0 )
+ {
+ if ( ctx.check_cache_complete )
+ rc = check_cache_complete( dir, &ctx );
+ else if ( ctx.truncate_cache )
+ rc = truncate_cache( dir, &ctx );
+ else if ( ctx.show_filesize )
+ rc = show_size( dir, &ctx );
+ else
+ rc = fetch( dir, &ctx );
+
+ KDirectoryRelease ( dir );
+ }
+ }
+ }
+ ArgsWhack ( args );
+ }
+ else
+ KOutMsg( "ArgsMakeAndHandle() failed %R\n", rc );
+
+ return rc;
+}
diff --git a/tools/kget/kget.vers b/tools/kget/kget.vers
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/tools/kget/kget.vers
@@ -0,0 +1 @@
+1.0.0
diff --git a/tools/kqsh/kqsh.c b/tools/kqsh/kqsh.c
index 47d26fb..bfeab9d 100644
--- a/tools/kqsh/kqsh.c
+++ b/tools/kqsh/kqsh.c
@@ -423,7 +423,7 @@ rc_t kqsh_main ( int argc, char *argv [] )
/* add each path in order */
for ( ix = 0; ix < pcount; ++ ix )
{
- rc = ArgsOptionValue ( args, OPTION_LIBPATH, ix, & pc );
+ rc = ArgsOptionValue ( args, OPTION_LIBPATH, ix, (const void **)& pc );
if ( rc != 0 )
{
PLOGERR ( klogInt, ( klogInt, rc, "failed to retrieve '$(option)' option value [ $(idx) ]", "option=" OPTION_LIBPATH ",idx=%u", ix ) );
diff --git a/tools/kqsh/kqsh.vers b/tools/kqsh/kqsh.vers
index 8e8299d..35d16fb 100644
--- a/tools/kqsh/kqsh.vers
+++ b/tools/kqsh/kqsh.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/meta-sync/meta-sync.c b/tools/meta-sync/meta-sync.c
deleted file mode 100644
index f652b7f..0000000
--- a/tools/meta-sync/meta-sync.c
+++ /dev/null
@@ -1,1517 +0,0 @@
-/*******************************************************************************
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * =============================================================================
- */
-
-#include "meta-sync.vers.h" /* META_SYNC_VERS */
-#include <kapp/main.h>
-#include <sra/srapath.h>
-#include <sra/sraschema.h> /* VDBManagerMakeSRASchema */
-#include <vdb/manager.h> /* VDBManager */
-#include <vdb/database.h> /* VDatabase */
-#include <vdb/table.h> /* VTable */
-#include <vdb/schema.h> /* VSchemaRelease */
-#include <vdb/cursor.h> /* VCursor */
-#include <rdbms/sybase.h> /* SybaseInit */
-#include <kfs/directory.h> /* KDirectory */
-#include <kfs/file.h> /* KFile */
-#include <kxml/xml.h> /* KXMLMgr */
-#include <klib/log.h> /* LOGERR */
-#include <klib/out.h> /* OUTMSG */
-#include <klib/debug.h> /* DBGMSG */
-#include <klib/rc.h> /* RC */
-#include <insdc/sra.h> /* SRA_READ_TYPE_... */
-#include <assert.h>
-#include <stdio.h> /* vspnrintf */
-#include <stdlib.h> /* malloc */
-#include <string.h> /* memset */
-#include <os-native.h> /* strncasecmp on Windows */
-
-#define DISP_RC(rc, msg) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, msg))
-#define DISP_RC2(rc, name, msg) (void)((rc == 0) ? 0 : \
- PLOGERR(klogInt, (klogInt,rc, "$(msg): $(name)","msg=%s,name=%s",msg,name)))
-#define DESTRUCT(type, obj) do { rc_t rc2 = type##Release(obj); \
- if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
-
-enum ECol {
- eFIXED_SPOT_LEN
- , eLABEL
- , eLABEL_LEN
- , eLABEL_START
- , eLINKER_SEQUENCE
-/* , eNREADS */
- , ePLATFORM
- , eREAD_LEN
- , eREAD_START
- , eREAD_TYPE
- , eSPOT_GROUP
- , eSPOT_LEN
-};
-/*enum EType {
- eUndetected
- , e454_2
- , e454_2_AA
- , e454_4_FollowingLinker
- , e454_4_ForwardForward
- , e454_4_RelativeOrder
- , e454_4_FFLF
- , eIllumina_1
-};*/
-enum EColType {
- eAscii
- , eReadType
- , eUint8
- , eUint32
-};
-typedef struct Col {
- enum ECol type;
- const char* name;
- enum EColType datatype;
- bool isArray;
- uint32_t idx;
- char buffer[70];
- uint32_t row_len;
- bool exists;
-} Col;
-enum ELabel {
- eUnsetL
- , eInvalidF7L
- , eAdapterL
- , eBarcodeL
- , eFL
- , eF3L
- , eForwardL
- , eFragmentL
- , eLinkerL
- , eMate1L
- , eMate2L
- , eR3L
- , eReverseL
- , erRNA_primerL
-};
-enum EReadType {
- eUnknownRT
- , eAdapterRT
- , eBarCodeRT
- , eForwardRT
- , eLinkerRT
- , ePrimerRT
- , eReverseRT
-};
-#define unsetReadClass -1
-#define MAX_NREADS 5
-typedef struct SReadXml {
- bool used;
-
- /* SRA_READ_TYPE_TECHNICAL SRA_READ_TYPE_BIOLOGICAL;
- unsetReadClass is a special value */
- int readClass;
-
- enum EReadType readType;
-} SReadXml;
-typedef struct MetaDataXml {
- INSDC_SRA_platform_id platform;
- uint32_t nreads;
- SReadXml read[MAX_NREADS];
- bool variableReadLen;
-} MetaDataXml;
-typedef struct SReadDb {
- bool used;
- /* READ_TYPE */
- /* SRA_READ_TYPE_TECHNICAL SRA_READ_TYPE_BIOLOGICAL;
- unsetReadClass is a special value */
- int readClass;
- int readOrientation; /* SRA_READ_TYPE_FORWARD SRA_READ_TYPE_REVERSE */
- enum ELabel label; /* LABEL, LABEL_START, LABEL_LEN */
-} SReadDb;
-typedef struct MetaDataDb {
- INSDC_SRA_platform_id platform; /* PLATFORM */
- uint32_t nreads; /* number of READ_START elements */
- SReadDb read[MAX_NREADS];
-} MetaDataDb;
-typedef struct CmdLine {
- bool notFound;
- const char* tbl;
- const char* exp;
- const char* run;
- const char* acc;
-} CmdLine;
-/* true if differ */
-static bool Compare(const char* sample, const char* test, size_t test_len)
-{
- if (!sample || !test || !test_len)
- { return true; }
- if (strlen(sample) != test_len)
- { return true; }
- return strncasecmp(test, sample, test_len);
-}
-typedef struct LabelStrEnum {
- enum ELabel e;
- const char* s;
-} LabelStrEnum;
-typedef struct PlatfromStrEnum {
- INSDC_SRA_platform_id e;
- const char* s;
-} PlatfromStrEnum;
-typedef struct ReadTypeStrEnum {
- enum EReadType e;
- const char* s;
-} ReadTypeStrEnum;
-#define TYPE_STR_CONVERT(name, elm, table) \
- static const char* Enum##name##2Str(elm e) { \
- int i = 0; \
- for (i = 0; i < sizeof table / sizeof table[0]; ++i) { \
- if (table[i].e == e) { return table[i].s; } \
- } \
- assert(0); \
- return NULL; \
- } \
- static elm Str##name##2Enum(const char* s) { \
- int i = 0; \
- assert(s); \
- for (i = 0; i < sizeof table / sizeof table[0]; ++i) { \
- assert(table[i].s); \
- if (strcmp(table[i].s, s) == 0) { return table[i].e; } \
- } \
- assert(0); \
- return 0; \
- } \
- static elm Strn##name##2Enum(const char* s, size_t l) { \
- int i = 0; \
- assert(s && l); \
- for (i = 0; i < sizeof table / sizeof table[0]; ++i) { \
- assert(table[i].s); \
- if (!Compare(table[i].s, s, l)) { return table[i].e; } \
- } \
- /* printf("Strn" #name "2Enum(%s) failed\n", s); \
- assert(0); */ \
- return 0; \
- }
-#define ENUM_STR_CONVERT(name, elm, table) TYPE_STR_CONVERT(name,enum elm,table)
-static PlatfromStrEnum platforms[] = {
- { SRA_PLATFORM_454 , "LS454" }
- , { SRA_PLATFORM_ABSOLID , "ABI_SOLID" }
- , { SRA_PLATFORM_COMPLETE_GENOMICS, "COMPLETE_GENOMICS" }
- , { SRA_PLATFORM_HELICOS , "HELICOS" }
- , { SRA_PLATFORM_ILLUMINA , "ILLUMINA" }
- , { SRA_PLATFORM_PACBIO_SMRT , "PACBIO_SMRT" }
-};
-TYPE_STR_CONVERT(Platform, INSDC_SRA_platform_id, platforms)
-static ReadTypeStrEnum readTypes[] = {
- { eAdapterRT, "Adapter" }
- , { eBarCodeRT, "Barcode" }
- , { eForwardRT, "Forward" }
- , { eLinkerRT , "Linker" }
- , { ePrimerRT , "Primer" }
- , { eReverseRT, "Reverse" }
-};
-ENUM_STR_CONVERT(ReadType, EReadType, readTypes)
-static LabelStrEnum labels[] = {
- { eAdapterL, "Adapter" }
- , { eBarcodeL , "Barcode" }
- , { eFL , "F" }
- , { eF3L , "F3" }
- , { eForwardL , "Fragment" }
- , { eFragmentL , "Forward" }
- , { eLinkerL , "Linker" }
- , { eMate1L , "Mate1" }
- , { eMate2L , "Mate2" }
- , { eR3L , "R3" }
- , { eReverseL , "Reverse" }
- , { erRNA_primerL, "Rrna_primer" }
-};
-ENUM_STR_CONVERT(Label, ELabel, labels)
-static rc_t ParseSpotDesc(const KXMLNode* nodeSpotDesc,
- MetaDataXml* md)
-{
- rc_t rc = 0;
- const KXMLNodeset* eNs = NULL;
- assert(nodeSpotDesc);
- if (rc == 0) {
- uint32_t i = 0;
- const char path[] = "SPOT_DECODE_SPEC/READ_SPEC";
- rc = KXMLNodeOpenNodesetRead(nodeSpotDesc, &eNs, path);
- DISP_RC2(rc, path, "while opening Nodeset");
- if (rc == 0) {
- uint32_t n = 0;
- rc = KXMLNodesetCount(eNs, &n);
- DISP_RC2(rc, path, "while counting Nodeset");
- if (rc == 0) {
- DBGMSG(DBG_APP, DBG_COND_1, ("READ_SPEC Count = %d\n", n));
- md->nreads = n;
- }
- for (i = 0; i < n && rc == 0; ++i) {
- const KXMLNode* node = NULL;
- rc = KXMLNodesetGetNodeRead(eNs, &node, i);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "while reading node $(n) of $(path)", "n=%d,path=%s",
- i, path));
- }
- else {
- uint32_t index = UINT32_MAX;
- char* rclass = NULL;
- char* rtype = NULL;
- uint32_t j, c = 0;
- rc = KXMLNodeCountChildNodes(node, &c);
- DISP_RC2(rc, path, "while counting Children");
- for (j = 0; j < c && rc == 0; ++j) {
- const KXMLNode* child;
- rc = KXMLNodeGetNodeRead(node, &child, j);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "while reading child $(n) of $(path)",
- "n=%d,path=%s", j, path));
- }
- else {
- const char* name = NULL;
- rc = KXMLNodeElementName(child, &name);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "while reading name of child $(n) of $(path)",
- "n=%d,path=%s", j, path));
- }
- else {
- if (!strcmp(name, "BASE_COORD")) {
- }
- else if (!strcmp(name,
- "EXPECTED_BASECALL"))
- {}
- else if (!strcmp(name,
- "EXPECTED_BASECALL_TABLE"))
- {}
- else if (!strcmp(name, "READ_CLASS")) {
- rc = KXMLNodeReadCStr(child, &rclass, NULL);
- DISP_RC2(rc, name, "reading as CString");
- DBGMSG(DBG_APP, DBG_COND_1,
- ("%d %s %s\n", i, name, rclass));
- }
- else if (!strcmp(name, "READ_INDEX")) {
- rc = KXMLNodeReadAsU32(child, &index);
- DISP_RC2(rc, name, "reading as U32");
- DBGMSG(DBG_APP, DBG_COND_1,
- ("%d %s %d\n", i, name, index));
- }
- else if (!strcmp(name, "READ_LABEL")) {
- }
- else if (!strcmp(name, "READ_TYPE")) {
- rc = KXMLNodeReadCStr(child, &rtype, NULL);
- DISP_RC2(rc, name, "reading as CString");
- DBGMSG(DBG_APP, DBG_COND_1,
- ("%d %s %s\n", i, name, rtype));
- }
- else if (!strcmp(name, "RELATIVE_ORDER")) {
- }
- else {
- rc = RC(rcExe, rcXmlDoc,
- rcParsing, rcName, rcUnexpected);
- PLOGERR(klogInt, (klogInt, rc,
- "$(name)", "name=%s", name));
- }
- }
- }
- {
- rc_t rc2 = KXMLNodeRelease(child);
- if (rc2 && !rc)
- { rc = rc2; }
- child = NULL;
- }
- }
- if (rc == 0) {
- /*if (bsCoord && expBCall) {
- rc = RC(rcExe,
- rcXmlDoc, rcParsing, rcData, rcInvalid);
- LOGERR(klogInt, rc,
- "both BASE_COORD and EXPECTED_BASECALL "
- "are present");
- }
- else*/ if (index == UINT32_MAX
- || index >= sizeof md->read / sizeof md->read[0]
- || md->read[index].used)
- {
- RC(rcExe, rcXmlDoc, rcParsing, rcData, rcDuplicate);
- PLOGERR(klogInt, (klogInt, rc,
- "READ_INDEX $(i)", "i=%d", index));
- }
- /*else if (bsCoord) {
- DBGMSG(DBG_APP, DBG_COND_1, ("%d %s %s bk=%d\n",
- index, rclass, rtype, bsCoord));
- md->read[index].baseCoord = bsCoord;
- }
- else if (expBCall) {
- md->read[index].expectedBaseCall = expBCall;
- DBGMSG(DBG_APP, DBG_COND_1,
- ("%d %s %s eb=%s(%lu)\n", index,
- rclass, rtype, expBCall, strlen(expBCall)));
- }
- else if (reltOrder) {
- md->read[index].relativeOrder = true;
- }
- else {
- rc = RC(rcExe,
- rcXmlDoc, rcParsing, rcData, rcUnexpected);
- LOGERR(klogInt, rc,
- "XML Read coordinates were not recognized");
- }*/
- }
- if (rc == 0) {
- SReadXml* r = &md->read[index];
- int readClass = unsetReadClass;
- enum EReadType readType = eUnknownRT;
-
- r->used = true;
-
- if (!strcmp(rclass, "Technical Read"))
- { readClass = SRA_READ_TYPE_TECHNICAL; }
- else if (!strcmp(rclass, "Application Read"))
- { readClass = SRA_READ_TYPE_BIOLOGICAL; }
- r->readClass = readClass;
-
-/* readType = ; TODO */
- if (!strcmp(rtype, "Adapter"))
- { readType = eAdapterRT; }
- else if (!strcmp(rtype, "BarCode"))
- { readType = eBarCodeRT; }
- else if (!strcmp(rtype, "Forward"))
- { readType = eForwardRT; }
- else if (!strcmp(rtype, "Linker"))
- { readType = eLinkerRT; }
- else if (!strcmp(rtype, "Primer"))
- { readType = ePrimerRT; }
- else if (!strcmp(rtype, "Reverse"))
- { readType = eReverseRT; }
- else {
- rc = RC(rcExe,
- rcXmlDoc, rcParsing, rcData, rcUnexpected);
- PLOGERR(klogInt, (klogInt, rc,
- "READ_TYPE value: $(val)", "val=%s", rtype));
- }
- r->readType = readType;
- }
- }
- {
- rc_t rc2 = KXMLNodeRelease(node);
- if (rc2 && !rc)
- { rc = rc2; }
- node = NULL;
- }
- }
- }
- for (i = 0; i < md->nreads && rc == 0; ++i) {
- SReadXml* r = &md->read[i];
- if (!r->used) {
- rc = RC(rcExe, rcXmlDoc, rcParsing, rcData, rcInconsistent);
- PLOGERR(klogInt,
- (klogInt, rc, "READ_INDEX $(i)", "i=%d", i));
- }
- }
- }
- DESTRUCT(KXMLNodeset, eNs);
- return rc;
-}
-
-static rc_t ParseXml(const KXMLDoc* exp, const KXMLDoc* run,
- MetaDataXml* md)
-{
- rc_t rc = 0;
- const KXMLNode* nodeExp = NULL;
- const KXMLNode* nodeSpotDesc = NULL;
- assert(exp && run && md);
- if (rc == 0) {
- const KXMLNodeset* ns = NULL;
- const char runNode[] = "/RUN/SPOT_DESCRIPTOR";
- rc = KXMLDocOpenNodesetRead(run, &ns, runNode);
- DISP_RC2(rc, runNode, "while opening Nodeset");
- if (rc == 0) {
- uint32_t n = 0;
- rc = KXMLNodesetCount(ns, &n);
- DISP_RC2(rc, runNode, "while counting Nodeset");
- if (rc == 0) {
- if (n > 1) {
- rc = RC(rcExe, rcXmlDoc, rcReading, rcNode, rcExcessive);
- PLOGERR(klogInt, (klogInt, rc,
- "$(path) in run XML", "path=%s", runNode));
- }
- else if (n == 1) {
- rc = KXMLNodesetGetNodeRead(ns, &nodeSpotDesc, 0);
- if (rc == 0) {
- DBGMSG(DBG_APP, DBG_COND_1,
- ("Use RUN SPOT_DECSRIPTOR\n"));
- rc = ParseSpotDesc(nodeSpotDesc, md);
- }
- }
- }
- }
- DESTRUCT(KXMLNodeset, ns);
- }
- if (rc == 0) {
- const char path[] = "/EXPERIMENT";
- const KXMLNodeset* ns = NULL;
- rc = KXMLDocOpenNodesetRead(exp, &ns, path);
- DISP_RC2(rc, path, "while opening Nodeset");
- if (rc == 0) {
- rc = KXMLNodesetGetNodeRead(ns, &nodeExp, 0);
- DISP_RC2(rc, path, "while accessing Node 0");
- }
- }
- if (rc == 0) {
- const char path[] = "PLATFORM";
- const KXMLNode* node = NULL;
- rc = KXMLNodeGetFirstChildNodeRead(nodeExp, &node, path);
- DISP_RC2(rc, path, "while accessing first Node");
- if (rc == 0) {
- const KNamelist* children = NULL;
- rc = KXMLNodeListChild(node, &children);
- DISP_RC2(rc, path, "while listing children");
- if (rc == 0) {
- const char* name = NULL;
- rc = KNamelistGet(children, 0, &name);
- DISP_RC2(rc, path, "while getting child[0]");
- if (rc == 0) {
- DBGMSG(DBG_APP, DBG_COND_1, ("PLATFORM = %s\n", name));
- md->platform = StrPlatform2Enum(name);
- if (md->platform == SRA_PLATFORM_UNDEFINED) {
- rc = RC(rcExe,
- rcXmlDoc, rcParsing, rcId, rcUnrecognized);
- DISP_RC2(rc, name, "PLATFORM");
- }
- }
- }
- }
- }
- if (rc == 0 && nodeSpotDesc == NULL) {
- const char path[] = "DESIGN/SPOT_DESCRIPTOR";
- rc = KXMLNodeGetFirstChildNodeRead(nodeExp, &nodeSpotDesc, path);
- DISP_RC2(rc, path, "while accessing first Node");
- if (rc == 0) {
- DBGMSG(DBG_APP, DBG_COND_1, ("Use EXPERIMENT SPOT_DESCRIPTOR\n"));
- rc = ParseSpotDesc(nodeSpotDesc, md);
- }
- }
- DESTRUCT(KXMLNode, nodeExp);
- DESTRUCT(KXMLNode, nodeSpotDesc);
- return rc;
-}
-
-static void ReportDiff(const CmdLine* aArgs, const MetaDataXml* xml,
- const char *format, ...)
-{
- static bool called = false;
- char buffer[256];
- va_list args;
- va_start(args, format);
- assert(aArgs && xml && format);
- if (!called) {
- called = true;
- OUTMSG(("%s %s ", aArgs->acc, EnumPlatform2Str(xml->platform)));
- }
- else { OUTMSG((" ")); }
- vsnprintf(buffer, sizeof buffer, format, args);
- OUTMSG((buffer));
- va_end(args);
-}
-
-static bool AnalyzeNReport(const MetaDataDb* db, const MetaDataXml* xml,
- const CmdLine* args)
-{
- ReportDiff(args, xml, "NREADS(X:%d/M:%d)", xml->nreads, db->nreads);
- if (db->platform == SRA_PLATFORM_ABSOLID
- && xml->nreads == 2 && db->nreads == 1
- && xml->read[0].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[0].readType == eForwardRT
- && xml->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[1].readType == eForwardRT
- && db ->read[0].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[0].readOrientation == 0
- &&(db ->read[0].label == eF3L || db ->read[0].label == eR3L)
- ) /* 1/2: BioFw,BioFw / BioF3|R3 */
- { ReportDiff(args, xml, "[FF-F]"); } /* ForwardForward-Fwd */
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 1 && db->nreads == 2
- && xml->read[0].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[0].readType == eForwardRT
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eFragmentL
- ) /* 1/2: BioFw / TecAda,BioFra */
- { ReportDiff(args, xml, "[F-AF]"); } /* Forward-AdapterFragment */
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 3 && db->nreads == 2
- && xml->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[0].readType == eAdapterRT
- && xml->read[1].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[1].readType == eBarCodeRT
- && xml->read[2].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[2].readType == eForwardRT
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eFragmentL
- ) /* 3/2: TecAd,TecBc,BioFw / TecAda,BioFra */
- { ReportDiff(args, xml, "[BF]"); } /* BarcodeForward */
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 3 && db->nreads == 2
- && xml->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[0].readType == eAdapterRT
- && xml->read[1].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[1].readType == ePrimerRT
- && xml->read[2].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[2].readType == eAdapterRT
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eFragmentL
- ) /* 3/2: TecAd,TecPr,BioAd / TecAda,BioFra */
- { ReportDiff(args, xml, "[PA]"); } /* PrimerAdapter */
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 4 && db->nreads == 2
- && xml->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[1].readType == eForwardRT
- && xml->read[2].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[2].readType == eLinkerRT
- && xml->read[3].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eFragmentL
- && (xml->read[0].readType == eAdapterRT ||
- xml->read[0].readType == eForwardRT)
- && (xml->read[3].readType == eForwardRT ||
- xml->read[3].readType == eReverseRT)
- ) /* 4/2: TecAd|Fw,BioFw,TecLn,BioFw|Rv, TecAda,BioFra */
- { ReportDiff(args, xml, "[FLF]"); } /* ForwardLinkerForward */
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 5 && db->nreads == 2
- && xml->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[0].readType == eAdapterRT
- && xml->read[1].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[1].readType == eBarCodeRT
- && xml->read[2].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[2].readType == eForwardRT
- && xml->read[3].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[3].readType == eLinkerRT
- && xml->read[4].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[4].readType == eReverseRT
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eFragmentL
- ) /* 5/2: TecAd,TecBc,BioFw,TecLn,BioRv / TecAda,BioFra */
- /* BarcodeForwardLinkerReverse-Fragment */
- { ReportDiff(args, xml, "[BFLR-F]"); }
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 2 && db->nreads == 4
- && xml->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[0].readType == eAdapterRT
- && xml->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[1].readType == eForwardRT
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eMate1L
- && db ->read[2].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[2].readOrientation == 0
- && db ->read[2].label == eLinkerL
- && db ->read[3].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[3].readOrientation == 0
- && db ->read[3].label == eMate2L
- ) /* 2/4: TecAd,BioFw / TecAda,BioMt1,TecLin,MioMt2 */
- { ReportDiff(args, xml, "[F-MLM]"); } /* Forward-MateLinkerMate */
- else if (db->platform == SRA_PLATFORM_454
- && xml->nreads == 5 && db->nreads == 4
- && xml->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[0].readType == eAdapterRT
- && xml->read[1].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[1].readType == eBarCodeRT
- && xml->read[2].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[2].readType == eForwardRT
- && xml->read[3].readClass == SRA_READ_TYPE_TECHNICAL
- && xml->read[3].readType == eLinkerRT
- && xml->read[4].readClass == SRA_READ_TYPE_BIOLOGICAL
- && xml->read[4].readType == eReverseRT
- && db ->read[0].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[0].readOrientation == 0
- && db ->read[0].label == eAdapterL
- && db ->read[1].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[1].readOrientation == 0
- && db ->read[1].label == eMate1L
- && db ->read[2].readClass == SRA_READ_TYPE_TECHNICAL
- && db ->read[2].readOrientation == 0
- && db ->read[2].label == eLinkerL
- && db ->read[3].readClass == SRA_READ_TYPE_BIOLOGICAL
- && db ->read[3].readOrientation == 0
- && db ->read[3].label == eMate2L
- ) /* 5/4: TecAd,TecBc,BioFw,TecLn,BioRv / TecAda,BioMt1,TecLin,MioMt2 */
- { ReportDiff(args, xml, "[BF-M]"); } /* BarcodeForward-Mate */
- else {
- int i = 0;
- ReportDiff(args, xml, "UNKNOWN X{");
- for (i = 0; i < xml->nreads; ++i) {
- if (i != 0)
- { ReportDiff(args, xml, ","); }
- ReportDiff(args, xml, "%d:", i);
- switch (xml->read[i].readClass) {
- case SRA_READ_TYPE_BIOLOGICAL:
- ReportDiff(args, xml, "BIO"); break;
- case SRA_READ_TYPE_TECHNICAL:
- ReportDiff(args, xml, "TEC"); break;
- default: assert(0);
- }
- switch (xml->read[i].readType) {
- case eAdapterRT:
- ReportDiff(args, xml, "AD"); break;
- case eBarCodeRT:
- ReportDiff(args, xml, "BC"); break;
- case eForwardRT:
- ReportDiff(args, xml, "FW"); break;
- case eLinkerRT:
- ReportDiff(args, xml, "LN"); break;
- case ePrimerRT:
- ReportDiff(args, xml, "PR"); break;
- case eReverseRT:
- ReportDiff(args, xml, "RV"); break;
- default: assert(0);
- }
- }
- ReportDiff(args, xml, "} D{");
- for (i = 0; i < db->nreads; ++i) {
- if (i != 0)
- { ReportDiff(args, xml, ","); }
- ReportDiff(args, xml, "%d:", i);
- switch (db->read[i].readClass) {
- case SRA_READ_TYPE_BIOLOGICAL:
- ReportDiff(args, xml, "BIO"); break;
- case SRA_READ_TYPE_TECHNICAL:
- ReportDiff(args, xml, "TEC"); break;
- default: assert(0);
- }
- {
- const char* l = EnumLabel2Str(db->read[i].label);
- assert(l);
- ReportDiff(args, xml, l);
- }
- /*switch (db->read[i].label) {
- case eAdapterL:
- ReportDiff(args, xml, "ADA"); break;
- case eF3L:
- ReportDiff(args, xml, "F3"); break;
- case eFragmentL:
- ReportDiff(args, xml, "FRA"); break;
- case eLinkerL:
- ReportDiff(args, xml, "LNK"); break;
- case eMate1L:
- ReportDiff(args, xml, "MT1"); break;
- case eMate2L:
- ReportDiff(args, xml, "MT2"); break;
- case eR3L:
- ReportDiff(args, xml, "R3"); break;
- default: assert(0);
- }*/
- }
- ReportDiff(args, xml, "}");
- return false;
- }
- return true;
-}
-static bool MDCompare(const MetaDataDb* db, const MetaDataXml* xml,
- const CmdLine* args)
-{
- bool res = true;
- if (db->platform != xml->platform) {
- ReportDiff(args, xml, "PLATFORM");
- res = false;
- }
- if (db->platform != SRA_PLATFORM_PACBIO_SMRT) {
- if (db->nreads != xml->nreads) {
- bool k = AnalyzeNReport(db, xml, args);
- if (!k)
- { OUTMSG(("\n")); }
- assert(k);
- res = false;
- }
- else {
- int i = 0;
- for (i = 0; i < db->nreads; ++i) {
- if (db->read[i].label == eInvalidF7L) {
- ReportDiff(args, xml, "ERROR label=F,length=7");
- res = false;
- }
- }
- }
- }
- if (res == true) {
- OUTMSG(("%s %s ", args->acc, EnumPlatform2Str(xml->platform)));
- if (db->platform != SRA_PLATFORM_PACBIO_SMRT)
- { OUTMSG(("%d ", db->nreads)); }
- OUTMSG(("EQUALS\n"));
- }
- else { OUTMSG(("\n")); }
- return res;
-}
-
-static void MetaDataXmlInit(MetaDataXml* md)
-{
- int i = 0;
- assert(md);
- memset(md, 0, sizeof *md);
- for (i = 0; i < sizeof md->read / sizeof md->read[0]; ++i) {
- SReadXml* r = &md->read[i];
- r->readClass = unsetReadClass;
- }
-}
-
-static void MetaDataDbInit(MetaDataDb* md)
-{
- int i = 0;
- assert(md);
- memset(md, 0, sizeof *md);
- for (i = 0; i < sizeof md->read / sizeof md->read[0]; ++i) {
- SReadDb* r = &md->read[i];
- r->readClass = unsetReadClass;
- }
-}
-
-static rc_t ReadColumns(const VTable* tbl,
- Col* columns, int n, MetaDataDb* md, bool variableReadLen)
-{
- rc_t rc = 0;
- const VCursor* curs = NULL;
- KNamelist* names = NULL;
- uint32_t nreads = 0;
- int i = 0;
- int colLabel = -1, colLabelLen = -1, colLabelStart = -1;
- assert(tbl && columns && md);
- rc = VTableListCol(tbl, &names);
- DISP_RC(rc, "while listing columns");
- if (rc == 0) {
- int i = 0;
- uint32_t count = 0;
- rc = KNamelistCount(names, &count);
- DISP_RC(rc, "while counting column list");
- for (i = 0; i < count && rc == 0; ++i) {
- int j = 0;
- const char* name = NULL;
- rc = KNamelistGet(names, i, &name);
- DISP_RC(rc, "while getting column name");
- for (j = 0; j < n && rc == 0; ++j) {
- if (strcmp(columns[j].name, name) == 0) {
- columns[j].exists = true;
- break;
- }
- }
- }
- DESTRUCT(KNamelist, names);
- }
- if (rc == 0) {
- rc = VTableCreateCursorRead(tbl, &curs);
- DISP_RC(rc, "while creating cursor");
- }
- for (i = 0; i < n && rc == 0; ++i) {
- Col* col = columns + i;
- assert(col);
- if (!col->exists)
- { continue; }
- rc = VCursorAddColumn(curs, &col->idx, "%s", col->name);
- DISP_RC2(rc, col->name, "while adding column to cursor");
- }
- if (rc == 0) {
- rc = VCursorOpen(curs);
- DISP_RC(rc, "while opening cursor");
- }
- for (i = 0; i < n && rc == 0; ++i) {
- Col* col = &columns[i];
- int64_t row_id = 1;
- uint32_t elem_bits = 8;
- col->row_len = 0;
- if (!col->exists)
- { continue; }
- if (rc == 0) {
- rc = VCursorReadDirect(curs, row_id, col->idx,
- elem_bits, col->buffer, sizeof col->buffer, &col->row_len);
- if (rc) {
- PLOGERR(klogInt,
- (klogInt, rc, "$(col): $(expected)", "col=%s,expected=%d",
- col->name, col->row_len));
- }
- }
- if (rc == 0 && col->type == eREAD_START) {
- nreads = col->row_len / 4;
- assert(col->row_len % 4 == 0);
- md->nreads = nreads;
- if (nreads > sizeof md->read / sizeof md->read[0]) {
- rc = RC
- (rcExe, rcColumn, rcReading, rcBuffer, rcInsufficient);
- PLOGERR(klogInt, (klogInt, rc,
- "while reading NREADS: received $(nreads)",
- "nreads=%d", nreads));
- }
- }
- if (rc == 0) {
- int64_t expected = -1;
- int64_t max = sizeof col->buffer;
- switch (col->datatype) {
- case eAscii:
- --max;
- expected = 0; break;
- case eReadType:
- case eUint8:
- expected = 1; break;
- case eUint32:
- expected = 4; break;
- default:
- assert(0) ; break;
- }
- if (expected) {
- assert((expected * (1 + col->isArray * (nreads - 1)))
- == col->row_len);
- }
- if (col->row_len > max) {
- rc = RC(rcExe, rcColumn, rcReading, rcBuffer, rcInsufficient);
- PLOGERR(klogInt, (klogInt, rc,
- "while reading $(column): received $(len)",
- "column=%s,len=%d", col->name, col->row_len));
- }
- else if (col->datatype == eAscii)
- { col->buffer[col->row_len] = '\0'; }
- switch (col->type) {
- case eLABEL:
- colLabel = i;
- break;
- case eLABEL_LEN:
- colLabelLen = i;
- break;
- case eLABEL_START:
- colLabelStart = i;
- break;
- default: /* does not matter */
- break;
- }
- }
- }
- for (i = 0; i < n && rc == 0; ++i) {
- Col* col = &columns[i];
- switch (col->datatype) {
- case eAscii: {
- const char* data = (char*)col->buffer;
- DBGMSG(DBG_APP, DBG_COND_1, ("%s: %s\n", col->name, data));
- break;
- }
- case eReadType: {
- uint8_t i = 0;
- char* data = (char*)col->buffer;
- for (i = 0; i < nreads; ++i) {
- char t[64] = "";
- if (data[i] & SRA_READ_TYPE_BIOLOGICAL) {
- strcpy(t, "SRA_READ_TYPE_BIOLOGICAL");
- data[i] &= ~SRA_READ_TYPE_BIOLOGICAL;
- md->read[i].readClass = SRA_READ_TYPE_BIOLOGICAL;
- } else {
- assert(SRA_READ_TYPE_TECHNICAL == 0);
- strcpy(t, "SRA_READ_TYPE_TECHNICAL");
- data[i] &= ~SRA_READ_TYPE_TECHNICAL;
- md->read[i].readClass = SRA_READ_TYPE_TECHNICAL;
- }
- if (data[i] & SRA_READ_TYPE_FORWARD) {
- strcat(t, "|SRA_READ_TYPE_FORWARD");
- data[i] &= ~SRA_READ_TYPE_FORWARD;
- md->read[i].readOrientation = SRA_READ_TYPE_FORWARD;
- }
- if (data[i] & SRA_READ_TYPE_REVERSE) {
- strcat(t, "|SRA_READ_TYPE_REVERSE");
- data[i] &= ~SRA_READ_TYPE_REVERSE;
- md->read[i].readOrientation = SRA_READ_TYPE_REVERSE;
- }
- if (data[i]) {
- rc = RC(rcExe,
- rcColumn, rcReading, rcData, rcUnexpected);
- LOGERR(klogInt, rc, "unexpected READ_TYPE value");
- }
- else {
- if (i == 0) {
- DBGMSG(DBG_APP, DBG_COND_1,
- ("%s: %s", col->name, t));
- } else { DBGMSG(DBG_APP, DBG_COND_1, (", %s", t)); }
- }
- }
- DBGMSG(DBG_APP, DBG_COND_1, ("\n"));
- break;
- }
- case eUint8:
- DBGMSG(DBG_APP, DBG_COND_1,
- ("%s: %d\n", col->name, *(uint8_t*)col->buffer));
- break;
- case eUint32: {
- uint8_t i = 0;
- uint32_t* data = (uint32_t*)col->buffer;
- DBGMSG(DBG_APP, DBG_COND_1, ("%s: %d", col->name, data[0]));
- for (i = 1; col->isArray && i < nreads; ++i)
- { DBGMSG(DBG_APP, DBG_COND_1, (", %d", data[i])); }
- DBGMSG(DBG_APP, DBG_COND_1, ("\n"));
- break;
- }
- default:
- assert(0);
- break;
- }
- switch (col->type) {
- case ePLATFORM:
- md->platform = *(uint8_t*)col->buffer;
- break;
- default: /* does not matter */
- break;
- }
- }
- if (rc == 0) {
- if (colLabel == -1 && colLabelLen == -1 && colLabelStart == -1)
- {}
- else if (colLabel == -1
- || colLabelLen == -1 || colLabelStart == -1)
- {
- rc = RC(rcExe, rcCursor, rcReading, rcColumn, rcNotFound);
- LOGERR(klogInt, rc, "(one of LABEL columns)");
- }
- else {
- int j = 0;
- int labell = 0;
- const char* label = (const char*)columns[colLabel].buffer;
- const uint32_t* labelLen
- = (const uint32_t*)columns[colLabelLen].buffer;
- const uint32_t* labelStart
- = (const uint32_t*)columns[colLabelStart].buffer;
- for (j = 0; j < nreads; ++j) {
- enum ELabel elabel = eUnsetL;
- if (labelLen[j]) {
- if (labelStart[j] == 0 && labelLen[j] == 7
- && label[0] == 'F' && label[1] == '\0')
- { elabel = eInvalidF7L; }
- else {
- elabel =
- StrnLabel2Enum(label + labelStart[j], labelLen[j]);
- }
- /* if (!Compare("Adapter", label + labelStart[j],
- labelLen[j]))
- { elabel = eAdapterL; }
- else if (!Compare("barcode", label + labelStart[j],
- labelLen[j]))
- { elabel = eBarcodeL; }
- else if (!Compare("F", label + labelStart[j],
- labelLen[j]))
- { elabel = eFL; }
- else if (!Compare("F3", label + labelStart[j],
- labelLen[j]))
- { elabel = eF3L; }
- else if (!Compare("Fragment", label + labelStart[j],
- labelLen[j]))
- { elabel = eFragmentL; }
- else if (!Compare("forward", label + labelStart[j],
- labelLen[j]))
- { elabel = eForwardL; }
- else if (!Compare("Linker", label + labelStart[j],
- labelLen[j]))
- { elabel = eLinkerL; }
- else if (!Compare("Mate1", label + labelStart[j],
- labelLen[j]))
- { elabel = eMate1L; }
- else if (!Compare("Mate2", label + labelStart[j],
- labelLen[j]))
- { elabel = eMate2L; }
- else if (!Compare("R3", label + labelStart[j],
- labelLen[j]))
- { elabel = eR3L; }
- else if (!Compare("reverse", label + labelStart[j],
- labelLen[j]))
- { elabel = eReverseL; }
- else if (!Compare("rRNA_primer", label + labelStart[j],
- labelLen[j]))
- { elabel = erRNA_primerL; }
- else*/
- if (elabel == eUnsetL) {
- rc = RC(rcExe, rcData, rcParsing, rcData, rcUnexpected);
- PLOGERR(klogInt,
- (klogInt, rc, "LABEL value: $(val), length: $(len)",
- "val=%.*s,len=%d",
- labelLen[j], label + labelStart[j], labelLen[j]));
- }
- }
- md->read[j].label = elabel;
- labell += labelLen[j];
- }
- }
- }
- DESTRUCT(VCursor, curs);
- return rc;
-}
-
-static rc_t MakeXmlDocs(const CmdLine* args,
- const KXMLDoc** exp, const KXMLDoc** run)
-{
- rc_t rc = 0;
- const KXMLMgr* mgr = NULL;
- assert(args && exp && run);
- rc = KXMLMgrMakeRead(&mgr);
- DISP_RC(rc, "while calling KXMLMgrMakeRead");
- if (rc)
- { return rc; }
- if (args->exp && args->run) {
- KDirectory* dir = NULL;
- const KFile* expF = NULL;
- const KFile* runF = NULL;
- if (rc == 0) {
- rc = KDirectoryNativeDir(&dir);
- DISP_RC(rc, "while calling KDirectoryNativeDir");
- }
- if (rc == 0) {
- rc = KDirectoryOpenFileRead(dir, &expF, args->exp);
- DISP_RC2(rc, args->exp, "while opening file");
- }
- if (rc == 0) {
- rc = KDirectoryOpenFileRead(dir, &runF, args->run);
- DISP_RC2(rc, args->run, "while opening file");
- }
- if (rc == 0) {
- rc = KXMLMgrMakeDocRead(mgr, exp, expF);
- DISP_RC2(rc, args->exp, "while making experiment XML doc");
- }
- if (rc == 0) {
- rc = KXMLMgrMakeDocRead(mgr, run, runF);
- DISP_RC2(rc, args->run, "while making run XML doc");
- }
- {
- rc_t rc2 = KFileRelease(runF);
- if (rc2 != 0 && rc == 0)
- { rc = rc2; }
- runF = NULL;
- }
- {
- rc_t rc2 = KFileRelease(expF);
- if (rc2 != 0 && rc == 0)
- { rc = rc2; }
- expF = NULL;
- }
- {
- rc_t rc2 = KDirectoryRelease(dir);
- if (rc2 != 0 && rc == 0)
- { rc = rc2; }
- dir = NULL;
- }
- }
- else if (args->acc) {
- const DBManager* dbMgr = NULL;
- if (rc == 0) {
- rc = SybaseInit(OS_CS_VERSION);
- if (rc != 0)
- { LOGERR(klogInt, rc, "failed to init Sybase"); }
- }
- if (rc == 0) {
- rc = DBManagerInit(&dbMgr, "sybase");
- if (rc != 0) {
- LOGERR(klogInt, rc, "failed to init Sybase");
- }
- }
- if (rc == 0) {
- Database* info = NULL;
- const char server[] = "NIHMS2";
- const char dbname[] = "SRA_Main";
- const char user[] = "anyone";
- rc = DBManagerConnect
- (dbMgr, server, dbname, user, "allowed", &info);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "failed to connect to $(srv).$(db) as user $(usr)",
- "srv=%s,db=%s,usr=%s", server, dbname, user));
- }
- if (rc == 0) {
- DBResultSet* rs = NULL;
- rc = DatabaseExecute(info, &rs, "set textsize 1310720");
- DISP_RC(rc, "while setting DB textsize");
- }
- if (rc == 0) {
- DBResultSet* rs = NULL;
- rc = DatabaseExecute(info, &rs,
- "SELECT r.meta, e.meta FROM Run r, Experiment e "
- "WHERE r.acc = '%s' AND e.acc = r.experiment_ref",
- args->acc);
- if (rc != 0) {
- LOGERR(klogInt, rc, "metadata selecting");
- }
- else {
- while (rc == 0) {
- const String* runS = NULL;
- const String* expS = NULL;
- DBRow* row = NULL;
- rc = DBResultSetNextRow(rs, &row);
- if (rc != 0) {
- if (rc == RC(rcRDBMS,
- rcData, rcRetrieving, rcData, rcNotAvailable))
- rc = 0;
- break;
- }
- rc = DBRowGetAsString(row, 0, &runS);
- if (rc != 0) {
- LOGERR(klogInt, rc, "accessing run meta");
- }
- else {
- rc = KXMLMgrMakeDocReadFromMemory
- (mgr, run, runS->addr, runS->size);
- if (rc != 0)
- { LOGERR(klogInt, rc, "run.xml"); }
- }
- if (rc == 0) {
- rc = DBRowGetAsString(row, 1, &expS);
- if (rc != 0) {
- LOGERR(klogInt, rc, "accessing exp meta");
- }
- else {
- rc = KXMLMgrMakeDocReadFromMemory
- (mgr, exp, expS->addr, expS->size);
- }
- }
- }
- if (rc != 0)
- { LOGERR(klogInt, rc, "metadata retieving"); }
- }
- }
- }
- {
- rc_t rc2 = DBManagerRelease(dbMgr);
- if (rc2 != 0 && rc == 0)
- { rc = rc2; }
- dbMgr = NULL;
- }
- }
- else { rc = RC(rcExe, rcArgv, rcParsing, rcArgv, rcInconsistent); }
- {
- rc_t rc2 = KXMLMgrRelease(mgr);
- if (rc2 != 0 && rc == 0)
- { rc = rc2; }
- mgr = NULL;
- }
- return rc;
-}
-
-rc_t CC UsageSummary (const char * progname)
-{
- rc_t rc = 0;
- return rc;
-}
-const char UsageDefaultName[] = "meta-sync";
-rc_t CC Usage(const Args* args)
-{
- rc_t rc = 0;
- return rc;
-}
-
-ver_t CC KAppVersion(void) { return META_SYNC_VERS; }
-
-static char *vdm_translate_accession( SRAPath* my_sra_path,
- const char* accession, const size_t bufsize, rc_t* rc)
-{
- char* res = malloc(bufsize);
- if (res != NULL) {
- *rc = SRAPathFind(my_sra_path, accession, res, bufsize);
- if (GetRCState(*rc) == rcNotFound) {
- free(res);
- res = NULL;
- }
- else if (GetRCState(*rc) == rcInsufficient) {
- DBGMSG(DBG_APP, 0, ("bufsize %lu was insufficient\n", bufsize));
- free(res);
- res = vdm_translate_accession
- (my_sra_path, accession, bufsize * 2, rc);
- }
- else if (*rc != 0) {
- free(res);
- res = NULL;
- }
- }
- return res;
-}
-
-#define OPTION_EXPERIMENT "experiment"
-#define ALIAS_EXPERIMENT "e"
-static const char* experiment_usage [] = { "experiment.xml file" , NULL };
-#define OPTION_RUN "run"
-#define ALIAS_RUN "r"
-static const char* run_usage [] = { "run.xml file" , NULL };
-#define OPTION_ACC "accession"
-#define ALIAS_ACC "a"
-static const char* acc_usage [] = { "run accession" , NULL };
-OptDef Options[] = {
- { OPTION_ACC, ALIAS_ACC, NULL, acc_usage , 1, true , true },
- { OPTION_EXPERIMENT,
- ALIAS_EXPERIMENT, NULL, experiment_usage, 1, true , true },
- { OPTION_RUN, ALIAS_RUN, NULL, run_usage , 1, true , true }
-};
-static rc_t CmdLineInit(const Args* args, CmdLine* cmdArgs)
-{
- rc_t rc = 0;
- bool acc_exp_runError = false;
- assert(args && cmdArgs);
-
- memset(cmdArgs, 0, sizeof *cmdArgs);
-
- while (rc == 0) {
- uint32_t pcount = 0;
- uint32_t acccount = 0;
- uint32_t expcount = 0;
- uint32_t runcount = 0;
-
- /* table path parameter */
- rc = ArgsParamCount(args, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure parsing table name");
- break;
- }
- if (pcount < 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
- LOGERR(klogErr, rc, "Missing table parameter");
- break;
- }
- if (pcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- LOGERR(klogErr, rc, "Too many table parameters");
- break;
- }
- rc = ArgsParamValue(args, 0, &cmdArgs->tbl);
- if (rc) {
- LOGERR(klogErr, rc, "Failure retrieving table name");
- break;
- }
-
- /* run accession parameter */
- rc = ArgsOptionCount(args, OPTION_ACC, &acccount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure parsing run accession");
- break;
- }
- if (acccount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- LOGERR(klogErr, rc, "Too many run accession parameters");
- break;
- }
-
- /* experiment file parameter */
- rc = ArgsOptionCount(args, OPTION_EXPERIMENT, &expcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure parsing experiment file name");
- break;
- }
- if (expcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- LOGERR(klogErr, rc, "Too many experiment parameters");
- break;
- }
-
- /* run file parameter */
- rc = ArgsOptionCount(args, OPTION_RUN, &runcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure parsing run file name");
- break;
- }
- if (runcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- LOGERR(klogErr, rc, "Too many run parameters");
- break;
- }
-
- if (acccount < 1) {
- if (expcount < 1) {
- acc_exp_runError = true;
- break;
- }
- else if (runcount < 1) {
- acc_exp_runError = true;
- break;
- }
- else {
- rc = ArgsOptionValue(args, OPTION_EXPERIMENT, 0, &cmdArgs->exp);
- if (rc) {
- LOGERR(klogErr, rc,
- "Failure retrieving experiment file name");
- break;
- }
-
- rc = ArgsOptionValue (args, OPTION_RUN, 0, &cmdArgs->run);
- if (rc) {
- LOGERR(klogErr, rc, "Failure retrieving run file name");
- break;
- }
- }
- }
- else {
- rc = ArgsOptionValue (args, OPTION_ACC, 0, &cmdArgs->acc);
- if (rc) {
- LOGERR(klogErr, rc, "Failure retrieving run accession");
- break;
- }
- }
-
- break;
- }
-
- if (rc == 0) {
- if (strchr(cmdArgs->tbl, '/') == NULL) {
- SRAPath* sraPath = NULL;
- rc = SRAPathMake(&sraPath, NULL);
- if (rc == 0) {
- if (!SRAPathTest(sraPath, cmdArgs->tbl)) {
- char* buf =
- vdm_translate_accession(sraPath, cmdArgs->tbl, 64, &rc);
- if (buf != NULL) {
- DBGMSG
- (DBG_APP, 0, ("sra-accession found! >%s<\n", buf));
- if (acc_exp_runError) {
- cmdArgs->acc = cmdArgs->tbl;
- acc_exp_runError = false;
- }
- cmdArgs->tbl = buf;
- }
- else if (GetRCState(rc) == rcNotFound) {
- OUTMSG(("%s NOT FOUND\n", cmdArgs->tbl));
- cmdArgs->notFound = true;
- rc = 0;
- }
- DISP_RC2(rc, cmdArgs->tbl, "while looking for table");
- }
- }
- else {
- if (GetRCState(rc) != rcNotFound ||
- GetRCTarget(rc) != rcDylib)
- { LOGERR( klogInt, rc, "SRAPathMake failed" ); }
- else { rc = 0; }
- }
- }
- }
- if (cmdArgs->notFound == false && rc == 0 && acc_exp_runError) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
- LOGERR(klogErr, rc,
- "Either accession or run/experiment XML should be specified");
- }
-/*
- else
- { MiniUsage (args); }
-*/
- return rc;
-}
-
-static void ShutUp(void) {
- StrLabel2Enum (0); StrnPlatform2Enum(0, 0);
- EnumReadType2Str(0); StrReadType2Enum(0); StrnReadType2Enum(0, 0);
-}
-
-rc_t CC KMain(int argc, char* argv[])
-{
- rc_t rc = 0;
- Args* args = NULL;
- CmdLine cmdArgs;
- Col columns[] = {
-/* { eNREADS ,"NREADS" , eUint8 , false } , */
- /* READ_START should be the first column in the array */
- { eREAD_START ,"READ_START" , eUint32 , true }
- , { eFIXED_SPOT_LEN ,"FIXED_SPOT_LEN" , eUint32 , false }
- , { eLABEL ,"LABEL" , eAscii , false }
- , { eLABEL_LEN ,"LABEL_LEN" , eUint32 , true }
- , { eLABEL_START ,"LABEL_START" , eUint32 , true }
- , { eLINKER_SEQUENCE,"LINKER_SEQUENCE", eAscii , false }
- , { ePLATFORM ,"PLATFORM" , eUint8 , false }
- , { eREAD_LEN ,"READ_LEN" , eUint32 , true }
- , { eREAD_TYPE ,"READ_TYPE" , eReadType, true }
- , { eSPOT_GROUP ,"SPOT_GROUP" , eAscii , false }
- , { eSPOT_LEN ,"SPOT_LEN" , eUint32 , false }
- };
- rc = ArgsMakeAndHandle
- (&args, argc, argv, 1, Options, sizeof Options / sizeof(Options[0]));
- if (rc == 0)
- { rc = CmdLineInit(args, &cmdArgs); }
- if (cmdArgs.notFound == false) {
- bool equal = true;
- const KXMLDoc* exp = NULL;
- const KXMLDoc* run = NULL;
- const VDBManager* mgr = NULL;
- const VDatabase *db = NULL;
- const VTable* tbl = NULL;
- VSchema* schema = NULL;
- MetaDataXml xmlMd;
- MetaDataDb runMd;
- MetaDataXmlInit(&xmlMd);
- MetaDataDbInit (&runMd);
- if (rc == 0)
- { rc = MakeXmlDocs(&cmdArgs, &exp, &run); }
- if (rc == 0)
- { rc = ParseXml(exp, run, &xmlMd); }
- if (rc == 0) {
- rc = VDBManagerMakeRead(&mgr, NULL);
- }
- if (rc == 0) {
- rc = VDBManagerMakeSRASchema(mgr, &schema);
- }
- if (rc == 0) {
- rc = VDBManagerOpenDBRead(mgr, &db, schema, "%s", cmdArgs.tbl);
- if (rc == 0) {
- const char path[] = "SEQUENCE";
- rc = VDatabaseOpenTableRead(db, &tbl, path);
- DISP_RC2(rc, cmdArgs.tbl, "while opening DB table SEQUENCE");
- }
- else {
- rc = VDBManagerOpenTableRead(mgr, &tbl, schema, "%s", cmdArgs.tbl);
- DISP_RC2(rc, cmdArgs.tbl, "while opening table");
- }
- }
- if (rc == 0) {
- rc = ReadColumns(tbl, columns, sizeof columns / sizeof columns[0],
- &runMd, xmlMd.variableReadLen);
- }
- if (rc == 0)
- { equal = MDCompare(&runMd, &xmlMd, &cmdArgs); }
- DESTRUCT(VTable, tbl);
- DESTRUCT(VDatabase, db);
- DESTRUCT(VSchema, schema);
- DESTRUCT(VDBManager, mgr);
- DESTRUCT(KXMLDoc, run);
- DESTRUCT(KXMLDoc, exp);
- }
- {
- rc_t rc2 = ArgsWhack(args);
- if (rc2 != 0 && rc == 0) {
- rc = rc2;
- ShutUp();
- }
- }
- return rc;
-}
-
-/************************************ EOF ****************** ******************/
diff --git a/tools/meta-sync/meta-sync.pl b/tools/meta-sync/meta-sync.pl
deleted file mode 100755
index 508ac7e..0000000
--- a/tools/meta-sync/meta-sync.pl
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/local/bin/perl -w
-use strict;
-use lib '/export/home/sybase/clients-mssql/current64/perl';
-use DBI;
-my $db;
-my $start = `date`;
-my $S = 1;
-my $L = 315142; # $S <= R <= $L
-my $INI = 'meta-sync.ini';
-if (-e $INI) {
- $S = `cat $INI`;
- chomp $S;
-}
-for (my $i = $S; $i <= $L; ++$i) {
- my $A = sprintf "SRR%06d", ($i);
- if ($i == 305952) {
-# print "$A SKIP\n"; next;
- }
- my $res = system "meta-sync $A"; # -+ APP
- if ($res) {
- `echo $i > $INI`;
- die "$i $A";
- }
- if ($i == $L) {
- $db = DBI->connect("dbi:Sybase:server=NIHMS2", "anyone", "allowed",
- { syb_err_handler => \&err_handler }) unless ($db);
- die unless ($db);
- my $sth = $db->prepare("select max(acc) from SRA_Main..Run") || die;
- my $rv = $sth->execute || die;
- my @res = $sth->fetchrow_array || die;
- die if ($#res != 0);
- $res[0] =~ /^(...)(.{7})$/;
- die unless ($1 && $2 && $1 eq 'SRR');
- $L = $2;
- }
-}
-sub err_handler {
- my($err, $sev, $state, $line, $server, $proc, $msg, $sql, $err_type) = @_;
- print "$msg $proc\n";
- 1
-}
diff --git a/tools/meta-sync/meta-sync.vers b/tools/meta-sync/meta-sync.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/meta-sync/meta-sync.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/nenctool/nenctest.c b/tools/nenctool/nenctest.c
index 081040a..677569c 100644
--- a/tools/nenctool/nenctest.c
+++ b/tools/nenctool/nenctest.c
@@ -107,6 +107,7 @@ static
const char * first_usage[] =
{
"The path to a file either in native format",
+ NULL
};
static
@@ -118,7 +119,8 @@ const char * second_usage[] =
"where 'enc' or 'encrypt' means the file is encrypted",
"and 'pwfile=<path>' points to a file to get the password",
"or 'pwfd=<fd>' refers to a file descriptor from which to",
- "read a password."
+ "read a password.",
+ NULL
};
rc_t CC Usage (const Args * args)
@@ -437,7 +439,7 @@ rc_t CC KMain ( int argc, char *argv [] )
{
const char * value;
- rc = ArgsOptionValue (args, OPTION_CIPHER, 0, &value);
+ rc = ArgsOptionValue (args, OPTION_CIPHER, 0, (const void **)&value);
if (rc)
LOGERR (klogInt, rc, "failed to examine cipher value");
else
@@ -474,13 +476,13 @@ rc_t CC KMain ( int argc, char *argv [] )
{
const char * src;
- rc = ArgsParamValue (args, 0, &src);
+ rc = ArgsParamValue (args, 0, (const void **)&src);
if (rc)
LOGERR (klogInt, rc, "failed to get source parameter");
else
{
const char * dst;
- rc = ArgsParamValue (args, 1, &dst);
+ rc = ArgsParamValue (args, 1, (const void **)&dst);
if (rc)
LOGERR (klogInt, rc, "failed to get destination parameter");
else
diff --git a/tools/nenctool/nenctest.vers b/tools/nenctool/nenctest.vers
index 8e8299d..35d16fb 100644
--- a/tools/nenctool/nenctest.vers
+++ b/tools/nenctool/nenctest.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/nenctool/nenctool.c b/tools/nenctool/nenctool.c
index 4ebab3c..7ce03f9 100644
--- a/tools/nenctool/nenctool.c
+++ b/tools/nenctool/nenctool.c
@@ -89,6 +89,7 @@ static
const char * first_usage[] =
{
"The path to a file either in native format",
+ NULL
};
static
@@ -100,7 +101,8 @@ const char * second_usage[] =
"where 'enc' or 'encrypt' means the file is encrypted",
"and 'pwfile=<path>' points to a file to get the password",
"or 'pwfd=<fd>' refers to a file descriptor from which to",
- "read a password."
+ "read a password.",
+ NULL
};
rc_t CC Usage (const Args * args)
@@ -414,19 +416,21 @@ rc_t CC KMain ( int argc, char *argv [] )
LOGERR (klogInt, rc, "failed to count parameters");
else
{
- if (pcount != 2)
+ if (pcount != 2) {
MiniUsage (args);
+ rc = RC(rcApp, rcArgv, rcParsing, rcParam, rcInsufficient);
+ }
else
{
const char * src;
- rc = ArgsParamValue (args, 0, &src);
+ rc = ArgsParamValue (args, 0, (const void **)&src);
if (rc)
LOGERR (klogInt, rc, "failed to get source parameter");
else
{
const char * dst;
- rc = ArgsParamValue (args, 1, &dst);
+ rc = ArgsParamValue (args, 1, (const void **)&dst);
if (rc)
LOGERR (klogInt, rc, "failed to get destination parameter");
else
diff --git a/tools/nenctool/nenctool.vers b/tools/nenctool/nenctool.vers
index 8e8299d..35d16fb 100644
--- a/tools/nenctool/nenctool.vers
+++ b/tools/nenctool/nenctool.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/nencvalid/nencvalid.c b/tools/nencvalid/nencvalid.c
index cc1fcf1..f18e341 100644
--- a/tools/nencvalid/nencvalid.c
+++ b/tools/nencvalid/nencvalid.c
@@ -448,7 +448,7 @@ rc_t HandleParams (VFSManager * mgr, Args * args, uint32_t pcount)
{
const char * pc;
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
PLOGERR (klogErr,
(klogErr, rc, "failed to retrieve "
@@ -484,9 +484,10 @@ rc_t HandleOptions (Args * args)
else
{
- if (pcount == 0)
+ if (pcount == 0) {
MiniUsage(args);
-
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
+ }
else
{
uint32_t ocount;
@@ -547,7 +548,7 @@ rc_t HandleOptions (Args * args)
{
const char * pc;
- rc = ArgsParamValue (args, 0, &pc);
+ rc = ArgsParamValue (args, 0, (const void **)&pc);
if (rc)
LOGERR (0, rc, "unable to retrieve file parameter");
diff --git a/tools/nencvalid/nencvalid.vers b/tools/nencvalid/nencvalid.vers
index 8e8299d..35d16fb 100644
--- a/tools/nencvalid/nencvalid.vers
+++ b/tools/nencvalid/nencvalid.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/fix-run-stat/Makefile b/tools/ngs-pileup/Makefile
similarity index 83%
rename from tools/fix-run-stat/Makefile
rename to tools/ngs-pileup/Makefile
index 3b39ff8..6d9672b 100644
--- a/tools/fix-run-stat/Makefile
+++ b/tools/ngs-pileup/Makefile
@@ -1,4 +1,4 @@
-# ==============================================================================
+# ===========================================================================
#
# PUBLIC DOMAIN NOTICE
# National Center for Biotechnology Information
@@ -26,18 +26,14 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/fix-run-stat
-include $(TOP)/build/Makefile.env
-INT_TOOLS = \
+MODULE = tools/ngs-pileup
-EXT_TOOLS = \
- fix-run-stat
+include $(TOP)/build/Makefile.env
ALL_TOOLS = \
- $(INT_TOOLS) \
- $(EXT_TOOLS)
+ ngs-pileup \
#-------------------------------------------------------------------------------
# outer targets
@@ -73,21 +69,23 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# fix-run-stat
-# update run STATS metadata
+# ngs-pileup
#
-FIX_RUN_STAT_SRC = \
- fix-run-stat
-
-FIX_RUN_STAT_OBJ = \
- $(addsuffix .$(OBJX),$(FIX_RUN_STAT_SRC))
-
-FIX_RUN_STAT_LIB = \
- -lkapp \
- -lncbi-wvdb \
-
-
-
-$(BINDIR)/fix-run-stat: $(FIX_RUN_STAT_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(FIX_RUN_STAT_LIB)
+NGS_PILEUP_SRC = \
+ ngs-pileup \
+ main \
+
+INCDIRS += -I $(TOP)/ngs/ngs-c++
+
+NGS_PILEUP_OBJ = \
+ $(addsuffix .$(OBJX),$(NGS_PILEUP_SRC))
+
+NGS_PILEUP_LIBS = \
+ -sncbi-ngs-c++ \
+ -sncbi-vdb-static \
+ -sngs-c++ \
+ -skapp \
+
+$(BINDIR)/ngs-pileup: $(NGS_PILEUP_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(NGS_PILEUP_LIBS)
diff --git a/tools/ngs-pileup/main.cpp b/tools/ngs-pileup/main.cpp
new file mode 100644
index 0000000..8a63c52
--- /dev/null
+++ b/tools/ngs-pileup/main.cpp
@@ -0,0 +1,164 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "ngs-pileup.vers.h"
+
+#include "ngs-pileup.hpp"
+
+#include <ngs/ErrorMsg.hpp>
+
+#include <kapp/main.h>
+#include <klib/out.h>
+#include <klib/rc.h>
+
+#include <sysalloc.h>
+#include <string.h>
+
+#include <iostream>
+
+#define OPTION_REF "aligned-region"
+#define ALIAS_REF "r"
+const char * ref_usage[] = { "Filter by position on genome.",
+ "Name can either be file specific or canonical",
+ "(ex: \"chr1\" or \"1\").",
+ "\"from\" and \"to\" are 1-based coordinates",
+ NULL };
+
+OptDef options[] =
+{ /*name, alias, hfkt, usage-help, maxcount, needs value, required */
+ { OPTION_REF, ALIAS_REF, NULL, ref_usage, 0, true, false },
+};
+
+
+const char UsageDefaultName[] = "ngs-pileup";
+
+rc_t CC UsageSummary ( const char * progname )
+{
+ return KOutMsg ("\n"
+ "Usage:\n"
+ " %s <path> [options]\n"
+ "\n", progname);
+}
+
+
+rc_t CC Usage ( const Args * args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ rc_t rc;
+
+ if ( args == NULL )
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
+ else
+ rc = ArgsProgram ( args, &fullpath, &progname );
+
+ if ( rc )
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary ( progname );
+ KOutMsg ( "Options:\n" );
+
+ HelpOptionsStandard ();
+ HelpVersion ( fullpath, KAppVersion() );
+
+ return rc;
+}
+
+/* Version EXTERN
+ * return 4-part version code: 0xMMmmrrrr, where
+ * MM = major release
+ * mm = minor release
+ * rrrr = bug-fix release
+ */
+ver_t CC KAppVersion ( void )
+{
+ return NGS_PILEUP_VERS;
+}
+rc_t CC KMain( int argc, char *argv [] )
+{
+ Args * args;
+
+ rc_t rc = ArgsMakeAndHandle( &args, argc, argv, 1, options, sizeof options / sizeof options [ 0 ] );
+ if ( rc == 0 )
+ {
+ try
+ {
+ NGS_Pileup::Settings settings;
+
+ uint32_t pcount;
+
+ rc = ArgsOptionCount ( args, OPTION_REF, &pcount );
+ if ( pcount > 1 )
+ {
+ throw ngs :: ErrorMsg ( "multiple positions are not supported at this point" );
+ }
+ if ( pcount == 1 )
+ {
+ const void * value;
+ rc = ArgsOptionValue ( args, OPTION_REF, 0, & value );
+ if ( rc != 0 )
+ {
+ throw ngs :: ErrorMsg ( "ArgsOptionValue (" OPTION_REF ") failed" );
+ }
+ settings . AddReference ( static_cast <char const*> (value) );
+ }
+
+ rc = ArgsParamCount ( args, &pcount );
+ if ( rc == 0 )
+ {
+ if ( pcount > 1 )
+ {
+ throw ngs :: ErrorMsg ( "multiple accessions are not supported at this point" );
+ }
+
+ settings . output = & std::cout;
+
+ void const *value;
+ rc = ArgsParamValue ( args, 0, &value );
+ if ( rc == 0 )
+ {
+ settings . AddInput ( static_cast <char const*> (value) );
+
+ NGS_Pileup ( settings ) . Run ();
+ }
+ else
+ {
+ throw ngs :: ErrorMsg ( "ArgsParamValue failed" );
+ }
+ }
+ }
+ catch (ngs :: ErrorMsg& ex)
+ {
+ std :: cerr << "Program aborted: " << ex.what() << std::endl;
+ exit ( -1 );
+ }
+
+ ArgsWhack( args );
+ }
+
+ return 0;
+}
+
diff --git a/tools/ngs-pileup/ngs-pileup.cpp b/tools/ngs-pileup/ngs-pileup.cpp
new file mode 100644
index 0000000..fdee801
--- /dev/null
+++ b/tools/ngs-pileup/ngs-pileup.cpp
@@ -0,0 +1,206 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "ngs-pileup.hpp"
+
+#include <iostream>
+
+#include <ngs/ncbi/NGS.hpp>
+#include <ngs/ReadCollection.hpp>
+#include <ngs/PileupIterator.hpp>
+
+using namespace std;
+
+struct NGS_Pileup::TargetReference
+{
+ typedef pair < int64_t, int64_t > Slice;
+ typedef vector < Slice > Slices;
+ typedef vector < ngs :: Reference > Targets;
+ typedef vector < ngs :: PileupIterator> Pileups;
+
+ string m_canonicalName;
+ Slices m_slices;
+ Targets m_targets;
+ Pileups m_pileups;
+ bool m_complete;
+
+ TargetReference ( ngs :: Reference p_ref )
+ : m_canonicalName ( p_ref . getCanonicalName() ), m_complete ( true )
+ {
+ AddReference ( p_ref );
+ }
+ TargetReference ( ngs :: Reference p_ref,
+ int64_t p_first,
+ int64_t p_last )
+ : m_canonicalName ( p_ref . getCanonicalName() ), m_complete ( false )
+ {
+ AddReference ( p_ref );
+ }
+ ~TargetReference ()
+ {
+ }
+
+ void AddSlice ( int64_t p_first, int64_t p_last )
+ {
+ }
+ void MakeComplete ()
+ {
+ m_complete = true;
+ m_slices . clear();
+ }
+
+ void AddReference ( ngs :: Reference p_ref )
+ {
+ m_targets. push_back ( p_ref );
+ }
+
+ void Process ( ostream& out )
+ {
+ int64_t firstPos = 0;
+ int64_t lastPos = m_targets . front () . getLength () - 1;
+
+ // create pileup iterators
+ for ( Targets::iterator i = m_targets.begin(); i != m_targets.end(); ++i )
+ {
+ m_pileups . push_back ( i -> getPileups ( ngs::Alignment::all ) );
+ }
+
+ int64_t curPos = firstPos;
+ while ( curPos <= lastPos )
+ {
+ uint32_t total_depth = 0;
+ for ( Pileups :: iterator i = m_pileups . begin (); i != m_pileups. end (); ++i )
+ {
+ bool next = i -> nextPileup ();
+ assert ( next );
+ total_depth += i -> getPileupDepth ();
+ }
+
+ if ( total_depth > 0 )
+ {
+ out << m_canonicalName
+ << '\t' << ( curPos + 1 ) // convert to 1-based position to emulate samtools
+ << '\t' << total_depth
+ << endl;
+ }
+
+ ++ curPos;
+ }
+ }
+};
+
+class NGS_Pileup::TargetReferences : public vector < TargetReference >
+{
+public :
+ void AddComplete ( ngs :: Reference ref )
+ {
+ string name = ref . getCanonicalName ();
+ for ( iterator i = begin(); i != end (); ++ i )
+ {
+ if ( i -> m_canonicalName == name )
+ {
+ i -> AddReference ( ref );
+ return;
+ }
+ }
+ // not found - add new reference
+ push_back ( TargetReference ( ref ) );
+ }
+};
+
+NGS_Pileup::NGS_Pileup ( const Settings& p_settings )
+: m_settings( p_settings )
+{
+}
+
+static
+bool FindReference ( const NGS_Pileup :: Settings :: References & requested, const ngs :: Reference & ref )
+{
+ for ( NGS_Pileup :: Settings :: References :: const_iterator i = requested . begin();
+ i != requested . end ();
+ ++i )
+ {
+ if ( i->m_name == ref . getCanonicalName () || i->m_name == ref . getCommonName () )
+ {
+ return true;
+ }
+ }
+ return false;
+}
+
+void
+NGS_Pileup::Run () const
+{
+ TargetReferences references;
+
+ // build the set of target references
+ for ( Settings :: Inputs :: const_iterator i = m_settings . inputs . begin();
+ i != m_settings . inputs . end ();
+ ++i )
+ {
+ ngs :: ReadCollection col = ncbi :: NGS :: openReadCollection ( *i );
+ ngs :: ReferenceIterator refIt = col . getReferences ();
+ while ( refIt . nextReference () )
+ {
+ if ( m_settings . references . empty () ) // all references requested
+ {
+ /* need to create a Reference object that is not attached to the iterator, so as
+ it is not invalidated on the next call to refIt.NextReference() */
+ references . AddComplete ( col . getReference ( refIt. getCommonName () ) );
+ }
+ else if ( FindReference ( m_settings . references, refIt ) )
+ {
+ //TODO: handle slices
+ references . AddComplete ( col . getReference ( refIt. getCommonName () ) );
+ }
+ }
+ }
+
+ ostream & out ( m_settings . output != (ostream*)0 ? * m_settings . output : cout );
+
+ // walk the references and output pileups
+ for ( TargetReferences :: iterator i = references . begin(); i != references . end (); ++i )
+ {
+ i -> Process ( out );
+ }
+}
+
+//// NGS_Pileup::Settings
+
+void
+NGS_Pileup::Settings::AddReference ( const string& commonOrCanonicalName )
+{
+ references . push_back ( ReferenceSlice ( commonOrCanonicalName ) );
+}
+
+void
+NGS_Pileup::Settings::AddReferenceSlice ( const string& commonOrCanonicalName,
+ int64_t firstPos,
+ int64_t lastPos )
+{
+ references . push_back ( ReferenceSlice ( commonOrCanonicalName, firstPos, lastPos ) );
+}
+
diff --git a/tools/ngs-pileup/ngs-pileup.hpp b/tools/ngs-pileup/ngs-pileup.hpp
new file mode 100644
index 0000000..0fb7a37
--- /dev/null
+++ b/tools/ngs-pileup/ngs-pileup.hpp
@@ -0,0 +1,97 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _sra_tools_hpp_ngs_pileup_
+#define _sra_tools_hpp_ngs_pileup_
+
+#include <klib/defs.h>
+
+#include <string>
+#include <vector>
+
+namespace ngs
+{
+ class Reference;
+}
+
+class NGS_Pileup
+{
+public:
+ struct Settings
+ {
+ struct ReferenceSlice
+ {
+ ReferenceSlice( const std::string& p_name ) /* entire reference */
+ : m_name ( p_name ),
+ m_firstPos ( 0 ),
+ m_lastPos ( 0 ),
+ m_full ( true )
+ {
+ }
+ ReferenceSlice( const std::string& p_name,
+ int64_t p_firstPos,
+ int64_t p_lastPos )
+ : m_name ( p_name ),
+ m_firstPos ( p_firstPos ),
+ m_lastPos ( p_lastPos ),
+ m_full ( false )
+ {
+ }
+
+ std::string m_name;
+ int64_t m_firstPos;
+ int64_t m_lastPos;
+ bool m_full;
+ };
+
+ void AddInput ( const std::string& accession ) { inputs . push_back ( accession ); }
+ void AddReference ( const std::string& commonOrCanonicalName );
+ void AddReferenceSlice ( const std::string& commonOrCanonicalName,
+ int64_t firstPos,
+ int64_t lastPos );
+
+
+ typedef std::vector < std::string > Inputs;
+ typedef std::vector < ReferenceSlice > References;
+
+ Inputs inputs;
+ std::ostream* output;
+ References references;
+ };
+
+public:
+ NGS_Pileup ( const Settings& p_settings );
+
+ void Run () const;
+
+private:
+ struct TargetReference;
+ class TargetReferences;
+
+ Settings m_settings;
+};
+
+#endif
diff --git a/tools/ngs-pileup/ngs-pileup.vers b/tools/ngs-pileup/ngs-pileup.vers
new file mode 100644
index 0000000..8acdd82
--- /dev/null
+++ b/tools/ngs-pileup/ngs-pileup.vers
@@ -0,0 +1 @@
+0.0.1
diff --git a/tools/ngs-pileup/ngs-pileup.vers.h b/tools/ngs-pileup/ngs-pileup.vers.h
new file mode 100644
index 0000000..125ccce
--- /dev/null
+++ b/tools/ngs-pileup/ngs-pileup.vers.h
@@ -0,0 +1 @@
+#define NGS_PILEUP_VERS 0x00000001
diff --git a/tools/pacbio-load/Makefile b/tools/pacbio-load/Makefile
index 6dc272b..095cc1b 100644
--- a/tools/pacbio-load/Makefile
+++ b/tools/pacbio-load/Makefile
@@ -125,12 +125,12 @@ PACBIO_LOAD_OBJ = \
$(addsuffix .$(OBJX),$(PACBIO_LOAD_SRC))
PACBIO_LOAD_LIB = \
- -lkapp \
+ -skapp \
-skdf5 \
- -lload \
+ -sload \
+ -shdf5 \
-sncbi-wvdb \
- -lm \
- -lhdf5 \
+ -sm \
ifdef HDF5_LIBDIR
PACBIO_LOAD_LIB += $(addprefix -L,$(HDF5_LIBDIR))
diff --git a/tools/pacbio-load/pacbio-load.vers b/tools/pacbio-load/pacbio-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/pacbio-load/pacbio-load.vers
+++ b/tools/pacbio-load/pacbio-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/pacbio-load/pl-consensus.c b/tools/pacbio-load/pl-consensus.c
index ed7c26f..c2c2565 100644
--- a/tools/pacbio-load/pl-consensus.c
+++ b/tools/pacbio-load/pl-consensus.c
@@ -61,17 +61,26 @@ static bool check_Consensus_totalcount( BaseCalls_cmn *tab, const uint64_t expec
{
bool res = check_table_count( &tab->Basecall, "Basecall", expected );
if ( res )
- res = check_table_count( &tab->DeletionQV, "DeletionQV", expected );
- if ( res )
- res = check_table_count( &tab->DeletionTag, "DeletionTag", expected );
- if ( res )
- res = check_table_count( &tab->InsertionQV, "InsertionQV", expected );
- if ( res )
res = check_table_count( &tab->QualityValue, "QualityValue", expected );
- if ( res )
- res = check_table_count( &tab->SubstitutionQV, "SubstitutionQV", expected );
- if ( res )
- res = check_table_count( &tab->SubstitutionTag, "SubstitutionTag", expected );
+
+ if ( res )
+ {
+ if ( tab->DeletionQV.extents != NULL )
+ res = check_table_count( &tab->DeletionQV, "DeletionQV", expected );
+
+ if ( tab->DeletionTag.extents != NULL )
+ res = check_table_count( &tab->DeletionTag, "DeletionTag", expected );
+
+ if ( tab->InsertionQV.extents != NULL )
+ res = check_table_count( &tab->InsertionQV, "InsertionQV", expected );
+
+ if ( tab->SubstitutionQV.extents != NULL )
+ res = check_table_count( &tab->SubstitutionQV, "SubstitutionQV", expected );
+
+ if ( tab->SubstitutionTag.extents != NULL )
+ res = check_table_count( &tab->SubstitutionTag, "SubstitutionTag", expected );
+ }
+
return res;
}
@@ -112,6 +121,8 @@ static rc_t consensus_load_spot_bases( VCursor *cursor, BaseCalls_cmn *tab,
const uint32_t *col_idx, zmw_row * spot )
{
rc_t rc = 0;
+ uint32_t column_idx, dummy_src;
+
/* we make a buffer to store NumEvent 8-bit-values
(that is so far the biggest value we have to read per DNA-BASE) */
char * buffer = malloc( spot->NumEvent );
@@ -129,27 +140,67 @@ static rc_t consensus_load_spot_bases( VCursor *cursor, BaseCalls_cmn *tab,
rc = transfer_bits( cursor, col_idx[ consensus_tab_QUALITY ],
&tab->QualityValue, buffer, spot->offset, spot->NumEvent,
QUALITY_VALUE_BITSIZE, "consensus.QualityValue" );
+
if ( rc == 0 )
- rc = transfer_bits( cursor, col_idx[ consensus_tab_INSERTION_QV ],
- &tab->InsertionQV, buffer, spot->offset, spot->NumEvent,
- INSERTION_QV_BITSIZE, "consensus.InsertionQV" );
+ {
+ column_idx = col_idx[ consensus_tab_INSERTION_QV ];
+ if ( tab->InsertionQV.extents != NULL )
+ rc = transfer_bits( cursor, column_idx,
+ &tab->InsertionQV, buffer, spot->offset, spot->NumEvent,
+ INSERTION_QV_BITSIZE, "consensus.InsertionQV" );
+ else
+ rc = vdb_write_value( cursor, column_idx,
+ &dummy_src, INSERTION_QV_BITSIZE, 0, "consensus.InsertionQV" );
+ }
+
if ( rc == 0 )
- rc = transfer_bits( cursor, col_idx[ consensus_tab_DELETION_QV ],
- &tab->DeletionQV, buffer, spot->offset, spot->NumEvent,
- DELETION_QV_BITSIZE, "consensus.DeletionQV" );
+ {
+ column_idx = col_idx[ consensus_tab_DELETION_QV ];
+ if ( tab->DeletionQV.extents != NULL )
+ rc = transfer_bits( cursor, column_idx,
+ &tab->DeletionQV, buffer, spot->offset, spot->NumEvent,
+ DELETION_QV_BITSIZE, "consensus.DeletionQV" );
+ else
+ rc = vdb_write_value( cursor, column_idx,
+ &dummy_src, DELETION_QV_BITSIZE, 0, "consensus.DeletionQV" );
+ }
+
if ( rc == 0 )
- rc = transfer_bits( cursor, col_idx[ consensus_tab_DELETION_TAG ],
- &tab->DeletionTag, buffer, spot->offset, spot->NumEvent,
- DELETION_TAG_BITSIZE, "consensus.DeletionTag" );
+ {
+ column_idx = col_idx[ consensus_tab_DELETION_TAG ];
+ if ( tab->DeletionTag.extents != NULL )
+ rc = transfer_bits( cursor, column_idx,
+ &tab->DeletionTag, buffer, spot->offset, spot->NumEvent,
+ DELETION_TAG_BITSIZE, "consensus.DeletionTag" );
+ else
+ rc = vdb_write_value( cursor, column_idx,
+ &dummy_src, DELETION_TAG_BITSIZE, 0, "consensus.DeletionTag" );
+ }
+
if ( rc == 0 )
- rc = transfer_bits( cursor, col_idx[ consensus_tab_SUBSTITUTION_QV ],
- &tab->SubstitutionQV, buffer, spot->offset, spot->NumEvent,
- SUBSTITUTION_QV_BITZISE, "consensus.SubstitutionQV" );
+ {
+ column_idx = col_idx[ consensus_tab_SUBSTITUTION_QV ];
+ if ( tab->SubstitutionQV.extents != NULL )
+ rc = transfer_bits( cursor, column_idx,
+ &tab->SubstitutionQV, buffer, spot->offset, spot->NumEvent,
+ SUBSTITUTION_QV_BITZISE, "consensus.SubstitutionQV" );
+ else
+ rc = vdb_write_value( cursor, column_idx,
+ &dummy_src, SUBSTITUTION_QV_BITZISE, 0, "consensus.SubstitutionQV" );
+ }
+
if ( rc == 0 )
- rc = transfer_bits( cursor, col_idx[ consensus_tab_SUBSTITUTION_TAG ],
- &tab->SubstitutionTag, buffer, spot->offset, spot->NumEvent,
- SUBSTITUTION_TAG_BITSIZE, "consensus.SubstitutionTag" );
-
+ {
+ column_idx = col_idx[ consensus_tab_SUBSTITUTION_TAG ];
+ if ( tab->SubstitutionTag.extents != NULL )
+ rc = transfer_bits( cursor, column_idx,
+ &tab->SubstitutionTag, buffer, spot->offset, spot->NumEvent,
+ SUBSTITUTION_TAG_BITSIZE, "consensus.SubstitutionTag" );
+ else
+ rc = vdb_write_value( cursor, column_idx,
+ &dummy_src, SUBSTITUTION_TAG_BITSIZE, 0, "consensus.SubstitutionTag" );
+ }
+
if ( buffer != NULL )
free( buffer );
return rc;
@@ -160,64 +211,68 @@ static rc_t consensus_load_spot( VCursor *cursor, const uint32_t *col_idx,
region_type_mapping *mapping, zmw_row * spot,
void * data )
{
- BaseCalls_cmn *tab = (BaseCalls_cmn *)data;
- rc_t rc = VCursorOpenRow( cursor );
- if ( rc != 0 )
- PLOGERR( klogErr, ( klogErr, rc, "cannot open consensus-row on spot# $(spotnr)",
- "spotnr=%u", spot->spot_nr ) );
-
- if ( rc == 0 )
- rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_HOLE_NUMBER ],
- spot->HoleNumber, "consensus.HOLE_NUMBER" );
- if ( rc == 0 )
- rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_HOLE_STATUS ],
- spot->HoleStatus, "consensus.HOLE_STATUS" );
- if ( rc == 0 )
- rc = vdb_write_value( cursor, col_idx[ consensus_tab_HOLE_XY ],
- &spot->HoleXY, HOLE_XY_BITSIZE, 2, "consensus.HOLE_XY" );
-
- /* has to be read ... from "PulseData/ConsensusBaesCalls/Passes/NumPasses" */
- if ( rc == 0 )
- rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_NUM_PASSES ],
- spot->NumPasses, "consensus.NUM_PASSES" );
-
- if ( rc == 0 )
- {
- if ( spot->NumEvent > 0 )
- rc = consensus_load_spot_bases( cursor, tab, col_idx, spot );
- else
- rc = consensus_load_zero_bases( cursor, col_idx );
- }
-
- if ( rc == 0 )
- rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_NREADS ],
- 1, "consensus.NREADS" );
- if ( rc == 0 )
- rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_READ_START ],
- 0, "consensus.READ_START" );
- if ( rc == 0 )
- rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_READ_LEN ],
- spot->NumEvent, "consensus.READ_LEN" );
- if ( rc == 0 )
- rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_READ_TYPE ],
- SRA_READ_TYPE_BIOLOGICAL, "consensus.READ_TYPE" );
-
- if ( rc == 0 )
- {
- rc = VCursorCommitRow( cursor );
- if ( rc != 0 )
- PLOGERR( klogErr, ( klogErr, rc, "cannot commit consensus-row on spot# $(spotnr)",
- "spotnr=%u", spot->spot_nr ) );
- }
-
- if ( rc == 0 )
- {
- rc = VCursorCloseRow( cursor );
- if ( rc != 0 )
- PLOGERR( klogErr, ( klogErr, rc, "cannot close consensus-row on spot# $(spotnr)",
- "spotnr=%u", spot->spot_nr ) );
-
- }
+ rc_t rc = 0;
+ if ( spot->NumEvent > 0 )
+ {
+ BaseCalls_cmn *tab = (BaseCalls_cmn *)data;
+ rc = VCursorOpenRow( cursor );
+ if ( rc != 0 )
+ PLOGERR( klogErr, ( klogErr, rc, "cannot open consensus-row on spot# $(spotnr)",
+ "spotnr=%u", spot->spot_nr ) );
+
+ if ( rc == 0 )
+ rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_HOLE_NUMBER ],
+ spot->HoleNumber, "consensus.HOLE_NUMBER" );
+ if ( rc == 0 )
+ rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_HOLE_STATUS ],
+ spot->HoleStatus, "consensus.HOLE_STATUS" );
+ if ( rc == 0 )
+ rc = vdb_write_value( cursor, col_idx[ consensus_tab_HOLE_XY ],
+ &spot->HoleXY, HOLE_XY_BITSIZE, 2, "consensus.HOLE_XY" );
+
+ /* has to be read ... from "PulseData/ConsensusBaesCalls/Passes/NumPasses" */
+ if ( rc == 0 )
+ rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_NUM_PASSES ],
+ spot->NumPasses, "consensus.NUM_PASSES" );
+
+ if ( rc == 0 )
+ {
+ if ( spot->NumEvent > 0 )
+ rc = consensus_load_spot_bases( cursor, tab, col_idx, spot );
+ else
+ rc = consensus_load_zero_bases( cursor, col_idx );
+ }
+
+ if ( rc == 0 )
+ rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_NREADS ],
+ 1, "consensus.NREADS" );
+ if ( rc == 0 )
+ rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_READ_START ],
+ 0, "consensus.READ_START" );
+ if ( rc == 0 )
+ rc = vdb_write_uint32( cursor, col_idx[ consensus_tab_READ_LEN ],
+ spot->NumEvent, "consensus.READ_LEN" );
+ if ( rc == 0 )
+ rc = vdb_write_uint8( cursor, col_idx[ consensus_tab_READ_TYPE ],
+ SRA_READ_TYPE_BIOLOGICAL, "consensus.READ_TYPE" );
+
+ if ( rc == 0 )
+ {
+ rc = VCursorCommitRow( cursor );
+ if ( rc != 0 )
+ PLOGERR( klogErr, ( klogErr, rc, "cannot commit consensus-row on spot# $(spotnr)",
+ "spotnr=%u", spot->spot_nr ) );
+ }
+
+ if ( rc == 0 )
+ {
+ rc = VCursorCloseRow( cursor );
+ if ( rc != 0 )
+ PLOGERR( klogErr, ( klogErr, rc, "cannot close consensus-row on spot# $(spotnr)",
+ "spotnr=%u", spot->spot_nr ) );
+
+ }
+ }
return rc;
}
@@ -267,11 +322,15 @@ static rc_t consensus_loader( ld_context *lctx, KDirectory * hdf5_src, VCursor *
KLogLevelSet( tmp_lvl );
if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) )
+ {
rc = zmw_for_each( &ConsensusTab.zmw, &lctx->xml_progress, cursor,
lctx->with_progress, col_idx, NULL,
true, consensus_load_spot, &ConsensusTab );
+ }
else
+ {
rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
+ }
close_BaseCalls_cmn( &ConsensusTab );
}
}
diff --git a/tools/pacbio-load/pl-context.c b/tools/pacbio-load/pl-context.c
index c789389..431b909 100644
--- a/tools/pacbio-load/pl-context.c
+++ b/tools/pacbio-load/pl-context.c
@@ -59,7 +59,7 @@ static rc_t ctx_get_params( const Args * args, context *ctx )
else for ( idx = 0; idx < count; ++idx )
{
const char *parameter = NULL;
- rc = ArgsParamValue( args, idx, ¶meter );
+ rc = ArgsParamValue( args, idx, (const void **)¶meter );
if ( rc != 0 )
LOGERR( klogErr, rc, "error reading commandline-parameter" );
else
@@ -86,7 +86,7 @@ static const char* ctx_get_str( const Args *args, const char *name, const char *
uint32_t count = 0;
if ( ArgsOptionCount( args, name, &count ) == 0 && count > 0 )
{
- if ( ArgsOptionValue( args, name, 0, &res ) != 0 )
+ if ( ArgsOptionValue( args, name, 0, (const void **)&res ) != 0 )
res = def;
}
return res;
diff --git a/tools/pacbio-load/pl-regions.c b/tools/pacbio-load/pl-regions.c
index 4cf66ea..4a4ea28 100644
--- a/tools/pacbio-load/pl-regions.c
+++ b/tools/pacbio-load/pl-regions.c
@@ -137,7 +137,7 @@ void rgn_free( regions *rgn )
static
-int CC rgn_sort_callback( const void *p1, const void *p2, void * data )
+int64_t CC rgn_sort_callback( const void *p1, const void *p2, void * data )
{
regions *rgn = ( regions * ) data;
int32_t idx1 = *( int32_t * ) p1;
@@ -249,7 +249,7 @@ static rc_t rgn_get_or_make( Vector * stock, region ** r )
}
-static int CC rgn_sort_by_start( const void *item, const void *n )
+static int64_t CC rgn_sort_by_start( const void *item, const void *n )
{
region * v1 = ( region * )item;
region * v2 = ( region * )n;
diff --git a/tools/pacbio-load/pl-tools.c b/tools/pacbio-load/pl-tools.c
index 96b1032..62437e1 100644
--- a/tools/pacbio-load/pl-tools.c
+++ b/tools/pacbio-load/pl-tools.c
@@ -432,13 +432,21 @@ rc_t add_columns( VCursor * cursor, uint32_t count, int32_t exclude_this,
bool check_table_count( af_data *tab, const char * name,
const uint64_t expected )
{
- bool res = ( tab->extents[ 0 ] == expected );
- if ( !res )
- {
- rc_t rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
- PLOGERR( klogErr, ( klogErr, rc, "'$(name)'.count != expected",
- "name=%s", name ) );
- }
+ bool res = ( tab->extents != NULL );
+ if ( res )
+ {
+ res = ( tab->extents[ 0 ] == expected );
+ if ( !res )
+ {
+ rc_t rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
+ PLOGERR( klogErr, ( klogErr, rc, "'$(name)'.count != expected",
+ "name=%s", name ) );
+ }
+ }
+ else
+ {
+ PLOGMSG( klogWarn, ( klogWarn, "Table ... '$(name)' not found", "name=%s", name ) );
+ }
return res;
}
diff --git a/tools/deseq/Makefile b/tools/pileup-stats/Makefile
similarity index 83%
rename from tools/deseq/Makefile
rename to tools/pileup-stats/Makefile
index 01614da..80d9dfb 100644
--- a/tools/deseq/Makefile
+++ b/tools/pileup-stats/Makefile
@@ -26,14 +26,14 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/deseq
+MODULE = tools/pileup-stats
include $(TOP)/build/Makefile.env
INT_TOOLS =
EXT_TOOLS = \
- csra-trim
+ pileup-stats
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -73,30 +73,23 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# align-info
+# pileup-stats
#
-CSRA_TRIM_SRC = \
- deseq
-
-CSRA_TRIM_OBJ = \
- $(addsuffix .$(OBJX),$(CSRA_TRIM_SRC))
-
-CSRA_TRIM_LIB = \
- -lkapp \
- -lalign-reader \
- -lwvdb \
- -lwkdb \
- $(UPDATE_SCHEMA_LIBS) \
- -lksrch \
- -lkproc \
- -lvfs \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lklib \
- -lm
-
-
-$(BINDIR)/csra-trim: $(CSRA_TRIM_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(CSRA_TRIM_LIB)
+PILEUP_STATS_SRC = \
+ pileup-stats
+PILEUP_STATS_OBJ = \
+ $(addsuffix .$(OBJX),$(PILEUP_STATS_SRC))
+
+PILEUP_STATS_LIB = \
+ -L$(LIBDIR) \
+ -L$(NGS_LIBDIR) \
+ -sgeneral-writer \
+ -sncbi-ngs-c++ \
+ -sncbi-vdb-static \
+ -sngs-c++ \
+ -skapp \
+
+
+$(BINDIR)/pileup-stats: $(PILEUP_STATS_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(PILEUP_STATS_LIB)
diff --git a/tools/pileup-stats/pileup-stats.cpp b/tools/pileup-stats/pileup-stats.cpp
new file mode 100644
index 0000000..6923e0e
--- /dev/null
+++ b/tools/pileup-stats/pileup-stats.cpp
@@ -0,0 +1,719 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <ngs/ncbi/NGS.hpp>
+#include <ngs/ReadCollection.hpp>
+#include <ngs/Reference.hpp>
+#include <ngs/Alignment.hpp>
+#include <ngs/Pileup.hpp>
+#include <ngs/PileupEvent.hpp>
+
+#include <kapp/main.h>
+#include <klib/printf.h>
+#include <iomanip>
+
+#define DFLT_BUFFER_SIZE ( 32 * 1024 )
+
+#include "../general-loader/general-writer.hpp"
+#include <arch-impl.h>
+
+#include "pileup-stats.vers.h"
+
+#include <iostream>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/types.h>
+#include <unistd.h>
+
+// log via general-writer API rather than stderr
+#define USE_GW_LOGMESSAGE 1
+
+using namespace ngs;
+
+namespace ncbi
+{
+ enum
+ {
+ col_RUN_NAME,
+ col_REFERENCE_SPEC,
+ col_REF_POS_TRANS,
+ col_DEPTH,
+ col_MISMATCH_COUNTS,
+ col_INSERTION_COUNTS,
+ col_DELETION_COUNT,
+
+ num_columns
+ };
+
+ static int tbl_id;
+ static int column_id [ num_columns ];
+ static uint8_t integer_column_flag_bits = 1; // pack integers by default
+ static int64_t zrow_id;
+
+ static uint32_t depth_cutoff = 1; // do not output if depth <= this value
+ static uint32_t event_cutoff = 1;
+ static uint32_t num_significant_bits = 4;
+
+ static uint32_t verbosity;
+
+ const bool need_write_true = false;
+
+ inline
+ uint32_t filter_significant_bits ( uint32_t val )
+ {
+ if ( val != 0 )
+ {
+ // we find the most significant bit in value
+ // which will be 0..31 for non-zero values
+ // from which we subtract the number of significant bits
+ // being retained ( - 1 to account for 0-based index )
+ //
+ // e.g.:
+ // val = 0x31, num_significant_bits = 4
+ // uint32_msbit ( val ) => 5, meaning the most significant bit is 5,
+ // which says there are 6 significant bits in total. we only want to
+ // keep 4, however.
+ // mask_index = 5 - ( 4 - 1 ) => 2
+ // 0xFFFFFFFF << mask_index => 0xFFFFFFFC
+ // val then becomes 0x30
+ int mask_index = uint32_msbit ( val ) - ( int ) ( num_significant_bits - 1 );
+ if ( mask_index > 0 )
+ {
+ // this is a positive quantity
+ assert ( mask_index <= 31 );
+ val = val & ( 0xFFFFFFFF << mask_index );
+ }
+ }
+
+ return val;
+ }
+
+ static
+ void run (
+ GeneralWriter & out,
+ const String & runName, const String & refName, PileupIterator & pileup, uint64_t refLength, uint64_t refLengthSubTotal, uint64_t refLengthTotal )
+ {
+ int64_t ref_zpos, last_writ = 0;
+ bool need_write = false;
+
+ for ( ref_zpos = -1; pileup . nextPileup (); ++ ref_zpos, ++ zrow_id )
+ {
+ if ( ref_zpos < 0 )
+ {
+ last_writ = ref_zpos = pileup . getReferencePosition ();
+ int64_t ref_pos_trans = ref_zpos - zrow_id;
+ out . columnDefault ( column_id [ col_REF_POS_TRANS ], 64, & ref_pos_trans, 1 );
+ need_write = need_write_true;
+ }
+
+ switch ( verbosity )
+ {
+ case 0:
+ break;
+ case 1:
+ if ( ( ref_zpos % 1000000 ) == 0 )
+#if USE_GW_LOGMESSAGE
+ out . progMsg ( refName, PILEUP_STATS_VERS, ref_zpos + refLengthSubTotal , refLengthTotal );
+#else
+ std :: cerr << "# " << std :: setw ( 9 ) << ref_zpos << '\n';
+#endif
+ break;
+ default:
+ if ( ( ref_zpos % 5000 ) == 0 )
+ {
+#if USE_GW_LOGMESSAGE
+ out . progMsg ( refName, PILEUP_STATS_VERS, ref_zpos + refLengthSubTotal , refLengthTotal );
+#else
+ if ( ( ref_zpos % 500000 ) == 0 )
+ std :: cerr << "\n# " << std :: setw ( 9 ) << ref_zpos << ' ';
+ std :: cerr << '.';
+#endif
+ }
+ }
+
+ uint32_t ref_base_idx = 0;
+ bool ref_ambiguous = false;
+ char ref_base = pileup . getReferenceBase ();
+ switch ( toupper ( ref_base ) )
+ {
+ case 'A': break;
+ case 'C': ref_base_idx = 1; break;
+ case 'G': ref_base_idx = 2; break;
+ case 'T': ref_base_idx = 3; break;
+ default:
+ ref_base_idx = 0;
+ ref_ambiguous = true;
+ }
+
+ uint32_t depth = pileup . getPileupDepth ();
+
+ uint32_t mismatch_counts [ 4 ];
+ memset ( mismatch_counts, 0, sizeof mismatch_counts );
+
+ uint32_t ins_counts [ 4 ];
+ memset ( ins_counts, 0, sizeof ins_counts );
+
+ uint32_t del_cnt = 0;
+
+ if ( depth > depth_cutoff )
+ {
+ char mismatch;
+ uint32_t mismatch_idx;
+
+ while ( pileup . nextPileupEvent () )
+ {
+ PileupEvent :: PileupEventType et = pileup . getEventType ();
+ switch ( et & 7 )
+ {
+ case PileupEvent :: match:
+
+ if ( ! ref_ambiguous )
+ {
+ handle_N_in_mismatch:
+ if ( ( et & PileupEvent :: insertion ) != 0 )
+ ++ ins_counts [ ref_base_idx ];
+ break;
+ }
+ // intentional fall-through
+
+ case PileupEvent :: mismatch:
+ mismatch = pileup . getAlignmentBase ();
+ mismatch_idx = 0;
+ switch ( toupper ( mismatch ) )
+ {
+ case 'A': break;
+ case 'C': mismatch_idx = 1; break;
+ case 'G': mismatch_idx = 2; break;
+ case 'T': mismatch_idx = 3; break;
+ default:
+ // treat N by removing this event from depth
+ -- depth;
+ goto handle_N_in_mismatch;
+ }
+
+ // first, assert that mismatch_idx cannot be ref_base_idx
+ assert ( ref_ambiguous || mismatch_idx != ref_base_idx );
+
+ // count insertions
+ if ( ( et & PileupEvent :: insertion ) != 0 )
+ ++ ins_counts [ mismatch_idx ];
+ // count the mismatches
+ ++ mismatch_counts [ mismatch_idx ];
+ break;
+
+ case PileupEvent :: deletion:
+ if ( pileup . getEventIndelType () == PileupEvent :: normal_indel )
+ ++ del_cnt;
+ else
+ -- depth;
+ break;
+ }
+ }
+ }
+
+ if ( depth > depth_cutoff )
+ {
+ int i;
+
+ if ( del_cnt <= event_cutoff )
+ del_cnt = 0;
+
+ bool have_mismatch = false;
+ for ( i = 0; i < 4; ++ i )
+ {
+ if ( mismatch_counts [ i ] <= event_cutoff )
+ mismatch_counts [ i ] = 0;
+ else
+ have_mismatch = true;
+ }
+
+ bool have_inserts = false;
+ for ( i = 0; i < 4; ++ i )
+ {
+ if ( ins_counts [ i ] <= event_cutoff )
+ ins_counts [ i ] = 0;
+ else
+ have_inserts = true;
+ }
+
+ if ( num_significant_bits != 0 )
+ {
+ depth = filter_significant_bits ( depth );
+ del_cnt = filter_significant_bits ( del_cnt );
+ for ( i = 0; i < 4; ++ i )
+ mismatch_counts [ i ] = filter_significant_bits ( mismatch_counts [ i ] );
+ for ( i = 0; i < 4; ++ i )
+ ins_counts [ i ] = filter_significant_bits ( ins_counts [ i ] );
+ }
+
+
+ if ( ref_zpos > last_writ )
+ out . moveAhead ( tbl_id, ref_zpos - last_writ );
+ out . write ( column_id [ col_DEPTH ], sizeof depth * 8, & depth, 1 );
+ if ( have_mismatch )
+ out . write ( column_id [ col_MISMATCH_COUNTS ], sizeof mismatch_counts [ 0 ] * 8, mismatch_counts, 4 );
+ if ( have_inserts )
+ out . write ( column_id [ col_INSERTION_COUNTS ], sizeof ins_counts [ 0 ] * 8, ins_counts, 4 );
+ if ( del_cnt != 0 )
+ out . write ( column_id [ col_DELETION_COUNT ], sizeof del_cnt * 8, & del_cnt, 1 );
+ out . nextRow ( tbl_id );
+ last_writ = ref_zpos + 1;
+ }
+ else if ( need_write )
+ {
+ out . nextRow ( tbl_id );
+ need_write = false;
+ }
+
+ }
+ if ( ref_zpos > last_writ )
+ out . moveAhead ( tbl_id, ref_zpos - last_writ );
+ }
+
+ static
+ void prepareOutput ( GeneralWriter & out, const String & runName )
+ {
+ // add table
+ tbl_id = out . addTable ( "STATS" );
+
+ // add each column
+ column_id [ col_RUN_NAME ] = out . addColumn ( tbl_id, "RUN_NAME", 8 );
+ column_id [ col_REFERENCE_SPEC ] = out . addColumn ( tbl_id, "REFERENCE_SPEC", 8 );
+ column_id [ col_REF_POS_TRANS ] = out . addColumn ( tbl_id, "REF_POS_TRANS", 64, 0 );
+ column_id [ col_DEPTH ] = out . addColumn ( tbl_id, "DEPTH", 32, integer_column_flag_bits );
+ column_id [ col_MISMATCH_COUNTS ] = out . addColumn ( tbl_id, "MISMATCH_COUNTS", 32, integer_column_flag_bits );
+ column_id [ col_INSERTION_COUNTS ] = out . addColumn ( tbl_id, "INSERTION_COUNTS", 32, integer_column_flag_bits );
+ column_id [ col_DELETION_COUNT ] = out . addColumn ( tbl_id, "DELETION_COUNT", 32, integer_column_flag_bits );
+
+ // open the stream
+ out . open ();
+
+ // set default values
+ out . columnDefault ( column_id [ col_RUN_NAME ], 8, runName . data (), runName . size () );
+ out . columnDefault ( column_id [ col_DEPTH ], 32, "", 0 );
+ out . columnDefault ( column_id [ col_MISMATCH_COUNTS ], 32, "", 0 );
+ out . columnDefault ( column_id [ col_INSERTION_COUNTS ], 32, "", 0 );
+ out . columnDefault ( column_id [ col_DELETION_COUNT ], 32, "", 0 );
+ }
+
+ static
+ void set_software ( GeneralWriter & out )
+ {
+ char vers_str [ 64 ];
+ ver_t version = KAppVersion ();
+ string_printf ( vers_str, sizeof vers_str, NULL, "%V", version );
+ out . setSoftwareName ( "pileup-stats", vers_str );
+ }
+
+ static
+ void run ( const char * spec, const char *outfile, const char *_remote_db, size_t buffer_size, Alignment :: AlignmentCategory cat )
+ {
+#if ! USE_GW_LOGMESSAGE
+ if ( verbosity > 0 )
+ std :: cerr << "# Opening run '" << spec << "'\n";
+#endif
+ ReadCollection obj = ncbi :: NGS :: openReadCollection ( spec );
+ String runName = obj . getName ();
+
+#if ! USE_GW_LOGMESSAGE
+ if ( verbosity > 0 )
+ {
+ std :: cerr << "# Preparing version " << GW_CURRENT_VERSION << " pipe to stdout\n";
+ if ( ( integer_column_flag_bits & 1 ) != 0 )
+ std :: cerr << "# USING INTEGER PACKING\n";
+ }
+#endif
+ std :: string remote_db;
+ if ( _remote_db == NULL )
+ remote_db = runName + ".pileup_stat";
+ else
+ remote_db = _remote_db;
+
+ GeneralWriter *outp = ( outfile == NULL ) ?
+ new GeneralWriter ( 1, buffer_size ) : new GeneralWriter ( outfile );
+
+ try
+ {
+ GeneralWriter &out = *outp;
+
+ // set software node
+ set_software ( out );
+
+ // add remote db event
+ out . setRemotePath ( remote_db );
+
+ // use schema
+ out . useSchema ( "align/pileup-stats.vschema", "NCBI:pileup:db:pileup_stats #1" );
+
+ prepareOutput ( out, runName );
+ if ( verbosity > 0 )
+#if USE_GW_LOGMESSAGE
+ out . logMsg ( "Accessing all references" );
+#else
+ std :: cerr << "# Accessing all references\n";
+#endif
+
+
+
+ // get the total number of references
+ uint64_t totalRefLength = 0;
+ {
+ ReferenceIterator rf = obj . getReferences ();
+ while ( rf . nextReference () )
+ totalRefLength += rf . getLength ();
+ }
+
+ ReferenceIterator ref = obj . getReferences ();
+
+ uint64_t refLengthSubTotal = 0;
+ while ( ref . nextReference () )
+ {
+ String refName = ref . getCanonicalName ();
+
+ //get the length of the reference so that progress can be given
+ uint64_t refLength = ref . getLength ();
+
+ if ( verbosity > 0 )
+#if USE_GW_LOGMESSAGE
+ {
+ out . logMsg ( "Processing reference '" + refName + "'" );
+ }
+#else
+ std :: cerr << "# Processing reference '" << refName << "'\n";
+#endif
+ out . columnDefault ( column_id [ col_REFERENCE_SPEC ], 8, refName . data (), refName . size () );
+
+ if ( verbosity > 0 )
+#if USE_GW_LOGMESSAGE
+ out . logMsg ( "Accessing all pileups" );
+#else
+ std :: cerr << "# Accessing all pileups\n";
+#endif
+ PileupIterator pileup = ref . getPileups ( cat );
+ run ( out, runName, refName, pileup, refLength, refLengthSubTotal, totalRefLength );
+
+ refLengthSubTotal += refLength;
+#if ! USE_GW_LOGMESSAGE
+ if ( verbosity > 1 )
+ std :: cerr << '\n';
+#endif
+ }
+ }
+ catch ( ErrorMsg & x )
+ {
+ outp -> logError ( x . what () );
+ delete outp;
+ throw;
+ }
+ catch ( const char x [] )
+ {
+ outp -> logError ( x );
+ delete outp;
+ throw;
+ }
+ catch ( ... )
+ {
+ outp -> logError ( "unknown exception" );
+ delete outp;
+ throw;
+ }
+ delete outp;
+ }
+}
+
+extern "C"
+{
+ ver_t CC KAppVersion ()
+ {
+ return PILEUP_STATS_VERS;
+ }
+
+ rc_t CC Usage ( struct Args const * args )
+ {
+ return 0;
+ }
+
+ static
+ const char * getArg ( int & i, int argc, char * argv [] )
+ {
+
+ if ( ++ i == argc )
+ throw "Missing argument";
+
+ return argv [ i ];
+ }
+
+ static
+ const char * findArg ( const char* &arg, int & i, int argc, char * argv [] )
+ {
+ if ( arg [ 1 ] != 0 )
+ {
+ const char * next = arg + 1;
+ arg = "\0";
+ return next;
+ }
+
+ return getArg ( i, argc, argv );
+ }
+
+ static void handle_help ( const char *appName )
+ {
+ const char *appLeaf = strrchr ( appName, '/' );
+ if ( appLeaf ++ == 0 )
+ appLeaf = appName;
+
+ ver_t vers = KAppVersion ();
+
+ std :: cout
+ << '\n'
+ << "Usage:\n"
+ << " " << appLeaf << " [options] <accession>"
+ << "\n\n"
+ << "Options:\n"
+ << " -o|--output-file file for output\n"
+ << " (default pipe to stdout)\n"
+ << " -r|--remote-db name of remote database to create\n"
+ << " (default <accession>.pileup_stat)\n"
+ << " -x|--depth-cutoff cutoff for depth <= value (default 1)\n"
+ << " -e|--event-cutoff cutoff for number of events > value (default " << ncbi::event_cutoff << ")\n"
+ << " -p|--num-significant-bits number of significant bits for depth and counts to store (default " << ncbi::num_significant_bits << ")\n"
+ << " -a|--align-category the types of alignments to pile up:\n"
+ << " { primary, secondary, all } (default primary)\n"
+ << " --buffer-size bytes size of output pipe buffer - default " << DFLT_BUFFER_SIZE/1024 << "K bytes\n"
+ << " -U|--unpack-integer don't pack integers in output pipe - uses more bandwidth\n"
+ << " -h|--help output brief explanation of the program\n"
+ << " -v|--verbose increase the verbosity of the program.\n"
+ << " use multiple times for more verbosity.\n"
+ << '\n'
+ << appName << " : "
+ << ( vers >> 24 )
+ << '.'
+ << ( ( vers >> 16 ) & 0xFF )
+ << '.'
+ << ( vers & 0xFFFF )
+ << '\n'
+ << '\n'
+ ;
+ }
+
+ static void CC handle_error ( const char *arg, void *message )
+ {
+ throw ( const char * ) message;
+ }
+
+ rc_t CC KMain ( int argc, char *argv [] )
+ {
+ rc_t rc = -1;
+ Alignment :: AlignmentCategory cat = Alignment :: primaryAlignment;
+ size_t buffer_size = DFLT_BUFFER_SIZE;
+ try
+ {
+ int num_runs = 0;
+ const char *outfile = NULL;
+ const char *remote_db = NULL;
+
+ for ( int i = 1; i < argc; ++ i )
+ {
+ const char * arg = argv [ i ];
+ if ( arg [ 0 ] != '-' )
+ {
+ // have an input run
+ argv [ ++ num_runs ] = ( char* ) arg;
+ }
+ else do switch ( ( ++ arg ) [ 0 ] )
+ {
+ case 'o':
+ outfile = findArg ( arg, i, argc, argv );
+ break;
+ case 'r':
+ remote_db = findArg ( arg, i, argc, argv );
+ break;
+ case 'x':
+ ncbi :: depth_cutoff = AsciiToU32 ( findArg ( arg, i, argc, argv ),
+ handle_error, ( void * ) "Invalid depth cutoff" );
+ break;
+ case 'e':
+ ncbi :: event_cutoff = AsciiToU32 ( findArg ( arg, i, argc, argv ),
+ handle_error, ( void * ) "Invalid event cutoff" );
+ break;
+ case 'p':
+ ncbi :: num_significant_bits = AsciiToU32 ( findArg ( arg, i, argc, argv ),
+ handle_error, ( void * ) "Invalid num-significant-bits" );
+ break;
+ case 'a':
+ {
+ const char * atype = findArg ( arg, i, argc, argv );
+ if ( strcmp ( atype, "all" ) == 0 )
+ cat = Alignment :: all;
+ else if ( strcmp ( atype, "primary" ) == 0 ||
+ strcmp ( atype, "primaryAlignment" ) == 0 )
+ cat = Alignment :: primaryAlignment;
+ else if ( strcmp ( atype, "secondary" ) == 0 ||
+ strcmp ( atype, "secondaryAlignment" ) == 0 )
+ cat = Alignment :: secondaryAlignment;
+ else
+ {
+ throw "Invalid alignment category";
+ }
+ break;
+ }
+ case 'U':
+ ncbi :: integer_column_flag_bits = 0;
+ break;
+ case 'v':
+ ++ ncbi :: verbosity;
+ break;
+ case 'h':
+ case '?':
+ handle_help ( argv [ 0 ] );
+ return 0;
+ case '-':
+ ++ arg;
+ if ( strcmp ( arg, "output-file" ) == 0 )
+ {
+ outfile = getArg ( i, argc, argv );
+ }
+ else if ( strcmp ( arg, "remote-db" ) == 0 )
+ {
+ remote_db = getArg ( i, argc, argv );
+ }
+ else if ( strcmp ( arg, "buffer-size" ) == 0 )
+ {
+ const char * str = getArg ( i, argc, argv );
+
+ char * end;
+ long new_buffer_size = strtol ( str, & end, 0 );
+ if ( new_buffer_size < 0 || str == ( const char * ) end || end [ 0 ] != 0 )
+ throw "Invalid buffer argument";
+
+ buffer_size = new_buffer_size;
+ }
+ else if ( strcmp ( arg, "depth-cutoff" ) == 0 )
+ {
+ ncbi :: depth_cutoff = AsciiToU32 ( getArg ( i, argc, argv ),
+ handle_error, ( void * ) "Invalid depth cutoff" );
+ }
+ else if ( strcmp ( arg, "event-cutoff" ) == 0 )
+ {
+ ncbi :: event_cutoff = AsciiToU32 ( getArg ( i, argc, argv ),
+ handle_error, ( void * ) "Invalid event cutoff" );
+ }
+ else if ( strcmp ( arg, "num-significant-bits" ) == 0 )
+ {
+ ncbi :: num_significant_bits = AsciiToU32 ( getArg ( i, argc, argv ),
+ handle_error, ( void * ) "Invalid num-significant-bits" );
+ }
+ else if ( strcmp ( arg, "align-category" ) == 0 )
+ {
+ const char * atype = getArg ( i, argc, argv );
+ if ( strcmp ( atype, "all" ) == 0 )
+ cat = Alignment :: all;
+ else if ( strcmp ( atype, "primary" ) == 0 ||
+ strcmp ( atype, "primaryAlignment" ) == 0 )
+ cat = Alignment :: primaryAlignment;
+ else if ( strcmp ( atype, "secondary" ) == 0 ||
+ strcmp ( atype, "secondaryAlignment" ) == 0 )
+ cat = Alignment :: secondaryAlignment;
+ else
+ {
+ throw "Invalid alignment category";
+ }
+ }
+ else if ( strcmp ( arg, "unpack-integer" ) == 0 )
+ {
+ ncbi :: integer_column_flag_bits = 0;
+ }
+ else if ( strcmp ( arg, "verbose" ) == 0 )
+ {
+ ++ ncbi :: verbosity;
+ }
+ else if ( strcmp ( arg, "help" ) == 0 )
+ {
+ handle_help ( argv [ 0 ] );
+ return 0;
+ }
+ else
+ {
+ throw "Invalid Argument";
+ }
+
+ arg = "\0";
+
+ break;
+ default:
+ throw "Invalid argument";
+ }
+ while ( arg [ 1 ] != 0 );
+ }
+
+ if ( num_runs > 1 )
+ throw "only one run may be processed at a time";
+ for ( int i = 1; i <= num_runs; ++ i )
+ {
+ ncbi :: run ( argv [ i ], outfile, remote_db, buffer_size, cat );
+ }
+
+ rc = 0;
+ }
+ catch ( ErrorMsg & x )
+ {
+#if ! USE_GW_LOGMESSAGE
+ std :: cerr
+ << "ERROR: "
+ << argv [ 0 ]
+ << ": "
+ << x . what ()
+ << '\n'
+ ;
+#endif
+ }
+ catch ( const char x [] )
+ {
+#if ! USE_GW_LOGMESSAGE
+ std :: cerr
+ << "ERROR: "
+ << argv [ 0 ]
+ << ": "
+ << x
+ << '\n'
+ ;
+#endif
+ }
+ catch ( ... )
+ {
+#if ! USE_GW_LOGMESSAGE
+ std :: cerr
+ << "ERROR: "
+ << argv [ 0 ]
+ << ": unknown\n"
+ ;
+#endif
+ }
+
+ return rc;
+ }
+}
diff --git a/tools/pileup-stats/pileup-stats.vers b/tools/pileup-stats/pileup-stats.vers
new file mode 100644
index 0000000..35d16fb
--- /dev/null
+++ b/tools/pileup-stats/pileup-stats.vers
@@ -0,0 +1 @@
+2.5.7
diff --git a/tools/pore-load/GeneralWriter.py b/tools/pore-load/GeneralWriter.py
new file mode 100644
index 0000000..d8296da
--- /dev/null
+++ b/tools/pore-load/GeneralWriter.py
@@ -0,0 +1,312 @@
+# General Writer to drive VDB General Loader
+
+import sys
+import os
+import struct
+import array
+
+def _paddedFormat(fmt):
+ l = struct.calcsize(fmt)
+ if l % 4 != 0:
+ fmt += (" %dx" % (4 - l % 4))
+ return fmt
+
+
+def _makeHeader():
+ fmt = "8s 4I"
+ size = struct.calcsize(fmt)
+ return struct.pack(fmt, "NCBIgnld".encode('ascii'), 1, 2, size, 0)
+
+
+def _makeSimpleEvent(eid):
+ """ used for { evt_end_stream, evt_open_stream, evt_next_row } """
+ return struct.pack("I", eid)
+
+
+def _make1StringEvent(eid, str1):
+ """ used for { evt_errmsg, evt_remote_path, evt_new_table, evt_software_name } """
+ fmt = _paddedFormat("I 1I {}s".format(len(str1)))
+ return struct.pack(fmt, eid, len(str1), str1)
+
+
+def _make2StringEvent(eid, str1, str2):
+ """ used for { evt_use_schema, evt_metadata_node } """
+ fmt = _paddedFormat("I 2I {}s {}s".format(len(str1), len(str2)))
+ return struct.pack(fmt, eid, len(str1), len(str2), str1, str2)
+
+
+def _makeColumnEvent(colid, tblid, bits, name):
+ """ used for { evt_new_column } """
+ fmt = _paddedFormat("I 3I {}s".format(len(name)))
+ return struct.pack(fmt, colid, tblid, bits, len(name), name)
+
+
+def _makeDataEvent(colid, count):
+ """ used for { evt_cell_default, evt_cell_data } """
+ return struct.pack("2I", colid, count)
+
+
+class GeneralWriter:
+ # pylint: disable=too-few-public-methods
+
+ evt_bad_event = 0
+
+ evt_errmsg = (1 << 24) + evt_bad_event
+ evt_end_stream = (1 << 24) + evt_errmsg
+
+ evt_remote_path = (1 << 24) + evt_end_stream
+ evt_use_schema = (1 << 24) + evt_remote_path
+ evt_new_table = (1 << 24) + evt_use_schema
+ evt_new_column = (1 << 24) + evt_new_table
+ evt_open_stream = (1 << 24) + evt_new_column
+
+ evt_cell_default = (1 << 24) + evt_open_stream
+ evt_cell_data = (1 << 24) + evt_cell_default
+ evt_next_row = (1 << 24) + evt_cell_data
+
+ evt_move_ahead = (1 << 24) + evt_next_row # this one is not used here
+ evt_errmsg2 = (1 << 24) + evt_move_ahead # this one is not used here
+ evt_remote_path2 = (1 << 24) + evt_errmsg2 # this one is not used here
+ evt_use_schema2 = (1 << 24) + evt_remote_path2 # this one is not used here
+ evt_new_table2 = (1 << 24) + evt_use_schema2 # this one is not used here
+ evt_cell_default2 = (1 << 24) + evt_new_table2 # this one is not used here
+ evt_cell_data2 = (1 << 24) + evt_cell_default2 # this one is not used here
+ evt_empty_default = (1 << 24) + evt_cell_data2 # this one is not used here
+
+ # BEGIN VERSION 2 MESSAGES
+ evt_software_name = (1 << 24) + evt_empty_default
+ evt_mdata_node_db = (1 << 24) + evt_software_name
+ evt_mdata_node_tbl = (1 << 24) + evt_mdata_node_db
+ evt_mdata_node_col = (1 << 24) + evt_mdata_node_tbl
+
+
+ def errorMessage(self, message):
+ os.write(sys.stdout.fileno(), _make1StringEvent(self.evt_errmsg, message.encode('utf-8')))
+
+
+ def write(self, spec):
+ tableId = spec['_tableId']
+ for k in spec:
+ if k.startswith('_'):
+ continue
+ c = spec[k]
+ if 'data' in c:
+ data = c['data']
+ try:
+ data = data()
+ except:
+ pass
+ try:
+ self._writeColumnData(c['_columnId'], len(data), data)
+ except:
+ sys.stderr.write("failed to write column #{}\n".format(c['_columnId']))
+ raise
+ self._writeNextRow(tableId)
+
+
+ @classmethod
+ def _writeHeader(cls, remoteDb, schemaFileName, schemaDbSpec):
+ os.write(sys.stdout.fileno(), _makeHeader())
+ os.write(sys.stdout.fileno(), _make1StringEvent(cls.evt_remote_path, remoteDb))
+ os.write(sys.stdout.fileno(), _make2StringEvent(cls.evt_use_schema, schemaFileName, schemaDbSpec))
+
+
+ @classmethod
+ def _writeSoftwareName(cls, name, version): # name is any string, version is like "2.1.5"
+ os.write(sys.stdout.fileno(), _make2StringEvent(cls.evt_software_name, name, version))
+
+
+ @classmethod
+ def _writeEndStream(cls):
+ os.write(sys.stdout.fileno(), _makeSimpleEvent(cls.evt_end_stream))
+
+
+ @classmethod
+ def _writeNewTable(cls, tableId, table):
+ os.write(sys.stdout.fileno(), _make1StringEvent(cls.evt_new_table + tableId, table))
+
+
+ @classmethod
+ def _writeNewColumn(cls, columnId, tableId, bits, column):
+ os.write(sys.stdout.fileno(), _makeColumnEvent(cls.evt_new_column + columnId, tableId, bits, column))
+
+
+ @classmethod
+ def _writeOpenStream(cls):
+ os.write(sys.stdout.fileno(), _makeSimpleEvent(cls.evt_open_stream))
+
+
+ @classmethod
+ def _writeColumnDefault(cls, colId, count, data):
+ l = os.write(sys.stdout.fileno(), _makeDataEvent(cls.evt_cell_default + colId, count))
+ l = (l + os.write(sys.stdout.fileno(), data)) % 4
+ if l != 0:
+ os.write(sys.stdout.fileno(), struct.pack("%dx" % (4 - l)))
+
+
+ @classmethod
+ def _writeDbMetadata(cls, dbId, nodeName, nodeValue):
+ os.write(sys.stdout.fileno(), _make2StringEvent(cls.evt_mdata_node_db + dbId, nodeName, nodeValue))
+
+
+ @classmethod
+ def _writeTableMetadata(cls, tblId, nodeName, nodeValue):
+ os.write(sys.stdout.fileno(), _make2StringEvent(cls.evt_mdata_node_tbl + tblId, nodeName, nodeValue))
+
+
+ @classmethod
+ def _writeColumnMetadata(cls, colId, nodeName, nodeValue):
+ os.write(sys.stdout.fileno(), _make2StringEvent(cls.evt_mdata_node_col + colId, nodeName, nodeValue))
+
+
+ @classmethod
+ def _writeColumnData(cls, colId, count, data):
+ l = os.write(sys.stdout.fileno(), _makeDataEvent(cls.evt_cell_data + colId, count))
+ l = (l + os.write(sys.stdout.fileno(), data)) % 4
+ if l != 0:
+ os.write(sys.stdout.fileno(), struct.pack("%dx" % (4 - l)))
+
+
+ @classmethod
+ def _writeNextRow(cls, tableId):
+ os.write(sys.stdout.fileno(), _makeSimpleEvent(cls.evt_next_row + tableId))
+
+
+ def writeDbMetadata(self, nodeName, nodeValue):
+ """ this only supports writing to the default database """
+ GeneralWriter._writeDbMetadata(0, nodeName.encode('ascii'), nodeValue.encode('utf-8'))
+
+
+ def writeTableMetadata(self, table, nodeName, nodeValue):
+ GeneralWriter._writeTableMetadata(table['_tableId'], nodeName.encode('ascii'), nodeValue.encode('utf-8'))
+
+
+ def writeColumnMetadata(self, column, nodeName, nodeValue):
+ GeneralWriter._writeColumnMetadata(column['_columnId'], nodeName.encode('ascii'), nodeValue.encode('utf-8'))
+
+
+ def __init__(self, fileName, schemaFileName, schemaDbSpec, softwareName, versionString, tbl):
+ """ Construct a General Writer object
+
+ writer may be None if no actual output is desired
+ else writer is expected to have a callable write attribute
+
+ fileName is a string with name of the database that will be created
+
+ schemaFileName is a string with the path to the file containing the schema
+
+ schemaDbSpec is a string with the schema name of the database that will be created
+
+ softwareName is a string
+
+ versionString is a three-part number like "2.1.5"
+ """
+
+ GeneralWriter._writeHeader(fileName.encode('utf-8')
+ , schemaFileName.encode('utf-8')
+ , schemaDbSpec.encode('ascii'))
+
+ GeneralWriter._writeSoftwareName(softwareName.encode('utf-8'), versionString.encode('ascii'))
+
+ tableId = 0
+ columnId = 0
+ for t in tbl:
+ tableId += 1
+ GeneralWriter._writeNewTable(tableId, t.encode('ascii'))
+ cols = tbl[t]
+ for c in cols:
+ columnId = columnId + 1
+ cols[c]['_columnId'] = columnId
+ expression = cols[c]['expression'] if 'expression' in cols[c] else c
+ bits = cols[c]['elem_bits']
+ GeneralWriter._writeNewColumn(columnId, tableId, bits, expression.encode('ascii'))
+ tbl[t]['_tableId'] = tableId
+
+ GeneralWriter._writeOpenStream()
+ for t in tbl.values():
+ for c in t.values():
+ try:
+ if 'default' in c:
+ try:
+ GeneralWriter._writeColumnDefault(c['_columnId'], len(c['default']), c['default'])
+ except:
+ sys.stderr.write("failed to set default for %s\n" % c)
+ raise
+ except TypeError:
+ pass
+
+
+ def __del__(self):
+ try: GeneralWriter._writeEndStream()
+ except: pass
+
+
+if __name__ == "__main__":
+ readData = [ "ACGT", "GTAACGT" ]
+ row = 0
+
+ def getRead():
+ global row
+ cur = row
+ row = row + 1
+ return readData[cur].encode('ascii')
+
+ spec = {
+ 'SEQUENCE': {
+ 'READ': {
+ 'expression': '(INSDC:dna:text)READ',
+ 'elem_bits': 8,
+ 'data': getRead
+ },
+ 'QUALITY': {
+ 'expression': '(INSDC:quality:phred)QUALITY',
+ 'elem_bits': 8,
+ },
+ 'LABEL': {
+ 'elem_bits': 8,
+ 'default': 'templatecomplement'.encode('ascii')
+ },
+ 'LABEL_START': {
+ 'elem_bits': 32,
+ },
+ 'LABEL_LENGTH': {
+ 'elem_bits': 32,
+ },
+ 'READ_START': {
+ 'elem_bits': 32,
+ },
+ 'READ_LENGTH': {
+ 'elem_bits': 32,
+ },
+ },
+ 'CONSENSUS': {
+ 'READ': {
+ 'expression': '(INSDC:dna:text)READ',
+ 'elem_bits': 8,
+ },
+ 'QUALITY': {
+ 'expression': '(INSDC:quality:phred)QUALITY',
+ 'elem_bits': 8,
+ },
+ 'LABEL': {
+ 'elem_bits': 8,
+ 'default': '2DFull'.encode('ascii')
+ },
+ 'LABEL_START': {
+ 'elem_bits': 32,
+ 'default': array.array('I', [ 0 ])
+ },
+ 'LABEL_LENGTH': {
+ 'elem_bits': 32,
+ 'default': array.array('I', [ 2 ])
+ }
+ },
+ }
+ gw = GeneralWriter(sys.argv[1], sys.argv[2], sys.argv[3], spec)
+
+ spec['SEQUENCE']['READ_LENGTH']['data'] = array.array('I', [ 1, 2 ])
+ gw.write(spec['SEQUENCE'])
+ gw.write(spec['SEQUENCE'])
+
+ gw = None
+
diff --git a/tools/pore-load/README.md b/tools/pore-load/README.md
new file mode 100644
index 0000000..01bdc01
--- /dev/null
+++ b/tools/pore-load/README.md
@@ -0,0 +1,10 @@
+### This tool relies on the following external python modules:
+- `h5py`
+- `poretools` https://github.com/arq5x/poretools
+- `GeneralWriter` which will be part of ngs, but is tempararily included here
+
+`poretools` also relies on `h5py` and `h5py` relies on `hdf5`. `python` will need to be able to locate the `hdf5` library at the time that the `h5py` module is loaded, for example, by setting LD_LIBRARY_PATH:
+```
+LD_LIBRARY_PATH=/usr/local/hdf5/1.8.10/lib python -O pore-load.py --tmpdir=/dev/shm --output=foo test-files/Ecoli_R7_NONI.tgz
+```
+Furthermore, to do anything useful, the output of this tool needs to be sent to `general-loader`, which is part of `sra-tools`.
diff --git a/tools/pore-load/pore-load.py b/tools/pore-load/pore-load.py
new file mode 100644
index 0000000..7fb0c19
--- /dev/null
+++ b/tools/pore-load/pore-load.py
@@ -0,0 +1,406 @@
+#!python
+"""Options:
+ tmpdir: Path to work space; this should be fast storage where lots of
+ small files can be quickly created and deleted, but it probably
+ doesn't need to be large, /dev/shm or an SSD would be perfect.
+ The work space will be a directory created here using mkdtemp.
+
+ output: The destination VDB database, default is 'pore-load.out' in the
+ work space.
+
+ progress: The update rate (in seconds) for periodic progress messages,
+ 0 to disable, initially set to 10 seconds if __debug__ else 0.
+ Regardless, one progress message will be printed at the end of
+ each input.
+
+ help: displays this message
+
+Note:
+ Input is expected to be .tar.gz files. Decompression is done by a subprocess
+ if zcat or gzcat can be found in the path, otherwise it's done in-process,
+ which is slower.
+
+ Input is taken from stdin if no files are given. If needed, it will be
+ decompressed in-process.
+
+ The args parsing in this script is terribly primitive. You must type exactly
+ as shown, with the '=' and no whitespace.
+"""
+
+import sys
+
+# Environment globals
+gzcat = None # what executable to use for out-of-process decompression
+files = [] # what files to operate on
+tmpdir = "/tmp" # default workspace
+outdir = None
+removeWorkDir = True
+showProgress = 10 if __debug__ else 0
+usage = "Usage: {} [ --tmpdir=<path> ] [ --output=<path> ] [ --progress=<number> ] [ --help ] [file ...]\n".format(sys.argv[0])
+
+for arg in sys.argv[1:]:
+ if arg[0:2] == '--':
+ if arg[2:9] == 'tmpdir=':
+ tmpdir = arg[9:]
+ elif arg[2:9] == 'output=':
+ outdir = arg[9:]
+ elif arg[2:11] == 'progress=':
+ showProgress = eval(arg[11:])
+ elif arg[2:] == 'help':
+ sys.stderr.write(usage)
+ sys.stderr.write(__doc__)
+ exit(0)
+ else:
+ sys.stderr.write(usage)
+ exit(1)
+ else:
+ files.append(arg)
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+import os, subprocess, tempfile
+
+if __debug__:
+ import traceback
+
+import time
+import array
+import tarfile
+import h5py
+import poretools
+import GeneralWriter
+
+
+workDir = tempfile.mkdtemp(suffix='.tmp', prefix='pore-load.', dir=tmpdir)
+if outdir == None:
+ outdir = workDir + '/pore-load.out'
+ removeWorkDir = False
+
+sys.stderr.write("Info: writing '{}'\n".format(outdir))
+sys.stderr.write("Info: using '{}' for work space\n".format(workDir))
+
+tbl = {
+ 'SEQUENCE': {
+ 'READ': {
+ 'expression': '(INSDC:dna:text)READ',
+ 'elem_bits': 8,
+ },
+ 'QUALITY': {
+ 'expression': '(INSDC:quality:text:phred_33)QUALITY',
+ 'elem_bits': 8,
+ },
+ 'SPOT_GROUP': {
+ 'elem_bits': 8,
+ },
+ 'CHANNEL': {
+ 'elem_bits': 32,
+ },
+ 'READ_NO': {
+ 'expression': 'READ_NUMBER',
+ 'elem_bits': 32,
+ },
+ 'READ_START': {
+ 'expression': '(INSDC:coord:zero)READ_START',
+ 'elem_bits': 32,
+ },
+ 'READ_LENGTH': {
+ 'expression': '(INSDC:coord:len)READ_LEN',
+ 'elem_bits': 32,
+ },
+ 'READ_TYPE': {
+ 'expression': '(U8)READ_TYPE',
+ 'elem_bits': 8
+ },
+ },
+ 'CONSENSUS': {
+ 'READ': {
+ 'expression': '(INSDC:dna:text)READ',
+ 'elem_bits': 8,
+ },
+ 'QUALITY': {
+ 'expression': '(INSDC:quality:text:phred_33)QUALITY',
+ 'elem_bits': 8,
+ },
+ 'SPOT_GROUP': {
+ 'elem_bits': 8,
+ },
+ 'CHANNEL': {
+ 'elem_bits': 32,
+ },
+ 'READ_NO': {
+ 'expression': 'READ_NUMBER',
+ 'elem_bits': 32,
+ },
+ 'HIGH_QUALITY': {
+ 'expression': 'HIGH_QUALITY',
+ 'elem_bits': 8,
+ },
+ 'READ_START': {
+ 'expression': '(INSDC:coord:zero)READ_START',
+ 'elem_bits': 32,
+ 'default': array.array('I', [0])
+ },
+ 'READ_LENGTH': {
+ 'expression': '(INSDC:coord:len)READ_LEN',
+ 'elem_bits': 32,
+ },
+ 'READ_TYPE': {
+ 'expression': '(U8)READ_TYPE',
+ 'elem_bits': 8
+ },
+ }
+}
+
+gw = GeneralWriter.GeneralWriter(
+ outdir
+ , 'sra/nanopore.vschema'
+ , 'NCBI:SRA:Nanopore:db'
+ , 'pore-load.py'
+ , '1.0.0'
+ , tbl)
+
+
+class FastQData:
+ """ To hold FastQ data """
+
+ # pylint: disable=too-many-instance-attributes
+ # pylint: disable=too-many-arguments
+ # what an idiotic warning
+
+ def __init__(self, source
+ , readLength
+ , sequence
+ , quality
+ , sequence_2d
+ , quality_2d
+ , channel
+ , readno
+ , isHighQuality
+ ):
+ self.source = source
+ self.readLength = readLength
+ self.sequence = sequence
+ self.quality = quality
+ self.sequence_2d = sequence_2d
+ self.quality_2d = quality_2d
+ self.channel = channel
+ self.readno = readno
+ self.isHighQuality = isHighQuality
+
+ def write(self):
+ """ Write FASTQ data to the General Writer
+ Writes 2D reads to CONSENSUS
+ Writes other reads to SEQUENCE
+ """
+ if self.sequence_2d != None:
+ tbl['CONSENSUS']['HIGH_QUALITY']['data'] = array.array('b', [ 1 if self.isHighQuality else 0 ])
+ tbl['CONSENSUS']['READ_LENGTH' ]['data'] = array.array('I', [len(self.sequence_2d)])
+ tbl['CONSENSUS']['READ_TYPE' ]['data'] = array.array('B', [1])
+ tbl['CONSENSUS']['READ' ]['data'] = self.sequence_2d.encode('ascii')
+ tbl['CONSENSUS']['QUALITY' ]['data'] = self.quality_2d.encode('ascii')
+ else:
+ tbl['CONSENSUS']['HIGH_QUALITY']['data'] = array.array('b', [False])
+ tbl['CONSENSUS']['READ_LENGTH' ]['data'] = array.array('I', [0])
+ tbl['CONSENSUS']['READ_TYPE' ]['data'] = array.array('B', [0])
+ tbl['CONSENSUS']['READ' ]['data'] = ''.encode('ascii')
+ tbl['CONSENSUS']['QUALITY' ]['data'] = ''.encode('ascii')
+
+ tbl['SEQUENCE']['READ_START' ]['data'] = array.array('I', [ 0, self.readLength[0] ])
+ tbl['SEQUENCE']['READ_LENGTH']['data'] = self.readLength
+ tbl['SEQUENCE']['READ_TYPE' ]['data'] = array.array('B', map((lambda length: 1 if length > 0 else 0), self.readLength))
+ tbl['SEQUENCE']['READ' ]['data'] = self.sequence.encode('ascii')
+ tbl['SEQUENCE']['QUALITY' ]['data'] = self.quality.encode('ascii')
+ tbl['SEQUENCE']['CHANNEL' ]['data'] = array.array('I', [self.channel])
+ tbl['SEQUENCE']['READ_NO' ]['data'] = array.array('I', [self.readno])
+ spotGroup = os.path.basename(self.source)
+ try:
+ at = spotGroup.rindex("_ch{}_".format(self.channel))
+ spotGroup = spotGroup[0:at]
+ except:
+ pass
+ tbl['SEQUENCE']['SPOT_GROUP']['data'] = spotGroup
+
+ tbl['CONSENSUS']['CHANNEL' ]['data'] = tbl['SEQUENCE']['CHANNEL' ]['data']
+ tbl['CONSENSUS']['READ_NO' ]['data'] = tbl['SEQUENCE']['READ_NO' ]['data']
+ tbl['CONSENSUS']['SPOT_GROUP']['data'] = tbl['SEQUENCE']['SPOT_GROUP']['data']
+
+ gw.write(tbl['SEQUENCE' ])
+ gw.write(tbl['CONSENSUS'])
+
+
+ @classmethod
+ def ReadFast5Data(cls, fname):
+ """ Read the FASTQ data from the fast5 file
+
+ This is segregated into its own function in order to catch errors
+ that poretools itself doesn't catch, e.g. malformed HDF5
+ """
+ f5 = poretools.Fast5File(fname)
+ try:
+ channel = int(f5.get_channel_number())
+ readno = int(f5.get_read_number())
+ sequence_2d = None
+ quality_2d = None
+ hiQ = False
+
+ if f5.has_2D():
+ twoD = f5.get_fastqs("2D")[0]
+ sequence_2d = twoD.seq
+ quality_2d = twoD.qual
+ hiQ = f5.is_high_quality()
+
+ fwd = f5.get_fastqs("fwd")[0]
+ rev = f5.get_fastqs("rev")[0]
+ if fwd != None or rev != None:
+ return FastQData(fname
+ , array.array('I', [ len(fwd.seq) if fwd != None else 0, len(rev.seq) if rev != None else 0 ])
+ , (fwd.seq if fwd != None else '') + (rev.seq if rev != None else '')
+ , (fwd.qual if fwd != None else '') + (rev.qual if rev != None else '')
+ , sequence_2d, quality_2d
+ , channel, readno, hiQ)
+
+ return None
+ except:
+ errMsg = "pore-tools reported an unspecific error while reading '{}'".format(os.path.basename(fname))
+ gw.errorMessage(errMsg)
+ sys.stderr.write(errMsg+"\n")
+ if __debug__:
+ traceback.print_exc()
+ return None
+ finally:
+ f5.close()
+
+
+def ProcessFast5(fname):
+ """ Read the FASTQ data from the fast5 file
+
+ Write it to the General Writer
+ """
+ data = FastQData.ReadFast5Data(fname)
+ if data:
+ data.write()
+ return True
+ else:
+ return False
+
+
+def isHDF5(fname):
+ """ Try to open it using h5py
+ If it works, assume it's an HDF5 file
+
+ This is needed because sometimes the .fast5 files
+ are actually HTML containing an error message
+ """
+ try:
+ with h5py.File(fname, 'r'):
+ return True
+ except IOError:
+ return False
+
+
+notRemoved = [] # because of errors
+
+def ExtractAndProcess(f, source):
+ """ Extract file to the working directory and process it
+ """
+ fname = os.path.join(workDir, source)
+ with open(fname, 'wb') as output:
+ output.write(f.read())
+
+ keep = False
+ if not isHDF5(fname):
+ errMsg = "Warning: skipping '{}': not an HDF5 file.".format(source)
+ gw.errorMessage(errMsg)
+ sys.stderr.write(errMsg+"\n")
+ elif not ProcessFast5(fname):
+ keep = __debug__
+
+ if keep:
+ notRemoved.append(source)
+ else:
+ os.remove(fname)
+
+
+processCounter = 0
+processStart = time.clock()
+nextReport = (processStart + showProgress) if showProgress > 0 else None
+
+def ProcessTar(tar):
+ """ Extract and process all fast5 files
+ """
+ global processCounter
+ global nextReport
+
+ for f in tar:
+ if f.name.endswith('.fast5'):
+ i = tar.extractfile(f)
+ try:
+ ExtractAndProcess(i, os.path.basename(f.name))
+ finally:
+ i.close()
+
+ processCounter = processCounter + 1
+ now = time.clock()
+ if nextReport and now >= nextReport:
+ nextReport = nextReport + showProgress
+ sys.stderr.write("Progress: processed {} spots; {} per sec.\n".
+ format(processCounter, processCounter/(now - processStart)))
+
+ elapsed = time.clock() - processStart
+ sys.stderr.write("Progress: processed {} spots in {} secs, {} per sec.\n".
+ format(processCounter, elapsed, processCounter/elapsed))
+
+
+def which(f):
+ PATH = os.environ["PATH"].split(":")
+ for fullname in map((lambda p: os.path.join(p, f)), PATH):
+ if os.path.exists(fullname):
+ return fullname
+ return None
+
+
+def main():
+ # check for gzcat or zcat
+
+ gzcat = which('gzcat')
+ if gzcat == None:
+ gzcat = which('zcat') # gnu-ish
+ if gzcat and not os.path.exists(gzcat):
+ gzcat = None
+
+ if len(files) == 0:
+ sys.stderr.write("Info: processing stdin\n")
+ with tarfile.open(mode='r|', fileobj=sys.stdin) as tar:
+ ProcessTar(tar)
+
+ for f in files:
+ sys.stderr.write("Info: processing {}\n".format(f))
+ if gzcat == None:
+ with tarfile.open(f) as tar:
+ ProcessTar(tar)
+ else:
+ p = subprocess.Popen([gzcat, f], stdout=subprocess.PIPE)
+ with tarfile.open(mode='r|', fileobj=p.stdout) as tar:
+ ProcessTar(tar)
+ p.stdout.close()
+ p.wait()
+
+
+def cleanup():
+ rmd = removeWorkDir
+ if len(notRemoved) != 0:
+ rmd = False
+ sys.stderr.write("Info: these files caused errors and were not removed "
+ "from the work space to aid in debugging:\n"
+ )
+ for f in notRemoved:
+ sys.stderr.write("Info:\t{}\n".format(f))
+
+ if rmd:
+ os.rmdir(workDir)
+
+
+main()
+gw = None # close stream and flush
+cleanup()
diff --git a/tools/prefetch/Makefile b/tools/prefetch/Makefile
index 301e9ee..1aba9ac 100644
--- a/tools/prefetch/Makefile
+++ b/tools/prefetch/Makefile
@@ -85,7 +85,8 @@ clean: stdclean
# prefetch
#
PREFETCH_SRC = \
- prefetch
+ prefetch \
+ kfile-no-q
PREFETCH_OBJ = \
$(addsuffix .$(OBJX),$(PREFETCH_SRC))
diff --git a/tools/prefetch/kfile-no-q.c b/tools/prefetch/kfile-no-q.c
new file mode 100644
index 0000000..073220a
--- /dev/null
+++ b/tools/prefetch/kfile-no-q.c
@@ -0,0 +1,215 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "kfile-no-q.h"
+
+#include <kapp/main.h>
+
+#include <klib/log.h>
+#include <klib/status.h>
+
+#include <kfs/file.h>
+#include <kfs/directory.h>
+#include <kfs/arc.h>
+#include <kfs/sra.h>
+
+#include <assert.h>
+#include <string.h>
+
+#define READ_CACHE_BLOCK_SIZE (1024*1024)
+
+#ifndef PATH_MAX
+#define PATH_MAX 4096
+#endif
+
+#define STS_FIN 3
+
+typedef rc_t ( * FileProcessFn ) ( const KDirectory_v1 * dir, const char * name );
+
+typedef struct {
+ FileProcessFn processFn;
+
+ bool elimQuals;
+} VisitParams;
+
+static
+bool qual_col_filter ( const KDirectory * self, const char * path, void * data )
+{
+ if ( strstr(path, "col/QUALITY") == NULL )
+ {
+ return true;
+ }
+
+ return false;
+}
+
+static
+rc_t sort_none (const KDirectory * self, struct Vector * v)
+{
+ return 0;
+}
+
+static
+rc_t visit_cb ( const KDirectory * self, uint32_t type, const char * name, void * data )
+{
+ rc_t rc = 0;
+ VisitParams * params = data;
+
+ assert(params);
+
+ if (params->processFn == NULL)
+ return rc;
+
+ if (type == kptFile)
+ {
+ if (params->elimQuals)
+ {
+ char path[PATH_MAX];
+ rc = KDirectoryResolvePath ( self, true, path, sizeof path, name );
+
+ if (rc == 0 && ( strstr(path, "/col/QUALITY/") == NULL ) )
+ {
+ rc = params->processFn(self, name);
+ }
+ }
+ else
+ {
+ rc = params->processFn(self, name);
+ }
+ }
+
+ return rc;
+}
+
+static
+rc_t file_read ( const KDirectory * dir, const char * name )
+{
+ rc_t rc;
+ const KFile * kfile;
+ char buffer[READ_CACHE_BLOCK_SIZE];
+ uint64_t pos = 0;
+ uint64_t prevPos = 0;
+ size_t num_read;
+
+
+ rc = KDirectoryOpenFileRead ( dir, &kfile, name );
+ if (rc != 0)
+ return rc;
+
+ do {
+ bool print = pos - prevPos > 200000000;
+ rc = Quitting();
+
+ if (rc == 0) {
+ if (print) {
+ STSMSG(STS_FIN,
+ ("Reading %lu bytes from pos. %lu", READ_CACHE_BLOCK_SIZE, pos));
+ }
+ rc = KFileRead(kfile,
+ pos, buffer, READ_CACHE_BLOCK_SIZE, &num_read);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc, "KFileRead failed for file $(name)",
+ "name=%S", name));
+ }
+ else {
+ pos += num_read;
+ }
+
+ if (print) {
+ prevPos = pos;
+ }
+ }
+ } while (rc == 0 && num_read > 0);
+
+ KFileRelease(kfile);
+
+ return rc;
+}
+
+rc_t CC KSraFileNoQuals( const struct KFile * self,
+ const struct KFile ** kfile )
+{
+ rc_t rc;
+ struct KDirectory * kdir_native, * kdir_virtual;
+ const struct KFile * new_kfile;
+
+ assert(self != NULL);
+ assert(kfile != NULL);
+
+ rc = KDirectoryNativeDir(&kdir_native);
+ if (rc != 0)
+ {
+ return rc;
+ }
+
+ rc = KDirectoryOpenArcDirRead_silent_preopened ( kdir_native, (const struct KDirectory **)&kdir_virtual, false, "/virtual", tocKFile,
+ ( void* ) self, KArcParseSRA, NULL, NULL );
+ if (rc == 0)
+ {
+ rc = KDirectoryOpenTocFileRead (kdir_virtual, &new_kfile, sraAlign4Byte, qual_col_filter, NULL, sort_none );
+ KDirectoryRelease (kdir_virtual);
+
+ if (rc == 0)
+ {
+ *kfile = new_kfile;
+ }
+ }
+
+ KDirectoryRelease (kdir_native);
+
+ return 0;
+}
+
+rc_t CC KSraReadCacheFile( const struct KFile * self, bool elimQuals )
+{
+ rc_t rc;
+ struct KDirectory * kdir_native, * kdir_virtual;
+
+
+ assert(self != NULL);
+
+ rc = KDirectoryNativeDir(&kdir_native);
+ if (rc != 0)
+ {
+ return rc;
+ }
+
+ rc = KDirectoryOpenArcDirRead_silent_preopened ( kdir_native, (const struct KDirectory **)&kdir_virtual, false, "/virtual", tocKFile,
+ ( void* ) self, KArcParseSRA, NULL, NULL );
+ if (rc == 0)
+ {
+ VisitParams params;
+ params.elimQuals = elimQuals;
+ params.processFn = &file_read;
+
+ rc = KDirectoryVisit(kdir_virtual, true, visit_cb, ¶ms, ".");
+ KDirectoryRelease (kdir_virtual);
+ }
+
+ KDirectoryRelease (kdir_native);
+
+ return 0;
+}
diff --git a/tools/bam-loader/loader-imp3.h b/tools/prefetch/kfile-no-q.h
similarity index 76%
rename from tools/bam-loader/loader-imp3.h
rename to tools/prefetch/kfile-no-q.h
index 9d2926d..c123720 100644
--- a/tools/bam-loader/loader-imp3.h
+++ b/tools/prefetch/kfile-no-q.h
@@ -23,16 +23,22 @@
* ===========================================================================
*
*/
-#ifndef _loader_imp3_h_
-#define _loader_imp3_h_
-struct CommonWriterSettings;
+#ifndef _h_kfile_no_q_
+#define _h_kfile_no_q_
-rc_t run(char const argv0[],
- struct CommonWriterSettings* G,
- unsigned countAligned,
- char const *bamFile[],
- unsigned countUnaligned,
- char const *unaligned[]);
+#include <klib/rc.h>
- #endif
+/*--------------------------------------------------------------------------
+ * forwards
+ */
+struct KFile;
+
+
+rc_t CC KSraFileNoQuals( const struct KFile * self,
+ const struct KFile ** kfile );
+
+rc_t CC KSraReadCacheFile( const struct KFile * self, bool elimQuals );
+
+
+#endif /* _h_kfile_no_q_ */
diff --git a/tools/prefetch/prefetch.c b/tools/prefetch/prefetch.c
index 5219a27..d9818f4 100644
--- a/tools/prefetch/prefetch.c
+++ b/tools/prefetch/prefetch.c
@@ -30,6 +30,8 @@
#include <kapp/main.h> /* KAppVersion */
+#include <kdb/manager.h> /* kptDatabase */
+
#include <kfg/config.h> /* KConfig */
#include <kfg/kart.h> /* Kart */
#include <kfg/repository.h> /* KRepositoryMgr */
@@ -37,8 +39,7 @@
#include <vdb/database.h> /* VDatabase */
#include <vdb/dependencies.h> /* VDBDependencies */
#include <vdb/manager.h> /* VDBManager */
-
-#include <kdb/manager.h> /* kptDatabase */
+#include <vdb/vdb-priv.h> /* VDatabaseIsCSRA */
#include <vfs/manager.h> /* VFSManager */
#include <vfs/path.h> /* VPath */
@@ -52,6 +53,7 @@
#include <kfs/file.h> /* KFile */
#include <kfs/gzip.h> /* KFileMakeGzipForRead */
#include <kfs/subfile.h> /* KFileMakeSubRead */
+#include <kfs/cacheteefile.h> /* KDirectoryMakeCacheTee */
#include <klib/container.h> /* BSTree */
#include <klib/data-buffer.h> /* KDataBuffer */
@@ -72,6 +74,8 @@
#include <stdio.h> /* printf */
+#include "kfile-no-q.h"
+
#define DISP_RC(rc, err) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, err))
#define DISP_RC2(rc, name, msg) (void)((rc == 0) ? 0 : \
@@ -90,6 +94,7 @@
#define STS_FIN 3
#define USE_CURL 0
+#define ALLOW_STRIP_QUALS 0
#define rcResolver rcTree
static bool NotFoundByResolver(rc_t rc) {
@@ -134,7 +139,7 @@ typedef struct {
VPathStr local;
const String *cache;
- const String *remote;
+ VPathStr remote;
const KFile *file;
uint64_t remoteSz;
@@ -142,11 +147,10 @@ typedef struct {
bool undersized; /* remoteSz < min allowed size */
bool oversized; /* remoteSz >= max allowed size */
- bool existing;
+ bool existing; /* the path is a path to an existing local file */
- /* path to the resolved object : either local or cache:
- should not be released */
- const char *path;
+ /* path to the resolved object : absolute path or resolved(local or cache) */
+ VPathStr path;
VPath *accession;
uint64_t project;
@@ -198,7 +202,11 @@ typedef struct {
const char *ascp;
const char *asperaKey;
String *ascpMaxRate;
+ const char *ascpParams; /* do not free! */
+ bool stripQuals; /* this will download file without quality columns */
+ bool eliminateQuals; /* this will download cache file with eliminated quality columns which could filled later */
+
#ifdef _DEBUGGING
const char *textkart;
#endif
@@ -215,6 +223,8 @@ typedef struct {
Resolved resolved;
int number;
+
+ bool isDependency;
Main *main; /* just a pointer, no refcount here, don't release it */
} Item;
@@ -291,7 +301,7 @@ static rc_t _KDirectoryMkTmpPrefix(const KDirectory *self,
if (num_writ > sz) {
rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
PLOGERR(klogInt, (klogInt, rc,
- "bad string_printf($(s).tmp) result", "s=%s", prefix->addr));
+ "bad string_printf($(s).tmp) result", "s=%S", prefix));
return rc;
}
@@ -318,8 +328,7 @@ static rc_t _KDirectoryMkTmpName(const KDirectory *self,
if (num_writ > sz) {
rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
PLOGERR(klogInt, (klogInt, rc,
- "bad string_printf($(s).tmp.rand) result",
- "s=%s", prefix->addr));
+ "bad string_printf($(s).tmp.rand) result", "s=%S", prefix));
return rc;
}
if (KDirectoryPathType(self, "%s", out) == kptNotFound) {
@@ -328,8 +337,8 @@ static rc_t _KDirectoryMkTmpName(const KDirectory *self,
if (++i > 999) {
rc = RC(rcExe, rcFile, rcCopying, rcName, rcUndefined);
PLOGERR(klogInt, (klogInt, rc,
- "cannot generate unique tmp file name for $(name)", "name=%s",
- prefix->addr));
+ "cannot generate unique tmp file name for $(name)", "name=%S",
+ prefix));
return rc;
}
}
@@ -351,7 +360,7 @@ static rc_t _KDirectoryMkLockName(const KDirectory *self,
if (rc == 0 && num_writ > sz) {
rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
PLOGERR(klogInt, (klogInt, rc,
- "bad string_printf($(s).lock) result", "s=%s", prefix->addr));
+ "bad string_printf($(s).lock) result", "s=%S", prefix));
return rc;
}
@@ -369,8 +378,7 @@ rc_t _KDirectoryCleanCache(KDirectory *self, const String *local)
assert(self && local && local->addr);
if (rc == 0) {
- rc = string_printf(cache, sizeof cache, &num_writ, "%s.cache",
- local->addr);
+ rc = string_printf(cache, sizeof cache, &num_writ, "%S.cache", local);
DISP_RC2(rc, "string_printf(.cache)", local->addr);
}
@@ -427,10 +435,11 @@ static rc_t _KDirectoryClean(KDirectory *self, const String *cache,
}
}
+ assert(cache);
if (rmSelf && KDirectoryPathType(self, "%s", cache->addr) != kptNotFound) {
rc_t rc3 = 0;
- STSMSG(STS_DBG, ("removing %s", cache->addr));
- rc3 = KDirectoryRemove(self, false, "%s", cache->addr);
+ STSMSG(STS_DBG, ("removing %S", cache));
+ rc3 = KDirectoryRemove(self, false, "%S", cache);
if (rc2 == 0 && rc3 != 0) {
rc2 = rc3;
}
@@ -508,7 +517,10 @@ static rc_t _VResolverRemote(VResolver *self, VRemoteProtocols protocols,
{
rc_t rc = 0;
const VPath *vcache = NULL;
- assert(vaccession && vremote && !*vremote);
+ assert(vaccession && vremote);
+ if (*vremote != NULL) {
+ RELEASE(VPath, *vremote);
+ }
rc = V_ResolverRemote(self, protocols, vaccession, vremote, &vcache);
if (rc == 0) {
char path[PATH_MAX] = "";
@@ -547,6 +559,9 @@ static rc_t _VResolverRemote(VResolver *self, VRemoteProtocols protocols,
DISP_RC2(rc, "VPathGetPath(VResolverCache)", name);
}
if (rc == 0) {
+ if (*cache != NULL) {
+ free((void*)*cache);
+ }
rc = StringCopy(cache, &path_str);
DISP_RC2(rc, "StringCopy(VResolverCache)", name);
}
@@ -570,8 +585,21 @@ static rc_t VPathStrFini(VPathStr *self) {
return rc;
}
+static
+rc_t VPathStrInitStr(VPathStr *self, const char *str, size_t len)
+{
+ String s;
+ assert(self);
+ if (len == 0) {
+ len = string_size(str);
+ }
+ StringInit(&s, str, len, (uint32_t)len);
+ VPathStrFini(self);
+ return StringCopy(&self->str, &s);
+}
+
/********** TreeNode **********/
-static int CC bstCmp(const void *item, const BSTNode *n) {
+static int64_t CC bstCmp(const void *item, const BSTNode *n) {
const char* path = item;
const TreeNode* sn = (const TreeNode*) n;
@@ -580,7 +608,7 @@ static int CC bstCmp(const void *item, const BSTNode *n) {
return strcmp(path, sn->path);
}
-static int CC bstSort(const BSTNode* item, const BSTNode* n) {
+static int64_t CC bstSort(const BSTNode* item, const BSTNode* n) {
const TreeNode* sn = (const TreeNode*) item;
return bstCmp(sn->path, n);
@@ -747,10 +775,16 @@ static bool NumIteratorNext(NumIterator *self, int64_t crnt) {
/********** Resolved **********/
static rc_t ResolvedFini(Resolved *self) {
rc_t rc = 0;
+ rc_t rc2 = 0;
assert(self);
- rc = VPathStrFini(&self->local);
+ rc = VPathStrFini(&self->local);
+ rc2 = VPathStrFini(&self->remote);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ rc2 = VPathStrFini(&self->path);
RELEASE(KFile, self->file);
RELEASE(VPath, self->accession);
@@ -758,14 +792,13 @@ static rc_t ResolvedFini(Resolved *self) {
RELEASE(KartItem, self->kartItem);
- RELEASE(String, self->remote);
RELEASE(String, self->cache);
free(self->name);
memset(self, 0, sizeof *self);
- return rc;
+ return rc != 0 ? rc : rc2;
}
static void ResolvedReset(Resolved *self, ERunType type) {
@@ -798,7 +831,8 @@ static rc_t ResolvedLocal(const Resolved *self,
rc = VPathReadPath(self->local.path, path, sizeof path, NULL);
DISP_RC(rc, "VPathReadPath");
- if (rc == 0 && KDirectoryPathType(dir, "%s", path) != kptFile) {
+ if (rc == 0 && (KDirectoryPathType(dir, "%s", path) & ~kptAlias) != kptFile)
+ {
if (force == eForceNo) {
STSMSG(STS_TOP,
("%s (not a file) is found locally: consider it complete",
@@ -814,7 +848,7 @@ static rc_t ResolvedLocal(const Resolved *self,
}
if (rc == 0) {
- if (! _StringIsFasp(self->remote, NULL) && self->file != NULL) {
+ if (! _StringIsFasp(self->remote.str, NULL) && self->file != NULL) {
rc = KFileSize(self->file, &sRemote);
DISP_RC2(rc, "KFileSize(remote)", self->name);
}
@@ -942,6 +976,7 @@ static rc_t MainDownloadFile(Resolved *self,
uint64_t prevPos = 0;
assert(self && main);
+ assert(!main->eliminateQuals);
if (rc == 0) {
STSMSG(STS_DBG, ("creating %s", to));
@@ -950,17 +985,29 @@ static rc_t MainDownloadFile(Resolved *self,
DISP_RC2(rc, "Cannot OpenFileWrite", to);
}
- assert(self->remote);
+ assert(self->remote.str);
if (self->file == NULL) {
- rc = _KFileOpenRemote(&self->file, main->kns, self->remote->addr);
+ rc = _KFileOpenRemote(&self->file, main->kns, self->remote.str->addr);
if (rc != 0) {
PLOGERR(klogInt, (klogInt, rc, "failed to open file for $(path)",
- "path=%s", self->remote->addr));
+ "path=%S", self->remote.str));
}
}
- STSMSG(STS_INFO, ("%s -> %s", self->remote->addr, to));
+ if (main->stripQuals)
+ {
+ const KFile * kfile;
+
+ rc = KSraFileNoQuals(self->file, &kfile);
+ if (rc == 0)
+ {
+ KFileRelease(self->file);
+ self->file = kfile;
+ }
+ }
+
+ STSMSG(STS_INFO, ("%S -> %s", self->remote.str, to));
do {
bool print = pos - prevPos > 200000000;
rc = Quitting();
@@ -973,7 +1020,7 @@ static rc_t MainDownloadFile(Resolved *self,
rc = KFileRead(self->file,
pos, main->buffer, main->bsize, &num_read);
if (rc != 0) {
- DISP_RC2(rc, "Cannot KFileRead", self->remote->addr);
+ DISP_RC2(rc, "Cannot KFileRead", self->remote.str->addr);
}
else {
pos += num_read;
@@ -1000,6 +1047,54 @@ static rc_t MainDownloadFile(Resolved *self,
return rc;
}
+static rc_t MainDownloadCacheFile(Resolved *self,
+ Main *main, const char *to, bool elimQuals)
+{
+ rc_t rc = 0;
+ const KFile *out = NULL;
+
+ assert(self && main);
+ assert(!main->stripQuals);
+
+ assert(self->remote.str);
+
+ if (self->file == ((void*)0)) {
+ rc = _KFileOpenRemote(&self->file, main->kns, self->remote.str->addr);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc, "failed to open file for $(path)",
+ "path=%S", self->remote.str));
+ return rc;
+ }
+ }
+
+ rc = KDirectoryMakeCacheTee(main->dir, &out, self->file, 0, "%s", to);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc, "failed to open cache file for $(path)",
+ "path=%S", to));
+ return rc;
+ }
+
+ STSMSG(STS_INFO, ("%S -> %s", self->remote.str, to));
+
+ rc = KSraReadCacheFile( out, elimQuals );
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc, "failed to read cache file at $(path)",
+ "path=%S", to));
+ }
+
+ RELEASE(KFile, out);
+
+ if (rc != 0) {
+ return rc;
+ }
+
+ if (rc == 0) {
+ STSMSG(STS_INFO, ("%s", to));
+ }
+
+ return rc;
+}
+
/* http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByR.../SRR125365.sra
anonftp at ftp-private.ncbi.nlm.nih.gov:/sra/sra-instant/reads/ByR.../SRR125365.sra
*/
@@ -1008,22 +1103,36 @@ static rc_t MainDownloadAscp(const Resolved *self, Main *main,
{
const char *src = NULL;
AscpOptions opt;
- assert(self && self->remote && self->remote->addr
+
+ assert(self && self->remote.str && self->remote.str->addr
&& main && main->ascp && main->asperaKey);
+
memset(&opt, 0, sizeof opt);
- if (!_StringIsFasp(self->remote, &src)) {
+
+ if (!_StringIsFasp(self->remote.str, &src)) {
return RC(rcExe, rcFile, rcCopying, rcSchema, rcInvalid);
}
- opt.target_rate
- = main->ascpMaxRate == NULL ? NULL : main->ascpMaxRate->addr;
+
+ if (main->ascpParams != NULL) {
+ opt.ascp_options = main->ascpParams;
+ }
+ else if (main->ascpMaxRate != NULL) {
+ size_t sz = string_copy(opt.target_rate, sizeof opt.target_rate,
+ main->ascpMaxRate->addr, main->ascpMaxRate->size);
+ if (sz < sizeof opt.target_rate) {
+ return RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
+ }
+ }
+
opt.name = self->name;
opt.src_size = self->remoteSz;
opt.heartbeat = main->heartbeat;
opt.quitting = Quitting;
+
return aspera_get(main->ascp, main->asperaKey, src, to, &opt);
}
-static rc_t MainDownload(Resolved *self, Main *main) {
+static rc_t MainDownload(Resolved *self, Main *main, bool isDependency) {
bool canceled = false;
rc_t rc = 0;
KFile *flock = NULL;
@@ -1045,7 +1154,7 @@ static rc_t MainDownload(Resolved *self, Main *main) {
rc = _KDirectoryMkLockName(main->dir, self->cache, lock, sizeof lock);
}
- if (rc == 0) {
+ if (rc == 0 && !main->eliminateQuals) {
rc = _KDirectoryMkTmpName(main->dir, self->cache, tmp, sizeof tmp);
}
@@ -1086,19 +1195,27 @@ static rc_t MainDownload(Resolved *self, Main *main) {
if (rc == 0) {
STSMSG(STS_DBG, ("creating %s", lock));
rc = KDirectoryCreateFile(main->dir, &flock,
- false, 0664, kcmInit | kcmParents, "%s", lock);
+ false, 0664, kcmInit | kcmParents, "%s", lock);
DISP_RC2(rc, "Cannot OpenFileWrite", lock);
}
assert(!main->noAscp || !main->noHttp);
if (rc == 0) {
- bool ascp = _StringIsFasp(self->remote, NULL);
+ bool ascp = _StringIsFasp(self->remote.str, NULL);
if (ascp) {
STSMSG(STS_TOP, (" Downloading via fasp..."));
if (main->forceAscpFail) {
rc = 1;
}
+ else if (main->eliminateQuals) {
+ LOGMSG(klogErr, "Cannot eliminate qualities during fasp download");
+ rc = 1;
+ }
+ else if (main->eliminateQuals) {
+ LOGMSG(klogErr, "Cannot remove QUALITY columns during fasp download");
+ rc = 1;
+ }
else {
rc = MainDownloadAscp(self, main, tmp);
}
@@ -1118,30 +1235,41 @@ static rc_t MainDownload(Resolved *self, Main *main) {
if (!ascp || (rc != 0 && GetRCObject(rc) != rcMemory
&& !canceled && !main->noHttp))
{
- const VPath *vremote = NULL;
STSMSG(STS_TOP, (" Downloading via http..."));
if (ascp) {
assert(self->resolver);
- RELEASE(String, self->remote);
+ {
+ rc_t rc2 = VPathStrFini(&self->remote);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
RELEASE(KFile, self->file);
- rc = _VResolverRemote(self->resolver, eProtocolHttp, self->name,
- self->accession, &vremote, &self->remote, &self->cache);
+ rc = _VResolverRemote(self->resolver,
+ eProtocolHttp, self->name, self->accession,
+ &self->remote.path, &self->remote.str, &self->cache);
}
if (rc == 0) {
- rc = MainDownloadFile(self, main, tmp);
+ /* when eliminateQuals is specified we will try newer algorithm for downloading files via cache,
+ but filter out qualities for main file, not for dependencies */
+ if (main->eliminateQuals) {
+ rc = MainDownloadCacheFile(self, main, self->cache->addr, main->eliminateQuals && !isDependency);
+ }
+ else {
+ rc = MainDownloadFile(self, main, tmp);
+ }
}
- RELEASE(VPath, vremote);
}
}
RELEASE(KFile, flock);
-
- if (rc == 0) {
- STSMSG(STS_DBG, ("renaming %s -> %s", tmp, self->cache->addr));
+
+ if (rc == 0 && !main->eliminateQuals) {
+ STSMSG(STS_DBG, ("renaming %s -> %S", tmp, self->cache));
rc = KDirectoryRename(main->dir, true, tmp, self->cache->addr);
if (rc != 0) {
PLOGERR(klogInt, (klogInt, rc, "cannot rename $(from) to $(to)",
- "from=%s,to=%s", tmp, self->cache->addr));
+ "from=%s,to=%S", tmp, self->cache));
}
}
@@ -1149,7 +1277,7 @@ static rc_t MainDownload(Resolved *self, Main *main) {
rc = MainDownloaded(main, self->cache->addr);
}
- if (rc == 0) {
+ if (rc == 0 && !main->eliminateQuals) {
rc_t rc2 = _KDirectoryCleanCache(main->dir, self->cache);
if (rc == 0 && rc2 != 0) {
rc = rc2;
@@ -1157,7 +1285,7 @@ static rc_t MainDownload(Resolved *self, Main *main) {
}
{
- rc_t rc2 = _KDirectoryClean(main->dir, self->cache, lock, tmp, rc != 0);
+ rc_t rc2 = _KDirectoryClean(main->dir, self->cache, lock, main->eliminateQuals ? NULL : tmp, rc != 0);
if (rc == 0 && rc2 != 0) {
rc = rc2;
}
@@ -1166,20 +1294,49 @@ static rc_t MainDownload(Resolved *self, Main *main) {
return rc;
}
+static rc_t _VDBManagerSetDbGapCtx(const VDBManager *self, VResolver *resolver)
+{
+ if (resolver == NULL) {
+ return 0;
+ }
+
+ return VDBManagerSetResolver(self, resolver);
+}
+
static rc_t MainDependenciesList(const Main *self,
- const char *path, const VDBDependencies **deps)
+ const Resolved *resolved, const VDBDependencies **deps)
{
rc_t rc = 0;
bool isDb = true;
const VDatabase *db = NULL;
+ const char *path = NULL;
+ const String *str = NULL;
+ KPathType type = kptNotFound;
+
+ assert(self && resolved && deps);
- assert(self && path && deps);
+ str = resolved->path.str;
+ assert(str && str->addr);
- if ((VDBManagerPathType(self->mgr, "%s", path) & ~kptAlias) != kptDatabase) {
+ path = str->addr;
+
+ rc = _VDBManagerSetDbGapCtx(self->mgr, resolved->resolver);
+
+ STSMSG(STS_DBG, ("Listing '%S's dependencies...", str));
+
+ type = VDBManagerPathType(self->mgr, "%s", resolved->name) & ~kptAlias;
+ if (type != kptDatabase) {
+ if (type == kptTable) {
+ STSMSG(STS_DBG, ("...'%S' is a table", str));
+ }
+ else {
+ STSMSG(STS_DBG,
+ ("...'%S' is not recognized as a database or a table", str));
+ }
return 0;
}
- rc = VDBManagerOpenDBRead(self->mgr, &db, NULL, "%s", path);
+ rc = VDBManagerOpenDBRead(self->mgr, &db, NULL, "%s", resolved->name);
if (rc != 0) {
if (rc == SILENT_RC(rcDB, rcMgr, rcOpening, rcDatabase, rcIncorrect)) {
isDb = false;
@@ -1188,17 +1345,17 @@ static rc_t MainDependenciesList(const Main *self,
else if (rc ==
SILENT_RC(rcKFG, rcEncryptionKey, rcRetrieving, rcItem, rcNotFound))
{
- STSMSG(STS_TOP, ("Cannot open encrypted file '%s'", path));
+ STSMSG(STS_TOP, ("Cannot open encrypted file '%s'", resolved->name));
isDb = false;
rc = 0;
}
- DISP_RC2(rc, "Cannot open database", path);
+ DISP_RC2(rc, "Cannot open database", resolved->name);
}
if (rc == 0 && isDb) {
bool all = self->check_all || self->force != eForceNo;
rc = VDatabaseListDependencies(db, deps, !all);
- DISP_RC2(rc, "VDatabaseListDependencies", path);
+ DISP_RC2(rc, "VDatabaseListDependencies", resolved->name);
}
RELEASE(VDatabase, db);
@@ -1210,13 +1367,17 @@ static rc_t MainDependenciesList(const Main *self,
static rc_t ItemRelease(Item *self) {
rc_t rc = 0;
- assert(self);
+ if (self == NULL) {
+ return 0;
+ }
rc = ResolvedFini(&self->resolved);
RELEASE(KartItem, self->item);
memset(self, 0, sizeof *self);
+ free(self);
+
return rc;
}
@@ -1254,6 +1415,29 @@ static char* ItemName(const Item *self) {
}
}
+static
+rc_t _KartItemToVPath(const KartItem *self, const VFSManager *vfs, VPath **path)
+{
+ uint64_t oid = 0;
+ rc_t rc = KartItemItemIdNumber(self, &oid);
+ if (rc == 0) {
+ rc = VFSManagerMakeOidPath(vfs, path, (uint32_t)oid);
+ }
+ else {
+ char path_str[PATH_MAX] = "";
+ const String *accession = NULL;
+ rc = KartItemAccession(self, &accession);
+ if (rc == 0) {
+ rc =
+ string_printf(path_str, sizeof path_str, NULL, "%S", accession);
+ }
+ if (rc == 0) {
+ rc = VFSManagerMakePath(vfs, path, path_str);
+ }
+ }
+ return rc;
+}
+
static rc_t _ItemSetResolverAndAssessionInResolved(Item *item,
VResolver *resolver, const KConfig *cfg, const KRepositoryMgr *repoMgr,
const VFSManager *vfs)
@@ -1276,15 +1460,14 @@ static rc_t _ItemSetResolverAndAssessionInResolved(Item *item,
}
}
else {
- uint64_t oid = 0;
rc = KartItemProjIdNumber(item->item, &resolved->project);
if (rc != 0) {
DISP_RC(rc, "KartItemProjIdNumber");
return rc;
}
- rc = KartItemItemIdNumber(item->item, &oid);
+ rc = _KartItemToVPath(item->item, vfs, &resolved->accession);
if (rc != 0) {
- DISP_RC(rc, "KartItemItemIdNumber");
+ DISP_RC(rc, "invalid kart file row");
return rc;
}
else {
@@ -1298,6 +1481,9 @@ static rc_t _ItemSetResolverAndAssessionInResolved(Item *item,
DISP_RC(rc, "KRepositoryMakeResolver");
return rc;
}
+ else {
+ VResolverCacheEnable(resolved->resolver, vrAlwaysEnable);
+ }
}
else {
PLOGERR(klogErr, (klogErr, rc,
@@ -1306,14 +1492,6 @@ static rc_t _ItemSetResolverAndAssessionInResolved(Item *item,
}
RELEASE(KRepository, p_protected);
}
- if (rc == 0) {
- rc =
- VFSManagerMakeOidPath(vfs, &resolved->accession, (uint32_t)oid);
- if (rc != 0) {
- DISP_RC(rc, "VFSManagerMakeOidPath");
- return rc;
- }
- }
}
return rc;
@@ -1321,20 +1499,20 @@ static rc_t _ItemSetResolverAndAssessionInResolved(Item *item,
/* resolve locations */
static rc_t _ItemResolveResolved(VResolver *resolver,
- VRemoteProtocols protocols, Item *item, const KRepositoryMgr *repoMgr,
- const KConfig *cfg, const VFSManager *vfs, KNSManager *kns, size_t minSize, size_t maxSize)
+ VRemoteProtocols protocols, Item *item,
+ const KRepositoryMgr *repoMgr, const KConfig *cfg,
+ const VFSManager *vfs, KNSManager *kns, size_t minSize, size_t maxSize)
{
Resolved *resolved = NULL;
rc_t rc = 0;
rc_t rc2 = 0;
- const VPath *vremote = NULL;
-
assert(resolver && item);
resolved = &item->resolved;
- memset(&resolved->local, 0, sizeof resolved->local);
+ VPathStrFini(&resolved->local);
+ VPathStrFini(&resolved->remote);
assert(resolved->accession == NULL);
@@ -1365,8 +1543,8 @@ static rc_t _ItemResolveResolved(VResolver *resolver,
protocols == eProtocolFaspHttp))
{
rc2 = _VResolverRemote(resolved->resolver, eProtocolHttp,
- resolved->name, resolved->accession,
- &vremote, &resolved->remote, NULL);
+ resolved->name, resolved->accession, &resolved->remote.path,
+ &resolved->remote.str, &resolved->cache);
if (rc2 != 0 && rc == 0) {
rc = rc2;
}
@@ -1374,18 +1552,16 @@ static rc_t _ItemResolveResolved(VResolver *resolver,
rc_t rc3 = 0;
if (resolved->file == NULL) {
rc3 = _KFileOpenRemote(&resolved->file,
- kns, resolved->remote->addr);
+ kns, resolved->remote.str->addr);
DISP_RC2(rc3,
- "cannot open remote file", resolved->remote->addr);
+ "cannot open remote file", resolved->remote.str->addr);
}
- RELEASE(VPath, vremote);
-
if (rc3 == 0 && resolved->file != NULL) {
rc3 = KFileSize(resolved->file, &resolved->remoteSz);
if (rc3 != 0) {
DISP_RC2(rc3, "cannot get remote file size",
- resolved->remote->addr);
+ resolved->remote.str->addr);
}
else if (resolved->remoteSz >= maxSize) {
return rc;
@@ -1399,8 +1575,8 @@ static rc_t _ItemResolveResolved(VResolver *resolver,
if (rc2 == 0) {
rc2 = _VResolverRemote(resolved->resolver, protocols,
- resolved->name, resolved->accession,
- &vremote, &resolved->remote, &resolved->cache);
+ resolved->name, resolved->accession, &resolved->remote.path,
+ &resolved->remote.str, &resolved->cache);
if (rc2 != 0 && rc == 0) {
rc = rc2;
}
@@ -1409,10 +1585,10 @@ static rc_t _ItemResolveResolved(VResolver *resolver,
if (rc == 0) {
rc2 = 0;
if (resolved->file == NULL) {
- assert(resolved->remote);
- if (!_StringIsFasp(resolved->remote, NULL)) {
+ assert(resolved->remote.str);
+ if (!_StringIsFasp(resolved->remote.str, NULL)) {
rc2 = _KFileOpenRemote(
- &resolved->file, kns, resolved->remote->addr);
+ &resolved->file, kns, resolved->remote.str->addr);
}
}
if (rc2 == 0 && resolved->file != NULL && resolved->remoteSz == 0) {
@@ -1422,51 +1598,50 @@ static rc_t _ItemResolveResolved(VResolver *resolver,
}
}
- RELEASE(VPath, vremote);
return rc;
}
/* Resolved: resolve locations */
-static rc_t ItemInitResolved(Item *self, VResolver *resolver,
- KDirectory *dir, bool ascp, const KRepositoryMgr *repoMgr,
- const KConfig *cfg, const VFSManager *vfs, KNSManager *kns, size_t minSize, size_t maxSize)
+static rc_t ItemInitResolved(Item *self, VResolver *resolver, KDirectory *dir,
+ bool ascp, const KRepositoryMgr *repoMgr, const KConfig *cfg,
+ const VFSManager *vfs, KNSManager *kns, size_t minSize, size_t maxSize)
{
Resolved *resolved = NULL;
rc_t rc = 0;
- KPathType type = kptNotFound;
VRemoteProtocols protocols = ascp ? eProtocolFaspHttp : eProtocolHttp;
assert(self);
resolved = &self->resolved;
resolved->name = ItemName(self);
-
+
assert(resolved->type != eRunTypeUnknown);
- type = KDirectoryPathType(dir, "%s", self->desc) & ~kptAlias;
- if (type == kptFile || type == kptDir) {
- resolved->path = self->desc;
- resolved->existing = true;
- if (resolved->type != eRunTypeDownload) {
- uint64_t s = -1;
- const KFile *f = NULL;
- rc = KDirectoryOpenFileRead(dir, &f, "%s", self->desc);
- if (rc == 0) {
- rc = KFileSize(f, &s);
- }
- if (s != -1) {
-/* OUTMSG(("%s\t%,zuB\n", self->desc, s)); */
- resolved->remoteSz = s;
+ if (self->desc != NULL) { /* object name is specified (not kart item) */
+ KPathType type = KDirectoryPathType(dir, "%s", self->desc) & ~kptAlias;
+ if (type == kptFile || type == kptDir) {
+ rc = VPathStrInitStr(&resolved->path, self->desc, 0);
+ resolved->existing = true;
+ if (resolved->type != eRunTypeDownload) {
+ uint64_t s = -1;
+ const KFile *f = NULL;
+ rc = KDirectoryOpenFileRead(dir, &f, "%s", self->desc);
+ if (rc == 0) {
+ rc = KFileSize(f, &s);
+ }
+ if (s != -1) {
+ resolved->remoteSz = s;
+ }
+ else {
+ OUTMSG(("%s\tunknown\n", self->desc));
+ }
+ RELEASE(KFile, f);
}
else {
- OUTMSG(("%s\tunknown\n", self->desc));
+ STSMSG(STS_TOP, ("'%s' is a local non-kart file", self->desc));
}
- RELEASE(KFile, f);
- }
- else {
- STSMSG(STS_TOP, ("'%s' is a local non-kart file", self->desc));
+ return 0;
}
- return 0;
}
rc = _ItemResolveResolved(resolver, protocols, self,
@@ -1478,7 +1653,7 @@ static rc_t ItemInitResolved(Item *self, VResolver *resolver,
STSMSG(STS_DBG, ("cache(%s)",
resolved->cache ? resolved->cache->addr : "NULL"));
STSMSG(STS_DBG, ("remote(%s:%,ld)",
- resolved->remote ? resolved->remote->addr : "NULL",
+ resolved->remote.str ? resolved->remote.str->addr : "NULL",
resolved->remoteSz));
if (rc == 0) {
@@ -1492,7 +1667,7 @@ static rc_t ItemInitResolved(Item *self, VResolver *resolver,
}
if (resolved->local.str == NULL
- && (resolved->cache == NULL || resolved->remote == NULL))
+ && (resolved->cache == NULL || resolved->remote.str == NULL))
{
rc = RC(rcExe, rcPath, rcValidating, rcParam, rcNull);
PLOGERR(klogInt, (klogInt, rc,
@@ -1517,7 +1692,9 @@ static rc_t ItemResolve(Item *item, int32_t row) {
assert(self->type);
++n;
- if (row > 0 && item->desc == NULL) {
+ if (row > 0 &&
+ item->desc == NULL) /* desc is NULL for kart items */
+ {
n = row;
}
@@ -1549,8 +1726,8 @@ static rc_t ItemDownload(Item *item) {
if (rc == 0) {
bool skip = false;
- if (self->existing) {
- self->path = item->desc;
+ if (self->existing) { /* the path is a path to an existing local file */
+ rc = VPathStrInitStr(&self->path, item->desc, 0);
return rc;
}
@@ -1581,10 +1758,13 @@ static rc_t ItemDownload(Item *item) {
if (isLocal) {
STSMSG(STS_TOP, ("%d) '%s' is found locally", n, self->name));
if (self->local.str != NULL) {
- self->path = self->local.str->addr;
+ VPathStrFini(&self->path);
+ rc = StringCopy(&self->path.str, self->local.str);
}
}
- else if (!_StringIsFasp(self->remote, NULL) && item->main->noHttp) {
+ else if (!_StringIsFasp(self->remote.str, NULL)
+ && item->main->noHttp)
+ {
rc = RC(rcExe, rcFile, rcCopying, rcFile, rcNotFound);
PLOGERR(klogErr, (klogErr, rc,
"cannot download '$(name)' using requested transport",
@@ -1592,16 +1772,17 @@ static rc_t ItemDownload(Item *item) {
}
else {
STSMSG(STS_TOP, ("%d) Downloading '%s'...", n, self->name));
- rc = MainDownload(self, item->main);
+ rc = MainDownload(self, item->main, item->isDependency);
if (rc == 0) {
STSMSG(STS_TOP,
("%d) '%s' was downloaded successfully", n, self->name));
if (self->cache != NULL) {
- self->path = self->cache->addr;
+ VPathStrFini(&self->path);
+ rc = StringCopy(&self->path.str, self->cache);
}
}
- else if (rc !=
- SILENT_RC(rcExe, rcProcess, rcExecuting, rcProcess, rcCanceled))
+ else if (rc != SILENT_RC(rcExe,
+ rcProcess, rcExecuting, rcProcess, rcCanceled))
{
STSMSG(STS_TOP, ("%d) failed to download %s", n, self->name));
}
@@ -1622,10 +1803,10 @@ static rc_t ItemPrintSized(const Item *self, int32_t row, size_t size) {
const String *name = NULL;
const String *itemDesc = NULL;
assert(self && row);
- if (self->desc != NULL) {
+ if (self->desc != NULL) { /* object */
OUTMSG(("%d\t%s\t%,zuB\n", row, self->desc, size));
}
- else {
+ else { /* kart item */
if (rc == 0) {
rc = KartItemProjId(self->item, &projId);
}
@@ -1669,10 +1850,9 @@ static rc_t ItemResolveResolvedAndDownloadOrProcess(Item *self, int32_t row) {
return ItemDownload(self);
}
-static rc_t ItemPostDownload(Item *item, int32_t row) {
+static rc_t ItemDownloadDependencies(Item *item) {
Resolved *resolved = NULL;
rc_t rc = 0;
- rc_t rc2 = 0;
const VDBDependencies *deps = NULL;
uint32_t count = 0;
uint32_t i = 0;
@@ -1681,31 +1861,22 @@ static rc_t ItemPostDownload(Item *item, int32_t row) {
resolved = &item->resolved;
- if (rc == 0) {
- if (resolved->type == eRunTypeList) {
- return rc;
- }
- else if (resolved->oversized) {
- item->main->oversized = true;
- }
- else if (resolved->undersized) {
- item->main->undersized = true;
- }
- if (resolved->path != NULL) {
- rc = MainDependenciesList(item->main, resolved->path, &deps);
- }
+ assert(resolved);
+
+ if (resolved->path.str != NULL) {
+ rc = MainDependenciesList(item->main, resolved, &deps);
}
/* resolve dependencies (refseqs) */
if (rc == 0 && deps != NULL) {
rc = VDBDependenciesCount(deps, &count);
if (rc == 0) {
- STSMSG(STS_TOP, ("'%s' has %d%s dependenc%s",
- item->desc, count, item->main->check_all ? "" : " unresolved",
+ STSMSG(STS_TOP, ("'%s' has %d%s dependenc%s", resolved->name,
+ count, item->main->check_all ? "" : " unresolved",
count == 1 ? "y" : "ies"));
}
else {
- DISP_RC2(rc, "Failed to check %s's dependencies", item->desc);
+ DISP_RC2(rc, "Failed to check %s's dependencies", resolved->name);
}
}
@@ -1715,7 +1886,7 @@ static rc_t ItemPostDownload(Item *item, int32_t row) {
if (rc == 0) {
rc = VDBDependenciesLocal(deps, &local, i);
- DISP_RC2(rc, "VDBDependenciesLocal", item->desc);
+ DISP_RC2(rc, "VDBDependenciesLocal", resolved->name);
if (local) {
continue;
}
@@ -1723,7 +1894,7 @@ static rc_t ItemPostDownload(Item *item, int32_t row) {
if (rc == 0) {
rc = VDBDependenciesSeqId(deps, &seq_id, i);
- DISP_RC2(rc, "VDBDependenciesSeqId", item->desc);
+ DISP_RC2(rc, "VDBDependenciesSeqId", resolved->name);
}
if (rc == 0) {
@@ -1751,6 +1922,7 @@ static rc_t ItemPostDownload(Item *item, int32_t row) {
ditem->desc = ncbiAcc;
ditem->main = item->main;
+ ditem->isDependency = true;
ResolvedReset(&ditem->resolved, eRunTypeDownload);
@@ -1761,15 +1933,427 @@ static rc_t ItemPostDownload(Item *item, int32_t row) {
}
}
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
+ RELEASE(VDBDependencies, deps);
+
+ return rc;
+}
+
+static bool ResolvedResetLocalFileAccession(
+ Resolved *self, const Main *main, const char *path)
+{
+ rc_t rc = 0;
+
+ assert(self);
+
+ if (!self->existing) {
+ /* need to check remote vdbcache when accession (no path) is specified */
+ return true;
}
- RELEASE(VDBDependencies, deps);
+ RELEASE(VPath, self->accession);
+ rc = VFSManagerMakePath(main->vfsMgr, &self->accession, "%s", path);
+ if (rc != 0) {
+ DISP_RC2(rc, "VFSManagerMakePath", path);
+ return rc;
+ }
+
+ if (!VPathIsAccessionOrOID(self->accession)) {
+ /* self->accession is not an accession but a path:
+ let's try to get the accession */
+ VPath *acc = NULL;
+ rc = VFSManagerExtractAccessionOrOID(
+ main->vfsMgr, &acc, self->accession);
+ if (rc == 0) {
+ char buffer[99] = "";
+ VPathReadPath(acc, buffer, sizeof buffer, NULL);
+ STSMSG(STS_INFO, ("Accession for '%s' = '%s': "
+ "checking vdbcache", path, buffer));
+
+ RELEASE(VPath, self->accession);
+ self->accession = acc;
+ }
+ else {
+ STSMSG(STS_TOP, ("Cannot extract accession for "
+ "'%s': skipped vdbcache download", path));
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static rc_t ItemResetRemoteToVdbcacheIfVdbcacheRemoteExists(
+ Item *self, char *remotePath, size_t remotePathLen, bool *exists)
+{
+ rc_t rc = 0;
+ size_t len = 0;
+ Resolved *resolved = NULL;
+ VPath *cremote = NULL;
+ assert(self && self->main && exists);
+ resolved = &self->resolved;
+ assert(resolved);
+ *exists = false;
+ if (resolved->remote.path == NULL) {
+#define TODO 1
+ rc_t rc = TODO;
+ DISP_RC(rc, "UNKNOWN REMOTE LOCATION WHEN TRYING TO FIND VDBCACHE");
+ return rc;
+ }
+ rc = VFSManagerMakePathWithExtension(self->main->vfsMgr,
+ &cremote, resolved->remote.path, ".vdbcache");
+ if (rc != 0) {
+ if (resolved->remote.str != NULL) {
+ DISP_RC2(rc, "VFSManagerMakePathWithExtension",
+ resolved->remote.str->addr);
+ }
+ else {
+ DISP_RC(rc, "VFSManagerMakePathWithExtension(remote)");
+ }
+ return rc;
+ }
+ rc = VPathReadUri(cremote, remotePath, remotePathLen, &len);
+ if (rc == 0) {
+ RELEASE(KFile, resolved->file);
+ rc = _KFileOpenRemote(&resolved->file, self->main->kns, remotePath);
+ if (rc == 0) {
+ char *query = string_chr(remotePath, len, '?');
+ if (query != NULL) {
+ *query = '\0';
+ }
+ STSMSG(STS_DBG, ("'%s' exists", remotePath));
+ STSMSG(STS_TOP, ("'%s' has remote vdbcache", resolved->name));
+ *exists = true;
+ rc = VPathStrInitStr(&resolved->remote, remotePath, len);
+ DISP_RC2(rc, "StringCopy(Remote.vdbcache)", remotePath);
+ resolved->remote.path = cremote;
+ cremote = NULL;
+ }
+ else if (rc == SILENT_RC(rcNS, rcFile, rcOpening, rcFile, rcNotFound)) {
+ STSMSG(STS_DBG, ("'%s' does not exist", remotePath));
+ *exists = false;
+ STSMSG(STS_TOP, ("'%s' has no remote vdbcache", resolved->name));
+ rc = 0;
+ }
+ else if
+ (rc == SILENT_RC(rcNS, rcFile, rcOpening, rcFile, rcUnauthorized))
+ {
+ STSMSG(STS_DBG, (
+ "Access to '%s's vdbcahe file is forbidden or it does not exist"
+ , resolved->name));
+ *exists = false;
+ STSMSG(STS_TOP, ("'%s' has no remote vdbcache", resolved->name));
+ rc = 0;
+ }
+ else {
+ DISP_RC2(rc, "Failed to check vdbcache", resolved->name);
+ }
+ }
+ RELEASE(VPath, cremote);
+ return rc;
+}
+
+static rc_t MainDetectVdbcacheCachePath(const Main *self,
+ const String *runCache, const VPath **runCachePath,
+ const String **vdbcacheCache)
+{
+ rc_t rc = 0;
+
+ VPath *vlocal = NULL;
+ VPath *clocal = NULL;
+
+ if (runCache == NULL && (runCachePath == NULL || *runCachePath == NULL))
+ {
+ rc_t rc = TODO;
+ DISP_RC(rc, "UNKNOWN CACHE LOCATION WHEN TRYING TO FIND VDBCACHE");
+ return rc;
+ }
+
+ if (runCachePath != NULL && *runCachePath != NULL) {
+ vlocal = (VPath*)*runCachePath;
+ }
+ else {
+ rc = VFSManagerMakePath(self->vfsMgr, &vlocal, "%S", runCache);
+ if (rc != 0) {
+ DISP_RC2(rc, "VFSManagerMakePath", runCache->addr);
+ return rc;
+ }
+ }
+
+ rc = VFSManagerMakePathWithExtension(
+ self->vfsMgr, &clocal, vlocal, ".vdbcache");
+ DISP_RC2(rc, "VFSManagerMakePathWithExtension", runCache->addr);
+
+ if (rc == 0) {
+ rc = VPathMakeString(clocal, vdbcacheCache);
+ }
+
+ RELEASE(VPath, clocal);
+
+ if (runCachePath == NULL) {
+ RELEASE(VPath, vlocal);
+ }
+ else if (*runCachePath == NULL) {
+ *runCachePath = vlocal;
+ }
return rc;
}
+static bool MainNeedDownload(const Main *self, const String *local,
+ const char *remotePath, const KFile *remote, size_t *remoteSz)
+{
+ KPathType type = kptNotFound;
+ assert(self && local);
+ type = KDirectoryPathType(self->dir, "%s", local->addr) & ~kptAlias;
+ if (type == kptNotFound) {
+ return false;
+ }
+ if (type != kptFile) {
+ if (self->force == eForceNo) {
+ STSMSG(STS_TOP, (
+ "%S (not a file) is found locally: consider it complete",
+ local));
+ return false;
+ }
+ else {
+ STSMSG(STS_TOP, (
+ "%S (not a file) is found locally and will be redownloaded",
+ local));
+ return true;
+ }
+ }
+ else if (self->force != eForceNo) {
+ return true;
+ }
+ else {
+ rc_t rc = 0;
+ size_t sLocal = 0;
+ assert(remoteSz);
+ rc = KFileSize(remote, remoteSz);
+ DISP_RC2(rc, "KFileSize(remote.vdbcache)", remotePath);
+ if (rc != 0) {
+ return true;
+ }
+ {
+ const KFile *f = NULL;
+ rc = KDirectoryOpenFileRead(self->dir, &f, "%S", local);
+ if (rc != 0) {
+ DISP_RC2(rc, "KDirectoryOpenFileRead", local->addr);
+ return true;
+ }
+ rc = KFileSize(f, &sLocal);
+ if (rc != 0) {
+ DISP_RC2(rc, "KFileSize", local->addr);
+ }
+ RELEASE(KFile, f);
+ if (rc != 0) {
+ return true;
+ }
+ }
+ if (sLocal == *remoteSz) {
+ STSMSG(STS_INFO, ("%S (%,lu) is found", local, sLocal));
+ return false;
+ }
+ else {
+ STSMSG(STS_INFO,
+ ("%S (%,lu) is found and will be redownloaded", local, sLocal));
+ return true;
+ }
+ }
+}
+
+static rc_t ItemDownloadVdbcache(Item *item) {
+ rc_t rc = 0;
+ Resolved *resolved = NULL;
+ bool checkRemote = true;
+ bool remoteExists = false;
+ char remotePath[PATH_MAX] = "";
+ const String *local = NULL;
+ const String *cache = NULL;
+ bool localExists = false;
+ bool download = true;
+ assert(item && item->main);
+ resolved = &item->resolved;
+ if (!resolved) {
+ STSMSG(STS_TOP,
+ ("CANNOT DOWNLOAD VDBCACHE FOR UNRESOLVED ITEM '%s'", item->desc));
+ /* TODO error? */
+ return 0;
+ }
+ {
+ bool csra = false;
+ const VDatabase *db = NULL;
+ KPathType type = VDBManagerPathType
+ (item->main->mgr, "%S", resolved->path.str) & ~kptAlias;
+ if (type == kptTable) {
+ STSMSG(STS_INFO, ("'%S' is a table", resolved->path.str));
+ }
+ else if (type != kptDatabase) {
+ STSMSG(STS_INFO, ("'%S' is not recognized as a database or a table",
+ resolved->path.str));
+ }
+ else {
+ rc_t rc = VDBManagerOpenDBRead(item->main->mgr,
+ &db, NULL, "%S", resolved->path.str);
+ if (rc == 0) {
+ csra = VDatabaseIsCSRA(db);
+ }
+ RELEASE(VDatabase, db);
+ if (csra) {
+ STSMSG(STS_INFO, ("'%s' is cSRA", resolved->name));
+ }
+ else {
+ STSMSG(STS_INFO, ("'%s' is not cSRA", resolved->name));
+ }
+ if (!csra) {
+ return 0;
+ }
+ }
+ }
+ if (resolved->remote.str == NULL ||
+ _StringIsFasp(resolved->remote.str, NULL))
+ {
+ if (item->main->noHttp) {
+ /* fasp download
+ return 0; ignore fasp-only transpot */
+ }
+ if (resolved->resolver == NULL) {
+ rc = VResolverAddRef(item->main->resolver);
+ if (rc == 0) {
+ resolved->resolver = item->main->resolver;
+ }
+ assert(!rc && resolved->resolver);
+ }
+ {
+ const char *path = item->desc;
+ if (path == NULL && resolved->path.str != NULL) {
+ path = resolved->path.str->addr;
+ }
+ checkRemote
+ = ResolvedResetLocalFileAccession(resolved, item->main, path);
+ }
+ if (checkRemote) {
+ rc_t rc2 = VPathStrFini(&resolved->remote);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ RELEASE(KFile, resolved->file);
+ rc = _VResolverRemote(resolved->resolver, eProtocolHttp,
+ resolved->name, resolved->accession, &resolved->remote.path,
+ &resolved->remote.str, &resolved->cache);
+ }
+ }
+ if (!checkRemote) {
+ return 0;
+ }
+ if (rc == 0) {
+ rc = ItemResetRemoteToVdbcacheIfVdbcacheRemoteExists(
+ item, remotePath, sizeof remotePath, &remoteExists);
+ }
+ if (!remoteExists) {
+ return 0;
+ }
+ {
+ bool cacheExists = false;
+ if (resolved->existing) {
+ /* resolved->path.str is a local file path */
+ rc = MainDetectVdbcacheCachePath(item->main,
+ resolved->path.str, &resolved->path.path, &local);
+ assert(local);
+ localExists = (VDBManagerPathType(item->main->mgr, "%S", local)
+ & ~kptAlias) != kptNotFound;
+ STSMSG(STS_DBG, ("'%S' %sexist%s", local,
+ localExists ? "" : "does not ", localExists ? "s" : ""));
+ }
+ /* check vdbcache file cache location and its existence */
+ rc = MainDetectVdbcacheCachePath(item->main,
+ resolved->cache, NULL, &cache);
+ cacheExists = (VDBManagerPathType(item->main->mgr, "%S", cache)
+ & ~kptAlias) != kptNotFound;
+ STSMSG(STS_DBG, ("'%S' %sexist%s", cache,
+ cacheExists ? "" : "does not ", cacheExists ? "s" : ""));
+ if (!localExists) {
+ localExists = cacheExists;
+ }
+ }
+ if (!remoteExists) {
+ return 0;
+ }
+ if (localExists) {
+ download = MainNeedDownload(item->main, local ? local : cache,
+ remotePath, resolved->file, &resolved->remoteSz);
+ }
+ RELEASE(String, local);
+ RELEASE(String, resolved->cache);
+ resolved->cache = cache;
+ if (download && rc == 0) {
+
+ /* ignore fasp transport request while ascp vdbcache address is unknown */
+ item->main->noHttp = false;
+
+ rc = MainDownload(&item->resolved, item->main, item->isDependency);
+ if (rc == 0) {
+ if (local && StringCompare(local, cache) != 0) {
+ STSMSG(STS_DBG, ("Removing '%S'", local));
+ /* TODO rm local vdbcache file
+ if full path is specified and it is not the cache path
+ rc = KDirectoryRemove(item->main->dir, false, "%S", local);
+ */
+ }
+ }
+ }
+ return rc;
+}
+
+static rc_t ItemPostDownload(Item *item, int32_t row) {
+ rc_t rc = 0;
+ Resolved *resolved = NULL;
+ KPathType type = kptNotFound;
+ assert(item);
+ resolved = &item->resolved;
+ if (resolved->type == eRunTypeList) {
+ return rc;
+ }
+ else if (resolved->oversized) {
+ item->main->oversized = true;
+ }
+ else if (resolved->undersized) {
+ item->main->undersized = true;
+ }
+
+ if (resolved->path.str != NULL) {
+ assert(item->main);
+ rc = _VDBManagerSetDbGapCtx(item->main->mgr, resolved->resolver);
+ type = VDBManagerPathType
+ (item->main->mgr, "%s", resolved->name) & ~kptAlias;
+ if (type != kptDatabase) {
+ if (type == kptTable) {
+ STSMSG(STS_DBG, ("...'%S' is a table", resolved->path.str));
+ }
+ else {
+ STSMSG(STS_DBG, ("...'%S' is not recognized "
+ "as a database or a table", resolved->path.str));
+ }
+ return rc;
+ }
+ else {
+ STSMSG(STS_DBG, ("...'%S' is a database", resolved->path.str));
+ }
+ }
+
+ rc = ItemDownloadDependencies(item);
+ if (true) {
+ rc_t rc2 = Quitting();
+ if (rc2 == 0) {
+ rc2 = ItemDownloadVdbcache(item);
+ }
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
+ return rc;
+}
+
static rc_t ItemProcess(Item *item, int32_t row) {
rc_t rc = 0;
@@ -1928,6 +2512,11 @@ static size_t _sizeFromString(const char *val) {
static const char* ASCP_USAGE[] =
{ "path to ascp program and private key file (asperaweb_id_dsa.putty)", NULL };
+#define ASCP_PAR_OPTION "ascp-options"
+#define ASCP_PAR_ALIAS NULL
+static const char* ASCP_PAR_USAGE[] =
+{ "arbitrary options to pass to ascp command line", NULL };
+
#define CHECK_ALL_OPTION "check-all"
#define CHECK_ALL_ALIAS "c"
static const char* CHECK_ALL_USAGE[] = { "double-check all refseqs", NULL };
@@ -1938,7 +2527,7 @@ static const char* FORCE_USAGE[] = {
"force object download - one of: no, yes, all.",
"no [default]: skip download if the object if found and complete;",
"yes: download it even if it is found and is complete;", "all: ignore lock "
- "files (stale locks or it is beeing downloaded by another process: "
+ "files (stale locks or it is being downloaded by another process: "
"use at your own risk!)", NULL };
#define FAIL_ASCP_OPTION "FAIL-ASCP"
@@ -1995,6 +2584,18 @@ static const char* SIZE_USAGE[] = {
"maximum file size to download in KB (exclusive).",
"Default: " DEFAULT_MAX_FILE_SIZE, NULL };
+#if ALLOW_STRIP_QUALS
+#define STRIP_QUALS_OPTION "strip-quals"
+#define STRIP_QUALS_ALIAS NULL
+static const char* STRIP_QUALS_USAGE[] =
+{ "remove QUALITY column from all tables", NULL };
+#endif
+
+#define ELIM_QUALS_OPTION "eliminate-quals"
+#define ELIM_QUALS_ALIAS NULL
+static const char* ELIM_QUALS_USAGE[] =
+{ "don't download QUALITY column", NULL };
+
#ifdef _DEBUGGING
#define TEXTKART_OPTION "text-kart"
static const char* TEXTKART_USAGE[] =
@@ -2003,22 +2604,27 @@ static const char* TEXTKART_USAGE[] =
static OptDef Options[] = {
/* needs_value required*/
- { FORCE_OPTION , FORCE_ALIAS , NULL, FORCE_USAGE , 1, true, false }
- ,{ TRANS_OPTION , TRASN_ALIAS , NULL, TRANS_USAGE , 1, true, false }
- ,{ LIST_OPTION , LIST_ALIAS , NULL, LIST_USAGE , 1, false,false }
- ,{ NM_L_OPTION , NM_L_ALIAS , NULL, NM_L_USAGE , 1, false,false }
- ,{ SZ_L_OPTION , SZ_L_ALIAS , NULL, SZ_L_USAGE , 1, false,false }
- ,{ ROWS_OPTION , ROWS_ALIAS , NULL, ROWS_USAGE , 1, true, false }
- ,{ MINSZ_OPTION , MINSZ_ALIAS , NULL, MINSZ_USAGE , 1, true ,false }
- ,{ SIZE_OPTION , SIZE_ALIAS , NULL, SIZE_USAGE , 1, true ,false }
- ,{ ORDR_OPTION , ORDR_ALIAS , NULL, ORDR_USAGE , 1, true ,false }
- ,{ ASCP_OPTION , ASCP_ALIAS , NULL, ASCP_USAGE , 1, true ,false }
- ,{ HBEAT_OPTION , HBEAT_ALIAS , NULL, HBEAT_USAGE , 1, true, false }
- ,{ FAIL_ASCP_OPTION, FAIL_ASCP_ALIAS, NULL, FAIL_ASCP_USAGE, 1, false, false}
+ { FORCE_OPTION , FORCE_ALIAS , NULL, FORCE_USAGE , 1, true, false }
+ ,{ TRANS_OPTION , TRASN_ALIAS , NULL, TRANS_USAGE , 1, true, false }
+ ,{ LIST_OPTION , LIST_ALIAS , NULL, LIST_USAGE , 1, false,false }
+ ,{ NM_L_OPTION , NM_L_ALIAS , NULL, NM_L_USAGE , 1, false,false }
+ ,{ SZ_L_OPTION , SZ_L_ALIAS , NULL, SZ_L_USAGE , 1, false,false }
+ ,{ ROWS_OPTION , ROWS_ALIAS , NULL, ROWS_USAGE , 1, true, false }
+ ,{ MINSZ_OPTION , MINSZ_ALIAS , NULL, MINSZ_USAGE , 1, true ,false }
+ ,{ SIZE_OPTION , SIZE_ALIAS , NULL, SIZE_USAGE , 1, true ,false }
+ ,{ ORDR_OPTION , ORDR_ALIAS , NULL, ORDR_USAGE , 1, true ,false }
+ ,{ ASCP_OPTION , ASCP_ALIAS , NULL, ASCP_USAGE , 1, true ,false }
+ ,{ ASCP_PAR_OPTION , ASCP_PAR_ALIAS , NULL, ASCP_PAR_USAGE, 1, true ,false }
+ ,{ HBEAT_OPTION , HBEAT_ALIAS , NULL, HBEAT_USAGE , 1, true, false }
+ ,{ FAIL_ASCP_OPTION , FAIL_ASCP_ALIAS , NULL, FAIL_ASCP_USAGE, 1, false, false}
+#if ALLOW_STRIP_QUALS
+ ,{ STRIP_QUALS_OPTION , STRIP_QUALS_ALIAS , NULL, STRIP_QUALS_USAGE , 1, false, false }
+#endif
+ ,{ ELIM_QUALS_OPTION , ELIM_QUALS_ALIAS , NULL, ELIM_QUALS_USAGE , 1, false, false }
#ifdef _DEBUGGING
- ,{ TEXTKART_OPTION , NULL , NULL, TEXTKART_USAGE , 1, true , false}
+ ,{ TEXTKART_OPTION , NULL , NULL, TEXTKART_USAGE , 1, true , false}
#endif
- ,{ CHECK_ALL_OPTION, CHECK_ALL_ALIAS, NULL, CHECK_ALL_USAGE, 1, false, false}
+ ,{ CHECK_ALL_OPTION , CHECK_ALL_ALIAS , NULL, CHECK_ALL_USAGE, 1, false, false}
};
static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
@@ -2045,7 +2651,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
if (pcount > 0) {
const char *val = NULL;
- rc = ArgsOptionValue(self->args, FORCE_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, FORCE_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" FORCE_OPTION "' argument value");
@@ -2136,7 +2742,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
}
if (pcount > 0) {
const char *val = NULL;
- rc = ArgsOptionValue(self->args, ASCP_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, ASCP_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" ASCP_OPTION "' argument value");
@@ -2162,6 +2768,23 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
}
}
+/* ASCP_PAR_OPTION */
+ rc = ArgsOptionCount(self->args, ASCP_PAR_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" ASCP_PAR_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ rc = ArgsOptionValue(self->args,
+ ASCP_PAR_OPTION, 0, (const void **)&self->ascpParams);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" ASCP_PAR_OPTION "' argument value");
+ break;
+ }
+ }
+
/* FAIL_ASCP_OPTION */
rc = ArgsOptionCount(self->args, FAIL_ASCP_OPTION, &pcount);
if (rc != 0) {
@@ -2183,7 +2806,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
if (pcount > 0) {
double f;
const char *val = NULL;
- rc = ArgsOptionValue(self->args, HBEAT_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, HBEAT_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" HBEAT_OPTION "' argument value");
@@ -2202,7 +2825,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
if (pcount > 0) {
const char *val = NULL;
- rc = ArgsOptionValue(self->args, ORDR_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, ORDR_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" ORDR_OPTION "' argument value");
@@ -2224,7 +2847,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
break;
}
if (pcount > 0) {
- rc = ArgsOptionValue(self->args, ROWS_OPTION, 0, &self->rows);
+ rc = ArgsOptionValue(self->args, ROWS_OPTION, 0, (const void **)&self->rows);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" ROWS_OPTION "' argument value");
@@ -2242,7 +2865,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
break;
}
if (pcount > 0) {
- rc = ArgsOptionValue(self->args, MINSZ_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, MINSZ_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" MINSZ_OPTION "' argument value");
@@ -2262,7 +2885,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
break;
}
if (pcount > 0) {
- rc = ArgsOptionValue(self->args, SIZE_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, SIZE_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" SIZE_OPTION "' argument value");
@@ -2292,7 +2915,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
if (pcount > 0) {
const char *val = NULL;
- rc = ArgsOptionValue(self->args, TRANS_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, TRANS_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" TRANS_OPTION "' argument value");
@@ -2307,6 +2930,38 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
}
}
+#if ALLOW_STRIP_QUALS
+/* STRIP_QUALS_OPTION */
+ rc = ArgsOptionCount(self->args, STRIP_QUALS_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" STRIP_QUALS_OPTION "' argument");
+ break;
+ }
+
+ if (pcount > 0) {
+ self->stripQuals = true;
+ }
+#endif
+
+/* ELIM_QUALS_OPTION */
+ rc = ArgsOptionCount(self->args, ELIM_QUALS_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" ELIM_QUALS_OPTION "' argument");
+ break;
+ }
+
+ if (pcount > 0) {
+ self->eliminateQuals = true;
+ }
+
+#if ALLOW_STRIP_QUALS
+ if (self->stripQuals && self->eliminateQuals) {
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
+ LOGERR(klogErr, rc, "Cannot set both '" STRIP_QUALS_OPTION "' and '" ELIM_QUALS_OPTION "'");
+ break;
+ }
+#endif
+
#ifdef _DEBUGGING
/* TEXTKART_OPTION */
rc = ArgsOptionCount(self->args, TEXTKART_OPTION, &pcount);
@@ -2318,7 +2973,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
if (pcount > 0) {
const char *val = NULL;
- rc = ArgsOptionValue(self->args, TEXTKART_OPTION, 0, &val);
+ rc = ArgsOptionValue(self->args, TEXTKART_OPTION, 0, (const void **)&val);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" TEXTKART_OPTION "' argument value");
@@ -2398,6 +3053,9 @@ rc_t CC Usage(const Args *args) {
param = "size";
}
}
+ else if (strcmp(Options[i].name, ASCP_PAR_OPTION) == 0) {
+ param = "value";
+ }
#ifdef _DEBUGGING
else if (strcmp(Options[i].name, TEXTKART_OPTION) == 0) {
param = "value";
@@ -2439,13 +3097,18 @@ static void CC bstKrtWhack(BSTNode *n, void *ignore) {
free(sn);
}
-static int CC bstKrtSort(const BSTNode *item, const BSTNode *n) {
+static int64_t CC bstKrtSort(const BSTNode *item, const BSTNode *n) {
const KartTreeNode *sn1 = (const KartTreeNode*)item;
const KartTreeNode *sn2 = (const KartTreeNode*)n;
assert(sn1 && sn2 && sn1->i && sn2->i);
- return sn1->i->resolved.remoteSz > sn2->i->resolved.remoteSz;
+ if (sn1->i->resolved.remoteSz < sn2->i->resolved.remoteSz)
+ return -1;
+ else if (sn1->i->resolved.remoteSz > sn2->i->resolved.remoteSz)
+ return 1;
+
+ return 0;
}
static void CC bstKrtDownload(BSTNode *n, void *data) {
@@ -2722,9 +3385,8 @@ static rc_t MainRun(Main *self, const char *arg, const char *realArg) {
}
}
}
- else {
- RELEASE(Item, item);
- }
+
+ RELEASE(Item, item);
}
if (type == eRunTypeList) {
@@ -2733,7 +3395,7 @@ static rc_t MainRun(Main *self, const char *arg, const char *realArg) {
}
}
else if (type == eRunTypeGetSize) {
- OUTMSG(("\nDownloading the files\n\n", realArg));
+ OUTMSG(("\nDownloading the files...\n\n", realArg));
BSTreeForEach(&trKrt, false, bstKrtDownload, NULL);
}
}
@@ -2789,7 +3451,7 @@ rc_t CC KMain(int argc, char *argv[]) {
for (i = 0; i < pcount; ++i) {
const char *obj = NULL;
- rc_t rc2 = ArgsParamValue(pars.args, i, &obj);
+ rc_t rc2 = ArgsParamValue(pars.args, i, (const void **)&obj);
DISP_RC(rc2, "ArgsParamValue");
if (rc2 == 0) {
rc2 = MainRun(&pars, obj, obj);
diff --git a/tools/prefetch/prefetch.vers b/tools/prefetch/prefetch.vers
index 8e8299d..35d16fb 100644
--- a/tools/prefetch/prefetch.vers
+++ b/tools/prefetch/prefetch.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/prefetch/prefetch.vers.h b/tools/prefetch/prefetch.vers.h
index 10ae6d4..dae07c4 100644
--- a/tools/prefetch/prefetch.vers.h
+++ b/tools/prefetch/prefetch.vers.h
@@ -1 +1 @@
-#define PREFETCH_VERS 0x02040002
+#define PREFETCH_VERS 0x02050007
diff --git a/tools/qual-recal/Makefile b/tools/qual-recal/Makefile
deleted file mode 100644
index cfc2d61..0000000
--- a/tools/qual-recal/Makefile
+++ /dev/null
@@ -1,101 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-default: std
-
-TOP ?= $(abspath ../..)
-MODULE = tools/qual-recal
-
-include $(TOP)/build/Makefile.env
-
-INT_TOOLS = \
- qual-recal
-
-EXT_TOOLS =
-
-ALL_TOOLS = \
- $(INT_TOOLS) \
- $(EXT_TOOLS)
-
-#-------------------------------------------------------------------------------
-# outer targets
-#
-all std: vers-includes
- @ $(MAKE_CMD) $(TARGDIR)/std
-
-$(ALL_TOOLS): vers-includes
- @ $(MAKE_CMD) $(BINDIR)/$@
-
-.PHONY: all std $(ALL_TOOLS)
-
-#-------------------------------------------------------------------------------
-# std
-#
-$(TARGDIR)/std: \
- $(addprefix $(BINDIR)/,$(ALL_TOOLS))
-
-.PHONY: $(TARGDIR)/std
-
-#-------------------------------------------------------------------------------
-# vers-includes
-#
-$(TARGDIR)/vers-includes: $(addsuffix .vers.h,$(EXT_TOOLS))
-
-.PHONY: $(TARGDIR)/vers-includes
-
-#-------------------------------------------------------------------------------
-# clean
-#
-clean: stdclean
-
-.PHONY: clean
-
-#-------------------------------------------------------------------------------
-# align-info
-#
-QUAL_RECAL_SRC = \
- qual-recal
-
-QUAL_RECAL_OBJ = \
- $(addsuffix .$(OBJX),$(QUAL_RECAL_SRC))
-
-QUAL_RECAL_LIB = \
- -lkapp \
- -lwvdb \
- -lwkdb \
- $(UPDATE_SCHEMA_LIBS) \
- -lksrch \
- -lkproc \
- -lvfs \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lklib \
- -lm
-
-
-$(BINDIR)/qual-recal: $(QUAL_RECAL_OBJ)
- $(LD) --exe -o $@ $^ $(QUAL_RECAL_LIB)
-
diff --git a/tools/qual-recal/make_histo_data.sh b/tools/qual-recal/make_histo_data.sh
deleted file mode 100644
index 886cf4b..0000000
--- a/tools/qual-recal/make_histo_data.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-echo qual-recalib-stat -p -o qstat.${$}.txt -m file -g 8 "${@}" || exit ${?};
-qual-recalib-stat -p -o qstat.${$}.txt -m file -g 8 "${@}" || exit ${?};
-tail -n +2 qstat.${$}.txt > qstat.${$}.headless.txt
-(echo -n "SPOTGROUPS: " ; cut -f 1 qstat.${$}.headless.txt | sort | uniq | wc -l) >qstat.${$}.hdr
-(echo -n "MAX_POS: " ; cut -f 2 qstat.${$}.headless.txt | sort -n | tail -n 1 ) >>qstat.${$}.hdr
-(echo -n "MAX_READ: " ; cut -f 3 qstat.${$}.headless.txt | sort -n | tail -n 1 ) >>qstat.${$}.hdr
-(cat qstat.${$}.hdr; echo ""; cat qstat.${$}.headless.txt) >qstat.${$}.hdr.text
-rm qstat.${$}.headless.txt qstat.${$}.hdr
-echo qual-recal -s qstat.${$}.hdr.text "${1}"
-qual-recal -s qstat.${$}.hdr.text "${1}"
diff --git a/tools/qual-recal/qual-recal.c b/tools/qual-recal/qual-recal.c
deleted file mode 100644
index e10b250..0000000
--- a/tools/qual-recal/qual-recal.c
+++ /dev/null
@@ -1,654 +0,0 @@
-/*==============================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- *==============================================================================
- */
-
-#include <vdb/manager.h> /* VDBManager */
-#include <vdb/database.h> /* VDatabase */
-#include <vdb/table.h> /* VTable */
-#include <vdb/schema.h> /* VSchema */
-#include <vdb/cursor.h> /* VCursor */
-#include <insdc/insdc.h>
-
-#include <kdb/meta.h>
-
-#include <kapp/main.h>
-
-#include <kfs/file.h>
-#include <klib/out.h> /* OUTMSG */
-#include <klib/log.h> /* (void)LOGERR */
-#include <klib/debug.h> /* DBGMSG */
-#include <klib/rc.h> /* RC */
-
-#include <assert.h>
-#include <stdlib.h> /* free */
-#include <string.h> /* strcmp */
-#include <stdio.h>
-#include <math.h>
-
-#define MIN_HITS (10000)
-
-struct Params {
- char const *statPath;
- char const *dbPath;
-} static Params;
-
-typedef struct stats_t {
- unsigned hits;
- unsigned miss;
-} stats_t;
-
-typedef struct {
- stats_t total;
- struct {
- stats_t total;
- struct {
- stats_t total;
- struct {
- stats_t hp_r[8];
- stats_t total;
- } gc_c[8];
- } dmer[25];
- unsigned qual[41];
- unsigned rcal[41];
- unsigned diff[82];
- } read[2];
-} read_node_t;
-
-unsigned qurc[41][41];
-
-typedef struct {
- char *spot_group;
- unsigned positions;
- unsigned reads;
- read_node_t position[1];
-} top_node_t;
-
-static unsigned spot_groups;
-static top_node_t **stats;
-
-static top_node_t *alloc_top_node_t(unsigned positions)
-{
- top_node_t *self = calloc(1, sizeof(*self) - sizeof(self->position) + positions * sizeof(self->position[0]));
- return self;
-}
-
-static top_node_t *new_top_node_t(unsigned positions, unsigned reads, char const sgnm[])
-{
- top_node_t *self = alloc_top_node_t(positions);
- if (self) {
- self->spot_group = strdup(sgnm);
- self->positions = positions;
- self->reads = reads;
- }
- return self;
-}
-
-unsigned base2number(int base)
-{
- switch (base) {
- case 'A':
- return 0;
- case 'C':
- return 1;
- case 'G':
- return 2;
- case 'T':
- return 3;
- default:
- return 4;
- }
-}
-
-static rc_t LoadStats(KFile const *kf)
-{
- char buf[16 * 4096];
- unsigned bsz = 0;
- uint64_t fpos = 0;
- bool inheader = true;
- unsigned reads;
- unsigned positions;
- unsigned nsgrp;
- unsigned line = 1;
-
- for ( ; ; ) {
- unsigned cur = 0;
- unsigned len;
-
- {
- size_t nread;
- rc_t rc = KFileRead(kf, fpos, &buf[bsz], sizeof(buf) - bsz, &nread);
-
- if (rc) return rc;
- if (nread == 0)
- return 0;
- bsz += nread;
- fpos += nread;
- }
- do {
- for (len = 0; len + cur < bsz; ++len) {
- if (buf[cur + len] == '\n') {
- buf[cur + len] = '\0';
- ++len;
- ++line;
- goto PROCESS_LINE;
- }
- }
- memmove(buf, &buf[cur], bsz -= cur);
- break;
- PROCESS_LINE:
- if (inheader) {
- unsigned val;
-
- if (buf[cur] == '\0')
- inheader = false;
- else if (sscanf(buf + cur, "MAX_POS: %u", &val) == 1)
- positions = val + 1;
- else if (sscanf(buf + cur, "SPOTGROUPS: %u", &val) == 1) {
- spot_groups = val;
- stats = malloc(spot_groups * sizeof(stats[0]));
- if (stats == NULL)
- return RC(rcApp, rcFile, rcReading, rcMemory, rcExhausted);
- nsgrp = 0;
- }
- else if (sscanf(buf + cur, "MAX_READ: %u", &val) == 1)
- reads = val + 1;
- else
- return RC(rcApp, rcFile, rcReading, rcData, rcInvalid);
- }
- else {
- unsigned hits;
- unsigned miss;
- unsigned bpos;
- unsigned read;
- unsigned hp_r;
- unsigned gc_c;
- unsigned dmer;
- char dimer[2];
- char sgnm[256];
- int i;
-
- i = sscanf(buf + cur, "%256s %u %u %2s %u %u %*u %*u %u %u",
- sgnm,
- &bpos,
- &read,
- dimer,
- &gc_c,
- &hp_r,
- &hits,
- &miss
- );
-
- if (i != 8) {
- fprintf(stderr, "error at line %u: %s\n", line, buf + cur);
- return RC(rcApp, rcFile, rcReading, rcData, rcInvalid);
- }
- dmer = base2number(dimer[0]) * 5 + base2number(dimer[1]);
- if (nsgrp == 0) {
- fprintf(stderr, "spot group: %s\n", sgnm);
- stats[0] = new_top_node_t(positions, reads, sgnm);
- if (stats[0] == NULL)
- return RC(rcApp, rcFile, rcReading, rcMemory, rcExhausted);
- ++nsgrp;
- }
- else {
- if (strcmp(stats[nsgrp-1]->spot_group, sgnm) != 0) {
- fprintf(stderr, "spot group: %s\n", sgnm);
- if (nsgrp == spot_groups)
- return RC(rcApp, rcFile, rcReading, rcData, rcInvalid);
- stats[nsgrp] = new_top_node_t(positions, reads, sgnm);
- if (stats[nsgrp] == NULL)
- return RC(rcApp, rcFile, rcReading, rcMemory, rcExhausted);
- ++nsgrp;
- }
- }
- assert(bpos < positions);
- assert(read < reads);
- assert(gc_c < 8);
- assert(hp_r < 8);
-
- stats[nsgrp-1]->position[bpos].total.hits += hits;
- stats[nsgrp-1]->position[bpos].total.miss += miss;
- stats[nsgrp-1]->position[bpos].read[read].total.hits += hits;
- stats[nsgrp-1]->position[bpos].read[read].total.miss += miss;
- stats[nsgrp-1]->position[bpos].read[read].dmer[dmer].total.hits += hits;
- stats[nsgrp-1]->position[bpos].read[read].dmer[dmer].total.miss += miss;
- stats[nsgrp-1]->position[bpos].read[read].dmer[dmer].gc_c[gc_c].total.hits += hits;
- stats[nsgrp-1]->position[bpos].read[read].dmer[dmer].gc_c[gc_c].total.miss += miss;
- stats[nsgrp-1]->position[bpos].read[read].dmer[dmer].gc_c[gc_c].hp_r[hp_r].hits += hits;
- stats[nsgrp-1]->position[bpos].read[read].dmer[dmer].gc_c[gc_c].hp_r[hp_r].miss += miss;
- }
- cur += len;
- } while (1);
- }
-}
-
-static rc_t compute(char const sgnm[], char const seqbin[], uint8_t const qual[], unsigned read, unsigned len)
-{
- unsigned last_base = 4;
- unsigned i;
- unsigned hp_r = 0;
- unsigned gc_c = 0;
- static unsigned sgrp = 0;
-
- if (strcmp(stats[sgrp]->spot_group, sgnm) != 0) {
- for (i = 1; i != spot_groups; ++i) {
- if (strcmp(stats[(i + sgrp)%spot_groups]->spot_group, sgnm) == 0) {
- sgrp = (i + sgrp)%spot_groups;
- break;
- }
- }
- if (sgrp == spot_groups)
- return RC(rcApp, rcTable, rcReading, rcData, rcInvalid);
- }
- assert(len <= stats[sgrp]->positions);
- assert(read < stats[sgrp]->reads);
-
- for (i = 0; i != len; ++i) {
- unsigned base = base2number(seqbin[i]);
- unsigned dmer = last_base * 5 + base;
- unsigned hits = stats[sgrp]->position[i].read[read].total.hits;
- unsigned miss = stats[sgrp]->position[i].read[read].total.miss;
- unsigned recalQ;
- int diff;
-
-#if 1
- if (last_base == base)
- ++hp_r;
- else
- hp_r = 0;
-
- if (i >= 8 && (seqbin[i - 8] == 'G' || seqbin[i - 8] == 'C')) {
- assert(gc_c > 0);
- --gc_c;
- }
- assert(gc_c < 8);
-
- if (stats[sgrp]->position[i].read[read].dmer[dmer].total.hits >= MIN_HITS) {
- hits = stats[sgrp]->position[i].read[read].dmer[dmer].total.hits;
- miss = stats[sgrp]->position[i].read[read].dmer[dmer].total.miss;
-
- if (stats[sgrp]->position[i].read[read].dmer[dmer].gc_c[gc_c].total.hits >= MIN_HITS) {
- unsigned const hpr = hp_r > 7 ? 7 : hp_r;
-
- hits = stats[sgrp]->position[i].read[read].dmer[dmer].gc_c[gc_c].total.hits;
- miss = stats[sgrp]->position[i].read[read].dmer[dmer].gc_c[gc_c].total.miss;
-
- if (stats[sgrp]->position[i].read[read].dmer[dmer].gc_c[gc_c].hp_r[hpr].hits >= MIN_HITS) {
- hits = stats[sgrp]->position[i].read[read].dmer[dmer].gc_c[gc_c].hp_r[hpr].hits;
- miss = stats[sgrp]->position[i].read[read].dmer[dmer].gc_c[gc_c].hp_r[hpr].miss;
- }
- }
- }
- if (seqbin[i] == 'G' || seqbin[i] == 'C')
- ++gc_c;
- last_base = base;
-#endif
- recalQ = floor(-10.0*log10(miss/(double)hits));
-
- diff = recalQ - qual[i];
- if (qual[i] == 2)
- diff = 41;
- if (diff < -40)
- diff = -40;
- else if (diff > 40)
- diff = 40;
-
- if (recalQ > 40)
- recalQ = 40;
-
- ++stats[sgrp]->position[i].read[read].qual[qual[i]];
- ++stats[sgrp]->position[i].read[read].rcal[recalQ];
- ++stats[sgrp]->position[i].read[read].diff[diff + 40];
- ++qurc[qual[i]][recalQ];
- }
- return 0;
-}
-
-static void print(void)
-{
- FILE *Q = fopen("qual.histo.txt", "w");
- if (Q) {
- FILE *R = fopen("rcal.histo.txt", "w");
- if (R) {
- FILE *D = fopen("diff.histo.txt", "w");
- if (D) {
- unsigned sg;
-
- for (sg = 0; sg < spot_groups; ++sg) {
- unsigned read;
-
- for (read = 0; read < stats[sg]->reads; ++read) {
- unsigned pos;
-
- for (pos = 0; pos < stats[sg]->positions; ++pos) {
- unsigned i;
-
- for (i = 0; i < 41; ++i) {
- if (stats[sg]->position[pos].read[read].qual[i])
- fprintf(Q, "%s\t%u\t%u\t%u\t%u\n", stats[sg]->spot_group, read, pos, i, stats[sg]->position[pos].read[read].qual[i]);
- if (stats[sg]->position[pos].read[read].rcal[i])
- fprintf(R, "%s\t%u\t%u\t%u\t%u\n", stats[sg]->spot_group, read, pos, i, stats[sg]->position[pos].read[read].rcal[i]);
- }
- for (i = 0; i < 81; ++i) {
- if (stats[sg]->position[pos].read[read].diff[i])
- fprintf(D, "%s\t%u\t%u\t%i\t%u\n", stats[sg]->spot_group, read, pos, (int)i - 40, stats[sg]->position[pos].read[read].diff[i]);
- }
- if (stats[sg]->position[pos].read[read].diff[81])
- fprintf(D, "%s\t%u\t%u\t-128\t%u\n", stats[sg]->spot_group, read, pos, stats[sg]->position[pos].read[read].diff[81]);
- }
- }
- }
- fclose(D);
- }
- fclose(R);
- }
- fclose(Q);
- }
- Q = fopen("qurc.histo.txt", "w");
- if (Q) {
- unsigned q, r;
-
- for (q = 0; q != 41; ++q) {
- for (r = 0; r != 41; ++r) {
- if (qurc[q][r])
- fprintf(Q, "%u\t%u\t%u\n", q, r, qurc[q][r]);
- }
- }
- fclose(Q);
- }
-}
-
-static rc_t process(VTable *dst, VTable const *src)
-{
- VCursor const *curs;
- rc_t rc = VTableCreateCursorRead(src, &curs);
-
- if (rc == 0) {
- uint32_t idx[5];
-
- while (1) {
- rc = VCursorAddColumn(curs, idx + 0, "SPOT_GROUP"); if (rc) break;
- rc = VCursorAddColumn(curs, idx + 1, "(INSDC:dna:text)READ"); if (rc) break;
- rc = VCursorAddColumn(curs, idx + 2, "QUALITY"); if (rc) break;
- rc = VCursorAddColumn(curs, idx + 3, "(INSDC:coord:len)READ_LEN"); if (rc) break;
- rc = VCursorAddColumn(curs, idx + 4, "(INSDC:coord:zero)READ_START"); if (rc) break;
- rc = VCursorOpen(curs);
- break;
- }
- while (rc == 0) {
- rc = VCursorOpenRow(curs);
- while (rc == 0) {
- char sgnm[256];
- char const *read;
- unsigned readlen;
- unsigned nreads;
- unsigned i;
- uint8_t const *qual;
- INSDC_coord_len const *read_len;
- INSDC_coord_zero const *read_start;
- uint32_t bits;
- uint32_t boff;
- uint32_t rlen;
-
- rc = VCursorRead(curs, idx[0], 8, sgnm, 256, &rlen); if (rc) break;
- sgnm[rlen] = '\0';
-
- rc = VCursorCellData(curs, idx[1], &bits, (void const **)&read, &boff, &rlen); if (rc) break;
- readlen = rlen;
- assert(bits == 8);
- assert(boff == 0);
-
- rc = VCursorCellData(curs, idx[2], &bits, (void const **)&qual, &boff, &rlen); if (rc) break;
- assert(readlen == rlen);
- assert(bits == 8);
- assert(boff == 0);
-
- rc = VCursorCellData(curs, idx[3], &bits, (void const **)&read_len, &boff, &rlen); if (rc) break;
- nreads = rlen;
- assert(bits == 32);
- assert(boff == 0);
-
- rc = VCursorCellData(curs, idx[4], &bits, (void const **)&read_start, &boff, &rlen); if (rc) break;
- assert(nreads == rlen);
- assert(bits == 32);
- assert(boff == 0);
-
- for (i = 0; i != nreads && rc == 0; ++i) {
- rc = compute(sgnm, read + read_start[i], qual + read_start[i], i, read_len[i]);
- }
- break;
- }
- if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
- print();
- rc = 0;
- break;
- }
- rc = VCursorCloseRow(curs);
- }
- VCursorRelease(curs);
- }
- return rc;
-}
-
-static rc_t ModifySchema(VSchema *schema)
-{
- VSchemaRuntimeTable *tschema;
- rc_t rc = VSchemaMakeRuntimeTable(schema, &tschema, "NCBI:align:tbl:seq:qc", NULL);
-
- if (rc == 0) {
- rc = VSchemaRuntimeTableAddIntegerColumn(tschema, 16, true, "DELTA_Q");
- if (rc == 0) {
- rc = VSchemaRuntimeTableCommit(tschema);
- }
- VSchemaRuntimeTableClose(tschema);
- }
- return rc;
-}
-
-static rc_t OpenDatabaseAndTable(VTable **tbl, VDatabase const **db)
-{
- VDBManager *vdb;
- rc_t rc = VDBManagerMakeUpdate(&vdb, NULL);
-
- if (rc == 0) {
- rc = VDBManagerOpenDBRead(vdb, db, NULL, "%s", Params.dbPath);
- if (rc == 0) {
-#if 0
- VSchema *schema;
-
- rc = VDBManagerMakeSchema(vdb, &schema);
- if (rc == 0) {
- rc = VSchemaParseFile(schema, "align/align.vschema");
- if (rc == 0) {
- rc = ModifySchema(schema);
- if (rc == 0) {
- rc = VDBManagerCreateTable(vdb, tbl, schema, "NCBI:align:tbl:seq:qc", kcmInit + kcmMD5, "%s/%s", Params.dbPath, ".TEMP");
- if (rc == 0)
- rc = VTableColumnCreateParams(*tbl, kcmInit, kcmMD5, 0);
- if (rc)
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create output '$(outname)'", "outname=%s/%s", Params.dbPath, ".TEMP"));
- }
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to load schema", ""));
- VSchemaRelease(schema);
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create schema", ""));
-#endif
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open input '$(inname)'", "inname=%s", Params.dbPath));
- VDBManagerRelease(vdb);
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to create manager", ""));
- return rc;
-}
-
-rc_t KFileOpenRead(KFile const **kfp, char const fname[])
-{
- KDirectory *dir;
- rc_t rc = KDirectoryNativeDir(&dir);
-
- if (rc == 0) {
- rc = KDirectoryOpenFileRead(dir, kfp, "%s", fname);
- KDirectoryRelease(dir);
- }
- return rc;
-}
-
-static rc_t run(void)
-{
- KFile const *kf;
- rc_t rc = KFileOpenRead(&kf, Params.statPath);
-
- if (rc == 0) {
- rc = LoadStats(kf);
- KFileRelease(kf);
- if (rc == 0) {
- VTable *dst = NULL;
- VDatabase const *src = NULL;
- rc_t rc = OpenDatabaseAndTable(&dst, &src);
-
- if (rc == 0) {
- VTable const *seq;
-
- rc = VDatabaseOpenTableRead(src, &seq, "SEQUENCE");
- if (rc == 0) {
- rc = process(dst, seq);
- VTableRelease(seq);
- }
- }
- VTableRelease(dst);
- VDatabaseRelease(src);
- }
- }
- return rc;
-}
-
-static const char* param_usage[] = { "Path to the database" };
-static const char* stats_usage[] = { "Path to the statistics file" };
-
-static OptDef const Options[] =
-{
- { "stats", "s", NULL, stats_usage, 1, true, true }
-};
-
-rc_t CC UsageSummary (const char * progname) {
- return KOutMsg (
-"Usage:\n"
-" %s <db-path>\n"
-, progname);
- }
-
-rc_t CC Usage(const Args* args) {
- rc_t rc = 0 ;
-
- const char* progname = UsageDefaultName;
- const char* fullpath = UsageDefaultName;
-
- if (args == NULL)
- { rc = RC(rcApp, rcArgv, rcAccessing, rcSelf, rcNull); }
- else
- { rc = ArgsProgram(args, &fullpath, &progname); }
-
- UsageSummary(progname);
-
- KOutMsg("Parameters:\n");
-
- HelpParamLine ("db-path", param_usage);
-
- KOutMsg ("\nOptions:\n");
-
- HelpOptionLine ("s", "stats", NULL, stats_usage);
-
- HelpOptionsStandard ();
-
- HelpVersion (fullpath, KAppVersion());
-
- return rc;
-}
-
-const char UsageDefaultName[] = "qual-recal";
-
-ver_t CC KAppVersion(void) { return 0x1000000; }
-
-static rc_t ArgsRelease(Args* self) { return ArgsWhack(self); }
-
-rc_t CC KMain(int argc, char* argv[]) {
- rc_t rc = 0;
- Args* args = NULL;
-
- do {
- uint32_t pcount = 0;
-
- rc = ArgsMakeAndHandle(&args, argc, argv, 1,
- Options, sizeof Options / sizeof (OptDef));
- if (rc) {
- (void)LOGERR(klogErr, rc, "While calling ArgsMakeAndHandle");
- break;
- }
- rc = ArgsParamCount(args, &pcount);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure parsing database name");
- break;
- }
- if (pcount < 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
- MiniUsage(args);
- break;
- }
- if (pcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- (void)LOGERR(klogErr, rc, "Too many database parameters");
- break;
- }
- rc = ArgsParamValue(args, 0, &Params.dbPath);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure retrieving database name");
- break;
- }
-
- rc = ArgsOptionCount (args, "stats", &pcount);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure to get 'stats' argument");
- break;
- }
- if (pcount > 1) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcAmbiguous);
- (void)LOGERR(klogErr, rc, "Too many output parameters");
- break;
- }
- rc = ArgsOptionValue(args, "stats", 0, &Params.statPath);
- if (rc) {
- (void)LOGERR(klogErr, rc, "Failure retrieving stats file name");
- break;
- }
- } while (false);
-
- rc = rc ? rc : run();
- ArgsRelease(args);
- return rc;
-}
diff --git a/tools/qual-recalib-stat/Makefile b/tools/qual-recalib-stat/Makefile
index e3c9215..2f05cea 100644
--- a/tools/qual-recalib-stat/Makefile
+++ b/tools/qual-recalib-stat/Makefile
@@ -77,8 +77,6 @@ clean: stdclean
#
Q_RECAL_STAT_SRC = \
namelist_tools \
- progressbar \
- num-gen \
context \
columns \
ref_exclude \
diff --git a/tools/qual-recalib-stat/context.c b/tools/qual-recalib-stat/context.c
index c9079f5..0eb462a 100644
--- a/tools/qual-recalib-stat/context.c
+++ b/tools/qual-recalib-stat/context.c
@@ -204,7 +204,7 @@ static rc_t context_evaluate_arguments( const Args *my_args, p_context ctx )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *value = NULL;
- rc = ArgsParamValue( my_args, idx, &value );
+ rc = ArgsParamValue( my_args, idx, (const void **)&value );
if ( rc != 0 )
{
OUTMSG(( "ArgsParamValue() failed %R\n", rc ));
@@ -245,7 +245,7 @@ static const char* context_get_str_option( const Args *my_args,
rc_t rc = ArgsOptionCount( my_args, name, &count );
if ( ( rc == 0 )&&( count > 0 ) )
{
- rc = ArgsOptionValue( my_args, name, 0, &res );
+ rc = ArgsOptionValue( my_args, name, 0, (const void **)&res );
}
return res;
}
@@ -260,7 +260,7 @@ static uint32_t context_get_int_option( const Args *my_args,
if ( ( rc == 0 )&&( count > 0 ) )
{
const char *s;
- rc = ArgsOptionValue( my_args, name, 0, &s );
+ rc = ArgsOptionValue( my_args, name, 0, (const void **)&s );
if ( rc == 0 ) res = atoi( s );
}
return res;
diff --git a/tools/qual-recalib-stat/context.h b/tools/qual-recalib-stat/context.h
index 861678f..cfb7ad2 100644
--- a/tools/qual-recalib-stat/context.h
+++ b/tools/qual-recalib-stat/context.h
@@ -34,8 +34,8 @@ extern "C" {
#include <kapp/args.h>
#include <klib/namelist.h>
#include <klib/out.h>
+#include <klib/num-gen.h>
#include "namelist_tools.h"
-#include "num-gen.h"
#define OPTION_ROWS "rows"
#define OPTION_SCHEMA "schema"
@@ -68,7 +68,7 @@ typedef struct context
char *output_mode;
char *exclude_file_path;
const KNamelist *src_schema_list;
- num_gen *row_generator;
+ struct num_gen * row_generator;
bool usage_requested;
bool show_progress;
bool info;
diff --git a/tools/qual-recalib-stat/num-gen.c b/tools/qual-recalib-stat/num-gen.c
deleted file mode 100644
index baa4aa2..0000000
--- a/tools/qual-recalib-stat/num-gen.c
+++ /dev/null
@@ -1,848 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "num-gen.h"
-#include <klib/printf.h>
-
-#include <sysalloc.h>
-#include <strtol.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-typedef struct num_gen_node
-{
- uint64_t start;
- uint64_t end;
- uint64_t count; /* 0 ... skip, >0 ... valid */
-} num_gen_node;
-typedef struct num_gen_node * p_num_gen_node;
-
-/* **************************************************************************
-{ start:5,count: 0 } ---> [ ]
-{ start:5,count: 1 } ---> [ 5 ]
-{ start:5,count: 2 } ---> [ 5, 6 ]
- ************************************************************************** */
-
-struct num_gen
-{
- Vector nodes;
-};
-
-
-struct num_gen_iter
-{
- Vector nodes;
- uint32_t curr_node;
- uint32_t curr_node_sub_pos;
- uint64_t total;
- uint64_t progress;
-};
-
-/* forward decl. for fixing-function */
-static rc_t num_gen_fix_overlaps( num_gen* self, uint32_t *count );
-
-
-/* helper function to destroy a node*/
-static void CC num_gen_node_destroy( void *item, void *data )
-{
- free( item );
-}
-
-
-/* helper function to create a node from start/count */
-static p_num_gen_node num_gen_make_node( const uint64_t start, const uint64_t count )
-{
- p_num_gen_node p = ( p_num_gen_node )malloc( sizeof( num_gen_node ) );
- if ( p )
- {
- p->start = start;
- p->end = start + count - 1;
- p->count = count;
- }
- return p;
-}
-
-
-/* helper callback to compare 2 nodes, lets VectorInsert create a sorted vector */
-static int CC num_gen_insert_helper( const void* item1, const void* item2 )
-{
- const p_num_gen_node node1 = ( p_num_gen_node )item1;
- const p_num_gen_node node2 = ( p_num_gen_node )item2;
- int res = 0;
- if ( node1->start == node2->start )
- {
- if ( node1->count < node2->count )
- res = -1;
- else if ( node1->count > node2->count )
- res = 1;
- }
- else if ( node1->start < node2->start )
- res = -1;
- else
- res = 1;
- return res;
-}
-
-
-/* helper callback to create a deep and conditional copy of a node-vector */
-static void CC num_gen_copy_cb( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- if ( node->count > 0 )
- {
- Vector * dst = ( Vector *)data;
- p_num_gen_node new_node = num_gen_make_node( node->start, node->count );
- if ( new_node != NULL )
- VectorInsert( dst, new_node, NULL, num_gen_insert_helper );
- }
-}
-
-
-/* helper function that creates a deep and conditional copy of a node-vector */
-static void num_gen_copy_vector( const Vector * src, Vector * dst )
-{
- if ( src == NULL || dst == NULL )
- return;
- VectorForEach ( src, false, num_gen_copy_cb, dst );
-}
-
-
-/* helper callback to add up all count values in the vector*/
-static void CC num_gen_total_count_cb( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- if ( node != NULL )
- {
- uint64_t * total = ( uint64_t *)data;
- if ( total != NULL )
- *total += node->count;
- }
-}
-
-
-/* helper function that adds up all count values in the vector*/
-static uint64_t num_gen_total_count( const Vector * src )
-{
- uint64_t res = 0;
- if ( src != NULL )
- VectorForEach ( src, false, num_gen_total_count_cb, &res );
- return res;
-}
-
-
-/* helper function for the parse-function */
-static rc_t num_gen_add_node( num_gen* self, const uint64_t from,
- const uint64_t to )
-{
- p_num_gen_node node = NULL;
- int64_t count = ( to - from );
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcSelf, rcNull );
-
- if ( count >= 0 )
- node = num_gen_make_node( from, count + 1 );
- else
- node = num_gen_make_node( to, -( count + 1 ) );
- if ( node == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcMemory, rcExhausted );
- return VectorInsert( &(self->nodes), node, NULL, num_gen_insert_helper );
-}
-
-
-#define MAX_NUM_STR 12
-/* helper-structure for num_gen_parse() */
-typedef struct num_gen_parse_ctx
-{
- uint32_t num_str_idx;
- bool this_is_the_first_number;
- uint64_t num1;
- uint64_t num2;
- char num_str[ MAX_NUM_STR + 1 ];
-} num_gen_parse_ctx;
-typedef num_gen_parse_ctx* p_num_gen_parse_ctx;
-
-
-/* helper for num_gen_parse() */
-static void num_gen_convert_ctx( p_num_gen_parse_ctx ctx )
-{
- char *endp;
-
- ctx->num_str[ ctx->num_str_idx ] = 0;
- ctx->num1 = strtou64( ctx->num_str, &endp, 10 );
- ctx->this_is_the_first_number = false;
- ctx->num_str_idx = 0;
-}
-
-
-/* helper for num_gen_parse() */
-static rc_t num_gen_convert_and_add_ctx( num_gen* self, p_num_gen_parse_ctx ctx )
-{
- char *endp;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcSelf, rcNull );
- if ( ctx == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcParam, rcNull );
- if ( ctx->num_str_idx == 0 )
- return RC( rcVDB, rcNoTarg, rcInserting, rcParam, rcEmpty );
-
- /* terminate the source-string */
- ctx->num_str[ ctx->num_str_idx ] = 0;
- /* convert the string into a uint64_t */
- if ( ctx->this_is_the_first_number )
- {
- ctx->num1 = strtou64( ctx->num_str, &endp, 10 );
- ctx->num2 = ctx->num1;
- }
- else
- ctx->num2 = strtou64( ctx->num_str, &endp, 10 );
- /* empty the source-string to be reused */
- ctx->num_str_idx = 0;
-
- ctx->this_is_the_first_number = true;
- return num_gen_add_node( self, ctx->num1, ctx->num2 );
-}
-
-
-/* parse the given string and insert the found ranges
- into the number-generator, fixes eventual overlaps */
-rc_t num_gen_parse( num_gen* self, const char* src )
-{
- size_t i, n;
- num_gen_parse_ctx ctx;
- rc_t rc = 0;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcSelf, rcNull );
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcParam, rcNull );
-
- n = string_measure ( src, NULL );
- if ( n == 0 )
- return RC( rcVDB, rcNoTarg, rcParsing, rcParam, rcEmpty );
-
- ctx.num_str_idx = 0;
- ctx.this_is_the_first_number = true;
- for ( i = 0; i < n && rc == 0; ++i )
- {
- switch ( src[ i ] )
- {
- /* a dash switches from N1-mode into N2-mode */
- case '-' :
- num_gen_convert_ctx( &ctx );
- break;
-
- /* a comma ends a single number or a range */
- case ',' :
- rc = num_gen_convert_and_add_ctx( self, &ctx );
- break;
-
- /* in both mode add the char to the temp string */
- default:
- if ( ( src[i]>='0' )&&( src[i]<='9' )&&( ctx.num_str_idx < MAX_NUM_STR ) )
- ctx.num_str[ ctx.num_str_idx++ ] = src[ i ];
- break;
- }
- }
- /* dont forget to add what is left in ctx.num_str ... */
- if ( ctx.num_str_idx > 0 )
- rc = num_gen_convert_and_add_ctx( self, &ctx );
- if ( rc == 0 )
- rc = num_gen_fix_overlaps( self, NULL );
- return rc;
-}
-
-
-/* inserts the given ranges into the number-generator,
- fixes eventual overlaps */
-rc_t num_gen_add( num_gen* self, const uint64_t first, const uint64_t count )
-{
- rc_t rc;
- uint64_t num_1 = first;
- uint64_t num_2 = first;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcInserting, rcSelf, rcNull );
-
- /* this is necessary because virtual columns which have a
- infinite row-range, get reported with first=1,count=0 */
- if ( count > 0 )
- num_2 = ( first + count - 1 );
- rc = num_gen_add_node( self, num_1, num_2 );
- if ( rc == 0 )
- rc = num_gen_fix_overlaps( self, NULL );
- return rc;
-}
-
-
-/* helper function for range-check */
-static bool CC num_gen_check_range_start( p_num_gen_node the_node,
- const uint64_t range_start )
-{
- bool res = true;
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( the_node->start < range_start )
- {
- the_node->start = range_start;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- res = false;
- }
- }
- return res;
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_check_range_end( p_num_gen_node the_node,
- const uint64_t last_tab_row )
-{
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( last_node_row > last_tab_row )
- {
- last_node_row = last_tab_row;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- }
- }
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_check_range_callback( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- p_num_gen_node the_range = ( p_num_gen_node )data;
- uint64_t last_tab_row = ( the_range->start + the_range->count - 1 );
-
- /* ignore invalid nodes... */
- if ( the_node->start == 0 || the_node->count == 0 )
- return;
-
- /* check if the start value is not out of range... */
- if ( num_gen_check_range_start( the_node, the_range->start ) )
- num_gen_check_range_end( the_node, last_tab_row );
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_count_invalid_nodes( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- uint32_t *invalid_count = ( uint32_t * )data;
-
- if ( ( the_node->start == 0 )&&( the_node->count == 0 ) )
- ( *invalid_count )++;
-}
-
-
-/* helper function for range-check */
-static void CC num_gen_copy_valid_nodes( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- Vector *dest = ( Vector * )data;
-
- if ( ( node->start != 0 )&&( node->count != 0 ) )
- VectorInsert ( dest, node, NULL, num_gen_insert_helper );
- else
- free ( node );
-}
-
-
-/* helper function for range-check */
-static void num_gen_remove_invalid_nodes( num_gen* self )
-{
- Vector temp_nodes;
- uint32_t count = VectorLength( &(self->nodes) );
-
- if ( count < 1 )
- return;
- /* create a temp. vector */
- VectorInit( &temp_nodes, 0, count );
-
- /* copy all valid nodes into the temp. vector */
- VectorForEach ( &(self->nodes), false,
- num_gen_copy_valid_nodes, &temp_nodes );
-
- /* clear all nodes so far...,
- DO NOT PASS num_gen_node_destroy into it */
- VectorWhack( &(self->nodes), NULL, NULL );
-
- /* initialize and copy (shallow) the valid nodes back
- into the generator */
- VectorCopy ( &temp_nodes, &(self->nodes) );
-
- /* destroy the temp-vector,
- DO NOT PASS num_gen_node_destroy into it */
- VectorWhack ( &temp_nodes, NULL, NULL );
-}
-
-
-/* helper function for trim */
-rc_t num_gen_trim( num_gen* self, const int64_t first, const uint64_t count )
-{
- num_gen_node trim_range;
- uint32_t invalid_nodes = 0;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcValidating, rcSelf, rcNull );
- if ( count == 0 )
- return RC( rcVDB, rcNoTarg, rcValidating, rcParam, rcNull );
-
- /* walk all nodes to check for boundaries... */
- trim_range.start = first;
- trim_range.count = count;
-
- VectorForEach ( &(self->nodes), false,
- num_gen_check_range_callback, &trim_range );
-
- VectorForEach ( &(self->nodes), false,
- num_gen_count_invalid_nodes, &invalid_nodes );
- if ( invalid_nodes > 0 )
- num_gen_remove_invalid_nodes( self );
-
- return 0;
-}
-
-
-rc_t num_gen_make( num_gen** self )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
-
- *self = calloc( 1, sizeof( num_gen ) );
- if ( *self == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
-
- VectorInit( &((*self)->nodes ), 0, 5 );
- return 0;
-}
-
-
-rc_t num_gen_make_from_str( num_gen** self, const char *src )
-{
- rc_t rc = num_gen_make( self );
- if ( rc == 0 )
- {
- rc = num_gen_parse( *self, src );
- if ( rc == 0 )
- rc = num_gen_fix_overlaps( *self, NULL );
- }
- return rc;
-}
-
-
-rc_t num_gen_make_from_range( num_gen** self,
- const int64_t first, const uint64_t count )
-{
- rc_t rc = num_gen_make( self );
- if ( rc != 0 )
- return rc;
- return num_gen_add( *self, first, count );
-}
-
-
-rc_t num_gen_clear( num_gen* self )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcClearing, rcSelf, rcNull );
-
- if ( VectorLength( &(self->nodes) ) > 0 )
- {
- /* clear all nodes so far... */
- VectorWhack( &(self->nodes), num_gen_node_destroy, NULL );
-
- /* re-init the vector */
- VectorInit( &(self->nodes ), 0, 5 );
- }
- return 0;
-}
-
-
-rc_t num_gen_destroy( num_gen* self )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
-
- VectorWhack( &(self->nodes), num_gen_node_destroy, NULL );
- free( self );
- return 0;
-}
-
-
-bool num_gen_empty( const num_gen* self )
-{
- if ( self == NULL )
- return true;
- return ( VectorLength( &(self->nodes) ) < 1 );
-}
-
-
-typedef struct overlap_ctx
-{
- p_num_gen_node prev;
- uint32_t overlaps;
-} overlap_ctx;
-typedef overlap_ctx* p_overlap_ctx;
-
-
-/* static bool CC num_gen_overlap_fix_cb( void *item, void *data ) */
-static bool CC num_gen_overlap_fix_cb( void *item, void *data )
-{
- p_num_gen_node node = ( p_num_gen_node )item;
- p_overlap_ctx ctx = ( p_overlap_ctx )data;
-
- /* skip invalid nodes */
- if ( node->count ==0 || node->start == 0 || node->end == 0 )
- return false;
- /* if we do not have a previous node, take this one... */
- if ( ctx->prev == NULL )
- {
- ctx->prev = node;
- return false;
- }
- /* if we do not have an overlap,
- take this node as prev-node and continue */
- if ( ctx->prev->end < node->start )
- {
- ctx->prev = node;
- return false;
- }
- /* we have a overlap, the end of the prev-node is inside
- the current-node, we fix it by expanding the prev-node
- to the end of this node, and later declaring this
- node as invalid */
- if ( ctx->prev->end < node->end )
- {
- ctx->prev->end = node->end;
- ctx->prev->count = ( ctx->prev->end - ctx->prev->start ) + 1;
- }
- /* if the prev-node ends after this node, all we have to
- do is declaring this node as invalid */
- node->count = 0;
- node->start = 0;
- node->end = 0;
- return true;
-}
-
-
-static rc_t num_gen_fix_overlaps( num_gen* self, uint32_t *count )
-{
- overlap_ctx ctx;
- bool fix_executed = false;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
-
- ctx.overlaps = 0;
- do
- {
- ctx.prev = NULL;
- fix_executed = VectorDoUntil ( &(self->nodes), false,
- num_gen_overlap_fix_cb, &ctx );
- } while ( fix_executed );
-
- if ( count )
- *count = ctx.overlaps;
- return 0;
-}
-
-
-typedef struct string_ctx
-{
- char *s;
- uint32_t len;
-} string_ctx;
-typedef string_ctx* p_string_ctx;
-
-
-static void string_ctx_add( p_string_ctx ctx, char *s )
-{
- uint32_t len = string_measure ( s, NULL );
- if ( len > 0 )
- {
- if ( ctx->len == 0 )
- ctx->s = malloc( len + 1 );
- else
- ctx->s = realloc( ctx->s, ctx->len + len );
- memcpy( &(ctx->s[ctx->len]), s, len );
- ctx->len += len;
- }
-}
-
-
-static void CC num_gen_as_string_cb( void *item, void *data )
-{
- char temp[40];
- p_num_gen_node node = ( p_num_gen_node )item;
- long unsigned int start = node->start;
- long unsigned int end = ( start + node->count - 1 );
- switch( node->count )
- {
- case 0 : temp[0]=0;
- break;
- case 1 : string_printf ( temp, sizeof temp, NULL, "%lu,", start );
- break;
- default: string_printf ( temp, sizeof temp, NULL, "%lu-%lu,", start, end );
- break;
- }
- string_ctx_add( ( p_string_ctx )data, temp );
-}
-
-
-rc_t num_gen_as_string( const num_gen* self, char **s )
-{
- string_ctx ctx;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( s == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
-
- ctx.s = NULL;
- ctx.len = 0;
- VectorForEach ( &(self->nodes), false, num_gen_as_string_cb, &ctx );
- if ( ctx.len == 0 )
- {
- *s = NULL;
- return RC( rcVDB, rcNoTarg, rcReading, rcData, rcEmpty );
- }
- ctx.s[ ctx.len ] = 0;
- *s = ctx.s;
- return 0;
-}
-
-
-static void CC num_gen_debug_cb( void *item, void *data )
-{
- char temp[40];
- p_num_gen_node node = ( p_num_gen_node )item;
- long unsigned int start = node->start;
- long unsigned int count = node->count;
- string_printf ( temp, sizeof temp, NULL, "[s:%lu c:%lu]", start, count );
- string_ctx_add( ( p_string_ctx )data, temp );
-}
-
-
-rc_t num_gen_debug( const num_gen* self, char **s )
-{
- string_ctx ctx;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( s == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
-
- ctx.s = NULL;
- ctx.len = 0;
- VectorForEach ( &(self->nodes), false, num_gen_debug_cb, &ctx );
- if ( ctx.len == 0 )
- {
- *s = NULL;
- return RC( rcVDB, rcNoTarg, rcReading, rcData, rcEmpty );
- }
- ctx.s[ ctx.len ] = 0;
- *s = ctx.s;
- return 0;
-}
-
-
-static bool CC num_gen_contains_cb( void *item, void *data )
-{
- bool res = false;
- p_num_gen_node node = ( p_num_gen_node )item;
- if ( node->count > 0 )
- {
- uint64_t *value = ( uint64_t * )data;
- uint64_t end = node->start + node->count - 1;
- res = ( node->start <= *value && *value <= end );
- }
- return res;
-}
-
-
-rc_t num_gen_contains_value( const num_gen* self, const uint64_t value )
-{
- uint64_t temp = value;
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( VectorDoUntil ( &(self->nodes), false,
- num_gen_contains_cb, &temp ) )
- return 0;
- else
- return RC( rcVDB, rcNoTarg, rcReading, rcData, rcEmpty );
-}
-
-
-rc_t num_gen_range_check( num_gen* self,
- const int64_t first, const uint64_t count )
-{
- /* if the user did not specify a row-range, take all rows */
- if ( num_gen_empty( self ) )
- return num_gen_add( self, first, count );
- /* if the user did specify a row-range, check the boundaries */
- else
- return num_gen_trim( self, first, count );
-}
-
-
-rc_t num_gen_iterator_make( const num_gen* self, const num_gen_iter **iter )
-{
- uint32_t count;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( iter == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
-
- *iter = NULL;
- count = VectorLength( &(self->nodes) );
- if ( count < 1 )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- else
- {
- num_gen_iter *temp = calloc( 1, sizeof( num_gen_iter ) );
- if ( temp == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- VectorInit( &(temp->nodes), 0, count );
- num_gen_copy_vector( &(self->nodes), &(temp->nodes ) );
- temp->total = num_gen_total_count( &(temp->nodes ) );
- *iter = temp;
- }
- return 0;
-}
-
-rc_t num_gen_iterator_destroy( const num_gen_iter *self )
-{
- num_gen_iter *temp;
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
-
- temp = (num_gen_iter *)self;
- VectorWhack( &(temp->nodes), num_gen_node_destroy, NULL );
- free( temp );
- return 0;
-}
-
-rc_t num_gen_iterator_next( const num_gen_iter* self, uint64_t* value )
-{
- num_gen_iter* temp;
- p_num_gen_node node;
-
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( value == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- if ( self->curr_node >= VectorLength( &(self->nodes) ) )
- return RC( rcVDB, rcNoTarg, rcReading, rcId, rcInvalid );
-
- temp = ( num_gen_iter *)self;
- *value = 0;
- node = (p_num_gen_node)VectorGet( &(temp->nodes), temp->curr_node );
- if ( node == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcItem, rcInvalid );
-
- *value = node->start;
- if ( node->count < 2 )
- /* the node is a single-number-node, next node for next time */
- temp->curr_node++;
- else
- {
- /* the node is a number range, add the sub-position */
- *value += temp->curr_node_sub_pos++;
- /* if the sub-positions are use up, switch to next node */
- if ( temp->curr_node_sub_pos >= node->count )
- {
- temp->curr_node++;
- temp->curr_node_sub_pos = 0;
- }
- }
- (temp->progress)++;
- return 0;
-}
-
-
-rc_t num_gen_iterator_count( const num_gen_iter* self, uint64_t* count )
-{
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( count == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- *count = self->total;
- return 0;
-}
-
-
-rc_t num_gen_iterator_percent( const num_gen_iter* self,
- const uint8_t fract_digits,
- uint32_t* value )
-{
- uint32_t factor = 100;
- if ( self == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcSelf, rcNull );
- if ( value == NULL )
- return RC( rcVDB, rcNoTarg, rcReading, rcParam, rcNull );
- if ( fract_digits > 0 )
- {
- if ( fract_digits > 1 )
- factor = 10000;
- else
- factor = 1000;
- }
-
- if ( self->total > 0 )
- {
- if ( self->progress >= self->total )
- *value = factor;
- else
- {
- uint64_t temp = self->progress;
- temp *= factor;
- temp /= self->total;
- *value = (uint16_t) temp;
- }
- }
- else
- *value = 0;
- return 0;
-}
diff --git a/tools/qual-recalib-stat/num-gen.h b/tools/qual-recalib-stat/num-gen.h
deleted file mode 100644
index 5b0324f..0000000
--- a/tools/qual-recalib-stat/num-gen.h
+++ /dev/null
@@ -1,225 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_num_gen_
-#define _h_num_gen_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef _h_klib_vector_
-#include <klib/vector.h>
-#endif
-
-#ifndef _h_klib_rc_
-#include <klib/rc.h>
-#endif
-
-#ifndef _h_klib_text_
-#include <klib/text.h>
-#endif
-
-
-/*--------------------------------------------------------------------------
- * A NUMBER GENERATOR
- *
- * input : string, for instance "3,6,8,12,44-49"
- * ouptut: sequence of integers, for instance 3,6,8,12,44,45,46,47,48,49
- */
-
-
-/*--------------------------------------------------------------------------
- * opaque number-generator and it's iterator
- */
-typedef struct num_gen num_gen;
-typedef struct num_gen_iter num_gen_iter;
-
-
-/*--------------------------------------------------------------------------
- * num_gen_make
- *
- * creates a empty number-generator
- * or creates a number-generator and parses the string
- * or creates and presets it with a range
- */
-rc_t num_gen_make( num_gen** self );
-rc_t num_gen_make_from_str( num_gen** self, const char *src );
-rc_t num_gen_make_from_range( num_gen** self,
- const int64_t first, const uint64_t count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_destroy
- *
- * destroys a number-generator
- */
-rc_t num_gen_destroy( num_gen* self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_clear
- *
- * resets a number-generator, to be empty just like after num_gen_make()
- */
-rc_t num_gen_clear( num_gen* self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_parse
- *
- * parses a given string in this form: "3,6,8,12,44-49"
- * does not clear the number-generator before parsing
- * eventual overlaps with the previous content are consolidated
- */
-rc_t num_gen_parse( num_gen* self, const char* src );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_add
- *
- * inserts the given interval into the number-generator
- *
- * num_gen_add( *g, 10, 30 )
- * is equivalent to:
- * num_gen_parse( *g, "10-39" );
- *
- * eventual overlaps with the previous content are consolidated
- */
-rc_t num_gen_add( num_gen* self, const uint64_t first, const uint64_t count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_trim
- *
- * checks if the content of the number-generator is inside the given interval
- * removes or shortens internal nodes if necessary
- */
-rc_t num_gen_trim( num_gen* self, const int64_t first, const uint64_t count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_empty
- *
- * checks if the generator has no ranges defined
- */
-bool num_gen_empty( const num_gen* self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_as_string
- *
- * allocates a string that contains the generator as text
- * *s = "1-5,20,24-25"
- * caller has to free the string
- */
-rc_t num_gen_as_string( const num_gen* self, char **s );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_debug
- *
- * allocates a string that contains the internal intervals as text
- * *s = "[s:1 c:5][s:20 c:1][s:24 c:2]"
- * [s...start-value c:count]
- * caller has to free the string
- */
-rc_t num_gen_debug( const num_gen* self, char **s );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_contains_value
- *
- * checks if the generator contains the given value
- */
-rc_t num_gen_contains_value( const num_gen* self, const uint64_t value );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_range_check
- *
- * if the generator is empty --> set it to the given range
- * if it is not empty ---------> trim it to the given range
- */
-rc_t num_gen_range_check( num_gen* self,
- const int64_t first, const uint64_t count );
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_make
- *
- * creates a iterator from the number-generator
- * the iterator contains a constant copy of the number-ranges
- * after this call it is safe to destroy or change the number-generator
- * returns an error-code if the number-generator was empty,
- * and *iter will be NULL
- */
-rc_t num_gen_iterator_make( const num_gen* self, const num_gen_iter **iter );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_destroy
- *
- * destroys the iterator
- */
-rc_t num_gen_iterator_destroy( const num_gen_iter *self );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_count
- *
- * returns how many values the iterator contains
- */
-rc_t num_gen_iterator_count( const num_gen_iter* self, uint64_t* count );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_next
- *
- * pulls the next value out of the iterator...
- * returns an error-code if the iterator has no more values
- */
-rc_t num_gen_iterator_next( const num_gen_iter* self, uint64_t* value );
-
-
-/*--------------------------------------------------------------------------
- * num_gen_iterator_percent
- *
- * return in value the percentage of the iterator...
- * depending on fract-digits the percentage will be:
- * fract_digits = 0 ... full percent's
- * fract_digits = 1 ... 1/10-th of a percent
- * fract_digits = 2 ... 1/100-th of a percent
- */
-rc_t num_gen_iterator_percent( const num_gen_iter* self,
- const uint8_t fract_digits,
- uint32_t* value );
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/qual-recalib-stat/progressbar.c b/tools/qual-recalib-stat/progressbar.c
deleted file mode 100644
index 7e0ab38..0000000
--- a/tools/qual-recalib-stat/progressbar.c
+++ /dev/null
@@ -1,182 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/out.h>
-#include <klib/rc.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-
-typedef struct progressbar
-{
- bool initialized;
- uint16_t percent;
-} progressbar;
-
-
-rc_t make_progressbar( progressbar ** pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- (*pb) = calloc( 1, sizeof( progressbar ) );
- if ( *pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- return 0;
-}
-
-
-rc_t destroy_progressbar( progressbar * pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
- free( pb );
- return 0;
-}
-
-static void progess_0a( const uint16_t percent )
-{
- OUTMSG (( "| %2u%%", percent ));
-}
-
-static void progess_0( const uint16_t percent )
-{
- if ( percent & 1 )
- OUTMSG (( "\b\b\b\b- %2u%%", percent ));
- else
- OUTMSG (( "\b\b\b%2u%%", percent ));
-}
-
-static void progess_1a( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- OUTMSG (( "| %2u.%01u%%", p1, p0 ));
-}
-
-static void progess_1( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- OUTMSG (( "\b\b\b\b\b\b- %2u.%01u%%", p1, p0 ));
- else
- OUTMSG (( "\b\b\b\b\b%2u.%01u%%", p1, p0 ));
-}
-
-static void progess_2a( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- OUTMSG (( "| %2u.%02u%%", p1, p0 ));
-}
-
-static void progess_2( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- OUTMSG (( "\b\b\b\b\b\b\b- %2u.%02u%%", p1, p0 ));
- else
- OUTMSG (( "\b\b\b\b\b\b%2u.%02u%%", p1, p0 ));
-}
-
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent )
-{
- uint8_t digits = fract_digits;
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcSelf, rcNull );
- if ( digits > 2 )
- digits = 2;
- if ( pb->initialized )
- {
- if ( pb->percent != percent )
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0( percent ); break;
- case 1 : progess_1( percent ); break;
- case 2 : progess_2( percent ); break;
- }
- }
- }
- else
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0a( percent ); break;
- case 1 : progess_1a( percent ); break;
- case 2 : progess_2a( percent ); break;
- }
- pb->initialized = true;
- }
- return 0;
-}
-
-
-uint8_t progressbar_calc_fract_digits( const uint64_t count )
-{
- uint8_t res = 0;
- if ( count > 10000 )
- {
- if ( count > 100000 )
- res = 2;
- else
- res = 1;
- }
- return res;
-}
-
-
-uint32_t progressbar_percent( const uint64_t count, const uint64_t value,
- const uint8_t fract_digits )
-{
- uint32_t factor = 100;
- uint32_t res = 0;
-
- if ( fract_digits > 0 )
- {
- if ( fract_digits > 1 )
- factor = 10000;
- else
- factor = 1000;
- }
-
- if ( count > 0 )
- {
- if ( value >= count )
- res = factor;
- else
- {
- uint64_t temp = value;
- temp *= factor;
- temp /= count;
- res = (uint16_t) temp;
- }
- }
- return res;
-}
diff --git a/tools/qual-recalib-stat/progressbar.h b/tools/qual-recalib-stat/progressbar.h
deleted file mode 100644
index f5f04bc..0000000
--- a/tools/qual-recalib-stat/progressbar.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_progressbar_
-#define _h_progressbar_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct progressbar progressbar;
-
-/*--------------------------------------------------------------------------
- * make_progressbar
- *
- * creates a progressbar with zero-values inside
- * does not output anything
- */
-rc_t make_progressbar( progressbar ** pb );
-
-
-/*--------------------------------------------------------------------------
- * destroy_progressbar
- *
- * destroy's the progressbar
- * does not output anything
- */
-rc_t destroy_progressbar( progressbar * pb );
-
-
-/*--------------------------------------------------------------------------
- * update_progressbar
- *
- * sets the progressbar to a specific percentage
- * outputs only if the percentage has changed from the last call
- * the precentage is in 1/10-th of a percent ( 21,6% = 216 )
- * expects the percents in increasing order ( does not jump back )
- * writes a growing bar made from '-'-chars every 2nd percent
- */
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent );
-
-uint8_t progressbar_calc_fract_digits( const uint64_t count );
-
-uint32_t progressbar_percent( const uint64_t count, const uint64_t value,
- const uint8_t fract_digits );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/qual-recalib-stat/qual-recalib-stat.c b/tools/qual-recalib-stat/qual-recalib-stat.c
index f926883..da63345 100644
--- a/tools/qual-recalib-stat/qual-recalib-stat.c
+++ b/tools/qual-recalib-stat/qual-recalib-stat.c
@@ -37,9 +37,10 @@
#include <klib/log.h>
#include <klib/out.h>
#include <klib/printf.h>
+#include <klib/num-gen.h>
+#include <klib/progressbar.h>
#include "context.h"
-#include "progressbar.h"
#include "stat_mod_2.h"
#include "reader.h"
#include "writer.h"
@@ -125,13 +126,19 @@ ver_t CC KAppVersion ( void )
}
-static uint8_t calc_fract_digits( const num_gen_iter *iter )
+static uint8_t calc_fract_digits( const struct num_gen_iter *iter )
{
uint8_t res = 0;
uint64_t count;
if ( num_gen_iterator_count( iter, &count ) == 0 )
{
- res = progressbar_calc_fract_digits( count );
+ if ( count > 10000 )
+ {
+ if ( count > 100000 )
+ res = 2;
+ else
+ res = 1;
+ }
}
return res;
}
@@ -164,26 +171,26 @@ static rc_t read_loop( statistic * data,
if ( rc == 0 )
{
- const num_gen_iter *iter;
+ const struct num_gen_iter *iter;
rc = num_gen_iterator_make( ctx->row_generator, &iter );
if ( rc != 0 )
LogErr( klogInt, rc, "num_gen_iterator_make() failed\n" );
else
{
- uint64_t row_id;
- progressbar * progress;
+ int64_t row_id;
+ uint8_t fract_digits = calc_fract_digits( iter );
+ struct progressbar * progress;
- rc = make_progressbar( &progress );
+
+ rc = make_progressbar( &progress, fract_digits );
if ( rc != 0 )
LogErr( klogInt, rc, "make_progressbar() failed\n" );
else
{
- uint8_t fract_digits = calc_fract_digits( iter );
uint32_t percent;
row_input row_data;
- while ( ( num_gen_iterator_next( iter, &row_id ) == 0 )&&
- ( rc == 0 ) )
+ while ( num_gen_iterator_next( iter, &row_id, &rc ) && rc == 0 )
{
rc = Quitting();
if ( rc == 0 )
@@ -195,9 +202,11 @@ static rc_t read_loop( statistic * data,
rc = extract_statistic_from_row( data, &row_data, row_id );
}
/* ******************************************** */
- if ( ctx->show_progress )
- if ( num_gen_iterator_percent( iter, fract_digits, &percent ) == 0 )
- update_progressbar( progress, fract_digits, percent );
+ if ( ctx->show_progress &&
+ num_gen_iterator_percent( iter, fract_digits, &percent ) == 0 )
+ {
+ update_progressbar( progress, percent );
+ }
}
}
destroy_progressbar( progress );
@@ -391,8 +400,10 @@ rc_t CC KMain( int argc, char * argv[] )
LogErr( klogInt, rc, "context_capture_arguments_and_options() failed\n" );
else
{
- if ( ctx->usage_requested )
+ if ( ctx->usage_requested ) {
MiniUsage( args );
+ rc = RC(rcApp, rcArgv, rcParsing, rcParam, rcInsufficient);
+ }
else
{
switch( ctx->output_mode[ 0 ] )
diff --git a/tools/qual-recalib-stat/qual-recalib-stat.vers b/tools/qual-recalib-stat/qual-recalib-stat.vers
index 8e8299d..35d16fb 100644
--- a/tools/qual-recalib-stat/qual-recalib-stat.vers
+++ b/tools/qual-recalib-stat/qual-recalib-stat.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/qual-recalib-stat/qual-recalib-stat.vers.h b/tools/qual-recalib-stat/qual-recalib-stat.vers.h
index bb1ba3d..fb98dec 100644
--- a/tools/qual-recalib-stat/qual-recalib-stat.vers.h
+++ b/tools/qual-recalib-stat/qual-recalib-stat.vers.h
@@ -1 +1 @@
-#define QUAL_RECALIB_STAT_VERS 0x02040002
+#define QUAL_RECALIB_STAT_VERS 0x02050007
diff --git a/tools/qual-recalib-stat/ref_exclude.c b/tools/qual-recalib-stat/ref_exclude.c
index 327b0c0..8512dc9 100644
--- a/tools/qual-recalib-stat/ref_exclude.c
+++ b/tools/qual-recalib-stat/ref_exclude.c
@@ -63,7 +63,7 @@ static void CC whack_trans_node( BSTNode *n, void *data )
}
-static int CC trans_node_find( const void *item, const BSTNode *n )
+static int64_t CC trans_node_find( const void *item, const BSTNode *n )
{
trans_node * node = ( trans_node * ) n;
return StringCompare ( ( String * ) item, &node->chromosome );
@@ -130,7 +130,7 @@ static const char * translations[ N_TRANS_NODES ] =
};
-static int CC trans_node_sort( const BSTNode *item, const BSTNode *n )
+static int64_t CC trans_node_sort( const BSTNode *item, const BSTNode *n )
{
trans_node * rn1 = ( trans_node* ) item;
trans_node * rn2 = ( trans_node* ) n;
@@ -432,7 +432,7 @@ static void CC whack_ref_node( BSTNode *n, void *data )
}
-static int CC ref_node_find( const void *item, const BSTNode *n )
+static int64_t CC ref_node_find( const void *item, const BSTNode *n )
{
ref_node * node = ( ref_node * ) n;
return StringCompare ( ( String * ) item, node->name );
@@ -485,7 +485,7 @@ rc_t make_ref_exclude( ref_exclude *exclude, KDirectory *dir,
}
-static int CC ref_node_sort( const BSTNode *item, const BSTNode *n )
+static int64_t CC ref_node_sort( const BSTNode *item, const BSTNode *n )
{
ref_node * rn1 = ( ref_node* ) item;
ref_node * rn2 = ( ref_node* ) n;
diff --git a/tools/qual-recalib-stat/stat_mod.c b/tools/qual-recalib-stat/stat_mod.c
index 0d86725..4cd927f 100644
--- a/tools/qual-recalib-stat/stat_mod.c
+++ b/tools/qual-recalib-stat/stat_mod.c
@@ -195,7 +195,7 @@ static spotgrp * make_spotgrp( const char *src, const size_t len )
}
-static int CC spotgroup_find( const void *item, const BSTNode *n )
+static int64_t CC spotgroup_find( const void *item, const BSTNode *n )
{
spotgrp * sg = ( spotgrp* ) n;
return StringCompare ( ( String* ) item, sg->name );
@@ -315,7 +315,7 @@ static rc_t spotgroup_enter_values( spotgrp * sg,
}
-static int CC spotgroup_sort( const BSTNode *item, const BSTNode *n )
+static int64_t CC spotgroup_sort( const BSTNode *item, const BSTNode *n )
{
spotgrp * sg1 = ( spotgrp* ) item;
spotgrp * sg2 = ( spotgrp* ) n;
diff --git a/tools/qual-recalib-stat/stat_mod_1.c b/tools/qual-recalib-stat/stat_mod_1.c
index 3862c33..f4df109 100644
--- a/tools/qual-recalib-stat/stat_mod_1.c
+++ b/tools/qual-recalib-stat/stat_mod_1.c
@@ -156,7 +156,7 @@ static spotgrp * make_spotgrp( const char *src, const size_t len )
}
-static int CC spotgroup_find( const void *item, const BSTNode *n )
+static int64_t CC spotgroup_find( const void *item, const BSTNode *n )
{
spotgrp * sg = ( spotgrp* ) n;
return StringCompare ( ( String* ) item, sg->name );
@@ -265,7 +265,7 @@ static rc_t spotgroup_enter_values( spotgrp * spotgroup,
}
-static int CC spotgroup_sort( const BSTNode *item, const BSTNode *n )
+static int64_t CC spotgroup_sort( const BSTNode *item, const BSTNode *n )
{
spotgrp * sg1 = ( spotgrp* ) item;
spotgrp * sg2 = ( spotgrp* ) n;
diff --git a/tools/qual-recalib-stat/stat_mod_2.c b/tools/qual-recalib-stat/stat_mod_2.c
index aeffbd3..a9d1f92 100644
--- a/tools/qual-recalib-stat/stat_mod_2.c
+++ b/tools/qual-recalib-stat/stat_mod_2.c
@@ -304,7 +304,7 @@ static spotgrp * make_spotgrp( const char *src, const size_t len )
}
-static int CC spotgroup_find( const void *item, const BSTNode *n )
+static int64_t CC spotgroup_find( const void *item, const BSTNode *n )
{
spotgrp * sg = ( spotgrp* ) n;
return StringCompare ( ( String* ) item, sg->name );
@@ -444,7 +444,7 @@ static rc_t spotgroup_enter_values( spotgrp * spotgroup,
}
-static int CC spotgroup_sort( const BSTNode *item, const BSTNode *n )
+static int64_t CC spotgroup_sort( const BSTNode *item, const BSTNode *n )
{
spotgrp * sg1 = ( spotgrp* ) item;
spotgrp * sg2 = ( spotgrp* ) n;
diff --git a/tools/qual-recalib-stat/writer.c b/tools/qual-recalib-stat/writer.c
index 92c5050..b133358 100644
--- a/tools/qual-recalib-stat/writer.c
+++ b/tools/qual-recalib-stat/writer.c
@@ -27,6 +27,7 @@
#include "writer.h"
#include <klib/printf.h>
+#include <klib/progressbar.h>
#include <kfs/file.h>
#include <sysalloc.h>
@@ -81,13 +82,56 @@ typedef struct writer_ctx
statistic_writer *writer;
rc_t rc;
- progressbar *progress;
+ struct progressbar * progress;
uint64_t entries;
uint64_t n;
uint8_t fract_digits;
} writer_ctx;
+static uint8_t progressbar_calc_fract_digits( const uint64_t count )
+{
+ uint8_t res = 0;
+ if ( count > 10000 )
+ {
+ if ( count > 100000 )
+ res = 2;
+ else
+ res = 1;
+ }
+ return res;
+}
+
+static uint32_t progressbar_percent( const uint64_t count, const uint64_t value,
+ const uint8_t fract_digits )
+{
+ uint32_t factor = 100;
+ uint32_t res = 0;
+
+ if ( fract_digits > 0 )
+ {
+ if ( fract_digits > 1 )
+ factor = 10000;
+ else
+ factor = 1000;
+ }
+
+ if ( count > 0 )
+ {
+ if ( value >= count )
+ res = factor;
+ else
+ {
+ uint64_t temp = value;
+ temp *= factor;
+ temp /= count;
+ res = (uint16_t) temp;
+ }
+ }
+ return res;
+}
+
+
static bool CC write_cb( stat_row * row, void * data )
{
writer_ctx * ctx = ( writer_ctx * ) data;
@@ -150,7 +194,7 @@ static bool CC write_cb( stat_row * row, void * data )
if ( ctx->progress != NULL && rc == 0 )
{
uint32_t percent = progressbar_percent( ctx->entries, ++( ctx->n ), ctx->fract_digits );
- update_progressbar( ctx->progress, ctx->fract_digits, percent );
+ update_progressbar( ctx->progress, percent );
}
return ( rc == 0 );
@@ -169,9 +213,9 @@ static rc_t write_statistic( statistic_writer *writer, statistic *data,
if ( show_progress )
{
- make_progressbar( &ctx.progress );
ctx.entries = data->entries;
ctx.fract_digits = progressbar_calc_fract_digits( ctx.entries );
+ make_progressbar( &ctx.progress, ctx.fract_digits );
ctx.n = 0;
}
@@ -204,7 +248,7 @@ typedef struct write_ctx
uint64_t pos;
uint64_t lines;
- progressbar *progress;
+ struct progressbar *progress;
uint64_t entries;
uint8_t fract_digits;
} write_ctx;
@@ -236,7 +280,7 @@ static bool CC write_to_file_cb( stat_row * row, void * f_data )
if ( rc == 0 )
{
uint32_t percent = progressbar_percent( wctx->entries, ++wctx->lines, wctx->fract_digits );
- update_progressbar( wctx->progress, wctx->fract_digits, percent );
+ update_progressbar( wctx->progress, percent );
wctx->pos += f_writ;
}
}
@@ -275,10 +319,10 @@ rc_t write_output_file( KDirectory *dir, statistic * data,
if ( written != NULL ) *written = f_writ;
wctx.pos += f_writ;
- make_progressbar( &wctx.progress );
wctx.entries = data->entries;
wctx.fract_digits = progressbar_calc_fract_digits( wctx.entries );
-
+ make_progressbar( &wctx.progress, wctx.fract_digits );
+
foreach_statistic( data, write_to_file_cb, &wctx );
destroy_progressbar( wctx.progress );
diff --git a/tools/qual-recalib-stat/writer.h b/tools/qual-recalib-stat/writer.h
index a3c4583..7dffc3f 100644
--- a/tools/qual-recalib-stat/writer.h
+++ b/tools/qual-recalib-stat/writer.h
@@ -41,7 +41,6 @@ extern "C" {
#include <vdb/database.h>
#include "columns.h"
#include "stat_mod_2.h"
-#include "progressbar.h"
#define WIDX_SPOT_GROUP 0
#define WIDX_KMER 1
diff --git a/tools/rd-filter-redact/rd-filter-redact.c b/tools/rd-filter-redact/rd-filter-redact.c
index cc12686..920cf29 100644
--- a/tools/rd-filter-redact/rd-filter-redact.c
+++ b/tools/rd-filter-redact/rd-filter-redact.c
@@ -1277,7 +1277,7 @@ rc_t KMain(int argc, char* argv[])
LOGERR (klogFatal, rc, "Too many table parameters");
break;
}
- rc = ArgsOptionValue (args, OPTION_TABLE, 0, &sargs.table);
+ rc = ArgsOptionValue (args, OPTION_TABLE, 0, (const void **)&sargs.table);
if (rc)
{
LOGERR (klogFatal, rc, "Failure retrieving table name");
@@ -1303,7 +1303,7 @@ rc_t KMain(int argc, char* argv[])
LOGERR (klogFatal, rc, "Too many file parameters");
break;
}
- rc = ArgsOptionValue (args, OPTION_FILE, 0, &context.file_path);
+ rc = ArgsOptionValue (args, OPTION_FILE, 0, (const void **)&context.file_path);
if (rc)
{
LOGERR (klogFatal, rc, "Failure retrieving file name");
@@ -1330,7 +1330,7 @@ rc_t KMain(int argc, char* argv[])
LOGERR (klogFatal, rc, "Too many schema parameters");
break;
}
- rc = ArgsOptionValue (args, OPTION_SCHEMA, 0, &context.schema_path);
+ rc = ArgsOptionValue (args, OPTION_SCHEMA, 0, (const void **)&context.schema_path);
if (rc)
{
LOGERR (klogFatal, rc, "Failure retrieving schema name");
diff --git a/tools/rd-filter-redact/rd-filter-redact.vers b/tools/rd-filter-redact/rd-filter-redact.vers
index 8e8299d..35d16fb 100644
--- a/tools/rd-filter-redact/rd-filter-redact.vers
+++ b/tools/rd-filter-redact/rd-filter-redact.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/rd-filter-redact/read-filter-redact.c b/tools/rd-filter-redact/read-filter-redact.c
index a6515d5..7e3a420 100644
--- a/tools/rd-filter-redact/read-filter-redact.c
+++ b/tools/rd-filter-redact/read-filter-redact.c
@@ -321,7 +321,7 @@ static rc_t SpotIteratorReadSpotToRedact(SpotIterator* self)
}
}
- sscanf(self->buffer, "%uld", &spot);
+ sscanf(self->buffer, "%ld", &spot);
if (spot == 0) {
rc = RC(rcExe, rcFile, rcReading, rcString, rcInvalid);
@@ -796,7 +796,7 @@ static rc_t CmdLineInit(const Args* args, CmdLine* cmdArgs)
LOGERR(klogErr, rc, "Too many file parameters");
break;
}
- rc = ArgsOptionValue (args, OPTION_FILE, 0, &cmdArgs->file);
+ rc = ArgsOptionValue (args, OPTION_FILE, 0, (const void **)&cmdArgs->file);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving file name");
break;
@@ -818,7 +818,7 @@ static rc_t CmdLineInit(const Args* args, CmdLine* cmdArgs)
LOGERR(klogErr, rc, "Too many table parameters");
break;
}
- rc = ArgsParamValue(args, 0, &cmdArgs->table);
+ rc = ArgsParamValue(args, 0, (const void **)&cmdArgs->table);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving table name");
break;
diff --git a/tools/rd-filter-redact/read-filter-redact.vers b/tools/rd-filter-redact/read-filter-redact.vers
index 8e8299d..35d16fb 100644
--- a/tools/rd-filter-redact/read-filter-redact.vers
+++ b/tools/rd-filter-redact/read-filter-redact.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/rd-filter-redact/read-filter-redact.vers.h b/tools/rd-filter-redact/read-filter-redact.vers.h
index 74de6b5..21caf52 100644
--- a/tools/rd-filter-redact/read-filter-redact.vers.h
+++ b/tools/rd-filter-redact/read-filter-redact.vers.h
@@ -1 +1 @@
-#define READ_FILTER_REDACT_VERS 0x02040002
+#define READ_FILTER_REDACT_VERS 0x02050007
diff --git a/tools/rd-filter-redact/test-read-write-cursor.vers b/tools/rd-filter-redact/test-read-write-cursor.vers
index 8e8299d..35d16fb 100644
--- a/tools/rd-filter-redact/test-read-write-cursor.vers
+++ b/tools/rd-filter-redact/test-read-write-cursor.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/check-blob-size/Makefile b/tools/ref-variation/Makefile
similarity index 70%
rename from tools/check-blob-size/Makefile
rename to tools/ref-variation/Makefile
index 0d213eb..eab4f1e 100644
--- a/tools/check-blob-size/Makefile
+++ b/tools/ref-variation/Makefile
@@ -26,24 +26,25 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/check-blob-size
+MODULE = tools/ref-variation
include $(TOP)/build/Makefile.env
-INT_TOOLS = \
+INT_TOOLS =
EXT_TOOLS = \
- check-blob-size
+ ref-variation \
+ var-expand
ALL_TOOLS = \
$(INT_TOOLS) \
$(EXT_TOOLS)
-#------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
# outer targets
#
all std: vers-includes
- @ $(MAKE_CMD) $(TARGDIR)/$@
+ @ $(MAKE_CMD) $(TARGDIR)/std
$(ALL_TOOLS): vers-includes
@ $(MAKE_CMD) $(BINDIR)/$@
@@ -51,14 +52,6 @@ $(ALL_TOOLS): vers-includes
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
-# all
-#
-$(TARGDIR)/all: \
- $(addprefix $(BINDIR)/,$(ALL_TOOLS))
-
-.PHONY: $(TARGDIR)/all
-
-#-------------------------------------------------------------------------------
# std
#
$(TARGDIR)/std: \
@@ -81,23 +74,53 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# check-blob-size
+# ref-variation
#
-CHECK-BLOB-SIZE_SRC = \
- check-blob-size \
-CHECK-BLOB-SIZE_OBJ = \
- $(addsuffix .$(OBJX),$(CHECK-BLOB-SIZE_SRC))
+CPFLAGS += -pthread
-CHECK-BLOB-SIZE_LIB = \
- -lkapp \
- -lncbi-vdb \
+REF_VARIATION_SRC = \
+ ref-variation \
+ common \
+ helper
+REF_VARIATION_OBJ = \
+ $(addsuffix .$(OBJX),$(REF_VARIATION_SRC))
-check-blob-size.vers.h: check-blob-size.vers
- $(TOP)/build/make-vers-inc.sh CHECK_BLOB_SIZE_VERS $^ > $@
+REF_VARIATION_LIB = \
+ -sncbi-ngs-c++ \
+ -sncbi-vdb-static \
+ -sngs-c++ \
+ -skapp \
+ -sncbi-vdb \
+ -sm \
+ -sload
-$(BINDIR)/check-blob-size: $(CHECK-BLOB-SIZE_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(CHECK-BLOB-SIZE_LIB)
+$(BINDIR)/ref-variation: $(REF_VARIATION_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(REF_VARIATION_LIB)
#-------------------------------------------------------------------------------
+# var-expand
+#
+
+#CPFLAGS += -pthread
+
+VAR_EXPAND_SRC = \
+ var-expand \
+ common \
+ helper
+
+VAR_EXPAND_OBJ = \
+ $(addsuffix .$(OBJX),$(VAR_EXPAND_SRC))
+
+VAR_EXPAND_LIB = \
+ -sncbi-ngs-c++ \
+ -sncbi-vdb-static \
+ -sngs-c++ \
+ -skapp \
+ -sncbi-vdb \
+ -sm \
+ -sload
+
+$(BINDIR)/var-expand: $(VAR_EXPAND_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(VAR_EXPAND_LIB)
diff --git a/tools/ref-variation/common.cpp b/tools/ref-variation/common.cpp
new file mode 100644
index 0000000..8a4f758
--- /dev/null
+++ b/tools/ref-variation/common.cpp
@@ -0,0 +1,70 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "common.h"
+#include "helper.h"
+#include <search/grep.h>
+
+namespace Common
+{
+ bool find_variation_core_step (KSearch::CVRefVariation& obj,
+ char const* ref_slice, size_t ref_slice_size,
+ size_t& ref_pos_in_slice,
+ char const* var, size_t var_len, size_t var_len_on_ref,
+ size_t chunk_size, size_t chunk_no_last,
+ size_t& bases_start, size_t& chunk_no_start, size_t& chunk_no_end)
+ {
+ bool cont = false;
+
+ if ( ref_pos_in_slice + var_len_on_ref >= ref_slice_size )
+ {
+ cont = true;
+ ++chunk_no_end;
+ }
+ else
+ {
+ obj = KSearch::VRefVariationIUPACMake ( refvarAlgRA,
+ ref_slice, ref_slice_size,
+ ref_pos_in_slice, var, var_len, var_len_on_ref, bases_start );
+
+ if ( obj.GetAlleleStartRelative() == 0 && chunk_no_start > 0 )
+ {
+ cont = true;
+ --chunk_no_start;
+ ref_pos_in_slice += chunk_size;
+ bases_start -= chunk_size;
+ }
+ if (obj.GetAlleleStartRelative() + obj.GetAlleleLenOnRef() == ref_slice_size &&
+ chunk_no_end < chunk_no_last )
+ {
+ cont = true;
+ ++chunk_no_end;
+ }
+ }
+
+ return cont;
+ }
+}
diff --git a/tools/vdb-dump/vdb_info.h b/tools/ref-variation/common.h
similarity index 73%
copy from tools/vdb-dump/vdb_info.h
copy to tools/ref-variation/common.h
index f0bde80..a01413d 100644
--- a/tools/vdb-dump/vdb_info.h
+++ b/tools/ref-variation/common.h
@@ -1,4 +1,4 @@
-/*===========================================================================
+/*==============================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
@@ -24,24 +24,19 @@
*
*/
-#ifndef _h_vdb_info_
-#define _h_vdb_info_
+#include <stdlib.h>
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
+namespace KSearch
+{
+ class CVRefVariation;
}
-#endif
-#include <klib/rc.h>
-#include <vdb/manager.h>
-
-rc_t vdb_info( Vector * schema_list, dump_format_t format, const VDBManager *mgr,
- const char * acc_or_path, num_gen * row_generator );
-
-#ifdef __cplusplus
+namespace Common
+{
+ bool find_variation_core_step (KSearch::CVRefVariation& obj,
+ char const* ref_slice, size_t ref_slice_size,
+ size_t& ref_pos_in_slice,
+ char const* var, size_t var_len, size_t var_len_on_ref,
+ size_t chunk_size, size_t chunk_no_last,
+ size_t& bases_start, size_t& chunk_no_start, size_t& chunk_no_end);
}
-#endif
-
-#endif
diff --git a/tools/ref-variation/helper.cpp b/tools/ref-variation/helper.cpp
new file mode 100644
index 0000000..2b31d0e
--- /dev/null
+++ b/tools/ref-variation/helper.cpp
@@ -0,0 +1,1404 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+// helper.cpp
+
+#include "helper.h"
+
+#include <algorithm>
+#if DEBUG_PRINT != 0
+#include <stdio.h>
+#endif
+
+#include <vdb/vdb-priv.h>
+#include <klib/rc.h>
+#include <search/grep.h>
+
+#include <kdb/table.h>
+#include <kproc/thread.h>
+#include <kproc/lock.h>
+
+#ifdef _WIN32
+#pragma warning (disable:4503)
+#endif
+
+// TODO: remove printfs
+namespace KLib
+{
+ CKVector::CKVector() : m_pSelf(NULL)
+ {
+ Make();
+ }
+
+ CKVector::~CKVector()
+ {
+ Release();
+ }
+
+ void CKVector::Make()
+ {
+ if (m_pSelf)
+ throw Utils::CErrorMsg (0, "Duplicated call to KVectorMake");
+
+ rc_t rc = ::KVectorMake(&m_pSelf);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorMake");
+#if DEBUG_PRINT != 0
+ printf("Created KVector %p\n", m_pSelf);
+#endif
+ }
+
+ void CKVector::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing KVector %p\n", m_pSelf);
+#endif
+ ::KVectorRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ size_t const RECORD_SIZE_IN_BITS = 2;
+ uint64_t const BIT_SET_MASK = 0x2;
+ uint64_t const BIT_VALUE_MASK = 0x1;
+ uint64_t const BIT_RECORD_MASK = BIT_SET_MASK | BIT_VALUE_MASK;
+
+ void CKVector::SetBool(uint64_t key, bool value)
+ {
+#if USING_UINT64_BITMAP == 1
+ uint64_t stored_bits = 0;
+ uint64_t key_qword = key / 64;
+ uint64_t key_bit = key % 64;
+ rc_t rc = ::KVectorGetU64 ( m_pSelf, key_qword, &stored_bits );
+ bool first_time = rc == RC ( rcCont, rcVector, rcAccessing, rcItem, rcNotFound ); // 0x1e615458
+ if ( !first_time && rc )
+ throw Utils::CErrorMsg(rc, "KVectorGetU64");
+
+ uint64_t new_bit = (uint64_t)value << key_bit;
+ uint64_t stored_bit = (uint64_t)1 << key_bit & stored_bits;
+
+ if ( first_time || new_bit != stored_bit )
+ {
+ if ( new_bit )
+ stored_bits |= new_bit;
+ else
+ stored_bits &= ~new_bit;
+
+ rc_t rc = ::KVectorSetU64 ( m_pSelf, key_qword, stored_bits );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorSetU64");
+ }
+#elif USING_UINT64_BITMAP == 2
+ uint64_t stored_bits = 0;
+ uint64_t key_qword = key / (sizeof(stored_bits) * 8 / RECORD_SIZE_IN_BITS);
+ uint64_t bit_offset_in_qword = (key % (sizeof(stored_bits) * 8 / RECORD_SIZE_IN_BITS)) * RECORD_SIZE_IN_BITS;
+ rc_t rc = ::KVectorGetU64 ( m_pSelf, key_qword, &stored_bits );
+ bool first_time = rc == RC ( rcCont, rcVector, rcAccessing, rcItem, rcNotFound ); // 0x1e615458;
+ if ( !first_time && rc )
+ throw Utils::CErrorMsg(rc, "KVectorGetU64");
+
+ uint64_t new_bit_record = (BIT_SET_MASK | (uint64_t)value) << bit_offset_in_qword;
+ uint64_t stored_bit_record = (uint64_t)BIT_RECORD_MASK << bit_offset_in_qword & stored_bits;
+
+ if ( first_time || new_bit_record != stored_bit_record )
+ {
+ stored_bits &= ~((uint64_t)BIT_RECORD_MASK << bit_offset_in_qword); // clear stored record to assign a new value by bitwise OR
+ stored_bits |= new_bit_record;
+
+ rc_t rc = ::KVectorSetU64 ( m_pSelf, key_qword, stored_bits );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorSetU64");
+ }
+#else
+
+ rc_t rc = ::KVectorSetBool ( m_pSelf, key, value );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KVectorSetBool");
+#endif
+ }
+
+ struct UserDataU64toBool
+ {
+ rc_t ( * f ) ( uint64_t key, bool value, void *user_data );
+ void* user_data;
+ };
+#if USING_UINT64_BITMAP == 1
+ rc_t VisitU64toBoolAdapter ( uint64_t key, uint64_t value, void *user_data )
+ {
+ rc_t ( * bool_callback ) ( uint64_t key, bool value, void *user_data );
+ bool_callback = ((UserDataU64toBool*) user_data) -> f;
+ void* original_user_data = ((UserDataU64toBool*) user_data) -> user_data;
+
+ rc_t rc = 0;
+ for ( size_t i = 0; i < sizeof (value) * 8; ++i )
+ {
+ rc = bool_callback ( key * 64 + i, (bool) ((uint64_t)1 << i & value), original_user_data );
+ if ( rc )
+ return rc;
+ }
+ return rc;
+ }
+#elif USING_UINT64_BITMAP == 2
+ rc_t VisitU64toBoolAdapter ( uint64_t key, uint64_t value, void *user_data )
+ {
+ rc_t ( * bool_callback ) ( uint64_t key, bool value, void *user_data );
+ bool_callback = ((UserDataU64toBool*) user_data) -> f;
+ void* original_user_data = ((UserDataU64toBool*) user_data) -> user_data;
+
+ rc_t rc = 0;
+ for ( size_t i = 0; i < sizeof (value) * 8 / RECORD_SIZE_IN_BITS; ++i )
+ {
+ uint64_t key_bool = key * sizeof(value) * 8 / RECORD_SIZE_IN_BITS + i;
+ uint64_t record = value >> i * RECORD_SIZE_IN_BITS & BIT_RECORD_MASK;
+ if ( record & BIT_SET_MASK )
+ {
+ rc = bool_callback ( key_bool, (bool) (record & BIT_VALUE_MASK), original_user_data );
+ if ( rc )
+ return rc;
+ }
+ }
+ return rc;
+ }
+#endif
+
+ void CKVector::VisitBool(rc_t ( * f ) ( uint64_t key, bool value, void *user_data ), void *user_data) const
+ {
+#if USING_UINT64_BITMAP == 1
+ UserDataU64toBool user_data_adapter = { f, user_data };
+ ::KVectorVisitU64 ( m_pSelf, false, VisitU64toBoolAdapter, &user_data_adapter );
+#elif USING_UINT64_BITMAP == 2
+ UserDataU64toBool user_data_adapter = { f, user_data };
+ ::KVectorVisitU64 ( m_pSelf, false, VisitU64toBoolAdapter, &user_data_adapter );
+#else
+ ::KVectorVisitBool ( m_pSelf, false, f, user_data );
+#endif
+ }
+}
+
+///////////////////////////////////////////////////////////////
+
+namespace VDBObjects
+{
+ CVCursor::CVCursor() : m_pSelf(NULL)
+ {}
+
+ CVCursor::~CVCursor()
+ {
+ Release();
+ }
+
+ CVCursor::CVCursor(CVCursor const& x)
+ {
+ Clone(x);
+ }
+
+ CVCursor& CVCursor::operator=(CVCursor const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVCursor::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VCursor %p\n", m_pSelf);
+#endif
+ ::VCursorRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVCursor::Clone(CVCursor const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VCursorAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VCursor %p\n", m_pSelf);
+#endif
+ }
+
+ void CVCursor::Open() const
+ {
+ rc_t rc = ::VCursorOpen(m_pSelf);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorOpen");
+ }
+
+ void CVCursor::PermitPostOpenAdd() const
+ {
+ rc_t rc = ::VCursorPermitPostOpenAdd ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorPermitPostOpenAdd");
+ }
+#if MANAGER_WRITABLE != 0
+ void CVCursor::InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount, bool set_default)
+ {
+ for (size_t i = 0; i < nCount; ++i)
+ {
+ rc_t rc = ::VCursorAddColumn(m_pSelf, & pColumnIndex[i], ColumnNames[i] );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorAddColumn - [%s]", ColumnNames[i]);
+
+ if ( set_default )
+ {
+ VTypedecl type;
+ VTypedesc desc;
+ uint32_t idx = pColumnIndex[i];
+
+ rc = ::VCursorDatatype ( m_pSelf, idx, & type, & desc );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorDatatype (column idx=%u [%s])", idx, ColumnNames[i]);
+
+ uint32_t elem_bits = ::VTypedescSizeof ( & desc );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTypedescSizeof (column idx=%u [%s])", idx, ColumnNames[i]);
+ rc = ::VCursorDefault ( m_pSelf, idx, elem_bits, "", 0, 0 );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorDefault (column idx=%u [%s])", idx, ColumnNames[i]);
+ }
+ }
+ }
+#else
+ void CVCursor::InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount)
+ {
+ for (size_t i = 0; i < nCount; ++i)
+ {
+ rc_t rc = ::VCursorAddColumn(m_pSelf, & pColumnIndex[i], ColumnNames[i] );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorAddColumn - [%s]", ColumnNames[i]);
+ }
+ }
+#endif
+
+ void CVCursor::GetIdRange(int64_t& idFirstRow, uint64_t& nRowCount) const
+ {
+ rc_t rc = ::VCursorIdRange(m_pSelf, 0, &idFirstRow, &nRowCount);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorIdRange");
+ }
+
+ int64_t CVCursor::GetRowId () const
+ {
+ int64_t row_id;
+ rc_t rc = ::VCursorRowId ( m_pSelf, & row_id );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorRowId");
+
+ return row_id;
+ }
+
+ void CVCursor::SetRowId (int64_t row_id) const
+ {
+ rc_t rc = ::VCursorSetRowId ( m_pSelf, row_id );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorSetRowId (%ld)", row_id);
+ }
+
+ void CVCursor::OpenRow () const
+ {
+ rc_t rc = ::VCursorOpenRow ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorOpenRow");
+ }
+
+#if MANAGER_WRITABLE != 0
+ void CVCursor::CommitRow ()
+ {
+ rc_t rc = ::VCursorCommitRow ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCommitRow");
+ }
+
+ void CVCursor::RepeatRow ( uint64_t count )
+ {
+ rc_t rc = ::VCursorRepeatRow ( m_pSelf, count );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorRepeatRow (%lu)", count);
+ }
+ void CVCursor::Commit ()
+ {
+ rc_t rc = ::VCursorCommit ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCommit");
+ }
+#endif
+ void CVCursor::CloseRow () const
+ {
+ rc_t rc = ::VCursorCloseRow ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCloseRow");
+ }
+
+///////////////////////////////////////////////////////////////////////////////////
+
+ CVTable::CVTable() : m_pSelf(NULL)
+ {
+ }
+
+ CVTable::~CVTable()
+ {
+ Release();
+ }
+
+ CVTable::CVTable(CVTable const& x)
+ {
+ Clone(x);
+ }
+
+ CVTable& CVTable::operator=(CVTable const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVTable::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VTable %p\n", m_pSelf);
+#endif
+ ::VTableRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVTable::Clone(CVTable const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VTableAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VTable %p\n", m_pSelf);
+#endif
+ }
+
+ CVCursor CVTable::CreateCursorRead ( ) const
+ {
+ CVCursor cursor;
+ rc_t rc = ::VTableCreateCursorRead(m_pSelf, const_cast<VCursor const**>(& cursor.m_pSelf));
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTableCreateCursorRead");
+
+#if DEBUG_PRINT != 0
+ printf("Created cursor (rd) %p\n", cursor.m_pSelf);
+#endif
+ return cursor;
+ }
+
+ CVCursor CVTable::CreateCursorRead ( size_t cache_size ) const
+ {
+ CVCursor cursor;
+ rc_t rc = ::VTableCreateCachedCursorRead(m_pSelf, const_cast<VCursor const**>(& cursor.m_pSelf), cache_size);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTableCreateCachedCursorRead (%zu)", cache_size);
+
+#if DEBUG_PRINT != 0
+ printf("Created cached cursor (rd) %p\n", cursor.m_pSelf);
+#endif
+ return cursor;
+ }
+
+#if MANAGER_WRITABLE != 0
+ CVCursor CVTable::CreateCursorWrite (::KCreateMode mode)
+ {
+ CVCursor cursor;
+ rc_t rc = ::VTableCreateCursorWrite ( m_pSelf, & cursor.m_pSelf, mode );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTableCreateCursorWrite");
+
+#if DEBUG_PRINT != 0
+ printf("Created cursor (wr) %p\n", cursor.m_pSelf);
+#endif
+ return cursor;
+ }
+#endif
+
+ KDBObjects::CKTable CVTable::OpenKTableRead () const
+ {
+ KDBObjects::CKTable obj;
+ rc_t rc = ::VTableOpenKTableRead(m_pSelf, const_cast<KTable const**>(& obj.m_pSelf));
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VTableOpenKTableRead");
+
+#if DEBUG_PRINT != 0
+ printf("Opened KTable %p\n", obj.m_pSelf);
+#endif
+ return obj;
+ }
+//////////////////////////////////////////////////////////////////////
+
+ CVDatabase::CVDatabase() : m_pSelf(NULL)
+ {}
+
+ CVDatabase::~CVDatabase()
+ {
+ Release();
+ }
+
+ CVDatabase::CVDatabase(CVDatabase const& x)
+ {
+ Clone(x);
+ }
+
+ CVDatabase& CVDatabase::operator=(CVDatabase const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVDatabase::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VDatabase %p\n", m_pSelf);
+#endif
+ ::VDatabaseRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVDatabase::Clone(CVDatabase const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VDatabaseAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VDatabase %p\n", m_pSelf);
+#endif
+ }
+
+ CVTable CVDatabase::OpenTable(char const* pszTableName) const
+ {
+ CVTable table;
+ rc_t rc = ::VDatabaseOpenTableRead(m_pSelf, const_cast<VTable const**>(& table.m_pSelf), pszTableName);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDatabaseOpenTableRead (%s)", pszTableName);
+
+#if DEBUG_PRINT != 0
+ printf("Opened table %p (%s)\n", table.m_pSelf, pszTableName);
+#endif
+ return table;
+ }
+
+#if MANAGER_WRITABLE != 0
+ CVTable CVDatabase::CreateTable ( char const* pszTableName )
+ {
+ CVTable table;
+ //rc_t rc = ::VDatabaseCreateTableDefault ( m_pSelf, & table.m_pSelf, pszTableName, pszTableName );
+ rc_t rc = ::VDatabaseCreateTableByMask ( m_pSelf, & table.m_pSelf, pszTableName, 0, 0, pszTableName );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDatabaseCreateTableDefault (%s)", pszTableName);
+
+#if DEBUG_PRINT != 0
+ printf("Created table %p (%s)\n", table.m_pSelf, pszTableName);
+#endif
+ return table;
+ }
+
+ void CVDatabase::ColumnCreateParams ( ::KCreateMode cmode, ::KChecksum checksum, size_t pgsize )
+ {
+ rc_t rc = ::VDatabaseColumnCreateParams ( m_pSelf, cmode, checksum, pgsize );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDatabaseColumnCreateParams");
+ }
+#endif
+
+//////////////////////////////////////////////////////////////////////
+
+ CVSchema::CVSchema() : m_pSelf (NULL)
+ {
+ }
+ CVSchema::~CVSchema()
+ {
+ Release();
+ }
+
+ CVSchema::CVSchema(CVSchema const& x)
+ {
+ Clone (x);
+ }
+
+ CVSchema& CVSchema::operator=(CVSchema const& x)
+ {
+ Clone (x);
+ return *this;
+ }
+
+ void CVSchema::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VSchema %p\n", m_pSelf);
+#endif
+ ::VSchemaRelease ( m_pSelf );
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVSchema::Clone ( CVSchema const& x )
+ {
+ if (m_pSelf)
+ Release();
+
+ m_pSelf = x.m_pSelf;
+ ::VSchemaAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING VSchema %p\n", m_pSelf);
+#endif
+ }
+
+ void CVSchema::VSchemaParseFile ( char const* pszFilePath )
+ {
+ rc_t rc = ::VSchemaParseFile ( m_pSelf, pszFilePath );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VSchemaParseFile (%s)", pszFilePath);
+ }
+
+//////////////////////////////////////////////////////////////////////
+
+ CVDBManager::CVDBManager() : m_pSelf(NULL)
+ {}
+
+ CVDBManager::~CVDBManager()
+ {
+ Release();
+ }
+
+ void CVDBManager::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VDBManager %p\n", m_pSelf);
+#endif
+ ::VDBManagerRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+#if MANGER_WRITABLE != 0
+ void CVDBManager::Make()
+ {
+ assert(m_pSelf == NULL);
+ if (m_pSelf)
+ throw Utils::CErrorMsg(0, "Double call to VDBManagerMakeUpdate");
+
+ rc_t rc = ::VDBManagerMakeUpdate ( & m_pSelf, NULL );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerMakeUpdate");
+
+ /*rc = VDBManagerDisablePagemapThread ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerDisablePagemapThread");*/
+
+#if DEBUG_PRINT != 0
+ printf("Created VDBManager (wr) %p\n", m_pSelf);
+#endif
+ }
+#else
+ void CVDBManager::Make()
+ {
+ assert(m_pSelf == NULL);
+ if (m_pSelf)
+ throw Utils::CErrorMsg(0, "Double call to VDBManagerMakeRead");
+
+ rc_t rc = ::VDBManagerMakeRead(const_cast<VDBManager const**>(&m_pSelf), NULL);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerMakeRead");
+
+#if DEBUG_PRINT != 0
+ printf("Created VDBManager (rd) %p\n", m_pSelf);
+#endif
+ }
+#endif
+
+ CVDatabase CVDBManager::OpenDB(char const* pszDBName) const
+ {
+ CVDatabase vdb;
+ rc_t rc = ::VDBManagerOpenDBRead(m_pSelf, const_cast<VDatabase const**>(& vdb.m_pSelf), NULL, pszDBName);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerOpenDBRead (%s)", pszDBName);
+
+#if DEBUG_PRINT != 0
+ printf("Opened database %p (%s)\n", vdb.m_pSelf, pszDBName);
+#endif
+ return vdb;
+ }
+
+ CVTable CVDBManager::OpenTable(char const* pszPath) const
+ {
+ CVTable table;
+ rc_t rc = ::VDBManagerOpenTableRead(m_pSelf, const_cast<VTable const**>(& table.m_pSelf), NULL, pszPath);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerOpenTableRead (%s)", pszPath);
+
+#if DEBUG_PRINT != 0
+ printf("Opened table directly from manager %p (%s)\n", vdb.m_pSelf, pszDBName);
+#endif
+ return table;
+ }
+
+#if MANAGER_WRITABLE != 0
+ CVDatabase CVDBManager::CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath )
+ {
+ CVDatabase vdb;
+ rc_t rc = ::VDBManagerCreateDB ( m_pSelf, & vdb.m_pSelf, schema.m_pSelf, pszTypeDesc, cmode, pszPath );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerCreateDB (%s)", pszPath);
+
+#if DEBUG_PRINT != 0
+ printf("Created database %p (%s)\n", vdb.m_pSelf, pszPath);
+#endif
+
+ // set creation mode of objects ( tables, columns, etc. ) to
+ // create new or re-initialize existing, plus attach md5 checksums
+ // to all files.
+ // set blob creation mode to record 32-bit CRC within blob
+ // continue to use default page size...
+ vdb.ColumnCreateParams ( kcmInit | kcmMD5, kcsCRC32, 0 );
+ return vdb;
+ }
+#endif
+
+ CVSchema CVDBManager::MakeSchema () const
+ {
+ CVSchema schema;
+ rc_t rc = ::VDBManagerMakeSchema ( m_pSelf, & schema.m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VDBManagerMakeSchema");
+
+#if DEBUG_PRINT != 0
+ printf("Created Schema %p\n", schema.m_pSelf);
+#endif
+ return schema;
+ }
+}
+
+namespace KDBObjects
+{
+//////////////////////////////////////////////////////////////
+ CKIndex::CKIndex() : m_pSelf(NULL)
+ {
+ }
+
+ CKIndex::~CKIndex()
+ {
+ Release();
+ }
+
+ CKIndex::CKIndex(CKIndex const& x)
+ {
+ Clone(x);
+ }
+
+ CKIndex& CKIndex::operator=(CKIndex const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CKIndex::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing KIndex %p\n", m_pSelf);
+#endif
+ ::KIndexRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CKIndex::Clone(CKIndex const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::KIndexAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING KIndex %p\n", m_pSelf);
+#endif
+ }
+
+ // returns false if not found
+ bool CKIndex::FindText(const char *key, int64_t *start_id, uint64_t *id_count,
+ int (CC * custom_cmp)(const void *item, struct PBSTNode const *n, void *data ),
+ void *data) const
+ {
+ rc_t rc = ::KIndexFindText(m_pSelf, key, start_id, id_count, NULL, NULL );
+ if (rc == SILENT_RC(rcDB,rcIndex,rcSelecting,rcString,rcNotFound))
+ return false;
+ else if (rc)
+ throw Utils::CErrorMsg(rc, "KIndexFindText: key=%s", key);
+ else
+ return true;
+ }
+
+
+//////////////////////////////////////////////////////////////
+ CKTable::CKTable() : m_pSelf(NULL)
+ {
+ }
+
+ CKTable::~CKTable()
+ {
+ Release();
+ }
+
+ CKTable::CKTable(CKTable const& x)
+ {
+ Clone(x);
+ }
+
+ CKTable& CKTable::operator=(CKTable const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CKTable::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing KTable %p\n", m_pSelf);
+#endif
+ ::KTableRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CKTable::Clone(CKTable const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::KTableAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING KTable %p\n", m_pSelf);
+#endif
+ }
+
+ CKIndex CKTable::OpenIndexRead(char const* name) const
+ {
+ KDBObjects::CKIndex obj;
+ rc_t rc = ::KTableOpenIndexRead(m_pSelf, const_cast<KIndex const**>(& obj.m_pSelf), name);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KTableOpenIndexRead(%s)", name);
+
+#if DEBUG_PRINT != 0
+ printf("Opened KIndex(%s) %p\n", name, obj.m_pSelf);
+#endif
+ return obj;
+ }
+
+
+//////////////////////////////////////////////////////////////
+}
+
+
+namespace KApp
+{
+ CArgs::CArgs () : m_pSelf(NULL)
+ {
+ }
+
+ CArgs::~CArgs()
+ {
+ Release();
+ }
+
+ void CArgs::MakeAndHandle (int argc, char** argv, ::OptDef const* pOptions, size_t option_count)
+ {
+ if (m_pSelf)
+ throw Utils::CErrorMsg (0, "Duplicated call to ArgsMakeAndHandle");
+
+ rc_t rc = ::ArgsMakeAndHandle (&m_pSelf, argc, argv, 1, pOptions, option_count);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsMakeAndHandle");
+#if DEBUG_PRINT != 0
+ printf("Created Args %p\n", m_pSelf);
+#endif
+ }
+ void CArgs::MakeAndHandle ( int argc, char** argv, ::OptDef const* pOptions1, size_t option_count1, ::OptDef const* pOptions2, size_t option_count2 )
+ {
+ if (m_pSelf)
+ throw Utils::CErrorMsg (0, "Duplicated call to ArgsMakeAndHandle");
+
+ rc_t rc = ::ArgsMakeAndHandle (&m_pSelf, argc, argv, 2, pOptions1, option_count1, pOptions2, option_count2);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsMakeAndHandle(2)");
+#if DEBUG_PRINT != 0
+ printf("Created Args(2) %p\n", m_pSelf);
+#endif
+ }
+
+ void CArgs::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing Args %p\n", m_pSelf);
+#endif
+ ::ArgsRelease (m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ ::Args const* CArgs::GetArgs () const
+ {
+ return m_pSelf;
+ }
+
+ uint32_t CArgs::GetParamCount () const
+ {
+ uint32_t ret = 0;
+ rc_t rc = ::ArgsParamCount ( m_pSelf, &ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsParamCount");
+
+ return ret;
+ }
+
+ char const* CArgs::GetParamValue ( uint32_t iteration ) const
+ {
+ void const* ret = NULL;
+ rc_t rc = ::ArgsParamValue ( m_pSelf, iteration, & ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsParamValue");
+
+ return static_cast <char const*> (ret);
+ }
+
+ uint32_t CArgs::GetOptionCount ( char const* option_name ) const
+ {
+ uint32_t ret = 0;
+ rc_t rc = ::ArgsOptionCount ( m_pSelf, option_name, &ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsOptionCount (%s)", option_name);
+
+ return ret;
+ }
+
+ char const* CArgs::GetOptionValue ( char const* option_name, uint32_t iteration ) const
+ {
+ void const* ret = NULL;
+ rc_t rc = ::ArgsOptionValue ( m_pSelf, option_name, iteration, & ret );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "ArgsOptionValue (%s)", option_name);
+
+ return static_cast <char const*> (ret);
+ }
+
+////////////////////////////////
+
+ CProgressBar::CProgressBar ( uint64_t size )
+ {
+ Make ( size );
+ }
+
+ CProgressBar::~CProgressBar ()
+ {
+ Release ();
+ }
+
+ void CProgressBar::Make ( uint64_t size )
+ {
+ rc_t rc = ::KLoadProgressbar_Make ( &m_pSelf, size );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLoadProgressbar_Make");
+#if DEBUG_PRINT != 0
+ printf ( "Created ProgressBar %p\n", m_pSelf );
+#endif
+ }
+ void CProgressBar::Release ()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing ProgressBar %p\n", m_pSelf);
+#endif
+ ::KLoadProgressbar_Release ( m_pSelf, true );
+ m_pSelf = NULL;
+ }
+ }
+
+ void CProgressBar::Append ( uint64_t chunk )
+ {
+ rc_t rc = ::KLoadProgressbar_Append ( m_pSelf, chunk );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLoadProgressbar_Append");
+ }
+
+ void CProgressBar::Process ( uint64_t chunk, bool force_report )
+ {
+ rc_t rc = ::KLoadProgressbar_Process ( m_pSelf, chunk, force_report );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLoadProgressbar_Process");
+ }
+
+///////////////////////////////////////////
+ CXMLLogger::CXMLLogger ( CArgs const& args )
+ {
+ Make ( args );
+ }
+
+ CXMLLogger::~CXMLLogger ()
+ {
+ Release ();
+ }
+
+ void CXMLLogger::Make ( CArgs const& args )
+ {
+ rc_t rc = ::XMLLogger_Make ( &m_pSelf, NULL, args.m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "XMLLogger_Make");
+#if DEBUG_PRINT != 0
+ printf ( "Created XMLLogger %p\n", m_pSelf );
+#endif
+ }
+
+ void CXMLLogger::Release ()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing XMLLogger %p\n", m_pSelf);
+#endif
+ ::XMLLogger_Release ( m_pSelf );
+ m_pSelf = NULL;
+ }
+ }
+}
+
+namespace Utils
+{
+ CErrorMsg::CErrorMsg(rc_t rc, char const* fmt_str, ...)
+ : m_rc(rc)
+ {
+ va_list args;
+ va_start(args, fmt_str);
+ string_vprintf (m_szDescr, countof(m_szDescr), NULL, fmt_str, args);
+ va_end(args);
+ }
+
+ CErrorMsg::CErrorMsg(char const* fmt_str, ...)
+ : m_rc(0)
+ {
+ va_list args;
+ va_start(args, fmt_str);
+ string_vprintf (m_szDescr, countof(m_szDescr), NULL, fmt_str, args);
+ va_end(args);
+ }
+
+ rc_t CErrorMsg::getRC() const
+ {
+ return m_rc;
+ }
+ char const* CErrorMsg::what() const throw()
+ {
+ return m_szDescr;
+ }
+
+
+ void HandleException ()
+ {
+ try
+ {
+ throw;
+ }
+ catch (Utils::CErrorMsg const& e)
+ {
+ char szBufErr[512] = "";
+ size_t rc = e.getRC();
+ rc_t res;
+ if (rc != 0)
+ res = string_printf(szBufErr, countof(szBufErr), NULL, "ERROR: %s failed with error 0x%08x (%u) [%R]", e.what(), rc, rc, rc);
+ else
+ res = string_printf(szBufErr, countof(szBufErr), NULL, "ERROR: %s", e.what());
+ if (res == rcBuffer || res == rcInsufficient)
+ szBufErr[countof(szBufErr) - 1] = '\0';
+ LOGMSG ( klogErr, szBufErr );
+ }
+ catch (std::exception const& e)
+ {
+ char szBufErr[512];
+ rc_t res = string_printf(szBufErr, countof(szBufErr), NULL,
+ "std::exception: %s", e.what());
+ if (res == rcBuffer || res == rcInsufficient)
+ szBufErr[countof(szBufErr) - 1] = '\0';
+ LOGMSG ( klogErr, szBufErr );
+ }
+ catch (...)
+ {
+ LOGMSG ( klogErr, "Unexpected exception occured" );
+ }
+ }
+}
+
+namespace KSearch
+{
+#if 0 // turning off old code
+ void FindRefVariationRegionAscii (
+ char const* ref, size_t ref_size, size_t ref_pos_var,
+ char const* variation, size_t variation_size, size_t var_len_on_ref,
+ size_t* p_ref_start, size_t* p_ref_len
+ )
+ {
+ rc_t rc = ::FindRefVariationRegionIUPAC ( ref, ref_size, ref_pos_var,
+ variation, variation_size, var_len_on_ref, p_ref_start, p_ref_len );
+
+ if (rc)
+ throw Utils::CErrorMsg(rc, "FindRefVariationRegionAscii");
+ }
+
+ // TODO: use pointers for return parameters, parameters returned by pointers should be
+ // in the beginning
+ void FindRefVariationRegionAscii (
+ std::string const& ref, size_t ref_pos_var,
+ char const* variation, size_t variation_size, size_t var_len_on_ref,
+ size_t& ref_start, size_t& ref_len
+ )
+ {
+ FindRefVariationRegionAscii ( ref.c_str(), ref.size(), ref_pos_var,
+ variation, variation_size, var_len_on_ref, & ref_start, & ref_len );
+ }
+#endif
+
+////////////////////////////////////////////////
+
+ CVRefVariation::CVRefVariation() : m_pSelf(NULL)
+ {
+ }
+
+ CVRefVariation::~CVRefVariation()
+ {
+ Release();
+ }
+
+ CVRefVariation::CVRefVariation(CVRefVariation const& x)
+ {
+ Clone(x);
+ }
+
+ CVRefVariation& CVRefVariation::operator=(CVRefVariation const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CVRefVariation::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing VRefVariation %p\n", m_pSelf);
+#endif
+ ::VRefVariationIUPACRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+
+ void CVRefVariation::Clone(CVRefVariation const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::VRefVariationIUPACAddRef ( m_pSelf );
+ m_bases_start = x.m_bases_start;
+
+#if DEBUG_PRINT != 0
+ printf ("CLONING VRefVariation %p\n", m_pSelf);
+#endif
+ }
+ char const* CVRefVariation::GetSearchQuery() const
+ {
+ if ( m_pSelf == NULL )
+ return "";
+ char const* ret = ::VRefVariationIUPACGetSearchQuery ( m_pSelf );
+ return ret == NULL ? "" : ret;
+ }
+
+ size_t CVRefVariation::GetSearchQueryStartRelative() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetSearchQueryStart ( m_pSelf );
+ }
+
+ size_t CVRefVariation::GetSearchQueryStartAbsolute() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetSearchQueryStart ( m_pSelf ) + m_bases_start;
+ }
+
+ size_t CVRefVariation::GetSearchQuerySize() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetSearchQuerySize ( m_pSelf );
+ }
+
+ size_t CVRefVariation::GetSearchQueryLenOnRef() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetSearchQueryLenOnRef ( m_pSelf );
+ }
+
+
+ char const* CVRefVariation::GetAllele( size_t& ret_size ) const
+ {
+ if ( m_pSelf == NULL )
+ {
+ ret_size = 0;
+ return "";
+ }
+ return ::VRefVariationIUPACGetAllele ( m_pSelf, & ret_size );
+ }
+
+ size_t CVRefVariation::GetAlleleStartRelative() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetAlleleStart ( m_pSelf );
+ }
+
+ size_t CVRefVariation::GetAlleleStartAbsolute() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetAlleleStart ( m_pSelf ) + m_bases_start;
+ }
+
+ size_t CVRefVariation::GetAlleleSize() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetAlleleSize ( m_pSelf );
+ }
+
+ size_t CVRefVariation::GetAlleleLenOnRef() const
+ {
+ if ( m_pSelf == NULL )
+ return 0;
+ return ::VRefVariationIUPACGetAlleleLenOnRef ( m_pSelf );
+ }
+
+ CVRefVariation VRefVariationIUPACMake ( uint32_t alg, char const* ref, size_t ref_size,
+ size_t ref_pos_var, char const* variation, size_t variation_size,
+ size_t var_len_on_ref, size_t bases_start)
+ {
+ CVRefVariation obj;
+ rc_t rc = ::VRefVariationIUPACMake ( & obj.m_pSelf, alg,
+ ref, ref_size, ref_pos_var, variation, variation_size, var_len_on_ref);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VRefVariationIUPACMake");
+
+#if DEBUG_PRINT != 0
+ printf("Created RefVariation (rd) %p\n", obj.m_pSelf);
+#endif
+ obj.m_bases_start = bases_start;
+ return obj;
+ }
+}
+
+namespace KProc
+{
+ rc_t KThreadFunc ( KThread const* , void* data )
+ {
+ CKThread* obj = static_cast<CKThread*>(data);
+
+ return ( * obj -> m_ThreadFunc ) ( obj -> m_pData );
+ }
+
+ CKThread::CKThread ()
+ : m_pSelf(NULL), m_ThreadFunc(NULL), m_pData(NULL)
+ {
+ }
+
+ CKThread::~CKThread()
+ {
+ Release();
+ }
+ CKThread::CKThread(CKThread const& x)
+ {
+ Clone(x);
+ }
+ CKThread& CKThread::operator=(CKThread const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CKThread::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing KThread %p\n", m_pSelf);
+#endif
+ ::KThreadRelease(m_pSelf);
+ m_pSelf = NULL;
+ m_ThreadFunc = NULL;
+ m_pData = NULL;
+ }
+ }
+ void CKThread::Clone(CKThread const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ m_ThreadFunc = x.m_ThreadFunc;
+ m_pData = x.m_pData;
+
+ if ( m_pSelf != NULL )
+ {
+ ::KThreadAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING KThread %p\n", m_pSelf);
+#endif
+ }
+ }
+
+ void CKThread::Make ( THREAD_FUNC thread_func, void* data )
+ {
+ assert (m_pSelf == NULL);
+ assert (m_ThreadFunc == NULL);
+ assert (m_pData == NULL);
+
+ m_ThreadFunc = thread_func;
+ m_pData = data;
+
+ rc_t rc = ::KThreadMake ( & m_pSelf, KThreadFunc, this );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KThreadMake");
+ }
+
+ rc_t CKThread::Wait ()
+ {
+ rc_t status;
+ rc_t rc = ::KThreadWait ( m_pSelf, & status );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KThreadWait");
+
+ return status;
+ }
+
+///////////////////////////////////////////////////////
+
+ CKLock::CKLock ( )
+ {
+ rc_t rc = ::KLockMake ( & m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLockMake");
+ }
+
+ CKLock::~CKLock()
+ {
+ Release();
+ }
+
+ CKLock::CKLock(CKLock const& x)
+ {
+ Clone(x);
+ }
+ CKLock& CKLock::operator=(CKLock const& x)
+ {
+ if (m_pSelf)
+ Release();
+
+ Clone(x);
+ return *this;
+ }
+
+ void CKLock::Release()
+ {
+ if (m_pSelf)
+ {
+#if DEBUG_PRINT != 0
+ printf("Releasing KLock %p\n", m_pSelf);
+#endif
+ ::KLockRelease(m_pSelf);
+ m_pSelf = NULL;
+ }
+ }
+ void CKLock::Clone(CKLock const& x)
+ {
+ m_pSelf = x.m_pSelf;
+ ::KLockAddRef ( m_pSelf );
+#if DEBUG_PRINT != 0
+ printf ("CLONING KLock %p\n", m_pSelf);
+#endif
+ }
+
+ void CKLock::Acquire ()
+ {
+ rc_t rc = ::KLockAcquire ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLockAcquire");
+ }
+
+ void CKLock::Lock ()
+ {
+ Acquire ();
+ }
+
+ void CKLock::Unlock ()
+ {
+ rc_t rc = ::KLockUnlock ( m_pSelf );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "KLockUnlock");
+ }
+
+ void CKLock::lock ()
+ {
+ Lock();
+ }
+ void CKLock::unlock ()
+ {
+ Unlock();
+ }
+}
\ No newline at end of file
diff --git a/tools/ref-variation/helper.h b/tools/ref-variation/helper.h
new file mode 100644
index 0000000..cadc28d
--- /dev/null
+++ b/tools/ref-variation/helper.h
@@ -0,0 +1,605 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+// helper.h
+#include <exception>
+
+#include <string>
+#include <string.h>
+
+#include <vdb/manager.h>
+#include <vdb/schema.h>
+#include <vdb/database.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+#include <klib/printf.h>
+#include <klib/vector.h>
+#include <kapp/args.h>
+#include <kapp/progressbar.h>
+#include <kapp/log-xml.h>
+#include <klib/out.h>
+
+#include <vdb/vdb-priv.h>
+#include <kdb/index.h>
+
+#ifndef countof
+#define countof(arr) (sizeof(arr)/sizeof(arr[0]))
+#endif
+
+#ifdef _WIN32
+#pragma warning (disable:4503)
+#endif
+
+#define USING_UINT64_BITMAP 0
+#define MANAGER_WRITABLE 0
+#define DEBUG_PRINT 0
+
+namespace KLib
+{
+ class CKVector
+ {
+ public:
+ CKVector();
+ CKVector (CKVector const& x);
+ CKVector& operator= (CKVector const& x);
+ ~CKVector();
+
+ void SetBool (uint64_t key, bool value);
+ void VisitBool (rc_t ( * f ) ( uint64_t key, bool value, void *user_data ), void *user_data) const;
+
+ private:
+ void Make();
+ void Release();
+
+ ::KVector* m_pSelf;
+ };
+}
+
+namespace Utils
+{
+ class CErrorMsg : public std::exception
+ {
+ public:
+ CErrorMsg(rc_t rc, char const* fmt_str, ...);
+ CErrorMsg(char const* fmt_str, ...);
+
+ rc_t getRC() const;
+ virtual char const* what() const throw();
+
+ private:
+ char m_szDescr[256];
+ rc_t m_rc;
+ };
+
+ template <typename T> T atoi_t ( char const* str_val )
+ {
+ if ( str_val [0] == '\0' )
+ throw Utils::CErrorMsg(0, "atoi_t: invalid input string (empty)");
+
+ size_t i = 0;
+ char sign = '+';
+ if ( str_val[0] == '-' || str_val[0] == '+' )
+ {
+ ++i;
+ sign = str_val[0];
+ }
+
+ T ret = 0;
+ for (; str_val[i] != '\0'; ++i )
+ {
+ if ( str_val[i] < '0' || str_val[i] > '9' )
+ throw Utils::CErrorMsg(0, "atoi_t: invalid input string \"%s\" (invalid character: '%c' at pos=%zu)", str_val, str_val[i], i+1);
+ ret = ret*10 + str_val[i] - '0';
+ }
+
+ return sign == '-' ? -ret : ret;
+ }
+
+ template <typename T> T atou_t ( char const* str_val )
+ {
+ if ( str_val [0] == '\0' )
+ throw Utils::CErrorMsg(0, "atou_t: invalid input string (empty)");
+
+ T ret = 0;
+ for ( size_t i = 0; str_val[i] != '\0'; ++i )
+ {
+ if ( str_val[i] < '0' || str_val[i] > '9' )
+ throw Utils::CErrorMsg(0, "atoi_t: invalid input string \"%s\" (invalid character: '%c' at pos=%zu)", str_val, str_val[i], i+1);
+ ret = ret*10 + str_val[i] - '0';
+ }
+
+ return ret;
+ }
+
+ void HandleException (); // This function must be called inside catch block only
+}
+
+namespace KDBObjects
+{
+ class CKTable;
+}
+
+namespace VDBObjects
+{
+ /* functor to remove trailing '\n' from char reads */
+ template<typename T> class CPostReadAction
+ {
+ T* m_pBuf;
+ uint32_t m_nCount;
+ public:
+ CPostReadAction(T* pBuf, uint32_t nCount) : m_pBuf(pBuf), m_nCount(nCount) {}
+ void operator()() const;
+ };
+ template<typename T> inline void CPostReadAction<T>::operator()() const {}
+ template<> inline void CPostReadAction<char>::operator()() const { m_pBuf[m_nCount] = '\0'; }
+ template<> inline void CPostReadAction<unsigned char>::operator()() const { m_pBuf[m_nCount] = '\0'; }
+
+ class CVCursor;
+ class CVTable;
+ class CVDatabase;
+ class CVSchema;
+
+/////////////////////////////////////////////////////////////////////////////////
+
+ class CVDBManager
+ {
+ public:
+ CVDBManager();
+ ~CVDBManager();
+ CVDBManager(CVDBManager const& x);
+ CVDBManager& operator=(CVDBManager const& x);
+
+ void Make();
+ void Release();
+ CVDatabase OpenDB ( char const* pszDBName ) const;
+ CVTable OpenTable(char const* pszPath) const;
+#if MANAGER_WRITABLE != 0
+ CVDatabase CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath );
+#endif
+ CVSchema MakeSchema () const;
+
+ private:
+ ::VDBManager* m_pSelf;
+ };
+
+/////////////////////////////////////////////////////////////////////////////////
+
+ class CVSchema
+ {
+ public:
+ friend CVSchema CVDBManager::MakeSchema () const;
+#if MANAGER_WRITABLE != 0
+ friend CVDatabase CVDBManager::CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath );
+#endif
+
+ CVSchema();
+ ~CVSchema();
+ CVSchema(CVSchema const& x);
+ CVSchema& operator=(CVSchema const& x);
+
+ void Make();
+ void Release();
+ void VSchemaParseFile(char const* pszFilePath);
+
+ private:
+ void Clone ( CVSchema const& x );
+ ::VSchema* m_pSelf;
+ };
+
+/////////////////////////////////////////////////////////////////////////////////
+
+ class CVDatabase
+ {
+ public:
+ friend CVDatabase CVDBManager::OpenDB ( char const* pszDBName ) const;
+#if MANAGER_WRITABLE != 0
+ friend CVDatabase CVDBManager::CreateDB ( CVSchema const& schema, char const* pszTypeDesc, ::KCreateMode cmode, char const* pszPath );
+#endif
+
+ CVDatabase();
+ ~CVDatabase();
+ CVDatabase(CVDatabase const& x);
+ CVDatabase& operator=(CVDatabase const& x);
+
+ void Release();
+ CVTable OpenTable ( char const* pszTableName ) const;
+#if MANAGER_WRITABLE != 0
+ CVTable CreateTable ( char const* pszTableName );
+ void ColumnCreateParams ( ::KCreateMode cmode, ::KChecksum checksum, size_t pgsize );
+#endif
+
+ private:
+ void Clone(CVDatabase const& x);
+ ::VDatabase* m_pSelf;
+ };
+
+//////////////////////////////////////////////////////////////
+
+ class CVTable
+ {
+ public:
+ friend CVTable CVDatabase::OpenTable(char const* pszTableName) const;
+ friend CVTable CVDBManager::OpenTable(char const* pszPath) const;
+#if MANAGER_WRITABLE != 0
+ friend CVTable CVDatabase::CreateTable ( char const* pszTableName );
+#endif
+
+ CVTable();
+ ~CVTable();
+ CVTable(CVTable const& x);
+ CVTable& operator=(CVTable const& x);
+
+ void Release();
+ CVCursor CreateCursorRead ( size_t cache_size ) const;
+ CVCursor CreateCursorRead ( ) const;
+ KDBObjects::CKTable OpenKTableRead () const;
+
+#if MANAGER_WRITABLE != 0
+ CVCursor CreateCursorWrite ( ::KCreateMode mode );
+#endif
+
+ private:
+ void Clone(CVTable const& x);
+ ::VTable* m_pSelf;
+ };
+
+////////////////////////////////////////////////////////////////////////////
+
+ class CVCursor
+ {
+ public:
+ friend CVCursor CVTable::CreateCursorRead ( ) const;
+ friend CVCursor CVTable::CreateCursorRead ( size_t cache_size ) const;
+#if MANAGER_WRITABLE != 0
+ friend CVCursor CVTable::CreateCursorWrite (::KCreateMode mode);
+#endif
+
+ CVCursor();
+ ~CVCursor();
+ CVCursor(CVCursor const& x);
+ CVCursor& operator=(CVCursor const& x);
+
+ void Release();
+ void PermitPostOpenAdd() const;
+#if MANAGER_WRITABLE != 0
+ void InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount, bool set_default);
+#else
+ void InitColumnIndex(char const* const* ColumnNames, uint32_t* pColumnIndex, size_t nCount);
+#endif
+ void Open() const;
+ void GetIdRange(int64_t& idFirstRow, uint64_t& nRowCount) const;
+
+ template <typename T> uint32_t ReadItems (int64_t idRow, uint32_t idxCol, T* pBuf, uint32_t nBufLen) const
+ {
+ uint32_t nItemsRead = 0;
+
+ rc_t rc = ::VCursorReadDirect(m_pSelf, idRow, idxCol, 8*sizeof(T), pBuf, nBufLen, &nItemsRead);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorReadDirect: row_id=%ld, idxCol=%u", idRow, idxCol);
+
+ //CPostReadAction<T>(pBuf, nItemsRead)();
+
+ return nItemsRead;
+ }
+
+ template <typename T> uint32_t CellDataDirect (int64_t idRow, uint32_t idxCol, T const** pBuf) const
+ {
+ uint32_t nItemsRead = 0;
+
+ assert ( pBuf != NULL );
+
+ void const* pv;
+
+ rc_t rc = ::VCursorCellDataDirect (m_pSelf, idRow, idxCol, NULL, & pv, 0, &nItemsRead);
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorCellDataDirect: row_id=%ld, idxCol=%u", idRow, idxCol);
+
+ *pBuf = static_cast <T const*>(pv);
+
+ return nItemsRead;
+ }
+
+ template <typename T> void Write (uint32_t idxCol, T const* pBuf, uint64_t count)
+ {
+ rc_t rc = ::VCursorWrite ( m_pSelf, idxCol, 8 * sizeof(T), pBuf, 0, count );
+ if (rc)
+ throw Utils::CErrorMsg(rc, "VCursorWrite: idxCol=%u", idxCol);
+ }
+
+ int64_t GetRowId () const;
+ void SetRowId (int64_t row_id) const;
+ void OpenRow () const;
+#if MANAGER_WRITABLE != 0
+ void CommitRow ();
+ void RepeatRow ( uint64_t count );
+ void Commit ();
+#endif
+ void CloseRow () const;
+
+ private:
+ void Clone(CVCursor const& x);
+ ::VCursor* m_pSelf;
+ };
+}
+
+namespace KDBObjects
+{
+//////////////////////////////////////////////////////////////
+ class CKIndex;
+
+ class CKTable
+ {
+ public:
+ friend CKTable VDBObjects::CVTable::OpenKTableRead () const;
+
+ CKTable();
+ ~CKTable();
+ CKTable(CKTable const& x);
+ CKTable& operator=(CKTable const& x);
+
+ void Release();
+
+ CKIndex OpenIndexRead(char const* name) const;
+
+ private:
+ void Clone(CKTable const& x);
+ ::KTable* m_pSelf;
+ };
+
+ class CKIndex
+ {
+ public:
+ friend CKIndex CKTable::OpenIndexRead(char const* name) const;
+
+ CKIndex();
+ ~CKIndex();
+ CKIndex(CKIndex const& x);
+ CKIndex& operator=(CKIndex const& x);
+
+ // returns false if not found
+ bool FindText(const char *key, int64_t *start_id, uint64_t *id_count,
+ int (CC * custom_cmp)(const void *item, struct PBSTNode const *n, void *data ),
+ void *data) const;
+
+ void Release();
+
+ private:
+ void Clone(CKIndex const& x);
+ ::KIndex* m_pSelf;
+ };
+}
+
+
+///////////////////////
+
+namespace KApp
+{
+ class CArgs;
+ class CXMLLogger
+ {
+ public:
+ CXMLLogger ( CArgs const& args );
+ CXMLLogger (CXMLLogger const& x);
+ CXMLLogger& operator= (CXMLLogger const& x);
+ ~CXMLLogger ();
+
+ void Make ( CArgs const& args );
+
+ private:
+ void Release ();
+
+ XMLLogger const* m_pSelf;
+ };
+
+/////////////////////////////////////////
+
+ class CArgs
+ {
+ public:
+ friend void CXMLLogger::Make ( CArgs const& args );
+ CArgs ();
+ CArgs ( CArgs const& x );
+ CArgs& operator= ( CArgs const& x );
+ ~CArgs ();
+
+ ::Args const* GetArgs () const;
+ uint32_t GetParamCount () const;
+ char const* GetParamValue ( uint32_t iteration ) const;
+ uint32_t GetOptionCount ( char const* option_name ) const;
+ char const* GetOptionValue ( char const* option_name, uint32_t iteration ) const;
+
+ template <typename T> T GetOptionValueInt ( char const* option_name, uint32_t iteration ) const
+ {
+ char const* str_val = GetOptionValue ( option_name, iteration );
+ return Utils::atoi_t <T> ( str_val );
+ }
+ template <typename T> T GetOptionValueUInt ( char const* option_name, uint32_t iteration ) const
+ {
+ char const* str_val = GetOptionValue ( option_name, iteration );
+ return Utils::atou_t <T> ( str_val );
+ }
+
+ void MakeAndHandle ( int argc, char** argv, ::OptDef const* pOptions, size_t option_count );
+ // TODO: it's better to make ::ArgsMakeAndHandle be able to take va_list
+ void MakeAndHandle ( int argc, char** argv, ::OptDef const* pOptions1, size_t option_count1, ::OptDef const* pOptions2, size_t option_count2 );
+
+ private:
+
+ void Release ();
+
+ ::Args* m_pSelf;
+ };
+
+ class CProgressBar
+ {
+ public:
+ CProgressBar ( uint64_t size );
+ CProgressBar ( CProgressBar const& x );
+ CProgressBar& operator= ( CProgressBar const& x );
+ ~CProgressBar ();
+
+ void Append ( uint64_t chunk );
+ void Process ( uint64_t chunk, bool force_report );
+
+ private:
+ void Make ( uint64_t size );
+ void Release ();
+
+ KLoadProgressbar const* m_pSelf;
+ };
+}
+
+struct VRefVariation;
+
+namespace KSearch
+{
+#if 0 // turning off old code
+ void FindRefVariationRegionAscii (
+ char const* ref, size_t ref_size, size_t ref_pos_var,
+ char const* variation, size_t variation_size, size_t var_len_on_ref,
+ size_t* p_ref_start, size_t* p_ref_len
+ );
+
+ void FindRefVariationRegionAscii (
+ std::string const& ref, size_t ref_pos_var,
+ char const* variation, size_t variation_size, size_t var_len_on_ref,
+ size_t& ref_start, size_t& ref_len
+ );
+#endif
+
+ class CVRefVariation
+ {
+ public:
+ friend CVRefVariation VRefVariationIUPACMake ( uint32_t alg,
+ char const* ref, size_t ref_size,
+ size_t ref_pos_var, char const* variation, size_t variation_size,
+ size_t var_len_on_ref, size_t bases_start);
+
+ CVRefVariation();
+ ~CVRefVariation();
+ CVRefVariation(CVRefVariation const& x);
+ CVRefVariation& operator=(CVRefVariation const& x);
+
+ void Release();
+
+ char const* GetSearchQuery() const;
+ size_t GetSearchQueryStartRelative() const;
+ size_t GetSearchQueryStartAbsolute() const;
+ size_t GetSearchQuerySize() const;
+ size_t GetSearchQueryLenOnRef() const;
+
+ char const* GetAllele( size_t& ret_size ) const;
+ size_t GetAlleleStartRelative() const; // relative to search region
+ size_t GetAlleleStartAbsolute() const; // in absolute reference coordinates
+ size_t GetAlleleSize() const;
+ size_t GetAlleleLenOnRef() const;
+
+ private:
+ void Clone(CVRefVariation const& x);
+ ::VRefVariation* m_pSelf;
+ size_t m_bases_start; // the absolute position on the reference
+ // starting at which the search was initiated
+ };
+
+ CVRefVariation VRefVariationIUPACMake ( uint32_t alg, char const* ref, size_t ref_size,
+ size_t ref_pos_var, char const* variation, size_t variation_size,
+ size_t var_len_on_ref, size_t bases_start);
+
+}
+
+struct KThread;
+struct KLock;
+
+namespace KProc
+{
+ class CKThread
+ {
+ public:
+
+ friend rc_t KThreadFunc ( ::KThread const* self, void* data );
+
+ typedef rc_t (*THREAD_FUNC) ( void* data );
+
+ CKThread ();
+ ~CKThread ();
+ CKThread (CKThread const& x);
+ CKThread& operator= (CKThread const& x);
+
+ void Make ( THREAD_FUNC thread_func, void* data );
+ void Release();
+
+ rc_t Wait ();
+
+ private:
+ void Clone(CKThread const& x);
+ ::KThread* m_pSelf;
+
+ THREAD_FUNC m_ThreadFunc;
+ void* m_pData;
+ };
+
+ class CKLock
+ {
+ public:
+ CKLock ();
+ ~CKLock ();
+ CKLock (CKLock const& x);
+ CKLock& operator= (CKLock const& x);
+
+ void Release();
+
+ void Acquire ();
+ void Lock ();
+ void Unlock ();
+
+ // for compatibility with standard library
+ void lock();
+ void unlock();
+
+ private:
+ void Clone(CKLock const& x);
+ ::KLock* m_pSelf;
+ };
+
+ template <class TLockable> class CLockGuard
+ {
+ public:
+ CLockGuard ( TLockable & lock ) : m_lock (lock)
+ {
+ m_lock.lock();
+ }
+ ~CLockGuard ( )
+ {
+ m_lock.unlock();
+ }
+
+ CLockGuard( CLockGuard<TLockable> const& x );
+ CLockGuard<TLockable>& operator=( CLockGuard<TLockable> const& x );
+
+ private:
+ TLockable & m_lock;
+ };
+}
\ No newline at end of file
diff --git a/tools/ref-variation/ref-variation.cpp b/tools/ref-variation/ref-variation.cpp
new file mode 100644
index 0000000..b41417d
--- /dev/null
+++ b/tools/ref-variation/ref-variation.cpp
@@ -0,0 +1,2102 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#define SECRET_OPTION 0
+
+#include "ref-variation.vers.h"
+
+#include <iostream>
+#include <string.h>
+#include <stdio.h>
+#include <stdint.h>
+
+#include <vector>
+#include <string>
+#include <fstream>
+
+#include <kapp/main.h>
+#include <klib/rc.h>
+#include <atomic.h>
+
+#include <ngs/ncbi/NGS.hpp>
+#include <ngs/ReferenceSequence.hpp>
+
+#include "helper.h"
+#include "common.h"
+#include <search/grep.h>
+
+#define CPP_THREADS 0
+
+class CNoMutex // empty class for singlethreaded lock_guard (no actions)
+{
+public:
+ CNoMutex() {}
+ ~CNoMutex() {}
+ CNoMutex(CNoMutex const& x);
+ CNoMutex const& operator=(CNoMutex const& x);
+
+ void lock() {}
+ void unlock() {}
+};
+
+#if CPP_THREADS != 0
+#include <thread>
+#include <mutex>
+
+#define LOCK_GUARD std::lock_guard<TLock>
+#define LOCK std::mutex
+
+#else
+
+#define LOCK_GUARD KProc::CLockGuard<TLock>
+#define LOCK KProc::CKLock
+
+#endif
+
+namespace RefVariation
+{
+
+#define COUNT_STRAND_NONE_STR "none"
+#define COUNT_STRAND_COUNTERALIGNED_STR "counteraligned"
+#define COUNT_STRAND_COALIGNED_STR "coaligned"
+#define PARAM_ALG_SW "sw"
+#define PARAM_ALG_RA "ra"
+
+ enum EnumCountStrand
+ {
+ COUNT_STRAND_NONE,
+ COUNT_STRAND_COUNTERALIGNED,
+ COUNT_STRAND_COALIGNED
+ };
+
+ struct Params
+ {
+ // command line params
+
+ // command line options
+ char const* ref_acc;
+ int64_t ref_pos_var;
+ char query[256];
+ size_t var_len_on_ref;
+ int verbosity;
+ size_t thread_count;
+ bool calc_coverage;
+ char const* input_file;
+ EnumCountStrand count_strand;
+ uint32_t query_min_rep;
+ uint32_t query_max_rep;
+ ::RefVarAlg alg;
+ } g_Params =
+ {
+ "",
+ -1,
+ "",
+ 0,
+ 0,
+ 1,
+ false,
+ "",
+ COUNT_STRAND_NONE,
+ 0,
+ 0,
+ ::refvarAlgSW
+ };
+
+ class CInputRun
+ {
+ public:
+
+ CInputRun (char const* run_name) : m_run_name (run_name == NULL ? "" : run_name)
+ {}
+
+ CInputRun (std::string const& run_name, std::string const& run_path, std::string const& pileup_stats_path)
+ : m_run_name (run_name),
+ m_run_path (run_path),
+ m_pileup_stats_path (pileup_stats_path)
+ {}
+
+ std::string const& GetRunName() const { return m_run_name; }
+ std::string const& GetRunPath() const { return m_run_path; }
+ std::string const& GetPileupStatsPath() const { return m_pileup_stats_path; }
+
+ bool IsValid () const { return m_run_name.size() > 0; }
+
+ private:
+
+ std::string m_run_name;
+ std::string m_run_path;
+ std::string m_pileup_stats_path;
+ };
+
+ class CInputRuns
+ {
+ public:
+ CInputRuns ();
+ CInputRuns (KApp::CArgs const& args);
+ CInputRuns (CInputRuns const& ); // no copy
+ CInputRuns& operator= (CInputRuns const& ); // no assignment
+
+ void Init ( KApp::CArgs const& args ); // not thread-safe!
+
+ size_t GetCount() const;
+ CInputRun GetNext () const;
+
+ size_t GetNextIndex() const;
+ CInputRun Get(size_t index) const;
+ private:
+
+ std::vector <CInputRun> m_input_runs;
+ mutable atomic_t m_param_index;
+ };
+
+ enum
+ {
+ VERBOSITY_DEFAULT = 0,
+ VERBOSITY_PRINT_VAR_SPEC = 1,
+ VERBOSITY_SOME_DETAILS = 2,
+ VERBOSITY_MORE_DETAILS = 3
+ };
+
+
+#if SECRET_OPTION != 0
+ char const OPTION_SECRET[] = "alt-ctrl-shift-f12";
+ char const* USAGE_SECRET[] = { "activate secret mode", NULL };
+#endif
+
+ char const OPTION_REFERENCE_ACC[] = "reference-accession";
+ char const ALIAS_REFERENCE_ACC[] = "r";
+ char const* USAGE_REFERENCE_ACC[] = { "look for the variation in this reference", NULL };
+
+ char const OPTION_REF_POS[] = "position";
+ char const ALIAS_REF_POS[] = "p";
+ char const* USAGE_REF_POS[] = { "look for the variation at this position on the reference", NULL };
+
+ char const OPTION_QUERY[] = "query";
+ //char const ALIAS_QUERY[] = "q"; // -q is deprecated
+ char const* USAGE_QUERY[] = { "query to find in the given reference (\"-\" is treated as an empty string, or deletion)."
+ " Optionally, for non-empty query, the variable number of repetitions "
+ "can be specified in the following way: "
+ "\"<query>[<min_rep>-<max_rep>]\" where <query> is the pattern which "
+ "should be repeated, <min_rep> is the minimum number of repetiotions and "
+ "<max_rep> is the maximum number of repetiotions to produce from query pattern. "
+ "E.g.: \"AT[1-3]\" produces 3 queries: \"AT\", \"ATAT\" and \"ATATAT\". "
+ "In this case the output counts will contain as many columns for matched "
+ "counts as many variations is produced from the given query.", NULL };
+
+ char const OPTION_VAR_LEN_ON_REF[] = "variation-length";
+ char const ALIAS_VAR_LEN_ON_REF[] = "l";
+ char const* USAGE_VAR_LEN_ON_REF[] = { "the length of the variation on the reference (0 means pure insertion)", NULL };
+
+ char const OPTION_VERBOSITY[] = "verbose";
+ char const ALIAS_VERBOSITY[] = "v";
+ char const* USAGE_VERBOSITY[] = { "increase the verbosity of the program. use multiple times for more verbosity.", NULL };
+
+ char const OPTION_THREADS[] = "threads";
+ char const ALIAS_THREADS[] = "t";
+ char const* USAGE_THREADS[] = { "the number of threads to run", NULL };
+
+ char const OPTION_COVERAGE[] = "coverage";
+ char const ALIAS_COVERAGE[] = "c";
+ char const* USAGE_COVERAGE[] = { "output coverage (the nubmer of alignments matching the given variation query) for each run", NULL };
+
+ char const OPTION_INPUT_FILE[] = "input-file";
+ char const ALIAS_INPUT_FILE[] = "i";
+ char const* USAGE_INPUT_FILE[] = { "take runs from input file rather than from command line. The file must be in text format, each line should contain three tab-separated values: <run-accession> <run-path> <pileup-stats-path>, the latter two are optional", NULL };
+
+ char const OPTION_COUNT_STRAND[] = "count-strand";
+ //char const ALIAS_COUNT_STRAND[] = "s";
+ char const* USAGE_COUNT_STRAND[] = { "controls relative orientation of 3' and 5' fragments. "
+ "\""COUNT_STRAND_NONE_STR"\" - do not count (default). "
+ "\""COUNT_STRAND_COUNTERALIGNED_STR"\" - as in Illumina. "
+ "\""COUNT_STRAND_COALIGNED_STR"\" - as in 454 or IonTorrent. ", NULL };
+
+ char const OPTION_ALG[] = "algorithm";
+ //char const ALIAS_ALG[] = "a";
+ char const* USAGE_ALG[] = { "the algorithm to use for searching. "
+ "\""PARAM_ALG_SW"\" means Smith-Waterman. "
+ "\""PARAM_ALG_RA"\" means Rolling bulldozer algorithm\n", NULL };
+
+ ::OptDef Options[] =
+ {
+ { OPTION_REFERENCE_ACC, ALIAS_REFERENCE_ACC, NULL, USAGE_REFERENCE_ACC, 1, true, true }
+ ,{ OPTION_REF_POS, ALIAS_REF_POS, NULL, USAGE_REF_POS, 1, true, true }
+ ,{ OPTION_QUERY, /*ALIAS_QUERY*/NULL, NULL, USAGE_QUERY, 1, true, true }
+ ,{ OPTION_VAR_LEN_ON_REF,ALIAS_VAR_LEN_ON_REF,NULL, USAGE_VAR_LEN_ON_REF,1, true, true }
+ ,{ OPTION_THREADS, ALIAS_THREADS, NULL, USAGE_THREADS, 1, true, false }
+ ,{ OPTION_COVERAGE, ALIAS_COVERAGE, NULL, USAGE_COVERAGE, 1, false,false }
+ ,{ OPTION_INPUT_FILE, ALIAS_INPUT_FILE, NULL, USAGE_INPUT_FILE, 1, true, false }
+ ,{ OPTION_COUNT_STRAND, NULL, NULL, USAGE_COUNT_STRAND, 1, true, false }
+ ,{ OPTION_ALG, NULL, NULL, USAGE_ALG, 1, true, false }
+#if SECRET_OPTION != 0
+ ,{ OPTION_SECRET, NULL, NULL, USAGE_SECRET, 1, true, false }
+#endif
+ //,{ OPTION_VERBOSITY, ALIAS_VERBOSITY, NULL, USAGE_VERBOSITY, 0, false, false }
+ };
+
+ void print_indel (char const* name, char const* text, size_t text_size, size_t indel_start, size_t indel_len)
+ {
+ int prefix_count = (int)(indel_start < 5 ? indel_start : 5);
+ int suffix_count = (int)((text_size - (indel_start + indel_len)) < 5 ?
+ (text_size - (indel_start + indel_len)) : 5);
+
+ char buf[512] = "";
+ rc_t res = string_printf(buf, countof(buf), NULL, "%s: %s%.*s[%.*s]%.*s%s",
+ name,
+ indel_start < 5 ? "" : "...",
+ prefix_count, text + indel_start - prefix_count, //(int)indel_start, text,
+ (int)indel_len, text + indel_start,
+ suffix_count, text + indel_start + indel_len, //(int)(text_size - (indel_start + indel_len)), text + indel_start + indel_len
+ (text_size - (indel_start + indel_len)) > 5 ? "..." : "");
+ if (res == rcBuffer || res == rcInsufficient)
+ buf[countof(buf) - 1] = '\0';
+
+ LOGMSG ( klogInfo, buf );
+ }
+
+#if SECRET_OPTION != 0
+ enum ColumnNameIndices
+ {
+// idx_SEQ_LEN,
+// idx_SEQ_START,
+ idx_MAX_SEQ_LEN,
+ idx_TOTAL_SEQ_LEN,
+ idx_READ
+ };
+ char const* g_ColumnNames[] =
+ {
+// "SEQ_LEN",
+// "SEQ_START", // 1-based
+ "MAX_SEQ_LEN",
+ "TOTAL_SEQ_LEN",
+ "READ"
+ };
+ uint32_t g_ColumnIndex [ countof (g_ColumnNames) ];
+#endif
+
+ // TODO: remove unused from here:
+ // idx_RUN_NAME - logging only for now
+ // idx_MISMATCH_COUNTS, idx_INSERTION_COUNTS, idx_DELETION_COUNT
+ // - intended for optimization but not used now
+ // idx_REFERENCE_SPEC, idx_REF_POS - not used
+
+ enum PileupColumnNameIndices
+ {
+ idx_DELETION_COUNT,
+ idx_DEPTH,
+ idx_INSERTION_COUNTS,
+ idx_MISMATCH_COUNTS,
+ idx_REFERENCE_SPEC,
+ idx_REF_POS,
+ idx_RUN_NAME
+ };
+ char const* g_PileupColumnNames[] =
+ {
+ "DELETION_COUNT",
+ "DEPTH",
+ "INSERTION_COUNTS",
+ "MISMATCH_COUNTS",
+ "REFERENCE_SPEC",
+ "REF_POS",
+ "RUN_NAME"
+ };
+
+ uint32_t base2na_to_index ( char const ch )
+ {
+ switch ( ch )
+ {
+ case 'A':
+ case 'a':
+ return 0;
+ case 'C':
+ case 'c':
+ return 1;
+ case 'G':
+ case 'g':
+ return 2;
+ case 'T':
+ case 't':
+ return 3;
+ default:
+ assert ( false );
+ return (uint32_t) -1;
+ }
+ }
+
+ enum
+ {
+ PILEUP_DEFINITELY_NOT_FOUND,
+ PILEUP_DEFINITELY_FOUND,
+ PILEUP_MAYBE_FOUND
+ };
+
+ struct count_pair
+ {
+ size_t count;
+ size_t count_posititve;
+ };
+
+ struct coverage_info
+ {
+ count_pair count_total;
+ std::vector <count_pair> counts_matched;
+ };
+
+ template <class TLock> void update_run_coverage ( char const* acc,
+ size_t alignments_total, size_t alignments_total_positive,
+ size_t alignments_matched, size_t alignments_matched_positive,
+ TLock* lock_cout, coverage_info* pcoverage_count, size_t index)
+ {
+ coverage_info& counts = *pcoverage_count;
+
+ if ( counts.count_total.count != (size_t)-1 )
+ {
+ if ( counts.count_total.count != alignments_total ||
+ counts.count_total.count_posititve != alignments_total_positive )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogWarn,
+ (klogWarn,
+ "Total counts don't match for $(ACC) for query # $(IDXPREV) and $(IDXCUR): "
+ "total: $(TOTALPREV) vs $(TOTALCUR), total positive: $(TOTALPOSPREV) vs $(TOTALPOSCUR)",
+ "ACC=%s,IDXPREV=%zu,IDXCUR=%zu,TOTALPREV=%zu,TOTALCUR=%zu,TOTALPOSPREV=%zu,TOTALPOSCUR=%zu",
+ acc, index, index + 1, alignments_total, counts.count_total.count,
+ alignments_total_positive, counts.count_total.count_posititve));
+ }
+ }
+
+ counts.count_total.count = alignments_total;
+ counts.count_total.count_posititve = alignments_total_positive;
+
+ assert ( index < counts.counts_matched.size() );
+ count_pair& matched_count = counts.counts_matched [index];
+
+ matched_count.count = alignments_matched;
+ matched_count.count_posititve = alignments_matched_positive;
+ }
+
+
+ template <class TLock> void report_run_coverage ( char const* acc,
+ coverage_info const* pcoverage_count, TLock* lock_cout)
+ {
+ coverage_info const& counts = *pcoverage_count;
+ if ( g_Params.calc_coverage )
+ {
+ LOCK_GUARD l(*lock_cout);
+
+ OUTMSG (( "%s", acc ));
+
+ std::vector <count_pair>::const_iterator cit = counts.counts_matched.begin();
+ std::vector <count_pair>::const_iterator cend = counts.counts_matched.end();
+ for (; cit != cend; ++cit)
+ {
+ count_pair const& c = *cit;
+ OUTMSG (( "\t%zu", c.count ));
+
+ if ( g_Params.count_strand != COUNT_STRAND_NONE )
+ OUTMSG (( ",%zu", c.count_posititve ));
+ }
+
+ OUTMSG (( "\t%zu", counts.count_total.count ));
+
+ if ( g_Params.count_strand != COUNT_STRAND_NONE )
+ OUTMSG (( ",%zu", counts.count_total.count_posititve ));
+
+ OUTMSG (("\n"));
+ }
+ else
+ {
+
+ std::vector <count_pair>::const_iterator cit = counts.counts_matched.begin();
+ std::vector <count_pair>::const_iterator cend = counts.counts_matched.end();
+ for (; cit != cend; ++cit)
+ {
+ count_pair const& c = *cit;
+ if ( c.count > 0 )
+ {
+ LOCK_GUARD l(*lock_cout);
+ OUTMSG (( "%s\n", acc ));
+ break;
+ }
+ }
+ }
+ }
+
+ template <class TLock> int find_alignment_in_pileup_db ( char const* acc,
+ char const* acc_pileup, char const* ref_name,
+ KSearch::CVRefVariation const* pobj, size_t ref_pos,
+ size_t query_len_on_ref,
+ TLock* lock_cout, size_t thread_num,
+ coverage_info* pcoverage_count, size_t index )
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] Processing $(ACC)...",
+ "THREAD_NUM=%zu,ACC=%s", thread_num, acc_pileup
+ ));
+ }
+
+ KSearch::CVRefVariation const& obj = *pobj;
+ VDBObjects::CVDBManager mgr;
+ mgr.Make();
+
+ try
+ {
+ VDBObjects::CVDatabase db = mgr.OpenDB ( acc_pileup );
+ VDBObjects::CVTable table = db.OpenTable ( "STATS" );
+ VDBObjects::CVCursor cursor = table.CreateCursorRead ();
+
+ uint32_t PileupColumnIndex [ countof (g_PileupColumnNames) ];
+ cursor.InitColumnIndex (g_PileupColumnNames, PileupColumnIndex, countof(g_PileupColumnNames));
+ cursor.Open();
+
+ int64_t id_first = 0;
+ uint64_t row_count = 0;
+
+ cursor.GetIdRange (id_first, row_count);
+
+ // Find Reference beginning
+
+ KDBObjects::CKTable ktbl = table.OpenKTableRead();
+ KDBObjects::CKIndex kindex = ktbl.OpenIndexRead("ref_spec");
+
+ int64_t ref_id_start;
+ uint64_t id_count;
+
+ bool found = kindex.FindText ( ref_name, & ref_id_start, & id_count, NULL, NULL );
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] $(FOUND)found $(REFNAME) row_id=$(ROWID), id_count=$(IDCOUNT)",
+ "THREAD_NUM=%zu,FOUND=%s,REFNAME=%s,ROWID=%ld,IDCOUNT=%lu",
+ thread_num, found ? "" : " not ", ref_name, ref_id_start, id_count
+ ));
+ }
+ if ( !found )
+ {
+ update_run_coverage ( acc, 0, 0, 0, 0, lock_cout, pcoverage_count, index );
+ return PILEUP_DEFINITELY_NOT_FOUND;
+ }
+
+ int64_t indel_cnt = (int64_t)obj.GetAlleleSize() - (int64_t)obj.GetAlleleLenOnRef();
+ //int64_t indel_check_cnt = indel_cnt > 0 ? indel_cnt : -indel_cnt;
+ size_t alignments_total = (size_t)((uint32_t)-1);
+
+ for ( int64_t pos = (int64_t)ref_pos; pos < (int64_t)( ref_pos + query_len_on_ref ); ++pos )
+ {
+ if ( pos + ref_id_start >= id_first + (int64_t)row_count )
+ {
+ // TODO: this is not expected normally, now
+ // nothing will be printed to the output
+ // but maybe we also need to report matches and total alignments
+ // here (0 0)
+
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogWarn,
+ "[$(THREAD_NUM)] OUT OF BOUNDS! filtering out",
+ "THREAD_NUM=%zu", thread_num
+ ));
+ }
+
+ update_run_coverage ( acc, 0, 0, 0, 0, lock_cout, pcoverage_count, index );
+ return PILEUP_DEFINITELY_NOT_FOUND; // went beyond the end of db, probably, it's a bug in db
+ }
+
+ uint32_t depth;
+ uint32_t count = cursor.ReadItems ( pos + ref_id_start, PileupColumnIndex[idx_DEPTH], & depth, sizeof depth );
+ if ( count == 0)
+ depth = 0;
+
+ if ( depth == 0 )
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] depth=0 at the ref_pos=$(POS) (id=$(ID)) filtering out",
+ "THREAD_NUM=%zu,POS=%ld,ID=%ld", thread_num, pos, pos + ref_id_start
+ ));
+ }
+ update_run_coverage ( acc, 0, 0, 0, 0, lock_cout, pcoverage_count, index );
+ return PILEUP_DEFINITELY_NOT_FOUND;
+ }
+
+ // if OPTION_COUNT_STRAND != none - cannot report any non-zero counts,
+ // so no further optimization is possible
+ if ( g_Params.count_strand != COUNT_STRAND_NONE )
+ continue;
+
+ if ( depth < alignments_total )
+ alignments_total = depth;
+
+ if ( indel_cnt == 0 ) // pure mismatch optimization
+ {
+ // 1. for query of length == 1 we can return PILEUP_DEFINITELY_FOUND
+ // and produce output (for total count can use DEPTH for -c option)
+
+ uint32_t mismatch[4];
+ count = cursor.ReadItems ( pos + ref_id_start, PileupColumnIndex[idx_MISMATCH_COUNTS], mismatch, sizeof mismatch );
+ assert ( count == 0 || count == 4 );
+
+ size_t allele_size;
+ char const* allele = obj.GetAllele(allele_size);
+ assert (count == 0 || pos - ref_pos < allele_size );
+ size_t alignments_matched = count == 0 ? 0 :
+ mismatch [base2na_to_index(allele[pos - ref_pos])];
+
+ if ( obj.GetAlleleLenOnRef() == 1 && obj.GetAlleleSize() == 1 )
+ {
+ update_run_coverage ( acc, alignments_total, 0, alignments_matched, 0, lock_cout, pcoverage_count, index );
+ return alignments_matched == 0 ?
+ PILEUP_DEFINITELY_NOT_FOUND : PILEUP_DEFINITELY_FOUND;
+ }
+
+ // 2. if at least one position has zero MISMATCH_COUNT - PILEUP_DEFINITELY_NOT_FOUND
+ // for the case when no -c option is specified
+ // otherwise we have to go into SRR to get actual
+ // alignments_total or alignments_matched
+
+ else if ( alignments_matched == 0 && ! g_Params.calc_coverage )
+ {
+ update_run_coverage ( acc, alignments_total, 0, alignments_matched, 0, lock_cout, pcoverage_count, index );
+ return PILEUP_DEFINITELY_NOT_FOUND;
+ }
+ }
+ // TODO: see if INSERTION_COUNTS or DELETION_COUNT can be also used
+ // for optimizations (at least for the case of lenght=1 indels).
+#if 0
+ else if ( indel_cnt > 0 ) // insertion
+ {
+ uint32_t counts[4];
+ count = cursor.ReadItems ( pos + ref_id_start, PileupColumnIndex[idx_INSERTION_COUNTS], counts, sizeof counts );
+ assert ( count == 0 || count == 4 );
+
+ LOCK_GUARD l(*lock_cout);
+ std::cout
+ << "pos=" << pos
+ << ", pileup row_id=" << pos + ref_id_start
+ << ", insertions=";
+ if ( count )
+ {
+ std::cout << "[";
+ for (size_t i = 0; i < count; ++i)
+ std::cout << ( i == 0 ? "" : ", ") << counts[i];
+ std::cout << "]";
+ }
+ else
+ std::cout << "<none>";
+
+ count = cursor.ReadItems ( pos + ref_id_start, PileupColumnIndex[idx_MISMATCH_COUNTS], counts, sizeof counts );
+ assert ( count == 0 || count == 4 );
+ std::cout << ", mismatches=";
+ if ( count )
+ {
+ std::cout << "[";
+ for (size_t i = 0; i < count; ++i)
+ std::cout << ( i == 0 ? "" : ", ") << counts[i];
+ std::cout << "]";
+ }
+ else
+ std::cout << "<none>";
+
+ count = cursor.ReadItems ( pos + ref_id_start, PileupColumnIndex[idx_DELETION_COUNT], counts, sizeof counts[0] );
+ assert ( count == 0 || count == 1 );
+
+ std::cout << ", deletions=";
+ if ( count > 0 )
+ std::cout << counts[0];
+ else
+ std::cout << "<none>";
+
+ std::cout << std::endl;
+ }
+ else
+ {
+ LOCK_GUARD l(*lock_cout);
+ std::cout << "DELETION" << std::endl;
+ }
+#endif
+ }
+
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_SOME_DETAILS )
+ {
+ char run_name[64];
+ uint32_t count = cursor.ReadItems ( id_first, PileupColumnIndex[idx_RUN_NAME], run_name, countof(run_name)-1 );
+ assert (count < countof(run_name));
+ run_name [count] = '\0';
+
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_SOME_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] $(RUNNAME) is suspicious",
+ "THREAD_NUM=%zu,RUNNAME=%s", thread_num, run_name
+ ));
+ }
+ //char const* p = run_name[0] == '/' ? run_name + 1 : run_name;
+ }
+
+ return PILEUP_MAYBE_FOUND;
+ }
+ catch ( Utils::CErrorMsg const& e )
+ {
+ if ( e.getRC() == SILENT_RC(rcVFS,rcMgr,rcOpening,rcDirectory,rcNotFound)
+ || e.getRC() == SILENT_RC(rcVFS,rcTree,rcResolving,rcPath,rcNotFound))
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] pileup db NOT FOUND, need to look into run itself",
+ "THREAD_NUM=%zu", thread_num
+ ));
+ }
+ return PILEUP_MAYBE_FOUND;
+ }
+ else if ( e.getRC() == SILENT_RC(rcDB,rcMgr,rcOpening,rcDatabase,rcIncorrect))
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "[$(THREAD_NUM)] BAD pileup db, need to look into run itself",
+ "THREAD_NUM=%zu", thread_num
+ ));
+ }
+ return PILEUP_MAYBE_FOUND;
+ }
+ else
+ throw;
+ }
+ }
+
+ bool is_primary_mate ( ngs::Fragment const& frag )
+ {
+ ngs::StringRef id = frag.getFragmentId();
+
+ // inplace strstr for non null-terminating string
+
+ char const pattern[] = { '.', 'F', 'A', '0', '.' };
+
+ if ( id.size() < countof ( pattern ) )
+ return false;
+
+ char const* data = id.data();
+ size_t stop = id.size() - countof ( pattern );
+
+ for (size_t i = 0; i <= stop; ++i )
+ {
+ if ( data [ i + 0 ] == pattern [ 0 ]
+ && data [ i + 1 ] == pattern [ 1 ]
+ && data [ i + 2 ] == pattern [ 2 ]
+ && data [ i + 3 ] == pattern [ 3 ]
+ && data [ i + 4 ] == pattern [ 4 ]
+ )
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ template <class TLock> void find_alignments_in_run_db ( char const* acc,
+ char const* path, char const* ref_name, size_t ref_start,
+ char const* variation, size_t var_size, size_t slice_size,
+ TLock* lock_cout, size_t thread_num,
+ coverage_info* pcoverage_count, size_t index)
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] Processing $(ACC)",
+ "THREAD_NUM=%zu,ACC=%s", thread_num, acc
+ ));
+ }
+
+ ncbi::ReadCollection run = ncbi::NGS::openReadCollection ( path && path[0] ? path : acc );
+
+ // TODO: if it works, it should me moved at the upper level, so
+ // it would not be calculated for each run. It should be done one time
+ // for each variation search
+ char const* pattern = g_Params.query_min_rep != 0 ? g_Params.query : NULL;
+ size_t pattern_len = g_Params.query_min_rep != 0 ? strlen (g_Params.query) : 0;
+
+ if ( run.hasReference ( ref_name ) )
+ {
+ if ( slice_size == 0 )
+ slice_size = 1; // for a pure insertion we at least a slice of length == 1 ?
+
+ ngs::Reference reference = run.getReference( ref_name );
+ //ngs::AlignmentIterator ai = reference.getAlignmentSlice (
+ // ref_start, slice_size + pattern_len, ngs::Alignment::all);
+
+ // TODO: remove C-cast to ngs::Alignment::AlignmentFilter
+ // when it's fixed in ngs api
+ ngs::AlignmentIterator ai = reference.getFilteredAlignmentSlice (
+ ref_start, slice_size + pattern_len, ngs::Alignment::all, (ngs::Alignment::AlignmentFilter)0, 0);
+
+ size_t alignments_total = 0, alignments_total_negative = 0;
+ size_t alignments_matched = 0, alignments_matched_negative = 0;
+ while ( ai.nextAlignment() )
+ {
+ uint64_t ref_pos_range = ai.getReferencePositionProjectionRange (ref_start);
+ if ( ref_pos_range == (uint64_t)-1 ) // effectively, checking that read doesn't start past ref_start
+ continue;
+
+ int64_t align_pos_first = (int64_t)( ref_pos_range >> 32);
+ int64_t align_pos_count = ref_pos_range & 0xFFFFFFFF;
+
+ // checking that read doesn't end before ref slice ends
+ if ( (int64_t) ai.getAlignmentLength() - align_pos_first < (int64_t)slice_size )
+ continue;
+
+ ngs::StringRef bases = ai.getAlignedFragmentBases ();
+
+ ++ alignments_total;
+ bool is_negative = g_Params.count_strand != COUNT_STRAND_NONE
+ && ai.getIsReversedOrientation();
+ if ( g_Params.count_strand == COUNT_STRAND_COUNTERALIGNED && ! is_primary_mate ( ai ) )
+ is_negative = ! is_negative;
+
+ if (is_negative)
+ ++ alignments_total_negative;
+
+ char const* bases_data = bases.data();
+ size_t bases_size = bases.size();
+
+ // trying all possible starting positions in
+ // the case when single reference position
+ // can have multiple alignment projections
+ for (int64_t i = 0; i < align_pos_count; ++i)
+ {
+ int64_t align_pos = align_pos_first + i;
+
+ bool match = bases_size + align_pos >= var_size && strncmp (variation, bases_data + align_pos, var_size) == 0;
+ if ( match && pattern != NULL )
+ {
+ char const* align_suffix = bases_data + align_pos + var_size;
+ size_t align_suffix_size = bases_size - (align_pos + var_size);
+
+ size_t min_size = align_suffix_size < pattern_len ? align_suffix_size : pattern_len;
+ match = strncmp ( pattern, align_suffix, min_size ) != 0;
+ }
+ if ( match )
+ {
+ ++ alignments_matched;
+ if ( ! g_Params.calc_coverage )
+ goto BREAK_ALIGNMENT_ITER; // -c option is for speed-up, so we sacrifice verbose output
+ if (is_negative)
+ ++ alignments_matched_negative;
+ }
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_SOME_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] id=$(ID): $(BASES)$(MATCH)",
+ "THREAD_NUM=%zu,ID=%s,BASES=%s,MATCH=%s",
+ thread_num, ai.getAlignmentId().data(),
+ bases.toString ( align_pos, var_size ).c_str(),
+ match ? " MATCH!" : ""
+ ));
+ }
+ if ( match )
+ break;
+ }
+ }
+
+BREAK_ALIGNMENT_ITER:
+ update_run_coverage ( acc,
+ alignments_total, alignments_total - alignments_total_negative,
+ alignments_matched, alignments_matched - alignments_matched_negative,
+ lock_cout, pcoverage_count, index );
+ }
+ else
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] reference $(REFNAME) NOT FOUND in $(ACC), skipping",
+ "THREAD_NUM=%zu,REFNAME=%s,ACC=%s", thread_num, ref_name, acc
+ ));
+ }
+
+ update_run_coverage ( acc, 0, 0, 0, 0, lock_cout, pcoverage_count, index );
+ }
+ }
+
+ template <class TLock> void find_alignments_in_single_run (char const* acc,
+ char const* path, char const* pileup_path, char const* ref_name,
+ KSearch::CVRefVariation const* pobj, size_t index,
+ TLock* lock_cout, size_t thread_num, coverage_info* pcoverage_count)
+ {
+ char const* variation;
+ size_t query_len_on_ref, var_start, var_size;
+
+ variation = pobj->GetSearchQuery();
+ var_size = pobj->GetSearchQuerySize();
+ query_len_on_ref = pobj->GetSearchQueryLenOnRef();
+ var_start = pobj->GetSearchQueryStartAbsolute();
+
+ int res = find_alignment_in_pileup_db ( acc, pileup_path, ref_name,
+ pobj, var_start, query_len_on_ref, lock_cout, thread_num, pcoverage_count, index );
+
+ if ( res == PILEUP_MAYBE_FOUND )
+ {
+ find_alignments_in_run_db( acc, path, ref_name, var_start,
+ variation, var_size, query_len_on_ref,
+ lock_cout, thread_num, pcoverage_count, index );
+ }
+ }
+
+ template <class TLock> void find_alignments_in_single_run ( char const* acc,
+ char const* path, char const* pileup_path, char const* ref_name,
+ std::vector <KSearch::CVRefVariation> const* pvec_obj,
+ TLock* lock_cout, size_t thread_num )
+ {
+ char const pileup_suffix[] = ".pileup";
+ char acc_pileup [ 128 ];
+
+ if ( pileup_path == NULL || pileup_path [0] == '\0' )
+ {
+ size_t acc_len = strlen(acc);
+ assert ( countof(acc_pileup) >= acc_len + countof(pileup_suffix) );
+ strncpy ( acc_pileup, acc, countof(acc_pileup) );
+ strncpy ( acc_pileup + acc_len, pileup_suffix, countof(acc_pileup) - acc_len );
+ pileup_path = acc_pileup;
+ }
+
+ std::vector <KSearch::CVRefVariation> const& vec_obj = *pvec_obj;
+
+ coverage_info coverage_counts = {};
+ coverage_counts.count_total.count = (size_t)-1;
+ coverage_counts.count_total.count_posititve = (size_t)-1;
+ coverage_counts.counts_matched.resize( vec_obj.size() );
+
+ std::vector <KSearch::CVRefVariation>::const_iterator cit = vec_obj.begin();
+ std::vector <KSearch::CVRefVariation>::const_iterator cend = vec_obj.end();
+ for ( size_t i = 0; cit != cend; ++cit, ++i )
+ {
+ KSearch::CVRefVariation const& obj = *cit;
+ find_alignments_in_single_run ( acc, path, pileup_path,
+ ref_name, & obj, i, lock_cout, thread_num, & coverage_counts );
+ }
+
+ report_run_coverage ( acc, & coverage_counts, lock_cout );
+ }
+
+ template <class TLock> void find_alignments ( char const* ref_name,
+ std::vector <KSearch::CVRefVariation> const* pvec_obj,
+ TLock* lock_cout, size_t thread_num, CInputRuns const* p_input_runs,
+ KApp::CProgressBar* progress_bar )
+ {
+ try
+ {
+ for ( ; ; )
+ {
+ size_t index = p_input_runs -> GetNextIndex();
+ CInputRun const& input_run = p_input_runs -> Get(index);
+
+ if ( ! input_run.IsValid() )
+ break;
+
+ char const* acc = input_run.GetRunName().c_str();
+ char const* path = input_run.GetRunPath().c_str();
+ char const* pileup_path = input_run.GetPileupStatsPath().c_str();
+
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "[$(THREAD_NUM)] Processing parameter # $(INDEX): $(ACC), path=[$(PATH)], pileup path=[$(PILEUPPATH)]",
+ "THREAD_NUM=%zu,INDEX=%zu,ACC=%s,PATH=%s,PILEUPPATH=%s",
+ thread_num, index,
+ acc, path, pileup_path
+ ));
+ }
+
+ try
+ {
+ find_alignments_in_single_run ( acc, path, pileup_path,
+ ref_name, pvec_obj, lock_cout, thread_num );
+ }
+ catch ( ngs::ErrorMsg const& e )
+ {
+ if ( strstr (e.what(), "Cannot open accession") == e.what() )
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_MORE_DETAILS )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "[$(THREAD_NUM)] $(WHAT), skipping",
+ "THREAD_NUM=%zu,WHAT=%s", thread_num, e.what()
+ ));
+ }
+ }
+ else
+ throw;
+ }
+
+ {
+ LOCK_GUARD l(*lock_cout);
+ progress_bar -> Process( 1, false );
+ }
+
+ if ( ::Quitting() )
+ {
+ LOCK_GUARD l(*lock_cout);
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "[$(THREAD_NUM)] Interrupted",
+ "THREAD_NUM=%zu", thread_num
+ ));
+ break;
+ }
+ }
+ }
+ catch ( ngs::ErrorMsg const& e )
+ {
+ LOCK_GUARD l(*lock_cout); // reuse cout mutex
+ PLOGMSG ( klogErr,
+ ( klogErr,
+ "[$(THREAD_NUM)] ngs::ErrorMsg: $(WHAT)",
+ "THREAD_NUM=%zu,WHAT=%s", thread_num, e.what()
+ ));
+ }
+ catch (...)
+ {
+ LOCK_GUARD l(*lock_cout); // reuse cout mutex
+ Utils::HandleException ();
+ }
+ }
+
+#if CPP_THREADS == 0
+ struct AdapterFindAlignment
+ {
+ KProc::CKThread thread;
+
+ size_t param_start, param_count;
+ char const* ref_name;
+ std::vector <KSearch::CVRefVariation> const* pvec_obj;
+ LOCK* lock_cout;
+ size_t thread_num;
+ CInputRuns const* p_input_runs;
+ KApp::CProgressBar* progress_bar;
+ };
+
+ void AdapterFindAlignment_Init (AdapterFindAlignment & params,
+ size_t param_start, size_t param_count,
+ char const* ref_name,
+ std::vector <KSearch::CVRefVariation> const* pvec_obj,
+ LOCK* lock_cout,
+ size_t thread_num,
+ CInputRuns const* p_input_runs,
+ KApp::CProgressBar* progress_bar
+ )
+ {
+ params.param_start = param_start;
+ params.param_count = param_count;
+ params.ref_name = ref_name;
+ params.pvec_obj = pvec_obj;
+ params.lock_cout = lock_cout;
+ params.thread_num = thread_num;
+ params.p_input_runs = p_input_runs;
+ params.progress_bar = progress_bar;
+ }
+
+ rc_t AdapterFindAlignmentFunc ( void* data )
+ {
+ AdapterFindAlignment& p = * (static_cast<AdapterFindAlignment*>(data));
+ find_alignments ( p.ref_name, p.pvec_obj, p.lock_cout, p.thread_num,
+ p.p_input_runs, p.progress_bar );
+ return 0;
+ }
+#endif
+
+ void finish_find_variation_region ( KApp::CArgs const & args,
+ std::vector <KSearch::CVRefVariation> const& vec_obj,
+ KApp::CProgressBar& progress_bar )
+ {
+ // Split further processing into multiple threads if there too many params
+ CInputRuns input_runs ( args );
+
+ size_t param_count = input_runs.GetCount();
+ progress_bar.Append ( param_count );
+
+ if ( param_count == 0 )
+ return;
+
+ if ( param_count >= 1 && param_count < g_Params.thread_count )
+ g_Params.thread_count = param_count;
+
+ size_t thread_count = g_Params.thread_count;
+
+ if ( thread_count == 1 )
+ {
+ CNoMutex mtx;
+ find_alignments (g_Params.ref_acc, & vec_obj, & mtx,
+ 0, & input_runs, & progress_bar);
+ }
+ else
+ {
+ // split
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_SOME_DETAILS )
+ {
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "Splitting $(PARAMCOUNT) jobs into $(THREADCOUNT) threads...",
+ "PARAMCOUNT=%zu,THREADCOUNT=%zu", param_count, thread_count
+ ));
+ }
+
+ LOCK mutex_cout;
+
+#if CPP_THREADS != 0
+ std::vector<std::thread> vec_threads;
+#else
+ std::vector<AdapterFindAlignment> vec_threads ( thread_count );
+#endif
+
+ size_t param_chunk_size = param_count / thread_count;
+ for (size_t i = 0; i < thread_count; ++i)
+ {
+ size_t current_chunk_size = i == thread_count - 1 ?
+ param_chunk_size + param_count % thread_count : param_chunk_size;
+#if CPP_THREADS != 0
+ vec_threads.push_back(
+ std::thread( find_alignments <LOCK>,
+ i * param_chunk_size,
+ current_chunk_size, g_Params.ref_acc, & vec_obj,
+ & mutex_cout, i + 1, & input_runs, & progress_bar
+ ));
+#else
+ AdapterFindAlignment & params = vec_threads [ i ];
+
+ AdapterFindAlignment_Init ( params, i * param_chunk_size,
+ current_chunk_size, g_Params.ref_acc, & vec_obj,
+ & mutex_cout, i + 1, & input_runs, & progress_bar );
+
+ params.thread.Make ( AdapterFindAlignmentFunc, & params );
+#endif
+ }
+#if CPP_THREADS != 0
+ for (std::thread& th : vec_threads)
+ th.join();
+#else
+ for (std::vector<AdapterFindAlignment>::iterator it = vec_threads.begin(); it != vec_threads.end(); ++it)
+ {
+ AdapterFindAlignment & params = *it;
+ params.thread.Wait();
+ }
+#endif
+
+ }
+ }
+
+ bool check_ref_slice ( char const* ref, size_t ref_size )
+ {
+ bool ret = ref_size == 0;
+ for ( size_t i = 0; i < ref_size; ++i )
+ {
+ if ( !(ref [i] == 'N' || ref [i] == 'n' || ref [i] == '.') )
+ {
+ ret = true; // at least one non-N base is OK
+ break;
+ }
+ }
+ return ret;
+ }
+
+ char const* get_query (char const* pattern, uint32_t repetitions, std::string& generated_query)
+ {
+ if ( repetitions == 0 )
+ return pattern;
+ else
+ {
+ size_t pattern_len = strlen (pattern);
+ generated_query.reserve ( pattern_len * repetitions );
+ generated_query.clear();
+
+ while ( repetitions-- > 0 )
+ generated_query.append ( pattern );
+
+ return generated_query.c_str();
+ }
+ }
+
+ void print_variation_specs ( char const* ref_slice, size_t ref_slice_size,
+ KSearch::CVRefVariation const& obj, const char* query, size_t query_len )
+ {
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_SOME_DETAILS )
+ {
+ size_t ref_start = obj.GetAlleleStartAbsolute();
+ size_t ref_len = obj.GetAlleleLenOnRef();
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "Found indel box at pos=$(REFSTART), length=$(REFLEN)",
+ "REFSTART=%zu,REFLEN=%zu", ref_start, ref_len
+ ));
+ print_indel ( "reference", ref_slice, ref_slice_size, obj.GetAlleleStartRelative(), ref_len );
+
+ PLOGMSG ( klogInfo,
+ ( klogInfo,
+ "var_query=$(VARIATION)", "VARIATION=%s", obj.GetSearchQuery()
+ ));
+ }
+
+ if ( g_Params.verbosity >= RefVariation::VERBOSITY_PRINT_VAR_SPEC )
+ {
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "Input variation spec : $(REFACC):$(REFPOSVAR):$(VARLENONREF):$(QUERY)",
+ "REFACC=%s,REFPOSVAR=%ld,VARLENONREF=%lu,QUERY=%.*s",
+ g_Params.ref_acc, g_Params.ref_pos_var,
+ g_Params.var_len_on_ref, query_len, query
+ ));
+
+ size_t allele_size;
+ char const* allele = obj.GetAllele ( allele_size );
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "Adjusted variation spec: $(REFACC):$(REFPOSVAR):$(VARLENONREF):$(ALLELE)",
+ "REFACC=%s,REFPOSVAR=%ld,VARLENONREF=%lu,ALLELE=%.*s",
+ g_Params.ref_acc, obj.GetAlleleStartAbsolute(),
+ obj.GetAlleleLenOnRef(), (int)allele_size, allele
+ ));
+ }
+ }
+
+ void get_ref_var_object (KSearch::CVRefVariation& obj,
+ char const* query, size_t query_len, ngs::ReferenceSequence const& ref_seq)
+ {
+ size_t var_len = query_len;
+
+ size_t chunk_size = 5000; // TODO: add the method Reference[Sequence].getChunkSize() to the API
+ size_t chunk_no = g_Params.ref_pos_var / chunk_size;
+ size_t ref_pos_in_slice = g_Params.ref_pos_var % chunk_size;
+ size_t bases_start = chunk_no * chunk_size;
+ size_t chunk_no_last = ref_seq.getLength() / chunk_size;
+
+ bool cont = false;
+ size_t chunk_no_start = chunk_no, chunk_no_end = chunk_no;
+
+ // optimization: first look into the current chunk only (using ngs::StringRef)
+ {
+ ngs::StringRef ref_chunk = ref_seq.getReferenceChunk ( bases_start );
+
+ if ( ! check_ref_slice (ref_chunk.data() + ref_pos_in_slice, g_Params.var_len_on_ref) )
+ {
+ throw Utils::CErrorMsg (
+ "The selected reference region [%.*s] does not contain valid bases, "
+ "exiting...",
+ (int)g_Params.var_len_on_ref, ref_chunk.data() + ref_pos_in_slice );
+ }
+
+ cont = Common::find_variation_core_step ( obj,
+ ref_chunk.data(), ref_chunk.size(), ref_pos_in_slice,
+ query, var_len, g_Params.var_len_on_ref,
+ chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
+
+ if ( !cont )
+ {
+ print_variation_specs ( ref_chunk.data(), ref_chunk.size(), obj, query, query_len );
+ }
+ }
+
+ // general case - expanding ref_slice to multiple chunks
+ if ( cont )
+ {
+ ngs::String ref_slice;
+ while ( cont )
+ {
+ ref_slice = ref_seq.getReferenceBases (
+ bases_start, (chunk_no_end - chunk_no_start + 1)*chunk_size );
+
+ cont = Common::find_variation_core_step ( obj,
+ ref_slice.c_str(), ref_slice.size(), ref_pos_in_slice,
+ query, var_len, g_Params.var_len_on_ref,
+ chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
+ }
+ print_variation_specs ( ref_slice.c_str(), ref_slice.size(), obj, query, query_len );
+ }
+ }
+
+ void check_var_objects (std::vector <KSearch::CVRefVariation> const& vec_obj)
+ {
+ assert (vec_obj.size() > 0);
+
+ // checking for invariants
+ size_t count = vec_obj.size();
+
+ if (count > 1)
+ {
+ size_t i_first_var = 0;
+ for (; i_first_var < count; ++i_first_var)
+ {
+ KSearch::CVRefVariation const& cur = vec_obj [i_first_var];
+ if ( cur.GetAlleleLenOnRef() != cur.GetAlleleSize() )
+ break;
+ }
+
+ KSearch::CVRefVariation const& first = vec_obj [i_first_var];
+
+ for (size_t i = i_first_var + 1; i < count; ++i)
+ {
+ KSearch::CVRefVariation const& cur = vec_obj[i];
+
+ // exception: if var_len_on_ref == var_size - skip
+ // this is probably the exact reference
+ if ( cur.GetAlleleLenOnRef() != cur.GetAlleleSize() )
+ {
+ if ( first.GetAlleleStartAbsolute() != cur.GetAlleleStartAbsolute()
+ || first.GetAlleleLenOnRef() != cur.GetAlleleLenOnRef() )
+ {
+ size_t allele_size_first, allele_size_cur;
+ char const* allele_first = first.GetAllele ( allele_size_first );
+ char const* allele_cur = cur.GetAllele ( allele_size_cur );
+
+ PLOGMSG( klogWarn, (klogWarn,
+ "Inconsistent variations found: (start=$(STARTFIRST), len=$(LENFIRST), allele=$(ALLELEFIRST)) vs (start=$(STARTCUR), len=$(LENCUR), allele=$(ALLELECUR))",
+ "STARTFIRST=%zu,LENFIRST=%zu,ALLELEFIRST=%.*s,STARTCUR=%zu,LENCUR=%zu,ALLELECUR=%.*s",
+ first.GetAlleleStartAbsolute(), first.GetAlleleLenOnRef(), (int)allele_size_first, allele_first,
+ cur.GetAlleleStartAbsolute(), cur.GetAlleleLenOnRef(), (int)allele_size_cur, allele_cur
+ ));
+ //throw Utils::CErrorMsg (
+ // "Inconsistent variations found: (start=%zu, len=%zu, variation=%s) vs (start=%zu, len=%zu, variation=%s)",
+ // first.GetVarStartAbsolute(), first.GetVarLenOnRef(), first.GetVariation(),
+ // cur.GetVarStartAbsolute(), cur.GetVarLenOnRef(), cur.GetVariation());
+ }
+ }
+ }
+ }
+ }
+
+ void find_reference_variation ( ngs::ReferenceSequence const& ref_seq )
+ {
+ uint64_t start = g_Params.ref_pos_var;
+ uint64_t len = g_Params.var_len_on_ref;
+
+ KSearch::CVRefVariation obj;
+
+ // this doesn't work for len = 0
+ ngs::StringRef query = ref_seq.getReferenceChunk ( start, len );
+ if ( query.size() == len )
+ {
+ get_ref_var_object ( obj, query.data(), query.size(), ref_seq );
+ }
+ else
+ {
+ ngs::String query_copy = ref_seq.getReferenceBases ( start, len );
+ get_ref_var_object ( obj, query_copy.c_str(), query_copy.size(), ref_seq );
+ }
+ }
+
+ int find_variation_region_impl (KApp::CArgs const& args)
+ {
+ // Adding 0% mark at the very beginning of the program
+ KApp::CProgressBar progress_bar(1);
+ progress_bar.Process ( 0, true );
+
+ ngs::ReferenceSequence ref_seq = ncbi::NGS::openReferenceSequence ( g_Params.ref_acc );
+
+ size_t variation_count = g_Params.query_min_rep == 0 ?
+ 1 : g_Params.query_max_rep - g_Params.query_min_rep + 1;
+
+ std::string generated_query;
+ if (g_Params.query_min_rep > 0)
+ generated_query.reserve ( strlen(g_Params.query) * g_Params.query_max_rep );
+
+ std::vector <KSearch::CVRefVariation> vec_obj ( variation_count );
+
+ // first, try to search against reference itself
+ // find_reference_variation ( ref_seq );
+
+ for (uint32_t i = 0; i < variation_count; ++i)
+ {
+ char const* query = get_query ( g_Params.query,
+ g_Params.query_min_rep + i, generated_query );
+ get_ref_var_object ( vec_obj [i], query, strlen(query), ref_seq );
+ }
+
+ check_var_objects (vec_obj);
+ finish_find_variation_region ( args, vec_obj, progress_bar );
+
+ return 0;
+ }
+
+#if SECRET_OPTION != 0
+ void get_ref_bases ( int64_t offset, uint64_t len, std::string & ret,
+ VDBObjects::CVCursor const& cursor,
+ int64_t id_first, uint64_t row_count,
+ uint32_t max_seq_len, uint64_t total_seq_len
+ )
+ {
+ if ( offset + len > total_seq_len )
+ len = total_seq_len - offset;
+
+ ret.clear();
+ if ( ret.capacity() < len )
+ ret.reserve ( ret.capacity() * 2 );
+
+ int64_t id_start = offset / max_seq_len + id_first;
+ int64_t id_end = (offset + len - 1) / max_seq_len + id_first;
+ if ( id_start == id_end ) // optimizing: have a separate branch for single-row slice
+ {
+ char const* pREAD;
+ cursor.CellDataDirect ( id_start, g_ColumnIndex[idx_READ], & pREAD );
+ int64_t offset_adj = offset % max_seq_len;
+ ret.assign( pREAD + offset_adj, len );
+ }
+ else // multi-row reference slice
+ {
+ // first id
+ char const* pREAD;
+ uint32_t count_read = cursor.CellDataDirect ( id_start, g_ColumnIndex[idx_READ], & pREAD );
+ int64_t offset_adj = offset % max_seq_len;
+ ret.assign( pREAD + offset_adj, count_read - offset_adj );
+
+ // intermediate id (full max_seq_len chunks)
+ for (int64_t id = id_start + 1; id < id_end; ++id )
+ {
+ count_read = cursor.CellDataDirect ( id, g_ColumnIndex[idx_READ], & pREAD );
+ ret.append ( pREAD, count_read );
+ }
+
+ // last id
+ count_read = cursor.CellDataDirect ( id_end, g_ColumnIndex[idx_READ], & pREAD );
+ ret.append( pREAD, len - ret.length() );
+ }
+ }
+
+ void test_ngs ()
+ {
+ std::cout << "Starting NGS test..." << std::endl;
+ ngs::ReferenceSequence ref = ncbi::NGS::openReferenceSequence ( g_Params.ref_acc );
+
+ uint64_t len = 1;
+ int64_t offset = ref.getLength()/2 - len;
+ std::string slice;
+ for ( ; offset >= 0; --offset, len += 2 )
+ {
+ slice = ref.getReferenceBases( offset, len );
+ // making something that prevents the optimizer from eliminating the code
+ if ( slice.length() != len )
+ std::cout << "slice.length() != len" << std::endl;
+ }
+ std::cout << "NGS test has SUCCEEDED" << std::endl;
+ }
+
+ void test_vdb ()
+ {
+ std::cout << "Starting VDB test..." << std::endl;
+
+ VDBObjects::CVDBManager mgr;
+ mgr.Make();
+
+ VDBObjects::CVTable table = mgr.OpenTable ( g_Params.ref_acc );
+ VDBObjects::CVCursor cursor = table.CreateCursorRead ();
+
+ cursor.InitColumnIndex (g_ColumnNames, g_ColumnIndex, countof(g_ColumnNames));
+ cursor.Open();
+
+ int64_t id_first = 0;
+ uint64_t row_count = 0;
+
+ cursor.GetIdRange (id_first, row_count);
+
+ uint32_t max_seq_len;
+ uint64_t total_seq_len;
+ cursor.ReadItems ( id_first, g_ColumnIndex[idx_MAX_SEQ_LEN], & max_seq_len, 1 );
+ cursor.ReadItems ( id_first, g_ColumnIndex[idx_TOTAL_SEQ_LEN], & total_seq_len, 1 );
+
+ uint64_t len = 1;
+ int64_t offset = total_seq_len/2 - len;
+ std::string slice;
+ int64_t offset_increment = 1;
+ for ( ; offset >= 0; offset -= offset_increment, len += 2 * offset_increment )
+ {
+ get_ref_bases ( offset, len, slice, cursor, id_first, row_count, max_seq_len, total_seq_len );
+ // making something that prevents the optimizer from eliminating the code
+ if ( slice.length() != len )
+ std::cout << "slice.length() != len" << std::endl;
+ }
+ std::cout << "VDB test has SUCCEEDED" << std::endl;
+
+ }
+
+ void test_correctness ()
+ {
+ std::cout << "Running correctness test..." << std::endl;
+ ngs::ReferenceSequence ref_ngs = ncbi::NGS::openReferenceSequence ( g_Params.ref_acc );
+
+ VDBObjects::CVDBManager mgr;
+ mgr.Make();
+
+ VDBObjects::CVTable table = mgr.OpenTable ( g_Params.ref_acc );
+ VDBObjects::CVCursor cursor = table.CreateCursorRead ();
+
+ cursor.InitColumnIndex (g_ColumnNames, g_ColumnIndex, countof(g_ColumnNames));
+ cursor.Open();
+
+ int64_t id_first = 0;
+ uint64_t row_count = 0;
+
+ cursor.GetIdRange (id_first, row_count);
+
+ uint32_t max_seq_len;
+ uint64_t total_seq_len;
+ cursor.ReadItems ( id_first, g_ColumnIndex[idx_MAX_SEQ_LEN], & max_seq_len, 1 );
+ cursor.ReadItems ( id_first, g_ColumnIndex[idx_TOTAL_SEQ_LEN], & total_seq_len, 1 );
+
+ if ( ref_ngs.getLength() != total_seq_len )
+ {
+ std::cout
+ << "Correctness test FAILED: length discrepancy: ngs="
+ << ref_ngs.getLength() << ", vdb=" << total_seq_len << std::endl;
+ return;
+ }
+
+ uint64_t len = 1;
+ int64_t offset = total_seq_len/2 - len;
+ std::string slice_vdb;
+ for ( ; offset >= 0; --offset, len += 2 )
+ {
+ ngs::String slice_ngs = ref_ngs.getReferenceBases( offset, len );
+ get_ref_bases ( offset, len, slice_vdb, cursor, id_first, row_count, max_seq_len, total_seq_len );
+
+ if (slice_ngs != slice_vdb)
+ {
+ std::cout
+ << "Correctness test FAILED: slice discrepancy at "
+ "offset=" << offset << ", length=" << len << std::endl
+ << "ngs=" << slice_ngs << std::endl
+ << "vdb=" << slice_vdb << std::endl;
+ return;
+ }
+ }
+ std::cout << "Correctness test has SUCCEEDED" << std::endl;
+ }
+
+ void test_getReferenceProjectionRange()
+ {
+ std::cout << "Starting getReferenceProjectionRange test..." << std::endl;
+ ngs::ReadCollection read_coll = ncbi::NGS::openReadCollection( "SRR1597772" );
+ int64_t ref_pos = 11601;
+ uint64_t stop = ref_pos + 92;
+
+ ngs::Reference ref = read_coll.getReference( "CM000663.1" );
+ ngs::Alignment align = read_coll.getAlignment("SRR1597772.PA.6");
+
+ ngs::String ref_str = ref.getReferenceBases( ref_pos, stop - ref_pos );
+ ngs::StringRef align_str = align.getAlignedFragmentBases();
+
+ std::cout
+ << "ref: " << ref_str << std::endl
+ << "seq: " << align_str << std::endl;
+
+ for ( ; ref_pos < (int64_t)stop; ++ ref_pos )
+ {
+ ngs::String ref_base = ref.getReferenceBases( ref_pos, 1 );
+ uint64_t range = align.getReferencePositionProjectionRange ( ref_pos );
+ uint32_t align_pos = range >> 32;
+ uint32_t range_len = range & 0xFFFFFFFF;
+
+ ngs::StringRef bases = align.getAlignedFragmentBases ( );
+
+ std::cout
+ << ref_pos << " "
+ << ref_base
+ << " (" << (int32_t)align_pos << ", " << range_len << ") "
+ << bases.toString( align_pos, range_len == 0 ? 1 : range_len )
+ << std::endl;
+ }
+
+ std::cout << "getReferenceProjectionRange test has SUCCEEDED" << std::endl;
+ }
+
+ void test_deletion_ambiguity ()
+ {
+ char const ref[] = "xABCABCy";
+ char const var[] = "";
+ size_t var_len_on_ref = g_Params.var_len_on_ref;
+ size_t pos = g_Params.ref_pos_var;
+
+ KSearch::CVRefVariation obj = KSearch::VRefVariationIUPACMake (
+ ref, countof(ref) - 1, pos, var, countof(var) - 1, var_len_on_ref, 0 );
+
+ std::cout
+ << "Found indel box at pos=" << obj.GetVarStartRelative()
+ << ", length=" << obj.GetVarLenOnRef()
+ << std::endl;
+ print_indel ( "reference", ref, countof(ref) - 1, obj.GetVarStartRelative(), obj.GetVarLenOnRef() );
+ }
+
+ void test_temp ()
+ {
+ char const ref[] = "ACACACTA";
+ char const var[] = "AGCACACA";
+ size_t var_len_on_ref = countof(ref) - 1;
+ size_t pos = 0;
+
+ KSearch::CVRefVariation obj = KSearch::VRefVariationIUPACMake (
+ ref, countof(ref) - 1, pos, var, countof(var) - 1, var_len_on_ref, 0 );
+
+ std::cout
+ << "Found indel box at pos=" << obj.GetVarStartRelative()
+ << ", length=" << obj.GetVarLenOnRef()
+ << std::endl;
+ print_indel ( "reference", ref, countof(ref) - 1, obj.GetVarStartRelative(), obj.GetVarLenOnRef() );
+ print_variation_specs ( ref, countof(ref), obj, var );
+ }
+#endif
+
+ // returns pointer to the character that cannot be parsed
+ char const* parse_query ( char const* str )
+ {
+ char const allowed[] = "ACGTNacgtn.-";
+ size_t i;
+ for ( i = 0; str [i] != '\0'; ++i )
+ {
+ if ( strchr ( allowed, str[i] ) == NULL )
+ break;
+ }
+
+ if ( i >= countof (g_Params.query) )
+ {
+ LOGMSG( klogErr, "The query is too long" );
+ return str + i;
+ }
+
+ strncpy ( g_Params.query, str, i );
+
+ // TODO: maybe CArgs should allow for empty option value
+ if (g_Params.query [0] == '-' && g_Params.query [1] == '\0' )
+ g_Params.query[0] = '\0';
+
+ if ( str [i] == '[' )
+ {
+ // the next substring must contain: "\[[0-9]+\-[0-9]+\]$"
+ ++i;
+ uint32_t num = 0;
+ size_t i_num_start = i;
+ for ( ; str[i] >= '0' && str[i] <= '9'; ++i )
+ {
+ num *= 10;
+ num += str[i] - '0';
+ }
+
+ if ( num == 0 )
+ return str + i_num_start;
+
+ g_Params.query_min_rep = num;
+
+ if ( str[i] != '-' )
+ return str + i;
+
+ ++i;
+ i_num_start = i;
+ for ( num = 0; str[i] >= '0' && str[i] <= '9'; ++i )
+ {
+ num *= 10;
+ num += str[i] - '0';
+ }
+
+ if ( num == 0 )
+ return str + i_num_start;
+
+ g_Params.query_max_rep = num;
+
+ if ( str[i] != ']' )
+ return str + i;
+
+ ++i;
+ if ( str[i] != '\0' )
+ return str + i;
+
+ return NULL;
+ }
+ else if ( str [i] != '\0' )
+ return str + i;
+ else
+ return NULL;
+ }
+
+ int find_variation_region_impl_safe ( KApp::CArgs const& args)
+ {
+ int ret = 0;
+ try
+ {
+ ret = find_variation_region_impl ( args );
+ }
+ catch ( ngs::ErrorMsg const& e )
+ {
+ PLOGMSG ( klogErr,
+ ( klogErr, "ngs::ErrorMsg: $(WHAT)", "WHAT=%s", e.what()
+ ));
+ ret = 3;
+ }
+ catch (...)
+ {
+ Utils::HandleException ();
+ ret = 3;
+ }
+
+ return ret;
+ }
+
+ int find_variation_region (int argc, char** argv)
+ {
+ int ret = 0;
+ try
+ {
+ KApp::CArgs args;
+ args.MakeAndHandle (argc, argv, Options, countof (Options), ::XMLLogger_Args, ::XMLLogger_ArgsQty);
+ KApp::CXMLLogger xml_logger ( args );
+
+ // Actually GetOptionCount check is not needed here since
+ // CArgs checks that exactly 1 option is required
+
+ if (args.GetOptionCount (OPTION_REFERENCE_ACC) == 1)
+ g_Params.ref_acc = args.GetOptionValue ( OPTION_REFERENCE_ACC, 0 );
+
+ if (args.GetOptionCount (OPTION_REF_POS) == 1)
+ g_Params.ref_pos_var = args.GetOptionValueInt<int64_t> ( OPTION_REF_POS, 0 );
+
+ if (args.GetOptionCount (OPTION_QUERY) == 1)
+ {
+ char const* query = args.GetOptionValue ( OPTION_QUERY, 0 );
+ char const* pInvalid = parse_query ( query );
+ if ( pInvalid != NULL )
+ {
+ PLOGMSG ( klogErr,
+ ( klogErr,
+ "Error: the given query ($(QUERY)) contains an invalid character ($(CHAR))",
+ "QUERY=%s,CHAR=%c", g_Params.query, *pInvalid
+ ));
+ return 3;
+ }
+ else if ( g_Params.query_min_rep != 0
+ && g_Params.query_min_rep >= g_Params.query_max_rep )
+ {
+ PLOGMSG ( klogErr,
+ ( klogErr,
+ "Error: the given query ($(QUERY)) contains an invalid repetition range [$(MIN)-$(MAX)] (min must be less than max and more than zero)",
+ "QUERY=%s,MIN=%u,MAX=%u",
+ g_Params.query, g_Params.query_min_rep, g_Params.query_max_rep
+ ));
+ return 3;
+ }
+ }
+
+ if (args.GetOptionCount (OPTION_VAR_LEN_ON_REF) == 1)
+ g_Params.var_len_on_ref = args.GetOptionValueUInt<size_t>( OPTION_VAR_LEN_ON_REF, 0 );
+
+ if (args.GetOptionCount (OPTION_THREADS) == 1)
+ g_Params.thread_count = args.GetOptionValueUInt<size_t>( OPTION_THREADS, 0 );
+
+ g_Params.calc_coverage = args.GetOptionCount (OPTION_COVERAGE) != 0;
+
+ g_Params.verbosity = (int)args.GetOptionCount (OPTION_VERBOSITY);
+
+ if (args.GetOptionCount (OPTION_INPUT_FILE) == 1)
+ {
+ g_Params.input_file = args.GetOptionValue ( OPTION_INPUT_FILE, 0 );
+ if (args.GetParamCount() > 0)
+ {
+ PLOGMSG ( klogErr,
+ ( klogErr,
+ "$(PROGNAME) $(OPTIONNAME) option must not be provided along with parameters,"
+ " use either command line parameters or input file but not both",
+ "PROGNAME=%s,OPTIONNAME=%s", argv [0], OPTION_INPUT_FILE
+ ));
+ return 3;
+ }
+ }
+
+ if (args.GetOptionCount (OPTION_COUNT_STRAND) == 1)
+ {
+ char const* val = args.GetOptionValue ( OPTION_COUNT_STRAND, 0 );
+ if ( strcmp (val, COUNT_STRAND_NONE_STR) == 0 )
+ g_Params.count_strand = COUNT_STRAND_NONE;
+ else if ( strcmp (val, COUNT_STRAND_COUNTERALIGNED_STR) == 0 )
+ g_Params.count_strand = COUNT_STRAND_COUNTERALIGNED;
+ else if ( strcmp (val, COUNT_STRAND_COALIGNED_STR) == 0 )
+ g_Params.count_strand = COUNT_STRAND_COALIGNED;
+ else
+ {
+ PLOGMSG ( klogErr,
+ ( klogErr,
+ "Error: unrecognized $(OPTIONNAME) option value: \"$(VALUE)\"",
+ "OPTIONNAME=%s,VALUE=%s", OPTION_COUNT_STRAND, val
+ ));
+ return 3;
+ }
+
+ if ( args.GetOptionCount (OPTION_COVERAGE) == 0 )
+ {
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "Warning: $(COUNTSTRAND) option has no effect if $(COVERAGE) is not specified",
+ "COUNTSTRAND=%s,COVERAGE=%s", OPTION_COUNT_STRAND, OPTION_COVERAGE
+ ));
+ g_Params.count_strand = COUNT_STRAND_NONE;
+ }
+ }
+
+ if (args.GetOptionCount (OPTION_ALG) == 1)
+ {
+ char const* alg = args.GetOptionValue ( OPTION_ALG, 0 );
+ if (!strcmp(alg, PARAM_ALG_SW))
+ g_Params.alg = ::refvarAlgSW;
+ else if (!strcmp(alg, PARAM_ALG_RA))
+ g_Params.alg = ::refvarAlgRA;
+ else
+ {
+ PLOGMSG ( klogErr, ( klogErr,
+ "Error: Unknown algorithm specified: \"$(ALG)\"", "ALG=%s", alg ));
+ return 3;
+ }
+ }
+
+#if SECRET_OPTION != 0
+ if ( args.GetOptionCount (OPTION_SECRET) > 0 )
+ {
+ std::cout << "Running ref-variation in secret mode" << std::endl;
+
+ switch ( args.GetOptionValueInt<int> (OPTION_SECRET, 0) )
+ {
+ case 1:
+ test_correctness ();
+ break;
+ case 2:
+ test_ngs ();
+ break;
+ case 3:
+ test_vdb ();
+ break;
+ case 4:
+ test_getReferenceProjectionRange ();
+ break;
+ case 5:
+ test_deletion_ambiguity ();
+ break;
+ case 6:
+ test_temp ();
+ break;
+ default:
+ std::cout
+ << "specify value for this option:" << std::endl
+ << "1 - run correctness test" << std::endl
+ << "2 - run ngs performance test" << std::endl
+ << "3 - run vdb performance test" << std::endl
+ << "4 - run getReferencePositionProjectionRange test" << std::endl
+ << "5 - run deletion ambiguity test" << std::endl
+ << "6 - run temporary test" << std::endl;
+ }
+ }
+ else
+#endif
+ {
+ ret = find_variation_region_impl_safe (args);
+ }
+
+ }
+ catch (...) // here we handle only exceptions in CArgs or CXMLLogger
+ {
+ ret = 3;
+ }
+
+ return ret;
+ }
+
+/////////////////////////////
+ // class CInputRuns
+
+ CInputRuns::CInputRuns (KApp::CArgs const& args)
+ {
+ m_param_index.counter = 0;
+
+ Init ( args );
+ }
+
+ CInputRuns::CInputRuns ()
+ {
+ m_param_index.counter = 0;
+ }
+
+
+ bool is_eol (char ch)
+ {
+ return ch == '\0' || ch == '\r' || ch == '\n';
+ }
+
+ bool is_sep (char ch)
+ {
+ return ch == '\t';
+ }
+
+ enum {PARSE_OK, PARSE_EMPTY, PARSE_ERROR};
+
+ int parse_input_line ( char const* line,
+ char const** p_acc, size_t* p_size_acc,
+ char const** p_run_path, size_t* p_size_run_path,
+ char const** p_pileup_path, size_t* p_size_pileup_path)
+ {
+ *p_acc = *p_run_path = *p_pileup_path = "";
+ *p_size_acc = *p_size_run_path = *p_size_pileup_path = 0;
+
+ char const* p = line;
+
+ if ( is_eol (*p) )
+ return PARSE_EMPTY;
+
+ *p_acc = p;
+ for ( ; ! is_sep (*p) && ! is_eol (*p); ++ p, ++ (*p_size_acc) );
+
+ if ( *p_size_acc == 0 )
+ return PARSE_ERROR;
+
+ if ( is_eol ( *p ) )
+ return PARSE_OK;
+
+ ++ p;
+ *p_run_path = p;
+ for ( ; ! is_sep (*p) && ! is_eol (*p); ++p, ++ (*p_size_run_path) );
+
+ if ( is_eol ( *p ) )
+ return PARSE_OK;
+
+ ++ p;
+ *p_pileup_path = p;
+ for ( ; ! is_sep (*p) && ! is_eol (*p); ++p, ++ (*p_size_pileup_path) );
+
+ return PARSE_OK;
+ }
+
+ void CInputRuns::Init ( KApp::CArgs const& args ) // not thread-safe!
+ {
+ if ( g_Params.input_file != NULL && g_Params.input_file [0] != '\0' )
+ {
+ std::ifstream input_file( g_Params.input_file );
+ if ( !input_file.good() )
+ throw Utils::CErrorMsg( "Failed to open file %s", g_Params.input_file );
+
+ std::string line;
+ size_t count = 0;
+
+ while ( std::getline ( input_file, line) )
+ {
+ ++ count;
+ if ( m_input_runs.capacity() < count )
+ m_input_runs.reserve ( m_input_runs.capacity() * 2 );
+
+ char const* p_acc, *p_path, *p_pileup_path;
+ size_t size_acc, size_path, size_pileup_path;
+
+ int res = parse_input_line ( line.c_str(),
+ & p_acc, & size_acc,
+ & p_path, & size_path,
+ & p_pileup_path, & size_pileup_path );
+ if ( res == PARSE_ERROR )
+ {
+ throw Utils::CErrorMsg(
+ "Failed to parse line # %lu from file %s",
+ count, g_Params.input_file );
+ }
+ else if ( res == PARSE_OK )
+ {
+ m_input_runs.push_back(
+ CInputRun(
+ std::string(p_acc, size_acc),
+ std::string(p_path, size_path),
+ std::string(p_pileup_path, size_pileup_path)));
+ }
+ else // res == PARSE_EMPTY
+ -- count;
+ }
+ }
+ else
+ {
+ m_input_runs.reserve (args.GetParamCount());
+
+ for ( uint32_t i = 0; i < args.GetParamCount(); ++i )
+ m_input_runs.push_back( CInputRun ( args.GetParamValue( i ) ) );
+ }
+ }
+
+ size_t CInputRuns::GetCount() const
+ {
+ return m_input_runs.size();
+ }
+
+ CInputRun CInputRuns::GetNext () const
+ {
+ size_t current_index = atomic_read_and_add( & m_param_index, 1 );
+ return current_index < m_input_runs.size() ? m_input_runs[current_index] : CInputRun("");
+ }
+ size_t CInputRuns::GetNextIndex () const
+ {
+ return atomic_read_and_add( & m_param_index, 1 );
+ }
+ CInputRun CInputRuns::Get(size_t index) const
+ {
+ return index < m_input_runs.size() ? m_input_runs[index] : CInputRun("");
+ }
+
+}
+
+extern "C"
+{
+ const char UsageDefaultName[] = "ref-variation";
+ ver_t CC KAppVersion ()
+ {
+ return REF_VARIATION_VERS;
+ }
+
+ rc_t CC UsageSummary (const char * progname)
+ {
+ OUTMSG ((
+ "Usage example:\n"
+ " %s -r <reference accession> -p <position on reference> -q <query to look for> -l 0 [<parameters>]\n"
+ "\n"
+ "Summary:\n"
+ " Find a possible indel window\n"
+ "\n", progname));
+ return 0;
+ }
+
+ rc_t CC Usage ( struct Args const * args )
+ {
+ rc_t rc = 0;
+ const char* progname = UsageDefaultName;
+ const char* fullpath = UsageDefaultName;
+
+ if (args == NULL)
+ rc = RC(rcExe, rcArgv, rcAccessing, rcSelf, rcNull);
+ else
+ rc = ArgsProgram(args, &fullpath, &progname);
+
+ UsageSummary (progname);
+
+
+ OUTMSG (("\nParameters: optional space-separated list of run accessions in which the query will be looked for\n\n"));
+
+ OUTMSG (("\nOptions:\n"));
+
+ HelpOptionLine (RefVariation::ALIAS_REFERENCE_ACC, RefVariation::OPTION_REFERENCE_ACC, "acc", RefVariation::USAGE_REFERENCE_ACC);
+ HelpOptionLine (RefVariation::ALIAS_REF_POS, RefVariation::OPTION_REF_POS, "value", RefVariation::USAGE_REF_POS);
+ HelpOptionLine (NULL, RefVariation::OPTION_QUERY, "string", RefVariation::USAGE_QUERY);
+ HelpOptionLine (RefVariation::ALIAS_VAR_LEN_ON_REF, RefVariation::OPTION_VAR_LEN_ON_REF, "value", RefVariation::USAGE_VAR_LEN_ON_REF);
+ HelpOptionLine (RefVariation::ALIAS_THREADS, RefVariation::OPTION_THREADS, "value", RefVariation::USAGE_THREADS);
+ HelpOptionLine (RefVariation::ALIAS_COVERAGE, RefVariation::OPTION_COVERAGE, "", RefVariation::USAGE_COVERAGE);
+ HelpOptionLine (RefVariation::ALIAS_INPUT_FILE, RefVariation::OPTION_INPUT_FILE, "string", RefVariation::USAGE_INPUT_FILE);
+ HelpOptionLine (NULL, RefVariation::OPTION_COUNT_STRAND, "value", RefVariation::USAGE_COUNT_STRAND);
+ HelpOptionLine (NULL, RefVariation::OPTION_ALG, "value", RefVariation::USAGE_ALG);
+ //HelpOptionLine (RefVariation::ALIAS_VERBOSITY, RefVariation::OPTION_VERBOSITY, "", RefVariation::USAGE_VERBOSITY);
+#if SECRET_OPTION != 0
+ HelpOptionLine (NULL, RefVariation::OPTION_SECRET, NULL, RefVariation::USAGE_SECRET);
+#endif
+ XMLLogger_Usage();
+
+ HelpOptionsStandard ();
+
+ HelpVersion (fullpath, KAppVersion());
+
+ return rc;
+ }
+
+ rc_t CC KMain ( int argc, char *argv [] )
+ {
+ /* command line examples:
+ -r NC_011752.1 -p 2018 --query CA -l 0
+ -r NC_011752.1 -p 2020 --query CA -l 0
+ -r NC_011752.1 -p 5000 --query CA -l 0
+
+ find insertion:
+ ref-variation -r NC_000013.10 -p 100635036 --query 'ACC' -l 0 /netmnt/traces04/sra33/SRZ/000793/SRR793062/SRR793062.pileup /netmnt/traces04/sra33/SRZ/000795/SRR795251/SRR795251.pileup
+ NEW: -r NC_000013.10 -p 100635036 --query ACC -l 0 SRR793062 SRR795251
+
+ windows example: -r NC_000002.11 -p 73613067 --query "-" -l 3 ..\..\..\tools\ref-variation\SRR618508.pileup
+
+ -r NC_000002.11 -p 73613071 --query "C" -l 1
+ -vv -t 16 -r NC_000007.13 -p 117292900 --query "-" -l 4
+
+ -vv -c -t 16 -r NC_000002.11 -p 73613067 --query "-" -l 3 /netmnt/traces04/sra33/SRZ/000867/SRR867061/SRR867061.pileup /netmnt/traces04/sra33/SRZ/000867/SRR867131/SRR867131.pileup
+ -vv -c -t 16 -r NC_000002.11 -p 73613067 --query "-" -l 3 ..\..\..\tools\ref-variation\SRR867061.pileup ..\..\..\tools\ref-variation\SRR867131.pileup
+
+ -vvv -c --count-strand counteraligned -t 16 -r NC_000002.11 -p 73613067 --query "-" -l 3 SRR867061 SRR867131
+
+ old problem cases (didn't stop) - now OK:
+ -v -c -r CM000671.1 -p 136131022 --query "T" -l 1 SRR1601768
+ -v -c -r NC_000001.11 -p 136131022 --query "T" -l 1 SRR1601768
+
+ NEW problebm - FIXED:
+ -t 16 -v -c -r CM000671.1 -p 136131021 --query "T" -l 1 SRR1596639
+
+ NEW problem:
+ -vvv -r NC_000002.11 -p 73613030 --query "AT[1-3]" -l 3
+ Inconsistent variations found
+
+ NEW problem - FIXED (not completely):
+ -c -r CM000664.1 -p 234668879 -l 14 --query "ATATATATATATAT" SRR1597895 ok, non zero 30/33
+ -c -r CM000664.1 -p 234668879 -l 14 --query "AT[1-8]" SRR1597895 - all counts 0 - FIXED
+ was different total count because of SRR1597895.PA.26088404
+ had wrong projected pos - fixed in ncbi-vdb
+ UPDATE: now the sum of counts for AT[1-8] < for "ATATATATATATAT"
+
+ NEW problem - FIXED
+ -r NC_000001.10 -p 1064999 -l 1 --query A -v - hangs up
+
+ NEW problem - FIXED
+ -r NC_000020.10 -p 137534 -l 2 --query - -v
+
+ NEW problem - FIXED
+ -c -r CM000663.1 -p 123452 -l 7 --query "GGGAGAAAT" -vv -L info
+ expanded variation has smaller window (6) but the variation returned is still of length 7
+
+ */
+
+ return RefVariation::find_variation_region ( argc, argv );
+ }
+}
diff --git a/tools/ref-variation/ref-variation.vers b/tools/ref-variation/ref-variation.vers
new file mode 100644
index 0000000..8acdd82
--- /dev/null
+++ b/tools/ref-variation/ref-variation.vers
@@ -0,0 +1 @@
+0.0.1
diff --git a/tools/ref-variation/ref-variation.vers.h b/tools/ref-variation/ref-variation.vers.h
new file mode 100644
index 0000000..d0b2ae4
--- /dev/null
+++ b/tools/ref-variation/ref-variation.vers.h
@@ -0,0 +1 @@
+#define REF_VARIATION_VERS 0x00000001
diff --git a/tools/ref-variation/var-expand.cpp b/tools/ref-variation/var-expand.cpp
new file mode 100644
index 0000000..ec9c34f
--- /dev/null
+++ b/tools/ref-variation/var-expand.cpp
@@ -0,0 +1,448 @@
+#include "var-expand.vers.h"
+
+#include <kapp/main.h>
+#include <klib/rc.h>
+#include <klib/printf.h>
+
+#include <iostream>
+#include <stdio.h>
+
+#include <ngs/ncbi/NGS.hpp>
+#include <ngs/ReferenceSequence.hpp>
+
+#include "helper.h"
+#include "common.h"
+#include <search/grep.h>
+
+#define PARAM_ALG_SW "sw"
+#define PARAM_ALG_RA "ra"
+
+namespace VarExpand
+{
+ struct Params
+ {
+ ::RefVarAlg alg;
+ } g_Params =
+ {
+ ::refvarAlgSW
+ };
+
+ char const OPTION_ALG[] = "algorithm";
+ //char const ALIAS_ALG[] = "a";
+ char const* USAGE_ALG[] = { "the algorithm to use for searching. "
+ "\""PARAM_ALG_SW"\" means Smith-Waterman. "
+ "\""PARAM_ALG_RA"\" means Rolling bulldozer algorithm\n", NULL };
+
+ ::OptDef Options[] =
+ {
+ { OPTION_ALG, NULL, NULL, USAGE_ALG, 1, true, false }
+ };
+
+
+ bool check_ref_slice ( char const* ref, size_t ref_size )
+ {
+ bool ret = ref_size == 0;
+ for ( size_t i = 0; i < ref_size; ++i )
+ {
+ if ( !(ref [i] == 'N' || ref [i] == 'n' || ref [i] == '.') )
+ {
+ ret = true; // at least one non-N base is OK
+ break;
+ }
+ }
+ return ret;
+ }
+
+ void get_ref_var_object (KSearch::CVRefVariation& obj, size_t ref_pos, size_t del_len,
+ char const* allele, size_t allele_len, ngs::ReferenceSequence const& ref_seq,
+ std::string & ref_allele)
+ {
+ size_t var_len = allele_len;
+
+ size_t chunk_size = 5000; // TODO: add the method Reference[Sequence].getChunkSize() to the API
+ size_t chunk_no = ref_pos / chunk_size;
+ size_t ref_pos_in_slice = ref_pos % chunk_size;
+ size_t bases_start = chunk_no * chunk_size;
+ size_t chunk_no_last = ref_seq.getLength() / chunk_size;
+
+ bool cont = false;
+ size_t chunk_no_start = chunk_no, chunk_no_end = chunk_no;
+
+ // optimization: first look into the current chunk only (using ngs::StringRef)
+ {
+ ngs::StringRef ref_chunk = ref_seq.getReferenceChunk ( bases_start );
+
+ if ( ! check_ref_slice (ref_chunk.data() + ref_pos_in_slice, del_len) )
+ {
+ PLOGMSG ( klogWarn,
+ ( klogWarn,
+ "The selected reference region [$(REFSLICE)] does not contain valid bases, skipping...",
+ "REFSLICE=%.*s",
+ (int)del_len, ref_chunk.data() + ref_pos_in_slice ));
+ }
+
+ cont = Common::find_variation_core_step ( obj,
+ ref_chunk.data(), ref_chunk.size(), ref_pos_in_slice,
+ allele, var_len, del_len,
+ chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
+
+ if ( !cont )
+ ref_allele.assign (ref_chunk.data() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef());
+ }
+
+ // general case - expanding ref_slice to multiple chunks
+ if ( cont )
+ {
+ ngs::String ref_slice;
+ while ( cont )
+ {
+ ref_slice = ref_seq.getReferenceBases (
+ bases_start, (chunk_no_end - chunk_no_start + 1)*chunk_size );
+
+ cont = Common::find_variation_core_step ( obj,
+ ref_slice.c_str(), ref_slice.size(), ref_pos_in_slice,
+ allele, var_len, del_len,
+ chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
+ }
+ ref_allele.assign (ref_slice.c_str() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef());
+ }
+ }
+
+ void expand_variation ( ngs::ReferenceSequence const& ref_seq,
+ char const* key, size_t key_len,
+ char const* ref_name, size_t ref_name_len,
+ size_t ref_pos, size_t del_len,
+ char const* allele, size_t allele_len )
+ {
+ KSearch::CVRefVariation obj;
+ std::string ref_allele;
+
+ get_ref_var_object ( obj, ref_pos, del_len, allele, allele_len, ref_seq, ref_allele );
+
+ if ( ref_allele.size() == 0 )
+ ref_allele = "-";
+
+ char buf[512];
+ size_t new_allele_size;
+ char const* new_allele = obj.GetAllele ( new_allele_size );
+
+ if ( new_allele_size == 0)
+ {
+ new_allele = "-";
+ new_allele_size = 1;
+ }
+
+ string_printf ( buf, countof(buf), NULL,
+ "%.*s\t%.*s:%zu:%zu:%.*s\t%.*s:%zu:%zu:%.*s\t%.*s:%zu:%zu:%s",
+ (int)key_len, key,
+
+ (int)ref_name_len, ref_name,
+ ref_pos, del_len,
+ (int)allele_len, allele,
+
+ (int)ref_name_len, ref_name,
+ obj.GetAlleleStartAbsolute(), obj.GetAlleleLenOnRef(),
+ (int)new_allele_size, new_allele,
+
+ (int)ref_name_len, ref_name,
+ obj.GetAlleleStartAbsolute(), obj.GetAlleleLenOnRef(),
+ ref_allele.c_str()
+ );
+ buf [countof(buf) - 1] = '\0';
+ printf ("%s\n", buf);
+ }
+
+
+ bool is_eol (char ch)
+ {
+ return ch == '\0' || ch == '\r' || ch == '\n';
+ }
+
+ bool is_sep (char ch, char const* sz_separators)
+ {
+ return strchr (sz_separators, ch) != NULL;
+ }
+
+ bool parse_number ( char const* num_str, size_t num_str_len, size_t* parsed_val )
+ {
+ * parsed_val = 0;
+ for ( size_t i = 0; i < num_str_len; ++i )
+ {
+ char ch = num_str[i];
+ if ( ch < '0' || ch > '9' )
+ return false;
+
+ size_t digit = ch - '0';
+ *parsed_val = *parsed_val * 10 + digit;
+ }
+ return true;
+ }
+
+ bool parse_input_line ( char const* line, size_t line_size,
+ char const** pkey, size_t* pkey_len,
+ char const** pref_name, size_t* pref_name_len,
+ char const** pallele, size_t* pallele_len,
+ size_t* pref_pos, size_t* pdel_len )
+ {
+ size_t i = 0;
+ char const* pstr;
+ size_t str_len;
+
+ char const SEP_SPACE[] = " \t\n\r";
+ char const SEP_COLON[] = ":\n\r";
+
+ // key
+ pstr = & line[i];
+ str_len = 0;
+ for (; i < line_size && !is_sep(line[i], SEP_SPACE) && !is_eol(line[i]); ++i, ++str_len);
+ if (i == line_size || str_len == 0 || is_eol(line[i]))
+ return false;
+ *pkey = pstr;
+ *pkey_len = str_len;
+ for (++i; i < line_size && is_sep(line[i], SEP_SPACE) && !is_eol(line[i]); ++i);
+
+ // ref_name
+ pstr = & line[i];
+ str_len = 0;
+ for (; i < line_size && !is_sep(line[i], SEP_COLON) && !is_eol(line[i]); ++i, ++str_len);
+ if (i == line_size || str_len == 0 || is_eol(line[i]))
+ return false;
+ *pref_name = pstr;
+ *pref_name_len = str_len;
+
+ // ref_pos
+ ++i;
+ pstr = & line[i];
+ str_len = 0;
+ for (; i < line_size && !is_sep(line[i], SEP_COLON) && !is_eol(line[i]); ++i, ++str_len);
+ if (i == line_size
+ || str_len == 0
+ || is_eol (line[i])
+ || ! parse_number (pstr, str_len, pref_pos))
+ {
+ return false;
+ }
+
+ // del_len
+ ++i;
+ pstr = & line[i];
+ str_len = 0;
+ for (; i < line_size && !is_sep(line[i], SEP_COLON) && !is_eol(line[i]); ++i, ++str_len);
+ if (i == line_size
+ || str_len == 0
+ || is_eol (line[i])
+ || ! parse_number (pstr, str_len, pdel_len))
+ {
+ return false;
+ }
+
+ // allele
+ ++i;
+ pstr = & line[i];
+ str_len = 0;
+ for (; i < line_size && !is_sep(line[i], SEP_COLON) && !is_eol(line[i]); ++i, ++str_len);
+ *pallele = pstr;
+ *pallele_len = str_len;
+ // treat "-" as ""
+ if (*pallele_len == 1 && (*pallele)[0] == '-')
+ *pallele_len = 0;
+
+ return true;
+ }
+#if 0
+ void process_input_line ( char const* line, size_t line_size )
+ {
+ char const* key, *ref_name, *allele;
+ size_t key_len, ref_name_len, allele_len, ref_pos, del_len;
+
+ if ( parse_input_line ( line, line_size,
+ & key, & key_len,
+ & ref_name, & ref_name_len,
+ & allele, & allele_len,
+ & ref_pos, & del_len ) )
+ {
+ expand_variation ( key, key_len,
+ ref_name, ref_name_len,
+ ref_pos, del_len,
+ allele, allele_len );
+ }
+ }
+#endif
+ int expand_variations_impl ( )
+ {
+ std::string line;
+ bool end_of_stream = false;
+
+ ncbi::String sref_name, sref_name_prev;
+ char const* key, *ref_name, *allele;
+ size_t key_len, ref_name_len, allele_len, ref_pos, del_len;
+
+ while ( ! end_of_stream )
+ {
+ end_of_stream = std::getline ( std::cin, line ).eof();
+ if (line.size() > 0 && parse_input_line ( line.c_str(), line.size(),
+ & key, & key_len,
+ & ref_name, & ref_name_len,
+ & allele, & allele_len,
+ & ref_pos, & del_len ))
+ {
+ try // really only trying to open the first reference
+ {
+ sref_name.assign ( ref_name, ref_name_len );
+ ngs::ReferenceSequence ref_seq = ncbi::NGS::openReferenceSequence(sref_name);
+ break;
+ }
+ catch (ngs::ErrorMsg const& e)
+ {
+ if ( strstr ( e.what(), "failed to open table" ) == NULL )
+ throw;
+ else
+ continue;
+ }
+ }
+ }
+
+ if ( end_of_stream )
+ return 0;
+
+ // here we have the first good reference name in sref_name (shall be opened with no exceptions)
+ ngs::ReferenceSequence ref_seq = ncbi::NGS::openReferenceSequence(sref_name);
+ expand_variation ( ref_seq, key, key_len,
+ ref_name, ref_name_len,
+ ref_pos, del_len,
+ allele, allele_len );
+ sref_name_prev = sref_name;
+
+ // process the next input lines
+ while ( std::getline ( std::cin, line ) )
+ {
+ if (line.size() > 0 && parse_input_line ( line.c_str(), line.size(),
+ & key, & key_len,
+ & ref_name, & ref_name_len,
+ & allele, & allele_len,
+ & ref_pos, & del_len ))
+ {
+ sref_name.assign ( ref_name, ref_name_len );
+ if (sref_name == sref_name_prev)
+ {
+ expand_variation ( ref_seq, key, key_len,
+ ref_name, ref_name_len,
+ ref_pos, del_len,
+ allele, allele_len );
+ }
+ else
+ {
+ try
+ {
+
+ ref_seq = ncbi::NGS::openReferenceSequence(sref_name);
+ expand_variation ( ref_seq, key, key_len,
+ ref_name, ref_name_len,
+ ref_pos, del_len,
+ allele, allele_len );
+ sref_name_prev = sref_name;
+ }
+ catch (ngs::ErrorMsg const& e)
+ {
+ if ( strstr ( e.what(), "failed to open table" ) == NULL )
+ throw;
+ else
+ continue;
+ }
+ }
+ }
+ }
+
+ return 0;
+ }
+
+ int expand_variations (int argc, char** argv)
+ {
+ int ret;
+ try
+ {
+ KApp::CArgs args;
+ args.MakeAndHandle (argc, argv, Options, countof (Options));
+
+ if (args.GetOptionCount (OPTION_ALG) == 1)
+ {
+ char const* alg = args.GetOptionValue ( OPTION_ALG, 0 );
+ if (!strcmp(alg, PARAM_ALG_SW))
+ g_Params.alg = ::refvarAlgSW;
+ else if (!strcmp(alg, PARAM_ALG_RA))
+ g_Params.alg = ::refvarAlgRA;
+ else
+ {
+ PLOGMSG ( klogErr, ( klogErr,
+ "Error: Unknown algorithm specified: \"$(ALG)\"", "ALG=%s", alg ));
+ return 3;
+ }
+ }
+
+ ret = expand_variations_impl ();
+ }
+ catch ( ngs::ErrorMsg const& e )
+ {
+ PLOGMSG ( klogErr,
+ ( klogErr, "ngs::ErrorMsg: $(WHAT)", "WHAT=%s", e.what()
+ ));
+ ret = 3;
+ }
+ catch (...)
+ {
+ Utils::HandleException ();
+ ret = 3;
+ }
+
+ return ret;
+ }
+}
+
+
+extern "C"
+{
+ const char UsageDefaultName[] = "var-expand";
+ ver_t CC KAppVersion ()
+ {
+ return VAR_EXPAND_VERS;
+ }
+
+ rc_t CC UsageSummary (const char * progname)
+ {
+ OUTMSG (("\nFor each pair (key, variation spec) in input produces the expanded variation spec\n\n"));
+ return 0;
+ }
+
+ rc_t CC Usage ( struct Args const * args )
+ {
+ rc_t rc = 0;
+ const char* progname = UsageDefaultName;
+ const char* fullpath = UsageDefaultName;
+
+ if (args == NULL)
+ rc = RC(rcExe, rcArgv, rcAccessing, rcSelf, rcNull);
+ else
+ rc = ArgsProgram(args, &fullpath, &progname);
+
+ UsageSummary (progname);
+
+
+ OUTMSG (("\nInput: the stream of lines in the format: <key> <tab> <input variation>\n\n"));
+ OUTMSG (("\nOptions:\n"));
+
+ HelpOptionLine (NULL, VarExpand::OPTION_ALG, "value", VarExpand::USAGE_ALG);
+
+ XMLLogger_Usage();
+
+ HelpOptionsStandard ();
+
+ HelpVersion (fullpath, KAppVersion());
+
+ return rc;
+ }
+
+ rc_t CC KMain ( int argc, char *argv [] )
+ {
+ return VarExpand::expand_variations (argc, argv);
+ }
+}
diff --git a/tools/ref-variation/var-expand.vers b/tools/ref-variation/var-expand.vers
new file mode 100644
index 0000000..8acdd82
--- /dev/null
+++ b/tools/ref-variation/var-expand.vers
@@ -0,0 +1 @@
+0.0.1
diff --git a/tools/ref-variation/var-expand.vers.h b/tools/ref-variation/var-expand.vers.h
new file mode 100644
index 0000000..faa4618
--- /dev/null
+++ b/tools/ref-variation/var-expand.vers.h
@@ -0,0 +1 @@
+#define VAR_EXPAND_VERS 0x00000001
diff --git a/tools/refseq-load/refseq-load.vers b/tools/refseq-load/refseq-load.vers
new file mode 100644
index 0000000..da6b0a8
--- /dev/null
+++ b/tools/refseq-load/refseq-load.vers
@@ -0,0 +1 @@
+2.5.6
diff --git a/tools/run-stat/.gitignore b/tools/run-stat/.gitignore
deleted file mode 100644
index c5f06c0..0000000
--- a/tools/run-stat/.gitignore
+++ /dev/null
@@ -1,6 +0,0 @@
-*.sh
-*.txt
-*.svg
-*.xml
-*.json
-*.csv
diff --git a/tools/run-stat/Makefile b/tools/run-stat/Makefile
deleted file mode 100644
index ec65302..0000000
--- a/tools/run-stat/Makefile
+++ /dev/null
@@ -1,120 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-default: std
-
-TOP ?= $(abspath ../..)
-MODULE = tools/run-stat
-
-include $(TOP)/build/Makefile.env
-
-INT_TOOLS =
-
-EXT_TOOLS = \
- run-stat
-
-ALL_TOOLS = \
- $(INT_TOOLS) \
- $(EXT_TOOLS)
-
-#-------------------------------------------------------------------------------
-# outer targets
-#
-all std: vers-includes
- @ $(MAKE_CMD) $(TARGDIR)/$@
-
-$(ALL_TOOLS): vers-includes
- @ $(MAKE_CMD) $(BINDIR)/$@
-
-.PHONY: all std $(ALL_TOOLS)
-
-#-------------------------------------------------------------------------------
-# all
-#
-$(TARGDIR)/all: \
- $(addprefix $(BINDIR)/,$(ALL_TOOLS))
-
-.PHONY: $(TARGDIR)/all
-
-#-------------------------------------------------------------------------------
-# std
-#
-$(TARGDIR)/std: \
- $(addprefix $(BINDIR)/,$(EXT_TOOLS))
-
-.PHONY: $(TARGDIR)/std
-
-#-------------------------------------------------------------------------------
-# vers-includes
-#
-$(TARGDIR)/vers-includes: $(addsuffix .vers.h,$(EXT_TOOLS))
-
-.PHONY: $(TARGDIR)/vers-includes
-
-#-------------------------------------------------------------------------------
-# clean
-#
-clean: stdclean
-
-.PHONY: clean
-
-#-------------------------------------------------------------------------------
-# run-stat
-#
-RUN_STAT_SRC = \
- num-gen \
- context \
- helper \
- progressbar \
- mod_cmn \
- key_value \
- rd_filter \
- mod_reads_helper \
- mod_reads \
- svg \
- chart \
- run-stat
-
-RUN_STAT_OBJ = \
- $(addsuffix .$(OBJX),$(RUN_STAT_SRC))
-
-RUN_STAT_LIB = \
- -ssrapath \
- -ssraschema \
- -dalign-reader \
- $(READONLY_SCHEMA_LIBS) \
- -lvdb \
- -lkdb \
- -lkapp \
- -lkfg \
- -lkfs \
- -lvfs \
- -lksrch \
- -lklib \
- -lm
-
-$(BINDIR)/run-stat: $(RUN_STAT_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(RUN_STAT_LIB)
-
diff --git a/tools/run-stat/chart.c b/tools/run-stat/chart.c
deleted file mode 100644
index 0c709ac..0000000
--- a/tools/run-stat/chart.c
+++ /dev/null
@@ -1,465 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/log.h>
-#include <klib/text.h>
-#include <klib/printf.h>
-#include <klib/out.h>
-
-#include <os-native.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-
-#include "chart.h"
-
-
-char * create_line_style( p_chart chart, const char * color, const uint32_t width )
-{
- size_t name_size = string_size( color ) + 10;
- char * name = malloc( name_size );
- if ( name != NULL )
- {
- rc_t rc = string_printf( name, name_size, NULL, ".line_%s", color );
- if ( rc == 0 )
- {
- size_t value_size = name_size + 48;
- char * value = malloc( value_size );
- if ( value != NULL )
- {
- rc = string_printf( value, value_size, NULL,
- "fill:none; stroke: %s; stroke-width: %d;",
- color, width );
- if ( rc == 0 )
- rc = svg_set_style( &chart->grafic, name, value );
- free( value );
- }
- else
- rc = -1;
- }
- if ( rc != 0 )
- {
- free( name );
- name = NULL;
- }
- }
- return name;
-}
-
-
-char * create_fill_style( p_chart chart, const char * line_color,
- const char * fill_color, const uint32_t width )
-{
- size_t name_size = string_size( fill_color ) + 10;
- char * name = malloc( name_size );
- if ( name != NULL )
- {
- rc_t rc = string_printf( name, name_size, NULL, ".fill_%s", fill_color );
- if ( rc == 0 )
- {
- size_t value_size = name_size + string_size( line_color ) + 48;
- char * value = malloc( value_size );
- if ( value != NULL )
- {
- rc = string_printf( value, value_size, NULL,
- "fill: %s; stroke: %s; stroke-width: %d;",
- fill_color, line_color, width );
- if ( rc == 0 )
- rc = svg_set_style( &chart->grafic, name, value );
- free( value );
- }
- else
- rc = -1;
- }
- if ( rc != 0 )
- {
- free( name );
- name = NULL;
- }
- }
- return name;
-}
-
-
-static void chart_styles( p_chart chart )
-{
- /* populate the styles */
- svg_set_style( &chart->grafic,
- ".thick", "stroke: black; stroke-width: 3;" );
- svg_set_style( &chart->grafic,
- ".medium", "stroke: black; stroke-width: 2;" );
- svg_set_style( &chart->grafic,
- ".thin", "stroke: black; stroke-width: 1;" );
-
- svg_set_style( &chart->grafic, ".big", "font-size: 24; text-anchor: middle;" );
- svg_set_style( &chart->grafic, ".small", "font-size: 12;" );
- svg_set_style( &chart->grafic, ".small_m", "font-size: 12; text-anchor: middle;" );
- svg_set_style( &chart->grafic, ".small_e", "font-size: 12; text-anchor: end;" );
-}
-
-
-static void chart_axis( p_chart chart )
-{
- svg_line( &chart->grafic,
- chart->xofs, chart->yofs,
- chart->xofs, chart->yofs + chart->dy,
- "thick" );
- svg_line( &chart->grafic,
- chart->xofs, chart->yofs + chart->dy,
- chart->xofs + chart->dx, chart->yofs + chart->dy,
- "thick" );
-}
-
-
-rc_t chart_init( p_chart chart,
- uint32_t xofs, uint32_t yofs, uint32_t dx, uint32_t dy )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL )
- {
- rc = svg_init( &chart->grafic );
- if ( rc == 0 )
- {
- chart->xofs = xofs;
- chart->yofs = yofs;
- chart->dx = dx;
- chart->dy = dy;
- chart->entries = 0;
-
- chart_styles( chart );
- chart_axis( chart );
- }
- }
- return rc;
-}
-
-
-void chart_destroy( p_chart chart )
-{
- if ( chart != NULL )
- svg_destroy( &chart->grafic );
-}
-
-
-rc_t chart_write( p_chart chart, const char * filename )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL && filename != NULL )
- {
- rc = svg_write( &chart->grafic, filename );
- }
- return rc;
-}
-
-
-rc_t chart_captions( p_chart chart,
- const char * caption,
- const char * caption_x,
- const char * caption_y )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL )
- {
- /* the main caption centered, 24pt below the upper line (center-aligned)*/
- svg_text( &chart->grafic,
- chart->xofs + ( chart->dx / 2 ), chart->yofs + 24,
- caption, "big" );
- /* caption of the y-axis, 12pt below the upper line (end-aligned)*/
- svg_text( &chart->grafic,
- chart->xofs -5, chart->yofs + 12,
- caption_y, "small_e" );
- /* caption of the x-axis, 6pt above the x-axis (end-aligned) */
- svg_text( &chart->grafic,
- chart->xofs + chart->dx, chart->yofs + chart->dy - 6,
- caption_x, "small_e" );
- }
- return rc;
-}
-
-
-long chart_round( double x )
-{
- if ( x >= 0 )
- return (long) ( x+0.5 );
- return (long) ( x-0.5 );
-}
-
-
-uint32_t chart_calc_y( const uint32_t value, double factor )
-{
- return ( chart_round( factor * value ) );
-}
-
-
-/* the scale of the y-axis */
-rc_t chart_vruler( p_chart chart,
- uint32_t from, uint32_t to,
- double factor,
- uint32_t spread,
- const char * ext )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL )
- {
- svg_path path;
- rc = svg_path_init( &path );
- if ( rc == 0 )
- {
- uint32_t idx;
- uint32_t x = chart->xofs;
- uint32_t y = chart->yofs + chart->dy;
-
- for ( idx = from; idx <= to && rc == 0; idx += spread )
- {
- char temp[ 32 ];
- uint32_t y1 = y - chart_calc_y( idx, factor );
-
- svg_path_move_to( &path, true, x, y1 );
- svg_path_hline_to( &path, false, -5 );
-
- if ( ext != NULL )
- rc = string_printf( temp, sizeof( temp ), NULL, "%u%s", idx, ext );
- else
- rc = string_printf( temp, sizeof( temp ), NULL, "%u", idx );
- if ( rc == 0 )
- svg_text( &chart->grafic, x - 10, y1, temp, "small_e" );
- }
- svg_set_path( &chart->grafic, &path, "thin" );
- svg_path_destroy( &path );
- }
- }
- return rc;
-}
-
-
-/* the scale of the y-axis */
-rc_t chart_vruler1( p_chart chart,
- uint32_t max_y, uint32_t step_y, uint32_t max_value,
- const char * ext )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL )
- {
- svg_path path;
- rc = svg_path_init( &path );
- if ( rc == 0 )
- {
- uint32_t idx;
- uint32_t x = chart->xofs;
- uint32_t y = chart->yofs + chart->dy;
-
- for ( idx = 0; idx <= max_y && rc == 0; idx += step_y )
- {
- char temp[ 32 ];
- uint64_t value = max_value;
- value *= idx;
- value /= max_y;
-
- svg_path_move_to( &path, true, x, y - idx );
- svg_path_hline_to( &path, false, -5 );
-
- if ( ext != NULL )
- rc = string_printf( temp, sizeof( temp ), NULL, "%u%s", value, ext );
- else
- rc = string_printf( temp, sizeof( temp ), NULL, "%u", value );
- if ( rc == 0 )
- svg_text( &chart->grafic, x - 10, y - idx, temp, "small_e" );
- }
- svg_set_path( &chart->grafic, &path, "thin" );
- svg_path_destroy( &path );
- }
- }
- return rc;
-}
-
-
-/* the scale of the x-axis */
-rc_t chart_hruler( p_chart chart,
- uint32_t *values, uint32_t n_values,
- uint32_t xofs, uint32_t spread )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL && values != NULL )
- {
- svg_path path;
- rc = svg_path_init( &path );
- if ( rc == 0 )
- {
- uint32_t idx;
- uint32_t y = chart->yofs + chart->dy;
- uint32_t x = chart->xofs + xofs;
- rc = 0;
- for ( idx = 0; idx < n_values && rc == 0; ++idx )
- {
- char temp[ 16 ];
-
- svg_path_move_to( &path, true, x, y );
- svg_path_vline_to( &path, false, 5 );
-
- rc = string_printf( temp, sizeof( temp ), NULL, "%u", values[ idx ] );
- if ( rc == 0 )
- {
- if ( ( idx & 1 ) == 0 )
- svg_text( &chart->grafic, x, y + 20, temp, "small_m" );
- else
- svg_text( &chart->grafic, x, y + 30, temp, "small_m" );
- }
-
- x += spread;
- }
- svg_set_path( &chart->grafic, &path, "thin" );
- svg_path_destroy( &path );
- }
- }
- return rc;
-}
-
-
-rc_t chart_line( p_chart chart,
- const uint32_t * values,
- uint32_t n_values,
- double factor,
- uint32_t spread,
- const char * color,
- const char * name )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL && values != NULL && color != NULL && name != NULL )
- {
- /* create a style for every line with the requested color */
- char * style_name = create_line_style( chart, color, 1 );
- if ( style_name != NULL )
- {
- /* translate the given values (factor) into an array of
- xy-coordinates, render that as polyline */
- uint32_t y = chart->yofs + chart->dy;
- uint32_t * points = malloc( sizeof( values[ 0 ] ) * n_values * 2 );
- if ( points != NULL )
- {
- uint32_t idx, x = chart->xofs;
- for ( idx = 0; idx < n_values; ++idx )
- {
- points[ idx * 2 ] = x;
- points[ idx * 2 + 1 ] = y - chart_calc_y( values[ idx ], factor );
- x += spread;
- }
- rc = svg_polyline( &chart->grafic, &style_name[ 1 ],
- points, n_values * 2 );
- free( points );
- }
-
- if ( rc == 0 )
- {
- /* print a short line with the right color, follow by the name */
- uint32_t x = chart->xofs + ( chart->entries * 100 );
- y += 50;
- svg_line( &chart->grafic, x, y, x + 20, y, &style_name[ 1 ] );
- svg_text( &chart->grafic, x +25, y, name, "small" );
- chart->entries++;
- }
- free( style_name );
- }
- }
- return rc;
-}
-
-
-rc_t chart_bar2d( p_chart chart,
- const uint32_t x,
- const uint32_t h,
- const uint32_t dx,
- double factor,
- const char * line_color,
- const char * fill_color,
- const char * name )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL && line_color != NULL && fill_color!= NULL && name != NULL )
- {
- /* create a style for every line with the requested color */
- char * style_name = create_fill_style( chart, line_color, fill_color, 1 );
- if ( style_name != NULL )
- {
- uint32_t y = chart->yofs + chart->dy;
- uint32_t dy = chart_calc_y( h, factor );
- uint32_t x1 = x + chart->xofs;
- rc = svg_rect( &chart->grafic, x1, y - dy, dx, dy,
- &(style_name[ 1 ]) );
- free( style_name );
- }
- }
- return rc;
-}
-
-
-rc_t chart_box_whisker( p_chart chart,
- const uint32_t x,
- const uint32_t dx,
- const uint32_t p_low,
- const uint32_t p_high,
- const uint32_t q_low,
- const uint32_t q_high,
- const uint32_t median,
- double factor,
- const char * box_style,
- const char * median_style )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( chart != NULL && box_style != NULL && median_style != NULL )
- {
- uint32_t y = chart->yofs + chart->dy;
- uint32_t y1 = y - chart_calc_y( p_low, factor );
- uint32_t y2 = y - chart_calc_y( p_high, factor );
- uint32_t x1 = x + chart->xofs + dx / 2;
-
- /* draw the "whisker"*/
- rc = svg_line( &chart->grafic, x1, y1, x1, y2, "thin" );
- if ( rc == 0 )
- rc = svg_line( &chart->grafic, x1 - ( dx / 4 ), y1,
- x1 + ( dx / 4 ), y1, "thin" );
- if ( rc == 0 )
- rc = svg_line( &chart->grafic, x1 - ( dx / 4 ), y2,
- x1 + ( dx / 4 ), y2, "thin" );
-
- /* draw the "box" over the "whisker" */
- if ( rc == 0 )
- {
- x1 = x + chart->xofs;
- y1 = y - chart_calc_y( q_low, factor );
- y2 = y - chart_calc_y( q_high, factor );
- rc = svg_rect( &chart->grafic, x1, y2, dx, y1-y2, box_style );
- }
-
- /* draw the "median" over the "box" */
- if ( rc == 0 )
- {
- y1 = y - chart_calc_y( median, factor );
- rc = svg_line( &chart->grafic, x1, y1, x1 + dx, y1, median_style );
- }
- }
- return rc;
-}
diff --git a/tools/run-stat/chart.h b/tools/run-stat/chart.h
deleted file mode 100644
index 972e7d9..0000000
--- a/tools/run-stat/chart.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_chart_
-#define _h_chart_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "svg.h"
-
-typedef struct chart
-{
- svg grafic;
- uint32_t xofs, yofs;
- uint32_t dx, dy;
- uint32_t entries;
-} chart;
-typedef chart* p_chart;
-
-rc_t chart_init( p_chart chart,
- uint32_t xofs, uint32_t yofs, uint32_t dx, uint32_t dy );
-
-void chart_destroy( p_chart chart );
-
-rc_t chart_write( p_chart chart, const char * filename );
-
-rc_t chart_captions( p_chart chart,
- const char * caption,
- const char * caption_x,
- const char * caption_y );
-
-long chart_round( double x );
-uint32_t chart_calc_y( const uint32_t value, double factor );
-
-rc_t chart_vruler( p_chart chart,
- uint32_t from, uint32_t to,
- double factor,
- uint32_t spread,
- const char * ext );
-
-rc_t chart_vruler1( p_chart chart,
- uint32_t max_y, uint32_t step_y, uint32_t max_value,
- const char * ext );
-
-rc_t chart_hruler( p_chart chart,
- uint32_t *values, uint32_t n_values,
- uint32_t xofs, uint32_t spread );
-
-char * create_line_style( p_chart chart, const char * color,
- const uint32_t width );
-
-char * create_fill_style( p_chart chart, const char * line_color,
- const char * fill_color, const uint32_t width );
-
-rc_t chart_line( p_chart chart,
- const uint32_t * values,
- uint32_t n_values,
- double factor,
- uint32_t spread,
- const char * color,
- const char * name );
-
-rc_t chart_bar2d( p_chart chart,
- const uint32_t x,
- const uint32_t h,
- const uint32_t dx,
- double factor,
- const char * line_color,
- const char * fill_color,
- const char * name );
-
-rc_t chart_box_whisker( p_chart chart,
- const uint32_t x,
- const uint32_t dx,
- const uint32_t p_low,
- const uint32_t p_high,
- const uint32_t q_low,
- const uint32_t q_high,
- const uint32_t median,
- double factor,
- const char * box_style,
- const char * median_style );
-
-#endif
diff --git a/tools/run-stat/context.c b/tools/run-stat/context.c
deleted file mode 100644
index 3a41419..0000000
--- a/tools/run-stat/context.c
+++ /dev/null
@@ -1,439 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "context.h"
-
-#include <klib/rc.h>
-#include <klib/log.h>
-#include <klib/status.h>
-#include <klib/text.h>
-#include <kapp/args.h>
-#include <os-native.h>
-#include <sysalloc.h>
-
-#include "helper.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-
-
-static rc_t ctx_set_str( char **dst, const char *src )
-{
- size_t len;
- if ( dst == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- }
- if ( *dst != NULL )
- {
- free( *dst );
- *dst = NULL;
- }
- if ( src == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- }
- len = strlen( src );
- if ( len == 0 )
- {
- return RC( rcVDB, rcNoTarg, rcWriting, rcItem, rcEmpty );
- }
- *dst = (char*)malloc( len + 1 );
- if ( *dst == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcWriting, rcMemory, rcExhausted );
- }
- strcpy( *dst, src );
- return 0;
-}
-
-
-static void ctx_init_values( p_stat_ctx ctx )
-{
- ctx->path = NULL;
- ctx->table = NULL;
- ctx->schema_list = NULL;
- ctx->module_list = NULL;
- ctx->output_path = NULL;
- ctx->name_prefix = NULL;
-
- ctx->usage_requested = false;
- ctx->dont_check_accession = false;
- ctx->show_progress = false;
-}
-
-rc_t ctx_init( stat_ctx **ctx )
-{
- rc_t rc = 0;
- if ( ctx == NULL )
- {
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
- }
- if ( rc == 0 )
- {
- (*ctx) = (p_stat_ctx)calloc( 1, sizeof( stat_ctx ) );
- if ( *ctx == NULL )
- {
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- if ( rc == 0 )
- {
- ctx_init_values( *ctx );
- rc = ng_make( &((*ctx)->row_generator) );
- DISP_RC( rc, "num_gen_make() failed" );
- }
- }
- return rc;
-}
-
-
-rc_t ctx_destroy( p_stat_ctx ctx )
-{
- rc_t rc = 0;
- if ( ctx == NULL )
- {
- rc = RC( rcVDB, rcNoTarg, rcDestroying, rcParam, rcNull );
- }
- if ( rc == 0 )
- {
- if ( ctx->path != NULL )
- {
- free( (void*)ctx->path );
- ctx->path = NULL;
- }
-
- if ( ctx->table != NULL )
- {
- free( (void*)ctx->table );
- ctx->table = NULL;
- }
-
- if ( ctx->name_prefix != NULL )
- {
- free( (void*)ctx->name_prefix );
- ctx->name_prefix = NULL;
- }
-
- if ( ctx->output_path != NULL )
- {
- free( (void*)ctx->output_path );
- ctx->output_path = NULL;
- }
-
- if ( ctx->schema_list != NULL )
- {
- KNamelistRelease( ctx->schema_list );
- ctx->schema_list = NULL;
- }
-
- if ( ctx->module_list != NULL )
- {
- KNamelistRelease( ctx->module_list );
- ctx->module_list = NULL;
- }
-
- ng_destroy( ctx->row_generator );
- free( ctx );
- }
- return rc;
-}
-
-
-static rc_t ctx_set_path( p_stat_ctx ctx, const char *src )
-{
- rc_t rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- if ( ctx != NULL && src != NULL )
- {
- rc = ctx_set_str( (char**)&(ctx->path), src );
- DISP_RC( rc, "dump_context_set_str() failed" );
- }
- return rc;
-}
-
-
-/* not static because can be called directly from run-stat.c */
-rc_t ctx_set_table( p_stat_ctx ctx, const char *src )
-{
- rc_t rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- if ( ctx != NULL && src != NULL )
- {
- rc = ctx_set_str( (char**)&(ctx->table), src );
- DISP_RC( rc, "stat_context_set_str() failed" );
- }
- return rc;
-}
-
-
-static rc_t ctx_set_row_range( p_stat_ctx ctx, const char *src )
-{
- rc_t rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- if ( ctx != NULL && src != NULL )
- {
- ng_parse( ctx->row_generator, src );
- rc = 0;
- }
- return rc;
-}
-
-
-static rc_t ctx_set_name_prefix( p_stat_ctx ctx, const char *src )
-{
- rc_t rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- if ( ctx != NULL )
- {
- if ( src != NULL )
- {
- rc = ctx_set_str( (char**)&(ctx->name_prefix), src );
- DISP_RC( rc, "dump_context_set_str() failed" );
- }
- else
- {
- ctx->name_prefix = string_dup_measure( DEFAULT_REPORT_PREFIX, NULL );
- rc = 0;
- }
- }
- return rc;
-}
-
-
-static rc_t ctx_set_output_path( p_stat_ctx ctx, const char *src )
-{
- rc_t rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- if ( ctx != NULL && src != NULL )
- {
- rc = ctx_set_str( (char**)&(ctx->output_path), src );
- DISP_RC( rc, "dump_context_set_str() failed" );
- }
- return rc;
-}
-
-
-static rc_t ctx_set_report_type( p_stat_ctx ctx, const char *src )
-{
- rc_t rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- if ( ctx != NULL )
- {
- ctx->report_type = RT_TXT; /* default */
- if ( src != NULL )
- {
- if ( strcmp( src, "csv" ) == 0 )
- ctx->report_type = RT_CSV;
- else if ( strcmp( src, "xml" ) == 0 )
- ctx->report_type = RT_XML;
- else if ( strcmp( src, "json" ) == 0 )
- ctx->report_type = RT_JSO;
- }
- rc = 0;
- }
- return rc;
-}
-
-
-static bool ctx_check_if_usage_necessary( p_stat_ctx ctx )
-{
- if ( ctx == NULL ) return false;
- if ( ctx->path == NULL ) ctx->usage_requested = true;
- return ctx->usage_requested;
-}
-
-
-static rc_t ctx_evaluate_arguments( const Args *my_args,
- p_stat_ctx ctx )
-{
- uint32_t count;
- rc_t rc = ArgsParamCount( my_args, &count );
- DISP_RC( rc, "ArgsParamCount() failed" );
- if ( rc == 0 )
- {
- uint32_t idx;
- for ( idx=0; idx < count; ++idx )
- {
- const char *value = NULL;
- rc = ArgsParamValue( my_args, idx, &value );
- DISP_RC( rc, "ArgsParamValue() failed" );
- if ( rc == 0 )
- {
- switch( idx )
- {
- case 0 : rc = ctx_set_path( ctx, value );
- DISP_RC( rc, "ctx_set_path() failed" );
- break;
- }
- }
- }
- }
- return rc;
-}
-
-
-static bool ctx_get_bool_option( const Args *my_args,
- const char *name,
- const bool def )
-{
- bool res = def;
- uint32_t count;
- rc_t rc = ArgsOptionCount( my_args, name, &count );
- DISP_RC( rc, "ArgsOptionCount() failed" );
- if ( rc == 0 )
- res = ( count > 0 );
- return res;
-}
-
-
-#if 0
-static bool ctx_get_bool_neg_option( const Args *my_args,
- const char *name,
- const bool def )
-{
- bool res = def;
- uint32_t count;
- rc_t rc = ArgsOptionCount( my_args, name, &count );
- DISP_RC( rc, "ArgsOptionCount() failed" );
- if ( rc == 0 )
- res = ( count == 0 );
- return res;
-}
-
-
-static uint16_t ctx_get_uint16_option( const Args *my_args,
- const char *name,
- const uint16_t def )
-{
- uint16_t res = def;
- uint32_t count;
- rc_t rc = ArgsOptionCount( my_args, name, &count );
- DISP_RC( rc, "ArgsOptionCount() failed" );
- if ( ( rc == 0 )&&( count > 0 ) )
- {
- const char *s;
- rc = ArgsOptionValue( my_args, name, 0, &s );
- DISP_RC( rc, "ArgsOptionValue() failed" );
- if ( rc == 0 ) res = atoi( s );
- }
- return res;
-}
-#endif
-
-static const char* ctx_get_str_option( const Args *my_args,
- const char *name )
-{
- const char* res = NULL;
- uint32_t count;
- rc_t rc = ArgsOptionCount( my_args, name, &count );
- DISP_RC( rc, "ArgsOptionCount() failed" );
- if ( ( rc == 0 )&&( count > 0 ) )
- {
- rc = ArgsOptionValue( my_args, name, 0, &res );
- DISP_RC( rc, "ArgsOptionValue() failed" );
- }
- return res;
-}
-
-
-uint32_t context_schema_count( p_stat_ctx ctx )
-{
- uint32_t res = 0;
- if ( ctx != NULL )
- if ( ctx->schema_list != 0 )
- {
- uint32_t count;
- if ( KNamelistCount( ctx->schema_list, &count ) == 0 )
- res = count;
- }
- return res;
-}
-
-
-static void ctx_evaluate_modules( const Args *my_args, p_stat_ctx ctx )
-{
- uint32_t count;
- rc_t rc = ArgsOptionCount( my_args, OPTION_MODULE, &count );
- DISP_RC( rc, "ArgsOptionCount( OPTION_MODULE ) failed" );
- if ( ( rc == 0 )&( count > 0 ) )
- {
- VNamelist *v_names;
- rc_t rc = VNamelistMake ( &v_names, 5 );
- if ( rc == 0 )
- {
- uint32_t i;
- for ( i = 0; i < count; ++i )
- {
- const char *s;
- rc = ArgsOptionValue( my_args, OPTION_MODULE, i, &s );
- DISP_RC( rc, "ArgsOptionValue(OPTION_MODULE) failed" );
- if ( rc == 0 )
- VNamelistAppend ( v_names, s );
- }
- VNamelistToConstNamelist ( v_names, &ctx->module_list );
- VNamelistRelease( v_names );
- }
- }
-}
-
-
-static void ctx_evaluate_options( const Args *my_args, p_stat_ctx ctx )
-{
- if ( my_args == NULL ) return;
- if ( ctx == NULL ) return;
-
- ctx->dont_check_accession = ctx_get_bool_option( my_args,
- OPTION_WITHOUT_ACCESSION, false );
- ctx->show_progress = ctx_get_bool_option( my_args,
- OPTION_PROGRESS, false );
-
- ctx_set_table( ctx, ctx_get_str_option( my_args, OPTION_TABLE ) );
- ctx_set_row_range( ctx, ctx_get_str_option( my_args, OPTION_ROWS ) );
-
- helper_make_namelist_from_string( &(ctx->schema_list),
- ctx_get_str_option( my_args, OPTION_SCHEMA ), ',' );
-
- ctx_evaluate_modules( my_args, ctx );
-
- ctx->produce_grafic = ctx_get_bool_option( my_args, OPTION_GRAFIC, false );
- ctx_set_report_type( ctx, ctx_get_str_option( my_args, OPTION_REPORT ) );
- ctx_set_output_path( ctx, ctx_get_str_option( my_args, OPTION_OUTPUT ) );
- ctx_set_name_prefix( ctx, ctx_get_str_option( my_args, OPTION_PREFIX ) );
-}
-
-
-rc_t ctx_capture_arguments_and_options( const Args * args, p_stat_ctx ctx )
-{
- rc_t rc;
-
- rc = ctx_evaluate_arguments( args, ctx );
- DISP_RC( rc, "ctx_evaluate_arguments() failed" );
- if ( rc == 0 )
- {
- ctx_evaluate_options( args, ctx );
- ctx_check_if_usage_necessary( ctx );
-
- rc = ArgsHandleLogLevel( args );
- DISP_RC( rc, "ArgsHandleLogLevel() failed" );
- }
- return rc;
-}
diff --git a/tools/run-stat/context.h b/tools/run-stat/context.h
deleted file mode 100644
index 841ee3b..0000000
--- a/tools/run-stat/context.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_run_stat_context_
-#define _h_run_stat_context_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include <kapp/args.h>
-#include <klib/vector.h>
-#include <klib/namelist.h>
-#include "definitions.h"
-#include "num-gen.h"
-
-#define OPTION_TABLE "table"
-#define OPTION_ROWS "rows"
-#define OPTION_SCHEMA "schema"
-#define OPTION_WITHOUT_ACCESSION "without_accession"
-#define OPTION_PROGRESS "progress"
-#define OPTION_MODULE "module"
-#define OPTION_GRAFIC "grafic"
-#define OPTION_REPORT "report"
-#define OPTION_OUTPUT "output"
-#define OPTION_PREFIX "prefix"
-
-
-#define ALIAS_TABLE "T"
-#define ALIAS_ROWS "R"
-#define ALIAS_SCHEMA "S"
-#define ALIAS_WITHOUT_ACCESSION "a"
-#define ALIAS_PROGRESS "p"
-#define ALIAS_MODULE "m"
-#define ALIAS_GRAFIC "g"
-#define ALIAS_RREPORT "r"
-#define ALIAS_OUTPUT "o"
-#define ALIAS_PREFIX "b"
-
-/********************************************************************
-the dump context contains all informations needed to execute the dump
-********************************************************************/
-typedef struct stat_ctx
-{
- const char *path;
- const KNamelist *schema_list;
- const KNamelist *module_list;
- const char *table;
- p_ng row_generator;
-
- bool usage_requested;
- bool dont_check_accession;
- bool show_progress;
- bool produce_grafic;
- uint32_t report_type;
-
- const char *output_path;
- const char *name_prefix;
-} stat_ctx;
-typedef stat_ctx* p_stat_ctx;
-
-
-rc_t ctx_init( stat_ctx **ctx );
-rc_t ctx_destroy( p_stat_ctx ctx );
-
-uint32_t context_schema_count( p_stat_ctx ctx );
-
-rc_t ctx_set_table( p_stat_ctx ctx, const char *src );
-
-rc_t ctx_capture_arguments_and_options( const Args * args, p_stat_ctx ctx );
-
-#endif
diff --git a/tools/run-stat/helper.c b/tools/run-stat/helper.c
deleted file mode 100644
index 2376e43..0000000
--- a/tools/run-stat/helper.c
+++ /dev/null
@@ -1,715 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "helper.h"
-#include "context.h"
-
-#include <kfs/file.h>
-#include <klib/printf.h>
-#include <os-native.h>
-
-#include <sysalloc.h>
-#include <bitstr.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-
-
-int helper_str_cmp( const char *a, const char *b )
-{
- size_t asize = string_size ( a );
- size_t bsize = string_size ( b );
- return strcase_cmp ( a, asize, b, bsize, ( asize > bsize ) ? asize : bsize );
-}
-
-
-rc_t char_buffer_init( p_char_buffer buffer, const size_t size )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL )
- {
- buffer->len = 0;
- rc = 0;
- if ( size > 0 )
- {
- buffer->ptr = malloc( size );
- if ( buffer->ptr != NULL )
- buffer->size = size;
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- else
- {
- buffer->ptr = NULL;
- buffer->size = 0;
- }
- buffer->trans_ptr = buffer->ptr;
- }
- return rc;
-}
-
-
-void char_buffer_destroy( p_char_buffer buffer )
-{
- if ( buffer != NULL )
- {
- free( buffer->ptr );
- buffer->ptr = NULL;
- buffer->len = 0;
- buffer->size = 0;
- }
-}
-
-
-rc_t char_buffer_realloc( p_char_buffer buffer, const size_t new_size )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL )
- {
- if ( buffer->ptr == NULL )
- rc = char_buffer_init( buffer, new_size );
- else
- {
- rc = 0;
- if ( new_size > buffer->size )
- {
- buffer->ptr = realloc( buffer->ptr, new_size );
- if ( buffer->ptr != NULL )
- {
- buffer->size = new_size;
- buffer->trans_ptr = buffer->ptr;
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- }
- }
- return rc;
-}
-
-
-rc_t char_buffer_append_cstring( p_char_buffer buffer, const char * s )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL )
- {
- size_t s_len = string_size( s );
- if ( s_len > 0 )
- {
- if ( buffer->ptr == NULL )
- {
- rc = char_buffer_init( buffer, s_len + 1 );
- if ( rc == 0 )
- {
- string_copy ( buffer->ptr, buffer->size, s, s_len );
- }
- }
- else
- {
- size_t needed = buffer->len + s_len + 1;
- if ( needed < buffer->size )
- {
- string_copy ( buffer->ptr + buffer->len,
- buffer->size - buffer->len, s, s_len );
- buffer->len += s_len;
- rc = 0;
- }
- else
- {
- rc = char_buffer_realloc( buffer, needed );
- if ( rc == 0 )
- {
- string_copy ( buffer->ptr + buffer->len,
- buffer->size - buffer->len, s, s_len );
- buffer->len += s_len;
- }
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t char_buffer_printfv( p_char_buffer buffer, const size_t estimated_len,
- const char * fmt, va_list args )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL )
- {
- bool done = false;
- size_t len = estimated_len + 1;
-
- while ( !done )
- {
- if ( buffer->ptr == NULL )
- rc = char_buffer_init( buffer, len );
- else
- {
- size_t needed = buffer->len + len;
- rc = 0;
- if ( needed > buffer->size )
- rc = char_buffer_realloc( buffer, needed );
- }
- done = ( rc != 0 );
- if ( !done )
- {
- size_t written;
- rc = string_vprintf ( buffer->ptr + buffer->len,
- buffer->size - buffer->len,
- &written,
- fmt,
- args );
- done = ( rc == 0 );
- if ( done )
- buffer->len += written;
- else
- len += len;
- }
- }
- }
- return rc;
-}
-
-
-rc_t char_buffer_printf( p_char_buffer buffer, const size_t estimated_len,
- const char * fmt, ... )
-{
- rc_t rc;
- va_list args;
- va_start ( args, fmt );
- rc = char_buffer_printfv( buffer, estimated_len, fmt, args );
- va_end ( args );
- return rc;
-}
-
-
-rc_t char_buffer_saveto( p_char_buffer buffer, const char * filename )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL && buffer->len > 0 && filename != NULL )
- {
- KDirectory * dir;
- rc = KDirectoryNativeDir ( &dir );
- if ( rc == 0 )
- {
- KFile * f;
- rc = KDirectoryCreateFile ( dir, &f, true, 0664, kcmInit, "%s", filename );
- if ( rc == 0 )
- {
- rc = KFileWrite ( f, 0, buffer->ptr, buffer->len, NULL );
- KFileRelease( f );
- }
- KDirectoryRelease( dir );
- }
- }
- return rc;
-}
-
-
-rc_t int_buffer_init( p_int_buffer buffer, const size_t size )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL )
- {
- buffer->len = 0;
- rc = 0;
- if ( size > 0 )
- {
- buffer->ptr = malloc( size * sizeof( buffer->ptr[ 0 ] ) );
- if ( buffer->ptr != NULL )
- buffer->size = size;
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- else
- {
- buffer->ptr = NULL;
- buffer->size = 0;
- }
- }
- return rc;
-
-}
-
-
-void int_buffer_destroy( p_int_buffer buffer )
-{
- if ( buffer != NULL )
- {
- free( buffer->ptr );
- buffer->ptr = NULL;
- buffer->len = 0;
- buffer->size = 0;
- }
-}
-
-
-rc_t int_buffer_realloc( p_int_buffer buffer, const size_t new_size )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( buffer != NULL )
- {
- if ( buffer->ptr == NULL )
- rc = int_buffer_init( buffer, new_size );
- else
- {
- rc = 0;
- if ( new_size > buffer->size )
- {
- buffer->ptr = realloc( buffer->ptr, new_size * sizeof( buffer->ptr[ 0 ] ) );
- if ( buffer->ptr != NULL )
- buffer->size = new_size;
- else
- rc= RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- }
- }
- return rc;
-}
-
-
-/*
- * calls the given manager to create a new SRA-schema
- * takes the list of user-supplied schema's (which can be empty)
- * and let the created schema parse all of them
-*/
-rc_t helper_parse_schema( const VDBManager *my_manager,
- VSchema **new_schema,
- const KNamelist *schema_list )
-{
- rc_t rc;
- if ( my_manager == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( new_schema == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
-
- rc = VDBManagerMakeSRASchema( my_manager, new_schema );
- DISP_RC( rc, "VDBManagerMakeSRASchema() failed" );
-
- if ( ( rc == 0 )&&( schema_list != NULL ) )
- {
- uint32_t count;
- if ( KNamelistCount( schema_list, &count ) == 0 )
- {
- uint32_t idx;
- for ( idx = 0; idx < count && rc == 0; ++idx )
- {
- const char *s;
- if ( KNamelistGet( schema_list, idx, &s ) == 0 )
- {
- rc = VSchemaParseFile( *new_schema, "%s", s );
- DISP_RC( rc, "VSchemaParseFile() failed" );
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t helper_make_namelist_from_string( const KNamelist **list,
- const char * src,
- const char split_char )
-{
- VNamelist *v_names;
- rc_t rc = VNamelistMake ( &v_names, 5 );
- if ( rc == 0 )
- {
- if ( ( src != NULL )&&( src[ 0 ] != 0 ) )
- {
- char * s = string_dup_measure ( src, NULL );
- if ( s )
- {
- uint32_t str_begin = 0;
- uint32_t str_end = 0;
- char c;
- do
- {
- c = s[ str_end ];
- if ( c == split_char || c == 0 )
- {
- if ( str_begin < str_end )
- {
- char c_temp = c;
- s[ str_end ] = 0;
- rc = VNamelistAppend ( v_names, &(s[str_begin]) );
- s[ str_end ] = c_temp;
- }
- str_begin = str_end + 1;
- }
- str_end++;
- } while ( c != 0 && rc == 0 );
- free( s );
- }
- }
- rc = VNamelistToConstNamelist ( v_names, list );
- VNamelistRelease( v_names );
- }
- return rc;
-}
-
-
-/*************************************************************************************
-helper-function to check if a given table is in the list of tables
-if found put that name into the dump-context
-*************************************************************************************/
-bool helper_take_this_table_from_db( const VDatabase * db,
- p_stat_ctx ctx,
- const char * table_to_find )
-{
- bool we_found_a_table = false;
- KNamelist *tbl_names;
- rc_t rc = VDatabaseListTbl( db, &tbl_names );
- DISP_RC( rc, "VDatabaseListTbl() failed" );
- if ( rc == 0 )
- {
- uint32_t n;
- rc = KNamelistCount( tbl_names, &n );
- DISP_RC( rc, "KNamelistCount() failed" );
- if ( ( rc == 0 )&&( n > 0 ) )
- {
- uint32_t i;
- for ( i = 0; i < n && rc == 0 && !we_found_a_table; ++i )
- {
- const char *tbl_name;
- rc = KNamelistGet( tbl_names, i, &tbl_name );
- DISP_RC( rc, "KNamelistGet() failed" );
- if ( rc == 0 )
- {
- if ( helper_str_cmp( tbl_name, table_to_find ) == 0 )
- {
- ctx_set_table( ctx, tbl_name );
- we_found_a_table = true;
- }
- }
- }
- }
- rc = KNamelistRelease( tbl_names );
- DISP_RC( rc, "KNamelistRelease() failed" );
- }
- return we_found_a_table;
-}
-
-
-bool helper_take_1st_table_from_db( const VDatabase *db,
- p_stat_ctx ctx )
-{
- bool we_found_a_table = false;
- KNamelist *tbl_names;
- rc_t rc = VDatabaseListTbl( db, &tbl_names );
- DISP_RC( rc, "VDatabaseListTbl() failed" );
- if ( rc == 0 )
- {
- uint32_t n;
- rc = KNamelistCount( tbl_names, &n );
- DISP_RC( rc, "KNamelistCount() failed" );
- if ( ( rc == 0 )&&( n > 0 ) )
- {
- const char *tbl_name;
- rc = KNamelistGet( tbl_names, 0, &tbl_name );
- DISP_RC( rc, "KNamelistGet() failed" );
- if ( rc == 0 )
- {
- ctx_set_table( ctx, tbl_name );
- we_found_a_table = true;
- }
- }
- rc = KNamelistRelease( tbl_names );
- DISP_RC( rc, "KNamelistRelease() failed" );
- }
- return we_found_a_table;
-}
-
-
-char * helper_concat( const char * s1, const char * s2 )
-{
- size_t l1 = string_size ( s1 );
- size_t l2 = string_size ( s2 );
- size_t l = l1 + l2 + 1;
- char * res = malloc( l );
- if ( res != NULL )
- {
- size_t l3 = string_copy ( res, l, s1, l1 );
- string_copy ( res + l3, l - l3, s2, l2 );
- }
- return res;
-}
-
-
-double percent( const uint64_t value, const uint64_t sum )
-{
- double res = 0.0;
- if ( sum > 0 && value > 0 )
- {
- res = value;
- res *= 100;
- res /= sum;
- }
- return res;
-}
-
-
-/*********************************************************************************
- "bases(bio)" ---> module = "bases", param = "bio"
-*********************************************************************************/
-rc_t helper_split_by_parenthesis( const char * src, char ** module, char ** param )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( src != NULL && module != NULL && param!= NULL )
- {
- size_t src_size = string_size( src );
- *module = NULL;
- *param = NULL;
- if ( src_size < 1 )
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcEmpty );
- else
- {
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- *module = malloc( src_size + 1 );
- if ( *module != NULL )
- {
- *param = malloc( src_size + 1 );
- if ( *param == NULL )
- {
- free( *module );
- *module = NULL;
- }
- else
- {
- size_t src_idx, dst_idx;
- bool b_module = true;
- dst_idx = 0;
- for ( src_idx = 0; src_idx < src_size; ++src_idx )
- {
- char c = src[ src_idx ];
- if ( b_module )
- {
- if ( c == '(' )
- {
- (*module)[ dst_idx ] = 0;
- dst_idx = 0;
- b_module = false;
- }
- else
- (*module)[ dst_idx++ ] = c;
- }
- else
- {
- if ( c != ')' )
- (*param)[ dst_idx++ ] = c;
- }
- }
- if ( b_module )
- {
- (*module)[ dst_idx ] = 0;
- free( *param );
- *param = NULL;
- }
- else
- (*param)[ dst_idx ] = 0;
-
- rc = 0;
- }
- }
- }
- }
- return rc;
-}
-
-
-static rc_t read_value( void * dst, const VCursor * cur,
- const uint32_t cur_idx, const uint32_t dst_bits )
-{
- uint32_t elem_bits, boff, elem_count;
- const void * base;
- rc_t rc = VCursorCellData ( cur, cur_idx, &elem_bits,
- &base, &boff, &elem_count );
- if ( rc == 0 )
- {
- if ( elem_bits > dst_bits )
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- else
- {
- if ( boff == 0 )
- memmove( dst, base, elem_bits >> 3 );
- else
- bitcpy ( dst, 0, base, boff, elem_bits );
- }
- }
- return rc;
-}
-
-
-rc_t helper_read_uint64( const VCursor * cur, const uint32_t cur_idx,
- uint64_t *value )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( cur != NULL && value != NULL )
- {
- *value = 0;
- rc = read_value( value, cur, cur_idx, 64 );
- }
- return rc;
-}
-
-
-rc_t helper_read_uint32( const VCursor * cur, const uint32_t cur_idx,
- uint32_t *value )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( cur != NULL && value != NULL )
- {
- *value = 0;
- rc = read_value( value, cur, cur_idx, 32 );
- }
- return rc;
-}
-
-
-static rc_t read_int32_values( p_int_buffer buf, const void * base,
- uint32_t boff, uint32_t elem_count )
-{
- rc_t rc = 0;
- if ( boff == 0 )
- {
- buf->trans_ptr = base;
- buf->len = elem_count;
- }
- else
- {
- if ( buf->size < elem_count )
- rc = int_buffer_realloc( buf, elem_count );
- if ( rc == 0 )
- {
- bitcpy ( buf->ptr, 0, base, boff, elem_count << 2 );
- buf->trans_ptr = buf->ptr;
- buf->len = elem_count;
- }
- }
- return rc;
-}
-
-
-static rc_t read_int8_values( p_int_buffer buf, const void * base,
- uint32_t boff, uint32_t elem_count )
-{
- rc_t rc = 0;
-
- if ( buf->size < elem_count )
- rc = int_buffer_realloc( buf, elem_count );
- if ( rc == 0 )
- {
- if ( boff == 0 )
- {
- const uint8_t * src = base;
- uint32_t idx;
- for ( idx = 0; idx < elem_count; idx++ )
- buf->ptr[ idx ] = src[ idx ];
- }
- else
- {
- uint8_t * src = malloc( elem_count );
- if ( src != NULL )
- {
- uint32_t idx;
- bitcpy ( src, 0, base, boff, elem_count );
- for ( idx = 0; idx < elem_count; idx++ )
- buf->ptr[ idx ] = src[ idx ];
- free( src );
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- }
- if ( rc == 0 )
- {
- buf->trans_ptr = buf->ptr;
- buf->len = elem_count;
- }
- return rc;
-}
-
-
-rc_t helper_read_int32_values( const VCursor * cur, const uint32_t cur_idx,
- p_int_buffer buf )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( cur != NULL && buf != NULL )
- {
- uint32_t elem_bits, boff, elem_count;
- const void * base;
- rc = VCursorCellData ( cur, cur_idx, &elem_bits,
- &base, &boff, &elem_count );
- if ( rc == 0 )
- {
- switch( elem_bits )
- {
- case 32 : rc = read_int32_values( buf, base, boff, elem_count ); break;
- case 8 : rc = read_int8_values( buf, base, boff, elem_count ); break;
- }
- }
- else
- buf->len = 0;
- }
- return rc;
-}
-
-
-rc_t helper_read_char_values( const VCursor * cur, const uint32_t cur_idx,
- p_char_buffer buf )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( cur != NULL && buf != NULL )
- {
- uint32_t elem_bits, boff, elem_count;
- const void * base;
- rc = VCursorCellData ( cur, cur_idx, &elem_bits,
- &base, &boff, &elem_count );
- if ( rc == 0 )
- {
- if ( boff == 0 )
- {
- buf->trans_ptr = base;
- buf->len = elem_count;
- }
- else
- {
- if ( buf->size < elem_count )
- rc = char_buffer_realloc( buf, elem_count );
- if ( rc == 0 )
- {
- bitcpy ( buf->ptr, 0, base, boff, elem_count * elem_bits );
- buf->trans_ptr = buf->ptr;
- buf->len = elem_count;
- }
- }
- }
- else
- buf->len = 0;
- }
- return rc;
-}
diff --git a/tools/run-stat/helper.h b/tools/run-stat/helper.h
deleted file mode 100644
index ce01552..0000000
--- a/tools/run-stat/helper.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_helper_
-#define _h_helper_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <klib/namelist.h>
-#include <klib/rc.h>
-#include <klib/log.h>
-#include <klib/text.h>
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/database.h>
-#include <vdb/cursor.h>
-#include <sra/sraschema.h>
-#include "definitions.h"
-#include "context.h"
-
-
-typedef struct char_buffer
-{
- char *ptr;
- const char *trans_ptr;
- size_t len; /* how many bytes are used... */
- size_t size; /* how much is allocated... */
-} char_buffer;
-typedef char_buffer* p_char_buffer;
-
-rc_t char_buffer_init( p_char_buffer buffer, const size_t size );
-void char_buffer_destroy( p_char_buffer buffer );
-rc_t char_buffer_realloc( p_char_buffer buffer, const size_t by );
-rc_t char_buffer_append_cstring( p_char_buffer buffer, const char * s );
-rc_t char_buffer_printfv( p_char_buffer buffer, const size_t estimated_len,
- const char * fmt, va_list args );
-rc_t char_buffer_printf( p_char_buffer buffer, const size_t estimated_len,
- const char * fmt, ... );
-rc_t char_buffer_saveto( p_char_buffer buffer, const char * filename );
-
-
-typedef struct int_buffer
-{
- uint32_t *ptr;
- const uint32_t *trans_ptr;
- size_t len; /* how many ints are used... */
- size_t size; /* how much is allocated... */
-} int_buffer;
-typedef int_buffer* p_int_buffer;
-
-rc_t int_buffer_init( p_int_buffer buffer, const size_t size );
-void int_buffer_destroy( p_int_buffer buffer );
-rc_t int_buffer_realloc( p_int_buffer buffer, const size_t by );
-
-int helper_str_cmp( const char *a, const char *b );
-
-/*
- * calls the given manager to create a new SRA-schema
- * takes the list of user-supplied schema's (which can be empty)
- * and let the created schema parse all of them
-*/
-rc_t helper_parse_schema( const VDBManager *my_manager,
- VSchema **new_schema,
- const KNamelist *schema_list );
-
-
-rc_t helper_make_namelist_from_string( const KNamelist **list,
- const char * src,
- const char split_char );
-
-bool helper_take_this_table_from_db( const VDatabase * db,
- p_stat_ctx ctx,
- const char * table_to_find );
-
-bool helper_take_1st_table_from_db( const VDatabase * db,
- p_stat_ctx ctx );
-
-char * helper_concat( const char * s1, const char * s2 );
-
-double percent( const uint64_t value, const uint64_t sum );
-
-rc_t helper_split_by_parenthesis( const char * src, char ** module, char ** param );
-
-rc_t helper_read_uint32( const VCursor * cur, const uint32_t cur_idx,
- uint32_t *value );
-
-rc_t helper_read_uint64( const VCursor * cur, const uint32_t cur_idx,
- uint64_t *value );
-
-rc_t helper_read_int32_values( const VCursor * cur, const uint32_t cur_idx,
- p_int_buffer buf );
-
-rc_t helper_read_char_values( const VCursor * cur, const uint32_t cur_idx,
- p_char_buffer buf );
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/run-stat/key_value.c b/tools/run-stat/key_value.c
deleted file mode 100644
index 5126350..0000000
--- a/tools/run-stat/key_value.c
+++ /dev/null
@@ -1,549 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/out.h>
-#include <klib/text.h>
-#include <klib/printf.h>
-#include <klib/namelist.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "key_value.h"
-
-
-rc_t report_init( p_report * self, uint32_t prealloc )
-{
- rc_t rc = 0;
- if ( self != NULL )
- {
- *self = calloc( 1, sizeof **self );
- if ( *self != NULL )
- VectorInit ( &( (*self)->data ), 0, prealloc );
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- return rc;
-}
-
-
-static void CC report_data_whack( void *item, void *data )
-{
- const VNamelist * values = item;
- if ( values != NULL )
- VNamelistRelease ( values );
-}
-
-
-rc_t report_clear( p_report self )
-{
- rc_t rc = 0;
- if ( self != NULL )
- {
- VectorWhack ( &(self->data), report_data_whack, NULL );
- VNamelistRelease ( self->columns ); /* ignores NULL */
- if ( self->max_width != NULL )
- {
- free( self->max_width );
- self->max_width = NULL;
- }
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- return rc;
-}
-
-
-rc_t report_destroy( p_report self )
-{
- rc_t rc;
- if ( self != NULL )
- {
- rc = report_clear( self );
- free( self );
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- return rc;
-}
-
-
-rc_t report_set_columns( p_report self, size_t count, ... )
-{
- rc_t rc;
- if ( self != NULL )
- {
- VNamelistRelease ( self->columns ); /* ignores NULL */
- rc = VNamelistMake ( &self->columns, count );
- if ( rc == 0 )
- {
- self->col_count = count;
- if ( self->max_width != NULL )
- free( self->max_width );
- self->max_width = malloc( count * sizeof( *self->max_width ) );
- if ( self->max_width == NULL )
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
-
- if ( rc == 0 )
- {
- size_t idx;
- va_list args;
-
- va_start ( args, count );
- for ( idx = 0; idx < count && rc == 0; ++idx )
- {
- const char * s = va_arg( args, const char * );
- if ( s != NULL )
- {
- rc = VNamelistAppend ( self->columns, s );
- self->max_width[ idx ] = string_size( s );
- }
- else
- {
- rc = VNamelistAppend ( self->columns, "." );
- self->max_width[ idx ] = 1;
- }
- }
- va_end ( args );
- }
- }
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- return rc;
-}
-
-
-rc_t report_new_data( p_report self )
-{
- rc_t rc;
- if ( self != NULL )
- {
- if ( self->columns != NULL )
- {
- uint32_t count;
- rc = VNameListCount ( self->columns, &count );
- if ( rc == 0 && count > 0 )
- {
- VNamelist * new_row;
- rc = VNamelistMake ( &new_row, count );
- if ( rc == 0 )
- rc = VectorAppend ( &self->data, NULL, new_row );
- }
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- return rc;
-}
-
-
-rc_t report_add_data( p_report self, const size_t estimated_len, const char * fmt, ... )
-{
- rc_t rc = 0;
- if ( self != NULL )
- {
- uint32_t n_data = VectorLength( &self->data );
- if ( n_data == 0 )
- {
- rc = report_new_data( self );
- n_data++;
- }
- if ( rc == 0 )
- {
- VNamelist * last_row = VectorGet ( &self->data, n_data - 1 );
- if ( last_row != NULL )
- {
- uint32_t n_col;
-
- VNameListCount ( last_row, &n_col );
- if ( n_col < self->col_count )
- {
- va_list args;
- size_t len = estimated_len;
- bool done = false;
- char * s;
-
- va_start ( args, fmt );
- while( !done )
- {
- s = malloc( len );
- done = ( s == NULL );
- if ( !done )
- {
- size_t written;
- rc = string_vprintf ( s, estimated_len, &written, fmt, args );
- done = ( rc == 0 );
- if ( done )
- {
- rc = VNamelistAppend ( last_row, s );
- if ( self->max_width[ n_col ] < written )
- self->max_width[ n_col ] = written;
- }
- else
- {
- free( s );
- len += len;
- }
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- va_end ( args );
- }
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- }
- }
- else
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- return rc;
-}
-
-
-static rc_t report_print_column_names( p_report self, p_char_buffer dst,
- const uint32_t n_col, const char delim )
-{
- rc_t rc = 0;
- uint32_t col_idx;
- /* print the column-names */
- for ( col_idx = 0; col_idx < n_col && rc == 0; ++col_idx )
- {
- const char * s;
- rc = VNameListGet ( self->columns, col_idx, &s );
- if ( rc == 0 )
- {
- size_t len = string_size( s );
- if ( col_idx < n_col - 1 )
- rc = char_buffer_printf( dst, len + 5, "%s%c", s, delim );
- else
- rc = char_buffer_printf( dst, len + 5, "%s\n", s );
- }
- }
- return rc;
-}
-
-static rc_t report_print_tabed_column_names( p_report self, p_char_buffer dst,
- const uint32_t n_col, const char delim )
-{
- rc_t rc = 0;
- uint32_t col_idx;
- /* print the column-names */
- for ( col_idx = 0; col_idx < n_col && rc == 0; ++col_idx )
- {
- const char * s;
- rc = VNameListGet ( self->columns, col_idx, &s );
- if ( rc == 0 )
- {
- uint32_t w = self->max_width[ col_idx ];
- if ( col_idx < n_col - 1 )
- rc = char_buffer_printf( dst, w + 5, "%*s%c", w, s, delim );
- else
- rc = char_buffer_printf( dst, w + 5, "%*s\n", w, s );
- }
- }
- return rc;
-}
-
-
-static rc_t report_print_data_rows( p_report self, p_char_buffer dst,
- const uint32_t n_data, const char delim )
-{
- rc_t rc = 0;
- uint32_t d_idx;
- for ( d_idx = 0; d_idx < n_data && rc == 0; ++d_idx )
- {
- VNamelist * row = VectorGet ( &self->data, d_idx );
- if ( row != NULL )
- {
- uint32_t nc_data;
- rc = VNameListCount ( row, &nc_data );
- if ( rc == 0 )
- {
- uint32_t col_idx;
- for ( col_idx = 0; col_idx < nc_data && rc == 0; ++col_idx )
- {
- const char * s;
- rc = VNameListGet ( row, col_idx, &s );
- if ( rc == 0 )
- {
- size_t len = string_size( s );
- if ( col_idx < nc_data - 1 )
- rc = char_buffer_printf( dst, len + 5, "%s%c", s, delim );
- else
- rc = char_buffer_printf( dst, len + 5, "%s\n", s );
- }
- }
- }
- }
- }
- return rc;
-}
-
-
-static rc_t report_print_tabed_data_rows( p_report self, p_char_buffer dst,
- const uint32_t n_data, const char delim )
-{
- rc_t rc = 0;
- uint32_t d_idx;
- for ( d_idx = 0; d_idx < n_data && rc == 0; ++d_idx )
- {
- VNamelist * row = VectorGet ( &self->data, d_idx );
- if ( row != NULL )
- {
- uint32_t nc_data;
- rc = VNameListCount ( row, &nc_data );
- if ( rc == 0 )
- {
- uint32_t col_idx;
- for ( col_idx = 0; col_idx < nc_data && rc == 0; ++col_idx )
- {
- const char * s;
- rc = VNameListGet ( row, col_idx, &s );
- if ( rc == 0 )
- {
- uint32_t w = self->max_width[ col_idx ];
- if ( col_idx < nc_data - 1 )
- rc = char_buffer_printf( dst, w + 5, "%*s%c", w, s, delim );
- else
- rc = char_buffer_printf( dst, w + 5, "%*s\n", w, s );
- }
- }
- }
- }
- }
- return rc;
-}
-
-
-static rc_t report_print_column_names_row0( p_report self, p_char_buffer dst,
- const uint32_t n_col )
-{
- rc_t rc = 0;
- uint32_t i, w = 0;
- VNamelist * row;
-
- for ( i = 0; i < n_col && rc == 0; ++i )
- {
- const char * s;
- rc = VNameListGet ( self->columns, i, &s );
- if ( rc == 0 )
- {
- size_t len = string_size( s );
- if ( len > w ) w = len;
- }
- }
- row = VectorGet ( &self->data, 0 );
- if ( row != NULL )
- {
- for ( i = 0; i < n_col && rc == 0; ++i )
- {
- const char * s_col;
- rc = VNameListGet ( self->columns, i, &s_col );
- if ( rc == 0 )
- {
- const char * s_value;
- rc = VNameListGet ( row, i, &s_value );
- if ( rc == 0 )
- {
- size_t len = string_size( s_value ) + w + 5;
- rc = char_buffer_printf( dst, len, "%*s : %s\n", w, s_col, s_value );
- }
- }
- }
- }
- return rc;
-}
-
-static rc_t report_print_txt( p_report self, p_char_buffer dst,
- const uint32_t n_col, const uint32_t n_data )
-{
- rc_t rc;
- uint32_t rows = VectorLength( &self->data );
- switch( rows )
- {
- case 0 : rc = report_print_tabed_column_names( self, dst, n_col, ' ' );
- break;
- case 1 : rc = report_print_column_names_row0( self, dst, n_col );
- break;
- default : rc = report_print_tabed_column_names( self, dst, n_col, ' ' );
- if ( rc == 0 )
- rc = report_print_tabed_data_rows( self, dst, n_data, ' ' );
- }
- return rc;
-}
-
-
-static rc_t report_print_csv( p_report self, p_char_buffer dst,
- const uint32_t n_col, const uint32_t n_data )
-{
- rc_t rc = report_print_column_names( self, dst, n_col, ',' );
- if ( rc == 0 )
- rc = report_print_data_rows( self, dst, n_data, ',' );
- return rc;
-}
-
-
-
-static rc_t report_print_xml( p_report self, p_char_buffer dst,
- const uint32_t n_col, const uint32_t n_data )
-{
- rc_t rc = 0;
- uint32_t d_idx;
- for ( d_idx = 0; d_idx < n_data && rc == 0; ++d_idx )
- {
- VNamelist * row = VectorGet ( &self->data, d_idx );
- if ( row != NULL )
- {
- /* open the row-xml node */
- rc = char_buffer_printf( dst, 20, "<row_%u>\n", d_idx );
- if ( rc == 0 )
- {
- uint32_t nc_data;
- rc = VNameListCount ( row, &nc_data );
- if ( rc == 0 )
- {
- uint32_t col_idx;
- for ( col_idx = 0; col_idx < nc_data && rc == 0; ++col_idx )
- {
- const char * s_col;
- rc = VNameListGet ( self->columns, col_idx, &s_col );
- if ( rc == 0 )
- {
- const char * s;
- size_t col_len = string_size( s_col );
- rc = VNameListGet ( row, col_idx, &s );
- if ( rc == 0 )
- {
- size_t len = string_size( s );
- /* print the value */
- rc = char_buffer_printf( dst, col_len * 2 + len + 20,
- " <%s>%s</%s>\n", s_col, s, s_col );
- }
- }
- }
- }
- /* close the row-xml node */
- rc = char_buffer_printf( dst, 20, "</row_%u>\n\n", d_idx );
- }
- }
- }
- return rc;
-}
-
-
-static rc_t report_print_jso( p_report self, p_char_buffer dst,
- const uint32_t n_col, const uint32_t n_data )
-{
- rc_t rc = 0;
- uint32_t d_idx;
- for ( d_idx = 0; d_idx < n_data && rc == 0; ++d_idx )
- {
- VNamelist * row = VectorGet ( &self->data, d_idx );
- if ( row != NULL )
- {
- /* open the json node */
- rc = char_buffer_printf( dst, 20, "{\n", d_idx );
- if ( rc == 0 )
- {
- uint32_t nc_data;
- rc = VNameListCount ( row, &nc_data );
- if ( rc == 0 )
- {
- uint32_t col_idx;
- for ( col_idx = 0; col_idx < nc_data && rc == 0; ++col_idx )
- {
- const char * s_col;
- rc = VNameListGet ( self->columns, col_idx, &s_col );
- if ( rc == 0 )
- {
- const char * s;
- size_t col_len = string_size( s_col );
- rc = VNameListGet ( row, col_idx, &s );
- if ( rc == 0 )
- {
- size_t len = string_size( s );
- /* print the value */
- if ( col_idx < nc_data -1 )
- rc = char_buffer_printf( dst, col_len + len + 10,
- " \"%s\":%s,\n", s_col, s );
- else
- rc = char_buffer_printf( dst, col_len + len + 10,
- " \"%s\":%s\n", s_col, s );
- }
- }
- }
- }
- /* close the json-node */
- if ( d_idx < n_data - 1 )
- rc = char_buffer_printf( dst, 20, "},\n\n", d_idx );
- else
- rc = char_buffer_printf( dst, 20, "}\n", d_idx );
- }
- }
- }
- return rc;
-}
-
-
-rc_t report_print( p_report self, p_char_buffer dst, uint32_t mode )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcSelf, rcNull );
- if ( self != NULL )
- {
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( dst != NULL )
- {
- uint32_t n_col;
- rc = VNameListCount ( self->columns, &n_col );
- if ( rc == 0 )
- {
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- if ( n_col > 0 )
- {
- uint32_t n_data = VectorLength( &self->data );
- if ( n_data >= 0 )
- {
- switch( mode )
- {
- case RT_TXT : rc = report_print_txt( self, dst, n_col, n_data ); break;
- case RT_CSV : rc = report_print_csv( self, dst, n_col, n_data ); break;
- case RT_XML : rc = report_print_xml( self, dst, n_col, n_data ); break;
- case RT_JSO : rc = report_print_jso( self, dst, n_col, n_data ); break;
- default : rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- }
- }
- }
- }
- }
- }
- return rc;
-}
\ No newline at end of file
diff --git a/tools/run-stat/mod_cmn.c b/tools/run-stat/mod_cmn.c
deleted file mode 100644
index 1fece24..0000000
--- a/tools/run-stat/mod_cmn.c
+++ /dev/null
@@ -1,329 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/out.h>
-#include <klib/text.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-
-#include "mod_cmn.h"
-#include "mod_reads.h"
-
-
-static rc_t module_init( p_module * new_module, const char * name )
-{
- rc_t rc = rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( new_module != NULL && name != NULL )
- {
- rc = RC( rcExe, rcData, rcAllocating, rcMemory, rcExhausted );
- (*new_module) = calloc( 1, sizeof( module ) );
- if ( *new_module != NULL )
- {
- char * s_module;
- char * param;
- /* we have to split the name into mod-name and parameters */
- rc = helper_split_by_parenthesis( name, &s_module, ¶m );
- if ( rc == 0 )
- {
- (*new_module)->name = string_dup_measure ( s_module, NULL );
- /* here we hardcode the build-in modules */
- if ( helper_str_cmp( s_module, "bases" ) == 0 )
- {
- rc = mod_reads_init( *new_module, param );
- }
- free( s_module );
- if ( param != NULL )
- free( param );
- }
- }
- }
- return rc;
-}
-
-
-static void destroy_module( p_module a_mod )
-{
- if ( a_mod != NULL )
- {
- /* free the name... */
- if ( a_mod->name != NULL )
- free( a_mod->name );
- /* call the function to free the context-pointer */
- if ( a_mod->f_free != NULL )
- a_mod->f_free( a_mod );
- free( a_mod );
- }
-}
-
-
-static void CC destroy_module_cb( void* node, void* data )
-{
- destroy_module( ( p_module )node );
-}
-
-
-static rc_t modules_init_loop( p_mod_list self, const KNamelist * names )
-{
- uint32_t count;
- rc_t rc = KNamelistCount ( names, &count );
- if ( rc == 0 )
- {
- uint32_t i;
- for ( i = 0; i < count && rc == 0; ++i )
- {
- const char * s;
- rc = KNamelistGet ( names, i, &s );
- if ( rc == 0 )
- {
- p_module a_module;
- rc = module_init( &a_module, s );
- if ( rc == 0 )
- {
- rc = VectorAppend( &(self->list), NULL, a_module );
- if ( rc != 0 )
- destroy_module( a_module );
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t mod_list_init( p_mod_list * self, const KNamelist * names )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL )
- {
- rc = RC( rcExe, rcData, rcAllocating, rcMemory, rcExhausted );
- (*self) = calloc( 1, sizeof( mod_list ) );
- if ( *self != NULL )
- {
- VectorInit( &( (*self)->list ), 0, 5 );
- rc = modules_init_loop( *self, names );
- if ( rc != 0 )
- {
- VectorWhack( &( (*self)->list ), destroy_module_cb, NULL );
- free( *self );
- *self = NULL;
- }
- }
- }
- return rc;
-}
-
-
-rc_t mod_list_destroy( p_mod_list list )
-{
- VectorWhack( &(list->list), destroy_module_cb, NULL );
- free( list );
- return 0;
-}
-
-
-rc_t mod_list_pre_open( p_mod_list self,
- const VCursor * cur )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && cur != NULL )
- {
- uint32_t len = VectorLength( &(self->list) );
- rc = 0;
- if ( len > 0 )
- {
- uint32_t m_idx;
- for ( m_idx = 0; m_idx < len && rc == 0; ++m_idx )
- {
- p_module a_module = VectorGet ( &(self->list), m_idx );
- if ( a_module != NULL && a_module->f_pre_open != NULL )
- {
- rc = a_module->f_pre_open( a_module, cur );
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t mod_list_post_rows( p_mod_list self )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL )
- {
- uint32_t len = VectorLength( &(self->list) );
- rc = 0;
- if ( len > 0 )
- {
- uint32_t m_idx;
- for ( m_idx = 0; m_idx < len && rc == 0; ++m_idx )
- {
- p_module a_module = VectorGet ( &(self->list), m_idx );
- if ( a_module != NULL && a_module->f_post_rows != NULL )
- {
- rc = a_module->f_post_rows( a_module );
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t mod_list_count( p_mod_list self, uint32_t * count )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && count != NULL )
- {
- *count = VectorLength( &(self->list) );
- rc = 0;
- }
- return rc;
-}
-
-
-static p_module mod_list_get( p_mod_list self, const uint32_t idx )
-{
- p_module res = NULL;
- if ( self != NULL )
- {
- if ( idx < VectorLength( &(self->list) ) )
- res = VectorGet ( &(self->list), idx );
- }
- return res;
-}
-
-
-rc_t mod_list_name( p_mod_list self, const uint32_t idx,
- char ** name )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && name != NULL )
- {
- p_module a_module = mod_list_get( self, idx );
- if ( a_module == NULL )
- rc = RC( rcExe, rcData, rcAllocating, rcParam, rcInvalid );
- else
- {
- (*name) = string_dup_measure ( a_module->name, NULL );
- rc = 0;
- }
- }
- return rc;
-}
-
-
-rc_t mod_list_subreport_count( p_mod_list self, const uint32_t idx,
- uint32_t * count )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && count != NULL )
- {
- p_module a_module = mod_list_get( self, idx );
- rc = RC( rcExe, rcData, rcAllocating, rcParam, rcInvalid );
- if ( a_module != NULL && a_module->f_count != NULL )
- rc = a_module->f_count( a_module, count );
- }
- return rc;
-}
-
-
-rc_t mod_list_subreport_name( p_mod_list self, const uint32_t m_idx,
- const uint32_t r_idx, char ** name )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && name != NULL )
- {
- p_module a_module = mod_list_get( self, m_idx );
- rc = RC( rcExe, rcData, rcAllocating, rcParam, rcInvalid );
- if ( a_module != NULL && a_module->f_name != NULL )
- rc = a_module->f_name( a_module, r_idx, name );
- }
- return rc;
-}
-
-
-rc_t mod_list_subreport( p_mod_list self,
- const uint32_t m_idx,
- const uint32_t r_idx,
- p_char_buffer dst,
- const uint32_t mode )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && dst != NULL )
- {
- p_module a_module = mod_list_get( self, m_idx );
- rc = RC( rcExe, rcData, rcAllocating, rcParam, rcInvalid );
- if ( a_module != NULL && a_module->f_report != NULL )
- {
- p_report report;
- rc = report_init( &report, 50 );
- if ( rc == 0 )
- {
- rc = a_module->f_report( a_module, report, r_idx );
- if ( rc == 0 )
- rc = report_print( report, dst, mode );
- report_destroy( report );
- }
- }
- }
- return rc;
-}
-
-
-rc_t mod_list_graph( p_mod_list self, const uint32_t m_idx,
- const uint32_t r_idx, const char * filename )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && filename != NULL )
- {
- p_module a_module = mod_list_get( self, m_idx );
- rc = RC( rcExe, rcData, rcAllocating, rcParam, rcInvalid );
- if ( a_module != NULL && a_module->f_graph != NULL )
- rc = a_module->f_graph( a_module, r_idx, filename );
- }
- return rc;
-}
-
-
-rc_t mod_list_row( p_mod_list self, const VCursor * cur )
-{
- rc_t rc = 0;
- uint32_t len = VectorLength( &(self->list) );
- if ( len > 0 )
- {
- uint32_t m_idx;
- for ( m_idx = 0; m_idx < len && rc == 0; ++m_idx )
- {
- p_module a_module = VectorGet ( &(self->list), m_idx );
- if ( a_module != NULL && a_module->f_row != NULL )
- {
- rc = a_module->f_row( a_module, cur );
- }
- }
- }
- return rc;
-}
diff --git a/tools/run-stat/mod_cmn.h b/tools/run-stat/mod_cmn.h
deleted file mode 100644
index 9d8559c..0000000
--- a/tools/run-stat/mod_cmn.h
+++ /dev/null
@@ -1,149 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_stat_mod_cmn_
-#define _h_stat_mod_cmn_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include <klib/rc.h>
-#include <klib/vector.h>
-#include <klib/namelist.h>
-#include <vdb/cursor.h>
-#include "helper.h"
-#include "key_value.h"
-
-struct module;
-
-/* the type of the callback-function to called before the cursor is open */
-typedef rc_t ( CC * mod_pre_open ) ( struct module * self,
- const VCursor * cur );
-
-/* the type of the callback-function to be performed on every row */
-typedef rc_t ( CC * mod_row ) ( struct module * self,
- const VCursor * cur );
-
-/* the type of the callback-function to be performed on after all rows are read */
-typedef rc_t ( CC * mod_post_rows ) ( struct module * self );
-
-
-/* the type of the callback-function to query how many sub-reports
- a statistic-module can produce */
-typedef rc_t ( CC * mod_query_report_count ) ( struct module * self,
- uint32_t * count );
-
-/* the type of the callback-function to query the name of one
- of the sub-reports */
-typedef rc_t ( CC * mod_query_report_name ) ( struct module * self,
- uint32_t n, char ** name );
-
-/* the type of the callback-function to produce a sub-report */
-typedef rc_t ( CC * mod_report ) ( struct module * self,
- p_report report,
- const uint32_t sub_select );
-
-/* the type of the callback-function to produce a grafic-file */
-typedef rc_t ( CC * mod_graph ) ( struct module * self,
- uint32_t n, const char * filename );
-
-
-/* the type of the callback-function to be performed free the module-context */
-typedef rc_t ( CC * mod_free ) ( struct module * self );
-
-
-/********************************************************************
-this structure represents a statistic-module
-********************************************************************/
-typedef struct module
-{
- char *name; /* every module puts its name in here */
- void * mod_ctx; /* every module puts its module-context here */
- bool active; /* if the module can be used (all necessary columns found)*/
-
- mod_pre_open f_pre_open; /* request the uses columns etc. */
-
- mod_row f_row; /* called for every row */
-
- mod_post_rows f_post_rows; /* called after all rows are read */
-
- mod_query_report_count f_count;
- /* how many reports does the module produce */
-
- mod_query_report_name f_name;
- /* name of a report */
-
- mod_report f_report; /* create a sub-report */
-
- mod_graph f_graph; /* create a grafic for a sub-report */
-
- mod_free f_free; /* called to free the context */
-} module;
-typedef module* p_module;
-
-
-typedef struct mod_list
-{
- Vector list; /* the list of modules... */
-} mod_list;
-typedef mod_list* p_mod_list;
-
-
-rc_t mod_list_init( p_mod_list * self, const KNamelist * names );
-
-rc_t mod_list_destroy( p_mod_list self );
-
-rc_t mod_list_pre_open( p_mod_list self, const VCursor * cur );
-
-rc_t mod_list_row( p_mod_list self, const VCursor * cur );
-
-rc_t mod_list_post_rows( p_mod_list self );
-
-rc_t mod_list_count( p_mod_list self, uint32_t * count );
-
-rc_t mod_list_name( p_mod_list self, const uint32_t idx,
- char ** name );
-
-rc_t mod_list_subreport_count( p_mod_list self, const uint32_t idx,
- uint32_t * count );
-
-rc_t mod_list_subreport_name( p_mod_list self, const uint32_t m_idx,
- const uint32_t r_idx, char ** name );
-
-rc_t mod_list_subreport( p_mod_list self,
- const uint32_t m_idx,
- const uint32_t r_idx,
- p_char_buffer dst,
- const uint32_t mode );
-
-rc_t mod_list_graph( p_mod_list self, const uint32_t m_idx,
- const uint32_t r_idx, const char * filename );
-
-#endif
diff --git a/tools/run-stat/mod_reads.c b/tools/run-stat/mod_reads.c
deleted file mode 100644
index ef1db56..0000000
--- a/tools/run-stat/mod_reads.c
+++ /dev/null
@@ -1,747 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <sysalloc.h>
-#include <stdlib.h>
-
-#include "key_value.h"
-#include "mod_cmn.h"
-#include "mod_reads.h"
-#include "mod_reads_defs.h"
-#include "mod_reads_helper.h"
-#include "helper.h"
-#include "chart.h"
-
-
-/* this is called for every single row in a table */
-static rc_t CC mod_reads_row ( p_module self,
- const VCursor * cur )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( self != NULL && cur != NULL )
- {
- if ( self->mod_ctx != NULL )
- {
- p_reads_data data = ( p_reads_data )self->mod_ctx;
- /* in here we read the data (bases/quality and read-start/len)... */
- rc = rd_filter_read( &data->filter, cur );
- if ( rc == 0 )
- {
- uint32_t base_pos;
- uint32_t read_len = filter_get_read_len( &data->filter );
- uint16_t kmer = 0;
-
- /* count the number of READS/SEQUENCES */
- data->sum_reads++;
-
- /* count the bases total and per sequence-position */
- for ( base_pos = 0; base_pos < read_len; ++base_pos )
- {
- char base = filter_get_base( &data->filter, base_pos );
- uint8_t qual = filter_get_quality( &data->filter, base_pos );
- uint8_t compressed_base_pos = compress_base_pos( base_pos );
-
- /* clip the quality to fit between MIN(2) and MAX(40) */
- if ( qual < MIN_QUALITY ) qual = MIN_QUALITY;
- if ( qual > MAX_QUALITY ) qual = MAX_QUALITY;
-
- /* count the base-postion-related base-data */
- count_base( &data->bp_data[ compressed_base_pos ], base, qual );
-
- /* count the total base-data */
- count_base( &data->total, base, qual );
-
- /* handle the kmer's */
- kmer = count_kmer( data, kmer, compressed_base_pos, base );
-
- /* detect min. and max. quality */
- if ( qual < data->min_quality )
- data->min_quality = qual;
- if ( qual > data->max_quality )
- data->max_quality = qual;
- }
-
- /* detect min. and max. read-len */
- if ( read_len < data->min_read_len )
- data->min_read_len = read_len;
- if ( read_len > data->max_read_len )
- data->max_read_len = read_len;
-
- /* count how often a particular read-len occurs */
- data->bp_data[ compress_base_pos( read_len ) ].read_count++;
- }
- }
- }
- return rc;
-}
-
-
-/* this is called once, after all rows are handled */
-static rc_t CC mod_reads_post_rows ( p_module self )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( self != NULL && self->mod_ctx != NULL )
- {
- p_reads_data data = self->mod_ctx;
- p_base_pos bp;
- uint32_t base_pos;
-
- /* first we calculate for every base-position:
- for quality: mean/median/quarat/centile
- for bases : percentage */
- for ( base_pos = 0; base_pos < MAX_COMPRESSED_BASE_POS; ++base_pos )
- {
- bp = &data->bp_data[ base_pos ];
- calculate_quality_mean_median_quart_centile( bp );
- calculate_bases_percentage( bp );
- }
-
- /* then we calculate for the sum:
- for quality: mean/median/quarat/centile
- for bases : percentage and probability
- for kmers : observed vs. expected occurance */
- bp = &data->total;
- calculate_quality_mean_median_quart_centile( bp );
- calculate_bases_percentage( bp );
- calculate_base_probability( data );
- calculate_kmer_observed_vs_expected( data );
-
- data->mean_read_len = bp->count;
- data->mean_read_len /= data->sum_reads;
-
- rc = 0;
- }
- return rc;
-}
-
-
-static rc_t print_totals( p_reads_data data, p_report report )
-{
- uint8_t i;
- p_base_pos bp = &( data->total );
-
- rc_t rc = report_set_columns( report, 21,
- "bases", "reads", "mean_readlen", "min_readlen", "max_readlen",
- "GC", "GC_percent", "A", "A_percent", "C", "C_percent",
- "G", "G_percent", "T", "T_percent", "N", "N_percent",
- "mean_quality", "median_quality", "min_quality", "max_quality" );
-
- if ( rc == 0 )
- rc = report_new_data( report );
-
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%lu", bp->count );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%lu", data->sum_reads );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", data->mean_read_len );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%lu", data->min_read_len );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%lu", data->max_read_len );
-
- if ( rc == 0 )
- rc = report_add_data( report, 40, "%lu",
- bp->base_sum[ IDX_G ] + bp->base_sum[ IDX_C ] );
- if ( rc == 0 )
- rc = report_add_data( report, 40, "%.2f",
- bp->base_percent[ IDX_G ] + bp->base_percent[ IDX_C ] );
-
- for ( i = IDX_A; i <= IDX_N && rc == 0 ; ++i )
- {
- rc = report_add_data( report, 20, "%lu", bp->base_sum[ i ] );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->base_percent[ i ] );
- }
-
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->mean );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->median );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", data->min_quality );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", data->max_quality );
-
- return rc;
-}
-
-
-static rc_t print_pos_bases( p_reads_data data, p_report report )
-{
- rc_t rc = report_set_columns( report, 8, "from", "to", "A", "C", "G", "T", "N", "reads" );
- if ( rc == 0 )
- {
- uint8_t i, max_i = compress_base_pos( data->max_read_len );
- for ( i = 0; i <= max_i && rc == 0 ; ++i )
- {
- uint8_t j;
- p_base_pos bp = &data->bp_data[ i ];
- uint32_t to = bp->to;
- if ( to > MAX_BASE_POS ) to = MAX_BASE_POS;
-
- if ( rc == 0 )
- rc = report_new_data( report );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", bp->from );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", to );
-
- for ( j = IDX_A; j <= IDX_N && rc == 0; ++j )
- rc = report_add_data( report, 20, "%.1f", bp->base_percent[ j ] );
-
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", bp->read_count );
- }
- }
- return rc;
-}
-
-
-static void prepare_hruler( uint32_t * pp, uint32_t n, uint32_t spread )
-{
- uint8_t idx;
- for ( idx = 0; idx < n; ++idx )
- pp[ idx ] = compress_start_lookup( idx * spread );
-}
-
-
-static void prepare_pos_bases_line( p_reads_data data, uint32_t * pp,
- uint32_t select )
-{
- uint8_t i;
- for ( i = 0; i <= MAX_COMPRESSED_BASE_POS; ++i )
- {
- p_base_pos bp = &data->bp_data[ i ];
- double value;
- if ( select <= IDX_N )
- value = bp->base_percent[ select ];
- else
- value = bp->base_percent[ IDX_C ] + bp->base_percent[ IDX_G ];
- if ( value > 60 )
- value = 0;
- pp[ i ] = ( value * 10 );
- }
-}
-
-
-static rc_t graph_pos_bases( p_reads_data data, const char * filename )
-{
- chart c;
- rc_t rc = chart_init( &c, 100, 100, 600, 500 );
- if ( rc == 0 )
- {
- double factor = 0.7;
- uint32_t pp[ MAX_COMPRESSED_BASE_POS + 1 ];
- uint8_t max_i = compress_base_pos( data->max_read_len );
- uint32_t h_spread = 600 / max_i;
-
- chart_captions( &c, "bases per position", "base-pos", "%" );
- chart_vruler( &c, 0, 60, factor * 10, 10, "%" );
- prepare_hruler( pp, max_i / 2, 2 );
- chart_hruler( &c, pp, max_i / 2, 0, h_spread * 2 );
-
- prepare_pos_bases_line( data, pp, IDX_A );
- chart_line( &c, pp, max_i, factor, h_spread, "red", "A" );
-
- prepare_pos_bases_line( data, pp, IDX_C );
- chart_line( &c, pp, max_i, factor, h_spread, "green", "C" );
-
- prepare_pos_bases_line( data, pp, IDX_G );
- chart_line( &c, pp, max_i, factor, h_spread, "blue", "G" );
-
- prepare_pos_bases_line( data, pp, IDX_T );
- chart_line( &c, pp, max_i, factor, h_spread, "yellow", "T" );
-
- prepare_pos_bases_line( data, pp, IDX_N );
- chart_line( &c, pp, max_i, factor, h_spread, "orange", "N" );
-
- prepare_pos_bases_line( data, pp, 5 );
- chart_line( &c, pp, max_i, factor, h_spread, "pink", "GC" );
-
- rc = chart_write( &c, filename );
- chart_destroy( &c );
- }
- return rc;
-}
-
-
-static rc_t graph_pos_quality( p_reads_data data, const char * filename )
-{
- chart c;
- rc_t rc = chart_init( &c, 100, 100, 600, 500 );
- if ( rc == 0 )
- {
- char * box_style = create_fill_style( &c, "black", "yellow", 1 );
- if ( box_style != NULL )
- {
- char * median_style = create_line_style( &c, "red", 4 );
- if ( median_style != NULL )
- {
- uint32_t i;
- uint32_t pp[ MAX_COMPRESSED_BASE_POS + 1 ];
- uint8_t max_i = compress_base_pos( data->max_read_len );
- uint32_t h_spread = ( 600 - 10 ) / max_i;
-
- chart_captions( &c, "quality per position", "base-pos", "phred" );
- chart_vruler( &c, 0, MAX_QUALITY, 10.0, 5, NULL );
- prepare_hruler( pp, ( max_i / 2 ) + 1, 2 );
- chart_hruler( &c, pp, ( max_i / 2 ) + 1,
- 5 + ( h_spread / 2 ), h_spread * 2 );
- for ( i = 0; i <= max_i; ++i )
- {
- p_base_pos bp = &data->bp_data[ i ];
- chart_box_whisker( &c,
- 5 + ( i * h_spread ) + 2, /* x */
- h_spread - 2, /* dx */
- bp->lower_centile, /* p_low */
- bp->upper_centile, /* p_high */
- bp->lower_quart, /* q_low */
- bp->upper_quart, /* q_high */
- bp->median, /* median */
- 10.0, /* factor */
- &( box_style[1] ),
- &( median_style[1] ) );
- }
- rc = chart_write( &c, filename );
- free( median_style );
- }
- free( box_style );
- }
- chart_destroy( &c );
- }
- return rc;
-}
-
-
-static rc_t print_pos_quality( p_reads_data data, p_report report )
-{
- rc_t rc = report_set_columns( report, 10, "from", "to", "mean", "median",
- "mode", "q-up", "q-low", "c-up", "c-low", "reads" );
- if ( rc == 0 )
- {
- uint8_t i, max_i = compress_base_pos( data->max_read_len );
- for ( i = 0; i <= max_i && rc == 0 ; ++i )
- {
- p_base_pos bp = &data->bp_data[ i ];
- uint32_t to = bp->to;
- if ( to > MAX_BASE_POS ) to = MAX_BASE_POS;
-
- if ( rc == 0 )
- rc = report_new_data( report );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", bp->from );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", to );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->mean );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->median );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->mode );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->upper_quart );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->lower_quart );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->upper_centile );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.2f", bp->lower_centile );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", bp->count );
- }
- }
- return rc;
-}
-
-
-static uint64_t max_count_of_read_length( p_reads_data data )
-{
- uint32_t res = 0;
- uint8_t i;
- for ( i = 0; i <= MAX_COMPRESSED_BASE_POS; ++i )
- {
- p_base_pos bp = &data->bp_data[ i ];
- if ( bp->read_count > res )
- res = bp->read_count;
- }
- return res;
-}
-
-
-static rc_t graph_read_length( p_reads_data data, const char * filename )
-{
- chart c;
- rc_t rc = chart_init( &c, 100, 100, 600, 500 );
- if ( rc == 0 )
- {
- double factor;
- uint8_t i;
- uint32_t pp[ MAX_COMPRESSED_BASE_POS + 1 ];
- uint32_t h_spread = 12;
- uint64_t max_value = max_count_of_read_length( data );
-
- chart_captions( &c, "distribution of read-length", "read-length", "count" );
- factor = max_value;
- factor /= 450;
-
- /* max_y, step_y, max_value */
- chart_vruler1( &c, 450, 50, max_value, NULL );
- prepare_hruler( pp, ( MAX_COMPRESSED_BASE_POS + 1 ) / 2, 2 );
- chart_hruler( &c, pp, ( MAX_COMPRESSED_BASE_POS + 1 ) / 2,
- 0, h_spread * 2 );
-
- /* prepare the one and only line to draw */
- for ( i = 0; i <= MAX_COMPRESSED_BASE_POS; ++i )
- {
- p_base_pos bp = &data->bp_data[ i ];
- uint64_t value = bp->read_count;
- value *= 450;
- value /= max_value;
- pp[ i ] = (uint32_t) value;
- }
- chart_line( &c, pp, MAX_COMPRESSED_BASE_POS + 1, 1.0,
- h_spread, "blue", "read-length" );
-
- rc = chart_write( &c, filename );
- chart_destroy( &c );
- }
- return rc;
-}
-
-
-static rc_t print_read_length( p_reads_data data, p_report report )
-{
- uint8_t i;
- rc_t rc = report_set_columns( report, 3, "from", "to", "reads" );
- if ( rc == 0 )
- {
- for ( i = 0; i <= MAX_COMPRESSED_BASE_POS && rc == 0 ; ++i )
- {
- p_base_pos bp = &data->bp_data[ i ];
- uint32_t to = bp->to;
- if ( to > MAX_BASE_POS ) to = MAX_BASE_POS;
- if ( rc == 0 )
- rc = report_new_data( report );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", bp->from );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", to );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", bp->read_count );
- }
- }
- return rc;
-}
-
-
-static uint64_t max_count_of_quality( p_reads_data data )
-{
- uint32_t res = 0;
- uint8_t i;
- for ( i = 0; i <= 40; ++i )
- {
- if ( data->total.qual[ i ] > res )
- res = data->total.qual[ i ];
- }
- return res;
-}
-
-
-static rc_t graph_qual_distr( p_reads_data data, const char * filename )
-{
- chart c;
- rc_t rc = chart_init( &c, 100, 100, 600, 500 );
- if ( rc == 0 )
- {
- double factor;
- uint8_t i;
- uint32_t pp[ 42 ];
- uint32_t h_spread = 12;
- uint32_t max_value = max_count_of_quality( data );
-
- chart_captions( &c, "distribution of quality", "quality", "count" );
- factor = max_value;
- factor /= 450;
-
- /* max_y, step_y, max_value */
- chart_vruler1( &c, 450, 50, max_value, NULL );
- for ( i = 0; i <= 21; ++i )
- pp[ i ] = i * 2;
- chart_hruler( &c, pp, 21, 0, h_spread * 2 );
-
- /* prepare the one and only line to draw */
- for ( i = 0; i <= 41; ++i )
- {
- uint64_t value = data->total.qual[ i ];
- value *= 450;
- value /= max_value;
- pp[ i ] = (uint32_t) value;
- }
- chart_line( &c, pp, 41, 1.0, h_spread, "blue", "count of quality" );
-
- rc = chart_write( &c, filename );
- chart_destroy( &c );
- }
- return rc;
-}
-
-
-static rc_t print_qual_distr( p_reads_data data, p_report report )
-{
- uint8_t i;
- rc_t rc = report_set_columns( report, 2, "quality_value", "count" );
- if ( rc == 0 )
- {
- for ( i = 0; i < 41 && rc == 0 ; ++i )
- {
- rc = report_new_data( report );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", i );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%u", data->total.qual[ i ] );
- }
- }
- return rc;
-}
-
-
-static rc_t print_kmers( p_reads_data data, p_report report )
-{
- uint16_t i;
- rc_t rc = report_set_columns( report, 7,
- "sequence", "count", "probability", "expected",
- "observed-vs-expected", "max-obs-vs-exp", "at" );
- if ( rc == 0 )
- {
- for ( i = 0; i < NKMER5 && rc == 0 ; ++i )
- {
- uint16_t idx = data->kmer5_idx[ i ];
- p_kmer5_count k = &( data->kmer5[ idx ] );
-
- rc = report_new_data( report );
- if ( rc == 0 )
- {
- char s[ 6 ];
- kmer_int_2_ascii( idx, s );
- rc = report_add_data( report, 10, "%s", s );
- }
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%lu", k->total_count );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%f", k->probability );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%.0f", k->expected );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%f", k->observed_vs_expected );
- if ( rc == 0 )
- rc = report_add_data( report, 20, "%f", k->max_bp_obs_vs_exp );
- if ( rc == 0 )
- {
- uint32_t from, to;
- from = compress_start_lookup( k->max_bp_obs_vs_exp_at );
- to = compress_end_lookup( k->max_bp_obs_vs_exp_at );
- if ( from == to )
- rc = report_add_data( report, 20, "%d", from );
- else
- rc = report_add_data( report, 20, "%d .. %d", from, to );
- }
- }
- }
- return rc;
-}
-
-
-static rc_t CC mod_reads_free ( p_module self )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( self != NULL )
- {
- if ( self->mod_ctx != NULL )
- {
- p_reads_data data = self->mod_ctx;
- rd_filter_destroy( &data->filter );
- free( self->mod_ctx );
- }
- rc = 0;
- }
- return rc;
-}
-
-
-static rc_t analyze_param( p_reads_data data, const char * param )
-{
- const KNamelist *plist;
- rc_t rc = helper_make_namelist_from_string( &plist, param, '/' );
- if ( rc == 0 )
- {
- uint32_t count;
- rc = KNamelistCount( plist, &count );
- if ( rc == 0 )
- {
- uint32_t i;
- for ( i = 0; i < count && rc == 0; ++i )
- {
- const char * s;
- rc = KNamelistGet( plist, i, &s );
- if ( rc == 0 )
- {
- if ( helper_str_cmp( s, "bio" ) == 0 )
- data->bio = true;
- else if ( helper_str_cmp( s, "trim" ) == 0 )
- data->trim = true;
- else if ( helper_str_cmp( s, "cut" ) == 0 )
- data->cut = true;
- }
- }
- }
- KNamelistRelease( plist );
- }
- return rc;
-}
-
-
-static rc_t CC mod_reads_count( p_module self, uint32_t * count )
-{
- *count = 6;
- return 0;
-}
-
-
-static rc_t CC mod_reads_name( p_module self, uint32_t n, char ** name )
-{
- switch ( n )
- {
- case 0 : *name = string_dup_measure ( "global", NULL ); break;
- case 1 : *name = string_dup_measure ( "bases", NULL ); break;
- case 2 : *name = string_dup_measure ( "quality", NULL ); break;
- case 3 : *name = string_dup_measure ( "readlength", NULL ); break;
- case 4 : *name = string_dup_measure ( "qual_dist", NULL ); break;
- case 5 : *name = string_dup_measure ( "k_mer", NULL ); break;
- default : *name = string_dup_measure ( "unknown", NULL ); break;
- }
- return 0;
-}
-
-
-static rc_t CC mod_reads_report( p_module self,
- p_report report,
- const uint32_t sub_select )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcInvalid );
- p_reads_data data = (p_reads_data)self->mod_ctx;
- if ( data != NULL && report != NULL )
- {
- switch( sub_select )
- {
- case 0 : rc = print_totals( data, report ); break;
- case 1 : rc = print_pos_bases( data, report ); break;
- case 2 : rc = print_pos_quality( data, report ); break;
- case 3 : rc = print_read_length( data, report ); break;
- case 4 : rc = print_qual_distr( data, report ); break;
- case 5 : rc = print_kmers( data, report ); break;
- }
- }
- return rc;
-}
-
-
-static rc_t CC mod_reads_graph( p_module self, uint32_t n, const char * filename )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcUnsupported );
- p_reads_data data = (p_reads_data)self->mod_ctx;
- switch( n )
- {
- case 1 : rc = graph_pos_bases( data, filename ); break;
- case 2 : rc = graph_pos_quality( data, filename ); break;
- case 3 : rc = graph_read_length( data, filename ); break;
- case 4 : rc = graph_qual_distr( data, filename ); break;
- }
- return rc;
-}
-
-
-static rc_t CC mod_reads_pre_open( p_module self, const VCursor * cur )
-{
- rc_t rc = RC( rcExe, rcData, rcAllocating, rcParam, rcNull );
- if ( self != NULL && cur != NULL )
- {
- p_reads_data data = self->mod_ctx;
- rc = rd_filter_pre_open( &data->filter, cur );
- }
- return rc;
-}
-
-
-rc_t mod_reads_init( p_module self, const char * param )
-{
- rc_t rc = 0;
-
- /* put in it's name */
- self->name = string_dup_measure ( "bases", NULL );
-
- /* connect the callback-functions */
- self->f_pre_open = mod_reads_pre_open;
- self->f_row = mod_reads_row;
- self->f_post_rows = mod_reads_post_rows;
- self->f_count = mod_reads_count;
- self->f_name = mod_reads_name;
- self->f_report = mod_reads_report;
- self->f_graph = mod_reads_graph;
- self->f_free = mod_reads_free;
-
- /* initialize the module context */
- self->mod_ctx = calloc( 1, sizeof( struct reads_data ) );
- if ( self->mod_ctx == NULL )
- {
- rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- else
- {
- p_reads_data data = self->mod_ctx;
- /* this is done in 2 steps to work on 32-bit-compilers too
- ( no 64-bit literals in 32-bit-compilers ) */
- data->min_read_len = 0xFFFFFFFF;
- data->min_read_len <<= 32;
- data->min_read_len |= 0xFFFFFFFF;
- data->min_quality = 0xFF;
- setup_bp_array( data->bp_data, MAX_COMPRESSED_BASE_POS );
-
- rc = rd_filter_init( &data->filter );
- if ( rc == 0 && param != NULL )
- rc = analyze_param( data, param );
- if ( rc == 0 )
- rc = rd_filter_set_flags( &data->filter, data->bio, data->trim, data->cut );
- if ( rc == 0 )
- {
- if ( param != NULL )
- OUTMSG(( "module 'bases' initialized (%s)\n", param ));
- else
- OUTMSG(( "module 'bases' initialized\n" ));
- }
- }
- return rc;
-}
diff --git a/tools/run-stat/mod_reads_defs.h b/tools/run-stat/mod_reads_defs.h
deleted file mode 100644
index 07e6b74..0000000
--- a/tools/run-stat/mod_reads_defs.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_stat_mod_reads_defs_
-#define _h_stat_mod_reads_defs_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include "rd_filter.h"
-
-#define MAX_COMPRESSED_BASE_POS 49
-#define MAX_BASE_POS 1899
-
-#define MIN_QUALITY 2
-#define MAX_QUALITY 40
-
-#define IDX_A 0
-#define IDX_C 1
-#define IDX_G 2
-#define IDX_T 3
-#define IDX_N 4
-
-typedef struct base_pos
-{
- uint32_t from;
- uint32_t to;
-
- uint64_t read_count;
- uint64_t count;
-
- uint64_t qual[ MAX_QUALITY + 1 ]; /* qual 0...MAX_QUALITY */
- double mean;
- double median;
- uint8_t mode;
- double upper_quart;
- double lower_quart;
- double upper_centile;
- double lower_centile;
-
- uint64_t base_sum[ IDX_N + 1 ];
- double base_percent[ IDX_N + 1 ];
-} base_pos;
-typedef base_pos* p_base_pos;
-
-
-#define NKMER5 1024
-
-
-typedef struct kmer5_count
-{
- uint64_t count[ MAX_COMPRESSED_BASE_POS + 1 ];
- double bp_expected[ MAX_COMPRESSED_BASE_POS + 1 ];
- double bp_obs_vs_exp[ MAX_COMPRESSED_BASE_POS + 1 ];
- uint64_t total_count;
- double probability;
- double expected;
- double observed_vs_expected;
- double max_bp_obs_vs_exp;
- uint16_t max_bp_obs_vs_exp_at;
-} kmer5_count;
-typedef kmer5_count* p_kmer5_count;
-
-
-typedef struct reads_data
-{
- /* the statistical data to be collected */
- uint64_t sum_reads;
- uint64_t min_read_len;
- uint64_t max_read_len;
- double mean_read_len;
-
- uint8_t min_quality;
- uint8_t max_quality;
-
- /* for the whole sequence */
- base_pos total;
- double base_prob[ IDX_N + 1 ];
-
- /* for a specific base-position */
- base_pos bp_data[ MAX_COMPRESSED_BASE_POS + 1 ];
-
- /* to count the Kmer's */
- uint64_t bp_total_kmers[ MAX_COMPRESSED_BASE_POS + 1 ];
- kmer5_count kmer5[ NKMER5 ];
- uint16_t kmer5_idx[ NKMER5 ];
- uint64_t total_kmers;
-
- /* the filtered read-data */
- rd_filter filter;
-
- /* flags how to process the data */
- bool bio;
- bool trim;
- bool cut;
-
-} reads_data;
-typedef reads_data* p_reads_data;
-
-
-#endif
diff --git a/tools/run-stat/mod_reads_helper.c b/tools/run-stat/mod_reads_helper.c
deleted file mode 100644
index 1a0b756..0000000
--- a/tools/run-stat/mod_reads_helper.c
+++ /dev/null
@@ -1,575 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/sort.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-#include "mod_reads_defs.h"
-
-uint8_t CompressedBasePos_Lookup[ MAX_BASE_POS + 1 ] =
-{
- /* 0 1 2 3 4 5 6 7 8 9 */
- /* 0 */ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
- /* 1 */ 10, 10, 10, 10, 10, 11, 11, 11, 11, 11,
- /* 2 */ 12, 12, 12, 12, 12, 13, 13, 13, 13, 13,
- /* 3 */ 14, 14, 14, 14, 14, 15, 15, 15, 15, 15,
- /* 4 */ 16, 16, 16, 16, 16, 17, 17, 17, 17, 17,
- /* 5 */ 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
- /* 6 */ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- /* 7 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
- /* 8 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
- /* 9 */ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
- /* 10 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- /* 11 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- /* 12 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- /* 13 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- /* 14 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- /* 15 */ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- /* 16 */ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- /* 17 */ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- /* 18 */ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- /* 19 */ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- /* 20 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
- /* 21 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
- /* 22 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
- /* 23 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
- /* 24 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
- /* 25 */ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
- /* 26 */ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
- /* 27 */ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
- /* 28 */ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
- /* 29 */ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
- /* 30 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
- /* 31 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
- /* 32 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
- /* 33 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
- /* 34 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
- /* 35 */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
- /* 36 */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
- /* 37 */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
- /* 38 */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
- /* 39 */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
- /* 40 */ 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
- /* 41 */ 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
- /* 42 */ 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
- /* 43 */ 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
- /* 44 */ 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
- /* 45 */ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
- /* 46 */ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
- /* 47 */ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
- /* 48 */ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
- /* 49 */ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
- /* 50 */ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
- /* 51 */ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
- /* 52 */ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
- /* 53 */ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
- /* 54 */ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
- /* 55 */ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
- /* 56 */ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
- /* 57 */ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
- /* 58 */ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
- /* 59 */ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
- /* 60 */ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- /* 61 */ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- /* 62 */ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- /* 63 */ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- /* 64 */ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- /* 65 */ 34, 34, 34, 34, 34, 34, 34, 34, 34, 34,
- /* 66 */ 34, 34, 34, 34, 34, 34, 34, 34, 34, 34,
- /* 67 */ 34, 34, 34, 34, 34, 34, 34, 34, 34, 34,
- /* 68 */ 34, 34, 34, 34, 34, 34, 34, 34, 34, 34,
- /* 69 */ 34, 34, 34, 34, 34, 34, 34, 34, 34, 34,
- /* 70 */ 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,
- /* 71 */ 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,
- /* 72 */ 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,
- /* 73 */ 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,
- /* 74 */ 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,
- /* 75 */ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
- /* 76 */ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
- /* 77 */ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
- /* 78 */ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
- /* 79 */ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
- /* 80 */ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
- /* 81 */ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
- /* 82 */ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
- /* 83 */ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
- /* 84 */ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
- /* 85 */ 38, 38, 38, 38, 38, 38, 38, 38, 38, 38,
- /* 86 */ 38, 38, 38, 38, 38, 38, 38, 38, 38, 38,
- /* 87 */ 38, 38, 38, 38, 38, 38, 38, 38, 38, 38,
- /* 88 */ 38, 38, 38, 38, 38, 38, 38, 38, 38, 38,
- /* 89 */ 38, 38, 38, 38, 38, 38, 38, 38, 38, 38,
- /* 90 */ 39, 39, 39, 39, 39, 39, 39, 39, 39, 39,
- /* 91 */ 39, 39, 39, 39, 39, 39, 39, 39, 39, 39,
- /* 92 */ 39, 39, 39, 39, 39, 39, 39, 39, 39, 39,
- /* 93 */ 39, 39, 39, 39, 39, 39, 39, 39, 39, 39,
- /* 94 */ 39, 39, 39, 39, 39, 39, 39, 39, 39, 39,
- /* 95 */ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
- /* 96 */ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
- /* 97 */ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
- /* 98 */ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
- /* 99 */ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
- /*100 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*101 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*102 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*103 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*104 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*105 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*106 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*107 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*108 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*109 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- /*110 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*111 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*112 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*113 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*114 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*115 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*116 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*117 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*118 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*119 */ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- /*120 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*121 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*122 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*123 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*124 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*125 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*126 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*127 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*128 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*129 */ 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
- /*130 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*131 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*132 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*133 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*134 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*135 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*136 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*137 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*138 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*139 */ 44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
- /*140 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*141 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*142 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*143 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*144 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*145 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*146 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*147 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*148 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*149 */ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
- /*150 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*151 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*152 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*153 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*154 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*155 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*156 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*157 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*158 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*159 */ 46, 46, 46, 46, 46, 46, 46, 46, 46, 46,
- /*160 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*161 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*162 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*163 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*164 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*165 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*166 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*167 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*168 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*169 */ 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,
- /*170 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*171 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*172 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*173 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*174 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*175 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*176 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*177 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*178 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*179 */ 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
- /*180 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*181 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*182 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*183 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*184 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*185 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*186 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*187 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*188 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
- /*189 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49
-};
-
-uint32_t CompressedStart_Lookup[ MAX_COMPRESSED_BASE_POS + 1 ] =
-{
- /* 0 1 2 3 4 5 6 7 8 9 */
- /* 0 */ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
- /* 1 */ 10, 15, 20, 25, 30, 35, 40, 45, 50, 60,
- /* 2 */ 70, 80, 90, 100, 150, 200, 250, 300, 350, 400,
- /* 3 */ 450, 500, 550, 600, 650, 700, 750, 800, 850, 900,
- /* 4 */ 950, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800
-};
-
-uint32_t CompressedEnd_Lookup[ MAX_COMPRESSED_BASE_POS + 1 ] =
-{
- /* 0 1 2 3 4 5 6 7 8 9 */
- /* 0 */ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
- /* 1 */ 14, 19, 24, 29, 34, 39, 44, 49, 59, 69,
- /* 2 */ 79, 89, 99, 149, 199, 249, 299, 349, 399, 449,
- /* 3 */ 499, 549, 599, 649, 699, 749, 799, 849, 899, 949,
- /* 4 */ 1000, 1099, 1199, 1299, 1399, 1499, 1599, 1699, 1799, 0xFFFFFFFF
-};
-
-uint8_t compress_base_pos( const uint32_t src )
-{
- if ( src > MAX_BASE_POS )
- return MAX_COMPRESSED_BASE_POS;
- else
- return CompressedBasePos_Lookup[ src ];
-}
-
-uint32_t compress_start_lookup( const uint8_t idx )
-{
- if ( idx <= MAX_COMPRESSED_BASE_POS )
- return CompressedStart_Lookup[ idx ];
- else
- return CompressedStart_Lookup[ MAX_COMPRESSED_BASE_POS ];
-}
-
-uint32_t compress_end_lookup( const uint8_t idx )
-{
- if ( idx <= MAX_COMPRESSED_BASE_POS )
- return CompressedEnd_Lookup[ idx ];
- else
- return CompressedEnd_Lookup[ MAX_COMPRESSED_BASE_POS ];
-}
-
-
-static uint8_t Kmer_Ascii_Lookup[ 26 ] =
-{
- /* A B C D E F G H I J K L M */
- 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0,
- /* N O P Q R S T U V W X Y Z */
- 4, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0
-};
-
-uint16_t kmer_ascii_2_int( char * s )
-{
- uint16_t res = Kmer_Ascii_Lookup[ s[ 0 ] - 'A' ];
- res <<= 2;
- res |= Kmer_Ascii_Lookup[ s[ 1 ] - 'A' ];
- res <<= 2;
- res |= Kmer_Ascii_Lookup[ s[ 2 ] - 'A' ];
- res <<= 2;
- res |= Kmer_Ascii_Lookup[ s[ 3 ] - 'A' ];
- res <<= 2;
- res |= Kmer_Ascii_Lookup[ s[ 4 ] - 'A' ];
- return res;
-}
-
-static char Kmer_int_Lookup[ 4 ] = { 'A', 'C', 'G', 'T' };
-
-void kmer_int_2_ascii( const uint16_t kmer, char * s )
-{
- uint16_t value = kmer;
- s[ 5 ] = 0;
- s[ 4 ] = Kmer_int_Lookup[ value & 3 ];
- value >>= 2;
- s[ 3 ] = Kmer_int_Lookup[ value & 3 ];
- value >>= 2;
- s[ 2 ] = Kmer_int_Lookup[ value & 3 ];
- value >>= 2;
- s[ 1 ] = Kmer_int_Lookup[ value & 3 ];
- value >>= 2;
- s[ 0 ] = Kmer_int_Lookup[ value & 3 ];
-}
-
-
-uint16_t kmer_add_base( const uint16_t kmer, const char c )
-{
- uint16_t res = ( ( kmer << 2 ) & 0x3FC );
- return ( res | ( Kmer_Ascii_Lookup[ c - 'A' ] & 0x03 ) );
-}
-
-
-static double calc_mean_qual( uint64_t * qual, uint64_t count )
-{
- uint8_t idx;
- double sum = 0.0;
- for ( idx = 1; idx <= MAX_QUALITY; ++ idx )
- {
- sum += ( qual[ idx ] * idx );
- }
- sum /= count;
- return sum;
-}
-
-
-static uint8_t get_qual_at_flat_pos( uint64_t * qual, uint64_t flat_pos )
-{
- uint8_t idx, res = 0;
- uint64_t pos = 0;
- bool found = false;
- for ( idx = 0; idx <= MAX_QUALITY && !found; ++ idx )
- {
- found = ( ( pos <= flat_pos ) && ( flat_pos < ( pos + qual[ idx ] ) ) );
- if ( found )
- res = idx;
- else
- pos += qual[ idx ];
- }
- return res;
-}
-
-
-static double calc_median_qual_between( uint64_t * qual, uint64_t from, uint64_t to )
-{
- double res;
- uint64_t count = ( to - from );
- if ( ( count & 1 ) == 0 )
- {
- uint64_t median_pos = ( from + ( count >> 1 ) ) - 1; /* even */
- res = get_qual_at_flat_pos( qual, median_pos );
- res += get_qual_at_flat_pos( qual, median_pos + 1 );
- res /= 2;
- }
- else
- {
- uint64_t median_pos = ( from + ( ( count + 1 ) >> 1 ) ) - 1; /* odd */
- res = get_qual_at_flat_pos( qual, median_pos );
- }
- return res;
-}
-
-
-static double calc_median_qual( uint64_t * qual, uint64_t count )
-{
- return calc_median_qual_between( qual, 0, count );
-}
-
-
-static uint8_t calc_mode( uint64_t * qual, uint64_t count )
-{
- uint8_t idx, max = 0, res = 0;
- for ( idx = 1; idx <= MAX_QUALITY; ++ idx )
- {
- if ( max < qual[ idx ] )
- {
- max = qual[ idx ];
- res = idx;
- }
- }
- return res;
-}
-
-
-static double calc_upper_quart( uint64_t * qual, uint64_t count )
-{
- uint64_t half = count >> 1;
- return calc_median_qual_between( qual, half, count );
-}
-
-
-static double calc_lower_quart( uint64_t * qual, uint64_t count )
-{
- uint64_t half = count >> 1;
- return calc_median_qual_between( qual, 0, half );
-}
-
-
-static double calc_upper_centile( uint64_t * qual, uint64_t count )
-{
- uint64_t tenth = count / 10;
- return get_qual_at_flat_pos( qual, count - tenth );
-}
-
-
-static double calc_lower_centile( uint64_t * qual, uint64_t count )
-{
- uint64_t tenth = count / 10;
- return get_qual_at_flat_pos( qual, tenth );
-}
-
-
-void setup_bp_array( p_base_pos bp, const uint8_t count )
-{
- uint8_t i;
- for ( i = 0; i <= count; ++i )
- {
- bp[ i ].from = compress_start_lookup( i );
- bp[ i ].to = compress_end_lookup( i );
- }
-}
-
-
-/* base_prob points to a array of base-probabilities in the order A,C,G,T */
-double kmer_probability( const uint16_t kmer, double * base_prob )
-{
- double res = 1.0;
- uint16_t temp = kmer;
-
- res *= base_prob[ temp & 3 ];
- temp >>= 2;
- res *= base_prob[ temp & 3 ];
- temp >>= 2;
- res *= base_prob[ temp & 3 ];
- temp >>= 2;
- res *= base_prob[ temp & 3 ];
- temp >>= 2;
- res *= base_prob[ temp & 3 ];
- return res;
-}
-
-
-void count_base( p_base_pos bp, const char base, const uint8_t quality )
-{
- bp->count++;
- switch( base )
- {
- case 'A' : bp->base_sum[ IDX_A ]++; break;
- case 'C' : bp->base_sum[ IDX_C ]++; break;
- case 'G' : bp->base_sum[ IDX_G ]++; break;
- case 'T' : bp->base_sum[ IDX_T ]++; break;
- case 'N' : bp->base_sum[ IDX_N ]++; break;
- }
- if ( quality <= MAX_QUALITY )
- bp->qual[ quality ]++;
- else
- bp->qual[ MAX_QUALITY ]++;
-}
-
-
-uint16_t count_kmer( p_reads_data data,
- const uint16_t kmer,
- const uint8_t base_pos,
- const char base )
-{
- uint16_t res = kmer_add_base( kmer, base );
- if ( base_pos > 3 )
- {
- /* pointer to the conter for this K-mer */
- p_kmer5_count cnt = &data->kmer5[ res ];
- /* count how often this K-mer occures at all */
- cnt->total_count++;
- /* count how often this K-mer occures at the given base-position */
- cnt->count[ base_pos ]++;
- /* count how many K-mers at all we have at this base-position */
- data->bp_total_kmers[ base_pos ]++;
- }
- return res;
-}
-
-
-void calculate_quality_mean_median_quart_centile( p_base_pos bp )
-{
- bp->mean = calc_mean_qual( bp->qual, bp->count );
- bp->median = calc_median_qual( bp->qual, bp->count );
- bp->mode = calc_mode( bp->qual, bp->count );
- bp->upper_quart = calc_upper_quart( bp->qual, bp->count );
- bp->lower_quart = calc_lower_quart( bp->qual, bp->count );
- bp->upper_centile = calc_upper_centile( bp->qual, bp->count );
- bp->lower_centile = calc_lower_centile( bp->qual, bp->count );
-}
-
-
-void calculate_bases_percentage( p_base_pos bp )
-{
- uint8_t i;
- for ( i = IDX_A; i <= IDX_N; ++i )
- bp->base_percent[ i ] = percent( bp->base_sum[ i ], bp->count );
-}
-
-
-void calculate_base_probability( p_reads_data data )
-{
- uint8_t i;
- uint64_t value, total_acgt = 0;
- for ( i = IDX_A; i <= IDX_T; ++i )
- {
- value = data->total.base_sum[ i ];
- total_acgt += value;
- data->base_prob[ i ] = value;
- }
- /* do not merge the 2 loops, in the first one we calculate total_acgt
- int the second loop we use it on every item */
- for ( i = IDX_A; i <= IDX_T; ++i )
- data->base_prob[ i ] /= total_acgt;
-}
-
-
-int CC kmer_sort_callback( const void *p1, const void *p2, void * data )
-{
- p_reads_data rdata = data;
- int16_t idx1 = *( int16_t * ) p1;
- int16_t idx2 = *( int16_t * ) p2;
- p_kmer5_count k1 = &( rdata->kmer5[ idx1 ] );
- p_kmer5_count k2 = &( rdata->kmer5[ idx2 ] );
- if ( k1->observed_vs_expected < k2->observed_vs_expected )
- return 1;
- if ( k1->observed_vs_expected > k2->observed_vs_expected )
- return -1;
- return ( idx1 - idx2 );
-}
-
-
-void calculate_kmer_observed_vs_expected( p_reads_data data )
-{
- uint16_t idx;
- for ( idx = 0; idx < NKMER5; ++idx )
- {
- data->kmer5[ idx ].probability = kmer_probability( idx, data->base_prob );
- data->total_kmers += data->kmer5[ idx ].total_count;
- data->kmer5_idx[ idx ] = idx;
- }
- /* do not merge the 2 loops, we need to calculate total_kmers first ! */
- for ( idx = 0; idx < NKMER5; ++idx )
- {
- uint16_t i;
- p_kmer5_count k = &( data->kmer5[ idx ] );
-
- k->expected = data->total_kmers;
- k->expected *= k->probability;
- k->observed_vs_expected = k->total_count;
- k->observed_vs_expected /= k->expected;
- for ( i = 0; i < MAX_COMPRESSED_BASE_POS; ++i )
- {
- k->bp_expected[ i ] = data->bp_total_kmers[ i ];
- k->bp_expected[ i ] *= k->probability;
- k->bp_obs_vs_exp[ i ] = k->count[ i ];
- k->bp_obs_vs_exp[ i ] /= k->bp_expected[ i ];
- if ( k->bp_obs_vs_exp[ i ] > k->max_bp_obs_vs_exp )
- {
- k->max_bp_obs_vs_exp = k->bp_obs_vs_exp[ i ];
- k->max_bp_obs_vs_exp_at = i;
- }
- }
- }
-
- /* sort the kmer-idx-array by the observed-vs-expected value of the kmer */
- ksort ( data->kmer5_idx, NKMER5, sizeof( uint16_t ),
- kmer_sort_callback, data );
-}
diff --git a/tools/run-stat/mod_reads_helper.h b/tools/run-stat/mod_reads_helper.h
deleted file mode 100644
index 5b21407..0000000
--- a/tools/run-stat/mod_reads_helper.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_stat_mod_reads_helper_
-#define _h_stat_mod_reads_helper_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
-
-#include <klib/rc.h>
-#include "mod_reads_defs.h"
-
-uint8_t compress_base_pos( const uint32_t src );
-uint32_t compress_start_lookup( const uint8_t idx );
-uint32_t compress_end_lookup( const uint8_t idx );
-
-uint16_t kmer_ascii_2_int( char * s );
-void kmer_int_2_ascii( const uint16_t kmer, char * s );
-uint16_t kmer_add_base( const uint16_t kmer, const char c );
-
-void setup_bp_array( p_base_pos bp, const uint8_t count );
-double kmer_probability( const uint16_t kmer, double * base_prob );
-void count_base( p_base_pos bp, const char base, const uint8_t quality );
-uint16_t count_kmer( p_reads_data data,
- const uint16_t kmer,
- const uint8_t base_pos,
- const char base );
-
-void calculate_quality_mean_median_quart_centile( p_base_pos bp );
-void calculate_bases_percentage( p_base_pos bp );
-void calculate_base_probability( p_reads_data data );
-void calculate_kmer_observed_vs_expected( p_reads_data data );
-
-#endif
diff --git a/tools/run-stat/num-gen.c b/tools/run-stat/num-gen.c
deleted file mode 100644
index adfa016..0000000
--- a/tools/run-stat/num-gen.c
+++ /dev/null
@@ -1,497 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "num-gen.h"
-
-#include <klib/vector.h>
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/out.h>
-#include <strtol.h>
-#include <sysalloc.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-
-static void CC ng_node_destroy( void *item, void *data )
-{
- free( item );
-}
-
-static p_ng_node ng_make_node( const uint64_t start, const uint64_t count )
-{
- p_ng_node p = ( p_ng_node )malloc( sizeof( ng_node ) );
- if ( p )
- {
- p->start = start;
- p->count = count;
- }
- return p;
-}
-
-
-static int CC ng_insert_helper( const void* item1, const void* item2 )
-{
- int res = 0;
- p_ng_node node1 = (p_ng_node)item1;
- p_ng_node node2 = (p_ng_node)item2;
- if ( node1->start < node2->start )
- res = -1;
- else if ( node1->start > node2->start )
- res = 1;
- return res;
-}
-
-
-static bool ng_add_node( p_ng generator, const uint64_t num_1, const uint64_t num_2 )
-{
- bool res = false;
-
- if ( generator != NULL )
- {
- p_ng_node node;
- if ( num_1 == num_2 )
- {
- node = ng_make_node( num_1, 1 );
- }
- else if ( num_1 < num_2 )
- {
- node = ng_make_node( num_1, ( num_2 - num_1 ) + 1 );
- }
- else
- {
- node = ng_make_node( num_2, ( num_1 - num_2 ) + 1 );
- }
- if ( node != NULL )
- {
- res = ( VectorInsert( &(generator->nodes), node, NULL, ng_insert_helper ) == 0 );
- if ( res )
- {
- generator->node_count++;
- }
- }
- }
- return res;
-}
-
-
-#define MAX_NUM_STR 12
-/* helper-structure for vdn_parse() */
-typedef struct ng_parse_ctx
-{
- size_t num_str_idx, num_count;
- uint64_t num[ 2 ];
- char num_str[ MAX_NUM_STR + 1 ];
-} ng_parse_ctx;
-typedef ng_parse_ctx* p_ng_parse_ctx;
-
-
-/* helper for ng_parse() */
-static void ng_convert_ctx( p_ng_parse_ctx ctx )
-{
- char *endp;
-
- ctx->num_str[ ctx->num_str_idx ] = 0;
- ctx->num[ 0 ] = strtou64( ctx->num_str, &endp, 10 );
- ctx->num_count = 1;
- ctx->num_str_idx = 0;
-}
-
-
-/* helper for ng_parse() */
-static uint32_t ng_convert_and_add_ctx( p_ng generator, p_ng_parse_ctx ctx )
-{
- uint32_t res = 0;
- char *endp;
-
- if ( ctx->num_str_idx > 0 )
- {
- ctx->num_str[ ctx->num_str_idx ] = 0;
- ctx->num[ ctx->num_count ] = strtou64( ctx->num_str, &endp, 10 );
- ctx->num_str_idx = 0;
- if ( ctx->num_count == 0 )
- {
- ctx->num[1] = ctx->num[0];
- }
- if ( ng_add_node( generator, ctx->num[0], ctx->num[1] ) )
- {
- res++;
- }
- ctx->num_count = 0;
- }
- return res;
-}
-
-
-uint32_t ng_parse( p_ng generator, const char* src )
-{
- size_t i, n;
- ng_parse_ctx ctx;
- uint32_t res = 0;
-
- if ( generator == NULL ) return res;
- if ( src == NULL ) return res;
- n = strlen( src );
- generator->node_count = 0;
- generator->curr_node = 0;
- generator->curr_node_sub_pos = 0;
- if ( n == 0 ) return res;
-
- ctx.num_str_idx = 0;
- ctx.num_count = 0;
-
- for ( i=0; i<n; ++i )
- {
- switch ( src[i] )
- {
- /* a dash switches from N1-mode into N2-mode */
- case '-' :
- ng_convert_ctx( &ctx );
- break;
-
- /* a comma ends a single number or a range */
- case ',' :
- res += ng_convert_and_add_ctx( generator, &ctx );
- break;
-
- /* in both mode add the char to the temp string */
- default:
- if ( ( src[i]>='0' )&&( src[i]<='9' )&&( ctx.num_str_idx < MAX_NUM_STR ) )
- {
- ctx.num_str[ ctx.num_str_idx++ ] = src[ i ];
- }
- break;
- }
- }
- res += ng_convert_and_add_ctx( generator, &ctx );
- return res;
-}
-
-
-bool ng_set_range( p_ng generator,
- const int64_t first, const uint64_t count )
-{
- bool res = ( generator != NULL );
- if ( res )
- {
- uint64_t num_1 = first;
- uint64_t num_2;
-
- /* this is necessary because virtual columns which have a
- infinite row-range, get reported with first=1,count=0 */
- if ( count > 0 )
- {
- num_2 = first + count - 1;
- }
- else
- {
- num_2 = first;
- }
-
- /* clear all nodes so far... */
- VectorWhack( &(generator->nodes), ng_node_destroy, NULL );
- /* re-init the vector */
- VectorInit( &(generator->nodes ), 0, 5 );
- generator->node_count = 0;
- generator->curr_node = 0;
- generator->curr_node_sub_pos = 0;
-
- res = ng_add_node( generator, num_1, num_2 );
- }
- return res;
-}
-
-
-static bool CC ng_check_range_start( p_ng_node the_node,
- const uint64_t range_start )
-{
- bool res = true;
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( the_node->start < range_start )
- {
- the_node->start = range_start;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- res = false;
- }
- }
- return res;
-}
-
-
-static void CC ng_check_range_end( p_ng_node the_node,
- const uint64_t last_tab_row )
-{
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( last_node_row > last_tab_row )
- {
- last_node_row = last_tab_row;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- }
- }
-}
-
-
-static void CC ng_check_range_callback( void *item, void *data )
-{
- p_ng_node the_node = ( p_ng_node )item;
- p_ng_node the_range = ( p_ng_node )data;
- uint64_t last_tab_row = ( the_range->start + the_range->count - 1 );
-
- /* check if the start value is not out of range... */
- if ( ng_check_range_start( the_node, the_range->start ) )
- {
- ng_check_range_end( the_node, last_tab_row );
- }
-}
-
-
-static void CC ng_count_invalid_nodes( void *item, void *data )
-{
- p_ng_node the_node = ( p_ng_node )item;
- uint32_t *invalid_count = ( uint32_t * )data;
-
- if ( ( the_node->start == 0 )&&( the_node->count == 0 ) )
- ( *invalid_count )++;
-}
-
-
-static void CC ng_copy_valid_nodes( void *item, void *data )
-{
- p_ng_node the_node = ( p_ng_node )item;
- Vector *p_temp_nodes = ( Vector * )data;
-
- if ( ( the_node->start != 0 )&&( the_node->count != 0 ) )
- {
- p_ng_node new_node = ng_make_node( the_node->start, the_node->count );
- if ( new_node )
- {
- VectorInsert( p_temp_nodes, new_node, NULL, ng_insert_helper );
- }
- }
-}
-
-
-static void ng_remove_invalid_nodes( p_ng generator, const uint32_t invalid_nodes )
-{
- Vector temp_nodes;
-
- /* create a temp. vector */
- VectorInit( &temp_nodes, 0, 5 );
-
- /* copy all valid nodes into the temp. vector */
- VectorForEach ( &(generator->nodes), false,
- ng_copy_valid_nodes, &temp_nodes );
-
- /* clear all nodes so far... */
- VectorWhack( &(generator->nodes), ng_node_destroy, NULL );
-
- /* re-init the vector */
- VectorInit( &(generator->nodes ), 0, 5 );
-
- /* copy (swallow) the valid nodes back into the generator */
- VectorCopy ( &temp_nodes, &(generator->nodes) );
-
- /* correct the node count */
- generator->node_count -= invalid_nodes;
-
- /* destroy the temp-vector, DO NOT PASS vdn_node_destroy into it */
- VectorWhack ( &temp_nodes, NULL, NULL );
-}
-
-
-bool ng_check_range( p_ng generator,
- const int64_t first, const uint64_t count )
-{
- bool res = ( generator != NULL );
- if ( res )
- {
- ng_node defined_range;
- uint32_t invalid_nodes = 0;
-
- if ( count > 0 )
- {
- /* walk all nodes to check for boundaries... */
- defined_range.start = first;
- defined_range.count = count;
-
- VectorForEach ( &(generator->nodes), false,
- ng_check_range_callback, &defined_range );
-
- VectorForEach ( &(generator->nodes), false,
- ng_count_invalid_nodes, &invalid_nodes );
- if ( invalid_nodes > 0 )
- {
- ng_remove_invalid_nodes( generator, invalid_nodes );
- }
- }
- }
- return res;
-}
-
-
-rc_t ng_make( ng** generator )
-{
- if ( generator == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
- }
- (*generator) = calloc( 1, sizeof( ng ) );
- if ( *generator == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- VectorInit( &((*generator)->nodes ), 0, 5 );
- return 0;
-}
-
-
-rc_t ng_destroy( p_ng generator )
-{
- if ( generator == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcDestroying, rcParam, rcNull );
- }
- VectorWhack( &(generator->nodes), ng_node_destroy, NULL );
- free( generator );
- return 0;
-}
-
-
-bool ng_start( p_ng generator )
-{
- if ( generator == NULL ) return false;
- generator->curr_node = ( generator->node_count > 0 ? 0 : 1 );
- generator->curr_node_sub_pos = 0;
- return true;
-}
-
-
-static bool ng_next_node( p_ng generator, uint64_t* value )
-{
- bool res = false;
- if ( generator->curr_node < generator->node_count )
- {
- p_ng_node node = (p_ng_node)VectorGet( &(generator->nodes),
- (uint32_t)generator->curr_node );
- if ( node != NULL )
- {
- uint64_t ret_value = node->start;
- if ( node->count < 2 )
- {
- /* the node is a single-number-node */
- generator->curr_node++;
- }
- else
- {
- /* the node is a number range */
- ret_value += generator->curr_node_sub_pos++;
- if ( generator->curr_node_sub_pos >= node->count )
- {
- generator->curr_node++;
- generator->curr_node_sub_pos = 0;
- }
- }
- if ( value ) *value = ret_value;
- res = true;
- }
- }
- return res;
-}
-
-
-bool ng_range_defined( p_ng generator )
-{
- bool res = false;
- if ( generator != NULL )
- {
- res = ( generator->node_count > 0 );
- }
- return res;
-}
-
-
-bool ng_next( p_ng generator, uint64_t* value )
-{
- bool res = false;
- if ( generator != NULL )
- {
- uint64_t ret_value = 0;
- if ( ng_range_defined( generator ) )
- {
- /* there are nodes (number-ranges) defined */
- res = ng_next_node( generator, &ret_value );
- }
- else
- {
- /* endless mode, there are NO nodes (number-ranges) defined */
- ret_value = generator->curr_node++;
- res = true;
- }
- if ( value ) *value = ret_value;
- }
- return res;
-}
-
-
-static void CC ng_count_callback( void *item, void *data )
-{
- p_ng_node the_node = ( p_ng_node )item;
- uint64_t *res = ( uint64_t * )data;
-
- *res += the_node->count;
-}
-
-
-uint64_t ng_count( p_ng generator )
-{
- uint64_t res = 0;
-
- if ( generator != NULL )
- {
- VectorForEach ( &(generator->nodes), false, ng_count_callback, &res );
- }
- return res;
-}
diff --git a/tools/run-stat/progressbar.c b/tools/run-stat/progressbar.c
deleted file mode 100644
index 65a7895..0000000
--- a/tools/run-stat/progressbar.c
+++ /dev/null
@@ -1,191 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/rc.h>
-#include <klib/out.h>
-#include <sysalloc.h>
-#include <stdlib.h>
-
-typedef struct progressbar
-{
- bool initialized;
- uint16_t percent;
-} progressbar;
-
-
-rc_t make_progressbar( progressbar ** pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- (*pb) = calloc( 1, sizeof( progressbar ) );
- if ( *pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- return 0;
-}
-
-
-rc_t destroy_progressbar( progressbar * pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
- free( pb );
- return 0;
-}
-
-
-uint8_t calc_progressbar_digits( uint64_t count )
-{
- uint8_t res = 0;
-
- if ( count > 10000 )
- {
- if ( count > 100000 )
- res = 2;
- else
- res = 1;
- }
- return res;
-}
-
-
-uint16_t percent_progressbar( const uint64_t total, const uint64_t progress,
- const uint8_t digits )
-{
- uint16_t res = 0;
- uint32_t factor = 100;
-
- if ( digits > 0 )
- {
- if ( digits > 1 )
- factor = 10000;
- else
- factor = 1000;
- }
-
- if ( total > 0 )
- {
- if ( progress >= total )
- res = factor;
- else
- {
- uint64_t temp = progress;
- temp *= factor;
- temp /= total;
- res = (uint16_t) temp;
- }
- }
-
- return res;
-}
-
-
-static void progess_0a( const uint16_t percent )
-{
- OUTMSG (( "| %2u%%", percent ));
-}
-
-
-static void progess_0( const uint16_t percent )
-{
- if ( percent & 1 )
- OUTMSG (( "\b\b\b\b- %2u%%", percent ));
- else
- OUTMSG (( "\b\b\b%2u%%", percent ));
-}
-
-
-static void progess_1a( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- OUTMSG (( "| %2u.%01u%%", p1, p0 ));
-}
-
-
-static void progess_1( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- OUTMSG (( "\b\b\b\b\b\b- %2u.%01u%%", p1, p0 ));
- else
- OUTMSG (( "\b\b\b\b\b%2u.%01u%%", p1, p0 ));
-}
-
-
-static void progess_2a( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- OUTMSG (( "| %2u.%02u%%", p1, p0 ));
-}
-
-
-static void progess_2( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- OUTMSG (( "\b\b\b\b\b\b\b- %2u.%02u%%", p1, p0 ));
- else
- OUTMSG (( "\b\b\b\b\b\b%2u.%02u%%", p1, p0 ));
-}
-
-
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent )
-{
- uint8_t digits = fract_digits;
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcSelf, rcNull );
- if ( digits > 2 )
- digits = 2;
- if ( pb->initialized )
- {
- if ( pb->percent != percent )
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0( percent ); break;
- case 1 : progess_1( percent ); break;
- case 2 : progess_2( percent ); break;
- }
- }
- }
- else
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0a( percent ); break;
- case 1 : progess_1a( percent ); break;
- case 2 : progess_2a( percent ); break;
- }
- pb->initialized = true;
- }
- return 0;
-}
diff --git a/tools/run-stat/progressbar.h b/tools/run-stat/progressbar.h
deleted file mode 100644
index f3681d8..0000000
--- a/tools/run-stat/progressbar.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_progressbar_
-#define _h_progressbar_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct progressbar progressbar;
-
-/*--------------------------------------------------------------------------
- * make_progressbar
- *
- * creates a progressbar with zero-values inside
- * does not output anything
- */
-rc_t make_progressbar( progressbar ** pb );
-
-
-/*--------------------------------------------------------------------------
- * destroy_progressbar
- *
- * destroy's the progressbar
- * does not output anything
- */
-rc_t destroy_progressbar( progressbar * pb );
-
-
-/*--------------------------------------------------------------------------
- * update_progressbar
- *
- * sets the progressbar to a specific percentage
- * outputs only if the percentage has changed from the last call
- * the precentage is in 1/10-th of a percent ( 21.6% = 216 )
- * expects the percents in increasing order ( does not jump back )
- * writes a growing bar made from '-'-chars every 2nd percent
- */
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent );
-
-
-uint8_t calc_progressbar_digits( uint64_t count );
-
-uint16_t percent_progressbar( const uint64_t total, const uint64_t progress,
- const uint8_t digits );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/run-stat/rd_filter.c b/tools/run-stat/rd_filter.c
deleted file mode 100644
index 69d3e28..0000000
--- a/tools/run-stat/rd_filter.c
+++ /dev/null
@@ -1,243 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "rd_filter.h"
-
-#include <os-native.h>
-/* #include <insdc/sra.h> */
-
-#include <sysalloc.h>
-#include <bitstr.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-
-
-static rc_t add_2_cur( p_rd_filter self, const VCursor * cur )
-{
- rc_t rc = VCursorAddColumn( cur, &self->idx_READ, "(INSDC:dna:text)READ" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_QUALITY, "(INSDC:quality:phred)QUALITY" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_TRIM_START, "TRIM_START" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_TRIM_LEN, "TRIM_LEN" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_READ_START, "READ_START" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_READ_LEN, "READ_LEN" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_READ_FILTER, "READ_FILTER" );
- if ( rc == 0 )
- rc = VCursorAddColumn( cur, &self->idx_READ_TYPE, "READ_TYPE" );
-
- return rc;
-}
-
-
-rc_t rd_filter_init( p_rd_filter self )
-{
- rc_t rc = char_buffer_init( &self->bases, 0 );
- if ( rc == 0 )
- rc = char_buffer_init( &self->filtered_bases, 5 );
- if ( rc == 0 )
- rc = char_buffer_init( &self->quality, 0 );
- if ( rc == 0 )
- rc = char_buffer_init( &self->filtered_quality, 0 );
- if ( rc == 0 )
- rc = char_buffer_init( &self->quality, 5 );
- if ( rc == 0 )
- rc = char_buffer_init( &self->filtered_quality, 5 );
- if ( rc == 0 )
- rc = int_buffer_init( &self->read_start, 5 );
- if ( rc == 0 )
- rc = int_buffer_init( &self->read_len, 5 );
- if ( rc == 0 )
- rc = int_buffer_init( &self->read_filter, 5 );
- if ( rc == 0 )
- rc = int_buffer_init( &self->read_type, 5 );
- return rc;
-}
-
-
-void rd_filter_destroy( p_rd_filter self )
-{
- char_buffer_destroy( &self->bases );
- char_buffer_destroy( &self->filtered_bases );
- char_buffer_destroy( &self->quality );
- char_buffer_destroy( &self->filtered_quality );
-
- int_buffer_destroy( &self->read_start );
- int_buffer_destroy( &self->read_len );
- int_buffer_destroy( &self->read_filter );
- int_buffer_destroy( &self->read_type );
-}
-
-
-rc_t rd_filter_pre_open( p_rd_filter self, const VCursor * cur )
-{
- return add_2_cur( self, cur );
- /* return get_cur_idx( self, cur ); */
-}
-
-#define SRA_READ_TYPE_BIOLOGICAL 1
-
-static bool rd_filter_restrict( p_rd_filter self, uint32_t ridx,
- uint32_t * start, uint32_t * end )
-{
- bool res = true;
-
- if ( self->bio )
- res = ( self->read_type.trans_ptr[ ridx ] == SRA_READ_TYPE_BIOLOGICAL );
- if ( res )
- {
- *start = self->read_start.trans_ptr[ ridx ];
- *end = *start + self->read_len.trans_ptr[ ridx ];
- if ( self->trim )
- {
- if ( *end < self->trim_start || *start > self->trim_end )
- res = false;
- else
- {
- if ( *start < self->trim_start )
- *start = self->trim_start;
- if ( *end > self->trim_end )
- *end = self->trim_end;
- }
- }
- if ( res && self->cut )
- {
- res = ( ( *end - *start ) > 25 );
- }
- }
-
- return res;
-}
-
-
-static rc_t rd_filter_apply( p_rd_filter self )
-{
- p_char_buffer b_src = &self->bases;
- p_char_buffer b_dst = &self->filtered_bases;
- p_char_buffer q_src = &self->quality;
- p_char_buffer q_dst = &self->filtered_quality;
-
- rc_t rc = char_buffer_realloc( b_dst, b_src->len );
- if ( rc == 0 )
- {
- b_dst->len = 0;
- rc = char_buffer_realloc( q_dst, q_src->len );
- if ( rc == 0 )
- {
- uint32_t ridx, n = self->read_start.len;
- q_dst->len = 0;
- for ( ridx = 0; ridx < n; ++ridx )
- {
- /* walk the READS */
- uint32_t start, end;
- if ( rd_filter_restrict( self, ridx, &start, &end ) )
- {
- uint32_t pos;
- for ( pos = start; pos < end; ++pos )
- {
- b_dst->ptr[ b_dst->len++ ] = b_src->trans_ptr[ pos ];
- q_dst->ptr[ q_dst->len++ ] = q_src->trans_ptr[ pos ];
- }
- }
- }
- }
- }
- return rc;
-}
-
-
-rc_t rd_filter_read( p_rd_filter self, const VCursor * cur )
-{
- /* first read the trim-values */
- rc_t rc = helper_read_uint32( cur, self->idx_TRIM_START, &self->trim_start );
- if ( rc == 0 )
- {
- rc = helper_read_uint32( cur, self->idx_TRIM_LEN, &self->trim_end );
- if ( rc == 0 )
- self->trim_end += self->trim_start;
- }
-
- /* then read the READ-SEGMENT-COLUMNs */
- if ( rc == 0 )
- rc = helper_read_int32_values( cur, self->idx_READ_START, &self->read_start );
- if ( rc == 0 )
- rc = helper_read_int32_values( cur, self->idx_READ_LEN, &self->read_len );
- if ( rc == 0 )
- rc = helper_read_int32_values( cur, self->idx_READ_FILTER, &self->read_filter );
- if ( rc == 0 )
- rc = helper_read_int32_values( cur, self->idx_READ_TYPE, &self->read_type );
-
- /* then read the BASES */
- if ( rc == 0 )
- rc = helper_read_char_values( cur, self->idx_READ, &self->bases );
-
- /* the read the QULAITY */
- if ( rc == 0 )
- rc = helper_read_char_values( cur, self->idx_QUALITY, &self->quality );
-
- if ( rc == 0 )
- rc = rd_filter_apply( self );
-
- return rc;
-}
-
-
-rc_t rd_filter_set_flags( p_rd_filter self, bool bio, bool trim, bool cut )
-{
- self->bio = bio;
- self->trim = trim;
- self->cut = cut;
- return 0;
-}
-
-
-uint32_t filter_get_read_len( p_rd_filter self )
-{
- return self->filtered_bases.len;
-}
-
-
-char filter_get_base( p_rd_filter self, const uint32_t idx )
-{
- if ( idx < self->filtered_bases.len )
- return self->filtered_bases.ptr[ idx ];
- else
- return 0;
-}
-
-
-uint8_t filter_get_quality( p_rd_filter self, const uint32_t idx )
-{
- if ( idx < self->filtered_quality.len )
- return (uint8_t)self->filtered_quality.ptr[ idx ];
- else
- return 0;
-}
diff --git a/tools/run-stat/rd_filter.h b/tools/run-stat/rd_filter.h
deleted file mode 100644
index 0f10128..0000000
--- a/tools/run-stat/rd_filter.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_rd_filter_
-#define _h_rd_filter_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <klib/namelist.h>
-#include <klib/rc.h>
-#include <klib/log.h>
-#include <klib/text.h>
-#include <vdb/cursor.h>
-#include "definitions.h"
-#include "helper.h"
-
-typedef struct rd_filter
-{
- /* the index of the columns needed */
- uint32_t idx_READ;
- uint32_t idx_QUALITY;
-
- uint32_t idx_TRIM_START;
- uint32_t idx_TRIM_LEN;
-
- uint32_t idx_READ_START;
- uint32_t idx_READ_LEN;
- uint32_t idx_READ_FILTER;
- uint32_t idx_READ_TYPE;
-
- /* the data to be read */
- char_buffer bases;
- char_buffer filtered_bases;
- char_buffer quality;
- char_buffer filtered_quality;
- uint32_t trim_start;
- uint32_t trim_end;
-
- int_buffer read_start;
- int_buffer read_len;
- int_buffer read_filter;
- int_buffer read_type;
-
- /* flags, how the read is to be processed */
- bool bio;
- bool trim;
- bool cut;
-
-} rd_filter;
-typedef rd_filter* p_rd_filter;
-
-
-rc_t rd_filter_init( p_rd_filter self );
-
-void rd_filter_destroy( p_rd_filter self );
-
-rc_t rd_filter_pre_open( p_rd_filter self, const VCursor * cur );
-
-rc_t rd_filter_read( p_rd_filter self, const VCursor * cur );
-
-rc_t rd_filter_set_flags( p_rd_filter self, bool bio, bool trim, bool cut );
-
-uint32_t filter_get_read_len( p_rd_filter self );
-
-char filter_get_base( p_rd_filter self, const uint32_t idx );
-
-uint8_t filter_get_quality( p_rd_filter self, const uint32_t idx );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/run-stat/run-stat.c b/tools/run-stat/run-stat.c
deleted file mode 100644
index 0a9c882..0000000
--- a/tools/run-stat/run-stat.c
+++ /dev/null
@@ -1,703 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "run-stat.vers.h"
-
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/table.h>
-#include <vdb/cursor.h>
-#include <vdb/database.h>
-#include <kfs/directory.h>
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <klib/container.h>
-#include <klib/vector.h>
-#include <klib/log.h>
-#include <klib/out.h>
-#include <klib/debug.h>
-#include <klib/status.h>
-#include <klib/text.h>
-#include <klib/rc.h>
-#include <klib/namelist.h>
-#include <sra/srapath.h>
-
-#include <os-native.h>
-#include <sysalloc.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <bitstr.h>
-
-#include "context.h"
-#include "helper.h"
-#include "progressbar.h"
-#include "mod_cmn.h"
-
-static const char * table_usage[] = { "table-name (if src is a database)", NULL };
-static const char * rows_usage[] = { "rows (default = all)", NULL };
-static const char * schema_usage[] = { "schema to be used on sourc-table", NULL };
-static const char * without_accession_usage[] = { "without accession-test", NULL };
-static const char * progress_usage[] = { "show a progress-bar (default: no)", NULL };
-static const char * module_usage[] = { "selects a statistic-module (default all build-in)", NULL };
-static const char * grafic_usage[] = { "enables output of grafic-files", NULL };
-static const char * report_usage[] = { "output-format of the report-files:",
- "txt..text (default), cvs..comma-sep.-format",
- "xml..xml-format, json...json-format",
- NULL };
-static const char * output_usage[] = { "location of report/grafics",
- "default = current directory",
- NULL };
-
-static const char * prefix_usage[] = { "name-prefix for generated files",
- "default = 'report'",
- NULL };
-
-OptDef StatOptions[] =
-{
- { OPTION_TABLE, ALIAS_TABLE, NULL, table_usage, 1, true, false },
- { OPTION_ROWS, ALIAS_ROWS, NULL, rows_usage, 1, true, false },
- { OPTION_SCHEMA, ALIAS_SCHEMA, NULL, schema_usage, 5, true, false },
- { OPTION_PROGRESS, ALIAS_PROGRESS, NULL, progress_usage, 1, false, false },
- { OPTION_MODULE, ALIAS_MODULE, NULL, module_usage, 0, true, false },
- { OPTION_WITHOUT_ACCESSION, ALIAS_WITHOUT_ACCESSION, NULL, without_accession_usage, 1, false, false },
- { OPTION_GRAFIC, ALIAS_GRAFIC, NULL, grafic_usage, 1, false, false },
- { OPTION_REPORT, ALIAS_RREPORT, NULL, report_usage, 1, true, false },
- { OPTION_OUTPUT, ALIAS_OUTPUT, NULL, output_usage, 1, true, false },
- { OPTION_PREFIX, ALIAS_PREFIX, NULL, prefix_usage, 1, true, false }
-};
-
-
-const char UsageDefaultName[] = "run-stat";
-
-rc_t CC UsageSummary ( const char * progname )
-{
- return KOutMsg ("\n"
- "Usage:\n"
- " %s <path> [options]\n"
- "\n", progname);
-}
-
-rc_t CC Usage (const Args * args)
-{
- const char * progname;
- const char * fullpath;
- rc_t rc;
-
- if (args == NULL)
- rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
- else
- rc = ArgsProgram ( args, &fullpath, &progname );
- if ( rc )
- progname = fullpath = UsageDefaultName;
-
- UsageSummary (progname);
-
- KOutMsg (( "Options:\n" ));
- HelpOptionLine ( ALIAS_TABLE, OPTION_TABLE, "table", table_usage );
- HelpOptionLine ( ALIAS_ROWS, OPTION_ROWS, "rows", rows_usage );
- HelpOptionLine ( ALIAS_SCHEMA, OPTION_SCHEMA, "schema", schema_usage );
- HelpOptionLine ( ALIAS_PROGRESS, OPTION_PROGRESS, "progress", progress_usage );
- HelpOptionLine ( ALIAS_MODULE, OPTION_MODULE, "module", module_usage );
- HelpOptionLine ( ALIAS_WITHOUT_ACCESSION, OPTION_WITHOUT_ACCESSION, NULL, without_accession_usage );
- HelpOptionLine ( ALIAS_GRAFIC, OPTION_GRAFIC, NULL, grafic_usage );
- HelpOptionLine ( ALIAS_REPORT, OPTION_REPORT, NULL, report_usage );
- HelpOptionLine ( ALIAS_OUTPUT, OPTION_OUTPUT, NULL, output_usage );
- HelpOptionLine ( ALIAS_PREFIX, OPTION_PREFIX, NULL, prefix_usage );
-
- HelpOptionsStandard ();
-
- HelpVersion (fullpath, KAppVersion());
-
- return rc;
-}
-
-
-/* Version EXTERN
- * return 4-part version code: 0xMMmmrrrr, where
- * MM = major release
- * mm = minor release
- * rrrr = bug-fix release
- */
-ver_t CC KAppVersion ( void )
-{
- return RUN_STAT_VERS;
-}
-
-
-#if TOOLS_USE_SRAPATH != 0
-static char *translate_accession( SRAPath *my_sra_path,
- const char *accession,
- const size_t bufsize )
-{
- char *res = malloc( bufsize );
- if ( res != NULL )
- {
- rc_t rc = SRAPathFind( my_sra_path, accession, res, bufsize );
- if ( GetRCState( rc ) == rcNotFound )
- {
- free( res );
- res = NULL;
- }
- else if ( GetRCState( rc ) == rcInsufficient )
- {
- DBGMSG ( DBG_APP, 0, ( "bufsize %lu was insufficient\n", bufsize ) );
- free( res );
- res = translate_accession( my_sra_path, accession, bufsize * 2 );
- }
- else if ( rc != 0 )
- {
- free( res );
- res = NULL;
- }
- }
- return res;
-}
-#endif
-
-
-#if TOOLS_USE_SRAPATH != 0
-static rc_t check_accession( p_stat_ctx ctx, const KDirectory *my_dir )
-{
- rc_t rc = 0;
- if ( strchr ( ctx->path, '/' ) == NULL )
- {
- SRAPath *my_sra_path;
- rc = SRAPathMake( &my_sra_path, my_dir );
- if ( rc != 0 )
- {
- if ( GetRCState ( rc ) != rcNotFound || GetRCTarget ( rc ) != rcDylib )
- LOGERR( klogInt, rc, "SRAPathMake() failed" );
- else
- rc = 0;
- }
- else
- {
- if ( !SRAPathTest( my_sra_path, ctx->path ) )
- {
- char *buf = translate_accession( my_sra_path, ctx->path, 64 );
- if ( buf != NULL )
- {
- DBGMSG ( DBG_APP, 0, ( "sra-accession found! >%s<\n", buf ) );
- free( (char*)ctx->path );
- ctx->path = buf;
- }
- }
- SRAPathRelease( my_sra_path );
- }
- }
- return rc;
-}
-#endif
-
-
-static void row_error( const char * fmt, rc_t rc, uint64_t row_id )
-{
- PLOGERR( klogInt, ( klogInt, rc, fmt, "row_nr=%lu", row_id ) );
-}
-
-
-static rc_t run_stat_rows( const VCursor * cur,
- p_mod_list list,
- p_ng row_generator,
- bool show_progress )
-{
- rc_t rc = 0;
- uint64_t row_id, requested;
- uint64_t processed = 0;
- progressbar *pb = NULL;
- uint8_t digits = 0;
-
- requested = ng_count( row_generator );
- OUTMSG(( "we will read: %lu rows\n", requested ));
-
- if ( show_progress )
- {
- rc = make_progressbar( &pb );
- DISP_RC( rc, "run_stat_rows:make_progressbar() failed" );
- if ( rc == 0 )
- digits = calc_progressbar_digits( requested );
- }
- if ( rc == 0 )
- {
- ng_start( row_generator );
- while ( ( ng_next( row_generator, &row_id ) )&&( rc == 0 ) )
- {
- rc = Quitting();
- if ( rc == 0 )
- {
- rc = VCursorSetRowId( cur, row_id );
- if ( rc != 0 )
- row_error( "VCursorSetRowId( row#$(row_nr) ) failed", rc, row_id );
- else
- {
- rc = VCursorOpenRow( cur );
- if ( rc != 0 )
- row_error( "VCursorOpenRow( row#$(row_nr) ) failed", rc, row_id );
- else
- {
- /*****************************************************/
- /* pass every row on to the list of modules */
- rc_t rc1 = mod_list_row( list, cur );
- /*****************************************************/
- if ( rc1 != 0 )
- row_error( "stat_module_row( row#$(row_nr) ) failed", rc1, row_id );
- else
- {
- if ( pb != NULL )
- {
- uint16_t percent = percent_progressbar( requested,
- ++processed,
- digits );
- update_progressbar( pb, digits, percent );
- }
- }
- rc = VCursorCloseRow( cur );
- if ( rc != 0 )
- row_error( "VCursorCloseRow( row#$(row_nr) ) failed", rc, row_id );
- else
- rc = rc1;
- }
- }
- }
- }
- if ( pb != 0 )
- {
- rc_t rc1 = destroy_progressbar( pb );
- DISP_RC( rc1, "run_stat_rows:destroy_progressbar() failed" );
- }
- }
- if ( show_progress )
- OUTMSG(( "\n" ));
- return rc;
-}
-
-
-static rc_t run_stat_check_range( p_ng row_generator, const VCursor * cur )
-{
- int64_t first;
- uint64_t count;
- rc_t rc = VCursorIdRange( cur, 0, &first, &count );
- DISP_RC( rc, "run_stat_check_range:VCursorIdRange() failed" );
- if ( rc == 0 )
- {
- /* if the user did not specify a row-range, take all rows */
- if ( ng_range_defined( row_generator ) == false )
- {
- ng_set_range( row_generator, first, count );
- }
- /* if the user did specify a row-range, check the boundaries */
- else
- {
- ng_check_range( row_generator, first, count );
- }
-
- if ( !ng_range_defined( row_generator ) )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
- DISP_RC( rc, "run_stat_check_range:ng_range_defined() failed" );
- }
-
- }
- return rc;
-}
-
-
-static rc_t run_stat_tab( p_stat_ctx ctx,
- const VTable * tab,
- p_mod_list m_list )
-{
- const VCursor *cur;
- rc_t rc = VTableCreateCursorRead( tab, &cur );
- DISP_RC( rc, "run_stat_tab:VTableCreateCursorRead() failed" );
- if ( rc == 0 )
- {
- rc = mod_list_pre_open( m_list, cur );
- if ( rc == 0 )
- {
- rc = VCursorOpen( cur );
- DISP_RC( rc, "run_stat_tab:VCursorOpen() failed" );
- if ( rc == 0 )
- {
- rc = run_stat_check_range( ctx->row_generator, cur );
- if ( rc == 0 )
- {
- /***************************************/
- rc = run_stat_rows( cur,
- m_list,
- ctx->row_generator,
- ctx->show_progress );
-
- if ( rc == 0 )
- rc = mod_list_post_rows( m_list );
- /***************************************/
- }
- }
- }
- {
- rc_t rc1 = VCursorRelease( cur );
- DISP_RC( rc1, "run_stat_tab:VCursorRelease() failed" );
- }
- }
- return 0;
-}
-
-
-static rc_t run_stat_db( p_stat_ctx ctx,
- const VDatabase * db,
- p_mod_list m_list )
-{
- rc_t rc = 0;
- bool table_defined = ( ctx->table != NULL );
- if ( !table_defined )
- {
- table_defined = helper_take_this_table_from_db( db, ctx, "SEQUENCE" );
- if ( !table_defined )
- table_defined = helper_take_1st_table_from_db( db, ctx );
- }
- if ( table_defined )
- {
- const VTable *tab;
- rc = VDatabaseOpenTableRead( db, &tab, "%s", ctx->table );
- DISP_RC( rc, "run_stat_db:VDatabaseOpenTableRead() failed" );
- if ( rc == 0 )
- {
- /************************************/
- rc = run_stat_tab( ctx, tab, m_list );
- /************************************/
- {
- rc_t rc1 = VTableRelease( tab );
- DISP_RC( rc1, "run_stat_db:VTableRelease() failed" );
- }
- }
- }
- else
- {
- rc = RC( rcVDB, rcNoTarg, rcCopying, rcItem, rcNotFound );
- LOGMSG( klogInfo, "opened as vdb-dabase, but no table found" );
- }
- return rc;
-}
-
-
-static rc_t run_stat_collect_data( p_stat_ctx ctx,
- const VDBManager * mgr,
- p_mod_list m_list )
-{
- VSchema * dflt_schema;
- rc_t rc = helper_parse_schema( mgr, &dflt_schema, ctx->schema_list );
- DISP_RC( rc, "run_stat_collect_data:helper_parse_schema() failed" );
- if ( rc == 0 )
- {
- const VDatabase * db;
- /* try to open it as a database */
- rc = VDBManagerOpenDBRead ( mgr, &db, dflt_schema, "%s", ctx->path );
- if ( rc == 0 )
- {
- /* if it succeeds it is a database, continue ... */
- /**********************************/
- rc = run_stat_db( ctx, db, m_list );
- /**********************************/
- VDatabaseRelease( db );
- }
- else
- {
- const VTable * tab;
- /* try to open it as a table */
- rc = VDBManagerOpenTableRead( mgr, &tab, dflt_schema, "%s", ctx->path );
- /* if it succeeds it is a table, continue ... */
- if ( rc == 0 )
- {
- /************************************/
- rc = run_stat_tab( ctx, tab, m_list );
- /************************************/
- VTableRelease( tab );
- }
- else
- {
- rc = RC( rcVDB, rcNoTarg, rcCopying, rcItem, rcNotFound );
- PLOGERR( klogInt, ( klogInt, rc,
- "\nthe path '$(path)' cannot be opened as vdb-database or vdb-table",
- "path=%s", ctx->path ));
- }
- }
- {
- rc_t rc1 = VSchemaRelease( dflt_schema );
- DISP_RC( rc1, "run_stat_perform:VSchemaRelease(dflt) failed" );
- }
- }
- return rc;
-}
-
-static const char * ext_txt = "txt";
-static const char * ext_csv = "csv";
-static const char * ext_xml = "xml";
-static const char * ext_jso = "json";
-
-static const char * run_stat_ext( const uint32_t report_type )
-{
- const char * res = ext_txt;
- switch( report_type )
- {
- case RT_CSV : res = ext_csv; break;
- case RT_XML : res = ext_xml; break;
- case RT_JSO : res = ext_jso; break;
- }
- return res;
-}
-
-static rc_t run_stat_assemble_filename( p_char_buffer buf, p_stat_ctx ctx,
- const char * module_name,
- const char * report_name,
- const char * ext )
-{
- rc_t rc;
- size_t size = string_size( ctx->output_path );
- size += string_size( ctx->name_prefix );
- size += string_size( module_name );
- size += string_size( report_name );
- size += string_size( ext );
- size += 6;
-
- if ( ctx->output_path == NULL )
- rc = char_buffer_printf( buf, size,
- "%s_%s_%s.%s", ctx->name_prefix, module_name, report_name, ext );
- else
- rc = char_buffer_printf( buf, size,
- "%s/%s_%s_%s.%s", ctx->output_path, ctx->name_prefix, module_name,
- report_name, ext );
- return rc;
-}
-
-
-static rc_t run_stat_module_sub_report( p_stat_ctx ctx, p_mod_list m_list,
- const char * module_name, uint32_t m_idx, uint32_t r_idx, p_char_buffer buffer )
-{
- char * report_name;
- rc_t rc = mod_list_subreport_name( m_list, m_idx, r_idx, &report_name );
- if ( rc == 0 )
- {
- buffer->len = 0;
- rc = mod_list_subreport( m_list, m_idx, r_idx, buffer, ctx->report_type );
- if ( rc == 0 )
- {
- char_buffer fn;
- rc = char_buffer_init( &fn, 1024 );
- if ( rc == 0 )
- {
- const char * ext = run_stat_ext( ctx->report_type );
- rc = run_stat_assemble_filename( &fn, ctx,
- module_name, report_name, ext );
- if ( rc == 0 )
- {
- if ( buffer->len > 0 )
- {
- rc = char_buffer_saveto( buffer, fn.ptr );
- if ( rc == 0 )
- OUTMSG(( "written: %s\n", fn.ptr ));
- }
- fn.len = 0;
- }
-
- if ( ctx->produce_grafic )
- {
- rc = run_stat_assemble_filename( &fn, ctx,
- module_name, report_name, "svg" );
- if ( rc == 0 )
- {
- rc_t rc1 = mod_list_graph( m_list, m_idx, r_idx, fn.ptr );
- if ( rc1 == 0 )
- OUTMSG(( "written: %s\n", fn.ptr ));
- }
- }
- char_buffer_destroy( &fn );
- }
- }
- free( report_name );
- }
- return rc;
-}
-
-
-static rc_t run_stat_module_report( p_stat_ctx ctx, p_mod_list m_list,
- uint32_t m_idx, p_char_buffer buffer )
-{
- char * module_name;
- rc_t rc = mod_list_name( m_list, m_idx, &module_name );
- if ( rc == 0 )
- {
- uint32_t r_count;
- rc = mod_list_subreport_count( m_list, m_idx, &r_count );
- if ( rc == 0 )
- {
- uint32_t r_idx;
- for ( r_idx = 0; r_idx < r_count && rc == 0; ++r_idx )
- {
- rc = run_stat_module_sub_report( ctx, m_list,
- module_name, m_idx, r_idx, buffer );
- }
- }
- free( module_name );
- }
- return rc;
-}
-
-
-static rc_t run_stat_report( p_stat_ctx ctx, p_mod_list m_list )
-{
- char_buffer buffer;
- rc_t rc = char_buffer_init( &buffer, 4096 ); /* preallocate */
- DISP_RC( rc, "run_stat_report:char_buffer_init(finalized) failed" );
- if ( rc == 0 )
- {
- uint32_t m_count;
- rc = mod_list_count( m_list, &m_count );
- if ( rc == 0 )
- {
- uint32_t m_idx;
- for ( m_idx = 0; m_idx < m_count && rc == 0; ++m_idx )
- {
- rc = run_stat_module_report( ctx, m_list, m_idx, &buffer );
- }
- }
- /* free the collected output... */
- char_buffer_destroy( &buffer );
- }
- return rc;
-}
-
-
-static rc_t run_stat_main( p_stat_ctx ctx )
-{
- KDirectory *dir;
- rc_t rc = KDirectoryNativeDir( &dir );
- DISP_RC( rc, "run_stat_main:KDirectoryNativeDir() failed" );
- if ( rc == 0 )
- {
- const VDBManager *mgr;
-
-#if TOOLS_USE_SRAPATH != 0
- if ( ctx->dont_check_accession == false )
- {
- rc_t rc1 = check_accession( ctx, dir );
- DISP_RC( rc1, "run_stat_main:check_accession() failed" );
- }
-#endif
-
- rc = VDBManagerMakeRead ( &mgr, dir );
- DISP_RC( rc, "run_stat_main:VDBManagerMakeRead() failed" );
- if ( rc == 0 )
- {
- p_mod_list m_list;
- rc = mod_list_init( &m_list, ctx->module_list );
- if ( rc == 0 )
- {
- /********************************/
- rc = run_stat_collect_data( ctx, mgr, m_list );
- /********************************/
-
- if ( rc == 0 )
- rc = run_stat_report( ctx, m_list );
-
- mod_list_destroy( m_list );
- }
- {
- rc_t rc1 = VDBManagerRelease( mgr );
- DISP_RC( rc1, "run_stat_main:VDBManagerRelease() failed" );
- }
- }
- {
- rc_t rc1 = KDirectoryRelease( dir );
- DISP_RC( rc1, "run_stat_main:KDirectoryRelease() failed" );
- }
- }
- return rc;
-}
-
-
-/* for KOutHandlerSet... */
-static
-rc_t CC write_to_FILE ( void *f, const char *buffer, size_t bytes, size_t *num_writ )
-{
- * num_writ = fwrite ( buffer, 1, bytes, f );
- if ( * num_writ != bytes )
- return RC ( rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete );
- return 0;
-}
-
-
-rc_t CC KMain ( int argc, char *argv [] )
-{
- Args * args;
-
- rc_t rc = KOutHandlerSet ( write_to_FILE, stdout );
- DISP_RC( rc, "KMain:KOutHandlerSet() failed" );
- if ( rc == 0 )
- rc = ArgsMakeAndHandle ( &args, argc, argv, 1,
- StatOptions, sizeof StatOptions / sizeof StatOptions [ 0 ] );
- if ( rc == 0 )
- {
- stat_ctx *ctx;
-
- rc = ctx_init( &ctx );
- DISP_RC( rc, "KMain:ctx_init() failed" );
-
- if ( rc == 0 )
- {
- rc = ctx_capture_arguments_and_options( args, ctx );
- DISP_RC( rc, "KMain:ctx_capture_arguments_and_options() failed" );
- if ( rc == 0 )
- {
- if ( ctx->usage_requested )
- {
- MiniUsage(args);
- }
- else
- {
- KLogHandlerSetStdErr();
- if ( ctx->module_list == NULL )
- {
- helper_make_namelist_from_string( &(ctx->module_list),
- "bases,qualities", ',' );
- }
- if ( rc == 0 )
- {
- /************************/
- rc = run_stat_main( ctx );
- /************************/
- }
- }
- }
- ctx_destroy( ctx );
- }
- ArgsWhack (args);
- }
-
- return rc;
-}
-
diff --git a/tools/run-stat/run-stat.vers b/tools/run-stat/run-stat.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/run-stat/run-stat.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/run-stat/svg.c b/tools/run-stat/svg.c
deleted file mode 100644
index 0b1cc0d..0000000
--- a/tools/run-stat/svg.c
+++ /dev/null
@@ -1,753 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <klib/log.h>
-#include <klib/text.h>
-#include <klib/printf.h>
-#include <klib/out.h>
-#include <kfs/directory.h>
-#include <kfs/file.h>
-
-#include <os-native.h>
-
-#include <sysalloc.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-
-#include "svg.h"
-
-rc_t svg_path_init( p_svg_path path )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( path != NULL )
- rc = char_buffer_init( &path->buf, 250 );
- return rc;
-}
-
-void svg_path_destroy( p_svg_path path )
-{
- if ( path != NULL )
- char_buffer_destroy( &path->buf );
-}
-
-
-rc_t svg_path_close( p_svg_path path )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( path != NULL )
- rc = char_buffer_printf( &path->buf, 2, "Z " );
- return rc;
-}
-
-
-static rc_t svg_path_2_to( p_svg_path path, const char c, int32_t x, int32_t y )
-{
- if ( path != NULL )
- return char_buffer_printf( &path->buf, 25, "%c%d %d ", c, x, y );
- else
- return RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
-}
-
-
-rc_t svg_path_move_to( p_svg_path path, bool absolute, int32_t x, int32_t y )
-{
- return svg_path_2_to( path, absolute ? 'M' : 'm', x, y );
-}
-
-
-rc_t svg_path_line_to( p_svg_path path, bool absolute, int32_t x, int32_t y )
-{
- return svg_path_2_to( path, absolute ? 'L' : 'l', x, y );
-}
-
-
-static rc_t svg_path_1_to( p_svg_path path, const char c, int32_t x )
-{
- if ( path != NULL )
- return char_buffer_printf( &path->buf, 25, "%c%d ", c, x );
- else
- return RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
-}
-
-
-rc_t svg_path_hline_to( p_svg_path path, bool absolute, int32_t x )
-{
- return svg_path_1_to( path, absolute ? 'H' : 'h', x );
-}
-
-
-rc_t svg_path_vline_to( p_svg_path path, bool absolute, int32_t y )
-{
- return svg_path_1_to( path, absolute ? 'V' : 'v', y );
-}
-
-
-rc_t svg_path_smooth_to( p_svg_path path, bool absolute, int32_t x, int32_t y )
-{
- return svg_path_2_to( path, absolute ? 'T' : 't', x, y );
-}
-
-
-static rc_t svg_path_4_to( p_svg_path path, const char c, int32_t x1, int32_t y1,
- int32_t x, int32_t y )
-{
- if ( path != NULL )
- return char_buffer_printf( &path->buf, 50, "%c%d,%d %d,%d ", c, x1, y1, x, y );
- else
- return RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
-}
-
-/* cubic bezier curve (x1,y1 ... control point / x,y ... endpoint ) */
-rc_t svg_path_bez3_to( p_svg_path path, bool absolute,
- int32_t x1, int32_t y1, int32_t x, int32_t y )
-{
- return svg_path_4_to( path, absolute ? 'S' : 's', x1, y1, x, y );
-}
-
-/* quadratic bezier curve (x1,y1 ... control point / x,y ... endpoint ) */
-rc_t svg_path_bez2_to( p_svg_path path, bool absolute,
- int32_t x1, int32_t y1, int32_t x, int32_t y )
-{
- return svg_path_4_to( path, absolute ? 'Q' : 'q', x1, y1, x, y );
-}
-
-
-/* quadratic bezier curve (x1,y1 x2,y2 ... control point / x,y ... endpoint ) */
-rc_t svg_path_curve_to( p_svg_path path, bool absolute,
- int32_t x1, int32_t y1, int32_t x2, int32_t y2, int32_t x, int32_t y )
-{
- if ( path != NULL )
- {
- char c = absolute ? 'C' : 'c';
- return char_buffer_printf( &path->buf, 75, "%c%d,%d %d,%d %d,%d ",
- c, x1, y1, x2, y2, x, y );
- }
- else
- return RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
-}
-
-
-/* arc ( rx, ry ....... elliptical radius in x/y-direction
- rotx ....... rotation around x-axis
- large-flag ... choose the large section of the arc
- sweep-flag ... draw in positive-angle-direction
- x,y .......... endpoint ) */
-rc_t svg_path_arc( p_svg_path path, bool absolute,
- int32_t rx, int32_t ry, int32_t rot_x,
- bool large_flag, bool sweep_flag,
- int32_t x, int32_t y )
-{
- if ( path != NULL )
- {
- char c = absolute ? 'A' : 'a';
- int32_t lf = large_flag ? 1 : 0;
- int32_t sf = sweep_flag ? 1 : 0;
- return char_buffer_printf( &path->buf, 75, "%c%d,%d %d %d,%d %d,%d ",
- c, rx, ry, rot_x, lf, sf, x, y );
- }
- else
- return RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
-}
-
-typedef struct svg_write_ctx
-{
- KFile * f;
- uint64_t pos;
-} svg_write_ctx;
-typedef svg_write_ctx* p_svg_write_ctx;
-
-
-static rc_t svg_write_txt( p_svg_write_ctx ctx, const char * s )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( ctx != NULL && s != NULL )
- {
- size_t num_writ;
- size_t len = string_size( s );
- rc = KFileWrite ( ctx->f, ctx->pos, s, len, &num_writ );
- if ( rc == 0 )
- ctx->pos += num_writ;
- }
- return rc;
-}
-
-typedef struct svg_style
-{
- char * name;
- char * style;
-} svg_style;
-typedef svg_style* p_svg_style;
-
-
-static p_svg_style svg_make_style( const char * name, const char * style )
-{
- p_svg_style res = calloc( 1, sizeof( struct svg_style ) );
- if ( res != NULL )
- {
- res->name = string_dup_measure( name, NULL );
- res->style = string_dup_measure( style, NULL );
- }
- return res;
-}
-
-
-static void CC svg_style_whack( void *item, void *data )
-{
- p_svg_style style = item;
- if ( style != NULL )
- {
- if ( style->name != NULL ) free( style->name );
- if ( style->style != NULL ) free( style->style );
- free( style );
- }
-}
-
-static const char * svg_style_fmt = "%s{ %s }\n";
-static void CC svg_write_style( void *item, void * data )
-{
- p_svg_style style = item;
- p_svg_write_ctx ctx = data;
- if ( style != NULL && ctx != NULL )
- {
- char temp[ 240 ];
- rc_t rc = string_printf( temp, sizeof( temp ), NULL, svg_style_fmt,
- style->name, style->style );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- }
-}
-
-
-static const char * svg_style_prefix = "<style type=\"text/css\"><![CDATA[\n";
-static const char * svg_style_postfix = "]]></style>\n\n";
-
-static rc_t svg_write_styles( Vector * styles, p_svg_write_ctx ctx )
-{
- rc_t rc = svg_write_txt( ctx, svg_style_prefix );
- if ( rc == 0 )
- {
- VectorForEach ( styles, false, svg_write_style, ctx );
- rc = svg_write_txt( ctx, svg_style_postfix );
- }
- return rc;
-}
-
-
-static rc_t svg_add_style( Vector * styles, p_svg_style style )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( styles != NULL && style != NULL )
- {
- rc = VectorAppend ( styles, NULL, style );
- if ( rc != 0 )
- svg_style_whack( style, NULL );
- }
- return rc;
-}
-
-struct svg_elem;
-
-/* the type of the callback-function to be performed on every row */
-typedef void ( CC * svg_elem_wr ) ( struct svg_elem * elem, p_svg_write_ctx ctx );
-
-typedef struct svg_elem
-{
- svg_elem_wr writer;
- uint32_t * values;
- uint32_t n_values;
- char * txt;
- char * style;
-} svg_elem;
-typedef svg_elem* p_svg_elem;
-
-
-static p_svg_elem svg_make_elem( const char * style,
- svg_elem_wr writer,
- const uint32_t * values,
- uint32_t n_values )
-{
- p_svg_elem res = calloc( 1, sizeof( struct svg_elem ) );
- if ( res != NULL )
- {
- if ( n_values > 0 )
- {
- size_t size = ( sizeof ( res->values[ 0 ] ) * n_values );
- res->values = malloc( size );
- if ( res->values != NULL )
- {
- memcpy( res->values, values, size );
- res->n_values = n_values;
- }
- }
- if ( style != NULL )
- res->style = string_dup_measure ( style, NULL );
- res->writer = writer;
- }
- return res;
-}
-
-
-static p_svg_elem svg_make_elemv( const char * style,
- svg_elem_wr writer,
- uint32_t n_values, ... )
-{
- p_svg_elem res = calloc( 1, sizeof( struct svg_elem ) );
- va_list v;
- va_start( v, n_values );
- if ( res != NULL )
- {
- size_t size = ( sizeof ( res->values[ 0 ] ) * n_values );
- res->values = malloc( size );
- if ( res->values != NULL )
- {
- int32_t i;
- for ( i = 0; i < n_values; ++i )
- res->values[ i ] = va_arg( v, uint32_t );
- res->n_values = n_values;
- }
- res->style = string_dup_measure ( style, NULL );
- res->writer = writer;
- }
- va_end( v );
- return res;
-}
-
-
-static void CC svg_elem_whack( void *item, void *data )
-{
- p_svg_elem elem = item;
- if ( elem )
- {
- if ( elem->values != NULL ) free( elem->values );
- if ( elem->style != NULL ) free( elem->style );
- if ( elem->txt != NULL ) free( elem->txt );
- free( elem );
- }
-}
-
-
-static rc_t svg_add_elem( Vector * elements, p_svg_elem elem )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( elements != NULL && elem != NULL )
- {
- rc = VectorAppend ( elements, NULL, elem );
- if ( rc != 0 )
- svg_elem_whack( elem, NULL );
- }
- return rc;
-}
-
-
-static const char * svg_circle_fmt_c =
- "<circle cx=\"%d\" cy=\"%d\" r=\"%d\" class=\"%s\"/>\n\n";
-static const char * svg_circle_fmt =
- "<circle cx=\"%d\" cy=\"%d\" r=\"%d\"/>\n\n";
-
-
-static void svg_elem_wr_circle( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL &&
- elem->values != NULL && elem->n_values > 2 )
- {
- char temp[ 120 ];
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, sizeof( temp ), NULL, svg_circle_fmt_c,
- elem->values[0], elem->values[1], elem->values[2], elem->style );
- else
- rc = string_printf( temp, sizeof( temp ), NULL, svg_circle_fmt,
- elem->values[0], elem->values[1], elem->values[2] );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- }
-}
-
-
-static const char * svg_rect_fmt_c =
- "<rect x=\"%d\" y=\"%d\" width=\"%d\" height=\"%d\" class=\"%s\"/>\n\n";
-static const char * svg_rect_fmt =
- "<rect x=\"%d\" y=\"%d\" width=\"%d\" height=\"%d\"/>\n\n";
-
-static void svg_elem_wr_rect( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL &&
- elem->values != NULL && elem->n_values > 3 )
- {
- char temp[ 120 ];
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, sizeof( temp ), NULL, svg_rect_fmt_c,
- elem->values[0], elem->values[1], elem->values[2],
- elem->values[3], elem->style );
- else
- rc = string_printf( temp, sizeof( temp ), NULL, svg_rect_fmt,
- elem->values[0], elem->values[1], elem->values[2],
- elem->values[3] );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- }
-}
-
-
-static const char * svg_ell_fmt_c =
- "<ellipse cx=\"%d\" cy=\"%d\" rx=\"%d\" ry=\"%d\" class=\"%s\"/>\n\n";
-static const char * svg_ell_fmt =
- "<ellipse cx=\"%d\" cy=\"%d\" rx=\"%d\" ry=\"%d\"/>\n\n";
-
-
-static void svg_elem_wr_ellipse( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL &&
- elem->values != NULL && elem->n_values > 3 )
- {
- char temp[ 120 ];
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, sizeof( temp ), NULL, svg_ell_fmt_c,
- elem->values[0], elem->values[1], elem->values[2],
- elem->values[3], elem->style );
- rc = string_printf( temp, sizeof( temp ), NULL, svg_ell_fmt,
- elem->values[0], elem->values[1], elem->values[2],
- elem->values[3] );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- }
-}
-
-
-static const char * svg_line_fmt_c =
- "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"%s\"/>\n\n";
-static const char * svg_line_fmt =
- "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\"/>\n\n";
-
-static void svg_elem_wr_line( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL &&
- elem->values != NULL && elem->n_values > 3 )
- {
- char temp[ 120 ];
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, sizeof( temp ), NULL, svg_line_fmt_c,
- elem->values[0], elem->values[1], elem->values[2],
- elem->values[3], elem->style );
- else
- rc = string_printf( temp, sizeof( temp ), NULL, svg_line_fmt,
- elem->values[0], elem->values[1], elem->values[2],
- elem->values[3] );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- }
-}
-
-
-static const char * svg_point_fmt ="%d,%d ";
-
-static char * svg_make_points_str( const uint32_t * values,
- const uint32_t n_values )
-{
- size_t size = ( n_values * 10 );
- char * res = malloc( size );
- if ( res )
- {
- rc_t rc = 0;
- size_t ofs = 0;
- uint32_t i, n = ( n_values >> 1 );
- for ( i = 0; i < n && rc == 0; ++i )
- {
- size_t written;
- rc = string_printf( &res[ofs], size-ofs, &written, svg_point_fmt,
- values[ i*2 ], values[ i*2 + 1 ] );
- if ( rc == 0 )
- ofs+=written;
- }
- }
- return res;
-}
-
-static const char * svg_pline_fmt_c =
- "<polyline points=\"%s\" class=\"%s\"/>\n\n";
-static const char * svg_pline_fmt =
- "<polyline points=\"%s\"/>\n\n";
-
-static void svg_elem_wr_polyline( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL &&
- elem->values != NULL && elem->n_values > 1 )
- {
- char * point_str = svg_make_points_str( elem->values, elem->n_values );
- if ( point_str != NULL )
- {
- size_t size = string_size( point_str ) + string_size( elem->style ) + 40;
- char * temp = malloc( size );
- if ( temp != NULL )
- {
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, size, NULL, svg_pline_fmt_c,
- point_str, elem->style );
- else
- rc = string_printf( temp, size, NULL, svg_pline_fmt,
- point_str );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- free( temp );
- }
- free( point_str );
- }
- }
-}
-
-
-static const char * svg_text_fmt_c =
- "<text x=\"%d\" y=\"%d\" class=\"%s\">%s</text>\n\n";
-static const char * svg_text_fmt =
- "<text x=\"%d\" y=\"%d\">%s</text>\n\n";
-
-static void svg_elem_wr_text( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL &&
- elem->values != NULL && elem->n_values > 1 )
- {
- size_t size = string_size( elem->txt ) + string_size( elem->style ) + 50;
- char * temp = malloc( size );
- if ( temp != NULL )
- {
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, size, NULL, svg_text_fmt_c,
- elem->values[0], elem->values[1], elem->style, elem->txt );
- else
- rc = string_printf( temp, size, NULL, svg_text_fmt,
- elem->values[0], elem->values[1], elem->txt );
-
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- free( temp );
- }
- }
-}
-
-
-static const char * svg_path_fmt_c =
- "<path d=\"%s\" class=\"%s\"/>\n\n";
-static const char * svg_path_fmt =
- "<path d=\"%s\"/>\n\n";
-
-static void svg_elem_wr_path( p_svg_elem elem, p_svg_write_ctx ctx )
-{
- if ( elem != NULL && ctx != NULL && elem->txt != NULL )
- {
- size_t size = string_size( elem->txt ) + string_size( elem->style ) + 50;
- char * temp = malloc( size );
- if ( temp != NULL )
- {
- rc_t rc;
- if ( elem->style != NULL )
- rc = string_printf( temp, size, NULL, svg_path_fmt_c,
- elem->txt, elem->style );
- else
- rc = string_printf( temp, size, NULL, svg_path_fmt,
- elem->txt );
- if ( rc == 0 )
- svg_write_txt( ctx, temp );
- free( temp );
- }
- }
-}
-
-
-rc_t svg_init( p_svg grafic )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( grafic != NULL )
- {
- VectorInit ( &grafic->elements, 0, 20 );
- VectorInit ( &grafic->styles, 0, 5 );
- rc = 0;
- }
- return rc;
-}
-
-
-void svg_destroy( p_svg grafic )
-{
- if ( grafic != NULL )
- {
- VectorWhack ( &grafic->styles, svg_style_whack, NULL );
- VectorWhack ( &grafic->elements, svg_elem_whack, NULL );
- }
-}
-
-
-static rc_t svg_write_lines( p_svg_write_ctx ctx, const char ** lines )
-{
- rc_t rc = 0;
- const char **s = lines;
- while ( *s != NULL && rc == 0 )
- {
- rc = svg_write_txt( ctx, *s );
- s++;
- }
- return rc;
-}
-
-
-static const char * svg_prefix[] =
-{
- "<?xml version=\"1.0\" standalone=\"no\"?>\n\n",
- "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
- "\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n\n",
- "<svg width=\"100%\" height=\"100%\" version=\"1.1\"\n",
- "xmlns=\"http://www.w3.org/2000/svg\">\n\n",
- NULL
-};
-
-
-static void CC svg_write_elem( void *item, void * data )
-{
- p_svg_elem elem = item;
- p_svg_write_ctx ctx = data;
- if ( elem != NULL && ctx != NULL )
- {
- if ( elem->writer != NULL )
- elem->writer( elem, ctx );
- }
-}
-
-rc_t svg_write( p_svg grafic, const char * filename )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcParam, rcNull );
- if ( grafic != NULL && filename != NULL )
- {
- KDirectory * dir;
- rc = KDirectoryNativeDir ( &dir );
- if ( rc == 0 )
- {
- svg_write_ctx ctx;
- ctx.pos = 0;
- rc = KDirectoryCreateFile ( dir, &ctx.f, true, 0664, kcmInit, "%s", filename );
- if ( rc == 0 )
- {
- rc = svg_write_lines( &ctx, svg_prefix );
- if ( rc == 0 )
- {
- rc = svg_write_styles( &grafic->styles, &ctx );
- if ( rc == 0 )
- {
- VectorForEach ( &grafic->elements, false, svg_write_elem, &ctx );
- rc = svg_write_txt( &ctx, "</svg>\n" );
- }
- }
- KFileRelease( ctx.f );
- }
- KDirectoryRelease( dir );
- }
- }
- return rc;
-}
-
-
-rc_t svg_set_style( p_svg grafic, const char * name, const char * style )
-{
- return svg_add_style( &grafic->styles,
- svg_make_style( name, style ) );
-}
-
-
-rc_t svg_circle( p_svg grafic, uint32_t x, uint32_t y, uint32_t r,
- const char * style )
-{
- return svg_add_elem( &grafic->elements,
- svg_make_elemv( style, svg_elem_wr_circle,
- 3, x, y, r ) );
-}
-
-
-rc_t svg_rect( p_svg grafic, uint32_t x, uint32_t y, uint32_t dx, uint32_t dy,
- const char * style )
-{
- return svg_add_elem( &grafic->elements,
- svg_make_elemv( style, svg_elem_wr_rect,
- 4, x, y, dx, dy ) );
-}
-
-
-rc_t svg_ellipse( p_svg grafic, uint32_t x, uint32_t y, uint32_t rx, uint32_t ry,
- const char * style )
-{
- return svg_add_elem( &grafic->elements,
- svg_make_elemv( style, svg_elem_wr_ellipse,
- 4, x, y, rx, ry ) );
-}
-
-
-rc_t svg_line( p_svg grafic, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2,
- const char * style )
-{
- return svg_add_elem( &grafic->elements,
- svg_make_elemv( style, svg_elem_wr_line,
- 4, x1, y1, x2, y2 ) );
-}
-
-
-rc_t svg_polyline( p_svg grafic, const char * style,
- const uint32_t * values, const uint32_t n_values )
-{
- return svg_add_elem( &grafic->elements,
- svg_make_elem( style, svg_elem_wr_polyline,
- values, n_values ) );
-}
-
-
-rc_t svg_text( p_svg grafic, uint32_t x, uint32_t y, const char * txt,
- const char * style )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- p_svg_elem elem = svg_make_elemv( style, svg_elem_wr_text, 2, x, y );
- if ( elem != NULL )
- {
- elem->txt = string_dup_measure ( txt, NULL );
- rc = svg_add_elem( &grafic->elements, elem );
- }
- return rc;
-}
-
-
-rc_t svg_set_path( p_svg grafic, p_svg_path path, const char * style )
-{
- rc_t rc = RC( rcExe, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- p_svg_elem elem = svg_make_elem( style, svg_elem_wr_path, NULL, 0 );
- if ( elem != NULL )
- {
- elem->txt = string_dup_measure ( path->buf.ptr, NULL );
- rc = svg_add_elem( &grafic->elements, elem );
- }
- return rc;
-
-}
diff --git a/tools/run-stat/svg.h b/tools/run-stat/svg.h
deleted file mode 100644
index a34ce5b..0000000
--- a/tools/run-stat/svg.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_svg_
-#define _h_svg_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <klib/rc.h>
-#include <klib/vector.h>
-#include <helper.h>
-
-typedef struct svg_path
-{
- char_buffer buf;
-} svg_path;
-typedef svg_path* p_svg_path;
-
-rc_t svg_path_init( p_svg_path path );
-
-void svg_path_destroy( p_svg_path path );
-
-rc_t svg_path_close( p_svg_path path );
-rc_t svg_path_move_to( p_svg_path path, bool absolute, int32_t x, int32_t y );
-rc_t svg_path_line_to( p_svg_path path, bool absolute, int32_t x, int32_t y );
-rc_t svg_path_hline_to( p_svg_path path, bool absolute, int32_t x );
-rc_t svg_path_vline_to( p_svg_path path, bool absolute, int32_t y );
-rc_t svg_path_smooth_to( p_svg_path path, bool absolute, int32_t x, int32_t y );
-
-/* cubic bezier curve (x1,y1 ... control point / x,y ... endpoint )*/
-rc_t svg_path_bez3_to( p_svg_path path, bool absolute,
- int32_t x1, int32_t y1, int32_t x, int32_t y );
-
-/* quadratic bezier curve (x1,y1 ... control point / x,y ... endpoint )*/
-rc_t svg_path_bez2_to( p_svg_path path, bool absolute,
- int32_t x1, int32_t y1, int32_t x, int32_t y );
-
-/* quadratic bezier curve (x1,y1 x2,y2 ... control point / x,y ... endpoint )*/
-rc_t svg_path_curve_to( p_svg_path path, bool absolute,
- int32_t x1, int32_t y1, int32_t x2, int32_t y2, int32_t x, int32_t y );
-
-/* arc ( rx, ry ....... elliptical radius in x/y-direction
- rotx ....... rotation around x-axis
- large-flag ... choose the large section of the arc
- sweep-flag ... draw in positive-angle-direction
- x,y .......... endpoint ) */
-rc_t svg_path_arc( p_svg_path path, bool absolute,
- int32_t rx, int32_t ry, int32_t rot_x,
- bool large_flag, bool sweep_flag,
- int32_t x, int32_t y );
-
-
-typedef struct svg
-{
- Vector styles;
- Vector elements;
-} svg;
-typedef svg* p_svg;
-
-rc_t svg_init( p_svg grafic );
-
-void svg_destroy( p_svg grafic );
-
-rc_t svg_set_style( p_svg grafic, const char * name, const char * style );
-
-rc_t svg_circle( p_svg grafic, uint32_t x, uint32_t y, uint32_t r,
- const char * style );
-
-rc_t svg_rect( p_svg grafic, uint32_t x, uint32_t y, uint32_t dx, uint32_t dy,
- const char * style );
-
-rc_t svg_ellipse( p_svg grafic, uint32_t x, uint32_t y, uint32_t rx, uint32_t ry,
- const char * style );
-
-rc_t svg_line( p_svg grafic, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2,
- const char * style );
-
-rc_t svg_polyline( p_svg grafic, const char * style,
- const uint32_t * values, const uint32_t n_values );
-
-rc_t svg_text( p_svg grafic, uint32_t x, uint32_t y, const char * txt,
- const char * style );
-
-rc_t svg_set_path( p_svg grafic, p_svg_path path, const char * style );
-
-rc_t svg_write( p_svg grafic, const char * filename );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/sam-dump/debug.h b/tools/sam-dump/debug.h
deleted file mode 100644
index d1a321b..0000000
--- a/tools/sam-dump/debug.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-#ifndef _tools_sam_dump_debug_h_
-#define _tools_sam_dump_debug_h_
-
-#include <klib/debug.h>
-
-#define SAM_DUMP_DBG_PASTE(a,b) a##b
-#define SAM_DUMP_DBG(flag,msg) DBGMSG(DBG_APP,DBG_FLAG(SAM_DUMP_DBG_PASTE(DBG_APP_,flag)), msg)
-
-#endif /* _tools_sam_dump_debug_h_ */
diff --git a/tools/sam-dump/sam-dump.c b/tools/sam-dump/sam-dump.c
deleted file mode 100644
index ee96043..0000000
--- a/tools/sam-dump/sam-dump.c
+++ /dev/null
@@ -1,4870 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-#include <klib/report.h>
-#include <klib/container.h>
-#include <klib/log.h>
-#include <klib/out.h>
-#include <klib/status.h>
-#include <klib/rc.h>
-#include <klib/vector.h>
-#include <klib/printf.h>
-#include <klib/data-buffer.h>
-#include <vfs/path.h>
-#include <vfs/path-priv.h>
-#include <kfs/file.h>
-#include <kfs/buffile.h>
-#include <kfs/gzip.h>
-#include <kfs/bzip.h>
-#include <kdb/meta.h>
-#include <kdb/namelist.h>
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <insdc/insdc.h>
-#include <insdc/sra.h>
-#include <vdb/report.h>
-#include <vdb/manager.h>
-#include <vdb/database.h>
-#include <vdb/table.h>
-#include <vdb/cursor.h>
-#include <vdb/vdb-priv.h>
-#include <vdb/schema.h>
-#include <vdb/dependencies.h>
-#include <sra/sraschema.h>
-#include <sra/srapath.h>
-#include <align/dna-reverse-cmpl.h>
-#include <align/iterator.h>
-#include <align/reference.h>
-#include <align/quality-quantizer.h>
-
-#include <kfs/directory.h>
-#include <os-native.h>
-#include <sysalloc.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <limits.h>
-#include <string.h>
-#include <strtol.h>
-#include <ctype.h>
-#include <assert.h>
-
-#include "debug.h"
-#include "sam-dump.vers.h"
-
-#if _ARCH_BITS == 64
-#define USE_MATE_CACHE 1
-#define CURSOR_CACHE (4 * 1024 * 1024 * 1024UL)
-#else
-#define USE_MATE_CACHE 0
-#define CURSOR_CACHE (256 * 1024 * 1024)
-#endif
-
-
-
-typedef struct TAlignedRegion_struct
-{
- char name[1024];
- struct {
- INSDC_coord_zero from;
- INSDC_coord_zero to;
- } r[10240];
- int rq;
- INSDC_coord_zero max_to;
- INSDC_coord_zero min_from;
- int printed;
-} TAlignedRegion;
-
-
-typedef struct TMatepairDistance_struct
-{
- uint32_t from;
- uint32_t to;
-} TMatepairDistance;
-
-
-struct params_s
-{
- /* which outputs are on */
- bool primaries;
- bool secondaries;
- bool unaligned;
- bool cg_evidence;
- bool cg_ev_dnb;
- bool cg_sam;
-
- bool use_seqid;
- bool long_cigar;
- bool reheader;
- bool noheader;
- bool hide_identical;
- bool fasta;
- bool fastq;
- bool spot_group_in_name;
- bool cg_friendly_names;
- bool reverse_unaligned;
- bool unaligned_spots;
-
- bool output_gzip;
- bool output_bz2;
-
- bool xi;
- int cg_style; /* 0: raw; 1: with B's; 2: without B's, fixed up SEQ/QUAL; */
- char const *name_prefix;
- /* region filter data */
- TAlignedRegion* region;
- uint32_t region_qty;
- /* distance filter data */
- bool mp_dist_unknown;
- TMatepairDistance* mp_dist;
- uint32_t mp_dist_qty;
- uint32_t test_rows;
-
- /* mate info cache */
- int64_t mate_row_gap_cachable;
-
- char const **comments;
-
- bool quantizeQual;
- uint8_t qualQuant[256];
- uint8_t qualQuantSingle; /*** the quality is quantized - single value, no need to retrieve **/
-};
-
-
-struct params_s const *param;
-ReferenceList const *gRefList;
-
-
-typedef union UData_union
-{
- void const *v;
- uint32_t const *u32;
- int32_t const *i32;
- int64_t const *i64;
- uint64_t const *u64;
- uint8_t const *u8;
- char const *str;
- bool const *tf;
- INSDC_coord_one* coord1;
- INSDC_coord_zero* coord0;
- INSDC_coord_len* coord_len;
- INSDC_coord_val* coord_val;
- INSDC_SRA_xread_type* read_type;
- INSDC_SRA_read_filter* read_filter;
-} UData;
-
-
-typedef struct SCol_struct
-{
- char const *name;
- uint32_t idx;
- UData base;
- uint32_t len;
- bool optional;
-} SCol;
-
-
-typedef struct STable_struct
-{
- char const *name;
- VTable const *vtbl;
-} STable;
-
-
-typedef struct SCursCache_struct
-{
- KVector* cache;
- KVector* cache_unaligned_mate; /* keeps unaligned-mate for a half-aligned spots */
- uint32_t sam_flags;
- INSDC_coord_zero pnext;
- int32_t tlen;
- ReferenceObj const *ref;
- /* cache stats */
- uint64_t projected;
- uint64_t added;
- uint64_t hit;
- uint64_t bad;
-} SCursCache;
-
-
-typedef struct SCurs_struct
-{
- STable const *tbl;
- VCursor const *vcurs;
- SCursCache* cache;
- SCursCache cache_local;
- uint64_t col_reads_qty;
-} SCurs;
-
-enum eDSTableType
-{
- edstt_NotSpecified,
- edstt_Reference,
- edstt_Sequence,
- edstt_PrimaryAlignment,
- edstt_SecondaryAlignment,
- edstt_EvidenceAlignment,
- edstt_EvidenceInterval
-};
-
-
-typedef struct DataSource_s {
- STable tbl;
- SCurs curs;
- SCol *cols;
- enum eDSTableType type;
-} DataSource;
-
-
-#define DATASOURCE_INIT(O, NAME) do { memset(&O, 0, sizeof(O)); O.tbl.name = NAME; O.curs.tbl = &O.tbl; } while(0)
-
-
-typedef struct SAM_dump_ctx_s
-{
- VDatabase const *db;
- char const *fullPath;
- char const *accession;
- char const *readGroup;
-
- DataSource ref;
- DataSource pri;
- DataSource sec;
- DataSource evi;
- DataSource eva;
- DataSource seq;
-} SAM_dump_ctx_t;
-
-
-enum ealg_col
-{
- alg_SEQ_SPOT_ID = 0,
- alg_SEQ_NAME,
- alg_MAPQ,
- alg_CIGAR,
- alg_READ,
- alg_READ_START,
- alg_READ_LEN,
- alg_CIGAR_LEN,
- alg_SAM_QUALITY,
- alg_SPOT_GROUP,
- alg_SEQ_SPOT_GROUP,
- alg_SEQ_READ_ID,
- alg_REVERSED,
- alg_ALIGNMENT_COUNT,
- alg_EDIT_DISTANCE,
- alg_READ_FILTER,
- alg_MATE_ALIGN_ID,
- alg_MATE_REF_NAME,
- alg_SAM_FLAGS,
- alg_REF_START,
- alg_MATE_REF_POS,
- alg_ALIGN_GROUP,
- alg_EVIDENCE_ALIGNMENT_IDS,
- alg_REF_PLOIDY,
- alg_REF_ID,
- alg_MATE_REF_ID,
- alg_HAS_MISMATCH,
- alg_REGION_FILTER,
- alg_REF_NAME = alg_REGION_FILTER,
- alg_REF_SEQ_ID,
- alg_REF_POS,
- alg_REF_LEN,
- alg_DISTANCE_FILTER,
- alg_TEMPLATE_LEN = alg_DISTANCE_FILTER,
- alg_CG_TAGS_STR
-};
-
-
-SCol const g_alg_col_tmpl[] =
-{
- { "SEQ_SPOT_ID", 0, {NULL}, 0, true },
- { "SEQ_NAME", 0, {NULL}, 0, false },
- { "MAPQ", 0, {NULL}, 0, false },
- { "?CIGAR column name?", 0, {NULL}, 0, false },
- { "?READ column name?", 0, {NULL}, 0, false },
- { "READ_START", 0, {NULL}, 0, false }, /* READ_START */
- { "READ_LEN", 0, {NULL}, 0, false }, /* READ_LEN */
- { "?CIGAR_LEN column name?", 0, {NULL}, 0, true }, /* CIGAR_LEN */
- { "SAM_QUALITY", 0, {NULL}, 0, false },
- { "SPOT_GROUP", 0, {NULL}, 0, true },
- { "SEQ_SPOT_GROUP", 0, {NULL}, 0, true },
- { "SEQ_READ_ID", 0, {NULL}, 0, true },
- { "REF_ORIENTATION", 0, {NULL}, 0, true },
- { "ALIGNMENT_COUNT", 0, {NULL}, 0, true },
- { "EDIT_DISTANCE", 0, {NULL}, 0, false },
- { "", 0, {NULL}, 0, false },
- /* start cols used as standalone in DumpUnaligned */
- /* MATE_ALIGN_ID col must preceeed
- MATE_REF_NAME, MATE_REF_POS, SAM_FLAGS, TEMPLATE_LEN for cache to work */
- { "MATE_ALIGN_ID", 0, {NULL}, 0, false },
- { "?MATE_REF_NAME column name?", 0, {NULL}, 0, false },
- { "SAM_FLAGS", 0, {NULL}, 0, false },
- { "REF_START", 0, {NULL}, 0, false }, /* priming cursor cache */
- { "MATE_REF_POS", 0, {NULL}, 0, false },
- { "ALIGN_GROUP", 0, {NULL}, 0, true },
- { "", 0, {NULL}, 0, true }, /* EVIDENCE_ALIGNMENT_IDS */
- { "", 0, {NULL}, 0, true }, /* REF_PLOIDY */
- { "REF_ID", 0, {NULL}, 0, true }, /* REF_ID */
- { "MATE_REF_ID", 0, {NULL}, 0, true }, /* priming cursor cache */
- { "(bool)HAS_MISMATCH", 0, {NULL}, 0, true },
- /* these are read before any other for filtering so they must be last */
- { "REF_NAME", 0, {NULL}, 0, false },
- { "REF_SEQ_ID", 0, {NULL}, 0, false },
- { "REF_POS", 0, {NULL}, 0, false },
- /* end cols used as standalone in DumpUnaligned */
- { "REF_LEN", 0, {NULL}, 0, false },
- { "TEMPLATE_LEN", 0, {NULL}, 0, false },
- { NULL, 0, {NULL}, 0, false }, /* alg_CG_TAGS_STR */
- { NULL, 0, {NULL}, 0, false }
-};
-
-enum eseq_col
-{
- seq_READ = 0,
- seq_QUALITY,
- seq_SPOT_GROUP,
- seq_READ_START,
- seq_READ_LEN,
- seq_READ_TYPE,
- seq_READ_FILTER,
- seq_NAME,
- seq_PRIMARY_ALIGNMENT_ID
-};
-
-static
-SCol const gSeqCol[] =
-{
- { "READ", 0, {NULL}, 0, false },
-/* { "(INSDC:quality:text:phred_33)QUALITY", 0, {NULL}, 0, false }, */
- { "(INSDC:quality:phred)QUALITY", 0, {NULL}, 0, false }, /* changed Jan 29 2013, because some runs don't have the phred_33-type in schema */
- { "SPOT_GROUP", 0, {NULL}, 0, true },
- { "READ_START", 0, {NULL}, 0, true },
- { "READ_LEN", 0, {NULL}, 0, true },
- { "READ_TYPE", 0, {NULL}, 0, true },
- { "READ_FILTER", 0, {NULL}, 0, true },
- { "NAME", 0, {NULL}, 0, true },
- /* must be last in list to avoid reading all columns */
- { "PRIMARY_ALIGNMENT_ID", 0, {NULL}, 0, true },
- { NULL, 0, {NULL}, 0, false }
-};
-
-
-static rc_t RefSeqPrint( void )
-{
- rc_t rc = 0;
- uint32_t i, count = 0;
-
- rc = ReferenceList_Count( gRefList, &count );
- for( i = 0; rc == 0 && i < count; i++ )
- {
- ReferenceObj const *obj;
- rc = ReferenceList_Get( gRefList, &obj, i );
- if ( rc == 0 )
- {
- char const *seqid = NULL;
- rc = ReferenceObj_SeqId( obj, &seqid );
- if ( rc == 0 )
- {
- char const *name = NULL;
- rc = ReferenceObj_Name( obj, &name );
- if ( rc == 0 )
- {
- INSDC_coord_len len;
- rc = ReferenceObj_SeqLength( obj, &len );
- if ( rc == 0 )
- {
- char const *nm;
- if ( param->use_seqid && seqid != NULL && seqid[ 0 ] != '\0' )
- {
- nm = seqid;
- }
- else
- {
- nm = name;
- }
- KOutMsg( "@SQ\tSN:%s", nm );
- if ( nm != seqid && seqid != NULL && seqid[ 0 ] != '\0' && strcmp( nm, seqid ) != 0)
- {
- KOutMsg( "\tAS:%s", seqid );
- }
- KOutMsg( "\tLN:%u\n", len );
- }
- }
- }
- ReferenceObj_Release( obj );
- }
- }
- return rc;
-}
-
-
-#if USE_MATE_CACHE
-static rc_t Cache_Init( SCursCache* c )
-{
- if ( c != NULL )
- {
- rc_t rc;
- memset( c, 0, sizeof( *c ) );
- rc=KVectorMake( &c->cache );
- if(rc == 0){
- rc=KVectorMake( &c->cache_unaligned_mate );
- }
- }
- return 0;
-}
-
-
-static void Cache_Close( char const *name, SCursCache* c )
-{
- if ( c != NULL )
- {
- KVectorRelease( c->cache );
- KVectorRelease( c->cache_unaligned_mate );
- if ( c->added > 0 )
- {
- SAM_DUMP_DBG( 2, ( "%s cache stats: projected %lu added of those %lu; "
- "hits %lu of those broken %lu;\n",
- name, c->projected, c->added, c->hit, c->bad ) );
- }
- }
- memset( c, 0, sizeof( *c ) );
-}
-
-
-static rc_t Cache_Add( uint64_t key, SCurs const *curs, SCol const *cols )
-{
- /* compact values for mate record to cache as:
- pos_delta - 32bit, ref_proj - 11bit, flags - 11bit, rnext_idx - 10bit = 64bit
- */
- rc_t rc = 0;
- ReferenceObj const *r = NULL;
- uint32_t rid = 0;
- uint64_t val = 0;
- int64_t mate_id = cols[ alg_MATE_ALIGN_ID ].len > 0 ? cols[ alg_MATE_ALIGN_ID ].base.i64[ 0 ] : 0;
-
- rc = ReferenceList_Find( gRefList, &r, cols[ alg_REF_NAME ].base.str, cols[ alg_REF_NAME ].len );
- if ( rc == 0 )
- {
- rc = ReferenceObj_Idx( r, &rid );
- }
-#if _DEBUGGING
- {
- char const *rname = NULL;
- curs->cache->projected++;
- ReferenceObj_Name( r, &rname );
- SAM_DUMP_DBG( 10, ( "to cache row %li for mate %li: %u,%s[%hu],%u,%u,%i",
- key, mate_id, cols[ alg_SAM_FLAGS ].base.u32[ 0 ], rname, rid,
- cols[ alg_REF_POS ].len ? cols[ alg_REF_POS ].base.coord0[ 0 ] : 0,
- cols[ alg_MATE_REF_POS ].len ? cols[ alg_MATE_REF_POS ].base.coord0[ 0 ] : 0,
- cols[ alg_TEMPLATE_LEN ].len > 0 ? cols[ alg_TEMPLATE_LEN ].base.i32[ 0 ] : 0));
- }
-#endif
- if ( rc == 0 && !( rid & 0xFC00 ) )
- {
- int64_t pos_delta64;
- int32_t pos_delta32;
-
- if ( mate_id != 0 )
- {
- ReferenceObj const *rm = NULL;
- uint32_t rm_id;
-
- rc = ReferenceList_Find( gRefList, &rm, cols[ alg_MATE_REF_NAME ].base.str, cols[ alg_MATE_REF_NAME ].len );
- if ( rc == 0 )
- {
- rc = ReferenceObj_Idx( rm, &rm_id );
- }
- assert( rm != NULL );
- if ( rc == 0 && rid != rm_id )
- {
- char const *rm_name = NULL;
- ReferenceObj_Name( rm, &rm_name );
- mate_id = 0;
- SAM_DUMP_DBG( 10, ( " mate ref differ: %s[%hu]!", rm_name, rm_id ) );
- }
- ReferenceObj_Release( rm );
- }
-
- if ( mate_id != 0 )
- {
- pos_delta64 = cols[ alg_MATE_REF_POS ].base.coord0[ 0 ] - cols[ alg_REF_POS ].base.coord0[ 0 ];
- }
- else
- {
- pos_delta64 = cols[ alg_REF_POS ].base.coord0[ 0 ];
- }
-
- pos_delta32 = pos_delta64;
- if ( pos_delta64 == pos_delta32 )
- {
- int64_t ref_proj;
- if ( mate_id == 0 )
- {
- ref_proj = 0; /* indicates full value in delta */
- }
- else if ( cols[ alg_TEMPLATE_LEN ].base.i32[ 0 ] < 0 )
- {
- assert( pos_delta32 <= 0 );
- ref_proj = pos_delta32 - cols[ alg_TEMPLATE_LEN ].base.i32[ 0 ];
- }
- else
- {
- assert( pos_delta32 >= 0 );
- ref_proj = cols[ alg_TEMPLATE_LEN ].base.i32[ 0 ] - pos_delta32;
- }
-
- if ( !( ref_proj & 0xFFFFF800 ) )
- {
- val = ( pos_delta64 << 32 ) | ( ref_proj << 21 ) | ( cols[ alg_SAM_FLAGS ].base.u32[ 0 ] << 10 ) | rid;
- rc = KVectorSetU64( curs->cache->cache, key, val );
- }
- }
- }
- ReferenceObj_Release( r );
-
-#if _DEBUGGING
- if ( val == 0 )
- {
- SAM_DUMP_DBG( 10, ( " --> out of range\n" ) );
- }
- else
- {
- SAM_DUMP_DBG( 10, ( " --> %016lX\n", val ) );
- curs->cache->added++;
- }
-#endif
- return rc;
-}
-
-
-static rc_t Cache_Get( SCurs const *curs, uint64_t key, uint64_t* val )
-{
- rc_t rc = KVectorGetU64( curs->cache->cache, key, val );
- if ( rc == 0 )
- {
- uint32_t id = ( *val & 0x3FF );
-#if _DEBUGGING
- curs->cache->hit++;
-#endif
- KVectorUnset( curs->cache->cache, key );
- rc = ReferenceList_Get( gRefList, &curs->cache->ref, id );
- if ( rc != 0 )
- {
- *val = 0;
- curs->cache->ref = NULL;
- rc = RC( rcExe, rcNoTarg, rcSearching, rcItem, rcNotFound );
-#if _DEBUGGING
- curs->cache->bad++;
-#endif
- }
- else
- {
- SAM_DUMP_DBG( 10, ( "from cache row %li %016lX", key, *val ) );
- }
- }
- return rc;
-}
-
-
-static void Cache_Unpack( uint64_t val, int64_t mate_id, SCurs const *curs, SCol* cols )
-{
- int32_t pos_delta = ( val & 0xFFFFFFFF00000000 ) >> 32;
- uint32_t ref_proj = ( val & 0x00000000FFE00000 ) >> 21;
- uint32_t flags = ( val & 0x00000000001FFC00 ) >> 10;
-
- if ( mate_id != 0 )
- {
- /* adjust flags for mate record */
- curs->cache->sam_flags = ( flags & 0x1 ) |
- ( flags & 0x2 ) |
- ( ( flags & 0x8 ) >> 1 ) |
- ( ( flags & 0x4 ) << 1 ) |
- ( ( flags & 0x20 ) >> 1 ) |
- ( ( flags & 0x10 ) << 1 ) |
- ( ( flags & 0x40 ) ? 0x80 : 0x40 ) |
- ( flags & 0x700 );
- }
- else
- {
- /* preserve flags as if original records is restored */
- curs->cache->sam_flags = flags;
- }
- cols[ alg_SAM_FLAGS ].base.u32 = &curs->cache->sam_flags;
- cols[ alg_SAM_FLAGS ].len = sizeof( curs->cache->sam_flags );
-
- if ( param->use_seqid )
- {
- ReferenceObj_SeqId( curs->cache->ref, &cols[ alg_MATE_REF_NAME ].base.str );
- }
- else
- {
- ReferenceObj_Name( curs->cache->ref, &cols[ alg_MATE_REF_NAME ].base.str );
- }
-
- cols[ alg_MATE_REF_NAME ].len = strlen( cols[ alg_MATE_REF_NAME ].base.str );
-
- if ( ref_proj == 0 )
- {
- curs->cache->pnext = pos_delta;
- curs->cache->tlen = 0;
- }
- else if ( pos_delta > 0 )
- {
- curs->cache->pnext = ( cols[ alg_REF_POS ].len > 0 ? cols[ alg_REF_POS ].base.coord0[ 0 ] : 0 ) - pos_delta;
- curs->cache->tlen = - ( ref_proj + pos_delta );
- }
- else
- {
- curs->cache->pnext = ( cols[ alg_REF_POS ].len > 0 ? cols[ alg_REF_POS ].base.coord0[ 0 ] : 0 ) - pos_delta;
- curs->cache->tlen = ref_proj - pos_delta;
- }
-
- cols[ alg_MATE_REF_POS ].base.coord0 = &curs->cache->pnext;
- cols[ alg_MATE_REF_POS ].len = sizeof( curs->cache->pnext );
- cols[ alg_TEMPLATE_LEN ].base.i32 = &curs->cache->tlen;
- cols[ alg_TEMPLATE_LEN ].len = sizeof( curs->cache->tlen );
- {
- uint32_t id;
- ReferenceObj_Idx( curs->cache->ref, &id );
- SAM_DUMP_DBG( 10, ( " --> mate %li: %u,%s[%hu],%u,%i\n",
- mate_id, curs->cache->sam_flags, cols[ alg_MATE_REF_NAME ].base.str,
- id, curs->cache->pnext, curs->cache->tlen ) );
- }
-}
-#endif /* USE_MATE_CACHE */
-
-#if 0
-static rc_t OpenVTable( VDatabase const *db, STable* tbl, char const *name, bool optional )
-{
- rc_t rc = VDatabaseOpenTableRead( db, &tbl->vtbl, "%s", name );
- if ( GetRCState( rc ) == rcNotFound && optional )
- {
- rc = 0;
- tbl->vtbl = NULL;
- }
- tbl->name = name;
- return rc;
-}
-#endif
-
-static rc_t Cursor_Open( STable const *const tbl, SCurs *const curs, SCol cols[], SCursCache* cache )
-{
- rc_t rc = 0;
- unsigned i;
-
- curs->vcurs = NULL;
- if ( tbl == NULL || tbl->vtbl == NULL )
- return 0;
-
- rc = VTableCreateCachedCursorRead( tbl->vtbl, &curs->vcurs, CURSOR_CACHE );
- if ( rc != 0 )
- return rc;
-
- rc = VCursorPermitPostOpenAdd( curs->vcurs );
- if ( rc != 0 )
- return rc;
-
- if ( rc == 0 )
- {
- rc = VCursorOpen( curs->vcurs );
- if ( rc == 0 )
- {
-#if USE_MATE_CACHE
- if ( cache != NULL )
- {
- curs->cache = cache;
- }
- else
- {
- curs->cache = &curs->cache_local;
- rc = Cache_Init( &curs->cache_local );
- }
-#endif /* USE_MATE_CACHE */
- curs->tbl = tbl;
- }
- }
-
- for ( i = 0; cols[ i ].name != NULL; ++i )
- {
- if ( cols[ i ].name[ 0 ] == 0 )
- continue;
- rc = VCursorAddColumn( curs->vcurs, &cols[ i ].idx, "%s", cols[ i ].name );
- if ( GetRCObject( rc ) == rcColumn )
- {
- switch ( GetRCState( rc ) )
- {
- case rcNotFound:
- case rcUndefined:
- if ( !cols[ i ].optional )
- break;
- case rcExists:
- rc = 0;
- default:
- break;
- }
- }
- if ( rc != 0 )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "table $(t) column $(c)", "t=%s,c=%s", tbl->name, cols[ i ].name ) );
- break;
- }
- }
- if ( rc != 0 )
- {
- VCursorRelease( curs->vcurs );
- curs->vcurs = NULL;
- if ( rc != KLogLastErrorCode() )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "table $(t)", "t=%s", tbl->name ) );
- }
- }
- else
- {
- SAM_DUMP_DBG( 2, ( "%s: table %s\n", __func__, curs->tbl->name ) );
- }
- return rc;
-}
-
-
-static void Cursor_Close( SCurs* curs )
-{
- if ( curs != NULL && curs->vcurs != NULL )
- {
- SAM_DUMP_DBG( 2, ( "%s: table %s, columns rows read %lu\n", __func__, curs->tbl->name, curs->col_reads_qty ) );
- VCursorRelease( curs->vcurs );
-#if USE_MATE_CACHE
- if ( curs->cache == &curs->cache_local )
- {
- Cache_Close( curs->tbl->name, curs->cache );
- }
-#endif /* USE_MATE_CACHE */
- memset( curs, 0, sizeof( *curs ) );
- }
-}
-
-
-static rc_t Cursor_Read( DataSource *ds, int64_t row_id, int firstCol, unsigned nCols )
-{
- rc_t rc = 0;
-
- if ( ds->curs.vcurs == NULL )
- {
- rc = Cursor_Open( &ds->tbl, &ds->curs, ds->cols, ds->curs.cache );
- if ( rc != 0 )
- return rc;
- }
- if (1)
- {
- SCol *const col = &ds->cols[ firstCol ];
- unsigned i;
-
- for ( i = 0; i < nCols && col[ i ].name; ++i )
- {
- uint32_t const idx = col[ i ].idx;
-
- if ( idx != 0 )
- {
- uint32_t elem_bits;
- uint32_t bit_offset;
- uint32_t elem_count;
- void const *base;
-
- rc = VCursorCellDataDirect( ds->curs.vcurs, row_id, idx, &elem_bits, &base, &bit_offset, &elem_count );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogWarn, ( klogWarn, rc, "reading $(t) row $(r), column $(c)", "t=%s,r=%li,c=%s",
- ds->tbl.name, row_id, col[ i ].name ) );
- return rc;
- }
-
- assert( bit_offset == 0 );
- assert( elem_bits % 8 == 0 );
-
- col[ i ].base.v = base;
- col[ i ].len = elem_count;
- }
- }
- }
- return rc;
-}
-
-struct
-{
- KWrtWriter writer;
- void* data;
- KFile* kfile;
- uint64_t pos;
-} g_out_writer = {NULL};
-
-
-static rc_t CC BufferedWriter( void *const self, char const buffer[], size_t const bufsize, size_t *const pnum_writ )
-{
- rc_t rc = 0;
- size_t written = 0;
-
- assert( buffer != NULL );
-
- while ( written < bufsize )
- {
- size_t n;
-
- rc = KFileWrite( g_out_writer.kfile, g_out_writer.pos + written, &buffer[ written ], bufsize - written, &n );
- if ( rc != 0 )
- break;
- written += n;
- }
- g_out_writer.pos += written;
- if ( pnum_writ != NULL )
- *pnum_writ = written;
- return rc;
-}
-
-
-static rc_t BufferedWriterMake( bool gzip, bool bzip2 )
-{
- rc_t rc = 0;
-
- if ( gzip && bzip2 )
- {
- rc = RC( rcApp, rcFile, rcConstructing, rcParam, rcAmbiguous );
- }
- else if ( g_out_writer.writer != NULL )
- {
- rc = RC( rcApp, rcFile, rcConstructing, rcParam, rcAmbiguous );
- }
- else
- {
- rc = KFileMakeStdOut( &g_out_writer.kfile );
- if ( rc == 0 )
- {
- g_out_writer.pos = 0;
- if ( gzip )
- {
- KFile* gz;
- rc = KFileMakeGzipForWrite( &gz, g_out_writer.kfile );
- if ( rc == 0 )
- {
- KFileRelease( g_out_writer.kfile );
- g_out_writer.kfile = gz;
- }
- }
- else if ( bzip2 )
- {
- KFile* bz;
- rc = KFileMakeBzip2ForWrite( &bz, g_out_writer.kfile );
- if ( rc == 0 )
- {
- KFileRelease( g_out_writer.kfile );
- g_out_writer.kfile = bz;
- }
- }
- if ( rc == 0 )
- {
- KFile* buf;
- rc = KBufFileMakeWrite( &buf, g_out_writer.kfile, false, 128 * 1024 );
- if ( rc == 0 )
- {
- KFileRelease( g_out_writer.kfile );
- g_out_writer.kfile = buf;
- g_out_writer.writer = KOutWriterGet();
- g_out_writer.data = KOutDataGet();
- rc = KOutHandlerSet( BufferedWriter, &g_out_writer );
- }
- }
- }
- }
- return rc;
-}
-
-
-static void BufferedWriterRelease( bool flush )
-{
- if ( flush )
- {
- /* avoid flushing buffered data after failure */
- KFileRelease( g_out_writer.kfile );
- }
- if ( g_out_writer.writer != NULL )
- {
- KOutHandlerSet( g_out_writer.writer, g_out_writer.data );
- }
- g_out_writer.writer = NULL;
-}
-
-
-typedef struct ReadGroup
-{
- BSTNode node;
- char name[ 1024 ];
-} ReadGroup;
-
-
-static int CC ReadGroup_sort( BSTNode const *item, BSTNode const *node )
-{
- return strcmp( ( ( ReadGroup const * )item )->name, ( ( ReadGroup const * ) node )->name );
-}
-
-
-static void ReadGroup_print( char const *nm )
-{
- if ( nm[ 0 ] != '\0' && strcasecmp( nm, "default" ) )
- {
- KOutMsg( "@RG\tID:%s\n", nm );
- }
-}
-
-
-static void CC ReadGroup_dump( BSTNode *n, void *data )
-{
- ReadGroup const *g = ( ReadGroup* )n;
- ReadGroup_print( g->name );
-}
-
-
-static rc_t CC DumpReadGroupsScan( STable const *tbl )
-{
- SCol cols[] =
- {
- { "SPOT_GROUP", 0, {NULL}, 0, false },
- { NULL, 0, {NULL}, 0, false }
- };
- rc_t rc = 0;
- BSTree tree;
- DataSource ds;
-
- memset( &ds, 0, sizeof( ds ) );
- ds.cols = cols;
-
- BSTreeInit( &tree );
- rc = Cursor_Open( tbl, &ds.curs, ds.cols, NULL );
- if ( rc == 0 )
- {
- int64_t start;
- uint64_t count;
-
- rc = VCursorIdRange( ds.curs.vcurs, 0, &start, &count );
- if ( rc == 0 )
- {
- ReadGroup* node = NULL;
- uint32_t last_len = 0;
-
- while ( count > 0 && rc == 0 )
- {
- rc = Cursor_Read( &ds, start, 0, ~(unsigned)0 );
- if ( rc == 0 && cols[ 0 ].len != 0 )
- {
- if ( node == NULL ||
- last_len != cols[ 0 ].len ||
- strncmp( cols[ 0 ].base.str, node->name, cols[ 0 ].len ) != 0 )
- {
- node = malloc( sizeof( *node ) );
- if ( node == NULL )
- {
- rc = RC( rcExe, rcNode, rcConstructing, rcMemory, rcExhausted );
- }
- else if ( cols[ 0 ].len > sizeof( node->name ) )
- {
- rc = RC( rcExe, rcString, rcCopying, rcBuffer, rcInsufficient );
- }
- else
- {
- last_len = cols[ 0 ].len;
- strncpy( node->name, cols[ 0 ].base.str, last_len );
- node->name[ last_len ] = '\0';
- rc = BSTreeInsertUnique( &tree, &node->node, NULL, ReadGroup_sort );
- if ( GetRCState( rc ) == rcExists )
- {
- free( node );
- rc = 0;
- }
- }
- }
- }
- else if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == rcRow )
- {
- rc = 0;
- }
- start++;
- count--;
- }
- }
- Cursor_Close( &ds.curs );
- }
-
- if ( rc == 0 )
- {
- BSTreeForEach( &tree, false, ReadGroup_dump, NULL );
- }
- else if ( rc != KLogLastErrorCode() )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "$(f)", "f=%s", __func__ ) );
- }
- BSTreeWhack( &tree, NULL, NULL );
- return rc;
-}
-
-
-rc_t CC DumpReadGroups( STable const *tbl )
-{
- rc_t rc = 0;
- KMetadata const *m;
-
- /* try getting list from stats meta */
- rc = VTableOpenMetadataRead( tbl->vtbl, &m );
- if ( rc == 0 )
- {
- KMDataNode const *n;
- rc = KMetadataOpenNodeRead( m, &n, "/STATS/SPOT_GROUP" );
- if ( rc == 0 )
- {
- KNamelist* names;
- rc = KMDataNodeListChild( n, &names );
- if ( rc == 0 )
- {
- uint32_t i, q;
- rc = KNamelistCount( names, &q );
- if ( rc == 0 )
- {
- for ( i = 0; rc == 0 && i < q; i++ )
- {
- char const *nm;
- rc = KNamelistGet( names, i, &nm );
- if ( rc == 0 )
- {
- ReadGroup_print( nm );
- }
- }
- }
- KNamelistRelease( names );
- }
- KMDataNodeRelease( n );
- }
- KMetadataRelease( m );
- }
-
- if ( GetRCState( rc ) == rcNotFound )
- {
- rc = DumpReadGroupsScan( tbl );
- }
- else if ( rc != 0 && rc != KLogLastErrorCode() )
- {
- (void)PLOGERR( klogErr, ( klogErr, rc, "$(f)", "f=%s", __func__ ) );
- }
- return rc;
-}
-
-static rc_t Cursor_ReadAlign( SCurs const *curs, int64_t row_id, SCol* cols, uint32_t idx )
-{
- rc_t rc = 0;
- SCol* c = NULL;
- SCol* mate_id = NULL;
-#if USE_MATE_CACHE
- uint64_t cache_val = 0;
- bool cache_miss = false;
-#endif /* USE_MATE_CACHE */
-
-
- for( ; rc == 0 && cols[ idx ].name != NULL; idx++ )
- {
- c = &cols[ idx ];
- if ( c->idx != 0 )
- {
-#if USE_MATE_CACHE
- if ( mate_id != NULL && curs->cache != NULL && !cache_miss &&
- ( idx == alg_SAM_FLAGS || idx == alg_MATE_REF_NAME || idx == alg_MATE_REF_POS || idx == alg_TEMPLATE_LEN ) &&
- mate_id->idx && mate_id->len > 0 && mate_id->base.i64[ 0 ] > 0 )
- {
- if ( cache_val != 0 )
- {
- continue;
- }
- rc = Cache_Get( curs, mate_id->base.u64[ 0 ], &cache_val );
- if ( rc == 0 )
- {
- continue;
- }
- else if ( !( GetRCObject( rc ) == rcItem && GetRCState( rc ) == rcNotFound ) )
- {
- break;
- }
- else
- {
- /* avoid multiple lookups in cache */
- cache_miss = true;
- }
- }
-#endif /* USE_MATE_CACHE */
- rc = VCursorCellDataDirect( curs->vcurs, row_id, c->idx, NULL, &c->base.v, NULL, &c->len );
- if ( rc == 0 )
- {
- if ( idx == alg_SEQ_SPOT_ID && ( c->len ==0 || c->base.i64[ 0 ] == 0 ) )
- {
- return RC( rcExe, rcColumn, rcReading, rcRow, rcNotFound );
- }
- if ( idx == alg_MATE_ALIGN_ID )
- {
- mate_id = &cols[ alg_MATE_ALIGN_ID ];
- }
-#if _DEBUGGING
- ( ( SCurs* )curs )->col_reads_qty++;
-#endif
- }
- }
- else
- {
- static INSDC_coord_zero readStart;
- static INSDC_coord_len readLen;
- static INSDC_coord_len cigarLen;
-
- switch ( (int)idx )
- {
- case alg_READ_START:
- readStart = 0;
- c->base.coord0 = &readStart;
- c->len = 1;
- break;
- case alg_READ_LEN:
- readLen = cols[ alg_READ ].len;
- c->base.coord_len = &readLen;
- c->len = 1;
- break;
- case alg_CIGAR_LEN:
- cigarLen = cols[ alg_CIGAR ].len;
- c->base.coord_len = &cigarLen;
- c->len = 1;
- break;
- }
- }
- }
-
- if ( rc != 0 )
- {
- (void)PLOGERR( klogWarn, ( klogWarn, rc, "reading $(t) row $(r), column $(c)", "t=%s,r=%li,c=%s",
- curs->tbl->name, row_id, c ? c->name : "<none>" ) );
-#if USE_MATE_CACHE
- }
- else if ( curs->cache == NULL )
- {
- }
- else if ( cache_val == 0 )
- {
- /* this row is not from cache */
- int64_t mate_align_id = ( mate_id != NULL && mate_id->len > 0 ) ? mate_id->base.i64[ 0 ] : 0;
- if ( cols[ 0 ].idx != 0 && /* we have full cursor which means we are reading alignment table */
- /* no mate -> unaligned (not secondary!) */
- ( ( mate_align_id == 0 && param->unaligned && curs->cache != &curs->cache_local ) ||
- /* 2nd mate with higher rowid and more than set gap rows away */
- ( mate_align_id != 0 && mate_align_id > row_id && ( mate_align_id - row_id) > param->mate_row_gap_cachable ) ) )
- {
- rc = Cache_Add( row_id, curs, cols );
- }
- if(param->unaligned == true && mate_align_id == 0 ){
- rc = KVectorSetBool( curs->cache->cache_unaligned_mate, cols[alg_SEQ_SPOT_ID].base.i64[0], true );
- }
- }
- else
- {
- Cache_Unpack( cache_val, row_id, curs, cols );
-#endif /* USE_MATE_CACHE */
- }
- return rc;
-}
-
-
-static void DumpName( char const *name, size_t name_len,
- const char spot_group_sep, char const *spot_group,
- size_t spot_group_len, int64_t spot_id )
-{
- if ( param->cg_friendly_names ) {
- KOutMsg("%.*s-1:%lu", spot_group_len, spot_group, spot_id);
- }
- else {
- if ( param->name_prefix != NULL )
- {
- KOutMsg( "%s.", param->name_prefix );
- }
- BufferedWriter( NULL, name, name_len, NULL );
- if ( param->spot_group_in_name && spot_group_len > 0 )
- {
- BufferedWriter( NULL, &spot_group_sep, 1, NULL );
- BufferedWriter( NULL, spot_group, spot_group_len, NULL );
- }
- }
-}
-
-
-static void DumpQuality( char const quality[], unsigned const count, bool const reverse, bool const quantize )
-{
- if ( quality == NULL )
- {
- unsigned i;
-
- for ( i = 0; i < count; ++i )
- {
- char const newValue = ((param->qualQuant && param->qualQuantSingle)?param->qualQuantSingle:30) + 33;
-
- BufferedWriter( NULL, &newValue, 1, NULL );
- }
- }
- else if ( reverse || quantize )
- {
- unsigned i;
-
- for ( i = 0; i < count; ++i )
- {
- char const qual = quality[ reverse ? ( count - i - 1 ) : i ];
-
- /* changed Jan 29 2013, because some runs dont have the phred_33 - type */
- /* char const newValue = quantize ? param->qualQuant[ qual - 33 ] + 33 : qual; */
- char const newValue = ( quantize ? param->qualQuant[ (unsigned int)qual ] : qual ) + 33 ;
- BufferedWriter( NULL, &newValue, 1, NULL );
- }
- }
- else
- {
- /* changed Jan 29 2013, because some runs dont have the phred_33 - type */
- /* BufferedWriter( NULL, quality, count, NULL ); */
- unsigned i;
- for ( i = 0; i < count; ++i )
- {
- char const qual = quality[ i ] + 33;
- BufferedWriter( NULL, &qual, 1, NULL );
- }
- }
-}
-
-
-static void DumpQuality_33( char const quality[], unsigned const count, bool const reverse, bool const quantize )
-{
- if ( quality == NULL )
- {
- unsigned i;
-
- for ( i = 0; i < count; ++i )
- {
- char const newValue = ((param->qualQuant && param->qualQuantSingle)?param->qualQuantSingle:30) + 33;
-
- BufferedWriter( NULL, &newValue, 1, NULL );
- }
- }
- else if ( reverse || quantize )
- {
- unsigned i;
-
- for ( i = 0; i < count; ++i )
- {
- char const qual = quality[ reverse ? ( count - i - 1 ) : i ];
- char const newValue = quantize ? param->qualQuant[ qual - 33 ] + 33 : qual;
- BufferedWriter( NULL, &newValue, 1, NULL );
- }
- }
- else
- {
- BufferedWriter( NULL, quality, count, NULL );
- }
-}
-
-
-static void DumpUnalignedFastX( const SCol cols[], uint32_t read_id, INSDC_coord_zero readStart, INSDC_coord_len readLen, int64_t row_id )
-{
- /* fast[AQ] represnted in SAM fields:
- [@|>]QNAME unaligned
- SEQ
- +
- QUAL
- */
- BufferedWriter( NULL, param->fastq ? "@" : ">", 1, NULL );
-
- /* QNAME: [PFX.]SEQUENCE:NAME[#SPOT_GROUP] */
- DumpName( cols[ seq_NAME ].base.str, cols[ seq_NAME ].len, '#',
- cols[ seq_SPOT_GROUP ].base.str, cols[ seq_SPOT_GROUP ].len, row_id );
- if ( read_id > 0 )
- {
- KOutMsg( "/%u", read_id );
- }
- BufferedWriter( NULL, " unaligned\n", 11, NULL );
-
- /* SEQ: SEQUENCE.READ */
- BufferedWriter( NULL, &cols[ seq_READ ].base.str[readStart], readLen, NULL );
- if ( param->fastq )
- {
- /* QUAL: SEQUENCE.QUALITY */
- BufferedWriter( NULL, "\n+\n", 3, NULL );
- DumpQuality( &cols[ seq_QUALITY ].base.str[ readStart ], readLen, false, param->quantizeQual );
- }
- BufferedWriter( NULL, "\n", 1, NULL );
-}
-
-
-static void DumpAlignedFastX( const SCol cols[], int64_t const alignId, uint32_t read_id, bool primary, bool secondary )
-{
- size_t nm;
- unsigned readId;
- unsigned const nreads = cols[ alg_READ_LEN ].len;
-
- for ( readId = 0; readId < nreads; ++readId )
- {
- char const *qname = cols[ alg_SEQ_NAME ].base.str;
- size_t qname_len = cols[ alg_SEQ_NAME ].len;
- char synth_qname[ 40 ];
- int64_t const spot_id = cols[alg_SEQ_SPOT_ID].len > 0 ? cols[alg_SEQ_SPOT_ID].base.i64[0] : 0;
- char const *const read = cols[ alg_READ ].base.str + cols[ alg_READ_START ].base.coord0[ readId ];
- char const *const qual = cols[ alg_SAM_QUALITY ].base.v
- ? cols[ alg_SAM_QUALITY ].base.str + cols[ alg_READ_START ].base.coord0[ readId ]
- : NULL;
- unsigned const readlen = cols[ alg_READ_LEN ].base.coord_len[ readId ];
-
- /* fast[AQ] represnted in SAM fields:
- [@|>]QNAME primary|secondary ref=RNAME pos=POS mapq=MAPQ
- SEQ
- +
- QUAL
- */
- BufferedWriter( NULL, param->fastq ? "@" : ">", 1, NULL );
- /* QNAME: [PFX.]SEQ_NAME[#SPOT_GROUP] */
- if ( qname_len == 0 || qname == NULL )
- {
- string_printf( synth_qname, sizeof( synth_qname ), &qname_len, "%li.%u", alignId, readId + 1 );
- qname = synth_qname;
- }
- nm = cols[ alg_SPOT_GROUP ].len ? alg_SPOT_GROUP : alg_SEQ_SPOT_GROUP;
- DumpName( qname, qname_len, '.', cols[ nm ].base.str, cols[ nm ].len, spot_id);
-
- if ( read_id > 0 )
- {
- KOutMsg( "/%u", read_id );
- }
-
- if ( primary )
- {
- BufferedWriter( NULL, " primary", 8, NULL );
- }
- else if ( secondary )
- {
- BufferedWriter( NULL, " secondary", 10, NULL );
- }
-
- /* RNAME: REF_NAME or REF_SEQ_ID */
- BufferedWriter( NULL, " ref=", 5, NULL );
- if ( param->use_seqid )
- {
- BufferedWriter( NULL, cols[ alg_REF_SEQ_ID ].base.str, cols[ alg_REF_SEQ_ID ].len, NULL );
- }
- else
- {
- BufferedWriter( NULL, cols[ alg_REF_NAME ].base.str, cols[ alg_REF_NAME ].len, NULL );
- }
-
- /* POS: REF_POS, MAPQ: MAPQ */
- KOutMsg( " pos=%u mapq=%i\n", cols[ alg_REF_POS ].base.coord0[ 0 ] + 1, cols[ alg_MAPQ ].base.i32[ 0 ] );
-
- /* SEQ: READ */
- BufferedWriter( NULL, read, readlen, NULL );
- if ( param->fastq )
- {
- /* QUAL: SAM_QUALITY */
- BufferedWriter( NULL, "\n+\n", 3, NULL );
- DumpQuality_33( qual, readlen, false, param->quantizeQual );
- }
- BufferedWriter( NULL, "\n", 1, NULL );
- }
-}
-
-
-static
-void DumpUnalignedSAM( const SCol cols[], uint32_t flags, INSDC_coord_zero readStart, INSDC_coord_len readLen,
- char const *rnext, uint32_t rnext_len, INSDC_coord_zero pnext, char const readGroup[], int64_t row_id )
-{
- unsigned i;
-
- /* QNAME: [PFX.]NAME[.SPOT_GROUP] */
- DumpName( cols[ seq_NAME ].base.str, cols[ seq_NAME ].len, '.',
- cols[ seq_SPOT_GROUP ].base.str, cols[ seq_SPOT_GROUP ].len, row_id );
-
- /* all these fields are const text for now */
- KOutMsg( "\t%u\t*\t0\t0\t*\t%.*s\t%u\t0\t",
- flags, rnext_len ? rnext_len : 1, rnext_len ? rnext : "*", pnext );
- /* SEQ: SEQUENCE.READ */
- if ( flags & 0x10 )
- {
- for( i = 0; i < readLen; i++ )
- {
- char base;
-
- DNAReverseCompliment( &cols[ seq_READ ].base.str[ readStart + readLen - 1 - i ], &base, 1 );
- BufferedWriter( NULL, &base, 1, NULL );
- }
- }
- else
- {
- BufferedWriter( NULL, &cols[ seq_READ ].base.str[ readStart ], readLen, NULL );
- }
-
- BufferedWriter( NULL, "\t", 1, NULL );
- /* QUAL: SEQUENCE.QUALITY */
- DumpQuality( &cols[ seq_QUALITY ].base.str[ readStart ], readLen, flags & 0x10, param->quantizeQual );
-
- /* optional fields: */
- if ( readGroup )
- {
- BufferedWriter( NULL, "\tRG:Z:", 6, NULL );
- BufferedWriter( NULL, readGroup, strlen( readGroup ), NULL );
- }
- else if ( cols[ seq_SPOT_GROUP ].len > 0 )
- {
- /* read group */
- BufferedWriter( NULL, "\tRG:Z:", 6, NULL );
- BufferedWriter( NULL, cols[ seq_SPOT_GROUP ].base.str, cols[ seq_SPOT_GROUP ].len, NULL );
- }
- BufferedWriter( NULL, "\n", 1, NULL );
-}
-
-
-static
-void DumpAlignedSAM(SAM_dump_ctx_t *const ctx,
- DataSource const *ds,
- int64_t alignId,
- char const readGroup[],
- int type)
-{
- unsigned const nreads = ds->cols[ alg_READ_LEN ].len;
- SCol const *const cols = ds->cols;
- int64_t const spot_id = cols[alg_SEQ_SPOT_ID].len > 0 ? cols[alg_SEQ_SPOT_ID].base.i64[0] : 0;
- INSDC_coord_one const read_id = cols[alg_SEQ_READ_ID].len > 0 ? cols[alg_SEQ_READ_ID].base.coord1[0] : 0;
- INSDC_SRA_read_filter const *align_filter = cols[alg_READ_FILTER].len == nreads ? cols[alg_READ_FILTER].base.read_filter : NULL;
- INSDC_SRA_read_filter seq_filter = 0;
- unsigned readId;
- unsigned cigOffset = 0;
-
- if (align_filter == NULL && spot_id && read_id && ctx->seq.cols) {
- rc_t rc;
-
- rc = Cursor_Read(&ctx->seq, spot_id, seq_READ_FILTER, 1);
- if (rc == 0 && ctx->seq.cols[seq_READ_FILTER].len >= read_id)
- seq_filter = ctx->seq.cols[seq_READ_FILTER].base.read_filter[read_id - 1];
- }
- for ( readId = 0; readId < nreads; ++readId ) {
- char const *qname = cols[ alg_SEQ_NAME ].base.str;
- size_t qname_len = cols[ alg_SEQ_NAME ].len;
- char const *const read = cols[ alg_READ ].base.str + cols[ alg_READ_START ].base.coord0[ readId ];
- char const *const qual = cols[ alg_SAM_QUALITY ].base.v
- ? cols[ alg_SAM_QUALITY ].base.str + cols[ alg_READ_START ].base.coord0[ readId ]
- : NULL;
- unsigned const readlen = nreads > 1 ? cols[ alg_READ_LEN ].base.coord_len[ readId ] : cols[ alg_READ ].len;
- unsigned const sflags = cols[ alg_SAM_FLAGS ].base.v ? cols[ alg_SAM_FLAGS ].base.u32[ readId ] : 0;
- INSDC_SRA_read_filter const filt = align_filter ? align_filter[readId] : seq_filter;
- unsigned const flags = (sflags & ~((unsigned)0x200)) | ((filt == SRA_READ_FILTER_REJECT) ? 0x200 : 0);
- char const *const cigar = cols[ alg_CIGAR ].base.str + cigOffset;
- unsigned const cigLen = nreads > 1 ? cols[ alg_CIGAR_LEN ].base.coord_len[ readId ] : cols[ alg_CIGAR ].len;
- size_t nm;
- char synth_qname[1024];
-
- cigOffset += cigLen;
- if ( qname_len == 0 || qname == NULL )
- {
- string_printf( synth_qname, sizeof( synth_qname ), &qname_len, "ALLELE_%li.%u", alignId, readId + 1 );
- qname = synth_qname;
- }
- else if (ds->type == edstt_EvidenceAlignment) {
- string_printf( synth_qname, sizeof( synth_qname ), &qname_len, "%u/ALLELE_%li.%u", spot_id, cols[ alg_REF_ID ].base.i64[ readId ], cols[ alg_REF_PLOIDY ].base.u32[ readId ] );
- qname = synth_qname;
- }
- nm = cols[ alg_SPOT_GROUP ].len ? alg_SPOT_GROUP : alg_SEQ_SPOT_GROUP;
- DumpName( qname, qname_len, '.', cols[ nm ].base.str, cols[ nm ].len, spot_id );
-
- /* FLAG: SAM_FLAGS */
- if (ds->type == edstt_EvidenceAlignment) {
- bool const cmpl = cols[alg_REVERSED].base.v && readId < cols[alg_REVERSED].len ? cols[alg_REVERSED].base.tf[readId] : false;
-
- KOutMsg( "\t%u\t", 1 | (cmpl ? 0x10 : 0) | (read_id == 1 ? 0x40 : 0x80) );
- }
- else if ( !param->unaligned /** not going to dump unaligned **/
- && ( flags & 0x1 ) /** but we have sequenced multiple fragments **/
- && ( flags & 0x8 ) ) /** and not all of them align **/
- {
- /*** remove flags talking about multiple reads **/
- /* turn off 0x001 0x008 0x040 0x080 */
- KOutMsg( "\t%u\t", flags & ~0xC9 );
- }
- else
- KOutMsg( "\t%u\t", flags );
-
- if ( ds->type == edstt_EvidenceAlignment && type == 0 )
- KOutMsg( "ALLELE_%li.%u", cols[ alg_REF_ID ].base.i64[ readId ], cols[ alg_REF_PLOIDY ].base.u32[ readId ] );
- else
- {
- /* RNAME: REF_NAME or REF_SEQ_ID */
- if ( param->use_seqid )
- BufferedWriter( NULL, cols[ alg_REF_SEQ_ID ].base.str, cols[ alg_REF_SEQ_ID ].len, NULL );
- else
- BufferedWriter( NULL, cols[ alg_REF_NAME ].base.str, cols[ alg_REF_NAME ].len, NULL );
- }
- BufferedWriter( NULL, "\t", 1, NULL );
-
- /* POS: REF_POS */
- KOutMsg( "%i\t", cols[ alg_REF_POS ].base.coord0[ 0 ] + 1 );
- /* MAPQ: MAPQ */
- KOutMsg( "%i\t", cols[ alg_MAPQ ].base.i32[ 0 ] );
- /* CIGAR: CIGAR_* */
- if (ds->type == edstt_EvidenceInterval) {
- unsigned i;
-
- for (i = 0; i != cigLen; ++i) {
- char ch = cigar[i];
-
- if (ch == 'S')
- ch = 'I';
- BufferedWriter(NULL, &ch, 1, NULL);
- }
- }
- else
- BufferedWriter( NULL, cigar, cigLen, NULL );
- BufferedWriter( NULL, "\t", 1, NULL );
-
- /* RNEXT: MATE_REF_NAME or '*' */
- /* PNEXT: MATE_REF_POS or 0 */
- if ( cols[ alg_MATE_REF_NAME ].len )
- {
- if ( cols[ alg_MATE_REF_NAME ].len == cols[ alg_REF_NAME ].len &&
- memcmp( cols[ alg_MATE_REF_NAME ].base.str, cols[ alg_REF_NAME ].base.str, cols[ alg_MATE_REF_NAME ].len ) == 0 )
- {
- BufferedWriter( NULL, "=\t", 2, NULL );
- }
- else
- {
- BufferedWriter( NULL, cols[ alg_MATE_REF_NAME ].base.str, cols[ alg_MATE_REF_NAME ].len, NULL );
- BufferedWriter( NULL, "\t", 1, NULL );
- }
- KOutMsg( "%u\t", cols[ alg_MATE_REF_POS ].base.coord0[ 0 ] + 1 );
- }
- else
- {
- BufferedWriter( NULL, "*\t0\t", 4, NULL );
- }
- /* TLEN: TEMPLATE_LEN */
- KOutMsg( "%i\t", cols[ alg_TEMPLATE_LEN ].base.v ? cols[ alg_TEMPLATE_LEN ].base.i32[ 0 ] : 0 );
- /* SEQ: READ */
- BufferedWriter( NULL, read, readlen, NULL );
- BufferedWriter( NULL, "\t", 1, NULL );
- /* QUAL: SAM_QUALITY */
- DumpQuality_33( qual, readlen, false, param->quantizeQual );
-
- /* optional fields: */
- if ( ds->type == edstt_EvidenceInterval )
- {
- KOutMsg( "\tRG:Z:ALLELE_%u", readId + 1 );
- }
- if ( readGroup )
- {
- BufferedWriter( NULL, "\tRG:Z:", 6, NULL );
- BufferedWriter( NULL, readGroup, strlen( readGroup ), NULL );
- }
- else if ( cols[ alg_SPOT_GROUP ].len > 0 )
- {
- /* read group */
- BufferedWriter( NULL, "\tRG:Z:", 6, NULL );
- BufferedWriter( NULL, cols[ alg_SPOT_GROUP ].base.str, cols[ alg_SPOT_GROUP ].len, NULL );
- }
- else if ( cols[ alg_SEQ_SPOT_GROUP ].len > 0 )
- {
- /* backward compatibility */
- BufferedWriter( NULL, "\tRG:Z:", 6, NULL );
- BufferedWriter( NULL, cols[ alg_SEQ_SPOT_GROUP ].base.str, cols[ alg_SEQ_SPOT_GROUP ].len, NULL );
- }
- if ( param->cg_style > 0 && cols[ alg_CG_TAGS_STR ].len > 0 )
- {
- BufferedWriter( NULL, cols[ alg_CG_TAGS_STR ].base.str, cols[ alg_CG_TAGS_STR ].len, NULL );
- }
-
- if ( param->cg_style > 0 && cols[ alg_ALIGN_GROUP ].len > 0 )
- {
- char const *ZI = cols[ alg_ALIGN_GROUP ].base.str;
- unsigned i;
-
- for ( i = 0; i < cols[ alg_ALIGN_GROUP ].len - 1; ++i )
- {
- if ( ZI[ i ] == '_' )
- {
- KOutMsg( "\tZI:i:%.*s\tZA:i:%.1s", i, ZI, ZI + i + 1 );
- break;
- }
- }
- }
- else if ( ds->type == edstt_EvidenceAlignment && type == 1 )
- KOutMsg( "\tZI:i:%li\tZA:i:%u", cols[ alg_REF_ID ].base.i64[ readId ], cols[ alg_REF_PLOIDY ].base.u32[ readId ] );
-
- /* align id */
- if ( param->xi )
- KOutMsg( "\tXI:i:%li", alignId );
-
- /* hit count */
- if (cols[alg_ALIGNMENT_COUNT].len)
- KOutMsg( "\tNH:i:%i", (int)cols[ alg_ALIGNMENT_COUNT ].base.u8[ readId ] );
-
- /* edit distance */
- if ( cols[ alg_EDIT_DISTANCE ].len )
- KOutMsg( "\tNM:i:%i", cols[ alg_EDIT_DISTANCE ].base.i32[ readId ] );
-
- KOutMsg( "\n" );
- }
-}
-
-
-static rc_t DumpUnalignedReads( SAM_dump_ctx_t *const ctx, SCol const calg_col[], int64_t row_id, uint64_t* rcount )
-{
- rc_t rc = 0;
- uint32_t i, nreads = 0;
-
- if ( calg_col != NULL )
- {
- if ( calg_col[ alg_SAM_FLAGS ].base.u32[ 0 ] & 0x02 )
- {
- /* skip all aligned rows by flag */
- return rc;
- }
- /* get primary alignments only */
- rc = Cursor_Read( &ctx->seq, calg_col[ alg_SEQ_SPOT_ID ].base.i64[ 0 ], seq_PRIMARY_ALIGNMENT_ID, ~(unsigned)0 );
- if ( rc == 0 )
- {
- for ( i = 0; i < ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].len; i++ )
- {
- if ( ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[ i ] != 0 )
- {
- if ( ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[ i ] < row_id )
- {
- /* unaligned were printed with 1st aligment */
- return rc;
- }
- }
- else
- {
- nreads++;
- }
- }
- if ( nreads == ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].len )
- {
- /* skip all aligned rows by actual data, if flag above is not set properly */
- return rc;
- }
- row_id = calg_col[ alg_SEQ_SPOT_ID ].base.i64[ 0 ];
- }
- }
- if ( rc == 0 )
- {
- rc = Cursor_Read( &ctx->seq, row_id, 0, ~(unsigned)0 );
- if ( rc == 0 )
- {
- unsigned non_empty_reads = 0;
-
- nreads = ctx->seq.cols[ seq_READ_LEN ].idx != 0 ? ctx->seq.cols[ seq_READ_LEN ].len : 1;
-
- for( i = 0; i < nreads; i++ ) {
- unsigned const readLen = ctx->seq.cols[ seq_READ_LEN ].idx
- ? ctx->seq.cols[ seq_READ_LEN ].base.coord_len[ i ]
- : 0;
-
- if (readLen)
- ++non_empty_reads;
- }
- for( i = 0; i < nreads; i++ )
- {
- INSDC_coord_zero readStart;
- INSDC_coord_len readLen;
-
- if ( ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].idx != 0 &&
- ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[ i ] != 0 )
- {
- continue;
- }
-
- if ( ctx->seq.cols[ seq_READ_TYPE ].idx != 0 &&
- !( ctx->seq.cols[ seq_READ_TYPE ].base.read_type[ i ] & SRA_READ_TYPE_BIOLOGICAL ) )
- {
- continue;
- }
-
- readLen = ctx->seq.cols[ seq_READ_LEN ].idx ?
- ctx->seq.cols[ seq_READ_LEN ].base.coord_len[ i ] :
- ctx->seq.cols[ seq_READ ].len;
- if ( readLen == 0 )
- {
- continue;
- }
-
- readStart = ctx->seq.cols[ seq_READ_START ].idx ?
- ctx->seq.cols[ seq_READ_START ].base.coord0[ i ] :
- 0;
- if ( param->fasta || param->fastq )
- {
- DumpUnalignedFastX( ctx->seq.cols, nreads > 1 ? i + 1 : 0, readStart, readLen, row_id );
- }
- else
- {
- uint32_t cflags = 0x4;
- if ( param->reverse_unaligned )
- {
- if ( ctx->seq.cols[ seq_READ_TYPE ].base.read_type[ i ] & SRA_READ_TYPE_REVERSE )
- {
- cflags |= 0x10;
- }
- if ( ctx->seq.cols[ seq_READ_TYPE ].base.read_type[ i == nreads - 1 ? 0 : ( i + 1 ) ] & SRA_READ_TYPE_REVERSE )
- {
- cflags |= 0x20;
- }
- }
- if ( ctx->seq.cols[ seq_READ_FILTER ].idx != 0 )
- {
- if ( ctx->seq.cols[ seq_READ_FILTER ].base.read_filter[ i ] == SRA_READ_FILTER_REJECT )
- {
- cflags |= 0x200;
- }
- else if ( ctx->seq.cols[ seq_READ_FILTER ].base.read_filter[ i ] == SRA_READ_FILTER_CRITERIA )
- {
- cflags |= 0x400;
- }
- }
- if ( calg_col == NULL )
- {
- DumpUnalignedSAM( ctx->seq.cols, cflags |
- ( non_empty_reads > 1 ? ( 0x1 | 0x8 | ( i == 0 ? 0x40 : 0x00 ) | ( i == nreads - 1 ? 0x80 : 0x00 ) ) : 0x00 ),
- readStart, readLen, NULL, 0, 0, ctx->readGroup, row_id );
- }
- else
- {
- int c = param->use_seqid ? alg_REF_SEQ_ID : alg_REF_NAME;
- uint16_t flags = cflags | 0x1 |
- ( ( calg_col[ alg_SAM_FLAGS ].base.u32[ 0 ] & 0x10 ) << 1 ) |
- ( ( calg_col[ alg_SAM_FLAGS ].base.u32[ 0 ] & 0x40 ) ? 0x80 : 0x40 );
- DumpUnalignedSAM( ctx->seq.cols, flags, readStart, readLen,
- calg_col[ c ].base.str, calg_col[ c ].len,
- calg_col[ alg_REF_POS ].base.coord0[ 0 ] + 1, ctx->readGroup, row_id );
- }
- }
- *rcount = *rcount + 1;
- }
- }
- }
- return rc;
-}
-
-
-static bool AlignRegionFilter( SCol const *cols )
-{
- if ( param->region_qty == 0 )
- return true;
- if ( cols[ alg_REF_NAME ].len != 0 || cols[ alg_REF_SEQ_ID ].len != 0 )
- {
- unsigned i;
-
- assert( cols[ alg_REF_POS ].len == cols[ alg_REF_LEN ].len );
-
- for ( i = 0; i < param->region_qty; i++ )
- {
- unsigned j;
-
- for ( j = 0; j < cols[ alg_REF_POS ].len; j++ )
- {
- unsigned const refStart_zero = cols[ alg_REF_POS ].base.coord0[ j ];
- unsigned const refEnd_exclusive = refStart_zero + cols[ alg_REF_LEN ].base.coord_len[ j ];
- unsigned k;
-
- for ( k = 0; k < param->region[ i ].rq; k++ )
- {
- unsigned const from_zero = param->region[ i ].r[ k ].from;
- unsigned const to_inclusive = param->region[ i ].r[ k ].to;
-
- if ( from_zero < refEnd_exclusive && refStart_zero <= to_inclusive )
- return true;
- }
- }
- }
- }
- return false;
-}
-
-
-static bool AlignDistanceFilter( SCol const *cols )
-{
- if ( param->mp_dist_qty != 0 || param->mp_dist_unknown )
- {
- if ( cols[ alg_TEMPLATE_LEN ].len == 0 && param->mp_dist_unknown )
- {
- return true;
- }
- else
- {
- uint32_t i, j;
- for( i = 0; i < param->mp_dist_qty; i++ )
- {
- for( j = 0; j < cols[ alg_TEMPLATE_LEN ].len; j++ )
- {
- if ( ( cols[ alg_TEMPLATE_LEN ].base.i32[ j ] == 0 && param->mp_dist_unknown ) ||
- ( param->mp_dist[ i ].from <= cols[ alg_TEMPLATE_LEN ].base.i32[ j ] &&
- cols[ alg_TEMPLATE_LEN ].base.i32[ j ] <= param->mp_dist[ i ].to ) )
- {
- return true;
- }
- }
- }
- }
- return false;
- }
- return true;
-}
-
-
-typedef struct cgOp_s
-{
- uint16_t length;
- uint8_t type; /* 0: match, 1: insert, 2: delete */
- char code;
-} cgOp;
-
-
-static void print_CG_cigar( int line, cgOp const op[], unsigned const ops, unsigned const gap[ 3 ] )
-{
-#if _DEBUGGING
- unsigned i;
-
- SAM_DUMP_DBG( 3, ( "%5i: ", line ) );
- for ( i = 0; i < ops; ++i )
- {
- if ( gap && ( i == gap[ 0 ] || i == gap[ 1 ] || i == gap[ 2 ] ) )
- {
- SAM_DUMP_DBG( 3, ( "%u%c", op[ i ].length, tolower( op[ i ].code ) ) );
- }
- else
- {
- SAM_DUMP_DBG( 3, ( "%u%c", op[ i ].length, toupper( op[ i ].code ) ) );
- }
- }
- SAM_DUMP_DBG( 3, ( "\n" ) );
-#endif
-}
-
-/* gap contains the indices of the wobbles in op
- * gap[0] is between read 1 and 2; it is the 'B'
- * gap[1] is between read 2 and 3; it is an 'N'
- * gap[2] is between read 3 and 4; it is an 'N'
- */
-
-static rc_t CIGAR_to_CG_Ops( cgOp op[], unsigned const maxOps,
- unsigned *const opCnt,
- unsigned gap[ 3 ],
- char const cigar[], unsigned const ciglen,
- unsigned *const S_adjust,
- unsigned *const CG_adjust,
- unsigned const read,
- bool const reversed )
-{
- unsigned i;
- unsigned ops = 0;
- unsigned gapno;
-
- *opCnt = 0;
- *S_adjust = 0;
- for ( i = 0; i < ciglen; ++ops )
- {
- char opChar=0;
- int opLen=0;
- int n;
-
- for(n=0;n+i<ciglen && isdigit(cigar[n+i]);n++){
- opLen = opLen * 10 + cigar[n+i] - '0';
- }
- if(n+i<ciglen){
- opChar = cigar[n+i];
- n++;
- }
- if ( ops + 1 >= maxOps )
- return RC( rcExe, rcData, rcReading, rcBuffer, rcInsufficient );
-
- i += n;
-
- op[ ops ].length = opLen;
- op[ ops ].code = opChar;
- switch ( opChar )
- {
- case 'M':
- case '=':
- case 'X':
- op[ ops ].type = 0;
- break;
- case 'S':
- *S_adjust += opLen;
- case 'I':
- op[ ops ].type = 1;
- op[ ops ].code = 'I';
- break;
- case 'D':
- op[ ops ].type = 2;
- break;
- default:
- return RC( rcExe, rcData, rcReading, rcConstraint, rcViolated );
- }
- }
- *opCnt = ops;
- gap[ 0 ] = gap[ 1 ] = gap[ 2 ] = ops;
- print_CG_cigar( __LINE__, op, ops, NULL );
- if ( ops < 3 )
- return RC( rcExe, rcData, rcReading, rcFormat, rcNotFound ); /* CG pattern not found */
-
- {
- unsigned fwd = 0; /* 5 10 10 10 */
- unsigned rev = 0; /* 10 10 10 5 */
- unsigned acc; /* accumulated length */
-
- if ( ( read == 1 && !reversed ) || ( read == 2 && reversed ) )
- {
- for ( i = 0, acc = 0; i < ops && acc <=5; ++i )
- {
- if ( op[ i ].type != 2 )
- {
- acc += op[ i ].length;
- if ( acc == 5 && op[ i + 1 ].type == 1 )
- {
- fwd = i + 1;
- break;
- }
- else if ( acc > 5 )
- {
- unsigned const right = acc - 5;
- unsigned const left = op[ i ].length - right;
-
- memmove( &op[ i + 2 ], &op[ i ], ( ops - i ) * sizeof( op[ 0 ] ) );
- ops += 2;
- op[ i ].length = left;
- op[ i + 2 ].length = right;
-
- op[ i + 1 ].type = 1;
- op[ i + 1 ].code = 'B';
- op[ i + 1 ].length = 0;
-
- fwd = i + 1;
- break;
- }
- }
- }
- }
- else if ( ( read == 2 && !reversed ) || ( read == 1 && reversed ) )
- {
- for ( i = ops, acc = 0; i > 0 && acc <= 5; )
- {
- --i;
- if ( op[ i ].type != 2 )
- {
- acc += op[ i ].length;
- if ( acc == 5 && op[ i ].type == 1 )
- {
- rev = i;
- break;
- }
- else if ( acc > 5 )
- {
- unsigned const left = acc - 5;
- unsigned const right = op[ i ].length - left;
-
- memmove( &op[ i + 2 ], &op[ i ], ( ops - i ) * sizeof( op[ 0 ] ) );
- ops += 2;
- op[ i ].length = left;
- op[ i + 2 ].length = right;
-
- op[ i + 1 ].type = 1;
- op[ i + 1 ].code = 'B';
- op[ i + 1 ].length = 0;
-
- rev = i + 1;
- break;
- }
- }
- }
- }
- else
- {
- /* fprintf(stderr, "guessing layout\n"); */
- for ( i = 0, acc = 0; i < ops && acc <= 5; ++i )
- {
- if ( op[ i ].type != 2 )
- {
- acc += op[ i ].length;
- if ( acc == 5 && op[ i + 1 ].type == 1 )
- {
- fwd = i + 1;
- }
- }
- }
- for ( i = ops, acc = 0; i > 0 && acc <= 5; )
- {
- --i;
- if ( op[ i ].type != 2 )
- {
- acc += op[ i ].length;
- if ( acc == 5 && op[i].type == 1 )
- {
- rev = i;
- }
- }
- }
- if ( ( fwd == 0 && rev == 0 ) || ( fwd != 0 && rev != 0 ) )
- {
- for ( i = 0; i < ops; ++i )
- {
- if ( op[ i ].type == 2 )
- {
- op[ i ].code = 'N';
- *CG_adjust += op[ i ].length;
- }
- }
- return RC( rcExe, rcData, rcReading, rcFormat, rcNotFound ); /* CG pattern not found */
- }
- }
- if ( fwd && op[ fwd ].type == 1 )
- {
- for ( i = ops, acc = 0; i > fwd + 1; )
- {
- --i;
- if ( op[ i ].type != 2 )
- {
- acc += op[ i ].length;
- if ( acc >= 10 )
- {
- if ( acc > 10 )
- {
- unsigned const r = 10 + op[ i ].length - acc;
- unsigned const l = op[ i ].length - r;
-
- if ( ops + 2 >= maxOps )
- return RC( rcExe, rcData, rcReading, rcBuffer, rcInsufficient );
- memmove( &op[ i + 2 ], &op[ i ], ( ops - i ) * sizeof( op[ 0 ] ) );
- ops += 2;
- op[ i + 2 ].length = r;
- op[ i ].length = l;
-
- op[ i + 1 ].length = 0;
- op[ i + 1 ].type = 2;
- op[ i + 1 ].code = 'N';
- i += 2;
- }
- else if ( i - 1 > fwd )
- {
- if ( op[ i - 1 ].type == 2 )
- op[ i - 1 ].code = 'N';
- else
- {
- if ( ops + 1 >= maxOps )
- return RC( rcExe, rcData, rcReading, rcBuffer, rcInsufficient );
- memmove( &op[ i + 1 ], &op[ i ], ( ops - i ) * sizeof( op[ 0 ] ) );
- ops += 1;
- op[ i ].length = 0;
- op[ i ].type = 2;
- op[ i ].code = 'N';
- i += 1;
- }
- }
- acc = 0;
- }
- }
- }
- /** change I to B+M **/
- op[ fwd ].code = 'B';
- memmove( &op[ fwd + 1 ], &op[ fwd ], ( ops - fwd ) * sizeof( op[ 0 ] ) );
- ops += 1;
- op[ fwd + 1 ].code = 'M';
- *opCnt = ops;
- /** set the gaps now **/
- for ( gapno = 3, i = ops; gapno > 1 && i > 0; )
- {
- --i;
- if ( op[ i ].code == 'N' )
- gap[ --gapno ] = i;
- }
- gap[ 0 ] = fwd;
- print_CG_cigar( __LINE__, op, ops, gap );
- return 0;
- }
- if ( rev && op[ rev ].type == 1 )
- {
- for ( acc = i = 0; i < rev; ++i )
- {
- if ( op[ i ].type != 2 )
- {
- acc += op[ i ].length;
- if ( acc >= 10 )
- {
- if ( acc > 10 )
- {
- unsigned const l = 10 + op[ i ].length - acc;
- unsigned const r = op[ i ].length - l;
-
- if ( ops + 2 >= maxOps )
- return RC( rcExe, rcData, rcReading, rcBuffer, rcInsufficient );
- memmove( &op[ i + 2 ], &op[ i ], ( ops - i ) * sizeof( op[ 0 ] ) );
- ops += 2;
- op[ i + 2 ].length = r;
- op[ i ].length = l;
-
- op[ i + 1 ].length = 0;
- op[ i + 1 ].type = 2;
- op[ i + 1 ].code = 'N';
- rev += 2;
- i += 2;
- }
- else if ( i + 1 < rev )
- {
- if ( op[ i + 1 ].type == 2)
- op[ i + 1 ].code = 'N';
- else
- {
- if ( ops + 1 >= maxOps )
- return RC( rcExe, rcData, rcReading, rcBuffer, rcInsufficient );
- memmove( &op[ i + 1 ], &op[ i ], ( ops - i ) * sizeof( op[ 0 ] ) );
- ops += 1;
- op[ i + 1 ].length = 0;
- op[ i + 1 ].type = 2;
- op[ i + 1 ].code = 'N';
- rev += 1;
- i += 1;
- }
- }
- acc = 0;
- }
- }
- }
- for ( gapno = 3, i = 0; gapno > 1 && i < ops; ++i )
- {
- if ( op[ i ].code == 'N' )
- gap[ --gapno ] = i;
- }
- gap[ 0 ] = rev;
- op[ rev ].code = 'B';
- memmove( &op[ rev + 1 ], &op[ rev ], ( ops - rev ) * sizeof( op[ 0 ] ) );
- ops += 1;
- op[ rev + 1 ].code = 'M';
- *opCnt = ops;
- print_CG_cigar( __LINE__, op, ops, gap );
- return 0;
- }
- }
- return RC( rcExe, rcData, rcReading, rcFormat, rcNotFound ); /* CG pattern not found */
-}
-
-
-static rc_t GenerateCGData( SCol cols[], unsigned style )
-{
- rc_t rc = 0;
-
- memset( &cols[ alg_CG_TAGS_STR], 0, sizeof( cols[ alg_CG_TAGS_STR ] ) );
-
- if ( cols[ alg_READ ].len == 35 && cols[ alg_SAM_QUALITY ].len == 35 )
- {
- static char newCIGAR[ 35 * 11 ];
- static int32_t newEditDistance;
- unsigned gap[ 3 ] = { 0, 0, 0 };
- cgOp cigOp[ 35 ];
- unsigned opCnt;
- unsigned i;
- unsigned j;
- size_t sz;
- unsigned S_adjust = 0;
- unsigned CG_adjust = 0;
- unsigned const read = cols[ alg_SEQ_READ_ID ].len && cols[ alg_REVERSED ].len ? cols[ alg_SEQ_READ_ID ].base.coord1[ 0 ] : 0;
- bool const reversed = cols[ alg_REVERSED ].len ? cols[ alg_REVERSED ].base.u8[ 0 ] : false;
-
- rc = CIGAR_to_CG_Ops( cigOp, sizeof( cigOp ) / sizeof( cigOp[ 0 ] ), &opCnt, gap,
- cols[ alg_CIGAR ].base.str, cols[ alg_CIGAR ].len,
- &S_adjust, &CG_adjust, read, reversed );
- if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == rcFormat )
- {
- rc = 0;
- if ( style == 1 )
- goto CLEAN_CIGAR;
- else
- goto PRINT_CIGAR;
- }
- if ( rc != 0 )
- return 0;
-
- if ( CG_adjust == 0 )
- CG_adjust = ( gap[ 0 ] < opCnt ? cigOp[ gap[ 0 ] ].length : 0 )
- + ( gap[ 1 ] < opCnt ? cigOp[ gap[ 1 ] ].length : 0 )
- + ( gap[ 2 ] < opCnt ? cigOp[ gap[ 2 ] ].length : 0 );
-
- print_CG_cigar( __LINE__, cigOp, opCnt, NULL );
- if ( style == 1 )
- {
- static char newSeq[ 35 ];
- static char newQual[ 35 ];
- static char tags[ 35*22 + 1 ];
- unsigned const B_len = cigOp[ gap[ 0 ] ].length;
- unsigned const B_at = gap[ 0 ] < gap[ 2 ] ? 5 : 30;
-
- if ( 0 < B_len && B_len < 5 )
- {
- memcpy( newSeq, cols[ alg_READ ].base.v, 35 );
- memcpy( newQual, cols[ alg_SAM_QUALITY ].base.v, 35 );
-
- cols[ alg_CG_TAGS_STR ].base.v = tags;
-
- cols[ alg_READ ].base.v = newSeq;
- cols[ alg_READ ].len = 35 - B_len;
-
- cols[ alg_SAM_QUALITY ].base.v = newQual;
- cols[ alg_SAM_QUALITY ].len = 35 - B_len;
-
- /* nBnM -> nB0M */
- cigOp[ gap[ 0 ] + 1 ].length -= B_len;
- if ( gap[ 0 ] < gap[ 2 ] )
- {
- rc = string_printf( tags, sizeof( tags ), &sz, "\tGC:Z:%uS%uG%uS\tGS:Z:%.*s\tGQ:Z:%.*s",
- 5 - B_len, B_len, 30 - B_len, 2 * B_len, &newSeq[ 5 - B_len ], 2 * B_len, &newQual[ 5 - B_len ] );
- if ( rc == 0 )
- {
- cols[ alg_CG_TAGS_STR ].len = sz;
- }
- memmove( &cigOp[ gap[ 0 ] ],
- &cigOp[ gap[ 0 ] + 1 ],
- ( opCnt - ( gap[ 0 ] + 1 ) ) * sizeof( cigOp[ 0 ] ) );
- --opCnt;
- }
- else
- {
- rc = string_printf( tags, sizeof( tags ), &sz, "\tGC:Z:%uS%uG%uS\tGS:Z:%.*s\tGQ:Z:%.*s",
- 30 - B_len, B_len, 5 - B_len, 2 * B_len, &newSeq[ 30 - B_len ], 2 * B_len, &newQual[ 30 - B_len ] );
- if ( rc == 0 )
- {
- cols[ alg_CG_TAGS_STR ].len = sz;
- }
- memmove( &cigOp[ gap[ 0 ] ],
- &cigOp[ gap[ 0 ] + 1 ],
- ( opCnt - ( gap[ 0 ] + 1 ) ) * sizeof( cigOp[ 0 ] ) );
- --opCnt;
- }
- if ( rc == 0 )
- {
- for ( i = B_at; i < B_at + B_len; ++i )
- {
- int const Lq = newQual[ i - B_len ];
- int const Rq = newQual[ i ];
-
- if ( Lq <= Rq )
- {
- newSeq[ i - B_len ] = newSeq[ i ];
- newQual[ i - B_len ] = Rq;
- }
- else
- {
- newSeq[ i ] = newSeq[ i - B_len ];
- newQual[ i ] = Lq;
- }
- }
- memmove( &newSeq [ B_at ], &newSeq [ B_at + B_len ], 35 - B_at - B_len );
- memmove( &newQual[ B_at ], &newQual[ B_at + B_len ], 35 - B_at - B_len );
- }
- }
- else
- {
- int len = cigOp[ gap[ 0 ] ].length;
-
- cigOp[ gap[ 0 ] ].code = 'I';
- for (i = gap[0] + 1; i < opCnt && len > 0; ++i) {
- if (cigOp[i].length <= len){
- len -= cigOp[i].length;
- cigOp[i].length = 0;
- }
- else {
- cigOp[i].length -= len;
- len = 0;
- }
- }
- CG_adjust -= cigOp[ gap[ 0 ] ].length;
- }
- }
- if ( rc == 0 )
- {
- PRINT_CIGAR:
- CLEAN_CIGAR:
- print_CG_cigar( __LINE__, cigOp, opCnt, NULL );
- /* remove zero length ops */
- for ( j = i = 0; i < opCnt; )
- {
- if ( cigOp[ j ].length == 0)
- {
- ++j;
- --opCnt;
- continue;
- }
- cigOp[ i++ ] = cigOp[ j++ ];
- }
- print_CG_cigar( __LINE__, cigOp, opCnt, NULL );
- if ( cols[ alg_EDIT_DISTANCE ].len )
- {
- int const edit_distance = cols[ alg_EDIT_DISTANCE ].base.i32[ 0 ];
- int const adjusted = edit_distance + S_adjust - CG_adjust;
-
- newEditDistance = adjusted > 0 ? adjusted : 0;
- SAM_DUMP_DBG( 4, ( "NM: before: %u, after: %u(+%u-%u)\n", edit_distance, newEditDistance, S_adjust, CG_adjust ) );
- cols[ alg_EDIT_DISTANCE ].base.v = &newEditDistance;
- cols[ alg_EDIT_DISTANCE ].len = 1;
- }
- /* merge adjacent ops */
- for ( i = opCnt; i > 1; )
- {
- --i;
- if ( cigOp[ i - 1 ].code == cigOp[ i ].code )
- {
- cigOp[ i - 1 ].length += cigOp[ i ].length;
- memmove( &cigOp[ i ], &cigOp[ i + 1 ], ( opCnt - 1 - i ) * sizeof( cigOp[ 0 ] ) );
- --opCnt;
- }
- }
- print_CG_cigar( __LINE__, cigOp, opCnt, NULL );
- for ( i = j = 0; i < opCnt && rc == 0; ++i )
- {
- rc = string_printf( &newCIGAR[ j ], sizeof( newCIGAR ) - j, &sz, "%u%c", cigOp[ i ].length, cigOp[ i ].code);
- j += sz;
- }
- cols[ alg_CIGAR ].base.v = newCIGAR;
- cols[ alg_CIGAR ].len = j;
- }
- }
- return rc;
-}
-
-
-static bool DumpAlignedRow( SAM_dump_ctx_t *const ctx, DataSource *const ds,
- int64_t row,
- bool const primary,
- int const cg_style,
- rc_t *prc )
-{
- rc_t rc;
-
- /* if SEQ_SPOT_ID is open then skip rows with no SEQ_SPOT_ID */
- if ( ds->cols[ alg_SEQ_SPOT_ID ].idx )
- {
- rc = Cursor_Read( ds, row, alg_SEQ_SPOT_ID, 1 );
- if ( rc != 0 )
- {
- if ( !( GetRCObject( rc ) == rcRow && GetRCState( rc ) == rcNotFound ) )
- *prc = rc;
- return false;
- }
- if ( ds->cols[ alg_SEQ_SPOT_ID ].len == 0 ||
- ds->cols[ alg_SEQ_SPOT_ID ].base.i64[ 0 ] == 0 )
- return false;
- }
-
- /* skip rows that fail mate distance filter */
- if ( param->mp_dist_qty != 0 || param->mp_dist_unknown )
- {
- rc = Cursor_Read( ds, row, alg_TEMPLATE_LEN, 1 );
- if ( rc != 0 )
- {
- if ( !( GetRCObject( rc ) == rcRow && GetRCState( rc ) == rcNotFound ) )
- *prc = rc;
- return false;
- }
- if ( !AlignDistanceFilter( ds->cols ) )
- return false;
- }
-
- rc = Cursor_ReadAlign( &ds->curs, row, ds->cols, 0 );
- if ( rc != 0 )
- {
- if ( !( GetRCObject( rc ) == rcRow && GetRCState( rc ) == rcNotFound ) )
- *prc = rc;
- return false;
- }
-
- if ( param->fasta || param->fastq )
- {
- unsigned const read_id = ds->cols[ alg_SEQ_READ_ID ].base.v ? ds->cols[ alg_SEQ_READ_ID ].base.coord1[ 0 ] : 0;
-
- DumpAlignedFastX( ctx->pri.cols, row, read_id, primary, false );
- }
- else
- {
- if ( cg_style != 0 )
- {
- rc = GenerateCGData( ds->cols, cg_style );
- if ( rc != 0 )
- {
- *prc = rc;
- return false;
- }
- }
- DumpAlignedSAM(ctx, ds, row, ctx->readGroup, 0 );
- }
- *prc = 0;
- return true;
-}
-
-
-static rc_t DumpAlignedRowList( SAM_dump_ctx_t *const ctx, DataSource *const ds, SCol const *const ids,
- int64_t *rcount,
- bool const primary,
- int const cg_style,
- bool const noFilter )
-{
- rc_t rc;
- unsigned i;
- unsigned const n = ids->len;
-
- for ( i = 0; ( rc = Quitting() ) == 0 && i < n; ++i )
- {
- int64_t const row = ids->base.i64[ i ];
-
- if ( ! noFilter )
- rc = Cursor_Read( ds, row, alg_REGION_FILTER, ~(unsigned)0 );
-
- if ( rc == 0 )
- {
- if ( noFilter || AlignRegionFilter(ds->cols) )
- {
- if ( DumpAlignedRow( ctx, ds, row, primary, cg_style, &rc ) )
- ++*rcount;
- if ( rc != 0 )
- break;
- }
- }
- }
- return rc;
-}
-
-static rc_t DumpAlignedRowListIndirect( SAM_dump_ctx_t *const ctx,
- DataSource *const indirectSource,
- DataSource *const directSource,
- SCol const *const directIDs,
- int64_t *rcount,
- bool const primary,
- int const cg_style )
-{
- unsigned i;
-
- for ( i = 0; i < directIDs->len; ++i )
- {
- int64_t const row = directIDs->base.i64[ i ];
- rc_t rc = Cursor_Read( directSource, row, alg_REGION_FILTER, ~(unsigned)0 );
-
- if ( rc != 0 )
- return rc;
-
- if ( !AlignRegionFilter( directSource->cols ) )
- continue;
-
- rc = Cursor_Read( directSource, row, alg_EVIDENCE_ALIGNMENT_IDS, 1 );
- if ( rc != 0 )
- return rc;
-
- rc = DumpAlignedRowList( ctx, indirectSource, &directSource->cols[ alg_EVIDENCE_ALIGNMENT_IDS ],
- rcount, primary, cg_style, true );
- if ( rc != 0 )
- return rc;
- }
- return 0;
-}
-
-
-enum e_tables
-{
- primary_alignment,
- secondary_alignment,
- evidence_interval,
- evidence_alignment
-};
-
-enum e_IDS_opts
-{
- primary_IDS = ( 1 << primary_alignment ),
- secondary_IDS = ( 1 << secondary_alignment ),
- evidence_interval_IDS = ( 1 << evidence_interval ),
- evidence_alignment_IDS = ( 1 << evidence_alignment )
-};
-
-
-static rc_t DumpAlignedRowList_cb( SAM_dump_ctx_t *const ctx, TAlignedRegion const *const rgn,
- int options, int which, int64_t *rcount, SCol const *const IDS )
-{
- /*SAM_DUMP_DBG(2, ("row %s index range is [%lu:%lu] pos %lu\n",
- param->region[r].name, start, start + count - 1, cur_pos));*/
- switch ( which )
- {
- case primary_IDS:
- return DumpAlignedRowList( ctx, &ctx->pri, IDS, rcount, true, param->cg_style, false );
-
- case secondary_IDS:
- return DumpAlignedRowList( ctx, &ctx->sec, IDS, rcount, false, param->cg_style, false );
-
- case evidence_interval_IDS:
- {
- rc_t rc = 0;
-
- if ( ( options & evidence_interval_IDS ) != 0 )
- rc = DumpAlignedRowList( ctx, &ctx->evi, IDS, rcount, true, 0, false );
-
- if ( rc == 0 && ( options & evidence_alignment_IDS ) != 0 )
- rc = DumpAlignedRowListIndirect( ctx, &ctx->eva, &ctx->evi, IDS, rcount, true, param->cg_style );
-
- return rc;
- }
- }
- return RC( rcExe, rcTable, rcReading, rcParam, rcUnexpected );
-}
-
-
-typedef struct CigOps{
- char op;
- int8_t ref_sign; /*** 0;+1;-1; ref_offset = ref_sign * offset ***/
- int8_t seq_sign; /*** 0;+1;-1; seq_offset = seq_sign * offset ***/
- uint32_t oplen;
-} CigOps;
-
-static void SetCigOp(CigOps *dst,char op,uint32_t oplen)
-{
- dst->op = op;
- dst->oplen = oplen;
- switch(op) { /*MX= DN B IS PH*/
- case 'M': case 'X': case '=':
- dst->ref_sign=+1;
- dst->seq_sign=+1;
- break;
- case 'D': case 'N':
- dst->ref_sign=+1;
- dst->seq_sign= 0;
- break;
- case 'B':
- dst->ref_sign=-1;
- dst->seq_sign= 0;
- break;
- case 'S': case 'I':
- dst->ref_sign= 0;
- dst->seq_sign=+1;
- break;
- case 'P': case 'H':
- case 0: /** terminating op **/
- dst->ref_sign= 0;
- dst->seq_sign= 0;
- break;
- default:
- assert(0);
- break;
- }
-}
-
-static unsigned ExplodeCIGAR( CigOps dst[], unsigned len, char const cigar[], unsigned ciglen )
-{
- unsigned i;
- unsigned j;
- int opLen;
-
- for ( i = j = opLen=0; i < ciglen; i++) {
- if(isdigit(cigar[i])){
- opLen = opLen * 10 + cigar[i] - '0';
- } else {
- assert(isalpha(cigar[i]));
- SetCigOp(dst+j,cigar[i],opLen);
- opLen=0;
- j++;
- }
- }
- SetCigOp(dst+j,0,0);
- j++;
- return j;
-}
-
-
-#define CG_CIGAR_STRING_MAX (35 * 11 + 1)
-
-
-#if 0
-static unsigned FormatCIGAR( char dst[], unsigned cp, unsigned oplen, int opcode )
-{
- size_t sz;
- string_printf( dst + cp, CG_CIGAR_STRING_MAX - cp, &sz, "%u%c", oplen, opcode );
- return sz;
-}
-
-
-static unsigned DeletedCIGAR( char dst[], unsigned const cp, int opcode, unsigned D,
- uint32_t const Op[], int ri, unsigned len )
-{
- unsigned i;
- unsigned curPos = 0;
- int curOp = opcode;
- unsigned oplen = 0;
-
- for ( i = 0; i < D && ri < len; ++i, ++ri )
- {
- uint32_t const op = Op[ ri ];
- int const d = op >> 2;
- int const m = op & 1;
- int const nxtOp = m ? opcode : 'P';
-
- if ( d != 0 )
- {
- if ( oplen != 0 )
- curPos += FormatCIGAR( dst, curPos + cp, oplen, curOp );
- curPos += FormatCIGAR( dst, cp, d, 'D' );
- oplen = 0;
- }
- if ( oplen == 0 || nxtOp == curOp )
- {
- ++oplen;
- }
- else
- {
- curPos += FormatCIGAR( dst, curPos + cp, oplen, curOp );
- oplen = 1;
- }
- curOp = nxtOp;
- }
-
- if ( oplen != 0 && curOp != opcode)
- {
- curPos += FormatCIGAR( dst, curPos + cp, oplen, curOp );
- oplen = 0;
- }
- if ( i < D )
- oplen += D - i;
- if ( oplen )
- curPos += FormatCIGAR( dst, curPos + cp, oplen, opcode );
- return curPos;
-}
-#endif
-
-static char merge_M_type_ops(char a,char b)
-{ /*MX=*/
- char c=0;
- switch(b){
- case 'X':
- switch(a){
- case '=': c='X'; break;
- case 'X': c='M'; break; /**we don't know - 2X may create '=' **/
- case 'M': c='M'; break;
- }
- break;
- case 'M':
- c='M';
- break;
- case '=':
- c=a;
- break;
- }
- assert(c!=0);
- return c;
-}
-
-
-static unsigned CombineCIGAR( char dst[], CigOps const seqOp[], unsigned seq_len,
- int refPos, CigOps const refOp[], unsigned ref_len )
-{
- bool done=false;
- unsigned ciglen=0,last_ciglen=0;
- char last_cig_op;
- uint32_t last_cig_oplen=0;
- int si=0,ri=0;
- CigOps seq_cop={0,0,0,0},ref_cop={0,0,0,0};
- int seq_pos=0; /** seq_pos is tracked roughly - with every extraction from seqOp **/
- int ref_pos=0; /** ref_pos is tracked precisely - with every delta and consumption in cigar **/
- int delta=refPos; /*** delta in relative positions of seq and ref **/
- /*** when delta < 0 - rewind or extend the reference ***/
- /*** wher delta > 0 - skip reference ***/
-#define MACRO_BUILD_CIGAR(OP,OPLEN) \
- if( last_cig_oplen > 0 && last_cig_op == OP){ \
- last_cig_oplen += OPLEN; \
- ciglen = last_ciglen + sprintf(dst+last_ciglen,"%d%c",last_cig_oplen,last_cig_op); \
- } else { \
- last_ciglen = ciglen; \
- last_cig_oplen = OPLEN; \
- last_cig_op = OP; \
- ciglen = ciglen + sprintf(dst+last_ciglen,"%d%c",last_cig_oplen,last_cig_op); \
- }
- while(!done){
- while(delta < 0){
- ref_pos += delta; /** we will make it to back up this way **/
- if(ri > 0){ /** backing up on ref if possible ***/
- int avail_oplen = refOp[ri-1].oplen - ref_cop.oplen;
- if(avail_oplen > 0 ){
- if((-delta) <= avail_oplen * ref_cop.ref_sign){ /*** rewind within last operation **/
- ref_cop.oplen -= delta;
- delta -= delta * ref_cop.ref_sign;
- } else { /*** rewind the whole ***/
- ref_cop.oplen += avail_oplen;
- delta += avail_oplen * ref_cop.ref_sign;
- }
- } else {
- ri--;
- /** pick the previous as used up **/
- ref_cop = refOp[ri-1];
- ref_cop.oplen =0;
- }
- } else { /** extending the reference **/
- SetCigOp(&ref_cop,'=',ref_cop.oplen - delta);
- delta = 0;
- }
- ref_pos -= delta;
- }
- if(ref_cop.oplen == 0){ /*** advance the reference ***/
- ref_cop = refOp[ri++];
- if(ref_cop.oplen == 0) { /** extending beyond the reference **/
- SetCigOp(&ref_cop,'=',1000);
- }
- assert(ref_cop.oplen > 0 );
- }
- if(delta > 0){ /*** skip refOps ***/
- ref_pos += delta; /** may need to back up **/
- if(delta >= ref_cop.oplen){ /** full **/
- delta -= ref_cop.oplen * ref_cop.ref_sign;
- ref_cop.oplen = 0;
- } else { /** partial **/
- ref_cop.oplen -= delta;
- delta -= delta * ref_cop.ref_sign;
- }
- ref_pos -= delta; /** if something left - restore ***/
- continue;
- }
- /*** seq and ref should be synchronized here **/
- assert(delta == 0);
- if(seq_cop.oplen == 0){ /*** advance sequence ***/
- if(seq_pos < seq_len){
- seq_cop = seqOp[si++];
- assert(seq_cop.oplen > 0 );
- seq_pos += seq_cop.oplen * seq_cop.seq_sign;
- } else {
- done=true;
- }
- }
- if(!done){
- int seq_seq_step = seq_cop.oplen * seq_cop.seq_sign; /** sequence movement**/
- int seq_ref_step = seq_cop.oplen * seq_cop.ref_sign; /** influence of sequence movement on intermediate reference **/
- int ref_seq_step = ref_cop.oplen * ref_cop.seq_sign; /** movement of the intermediate reference ***/
- int ref_ref_step = ref_cop.oplen * ref_cop.ref_sign; /** influence of the intermediate reference movement on final reference ***/
- assert( ref_ref_step >= 0); /** no B in the reference **/
- if( seq_ref_step <= 0){ /** BSIPH in the sequence against anything ***/
- MACRO_BUILD_CIGAR( seq_cop.op, seq_cop.oplen );
- seq_cop.oplen = 0;
- delta = seq_ref_step; /** if negative - will force rewind next cycle **/
- } else if(ref_ref_step <= 0){ /** MX=DN against SIPH in the reference***/
- if(ref_seq_step == 0){ /** MX=DN against PH **/
- MACRO_BUILD_CIGAR( ref_cop.op,ref_cop.oplen);
- ref_cop.oplen = 0;
- } else {
- int min_len = (seq_cop.oplen<ref_cop.oplen)?seq_cop.oplen:ref_cop.oplen;
- if(seq_seq_step == 0){ /** DN agains SI **/
- MACRO_BUILD_CIGAR('P',min_len);
- } else { /** MX= agains SI ***/
- MACRO_BUILD_CIGAR( ref_cop.op,min_len);
- }
- seq_cop.oplen -= min_len;
- ref_cop.oplen -= min_len;
- }
- } else { /*MX=DN against MX=DN*/
- int min_len = (seq_cop.oplen<ref_cop.oplen)?seq_cop.oplen:ref_cop.oplen;
- if(seq_seq_step == 0){ /* DN against MX=DN */
- if(ref_seq_step == 0){ /** padding DN against DN **/
- MACRO_BUILD_CIGAR('P',min_len);
- ref_cop.oplen -= min_len;
- seq_cop.oplen -= min_len;
- } else { /* DN against MX= **/
- MACRO_BUILD_CIGAR(seq_cop.op,min_len);
- seq_cop.oplen -= min_len;
- }
- } else if (ref_cop.seq_sign == 0){ /* MX= against DN - always wins */
- MACRO_BUILD_CIGAR(ref_cop.op,min_len);
- ref_cop.oplen -= min_len;
- } else { /** MX= against MX= ***/
- MACRO_BUILD_CIGAR(merge_M_type_ops(seq_cop.op,ref_cop.op),min_len);
- ref_cop.oplen -= min_len;
- seq_cop.oplen -= min_len;
- }
- ref_pos += min_len;
- }
- }
- }
- return ciglen;
-}
-
-
-static rc_t DumpCGSAM( SAM_dump_ctx_t *const ctx, TAlignedRegion const *const rgn, int options,
- int which, int64_t *rows, SCol const *const ids )
-{
- if ( which == evidence_interval_IDS )
- {
- rc_t rc = 0;
- unsigned i;
-
- for ( i = 0; i < ids->len; ++i )
- {
- int64_t const rowInterval = ids->base.i64[ i ];
-
- rc = Cursor_Read( &ctx->evi, rowInterval, alg_REGION_FILTER, ~(0u) );
- if ( rc == 0 )
- {
- if ( AlignRegionFilter( ctx->evi.cols ) )
- {
- rc = Cursor_ReadAlign( &ctx->evi.curs, rowInterval, ctx->evi.cols, 0 );
- if ( rc == 0 )
- {
- unsigned const evidence = ctx->evi.cols[ alg_EVIDENCE_ALIGNMENT_IDS ].len;
- if ( evidence != 0 )
- {
- INSDC_coord_len const *const refLen = ctx->evi.cols[ alg_READ_LEN ].base.coord_len;
- INSDC_coord_len const *const cigLen = ctx->evi.cols[ alg_CIGAR_LEN ].base.coord_len;
- unsigned const ploids = ctx->evi.cols[ alg_CIGAR_LEN ].len;
- unsigned const totalIntervalReadLen = ctx->evi.cols[ alg_READ ].len;
- CigOps refCigOps_stack[1024];
- CigOps *refCigOps_heap=NULL;
- CigOps *refCigOps;
-
- if( totalIntervalReadLen > 1000){
- refCigOps=refCigOps_heap = malloc( sizeof( refCigOps[ 0 ] ) * (totalIntervalReadLen+ploids) );
- } else {
- refCigOps=refCigOps_stack;
- }
-
- if ( refCigOps != NULL )
- {
- unsigned j;
- unsigned start;
- unsigned cigop_starts[256];
-
- assert(ploids < 256);
-
- for ( start = j = 0, cigop_starts[0]=0; j < ploids; ++j )
- {
- unsigned len = cigLen[ j ];
- cigop_starts[j+1] = cigop_starts[j] + ExplodeCIGAR( refCigOps + cigop_starts[j], refLen[ j ],
- ctx->evi.cols[ alg_CIGAR ].base.str + start, len );
- start += len;
- }
-
- for ( j = 0; j < evidence; ++j )
- {
- int64_t const rowAlign = ctx->evi.cols[ alg_EVIDENCE_ALIGNMENT_IDS ].base.i64[ j ];
-
- rc = Cursor_ReadAlign( &ctx->eva.curs, rowAlign, ctx->eva.cols, 0 );
- if ( rc == 0 )
- {
- if(param->cg_style != 0)
- rc = GenerateCGData( ctx->eva.cols, param->cg_style );
- if ( rc == 0 )
- {
- int const ploidy = ctx->eva.cols[ alg_REF_PLOIDY ].base.u32[ 0 ];
- int const readLen = ctx->eva.cols[ alg_READ ].len;
- INSDC_coord_zero refPos = ctx->eva.cols[ alg_REF_POS ].base.coord0[ 0 ];
- CigOps op[ 36 ];
- char cigbuf[ CG_CIGAR_STRING_MAX ];
-
- memset( cigbuf, 0, CG_CIGAR_STRING_MAX );
- ExplodeCIGAR( op, readLen, ctx->eva.cols[ alg_CIGAR ].base.str,
- ctx->eva.cols[ alg_CIGAR ].len );
- ctx->eva.cols[ alg_CIGAR ].len = CombineCIGAR( cigbuf, op, readLen, refPos,
- refCigOps + cigop_starts[ ploidy - 1 ], refLen[ ploidy - 1 ] );
- ctx->eva.cols[ alg_CIGAR ].base.str = cigbuf;
- ctx->eva.cols[ alg_REF_POS ].base.v = &refPos;
- refPos += ctx->evi.cols[ alg_REF_POS ].base.coord0[ 0 ] ;
- if(refPos < 0){
- ReferenceObj const *r = NULL;
- rc = ReferenceList_Find( gRefList, &r,
- ctx->evi.cols[ alg_REF_NAME ].base.str,
- ctx->evi.cols[ alg_REF_NAME ].len );
- if(rc == 0){
- bool circular=false;
- rc=ReferenceObj_Circular(r, &circular);
- if(rc == 0 && circular){
- INSDC_coord_len len;
- rc=ReferenceObj_SeqLength(r,&len);
- if(rc == 0)
- refPos += len;
- }
- ReferenceObj_Release(r);
- }
- }
- DumpAlignedSAM(ctx, &ctx->eva, rowAlign, ctx->readGroup, 1);
- ++*rows;
- }
- else
- break;
- }
- else
- break;
- }
- if(refCigOps_heap) free( refCigOps_heap );
- }
- else
- rc = RC( rcExe, rcTable, rcReading, rcMemory, rcExhausted );
- }
- }
- }
- }
- else
- break;
- rc = rc ? rc : Quitting();
- if ( rc != 0 )
- break;
- }
- return rc;
- }
- return RC( rcExe, rcTable, rcReading, rcParam, rcUnexpected );
-}
-
-
-static rc_t ForEachAlignedRegion( SAM_dump_ctx_t *const ctx, enum e_IDS_opts const Options,
- rc_t ( *user_func )( SAM_dump_ctx_t *ctx, TAlignedRegion const *rgn, int options, int which, int64_t *rows, SCol const *IDS ) )
-{
- SCol cols[] =
- {
- { "MAX_SEQ_LEN", 0, { NULL }, 0, false },
- { "OVERLAP_REF_POS", 0, { NULL }, 0, true },
- { "PRIMARY_ALIGNMENT_IDS", 0, { NULL }, 0, false },
- { "SECONDARY_ALIGNMENT_IDS", 0, { NULL }, 0, true },
- { "EVIDENCE_INTERVAL_IDS", 0, { NULL }, 0, true },
- { NULL, 0, { NULL }, 0, false }
- };
- enum eref_col
- {
- ref_MAX_SEQ_LEN = 0,
- ref_OVERLAP_REF_LEN,
- ref_PRIMARY_ALIGNMENT_IDS,
- ref_SECONDARY_ALIGNMENT_IDS,
- ref_EVIDENCE_INTERVAL_IDS
- };
-
- KIndex const *iname = NULL;
- int64_t rows = 0;
- rc_t rc = 0;
- int options = Options & 7;
-
- if ( Options & evidence_alignment_IDS )
- options |= evidence_interval_IDS;
-
- if ( ( options & primary_IDS ) == 0 )
- cols[ ref_PRIMARY_ALIGNMENT_IDS ].name = "";
- if ( ( options & secondary_IDS ) == 0 )
- cols[ ref_SECONDARY_ALIGNMENT_IDS ].name = "";
- if ( ( options & evidence_interval_IDS ) == 0 )
- cols[ ref_EVIDENCE_INTERVAL_IDS ].name = "";
-
- ctx->ref.cols = cols;
-
- rc = VTableOpenIndexRead( ctx->ref.tbl.vtbl, &iname, "i_name" );
- if ( rc == 0 )
- {
- rc = Cursor_Open( &ctx->ref.tbl, &ctx->ref.curs, ctx->ref.cols, NULL );
- if ( rc == 0 )
- {
- int64_t rowid = 1;
- uint64_t count = 0;
- char refname[ 1024 ];
- size_t sz;
-
- if ( ctx->ref.cols[ ref_PRIMARY_ALIGNMENT_IDS ].idx == 0 )
- options &= ~primary_IDS;
- if ( ctx->ref.cols[ ref_SECONDARY_ALIGNMENT_IDS ].idx == 0 )
- options &= ~secondary_IDS;
- if ( ctx->ref.cols[ ref_EVIDENCE_INTERVAL_IDS ].idx == 0 )
- options &= ~evidence_interval_IDS;
-
- /* new: if we have a list of regions, reset the their print-flag */
- if ( param->region_qty > 0 )
- {
- unsigned r;
- for ( r = 0; r < param->region_qty; ++r )
- {
- param->region[ r ].printed = 0;
- }
- }
-
- while ( ( rc = KIndexProjectText(iname, rowid + count, &rowid, &count, refname, sizeof( refname ), &sz ) ) == 0 )
- {
- bool include;
- unsigned r;
- unsigned max_to = UINT_MAX;
- unsigned min_from = 0;
-
- for ( include = false, r = 0; r < param->region_qty; ++r )
- {
- if ( sz == strlen( param->region[ r ].name ) &&
- memcmp( param->region[ r ].name, refname, sz ) == 0 )
- {
- include = true;
- max_to = param->region[ r ].max_to;
- min_from = param->region[ r ].min_from;
- param->region[ r ].printed++; /* new: mark a region as printed */
- break;
- }
- }
-
- if ( param->region_qty == 0 || include )
- {
- int64_t const endrow_exclusive = rowid + count;
- unsigned m;
- unsigned k;
-
- for ( k = 0, m = 1; rc == 0 && k < 3; ++k, m <<= 1 )
- {
- if ( m & options )
- {
- unsigned lookback = 0;
- int64_t row;
- int32_t row_start_offset;
- unsigned pos;
- unsigned const maxseqlen = 5000; /***** TODO: this code is to be rewritten anyway ****/
-
- if ( param->region_qty )
- {
- if ( ctx->ref.cols[ ref_OVERLAP_REF_LEN ].idx )
- {
- rc = Cursor_Read( &ctx->ref, rowid, 0, ~(0u) );
- if ( rc != 0 )
- break;
- if ( ctx->ref.cols[ ref_OVERLAP_REF_LEN ].len > k )
- {
- unsigned const overlap = ctx->ref.cols[ ref_OVERLAP_REF_LEN ].base.coord0[ k ];
-
- if ( overlap != 0 )
- {
-
- assert( overlap < rowid * maxseqlen );
- lookback = ( rowid - ( overlap / maxseqlen ) );
- }
- }
- }
- else
- lookback = 1;
- }
- row_start_offset = min_from/maxseqlen - lookback;
- if(row_start_offset < 0) row_start_offset=0;
-
-
- for ( pos = row_start_offset*maxseqlen, row = rowid + row_start_offset; rc == 0 && row < endrow_exclusive; ++row )
- {
- if ( row < 1 )
- {
- row = 0;
- continue;
- }
- rc = Cursor_Read( &ctx->ref, row, 0, ~(0u) );
- if ( rc != 0 )
- break;
- if ( ctx->ref.cols[ ref_PRIMARY_ALIGNMENT_IDS + k ].len > 0 )
- rc = user_func( ctx, ¶m->region[r], Options, m, &rows,
- &ctx->ref.cols[ ref_PRIMARY_ALIGNMENT_IDS + k ] );
- pos += ctx->ref.cols[ ref_MAX_SEQ_LEN ].base.u32[ 0 ];
- if ( pos >= max_to )
- break;
- if ( param->test_rows != 0 && rows > param->test_rows )
- break;
- }
- }
- }
- }
- if ( param->test_rows != 0 && rows > param->test_rows )
- break;
- if ( rc != 0 )
- break;
- } /* while */
-
- /* new: walk the list of regions and report what was NOT printed ( found ) ... */
- if ( param->region_qty > 0 && GetRCState( rc ) != rcCanceled )
- {
- unsigned r;
- for ( r = 0; r < param->region_qty; ++r )
- {
- if ( param->region[ r ].printed == 0 )
- {
- (void)PLOGMSG( klogWarn, ( klogWarn, "reference >$(a)< not found", "a=%s", param->region[ r ].name ) );
- }
- }
- }
-
- if ( GetRCObject( rc ) == rcId && GetRCState( rc ) == rcNotFound )
- {
- rc = 0;
- }
-
- }
- }
- Cursor_Close( &ctx->ref.curs );
- KIndexRelease( iname );
-
- return rc;
-}
-
-
-static rc_t DumpAlignedTable( SAM_dump_ctx_t *const ctx, DataSource *const ds,
- bool const primary, int const cg_style, unsigned *const rcount )
-{
- unsigned i;
- int64_t start;
- uint64_t count;
- rc_t rc = VCursorIdRange( ds->curs.vcurs, 0, &start, &count );
-
- if ( rc != 0 )
- return rc;
- for ( i = 0; i != (unsigned)count; ++i )
- {
- if ( DumpAlignedRow(ctx, ds, start + i, primary, cg_style, &rc ) )
- ++*rcount;
- if ( rc != 0 || ( rc = Quitting() ) != 0 )
- return rc;
- if ( param->test_rows && *rcount > param->test_rows )
- break;
- }
- return 0;
-}
-
-
-static rc_t DumpUnsorted( SAM_dump_ctx_t *const ctx )
-{
- rc_t rc = 0;
- unsigned rcount;
-
- if ( rc == 0 && param->cg_evidence )
- {
- SAM_DUMP_DBG( 2, ( "%s EVIDENCE_INTERVAL\n", ctx->accession ) );
- rcount = 0;
- rc = DumpAlignedTable( ctx, &ctx->evi, false, 0, &rcount );
- (void)PLOGMSG( klogInfo, ( klogInfo, "$(a): $(c) allele sequences", "a=%s,c=%lu", ctx->accession, rcount ) );
- }
- if ( rc == 0 && param->cg_ev_dnb )
- {
- SAM_DUMP_DBG( 2, ( "%s EVIDENCE_ALIGNMENT\n", ctx->accession ) );
- rcount = 0;
- rc = DumpAlignedTable( ctx, &ctx->eva, false, param->cg_style, &rcount );
- (void)PLOGMSG( klogInfo, ( klogInfo, "$(a): $(c) support sequences", "a=%s,c=%lu", ctx->accession, rcount ) );
- }
- if ( rc == 0 && ctx->pri.curs.vcurs )
- {
- SAM_DUMP_DBG( 2, ( "%s PRIMARY_ALIGNMENT\n", ctx->accession ) );
- rcount = 0;
- rc = DumpAlignedTable( ctx, &ctx->pri, true, param->cg_style, &rcount );
- (void)PLOGMSG( klogInfo, ( klogInfo, "$(a): $(c) primary sequences", "a=%s,c=%lu", ctx->accession, rcount ) );
- }
- if ( rc == 0 && ctx->sec.curs.vcurs )
- {
- SAM_DUMP_DBG( 2, ( "%s SECONDARY_ALIGNMENT\n", ctx->accession ) );
- rcount = 0;
- rc = DumpAlignedTable( ctx, &ctx->sec, false, param->cg_style, &rcount );
- (void)PLOGMSG( klogInfo, ( klogInfo, "$(a): $(c) secondary sequences", "a=%s,c=%lu", ctx->accession, rcount ) );
- }
- return rc;
-}
-
-
-static rc_t DumpUnaligned( SAM_dump_ctx_t *const ctx, bool const aligned )
-{
- rc_t rc = 0;
- int64_t start = 0;
- uint64_t count = 0;
-
- rc = VCursorIdRange( ctx->seq.curs.vcurs, 0, &start, &count );
- if ( rc == 0 )
- {
- uint64_t rcount = 0;
- if ( param->test_rows != 0 && count > param->test_rows )
- {
- count = param->test_rows;
- }
- SAM_DUMP_DBG( 2, ( "%s SEQUENCE table range from %ld qty %lu\n", ctx->accession, start, count ) );
- while ( count > 0 && rc == 0 )
- {
- uint32_t i;
-
- if ( ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].idx == 0)
- {
- rc = DumpUnalignedReads( ctx, NULL, start, &rcount );
- }
- else
- {
- /* avoid reading whole sequence cursor data unnecessarily */
- rc = Cursor_Read( &ctx->seq, start, seq_PRIMARY_ALIGNMENT_ID, ~(unsigned)0 );
- if ( rc == 0 )
- {
- /* find if its completely unaligned */
- int64_t min_prim_id = 0;
- bool has_unaligned = false;
- for ( i = 0; rc == 0 && i < ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].len; i++ )
- {
- int64_t x = ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[ i ];
- has_unaligned |= x == 0;
- if ( ( min_prim_id == 0 && x != 0 ) || min_prim_id < x )
- {
- min_prim_id = x;
- }
- }
- if ( min_prim_id == 0 )
- {
- /* fully unaligned spot */
- rc = DumpUnalignedReads( ctx, NULL, start, &rcount );
- }
- else if ( has_unaligned && !param->unaligned_spots )
- {
- if ( rc == 0 )
- {
-#if USE_MATE_CACHE
- uint64_t val;
- rc = Cache_Get( &ctx->pri.curs, min_prim_id, &val );
- if ( rc == 0 )
- {
- ctx->pri.cols[ alg_REF_POS ].len = 0;
- Cache_Unpack( val, 0, &ctx->pri.curs, ctx->pri.cols );
- ctx->pri.cols[ alg_SEQ_SPOT_ID ].base.i64 = &start;
- ctx->pri.cols[ alg_SEQ_SPOT_ID ].len = 1;
- memcpy( &ctx->pri.cols[ alg_REF_NAME ], &ctx->pri.cols[ alg_MATE_REF_NAME ], sizeof( SCol ) );
- memcpy( &ctx->pri.cols[ alg_REF_SEQ_ID ], &ctx->pri.cols[ alg_MATE_REF_NAME ], sizeof( SCol ) );
- memcpy( &ctx->pri.cols[ alg_REF_POS ], &ctx->pri.cols[ alg_MATE_REF_POS ], sizeof( SCol ) );
- }
- else if ( !( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == rcItem ) )
- {
- break;
- }
- else
- {
-#endif /* USE_MATE_CACHE */
- rc = Cursor_ReadAlign( &ctx->pri.curs, min_prim_id, ctx->pri.cols, alg_MATE_ALIGN_ID );
-#if USE_MATE_CACHE
- }
-#endif /* USE_MATE_CACHE */
- rc = rc ? rc : DumpUnalignedReads( ctx, ctx->pri.cols, min_prim_id, &rcount );
- }
- }
- } else if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == rcRow )
- {
- rc = 0;
- }
- }
- start++;
- count--;
- rc = rc ? rc : Quitting();
- }
- (void)PLOGMSG( klogInfo, ( klogInfo, "$(a): $(c) unaligned sequences", "a=%s,c=%lu", ctx->accession, rcount ) );
- }
- return rc;
-}
-
-#if USE_MATE_CACHE
-static rc_t FlushUnalignedRead_cb (uint64_t key, bool value,void *user_data)
-{
- rc_t rc;
- int64_t spot_id = (int64_t) key;
- int64_t aligned_mate_id = (int64_t) value;
- SAM_dump_ctx_t *const ctx = user_data;
- rc = Cursor_Read( &ctx->seq, spot_id, seq_PRIMARY_ALIGNMENT_ID, ~(unsigned)0 );
- if ( rc == 0 ) {
- uint64_t rcount;
- uint64_t val;
- assert(ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].len == 2);
- if(ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[0]==0){
- aligned_mate_id = ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[1];
- assert(aligned_mate_id != 0);
- } else if(ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[1] == 0){
- aligned_mate_id = ctx->seq.cols[ seq_PRIMARY_ALIGNMENT_ID ].base.i64[0];
- assert(aligned_mate_id != 0);
- } else {
- assert(0);
- }
- rc = Cache_Get( &ctx->pri.curs, aligned_mate_id, &val );
- if ( rc == 0 ) {
- ctx->pri.cols[ alg_REF_POS ].len = 0;
- Cache_Unpack( val, 0, &ctx->pri.curs, ctx->pri.cols );
- ctx->pri.cols[ alg_SEQ_SPOT_ID ].base.i64 = &spot_id;
- ctx->pri.cols[ alg_SEQ_SPOT_ID ].len = 1;
- memcpy( &ctx->pri.cols[ alg_REF_NAME ], &ctx->pri.cols[ alg_MATE_REF_NAME ], sizeof( SCol ) );
- memcpy( &ctx->pri.cols[ alg_REF_SEQ_ID ], &ctx->pri.cols[ alg_MATE_REF_NAME ], sizeof( SCol ) );
- memcpy( &ctx->pri.cols[ alg_REF_POS ], &ctx->pri.cols[ alg_MATE_REF_POS ], sizeof( SCol ) );
- rc =DumpUnalignedReads( ctx, ctx->pri.cols, aligned_mate_id, &rcount );
- }
- }
- return rc;
-}
-static rc_t FlushUnaligned( SAM_dump_ctx_t *const ctx, const SCursCache * cache )
-{
- return KVectorVisitBool (cache->cache_unaligned_mate, false, FlushUnalignedRead_cb, ctx );
-}
-#endif
-
-static rc_t DumpHeader( SAM_dump_ctx_t const *ctx )
-{
- bool reheader = param->reheader;
- rc_t rc = 0;
-
- if ( !reheader )
- {
- if ( ctx->db )
- {
- /* grab header from db meta node */
- KMetadata const *meta;
-
- rc = VDatabaseOpenMetadataRead( ctx->db, &meta );
- if ( rc == 0 )
- {
- KMDataNode const *node;
-
- rc = KMetadataOpenNodeRead( meta, &node, "BAM_HEADER" );
- KMetadataRelease( meta );
-
- if ( rc == 0 )
- {
- size_t offset = 0;
-
- for ( offset = 0; ; )
- {
- size_t remain;
- size_t nread;
- char buffer[ 4096 ];
-
- rc = KMDataNodeRead( node, offset, buffer, sizeof(buffer), &nread, &remain );
- if ( rc != 0 )
- break;
- BufferedWriter( NULL, buffer, nread, NULL );
- if ( remain == 0 )
- {
- if ( buffer[nread - 1] != '\n' )
- {
- buffer[ 0 ] = '\n';
- BufferedWriter( NULL, buffer, 1, NULL );
- }
- break;
- }
- offset += nread;
- }
- KMDataNodeRelease(node);
- }
- else if ( GetRCState( rc ) == rcNotFound )
- {
- reheader = true;
- rc = 0;
- }
- }
- }
- else
- {
- reheader = true;
- rc = 0;
- }
- }
-
- if ( rc == 0 && reheader )
- {
- KOutMsg( "@HD\tVN:1.3\n" );
- if ( gRefList )
- rc = RefSeqPrint();
- if ( ctx->readGroup )
- ReadGroup_print( ctx->readGroup );
- if ( ctx->seq.tbl.vtbl )
- rc = DumpReadGroups( &ctx->seq.tbl );
- }
- if ( rc == 0 && param->comments )
- {
- unsigned i;
-
- for( i = 0; param->comments[ i ]; ++i )
- KOutMsg( "@CO\t%s\n", param->comments[ i ] );
- }
- return rc;
-}
-
-
-static rc_t DumpDB( SAM_dump_ctx_t *const ctx )
-{
- rc_t rc = 0;
-
- if ( ctx->ref.tbl.vtbl != NULL )
- rc = ReferenceList_MakeTable( &gRefList, ctx->ref.tbl.vtbl, 0, CURSOR_CACHE, NULL, 0 );
- if ( !param->noheader )
- rc = DumpHeader( ctx );
- if ( rc == 0 )
- {
- if ( param->region_qty ){
- rc = ForEachAlignedRegion( ctx
- , ( param->primaries ? primary_IDS : 0 )
- | ( param->secondaries ? secondary_IDS : 0 )
- | ( param->cg_evidence ? evidence_interval_IDS : 0 )
- | ( param->cg_ev_dnb ? evidence_alignment_IDS : 0 )
- , DumpAlignedRowList_cb );
-#if USE_MATE_CACHE
- if ( rc == 0 && param->unaligned ){
- rc = FlushUnaligned( ctx,ctx->pri.curs.cache);
- }
-#endif
- }
- if ( param->cg_sam )
- rc = ForEachAlignedRegion( ctx, evidence_interval_IDS, DumpCGSAM );
-
- if ( param->region_qty == 0 )
- {
- rc = DumpUnsorted( ctx );
- if ( rc == 0 && param->unaligned )
- rc = DumpUnaligned( ctx, ctx->pri.tbl.vtbl != NULL );
- }
- }
- ReferenceList_Release( gRefList );
- return rc;
-}
-
-
-static rc_t DumpTable( SAM_dump_ctx_t *ctx )
-{
- rc_t rc = 0;
-
- if ( !param->noheader )
- rc = DumpHeader( ctx );
- if ( rc == 0 )
- rc = DumpUnaligned( ctx, false );
- return rc;
-}
-
-
-static rc_t ProcessTable( VDBManager const *mgr, char const fullPath[],
- char const accession[], char const readGroup[] )
-{
- VTable const *tbl;
- rc_t rc = VDBManagerOpenTableRead( mgr, &tbl, 0, "%s", fullPath );
-
- if ( rc != 0 )
- {
- VSchema *schema;
-
- rc = VDBManagerMakeSRASchema( mgr, &schema );
- if ( rc == 0 )
- {
- rc = VDBManagerOpenTableRead( mgr, &tbl, schema, "%s", fullPath );
- VSchemaRelease( schema );
- }
- }
-
- if ( rc == 0 )
- {
- SAM_dump_ctx_t ctx;
- SCol seq_cols[ sizeof( gSeqCol ) / sizeof( gSeqCol[ 0 ] ) ];
-
- memset( &ctx, 0, sizeof( ctx ) );
-
- ctx.fullPath = fullPath;
- ctx.accession = accession;
- ctx.readGroup = readGroup;
-
- DATASOURCE_INIT( ctx.seq, accession );
- ctx.seq.tbl.vtbl = tbl;
- ctx.seq.type = edstt_Sequence;
-
- ReportResetTable( fullPath, tbl );
-
- ctx.seq.cols = seq_cols;
- memcpy( ctx.seq.cols, gSeqCol, sizeof( gSeqCol ) );
- rc = Cursor_Open( &ctx.seq.tbl, &ctx.seq.curs, ctx.seq.cols, NULL );
- if ( rc == 0 ) {
- rc = DumpTable( &ctx );
- Cursor_Close(&ctx.seq.curs);
- }
- VTableRelease( tbl );
- }
- return rc;
-}
-
-
-static void SetupColumns( DataSource *ds, enum eDSTableType type )
-{
- memcpy( ds->cols, g_alg_col_tmpl, sizeof( g_alg_col_tmpl ) );
-
- ds->type = type;
- if ( param->use_seqid )
- ds->cols[ alg_MATE_REF_NAME ].name = "MATE_REF_SEQ_ID";
- else
- ds->cols[ alg_MATE_REF_NAME ].name = "MATE_REF_NAME";
-
- if ( param->fasta || param->fastq )
- ds->cols[ alg_READ ].name = "RAW_READ";
- else if ( param->hide_identical )
- ds->cols[ alg_READ ].name = "MISMATCH_READ";
- else
- ds->cols[ alg_READ ].name = "READ";
-
- if ( param->long_cigar )
- {
- ds->cols[ alg_CIGAR ].name = "CIGAR_LONG";
- ds->cols[ alg_CIGAR_LEN ].name = "CIGAR_LONG_LEN";
- }
- else
- {
- ds->cols[ alg_CIGAR ].name = "CIGAR_SHORT";
- ds->cols[ alg_CIGAR_LEN ].name = "CIGAR_SHORT_LEN";
- }
-
- if(param->qualQuant && param->qualQuantSingle){ /** we don't really need quality ***/
- ds->cols[ alg_SAM_QUALITY ].name = "";
- }
-
- switch ( type )
- {
- case edstt_EvidenceInterval:
-
- ds->cols[ alg_SEQ_SPOT_ID ].name = "";
- ds->cols[ alg_SEQ_NAME ].name = "";
- ds->cols[ alg_SAM_QUALITY ].name = "";
- ds->cols[ alg_SPOT_GROUP ].name = "";
- ds->cols[ alg_SEQ_SPOT_GROUP].name = "";
- ds->cols[ alg_SEQ_READ_ID ].name = "";
- ds->cols[ alg_MATE_ALIGN_ID ].name = "";
- ds->cols[ alg_MATE_REF_ID ].name = "";
- ds->cols[ alg_MATE_REF_NAME ].name = "";
- ds->cols[ alg_SAM_FLAGS ].name = "";
- ds->cols[ alg_MATE_REF_POS ].name = "";
- ds->cols[ alg_ALIGN_GROUP ].name = "";
- ds->cols[ alg_TEMPLATE_LEN ].name = "";
- if ( param->cg_sam || param->cg_ev_dnb )
- ds->cols[ alg_EVIDENCE_ALIGNMENT_IDS ].name = "EVIDENCE_ALIGNMENT_IDS";
- break;
-
- case edstt_EvidenceAlignment:
- ds->cols[ alg_REF_PLOIDY ].name = "REF_PLOIDY";
- ds->cols[ alg_REF_ID ].name = "REF_ID";
- ds->cols[ alg_MATE_ALIGN_ID ].name = "";
- ds->cols[ alg_MATE_REF_ID ].name = "";
- ds->cols[ alg_MATE_REF_NAME ].name = "";
- ds->cols[ alg_SAM_FLAGS ].name = "";
- ds->cols[ alg_MATE_REF_POS ].name = "";
- ds->cols[ alg_ALIGN_GROUP ].name = "";
- ds->cols[ alg_TEMPLATE_LEN ].name = "";
- break;
- default:
- break;
- }
-}
-
-
-static rc_t ProcessDB( VDatabase const *db, char const fullPath[],
- char const accession[], char const readGroup[] )
-{
- char const *const refTableName = ( param->region_qty > 0
- || param->cg_evidence
- || param->cg_ev_dnb
- || param->cg_sam
- || param->reheader
- || param->primaries
- || param->secondaries
- )
- ? "REFERENCE"
- : 0;
- char const *const priTableName = param->primaries ? "PRIMARY_ALIGNMENT" : 0;
- char const *const secTableName = param->secondaries ? "SECONDARY_ALIGNMENT" : 0;
- char const *const seqTableName = ( param->unaligned || param->reheader ) ? "SEQUENCE" : 0;
- char const *const eviTableName = ( param->cg_evidence || ( param->cg_ev_dnb & ( param->region_qty > 0 ) ) || param->cg_sam ) ? "EVIDENCE_INTERVAL" : 0;
- char const *const evaTableName = ( param->cg_ev_dnb || param->cg_sam ) ? "EVIDENCE_ALIGNMENT" : 0;
-
- SAM_dump_ctx_t ctx;
- SCol align_cols[ ( sizeof( g_alg_col_tmpl ) / sizeof( g_alg_col_tmpl[ 0 ] ) ) * 4 ];
- SCol seq_cols[ sizeof( gSeqCol ) / sizeof( gSeqCol[ 0 ] ) ];
- rc_t rc = 0;
-
- memset( &ctx, 0, sizeof( ctx ) );
- memset( align_cols, 0, sizeof( align_cols ) );
- memset( seq_cols, 0, sizeof( seq_cols ) );
-
- ctx.db = db;
- ctx.fullPath = fullPath;
- ctx.accession = accession;
- ctx.readGroup = readGroup;
-
- DATASOURCE_INIT( ctx.seq, seqTableName );
- DATASOURCE_INIT( ctx.ref, refTableName );
- DATASOURCE_INIT( ctx.pri, priTableName );
- DATASOURCE_INIT( ctx.sec, secTableName );
- DATASOURCE_INIT( ctx.evi, eviTableName );
- DATASOURCE_INIT( ctx.eva, evaTableName );
-
- if ( ctx.seq.tbl.name )
- {
- rc = VDatabaseOpenTableRead( db, &ctx.seq.tbl.vtbl, "%s", ctx.seq.tbl.name );
- if ( rc == 0 )
- {
- ctx.seq.cols = seq_cols;
- memcpy(seq_cols, gSeqCol, sizeof(gSeqCol));
- rc = Cursor_Open( &ctx.seq.tbl, &ctx.seq.curs, ctx.seq.cols, NULL );
- }
- }
- ctx.pri.cols = &align_cols[ 0 * ( sizeof( g_alg_col_tmpl ) / sizeof( g_alg_col_tmpl[ 0 ] ) ) ];
- ctx.sec.cols = &align_cols[ 1 * ( sizeof( g_alg_col_tmpl ) / sizeof( g_alg_col_tmpl[ 0 ] ) ) ];
- ctx.evi.cols = &align_cols[ 2 * ( sizeof( g_alg_col_tmpl ) / sizeof( g_alg_col_tmpl[ 0 ] ) ) ];
- ctx.eva.cols = &align_cols[ 3 * ( sizeof( g_alg_col_tmpl ) / sizeof( g_alg_col_tmpl[ 0 ] ) ) ];
-
- SetupColumns( &ctx.pri, edstt_PrimaryAlignment );
- SetupColumns( &ctx.sec, edstt_SecondaryAlignment );
- SetupColumns( &ctx.evi, edstt_EvidenceInterval );
- SetupColumns( &ctx.eva, edstt_EvidenceAlignment );
-
- if ( ctx.pri.tbl.name )
- VDatabaseOpenTableRead( db, &ctx.pri.tbl.vtbl, "%s", ctx.pri.tbl.name );
- if ( ctx.sec.tbl.name )
- VDatabaseOpenTableRead( db, &ctx.sec.tbl.vtbl, "%s", ctx.sec.tbl.name );
- if ( ctx.evi.tbl.name )
- VDatabaseOpenTableRead( db, &ctx.evi.tbl.vtbl, "%s", ctx.evi.tbl.name );
- if ( ctx.eva.tbl.name )
- VDatabaseOpenTableRead( db, &ctx.eva.tbl.vtbl, "%s", ctx.eva.tbl.name );
-
- if ( ctx.pri.tbl.vtbl == NULL
- && ctx.sec.tbl.vtbl == NULL
- && ctx.evi.tbl.vtbl == NULL
- && ctx.eva.tbl.vtbl == NULL )
- {
- memset (&ctx.pri, 0, sizeof( ctx.pri ) * 4 );
- if ( ctx.seq.tbl.name == NULL )
- {
- ctx.seq.tbl.name = "SEQUENCE";
- rc = VDatabaseOpenTableRead( db, &ctx.seq.tbl.vtbl, "%s", ctx.seq.tbl.name );
- }
- if ( rc == 0 )
- {
- ctx.seq.cols = seq_cols;
- memcpy(seq_cols, gSeqCol, sizeof(gSeqCol));
- rc = Cursor_Open(&ctx.seq.tbl, &ctx.seq.curs, ctx.seq.cols, NULL);
- if ( rc == 0 )
- {
- rc = DumpTable( &ctx );
- Cursor_Close(&ctx.seq.curs);
- }
- VTableRelease(ctx.seq.tbl.vtbl);
- }
- VDatabaseRelease( db );
- return rc;
- }
- ReportResetDatabase( fullPath, db );
-
- if ( ctx.ref.tbl.name )
- {
- rc = VDatabaseOpenTableRead( db, &ctx.ref.tbl.vtbl, "%s", ctx.ref.tbl.name );
- ctx.ref.type = edstt_Reference;
- }
- if ( rc == 0 )
- rc = Cursor_Open( &ctx.pri.tbl, &ctx.pri.curs, ctx.pri.cols, ctx.seq.curs.cache );
- if ( rc == 0 )
- rc = Cursor_Open( &ctx.sec.tbl, &ctx.sec.curs, ctx.sec.cols, NULL );
- if ( rc == 0 )
- rc = Cursor_Open( &ctx.evi.tbl, &ctx.evi.curs, ctx.evi.cols, NULL );
- if ( rc == 0 )
- rc = Cursor_Open( &ctx.eva.tbl, &ctx.eva.curs, ctx.eva.cols, NULL );
- if ( rc == 0 )
- rc = DumpDB( &ctx );
-
- Cursor_Close(&ctx.ref.curs);
- Cursor_Close(&ctx.pri.curs);
- Cursor_Close(&ctx.sec.curs);
- Cursor_Close(&ctx.evi.curs);
- Cursor_Close(&ctx.eva.curs);
- Cursor_Close(&ctx.seq.curs);
-
- VTableRelease( ctx.ref.tbl.vtbl );
- VTableRelease( ctx.pri.tbl.vtbl );
- VTableRelease( ctx.sec.tbl.vtbl );
- VTableRelease( ctx.evi.tbl.vtbl );
- VTableRelease( ctx.eva.tbl.vtbl );
- VTableRelease( ctx.seq.tbl.vtbl );
-
- return rc;
-}
-
-#if 0
-rc_t ResolvePath( char const *accession, char const ** path )
-{
- rc_t rc = 0;
- static char tblpath[ 4096 ];
-#if TOOLS_USE_SRAPATH != 0
- static SRAPath* pmgr = NULL;
-
- if ( accession == NULL && path == NULL )
- {
- SRAPathRelease( pmgr );
- return 0;
- }
- if ( pmgr != NULL
- || ( rc = SRAPathMake( &pmgr, NULL ) ) == 0
- || ( GetRCState( rc ) == rcNotFound && GetRCTarget( rc ) == rcDylib ) )
- {
- *path = tblpath;
- tblpath[ 0 ] = '\0';
- rc = 0;
- do {
- if ( pmgr != NULL && !SRAPathTest( pmgr, accession ) )
- {
- /* try to resolve the path using mgr */
- rc = SRAPathFind( pmgr, accession, tblpath, sizeof( tblpath ) );
- if ( rc == 0 )
- break;
- }
- if ( strlen( accession ) >= sizeof( tblpath ) )
- {
- rc = RC( rcExe, rcPath, rcResolving, rcBuffer, rcInsufficient );
- }
- else
- {
- strcpy( tblpath, accession );
- rc = 0;
- }
- } while( false );
- }
-#else
- if ( accession != NULL && path != NULL )
- {
- *path = tblpath;
- strcpy( tblpath, accession );
- }
-#endif
-
- return rc;
-}
-#endif
-
-
-static rc_t ProcessPath( VDBManager const *mgr, char const Path[] )
-{
-#if 0
- unsigned pathlen = strlen( Path );
- char *readGroup = NULL;
- char *path = NULL;
- unsigned i;
- rc_t rc;
-
- /* strip trailing path seperators */
- for ( i = pathlen; i > 0; )
- {
- int const ch = Path[ --i ];
-
- if ( ch == '/' || ch == '\\' )
- --pathlen;
- else
- break;
- }
-
- /* find read group alias */
- for ( i = pathlen; i > 0; )
- {
- int const ch = Path[ --i ];
-
- if ( ch == '=' )
- {
- unsigned const pos = i + 1;
- unsigned const len = pathlen - pos;
-
- pathlen = i;
-
- readGroup = malloc( len + 1 );
- if ( readGroup == NULL )
- return RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
- memcpy( readGroup, &Path[ pos ], len );
- readGroup[ len ] = '\0';
- break;
- }
- }
-
- path = malloc( pathlen + 1 );
- if ( path == NULL )
- return RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
- memcpy( path, Path, pathlen );
- path[ pathlen ] = '\0';
-
- rc = ReportResetObject( path );
- if ( rc == 0 )
- {
- char const *fullPath;
-
- rc = ResolvePath( path, &fullPath );
- if ( rc == 0 )
- {
- char const *accession = fullPath;
- VDatabase const *db;
-
- /* use last path element as accession */
- for ( i = pathlen; i > 0; )
- {
- int const ch = path[ --i ];
-
- if ( ch == '/' || ch == '\\' )
- {
- accession = &fullPath[ i + i ];
- break;
- }
- }
- rc = VDBManagerOpenDBRead( mgr, &db, NULL, "%s", fullPath );
- if ( rc == 0 )
- {
- rc = ProcessDB( db, fullPath, accession, readGroup );
- VDatabaseRelease( db );
- }
- else
- rc = ProcessTable( mgr, fullPath, accession, readGroup );
- }
- }
- free( path );
- free( readGroup );
- return rc;
-
-#else
-
- char * pc;
- rc_t rc = 0;
-
- pc = strchr (Path, ':');
- if (pc == NULL)
- {
- unsigned pathlen;
- char *readGroup;
- char *path;
- unsigned i;
-
- old:
- pathlen = strlen( Path );
- readGroup = NULL;
- path = NULL;
-
- /* strip trailing path seperators */
- for ( i = pathlen; i > 0; )
- {
- int const ch = Path[ --i ];
-
- if ( ch == '/' || ch == '\\' )
- --pathlen;
- else
- break;
- }
-
- /* find read group alias */
- for ( i = pathlen; i > 0; )
- {
- int const ch = Path[ --i ];
-
- if ( ch == '=' )
- {
- unsigned const pos = i + 1;
- unsigned const len = pathlen - pos;
-
- pathlen = i;
-
- readGroup = malloc( len + 1 );
- if ( readGroup == NULL )
- return RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
- memcpy( readGroup, &Path[ pos ], len );
- readGroup[ len ] = '\0';
- break;
- }
- }
-
- path = malloc( pathlen + 1 );
- if ( path == NULL )
- return RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
- memcpy( path, Path, pathlen );
- path[ pathlen ] = '\0';
-
- rc = ReportResetObject( path );
- if ( rc == 0 )
- {
- char const *accession = path;
- VDatabase const *db;
-
- /* use last path element as accession */
- for ( i = pathlen; i > 0; )
- {
- int const ch = path[ --i ];
-
- if ( ch == '/' || ch == '\\' )
- {
- accession = &path[ i + 1 ];
- break;
- }
- }
- rc = VDBManagerOpenDBRead( mgr, &db, NULL, "%s", path );
- if ( rc == 0 )
- {
- rc = ProcessDB( db, path, accession, readGroup );
- if (UIError(rc, db, NULL)) {
- UIDatabaseLOGError(rc, db, true);
-/* *error_reported = true; */
- }
- VDatabaseRelease( db );
- }
- else {
- rc = ProcessTable( mgr, path, accession, readGroup );
- if (UIError(rc, NULL, NULL)) {
- UITableLOGError(rc, NULL, true);
-/* *error_reported = true; */
- }
- }
- }
- free( path );
- free( readGroup );
- return rc;
- }
- else
- {
- VPath * vpath;
- char * basename;
- size_t zz;
- char buffer [8193];
- char readgroup_ [257];
- char * readgroup;
-
- rc = VPathMake (&vpath, Path);
- if (rc)
- ; /* LOGERR */
- else
- {
- VPUri_t uri_t;
- switch (uri_t = VPathGetUri_t (vpath))
- {
- default:
- case vpuri_invalid:
- rc = RC (rcExe, rcParam, rcAccessing, rcPath, rcInvalid);
- break;
- case vpuri_not_supported:
- rc = RC (rcExe, rcParam, rcAccessing, rcPath, rcUnsupported);
- break;
- case vpuri_none:
- goto old;
-
- case vpuri_ncbi_vfs:
- case vpuri_file:
- case vpuri_ncbi_acc:
- case vpuri_http:
- rc = VPathReadPath (vpath, buffer, sizeof buffer - 1, &zz);
- if (rc)
- ;
- else
- {
- loop:
- basename = strrchr (buffer, '/');
- if (basename)
- {
- if (basename[1] == '\0')
- {
- basename[0] = '\0';
- goto loop;
- }
- else
- ++basename;
- }
- else
- basename = buffer;
-
- {
- char * ext;
-
- /* cut off [.lite].[c]sra[.nenc||.ncbi_enc] if any */
- ext = strrchr( basename, '.' );
- if ( ext != NULL )
- {
- if ( strcasecmp( ext, ".nenc" ) == 0 || strcasecmp( ext, ".ncbi_enc" ) == 0 )
- {
- *ext = '\0';
- ext = strrchr( basename, '.' );
- }
- if ( ext != NULL )
- {
- if ( strcasecmp( ext, ".sra" ) == 0 || strcasecmp( ext, ".csra" ) == 0 )
- {
- *ext = '\0';
- ext = strrchr( basename, '.' );
- if ( ext != NULL && strcasecmp( ext, ".lite" ) == 0 )
- *ext = '\0';
- }
- }
- }
- }
-
- rc = VPathOption (vpath, vpopt_readgroup, readgroup_,
- sizeof readgroup_ - 1, &zz);
- if (rc)
- {
- if (GetRCState (rc) == rcNotFound)
- {
- rc = 0;
- readgroup = NULL;
- }
- }
- else
- readgroup = readgroup_;
-
-
- if (rc)
- ;
- else
- {
- VDatabase const *db;
-
- rc = VDBManagerOpenDBRead( mgr, &db, NULL, "%s", Path );
- if ( rc == 0 )
- {
- rc = ProcessDB( db, Path, basename, readgroup );
- VDatabaseRelease( db );
- }
- else
- rc = ProcessTable( mgr, Path, basename, readgroup );
- }
- }
- break;
- }
- VPathRelease (vpath);
- }
- }
- return rc;
-#endif
-}
-
-
-ver_t CC KAppVersion( void )
-{
- return SAM_DUMP_VERS;
-}
-
-char const *unaligned_usage[] = {"Output unaligned reads", NULL};
-char const *unaligned_only_usage[] = {"Only output unaligned spots", NULL};
-char const *primaryonly_usage[] = {"Output only primary alignments", NULL};
-char const *cigartype_usage[] = {"Output long version of CIGAR", NULL};
-char const *cigarCG_usage[] = {"Output CG version of CIGAR", NULL};
-char const *cigarCGMerge_usage[] = {"Apply CG fixups to CIGAR/SEQ/QUAL and outputs CG-specific columns", NULL};
-char const *CG_evidence[] = {"Output CG evidence aligned to reference", NULL};
-char const *CG_ev_dnb[] = {"Output CG evidence DNB's aligned to evidence", NULL};
-char const *CG_SAM[] = {"Output CG evidence DNB's aligned to reference ", NULL};
-char const *CG_mappings[] = {"Output CG sequences aligned to reference ", NULL};
-char const *header_usage[] = {"Replace original BAM header with a reconstructed one", NULL};
-char const *noheader_usage[] = {"Do not output headers", NULL};
-char const *region_usage[] = {"Filter by position on genome.",
- "Name can either be file specific name (ex: \"chr1\" or \"1\").",
- "\"from\" and \"to\" are 1-based coordinates", NULL};
-char const *distance_usage[] = {"Filter by distance between matepairs.",
- "Use \"unknown\" to find matepairs split between the references.",
- "Use from-to to limit matepair distance on the same reference", NULL};
-char const *seq_id_usage[] = {"Print reference SEQ_ID in RNAME instead of NAME", NULL};
-char const *identicalbases_usage[] = {"Output '=' if base is identical to reference", NULL};
-char const *gzip_usage[] = {"Compress output using gzip", NULL};
-char const *bzip2_usage[] = {"Compress output using bzip2", NULL};
-char const *qname_usage[] = {"Add .SPOT_GROUP to QNAME", NULL};
-char const *fasta_usage[] = {"Produce Fasta formatted output", NULL};
-char const *fastq_usage[] = {"Produce FastQ formatted output", NULL};
-char const *prefix_usage[] = {"Prefix QNAME: prefix.QNAME", NULL};
-char const *reverse_usage[] = {"Reverse unaligned reads according to read type", NULL};
-char const *comment_usage[] = {"Add comment to header. Use multiple times for several lines. Use quotes", NULL};
-char const *XI_usage[] = {"Output cSRA alignment id in XI column", NULL};
-char const *qual_quant_usage[] = {"Quality scores quantization level",
- "a string like '1:10,10:20,20:30,30:-'", NULL};
-char const *CG_names[] = { "Generate CG friendly read names", NULL};
-
-char const *usage_params[] =
-{
- NULL, /* unaligned */
- NULL, /* unaligned-only */
- NULL, /* primaryonly */
- NULL, /* cigartype */
- NULL, /* cigarCG */
- NULL, /* header */
- NULL, /* no-header */
- "'text'", /* comment */
- "name[:from-to]", /* region */
- "from-to|unknown", /* distance */
- NULL, /* seq-id */
- NULL, /* identical-bases */
- NULL, /* gzip */
- NULL, /* bzip2 */
- NULL, /* qname */
- NULL, /* fasta */
- NULL, /* fastq */
- "prefix", /* prefix */
- NULL, /* reverse */
- NULL, /* test-rows */
- NULL, /* mate-row-gap-cacheable */
- NULL, /* cigarCGMerge */
- NULL, /* XI */
- "quantization string", /* qual-quant */
- NULL, /* cigarCGMerge */
- NULL, /* CG-evidence */
- NULL, /* CG-ev-dnb */
- NULL, /* CG-mappings */
- NULL, /* CG-SAM */
- NULL /* CG-names */
-};
-
-enum eArgs
-{
- earg_unaligned = 0, /* unaligned */
- earg_unaligned_only, /* unaligned */
- earg_prim_only, /* primaryonly */
- earg_cigartype, /* cigartype */
- earg_cigarCG, /* cigarCG */
- earg_header, /* header */
- earg_noheader, /* no-header */
- earg_comment, /* comment */
- earg_region, /* region */
- earg_distance, /* distance */
- earg_seq_id, /* seq-id */
- earg_identicalbases, /* identical-bases */
- earg_gzip, /* gzip */
- earg_bzip2, /* bzip2 */
- earg_qname, /* qname */
- earg_fastq, /* fasta */
- earg_fasta, /* fastq */
- earg_prefix, /* prefix */
- earg_reverse, /* reverse */
- earg_test_rows, /* test-rows */
- earg_mate_row_gap_cachable, /* mate-row-gap-cacheable */
- earg_cigarCG_merge, /* cigarCGMerge */
- earg_XI, /* XI */
- earg_QualQuant, /* qual-quant */
- earg_CG_evidence, /* CG-evidence */
- earg_CG_ev_dnb, /* CG-ev-dnb */
- earg_CG_mappings, /* CG-mappings */
- earg_CG_SAM, /* CG-SAM */
- earg_CG_names /* CG-names */
-};
-
-OptDef DumpArgs[] =
-{
- { "unaligned", "u", NULL, unaligned_usage, 0, false, false }, /* unaligned */
- { "unaligned-only", "U", NULL, unaligned_only_usage, 0, false, false }, /* unaligned-only */
- { "primary", "1", NULL, primaryonly_usage, 0, false, false }, /* primaryonly */
- { "cigar-long", "c", NULL, cigartype_usage, 0, false, false }, /* cigartype */
- { "cigar-CG", NULL, NULL, cigarCG_usage, 0, false, false }, /* cigarCG */
- { "header", "r", NULL, header_usage, 0, false, false }, /* header */
- { "no-header", "n", NULL, noheader_usage, 0, false, false }, /* no-header */
- { "header-comment", NULL, NULL, comment_usage, 0, true, false }, /* comment */
- { "aligned-region", NULL, NULL, region_usage, 0, true, false }, /* region */
- { "matepair-distance", NULL, NULL, distance_usage, 0, true, false }, /* distance */
- { "seqid", "s", NULL, seq_id_usage, 0, false, false }, /* seq-id */
- { "hide-identical", "=", NULL, identicalbases_usage, 0, false, false }, /* identical-bases */
- { "gzip", NULL, NULL, gzip_usage, 0, false, false }, /* gzip */
- { "bzip2", NULL, NULL, bzip2_usage, 0, false, false }, /* bzip2 */
- { "spot-group", "g", NULL, qname_usage, 0, false, false }, /* qname */
- { "fastq", NULL, NULL, fastq_usage, 0, false, false }, /* fasta */
- { "fasta", NULL, NULL, fasta_usage, 0, false, false }, /* fastq */
- { "prefix", "p", NULL, prefix_usage, 0, true, false }, /* prefix */
- { "reverse", NULL, NULL, reverse_usage, 0, false, false }, /* reverse */
- { "test-rows", NULL, NULL, NULL, 0, true, false }, /* test-rows */
- { "mate-cache-row-gap", NULL, NULL, NULL, 0, true, false }, /* mate-row-gap-cacheable */
- { "cigar-CG-merge", NULL, NULL, cigarCGMerge_usage, 0, false, false }, /* cigarCGMerge */
- { "XI", NULL, NULL, XI_usage, 0, false, false }, /* XI */
- { "qual-quant", "Q", NULL, qual_quant_usage, 0, true, false }, /* qual-quant */
- { "CG-evidence", NULL, NULL, CG_evidence, 0, false, false }, /* CG-evidence */
- { "CG-ev-dnb" , NULL, NULL, CG_ev_dnb , 0, false, false }, /* CG-ev-dnb */
- { "CG-mappings", NULL, NULL, CG_mappings, 0, false, false }, /* CG-mappings */
- { "CG-SAM", NULL, NULL, CG_SAM, 0, false, false }, /* CG-SAM */
- { "CG-names", NULL, NULL, CG_names, 0, false, false } /* CG-names */
-};
-
-
-const char UsageDefaultName[] = "sam-dump";
-
-
-rc_t CC UsageSummary( char const *progname )
-{
- return KOutMsg( "Usage:\n"
- "\t%s [options] path-to-run[ path-to-run ...] > output.sam\n\n", progname );
-}
-
-
-rc_t CC Usage( Args const *args)
-{
- char const *progname = UsageDefaultName;
- char const *fullpath = UsageDefaultName;
- rc_t rc;
- unsigned i;
-
- rc = ArgsProgram( args, &fullpath, &progname );
-
- UsageSummary( progname );
-
- KOutMsg( "Options:\n" );
- for( i = 0; i < sizeof( DumpArgs ) / sizeof( DumpArgs[ 0 ] ); i++ )
- {
- if ( DumpArgs[ i ].help != NULL )
- {
- HelpOptionLine( DumpArgs[ i ].aliases, DumpArgs[ i ].name,
- usage_params[ i ], DumpArgs[ i ].help);
- }
- }
- KOutMsg( "\n" );
- HelpOptionsStandard();
-
- HelpVersion( fullpath, KAppVersion() );
-
- return rc;
-}
-
-
-#define ARGS_COUNT (sizeof(DumpArgs)/sizeof(DumpArgs[0]))
-
-
-static unsigned ParamCount( Args const *const args, rc_t *const prc )
-{
- uint32_t y = 0;
- rc_t rc = ArgsParamCount( args, &y );
-
- if ( prc != NULL )
- *prc = rc;
- return y;
-}
-
-
-static unsigned ArgCount( Args const *const args, rc_t *const prc )
-{
- uint32_t y = 0;
- rc_t rc = ArgsArgvCount( args, &y );
-
- if ( prc != NULL )
- *prc = rc;
- return y;
-}
-
-
-static rc_t CountArgs( Args const *const args, unsigned count[],
- char const **const errmsg )
-{
- rc_t rc;
- unsigned const pcount = ParamCount( args, &rc );
-
- memset( count, 0, ARGS_COUNT );
- if ( rc != 0 || pcount == 0 )
- {
- *errmsg = "";
- MiniUsage( args );
- return ( ArgCount( args, NULL ) < 2 )
- ? 0
- : RC( rcExe, rcArgv, rcParsing, rcParam, rcInsufficient );
- }
-#define COUNT_ARG(E) do {\
- char const *const optname = DumpArgs[ (E) ].name; \
- if ( ( rc = ArgsOptionCount( args, optname, &count[ (E) ] ) ) != 0 ) { \
- *errmsg = optname; \
- return rc; \
- } } while( 0 )
-
-
- /* record source options */
- COUNT_ARG( earg_unaligned );
- COUNT_ARG( earg_unaligned_only );
- COUNT_ARG( earg_prim_only );
- COUNT_ARG( earg_CG_evidence );
- COUNT_ARG( earg_CG_ev_dnb );
- COUNT_ARG( earg_CG_mappings );
- COUNT_ARG( earg_CG_SAM );
-
- /* record filter options */
- COUNT_ARG( earg_region );
- COUNT_ARG( earg_distance );
-
- /* output format options */
- COUNT_ARG( earg_fastq );
- COUNT_ARG( earg_fasta );
-
- /* SAM header options */
- COUNT_ARG( earg_header );
- COUNT_ARG( earg_noheader );
- COUNT_ARG( earg_comment );
-
- /* SAM field options */
- COUNT_ARG( earg_prefix );
- COUNT_ARG( earg_qname );
- COUNT_ARG( earg_seq_id );
- COUNT_ARG( earg_CG_names );
-
- COUNT_ARG( earg_cigartype );
- COUNT_ARG( earg_cigarCG );
- COUNT_ARG( earg_cigarCG_merge );
-
- COUNT_ARG( earg_identicalbases );
- COUNT_ARG( earg_reverse );
- COUNT_ARG( earg_QualQuant );
-
- /* output encoding options */
- COUNT_ARG( earg_gzip );
- COUNT_ARG( earg_bzip2 );
-
- COUNT_ARG( earg_mate_row_gap_cachable );
-
- /* debug options */
- COUNT_ARG( earg_XI );
- COUNT_ARG( earg_test_rows );
-#undef COUNT_ARG
- return 0;
-}
-
-
-static unsigned GetOptValU( Args const *const args, char const *const argname,
- unsigned const defval, rc_t *const prc )
-{
- unsigned y = defval;
- char const *valstr;
- rc_t rc = ArgsOptionValue( args, argname, 0, &valstr );
-
- if ( rc == 0 )
- {
- char *endp;
-
- y = strtou32( valstr, &endp, 0 );
- if ( *endp != '\0' )
- {
- rc = RC( rcExe, rcArgv, rcProcessing, rcParam, rcInvalid );
- y = 0;
- }
- if ( y == 0 )
- y = defval;
- }
- if ( prc )
- *prc = rc;
- return y;
-}
-
-
-static rc_t GetComments( Args const *const args, struct params_s *const parms, unsigned const n )
-{
- parms->comments = calloc( n + 1, sizeof( parms->comments[ 0 ] ) );
- if ( parms->comments != NULL )
- {
- unsigned i;
-
- for ( i = 0; i < n; ++i )
- {
- rc_t const rc = ArgsOptionValue( args, DumpArgs[ earg_comment ].name, i, &parms->comments[ i ] );
- if ( rc != 0 )
- {
- free( parms->comments );
- parms->comments = NULL;
- return rc;
- }
- }
- return 0;
- }
- else
- return RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
-}
-
-
-static rc_t ParseFromTo( int *const pFrom, int *const pTo, char const str[] )
-{
- /* $str =~ /((\d+)-(\d+))|(-(\d+))|(\d+)/ */
- int n;
- char fr_str[ 16 ];
- char to_str[ 16 ];
- uint32_t fr = 0;
- uint32_t to = 0;
- int i = sscanf( str, "%15[0-9]-%15[0-9]%n", fr_str, to_str, &n );
-
- if ( i != 2 )
- {
- unsigned const offset = ( str[ 0 ] == '-' ) ? 1 : 0;
-
- fr = 0;
- i = sscanf( str + offset, "%15[0-9]%n", to_str, &n );
- if ( i != 1 )
- return RC( rcExe, rcArgv, rcProcessing, rcParam, rcOutofrange );
- to = strtou32( to_str, 0, 0 );
- }
- else
- {
- fr = strtou32( fr_str, 0, 0 );
- to = strtou32( to_str, 0, 0 );
-
- if ( to < fr )
- {
- uint32_t const tmp = to;
- to = fr;
- fr = tmp;
- }
- }
- /* was the entire string consumed */
- if ( str[ n ] != 0 )
- return RC( rcExe, rcArgv, rcProcessing, rcParam, rcInvalid );
- *pFrom = ( int )fr;
- *pTo = ( int )to;
- return 0;
-}
-
-
-static rc_t GetDistance( Args const *const args, struct params_s *const parms, unsigned const n )
-{
- rc_t rc;
- TMatepairDistance *const mpd = calloc( n, sizeof( mpd[ 0 ] ) );
-
- if ( mpd == NULL )
- rc = RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
- else
- {
- unsigned i;
- unsigned j;
-
- for ( j = i = 0; i < n; ++i )
- {
- char const *valstr;
- int from = 0;
- int to = 0;
-
- rc = ArgsOptionValue( args, DumpArgs[earg_distance].name, i, &valstr );
- if ( rc != 0 )
- break;
- if ( strcasecmp( valstr, "unknown" ) == 0 )
- {
- parms->mp_dist_unknown = true;
- continue;
- }
- rc = ParseFromTo( &from, &to, valstr );
- if ( rc != 0 )
- break;
- if ( ( from != 0 ) || ( to != 0 ) )
- {
- mpd[ j ].from = from;
- mpd[ j ].to = to;
- ++j;
- }
- }
- if ( rc == 0 )
- {
- parms->mp_dist = mpd;
- parms->mp_dist_qty = j;
-
- SAM_DUMP_DBG( 2, ( "filter by mate pair distance\n" ) );
- if ( parms->mp_dist_unknown )
- SAM_DUMP_DBG( 2, ( " distance: unknown\n" ) );
- for ( i = 0; i < j; ++i )
- {
- SAM_DUMP_DBG( 2, ( " distance [%lu-%lu]\n",
- parms->mp_dist[ i ].from, parms->mp_dist[ i ].to ) );
- }
- }
- else
- free( mpd );
- }
- return rc;
-}
-
-
-static rc_t GetRegion( Args const *const args, struct params_s *const parms, unsigned const n )
-{
- rc_t rc;
- TAlignedRegion *const reg = calloc( n, sizeof( reg[ 0 ] ) );
-
- if ( reg == NULL )
- rc = RC( rcExe, rcArgv, rcProcessing, rcMemory, rcExhausted );
- else
- {
- unsigned i;
- unsigned j;
-
- for ( rc = 0, j = i = 0; i < n; ++i )
- {
- char const *valstr;
- char const *sep;
- unsigned namelen;
- unsigned f;
- unsigned e;
- TAlignedRegion r;
-
- memset(&r,0,sizeof(r));
-
- rc = ArgsOptionValue( args, DumpArgs[ earg_region ].name, i, &valstr );
- if ( rc != 0 ) break;
-
- sep = strchr( valstr, ':' );
- if ( sep != NULL )
- namelen = sep - valstr;
- else
- namelen = strlen( valstr );
-
- if ( namelen >= sizeof( r.name ) )
- return RC( rcExe, rcArgv, rcProcessing, rcParam, rcTooLong );
-
- memcpy( r.name, valstr, namelen );
- r.name[ namelen ] = '\0';
-
- r.rq = UINT_MAX;
- if ( sep != NULL )
- {
- int from = -1;
- int to = -1;
-
- rc = ParseFromTo( &from, &to, sep + 1 );
- if ( rc != 0 )
- break;
- if ( from > 0 ) /* convert to 0-based */
- --from;
- else if ( to > 0 )
- from = 0;
- if ( to > 0 )
- --to;
- else if ( from >= 0 && to < 0 )
- to = from;
- if ( from >= 0 || to >= 0 )
- {
- if ( from > to )
- {
- int tmp = to;
- to = from;
- from = tmp;
- }
- r.r[ 0 ].from = from;
- r.r[ 0 ].to = to;
- r.rq = 1;
- }
- }
-
- for ( f = 0, e = j; f < e; )
- {
- unsigned const m = ( ( f + e ) / 2 );
- int const diff = strcmp( r.name, reg[ m ].name );
-
- if ( diff < 0 )
- e = m;
- else
- f = m + 1;
- }
- if ( 0 < e && e < j && strcmp( r.name, reg[ e-1 ].name ) == 0 )
- {
- TAlignedRegion *const fnd = ®[ e - 1 ];
-
- if ( fnd->rq != UINT_MAX )
- {
- for ( f = 0, e = fnd->rq; f < e; )
- {
- unsigned const m = ( ( f + e ) / 2 );
-
- if ( r.r[ 0 ].from < fnd->r[ m ].from )
- e = m;
- else
- f = m + 1;
- }
- if ( fnd->rq >= ( sizeof( r.r ) / sizeof( r.r[0] ) ) )
- {
- rc = RC( rcExe, rcArgv, rcProcessing, rcBuffer, rcInsufficient );
- break;
- }
- memmove( &fnd->r[ e +1 ], &fnd->r[ e ], ( fnd->rq - e ) * sizeof( fnd->r[ 0 ] ) );
- fnd->r[ e ] = r.r[ 0 ];
- ++fnd->rq;
- }
- }
- else
- {
- memmove( ®[ e + 1 ], ®[ e ], ( j - e ) * sizeof( reg[ 0 ] ) );
- reg[ e ] = r;
- ++j;
- }
- }
- if ( rc == 0 )
- {
- parms->region = reg;
- parms->region_qty = j;
-
- for ( i = 0; i < parms->region_qty; ++i )
- {
- TAlignedRegion *const r = &parms->region[ i ];
-
- SAM_DUMP_DBG( 2, ( "filter by %s\n", r->name ) );
- if ( r->rq == UINT_MAX )
- {
- r->rq = 1;
- r->r[ 0 ].from = 0;
- r->r[ 0 ].to = UINT_MAX;
- }
- for ( j = 0; j < r->rq; ++j )
- {
- unsigned const to = r->r[ j ].to;
- unsigned const from = r->r[ j ].from;
-
- SAM_DUMP_DBG( 2, ( " range: [%u:%u]\n", r->r[ j ].from, to ) );
- if ( r->max_to < to ) r->max_to = to;
- if ( r->min_from < from ) r->min_from = from;
- }
- }
- }
- else
- free( reg );
- }
- return rc;
-}
-
-
-static rc_t GetQualQuant( Args const *const args, struct params_s *const parms )
-{
- char const *valstr;
- int i;
- rc_t rc = ArgsOptionValue( args, DumpArgs[ earg_QualQuant ].name, 0, &valstr );
-
- if ( rc == 0 && strcmp( valstr, "0" ) != 0 )
- parms->quantizeQual = QualityQuantizerInitMatrix( parms->qualQuant, valstr );
- for(i=1; parms->qualQuant[i]==parms->qualQuant[0] && i<256;i++){}
- if(i<256) parms->qualQuantSingle=0;
- else parms->qualQuantSingle = parms->qualQuant[0];
- return rc;
-}
-
-
-static rc_t GetArgs( Args const *const args, unsigned const count[],
- char const **const errmsg )
-{
- static struct params_s parms;
- bool const multipass = ParamCount( args, 0 ) > 1;
- rc_t rc;
-
- memset( &parms, 0, sizeof( parms ) );
-
- if ( count[ earg_comment ] )
- {
- rc = GetComments( args, &parms, count[ earg_comment ] );
- if ( rc != 0 )
- {
- *errmsg = DumpArgs[ earg_comment ].name;
- return rc;
- }
- }
-
- if ( count[ earg_region ] && count[ earg_unaligned_only ] == 0 )
- {
- rc = GetRegion( args, &parms, count[ earg_region ] );
- if ( rc != 0 )
- {
- *errmsg = DumpArgs[ earg_region ].name;
- return rc;
- }
- }
-
- if ( count[ earg_distance ] && count[ earg_unaligned_only ] == 0 )
- {
- rc = GetDistance( args, &parms, count[ earg_distance ] );
- if ( rc != 0 )
- {
- *errmsg = DumpArgs[ earg_distance ].name;
- return rc;
- }
- }
-
- if ( count[ earg_QualQuant ] )
- {
- rc = GetQualQuant( args, &parms );
- if ( rc != 0 )
- {
- *errmsg = DumpArgs[ earg_QualQuant ].name;
- return rc;
- }
- }
-
- parms.cg_sam = ( count[ earg_CG_SAM ] != 0 );
- parms.cg_evidence = ( count[ earg_CG_evidence ] != 0 );
- parms.cg_ev_dnb = ( count[ earg_CG_ev_dnb ] != 0 );
-
- if ( count[ earg_CG_mappings ] == 0 &&
- ( parms.cg_sam || parms.cg_evidence || parms.cg_ev_dnb ) )
- {
- parms.primaries = false;
- parms.secondaries = false;
- parms.unaligned = false;
- }
- else
- {
- parms.primaries = true;
- parms.secondaries = ( count[ earg_prim_only ] == 0 );
- parms.unaligned = ( ( count[ earg_unaligned ] != 0 ) /*&& ( parms.region_qty == 0 )*/ );
- }
-
- parms.long_cigar = ( count[ earg_cigartype ] != 0 );
- parms.use_seqid = ( ( count[ earg_seq_id ] != 0 ) || multipass );
- parms.hide_identical = ( count[ earg_identicalbases ] != 0 );
- parms.fasta = ( count[ earg_fasta ] != 0 );
- parms.fastq = ( count[ earg_fastq ] != 0 );
- parms.reverse_unaligned = ( count[ earg_reverse ] != 0 );
- parms.cg_friendly_names = count[ earg_CG_names ] != 0;
- parms.spot_group_in_name = ( count[ earg_qname ] != 0 || multipass );
- parms.noheader = ( ( count[ earg_noheader ] != 0 ) || parms.fasta || parms.fastq || multipass );
- parms.reheader = ( ( count[ earg_header ] != 0 ) && !parms.noheader );
- parms.xi = ( count[ earg_XI ] != 0 );
- if ( ( count[ earg_cigarCG ] == 0 ) && ( count[ earg_cigarCG_merge ] == 0 ) )
- {
- parms.cg_style = 0;
- }
- else if ( count[ earg_cigarCG ] == 0 )
- {
- parms.cg_style = 1;
- parms.long_cigar = false;
- }
- else if ( count[ earg_cigarCG_merge ] == 0 )
- {
- parms.cg_style = 2;
- }
- else
- {
- rc = RC( rcExe, rcArgv, rcProcessing, rcParam, rcInconsistent );
- *errmsg = "cigar-CG and CG are mutually exclusive";
- parms.cg_style = 0;
- }
-
- if (count[ earg_unaligned_only ] > 0) {
- parms.unaligned_spots = true;
-
- parms.primaries = false;
- parms.secondaries = false;
- parms.unaligned = true;
- parms.cg_ev_dnb = false;
- parms.cg_evidence = false;
- parms.cg_sam = false;
- parms.cg_style = 0;
- }
-
- parms.test_rows = GetOptValU( args, DumpArgs[ earg_test_rows ].name, 0, NULL );
- parms.mate_row_gap_cachable = GetOptValU( args, DumpArgs[ earg_mate_row_gap_cachable ].name, 1000000, NULL );
-
- param = &parms;
- return 0;
-}
-
-
-static rc_t GetParams( Args const *const args, char const **const errmsg )
-{
- unsigned count[ ARGS_COUNT ];
- rc_t rc = CountArgs( args, count, errmsg );
-
- if ( rc == 0 )
- rc = GetArgs( args, count, errmsg );
- return rc;
-}
-
-
-rc_t CC SAM_Dump_Main( int argc, char* argv[] )
-{
- rc_t rc = 0;
- Args* args;
- char const *errmsg = "stop";
- bool error_reported = false;
-
- memset( &g_out_writer, 0, sizeof( g_out_writer ) );
- KOutHandlerSetStdOut();
- KStsHandlerSetStdErr();
- KLogHandlerSetStdErr();
- (void)KDbgHandlerSetStdErr();
-
- ReportBuildDate( __DATE__ );
-
- rc = ArgsMakeAndHandle( &args, argc, argv, 1, DumpArgs, ARGS_COUNT );
- if ( rc == 0 )
- {
- uint32_t pcount;
-
- rc = ArgsParamCount( args, &pcount );
- if ( rc != 0 || pcount < 1 )
- {
- errmsg = "";
- rc = argc < 2 ? 0 : RC( rcExe, rcArgv, rcParsing, rcParam, rcInsufficient );
- MiniUsage( args );
- }
- else if ( ( rc = GetParams( args, &errmsg ) ) != 0 )
- {
- }
- else
- {
- VDBManager const *mgr;
-
- rc = VDBManagerMakeRead( &mgr, NULL );
- if ( rc == 0 )
- {
- rc = BufferedWriterMake( param->output_gzip, param->output_bz2 );
- if ( rc == 0 )
- {
- unsigned i;
-
- for ( i = 0; i < pcount; ++i )
- {
- char const *arg;
-
- rc = ArgsParamValue( args, i, &arg );
- if ( rc != 0 ) break;
- rc = ProcessPath( mgr, arg );
-#if _ARCH_BITS < 64
- /* look for "param excessive while writing vector within container module" */
- if (GetRCState(rc) == rcExcessive &&
- GetRCObject(rc) == rcParam &&
- GetRCContext(rc) == rcWriting &&
- GetRCTarget(rc) == rcVector &&
- GetRCModule(rc) == rcCont)
- {
- ( void )PLOGMSG( klogErr, (klogErr, "This run '$(run)' contains too many reads to be processed with a $(bits)-bit executable; please try again with a 64-bit executable.", "run=%s,bits=%u", arg, _ARCH_BITS));
- }
-#endif
- if ( rc != 0 ) break;
- }
- BufferedWriterRelease( rc == 0 );
- }
- VDBManagerRelease( mgr );
- }
- }
- ArgsWhack( args );
- }
-
- if ( rc != 0 && !error_reported )
- {
- if ( errmsg[ 0 ] )
- {
- ( void )LOGERR( klogErr, rc, errmsg );
- } else if ( KLogLastErrorCode() != rc )
- {
- ( void )LOGERR( klogErr, rc, "stop" );
- }
- }
-
- {
- /* Report execution environment if necessary */
- rc_t rc2 = ReportFinalize( rc );
- if ( rc == 0 )
- rc = rc2;
- }
- return rc;
-}
-
-
-rc_t CC KMain( int argc, char* argv[] )
-{
- char filename[ 4096 ];
- /*
- Try to find a option-file - parameter in the original parameters
- This is a hack to teach sam-dump to read it's parameters from a file/pipe instead
- of the commandline
- It is neccessary because the code above does not use libkapp to parse parameters!
- */
- rc_t rc = Args_find_option_in_argv( argc, argv, "--option-file", filename, sizeof filename );
- if ( rc == 0 )
- {
- int new_argc;
- char ** new_argv;
- /* we found it! Now comes the special treatment: we fake a new argc/argv-pair! */
- rc = Args_tokenize_file_and_progname_into_argv( filename, "sam-dump", &new_argc, &new_argv );
- if ( rc == 0 )
- {
- /* pass the faked input-parameters from */
- rc = SAM_Dump_Main( new_argc, new_argv );
-
- Args_free_token_argv( new_argc, new_argv );
- }
- }
- else
- {
- /* we did not found the special option, just use the commandline-parameters unchanged */
- rc = SAM_Dump_Main( argc, argv );
- }
- return rc;
-}
diff --git a/tools/sam-dump/sam-dump.vers b/tools/sam-dump/sam-dump.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/sam-dump/sam-dump.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/sra-dbcc/sra-dbcc.c b/tools/sra-dbcc/sra-dbcc.c
deleted file mode 100644
index 50a90b4..0000000
--- a/tools/sra-dbcc/sra-dbcc.c
+++ /dev/null
@@ -1,1577 +0,0 @@
-/*===========================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * ===========================================================================
- *
- */
-
-#include "sra-dbcc.vers.h"
-
-#include <kapp/main.h>
-#include <kapp/args.h>
-
-#include <insdc/insdc.h>
-#include <insdc/sra.h>
-
-#include <sra/sradb.h>
-#include <sra/sraschema.h>
-#include <sra/srapath.h>
-
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/database.h>
-#include <vdb/table.h>
-#include <vdb/cursor.h>
-
-#include <kdb/manager.h>
-#include <kdb/database.h>
-#include <kdb/table.h>
-#include <kdb/meta.h>
-#include <kdb/consistency-check.h>
-#include <kdb/namelist.h>
-
-#include <vfs/manager.h> /* VFSManager */
-#include <vfs/resolver.h> /* VResolver */
-#include <vfs/path.h> /* VPath */
-
-#include <kfs/sra.h>
-#include <kfs/tar.h>
-#include <kfs/kfs-priv.h>
-
-#include <klib/container.h>
-#include <klib/sort.h>
-#include <klib/namelist.h>
-#include <klib/data-buffer.h>
-#include <klib/text.h> /* String */
-#include <klib/out.h>
-#include <klib/log.h>
-#include <klib/debug.h>
-#include <klib/rc.h>
-
-#include <sysalloc.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <assert.h>
-
-#ifndef PATH_MAX
- #define PATH_MAX 4096
-#endif
-
-#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
- if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
-
-static bool exhaustive;
-static bool md5_required;
-static bool ref_int_check;
-static bool s_IndexOnly = false;
-
-struct node_s {
- int parent;
- int prvSibl;
- int nxtSibl;
- int firstChild;
- unsigned depth;
- unsigned name;
- uint32_t objType;
-};
-typedef struct node_s node_t;
-
-typedef struct cc_context_s {
- node_t *nodes;
- char *names;
- rc_t rc;
- unsigned num_columns;
- unsigned nextNode;
- unsigned nextName;
-} cc_context_t;
-
-static
-rc_t report_rtn ( rc_t rc )
-{
- return exhaustive ? 0 : rc;
-}
-
-static rc_t report_index(CCReportInfoBlock const *what, cc_context_t *ctx)
-{
- switch (what->type) {
- case ccrpt_Done:
- if (what->info.done.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.done.rc, "Index '$(index)': $(mesg)",
- "index=%s,mesg=%s", what->objName, what->info.done.mesg));
- if (ctx->rc == 0)
- ctx->rc = what->info.done.rc;
- }
- return report_rtn (what->info.done.rc);
- case ccrpt_Index:
- return 0; /* continue with check */
- case ccrpt_MD5:
- if (what->info.MD5.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.MD5.rc, "File '$(file)' of index '$(index)' failed MD5 check",
- "file=%s,index=%s", what->info.MD5.file, what->objName));
- if (ctx->rc == 0)
- ctx->rc = what->info.MD5.rc;
- }
- return report_rtn (what->info.MD5.rc);
- default:
- return RC(rcExe, rcTable, rcVisiting, rcParam, rcUnexpected);
- }
-}
-
-static rc_t report_column(CCReportInfoBlock const *what, cc_context_t *ctx)
-{
- switch (what->type) {
- case ccrpt_Done:
- if (what->info.done.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.done.rc, "Column '$(column)': $(mesg)",
- "column=%s,mesg=%s", what->objName, what->info.done.mesg));
- if (ctx->rc == 0)
- ctx->rc = what->info.done.rc;
- }
- else {
- (void)PLOGMSG(klogInfo, (klogInfo, "Column '$(column)': $(mesg)", "column=%s,mesg=%s",
- what->objName, what->info.done.mesg ? what->info.done.mesg : "checked"));
- ++ctx->num_columns;
- }
- return report_rtn (what->info.done.rc);
- case ccrpt_Blob:
- return 0; /* continue with check */
- case ccrpt_MD5:
- if (what->info.MD5.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.MD5.rc, "File '$(file)' of column '$(column)' failed MD5 check",
- "file=%s,column=%s", what->info.MD5.file, what->objName));
- if (ctx->rc == 0)
- ctx->rc = what->info.MD5.rc;
- }
- return report_rtn (what->info.MD5.rc);
- default:
- return RC(rcExe, rcTable, rcVisiting, rcParam, rcUnexpected);
- }
-}
-
-static rc_t report_table(CCReportInfoBlock const *what, cc_context_t *ctx)
-{
- switch (what->type) {
- case ccrpt_Done:
- if (what->info.done.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.done.rc, "Table '$(table)': $(mesg)",
- "table=%s,mesg=%s", what->objName, what->info.done.mesg));
- if (ctx->rc == 0)
- ctx->rc = what->info.done.rc;
- }
- else if (what->info.done.mesg) {
- if (strcmp(what->info.done.mesg, "missing md5 file") == 0 && md5_required) {
- rc_t rc = RC(0, rcTable, rcValidating, rcChecksum, rcNotFound);
- (void)PLOGERR(klogErr, (klogErr, rc, "Table '$(table)': is missing required md5 files",
- "table=%s", what->objName));
- return rc;
- }
- else if (strncmp("unexpected object ", what->info.done.mesg, 18) == 0) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Table '$(tbl)': $(mesg)", "tbl=%s,mesg=%s",
- what->objName, what->info.done.mesg));
- }
- else {
- (void)PLOGMSG(klogInfo, (klogInfo, "Table '$(tbl)' metadata: $(mesg)",
- "tbl=%s,mesg=%s", what->objName, what->info.done.mesg));
- }
- }
- return report_rtn (what->info.done.rc);
- case ccrpt_MD5:
- if (what->info.MD5.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.MD5.rc, "File '$(file)' of table '$(table)' failed MD5 check",
- "file=%s,table=%s", what->info.MD5.file, what->objName));
- if (ctx->rc == 0)
- ctx->rc = what->info.MD5.rc;
- }
- return report_rtn (what->info.MD5.rc);
- default:
- return RC(rcExe, rcTable, rcVisiting, rcParam, rcUnexpected);
- }
-}
-
-static rc_t report_database(CCReportInfoBlock const *what, cc_context_t *ctx)
-{
- switch (what->type) {
- case ccrpt_Done:
- if (what->info.done.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.done.rc, "Database '$(db)': $(mesg)",
- "db=%s,mesg=%s", what->objName, what->info.done.mesg));
- if (ctx->rc == 0)
- ctx->rc = what->info.done.rc;
- }
- else if (what->info.done.mesg) {
- if (strcmp(what->info.done.mesg, "missing md5 file") == 0 && md5_required) {
- rc_t rc = RC(0, rcTable, rcValidating, rcChecksum, rcNotFound);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(table)': is missing required md5 files",
- "table=%s", what->objName));
- return rc;
- }
- else {
- (void)PLOGMSG(klogInfo, (klogInfo, "Database '$(db)' metadata: $(mesg)",
- "db=%s,mesg=%s", what->objName, what->info.done.mesg));
- }
- }
- return report_rtn (what->info.done.rc);
- case ccrpt_MD5:
- if (what->info.MD5.rc) {
- (void)PLOGERR(klogErr, (klogErr, what->info.MD5.rc, "File '$(file)' of database '$(db)' failed MD5 check",
- "file=%s,db=%s", what->info.MD5.file, what->objName));
- if (ctx->rc == 0)
- ctx->rc = what->info.MD5.rc;
- }
- return report_rtn (what->info.MD5.rc);
- default:
- return RC(rcExe, rcTable, rcVisiting, rcParam, rcUnexpected);
- }
-}
-
-static rc_t visiting(CCReportInfoBlock const *what, cc_context_t *ctx)
-{
- unsigned const nn = ctx->nextNode++;
- node_t *const nxt = &ctx->nodes[nn];
- node_t *const cur = nxt - 1;
-
- nxt->parent = nxt->prvSibl = nxt->nxtSibl = nxt->firstChild = -1;
- nxt->depth = what->info.visit.depth;
- nxt->objType = what->objType;
- nxt->name = ctx->nextName;
- ctx->nextName += strlen(what->objName) + 1;
- strcpy(&ctx->names[nxt->name], what->objName);
-
- if (nn) {
- if (cur->depth == nxt->depth) {
- nxt->parent = cur->parent;
- nxt->prvSibl = nn - 1;
- cur->nxtSibl = nn;
- }
- else if (cur->depth < nxt->depth) {
- nxt->parent = nn - 1;
- cur->firstChild = nn;
- }
- else {
- unsigned sibling = cur->parent;
-
- while (ctx->nodes[sibling].depth > nxt->depth)
- sibling = ctx->nodes[sibling].parent;
- nxt->parent = ctx->nodes[sibling].parent;
- nxt->prvSibl = sibling;
- ctx->nodes[sibling].nxtSibl = nn;
- }
- }
- return 0;
-}
-
-static rc_t CC report(CCReportInfoBlock const *what, void *Ctx)
-{
- cc_context_t *ctx = Ctx;
- rc_t rc = Quitting();
-
- if (rc)
- return rc;
-
- if (what->type == ccrpt_Visit)
- return visiting(what, ctx);
-
- switch (what->objType) {
- case kptDatabase:
- return report_database(what, ctx);
- case kptTable:
- return report_table(what, ctx);
- case kptColumn:
- return report_column(what, ctx);
- case kptIndex:
- return report_index(what, ctx);
- default:
- return RC(rcExe, rcTable, rcVisiting, rcParam, rcUnexpected);
- }
-}
-
-static
-rc_t kdbcc(const KDirectory *dir, char const name[], uint32_t mode, bool *is_db,
- bool is_file, node_t nodes[], char names[], INSDC_SRA_platform_id platform)
-{
- const KDBManager *mgr;
- rc_t rc = KDBManagerMakeRead ( & mgr, NULL );
-
- if ( rc == 0 )
- {
- cc_context_t ctx;
- char const *objtype;
-
- uint32_t level = ( mode & 4 ) ? 3 : ( mode & 2 ) ? 1 : 0;
- if (s_IndexOnly) {
- level |= CC_INDEX_ONLY;
- }
-
- memset(&ctx, 0, sizeof(ctx));
- ctx.nodes = &nodes[0];
- ctx.names = &names[0];
-
- * is_db = KDBManagerExists ( mgr, kptDatabase, "%s", name );
- if ( * is_db )
- {
- const KDatabase *db;
-
- objtype = "database";
- rc = KDBManagerOpenDBRead ( mgr, & db, "%s", name );
- if ( rc == 0 )
- {
- rc = KDatabaseConsistencyCheck ( db, 0, level, report, & ctx );
- if ( rc == 0 ) {
- rc = ctx.rc;
- if (s_IndexOnly)
- { (void)LOGMSG(klogInfo, "Indices: checked"); }
- }
- KDatabaseRelease ( db );
- }
- }
- else
- {
- const KTable *tbl;
-
- objtype = "table";
- rc = KDBManagerOpenTableRead ( mgr, & tbl, "%s", name );
- if ( rc == 0 )
- {
- rc = KTableConsistencyCheck ( tbl, 0, level, report, & ctx,
- platform );
- if ( rc == 0 ) {
- rc = ctx.rc;
- }
- if ( rc == 0 && s_IndexOnly ) {
- (void)LOGMSG(klogInfo, "Index: checked");
- }
- KTableRelease ( tbl );
- }
- }
- if (rc == 0 && ctx.num_columns == 0 && !s_IndexOnly) {
- if (is_file) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Nothing to validate; the file '$(file)' has no checksums or is truncated.", "file=%s", name));
- }
- else {
- (void)PLOGMSG(klogWarn, (klogWarn, "Nothing to validate; the $(type) '$(file)' has no checksums or is empty.", "type=%s,file=%s", objtype, name));
- }
- }
-
- KDBManagerRelease ( mgr );
- }
- return rc;
-}
-
-static
-rc_t vdbcc(const KDirectory *dir, char const name[], uint32_t mode, bool is_db, bool is_file)
-{
-#if 0
- if (mode & 8) {
- const VDBManager *mgr;
- rc_t rc = VDBManagerMakeRead(&mgr, dir);
-
- if (rc == 0) {
- const VTable *tbl;
-
- rc = VDBManagerOpenTableRead(mgr, &tbl, NULL, "%s", name);
- if (rc == 0)
- rc = VTableConsistencyCheck(tbl, 2);
- }
- return rc;
- }
-#endif
- return 0;
-}
-
-typedef struct ColumnInfo_s {
- char const *name;
- union {
- void const *vp;
- char const *string;
- bool const *tf;
- int8_t const *i8;
- uint8_t const *u8;
- int16_t const *i16;
- uint16_t const *u16;
- int32_t const *i32;
- uint32_t const *u32;
- int64_t const *i64;
- uint64_t const *u64;
- float const *f32;
- double const *f64;
- } value;
- uint32_t idx;
- uint32_t elem_bits;
- uint32_t elem_count;
-} ColumnInfo;
-
-static rc_t CC get_sizes_cb(const KDirectory *dir, uint32_t type, const char *name, void *data)
-{
- struct {
- unsigned count;
- size_t size;
- } *pb = data;
-
- ++pb->count;
- pb->size += strlen(name) + 1;
-
- return 0;
-}
-
-static rc_t get_sizes(KDirectory const *dir, unsigned *nobj, size_t *namesz)
-{
- rc_t rc;
- struct {
- unsigned count;
- size_t size;
- } pb;
-
- memset(&pb, 0, sizeof(pb));
- rc = KDirectoryVVisit(dir, true, get_sizes_cb, &pb, NULL, NULL);
- if (rc)
- memset(&pb, 0, sizeof(pb));
- *nobj = pb.count;
- *namesz = pb.size;
- return rc;
-}
-
-static rc_t init_dbcc(KDirectory const *dir, char const name[], bool is_file, node_t **nodes, char **names)
-{
- KDirectory const *obj;
- unsigned nobj;
- size_t namesz;
- rc_t rc;
-
- if (is_file) {
- rc = KDirectoryOpenSraArchiveRead_silent(dir, &obj, false, "%s", name);
- if (rc)
- rc = KDirectoryOpenTarArchiveRead_silent(dir, &obj, false, "%s", name);
- }
- else {
- rc = KDirectoryOpenDirRead(dir, &obj, false, "%s", name);
- }
- if (rc)
- return rc;
- rc = get_sizes(obj, &nobj, &namesz);
- KDirectoryRelease(obj);
- if (rc) {
- *nodes = NULL;
- *names = NULL;
- }
- else {
- *nodes = malloc(nobj * sizeof(**nodes) + namesz);
- if (nodes)
- *names = (char *)&(*nodes)[nobj];
- else
- rc = RC(rcExe, rcSelf, rcConstructing, rcMemory, rcExhausted);
- }
- return rc;
-}
-
-static rc_t get_schema_info(KMetadata const *meta, char buffer[], size_t bsz, char **vers)
-{
- KMDataNode const *node;
- rc_t rc = KMetadataOpenNodeRead(meta, &node, "schema");
-
- if (rc == 0) {
- size_t sz;
-
- rc = KMDataNodeReadAttr(node, "name", buffer, bsz, &sz);
- if (rc == 0) {
- buffer[sz] = '\0';
- *vers = &buffer[sz];
- while (sz) {
- --sz;
- if (buffer[sz] == '#') {
- buffer[sz] = '\0';
- *vers = &buffer[sz + 1];
- break;
- }
- }
- }
- }
- return rc;
-}
-
-static rc_t get_tbl_schema_info(VTable const *tbl, char buffer[], size_t bsz, char **vers)
-{
- KMetadata const *meta;
- rc_t rc = VTableOpenMetadataRead(tbl, &meta);
-
- *(*vers = &buffer[0]) = '\0';
- if (rc == 0) rc = get_schema_info(meta, buffer, bsz, vers);
- return 0;
-}
-
-static rc_t get_db_schema_info(VDatabase const *db, char buffer[], size_t bsz, char **vers)
-{
- KMetadata const *meta;
- rc_t rc = VDatabaseOpenMetadataRead(db, &meta);
-
- *(*vers = &buffer[0]) = '\0';
- if (rc == 0) rc = get_schema_info(meta, buffer, bsz, vers);
- return rc;
-}
-
-static rc_t sra_dbcc_454(VTable const *tbl, char const name[])
-{
- /* TODO: complete this */
- return 0;
-}
-
-static rc_t sra_dbcc_fastq(VTable const *tbl, char const name[])
-{
- static char const *const cn_FastQ[] = {
- "READ", "QUALITY", "SPOT_LEN", "READ_START", "READ_LEN", "READ_TYPE"
- };
-
- VCursor const *curs;
- rc_t rc = VTableCreateCursorRead(tbl, &curs);
-
- if (rc == 0) {
- unsigned const n = sizeof(cn_FastQ)/sizeof(cn_FastQ[0]);
- ColumnInfo cols[sizeof(cn_FastQ)/sizeof(cn_FastQ[0])];
- unsigned i;
-
- memset(cols, 0, sizeof(cols));
- for (i = 0; i < n; ++i) {
- cols[i].name = cn_FastQ[i];
- VCursorAddColumn(curs, &cols[i].idx, "%s", cols[i].name);
- }
- rc = VCursorOpen(curs);
- if (rc == 0) {
- rc = VCursorOpenRow(curs);
- for (i = 0; i < n && rc == 0; ++i) {
- VCursorCellData(curs, cols[i].idx, &cols[i].elem_bits, &cols[i].value.vp, NULL, &cols[i].elem_count);
- }
- if ( cols[0].idx == 0 || cols[0].elem_bits == 0 || cols[0].value.vp == NULL
- || cols[1].idx == 0 || cols[1].elem_bits == 0 || cols[1].value.vp == NULL)
- {
- rc = RC(rcExe, rcTable, rcValidating, rcColumn, rcNotFound);
- }
- else if (cols[2].idx == 0 || cols[2].elem_bits == 0 || cols[2].value.vp == NULL) {
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcExe, rcTable, rcValidating, rcColumn, rcNotFound),
- "Table '$(name)' is usable for fasta only; no quality data", "name=%s", name));
- }
- }
- VCursorRelease(curs);
- }
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Table '$(name)' is damaged beyond any use", "name=%s", name));
- }
- return rc;
-}
-
-static rc_t VTable_get_platform(VTable const *tbl, INSDC_SRA_platform_id *rslt)
-{
- rc_t rc;
- VCursor const *curs;
- INSDC_SRA_platform_id platform = -1;
-
- rc = VTableCreateCursorRead(tbl, &curs);
- if (rc == 0) {
- uint32_t cid;
-
- rc = VCursorAddColumn(curs, &cid, "("sra_platform_id_t")PLATFORM");
- if (rc == 0) {
- rc = VCursorOpen(curs);
- if (rc == 0) {
- uint32_t ebits;
- void const *data;
- uint32_t boff;
- uint32_t ecnt;
-
- rc = VCursorCellDataDirect(curs, 1, cid, &ebits, &data, &boff, &ecnt);
- if (rc == 0) {
- if (ebits == sizeof(platform) * 8 && boff == 0 && ecnt == 1)
- platform = ((INSDC_SRA_platform_id *)data)[0];
- else
- rc = RC(rcExe, rcTable, rcReading, rcType, rcUnexpected);
- }
- }
- }
- else
- rc = 0;
- VCursorRelease(curs);
- }
- rslt[0] = platform;
- return rc;
-}
-
-static rc_t verify_table(VTable const *tbl, char const name[])
-{
- char schemaName[1024];
- char *schemaVers = NULL;
- rc_t rc = 0;
-
- get_tbl_schema_info(tbl, schemaName, sizeof(schemaName), &schemaVers);
-
- if (schemaName[0] == '\0' || strncmp(schemaName, "NCBI:SRA:", 9) == 0) {
- /* SRA or legacy SRA */
- INSDC_SRA_platform_id platform;
-
- rc = VTable_get_platform(tbl, &platform);
- if (rc == 0) {
- if (platform == (INSDC_SRA_platform_id)-1) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Couldn't determine SRA Platform; type of table '$(name)' is indeterminate.", "name=%s", name));
- }
- rc = sra_dbcc_fastq(tbl, name);
- if (rc == 0 && platform == SRA_PLATFORM_454) {
- rc = sra_dbcc_454(tbl, name);
- }
- }
- else {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to read table '$(name)'", "name=%s", name));
- }
- }
- return rc;
-}
-
-static rc_t verify_mgr_table(VDBManager const *mgr, char const name[])
-{
- VTable const *tbl;
- VSchema *sra_schema = NULL;
-
- for ( ; ; ) {
- rc_t rc = VDBManagerOpenTableRead(mgr, &tbl, sra_schema, "%s", name);
- VSchemaRelease(sra_schema);
- if (rc == 0) {
- rc = verify_table(tbl, name);
- VTableRelease(tbl);
- return rc;
- }
- else if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcSchema && sra_schema == NULL) {
- rc = VDBManagerMakeSRASchema(mgr, &sra_schema);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open table '$(name)'", "name=%s", name));
- return rc;
- }
- }
- else {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open table '$(name)'", "name=%s", name));
- return rc;
- }
- }
-}
-
-#if 0
-static rc_t verify_db_table(VDatabase const *db, char const name[])
-{
- VTable const *tbl;
- rc_t rc = VDatabaseOpenTableRead(db, &tbl, "%s", name);
-
- if (rc == 0) {
- rc = verify_table(tbl, name);
- VTableRelease(tbl);
- }
- else {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to open table '$(name)'", "name=%s", name));
- }
- return rc;
-}
-
-static rc_t align_dbcc_primary_alignment(VTable const *tbl, char const name[])
-{
- VCursor const *curs;
- rc_t rc = VTableCreateCursorRead(tbl, &curs);
-
- if (rc == 0) {
- static char const *const cn_SAM[] = {
- "SEQ_NAME",
- "SAM_FLAGS",
- "REF_NAME",
- "REF_POS",
- "MAPQ",
- "HAS_MISMATCH",
- "HAS_REF_OFFSET",
- "REF_OFFSET",
- "MATE_REF_NAME",
- "MATE_REF_POS",
- "TEMPLATE_LEN",
- "READ",
- "QUALITY"
- };
- unsigned const n = sizeof(cn_SAM)/sizeof(cn_SAM[0]);
- ColumnInfo cols[sizeof(cn_SAM)/sizeof(cn_SAM[0])];
- unsigned i;
-
- memset(cols, 0, sizeof(cols));
- for (i = 0; i < n; ++i) {
- cols[i].name = cn_SAM[i];
- VCursorAddColumn(curs, &cols[i].idx, "%s", cols[i].name);
- }
- rc = VCursorOpen(curs);
- if (rc == 0) {
- rc = VCursorOpenRow(curs);
- for (i = 0; i < n && rc == 0; ++i) {
- VCursorCellData(curs, cols[i].idx, &cols[i].elem_bits, &cols[i].value.vp, NULL, &cols[i].elem_count);
- }
- for (i = 0; i < n && rc == 0; ++i) {
- if (cols[i].idx == 0 || cols[i].elem_bits == 0) {
- (void)PLOGMSG(klogInfo, (klogInfo, "Database '$(name)' could not be used to generate SAM", "name=%s", name));
- break;
- }
- }
- }
- VCursorRelease(curs);
- }
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)' is damaged beyond any use", "name=%s", name));
- }
- return rc;
-}
-#endif
-
-typedef struct id_pair_s {
- int64_t first;
- int64_t second;
-} id_pair_t;
-
-static int CC id_pair_cmp(void const *A, void const *B, void *ignored)
-{
- id_pair_t const *a = A;
- id_pair_t const *b = B;
-
- return a->first < b->first ? -1 : a->first == b->first ? a->second < b->second ? -1 : a->second == b->second ? 0 : 1 : 1;
-}
-
-static rc_t ric_align_ref_and_align(char const dbname[],
- VTable const *ref,
- VTable const *align,
- int which)
-{
- char const *const id_col_name = which == 0 ? "PRIMARY_ALIGNMENT_IDS"
- : which == 1 ? "SECONDARY_ALIGNMENT_IDS"
- : which == 2 ? "EVIDENCE_ALIGNMENT_IDS"
- : NULL;
- rc_t rc;
- VCursor const *curs = NULL;
- ColumnInfo ci;
- int64_t startId;
- uint64_t count;
-
- rc = VTableCreateCursorRead(align, &curs);
- if (rc == 0) {
- rc = VCursorAddColumn(curs, &ci.idx, "REF_ID");
- if (rc == 0)
- rc = VCursorOpen(curs);
- if (rc == 0)
- rc = VCursorIdRange(curs, ci.idx, &startId, &count);
- }
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': alignment table can not be read", "name=%s", dbname));
- }
- else {
- id_pair_t *const id_pair = malloc(sizeof(id_pair_t) * count);
-
- if (id_pair) {
- uint64_t i;
- uint64_t j;
-
- for (j = i = 0; i < count && rc == 0; ++i) {
- int64_t const row = startId + i;
-
- rc = VCursorCellDataDirect(curs, row, ci.idx, &ci.elem_bits, &ci.value.vp, NULL, &ci.elem_count);
- if (rc == 0) {
- if (ci.elem_count != 1) {
- rc = RC(rcExe, rcDatabase, rcValidating, rcData, rcUnexpected);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': failed referential integrity check", "name=%s", dbname));
- break;
- }
- else {
- id_pair[j].second = row;
- id_pair[j].first = ci.value.i64[0];
- ++j;
- }
- }
- else if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound)
- rc = 0;
- }
- VCursorRelease(curs); curs = NULL;
- if (rc == 0) {
- bool ooo_warned = false;
- bool failed = false;
-
- ksort(id_pair, count, sizeof(id_pair_t), id_pair_cmp, NULL);
-
- rc = VTableCreateCursorRead(ref, &curs);
- if (rc == 0)
- rc = VCursorAddColumn(curs, &ci.idx, "%s", id_col_name);
- if (rc == 0)
- rc = VCursorOpen(curs);
- if (rc == 0)
- rc = VCursorIdRange(curs, ci.idx, &startId, &count);
- if (rc == 0) {
- for (i = j = 0; rc == 0 && i < count; ++i) {
- int64_t const row = startId + i;
-
- rc = VCursorCellDataDirect(curs, row, ci.idx, &ci.elem_bits, &ci.value.vp, NULL, &ci.elem_count);
- if (rc == 0) {
- unsigned k;
- int64_t prvId = ci.value.i64[0];
-
- for (k = 0; rc == 0 && k < ci.elem_count; ++k, ++j) {
- int64_t const alignId = ci.value.i64[k];
-
- if (!ooo_warned && prvId > alignId) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Database '$(name)': column '$(idcol)' is not ordered", "name=%s,idcol=%s", dbname, id_col_name));
- ooo_warned = true;
- }
- if (id_pair[j].first != row) {
- if (!failed) {
- rc = RC(rcExe, rcDatabase, rcValidating, rcData, rcInconsistent);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': column '$(idcol)' failed referential integrity check", "name=%s,idcol=%s", dbname, id_col_name));
- }
- failed = true;
- }
- else if (id_pair[j].second != alignId) {
- if (!ooo_warned) {
- (void)PLOGMSG(klogWarn, (klogWarn, "Database '$(name)': column '$(idcol)' might fail referential integrity check", "name=%s,idcol=%s", dbname, id_col_name));
- }
- }
- prvId = alignId;
- }
- }
- else if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound)
- rc = 0;
- }
- if (!failed && i < count) {
- rc = RC(rcExe, rcDatabase, rcValidating, rcData, rcInconsistent);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': column '$(idcol)' failed referential integrity check", "name=%s,idcol=%s", dbname, id_col_name));
- }
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': reference table can not be read", "name=%s", dbname));
- }
- free(id_pair);
- VCursorRelease(curs);
- }
- else {
- rc = RC(rcExe, rcDatabase, rcValidating, rcMemory, rcExhausted);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': referential integrity could not be checked", "name=%s", dbname));
- }
- }
- return rc;
-}
-
-static rc_t ric_align_seq_and_pri(char const dbname[],
- VTable const *seq,
- VTable const *pri)
-{
- rc_t rc;
- VCursor const *curs = NULL;
- ColumnInfo ci;
- int64_t startId;
- uint64_t count;
-
- rc = VTableCreateCursorRead(pri, &curs);
- if (rc == 0) {
- rc = VCursorAddColumn(curs, &ci.idx, "SEQ_SPOT_ID");
- if (rc == 0)
- rc = VCursorOpen(curs);
- if (rc == 0)
- rc = VCursorIdRange(curs, ci.idx, &startId, &count);
- }
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': alignment table can not be read", "name=%s", dbname));
- }
- else {
- id_pair_t *const id_pair = malloc(sizeof(id_pair_t) * count);
-
- if (id_pair) {
- uint64_t i;
- uint64_t j;
-
- for (j = i = 0; i < count && rc == 0; ++i) {
- int64_t const row = startId + i;
-
- rc = VCursorCellDataDirect(curs, row, ci.idx, &ci.elem_bits, &ci.value.vp, NULL, &ci.elem_count);
- if (rc == 0) {
- if (ci.elem_count != 1) {
- rc = RC(rcExe, rcDatabase, rcValidating, rcData, rcUnexpected);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': failed referential integrity check", "name=%s", dbname));
- break;
- }
- else {
- id_pair[j].second = row;
- id_pair[j].first = ci.value.i64[0];
- ++j;
- }
- }
- else if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound)
- rc = 0;
- }
- VCursorRelease(curs); curs = NULL;
- if (rc == 0) {
- ksort(id_pair, count, sizeof(id_pair_t), id_pair_cmp, NULL);
-
- rc = VTableCreateCursorRead(seq, &curs);
- if (rc == 0)
- rc = VCursorAddColumn(curs, &ci.idx, "PRIMARY_ALIGNMENT_ID");
- if (rc == 0)
- rc = VCursorOpen(curs);
- if (rc == 0) {
- for (i = 0; rc == 0 && i < count; ++i) {
- int64_t const row = id_pair[i].first;
- int64_t const alignId = id_pair[i].second;
-
- rc = VCursorCellDataDirect(curs, row, ci.idx, &ci.elem_bits, &ci.value.vp, NULL, &ci.elem_count);
- if (rc == 0) {
- bool found = false;
-
- for (j = 0; j < ci.elem_count; ++j)
- found |= (alignId == ci.value.i64[j]);
-
- if (!found) {
- rc = RC(rcExe, rcDatabase, rcValidating, rcData, rcInconsistent);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': column 'SEQ_SPOT_ID' failed referential integrity check", "name=%s", dbname));
- }
- }
- else if (GetRCObject(rc) == rcRow && GetRCState(rc) == rcNotFound) {
- rc = RC(rcExe, rcDatabase, rcValidating, rcData, rcInconsistent);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': column 'SEQ_SPOT_ID' failed referential integrity check", "name=%s", dbname));
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': sequence table can not be read", "name=%s", dbname));
- }
- }
- else
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': sequence table can not be read", "name=%s", dbname));
- }
- free(id_pair);
- }
- else {
- rc = RC(rcExe, rcDatabase, rcValidating, rcMemory, rcExhausted);
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(name)': referential integrity could not be checked", "name=%s", dbname));
- }
- VCursorRelease(curs);
- }
- return rc;
-}
-
-/* database referential integrity check for alignment database */
-static rc_t dbric_align(char const dbname[],
- VTable const *pri,
- VTable const *seq,
- VTable const *ref)
-{
- rc_t rc = 0;
-
- if ((rc == 0 || exhaustive) && (pri != NULL && seq != NULL)) {
- rc_t rc2 = ric_align_seq_and_pri(dbname, seq, pri);
-
- if (rc2 == 0) {
- (void)PLOGMSG(klogInfo, (klogInfo, "Database '$(dbname)': SEQUENCE.PRIMARY_ALIGNMENT_ID <-> PRIMARY_ALIGNMENT.SEQ_SPOT_ID referential integrity ok", "dbname=%s", dbname));
- }
- if (rc == 0) {
- rc = rc2;
- }
- }
- if ((rc == 0 || exhaustive) && (pri != NULL && ref != NULL)) {
- rc_t rc2 = ric_align_ref_and_align(dbname, ref, pri, 0);
-
- if (rc2 == 0) {
- (void)PLOGMSG(klogInfo, (klogInfo, "Database '$(dbname)': REFERENCE.PRIMARY_ALIGNMENT_IDS <-> PRIMARY_ALIGNMENT.REF_ID referential integrity ok", "dbname=%s", dbname));
- }
- if (rc == 0) {
- rc = rc2;
- }
- }
- return rc;
-}
-
-
-static rc_t verify_database_align(VDatabase const *db, char const name[], node_t const nodes[], char const names[])
-{
- rc_t rc = 0;
- unsigned tables = 0;
- node_t const *tbl = &nodes[nodes[0].firstChild];
- enum table_bits {
- tbEvidenceInterval = ( 1u << 0 ),
- tbEvidenceAlignment = ( 1u << 1 ),
- tbPrimaryAlignment = ( 1u << 2 ),
- tbReference = ( 1u << 3 ),
- tbSequence = ( 1u << 4 ),
- tbSecondaryAlignment = ( 1u << 5 )
- };
-
- if (nodes[0].firstChild) {
- for ( ; ; ) {
- char const *tname = &names[tbl->name];
- unsigned this_table = 0;
-
- if (tbl->objType == kptTable) {
- switch (tname[0]) {
- case 'E':
- if (strcmp(tname, "EVIDENCE_INTERVAL") == 0)
- this_table |= tbEvidenceInterval;
- else if (strcmp(tname, "EVIDENCE_ALIGNMENT") == 0)
- this_table |= tbEvidenceAlignment;
- break;
- case 'P':
- if (strcmp(tname, "PRIMARY_ALIGNMENT") == 0)
- this_table |= tbPrimaryAlignment;
- break;
- case 'R':
- if (strcmp(tname, "REFERENCE") == 0)
- this_table |= tbReference;
- break;
- case 'S':
- if (strcmp(tname, "SEQUENCE") == 0)
- this_table |= tbSequence;
- else if (strcmp(tname, "SECONDARY_ALIGNMENT") == 0)
- this_table |= tbSecondaryAlignment;
- break;
- }
- if (this_table == 0) {
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcExe, rcDatabase, rcValidating, rcTable, rcUnexpected),
- "Database '$(name)' contains unexpected table '$(table)'",
- "name=%s,table=%s", name, tname));
- }
- tables |= this_table;
- }
- else {
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcExe, rcDatabase, rcValidating, rcType, rcUnexpected),
- "Database '$(name)' contains unexpected object '$(obj)'",
- "name=%s,obj=%s", name, tname));
- }
- if (tbl->nxtSibl > 0)
- tbl = &nodes[tbl->nxtSibl];
- else
- break;
- }
- }
- if (tables == tbSequence) {
- /* sequence data only */
- (void)PLOGMSG(klogInfo, (klogInfo, "Database '$(name)' contains only unaligned reads", "name=%s", name));
- }
- else if ( (tables & tbReference) == 0
- || (tables & tbPrimaryAlignment) == 0)
- {
- /* missing reference or primary alignment */
- (void)PLOGERR(klogWarn, (klogWarn, rc = RC(rcExe, rcDatabase, rcValidating, rcDatabase, rcIncomplete),
- "Database '$(name)' does not contain all required tables",
- "name=%s", name));
- }
- else if ( ((tables & tbEvidenceAlignment) != 0)
- != ((tables & tbEvidenceInterval ) != 0))
- {
- /* both must be present or both must be absent */
- (void)PLOGERR(klogWarn, (klogWarn, rc = RC(rcExe, rcDatabase, rcValidating, rcDatabase, rcIncomplete),
- "Database '$(name)' does not contain all required tables",
- "name=%s", name));
- }
- while (ref_int_check) {
- VTable const *pri = NULL;
- VTable const *seq = NULL;
- VTable const *ref = NULL;
-
- if ((tables & tbPrimaryAlignment) != 0) {
- rc = VDatabaseOpenTableRead(db, &pri, "PRIMARY_ALIGNMENT");
- if (rc) break;
- }
- if ((tables & tbSequence) != 0) {
- rc = VDatabaseOpenTableRead(db, &seq, "SEQUENCE");
- if (rc) break;
- }
- if ((tables & tbReference) != 0) {
- rc = VDatabaseOpenTableRead(db, &ref, "REFERENCE");
- if (rc) break;
- }
- rc = dbric_align(name, pri, seq, ref);
- break;
- }
- return rc;
-}
-
-static rc_t verify_database(VDatabase const *db, char const name[], node_t const nodes[], char const names[])
-{
- char schemaName[1024];
- char *schemaVers = NULL;
- rc_t rc;
-
- rc = get_db_schema_info(db, schemaName, sizeof(schemaName), &schemaVers);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc, "Failed to find database schema for '$(name)'", "name=%s", name));
- }
- else if (strncmp(schemaName, "NCBI:var:db:", 12) == 0) {
- /* TODO: verify NCBI:var:db:* */
- }
- else if (strncmp(schemaName, "NCBI:WGS:db:", 12) == 0) {
- /* TODO: verify NCBI:WGS:db:* */
- }
- else if (strncmp(schemaName, "NCBI:align:db:", 14) == 0) {
- rc = verify_database_align(db, name, nodes, names);
- }
- else if (strcmp(schemaName, "NCBI:SRA:PacBio:smrt:db") == 0) {
- /* TODO: verify NCBI:SRA:PacBio:smrt:db */
- }
- else {
- (void)PLOGERR(klogWarn, (klogWarn, RC(rcExe, rcDatabase, rcValidating, rcType, rcUnrecognized),
- "Database '$(name)' has unrecognized type '$(type)'",
- "name=%s,type=%s", name, schemaName));
- }
- return rc;
-}
-
-static rc_t verify_mgr_database(VDBManager const *mgr, char const name[], node_t const nodes[], char const names[])
-{
- VDatabase const *child;
- rc_t rc = VDBManagerOpenDBRead(mgr, &child, NULL, "%s", name);
-
- if (rc == 0) {
- rc = verify_database(child, name, nodes, names);
- VDatabaseRelease(child);
- }
- return rc;
-}
-
-static rc_t sra_dbcc(KDirectory const *dir, char const name[], node_t const nodes[], char const names[])
-{
- const VDBManager *mgr;
- rc_t rc = VDBManagerMakeRead(&mgr, NULL);
-
- if (rc) return rc;
-
- if (nodes[0].objType == kptDatabase)
- rc = verify_mgr_database(mgr, name, nodes, names);
- else
- rc = verify_mgr_table(mgr, name);
- VDBManagerRelease(mgr);
-
- return rc;
-}
-
-static
-rc_t get_platform(const KDirectory *dir, const VDBManager *aMgr,
- const VTable *aTbl, char const name[], INSDC_SRA_platform_id *platform)
-{
- rc_t rc = 0;
- const VDBManager *mgr = aMgr;
- const VTable *tbl = aTbl;
- assert(name && platform);
- if (mgr == NULL) {
- rc = VDBManagerMakeRead(&mgr, NULL);
- if (rc != 0) {
- return rc;
- }
- }
- if (tbl == NULL) {
- VSchema *sra_schema = NULL;
- for ( ; rc == 0; ) {
- rc = VDBManagerOpenTableRead(mgr, &tbl, sra_schema, "%s", name);
- VSchemaRelease(sra_schema);
- if (rc == 0) {
- rc = VTable_get_platform(tbl, platform);
- break;
- }
- else if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcSchema
- && sra_schema == NULL)
- {
- rc = VDBManagerMakeSRASchema(mgr, &sra_schema);
- }
- }
- }
- if (aTbl == NULL) {
- VTableRelease(tbl);
- }
- if (aMgr == NULL) {
- VDBManagerRelease(mgr);
- }
- return rc;
-}
-
-static
-rc_t dbcc(const char *src_path, const KDirectory *dir,
- char const name[], uint32_t mode, bool *is_db, bool is_file)
-{
- rc_t rc;
- node_t *nodes = NULL;
- char *names;
-
- assert(src_path);
-
- rc = init_dbcc(dir, name, is_file, &nodes, &names);
- if (rc == 0) {
- INSDC_SRA_platform_id platform = SRA_PLATFORM_UNDEFINED;
- get_platform(dir, NULL, NULL, src_path, &platform);
- rc = kdbcc(dir, src_path, mode, is_db, is_file, nodes, names, platform);
- if ( rc == 0 )
- rc = vdbcc(dir, src_path, mode, * is_db, is_file);
- }
- if (rc == 0)
- rc = sra_dbcc(dir, src_path, nodes, names);
-
- free(nodes);
-
- return rc;
-}
-
-static char const* const defaultLogLevel =
-#if _DEBUGGING
-"debug5";
-#else
-"info";
-#endif
-
-/******************************************************************************
- * Usage
- ******************************************************************************/
-const char UsageDefaultName[] = "sra-dbcc";
-
-rc_t CC UsageSummary(const char *prog_name)
-{
- return KOutMsg("Usage: %s [options] table\n\n", prog_name);
-}
-
-char const *help_text[] =
-{
- "Check components md5s if present, fail unless other checks are requested (default: yes)", NULL,
- "Check blobs CRC32 (default: no)", NULL,
- "Check 'skey' index (default: no)", NULL,
- "Continue checking table for all possible errors (default: no)", NULL,
- "Check data referential integrity for databases (default: no)", NULL,
- "Check index-only with blobs CRC32 (default: no)", NULL
-};
-
-OptDef Options[] =
-{
- { "md5" , "5", NULL, &help_text[0], 1, false, false },
- { "blob-crc" , "b", NULL, &help_text[2], 1, false, false },
-#if CHECK_INDEX
- { "index" , "i", NULL, &help_text[4], 1, false, false },
-#endif
- { "exhaustive", "x", NULL, &help_text[6], 1, false, false },
- { "dri" , "d", NULL, &help_text[8], 1, false, false },
-
- /* should be the last one here: it is not printed by --help */
- { "index-only",NULL, NULL, &help_text[10], 1, false, false }
-};
-#define NOPTS ( sizeof(Options)/sizeof(Options[0]))
-
-rc_t CC Usage (const Args * args)
-{
- const char * progname = UsageDefaultName;
- const char * fullpath = UsageDefaultName;
- rc_t rc;
- unsigned i;
-
- if (args == NULL)
- rc = RC (rcApp, rcArgv, rcAccessing, rcSelf, rcNull);
- else
- rc = ArgsProgram (args, &fullpath, &progname);
- if (rc)
- progname = fullpath = UsageDefaultName;
-
- UsageSummary (progname);
-
- KOutMsg ("Options:\n");
-
- for (i = 0; i != NOPTS - 1; ++i) {
- HelpOptionLine(Options[i].aliases, Options[i].name, NULL, Options[i].help);
- }
-
- HelpOptionsStandard ();
-
- HelpVersion (fullpath, KAppVersion());
-
- return 0;
-}
-
-uint32_t CC KAppVersion(void)
-{
- return SRA_DBCC_VERS;
-}
-
-#ifdef MAC
-#define SRAPathMake(...) (RC(rcRuntime, rcDylib, rcSearching, rcFunction, rcUnsupported))
-#define SRAPathFind(...) (RC(rcRuntime, rcDylib, rcSearching, rcFunction, rcUnsupported))
-#define SRAPathRelease(...) ((rc_t)(0))
-#endif
-
-rc_t CC KMain ( int argc, char *argv [] )
-{
- Args * args;
- char *src_dir_path = NULL;
- char *src_path = NULL;
- char *obj_name = NULL;
- bool md5_chk = true, md5_chk_explicit = false;
- bool blob_crc = false;
- bool index_chk = false;
- bool is_file = false;
- const KDirectory *src_dir = NULL;
-/* char path_buffer [ 4096 ]; */
- VFSManager *mgr = NULL;
- VResolver *resolver = NULL;
-
- rc_t rc = KLogLevelSet ( klogInfo );
-
- if ( rc == 0 )
- rc = ArgsMakeAndHandle(&args, argc, argv, 1, Options, NOPTS);
-
- if (rc == 0) {
- rc = VFSManagerMake(&mgr);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Failed to VFSManagerMake()");
- }
- }
-
- if (rc == 0) {
- rc = VFSManagerGetResolver(mgr, &resolver);
- if (rc != 0) {
- LOGERR(klogInt, rc, "Cannot VFSManagerGetResolver");
- }
- else {
- VResolverRemoteEnable(resolver, vrAlwaysDisable);
- }
- }
-
- RELEASE(VFSManager, mgr);
-
- while (rc == 0) {
- uint32_t pcount;
-
- rc = ArgsParamCount(args, &pcount); if (rc) break;
- if (pcount == 1) {
- KDirectory *dir;
- rc = KDirectoryNativeDir(&dir);
- if (rc == 0)
- {
- char const *value;
-
- rc = ArgsParamValue (args, 0, &value); if (rc) break;
- src_dir_path = strdup(value);
- src_path = strdup(value);
- obj_name = strrchr(src_dir_path, '/');
- if ( obj_name != NULL && obj_name [ 1 ] == 0 )
- {
- * obj_name = 0;
- obj_name = strrchr(src_dir_path, '/');
- }
- if (obj_name != NULL)
- {
- if (obj_name == src_dir_path) {
- src_dir_path[0] = '/';
- src_dir_path[1] = '\0';
- }
- *obj_name++ = '\0';
- }
- else
- {
- uint32_t type
- = KDirectoryPathType(dir, "%s", src_path) & ~kptAlias;
- if (type != kptFile && type != kptDir) {
- /* check for accession */
- VPath *acc = NULL;
- const VPath *pLocal = NULL;
- const String *local = NULL;
-
- if (rc == 0) {
- rc = VPathMake(&acc, src_path);
- if (rc != 0) {
- PLOGERR(klogErr, (klogErr, rc,
- "VPathMake($(path)) failed",
- PLOG_S(path), src_path));
- }
- else {
- rc = VResolverLocal(resolver, acc, &pLocal);
- }
- }
- if (rc == 0) {
- rc = VPathMakeString(pLocal, &local);
- if (rc != 0) {
- PLOGERR(klogErr, (klogErr, rc,
- "VPathMake(local $(path)) failed",
- PLOG_S(path), src_path));
- }
- }
- if ( rc == 0 )
- {
- PLOGMSG(klogInfo, (klogInfo,
- "Validating '$(path)'...", PLOG_S(path),
- local->addr));
- /* use mapped path */
- free(src_path);
- src_path = string_dup(local->addr, local->size);
- free(src_dir_path);
- src_dir_path = string_dup(local->addr, local->size);
- obj_name = strrchr(src_dir_path, '/');
- if ( obj_name == NULL )
- obj_name = src_dir_path;
- else
- {
- if ( obj_name == src_dir_path ) {
- src_dir_path[0] = '/';
- src_dir_path[1] = '\0';
- }
- * obj_name ++ = '\0';
- }
- }
-
- RELEASE(VPath, pLocal);
- RELEASE(VPath, acc);
- free((void*)local);
- }
- else {
- char full[PATH_MAX];
- rc = KDirectoryResolvePath(dir, true,
- full, sizeof full, "%s", src_path);
- if (rc == 0) {
- PLOGMSG(klogInfo, (klogInfo,
- "Validating '$(path)'...", PLOG_S(path),
- full));
- /* use mapped path */
- free(src_path);
- src_path = string_dup(full, strlen(full));
- free(src_dir_path);
- src_dir_path = string_dup(full, strlen(full));
- obj_name = strrchr(src_dir_path, '/');
- if ( obj_name == NULL )
- obj_name = src_dir_path;
- else
- {
- if ( obj_name == src_dir_path ) {
- src_dir_path[0] = '/';
- src_dir_path[1] = '\0';
- }
- * obj_name ++ = '\0';
- }
- }
- }
-
- if ( rc != 0 )
- {
- /* appears to be a simple name */
- obj_name = strdup(src_dir_path);
- src_dir_path[0] = '.';
- src_dir_path[1] = '\0';
- }
- }
-
- rc = KDirectoryOpenDirRead(dir, &src_dir, false, "%s", src_dir_path);
- KDirectoryRelease(dir);
- if (rc) {
- (void)PLOGERR(klogErr, (klogErr, rc,
- "Failed to open '$(dir)'", "dir=%s", src_dir_path));
- break;
- }
- else {
- uint32_t const obj_type = KDirectoryPathType(src_dir, "%s", obj_name) & (~((uint32_t)kptAlias));
-
- switch (obj_type) {
- case kptFile:
- is_file = true;
- break;
- default:
- break;
- }
- }
- }
- }
- else {
- MiniUsage(args);
- ArgsWhack(args);
- exit(EXIT_FAILURE);
- }
-
- rc = ArgsOptionCount(args, "md5", &pcount); if (rc) break;
- md5_chk_explicit = md5_required = (pcount == 1);
- if (pcount > 1) {
- KOutMsg("argument given too many times\n");
- MiniUsage(args);
- break;
- }
-
- rc = ArgsOptionCount(args, "blob-crc", &pcount); if (rc) break;
- blob_crc = (pcount == 1);
- if (pcount > 1) {
- KOutMsg("argument given too many times\n");
- MiniUsage(args);
- break;
- }
-
- rc = ArgsOptionCount(args, "exhaustive", &pcount); if (rc) break;
- exhaustive = (pcount == 1);
- if (pcount > 1) {
- KOutMsg("argument given too many times\n");
- MiniUsage(args);
- break;
- }
-
- rc = ArgsOptionCount(args, "dri", &pcount); if (rc) break;
- ref_int_check = (pcount == 1);
- if (pcount > 1) {
- KOutMsg("argument given too many times\n");
- MiniUsage(args);
- break;
- }
-
-#if CHECK_INDEX
- rc = ArgsOptionCount(args, "index", &pcount); if (rc) break;
- index_chk = (pcount == 1);
- if (pcount > 1) {
- KOutMsg("argument given too many times\n");
- MiniUsage(args);
- break;
- }
-#endif
-
- rc = ArgsOptionCount(args, "index-only", &pcount);
- if (rc) {
- break;
- }
- if (pcount > 0) {
- s_IndexOnly = blob_crc = true;
- }
-
-
- if (blob_crc || index_chk)
- {
- /* if blob or index is requested, md5 is off unless explicitly requested */
- md5_chk = md5_chk_explicit;
- }
-
- {
- bool is_db = false;
- rc = dbcc(src_path, src_dir, obj_name, (md5_chk ? 1 : 0)|(blob_crc ? 2 : 0)|(index_chk ? 4 : 0), & is_db, is_file);
- if ( is_db )
- {
- if ( rc != 0 )
- (void)PLOGERR(klogErr, (klogErr, rc, "Database '$(db)' check failed", PLOG_S(db), obj_name));
- else
- (void)PLOGMSG(klogInfo, (klogInfo, "Database '$(db)' is consistent", PLOG_S(db), obj_name));
- }
- else
- {
- if ( rc != 0 )
- (void)PLOGERR(klogErr, (klogErr, rc, "Table '$(table)' check failed", PLOG_S(table), obj_name));
- else
- (void)PLOGMSG(klogInfo, (klogInfo, "Table '$(table)' is consistent", PLOG_S(table), obj_name));
- }
- }
- break;
- }
-
- RELEASE(VResolver, resolver);
- RELEASE(KDirectory, src_dir);
- free(src_path);
-
- return rc;
-}
diff --git a/tools/sra-dbcc/sra-dbcc.vers b/tools/sra-dbcc/sra-dbcc.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/sra-dbcc/sra-dbcc.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/sra-dump/abi-dump.vers b/tools/sra-dump/abi-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-dump/abi-dump.vers
+++ b/tools/sra-dump/abi-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-dump/abi-dump.vers.h b/tools/sra-dump/abi-dump.vers.h
index bfbcf41..5b8896f 100644
--- a/tools/sra-dump/abi-dump.vers.h
+++ b/tools/sra-dump/abi-dump.vers.h
@@ -1 +1 @@
-#define ABI_DUMP_VERS 0x02040002
+#define ABI_DUMP_VERS 0x02050007
diff --git a/tools/sra-dump/abi.c b/tools/sra-dump/abi.c
index 7fd50b9..d5ca2ac 100644
--- a/tools/sra-dump/abi.c
+++ b/tools/sra-dump/abi.c
@@ -158,8 +158,8 @@ rc_t Absolid2BioFilterFactory_Make(const SRASplitterFactory** cself, const char*
typedef struct AbsolidLabelerFilter_struct {
const AbsolidReader* reader;
bool is_platform_cs_native;
- SRASplitter_Keys keys[7];
- size_t key_sz[7];
+ SRASplitter_Keys keys[8];
+ size_t key_sz[8];
} AbsolidLabelerFilter;
static
diff --git a/tools/sra-dump/core.c b/tools/sra-dump/core.c
index fdf807b..9224799 100644
--- a/tools/sra-dump/core.c
+++ b/tools/sra-dump/core.c
@@ -36,7 +36,6 @@
#include <klib/container.h>
#include <klib/log.h>
-#include <klib/misc.h> /* SetHttpTries */
#include <klib/report.h> /* ReportInit */
#include <klib/out.h>
#include <klib/status.h>
@@ -687,7 +686,6 @@ static const SRADumperFmt_Arg KMainArgs[] =
"Current/default is warn", NULL } },
{ "v", "verbose", NULL, { "Increase the verbosity level of the program",
"Use multiple times for more verbosity", NULL } },
- { NULL, TRIES_OPTION, TRIES_ARGUMENT, { TRIES_HELP, NULL } },
{ NULL, OPTION_REPORT, NULL, { "Control program execution environment report generation (if implemented).",
"One of (never|error|always). Default is error", NULL } },
#if _DEBUGGING
@@ -978,7 +976,7 @@ rc_t CC KMain ( int argc, char* argv[] )
bool to_stdout = false, do_gzip = false, do_bzip2 = false;
char const* outdir = NULL;
spotid_t minSpotId = 1;
- spotid_t maxSpotId = ~0;
+ spotid_t maxSpotId = 0x7FFFFFFFFFFFFFFF; /* 9,223,372,036,854,775,807 max int64_t value !!! ~0 is wrong !!! */
bool sub_dir = false;
bool keep_empty = false;
const char* table_path[10240];
@@ -1070,11 +1068,6 @@ rc_t CC KMain ( int argc, char* argv[] )
goto Catch;
}
}
- else if
- (SRADumper_GetArg(&fmt, NULL, TRIES_OPTION, &i, argc, argv, &arg ))
- {
- SetHttpTries(AsciiToU32(arg, NULL, NULL));
- }
else if ( SRADumper_GetArg( &fmt, NULL, "disable-multithreading", &i, argc, argv, NULL ) )
{
no_mt = true;
diff --git a/tools/sra-dump/factory.c b/tools/sra-dump/factory.c
index 9404435..771788b 100644
--- a/tools/sra-dump/factory.c
+++ b/tools/sra-dump/factory.c
@@ -605,7 +605,7 @@ void CC SRASplitter_Child_Whack(BSTNode* node, void* data)
}
static
-int CC SRASplitter_Child_Cmp(const BSTNode* item, const BSTNode* node)
+int64_t CC SRASplitter_Child_Cmp(const BSTNode* item, const BSTNode* node)
{
const SRASplitter_Child* i = (const SRASplitter_Child*)item;
const SRASplitter_Child* n = (const SRASplitter_Child*)node;
@@ -613,7 +613,7 @@ int CC SRASplitter_Child_Cmp(const BSTNode* item, const BSTNode* node)
}
static
-int CC SRASplitter_Child_Find(const void* item, const BSTNode* node)
+int64_t CC SRASplitter_Child_Find(const void* item, const BSTNode* node)
{
const char* key = (const char*)item;
const SRASplitter_Child* n = (const SRASplitter_Child*)node;
diff --git a/tools/sra-dump/fasta_dump.c b/tools/sra-dump/fasta_dump.c
index 41e7b58..2ef3ac9 100644
--- a/tools/sra-dump/fasta_dump.c
+++ b/tools/sra-dump/fasta_dump.c
@@ -151,7 +151,7 @@ static rc_t get_str_option( const Args *args, const char *name, const char ** re
{
if ( count > 0 )
{
- rc = ArgsOptionValue( args, name, 0, res );
+ rc = ArgsOptionValue( args, name, 0, (const void **)res );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsOptionValue() failed" );
@@ -560,7 +560,7 @@ static rc_t foreach_refname( Args * args, fasta_options *opts, fasta_ctx *ctx, c
for ( i = 0; i < count && rc == 0; ++i )
{
const char * refname;
- rc = ArgsOptionValue( args, OPTION_REFNAME, 0, &refname );
+ rc = ArgsOptionValue( args, OPTION_REFNAME, 0, (const void **)&refname );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsOptionValue( REFNAME ) failed" );
@@ -601,7 +601,7 @@ static rc_t foreach_argument( Args * args, fasta_options *opts, fasta_ctx *ctx )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *param = NULL;
- rc = ArgsParamValue( args, idx, ¶m );
+ rc = ArgsParamValue( args, idx, (const void **)¶m );
if ( rc != 0 )
LOGERR( klogInt, rc, "ArgsParamvalue() failed" );
else
diff --git a/tools/sra-dump/fastq-dump.vers b/tools/sra-dump/fastq-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-dump/fastq-dump.vers
+++ b/tools/sra-dump/fastq-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-dump/fastq-dump.vers.h b/tools/sra-dump/fastq-dump.vers.h
index c74e051..22da127 100644
--- a/tools/sra-dump/fastq-dump.vers.h
+++ b/tools/sra-dump/fastq-dump.vers.h
@@ -1 +1 @@
-#define FASTQ_DUMP_VERS 0x02040002
+#define FASTQ_DUMP_VERS 0x02050007
diff --git a/tools/sra-dump/fastq.c b/tools/sra-dump/fastq.c
index b415d64..b74edec 100644
--- a/tools/sra-dump/fastq.c
+++ b/tools/sra-dump/fastq.c
@@ -209,7 +209,7 @@ static bool CC Defline_Builder( SLNode *node, void *data )
if ( d->values[ n->type ].id != NULL &&
( !d->optional || *d->values[ n->type ].id > 0 ) )
{
- x = snprintf( s, sizeof( s ), "%u", *d->values[ n->type ].id );
+ x = snprintf( s, sizeof( s ), "%ld", *d->values[ n->type ].id );
if ( x < 0 || x >= sizeof( s ) )
{
d->rc = RC( rcExe, rcNamelist, rcExecuting, rcBuffer, rcInsufficient );
@@ -3731,7 +3731,7 @@ rc_t SRADumper_Init( SRADumperFmt* fmt )
{"I", "readids", NULL, {"Append read id after spot id as 'accession.spot.readid' on defline", NULL}}, /* H_readids = 12 */
{NULL, "helicos", NULL, {"Helicos style defline", NULL}}, /* H_helicos = 13 */
{NULL, "defline-seq", "fmt", {"Defline format specification for sequence.", NULL}}, /* H_defline_seq = 14 */
- {NULL, "defline-qual", "fmt", {"Defline format specification for quailty.", /* H_defline_qual = 15 */
+ {NULL, "defline-qual", "fmt", {"Defline format specification for quality.", /* H_defline_qual = 15 */
"<fmt> is a string of characters and/or variables. The variables can be one of:",
"$ac - accession, $si - spot id, $sn - spot name, $sg - spot group (barcode),",
"$sl - spot length in bases, $ri - read number, $rn - read name, $rl - read length in bases.",
diff --git a/tools/sra-dump/illumina-dump.vers b/tools/sra-dump/illumina-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-dump/illumina-dump.vers
+++ b/tools/sra-dump/illumina-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-dump/illumina-dump.vers.h b/tools/sra-dump/illumina-dump.vers.h
index d977721..2b6b70b 100644
--- a/tools/sra-dump/illumina-dump.vers.h
+++ b/tools/sra-dump/illumina-dump.vers.h
@@ -1 +1 @@
-#define ILLUMINA_DUMP_VERS 0x02040002
+#define ILLUMINA_DUMP_VERS 0x02050007
diff --git a/tools/sra-dump/sff-dump.vers b/tools/sra-dump/sff-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-dump/sff-dump.vers
+++ b/tools/sra-dump/sff-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-dump/sff-dump.vers.h b/tools/sra-dump/sff-dump.vers.h
index 1589515..ef4cf1f 100644
--- a/tools/sra-dump/sff-dump.vers.h
+++ b/tools/sra-dump/sff-dump.vers.h
@@ -1 +1 @@
-#define SFF_DUMP_VERS 0x02040002
+#define SFF_DUMP_VERS 0x02050007
diff --git a/tools/sra-dump/sra-dump.c b/tools/sra-dump/sra-dump.c
index c2a8f09..182f5de 100644
--- a/tools/sra-dump/sra-dump.c
+++ b/tools/sra-dump/sra-dump.c
@@ -498,25 +498,21 @@ void CC basetype_whack ( BSTNode *n, void *ignore )
}
static
-int CC basetype_cmp ( const void *item, const BSTNode *n )
+int64_t CC basetype_cmp ( const void *item, const BSTNode *n )
{
const VTypedecl *td = item;
const basetype *bt = ( const basetype* ) n;
- if ( td -> type_id < bt -> type_id )
- return -1;
- return td -> type_id - bt -> type_id;
+ return (int64_t) td -> type_id - (int64_t) bt -> type_id;
}
static
-int CC basetype_sort ( const BSTNode *item, const BSTNode *n )
+int64_t CC basetype_sort ( const BSTNode *item, const BSTNode *n )
{
const basetype *td = ( const basetype* ) item;
const basetype *bt = ( const basetype* ) n;
- if ( td -> type_id < bt -> type_id )
- return -1;
- return td -> type_id - bt -> type_id;
+ return (int64_t) td -> type_id - (int64_t) bt -> type_id;
}
static
@@ -950,7 +946,7 @@ rc_t CC KMain ( int argc, char *argv [] )
{
const char * pc;
- rc = ArgsOptionValue (args, OPTION_START, 0, &pc);
+ rc = ArgsOptionValue (args, OPTION_START, 0, (const void **)&pc);
if (rc)
break;
@@ -965,7 +961,7 @@ rc_t CC KMain ( int argc, char *argv [] )
{
const char * pc;
- rc = ArgsOptionValue (args, OPTION_STOP, 0, &pc);
+ rc = ArgsOptionValue (args, OPTION_STOP, 0, (const void **)&pc);
if (rc)
break;
@@ -983,7 +979,7 @@ rc_t CC KMain ( int argc, char *argv [] )
exit(EXIT_FAILURE);
}
- rc = ArgsParamValue (args, 0, &pb.src_path);
+ rc = ArgsParamValue (args, 0, (const void **)&pb.src_path);
if (rc)
break;
@@ -1004,7 +1000,7 @@ rc_t CC KMain ( int argc, char *argv [] )
ppc = pb.columns;
for (ix = 1; ix < pcount; ix++)
{
- rc = ArgsParamValue (args, ix, ppc);
+ rc = ArgsParamValue (args, ix, (const void **)ppc);
if (rc)
break;
ppc++;
diff --git a/tools/sra-dump/sra-dump.vers b/tools/sra-dump/sra-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-dump/sra-dump.vers
+++ b/tools/sra-dump/sra-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/Makefile b/tools/sra-load/Makefile
index fa486fd..0fca0cc 100644
--- a/tools/sra-load/Makefile
+++ b/tools/sra-load/Makefile
@@ -47,17 +47,24 @@ ALL_TOOLS = \
#-------------------------------------------------------------------------------
# outer targets
#
-ifeq (win,$(OS))
-all std:
- @ echo "not building loaders under Windows"
-else
+ifeq (1,$(HAVE_XML2))
+
all std: vers-includes
@ $(MAKE_CMD) $(TARGDIR)/std
-endif
$(ALL_TOOLS): vers-includes
@ $(MAKE_CMD) $(BINDIR)/$@
+else
+
+all std: $(ALL_TOOLS)
+
+$(ALL_TOOLS):
+ @ echo "NOTE - $(@F) cannot be built:" \
+ "It requires our internal library 'libkxml'" \
+ "which requires 'libxml2' and its development headers."
+endif
+
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
@@ -100,7 +107,7 @@ LOADER_OBJ = \
$(addsuffix .$(OBJX),$(LOADER_SRC))
LIBXML = -lkxfs -lkxml -lxml2
-
+
LOADER_LIB = \
-lkapp \
-lload \
diff --git a/tools/sra-load/abi-load.vers b/tools/sra-load/abi-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/abi-load.vers
+++ b/tools/sra-load/abi-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/common-xml.c b/tools/sra-load/common-xml.c
index 51366ad..5c9439d 100644
--- a/tools/sra-load/common-xml.c
+++ b/tools/sra-load/common-xml.c
@@ -443,7 +443,7 @@ rc_t parse_EXPECTED_BASECALL_TABLE(const KXMLNode* READ_SPEC, ReadSpecXML_read*
}
} else {
rc = RC(rcExe, rcFormatter, rcConstructing, rcData, rcIncomplete);
- LOGERR(klogErr, rc, "need at leaset one BASECALL element");
+ LOGERR(klogErr, rc, "need at least one BASECALL element");
}
KXMLNodesetRelease(BASECALLS);
} else {
@@ -665,7 +665,12 @@ rc_t PlatformXML_Make(const PlatformXML** cself, const KXMLNode* node, uint32_t*
platform->id = SRA_PLATFORM_PACBIO_SMRT;
} else if( strcmp(name, "COMPLETE_GENOMICS") == 0 ) {
platform->id = SRA_PLATFORM_COMPLETE_GENOMICS;
+ } else if( strcmp(name, "CAPILLARY") == 0 ) {
+ platform->id = SRA_PLATFORM_CAPILLARY;
+ } else if( strcmp(name, "OXFORD_NANOPORE") == 0 ) {
+ platform->id = SRA_PLATFORM_OXFORD_NANOPORE;
}
+
if( rc != 0 || platform->id == SRA_PLATFORM_UNDEFINED ) {
rc = rc ? rc : RC(rcExe, rcFormatter, rcConstructing, rcId, rcUnrecognized);
PLOGERR(klogErr, (klogErr, rc, "PLATFORM '$(n)'", PLOG_S(n), name));
@@ -711,6 +716,10 @@ void PlatformXML_Whack(const PlatformXML* cself)
break;
case SRA_PLATFORM_COMPLETE_GENOMICS:
break;
+ case SRA_PLATFORM_CAPILLARY:
+ break;
+ case SRA_PLATFORM_OXFORD_NANOPORE:
+ break;
case SRA_PLATFORM_UNDEFINED:
break;
}
diff --git a/tools/sra-load/experiment-xml.c b/tools/sra-load/experiment-xml.c
index 567ee1b..9c50fff 100644
--- a/tools/sra-load/experiment-xml.c
+++ b/tools/sra-load/experiment-xml.c
@@ -78,7 +78,7 @@ void CC PoolMember_Whack(BSTNode* node, void* data)
}
static
-int CC PoolMember_StrCmp(const char* s1, const char* s2)
+int64_t CC PoolMember_StrCmp(const char* s1, const char* s2)
{
if( s1 == NULL && s2 == NULL ) {
return 0;
@@ -91,7 +91,7 @@ int CC PoolMember_StrCmp(const char* s1, const char* s2)
}
static
-int CC PoolMember_Cmp(const BSTNode* item, const BSTNode* node)
+int64_t CC PoolMember_Cmp(const BSTNode* item, const BSTNode* node)
{
const PoolMember* i = (const PoolMember*)item;
const PoolMember* n = (const PoolMember*)node;
@@ -100,7 +100,7 @@ int CC PoolMember_Cmp(const BSTNode* item, const BSTNode* node)
}
static
-int CC PoolMember_FindByName(const void* item, const BSTNode* node)
+int64_t CC PoolMember_FindByName(const void* item, const BSTNode* node)
{
const char* name = (const char*)item;
const PoolMember* n = (const PoolMember*)node;
diff --git a/tools/sra-load/fastq-load.vers b/tools/sra-load/fastq-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/fastq-load.vers
+++ b/tools/sra-load/fastq-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/helicos-load.vers b/tools/sra-load/helicos-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/helicos-load.vers
+++ b/tools/sra-load/helicos-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/illumina-load.vers b/tools/sra-load/illumina-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/illumina-load.vers
+++ b/tools/sra-load/illumina-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/loader.c b/tools/sra-load/loader.c
index b838444..d721263 100644
--- a/tools/sra-load/loader.c
+++ b/tools/sra-load/loader.c
@@ -167,31 +167,31 @@ rc_t TArgsParse(TArgs *args, int argc, char *argv[])
/* no parameters accepted */
if( (rc = ArgsParamCount(args->args, &count)) != 0 || count != 0 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcUnexpected);
- ArgsParamValue(args->args, 0, &errmsg);
+ ArgsParamValue(args->args, 0, (const void **)&errmsg);
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_RunXML].name, &count)) != 0 || count != 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, count ? rcExcessive : rcInsufficient);
errmsg = TArgsDef[targs_RunXML].name;
- } else if( (rc = ArgsOptionValue(args->args, TArgsDef[targs_RunXML].name, 0, &args->_runXmlPath)) != 0 ) {
+ } else if( (rc = ArgsOptionValue(args->args, TArgsDef[targs_RunXML].name, 0, (const void **)&args->_runXmlPath)) != 0 ) {
errmsg = TArgsDef[targs_RunXML].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_ExperimentXML].name, &count)) != 0 || count != 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, count ? rcExcessive : rcInsufficient);
errmsg = TArgsDef[targs_ExperimentXML].name;
- } else if( (rc = ArgsOptionValue(args->args, TArgsDef[targs_ExperimentXML].name, 0, &args->_experimentXmlPath)) != 0 ) {
+ } else if( (rc = ArgsOptionValue(args->args, TArgsDef[targs_ExperimentXML].name, 0, (const void **)&args->_experimentXmlPath)) != 0 ) {
errmsg = TArgsDef[targs_ExperimentXML].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_Target].name, &count)) != 0 || count != 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, count ? rcExcessive : rcInsufficient);
errmsg = TArgsDef[targs_Target].name;
- } else if( (rc = ArgsOptionValue(args->args, TArgsDef[targs_Target].name, 0, &args->_target)) != 0 ) {
+ } else if( (rc = ArgsOptionValue(args->args, TArgsDef[targs_Target].name, 0, (const void **)&args->_target)) != 0 ) {
errmsg = TArgsDef[targs_Target].name;
/* optional */
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_InputPath].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = TArgsDef[targs_InputPath].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_InputPath].name, 0, &args->_input_path)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_InputPath].name, 0, (const void **)&args->_input_path)) != 0 ) {
errmsg = TArgsDef[targs_InputPath].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_InputUnpacked].name, &args->_input_unpacked)) != 0 ) {
@@ -206,31 +206,31 @@ rc_t TArgsParse(TArgs *args, int argc, char *argv[])
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_SpotsNumber].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = TArgsDef[targs_SpotsNumber].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_SpotsNumber].name, 0, &spot_qty)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_SpotsNumber].name, 0, (const void **)&spot_qty)) != 0 ) {
errmsg = TArgsDef[targs_SpotsNumber].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_BadSpotsNumber].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = TArgsDef[targs_BadSpotsNumber].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_BadSpotsNumber].name, 0, &bad_spots)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_BadSpotsNumber].name, 0, (const void **)&bad_spots)) != 0 ) {
errmsg = TArgsDef[targs_BadSpotsNumber].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_BadSpotPercentage].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = TArgsDef[targs_BadSpotPercentage].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_BadSpotPercentage].name, 0, &bad_percent)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_BadSpotPercentage].name, 0, (const void **)&bad_percent)) != 0 ) {
errmsg = TArgsDef[targs_BadSpotPercentage].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_ExpectedXML].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = TArgsDef[targs_ExpectedXML].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_ExpectedXML].name, 0, &args->_expectedXmlPath)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_ExpectedXML].name, 0, (const void **)&args->_expectedXmlPath)) != 0 ) {
errmsg = TArgsDef[targs_ExpectedXML].name;
} else if( (rc = ArgsOptionCount(args->args, TArgsDef[targs_Intensities].name, &count)) != 0 || count > 1 ) {
rc = rc ? rc : RC(rcExe, rcArgv, rcParsing, rcParam, rcExcessive);
errmsg = TArgsDef[targs_Intensities].name;
- } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_Intensities].name, 0, &intense)) != 0 ) {
+ } else if( count > 0 && (rc = ArgsOptionValue(args->args, TArgsDef[targs_Intensities].name, 0, (const void **)&intense)) != 0 ) {
errmsg = TArgsDef[targs_Intensities].name;
}
while( rc == 0 ) {
@@ -951,7 +951,7 @@ rc_t KMain(int argc, char *argv[])
uint32_t j;
for(j = 0; j < qty; j++) {
const char* v;
- if( ArgsOptionValue(args->args, "+", j, &v) == 0 ) {
+ if( ArgsOptionValue(args->args, "+", j, (const void **)&v) == 0 ) {
eargs[i++] = "-+";
eargs[i++] = v;
}
@@ -961,7 +961,7 @@ rc_t KMain(int argc, char *argv[])
uint32_t j;
for(j = 0; j < qty; j++) {
const char* v;
- if( ArgsOptionValue(args->args, "debug", j, &v) == 0 ) {
+ if( ArgsOptionValue(args->args, "debug", j, (const void **)&v) == 0 ) {
eargs[i++] = "-+";
eargs[i++] = v;
}
@@ -973,7 +973,7 @@ rc_t KMain(int argc, char *argv[])
if( XMLLogger_Args[j].aliases != NULL &&
ArgsOptionCount(args->args, XMLLogger_Args[j].aliases, &qty) == 0 &&
qty > 0 &&
- ArgsOptionValue(args->args, XMLLogger_Args[j].aliases, 0, &v) == 0 ) {
+ ArgsOptionValue(args->args, XMLLogger_Args[j].aliases, 0, (const void **)&v) == 0 ) {
eargs[i++] = &extra[x];
extra[x++] = '-';
extra[x++] = '-';
@@ -983,7 +983,7 @@ rc_t KMain(int argc, char *argv[])
} else if( XMLLogger_Args[j].name != NULL &&
ArgsOptionCount(args->args, XMLLogger_Args[j].name, &qty) == 0 &&
qty > 0 &&
- ArgsOptionValue(args->args, XMLLogger_Args[j].name, 0, &v) == 0 ) {
+ ArgsOptionValue(args->args, XMLLogger_Args[j].name, 0, (const void **)&v) == 0 ) {
eargs[i++] = &extra[x];
extra[x++] = '-';
extra[x++] = '-';
diff --git a/tools/sra-load/pacbio-loadxml.vers b/tools/sra-load/pacbio-loadxml.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/pacbio-loadxml.vers
+++ b/tools/sra-load/pacbio-loadxml.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/sff-load.vers b/tools/sra-load/sff-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/sff-load.vers
+++ b/tools/sra-load/sff-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/srf-absolid.c b/tools/sra-load/srf-absolid.c
index 6e6a67f..50981f7 100644
--- a/tools/sra-load/srf-absolid.c
+++ b/tools/sra-load/srf-absolid.c
@@ -58,10 +58,10 @@ typedef struct fe_context_t_struct {
struct {
uint8_t nreads;
- uint16_t start[ABSOLID_FMT_MAX_NUM_READS];
- pstring label[ABSOLID_FMT_MAX_NUM_READS];
- char cs_key[ABSOLID_FMT_MAX_NUM_READS];
- EAbisolidReadType type[ABSOLID_FMT_MAX_NUM_READS];
+ uint16_t start[ABSOLID_FMT_MAX_NUM_READS+1];
+ pstring label[ABSOLID_FMT_MAX_NUM_READS+1];
+ char cs_key[ABSOLID_FMT_MAX_NUM_READS+1];
+ EAbisolidReadType type[ABSOLID_FMT_MAX_NUM_READS+1];
} region;
} fe_context_t;
@@ -95,7 +95,7 @@ rc_t fe_new_region(fe_context_t *self, size_t region_count, const region_t regio
rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnsupported);
SRALoaderFile_LOG(self->ctx.file, klogErr, rc, "read count $(c)", PLOG_U8(c), self->region.nreads);
}
- for(i = 0; rc == 0 && i < self->region.nreads; i++ ) {
+ for(i = 0; rc == 0 && i < self->region.nreads ; i++ ) {
int j = i * 2 + 1;
self->region.start[i] = region[j].start;
if( (rc = set_label_type(region[j].name, &self->region.label[i], &self->region.type[i])) != 0 ) {
@@ -356,6 +356,11 @@ rc_t parse_v1_read(SRF_context *ctx, ZTR_Context *ztr_ctx, const uint8_t *data,
SRALoaderFile_LOG(ctx->file, klogErr, rc, "copying signal", NULL);
}
} else {
+ if( fe->region.nreads <= 0 || fe->region.nreads > ABSOLID_FMT_MAX_NUM_READS ) {
+ rc = RC(rcSRA, rcFormatter, rcParsing, rcData, rcUnsupported);
+ SRALoaderFile_LOG(fe->ctx.file, klogErr, rc, "read count $(c)", PLOG_U8(c), fe->region.nreads);
+ }
+
for(i = 0; rc == 0 && i < fe->region.nreads; i++) {
pstring* d = NULL;
int read_number = AbisolidReadType2ReadNumber[fe->region.type[i]];
diff --git a/tools/sra-load/srf-load.vers b/tools/sra-load/srf-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-load/srf-load.vers
+++ b/tools/sra-load/srf-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-load/writer-illumina.c b/tools/sra-load/writer-illumina.c
index b7162ac..8362abe 100644
--- a/tools/sra-load/writer-illumina.c
+++ b/tools/sra-load/writer-illumina.c
@@ -353,7 +353,7 @@ typedef struct IlluminaWriterSpot_struct {
pstring noise;
pstring intensity;
pstring signal;
- SRAReadFilter filter[ILLUMINAWRITER_MAX_NUM_READS];
+ SRAReadFilter filter[ILLUMINAWRITER_MAX_NUM_READS+1];
} IlluminaWriterSpot;
static
diff --git a/tools/sra-load/ztr-huffman.c b/tools/sra-load/ztr-huffman.c
index b525807..2587cc6 100644
--- a/tools/sra-load/ztr-huffman.c
+++ b/tools/sra-load/ztr-huffman.c
@@ -74,19 +74,14 @@ struct index_t {
uint16_t i, len;
};
-static int idx_cmp(const void *A, const void *B, void * ignored) {
+static int64_t idx_cmp(const void *A, const void *B, void * ignored) {
const struct index_t *a = A;
const struct index_t *b = B;
- if (a->len < b->len)
- return -1;
- if (a->len > b->len)
- return 1;
- if (a->i < b->i)
- return -1;
- if (a->i > b->i)
- return 1;
- return 0;
+ if ( a->len != b->len )
+ return (int64_t)a->len - (int64_t)b->len;
+
+ return (int64_t)a->i - (int64_t)b->i;
}
#if 0
diff --git a/tools/sra-pileup/Makefile b/tools/sra-pileup/Makefile
index 194656d..c58bb57 100644
--- a/tools/sra-pileup/Makefile
+++ b/tools/sra-pileup/Makefile
@@ -35,7 +35,6 @@ INT_TOOLS = \
EXT_TOOLS = \
sra-pileup \
sam-dump
-# sam-dump2
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -88,6 +87,7 @@ clean: stdclean
TOOL_SRC = \
dyn_string \
cmdline_cmn \
+ out_redir \
perf_log \
reref \
cg_tools \
@@ -99,6 +99,7 @@ TOOL_SRC = \
walk_debug \
pileup_counters \
pileup_index \
+ pileup_indels \
pileup_varcount \
pileup_stat \
pileup_v2 \
@@ -115,28 +116,6 @@ TOOL_LIB = \
$(BINDIR)/sra-pileup: $(TOOL_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(TOOL_LIB)
-#-------------------------------------------------------------------------------
-# sam-dump2
-#
-SAMDUMP2_SRC = \
- cmdline_cmn \
- writer \
- sam-dump
-
-SAMDUMP2_OBJ = \
- $(addsuffix .$(OBJX),$(SAMDUMP2_SRC))
-
-SAMDUMP2_LIB = \
- -lkapp \
- -sncbi-vdb \
- -lm
-
-ifdef NCBI
-CFLAGS += -DNCBI
-endif
-
-$(BINDIR)/sam-dump2: $(SAMDUMP2_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SAMDUMP2_LIB)
#-------------------------------------------------------------------------------
# sam-dump
@@ -148,10 +127,12 @@ SAMDUMP3_SRC = \
sam-dump-opts \
out_redir \
sam-hdr \
+ sam-hdr1 \
matecache \
read_fkt \
sam-aligned \
sam-unaligned \
+ md_flag \
cg_tools \
sam-dump \
sam-dump3
diff --git a/tools/sra-pileup/cg_tools.c b/tools/sra-pileup/cg_tools.c
index 667af70..df123d7 100644
--- a/tools/sra-pileup/cg_tools.c
+++ b/tools/sra-pileup/cg_tools.c
@@ -1638,7 +1638,7 @@ rc_t discover_rna_splicing_candidates( uint32_t cigar_len, const char * cigar, u
{
char op_code = cigops[ op_idx ].op;
uint32_t op_len = cigops[ op_idx ].oplen;
- if ( op_code == 'D' && op_len >= min_len && candidates->count < MAX_RNA_SPLICE_CANDIDATES )
+ if ( (op_code == 'D' || op_code == 'N') && op_len >= min_len && candidates->count < MAX_RNA_SPLICE_CANDIDATES )
{
rna_splice_candidate * rsc = &candidates->candidates[ candidates->count++ ];
rsc->ref_offset = ref_offset;
diff --git a/tools/sra-pileup/cmdline_cmn.c b/tools/sra-pileup/cmdline_cmn.c
index 30ce3da..c9cc9e5 100644
--- a/tools/sra-pileup/cmdline_cmn.c
+++ b/tools/sra-pileup/cmdline_cmn.c
@@ -124,7 +124,7 @@ static rc_t get_str_option( const Args *args, const char *name, const char ** re
{
if ( count > 0 )
{
- rc = ArgsOptionValue( args, name, 0, res );
+ rc = ArgsOptionValue( args, name, 0, (const void **)res );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsOptionValue() failed" );
@@ -259,7 +259,7 @@ rc_t init_ref_regions( BSTree * tree, Args * args )
for ( i = 0; i < count && rc == 0; ++i )
{
const char * s;
- rc = ArgsOptionValue( args, OPTION_REF, i, &s );
+ rc = ArgsOptionValue( args, OPTION_REF, i, (const void **)&s );
if ( rc != 0 )
LOGERR( klogInt, rc, "ArgsOptionValue() failed" );
else
@@ -499,7 +499,7 @@ rc_t foreach_argument( Args * args, KDirectory *dir, bool div_by_spotgrp, bool *
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *param = NULL;
- rc = ArgsParamValue( args, idx, ¶m );
+ rc = ArgsParamValue( args, idx, (const void **)¶m );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsParamvalue() failed" );
diff --git a/tools/sra-pileup/dyn_string.c b/tools/sra-pileup/dyn_string.c
index 6cb4bd7..c8f5348 100644
--- a/tools/sra-pileup/dyn_string.c
+++ b/tools/sra-pileup/dyn_string.c
@@ -136,6 +136,26 @@ rc_t add_string_2_dyn_string( struct dyn_string *self, const char * s )
}
+rc_t add_dyn_string_2_dyn_string( struct dyn_string *self, struct dyn_string *other )
+{
+ rc_t rc = 0;
+ size_t size = other->data_len;
+ if ( size > 0 )
+ {
+ /* does nothing if self->data_len + size + 1 < self->allocated */
+ rc = expand_dyn_string( self, self->data_len + size + 1 );
+ if ( rc == 0 )
+ {
+ string_copy ( &(self->data[ self->data_len ]), self->allocated, other->data, size );
+ self->data_len += size;
+ self->data[ self->data_len ] = 0;
+ }
+ }
+ return rc;
+
+}
+
+
rc_t print_2_dyn_string( struct dyn_string * self, const char *fmt, ... )
{
rc_t rc = 0;
diff --git a/tools/sra-pileup/dyn_string.h b/tools/sra-pileup/dyn_string.h
index b51cd99..f3806d6 100644
--- a/tools/sra-pileup/dyn_string.h
+++ b/tools/sra-pileup/dyn_string.h
@@ -43,6 +43,7 @@ rc_t expand_dyn_string( struct dyn_string *self, size_t new_size );
rc_t add_char_2_dyn_string( struct dyn_string *self, const char c );
char * dyn_string_char( struct dyn_string *self, uint32_t idx );
rc_t add_string_2_dyn_string( struct dyn_string *self, const char * s );
+rc_t add_dyn_string_2_dyn_string( struct dyn_string *self, struct dyn_string *other );
rc_t print_2_dyn_string( struct dyn_string * self, const char *fmt, ... );
rc_t print_dyn_string( struct dyn_string * self );
size_t dyn_string_len( struct dyn_string * self );
diff --git a/tools/sra-pileup/inputfiles.c b/tools/sra-pileup/inputfiles.c
index 12c9584..a232677 100644
--- a/tools/sra-pileup/inputfiles.c
+++ b/tools/sra-pileup/inputfiles.c
@@ -189,6 +189,7 @@ static rc_t contains_ref_and_alignments( const VDatabase * db, const char * path
}
*res = ( has_ref && has_alignment );
}
+ KNamelistRelease( tables );
}
return rc;
}
diff --git a/tools/sra-pileup/md_flag.c b/tools/sra-pileup/md_flag.c
new file mode 100644
index 0000000..491d475
--- /dev/null
+++ b/tools/sra-pileup/md_flag.c
@@ -0,0 +1,268 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <klib/printf.h>
+#include <klib/out.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+#include <ctype.h>
+#include <string.h>
+
+#include "md_flag.h"
+
+struct cigar_t
+{
+ char * op;
+ int * count;
+ size_t size, length;
+};
+
+
+static void init_cigar_t( struct cigar_t * c, size_t size )
+{
+ if ( c != NULL )
+ {
+ c->size = 0;
+ c->length = 0;
+ c->op = malloc( sizeof( c->op[ 0 ] ) * size );
+ if ( c->op != NULL )
+ {
+ c->count = malloc( sizeof( c->count[ 0 ] ) * size );
+ if ( c->count != NULL )
+ c->size = size;
+ else
+ free( ( void * ) c->op );
+ }
+ }
+}
+
+static void resize_cigar_t( struct cigar_t * c, size_t new_size )
+{
+ if ( c != NULL )
+ {
+ if ( c->size == 0 )
+ init_cigar_t( c, new_size );
+ else if ( c->size < new_size )
+ {
+ char * temp_op = c->op;
+ c->op = realloc( c->op, sizeof( c->op[ 0 ] ) * new_size );
+ if ( c->op != NULL )
+ {
+ int * temp_count = c->count;
+ c->count = realloc( c->count, sizeof( c->count[ 0 ] ) * new_size );
+ if ( c->count != NULL )
+ c->size = new_size;
+ else
+ c->count = temp_count;
+ }
+ else
+ c->op = temp_op;
+ }
+ }
+}
+
+static void append_to_cigar_t( struct cigar_t * c, char op, int count )
+{
+ if ( c->length < c->size )
+ {
+ c->op[ c->length ] = op;
+ c->count[ c->length ++ ] = count;
+ }
+}
+
+void parse_cigar_t( struct cigar_t * c, const char * cigar_str )
+{
+ if ( c != NULL && cigar_str != NULL && cigar_str[ 0 ] != 0 )
+ {
+ resize_cigar_t( c, strlen( cigar_str ) );
+ if ( c->size > 0 )
+ {
+ int count = 0;
+ while ( *cigar_str != 0 && c->length < c->size )
+ {
+ if ( isdigit( *cigar_str ) )
+ {
+ count = ( count * 10 ) + ( *cigar_str - '0' );
+ }
+ else
+ {
+ if ( count == 0 ) count = 1;
+ append_to_cigar_t( c, *cigar_str, count );
+ count = 0;
+ }
+ cigar_str++;
+ }
+ }
+ }
+}
+
+struct cigar_t * make_cigar_t( const char * cigar_str, const size_t cigar_len )
+{
+ struct cigar_t * res = malloc( sizeof * res );
+ if ( res != NULL )
+ {
+ size_t size;
+ if ( cigar_str != NULL && cigar_str[ 0 ] != 0 && cigar_len > 0 )
+ size = cigar_len;
+ else
+ size = 1024;
+ init_cigar_t( res, size );
+ if ( res->size == size )
+ parse_cigar_t( res, cigar_str );
+ }
+ return res;
+}
+
+
+void free_cigar_t( struct cigar_t * c )
+{
+ if ( c != NULL )
+ {
+ if ( c->op != NULL )
+ {
+ free( ( void * ) c->op );
+ c->op = NULL;
+ }
+ if ( c->count != NULL )
+ {
+ free( ( void * ) c->count );
+ c->count = NULL;
+ }
+ free( ( void * ) c );
+ }
+}
+
+
+static rc_t kout_delete( int count, int *match_count,
+ const uint8_t * ref, const INSDC_coord_len ref_len, int *ref_idx )
+{
+ rc_t rc = 0;
+
+ if ( *match_count > 0 )
+ {
+ rc = KOutMsg( "%d", *match_count );
+ *match_count = 0;
+ }
+
+ if ( rc == 0 )
+ {
+ if ( ( *ref_idx + count ) < ref_len )
+ {
+ rc = KOutMsg( "^%.*s", count, &(ref[ *ref_idx ] ) );
+ (*ref_idx) += count;
+ }
+ else
+ rc = RC( rcExe, rcNoTarg, rcAllocating, rcItem, rcIncomplete );
+ }
+ return rc;
+}
+
+
+static rc_t kout_match( int count, int *match_count,
+ const char * read, size_t read_len, int *read_idx,
+ const uint8_t *ref, const INSDC_coord_len ref_len, int *ref_idx )
+{
+ rc_t rc = 0;
+ int i;
+ for ( i = 0; i < count && rc == 0; ++i )
+ {
+ if ( *read_idx < read_len && *ref_idx < ref_len )
+ {
+ if ( read[ (*read_idx)++ ] == ref[ *ref_idx ] )
+ {
+ (*match_count)++;
+ }
+ else
+ {
+ rc = KOutMsg( "%d%c", *match_count, ref[ *ref_idx ] );
+ *match_count = 0;
+ }
+ (*ref_idx)++;
+ }
+ else
+ rc = RC( rcExe, rcNoTarg, rcAllocating, rcItem, rcIncomplete );
+ }
+ return rc;
+}
+
+
+static rc_t kout_tag( const struct cigar_t * c,
+ const char * read,
+ const size_t read_len,
+ const uint8_t * ref,
+ const INSDC_coord_len ref_len )
+{
+ rc_t rc = 0;
+ if ( c != NULL && read != NULL && read_len > 0 && ref != NULL && ref_len > 0 )
+ {
+ rc = KOutMsg( "\tMD:Z:" );
+ if ( rc == 0 )
+ {
+ int read_idx = 0;
+ int ref_idx = 0;
+ int match_count = 0;
+ int cigar_idx;
+ for ( cigar_idx = 0; cigar_idx < c->length && rc == 0; ++cigar_idx )
+ {
+ int count = c->count[ cigar_idx ];
+ switch ( c->op[ cigar_idx ] )
+ {
+ case 'D' : rc = kout_delete( count, &match_count, ref, ref_len, &ref_idx ); break;
+
+ case 'I' : read_idx += count; break;
+
+ case 'M' : rc = kout_match( count, &match_count, read, read_len, &read_idx, ref, ref_len, &ref_idx ); break;
+ }
+ }
+ if ( rc == 0 && match_count > 0 )
+ rc = KOutMsg( "%d", match_count );
+ }
+ }
+ else
+ rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcIncomplete );
+ return rc;
+}
+
+
+rc_t kout_md_tag_from_cigar_string( const char * cigar_str,
+ const size_t cigar_len,
+ const char * read,
+ const size_t read_len,
+ const uint8_t * ref,
+ const INSDC_coord_len ref_len )
+{
+ rc_t rc = 0;
+ struct cigar_t * cigar = make_cigar_t( cigar_str, cigar_len );
+ if ( cigar == NULL )
+ rc = RC( rcExe, rcNoTarg, rcAllocating, rcItem, rcIncomplete );
+ else
+ {
+ rc = kout_tag( cigar, read, read_len, ref, ref_len );
+ free_cigar_t( cigar );
+ }
+ return rc;
+}
diff --git a/tools/cg-dump/last_rowid.h b/tools/sra-pileup/md_flag.h
similarity index 82%
copy from tools/cg-dump/last_rowid.h
copy to tools/sra-pileup/md_flag.h
index 5ae92f0..f430e94 100644
--- a/tools/cg-dump/last_rowid.h
+++ b/tools/sra-pileup/md_flag.h
@@ -24,18 +24,22 @@
*
*/
-#ifndef _h_last_rowid_
-#define _h_last_rowid_
+#ifndef _h_md_flag_
+#define _h_md_flag_
#ifdef __cplusplus
extern "C" {
#endif
#include <klib/rc.h>
-#include <klib/text.h>
-#include <klib/container.h>
+#include <insdc/insdc.h>
-rc_t discover_last_rowid( const char * src, int64_t * last_row_id );
+rc_t kout_md_tag_from_cigar_string( const char * cigar_str,
+ const size_t cigar_len,
+ const char * read,
+ const size_t read_len,
+ const uint8_t * ref,
+ const INSDC_coord_len ref_len );
#ifdef __cplusplus
}
diff --git a/tools/sra-pileup/pileup_counters.c b/tools/sra-pileup/pileup_counters.c
index f63cbab..b490139 100644
--- a/tools/sra-pileup/pileup_counters.c
+++ b/tools/sra-pileup/pileup_counters.c
@@ -92,7 +92,7 @@ typedef struct find_fragment_ctx
} find_fragment_ctx;
-static int CC cmp_fragment_vs_find_ctx( const void *item, const BSTNode *n )
+static int64_t CC cmp_fragment_vs_find_ctx( const void *item, const BSTNode *n )
{
const indel_fragment * fragment = ( const indel_fragment * )n;
const find_fragment_ctx * fctx = ( const find_fragment_ctx * )item;
@@ -100,7 +100,7 @@ static int CC cmp_fragment_vs_find_ctx( const void *item, const BSTNode *n )
}
-static int CC cmp_fragment_vs_fragment( const BSTNode *item, const BSTNode *n )
+static int64_t CC cmp_fragment_vs_fragment( const BSTNode *item, const BSTNode *n )
{
const indel_fragment * f1 = ( const indel_fragment * )item;
const indel_fragment * f2 = ( const indel_fragment * )n;
diff --git a/tools/sra-pileup/pileup_varcount.c b/tools/sra-pileup/pileup_indels.c
similarity index 55%
copy from tools/sra-pileup/pileup_varcount.c
copy to tools/sra-pileup/pileup_indels.c
index a905e01..ea10cea 100644
--- a/tools/sra-pileup/pileup_varcount.c
+++ b/tools/sra-pileup/pileup_indels.c
@@ -29,32 +29,31 @@
#include "ref_walker_0.h"
#include "4na_ascii.h"
-typedef struct var_counters
+typedef struct indel_counters
{
uint32_t coverage;
- uint32_t base_counts[ 4 ]; /* 0...A, 1...C, 2...G, 3...T */
uint32_t deletes;
uint32_t inserts;
- uint32_t insert_after[ 4 ]; /* 0...A, 1...C, 2...G, 3...T */
-} var_counters;
+} indel_counters;
-static rc_t CC walk_varcount_enter_ref_pos( walk_data * data )
+static rc_t CC walk_indels_enter_ref_pos( walk_data * data )
{
- var_counters * vc = data->data;
+ indel_counters * vc = data->data;
memset( vc, 0, sizeof *vc );
return 0;
}
-static rc_t CC walk_varcount_exit_ref_pos( walk_data * data )
+static rc_t CC walk_indels_exit_ref_pos( walk_data * data )
{
- if ( data->depth == 0 )
- return 0;
- else
+ rc_t rc = 0;
+ if ( data->depth > 0 )
{
- var_counters * vc = data->data;
- char ref_base = _4na_to_ascii( data->ref_base, false );
+ indel_counters * vc = data->data;
+ if ( ( vc -> deletes + vc -> inserts ) > 0 )
+ {
+ char ref_base = _4na_to_ascii( data->ref_base, false );
/*
A ... ref-name
@@ -62,60 +61,42 @@ static rc_t CC walk_varcount_exit_ref_pos( walk_data * data )
C ... ref-base
D ... coverage
- E ... A ( match or mismatch )
- F ... C ( match or mismatch )
- G ... G ( match or mismatch )
- H ... T ( match or mismatch )
-
- I ... total deletes
- J ... total insertes
-
- K ... inserts after A
- L ... inserts after C
- M ... inserts after G
- N ... inserts after T
-
- A B C D E F G H I J K L M N
-*/
- return KOutMsg( "%s\t%u\t%c\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\n",
+ E ... total deletes
+ F ... total insertes
+ A B C D E F
+*/
+ rc = KOutMsg( "%s\t%u\t%c\t%u\t%u\t%u\n",
data->ref_name, data->ref_pos + 1, ref_base, data->depth,
-
- vc->base_counts[ 0 ], vc->base_counts[ 1 ], vc->base_counts[ 2 ], vc->base_counts[ 3 ],
- vc->deletes, vc->inserts,
- vc->insert_after[ 0 ], vc->insert_after[ 1 ], vc->insert_after[ 2 ], vc->insert_after[ 3 ] );
+ vc->deletes, vc->inserts );
+ }
}
+ return rc;
}
-static rc_t CC walk_varcount_placement( walk_data * data )
+static rc_t CC walk_indels_placement( walk_data * data )
{
int32_t state = data->state;
if ( ( state & align_iter_invalid ) != align_iter_invalid )
{
- var_counters * vc = data->data;
- uint32_t idx = _4na_to_index( state );
+ indel_counters * vc = data->data;
if ( ( state & align_iter_skip ) == align_iter_skip )
( vc->deletes ) ++;
- else if ( ( state & align_iter_match ) != align_iter_match )
- vc->base_counts[ idx ] ++;
if ( ( state & align_iter_insert ) == align_iter_insert )
- {
( vc->inserts )++;
- vc->insert_after[ idx ] ++;
- }
}
return 0;
}
-rc_t walk_varcount( ReferenceIterator *ref_iter, pileup_options * options )
+rc_t walk_indels( ReferenceIterator *ref_iter, pileup_options * options )
{
walk_data data;
walk_funcs funcs;
- var_counters v_counters;
+ indel_counters v_counters;
data.ref_iter = ref_iter;
data.options = options;
@@ -127,13 +108,13 @@ rc_t walk_varcount( ReferenceIterator *ref_iter, pileup_options * options )
funcs.on_enter_ref_window = NULL;
funcs.on_exit_ref_window = NULL;
- funcs.on_enter_ref_pos = walk_varcount_enter_ref_pos;
- funcs.on_exit_ref_pos = walk_varcount_exit_ref_pos;
+ funcs.on_enter_ref_pos = walk_indels_enter_ref_pos;
+ funcs.on_exit_ref_pos = walk_indels_exit_ref_pos;
funcs.on_enter_spotgroup = NULL;
funcs.on_exit_spotgroup = NULL;
- funcs.on_placement = walk_varcount_placement;
+ funcs.on_placement = walk_indels_placement;
return walk_0( &data, &funcs );
}
diff --git a/tools/vtblcp/vtblcp-priv.h b/tools/sra-pileup/pileup_indels.h
similarity index 89%
rename from tools/vtblcp/vtblcp-priv.h
rename to tools/sra-pileup/pileup_indels.h
index b8e413b..2eefc1c 100644
--- a/tools/vtblcp/vtblcp-priv.h
+++ b/tools/sra-pileup/pileup_indels.h
@@ -24,17 +24,17 @@
*
*/
-#ifndef _h_vtblcp_priv_
-#define _h_vtblcp_priv_
+#ifndef _h_pileup_indels_
+#define _h_pileup_indels_
#ifdef __cplusplus
extern "C" {
#endif
-
+rc_t walk_indels( ReferenceIterator *ref_iter, pileup_options * options );
#ifdef __cplusplus
}
#endif
-#endif /* _h_vtblcp_priv_ */
+#endif /* _h_pileup_indels_ */
diff --git a/tools/sra-pileup/pileup_options.h b/tools/sra-pileup/pileup_options.h
index 6d66595..9b02adc 100644
--- a/tools/sra-pileup/pileup_options.h
+++ b/tools/sra-pileup/pileup_options.h
@@ -43,6 +43,7 @@ typedef struct pileup_options
bool no_skip;
bool show_id;
bool div_by_spotgrp;
+ bool depth_per_spotgrp;
bool use_seq_name;
uint32_t minmapq;
uint32_t min_mismatch;
diff --git a/tools/sra-pileup/pileup_v2.c b/tools/sra-pileup/pileup_v2.c
index 937052d..015bf6a 100644
--- a/tools/sra-pileup/pileup_v2.c
+++ b/tools/sra-pileup/pileup_v2.c
@@ -102,7 +102,7 @@ static rc_t pileup_test( Args * args, pileup_options *options )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char * src = NULL;
- rc = ArgsParamValue( args, idx, &src );
+ rc = ArgsParamValue( args, idx, (const void **)&src );
if ( rc == 0 && src != NULL )
rc = ref_walker_add_source( walker, src );
}
@@ -113,7 +113,7 @@ static rc_t pileup_test( Args * args, pileup_options *options )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char * s = NULL;
- rc = ArgsOptionValue( args, OPTION_REF, idx, &s );
+ rc = ArgsOptionValue( args, OPTION_REF, idx, (const void **)&s );
if ( rc == 0 && s != NULL )
rc = ref_walker_parse_and_add_range( walker, s );
}
@@ -297,7 +297,7 @@ rc_t pileup_v2( Args * args, pileup_options *options )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char * src = NULL;
- rc = ArgsParamValue( args, idx, &src );
+ rc = ArgsParamValue( args, idx, (const void **)&src );
if ( rc == 0 && src != NULL )
rc = ref_walker_add_source( walker, src );
}
@@ -309,7 +309,7 @@ rc_t pileup_v2( Args * args, pileup_options *options )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char * s = NULL;
- rc = ArgsOptionValue( args, OPTION_REF, idx, &s );
+ rc = ArgsOptionValue( args, OPTION_REF, idx, (const void **)&s );
if ( rc == 0 && s != NULL )
rc = ref_walker_parse_and_add_range( walker, s );
}
diff --git a/tools/sra-pileup/pileup_varcount.c b/tools/sra-pileup/pileup_varcount.c
index a905e01..d5834cf 100644
--- a/tools/sra-pileup/pileup_varcount.c
+++ b/tools/sra-pileup/pileup_varcount.c
@@ -62,13 +62,13 @@ static rc_t CC walk_varcount_exit_ref_pos( walk_data * data )
C ... ref-base
D ... coverage
- E ... A ( match or mismatch )
- F ... C ( match or mismatch )
- G ... G ( match or mismatch )
- H ... T ( match or mismatch )
+ E ... A ( if mismatch )
+ F ... C ( if mismatch )
+ G ... G ( if mismatch )
+ H ... T ( if mismatch )
I ... total deletes
- J ... total insertes
+ J ... total inserts
K ... inserts after A
L ... inserts after C
diff --git a/tools/sra-pileup/ref_regions.c b/tools/sra-pileup/ref_regions.c
index 5a39432..c628927 100644
--- a/tools/sra-pileup/ref_regions.c
+++ b/tools/sra-pileup/ref_regions.c
@@ -103,29 +103,38 @@ static void free_range( struct reference_range * self )
free( self );
}
-static int cmp_range( const struct reference_range * a, const struct reference_range * b )
-{
-
- int64_t res = ( a->start - b->start );
- if ( res == 0 )
- res = ( a->end - b->end );
- if ( res < 0 )
- return -1;
- else if ( res > 0 )
- return 1;
- else return 0;
+static int64_t cmp_range( const struct reference_range * a, const struct reference_range * b )
+{
+ if ( a != NULL && b != NULL )
+ {
+ if ( a->start < b->start )
+ return -1;
+ else if ( a->start > b->start )
+ return 1;
+ else if ( a->end < b->end )
+ return -1;
+ else if ( a->end > b->end )
+ return 1;
+ else
+ return 0;
+ }
+ return 1; /* question from ukrainch: so, cmp_range cannot be used for sorting? */
}
static bool range_overlapp( const struct reference_range * a, const struct reference_range * b )
{
- return ( !( ( b->end < a->start ) || ( b->start > a->end ) ) );
+ if ( a != NULL && b != NULL )
+ return ( !( ( b->end < a->start ) || ( b->start > a->end ) ) );
+ return false;
}
static uint64_t range_distance( const struct reference_range * a, const struct reference_range * b )
{
- return ( b->start - a->end );
+ if ( a != NULL && b != NULL )
+ return ( b->start - a->end );
+ return 0;
}
/* =========================================================================================== */
@@ -150,7 +159,7 @@ static struct reference_region * make_reference_region( const char *name )
}
-static int CC cmp_range_wrapper( const void *item, const void *n )
+static int64_t CC cmp_range_wrapper( const void *item, const void *n )
{ return cmp_range( item, n ); }
@@ -213,7 +222,7 @@ static void parse_definition( const char *s, char * name, size_t len,
if ( n > 0 )
{
size_t i, st, dst = 0;
- char tmp[ 32 ];
+ char tmp[ 4096 ];
st = RR_NAME;
for ( i = 0; i < n; ++i )
{
@@ -346,7 +355,7 @@ static void merge_close_ranges_and_create_filter( struct reference_region * self
/* =========================================================================================== */
-static int CC reference_vs_pchar_wrapper( const void *item, const BSTNode *n )
+static int64_t CC reference_vs_pchar_wrapper( const void *item, const BSTNode *n )
{
const struct reference_region * r = ( const struct reference_region * )n;
return cmp_pchar( (const char *)item, r->name );
@@ -357,7 +366,7 @@ static struct reference_region * find_reference_region( BSTree * regions, const
return ( struct reference_region * ) BSTreeFind ( regions, name, reference_vs_pchar_wrapper );
}
-static int CC ref_vs_ref_wrapper( const BSTNode *item, const BSTNode *n )
+static int64_t CC ref_vs_ref_wrapper( const BSTNode *item, const BSTNode *n )
{
const struct reference_region * a = ( const struct reference_region * )item;
const struct reference_region * b = ( const struct reference_region * )n;
@@ -393,7 +402,7 @@ rc_t add_region( BSTree * regions, const char * name, const uint64_t start, cons
rc_t parse_and_add_region( BSTree * regions, const char * s )
{
uint64_t start, end;
- char name[ 64 ];
+ char name[ 4096 ];
parse_definition( s, name, sizeof name, &start, &end );
if ( name[ 0 ] == 0 )
return RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
@@ -617,12 +626,18 @@ struct skiplist
static bool reference_region_has_skip_ranges( const struct reference_region * r )
{
bool res = false;
- uint32_t i, n = VectorLength( &r->ranges );
- for ( i = 0; i < n && !res; ++i )
- {
- const struct reference_range * rr = VectorGet ( &( r->ranges ), i );
- if ( VectorLength( &rr->skip ) > 0 ) res = true;
- }
+ if ( r != NULL )
+ {
+ uint32_t i, n = VectorLength( &r->ranges );
+ for ( i = 0; i < n && !res; ++i )
+ {
+ const struct reference_range * rr = VectorGet ( &( r->ranges ), i );
+ if ( rr != NULL )
+ {
+ if ( VectorLength( &rr->skip ) > 0 ) res = true;
+ }
+ }
+ }
return res;
}
@@ -661,7 +676,7 @@ static struct skiplist_ref_node * make_skiplist_ref_node( const struct reference
/* helper call back for BSTreeInsert into skiplist->nodes */
-static int CC srn_vs_srn_wrapper( const BSTNode *item, const BSTNode *n )
+static int64_t CC srn_vs_srn_wrapper( const BSTNode *item, const BSTNode *n )
{
const struct skiplist_ref_node * a = ( const struct skiplist_ref_node * )item;
const struct skiplist_ref_node * b = ( const struct skiplist_ref_node * )n;
@@ -729,7 +744,7 @@ void skiplist_release( struct skiplist * list )
}
-static int CC pchar_vs_srn_cmp( const void * item, const BSTNode * n )
+static int64_t CC pchar_vs_srn_cmp( const void * item, const BSTNode * n )
{
const char * name = item;
const struct skiplist_ref_node * b = ( const struct skiplist_ref_node * )n;
@@ -748,8 +763,11 @@ void skiplist_enter_ref( struct skiplist * list, const char * name )
{
struct skiplist_ref_node * cur_node = ( struct skiplist_ref_node * )BSTreeFind ( &( list->nodes ), name, pchar_vs_srn_cmp );
list->current = cur_node;
- cur_node->current_id = 0;
- cur_node->current_skip_range = VectorGet ( &( cur_node->skip_ranges ), 0 );
+ if ( cur_node != NULL )
+ {
+ cur_node->current_id = 0;
+ cur_node->current_skip_range = VectorGet ( &( cur_node->skip_ranges ), 0 );
+ }
}
}
}
diff --git a/tools/sra-pileup/ref_walker_0.c b/tools/sra-pileup/ref_walker_0.c
index d5b3886..a620ec0 100644
--- a/tools/sra-pileup/ref_walker_0.c
+++ b/tools/sra-pileup/ref_walker_0.c
@@ -207,12 +207,20 @@ rc_t walk_0( walk_data * data, walk_funcs * funcs )
}
else
{
- if ( funcs->on_enter_ref != NULL )
- rc = funcs->on_enter_ref( data );
- if ( rc == 0 )
- rc = walk_ref_window( data, funcs );
- if ( rc == 0 && funcs->on_exit_ref != NULL )
- rc = funcs->on_exit_ref( data );
+ rc = ReferenceObj_SeqLength( data->ref_obj, &data->ref_len );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "ReferenceObj_SeqLength() failed" );
+ }
+ else
+ {
+ if ( funcs->on_enter_ref != NULL )
+ rc = funcs->on_enter_ref( data );
+ if ( rc == 0 )
+ rc = walk_ref_window( data, funcs );
+ if ( rc == 0 && funcs->on_exit_ref != NULL )
+ rc = funcs->on_exit_ref( data );
+ }
}
}
}
diff --git a/tools/sra-pileup/report_deletes.c b/tools/sra-pileup/report_deletes.c
index 3aef209..21e376e 100644
--- a/tools/sra-pileup/report_deletes.c
+++ b/tools/sra-pileup/report_deletes.c
@@ -238,7 +238,7 @@ rc_t report_deletes( Args * args, uint32_t min_len )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *param = NULL;
- rc = ArgsParamValue( args, idx, ¶m );
+ rc = ArgsParamValue( args, idx, (const void **)¶m );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsParamvalue() failed" );
diff --git a/tools/sra-pileup/reref.c b/tools/sra-pileup/reref.c
index 5dd034b..d05dae9 100644
--- a/tools/sra-pileup/reref.c
+++ b/tools/sra-pileup/reref.c
@@ -251,7 +251,7 @@ static const char * path_type_2_str( const uint32_t pt )
return res;
}
-
+/*
static rc_t resolve_accession( VFSManager * vfs_mgr, const char * accession, const String ** path )
{
VResolver * resolver;
@@ -289,6 +289,51 @@ static rc_t resolve_accession( VFSManager * vfs_mgr, const char * accession, con
}
return rc;
}
+*/
+
+static rc_t resolve_accession( VFSManager * vfs_mgr, const char * acc, const String ** resolved )
+{
+ VResolver * resolver;
+ rc_t rc = VFSManagerGetResolver( vfs_mgr, &resolver );
+ if ( rc != 0 )
+ {
+ (void)LOGERR( klogErr, rc, "VFSManagerGetResolver() failed" );
+ }
+ else
+ {
+ VPath * acc_vpath;
+ rc = VFSManagerMakePath( vfs_mgr, &acc_vpath, "ncbi-acc:%s", acc );
+ if ( rc != 0 )
+ {
+ (void)LOGERR( klogErr, rc, "VFSManagerMakePath() failed" );
+ }
+ else
+ {
+ const VPath * local = NULL;
+ const VPath * remote = NULL;
+ rc = VResolverQuery ( resolver, eProtocolHttp, acc_vpath, &local, &remote, NULL );
+ if ( rc != 0 )
+ {
+ (void)LOGERR( klogErr, rc, "VResolverQuery() failed" );
+ }
+ else
+ {
+ if ( local != NULL )
+ rc = VPathMakeString( local, resolved );
+ else if ( remote != NULL )
+ rc = VPathMakeString( remote, resolved );
+ else
+ rc = KOutMsg( "cannot resolve '%s'\n", acc );
+
+ if ( local != NULL ) VPathRelease ( local );
+ if ( remote != NULL ) VPathRelease ( remote );
+ }
+ VPathRelease ( acc_vpath );
+ }
+ VResolverRelease( resolver );
+ }
+ return rc;
+}
static rc_t report_ref_loc( const VDBManager *vdb_mgr, VFSManager * vfs_mgr, const char * seq_id )
@@ -361,100 +406,87 @@ static rc_t report_ref_obj( const VDBManager *vdb_mgr, VFSManager * vfs_mgr, con
static rc_t report_ref_database( const VDBManager *vdb_mgr, VFSManager * vfs_mgr, const char * path, bool extended )
{
- const ReferenceList* reflist;
- uint32_t options = ( ereferencelist_usePrimaryIds | ereferencelist_useSecondaryIds | ereferencelist_useEvidenceIds );
- rc_t rc = ReferenceList_MakePath( &reflist, vdb_mgr, path, options, 0, NULL, 0 );
+ const VDatabase *db;
+ rc_t rc = VDBManagerOpenDBRead ( vdb_mgr, &db, NULL, "%s", path );
if ( rc != 0 )
{
- (void)LOGERR( klogErr, rc, "cannot create ReferenceList" );
+ (void)LOGERR( klogErr, rc, "VDBManagerOpenDBRead() failed" );
}
- else
- {
- uint32_t count;
- rc = ReferenceList_Count( reflist, &count );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "ReferenceList_Count() failed" );
- }
- else
- {
- rc = KOutMsg( "this object uses %u references\n", count );
- if ( rc == 0 )
- {
- uint32_t idx;
- for ( idx = 0; idx < count && rc == 0; ++idx )
- {
- const ReferenceObj* ref_obj;
- rc = ReferenceList_Get( reflist, &ref_obj, idx );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "ReferenceList_Get() failed" );
- }
- else
- {
- rc = report_ref_obj( vdb_mgr, vfs_mgr, path, idx, ref_obj, extended );
- ReferenceObj_Release( ref_obj );
- }
- }
- }
- }
- ReferenceList_Release( reflist );
+ else
+ {
+ const ReferenceList * reflist;
+ uint32_t options = ( ereferencelist_usePrimaryIds | ereferencelist_useSecondaryIds | ereferencelist_useEvidenceIds );
+ rc = ReferenceList_MakeDatabase( &reflist, db, options, 0, NULL, 0 );
+ if ( rc != 0 )
+ {
+ (void)LOGERR( klogErr, rc, "ReferenceList_MakeDatabase() failed" );
+ }
+ else
+ {
+ uint32_t count;
+ rc = ReferenceList_Count( reflist, &count );
+ if ( rc != 0 )
+ {
+ (void)LOGERR( klogErr, rc, "ReferenceList_Count() failed" );
+ }
+ else
+ {
+ rc = KOutMsg( "this object uses %u references\n", count );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const ReferenceObj* ref_obj;
+ rc = ReferenceList_Get( reflist, &ref_obj, idx );
+ if ( rc != 0 )
+ {
+ (void)LOGERR( klogErr, rc, "ReferenceList_Get() failed" );
+ }
+ else
+ {
+ rc = report_ref_obj( vdb_mgr, vfs_mgr, path, idx, ref_obj, extended );
+ ReferenceObj_Release( ref_obj );
+ }
+ }
+ }
+ }
+ ReferenceList_Release( reflist );
+ }
+ VDatabaseRelease( db );
}
return rc;
}
-
-static rc_t report_references( const VDBManager *vdb_mgr, VFSManager * vfs_mgr, const char * spec,
- bool extended )
+static rc_t report_references( const VDBManager *vdb_mgr, VFSManager * vfs_mgr,
+ const char * spec, bool extended )
{
- rc_t rc = KOutMsg( "\nreporting references of '%s'\n", spec );
- if ( rc == 0 )
- {
- VPath * path = NULL;
- const VPath * local_cache = NULL;
- const KFile * remote_file = NULL;
- rc = VFSManagerResolveSpec ( vfs_mgr, spec, &path, &remote_file, &local_cache, true );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot resolve spec via VFSManager" );
- }
- else
- {
- char buffer[ 4096 ];
- size_t num_read;
- rc = VPathReadPath ( path, buffer, sizeof buffer, &num_read );
- if ( rc != 0 )
- {
- (void)LOGERR( klogErr, rc, "cannot read path from vpath" );
- }
- else
- {
- rc = KOutMsg( "resolved into '%s'\n", buffer );
- if ( rc == 0 )
- {
- int path_type = ( VDBManagerPathType ( vdb_mgr, "%s", buffer ) & ~ kptAlias );
- switch( path_type )
- {
- case kptDatabase : rc = report_ref_database( vdb_mgr, vfs_mgr, buffer, extended );
- break;
-
- case kptTable : KOutMsg( "cannot report references on a table-object\n" );
- rc = RC ( rcApp, rcNoTarg, rcAccessing, rcParam, rcInvalid );
- (void)LOGERR( klogErr, rc, "cannot report references on a table-object" );
- break;
-
- default : KOutMsg( "the given object is not a vdb-database\n" );
- rc = RC ( rcApp, rcNoTarg, rcAccessing, rcParam, rcInvalid );
- (void)LOGERR( klogErr, rc, "the given object is not a vdb-database" );
- break;
- }
- }
- }
- KFileRelease( remote_file );
- VPathRelease ( local_cache );
- VPathRelease ( path );
- }
- }
+ const String * resolved = NULL;
+ rc_t rc = resolve_accession( vfs_mgr, spec, &resolved );
+ if ( rc == 0 && resolved != NULL )
+ {
+ rc = KOutMsg( "resolved into '%S'\n", resolved );
+ if ( rc == 0 )
+ {
+ int path_type = ( VDBManagerPathType ( vdb_mgr, "%s", spec ) & ~ kptAlias );
+ switch( path_type )
+ {
+ case kptDatabase : rc = report_ref_database( vdb_mgr, vfs_mgr, spec, extended );
+ break;
+
+ case kptTable : KOutMsg( "cannot report references on a table-object\n" );
+ rc = RC ( rcApp, rcNoTarg, rcAccessing, rcParam, rcInvalid );
+ (void)LOGERR( klogErr, rc, "cannot report references on a table-object" );
+ break;
+
+ default : KOutMsg( "the given object is not a vdb-database\n" );
+ rc = RC ( rcApp, rcNoTarg, rcAccessing, rcParam, rcInvalid );
+ (void)LOGERR( klogErr, rc, "the given object is not a vdb-database" );
+ break;
+ }
+ }
+ StringWhack ( resolved );
+ }
return rc;
}
@@ -497,7 +529,7 @@ rc_t report_on_reference( Args * args, bool extended )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *param = NULL;
- rc = ArgsParamValue( args, idx, ¶m );
+ rc = ArgsParamValue( args, idx, (const void **)¶m );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsParamvalue() failed" );
diff --git a/tools/sra-pileup/sam-aligned.c b/tools/sra-pileup/sam-aligned.c
index cda6494..aa84c7d 100644
--- a/tools/sra-pileup/sam-aligned.c
+++ b/tools/sra-pileup/sam-aligned.c
@@ -34,6 +34,7 @@
#include "cg_tools.h"
#include "rna_splice_log.h"
#include "sam-aligned.h"
+#include "md_flag.h"
const char * PRIM_TABLE = "PRIMARY_ALIGNMENT";
const char * SEC_TABLE = "SECONDARY_ALIGNMENT";
@@ -616,6 +617,13 @@ static rc_t add_table_pl_iter( const samdump_opts * const opts,
if ( rc == 0 )
{
rc = PlacementSetIteratorAddPlacementIterator ( set_iter, pl_iter );
+ /* if the iterator-set was not able to take ownership of the new iterator
+ we have to release the iterator right here! */
+ if ( rc != 0 )
+ PlacementIteratorRelease( pl_iter );
+
+ /* if the new iterator has actually no placements inside, the call
+ to PlacementSetIteratorAddPlacementIterator() returned rcDone, which is OK - we continue... */
if ( GetRCState( rc ) == rcDone ) { rc = 0; }
}
}
@@ -1542,13 +1550,15 @@ static rc_t print_alignment_sam_ps( const samdump_opts * const opts,
if ( opts->rna_splicing )
{
- { /*** reset previous identification of N to D ***/
- int i;
- char *c=cgc_output.p_cigar.ptr;
- for(i=0;i< cgc_output.p_cigar.len;i++){
- if(c[i]=='N') c[i]='D';
- }
- }
+ { /*** reset previous identification of N to D ***/
+ int i;
+ char *c = ( char * )cgc_output.p_cigar.ptr;
+ for( i = 0; i < cgc_output.p_cigar.len; i++ )
+ {
+ if ( c[ i ] == 'N' ) c[ i ] = 'D';
+ }
+ }
+
/* discover which cigar-operations could be a RNA-splice ( it is a D-operation with min length of 10 ) */
rc = discover_rna_splicing_candidates( cgc_output.p_cigar.len, cgc_output.p_cigar.ptr, 10, &candidates ); /* cg_tools.c */
if ( rc == 0 && candidates.count > 0 )
@@ -1729,6 +1739,26 @@ static rc_t print_alignment_sam_ps( const samdump_opts * const opts,
}
}
+ /* OPT SAM_FIELD: MD reports Mismatches and Deletions */
+ if ( rc == 0 && opts->with_md_flag )
+ {
+ uint8_t * alig_ref = malloc( rec->len );
+ if ( alig_ref == NULL )
+ rc = RC( rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted );
+ else
+ {
+ INSDC_coord_len ref_len;
+ rc = ReferenceObj_Read( rec->ref, pos, rec->len, alig_ref, &ref_len );
+ if ( rc == 0 )
+ {
+ rc = kout_md_tag_from_cigar_string( cgc_output.p_cigar.ptr, cgc_output.p_cigar.len, /* cigar */
+ cgc_output.p_read.ptr, cgc_output.p_read.len, /* read */
+ alig_ref, ref_len ); /* reference */
+ }
+ free( alig_ref );
+ }
+ }
+
if ( rc == 0 )
rc = KOutMsg( "\n" );
diff --git a/tools/sra-pileup/sam-dump-opts.c b/tools/sra-pileup/sam-dump-opts.c
index e0e63e7..24e262c 100644
--- a/tools/sra-pileup/sam-dump-opts.c
+++ b/tools/sra-pileup/sam-dump-opts.c
@@ -65,13 +65,18 @@ static range * make_range( const uint64_t start, const uint64_t end )
}
-static int cmp_range( const range * a, const range * b )
+static int64_t cmp_range( const range * a, const range * b )
{
-
- int res = ( a->start - b->start );
- if ( res == 0 )
- res = ( a->end - b->end );
- return res;
+ if ( a->start < b->start )
+ return -1;
+ else if ( a->start > b->start )
+ return 1;
+ else if ( a->end < b->end )
+ return -1;
+ else if ( a->end > b->end )
+ return 1;
+ else
+ return 0;
}
@@ -102,7 +107,7 @@ static reference_region * make_reference_region( const char *name )
}
-static int CC cmp_range_wrapper( const void *item, const void *n )
+static int64_t CC cmp_range_wrapper( const void *item, const void *n )
{ return cmp_range( item, n ); }
@@ -263,7 +268,7 @@ static void check_ref_region_ranges( reference_region * self )
/* =========================================================================================== */
-static int CC reference_vs_pchar_wrapper( const void *item, const BSTNode *n )
+static int64_t CC reference_vs_pchar_wrapper( const void *item, const BSTNode *n )
{
const reference_region * r = ( const reference_region * )n;
return cmp_pchar( (const char *)item, r->name );
@@ -293,7 +298,7 @@ static int cmp_pchar_vs_len( const char * a, const char * b, size_t len_b )
return res;
}
-static int CC reference_vs_frr_wrapper( const void *item, const BSTNode *n )
+static int64_t CC reference_vs_frr_wrapper( const void *item, const BSTNode *n )
{
const reference_region * r = ( const reference_region * )n;
const frrl * ctx = item;
@@ -309,7 +314,7 @@ static reference_region * find_reference_region_len( BSTree * regions, const cha
}
-static int CC ref_vs_ref_wrapper( const BSTNode *item, const BSTNode *n )
+static int64_t CC ref_vs_ref_wrapper( const BSTNode *item, const BSTNode *n )
{
const reference_region * a = ( const reference_region * )item;
const reference_region * b = ( const reference_region * )n;
@@ -345,7 +350,7 @@ static rc_t parse_and_add_region( BSTree * regions, const char * s )
{
rc_t rc = 0;
uint64_t start, end;
- char name[ 64 ];
+ char name[ 4096 ];
parse_definition( s, name, sizeof name, &start, &end );
if ( name[ 0 ] == 0 )
rc = RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
@@ -564,7 +569,7 @@ static rc_t gather_region_options( Args * args, samdump_opts * opts )
for ( i = 0; i < count && rc == 0; ++i )
{
const char * s;
- rc = ArgsOptionValue( args, OPT_REGION, i, &s );
+ rc = ArgsOptionValue( args, OPT_REGION, i, (const void **)&s );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "error retrieving comandline option '$(t)'", "t=%s", OPT_REGION ) );
@@ -598,7 +603,7 @@ static rc_t gather_matepair_distances( Args * args, samdump_opts * opts )
for ( i = 0; i < count && rc == 0; ++i )
{
const char * s;
- rc = ArgsOptionValue( args, OPT_MATE_DIST, i, &s );
+ rc = ArgsOptionValue( args, OPT_MATE_DIST, i, (const void **)&s );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "error retrieving comandline option '$(t)'", "t=%s", OPT_MATE_DIST ) );
@@ -806,6 +811,9 @@ static rc_t gather_flag_options( Args * args, samdump_opts * opts )
/* do we disable multi-threading */
rc = get_bool_option( args, OPT_NO_MT, &opts->no_mt );
+ /* do we have to generate the MD-flag */
+ rc = get_bool_option( args, OPT_MD_FLAG, &opts->with_md_flag );
+
/* forcing to use the legacy code in case of Evidence-Dnb was requested */
if ( rc == 0 )
{
@@ -831,7 +839,7 @@ static rc_t get_str_option( Args * args, const char * name, const char ** s )
}
else if ( count > 0 )
{
- rc = ArgsOptionValue( args, name, 0, s );
+ rc = ArgsOptionValue( args, name, 0, (const void **)s );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "error retrieving comandline option '$(t)'", "t=%s", name ) );
@@ -1008,7 +1016,7 @@ static rc_t gather_string_options( Args * args, samdump_opts * opts )
for ( i = 0; i < count && rc == 0; ++i )
{
const char * src;
- rc = ArgsOptionValue( args, OPT_HDR_COMMENT, i, &src );
+ rc = ArgsOptionValue( args, OPT_HDR_COMMENT, i, (const void **)&src );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "error retrieving comandline option '$(t)' #$(n)",
@@ -1042,7 +1050,7 @@ static rc_t gather_string_options( Args * args, samdump_opts * opts )
for ( i = 0; i < count && rc == 0; ++i )
{
const char * src;
- rc = ArgsParamValue( args, i, &src );
+ rc = ArgsParamValue( args, i, (const void **)&src );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "error retrieving comandline param #$(n)",
@@ -1334,8 +1342,9 @@ void report_options( const samdump_opts * opts )
KOutMsg( "rna-splice-level : %u\n", opts->rna_splice_level );
KOutMsg( "rna-splice-log : %s\n", opts->rna_splice_log_file );
- KOutMsg( "multithreading : %s\n", opts->no_mt ? "NO" : "YES" );
-
+ KOutMsg( "multithreading : %s\n", opts->no_mt ? "NO" : "YES" );
+ KOutMsg( "with-MD-flag : %s\n", opts->with_md_flag ? "NO" : "YES" );
+
#if _DEBUGGING
if ( opts->timing_file != NULL )
KOutMsg( "timing-file : '%s'\n", opts->timing_file );
@@ -1486,29 +1495,58 @@ rc_t dump_name_legacy( const samdump_opts * opts, const char * name, size_t name
return rc;
}
+#define USE_KWRT_HANDLER 1
rc_t dump_quality( const samdump_opts * opts, char const *quality, uint32_t qual_len, bool reverse )
{
+ uint32_t i;
rc_t rc = 0;
bool quantize = ( opts->qual_quant != NULL );
- uint32_t i;
+
+ size_t size = 0;
+ char buffer [ 4096 ];
+#if USE_KWRT_HANDLER
+ size_t num_writ;
+ KWrtHandler * kout_msg_handler = KOutHandlerGet ();
+ assert ( kout_msg_handler != NULL );
+#endif
if ( reverse )
{
if ( quantize )
{
- for ( i = 0; i < qual_len && rc == 0; ++i )
+ for ( i = qual_len; i > 0; )
{
- uint32_t qual = quality[ qual_len - i - 1 ];
- char newValue = ( opts->qual_quant_matrix[ qual ] + 33 );
- rc = KOutMsg( "%c", newValue );
+ uint32_t qual = quality[ -- i ];
+ buffer [ size ] = ( opts->qual_quant_matrix[ qual ] + 33 );
+ if ( ++ size == sizeof buffer )
+ {
+#if USE_KWRT_HANDLER
+ rc = ( * kout_msg_handler -> writer ) ( kout_msg_handler -> data, buffer, size, & num_writ );
+#else
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+#endif
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
else
{
- for ( i = 0; i < qual_len && rc == 0; ++i )
+ for ( i = qual_len; i > 0; )
{
- char qual = quality[ qual_len - i - 1 ] + 33;
- rc = KOutMsg( "%c", qual );
+ buffer [ size ] = quality[ -- i ] + 33;
+ if ( ++ size == sizeof buffer )
+ {
+#if USE_KWRT_HANDLER
+ rc = ( * kout_msg_handler -> writer ) ( kout_msg_handler -> data, buffer, size, & num_writ );
+#else
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+#endif
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
}
@@ -1516,11 +1554,21 @@ rc_t dump_quality( const samdump_opts * opts, char const *quality, uint32_t qual
{
if ( quantize )
{
- for ( i = 0; i < qual_len && rc == 0; ++i )
+ for ( i = 0; i < qual_len; ++i )
{
uint32_t qual = quality[ i ];
- char newValue = opts->qual_quant_matrix[ qual ] + 33;
- rc = KOutMsg( "%c", newValue );
+ buffer [ size ] = opts->qual_quant_matrix[ qual ] + 33;
+ if ( ++ size == sizeof buffer )
+ {
+#if USE_KWRT_HANDLER
+ rc = ( * kout_msg_handler -> writer ) ( kout_msg_handler -> data, buffer, size, & num_writ );
+#else
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+#endif
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
@@ -1528,20 +1576,43 @@ rc_t dump_quality( const samdump_opts * opts, char const *quality, uint32_t qual
{
for ( i = 0; i < qual_len && rc == 0; ++i )
{
- char qual = quality[ i ] + 33;
- rc = KOutMsg( "%c", qual );
+ buffer [ size ] = quality[ i ] + 33;
+ if ( ++ size == sizeof buffer )
+ {
+#if USE_KWRT_HANDLER
+ rc = ( * kout_msg_handler -> writer ) ( kout_msg_handler -> data, buffer, size, & num_writ );
+#else
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+#endif
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
}
+
+ if ( rc == 0 && size != 0 )
+ {
+#if USE_KWRT_HANDLER
+ rc = ( * kout_msg_handler -> writer ) ( kout_msg_handler -> data, buffer, size, & num_writ );
+#else
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+#endif
+ }
return rc;
}
rc_t dump_quality_33( const samdump_opts * opts, char const *quality, uint32_t qual_len, bool reverse )
{
+ uint32_t i;
rc_t rc = 0;
bool quantize = ( opts->qual_quant != NULL );
- uint32_t i;
+
+ size_t size = 0;
+ char buffer [ 4096 ];
+
if ( reverse )
{
if ( quantize )
@@ -1549,16 +1620,28 @@ rc_t dump_quality_33( const samdump_opts * opts, char const *quality, uint32_t q
for ( i = 0; i < qual_len && rc == 0; ++i )
{
uint32_t qual = quality[ qual_len - i - 1 ] - 33;
- char newValue = ( opts->qual_quant_matrix[ qual ] + 33 );
- rc = KOutMsg( "%c", newValue );
+ buffer [ size ] = ( opts->qual_quant_matrix[ qual ] + 33 );
+ if ( ++ size == sizeof buffer )
+ {
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
else
{
for ( i = 0; i < qual_len && rc == 0; ++i )
{
- char qual = quality[ qual_len - i - 1 ];
- rc = KOutMsg( "%c", qual );
+ buffer [ size ] = quality[ qual_len - i - 1 ];
+ if ( ++ size == sizeof buffer )
+ {
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
}
@@ -1569,8 +1652,14 @@ rc_t dump_quality_33( const samdump_opts * opts, char const *quality, uint32_t q
for ( i = 0; i < qual_len && rc == 0; ++i )
{
uint32_t qual = quality[ i ] - 33;
- char newValue = opts->qual_quant_matrix[ qual ] + 33;
- rc = KOutMsg( "%c", newValue );
+ buffer [ size ] = opts->qual_quant_matrix[ qual ] + 33;
+ if ( ++ size == sizeof buffer )
+ {
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+ if ( rc != 0 )
+ break;
+ size = 0;
+ }
}
}
@@ -1579,7 +1668,9 @@ rc_t dump_quality_33( const samdump_opts * opts, char const *quality, uint32_t q
rc = KOutMsg( "%.*s", qual_len, quality );
}
}
- return rc;
-}
+ if ( rc == 0 && size != 0 )
+ rc = KOutMsg( "%.*s", ( uint32_t ) size, buffer );
+ return rc;
+}
diff --git a/tools/sra-pileup/sam-dump-opts.h b/tools/sra-pileup/sam-dump-opts.h
index 09c66cf..71c0ba8 100644
--- a/tools/sra-pileup/sam-dump-opts.h
+++ b/tools/sra-pileup/sam-dump-opts.h
@@ -98,6 +98,7 @@ extern "C" {
#define OPT_RNA_SPLICE_LOG "rna-splice-log"
#define OPT_NO_MT "disable-multithreading"
#define OPT_TIMING "timing"
+#define OPT_MD_FLAG "with-md-flag"
typedef struct range
{
@@ -262,6 +263,8 @@ typedef struct samdump_opts
/* option to disable multi-threading */
bool no_mt;
+ bool with_md_flag;
+
uint8_t qual_quant_matrix[ 256 ];
} samdump_opts;
diff --git a/tools/sra-pileup/sam-dump.c b/tools/sra-pileup/sam-dump.c
index 203a38a..444ebf0 100644
--- a/tools/sra-pileup/sam-dump.c
+++ b/tools/sra-pileup/sam-dump.c
@@ -925,7 +925,7 @@ typedef struct ReadGroup
} ReadGroup;
-static int CC ReadGroup_sort( BSTNode const *item, BSTNode const *node )
+static int64_t CC ReadGroup_sort( BSTNode const *item, BSTNode const *node )
{
return strcmp( ( ( ReadGroup const * )item )->name, ( ( ReadGroup const * ) node )->name );
}
@@ -4350,7 +4350,7 @@ static unsigned GetOptValU( Args const *const args, char const *const argname,
{
unsigned y = defval;
char const *valstr;
- rc_t rc = ArgsOptionValue( args, argname, 0, &valstr );
+ rc_t rc = ArgsOptionValue( args, argname, 0, (const void **)&valstr );
if ( rc == 0 )
{
@@ -4380,7 +4380,7 @@ static rc_t GetComments( Args const *const args, struct params_s *const parms, u
for ( i = 0; i < n; ++i )
{
- rc_t const rc = ArgsOptionValue( args, DumpArgs[ earg_comment ].name, i, &parms->comments[ i ] );
+ rc_t const rc = ArgsOptionValue( args, DumpArgs[ earg_comment ].name, i, (const void **)&parms->comments[ i ] );
if ( rc != 0 )
{
free( ( void * )parms->comments );
@@ -4454,7 +4454,7 @@ static rc_t GetDistance( Args const *const args, struct params_s *const parms, u
int from = 0;
int to = 0;
- rc = ArgsOptionValue( args, DumpArgs[earg_distance].name, i, &valstr );
+ rc = ArgsOptionValue( args, DumpArgs[earg_distance].name, i, (const void **)&valstr );
if ( rc != 0 )
break;
if ( strcasecmp( valstr, "unknown" ) == 0 )
@@ -4516,7 +4516,7 @@ static rc_t GetRegion( Args const *const args, struct params_s *const parms, uns
memset(&r,0,sizeof(r));
- rc = ArgsOptionValue( args, DumpArgs[ earg_region ].name, i, &valstr );
+ rc = ArgsOptionValue( args, DumpArgs[ earg_region ].name, i, (const void **)&valstr );
if ( rc != 0 ) break;
sep = strchr( valstr, ':' );
@@ -4642,7 +4642,7 @@ static rc_t GetQualQuant( Args const *const args, struct params_s *const parms )
{
char const *valstr;
int i;
- rc_t rc = ArgsOptionValue( args, DumpArgs[ earg_QualQuant ].name, 0, &valstr );
+ rc_t rc = ArgsOptionValue( args, DumpArgs[ earg_QualQuant ].name, 0, (const void **)&valstr );
if ( rc == 0 && strcmp( valstr, "0" ) != 0 )
parms->quantizeQual = QualityQuantizerInitMatrix( parms->qualQuant, valstr );
@@ -4828,7 +4828,7 @@ rc_t CC SAM_Dump_Main( int argc, char* argv[] )
{
char const *arg;
- rc = ArgsParamValue( args, i, &arg );
+ rc = ArgsParamValue( args, i, (const void **)&arg );
if ( rc != 0 ) break;
rc = ProcessPath( mgr, arg );
#if _ARCH_BITS < 64
diff --git a/tools/sra-pileup/sam-dump.vers b/tools/sra-pileup/sam-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-pileup/sam-dump.vers
+++ b/tools/sra-pileup/sam-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-pileup/sam-dump.vers.h b/tools/sra-pileup/sam-dump.vers.h
index d8a5034..5724448 100644
--- a/tools/sra-pileup/sam-dump.vers.h
+++ b/tools/sra-pileup/sam-dump.vers.h
@@ -1 +1 @@
-#define SAM_DUMP_VERS 0x02040002
+#define SAM_DUMP_VERS 0x02050007
diff --git a/tools/sra-pileup/sam-dump2.vers b/tools/sra-pileup/sam-dump2.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-pileup/sam-dump2.vers
+++ b/tools/sra-pileup/sam-dump2.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-pileup/sam-dump3.c b/tools/sra-pileup/sam-dump3.c
index 67087d2..b3219c4 100644
--- a/tools/sra-pileup/sam-dump3.c
+++ b/tools/sra-pileup/sam-dump3.c
@@ -34,6 +34,7 @@
#include "sam-dump-opts.h"
#include "inputfiles.h"
#include "sam-hdr.h"
+#include "sam-hdr1.h"
#include "matecache.h"
#include "cg_tools.h"
#include "out_redir.h"
@@ -139,7 +140,7 @@ char const *sd_cachereport_usage[] = { "print report about mate-pair-cache",
char const *sd_unaligned_only_usage[] = { "output reads for spots with no aligned reads",
NULL };
-char const *sd_cg_names_usage[] = { "prints cg-style spotgroup.spotid fromed names",
+char const *sd_cg_names_usage[] = { "prints cg-style spotgroup.spotid formed names",
NULL };
char const *sd_cur_cache_usage[] = { "open cached cursor with this size",
@@ -163,8 +164,9 @@ char const *rna_splicel_usage[] = { "level of rna-splicing detection (0,1,
char const *rna_splice_log_usage[] = { "file, into which rna-splice events are written", NULL };
-char const *no_mt_usage[] = { "disable multithreading", NULL };
+char const *no_mt_usage[] = { "disable multithreading", NULL };
+char const *with_md_flag_usage[] = { "print MD-flag", NULL };
OptDef SamDumpArgs[] =
{
@@ -208,6 +210,7 @@ OptDef SamDumpArgs[] =
{ OPT_RNA_SPLICEL, NULL, NULL, rna_splicel_usage, 0, true, false }, /* level of rna-splicing detection */
{ OPT_RNA_SPLICE_LOG, NULL, NULL, rna_splice_log_usage, 0, true, false }, /* filename to log rna-splice events into */
{ OPT_NO_MT, NULL, NULL, no_mt_usage, 0, false, false }, /* force new code-path */
+ { OPT_MD_FLAG, NULL, NULL, with_md_flag_usage, 0, false, false }, /* print the MD-flag */
{ OPT_DUMP_MODE, NULL, NULL, NULL, 0, true, false }, /* how to produce aligned reads if no regions given */
{ OPT_CIGAR_TEST, NULL, NULL, NULL, 0, true, false }, /* test cg-treatment of cigar string */
{ OPT_LEGACY, NULL, NULL, NULL, 0, false, false }, /* force legacy code-path */
@@ -256,7 +259,8 @@ char const *sd_usage_params[] =
NULL, /* detect rna-splicing in sequence */
NULL, /* level of rna-splicing detection */
NULL, /* file to log rna-splice-events into */
- NULL, /* no-mt */
+ NULL, /* no-mt */
+ NULL, /* with-md-flag */
NULL, /* dump_mode */
NULL, /* cigar test */
NULL, /* force legacy code path */
@@ -397,7 +401,7 @@ static rc_t print_samdump( const samdump_opts * const opts )
( opts->header_mode != hm_none ) &&
!opts->dump_unaligned_only )
/* ------------------------------------------------------ */
- rc = print_headers( opts, ifs ); /* sam-hdr.c */
+ rc = print_headers_1( opts, ifs ); /* sam-hdr.c */
/* ------------------------------------------------------ */
diff --git a/tools/sra-pileup/sam-hdr.c b/tools/sra-pileup/sam-hdr.c
index ab9ec8e..1e5ca02 100644
--- a/tools/sra-pileup/sam-hdr.c
+++ b/tools/sra-pileup/sam-hdr.c
@@ -141,7 +141,7 @@ static void free_seq_id_tree( BSTree * tree )
}
-static int CC seq_id_node_vs_pchar_wrapper( const void *item, const BSTNode *n )
+static int64_t CC seq_id_node_vs_pchar_wrapper( const void *item, const BSTNode *n )
{
const seq_id_node * node = ( const seq_id_node * )n;
return cmp_pchar_0( (const char *)item, node->seq_id );
@@ -154,7 +154,7 @@ static seq_id_node * find_seq_id_node( BSTree * tree, const char * key )
}
-static int CC node_vs_node_wrapper( const BSTNode *item, const BSTNode *n )
+static int64_t CC node_vs_node_wrapper( const BSTNode *item, const BSTNode *n )
{
const seq_id_node * a = ( const seq_id_node * )item;
const seq_id_node * b = ( const seq_id_node * )n;
diff --git a/tools/sra-pileup/sam-hdr1.c b/tools/sra-pileup/sam-hdr1.c
new file mode 100644
index 0000000..6b67fbe
--- /dev/null
+++ b/tools/sra-pileup/sam-hdr1.c
@@ -0,0 +1,853 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <klib/rc.h>
+#include <klib/text.h>
+#include <klib/printf.h>
+#include <kfs/filetools.h>
+#include <kdb/meta.h>
+#include <kdb/namelist.h>
+#include <sysalloc.h>
+
+#include "sam-hdr1.h"
+
+
+typedef struct headers
+{
+ VNamelist * SQ_Lines_1;
+ VNamelist * SQ_Lines_2;
+
+ VNamelist * RG_Lines_1;
+ VNamelist * RG_Lines_2;
+
+ VNamelist * Other_Lines;
+ VNamelist * HD_Lines;
+} headers;
+
+
+static void release_lines( VNamelist ** lines )
+{
+ if ( *lines != NULL )
+ {
+ VNamelistRelease ( *lines );
+ lines = NULL;
+ }
+}
+
+
+static void release_headers( headers * h )
+{
+ release_lines( &h->HD_Lines );
+ release_lines( &h->Other_Lines );
+ release_lines( &h->RG_Lines_1 );
+ release_lines( &h->RG_Lines_2 );
+ release_lines( &h->SQ_Lines_1 );
+ release_lines( &h->SQ_Lines_2 );
+}
+
+
+static rc_t init_headers( headers * h, uint32_t blocksize )
+{
+ rc_t rc;
+ h->SQ_Lines_1 = NULL;
+ h->SQ_Lines_2 = NULL;
+ h->RG_Lines_1 = NULL;
+ h->RG_Lines_2 = NULL;
+ h->Other_Lines = NULL;
+ h->HD_Lines = NULL;
+
+ rc = VNamelistMake( &h->SQ_Lines_1, blocksize );
+ if ( rc == 0 )
+ rc = VNamelistMake( &h->SQ_Lines_2, blocksize );
+ if ( rc == 0 )
+ rc = VNamelistMake( &h->RG_Lines_1, blocksize );
+ if ( rc == 0 )
+ rc = VNamelistMake( &h->RG_Lines_2, blocksize );
+ if ( rc == 0 )
+ rc = VNamelistMake( &h->Other_Lines, blocksize );
+ if ( rc == 0 )
+ rc = VNamelistMake( &h->HD_Lines, blocksize );
+
+ if ( rc != 0 )
+ release_headers( h );
+
+ return rc;
+}
+
+
+static void process_line( headers * h, int idx, const char * line, size_t len )
+{
+ if ( len > 3 && line[ 0 ] == '@' )
+ {
+ if ( line[ 1 ] == 'S' && line[ 2 ] == 'Q' )
+ {
+ if ( idx == 1 )
+ VNamelistAppend( h->SQ_Lines_1, line );
+ else
+ VNamelistAppend( h->SQ_Lines_2, line );
+ }
+ else if ( line[ 1 ] == 'R' && line[ 2 ] == 'G' )
+ {
+ if ( idx == 1 )
+ VNamelistAppend( h->RG_Lines_1, line );
+ else
+ VNamelistAppend( h->RG_Lines_2, line );
+ }
+ else if ( line[ 1 ] == 'H' && line[ 2 ] == 'D' )
+ VNamelistAppend( h->HD_Lines, line );
+ else
+ VNamelistAppend( h->Other_Lines, line );
+ }
+}
+
+
+static rc_t process_lines( headers * h, int idx, VNamelist * content, const char * identifier )
+{
+ uint32_t i, count;
+ rc_t rc = VNameListCount( content, &count );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant get count for content of '$(t)'", "t=%s", identifier ) );
+ }
+ else
+ {
+ const char * line = NULL;
+ for ( i = 0; i < count && rc == 0; ++i )
+ {
+ rc = VNameListGet( content, i, &line );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant get line #$(t) from content", "t=%u", i ) );
+ }
+ else
+ process_line( h, idx, line, string_measure( line, NULL ) );
+ }
+ }
+ return rc;
+}
+
+
+#define STATE_ALPHA 0
+#define STATE_LF 1
+#define STATE_NL 2
+
+typedef struct buffer_range
+{
+ const char * start;
+ uint32_t processed, count, state;
+} buffer_range;
+
+
+static const char empty_str[ 2 ] = { ' ', 0 };
+
+
+static void LoadFromBuffer( VNamelist * nl, buffer_range * range )
+{
+ uint32_t idx;
+ const char * p = range->start;
+ String S;
+
+ S.addr = p;
+ S.len = S.size = range->processed;
+ for ( idx = range->processed; idx < range->count; ++idx )
+ {
+ switch( p[ idx ] )
+ {
+ case 0x0A : switch( range->state )
+ {
+ case STATE_ALPHA : /* ALPHA --> LF */
+ VNamelistAppendString ( nl, &S );
+ range->state = STATE_LF;
+ break;
+
+ case STATE_LF : /* LF --> LF */
+ VNamelistAppend ( nl, empty_str );
+ break;
+
+ case STATE_NL : /* NL --> LF */
+ VNamelistAppend ( nl, empty_str );
+ range->state = STATE_LF;
+ break;
+ }
+ break;
+
+ case 0x0D : switch( range->state )
+ {
+ case STATE_ALPHA : /* ALPHA --> NL */
+ VNamelistAppendString ( nl, &S );
+ range->state = STATE_NL;
+ break;
+
+ case STATE_LF : /* LF --> NL */
+ range->state = STATE_NL;
+ break;
+
+ case STATE_NL : /* NL --> NL */
+ VNamelistAppend ( nl, empty_str );
+ break;
+ }
+ break;
+
+ default : switch( range->state )
+ {
+ case STATE_ALPHA : /* ALPHA --> ALPHA */
+ S.len++; S.size++;
+ break;
+
+ case STATE_LF : /* LF --> ALPHA */
+ S.addr = &p[ idx ]; S.len = S.size = 1;
+ range->state = STATE_ALPHA;
+ break;
+
+ case STATE_NL : /* NL --> ALPHA */
+ S.addr = &p[ idx ]; S.len = S.size = 1;
+ range->state = STATE_ALPHA;
+ break;
+ }
+ break;
+ }
+ }
+ if ( range->state == STATE_ALPHA )
+ {
+ range->start = S.addr;
+ range->count = S.len;
+ }
+ else
+ range->count = 0;
+}
+
+
+static rc_t Load_Namelist_From_Node( VNamelist * nl, const KMDataNode * node )
+{
+ rc_t rc = 0;
+ size_t pos = 0, num_read, remaining = ~0;
+ char buffer[ 4096 ];
+ buffer_range range;
+
+ range.start = buffer;
+ range.count = 0;
+ range.processed = 0;
+ range.state = STATE_ALPHA;
+
+ do
+ {
+ rc = KMDataNodeRead( node, pos, buffer, sizeof( buffer ), &num_read, &remaining );
+ if ( rc == 0 )
+ {
+ if ( num_read > 0 )
+ {
+ range.start = buffer;
+ range.count = range.processed + num_read;
+
+ LoadFromBuffer( nl, &range );
+ if ( range.count > 0 )
+ {
+ memmove ( buffer, range.start, range.count );
+ }
+ range.start = buffer;
+ range.processed = range.count;
+
+ pos += num_read;
+ }
+
+ if ( remaining == 0 && range.state == STATE_ALPHA )
+ {
+ String S;
+ S.addr = range.start;
+ S.len = S.size = range.count;
+ VNamelistAppendString ( nl, &S );
+ }
+ }
+ } while ( rc == 0 && remaining > 0 );
+
+ return rc;
+}
+
+
+static rc_t collect_from_BAM_HEADER( headers * h, int hdr_idx, input_files * ifs )
+{
+ rc_t rc = 0;
+ if ( ifs->database_count > 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < ifs->database_count && rc == 0; ++idx )
+ {
+ input_database * id = VectorGet( &ifs->dbs, idx );
+ if ( id != NULL )
+ {
+ const KMetadata * meta;
+ rc = VDatabaseOpenMetadataRead( id->db, &meta );
+ if ( rc == 0 )
+ {
+ const KMDataNode * node;
+ rc = KMetadataOpenNodeRead( meta, &node, "BAM_HEADER" );
+ if ( rc == 0 )
+ {
+ VNamelist * content;
+ rc = VNamelistMake ( &content, 25 );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant create container for '$(t)'", "t=%s", id->path ) );
+ }
+ else
+ {
+ rc = Load_Namelist_From_Node( content, node );
+ if ( rc == 0 )
+ rc = process_lines( h, hdr_idx, content, id->path );
+ VNamelistRelease( content );
+ }
+ KMDataNodeRelease( node );
+ }
+ else
+ rc = 0;
+ KMetadataRelease( meta );
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t collect_from_spotgroup_stats( headers * h, int hdr_idx, const KMDataNode * node, const KNamelist * spot_groups )
+{
+ uint32_t count;
+ rc_t rc = KNamelistCount( spot_groups, &count );
+ if ( rc == 0 && count > 0 )
+ {
+ uint32_t i;
+ for ( i = 0; i < count && rc == 0; ++i )
+ {
+ const char * name = NULL; /* this is the name of the node !!!NOT!!! the name of the spotgroup! */
+ rc = KNamelistGet( spot_groups, i, &name );
+ if ( rc == 0 && name != NULL )
+ {
+ const KMDataNode * spot_count_node;
+ rc = KMDataNodeOpenNodeRead( node, &spot_count_node, "%s/SPOT_COUNT", name );
+ if ( rc == 0 )
+ {
+ uint64_t spot_count = 0;
+ rc = KMDataNodeReadAsU64( spot_count_node, &spot_count );
+ if ( rc == 0 )
+ {
+ if ( spot_count > 0 )
+ {
+ const KMDataNode * spot_group_node;
+ rc = KMDataNodeOpenNodeRead( node, &spot_group_node, name );
+ if ( rc == 0 )
+ {
+ char name_attr[ 2048 ];
+ char buffer[ 2048 ];
+ size_t num_writ;
+ rc = KMDataNodeReadAttr( spot_group_node, "name", name_attr, sizeof name_attr, &num_writ );
+ rc = string_printf( buffer, sizeof buffer, &num_writ,
+ "@RG\tID:%s", rc == 0 ? name_attr : name );
+ if ( rc == 0 )
+ process_line( h, hdr_idx, buffer, num_writ );
+ KMDataNodeRelease( spot_group_node );
+ }
+ }
+ }
+ else
+ rc = 0;
+ KMDataNodeRelease( spot_count_node );
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t collect_from_stats( headers * h, int hdr_idx, input_files * ifs )
+{
+ rc_t rc = 0;
+ if ( ifs->database_count > 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < ifs->database_count && rc == 0; ++idx )
+ {
+ input_database * id = VectorGet( &ifs->dbs, idx );
+ if ( id != NULL )
+ {
+ const VTable * seq_table = NULL;
+ rc = VDatabaseOpenTableRead( id->db, &seq_table, "SEQUENCE" );
+ if ( rc == 0 )
+ {
+ const KMetadata * meta;
+ rc = VTableOpenMetadataRead( seq_table, &meta );
+ if ( rc == 0 )
+ {
+ const KMDataNode * node;
+ rc = KMetadataOpenNodeRead( meta, &node, "STATS/SPOT_GROUP" );
+ if ( rc == 0 )
+ {
+ KNamelist * spot_groups;
+ rc = KMDataNodeListChildren( node, &spot_groups );
+ if ( rc == 0 )
+ {
+ rc = collect_from_spotgroup_stats( h, hdr_idx, node, spot_groups );
+ KNamelistRelease( spot_groups );
+ }
+ KMDataNodeRelease( node );
+ }
+ KMetadataRelease( meta );
+ }
+ VTableRelease( seq_table );
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t collect_from_file( headers * h, int hdr_idx, const char * filename )
+{
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir ( &dir );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant created native directory for file '$(t)'", "t=%s", filename ) );
+ }
+ else
+ {
+ const struct KFile * f;
+ rc = KDirectoryOpenFileRead ( dir, &f, "%s", filename );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant open file '$(t)'", "t=%s", filename ) );
+ }
+ else
+ {
+ VNamelist * content;
+ rc = VNamelistMake ( &content, 25 );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant create container for file '$(t)'", "t=%s", filename ) );
+ }
+ else
+ {
+ rc = LoadKFileToNameList( f, content );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogErr, ( klogErr, rc, "cant load file '$(t)' into container", "t=%s", filename ) );
+ }
+ else
+ rc = process_lines( h, hdr_idx, content, filename );
+ VNamelistRelease( content );
+ }
+ KFileRelease ( f );
+ }
+ KDirectoryRelease ( dir );
+ }
+ return rc;
+}
+
+
+static rc_t collect_from_references( headers * h, int hdr_idx, input_files * ifs )
+{
+ rc_t rc = 0;
+ uint32_t i;
+ for ( i = 0; i < ifs->database_count && rc == 0; ++i )
+ {
+ input_database * id = VectorGet( &ifs->dbs, i );
+ if ( id != NULL )
+ {
+ uint32_t rcount;
+ rc = ReferenceList_Count( id->reflist, &rcount );
+ if ( rc == 0 && rcount > 0 )
+ {
+ uint32_t r_idx;
+ for ( r_idx = 0; r_idx < rcount && rc == 0; ++r_idx )
+ {
+ const ReferenceObj * ref_obj;
+ rc = ReferenceList_Get( id->reflist, &ref_obj, r_idx );
+ if ( rc == 0 && ref_obj != NULL )
+ {
+ const char * seqid;
+ rc = ReferenceObj_SeqId( ref_obj, &seqid );
+ if ( rc == 0 )
+ {
+ const char * name;
+ rc = ReferenceObj_Name( ref_obj, &name );
+ if ( rc == 0 )
+ {
+ INSDC_coord_len seq_len;
+ rc = ReferenceObj_SeqLength( ref_obj, &seq_len );
+ if ( rc == 0 )
+ {
+ char buffer[ 2048 ];
+ size_t num_writ;
+ rc = string_printf( buffer, sizeof buffer, &num_writ, "@SQ\tSN:%s\tLN:%lu", name, seq_len );
+ if ( rc == 0 )
+ process_line( h, hdr_idx, buffer, num_writ );
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+typedef struct hdr_tag
+{
+ String key, value;
+} hdr_tag;
+
+
+typedef struct hdr_line
+{
+ String line_key;
+ hdr_tag tags[ 32 ];
+ uint32_t n_tags;
+} hdr_line;
+
+
+static bool parse_hdr_line( hdr_line * hl, const char * line )
+{
+ size_t i, tl = 0, start = 3, len = string_size( line );
+ bool res = ( len > start && line[ 0 ] == '@' );
+ hl->n_tags = 0;
+ if ( res )
+ {
+ uint32_t colons = 0;
+ StringInit( &hl->line_key, &line[ 1 ], 2, 2 );
+ for( i = start; i < len; ++i )
+ {
+ switch ( line[ i ] )
+ {
+ case '\t' : if ( tl > 0 )
+ {
+ StringInit( &hl->tags[ hl->n_tags ].value, &line[ start ], tl, tl );
+ ( hl->n_tags )++;
+ tl = 0;
+ }
+ start = i + 1;
+ colons = 0;
+ break;
+
+ case ':' : if ( colons == 0 )
+ {
+ if ( tl > 0 )
+ {
+ StringInit( &hl->tags[ hl->n_tags ].key, &line[ start ], tl, tl );
+ tl = 0;
+ }
+ start = i + 1;
+ }
+ else
+ tl++;
+ colons++;
+ break;
+
+ default : tl++;
+ break;
+ }
+ }
+ if ( tl > 0 )
+ {
+ StringInit( &hl->tags[ hl->n_tags ].value, &line[ start ], tl, tl );
+ ( hl->n_tags )++;
+ }
+ }
+ else
+ StringInit( &hl->line_key, NULL, 0, 0 );
+ return res;
+}
+
+
+static rc_t print_hdr_line( char * buffer, size_t buflen, const hdr_line * hl )
+{
+ rc_t rc = 0;
+ buffer[ 0 ] = 0;
+ if ( hl->line_key.len > 0 )
+ {
+ size_t num_writ, total_writ = 0, i;
+ rc = string_printf( buffer, buflen, &num_writ, "@%S", &hl->line_key );
+ for ( i = 0; i < hl->n_tags && rc == 0; ++i )
+ {
+ total_writ += num_writ;
+ if ( buflen > total_writ )
+ rc = string_printf( &buffer[ total_writ ], buflen - total_writ, &num_writ,
+ "\t%S:%S", &hl->tags[ i ].key, &hl->tags[ i ].value );
+ }
+ }
+ return rc;
+}
+
+
+static rc_t append_hdr_line( VNamelist * dst, const hdr_line * hl )
+{
+ rc_t rc = 0;
+ if ( dst != NULL && hl != NULL && hl->line_key.len > 0 )
+ {
+ char buffer[ 2048 ];
+ rc = print_hdr_line( buffer, sizeof buffer, hl );
+ if ( rc == 0 && buffer[ 0 ] != 0 )
+ rc = VNamelistAppend( dst, buffer );
+ }
+ return rc;
+}
+
+
+static bool sam_hdr_id( const hdr_line * hl1, const hdr_line * hl2 )
+{
+ bool res = ( hl1 != NULL && hl2 != NULL );
+ if ( res )
+ res = ( ( hl1->n_tags > 0 ) && ( hl2->n_tags > 0 ) );
+ if ( res )
+ res = ( 0 == StringCompare( &hl1->tags[ 0 ].value, &hl2->tags[ 0 ].value ) );
+ return res;
+}
+
+
+#if 0
+static void copy_hdr_line( const hdr_line * src, hdr_line * dst )
+{
+ if ( src != NULL && dst != NULL )
+ {
+ uint32_t i;
+
+ memset( dst, 0, sizeof *dst );
+ dst->line_key = src->line_key;
+ for ( i = 0; i < src->n_tags; ++i )
+ dst->tags[ i ] = src->tags[ i ];
+ dst->n_tags = src->n_tags;
+ }
+}
+#endif
+
+
+static bool has_tag( const String * tag_id, hdr_line * dst )
+{
+ bool res = false;
+ uint32_t i;
+ for ( i = 0; i < dst->n_tags && !res; ++i )
+ res = ( 0 == StringCompare( tag_id, &dst->tags[ i ].key ) );
+ return res;
+}
+
+
+static void merge_tag( const hdr_tag * tag, hdr_line * dst )
+{
+ if ( !has_tag( &tag->key, dst ) )
+ {
+ dst->tags[ dst->n_tags ] = *tag;
+ ( dst->n_tags )++;
+ }
+}
+
+static void merge_hdr_line( const hdr_line * src, hdr_line * dst )
+{
+ if ( src != NULL && dst != NULL )
+ {
+ uint32_t i;
+ for ( i = 0; i < src->n_tags; ++i )
+ merge_tag( &src->tags[ i ], dst );
+ }
+}
+
+
+static rc_t for_each_line( const VNamelist * src,
+ rc_t ( * f ) ( const char * line, void * context ),
+ void * context )
+{
+ uint32_t idx, count;
+ rc_t rc = VNameListCount( src, &count );
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * line = NULL;
+ rc = VNameListGet( src, idx, &line );
+ if ( rc == 0 )
+ rc = f( line, context );
+ }
+ return rc;
+}
+
+
+static rc_t merge_callback_2( const char * line, void * context )
+{
+ hdr_line * h_parent = context;
+ hdr_line h;
+ if ( parse_hdr_line( &h, line ) )
+ {
+ if ( sam_hdr_id( h_parent, &h ) )
+ {
+ /* merge the tags! */
+ merge_hdr_line( h_parent, &h );
+ }
+ }
+ return 0;
+}
+
+
+typedef struct merge_ctx
+{
+ VNamelist * dst;
+ VNamelist * other;
+} merge_ctx;
+
+
+static rc_t merge_callback_1( const char * line, void * context )
+{
+ rc_t rc = 0;
+ merge_ctx * mc = context;
+ hdr_line h;
+ if ( parse_hdr_line( &h, line ) )
+ {
+ rc = for_each_line( mc->other, merge_callback_2, &h );
+ if ( rc == 0 )
+ rc = append_hdr_line( mc->dst, &h );
+ }
+ return rc;
+}
+
+
+/* SQ-lines have to be uniue by the SN-tag */
+/* RG-lines have to be uniue by the ID-tag */
+static rc_t merge_lines( VNamelist ** lines_1, VNamelist * lines_2 )
+{
+ rc_t rc;
+ merge_ctx mc;
+ memset( &mc, 0, sizeof mc );
+ rc = VNamelistMake( &mc.dst, 25 );
+ if ( rc == 0 )
+ {
+ mc.other = lines_2;
+ rc = for_each_line( *lines_1, merge_callback_1, &mc );
+ if ( rc == 0 )
+ {
+ VNamelistRelease( *lines_1 );
+ *lines_1 = mc.dst;
+ }
+ }
+ return rc;
+}
+
+
+static rc_t collect_from_bam_hdr( headers * h, input_files * ifs )
+{
+ uint32_t count;
+ rc_t rc = collect_from_BAM_HEADER( h, 1, ifs );
+ if ( rc == 0 )
+ {
+ rc = VNameListCount( h->SQ_Lines_1, &count );
+ if ( rc == 0 && count == 0 )
+ rc = collect_from_references( h, 1, ifs );
+ }
+ if ( rc == 0 )
+ {
+ rc = VNameListCount( h->RG_Lines_1, &count );
+ if ( rc == 0 && count == 0 )
+ rc = collect_from_stats( h, 1, ifs );
+ }
+ return rc;
+}
+
+
+static rc_t collect_by_recalc( headers * h, input_files * ifs )
+{
+ rc_t rc = collect_from_references( h, 1, ifs );
+ if ( rc == 0 )
+ rc = collect_from_stats( h, 1, ifs );
+ return rc;
+}
+
+
+static rc_t collect_from_src_and_files( headers * h, input_files * ifs, const char * filename )
+{
+ rc_t rc = collect_from_bam_hdr( h, ifs );
+ if ( rc == 0 && filename != NULL )
+ rc = collect_from_file( h, 2, filename );
+ return rc;
+}
+
+
+static rc_t print_HD_line( const VNamelist * lines )
+{
+ uint32_t count;
+ rc_t rc = VNameListCount( lines, &count );
+ if ( rc == 0 && count > 0 )
+ {
+ const char * line = NULL;
+ rc = VNameListGet( lines, 0, &line );
+ if ( rc == 0 && line != NULL )
+ rc = KOutMsg( "%s\n", line );
+ else
+ rc = KOutMsg( "@HD\tVN:1.2\tSO:coordinate\n" );
+ }
+ else
+ rc = KOutMsg( "@HD\tVN:1.2\tSO:coordinate\n" );
+ return rc;
+}
+
+static rc_t print_callback( const char * line, void * context ) { return KOutMsg( "%s\n", line ); }
+
+rc_t print_headers_1( const samdump_opts * opts, input_files * ifs )
+{
+ headers h;
+ rc_t rc = init_headers( &h, 25 );
+ if ( rc == 0 )
+ {
+ /* collect ... */
+
+ switch( opts->header_mode )
+ {
+ case hm_dump : rc = collect_from_bam_hdr( &h, ifs ); break;
+
+ case hm_recalc : rc = collect_by_recalc( &h, ifs ); break;
+
+ case hm_file : rc = collect_from_src_and_files( &h, ifs, opts->header_file ); break;
+
+ case hm_none : break; /* to not let the compiler complain about not handled enum */
+ }
+
+ /* merge ... */
+ if ( rc == 0 )
+ rc = merge_lines( &h.SQ_Lines_1, h.SQ_Lines_2 );
+ if ( rc == 0 )
+ rc = merge_lines( &h.RG_Lines_1, h.RG_Lines_2 );
+
+
+ /* print ... */
+ if ( rc == 0 )
+ rc = print_HD_line( h.HD_Lines );
+ if ( rc == 0 )
+ rc = for_each_line( h.SQ_Lines_1, print_callback, NULL );
+ if ( rc == 0 )
+ rc = for_each_line( h.RG_Lines_1, print_callback, NULL );
+ if ( rc == 0 )
+ rc = for_each_line( h.Other_Lines, print_callback, NULL );
+
+ release_headers( &h );
+ }
+ return rc;
+}
\ No newline at end of file
diff --git a/tools/cg-dump/last_rowid.h b/tools/sra-pileup/sam-hdr1.h
similarity index 87%
copy from tools/cg-dump/last_rowid.h
copy to tools/sra-pileup/sam-hdr1.h
index 5ae92f0..4e7f645 100644
--- a/tools/cg-dump/last_rowid.h
+++ b/tools/sra-pileup/sam-hdr1.h
@@ -23,22 +23,20 @@
* ===========================================================================
*
*/
-
-#ifndef _h_last_rowid_
-#define _h_last_rowid_
+#ifndef _h_sam_headers_1_
+#define _h_sam_headers_1_
#ifdef __cplusplus
extern "C" {
#endif
+#if 0
+}
+#endif
#include <klib/rc.h>
-#include <klib/text.h>
-#include <klib/container.h>
-
-rc_t discover_last_rowid( const char * src, int64_t * last_row_id );
+#include "sam-dump-opts.h"
+#include "inputfiles.h"
-#ifdef __cplusplus
-}
-#endif
+rc_t print_headers_1( const samdump_opts * opts, input_files * ifs );
#endif
diff --git a/tools/sra-pileup/sra-pileup.c b/tools/sra-pileup/sra-pileup.c
index e93dfc3..4981215 100644
--- a/tools/sra-pileup/sra-pileup.c
+++ b/tools/sra-pileup/sra-pileup.c
@@ -28,6 +28,7 @@
#include "ref_regions.h"
#include "cmdline_cmn.h"
+#include "out_redir.h"
#include "pileup_options.h"
#include "dyn_string.h"
#include "reref.h"
@@ -39,6 +40,7 @@
#include "pileup_counters.h"
#include "pileup_index.h"
#include "pileup_varcount.h"
+#include "pileup_indels.h"
#include "pileup_stat.h"
#include "pileup_v2.h"
@@ -99,6 +101,8 @@
#define OPTION_MIN_M "minmismatch"
#define OPTION_MERGE "merge-dist"
+#define OPTION_DEPTH_PER_SPOTGRP "depth-per-spotgroup"
+
#define OPTION_FUNC "function"
#define ALIAS_FUNC NULL
@@ -112,6 +116,7 @@
#define FUNC_TEST "test"
#define FUNC_VARCOUNT "varcount"
#define FUNC_DELETES "deletes"
+#define FUNC_INDELS "indels"
enum
{
@@ -125,7 +130,8 @@ enum
sra_pileup_index = 7,
sra_pileup_test = 8,
sra_pileup_varcount = 9,
- sra_pileup_deletes = 10
+ sra_pileup_deletes = 10,
+ sra_pileup_indels = 11
};
static const char * minmapq_usage[] = { "Minimum mapq-value, ",
@@ -142,6 +148,8 @@ static const char * showid_usage[] = { "Shows alignment-id for every ba
static const char * spotgrp_usage[] = { "divide by spotgroups", NULL };
+static const char * dpgrp_usage[] = { "print depth per spotgroup", NULL };
+
static const char * seqname_usage[] = { "use original seq-name", NULL };
static const char * min_m_usage[] = { "min percent of mismatches used in function mismatch, default is 5%", NULL };
@@ -150,12 +158,15 @@ static const char * merge_usage[] = { "If adjacent slices are closer t
"they are merged and a skiplist is created. ",
"a value of zero disables the feature, default is 10000", NULL };
+static const char * no_qual_usage[] = { "omit qualities", NULL };
+
static const char * func_ref_usage[] = { "list references", NULL };
static const char * func_ref_ex_usage[] = { "list references + coverage", NULL };
static const char * func_count_usage[] = { "sort pileup with counters", NULL };
static const char * func_stat_usage[] = { "strand/tlen statistic", NULL };
static const char * func_mismatch_usage[] = { "only lines with mismatch", NULL };
static const char * func_index_usage[] = { "list deletion counts", NULL };
+static const char * func_indels_usage[] = { "list only inserts/deletions", NULL };
static const char * func_varcount_usage[] = { "variation counters: ",
"ref-name, ref-pos, ref-base, coverage, ",
@@ -169,17 +180,18 @@ static const char * func_usage[] = { "alternative functionality", NUL
OptDef MyOptions[] =
{
- /*name, alias, hfkt, usage-help, maxcount, needs value, required */
- { OPTION_MINMAPQ, ALIAS_MINMAPQ, NULL, minmapq_usage, 1, true, false },
- { OPTION_DUPS, ALIAS_DUPS, NULL, dups_usage, 1, true, false },
- { OPTION_NOQUAL, ALIAS_NOQUAL, NULL, noqual_usage, 1, false, false },
- { OPTION_NOSKIP, ALIAS_NOSKIP, NULL, noskip_usage, 1, false, false },
- { OPTION_SHOWID, ALIAS_SHOWID, NULL, showid_usage, 1, false, false },
- { OPTION_SPOTGRP, ALIAS_SPOTGRP, NULL, spotgrp_usage, 1, false, false },
- { OPTION_SEQNAME, ALIAS_SEQNAME, NULL, seqname_usage, 1, false, false },
- { OPTION_MIN_M, NULL, NULL, min_m_usage, 1, true, false },
- { OPTION_MERGE, NULL, NULL, merge_usage, 1, true, false },
- { OPTION_FUNC, ALIAS_FUNC, NULL, func_usage, 1, true, false }
+ /*name, alias, hfkt, usage-help, maxcount, needs value, required */
+ { OPTION_MINMAPQ, ALIAS_MINMAPQ, NULL, minmapq_usage, 1, true, false },
+ { OPTION_DUPS, ALIAS_DUPS, NULL, dups_usage, 1, true, false },
+ { OPTION_NOQUAL, ALIAS_NOQUAL, NULL, noqual_usage, 1, false, false },
+ { OPTION_NOSKIP, ALIAS_NOSKIP, NULL, noskip_usage, 1, false, false },
+ { OPTION_SHOWID, ALIAS_SHOWID, NULL, showid_usage, 1, false, false },
+ { OPTION_SPOTGRP, ALIAS_SPOTGRP, NULL, spotgrp_usage, 1, false, false },
+ { OPTION_DEPTH_PER_SPOTGRP, NULL, NULL, dpgrp_usage, 1, false, false },
+ { OPTION_SEQNAME, ALIAS_SEQNAME, NULL, seqname_usage, 1, false, false },
+ { OPTION_MIN_M, NULL, NULL, min_m_usage, 1, true, false },
+ { OPTION_MERGE, NULL, NULL, merge_usage, 1, true, false },
+ { OPTION_FUNC, ALIAS_FUNC, NULL, func_usage, 1, true, false }
};
/* =========================================================================================== */
@@ -206,7 +218,7 @@ static rc_t get_str_option( const Args *args, const char *name, const char ** re
{
if ( count > 0 )
{
- rc = ArgsOptionValue( args, name, 0, res );
+ rc = ArgsOptionValue( args, name, 0, (const void **)res );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsOptionValue() failed" );
@@ -291,6 +303,9 @@ static rc_t get_pileup_options( Args * args, pileup_options *opts )
rc = get_bool_option( args, OPTION_SPOTGRP, &opts->div_by_spotgrp, false );
if ( rc == 0 )
+ rc = get_bool_option( args, OPTION_DEPTH_PER_SPOTGRP, &opts->depth_per_spotgrp, false );
+
+ if ( rc == 0 )
rc = get_bool_option( args, OPTION_SEQNAME, &opts->use_seq_name, false );
if ( rc == 0 )
@@ -319,7 +334,8 @@ static rc_t get_pileup_options( Args * args, pileup_options *opts )
opts->function = sra_pileup_varcount;
else if ( cmp_pchar( fkt, FUNC_DELETES ) == 0 )
opts->function = sra_pileup_deletes;
-
+ else if ( cmp_pchar( fkt, FUNC_INDELS ) == 0 )
+ opts->function = sra_pileup_indels;
}
}
return rc;
@@ -363,11 +379,13 @@ rc_t CC Usage ( const Args * args )
print_common_helplines();
HelpOptionLine ( ALIAS_MINMAPQ, OPTION_MINMAPQ, "min. mapq", minmapq_usage );
HelpOptionLine ( ALIAS_DUPS, OPTION_DUPS, "dup-mode", dups_usage );
- HelpOptionLine ( ALIAS_SPOTGRP, OPTION_SPOTGRP, "spotgroups-modes", spotgrp_usage );
+ HelpOptionLine ( ALIAS_SPOTGRP, OPTION_SPOTGRP, NULL, spotgrp_usage );
+ HelpOptionLine ( NULL, OPTION_DEPTH_PER_SPOTGRP, NULL, dpgrp_usage );
HelpOptionLine ( ALIAS_SEQNAME, OPTION_SEQNAME, NULL, seqname_usage );
HelpOptionLine ( NULL, OPTION_MIN_M, NULL, min_m_usage );
HelpOptionLine ( NULL, OPTION_MERGE, NULL, merge_usage );
-
+ HelpOptionLine ( ALIAS_NOQUAL, OPTION_NOQUAL, NULL, no_qual_usage );
+
HelpOptionLine ( NULL, "function ref", NULL, func_ref_usage );
HelpOptionLine ( NULL, "function ref-ex", NULL, func_ref_ex_usage );
HelpOptionLine ( NULL, "function count", NULL, func_count_usage );
@@ -376,7 +394,8 @@ rc_t CC Usage ( const Args * args )
HelpOptionLine ( NULL, "function index", NULL, func_index_usage );
HelpOptionLine ( NULL, "function varcount", NULL, func_varcount_usage );
HelpOptionLine ( NULL, "function deletes", NULL, func_deletes_usage );
-
+ HelpOptionLine ( NULL, "function indels", NULL, func_indels_usage );
+
KOutMsg ( "\nGrouping of accessions into artificial spotgroups:\n" );
KOutMsg ( " sra-pileup SRRXXXXXX=a SRRYYYYYY=b SRRZZZZZZ=a\n\n" );
@@ -400,7 +419,7 @@ ver_t CC KAppVersion ( void )
/* =========================================================================================== */
-
+/*
static rc_t CC BufferedWriter ( void* self, const char* buffer, size_t bufsize, size_t* num_writ )
{
rc_t rc = 0;
@@ -494,7 +513,7 @@ static void release_stdout_redirection( void )
}
g_out_writer.org_writer = NULL;
}
-
+*/
static rc_t CC write_to_FILE( void *f, const char *buffer, size_t bytes, size_t *num_writ )
{
@@ -718,21 +737,29 @@ static rc_t walk_ref_position( ReferenceIterator *ref_iter,
static rc_t walk_alignments( ReferenceIterator *ref_iter,
struct dyn_string *line,
+ struct dyn_string *events,
struct dyn_string *qualities,
pileup_options *options )
{
uint32_t depth = 0;
rc_t rc;
+
+ reset_dyn_string( events );
do
{
const PlacementRecord *rec;
rc = ReferenceIteratorNextPlacement ( ref_iter, &rec );
if ( rc == 0 )
- rc = walk_ref_position( ref_iter, rec, line, dyn_string_char( qualities, depth++ ), options );
+ rc = walk_ref_position( ref_iter, rec, events, dyn_string_char( qualities, depth++ ), options );
if ( rc == 0 )
rc = Quitting();
} while ( rc == 0 );
+ if ( options->depth_per_spotgrp )
+ print_2_dyn_string( line, "%d\t", depth );
+
+ add_dyn_string_2_dyn_string( line, events );
+
if ( !options->omit_qualities )
{
uint32_t i;
@@ -751,17 +778,19 @@ static rc_t walk_alignments( ReferenceIterator *ref_iter,
static rc_t walk_spot_groups( ReferenceIterator *ref_iter,
struct dyn_string *line,
+ struct dyn_string *events,
struct dyn_string *qualities,
pileup_options *options )
{
rc_t rc;
+ reset_dyn_string( events );
do
{
rc = ReferenceIteratorNextSpotGroup ( ref_iter, NULL, NULL );
if ( rc == 0 )
add_char_2_dyn_string( line, '\t' );
if ( rc == 0 )
- rc = walk_alignments( ref_iter, line, qualities, options );
+ rc = walk_alignments( ref_iter, line, events, qualities, options );
} while ( rc == 0 );
if ( GetRCState( rc ) == rcDone ) { rc = 0; }
@@ -772,6 +801,7 @@ static rc_t walk_spot_groups( ReferenceIterator *ref_iter,
static rc_t walk_position( ReferenceIterator *ref_iter,
const char * refname,
struct dyn_string *line,
+ struct dyn_string *events,
struct dyn_string *qualities,
pileup_options *options )
{
@@ -795,26 +825,35 @@ static rc_t walk_position( ReferenceIterator *ref_iter,
rc = expand_dyn_string( line, ( 5 * depth ) + 100 );
if ( rc == 0 )
{
- rc = expand_dyn_string( qualities, depth + 100 );
- if ( rc == 0 )
- {
- char c = _4na_to_ascii( base, false );
-
- reset_dyn_string( line );
- rc = print_2_dyn_string( line, "%s\t%u\t%c\t%u", refname, pos + 1, c, depth );
- if ( rc == 0 )
- {
- if ( depth > 0 )
- rc = walk_spot_groups( ref_iter, line, qualities, options );
-
- /* only one KOutMsg() per line... */
- if ( rc == 0 )
- rc = KOutMsg( "%s\n", dyn_string_char( line, 0 ) );
-
- if ( GetRCState( rc ) == rcDone )
- rc = 0;
- }
- }
+ rc = expand_dyn_string( events, ( 5 * depth ) + 100 );
+ if ( rc == 0 )
+ {
+ rc = expand_dyn_string( qualities, depth + 100 );
+ if ( rc == 0 )
+ {
+ char c = _4na_to_ascii( base, false );
+
+ reset_dyn_string( line );
+
+ if ( options->depth_per_spotgrp )
+ rc = print_2_dyn_string( line, "%s\t%u\t%c", refname, pos + 1, c );
+ else
+ rc = print_2_dyn_string( line, "%s\t%u\t%c\t%u", refname, pos + 1, c, depth );
+
+ if ( rc == 0 )
+ {
+ if ( depth > 0 )
+ rc = walk_spot_groups( ref_iter, line, events, qualities, options );
+
+ /* only one KOutMsg() per line... */
+ if ( rc == 0 )
+ rc = KOutMsg( "%s\n", dyn_string_char( line, 0 ) );
+
+ if ( GetRCState( rc ) == rcDone )
+ rc = 0;
+ }
+ }
+ }
}
}
}
@@ -825,6 +864,7 @@ static rc_t walk_position( ReferenceIterator *ref_iter,
static rc_t walk_reference_window( ReferenceIterator *ref_iter,
const char * refname,
struct dyn_string *line,
+ struct dyn_string *events,
struct dyn_string *qualities,
pileup_options *options )
{
@@ -841,7 +881,7 @@ static rc_t walk_reference_window( ReferenceIterator *ref_iter,
}
else
{
- rc = walk_position( ref_iter, refname, line, qualities, options );
+ rc = walk_position( ref_iter, refname, line, events, qualities, options );
}
if ( rc == 0 )
{
@@ -861,31 +901,37 @@ static rc_t walk_reference( ReferenceIterator *ref_iter,
rc_t rc = allocated_dyn_string ( &line, 4096 );
if ( rc == 0 )
{
- struct dyn_string * qualities;
- rc = allocated_dyn_string ( &qualities, 4096 );
- if ( rc == 0 )
- {
- while ( rc == 0 )
- {
- rc = Quitting ();
- if ( rc == 0 )
- {
- INSDC_coord_zero first_pos;
- INSDC_coord_len len;
- rc = ReferenceIteratorNextWindow ( ref_iter, &first_pos, &len );
- if ( rc != 0 )
- {
- if ( GetRCState( rc ) != rcDone )
- {
- LOGERR( klogInt, rc, "ReferenceIteratorNextWindow() failed" );
- }
- }
- else
- rc = walk_reference_window( ref_iter, refname, line, qualities, options );
- }
- }
- free_dyn_string ( qualities );
- }
+ struct dyn_string * events;
+ rc_t rc = allocated_dyn_string ( &events, 4096 );
+ if ( rc == 0 )
+ {
+ struct dyn_string * qualities;
+ rc = allocated_dyn_string ( &qualities, 4096 );
+ if ( rc == 0 )
+ {
+ while ( rc == 0 )
+ {
+ rc = Quitting ();
+ if ( rc == 0 )
+ {
+ INSDC_coord_zero first_pos;
+ INSDC_coord_len len;
+ rc = ReferenceIteratorNextWindow ( ref_iter, &first_pos, &len );
+ if ( rc != 0 )
+ {
+ if ( GetRCState( rc ) != rcDone )
+ {
+ LOGERR( klogInt, rc, "ReferenceIteratorNextWindow() failed" );
+ }
+ }
+ else
+ rc = walk_reference_window( ref_iter, refname, line, events, qualities, options );
+ }
+ }
+ free_dyn_string ( qualities );
+ }
+ free_dyn_string( events );
+ }
free_dyn_string ( line );
}
if ( GetRCState( rc ) == rcDone ) rc = 0;
@@ -1443,15 +1489,15 @@ static rc_t pileup_main( Args * args, pileup_options *options )
case sra_pileup_samtools : options->read_tlen = false;
break;
- case sra_pileup_mismatch : options->omit_qualities = true;
+ case sra_pileup_mismatch : options->omit_qualities = true;
options->read_tlen = false;
break;
- case sra_pileup_index : options->omit_qualities = true;
+ case sra_pileup_index : options->omit_qualities = true;
options->read_tlen = false;
break;
- case sra_pileup_varcount : options->omit_qualities = true;
+ case sra_pileup_varcount : options->omit_qualities = true;
options->read_tlen = false;
break;
}
@@ -1474,6 +1520,7 @@ static rc_t pileup_main( Args * args, pileup_options *options )
if ( empty )
{
Usage ( args );
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcInsufficient );
}
free_ref_regions( ®ions );
}
@@ -1491,6 +1538,7 @@ static rc_t pileup_main( Args * args, pileup_options *options )
case sra_pileup_mismatch : rc = walk_mismatches( arg_ctx.ref_iter, options ); break;
case sra_pileup_index : rc = walk_index( arg_ctx.ref_iter, options ); break;
case sra_pileup_varcount : rc = walk_varcount( arg_ctx.ref_iter, options ); break;
+ case sra_pileup_indels : rc = walk_indels( arg_ctx.ref_iter, options ); break;
default : rc = walk_ref_iter( arg_ctx.ref_iter, options ); break;
}
/* ============================================== */
@@ -1522,7 +1570,7 @@ rc_t CC KMain( int argc, char *argv [] )
{
Args * args;
- KLogHandlerSetStdErr();
+ /* KLogHandlerSetStdErr(); */
rc = ArgsMakeAndHandle( &args, argc, argv, 2,
MyOptions, sizeof MyOptions / sizeof MyOptions [ 0 ],
CommonOptions_ptr(), CommonOptions_count() );
@@ -1535,7 +1583,21 @@ rc_t CC KMain( int argc, char *argv [] )
rc = get_pileup_options( args, &options );
if ( rc == 0 )
{
- options.skiplist = NULL;
+ out_redir redir; /* from out_redir.h */
+ enum out_redir_mode mode;
+
+ options.skiplist = NULL;
+
+ if ( options.cmn.gzip_output )
+ mode = orm_gzip;
+ else if ( options.cmn.bzip_output )
+ mode = orm_bzip2;
+ else
+ mode = orm_uncompressed;
+
+ rc = init_out_redir( &redir, mode, options.cmn.output_file, 32 * 1024 ); /* from out_redir.c */
+
+ /*
if ( options.cmn.output_file != NULL )
{
rc = set_stdout_to( options.cmn.gzip_output,
@@ -1543,7 +1605,8 @@ rc_t CC KMain( int argc, char *argv [] )
options.cmn.output_file,
32 * 1024 );
}
-
+ */
+
if ( rc == 0 )
{
if ( options.function == sra_pileup_report_ref ||
@@ -1567,9 +1630,12 @@ rc_t CC KMain( int argc, char *argv [] )
}
}
+ /*
if ( options.cmn.output_file != NULL )
release_stdout_redirection();
-
+ */
+ release_out_redir( &redir ); /* from out_redir.c */
+
if ( options.skiplist != NULL )
skiplist_release( options.skiplist );
}
diff --git a/tools/sra-pileup/sra-pileup.vers b/tools/sra-pileup/sra-pileup.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-pileup/sra-pileup.vers
+++ b/tools/sra-pileup/sra-pileup.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-pileup/sra-pileup.vers.h b/tools/sra-pileup/sra-pileup.vers.h
index 5329110..4de089c 100644
--- a/tools/sra-pileup/sra-pileup.vers.h
+++ b/tools/sra-pileup/sra-pileup.vers.h
@@ -1 +1 @@
-#define SRA_PILEUP_VERS 0x02040002
+#define SRA_PILEUP_VERS 0x02050007
diff --git a/tools/sra-pileup/walk_debug.c b/tools/sra-pileup/walk_debug.c
index c993362..18d7ae7 100644
--- a/tools/sra-pileup/walk_debug.c
+++ b/tools/sra-pileup/walk_debug.c
@@ -28,24 +28,25 @@
#include "4na_ascii.h"
#include <klib/out.h>
+#include <klib/printf.h>
static rc_t CC walk_debug_enter_ref( walk_data * data )
-{ return KOutMsg( "ENTER REF '%s' ( %u.%u )\n", data->ref_name, data->ref_start, data->ref_len ); }
+{ return KOutMsg( "ENTER REF '%s' ( start:%,u / len:%,u )\n", data->ref_name, data->ref_start, data->ref_len ); }
static rc_t CC walk_debug_exit_ref( walk_data * data )
-{ return KOutMsg( "EXIT REF '%s' ( %u.%u )\n", data->ref_name, data->ref_start, data->ref_len ); }
+{ return KOutMsg( "EXIT REF '%s' ( start: %,u / len:%,u )\n", data->ref_name, data->ref_start, data->ref_len ); }
static rc_t CC walk_debug_enter_ref_window( walk_data * data )
-{ return KOutMsg( " ENTER REF-WINDOW ( %u.%u )\n", data->ref_window_start, data->ref_window_len ); }
+{ return KOutMsg( " ENTER REF-WINDOW ( start: %,u / len:%,u )\n", data->ref_window_start, data->ref_window_len ); }
static rc_t CC walk_debug_exit_ref_window( walk_data * data )
-{ return KOutMsg( " EXIT REF-WINDOW ( %u.%u )\n", data->ref_window_start, data->ref_window_len ); }
+{ return KOutMsg( " EXIT REF-WINDOW ( start: %,u / len:%,u )\n", data->ref_window_start, data->ref_window_len ); }
static rc_t CC walk_debug_enter_ref_pos( walk_data * data )
-{ return KOutMsg( " ENTER REF-POS ( %u / d=%u / '%c' )\n", data->ref_pos, data->depth, _4na_to_ascii( data->ref_base, false ) ); }
+{ return KOutMsg( " ENTER REF-POS ( %u / depth=%u / '%c' )\n", data->ref_pos, data->depth, _4na_to_ascii( data->ref_base, false ) ); }
static rc_t CC walk_debug_exit_ref_pos( walk_data * data )
-{ return KOutMsg( " EXIT REF-POS ( %u / d=%u / '%c' )\n", data->ref_pos, data->depth, _4na_to_ascii( data->ref_base, false ) ); }
+{ return KOutMsg( " EXIT REF-POS ( %u / depth=%u / '%c' )\n", data->ref_pos, data->depth, _4na_to_ascii( data->ref_base, false ) ); }
static rc_t CC walk_debug_enter_sg( walk_data * data )
{ return KOutMsg( " ENTER SPOTGROUP '%s'\n", data->spotgroup ); }
@@ -55,8 +56,66 @@ static rc_t CC walk_debug_exit_sg( walk_data * data )
static rc_t CC walk_debug_placement( walk_data * data )
{
- char c = ( data->xrec->reverse ? 'R' : 'F' );
- return KOutMsg( " PLACEMENT #%lu %c ( TLEN %i )\n", data->rec->id, c, data->xrec->tlen );
+ int32_t state = data->state;
+ bool m = ( ( state & align_iter_match ) == align_iter_match );
+
+ rc_t rc = KOutMsg( " #%.08lu %c %c %c ",
+ data->rec->id,
+ ( data->xrec->reverse ? 'R' : 'F' ),
+ _4na_to_ascii( data->state, data->xrec->reverse ),
+ ( m ? 'M' : '!' ) );
+ if ( rc == 0 )
+ {
+ if ( ( state & align_iter_first ) == align_iter_first )
+ {
+ char c = data->rec->mapq + 33;
+ if ( c > '~' ) { c = '~'; }
+ if ( c < 33 ) { c = 33; }
+ rc = KOutMsg( "^%c ", c );
+ }
+ else if ( ( state & align_iter_last ) == align_iter_last )
+ {
+ rc = KOutMsg( "$ " );
+ }
+ }
+
+ if ( rc == 0 && ( ( state & align_iter_skip ) == align_iter_skip ) )
+ {
+ rc = KOutMsg( "%c ", ( data->xrec->reverse ? '<' : '>' ) );
+ }
+
+ if ( rc == 0 && ( ( state & align_iter_insert ) == align_iter_insert ) )
+ {
+ const INSDC_4na_bin *bases;
+ uint32_t i, n = ReferenceIteratorBasesInserted ( data -> ref_iter, &bases );
+ if ( bases != NULL )
+ {
+ char temp[ 4096 ];
+ if ( n > sizeof temp ) n = sizeof temp;
+ for ( i = 0; i < n; ++i ) temp[ i ] = _4na_to_ascii( bases[ i ], data->xrec->reverse );
+ rc = KOutMsg( "+%u{%.*s} ", n, n, temp );
+ }
+ }
+
+ if ( rc == 0 && ( ( state & align_iter_delete ) == align_iter_delete ) )
+ {
+ INSDC_coord_zero ref_pos;
+ const INSDC_4na_bin *bases;
+ uint32_t i, n = ReferenceIteratorBasesDeleted ( data -> ref_iter, &ref_pos, &bases );
+ if ( bases != NULL )
+ {
+ char temp[ 4096 ];
+ if ( n > sizeof temp ) n = sizeof temp;
+ for ( i = 0; i < n; ++i ) temp[ i ] = _4na_to_ascii( bases[ i ], data->xrec->reverse );
+ rc = KOutMsg( "-%u{%.*s} ", n, n, temp );
+ free( (void *) bases );
+ }
+ }
+
+ if ( rc == 0 )
+ rc = KOutMsg( "( TLEN %,i )\n", data->xrec->tlen );
+
+ return rc;
}
diff --git a/tools/sam-dump/Makefile b/tools/sra-seq-count/Makefile
similarity index 82%
rename from tools/sam-dump/Makefile
rename to tools/sra-seq-count/Makefile
index 03f7d69..6234323 100644
--- a/tools/sam-dump/Makefile
+++ b/tools/sra-seq-count/Makefile
@@ -26,14 +26,13 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/sam-dump
-
+MODULE = tools/sra-seq-count
include $(TOP)/build/Makefile.env
-INT_TOOLS =
+INT_TOOLS = \
EXT_TOOLS = \
- sam-dump
+ sra-seq-count
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -73,31 +72,25 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# sam-dump
+# Common dumper definitions
#
-SAMDUMP_SRC = \
- sam-dump
-
-SAMDUMP_OBJ = \
- $(addsuffix .$(OBJX),$(SAMDUMP_SRC))
-
-SAMDUMP_LIB = \
- -lkapp \
- -lalign-reader \
- $(READONLY_SCHEMA_LIBS) \
- -lsraschema \
- -lvdb \
- -lkdb \
- -lvfs \
- -ssrapath \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lksrch \
- -lkproc \
- -lklib \
- -lm
-
-$(BINDIR)/sam-dump: $(SAMDUMP_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SAMDUMP_LIB)
+TOOL_LIBS = \
+ -skapp \
+ -sngs-c++ \
+ -sncbi-ngs-c++ \
+ -sncbi-vdb-static
+
+#-------------------------------------------------------------------------------
+# fastq-dump
+#
+TOOL_SRC = \
+ main_fkt \
+ sra-seq-count
+
+INCDIRS += -I $(TOP)/ngs/ngs-c++
+
+TOOL_OBJ = \
+ $(addsuffix .$(OBJX),$(TOOL_SRC))
+$(BINDIR)/sra-seq-count: $(TOOL_OBJ)
+ $(LP) --exe --vers $(SRCDIR) -o $@ $^ $(TOOL_LIBS)
diff --git a/tools/sra-seq-count/main_fkt.c b/tools/sra-seq-count/main_fkt.c
new file mode 100644
index 0000000..fddba89
--- /dev/null
+++ b/tools/sra-seq-count/main_fkt.c
@@ -0,0 +1,211 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "sra-seq-count.vers.h"
+
+#include "options.h"
+
+#include <kapp/main.h>
+#include <kapp/args.h>
+
+#include <klib/rc.h>
+#include <klib/out.h>
+#include <klib/text.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <stdlib.h>
+#include <string.h>
+
+#define OPTION_ID_ATTR "id_attr"
+#define OPTION_FEATURE_TYPE "feature_type"
+#define OPTION_MODE "mode"
+
+#define ALIAS_ID_ATTR "i"
+#define ALIAS_FEATURE_TYPE "f"
+#define ALIAS_MODE "m"
+
+#define DEFAULT_ID_ATTR "gene_id"
+#define DEFAULT_FEATURE_TYPE "exon"
+
+static const char * id_attr_usage[] = { "id-attr (default gene_id)", NULL };
+static const char * feature_type_usage[] = { "feature-type (default exon)", NULL };
+static const char * mode_usage[] = { "output-mode (norm, debug)", NULL };
+
+OptDef sra_seq_count_options[] =
+{
+ { OPTION_ID_ATTR, ALIAS_ID_ATTR, NULL, id_attr_usage, 1, true, false },
+ { OPTION_FEATURE_TYPE, ALIAS_FEATURE_TYPE, NULL, feature_type_usage, 1, true, false },
+ { OPTION_MODE, ALIAS_MODE, NULL, mode_usage, 1, true, false }
+};
+
+const char UsageDefaultName[] = "sra-seq-count";
+
+rc_t CC UsageSummary ( const char * progname )
+{
+ return KOutMsg ( "\n"
+ "Usage:\n"
+ " %s <sra-accession> <gtf-file> [options]\n"
+ "\n", progname );
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ rc_t rc;
+
+ if ( args == NULL )
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
+ else
+ rc = ArgsProgram ( args, &fullpath, &progname );
+
+ if ( rc )
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary ( progname );
+
+ KOutMsg ( "Options:\n" );
+ HelpOptionLine ( ALIAS_ID_ATTR, OPTION_ID_ATTR, NULL, id_attr_usage );
+ HelpOptionLine ( ALIAS_FEATURE_TYPE, OPTION_FEATURE_TYPE, NULL, feature_type_usage );
+ HelpOptionLine ( ALIAS_MODE, OPTION_MODE, NULL, mode_usage );
+
+ KOutMsg ( "\n" );
+ HelpOptionsStandard ();
+ HelpVersion ( fullpath, KAppVersion() );
+
+ return rc;
+}
+
+ver_t CC KAppVersion ( void )
+{
+ return SRA_SEQ_COUNT_VERS;
+}
+
+
+static rc_t get_str_option( const Args * args, const char * option_name, const char ** dst, const char * default_value )
+{
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, option_name, &count );
+ if ( ( rc == 0 )&&( count > 0 ) )
+ rc = ArgsOptionValue( args, option_name, 0, (const void **)dst );
+ else
+ (*dst) = string_dup_measure ( default_value, NULL );
+ return rc;
+}
+
+
+static rc_t gather_options( const Args * args, struct sra_seq_count_options * options )
+{
+ rc_t rc;
+
+ memset ( options, 0, sizeof *options );
+ rc = get_str_option( args, OPTION_ID_ATTR, &options->id_attrib, DEFAULT_ID_ATTR );
+ if ( rc == 0 )
+ rc = get_str_option( args, OPTION_FEATURE_TYPE, &options->feature_type, DEFAULT_FEATURE_TYPE );
+ if ( rc == 0 )
+ {
+ const char * mode;
+ rc = get_str_option( args, OPTION_MODE, &mode, "norm" );
+ if ( rc == 0 )
+ {
+ if ( mode[ 0 ] == 'd' && mode[ 1 ] == 'e' && mode[ 2 ] == 'b' && mode[ 3 ] == 'u' &&
+ mode[ 4 ] == 'g' && mode[ 5 ] == 0 )
+ {
+ options -> output_mode = SSC_MODE_DEBUG;
+ }
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = ArgsParamCount( args, &count );
+ if ( rc == 0 )
+ {
+ if ( count == 2 )
+ {
+ rc = ArgsParamValue( args, 0, (const void **)&options->sra_accession );
+ if ( rc == 0 )
+ rc = ArgsParamValue( args, 1, (const void **)&options->gtf_file );
+ if ( rc == 0 )
+ options -> valid = true;
+ }
+ else
+ {
+ UsageSummary ( UsageDefaultName );
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t report_options( const struct sra_seq_count_options * options )
+{
+ rc_t rc = KOutMsg( "accession : %s\n", options->sra_accession );
+ if ( rc == 0 )
+ rc = KOutMsg( "gtf-file : %s\n", options->gtf_file );
+ if ( rc == 0 )
+ rc = KOutMsg( "id-attr : %s\n", options->id_attrib );
+ if ( rc == 0 )
+ rc = KOutMsg( "feature-type : %s\n", options->feature_type );
+ if ( rc == 0 )
+ {
+ switch ( options->output_mode )
+ {
+ case SSC_MODE_NORMAL : rc = KOutMsg( "output-mode : mormal\n" ); break;
+ case SSC_MODE_DEBUG : rc = KOutMsg( "output-mode : debug\n" ); break;
+ default : rc = KOutMsg( "output-mode : unknown\n" ); break;
+ }
+ }
+ return rc;
+}
+
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ Args * args;
+ rc_t rc = ArgsMakeAndHandle ( &args, argc, argv, 1,
+ sra_seq_count_options,
+ ( sizeof sra_seq_count_options / sizeof sra_seq_count_options [ 0 ] ) );
+ if ( rc == 0 )
+ {
+ struct sra_seq_count_options options;
+ rc = gather_options( args, &options );
+ if ( rc == 0 && options.valid )
+ {
+ rc = report_options( &options );
+ if ( rc == 0 )
+ {
+ rc = matching( &options ); /* here we are calling into C++ */
+ }
+ }
+ ArgsWhack ( args );
+ }
+ return rc;
+}
diff --git a/tools/cg-dump/last_rowid.h b/tools/sra-seq-count/options.h
similarity index 77%
rename from tools/cg-dump/last_rowid.h
rename to tools/sra-seq-count/options.h
index 5ae92f0..95d4c57 100644
--- a/tools/cg-dump/last_rowid.h
+++ b/tools/sra-seq-count/options.h
@@ -24,21 +24,33 @@
*
*/
-#ifndef _h_last_rowid_
-#define _h_last_rowid_
+#ifndef _h_sra_seq_count_options_
+#define _h_sra_seq_count_options_
+
+#include <os-native.h>
#ifdef __cplusplus
extern "C" {
#endif
-#include <klib/rc.h>
-#include <klib/text.h>
-#include <klib/container.h>
+#define SSC_MODE_NORMAL 0
+#define SSC_MODE_DEBUG 1
+
+struct sra_seq_count_options
+{
+ const char * sra_accession;
+ const char * gtf_file;
+ const char * id_attrib;
+ const char * feature_type;
+ int output_mode;
+ bool valid;
+};
-rc_t discover_last_rowid( const char * src, int64_t * last_row_id );
+int matching( const struct sra_seq_count_options * options );
#ifdef __cplusplus
}
#endif
+
#endif
diff --git a/tools/sra-seq-count/range.hpp b/tools/sra-seq-count/range.hpp
new file mode 100644
index 0000000..c91498b
--- /dev/null
+++ b/tools/sra-seq-count/range.hpp
@@ -0,0 +1,307 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _hpp_seq_ranges_
+#define _hpp_seq_ranges_
+
+#include <list>
+
+namespace seq_ranges {
+
+enum e_range_relation {
+ rr_before,
+ rr_begin,
+ rr_span,
+ rr_inside,
+ rr_end,
+ rr_after
+};
+
+
+/* -----------------------------------------------------------------------
+ range-relations:
+
+ A |-------------|
+ B |-----|
+ A.range_rel( B ) = rr_before
+
+ A |-------------|
+ B |-----|
+ A.range_rel( B ) = rr_begin
+
+ A |-------------|
+ B |-------------------|
+ A.range_rel( B ) = rr_span
+
+ A |-------------|
+ B |-----|
+ A.range_rel( B ) = rr_inside
+
+ A |-------------|
+ B |-----|
+ A.range_rel( B ) = rr_end
+
+ A |-------------|
+ B |-----|
+ A.range_rel( B ) = rr_after
+
+ ----------------------------------------------------------------------- */
+
+class range
+{
+ private :
+ long start;
+ long end;
+
+ public :
+ range( void ) : start( 0 ), end( 0 ) { }
+ range( long start_, long end_ ) : start( start_ ), end( end_ ) { }
+ range( const range &other ) : start( other.start ), end( other.end ) { }
+
+ range( const std::string &start_, const std::string &end_ ) : start( 0 ), end( 0 )
+ {
+ bool valid = ( convert_value( start_, start ) && convert_value( end_, end ) );
+ if ( !valid ) set( 0, 0 );
+ }
+
+ range& operator= ( const range &other )
+ {
+ if ( this == &other ) return *this;
+ start = other.start;
+ end = other.end;
+ return *this;
+ }
+
+ void set( long start_, long end_ ) { start = start_; end = end_; }
+
+ static bool convert_value( const std::string value_, long &value )
+ {
+ bool res = true;
+ std::istringstream ( value_ ) >> value;
+ if ( value < 1 )
+ res = false;
+ return res;
+ }
+
+ /* does this range end before the other range starts ? */
+ bool ends_before( const range &other ) const { return ( end < other.start ); }
+
+ /* does this range start after the other range ends ? */
+ bool starts_after( const range &other ) const { return ( start > other.end ); }
+
+ bool intersect( const range &other ) const
+ {
+ e_range_relation rr = range_relation( other );
+ return ( rr == rr_begin || rr == rr_span || rr == rr_inside || rr == rr_end );
+ }
+
+ bool intersect( const long value ) const
+ { return ( value >= start && value <= end ); }
+
+ void include( const range &other )
+ {
+ if ( other.start < start ) start = other.start;
+ if ( other.end > end ) end = other.end;
+ }
+
+ bool merge( const range &other )
+ {
+ bool res = intersect( other );
+ if ( res ) include( other );
+ return res;
+ }
+
+ enum e_range_relation range_relation( const range &other ) const
+ {
+ enum e_range_relation res;
+ if ( other.start < start )
+ {
+ if ( other.end < start )
+ res = rr_before;
+ else if ( other.end <= end )
+ res = rr_begin;
+ else
+ res = rr_span;
+ }
+ else
+ {
+ if ( other.start <= end )
+ {
+ if ( other.end <= end )
+ res = rr_inside;
+ else
+ res = rr_end;
+ }
+ else
+ res = rr_after;
+ }
+
+ return res;
+ }
+
+ long get_start( void ) const { return start; }
+ long get_end( void ) const { return end; }
+ bool empty( void ) const { return ( start == 0 && end == 0 ); }
+
+ void print( std::ostream &stream ) const { stream << "[ " << start << " .. " << end << " ]"; }
+
+ friend std::ostream& operator<< ( std::ostream &stream, const range &other )
+ {
+ other.print( stream );
+ return stream;
+ }
+
+ static std::string range_relation_2_str( enum e_range_relation rr )
+ {
+ switch ( rr )
+ {
+ case rr_before : return std::string( "before" ); break;
+ case rr_begin : return std::string( "begin" ); break;
+ case rr_span : return std::string( "span" ); break;
+ case rr_inside : return std::string( "inside" ); break;
+ case rr_end : return std::string( "end" ); break;
+ case rr_after : return std::string( "after" ); break;
+ }
+ return std::string( "unknown" );
+ }
+
+};
+
+
+bool compare_ranges ( const range * first, const range * second )
+{
+ return ( first -> get_start() < second -> get_start() );
+}
+
+
+struct ranges_relation
+{
+ bool has_inside;
+ bool has_partial;
+
+ void clear( void ) { has_inside = has_partial = false; }
+
+ void print( std::ostream &stream ) const { stream << "inside: " << has_inside << ", partial: " << has_partial; }
+
+ friend std::ostream& operator<< ( std::ostream &stream, const ranges_relation &other )
+ {
+ other.print( stream );
+ return stream;
+ }
+};
+
+
+class ranges
+{
+ private :
+ std::list< range * > range_list;
+ long count;
+
+ public :
+ ranges( void ) : count( 0 ) {}
+ ~ranges( void ) { clear(); }
+
+ void clear( void )
+ {
+ while ( !range_list.empty() )
+ {
+ range * r = range_list.front();
+ range_list.pop_front();
+ if ( r != NULL ) delete r;
+ }
+ count = 0;
+ }
+
+ void add( range * r ){ if ( r != NULL ) { range_list.push_back( r ); count++; } }
+ void add( const range &r ){ add( new range( r ) ); }
+
+ void merge( const range &r1 )
+ {
+ bool merged = false;
+ std::list< range * >::iterator it;
+ for ( it = range_list.begin(); it != range_list.end() && !merged; ++it )
+ {
+ range * r = *it;
+ if ( r != NULL )
+ merged = r -> merge( r1 );
+ }
+ if ( !merged ) add( r1 );
+ }
+
+ void sort( void ) { range_list.sort( compare_ranges ); }
+ long get_count( void ) { return count; }
+
+ void compare_sample( const ranges &sample, ranges_relation &res ) const
+ {
+ res.clear();
+ bool done = false;
+
+ // we take each range of the sample and compare it against each range of self
+ std::list< range * >::const_iterator sample_it;
+ for ( sample_it = sample.range_list.begin(); sample_it != sample.range_list.end() && !done; ++sample_it )
+ {
+ const range * sample_range = *sample_it;
+ std::list< range * >::const_iterator pattern_it;
+ for ( pattern_it = range_list.begin(); pattern_it != range_list.end() && !done; ++pattern_it )
+ {
+ const range * pattern_range( *pattern_it );
+ enum e_range_relation rr = pattern_range -> range_relation( *sample_range );
+ switch( rr )
+ {
+ case rr_before : break;
+ case rr_begin : res.has_partial = true; break;
+ case rr_span : res.has_partial = true; break;
+ case rr_inside : res.has_inside = true; break;
+ case rr_end : res.has_partial = true; break;
+ case rr_after : break;
+ }
+ done = ( res.has_partial && res.has_inside );
+ }
+ }
+ }
+
+ void print( std::ostream &stream ) const
+ {
+ std::list< range * >::const_iterator it;
+ for ( it = range_list.begin(); it != range_list.end(); ++it )
+ {
+ range * r = *it;
+ if ( r != NULL ) stream << *r << " ";
+ }
+ }
+
+ friend std::ostream& operator<< ( std::ostream &stream, const ranges &other )
+ {
+ other.print( stream );
+ return stream;
+ }
+
+};
+
+}; // namespace seq_ranges
+
+#endif // _hpp_seq_ranges_
+
diff --git a/tools/sra-seq-count/sra-seq-count.cpp b/tools/sra-seq-count/sra-seq-count.cpp
new file mode 100644
index 0000000..ccf1faa
--- /dev/null
+++ b/tools/sra-seq-count/sra-seq-count.cpp
@@ -0,0 +1,688 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <ngs/ncbi/NGS.hpp>
+#include <ngs/ErrorMsg.hpp>
+#include <ngs/ReadCollection.hpp>
+#include <ngs/AlignmentIterator.hpp>
+#include <ngs/Alignment.hpp>
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <stdexcept>
+#include <vector>
+#include <list>
+#include <algorithm>
+
+#include "options.h"
+#include "range.hpp"
+
+using namespace seq_ranges;
+
+
+struct feature_range
+{
+ const std::string ref_name;
+ const std::string feature_id;
+ const char strand;
+ range ft_range;
+
+ feature_range( const std::string &ref_name_, const std::string &feature_id_,
+ const long start_, const long end_, const char strand_ )
+ : ref_name( ref_name_ ), feature_id( feature_id_ ), strand( strand_ ), ft_range( start_, end_ )
+ { }
+};
+
+
+class feature
+{
+ private :
+ const std::string ref_name;
+ const std::string feature_id;
+ const char strand;
+ ranges feature_ranges;
+ range outer;
+ long counter;
+
+ public :
+ feature( const feature_range &fr ) : ref_name( fr.ref_name ), feature_id( fr.feature_id ),
+ strand( fr.strand ), outer( fr.ft_range ), counter( 0 )
+ {
+ feature_ranges.add( fr.ft_range );
+ }
+
+ bool add( const feature_range &fr )
+ {
+ bool res = ( feature_id == fr.feature_id );
+ if ( res )
+ {
+ feature_ranges.merge( fr.ft_range );
+ outer.include( fr.ft_range );
+ }
+ return res;
+ }
+
+ void debug_report ( void )
+ {
+ std::cout << "FEATURE: " << feature_id << " ( refname: " << ref_name << " ) strand = '" << strand << "' " << outer << std::endl;
+ std::cout << feature_ranges << std::endl;
+ std::cout << "counter : " << counter << std::endl;
+ std::cout << std::endl;
+ }
+
+ void report ( int output_mode )
+ {
+ if ( counter > 0 )
+ {
+ if ( output_mode == SSC_MODE_NORMAL )
+ {
+ std::cout << feature_id << "\t" << counter << std::endl;
+ }
+ else
+ {
+ std::cout << ref_name << "." << outer << "(" << feature_ranges.get_count() << ") "
+ << feature_id << "\t" << counter << std::endl;
+ }
+ }
+ }
+
+ void sort_ranges( void ) { feature_ranges.sort(); }
+ void get_ref_name( std::string &s ) { s = ref_name; }
+ void get_outer_range( range &r ) { r = outer; }
+
+ /* does this feature end before the given range */
+ bool ends_before( const range &r ) const { return outer.ends_before( r ); }
+ bool is_ref( const std::string &r_name ) { return ( ref_name == r_name ); }
+ long start( void ) { return outer.get_start(); }
+
+ void check( const range &r ) { if ( outer.intersect( r ) ) counter++; }
+};
+
+
+void trim( std::string &s )
+{
+ bool to_trim = false;
+ std::size_t pos = 0;
+ std::size_t len = s.length();
+ while ( s[ pos ] == ' ' && len > 0 )
+ {
+ pos++;
+ len--;
+ to_trim = true;
+ }
+ while ( s[ pos + len - 1 ] == ' ' )
+ {
+ len--;
+ to_trim = true;
+ }
+ if ( to_trim )
+ s = s.substr( pos, len );
+}
+
+
+void split_attr( const std::string &attr, const std::string &idattr, std::string &feature_id )
+{
+ std::size_t from = 0;
+ std::size_t pos = 0;
+ bool done = false;
+ feature_id.clear();
+ while ( !done )
+ {
+ pos = attr.find( ";", from );
+ done = ( pos == std::string::npos );
+ std::string sub_attr = attr.substr( from, pos - from );
+ std::size_t sub_pos = sub_attr.find( "\"" );
+ if ( sub_pos != std::string::npos )
+ {
+ std::string sub_name = sub_attr.substr( 0, sub_pos - 1 );
+ trim( sub_name );
+ if ( sub_name == idattr )
+ {
+ feature_id = sub_attr.substr( sub_pos + 1, sub_attr.length() - ( sub_pos + 2 ) );
+ done = true;
+ }
+ }
+ if ( !done ) from = pos + 1;
+ }
+}
+
+
+bool split_line( const std::string &line, const std::string feature_type, const std::string &idattr,
+ std::string &ref_name, std::string &feature_id,
+ long &start, long &end, char &strand )
+{
+ bool res = ( ( line.length() > 0 )&&( line[ 0 ] != '#' ) );
+ if ( res )
+ {
+ int field = 0;
+ std::string attr;
+ std::size_t from = 0;
+ std::size_t pos = 0;
+ start = end = 0;
+ long value;
+ bool done = false;
+ while ( !done )
+ {
+ pos = line.find( "\t", from );
+ done = ( pos == std::string::npos );
+ switch( field++ )
+ {
+ case 0 : ref_name = line.substr( from, pos - from ); break;
+ case 2 : if ( feature_type != line.substr( from, pos - from ) )
+ {
+ done = true;
+ res = false;
+ }
+ break;
+ case 3 : if ( range::convert_value( line.substr( from, pos - from ), value ) ) start = value; break;
+ case 4 : if ( range::convert_value( line.substr( from, pos - from ), value ) ) end = value; break;
+ case 6 : if ( ( pos - from ) > 0 ) strand = line[ from ]; else strand = '?'; break;
+ case 8 : attr = line.substr( from, pos - from ); break;
+ }
+ if ( !done ) from = pos + 1;
+ }
+
+ if ( res ) split_attr( attr, idattr, feature_id );
+ }
+ return res;
+}
+
+
+class gtf_iter
+{
+ private :
+ const std::string filename;
+ const std::string idattr;
+ const std::string feature_type;
+ feature_range * stored_feature_range;
+ feature * stored_next_feature;
+ std::ifstream inputstream;
+
+ feature_range * read_next_feature_range( void )
+ {
+ feature_range * res = NULL;
+ if ( inputstream.good() )
+ {
+ std::string line;
+ if ( std::getline( inputstream, line ) )
+ {
+ std::string ref_name, feature_id;
+ long start, end;
+ char strand;
+ if ( split_line( line, feature_type, idattr,
+ ref_name, feature_id, start, end, strand ) )
+ {
+ res = new feature_range( ref_name, feature_id, start, end, strand );
+ }
+ }
+ }
+ return res;
+ }
+
+ feature_range * next_feature_range( void )
+ {
+ feature_range * res = NULL;
+ if ( stored_feature_range != NULL )
+ {
+ res = stored_feature_range;
+ stored_feature_range = NULL;
+ }
+ else while ( inputstream.good() && res == NULL )
+ {
+ /* because read_next_feature_range can return NULL if the line contains the wrong feature_type */
+ res = read_next_feature_range();
+ }
+ return res;
+ }
+
+ feature * read_next_feature( void )
+ {
+ feature * res = NULL;
+ feature_range * fr = next_feature_range();
+ if ( fr != NULL )
+ {
+ res = new feature( *fr );
+ delete fr;
+ bool done = ( res == NULL );
+ while ( !done )
+ {
+ fr = next_feature_range();
+ done = ( fr == NULL );
+ if ( !done )
+ {
+ done = !( res -> add( *fr ) );
+ if ( done )
+ stored_feature_range = fr;
+ else
+ delete fr;
+ }
+ }
+ if ( done ) res->sort_ranges();
+ }
+ return res;
+ }
+
+ public :
+ gtf_iter( const char * filename_, const std::string &idattr_, const std::string &feature_type_ ) :
+ filename( filename_ ), idattr( idattr_ ), feature_type( feature_type_ ),
+ stored_feature_range( NULL ), inputstream( filename_ )
+ {
+ stored_next_feature = read_next_feature();
+ }
+
+ bool good( void ) { return ( stored_next_feature != NULL ); }
+
+ feature * next_feature( void )
+ {
+ feature * res = stored_next_feature;
+ if ( res != NULL )
+ stored_next_feature = read_next_feature();
+ return res;
+ }
+
+ feature * forward_to_next_ref( const std::string & ref )
+ {
+ feature * res = NULL;
+ bool done = false;
+ while ( !done )
+ {
+ res = next_feature();
+ done = ( res == NULL );
+ if ( !done )
+ {
+ if ( res -> is_ref( ref ) )
+ delete res;
+ else
+ done = true;
+ }
+ }
+ return res;
+ }
+
+ long peek_next_feature_start( void )
+ {
+ if ( stored_next_feature != NULL )
+ return stored_next_feature -> start();
+ else
+ return 0;
+ }
+
+ bool peek_next_feature_range( range &r )
+ {
+ bool res = ( stored_next_feature != NULL );
+ if ( res ) stored_next_feature -> get_outer_range( r );
+ return res;
+ }
+
+ bool peek_next_feature_is_ref( const std::string &ref_name )
+ {
+ if ( stored_next_feature != NULL )
+ return stored_next_feature -> is_ref( ref_name );
+ else
+ return false;
+ }
+};
+
+
+class global_counter
+{
+ private:
+ long refs;
+ long total_alignments;
+ long no_feature;
+ long ambiguous;
+ long too_low_qual;
+ long not_aligned;
+ long not_unique;
+
+ public:
+ global_counter( void ) : refs( 0 ), total_alignments( 0 ), no_feature( 0 ), ambiguous( 0 ),
+ too_low_qual( 0 ), not_aligned( 0 ), not_unique( 0 ) {}
+
+ void inc_refs( void ) { refs++; }
+ void inc_total_alignments( void ) { total_alignments++; }
+ void inc_no_feature( void ) { no_feature++; }
+ void inc_ambiguous( void ) { ambiguous++; }
+ void inc_too_low_qual( void ) { too_low_qual++; }
+ void inc_not_aligned( void ) { not_aligned++; }
+ void inc_not_unique( void ) { not_unique++; }
+
+ void report( void )
+ {
+ std::cout << std::endl;
+ std::cout << "__no_feature\t" << no_feature << std::endl;
+ std::cout << "__ambiguous\t" << ambiguous << std::endl;
+ std::cout << "__too_low_aQual\t" << too_low_qual << std::endl;
+ std::cout << "__not_aligned\t" << not_aligned << std::endl;
+ std::cout << "__alignment_not_unique\t" << not_unique << std::endl;
+
+ std::cout << std::endl << "total\t" << total_alignments << std::endl;
+ std::cout << "refs\t" << refs << std::endl << std::endl;
+ }
+};
+
+
+class feature_list
+{
+ private :
+ const std::string &ref_name;
+ int output_mode;
+ std::list< feature * > flist;
+
+ void clear( void )
+ {
+ while ( !flist.empty() )
+ {
+ feature * f = flist.front();
+ flist.pop_front();
+ if ( f != NULL ) delete f;
+ }
+ }
+
+ void add( feature * f )
+ {
+ flist.push_back( f );
+ }
+
+ public :
+ feature_list( const std::string &ref_name_, int output_mode_, feature * f )
+ : ref_name( ref_name_ ), output_mode( output_mode_ ) { add( f ); }
+ ~feature_list( void ) { clear(); }
+
+ bool empty( void ) { return flist.empty(); }
+
+ void remove_all_features_before( const range &al_range )
+ {
+ bool done = flist.empty();
+ while ( !done )
+ {
+ feature * f = flist.front();
+ if ( f -> ends_before( al_range ) )
+ {
+ flist.pop_front();
+ f -> report( output_mode );
+ done = flist.empty();
+ delete f;
+ }
+ else
+ done = true;
+ }
+ }
+
+ void load_all_features_starting_inside( const range &al_range, gtf_iter >f_it )
+ {
+ bool done = false;
+ while ( !done )
+ {
+ done = !gtf_it.peek_next_feature_is_ref( ref_name );
+ if ( !done )
+ {
+ range r;
+ done = !gtf_it.peek_next_feature_range( r );
+ if ( !done )
+ {
+ if ( r.ends_before( al_range ) )
+ {
+ /* we can skip this feature! */
+ feature * f = gtf_it.next_feature();
+ delete f;
+ }
+ else if ( r.starts_after( al_range ) )
+ {
+ /* we are done, this feature if for alignments that come later */
+ done = true;
+ }
+ else
+ {
+ feature * f = gtf_it.next_feature();
+ if ( f != NULL ) add( f );
+ }
+ }
+ }
+ }
+ }
+
+ void count_matches_between_alignment_and_features( const range &al_range )
+ {
+ std::list< feature * >::iterator it;
+ for ( it = flist.begin(); it != flist.end(); ++it )
+ {
+ feature * f = *it;
+ if ( f != NULL )
+ {
+ f -> check( al_range );
+ }
+ }
+ }
+};
+
+
+class iter_window
+{
+ private :
+ ngs::ReadCollection &run;
+ gtf_iter >f_it;
+ int output_mode;
+ global_counter counter;
+
+ public :
+ iter_window( ngs::ReadCollection &run_, gtf_iter >f_it_, int output_mode_ )
+ : run( run_ ), gtf_it( gtf_it_ ), output_mode( output_mode_ ) {}
+
+ void walk( void )
+ {
+ feature * f = gtf_it.next_feature();
+ while ( f != NULL )
+ {
+ /* get the name of the reference of the fist feature in here... */
+ std::string ref_name;
+ f -> get_ref_name( ref_name );
+ try
+ {
+ ngs::Reference ref = run.getReference ( ref_name );
+ try
+ {
+ bool done = false;
+ feature_list flist( ref_name, output_mode, f );
+ ngs::AlignmentIterator al_iter = ref.getAlignments( ngs::Alignment::primaryAlignment );
+
+ std::cout << std::endl << "processing ref: " << ref_name << std::endl;
+ std::cout << "-------------------------------------------" << std::endl;
+ counter.inc_refs();
+
+ /* now walk all alignments of this al_iter */
+ while ( al_iter.nextAlignment() && !done )
+ {
+ int64_t pos = al_iter.getAlignmentPosition() + 1; /* al_iter returns 0-based ! */
+ uint64_t len = al_iter.getAlignmentLength();
+
+ const range al_range( pos, pos + len - 1 );
+
+ /* remove and report all features that end before this alignment */
+ flist.remove_all_features_before( al_range );
+ flist.load_all_features_starting_inside( al_range, gtf_it );
+
+ /* now we can count matches between the feature-list and this alignment */
+ flist.count_matches_between_alignment_and_features( al_range );
+
+ /* look if we have to cancel the loop if the feature-list is empty
+ and we have no more features in this reference */
+ if ( flist.empty() )
+ {
+ done = ( gtf_it.peek_next_feature_start() == 0 );
+ if ( !done )
+ done = !gtf_it.peek_next_feature_is_ref( ref_name );
+ }
+ counter.inc_total_alignments();
+ }
+ f = gtf_it.forward_to_next_ref( ref_name );
+ }
+ catch ( ngs::ErrorMsg e )
+ {
+ std::cout << "error in ref " << ref_name << " : " << e.what() << std::endl;
+ f = NULL;
+ }
+ }
+ catch ( ngs::ErrorMsg e )
+ {
+ f = gtf_it.forward_to_next_ref( ref_name );
+ }
+
+ }
+ counter.report();
+ }
+};
+
+
+int matching( const struct sra_seq_count_options * options )
+{
+ int res = 0;
+
+ std::string id_attr( options->id_attrib );
+ std::string feature_type( options->feature_type );
+
+ /* create the ngs-iterator, which delivers references and alignments */
+ try
+ {
+ ngs::ReadCollection run ( ncbi::NGS::openReadCollection( options->sra_accession ) );
+
+ /* create the gtf-iterator, which delivers gtf-features */
+ gtf_iter gtf_it( options->gtf_file, id_attr, feature_type );
+
+ /* create a iterator window that walks both iterators to count alignments for the features */
+ iter_window window( run, gtf_it, options->output_mode );
+
+ /* walk the window... */
+ window.walk();
+ }
+ catch ( ngs::ErrorMsg e )
+ {
+ std::cout << "cannot open " << options->sra_accession << " because " << e.what() << std::endl;
+ }
+ return res;
+}
+
+
+int list_refs_in_gtf( const char * gtf )
+{
+ int res = 0;
+ long feature_count = 0;
+ long feature_ref_count = 0;
+ std::string ref_name;
+ std::string id_attr( "gene_id" );
+ std::string feature_type( "exon" );
+
+ /* create the gtf-iterator, which delivers gtf-features */
+ gtf_iter gtf_it( gtf, id_attr, feature_type );
+
+ feature * f = gtf_it.next_feature();
+ while ( f != NULL )
+ {
+ /* count feature */
+ feature_count ++;
+
+ if ( f -> is_ref( ref_name ) )
+ {
+ feature_ref_count++;
+ }
+ else
+ {
+ if ( feature_ref_count > 0 )
+ {
+ std::cout << ref_name << "\t" << feature_ref_count << std::endl;
+ feature_ref_count = 0;
+ }
+ f -> get_ref_name( ref_name );
+ }
+
+ /* dispose feature */
+ delete f;
+ f = gtf_it.next_feature();
+ }
+ std::cout << feature_count << " features" << std::endl;
+
+ return res;
+}
+
+
+void print_rr( const range &A, const range &B )
+{
+ std::cout << A << " --- " << B << " : " << range::range_relation_2_str( A.range_relation( B ) ) << std::endl;
+}
+
+void rr_test( void )
+{
+ /*
+ print_rr( range( 10, 20 ), range( 5, 7 ) );
+ print_rr( range( 10, 20 ), range( 5, 10 ) );
+
+ print_rr( range( 10, 20 ), range( 5, 12 ) );
+ print_rr( range( 10, 20 ), range( 5, 20 ) );
+
+ print_rr( range( 10, 20 ), range( 5, 22 ) );
+
+ print_rr( range( 10, 20 ), range( 10, 20 ) );
+ print_rr( range( 10, 20 ), range( 5, 25 ) );
+
+ print_rr( range( 10, 20 ), range( 15, 25 ) );
+ print_rr( range( 10, 20 ), range( 20, 25 ) );
+
+ print_rr( range( 10, 20 ), range( 25, 35 ) );
+ */
+
+ ranges rl;
+ rl.add( range( 10, 20 ) );
+ rl.add( range( 70, 80 ) );
+ rl.add( range( 30, 40 ) );
+ rl.add( range( 50, 60 ) );
+ rl.merge( range( 18, 22 ) );
+
+ rl.sort();
+
+ std::cout << rl << std::endl;
+}
+
+/*
+int main( int argc, const char* argv[] )
+{
+ int res = 0;
+ if ( argc < 3 )
+ {
+ std::cout << "\nusage : gene_count sra_accession genes.gtf\n";
+ res = -1;
+ }
+ else
+ {
+ matching( argv[ 1 ], argv[ 2 ] );
+ // list_refs_in_gtf( argv[ 2 ] );
+ // rr_test();
+ }
+
+ return res;
+}
+*/
diff --git a/tools/sra-seq-count/sra-seq-count.vers b/tools/sra-seq-count/sra-seq-count.vers
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/tools/sra-seq-count/sra-seq-count.vers
@@ -0,0 +1 @@
+1.0.0
diff --git a/tools/sra-sort/col-pair.c b/tools/sra-sort/col-pair.c
index 6cedbb3..3bde514 100644
--- a/tools/sra-sort/col-pair.c
+++ b/tools/sra-sort/col-pair.c
@@ -395,7 +395,7 @@ void SimpleColumnWriterWriteStatic ( SimpleColumnWriter *self, const ctx_t *ctx,
else if ( count > 1 )
{
/* setting it small since threshold is detected in CommitRow, but execused on CloseRow */
- uint64_t cnt = ( count < 0x20000000U ) ? count : 0x20000000U;
+ uint64_t cnt = ( count < 0x10000000U ) ? count : 0x10000000U;
rc = VCursorRepeatRow ( self -> curs, cnt - 1 );
if ( rc != 0 )
SYSTEM_ERROR ( rc, "VCursorRepeatRow failed for column '%s'", self -> full_spec );
diff --git a/tools/sra-sort/idx-mapping.c b/tools/sra-sort/idx-mapping.c
index d1434aa..5552415 100644
--- a/tools/sra-sort/idx-mapping.c
+++ b/tools/sra-sort/idx-mapping.c
@@ -39,24 +39,20 @@ FILE_ENTRY ( idx-mapping );
#if USE_OLD_KSORT
-int CC IdxMappingCmpOld ( const void *a, const void *b, void *data )
+int64_t CC IdxMappingCmpOld ( const void *a, const void *b, void *data )
{
const IdxMapping *ap = a;
const IdxMapping *bp = b;
- if ( ap -> old_id < bp -> old_id )
- return -1;
- return ap -> old_id > bp -> old_id;
+ return ap -> old_id < bp -> old_id ? -1 : ap -> old_id > bp -> old_id;
}
-int CC IdxMappingCmpNew ( const void *a, const void *b, void *data )
+int64_t CC IdxMappingCmpNew ( const void *a, const void *b, void *data )
{
const IdxMapping *ap = a;
const IdxMapping *bp = b;
- if ( ap -> new_id < bp -> new_id )
- return -1;
- return ap -> new_id > bp -> new_id;
+ return ap -> new_id < bp -> new_id ? -1 : ap -> new_id > bp -> new_id;
}
#else /* USE_OLD_KSORT */
diff --git a/tools/sra-sort/idx-mapping.h b/tools/sra-sort/idx-mapping.h
index 7691526..bbd031b 100644
--- a/tools/sra-sort/idx-mapping.h
+++ b/tools/sra-sort/idx-mapping.h
@@ -45,8 +45,8 @@ struct IdxMapping
#if USE_OLD_KSORT
/* ksort callbacks */
-int CC IdxMappingCmpOld ( const void *a, const void *b, void *data );
-int CC IdxMappingCmpNew ( const void *a, const void *b, void *data );
+int64_t CC IdxMappingCmpOld ( const void *a, const void *b, void *data );
+int64_t CC IdxMappingCmpNew ( const void *a, const void *b, void *data );
#else
diff --git a/tools/sra-sort/meta-pair.c b/tools/sra-sort/meta-pair.c
index ac37903..a067fe2 100644
--- a/tools/sra-sort/meta-pair.c
+++ b/tools/sra-sort/meta-pair.c
@@ -209,7 +209,7 @@ void copy_meta_node_value ( const KMDataNode *src, const ctx_t *ctx, KMDataNode
FUNC_ENTRY ( ctx );
rc_t rc;
- char buff [ 4096 ];
+ char buff [ 32*1024 ];
size_t num_read, remaining;
rc = KMDataNodeRead ( src, 0, buff, sizeof buff, & num_read, & remaining );
@@ -249,7 +249,7 @@ void copy_meta_node_attr ( const KMDataNode *src, const ctx_t *ctx, KMDataNode *
rc_t rc;
size_t num_read;
- char buff [ 4096 ];
+ char buff [ 32*1024 ];
rc = KMDataNodeReadAttr ( src, name, buff, sizeof buff, & num_read );
if ( rc != 0 )
@@ -293,7 +293,7 @@ const char *filter_node_name ( const char *name, size_t nsize, const char **excl
static
void copy_meta_node ( const KMDataNode *src, const ctx_t *ctx, KMDataNode *dst,
- char path [ 4096 ], size_t psize, const char *owner_spec, const char **exclude )
+ char path [ 32*1024 ], size_t psize, const char *owner_spec, const char **exclude )
{
FUNC_ENTRY ( ctx );
@@ -341,7 +341,7 @@ void copy_meta_node ( const KMDataNode *src, const ctx_t *ctx, KMDataNode *dst,
rc = KNamelistCount ( names, & count );
if ( rc != 0 )
INTERNAL_ERROR ( rc, "KNamelistCount failed" );
- else if ( psize + 1 == 4096 )
+ else if ( psize + 1 == 32*1024 )
{
rc = RC ( rcExe, rcNode, rcCopying, rcBuffer, rcInsufficient );
ERROR ( rc, "metadata path too long" );
@@ -369,7 +369,7 @@ void copy_meta_node ( const KMDataNode *src, const ctx_t *ctx, KMDataNode *dst,
}
nsize = string_size ( name );
- if ( psize + nsize + 1 >= 4096 )
+ if ( psize + nsize + 1 >= 32*1024 )
{
WARN ( "skipping node '%.*s/%s' - path too long", ( uint32_t ) psize, path, name );
continue;
@@ -429,7 +429,7 @@ void MetaPairCopy ( MetaPair *self, const ctx_t *ctx, const char *owner_spec, co
FUNC_ENTRY ( ctx );
rc_t rc;
- char path [ 4096 ];
+ char path [ 32*1024 ];
const KMDataNode *src;
STATUS ( 3, "copying '%s' metadata", owner_spec );
diff --git a/tools/sra-sort/poslen-col-pair.c b/tools/sra-sort/poslen-col-pair.c
index 452b134..4d9ce46 100644
--- a/tools/sra-sort/poslen-col-pair.c
+++ b/tools/sra-sort/poslen-col-pair.c
@@ -431,7 +431,7 @@ void PoslenColWriterWriteStatic ( PoslenColWriter *self, const ctx_t *ctx,
SYSTEM_ERROR ( rc, "VCursorCommit failed for column '%s'", self -> full_spec );
else if ( count > 1 )
{
- uint64_t cnt = (count < 0x20000000U ) ? count : 0x20000000U;
+ uint64_t cnt = (count < 0x10000000U ) ? count : 0x10000000U;
rc = VCursorRepeatRow ( self -> curs, cnt - 1 );
if ( rc != 0 )
SYSTEM_ERROR ( rc, "VCursorRepeatRow failed for column '%s'", self -> full_spec );
diff --git a/tools/sra-sort/ref-alignid-col.c b/tools/sra-sort/ref-alignid-col.c
index 59570df..89e4b46 100644
--- a/tools/sra-sort/ref-alignid-col.c
+++ b/tools/sra-sort/ref-alignid-col.c
@@ -63,7 +63,7 @@ struct IdPosLen
#if USE_OLD_KSORT
static
-int CC IdPosLenCmpPos ( const void *a, const void *b, void *data )
+int64_t CC IdPosLenCmpPos ( const void *a, const void *b, void *data )
{
const IdPosLen *ap = a;
const IdPosLen *bp = b;
@@ -72,20 +72,17 @@ int CC IdPosLenCmpPos ( const void *a, const void *b, void *data )
return -1;
if ( ap -> poslen > bp -> poslen )
return 1;
- if ( ap -> id < bp -> id )
- return -1;
- return ap -> id > bp -> id;
+
+ return ap -> id < bp -> id ? -1 : ap -> id > bp -> id;
}
static
-int CC cmp_int64_t ( const void *a, const void *b, void *data )
+int64_t CC cmp_int64_t ( const void *a, const void *b, void *data )
{
const int64_t *ap = a;
const int64_t *bp = b;
- if ( * ap < * bp )
- return -1;
- return * ap > * bp;
+ return * ap < * bp ? -1 : * ap > * bp;
}
#else
diff --git a/tools/sra-sort/sra-sort.c b/tools/sra-sort/sra-sort.c
index 0c72310..674f7ce 100644
--- a/tools/sra-sort/sra-sort.c
+++ b/tools/sra-sort/sra-sort.c
@@ -217,7 +217,7 @@ const char *ArgsGetOptStr ( Args *self, const ctx_t *ctx, const char *optname, u
rc = ArgsOptionCount ( self, optname, count );
if ( rc == 0 && * count != 0 )
{
- rc = ArgsOptionValue ( self, optname, 0, & val );
+ rc = ArgsOptionValue ( self, optname, 0, (const void **)& val );
if ( rc != 0 )
INTERNAL_ERROR ( rc, "failed to retrieve '%s' parameter", optname );
}
@@ -239,7 +239,7 @@ uint64_t ArgsGetOptU64 ( Args *self, const ctx_t *ctx, const char *optname, uint
if ( rc == 0 && * count != 0 )
{
const char *str;
- rc = ArgsOptionValue ( self, optname, 0, & str );
+ rc = ArgsOptionValue ( self, optname, 0, (const void **)& str );
if ( rc != 0 )
INTERNAL_ERROR ( rc, "failed to retrieve '%s' parameter", optname );
else
@@ -586,7 +586,7 @@ rc_t CC KMain ( int argc, char *argv [] )
{
/* check type of last parameter */
const char *dst;
- rc = ArgsParamValue ( args, count - 1, & dst );
+ rc = ArgsParamValue ( args, count - 1, (const void **)& dst );
if ( rc != 0 )
ERROR ( rc, "ArgsParamValue [ %u ] failed", count - 1 );
else
@@ -644,7 +644,7 @@ rc_t CC KMain ( int argc, char *argv [] )
for ( issue_divider_line = false, i = 0; i < count - 1; issue_divider_line = true, ++ i )
{
- rc = ArgsParamValue ( args, i, & tp . src_path );
+ rc = ArgsParamValue ( args, i, (const void **)& tp . src_path );
if ( rc != 0 )
ERROR ( rc, "ArgParamValue [ %u ] failed", i );
else
@@ -700,7 +700,7 @@ rc_t CC KMain ( int argc, char *argv [] )
else
{
tp . dst_path = dst;
- rc = ArgsParamValue ( args, 0, & tp . src_path );
+ rc = ArgsParamValue ( args, 0, (const void **)& tp . src_path );
if ( rc != 0 )
ERROR ( rc, "ArgParamValue [ 0 ] failed" );
else
diff --git a/tools/sra-sort/sra-sort.vers b/tools/sra-sort/sra-sort.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-sort/sra-sort.vers
+++ b/tools/sra-sort/sra-sort.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-stat/Makefile b/tools/sra-stat/Makefile
index 65224db..a7611be 100644
--- a/tools/sra-stat/Makefile
+++ b/tools/sra-stat/Makefile
@@ -33,7 +33,7 @@ include $(TOP)/build/Makefile.env
INT_TOOLS =
EXT_TOOLS = \
- sra-stat
+ sra-stat \
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -77,7 +77,8 @@ clean: stdclean
# sra statistics
#
SRASTAT_SRC = \
- sra-stat
+ sra \
+ sra-stat \
SRASTAT_OBJ = \
$(addsuffix .$(OBJX),$(SRASTAT_SRC))
@@ -85,7 +86,8 @@ SRASTAT_OBJ = \
SRASTAT_LIB = \
-lkapp \
-sncbi-vdb \
- -lm
+ -sload \
+ -lm \
$(BINDIR)/sra-stat: $(SRASTAT_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SRASTAT_LIB)
diff --git a/tools/sra-stat/sra-stat.c b/tools/sra-stat/sra-stat.c
index fa517e9..dedebe5 100644
--- a/tools/sra-stat/sra-stat.c
+++ b/tools/sra-stat/sra-stat.c
@@ -26,35 +26,47 @@
#include "sra-stat.vers.h"
-#include <kapp/main.h>
+#include "sra-stat.h" /* VTableMakeSingleFileArchive_ */
-#include <sra/wsradb.h>
-#include <sra/sradb-priv.h>
-#include <sra/types.h>
+#include <insdc/sra.h> /* SRA_READ_TYPE_BIOLOGICAL */
-#include <vdb/dependencies.h> /* VDBDependencies */
-#include <vdb/database.h> /* VDatabaseRelease */
-#include <vdb/table.h> /* VTableRelease */
-#include <vdb/cursor.h> /* VCursor */
+#include <kapp/log-xml.h> /* XMLLogger */
+#include <kapp/main.h>
+#include <kapp/progressbar.h> /* KLoadProgressbar */
#include <kdb/database.h> /* KDatabase */
-#include <kdb/table.h>
+#include <kdb/kdb-priv.h>
+#include <kdb/manager.h> /* KDBManager */
#include <kdb/meta.h> /* KMetadata */
#include <kdb/namelist.h> /* KMDataNodeListChild */
-#include <kdb/kdb-priv.h>
+#include <kdb/table.h>
-#include <kfs/directory.h>
-#include <kfs/file.h>
+#include <kfs/directory.h> /* KDirectory */
+#include <kfs/file.h> /* KFile */
-#include <klib/sort.h> /* ksort */
#include <klib/checksum.h>
#include <klib/container.h>
+#include <klib/debug.h> /* DBGMSG */
+#include <klib/log.h> /* LOGERR */
#include <klib/namelist.h>
-#include <klib/printf.h>
-#include <klib/log.h>
#include <klib/out.h>
-#include <klib/debug.h> /* DBGMSG */
+#include <klib/printf.h>
#include <klib/rc.h>
+#include <klib/sort.h> /* ksort */
+
+#include <sra/sraschema.h> /* VDBManagerMakeSRASchema */
+
+#include <vdb/cursor.h> /* VCursor */
+#include <vdb/database.h> /* VDatabaseRelease */
+#include <vdb/dependencies.h> /* VDBDependencies */
+#include <vdb/manager.h> /* VDBManager */
+#include <vdb/schema.h> /* VSchema */
+#include <vdb/table.h> /* VTableRelease */
+#include <vdb/vdb-priv.h> /* VDBManagerGetKDBManagerRead */
+
+#include <vfs/manager.h> /* VFSManager */
+#include <vfs/path.h> /* VPath */
+#include <vfs/resolver.h> /* VResolver */
#include <os-native.h> /* strtok_r on Windows */
@@ -64,7 +76,7 @@
#include <string.h>
#include <time.h>
-#include <stdio.h> /* stderr */
+/* #include <stdio.h> */ /* stderr */
#define DISP_RC(rc, msg) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, msg))
@@ -81,10 +93,10 @@
#define MAX_NREADS 2*1024
-/*********** XMLLogger_Encode : copied from kapp/log-xml.c (-lload) ***********/
+/********** _XMLLogger_Encode : copied from kapp/log-xml.c (-lload) ***********/
static
-rc_t CC XMLLogger_Encode(const char* src,
+rc_t CC _XMLLogger_Encode(const char* src,
char* aDst, size_t dst_sz, size_t *num_writ)
{
rc_t rc = 0;
@@ -152,21 +164,28 @@ rc_t BAM_HEADER_RG_copy(BAM_HEADER_RG* dest, const BAM_HEADER_RG* src)
{
assert(dest && !dest->ID && !dest->LB && !dest->SM
&& src && src ->ID && src ->LB && src ->SM);
+
dest->ID = strdup(src->ID);
dest->LB = strdup(src->LB);
dest->SM = strdup(src->SM);
+
if (!dest->ID || !dest->LB || !dest->SM) {
return RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
+
return 0;
}
+
static void BAM_HEADER_RG_free(BAM_HEADER_RG* rg) {
assert(rg);
+
free(rg->ID);
free(rg->LB);
free(rg->SM);
+
rg->ID = rg->LB = rg->SM = NULL;
}
+
static void BAM_HEADER_RG_print_xml(const BAM_HEADER_RG* rg) {
rc_t rc = 0;
char library[1024] = "";
@@ -174,28 +193,34 @@ static void BAM_HEADER_RG_print_xml(const BAM_HEADER_RG* rg) {
char* l = library;
char* s = sample;
size_t num_writ = 0;
+
assert(rg);
+
if (rg->LB) {
- rc = XMLLogger_Encode(rg->LB, library, sizeof library, &num_writ);
- if (rc) {
+ rc = _XMLLogger_Encode(rg->LB, library, sizeof library, &num_writ);
+ if (rc != 0) {
l = rg->LB;
PLOGMSG(klogWarn, (klogWarn, "Library value is too long: "
"XML-safe conversion was disabled for @RG\tID:$(ID)\tLB:$(LB)"
"ID=%s,LB=%s", rg->ID, rg->LB));
}
+
OUTMSG((" library=\"%s\"", l ? l : ""));
}
+
if (rg->SM) {
- rc = XMLLogger_Encode(rg->SM, sample, sizeof sample, &num_writ);
- if (rc) {
+ rc = _XMLLogger_Encode(rg->SM, sample, sizeof sample, &num_writ);
+ if (rc != 0) {
s = rg->SM;
PLOGMSG(klogWarn, (klogWarn, "Sample value is too long: "
"XML-safe conversion was disabled for @RG\tID:$(ID)\tSM:$(SM)"
"ID=%s,SM=%s", rg->ID, rg->SM));
}
+
OUTMSG((" sample=\"%s\"" , s ? s : ""));
}
}
+
typedef struct SraStats {
BSTNode n;
char spot_group[1024]; /* SPOT_GROUP Column */
@@ -229,7 +254,8 @@ typedef struct Statistics2 {
double diff_sq_sum;
} Statistics2;
static
-void Statistics2Init(Statistics2* self, double sum, spotid_t count) {
+void Statistics2Init(Statistics2* self, double sum, int64_t count)
+{
assert(self);
if (count) {
@@ -248,8 +274,7 @@ static void Statistics2Add(Statistics2* self, double value) {
self->diff_sq_sum += diff * diff;
}
-static
-void Statistics2Print(const Statistics2* selfs,
+static void Statistics2Print(const Statistics2* selfs,
uint32_t nreads, const char* indent,
/* the same as in <Statistics: just to make <Statistics> and <Statistics2>
@@ -284,117 +309,145 @@ void Statistics2Print(const Statistics2* selfs,
typedef struct {
uint64_t cnt[5];
bool CS_NATIVE;
- const SRAColumn *col;
+ const VCursor *curs;
+ uint32_t idx;
+
bool finalized;
} Bases;
-static
-void BasesInit(Bases *self, const SRATable *tbl)
-{
+static rc_t BasesInit(Bases *self, const VTable *vtbl) {
rc_t rc = 0;
- const char name[] = "CS_NATIVE";
- const SRAColumn *col = NULL;
- const void *base = NULL;
- bitsz_t boff = ~0;
- bitsz_t row_bits = ~0;
assert(self);
memset(self, 0, sizeof *self);
- self->CS_NATIVE = true;
-
- rc = SRATableOpenColumnRead(tbl, &col, name, NULL);
- if (rc != 0) {
- PLOGERR(klogInt, (klogErr, rc,
- "while SRATableOpenColumnRead($(name))", "name=%s", name));
- }
if (rc == 0) {
- rc = SRAColumnRead(col, 1, &base, &boff, &row_bits);
- if (rc != 0) {
- PLOGERR(klogInt, (klogErr, rc,
- "while SRAColumnRead($(name))", "name=%s", name));
+ const char name[] = "CS_NATIVE";
+ const void *base = NULL;
+
+ const VCursor *curs = NULL;
+ uint32_t idx = 0;
+
+ self->CS_NATIVE = true;
+
+ rc = VTableCreateCursorRead(vtbl, &curs);
+ DISP_RC(rc, "Cannot VTableCreateCursorRead");
+
+ if (rc == 0) {
+ rc = VCursorAddColumn(curs, &idx, "%s", name);
+ DISP_RC(rc, "Cannot VCursorAddColumn(CS_NATIVE)");
}
- if (boff != 0 || row_bits != 8) {
- rc = RC(rcExe, rcColumn, rcReading, rcData, rcInvalid);
- PLOGERR(klogInt, (klogErr, rc,
- "invalid boff or row_bits while SRAColumnRead($(name))",
- "name=%s", name));
+
+ if (rc == 0) {
+ rc = VCursorOpen(curs);
+ DISP_RC(rc, "Cannot VCursorOpen(CS_NATIVE)");
}
- }
- if (rc == 0) {
- self->CS_NATIVE = *((bool*)base);
- }
+ if (rc == 0) {
+ bitsz_t boff = ~0;
+ bitsz_t row_bits = ~0;
+
+ uint32_t elem_bits = 0, elem_off = 0, elem_cnt = 0;
+ rc = VCursorCellDataDirect(curs, 1, idx,
+ &elem_bits, &base, &elem_off, &elem_cnt);
+ boff = elem_off * elem_bits;
+ row_bits = elem_cnt * elem_bits;
+
+ if (boff != 0 || row_bits != 8) {
+ rc = RC(rcExe, rcColumn, rcReading, rcData, rcInvalid);
+ PLOGERR(klogInt, (klogErr, rc, "invalid boff or row_bits "
+ "while VCursorCellDataDirect($(name))", "name=%s", name));
+ }
+ }
+
+ if (rc == 0) {
+ self->CS_NATIVE = *((bool*)base);
+ }
- RELEASE(SRAColumn, col);
+ RELEASE(VCursor, curs);
+ }
{
const char *name = self->CS_NATIVE ? "CSREAD" : "READ";
const char *datatype
= self->CS_NATIVE ? "INSDC:x2cs:bin" : "INSDC:x2na:bin";
- rc = SRATableOpenColumnRead(tbl, &self->col, name, datatype);
- if (rc != 0) {
- PLOGERR(klogInt, (klogErr, rc,
- "while SRATableOpenColumnRead($(name), $(type))",
- "name=%s,type=%s", name, datatype));
+ rc = VTableCreateCursorRead(vtbl, &self->curs);
+ DISP_RC(rc, "Cannot VTableCreateCursorRead");
+ if (rc == 0) {
+ rc = VCursorAddColumn(self->curs,
+ &self->idx, "(%s)%s", datatype, name);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc,
+ "Cannot VCursorAddColumn(($(type)),$(name)",
+ "type=%s,name=%s", datatype, name));
+ }
+ }
+ if (rc == 0) {
+ rc = VCursorOpen(self->curs);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc,
+ "Cannot VCursorOpen(($(type)),$(name)))",
+ "type=%s,name=%s", datatype, name));
+ }
}
}
+
+ return rc;
}
-static
-void BasesFinalize(Bases *self)
-{
+static void BasesFinalize(Bases *self) {
assert(self);
- if (self->col == NULL) {
+ if (self->curs == NULL) {
LOGMSG(klogInfo, "Bases statistics will not be printed : "
- "READ column was not opened during BasesFinalize()");
+ "READ cursor was not opened during BasesFinalize()");
return;
}
self->finalized = true;
}
-static
-rc_t BasesRelease(Bases *self)
-{
+static rc_t BasesRelease(Bases *self) {
rc_t rc = 0;
assert(self);
- RELEASE(SRAColumn, self->col);
+ RELEASE(VCursor , self->curs);
return rc;
}
-static
-void BasesAdd(Bases *self, spotid_t spotid)
-{
+static void BasesAdd(Bases *self, int64_t spotid) {
rc_t rc = 0;
const void *base = NULL;
- bitsz_t boff = ~0;
bitsz_t row_bits = ~0;
bitsz_t i = ~0;
const unsigned char *bases = NULL;
assert(self);
- if (self->col == NULL) {
+ if (self->curs == NULL) {
return;
}
- rc = SRAColumnRead(self->col, spotid, &base, &boff, &row_bits);
- if (rc != 0) {
- PLOGERR(klogInt, (klogErr, rc,
- "while SRAColumnRead(READ, $(type))",
- "type=%s", self->CS_NATIVE ? "CS_NATIVE" : "not CS_NATIVE"));
- BasesRelease(self);
- return;
+ {
+ uint32_t elem_bits = 0, elem_off = 0, elem_cnt = 0;
+ rc = VCursorCellDataDirect(self->curs, spotid, self->idx,
+ &elem_bits, &base, &elem_off, &elem_cnt);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogErr, rc,
+ "while VCursorCellDataDirect(READ, $(type))",
+ "type=%s", self->CS_NATIVE ? "CS_NATIVE" : "not CS_NATIVE"));
+ BasesRelease(self);
+ return;
+ }
+
+ row_bits = elem_cnt * elem_bits;
}
if ((row_bits % 8) != 0) {
rc = RC(rcExe, rcColumn, rcReading, rcData, rcInvalid);
PLOGERR(klogInt, (klogErr, rc, "Invalid row_bits '$(row_bits) "
- "while SRAColumnRead(READ, $(type), spotid=$(spotid))",
+ "while VCursorCellDataDirect(READ, $(type), spotid=$(spotid))",
"row_bits=%lu,type=%s,spotid=%lu",
row_bits, self->CS_NATIVE ? "CS_NATIVE" : "not CS_NATIVE", spotid));
BasesRelease(self);
@@ -408,8 +461,8 @@ void BasesAdd(Bases *self, spotid_t spotid)
if (base > 4) {
rc = RC(rcExe, rcColumn, rcReading, rcData, rcInvalid);
PLOGERR(klogInt, (klogErr, rc,
- "Invalid READ column value '$(base) "
- "while SRAColumnRead($(type), spotid=$(spotid), index=$(i))",
+ "Invalid READ column value '$(base) while VCursorCellDataDirect"
+ "($(type), spotid=$(spotid), index=$(i))",
"base=%d,type=%s,spotid=%lu,index=%lu",
base, self->CS_NATIVE ? "CS_NATIVE" : "not CS_NATIVE",
spotid, i));
@@ -450,10 +503,12 @@ static rc_t BasesPrint(const Bases *self,
OUTMSG(("%s<%s cs_native=\"%s\" count=\"%lu\">\n",
indent, tag, self->CS_NATIVE ? "true" : "false", base_count));
+
for (i = 0; i < 5; ++i) {
OUTMSG(("%s <Base value=\"%c\" count=\"%lu\"/>\n",
indent, name[i], self->cnt[i]));
}
+
OUTMSG(("%s</%s>\n", indent, tag));
return rc;
@@ -633,8 +688,7 @@ void print_double_or_int(const char* name, double val, bool variable) {
}
}
-static
-void StatisticsPrint(const Statistics* selfs,
+static void StatisticsPrint(const Statistics* selfs,
uint32_t nreads, const char* indent)
{
int i = ~0;
@@ -655,6 +709,7 @@ void StatisticsPrint(const Statistics* selfs,
OUTMSG(("/>\n"));
}
}
+
static rc_t StatisticsDiff(const Statistics* ss,
const Statistics2* ss2, uint32_t nreads)
{
@@ -701,9 +756,8 @@ static rc_t StatisticsDiff(const Statistics* ss,
return rc;
}
-static
-rc_t SraStatsTotalPrintStatistics(const SraStatsTotal* self,
- const char* indent, bool test)
+static rc_t SraStatsTotalPrintStatistics(
+ const SraStatsTotal* self, const char* indent, bool test)
{
rc_t rc = 0;
@@ -737,12 +791,15 @@ typedef struct srastat_parms {
bool printMeta;
bool quick; /* quick mode: stats from meta */
bool skip_members; /* not to print spot_group statistics */
+ bool progress; /* show progress */
bool skip_alignment; /* not to print alignment info */
bool print_arcinfo;
bool statistics; /* calculate average and stdev */
bool test; /* test stdev */
- spotid_t start, stop;
+ const XMLLogger *logger;
+
+ int64_t start, stop;
bool hasSPOT_GROUP;
bool variableReadLength;
@@ -831,9 +888,7 @@ typedef struct Ctx {
QualityStats quality;
TableCounts tables;
} Ctx;
-
typedef rc_t (CC * RG_callback)(const BAM_HEADER_RG* rg, const void* data);
-
static
rc_t CC meta_RG_callback(const BAM_HEADER_RG* rg, const void* data)
{
@@ -866,9 +921,7 @@ rc_t CC meta_RG_callback(const BAM_HEADER_RG* rg, const void* data)
return rc;
}
-static
-int CC srastats_cmp ( const void *item, const BSTNode *n )
-{
+static int64_t CC srastats_cmp(const void *item, const BSTNode *n) {
const char *sg = item;
const SraStats *ss = ( const SraStats* ) n;
@@ -897,24 +950,22 @@ rc_t CC tree_RG_callback(const BAM_HEADER_RG* rg, const void* data)
return rc;
}
-static
-int CC QualityCmp(const void* s1, const void* s2, void *data) {
+static int64_t CC QualityCmp(const void* s1, const void* s2, void *data) {
const Quality* q1 = s1;
const Quality* q2 = s2;
assert(q1 && q2);
- return q1->value < q2->value ? -1 : q1->value == q2->value ? 0 : 1;
+ return (int64_t)q1->value - (int64_t)q2->value;
}
-static
-int CC CountsCmp(const void* v1, const void* v2, void *data) {
+static int64_t CC CountsCmp(const void* v1, const void* v2, void *data) {
const Counts* e1 = v1;
const Counts* e2 = v2;
assert(e1 && e2 && e1->tableName && e2->tableName);
return strcmp(e1->tableName, e2->tableName);
}
-static
-rc_t QualityParse(Quality* self, const KMDataNode* node, const char* name)
+static rc_t QualityParse(Quality* self,
+ const KMDataNode* node, const char* name)
{
rc_t rc = 0;
const char start[] = "PHRED_";
@@ -950,8 +1001,7 @@ rc_t QualityParse(Quality* self, const KMDataNode* node, const char* name)
return rc;
}
-static
-rc_t QualityStatsRead1(QualityStats* self,
+static rc_t QualityStatsRead1(QualityStats* self,
const KMDataNode* parent, const char* name)
{
const KMDataNode* node = NULL;
@@ -996,9 +1046,7 @@ rc_t QualityStatsRead1(QualityStats* self,
return rc;
}
-static
-rc_t QualityStatsRead(QualityStats* self, const KMetadata* meta)
-{
+static rc_t QualityStatsRead(QualityStats* self, const KMetadata* meta) {
rc_t rc = 0;
const char name[] = "STATS/QUALITY";
const KMDataNode* node = NULL;
@@ -1042,7 +1090,9 @@ rc_t QualityStatsRead(QualityStats* self, const KMetadata* meta)
}
RELEASE(KNamelist, names);
}
+
RELEASE(KMDataNode, node);
+
return 0;
}
@@ -1486,19 +1536,70 @@ rc_t CC fileSizeVisitor(const KDirectory* dir,
return rc;
}
-static
-rc_t get_arc_info(const SRAMgr* mgr, const char* path,
- const SRATable* tbl, ArcInfo* arc_info)
+static rc_t GetTableModDate(const VDBManager *mgr,
+ KTime_t *mtime, const char *spec)
+{
+ VFSManager *vfs = NULL;
+ VResolver *resolver = NULL;
+ VPath *accession = NULL;
+ const VPath *tblpath = NULL;
+ const KDBManager *kmgr = NULL;
+ char aPath[4096] = "";
+ const char *path = aPath;
+ rc_t rc = VFSManagerMake(&vfs);
+ DISP_RC(rc, "VFSManagerMake");
+ assert(mtime);
+ *mtime = 0;
+ if (rc == 0) {
+ rc = VFSManagerGetResolver(vfs, &resolver);
+ DISP_RC(rc, "VFSManagerGetResolver");
+ }
+ if (rc == 0) {
+ rc = VFSManagerMakePath(vfs, &accession, spec);
+ DISP_RC(rc, "VFSManagerMakePath");
+ }
+ if (rc == 0) {
+ assert(accession);
+ if (VPathIsAccessionOrOID(accession)) {
+ rc = VResolverLocal(resolver, accession, &tblpath);
+ DISP_RC(rc, "VResolverLocal");
+ if (rc == 0) {
+ //size_t size;
+ rc = VPathReadPath(tblpath, aPath, sizeof aPath, NULL);//& size)
+ path = aPath;
+ }
+ }
+ else {
+ path = spec;
+ }
+ }
+ if (rc == 0) {
+ rc = VDBManagerGetKDBManagerRead(mgr, &kmgr);
+ DISP_RC(rc, "VDBManagerGetKDBManagerRead");
+ }
+ if (rc == 0) {
+ rc = KDBManagerGetTableModDate(kmgr, mtime, "%s", path);
+ }
+ RELEASE(KDBManager, kmgr);
+ RELEASE(VPath, tblpath);
+ RELEASE(VPath, accession);
+ RELEASE(VResolver, resolver);
+ RELEASE(VFSManager, vfs);
+ return rc;
+}
+
+static rc_t get_arc_info(const char *path, ArcInfo *arc_info,
+ const VDBManager *vmgr, const VTable *vtbl)
{
rc_t rc = 0;
memset(arc_info, 0, sizeof(*arc_info));
- if( (rc = SRAMgrGetTableModDate(mgr, &arc_info->timestamp, "%s", path)) == 0 ) {
+ if ((rc = GetTableModDate(vmgr, &arc_info->timestamp, path)) == 0 ) {
uint32_t i;
for(i = 0; i < sizeof(arc_info->i) / sizeof(arc_info->i[0]); i++) {
const KFile* kfile;
arc_info->i[i].tag = i == 0 ? "sra" : "lite.sra";
- if( (rc = SRATableMakeSingleFileArchive(tbl, &kfile, i == 1, NULL)) == 0 ) {
+ if ((rc = VTableMakeSingleFileArchive(vtbl, &kfile, i == 1)) == 0) {
MD5State md5;
uint64_t pos = 0;
uint8_t buffer[256 * 1024];
@@ -1510,8 +1611,14 @@ rc_t get_arc_info(const SRAMgr* mgr, const char* path,
MD5StateAppend(&md5, buffer, num_read);
pos += num_read;
}
+ rc = Quitting();
+ if (rc != 0) {
+ LOGMSG(klogWarn, "Interrupted");
+ }
} while(rc == 0 && num_read != 0);
- if( rc == 0 && (rc = KFileSize(kfile, &arc_info->i[i].size)) == 0 ) {
+ if (rc == 0 &&
+ (rc = KFileSize(kfile, &arc_info->i[i].size)) == 0)
+ {
uint8_t digest[16];
MD5StateFinish(&md5, digest);
for(pos = 0, x = 0; rc == 0 && pos < sizeof(digest); pos++) {
@@ -1526,19 +1633,16 @@ rc_t get_arc_info(const SRAMgr* mgr, const char* path,
return rc;
}
-static
-rc_t get_size(const SRATable* tbl, SraSizeStats* sizes)
-{
+static rc_t get_size( SraSizeStats* sizes, const VTable *vtbl) {
rc_t rc = 0;
const KDatabase* kDb = NULL;
const KTable* kTbl = NULL;
const KDirectory* dir = NULL;
- assert(tbl && sizes);
-
- rc = SRATableGetKTableRead(tbl, &kTbl);
- DISP_RC(rc, "while calling SRATableGetKTableRead");
+ assert(vtbl && sizes);
+ rc = VTableOpenKTableRead(vtbl, &kTbl);
+ DISP_RC(rc, "while calling VTableOpenKTableRead");
if (rc == 0) {
rc = KTableOpenParentRead(kTbl, &kDb);
@@ -1753,7 +1857,7 @@ rc_t get_load_info(const KMetadata* meta, SraMeta* info)
static
rc_t readStatsMetaNode(const KMDataNode* parent, const char* parentName,
- const char* name, uint64_t* result, bool quick, bool optional)
+ const char* name, uint64_t* result, bool quick, bool optional)
{
rc_t rc = 0;
@@ -2404,18 +2508,24 @@ void CC bst_whack_free ( BSTNode *n, void *ignore )
}
static
-int CC srastats_sort ( const BSTNode *item, const BSTNode *n )
+int64_t CC srastats_sort ( const BSTNode *item, const BSTNode *n )
{
const SraStats *ss = ( const SraStats* ) item;
return srastats_cmp(ss->spot_group,n);
}
-static
-rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
- BSTree* tr, SraStatsTotal* total)
+static bool columnUndefined(rc_t rc) {
+ return rc == SILENT_RC(rcVDB, rcCursor, rcOpening , rcColumn, rcUndefined)
+ || rc == SILENT_RC(rcVDB, rcCursor, rcUpdating, rcColumn, rcNotFound );
+}
+
+static rc_t sra_stat(srastat_parms* pb, BSTree* tr,
+ SraStatsTotal* total, const VTable *vtbl)
{
rc_t rc = 0;
+ const VCursor *curs = NULL;
+
/* const char CMP_READ [] = "CMP_READ"; */
const char PRIMARY_ALIGNMENT_ID[] = "PRIMARY_ALIGNMENT_ID";
const char RD_FILTER [] = "RD_FILTER";
@@ -2423,16 +2533,16 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
const char READ_TYPE [] = "READ_TYPE";
const char SPOT_GROUP[] = "SPOT_GROUP";
-/* const SRAColumn* cCMP_READ = NULL; */
- const SRAColumn* cPRIMARY_ALIGNMENT_ID = NULL;
- const SRAColumn* cRD_FILTER = NULL;
- const SRAColumn* cREAD_LEN = NULL;
- const SRAColumn* cREAD_TYPE = NULL;
- const SRAColumn* cSPOT_GROUP = NULL;
+ uint32_t idxPRIMARY_ALIGNMENT_ID = 0;
+ uint32_t idxRD_FILTER = 0;
+ uint32_t idxREAD_LEN = 0;
+ uint32_t idxREAD_TYPE = 0;
+ uint32_t idxSPOT_GROUP = 0;
int g_nreads = 0;
- spotid_t n_spots = 0;
- spotid_t start = pb->start, stop = pb->stop;
+ int64_t n_spots = 0;
+ int64_t start = 0;
+ int64_t stop = 0;
/* filled with dREAD_LEN[i] for (spotid == start);
used to check fixedReadLength */
@@ -2441,35 +2551,51 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
memset(g_totalREAD_LEN, 0, sizeof g_totalREAD_LEN);
memset(g_nonZeroLenReads, 0, sizeof g_nonZeroLenReads);
- assert(pb && tbl && tr && total);
+ rc = VTableCreateCursorRead(vtbl, &curs);
+ DISP_RC(rc, "Cannot VTableCreateCursorRead");
+
+ if (rc == 0) {
+ rc = VCursorPermitPostOpenAdd(curs);
+ DISP_RC(rc, "Cannot VCursorPermitPostOpenAdd");
+ }
+
+ if (rc == 0) {
+ rc = VCursorOpen(curs);
+ DISP_RC(rc, "Cannot VCursorOpen");
+ }
+
+ assert(pb && vtbl && tr && total);
if (rc == 0) {
const char* name = READ_LEN;
- rc = SRATableOpenColumnRead(tbl, &cREAD_LEN, name, vdb_uint32_t);
- DISP_RC2(rc, name, "while calling SRATableOpenColumnRead");
+ rc = VCursorAddColumn(curs, &idxREAD_LEN, "%s", name);
+ DISP_RC2(rc, name, "while calling VCursorAddColumn");
}
if (rc == 0) {
const char* name = READ_TYPE;
- rc = SRATableOpenColumnRead(tbl, &cREAD_TYPE, name, sra_read_type_t);
- DISP_RC2(rc, name, "while calling SRATableOpenColumnRead");
+ rc = VCursorAddColumn(curs, &idxREAD_TYPE, "%s", name);
+ DISP_RC2(rc, name, "while calling VCursorAddColumn");
}
if (rc == 0) {
{
const char* name = SPOT_GROUP;
- rc = SRATableOpenColumnRead(tbl, &cSPOT_GROUP, name, vdb_ascii_t);
- if (GetRCState(rc) == rcNotFound)
- { rc = 0; }
- DISP_RC2(rc, name, "while calling SRATableOpenColumnRead");
+ rc = VCursorAddColumn(curs, &idxSPOT_GROUP, "%s", name);
+ if (columnUndefined(rc)) {
+ idxSPOT_GROUP = 0;
+ rc = 0;
+ }
+ DISP_RC2(rc, name, "while calling VCursorAddColumn");
}
if (rc == 0) {
if (rc == 0) {
const char* name = RD_FILTER;
- rc = SRATableOpenColumnRead
- (tbl, &cRD_FILTER, name, sra_read_filter_t);
- if (GetRCState(rc) == rcNotFound)
- { rc = 0; }
- DISP_RC2(rc, name, "while calling SRATableOpenColumnRead");
+ rc = VCursorAddColumn(curs, &idxRD_FILTER, "%s", name);
+ if (columnUndefined(rc)) {
+ idxRD_FILTER = 0;
+ rc = 0;
+ }
+ DISP_RC2(rc, name, "while calling VCursorAddColumn");
}
/* if (rc == 0) {
const char* name = CMP_READ;
@@ -2481,21 +2607,26 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
} */
if (rc == 0) {
const char* name = PRIMARY_ALIGNMENT_ID;
- rc = SRATableOpenColumnRead
- (tbl, &cPRIMARY_ALIGNMENT_ID, name, "I64");
- if (GetRCState(rc) == rcNotFound)
- { rc = 0; }
- DISP_RC2(rc, name, "while calling SRATableOpenColumnRead");
+ rc = VCursorAddColumn(curs, &idxPRIMARY_ALIGNMENT_ID,
+ "%s", name);
+ if (columnUndefined(rc)) {
+ idxPRIMARY_ALIGNMENT_ID = 0;
+ rc = 0;
+ }
+ DISP_RC2(rc, name, "while calling VCursorAddColumn");
}
if (rc == 0) {
- spotid_t spotid;
+ int64_t first = 0;
+ uint64_t count = 0;
+ int64_t spotid;
pb->hasSPOT_GROUP = 0;
- rc = SRATableMaxSpotId(tbl, &spotid);
- DISP_RC(rc, "failed to read max spot id");
+ rc = VCursorIdRange(curs, 0, &first, &count);
+ DISP_RC(rc, "VCursorIdRange() failed");
if (rc == 0) {
- BasesInit(&total->bases_count, tbl);
+ rc = BasesInit(&total->bases_count, vtbl);
}
if (rc == 0) {
+ const KLoadProgressbar *pr = NULL;
bool bad_read_filter = false;
bool fixedNReads = true;
bool fixedReadLength = true;
@@ -2504,14 +2635,27 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
memset(g_dREAD_LEN, 0, sizeof g_dREAD_LEN);
- if (start == 0)
- { start = 1; }
- if (stop == 0 || pb -> stop > spotid)
- { stop = spotid; }
+ if (pb->start > 0) {
+ start = pb->start;
+ if (start < first) {
+ start = first;
+ }
+ }
+ else {
+ start = first;
+ }
- for (spotid = start; spotid <= stop && rc == 0;
- ++spotid)
- {
+ if (pb->stop > 0) {
+ stop = pb->stop;
+ if (stop > first + count) {
+ stop = first + count;
+ }
+ }
+ else {
+ stop = first + count;
+ }
+
+ for (spotid = start; spotid < stop && rc == 0; ++spotid) {
SraStats* ss;
uint32_t dREAD_LEN [MAX_NREADS];
uint8_t dREAD_TYPE [MAX_NREADS];
@@ -2523,25 +2667,48 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
int nreads;
rc = Quitting();
- if (rc)
- { LOGMSG(klogWarn, "Interrupted"); }
+ if (rc != 0) {
+ LOGMSG(klogWarn, "Interrupted");
+ }
+
+ if (rc == 0 && pb->progress && pr == NULL) {
+ rc = KLoadProgressbar_Make(&pr, stop + 1 - start);
+ if (rc != 0) {
+ DISP_RC(rc, "cannot initialize progress bar");
+ rc = 0;
+ pr = NULL;
+ }
+ else if (stop - start > 99) {
+ KLoadProgressbar_Process(pr, 0, true);
+ }
+ }
if (rc == 0) {
- rc = SRAColumnRead(cREAD_LEN, spotid, &base, &boff, &row_bits);
- DISP_RC_Read(rc, READ_LEN, spotid, "while calling SRAColumnRead");
+ rc = VCursorColumnRead(curs, spotid,
+ idxREAD_LEN, &base, &boff, &row_bits);
+ DISP_RC_Read(rc, READ_LEN, spotid,
+ "while calling VCursorColumnRead");
}
if (rc == 0) {
- if (boff & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
- if (row_bits & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
- if ((row_bits >> 3) > sizeof(dREAD_LEN))
- { rc = RC(rcExe, rcColumn, rcReading, rcBuffer, rcInsufficient); }
- DISP_RC_Read(rc, READ_LEN, spotid, "after calling SRAColumnRead");
+ if (boff & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcOffset, rcInvalid);
+ }
+ if (row_bits & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcSize, rcInvalid);
+ }
+ if ((row_bits >> 3) > sizeof(dREAD_LEN)) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcBuffer, rcInsufficient);
+ }
+ DISP_RC_Read(rc, READ_LEN, spotid,
+ "after calling VCursorColumnRead");
}
if (rc == 0) {
int i, bio_len, bio_count, bad_cnt, filt_cnt;
- memcpy(dREAD_LEN, ((const char*)base) + (boff>>3), row_bits>>3);
+ memcpy(dREAD_LEN,
+ ((const char*)base) + (boff>>3), row_bits >> 3);
nreads = (row_bits >> 3) / sizeof(*dREAD_LEN);
if (spotid == start) {
g_nreads = nreads;
@@ -2555,100 +2722,167 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
}
if (rc == 0) {
- rc = SRAColumnRead(cREAD_TYPE, spotid, &base, &boff, &row_bits);
- DISP_RC_Read(rc, READ_TYPE, spotid, "while calling SRAColumnRead");
+ rc = VCursorColumnRead(curs, spotid,
+ idxREAD_TYPE, &base, &boff, &row_bits);
+ DISP_RC_Read(rc, READ_TYPE, spotid,
+ "while calling VCursorColumnRead");
if (rc == 0) {
- if (boff & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
- if (row_bits & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
- if ((row_bits >> 3) > sizeof(dREAD_TYPE))
- { rc = RC(rcExe, rcColumn, rcReading, rcBuffer, rcInsufficient); }
- if ((row_bits >> 3) != nreads)
- { rc = RC(rcExe, rcColumn, rcReading, rcData, rcIncorrect); }
- DISP_RC_Read(rc, READ_TYPE, spotid, "after calling SRAColumnRead");
+ if (boff & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcOffset, rcInvalid);
+ }
+ if (row_bits & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcSize, rcInvalid);
+ }
+ if ((row_bits >> 3) > sizeof(dREAD_TYPE)) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcBuffer, rcInsufficient);
+ }
+ if ((row_bits >> 3) != nreads) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcData, rcIncorrect);
+ }
+ DISP_RC_Read(rc, READ_TYPE, spotid,
+ "after calling VCursorColumnRead");
}
}
if (rc == 0) {
- memcpy(dREAD_TYPE, ((const char*)base) + (boff >> 3), row_bits >> 3);
- if (cSPOT_GROUP) {
- rc = SRAColumnRead(cSPOT_GROUP, spotid, &base, &boff, &row_bits);
- DISP_RC_Read(rc, SPOT_GROUP, spotid, "while calling SRAColumnRead");
+ memcpy(dREAD_TYPE,
+ ((const char*)base) + (boff >> 3),
+ row_bits >> 3);
+ if (idxSPOT_GROUP != 0) {
+ rc = VCursorColumnRead(curs, spotid,
+ idxSPOT_GROUP, &base, &boff, &row_bits);
+ DISP_RC_Read(rc, SPOT_GROUP, spotid,
+ "while calling VCursorColumnRead");
if (rc == 0) {
if (row_bits > 0) {
- if (boff & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
- if (row_bits & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
- if ((row_bits >> 3) > sizeof(dSPOT_GROUP))
- { rc = RC(rcExe, rcColumn, rcReading, rcBuffer, rcInsufficient); }
- DISP_RC_Read(rc, SPOT_GROUP, spotid, "after calling SRAColumnRead");
+ if (boff & 7) {
+ rc = RC(rcExe, rcColumn,
+ rcReading,
+ rcOffset, rcInvalid);
+ }
+ if (row_bits & 7) {
+ rc = RC(rcExe, rcColumn,
+ rcReading,
+ rcSize, rcInvalid); }
+ if ((row_bits >> 3)
+ > sizeof(dSPOT_GROUP))
+ {
+ rc = RC(rcExe, rcColumn,
+ rcReading,
+ rcBuffer, rcInsufficient);
+ }
+ DISP_RC_Read(rc, SPOT_GROUP, spotid,
+ "after calling VCursorColumnRead"
+ );
if (rc == 0) {
int n = row_bits >> 3;
- memcpy(dSPOT_GROUP,((const char*)base) + (boff>>3),row_bits>>3);
+ memcpy(dSPOT_GROUP,
+ ((const char*)base)+(boff>>3),
+ row_bits>>3);
dSPOT_GROUP[n]='\0';
if (n > 1 ||
(n == 1 && dSPOT_GROUP[0]))
- { pb -> hasSPOT_GROUP = 1; }
+ {
+ pb -> hasSPOT_GROUP = 1;
+ }
}
}
- else { dSPOT_GROUP[0]='\0'; }
- } else { break; }
+ else {
+ dSPOT_GROUP[0]='\0';
+ }
+ }
+ else {
+ break;
+ }
}
}
if (rc == 0) {
uint64_t cmp_len = 0; /* CMP_READ */
- if (cRD_FILTER) {
- rc = SRAColumnRead(cRD_FILTER, spotid, &base, &boff, &row_bits);
- DISP_RC_Read(rc, RD_FILTER, spotid, "while calling SRAColumnRead");
+ if (idxRD_FILTER != 0) {
+ rc = VCursorColumnRead(curs, spotid,
+ idxRD_FILTER, &base, &boff, &row_bits);
+ DISP_RC_Read(rc, RD_FILTER, spotid,
+ "while calling VCursorColumnRead");
if (rc == 0) {
int size = row_bits >> 3;
- if (boff & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
- if (row_bits & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
- if (size > sizeof dRD_FILTER)
- { rc = RC(rcExe, rcColumn, rcReading, rcBuffer, rcInsufficient); }
- DISP_RC_Read(rc, RD_FILTER, spotid, "after calling SRAColumnRead");
+ if (boff & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcOffset, rcInvalid); }
+ if (row_bits & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcSize, rcInvalid);
+ }
+ if (size > sizeof dRD_FILTER) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcBuffer, rcInsufficient);
+ }
+ DISP_RC_Read(rc, RD_FILTER, spotid,
+ "after calling VCursorColumnRead");
if (rc == 0) {
- memcpy(dRD_FILTER,((const char*)base) + (boff>>3), size);
+ memcpy(dRD_FILTER,
+ ((const char*)base) + (boff>>3),
+ size);
if (size < nreads) {
- /* RD_FILTER is expected to have nreads elements */
+ /* RD_FILTER is expected to have nreads elements */
if (size == 1) {
- /* fill all RD_FILTER elements with RD_FILTER[0] */
+ /* fill all RD_FILTER elements with RD_FILTER[0] */
int i = 0;
- for (i = 1; i < nreads; ++i) {
+ for (i = 1; i < nreads;
+ ++i)
+ {
memcpy(dRD_FILTER + i,
- ((const char*)base) + (boff>>3), 1);
+ ((const char*)base)+(boff>>3),
+ 1);
}
- if (!bad_read_filter) {
+ if
+ (!bad_read_filter)
+ {
bad_read_filter = true;
- PLOGMSG(klogWarn, (klogWarn, "RD_FILTER column"
- " size is 1 but it is expected to be $(n)",
+ PLOGMSG(klogWarn,
+ (klogWarn,
+ "RD_FILTER column size is 1 but it is expected to be $(n)",
"n=%d", nreads));
}
}
- else { /* something really bad with RD_FILTER column:
- let's pretend it does not exist */
- cRD_FILTER = NULL;
+ else {
+ /* something really bad with RD_FILTER column:
+ let's pretend it does not exist */
+ idxRD_FILTER = 0;
bad_read_filter = true;
- PLOGMSG(klogWarn, (klogWarn, "RD_FILTER column size"
- " is $(real) but it is expected to be $(exp)",
- "real=%d,exp=%d", size, nreads));
+ PLOGMSG(klogWarn,
+ (klogWarn,
+ "RD_FILTER column size is $(real) but it is expected to be $(exp)",
+ "real=%d,exp=%d",
+ size, nreads));
}
}
}
- } else { break; }
+ }
+ else {
+ break;
+ }
}
-
- if (cPRIMARY_ALIGNMENT_ID) {
- rc = SRAColumnRead(cPRIMARY_ALIGNMENT_ID, spotid, &base, &boff, &row_bits);
- DISP_RC_Read(rc, PRIMARY_ALIGNMENT_ID, spotid, "while calling SRAColumnRead");
- if (boff & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
- if (row_bits & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
- DISP_RC_Read(rc, PRIMARY_ALIGNMENT_ID, spotid, "after calling calling SRAColumnRead");
+ if (idxPRIMARY_ALIGNMENT_ID != 0) {
+ rc = VCursorColumnRead(curs, spotid,
+ idxPRIMARY_ALIGNMENT_ID,
+ &base, &boff, &row_bits);
+ DISP_RC_Read(rc, PRIMARY_ALIGNMENT_ID,
+ spotid,
+ "while calling VCursorColumnRead");
+ if (boff & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcOffset, rcInvalid); }
+ if (row_bits & 7) {
+ rc = RC(rcExe, rcColumn, rcReading,
+ rcSize, rcInvalid);
+ }
+ DISP_RC_Read(rc, PRIMARY_ALIGNMENT_ID,
+ spotid,
+ "after calling calling VCursorColumnRead"
+ );
if (rc == 0) {
int i = 0;
const int64_t* pii = base;
@@ -2661,27 +2895,30 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
}
}
/* if (cCMP_READ) {
- rc = SRAColumnRead(cCMP_READ, spotid, &base, &boff, &row_bits);
- DISP_RC_Read(rc, CMP_READ, spotid, "while calling SRAColumnRead");
- if (boff & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
- if (row_bits & 7)
- { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
- DISP_RC_Read(rc, CMP_READ, spotid, "after calling calling SRAColumnRead");
- if (rc == 0)
- { assert(cmp_len == row_bits >> 3); }
- } */
-
- ss = (SraStats*)BSTreeFind(tr, dSPOT_GROUP, srastats_cmp);
+ rc = SRAColumnRead(cCMP_READ, spotid, &base, &boff, &row_bits);
+ DISP_RC_Read(rc, CMP_READ, spotid, "while calling SRAColumnRead");
+ if (boff & 7)
+ { rc = RC(rcExe, rcColumn, rcReading, rcOffset, rcInvalid); }
+ if (row_bits & 7)
+ { rc = RC(rcExe, rcColumn, rcReading, rcSize, rcInvalid); }
+ DISP_RC_Read(rc, CMP_READ, spotid, "after calling calling SRAColumnRead");
+ if (rc == 0)
+ { assert(cmp_len == row_bits >> 3); }
+ } */
+
+ ss = (SraStats*)BSTreeFind
+ (tr, dSPOT_GROUP, srastats_cmp);
if (ss == NULL) {
ss = calloc(1, sizeof(*ss));
if (ss == NULL) {
- rc = RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
+ rc = RC(rcExe, rcStorage, rcAllocating,
+ rcMemory, rcExhausted);
break;
}
else {
strcpy(ss->spot_group, dSPOT_GROUP);
- BSTreeInsert(tr, (BSTNode*)ss, srastats_sort);
+ BSTreeInsert
+ (tr, (BSTNode*)ss, srastats_sort);
}
}
++ss->spot_count;
@@ -2695,7 +2932,10 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
if (pb->statistics) {
SraStatsTotalAdd(total, dREAD_LEN, nreads);
}
- for (bio_len = bio_count = i = bad_cnt = filt_cnt = 0; (i < nreads) && (rc == 0); i++) {
+ for (bio_len = bio_count = i = bad_cnt
+ = filt_cnt = 0;
+ (i < nreads) && (rc == 0); i++)
+ {
if (dREAD_LEN[i] > 0) {
g_totalREAD_LEN[i] += dREAD_LEN[i];
++g_nonZeroLenReads[i];
@@ -2703,42 +2943,53 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
if (spotid == start) {
g_dREAD_LEN[i] = dREAD_LEN[i];
}
- else if (g_dREAD_LEN[i] != dREAD_LEN[i])
- { fixedReadLength = false; }
+ else if (g_dREAD_LEN[i] != dREAD_LEN[i]) {
+ fixedReadLength = false;
+ }
if (dREAD_LEN[i] > 0) {
bool biological = false;
ss->total_len += dREAD_LEN[i];
total->BASE_COUNT += dREAD_LEN[i];
- if ((dREAD_TYPE[i] & SRA_READ_TYPE_BIOLOGICAL) != 0) {
+ if ((dREAD_TYPE[i]
+ & SRA_READ_TYPE_BIOLOGICAL) != 0)
+ {
biological = true;
bio_len += dREAD_LEN[i];
bio_count++;
}
- if (cRD_FILTER) {
+ if (idxRD_FILTER != 0) {
switch (dRD_FILTER[i]) {
case SRA_READ_FILTER_PASS:
break;
case SRA_READ_FILTER_REJECT:
case SRA_READ_FILTER_CRITERIA:
if (biological) {
- ss->bad_bio_len += dREAD_LEN[i];
- total->bad_bio_len += dREAD_LEN[i];
+ ss->bad_bio_len
+ += dREAD_LEN[i];
+ total->bad_bio_len
+ += dREAD_LEN[i];
}
bad_cnt++;
break;
case SRA_READ_FILTER_REDACTED:
if (biological) {
- ss->filtered_bio_len += dREAD_LEN[i];
- total->filtered_bio_len += dREAD_LEN[i];
+ ss->filtered_bio_len
+ += dREAD_LEN[i];
+ total->filtered_bio_len
+ += dREAD_LEN[i];
}
filt_cnt++;
break;
default:
- rc = RC(rcExe, rcColumn, rcReading, rcData, rcUnexpected);
- PLOGERR(klogInt, (klogInt, rc,
- "spot=$(spot), read=$(read), READ_FILTER=$(val)", "spot=%lu,read=%d,val=%d",
- spotid, i, dRD_FILTER[i]));
+ rc = RC(rcExe, rcColumn,
+ rcReading,
+ rcData, rcUnexpected);
+ PLOGERR(klogInt,
+ (klogInt, rc,
+ "spot=$(spot), read=$(read), READ_FILTER=$(val)", "spot=%lu,read=%d,val=%d",
+ spotid, i,
+ dRD_FILTER[i]));
break;
}
}
@@ -2761,18 +3012,23 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
total->filtered_spot_count++;
}
}
+
+ if (rc == 0 && pb->progress) {
+ KLoadProgressbar_Process(pr, 1, false);
+ }
}
- } /* for (spotid = start; spotid <= stop && rc == 0; ++spotid) */
+ } /* for (spotid = start; spotid <= stop && rc == 0;
+ ++spotid) */
if (rc == 0) {
BasesFinalize(&total->bases_count);
pb->variableReadLength = !fixedReadLength;
- /* --- g_totalREAD_LEN[i] is sum(READ_LEN[i]) for all spots --- */
+ /* --- g_totalREAD_LEN[i] is sum(READ_LEN[i]) for all spots --- */
if (fixedNReads) {
int i = 0;
if (stop >= start) {
- n_spots = stop - start + 1;
+ n_spots = stop - start;
}
if (n_spots > 0) {
for (i = 0; i < g_nreads && rc == 0; ++i) {
@@ -2784,20 +3040,22 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
}
}
}
+ if (rc == 0) {
+ KLoadProgressbar_Release(pr, true);
+ pr = NULL;
+ }
}
- RELEASE(SRAColumn, cSPOT_GROUP);
}
}
}
- RELEASE(SRAColumn, cPRIMARY_ALIGNMENT_ID);
- RELEASE(SRAColumn, cRD_FILTER);
- RELEASE(SRAColumn, cREAD_TYPE);
- RELEASE(SRAColumn, cREAD_LEN);
+ RELEASE(VCursor, curs);
- if (pb->test) {
+ if (pb->test && rc == 0) {
+ uint32_t idx = 0;
int i = 0;
- spotid_t spotid = 0;
+ int64_t spotid = 0;
+
double average[MAX_NREADS];
double diff_sq[MAX_NREADS];
SraStatsTotalStatistics2Init(total,
@@ -2806,19 +3064,32 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
for (i = 0; i < g_nreads; ++i) {
average[i] = (double)g_totalREAD_LEN[i] / n_spots;
}
+
+ rc = VTableCreateCursorRead(vtbl, &curs);
+ DISP_RC(rc, "Cannot VTableCreateCursorRead");
+
if (rc == 0) {
const char* name = READ_LEN;
- rc = SRATableOpenColumnRead(tbl, &cREAD_LEN, name, vdb_uint32_t);
- DISP_RC2(rc, name, "while calling SRATableOpenColumnRead");
+ rc = VCursorAddColumn(curs, &idx, "%s", name);
+ DISP_RC(rc, "Cannot VCursorAddColumn(READ_LEN)");
+ if (rc == 0) {
+ rc = VCursorOpen(curs);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt,
+ rc, "Cannot VCursorOpen($(name))", "name=%s", name));
+ }
+ }
}
- for (spotid = start; spotid <= stop && rc == 0; ++spotid) {
+
+ for (spotid = start; spotid < stop && rc == 0; ++spotid) {
uint32_t dREAD_LEN[MAX_NREADS];
const void* base;
bitsz_t boff, row_bits;
if (rc == 0) {
- rc = SRAColumnRead(cREAD_LEN, spotid, &base, &boff, &row_bits);
+ rc = VCursorColumnRead(curs, spotid,
+ idx, &base, &boff, &row_bits);
DISP_RC_Read(rc, READ_LEN, spotid,
- "while calling SRAColumnRead");
+ "while calling VCursorColumnRead");
}
if (rc == 0) {
memcpy(dREAD_LEN, ((const char*)base) + (boff>>3), row_bits>>3);
@@ -2829,7 +3100,7 @@ rc_t sra_stat(srastat_parms* pb, const SRATable* tbl,
}
SraStatsTotalAdd2(total, dREAD_LEN);
}
- RELEASE(SRAColumn, cREAD_LEN);
+ RELEASE(VCursor, curs);
}
return rc;
@@ -2849,32 +3120,54 @@ static
rc_t run(srastat_parms* pb)
{
rc_t rc = 0;
- const SRAMgr* mgr = NULL;
+ const VDBManager* vmgr = NULL;
assert(pb && pb->table_path);
- rc = SRAMgrMakeRead(&mgr);
-
+ rc = VDBManagerMakeRead(&vmgr, NULL);
if (rc != 0) {
- LOGERR(klogInt, rc, "failed to open SRAMgr");
+ LOGERR(klogInt, rc, "failed to open VDBManager");
}
else {
SraSizeStats sizes;
ArcInfo arc_info;
SraMeta info;
- const SRATable* tbl = NULL;
+ const VTable* vtbl = NULL;
- rc = SRAMgrOpenTableRead(mgr, &tbl, "%s", pb->table_path);
+ VSchema *schema = NULL;
+
+ rc = VDBManagerMakeSRASchema(vmgr, &schema);
if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc,
- "'$(spec)'", "spec=%s", pb->table_path));
+ LOGERR(klogInt, rc, "cannot VDBManagerMakeSRASchema");
}
- else {
+ if (rc == 0) {
+ rc = VDBManagerOpenTableRead(vmgr, &vtbl,
+ schema, "%s", pb->table_path);
+ if (rc != 0 && GetRCObject(rc) == (enum RCObject)rcTable
+ && GetRCState (rc) == rcIncorrect)
+ {
+ const char altname[] = "SEQUENCE";
+ const VDatabase *db = NULL;
+ rc_t rc2 = VDBManagerOpenDBRead(vmgr,
+ &db, schema, pb->table_path);
+ if (rc2 == 0) {
+ rc2 = VDatabaseOpenTableRead(db, &vtbl, "%s", altname);
+ if (rc2 == 0) {
+ rc = 0;
+ }
+ }
+ VDatabaseRelease ( db );
+ }
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc,
+ "'$(spec)'", "spec=%s", pb->table_path));
+ }
+ }
+ if (rc == 0) {
MetaDataStats stats;
SraStatsTotal total;
const KTable* ktbl = NULL;
const KMetadata* meta = NULL;
- const VTable* vtbl = NULL;
const VDatabase* db = NULL;
BSTree tr;
@@ -2885,18 +3178,13 @@ rc_t run(srastat_parms* pb)
memset(&total, 0, sizeof total);
- rc = SRATableGetKTableRead(tbl, &ktbl);
- DISP_RC(rc, "While calling SRATableGetKTableRead");
+ rc = VTableOpenKTableRead(vtbl, &ktbl);
+ DISP_RC(rc, "While calling VTableOpenKTableRead");
if (rc == 0) {
rc = KTableOpenMetadataRead(ktbl, &meta);
DISP_RC(rc, "While calling KTableOpenMetadataRead");
}
if (rc == 0) {
- rc = SRATableGetVTableRead(tbl, &vtbl);
- DISP_RC2(rc, pb->table_path,
- "while calling SRATableGetVTableRead");
- }
- if (rc == 0) {
rc = VTableOpenParentRead(vtbl, &db);
DISP_RC2(rc, pb->table_path,
"while calling VTableOpenParentRead");
@@ -2912,24 +3200,29 @@ rc_t run(srastat_parms* pb)
}
rc = 0;
}
- if (rc == 0)
- { rc = get_size(tbl, &sizes); }
- if (rc == 0 && pb->printMeta)
- { rc = get_load_info(meta, &info); }
- if (rc == 0 && !pb->quick)
- { rc = sra_stat(pb, tbl, &tr, &total); }
+ if (rc == 0) {
+ rc = get_size(&sizes, vtbl);
+ }
+ if (rc == 0 && pb->printMeta) {
+ rc = get_load_info(meta, &info);
+ }
+ if (rc == 0 && !pb->quick) {
+ rc = sra_stat(pb, &tr, &total, vtbl);
+ }
if (rc == 0 && pb->print_arcinfo ) {
- rc = get_arc_info(mgr, pb->table_path, tbl, &arc_info);
+ rc = get_arc_info(pb->table_path, &arc_info, vmgr, vtbl);
}
if (rc == 0) {
rc = QualityStatsRead(&ctx.quality, meta);
- if (rc == 0)
- { QualityStatsSort(&ctx.quality); }
+ if (rc == 0) {
+ QualityStatsSort(&ctx.quality);
+ }
}
if (rc == 0) {
rc = TableCountsRead(&ctx.tables, db);
- if (rc == 0)
- { TableCountsSort(&ctx.tables); }
+ if (rc == 0) {
+ TableCountsSort(&ctx.tables);
+ }
}
if (rc == 0) {
ctx.db = db;
@@ -2945,7 +3238,6 @@ rc_t run(srastat_parms* pb)
BSTreeWhack(&tr, bst_whack_free, NULL);
SraStatsTotalFree(&total);
RELEASE(VDatabase, db);
- RELEASE(VTable, vtbl);
RELEASE(KTable, ktbl);
{
int i;
@@ -2961,10 +3253,11 @@ rc_t run(srastat_parms* pb)
CtxRelease(&ctx);
RELEASE(KMetadata, meta);
}
- RELEASE(SRATable, tbl);
+ RELEASE(VSchema, schema);
+ RELEASE(VTable, vtbl);
}
- RELEASE(SRAMgr, mgr);
+ RELEASE(VDBManager, vmgr);
return rc;
}
@@ -2981,57 +3274,76 @@ ver_t CC KAppVersion ( void )
}
-/* Usage
- */
-#define OPTION_ALIGN "alignment"
-#define OPTION_SPT_D "spot-desc"
-#define OPTION_MEMBR "member-stats"
-#define OPTION_META "meta"
-#define OPTION_QUICK "quick"
-#define OPTION_START "start"
-#define OPTION_STATS "statistics"
-#define OPTION_STOP "stop"
-#define OPTION_TEST "test"
-#define OPTION_XML "xml"
+/* Usage */
+#define ALIAS_ALIGN "a"
+#define OPTION_ALIGN "alignment"
+
+#define ALIAS_ARCINFO NULL
#define OPTION_ARCINFO "archive-info"
-#define ALIAS_ALIGN "a"
-#define ALIAS_SPT_D "d"
-#define ALIAS_MEMBR NULL
-#define ALIAS_META "m"
-#define ALIAS_QUICK "q"
-#define ALIAS_START "b"
-#define ALIAS_STATS "s"
-#define ALIAS_STOP "e"
-#define ALIAS_TEST "t"
-#define ALIAS_XML "x"
-#define ALIAS_ARCINFO NULL
-
-static const char * align_usage[] = { "print alignment info, default is on", NULL };
+#define ALIAS_START "b"
+#define OPTION_START "start"
+
+#define ALIAS_STOP "e"
+#define OPTION_STOP "stop"
+
+#define ALIAS_SPT_D "d"
+#define OPTION_SPT_D "spot-desc"
+
+#define ALIAS_META "m"
+#define OPTION_META "meta"
+
+#define ALIAS_MEMBR NULL
+#define OPTION_MEMBR "member-stats"
+
+#define ALIAS_PROGRESS "p"
+#define OPTION_PROGRESS "show_progress"
+
+#define ALIAS_QUICK "q"
+#define OPTION_QUICK "quick"
+
+#define ALIAS_STATS "s"
+#define OPTION_STATS "statistics"
+
+#define ALIAS_TEST "t"
+#define OPTION_TEST "test"
+
+#define ALIAS_XML "x"
+#define OPTION_XML "xml"
+
+static const char * align_usage[] = { "print alignment info, default is on"
+ , NULL };
static const char * spt_d_usage[] = { "print table spot descriptor", NULL };
-static const char * membr_usage[] = { "print member stats, default is on", NULL };
+static const char * membr_usage[] = { "print member stats, default is on"
+ , NULL };
+static const char *progress_usage[] = { "show the percentage of completion"
+ , NULL };
static const char * meta_usage[] = { "print load metadata", NULL };
static const char * start_usage[] = { "starting spot id, default is 1", NULL };
static const char * stop_usage[] = { "ending spot id, default is max", NULL };
-static const char * stats_usage[] = { "calculate READ_LEN average and standard deviation", NULL };
-static const char * quick_usage[] = { "quick mode: get statistics from metadata;", "do not scan the table", NULL };
-static const char * test_usage[] = { "test READ_LEN average and standard deviation calculation", NULL };
+static const char * stats_usage[] = {
+ "calculate READ_LEN average and standard deviation", NULL };
+static const char * quick_usage[] = {
+ "quick mode: get statistics from metadata;", "do not scan the table", NULL };
+static const char * test_usage[] = {
+ "test READ_LEN average and standard deviation calculation", NULL };
static const char * xml_usage[] = { "output as XML, default is text", NULL };
-static const char * arcinfo_usage[] = { "output archive info, default is off", NULL };
-
-OptDef Options[] =
-{
- { OPTION_ALIGN, ALIAS_ALIGN, NULL, align_usage, 1, true , false }
- , { OPTION_SPT_D, ALIAS_SPT_D, NULL, spt_d_usage, 1, false, false }
- , { OPTION_MEMBR, ALIAS_MEMBR, NULL, membr_usage, 1, true , false }
- , { OPTION_ARCINFO, ALIAS_ARCINFO, NULL, arcinfo_usage, 0, false, false }
- , { OPTION_META, ALIAS_META, NULL, meta_usage, 1, false, false }
- , { OPTION_QUICK, ALIAS_QUICK, NULL, quick_usage, 1, false, false }
- , { OPTION_START, ALIAS_START, NULL, start_usage, 1, true, false }
- , { OPTION_STATS, ALIAS_STATS, NULL, stats_usage, 1, false, false }
- , { OPTION_STOP, ALIAS_STOP, NULL, stop_usage, 1, true, false }
- , { OPTION_TEST , ALIAS_TEST , NULL, test_usage, 1, false, false }
- , { OPTION_XML, ALIAS_XML, NULL, xml_usage, 1, false, false }
+static const char * arcinfo_usage[] = { "output archive info, default is off"
+ , NULL };
+
+OptDef Options[] = {
+ { OPTION_ALIGN , ALIAS_ALIGN , NULL, align_usage , 1, true , false }
+ , { OPTION_SPT_D , ALIAS_SPT_D , NULL, spt_d_usage , 1, false, false }
+ , { OPTION_MEMBR , ALIAS_MEMBR , NULL, membr_usage , 1, true , false }
+ , { OPTION_PROGRESS, ALIAS_PROGRESS, NULL, progress_usage, 1, false, false }
+ , { OPTION_ARCINFO , ALIAS_ARCINFO , NULL, arcinfo_usage , 0, false, false }
+ , { OPTION_META , ALIAS_META , NULL, meta_usage , 1, false, false }
+ , { OPTION_QUICK , ALIAS_QUICK , NULL, quick_usage , 1, false, false }
+ , { OPTION_START , ALIAS_START , NULL, start_usage , 1, true, false }
+ , { OPTION_STATS , ALIAS_STATS , NULL, stats_usage , 1, false, false }
+ , { OPTION_STOP , ALIAS_STOP , NULL, stop_usage , 1, true, false }
+ , { OPTION_TEST , ALIAS_TEST , NULL, test_usage , 1, false, false }
+ , { OPTION_XML , ALIAS_XML , NULL, xml_usage , 1, false, false }
};
rc_t CC UsageSummary (const char * progname)
@@ -3064,18 +3376,24 @@ rc_t CC Usage (const Args * args)
KOutMsg ("Options:\n");
- HelpOptionLine (ALIAS_XML, OPTION_XML, NULL, xml_usage);
- HelpOptionLine (ALIAS_START, OPTION_START, "row-id", start_usage);
- HelpOptionLine (ALIAS_STOP, OPTION_STOP, "row-id", stop_usage);
- HelpOptionLine (ALIAS_META, OPTION_META, NULL, meta_usage);
- HelpOptionLine (ALIAS_QUICK, OPTION_QUICK, NULL, quick_usage);
- HelpOptionLine (ALIAS_MEMBR, OPTION_MEMBR, "on | off", membr_usage);
- HelpOptionLine (ALIAS_ARCINFO, OPTION_ARCINFO, NULL, arcinfo_usage);
- HelpOptionLine (ALIAS_STATS, OPTION_STATS, NULL, stats_usage);
- HelpOptionLine (ALIAS_ALIGN, OPTION_ALIGN, "on | off", align_usage);
+ HelpOptionLine(ALIAS_XML , OPTION_XML , NULL , xml_usage);
+ HelpOptionLine(ALIAS_START , OPTION_START , "row-id" , start_usage);
+ HelpOptionLine(ALIAS_STOP , OPTION_STOP , "row-id" , stop_usage);
+ HelpOptionLine(ALIAS_META , OPTION_META , NULL , meta_usage);
+ HelpOptionLine(ALIAS_QUICK , OPTION_QUICK , NULL , quick_usage);
+ HelpOptionLine(ALIAS_MEMBR , OPTION_MEMBR , "on | off", membr_usage);
+ HelpOptionLine(ALIAS_ARCINFO , OPTION_ARCINFO , NULL , arcinfo_usage);
+ HelpOptionLine(ALIAS_STATS , OPTION_STATS , NULL , stats_usage);
+ HelpOptionLine(ALIAS_ALIGN , OPTION_ALIGN , "on | off", align_usage);
+ HelpOptionLine(ALIAS_PROGRESS, OPTION_PROGRESS, NULL , progress_usage);
+ XMLLogger_Usage();
+
KOutMsg ("\n");
+
HelpOptionsStandard ();
+
HelpVersion (fullpath, KAppVersion());
+
return rc;
}
@@ -3104,143 +3422,210 @@ rc_t CC KMain ( int argc, char *argv [] )
srastat_parms pb;
memset(&pb, 0, sizeof pb);
- rc = ArgsMakeAndHandle(&args, argc, argv, 1,
- Options, sizeof Options / sizeof (OptDef));
- if (rc == 0)
- {
- do
- {
- uint32_t pcount;
- const char* pc;
-
- rc = ArgsOptionCount (args, OPTION_START, &pcount);
- if (rc)
- break;
+ rc = ArgsMakeAndHandle(&args, argc, argv, 2, Options,
+ sizeof Options / sizeof(OptDef), XMLLogger_Args, XMLLogger_ArgsQty);
+ if (rc == 0) {
+ do {
+ uint32_t pcount = 0;
+ const char* pc = NULL;
- if (pcount == 1)
{
- rc = ArgsOptionValue (args, OPTION_START, 0, &pc);
- if (rc)
+ rc = ArgsOptionCount (args, OPTION_START, &pcount);
+ if (rc != 0) {
break;
+ }
- pb.start = AsciiToU32 (pc, NULL, NULL);
- }
+ if (pcount == 1) {
+ rc = ArgsOptionValue (args, OPTION_START, 0, (const void **)&pc);
+ if (rc != 0) {
+ break;
+ }
- rc = ArgsOptionCount (args, OPTION_STOP, &pcount);
- if (rc)
- break;
+ pb.start = AsciiToU32 (pc, NULL, NULL);
+ }
- if (pcount == 1)
- {
- rc = ArgsOptionValue (args, OPTION_STOP, 0, &pc);
- if (rc)
+
+ rc = ArgsOptionCount (args, OPTION_STOP, &pcount);
+ if (rc != 0) {
break;
+ }
- pb.stop = AsciiToU32 (pc, NULL, NULL);
- }
- rc = ArgsOptionCount (args, OPTION_XML, &pcount);
- if (rc)
- break;
+ if (pcount == 1) {
+ rc = ArgsOptionValue (args, OPTION_STOP, 0, (const void **)&pc);
+ if (rc != 0) {
+ break;
+ }
- if (pcount)
- pb.xml = true;
+ pb.stop = AsciiToU32 (pc, NULL, NULL);
+ }
- rc = ArgsOptionCount (args, OPTION_QUICK, &pcount);
- if (rc)
- break;
- if (pcount)
- pb.quick = true;
+ rc = ArgsOptionCount (args, OPTION_XML, &pcount);
+ if (rc != 0) {
+ break;
+ }
- rc = ArgsOptionCount (args, OPTION_META, &pcount);
- if (rc)
- break;
+ if (pcount > 0) {
+ pb.xml = true;
+ }
+ }
- if (pcount)
- pb.printMeta = true;
+ {
+ rc = ArgsOptionCount (args, OPTION_QUICK, &pcount);
+ if (rc != 0) {
+ break;
+ }
- rc = ArgsOptionCount (args, OPTION_MEMBR, &pcount);
- if (rc)
- break;
+ if (pcount > 0) {
+ pb.quick = true;
+ }
- if (pcount) {
- const char* v = NULL;
- rc = ArgsOptionValue (args, OPTION_MEMBR, 0, &v);
- if (rc)
+
+ rc = ArgsOptionCount (args, OPTION_META, &pcount);
+ if (rc != 0) {
break;
- if (!strcmp(v, "off")) {
- pb.skip_members = true;
}
- }
- rc = ArgsOptionCount (args, OPTION_ARCINFO, &pcount);
- if (rc)
- break;
+ if (pcount > 0) {
+ pb.printMeta = true;
+ }
- pb.print_arcinfo = pcount > 0;
- rc = ArgsOptionCount (args, OPTION_ALIGN, &pcount);
- if (rc)
- break;
+ rc = ArgsOptionCount (args, OPTION_MEMBR, &pcount);
+ if (rc != 0) {
+ break;
+ }
+
+ if (pcount > 0) {
+ const char* v = NULL;
+ rc = ArgsOptionValue (args, OPTION_MEMBR, 0, (const void **)&v);
+ if (rc != 0) {
+ break;
+ }
+ if (!strcmp(v, "off")) {
+ pb.skip_members = true;
+ }
+ }
+ }
- if (pcount) {
- const char* v = NULL;
- rc = ArgsOptionValue (args, OPTION_ALIGN, 0, &v);
- if (rc)
+ {
+ rc = ArgsOptionCount(args, OPTION_PROGRESS, &pcount);
+ if (rc != 0) {
break;
- if (!strcmp(v, "off")) {
- pb.skip_alignment = true;
}
+ if (pcount > 0) {
+ KLogLevel l = KLogLevelGet();
+ pb.progress = true;
+ rc = ArgsOptionCount(args, OPTION_LOG_LEVEL, &pcount);
+ if (rc == 0) {
+ if (pcount == 0) {
+ if (l < klogInfo) {
+ KLogLevelSet(klogInfo);
+ }
+ }
+ else if (l < klogInfo) {
+ LOGMSG(klogWarn, "log-level was set: "
+ "progress will not be shown");
+ }
+ }
+ }
+
+
+ rc = ArgsOptionCount (args, OPTION_ARCINFO, &pcount);
+ if (rc != 0) {
+ break;
+ }
+
+ pb.print_arcinfo = pcount > 0;
}
- rc = ArgsOptionCount (args, OPTION_STATS, &pcount);
- if (rc)
- break;
+ {
+ rc = ArgsOptionCount (args, OPTION_ALIGN, &pcount);
+ if (rc != 0) {
+ break;
+ }
- if (pcount)
- pb.statistics = true;
+ if (pcount > 0) {
+ const char* v = NULL;
+ rc = ArgsOptionValue (args, OPTION_ALIGN, 0, (const void **)&v);
+ if (rc != 0) {
+ break;
+ }
+ if (!strcmp(v, "off")) {
+ pb.skip_alignment = true;
+ }
+ }
- rc = ArgsOptionCount (args, OPTION_TEST, &pcount);
- if (rc)
- break;
- if (pcount)
- pb.test = pb.statistics = true;
+ rc = ArgsOptionCount (args, OPTION_STATS, &pcount);
+ if (rc != 0) {
+ break;
+ }
- rc = ArgsParamCount (args, &pcount);
- if (rc)
- break;
+ if (pcount > 0) {
+ pb.statistics = true;
+ }
+
+
+ rc = ArgsOptionCount (args, OPTION_TEST, &pcount);
+ if (rc != 0) {
+ break;
+ }
- if (pcount == 0) {
- MiniUsage (args);
- exit(1);
+ if (pcount > 0) {
+ pb.test = pb.statistics = true;
+ }
}
- rc = ArgsParamValue (args, 0, &pb.table_path);
- if (rc)
- break;
+ {
+ rc = ArgsParamCount (args, &pcount);
+ if (rc != 0) {
+ break;
+ }
+
+ if (pcount == 0) {
+ MiniUsage (args);
+ exit(1);
+ }
+
+
+ rc = ArgsParamValue (args, 0, (const void **)&pb.table_path);
+ if (rc != 0) {
+ break;
+ }
+
+ if (pb.statistics && (pb.quick || ! pb.xml)) {
+ KOutMsg("\n--" OPTION_STATS
+ " option can be used just in XML NON-QUICK mode\n");
+ MiniUsage (args);
+ exit(1);
+ }
- if (pb.statistics && (pb.quick || ! pb.xml)) {
- KOutMsg("\n--" OPTION_STATS
- " option can be used just in XML NON-QUICK mode\n");
- MiniUsage (args);
- exit(1);
+ rc = XMLLogger_Make(&pb.logger, NULL, args);
+ if (rc != 0) {
+ DISP_RC(rc, "cannot initialize XMLLogger");
+ rc = 0;
+ }
}
} while (0);
}
- if (rc == 0)
- { rc = run(&pb); }
- else { DISP_RC(rc, "while processing command line"); }
+ if (rc == 0) {
+ rc = run(&pb);
+ }
+ else {
+ DISP_RC(rc, "while processing command line");
+ }
+
+ XMLLogger_Release(pb.logger);
{
rc_t rc2 = ArgsWhack(args);
- if (rc == 0)
- { rc = rc2; }
+ if (rc == 0) {
+ rc = rc2;
+ }
}
return rc;
}
-
-/* EOF */
diff --git a/tools/run-stat/mod_reads.h b/tools/sra-stat/sra-stat.h
similarity index 74%
rename from tools/run-stat/mod_reads.h
rename to tools/sra-stat/sra-stat.h
index 4dec794..1415ae2 100644
--- a/tools/run-stat/mod_reads.h
+++ b/tools/sra-stat/sra-stat.h
@@ -1,4 +1,7 @@
-/*===========================================================================
+#ifndef _h_sra_stat_tools_
+#define _h_sra_stat_tools_
+
+/*==============================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
@@ -24,19 +27,16 @@
*
*/
-#ifndef _h_stat_mod_reads_
-#define _h_stat_mod_reads_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
+#include <klib/defs.h> /* rc_t */
-#include <klib/rc.h>
+struct KFile;
+struct VCursor;
+struct VTable;
-rc_t mod_reads_init( p_module self, const char *param );
+rc_t CC VCursorColumnRead(const struct VCursor *self, int64_t id,
+ uint32_t idx, const void **base, bitsz_t *offset, bitsz_t *size);
+rc_t CC VTableMakeSingleFileArchive(const struct VTable *self,
+ const struct KFile **sfa, bool lightweight);
-#endif
+#endif /* _h_sra_stat_tools_ */
diff --git a/tools/sra-stat/sra-stat.vers b/tools/sra-stat/sra-stat.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra-stat/sra-stat.vers
+++ b/tools/sra-stat/sra-stat.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-stat/sra-stat.vers.h b/tools/sra-stat/sra-stat.vers.h
index 62358e7..bbfbbe5 100644
--- a/tools/sra-stat/sra-stat.vers.h
+++ b/tools/sra-stat/sra-stat.vers.h
@@ -1 +1 @@
-#define SRA_STAT_VERS 0x02040002
+#define SRA_STAT_VERS 0x02050007
diff --git a/tools/sra-stat/sra.c b/tools/sra-stat/sra.c
new file mode 100644
index 0000000..b68e533
--- /dev/null
+++ b/tools/sra-stat/sra.c
@@ -0,0 +1,446 @@
+/*==============================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+* A couple of function to ease migration from libsra to libvdb use.
+* Maybe could be improved/optimized/removed later.
+*/
+
+#include "sra-stat.h" /* VTableMakeSingleFileArchive_ */
+
+#include <kdb/database.h> /* KDatabase */
+#include <kdb/kdb-priv.h> /* KDatabaseOpenDirectoryRead */
+#include <kdb/table.h> /* KTableRelease */
+
+#include <kfs/directory.h> /* KDirectory */
+#include <kfs/file.h> /* KFile */
+#include <kfs/toc.h> /* KDirectoryOpenTocFileRead */
+
+#include <klib/debug.h> /* DBGMSG */
+#include <klib/log.h> /* LOGERR */
+#include <klib/rc.h>
+#include <klib/vector.h> /* Vector */
+
+#include <vdb/cursor.h> /* VCursor */
+#include <vdb/database.h> /* VDatabase */
+#include <vdb/table.h> /* VTableOpenParentRead */
+#include <vdb/vdb-priv.h> /* VDatabaseOpenKDatabaseRead */
+
+#include <string.h> /* memcmp */
+
+#define DISP_RC(rc, msg) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, msg))
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
+
+/* sfa_sort
+ * reorders list
+ */
+enum sfa_path_type_id {
+ sfa_not_set = -1,
+ sfa_exclude,
+ sfa_non_column,
+ sfa_required,
+ sfa_preferred,
+ sfa_optional
+};
+#if _DEBUGGING
+static const char* sfa_path_type_id[] = {
+ "not_set",
+ "exclude",
+ "non_column",
+ "required",
+ "preferred",
+ "optional"
+};
+#endif
+
+#define MATCH( ptr, str ) \
+ ( (memcmp(ptr, str, sizeof(str) - 2) == 0 && \
+ ((ptr)[sizeof(str) - 2] == '\0' || (ptr)[sizeof(str) - 2] == '/')) \
+ ? (ptr) += sizeof(str) - (((ptr)[sizeof(str) - 2] == '/') ? 1 : 2) \
+ : (const char*) 0)
+
+static
+enum sfa_path_type_id CC sfa_path_type_tbl( const char *path )
+{
+ /* use first character as distinguisher for match */
+ switch ( path [ 0 ] )
+ {
+ case 'c':
+ /* perhaps it's a column */
+ if ( MATCH ( path, "col/" ) )
+ {
+ switch ( path [ 0 ] )
+ {
+ case 'D':
+ if ( MATCH ( path, "DELETION_QV/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "DELETION_TAG/" ) )
+ return sfa_optional;
+ break;
+ case 'H':
+ if ( MATCH ( path, "HOLE_NUMBER/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "HOLE_STATUS/" ) )
+ return sfa_optional;
+ break;
+ case 'I':
+ if ( MATCH ( path, "INTENSITY/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "INSERTION_QV/" ) )
+ return sfa_optional;
+ break;
+ case 'N':
+ if ( MATCH ( path, "NAME_FMT/" ) )
+ return sfa_preferred;
+ if ( MATCH ( path, "NAME/" ) )
+ return sfa_preferred;
+ if ( MATCH ( path, "NOISE/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "NUM_PASSES/" ) )
+ return sfa_optional;
+ break;
+ case 'P':
+ if ( MATCH ( path, "POSITION/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "PRE_BASE_FRAMES/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "PULSE_INDEX/" ) )
+ return sfa_optional;
+ break;
+ case 'Q':
+ if ( MATCH ( path, "QUALITY2/" ) )
+ return sfa_optional;
+ break;
+ case 'S':
+ if ( MATCH ( path, "SIGNAL/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "SPOT_NAME/" ) )
+ return sfa_preferred;
+ if ( MATCH ( path, "SUBSTITUTION_QV/" ) )
+ return sfa_optional;
+ if ( MATCH ( path, "SUBSTITUTION_TAG/" ) )
+ return sfa_optional;
+ break;
+ case 'W':
+ if ( MATCH ( path, "WIDTH_IN_FRAMES/" ) )
+ return sfa_optional;
+ break;
+ case 'X':
+ case 'Y':
+ if ( path [ 1 ] == '/' )
+ return sfa_preferred;
+ break;
+ }
+ }
+ return sfa_required;
+
+ case 'i':
+ /* look for skey index */
+ if ( MATCH ( path, "idx/skey" ) )
+ if ( path [ 0 ] == 0 || strcmp ( path, ".md5" ) == 0 )
+ return sfa_preferred;
+ if ( MATCH ( path, "idx/fuse-" ) )
+ return sfa_exclude;
+ break;
+
+ case 's':
+ /* look for old skey index */
+ if ( MATCH ( path, "skey" ) )
+ if ( path [ 0 ] == 0 || strcmp ( path, ".md5" ) == 0 )
+ return sfa_preferred;
+ break;
+ }
+ /* anything not recognized is non-column required */
+ return sfa_non_column;
+}
+
+static
+enum sfa_path_type_id CC sfa_path_type_db ( const char *path )
+{
+ /* use first character as distinguisher for match */
+ switch ( path [ 0 ] )
+ {
+ case 't':
+ /* perhaps it's a table */
+ if ( MATCH ( path, "tbl/" ) )
+ {
+ switch ( path [ 0 ] )
+ {
+ case 0:
+ return sfa_non_column;
+ case 'S':
+ if ( MATCH ( path, "SEQUENCE/" ) )
+ return sfa_path_type_tbl(path);
+ break;
+ case 'C':
+ if ( MATCH ( path, "CONSENSUS/" ) )
+ return sfa_path_type_tbl(path);
+ break;
+ case 'P':
+ if ( MATCH ( path, "PRIMARY_ALIGNMENT/" ) )
+ return sfa_path_type_tbl(path);
+ break;
+ case 'R':
+ if ( MATCH ( path, "REFERENCE/" ) )
+ return sfa_path_type_tbl(path);
+ break;
+ }
+ /* all other tables are optional */
+ return sfa_optional;
+ }
+ }
+ /* anything not recognized is non-column required */
+ return sfa_non_column;
+}
+
+typedef enum sfa_path_type_id (CC *sfa_path_type_func)( const char *path );
+
+typedef
+struct to_nv_data_struct
+{
+ const KDirectory * d;
+ Vector * v;
+ rc_t rc;
+ sfa_path_type_func path_type;
+} to_nv_data;
+
+/* sfa_filter
+ * if a name is found in list, exclude it
+ */
+#define DEBUG_SORT(msg) DBGMSG (DBG_APP, DBG_FLAG(DBG_APP_0), msg)
+
+typedef struct reorder_t_struct {
+ const char * path;
+ uint64_t size;
+ enum sfa_path_type_id type_id;
+} reorder_t;
+
+static
+void CC to_nv (void * _item, void * _data)
+{
+ const char* path = _item;
+ to_nv_data* data = _data;
+ reorder_t* obj;
+
+ if (data->rc == 0)
+ {
+ obj = malloc (sizeof (*obj));
+ if (obj == NULL)
+ data->rc
+ = RC(rcSRA, rcVector, rcConstructing, rcMemory, rcExhausted);
+ else
+ {
+ rc_t rc = KDirectoryFileSize (data->d, &obj->size, "%s", path);
+ if (rc == 0)
+ {
+ obj->path = path;
+ obj->type_id = data->path_type(path);
+ rc = VectorAppend (data->v, NULL, obj);
+ }
+
+ if (rc)
+ {
+ free (obj);
+ data->rc = rc;
+ }
+ }
+ }
+}
+
+static
+int64_t CC sfa_path_cmp ( const void **_a, const void **_b, void * ignored )
+{
+ const reorder_t * a = *_a;
+ const reorder_t * b = *_b;
+ int ret;
+
+ DEBUG_SORT(("%s enter\t%s %s %lu \t%s %s %lu", __func__,
+ a->path, sfa_path_type_id[a->type_id + 1], a->size,
+ b->path, sfa_path_type_id[b->type_id + 1], b->size));
+
+ ret = a->type_id - b->type_id;
+ if (ret == 0)
+ {
+ if (a->size > b->size)
+ ret = 1;
+ else if (a->size < b->size)
+ ret = -1;
+ else
+ ret = strcmp (a->path, b->path);
+ }
+ DEBUG_SORT(("\t%d\n", ret));
+ return ret;
+}
+
+static
+void CC item_whack (void * item, void * ignored)
+{
+ free (item);
+}
+
+static
+rc_t CC sfa_sort(const KDirectory *dir, Vector *v, sfa_path_type_func func) {
+ /* assume "v" is a vector of paths - hopefully relative to "dir" */
+ Vector nv;
+ to_nv_data data;
+ uint32_t base;
+
+ DEBUG_SORT(("%s enter\n", __func__));
+
+ base = VectorStart (v);
+ VectorInit (&nv, base, VectorLength (v));
+ data.d = dir;
+ data.v = &nv;
+ data.rc = 0;
+ data.path_type = func;
+
+ VectorForEach (v, false, to_nv, &data);
+
+ if(data.rc == 0) {
+ uint32_t idx = 0;
+ uint32_t limit = VectorLength (v) + base;
+
+ VectorReorder(&nv, sfa_path_cmp, NULL);
+
+ for (idx = base; idx < limit; ++idx) {
+ const reorder_t * tmp;
+ void * ignore;
+
+ tmp = VectorGet (&nv, idx);
+ data.rc = VectorSwap (v, idx + base, tmp->path, &ignore);
+ if(data.rc) {
+ break;
+ }
+ }
+ }
+ VectorWhack (&nv, item_whack, NULL);
+ DEBUG_SORT(("%s exit %d %R\n", __func__, data.rc, data.rc));
+ return data.rc;
+}
+
+static rc_t CC sfa_sort_db(const KDirectory *dir, Vector *v) {
+ return sfa_sort(dir, v, sfa_path_type_db);
+}
+
+static
+bool CC sfa_filter(const KDirectory *dir, const char *leaf, void* func)
+{
+ bool ret = true;
+ sfa_path_type_func f = (sfa_path_type_func)func;
+ enum sfa_path_type_id type = f(leaf);
+
+ ret = type >= sfa_non_column;
+ DEBUG_SORT(("%s: %s %s %s\n",
+ __func__, leaf, sfa_path_type_id[type + 1], ret ? "keep" : "drop"));
+ return ret;
+}
+
+static
+bool CC sfa_filter_light(const KDirectory *dir, const char *leaf, void* func)
+{
+ bool ret = true;
+ sfa_path_type_func f = (sfa_path_type_func)func;
+
+ enum sfa_path_type_id type = f(leaf);
+
+ ret = type >= sfa_non_column && type < sfa_optional;
+ DEBUG_SORT(("%s: %s %s %s\n",
+ __func__, leaf, sfa_path_type_id[type + 1], ret ? "keep" : "drop"));
+ return ret;
+}
+
+static rc_t CC sfa_sort_tbl(const KDirectory *dir, Vector *v) {
+ return sfa_sort(dir, v, sfa_path_type_tbl);
+}
+
+rc_t CC VTableMakeSingleFileArchive(const VTable *self,
+ const KFile **sfa, bool lightweight)
+{
+ const VDatabase *db = NULL;
+ rc_t rc = VTableOpenParentRead(self, &db);
+
+ assert(sfa);
+
+ *sfa = NULL;
+
+ if (rc == 0 && db != NULL) {
+ const KDatabase *kdb = NULL;
+ const KDirectory *db_dir = NULL;
+ rc = VDatabaseOpenKDatabaseRead(db, &kdb);
+ DISP_RC(rc, "VDatabaseOpenKDatabaseRead");
+
+ if (rc == 0) {
+ rc = KDatabaseOpenDirectoryRead(kdb, &db_dir);
+ DISP_RC(rc, "KDatabaseOpenDirectoryRead");
+ }
+
+ if (rc == 0) {
+ rc = KDirectoryOpenTocFileRead(db_dir, sfa, sraAlign4Byte,
+ lightweight ? sfa_filter_light : sfa_filter,
+ (void *)sfa_path_type_db, sfa_sort_db);
+ }
+
+ RELEASE(KDirectory, db_dir);
+ RELEASE(KDatabase, kdb);
+ }
+ else {
+ const KTable *ktbl = NULL;
+ rc = VTableOpenKTableRead(self, &ktbl);
+ if (rc == 0) {
+ const KDirectory *tbl_dir = NULL;
+ rc = KTableGetDirectoryRead(ktbl, &tbl_dir);
+ if (rc == 0) {
+ rc = KDirectoryOpenTocFileRead(tbl_dir, sfa, sraAlign4Byte,
+ lightweight ? sfa_filter_light : sfa_filter,
+ (void *)sfa_path_type_tbl, sfa_sort_tbl);
+ KDirectoryRelease( tbl_dir );
+ KTableRelease(ktbl);
+ }
+ }
+ }
+
+ RELEASE(VDatabase, db);
+
+ return rc;
+}
+
+rc_t CC VCursorColumnRead(const VCursor *self, int64_t id,
+ uint32_t idx, const void **base, bitsz_t *offset, bitsz_t *size)
+{
+ uint32_t elem_bits = 0, elem_off = 0, elem_cnt = 0;
+ rc_t rc = VCursorCellDataDirect(self, id, idx,
+ &elem_bits, base, &elem_off, &elem_cnt);
+
+ assert(offset && size);
+
+ if (rc == 0) {
+ *offset = elem_off * elem_bits;
+ *size = elem_cnt * elem_bits;
+ }
+ else {
+ *offset = *size = 0;
+ }
+
+ return rc;
+}
diff --git a/tools/sra/.cvsignore b/tools/sra/.cvsignore
new file mode 100644
index 0000000..29f1a27
--- /dev/null
+++ b/tools/sra/.cvsignore
@@ -0,0 +1 @@
+*.vers.h
diff --git a/tools/sra/Makefile b/tools/sra/Makefile
index 3b0ef8f..cd98391 100644
--- a/tools/sra/Makefile
+++ b/tools/sra/Makefile
@@ -90,15 +90,8 @@ TABLE_VERS_OBJ = \
$(addsuffix .$(OBJX),$(TABLE_VERS_SRC))
TABLE_VERS_LIB = \
- -dvxf \
- -lvdb \
- -lkdb \
- -lvfs \
-lkapp \
- -lkfg \
- -lkfs \
- -lkrypto \
- -lklib \
+ -lncbi-vdb \
-lm
table-vers.vers.h: table-vers.vers
diff --git a/tools/sra/table-vers.c b/tools/sra/table-vers.c
index c41335f..c05e904 100644
--- a/tools/sra/table-vers.c
+++ b/tools/sra/table-vers.c
@@ -125,7 +125,7 @@ static rc_t CmdLineInit(const Args* args, CmdLine* cmdArgs)
break;
}
else if (pcount == 1) {
- rc = ArgsOptionValue(args, OPTION_OUTPUT, 0, &outType);
+ rc = ArgsOptionValue(args, OPTION_OUTPUT, 0, (const void **)&outType);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving output type");
break;
@@ -161,7 +161,7 @@ static rc_t CmdLineInit(const Args* args, CmdLine* cmdArgs)
LOGERR(klogErr, rc, "Too many table parameters");
break;
}
- rc = ArgsParamValue(args, 0, &cmdArgs->table);
+ rc = ArgsParamValue(args, 0, (const void **)&cmdArgs->table);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving table name");
break;
diff --git a/tools/sra/table-vers.vers b/tools/sra/table-vers.vers
index 8e8299d..35d16fb 100644
--- a/tools/sra/table-vers.vers
+++ b/tools/sra/table-vers.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/srapath/srapath.c b/tools/srapath/srapath.c
index bf60b87..7583f2c 100644
--- a/tools/srapath/srapath.c
+++ b/tools/srapath/srapath.c
@@ -53,6 +53,10 @@
#define PATH_MAX 4096
#endif
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
+
+
/* Version EXTERN
* return 4-part version code: 0xMMmmrrrr, where
* MM = major release
@@ -140,7 +144,8 @@ rc_t CC KMain ( int argc, char *argv [] )
if (acount == 0)
{
- rc = MiniUsage (args);
+ /* That useless: rc = */ MiniUsage (args);
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
break;
}
else
@@ -160,13 +165,13 @@ rc_t CC KMain ( int argc, char *argv [] )
for ( ix = 0; ix < acount; ++ ix )
{
const char * pc;
- rc = ArgsParamValue (args, ix, &pc );
+ rc = ArgsParamValue (args, ix, (const void **)&pc );
if (rc)
LOGERR (klogInt, rc,
"failed to retrieve parameter value");
else
{
- const VPath * upath;
+ const VPath * upath = NULL;
rc = VFSManagerMakePath ( mgr, (VPath**)&upath, "%s", pc);
if (rc == 0)
{
@@ -220,32 +225,34 @@ rc_t CC KMain ( int argc, char *argv [] )
}
}
if (orc == 0 && rc == 0) {
- if (rc != 0) {
- PLOGMSG(klogErr, (klogErr,
- "'$(name)': not found",
- "name=%s", pc));
- }
- else {
- char resolved[PATH_MAX] = "";
- rc = KDirectoryResolvePath(cwd, true,
- resolved, sizeof resolved, "%s", pc);
- if (rc == 0) {
- STSMSG(1, ("'%s': found in "
- "the current directory at '%s'",
- pc, resolved));
- OUTMSG (("%s\n", resolved));
+ if (rc != 0) {
+ PLOGMSG(klogErr, (klogErr,
+ "'$(name)': not found",
+ "name=%s", pc));
}
else {
- STSMSG(1, ("'%s': cannot resolve "
- "in the current directory",
- pc));
- OUTMSG (("./%s\n", pc));
+ char resolved[PATH_MAX] = "";
+ rc = KDirectoryResolvePath(cwd, true,
+ resolved, sizeof resolved, "%s", pc);
+ if (rc == 0) {
+ STSMSG(1, ("'%s': found in "
+ "the current directory at '%s'",
+ pc, resolved));
+ OUTMSG (("%s\n", resolved));
+ }
+ else {
+ STSMSG(1, ("'%s': cannot resolve "
+ "in the current directory",
+ pc));
+ OUTMSG (("./%s\n", pc));
+ }
}
}
- }
KDirectoryRelease(cwd);
}
}
+
+ RELEASE(VPath, upath);
}
}
VResolverRelease (resolver);
diff --git a/tools/srapath/srapath.vers b/tools/srapath/srapath.vers
index 8e8299d..35d16fb 100644
--- a/tools/srapath/srapath.vers
+++ b/tools/srapath/srapath.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/srapath/srapath.vers.h b/tools/srapath/srapath.vers.h
index 6718499..217c0dd 100644
--- a/tools/srapath/srapath.vers.h
+++ b/tools/srapath/srapath.vers.h
@@ -1 +1 @@
-#define SRAPATH_VERS 0x02040002
+#define SRAPATH_VERS 0x02050007
diff --git a/tools/update-schema/.cvsignore b/tools/update-schema/.cvsignore
new file mode 100644
index 0000000..29f1a27
--- /dev/null
+++ b/tools/update-schema/.cvsignore
@@ -0,0 +1 @@
+*.vers.h
diff --git a/tools/update-schema/Makefile b/tools/update-schema/Makefile
index 31b480e..f0acb72 100644
--- a/tools/update-schema/Makefile
+++ b/tools/update-schema/Makefile
@@ -90,8 +90,8 @@ UPDATE_SCHEMA_OBJ = \
$(addsuffix .$(OBJX),$(UPDATE_SCHEMA_SRC))
UPDATE_SCHEMA_LIB = \
- -lkapp \
- -lncbi-wvdb \
+ -skapp \
+ -sncbi-wvdb \
update-schema.vers.h: update-schema.vers
$(TOP)/build/make-vers-inc.sh UPDATE_SCHEMA_VERS $^ > $@
diff --git a/tools/update-schema/update-schema.c b/tools/update-schema/update-schema.c
index 962aef2..ac17f7a 100644
--- a/tools/update-schema/update-schema.c
+++ b/tools/update-schema/update-schema.c
@@ -66,7 +66,7 @@ ver_t CC KAppVersion(void) { return VDB_UPDATE_SCHEMA_VERS; }
#define OPTION_PLATF "platform"
#define ALIAS_PLATF "p"
static const char* platf_usage[] = { "Platform: LS454 | ILLUMINA | HELICOS",
- " | ABI_SOLID | COMPLETE_GENOMICS", " | PACBIO_SMRT | ION_TORRENT", NULL };
+ " | ABI_SOLID | COMPLETE_GENOMICS", " | PACBIO_SMRT | ION_TORRENT | CAPILLARY | OXFORD_NANOPORE", NULL };
#define OPTION_SCHEMA "schema"
#define ALIAS_SCHEMA "s"
static const char* schema_usage[] = { "Schema", NULL };
@@ -148,7 +148,7 @@ static rc_t CmdArgsInit(int argc, char** argv, CmdArgs* cmdArgs)
LOGERR(klogErr, rc, "Too many run parameters");
break;
}
- rc = ArgsOptionValue (args, OPTION_RUN, 0, &cmdArgs->run);
+ rc = ArgsOptionValue (args, OPTION_RUN, 0, (const void **)&cmdArgs->run);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving run name");
break;
@@ -169,7 +169,7 @@ static rc_t CmdArgsInit(int argc, char** argv, CmdArgs* cmdArgs)
if (pcount == 1) {
uint8_t p = SRA_PLATFORM_UNDEFINED;
const char* arg = NULL;
- rc = ArgsOptionValue(args, OPTION_PLATF, 0, &arg);
+ rc = ArgsOptionValue(args, OPTION_PLATF, 0, (const void **)&arg);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving platform");
break;
@@ -190,6 +190,10 @@ static rc_t CmdArgsInit(int argc, char** argv, CmdArgs* cmdArgs)
{ p = SRA_PLATFORM_PACBIO_SMRT; }
else if (!strcmp(arg, "ION_TORRENT"))
{ p = SRA_PLATFORM_ION_TORRENT; }
+ else if (!strcmp(arg, "CAPILLARY"))
+ { p = SRA_PLATFORM_CAPILLARY; }
+ else if (!strcmp(arg, "OXFORD_NANOPORE"))
+ { p = SRA_PLATFORM_OXFORD_NANOPORE; }
else {
rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
PLOGERR(klogInt, (klogInt, rc, "Invalid platform: $(name)",
@@ -212,7 +216,7 @@ static rc_t CmdArgsInit(int argc, char** argv, CmdArgs* cmdArgs)
break;
}
else if (pcount == 1) {
- rc = ArgsOptionValue(args, OPTION_SCHEMA, 0, &cmdArgs->schema);
+ rc = ArgsOptionValue(args, OPTION_SCHEMA, 0, (const void **)&cmdArgs->schema);
if (rc) {
LOGERR(klogErr, rc, "Failure retrieving schema");
break;
@@ -255,7 +259,7 @@ static rc_t CmdArgsGetNextParam(CmdArgs* args, const char** param)
*param = NULL;
if (args->i >= args->count) /* no more parameters to give :( */
{ return rc; }
- rc = ArgsParamValue(args->args, args->i, param);
+ rc = ArgsParamValue(args->args, args->i, (const void **)param);
if (rc) {
PLOGERR(klogInt, (klogInt, rc,
"while calling ArgsParamValue($(i))", "i=%d", args->i));
diff --git a/tools/update-schema/vdb-update-schema.vers b/tools/update-schema/vdb-update-schema.vers
index 8e8299d..35d16fb 100644
--- a/tools/update-schema/vdb-update-schema.vers
+++ b/tools/update-schema/vdb-update-schema.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/Makefile b/tools/util/Makefile
index 24f7f90..0e64e0a 100644
--- a/tools/util/Makefile
+++ b/tools/util/Makefile
@@ -37,6 +37,8 @@ INT_TOOLS = \
sortreadtest \
pacbio-correct \
crc32sum \
+ samview \
+ kdb-index
EXT_TOOLS = \
rcexplain \
@@ -82,7 +84,7 @@ $(TARGDIR)/std: \
#-------------------------------------------------------------------------------
# vers-includes
#
-$(TARGDIR)/vers-includes: $(addsuffix .vers.h,$(EXT_TOOLS))
+$(TARGDIR)/vers-includes: $(addsuffix .vers.h,$(EXT_TOOLS) kdb-index)
.PHONY: $(TARGDIR)/vers-includes
@@ -275,6 +277,25 @@ $(BINDIR)/crc32sum: $(CRC32SUM_OBJ)
$(LD) --exe -o $@ $^ $(CRC32SUM_LIB)
+#-------------------------------------------------------------------------------
+# samview
+# for testing bam/sam code
+#
+SAMVIEW_SRC = \
+ samview
+
+SAMVIEW_OBJ = \
+ $(addsuffix .$(OBJX),$(SAMVIEW_SRC))
+
+SAMVIEW_LIB = \
+ -lkapp \
+ -sncbi-vdb \
+
+
+$(BINDIR)/samview: $(SAMVIEW_OBJ)
+ $(LD) --exe -o $@ $^ $(SAMVIEW_LIB)
+
+
#----------------------------------------------------------------
# sortreadtest
#
@@ -345,3 +366,19 @@ PACBIO_CORRECT_LIB = \
$(BINDIR)/pacbio-correct: $(PACBIO_CORRECT_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(PACBIO_CORRECT_LIB)
+
+#-------------------------------------------------------------------------------
+# kdb-index
+#
+KDBINDEX_SRC = \
+ kdb-index
+
+KDBINDEX_OBJ = \
+ $(addsuffix .$(OBJX),$(KDBINDEX_SRC))
+
+KDBINDEX_LIB = \
+ -skapp \
+ -sncbi-vdb \
+
+$(BINDIR)/kdb-index: $(KDBINDEX_OBJ)
+ $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(KDBINDEX_LIB)
diff --git a/tools/util/crc32sum.c b/tools/util/crc32sum.c
index 887a177..d9d17ad 100644
--- a/tools/util/crc32sum.c
+++ b/tools/util/crc32sum.c
@@ -252,7 +252,7 @@ rc_t CC KMain (int argc, char * argv [])
const char *fname;
FILE *in;
- rc = ArgsParamValue (args, i, &fname);
+ rc = ArgsParamValue (args, i, (const void **)&fname);
in = fopen ( fname, bin ? "rb" : "r" );
diff --git a/tools/util/kdb-index.c b/tools/util/kdb-index.c
new file mode 100644
index 0000000..ba20b77
--- /dev/null
+++ b/tools/util/kdb-index.c
@@ -0,0 +1,264 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <kapp/main.h>
+#include <kdb/manager.h>
+#include <kdb/database.h>
+#include <kdb/table.h>
+#include <kdb/index.h>
+#include <klib/text.h>
+#include <klib/out.h>
+#include <klib/rc.h>
+
+#include <string.h>
+#include <stdio.h>
+
+#include "kdb-index.vers.h"
+
+static bool keys_only;
+static const char * app_name;
+
+static
+rc_t run_idx ( const KIndex * idx, const char * spec )
+{
+ rc_t rc = 0;
+ uint64_t span;
+ int64_t id, start_id;
+
+ for ( id = 1; ; id = start_id + span )
+ {
+ size_t ksize;
+ char key [ 256 ];
+
+ rc = KIndexProjectText ( idx, id, & start_id, & span, key, sizeof key, & ksize );
+ if ( rc != 0 )
+ {
+ if ( GetRCState ( rc ) == rcNotFound && GetRCObject ( rc ) == rcId )
+ rc = 0;
+ break;
+ }
+
+ if ( keys_only )
+ KOutMsg ( "%s\n", key );
+ else
+ KOutMsg ( "'%s': start = %'ld, span = %'lu\n", key, start_id, span );
+
+#if 0
+ /* just for the fun of it, now try to look up the key */
+ {
+ int64_t start2;
+ uint64_t count2;
+ rc_t rc2 = KIndexFindText ( idx, key, & start2, & count2, NULL, NULL );
+ if ( rc2 != 0 )
+ fprintf ( stderr, "failed to find key '%s' in index\n", key );
+ else
+ KOutMsg ( " key lookup returned start = %'ld, span = %'lu\n", start2, count2 );
+ }
+#endif
+ }
+
+ return rc;
+}
+
+static
+rc_t run_tbl ( const KTable * tbl, const char * spec, const char * idx_spec )
+{
+ const KIndex * idx;
+ rc_t rc = KTableOpenIndexRead ( tbl, & idx, "%s", idx_spec );
+ if ( rc == 0 )
+ {
+ rc = run_idx ( idx, spec );
+ KIndexRelease ( idx );
+ }
+
+ return rc;
+}
+
+static
+rc_t run_db ( const KDatabase * db, const char * spec, const char * tbl_idx )
+{
+ rc_t rc = 0;
+ const KTable * tbl;
+
+ size_t tlen = string_size ( tbl_idx );
+ const char * idx = string_chr ( tbl_idx, tlen, ':' );
+ if ( idx != NULL )
+ {
+ tlen = idx - tbl_idx;
+ ++ idx;
+ }
+
+ rc = KDatabaseOpenTableRead ( db, & tbl, "%.*s", ( uint32_t ) tlen, tbl_idx );
+ if ( rc == 0 )
+ {
+ rc = run_tbl ( tbl, spec, idx ? idx : "ref_spec" );
+ KTableRelease ( tbl );
+ }
+
+ return rc;
+}
+
+static
+rc_t run ( const KDBManager * mgr, const char * spec )
+{
+ rc_t rc = 0;
+
+ const KTable * tbl;
+ const KDatabase * db;
+
+ size_t slen = string_size ( spec );
+ const char * tbl_idx = string_chr ( spec, slen, ':' );
+ if ( tbl_idx != NULL )
+ {
+ slen = tbl_idx - spec;
+ ++ tbl_idx;
+ }
+
+ switch ( KDBManagerPathType ( mgr, "%.*s", ( uint32_t ) slen, spec ) )
+ {
+ case kptDatabase:
+ rc = KDBManagerOpenDBRead ( mgr, & db, "%.*s", ( uint32_t ) slen, spec );
+ if ( rc == 0 )
+ {
+ rc = run_db ( db, spec, tbl_idx ? tbl_idx : "STATS:ref_spec" );
+ KDatabaseRelease ( db );
+ }
+ break;
+ case kptTable:
+ rc = KDBManagerOpenTableRead ( mgr, & tbl, "%.*s", ( uint32_t ) slen, spec );
+ if ( rc == 0 )
+ {
+ rc = run_tbl ( tbl, spec, tbl_idx ? tbl_idx : "ref_spec" );
+ KTableRelease ( tbl );
+ }
+ break;
+ default:
+ rc = RC ( rcApp, rcMgr, rcAccessing, rcPath, rcIncorrect );
+ }
+ return rc;
+}
+
+ver_t CC KAppVersion ( void )
+{
+ return KDB_INDEX_VERS;
+}
+
+static
+void print_help ( void )
+{
+ size_t nlen = string_size ( app_name );
+ const char * short_name = string_rchr ( app_name, nlen, '/' );
+ if ( short_name ++ == NULL )
+ short_name = app_name;
+
+ KOutMsg ( "\n"
+ "Usage:\n"
+ " %s [ options ] obj-spec [ obj-spec.. ]\n"
+ "\n"
+ "Object Specification:\n"
+ " <db-path-or-accession> [ ':' <table-name> [ ':' <index-name> ] ]\n"
+ " <tbl-path-or-accession> [ ':' <index-name> ]\n"
+ "\n"
+ "Options:\n"
+ " -k|--keys-only only list keys\n"
+ " -h|--help print this message\n"
+ "\n"
+ "%s : %V\n"
+ "\n"
+ , short_name
+ , app_name
+ , KAppVersion ()
+ );
+}
+
+rc_t CC Usage ( struct Args const * args )
+{
+ return 0;
+}
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ rc_t rc = 0;
+ int i, count = 0;
+ const KDBManager * mgr;
+
+ app_name = argv [ 0 ];
+
+ for ( i = 1; i < argc; ++ i )
+ {
+ const char * arg = argv [ i ];
+ if ( arg [ 0 ] != '-' )
+ argv [ ++ count ] = ( char * ) arg;
+ else do switch ( ( ++ arg ) [ 0 ] )
+ {
+ case 'k':
+ keys_only = true;
+ break;
+ case 'h':
+ case '?':
+ print_help ();
+ return 0;
+ case '-':
+ ++ arg;
+ if ( strcmp ( arg, "keys-only" ) == 0 )
+ keys_only = true;
+ else if ( strcmp ( arg, "help" ) == 0 )
+ {
+ print_help ();
+ return 0;
+ }
+ else
+ {
+ fprintf ( stderr, "%s: unrecognized switch: '--%s'\n", argv [ 0 ], arg );
+ return -1;
+ }
+ arg = "\0";
+ break;
+ default:
+ fprintf ( stderr, "%s: unrecognized switch: '-%c'\n", argv [ 0 ], arg [ 0 ] );
+ return -1;
+ }
+ while ( arg [ 1 ] != 0 );
+ }
+
+ if ( count == 0 )
+ {
+ fprintf ( stderr, "%s: at least one object spec is required\n", argv [ 0 ] );
+ return -1;
+ }
+
+ rc = KDBManagerMakeRead ( & mgr, NULL );
+ if ( rc == 0 )
+ {
+
+ for ( i = 1; rc == 0 && i <= count; ++ i )
+ rc = run ( mgr, argv [ i ] );
+
+ KDBManagerRelease ( mgr );
+ }
+
+ return rc;
+}
diff --git a/tools/util/kdb-index.vers b/tools/util/kdb-index.vers
new file mode 100644
index 0000000..35d16fb
--- /dev/null
+++ b/tools/util/kdb-index.vers
@@ -0,0 +1 @@
+2.5.7
diff --git a/tools/util/kdbmeta.c b/tools/util/kdbmeta.c
index f403042..899f296 100644
--- a/tools/util/kdbmeta.c
+++ b/tools/util/kdbmeta.c
@@ -1158,7 +1158,7 @@ rc_t CC KMain ( int argc, char *argv [] )
break;
}
if (pcount) {
- rc = ArgsOptionValue (args, OPTION_TABLE, 0, &table_arg);
+ rc = ArgsOptionValue (args, OPTION_TABLE, 0, (const void **)&table_arg);
if (rc) {
LOGERR(klogErr, rc,
"Failure to get '" OPTION_TABLE "' argument");
@@ -1173,7 +1173,7 @@ rc_t CC KMain ( int argc, char *argv [] )
}
if (pcount) {
const char* dummy = NULL;
- rc = ArgsOptionValue (args, OPTION_OUT, 0, &dummy);
+ rc = ArgsOptionValue (args, OPTION_OUT, 0, (const void **)&dummy);
if (rc) {
LOGERR(klogErr, rc,
"Failure to get '" OPTION_OUT "' argument");
@@ -1214,7 +1214,7 @@ rc_t CC KMain ( int argc, char *argv [] )
else
{
/* get target */
- rc = ArgsParamValue (args, 0, &pc);
+ rc = ArgsParamValue (args, 0, (const void **)&pc);
if (rc)
LOGERR (klogInt, rc, "Unable to read target parameter");
else
@@ -1310,11 +1310,11 @@ rc_t CC KMain ( int argc, char *argv [] )
}
else for (ix = 1; ix < pcount; ++ix)
{
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
break;
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
break;
diff --git a/tools/util/kdbmeta.vers b/tools/util/kdbmeta.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/kdbmeta.vers
+++ b/tools/util/kdbmeta.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/kdbmeta.vers.h b/tools/util/kdbmeta.vers.h
index c16defb..a7a4be3 100644
--- a/tools/util/kdbmeta.vers.h
+++ b/tools/util/kdbmeta.vers.h
@@ -1 +1 @@
-#define KDBMETA_VERS 0x02040002
+#define KDBMETA_VERS 0x02050007
diff --git a/tools/util/md5cp.c b/tools/util/md5cp.c
index a0ccb25..548f167 100644
--- a/tools/util/md5cp.c
+++ b/tools/util/md5cp.c
@@ -556,7 +556,7 @@ rc_t run (Args * args)
break;
}
- rc = ArgsParamValue (args, 0, &outname);
+ rc = ArgsParamValue (args, 0, (const void **)&outname);
if (rc)
break;
@@ -576,7 +576,7 @@ rc_t run (Args * args)
for (ix = 1; ix < pcount; ++ix)
{
- rc = ArgsParamValue (args, ix, &source);
+ rc = ArgsParamValue (args, ix, (const void **)&source);
if (rc)
break;
@@ -617,7 +617,7 @@ rc_t run (Args * args)
break;
}
- rc = ArgsParamValue (args, 1, &source);
+ rc = ArgsParamValue (args, 1, (const void **)&source);
if (rc)
break;
@@ -641,7 +641,7 @@ rc_t run (Args * args)
break;
}
- rc = ArgsParamValue (args, 1, &source);
+ rc = ArgsParamValue (args, 1, (const void **)&source);
if (rc)
break;
diff --git a/tools/util/md5cp.vers b/tools/util/md5cp.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/md5cp.vers
+++ b/tools/util/md5cp.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/md5cp.vers.h b/tools/util/md5cp.vers.h
index 83cdbf6..eebf593 100644
--- a/tools/util/md5cp.vers.h
+++ b/tools/util/md5cp.vers.h
@@ -1 +1 @@
-#define MD5CP_VERS 0x02040002
+#define MD5CP_VERS 0x02050007
diff --git a/tools/util/pacbio-correct.c b/tools/util/pacbio-correct.c
index d5eb5c0..6db76a3 100644
--- a/tools/util/pacbio-correct.c
+++ b/tools/util/pacbio-correct.c
@@ -206,7 +206,7 @@ rc_t CC KMain ( int argc, char *argv [] )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *obj;
- rc = ArgsParamValue( args, idx, &obj );
+ rc = ArgsParamValue( args, idx, (const void **)&obj );
if ( rc != 0 )
LOGERR( klogErr, rc, "error reading commandline-parameter" );
else
diff --git a/tools/util/pacbio-correct.vers b/tools/util/pacbio-correct.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/pacbio-correct.vers
+++ b/tools/util/pacbio-correct.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/rcexplain.c b/tools/util/rcexplain.c
index c3fd4da..f6d63d0 100644
--- a/tools/util/rcexplain.c
+++ b/tools/util/rcexplain.c
@@ -119,7 +119,7 @@ rc_t CC KMain ( int argc, char *argv [] )
for (ix = 0; ix < pcount; ++ix)
{
- rc = ArgsParamValue (args, ix, &pc);
+ rc = ArgsParamValue (args, ix, (const void **)&pc);
if (rc)
break;
diff --git a/tools/util/rcexplain.vers b/tools/util/rcexplain.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/rcexplain.vers
+++ b/tools/util/rcexplain.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/rcexplain.vers.h b/tools/util/rcexplain.vers.h
index 3afa312..e27a8b3 100644
--- a/tools/util/rcexplain.vers.h
+++ b/tools/util/rcexplain.vers.h
@@ -1 +1 @@
-#define RCEXPLAIN_VERS 0x02040002
+#define RCEXPLAIN_VERS 0x02050007
diff --git a/tools/util/rowwritetest.c b/tools/util/rowwritetest.c
index 2758495..719f469 100644
--- a/tools/util/rowwritetest.c
+++ b/tools/util/rowwritetest.c
@@ -381,7 +381,7 @@ rc_t CC KMain ( int argc, char *argv[] )
{
const char * pc;
- ArgsOptionValue (args, OPTION_TABLE, 0, &pc);
+ ArgsOptionValue (args, OPTION_TABLE, 0, (const void **)&pc);
strncpy (tablePath, pc, sizeof (tablePath));
}
@@ -416,7 +416,7 @@ rc_t CC KMain ( int argc, char *argv[] )
{
const char * pc;
- rc = ArgsOptionValue (args, OPTION_ROW, 0, &pc);
+ rc = ArgsOptionValue (args, OPTION_ROW, 0, (const void **)&pc);
if (rc)
break;
diff --git a/tools/util/rowwritetest.vers b/tools/util/rowwritetest.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/rowwritetest.vers
+++ b/tools/util/rowwritetest.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/sra-sort/idx-mapping.h b/tools/util/samview.c
similarity index 50%
copy from tools/sra-sort/idx-mapping.h
copy to tools/util/samview.c
index 7691526..e3ef60a 100644
--- a/tools/sra-sort/idx-mapping.h
+++ b/tools/util/samview.c
@@ -1,4 +1,4 @@
-/*===========================================================================
+/* ===========================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
@@ -24,36 +24,62 @@
*
*/
-#ifndef _h_sra_sort_idx_mapping_
-#define _h_sra_sort_idx_mapping_
+#include <kapp/args.h>
+#include <kapp/main.h>
+#include <klib/log.h>
-#ifndef _h_sra_sort_defs_
-#include "sort-defs.h"
-#endif
+#include <stdlib.h>
+#include <stdio.h>
+#include <align/bam.h>
-/*--------------------------------------------------------------------------
- * IdxMapping
- *
- */
-typedef struct IdxMapping IdxMapping;
-struct IdxMapping
+static
+void samview(char const file[])
{
- int64_t old_id, new_id;
-};
+ char buffer[64*1024];
+ BAMFile const *bam;
+ rc_t rc = BAMFileMake(&bam, file);
+
+ if (rc == 0) {
+ char const *header;
+ size_t hsize;
+ BAMAlignment const *rec;
-#if USE_OLD_KSORT
+ BAMFileGetHeaderText(bam, &header, &hsize);
+ fwrite(header, 1, hsize, stdout);
-/* ksort callbacks */
-int CC IdxMappingCmpOld ( const void *a, const void *b, void *data );
-int CC IdxMappingCmpNew ( const void *a, const void *b, void *data );
+ while ((rc = BAMFileRead2(bam, &rec)) == 0) {
+ size_t actsize = 0;
-#else
+ if (BAMAlignmentFormatSAM(rec, &actsize, sizeof(buffer), buffer) != 0)
+ break;
+ fwrite(buffer, 1, actsize, stdout);
+ BAMAlignmentRelease(rec);
+ }
+ BAMFileRelease(bam);
+ }
+ LOGERR(klogWarn, rc, "Final RC");
+}
-/* ksort_inlines */
-void IdxMappingSortOld ( IdxMapping *self, const ctx_t *ctx, size_t count );
-void IdxMappingSortNew ( IdxMapping *self, const ctx_t *ctx, size_t count );
+rc_t CC UsageSummary(char const *name)
+{
+ return 0;
+}
-#endif
+rc_t CC Usage(Args const *args)
+{
+ return 0;
+}
-#endif /* _h_sra_sort_idx_mapping_ */
+ver_t CC KAppVersion(void)
+{
+ return 0;
+}
+
+rc_t CC KMain(int argc, char *argv[])
+{
+ while (--argc) {
+ samview(*++argv);
+ }
+ return 0;
+}
diff --git a/tools/util/schema-replace.c b/tools/util/schema-replace.c
index 53137cb..36791f6 100644
--- a/tools/util/schema-replace.c
+++ b/tools/util/schema-replace.c
@@ -132,7 +132,7 @@ static const char* get_str_option( const Args *my_args,
rc_t rc = ArgsOptionCount( my_args, name, &count );
if ( ( rc == 0 )&&( count > 0 ) )
{
- rc = ArgsOptionValue( my_args, name, 0, &res );
+ rc = ArgsOptionValue( my_args, name, 0, (const void **)&res );
}
return res;
}
@@ -226,7 +226,7 @@ static rc_t process_files( p_ctx ctx, Args * args )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *csra = NULL;
- rc = ArgsParamValue( args, idx, &csra );
+ rc = ArgsParamValue( args, idx, (const void **)&csra );
if ( rc == 0 )
{
OUTMSG(( "\nprocessing: %s\n", csra ));
diff --git a/tools/util/schema-replace.vers b/tools/util/schema-replace.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/schema-replace.vers
+++ b/tools/util/schema-replace.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/schema-replace.vers.h b/tools/util/schema-replace.vers.h
index c00938b..a617fcb 100644
--- a/tools/util/schema-replace.vers.h
+++ b/tools/util/schema-replace.vers.h
@@ -1 +1 @@
-#define SCHEMA_REPLACE_VERS 0x02040002
+#define SCHEMA_REPLACE_VERS 0x02050007
diff --git a/tools/util/sortreadtest.c b/tools/util/sortreadtest.c
index 1342791..4ee8309 100644
--- a/tools/util/sortreadtest.c
+++ b/tools/util/sortreadtest.c
@@ -317,14 +317,14 @@ rc_t CC KMain ( int argc, char *argv[] )
break;
}
- rc = ArgsParamValue (args, 0, &pb.file_path);
+ rc = ArgsParamValue (args, 0, (const void **)&pb.file_path);
if (rc)
{
LOGERR (klogInt,rc, "Failed to get file path");
break;
}
- rc = ArgsParamValue (args, 1, &pb.table_path);
+ rc = ArgsParamValue (args, 1, (const void **)&pb.table_path);
if (rc)
{
LOGERR (klogInt,rc, "Failed to get table path");
@@ -332,7 +332,7 @@ rc_t CC KMain ( int argc, char *argv[] )
}
if (pb.schema_path == NULL)
{
- rc = ArgsParamValue (args, 2, &pb.schema_path);
+ rc = ArgsParamValue (args, 2, (const void **)&pb.schema_path);
if (rc)
{
LOGERR (klogInt,rc, "Failed to get schema path");
diff --git a/tools/util/test-sra.c b/tools/util/test-sra.c
index 0ecca77..e0db407 100644
--- a/tools/util/test-sra.c
+++ b/tools/util/test-sra.c
@@ -25,6 +25,7 @@
*/
#include "test-sra-priv.h" /* PrintOS */
+#include "test-sra.vers.h" /* TEST_SRA_VERS */
#include <kapp/main.h> /* KMain */
@@ -32,6 +33,13 @@
#include <kfg/kart.h> /* Kart */
#include <kfg/repository.h> /* KRepositoryMgr */
+#include <kfs/dyload.h> /* KDyld */
+
+#include <klib/time.h> /* KTimeMsStamp */
+
+#include <kns/endpoint.h> /* KEndPoint */
+#include <kns/kns-mgr-priv.h> /* KNSManagerMakeReliableClientRequest */
+
#include <vfs/manager.h> /* VFSManager */
#include <vfs/path.h> /* VPath */
#include <vfs/resolver.h> /* VResolver */
@@ -44,6 +52,7 @@
#include <vdb/report.h> /* ReportSetVDBManager */
#include <vdb/schema.h> /* VSchemaRelease */
#include <vdb/table.h> /* VDBManagerOpenTableRead */
+#include <vdb/vdb-priv.h> /* VDBManagerSetResolver */
#include <kns/ascp.h> /* ascp_locate */
#include <kns/http.h>
@@ -84,34 +93,38 @@ VFS_EXTERN rc_t CC VResolverProtocols ( VResolver * self,
if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
typedef enum {
- eCfg = 1,
-/* eType = 2, */
- eResolve = 2,
- eDependMissing = 4,
- eDependAll = 8,
-/* eCurl = 16, */
- eVersion = 32,
- eNewVersion = 64,
- eOpenTable = 128,
- eOpenDB = 256,
- eType = 512,
- eNcbiReport = 1024,
- eOS = 2048,
- eAscp = 4096,
+ eCfg = 1,
+ eResolve = 2,
+ eDependMissing = 4,
+ eDependAll = 8,
+ eNetwork = 16,
+ eVersion = 32,
+ eNewVersion = 64,
+ eOpenTable = 128,
+ eOpenDB = 256,
+ eType = 512,
+ eNcbiReport = 1024,
+ eOS = 2048,
+ eAscp = 4096,
eAscpVerbose = 8192,
- eAll = 16384,
- eNoTestArg = 32768
+ eNgs = 16384,
+ eAll = 32768,
+ eNoTestArg = 65536,
} Type;
typedef uint16_t TTest;
-static const char TESTS[] = "crdDksSoOtnufFa";
+static const char TESTS[] = "crdDwsSoOtnufFga";
typedef struct {
KConfig *cfg;
KDirectory *dir;
KNSManager *knsMgr;
+
const VDBManager *mgr;
+ uint32_t projectId;
+
VFSManager *vMgr;
const KRepositoryMgr *repoMgr;
VResolver *resolver;
+ VSchema *schema;
TTest tests;
bool recursive;
@@ -119,11 +132,12 @@ typedef struct {
bool noRfs;
bool full;
bool xml;
+ bool network;
bool allowCaching;
VResolverEnableState cacheState;
} Main;
-uint32_t CC KAppVersion(void) { return 0; }
+uint32_t CC KAppVersion(void) { return TEST_SRA_VERS; }
const char UsageDefaultName[] = "test-sra";
@@ -168,8 +182,8 @@ rc_t CC Usage(const Args *args) {
" n - print NCBI error report\n"
" f - print ascp information\n"
" F - print verbose ascp information\n"
- " t - print object types\n");
-// " k - print curl info\n"
+ " t - print object types\n"
+ " w - run network test\n");
if (rc == 0 && rc2 != 0) {
rc = rc2;
}
@@ -390,6 +404,24 @@ static rc_t MainInitObjects(Main *self) {
if (rc == 0) {
rc = KConfigMakeRepositoryMgrRead(self->cfg, &self->repoMgr);
+ if (rc == 0) {
+ bool has_project_id = self->projectId != 0;
+ if (has_project_id) {
+ const KRepository *repository = NULL;
+ rc = KRepositoryMgrGetProtectedRepository(
+ self->repoMgr, self->projectId, &repository);
+ if (rc == 0) {
+ VResolver *resolver = NULL;
+ rc = KRepositoryMakeResolver(
+ repository, &resolver, self->cfg);
+ if (rc == 0) {
+ rc = VDBManagerSetResolver(self->mgr, resolver);
+ }
+ RELEASE(VResolver, resolver);
+ }
+ RELEASE(KRepository, repository);
+ }
+ }
}
if (rc == 0) {
@@ -414,6 +446,12 @@ static rc_t MainInitObjects(Main *self) {
rc = VFSManagerMakeResolver(self->vMgr, &self->resolver, self->cfg);
}
+ if (rc == 0) {
+ rc = VDBManagerMakeSRASchema(self->mgr, &self->schema);
+ if (rc != 0) {
+ OUTMSG(("VDBManagerMakeSRASchema() = %R\n", rc));
+ }
+ }
RELEASE(VResolver, resolver);
@@ -517,13 +555,15 @@ static rc_t MainCallCgiImpl(const Main *self,
uint64_t avail = result . elem_count - total;
if ( avail < 256 )
{
- rc = KDataBufferResize ( & result, result . elem_count + 4096 );
+ rc = KDataBufferResize
+ (&result, result.elem_count + 4096);
if ( rc != 0 )
break;
}
base = result . base;
- rc = KStreamRead ( response, & base [ total ], result . elem_count - total, & num_read );
+ rc = KStreamRead(response, &base[total],
+ result.elem_count - total, &num_read);
if ( rc != 0 )
{
if ( num_read > 0 )
@@ -868,6 +908,36 @@ static rc_t _VDBManagerReport(const VDBManager *self,
return _KDBPathTypePrint("", *type, " ");
}
+static rc_t _VDBManagerReportRemote
+ (const VDBManager *self, const char *name, const VSchema *schema)
+{
+ bool notFound = false;
+ const VDatabase *db = NULL;
+ const VTable *tbl = NULL;
+ rc_t rc = VDBManagerOpenDBRead(self, &db, schema, name);
+ if (rc == 0) {
+ RELEASE(VDatabase, db);
+ return _KDBPathTypePrint("", kptDatabase, " ");
+ }
+ else if (GetRCState(rc) == rcNotFound) {
+ notFound = true;
+ }
+ rc = VDBManagerOpenTableRead(self, &tbl, schema, name);
+ if (rc == 0) {
+ RELEASE(VTable, tbl);
+ return _KDBPathTypePrint("", kptTable, " ");
+ }
+ else if (GetRCState(rc) == rcNotFound) {
+ notFound = true;
+ }
+ if (notFound) {
+ return OUTMSG(("NotFound "));
+ }
+ else {
+ return OUTMSG(("Unknown "));
+ }
+}
+
static
rc_t _KDirectoryFileHeaderReport(const KDirectory *self, const char *path)
{
@@ -924,25 +994,34 @@ static rc_t MainReport(const Main *self,
return rc;
}
+static rc_t MainReportRemote(const Main *self, const char *name, int64_t size) {
+ rc_t rc = 0;
+
+ assert(self);
+
+ OUTMSG(("%,lu ", size));
+
+ if (!self->noVDBManagerPathType) {
+ _VDBManagerReportRemote(self->mgr, name, self->schema);
+ }
+
+ return rc;
+}
+
static rc_t MainOpenAs(const Main *self, const char *name, bool isDb) {
rc_t rc = 0;
const VTable *tbl = NULL;
const VDatabase *db = NULL;
- VSchema *schema = NULL;
- assert(self);
- rc = VDBManagerMakeSRASchema(self->mgr, &schema);
- if (rc != 0) {
- OUTMSG(("VDBManagerMakeSRASchema() = %R\n", rc));
- }
+ assert(self);
if (isDb) {
- rc = VDBManagerOpenDBRead(self->mgr, &db, schema, "%s", name);
+ rc = VDBManagerOpenDBRead(self->mgr, &db, self->schema, "%s", name);
ReportResetDatabase(name, db);
OUTMSG(("VDBManagerOpenDBRead(%s) = ", name));
}
else {
- rc = VDBManagerOpenTableRead(self->mgr, &tbl, schema, "%s", name);
+ rc = VDBManagerOpenTableRead(self->mgr, &tbl, self->schema, "%s", name);
ReportResetTable(name, tbl);
OUTMSG(("VDBManagerOpenTableRead(%s) = ", name));
}
@@ -954,7 +1033,6 @@ static rc_t MainOpenAs(const Main *self, const char *name, bool isDb) {
}
RELEASE(VDatabase, db);
RELEASE(VTable, tbl);
- RELEASE(VSchema, schema);
return rc;
}
@@ -982,20 +1060,24 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
bool fasp, const KFile *fRemote)
{
const char *eol = "\n";
+ const char *bol = "";
assert(self);
- eol = self->xml ? "<br/>\n" : "\n";
+ if (self->xml) {
+ eol = "<br/>\n";
+ bol = " ";
+ }
switch (type) {
case ePathLocal:
- OUTMSG(("Local:\t\t "));
+ OUTMSG(("%sLocal:\t\t ", bol));
break;
case ePathRemote:
- OUTMSG(("Remote %s:\t ", fasp ? "fasp" : "http"));
+ OUTMSG(("%sRemote %s:\t ", bol, fasp ? "FaspHttp" : "HttpFasp"));
break;
case ePathCache:
- OUTMSG(("Cache %s:\t ", fasp ? "fasp" : "http"));
+ OUTMSG(("%sCache %s:\t ", bol, fasp ? "FaspHttp" : "HttpFasp"));
if (remote == NULL) {
OUTMSG(("skipped%s", eol));
return rc;
@@ -1051,7 +1133,7 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
OUTMSG(("KFileSize(%s)=%R ", name, rc));
}
else {
- OUTMSG(("%,lu ", sz));
+ MainReportRemote(self, fPath, sz);
*size = sz;
}
}
@@ -1089,7 +1171,8 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
}
if (rc == 0) {
- OUTMSG(("Cache.cache %s: ", fasp ? "fasp" : "http"));
+ OUTMSG((
+ "%sCache.cache %s: ", bol, fasp ? "FaspHttp" : "HttpFasp"));
OUTMSG(("%s ", cachecache));
rc = MainReport(self, cachecache, NULL, NULL, NULL);
OUTMSG(("%s", eol));
@@ -1112,8 +1195,6 @@ static rc_t MainResolveLocal(const Main *self, const VResolver *resolver,
if (resolver == NULL) {
resolver = self->resolver;
}
-/*
- OUTMSG(("Local: "));*/
rc = VResolverLocal(resolver, acc, &local);
rc = MainPathReport(self,
@@ -1165,7 +1246,7 @@ static rc_t MainResolveRemote(const Main *self, VResolver *resolver,
}
}
}
-
+
rc = MainPathReport(self,
rc, *remote, ePathRemote, name, NULL, size, fasp, f);
RELEASE(KFile, f);
@@ -1217,6 +1298,7 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
bool fasp, VRemoteProtocols protocols, EQueryType type)
{
const char *eol = "\n";
+ const char *bol = "";
rc_t rc = 0;
@@ -1229,7 +1311,10 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
assert(self);
- eol = self->xml ? "<br/>\n" : "\n";
+ if (self->xml) {
+ eol = "<br/>\n";
+ bol = " ";
+ }
switch (type) {
case eQueryLocal:
@@ -1253,8 +1338,8 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
}
rc = VResolverQuery(resolver, protocols, query, pLocal, pRemote, pCache);
- OUTMSG(("\nVResolverQuery(%s, %s, local%s, remote%s, cache%s)= %R%s",
- name, protocols == eProtocolHttp ? "Http" : "FaspHttp",
+ OUTMSG(("%sVResolverQuery(%s, %s, local%s, remote%s, cache%s)= %R%s",
+ bol, name, protocols == eProtocolHttp ? "Http" : "FaspHttp",
pLocal == NULL ? "=NULL" : "", pRemote == NULL ? "=NULL" : "",
pCache == NULL ? "=NULL" : "", rc, eol));
if (rc == 0) {
@@ -1267,11 +1352,13 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
rc, remote, ePathRemote, name, NULL, NULL, fasp, NULL);
}
if (cache != NULL) {
-/* rc2 =*/ MainPathReport(self,
+ MainPathReport(self,
rc, cache, ePathCache, name, remote, NULL, fasp, NULL);
}
}
+ OUTMSG(("\n"));
+
RELEASE(VPath, local);
RELEASE(VPath, remote);
RELEASE(VPath, cache);
@@ -1286,8 +1373,11 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
static rc_t MainResolveQuery(const Main *self, const VResolver *resolver,
const char *name, const VPath *query, bool fasp)
{
+ const char root[] = "Timer";
rc_t rc = 0;
rc_t rc2 = 0;
+ KTimeMs_t time = 0;
+ KTimeMs_t start_time = 0;
VRemoteProtocols protocols = eProtocolHttp;
if (fasp) {
@@ -1307,6 +1397,14 @@ static rc_t MainResolveQuery(const Main *self, const VResolver *resolver,
rc = rc2;
}
+ if (self->xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ else {
+ OUTMSG(("time started...\n"));
+ }
+ start_time = KTimeMsStamp();
+
rc2 = VResolverQueryByType(self, resolver, name, query, fasp, protocols,
eQueryLocal);
if (rc2 != 0 && rc == 0) {
@@ -1319,9 +1417,45 @@ static rc_t MainResolveQuery(const Main *self, const VResolver *resolver,
rc = rc2;
}
+/* TODO check vdbcache */
+
+ time = KTimeMsStamp() - start_time;
+ if (self->xml) {
+ OUTMSG((" <Time time=\"%d ms\"/>\n", time));
+ OUTMSG((" </%s>\n", root));
+ }
+ else {
+ OUTMSG(("...elapsed time=\"%d ms\"\n\n", time));
+ }
+
return rc;
}
+static rc_t _KartItemToVPath(const KartItem *self,
+ const VFSManager *vfs, VPath **path)
+{
+ uint64_t oid = 0;
+ rc_t rc = KartItemItemIdNumber(self, &oid);
+ if (rc == 0) {
+ rc = VFSManagerMakeOidPath(vfs, path, (uint32_t)oid);
+ }
+ else {
+ char path_str[PATH_MAX] = "";
+ const String *accession = NULL;
+ rc = KartItemAccession(self, &accession);
+ if (rc == 0) {
+ rc =
+ string_printf(path_str, sizeof path_str, NULL, "%S", accession);
+ }
+ if (rc == 0) {
+ rc = VFSManagerMakePath(vfs, path, path_str);
+ }
+ }
+ return rc;
+}
+
+static rc_t perform_read_test(void) { return 0; }
+
static rc_t MainResolve(const Main *self, const KartItem *item,
const char *name, int64_t *localSz, int64_t *remoteSz, bool refseqCtx)
{
@@ -1352,7 +1486,6 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
}
else {
const KRepository *p_protected = NULL;
- uint64_t oid = 0;
uint64_t project = 0;
if (rc == 0) {
rc = KartItemProjIdNumber(item, &project);
@@ -1361,15 +1494,9 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
}
}
if (rc == 0) {
- rc = KartItemItemIdNumber(item, &oid);
- if (rc != 0) {
- OUTMSG(("KartItemItemIdNumber = %R\n", rc));
- }
- }
- if (rc == 0) {
- rc = VFSManagerMakeOidPath(self->vMgr, &acc, (uint32_t)oid);
+ rc = _KartItemToVPath(item, self->vMgr, &acc);
if (rc != 0) {
- OUTMSG(("VFSManagerMakePath(%d) = %R\n", oid, rc));
+ OUTMSG(("Invalid kart file row: %R\n", rc));
}
}
if (rc == 0) {
@@ -1394,9 +1521,34 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
}
if (rc == 0) {
+ const String *id = NULL;
+ const char root[] = "Query";
+ rc_t rc2 = 0;
+
const VPath* remote = NULL;
- rc_t rc2 = MainResolveLocal(self, resolver, name, acc, localSz);
+ const char *attr = name;
+ if (attr == NULL) {
+ rc2 = KartItemItemId(item, &id);
+ if (rc2 != 0) {
+ OUTMSG(("KartItemItemId = %R\n", rc2));
+ rc2 = 0;
+ attr = "Accession";
+ }
+ }
+
+ if (self->xml) {
+ if (attr != NULL) {
+ OUTMSG((" <%s name=\"%s\">\n", root, attr));
+ }
+ else {
+ OUTMSG((" <%s name=\"%S\">\n", root, id));
+ }
+ }
+
+ rc = VDBManagerSetResolver(self->mgr, resolver);
+
+ rc2 = MainResolveLocal(self, resolver, name, acc, localSz);
if (rc2 != 0 && rc == 0) {
rc = rc2;
}
@@ -1412,6 +1564,8 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
rc = rc2;
}
+ RELEASE(VPath, remote);
+
rc2 = MainResolveRemote(self, resolver, name, acc, &remote, remoteSz,
true);
if (rc2 != 0 && rc == 0) {
@@ -1423,6 +1577,8 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
rc = rc2;
}
+ OUTMSG(("\n"));
+
rc2 = MainResolveQuery(self, resolver, name, acc, false);
if (rc2 != 0 && rc == 0) {
rc = rc2;
@@ -1434,6 +1590,14 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
}
RELEASE(VPath, remote);
+
+ if (MainHasTest(self, eNetwork)) {
+ perform_read_test();
+ }
+
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
}
RELEASE(VPath, acc);
@@ -1465,7 +1629,7 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
}
if (rc == 0) {
- rc = VDBManagerOpenDBRead(self->mgr, &db, NULL, "%s", name);
+ rc = VDBManagerOpenDBRead(self->mgr, &db, self->schema, "%s", name);
if (rc != 0) {
if (rc == SILENT_RC(rcVFS,rcMgr,rcOpening,rcDirectory,rcNotFound)) {
return 0;
@@ -1670,10 +1834,10 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
if (MainHasTest(self, eResolve) && seqId != NULL) {
int64_t remoteSz = 0;
OUTMSG(("%s", eol));
- rc2 = MainResolve(self, NULL, seqId, NULL, &remoteSz, true);
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
- }
+
+ /* ignore returned value :
+ resolver's errors are detected but not reported as test-sra's failure */
+ MainResolve(self, NULL, seqId, NULL, &remoteSz, true);
}
if (self->xml) {
@@ -1814,6 +1978,442 @@ static rc_t PrintCurl(bool full, bool xml) {
#define kptKartITEM (kptAlias - 1)
+static rc_t _KartItemPrint(const KartItem *self, bool xml) {
+ const char root[] = "KartRow";
+ const String *elem = NULL;
+ if (xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ {
+ const char root[] = "ProjId";
+ rc_t rc = KartItemProjId(self, &elem);
+ if (rc != 0) {
+ OUTMSG(("KartItem%s = %R\n", root, rc));
+ }
+ else if (xml) {
+ OUTMSG((" <%s>%S</%s>\n", root, elem, root));
+ }
+ else {
+ OUTMSG(("%s: %S\n", root, elem));
+ }
+ }
+ {
+ const char root[] = "ItemId";
+ rc_t rc = KartItemItemId(self, &elem);
+ if (rc != 0) {
+ OUTMSG(("KartItem%s = %R\n", root, rc));
+ }
+ else if (xml) {
+ OUTMSG((" <%s>%S</%s>\n", root, elem, root));
+ }
+ else {
+ OUTMSG(("%s: %S\n", root, elem));
+ }
+ }
+ {
+ const char root[] = "Accession";
+ rc_t rc = KartItemAccession(self, &elem);
+ if (rc != 0) {
+ OUTMSG(("KartItem%s = %R\n", root, rc));
+ }
+ else if (xml) {
+ OUTMSG((" <%s>%S</%s>\n", root, elem, root));
+ }
+ else {
+ OUTMSG(("%s: %S\n", root, elem));
+ }
+ }
+ {
+ const char root[] = "Name";
+ rc_t rc = KartItemName(self, &elem);
+ if (rc != 0) {
+ OUTMSG(("KartItem%s = %R\n", root, rc));
+ }
+ else if (xml) {
+ OUTMSG((" <%s>%S</%s>\n", root, elem, root));
+ }
+ else {
+ OUTMSG(("%s: %S\n", root, elem));
+ }
+ }
+ {
+ const char root[] = "ItemDesc";
+ rc_t rc = KartItemItemDesc(self, &elem);
+ if (rc != 0) {
+ OUTMSG(("KartItem%s = %R\n", root, rc));
+ }
+ else if (xml) {
+ OUTMSG((" <%s>%S</%s>\n", root, elem, root));
+ }
+ else {
+ OUTMSG(("%s: %S\n", root, elem));
+ }
+ }
+ if (xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ return 0;
+}
+
+/*static rc_t _KartPrint(const Kart *self, bool xml) {
+ const char root[] = "Kart";
+ if (xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ rc_t rc = KartPrint(self);
+ if (rc != 0) {
+ OUTMSG(("KartPrint = %R\n", rc));
+ }
+ if (xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ return 0;
+}
+static rc_t _KartPrintSized(const Kart *self, bool xml) {
+ return 0;
+}
+*/
+
+static rc_t _KartPrintNumbered(const Kart *self, bool xml) {
+ const char root[] = "Kart";
+ if (xml) {
+ OUTMSG((" <%s numbered=\"true\">\n", root));
+ }
+ {
+ rc_t rc = KartPrintNumbered(self);
+ if (rc != 0) {
+ OUTMSG(("KartPrint = %R\n", rc));
+ }
+ }
+ if (xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ return 0;
+}
+
+
+static rc_t ipv4_endpoint_to_string(char *buffer, size_t buflen, KEndPoint *ep)
+{
+ uint32_t b[4];
+ b[0] = ( ep->u.ipv4.addr & 0xFF000000 ) >> 24;
+ b[1] = ( ep->u.ipv4.addr & 0xFF0000 ) >> 16;
+ b[2] = ( ep->u.ipv4.addr & 0xFF00 ) >> 8;
+ b[3] = ( ep->u.ipv4.addr & 0xFF );
+ return string_printf( buffer, buflen, NULL, "ipv4: %d.%d.%d.%d : %d",
+ b[0], b[1], b[2], b[3], ep->u.ipv4.port );
+}
+
+static rc_t ipv6_endpoint_to_string(char *buffer, size_t buflen, KEndPoint *ep)
+{
+ uint32_t b[8];
+ b[0] = ( ep->u.ipv6.addr[ 0 ] << 8 ) | ep->u.ipv6.addr[ 1 ];
+ b[1] = ( ep->u.ipv6.addr[ 2 ] << 8 ) | ep->u.ipv6.addr[ 3 ];
+ b[2] = ( ep->u.ipv6.addr[ 4 ] << 8 ) | ep->u.ipv6.addr[ 5 ];
+ b[3] = ( ep->u.ipv6.addr[ 6 ] << 8 ) | ep->u.ipv6.addr[ 7 ];
+ b[4] = ( ep->u.ipv6.addr[ 8 ] << 8 ) | ep->u.ipv6.addr[ 9 ];
+ b[5] = ( ep->u.ipv6.addr[ 10 ] << 8 ) | ep->u.ipv6.addr[ 11 ];
+ b[6] = ( ep->u.ipv6.addr[ 12 ] << 8 ) | ep->u.ipv6.addr[ 13 ];
+ b[7] = ( ep->u.ipv6.addr[ 14 ] << 8 ) | ep->u.ipv6.addr[ 15 ];
+ return string_printf( buffer, buflen, NULL,
+ "ipv6: %.04X:%.04X:%.04X:%.04X:%.04X:%.04X:%.04X:%.04X: :%d",
+ b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], ep->u.ipv6.port );
+}
+
+static rc_t ipc_endpoint_to_string(char *buffer, size_t buflen, KEndPoint *ep)
+{
+ return string_printf( buffer, buflen, NULL, "ipc: %s", ep->u.ipc_name );
+}
+
+static rc_t endpoint_to_string( char * buffer, size_t buflen, KEndPoint * ep )
+{
+ rc_t rc;
+ switch( ep->type )
+ {
+ case epIPV4 : rc = ipv4_endpoint_to_string( buffer, buflen, ep ); break;
+ case epIPV6 : rc = ipv6_endpoint_to_string( buffer, buflen, ep ); break;
+ case epIPC : rc = ipc_endpoint_to_string( buffer, buflen, ep ); break;
+ default : rc = string_printf( buffer, buflen, NULL,
+ "unknown endpoint-tyep %d", ep->type ); break;
+ }
+ return rc;
+}
+
+static rc_t perfrom_dns_test(const Main *self, const char *eol) {
+ rc_t rc = 0;
+ const char domain[] = "www.ncbi.nlm.nih.gov";
+ const uint16_t port = 80;
+ KEndPoint ep;
+ String s_domain;
+ KTimeMs_t time = 0;
+ KTimeMs_t start_time = KTimeMsStamp();
+ assert(self);
+ StringInitCString(&s_domain, domain);
+ rc = KNSManagerInitDNSEndpoint(self->knsMgr, &ep, &s_domain, port);
+ time = KTimeMsStamp() - start_time;
+ if (rc != 0) {
+ OUTMSG
+ (("KNSManagerInitDNSEndpoint(%s, %d)=%R%s", domain, port, rc, eol));
+ }
+ else {
+ const char root[] = "DnsEndpoint";/*"www.ncbi.nlm.nih.gov", 80*/
+ char s_endpoint[1024] = "";
+ rc = endpoint_to_string(s_endpoint, sizeof s_endpoint, &ep);
+ if (self->xml) {
+ OUTMSG((" <%s "
+ "domain=\"%s\" port=\"%d\" address=\"%s\" time=\"%d ms\"/>\n",
+ root, domain, port, s_endpoint, time));
+ }
+ else {
+ OUTMSG((
+ "%s domain=\"%s\" port=\"%d\" address=\"%s\" time=\"%d ms\"\n",
+ root, domain, port, s_endpoint, time));
+ }
+ }
+ return rc;
+}
+
+static rc_t read_stream_into_databuffer(
+ KStream *stream, KDataBuffer *databuffer)
+{
+ rc_t rc;
+
+ size_t total = 0;
+ KDataBufferMakeBytes( databuffer, 4096 );
+ while ( 1 )
+ {
+ size_t num_read;
+ uint8_t * base;
+ uint64_t avail = databuffer->elem_count - total;
+ if ( avail < 256 )
+ {
+ rc = KDataBufferResize( databuffer, databuffer->elem_count + 4096 );
+ if ( rc != 0 )
+ {
+ LogErr( klogErr, rc, "CGI: KDataBufferResize failed" );
+ break;
+ }
+ }
+
+ base = databuffer->base;
+ rc = KStreamRead(stream, &base[total],
+ (size_t) databuffer->elem_count - total, &num_read);
+ if ( rc != 0 )
+ {
+ /* TBD - look more closely at rc */
+ if ( num_read > 0 )
+ {
+ LogErr( klogErr, rc, "CGI: KStreamRead failed" );
+ rc = 0;
+ }
+ else
+ break;
+ }
+
+ if ( num_read == 0 )
+ break;
+
+ total += num_read;
+ }
+
+ if ( rc == 0 )
+ databuffer->elem_count = total;
+ return rc;
+}
+
+static rc_t call_cgi(const Main *self, const char *cgi_url,
+ uint32_t ver_major, uint32_t ver_minor, const char *protocol,
+ const char *acc, KDataBuffer *databuffer, const char *eol)
+{
+ KHttpRequest *req = NULL;
+ rc_t rc = 0;
+ assert(self);
+ rc = KNSManagerMakeReliableClientRequest
+ (self->knsMgr, &req, 0x01000000, NULL, cgi_url);
+ if (rc != 0) {
+ OUTMSG(
+ ("KNSManagerMakeReliableClientRequest(%s)=%R%s", cgi_url, rc, eol));
+ }
+ if (rc == 0) {
+ const char param[] = "acc";
+ rc = KHttpRequestAddPostParam( req, "%s=%s", param, acc);
+ if (rc != 0) {
+ OUTMSG(("KHttpRequestAddPostParam(%s)=%R%s", param, rc, eol));
+ }
+ }
+ if (rc == 0) {
+ const char param[] = "accept-proto";
+ rc = KHttpRequestAddPostParam( req, "%s=%s", param, protocol);
+ if (rc != 0) {
+ OUTMSG(("KHttpRequestAddPostParam(%s)=%R%s", param, rc, eol));
+ }
+ }
+ if (rc == 0) {
+ const char param[] = "version";
+ rc = KHttpRequestAddPostParam
+ (req, "%s=%u.%u", param, ver_major, ver_minor);
+ if (rc != 0) {
+ OUTMSG(("KHttpRequestAddPostParam(%s)=%R%s", param, rc, eol));
+ }
+ }
+ if (rc == 0) {
+ KHttpResult *rslt = NULL;
+ rc = KHttpRequestPOST(req, &rslt);
+ if (rc != 0) {
+ OUTMSG(("KHttpRequestPOST(%s)=%R%s", cgi_url, rc, eol));
+ }
+ else {
+ const char root[] = "StatusCode";
+ uint32_t code = 0;
+ rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL);
+ if (rc != 0) {
+ OUTMSG(("KHttpResultStatus(%s)=%R%s", cgi_url, rc, eol));
+ }
+ else if (code != 200) {
+ if (self->xml) {
+ OUTMSG((" <%s>%d</%s>\n", root, code, root));
+ }
+ else {
+ OUTMSG(("%s=%d\n", root, code));
+ }
+ rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid);
+ }
+ else {
+ KStream *response = NULL;
+ rc = KHttpResultGetInputStream(rslt, &response);
+ if (rc != 0) {
+ OUTMSG(("KHttpResultGetInputStream(%s)=%R%s",
+ cgi_url, rc, eol));
+ }
+ else {
+ rc = read_stream_into_databuffer(response, databuffer);
+ }
+ RELEASE(KStream, response);
+ }
+ }
+
+ RELEASE(KHttpResult, rslt);
+ }
+ RELEASE(KHttpRequest, req);
+ return rc;
+}
+
+static rc_t perform_cgi_test(const Main *self, const char *eol, const char *acc)
+{
+ rc_t rc = 0;
+ const char root[] = "Cgi";
+ KDataBuffer databuffer;
+ KTimeMs_t start_time = KTimeMsStamp();
+ if (acc == NULL) {
+ return 0;
+ }
+ assert(self);
+ memset(&databuffer, 0, sizeof databuffer);
+ if (self->xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ {
+ KTimeMs_t time = 0;
+ const char root[] = "Response";
+ rc = call_cgi(self, "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
+ , 1, 1, "http", acc, &databuffer, eol);
+ time = KTimeMsStamp() - start_time;
+ if (rc == 0) {
+ const char *start = databuffer.base;
+ size_t size = KDataBufferBytes(&databuffer);
+ if (self->xml) {
+ OUTMSG((" <%s time=\"%d ms\">%.*s</%s>\n",
+ root, time, size, start, root));
+ }
+ else {
+ OUTMSG(("%s = \"%.*s\" time=\"%d ms\"\n",
+ root, size, start, time));
+ }
+ }
+ else {/*
+ if (self->xml) {
+ //OUTMSG((" <%s/>\n", root, size, start, root));
+ }
+ else {
+ //OUTMSG(("%s = '%.*s'\n", root, size, start));
+ }*/
+ }
+ }
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ return rc;
+}
+
+static rc_t MainNetwotk(const Main *self, const char *arg, const char *eol)
+{
+ const char root[] = "Network";
+ assert(self);
+ if (self->xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+ {
+ const char root[] = "HttpProxy";
+ bool enabled = KNSManagerGetHTTPProxyEnabled(self->knsMgr);
+ if (!enabled) {
+ if (self->xml) {
+ OUTMSG((" <%s enabled=\"false\">\n", root));
+ }
+ else {
+ OUTMSG(("HTTPProxyEnabled=\"false\"\n", root));
+ }
+ }
+ else {
+ if (self->xml) {
+ OUTMSG((" <%s enabled=\"true\">\n", root));
+ }
+ else {
+ OUTMSG(("HTTPProxyEnabled=\"true\"\n", root));
+ }
+ }
+ {
+ const String *proxy = NULL;
+ rc_t rc = KNSManagerGetHTTPProxyPath(self->knsMgr, &proxy);
+ if (rc != 0) {
+ OUTMSG(("KNSManagerGetHTTPProxyPath()=%R%s", rc, eol));
+ }
+ else {
+ const char root[] = "Path";
+ if (self->xml) {
+ OUTMSG((" <%s>%S</%s>\n", root, proxy, root));
+ }
+ else {
+ OUTMSG(("HTTPProxyPath=\"%S\"\n", proxy));
+ }
+ }
+ }
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ }
+ {
+ const char *user_agent = NULL;
+ rc_t rc = KNSManagerGetUserAgent(&user_agent);
+ if (rc != 0) {
+ OUTMSG(("KNSManagerGetUserAgent()=%R%s", rc, eol));
+ }
+ else {
+ const char root[] = "UserAgent";
+ if (self->xml) {
+ OUTMSG((" <%s>%s</%s>\n", root, user_agent, root));
+ }
+ else {
+ OUTMSG(("UserAgent=\"%s\"\n", user_agent));
+ }
+ }
+ }
+ perfrom_dns_test(self, eol);
+ perform_cgi_test(self, eol, arg);
+ if (self->xml) {
+ OUTMSG(("</%s>\n", root));
+ }
+ return 0;
+}
+
static
rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
{
@@ -1830,22 +2430,26 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
size_t num_writ = 0;
char arg[PATH_MAX] = "";
+ const char *eol = NULL;
+
va_list args;
va_start(args, aArg);
assert(self);
+ eol = self->xml ? "<br/>\n" : "\n";
+
if (self->xml) {
OUTMSG(("<%s>\n", root));
}
if (item != NULL) {
type = kptKartITEM;
+
+ _KartItemPrint(item, self->xml);
}
else {
- const char *eol = self->xml ? "<br/>\n" : "\n";
-
rc = string_vprintf(arg, sizeof arg, &num_writ, aArg, args);
if (rc != 0) {
OUTMSG(("string_vprintf(%s)=%R%s", aArg, rc, eol));
@@ -1863,19 +2467,16 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
OUTMSG((" "));
rc = MainReport(self, arg, &directSz, &type, &alias);
}
+ else {
+ type = KDirectoryPathType(self->dir, "%s", arg);
+ }
OUTMSG(("%s", eol));
if (MainHasTest(self, eOpenTable)) {
- rc_t rc2 = MainOpenAs(self, arg, false);
- if (rce == 0 && rc2 != 0) {
- rce = rc2;
- }
+ MainOpenAs(self, arg, false);
}
if (MainHasTest(self, eOpenDB)) {
- rc_t rc2 = MainOpenAs(self, arg, true);
- if (rce == 0 && rc2 != 0) {
- rce = rc2;
- }
+ MainOpenAs(self, arg, true);
}
}
@@ -1895,6 +2496,10 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
}
for (i = 0; i < count && rc == 0; ++i) {
const char *name = NULL;
+ rc = Quitting();
+ if (rc != 0) {
+ break;
+ }
rc = KNamelistGet(list, i, &name);
if (rc != 0) {
OUTMSG(("KNamelistGet(KDirectoryList(%s), %d)=%R ",
@@ -1924,6 +2529,13 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
while (true) {
rc_t rc2 = 0;
RELEASE(KartItem, item);
+ rc2 = Quitting();
+ if (rc2 != 0) {
+ if (rce == 0) {
+ rce = rc2;
+ }
+ break;
+ }
rc2 = KartMakeNextItem(kart, &item);
if (rc2 != 0) {
OUTMSG(("KartMakeNextItem = %R\n", rc2));
@@ -1940,19 +2552,30 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
rce = rc2;
}
}
+ if (true) {
+ _KartPrintNumbered(kart, self->xml);
+ }
+ /*if (true) {
+ _KartPrint(kart, self->xml);
+ }
+ if (true) {
+ _KartPrintSized(kart, self->xml);
+ }*/
KartRelease(kart);
kart = NULL;
}
else {
if (MainHasTest(self, eResolve)) {
- rc_t rc2 = MainResolve(self, item, arg, &localSz, &remoteSz,
- false);
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
- }
+ /* ignore returned value :
+ resolver's errors are detected but not reported as test-sra's failure */
+ MainResolve(self, item, arg, &localSz, &remoteSz, false);
+ }
+
+ if (MainHasTest(self, eNetwork)) {
+ MainNetwotk(self, arg, eol);
}
- if (item == NULL) {
+ if (item == NULL) { /* TODO || kartitem & database */
if (type == kptDatabase || type == kptNotFound) {
if (MainHasTest(self, eDependMissing)) {
rc_t rc2 = MainDepend(self, arg, true);
@@ -1989,7 +2612,6 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
OUTMSG(("\n"));
}
- OUTMSG(("\n"));
}
}
@@ -2001,6 +2623,7 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
OUTMSG(("</%s>\n", root));
}
+ va_end(args);
return rc;
}
@@ -2025,6 +2648,295 @@ rc_t _SraReleaseVersionPrint(const SraReleaseVersion *self, rc_t rc, bool xml,
return rc;
}
+static rc_t _KDyldLoadLib(KDyld *self, char *name, size_t sz,
+ const char *path, bool xml, int indent)
+{
+ rc_t rc = 0;
+ KDylib *lib = NULL;
+ rc = KDyldLoadLib(self, &lib, "%.*s", sz, path);
+ if (rc == 0) {
+ rc = OUTMSG(("LOADED %.*s\n", sz, path));
+ }
+ else {
+ rc = OUTMSG(("NOT LOADED %.*s\n", sz, path));
+ }
+ RELEASE(KDylib, lib);
+ return rc;
+}
+
+static
+rc_t _KHttpRequestPOST(KHttpRequest *self, KDataBuffer *result, size_t *total) {
+ rc_t rc = 0;
+ KHttpResult *rslt = NULL;
+ assert(result && total);
+ memset(result, 0, sizeof *result);
+ *total = 0;
+ rc = KHttpRequestPOST(self, &rslt);
+ if (rc == 0) {
+ uint32_t code = 0;
+ size_t msg_size = 0;
+ char msg_buff[256] = "";
+ rc = KHttpResultStatus(rslt,
+ &code, msg_buff, sizeof msg_buff, &msg_size);
+ if (rc == 0 && code == 200) {
+ KStream *response = NULL;
+ rc = KHttpResultGetInputStream( rslt, &response);
+ if (rc == 0) {
+ size_t num_read = 0;
+ KDataBufferMakeBytes(result, 4096);
+ while (true) {
+ uint8_t *base = NULL;
+ uint64_t avail = result->elem_count - *total;
+ if (avail < 256) {
+ rc = KDataBufferResize
+ (result, result->elem_count + 4096);
+ if (rc != 0) {
+ break;
+ }
+ }
+ base = result->base;
+ rc = KStreamRead(response, &base[*total],
+ result->elem_count - *total, &num_read);
+ if (rc != 0) {
+ if (num_read > 0) {
+ rc = 0;
+ }
+ else {
+ break;
+ }
+ }
+ if (num_read == 0) {
+ break;
+ }
+ *total += num_read;
+ }
+ RELEASE(KStream, response);
+ }
+ }
+ }
+ RELEASE(KHttpResult, rslt);
+ return rc;
+}
+
+static
+rc_t _MainPost(const Main *self, const char *name, char *buffer, size_t sz)
+{
+ rc_t rc = 0;
+
+ KHttpRequest *req = NULL;
+ KDataBuffer result;
+
+ size_t total = 0;
+
+ assert(self && buffer && sz);
+
+ buffer[0] = '\0';
+
+ rc = KNSManagerMakeRequest(self->knsMgr, &req, 0x01000000, NULL,
+ "http://trace.ncbi.nlm.nih.gov/Traces/sratoolkit/sratoolkit.cgi");
+
+ if (rc == 0) {
+ rc = KHttpRequestAddPostParam(req, "cmd=vers");
+ }
+ if (rc == 0) {
+ rc = KHttpRequestAddPostParam(req, "libname=%s", name);
+ }
+ if (rc == 0) {
+ rc = _KHttpRequestPOST(req, &result, &total);
+ }
+
+ if (rc == 0) {
+ const char *start = (const void*)(result.base);
+ if (total > 0) {
+ if (*(start + total) != '\0') {
+ rc = RC(rcExe, rcString, rcParsing, rcString, rcUnexpected);
+ }
+ else if (*(start + total - 1) != '\n') {
+ rc = RC(rcExe, rcString, rcParsing, rcString, rcUnexpected);
+ }
+ else {
+ string_copy(buffer, sz, start, total - 1);
+ }
+ }
+ }
+
+ KDataBufferWhack(&result);
+ RELEASE(KHttpRequest, req);
+
+ return rc;
+}
+
+static rc_t _MainPrintNgsInfo(const Main* self) {
+ rc_t rc = 0;
+ const char root[] = "Ngs";
+ KDyld *l = NULL;
+ assert(self);
+ if (self->xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ {
+ const char root[] = "Latest";
+ char v[512];
+ if (self->xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ {
+ const char name[] = "ncbi-vdb";
+ _MainPost(self, name, v, sizeof v);
+ if (self->xml) {
+ OUTMSG((" <%s version=\"%s\"/>\n", name, v));
+ }
+ else {
+ OUTMSG(("%s latest version=\"%s\"\n", name, v));
+ }
+ }
+ {
+ const char name[] = "ngs-sdk";
+ _MainPost(self, name, v, sizeof v);
+ if (self->xml) {
+ OUTMSG((" <%s version=\"%s\"/>\n", name, v));
+ }
+ else {
+ OUTMSG(("%s latest version=\"%s\"\n", name, v));
+ OUTMSG(("\n"));
+ }
+ }
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ }
+ rc = KDyldMake(&l);
+ {
+ String *result = NULL;
+ rc = KConfigReadString(self->cfg, "NCBI_HOME", &result);
+ if (rc == 0) {
+ bool found = false;
+ const KFile *f = NULL;
+ char *buffer = NULL;
+ char *ps = NULL;
+ size_t ls = 0;
+ char *pv = NULL;
+ size_t lv = 0;
+ assert(result);
+ rc = KDirectoryOpenFileRead
+ (self->dir, &f, "%s/LibManager.properties", result->addr);
+ if (rc == 0) {
+ uint64_t size = 0;
+ rc = KFileSize(f, &size);
+ if (rc == 0) {
+ buffer = malloc(size + 1);
+ if (buffer == NULL) {
+ rc = RC(rcExe, rcData, rcAllocating,
+ rcMemory, rcExhausted);
+ }
+ else {
+ size_t num_read = 0;
+ rc = KFileRead(f, 0, buffer, size + 1, &num_read);
+ if (rc == 0) {
+#if _ARCH_BITS == 32
+ const char* sneed = "/dll/ngs-sdk/32/loaded/path=";
+ const char* vneed = "/dll/ncbi-vdb/32/loaded/path=";
+#else
+ const char* sneed = "/dll/ngs-sdk/64/loaded/path=";
+ const char* vneed = "/dll/ncbi-vdb/64/loaded/path=";
+#endif
+
+ assert(num_read <= size);
+ buffer[num_read] = '\0';
+ found = true;
+
+ ps = strstr(buffer, sneed);
+ if (ps != NULL) {
+ ps += strlen(sneed);
+ }
+ if (ps != NULL) {
+ const char *e = strchr(ps, '\n');
+ if (e != NULL) {
+ ls = e - ps;
+ }
+ }
+
+ pv = strstr(buffer, vneed);
+ if (pv != NULL) {
+ pv += strlen(vneed);
+ }
+ if (pv != NULL) {
+ const char *e = strchr(pv, '\n');
+ if (e != NULL) {
+ lv = e - pv;
+ }
+ }
+ }
+ }
+ }
+ RELEASE(KFile, f);
+ }
+ if (self->xml) {
+ if (found) {
+ OUTMSG((" <LibManager>\n"
+ " <Properties>\n%s"
+ " </Properties>\n", buffer));
+ if (ls != 0) {
+ OUTMSG((" <ngs-sdk>\n"
+ " <Path>%.*s</Path>\n", ls, ps));
+ _KDyldLoadLib(l, "ngs-sdk", ls, ps, self->xml, 8);
+ OUTMSG((" </ngs-sdk>\n"));
+ }
+ if (lv != 0) {
+ OUTMSG((" <ncbi-vdb>\n"
+ " <Path>%.*s</Path>\n", lv, pv));
+ _KDyldLoadLib(l, "ncbi-vdb", ls, ps, self->xml, 8);
+ OUTMSG((" </ncbi-vdb>\n"));
+ }
+ OUTMSG((" </LibManager>\n"));
+ }
+ else {
+ OUTMSG((" <LibManager.properties/>\n"));
+ }
+ }
+ else {
+ OUTMSG(("LibManager.properties=\n"));
+ if (found) {
+ OUTMSG(("%s\n", buffer));
+ if (ls != 0) {
+ OUTMSG(("LibManager.ngs-sdk='%.*s'\n", ls, ps));
+ }
+ if (lv != 0) {
+ OUTMSG(("LibManager.ngs-sdk='%.*s'\n", lv, pv));
+ }
+ }
+ }
+ free(buffer);
+ buffer = NULL;
+ free(result);
+ result = NULL;
+ }
+ }
+ {
+ const char root[] = "ncbi-vdb";
+ if (self->xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ }
+ {
+ const char root[] = "ngs-sdk";
+ if (self->xml) {
+ OUTMSG((" <%s>\n", root));
+ }
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ }
+ if (self->xml) {
+ OUTMSG((" </%s>\n", root));
+ }
+ RELEASE(KDyld, l);
+ return rc;
+}
+
static rc_t MainPrintVersion(Main *self) {
const char root[] = "Version";
@@ -2045,7 +2957,7 @@ static rc_t MainPrintVersion(Main *self) {
memset(&version, 0, sizeof version);
memset(&newVersion, 0, sizeof newVersion);
if (self->xml) {
- OUTMSG(("<%s>\n", root));
+ OUTMSG((" <%s>\n", root));
}
rc = SraReleaseVersionGet(&version);
rc = _SraReleaseVersionPrint(&version, rc, self->xml,
@@ -2064,7 +2976,7 @@ static rc_t MainPrintVersion(Main *self) {
if (rc == 0) {
if (isNew > 0) {
OUTMSG((
- "New version of SRA Toolkit is available for download from\n"
+ "A new version of SRA Toolkit is available for download from\n"
"\"http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=software\".\n"
));
}
@@ -2080,7 +2992,7 @@ static rc_t MainPrintVersion(Main *self) {
}
if (self->xml) {
- OUTMSG(("</%s>\n", root));
+ OUTMSG((" </%s>\n", root));
}
else if (self->full) {
OUTMSG(("\n"));
@@ -2101,6 +3013,7 @@ static rc_t MainFini(Main *self) {
RELEASE(VFSManager, self->vMgr);
RELEASE(VDBManager, self->mgr);
RELEASE(KDirectory, self->dir);
+ RELEASE(VSchema, self->schema);
return rc;
}
@@ -2125,6 +3038,10 @@ static const char* USAGE_NO_RFS[]
#define ALIAS_NO_VDB "N"
static const char* USAGE_NO_VDB[] = { "do not call VDBManagerPathType", NULL };
+#define OPTION_PRJ "project-id"
+#define ALIAS_PRJ "p"
+static const char* USAGE_PRJ[] = { "set project context", NULL };
+
#define OPTION_REC "recursive"
#define ALIAS_REC "R"
static const char* USAGE_REC[] = { "check object type recursively", NULL };
@@ -2139,8 +3056,9 @@ OptDef Options[] = { /* needs_value, required */
{ OPTION_NO_RFS, NULL , NULL, USAGE_NO_RFS, 1, false, false },
{ OPTION_NO_VDB, ALIAS_NO_VDB, NULL, USAGE_NO_VDB, 1, false, false },
{ OPTION_OUT , ALIAS_OUT , NULL, USAGE_OUT , 1, true , false },
+ { OPTION_PRJ , ALIAS_PRJ , NULL, USAGE_PRJ , 1, true , false },
{ OPTION_QUICK , ALIAS_QUICK , NULL, USAGE_QUICK , 1, false, false },
- { OPTION_REC , ALIAS_REC , NULL, USAGE_REC , 1, false, false }
+ { OPTION_REC , ALIAS_REC , NULL, USAGE_REC , 1, false, false },
};
rc_t CC KMain(int argc, char *argv[]) {
@@ -2171,6 +3089,26 @@ rc_t CC KMain(int argc, char *argv[]) {
}
}
+ if (rc == 0) {
+ rc = ArgsOptionCount(args, OPTION_PRJ, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_PRJ "' argument");
+ }
+ else {
+ if (pcount > 0) {
+ const char *dummy = NULL;
+ rc = ArgsOptionValue(args, OPTION_PRJ, 0, (const void **)&dummy);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" OPTION_PRJ "' argument");
+ }
+ else {
+ prms.projectId = AsciiToU32(dummy, NULL, NULL);
+ }
+ }
+ }
+ }
+
prms.full = true;
if (rc == 0) {
@@ -2213,9 +3151,10 @@ rc_t CC KMain(int argc, char *argv[]) {
}
else {
if (pcount > 0) {
- const char* dummy = NULL;
- rc = ArgsOptionValue(args, OPTION_OUT, 0, &dummy);
- if (rc) {
+ const char *dummy = NULL;
+ rc =
+ ArgsOptionValue(args, OPTION_OUT, 0, (const void **)&dummy);
+ if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" OPTION_OUT "' argument");
}
@@ -2236,6 +3175,10 @@ rc_t CC KMain(int argc, char *argv[]) {
OUTMSG(("<%s>\n", root));
}
+ if (MainHasTest(&prms, eNgs)) {
+ _MainPrintNgsInfo(&prms);
+ }
+
MainPrintVersion(&prms);
if (MainHasTest(&prms, eCfg)) {
@@ -2253,12 +3196,6 @@ rc_t CC KMain(int argc, char *argv[]) {
MainPrintAscp(&prms);
}
-#if 0
- if (MainHasTest(&prms, eCurl)) {
- PrintCurl(prms.full, prms.xml);
- }
-#endif
-
if (!prms.full) {
rc_t rc2 = MainQuickCheck(&prms);
if (rc == 0 && rc2 != 0) {
@@ -2307,12 +3244,18 @@ rc_t CC KMain(int argc, char *argv[]) {
if (rc == 0) {
rc = ArgsParamCount(args, &pcount);
}
-
+/* TODO if pcount == 0 && there are no test type options use a small run as name[0]*/
for (i = 0; i < pcount; ++i) {
const char *name = NULL;
- rc3 = ArgsParamValue(args, i, &name);
+ rc3 = ArgsParamValue(args, i, (const void **)&name);
if (rc3 == 0) {
- rc_t rc2 = 0;
+ rc_t rc2 = Quitting();
+ if (rc2 != 0) {
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ break;
+ }
ReportResetObject(name);
rc2 = MainExec(&prms, NULL, name);
if (rc == 0 && rc2 != 0) {
@@ -2347,7 +3290,5 @@ rc_t CC KMain(int argc, char *argv[]) {
}
free(argv2);
- OUTMSG(("\n"));
-
return rc;
}
diff --git a/tools/util/test-sra.vers b/tools/util/test-sra.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/test-sra.vers
+++ b/tools/util/test-sra.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/test-sra.vers.h b/tools/util/test-sra.vers.h
index a76cf32..8a4eb6a 100644
--- a/tools/util/test-sra.vers.h
+++ b/tools/util/test-sra.vers.h
@@ -1 +1 @@
-#define TEST_SRA_VERS 0x02040000
+#define TEST_SRA_VERS 0x02050007
diff --git a/tools/util/testld.c b/tools/util/testld.c
index c89603d..e8daa47 100644
--- a/tools/util/testld.c
+++ b/tools/util/testld.c
@@ -210,7 +210,7 @@ rc_t CC KMain ( int argc, char *argv [] )
for (ix = 0; ix < num_paths; ix++)
{
- rc = ArgsOptionValue (args, OPTION_LOAD, ix, &path);
+ rc = ArgsOptionValue (args, OPTION_LOAD, ix, (const void **)&path);
if (rc)
{
LOGERR (klogInt, rc, "failed to access a path option");
@@ -239,7 +239,7 @@ rc_t CC KMain ( int argc, char *argv [] )
/* test it */
for (ix = 0; ix < num_libs; ++ ix )
{
- rc = ArgsParamValue (args, ix, &path);
+ rc = ArgsParamValue (args, ix, (const void **)&path);
if (rc)
break;
diff --git a/tools/util/testld.vers b/tools/util/testld.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/testld.vers
+++ b/tools/util/testld.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/txt2kdb.c b/tools/util/txt2kdb.c
index 498994e..42d3229 100644
--- a/tools/util/txt2kdb.c
+++ b/tools/util/txt2kdb.c
@@ -486,11 +486,11 @@ rc_t CC KMain (int argc, char *argv [])
break;
}
- rc = ArgsParamValue (args, 0, &G.txtpath);
+ rc = ArgsParamValue (args, 0, (const void **)&G.txtpath);
if (rc)
break;
- rc = ArgsParamValue (args, 1, &G.colpath);
+ rc = ArgsParamValue (args, 1, (const void **)&G.colpath);
if (rc)
break;
@@ -504,7 +504,7 @@ rc_t CC KMain (int argc, char *argv [])
{
const char * pc;
- rc = ArgsOptionValue (args, OPTION_BEGIN, 0, &pc);
+ rc = ArgsOptionValue (args, OPTION_BEGIN, 0, (const void **)&pc);
if (rc)
break;
@@ -525,7 +525,7 @@ rc_t CC KMain (int argc, char *argv [])
{
const char * pc;
- rc = ArgsOptionValue (args, OPTION_END, 0, &pc);
+ rc = ArgsOptionValue (args, OPTION_END, 0, (const void **)&pc);
if (rc)
break;
diff --git a/tools/util/txt2kdb.vers b/tools/util/txt2kdb.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/txt2kdb.vers
+++ b/tools/util/txt2kdb.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/vdb-lock.c b/tools/util/vdb-lock.c
index a440e4f..709aee0 100644
--- a/tools/util/vdb-lock.c
+++ b/tools/util/vdb-lock.c
@@ -109,7 +109,7 @@ rc_t CC KMain ( int argc, char *argv [] )
else
{
const char *target;
- rc = ArgsParamValue ( args, 0, & target );
+ rc = ArgsParamValue ( args, 0, (const void **)& target );
if ( rc != 0 )
LogErr ( klogInt, rc, "failed to obtain param value" );
else
diff --git a/tools/util/vdb-lock.vers b/tools/util/vdb-lock.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/vdb-lock.vers
+++ b/tools/util/vdb-lock.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/vdb-lock.vers.h b/tools/util/vdb-lock.vers.h
index 33b803c..40e9880 100644
--- a/tools/util/vdb-lock.vers.h
+++ b/tools/util/vdb-lock.vers.h
@@ -1 +1 @@
-#define VDB_LOCK_VERS 0x02040002
+#define VDB_LOCK_VERS 0x02050007
diff --git a/tools/util/vdb-passwd.c b/tools/util/vdb-passwd.c
index cdecedb..a9a664c 100644
--- a/tools/util/vdb-passwd.c
+++ b/tools/util/vdb-passwd.c
@@ -47,7 +47,7 @@
#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
-
+#if 0
/******************************************************************************/
static
rc_t SetPwd(const char *password, bool no_prompt)
@@ -135,8 +135,10 @@ rc_t SetPwd(const char *password, bool no_prompt)
"Cannot set the password.\n"));
}
else {
- OUTMSG(("\nCannot set the password. Please run "
- "\"perl configuration-assistant.perl\" and try again.\n"));
+ OUTMSG(("\nCannot set the password. Please consult\n"
+ "configuration page at\n"
+ "http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=toolkit_doc&f=std or\n"
+ "https://github.com/ncbi/sra-tools/wiki/Toolkit-Configuration\n"));
}
}
}
@@ -217,11 +219,11 @@ rc_t run(bool quiet)
return rc;
}
-
+#endif
#define SL_OPTION "noprompt"
#define SL_ALIAS "n"
static const char *SL_USAGE[]
- = { "Do not suggest running configuration-assistant", NULL };
+ = { "Do not suggest consulting configuration page at http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=toolkit_doc&f=std or https://github.com/ncbi/sra-tools/wiki/Toolkit-Configuration", NULL };
OptDef Options[] = { { SL_OPTION, SL_ALIAS, NULL, SL_USAGE, 1, false, false } };
@@ -248,7 +250,7 @@ rc_t CC Usage(const Args *args) {
}
ver_t CC KAppVersion(void) { return VDB_PASSWD_VERS; }
-
+#if 0
rc_t CC KMain(int argc, char *argv[]) {
Args *args;
rc_t rc; /*, orc; */
@@ -274,3 +276,35 @@ rc_t CC KMain(int argc, char *argv[]) {
return rc;
}
+#endif
+rc_t CC KMain(int argc, char *argv[]) {
+ OUTMSG((
+"WARNING: vdb-passwd IS OBSOLETE AND SHOULD NOT BE USED.\n"
+"\n"
+"\n"
+"To access dbGaP data you need:\n"
+"\n"
+"- Make sure you have the latest version of SRA Toolkit installed:\n"
+"https://github.com/ncbi/sra-tools/wiki/Downloads\n"
+"\n"
+"- Have permission to access controlled-access data for a dbGaP project;\n"
+"\n"
+"- Get dbGaP repository key (ngc file);\n"
+"\n"
+"- Import the dbGaP repository key to SRA Toolkit.\n"
+" It will set up the project's workspace directory.\n"
+"\n"
+"- Change directory to the project's workspace.\n"
+"\n"
+"N.B. MAKE SURE YOU DO NOT HAVE A VDB_PWFILE ENVIRONMENT VARIABLE SET !!!\n"
+"\n"
+"Now you should be able to work with encrypted data.\n"
+"The SRA Toolkit will work with encrypted SRA data files,\n"
+"there is no need to decrypt the read data.\n"
+"\n"
+"The complete instructions are:\n"
+"http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=toolkit_doc&f=dbgap_use\n"
+"\n"
+"Send questions/bug reports to sra-tools at ncbi.nlm.nih.gov\n"));
+ return RC(rcExe, rcProcess, rcExecuting, rcProcess, rcUnsupported);
+}
diff --git a/tools/util/vdb-passwd.vers b/tools/util/vdb-passwd.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/vdb-passwd.vers
+++ b/tools/util/vdb-passwd.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/vdb-passwd.vers.h b/tools/util/vdb-passwd.vers.h
index 45ef0cc..c206a34 100644
--- a/tools/util/vdb-passwd.vers.h
+++ b/tools/util/vdb-passwd.vers.h
@@ -1 +1 @@
-#define VDB_PASSWD_VERS 0x02040002
+#define VDB_PASSWD_VERS 0x02050007
diff --git a/tools/util/vdb-unlock.c b/tools/util/vdb-unlock.c
index ca481b6..16a3e4f 100644
--- a/tools/util/vdb-unlock.c
+++ b/tools/util/vdb-unlock.c
@@ -111,7 +111,7 @@ rc_t CC KMain ( int argc, char *argv [] )
else
{
const char *target;
- rc = ArgsParamValue ( args, 0, & target );
+ rc = ArgsParamValue ( args, 0, (const void **)& target );
if ( rc != 0 )
LogErr ( klogInt, rc, "failed to obtain param value" );
else
diff --git a/tools/util/vdb-unlock.vers b/tools/util/vdb-unlock.vers
index 8e8299d..35d16fb 100644
--- a/tools/util/vdb-unlock.vers
+++ b/tools/util/vdb-unlock.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/util/vdb-unlock.vers.h b/tools/util/vdb-unlock.vers.h
index 8f72686..e9e1a51 100644
--- a/tools/util/vdb-unlock.vers.h
+++ b/tools/util/vdb-unlock.vers.h
@@ -1 +1 @@
-#define VDB_UNLOCK_VERS 0x02040002
+#define VDB_UNLOCK_VERS 0x02050007
diff --git a/tools/vcf-loader/.gitignore b/tools/vcf-loader/.gitignore
new file mode 100644
index 0000000..ec76f86
--- /dev/null
+++ b/tools/vcf-loader/.gitignore
@@ -0,0 +1 @@
+vcf-grammar.output
diff --git a/tools/vcf-loader/Makefile b/tools/vcf-loader/Makefile
index f20f0f6..065bdb2 100644
--- a/tools/vcf-loader/Makefile
+++ b/tools/vcf-loader/Makefile
@@ -102,8 +102,15 @@ LIBVCF_SRC = \
vcf-reader \
vcf-database
+# flex/bison should only be invoked manually in an environment ensures the correct versions:
+# bison 2.5, flex 2.5.35
+bison:
+ bison -o $(SRCDIR)/vcf-grammar.c --defines=$(SRCDIR)/vcf-grammar.h -v --no-lines $(SRCDIR)/vcf-grammar.y
+
+flex:
+ flex -t $(SRCDIR)/vcf-lex.l | grep -v '^#line' > $(SRCDIR)/vcf-lex.c
+
$(SRCDIR)/vcf-grammar.c $(SRCDIR)/vcf-grammar.h: $(SRCDIR)/vcf-grammar.y
- $(YACC) -o $(SRCDIR)/vcf-grammar.c --defines=$(SRCDIR)/vcf-grammar.h -v $^
LIBVCF_OBJ = \
$(addsuffix .$(LOBX),$(LIBVCF_SRC))
diff --git a/tools/vcf-loader/vcf-database.c b/tools/vcf-loader/vcf-database.c
index b4ec7ca..ac18c74 100644
--- a/tools/vcf-loader/vcf-database.c
+++ b/tools/vcf-loader/vcf-database.c
@@ -101,12 +101,15 @@ rc_t SaveVariants( const VcfReader* reader, const char configPath[], VDatabase*
const ReferenceSeq* seq;
#define MAX_CHROMOSOME_NAME_LENGTH 1024
char chromName[MAX_CHROMOSOME_NAME_LENGTH];
+ bool shouldUnmap = false;
+ bool wasRenamed = false;
string_copy(chromName, sizeof(chromName), line->chromosome.addr, line->chromosome.size);
- rc = (ReferenceMgr_GetSeq(refMgr, &seq, chromName));
+ rc = (ReferenceMgr_GetSeq(refMgr, &seq, chromName, &shouldUnmap, true, &wasRenamed));
if (rc == 0)
{
int64_t ref_id;
INSDC_coord_zero ref_start;
+ assert(shouldUnmap == false);
rc = ReferenceSeq_TranslateOffset_int(seq, line->position, &ref_id, &ref_start, NULL);
if (rc == 0)
{
diff --git a/tools/vcf-loader/vcf-grammar.c b/tools/vcf-loader/vcf-grammar.c
index 5a9f56f..16f6e3a 100644
--- a/tools/vcf-loader/vcf-grammar.c
+++ b/tools/vcf-loader/vcf-grammar.c
@@ -1678,7 +1678,7 @@ yyreturn:
-void CC VCF_error(struct VCFParseBlock* pb, const char* msg)
+void VCF_error(struct VCFParseBlock* pb, const char* msg)
{
if (pb && pb->error)
pb->error(pb, msg);
diff --git a/tools/vcf-loader/vcf-grammar.y b/tools/vcf-loader/vcf-grammar.y
index f277c36..03d20b6 100644
--- a/tools/vcf-loader/vcf-grammar.y
+++ b/tools/vcf-loader/vcf-grammar.y
@@ -116,7 +116,7 @@ dataItems:
%%
-void CC VCF_error(struct VCFParseBlock* pb, const char* msg)
+void VCF_error(struct VCFParseBlock* pb, const char* msg)
{
if (pb && pb->error)
pb->error(pb, msg);
diff --git a/tools/vcf-loader/vcf-lex.c b/tools/vcf-loader/vcf-lex.c
index 49ae8df..356a517 100644
--- a/tools/vcf-loader/vcf-lex.c
+++ b/tools/vcf-loader/vcf-lex.c
@@ -1,6 +1,5 @@
-#define YY_BUF_SIZE 16777216
#define YY_INT_ALIGNED short int
/* A lexical scanner generated by flex */
@@ -439,7 +438,7 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
#define YY_DO_BEFORE_ACTION \
yyg->yytext_ptr = yy_bp; \
/* %% [2.0] code to fiddle yytext and yyleng for yymore() goes here \ */\
- yyleng = (size_t) (yy_cp - yy_bp); \
+ yyleng = (int) ((size_t) (yy_cp - yy_bp)); \
yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \
/* %% [3.0] code to copy yytext_ptr to yytext[] goes here, if %array \ */\
@@ -584,8 +583,8 @@ static yyconst flex_int32_t yy_rule_can_match_eol[18] =
static yyconst flex_int16_t yy_rule_linenum[17] =
{ 0,
- 76, 77, 79, 81, 83, 84, 85, 89, 93, 94,
- 99, 100, 101, 102, 103, 106
+ 73, 74, 76, 78, 80, 81, 82, 86, 90, 91,
+ 96, 97, 98, 99, 100, 103
} ;
/* The intent behind this definition is that it'll catch
@@ -620,9 +619,6 @@ static yyconst flex_int16_t yy_rule_linenum[17] =
* ===========================================================================
*
*/
- #include <klib/rc.h>
- #include <klib/text.h>
-
#include "vcf-parse.h"
#include "vcf-grammar.h"
@@ -644,13 +640,18 @@ static yyconst flex_int16_t yy_rule_linenum[17] =
return vcfENDLINE;
#define YY_INPUT(buf, result, max_size) \
- result = yyextra->input(yyextra, buf, max_size)
+ result = (int) yyextra->input(yyextra, buf, max_size)
#define ECHO
#define YY_NO_INPUT
#define YY_NO_UNISTD_H 1
+
+
+
+
+
#define INITIAL 0
#define HEADER 1
#define DATA 2
@@ -954,6 +955,8 @@ YY_DECL
/* %% [7.0] user's declarations go here */
+
+
yylval = yylval_param;
if ( !yyg->yy_init )
@@ -1116,11 +1119,14 @@ YY_RULE_SETUP
{ BEGIN META_COMPOSITE; return vcfMETAKEY; }
YY_BREAK
+
case 8:
YY_RULE_SETUP
{ return vcfMETAVALUE; }
YY_BREAK
+
+
case 9:
*yy_cp = yyg->yy_hold_char; /* undo effects of setting up yytext */
yyg->yy_c_buf_p = yy_cp -= 1;
@@ -1133,6 +1139,8 @@ YY_RULE_SETUP
{ return vcfMETAVALUE; }
YY_BREAK
+
+
case 11:
YY_RULE_SETUP
@@ -1373,7 +1381,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
else
{
int num_to_read =
- YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1;
+ (int) YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1;
while ( num_to_read <= 0 )
{ /* Not enough room in the buffer - grow it. */
@@ -1386,7 +1394,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
if ( b->yy_is_our_buffer )
{
- int new_size = b->yy_buf_size * 2;
+ int new_size = (int) b->yy_buf_size * 2;
if ( new_size <= 0 )
b->yy_buf_size += b->yy_buf_size / 8;
@@ -1407,7 +1415,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
yyg->yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset];
- num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size -
+ num_to_read = (int) YY_CURRENT_BUFFER_LVALUE->yy_buf_size -
number_to_move - 1;
}
@@ -1561,7 +1569,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
yy_cp += (int) (dest - source);
yy_bp += (int) (dest - source);
YY_CURRENT_BUFFER_LVALUE->yy_n_chars =
- yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_buf_size;
+ yyg->yy_n_chars = (int) YY_CURRENT_BUFFER_LVALUE->yy_buf_size;
if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 )
YY_FATAL_ERROR( "flex scanner push-back overflow" );
@@ -1960,7 +1968,7 @@ static void VCF_ensure_buffer_stack (yyscan_t yyscanner)
/* %if-c++-only */
/* %endif */
{
- int num_to_alloc;
+ size_t num_to_alloc;
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if (!yyg->yy_buffer_stack) {
@@ -2028,7 +2036,7 @@ YY_BUFFER_STATE VCF__scan_buffer (char * base, yy_size_t size , yyscan_t yysca
b->yy_buf_pos = b->yy_ch_buf = base;
b->yy_is_our_buffer = 0;
b->yy_input_file = 0;
- b->yy_n_chars = b->yy_buf_size;
+ b->yy_n_chars = (int) b->yy_buf_size;
b->yy_is_interactive = 0;
b->yy_at_bol = 1;
b->yy_fill_buffer = 0;
@@ -2052,7 +2060,7 @@ YY_BUFFER_STATE VCF__scan_buffer (char * base, yy_size_t size , yyscan_t yysca
YY_BUFFER_STATE VCF__scan_string (yyconst char * yystr , yyscan_t yyscanner)
{
- return VCF__scan_bytes(yystr,strlen(yystr) ,yyscanner);
+ return VCF__scan_bytes(yystr, (int)strlen(yystr), yyscanner);
}
/* %endif */
@@ -2488,11 +2496,15 @@ void VCF_free (void * ptr , yyscan_t yyscanner)
/* %ok-for-header */
-rc_t CC VCFScan_yylex_init(VCFParseBlock* sb, bool debug)
+
+
+
+bool VCFScan_yylex_init(VCFParseBlock* sb, bool debug)
{
if (VCF_lex_init_extra(sb,&sb->scanner) != 0)
{
- return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );
+ return false;
+ /*return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );*/
}
sb->offset = 0;
@@ -2502,10 +2514,10 @@ rc_t CC VCFScan_yylex_init(VCFParseBlock* sb, bool debug)
VCF_set_debug(debug ? 1 : 0,sb->scanner);
- return 0;
+ return true;
}
-void CC VCFScan_yylex_destroy(VCFParseBlock* sb)
+void VCFScan_yylex_destroy(VCFParseBlock* sb)
{
if (sb->scanner)
{
diff --git a/tools/vcf-loader/vcf-lex.l b/tools/vcf-loader/vcf-lex.l
index 17925b3..721fbc6 100644
--- a/tools/vcf-loader/vcf-lex.l
+++ b/tools/vcf-loader/vcf-lex.l
@@ -25,9 +25,6 @@
*/
%{
- #include <klib/rc.h>
- #include <klib/text.h>
-
#include "vcf-parse.h"
#include "vcf-grammar.h"
@@ -107,11 +104,12 @@ alphanum [A-Za-z0-9\-_]+
%%
-rc_t CC VCFScan_yylex_init(VCFParseBlock* sb, bool debug)
+bool VCFScan_yylex_init(VCFParseBlock* sb, bool debug)
{
if (yylex_init_extra(sb, &sb->scanner) != 0)
{
- return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );
+ return false;
+ /*return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );*/
}
sb->offset = 0;
@@ -121,10 +119,10 @@ rc_t CC VCFScan_yylex_init(VCFParseBlock* sb, bool debug)
yyset_debug(debug ? 1 : 0, sb->scanner);
- return 0;
+ return true;
}
-void CC VCFScan_yylex_destroy(VCFParseBlock* sb)
+void VCFScan_yylex_destroy(VCFParseBlock* sb)
{
if (sb->scanner)
{
diff --git a/tools/vcf-loader/vcf-load.vers b/tools/vcf-loader/vcf-load.vers
index 8e8299d..35d16fb 100644
--- a/tools/vcf-loader/vcf-load.vers
+++ b/tools/vcf-loader/vcf-load.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vcf-loader/vcf-parse.h b/tools/vcf-loader/vcf-parse.h
index 5398c0f..3798737 100644
--- a/tools/vcf-loader/vcf-parse.h
+++ b/tools/vcf-loader/vcf-parse.h
@@ -27,7 +27,8 @@
#ifndef _h_vcf_parse_
#define _h_vcf_parse_
-#include <klib/defs.h>
+#include <stdbool.h>
+#include <stdlib.h>
typedef struct VCFToken
{
@@ -70,7 +71,7 @@ typedef struct VCFParseBlock
} VCFParseBlock;
-extern rc_t VCFScan_yylex_init(VCFParseBlock* context, bool debug);
+extern bool VCFScan_yylex_init(VCFParseBlock* context, bool debug); /* false - out of memory */
extern void VCFScan_yylex_destroy(VCFParseBlock* context);
extern void VCF_set_lineno (int line_number, void* scanner);
diff --git a/tools/vdb-config/configure.cpp b/tools/vdb-config/configure.cpp
index 901f21f..794bc57 100644
--- a/tools/vdb-config/configure.cpp
+++ b/tools/vdb-config/configure.cpp
@@ -74,7 +74,14 @@ class CConfigurator : CNoncopyable {
const string name("/repository/user/default-path");
CString node(m_Cfg.ReadString(name.c_str()));
if (node.Empty()) {
-/* this rc is ignored */ m_Cfg.UpdateNode(name.c_str(), "$(HOME)/ncbi");
+ CString home(m_Cfg.ReadString("HOME"));
+ if (!home.Empty()) {
+/* this rc is ignored */ m_Cfg.UpdateNode(name.c_str(),
+ (home.GetString() + "/ncbi").c_str());
+ }
+ else {
+ m_Cfg.UpdateNode(name.c_str(), "$(HOME)/ncbi");
+ }
}
}
rc_t rc = 0;
@@ -118,16 +125,30 @@ class CConfigurator : CNoncopyable {
if (len == 0) {
noUser = true;
}
- else if (len == 1) {
- const KRepository *repo = static_cast<const KRepository*>
- (VectorGet(&repositories, 0));
- if (repo != NULL) {
- char buffer[PATH_MAX] = "";
- size_t size = 0;
- rc =
- KRepositoryName(repo, buffer, sizeof buffer, &size);
- if (rc == 0) {
- rc = 0;
+ else {
+ uint32_t i = 0;
+ noUser = true;
+ for (i = 0; i < len; ++i) {
+ const KRepository *repo
+ = static_cast<const KRepository*>
+ (VectorGet(&repositories, i));
+ if (repo != NULL) {
+ char buffer[PATH_MAX] = "";
+ size_t size = 0;
+ rc = KRepositoryName(repo,
+ buffer, sizeof buffer, &size);
+ if (rc == 0) {
+ const char p[] = "public";
+ if (strcase_cmp(p, sizeof p - 1, buffer,
+ size, sizeof buffer) == 0)
+ {
+ noUser = false;
+ }
+ if (fix) {
+ rc = m_Cfg.CreateUserRepository
+ (buffer, fix);
+ }
+ }
}
}
rc = 0;
@@ -140,10 +161,7 @@ class CConfigurator : CNoncopyable {
noUser = true;
}
if (noUser) {
- rc = m_Cfg.CreateUserRepositories();
- }
- else if (fix) {
- rc = m_Cfg.CreateUserRepositories(fix);
+ rc = m_Cfg.CreateUserRepository();
}
}
RELEASE(KRepositoryMgr, mgr);
@@ -162,7 +180,7 @@ class CConfigurator : CNoncopyable {
const string name("/tools/ascp/max_rate");
CString node(m_Cfg.ReadString(name.c_str()));
if (node.Empty()) {
- rc = m_Cfg.UpdateNode(name.c_str(), "300m");
+ rc = m_Cfg.UpdateNode(name.c_str(), "1000m");
}
}
if (rc == 0) {
@@ -392,7 +410,7 @@ class CTextualConfigurator : public CConfigurator {
static const string magic;
public:
static STrinity Id2Seq(uint32_t id) {
- int d = id * 2 - magic.size();
+ int d = id * 2 - (int)magic.size();
if (d < 0) {
return STrinity(magic.substr(id * 2, 2));
}
@@ -416,7 +434,7 @@ class CTextualConfigurator : public CConfigurator {
case 1: c = eUserRoot ; break;
default: assert(0); break;
}
- return SChoice(c, p / 2);
+ return SChoice(c, (int)p / 2);
}
}
else {
@@ -424,7 +442,7 @@ class CTextualConfigurator : public CConfigurator {
if (!isdigit(s[0]) || !!isdigit(s[0]) || s[0] == '0') {
return SChoice(eUnknown);
}
- int id = (s[0] - '0') * 10 + s[1] - '0' + magic.size();
+ int id = (s[0] - '0') * 10 + s[1] - '0' + (int)magic.size();
EChoice c = eUnknown;
switch (id % 2) {
case 0: c = eUserCacheEnable; break;
diff --git a/tools/vdb-config/interactive.cpp b/tools/vdb-config/interactive.cpp
index 7fabc4d..126ba78 100644
--- a/tools/vdb-config/interactive.cpp
+++ b/tools/vdb-config/interactive.cpp
@@ -55,8 +55,9 @@ using namespace tui;
#define CB_COLOR_BG KTUI_c_cyan
#define BTN_COLOR_BG KTUI_c_cyan
-#define TOP_H 7
+#define TOP_H 9
#define SRC_W 36
+#define PROXY_EN_W 15
#define SAVE_W 14
#define EXIT_W 14
#define WKSP_NAME_W 32
@@ -65,6 +66,8 @@ using namespace tui;
#define CB_TXT_REMOTE "Enable Remote Access (1)"
#define CB_TXT_CACHE "Enable Local File Caching (2)"
#define CB_TXT_SITE "Use Site Installation (3)"
+#define CB_TXT_PROXY "Use Proxy"
+
#define BTN_TXT_IMPORT_NGC "Import Repository Key (4)"
#define BTN_TXT_DFLT_IMPORT_PATH "Set Default Import Path (5)"
#define B_SAVE_TXT "Save (6)"
@@ -82,6 +85,8 @@ using namespace tui;
#define FOCUS_TXT_CB_REMOTE "Press SPACE | ENTER to enable/disable access to the servers at NCBI"
#define FOCUS_TXT_CB_CACHED "Press SPACE | ENTER to enable/disable caching"
#define FOCUS_TXT_CB_SITE "Press SPACE | ENTER to enable/disable access to site repositories"
+#define FOCUS_TXT_CB_PROXY "Press SPACE | ENTER to enable/disable http proxy"
+#define FOCUS_TXT_B_PROXY "Press SPACE | ENTER to edit proxy"
#define FOCUS_TXT_B_PUBLIC_LOC "Press SPACE | ENTER to change location of public data"
#define FOCUS_TXT_B_IMPORT_NGC "Press SPACE | ENTER to import a dbGaP project"
#define FOCUS_TXT_B_USR_DFLT_PATH "Press SPACE | ENTER to change default repository location"
@@ -99,23 +104,26 @@ using namespace tui;
#define ID_CB_REMOTE 104
#define ID_CB_CACHED 105
#define ID_CB_SITE 106
+#define ID_CB_PROXY 107
+#define ID_B_PROXY 108
+#define ID_L_PROXY 109
-#define ID_B_IMPORT_NGC 107
-#define ID_B_USR_DFLT_PATH 108
+#define ID_B_IMPORT_NGC 120
+#define ID_B_USR_DFLT_PATH 121
-#define ID_L_WKSP_NAME 109
-#define ID_L_WKSP_LOC 110
+#define ID_L_WKSP_NAME 122
+#define ID_L_WKSP_LOC 123
-#define ID_L_PUBLIC 111
-#define ID_B_PUBLIC_LOC 112
-#define ID_L_PUBLIC_LOC 113
+#define ID_L_PUBLIC 123
+#define ID_B_PUBLIC_LOC 124
+#define ID_L_PUBLIC_LOC 125
-#define ID_B_SAVE 114
-#define ID_B_EXIT 115
-#define ID_B_RELOAD 116
-#define ID_B_FACT_DFLT 117
+#define ID_B_SAVE 126
+#define ID_B_EXIT 127
+#define ID_B_RELOAD 128
+#define ID_B_FACT_DFLT 129
-#define ID_L_NUMBER_EXPLAIN 118
+#define ID_L_NUMBER_EXPLAIN 130
#define ID_L_PROT 200
#define ID_L_PROT_LOC 300
@@ -163,7 +171,11 @@ class vdbconf_view : public Dlg
Tui_Rect remote_cb_rect( Tui_Rect const &r ) const { return h1_w_rect( r, 1, 1, SRC_W ); }
Tui_Rect cache_cb_rect( Tui_Rect const &r ) const { return h1_w_rect( r, 1, 3, SRC_W ); }
Tui_Rect site_cb_rect( Tui_Rect const &r ) const { return h1_w_rect( r, 1, 5, SRC_W ); }
-
+ Tui_Rect proxy_cb_rect( Tui_Rect const &r ) const { return h1_w_rect( r, 1, 7, PROXY_EN_W ); }
+ Tui_Rect proxy_btn_rect( Tui_Rect const &r ) const { return h1_w_rect( r, PROXY_EN_W + 2, 7, WKSP_B_LOC_W ); }
+ Tui_Rect proxy_lb_rect( Tui_Rect const &r ) const
+ { return h1_w_rect( r, PROXY_EN_W + 3 + WKSP_B_LOC_W, 7, r.get_w() - ( PROXY_EN_W + 4 + WKSP_B_LOC_W ) ); }
+
Tui_Rect import_ngc_rect( Tui_Rect const &r ) const { return h1_w_rect( r, 1, 1, WKSP_NAME_W ); }
Tui_Rect usr_dflt_path_rect( Tui_Rect const &r ) const { return h1_w_rect( r, WKSP_NAME_W + 2, 1, WKSP_NAME_W + 3 ); }
@@ -267,6 +279,9 @@ void vdbconf_view::populate_top_left( Tui_Rect const &r, bool resize )
setup_checkbox( cache_cb_rect( r ), resize, ID_CB_CACHED, CB_TXT_CACHE, priv_model.is_global_cache_enabled() );
if ( priv_model.does_site_repo_exist() )
setup_checkbox( site_cb_rect( r ), resize, ID_CB_SITE, CB_TXT_SITE, priv_model.is_site_enabled() );
+ setup_checkbox( proxy_cb_rect( r ), resize, ID_CB_PROXY, CB_TXT_PROXY, priv_model.is_http_proxy_enabled() );
+ setup_button( proxy_btn_rect( r ), resize, ID_B_PROXY, BTN_TXT_CHANGE );
+ setup_label( proxy_lb_rect( r ), resize, ID_L_PROXY, priv_model.get_http_proxy_path().c_str() );
}
@@ -510,6 +525,8 @@ class vdbconf_controller : public Dlg_Runner
bool toggle_site( Dlg &dlg );
bool toggle_cached( Dlg &dlg );
+ bool change_proxy( Dlg &dlg );
+
bool on_set_location_error( Dlg &dlg, ESetRootState s );
bool on_pick_public_location( Dlg &dlg );
@@ -673,6 +690,7 @@ bool vdbconf_controller::import_this_ngc_into_this_location( Dlg &dlg,
{
m.commit();
update_view( dlg );
+ m.mkdir(ngc);
}
}
else if ( result_flags == 0 )
@@ -831,6 +849,18 @@ bool vdbconf_controller::on_change_dflt_import_path( Dlg &dlg )
}
+bool vdbconf_controller::change_proxy( Dlg &dlg )
+{
+ std::string proxy = priv_model.get_http_proxy_path();
+ bool res = vdbconf_input( dlg, Tui_Rect( 5, 5, 100, 6 ), "change proxy", proxy );
+ if ( res )
+ {
+ priv_model.set_http_proxy_path( proxy.c_str() );
+ dlg.SetWidgetCaption( ID_L_PROXY, proxy );
+ }
+ return res;
+}
+
bool vdbconf_controller::on_set_location_error( Dlg &dlg, ESetRootState s )
{
bool result = false;
@@ -976,6 +1006,9 @@ bool vdbconf_controller::on_select( Dlg &dlg, void * data, Tui_Dlg_Event &dev )
res = true;
break;
+ case ID_CB_PROXY : m.set_http_proxy_enabled( dev.get_value_1() == 1 ); res = true; break;
+ case ID_B_PROXY : res = change_proxy( dlg ); break;
+
case ID_B_PUBLIC_LOC : res = on_pick_public_location( dlg ); break;
case ID_B_IMPORT_NGC : res = on_import_ngc( dlg ); break;
@@ -1010,6 +1043,9 @@ bool vdbconf_controller::on_focused( Dlg &dlg, uint32_t widget_id )
case ID_CB_REMOTE : status_txt( dlg, FOCUS_TXT_CB_REMOTE ); break;
case ID_CB_CACHED : status_txt( dlg, FOCUS_TXT_CB_CACHED ); break;
case ID_CB_SITE : status_txt( dlg, FOCUS_TXT_CB_SITE ); break;
+ case ID_CB_PROXY : status_txt( dlg, FOCUS_TXT_CB_PROXY ); break;
+ case ID_B_PROXY : status_txt( dlg, FOCUS_TXT_B_PROXY ); break;
+
case ID_B_PUBLIC_LOC : status_txt( dlg, FOCUS_TXT_B_PUBLIC_LOC ); break;
case ID_B_IMPORT_NGC : status_txt( dlg, FOCUS_TXT_B_IMPORT_NGC ); break;
case ID_B_USR_DFLT_PATH : status_txt( dlg, FOCUS_TXT_B_USR_DFLT_PATH ); break;
diff --git a/tools/vdb-config/util.cpp b/tools/vdb-config/util.cpp
index 894909b..2ab5115 100644
--- a/tools/vdb-config/util.cpp
+++ b/tools/vdb-config/util.cpp
@@ -1,6 +1,34 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
#include "util.hpp" // CStdIn
#include <klib/printf.h> /* string_printf */
+#include <sstream> // ostringstream
+
#include <climits> /* PATH_MAX */
#ifndef PATH_MAX
#define PATH_MAX 4096
@@ -290,13 +318,13 @@ rc_t CKConfig::CreateRemoteRepositories(bool fix) {
}
r2 = UpdateNode(
- "/repository/remote/aux/NCBI/apps/nakmer/volumes/fuseNANNOT",
+ "/repository/remote/aux/NCBI/apps/nannot/volumes/fuseNANNOT",
"sadb");
if (r2 != 0 && rc == 0) {
rc = r2;
}
- r2 = UpdateNode("/repository/remote/aux/NCBI/apps/sra/volumes/refseq",
+ r2 = UpdateNode("/repository/remote/aux/NCBI/apps/refseq/volumes/refseq",
"refseq");
if (r2 != 0 && rc == 0) {
rc = r2;
@@ -308,7 +336,7 @@ rc_t CKConfig::CreateRemoteRepositories(bool fix) {
rc = r2;
}
- r2 = UpdateNode("/repository/remote/aux/NCBI/apps/sra/volumes/fuseWGS",
+ r2 = UpdateNode("/repository/remote/aux/NCBI/apps/wgs/volumes/fuseWGS",
"wgs");
if (r2 != 0 && rc == 0) {
rc = r2;
@@ -343,7 +371,10 @@ rc_t CKConfig::CreateRemoteRepositories(bool fix) {
return rc;
}
-rc_t CKConfig::CreateUserRepositories(bool fix) {
+rc_t CKConfig::CreateUserRepository(string repoName, bool fix) {
+ if (repoName.size() == 0) {
+ repoName = "public";
+ }
CString cRoot(ReadString("/repository/user/default-path"));
string root;
if (cRoot.Empty()) {
@@ -353,55 +384,53 @@ rc_t CKConfig::CreateUserRepositories(bool fix) {
root = cRoot.GetString();
}
- string name("/repository/user/main/public/root");
+ std::ostringstream s;
+ s << "/repository/user/" << (repoName == "public" ? "main" : "protected")
+ << "/" << repoName;
+ string repoNode(s.str());
+ string name(repoNode + "/root");
bool toFix = true;
if (fix) {
toFix = !NodeExists(name);
}
rc_t rc = 0;
if (toFix) {
- rc = UpdateNode(name.c_str(), (root + "/public").c_str());
+ rc = UpdateNode(name, (root + "/public").c_str());
}
- rc_t r2 = UpdateNode("/repository/user/main/public/apps/file/volumes/flat",
- "files");
+ rc_t r2 = UpdateNode(repoNode + "/apps/file/volumes/flat", "files");
if (r2 != 0 && rc == 0) {
rc = r2;
}
- r2 = UpdateNode(
- "/repository/user/main/public/apps/nakmer/volumes/nakmerFlat",
- "nannot");
+ r2 = UpdateNode(repoNode + "/apps/sra/volumes/sraFlat", "sra");
if (r2 != 0 && rc == 0) {
rc = r2;
}
- r2 = UpdateNode(
- "/repository/user/main/public/apps/nakmer/volumes/nannotFlat",
- "nannot");
- if (r2 != 0 && rc == 0) {
- rc = r2;
- }
+ if (repoName == "public") {
+ r2 = UpdateNode(repoNode + "/apps/nakmer/volumes/nakmerFlat", "nannot");
+ if (r2 != 0 && rc == 0) {
+ rc = r2;
+ }
- r2 = UpdateNode("/repository/user/main/public/apps/refseq/volumes/refseq",
- "refseq");
- if (r2 != 0 && rc == 0) {
- rc = r2;
- }
+ r2 = UpdateNode(repoNode + "/apps/nannot/volumes/nannotFlat", "nannot");
+ if (r2 != 0 && rc == 0) {
+ rc = r2;
+ }
- r2 = UpdateNode("/repository/user/main/public/apps/sra/volumes/sraFlat",
- "sra");
- if (r2 != 0 && rc == 0) {
- rc = r2;
- }
+ r2 = UpdateNode(repoNode + "/apps/refseq/volumes/refseq", "refseq");
+ if (r2 != 0 && rc == 0) {
+ rc = r2;
+ }
- r2 = UpdateNode("/repository/user/main/public/apps/wgs/volumes/wgsFlat",
- "wgs");
- if (r2 != 0 && rc == 0) {
- rc = r2;
+ r2 = UpdateNode(repoNode + "/apps/wgs/volumes/wgsFlat", "wgs");
+ if (r2 != 0 && rc == 0) {
+ rc = r2;
+ }
}
- r2 = UpdateNode("/repository/user/main/public/cache-enabled", "true");
+ r2 = UpdateNode(repoNode + "/cache-enabled", "true");
if (r2 != 0 && rc == 0) {
rc = r2;
}
diff --git a/tools/vdb-config/util.hpp b/tools/vdb-config/util.hpp
index 31d9c2c..f21cae5 100644
--- a/tools/vdb-config/util.hpp
+++ b/tools/vdb-config/util.hpp
@@ -105,7 +105,7 @@ public:
}
String s;
- StringInit(&s, buffer, bsize, bsize + 1);
+ StringInit(&s, buffer, bsize, (uint32_t)bsize + 1);
return StringEqual(m_Self, &s);
}
@@ -282,8 +282,11 @@ public:
rc_t UpdateNode(bool verbose, const char *value, const char *name, ...);
rc_t UpdateNode(const char *path, const char *buffer,
bool verbose = false, size_t size = ~0);
+ rc_t UpdateNode(const std::string &path, const char *buffer) {
+ return UpdateNode(path.c_str(), buffer);
+ }
rc_t CreateRemoteRepositories(bool fix = false);
- rc_t CreateUserRepositories(bool fix = false);
+ rc_t CreateUserRepository(std::string name = "", bool fix = false);
const String* ReadString(const char *path) const;
};
diff --git a/tools/vdb-config/vdb-config-model.cpp b/tools/vdb-config/vdb-config-model.cpp
index 7e7f2c5..38694db 100644
--- a/tools/vdb-config/vdb-config-model.cpp
+++ b/tools/vdb-config/vdb-config-model.cpp
@@ -49,7 +49,7 @@ std::string vdbconf_model::native_to_internal( const std::string &s ) const
if ( rc == 0 )
{
size_t written;
- char buffer[ 4096 ];
+ char buffer[ PATH_MAX ];
rc = VPathReadPath ( temp_v_path, buffer, sizeof buffer, &written );
if ( rc == 0 )
res.assign( buffer, written );
@@ -66,7 +66,7 @@ std::string vdbconf_model::internal_to_native( const std::string &s ) const
if ( rc == 0 )
{
size_t written;
- char buffer[ 4096 ];
+ char buffer[ PATH_MAX ];
rc = VPathReadSysPath ( temp_v_path, buffer, sizeof buffer, &written );
if ( rc == 0 )
res.assign( buffer, written );
@@ -76,6 +76,18 @@ std::string vdbconf_model::internal_to_native( const std::string &s ) const
}
+std::string vdbconf_model::get_http_proxy_path( void ) const {
+ char buffer[ PATH_MAX ] = "";
+ rc_t rc = KConfig_Get_Http_Proxy_Path(_config, buffer, sizeof buffer, NULL);
+ if (rc == 0) {
+ return buffer;
+ }
+ else {
+ return "";
+ }
+}
+
+
bool vdbconf_model::does_site_repo_exist( void ) const
{
KRepositoryVector repositories;
diff --git a/tools/vdb-config/vdb-config-model.hpp b/tools/vdb-config/vdb-config-model.hpp
index 31e1732..460fc83 100644
--- a/tools/vdb-config/vdb-config-model.hpp
+++ b/tools/vdb-config/vdb-config-model.hpp
@@ -108,6 +108,31 @@ class vdbconf_model
bool get_config_changed( void ) const { return _config_changed; }
// ----------------------------------------------------------------
+
+ bool is_http_proxy_enabled( void ) const {
+ bool enabled = true;
+ KConfig_Get_Http_Proxy_Enabled(_config, &enabled, true);
+ if ( enabled )
+ {
+ std::string path = get_http_proxy_path();
+ if ( path.empty() ) enabled = false;
+ }
+ return enabled;
+ }
+
+ void set_http_proxy_enabled( bool enabled ) {
+ KConfig_Set_Http_Proxy_Enabled(_config, enabled);
+ _config_changed = true;
+ }
+
+ std::string get_http_proxy_path( void ) const;
+
+ void set_http_proxy_path(const std::string &path) {
+ KConfig_Set_Http_Proxy_Path(_config, path.c_str());
+ _config_changed = true;
+ }
+
+ // ----------------------------------------------------------------
bool is_remote_enabled( void ) const
{
bool res = false;
@@ -353,21 +378,26 @@ class vdbconf_model
}
// ----------------------------------------------------------------
- bool import_ngc( const std::string &native_location, const KNgcObj *ngc, uint32_t permissions, uint32_t * result_flags )
+ bool import_ngc( const std::string &native_location,
+ const KNgcObj *ngc, uint32_t permissions, uint32_t * result_flags )
{
bool res = false;
+
if ( _config_valid )
{
KRepositoryMgr * repo_mgr;
rc_t rc = KConfigMakeRepositoryMgrUpdate ( _config, &repo_mgr );
if ( rc == 0 )
{
- std::string location = native_to_internal( native_location );
- rc = KRepositoryMgrImportNgcObj( repo_mgr, ngc, location.c_str(), permissions, result_flags );
+ std::string location = native_to_internal( native_location);
+
+ rc = KRepositoryMgrImportNgcObj( repo_mgr, ngc,
+ location.c_str(), permissions, result_flags );
res = ( rc == 0 );
KRepositoryMgrRelease( repo_mgr );
}
}
+
return res;
}
@@ -388,16 +418,35 @@ class vdbconf_model
return res;
}
- bool does_path_exist( std::string &path )
- {
- bool res = false;
- if ( _dir != NULL )
- {
- KPathType type = KDirectoryPathType( _dir, path.c_str() );
- res = ( ( type & ~kptAlias ) == kptDir );
- }
- return res;
- }
+ bool mkdir(const KNgcObj *ngc) {
+ uint32_t id = 0;
+ if (!get_id_of_ngc_obj(ngc, &id)) {
+ return false;
+ }
+
+ const std::string root(get_repo_location(id));
+ if (root.size() == 0) {
+ return false;
+ }
+
+ if (KDirectoryPathType(_dir, root.c_str()) != kptNotFound) {
+ return false;
+ }
+
+ return KDirectoryCreateDir(_dir, 0775,
+ kcmCreate | kcmParents, root.c_str()) == 0;
+ }
+
+ bool does_path_exist( std::string &path )
+ {
+ bool res = false;
+ if ( _dir != NULL )
+ {
+ KPathType type = KDirectoryPathType( _dir, path.c_str() );
+ res = ( ( type & ~kptAlias ) == kptDir );
+ }
+ return res;
+ }
bool reload( void )
{
diff --git a/tools/vdb-config/vdb-config.c b/tools/vdb-config/vdb-config.c
index e312f98..f24178a 100644
--- a/tools/vdb-config/vdb-config.c
+++ b/tools/vdb-config/vdb-config.c
@@ -28,6 +28,7 @@
#include "configure.h"
#include <kapp/main.h>
+#include <kapp/args-conv.h>
#include <vdb/vdb-priv.h> /* VDBManagerListExternalSchemaModules */
#include <vdb/manager.h> /* VDBManager */
@@ -57,6 +58,7 @@
#include <os-native.h> /* SHLX */
#include <assert.h>
+#include <ctype.h> /* tolower */
#include <errno.h>
#include <stdio.h> /* scanf */
#include <stdlib.h> /* getenv */
@@ -125,6 +127,15 @@ static const char* USAGE_OUT[] = { "output type: one of (x n), "
#define OPTION_PCF "cfg"
static const char* USAGE_PCF[] = { "print current configuration", NULL };
+#define ALIAS_PRD NULL
+#define OPTION_PRD "proxy-disable"
+static const char* USAGE_PRD[] = { "enable/disable using HTTP proxy", NULL };
+
+#define ALIAS_PRX NULL
+#define OPTION_PRX "proxy"
+static const char* USAGE_PRX[]
+ = { "set HTTP proxy server configuration", NULL };
+
#define ALIAS_ROOT NULL
#define OPTION_ROOT "root"
static const char* USAGE_ROOT[] =
@@ -134,31 +145,57 @@ static const char* USAGE_ROOT[] =
#define OPTION_SET "set"
static const char* USAGE_SET[] = { "set configuration node value", NULL };
+rc_t WorkspaceDirPathConv(const Args * args, uint32_t arg_index, const char * arg, size_t arg_len, void ** result, WhackParamFnP * whack)
+{
+ rc_t rc;
+ uint32_t imp_count;
+
+ rc = ArgsOptionCount(args, OPTION_IMP, &imp_count);
+ if (rc != 0)
+ return rc;
+
+ // first parameter is a directory only if OPTION_IMP is present; otherwise it is a query
+ if (imp_count > 0)
+ {
+ return ArgsConvFilepath(args, arg_index, arg, arg_len, result, whack);
+ }
+
+ return ArgsConvDefault(args, arg_index, arg, arg_len, result, whack);
+}
+
OptDef Options[] =
-{ /* needs_value, required */
- { OPTION_ALL, ALIAS_ALL, NULL, USAGE_ALL, 1, false, false }
- , { OPTION_CFG, ALIAS_CFG, NULL, USAGE_CFG, 1, false, false }
- , { OPTION_CFM, ALIAS_CFM, NULL, USAGE_CFM, 1, true , false }
- , { OPTION_DIR, ALIAS_DIR, NULL, USAGE_DIR, 1, false, false }
- , { OPTION_ENV, ALIAS_ENV, NULL, USAGE_ENV, 1, false, false }
- , { OPTION_FIL, ALIAS_FIL, NULL, USAGE_FIL, 1, false, false }
- , { OPTION_FIX, ALIAS_FIX, NULL, USAGE_FIX, 1, false, false }
- , { OPTION_IMP, ALIAS_IMP, NULL, USAGE_IMP, 1, true , false }
- , { OPTION_MOD, ALIAS_MOD, NULL, USAGE_MOD, 1, false, false }
- , { OPTION_OUT, ALIAS_OUT, NULL, USAGE_OUT, 1, true , false }
- , { OPTION_PCF, ALIAS_PCF, NULL, USAGE_PCF, 1, false, false }
- , { OPTION_SET, ALIAS_SET, NULL, USAGE_SET, 1, true , false }
- , { OPTION_ROOT,ALIAS_ROOT,NULL, USAGE_ROOT,1, false, false }
+{ /* needs_value, required, converter */
+ { OPTION_ALL, ALIAS_ALL, NULL, USAGE_ALL, 1, false, false, NULL }
+ , { OPTION_CFG, ALIAS_CFG, NULL, USAGE_CFG, 1, false, false, NULL }
+ , { OPTION_CFM, ALIAS_CFM, NULL, USAGE_CFM, 1, true , false, NULL }
+ , { OPTION_DIR, ALIAS_DIR, NULL, USAGE_DIR, 1, false, false, NULL }
+ , { OPTION_ENV, ALIAS_ENV, NULL, USAGE_ENV, 1, false, false, NULL }
+ , { OPTION_FIL, ALIAS_FIL, NULL, USAGE_FIL, 1, false, false, NULL }
+ , { OPTION_FIX, ALIAS_FIX, NULL, USAGE_FIX, 1, false, false, NULL }
+ , { OPTION_IMP, ALIAS_IMP, NULL, USAGE_IMP, 1, true , false, ArgsConvFilepath }
+ , { OPTION_MOD, ALIAS_MOD, NULL, USAGE_MOD, 1, false, false, NULL }
+ , { OPTION_OUT, ALIAS_OUT, NULL, USAGE_OUT, 1, true , false, NULL }
+ , { OPTION_PCF, ALIAS_PCF, NULL, USAGE_PCF, 1, false, false, NULL }
+ , { OPTION_PRD, ALIAS_PRD, NULL, USAGE_PRD, 1, true , false, NULL }
+ , { OPTION_PRX, ALIAS_PRX, NULL, USAGE_PRX, 1, true , false, NULL }
+ , { OPTION_SET, ALIAS_SET, NULL, USAGE_SET, 1, true , false, NULL }
+ , { OPTION_ROOT,ALIAS_ROOT,NULL, USAGE_ROOT,1, false, false, NULL }
+};
+
+ParamDef Parameters[] =
+{
+ { WorkspaceDirPathConv }
};
rc_t CC UsageSummary (const char * progname) {
return KOutMsg (
"Usage:\n"
- " %s [options] [<query> ...]\n"
+ " %s [options] [<query> ...]\n\n"
+ " %s [options] --import <ngc-file> [<workspace directory path>]\n"
"\n"
"Summary:\n"
" Manage VDB configuration\n"
- "\n", progname);
+ , progname, progname);
}
rc_t CC Usage(const Args* args) {
@@ -188,12 +225,14 @@ rc_t CC Usage(const Args* args) {
KOutMsg ("\n");
HelpOptionLine (ALIAS_CFG, OPTION_CFG, NULL, USAGE_CFG);
HelpOptionLine (ALIAS_CFM, OPTION_CFM, "mode", USAGE_CFM);
- HelpOptionLine (ALIAS_FIX, OPTION_FIX, NULL, USAGE_FIX);
KOutMsg ("\n");
- HelpOptionLine (ALIAS_IMP, OPTION_IMP, "ngc-file", USAGE_IMP);
+ HelpOptionLine (ALIAS_FIX, OPTION_FIX, NULL, USAGE_FIX);
KOutMsg ("\n");
HelpOptionLine (ALIAS_OUT, OPTION_OUT, "x | n", USAGE_OUT);
KOutMsg ("\n");
+ HelpOptionLine (ALIAS_PRX, OPTION_PRX, "uri[:port]", USAGE_PRX);
+ HelpOptionLine (ALIAS_PRD, OPTION_PRD, "yes | no", USAGE_PRD);
+ KOutMsg ("\n");
HelpOptionLine (ALIAS_ROOT,OPTION_ROOT,NULL, USAGE_ROOT);
KOutMsg ("\n");
@@ -231,11 +270,29 @@ static rc_t KConfigNodeReadData(const KConfigNode* self,
static
rc_t _printNodeData(const char *name, const char *data, size_t dlen)
{
- const char ticket[] = "download-ticket";
- size_t l = sizeof ticket - 1;
- if (string_cmp(name, string_measure(name, NULL),
- ticket, l, (uint32_t)l) == 0)
+ bool secret = false;
{
+ const char d1[] = "download-ticket";
+ size_t l1 = sizeof d1 - 1;
+
+ const char d2[] = "aws_access_key_id";
+ size_t l2 = sizeof d2 - 1;
+
+ const char d3[] = "aws_secret_access_key";
+ size_t l3 = sizeof d3 - 1;
+
+ if ((string_cmp(name,
+ string_measure(name, NULL), d1, l1, (uint32_t)l1) == 0) ||
+ (string_cmp(name,
+ string_measure(name, NULL), d2, l2, (uint32_t)l2) == 0) ||
+ (string_cmp(name,
+ string_measure(name, NULL), d3, l3, (uint32_t)l3) == 0))
+ {
+ secret = true;
+ }
+ }
+
+ if (secret) {
const char *ellipsis = "";
const char replace[] =
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
@@ -263,6 +320,7 @@ static rc_t KConfigNodePrintChildNames(bool xml, const KConfigNode* self,
bool hasData = false;
const KConfigNode* node = NULL;
KNamelist* names = NULL;
+ bool beginsWithNumberinXml = false;
assert(self && name);
if (rc == 0)
@@ -274,19 +332,30 @@ static rc_t KConfigNodePrintChildNames(bool xml, const KConfigNode* self,
/* VDB_CONGIG_OUTMSG(("\n%s = \"%.*s\"\n\n", aFullpath, num_read, buffer)); */
}
}
- if (rc == 0)
- { rc = KConfigNodeListChild(node, &names); }
+ if (rc == 0) {
+ rc = KConfigNodeListChild(node, &names);
+ }
if (rc == 0) {
rc = KNamelistCount(names, &count);
hasChildren = count;
}
Indent(xml, indent);
- VDB_CONGIG_OUTMSG(("<%s", name));
- if (!hasChildren && !hasData)
- { VDB_CONGIG_OUTMSG(("/>\n")); }
- else
- { VDB_CONGIG_OUTMSG((">")); }
+ if (xml) {
+ beginsWithNumberinXml = isdigit(name[0]);
+ if (! beginsWithNumberinXml) {
+ VDB_CONGIG_OUTMSG(("<%s", name));
+ }
+ else {
+ /* XML node names cannot start with a number */
+ VDB_CONGIG_OUTMSG(("<_%s", name));
+ }
+ }
+ if (!hasChildren && !hasData) {
+ VDB_CONGIG_OUTMSG(("/>\n"));
+ }
+ else { VDB_CONGIG_OUTMSG((">"));
+ }
if (hasData) {
if (xml) {
_printNodeData(name, buffer, num_read);
@@ -328,7 +397,14 @@ static rc_t KConfigNodePrintChildNames(bool xml, const KConfigNode* self,
if (hasChildren)
{ Indent(xml, indent); }
if (hasChildren || hasData)
- { VDB_CONGIG_OUTMSG(("</%s>\n",name)); }
+ {
+ if (! beginsWithNumberinXml) {
+ VDB_CONGIG_OUTMSG(("</%s>\n",name));
+ }
+ else {
+ VDB_CONGIG_OUTMSG(("</_%s>\n",name));
+ }
+ }
RELEASE(KNamelist, names);
RELEASE(KConfigNode, node);
@@ -342,9 +418,9 @@ typedef struct Params {
bool xml;
- const char* setValue;
+ const char *setValue;
- const char* ngc;
+ const char *ngc;
bool modeSetNode;
bool modeConfigure;
@@ -358,6 +434,13 @@ typedef struct Params {
bool modeRoot;
bool showMultiple;
+
+ enum {
+ eUndefined,
+ eNo,
+ eYes
+ } proxyDisabled;
+ const char *proxy;
} Params;
static rc_t ParamsConstruct(int argc, char* argv[], Params* prm) {
rc_t rc = 0;
@@ -368,12 +451,12 @@ static rc_t ParamsConstruct(int argc, char* argv[], Params* prm) {
args = prm->args;
do {
uint32_t pcount = 0;
- rc = ArgsMakeAndHandle(&args, argc, argv, 1,
- Options, sizeof Options / sizeof (OptDef));
+ rc = ArgsMakeAndHandle2(&args, argc, argv, Parameters, sizeof Parameters / sizeof Parameters[0], 1, Options, sizeof Options / sizeof Options[0]);
if (rc) {
- LOGERR(klogErr, rc, "While calling ArgsMakeAndHandle");
+ LOGERR(klogErr, rc, "While calling ArgsMakeAndHandle2");
break;
}
+
prm->args = args;
rc = ArgsParamCount(args, &prm->argsParamCnt);
if (rc) {
@@ -385,249 +468,285 @@ static rc_t ParamsConstruct(int argc, char* argv[], Params* prm) {
++count;
}
- { // OPTION_OUT
- prm->xml = true;
- rc = ArgsOptionCount(args, OPTION_OUT, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_OUT "' argument");
- break;
- }
- if (pcount) {
- const char* dummy = NULL;
- rc = ArgsOptionValue(args, OPTION_OUT, 0, &dummy);
+ { // OPTION_OUT
+ prm->xml = true;
+ rc = ArgsOptionCount(args, OPTION_OUT, &pcount);
if (rc) {
LOGERR(klogErr, rc, "Failure to get '" OPTION_OUT "' argument");
break;
}
- if (!strcmp(dummy, "n")) {
- prm->xml = false;
+ if (pcount) {
+ const char* dummy = NULL;
+ rc = ArgsOptionValue(args, OPTION_OUT, 0, (const void **)&dummy);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_OUT "' argument");
+ break;
+ }
+ if (!strcmp(dummy, "n")) {
+ prm->xml = false;
+ }
+ else if (strcmp(dummy, "x")) {
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
+ LOGERR(klogErr, rc, "Bad " OPTION_OUT " value");
+ break;
+ }
}
- else if (strcmp(dummy, "x")) {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
- LOGERR(klogErr, rc, "Bad " OPTION_OUT " value");
+ }
+ { // OPTION_ENV
+ rc = ArgsOptionCount(args, OPTION_ENV, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_ENV "' argument");
break;
}
+ if (pcount) {
+ prm->modeShowEnv = true;
+ ++count;
+ }
}
- }
- { // OPTION_ENV
- rc = ArgsOptionCount(args, OPTION_ENV, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_ENV "' argument");
- break;
- }
- if (pcount) {
- prm->modeShowEnv = true;
- ++count;
- }
- }
- { // OPTION_FIL
- rc = ArgsOptionCount(args, OPTION_FIL, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_FIL "' argument");
- break;
- }
- if (pcount) {
- prm->modeShowFiles = true;
- ++count;
- }
- }
- { // OPTION_IMP
- rc = ArgsOptionCount(args, OPTION_IMP, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_IMP "' argument");
- break;
- }
- if (pcount) {
- rc = ArgsOptionValue(args, OPTION_IMP, 0, &prm->ngc);
+ { // OPTION_FIL
+ rc = ArgsOptionCount(args, OPTION_FIL, &pcount);
if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_IMP "' argument");
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_FIL "' argument");
break;
}
- else {
- prm->modeShowCfg = false;
+ if (pcount > 0) {
+ prm->modeShowFiles = true;
+ ++count;
}
}
- }
- { // OPTION_MOD
- rc = ArgsOptionCount(args, OPTION_MOD, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_MOD "' argument");
- break;
+ { // OPTION_IMP
+ rc = ArgsOptionCount(args, OPTION_IMP, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_IMP "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ rc = ArgsOptionValue(args, OPTION_IMP, 0, (const void **)&prm->ngc);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_IMP "' argument");
+ break;
+ }
+ else {
+ prm->modeShowCfg = false;
+ }
+ }
}
- if (pcount) {
- prm->modeShowModules = true;
- ++count;
+ { // OPTION_MOD
+ rc = ArgsOptionCount(args, OPTION_MOD, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_MOD "' argument");
+ break;
+ }
+ if (pcount) {
+ prm->modeShowModules = true;
+ ++count;
+ }
}
- }
#if 0
- { // OPTION_NEW
- rc = ArgsOptionCount(args, OPTION_NEW, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_NEW "' argument");
- break;
- }
- if (pcount) {
- prm->modeCreate = true;
- ++count;
- }
- }
-#endif
- { // OPTION_PCF
- rc = ArgsOptionCount(args, OPTION_PCF, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_PCF "' argument");
- break;
- }
- if (pcount) {
- if (!prm->modeShowCfg) {
- prm->modeShowCfg = true;
+ { // OPTION_NEW
+ rc = ArgsOptionCount(args, OPTION_NEW, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_NEW "' argument");
+ break;
+ }
+ if (pcount) {
+ prm->modeCreate = true;
++count;
}
}
- }
- { // OPTION_DIR
- rc = ArgsOptionCount(args, OPTION_DIR, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_DIR "' argument");
- break;
- }
- if (pcount) {
- prm->modeShowLoadPath = true;
- ++count;
- }
- }
- { // OPTION_ROOT
- rc = ArgsOptionCount(args, OPTION_ROOT, &pcount);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_ROOT "' argument");
- break;
- }
- if (pcount) {
- prm->modeRoot = true;
+#endif
+ { // OPTION_PCF
+ rc = ArgsOptionCount(args, OPTION_PCF, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_PCF "' argument");
+ break;
+ }
+ if (pcount) {
+ if (!prm->modeShowCfg) {
+ prm->modeShowCfg = true;
+ ++count;
+ }
+ }
}
- }
- { // OPTION_SET
- rc = ArgsOptionCount(args, OPTION_SET, &pcount);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_SET "' argument");
- break;
+ { // OPTION_PRD
+ rc = ArgsOptionCount(args, OPTION_PRD, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_PRD "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ const char *dummy = NULL;
+ rc = ArgsOptionValue(args, OPTION_PRD, 0, (const void **)&dummy);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_PRD "' argument");
+ break;
+ }
+ if (tolower(dummy[0]) == 'y') {
+ prm->proxyDisabled = eYes;
+ }
+ else if (tolower(dummy[0]) == 'n') {
+ prm->proxyDisabled = eNo;
+ }
+ }
}
- if (pcount) {
- rc = ArgsOptionValue(args, OPTION_SET, 0, &prm->setValue);
- if (rc == 0) {
- const char* p = strchr(prm->setValue, '=');
- if (p == NULL || *(p + 1) == '\0') {
- rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
- LOGERR(klogErr, rc, "Bad " OPTION_SET " value");
+ { // OPTION_PRX
+ rc = ArgsOptionCount(args, OPTION_PRX, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_PRX "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ rc = ArgsOptionValue(args, OPTION_PRX, 0, (const void **)&prm->proxy);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_PRX "' argument");
break;
}
- prm->modeSetNode = true;
- prm->modeCreate = prm->modeShowCfg = prm->modeShowEnv
- = prm->modeShowFiles = prm->modeShowLoadPath
- = prm->modeShowModules = false;
- count = 1;
}
}
- }
- { // OPTION_FIX
- rc = ArgsOptionCount(args, OPTION_FIX, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_FIX "' argument");
- break;
+ { // OPTION_DIR
+ rc = ArgsOptionCount(args, OPTION_DIR, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_DIR "' argument");
+ break;
+ }
+ if (pcount) {
+ prm->modeShowLoadPath = true;
+ ++count;
+ }
}
- if (pcount) {
- prm->modeConfigure = true;
- prm->modeShowCfg = false;
- count = 1;
- prm->configureMode = eCfgModeDefault;
+ { // OPTION_ROOT
+ rc = ArgsOptionCount(args, OPTION_ROOT, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_ROOT "' argument");
+ break;
+ }
+ if (pcount) {
+ prm->modeRoot = true;
+ }
}
- }
- { // OPTION_CFG
- rc = ArgsOptionCount(args, OPTION_CFG, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_CFG "' argument");
- break;
+ { // OPTION_SET
+ rc = ArgsOptionCount(args, OPTION_SET, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_SET "' argument");
+ break;
+ }
+ if (pcount) {
+ rc = ArgsOptionValue(args, OPTION_SET, 0, (const void **)&prm->setValue);
+ if (rc == 0) {
+ const char* p = strchr(prm->setValue, '=');
+ if (p == NULL || *(p + 1) == '\0') {
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
+ LOGERR(klogErr, rc, "Bad " OPTION_SET " value");
+ break;
+ }
+ prm->modeSetNode = true;
+ prm->modeCreate = prm->modeShowCfg = prm->modeShowEnv
+ = prm->modeShowFiles = prm->modeShowLoadPath
+ = prm->modeShowModules = false;
+ count = 1;
+ }
+ }
+ }
+ { // OPTION_FIX
+ rc = ArgsOptionCount(args, OPTION_FIX, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_FIX "' argument");
+ break;
+ }
+ if (pcount) {
+ prm->modeConfigure = true;
+ prm->modeShowCfg = false;
+ count = 1;
+ prm->configureMode = eCfgModeDefault;
+ }
}
- if (pcount) {
+ { // OPTION_CFG
+ rc = ArgsOptionCount(args, OPTION_CFG, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_CFG "' argument");
+ break;
+ }
+ if (pcount) {
#if 1
- prm->modeConfigure = true;
- prm->modeShowCfg = false;
- count = 1;
- prm->configureMode = eCfgModeVisual;
+ prm->modeConfigure = true;
+ prm->modeShowCfg = false;
+ count = 1;
+ prm->configureMode = eCfgModeVisual;
#else
- const char* dummy = NULL;
- rc = ArgsOptionValue(args, OPTION_CFG, 0, &dummy);
+ const char* dummy = NULL;
+ rc = ArgsOptionValue(args, OPTION_CFG, 0, (const void **)&dummy);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_CFG "' argument");
+ break;
+ }
+ prm->modeConfigure = true;
+ prm->modeShowCfg = false;
+ count = 1;
+ switch (dummy[0]) {
+ case 't':
+ prm->configureMode = eCfgModeTextual;
+ break;
+ default:
+ prm->configureMode = eCfgModeDefault;
+ break;
+ }
+#endif
+ }
+ }
+ { // OPTION_CFM
+ rc = ArgsOptionCount(args, OPTION_CFM, &pcount);
if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_CFG "' argument");
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_CFM "' argument");
break;
}
- prm->modeConfigure = true;
- prm->modeShowCfg = false;
- count = 1;
- switch (dummy[0]) {
- case 't':
- prm->configureMode = eCfgModeTextual;
+ if (pcount) {
+ const char* dummy = NULL;
+ size_t dummy_len;
+ rc = ArgsOptionValue(args, OPTION_CFM, 0, (const void **)&dummy);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_OUT "' argument");
break;
- default:
- prm->configureMode = eCfgModeDefault;
+ }
+ prm->modeShowCfg = false;
+ count = 1;
+ prm->modeConfigure = true;
+
+ dummy_len = strlen( dummy );
+ if ( dummy_len == 0 )
+ dummy_len = 1;
+
+ if ( strncmp( dummy, "textual", dummy_len ) == 0 )
+ prm->configureMode = eCfgModeTextual;
+ else if ( strncmp( dummy, "graphical", dummy_len ) == 0 )
+ prm->configureMode = eCfgModeVisual;
+ else
+ {
+ rc = RC( rcExe, rcArgv, rcEvaluating, rcParam, rcInvalid );
+ LOGERR(klogErr, rc, "Unrecognized '" OPTION_CFM "' argument");
break;
+ }
}
-#endif
- }
- }
- { // OPTION_CFM
- rc = ArgsOptionCount(args, OPTION_CFM, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_CFM "' argument");
- break;
}
- if (pcount) {
- const char* dummy = NULL;
- size_t dummy_len;
- rc = ArgsOptionValue(args, OPTION_CFM, 0, &dummy);
+ { // OPTION_ALL
+ rc = ArgsOptionCount(args, OPTION_ALL, &pcount);
if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_OUT "' argument");
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_ALL "' argument");
break;
}
- prm->modeShowCfg = false;
- count = 1;
- prm->modeConfigure = true;
-
- dummy_len = strlen( dummy );
- if ( dummy_len == 0 )
- dummy_len = 1;
-
- if ( strncmp( dummy, "textual", dummy_len ) == 0 )
- prm->configureMode = eCfgModeTextual;
- else if ( strncmp( dummy, "graphical", dummy_len ) == 0 )
- prm->configureMode = eCfgModeVisual;
- else
+ if (pcount
+ || ( !prm->modeConfigure
+ && !prm->modeShowCfg && ! prm->modeShowLoadPath
+ && !prm->modeShowEnv && !prm->modeShowFiles
+ && !prm->modeShowModules && !prm->modeCreate
+ && !prm->modeSetNode && prm->ngc == NULL
+ && prm->proxy == NULL && prm->proxyDisabled == eUndefined))
+ /* show all by default */
{
- rc = RC( rcExe, rcArgv, rcEvaluating, rcParam, rcInvalid );
- LOGERR(klogErr, rc, "Unrecognized '" OPTION_CFM "' argument");
- break;
+ prm->modeShowCfg = prm->modeShowEnv = prm->modeShowFiles = true;
+ count += 2;
}
}
- }
- { // OPTION_ALL
- rc = ArgsOptionCount(args, OPTION_ALL, &pcount);
- if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_ALL "' argument");
- break;
- }
- if (pcount
- || ( !prm->modeConfigure
- && !prm->modeShowCfg && ! prm->modeShowLoadPath
- && !prm->modeShowEnv && !prm->modeShowFiles
- && !prm->modeShowModules && !prm->modeCreate
- && !prm->modeSetNode && !prm->ngc))
- /* show all by default */
- {
- prm->modeShowCfg = prm->modeShowEnv = prm->modeShowFiles = true;
- count += 2;
- }
- }
if (count > 1) {
prm->showMultiple = true;
@@ -642,7 +761,7 @@ static rc_t ParamsGetNextParam(Params* prm, const char** param) {
assert(prm && param);
*param = NULL;
if (prm->argsParamIdx < prm->argsParamCnt) {
- rc = ArgsParamValue(prm->args, prm->argsParamIdx++, param);
+ rc = ArgsParamValue(prm->args, prm->argsParamIdx++, (const void **)param);
if (rc)
{ LOGERR(klogErr, rc, "Failure retrieving query"); }
}
@@ -999,6 +1118,62 @@ static rc_t SetNode(KConfig* cfg, const Params* prm) {
return rc;
}
+static rc_t SetProxy(KConfig *cfg, const Params *prm) {
+ rc_t rc = 0;
+
+ bool disabled = false;
+ bool set = false;
+ bool disableSet = false;
+
+ assert(prm);
+
+ switch (prm->proxyDisabled) {
+ case eNo:
+ disabled = false;
+ break;
+ case eYes:
+ disabled = true;
+ break;
+ default:
+ break;
+ }
+
+ if (prm->proxy != NULL) {
+ rc = KConfig_Set_Http_Proxy_Path(cfg, prm->proxy);
+ set = true;
+
+ if (rc == 0 && prm->proxyDisabled == eUndefined) {
+ rc = KConfig_Set_Http_Proxy_Enabled(cfg, true);
+ }
+ }
+
+ if (prm->proxyDisabled != eUndefined) {
+ rc_t r2 = KConfig_Set_Http_Proxy_Enabled(cfg, ! disabled);
+ disableSet = true;
+ if (r2 != 0 && rc == 0) {
+ rc = r2;
+ }
+ }
+
+ if (rc == 0) {
+ rc = KConfigCommit(cfg);
+ DISP_RC(rc, "while calling KConfigCommit");
+ }
+ if (rc == 0) {
+ if (set) {
+ OUTMSG(("Use HTTP proxy server configuration '%s'\n", prm->proxy));
+ }
+ if (disableSet) {
+ OUTMSG(("HTTP proxy was %s\n", disabled ? "disabled" : "enabled"));
+ }
+ }
+ else {
+ LOGERR(klogErr, rc, "Failed to update HTTP proxy configuration");
+ }
+
+ return rc;
+}
+
static rc_t ShowConfig(const KConfig* cfg, Params* prm) {
rc_t rc = 0;
bool hasAny = false;
@@ -1168,28 +1343,40 @@ static rc_t ShowFiles(const KConfig* cfg, const Params* prm) {
uint32_t count = 0;
KNamelist* names = NULL;
rc = KConfigListIncluded(cfg, &names);
- if (rc == 0)
- { rc = KNamelistCount(names, &count); }
+ if (rc == 0) {
+ rc = KNamelistCount(names, &count);
+ }
if (rc == 0) {
uint32_t i = 0;
if (prm->showMultiple) {
- OUTMSG(("<!-- Configuration files -->\n"));
+ if (prm->xml) {
+ OUTMSG(("<ConfigurationFiles>\n"));
+ }
+ else {
+ OUTMSG(("<!-- Configuration files -->\n"));
+ }
hasAny = true;
}
for (i = 0; i < count && rc == 0; ++i) {
const char* name = NULL;
- if (rc == 0)
- { rc = KNamelistGet(names, i, &name); }
+ if (rc == 0) {
+ rc = KNamelistGet(names, i, &name);
+ }
if (rc == 0) {
OUTMSG(("%s\n", name));
hasAny = true;
}
}
}
- if (rc == 0 && hasAny)
- { OUTMSG(("\n")); }
+ if (prm->showMultiple && prm->xml) {
+ OUTMSG(("</ConfigurationFiles>"));
+ }
+
+ if (rc == 0 && hasAny) {
+ OUTMSG(("\n"));
+ }
RELEASE(KNamelist, names);
@@ -1207,7 +1394,12 @@ static void ShowEnv(const Params* prm) {
int i = 0;
if (prm->showMultiple) {
- OUTMSG(("<!-- Environment -->\n"));
+ if (prm->xml) {
+ OUTMSG(("<Environment>\n"));
+ }
+ else {
+ OUTMSG(("<!-- Environment -->\n"));
+ }
hasAny = true;
}
@@ -1218,6 +1410,9 @@ static void ShowEnv(const Params* prm) {
hasAny = true;
}
}
+ if (prm->showMultiple && prm->xml) {
+ OUTMSG(("</Environment>"));
+ }
if (hasAny) {
OUTMSG(("\n"));
}
@@ -1239,11 +1434,65 @@ static rc_t _VFSManagerSystem2PosixPath(const VFSManager *self,
return rc;
}
-static rc_t ImportNgc(KConfig *cfg, Params *prm, const char **newRepoParentPath)
+static rc_t DefaultPepoLocation(const KConfig *cfg,
+ uint32_t id, char *buffer, size_t bsize)
{
- VFSManager *vmgr = NULL;
- rc_t rc = VFSManagerMake(&vmgr);
- char ngcPath[PATH_MAX] = "";
+ rc_t rc = 0;
+ char home[PATH_MAX] = "";
+ size_t written = 0;
+ assert(buffer && bsize);
+ rc = KConfig_Get_Default_User_Path(cfg, home, sizeof home, &written);
+ if (rc == 0 && written > 0) {
+ rc = string_printf(buffer, bsize, &written, "%s/dbGaP-%u", home, id);
+ if (rc == 0) {
+ return rc;
+ }
+ }
+
+ rc = KConfig_Get_Home(cfg, home, sizeof home, &written);
+ if (rc == 0 && written > 0) {
+ rc = string_printf(buffer, bsize, &written,
+ "%s/ncbi/dbGaP-%u", home, id);
+ if (rc == 0) {
+ return rc;
+ }
+ }
+
+ {
+ VFSManager *vmgr = NULL;
+ rc = VFSManagerMake(&vmgr);
+ if (rc == 0) {
+ const char *home = getenv("HOME");
+ if (home == NULL) {
+ home = getenv("USERPROFILE");
+ }
+ if (home == NULL) {
+#define TODO 1
+ rc = TODO;
+ }
+ else {
+ size_t num_writ = 0;
+ char posix[PATH_MAX] = "";
+ rc = _VFSManagerSystem2PosixPath(vmgr, home, posix);
+ if (rc == 0) {
+ rc = string_printf(buffer, bsize, &num_writ,
+ "%s/ncbi/dbGaP-%u", posix, id);
+ if (rc == 0) {
+ return rc;
+ }
+ }
+ }
+ }
+ RELEASE(VFSManager, vmgr);
+ }
+
+ return rc;
+}
+
+static rc_t DoImportNgc(KConfig *cfg, Params *prm,
+ const char **newRepoParentPath, uint32_t *result_flags)
+{
+ rc_t rc = 0;
KDirectory *dir = NULL;
const KFile *src = NULL;
const KNgcObj *ngc = NULL;
@@ -1253,86 +1502,119 @@ static rc_t ImportNgc(KConfig *cfg, Params *prm, const char **newRepoParentPath)
assert(prm);
if (rc == 0) {
- rc = _VFSManagerSystem2PosixPath(vmgr, prm->ngc, ngcPath);
- }
- if (rc == 0) {
rc = KDirectoryNativeDir(&dir);
}
if (rc == 0) {
- rc = KDirectoryOpenFileRead(dir, &src, "%s", ngcPath);
+ rc = KDirectoryOpenFileRead(dir, &src, "%s", prm->ngc);
}
- RELEASE(KDirectory, dir);
if (rc == 0) {
rc = KNgcObjMakeFromFile(&ngc, src);
}
RELEASE(KFile, src);
+
if (rc == 0) {
uint32_t id = 0;
- size_t written = 0;
rc = KNgcObjGetProjectId(ngc, &id);
if (rc == 0) {
- rc = ParamsGetNextParam(prm, &root);
- root = NULL; // TODO: we cannot accept repo default path from cmd ln
- if (rc != 0 || root == NULL) {
- char home[PATH_MAX] = "";
- rc = KConfig_Get_Default_User_Path(cfg,
- home, sizeof home, &written);
- if (rc == 0 && written > 0) {
- rc = string_printf(buffer, sizeof buffer,
- &written, "%s/dbGaP-%u", home, id);
+ const char *p = NULL;
+ rc = ParamsGetNextParam(prm, &p);
+ if (rc == 0 && p != NULL) {
+ rc = KDirectoryResolvePath(dir, true, buffer, sizeof buffer, p);
+ if (rc == 0) {
root = buffer;
}
}
- rc = 0;
- if (root == NULL && rc == 0) {
- char home[PATH_MAX] = "";
- rc = KConfig_Get_Home(cfg, home, sizeof home, &written);
- if (rc == 0 && written > 0) {
- rc = string_printf(buffer, sizeof buffer,
- &written, "%s/ncbi/dbGaP-%u", home, id);
+ else {
+ rc = DefaultPepoLocation(cfg, id, buffer, sizeof buffer);
+ if (rc == 0) {
root = buffer;
}
}
- if (root == NULL && rc == 0) {
- const char *home = getenv("HOME");
- if (home == NULL) {
- home = getenv("USERPROFILE");
- }
- if (home == NULL) {
-#define TODO 1
- rc = TODO;
- }
- else {
- size_t num_writ = 0;
- char posix[PATH_MAX] = "";
- rc = _VFSManagerSystem2PosixPath(vmgr, root, posix);
- if (rc == 0) {
- rc = string_printf(buffer, sizeof buffer,
- &num_writ, "%s/ncbi/dbGaP-%u", posix, id);
- }
- if (rc == 0) {
- root = buffer;
- }
- }
- }
}
}
- RELEASE(VFSManager, vmgr);
+ RELEASE(KDirectory, dir);
if (rc == 0) {
rc = KConfigMakeRepositoryMgrUpdate(cfg, &rmgr);
}
if (rc == 0) {
- uint32_t result_flags = 0;
assert(root);
rc = KRepositoryMgrImportNgcObj(rmgr, ngc, root,
- INP_CREATE_REPOSITORY, &result_flags);
+ INP_CREATE_REPOSITORY | INP_UPDATE_DNLD_TICKET | INP_UPDATE_ROOT,
+ result_flags);
}
+
*newRepoParentPath = root;
RELEASE(KNgcObj, ngc);
RELEASE(KRepositoryMgr, rmgr);
return rc;
}
+static rc_t ImportNgc(KConfig *cfg, Params *prm) {
+ const char *newRepoParentPath = NULL;
+ uint32_t result_flags = 0;
+ rc_t rc = 0;
+ assert(prm);
+ rc = DoImportNgc(cfg, prm, &newRepoParentPath, &result_flags);
+ DISP_RC2(rc, "cannot import ngc file", prm->ngc);
+ if (rc == 0 && result_flags != 0) {
+ rc = KConfigCommit(cfg);
+ }
+ if (rc == 0) {
+#if WINDOWS
+ char ngcPath[PATH_MAX] = "";
+ char system[MAX_PATH] = "";
+#endif
+ KDirectory *wd = NULL;
+ const char *ngc = prm->ngc;
+ rc_t rc = KDirectoryNativeDir(&wd);
+#if WINDOWS
+ if (rc == 0) {
+ rc = KDirectoryPosixStringToSystemString(wd,
+ system, sizeof system, "%s", newRepoParentPath);
+ if (rc == 0) {
+ newRepoParentPath = system;
+ }
+ rc = KDirectoryPosixStringToSystemString(wd,
+ ngcPath, sizeof ngcPath, "%s", ngc);
+ if (rc == 0) {
+ ngc = ngcPath;
+ }
+ }
+#endif
+ if (wd) {
+ if (KDirectoryPathType(wd, newRepoParentPath)
+ == kptNotFound)
+ {
+ KDirectoryCreateDir(wd, 0775,
+ kcmCreate | kcmParents, newRepoParentPath);
+ }
+ }
+
+ if (result_flags == 0) {
+ OUTMSG((
+ "%s was already imported.\nProtected repository is: '%s'.\n",
+ ngc, newRepoParentPath));
+ }
+ else {
+ if (result_flags & INP_CREATE_REPOSITORY) {
+ OUTMSG((
+ "%s was imported.\nNew protected repository was created.\n"
+ "Repository directory is: '%s'.\n",
+ ngc, newRepoParentPath));
+ }
+ else {
+ OUTMSG((
+ "%s was imported.\nProtected repository was updated.\n"
+ "Repository directory is: '%s'.\n",
+ ngc, newRepoParentPath));
+ }
+ }
+
+ RELEASE(KDirectory, wd);
+ }
+ return rc;
+}
+
rc_t CC KMain(int argc, char* argv[]) {
rc_t rc = 0;
@@ -1348,34 +1630,16 @@ rc_t CC KMain(int argc, char* argv[]) {
DISP_RC(rc, "while calling KConfigMake");
}
+ if (rc == 0 && prm.showMultiple && prm.xml) {
+ OUTMSG(("<VdbConfig>\n"));
+ }
+
if (rc == 0) {
if (prm.modeConfigure) {
rc = configure(prm.configureMode);
}
else if (prm.ngc) {
- const char *newRepoParentPath = NULL;
- rc = ImportNgc (cfg,&prm , &newRepoParentPath);
- DISP_RC2(rc, "cannot import ngc file", prm.ngc);
- if (rc == 0) {
- rc = KConfigCommit(cfg);
- }
- if (rc == 0) {
-#if WINDOWS
- KDirectory *wd = NULL;
- char system[MAX_PATH] = "";
- rc_t rc = KDirectoryNativeDir(&wd);
- if (rc == 0) {
- rc = KDirectoryPosixStringToSystemString(wd,
- system, sizeof system, "%s", newRepoParentPath);
- if (rc == 0) {
- newRepoParentPath = system;
- }
- }
-#endif
- OUTMSG((
- "%s was imported\nThe new protected repository is: %s\n",
- prm.ngc, newRepoParentPath));
- }
+ rc = ImportNgc(cfg, &prm);
}
else if (prm.modeSetNode) {
rc_t rc3 = SetNode(cfg, &prm);
@@ -1383,26 +1647,24 @@ rc_t CC KMain(int argc, char* argv[]) {
rc = rc3;
}
}
- if (prm.modeShowCfg) {
- rc_t rc3 = ShowConfig(cfg, &prm);
+ if (prm.proxy != NULL || prm.proxyDisabled != eUndefined) {
+ rc_t rc3 = SetProxy(cfg, &prm);
if (rc3 != 0 && rc == 0) {
rc = rc3;
}
}
- if (prm.modeShowFiles) {
- rc_t rc3 = ShowFiles(cfg, &prm);
+ if (prm.modeShowCfg) {
+ rc_t rc3 = ShowConfig(cfg, &prm);
if (rc3 != 0 && rc == 0) {
rc = rc3;
}
}
-#if 0
- if (prm.modeShowModules) {
- rc_t rc3 = ShowModules(cfg, &prm);
+ if (prm.modeShowFiles) {
+ rc_t rc3 = ShowFiles(cfg, &prm);
if (rc3 != 0 && rc == 0) {
rc = rc3;
}
}
-#endif
if (prm.modeShowLoadPath) {
const char* path = NULL;
rc_t rc3 = KConfigGetLoadPath(cfg, &path);
@@ -1421,10 +1683,15 @@ rc_t CC KMain(int argc, char* argv[]) {
ShowEnv(&prm);
}
+ if (rc == 0 && prm.showMultiple && prm.xml) {
+ OUTMSG(("</VdbConfig>\n"));
+ }
+
RELEASE(KConfig, cfg);
- if (rc == 0 && prm.modeCreate)
- { rc = CreateConfig(argv[0]); }
+ if (rc == 0 && prm.modeCreate) {
+ rc = CreateConfig(argv[0]);
+ }
ParamsDestruct(&prm);
return rc;
diff --git a/tools/vdb-config/vdb-config.vers b/tools/vdb-config/vdb-config.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-config/vdb-config.vers
+++ b/tools/vdb-config/vdb-config.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-config/vdb-config.vers.h b/tools/vdb-config/vdb-config.vers.h
index 7cf38bf..1abbcbd 100644
--- a/tools/vdb-config/vdb-config.vers.h
+++ b/tools/vdb-config/vdb-config.vers.h
@@ -1 +1 @@
-#define VDB_CONFIG_VERS 0x02040002
+#define VDB_CONFIG_VERS 0x02050007
diff --git a/tools/vdb-copy/Makefile b/tools/vdb-copy/Makefile
index 7356529..bf52ffc 100644
--- a/tools/vdb-copy/Makefile
+++ b/tools/vdb-copy/Makefile
@@ -90,13 +90,11 @@ $(NCBIDIR)/vdb-copy.kfg: $(SRCDIR)/vdb-copy.kfg
cp $^ $@
VDB_COPY_SRC = \
- num-gen \
context \
helper \
coldefs \
get_platform \
namelist_tools \
- progressbar \
copy_meta \
type_matcher \
redactval \
diff --git a/tools/vdb-copy/context.c b/tools/vdb-copy/context.c
index 6dfe569..992bb41 100644
--- a/tools/vdb-copy/context.c
+++ b/tools/vdb-copy/context.c
@@ -318,9 +318,12 @@ static rc_t context_set_blob_checksum( p_context ctx, const char *src )
static rc_t context_set_row_range( p_context ctx, const char *src )
{
+ rc_t rc;
if ( ( ctx == NULL )||( src == NULL ) )
- return RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
- return num_gen_parse( ctx->row_generator, src );
+ rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
+ else
+ rc = num_gen_parse( ctx->row_generator, src );
+ return rc;
}
@@ -344,7 +347,7 @@ static rc_t context_evaluate_arguments( const Args *my_args, p_context ctx )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *value = NULL;
- rc = ArgsParamValue( my_args, idx, &value );
+ rc = ArgsParamValue( my_args, idx, (const void **)&value );
DISP_RC( rc, "ArgsParamValue() failed" );
if ( rc == 0 )
{
@@ -386,7 +389,7 @@ static const char* context_get_str_option( const Args *my_args,
rc_t rc = ArgsOptionCount( my_args, name, &count );
if ( ( rc == 0 )&&( count > 0 ) )
{
- rc = ArgsOptionValue( my_args, name, 0, &res );
+ rc = ArgsOptionValue( my_args, name, 0, (const void **)&res );
}
return res;
}
@@ -438,7 +441,11 @@ static void context_evaluate_options( const Args *my_args, p_context ctx )
context_set_columns( ctx, context_get_str_option( my_args, OPTION_COLUMNS ) );
#endif
context_set_excluded_columns( ctx, context_get_str_option( my_args, OPTION_EXCLUDED_COLUMNS ) );
- context_set_row_range( ctx, context_get_str_option( my_args, OPTION_ROWS ) );
+
+ {
+ const char * row_range = context_get_str_option( my_args, OPTION_ROWS );
+ context_set_row_range( ctx, row_range );
+ }
nlt_make_namelist_from_string( &(ctx->src_schema_list),
context_get_str_option( my_args, OPTION_SCHEMA ) );
}
diff --git a/tools/vdb-copy/context.h b/tools/vdb-copy/context.h
index dbeb202..2ca14ea 100644
--- a/tools/vdb-copy/context.h
+++ b/tools/vdb-copy/context.h
@@ -43,9 +43,10 @@ extern "C" {
#include "redactval.h"
#endif
+#include <klib/num-gen.h>
+
#include <kapp/args.h>
#include "definitions.h"
-#include "num-gen.h"
#define OPTION_TABLE "table"
#define OPTION_ROWS "rows"
@@ -110,7 +111,7 @@ typedef struct context
const char *table;
const char *columns;
const char *excluded_columns;
- num_gen *row_generator;
+ struct num_gen * row_generator;
bool usage_requested;
bool dont_check_accession;
uint64_t platform_id;
diff --git a/tools/vdb-copy/definitions.h b/tools/vdb-copy/definitions.h
index 0628692..de539df 100644
--- a/tools/vdb-copy/definitions.h
+++ b/tools/vdb-copy/definitions.h
@@ -43,6 +43,10 @@ extern "C" {
#include <klib/rc.h>
#endif
+#ifndef _h_klib_num_gen_
+#include <klib/num-gen.h>
+#endif
+
#define DISP_RC(rc,err) (void)((rc == 0) ? 0 : LogErr( klogInt, rc, err ))
#define DISP_RC2(rc,err,succ) \
diff --git a/tools/vdb-copy/num-gen.c b/tools/vdb-copy/num-gen.c
index 93125ac..e3c1607 100644
--- a/tools/vdb-copy/num-gen.c
+++ b/tools/vdb-copy/num-gen.c
@@ -88,11 +88,11 @@ static p_num_gen_node num_gen_make_node( const uint64_t start, const uint64_t co
/* helper callback to compare 2 nodes, lets VectorInsert create a sorted vector */
-static int CC num_gen_insert_helper( const void* item1, const void* item2 )
+static int64_t CC num_gen_insert_helper( const void* item1, const void* item2 )
{
const p_num_gen_node node1 = ( p_num_gen_node )item1;
const p_num_gen_node node2 = ( p_num_gen_node )item2;
- int res = 0;
+ int64_t res = 0;
if ( node1->start == node2->start )
{
if ( node1->count < node2->count )
diff --git a/tools/vdb-copy/progressbar.c b/tools/vdb-copy/progressbar.c
deleted file mode 100644
index 85a793a..0000000
--- a/tools/vdb-copy/progressbar.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "vdb-copy-includes.h"
-#include <sysalloc.h>
-#include <stdlib.h>
-
-typedef struct progressbar
-{
- bool initialized;
- uint16_t percent;
-} progressbar;
-
-
-rc_t make_progressbar( progressbar ** pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
- (*pb) = calloc( 1, sizeof( progressbar ) );
- if ( *pb == NULL )
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- return 0;
-}
-
-
-rc_t destroy_progressbar( progressbar * pb )
-{
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
- free( pb );
- return 0;
-}
-
-static void progess_0a( const uint16_t percent )
-{
- KOutMsg( "| %2u%%", percent );
-}
-
-static void progess_0( const uint16_t percent )
-{
- if ( percent & 1 )
- KOutMsg( "\b\b\b\b- %2u%%", percent );
- else
- KOutMsg( "\b\b\b%2u%%", percent );
-}
-
-static void progess_1a( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- KOutMsg( "| %2u.%01u%%", p1, p0 );
-}
-
-static void progess_1( const uint16_t percent )
-{
- uint16_t p1 = percent / 10;
- uint16_t p0 = percent - ( p1 * 10 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- KOutMsg( "\b\b\b\b\b\b- %2u.%01u%%", p1, p0 );
- else
- KOutMsg( "\b\b\b\b\b%2u.%01u%%", p1, p0 );
-}
-
-static void progess_2a( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- KOutMsg( "| %2u.%02u%%", p1, p0 );
-}
-
-static void progess_2( const uint16_t percent )
-{
- uint16_t p1 = percent / 100;
- uint16_t p0 = percent - ( p1 * 100 );
- if ( ( p1 & 1 )&&( p0 == 0 ) )
- KOutMsg( "\b\b\b\b\b\b\b- %2u.%02u%%", p1, p0 );
- else
- KOutMsg( "\b\b\b\b\b\b%2u.%02u%%", p1, p0 );
-}
-
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent )
-{
- uint8_t digits = fract_digits;
- if ( pb == NULL )
- return RC( rcVDB, rcNoTarg, rcParsing, rcSelf, rcNull );
- if ( digits > 2 )
- digits = 2;
- if ( pb->initialized )
- {
- if ( pb->percent != percent )
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0( percent ); break;
- case 1 : progess_1( percent ); break;
- case 2 : progess_2( percent ); break;
- }
- }
- }
- else
- {
- pb->percent = percent;
- switch( digits )
- {
- case 0 : progess_0a( percent ); break;
- case 1 : progess_1a( percent ); break;
- case 2 : progess_2a( percent ); break;
- }
- pb->initialized = true;
- }
- return 0;
-}
diff --git a/tools/vdb-copy/progressbar.h b/tools/vdb-copy/progressbar.h
deleted file mode 100644
index f8bef61..0000000
--- a/tools/vdb-copy/progressbar.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _h_progressbar_
-#define _h_progressbar_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct progressbar progressbar;
-
-/*--------------------------------------------------------------------------
- * make_progressbar
- *
- * creates a progressbar with zero-values inside
- * does not output anything
- */
-rc_t make_progressbar( progressbar ** pb );
-
-
-/*--------------------------------------------------------------------------
- * destroy_progressbar
- *
- * destroy's the progressbar
- * does not output anything
- */
-rc_t destroy_progressbar( progressbar * pb );
-
-
-/*--------------------------------------------------------------------------
- * update_progressbar
- *
- * sets the progressbar to a specific percentage
- * outputs only if the percentage has changed from the last call
- * the precentage is in 1/10-th of a percent ( 21,6% = 216 )
- * expects the percents in increasing order ( does not jump back )
- * writes a growing bar made from '-'-chars every 2nd percent
- */
-rc_t update_progressbar( progressbar * pb, const uint8_t fract_digits,
- const uint16_t percent );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/tools/vdb-copy/type_matcher.c b/tools/vdb-copy/type_matcher.c
index 89d1b91..743fda0 100644
--- a/tools/vdb-copy/type_matcher.c
+++ b/tools/vdb-copy/type_matcher.c
@@ -588,15 +588,15 @@ static rc_t matcher_make_type_matrix( matcher* self )
}
-static int CC matcher_match_cb( const void ** p1, const void ** p2, void *data )
+static int64_t CC matcher_match_cb( const void ** p1, const void ** p2, void *data )
{
- int res = 0;
+ int64_t res = 0;
const p_mpair pair_1 = (p_mpair)(*p1);
const p_mpair pair_2 = (p_mpair)(*p2);
if ( pair_1 == NULL || pair_2 == NULL )
return res;
/* first we order by compatibility */
- res = ( pair_1->compatible ) - ( pair_2->compatible );
+ res = (int64_t)( pair_1->compatible ) - (int64_t)( pair_2->compatible );
if ( res == 0 )
{
/* second we order by lossy-ness, lowest value first */
@@ -604,14 +604,14 @@ static int CC matcher_match_cb( const void ** p1, const void ** p2, void *data )
if ( res == 0 )
{
/* if the lossy-ness is the same, we order by distance */
- res = ( pair_1->distance ) - ( pair_2->distance );
+ res = (int64_t)( pair_1->distance ) - (int64_t)( pair_2->distance );
if ( res == 0 )
{
/* if the distance is the same, we order by default-value */
- res = ( pair_1->src->dflt ) - ( pair_2->src->dflt );
+ res = (int64_t)( pair_1->src->dflt ) - (int64_t)( pair_2->src->dflt );
if ( res == 0 )
/* if there is not default-value, we use the org. order */
- res = ( pair_1->src->order ) - ( pair_2->src->order );
+ res = (int64_t)( pair_1->src->order ) - (int64_t)( pair_2->src->order );
}
}
}
diff --git a/tools/vdb-copy/vdb-copy.c b/tools/vdb-copy/vdb-copy.c
index 41cd4f8..3c0d17e 100644
--- a/tools/vdb-copy/vdb-copy.c
+++ b/tools/vdb-copy/vdb-copy.c
@@ -31,12 +31,12 @@
#include "helper.h"
#include "coldefs.h"
#include "get_platform.h"
-#include "progressbar.h"
#include "copy_meta.h"
#include "type_matcher.h"
#include "redactval.h"
#include <kapp/main.h>
+#include <klib/progressbar.h>
#include <sysalloc.h>
/*
@@ -379,24 +379,6 @@ static rc_t vdb_copy_read_row_flags( const p_context ctx,
}
-static uint8_t vdb_copy_calc_fract_digits( const num_gen_iter *iter )
-{
- uint8_t res = 0;
- uint64_t count;
- if ( num_gen_iterator_count( iter, &count ) == 0 )
- {
- if ( count > 10000 )
- {
- if ( count > 100000 )
- res = 2;
- else
- res = 1;
- }
- }
- return res;
-}
-
-
static rc_t vdb_copy_row_loop( const p_context ctx,
const VCursor * src_cursor,
VCursor * dst_cursor,
@@ -404,13 +386,13 @@ static rc_t vdb_copy_row_loop( const p_context ctx,
redact_vals * rvals )
{
rc_t rc;
- const num_gen_iter *iter;
- uint64_t count, row_id;
+ const struct num_gen_iter * iter;
+ int64_t row_id;
+ uint64_t count;
uint32_t percent;
- uint8_t fract_digits;
p_col_def filter_col_def = NULL;
redact_buffer rbuf;
- progressbar * progress;
+ struct progressbar * progress = NULL;
if ( columns->filter_idx != -1 )
filter_col_def = col_defs_get( columns, columns->filter_idx );
@@ -418,19 +400,18 @@ static rc_t vdb_copy_row_loop( const p_context ctx,
rc = num_gen_iterator_make( ctx->row_generator, &iter );
if ( rc != 0 ) return rc;
- rc = make_progressbar( &progress );
+ rc = make_progressbar( &progress, 2 );
DISP_RC( rc, "vdb_copy_row_loop:make_progressbar() failed" );
if ( rc != 0 ) return rc;
redact_buf_init( &rbuf );
col_defs_find_redact_vals( columns, rvals );
-
- fract_digits = vdb_copy_calc_fract_digits( iter );
+
count = 0;
- rc = num_gen_iterator_next( iter, &row_id );
- while ( rc == 0 )
+ while ( rc == 0 && num_gen_iterator_next( iter, &row_id, &rc ) )
{
- rc = Quitting(); /* to be able to cancel the loop by signal */
+ if ( rc == 0 )
+ rc = Quitting(); /* to be able to cancel the loop by signal */
if ( rc == 0 )
{
rc = VCursorSetRowId( src_cursor, row_id );
@@ -469,13 +450,12 @@ static rc_t vdb_copy_row_loop( const p_context ctx,
"row_nr=%lu", row_id ) );
}
}
- rc = num_gen_iterator_next( iter, &row_id );
+
if ( ctx->show_progress )
{
- if ( num_gen_iterator_percent( iter, fract_digits, &percent ) == 0 )
- update_progressbar( progress, fract_digits, percent );
+ if ( num_gen_iterator_percent( iter, 2, &percent ) == 0 )
+ update_progressbar( progress, percent );
}
-
}
}
}
@@ -1473,8 +1453,10 @@ rc_t CC KMain ( int argc, char *argv [] )
DISP_RC( rc, "KMain:context_capture_arguments_and_options() failed" );
if ( rc == 0 )
{
- if ( ctx->usage_requested )
+ if ( ctx->usage_requested ) {
MiniUsage( args );
+ rc = RC(rcApp, rcArgv, rcParsing, rcParam, rcInsufficient);
+ }
else
/************************/
rc = vdb_copy_main( ctx );
diff --git a/tools/vdb-copy/vdb-copy.vers b/tools/vdb-copy/vdb-copy.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-copy/vdb-copy.vers
+++ b/tools/vdb-copy/vdb-copy.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-copy/vdb-copy.vers.h b/tools/vdb-copy/vdb-copy.vers.h
index a5a0f15..fda4806 100644
--- a/tools/vdb-copy/vdb-copy.vers.h
+++ b/tools/vdb-copy/vdb-copy.vers.h
@@ -1 +1 @@
-#define VDB_COPY_VERS 0x02040002
+#define VDB_COPY_VERS 0x02050007
diff --git a/tools/vdb-decrypt/shared.c b/tools/vdb-decrypt/shared.c
index b74e0a0..01545dd 100644
--- a/tools/vdb-decrypt/shared.c
+++ b/tools/vdb-decrypt/shared.c
@@ -225,9 +225,10 @@ rc_t CC Usage (const Args * args)
"Encryption key (file password):\n"
" The encryption key or file password is handled by configuration. If not yet\n"
" set, this program will fail.\n\n"
- " You can set the encryption key by running:\n\n"
- " perl configuration-assistant.perl\n"
- "\n");
+ " Please consult configuration page at\n"
+ " http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=toolkit_doc&f=std or\n"
+ " https://github.com/ncbi/sra-tools/wiki/Toolkit-Configuration\n"
+ );
HelpVersion (fullpath, KAppVersion());
@@ -493,7 +494,7 @@ rc_t FileInPlace (KDirectory * cwd, const char * leaf, bool try_rename)
z = string_size (leaf);
/* vdb-decrypt and vdb-encrypt both ignore repository cache files. */
- irc = GetCacheTruncatedSize (infile, &ignored, true);
+ irc = GetCacheTruncatedSize (infile, &ignored);
if (irc == 0)
STSMSG (1, ("skipping cache download file %s", leaf));
else if ((fz == 0) &&
@@ -912,7 +913,7 @@ rc_t Start (KDirectory * cwd, const char * src, const char * dst)
return rc;
}
- stype = KDirectoryPathType (cwd, "%s", spath);
+ stype = KDirectoryPathType (cwd, "%s", spath) & ~ kptAlias;
switch (stype)
{
@@ -1024,7 +1025,7 @@ rc_t Start (KDirectory * cwd, const char * src, const char * dst)
LOGERR (klogErr, rc, "can't resolve destination");
return rc;
}
- dtype = KDirectoryPathType (cwd, "%s", dpath);
+ dtype = KDirectoryPathType (cwd, "%s", dpath) & ~ kptAlias;
switch (dtype)
{
default:
@@ -1239,7 +1240,7 @@ rc_t CommonMain (Args * args)
* letting comp put src in register
* only if it wants
*/
- rc = ArgsParamValue (args, 0, &dst);
+ rc = ArgsParamValue (args, 0, (const void **)&dst);
if (rc)
LOGERR (klogInt, rc, "Failure to fetch "
"source parameter");
@@ -1253,7 +1254,7 @@ rc_t CommonMain (Args * args)
else
{
- rc = ArgsParamValue (args, 1, &dst);
+ rc = ArgsParamValue (args, 1, (const void **)&dst);
if (rc)
LOGERR (klogInt, rc, "Failure to fetch "
"destination parameter");
diff --git a/tools/vdb-decrypt/vdb-decrypt.vers b/tools/vdb-decrypt/vdb-decrypt.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-decrypt/vdb-decrypt.vers
+++ b/tools/vdb-decrypt/vdb-decrypt.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-decrypt/vdb-decrypt.vers.h b/tools/vdb-decrypt/vdb-decrypt.vers.h
index 8cf32dc..b09ac25 100644
--- a/tools/vdb-decrypt/vdb-decrypt.vers.h
+++ b/tools/vdb-decrypt/vdb-decrypt.vers.h
@@ -1 +1 @@
-#define VDB_DECRYPT_VERS 0x02040002
+#define VDB_DECRYPT_VERS 0x02050007
diff --git a/tools/vdb-decrypt/vdb-encrypt.vers b/tools/vdb-decrypt/vdb-encrypt.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-decrypt/vdb-encrypt.vers
+++ b/tools/vdb-decrypt/vdb-encrypt.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-decrypt/vdb-encrypt.vers.h b/tools/vdb-decrypt/vdb-encrypt.vers.h
index c0446ba..419c60c 100644
--- a/tools/vdb-decrypt/vdb-encrypt.vers.h
+++ b/tools/vdb-decrypt/vdb-encrypt.vers.h
@@ -1 +1 @@
-#define VDB_ENCRYPT_VERS 0x02040002
+#define VDB_ENCRYPT_VERS 0x02050007
diff --git a/tools/sra-dbcc/Makefile b/tools/vdb-diff/Makefile
similarity index 87%
rename from tools/sra-dbcc/Makefile
rename to tools/vdb-diff/Makefile
index 4d4646e..34ea086 100644
--- a/tools/sra-dbcc/Makefile
+++ b/tools/vdb-diff/Makefile
@@ -26,21 +26,20 @@
default: std
TOP ?= $(abspath ../..)
-MODULE = tools/sra-dbcc
+MODULE = tools/vdb-diff
include $(TOP)/build/Makefile.env
INT_TOOLS = \
-
EXT_TOOLS = \
- sra-dbcc
+ vdb-diff
ALL_TOOLS = \
$(INT_TOOLS) \
$(EXT_TOOLS)
-#------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
# outer targets
#
all std: vers-includes
@@ -82,21 +81,21 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# sra-dbcc
+# vdb-diff
#
-SRA_DBCC_SRC = \
- sra-dbcc
-SRA_DBCC_OBJ = \
- $(addsuffix .$(OBJX),$(SRA_DBCC_SRC))
+VDB_DIFF_SRC = \
+ namelist_tools \
+ coldefs \
+ vdb-diff
-SRA_DBCC_LIB = \
- -lkapp \
- -lncbi-vdb \
+VDB_DIFF_OBJ = \
+ $(addsuffix .$(OBJX),$(VDB_DIFF_SRC))
-sra-dbcc.vers.h: sra-dbcc.vers
- $(TOP)/build/make-vers-inc.sh SRA_DBCC_VERS $^ > $@
-
-$(BINDIR)/sra-dbcc: $(SRA_DBCC_OBJ)
- $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SRA_DBCC_LIB)
+VDB_DIFF_LIB = \
+ -lkapp \
+ -sncbi-vdb \
+ -lm
+$(BINDIR)/vdb-diff: $(VDB_DIFF_OBJ)
+ $(LD) --exe --vers $(SRCDIR) -o $@ $^ $(VDB_DIFF_LIB)
diff --git a/tools/vdb-diff/coldefs.c b/tools/vdb-diff/coldefs.c
new file mode 100644
index 0000000..d067d8f
--- /dev/null
+++ b/tools/vdb-diff/coldefs.c
@@ -0,0 +1,211 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "coldefs.h"
+#include <klib/text.h>
+#include <klib/out.h>
+
+#include <vdb/table.h>
+
+#include <sysalloc.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+
+/*
+ * helper-function for: col_defs_destroy
+ * - free's everything a node owns
+ * - free's the node
+*/
+static void CC col_defs_destroy_pair( void * node, void * data )
+{
+ col_pair * pair = node;
+ if ( pair != NULL )
+ {
+
+ if ( pair -> name != NULL )
+ {
+ free( pair -> name );
+ pair -> name = NULL;
+ }
+
+ free( pair );
+ }
+}
+
+
+/*
+ * initializes a column-definitions-list
+*/
+rc_t col_defs_init( col_defs ** defs )
+{
+ rc_t rc = 0;
+
+ if ( defs == NULL )
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
+ else
+ {
+ col_defs * tmp = calloc( 1, sizeof( * tmp ) );
+ if ( tmp == NULL )
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ VectorInit( &( tmp ->cols ), 0, 5 );
+ *defs = tmp;
+ }
+ }
+ return rc;
+}
+
+
+/*
+ * destroys the column-definitions-list
+*/
+rc_t col_defs_destroy( col_defs * defs )
+{
+ rc_t rc = 0;
+ if ( defs == NULL )
+ rc = RC( rcVDB, rcNoTarg, rcDestroying, rcSelf, rcNull );
+ else
+ {
+ VectorWhack( &( defs -> cols ), col_defs_destroy_pair, NULL );
+ free( defs );
+ }
+ return rc;
+}
+
+
+/*
+ * helper-function for: col_defs_parse_string / col_defs_extract_from_table
+ * - creates a column-definition by the column-name
+ * - adds the definition to the column-definition-vector
+*/
+static rc_t col_defs_append_col_pair( col_defs * defs, const char * name )
+{
+ rc_t rc = 0;
+
+ if ( name == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
+ else if ( name[ 0 ] == 0 )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcEmpty );
+ else
+ {
+ col_pair * new_pair = calloc( 1, sizeof( * new_pair ) );
+ if ( new_pair == NULL )
+ rc = RC( rcVDB, rcNoTarg, rcParsing, rcMemory, rcExhausted );
+ else
+ {
+ new_pair -> name = string_dup_measure ( name, NULL );
+ rc = VectorAppend( &( defs -> cols ), NULL, new_pair );
+ if ( rc != 0 )
+ col_defs_destroy_pair( new_pair, NULL );
+ }
+ }
+ return rc;
+}
+
+
+/*
+*/
+rc_t col_defs_fill( col_defs * defs, const KNamelist * list )
+{
+ rc_t rc = 0;
+
+ if ( defs == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcSelf, rcNull );
+ else if ( list == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
+ else
+ {
+ uint32_t count;
+ rc = KNamelistCount( list, &count );
+ if ( rc != 0 )
+ KOutMsg( "col_defs_extract_from_table:KNamelistCount() failed %R\n", rc );
+ else
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * name;
+ rc = KNamelistGet( list, idx, &name );
+ if ( rc != 0 )
+ KOutMsg( "col_defs_extract_from_table:KNamelistGet() failed %R\n", rc );
+ else
+ rc = col_defs_append_col_pair( defs, name );
+ }
+ }
+ }
+ return rc;
+}
+
+
+/*
+ * how many columns do we have in here?
+*/
+rc_t col_defs_count( const col_defs * defs, uint32_t * count )
+{
+ rc_t rc = 0;
+ if ( defs == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcSelf, rcNull );
+ else if ( count == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
+ else
+ {
+ *count = VectorLength( &( defs -> cols ) );
+ }
+ return rc;
+}
+
+
+/*
+ * add the pairs in defs to the given cursor
+*/
+rc_t col_defs_add_to_cursor( col_defs * defs, const VCursor * cur, int idx )
+{
+ rc_t rc = 0;
+
+ if ( defs == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcSelf, rcNull );
+ else if ( cur == NULL )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
+ else if ( idx < 0 || idx > 1 )
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcInvalid );
+ else
+ {
+ uint32_t i;
+ uint32_t count = VectorLength( &( defs -> cols ) );
+ for ( i = 0; i < count && rc == 0; ++i )
+ {
+ col_pair * pair = VectorGet( &( defs -> cols ), i );
+ if ( pair != NULL )
+ rc = VCursorAddColumn( cur, &( pair -> pair[ idx ].idx ), "%s", pair -> name );
+ }
+ }
+ return rc;
+}
diff --git a/tools/vdb-dump/vdb-dump-num-gen.h b/tools/vdb-diff/coldefs.h
similarity index 50%
rename from tools/vdb-dump/vdb-dump-num-gen.h
rename to tools/vdb-diff/coldefs.h
index 12fc530..e964568 100644
--- a/tools/vdb-dump/vdb-dump-num-gen.h
+++ b/tools/vdb-diff/coldefs.h
@@ -24,54 +24,79 @@
*
*/
-#ifndef _h_vdb_dump_num_gen_
-#define _h_vdb_dump_num_gen_
+#ifndef _h_vdb_coldefs_
+#define _h_vdb_coldefs_
+
+#include <klib/defs.h>
+#include <klib/rc.h>
+#include <klib/vector.h>
+#include <klib/namelist.h>
+
+#include <vdb/schema.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
#ifdef __cplusplus
extern "C" {
#endif
-#include <klib/vector.h>
+/********************************************************************
+col-def is the definition of a single column: index/type
+********************************************************************/
+typedef struct col_def
+{
+ uint32_t idx; /* index of this column in curosr */
+} col_def;
-/********************************************************************************
- A NUMBER GENERATOR
+/********************************************************************
+col-pair is the definition of pair of columns with same name
+********************************************************************/
+typedef struct col_pair
+{
+ char * name;
+ col_def pair[ 2 ]; /* a pair of sub-col-defs */
+} col_pair;
-input : string, for instance "3,6,8,12,44-49"
-ouptut: sequence of integers, for instance 3,6,8,12,44,45,46,47,48,49
-********************************************************************************/
-typedef struct num_gen_node
+/********************************************************************
+the col-defs are a vector of single column-pairs
+********************************************************************/
+typedef struct col_defs
{
- uint64_t start;
- uint64_t count;
-} num_gen_node;
-typedef num_gen_node* p_num_gen_node;
+ Vector cols;
+} col_defs;
-typedef struct num_gen
-{
- Vector nodes;
- uint32_t node_count;
- uint64_t curr_node;
- uint64_t curr_node_sub_pos;
-} num_gen;
+/*
+ * initializes a column-definitions-list
+*/
+rc_t col_defs_init( col_defs ** defs );
-rc_t vdn_make( num_gen** generator );
-rc_t vdn_destroy( num_gen* generator );
-uint32_t vdn_parse( num_gen* generator, const char* src );
+/*
+ * destroys the column-definitions-list
+*/
+rc_t col_defs_destroy( col_defs * defs );
+
+
+/*
+ * setup the list with pairs made from the given list
+*/
+rc_t col_defs_fill( col_defs * defs, const KNamelist * list );
-bool vdn_set_range( num_gen* generator,
- const int64_t first, const uint64_t count );
-bool vdn_check_range( num_gen* generator,
- const int64_t first, const uint64_t count );
-bool vdn_start( num_gen* generator );
-bool vdn_next( num_gen* generator, uint64_t* value );
-bool vdn_range_defined( num_gen* generator );
+/*
+ * how many columns do we have in here?
+*/
+rc_t col_defs_count( const col_defs * defs, uint32_t * count );
+
+
+/*
+ * add the pairs in defs to the given cursor
+*/
+rc_t col_defs_add_to_cursor( col_defs * defs, const VCursor * cur, int idx );
-uint64_t vdn_max( num_gen* generator );
#ifdef __cplusplus
}
diff --git a/tools/vdb-diff/namelist_tools.c b/tools/vdb-diff/namelist_tools.c
new file mode 100644
index 0000000..1871512
--- /dev/null
+++ b/tools/vdb-diff/namelist_tools.c
@@ -0,0 +1,299 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+#include "namelist_tools.h"
+
+#include <sysalloc.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+rc_t nlt_make_namelist_from_string( const KNamelist **list, const char * src )
+{
+ VNamelist *v_names;
+ rc_t rc = VNamelistMake ( &v_names, 5 );
+ if ( rc == 0 )
+ {
+ char * s = string_dup_measure ( src, NULL );
+ if ( s )
+ {
+ uint32_t str_begin = 0;
+ uint32_t str_end = 0;
+ char c;
+ do
+ {
+ c = s[ str_end ];
+ if ( c == ',' || c == 0 )
+ {
+ if ( str_begin < str_end )
+ {
+ char c_temp = c;
+ s[ str_end ] = 0;
+ rc = VNamelistAppend ( v_names, &(s[str_begin]) );
+ s[ str_end ] = c_temp;
+ }
+ str_begin = str_end + 1;
+ }
+ str_end++;
+ } while ( c != 0 && rc == 0 );
+ free( s );
+ }
+ if ( rc == 0 )
+ rc = VNamelistToConstNamelist ( v_names, list );
+ VNamelistRelease( v_names );
+ }
+ return rc;
+}
+
+
+int nlt_strcmp( const char* s1, const char* s2 )
+{
+ size_t n1 = string_size ( s1 );
+ size_t n2 = string_size ( s2 );
+ return string_cmp
+ ( s1, n1, s2, n2, ( n1 < n2 ) ? ( uint32_t ) n2 : ( uint32_t ) n1 );
+}
+
+
+bool nlt_is_name_in_namelist( const KNamelist *list, const char *name_to_find )
+{
+ uint32_t count, idx;
+ bool res = false;
+ if ( list == NULL || name_to_find == NULL )
+ return res;
+ if ( KNamelistCount( list, &count ) == 0 )
+ {
+ for ( idx = 0; idx < count && res == false; ++idx )
+ {
+ const char *item_name;
+ if ( KNamelistGet( list, idx, &item_name ) == 0 )
+ {
+ if ( nlt_strcmp( item_name, name_to_find ) == 0 )
+ res = true;
+ }
+ }
+ }
+ return res;
+}
+
+/*
+ - list1 and list2 containts strings
+ - if one of the strings in list2 is contained ( partial match, strstr() )
+ in one of the strings in list1 the function returns true...
+*/
+bool nlt_namelist_intersect( const KNamelist *list1, const KNamelist *list2 )
+{
+ uint32_t count1;
+ bool res = false;
+ if ( list1 == NULL || list2 == NULL )
+ return res;
+ if ( KNamelistCount( list1, &count1 ) == 0 )
+ {
+ uint32_t idx1;
+ for ( idx1 = 0; idx1 < count1 && res == false; ++idx1 )
+ {
+ const char *string1;
+ if ( KNamelistGet( list1, idx1, &string1 ) == 0 )
+ {
+ uint32_t count2;
+ if ( KNamelistCount( list2, &count2 ) == 0 )
+ {
+ uint32_t idx2;
+ for ( idx2 = 0; idx2 < count2 && res == false; ++idx2 )
+ {
+ const char *string2;
+ if ( KNamelistGet( list2, idx2, &string2 ) == 0 )
+ {
+ if ( strstr( string1, string2 ) != NULL )
+ res = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ return res;
+}
+
+rc_t nlt_remove_names_from_namelist( const KNamelist *source,
+ const KNamelist **dest, const KNamelist *to_remove )
+{
+ rc_t rc = 0;
+ uint32_t count;
+
+ if ( source == NULL || dest == NULL || to_remove == NULL )
+ return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
+ *dest = NULL;
+ rc = KNamelistCount( source, &count );
+ if ( rc == 0 && count > 0 )
+ {
+ VNamelist *cleaned;
+ rc = VNamelistMake ( &cleaned, count );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char *s;
+ rc = KNamelistGet( source, idx, &s );
+ if ( rc == 0 )
+ {
+ if ( !nlt_is_name_in_namelist( to_remove, s ) )
+ rc = VNamelistAppend ( cleaned, s );
+ }
+ rc = VNamelistToConstNamelist ( cleaned, dest );
+ }
+ }
+ }
+ return rc;
+}
+
+rc_t nlt_remove_strings_from_namelist( const KNamelist *source,
+ const KNamelist **dest, const char *items_to_remove )
+{
+ rc_t rc = 0;
+ const KNamelist *to_remove;
+
+ if ( source == NULL || dest == NULL || items_to_remove == NULL )
+ return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
+ rc = nlt_make_namelist_from_string( &to_remove, items_to_remove );
+ if ( rc == 0 )
+ {
+ rc = nlt_remove_names_from_namelist( source, dest, to_remove );
+ KNamelistRelease( to_remove );
+ }
+ return rc;
+}
+
+
+bool nlt_compare_namelists( const KNamelist *nl1, const KNamelist *nl2, uint32_t * found )
+{
+ bool res = false;
+ if ( nl1 != NULL && nl2 != NULL )
+ {
+ uint32_t count_1;
+ rc_t rc = KNamelistCount( nl1, &count_1 );
+ if ( rc == 0 )
+ {
+ uint32_t count_2;
+ rc = KNamelistCount( nl2, &count_2 );
+ if ( rc == 0 && count_1 == count_2 )
+ {
+ uint32_t idx;
+ uint32_t in_nl2 = 0;
+ for ( idx = 0; idx < count_1 && rc == 0; ++idx )
+ {
+ const char *s;
+ rc = KNamelistGet( nl1, idx, &s );
+ if ( rc == 0 && s != NULL && nlt_is_name_in_namelist( nl2, s ) )
+ in_nl2++;
+ }
+ res = ( rc == 0 && in_nl2 == count_1 );
+ if ( found != NULL ) *found = in_nl2;
+ }
+ }
+ }
+ return res;
+}
+
+
+rc_t nlt_copy_namelist( const KNamelist *src, const KNamelist ** dst )
+{
+ VNamelist *v_names;
+ rc_t rc = VNamelistMake ( &v_names, 5 );
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = KNamelistCount( src, &count );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char *s;
+ rc = KNamelistGet( src, idx, &s );
+ if ( rc == 0 && s != NULL )
+ rc = VNamelistAppend ( v_names, s );
+ }
+ }
+ if ( rc == 0 )
+ rc = VNamelistToConstNamelist ( v_names, dst );
+ VNamelistRelease( v_names );
+ }
+ return rc;
+}
+
+
+rc_t nlt_build_intersect( const KNamelist *nl1, const KNamelist *nl2, const KNamelist ** dst )
+{
+ VNamelist *v_names;
+ rc_t rc = VNamelistMake ( &v_names, 5 );
+ if ( rc == 0 )
+ {
+ /* loop through nl1: if a entry is found in nl2 -> add it to dst */
+ uint32_t count;
+ rc = KNamelistCount( nl1, &count );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char *s;
+ rc = KNamelistGet( nl1, idx, &s );
+ if ( rc == 0 && s != NULL )
+ {
+ if ( nlt_is_name_in_namelist( nl2, s ) )
+ rc = VNamelistAppend ( v_names, s );
+ }
+ }
+ }
+ if ( rc == 0 )
+ rc = VNamelistToConstNamelist ( v_names, dst );
+ VNamelistRelease( v_names );
+
+ }
+ return rc;
+}
+
+bool nlt_namelist_is_sub_set_in_full_set( const KNamelist * sub_set, const KNamelist * full_set )
+{
+ bool res = false;
+ uint32_t count;
+ rc_t rc = KNamelistCount( sub_set, &count );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ uint32_t found = 0;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char *s;
+ rc = KNamelistGet( sub_set, idx, &s );
+ if ( rc == 0 && s != NULL && nlt_is_name_in_namelist( full_set, s ) )
+ found++;
+ }
+ res = ( rc == 0 && count == found );
+ }
+ return res;
+}
diff --git a/tools/cg-dump/sg_lookup.h b/tools/vdb-diff/namelist_tools.h
similarity index 57%
rename from tools/cg-dump/sg_lookup.h
rename to tools/vdb-diff/namelist_tools.h
index 8191c10..6819960 100644
--- a/tools/cg-dump/sg_lookup.h
+++ b/tools/vdb-diff/namelist_tools.h
@@ -23,40 +23,34 @@
* ===========================================================================
*
*/
-#ifndef _h_sg_lookup_
-#define _h_sg_lookup_
+#ifndef _h_namelist_tools_
+#define _h_namelist_tools_
+
+#include <klib/rc.h>
+#include <klib/namelist.h>
#ifdef __cplusplus
extern "C" {
#endif
-#include <klib/rc.h>
-#include <klib/text.h>
-#include <klib/container.h>
-
-#include <vdb/database.h>
-
-struct sg_lookup;
-
-typedef struct sg
-{
- BSTNode node;
-
- const String name;
- const String field_size;
- const String lib;
- const String sample;
-} sg;
+int nlt_strcmp( const char* s1, const char* s2 );
+rc_t nlt_make_namelist_from_string( const KNamelist **list, const char * src );
+bool nlt_is_name_in_namelist( const KNamelist *list, const char *name_to_find );
+bool nlt_namelist_intersect( const KNamelist *list1, const KNamelist *list2 );
+rc_t nlt_remove_names_from_namelist( const KNamelist *source,
+ const KNamelist **dest, const KNamelist *to_remove );
-rc_t make_sg_lookup( struct sg_lookup ** self );
+rc_t nlt_remove_strings_from_namelist( const KNamelist *source,
+ const KNamelist **dest, const char *items_to_remove );
-rc_t parse_sg_lookup( struct sg_lookup * self, const VDatabase * db );
+bool nlt_compare_namelists( const KNamelist *nl1, const KNamelist *nl2, uint32_t * found );
-rc_t perform_sg_lookup( struct sg_lookup * self, const String * name, sg ** entry );
+rc_t nlt_copy_namelist( const KNamelist *src, const KNamelist ** dst );
-rc_t destroy_sg_lookup( struct sg_lookup * self );
+rc_t nlt_build_intersect( const KNamelist *nl1, const KNamelist *nl2, const KNamelist ** dst );
+bool nlt_namelist_is_sub_set_in_full_set( const KNamelist * sub_set, const KNamelist * full_set );
#ifdef __cplusplus
}
diff --git a/tools/vdb-diff/vdb-diff.c b/tools/vdb-diff/vdb-diff.c
new file mode 100644
index 0000000..5efa741
--- /dev/null
+++ b/tools/vdb-diff/vdb-diff.c
@@ -0,0 +1,1017 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "vdb-diff.vers.h"
+
+#include <kapp/main.h>
+
+#include <klib/rc.h>
+#include <klib/out.h>
+#include <klib/log.h>
+#include <klib/num-gen.h>
+#include <klib/progressbar.h>
+
+#include <vdb/manager.h>
+#include <vdb/schema.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+#include <vdb/database.h>
+
+#include <sra/sraschema.h>
+
+#include "coldefs.h"
+#include "namelist_tools.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+
+#define OPTION_ROWS "rows"
+#define ALIAS_ROWS "R"
+
+#define OPTION_COLUMNS "columns"
+#define ALIAS_COLUMNS "C"
+
+#define OPTION_TABLE "table"
+#define ALIAS_TABLE "T"
+
+#define OPTION_PROGRESS "progress"
+#define ALIAS_PROGRESS "p"
+
+#define OPTION_MAXERR "maxerr"
+#define ALIAS_MAXERR "e"
+
+#define OPTION_INTERSECT "intersect"
+#define ALIAS_INTERSECT "i"
+
+#define OPTION_EXCLUDE "exclude"
+#define ALIAS_EXCLUDE "x"
+
+static const char * rows_usage[] = { "set of rows to be comparend (default = all)", NULL };
+static const char * columns_usage[] = { "set of columns to be compared (default = all)", NULL };
+static const char * table_usage[] = { "name of table (in case of database ) to be compared", NULL };
+static const char * progress_usage[] = { "show progress in percent", NULL };
+static const char * maxerr_usage[] = { "max errors im comparing (default = 1)", NULL };
+static const char * intersect_usage[] = { "intersect column-set from both runs", NULL };
+static const char * exclude_usage[] = { "exclude these columns from comapring", NULL };
+
+OptDef MyOptions[] =
+{
+ { OPTION_ROWS, ALIAS_ROWS, NULL, rows_usage, 1, true, false },
+ { OPTION_COLUMNS, ALIAS_COLUMNS, NULL, columns_usage, 1, true, false },
+ { OPTION_TABLE, ALIAS_TABLE, NULL, table_usage, 1, true, false },
+ { OPTION_PROGRESS, ALIAS_PROGRESS, NULL, progress_usage, 1, false, false },
+ { OPTION_MAXERR, ALIAS_MAXERR, NULL, maxerr_usage, 1, true, false },
+ { OPTION_INTERSECT, ALIAS_INTERSECT, NULL, intersect_usage, 1, false, false },
+ { OPTION_EXCLUDE, ALIAS_EXCLUDE, NULL, exclude_usage, 1, true, false }
+};
+
+
+const char UsageDefaultName[] = "vdb-diff";
+
+
+rc_t CC UsageSummary ( const char * progname )
+{
+ return KOutMsg (
+ "\n"
+ "Usage:\n"
+ " %s <src_path> <dst_path> [options]\n"
+ "\n", progname );
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ rc_t rc;
+
+ if ( args == NULL )
+ rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
+ else
+ rc = ArgsProgram ( args, &fullpath, &progname );
+
+ if ( rc != 0 )
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary ( progname );
+
+ KOutMsg ( "Options:\n" );
+ HelpOptionLine ( ALIAS_ROWS, OPTION_ROWS, "row-range", rows_usage );
+ HelpOptionLine ( ALIAS_COLUMNS, OPTION_COLUMNS, "column-set", columns_usage );
+ HelpOptionLine ( ALIAS_TABLE, OPTION_TABLE, "table-name", table_usage );
+ HelpOptionLine ( ALIAS_PROGRESS, OPTION_PROGRESS, NULL, progress_usage );
+ HelpOptionLine ( ALIAS_MAXERR, OPTION_MAXERR, "max value", maxerr_usage );
+ HelpOptionLine ( ALIAS_INTERSECT, OPTION_INTERSECT, NULL, intersect_usage );
+ HelpOptionLine ( ALIAS_EXCLUDE, OPTION_EXCLUDE, "column-set", exclude_usage );
+
+ HelpOptionsStandard ();
+ HelpVersion ( fullpath, KAppVersion() );
+
+ return rc;
+}
+
+
+/* Version EXTERN
+ * return 4-part version code: 0xMMmmrrrr, where
+ * MM = major release
+ * mm = minor release
+ * rrrr = bug-fix release
+ */
+ver_t CC KAppVersion ( void )
+{
+ return VDB_DIFF_VERS;
+}
+
+
+struct diff_ctx
+{
+ const char * src1;
+ const char * src2;
+ const char * columns;
+ const char * excluded;
+ const char * table;
+
+ struct num_gen * rows;
+ uint32_t max_err;
+ bool show_progress;
+ bool intersect;
+};
+
+
+static void init_diff_ctx( struct diff_ctx * dctx )
+{
+ dctx -> src1 = NULL;
+ dctx -> src2 = NULL;
+ dctx -> columns = NULL;
+ dctx -> excluded = NULL;
+ dctx -> table = NULL;
+
+ dctx -> rows = NULL;
+ dctx -> show_progress = false;
+ dctx -> intersect = false;
+}
+
+
+static void release_diff_ctx( struct diff_ctx * dctx )
+{
+ /* do not release the src1,src2,columns,table pointers! they are owned by the Args-obj. */
+ if ( dctx -> rows != 0 )
+ {
+ num_gen_destroy( dctx -> rows );
+ }
+}
+
+
+static const char * get_str_option( const Args *args, const char * option_name )
+{
+ const char * res = NULL;
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, option_name, &count );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsOptionCount() failed" );
+ }
+ else if ( count > 0 )
+ {
+ rc = ArgsOptionValue( args, option_name, 0, (const void **)&res );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsOptionValue( 0 ) failed" );
+ }
+ }
+ return res;
+}
+
+
+static bool get_bool_option( const Args *args, const char * option_name, bool dflt )
+{
+ bool res = dflt;
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( args, option_name, &count );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsOptionCount() failed" );
+ }
+ else if ( count > 0 )
+ {
+ res = true;
+ }
+ return res;
+}
+
+
+static uint32_t get_uint32t_option( const Args *args, const char * option_name, uint32_t dflt )
+{
+ uint32_t res = dflt;
+ const char * s = get_str_option( args, option_name );
+ if ( s != NULL ) res = atoi( s );
+ return res;
+}
+
+
+static rc_t gather_diff_ctx( struct diff_ctx * dctx, Args * args )
+{
+ uint32_t count;
+ rc_t rc = ArgsParamCount( args, &count );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsParamCount() failed" );
+ }
+ else if ( count < 2 )
+ {
+ rc = RC( rcExe, rcNoTarg, rcReading, rcParam, rcIncorrect );
+ LOGERR ( klogInt, rc, "This tool needs 2 arguments: source1 and source2" );
+ }
+ else
+ {
+ rc = ArgsParamValue( args, 0, (const void **)&dctx->src1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsParamValue( 1 ) failed" );
+ }
+ else
+ {
+ rc = ArgsParamValue( args, 1, (const void **)&dctx->src2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsParamValue( 2 ) failed" );
+ }
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ const char * s = get_str_option( args, OPTION_ROWS );
+ if ( s != NULL )
+ {
+ rc = num_gen_make_from_str( &dctx->rows, s );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "num_gen_make_from_str() failed" );
+ }
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ dctx -> columns = get_str_option( args, OPTION_COLUMNS );
+ dctx -> excluded = get_str_option( args, OPTION_EXCLUDE );
+ dctx -> show_progress = get_bool_option( args, OPTION_PROGRESS, false );
+ dctx -> intersect = get_bool_option( args, OPTION_INTERSECT, false );
+ dctx -> max_err = get_uint32t_option( args, OPTION_MAXERR, 1 );
+ }
+
+ return rc;
+}
+
+
+static rc_t report_diff_ctx( struct diff_ctx * dctx )
+{
+ rc_t rc = KOutMsg( "src[ 1 ] : %s\n", dctx -> src1 );
+ if ( rc == 0 )
+ rc = KOutMsg( "src[ 2 ] : %s\n", dctx -> src2 );
+
+ if ( rc == 0 )
+ {
+ if ( dctx -> rows != NULL )
+ {
+ char buffer[ 4096 ];
+ rc = num_gen_as_string( dctx -> rows, buffer, sizeof buffer, NULL, false );
+ if ( rc == 0 )
+ rc = KOutMsg( "- rows : %s\n", buffer );
+ }
+ else
+ rc = KOutMsg( "- rows : all\n" );
+ }
+
+ if ( rc == 0 && dctx -> columns != NULL )
+ rc = KOutMsg( "- columns : %s\n", dctx -> columns );
+ if ( rc == 0 && dctx -> excluded != NULL )
+ rc = KOutMsg( "- exclude : %s\n", dctx -> excluded );
+ if ( rc == 0 && dctx -> table != NULL )
+ rc = KOutMsg( "- table : %s\n", dctx -> table );
+ if ( rc == 0 )
+ rc = KOutMsg( "- progress : %s\n", dctx -> show_progress ? "show" : "hide" );
+ if ( rc == 0 )
+ rc = KOutMsg( "- intersect: %s\n", dctx -> intersect ? "yes" : "no" );
+ if ( rc == 0 )
+ rc = KOutMsg( "- max err : %u\n", dctx -> max_err );
+
+ if ( rc == 0 )
+ rc = KOutMsg( "\n" );
+ return rc;
+}
+
+
+static rc_t diff_columns_iter( col_defs * defs, const VCursor * cur_1, const VCursor * cur_2,
+ struct diff_ctx * dctx, const struct num_gen_iter * iter )
+{
+ uint32_t column_count;
+ rc_t rc = col_defs_count( defs, &column_count );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "col_defs_count() failed" );
+ }
+ else
+ {
+ struct progressbar * progress = NULL;
+ int64_t row_id;
+ uint64_t rows_checked = 0;
+ uint64_t rows_different = 0;
+
+ if ( dctx -> show_progress )
+ make_progressbar( &progress, 2 );
+
+ while ( rc == 0 && num_gen_iterator_next( iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 ) rc = Quitting(); /* to be able to cancel the loop by signal */
+ if ( rc == 0 )
+ {
+ bool row_equal = true;
+
+ uint32_t col_id;
+ for ( col_id = 0; col_id < column_count && rc == 0; ++col_id )
+ {
+ col_pair * pair = VectorGet( &( defs -> cols ), col_id );
+ if ( pair != NULL )
+ {
+ uint32_t elem_bits_1, boff_1, row_len_1;
+ const void * base_1;
+ rc = VCursorCellDataDirect ( cur_1, row_id, pair->pair[ 0 ].idx,
+ &elem_bits_1, &base_1, &boff_1, &row_len_1 );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogInt, ( klogInt, rc,
+ "VCursorCellDataDirect( #1 [$(col)].$(row) ) failed",
+ "col=%s,row=%ld", pair->name, row_id ) );
+ }
+ else
+ {
+ uint32_t elem_bits_2, boff_2, row_len_2;
+ const void * base_2;
+ rc = VCursorCellDataDirect ( cur_2, row_id, pair->pair[ 1 ].idx,
+ &elem_bits_2, &base_2, &boff_2, &row_len_2 );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogInt, ( klogInt, rc,
+ "VCursorCellDataDirect( #2 [$(col)].$(row) ) failed",
+ "col=%s,row=%ld", pair->name, row_id ) );
+ }
+ else
+ {
+ bool bin_diff = true;
+
+ if ( elem_bits_1 != elem_bits_2 )
+ {
+ bin_diff = row_equal = false;
+ rc = KOutMsg( "%s[ %ld ].elem_bits %u != %u\n", pair->name, row_id, elem_bits_1, elem_bits_2 );
+ }
+
+ if ( row_len_1 != row_len_2 )
+ {
+ bin_diff = row_equal = false;
+ if ( rc == 0 )
+ {
+ rc = KOutMsg( "%s[ %ld ].row_len %u != %u\n", pair->name, row_id, row_len_1, row_len_2 );
+ }
+ }
+
+ if ( boff_1 != 0 || boff_2 != 0 )
+ {
+ bin_diff = row_equal = false;
+ if ( rc == 0 )
+ {
+ rc = KOutMsg( "%s[ %ld ].bit_offset: %u, %u\n", pair->name, row_id, boff_1, boff_2 );
+ }
+ }
+
+ if ( bin_diff )
+ {
+ size_t num_bits = ( row_len_1 * elem_bits_1 );
+ if ( num_bits & 0x07 )
+ {
+ if ( rc == 0 )
+ {
+ rc = KOutMsg( "%s[ %ld ].bits_total %% 8 = %u\n", pair->name, row_id, ( num_bits % 8 ) );
+ }
+ }
+ else
+ {
+ size_t num_bytes = ( num_bits >> 3 );
+ int cmp = memcmp ( base_1, base_2, num_bytes );
+ if ( cmp != 0 )
+ {
+ if ( rc == 0 )
+ {
+ rc = KOutMsg( "%s[ %ld ] differ\n", pair->name, row_id );
+ }
+ row_equal = false;
+ }
+ }
+ }
+ }
+ }
+ }
+ } /* for ( col_id ... ) */
+
+ if ( !row_equal )
+ {
+ if ( rc == 0 ) rc = KOutMsg( "\n" );
+ rows_different ++;
+ if ( rows_different >= dctx -> max_err )
+ rc = RC( rcExe, rcNoTarg, rcComparing, rcRow, rcInconsistent );
+ }
+ rows_checked ++;
+
+ if ( progress != NULL )
+ {
+ uint32_t progress_value;
+ if ( num_gen_iterator_percent( iter, 2, &progress_value ) == 0 )
+ update_progressbar( progress, progress_value );
+ }
+
+ } /* if (!Quitting) */
+ } /* while ( num_gen_iterator_next() ) */
+
+ if ( rc == 0 )
+ rc = KOutMsg( "\n%,lu rows checked ( %d columns each ), %,lu rows differ\n",
+ rows_checked, column_count, rows_different );
+
+ if ( rows_different > 0 )
+ rc = RC( rcExe, rcNoTarg, rcComparing, rcRow, rcInconsistent );
+
+ if ( progress != NULL )
+ destroy_progressbar( progress );
+
+ } /* if ( col_defs_count == 0 )*/
+
+ return rc;
+}
+
+
+static rc_t diff_columns_cursor( col_defs * defs, const VCursor * cur_1, const VCursor * cur_2, struct diff_ctx * dctx )
+{
+ int64_t first_1;
+ uint64_t count_1;
+ rc_t rc = VCursorIdRange( cur_1, 0, &first_1, &count_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VCursorIdRange( acc #1 ) failed" );
+ }
+ else
+ {
+ int64_t first_2;
+ uint64_t count_2;
+ rc = VCursorIdRange( cur_2, 0, &first_2, &count_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VCursorIdRange( acc #2 ) failed" );
+ }
+ else
+ {
+ struct num_gen * rows_to_diff = NULL;
+ if ( dctx -> rows != NULL )
+ {
+ num_gen_copy( dctx -> rows, &rows_to_diff );
+ }
+
+ if ( rows_to_diff == NULL )
+ {
+ /* no row-range given create the number generator from the discovered range, if it is the same */
+ if ( first_1 != first_2 || count_1 != count_2 )
+ {
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcInvalid );
+ PLOGERR( klogInt, ( klogInt, rc, "row-ranges differ: $(first1).$(count1) != $(first2).$(count2)",
+ "first1=%ld,count1=%lu,first2=%ld,count2=%lu",
+ first_1, count_1, first_2, count_2 ) );
+ }
+ else
+ {
+ rc = num_gen_make_from_range( &rows_to_diff, first_1, count_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "num_gen_make_from_range() failed" );
+ }
+ }
+ }
+ else
+ {
+ /* row-range given, clip the rows be the 2 ranges ( even if they are not the same ) */
+ rc = num_gen_trim( rows_to_diff, first_1, count_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "num_gen_trim( acc #1 ) failed" );
+ }
+ else if ( first_1 != first_2 || count_1 != count_2 )
+ {
+ rc = num_gen_trim( rows_to_diff, first_2, count_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "num_gen_trim( acc #2 ) failed" );
+ }
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ const struct num_gen_iter * iter;
+ rc = num_gen_iterator_make( rows_to_diff, &iter );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "num_gen_iterator_make() failed" );
+ }
+ else
+ {
+ /* *************************************************************** */
+ rc = diff_columns_iter( defs, cur_1, cur_2, dctx, iter );
+ /* *************************************************************** */
+ num_gen_iterator_destroy( iter );
+ }
+ }
+
+ if ( rows_to_diff != NULL )
+ num_gen_destroy( rows_to_diff );
+ }
+ }
+ return rc;
+}
+
+
+static rc_t diff_columns( col_defs * defs, const VTable * tab_1, const VTable * tab_2, struct diff_ctx * dctx )
+{
+ const VCursor * cur_1;
+ rc_t rc = VTableCreateCursorRead( tab_1, &cur_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VTableCreateCursorRead( acc #1 ) failed" );
+ }
+ else
+ {
+ const VCursor * cur_2;
+ rc = VTableCreateCursorRead( tab_2, &cur_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VTableCreateCursorRead( acc #2 ) failed" );
+ }
+ else
+ {
+ rc = col_defs_add_to_cursor( defs, cur_1, 0 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "failed to add all requested columns to cursor of 1st accession" );
+ }
+ else
+ {
+ rc = col_defs_add_to_cursor( defs, cur_2, 1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "failed to add all requested columns to cursor of 2nd accession" );
+ }
+ else
+ {
+ rc = VCursorOpen( cur_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VCursorOpen( acc #1 ) failed" );
+ }
+ else
+ {
+ rc = VCursorOpen( cur_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VCursorOpen( acc #2 ) failed" );
+ }
+ else
+ {
+ /* ************************************************** */
+ rc = diff_columns_cursor( defs, cur_1, cur_2, dctx );
+ /* ************************************************** */
+ }
+ }
+ }
+ }
+ VCursorRelease( cur_2 );
+ }
+ VCursorRelease( cur_1 );
+ }
+ return 0;
+}
+
+
+static rc_t compare_2_namelists( const KNamelist * cols_1, const KNamelist * cols_2, const KNamelist ** cols_to_diff, bool intersect )
+{
+ uint32_t count_1;
+ rc_t rc = KNamelistCount( cols_1, &count_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "KNamelistCount() failed" );
+ }
+ else
+ {
+ uint32_t count_2;
+ rc = KNamelistCount( cols_2, &count_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "KNamelistCount() failed" );
+ }
+ else if ( count_1 != count_2 )
+ {
+ if ( intersect )
+ {
+ rc = nlt_build_intersect( cols_1, cols_2, cols_to_diff );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "nlt_build_intersect() failed" );
+ }
+ }
+ else
+ {
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcInvalid );
+ PLOGERR( klogInt, ( klogInt, rc,
+ "the accessions have not the same number of columns! ( $(count1) != $(count2) )\n",
+ "count1=%u,count2=%u", count_1, count_2 ) );
+ *cols_to_diff = NULL;
+ }
+ }
+ else
+ {
+ uint32_t found_in_both;
+ bool equal = nlt_compare_namelists( cols_1, cols_2, &found_in_both );
+ if ( equal )
+ {
+ rc = nlt_copy_namelist( cols_1, cols_to_diff );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "nlt_copy_namelist() failed" );
+ }
+ }
+ else
+ {
+ if ( intersect )
+ {
+ rc = nlt_build_intersect( cols_1, cols_2, cols_to_diff );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "nlt_build_intersect() failed" );
+ }
+ }
+ else
+ {
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcInvalid );
+ KOutMsg( "the 2 accessions have not the same set of columns! ( %u found in both )\n", found_in_both );
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t extract_columns_from_2_namelists( const KNamelist * cols_1, const KNamelist * cols_2, const char * sub_set,
+ const KNamelist ** cols_to_diff )
+{
+ rc_t rc = nlt_make_namelist_from_string( cols_to_diff, sub_set );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "cannot parse set of requested columns" );
+ }
+ else
+ {
+ if ( nlt_namelist_is_sub_set_in_full_set( *cols_to_diff, cols_1 ) )
+ {
+ if ( nlt_namelist_is_sub_set_in_full_set( *cols_to_diff, cols_2 ) )
+ {
+ rc = 0;
+ }
+ else
+ {
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcInvalid );
+ LOGERR ( klogInt, rc, "accession #2 is missing some of the requested columns" );
+ }
+ }
+ else
+ {
+ rc = RC( rcExe, rcNoTarg, rcResolving, rcParam, rcInvalid );
+ LOGERR ( klogInt, rc, "accession #1 is missing some of the requested columns" );
+ }
+ }
+ return rc;
+}
+
+
+static rc_t perform_table_diff( const VTable * tab_1, const VTable * tab_2, struct diff_ctx * dctx )
+{
+ col_defs * defs;
+ rc_t rc = col_defs_init( &defs );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "col_defs_init() failed" );
+ }
+ else
+ {
+ KNamelist * cols_1;
+ rc_t rc = VTableListReadableColumns( tab_1, &cols_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VTableListReadableColumns( acc #1 ) failed" );
+ }
+ else
+ {
+ KNamelist * cols_2;
+ rc = VTableListReadableColumns( tab_2, &cols_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VTableListReadableColumns( acc #2 ) failed" );
+ }
+ else
+ {
+ const KNamelist * cols_to_diff;
+
+ if ( dctx -> columns == NULL )
+ rc = compare_2_namelists( cols_1, cols_2, &cols_to_diff, dctx -> intersect );
+ else
+ rc = extract_columns_from_2_namelists( cols_1, cols_2, dctx -> columns, &cols_to_diff );
+
+ if ( rc == 0 && dctx -> excluded != NULL )
+ {
+ const KNamelist * temp;
+ rc = nlt_remove_strings_from_namelist( cols_to_diff, &temp, dctx -> excluded );
+ KNamelistRelease( cols_to_diff );
+ cols_to_diff = temp;
+ }
+
+ if ( rc == 0 )
+ {
+ rc = col_defs_fill( defs, cols_to_diff );
+ if ( rc == 0 )
+ {
+ /* ******************************************* */
+ rc = diff_columns( defs, tab_1, tab_2, dctx );
+ /* ******************************************* */
+ }
+ KNamelistRelease( cols_to_diff );
+ }
+
+ KNamelistRelease( cols_2 );
+ }
+ KNamelistRelease( cols_1 );
+ }
+ col_defs_destroy( defs );
+ }
+ return 0;
+}
+
+
+static rc_t perform_database_diff_on_this_table( const VDatabase * db_1, const VDatabase * db_2,
+ struct diff_ctx * dctx, const char * table_name )
+{
+ /* we want to compare only the table wich name was given at the commandline */
+ const VTable * tab_1;
+ rc_t rc = VDatabaseOpenTableRead ( db_1, &tab_1, "%s", table_name );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogInt, ( klogInt, rc,
+ "VDatabaseOpenTableRead( [$(acc)].$(tab) ) failed",
+ "acc=%s,tab=%s", dctx -> src1, table_name ) );
+ }
+ else
+ {
+ const VTable * tab_2;
+ rc = VDatabaseOpenTableRead ( db_2, &tab_2, "%s", table_name );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogInt, ( klogInt, rc,
+ "VDatabaseOpenTableRead( [$(acc)].$(tab) ) failed",
+ "acc=%s,tab=%s", dctx -> src2, table_name ) );
+ }
+ else
+ {
+ /* ******************************************* */
+ rc = perform_table_diff( tab_1, tab_2, dctx );
+ /* ******************************************* */
+ VTableRelease( tab_2 );
+ }
+ VTableRelease( tab_1 );
+ }
+ return rc;
+}
+
+
+static rc_t perform_database_diff( const VDatabase * db_1, const VDatabase * db_2, struct diff_ctx * dctx )
+{
+ rc_t rc = 0;
+ if ( dctx -> table != NULL )
+ {
+ /* we want to compare only the table wich name was given at the commandline */
+ /* ************************************************************************** */
+ rc = perform_database_diff_on_this_table( db_1, db_2, dctx, dctx -> table );
+ /* ************************************************************************** */
+ }
+ else
+ {
+ /* we want to compare all tables of the 2 accession, if they have the same set of tables */
+ KNamelist * table_set_1;
+ rc = VDatabaseListTbl ( db_1, &table_set_1 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VDatabaseListTbl( acc #1 ) failed" );
+ }
+ else
+ {
+ KNamelist * table_set_2;
+ rc = VDatabaseListTbl ( db_2, &table_set_2 );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VDatabaseListTbl( acc #2 ) failed" );
+ }
+ else
+ {
+ if ( nlt_compare_namelists( table_set_1, table_set_2, NULL ) )
+ {
+ /* the set of tables match, walk the first set, and perform a diff on each table */
+ uint32_t count;
+ rc = KNamelistCount( table_set_1, &count );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "KNamelistCount() failed" );
+ }
+ else
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * table_name;
+ rc = KNamelistGet( table_set_1, idx, &table_name );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "KNamelistGet() failed" );
+ }
+ else
+ {
+ rc = KOutMsg( "\ncomparing table: %s\n", table_name );
+ if ( rc == 0 )
+ {
+ /* ********************************************************************* */
+ rc = perform_database_diff_on_this_table( db_1, db_2, dctx, table_name );
+ /* ********************************************************************* */
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ LOGERR ( klogInt, rc, "the 2 databases have not the same set of tables" );
+ }
+ KNamelistRelease( table_set_2 );
+ }
+ KNamelistRelease( table_set_1 );
+ }
+ }
+ return rc;
+}
+
+
+/***************************************************************************
+ perform the actual diffing
+***************************************************************************/
+static rc_t perform_diff( struct diff_ctx * dctx )
+{
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "KDirectoryNativeDir() failed" );
+ }
+ else
+ {
+ const VDBManager * vdb_mgr;
+ rc = VDBManagerMakeRead ( &vdb_mgr, dir );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VDBManagerMakeRead() failed" );
+ }
+ else
+ {
+ VSchema * vdb_schema = NULL;
+ rc = VDBManagerMakeSRASchema( vdb_mgr, &vdb_schema );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "VDBManagerMakeSRASchema() failed" );
+ }
+ else
+ {
+ const VDatabase * db_1;
+ /* try to open it as a database */
+ rc = VDBManagerOpenDBRead ( vdb_mgr, &db_1, vdb_schema, "%s", dctx -> src1 );
+ if ( rc == 0 )
+ {
+ const VDatabase * db_2;
+ rc = VDBManagerOpenDBRead ( vdb_mgr, &db_2, vdb_schema, "%s", dctx -> src2 );
+ if ( rc == 0 )
+ {
+ /* ******************************************** */
+ rc = perform_database_diff( db_1, db_2, dctx );
+ /* ******************************************** */
+ VDatabaseRelease( db_2 );
+ }
+ else
+ {
+ LOGERR ( klogInt, rc, "accession #1 opened as database, but accession #2 cannot be opened as database" );
+ }
+ VDatabaseRelease( db_1 );
+ }
+ else
+ {
+ const VTable * tab_1;
+ rc = VDBManagerOpenTableRead( vdb_mgr, &tab_1, vdb_schema, "%s", dctx -> src1 );
+ if ( rc == 0 )
+ {
+ const VTable * tab_2;
+ rc = VDBManagerOpenTableRead( vdb_mgr, &tab_2, vdb_schema, "%s", dctx -> src2 );
+ if ( rc == 0 )
+ {
+ /* ******************************************** */
+ rc = perform_table_diff( tab_1, tab_2, dctx );
+ /* ******************************************** */
+ VTableRelease( tab_2 );
+ }
+ else
+ {
+ LOGERR ( klogInt, rc, "accession #1 opened as table, but accession #2 cannot be opened as table" );
+ }
+ VTableRelease( tab_1 );
+ }
+ else
+ {
+ LOGERR ( klogInt, rc, "accession #1 cannot be opened as table or database" );
+ }
+ }
+ VSchemaRelease( vdb_schema );
+ }
+ VDBManagerRelease( vdb_mgr );
+ }
+ KDirectoryRelease( dir );
+ }
+ return rc;
+}
+
+
+/***************************************************************************
+ Main:
+***************************************************************************/
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ Args * args;
+ rc_t rc = ArgsMakeAndHandle ( &args, argc, argv, 1,
+ MyOptions, sizeof MyOptions / sizeof ( OptDef ) );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogInt, rc, "ArgsMakeAndHandle failed()" );
+ }
+ else
+ {
+ struct diff_ctx dctx;
+
+ KLogHandlerSetStdErr();
+ init_diff_ctx( &dctx );
+ rc = gather_diff_ctx( &dctx, args );
+ if ( rc == 0 )
+ {
+ rc = report_diff_ctx( &dctx );
+ if ( rc == 0 )
+ {
+ /* ***************************** */
+ rc = perform_diff( &dctx );
+ /* ***************************** */
+ }
+ }
+ release_diff_ctx( &dctx );
+
+ ArgsWhack ( args );
+ }
+ return rc;
+}
diff --git a/tools/vdb-diff/vdb-diff.vers b/tools/vdb-diff/vdb-diff.vers
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/tools/vdb-diff/vdb-diff.vers
@@ -0,0 +1 @@
+1.0.0
diff --git a/tools/vdb-diff/vdb-diff.vers.h b/tools/vdb-diff/vdb-diff.vers.h
new file mode 100644
index 0000000..bb0fb09
--- /dev/null
+++ b/tools/vdb-diff/vdb-diff.vers.h
@@ -0,0 +1 @@
+#define VDB_DIFF_VERS 0x01000000
diff --git a/tools/vdb-dump/Makefile b/tools/vdb-dump/Makefile
index 0922e99..d789ebf 100644
--- a/tools/vdb-dump/Makefile
+++ b/tools/vdb-dump/Makefile
@@ -85,7 +85,6 @@ clean: stdclean
# vdb dump tool
#
VDB_DUMP_SRC = \
- vdb-dump-num-gen \
vdb-dump-context \
vdb-dump-coldefs \
vdb-dump-tools \
diff --git a/tools/vdb-dump/help.txt b/tools/vdb-dump/help.txt
index f40e468..d3d6f7e 100644
--- a/tools/vdb-dump/help.txt
+++ b/tools/vdb-dump/help.txt
@@ -2,70 +2,70 @@ vdb-dump extended help
(1) dumping a vdb-table:
-the only mandatory option to vdb-dump is the name of the object to dump:
+ the only mandatory option to vdb-dump is the name of the object to dump:
-vdb-dump object
+ vdb-dump OBJECT
-the object can be:
+ the OBJECT can be:
-a) absolute or relative path to a vdb-table (a directory)
+ a) absolute or relative path to a vdb-table (a directory)
-on linux:
-vdb-dump /panfs/traces/sra0/SRR/000000/SRR000001
-or
-vdb-dump `srapath SRR000001`
-(only at NCBI, same infrastructure as for a accession needed )
+ on linux: vdb-dump /panfs/traces/sra0/SRR/000000/SRR000001
-on windows
-vdb-dump \\panfs\traces\sra0\SRR\000000\SRR000001
-or
-vdb-dump Y:\sra0\SRR\000000\SRR000001
-(if "\\panfs\traces" is mapped to the driveletter Y on your windows-pc )
-b) absolute or relative path to a file containing a vdb-table
-on linux/windows:
-vdb-dump SRR044989.lite.sra
+ on windows: vdb-dump \\panfs\traces\sra0\SRR\000000\SRR000001
+ vdb-dump Y:\sra0\SRR\000000\SRR000001
+ ( if "\\panfs\traces" is mapped to the driveletter Y on your windows-pc )
-c) an accession ( only at NCBI )
-on linux/windows:
-vdb-dump SRR000001
-(you need: for linux libsra-path.lib / for windows libsra-path.dll in your search-path,
-a subdir "ncbi" in the same directory where the lib/dll is located,
-in this "ncbi"-subdir you need a config-file "config.kfg"
-in this config-file you need servers and volumes to be defined )
+ b) absolute or relative path to a file containing a vdb-table
+
+ on linux/windows: vdb-dump SRR044989.sra
+
+ c) an accession
+
+ on linux/windows: vdb-dump SRR000001
+
+ outside NCBI you need internet access to reach accessions stored at NCBI and you need
+ remote access enabled in your configuration
+
+ If you specify only the object, vdb-dump will dump all columns for all rows to the standard-output.
-If you specify only the object, vdb-dump will dump all columns for all rows to the standard-output.
The --table / -T option:
-This is for future extensions. Vdb-dump is designed to operate on a vdb-database. A vdb-database can
-contain more then one table. Right now it contains only one table. If you do not specify the table-name,
-vdb-dump will first try to interpret the given object as a vdb-database (and try to dump the first table
-it finds in this database). If this try (silently) fails, because the given object is not a database,
-it is a table instead, vdb-dump will try to interpret the given object as a table.
-That is what happens right now when you use vdb-dump.
+========================
+vdb-dump is designed to operate on a vdb-database. A vdb-database can contain more then one table.
+If you do not specify the table-name, vdb-dump will first try to interpret the given object as a vdb-database
+( and try to dump the table "SEQUENCE", if that table does not exist: the first table it finds in this database ).
+If this try (silently) fails, because the given object is not a database, vdb-dump will try to interpret
+the given object as a table. If the object is not a vdb-database or vdb-table, the tool will fail.
+
The --rows / -R option:
+=======================
With this option you can restrict which rows will be dumped.
-vdb-dump file.sra -R 5 ... will dump only row number 5
-vdb-dump file.sra -R 5-20 ... will dump rows number 5 to number 20 (15 rows)
+vdb-dump file.sra -R 5 ... will dump only row number 5
+vdb-dump file.sra -R 5-20 ... will dump rows number 5 to number 20 (15 rows)
The ranges can be mixed:
vdb-dump file.sra -R 5,7-20,200-201,300,305 ... will dump these rows/ranges
+If you omit the range, vdb-dump will output all rows.
+
The --columns -C option:
+========================
With this option you can restrict which columns per row will be dumped.
vdb-dump file.sra -C NAME,READ ... will dump only the columns NAME and READ per row
+
the --exclude -x option:
+========================
If you want to dump all columns, except some specific ones.
vdb-dump file.sra -x READ,RD_FILTER ... will dump all columns but the READ-column
and the RD_FILTER-column.
-The --schema -S option:
-With this option you can specify one or more additional schema's to be used for dumping
-a table. For instance to reinterpret the content of columns in a new way.
The --row_id_on -I option:
-Vdb-dump does not output the row-id per default, it has to be switched on with this option:
+==========================
+vdb-dump does not output the row-id per default, it has to be switched on with this option:
vdb-dump SRR000001 -R1 -CNAME,SPOT_LEN
NAME: EM7LVYS01C1LWG
@@ -76,8 +76,10 @@ ROW-ID = 1
NAME: EM7LVYS01C1LWG
SPOT_LEN: 255
+
The --line_feed -l option:
-Vdb-dump separates the rows by one empty line (line-feed) per default:
+==========================
+vdb-dump separates the rows by one empty line (line-feed) per default:
vdb-dump SRR000001 -R1-3 -CNAME,SPOT_LEN
NAME: EM7LVYS01C1LWG
@@ -105,7 +107,8 @@ SPOT_LEN: 307
The --colname_off -N option:
-Vdb-dump prints the name of every column in front of the it's data:
+============================
+vdb-dump prints the name of every column in front of the it's data:
vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN
NAME: EM7LVYS01C1LWG
@@ -123,7 +126,9 @@ EM7LVYS01C1LWG
EM7LVYS01B2EMP
248
+
The --in_hex -X option:
+=======================
With this option all numeric outputs are printed as hexadecimal numbers:
$vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -X
@@ -133,13 +138,17 @@ SPOT_LEN: 0xFF
NAME: EM7LVYS01B2EMP
SPOT_LEN: 0xF8
+
The --dna_baese -D option:
+==========================
With this option you can force columns into printed as DNA-base "ACGT",
but only if the column has a datatype with more than one dimension.
If a column has a datatype with a dimension of 2, each dimension 1 bit,
it is automatically printed as DNA-base.
+
The --max_length -M option:
+===========================
With this options you can truncate the output of columns longer than this limit.
vdb-dump SRR000001 -R1-2 -CREAD
@@ -152,7 +161,9 @@ READ: TCAGGGGGGAGCTTAAATTTGAAACTAGAA ...
READ: TCAGGGGGGGGTTACACGTGCAGATTTGTT ...
+
The --indent_with -i option:
+============================
With this option you can limit the length of the output-line and force a left-edge
indenting.
@@ -167,18 +178,19 @@ READ: TCAGGGGGGGGTTACACGTGCAGATTTGTTACACGGGTGTACTGTGAGGTTTGGGGTACGAATGATCCCGTTAC
GGGCTCAGGTCAGCATTAGGGTCAGGTTCTTAGGAAAAGAAAGAGCAAAAACAATGAAACACAATACAAAGTAA
AGAACACTGAGCGGGCTGGCAAGGCN
-The --filter -F option:
-Not implemented yet.
The --format -f option:
+=======================
This selects other than the default-output formating:
csv = comma-separated on one line
+---------------------------------
vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -fcsv
EM7LVYS01C1LWG,255
EM7LVYS01B2EMP,248
xml = xml-section
+-----------------
vdb-dump $vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -fxml
<row_1>
<NAME>
@@ -199,6 +211,7 @@ EM7LVYS01B2EMP
</row_2>
json = json format
+------------------
vdb-dump $vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -fjson
{
"row_id": 1,
@@ -212,7 +225,47 @@ vdb-dump $vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -fjson
"SPOT_LEN":248
},
+piped = format friendly to beeing piped into other processes
+------------------------------------------------------------
+vdb-dump $vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -fpiped
+1, NAME: "EM7LVYS02FOYNU"
+1, SPOT_LEN: 284
+
+2, NAME: "EM7LVYS02GCAPL"
+2, SPOT_LEN: 262
+
+sra-dump = simulates the output of a deprecated tool
+------------------------------------------------------------
+vdb-dump $vdb-dump SRR000001 -R1-2 -CNAME,SPOT_LEN -fsra-dump
+1, NAME: EM7LVYS02FOYNU
+1, SPOT_LEN: 284
+
+2, NAME: EM7LVYS02GCAPL
+2, SPOT_LEN: 262
+
+fastq = produces fastq-output
+( the table needs to have a READ- and a QUALITY column, no splitting supported )
+-------------------------------------------------------
+vdb-dump $vdb-dump SRR000001 -R1 -ffastq
+ at SRR000001.1 EM7LVYS02FOYNU length=284
+TCAGATTCTCCTAGCCTACATCCGTACGAGTTAGCGTGGGATTACGAGGTGCACACCATTTCATTCCGTACGGGTAAATTTTTGTATTTTTAGCAGACGGCAGGGTTTCACCATGGTTGACCAACGTACTAATCTTGAACTCCTGACCTCAAGTGATTTGCCTGCCTTCAGCCTCCCAAAGTGACTGGGTATTACAGATGTGAGCGAGTTTGTGCCCAAGCCTTATAAGTAAATTTATAAATTTACATAATTTAAATGACTTATGCTTAGCGAAATAGGGTAAG
++SRR000001.1 EM7LVYS02FOYNU length=284
+=<8<85)9=9/3-8?68<7=8<3657747==49==+;FB2;A;5:'*>69<:74)9.;C?+;<B<B;(<';FA/;C>*GC8/%9<=GC8.#=2:5:16D==<EA2EA.;5=44<;2C=5;@73&<<2;5;6+9<?776+:24'26:7,<9A;=:;0C>*6?7<<C=D=<52?:9CA2CA23<2<;3CA12:A<9414<7<<6;99<2/=9#<;9B at 27.;=6>:77>:1<A>+CA138?<)C at 2166:A<B?->:%<<9<;33<;6?9;<;4=:%<$CA1+1%1
+
+fasta = produces fasta-output
+( the table needs to have a READ column )
+-------------------------------------------------------
+vdb-dump SRR000001 -R1 -f fasta
+>SRR000001.1 EM7LVYS02FOYNU length=284
+TCAGATTCTCCTAGCCTACATCCGTACGAGTTAGCGTGGGATTACGAGGTGCACACCATTTCATTCCGTA
+CGGGTAAATTTTTGTATTTTTAGCAGACGGCAGGGTTTCACCATGGTTGACCAACGTACTAATCTTGAAC
+TCCTGACCTCAAGTGATTTGCCTGCCTTCAGCCTCCCAAAGTGACTGGGTATTACAGATGTGAGCGAGTT
+TGTGCCCAAGCCTTATAAGTAAATTTATAAATTTACATAATTTAAATGACTTATGCTTAGCGAAATAGGG
+TAAG
+
+
The --without_sra -n option:
+============================
With this option you can switch off the special treatment (translation) of certain column-types
vdb-dump SRR000001 -R1 -C SPOT_DESC,PLATFORM
@@ -223,50 +276,77 @@ vdb-dump SRR000001 -R1 -C SPOT_DESC,PLATFORM -n
SPOT_DESC: [255, 0, 0, 0, 144, 1, 235, 0, 4, 0, 0, 0, 0, 0, 0, 0]
PLATFORM: 1
-The --no_accession -a option:
-With this option you can switch off the test if a given object is a sra-accession.
-It can speed up executing the tool.
-
-(2) printing other informations about a table:
-
-The --schema_dump -A option:
-With this option you can use vdb-dump to print the schema of a table instead of it's content.
-
-vdb-dump SRR000001 -A
The --table_enum -E option:
-For future use: if the object is a vdb-database, enumerate the tables it contains.
+===========================
+If the object is a vdb-database, enumerate the tables it contains.
+
The --version -V option:
+========================
Print the version of the vdb-manager used by vdb-dump.
vdb-dump -V
-vdb-dump: 1.0.0
+vdb-dump: 2.5.1
+
+
+The column_enum_short -o option:
+================================
+Enumerates the columns and the default type of each column
+vdb-dump SRR000001 -o
+
+BASE_COUNT (U64)
+BIO_BASE_COUNT (U64)
+CLIP_ADAPTER_LEFT (INSDC:coord:one)
+etc.
+
The column_enum -O option:
-Enumerates the columns and the types of columns of a table.
+==========================
+Enumerates the columns and all available type of each column
vdb-dump SRR000001 -O
-/panfs/traces01/sra0/SRR/000000/SRR000001.01 : (032 bits [01], Int) CLIP_QUALITY_LEFT
+SRR000001.01 : (032 bits [01], Int) CLIP_QUALITY_LEFT
(INSDC:coord:one)
CLIP_QUALITY_LEFT.type[0] = INSDC:coord:one (dflt)
CLIP_QUALITY_LEFT.type[1] = U16
CLIP_QUALITY_LEFT.type[2] = INSDC:coord:zero
-/panfs/traces01/sra0/SRR/000000/SRR000001.02 : (032 bits [01], Int) CLIP_QUALITY_RIGHT
+SRR000001.02 : (032 bits [01], Int) CLIP_QUALITY_RIGHT
(INSDC:coord:one)
CLIP_QUALITY_RIGHT.type[0] = INSDC:coord:one (dflt)
CLIP_QUALITY_RIGHT.type[1] = U16
CLIP_QUALITY_RIGHT.type[2] = INSDC:coord:zero
-/panfs/traces01/sra0/SRR/000000/SRR000001.03 : (008 bits [01], Uint) COLOR_MATRIX
+SRR000001.03 : (008 bits [01], Uint) COLOR_MATRIX
(U8)
COLOR_MATRIX.type[0] = U8 (dflt)
etc.
The --id_range -r option:
+=========================
Print the row-range that a table contains.
vdb-dump SRR000001 -r
id-range: first-row = 1, row-count = 470985
+
+
+The --info option:
+==================
+prints a summary of meta-data about the accession
+
+vdb-dump SRR000001 --info
+acc : SRR000001
+path : /somepath/SRR/000000/SRR000001
+size : 312,527,083
+type : Table
+platf : SRA_PLATFORM_454
+SEQ : 470,985
+SCHEMA : NCBI:SRA:_454_:tbl:v2#1.0.7
+TIME : 0x0000000055248a41 (04/07/2015 21:54)
+FMT : SFF
+FMTVER : 2.4.5
+LDR : sff-load.2.4.5
+LDRVER : 2.4.5
+LDRDATE: Feb 25 2015 (2/25/2015 0:0)
diff --git a/tools/vdb-dump/vdb-boot.vers b/tools/vdb-dump/vdb-boot.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-dump/vdb-boot.vers
+++ b/tools/vdb-dump/vdb-boot.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-dump/vdb-dump-bin.c b/tools/vdb-dump/vdb-dump-bin.c
index 06ffa2b..ea8cf85 100644
--- a/tools/vdb-dump/vdb-dump-bin.c
+++ b/tools/vdb-dump/vdb-dump-bin.c
@@ -28,6 +28,7 @@
#include <insdc/insdc.h>
+#include <kdb/manager.h>
#include <vdb/table.h>
#include <vdb/cursor.h>
@@ -37,8 +38,8 @@
#include <klib/text.h>
#include <klib/printf.h>
+#include <klib/num-gen.h>
-#include "vdb-dump-num-gen.h"
#include "vdb-dump-context.h"
#include "vdb-dump-coldefs.h"
@@ -257,7 +258,7 @@ static rc_t vdi_create_dir( const char * path, KDirectory ** dir )
}
-static rc_t vdi_dump_column_rows( const char * path, const VCursor *cur, p_col_def col, num_gen * rows )
+static rc_t vdi_dump_column_rows( const char * path, const VCursor *cur, p_col_def col, struct num_gen * rows )
{
KDirectory * dir;
@@ -269,30 +270,36 @@ static rc_t vdi_dump_column_rows( const char * path, const VCursor *cur, p_col_d
rc = create_wr_bin_idx( dir, col->name, &wr );
if ( rc == 0 )
{
- uint64_t row_id;
-
- vdn_start( rows );
- while ( vdn_next( rows, &row_id ) && rc == 0 )
+ const struct num_gen_iter * iter;
+ rc = num_gen_iterator_make( rows, &iter );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "VCursorIdRange() failed" );
+ else
{
- rc = Quitting();
- if ( rc == 0 )
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( iter, &row_id, &rc ) )
{
- const void * base;
- uint32_t elem_bits, boff, row_len;
- rc = VCursorCellDataDirect ( cur, (int64_t)row_id, col->idx,
- &elem_bits, &base, &boff, &row_len );
- if ( rc != 0 )
- {
- PLOGERR( klogInt, ( klogInt, rc,
- "VCursorCellData( col:$(col_name) at row #$(row_nr) ) failed",
- "col_name=%s,row_nr=%lu", col->name, row_id ) );
- }
- else
+ rc = Quitting();
+ if ( rc == 0 )
{
- uint32_t len = ( elem_bits >> 3 ) * row_len;
- rc = write_bin_idx( &wr, base, len );
+ const void * base;
+ uint32_t elem_bits, boff, row_len;
+ rc = VCursorCellDataDirect ( cur, row_id, col->idx,
+ &elem_bits, &base, &boff, &row_len );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogInt, ( klogInt, rc,
+ "VCursorCellData( col:$(col_name) at row #$(row_nr) ) failed",
+ "col_name=%s,row_nr=%ld", col->name, row_id ) );
+ }
+ else
+ {
+ uint32_t len = ( elem_bits >> 3 ) * row_len;
+ rc = write_bin_idx( &wr, base, len );
+ }
}
}
+ num_gen_iterator_destroy( iter );
}
if ( rc == 0 && !wr.multi_value )
@@ -318,48 +325,45 @@ static rc_t vdi_dump_column( const p_dump_context ctx, const VCursor *cur, p_col
}
else if ( count > 0 )
{
- num_gen * rows;
+ struct num_gen * rows = NULL;
- rc = vdn_make( &rows );
- if ( rc != 0 )
+
+ if ( ctx->row_range == NULL )
{
- LOGERR( klogInt, rc, "vdn_make() failed" );
+ /* the user did not give us a row-range, we take all rows of this column... */
+ rc = num_gen_make_from_range( &rows, first, count );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_make_from_range() failed" );
}
else
{
- if ( ctx->row_range == NULL )
- {
- /* the user did not give us a row-range, we take all rows of this column... */
- bool success = vdn_set_range( rows, first, count );
- if ( !success )
- {
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- LOGERR( klogInt, rc, "vdn_set_range() failed" );
- }
- }
+ /* the gave us a row-range, we parse that string and check agains the real row-count... */
+ rc = num_gen_make_from_str( &rows, ctx->row_range );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_make_from_str() failed" );
else
{
- /* the gave us a row-range, we parse that string and check agains the real row-count... */
- uint32_t num_ranges = vdn_parse( rows, ctx->row_range );
- if ( num_ranges < 1 )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
- LOGERR( klogInt, rc, "vdn_parse() failed" );
- }
- else
- vdn_check_range( rows, first, count );
+ rc = num_gen_trim( rows, first, count );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_trim() failed" );
}
+ }
- if ( !vdn_range_defined( rows ) )
+ if ( rc == 0 )
+ {
+ if ( num_gen_empty( rows ) )
{
rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
LOGERR( klogInt, rc, "no row-range(s) defined" );
}
else
+ {
rc = vdi_dump_column_rows( ctx->output_path, cur, col, rows ); /* <---- */
-
- vdn_destroy( rows );
+ }
}
+
+ if ( rows != NULL )
+ num_gen_destroy( rows );
}
return rc;
}
@@ -995,7 +999,7 @@ typedef struct alignment
/* ----------------------------------------------------------------------------------------------------------- */
-static rc_t vdi_bin_phase_1_row( const p_dump_context ctx, p1_ctx * p1_ctx, uint64_t row_id )
+static rc_t vdi_bin_phase_1_row( const p_dump_context ctx, p1_ctx * p1_ctx, int64_t row_id )
{
rc_t rc;
@@ -1172,60 +1176,58 @@ static rc_t vdi_bin_phase_1( KDirectory * dir, const p_dump_context ctx )
rc_t rc = init_p1_ctx( dir, &p1_ctx );
if ( rc == 0 )
{
- num_gen * rows;
+ struct num_gen * rows = NULL;
+ uint64_t row_count = p1_ctx.REF_POS.row_count;
- rc = vdn_make( &rows );
- if ( rc != 0 )
+ if ( ctx->row_range == NULL )
{
- LOGERR( klogInt, rc, "vdn_make() failed" );
+ /* the user did not give us a row-range, we take all rows of this column... */
+ rc = num_gen_make_from_range( &rows, 0, row_count );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_make_from_range() failed" );
}
else
{
- uint64_t row_count = p1_ctx.REF_POS.row_count;
-
- if ( ctx->row_range == NULL )
- {
- /* the user did not give us a row-range, we take all rows of this column... */
- bool success = vdn_set_range( rows, 0, row_count );
- if ( !success )
- {
- rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
- LOGERR( klogInt, rc, "vdn_set_range() failed" );
- }
- }
+ /* the gave us a row-range, we parse that string and check agains the real row-count... */
+ rc = num_gen_make_from_str( &rows, ctx->row_range );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_make_from_str() failed" );
else
{
- /* the gave us a row-range, we parse that string and check agains the real row-count... */
- uint32_t num_ranges = vdn_parse( rows, ctx->row_range );
- if ( num_ranges < 1 )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
- LOGERR( klogInt, rc, "vdn_parse() failed" );
- }
- else
- vdn_check_range( rows, 0, row_count );
+ rc = num_gen_trim( rows, 0, row_count );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_trim() failed" );
}
+ }
- if ( !vdn_range_defined( rows ) )
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
- LOGERR( klogInt, rc, "no row-range(s) defined" );
- }
+ if ( rc == 0 && num_gen_empty( rows ) )
+ {
+ rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
+ LOGERR( klogInt, rc, "no row-range(s) defined" );
+ }
+
+ if ( rc == 0 )
+ {
+ const struct num_gen_iter * iter;
+ rc = num_gen_iterator_make( rows, &iter );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "num_gen_iterator_make() failed" );
else
{
- uint64_t row_id;
-
- vdn_start( rows );
- while ( vdn_next( rows, &row_id ) && rc == 0 )
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( iter, &row_id, &rc ) )
{
rc = Quitting();
if ( rc == 0 )
rc = vdi_bin_phase_1_row( ctx, &p1_ctx, row_id );
}
+ num_gen_iterator_destroy( iter );
}
-
- vdn_destroy( rows );
}
+
+ if ( rows != NULL )
+ num_gen_destroy( rows );
+
release_p1_ctx( &p1_ctx );
}
return rc;
@@ -1234,16 +1236,13 @@ static rc_t vdi_bin_phase_1( KDirectory * dir, const p_dump_context ctx )
/* ----------------------------------------------------------------------------------------------------------- */
-
-/* ----------------------------------------------------------------------------------------------------------- */
-
rc_t vdi_bin_phase( const p_dump_context ctx, Args * args )
{
uint32_t count;
rc_t rc = ArgsParamCount( args, &count );
if ( rc != 0 )
{
- LOGERR( klogInt, rc, "VCursorOpen() failed" );
+ LOGERR( klogInt, rc, "ArgsParamCount() failed" );
}
else if ( count < 1 )
{
@@ -1253,7 +1252,7 @@ rc_t vdi_bin_phase( const p_dump_context ctx, Args * args )
else
{
const char *bin_path = NULL;
- rc = ArgsParamValue( args, 0, &bin_path );
+ rc = ArgsParamValue( args, 0, (const void**)&bin_path );
if ( rc != 0 )
{
LOGERR( klogInt, rc, "ArgsParamValue() failed" );
@@ -1275,3 +1274,223 @@ rc_t vdi_bin_phase( const p_dump_context ctx, Args * args )
}
return rc;
}
+
+
+/* ----------------------------------------------------------------------------------------------------------- */
+#define SLICE_COL_CG_LOW 0
+#define SLICE_COL_CG_HIGH 1
+#define SLICE_COL_SEQ_ID 2
+#define SLICE_COL_SEQ_START 3
+#define SLICE_COL_SEQ_LEN 4
+
+static rc_t get_seq_id( const VCursor * ref_cur, int64_t row, uint32_t *col_idx, char * buffer, size_t buflen )
+{
+ const char * value;
+ uint32_t len;
+ rc_t rc = VCursorCellDataDirect ( ref_cur, row, col_idx[ SLICE_COL_SEQ_ID ], NULL, ( const void ** )&value, NULL, &len );
+ if ( rc == 0 )
+ {
+ if ( len == 0 )
+ buffer[ 0 ] = 0;
+ else
+ string_copy( buffer, buflen, value, len );
+ }
+ return rc;
+}
+
+static INSDC_coord_one get_seq_start( const VCursor * ref_cur, int64_t row, uint32_t *col_idx )
+{
+ INSDC_coord_one * value;
+ uint32_t len;
+ rc_t rc = VCursorCellDataDirect ( ref_cur, row, col_idx[ SLICE_COL_SEQ_START ], NULL, ( const void ** )&value, NULL, &len );
+ if ( rc == 0 && len > 0 )
+ return *value;
+ return 0;
+}
+
+
+static INSDC_coord_len get_seq_len( const VCursor * ref_cur, int64_t row, uint32_t *col_idx )
+{
+ INSDC_coord_len * value;
+ uint32_t len;
+ rc_t rc = VCursorCellDataDirect ( ref_cur, row, col_idx[ SLICE_COL_SEQ_LEN ], NULL, ( const void ** )&value, NULL, &len );
+ if ( rc == 0 && len > 0 )
+ return *value;
+ return 0;
+}
+
+
+static rc_t find_slice_in_ref( const p_dump_context ctx, const VTable * ref_tab, const VTable * prim_tab )
+{
+ const VCursor * ref_cur;
+ rc_t rc = VTableCreateCachedCursorRead( ref_tab, &ref_cur, ctx->cur_cache_size );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VTableCreateCachedCursorRead( REFERENCE ) failed" );
+ }
+ else
+ {
+ uint32_t col_idx[ 5 ];
+ char seq_id[ 512 ];
+ uint32_t cgraph_len;
+ int64_t first = 0, row = 0;
+ uint64_t count = 0;
+ const uint8_t * cgraph_value;
+ bool done = false;
+
+ rc = VCursorAddColumn( ref_cur, &col_idx[ SLICE_COL_CG_LOW ], "CGRAPH_LOW" );
+ if ( rc == 0 )
+ rc = VCursorAddColumn( ref_cur, &col_idx[ SLICE_COL_CG_HIGH ], "CGRAPH_HIGH" );
+ if ( rc == 0 )
+ rc = VCursorAddColumn( ref_cur, &col_idx[ SLICE_COL_SEQ_ID ], "SEQ_ID" );
+ if ( rc == 0 )
+ rc = VCursorAddColumn( ref_cur, &col_idx[ SLICE_COL_SEQ_START ], "SEQ_START" );
+ if ( rc == 0 )
+ rc = VCursorAddColumn( ref_cur, &col_idx[ SLICE_COL_SEQ_LEN ], "SEQ_LEN" );
+ if ( rc == 0 )
+ rc = VCursorOpen( ref_cur );
+ if ( rc == 0 )
+ rc = VCursorIdRange( ref_cur, col_idx[ SLICE_COL_CG_LOW ], &first, &count );
+
+ for ( row = first; rc == 0 && !done && ( row < ( first + count ) ); ++row )
+ {
+ rc = VCursorCellDataDirect ( ref_cur, row, col_idx[ SLICE_COL_CG_LOW ], NULL, ( const void ** )&cgraph_value, NULL, &cgraph_len );
+ if ( rc == 0 && cgraph_len > 0 )
+ {
+ if ( *cgraph_value >= ctx->slice_depth )
+ {
+
+ rc = get_seq_id( ref_cur, row, col_idx, seq_id, sizeof seq_id );
+ if ( rc == 0 )
+ {
+ INSDC_coord_one seq_start = get_seq_start( ref_cur, row, col_idx );
+ INSDC_coord_len seq_len = get_seq_len( ref_cur, row, col_idx );
+ if ( ctx->indented_line_len > 0 && ctx->indented_line_len < seq_len )
+ rc = KOutMsg( "%s:%d-%d\n", seq_id, seq_start, seq_start + ctx->indented_line_len );
+ else
+ rc = KOutMsg( "%s:%d-%d\n", seq_id, seq_start, seq_start + seq_len - 1 );
+ }
+ done = true;
+ }
+ }
+ }
+
+ for ( row = first; rc == 0 && !done && ( row < ( first + count ) ); ++row )
+ {
+ rc = VCursorCellDataDirect ( ref_cur, row, col_idx[ SLICE_COL_CG_HIGH ], NULL, ( const void ** )&cgraph_value, NULL, &cgraph_len );
+ if ( rc == 0 && cgraph_len > 0 )
+ {
+ if ( *cgraph_value >= ctx->slice_depth )
+ {
+
+ rc = get_seq_id( ref_cur, row, col_idx, seq_id, sizeof seq_id );
+ if ( rc == 0 )
+ {
+ INSDC_coord_one seq_start = get_seq_start( ref_cur, row, col_idx );
+ INSDC_coord_len seq_len = get_seq_len( ref_cur, row, col_idx );
+ rc = KOutMsg( "%s:%d-%d\n", seq_id, seq_start, seq_start + seq_len - 1 );
+ }
+ done = true;
+ }
+ }
+ }
+
+
+ if ( !done || rc != 0 )
+ KOutMsg( "none\n" );
+
+ VCursorRelease( ref_cur );
+ }
+ return rc;
+}
+
+
+static rc_t find_slice_in( const p_dump_context ctx, const VDBManager *mgr, const char * acc )
+{
+ rc_t rc;
+ int path_type = ( VDBManagerPathType ( mgr, "%s", acc ) & ~ kptAlias );
+ /* types defined in <kdb/manager.h> */
+ if ( path_type == kptDatabase )
+ {
+ const VDatabase * db;
+ rc = VDBManagerOpenDBRead( mgr, &db, NULL, "%s", acc );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VDBManagerOpenDBRead() failed" );
+ }
+ else
+ {
+ const VTable * ref_tab;
+ rc = VDatabaseOpenTableRead( db, & ref_tab, "REFERENCE" );
+ if ( rc == 0 )
+ {
+ const VTable * prim_tab;
+ rc = VDatabaseOpenTableRead( db, & prim_tab, "PRIMARY_ALIGNMENT" );
+ if ( rc == 0 )
+ {
+ rc = find_slice_in_ref( ctx, ref_tab, prim_tab );
+ VTableRelease ( prim_tab );
+ }
+ VTableRelease ( ref_tab );
+ }
+ VDatabaseRelease( db );
+ }
+ if ( rc != 0 )
+ rc = KOutMsg( "none\n" );
+ }
+ else
+ rc = KOutMsg( "none\n" );
+ return rc;
+}
+
+
+rc_t find_slice( const p_dump_context ctx, Args * args )
+{
+ uint32_t count;
+ rc_t rc = ArgsParamCount( args, &count );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "ArgsParamCount() failed" );
+ }
+ else if ( count < 1 )
+ {
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
+ LOGERR( klogInt, rc, "parameter missing ( accession to find slice in )" );
+ }
+ else
+ {
+ KDirectory * dir;
+ rc = KDirectoryNativeDir( &dir );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "KDirectoryNativeDir() failed" );
+ }
+ else
+ {
+ const VDBManager *mgr;
+ rc = VDBManagerMakeRead ( &mgr, dir );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VDBManagerMakeRead() failed" );
+ }
+ else
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char *acc = NULL;
+ rc = ArgsParamValue( args, 0, (const void**)&acc );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "ArgsParamValue() failed" );
+ }
+ else
+ rc = find_slice_in( ctx, mgr, acc );
+ }
+ VDBManagerRelease( mgr );
+ }
+ KDirectoryRelease( dir );
+ }
+ }
+ return rc;
+}
diff --git a/tools/vdb-dump/vdb-dump-bin.h b/tools/vdb-dump/vdb-dump-bin.h
index 2caf4b4..0af510f 100644
--- a/tools/vdb-dump/vdb-dump-bin.h
+++ b/tools/vdb-dump/vdb-dump-bin.h
@@ -37,6 +37,8 @@ extern "C" {
rc_t vdi_dump_opened_table( const p_dump_context ctx, const VTable *my_table );
rc_t vdi_bin_phase( const p_dump_context ctx, Args * args );
+rc_t find_slice( const p_dump_context ctx, Args * args );
+
#ifdef __cplusplus
}
#endif
diff --git a/tools/vdb-dump/vdb-dump-coldefs.c b/tools/vdb-dump/vdb-dump-coldefs.c
index 5d819c3..af22165 100644
--- a/tools/vdb-dump/vdb-dump-coldefs.c
+++ b/tools/vdb-dump/vdb-dump-coldefs.c
@@ -33,7 +33,9 @@
#include <klib/printf.h>
#include <klib/log.h>
#include <klib/rc.h>
+
#include <vdb/vdb-priv.h>
+
#include <sra/sradb.h>
#include <sra/pacbio.h>
#include <os-native.h>
@@ -46,6 +48,9 @@
#include <stdio.h>
#include <string.h>
#include <assert.h>
+#include <math.h>
+
+rc_t Quitting();
/* once we get used to having moved the read descriptor
out of SRA, we should begin using those names.
@@ -108,7 +113,8 @@ const char *vdcd_get_platform_txt( const uint32_t id )
CASE ( SRA_PLATFORM_HELICOS );
CASE ( SRA_PLATFORM_PACBIO_SMRT );
CASE ( SRA_PLATFORM_ION_TORRENT );
- CASE ( SRA_PLATFORM_SANGER );
+ CASE ( SRA_PLATFORM_CAPILLARY );
+ CASE ( SRA_PLATFORM_OXFORD_NANOPORE );
}
#undef CASE
@@ -419,6 +425,45 @@ uint32_t vdcd_parse_string( col_defs* defs, const char* src, const VTable *my_ta
}
+bool vdcd_table_has_column( const VTable *my_table, const char * to_find )
+{
+ bool res = false;
+ if ( my_table != NULL && to_find != NULL )
+ {
+ size_t to_find_len = string_size( to_find );
+ if ( to_find_len > 0 )
+ {
+ KNamelist * names;
+ rc_t rc = VTableListCol( my_table, &names );
+ DISP_RC( rc, "VTableListCol() failed" );
+ if ( rc == 0 )
+ {
+ uint32_t n;
+ rc = KNamelistCount( names, &n );
+ DISP_RC( rc, "KNamelistCount() failed" );
+ if ( rc == 0 )
+ {
+ uint32_t i;
+ for ( i = 0; i < n && rc == 0 && !res; ++i )
+ {
+ const char * col_name;
+ rc = KNamelistGet( names, i, &col_name );
+ DISP_RC( rc, "KNamelistGet() failed" );
+ if ( rc == 0 )
+ {
+ size_t col_name_len = string_size( col_name );
+ if ( col_name_len == to_find_len )
+ res = ( string_cmp( to_find, to_find_len, col_name, col_name_len, col_name_len ) == 0 );
+ }
+ }
+ }
+ KNamelistRelease( names );
+ }
+ }
+ }
+ return res;
+}
+
uint32_t vdcd_extract_from_table( col_defs* defs, const VTable *my_table )
{
uint32_t found = 0;
@@ -438,7 +483,7 @@ uint32_t vdcd_extract_from_table( col_defs* defs, const VTable *my_table )
if ( rc == 0 )
{
uint32_t i;
- for ( i = 0; i < n && rc ==0; ++i )
+ for ( i = 0; i < n && rc == 0; ++i )
{
const char *col_name;
rc = KNamelistGet( names, i, &col_name );
@@ -668,3 +713,134 @@ bool vdcd_get_first_none_static_column_idx( col_defs* defs, const VCursor * cur,
}
return res;
}
+
+/* ******************************************************************************************************** */
+typedef struct spread
+{
+ uint64_t count;
+ double sum, sum_sq;
+ int64_t min, max;
+} spread;
+
+
+/*
+ s ... spread * s
+ b ... const void * base
+ l ... uint32_t row_len
+ t ... type ( int64_t, uint64_t ... )
+*/
+#define COUNTVALUES( S, b, l, t ) \
+ { \
+ const t * values = base; \
+ uint32_t i; \
+ for ( i = 0; i < l; ++i ) \
+ { \
+ t value = values[ i ]; \
+ if ( value != 0 ) \
+ { \
+ double value_d = value; \
+ if ( value < (S)->min ) (S)->min = value; \
+ if ( value > (S)->max ) (S)->max = value; \
+ (S)->sum += value_d; \
+ (S)->sum_sq += ( value_d * value_d ); \
+ (S)->count++; \
+ } \
+ } \
+ } \
+
+static uint64_t round_to_uint64_t( double value )
+{
+ double floor_value = floor( value );
+ double x = ( value - floor_value ) > 0.5 ? ceil( value ) : floor_value;
+ return ( uint64_t )x;
+}
+
+static rc_t vdcd_collect_spread_col( const struct num_gen * row_set, col_def * cd, const VCursor * cursor )
+{
+ const struct num_gen_iter * iter;
+ rc_t rc = num_gen_iterator_make( row_set, &iter );
+ if ( rc == 0 )
+ {
+ const void * base;
+ uint32_t row_len, elem_bits;
+ int64_t row_id;
+ spread s;
+ spread * sp = &s;
+
+ s.max = s.sum = s.sum_sq = s.count = 0;
+ s.min = INT64_MAX;
+
+ while ( ( rc == 0 ) && num_gen_iterator_next( iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 ) rc = Quitting();
+ if ( rc != 0 ) break;
+ rc = VCursorCellDataDirect( cursor, row_id, cd->idx, &elem_bits, &base, NULL, &row_len );
+ if ( rc == 0 )
+ {
+ if ( cd->type_desc.domain == vtdUint )
+ {
+ /* unsigned int's */
+ switch( elem_bits )
+ {
+ case 64 : COUNTVALUES( sp, base, row_len, uint64_t ) break;
+ case 32 : COUNTVALUES( sp, base, row_len, uint32_t ) break;
+ case 16 : COUNTVALUES( sp, base, row_len, uint16_t ) break;
+ case 8 : COUNTVALUES( sp, base, row_len, uint8_t ) break;
+ }
+ }
+ else
+ {
+ /* signed int's */
+ switch( elem_bits )
+ {
+ case 64 : COUNTVALUES( sp, base, row_len, int64_t ) break;
+ case 32 : COUNTVALUES( sp, base, row_len, int32_t ) break;
+ case 16 : COUNTVALUES( sp, base, row_len, int16_t ) break;
+ case 8 : COUNTVALUES( sp, base, row_len, int8_t ) break;
+ }
+ }
+ }
+ }
+
+ if ( s.count > 0 )
+ {
+ rc = KOutMsg( "\n[%s]\n", cd->name );
+ if ( rc == 0 )
+ rc = KOutMsg( "min = %,ld\n", s.min );
+ if ( rc == 0 )
+ rc = KOutMsg( "max = %,ld\n", s.max );
+ if ( rc == 0 )
+ rc = KOutMsg( "count = %,ld\n", s.count );
+ if ( rc == 0 )
+ {
+ double median = ( s.sum / s.count );
+ rc = KOutMsg( "median = %,ld\n", round_to_uint64_t( median ) );
+ if ( rc == 0 )
+ {
+ double stdev = sqrt( ( ( s.sum_sq - ( s.sum * s.sum ) / s.count ) ) / ( s.count - 1 ) );
+ rc = KOutMsg( "stdev = %,ld\n", round_to_uint64_t( stdev ) );
+ }
+ }
+ }
+
+ num_gen_iterator_destroy( iter );
+ }
+ return rc;
+}
+#undef COUNTVALUES
+
+rc_t vdcd_collect_spread( const struct num_gen * row_set, col_defs * cols, const VCursor * cursor )
+{
+ rc_t rc = 0;
+ uint32_t i, n = VectorLength( &cols->cols );
+ for ( i = 0; i < n && rc == 0; ++i )
+ {
+ col_def * cd = VectorGet( &cols->cols, i );
+ if ( cd != NULL )
+ {
+ if ( cd->type_desc.domain == vtdUint || cd->type_desc.domain == vtdInt )
+ rc = vdcd_collect_spread_col( row_set, cd, cursor );
+ }
+ }
+ return rc;
+}
diff --git a/tools/vdb-dump/vdb-dump-coldefs.h b/tools/vdb-dump/vdb-dump-coldefs.h
index d2dc6fd..fb81d04 100644
--- a/tools/vdb-dump/vdb-dump-coldefs.h
+++ b/tools/vdb-dump/vdb-dump-coldefs.h
@@ -32,8 +32,10 @@
#include <vdb/table.h>
#include <vdb/cursor.h>
#include <vdb/database.h>
+
#include <klib/vector.h>
#include <klib/text.h>
+#include <klib/num-gen.h>
#include "vdb-dump-str.h"
@@ -91,6 +93,7 @@ void vdcd_destroy( col_defs* defs );
uint32_t vdcd_parse_string( col_defs* defs, const char* src, const VTable *my_table );
uint32_t vdcd_extract_from_table( col_defs* defs, const VTable *my_table );
+bool vdcd_table_has_column( const VTable *my_table, const char * to_find );
bool vdcd_extract_from_phys_table( col_defs* defs, const VTable *my_table );
uint32_t vdcd_add_to_cursor( col_defs* defs, const VCursor *my_cursor );
void vdcd_reset_content( col_defs* defs );
@@ -98,6 +101,8 @@ void vdcd_ins_trans_fkt( col_defs* defs, const VSchema *my_schema );
void vdcd_exclude_these_columns( col_defs* defs, const char* column_names );
bool vdcd_get_first_none_static_column_idx( col_defs* defs, const VCursor * cur, uint32_t * idx );
+rc_t vdcd_collect_spread( const struct num_gen * row_set, col_defs * cols, const VCursor * cursor );
+
#ifdef __cplusplus
}
#endif
diff --git a/tools/vdb-dump/vdb-dump-context.c b/tools/vdb-dump/vdb-dump-context.c
index 32b19e9..3ce366c 100644
--- a/tools/vdb-dump/vdb-dump-context.c
+++ b/tools/vdb-dump/vdb-dump-context.c
@@ -31,6 +31,7 @@
#include <klib/log.h>
#include <klib/status.h>
#include <klib/text.h>
+#include <klib/num-gen.h>
#include <kapp/args.h>
#include <os-native.h>
#include <sysalloc.h>
@@ -86,6 +87,7 @@ static void vdco_init_values( p_dump_context ctx )
ctx->idx_range = NULL;
ctx->output_file = NULL;
ctx->output_path = NULL;
+ ctx->rows = NULL;
ctx->print_row_id = true;
ctx->print_in_hex = false;
@@ -105,7 +107,6 @@ static void vdco_init_values( p_dump_context ctx )
ctx->column_enum_short = false;
ctx->id_range_requested = false;
ctx->without_sra_types = false;
- ctx->dont_check_accession = false;
ctx->print_num_elem = false;
ctx->objver_requested = false;
ctx->objts_requested = false;
@@ -113,6 +114,11 @@ static void vdco_init_values( p_dump_context ctx )
ctx->idx_enum_requested = false;
ctx->idx_range_requested = false;
ctx->disable_multithreading = false;
+ ctx->table_defined = false;
+ ctx->diff = false;
+ ctx->show_spotgroups = false;
+ /*ctx->force_sra_schema = false;*/
+ ctx->show_spread = false;
}
rc_t vdco_init( dump_context **ctx )
@@ -133,8 +139,6 @@ rc_t vdco_init( dump_context **ctx )
{
VectorInit( &((*ctx)->schema_list), 0, 5 );
vdco_init_values( *ctx );
- rc = vdn_make( &((*ctx)->row_generator) );
- DISP_RC( rc, "num_gen_make() failed" );
}
}
return rc;
@@ -194,7 +198,11 @@ rc_t vdco_destroy( p_dump_context ctx )
ctx->output_file = NULL;
}
- vdn_destroy( ctx->row_generator );
+ if ( ctx->rows != NULL )
+ {
+ num_gen_destroy( ctx->rows );
+ ctx->rows = NULL;
+ }
free( ctx );
}
return rc;
@@ -299,7 +307,15 @@ static rc_t vdco_set_row_range( p_dump_context ctx, const char *src )
rc = vdco_set_str( (char**)&(ctx->row_range), src );
DISP_RC( rc, "vdco_set_str() failed" );
if ( rc == 0 )
- vdn_parse( ctx->row_generator, src );
+ {
+ if ( ctx->rows != NULL )
+ {
+ num_gen_destroy( ctx->rows );
+ ctx->rows = NULL;
+ }
+ rc = num_gen_make_from_str_sorted( &ctx->rows, src, ctx->merge_ranges );
+ DISP_RC( rc, "num_gen_make_from_str() failed" );
+ }
}
return rc;
}
@@ -365,13 +381,21 @@ static bool vdco_set_format( p_dump_context ctx, const char *src )
ctx->format = df_json;
else if ( strcmp( src, "piped" ) == 0 )
ctx->format = df_piped;
+ else if ( strcmp( src, "sra-dump" ) == 0 )
+ ctx->format = df_sra_dump;
else if ( strcmp( src, "tab" ) == 0 )
ctx->format = df_tab;
else if ( strcmp( src, "fastq" ) == 0 )
ctx->format = df_fastq;
- else if ( strcmp( src, "fasta" ) == 0 )
+ else if ( strcmp( src, "fastq1" ) == 0 )
+ ctx->format = df_fastq1;
+ else if ( strcmp( src, "fasta" ) == 0 )
ctx->format = df_fasta;
- else if ( strcmp( src, "bin" ) == 0 )
+ else if ( strcmp( src, "fasta1" ) == 0 )
+ ctx->format = df_fasta1;
+ else if ( strcmp( src, "fasta2" ) == 0 )
+ ctx->format = df_fasta2;
+ else if ( strcmp( src, "bin" ) == 0 )
ctx->format = df_bin;
else if ( strcmp( src, "sql" ) == 0 )
ctx->format = df_sql;
@@ -430,7 +454,7 @@ static uint16_t vdco_get_uint16_option( const Args *my_args,
if ( ( rc == 0 )&&( count > 0 ) )
{
const char *s;
- rc = ArgsOptionValue( my_args, name, 0, &s );
+ rc = ArgsOptionValue( my_args, name, 0, (const void **)&s );
DISP_RC( rc, "ArgsOptionValue() failed" );
if ( rc == 0 ) res = atoi( s );
}
@@ -449,7 +473,7 @@ static size_t vdco_get_size_t_option( const Args *my_args,
if ( ( rc == 0 )&&( count > 0 ) )
{
const char *s;
- rc = ArgsOptionValue( my_args, name, 0, &s );
+ rc = ArgsOptionValue( my_args, name, 0, (const void **)&s );
DISP_RC( rc, "ArgsOptionValue() failed" );
if ( rc == 0 )
{
@@ -470,7 +494,7 @@ static const char* vdco_get_str_option( const Args *my_args,
DISP_RC( rc, "ArgsOptionCount() failed" );
if ( ( rc == 0 )&&( count > 0 ) )
{
- rc = ArgsOptionValue( my_args, name, 0, &res );
+ rc = ArgsOptionValue( my_args, name, 0, (const void**)&res );
DISP_RC( rc, "ArgsOptionValue() failed" );
}
return res;
@@ -487,7 +511,7 @@ void vdco_set_schemas( const Args *my_args, p_dump_context ctx )
for ( i=0; i<count; ++i )
{
const char* txt = NULL;
- rc = ArgsOptionValue( my_args, OPTION_SCHEMA, i, &txt );
+ rc = ArgsOptionValue( my_args, OPTION_SCHEMA, i, (const void**)&txt );
DISP_RC( rc, "ArgsOptionValue() failed" );
if ( ( rc == 0 )&&( txt != NULL ) )
{
@@ -531,15 +555,20 @@ static void vdco_evaluate_options( const Args *my_args,
ctx->id_range_requested = vdco_get_bool_option( my_args, OPTION_ID_RANGE, false );
vdco_set_format( ctx, vdco_get_str_option( my_args, OPTION_FORMAT ) );
ctx->without_sra_types = vdco_get_bool_option( my_args, OPTION_WITHOUT_SRA, false );
- ctx->dont_check_accession = vdco_get_bool_option( my_args, OPTION_WITHOUT_ACCESSION, false );
ctx->print_num_elem = vdco_get_bool_option( my_args, OPTION_NUMELEM, false );
ctx->sum_num_elem = vdco_get_bool_option( my_args, OPTION_NUMELEMSUM, false );
ctx->show_blobbing = vdco_get_bool_option( my_args, OPTION_SHOW_BLOBBING, false );
ctx->enum_phys = vdco_get_bool_option( my_args, OPTION_ENUM_PHYS, false );
+ ctx->enum_readable = vdco_get_bool_option( my_args, OPTION_ENUM_READABLE, false );
ctx->idx_enum_requested = vdco_get_bool_option( my_args, OPTION_IDX_ENUM, false );
ctx->disable_multithreading = vdco_get_bool_option( my_args, OPTION_NO_MULTITHREAD, false );
ctx->print_info = vdco_get_bool_option( my_args, OPTION_INFO, false );
-
+ ctx->diff = vdco_get_bool_option( my_args, OPTION_DIFF, false );
+ ctx->show_spotgroups = vdco_get_bool_option( my_args, OPTION_SPOTGROUPS, false );
+ /*ctx->force_sra_schema = vdco_get_bool_option( my_args, OPTION_SRASCHEMA, false );*/
+ ctx->merge_ranges = vdco_get_bool_option( my_args, OPTION_MERGE_RANGES, false );
+ ctx->show_spread = vdco_get_bool_option( my_args, OPTION_SPREAD, false );
+
ctx->cur_cache_size = vdco_get_size_t_option( my_args, OPTION_CUR_CACHE, CURSOR_CACHE_SIZE );
ctx->output_buffer_size = vdco_get_size_t_option( my_args, OPTION_OUT_BUF_SIZE, DEF_OPTION_OUT_BUF_SIZE );
@@ -551,6 +580,8 @@ static void vdco_evaluate_options( const Args *my_args,
ctx->compress_mode = orm_uncompressed;
vdco_set_table( ctx, vdco_get_str_option( my_args, OPTION_TABLE ) );
+ ctx->table_defined = ( ctx->table != NULL );
+
vdco_set_columns( ctx, vdco_get_str_option( my_args, OPTION_COLUMNS ) );
vdco_set_excluded_columns( ctx, vdco_get_str_option( my_args, OPTION_EXCLUDED_COLUMNS ) );
vdco_set_row_range( ctx, vdco_get_str_option( my_args, OPTION_ROWS ) );
@@ -562,6 +593,9 @@ static void vdco_evaluate_options( const Args *my_args,
vdco_set_schemas( my_args, ctx );
vdco_set_filter( ctx, vdco_get_str_option( my_args, OPTION_FILTER ) );
vdco_set_boolean_char( ctx, vdco_get_str_option( my_args, OPTION_BOOLEAN ) );
+
+ if ( ctx->format == df_sra_dump )
+ ctx->without_sra_types = true;
}
rc_t vdco_capture_arguments_and_options( const Args * args, dump_context *ctx)
diff --git a/tools/vdb-dump/vdb-dump-context.h b/tools/vdb-dump/vdb-dump-context.h
index 26ec382..edb9ddf 100644
--- a/tools/vdb-dump/vdb-dump-context.h
+++ b/tools/vdb-dump/vdb-dump-context.h
@@ -36,7 +36,7 @@ extern "C" {
#include <kapp/args.h>
#include <klib/vector.h>
-#include "vdb-dump-num-gen.h"
+#include <klib/num-gen.h>
#include "vdb-dump-redir.h"
#define OPTION_ROW_ID_ON "row_id_on"
@@ -58,7 +58,6 @@ extern "C" {
#define OPTION_FORMAT "format"
#define OPTION_ID_RANGE "id_range"
#define OPTION_WITHOUT_SRA "without_sra"
-#define OPTION_WITHOUT_ACCESSION "without_accession"
#define OPTION_EXCLUDED_COLUMNS "exclude"
#define OPTION_BOOLEAN "boolean"
#define OPTION_OBJVER "obj_version"
@@ -68,6 +67,7 @@ extern "C" {
#define OPTION_NUMELEMSUM "numelemsum"
#define OPTION_SHOW_BLOBBING "blobbing"
#define OPTION_ENUM_PHYS "phys"
+#define OPTION_ENUM_READABLE "readable"
#define OPTION_IDX_ENUM "idx-report"
#define OPTION_IDX_RANGE "idx-range"
#define OPTION_CUR_CACHE "cur-cache"
@@ -79,6 +79,12 @@ extern "C" {
#define OPTION_OUT_BUF_SIZE "output-buffer-size"
#define OPTION_NO_MULTITHREAD "disable-multithreading"
#define OPTION_INFO "info"
+#define OPTION_DIFF "diff"
+#define OPTION_SPOTGROUPS "spotgroups"
+/*#define OPTION_SRASCHEMA "sraschema"*/
+#define OPTION_MERGE_RANGES "merge-ranges"
+#define OPTION_SPREAD "spread"
+#define OPTION_SLICE "slice"
#define ALIAS_ROW_ID_ON "I"
#define ALIAS_LINE_FEED "l"
@@ -99,7 +105,6 @@ extern "C" {
#define ALIAS_FORMAT "f"
#define ALIAS_ID_RANGE "r"
#define ALIAS_WITHOUT_SRA "n"
-#define ALIAS_WITHOUT_ACCESSION "a"
#define ALIAS_EXCLUDED_COLUMNS "x"
#define ALIAS_BOOLEAN "b"
#define ALIAS_OBJVER "j"
@@ -118,9 +123,13 @@ typedef enum dump_format_t
df_xml,
df_json,
df_piped,
+ df_sra_dump,
df_tab,
df_fastq,
+ df_fastq1,
df_fasta,
+ df_fasta1,
+ df_fasta2,
df_bin,
df_sql
} dump_format_t;
@@ -140,13 +149,14 @@ typedef struct dump_context
const char *row_range;
const char *output_file;
const char *output_path;
- num_gen *row_generator;
+ struct num_gen * rows;
bool print_row_id;
uint16_t lf_after_row;
uint16_t max_line_len;
uint16_t indented_line_len;
uint16_t phase;
uint32_t generic_idx;
+ uint32_t slice_depth;
size_t cur_cache_size;
size_t output_buffer_size;
dump_format_t format;
@@ -165,7 +175,6 @@ typedef struct dump_context
bool column_enum_short;
bool id_range_requested;
bool without_sra_types;
- bool dont_check_accession;
bool objver_requested;
bool objts_requested;
bool objtype_requested;
@@ -173,10 +182,17 @@ typedef struct dump_context
bool sum_num_elem;
bool show_blobbing;
bool enum_phys;
+ bool enum_readable;
bool idx_enum_requested;
bool idx_range_requested;
bool disable_multithreading;
bool print_info;
+ bool table_defined;
+ bool diff;
+ bool show_spotgroups;
+ /* bool force_sra_schema; */
+ bool merge_ranges;
+ bool show_spread;
} dump_context;
typedef dump_context* p_dump_context;
diff --git a/tools/vdb-dump/vdb-dump-fastq.c b/tools/vdb-dump/vdb-dump-fastq.c
index e9a9bb3..58c2cbe 100644
--- a/tools/vdb-dump/vdb-dump-fastq.c
+++ b/tools/vdb-dump/vdb-dump-fastq.c
@@ -26,7 +26,6 @@
#include "vdb-dump-fastq.h"
#include "vdb-dump-helper.h"
-#include "vdb-dump-num-gen.h"
#include <stdlib.h>
@@ -34,6 +33,10 @@
#include <vdb/vdb-priv.h>
#include <klib/log.h>
#include <klib/out.h>
+#include <klib/num-gen.h>
+
+#include <insdc/sra.h> /* for filter/types */
+
rc_t CC Quitting ( void );
@@ -42,294 +45,744 @@ rc_t CC Quitting ( void );
typedef struct fastq_ctx
{
- const char *run_name;
+ const char * run_name;
+ const VTable * tbl;
const VCursor * cursor;
+ const struct num_gen_iter * row_iter;
+ dump_format_t format;
+ size_t cur_cache_size;
+ uint32_t max_line_len;
uint32_t idx_read;
uint32_t idx_qual;
- uint32_t idx_name;
+ uint32_t idx_name;
+ uint32_t idx_read_start;
+ uint32_t idx_read_len;
+ uint32_t idx_read_type;
} fastq_ctx;
+static char * vdb_fastq_extract_run_name( const char * acc_or_path )
+{
+ char * delim = string_rchr ( acc_or_path, string_size( acc_or_path ), '/' );
+ if ( delim == NULL )
+ return string_dup_measure ( acc_or_path, NULL );
+ else
+ return string_dup_measure ( delim + 1, NULL );
+}
+
+
+static void init_fastq_ctx( const p_dump_context ctx, fastq_ctx * fctx, const char * acc_or_path )
+{
+ fctx->run_name = vdb_fastq_extract_run_name( acc_or_path );
+ fctx->tbl = NULL;
+ fctx->cursor = NULL;
+ fctx->row_iter = NULL;
+ fctx->max_line_len = ctx->max_line_len;
+ fctx->format = ctx->format;
+ fctx->cur_cache_size = ctx->cur_cache_size;
+ fctx->idx_read = INVALID_COLUMN;
+ fctx->idx_qual = INVALID_COLUMN;
+ fctx->idx_name = INVALID_COLUMN;
+ fctx->idx_read_start = INVALID_COLUMN;
+ fctx->idx_read_len = INVALID_COLUMN;
+ fctx->idx_read_type = INVALID_COLUMN;
+}
+
+
static void vdb_fastq_row_error( const char * fmt, rc_t rc, int64_t row_id )
{
PLOGERR( klogInt, ( klogInt, rc, fmt, "row_nr=%li", row_id ) );
}
-static rc_t vdb_fastq_loop_with_name( const p_dump_context ctx, const fastq_ctx * fctx )
+static bool is_name_in_list( KNamelist * col_names, const char * to_find )
{
- rc_t rc = 0;
- int64_t row_id;
+ bool res = false;
+ uint32_t count;
+ rc_t rc = KNamelistCount( col_names, &count );
+ DISP_RC( rc, "KNamelistCount() failed" );
+ if ( rc == 0 )
+ {
+ uint32_t i;
+ size_t to_find_len = string_size( to_find );
+ for ( i = 0; i < count && rc == 0 && !res; ++i )
+ {
+ const char * col_name;
+ rc = KNamelistGet( col_names, i, &col_name );
+ DISP_RC( rc, "KNamelistGet() failed" );
+ if ( rc == 0 )
+ {
+ size_t col_name_len = string_size( col_name );
+ if ( col_name_len == to_find_len )
+ res = ( string_cmp( to_find, to_find_len, col_name, col_name_len, col_name_len ) == 0 );
+ }
+ }
+ }
+ return res;
+}
- vdn_start( ctx->row_generator );
- while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
- {
- rc = Quitting();
- if ( rc == 0 )
- {
- uint32_t elem_bits, boff, row_len, name_len;
- const char * data;
- const char * name;
-
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_name, &elem_bits,
- (const void**)&name, &boff, &name_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), NAME ) failed", rc, row_id );
- else
- {
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
- (const void**)&data, &boff, &row_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
- else
- {
- rc = KOutMsg( "@%s.%li %.*s length=%u\n%.*s\n",
- fctx->run_name, row_id, name_len, name, row_len, row_len, data );
- if ( rc == 0 )
- {
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_qual, &elem_bits,
- (const void**)&data, &boff, &row_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), QUALITY ) failed", rc, row_id );
- else
- rc = KOutMsg( "+%s.%li %.*s length=%u\n%.*s\n",
- fctx->run_name, row_id, name_len, name, row_len, row_len, data );
- }
- }
- }
- }
- }
+
+static rc_t prepare_column( fastq_ctx * fctx, KNamelist * col_names, uint32_t * col_idx,
+ const char * to_find, const char * col_spec )
+{
+ rc_t rc = 0;
+ if ( is_name_in_list( col_names, to_find ) )
+ {
+ rc = VCursorAddColumn( fctx->cursor, col_idx, col_spec );
+ if ( rc != 0 )
+ {
+ *col_idx = INVALID_COLUMN;
+ PLOGERR( klogInt, ( klogInt, rc, "VCurosrAddColumn( '$(col)' ) failed", "col=%s", col_spec ) );
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_prepare_cursor( fastq_ctx * fctx )
+{
+ KNamelist * col_names;
+ rc_t rc = VTableListCol( fctx->tbl, &col_names );
+ DISP_RC( rc, "VTableListCol() failed" );
+ if ( rc == 0 )
+ {
+ rc = VTableCreateCachedCursorRead( fctx->tbl, &fctx->cursor, fctx->cur_cache_size );
+ DISP_RC( rc, "VTableCreateCursorRead( fasta/fastq ) failed" );
+ if ( rc == 0 )
+ rc = prepare_column( fctx, col_names, &fctx->idx_read, "READ", "(INSDC:dna:text)READ" );
+
+ if ( rc == 0 && ( fctx->format == df_fastq || fctx->format == df_fastq1 ) )
+ rc = prepare_column( fctx, col_names, &fctx->idx_qual, "QUALITY", "(INSDC:quality:text:phred_33)QUALITY" );
+
+ if ( rc == 0 )
+ {
+ if ( fctx->format == df_fasta2 )
+ rc = prepare_column( fctx, col_names, &fctx->idx_name, "SEQ_ID", "(ascii)SEQ_ID" );
+ if ( rc == 0 && fctx->idx_name == INVALID_COLUMN )
+ rc = prepare_column( fctx, col_names, &fctx->idx_name, "NAME", "(ascii)NAME" );
+ }
+
+ if ( rc == 0 )
+ rc = prepare_column( fctx, col_names, &fctx->idx_read_start, "READ_START", "(INSDC:coord:zero)READ_START" );
+
+ if ( rc == 0 )
+ rc = prepare_column( fctx, col_names, &fctx->idx_read_len, "READ_LEN", "(INSDC:coord:len)READ_LEN" );
+
+ if ( rc == 0 )
+ rc = prepare_column( fctx, col_names, &fctx->idx_read_type, "READ_TYPE", "(INSDC:SRA:xread_type)READ_TYPE" );
+
+ if ( rc == 0 )
+ {
+ rc = VCursorOpen ( fctx->cursor );
+ DISP_RC( rc, "VCursorOpen( fasta/fastq ) failed" );
+ }
+ KNamelistRelease( col_names );
+ }
return rc;
}
-static rc_t vdb_fasta_loop_with_name( const p_dump_context ctx, const fastq_ctx * fctx )
+typedef struct fastq_spot
{
- rc_t rc = 0;
- int64_t row_id;
+ const char * name;
+ const char * bases;
+ const char * qual;
+ const uint32_t * rd_start;
+ const uint32_t * rd_len;
+ const uint8_t * rd_type;
+ uint32_t name_len;
+ uint32_t num_bases;
+ uint32_t num_qual;
+ uint32_t num_rd_start;
+ uint32_t num_rd_len;
+ uint32_t num_rd_type;
+} fastq_spot;
+
+
+static rc_t read_spot( const fastq_ctx * fctx, int64_t row_id, fastq_spot * spot )
+{
+ rc_t rc = 0;
+ uint32_t elem_bits, boff;
+ if ( fctx->idx_name != INVALID_COLUMN )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_name, &elem_bits,
+ (const void**)&spot->name, &boff, &spot->name_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), NAME ) failed", rc, row_id );
+ }
+
+ if ( rc == 0 && fctx->idx_read != INVALID_COLUMN )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
+ (const void**)&spot->bases, &boff, &spot->num_bases );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
+ }
+
+ if ( rc == 0 && fctx->idx_qual != INVALID_COLUMN )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_qual, &elem_bits,
+ (const void**)&spot->qual, &boff, &spot->num_qual );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), QUALITY ) failed", rc, row_id );
+ }
+
+ if ( rc == 0 && fctx->idx_read_start != INVALID_COLUMN )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read_start, &elem_bits,
+ (const void**)&spot->rd_start, &boff, &spot->num_rd_start );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ_START ) failed", rc, row_id );
+ }
+
+ if ( rc == 0 && fctx->idx_read_len != INVALID_COLUMN )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read_len, &elem_bits,
+ (const void**)&spot->rd_len, &boff, &spot->num_rd_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ_LEN ) failed", rc, row_id );
+ }
+
+ if ( rc == 0 && fctx->idx_read_type != INVALID_COLUMN )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read_type, &elem_bits,
+ (const void**)&spot->rd_type, &boff, &spot->num_rd_type );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ_TYPE ) failed", rc, row_id );
+ }
+
+ return rc;
+}
- vdn_start( ctx->row_generator );
- while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
- {
- rc = Quitting();
- if ( rc == 0 )
- {
- uint32_t elem_bits, boff, row_len, name_len;
- const char * data;
- const char * name;
-
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_name, &elem_bits,
- (const void**)&name, &boff, &name_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), NAME ) failed", rc, row_id );
- else
- {
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
- (const void**)&data, &boff, &row_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
- else
- {
- uint32_t idx = 0;
- int32_t to_print = row_len;
-
- rc = KOutMsg( ">%s.%li %.*s length=%u\n",
- fctx->run_name, row_id, name_len, name, row_len );
- if ( to_print > ctx->max_line_len )
- to_print = ctx->max_line_len;
- while ( rc == 0 && to_print > 0 )
- {
- rc = KOutMsg( "%.*s\n", to_print, &data[ idx ] );
- if ( rc == 0 )
- {
- idx += ctx->max_line_len;
- to_print = ( row_len - idx );
- if ( to_print > ctx->max_line_len )
- to_print = ctx->max_line_len;
- }
- }
- }
- }
- }
- }
- return rc;
+
+static rc_t vdb_fastq1_frag_type_checked( fastq_spot * spot, int64_t row_id, const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ if ( spot->num_bases != spot->num_qual )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ PLOGERR( klogInt,
+ ( klogInt, rc, "invalid spot #$(row): bases.len( $(n_bases) ) != qual.len( $(n_qual)",
+ "row=%li,n_bases=%d,n_qual=%d", row_id, spot->num_bases, spot->num_qual ) );
+ }
+ else if ( spot->num_rd_start != spot->num_rd_len ||
+ spot->num_rd_start != spot->num_rd_type )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ PLOGERR( klogInt,
+ ( klogInt, rc,
+ "invalid spot #$(row): #READ_START=$(rd_start), #READ_LEN=$(rd_len), #READ_TYPE=$(rd_type)",
+ "row=%li,rd_start=%d,rd_len=%d,rd_type=%d",
+ row_id, spot->num_rd_start, spot->num_rd_len, spot->num_rd_type ) );
+ }
+ else
+ {
+ uint32_t idx, frag, ofs;
+ for ( idx = 0, frag = 1, ofs = 0; rc == 0 && idx < spot->num_rd_start; ++idx )
+ {
+ if ( ( ( spot->rd_type[ idx ] & READ_TYPE_BIOLOGICAL ) == READ_TYPE_BIOLOGICAL ) &&
+ spot->rd_len[ idx ] > 0 )
+ {
+ rc = KOutMsg( "@%s.%li.%d %.*s length=%u\n%.*s\n+%s.%li.%d %.*s length=%u\n%.*s\n",
+ fctx->run_name, row_id, frag, spot->name_len, spot->name, spot->rd_len[ idx ],
+ spot->rd_len[ idx ], &( spot->bases[ ofs ] ),
+ fctx->run_name, row_id, frag, spot->name_len, spot->name, spot->rd_len[ idx ],
+ spot->rd_len[ idx ], &( spot->qual[ ofs ] )
+ );
+ frag++;
+ }
+ ofs += spot->rd_len[ idx ];
+ }
+ }
+ return rc;
}
-static rc_t vdb_fastq_loop_without_name( const p_dump_context ctx, const fastq_ctx * fctx )
+static rc_t vdb_fastq1_frag_not_type_checked( fastq_spot * spot, int64_t row_id, const fastq_ctx * fctx )
{
- rc_t rc = 0;
- int64_t row_id;
+ rc_t rc = 0;
+ if ( spot->num_bases != spot->num_qual )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ PLOGERR( klogInt,
+ ( klogInt, rc, "invalid spot #$(row): bases.len( $(n_bases) ) != qual.len( $(n_qual)",
+ "row=%li,n_bases=%d,n_qual=%d", row_id, spot->num_bases, spot->num_qual ) );
+ }
+ else if ( spot->num_rd_start != spot->num_rd_len )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ PLOGERR( klogInt,
+ ( klogInt, rc,
+ "invalid spot #$(row): #READ_START=$(rd_start), #READ_LEN=$(rd_len)",
+ "row=%li,rd_start=%d,rd_len=%d",
+ row_id, spot->num_rd_start, spot->num_rd_len ) );
+ }
+ else
+ {
+ uint32_t idx, frag, ofs;
+ for ( idx = 0, frag = 1, ofs = 0; rc == 0 && idx < spot->num_rd_start; ++idx )
+ {
+ if ( spot->rd_len[ idx ] > 0 )
+ {
+ rc = KOutMsg( "@%s.%li.%d %.*s length=%u\n%.*s\n+%s.%li.%d %.*s length=%u\n%.*s\n",
+ fctx->run_name, row_id, frag, spot->name_len, spot->name, spot->rd_len[ idx ],
+ spot->rd_len[ idx ], &( spot->bases[ ofs ] ),
+ fctx->run_name, row_id, frag, spot->name_len, spot->name, spot->rd_len[ idx ],
+ spot->rd_len[ idx ], &( spot->qual[ ofs ] )
+ );
+ frag++;
+ }
+ ofs += spot->rd_len[ idx ];
+ }
+ }
+ return rc;
+}
- vdn_start( ctx->row_generator );
- while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
- {
- rc = Quitting();
- if ( rc == 0 )
- {
- uint32_t elem_bits, boff, row_len;
- const char * data;
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
- (const void**)&data, &boff, &row_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
- else
- {
- rc = KOutMsg( "@%s.%li %li length=%u\n%.*s\n",
- fctx->run_name, row_id, row_id, row_len, row_len, data );
- if ( rc == 0 )
- {
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_qual, &elem_bits,
- (const void**)&data, &boff, &row_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), QUALITY ) failed", rc, row_id );
- else
- rc = KOutMsg( "+%s.%li %li length=%u\n%.*s\n",
- fctx->run_name, row_id, row_id, row_len, row_len, data );
- }
- }
- }
- }
+static rc_t vdb_fastq1_loop( const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ if ( fctx->idx_read == INVALID_COLUMN || fctx->idx_name == INVALID_COLUMN ||
+ fctx->idx_qual == INVALID_COLUMN || fctx->idx_read_start == INVALID_COLUMN ||
+ fctx->idx_read_len == INVALID_COLUMN )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ DISP_RC( rc, "cannot generate fasta-format, at least one of these columns not found: READ, NAME, QUALITY, READ_START, READ_LEN" );
+ }
+ else
+ {
+ bool has_type = ( fctx->idx_read_type == INVALID_COLUMN );
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ {
+ if ( has_type )
+ rc = vdb_fastq1_frag_type_checked( &spot, row_id, fctx );
+ else
+ rc = vdb_fastq1_frag_not_type_checked( &spot, row_id, fctx );
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fastq_loop( const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ if ( fctx->idx_read == INVALID_COLUMN || fctx->idx_qual == INVALID_COLUMN )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ DISP_RC( rc, "cannot generate fasta-format: READ and/or QUALITY column not found" );
+ }
+ else
+ {
+ bool has_name = ( fctx->idx_name != INVALID_COLUMN );
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ {
+ if ( has_name )
+ rc = KOutMsg( "@%s.%li %.*s length=%u\n%.*s\n+%s.%li %.*s length=%u\n%.*s\n",
+ fctx->run_name, row_id, spot.name_len, spot.name, spot.num_bases,
+ spot.num_bases, spot.bases,
+ fctx->run_name, row_id, spot.name_len, spot.name, spot.num_qual,
+ spot.num_qual, spot.qual );
+ else
+
+ rc = KOutMsg( "@%s.%li %li length=%u\n%.*s\n+%s.%li %li length=%u\n%.*s\n",
+ fctx->run_name, row_id, row_id, spot.num_bases,
+ spot.num_bases, spot.bases,
+ fctx->run_name, row_id, row_id, spot.num_bases,
+ spot.num_qual, spot.qual );
+ }
+ }
+ }
+ }
return rc;
}
-static rc_t vdb_fasta_loop_without_name( const p_dump_context ctx, const fastq_ctx * fctx )
+static rc_t print_bases( const char * bases, uint32_t num_bases, uint32_t max_line_len )
{
- rc_t rc = 0;
- int64_t row_id;
+ rc_t rc;
+ if ( max_line_len == 0 )
+ rc = KOutMsg( "%.*s\n", num_bases, bases );
+ else
+ {
+ uint32_t idx = 0, to_print = num_bases;
+ rc = 0;
+ while ( rc == 0 && idx < num_bases )
+ {
+ if ( to_print > max_line_len )
+ to_print = max_line_len;
+
+ rc = KOutMsg( "%.*s\n", to_print, &bases[ idx ] );
+ if ( rc == 0 )
+ {
+ idx += to_print;
+ to_print = ( num_bases - idx );
+ }
+ }
+ }
+ return rc;
+}
- vdn_start( ctx->row_generator );
- while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
- {
- rc = Quitting();
- if ( rc == 0 )
- {
- uint32_t elem_bits, boff, row_len;
- const char * data;
- rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
- (const void**)&data, &boff, &row_len );
- if ( rc != 0 )
- vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
- else
- {
- uint32_t idx = 0;
- int32_t to_print = row_len;
-
- rc = KOutMsg( ">%s.%li %li length=%u\n",
- fctx->run_name, row_id, row_id, row_len );
- if ( to_print > ctx->max_line_len )
- to_print = ctx->max_line_len;
- while ( rc == 0 && to_print > 0 )
- {
- rc = KOutMsg( "%.*s\n", to_print, &data[ idx ] );
- if ( rc == 0 )
- {
- idx += ctx->max_line_len;
- to_print = ( row_len - idx );
- if ( to_print > ctx->max_line_len )
- to_print = ctx->max_line_len;
- }
- }
- }
- }
- }
+static rc_t vdb_fasta_frag_type_checked_loop( const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ bool has_name = ( fctx->idx_name != INVALID_COLUMN );
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ {
+ uint32_t idx, frag, ofs;
+ for ( idx = 0, frag = 1, ofs = 0; rc == 0 && idx < spot.num_rd_start; ++idx )
+ {
+ uint32_t frag_len = spot.rd_len[ idx ];
+ if ( frag_len > 0 &&
+ ( ( spot.rd_type[ idx ] & READ_TYPE_BIOLOGICAL ) == READ_TYPE_BIOLOGICAL ) )
+ {
+ if ( has_name )
+ rc = KOutMsg( ">%s.%li.%d %.*s length=%u\n",
+ fctx->run_name, row_id, frag, spot.name_len, spot.name, frag_len );
+ else
+ rc = KOutMsg( ">%s.%li.%d %li length=%u\n",
+ fctx->run_name, row_id, frag, row_id, frag_len );
+
+ if ( rc == 0 )
+ rc = print_bases( &( spot.bases[ ofs ] ), frag_len, fctx->max_line_len );
+
+ frag++;
+ }
+ ofs += frag_len;
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fasta_frag_no_type_check_loop( const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ bool has_name = ( fctx->idx_name != INVALID_COLUMN );
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ {
+ uint32_t idx, frag, ofs;
+ for ( idx = 0, frag = 1, ofs = 0; rc == 0 && idx < spot.num_rd_start; ++idx )
+ {
+ uint32_t frag_len = spot.rd_len[ idx ];
+ if ( frag_len > 0 )
+ {
+ if ( has_name )
+ rc = KOutMsg( ">%s.%li.%d %.*s length=%u\n",
+ fctx->run_name, row_id, frag, spot.name_len, spot.name, frag_len );
+ else
+ rc = KOutMsg( ">%s.%li.%d %li length=%u\n",
+ fctx->run_name, row_id, frag, row_id, frag_len );
+
+ if ( rc == 0 )
+ rc = print_bases( &( spot.bases[ ofs ] ), frag_len, fctx->max_line_len );
+
+ frag++;
+ }
+ ofs += frag_len;
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fasta_spot_loop( const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ bool has_name = ( fctx->idx_name != INVALID_COLUMN );
+ int64_t row_id;
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ {
+ if ( has_name )
+ rc = KOutMsg( ">%s.%li %.*s length=%u\n",
+ fctx->run_name, row_id, spot.name_len, spot.name, spot.num_bases );
+ else
+ rc = KOutMsg( ">%s.%li %li length=%u\n", fctx->run_name, row_id, row_id, spot.num_bases );
+
+ if ( rc == 0 )
+ rc = print_bases( spot.bases, spot.num_bases, fctx->max_line_len );
+ }
+ }
+ }
+ return rc;
+}
+
+static rc_t vdb_fasta_loop( const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ if ( fctx->idx_read == INVALID_COLUMN )
+ {
+ /* we actually only need a READ-column, everything else name/splitting etc. is optional... */
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ DISP_RC( rc, "cannot generate fasta-format: READ column not found" );
+ }
+ else
+ {
+ bool can_split = ( fctx->idx_read_start != INVALID_COLUMN && fctx->idx_read_len != INVALID_COLUMN );
+ if ( can_split )
+ {
+ bool has_type = ( fctx->idx_read_type != INVALID_COLUMN );
+ if ( has_type )
+ rc = vdb_fasta_frag_type_checked_loop( fctx );
+ else
+ rc = vdb_fasta_frag_no_type_check_loop( fctx );
+ }
+ else
+ rc = vdb_fasta_spot_loop( fctx );
+ }
return rc;
}
-static rc_t vdb_prepare_cursor( const p_dump_context ctx, const VTable * tbl, fastq_ctx * fctx )
+static rc_t vdb_fasta_accumulated( const char * bases, uint32_t num_bases,
+ int32_t * chars_left_on_line, uint32_t max_line_len )
{
- rc_t rc;
+ rc_t rc = 0;
+ if ( num_bases < ( *chars_left_on_line ) )
+ {
+ rc = KOutMsg( "%.*s", num_bases, bases );
+ ( *chars_left_on_line ) -= num_bases;
+ }
+ else if ( num_bases == ( *chars_left_on_line ) )
+ {
+ rc = KOutMsg( "%.*s\n", num_bases, bases );
+ ( *chars_left_on_line ) = max_line_len;
+ }
+ else
+ {
+ uint32_t ofs = 0;
+ int32_t remaining = num_bases;
+ while( rc == 0 && ofs < num_bases )
+ {
+ if ( remaining >= ( *chars_left_on_line ) )
+ {
+ rc = KOutMsg( "%.*s\n", ( *chars_left_on_line ), &bases[ ofs ] );
+ ofs += ( *chars_left_on_line );
+ remaining -= ( *chars_left_on_line );
+ ( *chars_left_on_line ) = max_line_len;
+ }
+ else
+ {
+ rc = KOutMsg( "%.*s", remaining, &bases[ ofs ] );
+ ofs += remaining;
+ ( *chars_left_on_line ) -= remaining;
+ remaining = 0;
+ }
+ }
+ }
+ return rc;
+}
- /* first we try to open READ/QUALITY/NAME */
- rc = VTableCreateCachedCursorRead( tbl, &fctx->cursor, ctx->cur_cache_size );
- DISP_RC( rc, "VTableCreateCursorRead( 1st ) failed" );
- if ( rc == 0 )
- {
- rc = VCursorAddColumn( fctx->cursor, &fctx->idx_read, "(INSDC:dna:text)READ" );
- if ( rc == 0 && ctx->format == df_fastq )
- rc = VCursorAddColumn( fctx->cursor, &fctx->idx_qual, "(INSDC:quality:text:phred_33)QUALITY" );
- else
- fctx->idx_qual = INVALID_COLUMN;
- if ( rc == 0 )
- rc = VCursorAddColumn( fctx->cursor, &fctx->idx_name, "(ascii)NAME" );
- if ( rc == 0 )
- rc = VCursorPermitPostOpenAdd ( fctx->cursor );
- if ( rc == 0 )
- rc = VCursorOpen ( fctx->cursor );
- if ( rc != 0 )
- {
- VCursorRelease( fctx->cursor );
- rc = VTableCreateCachedCursorRead( tbl, &fctx->cursor, ctx->cur_cache_size );
- DISP_RC( rc, "VTableCreateCursorRead( 2nd ) failed" );
- if ( rc == 0 )
- {
- rc = VCursorAddColumn( fctx->cursor, &fctx->idx_read, "(INSDC:dna:text)READ" );
- if ( rc == 0 && ctx->format == df_fastq )
- rc = VCursorAddColumn( fctx->cursor, &fctx->idx_qual, "(INSDC:quality:text:phred_33)QUALITY" );
- else
- fctx->idx_qual = INVALID_COLUMN;
- if ( rc == 0 )
- rc = VCursorOpen ( fctx->cursor );
- fctx->idx_name = INVALID_COLUMN;
- }
- }
- }
+static rc_t vdb_fasta1_loop( const fastq_ctx * fctx )
+{
+ rc_t rc;
+ if ( fctx->idx_read == INVALID_COLUMN )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ DISP_RC( rc, "cannot generate fasta1-format: READ column not found" );
+ }
+ else
+ {
+ int64_t row_id;
+ int32_t chars_left_on_line = fctx->max_line_len;
+
+ rc = KOutMsg( ">%s\n", fctx->run_name );
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ rc = vdb_fasta_accumulated( spot.bases, spot.num_bases, &chars_left_on_line, fctx->max_line_len );
+ }
+ }
+ rc = KOutMsg( "\n" );
+ }
return rc;
}
-static rc_t vdb_fastq_tbl( const p_dump_context ctx,
- const VTable * tbl,
- fastq_ctx * fctx )
+static rc_t vdb_fasta2_loop( const fastq_ctx * fctx )
{
- rc_t rc = vdb_prepare_cursor( ctx, tbl, fctx );
+ rc_t rc = 0;
+ if ( fctx->idx_name == INVALID_COLUMN || fctx->idx_read == INVALID_COLUMN )
+ {
+ rc = RC( rcExe, rcNoTarg, rcConstructing, rcNoObj, rcInvalid );
+ DISP_RC( rc, "cannot generate fasta2-format: READ and/or NAME column not found" );
+ }
+ else
+ {
+ char last_name[ 1024 ];
+ size_t last_name_len = 0;
+ int64_t row_id;
+ int32_t chars_left_on_line = fctx->max_line_len;
+
+ while ( rc == 0 && num_gen_iterator_next( fctx->row_iter, &row_id, &rc ) )
+ {
+ if ( rc == 0 )
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ fastq_spot spot;
+ rc = read_spot( fctx, row_id, &spot );
+ if ( rc == 0 )
+ {
+ bool print_ref_name = ( last_name_len == 0 );
+ if ( !print_ref_name )
+ {
+ print_ref_name = ( last_name_len != spot.name_len );
+ if ( !print_ref_name )
+ print_ref_name = ( string_cmp( last_name, last_name_len, spot.name, spot.name_len, spot.name_len ) != 0 );
+ }
+
+ if ( print_ref_name )
+ {
+ if ( chars_left_on_line == fctx->max_line_len )
+ rc = KOutMsg( ">%.*s\n", spot.name_len, spot.name );
+ else
+ {
+ rc = KOutMsg( "\n>%.*s\n", spot.name_len, spot.name );
+ chars_left_on_line = fctx->max_line_len;
+ }
+ last_name_len = string_copy ( last_name, sizeof last_name, spot.name, spot.name_len );
+ }
+
+ if ( rc == 0 )
+ rc = vdb_fasta_accumulated( spot.bases, spot.num_bases, &chars_left_on_line, fctx->max_line_len );
+ }
+ }
+ }
+ rc = KOutMsg( "\n" );
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fastq_tbl( const p_dump_context ctx, fastq_ctx * fctx )
+{
+ rc_t rc = vdb_prepare_cursor( fctx );
DISP_RC( rc, "the table lacks READ and/or QUALITY column" );
if ( rc == 0 )
{
int64_t first;
uint64_t count;
+ /* READ is the colum we have in all cases... */
rc = VCursorIdRange( fctx->cursor, fctx->idx_read, &first, &count );
DISP_RC( rc, "VCursorIdRange() failed" );
if ( rc == 0 )
{
- /* if the user did not specify a row-range, take all rows */
- if ( vdn_range_defined( ctx->row_generator ) == false )
+ if ( count == 0 )
{
- vdn_set_range( ctx->row_generator, first, count );
+ KOutMsg( "this table is empty\n" );
}
- /* if the user did specify a row-range, check the boundaries */
else
{
- vdn_check_range( ctx->row_generator, first, count );
- }
-
- if ( vdn_range_defined( ctx->row_generator ) )
- {
- if ( ctx->format == df_fastq )
+ /* if the user did not specify a row-range, take all rows */
+ if ( ctx->rows == NULL )
{
- if ( fctx->idx_name == INVALID_COLUMN)
- rc = vdb_fastq_loop_without_name( ctx, fctx ); /* <--- */
- else
- rc = vdb_fastq_loop_with_name( ctx, fctx ); /* <--- */
+ rc = num_gen_make_from_range( &ctx->rows, first, count );
+ DISP_RC( rc, "num_gen_make_from_range() failed" );
}
- else if ( ctx->format == df_fasta )
+ /* if the user did specify a row-range, check the boundaries */
+ else
{
- if ( ctx->max_line_len == 0 )
- ctx->max_line_len = DEF_FASTA_LEN;
- if ( fctx->idx_name == INVALID_COLUMN)
- rc = vdb_fasta_loop_without_name( ctx, fctx ); /* <--- */
- else
- rc = vdb_fasta_loop_with_name( ctx, fctx ); /* <--- */
+ rc = num_gen_trim( ctx->rows, first, count );
+ DISP_RC( rc, "num_gen_trim() failed" );
}
- }
- else
- {
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
+
+ if ( rc == 0 && !num_gen_empty( ctx->rows ) )
+ {
+ rc = num_gen_iterator_make( ctx->rows, &fctx->row_iter );
+ DISP_RC( rc, "num_gen_iterator_make() failed" );
+ if ( rc == 0 )
+ {
+ if ( fctx->max_line_len == 0 )
+ fctx->max_line_len = DEF_FASTA_LEN;
+
+ switch( fctx->format )
+ {
+ /* one FASTQ-record ( 4 liner ) per READ/SPOT */
+ case df_fastq : rc = vdb_fastq_loop( fctx ); /* <--- */
+ break;
+
+ /* one FASTQ-record ( 4 liner ) per FRAGMENT/ALIGNMENT */
+ case df_fastq1 : rc = vdb_fastq1_loop( fctx ); /* <--- */
+ break;
+
+ /* one FASTA-record ( 2 liner ) per READ/SPOT */
+ case df_fasta : rc = vdb_fasta_loop( fctx ); /* <--- */
+ break;
+
+ /* one FASTA-record ( many lines ) for the whole accession ( REFSEQ-accession ) */
+ case df_fasta1 : rc = vdb_fasta1_loop( fctx ); /* <--- */
+ break;
+
+ /* one FASTA-record ( many lines ) for each REFERENCE used in a cSRA-database */
+ case df_fasta2 : rc = vdb_fasta2_loop( fctx ); /* <--- */
+ break;
+
+ default : break;
+ }
+ num_gen_iterator_destroy( fctx->row_iter );
+ }
+ }
+ else
+ rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
}
}
VCursorRelease( fctx->cursor );
@@ -342,20 +795,21 @@ static rc_t vdb_fastq_table( const p_dump_context ctx,
const VDBManager *mgr,
fastq_ctx * fctx )
{
- const VTable * tbl;
- VSchema * schema = NULL;
+ VSchema * schema = NULL;
rc_t rc;
- vdh_parse_schema( mgr, &schema, &(ctx->schema_list) );
+ vdh_parse_schema( mgr, &schema, &(ctx->schema_list), true /* ctx->force_sra_schema */ );
- rc = VDBManagerOpenTableRead( mgr, &tbl, schema, "%s", ctx->path );
+ rc = VDBManagerOpenTableRead( mgr, &fctx->tbl, schema, "%s", ctx->path );
DISP_RC( rc, "VDBManagerOpenTableRead() failed" );
if ( rc == 0 )
{
- rc = vdb_fastq_tbl( ctx, tbl, fctx );
- VTableRelease( tbl );
+ rc = vdb_fastq_tbl( ctx, fctx );
+ VTableRelease( fctx->tbl );
}
- VSchemaRelease( schema );
+
+ if ( schema != NULL )
+ VSchemaRelease( schema );
return rc;
}
@@ -369,7 +823,7 @@ static rc_t vdb_fastq_database( const p_dump_context ctx,
VSchema *schema = NULL;
rc_t rc;
- vdh_parse_schema( mgr, &schema, &(ctx->schema_list) );
+ vdh_parse_schema( mgr, &schema, &(ctx->schema_list), true /* ctx->force_sra_schema */ );
rc = VDBManagerOpenDBRead( mgr, &db, schema, "%s", ctx->path );
DISP_RC( rc, "VDBManagerOpenDBRead() failed" );
@@ -381,14 +835,12 @@ static rc_t vdb_fastq_database( const p_dump_context ctx,
if ( table_defined )
{
- const VTable * tbl;
-
- rc = VDatabaseOpenTableRead( db, &tbl, "%s", ctx->table );
+ rc = VDatabaseOpenTableRead( db, &fctx->tbl, "%s", ctx->table );
DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
if ( rc == 0 )
{
- rc = vdb_fastq_tbl( ctx, tbl, fctx );
- VTableRelease( tbl );
+ rc = vdb_fastq_tbl( ctx, fctx );
+ VTableRelease( fctx->tbl );
}
}
else
@@ -399,7 +851,8 @@ static rc_t vdb_fastq_database( const p_dump_context ctx,
VDatabaseRelease( db );
}
- VSchemaRelease( schema );
+ if ( schema != NULL )
+ VSchemaRelease( schema );
return rc;
}
@@ -466,35 +919,20 @@ static rc_t vdb_fastq_by_probing( const p_dump_context ctx,
}
-static char * vdb_fastq_extract_run_name( const char * acc_or_path )
-{
- char * delim = string_rchr ( acc_or_path, string_size( acc_or_path ), '/' );
- if ( delim == NULL )
- return string_dup_measure ( acc_or_path, NULL );
- else
- return string_dup_measure ( delim + 1, NULL );
-}
-
-
rc_t vdf_main( const p_dump_context ctx, const VDBManager * mgr, const char * acc_or_path )
{
rc_t rc = 0;
fastq_ctx fctx;
- fctx.run_name = vdb_fastq_extract_run_name( acc_or_path );
-
+ init_fastq_ctx( ctx, &fctx, acc_or_path );
ctx->path = string_dup_measure ( acc_or_path, NULL );
-
+
if ( USE_PATHTYPE_TO_DETECT_DB_OR_TAB ) /* in vdb-dump-context.h */
- {
rc = vdb_fastq_by_pathtype( ctx, mgr, &fctx );
- }
else
- {
rc = vdb_fastq_by_probing( ctx, mgr, &fctx );
- }
- free( (char*)ctx->path );
- free( (void*)fctx.run_name );
+ free( ( char* )ctx->path );
+ free( ( void* )fctx.run_name );
ctx->path = NULL;
return rc;
diff --git a/tools/vdb-dump/vdb-dump-formats.c b/tools/vdb-dump/vdb-dump-formats.c
index 4b66d7a..1a820dc 100644
--- a/tools/vdb-dump/vdb-dump-formats.c
+++ b/tools/vdb-dump/vdb-dump-formats.c
@@ -114,7 +114,7 @@ static void CC vdfo_print_col_csv( void *item, void *data )
if ( my_col_def->valid == false ) return;
if ( my_col_def->excluded == true ) return;
- if ( r_ctx->col_nr > 0 )
+ if ( r_ctx->col_nr > 0 || r_ctx->ctx->print_row_id )
{
rc = vds_append_str( &(r_ctx->s_col), "," );
DISP_RC( rc, "dump_str_append_str() failed" )
@@ -136,6 +136,9 @@ static rc_t vdfo_print_row_csv( const p_row_context r_ctx )
{
rc_t rc = vds_clear( &(r_ctx->s_col) );
DISP_RC( rc, "dump_str_clear() failed" )
+ if ( rc == 0 && r_ctx->ctx->print_row_id )
+ rc = KOutMsg( "%u", r_ctx->row_id );
+
if ( rc == 0 )
{
r_ctx->col_nr = 0;
@@ -273,6 +276,26 @@ static void CC vdfo_print_col_piped( void *item, void *data )
/*************************************************************************************
+ like legacy sra-dump
+*************************************************************************************/
+static void CC vdfo_print_col_sra_dump( void *item, void *data )
+{
+ rc_t rc = 0;
+ p_col_def my_col_def = (p_col_def)item;
+ p_row_context r_ctx = (p_row_context)data;
+
+ if ( my_col_def->valid == false ) return;
+ if ( my_col_def->excluded == true ) return;
+
+ /* first we print the row_id and the column-name for every column! */
+ KOutMsg( "%lu. %s: ", r_ctx->row_id, my_col_def->name );
+
+ if ( rc == 0 )
+ KOutMsg( "%s\n", my_col_def->content.buf );
+}
+
+
+/*************************************************************************************
TAB-delimited
*************************************************************************************/
static void CC vdfo_print_col_tab( void *item, void *data )
@@ -284,7 +307,7 @@ static void CC vdfo_print_col_tab( void *item, void *data )
if ( my_col_def->valid == false ) return;
if ( my_col_def->excluded == true ) return;
- if ( r_ctx->col_nr > 0 )
+ if ( r_ctx->col_nr > 0 || r_ctx->ctx->print_row_id )
{
rc = vds_append_str( &(r_ctx->s_col), "\t" );
DISP_RC( rc, "dump_str_append_str() failed" )
@@ -310,10 +333,26 @@ static rc_t vdfo_print_row_piped( const p_row_context r_ctx )
return rc;
}
+
+static rc_t vdfo_print_row_sra_dump( const p_row_context r_ctx )
+{
+ rc_t rc = vds_clear( &(r_ctx->s_col) );
+ DISP_RC( rc, "dump_str_clear() failed" )
+ if ( rc == 0 )
+ {
+ VectorForEach( &(r_ctx->col_defs->cols), false, vdfo_print_col_sra_dump, r_ctx );
+ rc = KOutMsg( "\n" );
+ }
+ return rc;
+}
+
static rc_t vdfo_print_row_tab( const p_row_context r_ctx )
{
rc_t rc = vds_clear( &(r_ctx->s_col) );
DISP_RC( rc, "dump_str_clear() failed" )
+ if ( rc == 0 && r_ctx->ctx->print_row_id )
+ rc = KOutMsg( "%u", r_ctx->row_id );
+
if ( rc == 0 )
{
r_ctx->col_nr = 0;
@@ -331,13 +370,14 @@ rc_t vdfo_print_row( const p_row_context r_ctx )
rc_t rc = 0;
switch( r_ctx->ctx->format )
{
- case df_default : rc = vdfo_print_row_default( r_ctx ); break;
- case df_csv : rc = vdfo_print_row_csv( r_ctx ); break;
- case df_xml : rc = vdfo_print_row_xml( r_ctx ); break;
- case df_json : rc = vdfo_print_row_json( r_ctx ); break;
- case df_piped : rc = vdfo_print_row_piped( r_ctx ); break;
- case df_tab : rc = vdfo_print_row_tab( r_ctx ); break;
- default : rc = vdfo_print_row_default( r_ctx ); break;
+ case df_default : rc = vdfo_print_row_default( r_ctx ); break;
+ case df_csv : rc = vdfo_print_row_csv( r_ctx ); break;
+ case df_xml : rc = vdfo_print_row_xml( r_ctx ); break;
+ case df_json : rc = vdfo_print_row_json( r_ctx ); break;
+ case df_piped : rc = vdfo_print_row_piped( r_ctx ); break;
+ case df_sra_dump : rc = vdfo_print_row_sra_dump( r_ctx ); break;
+ case df_tab : rc = vdfo_print_row_tab( r_ctx ); break;
+ default : rc = vdfo_print_row_default( r_ctx ); break;
}
return rc;
}
diff --git a/tools/vdb-dump/vdb-dump-helper.c b/tools/vdb-dump/vdb-dump-helper.c
index c937e7f..d3ef011 100644
--- a/tools/vdb-dump/vdb-dump-helper.c
+++ b/tools/vdb-dump/vdb-dump-helper.c
@@ -24,11 +24,14 @@
*
*/
-#include <kfs/directory.h>
#include <klib/log.h>
#include <klib/rc.h>
#include <klib/text.h>
+#include <kfs/directory.h>
+#include <kfs/file.h>
+#include <kfs/cacheteefile.h>
+
#include <vdb/manager.h>
#include <vdb/database.h>
#include <vdb/schema.h>
@@ -78,9 +81,10 @@ static void CC vdh_parse_1_schema( void *item, void *data )
rc_t vdh_parse_schema( const VDBManager *my_manager,
VSchema **new_schema,
- Vector *schema_list )
+ Vector *schema_list,
+ bool with_sra_schema )
{
- rc_t rc;
+ rc_t rc = 0;
if ( my_manager == NULL )
{
@@ -91,16 +95,24 @@ rc_t vdh_parse_schema( const VDBManager *my_manager,
return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
}
- rc = VDBManagerMakeSRASchema( my_manager, new_schema );
- DISP_RC( rc, "VDBManagerMakeSRASchema() failed" );
-
+ *new_schema = NULL;
+
+ if ( with_sra_schema )
+ {
+ rc = VDBManagerMakeSRASchema( my_manager, new_schema );
+ DISP_RC( rc, "VDBManagerMakeSRASchema() failed" );
+ }
+
if ( ( rc == 0 )&&( schema_list != NULL ) )
{
- VectorForEach( schema_list, false, vdh_parse_1_schema, *new_schema );
+ if ( *new_schema == NULL )
+ {
+ rc = VDBManagerMakeSchema( my_manager, new_schema );
+ DISP_RC( rc, "VDBManagerMakeSchema() failed" );
+ }
+ if ( rc == 0 )
+ VectorForEach( schema_list, false, vdh_parse_1_schema, *new_schema );
}
-
-/* rc = VDBManagerMakeSchema( my_manager, new_schema );
- display_rescode( rc, "failed to make a schema", NULL ); */
return rc;
}
@@ -115,7 +127,7 @@ bool vdh_is_path_table( const VDBManager *my_manager, const char *path,
VSchema *my_schema = NULL;
rc_t rc;
- rc = vdh_parse_schema( my_manager, &my_schema, schema_list );
+ rc = vdh_parse_schema( my_manager, &my_schema, schema_list, false );
DISP_RC( rc, "helper_parse_schema() failed" );
rc = VDBManagerOpenTableRead( my_manager, &my_table, my_schema, "%s", path );
@@ -182,7 +194,7 @@ bool vdh_is_path_database( const VDBManager *my_manager, const char *path,
VSchema *my_schema = NULL;
rc_t rc;
- rc = vdh_parse_schema( my_manager, &my_schema, schema_list );
+ rc = vdh_parse_schema( my_manager, &my_schema, schema_list, false );
DISP_RC( rc, "helper_parse_schema() failed" );
rc = VDBManagerOpenDBRead( my_manager, &my_database, my_schema, "%s", path );
@@ -448,3 +460,85 @@ rc_t resolve_accession( const char * accession, char * dst, size_t dst_size, boo
}
return rc;
}
+
+
+rc_t resolve_cache( const char * accession, char * dst, size_t dst_size )
+{
+ VFSManager * vfs_mgr;
+ rc_t rc = VFSManagerMake( &vfs_mgr );
+ dst[ 0 ] = 0;
+ if ( rc == 0 )
+ {
+ VResolver * resolver;
+ rc = VFSManagerGetResolver( vfs_mgr, &resolver );
+ if ( rc == 0 )
+ {
+ VPath * vpath;
+ rc = VFSManagerMakePath( vfs_mgr, &vpath, "ncbi-acc:%s", accession );
+ if ( rc == 0 )
+ {
+ const VPath * local = NULL;
+ const VPath * remote = NULL;
+ const VPath * cache = NULL;
+ rc = VResolverQuery ( resolver, eProtocolHttp, vpath, &local, &remote, &cache );
+ if ( rc == 0 && cache != NULL )
+ {
+ const String * path;
+ rc = VPathMakeString( cache, &path );
+
+ if ( rc == 0 && path != NULL )
+ {
+ string_copy ( dst, dst_size, path->addr, path->size );
+ dst[ path->size ] = 0;
+ StringWhack ( path );
+ }
+
+ if ( local != NULL )
+ VPathRelease ( local );
+ if ( remote != NULL )
+ VPathRelease ( remote );
+ if ( remote != NULL )
+ VPathRelease ( cache );
+ }
+ VPathRelease ( vpath );
+ }
+ VResolverRelease( resolver );
+ }
+ VFSManagerRelease ( vfs_mgr );
+ }
+ return rc;
+}
+
+
+rc_t check_cache_comleteness( const char * path, float * percent, uint64_t * bytes_in_cache )
+{
+ rc_t rc = 0;
+ if ( percent != NULL ) { ( * percent ) = 0.0; }
+ if ( bytes_in_cache != NULL ) { ( * bytes_in_cache ) = 0; }
+ if ( path != NULL && path[ 0 ] != 0 )
+ {
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+ const KFile * f = NULL;
+ rc = KDirectoryOpenFileRead( dir, &f, "%s.cache", path );
+ if ( rc == 0 )
+ {
+ rc = GetCacheCompleteness( f, percent, bytes_in_cache );
+ }
+ else
+ {
+ rc = KDirectoryOpenFileRead( dir, &f, "%s", path );
+ if ( rc == 0 )
+ {
+ if ( percent != NULL ) ( * percent ) = 100.0;
+ if ( bytes_in_cache != NULL ) rc = KFileSize ( f, bytes_in_cache );
+ }
+ }
+ if ( f != NULL ) KFileRelease( f );
+ KDirectoryRelease( dir );
+ }
+ }
+ return rc;
+}
diff --git a/tools/vdb-dump/vdb-dump-helper.h b/tools/vdb-dump/vdb-dump-helper.h
index 72fe1cf..2337c81 100644
--- a/tools/vdb-dump/vdb-dump-helper.h
+++ b/tools/vdb-dump/vdb-dump-helper.h
@@ -62,7 +62,8 @@ rc_t vdh_show_manager_version( const VDBManager *my_manager );
rc_t vdh_parse_schema( const VDBManager *my_manager,
VSchema **new_schema,
- Vector *schema_list );
+ Vector *schema_list,
+ bool with_sra_schema );
bool vdh_is_path_table( const VDBManager *my_manager, const char *path,
Vector *schema_list );
@@ -82,6 +83,8 @@ rc_t vdh_print_col_info( dump_context *ctx,
const VSchema *my_schema );
rc_t resolve_accession( const char * accession, char * dst, size_t dst_size, bool remotely );
+rc_t resolve_cache( const char * accession, char * dst, size_t dst_size );
+rc_t check_cache_comleteness( const char * path, float * percent, uint64_t * bytes_in_cache );
#ifdef __cplusplus
}
diff --git a/tools/vdb-dump/vdb-dump-num-gen.c b/tools/vdb-dump/vdb-dump-num-gen.c
deleted file mode 100644
index 1b237e5..0000000
--- a/tools/vdb-dump/vdb-dump-num-gen.c
+++ /dev/null
@@ -1,477 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "vdb-dump-num-gen.h"
-
-#include <klib/vector.h>
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/out.h>
-#include <klib/text.h>
-#include <strtol.h>
-#include <sysalloc.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-
-static void CC vdn_node_destroy( void *item, void *data )
-{
- free( item );
-}
-
-static p_num_gen_node vdn_make_node( const uint64_t start, const uint64_t count )
-{
- p_num_gen_node p = ( p_num_gen_node )malloc( sizeof( num_gen_node ) );
- if ( p )
- {
- p->start = start;
- p->count = count;
- }
- return p;
-}
-
-static int CC vdn_insert_helper( const void* item1, const void* item2 )
-{
- int res = 0;
- p_num_gen_node node1 = (p_num_gen_node)item1;
- p_num_gen_node node2 = (p_num_gen_node)item2;
- if ( node1->start < node2->start )
- res = -1;
- else if ( node1->start > node2->start )
- res = 1;
- return res;
-}
-
-static bool vdn_add_node( num_gen* generator, const uint64_t num_1,
- const uint64_t num_2 )
-{
- rc_t res;
- p_num_gen_node node = NULL;
- if ( generator == NULL ) return false;
- if ( num_1 == num_2 )
- {
- node = vdn_make_node( num_1, 1 );
- }
- else if ( num_1 < num_2 )
- {
- node = vdn_make_node( num_1, ( num_2 - num_1 ) + 1 );
- }
- else
- {
- node = vdn_make_node( num_2, ( num_1 - num_2 ) + 1 );
- }
- if ( node == NULL ) return false;
- res = VectorInsert( &(generator->nodes), node, NULL, vdn_insert_helper );
- if ( res == 0 ) generator->node_count++;
- return ( res == 0 );
-}
-
-#define MAX_NUM_STR 12
-/* helper-structure for vdn_parse() */
-typedef struct num_gen_parse_ctx
-{
- size_t num_str_idx, num_count;
- uint64_t num[ 2 ];
- char num_str[ MAX_NUM_STR + 1 ];
-} num_gen_parse_ctx;
-typedef num_gen_parse_ctx* p_num_gen_parse_ctx;
-
-/* helper for vdn_parse() */
-static void vdb_convert_ctx( p_num_gen_parse_ctx ctx )
-{
- char *endp;
-
- ctx->num_str[ ctx->num_str_idx ] = 0;
- ctx->num[ 0 ] = strtou64( ctx->num_str, &endp, 10 );
- ctx->num_count = 1;
- ctx->num_str_idx = 0;
-}
-
-/* helper for vdn_parse() */
-static uint32_t vdb_convert_and_add_ctx( num_gen* generator, p_num_gen_parse_ctx ctx )
-{
- uint32_t res = 0;
- char *endp;
-
- if ( ctx->num_str_idx > 0 )
- {
- ctx->num_str[ ctx->num_str_idx ] = 0;
- ctx->num[ ctx->num_count ] = strtou64( ctx->num_str, &endp, 10 );
- ctx->num_str_idx = 0;
- if ( ctx->num_count == 0 )
- {
- ctx->num[1] = ctx->num[0];
- }
- if ( vdn_add_node( generator, ctx->num[0], ctx->num[1] ) )
- {
- res++;
- }
- ctx->num_count = 0;
- }
- return res;
-}
-
-uint32_t vdn_parse( num_gen* generator, const char* src )
-{
- size_t i, n;
- num_gen_parse_ctx ctx;
- uint32_t res = 0;
-
- if ( generator == NULL ) return res;
- if ( src == NULL ) return res;
- n = string_size( src );
- generator->node_count = 0;
- generator->curr_node = 0;
- generator->curr_node_sub_pos = 0;
- if ( n == 0 ) return res;
-
- ctx.num_str_idx = 0;
- ctx.num_count = 0;
-
- for ( i=0; i<n; ++i )
- {
- switch ( src[i] )
- {
- /* a dash switches from N1-mode into N2-mode */
- case '-' :
- vdb_convert_ctx( &ctx );
- break;
-
- /* a comma ends a single number or a range */
- case ',' :
- res += vdb_convert_and_add_ctx( generator, &ctx );
- break;
-
- /* in both mode add the char to the temp string */
- default:
- if ( ( src[i]>='0' )&&( src[i]<='9' )&&( ctx.num_str_idx < MAX_NUM_STR ) )
- {
- ctx.num_str[ ctx.num_str_idx++ ] = src[ i ];
- }
- break;
- }
- }
- res += vdb_convert_and_add_ctx( generator, &ctx );
- return res;
-}
-
-bool vdn_set_range( num_gen* generator,
- const int64_t first, const uint64_t count )
-{
- bool res = ( generator != NULL );
- if ( res )
- {
- uint64_t num_1 = first;
- uint64_t num_2;
-
- /* this is necessary because virtual columns which have a
- infinite row-range, get reported with first=1,count=0 */
- if ( count > 0 )
- {
- num_2 = first + count - 1;
- }
- else
- {
- num_2 = first;
- }
-
- /* clear all nodes so far... */
- VectorWhack( &(generator->nodes), vdn_node_destroy, NULL );
- /* re-init the vector */
- VectorInit( &(generator->nodes ), 0, 5 );
- generator->node_count = 0;
- generator->curr_node = 0;
- generator->curr_node_sub_pos = 0;
-
- res = vdn_add_node( generator, num_1, num_2 );
- }
- return res;
-}
-
-static bool CC vdn_check_range_start( p_num_gen_node the_node,
- const uint64_t range_start )
-{
- bool res = true;
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( the_node->start < range_start )
- {
- the_node->start = range_start;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- res = false;
- }
- }
- return res;
-}
-
-static void CC vdn_check_range_end( p_num_gen_node the_node,
- const uint64_t last_tab_row )
-{
- uint64_t last_node_row = ( the_node->start + the_node->count - 1 );
-
- if ( last_node_row > last_tab_row )
- {
- last_node_row = last_tab_row;
- if ( the_node->start <= last_node_row )
- {
- the_node->count = ( last_node_row - the_node->start ) + 1;
- }
- else
- {
- /* the node becomes invalid ... */
- the_node->start = 0;
- the_node->count = 0;
- }
- }
-}
-
-static void CC vdn_check_range_callback( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- p_num_gen_node the_range = ( p_num_gen_node )data;
- uint64_t last_tab_row = ( the_range->start + the_range->count - 1 );
-
- /* check if the start value is not out of range... */
- if ( vdn_check_range_start( the_node, the_range->start ) )
- vdn_check_range_end( the_node, last_tab_row );
-}
-
-static void CC vdn_count_invalid_nodes( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- uint32_t *invalid_count = ( uint32_t * )data;
-
- if ( ( the_node->start == 0 )&&( the_node->count == 0 ) )
- ( *invalid_count )++;
-}
-
-static void CC vdn_copy_valid_nodes( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- Vector *p_temp_nodes = ( Vector * )data;
-
- if ( ( the_node->start != 0 )&&( the_node->count != 0 ) )
- {
- p_num_gen_node new_node = vdn_make_node( the_node->start, the_node->count );
- if ( new_node )
- {
- VectorInsert( p_temp_nodes, new_node, NULL, vdn_insert_helper );
- }
- }
-}
-
-static void vdn_remove_invalid_nodes( num_gen* generator, const uint32_t invalid_nodes )
-{
- Vector temp_nodes;
-
- /* create a temp. vector */
- VectorInit( &temp_nodes, 0, 5 );
-
- /* copy all valid nodes into the temp. vector */
- VectorForEach ( &(generator->nodes), false,
- vdn_copy_valid_nodes, &temp_nodes );
-
- /* clear all nodes so far... */
- VectorWhack( &(generator->nodes), vdn_node_destroy, NULL );
-
- /* re-init the vector */
- VectorInit( &(generator->nodes ), 0, 5 );
-
- /* copy (swallow) the valid nodes back into the generator */
- VectorCopy ( &temp_nodes, &(generator->nodes) );
-
- /* correct the node count */
- generator->node_count -= invalid_nodes;
-
- /* destroy the temp-vector, DO NOT PASS vdn_node_destroy into it */
- VectorWhack ( &temp_nodes, NULL, NULL );
-}
-
-
-bool vdn_check_range( num_gen* generator,
- const int64_t first, const uint64_t count )
-{
- bool res = ( generator != NULL );
- if ( res )
- {
- num_gen_node defined_range;
- uint32_t invalid_nodes = 0;
-
- if ( count > 0 )
- {
- /* walk all nodes to check for boundaries... */
- defined_range.start = first;
- defined_range.count = count;
-
- VectorForEach ( &(generator->nodes), false,
- vdn_check_range_callback, &defined_range );
-
- VectorForEach ( &(generator->nodes), false,
- vdn_count_invalid_nodes, &invalid_nodes );
-
- if ( invalid_nodes > 0 )
- {
- vdn_remove_invalid_nodes( generator, invalid_nodes );
- }
- }
- }
- return res;
-}
-
-
-rc_t vdn_make( num_gen** generator )
-{
- if ( generator == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
- }
- (*generator) = calloc( 1, sizeof(num_gen) );
- if ( *generator == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- }
- VectorInit( &((*generator)->nodes ), 0, 5 );
- return 0;
-}
-
-
-rc_t vdn_destroy( num_gen* generator )
-{
- if ( generator == NULL )
- {
- return RC( rcVDB, rcNoTarg, rcDestroying, rcParam, rcNull );
- }
- VectorWhack( &(generator->nodes), vdn_node_destroy, NULL );
- free( generator );
- return 0;
-}
-
-
-bool vdn_start( num_gen* generator )
-{
- if ( generator == NULL ) return false;
- generator->curr_node = ( generator->node_count > 0 ? 0 : 1 );
- generator->curr_node_sub_pos = 0;
- return true;
-}
-
-
-static bool vdn_next_node( num_gen* generator, uint64_t* value )
-{
- bool res = false;
- if ( generator->curr_node < generator->node_count )
- {
- p_num_gen_node node = (p_num_gen_node)VectorGet( &(generator->nodes),
- (uint32_t)generator->curr_node );
- if ( node != NULL )
- {
- uint64_t ret_value = node->start;
- if ( node->count < 2 )
- {
- /* the node is a single-number-node */
- generator->curr_node++;
- }
- else
- {
- /* the node is a number range */
- ret_value += generator->curr_node_sub_pos++;
- if ( generator->curr_node_sub_pos >= node->count )
- {
- generator->curr_node++;
- generator->curr_node_sub_pos = 0;
- }
- }
- if ( value ) *value = ret_value;
- res = true;
- }
- }
- return res;
-}
-
-
-bool vdn_range_defined( num_gen* generator )
-{
- bool res = false;
- if ( generator != NULL )
- {
- res = ( generator->node_count > 0 );
- }
- return res;
-}
-
-
-bool vdn_next( num_gen* generator, uint64_t* value )
-{
- bool res = false;
- if ( generator != NULL )
- {
- uint64_t ret_value = 0;
- if ( vdn_range_defined( generator ) )
- {
- /* there are nodes (number-ranges) defined */
- res = vdn_next_node( generator, &ret_value );
- }
- else
- {
- /* endless mode, there are NO nodes (number-ranges) defined */
- ret_value = generator->curr_node++;
- res = true;
- }
- if ( value ) *value = ret_value;
- }
- return res;
-}
-
-
-static void CC vdn_detect_max( void *item, void *data )
-{
- p_num_gen_node the_node = ( p_num_gen_node )item;
- uint64_t * max_value = ( uint64_t * )data;
- uint64_t node_max = ( the_node->start + the_node->count ) - 1;
- if ( node_max > * max_value )
- {
- * max_value = node_max;
- }
-}
-
-
-uint64_t vdn_max( num_gen* generator )
-{
- uint64_t res = 0;
- if ( generator != NULL )
- VectorForEach ( &(generator->nodes), false, vdn_detect_max, &res );
- return res;
-}
\ No newline at end of file
diff --git a/tools/vdb-dump/vdb-dump-row-context.h b/tools/vdb-dump/vdb-dump-row-context.h
index 3f2579c..de282f1 100644
--- a/tools/vdb-dump/vdb-dump-row-context.h
+++ b/tools/vdb-dump/vdb-dump-row-context.h
@@ -57,7 +57,7 @@ typedef struct row_context
p_col_defs col_defs;
p_dump_context ctx;
dump_str s_col;
- uint64_t row_id;
+ int64_t row_id;
uint32_t col_nr;
rc_t rc;
} row_context;
diff --git a/tools/vdb-dump/vdb-dump-str.c b/tools/vdb-dump/vdb-dump-str.c
index 5031ebe..488cc7e 100644
--- a/tools/vdb-dump/vdb-dump-str.c
+++ b/tools/vdb-dump/vdb-dump-str.c
@@ -27,6 +27,12 @@
#include "vdb-dump-str.h"
#include <klib/text.h>
#include <klib/printf.h>
+#include <klib/out.h>
+#include <klib/namelist.h>
+
+#include <kfs/directory.h>
+#include <kfs/file.h>
+
#include <sysalloc.h>
#include <stdlib.h>
@@ -368,3 +374,157 @@ rc_t vds_2_csv( p_dump_str s )
}
return rc;
}
+
+
+/* ========================================================================================= */
+static rc_t vds_add_to_sections( const char * path, uint32_t start, uint32_t len, VNamelist * sections )
+{
+ String S;
+ StringInit( &S, &( path[ start ] ), len, len );
+ return VNamelistAppendString ( sections, &S );
+}
+
+rc_t vds_path_to_sections( const char * path, char delim, VNamelist ** sections )
+{
+ rc_t rc = 0;
+ if ( path == NULL || sections == NULL )
+ rc = RC( rcVDB, rcNoTarg, rcInserting, rcParam, rcNull );
+ else
+ {
+ rc = VNamelistMake ( sections, 5 );
+ if ( rc == 0 )
+ {
+ uint32_t idx = 0, start = 0, len = 0;
+ while( rc== 0 && path[ idx ] != 0 )
+ {
+ if ( path[ idx ] == delim )
+ {
+ if ( len > 0 )
+ rc = vds_add_to_sections( path, start, len, *sections );
+ start = idx + 1;
+ len = 0;
+ }
+ else
+ {
+ len++;
+ }
+ idx++;
+ }
+ if ( rc == 0 && len > 0 )
+ rc = vds_add_to_sections( path, start, len, *sections );
+ }
+ }
+ return rc;
+}
+
+/* ========================================================================================= */
+
+static void clear_recorded_errors( void )
+{
+ rc_t rc;
+ const char * filename;
+ const char * funcname;
+ uint32_t line_nr;
+ while ( GetUnreadRCInfo ( &rc, &filename, &funcname, &line_nr ) )
+ {
+ }
+}
+
+static rc_t vds_read_line( const KFile * f, uint64_t * pos, size_t * len, char * buffer, size_t buflen )
+{
+ size_t num_read;
+ rc_t rc = KFileRead( f, *pos, buffer, buflen, &num_read );
+ if ( rc == 0 && num_read > 0 )
+ {
+ char * nl = string_chr ( buffer, num_read, '\n' );
+ char * cr = string_chr ( buffer, num_read, '\r' );
+ if ( nl != NULL )
+ {
+ if ( cr != NULL )
+ {
+ if ( nl < cr )
+ *len = ( nl - buffer );
+ else
+ *len = ( cr - buffer );
+ }
+ else
+ *len = ( nl - buffer );
+ }
+ else if ( cr != NULL )
+ {
+ *len = ( cr - buffer );
+ }
+ else
+ *len = 0;
+ *pos += ( *len + 1 );
+ }
+ return rc;
+}
+
+static rc_t vds_print_diff( const char * buffer1, const char * buffer2, size_t len1, size_t len2 )
+{
+ rc_t rc;
+ String S1, S2;
+
+ StringInit( &S1, buffer1, len1, len1 );
+ StringInit( &S2, buffer2, len2, len2 );
+ rc = KOutMsg( "\n[A] %S\n", &S1 );
+ if ( rc == 0 )
+ rc = KOutMsg( "[B] %S\n", &S2 );
+ return rc;
+}
+
+static rc_t vds_diff_files( const KFile * f1, const KFile * f2 )
+{
+ rc_t rc;
+ char buffer1[ 2048 ];
+ char buffer2[ 2048 ];
+ uint64_t pos1 = 0;
+ uint64_t pos2 = 0;
+ size_t len1, len2;
+
+ do
+ {
+ len1 = len2 = 0;
+ rc = vds_read_line( f1, &pos1, &len1, buffer1, sizeof buffer1 );
+ if ( rc == 0 )
+ rc = vds_read_line( f2, &pos2, &len2, buffer2, sizeof buffer2 );
+ if ( rc == 0 && ( len1 > 0 || len2 > 0 ) )
+ rc = vds_print_diff( buffer1, buffer2, len1, len2 );
+
+ } while( rc == 0 && ( len1 > 0 || len2 > 0 ) );
+ return rc;
+}
+
+
+rc_t vds_diff( const char * f1, const char * f2 )
+{
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc != 0 )
+ KOutMsg( "KDirectoryNativeDir() failed\n" );
+ else
+ {
+ const KFile * kf1;
+ rc = KDirectoryOpenFileRead ( dir, &kf1, "%s", f1 );
+ if ( rc != 0 )
+ KOutMsg( "cannot open file '%s'\n", f1 );
+ else
+ {
+ const KFile * kf2;
+ rc = KDirectoryOpenFileRead ( dir, &kf2, "%s", f2 );
+ if ( rc != 0 )
+ KOutMsg( "cannot open file '%s'\n", f2 );
+ else
+ {
+ rc = vds_diff_files( kf1, kf2 );
+ KFileRelease( kf2 );
+ }
+ KFileRelease( kf1 );
+ }
+ KDirectoryRelease( dir );
+ }
+ clear_recorded_errors();
+
+ return rc;
+}
\ No newline at end of file
diff --git a/tools/vdb-dump/vdb-dump-str.h b/tools/vdb-dump/vdb-dump-str.h
index 71e2678..b157204 100644
--- a/tools/vdb-dump/vdb-dump-str.h
+++ b/tools/vdb-dump/vdb-dump-str.h
@@ -36,6 +36,7 @@ extern "C" {
#endif
#include <klib/rc.h>
+#include <klib/namelist.h>
typedef struct dump_str
{
@@ -88,6 +89,10 @@ rc_t vds_2_csv( p_dump_str s );
rc_t vds_enclose_string( p_dump_str s, const char c_left, const char c_right );
rc_t vds_escape( p_dump_str s, const char to_escape, const char escape_char );
+rc_t vds_path_to_sections( const char * path, char delim, VNamelist ** sections );
+
+rc_t vds_diff( const char * f1, const char * f2 );
+
#ifdef __cplusplus
}
#endif
diff --git a/tools/vdb-dump/vdb-dump-tools.c b/tools/vdb-dump/vdb-dump-tools.c
index 197b076..bbcee5c 100644
--- a/tools/vdb-dump/vdb-dump-tools.c
+++ b/tools/vdb-dump/vdb-dump-tools.c
@@ -468,7 +468,7 @@ my_col_def [IN] ... the definition of the column to be dumped
dumps one data-element (or a vector of it)
*************************************************************************************/
-rc_t vdt_dump_element( const p_dump_src src, const p_col_def def )
+rc_t vdt_dump_element( const p_dump_src src, const p_col_def def, bool bracket )
{
int dimension, selection;
rc_t rc = 0;
@@ -501,7 +501,8 @@ rc_t vdt_dump_element( const p_dump_src src, const p_col_def def )
if ( paren )
{
- rc = vds_append_str( &(def->content), "[" );
+ rc = vds_append_str( &(def->content), bracket ? "[" : "{" );
+
DISP_RC( rc, "dump_str_append_str() failed" )
}
@@ -517,7 +518,7 @@ rc_t vdt_dump_element( const p_dump_src src, const p_col_def def )
if ( paren && ( rc == 0 ) )
{
- rc = vds_append_str( &(def->content), "]" );
+ rc = vds_append_str( &(def->content), bracket ? "]" : "}" );
DISP_RC( rc, "dump_str_append_str() failed" )
}
}
diff --git a/tools/vdb-dump/vdb-dump-tools.h b/tools/vdb-dump/vdb-dump-tools.h
index ac0834b..d8b7ac7 100644
--- a/tools/vdb-dump/vdb-dump-tools.h
+++ b/tools/vdb-dump/vdb-dump-tools.h
@@ -51,7 +51,7 @@ typedef struct dump_src
} dump_src;
typedef dump_src* p_dump_src;
-rc_t vdt_dump_element( const p_dump_src src, const p_col_def def );
+rc_t vdt_dump_element( const p_dump_src src, const p_col_def def, bool bracket );
#ifdef __cplusplus
}
diff --git a/tools/vdb-dump/vdb-dump.c b/tools/vdb-dump/vdb-dump.c
index 37ed36b..597f190 100644
--- a/tools/vdb-dump/vdb-dump.c
+++ b/tools/vdb-dump/vdb-dump.c
@@ -38,12 +38,14 @@
#include <kdb/column.h>
#include <kdb/manager.h>
#include <kdb/namelist.h>
+#include <kdb/meta.h>
#include <kfs/directory.h>
#include <kns/manager.h>
#include <kapp/main.h>
#include <kapp/args.h>
+#include <kapp/args-conv.h>
#include <klib/container.h>
#include <klib/vector.h>
@@ -56,6 +58,7 @@
#include <klib/rc.h>
#include <klib/namelist.h>
#include <klib/time.h>
+#include <klib/num-gen.h>
#include <os-native.h>
#include <sysalloc.h>
@@ -63,7 +66,6 @@
#include <stdlib.h>
#include <bitstr.h>
-#include "vdb-dump-num-gen.h"
#include "vdb-dump-context.h"
#include "vdb-dump-coldefs.h"
#include "vdb-dump-tools.h"
@@ -91,10 +93,9 @@ static const char * dna_bases_usage[] = { "print dna-bases", NULL };
static const char * max_line_len_usage[] = { "limits line length", NULL };
static const char * line_indent_usage[] = { "indents the line", NULL };
static const char * filter_usage[] = { "filters lines", NULL };
-static const char * format_usage[] = { "dump format (csv,xml,json,piped,tab,fastq,fasta,bin)", NULL };
+static const char * format_usage[] = { "output format:", NULL };
static const char * id_range_usage[] = { "prints id-range", NULL };
static const char * without_sra_usage[] = { "without sra-type-translation", NULL };
-static const char * without_accession_usage[] = { "without accession-test", NULL };
static const char * excluded_columns_usage[] = { "exclude these columns", NULL };
static const char * boolean_usage[] = { "defines how boolean's are printed (1,T)", NULL };
static const char * objver_usage[] = { "request vdb-version", NULL };
@@ -103,6 +104,7 @@ static const char * numelem_usage[] = { "print only element-count", NULL };
static const char * numelemsum_usage[] = { "sum element-count", NULL };
static const char * show_blobbing_usage[] = { "show blobbing", NULL };
static const char * enum_phys_usage[] = { "enumerate physical columns", NULL };
+static const char * enum_readable_usage[] = { "enumerate readable columns", NULL };
static const char * objtype_usage[] = { "report type of object", NULL };
static const char * idx_enum_usage[] = { "enumerate all available index", NULL };
static const char * idx_range_usage[] = { "enumerate values and row-ranges of one index", NULL };
@@ -114,6 +116,11 @@ static const char * bzip2_usage[] = { "compress output using bzip2", NULL };
static const char * outbuf_size_usage[] = { "size of output-buffer, 0...none", NULL };
static const char * disable_mt_usage[] = { "disable multithreading", NULL };
static const char * info_usage[] = { "print info about run", NULL };
+static const char * spotgroup_usage[] = { "show spotgroups", NULL };
+/*static const char * sraschema_usage[] = { "force use of dflt. sra-schema", NULL }; */
+static const char * merge_ranges_usage[] = { "merge and sort row-ranges", NULL };
+static const char * spread_usage[] = { "show spread of integer values", NULL };
+static const char * slice_usage[] = { "find a slice of given depth", NULL };
OptDef DumpOptions[] =
{
@@ -137,13 +144,13 @@ OptDef DumpOptions[] =
{ OPTION_FORMAT, ALIAS_FORMAT, NULL, format_usage, 1, true, false },
{ OPTION_ID_RANGE, ALIAS_ID_RANGE, NULL, id_range_usage, 1, false, false },
{ OPTION_WITHOUT_SRA, ALIAS_WITHOUT_SRA, NULL, without_sra_usage, 1, false, false },
- { OPTION_WITHOUT_ACCESSION, ALIAS_WITHOUT_ACCESSION, NULL, without_accession_usage, 1, false, false },
{ OPTION_EXCLUDED_COLUMNS, ALIAS_EXCLUDED_COLUMNS, NULL, excluded_columns_usage, 1, true, false },
{ OPTION_BOOLEAN, ALIAS_BOOLEAN, NULL, boolean_usage, 1, true, false },
{ OPTION_NUMELEM, ALIAS_NUMELEM, NULL, numelem_usage, 1, false, false },
{ OPTION_NUMELEMSUM, ALIAS_NUMELEMSUM, NULL, numelemsum_usage, 1, false, false },
{ OPTION_SHOW_BLOBBING, NULL, NULL, show_blobbing_usage, 1, false, false },
{ OPTION_ENUM_PHYS, NULL, NULL, enum_phys_usage, 1, false, false },
+ { OPTION_ENUM_READABLE, NULL, NULL, enum_readable_usage, 1, false, false },
{ OPTION_OBJVER, ALIAS_OBJVER, NULL, objver_usage, 1, false, false },
{ OPTION_OBJTS, NULL, NULL, objts_usage, 1, false, false },
{ OPTION_OBJTYPE, ALIAS_OBJTYPE, NULL, objtype_usage, 1, false, false },
@@ -157,7 +164,13 @@ OptDef DumpOptions[] =
{ OPTION_BZIP2, NULL, NULL, bzip2_usage, 1, false, false },
{ OPTION_OUT_BUF_SIZE, NULL, NULL, outbuf_size_usage, 1, true, false },
{ OPTION_NO_MULTITHREAD, NULL, NULL, disable_mt_usage, 1, false, false },
- { OPTION_INFO, NULL, NULL, info_usage, 1, false, false }
+ { OPTION_INFO, NULL, NULL, info_usage, 1, false, false },
+ { OPTION_DIFF, NULL, NULL, NULL, 1, false, false },
+ { OPTION_SPOTGROUPS, NULL, NULL, spotgroup_usage, 1, false, false },
+ /*{ OPTION_SRASCHEMA, NULL, NULL, sraschema_usage, 1, false, false }, */
+ { OPTION_MERGE_RANGES, NULL, NULL, merge_ranges_usage, 1, false, false },
+ { OPTION_SPREAD, NULL, NULL, spread_usage, 1, false, false },
+ { OPTION_SLICE, NULL, NULL, slice_usage, 1, true, false }
};
const char UsageDefaultName[] = "vdb-dump";
@@ -206,9 +219,20 @@ rc_t CC Usage ( const Args * args )
HelpOptionLine ( ALIAS_MAX_LINE_LEN, OPTION_MAX_LINE_LEN, "max_length", max_line_len_usage );
HelpOptionLine ( ALIAS_LINE_INDENT, OPTION_LINE_INDENT, "indent_width", line_indent_usage );
HelpOptionLine ( ALIAS_FORMAT, OPTION_FORMAT, "format", format_usage );
+
+ KOutMsg( " csv ..... comma separated values on one line\n" );
+ KOutMsg( " xml ..... xml-style without complete xml-frame\n" );
+ KOutMsg( " json .... json-style\n" );
+ KOutMsg( " piped ... 1 line per cell: row-id, column-name: value\n" );
+ KOutMsg( " tab ..... 1 line per row: tab-separated values only\n" );
+ KOutMsg( " fastq ... FASTQ( 4 lines ) for each row\n" );
+ KOutMsg( " fastq1 .. FASTQ( 4 lines ) for each fragment\n" );
+ KOutMsg( " fasta ... FASTA( 2 lines ) for each fragment if possible\n" );
+ KOutMsg( " fasta1 .. one FASTA-record for the whole accession (REFSEQ)\n" );
+ KOutMsg( " fasta2 .. one FASTA-record for each REFERENCE in cSRA\n\n" );
+
HelpOptionLine ( ALIAS_ID_RANGE, OPTION_ID_RANGE, NULL, id_range_usage );
HelpOptionLine ( ALIAS_WITHOUT_SRA, OPTION_WITHOUT_SRA, NULL, without_sra_usage );
- HelpOptionLine ( ALIAS_WITHOUT_ACCESSION, OPTION_WITHOUT_ACCESSION, NULL, without_accession_usage );
HelpOptionLine ( ALIAS_EXCLUDED_COLUMNS, OPTION_EXCLUDED_COLUMNS, NULL, excluded_columns_usage );
HelpOptionLine ( ALIAS_BOOLEAN, OPTION_BOOLEAN, NULL, boolean_usage );
HelpOptionLine ( ALIAS_OBJVER, OPTION_OBJVER, NULL, objver_usage );
@@ -218,6 +242,7 @@ rc_t CC Usage ( const Args * args )
HelpOptionLine ( ALIAS_NUMELEMSUM, OPTION_NUMELEMSUM, NULL, numelemsum_usage );
HelpOptionLine ( NULL, OPTION_SHOW_BLOBBING, NULL, show_blobbing_usage );
HelpOptionLine ( NULL, OPTION_ENUM_PHYS, NULL, enum_phys_usage );
+ HelpOptionLine ( NULL, OPTION_ENUM_READABLE, NULL, enum_readable_usage );
HelpOptionLine ( NULL, OPTION_IDX_ENUM, NULL, idx_enum_usage );
HelpOptionLine ( NULL, OPTION_IDX_RANGE, NULL, idx_range_usage );
HelpOptionLine ( NULL, OPTION_CUR_CACHE, NULL, cur_cache_usage );
@@ -228,7 +253,11 @@ rc_t CC Usage ( const Args * args )
HelpOptionLine ( NULL, OPTION_OUT_BUF_SIZE, NULL, outbuf_size_usage );
HelpOptionLine ( NULL, OPTION_NO_MULTITHREAD, NULL, disable_mt_usage );
HelpOptionLine ( NULL, OPTION_INFO, NULL, info_usage );
-
+ HelpOptionLine ( NULL, OPTION_SPOTGROUPS, NULL, spotgroup_usage );
+ /* HelpOptionLine ( NULL, OPTION_SRASCHEMA, NULL, sraschema_usage ); */
+ HelpOptionLine ( NULL, OPTION_MERGE_RANGES, NULL, merge_ranges_usage );
+ HelpOptionLine ( NULL, OPTION_SPREAD, NULL, spread_usage );
+
HelpOptionsStandard ();
HelpVersion ( fullpath, KAppVersion() );
@@ -302,6 +331,7 @@ static void CC vdm_read_cell_data( void *item, void *data )
else
{
bool print_comma = true;
+ bool sra_dump_format;
/* initialize the element-idx ( for dimension > 1 ) */
src.element_idx = 0;
@@ -310,6 +340,9 @@ static void CC vdm_read_cell_data( void *item, void *data )
src.in_hex = r_ctx->ctx->print_in_hex;
src.without_sra_types = r_ctx->ctx->without_sra_types;
+ /* special treatment to suppress spaces between values */
+ sra_dump_format = ( r_ctx->ctx->format == df_sra_dump );
+
/* hardcoded printing of dna-bases if the column-type fits */
src.print_dna_bases = ( r_ctx->ctx->print_dna_bases &
( my_col_def->type_desc.intrinsic_dim == 2 ) &
@@ -344,7 +377,10 @@ static void CC vdm_read_cell_data( void *item, void *data )
uint32_t eidx = src.element_idx;
if ( ( eidx > 0 )&& ( src.print_dna_bases == false ) && print_comma )
{
- vds_append_str( &(my_col_def->content), ", " );
+ if ( sra_dump_format )
+ vds_append_str( &(my_col_def->content), "," );
+ else
+ vds_append_str( &(my_col_def->content), ", " );
}
/* dumps the basic data-types, implementation in vdb-dump-tools.c
@@ -354,7 +390,7 @@ static void CC vdm_read_cell_data( void *item, void *data )
dump_element is also responsible for incrementing
the src.element_idx by: 1...bool/int/uint/float
n...string/unicode-string */
- r_ctx->rc = vdt_dump_element( &src, my_col_def );
+ r_ctx->rc = vdt_dump_element( &src, my_col_def, !sra_dump_format );
/* insurance against endless loop */
if ( eidx == src.element_idx )
@@ -411,68 +447,76 @@ static rc_t vdm_dump_rows( p_row_context r_ctx )
/* the important row_id is a member of r_ctx ! */
r_ctx->rc = vds_make( &(r_ctx->s_col), r_ctx->ctx->max_line_len, 512 );
if ( r_ctx->rc != 0 )
- vdm_row_error( "dump_str_make( row#$(row_nr) ) failed",
- r_ctx->rc, r_ctx->row_id );
- if ( r_ctx->rc == 0 )
+ vdm_row_error( "dump_str_make( row#$(row_nr) ) failed", r_ctx->rc, r_ctx->row_id );
+ else
{
- vdn_start( r_ctx->ctx->row_generator );
- while ( ( vdn_next( r_ctx->ctx->row_generator, &(r_ctx->row_id )) )&&
- ( r_ctx->rc == 0 ) )
+ const struct num_gen_iter * iter;
+
+ r_ctx->rc = num_gen_iterator_make( r_ctx->ctx->rows, &iter );
+ if ( r_ctx->rc != 0 )
+ vdm_row_error( "num_gen_iterator_make( row#$(row_nr) ) failed", r_ctx->rc, r_ctx->row_id );
+ else
{
- r_ctx-> rc = Quitting();
- if ( r_ctx->rc != 0 )
- break;
- r_ctx->rc = VCursorSetRowId( r_ctx->cursor, r_ctx->row_id );
- if ( r_ctx->rc != 0 )
+ while ( ( r_ctx->rc == 0 ) && num_gen_iterator_next( iter, &(r_ctx->row_id), &(r_ctx->rc) ) )
{
- vdm_row_error( "VCursorSetRowId( row#$(row_nr) ) failed",
- r_ctx->rc, r_ctx->row_id );
- }
- else
- {
- r_ctx->rc = VCursorOpenRow( r_ctx->cursor );
+ if ( r_ctx-> rc == 0 )
+ r_ctx-> rc = Quitting();
+ if ( r_ctx->rc != 0 )
+ break;
+ r_ctx->rc = VCursorSetRowId( r_ctx->cursor, r_ctx->row_id );
if ( r_ctx->rc != 0 )
{
- vdm_row_error( "VCursorOpenRow( row#$(row_nr) ) failed",
+ vdm_row_error( "VCursorSetRowId( row#$(row_nr) ) failed",
r_ctx->rc, r_ctx->row_id );
}
else
{
- /* first reset the string and valid-flag for every column */
- vdcd_reset_content( r_ctx->col_defs );
+ r_ctx->rc = VCursorOpenRow( r_ctx->cursor );
+ if ( r_ctx->rc != 0 )
+ {
+ vdm_row_error( "VCursorOpenRow( row#$(row_nr) ) failed",
+ r_ctx->rc, r_ctx->row_id );
+ }
+ else
+ {
+ /* first reset the string and valid-flag for every column */
+ vdcd_reset_content( r_ctx->col_defs );
- /* read the data of every column and create a string for it */
- VectorForEach( &(r_ctx->col_defs->cols),
- false, vdm_read_cell_data, r_ctx );
+ /* read the data of every column and create a string for it */
+ VectorForEach( &(r_ctx->col_defs->cols),
+ false, vdm_read_cell_data, r_ctx );
- if ( r_ctx->rc == 0 )
- {
- /* prints the collected strings, in vdb-dump-formats.c */
- if ( !r_ctx->ctx->sum_num_elem )
+ if ( r_ctx->rc == 0 )
{
- r_ctx->rc = vdfo_print_row( r_ctx );
- if ( r_ctx->rc != 0 )
- vdm_row_error( "vdfo_print_row( row#$(row_nr) ) failed",
- r_ctx->rc, r_ctx->row_id );
+ /* prints the collected strings, in vdb-dump-formats.c */
+ if ( !r_ctx->ctx->sum_num_elem )
+ {
+ r_ctx->rc = vdfo_print_row( r_ctx );
+ if ( r_ctx->rc != 0 )
+ vdm_row_error( "vdfo_print_row( row#$(row_nr) ) failed",
+ r_ctx->rc, r_ctx->row_id );
+ }
}
+ r_ctx->rc = VCursorCloseRow( r_ctx->cursor );
+ if ( r_ctx->rc != 0 )
+ vdm_row_error( "VCursorCloseRow( row#$(row_nr) ) failed",
+ r_ctx->rc, r_ctx->row_id );
}
- r_ctx->rc = VCursorCloseRow( r_ctx->cursor );
- if ( r_ctx->rc != 0 )
- vdm_row_error( "VCursorCloseRow( row#$(row_nr) ) failed",
- r_ctx->rc, r_ctx->row_id );
}
}
}
+ num_gen_iterator_destroy( iter );
+
if ( r_ctx->rc == 0 && r_ctx->ctx->sum_num_elem )
{
- VectorForEach( &(r_ctx->col_defs->cols),
- false, vdm_print_elem_sum, r_ctx );
+ VectorForEach( &( r_ctx->col_defs->cols ), false, vdm_print_elem_sum, r_ctx );
if ( r_ctx->rc == 0 )
{
r_ctx->rc = vdfo_print_row( r_ctx );
DISP_RC( r_ctx->rc, "VTableOpenSchema() failed" );
}
}
+
vds_free( &(r_ctx->s_col) );
}
return r_ctx->rc;
@@ -515,7 +559,7 @@ static bool vdm_extract_or_parse_phys_columns( const p_dump_context ctx,
res = vdcd_extract_from_phys_table( my_col_defs, my_table );
else
/* the user knows the names of the wanted columns... */
- res = vdcd_parse_string( my_col_defs, ctx->columns, NULL );
+ res = vdcd_parse_string( my_col_defs, ctx->columns, my_table );
if ( ctx->excluded_columns != NULL )
vdcd_exclude_these_columns( my_col_defs, ctx->excluded_columns );
@@ -596,24 +640,37 @@ static rc_t vdm_dump_opened_table( const p_dump_context ctx, const VTable *my_ta
DISP_RC( rc, "VCursorIdRange() failed" );
if ( rc == 0 )
{
- if ( vdn_range_defined( ctx->row_generator ) == false )
+ if ( ctx->rows == NULL )
{
/* if the user did not specify a row-range, take all rows */
- vdn_set_range( ctx->row_generator, first, count );
+ rc = num_gen_make_from_range( &ctx->rows, first, count );
+ DISP_RC( rc, "num_gen_make_from_range() failed" );
}
else
{
/* if the user did specify a row-range, check the boundaries */
- vdn_check_range( ctx->row_generator, first, count );
+ if ( count > 0 )
+ {
+ /* trim only if the row-range is not zero, otherwise
+ we will not get data if the user specified only static columns
+ because they report a row-range of zero! */
+ rc = num_gen_trim( ctx->rows, first, count );
+ DISP_RC( rc, "num_gen_trim() failed" );
+ }
}
- if ( vdn_range_defined( ctx->row_generator ) )
+ if ( rc == 0 )
{
- r_ctx.ctx = ctx;
- rc = vdm_dump_rows( &r_ctx ); /* <--- */
+ if ( num_gen_empty( ctx->rows ) )
+ {
+ rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
+ }
+ else
+ {
+ r_ctx.ctx = ctx;
+ rc = vdm_dump_rows( &r_ctx ); /* <--- */
+ }
}
- else
- rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
}
}
}
@@ -636,14 +693,117 @@ static rc_t vdm_dump_opened_table( const p_dump_context ctx, const VTable *my_ta
ctx [IN] ... contains path, tablename, columns, row-range etc.
my_database [IN] ... open database needed for vdb-calls
*************************************************************************************/
+static rc_t vdm_walk_sections( const VDatabase * base_db, const VDatabase ** sub_db,
+ const VNamelist * sections, uint32_t count )
+{
+ rc_t rc = 0;
+ const VDatabase * parent_db = base_db;
+ if ( count == 0 )
+ {
+ rc = VDatabaseAddRef ( parent_db );
+ DISP_RC( rc, "VDatabaseAddRef() failed" );
+ }
+ else
+ {
+ uint32_t idx;
+ for ( idx = 0; rc == 0 && idx < count; ++idx )
+ {
+ const char * dbname;
+ rc = VNameListGet ( sections, idx, &dbname );
+ DISP_RC( rc, "VNameListGet() failed" );
+ if ( rc == 0 )
+ {
+ const VDatabase * temp;
+ rc = VDatabaseOpenDBRead ( parent_db, &temp, "%s", dbname );
+ DISP_RC( rc, "VDatabaseOpenDBRead() failed" );
+ if ( rc == 0 && idx > 0 )
+ {
+ rc = VDatabaseRelease ( parent_db );
+ DISP_RC( rc, "VDatabaseRelease() failed" );
+ }
+ if ( rc == 0 )
+ parent_db = temp;
+ }
+ }
+ }
+
+ if ( rc == 0 ) *sub_db = parent_db;
+ return rc;
+}
+
+
+static void vdm_clear_recorded_errors( void )
+{
+ rc_t rc;
+ const char * filename;
+ const char * funcname;
+ uint32_t line_nr;
+ while ( GetUnreadRCInfo ( &rc, &filename, &funcname, &line_nr ) )
+ {
+ }
+}
+
+
+static rc_t vdm_check_table_empty( const VTable * tab )
+{
+ bool empty;
+ rc_t rc = VTableIsEmpty( tab, &empty );
+ DISP_RC( rc, "VTableIsEmpty() failed" );
+ if ( rc == 0 && empty )
+ {
+ vdm_clear_recorded_errors();
+ KOutMsg( "the requested table is empty!\n" );
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcTable, rcEmpty );
+ }
+ return rc;
+}
+
+static rc_t vdm_open_table_by_path( const VDatabase * db, const char * path, const VTable ** tab )
+{
+ VNamelist * sections;
+ rc_t rc = vds_path_to_sections( path, '.', §ions );
+ DISP_RC( rc, "vds_path_to_sections() failed" );
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = VNameListCount ( sections, &count );
+ DISP_RC( rc, "VNameListCount() failed" );
+ if ( rc == 0 && count > 0 )
+ {
+ const VDatabase * sub_db;
+ rc = vdm_walk_sections( db, &sub_db, sections, count - 1 );
+ if ( rc == 0 )
+ {
+ const char * tabname;
+ rc = VNameListGet ( sections, count - 1, &tabname );
+ DISP_RC( rc, "VNameListGet() failed" );
+ if ( rc == 0 )
+ {
+ rc = VDatabaseOpenTableRead( sub_db, tab, "%s", tabname );
+ DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ if ( rc == 0 )
+ {
+ rc = vdm_check_table_empty( *tab );
+ if ( rc != 0 )
+ {
+ VTableRelease( *tab );
+ tab = NULL;
+ }
+ }
+ }
+ VDatabaseRelease ( sub_db );
+ }
+ }
+ VNamelistRelease ( sections );
+ }
+ return rc;
+}
+
static rc_t vdm_dump_opened_database( const p_dump_context ctx,
const VDatabase *my_database )
{
- rc_t rc;
const VTable *my_table;
-
- rc = VDatabaseOpenTableRead( my_database, &my_table, "%s", ctx->table );
- DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
if ( rc == 0 )
{
rc = vdm_dump_opened_table( ctx, my_table );
@@ -652,6 +812,90 @@ static rc_t vdm_dump_opened_database( const p_dump_context ctx,
return rc;
}
+/* ********************************************************************** */
+
+static rc_t vdm_show_tab_spread( const p_dump_context ctx,
+ const VTable *my_table )
+{
+ const VCursor * cursor;
+ rc_t rc = VTableCreateCachedCursorRead( my_table, &cursor, ctx->cur_cache_size );
+ DISP_RC( rc, "VTableCreateCursorRead() failed" );
+ if ( rc == 0 )
+ {
+ col_defs * cols;
+ if ( !vdcd_init( &cols, ctx->max_line_len ) )
+ {
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
+ DISP_RC( rc, "col_defs_init() failed" );
+ }
+ if ( rc == 0 )
+ {
+ uint32_t n = vdm_extract_or_parse_columns( ctx, my_table, cols );
+ if ( n < 1 )
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
+ else
+ {
+ n = vdcd_add_to_cursor( cols, cursor );
+ if ( n < 1 )
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
+ else
+ {
+ rc = VCursorOpen( cursor );
+ DISP_RC( rc, "VCursorOpen() failed" );
+ if ( rc == 0 )
+ {
+ int64_t first;
+ uint64_t count;
+ rc = VCursorIdRange( cursor, 0, &first, &count );
+ DISP_RC( rc, "VCursorIdRange( spread ) failed" );
+ if ( rc == 0 )
+ {
+ if ( ctx->rows == NULL )
+ {
+ rc = num_gen_make_from_range( &ctx->rows, first, count );
+ DISP_RC( rc, "num_gen_make_from_range() failed" );
+ }
+ else
+ {
+ if ( count > 0 )
+ {
+ rc = num_gen_trim( ctx->rows, first, count );
+ DISP_RC( rc, "num_gen_trim() failed" );
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ if ( num_gen_empty( ctx->rows ) )
+ rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
+ else
+ rc = vdcd_collect_spread( ctx->rows, cols, cursor ); /* is in vdb-dump-coldefs.c */
+ }
+ }
+ }
+ }
+ }
+ vdcd_destroy( cols );
+ }
+ VCursorRelease( cursor );
+ }
+ return rc;
+}
+
+static rc_t vdm_show_db_spread( const p_dump_context ctx,
+ const VDatabase *my_database )
+{
+ const VTable *my_table;
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
+ if ( rc == 0 )
+ {
+ rc = vdm_show_tab_spread( ctx, my_table );
+ VTableRelease( my_table );
+ }
+ return rc;
+}
+/* ********************************************************************** */
+
/********************************************************************
helper function, needed by "VSchemaDump()"
********************************************************************/
@@ -680,9 +924,24 @@ static rc_t vdm_dump_tab_schema( const p_dump_context ctx,
DISP_RC( rc, "VTableOpenSchema() failed" );
if ( rc == 0 )
{
- const char * decl = ctx->columns;
- rc = VSchemaDump( my_schema, sdmPrint, decl,
- vdm_schema_dump_flush, stdout );
+ if ( ctx->columns == NULL )
+ {
+ /* the user did not ask to inspect a specific object, we look for
+ the Typespec of the table... */
+ char buffer[ 4096 ];
+ rc = VTableTypespec ( my_table, buffer, sizeof buffer );
+ DISP_RC( rc, "VTableTypespec() failed" );
+ if ( rc == 0 )
+ rc = VSchemaDump( my_schema, sdmPrint, buffer,
+ vdm_schema_dump_flush, stdout );
+
+ }
+ else
+ {
+ /* the user did ask to inspect a specific object */
+ rc = VSchemaDump( my_schema, sdmPrint, ctx->columns,
+ vdm_schema_dump_flush, stdout );
+ }
DISP_RC( rc, "VSchemaDump() failed" );
VSchemaRelease( my_schema );
}
@@ -702,14 +961,47 @@ my_database [IN] ... open database needed for vdb-calls
static rc_t vdm_dump_db_schema( const p_dump_context ctx,
const VDatabase *my_database )
{
- const VTable *my_table;
- rc_t rc = VDatabaseOpenTableRead( my_database, &my_table, "%s", ctx->table );
- DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
- if ( rc == 0 )
- {
- rc = vdm_dump_tab_schema( ctx, my_table );
- VTableRelease( my_table );
- }
+ rc_t rc = 0;
+ if ( ctx->table_defined )
+ {
+ /* the user has given a database as object, but asks to inspect a given table */
+ const VTable *my_table;
+ rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
+ if ( rc == 0 )
+ {
+ rc = vdm_dump_tab_schema( ctx, my_table );
+ VTableRelease( my_table );
+ }
+ }
+ else
+ {
+ /* the user has given a database as object, but did not ask for a specific table */
+ const VSchema * my_schema;
+ rc = VDatabaseOpenSchema( my_database, &my_schema );
+ DISP_RC( rc, "VDatabaseOpenSchema() failed" );
+ if ( rc == 0 )
+ {
+ if ( ctx->columns == NULL )
+ {
+ /* the used did not ask to inspect a specifiy object, we look for
+ the Typespec of the database... */
+ char buffer[ 4096 ];
+ rc = VDatabaseTypespec ( my_database, buffer, sizeof buffer );
+ DISP_RC( rc, "VDatabaseTypespec() failed" );
+ if ( rc == 0 )
+ rc = VSchemaDump( my_schema, sdmPrint, buffer,
+ vdm_schema_dump_flush, stdout );
+ }
+ else
+ {
+ /* the user did ask to inspect a specific object */
+ rc = VSchemaDump( my_schema, sdmPrint, ctx->columns,
+ vdm_schema_dump_flush, stdout );
+ }
+ DISP_RC( rc, "VSchemaDump() failed" );
+ VSchemaRelease( my_schema );
+ }
+ }
return rc;
}
@@ -722,38 +1014,91 @@ static rc_t vdm_dump_db_schema( const p_dump_context ctx,
ctx [IN] ... contains path, tablename, columns, row-range etc.
my_database [IN] ... open database needed for vdb-calls
*************************************************************************************/
-static rc_t vdm_enum_tables( const p_dump_context ctx,
- const VDatabase *my_database )
+static rc_t vdm_enum_sub_dbs_and_tabs( const VDatabase * db, uint32_t indent );
+
+static rc_t vdm_report_tab_or_db( const VDatabase * db, const KNamelist * list,
+ const char * prefix, uint32_t indent )
{
- rc_t rc = KOutMsg( "enumerating the tables of database '%s'\n", ctx->path );
+ uint32_t n;
+ rc_t rc = KNamelistCount( list, &n );
+ DISP_RC( rc, "KNamelistCount() failed" );
if ( rc == 0 )
{
- KNamelist *tbl_names;
- rc = VDatabaseListTbl( my_database, &tbl_names );
- DISP_RC( rc, "VDatabaseListTbl() failed" );
- if ( rc == 0 )
+ uint32_t i;
+ for ( i = 0; i < n && rc == 0; ++i )
{
- uint32_t n;
- rc = KNamelistCount( tbl_names, &n );
- DISP_RC( rc, "KNamelistCount() failed" );
+ const char * entry;
+ rc = KNamelistGet( list, i, &entry );
+ DISP_RC( rc, "KNamelistGet() failed" );
if ( rc == 0 )
{
- uint32_t i;
- for ( i = 0; i < n && rc == 0; ++i )
+ rc = KOutMsg( "%*s : %s\n", indent + 3, prefix, entry );
+ if ( rc == 0 && db != NULL )
{
- const char *tbl_name;
- rc = KNamelistGet( tbl_names, i, &tbl_name );
- DISP_RC( rc, "KNamelistGet() failed" );
+ const VDatabase * sub_db;
+ rc = VDatabaseOpenDBRead ( db, &sub_db, entry );
+ DISP_RC( rc, "VDatabaseOpenDBRead() failed" );
if ( rc == 0 )
- rc = KOutMsg( "tbl #%u: %s\n", i+1, tbl_name );
+ {
+ rc = vdm_enum_sub_dbs_and_tabs( sub_db, indent + 3 ); /* recursion here... */
+ VDatabaseRelease( sub_db );
+ }
}
}
- KNamelistRelease( tbl_names );
}
}
return rc;
}
+static rc_t vdm_enum_tabs_of_db( const VDatabase * db, uint32_t indent )
+{
+ KNamelist *tbl_names;
+ rc_t rc = VDatabaseListTbl( db, &tbl_names );
+ if ( rc == 0 )
+ {
+ rc = vdm_report_tab_or_db( NULL, tbl_names, "tbl", indent );
+ KNamelistRelease( tbl_names );
+ }
+ else
+ {
+ rc = 0;
+ }
+ return rc;
+}
+
+static rc_t vdm_enum_sub_dbs_of_db( const VDatabase * db, uint32_t indent )
+{
+ KNamelist *db_names;
+ rc_t rc = VDatabaseListDB( db, &db_names );
+ if ( rc == 0 )
+ {
+ rc = vdm_report_tab_or_db( db, db_names, "db ", indent );
+ KNamelistRelease( db_names );
+ }
+ else
+ {
+ rc = 0;
+ }
+ return rc;
+}
+
+static rc_t vdm_enum_sub_dbs_and_tabs( const VDatabase * db, uint32_t indent )
+{
+ rc_t rc = vdm_enum_sub_dbs_of_db( db, indent );
+ if ( rc == 0 )
+ rc = vdm_enum_tabs_of_db( db, indent );
+ return rc;
+}
+
+static rc_t vdm_enum_tables( const p_dump_context ctx, const VDatabase * db )
+{
+ rc_t rc = KOutMsg( "enumerating the tables of database '%s'\n", ctx->path );
+ if ( rc == 0 )
+ rc = vdm_enum_sub_dbs_and_tabs( db, 0 );
+ return rc;
+}
+
+
typedef struct col_info_context
{
p_dump_context ctx;
@@ -820,7 +1165,7 @@ static rc_t vdm_show_row_blobbing( const p_col_def col_def,
DISP_RC( rc, "KColumnIdRange() failed" );
if ( rc == 0 )
{
- int64_t last = first + count;
+ int64_t last = first + count - 1;
rc = KOutMsg( "range: %,ld ... %,ld\n", first, last );
if ( rc == 0 )
{
@@ -845,7 +1190,7 @@ static rc_t vdm_show_row_blobbing( const p_col_def col_def,
DISP_RC( rc, "KColumnBlobRead() failed" );
if ( rc == 0 )
{
- rc = KOutMsg( "blob[ %,ld ... %,ld] size = %,zu\n", first_id_in_blob, last_id_in_blob, remaining );
+ rc = KOutMsg( "blob[ %,ld ... %,ld] size = %,zu\n", first_id_in_blob, last_id_in_blob, remaining + num_read );
}
id = last_id_in_blob + 1;
}
@@ -925,6 +1270,45 @@ static rc_t vdm_enum_phys_columns( const VTable *my_table )
return rc;
}
+
+static rc_t vdm_enum_readable_columns( const VTable *my_table )
+{
+ rc_t rc = KOutMsg( "readable columns:\n" );
+ if ( rc == 0 )
+ {
+ KNamelist *readable;
+ rc = VTableListReadableColumns( my_table, &readable );
+ DISP_RC( rc, "VTableListReadableColumns() failed" );
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = KNamelistCount( readable, &count );
+ DISP_RC( rc, "KNamelistCount( readable columns ) failed" );
+ if ( rc == 0 )
+ {
+ if ( count > 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * name;
+ rc = KNamelistGet( readable, idx, &name );
+ DISP_RC( rc, "KNamelistGet( readable columns ) failed" );
+ if ( rc == 0 )
+ rc = KOutMsg( "[%.02d] = %s\n", idx, name );
+ }
+ }
+ else
+ {
+ rc = KOutMsg( "... list is empty!\n" );
+ }
+ }
+ KNamelistRelease( readable );
+ }
+ }
+ return rc;
+}
+
/*************************************************************************************
enum_tab_columns:
* called by "enum_db_columns()" and "dump_table()" as fkt-pointer
@@ -944,6 +1328,10 @@ static rc_t vdm_enum_tab_columns( const p_dump_context ctx, const VTable *my_tab
{
rc = vdm_enum_phys_columns( my_table );
}
+ else if ( ctx->enum_readable )
+ {
+ rc = vdm_enum_readable_columns( my_table );
+ }
else
{
col_defs *my_col_defs;
@@ -1017,11 +1405,8 @@ my_database [IN] ... open database needed for vdb-calls
*************************************************************************************/
static rc_t vdm_enum_db_columns( const p_dump_context ctx, const VDatabase *my_database )
{
- rc_t rc;
const VTable *my_table;
-
- rc = VDatabaseOpenTableRead( my_database, &my_table, "%s", ctx->table );
- DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
if ( rc == 0 )
{
rc = vdm_enum_tab_columns( ctx, my_table );
@@ -1060,8 +1445,13 @@ static rc_t vdm_print_tab_id_range( const p_dump_context ctx, const VTable *my_t
uint64_t count;
uint32_t idx = 0;
- vdcd_get_first_none_static_column_idx( my_col_defs, my_cursor, &idx );
-
+
+ /* calling with idx = 0 means: let the cursor find out the min/max values of
+ all open columns...
+
+ vdcd_get_first_none_static_column_idx( my_col_defs, my_cursor, &idx );
+ */
+
rc = VCursorIdRange( my_cursor, idx, &first, &count );
DISP_RC( rc, "VCursorIdRange() failed" );
if ( rc == 0 )
@@ -1088,11 +1478,8 @@ my_database [IN] ... open database needed for vdb-calls
*************************************************************************************/
static rc_t vdm_print_db_id_range( const p_dump_context ctx, const VDatabase *my_database )
{
- rc_t rc;
const VTable *my_table;
-
- rc = VDatabaseOpenTableRead( my_database, &my_table, "%s", ctx->table );
- DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
if ( rc == 0 )
{
rc = vdm_print_tab_id_range( ctx, my_table );
@@ -1192,11 +1579,8 @@ static rc_t vdm_enum_tab_index( const p_dump_context ctx, const VTable *my_table
static rc_t vdm_enum_db_index( const p_dump_context ctx, const VDatabase *my_database )
{
- rc_t rc;
const VTable *my_table;
-
- rc = VDatabaseOpenTableRead( my_database, &my_table, "%s", ctx->table );
- DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
if ( rc == 0 )
{
rc = vdm_enum_tab_index( ctx, my_table );
@@ -1244,11 +1628,8 @@ static rc_t vdm_range_tab_index( const p_dump_context ctx, const VTable *my_tabl
static rc_t vdm_range_db_index( const p_dump_context ctx, const VDatabase *my_database )
{
- rc_t rc;
const VTable *my_table;
-
- rc = VDatabaseOpenTableRead( my_database, &my_table, "%s", ctx->table );
- DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
if ( rc == 0 )
{
rc = vdm_range_tab_index( ctx, my_table );
@@ -1258,6 +1639,89 @@ static rc_t vdm_range_db_index( const p_dump_context ctx, const VDatabase *my_da
}
+/* ************************************************************************************ */
+
+static rc_t vdm_show_tab_spotgroups( const p_dump_context ctx, const VTable *my_table )
+{
+ const KMetadata * meta = NULL;
+ rc_t rc = VTableOpenMetadataRead( my_table, &meta );
+ DISP_RC( rc, "VTableOpenMetadataRead() failed" );
+ if ( rc == 0 )
+ {
+ const KMDataNode * spot_groups_node;
+ rc = KMetadataOpenNodeRead( meta, &spot_groups_node, "STATS/SPOT_GROUP" );
+ DISP_RC( rc, "KMetadataOpenNodeRead( STATS/SPOT_GROUP ) failed" );
+ if ( rc == 0 )
+ {
+ KNamelist * spot_groups;
+ rc = KMDataNodeListChildren( spot_groups_node, &spot_groups );
+ DISP_RC( rc, "KMDataNodeListChildren() failed" );
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = KNamelistCount( spot_groups, &count );
+ if ( rc == 0 && count > 0 )
+ {
+ uint32_t i;
+ for ( i = 0; i < count && rc == 0; ++i )
+ {
+ const char * name = NULL;
+ rc = KNamelistGet( spot_groups, i, &name );
+ if ( rc == 0 && name != NULL )
+ {
+ const KMDataNode * spot_count_node;
+ rc = KMDataNodeOpenNodeRead( spot_groups_node, &spot_count_node, "%s/SPOT_COUNT", name );
+ DISP_RC( rc, "KMDataNodeOpenNodeRead() failed" );
+ if ( rc == 0 )
+ {
+ uint64_t spot_count = 0;
+ rc = KMDataNodeReadAsU64( spot_count_node, &spot_count );
+ if ( rc == 0 )
+ {
+ if ( spot_count > 0 )
+ {
+ const KMDataNode * spot_group_node;
+ rc = KMDataNodeOpenNodeRead( spot_groups_node, &spot_group_node, name );
+ if ( rc == 0 )
+ {
+ char name_attr[ 2048 ];
+ size_t num_writ;
+ rc = KMDataNodeReadAttr( spot_group_node, "name", name_attr, sizeof name_attr, &num_writ );
+ rc = KOutMsg( "%s\t%,lu\n", rc == 0 ? name_attr : name, spot_count );
+ KMDataNodeRelease( spot_group_node );
+ }
+
+ }
+ }
+ else
+ vdm_clear_recorded_errors();
+
+ KMDataNodeRelease( spot_count_node );
+ }
+ }
+ }
+ }
+ KNamelistRelease( spot_groups );
+ }
+ KMDataNodeRelease( spot_groups_node );
+ }
+ KMetadataRelease ( meta );
+ }
+ return rc;
+}
+
+static rc_t vdm_show_db_spotgroups( const p_dump_context ctx, const VDatabase *my_database )
+{
+ const VTable *my_table;
+ rc_t rc = vdm_open_table_by_path( my_database, ctx->table, &my_table );
+ if ( rc == 0 )
+ {
+ rc = vdm_show_tab_spotgroups( ctx, my_table );
+ VTableRelease( my_table );
+ }
+ return rc;
+}
+
typedef rc_t (*db_tab_t)( const p_dump_context ctx, const VTable *a_tab );
@@ -1284,13 +1748,15 @@ static rc_t vdm_dump_tab_fkt( const p_dump_context ctx,
VSchema *my_schema = NULL;
rc_t rc;
- vdh_parse_schema( my_manager, &my_schema, &(ctx->schema_list) );
+ vdh_parse_schema( my_manager, &my_schema, &(ctx->schema_list), true /*ctx->force_sra_schema*/ );
rc = VDBManagerOpenTableRead( my_manager, &my_table, my_schema, "%s", ctx->path );
DISP_RC( rc, "VDBManagerOpenTableRead() failed" );
if ( rc == 0 )
{
- rc = tab_fkt( ctx, my_table ); /* fkt-pointer is called */
+ rc = vdm_check_table_empty( my_table );
+ if ( rc == 0 )
+ rc = tab_fkt( ctx, my_table ); /* fkt-pointer is called */
VTableRelease( my_table );
}
@@ -1304,7 +1770,7 @@ static rc_t vdm_dump_tab_fkt( const p_dump_context ctx,
static bool enum_col_request( const p_dump_context ctx )
{
return ( ctx->column_enum_requested || ctx->column_enum_short ||
- ctx->show_blobbing || ctx->enum_phys );
+ ctx->show_blobbing || ctx->enum_phys || ctx->enum_readable );
}
/***************************************************************************
@@ -1332,7 +1798,8 @@ static rc_t vdm_dump_table( const p_dump_context ctx, const VDBManager *my_manag
else if ( ctx->table_enum_requested )
{
KOutMsg( "cannot enum tables of a table-object\n" );
- rc = RC ( rcApp, rcArgv, rcAccessing, rcParam, rcInvalid );
+ vdm_clear_recorded_errors();
+ rc = 0;
}
else if ( enum_col_request( ctx ) )
{
@@ -1350,6 +1817,14 @@ static rc_t vdm_dump_table( const p_dump_context ctx, const VDBManager *my_manag
{
rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_range_tab_index );
}
+ else if ( ctx->show_spotgroups )
+ {
+ rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_show_tab_spotgroups );
+ }
+ else if ( ctx->show_spread )
+ {
+ rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_show_tab_spread );
+ }
else
{
rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_dump_opened_table );
@@ -1388,7 +1863,7 @@ static rc_t vdm_dump_db_fkt( const p_dump_context ctx,
VSchema *my_schema = NULL;
rc_t rc;
- vdh_parse_schema( my_manager, &my_schema, &(ctx->schema_list) );
+ vdh_parse_schema( my_manager, &my_schema, &(ctx->schema_list), true /* ctx->force_sra_schema */ );
rc = VDBManagerOpenDBRead( my_manager, &my_database, my_schema, "%s", ctx->path );
DISP_RC( rc, "VDBManagerOpenDBRead() failed" );
@@ -1401,7 +1876,7 @@ static rc_t vdm_dump_db_fkt( const p_dump_context ctx,
if ( !table_defined )
table_defined = vdh_take_1st_table_from_db( ctx, my_database );
}
- if ( table_defined )
+ if ( table_defined || ctx->table_enum_requested )
{
rc = db_fkt( ctx, my_database ); /* fkt-pointer is called */
}
@@ -1459,6 +1934,14 @@ static rc_t vdm_dump_database( const p_dump_context ctx, const VDBManager *my_ma
{
rc = vdm_dump_db_fkt( ctx, my_manager, vdm_range_db_index );
}
+ else if ( ctx->show_spotgroups )
+ {
+ rc = vdm_dump_db_fkt( ctx, my_manager, vdm_show_db_spotgroups );
+ }
+ else if ( ctx->show_spread )
+ {
+ rc = vdm_dump_db_fkt( ctx, my_manager, vdm_show_db_spread );
+ }
else
{
rc = vdm_dump_db_fkt( ctx, my_manager, vdm_dump_opened_database );
@@ -1535,14 +2018,10 @@ static rc_t vdb_main_one_obj_by_pathtype( const p_dump_context ctx,
/* types defined in <kdb/manager.h> */
switch ( path_type )
{
- case kptDatabase : rc = vdm_dump_database( ctx, mgr );
- DISP_RC( rc, "dump_database() failed" );
- break;
+ case kptDatabase : rc = vdm_dump_database( ctx, mgr ); break;
case kptPrereleaseTbl:
- case kptTable : rc = vdm_dump_table( ctx, mgr );
- DISP_RC( rc, "dump_table() failed" );
- break;
+ case kptTable : rc = vdm_dump_table( ctx, mgr ); break;
default : rc = RC( rcVDB, rcNoTarg, rcConstructing, rcItem, rcNotFound );
PLOGERR( klogInt, ( klogInt, rc,
@@ -1565,12 +2044,10 @@ static rc_t vdb_main_one_obj_by_probing( const p_dump_context ctx,
if ( vdh_is_path_database( mgr, ctx->path, &(ctx->schema_list) ) )
{
rc = vdm_dump_database( ctx, mgr );
- DISP_RC( rc, "dump_database() failed" );
}
else if ( vdh_is_path_table( mgr, ctx->path, &(ctx->schema_list) ) )
{
rc = vdm_dump_table( ctx, mgr );
- DISP_RC( rc, "dump_table() failed" );
}
else
{
@@ -1682,19 +2159,22 @@ static rc_t vdm_main( const p_dump_context ctx, Args * args )
for ( idx = 0; idx < count && rc == 0; ++idx )
{
const char *value = NULL;
- rc = ArgsParamValue( args, idx, &value );
+ rc = ArgsParamValue( args, idx, (const void **)&value );
DISP_RC( rc, "ArgsParamValue() failed" );
if ( rc == 0 )
{
if ( ctx->print_info )
{
rc = vdb_info( &(ctx->schema_list), ctx->format, mgr,
- value, ctx->row_generator ); /* in vdb_info.c */
+ value, ctx->rows ); /* in vdb_info.c */
}
else switch( ctx->format )
{
- case df_fastq : ;
- case df_fasta : vdf_main( ctx, mgr, value ); break;
+ case df_fastq : ;
+ case df_fastq1 : ;
+ case df_fasta : ;
+ case df_fasta1 : ;
+ case df_fasta2 : vdf_main( ctx, mgr, value ); break;
default : rc = vdm_main_one_obj( ctx, mgr, value ); break;
}
}
@@ -1702,7 +2182,8 @@ static rc_t vdm_main( const p_dump_context ctx, Args * args )
}
else
{
- rc = UsageSummary ( UsageDefaultName );
+ UsageSummary ( UsageDefaultName );
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInsufficient);
}
}
}
@@ -1716,6 +2197,34 @@ static rc_t vdm_main( const p_dump_context ctx, Args * args )
}
+static rc_t diff_files( Args * args )
+{
+ uint32_t count;
+ rc_t rc = ArgsParamCount( args, &count );
+ DISP_RC( rc, "ArgsParamCount() failed" );
+ if ( rc == 0 )
+ {
+ if ( count != 2 )
+ KOutMsg( "this function needs exactly 2 files to diff\n" );
+ else
+ {
+ const char * f1;
+ rc = ArgsParamValue( args, 0, (const void **)&f1 );
+ DISP_RC( rc, "ArgsParamValue( 0 ) failed" );
+ if ( rc == 0 )
+ {
+ const char * f2;
+ rc = ArgsParamValue( args, 1, (const void **)&f2 );
+ DISP_RC( rc, "ArgsParamValue( 1 ) failed" );
+ if ( rc == 0 )
+ rc = vds_diff( f1, f2 ); /* in vdb-dump-str.c */
+ }
+ }
+ }
+ return rc;
+}
+
+
/***************************************************************************
Main:
* create the dump-context
@@ -1741,9 +2250,11 @@ rc_t CC KMain ( int argc, char *argv [] )
rc = KOutHandlerSet ( write_to_FILE, stdout );
if ( rc == 0 )
- rc = ArgsMakeAndHandle (&args, argc, argv, 1,
- DumpOptions, sizeof DumpOptions / sizeof DumpOptions [ 0 ] );
- if (rc == 0)
+ {
+ rc = ArgsMakeAndHandle (&args, argc, argv,
+ 1, DumpOptions, sizeof DumpOptions / sizeof DumpOptions [ 0 ] );
+ }
+ if ( rc == 0 )
{
dump_context *ctx;
@@ -1765,9 +2276,14 @@ rc_t CC KMain ( int argc, char *argv [] )
if ( rc == 0 )
{
if ( ctx->phase > 0 )
- rc = vdi_bin_phase( ctx, args );
- else
+ rc = vdi_bin_phase( ctx, args ); /* vdb-dump-bin.c */
+ else if ( ctx->diff )
+ rc = diff_files( args ); /* above calls into vdb-dump-str.c */
+ else if ( ctx->slice_depth > 0 )
+ rc = find_slice( ctx, args ); /* vdb-dump-str.c */
+ else
rc = vdm_main( ctx, args );
+
release_out_redir( &redir ); /* vdb-dump-redir.c */
}
}
diff --git a/tools/vdb-dump/vdb-dump.vers b/tools/vdb-dump/vdb-dump.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-dump/vdb-dump.vers
+++ b/tools/vdb-dump/vdb-dump.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-dump/vdb-dump.vers.h b/tools/vdb-dump/vdb-dump.vers.h
index a0148a1..ce77f40 100644
--- a/tools/vdb-dump/vdb-dump.vers.h
+++ b/tools/vdb-dump/vdb-dump.vers.h
@@ -1 +1 @@
-#define VDB_DUMP_VERS 0x02040002
+#define VDB_DUMP_VERS 0x02050007
diff --git a/tools/vdb-dump/vdb_info.c b/tools/vdb-dump/vdb_info.c
index 27bb19e..252f583 100644
--- a/tools/vdb-dump/vdb_info.c
+++ b/tools/vdb-dump/vdb_info.c
@@ -28,10 +28,15 @@
#include <klib/out.h>
#include <klib/time.h>
#include <klib/printf.h>
+#include <klib/num-gen.h>
+#include <klib/text.h>
#include <kfs/directory.h>
#include <kfs/file.h>
+#include <kns/manager.h>
+#include <kns/http.h>
+
#include <kdb/manager.h>
#include <kdb/meta.h>
@@ -80,6 +85,16 @@ typedef struct vdb_info_event
vdb_info_date run_date;
} vdb_info_event;
+typedef struct vdb_info_bam_hdr
+{
+ bool present;
+ size_t hdr_bytes;
+ uint32_t total_lines;
+ uint32_t HD_lines;
+ uint32_t SQ_lines;
+ uint32_t RG_lines;
+ uint32_t PG_lines;
+} vdb_info_bam_hdr;
typedef struct vdb_info_data
{
@@ -88,14 +103,21 @@ typedef struct vdb_info_data
const char * s_platform;
char path[ 1024 ];
+ char cache[ 1024 ];
char schema_name[ 1024 ];
-
+ char species[ 1024 ];
+
vdb_info_event formatter;
vdb_info_event loader;
vdb_info_event update;
vdb_info_date ts;
-
+
+ vdb_info_bam_hdr bam_hdr;
+
+ float cache_percent;
+ uint64_t bytes_in_cache;
+
uint64_t seq_rows;
uint64_t ref_rows;
uint64_t prim_rows;
@@ -333,35 +355,103 @@ static void split_date( vdb_info_date * d )
}
/* ----------------------------------------------------------------------------- */
+static bool has_col( const VTable * tab, const char * colname )
+{
+ bool res = false;
+ struct KNamelist * columns;
+ rc_t rc = VTableListReadableColumns( tab, &columns );
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = KNamelistCount( columns, &count );
+ if ( rc == 0 && count > 0 )
+ {
+ uint32_t idx;
+ size_t colname_size = string_size( colname );
+ for ( idx = 0; idx < count && rc == 0 && !res; ++idx )
+ {
+ const char * a_name;
+ rc = KNamelistGet ( columns, idx, &a_name );
+ if ( rc == 0 )
+ {
+ int cmp;
+ size_t a_name_size = string_size( a_name );
+ uint32_t max_chars = ( uint32_t )colname_size;
+ if ( a_name_size > max_chars ) max_chars = ( uint32_t )a_name_size;
+ cmp = strcase_cmp ( colname, colname_size,
+ a_name, a_name_size,
+ max_chars );
+ res = ( cmp == 0 );
+ }
+ }
+ }
+ KNamelistRelease( columns );
+ }
+ return res;
+}
static const char * get_platform( const VTable * tab )
{
const char * res = PT_NONE;
- const VCursor * cur;
- rc_t rc = VTableCreateCursorRead( tab, &cur );
- if ( rc == 0 )
- {
- uint32_t idx;
- rc = VCursorAddColumn( cur, &idx, "PLATFORM" );
- if ( rc == 0 )
- {
- rc = VCursorOpen( cur );
- if ( rc == 0 )
- {
- const uint8_t * pf;
- rc = VCursorCellDataDirect( cur, 1, idx, NULL, (const void**)&pf, NULL, NULL );
- if ( rc == 0 )
- {
- res = vdcd_get_platform_txt( *pf );
- }
- }
- }
- VCursorRelease( cur );
- }
+ if ( has_col( tab, "PLATFORM" ) )
+ {
+ const VCursor * cur;
+ rc_t rc = VTableCreateCursorRead( tab, &cur );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ rc = VCursorAddColumn( cur, &idx, "PLATFORM" );
+ if ( rc == 0 )
+ {
+ rc = VCursorOpen( cur );
+ if ( rc == 0 )
+ {
+ const uint8_t * pf;
+ rc = VCursorCellDataDirect( cur, 1, idx, NULL, (const void**)&pf, NULL, NULL );
+ if ( rc == 0 )
+ {
+ res = vdcd_get_platform_txt( *pf );
+ }
+ }
+ }
+ VCursorRelease( cur );
+ }
+ }
return res;
}
+static void get_string_cell( char * buffer, size_t buffer_size, const VTable * tab, int64_t row, const char * column )
+{
+ if ( has_col( tab, column ) )
+ {
+ const VCursor * cur;
+ rc_t rc = VTableCreateCursorRead( tab, &cur );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ rc = VCursorAddColumn( cur, &idx, column );
+ if ( rc == 0 )
+ {
+ rc = VCursorOpen( cur );
+ if ( rc == 0 )
+ {
+ const char * src;
+ uint32_t row_len;
+ rc = VCursorCellDataDirect( cur, row, idx, NULL, (const void**)&src, NULL, &row_len );
+ if ( rc == 0 )
+ {
+ size_t num_writ;
+ string_printf( buffer, buffer_size, &num_writ, "%.*s", row_len, src );
+ }
+ }
+ }
+ VCursorRelease( cur );
+ }
+ }
+}
+
+
static uint64_t get_rowcount( const VTable * tab )
{
uint64_t res = 0;
@@ -433,6 +523,96 @@ static void get_meta_event( const KMetadata * meta, const char * node_path, vdb_
}
}
+static size_t get_node_size( const KMDataNode * node )
+{
+ char buffer[ 10 ];
+ size_t num_read, remaining, res = 0;
+ rc_t rc = KMDataNodeRead( node, 0, buffer, sizeof( buffer ), &num_read, &remaining );
+ if ( rc == 0 ) res = num_read + remaining;
+ return res;
+}
+
+static bool is_newline( const char c ) { return ( c == 0x0A || c == 0x0D ); }
+
+static void inspect_line( vdb_info_bam_hdr * bam_hdr, char * line, size_t len )
+{
+ bam_hdr->total_lines++;
+ if ( len > 3 && line[ 0 ] == '@' )
+ {
+ switch( line[ 1 ] )
+ {
+ case 'H' : if ( line[ 2 ] == 'D' ) bam_hdr->HD_lines++; break;
+ case 'S' : if ( line[ 2 ] == 'Q' ) bam_hdr->SQ_lines++; break;
+ case 'R' : if ( line[ 2 ] == 'G' ) bam_hdr->RG_lines++; break;
+ case 'P' : if ( line[ 2 ] == 'G' ) bam_hdr->PG_lines++; break;
+ }
+ }
+}
+
+static void parse_buffer( vdb_info_bam_hdr * bam_hdr, char * buffer, size_t len )
+{
+ char * line;
+ size_t idx, line_len, state = 0;
+ for ( idx = 0; idx < len; ++idx )
+ {
+ switch( state )
+ {
+ case 0 : if ( is_newline( buffer[ idx ] ) ) /* init */
+ state = 2;
+ else
+ {
+ line = &( buffer[ idx ] );
+ line_len = 1;
+ state = 1;
+ }
+ break;
+
+ case 1 : if ( is_newline( buffer[ idx ] ) ) /* content */
+ {
+ inspect_line( bam_hdr, line, line_len );
+ state = 2;
+ }
+ else
+ line_len++;
+ break;
+
+ case 2 : if ( !is_newline( buffer[ idx ] ) ) /* newline */
+ {
+ line = &( buffer[ idx ] );
+ line_len = 1;
+ state = 1;
+ }
+ break;
+ }
+ }
+}
+
+static void get_meta_bam_hdr( vdb_info_bam_hdr * bam_hdr, const KMetadata * meta )
+{
+ const KMDataNode * node;
+ rc_t rc = KMetadataOpenNodeRead ( meta, &node, "BAM_HEADER" );
+ bam_hdr -> present = ( rc == 0 );
+ if ( bam_hdr -> present )
+ {
+ bam_hdr->hdr_bytes = get_node_size( node );
+ if ( bam_hdr->hdr_bytes > 0 )
+ {
+ char * buffer = malloc( bam_hdr->hdr_bytes );
+ if ( buffer != NULL )
+ {
+ size_t num_read, remaining;
+ rc = KMDataNodeRead( node, 0, buffer, bam_hdr->hdr_bytes, &num_read, &remaining );
+ if ( rc == 0 )
+ {
+ parse_buffer( bam_hdr, buffer, bam_hdr->hdr_bytes );
+ }
+ free( buffer );
+ }
+ }
+ KMDataNodeRelease ( node );
+ }
+}
+
static void get_meta_info( vdb_info_data * data, const KMetadata * meta )
{
const KMDataNode * node;
@@ -466,6 +646,7 @@ static void get_meta_info( vdb_info_data * data, const KMetadata * meta )
get_meta_event( meta, "SOFTWARE/formatter", &data->formatter );
get_meta_event( meta, "SOFTWARE/loader", &data->loader );
get_meta_event( meta, "SOFTWARE/update", &data->update );
+ get_meta_bam_hdr( &data->bam_hdr, meta );
}
@@ -487,22 +668,44 @@ static const char * get_path_type( const VDBManager *mgr, const char * acc_or_pa
}
-static uint64_t get_file_size( const char * path )
+static rc_t make_remote_file( const KFile ** f, const char * url )
+{
+ KNSManager * kns_mgr;
+ rc_t rc = KNSManagerMake ( & kns_mgr );
+ *f = NULL;
+ if ( rc == 0 )
+ {
+ rc = KNSManagerMakeHttpFile ( kns_mgr, f, NULL, 0x01010000, "%s", url );
+ KNSManagerRelease ( kns_mgr );
+ }
+ return rc;
+}
+
+
+static rc_t make_local_file( const KFile ** f, const char * path )
+{
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ *f = NULL;
+ if ( rc == 0 )
+ {
+ rc = KDirectoryOpenFileRead( dir, f, "%s", path );
+ KDirectoryRelease( dir );
+ }
+ return rc;
+}
+
+
+static uint64_t get_file_size( const char * path, bool remotely )
{
uint64_t res = 0;
- KDirectory * dir;
- rc_t rc = KDirectoryNativeDir( &dir );
- if ( rc == 0 )
- {
- const KFile * f;
- rc = KDirectoryOpenFileRead( dir, &f, "%s", path );
- if ( rc == 0 )
- {
- KFileSize ( f, &res );
- KFileRelease( f );
- }
- KDirectoryRelease( dir );
- }
+ const KFile * f;
+ rc_t rc = ( remotely ) ? make_remote_file( &f, path ) : make_local_file( &f, path );
+ if ( rc == 0 )
+ {
+ KFileSize ( f, &res );
+ KFileRelease( f );
+ }
return res;
}
@@ -517,6 +720,7 @@ static rc_t vdb_info_tab( vdb_info_data * data, VSchema * schema, const VDBManag
data->s_platform = get_platform( tab );
data->seq_rows = get_rowcount( tab );
+ get_string_cell( data->species, sizeof data->species, tab, 1, "DEF_LINE" );
rc = VTableOpenMetadataRead ( tab, &meta );
if ( rc == 0 )
@@ -545,6 +749,29 @@ static uint64_t get_tab_row_count( const VDatabase * db, const char * table_name
}
+static void get_species( char * buffer, size_t buffer_size, const VDatabase * db, const VDBManager *mgr )
+{
+ const VTable * tab;
+ rc_t rc = VDatabaseOpenTableRead( db, &tab, "REFERENCE" );
+ if ( rc == 0 )
+ {
+ char seq_id[ 1024 ];
+
+ seq_id[ 0 ] = 0;
+ get_string_cell( seq_id, sizeof seq_id, tab, 1, "SEQ_ID" );
+ VTableRelease( tab );
+ if ( seq_id[ 0 ] != 0 )
+ {
+ rc = VDBManagerOpenTableRead( mgr, &tab, NULL, "%s", seq_id );
+ if ( rc == 0 )
+ {
+ get_string_cell( buffer, buffer_size, tab, 1, "DEF_LINE" );
+ VTableRelease( tab );
+ }
+ }
+ }
+}
+
static rc_t vdb_info_db( vdb_info_data * data, VSchema * schema, const VDBManager *mgr )
{
const VDatabase * db;
@@ -571,6 +798,9 @@ static rc_t vdb_info_db( vdb_info_data * data, VSchema * schema, const VDBManage
data->passes_rows = get_tab_row_count( db, "PASSES" );
data->metrics_rows = get_tab_row_count( db, "ZMW_METRICS" );
+ if ( data->ref_rows > 0 )
+ get_species( data->species, sizeof data->species, db, mgr );
+
rc = VDatabaseOpenMetadataRead ( db, &meta );
if ( rc == 0 )
{
@@ -677,6 +907,9 @@ static rc_t vdb_info_print_xml( vdb_info_data * data )
rc = KOutMsg( "<MINUTE>%.02d</MINUTE>\n", data->ts.minute );
}
+ if ( rc == 0 && data->species[ 0 ] != 0 )
+ rc = vdb_info_print_xml_s( "SPECIES", data->species );
+
if ( rc == 0 )
rc = vdb_info_print_xml_event( "FORMATTER", &data->formatter );
if ( rc == 0 )
@@ -781,6 +1014,9 @@ static rc_t vdb_info_print_json( vdb_info_data * data )
rc = KOutMsg( "\"MINUTE\":%d,\n", data->ts.minute );
}
+ if ( rc == 0 && data->species[ 0 ] != 0 )
+ rc = vdb_info_print_json_s( "SPECIES", data->species );
+
if ( rc == 0 )
rc = vdb_info_print_json_event( "FORMATTER", &data->formatter );
if ( rc == 0 )
@@ -797,22 +1033,27 @@ static rc_t vdb_info_print_json( vdb_info_data * data )
/* ----------------------------------------------------------------------------- */
+static const char dflt_event_name[] = "-";
static rc_t vdb_info_print_sep_event( vdb_info_event * event, const char sep, bool last )
{
rc_t rc;
+ const char * ev_name = event->name;
+ if ( ev_name == NULL || ev_name[ 0 ] == 0 )
+ ev_name = dflt_event_name;
+
if ( last )
{
- rc = KOutMsg( "%s%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d",
- event->name, sep,
+ rc = KOutMsg( "'%s'%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d",
+ ev_name, sep,
event->vers.major, sep, event->vers.minor, sep, event->vers.release, sep,
event->tool_date.month, sep, event->tool_date.day, sep, event->tool_date.year, sep,
event->run_date.month, sep, event->run_date.day, sep, event->run_date.year );
}
else
{
- rc = KOutMsg( "%s%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c",
- event->name, sep,
+ rc = KOutMsg( "'%s'%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c%d%c",
+ ev_name, sep,
event->vers.major, sep, event->vers.minor, sep, event->vers.release, sep,
event->tool_date.month, sep, event->tool_date.day, sep, event->tool_date.year, sep,
event->run_date.month, sep, event->run_date.day, sep, event->run_date.year, sep );
@@ -823,7 +1064,7 @@ static rc_t vdb_info_print_sep_event( vdb_info_event * event, const char sep, bo
static rc_t vdb_info_print_sep( vdb_info_data * data, const char sep )
{
- rc_t rc = KOutMsg( "%s%c%lu%c%c%c%s%c",
+ rc_t rc = KOutMsg( "'%s'%c%lu%c%c%c'%s'%c",
data->acc, sep, data->file_size, sep,
data->s_path_type[0], sep, &(data->s_platform[13]), sep );
if ( rc == 0 )
@@ -833,16 +1074,26 @@ static rc_t vdb_info_print_sep( vdb_info_data * data, const char sep )
data->ev_int_rows, sep, data->consensus_rows, sep,
data->passes_rows, sep, data->metrics_rows, sep );
if ( rc == 0 )
- rc = KOutMsg( "%s%c%d%c%d%c%d%c%d%c%d%c", data->schema_name, sep,
+ rc = KOutMsg( "'%s'%c%d%c%d%c%d%c%d%c%d%c",
+ data->schema_name, sep,
data->ts.month, sep, data->ts.day, sep, data->ts.year, sep,
data->ts.hour, sep, data->ts.minute, sep );
if ( rc == 0 )
+ {
+ if ( data->species[ 0 ] != 0 )
+ rc = KOutMsg( "'%s'%c", data->species, sep );
+ else
+ rc = KOutMsg( "-%c", sep );
+ }
+
+ if ( rc == 0 )
rc = vdb_info_print_sep_event( &data->formatter, sep, false );
if ( rc == 0 )
rc = vdb_info_print_sep_event( &data->loader, sep, false );
if ( rc == 0 )
rc = vdb_info_print_sep_event( &data->update, sep, true );
+
if ( rc == 0 )
rc = KOutMsg( "\n" );
@@ -882,30 +1133,52 @@ static rc_t vdb_info_print_dflt_event( vdb_info_event * event, const char *prefi
static rc_t vdb_info_print_dflt( vdb_info_data * data )
{
rc_t rc= KOutMsg( "acc : %s\n", data->acc );
+
if ( rc == 0 && data->path[ 0 ] != 0 )
rc = KOutMsg( "path : %s\n", data->path );
- if ( rc == 0 && data->file_size != 0 )
+
+ if ( rc == 0 && data->file_size != 0 )
rc = KOutMsg( "size : %,lu\n", data->file_size );
- if ( rc == 0 && data->s_path_type[ 0 ] != 0 )
+
+ if ( rc == 0 && data->cache[ 0 ] != 0 )
+ {
+ rc = KOutMsg( "cache : %s\n", data->cache );
+ if ( rc == 0 )
+ rc = KOutMsg( "percent: %f\n", data->cache_percent );
+ if ( rc == 0 )
+ rc = KOutMsg( "bytes : %,lu\n", data->bytes_in_cache );
+ }
+
+ if ( rc == 0 && data->s_path_type[ 0 ] != 0 )
rc = KOutMsg( "type : %s\n", data->s_path_type );
- if ( rc == 0 && data->s_platform[ 0 ] != 0 )
+
+ if ( rc == 0 && data->s_platform[ 0 ] != 0 )
rc = KOutMsg( "platf : %s\n", data->s_platform );
+
if ( rc == 0 && data->seq_rows != 0 )
rc = KOutMsg( "SEQ : %,lu\n", data->seq_rows );
+
if ( rc == 0 && data->ref_rows != 0 )
rc = KOutMsg( "REF : %,lu\n", data->ref_rows );
+
if ( rc == 0 && data->prim_rows != 0 )
rc = KOutMsg( "PRIM : %,lu\n", data->prim_rows );
+
if ( rc == 0 && data->sec_rows != 0 )
rc = KOutMsg( "SEC : %,lu\n", data->sec_rows );
+
if ( rc == 0 && data->ev_rows != 0 )
rc = KOutMsg( "EVID : %,lu\n", data->ev_rows );
+
if ( rc == 0 && data->ev_int_rows != 0 )
rc = KOutMsg( "EVINT : %,lu\n", data->ev_int_rows );
+
if ( rc == 0 && data->consensus_rows != 0 )
rc = KOutMsg( "CONS : %,lu\n", data->consensus_rows );
+
if ( rc == 0 && data->passes_rows != 0 )
rc = KOutMsg( "PASS : %,lu\n", data->passes_rows );
+
if ( rc == 0 && data->metrics_rows != 0 )
rc = KOutMsg( "METR : %,lu\n", data->metrics_rows );
@@ -917,6 +1190,9 @@ static rc_t vdb_info_print_dflt( vdb_info_data * data )
data->ts.timestamp, data->ts.month, data->ts.day, data->ts.year,
data->ts.hour, data->ts.minute );
+ if ( rc == 0 && data->species[ 0 ] != 0 )
+ rc = KOutMsg( "SPECIES: %s\n", data->species );
+
if ( rc == 0 )
vdb_info_print_dflt_event( &data->formatter, "FMT" );
if ( rc == 0 )
@@ -924,6 +1200,19 @@ static rc_t vdb_info_print_dflt( vdb_info_data * data )
if ( rc == 0 )
vdb_info_print_dflt_event( &data->update, "UPD" );
+ if ( rc == 0 && data->bam_hdr.present )
+ {
+ rc = KOutMsg( "BAMHDR : %d bytes / %d lines\n", data->bam_hdr.hdr_bytes, data->bam_hdr.total_lines );
+ if ( rc == 0 && data->bam_hdr.HD_lines > 0 )
+ rc = KOutMsg( "BAMHDR : %d HD-lines\n", data->bam_hdr.HD_lines );
+ if ( rc == 0 && data->bam_hdr.SQ_lines > 0 )
+ rc = KOutMsg( "BAMHDR : %d SQ-lines\n", data->bam_hdr.SQ_lines );
+ if ( rc == 0 && data->bam_hdr.RG_lines > 0 )
+ rc = KOutMsg( "BAMHDR : %d RG-lines\n", data->bam_hdr.RG_lines );
+ if ( rc == 0 && data->bam_hdr.PG_lines > 0 )
+ rc = KOutMsg( "BAMHDR : %d PG-lines\n", data->bam_hdr.PG_lines );
+ }
+
return rc;
}
@@ -1137,10 +1426,27 @@ static rc_t vdb_info_1( VSchema * schema, dump_format_t format, const VDBManager
case 'T' : vdb_info_tab( &data, schema, mgr ); break;
}
- rc1 = resolve_accession( acc_or_path, data.path, sizeof data.path, false );
+ /* try to resolve the path locally */
+ rc1 = resolve_accession( acc_or_path, data.path, sizeof data.path, false ); /* vdb-dump-helper.c */
if ( rc1 == 0 )
- data.file_size = get_file_size( data.path );
-
+ data.file_size = get_file_size( data.path, false );
+ else
+ {
+ /* try to resolve the path remotely */
+ rc1 = resolve_accession( acc_or_path, data.path, sizeof data.path, true ); /* vdb-dump-helper.c */
+ if ( rc1 == 0 )
+ {
+ data.file_size = get_file_size( data.path, true );
+ /* try to find out the cache-file */
+ rc1 = resolve_cache( acc_or_path, data.cache, sizeof data.cache ); /* vdb-dump-helper.c */
+ if ( rc1 == 0 )
+ {
+ /* try to find out cache completeness */
+ check_cache_comleteness( data.cache, &data.cache_percent, &data.bytes_in_cache );
+ }
+ }
+ }
+
switch ( format )
{
case df_xml : rc = vdb_info_print_xml( &data ); break;
@@ -1157,42 +1463,52 @@ static rc_t vdb_info_1( VSchema * schema, dump_format_t format, const VDBManager
rc_t vdb_info( Vector * schema_list, dump_format_t format, const VDBManager *mgr,
- const char * acc_or_path, num_gen * row_generator )
+ const char * acc_or_path, struct num_gen * rows )
{
rc_t rc = 0;
VSchema * schema = NULL;
- vdh_parse_schema( mgr, &schema, schema_list );
+ vdh_parse_schema( mgr, &schema, schema_list, false );
if ( format == df_sql )
rc = vdb_info_print_sql_header( acc_or_path );
- if ( row_generator != NULL && vdn_range_defined( row_generator ) )
+ if ( rows != NULL && !num_gen_empty( rows ) )
{
- uint64_t id;
- uint8_t digits = digits_of( vdn_max( row_generator ) );
-
- vdn_start( row_generator );
- while ( vdn_next( row_generator, &id ) && rc == 0 )
+ const struct num_gen_iter * iter;
+ rc = num_gen_iterator_make( rows, &iter );
+ if ( rc == 0 )
{
- char acc[ 64 ];
- size_t num_writ;
- rc_t rc1 = -1;
- switch ( digits )
+ int64_t max_row;
+ rc = num_gen_iterator_max( iter, &max_row );
+ if ( rc == 0 )
{
- case 1 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%lu", acc_or_path, id ); break;
- case 2 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.02lu", acc_or_path, id ); break;
- case 3 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.03lu", acc_or_path, id ); break;
- case 4 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.04lu", acc_or_path, id ); break;
- case 5 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.05lu", acc_or_path, id ); break;
- case 6 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.06lu", acc_or_path, id ); break;
- case 7 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.07lu", acc_or_path, id ); break;
- case 8 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.08lu", acc_or_path, id ); break;
- case 9 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.09lu", acc_or_path, id ); break;
- }
+ int64_t id;
+ uint8_t digits = digits_of( max_row );
+
+ while ( rc == 0 && num_gen_iterator_next( iter, &id, &rc ) )
+ {
+ char acc[ 64 ];
+ size_t num_writ;
+ rc_t rc1 = -1;
+ switch ( digits )
+ {
+ case 1 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%ld", acc_or_path, id ); break;
+ case 2 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.02ld", acc_or_path, id ); break;
+ case 3 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.03ld", acc_or_path, id ); break;
+ case 4 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.04ld", acc_or_path, id ); break;
+ case 5 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.05ld", acc_or_path, id ); break;
+ case 6 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.06ld", acc_or_path, id ); break;
+ case 7 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.07ld", acc_or_path, id ); break;
+ case 8 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.08ld", acc_or_path, id ); break;
+ case 9 : rc1 = string_printf ( acc, sizeof acc, &num_writ, "%s%.09ld", acc_or_path, id ); break;
+ }
- if ( rc1 == 0 )
- rc = vdb_info_1( schema, format, mgr, acc, acc_or_path );
+ if ( rc1 == 0 )
+ rc = vdb_info_1( schema, format, mgr, acc, acc_or_path );
+ }
+ }
+ num_gen_iterator_destroy( iter );
}
}
else
diff --git a/tools/vdb-dump/vdb_info.h b/tools/vdb-dump/vdb_info.h
index f0bde80..b83dfcd 100644
--- a/tools/vdb-dump/vdb_info.h
+++ b/tools/vdb-dump/vdb_info.h
@@ -38,7 +38,7 @@ extern "C" {
#include <vdb/manager.h>
rc_t vdb_info( Vector * schema_list, dump_format_t format, const VDBManager *mgr,
- const char * acc_or_path, num_gen * row_generator );
+ const char * acc_or_path, struct num_gen * rows );
#ifdef __cplusplus
}
diff --git a/tools/vdb-validate/vdb-validate.c b/tools/vdb-validate/vdb-validate.c
index e4516be..6953772 100644
--- a/tools/vdb-validate/vdb-validate.c
+++ b/tools/vdb-validate/vdb-validate.c
@@ -673,7 +673,7 @@ static rc_t init_dbcc(KDirectory const *dir, char const name[], bool is_file,
*names = NULL;
}
else {
- *nodes = malloc(nobj * sizeof(**nodes) + namesz);
+ *nodes = calloc(1, nobj * sizeof(**nodes) + namesz);
if (nodes)
*names = (char *)&(*nodes)[nobj];
else
@@ -1668,8 +1668,14 @@ static rc_t verify_database_align(VDatabase const *db,
if (rc) break;
}
rc = dbric_align(name, pri, seq, ref);
+
+ RELEASE(VTable, ref);
+ RELEASE(VTable, seq);
+ RELEASE(VTable, pri);
+
break;
}
+
return rc;
}
@@ -2211,7 +2217,7 @@ static OptDef options [] =
, { "index-only" ,NULL , NULL, USAGE_IND_ONLY, 1, false, false }
/* obsolete options for backward compatibility */
- , { OPTION_md5 , ALIAS_md5 , NULL, USAGE_MD5 , 1, false, false }
+ , { OPTION_md5 , ALIAS_md5 , NULL, USAGE_MD5 , 1, true , false }
, { OPTION_blob_crc, ALIAS_blob_crc, NULL, USAGE_BLOB_CRC, 1, false, false }
, { OPTION_ref_int , ALIAS_ref_int , NULL, USAGE_REF_INT , 1, false, false }
};
@@ -2291,44 +2297,7 @@ rc_t parse_args ( vdb_validate_params *pb, Args *args )
pb->consist_check = false;
ref_int_check = pb -> blob_crc
= pb -> md5_chk_explicit = md5_required = true;
-/*
- {
- rc = ArgsOptionCount(args, OPTION_MD5, &cnt);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_MD5 "' argument");
- return rc;
- }
- if (cnt != 0) {
- rc = ArgsOptionValue(args, OPTION_MD5, 0, &dummy);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_MD5 "' argument");
- return rc;
- }
- assert(dummy && dummy[0]);
- if (dummy[0] == 'n') {
- pb -> md5_chk_explicit = md5_required = false;
- }
- }
- }
-*/{
- rc = ArgsOptionCount( args, OPTION_BLOB_CRC, &cnt);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_BLOB_CRC "' argument");
- return rc;
- }
- if (cnt != 0) {
- rc = ArgsOptionValue(args, OPTION_BLOB_CRC, 0, &dummy);
- if (rc != 0) {
- LOGERR(klogErr, rc,
- "Failure to get '" OPTION_BLOB_CRC "' argument");
- return rc;
- }
- assert(dummy && dummy[0]);
- if (dummy[0] == 'n') {
- pb -> blob_crc = false;
- }
- }
- }
+
{
rc = ArgsOptionCount(args, OPTION_CNS_CHK, &cnt);
if (rc != 0) {
@@ -2336,7 +2305,7 @@ rc_t parse_args ( vdb_validate_params *pb, Args *args )
return rc;
}
if (cnt != 0) {
- rc = ArgsOptionValue(args, OPTION_CNS_CHK, 0, &dummy);
+ rc = ArgsOptionValue(args, OPTION_CNS_CHK, 0, (const void **)&dummy);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" OPTION_CNS_CHK "' argument");
@@ -2361,7 +2330,7 @@ rc_t parse_args ( vdb_validate_params *pb, Args *args )
return rc;
}
if (cnt != 0) {
- rc = ArgsOptionValue(args, OPTION_REF_INT, 0, &dummy);
+ rc = ArgsOptionValue(args, OPTION_REF_INT, 0, (const void **)&dummy);
if (rc != 0) {
LOGERR(klogErr, rc,
"Failure to get '" OPTION_REF_INT "' argument");
@@ -2396,6 +2365,43 @@ rc_t parse_args ( vdb_validate_params *pb, Args *args )
if ( cnt != 0 )
s_IndexOnly = pb -> blob_crc = true;
}
+ {
+ rc = ArgsOptionCount( args, OPTION_BLOB_CRC, &cnt);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_BLOB_CRC "' argument");
+ return rc;
+ }
+ if (cnt != 0) {
+ rc = ArgsOptionValue(args, OPTION_BLOB_CRC, 0, (const void **)&dummy);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" OPTION_BLOB_CRC "' argument");
+ return rc;
+ }
+ assert(dummy && dummy[0]);
+ if (dummy[0] == 'n') {
+ pb -> blob_crc = false;
+ }
+ }
+ }
+ {
+ rc = ArgsOptionCount(args, OPTION_md5, &cnt);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_md5 "' argument");
+ return rc;
+ }
+ if (cnt != 0) {
+ rc = ArgsOptionValue(args, OPTION_md5, 0, (const void **)&dummy);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_md5 "' argument");
+ return rc;
+ }
+ assert(dummy && dummy[0]);
+ if (dummy[0] == 'n') {
+ pb -> md5_chk = pb -> md5_chk_explicit = md5_required = false;
+ }
+ }
+ }
if ( pb -> blob_crc || pb -> index_chk )
pb -> md5_chk = pb -> md5_chk_explicit;
@@ -2499,7 +2505,7 @@ rc_t CC KMain ( int argc, char *argv [] )
{
rc_t rc2;
const char *path;
- rc = ArgsParamValue ( args, i, & path );
+ rc = ArgsParamValue ( args, i, (const void **)& path );
if ( rc != 0 )
{
LOGERR ( klogErr, rc, "Failed to extract command line options" );
diff --git a/tools/vdb-validate/vdb-validate.vers b/tools/vdb-validate/vdb-validate.vers
index 8e8299d..35d16fb 100644
--- a/tools/vdb-validate/vdb-validate.vers
+++ b/tools/vdb-validate/vdb-validate.vers
@@ -1 +1 @@
-2.4.2
+2.5.7
diff --git a/tools/vdb-validate/vdb-validate.vers.h b/tools/vdb-validate/vdb-validate.vers.h
index d4aa6ce..7d1cfb0 100644
--- a/tools/vdb-validate/vdb-validate.vers.h
+++ b/tools/vdb-validate/vdb-validate.vers.h
@@ -1 +1 @@
-#define VDB_VALIDATE_VERS 0x02040002
+#define VDB_VALIDATE_VERS 0x02050007
diff --git a/tools/vdbview/b_first.png b/tools/vdbview/b_first.png
deleted file mode 100644
index 1dcc2b4..0000000
Binary files a/tools/vdbview/b_first.png and /dev/null differ
diff --git a/tools/vdbview/b_goto.png b/tools/vdbview/b_goto.png
deleted file mode 100644
index 0515cbb..0000000
Binary files a/tools/vdbview/b_goto.png and /dev/null differ
diff --git a/tools/vdbview/b_last.png b/tools/vdbview/b_last.png
deleted file mode 100644
index 0d7522b..0000000
Binary files a/tools/vdbview/b_last.png and /dev/null differ
diff --git a/tools/vdbview/b_next.png b/tools/vdbview/b_next.png
deleted file mode 100644
index 4abecd8..0000000
Binary files a/tools/vdbview/b_next.png and /dev/null differ
diff --git a/tools/vdbview/b_nextpage.png b/tools/vdbview/b_nextpage.png
deleted file mode 100644
index fd24611..0000000
Binary files a/tools/vdbview/b_nextpage.png and /dev/null differ
diff --git a/tools/vdbview/b_prev.png b/tools/vdbview/b_prev.png
deleted file mode 100644
index da478a8..0000000
Binary files a/tools/vdbview/b_prev.png and /dev/null differ
diff --git a/tools/vdbview/b_prevpage.png b/tools/vdbview/b_prevpage.png
deleted file mode 100644
index c64d514..0000000
Binary files a/tools/vdbview/b_prevpage.png and /dev/null differ
diff --git a/tools/vdbview/b_search.png b/tools/vdbview/b_search.png
deleted file mode 100644
index c4b8e7e..0000000
Binary files a/tools/vdbview/b_search.png and /dev/null differ
diff --git a/tools/vdbview/background.pas b/tools/vdbview/background.pas
deleted file mode 100644
index 83092e3..0000000
--- a/tools/vdbview/background.pas
+++ /dev/null
@@ -1,277 +0,0 @@
-unit background;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, jobdef, linkedlist, svdb_obj, selector_types;
-
-Type
-
- TOnJobDone = procedure( aJob : PJob ) of Object;
-
- { TBackground }
-
- TBackground = class( TThread )
- private
- FOnJobDone : TOnJobDone;
- FInputQ : TMutexJobQ;
- FJob : PJob;
- FSvdb : TSvdb_obj;
- FIsOpen : boolean;
-
- procedure exec_cmd_cell;
- procedure exec_cmd_colcount;
- procedure exec_cmd_colname;
- procedure exec_cmd_colsdone;
- procedure exec_cmd_open;
- procedure exec_cmd_close;
- procedure exec_cmd_opened;
- procedure exec_cmd_opentab;
- procedure exec_cmd_search;
- procedure exec_cmd_tablename;
-
- protected
- procedure Execute; override;
- procedure Notify;
- public
- constructor Create( OnJobDone : TOnJobDone );
- destructor Destroy; override;
- function PutJob( aJob : PJob ) : boolean;
- procedure FlushJobs( q : TJobQ );
- end;
-
-implementation
-
-function mask_the_backslash( const s : string ) : string;
-var i : Integer;
-begin
- Result := '';
- for i:= 1 to Length( s )
- do begin
- if ( s[ i ] = '\' )
- then Result := Result + '/'
- else Result := Result + s[ i ];
- end;
-end;
-
-{ TBackground }
-
-{ **************************************************
- IN : S1...acc, S2...path,
- OUT: I1...open( 0=no, 1=yes )
- I2...isDb( 0=no, 1=yes )
- I3...Tablecount
- ************************************************** }
-procedure TBackground.exec_cmd_open;
-begin
- FJob^.I1 := 0;
- FJob^.I2 := 0;
- if ( FIsOpen )
- then exec_cmd_close;
- if ( Length( FJob^.S1 ) > 0 )
- then begin
- FJob^.S2 := FSvdb.Acc2Path( FJob^.S1 );
- end;
- if ( Length( FJob^.S2 ) > 0 )
- then if FSvdb.OpenPath( mask_the_backslash( FJob^.S2 ) )
- then FIsOpen := True;
- if FIsOpen
- then begin
- FJob^.I1 := 1;
- if FSvdb.IsDb
- then FJob^.I2 := 1;
- FJob^.I3 := FSvdb.CountTabs;
- end;
-end;
-
-
-procedure TBackground.exec_cmd_close;
-begin
- if ( FIsOpen )
- then begin
- FSvdb.Close;
- FIsOpen := false;
- end;
-end;
-
-
-{ **************************************************
- IN : I1...index
- OUT: S1...Tablename
- ************************************************** }
-procedure TBackground.exec_cmd_tablename;
-begin
- if ( FIsOpen )
- then FJob^.S1 := FSvdb.TabName( FJob^.I1 );
-end;
-
-
-{ **************************************************
- IN : I1...tab-idx
- : I2...selector (all,visible)
- OUT: I3...count
- ************************************************** }
-procedure TBackground.exec_cmd_colcount;
-begin
- if ( FIsOpen )
- then FJob^.I3 := FSvdb.CountCols( FJob^.I1, FJob^.I2 );
-end;
-
-
-{ *** do nothing, just pass through as a signal *** }
-procedure TBackground.exec_cmd_opened; begin end;
-
-
-{ **************************************************
- IN : I1...tab-idx
- I2...selector (all,visible)
- I3...idx
- OUT: S1...name
- ************************************************** }
-procedure TBackground.exec_cmd_colname;
-begin
- if ( FIsOpen )
- then FJob^.S1 := FSvdb.ColName( FJob^.I1, FJob^.I2, FJob^.I3 );
-end;
-
-
-{ **************************************************
- IN : I1...tab-idx
- OUT: I2...result-code
- I3...RowRange
- S1...Non-Static-Columns as String 'XX-X-X--'
- ************************************************** }
-procedure TBackground.exec_cmd_opentab;
-var i, n : Integer;
- s : String;
-begin
- if ( FIsOpen )
- then begin
- FJob^.I2 := FSvdb.OpenTable( FJob^.I1, '' );
- FJob^.I3 := FSvdb.RowRange( FJob^.I1 );
- n := FSvdb.CountCols( FJob^.I1, ALL_COLUMNS );
- for i := 0 to n - 1
- do begin
- s := FSvdb.ColName( FJob^.I1, ALL_COLUMNS, i );
- if ( FSvdb.ColIdx( FJob^.I1, NON_STATIC_COLUMNS, s ) >= 0 )
- then FJob^.S1 := FJob^.S1 + 'X'
- else FJob^.S1 := FJob^.S1 + '-';
- end;
- end;
-end;
-
-
-{ **************************************************
- IN : I1...tab-idx
- I2...selector (all,visible)
- I3...aCol
- I4...aRow
- OUT: S1...name
- ************************************************** }
-procedure TBackground.exec_cmd_cell;
-begin
- if ( FIsOpen )
- then FJob^.S1 := FSvdb.GetCell( FJob^.I1, FJob^.I2, FJob^.I3, FJob^.I4 );
-end;
-
-
-{ **************************************************
- IN : I1...tab-idx
- I2...column to search in
- I3...row to start searching at
- I4...forward or backward
- I5...search-chunk
- S1...text to search for
- OUT: I6... >= 0 ... pattern found at this row,
- -1 ... pattern not found
- -2 ... chunk exhausted, pattern not found
- ************************************************** }
-procedure TBackground.exec_cmd_search;
-begin
- FJob^.I6 := -1;
- if ( FIsOpen )
- then if ( FJob^.I4 = 0 )
- then FJob^.I6 := FSvdb.FindBwd( FJob^.I1, ALL_COLUMNS,
- FJob^.I2, FJob^.I3, FJob^.I5, FJob^.S1 )
- else FJob^.I6 := FSvdb.FindFwd( FJob^.I1, ALL_COLUMNS,
- FJob^.I2, FJob^.I3, FJob^.I5, FJob^.S1 );
-end;
-
-
-{ *** do nothing, just pass through as a signal *** }
-procedure TBackground.exec_cmd_colsdone; begin end;
-
-procedure TBackground.Execute;
-var working : boolean;
-begin
- working := false;
- while ( not Terminated ) do
- begin
- if ( FInputQ.Empty )
- then begin
- if working
- then begin
- working := false;
- FJob := Nil;
- Synchronize( @Notify );
- end;
- Sleep( 200 );
- end
- else begin
- working := true;
- FJob := FInputQ.Get;
- case FJob^.cmd of
- CMD_OPEN : exec_cmd_open;
- CMD_CLOSE : exec_cmd_close;
- CMD_TABLENAME : exec_cmd_tablename;
- CMD_OPENED : exec_cmd_opened;
- CMD_COLCOUNT : exec_cmd_colcount;
- CMD_COLNAME : exec_cmd_colname;
- CMD_COLSDONE : exec_cmd_colsdone;
- CMD_OPENTAB : exec_cmd_opentab;
- CMD_CELL : exec_cmd_cell;
- CMD_SEARCH : exec_cmd_search;
- end;
- Synchronize( @Notify );
- end;
- end;
-end;
-
-procedure TBackground.Notify;
-begin
- if Assigned( FOnJobDone )
- then FOnJobDone( FJob );
-end;
-
-constructor TBackground.Create( OnJobDone : TOnJobDone );
-begin
- FreeOnTerminate := True;
- FOnJobDone := OnJobDone;
- FInputQ := TMutexJobQ.Create;
- FSvdb := TSvdb_obj.Create;
- FIsOpen := false;
- inherited Create( false );
-end;
-
-destructor TBackground.Destroy;
-begin
- inherited Destroy;
- FSvdb.Destroy;
- FInputQ.Free;
-end;
-
-function TBackground.PutJob( aJob : PJob ) : boolean;
-begin
- Result := FInputQ.Empty;
- FInputQ.Put( aJob );
-end;
-
-procedure TBackground.FlushJobs( q : TJobQ );
-begin
- FInputQ.Flush( q );
-end;
-
-end.
-
diff --git a/tools/vdbview/buttons.svg b/tools/vdbview/buttons.svg
deleted file mode 100644
index c298afa..0000000
--- a/tools/vdbview/buttons.svg
+++ /dev/null
@@ -1,268 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!-- Created with Inkscape (http://www.inkscape.org/) -->
-<svg
- xmlns:dc="http://purl.org/dc/elements/1.1/"
- xmlns:cc="http://web.resource.org/cc/"
- xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
- xmlns:svg="http://www.w3.org/2000/svg"
- xmlns="http://www.w3.org/2000/svg"
- xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
- xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
- width="744.09448819"
- height="1052.3622047"
- id="svg2"
- sodipodi:version="0.32"
- inkscape:version="0.45pre1"
- sodipodi:docbase="C:\Users\raetzw\Documents\Lazarus_prj\vdbview"
- sodipodi:docname="buttons.svg"
- inkscape:output_extension="org.inkscape.output.svg.inkscape"
- sodipodi:modified="TRUE">
- <defs
- id="defs4" />
- <sodipodi:namedview
- id="base"
- pagecolor="#ffffff"
- bordercolor="#666666"
- borderopacity="1.0"
- gridtolerance="10000"
- guidetolerance="10"
- objecttolerance="10"
- inkscape:pageopacity="0.0"
- inkscape:pageshadow="2"
- inkscape:zoom="2.9623907"
- inkscape:cx="214.19993"
- inkscape:cy="613.92108"
- inkscape:document-units="px"
- inkscape:current-layer="layer1"
- showgrid="true"
- gridspacingx="20px"
- gridspacingy="20px"
- inkscape:grid-points="true"
- inkscape:window-width="1335"
- inkscape:window-height="981"
- inkscape:window-x="210"
- inkscape:window-y="93" />
- <metadata
- id="metadata7">
- <rdf:RDF>
- <cc:Work
- rdf:about="">
- <dc:format>image/svg+xml</dc:format>
- <dc:type
- rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
- </cc:Work>
- </rdf:RDF>
- </metadata>
- <g
- inkscape:label="Layer 1"
- inkscape:groupmode="layer"
- id="layer1">
- <g
- id="g7117"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_nextpage.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- transform="matrix(2.9134226,0,0,2.9928606,-74.805361,-183.92217)"
- y="112.36218"
- x="60"
- height="40.095417"
- width="41.188671"
- id="rect3131"
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
- <path
- transform="translate(39.999999,39.999993)"
- id="path4105"
- d="M 80,132.36218 L 140,172.36218 L 80,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" />
- <path
- transform="translate(39.999999,39.999993)"
- id="path4109"
- d="M 100,132.36218 L 160,172.36218 L 100,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" />
- </g>
- <g
- id="g7122"
- transform="translate(-2.8184434e-7,3.3322021e-6)"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_prevpage.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- id="rect5080"
- width="41.188671"
- height="40.095417"
- x="60"
- y="112.36218"
- transform="matrix(2.9134226,0,0,2.9928606,65.194639,-183.92217)" />
- <path
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 80,132.36218 L 140,172.36218 L 80,212.36218"
- id="path5082"
- transform="matrix(-1,0,0,-1,420,384.72436)" />
- <path
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 100,132.36218 L 160,172.36218 L 100,212.36218"
- id="path5084"
- transform="matrix(-1,0,0,-1,420,384.72436)" />
- </g>
- <g
- id="g7109"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_next.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- id="rect5086"
- width="41.188671"
- height="40.095417"
- x="60"
- y="112.36218"
- transform="matrix(2.9134226,0,0,2.9928606,-74.805361,-43.922167)" />
- <path
- sodipodi:nodetypes="ccc"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 80,132.36218 L 160,172.36218 L 80,212.36218"
- id="path5088"
- transform="translate(39.999999,180)" />
- </g>
- <g
- id="g7136">
- <rect
- transform="matrix(2.9134226,0,0,2.9928606,65.194639,-43.922167)"
- y="112.36218"
- x="60"
- height="40.095417"
- width="41.188671"
- id="rect5092"
- style="color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
- <path
- sodipodi:nodetypes="ccc"
- transform="matrix(-1,0,0,-1,420,524.72436)"
- id="path5094"
- d="M 80,132.36218 L 160,172.36218 L 80,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
- </g>
- <g
- id="g7086"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_last.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- id="rect5098"
- width="41.188671"
- height="40.095417"
- x="60"
- y="112.36218"
- transform="matrix(2.9134226,0,0,2.9928606,-74.805361,96.077833)" />
- <path
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 80,132.36218 L 140,172.36218 L 80,212.36218"
- id="path5100"
- transform="translate(39.999999,320)" />
- <path
- sodipodi:nodetypes="cc"
- transform="translate(120,320)"
- id="path5112"
- d="M 80,132.36218 L 80,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
- </g>
- <g
- id="g7081"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_first.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- transform="matrix(2.9134226,0,0,2.9928606,65.194639,96.077833)"
- y="112.36218"
- x="60"
- height="40.095417"
- width="41.188671"
- id="rect5104"
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
- <path
- transform="matrix(-1,0,0,-1,420,664.72436)"
- id="path5106"
- d="M 80,132.36218 L 140,172.36218 L 80,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
- <path
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 80,132.36218 L 80,212.36218"
- id="path5115"
- transform="translate(180,320)"
- sodipodi:nodetypes="cc" />
- </g>
- <g
- id="g7127"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_search.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- transform="matrix(2.9134226,0,0,2.9928606,205.19464,-183.92217)"
- y="112.36218"
- x="60"
- height="40.095417"
- width="41.188671"
- id="rect5135"
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
- <path
- transform="translate(4.5,-124.5)"
- d="M 480 322.36218 A 30 30 0 1 1 420,322.36218 A 30 30 0 1 1 480 322.36218 z"
- sodipodi:ry="30"
- sodipodi:rx="30"
- sodipodi:cy="322.36218"
- sodipodi:cx="450"
- id="path5137"
- style="opacity:1;color:#000000;fill:none;fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- sodipodi:type="arc" />
- <path
- sodipodi:nodetypes="cc"
- id="path6108"
- d="M 434.26139,220.46374 L 400,252.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" />
- </g>
- <g
- id="g7146"
- inkscape:export-filename="C:\Users\raetzw\Documents\Lazarus_prj\vdbview\b_goto.png"
- inkscape:export-xdpi="16.5"
- inkscape:export-ydpi="16.5">
- <rect
- style="opacity:1;color:#000000;fill:#ff6600;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- id="rect7079"
- width="41.188671"
- height="40.095417"
- x="60"
- y="112.36218"
- transform="matrix(2.9134226,0,0,2.9928606,205.19464,-43.922167)" />
- <path
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 80,132.36218 L 80,212.36218"
- id="path7134"
- transform="matrix(-1,0,0,-1,500,524.72436)"
- sodipodi:nodetypes="cc" />
- <path
- sodipodi:nodetypes="cc"
- transform="matrix(-1,0,0,-1,540,524.72436)"
- id="path7140"
- d="M 80,132.36218 L 80,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
- <path
- inkscape:transform-center-y="19.999997"
- inkscape:transform-center-x="-60"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- d="M 80,132.36218 L 80,212.36218"
- id="path7142"
- transform="matrix(0,1,-1,0,612.36218,252.36218)"
- sodipodi:nodetypes="cc" />
- <path
- sodipodi:nodetypes="cc"
- transform="matrix(0,1,-1,0,612.36218,292.36218)"
- id="path7144"
- d="M 80,132.36218 L 80,212.36218"
- style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:9;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
- inkscape:transform-center-x="-60"
- inkscape:transform-center-y="19.999997" />
- </g>
- </g>
-</svg>
diff --git a/tools/vdbview/columnsel.lfm b/tools/vdbview/columnsel.lfm
deleted file mode 100644
index b2c0f76..0000000
--- a/tools/vdbview/columnsel.lfm
+++ /dev/null
@@ -1,90 +0,0 @@
-object Columnform: TColumnform
- Left = 375
- Height = 472
- Top = 154
- Width = 350
- BorderStyle = bsSingle
- Caption = 'columns...'
- ClientHeight = 472
- ClientWidth = 350
- LCLVersion = '0.9.30'
- object ColumnBox: TCheckListBox
- Left = 0
- Height = 376
- Top = 0
- Width = 350
- Align = alClient
- ItemHeight = 0
- OnItemClick = ColumnBoxItemClick
- TabOrder = 0
- end
- object Footer: TPanel
- Left = 0
- Height = 50
- Top = 422
- Width = 350
- Align = alBottom
- ClientHeight = 50
- ClientWidth = 350
- TabOrder = 1
- object BOK: TBitBtn
- Left = 16
- Height = 30
- Top = 10
- Width = 144
- Caption = '&OK'
- Default = True
- Kind = bkOK
- ModalResult = 1
- TabOrder = 0
- end
- object BCancel: TBitBtn
- Left = 192
- Height = 30
- Top = 10
- Width = 144
- Cancel = True
- Caption = 'Cancel'
- Kind = bkCancel
- ModalResult = 2
- TabOrder = 1
- end
- end
- object Panel1: TPanel
- Left = 0
- Height = 46
- Top = 376
- Width = 350
- Align = alBottom
- ClientHeight = 46
- ClientWidth = 350
- TabOrder = 2
- object BClear: TButton
- Left = 16
- Height = 25
- Top = 8
- Width = 104
- Caption = 'Clear all'
- OnClick = BClearClick
- TabOrder = 0
- end
- object BSetAll: TButton
- Left = 128
- Height = 25
- Top = 8
- Width = 99
- Caption = 'Set all'
- OnClick = BSetAllClick
- TabOrder = 1
- end
- object BNonStatic: TButton
- Left = 237
- Height = 25
- Top = 8
- Width = 99
- Caption = 'Set non static'
- OnClick = BNonStaticClick
- TabOrder = 2
- end
- end
-end
diff --git a/tools/vdbview/columnsel.pas b/tools/vdbview/columnsel.pas
deleted file mode 100644
index bc0c9c4..0000000
--- a/tools/vdbview/columnsel.pas
+++ /dev/null
@@ -1,115 +0,0 @@
-unit ColumnSel;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, FileUtil, Forms, Controls, Graphics, Dialogs, CheckLst,
- ExtCtrls, Buttons, StdCtrls, DataProxy, selector_types, support_unit;
-
-type
-
- { TColumnform }
-
- TColumnform = class(TForm)
- BOK: TBitBtn;
- BCancel: TBitBtn;
- BClear: TButton;
- BSetAll: TButton;
- BNonStatic: TButton;
- ColumnBox: TCheckListBox;
- Footer: TPanel;
- Panel1: TPanel;
- procedure BClearClick(Sender: TObject);
- procedure BNonStaticClick(Sender: TObject);
- procedure BSetAllClick(Sender: TObject);
- procedure ColumnBoxItemClick( Sender : TObject; Index : integer );
- private
- { private declarations }
- FOrgChecked : String;
- FNonStatic : String;
- procedure setup_columns( aProxy : TProxy );
- procedure enable_bok;
- public
- { public declarations }
- function present( const p : TPoint; const h : Integer;
- aProxy : TProxy ) : boolean;
- function checked_items : String;
- end;
-
-var
- Columnform: TColumnform;
-
-
-implementation
-
-{$R *.lfm}
-
-{ TColumnform }
-
-procedure TColumnform.BClearClick( Sender : TObject );
-var i : Integer;
-begin
- for i:=0 to ColumnBox.Items.Count -1
- do ColumnBox.Checked[ i ] := false;
- enable_bok;
-end;
-
-procedure TColumnform.BNonStaticClick( Sender : TObject );
-var i : Integer;
-begin
- for i:=0 to ColumnBox.Items.Count -1
- do ColumnBox.Checked[ i ] := ( FNonStatic[ i + 1 ] = 'X' );
- enable_bok;
-end;
-
-procedure TColumnform.BSetAllClick( Sender : TObject );
-var i : Integer;
-begin
- for i:=0 to ColumnBox.Items.Count -1
- do ColumnBox.Checked[ i ] := true;
- enable_bok;
-end;
-
-procedure TColumnform.ColumnBoxItemClick( Sender : TObject; Index : integer );
-begin
- enable_bok;
-end;
-
-procedure TColumnform.setup_columns( aProxy : TProxy );
-var i : Integer;
-begin
- ColumnBox.Clear;
- for i:=0 to aProxy.ColCount[ ALL_COLUMNS ] - 1
- do begin
- ColumnBox.Items.Add( aProxy.ColName[ ALL_COLUMNS, i ] );
- ColumnBox.Checked[ i ] := aProxy.ColVisibleI[ i ];
- end;
- FNonStatic := aProxy.NonStaticColumns;
-end;
-
-procedure TColumnform.enable_bok;
-begin
- BOK.Enabled := ( FOrgChecked <> checked_items );
-end;
-
-function TColumnform.present( const p : TPoint; const h : Integer;
- aProxy : TProxy ) : boolean;
-begin
- Left := p.x;
- Top := p.y;
- Height := h;
- setup_columns( aProxy );
- FOrgChecked := CheckListBox2String( ColumnBox );
- BOK.Enabled := False;
- Result := ( ShowModal = mrOK );
-end;
-
-function TColumnform.checked_items: String;
-begin
- Result := CheckListBox2String( ColumnBox );
-end;
-
-end.
-
diff --git a/tools/vdbview/dataproxy.pas b/tools/vdbview/dataproxy.pas
deleted file mode 100644
index 39fc670..0000000
--- a/tools/vdbview/dataproxy.pas
+++ /dev/null
@@ -1,602 +0,0 @@
-unit DataProxy;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, Dialogs, math, jobdef, background, linkedlist,
- selector_types;
-
-type
-
- cell = record
- value : String;
- valid : boolean;
- end;
-
- row = record
- cells : array of cell;
- end;
-
- TOnCellValid = procedure( const aCol, aRow : Integer ) of Object;
- TOnSearchDone = procedure( const aCol, aRow : Integer;
- const pattern : String ) of Object;
- TOnOpened = procedure( const path : String; success : Boolean ) of Object;
- TOnTableSwitched = procedure of Object;
- TOnClosed = procedure of Object;
- TOnLED = procedure( state : boolean ) of Object;
- TOnLog = procedure( const s : String ) of Object;
-
- { TProxy }
-
- TProxy = class
- private
- rows : array of row;
- FOffset : Integer;
- FRows : Integer;
- FCols : Integer;
- FVisCols : Integer;
-
- FBackground : TBackground;
- FJobStock : TJobQ;
- FTableNames : TStringList;
-
- FAllColumnNames : TStringList;
- FVisIdx2AllIdx : array of Integer;
- FAllIdx2VisIdx : array of Integer;
-
- FOnCellValid : TOnCellValid;
- FOnOpened : TOnOpened;
- FOnTableSwitched : TOnTableSwitched;
- FOnClosed : TOnClosed;
- FOnSearchDone : TOnSearchDone;
- FOnLED : TOnLED;
-
- FAccession : String;
- FOpenPath : String;
- FNonStatic : String;
- FTabId : Integer;
- FIsOpen : boolean;
- FIsDb : boolean;
- FTables : Integer;
- FTableRows : Integer;
- FSearchChunk : Integer;
- FSearchCancel: boolean;
-
- procedure CopyRow( const dst, src : Integer );
- function GetCell( aCol, aRow : Integer ): String;
- function GetColCount( sel : Integer ): Integer;
- function GetColName( sel, idx : Integer ): String;
- function GetColVisibleI( idx : Integer ): boolean;
- function GetColVisibleS( aName : String ): boolean;
- function GetTableName( idx : Integer ): String;
- procedure Invalidate( const firstrow, lastrow : Integer );
- procedure SetColVisibleI( idx : Integer ; const AValue : boolean );
- procedure SetColVisibleS( aName : String ; const AValue : boolean );
- procedure SetOffset( const n : Integer );
- procedure AdjustCacheColumnCount( const n : Integer );
-
- procedure SendJob( cmd : TCmdType;
- i1, i2, i3, i4, i5, i6 : Integer;
- s1, s2 : String );
- procedure QueryColName( sel, idx : Integer );
- procedure QueryOpened;
- procedure QueryTablename( idx : Integer );
- procedure QueryColCount( sel : Integer );
- procedure QueryColsDone( sel : Integer );
- procedure QueryCell( sel, aCol, aRow : Integer );
- procedure QueryOpenTab;
-
- procedure OnJobDone( job : PJob );
- procedure OnOpenJobDone( job : PJob );
- procedure OnOpenedReceived;
- procedure OnCloseJobDone;
- procedure OnColCountReceived( job : PJob );
- procedure OnColNameReceived( job : PJob );
- procedure OnColsDoneReceived( job : PJob );
- procedure OnTableName( job : PJob );
- procedure OnOpenTabReceived( job : PJob );
- procedure OnCellReceived( job : PJob );
- procedure OnSearchReceived( job : PJob );
-
- procedure SetTabId( const AValue : Integer );
- procedure ShiftRows( const by : Integer );
- public
- constructor Create;
- destructor Destroy; override;
- procedure AdjustCacheRows( const n : Integer );
- procedure Open( path, acc : String );
- procedure AjustVisibleColumns( checked : String );
- procedure Search( fwd : boolean; colidx, startrow : Integer;
- pattern : String );
- procedure CancelSearch;
-
- property OnCellValid : TOnCellValid read FOnCellValid write FOnCellValid;
- property OnOpened : TOnOpened read FOnOpened write FOnOpened;
- property OnTableSwitched : TOnTableSwitched read FOnTableSwitched write FOnTableSwitched;
- property OnClosed : TOnClosed read FOnClosed write FOnClosed;
- property OnLED : TOnLED read FOnLED write FOnLED;
- property OnSearchDone : TOnSearchDone read FOnSearchDone write FOnSearchDone;
-
- property Tables : Integer read FTables;
- property TableName[ idx : Integer ] : String read GetTableName;
- property TableRows : Integer read FTableRows;
- property IsDb : Boolean read FIsDb;
- property Accession : String read FAccession;
- property IsOpen : Boolean read FIsOpen;
- property Path : String read FOpenPath;
- property TabId : Integer read FTabId write SetTabId;
- property ColCount[ sel : Integer ] : Integer read GetColCount;
- property ColName[ sel, idx : Integer ] : String read GetColName;
- property Cell[ aCol, aRow : Integer ] : String read GetCell;
- property RowCount : Integer read FRows;
- property Offset : Integer read FOffset write SetOffset;
- property ColVisibleS[ aName : String ] : boolean read GetColVisibleS write SetColVisibleS;
- property ColVisibleI[ idx : Integer ] : boolean read GetColVisibleI write SetColVisibleI;
- property NonStaticColumns : String read FNonStatic;
- end;
-
-
-implementation
-
-{ TProxy }
-
-constructor TProxy.Create;
-begin
- FOnCellValid := NIL;
- FOnOpened := NIL;
- FOnTableSwitched := NIL;
- FOnClosed := NIL;
- FOnLED := NIL;
- FOnSearchDone := NIL;
-
- FOffset := 0;
- FRows := 0;
- FCols := 1;
- FVisCols:= 1;
- FTabId := 0;
- FTableRows := 0;
- FOpenPath := '';
- FAccession := '';
- FNonStatic := '';
- FSearchChunk := 1000;
- AdjustCacheRows( 10 );
-
- FTableNames := TStringList.Create;
- FAllColumnNames := TStringList.Create;
-
- SetLength( FVisIdx2AllIdx, 1 );
- FVisIdx2AllIdx[ 0 ] := 0;
- SetLength( FAllIdx2VisIdx, 1 );
- FAllIdx2VisIdx[ 0 ] := 0;
-
- FJobStock := TJobQ.Create;
- FBackground := TBackground.Create( @OnJobDone );
-end;
-
-destructor TProxy.Destroy;
-begin
- FBackground.Terminate;
- FJobStock.Free;
- FTableNames.Free;
- FAllColumnNames.Free;
- SetLength( FVisIdx2AllIdx, 0 );
- SetLength( FAllIdx2VisIdx, 0 );
- inherited Destroy;
-end;
-
-procedure TProxy.AdjustCacheRows( const n : Integer );
-var i : Integer;
-begin
- if ( n < FRows )
- then for i := n to FRows-1
- do SetLength( rows[ i ].cells, 0 );
- SetLength( rows, n );
- if ( n > FRows )
- then for i := FRows to n-1
- do SetLength( rows[ i ].cells, FCols );
- FRows := n;
-end;
-
-procedure TProxy.AdjustCacheColumnCount( const n : Integer );
-var i : Integer;
-begin
- for i := 0 to FRows - 1
- do SetLength( rows[i].cells, n );
- SetLength( FVisIdx2AllIdx, n );
- SetLength( FAllIdx2VisIdx, n );
- for i := 0 to n - 1
- do begin
- FVisIdx2AllIdx[ i ] := i;
- FAllIdx2VisIdx[ i ] := i;
- end;
- FCols := n;
- FVisCols := n;
-end;
-
-procedure TProxy.Invalidate( const firstrow, lastrow : Integer );
-var i, j : Integer;
-begin
- for i := firstrow to lastrow
- do begin
- for j:= 0 to FCols - 1
- do begin
- rows[ i ].cells[ j ].value := '';
- rows[ i ].cells[ j ].valid := false;
- end;
- end;
-end;
-
-procedure TProxy.SetColVisibleI( idx : Integer ; const AValue: boolean );
-var i, j : Integer;
-begin
- if ( idx >= 0 )and( idx < FCols )
- then begin
- if ( AValue )
- then FAllIdx2VisIdx[ idx ] := idx
- else FAllIdx2VisIdx[ idx ] := -1;
- j := 0;
- for i := 0 to FCols - 1
- do begin
- if ( FAllIdx2VisIdx[ i ] >= 0 )
- then begin
- FAllIdx2VisIdx[ i ] := j;
- FVisIdx2AllIdx[ j ] := i;
- inc( j );
- end;
- end;
- FVisCols := j;
- end;
-end;
-
-procedure TProxy.SetColVisibleS( aName : String ; const AValue : boolean );
-begin
- SetColVisibleI( FAllColumnNames.IndexOf( aName ), AValue );
-end;
-
-procedure TProxy.CopyRow( const dst, src : Integer );
-var i : Integer;
-begin
- for i := 0 to ( FCols - 1 )
- do rows[ dst ].cells[ i ] := rows[ src ].cells[ i ];
-end;
-
-function TProxy.GetCell( aCol, aRow : Integer ): String;
-var eRow, absCol : Integer;
-begin
- Result := '';
- if ( aCol >= 0 )and( aCol < FCols )
- then begin
- eRow := aRow - FOffset;
- absCol := FVisIdx2AllIdx[ aCol ];
- if ( absCol >= 0 )and( absCol < Length( rows[0].cells ) )and
- ( eRow >= 0 )and( eRow < Length( rows ) )
- then if ( rows[ eRow ].cells[ absCol ].valid )
- then Result := rows[ eRow ].cells[ absCol ].value
- else QueryCell( ALL_COLUMNS, absCol, aRow );
- end;
-end;
-
-function TProxy.GetColCount( sel : Integer ) : Integer;
-begin
- Result := 0;
- case sel of
- ALL_COLUMNS : Result := FCols;
- VISIBLE_COLUMNS : Result := FVisCols;
- end;
-end;
-
-function TProxy.GetColName( sel, idx : Integer ) : String;
-begin
- Result := '';
- if ( idx >= 0 )and ( idx < FCols )
- then begin
- if ( sel = VISIBLE_COLUMNS )
- then idx := FVisIdx2AllIdx[ idx ];
- if ( idx >= 0 )and( idx < FAllColumnNames.Count )
- then Result := FAllColumnNames[ idx ];
- end;
-end;
-
-function TProxy.GetColVisibleI( idx : Integer ): boolean;
-begin
- if ( idx >= 0 ) and ( idx < FCols )
- then Result := ( FAllIdx2VisIdx[ idx ] >= 0 )
- else Result := false;
-end;
-
-function TProxy.GetColVisibleS( aName : String ) : boolean;
-begin
- Result := GetColVisibleI( FAllColumnNames.IndexOf( aName ) );
-end;
-
-function TProxy.GetTableName( idx : Integer ) : String;
-begin
- if ( idx >= 0 )and( idx < FTableNames.Count )
- then Result := FTableNames[ idx ]
- else Result := '?';
-end;
-
-procedure TProxy.ShiftRows( const by : Integer );
-var i : Integer;
-begin
- if ( by > 0 )
- then for i := 0 to ( ( FRows - 1 ) - by )
- do CopyRow( i, i + by )
- else for i := FRows - 1 downto ( - by )
- do CopyRow( i, i + by )
-end;
-
-procedure TProxy.SetOffset( const n : Integer );
-var diff : Integer;
-begin
- diff := ( n - FOffset );
- if ( diff <> 0 )
- then begin
- FOffset := n;
- if ( abs( diff ) >= FRows )
- then begin
- FBackground.FlushJobs( FJobStock );
- Invalidate( 0, FRows - 1 ); { invalidate all }
- end
- else begin
- ShiftRows( diff );
- if ( diff > 0 )
- then Invalidate( FRows - diff, FRows - 1 )
- else Invalidate( 0, ( -diff ) - 1 );
- end;
- end;
-end;
-
-procedure TProxy.SendJob( cmd : TCmdType;
- i1, i2, i3, i4, i5, i6 : Integer;
- s1, s2 : String );
-var job : PJob;
-begin
- job := FJobStock.GetOrMake;
- if Assigned( job )
- then begin
- job^.cmd := cmd;
- job^.I1 := i1;
- job^.I2 := i2;
- job^.I3 := i3;
- job^.I4 := i4;
- job^.I5 := i5;
- job^.I6 := i6;
- job^.S1 := s1;
- job^.S2 := s2;
- if FBackground.PutJob( job )
- then if Assigned( FOnLED )
- then FOnLED( true );
- end;
-end;
-
-procedure TProxy.Open( path, acc : String );
-begin SendJob( CMD_OPEN, 0, 0, 0, 0, 0, 0, acc, path ); end;
-
-procedure TProxy.AjustVisibleColumns( checked : String );
-var i, l : Integer;
- vis : boolean;
-begin
- l := Min( Length( checked ), FCols );
- for i := 0 to l - 1
- do begin
- vis := ( checked[ i + 1 ] = 'X' );
- if ( vis <> ColVisibleI[ i ] )
- then ColVisibleI[ i ] := vis;
- end;
-end;
-
-procedure TProxy.Search( fwd : boolean;
- colidx, startrow : Integer;
- pattern: String );
-var i_fwd, absCol : Integer;
-begin
- if fwd then i_fwd := 1 else i_fwd := 0;
- if ( colidx >= 0 )and( colidx < FCols )
- then begin
- absCol := FVisIdx2AllIdx[ colidx ];
- if ( absCol >= 0 )
- then begin
- SendJob( CMD_SEARCH,
- FTabId, absCol, startrow, i_fwd, FSearchChunk, 0,
- pattern, '' );
- FSearchCancel := false;
- end;
- end;
-end;
-
-procedure TProxy.CancelSearch;
-begin
- FSearchCancel := true;
-end;
-
-procedure TProxy.QueryTablename( idx : Integer );
-begin SendJob( CMD_TABLENAME, idx, 0, 0, 0, 0, 0, '', '' ); end;
-
-procedure TProxy.QueryColCount( sel : Integer );
-begin SendJob( CMD_COLCOUNT, FTabId, sel, 0, 0, 0, 0, '', '' ); end;
-
-procedure TProxy.QueryColsDone( sel : Integer );
-begin SendJob( CMD_COLSDONE, sel, 0, 0, 0, 0, 0, '', '' ); end;
-
-procedure TProxy.QueryCell( sel, aCol, aRow : Integer );
-begin SendJob( CMD_CELL, FTabId, sel, aCol, aRow, 0, 0, '', '' ); end;
-
-procedure TProxy.QueryOpenTab;
-begin SendJob( CMD_OPENTAB, FTabId, 0, 0, 0, 0, 0, '', '' ); end;
-
-procedure TProxy.QueryOpened;
-begin SendJob( CMD_OPENED, 0, 0, 0, 0, 0, 0, '', '' ); end;
-
-procedure TProxy.QueryColName( sel, idx : Integer );
-begin SendJob( CMD_COLNAME, FTabId, sel, idx, 0, 0, 0, '', '' ); end;
-
-
-{ **************************************************
- S1...acc, S2...path,
- I1...open( 0=no, 1=yes )
- I2...isDb( 0=no, 1=yes )
- I3...number of tables
- ************************************************** }
-procedure TProxy.OnOpenJobDone( job : PJob );
-var i : Integer;
-begin
- FTableNames.Clear;
- if ( job^.I1 > 0 )
- then begin
- FAccession := job^.S1;
- FOpenPath := job^.S2;
- FIsOpen := true;
- FIsDb := ( job^.I2 > 0 );
- FTables := job^.I3;
- for i := 0 to FTables-1
- do QueryTablename( i );
- QueryOpened;
- end
- else begin
- FAccession := '';
- FOpenPath := '';
- FIsOpen := false;
- FIsDb := false;
- FTables := 0;
- if Assigned( FOnOpened )
- then FOnOpened( job^.S2, false );
- end;
-end;
-
-procedure TProxy.OnCloseJobDone;
-begin
- if Assigned( FOnClosed )
- then FOnClosed;
-end;
-
-procedure TProxy.OnTableName( job : PJob );
-begin
- FTableNames.Add( job^.S1 );
-end;
-
-procedure TProxy.OnOpenedReceived;
-begin
- if Assigned( FOnOpened )
- then FOnOpened( FOpenPath, true );
-end;
-
-procedure TProxy.OnColCountReceived( job : PJob );
-var i : Integer;
-begin
- for i:=0 to job^.I3 - 1
- do QueryColName( job^.I2, i );
- QueryColsDone( job^.I2 );
-end;
-
-procedure TProxy.OnColNameReceived( job : PJob );
-begin
- case job^.I2 of
- ALL_COLUMNS : FAllColumnNames.Add( job^.S1 );
- end;
-end;
-
-procedure TProxy.OnColsDoneReceived( job : PJob );
-begin
- case job^.I1 of
- ALL_COLUMNS : QueryOpenTab;
- end;
-end;
-
-procedure TProxy.OnOpenTabReceived( job : PJob );
-begin
- AdjustCacheColumnCount( FAllColumnNames.Count );
- FTableRows := job^.I3;
- FNonStatic := job^.S1;
- if Assigned( FOnTableSwitched )
- then FOnTableSwitched;
-end;
-
-procedure TProxy.OnCellReceived( job : PJob );
-var aRow, aCol, visCol : Integer;
-begin
- aCol := job^.I3; { is in all-columns-index }
- aRow := job^.I4 - FOffset;
- if ( aRow >= 0 )and( aRow < FRows )and( aCol >= 0 )and( aCol < FCols )
- then begin
- rows[ aRow ].cells[ aCol ].value := job^.S1;
- rows[ aRow ].cells[ aCol ].valid := true;
- visCol := FAllIdx2VisIdx[ aCol ];
- if Assigned( FOnCellValid ) and ( visCol >= 0 )
- then FOnCellValid( visCol, aRow );
- end;
-end;
-
-procedure TProxy.OnSearchReceived( job : PJob );
-var aCol, visCol, aRow : Integer;
- fwd, call_handler : boolean;
-begin
- if ( not FSearchCancel )
- then begin
- visCol := 0;
- aRow := -1;
- call_handler := true;
- aCol := job^.I2; { is in all-columns-index }
- if ( aCol >= 0 )and( aCol < FCols )
- then begin
- visCol := FAllIdx2VisIdx[ aCol ];
- if ( visCol >= 0 )
- then aRow := job^.I6;
- if ( aRow = -2 )
- then begin
- { search of next chunk if not found... }
- fwd := ( job^.I4 = 1 );
- if ( fwd )
- then aRow := job^.I3 + FSearchChunk
- else aRow := job^.I3 - FSearchChunk;
- if ( aRow >= 0 )and( aRow < FTableRows )
- then begin
- Search( fwd, visCol, aRow, job^.S1 );
- call_handler := false;
- end
- else begin
- aRow := -1; { not found... }
- end;
- end;
- end;
- if Assigned( FOnSearchDone ) and ( call_handler )
- then FOnSearchDone( visCol, aRow, job^.S1 );
- end;
-end;
-
-procedure TProxy.SetTabId( const AValue : Integer );
-begin
- FTabId := AValue;
- FAllColumnNames.Clear;
- Invalidate( 0, FRows - 1 ); { InvalidateAll }
- QueryColCount( ALL_COLUMNS );
-end;
-
-procedure TProxy.OnJobDone( job : PJob );
-begin
- if Assigned( job )
- then begin
- case job^.cmd of
- CMD_OPEN : OnOpenJobDone( job );
- CMD_CLOSE : OnCloseJobDone;
- CMD_TABLENAME : OnTableName( job );
- CMD_OPENED : OnOpenedReceived;
- CMD_COLCOUNT : OnColCountReceived( job );
- CMD_COLNAME : OnColNameReceived( job );
- CMD_COLSDONE : OnColsDoneReceived( job );
- CMD_OPENTAB : OnOpenTabReceived( job );
- CMD_CELL : OnCellReceived( job );
- CMD_SEARCH : OnSearchReceived( job );
- end;
- FJobStock.Put( job );
- end
- else begin
- if Assigned( FOnLED )
- then FOnLED( false );
- end;
-end;
-
-end.
-
diff --git a/tools/vdbview/gotoform.lfm b/tools/vdbview/gotoform.lfm
deleted file mode 100644
index b1460a4..0000000
--- a/tools/vdbview/gotoform.lfm
+++ /dev/null
@@ -1,56 +0,0 @@
-object GotoForm: TGotoForm
- Left = 416
- Height = 101
- Top = 502
- Width = 186
- ActiveControl = ERow
- Caption = 'Goto Row...'
- ClientHeight = 101
- ClientWidth = 186
- LCLVersion = '0.9.30'
- object Panel1: TPanel
- Left = 0
- Height = 50
- Top = 51
- Width = 186
- Align = alBottom
- ClientHeight = 50
- ClientWidth = 186
- TabOrder = 0
- object BOK: TBitBtn
- Left = 8
- Height = 30
- Top = 9
- Width = 83
- Caption = '&OK'
- Default = True
- Kind = bkOK
- ModalResult = 1
- TabOrder = 0
- end
- object BCancel: TBitBtn
- Left = 96
- Height = 30
- Top = 9
- Width = 83
- Cancel = True
- Caption = 'Cancel'
- Kind = bkCancel
- ModalResult = 2
- TabOrder = 1
- end
- end
- object ERow: TMaskEdit
- Left = 8
- Height = 21
- Top = 16
- Width = 168
- AutoSelect = False
- AutoSize = False
- CharCase = ecNormal
- MaxLength = 10
- TabOrder = 1
- EditMask = '##########;0;_'
- SpaceChar = '_'
- end
-end
diff --git a/tools/vdbview/gotoform.pas b/tools/vdbview/gotoform.pas
deleted file mode 100644
index 2ce7341..0000000
--- a/tools/vdbview/gotoform.pas
+++ /dev/null
@@ -1,47 +0,0 @@
-unit gotoform;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, FileUtil, Forms, Controls, Graphics, Dialogs, ExtCtrls,
- Buttons, MaskEdit;
-
-type
-
- { TGotoForm }
-
- TGotoForm = class(TForm)
- BOK: TBitBtn;
- BCancel: TBitBtn;
- ERow: TMaskEdit;
- Panel1: TPanel;
- private
- { private declarations }
- public
- { public declarations }
- function present( const p : TPoint; var row_nr : Integer ) : boolean;
- end;
-
-var
- MyGotoForm : TGotoForm;
-
-implementation
-
-{$R *.lfm}
-
-{ TGotoForm }
-
-function TGotoForm.present( const p : TPoint; var row_nr : Integer ) : boolean;
-begin
- Left := p.x;
- Top := p.y;
- ERow.Text := IntToStr( row_nr );
- Result := ( ShowModal = mrOK );
- if ( Result )
- then row_nr := StrToIntDef( ERow.Text, row_nr );
-end;
-
-end.
-
diff --git a/tools/vdbview/jobdef.pas b/tools/vdbview/jobdef.pas
deleted file mode 100644
index 2b18b71..0000000
--- a/tools/vdbview/jobdef.pas
+++ /dev/null
@@ -1,27 +0,0 @@
-unit jobdef;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils;
-
-type
- TCmdType = ( CMD_OPEN, CMD_CLOSE, CMD_TABLENAME,
- CMD_OPENED, CMD_COLCOUNT, CMD_COLNAME, CMD_COLSDONE,
- CMD_OPENTAB, CMD_CELL, CMD_SEARCH );
-
- { TJob }
- PJob = ^TJob;
- TJob = record
- cmd : TCmdType;
- I1, I2, I3, I4, I5, I6 : Integer;
- S1, S2 : String;
- next : PJob;
- end;
-
-implementation
-
-end.
-
diff --git a/tools/vdbview/linkedlist.pas b/tools/vdbview/linkedlist.pas
deleted file mode 100644
index 59dc086..0000000
--- a/tools/vdbview/linkedlist.pas
+++ /dev/null
@@ -1,186 +0,0 @@
-unit linkedlist;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, jobdef;
-
-type
-
- { TJobQ }
- TJobQ = class
- private
- FHead, FTail : PJob;
- public
- constructor Create;
- destructor Destroy; override;
- function Empty : Boolean; virtual;
- procedure Put( job : PJob ); virtual;
- function Get : PJob; virtual;
- function GetOrMake : PJob; virtual;
- procedure Flush( dst : TJobQ ); virtual;
- procedure Clear;
- end;
-
- { TMutexJobQ }
-
- TMutexJobQ = class( TJobQ )
- private
- cs : TRTLCriticalSection;
- public
- constructor Create;
- destructor Destroy; override;
- function Empty : Boolean; override;
- procedure Put( job : PJob ); override;
- function Get : PJob; override;
- function GetOrMake : PJob; override;
- procedure Flush( dst : TJobQ ); override;
- end;
-
-implementation
-
-{ TJobQ }
-
-constructor TJobQ.Create;
-begin
- inherited Create;
- FHead := Nil;
- FTail := Nil;
-end;
-
-destructor TJobQ.Destroy;
-begin
- Clear;
- inherited Destroy;
-end;
-
-function TJobQ.Empty : Boolean;
-begin
- Result := ( FHead = Nil );
-end;
-
-procedure TJobQ.Put( job : PJob );
-begin
- job^.next := Nil;
- if ( FHead = Nil )
- then FHead := job
- else FTail^.next := job;
- FTail := job;
-end;
-
-function TJobQ.Get : PJob;
-begin
- if ( FHead <> Nil )
- then begin
- Result := FHead;
- FHead := Result^.next;
- end
- else begin
- Result := Nil;
- end;
-end;
-
-function TJobQ.GetOrMake : PJob;
-begin
- if ( FHead <> Nil )
- then begin
- Result := FHead;
- FHead := Result^.next;
- end
- else begin
- new ( Result );
- end;
-end;
-
-procedure TJobQ.Flush( dst : TJobQ );
-var job : PJob;
-begin
- while ( not Empty )
- do begin
- job := Get;
- if Assigned( dst )
- then dst.Put( job )
- else Dispose( job );
- end;
-end;
-
-procedure TJobQ.Clear;
-var job : PJob;
-begin
- while ( not Empty )
- do begin
- job := Get;
- Dispose( job );
- end;
-end;
-
-{ TMutexJobQ }
-
-constructor TMutexJobQ.Create;
-begin
- inherited Create;
- InitCriticalSection( cs );
-end;
-
-destructor TMutexJobQ.Destroy;
-begin
- inherited Destroy;
- DoneCriticalsection( cs );
-end;
-
-function TMutexJobQ.Empty : Boolean;
-begin
- EnterCriticalSection( cs );
- try
- Result := inherited Empty;
- finally
- LeaveCriticalSection( cs );
- end;
-end;
-
-procedure TMutexJobQ.Put( job : PJob );
-begin
- EnterCriticalSection( cs );
- try
- inherited Put( job );
- finally
- LeaveCriticalSection( cs );
- end;
-end;
-
-function TMutexJobQ.Get : PJob;
-begin
- Result := Nil;
- EnterCriticalSection( cs );
- try
- Result := inherited Get;
- finally
- LeaveCriticalSection( cs );
- end
-end;
-
-function TMutexJobQ.GetOrMake: PJob;
-begin
- Result := Nil;
- EnterCriticalSection( cs );
- try
- Result := inherited GetOrMake;
- finally
- LeaveCriticalSection( cs );
- end
-end;
-
-procedure TMutexJobQ.Flush( dst : TJobQ );
-begin
- EnterCriticalSection( cs );
- try
- inherited Flush( dst );
- finally
- LeaveCriticalSection( cs );
- end
-end;
-
-end.
-
diff --git a/tools/vdbview/logo_128.png b/tools/vdbview/logo_128.png
deleted file mode 100644
index 69c0aa6..0000000
Binary files a/tools/vdbview/logo_128.png and /dev/null differ
diff --git a/tools/vdbview/logo_16.png b/tools/vdbview/logo_16.png
deleted file mode 100644
index 1989f95..0000000
Binary files a/tools/vdbview/logo_16.png and /dev/null differ
diff --git a/tools/vdbview/logo_256.png b/tools/vdbview/logo_256.png
deleted file mode 100644
index 1db3949..0000000
Binary files a/tools/vdbview/logo_256.png and /dev/null differ
diff --git a/tools/vdbview/logo_32.png b/tools/vdbview/logo_32.png
deleted file mode 100644
index 4d74dab..0000000
Binary files a/tools/vdbview/logo_32.png and /dev/null differ
diff --git a/tools/vdbview/logo_64.png b/tools/vdbview/logo_64.png
deleted file mode 100644
index 762d22d..0000000
Binary files a/tools/vdbview/logo_64.png and /dev/null differ
diff --git a/tools/vdbview/mainwindow.lfm b/tools/vdbview/mainwindow.lfm
deleted file mode 100644
index 8b2135e..0000000
--- a/tools/vdbview/mainwindow.lfm
+++ /dev/null
@@ -1,973 +0,0 @@
-object Mainform: TMainform
- Left = 367
- Height = 665
- Top = 131
- Width = 641
- ActiveControl = Grid
- Caption = 'vdbview'
- ClientHeight = 646
- ClientWidth = 641
- DragMode = dmAutomatic
- Menu = MainMenu
- OnActivate = FormActivate
- OnClose = FormClose
- OnCreate = FormCreate
- OnDestroy = FormDestroy
- OnResize = FormResize
- LCLVersion = '0.9.30'
- object Grid: TDrawGrid
- Left = 0
- Height = 603
- Top = 23
- Width = 625
- Align = alClient
- AutoEdit = False
- BorderStyle = bsNone
- ExtendedSelect = False
- Flat = True
- Font.CharSet = ANSI_CHARSET
- Font.Height = -13
- Font.Name = 'Consolas'
- Font.Pitch = fpFixed
- Font.Quality = fqDraft
- Options = [goFixedVertLine, goFixedHorzLine, goVertLine, goHorzLine, goRangeSelect, goDrawFocusSelected, goRowSizing, goColSizing, goSmoothScroll]
- ParentFont = False
- PopupMenu = PopupMenu1
- ScrollBars = ssHorizontal
- TabOrder = 0
- TitleFont.CharSet = ANSI_CHARSET
- TitleFont.Height = -13
- TitleFont.Name = 'Consolas'
- TitleFont.Pitch = fpFixed
- TitleFont.Quality = fqDraft
- UseXORFeatures = True
- OnDrawCell = GridDrawCell
- OnHeaderSized = GridHeaderSized
- OnKeyDown = GridKeyDown
- OnMouseDown = GridMouseDown
- OnMouseWheelDown = GridMouseWheelDown
- OnMouseWheelUp = GridMouseWheelUp
- OnSelectCell = GridSelectCell
- end
- object Panel1: TPanel
- Left = 0
- Height = 23
- Top = 0
- Width = 641
- Align = alTop
- ClientHeight = 23
- ClientWidth = 641
- TabOrder = 1
- object ActivateRecord: TSpeedButton
- Left = 594
- Height = 22
- Top = 0
- Width = 47
- Action = A_ActivateRecord
- Anchors = [akTop, akRight]
- GroupIndex = 1
- NumGlyphs = 0
- ShowHint = True
- ParentShowHint = False
- end
- object ActivateGrid: TSpeedButton
- Left = 546
- Height = 22
- Top = 0
- Width = 48
- Action = A_ActivateGrid
- Anchors = [akTop, akRight]
- Down = True
- GroupIndex = 1
- NumGlyphs = 0
- ShowHint = True
- ParentShowHint = False
- end
- object Tables: TComboBox
- Left = 0
- Height = 21
- Hint = 'select table'
- Top = 1
- Width = 144
- Color = 33023
- ItemHeight = 13
- OnSelect = TablesSelect
- ParentShowHint = False
- ShowHint = True
- Style = csDropDownList
- TabOrder = 0
- end
- object LED: TShape
- Left = 152
- Height = 13
- Top = 5
- Width = 23
- Brush.Color = clDefault
- Pen.Style = psClear
- end
- object B_First: TSpeedButton
- Left = 180
- Height = 22
- Top = 0
- Width = 23
- Action = A_First
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F2FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FEFF0062F4FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF005EECFF000308FF003E9CFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0057D9FF00173AFF00040BFF0062F4FF0066FFFF0066FFFF0066FFFF0066
- FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0065FDFF00388DFF000409FF0001
- 03FF003079FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF0000
- 00FF00337FFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0057DAFF00183BFF000000FF001029FF0050C9FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0065FDFF00398EFF0004
- 09FF000103FF003077FF0063F8FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0057DAFF00183BFF000000FF001029FF0050C8FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0065FDFF0039
- 8FFF000409FF000103FF002F76FF0063F8FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF0000
- 00FF00337FFF0066FFFF0066FFFF0058DBFF00183CFF000000FF001028FF0050
- C8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066
- FFFF004EC4FF00040AFF000103FF002F75FF0063F7FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF004FC6FF0004
- 0AFF000103FF002F75FF0063F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0058DBFF00183CFF0000
- 00FF001028FF0050C8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF0000
- 00FF00337FFF0066FFFF0066FFFF0066FFFF0065FDFF00398FFF000409FF0001
- 03FF002F76FF0063F8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0057DAFF00183BFF000000FF0010
- 29FF0050C8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0065FDFF00398EFF000409FF000103FF0030
- 77FF0063F8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0057DAFF00183BFF000000FF001029FF0050
- C9FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF0000
- 00FF00337FFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0065FDFF00388DFF000409FF000103FF00327EFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF005CE7FF00040AFF003E9CFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0057D9FF00173AFF000308FF0061F3FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0061F2FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0065FDFF0061F3FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Prev: TSpeedButton
- Left = 203
- Height = 22
- Top = 0
- Width = 23
- Action = A_PageUp
- Caption = '<<'
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0061F3FF0066FFFF0066FFFF0066FEFF0062F4FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0062F6FF002E72FF000102FF0043
- A7FF0057D9FF00173AFF00040BFF0062F4FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF004FC6FF000F26FF000000FF00193EFF00327FFF000409FF0001
- 03FF003079FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F7FF002E73FF0001
- 02FF00040AFF00327DFF00183BFF000000FF001029FF0050C9FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF004FC6FF001027FF000000FF00193EFF00317BFF0004
- 09FF000103FF003077FF0063F8FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F7FF002E
- 74FF000102FF00040AFF00317CFF00183BFF000000FF001029FF0050C8FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0050C7FF001027FF000000FF00183DFF0031
- 7BFF000409FF000103FF002F76FF0063F8FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063
- F7FF002F75FF000102FF00040AFF00327CFF00183CFF000000FF001028FF0050
- C8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0064FAFF00122CFF000000FF0018
- 3CFF0043A8FF00040AFF000103FF002F75FF0063F7FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0064F9FF00122EFF000000FF00183CFF0044AAFF0004
- 0AFF000103FF002F75FF0063F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0063F7FF002F75FF000102FF00040AFF00327CFF00183CFF0000
- 00FF001028FF0050C8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0050C7FF001027FF000000FF00183DFF00317BFF000409FF0001
- 03FF002F76FF0063F8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0063F7FF002E74FF000102FF00040AFF00317CFF00183BFF000000FF0010
- 29FF0050C8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF004FC6FF001027FF000000FF00193EFF00317BFF000409FF000103FF0030
- 77FF0063F8FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063
- F7FF002E73FF000102FF00040AFF00327DFF00183BFF000000FF001029FF0050
- C9FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF004F
- C6FF000F26FF000000FF00193EFF003482FF000409FF000103FF00327EFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0062F6FF002E
- 72FF000102FF0042A4FF0057D9FF00173AFF000308FF0061F3FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0060F1FF0066
- FFFF0066FFFF0065FDFF0061F3FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Up: TSpeedButton
- Left = 226
- Height = 22
- Top = 0
- Width = 23
- Action = A_Up
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FEFF0062F4FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FEFF0044ABFF00122DFF000409FF0061F2FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FEFF0044ABFF00122CFF000000FF0006
- 0FFF003585FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FEFF0044AAFF00122CFF000000FF00060FFF00337FFF0060EFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FEFF0044A9FF00122CFF000000FF0006
- 0FFF00327EFF0060EFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0065
- FDFF0044A9FF00112BFF000000FF00060EFF00327EFF0060EFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0065FDFF0043A8FF00112BFF000000FF0006
- 0EFF00327DFF005FEEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0065
- FDFF0043A8FF00112AFF000000FF00060EFF00327CFF005FEEFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0064FBFF00132FFF000000FF0006
- 0EFF00327CFF005FEEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0063F7FF001534FF000000FF00060EFF00327CFF005F
- EEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0065FDFF0043A8FF00112AFF000000FF00060EFF00327CFF005F
- EEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0065FDFF0043A8FF00112BFF000000FF00060EFF00327DFF005F
- EEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0065FDFF0044A9FF00112BFF000000FF00060EFF00327EFF0060
- EFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FEFF0044A9FF00122CFF000000FF00060FFF00327EFF0060
- EFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FEFF0044AAFF00122CFF000000FF00060FFF00337FFF0060
- EFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FEFF0044ABFF00122CFF000000FF00060FFF003789FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FEFF0044ABFF00122DFF000308FF0061F3FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FEFF0060F0FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Down: TSpeedButton
- Left = 249
- Height = 22
- Top = 0
- Width = 23
- Action = A_Down
- Caption = '>'
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0060F0FF0066FEFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0061F3FF000308FF00122DFF0044ABFF0066FEFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF003789FF00060FFF000000FF00122CFF0044ABFF0066FEFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0060EFFF00337FFF00060FFF000000FF00122CFF0044AAFF0066FEFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0060EFFF00327EFF00060FFF000000FF00122CFF0044A9FF0066FEFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0060EFFF00327EFF00060EFF000000FF00112BFF0044A9FF0065FDFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF005FEEFF00327DFF00060EFF000000FF00112BFF0043A8FF0065FDFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF005FEEFF00327CFF00060EFF000000FF00112AFF0043A8FF0065FDFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF005FEEFF00327CFF00060EFF000000FF001534FF0063F7FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF005FEEFF0032
- 7CFF00060EFF000000FF00132FFF0064FBFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF005FEEFF00327CFF00060EFF000000FF00112AFF0043
- A8FF0065FDFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF005FEEFF0032
- 7DFF00060EFF000000FF00112BFF0043A8FF0065FDFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0060EFFF00327EFF00060EFF000000FF00112BFF0044
- A9FF0065FDFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0060EFFF0032
- 7EFF00060FFF000000FF00122CFF0044A9FF0066FEFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0060EFFF00337FFF00060FFF000000FF00122CFF0044
- AAFF0066FEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0035
- 85FF00060FFF000000FF00122CFF0044ABFF0066FEFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F2FF000409FF00122DFF0044
- ABFF0066FEFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0062F4FF0066FEFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Next: TSpeedButton
- Left = 272
- Height = 22
- Top = 0
- Width = 23
- Action = A_PageDn
- Caption = '>>'
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F3FF0065FDFF0066
- FFFF0066FFFF0060F1FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0061F3FF000308FF00173AFF0057D9FF0042A4FF0001
- 02FF002E72FF0062F6FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF00327EFF000103FF000409FF003482FF00193EFF000000FF000F
- 26FF004FC6FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0050C9FF001029FF000000FF00183BFF00327DFF00040AFF000102FF002E
- 73FF0063F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063
- F8FF003077FF000103FF000409FF00317BFF00193EFF000000FF001027FF004F
- C6FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0050
- C8FF001029FF000000FF00183BFF00317CFF00040AFF000102FF002E74FF0063
- F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F8FF002F
- 76FF000103FF000409FF00317BFF00183DFF000000FF001027FF0050C7FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0050C8FF0010
- 28FF000000FF00183CFF00327CFF00040AFF000102FF002F75FF0063F7FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F7FF002F75FF0001
- 03FF00040AFF0044AAFF00183CFF000000FF00122EFF0064F9FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0063F7FF002F75FF000103FF00040AFF0043
- A8FF00183CFF000000FF00122CFF0064FAFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0050C8FF001028FF000000FF00183CFF00327CFF00040AFF000102FF002F
- 75FF0063F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F8FF002F76FF000103FF0004
- 09FF00317BFF00183DFF000000FF001027FF0050C7FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0050C8FF001029FF000000FF00183BFF00317CFF00040AFF0001
- 02FF002E74FF0063F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F8FF003077FF0001
- 03FF000409FF00317BFF00193EFF000000FF001027FF004FC6FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0050C9FF001029FF000000FF00183BFF00327DFF0004
- 0AFF000102FF002E73FF0063F7FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0030
- 79FF000103FF000409FF00327FFF00193EFF000000FF000F26FF004FC6FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0062F4FF00040BFF00173AFF0057
- D9FF0043A7FF000102FF002E72FF0062F6FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0062F4FF0066FEFF0066FFFF0066FFFF0061
- F3FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Last: TSpeedButton
- Left = 295
- Height = 22
- Top = 0
- Width = 23
- Action = A_Last
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F3FF0065FDFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F2FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0061F3FF000308FF00173AFF0057D9FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF003E9CFF00040AFF005CE7FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF00327EFF000103FF000409FF00388DFF0065FDFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0033
- 7FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0050C9FF001029FF000000FF00183BFF0057DAFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056
- D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063
- F8FF003077FF000103FF000409FF00398EFF0065FDFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0050
- C8FF001029FF000000FF00183BFF0057DAFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F8FF002F
- 76FF000103FF000409FF00398FFF0065FDFF0066FFFF0066FFFF0066FFFF0033
- 7FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0050C8FF0010
- 28FF000000FF00183CFF0058DBFF0066FFFF0066FFFF00337FFF000000FF0056
- D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F7FF002F75FF0001
- 03FF00040AFF004FC6FF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0063F7FF002F75FF000103FF00040AFF004E
- C4FF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0050C8FF001028FF000000FF00183CFF0058DBFF0066FFFF0066FFFF0033
- 7FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F8FF002F76FF000103FF0004
- 09FF00398FFF0065FDFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056
- D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0050C8FF001029FF000000FF00183BFF0057DAFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0063F8FF003077FF0001
- 03FF000409FF00398EFF0065FDFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0050C9FF001029FF000000FF00183BFF0057DAFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0033
- 7FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0030
- 79FF000103FF000409FF00388DFF0065FDFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056
- D7FF0066FFFF0066FFFF0066FFFF0066FFFF0062F4FF00040BFF00173AFF0057
- D9FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF003E9CFF000308FF005EECFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0062F4FF0066FEFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0061F2FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Search: TSpeedButton
- Left = 336
- Height = 22
- Top = 0
- Width = 23
- Action = A_Search
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0060F1FF0065FDFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0061F2FF00040AFF002255FF0064F9FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF00245BFF000000FF001C47FF0062F5FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0065
- FDFF002A69FF000000FF00173AFF0060EFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0031
- 7AFF000000FF00122EFF005CE7FF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF00388CFF0000
- 00FF000E23FF0059DFFF0066FFFF004EC3FF00388BFF00265FFF00317BFF0046
- AFFF0062F4FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF003F9DFF000103FF0009
- 17FF00193EFF000000FF000000FF000000FF000000FF000000FF000D20FF004C
- BFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF003E9BFF000001FF000000FF0029
- 66FF004EC3FF0060F1FF0055D4FF003A90FF00060EFF000001FF0044AAFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF001E4AFF000000FF003C96FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF005AE1FF000C1EFF000712FF005CE7FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF004E
- C3FF000000FF002966FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF004AB8FF000000FF002E73FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF00388BFF000000FF004E
- C3FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF000814FF00183BFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF00265FFF000000FF0060F1FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF001A41FF0006
- 0FFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF00317BFF000000FF0055D4FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF000E24FF00112BFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0046
- AFFF000000FF003A90FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF005AE0FF000000FF00265FFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0062F4FF000D20FF0006
- 0EFF005AE1FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF001C
- 47FF000001FF004EC3FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF004CBFFF000001FF000C1EFF004A
- B8FF0066FFFF0066FFFF0066FFFF005AE0FF001C47FF000000FF002D71FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0044AAFF000712FF000000FF000814FF001A
- 41FF000E24FF000000FF000001FF002D71FF0064F9FF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF005CE7FF002E73FF00183BFF00060FFF00112BFF0026
- 5FFF004EC3FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- object B_Goto: TSpeedButton
- Left = 359
- Height = 22
- Top = 0
- Width = 23
- Action = A_Goto
- Glyph.Data = {
- C6070000424DC607000000000000360000002800000016000000160000000100
- 20000000000090070000640000006400000000000000000000000066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0061F2FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0061F2FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF003E9CFF0004
- 0AFF005CE7FF0066FFFF0066FFFF0066FFFF0066FFFF005EECFF000308FF003E
- 9CFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066
- FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066
- FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF003E9CFF00337FFF0033
- 7FFF00193FFF000000FF002B6BFF00337FFF00337FFF00337FFF00337FFF002B
- 6BFF000000FF00193FFF00337FFF00337FFF003E9CFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0061F2FF000308FF000000FF000000FF000000FF0000
- 00FF000000FF000000FF000000FF000000FF000000FF000000FF000000FF0000
- 00FF000000FF000000FF00040AFF0061F2FF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF005EECFF0056D7FF0056D7FF002B6BFF000000FF0049B5FF0056
- D7FF0056D7FF0056D7FF0056D7FF0049B5FF000000FF002B6BFF0056D7FF0056
- D7FF005CE7FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066
- FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066FFFF0066FFFF0056
- D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF00337FFF0000
- 00FF0056D7FF0066FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF0033
- 7FFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066
- FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF005C
- E7FF0056D7FF0056D7FF002B6BFF000000FF0049B5FF0056D7FF0056D7FF0056
- D7FF0056D7FF0049B5FF000000FF002B6BFF0056D7FF0056D7FF005EECFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F2FF00040AFF000000FF0000
- 00FF000000FF000000FF000000FF000000FF000000FF000000FF000000FF0000
- 00FF000000FF000000FF000000FF000000FF000308FF0061F2FF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF003E9CFF00337FFF00337FFF00193FFF0000
- 00FF002B6BFF00337FFF00337FFF00337FFF00337FFF002B6BFF000000FF0019
- 3FFF00337FFF00337FFF003E9CFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066
- FFFF0066FFFF0066FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF00337FFF000000FF0056D7FF0066FFFF0066FFFF0066
- FFFF0066FFFF0056D7FF000000FF00337FFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF003E9CFF000308FF005EECFF0066FFFF0066FFFF0066FFFF0066FFFF005C
- E7FF00040AFF003E9CFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061
- F2FF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0061F2FF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066FFFF0066
- FFFF0066FFFF0066FFFF
- }
- NumGlyphs = 0
- Transparent = False
- ShowCaption = False
- ShowHint = True
- ParentShowHint = False
- end
- end
- object RowScroll: TScrollBar
- Left = 625
- Height = 603
- Top = 23
- Width = 16
- Align = alRight
- Kind = sbVertical
- Max = 20
- PageSize = 0
- TabOrder = 2
- TabStop = False
- OnChange = RowScrollChange
- end
- object StatusBar: TStatusBar
- Left = 0
- Height = 20
- Top = 626
- Width = 641
- Panels = <
- item
- Alignment = taCenter
- Width = 120
- end
- item
- Alignment = taCenter
- Width = 120
- end
- item
- Width = 50
- end>
- SimplePanel = False
- end
- object MainMenu: TMainMenu
- left = 472
- top = 96
- object MenueFile: TMenuItem
- Caption = '&File'
- object MI_Open_Accession: TMenuItem
- Action = A_OpenAcc
- end
- object MI_Open_Directory: TMenuItem
- Action = A_OpenDir
- end
- object MI_Open_File: TMenuItem
- Action = A_OpenFile
- end
- object MI_Recent: TMenuItem
- Caption = 'Open Recent'
- end
- object MenuItem1: TMenuItem
- Action = A_ClearHistory
- end
- object MI_Exit: TMenuItem
- Action = A_Exit
- end
- end
- object MenueView: TMenuItem
- Caption = '&View'
- object MenuGrid: TMenuItem
- Action = A_ActivateGrid
- AutoCheck = True
- end
- object MenuRecord: TMenuItem
- Action = A_ActivateRecord
- AutoCheck = True
- end
- object MenuColumns: TMenuItem
- Action = A_Columns
- end
- end
- object MenuItem2: TMenuItem
- Action = A_Version
- end
- end
- object ActionList: TActionList
- left = 472
- top = 48
- object A_ActivateGrid: TAction
- AutoCheck = True
- Caption = '&grid'
- Checked = True
- GroupIndex = 1
- Hint = 'grid view'
- OnExecute = A_ActivateGridExecute
- end
- object A_ActivateRecord: TAction
- AutoCheck = True
- Caption = '&record'
- GroupIndex = 1
- Hint = 'record view'
- OnExecute = A_ActivateRecordExecute
- end
- object A_Exit: TAction
- Caption = 'E&xit'
- OnExecute = A_ExitExecute
- ShortCut = 16472
- end
- object A_OpenAcc: TAction
- Caption = 'Open &Accession'
- OnExecute = A_OpenAccExecute
- ShortCut = 16449
- end
- object A_OpenFile: TAction
- Caption = 'Open &File'
- OnExecute = A_OpenFileExecute
- ShortCut = 16454
- end
- object A_OpenDir: TAction
- Caption = 'Open &Directory'
- OnExecute = A_OpenDirExecute
- ShortCut = 16452
- end
- object A_Columns: TAction
- Caption = '&columns...'
- OnExecute = A_ColumnsExecute
- ShortCut = 32835
- end
- object A_Up: TAction
- Caption = 'Up'
- Hint = 'previous row'
- OnExecute = A_UpExecute
- end
- object A_Down: TAction
- Caption = 'Down'
- Hint = 'next row'
- OnExecute = A_DownExecute
- end
- object A_PageUp: TAction
- Caption = 'Page Up'
- Hint = 'previous page'
- OnExecute = A_PageUpExecute
- end
- object A_PageDn: TAction
- Caption = 'Page Down'
- Hint = 'next page'
- OnExecute = A_PageDnExecute
- end
- object A_First: TAction
- Caption = 'First'
- Hint = 'first row'
- OnExecute = A_FirstExecute
- end
- object A_Last: TAction
- Caption = 'Last'
- Hint = 'last row'
- OnExecute = A_LastExecute
- end
- object A_Goto: TAction
- Caption = 'Goto...'
- Hint = 'goto row'
- OnExecute = A_GotoExecute
- ShortCut = 16455
- end
- object A_Search: TAction
- Caption = '&Find'
- Hint = 'Search'
- OnExecute = A_SearchExecute
- ShortCut = 16467
- end
- object A_AdjustHeader: TAction
- Caption = 'Adjust Header'
- OnExecute = A_AdjustHeaderExecute
- end
- object A_Adjust_All_Headers: TAction
- Caption = 'Adjust All Headers'
- OnExecute = A_Adjust_All_HeadersExecute
- end
- object A_AdjustCellWidth: TAction
- Caption = 'Adjust CellWidth'
- OnExecute = A_AdjustCellWidthExecute
- end
- object A_Adjust_All_Cellwidths: TAction
- Caption = 'Adjust All Cellwidths'
- OnExecute = A_Adjust_All_CellwidthsExecute
- end
- object A_Copy: TAction
- Caption = 'Copy to Clipboard'
- OnExecute = A_CopyExecute
- ShortCut = 16451
- end
- object A_ClearHistory: TAction
- Caption = 'Clear History'
- OnExecute = A_ClearHistoryExecute
- end
- object A_Version: TAction
- Caption = 'About'
- OnExecute = A_VersionExecute
- end
- end
- object cfg: TXMLConfig
- Filename = 'vdbview.cfg'
- StartEmpty = False
- RootName = 'CONFIG'
- left = 344
- top = 48
- end
- object SelDirDlg: TSelectDirectoryDialog
- Options = [ofReadOnly, ofPathMustExist, ofFileMustExist, ofCreatePrompt, ofNoTestFileCreate, ofEnableSizing, ofViewDetail]
- left = 408
- top = 49
- end
- object SelFileDlg: TOpenDialog
- Options = [ofReadOnly, ofNoValidate, ofEnableSizing, ofViewDetail]
- left = 408
- top = 96
- end
- object LEDTimer: TTimer
- Enabled = False
- Interval = 300
- OnTimer = LEDTimerTimer
- left = 344
- top = 96
- end
- object PopupMenu1: TPopupMenu
- left = 536
- top = 96
- object MenuItem4: TMenuItem
- Action = A_Goto
- end
- object MenuItem5: TMenuItem
- Action = A_Columns
- end
- object MenuItem6: TMenuItem
- Action = A_Search
- end
- object MenuItem7: TMenuItem
- Action = A_AdjustHeader
- end
- object MenuItem8: TMenuItem
- Action = A_Adjust_All_Headers
- end
- object MenuItem9: TMenuItem
- Action = A_AdjustCellWidth
- end
- object MenuItem10: TMenuItem
- Action = A_Adjust_All_Cellwidths
- end
- object MenuItem11: TMenuItem
- Action = A_Copy
- end
- end
- object ApplicationProperties1: TApplicationProperties
- left = 537
- top = 49
- end
-end
diff --git a/tools/vdbview/mainwindow.pas b/tools/vdbview/mainwindow.pas
deleted file mode 100644
index 19b66b0..0000000
--- a/tools/vdbview/mainwindow.pas
+++ /dev/null
@@ -1,793 +0,0 @@
-unit MainWindow;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, XMLCfg, FileUtil, Forms, Controls, Graphics, Dialogs,
- ComCtrls, ExtCtrls, Grids, StdCtrls, Buttons, Menus, ActnList, LCLType,
- support_unit, math, selector_types, DataProxy, columnSel, gotoform,
- searchform, clipbrd, types, version;
-
-type
-
- { TMainform }
-
- TMainform = class(TForm)
- A_Version: TAction;
- A_ClearHistory: TAction;
- ApplicationProperties1: TApplicationProperties;
- A_Copy: TAction;
- A_Adjust_All_Cellwidths: TAction;
- A_AdjustCellWidth: TAction;
- A_Adjust_All_Headers: TAction;
- A_AdjustHeader: TAction;
- A_Search: TAction;
- A_Goto: TAction;
- A_First: TAction;
- A_Last: TAction;
- A_PageDn: TAction;
- A_PageUp: TAction;
- A_Up: TAction;
- A_Down: TAction;
- A_Columns: TAction;
- A_OpenDir: TAction;
- A_OpenFile: TAction;
- A_OpenAcc: TAction;
- A_Exit: TAction;
- A_ActivateRecord: TAction;
- A_ActivateGrid: TAction;
- ActionList: TActionList;
- LED: TShape;
- MenuColumns: TMenuItem;
- MenuItem1: TMenuItem;
- MenuItem10: TMenuItem;
- MenuItem11: TMenuItem;
- MenuItem2: TMenuItem;
- MI_Recent: TMenuItem;
- MenuItem4: TMenuItem;
- MenuItem5: TMenuItem;
- MenuItem6: TMenuItem;
- MenuItem7: TMenuItem;
- MenuItem8: TMenuItem;
- MenuItem9: TMenuItem;
- PopupMenu1: TPopupMenu;
- B_First: TSpeedButton;
- B_Prev: TSpeedButton;
- B_Up: TSpeedButton;
- B_Down: TSpeedButton;
- B_Next: TSpeedButton;
- B_Last: TSpeedButton;
- B_Search: TSpeedButton;
- B_Goto: TSpeedButton;
- Tables: TComboBox;
- Grid: TDrawGrid;
- MainMenu: TMainMenu;
- MenueFile: TMenuItem;
- MenueView: TMenuItem;
- MI_Exit: TMenuItem;
- MenuGrid: TMenuItem;
- MI_Open_Accession: TMenuItem;
- MI_Open_File: TMenuItem;
- MI_Open_Directory: TMenuItem;
- MenuRecord: TMenuItem;
- SelFileDlg: TOpenDialog;
- Panel1: TPanel;
- RowScroll: TScrollBar;
- ActivateRecord: TSpeedButton;
- ActivateGrid: TSpeedButton;
- cfg: TXMLConfig;
- SelDirDlg: TSelectDirectoryDialog;
- StatusBar: TStatusBar;
- LEDTimer: TTimer;
- procedure A_AdjustCellWidthExecute(Sender: TObject);
- procedure A_AdjustHeaderExecute(Sender: TObject);
- procedure A_Adjust_All_CellwidthsExecute(Sender: TObject);
- procedure A_Adjust_All_HeadersExecute(Sender: TObject);
- procedure A_ClearHistoryExecute(Sender: TObject);
- procedure A_CopyExecute(Sender: TObject);
- procedure A_SearchExecute(Sender: TObject);
- procedure A_ActivateGridExecute(Sender: TObject);
- procedure A_ActivateRecordExecute(Sender: TObject);
- procedure A_ColumnsExecute(Sender: TObject);
- procedure A_DownExecute(Sender: TObject);
- procedure A_ExitExecute(Sender: TObject);
- procedure A_FirstExecute(Sender: TObject);
- procedure A_GotoExecute(Sender: TObject);
- procedure A_LastExecute(Sender: TObject);
- procedure A_OpenAccExecute(Sender: TObject);
- procedure A_OpenDirExecute(Sender: TObject);
- procedure A_OpenFileExecute(Sender: TObject);
- procedure A_PageDnExecute(Sender: TObject);
- procedure A_PageUpExecute(Sender: TObject);
- procedure A_UpExecute(Sender: TObject);
- procedure A_VersionExecute(Sender: TObject);
- procedure FormActivate(Sender: TObject);
- procedure FormClose(Sender: TObject; var CloseAction: TCloseAction);
- procedure FormCreate(Sender: TObject);
- procedure FormDestroy(Sender: TObject);
- procedure FormResize(Sender: TObject);
- procedure GridDrawCell(Sender: TObject; aCol, aRow: Integer; aRect: TRect;
- aState: TGridDrawState);
- procedure GridHeaderSized(Sender: TObject; IsColumn: Boolean; Index: Integer
- );
- procedure GridKeyDown(Sender: TObject; var Key: Word; Shift: TShiftState);
- procedure GridMouseDown(Sender: TObject; Button: TMouseButton;
- Shift: TShiftState; X, Y: Integer);
- procedure GridMouseWheelDown(Sender: TObject; Shift: TShiftState;
- MousePos: TPoint; var Handled: Boolean);
- procedure GridMouseWheelUp(Sender: TObject; Shift: TShiftState;
- MousePos: TPoint; var Handled: Boolean);
- procedure GridSelectCell(Sender: TObject; aCol, aRow: Integer;
- var CanSelect: Boolean);
- procedure LEDTimerTimer(Sender: TObject);
- procedure RowScrollChange(Sender: TObject);
- procedure TablesSelect(Sender: TObject);
- private
- { private declarations }
- fProxy : TProxy;
- fGridRow : Integer;
- fGridCol : Integer;
- fRecRow : Integer;
- fRecCol : Integer;
- fSearch : String;
- fDrawCtx : TDrawCtx;
- fMouseShift : TShiftState;
- fColWidths : TStringList;
-
- function CopyGrid: String;
- function CopyRecord: String;
- procedure DrawDataCell( aCol, aRow : Integer; ct: TCellType; aRect: TRect );
- procedure DrawRecordCell( aRow : Integer; ct: TCellType; aRect: TRect );
- procedure GotoRow( const aRow : Integer );
- procedure OnLRUClick( Sender : TObject );
- procedure OnOpened( const path : String; success : boolean );
- procedure OnTableSwitched;
- procedure OnCellInvalidate( const aCol, aRow : Integer );
- procedure OnLED( state : boolean );
- procedure OnSearchDone( const aCol, aRow : Integer;
- const pattern : String );
- procedure PerformOpen( const path, acc : String );
- procedure AdjustColWidth( aCol : Integer; header : boolean );
- procedure SaveColWidths;
- procedure RestoreColWidths;
- public
- { public declarations }
- end;
-
-var
- Mainform: TMainform;
-
-implementation
-
-{$R *.lfm}
-
-{ TMainform }
-
-procedure TMainform.FormCreate(Sender: TObject);
-var s : String;
-begin
- s := ExtractFilePath( Application.ExeName );
- cfg.Filename := s + '\' + cfg.Filename;
- SelDirDlg.InitialDir := s;
- SelFileDlg.InitialDir := s;
- fColWidths := TStringList.Create;
- fProxy := TProxy.Create;
- fProxy.OnOpened := @OnOpened;
- fProxy.OnTableSwitched := @OnTableSwitched;
- fProxy.OnCellValid := @OnCellInvalidate;
- fProxy.OnLED := @OnLED;
- fProxy.OnSearchDone := @OnSearchDone;
- fGridRow := 1;
- fGridCol := 1;
- fRecRow := 1;
- fRecCol := 1;
- fSearch := '';
- A_ActivateGrid.Execute;
-end;
-
-procedure TMainform.FormDestroy(Sender: TObject);
-begin
- fProxy.Free;
- fColWidths.Free;
-end;
-
-procedure TMainform.FormActivate( Sender : TObject );
-begin
- setup_position( Mainform, cfg, Screen ); {support_unit}
- PopulateLRUMenue( cfg, MainMenu, MI_Recent, @OnLRUClick );
- fDrawCtx.h := Grid.Canvas.TextHeight( 'X' );
- fDrawCtx.w := Grid.Canvas.TextWidth( 'W' );
- fDrawCtx.g := Grid;
- if ( ParamCount > 0 )
- then PerformOpen( ParamStr( 1 ), '' );
-end;
-
-procedure TMainform.FormClose(Sender: TObject; var CloseAction: TCloseAction);
-begin
- store_position( Mainform, cfg ); {support_unit}
-end;
-
-procedure TMainform.FormResize( Sender : TObject );
-var n : Integer;
-begin
- if ( ActivateGrid.Down )
- then begin
- adjust_gridrowcount( Grid, fProxy ); {support_unit}
- n := fProxy.TableRows - Grid.RowCount + 1;
- if ( n < Grid.RowCount )
- then n := Grid.RowCount;
- RowScroll.Max := n;
- end
- else Grid.ColWidths[ 1 ] := Grid.ClientWidth - Grid.ColWidths[ 0 ];
-end;
-
-procedure TMainform.A_ActivateGridExecute( Sender : TObject );
-begin
- { save the row/col for record-view }
- fRecRow := Grid.Row;
- fRecCol := Grid.Col;
- adjust_gridrowcount( Grid, fProxy ); {support_unit}
- Grid.ColCount := fProxy.ColCount[ VISIBLE_COLUMNS ] + 1;
- Grid.ScrollBars := ssHorizontal;
- RowScroll.Visible := True;
- { restore row/col for grid-view }
- Grid.Row := fGridRow;
- Grid.Col := fGridCol;
- Grid.ColWidths[ 0 ] := 96;
- A_Adjust_All_HeadersExecute( Sender );
- RestoreColWidths;
- ActivateGrid.Down := True;
-end;
-
-procedure TMainform.A_SearchExecute( Sender : TObject );
-var aCol, aRow : Integer;
- p : TPoint;
-begin
- if ActivateGrid.Down
- then begin
- aCol := Grid.Col - 1;
- aRow := Grid.Row + RowScroll.Position - 1;
- end
- else begin
- aCol := Grid.Row - 1;
- aRow := fRecRow;
- end;
- if ( Length( fSearch ) < 1 )
- then fSearch := fProxy.Cell[ aCol, aRow ];
- p := Point( Grid.Left, Grid.Top );
- SForm.present( ClientToScreen( p ), aCol, aRow, fSearch, fProxy );
-end;
-
-procedure TMainform.SaveColWidths;
-var i, idx : Integer;
- s : String;
-begin
- for i := 1 to Grid.ColCount - 1
- do begin
- s := fProxy.ColName[ VISIBLE_COLUMNS, i - 1 ];
- idx := fColWidths.IndexOf( s );
- if ( idx < 0 )
- then begin
- fColWidths.Add( s );
- idx := fColWidths.IndexOf( s );
- end;
- if ( idx >= 0 )
- then fColWidths.Objects[ idx ] := TObject( Grid.ColWidths[ i ] );
- end;
-end;
-
-procedure TMainform.RestoreColWidths;
-var i, idx : Integer;
-begin
- if ( ActivateGrid.Down )
- then for i := 1 to Grid.ColCount - 1
- do begin
- idx := fColWidths.IndexOf( fProxy.ColName[ VISIBLE_COLUMNS, i - 1 ] );
- if ( idx >= 0 )
- then Grid.ColWidths[ i ] := Integer( fColWidths.Objects[ idx ] );
- end;
-end;
-
-procedure TMainform.A_AdjustHeaderExecute( Sender : TObject );
-begin
- if ActivateGrid.Down
- then AdjustColWidth( Grid.Col, true );
-end;
-
-procedure TMainform.A_Adjust_All_CellwidthsExecute(Sender: TObject);
-var i : Integer;
-begin
- if ActivateGrid.Down
- then for i:= 1 to Grid.ColCount - 1
- do AdjustColWidth( i, false );
-end;
-
-procedure TMainform.A_AdjustCellWidthExecute( Sender : TObject );
-begin
- if ActivateGrid.Down
- then AdjustColWidth( Grid.Col, false );
-end;
-
-procedure TMainform.A_Adjust_All_HeadersExecute( Sender : TObject );
-var i : Integer;
-begin
- if ActivateGrid.Down
- then for i:= 1 to Grid.ColCount - 1
- do AdjustColWidth( i, true );
-end;
-
-procedure TMainform.A_ClearHistoryExecute( Sender : TObject );
-begin
- ClearLRU( cfg, MI_Recent ); {support_unit}
-end;
-
-function TMainform.CopyGrid : String;
-var gr : TGridRect;
- row, col, n : Integer;
-begin
- Result := '';
- gr := Grid.Selection;
- for row := gr.Top to gr.Bottom
- do begin
- if ( row > gr.Top )
- then Result := Result + #13;
- for col := gr.Left to gr.Right
- do begin
- if ( col > gr.Left )
- then Result := Result + #9;
- if ( col = 0 )
- then begin
- n := row + RowScroll.Position;
- if ( n <= RowScroll.Max )
- then Result := Result + FloatToStrF( n, ffnumber, 0, 0 );
- end
- else Result := Result + fProxy.Cell[ col - 1, row - 1 ];
- end;
- end;
-end;
-
-function TMainform.CopyRecord : String;
-var gr : TGridRect;
- row, col : Integer;
-begin
- Result := '';
- gr := Grid.Selection;
- for row := gr.Top to gr.Bottom
- do begin
- if ( row > gr.Top )
- then Result := Result + #13;
- for col := gr.Left to gr.Right
- do begin
- if ( col > gr.Left )
- then Result := Result + #9;
- case col of
- 0 : Result := Result + fProxy.ColName[ VISIBLE_COLUMNS, row - 1 ];
- 1 : Result := Result + fProxy.Cell[ row -1, fGridRow + RowScroll.Position - 1 ];
- end;
- end;
- end;
-end;
-
-procedure TMainform.A_CopyExecute( Sender : TObject );
-var s : String;
-begin
- if ActivateGrid.Down
- then s := CopyGrid
- else s := CopyRecord;
- if ( Length( s ) > 0 )
- then Clipboard.AsText := s;
-end;
-
-procedure TMainform.A_ActivateRecordExecute(Sender: TObject);
-begin
- { save the row/col for record-view }
- fGridRow := Grid.Row;
- fGridCol := Grid.Col;
- SaveColWidths;
- Grid.RowCount := fProxy.ColCount[ VISIBLE_COLUMNS ] + 1;
- Grid.ColCount := 2;
- Grid.ScrollBars := ssVertical;
- RowScroll.Visible := False;
- { restore row/col for record-view }
- Grid.Row := fRecRow;
- Grid.Col := fRecCol;
- Grid.ColWidths[ 0 ] := 120;
- Grid.ColWidths[ 1 ] := Grid.ClientWidth - Grid.ColWidths[ 0 ];
- ActivateRecord.Down := True;
- Grid.Invalidate;
-end;
-
-procedure TMainform.A_ColumnsExecute( Sender : TObject );
-var p : TPoint;
-begin
- p := Point( Grid.Left, Grid.Top );
- if ( Columnform.present( ClientToScreen( p ), Grid.ClientHeight, fProxy ) )
- then begin
- fProxy.AjustVisibleColumns( Columnform.checked_items );
- if ( ActivateGrid.Down )
- then Grid.ColCount := FProxy.ColCount[ VISIBLE_COLUMNS ] + 1
- else Grid.RowCount := FProxy.ColCount[ VISIBLE_COLUMNS ] + 1;
- Grid.Invalidate;
- end;
-end;
-
-procedure TMainform.A_DownExecute( Sender : TObject );
-begin
- if ( Grid.Row < ( Grid.RowCount - 2 ) )
- then Grid.Row := Grid.Row + 1
- else if ( RowScroll.Position < RowScroll.Max )
- then begin
- RowScroll.Position := RowScroll.Position + 1;
- Grid.Invalidate;
- end
-
-end;
-
-procedure TMainform.A_ExitExecute(Sender: TObject);
-begin
- Close;
-end;
-
-procedure TMainform.A_FirstExecute( Sender : TObject );
-begin
- RowScroll.Position := 0;
- Grid.Row := 1;
- Grid.Invalidate;
-end;
-
-procedure TMainform.GotoRow( const aRow : Integer );
-var nr, n1 : Integer;
-begin
- nr := Min( aRow, fProxy.TableRows );
- n1 := ( nr - Grid.Row );
- if ( n1 <= RowScroll.Max )and( n1 >= 0 )
- then RowScroll.Position := n1
- else if ( n1 < 0 )
- then begin
- RowScroll.Position := 0;
- Grid.Row := nr;
- end
- else begin
- RowScroll.Position := RowScroll.Max;
- Grid.Row := nr - RowScroll.Position;
- end;
- Grid.Invalidate;
-end;
-
-procedure TMainform.A_GotoExecute( Sender : TObject );
-var aRow : Integer;
- p : TPoint;
-begin
- p := Point( Grid.Left, Grid.Top );
- aRow := Grid.Row + RowScroll.Position;
- if ( MyGotoForm.present( ClientToScreen( p ), aRow ) )
- then GotoRow( aRow );
-end;
-
-procedure TMainform.A_LastExecute( Sender : TObject );
-begin
- RowScroll.Position := RowScroll.Max;
- Grid.Row := Grid.RowCount - 1;
- Grid.Invalidate;
-end;
-
-procedure TMainform.PerformOpen( const path, acc : String );
-begin
- Tables.Clear; { the table-selector ... }
- fProxy.Open( path, acc );
-end;
-
-procedure TMainform.OnLRUClick( Sender : TObject );
-var s : String;
-begin
- s := ( Sender as TMenuItem ).Caption;
- PerformOpen( s, '' );
-end;
-
-procedure TMainform.AdjustColWidth( aCol : Integer; header : boolean );
-var s : String;
- r, w : Integer;
-begin
- w := 0;
- if ( header )
- then begin
- if ( aCol > 0 )
- then begin
- s := fProxy.ColName[ VISIBLE_COLUMNS, aCol - 1 ];
- w := Grid.Canvas.TextWidth( s );
- end;
- end
- else begin
- for r := 0 to Grid.RowCount - 1
- do begin
- s := fProxy.Cell[ aCol-1, r + RowScroll.Position ];
- w := Max( w, Grid.Canvas.TextWidth( s ) );
- end;
- end;
- if ( w > 0 )
- then Grid.ColWidths[ aCol ] := w + 10;
-end;
-
-procedure TMainform.A_OpenAccExecute( Sender : TObject );
-var acc : String;
-begin
- acc := '';
- if ( InputQuery( 'Open accession', 'accession', acc ) )
- then PerformOpen( '', acc );
-end;
-
-procedure TMainform.A_OpenDirExecute(Sender: TObject);
-begin
- if SelDirDlg.Execute
- then PerformOpen( SelDirDlg.FileName, '' );
-end;
-
-procedure TMainform.A_OpenFileExecute(Sender: TObject);
-begin
- if SelFileDlg.Execute
- then PerformOpen( SelFileDlg.FileName, '' );
-end;
-
-procedure TMainform.A_PageDnExecute( Sender : TObject );
-begin
- if ( Grid.Row + 10 <= Grid.RowCount )
- then Grid.Row := Grid.Row + 10
- else if ( RowScroll.Position < ( RowScroll.Max - 10 ) )
- then begin
- RowScroll.Position := RowScroll.Position + 10;
- Grid.Invalidate;
- end
- else begin
- RowScroll.Position := RowScroll.Max;
- Grid.Row := Grid.RowCount - 1;
- Grid.Invalidate;
- end;
-end;
-
-procedure TMainform.A_PageUpExecute( Sender : TObject );
-begin
- if Grid.Row > 10
- then Grid.Row := Grid.Row - 10
- else if ( RowScroll.Position > 10 )
- then begin
- RowScroll.Position := RowScroll.Position - 10;
- Grid.Invalidate;
- end
- else begin
- RowScroll.Position := 0;
- Grid.Row := 0;
- Grid.Invalidate;
- end;
-end;
-
-procedure TMainform.A_UpExecute( Sender : TObject );
-begin
- if Grid.Row > 1
- then Grid.Row := Grid.Row - 1
- else if ( RowScroll.Position > 0 )
- then begin
- RowScroll.Position := RowScroll.Position - 1;
- Grid.Invalidate;
- end;
-end;
-
-procedure TMainform.A_VersionExecute(Sender: TObject);
-begin
- VersionForm.ShowModal;
-end;
-
-procedure TMainform.DrawDataCell( aCol, aRow : Integer;
- ct : TCellType; aRect : TRect );
-var s : String;
- effective_row : Integer;
- center, clip : boolean;
-begin
- effective_row := aRow + RowScroll.Position - 1;
- case ct of
- ct_topleft : s := 'row';
- ct_row : s := Format( '%s', [ FloatToStrF(
- effective_row + 1, ffnumber, 0, 0 ) ] );
- ct_col : s := FProxy.ColName[ VISIBLE_COLUMNS, aCol - 1 ];
- ct_content : s := FProxy.Cell[ aCol - 1, effective_row ];
- end;
- clip := ( ct = ct_content );
- center := ( ct = ct_topleft )or( ct = ct_row );
- draw_cell( fDrawCtx, aRect, center, clip, s );
-end;
-
-procedure TMainform.DrawRecordCell( aRow : Integer;
- ct : TCellType; aRect : TRect );
-var s : String;
- clip : boolean;
-begin
- case ct of
- ct_topleft : s := 'column';
- ct_row : s := FProxy.ColName[ VISIBLE_COLUMNS, aRow - 1 ];
- ct_col : s := 'value';
- ct_content : s := FProxy.Cell[ aRow - 1, fGridRow + RowScroll.Position - 1 ];
- end;
- clip := ( ct = ct_content );
- draw_cell( fDrawCtx, aRect, false, clip, s );
-end;
-
-procedure TMainform.GridDrawCell(Sender: TObject; aCol, aRow: Integer;
- aRect: TRect; aState: TGridDrawState);
-begin
- if ActivateGrid.Down
- then DrawDataCell( aCol, aRow, get_celltype( aCol, aRow ), aRect )
- else DrawRecordCell( aRow, get_celltype( aCol, aRow ), aRect );
-end;
-
-procedure TMainform.GridHeaderSized( Sender : TObject; IsColumn : Boolean;
- Index : Integer );
-var h, i : Integer;
-begin
- if ( ( not IsColumn )and ( ActivateGrid.Down ) )
- then begin
- if ( ssShift in fMouseShift )
- then begin
- h := Grid.RowHeights[ Index ];
- Grid.RowCount := ( Grid.ClientHeight div h ) + 1;
- for i := 1 to Grid.RowCount - 1
- do Grid.RowHeights[ i ] := h;
- adjust_gridrowcount_to( Grid, fProxy, h );
- end
- else begin
- adjust_gridrowcount( Grid, fProxy );
- end;
- end;
-end;
-
-procedure TMainform.GridKeyDown(Sender: TObject; var Key: Word;
- Shift: TShiftState);
-begin
- case Key of
- vk_up : begin A_Up.Execute; Key := 0; end;
- vk_down : begin A_Down.Execute; Key := 0; end;
- vk_prior : begin A_PageUp.Execute; Key := 0; end;
- vk_next : begin A_PageDn.Execute; Key := 0; end;
- vk_home : begin A_First.Execute; Key := 0; end;
- vk_end : begin A_Last.Execute; Key := 0; end;
- end;
-end;
-
-procedure TMainform.GridMouseDown(Sender: TObject; Button: TMouseButton;
- Shift: TShiftState; X, Y: Integer);
-begin
- fMouseShift := Shift;
-end;
-
-procedure TMainform.GridMouseWheelDown( Sender : TObject; Shift : TShiftState;
- MousePos : TPoint; var Handled : Boolean );
-begin
- A_DownExecute( Sender );
- Handled := True;
-end;
-
-procedure TMainform.GridMouseWheelUp( Sender : TObject; Shift : TShiftState;
- MousePos : TPoint; var Handled : Boolean );
-begin
- A_UpExecute( Sender );
- Handled := True;
-end;
-
-procedure TMainform.GridSelectCell(Sender: TObject; aCol, aRow: Integer;
- var CanSelect: Boolean);
-begin
- if ActivateGrid.Down
- then StatusBar.Panels[ 0 ].Text := Format( '%s',
- [ FloatToStrF( aRow + RowScroll.Position, ffnumber, 0, 0 ) ] );
-
-end;
-
-procedure TMainform.LEDTimerTimer( Sender : TObject );
-begin
- if ( LEDTimer.Tag = 0 )
- then begin
- LED.Brush.Color := clLime;
- LEDTimer.Tag := 1;
- end
- else begin
- LED.Brush.Color := clGreen;
- LEDTimer.Tag := 0;
- end
-end;
-
-procedure TMainform.RowScrollChange( Sender : TObject );
-begin
- FProxy.Offset := RowScroll.Position;
- StatusBar.Panels[ 0 ].Text := Format( '%s',
- [ FloatToStrF( Grid.Row + RowScroll.Position, ffnumber, 0, 0 ) ] );
- Grid.Invalidate;
-end;
-
-procedure TMainform.TablesSelect( Sender : TObject );
-begin
- FProxy.TabId := Tables.ItemIndex; { triggers table-open... }
- Tables.Enabled := false;
-end;
-
-procedure TMainform.OnOpened( const path : String; success : boolean );
-var i : Integer;
-begin
- if ( success )
- then begin
- Mainform.Caption := path;
- Tables.Clear;
- for i:=0 to FProxy.Tables - 1
- do Tables.Items.Add( FProxy.TableName[ i ] );
- Tables.ItemIndex := 0;
- TablesSelect( Tables );
- SaveLRU( path, cfg );
- PopulateLRUMenue( cfg, MainMenu, MI_Recent, @OnLRUClick );
- end
- else begin
- ShowMessage( Format( 'open "%s" failed', [ path ] ) );
- end;
-end;
-
-procedure TMainform.OnTableSwitched;
-begin
- if ( ActivateGrid.Down )
- then begin
- Grid.ColCount := fProxy.ColCount[ VISIBLE_COLUMNS ] + 1;
- adjust_gridrowcount( Grid, fProxy ); {support_unit}
- end
- else begin
- Grid.ColCount := 2;
- Grid.RowCount := fProxy.ColCount[ VISIBLE_COLUMNS ] + 1;
- end;
- RowScroll.Max := fProxy.TableRows - Grid.RowCount + 1;
- StatusBar.Panels[ 1 ].Text := Format( 'of %s',
- [ FloatToStrF( fProxy.TableRows, ffnumber, 0, 0 ) ] );
- Grid.Invalidate;
- Tables.Enabled := true;
-end;
-
-procedure TMainform.OnCellInvalidate( const aCol, aRow : Integer );
-var eCol, eRow : Integer;
-begin
- if ( ActivateGrid.Down )
- then begin
- eRow := aRow + 1;
- eCol := aCol + 1;
- end
- else begin
- eRow := aCol + 1;
- eCol := 1;
- end;
- if ( eRow > 0 )and( eRow < Grid.RowCount )and
- ( eCol > 0 )and( eCol < Grid.ColCount )
- then Grid.InvalidateCell( eCol, eRow );
-end;
-
-procedure TMainform.OnLED( state : boolean );
-begin
- if State
- then begin
- LEDTimer.Tag := 1;
- LED.Brush.Color := clLime;
- LEDTimer.Enabled:= true;
- end
- else begin
- LEDTimer.Tag := 0;
- LEDTimer.Enabled:= false;
- LED.Brush.Color := clDefault;
- end;
-end;
-
-procedure TMainform.OnSearchDone( const aCol, aRow : Integer;
- const pattern : String );
-begin
- if ( aRow < 0 )
- then ShowMessage( Format( '"%s" not found', [ pattern ] ) )
- else begin
- SForm.update_row( aRow );
- GotoRow( aRow + 1 );
- end;
-end;
-
-end.
-
diff --git a/tools/vdbview/searchform.lfm b/tools/vdbview/searchform.lfm
deleted file mode 100644
index 83440a9..0000000
--- a/tools/vdbview/searchform.lfm
+++ /dev/null
@@ -1,69 +0,0 @@
-object SForm: TSForm
- Left = 494
- Height = 158
- Top = 314
- Width = 354
- Caption = 'SForm'
- ClientHeight = 158
- ClientWidth = 354
- FormStyle = fsStayOnTop
- LCLVersion = '0.9.30'
- object Panel1: TPanel
- Left = 0
- Height = 50
- Top = 108
- Width = 354
- Align = alBottom
- ClientHeight = 50
- ClientWidth = 354
- TabOrder = 0
- object B_OK: TBitBtn
- Left = 168
- Height = 30
- Top = 9
- Width = 83
- Caption = '&Search'
- Kind = bkOK
- ModalResult = 1
- OnClick = B_OKClick
- TabOrder = 0
- end
- object BCancel: TBitBtn
- Left = 264
- Height = 30
- Top = 9
- Width = 83
- Cancel = True
- Caption = '&Close'
- Kind = bkCancel
- ModalResult = 2
- OnClick = BCancelClick
- TabOrder = 1
- end
- end
- object EPattern: TEdit
- Left = 16
- Height = 21
- Top = 17
- Width = 320
- TabOrder = 1
- end
- object RB_Forward: TRadioButton
- Left = 16
- Height = 17
- Top = 47
- Width = 60
- Caption = 'Forward'
- Checked = True
- TabOrder = 3
- TabStop = True
- end
- object RB_Backward: TRadioButton
- Left = 16
- Height = 17
- Top = 68
- Width = 66
- Caption = 'Backward'
- TabOrder = 2
- end
-end
diff --git a/tools/vdbview/searchform.pas b/tools/vdbview/searchform.pas
deleted file mode 100644
index 7e62529..0000000
--- a/tools/vdbview/searchform.pas
+++ /dev/null
@@ -1,79 +0,0 @@
-unit searchform;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, FileUtil, Forms, Controls, Graphics, Dialogs, ExtCtrls,
- Buttons, StdCtrls, DataProxy;
-
-type
-
- { TSForm }
-
- TSForm = class(TForm)
- B_OK: TBitBtn;
- BCancel: TBitBtn;
- EPattern: TEdit;
- Panel1: TPanel;
- RB_Backward: TRadioButton;
- RB_Forward: TRadioButton;
- procedure BCancelClick(Sender: TObject);
- procedure B_OKClick(Sender: TObject);
- private
- { private declarations }
- fRow, fCol : Integer;
- fProxy : TProxy;
- public
- { public declarations }
- procedure present( const p : TPoint;
- aCol, aRow : Integer;
- pattern : String;
- aProxy : TProxy );
- procedure update_row( aRow : Integer );
- end;
-
-var
- SForm: TSForm;
-
-implementation
-
-{$R *.lfm}
-
-{ TSForm }
-
-procedure TSForm.B_OKClick( Sender : TObject );
-begin
- if ( RB_Forward.Checked )
- then fProxy.Search( true, fCol, fRow + 1, EPattern.Text )
- else fProxy.Search( false, fCol, fRow - 1, EPattern.Text );
-end;
-
-procedure TSForm.BCancelClick(Sender: TObject);
-begin
- fProxy.CancelSearch;
- Hide;
-end;
-
-procedure TSForm.present( const p : TPoint;
- aCol, aRow : Integer;
- pattern : String;
- aProxy : TProxy );
-begin
- Left := p.x;
- Top := p.y;
- EPattern.Text := pattern;
- fRow := aRow;
- fCol := aCol;
- fProxy := aProxy;
- Show;
-end;
-
-procedure TSForm.update_row( aRow : Integer );
-begin
- fRow := aRow;
-end;
-
-end.
-
diff --git a/tools/vdbview/selector_types.pas b/tools/vdbview/selector_types.pas
deleted file mode 100644
index aa0c37b..0000000
--- a/tools/vdbview/selector_types.pas
+++ /dev/null
@@ -1,20 +0,0 @@
-unit selector_types;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils;
-
-const
- ALL_COLUMNS = Integer( 0 );
- STATIC_COLUMNS = Integer( 1 );
- NON_STATIC_COLUMNS = Integer( 2 );
- VISIBLE_COLUMNS = Integer( 3 );
- NON_VISIBLE_COLUMNS = Integer( 4 );
-
-implementation
-
-end.
-
diff --git a/tools/vdbview/support_unit.pas b/tools/vdbview/support_unit.pas
deleted file mode 100644
index d1f757b..0000000
--- a/tools/vdbview/support_unit.pas
+++ /dev/null
@@ -1,217 +0,0 @@
-unit support_unit;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, Forms, Grids, XMLCfg, DataProxy, CheckLst,
- Menus, Graphics, StrUtils;
-
-type
- TCellType = ( ct_topleft, ct_row, ct_col, ct_content );
- TOnLRUClick = procedure( Sender : TObject ) of Object;
-
- TDrawCtx = record
- w, h : Integer;
- g : TDrawGrid;
- end;
-
-procedure setup_position( M : TForm; cfg : TXMLConfig; S : TScreen );
-procedure store_position( M : TForm; cfg : TXMLConfig );
-procedure adjust_gridrowcount_to( G : TDrawGrid; P : TProxy; h : Integer );
-procedure adjust_gridrowcount( G : TDrawGrid; P : TProxy );
-function get_celltype( aCol, aRow: Integer ) : TCellType;
-procedure draw_cell( ctx : TDrawCtx; aRect : TRect;
- center, clip : boolean; s : String );
-function CheckListBox2String( aBox : TCheckListBox ) : String;
-
-procedure SaveLRU( path : string; cfg : TXMLConfig );
-procedure PopulateLRUMenue( cfg : TXMLConfig; mm : TMainMenu;
- item : TMenuItem; event : TOnLRUClick );
-procedure ClearLRU( cfg : TXMLConfig; item : TMenuItem );
-
-implementation
-
-procedure setup_position( M : TForm; cfg : TXMLConfig; S : TScreen );
-begin
- M.Left := cfg.GetValue( 'xpos', M.Left );
- M.Width:= cfg.GetValue( 'width', M.Width );
- if ( M.Left + M.Width > S.Width )
- then M.Left := S.Width - M.Width;
- if ( M.Left < 0 ) then M.Left := 0;
- if ( M.Width > S.Width ) then M.Width := S.Width;
- M.Top := cfg.GetValue( 'ypos', M.Top );
- M.Height:= cfg.Getvalue( 'height', M.Height );
- if ( M.Top + M.Height > S.Height )
- then M.Top := S.Height - M.Height;
- if ( M.Top < 0 ) then M.Top := 0;
- if ( M.Height > S.Height ) then M.Height := S.Height;
- if ( cfg.GetValue( 'windowstate', 'N' ) = 'M' )
- then M.WindowState := wsMaximized;
-end;
-
-procedure store_position( M : TForm; cfg : TXMLConfig );
-begin
- case M.WindowState of
- wsNormal : begin
- cfg.SetValue( 'xpos', M.Left );
- cfg.SetValue( 'width', M.Width );
- cfg.SetValue( 'ypos', M.Top );
- cfg.SetValue( 'height', M.Height );
- cfg.SetValue( 'windowstate', 'N' );
- end;
- wsMaximized : cfg.SetValue( 'windowstate', 'M' );
- wsMinimized : cfg.SetValue( 'windowstate', 'N' );
- end;
-end;
-
-procedure adjust_gridrowcount_to( G : TDrawGrid; P : TProxy; h : Integer );
-begin
- G.RowCount := ( G.ClientHeight div h ) + 1;
- G.RowCount := G.VisibleRowCount + 2;
- P.AdjustCacheRows( G.RowCount );
-end;
-
-procedure adjust_gridrowcount( G : TDrawGrid; P : TProxy );
-begin
- adjust_gridrowcount_to( G, P, G.DefaultRowHeight );
-end;
-
-function get_celltype( aCol, aRow : Integer ) : TCellType;
-begin
- if ( aCol = 0 )
- then if ( aRow = 0 )
- then Result := ct_topleft
- else Result := ct_row
- else if ( aRow = 0 )
- then Result := ct_col
- else Result := ct_content;
-end;
-
-procedure draw_multi_line( ctx : TDrawCtx; aRect : TRect;
- center : boolean; s : String; cw, tw, ch, cpl : Integer );
-var n_lines, v_lines, i : Integer;
-begin
- if ( cpl > 0 )
- then begin
- n_lines := ( Length( s ) div cpl );
- v_lines := ( ch div ctx.h );
- if ( v_lines < n_lines )
- then begin
- { not all lines are visible ... }
- if ( v_lines > 0 )
- then begin
- for i := 0 to v_lines - 2
- do ctx.g.Canvas.TextOut( aRect.Left + 5,
- aRect.Top + 3 + ( i * ctx.h ),
- MidStr( s, i * cpl, cpl ) );
- i := v_lines - 1;
- ctx.g.Canvas.TextOut( aRect.Left + 5,
- aRect.Top + 3 + ( i * ctx.h ),
- MidStr( s, i * cpl, cpl - 2 ) + '..' );
-
- end;
- end
- else begin
- { all lines are visible ... }
- if ( n_lines > 0 )
- then for i := 0 to n_lines
- do ctx.g.Canvas.TextOut( aRect.Left + 5,
- aRect.Top + 3 + ( i * ctx.h ),
- MidStr( s, i * cpl, cpl ) );
- end;
- end;
-end;
-
-procedure draw_single_line( ctx : TDrawCtx; aRect : TRect;
- center, clip : boolean; s : String; cw, tw, cpl : Integer );
-var x, y : Integer;
-begin
- y := aRect.top + 3;
- if ( center )
- then x := aRect.Left + ( ( cw - tw ) div 2 )
- else x := aRect.left + 5;
- if ( ( Length( s ) > cpl )and( clip ) )
- then ctx.g.Canvas.TextRect( aRect, x, y, MidStr( s, 0, cpl-2 )+'..' )
- else ctx.g.Canvas.TextRect( aRect, x, y, s );
-end;
-
-procedure draw_cell( ctx : TDrawCtx; aRect : TRect;
- center, clip : boolean; s : String );
-var ch, cw, tw, cpl : Integer;
-begin
- ch := aRect.Bottom - aRect.Top; { cell height }
- cw := aRect.Right - aRect.Left; { cell width }
- tw := ctx.g.Canvas.TextWidth( s ); { text width }
- cpl := ( ( cw - 10 ) div ctx.w ); { chars per line }
-
- if ( tw > ( cw - 10 ) )and( ch > ( ctx.h + ctx.h ) )
- then draw_multi_line( ctx, aRect, center, s, cw, tw, ch, cpl )
- else draw_single_line( ctx, aRect, center, clip, s, cw, tw, cpl );
-end;
-
-function CheckListBox2String( aBox : TCheckListBox ) : String;
-var i : Integer;
-begin
- Result := '';
- if Assigned( aBox )
- then begin
- for i := 0 to aBox.Items.Count - 1
- do if ( aBox.Checked[ i ] )
- then Result := Result + 'X'
- else Result := Result + '-';
- end;
-end;
-
-procedure SaveLRU( path : string; cfg : TXMLConfig );
-var L : TStringList;
-begin
- L := TStringList.Create;
- if Assigned( L )
- then try
- L.Text := cfg.GetValue( 'LRU/path', '' );
- if ( L.IndexOf( path ) < 0 )
- then begin
- if ( L.Count > 20 )
- then L.Delete( 0 );
- L.Add( path );
- cfg.SetValue( 'LRU/path', L.Text );
- end;
- finally
- L.Free;
- end;
-end;
-
-procedure PopulateLRUMenue( cfg : TXMLConfig; mm : TMainMenu;
- item : TMenuItem; event : TOnLRUClick );
-var M : TMenuItem;
- L : TStringList;
- I : Integer;
-begin
- L := TStringList.Create;
- if Assigned( L )
- then try
- L.Text := cfg.GetValue( 'LRU/path', '' );
- item.Clear;
- for I := 0 to L.Count-1
- do begin
- M := TMenuItem.Create( mm );
- M.Name := Format( 'LRU_Item_%d', [ I ] );
- M.Caption := L.Strings[ I ];
- M.OnClick := event;
- item.Add( M );
- end;
- finally
- L.Free;
- end;
-end;
-
-procedure ClearLRU( cfg : TXMLConfig; item : TMenuItem );
-begin
- cfg.SetValue( 'LRU/path', '' );
- item.Clear;
-end;
-
-end.
-
diff --git a/tools/vdbview/svdb.pas b/tools/vdbview/svdb.pas
deleted file mode 100644
index 2ab2dc5..0000000
--- a/tools/vdbview/svdb.pas
+++ /dev/null
@@ -1,676 +0,0 @@
-unit svdb;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, dynlibs;
-
-type
-
- TAcc2Path = function( acc : PChar ) : PChar; cdecl;
- TOpenPath = function( path : PChar ) : Pointer; cdecl;
- TClose = procedure( obj : Pointer ); cdecl;
- TLastErr = function : PChar; cdecl;
- TIsDb = function( obj : Pointer ) : Integer; cdecl;
- TCountTabs = function( obj : Pointer ) : Integer; cdecl;
- TTabName = function( obj : Pointer; tabid : Integer ) : PChar; cdecl;
- TTabIdx = function( obj : Pointer; name : PChar ) : Integer; cdecl;
- TCountCols = function( obj : Pointer; tabid, sel : Integer ) : Integer; cdecl;
- TColName = function( obj : Pointer;
- tabid, sel, colid : Integer ) : PChar; cdecl;
- TColIdx = function( obj : Pointer; tabid, sel : Integer;
- name : PChar ) : Integer; cdecl;
- TColVisibility = function( obj : Pointer; tabid, sel,
- col_id, visible : Integer ) : Integer; cdecl;
- TDefTypeIdx = function( obj : Pointer; tabid, sel,
- colid : Integer ) : Integer; cdecl;
- TCountTypes = function( obj : Pointer; tabid, sel,
- colid : Integer ) : Integer; cdecl;
- TTypeName = function( obj : Pointer; tabid, sel,
- colid, typeid : Integer ) : PChar; cdecl;
- TTypeIdx = function( obj : Pointer; tabid, sel, colid : Integer;
- name : PChar ) : Integer; cdecl;
- TTypeDomain = function( obj : Pointer; tabid, sel,
- colid, typeid : Integer ) : Integer; cdecl;
- TTypeBits = function( obj : Pointer; tabid, sel,
- colid, typeid : Integer ) : Integer; cdecl;
- TTypeDim = function( obj : Pointer; tabid, sel,
- colid, typeid : Integer ) : Integer; cdecl;
-
- TOpenTable = function( obj : Pointer; tabid : Integer;
- defline : PChar ) : Integer; cdecl;
- TMaxColnameLen = function( obj : Pointer;
- tabid, sel : Integer ) : Integer; cdecl;
- TSetElemSep = procedure( obj : Pointer; tabid, sel,
- colid : Integer; sep : PChar ); cdecl;
- TSetDimSep = procedure( obj : Pointer; tabid, sel,
- colid : Integer; sep : PChar ); cdecl;
- TIsColEnabled = function( obj : Pointer; tabid, sel,
- colid : Integer ) : Integer; cdecl;
- TGetRowRange = function( obj : Pointer; tabid : Integer ) : Int64; cdecl;
- TGetFirstRow = function( obj : Pointer; tabid : Integer ) : Int64; cdecl;
- TGetCell = function( obj : Pointer; dst : PChar;
- dst_len, tab_id, sel, col_id : Integer;
- row : Int64 ) : Integer; cdecl;
-
- TFindFwd = function( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : PChar ): Int64; cdecl;
- TFindBwd = function( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : PChar ): Int64; cdecl;
-
- TMetaRoot = function( obj : Pointer; tabid : Integer ) : Integer; cdecl;
- TMetaName = function( obj : Pointer; tabid, metaid : Integer;
- dst :PChar; dst_len : Integer ) : Integer; cdecl;
- TMetaValueLen = function( obj : Pointer;
- tabid, metaid : Integer ) : Integer; cdecl;
- TMetaValuePrintable = function( obj : Pointer;
- tabid, metaid : Integer ) : Integer; cdecl;
- TMetaValuePtr = function( obj : Pointer;
- tabid, metaid : Integer ) : Pointer; cdecl;
- TMetaValue = function( obj : Pointer; tabid, metaid : Integer;
- dst :PChar; dst_len, trim : Integer ) : Integer; cdecl;
- TMetaChildCount = function( obj : Pointer;
- tabid, metaid : Integer ) : Integer; cdecl;
- TMetaChildId = function( obj : Pointer;
- tabid, metaid, childid : Integer ) : Integer; cdecl;
-
- { TSvdb }
-
- TSvdb = class
- private
- { private declarations }
- f_mylib : TLibHandle;
-
- f_acc2path : TAcc2Path;
- f_openpath : TOpenPath;
- f_lasterr : TLastErr;
- f_close : TClose;
- f_isdb : TIsDb;
- f_countTabs : TCountTabs;
- f_tabName : TTabName;
- f_tabIdx : TTabIdx;
- f_countCols : TCountCols;
- f_colName : TColName;
- f_colIdx : TColIdx;
- f_colVisible : TColVisibility;
- f_defTypeIdx : TDefTypeIdx;
- f_countTypes : TCountTypes;
- f_typeName : TTypeName;
- f_typeIdx : TTypeIdx;
- f_typeDomain : TTypeDomain;
- f_typeBits : TTypeBits;
- f_typeDim : TTypeDim;
-
- f_openTab : TOpenTable;
- f_maxColNameLen : TMaxColnameLen;
- f_setElemSep : TSetElemSep;
- f_setDimSep : TSetDimSep;
- f_isColEnabled : TIsColEnabled;
- f_getRowRange : TGetRowRange;
- f_getFirstRow : TGetFirstRow;
- f_getCell : TGetCell;
- f_findFwd : TFindFwd;
- f_findBwd : TFindBwd;
-
- f_metaRoot : TMetaRoot;
- f_metaName : TMetaName;
- f_metaValueLen : TMetaValueLen;
- f_metaValuePrintable : TMetaValuePrintable;
- f_metaValuePtr : TMetaValuePtr;
- f_metaValue : TMetaValue;
- f_metaChildCount : TMetaChildCount;
- f_metaChildId : TMetaChildId;
-
- function connect( dll_name : String ) : boolean;
- procedure disconnect;
- procedure clear;
- function GetCellN( obj : Pointer; tabid, sel, colid, size : Integer;
- row : Int64; var needed : Integer ) : String;
- function MetaNameN( obj : Pointer; tabid, metaid, size : Integer) : String;
- function MetaValueN( obj : Pointer; tabid, metaid, size, trim : Integer ) : String;
- public
- { public declarations }
- constructor Create;
- destructor Destroy; override;
-
- function LastErr : String;
- function Acc2Path( acc : String ) : String;
- function OpenPath( path : String ) : Pointer;
- procedure Close( obj : Pointer );
- function IsDb( obj : Pointer ) : boolean;
- function CountTabs( obj : Pointer ) : Integer;
- function TabName( obj : Pointer; tabid : Integer ) : String;
- function TabIdx( obj : Pointer; name : String ) : Integer;
- function CountCols( obj : Pointer; tabid, sel : Integer ) : Integer;
- function ColName( obj : Pointer; tabid, sel, colid : Integer ) : String;
- function ColIdx( obj : Pointer; tabid, sel : Integer; name : String ) : Integer;
- function SetColVisible( obj : Pointer; tabid, sel, colid, visible : Integer ) : Integer;
- function DefTypeIdx( obj : Pointer; tabid, sel, colid : Integer ) : Integer;
- function CountTypes( obj : Pointer; tabid, sel, colid : Integer ) : Integer;
- function TypeName( obj : Pointer; tabid, sel, colid, typeid : Integer ) : String;
- function TypeIdx( obj : Pointer; tabid, sel, colid : Integer; name : String ) : Integer;
- function TypeDomain( obj : Pointer; tabid, sel, colid, typeid : Integer ) : Integer;
- function TypeBits( obj : Pointer; tabid, sel, colid, typeid : Integer ) : Integer;
- function TypeDim( obj : Pointer; tabid, sel, colid, typeid : Integer ) : Integer;
-
- function OpenTable( obj : Pointer; tabid : Integer; defline : String ) : Integer;
- function MaxColnameLen( obj : Pointer; tabid, sel : Integer ) : Integer;
- procedure SetElemSep( obj : Pointer; tabid, sel, colid : Integer; sep : String );
- procedure SetDimSep( obj : Pointer; tabid, sel, colid : Integer; sep : String );
- function IsColEnabled( obj : Pointer; tabid, sel, colid : Integer ) : boolean;
- function RowRange( obj : Pointer; tabid : Integer ) : Int64;
- function FirstRow( obj : Pointer; tabid : Integer ) : Int64;
- function GetCell( obj : Pointer; tabid, sel, colid : Integer; row : Int64 ) : String;
- function FindFwd( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : String ) : Int64;
- function FindBwd( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : String ) : Int64;
-
- function MetaRoot( obj : Pointer; tabid : Integer ) : Integer;
- function MetaName( obj : Pointer; tabid, metaid : Integer ) : String;
- function MetaValueLen( obj : Pointer; tabid, metaid : Integer ) : Integer;
- function MetaValuePrintable( obj : Pointer; tabid, metaid : Integer ) : boolean;
- function MetaValuePtr( obj : Pointer; tabid, metaid : Integer ) : Pointer;
- function MetaValue( obj : Pointer; tabid, metaid, trim : Integer ) : String;
- function MetaChildCount( obj : Pointer; tabid, metaid : Integer ) : Integer;
- function MetaChildId( obj : Pointer; tabid, metaid, childid : Integer ) : Integer;
-
- end;
-
-implementation
-
-{ TSvdb }
-
-constructor TSvdb.Create;
-begin
- inherited;
- clear;
- connect( 'libsvdb.dll' );
-end;
-
-destructor TSvdb.Destroy;
-begin
- disconnect;
- inherited Destroy;
-end;
-
-function TSvdb.LastErr : String;
-var s : PChar;
-begin
- Result := '';
- if ( f_lasterr <> Nil )
- then begin
- s := f_lasterr();
- if ( s <> Nil )
- then Result := String( s );
- end;
-end;
-
-function TSvdb.Acc2Path( acc : String ) : String;
-var s : PChar;
-begin
- Result := '';
- if ( f_acc2path <> Nil )
- then begin
- s := f_acc2path( PChar( acc ) );
- if ( s <> Nil )
- then Result := String( s );
- end;
-end;
-
-function TSvdb.OpenPath( path : String ) : Pointer;
-begin
- Result := Nil;
- if ( f_openpath <> Nil )
- then Result := f_openpath( PChar( path ) );
-end;
-
-procedure TSvdb.Close( obj : Pointer );
-begin
- if ( f_close <> Nil )
- then f_close( obj );
-end;
-
-function TSvdb.IsDb( obj : Pointer ) : boolean;
-begin
- Result := false;
- if ( f_isdb <> Nil )
- then Result := ( f_isdb( obj ) <> 0 );
-end;
-
-function TSvdb.CountTabs( obj : Pointer ) : Integer;
-begin
- Result := 0;
- if ( f_countTabs <> Nil )
- then Result := f_countTabs( obj );
-end;
-
-function TSvdb.TabName( obj : Pointer; tabid : Integer ) : String;
-var p : PChar;
-begin
- Result := '';
- if ( f_tabName <> Nil )
- then begin
- p := f_tabName( obj, tabid );
- if ( p <> Nil )
- then Result := String( p );
- end;
-end;
-
-function TSvdb.TabIdx( obj : Pointer; name : String ) : Integer;
-begin
- Result := -1;
- if ( f_tabIdx <> Nil )
- then Result := f_tabIdx( obj, PChar( name ) );
-end;
-
-function TSvdb.CountCols( obj : Pointer; tabid, sel : Integer ) : Integer;
-begin
- Result := 0;
- if ( f_countCols <> Nil )
- then Result := f_countCols( obj, tabid, sel );
-end;
-
-function TSvdb.ColName( obj : Pointer; tabid, sel, colid : Integer ) : String;
-var p : PChar;
-begin
- Result := '';
- if ( f_colName <> Nil )
- then begin
- p := f_colName( obj, tabid, sel, colid );
- if ( p <> Nil )
- then Result := String( p );
- end;
-end;
-
-function TSvdb.ColIdx( obj : Pointer; tabid, sel : Integer; name : String ) : Integer;
-begin
- Result := -1;
- if ( f_colIdx <> Nil )
- then Result := f_colIdx( obj, tabid, sel, PChar( name ) );
-end;
-
-function TSvdb.SetColVisible( obj : Pointer; tabid, sel, colid, visible : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_colVisible <> Nil )
- then Result := f_colVisible( obj, tabid, sel, colid, visible );
-end;
-
-function TSvdb.DefTypeIdx( obj : Pointer; tabid, sel, colid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_defTypeIdx <> Nil )
- then Result := f_defTypeIdx( obj, tabid, sel, colid );
-end;
-
-function TSvdb.CountTypes( obj : Pointer; tabid, sel, colid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_countTypes <> Nil )
- then Result := f_countTypes( obj, tabid, sel, colid );
-end;
-
-function TSvdb.TypeName( obj : Pointer; tabid, sel, colid, typeid : Integer ) : String;
-var p : PChar;
-begin
- Result := '';
- if ( f_typeName <> Nil )
- then begin
- p := f_typeName( obj, tabid, sel, colid, typeid );
- if ( p <> Nil )
- then Result := String( p );
- end;
-end;
-
-function TSvdb.TypeIdx( obj : Pointer; tabid, sel, colid : Integer; name : String ) : Integer;
-begin
- Result := -1;
- if ( f_typeIdx <> Nil )
- then Result := f_typeIdx( obj, tabid, sel, colid, PChar( name ) );
-end;
-
-function TSvdb.TypeDomain( obj : Pointer; tabid, sel, colid, typeid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_typeDomain <> Nil )
- then Result := f_typeDomain( obj, tabid, sel, colid, typeid );
-end;
-
-function TSvdb.TypeBits( obj : Pointer; tabid, sel, colid, typeid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_typeBits <> Nil )
- then Result := f_typeBits( obj, tabid, sel, colid, typeid );
-end;
-
-function TSvdb.TypeDim( obj : Pointer; tabid, sel, colid, typeid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_typeDim <> Nil )
- then Result := f_typeDim( obj, tabid, sel, colid, typeid );
-end;
-
-function TSvdb.OpenTable( obj : Pointer; tabid : Integer; defline : String ) : Integer;
-begin
- Result := -1;
- if ( f_openTab <> Nil )
- then Result := f_openTab( obj, tabid, PChar( defline ) );
-end;
-
-function TSvdb.MaxColnameLen( obj : Pointer; tabid, sel : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_maxColNameLen <> Nil )
- then Result := f_maxColNameLen( obj, tabid, sel );
-end;
-
-procedure TSvdb.SetElemSep( obj : Pointer; tabid, sel, colid : Integer; sep : String );
-begin
- if ( f_setElemSep <> Nil )
- then f_setElemSep( obj, tabid, sel, colid, PChar( sep ) );
-end;
-
-procedure TSvdb.SetDimSep( obj : Pointer; tabid, sel, colid : Integer; sep : String );
-begin
- if ( f_setDimSep <> Nil )
- then f_setDimSep( obj, tabid, sel, colid, PChar( sep ) );
-end;
-
-function TSvdb.IsColEnabled( obj : Pointer; tabid, sel, colid : Integer ) : boolean;
-begin
- Result := false;
- if ( f_isColEnabled <> Nil )
- then Result := ( f_isColEnabled( obj, tabid, sel, colid ) <> 0 );
-end;
-
-function TSvdb.RowRange( obj : Pointer; tabid : Integer ) : Int64;
-begin
- Result := 0;
- if ( f_getRowRange <> Nil )
- then Result := f_getRowRange( obj, tabid );
-end;
-
-function TSvdb.FirstRow( obj : Pointer; tabid : Integer ) : Int64;
-begin
- Result := 0;
- if ( f_getFirstRow <> Nil )
- then Result := f_getFirstRow( obj, tabid );
-end;
-
-function TSvdb.GetCellN( obj : Pointer; tabid, sel, colid, size : Integer;
- row : Int64; var needed : Integer ) : String;
-var p : PChar;
-begin
- Result := '';
- needed := size;
- p := StrAlloc( size );
- if ( p <> Nil )
- then begin
- needed := f_getCell( obj, p, size, tabid, sel, colid, row );
- Result := String( p );
- StrDispose( p );
- end;
-end;
-
-
-function TSvdb.GetCell( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64 ) : String;
-var bufsize, needed : Integer;
- p : PChar;
-begin
- Result := '';
- bufsize := 250;
- needed := bufsize;
- if ( f_getCell <> Nil )
- then begin
- p := StrAlloc( bufsize );
- if ( p <> Nil )
- then begin
- needed := f_getCell( obj, p, bufsize, tabid, sel, colid, row );
- if ( needed >= bufsize )
- then begin
- StrDispose( p );
- p := StrAlloc( needed + 1 );
- if ( p <> Nil )
- then f_getCell( obj, p, needed + 1, tabid, sel, colid, row );
- end;
- end;
- if ( p <> Nil )
- then begin
- Result := String( p );
- StrDispose( p );
- end;
- end;
-end;
-
-function TSvdb.FindFwd( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : String ) : Int64;
-begin
- Result := -1;
- if ( f_findFwd <> Nil )
- then Result := f_findFwd( obj, tabid, sel, colid, row, chunk, PChar( pattern ) );
-end;
-
-function TSvdb.FindBwd( obj : Pointer; tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : String ) : Int64;
-begin
- Result := -1;
- if ( f_findBwd <> Nil )
- then Result := f_findBwd( obj, tabid, sel, colid, row, chunk, PChar( pattern ) );
-end;
-
-function TSvdb.MetaRoot( obj : Pointer; tabid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_metaRoot <> Nil )
- then Result := f_metaRoot( obj, tabid );
-end;
-
-
-function TSvdb.MetaNameN( obj : Pointer; tabid, metaid, size : Integer ) : String;
-var p : PChar;
- written : Integer;
-begin
- Result := '';
- p := StrAlloc( size );
- if ( p <> Nil )
- then begin
- written := f_metaName( obj, tabid, metaid, p, size );
- if ( written < ( size - 1 ) )
- then Result := String( p );
- StrDispose( p );
- end;
-end;
-
-
-function TSvdb.MetaName( obj: Pointer ; tabid, metaid : Integer ) : String;
-var size : Integer;
-begin
- Result := '';
- size := 512;
- if ( f_metaName <> Nil )
- then while ( ( Length( Result ) = 0 )and( size < 128000 ) )
- do begin
- size := size + size;
- Result := MetaNameN( obj, tabid, metaid, size );
- end;
-end;
-
-function TSvdb.MetaValueLen( obj : Pointer;
- tabid, metaid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_metaValueLen <> Nil )
- then Result := f_metaValueLen( obj, tabid, metaid );
-end;
-
-function TSvdb.MetaValuePrintable( obj : Pointer;
- tabid, metaid : Integer ) : boolean;
-begin
- Result := false;
- if ( f_metaValuePrintable <> Nil )
- then Result := ( f_metaValuePrintable( obj, tabid, metaid ) <> 0 );
-end;
-
-function TSvdb.MetaValuePtr( obj : Pointer; tabid, metaid : Integer ) : Pointer;
-begin
- Result := Nil;
- if ( f_metaValuePtr <> Nil )
- then Result := f_metaValuePtr( obj, tabid, metaid );
-end;
-
-function TSvdb.MetaValueN( obj : Pointer; tabid, metaid, size, trim : Integer ) : String;
-var p : PChar;
- written : Integer;
-begin
- Result := '';
- p := StrAlloc( size );
- if ( p <> Nil )
- then begin
- written := f_metaValue( obj, tabid, metaid, p, size, trim );
- if ( written < ( size - 1 ) )
- then Result := String( p );
- StrDispose( p );
- end;
-end;
-
-function TSvdb.MetaValue( obj : Pointer; tabid, metaid, trim : Integer ) : String;
-var size : Integer;
-begin
- Result := '';
- size := 512;
- if ( f_metaValue <> Nil )
- then while ( ( Length( Result ) = 0 )and( size < 128000 ) )
- do begin
- size := size + size;
- Result := MetaValueN( obj, tabid, metaid, size, trim );
- end;
-end;
-
-function TSvdb.MetaChildCount( obj : Pointer;
- tabid, metaid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_metaChildCount <> Nil )
- then Result := f_metaChildCount( obj, tabid, metaid );
-end;
-
-function TSvdb.MetaChildId( obj : Pointer;
- tabid, metaid, childid : Integer ) : Integer;
-begin
- Result := -1;
- if ( f_metaChildId <> Nil )
- then Result := f_metaChildId( obj, tabid, metaid, childid );
-
-end;
-
-
-function TSvdb.connect( dll_name: String ) : boolean;
-begin
- f_mylib := LoadLibrary( dll_name );
- Result := ( f_mylib <> NilHandle );
- if ( Result )
- then begin
- f_acc2path := TAcc2Path( GetProcedureAddress( f_mylib, 'svdb_accession_2_path' ) );
- f_openpath := TOpenPath( GetProcedureAddress( f_mylib, 'svdb_open_path' ) );
- f_lasterr := TLastErr( GetProcedureAddress( f_mylib, 'svdb_last_err' ) );
- f_close := TClose( GetProcedureAddress( f_mylib, 'svdb_close' ) );
- f_isdb := TIsDb( GetProcedureAddress( f_mylib, 'svdb_is_db' ) );
- f_countTabs := TCountTabs( GetProcedureAddress( f_mylib, 'svdb_count_tabs' ) );
- f_tabName := TTabName( GetProcedureAddress( f_mylib, 'svdb_tabname' ) );
- f_tabIdx := TTabIdx( GetProcedureAddress( f_mylib, 'svdb_tab_idx' ) );
- f_countCols := TCountCols( GetProcedureAddress( f_mylib, 'svdb_count_cols' ) );
- f_colName := TColName( GetProcedureAddress( f_mylib, 'svdb_colname' ) );
- f_colIdx := TColIdx( GetProcedureAddress( f_mylib, 'svdb_col_idx' ) );
- f_colVisible := TColVisibility( GetProcedureAddress( f_mylib, 'svdb_set_column_visibility' ) );
- f_defTypeIdx := TDefTypeIdx( GetProcedureAddress( f_mylib, 'svdb_dflt_type_idx' ) );
- f_countTypes := TCountTypes( GetProcedureAddress( f_mylib, 'svdb_count_types' ) );
- f_typeName := TTypeName( GetProcedureAddress( f_mylib, 'svdb_typename' ) );
- f_typeIdx := TTypeIdx( GetProcedureAddress( f_mylib, 'svdb_type_idx' ) );
- f_typeDomain := TTypeDomain( GetProcedureAddress( f_mylib, 'svdb_typedomain' ) );
- f_typeBits := TTypeBits( GetProcedureAddress( f_mylib, 'svdb_typebits' ) );
- f_typeDim := TTypeDim( GetProcedureAddress( f_mylib, 'svdb_typedim' ) );
-
- f_openTab := TOpenTable( GetProcedureAddress( f_mylib, 'svdb_open_table' ) );
- f_maxColNameLen := TMaxColnameLen( GetProcedureAddress( f_mylib, 'svdb_max_colname_length' ) );
- f_setElemSep := TSetElemSep( GetProcedureAddress( f_mylib, 'svdb_set_elem_separator' ) );
- f_setDimSep := TSetDimSep( GetProcedureAddress( f_mylib, 'svdb_set_dim_separator' ) );
- f_isColEnabled := TIsColEnabled( GetProcedureAddress( f_mylib, 'svdb_is_enabled' ) );
- f_getRowRange := TGetRowRange( GetProcedureAddress( f_mylib, 'svdb_row_range' ) );
- f_getFirstRow := TGetFirstRow( GetProcedureAddress( f_mylib, 'svdb_first_row' ) );
- f_getCell := TGetCell( GetProcedureAddress( f_mylib, 'svdb_cell' ) );
- f_findFwd := TFindFwd( GetProcedureAddress( f_mylib, 'svdb_find_fwd' ) );
- f_findBwd := TFindBwd( GetProcedureAddress( f_mylib, 'svdb_find_bwd' ) );
-
- f_metaRoot := TMetaRoot( GetProcedureAddress( f_mylib, 'svdb_tab_meta_root' ) );
- f_metaName := TMetaName( GetProcedureAddress( f_mylib, 'svdb_tab_meta_name' ) );
- f_metaValueLen := TMetaValueLen( GetProcedureAddress( f_mylib, 'svdb_tab_meta_value_len' ) );
- f_metaValuePrintable := TMetaValuePrintable( GetProcedureAddress( f_mylib, 'svdb_tab_meta_value_printable' ) );
- f_metaValuePtr := TMetaValuePtr( GetProcedureAddress( f_mylib, 'svdb_tab_meta_value_ptr' ) );
- f_metaValue := TMetaValue( GetProcedureAddress( f_mylib, 'svdb_tab_meta_value' ) );
- f_metaChildCount := TMetaChildCount( GetProcedureAddress( f_mylib, 'svdb_tab_meta_child_count' ) );
- f_metaChildId := TMetaChildId( GetProcedureAddress( f_mylib, 'svdb_tab_meta_child_id' ) );
-
- end;
-end;
-
-procedure TSvdb.disconnect;
-begin
- if ( f_mylib <> NilHandle )
- then begin
- FreeLibrary( f_mylib );
- clear;
- end;
-end;
-
-procedure TSvdb.clear;
-begin
- f_mylib := NilHandle;
-
- f_acc2path := Nil;
- f_openpath := Nil;
- f_lasterr := Nil;
- f_close := Nil;
- f_isdb := Nil;
- f_countTabs := Nil;
- f_tabName := Nil;
- f_tabIdx := Nil;
- f_countCols := Nil;
- f_colName := Nil;
- f_colIdx := Nil;
- f_colVisible := Nil;
- f_defTypeIdx := Nil;
- f_countTypes := Nil;
- f_typeName := Nil;
- f_typeIdx := Nil;
- f_typeDomain := Nil;
- f_typeBits := Nil;
- f_typeDim := Nil;
-
- f_openTab := Nil;
- f_maxColNameLen := Nil;
- f_setElemSep := Nil;
- f_setDimSep := Nil;
- f_isColEnabled := Nil;
- f_getRowRange := Nil;
- f_getFirstRow := Nil;
- f_getCell := Nil;
- f_findFwd := Nil;
- f_findBwd := Nil;
-
- f_metaRoot := Nil;
- f_metaName := Nil;
- f_metaValueLen := Nil;
- f_metaValuePrintable := Nil;
- f_metaValuePtr := Nil;
- f_metaValue := Nil;
- f_metaChildCount := Nil;
- f_metaChildId := Nil;
-
-end;
-
-end.
-
diff --git a/tools/vdbview/svdb_obj.pas b/tools/vdbview/svdb_obj.pas
deleted file mode 100644
index e6b77e2..0000000
--- a/tools/vdbview/svdb_obj.pas
+++ /dev/null
@@ -1,339 +0,0 @@
-unit svdb_obj;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, selector_types, svdb;
-
-type
-
- { TSvdb_obj }
-
- TSvdb_obj = class
- private
- { private declarations }
- f_svdb : TSvdb;
- f_obj : Pointer;
-
- public
- { public declarations }
- constructor Create;
- destructor Destroy; override;
-
- function Acc2Path( acc : String ) : String;
- function OpenPath( path : String ) : boolean;
- procedure Close;
- function IsDb : boolean;
- function CountTabs : Integer;
- function TabName( tabid : Integer ) : String;
- function TabIdx( name : String ) : Integer;
- function CountCols( tabid, sel : Integer ) : Integer;
- function ColName( tabid, sel, colid : Integer ) : String;
- function ColIdx( tabid, sel : Integer; name : String ) : Integer;
- function SetColVisible( tabid, sel, colid, visible : Integer ) : Integer;
- function DefTypeIdx( tabid, sel, colid : Integer ) : Integer;
- function CountTypes( tabid, sel, colid : Integer ) : Integer;
- function TypeName( tabid, sel, colid, typeid : Integer ) : String;
- function TypeIdx( tabid, sel, colid : Integer; name : String ) : Integer;
- function TypeDomain( tabid, sel, colid, typeid : Integer ) : Integer;
- function TypeBits( tabid, sel, colid, typeid : Integer ) : Integer;
- function TypeDim( tabid, sel, colid, typeid : Integer ) : Integer;
-
- function OpenTable( tabid : Integer; defline : String ) : Integer;
- function MaxColnameLen( tabid, sel : Integer ) : Integer;
- procedure SetElemSep( tabid, sel, colid : Integer; sep : String );
- procedure SetDimSep( tabid, sel, colid : Integer; sep : String );
- function IsColEnabled( tabid, sel, colid : Integer ) : boolean;
- function RowRange( tabid : Integer ) : Int64;
- function FirstRow( tabid : Integer ) : Int64;
- function GetCell( tabid, sel, colid : Integer; row : Int64 ) : String;
- function FindFwd( tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : String ): Int64;
- function FindBwd( tabid, sel, colid : Integer;
- row : Int64; chunk : Integer; pattern : String ): Int64;
-
- function MetaRoot( tabid : Integer ) : Integer;
- function MetaName( tabid, metaid : Integer ) : String;
- function MetaValueLen( tabid, metaid : Integer ) : Integer;
- function MetaValuePrintable( tabid, metaid : Integer ) : boolean;
- function MetaValuePtr( tabid, metaid : Integer ) : Pointer;
- function MetaValue( tabid, metaid, trim : Integer ) : String;
- function MetaChildCount( tabid, metaid : Integer ) : Integer;
- function MetaChildId( tabid, metaid, childid : Integer ) : Integer;
-
- end;
-
-
-implementation
-
-{ TSvdb_obj }
-
-constructor TSvdb_obj.Create;
-begin
- inherited Create;
- f_svdb := TSvdb.Create;
- f_obj := Nil;
-end;
-
-destructor TSvdb_obj.Destroy;
-begin
- f_svdb.Free;
- inherited Destroy;
-end;
-
-function TSvdb_obj.Acc2Path( acc : String ) : String;
-begin
- Result := f_svdb.Acc2Path( acc );
-end;
-
-function TSvdb_obj.OpenPath( path : String ) : boolean;
-begin
- Close;
- f_obj := f_svdb.OpenPath( path );
- Result := ( f_obj <> Nil );
-end;
-
-procedure TSvdb_obj.Close;
-begin
- if ( f_obj <> Nil )
- then begin
- f_svdb.Close( f_obj );
- f_obj := Nil;
- end;
-end;
-
-function TSvdb_obj.IsDb : boolean;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.IsDb( f_obj )
- else Result := false;
-end;
-
-function TSvdb_obj.CountTabs : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.CountTabs( f_obj )
- else Result := 0;
-end;
-
-function TSvdb_obj.TabName( tabid : Integer ) : String;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TabName( f_obj, tabid )
- else Result := '';
-end;
-
-function TSvdb_obj.TabIdx( name : String ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TabIdx( f_obj, name )
- else Result := -1;
-end;
-
-function TSvdb_obj.CountCols( tabid, sel : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.CountCols( f_obj, tabid, sel )
- else Result := 0;
-end;
-
-function TSvdb_obj.ColName( tabid, sel, colid : Integer ) : String;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.ColName( f_obj, tabid, sel, colid )
- else Result := '';
-end;
-
-function TSvdb_obj.ColIdx( tabid, sel : Integer; name : String ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.ColIdx( f_obj, tabid, sel, name )
- else Result := -1;
-end;
-
-function TSvdb_obj.SetColVisible( tabid, sel, colid, visible : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.SetColVisible( f_obj, tabid, sel, colid, visible )
- else Result := 0;
-end;
-
-function TSvdb_obj.DefTypeIdx( tabid, sel, colid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.DefTypeIdx( f_obj, tabid, sel, colid )
- else Result := 0;
-end;
-
-function TSvdb_obj.CountTypes( tabid, sel, colid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.CountTypes( f_obj, tabid, sel, colid )
- else Result := 0;
-end;
-
-function TSvdb_obj.TypeName( tabid, sel, colid, typeid : Integer ) : String;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TypeName( f_obj, tabid, sel, colid, typeid )
- else Result := '';
-end;
-
-function TSvdb_obj.TypeIdx( tabid, sel, colid : Integer; name : String ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TypeIdx( f_obj, tabid, sel, colid, name )
- else Result := -1;
-end;
-
-function TSvdb_obj.TypeDomain( tabid, sel, colid, typeid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TypeDomain( f_obj, tabid, sel, colid, typeid )
- else Result := 0;
-end;
-
-function TSvdb_obj.TypeBits( tabid, sel, colid, typeid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TypeBits( f_obj, tabid, sel, colid, typeid )
- else Result := 0;
-end;
-
-function TSvdb_obj.TypeDim( tabid, sel, colid, typeid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.TypeDim( f_obj, tabid, sel, colid, typeid )
- else Result := 0;
-end;
-
-function TSvdb_obj.OpenTable( tabid : Integer; defline : String ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.OpenTable( f_obj, tabid, defline )
- else Result := 0;
-end;
-
-function TSvdb_obj.MaxColnameLen( tabid, sel : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MaxColnameLen( f_obj, tabid, sel )
- else Result := 0;
-end;
-
-procedure TSvdb_obj.SetElemSep( tabid, sel, colid : Integer; sep : String);
-begin
- if ( f_obj <> Nil )
- then f_svdb.SetElemSep( f_obj, tabid, sel, colid, sep );
-end;
-
-procedure TSvdb_obj.SetDimSep( tabid, sel, colid : Integer; sep : String );
-begin
- if ( f_obj <> Nil )
- then f_svdb.SetDimSep( f_obj, tabid, sel, colid, sep );
-end;
-
-function TSvdb_obj.IsColEnabled( tabid, sel, colid : Integer ) : boolean;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.IsColEnabled( f_obj, tabid, sel, colid )
- else Result := false;
-end;
-
-function TSvdb_obj.RowRange( tabid : Integer ) : Int64;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.RowRange( f_obj, tabid )
- else Result := 0;
-end;
-
-function TSvdb_obj.FirstRow( tabid : Integer ) : Int64;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.FirstRow( f_obj, tabid )
- else Result := 0;
-end;
-
-function TSvdb_obj.GetCell( tabid, sel, colid : Integer; row : Int64 ) : String;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.GetCell( f_obj, tabid, sel, colid, row )
- else Result := '';
-end;
-
-function TSvdb_obj.FindFwd( tabid, sel, colid : Integer; row : Int64;
- chunk : Integer; pattern : String ) : Int64;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.FindFwd( f_obj, tabid, sel, colid, row, chunk, pattern )
- else Result := -1;
-end;
-
-function TSvdb_obj.FindBwd( tabid, sel, colid : Integer; row : Int64;
- chunk : Integer; pattern : String ) : Int64;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.FindBwd( f_obj, tabid, sel, colid, row, chunk, pattern )
- else Result := -1;
-end;
-
-function TSvdb_obj.MetaRoot( tabid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaRoot( f_obj, tabid )
- else Result := -1;
-end;
-
-function TSvdb_obj.MetaName( tabid, metaid : Integer ) : String;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaName( f_obj, tabid, metaid )
- else Result := '';
-end;
-
-function TSvdb_obj.MetaValueLen( tabid, metaid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaValueLen( f_obj, tabid, metaid )
- else Result := -1;
-end;
-
-function TSvdb_obj.MetaValuePrintable( tabid, metaid : Integer ) : boolean;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaValuePrintable( f_obj, tabid, metaid )
- else Result := false;
-end;
-
-function TSvdb_obj.MetaValuePtr( tabid, metaid : Integer ) : Pointer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaValuePtr( f_obj, tabid, metaid )
- else Result := Nil;
-end;
-
-function TSvdb_obj.MetaValue( tabid, metaid, trim : Integer ) : String;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaValue( f_obj, tabid, metaid, trim )
- else Result := '';
-end;
-
-function TSvdb_obj.MetaChildCount( tabid, metaid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaChildCount( f_obj, tabid, metaid )
- else Result := -1;
-end;
-
-function TSvdb_obj.MetaChildId( tabid, metaid, childid : Integer ) : Integer;
-begin
- if ( f_obj <> Nil )
- then Result := f_svdb.MetaChildId( f_obj, tabid, metaid, childid )
- else Result := -1;
-end;
-
-end.
-
diff --git a/tools/vdbview/vdbview.ico b/tools/vdbview/vdbview.ico
deleted file mode 100644
index a2338d4..0000000
Binary files a/tools/vdbview/vdbview.ico and /dev/null differ
diff --git a/tools/vdbview/vdbview.lpi b/tools/vdbview/vdbview.lpi
deleted file mode 100644
index 6da3012..0000000
--- a/tools/vdbview/vdbview.lpi
+++ /dev/null
@@ -1,343 +0,0 @@
-<?xml version="1.0"?>
-<CONFIG>
- <ProjectOptions>
- <Version Value="9"/>
- <PathDelim Value="\"/>
- <General>
- <MainUnit Value="0"/>
- <UseAppBundle Value="False"/>
- <ResourceType Value="res"/>
- <UseXPManifest Value="True"/>
- <Icon Value="0"/>
- <ActiveWindowIndexAtStart Value="0"/>
- </General>
- <i18n>
- <EnableI18N LFM="False"/>
- </i18n>
- <VersionInfo>
- <StringTable ProductVersion=""/>
- </VersionInfo>
- <BuildModes Count="1" Active="Default">
- <Item1 Name="Default" Default="True"/>
- </BuildModes>
- <PublishOptions>
- <Version Value="2"/>
- <IncludeFileFilter Value="*.(pas|pp|inc|lfm|lpr|lrs|lpi|lpk|sh|xml)"/>
- <ExcludeFileFilter Value="*.(bak|ppu|o|so);*~;backup"/>
- </PublishOptions>
- <RunParams>
- <local>
- <FormatVersion Value="1"/>
- </local>
- </RunParams>
- <RequiredPackages Count="4">
- <Item1>
- <PackageName Value="FCL"/>
- <MinVersion Major="1" Valid="True"/>
- </Item1>
- <Item2>
- <PackageName Value="IDEIntf"/>
- <MinVersion Major="1" Valid="True"/>
- </Item2>
- <Item3>
- <PackageName Value="LazControls"/>
- <MinVersion Valid="True"/>
- </Item3>
- <Item4>
- <PackageName Value="LCL"/>
- </Item4>
- </RequiredPackages>
- <Units Count="25">
- <Unit0>
- <Filename Value="vdbview.lpr"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="vdbview"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1"/>
- <CursorPos X="46" Y="10"/>
- <UsageCount Value="202"/>
- </Unit0>
- <Unit1>
- <Filename Value="mainwindow.pas"/>
- <IsPartOfProject Value="True"/>
- <ComponentName Value="Mainform"/>
- <ResourceBaseClass Value="Form"/>
- <UnitName Value="MainWindow"/>
- <IsVisibleTab Value="True"/>
- <EditorIndex Value="0"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1"/>
- <CursorPos X="38" Y="11"/>
- <UsageCount Value="202"/>
- <Bookmarks Count="1">
- <Item0 X="5" Y="630" ID="0"/>
- </Bookmarks>
- <Loaded Value="True"/>
- <LoadedDesigner Value="True"/>
- </Unit1>
- <Unit2>
- <Filename Value="support_unit.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="support_unit"/>
- <WindowIndex Value="0"/>
- <TopLine Value="116"/>
- <CursorPos X="36" Y="135"/>
- <UsageCount Value="202"/>
- </Unit2>
- <Unit3>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\grids.pas"/>
- <UnitName Value="Grids"/>
- <WindowIndex Value="0"/>
- <TopLine Value="3020"/>
- <CursorPos X="1" Y="3048"/>
- <UsageCount Value="1"/>
- </Unit3>
- <Unit4>
- <Filename Value="background.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="background"/>
- <WindowIndex Value="0"/>
- <TopLine Value="153"/>
- <CursorPos X="1" Y="187"/>
- <UsageCount Value="201"/>
- </Unit4>
- <Unit5>
- <Filename Value="dataproxy.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="DataProxy"/>
- <WindowIndex Value="0"/>
- <TopLine Value="517"/>
- <CursorPos X="3" Y="520"/>
- <UsageCount Value="201"/>
- </Unit5>
- <Unit6>
- <Filename Value="linkedlist.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="linkedlist"/>
- <WindowIndex Value="0"/>
- <TopLine Value="52"/>
- <CursorPos X="3" Y="53"/>
- <UsageCount Value="201"/>
- </Unit6>
- <Unit7>
- <Filename Value="svdb_obj.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="svdb_obj"/>
- <WindowIndex Value="0"/>
- <TopLine Value="250"/>
- <CursorPos X="31" Y="262"/>
- <UsageCount Value="201"/>
- </Unit7>
- <Unit8>
- <Filename Value="svdb.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="svdb"/>
- <WindowIndex Value="0"/>
- <TopLine Value="408"/>
- <CursorPos X="1" Y="447"/>
- <UsageCount Value="201"/>
- </Unit8>
- <Unit9>
- <Filename Value="selector_types.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="selector_types"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1"/>
- <CursorPos X="6" Y="13"/>
- <UsageCount Value="206"/>
- </Unit9>
- <Unit10>
- <Filename Value="jobdef.pas"/>
- <IsPartOfProject Value="True"/>
- <UnitName Value="jobdef"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1"/>
- <CursorPos X="49" Y="13"/>
- <UsageCount Value="202"/>
- </Unit10>
- <Unit11>
- <Filename Value="columnsel.pas"/>
- <IsPartOfProject Value="True"/>
- <ComponentName Value="Columnform"/>
- <ResourceBaseClass Value="Form"/>
- <UnitName Value="ColumnSel"/>
- <WindowIndex Value="0"/>
- <TopLine Value="59"/>
- <CursorPos X="28" Y="93"/>
- <UsageCount Value="132"/>
- </Unit11>
- <Unit12>
- <Filename Value="C:\Apps\User_Installs\lazarus\fpc\2.4.2\source\rtl\objpas\classes\classesh.inc"/>
- <WindowIndex Value="0"/>
- <TopLine Value="663"/>
- <CursorPos X="15" Y="691"/>
- <UsageCount Value="16"/>
- </Unit12>
- <Unit13>
- <Filename Value="C:\Apps\User_Installs\lazarus\fpc\2.4.2\source\rtl\objpas\classes\stringl.inc"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1145"/>
- <CursorPos X="3" Y="1148"/>
- <UsageCount Value="10"/>
- </Unit13>
- <Unit14>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\forms.pp"/>
- <UnitName Value="Forms"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1380"/>
- <CursorPos X="3" Y="1174"/>
- <UsageCount Value="10"/>
- </Unit14>
- <Unit15>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\checklst.pas"/>
- <UnitName Value="CheckLst"/>
- <WindowIndex Value="0"/>
- <TopLine Value="57"/>
- <CursorPos X="3" Y="85"/>
- <UsageCount Value="2"/>
- </Unit15>
- <Unit16>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\actnlist.pas"/>
- <UnitName Value="ActnList"/>
- <WindowIndex Value="0"/>
- <TopLine Value="175"/>
- <CursorPos X="14" Y="203"/>
- <UsageCount Value="3"/>
- </Unit16>
- <Unit17>
- <Filename Value="gotoform.pas"/>
- <IsPartOfProject Value="True"/>
- <ComponentName Value="GotoForm"/>
- <ResourceBaseClass Value="Form"/>
- <UnitName Value="gotoform"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1"/>
- <CursorPos X="1" Y="20"/>
- <UsageCount Value="116"/>
- </Unit17>
- <Unit18>
- <Filename Value="C:\Apps\User_Installs\lazarus\fpc\2.4.2\source\rtl\objpas\sysutils\sysstrh.inc"/>
- <WindowIndex Value="0"/>
- <TopLine Value="92"/>
- <CursorPos X="66" Y="125"/>
- <UsageCount Value="3"/>
- </Unit18>
- <Unit19>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\include\customform.inc"/>
- <WindowIndex Value="0"/>
- <TopLine Value="344"/>
- <CursorPos X="1" Y="372"/>
- <UsageCount Value="3"/>
- </Unit19>
- <Unit20>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\dialogs.pp"/>
- <UnitName Value="Dialogs"/>
- <WindowIndex Value="0"/>
- <TopLine Value="480"/>
- <CursorPos X="11" Y="508"/>
- <UsageCount Value="10"/>
- </Unit20>
- <Unit21>
- <Filename Value="searchform.pas"/>
- <IsPartOfProject Value="True"/>
- <ComponentName Value="SForm"/>
- <ResourceBaseClass Value="Form"/>
- <UnitName Value="searchform"/>
- <WindowIndex Value="0"/>
- <TopLine Value="23"/>
- <CursorPos X="23" Y="55"/>
- <UsageCount Value="75"/>
- </Unit21>
- <Unit22>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\menus.pp"/>
- <UnitName Value="Menus"/>
- <WindowIndex Value="0"/>
- <TopLine Value="90"/>
- <CursorPos X="3" Y="118"/>
- <UsageCount Value="8"/>
- </Unit22>
- <Unit23>
- <Filename Value="C:\Apps\User_Installs\lazarus\lcl\graphics.pp"/>
- <UnitName Value="Graphics"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1111"/>
- <CursorPos X="15" Y="1139"/>
- <UsageCount Value="16"/>
- </Unit23>
- <Unit24>
- <Filename Value="version.pas"/>
- <IsPartOfProject Value="True"/>
- <ComponentName Value="VersionForm"/>
- <ResourceBaseClass Value="Form"/>
- <UnitName Value="version"/>
- <WindowIndex Value="0"/>
- <TopLine Value="1"/>
- <CursorPos X="3" Y="26"/>
- <UsageCount Value="20"/>
- </Unit24>
- </Units>
- <JumpHistory Count="6" HistoryIndex="5">
- <Position1>
- <Filename Value="mainwindow.pas"/>
- <Caret Line="613" Column="3" TopLine="562"/>
- </Position1>
- <Position2>
- <Filename Value="mainwindow.pas"/>
- <Caret Line="598" Column="45" TopLine="566"/>
- </Position2>
- <Position3>
- <Filename Value="mainwindow.pas"/>
- <Caret Line="611" Column="45" TopLine="583"/>
- </Position3>
- <Position4>
- <Filename Value="mainwindow.pas"/>
- <Caret Line="618" Column="45" TopLine="590"/>
- </Position4>
- <Position5>
- <Filename Value="mainwindow.pas"/>
- <Caret Line="577" Column="1" TopLine="575"/>
- </Position5>
- <Position6>
- <Filename Value="mainwindow.pas"/>
- <Caret Line="11" Column="38" TopLine="1"/>
- </Position6>
- </JumpHistory>
- </ProjectOptions>
- <CompilerOptions>
- <Version Value="9"/>
- <PathDelim Value="\"/>
- <Target>
- <Filename Value="vdbview"/>
- </Target>
- <SearchPaths>
- <IncludeFiles Value="$(ProjOutDir)"/>
- <UnitOutputDirectory Value="lib\$(TargetCPU)-$(TargetOS)"/>
- </SearchPaths>
- <Linking>
- <Options>
- <Win32>
- <GraphicApplication Value="True"/>
- </Win32>
- </Options>
- </Linking>
- <Other>
- <CompilerMessages>
- <UseMsgFile Value="True"/>
- </CompilerMessages>
- <CompilerPath Value="$(CompPath)"/>
- </Other>
- </CompilerOptions>
- <Debugging>
- <Exceptions Count="3">
- <Item1>
- <Name Value="EAbort"/>
- </Item1>
- <Item2>
- <Name Value="ECodetoolError"/>
- </Item2>
- <Item3>
- <Name Value="EFOpenError"/>
- </Item3>
- </Exceptions>
- </Debugging>
-</CONFIG>
diff --git a/tools/vdbview/vdbview.lpr b/tools/vdbview/vdbview.lpr
deleted file mode 100644
index 74c7f79..0000000
--- a/tools/vdbview/vdbview.lpr
+++ /dev/null
@@ -1,24 +0,0 @@
-program vdbview;
-
-{$mode objfpc}{$H+}
-
-uses
- {$IFDEF UNIX}{$IFDEF UseCThreads}
- cthreads,
- {$ENDIF}{$ENDIF}
- Interfaces, // this includes the LCL widgetset
- Forms, lazcontrols, MainWindow, ColumnSel, gotoform, searchform, version
- { you can add units after this };
-
-{$R *.res}
-
-begin
- Application.Initialize;
- Application.CreateForm(TMainform, Mainform);
- Application.CreateForm(TColumnform, Columnform);
- Application.CreateForm(TGotoForm, MyGotoForm);
- Application.CreateForm(TSForm, SForm);
- Application.CreateForm(TVersionForm, VersionForm);
- Application.Run;
-end.
-
diff --git a/tools/vdbview/vdbview.res b/tools/vdbview/vdbview.res
deleted file mode 100644
index 4b8fabf..0000000
Binary files a/tools/vdbview/vdbview.res and /dev/null differ
diff --git a/tools/vdbview/vdbview1.ico b/tools/vdbview/vdbview1.ico
deleted file mode 100644
index a2338d4..0000000
Binary files a/tools/vdbview/vdbview1.ico and /dev/null differ
diff --git a/tools/vdbview/version.lfm b/tools/vdbview/version.lfm
deleted file mode 100644
index dae1b99..0000000
--- a/tools/vdbview/version.lfm
+++ /dev/null
@@ -1,40 +0,0 @@
-object VersionForm: TVersionForm
- Left = 498
- Height = 215
- Top = 345
- Width = 346
- BorderStyle = bsDialog
- Caption = 'version...'
- ClientHeight = 215
- ClientWidth = 346
- LCLVersion = '0.9.30'
- object Panel1: TPanel
- Left = 0
- Height = 50
- Top = 165
- Width = 346
- Align = alBottom
- ClientHeight = 50
- ClientWidth = 346
- TabOrder = 0
- object BitBtn1: TBitBtn
- Left = 216
- Height = 30
- Top = 11
- Width = 122
- Caption = '&OK'
- Default = True
- Kind = bkOK
- ModalResult = 1
- TabOrder = 0
- end
- end
- object StaticText1: TStaticText
- Left = 8
- Height = 144
- Top = 8
- Width = 329
- Caption = 'Version 1.0.3'#13#10'December29, 2011'
- TabOrder = 1
- end
-end
diff --git a/tools/vdbview/version.pas b/tools/vdbview/version.pas
deleted file mode 100644
index f819c49..0000000
--- a/tools/vdbview/version.pas
+++ /dev/null
@@ -1,33 +0,0 @@
-unit version;
-
-{$mode objfpc}{$H+}
-
-interface
-
-uses
- Classes, SysUtils, FileUtil, Forms, Controls, Graphics, Dialogs, ExtCtrls,
- Buttons, StdCtrls;
-
-type
-
- { TVersionForm }
-
- TVersionForm = class(TForm)
- BitBtn1: TBitBtn;
- Panel1: TPanel;
- StaticText1: TStaticText;
- private
- { private declarations }
- public
- { public declarations }
- end;
-
-var
- VersionForm: TVersionForm;
-
-implementation
-
-{$R *.lfm}
-
-end.
-
diff --git a/tools/vtblcp/vtblcp.c b/tools/vtblcp/vtblcp.c
deleted file mode 100644
index a96771b..0000000
--- a/tools/vtblcp/vtblcp.c
+++ /dev/null
@@ -1,979 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "vtblcp.vers.h"
-#include "vtblcp-priv.h"
-
-#include <vdb/manager.h>
-#include <vdb/schema.h>
-#include <vdb/table.h>
-#include <vdb/cursor.h>
-#include <kapp/main.h>
-#include <klib/container.h>
-#include <klib/vector.h>
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <fmtdef.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-
-
-/* param block
- */
-typedef struct vtblcp_parms vtblcp_parms;
-struct vtblcp_parms
-{
- /* manager params, e.g. global schema, library include paths */
-
- /* schema params, e.g. include paths, source file */
- const char *schema_src;
-
- /* source table params */
- const char *src_path;
-
- /* destination table params */
- const char *dst_type;
- const char *dst_path;
-
- /* column list */
- const char **columns;
- uint32_t column_cnt;
-
- /* options */
- bool redact_sensitive;
-};
-
-/* column map
- * the write cursor determines which columns get copied
- * the read cursor should have a similar structure
- * each cursor will have its own column indices
- * this structure allows a simple ordinal mapping of the pairs
- */
-typedef struct vtblcp_column_map vtblcp_column_map;
-struct vtblcp_column_map
-{
- const void *redact_value;
- uint32_t wr, rd;
- bool sensitive;
- bool rd_filter;
- uint8_t align [ 2 ];
-};
-
-
-static
-rc_t copy_row ( const vtblcp_parms *pb, VCursor *dcurs, const VCursor *scurs,
- const vtblcp_column_map *cm, uint32_t count, uint32_t rdfilt_idx, int64_t row )
-{
- rc_t rc = VCursorOpenRow ( scurs );
- if ( rc != 0 )
- PLOGERR ( klogErr, (klogErr, rc, "failed to open source row '$(row)'", "row=%" LD64, row ));
- else
- {
- rc_t rc2;
- rc = VCursorOpenRow ( dcurs );
- if ( rc != 0 )
- PLOGERR ( klogErr, (klogErr, rc, "failed to open destination row '$(row)'", "row=%" LD64, row ));
- else
- {
- uint32_t i;
- const void *base;
- uint32_t elem_bits, boff, row_len;
-
- bool redact = false;
- if ( rdfilt_idx != 0 && pb -> redact_sensitive )
- {
- rc = VCursorCellData ( scurs, rdfilt_idx,
- & elem_bits, & base, & boff, & row_len );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to read cell data for read filter, row '$(row)'",
- "row=%" LD64, row ));
- }
- else
- {
- const uint8_t *rd_filter = base;
- for ( i = 0; i < row_len; ++ i )
- {
- if ( rd_filter [ i ] == 3 )
- {
- redact = pb -> redact_sensitive;
- break;
- }
- }
- }
- }
-
- for ( i = 0; i < count && rc != 0; ++ i )
- {
- /* get column data */
- rc = VCursorCellData ( scurs, cm [ i ] . rd,
- & elem_bits, & base, & boff, & row_len );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to read cell data for column '$(idx)', row '$(row)'",
- "idx=%u,row=%" LD64, i, row ));
- break;
- }
-
- if ( redact && cm [ i ] . sensitive )
- {
- uint32_t j;
-
- /* substitute base pointer with redact value */
- base = cm [ i ] . redact_value;
-
- /* redact destination */
- for ( j = 0; j < row_len; ++ j )
- {
- rc = VCursorWrite ( dcurs, cm [ i ] . wr, elem_bits, base, 0, 1 );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to redact cell data for column '$(idx)', row '$(row)'",
- "idx=%u,row=%" LD64, i, row ));
- break;
- }
- }
- }
- else
- {
- /* write to destination */
- rc = VCursorWrite ( dcurs, cm [ i ] . wr, elem_bits, base, boff, row_len );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to write cell data for column '$(idx)', row '$(row)'",
- "idx=%u,row=%" LD64, i, row ));
- break;
- }
- }
- }
-
- /* commit row */
- if ( rc == 0 )
- rc = VCursorCommitRow ( dcurs );
-
- rc2 = VCursorCloseRow ( dcurs );
- if ( rc == 0 )
- rc = rc2;
- }
-
- rc2 = VCursorCloseRow ( scurs );
- if ( rc == 0 )
- rc = rc2;
- }
-
- return rc;
-}
-
-
-static
-rc_t copy_table ( const vtblcp_parms *pb, VCursor *dcurs, const VCursor *scurs,
- const vtblcp_column_map *cm, uint32_t count, uint32_t rdfilt_idx )
-{
- /* open source */
- rc_t rc = VCursorOpen ( scurs );
- if ( rc != 0 )
- LOGERR ( klogErr, rc, "failed to open source cursor" );
- else
- {
- /* get row range */
- int64_t row, last;
- uint64_t range_count;
-
- rc = VCursorIdRange ( scurs, 0, & row, & range_count );
- last = row + range_count;
- if ( rc != 0 )
- LOGERR ( klogInt, rc, "failed to determine row range for source cursor" );
- else
- {
- /* open desination cursor */
- rc = VCursorOpen ( dcurs );
- if ( rc != 0 )
- LOGERR ( klogErr, rc, "failed to open destination cursor" );
- else
- {
- /* focus destination on initial source row */
- rc = VCursorSetRowId ( dcurs, row );
- if ( rc != 0 )
- PLOGERR ( klogErr, (klogErr, rc, "failed to set destination cursor row to id '$(row)'", "row=%" LD64, row ));
- else
- {
- /* copy each row */
- for ( ; row <= last; ++ row )
- {
- rc = copy_row ( pb, dcurs, scurs, cm, count, rdfilt_idx, row );
- if ( rc != 0 )
- break;
- }
-
- /* commit changes */
- if ( rc == 0 )
- rc = VCursorCommit ( dcurs );
- }
- }
- }
- }
-
- return rc;
-}
-
-typedef struct stype_id stype_id;
-struct stype_id
-{
- BSTNode n;
- const void *redact_value;
- uint32_t type_id;
- uint32_t elem_size;
-};
-
-
-static
-void CC stype_id_whack ( BSTNode *n, void *ignore )
-{
- free ( n );
-}
-
-static
-int CC stype_id_cmp ( const void *item, const BSTNode *n )
-{
- const VTypedecl *a = item;
- const stype_id *b = ( const stype_id* ) n;
-
- if ( a -> type_id < b -> type_id )
- return -1;
- return a -> type_id > b -> type_id;
-}
-
-static
-int CC stype_id_sort ( const BSTNode *item, const BSTNode *n )
-{
- const stype_id *a = ( const stype_id* ) item;
- const stype_id *b = ( const stype_id* ) n;
-
- if ( a -> type_id < b -> type_id )
- return -1;
- return a -> type_id > b -> type_id;
-}
-
-static
-rc_t mark_type_sensitivity ( const BSTree *stype_tbl, const VSchema *schema,
- const VTypedecl *td, vtblcp_column_map *cm )
-{
- const stype_id *node;
-
- /* simple case - look for exact matches */
- if ( BSTreeFind ( stype_tbl, td, stype_id_cmp ) != NULL )
- {
- cm -> sensitive = true;
- return 0;
- }
-
- /* exhaustive case - perform one by one ancestry test */
- for ( node = ( const stype_id* ) BSTreeFirst ( stype_tbl );
- node != NULL;
- node = ( const stype_id* ) BSTNodeNext ( & node -> n ) )
- {
- cm -> redact_value = NULL;
- if ( td -> type_id > node -> type_id )
- {
- VTypedecl cast;
-
- /* test for our type being a subtype */
- if ( VTypedeclToType ( td, schema, node -> type_id, & cast, NULL ) )
- {
- cm -> redact_value = node -> redact_value;
- cm -> sensitive = true;
- return 0;
- }
- }
- }
-
- /* doesn't appear to be sensitive */
- cm -> sensitive = false;
- return 0;
-}
-
-static
-rc_t populate_stype_tbl ( BSTree *stype_tbl, const VSchema *schema )
-{
- rc_t rc;
- uint32_t i;
- static struct
- {
- const char *typename;
- const char *redact_value;
- } sensitive_types [] =
- {
- /* original SRA types */
- { "INSDC:fasta", "N" },
- { "INSDC:csfasta", "." },
- { "NCBI:2na", "\x00" },
- { "NCBI:2cs", "\x00" },
- { "NCBI:4na", "\xFF" },
- { "NCBI:qual1", "\x00" },
- { "NCBI:qual4", "\xFB\xFB\xFB\xFB" },
- { "NCBI:isamp1", "\x00\x00\x00" },
- { "NCBI:isamp4", "\x00\x00\x00" },
- { "NCBI:fsamp1", "\x00\x00\x00" },
- { "NCBI:fsamp4", "\x00\x00\x00" },
- { "INSDC:dna:text", "N" },
- { "INSDC:dna:bin", "\x04" },
- { "INSDC:dna:4na", "\xFF" },
- { "INSDC:dna:2na", "\x00" },
- { "INSDC:color:text", "." },
- { "INSDC:color:bin", "\x04" },
- { "INSDC:color:2cs", "\x00" },
- { "INSDC:quality:phred", "\x00" },
- { "INSDC:quality:log_odds", "\x00\x00\x00" }
- /* signal types TBD */
- };
-
- BSTreeInit ( stype_tbl );
-
- for ( rc = 0, i = 0; i < sizeof sensitive_types / sizeof sensitive_types [ 0 ]; ++ i )
- {
- VTypedecl td;
- const char *decl = sensitive_types [ i ] . typename;
- rc = VSchemaResolveTypedecl ( schema, & td, "%s", decl );
- if ( rc == 0 )
- {
- stype_id *n;
- BSTNode *exist;
-
- VTypedesc desc;
- rc = VSchemaDescribeTypedecl ( schema, & desc, & td );
- if ( rc != 0 )
- {
- PLOGERR ( klogInt, (klogInt, rc, "failed to describe type '$(type)'", "type=%s", decl ));
- break;
- }
-
- n = malloc ( sizeof * n );
- if ( n == NULL )
- {
- rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
- LOGERR ( klogInt, rc, "failed to record sensitive data type" );
- break;
- }
-
- n -> redact_value = sensitive_types [ i ] . redact_value;
- n -> type_id = td . type_id;
- n -> elem_size = VTypedescSizeof ( & desc );
-
- rc = BSTreeInsertUnique ( stype_tbl, & n -> n, & exist, stype_id_sort );
- if ( rc != 0 )
- {
- free ( n );
- if ( GetRCState ( rc ) != rcExists )
- {
- LOGERR ( klogInt, rc, "failed to record sensitive data type" );
- break;
- }
- rc = 0;
- }
- }
- else if ( GetRCState ( rc ) == rcNotFound )
- {
- rc = 0;
- }
- else
- {
- break;
- }
- }
-
- return rc;
-}
-
-static
-rc_t populate_rdfilt_tbl ( BSTree *rftype_tbl, const VSchema *schema )
-{
- rc_t rc;
- uint32_t i;
- const char *rftypes [] =
- {
- "NCBI:SRA:read_filter",
- "INSDC:SRA:read_filter"
- };
-
- BSTreeInit ( rftype_tbl );
-
- for ( rc = 0, i = 0; i < sizeof rftypes / sizeof rftypes [ 0 ]; ++ i )
- {
- VTypedecl td;
- const char *decl = rftypes [ i ];
- rc = VSchemaResolveTypedecl ( schema, & td, "%s", decl );
- if ( rc == 0 )
- {
- BSTNode *exist;
-
- stype_id *n = malloc ( sizeof * n );
- if ( n == NULL )
- {
- rc = RC ( rcExe, rcNode, rcAllocating, rcMemory, rcExhausted );
- LOGERR ( klogInt, rc, "failed to record read_filter data type" );
- break;
- }
-
- n -> redact_value = NULL;
- n -> type_id = td . type_id;
- n -> elem_size = 8;
-
- rc = BSTreeInsertUnique ( rftype_tbl, & n -> n, & exist, stype_id_sort );
- if ( rc != 0 )
- {
- free ( n );
- if ( GetRCState ( rc ) != rcExists )
- {
- LOGERR ( klogInt, rc, "failed to record read_filter data type" );
- break;
- }
- rc = 0;
- }
- }
- else if ( GetRCState ( rc ) == rcNotFound )
- {
- rc = 0;
- }
- else
- {
- break;
- }
- }
-
- return rc;
-}
-
-
-static
-rc_t populate_cursors ( VTable *dtbl, VCursor *dcurs, const VCursor *scurs,
- vtblcp_column_map *cm, const Vector *v, uint32_t *rd_filt )
-{
- uint32_t end = VectorLength ( v );
- uint32_t i = VectorStart ( v );
-
- BSTree stype_tbl, rftype_tbl;
-
- const VSchema *schema;
- rc_t rc = VTableOpenSchema ( dtbl, & schema );
- if ( rc != 0 )
- {
- LOGERR ( klogInt, rc, "failed to open destination table schema" );
- return rc;
- }
-
- /* populate sensitive type table */
- rc = populate_stype_tbl ( & stype_tbl, schema );
- if ( rc != 0 )
- {
- VSchemaRelease ( schema );
- return rc;
- }
-
- /* populate read filter type table */
- rc = populate_rdfilt_tbl ( & rftype_tbl, schema );
- if ( rc != 0 )
- {
- BSTreeWhack ( & stype_tbl, stype_id_whack, NULL );
- VSchemaRelease ( schema );
- return rc;
- }
-
- for ( end += i, rc = 0, * rd_filt = 0; i < end; ++ i )
- {
- VTypedecl td;
- char typedecl [ 128 ];
-
- const char *spec = ( const void* ) VectorGet ( v, i );
-
- /* request column in destination */
- rc = VCursorAddColumn ( dcurs, & cm [ i ] . wr, "%s", spec );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to destination cursor", "spec=%s", spec ));
- break;
- }
-
- /* always retrieve data type */
- rc = VCursorDatatype ( dcurs, cm [ i ] . wr, & td, NULL );
- if ( rc != 0 )
- {
- PLOGERR ( klogInt, (klogInt, rc, "failed to determine datatype of destination column '$(name)'", "name=%s", spec ));
- break;
- }
-
- /* mark column as sensitive or not */
- rc = mark_type_sensitivity ( & stype_tbl, schema, & td, & cm [ i ] );
- if ( rc != 0 )
- break;
-
- /* if spec is already typed, request it in source */
- if ( spec [ 0 ] == '(' )
- {
- rc = VCursorAddColumn ( scurs, & cm [ i ] . rd, "%s", spec );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to source cursor", "spec=%s", spec ));
- break;
- }
- }
- else
- {
- rc = VTypedeclToText ( & td, schema, typedecl, sizeof typedecl );
- if ( rc != 0 )
- {
- PLOGERR ( klogInt, (klogInt, rc, "failed to print datatype of destination column '$(name)'", "name=%s", spec ));
- break;
- }
-
- rc = VCursorAddColumn ( scurs, & cm [ i ] . rd, "(%s)%s", typedecl, spec );
- if ( rc != 0 )
- {
- PLOGERR ( klogErr, (klogErr, rc, "failed to add '$(spec)' to source cursor", "spec=(%s)%s", typedecl, spec ));
- break;
- }
- }
-
- /* check if column is a read filter */
- cm [ i ] . rd_filter = false;
- if ( ! cm [ i ] . sensitive )
- {
- if ( BSTreeFind ( & rftype_tbl, & td, stype_id_cmp ) != NULL )
- {
- if ( * rd_filt != 0 )
- {
- rc = RC ( rcExe, rcColumn, rcOpening, rcColumn, rcExists );
- PLOGERR ( klogInt, (klogInt, rc, "can't use column '$(name)' as read filter", "name=%s", spec ));
- break;
- }
-
- * rd_filt = cm [ i ] . rd;
- cm [ i ] . rd_filter = true;
- }
- }
- }
-
- BSTreeWhack ( & rftype_tbl, stype_id_whack, NULL );
- BSTreeWhack ( & stype_tbl, stype_id_whack, NULL );
- VSchemaRelease ( schema );
-
- /* add read filter to input if not already there in some way */
- if ( * rd_filt == 0 )
- {
- rc = VCursorAddColumn ( scurs, rd_filt, "RD_FILTER" );
- if ( rc != 0 && GetRCState ( rc ) == rcNotFound )
- rc = 0;
- }
-
- return rc;
-}
-
-
-/* NOTES
-
- 1. default schema from code
- a. discovered by VDBManager upon "Make" ? look for "libsraschema.so"
- b. discovered by client?
- c. explicitly added when making "empty" schema?
-
- 2. need support for discovering all openable columns
- a. declaration in schema for creating a table from another,
- or actually from some set of base tables
-
- 3. need support for indicating/discovering native columns
- a. this is related in some way to knowing whether the vcolumn
- represents a kcolumn, and so is not likely to be a manually
- indicated property of the vcolumn, especially with subclasses.
-
-*/
-
-/* vtblcp
- */
-static
-rc_t vtblcp ( const vtblcp_parms *pb, const VTable *stbl, VTable *dtbl, const Vector *v )
-{
- const VCursor *scurs;
- rc_t rc = VTableCreateCursorRead ( stbl, & scurs );
- if ( rc != 0 )
- LOGERR ( klogInt, rc, "failed to create empty destination cursor" );
- else
- {
- VCursor *dcurs;
- rc = VTableCreateCursorWrite ( dtbl, & dcurs, kcmInsert );
- if ( rc != 0 )
- LOGERR ( klogInt, rc, "failed to create empty destination cursor" );
- else
- {
- uint32_t count = VectorLength ( v );
- vtblcp_column_map *cm = malloc ( sizeof *cm * count );
- if ( cm == NULL )
- {
- rc = RC ( rcExe, rcVector, rcAllocating, rcMemory, rcExhausted );
- LOGERR ( klogInt, rc, "failed to create column index map" );
- }
- else
- {
- uint32_t rd_filt;
- rc = populate_cursors ( dtbl, dcurs, scurs, cm, v, & rd_filt );
-
- if ( rc == 0 )
- rc = copy_table ( pb, dcurs, scurs, cm, count, rd_filt );
-
- free ( cm );
- }
-
- VCursorRelease ( dcurs );
- }
-
- VCursorRelease ( scurs );
- }
-
- return rc;
-}
-
-
-/* get_column_specs
- */
-static
-void CC free_column_spec ( void *elem, void *ignore )
-{
- free ( elem );
-}
-
-static
-rc_t get_column_specs ( const vtblcp_parms *pb, Vector *v, const VTable *stbl, VTable *dtbl )
-{
- rc_t rc;
-
- /* always prepare the vector */
- VectorInit ( v, 0, pb -> column_cnt );
-
- /* unable at this moment to auto-determine column list */
- if ( pb -> column_cnt == 0 )
- {
- rc = RC ( rcExe, rcSchema, rcEvaluating, rcFunction, rcUnsupported );
- LOGERR ( klogInt, rc, "failed to determine column specs" );
- }
- else
- {
- uint32_t i;
-
- /* process command line arguments */
- for ( rc = 0, i = 0; i < pb -> column_cnt; ++ i )
- {
- const char *src = pb -> columns [ i ];
-
- char *dst = malloc ( strlen ( src ) + 2 );
- if ( dst == NULL )
- {
- rc = RC ( rcExe, rcString, rcAllocating, rcMemory, rcExhausted );
- break;
- }
-
- strcpy ( dst, src );
-
- rc = VectorAppend ( v, NULL, dst );
- if ( rc != 0 )
- {
- free ( dst );
- break;
- }
- }
-
- /* failure */
- if ( rc != 0 )
- VectorWhack ( v, free_column_spec, NULL );
- }
-
- return rc;
-}
-
-
-/* close_table
- */
-static
-rc_t close_dst_table ( const vtblcp_parms *pb, VTable *tbl, rc_t entry_rc )
-{
- rc_t rc;
-
- /* perform "re-indexing" on columns
- this will go away soon once cursor sessions are supported */
- if ( entry_rc == 0 )
- entry_rc = VTableReindex ( tbl );
-
- /* close the table */
- rc = VTableRelease ( tbl );
-
- /* return most important return code */
- return entry_rc ? entry_rc : rc;
-}
-
-/* open_table
- */
-static
-rc_t open_src_table ( const vtblcp_parms *pb,
- const VDBManager *mgr, const VSchema *schema, const VTable **tblp )
-{
- return VDBManagerOpenTableRead ( mgr, tblp, schema, "%s", pb -> src_path );
-}
-
-static
-rc_t open_dst_table ( const vtblcp_parms *pb,
- VDBManager *mgr, const VSchema *schema, VTable **tblp )
-{
- return VDBManagerCreateTable ( mgr, tblp, schema,
- pb -> dst_type, kcmInit, "%s", pb -> dst_path );
-}
-
-
-/* init_schema
- */
-static
-rc_t init_schema ( const vtblcp_parms *pb,
- const VDBManager *mgr, VSchema **schemap )
-{
- VSchema *schema;
- rc_t rc = VDBManagerMakeSchema ( mgr, & schema );
- if ( rc != 0 )
- LOGERR ( klogInt, rc, "failed to create empty schema" );
- else
- {
- /* parse schema file */
- rc = VSchemaParseFile ( schema, "%s", pb -> schema_src );
- if ( rc != 0 )
- PLOGERR ( klogErr, (klogErr, rc, "failed to parse schema file '$(file)'", "file=%s", pb -> schema_src ));
- else
- {
- * schemap = schema;
- return 0;
- }
-
- VSchemaRelease ( schema );
- }
-
- return rc;
-}
-
-
-/* init_mgr
- */
-static
-rc_t init_mgr ( const vtblcp_parms *pb, VDBManager **mgrp )
-{
- VDBManager *mgr;
- rc_t rc = VDBManagerMakeUpdate ( & mgr, NULL );
- if ( rc != 0 )
- LOGERR ( klogInt, rc, "failed to open vdb library" );
- else
- {
- /* currently have no manager parameters */
- * mgrp = mgr;
- }
-
- return rc;
-}
-
-
-/* run
- */
-static
-rc_t run ( const vtblcp_parms *pb )
-{
- VDBManager *mgr;
- rc_t rc = init_mgr ( pb, & mgr );
- if ( rc == 0 )
- {
- VSchema *schema;
- rc = init_schema ( pb, mgr, & schema );
- if ( rc == 0 )
- {
- const VTable *stbl;
- rc = open_src_table ( pb, mgr, schema, & stbl );
- if ( rc == 0 )
- {
- VTable *dtbl;
- rc = open_dst_table ( pb, mgr, schema, & dtbl );
- if ( rc == 0 )
- {
- /* determine columns */
- Vector v;
- rc = get_column_specs ( pb, & v, stbl, dtbl );
-
- /* perform the copy */
- if ( rc == 0 )
- {
- rc = vtblcp ( pb, stbl, dtbl, & v );
- VectorWhack ( & v, free_column_spec, NULL );
- }
-
- /* cleanup */
- rc = close_dst_table ( pb, dtbl, rc );
- }
-
- VTableRelease ( stbl );
- }
-
- VSchemaRelease ( schema );
- }
-
- VDBManagerRelease ( mgr );
- }
-
- return rc;
-}
-
-
-/* Version
- */
-ver_t CC KAppVersion ( void )
-{
- return VTBLCP_VERS;
-}
-
-/* Usage
- */
-static
-void s_Usage ( const char *app_name )
-{
- printf ( "\n"
- "Usage: %s [ options ] src-table dst-table [ column-spec ... ]\n"
- "\n%s%s"
- " options:\n"
- " -h give tool help\n"
- " -K <path> schema file for output\n"
- " -T <table-name-expr> fully qualified table name expression for output table\n"
- "\n"
- " column-spec:\n"
- " NAME simple column name\n"
- " (typedecl)NAME specifically typed column name\n"
- "\n",
- app_name,
- " copy data from one table to another. default behavior is to use\n"
- " schema from source table to create copy. if a new schema is to be\n"
- " used, its identifier must be given with the '-T' option.\n"
- "\n"
- " the '-K' option allows external schema definitions to be loaded.\n"
- "\n",
- " the list of columns to copy may be given on the command line as\n"
- " white-space separated arguments. the syntax used differs from VDB proper\n"
- " in order to ease quoting.\n"
- "\n"
- " if no columns are given the intent is to automatically derive them\n"
- " from schema - NOT YET IMPLEMENTED.\n"
- "\n"
- );
-}
-
-static
-void MiniUsage ( const char *app_name )
-{
- printf ( "\n"
- "Usage: %s [ options ] src-table dst-table\n"
- " run with option '-h' for help\n"
- , app_name
- );
-}
-
-
-/* KMain
- */
-rc_t CC KMain ( int argc, char *argv [] )
-{
- int i;
- rc_t rc;
-
- /* expect paths and schema types */
- vtblcp_parms pb;
- memset ( & pb, 0, sizeof pb );
- pb . columns = ( const char** ) & argv [ 1 ];
-
- /* parse arguments */
- for ( rc = 0, i = 1; i < argc; ++ i )
- {
- const char *arg = argv [ i ];
- if ( arg [ 0 ] != '-' )
- {
- if ( pb . src_path == NULL )
- pb . src_path = arg;
- else if ( pb . dst_path == NULL )
- pb . dst_path = NULL;
- else
- {
- /* capture column name/spec */
- pb . columns [ pb . column_cnt ++ ] = arg;
- }
- }
- else do switch ( ( ++ arg ) [ 0 ] )
- {
- case 'K':
- pb . schema_src = NextArg ( & arg, & i, argc, argv, NULL, NULL );
- break;
- case 'T':
- pb . dst_type = NextArg ( & arg, & i, argc, argv, NULL, NULL );
- break;
- case 'h':
- case '?':
- s_Usage ( argv [ 0 ] );
- return 0;
- default:
- fprintf ( stderr, "unrecognized switch: '%s'\n", argv [ i ] );
- MiniUsage ( argv [ 0 ] );
- return RC ( rcExe, rcArgv, rcReading, rcParam, rcInvalid );
- }
- while ( arg [ 1 ] != 0 );
- }
-
- /* check arguments */
- if ( pb . src_path == NULL )
- {
- fprintf ( stderr, "missing source table path\n" );
- MiniUsage ( argv [ 0 ] );
- return RC ( rcExe, rcArgv, rcReading, rcParam, rcNotFound );
- }
- if ( pb . dst_path == NULL )
- {
- fprintf ( stderr, "missing destination table path\n" );
- MiniUsage ( argv [ 0 ] );
- return RC ( rcExe, rcArgv, rcReading, rcParam, rcNotFound );
- }
- if ( pb . schema_src == NULL )
- {
- fprintf ( stderr, "missing schema source file\n" );
- MiniUsage ( argv [ 0 ] );
- return RC ( rcExe, rcArgv, rcReading, rcParam, rcNotFound );
- }
- if ( pb . dst_type == NULL )
- {
- fprintf ( stderr, "missing destination table type description\n" );
- MiniUsage ( argv [ 0 ] );
- return RC ( rcExe, rcArgv, rcReading, rcParam, rcNotFound );
- }
-
- /* run tool */
- return run ( & pb );
-}
diff --git a/tools/vtblcp/vtblcp.vers b/tools/vtblcp/vtblcp.vers
deleted file mode 100644
index 8e8299d..0000000
--- a/tools/vtblcp/vtblcp.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.4.2
diff --git a/tools/workspace/.gitignore b/tools/workspace/.gitignore
deleted file mode 100644
index 6fa85f1..0000000
--- a/tools/workspace/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-prefs-priv.[ch]
-prefs-yacc.[ch]
-*.kar
diff --git a/tools/workspace/path-to-kar-file.cpp b/tools/workspace/path-to-kar-file.cpp
deleted file mode 100644
index 17dce54..0000000
--- a/tools/workspace/path-to-kar-file.cpp
+++ /dev/null
@@ -1,117 +0,0 @@
-#include <kapp/main.h>
-#include <klib/rc.h>
-#include <vfs/manager.h>
-#include <vfs/manager-priv.h>
-#include <vfs/path.h>
-#include <kfs/directory.h>
-
-#include "sratoolkit-exception.hpp"
-
-#include <iostream>
-
-namespace sra
-{
-
- static VFSManager *vfs;
- static KDirectory *wd;
-
-
- static
- rc_t CC HandleSourceFiles ( const KDirectory *dir, uint32_t type, const char *name, void *data )
- {
- rc_t rc = 0;
-
- unsigned int *indent = ( unsigned int * ) data;
- for ( unsigned int i = 0; i < * indent; ++ i )
- std :: cout << " ";
-
- switch ( type )
- {
- case kptFile:
- std :: cout << "File: " << name << std :: endl;
- break;
- case kptDir:
- {
- std :: cout << "Dir: " << name << std :: endl;
-
- unsigned int sub = * indent + 1;
-
- rc = KDirectoryVisit ( dir, false, HandleSourceFiles, & sub, "%s", name );
- break;
- }
- case kptFile | kptAlias:
- case kptDir | kptAlias:
- std :: cout << "Alias: " << name << std :: endl;
- break;
- default:
- return RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcUnknown );
- }
- return rc;
- }
-
- static
- void run ( int argc, char *argv [] )
- {
- assert ( argc > 1 );
-
- rc_t rc = VFSManagerMake ( &vfs );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "aaah" );
-
- rc = KDirectoryNativeDir ( &wd );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "aaah" );
-
- VPath *url;
- rc = VFSManagerMakePath ( vfs, & url, "%s", argv [ 1 ] );
- if ( rc == 0 )
- {
- const KDirectory *src;
-
- rc = VFSManagerOpenDirectoryReadDirectoryRelative ( vfs, wd, &src, url );
- if ( rc == 0 )
- {
- unsigned int indent = 0;
- rc = KDirectoryVisit ( src, false, HandleSourceFiles, & indent, "." );
-
- KDirectoryRelease ( src );
- }
-
- VPathRelease ( url );
- }
- }
-}
-
-ver_t CC KAppVersion ( void )
-{
- return 0;
-}
-
-const char UsageDefaultName[] = "aaaah";
-
-rc_t CC UsageSummary ( const char *progname )
-{
- return 0;
-}
-
-rc_t CC Usage ( const Args *args )
-{
- return 0;
-}
-
-
-rc_t CC KMain ( int argc, char *argv [] )
-{
- rc_t rc = 0;
-
- try
- {
- sra :: run ( argc, argv );
- }
- catch ( ... )
- {
- rc = RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown );
- }
-
- return rc;
-}
diff --git a/tools/workspace/prefs-lex.l b/tools/workspace/prefs-lex.l
deleted file mode 100644
index c06c072..0000000
--- a/tools/workspace/prefs-lex.l
+++ /dev/null
@@ -1,181 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-%{
- #include "token.h"
- #include "prefs-yacc.h"
- #include <sysalloc.h>
-
- #define YYVAL_TYPE( code ) \
- yylval -> t . var = code
-
- #define YYVAL_NONE() \
- YYVAL_TYPE ( val_none )
-
- #define YY_RETURN( tok ) \
- YYVAL_NONE (); \
- return tok
-
- #define YYVAL_RETURN( code, tok ) \
- YYVAL_TYPE ( code ); \
- return tok
-
- #define YYTXT_RETURN( code, tok ) \
- yylval -> t . val . c = yytext; \
- yylval -> t . len = yyleng; \
- YYVAL_RETURN ( code, tok )
-
-%}
-
-%option never-interactive nounistd yylineno
-
-%option prefix="prefs_yy"
-
-/* non-re-entrant scanner
-%option reentrant
- */
-
-/* bison options */
-%option bison-bridge
-
- /* maintain state stack */
-%option stack
-
- /* let parser handle EOF */
-%option noyywrap
-
- /* comment states */
-%x SLASH_STAR
-%x LINE_CMT
-
- /* common patterns */
-year [0-9][0-9][0-9][0-9]
-month [0-9][0-9]
-day [0-9][0-9]
-date {year}-{month}-{day}
-hour [0-9][0-9]
-minute [0-9][0-9]
-second [0-9][0-9]
-time {hour}:{minute}:{second}
-zone {hour}:{minute}
-
-%%
-
- /* multi-line comments */
-\/\* { yy_push_state ( SLASH_STAR ); }
-<SLASH_STAR>[^*\n]+ { /* ignore */ }
-<SLASH_STAR>\*+[^*/\n]+ { /* ignore */ }
-<SLASH_STAR>\**\n { ++ yylineno; }
-<SLASH_STAR>\*+\/ { yy_pop_state (); }
-
- /* line comments */
-"#" { yy_push_state ( LINE_CMT ); }
-<LINE_CMT>[^\\\n]+ { /* ignore */ }
-<LINE_CMT>\\[^\r\n] { /* ignore */ }
-<LINE_CMT>\\\r[^\n] { /* ignore */ }
-<LINE_CMT>\\\r\n { ++ yylineno; }
-<LINE_CMT>\\\n { ++ yylineno; }
-<LINE_CMT>\n {
- ++ yylineno;
- yy_pop_state ();
- YY_RETURN ( EOLN );
- }
-
- /* whitespace characters */
-[ \t\f\v\r] { /* ignore */ }
-\\\r\n { ++ yylineno; }
-\\\n { ++ yylineno; }
-\n { ++ yylineno; YY_RETURN ( EOLN ); }
-
- /* prefs members */
-"download-date" { YY_RETURN ( PD_DOWNLOAD_DATE ); }
-"last-check-date" { YY_RETURN ( PD_LAST_CHECK_DATE ); }
-"latest-version" { YY_RETURN ( PD_LATEST_VERS ); }
-"current-version" { YY_RETURN ( PD_CURRENT_VERS ); }
-"download-url" { YY_RETURN ( PD_DOWNLOAD_URL ); }
-"latest-vers-url" { YY_RETURN ( PD_LATEST_VERS_URL ); }
-"path-to-installation" { YY_RETURN ( PD_PATH_TO_INSTALLATION ); }
-"auto-download-enabled" { YY_RETURN ( PD_AUTO_DOWNLOAD_ENABLED ); }
-
- /* symbolic values */
-"true" { YY_RETURN ( KW_TRUE ); }
-"TRUE" { YY_RETURN ( KW_TRUE ); }
-"false" { YY_RETURN ( KW_FALSE ); }
-"FALSE" { YY_RETURN ( KW_FALSE ); }
-
- /* version tokens */
-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ {
- if ( toolkit_atov ( & yylval -> t . val . v, yytext, yyleng ) )
- { YYVAL_RETURN ( val_vers, MAJMINRELBUILD ); }
- YYTXT_RETURN ( val_txt, OVER_VERSION );
- }
-[0-9]+\.[0-9]+\.[0-9]+ {
- if ( toolkit_atov ( & yylval -> t . val . v, yytext, yyleng ) )
- { YYVAL_RETURN ( val_vers, MAJMINREL ); }
- YYTXT_RETURN ( val_txt, OVER_VERSION );
- }
-
- /* real numbers or two-part versions */
-[0-9]+\.[0-9]+ { YYTXT_RETURN ( val_txt, REAL ); }
-
- /* unsigned integers or one-part versions */
-[0-9]+ {
- if ( toolkit_dtoi ( & yylval -> t . val . u, yytext, yyleng ) )
- { YYVAL_RETURN ( val_u64, DECIMAL ); }
- YYTXT_RETURN ( val_txt, OVER_DECIMAL );
- }
- /* a string */
-\"[^\\"\f\r\n]*\" { YYTXT_RETURN ( val_quot, STRING ); }
-\"(\\.|[^\\"\f\r\n])+\" { YYTXT_RETURN ( val_esc, STRING ); }
-
-
- /* a current date */
-{date}T{time}Z {
- if ( toolkit_atotm ( & yylval -> t . val . t, yytext, yyleng ) )
- { YYVAL_RETURN ( val_time, DATETIME ); }
- YYTXT_RETURN ( val_txt, BAD_DATETIME );
- }
-{date}T{time}[+-]{zone} { YYTXT_RETURN ( val_txt, TZ_DATETIME ); }
-
- /* random punctuation */
-[-!%^&*+=|~/;:?,.@(){}<>[\]] { YY_RETURN ( yytext [ 0 ] ); }
-
-
- /* unrecognized input */
-. { YYTXT_RETURN ( val_txt, UNRECOGNIZED ); }
-
-%%
-
-rc_t PrefsInitFlex ( FILE * prefs )
-{
- yyin = prefs;
- return 0;
-}
-
-void PrefsDestroyFlex ( void )
-{
- yyin = NULL;
-}
diff --git a/tools/workspace/prefs-yacc.y b/tools/workspace/prefs-yacc.y
deleted file mode 100644
index 47f669c..0000000
--- a/tools/workspace/prefs-yacc.y
+++ /dev/null
@@ -1,189 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-%pure-parser
-
-%defines
-%debug
-%error-verbose
-%name-prefix="prefs_"
-
-%{
-
- /* general prelude for auto-generated C code */
- #include "token.h"
- #include "prefs-priv.h"
- #include <sysalloc.h>
-
-%}
-
-/* token and production values */
-%union
-{
- /* token from flex */
- Token t;
-
- /* a boolean */
- bool b;
-}
-
-%token < t > UNRECOGNIZED
-
-/* punctuation */
-%token < t > EOLN
-
-/* value tokens */
-%token < t > DECIMAL REAL MAJMINREL MAJMINRELBUILD DATETIME STRING
-%token < t > OVER_DECIMAL OVER_VERSION BAD_DATETIME TZ_DATETIME
-
-/* keywords */
-%token < t > KW_TRUE KW_FALSE
-%token < t > PD_DOWNLOAD_DATE PD_LAST_CHECK_DATE
-%token < t > PD_LATEST_VERS PD_CURRENT_VERS
-%token < t > PD_DOWNLOAD_URL PD_LATEST_VERS_URL
-%token < t > PD_PATH_TO_INSTALLATION
-%token < t > PD_AUTO_DOWNLOAD_ENABLED
-
-/* productions */
-%type < t > version version_value
-%type < t > timestamp
-%type < t > url
-%type < t > path
-%type < b > auto boolean
-
-%start prefs
-
-%%
-
-prefs
- : /* empty */
- | prefsline_seq
- ;
-
-prefsline_seq
- : prefsline
- | prefsline_seq prefsline
- ;
-
-prefsline
- : EOLN /* ignore empty lines */
- | PD_DOWNLOAD_DATE download_date
- | PD_LAST_CHECK_DATE last_check_date
- | PD_LATEST_VERS latest_version
- | PD_CURRENT_VERS current_version
- | PD_DOWNLOAD_URL download_url
- | PD_LATEST_VERS_URL latest_version_url
- | PD_PATH_TO_INSTALLATION path_to_installation
- | PD_AUTO_DOWNLOAD_ENABLED auto_download
- ;
-
-download_date
- : timestamp { PrefsDataSetDownloadDate ( & $1 ); }
- | malformed_timestamp EOLN
- ;
-
-last_check_date
- : timestamp { PrefsDataSetLastCheckDate ( & $1 ); }
- | malformed_timestamp EOLN
- ;
-
-latest_version
- : version { PrefsDataSetLatestVersion ( & $1 ); }
- | bad_version EOLN
- ;
-
-current_version
- : version { PrefsDataSetCurrentVersion ( & $1 ); }
- | bad_version EOLN
- ;
-
-download_url
- : url { PrefsDataSetDownloadURL ( & $1 ); }
- ;
-
-latest_version_url
- : url { PrefsDataSetLatestVersionURL ( & $1 ); }
- ;
-
-path_to_installation
- : path { PrefsDataSetPathToInstallation ( & $1 ); }
- ;
-
-auto_download
- : auto { PrefsDataSetAutoDownloadEnabled ( & $1 ); }
- ;
-
-auto
- : boolean { $$ = $1; }
- | '=' boolean { $$ = $2; }
- ;
-
-
-timestamp
- : DATETIME EOLN { $$ = $1; }
- | '=' DATETIME EOLN { $$ = $2; }
- ;
-
-malformed_timestamp
- : BAD_DATETIME { prefs_token_error ( & $1, "badly formed timestamp" ); }
- | TZ_DATETIME { prefs_token_error ( & $1, "timestamp must use GMT" ); }
- ;
-
-version
- : version_value EOLN { $$ = $1; }
- | '=' version_value EOLN { $$ = $2; }
- ;
-
-version_value
- : MAJMINRELBUILD { $$ = $1; }
- | MAJMINREL { $$ = $1; }
- | REAL { $$ = toolkit_rtov ( & $1 ); }
- | DECIMAL { $$ = toolkit_itov ( & $1 ); }
- ;
-
-bad_version
- : OVER_VERSION { prefs_token_error ( & $1, "version component overflow" ); }
- ;
-
-url
- : STRING
- ;
-
-path
- : STRING
- ;
-
-boolean
- : KW_TRUE { $$ = true; }
- | KW_FALSE { $$ = false; }
- ;
-
-%%
-
-void prefs_set_debug ( int enabled )
-{
- prefs_debug = enabled;
-}
diff --git a/tools/workspace/sra.cpp b/tools/workspace/sra.cpp
deleted file mode 100644
index af6229a..0000000
--- a/tools/workspace/sra.cpp
+++ /dev/null
@@ -1,1514 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <klib/text.h>
-#include <klib/out.h>
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
-#include <klib/debug.h>
-#include <klib/time.h>
-#include <klib/text.h>
-#include <klib/container.h>
-#include <klib/data-buffer.h>
-#include <kfg/config.h>
-#include <kns/manager.h>
-#include <kns/http.h>
-#include <kns/stream.h>
-#include <kfs/file.h>
-#include <kfs/directory.h>
-#include <kfs/gzip.h>
-#include <kfs/bzip.h>
-#include <kfs/defs.h>
-#include <vfs/manager.h>
-#include <vfs/manager-priv.h>
-#include <vfs/path.h>
-
-#include "prefs-priv.h"
-#include "token.h"
-#include "sra.vers.h"
-#include "sratoolkit-exception.hpp"
-
-#include <cerrno>
-#include <cstdlib>
-#include <cstring>
-#include <cstdio>
-#include <ctime>
-#include <string>
-#include <iostream>
-#include <sys/wait.h>
-#include <unistd.h>
-#include <time.h>
-
-
-namespace sra
-{
- struct PrefsData
- {
- PrefsData ();
- ~ PrefsData ();
-
- PrefsData & operator = ( const PrefsData & pd );
- PrefsData ( const PrefsData & pd );
-
- KTime_t dl_date; // download date
- KTime_t last_check_date;
- KTime_t last_check_date_interval_limit; // amount of time allowed between checks
- ver_t l_vers; // latest version available from NCBI
- ver_t c_vers; // current version on user's system
- std :: string download_url;
- std :: string latest_vers_url;
- std :: string path_to_installation;
- bool auto_download_enabled;
- };
-
- PrefsData :: PrefsData ()
- : dl_date ( 0 )
- , last_check_date ( 0 )
- , last_check_date_interval_limit ( 7 * 24 * 60 * 60 ) // 1 week
- , l_vers ( 0 )
- , c_vers ( 0 )
- , download_url ( "http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk" )
- // NB - this is currently a static file
- // needs to be replaced with CGI
- , latest_vers_url ( "http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version" )
- , auto_download_enabled ( true )
- {
- }
-
- PrefsData :: ~ PrefsData ()
- {
- }
-
- PrefsData & PrefsData :: operator = ( const PrefsData & pd )
- {
- dl_date = pd . dl_date;
- last_check_date = pd . last_check_date;
- last_check_date_interval_limit = pd . last_check_date_interval_limit;
- l_vers = pd . l_vers;
- c_vers = pd . c_vers;
- download_url = pd . download_url;
- latest_vers_url = pd . latest_vers_url;
- path_to_installation = pd . path_to_installation;
- auto_download_enabled = pd . auto_download_enabled;
-
- return * this;
- }
-
- PrefsData :: PrefsData ( const PrefsData & pd )
- : dl_date ( pd . dl_date )
- , last_check_date ( pd . last_check_date )
- , last_check_date_interval_limit ( pd . last_check_date_interval_limit )
- , l_vers ( pd . l_vers )
- , c_vers ( pd . c_vers )
- , download_url ( pd . download_url )
- , latest_vers_url ( pd . latest_vers_url )
- , path_to_installation ( pd . path_to_installation )
- , auto_download_enabled ( pd . auto_download_enabled )
- {
- }
-
-
- enum WhereFound { found_by_prefs, found_by_PATH };
-
- static PrefsData prefs_data;
- static bool dirty;
-
- static KTime_t current_time; // current time of running process
- static KTime_t lower_date_limit; //dawn of time for this program
- static std :: string prgm_name;
- static std :: string PATH;
- static std :: string path_to_binaries;
- static std :: string latest_vers_string; // TBD - record this from NCBI
- static std :: string path_to_prefs;
- static std :: string path_to_url;
- static KDirectory * wd;
- static VFSManager * vfs;
- static KNSManager * kns;
- static KConfig * kfg;
- static WhereFound where_found;
-
-
- static int g_argc;
- static char ** g_argv;
-
- enum SRAOption
- {
- opt_path_to_prefs,
- opt_offline_mode,
- opt_verbose,
- opt_auto_update,
-
- opt_unrecognized,
- opt_no_more_args
- };
-
- struct SRAOption_Iterator
- {
- SRAOption_Iterator ();
-
- int idx, num_params;
- const char *param;
-
- enum SRAOption next ();
- void remove ();
- };
-
- SRAOption_Iterator :: SRAOption_Iterator ()
- : idx ( 0 )
- , num_params ( 0 )
- , param ( "" )
- {
- }
-
- enum SRAOption SRAOption_Iterator :: next ()
- {
- idx += num_params + 1;
- num_params = 0;
-
- if ( idx >= g_argc )
- return opt_no_more_args;
-
- static struct
- {
- const char *opt_name;
- const char *alias;
- size_t num_params;
- const char *xtext;
- SRAOption opt_enum;
- }
- formal_opts [] =
- {
- { "--sra-prefs", NULL, 1, "expected path to prefs", opt_path_to_prefs },
- { "--offline-mode", NULL, 0, NULL, opt_offline_mode },
- { "--verbose", NULL, 0, NULL, opt_verbose },
- { "--auto-update", NULL, 1, "expected 'yes' or 'no'", opt_auto_update }
- };
-
- for ( uint32_t i = 0; i < sizeof formal_opts / sizeof formal_opts [ 0 ]; ++ i )
- {
- // "strcmp" is safe here because we know the formal_opt name
- // is properly NUL terminated and has a known, controlled length
- if ( strcmp ( g_argv [ idx ], formal_opts [ i ] . opt_name ) == 0 )
- {
- // right now we only support 0 or 1 option parameters
- // if we ever allow > 1, the class structure needs to change
- assert ( formal_opts [ i ] . num_params <= 1 );
-
- // look for option parameters
- for ( uint32_t p = 0; p < formal_opts [ i ] . num_params; ++ p )
- {
- // make sure the command line has supplied the parameter
- if ( ++ num_params + idx == g_argc )
- {
- rc_t rc = RC ( rcExe, rcNoTarg, rcNotFound, rcPath, rcEmpty );
- throw Exception ( rc, __FILE__, __LINE__, formal_opts [ i ] . xtext );
- }
-
- // record the option parameter
- param = g_argv [ idx + num_params ];
- }
-
- // return the option enum
- return formal_opts [ i ] . opt_enum;
- }
- }
-
- // unrecognized option - start of sra-shell parameters
- return opt_unrecognized;
- }
-
- void SRAOption_Iterator :: remove ()
- {
- int next_opt = idx + num_params + 1;
- int remaining = g_argc - next_opt;
- if ( remaining <= 0 )
- g_argc = idx;
- else
- {
- memmove ( & g_argv [ idx ], & g_argv [ next_opt ], remaining * sizeof g_argv [ 0 ] );
- g_argc -= num_params + 1;
- }
-
- g_argv [ g_argc ] = NULL;
- }
-
-
- static
- void InitFactoryDefaults ( void )
- {
- // create managers;
-
- rc_t rc = VFSManagerMake ( & vfs );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to create VFS manager" );
-
- rc = VFSManagerGetKNSMgr ( vfs, & kns );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to retrieve KNS manager" );
-
- // create Native Dir
- rc = KDirectoryNativeDir ( & wd );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to create native KDirectory" );
-
- // create KConfig
- rc = KConfigMake ( &kfg, NULL );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to create KConfig" );
-
- // locate NCBI_HOME directory from KConfig
- String *ncbi_home;
- rc = KConfigReadString ( kfg, "NCBI_HOME", &ncbi_home );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to determine NCBI_HOME" );
-
- // create path to preferences file
- path_to_prefs = std :: string ( ncbi_home -> addr, ncbi_home -> size );
- StringWhack ( ncbi_home );
- path_to_prefs += "/sra.prefs";
-
- // capture $PATH
- const char *p = getenv ( "PATH" );
- if ( p == NULL )
- throw Exception ( __FILE__, __LINE__, "failed to determine PATH" );
-
- PATH = std :: string ( p );
-
- // capture current timestamp
- current_time = KTimeStamp ();
-
- //cutoff time
- KTime cutoff = { 2014 };
- lower_date_limit = KTimeMakeTime ( &cutoff );
-
-
- }
-
- // user may want to indicate a different location from which
- // to load user preferences
- static
- void OverridePathToPrefs ( void )
- {
- SRAOption_Iterator it;
-
- while ( 1 )
- {
- switch ( it . next () )
- {
- case opt_path_to_prefs:
- path_to_prefs = std :: string ( it . param );
- it . remove ();
- break;
-
- case opt_unrecognized:
- case opt_no_more_args:
- return;
-
- default:
- break;
- }
- }
- }
-
- static
- void LoadPrefsFromFile ()
- {
- // open prefs file for read
- FILE *prefs = fopen ( path_to_prefs . c_str (), "r" );
- if ( prefs != NULL )
- {
- // init flex with file
- rc_t rc = PrefsInitFlex ( prefs );
- if ( rc != 0 )
- {
- fclose ( prefs );
- throw Exception ( rc, __FILE__, __LINE__, "failed to initialize prefs parser" );
- }
-
- // parse contents with prefs_parse
- prefs_parse ();
-
- // destroy flex
- PrefsDestroyFlex ();
-
- // close prefs
- fclose ( prefs );
- }
- }
-
- static
- void Init ( int argc, char *argv [] )
- {
- // capture in global variables
- g_argc = argc;
- g_argv = argv;
-
- prgm_name = argv [ 0 ];
-
- InitFactoryDefaults ();
- OverridePathToPrefs ();
- LoadPrefsFromFile ();
- }
-
- static
- void OverrideParamsFromCmdLine ( PrefsData & params )
- {
- SRAOption_Iterator it;
- while ( 1 )
- {
- // scan argv for options
- switch ( it . next () )
- {
- // path to installation
- case opt_path_to_prefs:
- // impossible - we already removed them all above.
- // good case for a debug msg... but doesn't hurt anything
- it . remove ();
- break;
-
- // offline non interactive mode
- case opt_offline_mode:
- // TBD - set offline mode, either to true
- // or if this option is made to take parameters,
- // interpret the parameter
- it . remove ();
- break;
-
- // verbocity
- case opt_verbose:
- // TBD - set verbosity level
- // this is either {none, some} or takes a parameter
- it . remove ();
- break;
-
- // no auto update
- case opt_auto_update:
- {
- std :: string up = std :: string ( it . param );
- if ( up . compare ( "yes" ) )
- params . auto_download_enabled = true;
- else if ( up . compare ( "no" ) )
- params . auto_download_enabled = false;
- else
- throw Exception ( __FILE__, __LINE__, "invalid option for '--auto-update'\n" );
- it . remove ();
- break;
- }
-
- case opt_unrecognized:
- case opt_no_more_args:
- return;
- }
- }
- }
-
- static
- void SetPrefsCurrentVersion () // !!!
- {
- // set version in prefs and working params
-
-
- // mark dirty
- dirty = true;
- }
-
- static
- bool IsPathToInstallation ( const char * path, const char *ext )
- {
- uint32_t path_type = KDirectoryPathType ( wd,
- "%s/%s"
-#if WINDOWS
- ".exe"
-#endif
- , path
- , ext );
- switch ( path_type )
- {
- case kptFile:
- case kptFile | kptAlias:
- // TBD - test for executable mode ??
- return true;
- }
-
- return false;
- }
-
- static
- bool IsPathToInstallation ( const char * path, unsigned int len, const char *ext )
- {
- uint32_t path_type = KDirectoryPathType ( wd,
- "%.*s/%s"
-#if WINDOWS
- ".exe"
-#endif
- , len
- , path
- , ext );
- switch ( path_type )
- {
- case kptFile:
- case kptFile | kptAlias:
- // TBD - test for executable mode ??
- return true;
- }
-
- return false;
- }
-
- static
- bool LocateUsingPrefsPath ( const PrefsData & params )
- {
- //# path names a directory, e.g. $HOME/sratoolkit-2.3.4
- //# on Windows, add ".exe" to end of file names
- //# implication of using existing prefs path is that we know the version
-
- if ( IsPathToInstallation ( params . path_to_installation . c_str (), "bin/sra-version" ) )
- path_to_binaries = params . path_to_installation + "/bin";
- else
- {
- if ( IsPathToInstallation ( params . path_to_installation . c_str (), "bin/fastq-dump" ) )
- path_to_binaries = params . path_to_installation + "/bin";
- else
- {
- if ( ! IsPathToInstallation ( params . path_to_installation . c_str (), "fastq-dump" ) )
- return false;
-
- path_to_binaries = params . path_to_installation;
- }
- }
-
-
- // record found in user-installation
- where_found = found_by_prefs;
-
- // !!! if ! version, obtain version by running tool
- // run sra-version to capture simple version
- // or run fastq-dump -V to capture and parse version
- SetPrefsCurrentVersion ();
-
- return true;
- }
-
- static
- bool LocateUsingPATH ( const PrefsData ¶ms )
- {
- //loop over $PATH using string_chr
- const char *sep, *dir = PATH . c_str ();
- while ( 1 )
- {
- // find next separator
- sep = strchr ( dir, ':' );
-
- // get length of this directory
- // if separator was found, distance from "dir" to ':'
- // otherwise, it's the remaining length in "dir"
- unsigned int len = ( sep != NULL ) ? sep - dir : strlen ( dir );
- if ( len != 0 )
- {
- if ( IsPathToInstallation ( dir, len, "sra-version" ) ||
- IsPathToInstallation ( dir, len, "fastq-dump" ) )
- {
- path_to_binaries = std :: string ( dir, len );
- break;
- }
- }
-
- // detect no more directories
- if ( sep == NULL )
- return false;
-
- // advance to next directory
- dir = sep + 1;
- }
-
- //record found in $PATH
- where_found = found_by_PATH;
-
- // !!! run sra-version to capture simple version
- //or run fastq-dump -V to capture and parse version
- SetPrefsCurrentVersion ();
-
- return true;
- }
-
- static
- bool LocateWithUserAssistance ( const PrefsData ¶ms )
- {
- //save prefs.path-to-installation
- std :: string prev_path = prefs_data . path_to_installation;
- //tell user that we cannot locate the installation
-
- std :: cout << "Unable to find a previous installation of SRAToolkit." << std :: endl;
-
- //loop
- while ( 1 )
- {
- std :: cout << "Would you like to provide a path to an installation of SRAToolkit? [ y / N ] ";
- std :: cout . flush ();
-
- std :: string ch;
- std :: getline ( std :: cin, ch );
-
- // anything other than y or Y exits loop/function
- if ( ch . empty () )
- break;
- if ( ch . compare ( "y" ) != 0 && ch . compare ( "Y" ) != 0 )
- break;
-
- std :: string path;
- std :: cout << "Please provide the path to the installation: "; // !!! should we give an example?
- std :: cout . flush ();
-
- std :: getline ( std :: cin, path );
-
- // empty == retry.
- if ( path . empty () )
- {
- std :: cout << "Empty path." << std :: endl;
- continue;
- }
-
- // slam in new potential value
- prefs_data . path_to_installation = path;
- prefs_data . c_vers = 0;
-
- // test user's input
- if ( LocateUsingPrefsPath ( prefs_data ) )
- {
- // found it - need to update the prefs
- dirty = true;
- return true;
- }
-
- // restore previous information
- prefs_data . path_to_installation = prev_path;
- std :: cout
- << " Unable to find SRAToolkit at: "
- << path
- << std :: endl;
- }
-
- return false;
- }
-
- static
- bool LocateSRAToolkit ( const PrefsData & params )
- {
- // first, try from previously written prefs
- // this cannot succeed the first time through,
- // because we have not created prefs yet.
- if ( LocateUsingPrefsPath ( params ) )
- return true;
-
- // MOST systems will not have an admin installation
- // but for those systems that do, check to see what's there
- if ( LocateUsingPATH ( params ) )
- return true;
-
- // the first time through, we will ask the user
- // to locate a previous installation, but the most
- // likely scenario is that there is none to be found
- if ( LocateWithUserAssistance ( params ) )
- return true;
-
- // probably a first time user
- return false;
- }
-
- static
- void UpdatePrefsFromCGI ( const char *data ) // !!!
- {
- // parse data buffer to get data
- // assign into prefs_data
- }
-
- static
- bool ContactNCBIForVersion ( const PrefsData & params )
- {
- bool success = false;
-
- assert ( prefs_data . latest_vers_url . c_str () != 0 ); // assume the url has been properly set
-
- //use HTTP to contact CGI
- KHttpRequest *req;
- rc_t rc = KNSManagerMakeRequest ( kns, &req, 0x01000000, NULL,
- "%s", prefs_data . latest_vers_url . c_str () );
-
-
- if ( rc == 0 )
- {
- try
- {
- // pass OS & version, architecture, current "sra" tool version
- // OS by KConfig "/OS"
- String *os;
- rc = KConfigReadString ( kfg, "OS", &os );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Could not determine operating system" );
-
- // add the POST parameter
- rc = KHttpRequestAddPostParam ( req, "OS=%S", os );
- StringWhack ( os );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Failed to post operating system" );
-
- // architecture bits by KConfig "/kfg/arch/bits"
- String *arch;
- rc = KConfigReadString ( kfg, "/kfg/arch/bits", &arch );
- if ( rc != 0 )
-
- // add the POST parameter
- rc = KHttpRequestAddPostParam ( req, "ARCH=%S", arch );
-
- StringWhack ( arch );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Failed to post system architecture" );
-
- // TBD - OS version we do not have - should add to KConfig
-
- // our version is KAppVersion
- ver_t vers = KAppVersion ();
-
- // add the POST parameter
- rc = KHttpRequestAddPostParam ( req, "SELF_VERSION=%V", vers );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Failed to post self version" );
-
- KHttpResult *rslt;
- rc = KHttpRequestPOST ( req, &rslt );
- if ( rc == 0 )
- {
- try
- {
- printf ( " Success posting a request\n" );
-
- uint32_t code;
- size_t msg_size;
- char msg_buff [ 1024 ];
- rc = KHttpResultStatus ( rslt, &code, msg_buff, sizeof msg_buff, &msg_size );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Could not retrieve status code" );
- if ( code != 200 )
- throw Exception ( rc, __FILE__, __LINE__, "Connection failed with status code: %u", code );
-
- KStream *response;
- rc = KHttpResultGetInputStream ( rslt, &response );
- if ( rc == 0 )
- {
- try
- {
- size_t num_read = 0;
- // TBD - since we are going to control the response
- // we can modify this size to fit accordingly
- char buffer [ 4096 ];
-
- rc = KStreamRead ( response, buffer, sizeof buffer, &num_read );
- if ( rc != 0 || num_read == 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to read response from NCBI" );
-
- // TBD - information to be obtained from CGI
-
- //store update information in prefs
- UpdatePrefsFromCGI ( buffer );
- }
- catch ( ... )
- {
- KStreamRelease ( response );
- throw;
- }
-
- KStreamRelease ( response );
- }
- }
- catch ( ... )
- {
- KHttpResultRelease ( rslt );
- throw;
- }
-
- KHttpResultRelease ( rslt );
- }
- }
- catch ( ... )
- {
- KHttpRequestRelease ( req );
- throw;
- }
-
- KHttpRequestRelease ( req );
- }
-
- return success;
- }
-
-
- // copy data from source file at NCBI to dest file on users system
- static
- rc_t CC CopyData ( const KFile *src, KFile *dst )
- {
- KDataBuffer buffer;
- rc_t rc = KDataBufferMakeBytes ( &buffer, 1024 * 1024 );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed create 1Mb kdatabuffer " );
- else
- {
- uint64_t read_pos = 0, write_pos = 0;
- size_t num_read = 0, num_writ = 0;
-
- while ( 1 )
- {
- rc = KFileReadAll ( src, read_pos, buffer . base, ( size_t ) buffer . elem_count, &num_read );
- if ( rc != 0 )
- {
- LogErr ( klogErr, rc, "Failed to read bytes for source file" );
- break;
- }
- if ( num_read == 0 )
- break;
-
-
- read_pos += num_read;
-
- rc = KFileWriteAll ( dst, write_pos, buffer . base, ( size_t ) buffer . elem_count, &num_writ );
- if ( rc != 0 )
- {
- LogErr ( klogErr, rc, "Failed to write bytes to dest file" );
- break;
- }
-
- if ( num_writ != num_read )
- {
- rc = RC ( rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete );
- LogErr ( klogErr, rc, "Failed to transfer all bytes to dest file" );
- break;
- }
-
- write_pos += num_writ;
- }
-
- KDataBufferWhack ( &buffer );
- }
-
- return rc;
- }
-
-#define string_endswith( str, len, ending ) \
- ( ( len ) >= sizeof ( ending ) && \
- memcmp ( & ( str ) [ ( len ) - sizeof ( ending ) + 1 ], ending, sizeof ( ending ) - 1 ) == 0 )
-
-
- static
- rc_t CC HandleFile ( const KDirectory *src_dir, KDirectory *dest_dir, const char *fname )
- {
- bool gz = false, bz2 = false;
- size_t fname_size = string_size ( fname );
- size_t dname_size = fname_size;
-
- // look for compressed files
- if ( string_endswith ( fname, fname_size, ".gz" ) )
- {
- gz = true;
- dname_size = fname_size - sizeof ".gz" - 1;
- }
- else if ( string_endswith ( fname, fname_size, ".bz2" ) )
- {
- bz2 = true;
- dname_size = fname_size - sizeof ".bz2" - 1;
- }
-
- const KFile *fsrc;
- rc_t rc = KDirectoryOpenFileRead ( src_dir, &fsrc, "%s", fname );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed open file for read" );
- else
- {
- // if file was compressed, uncompress
- if ( gz )
- {
- const KFile *gz;
- rc = KFileMakeGzipForRead ( & gz, fsrc );
- if ( rc == 0 )
- {
- KFileRelease ( fsrc );
- fsrc = gz;
- }
- }
- else if ( bz2 )
- {
- const KFile *bz2;
- rc = KFileMakeBzip2ForRead ( & bz2, fsrc );
- if ( rc == 0 )
- {
- KFileRelease ( fsrc );
- fsrc = bz2;
- }
- }
-
- // create the destination file on users system
- KFile *fdest;
- rc = KDirectoryCreateFile ( dest_dir, &fdest, false, 0600, kcmInit | kcmParents
- , "%.*s.tmp"
- , ( int ) dname_size
- , fname );
-
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed create file" );
- else
- {
- rc = CopyData ( fsrc, fdest );
-
- KFileRelease ( fdest );
- }
-
- KFileRelease ( fsrc );
-
-
- if ( rc != 0 )
- return rc;
-
-
-
- // rename dest.tmp to final name and setting mode bits
- uint32_t access;
- rc = KDirectoryAccess ( src_dir, &access, "%s", fname );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed to acquire directory access" );
- else
- {
- // TBD may need to tune up
- rc = KDirectorySetAccess ( dest_dir, false, access, 0777, "%.*s.tmp", ( int ) dname_size, fname );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed to set directory access" );
- else
- {
- KDataBuffer names;
- rc = KDataBufferMakeBytes ( & names, dname_size + dname_size + sizeof ".tmp" - 1 + 2 );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed create KDataBuffer" );
- else
- {
- char * from = ( char * ) names . base;
- char * to = from + dname_size + sizeof ".tmp";
-
- string_printf ( from, dname_size + sizeof ".tmp", NULL, "%.*s.tmp", ( uint32_t ) dname_size, fname );
- string_printf ( to, dname_size + 1, NULL, "%.*s", ( uint32_t ) dname_size, fname );
-
- rc = KDirectoryRename ( dest_dir, true, from, to );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed to rename directory" );
-
- KDataBufferWhack ( & names );
- }
- }
- }
- }
-
- return rc;
- }
-
- static
- rc_t CC HandleAlias ( const KDirectory *src_dir, void *dest_dir, const char *name )
- {
- char resolved [ 4096 ];
- // TBD - check shallow resolution of alias
- rc_t rc = KDirectoryResolveAlias ( src_dir, false, resolved, sizeof resolved, "%s", name );
- if ( rc == 0 )
- rc = KDirectoryCreateAlias ( ( KDirectory * ) dest_dir, 0777, kcmInit | kcmParents, resolved, name );
-
- return rc;
- }
-
- /* HandleSourceFiles
- * dir = source dir
- * name = name of file
- * data = destination dir
- */
- static
- rc_t CC HandleSourceFiles ( const KDirectory *src, uint32_t type, const char *name, void *data )
- {
- KDirectory *dst = ( KDirectory * ) data;
-
- // check for file type
- switch ( type )
- {
- case kptFile:
- return HandleFile ( src, dst, name );
- case kptDir:
- break;
- case kptFile | kptAlias:
- case kptDir | kptAlias:
- return HandleAlias ( src, dst, name );
- default:
- return RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- }
-
-
- // handle directory here. recursive.
- rc_t rc = KDirectoryCreateDir ( dst, 0777, kcmOpen | kcmParents, "%s", name );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed to create directory directory" );
- else
- {
- KDirectory *sub;
- rc = KDirectoryOpenDirUpdate ( dst, & sub, false, "%s", name );
- if ( rc != 0 )
- LogErr ( klogErr, rc, "Failed to open directory for update" );
- else
- {
- rc = KDirectoryVisit ( src, false, HandleSourceFiles, sub, "%s", name );
-
- KDirectoryRelease ( sub );
- }
- }
-
- return rc;
- }
-
- static
- void DownloadAndInstallToolkit ( const PrefsData & params )
- {
- // create the destination directory of the installation
- rc_t rc = KDirectoryCreateDir ( wd, 0775, kcmOpen | kcmParents,
- "%s", params . path_to_installation . c_str () );
- if ( rc == 0 )
- {
- // open for update
- KDirectory *dest_dir;
- rc = KDirectoryOpenDirUpdate ( wd, &dest_dir, false,
- "%s", params . path_to_installation . c_str () );
- if ( rc == 0 )
- {
- // establish connection to NCBI to access source files
- const KFile *src_file;
- rc = KNSManagerMakeReliableHttpFile ( kns, &src_file, NULL, 0x01010000,
- "%s", path_to_url . c_str () );
- if ( rc == 0 )
- {
- VPath *url;
- rc = VFSManagerMakePath ( vfs, & url, "%s", path_to_url . c_str () );
- if ( rc == 0 )
- {
- const KDirectory *src_dir;
- // hack to mount kar file in memory
- rc = VFSManagerOpenDirectoryReadDirectoryRelative ( vfs, wd, &src_dir, url );
- if ( rc == 0 )
- {
- // visit each entry in the directory and handle
- rc = KDirectoryVisit ( src_dir, false, HandleSourceFiles, dest_dir, "." );
-
- KDirectoryRelease ( src_dir );
- }
-
- VPathRelease ( url );
- }
-
- KFileRelease ( src_file );
- }
-
- KDirectoryRelease ( dest_dir );
- }
- }
- if ( rc != 0 )
- throw Exception ( __FILE__, __LINE__, "failed to get visit src_dir" );
- }
-
-
- static
- bool LatestVersionValid ( const PrefsData & params )
- {
- // latest version has not yet been set
- if ( params . l_vers == 0 )
- return false;
-
- // the params should contain only valid dates
- if ( current_time < params . last_check_date )
- throw Exception ( __FILE__, __LINE__, "%s", "invalid date" );
-
- // TBD - allow for some time interval
- // this may also be part of prefs - probably should be
- // default might be 1 week... 1 day?
- if ( current_time - params . last_check_date <=
- params . last_check_date_interval_limit );
- return true;
-
- // failed
- return false;
- }
-
- static
- bool CheckLatestVersion ( const PrefsData & params )
- {
- if ( LatestVersionValid ( params ) )
- return true;
-
- if ( ContactNCBIForVersion ( params ) )
- {
- prefs_data . last_check_date = current_time;
- dirty = true;
-
- return true;
- }
-
- return false;
- }
-
- static
- void EnsureLatestVersion ( const PrefsData & params )
- {
- if ( ! CheckLatestVersion ( params ) )
- return;
-
- //if ! user wants to be asked prefs.dont-ask-before-download
- // TBD maybe notify an installation is happening?
- if ( params . auto_download_enabled )
- DownloadAndInstallToolkit ( params );
- else
- {
- //ask if they want to download? [Yn]
-
- // first time around
- if ( params . c_vers == 0 )
- std :: cout << "Would you like to download and install the SRAToolkit? [Y/n] ";
- else
- std :: cout << "A newer version of SRAToolkit is available. Would you like to download and install? [Y/n] ";
-
- std :: cout . flush ();
-
- std :: string ch;
- std :: getline ( std :: cin, ch );
-
- if ( ch . empty () || ch . compare ( "y" ) == 0 || ch . compare ( "Y" ) == 0 )
- DownloadAndInstallToolkit ( params );
- }
- }
-
- static
- void EnsureVDBConfig ( const PrefsData & params )
- {
- //must have user public repository
- String *str;
- rc_t rc = KConfigReadString ( kfg, "/user/public", &str );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Failed to determine public repository" );
- StringWhack ( str );
-
- //must have ncbi remote
- rc = KConfigReadString ( kfg, "/ncbi/remote", &str );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "Failed to determine remote repository" );
- StringWhack ( str );
-
- //update KConfig with anything missing
- }
-
- static
- void SavePrefs ( void )
- {
- //return if ! dirty
- if ( ! dirty )
- return;
-
- //open prefs file for write
- FILE *prefs = fopen ( path_to_prefs . c_str (), "w" );
- if ( prefs == NULL )
- throw Exception ( __FILE__, __LINE__, "Failed to open file for save" );
-
- //write all prefs to prefs file
- char buffer [ 256 ];
-
- string_printf ( buffer, sizeof buffer, NULL, "%T", prefs_data . dl_date );
- fprintf ( prefs, "download-date = %s\n", buffer );
- string_printf ( buffer, sizeof buffer, NULL, "%T", prefs_data . last_check_date );
- fprintf ( prefs, "last-check-date = %s\n", buffer );
-
- string_printf ( buffer, sizeof buffer, NULL, "%V", prefs_data . l_vers );
- fprintf ( prefs, "latest-version = %s\n", buffer );
- string_printf ( buffer, sizeof buffer, NULL, "%V", prefs_data . c_vers );
- fprintf ( prefs, "current-version = %s\n", buffer );
-
- fprintf ( prefs, "download-url = %s\n", prefs_data . download_url . c_str () );
- fprintf ( prefs, "latest-ver-url = %s\n", prefs_data . latest_vers_url . c_str () );
- fprintf ( prefs, "path-to-installation = %s\n", prefs_data . path_to_installation . c_str () );
-
- fprintf ( prefs, "auto-download-enabled = %s\n", prefs_data . auto_download_enabled ? "true" : "false" );
-
- //close prefs file
- fclose ( prefs );
-
- //mark not dirty
- dirty = false;
- }
-
- //!!!
- static
- void RunSRAShell ( const PrefsData & params )
- {
- // append toolkit directory to PATH
- std :: string NEW_PATH ( path_to_binaries );
- NEW_PATH += ":";
- NEW_PATH += PATH;
- // TBD - use putenv or setenv to update PATH
-
- // launch sra-shell with remaining g_argv
-
- // wait upon return
-
- // throw exception if error
- }
-
- static
- void SRAMain ( void )
- throw ()
- {
- try
- {
- // make a copy of prefs data for current session
- PrefsData params ( prefs_data );
-
- // allow user to override prefs from command line
- OverrideParamsFromCmdLine ( params );
-
- // *** now have a complete idea of what we want to do ***
-
- // try to find the toolkit
- // not guaranteed to find anything
- LocateSRAToolkit ( params );
-
- // make sure that the software we have is up to date
- EnsureLatestVersion ( params );
-
- EnsureVDBConfig ( params );
-
- SavePrefs ();
-
- RunSRAShell ( params );
- }
- catch ( Exception & x )
- {
- // report nature of problem
- }
- catch ( ... )
- {
- // report unknown error
- }
- }
-
- static
- void Cleanup ( void )
- {
- // close managers
- KNSManagerRelease ( kns );
- kns = NULL;
-
- KDirectoryRelease ( wd );
- wd = NULL;
-
- KConfigRelease ( kfg );
- kfg = NULL;
- }
-
- static
- void run ( int argc, char *argv [] )
- {
- Init ( argc, argv );
- SRAMain ();
- Cleanup ();
- }
-}
-
-extern "C"
-{
- int prefs_yyget_lineno ( void );
- int prefs_yylex ( union YYSTYPE *val );
-
- int prefs_lex ( YYSTYPE *val )
- {
- return prefs_yylex ( val );
- }
-
- void prefs_error ( const char *msg )
- {
- std :: cerr
- << sra :: prgm_name
- << ':'
- << prefs_yyget_lineno ()
- << ": Error in preferences file: "
- << msg
- << std :: endl;
- }
-
- void prefs_token_error ( const Token *token, const char *msg )
- {
- std :: cerr
- << sra :: prgm_name
- << ':'
- << token -> lineno
- << ": Error in preferences file: "
- << msg
- << std :: endl;
- }
-
- static
- bool CheckValidTimeStamp ( const Token *token )
- {
- if ( token -> val . t < sra :: lower_date_limit )
- {
- prefs_token_error ( token, "timestamp is too old" );
- return false;
- }
- if ( token -> val . t > sra :: current_time )
- {
- prefs_token_error ( token, "timestamp is in the future" );
- return false;
- }
-
- return true;
- }
-
- static
- bool CheckValidToolkitVersion ( const Token *token )
- {
- const ver_t two_three = ( 2 << 24 ) | ( 3 << 16 );
- if ( token -> val . v < two_three )
- {
- prefs_token_error ( token, "version is too old" );
- return false;
- }
- // To the reader:
- // We have no way of knowing if the version beyond the latest
- // without contacting NCBI.
- // All version will have to be post-validated.
-
- return true;
- }
-
- static
- bool CheckValidURL ( const Token *token )
- {
- bool valid = false;
- // create VPath using VFSManagerMakePath
- VPath *path;
- rc_t rc = VFSManagerMakePath ( sra :: vfs, &path, "%s", token -> val . c );
- if ( rc != 0 )
- prefs_token_error ( token, "could not establish a valid path to check url" );
- else
- {
- // check that the scheme is http using VPathGetScheme.
- String scheme;
- rc = VPathGetScheme ( path, &scheme );
- if ( rc != 0 )
- prefs_token_error ( token, "could not establish scheme of url" );
- else
- {
- String http;
- CONST_STRING ( & http, "http" );
- if ( ! StringCaseEqual ( & scheme, & http ) )
- prefs_token_error ( token, "not a valid http url" );
- else
- {
- // check that the host is NCBI
- String host;
- rc = VPathGetHost ( path, &host );
- if ( rc != 0 )
- prefs_token_error ( token, "could not establish host of url" );
- else
- {
- String ncbi;
- CONST_STRING ( &ncbi, "www.ncbi.nlm.nih.gov" );
- if ( ! StringCaseEqual ( &host, &ncbi ) )
- prefs_token_error ( token, "not a valid host for url" );
- else
- valid = true;
- }
- }
- }
- VPathRelease ( path );
- }
- return valid;
- }
-
- static
- bool CheckValidPath ( const Token *token )
- {
- uint32_t type = KDirectoryPathType ( sra :: wd, "%.*s", token -> len, token -> val . c );
- switch ( type )
- {
- case kptFile:
- case kptDir:
- case kptFile | kptAlias:
- case kptDir | kptAlias:
- return true;
- }
- return false;
- }
-
- void PrefsDataSetDownloadDate ( const Token *download_date )
- {
- if ( CheckValidTimeStamp ( download_date ) )
- sra :: prefs_data . dl_date = download_date -> val . t;
- }
-
- void PrefsDataSetLastCheckDate ( const Token *last_check_date )
- {
- if ( CheckValidTimeStamp ( last_check_date ) )
- sra :: prefs_data . last_check_date = last_check_date -> val . t;
- }
-
- void PrefsDataSetLatestVersion ( const Token *latest_version )
- {
- if ( CheckValidToolkitVersion ( latest_version ) )
- sra :: prefs_data . l_vers = latest_version -> val . v;
- }
-
- void PrefsDataSetCurrentVersion ( const Token *current_version )
- {
- if ( CheckValidToolkitVersion ( current_version ) )
- sra :: prefs_data . c_vers = current_version -> val . v;
- }
-
- void PrefsDataSetDownloadURL ( const Token *download_url )
- {
- if ( CheckValidURL ( download_url ) )
- sra :: prefs_data . download_url = std :: string ( download_url -> val . c,
- download_url -> len );
- }
-
- void PrefsDataSetLatestVersionURL ( const Token *latest_version_url )
- {
- if ( CheckValidURL ( latest_version_url ) )
- sra :: prefs_data . latest_vers_url = std :: string ( latest_version_url -> val . c,
- latest_version_url -> len );
- }
-
- void PrefsDataSetPathToInstallation ( const Token *path_to_installation )
- {
- if ( CheckValidPath ( path_to_installation ) )
- sra :: prefs_data . path_to_installation = std :: string ( path_to_installation -> val . c,
- path_to_installation -> len );
- }
-
- void PrefsDataSetAutoDownloadEnabled ( bool which )
- {
- sra :: prefs_data . auto_download_enabled = which;
- }
-
- /* Version EXTERN
- * return 4-part version code: 0xMMmmrrrr, where
- * MM = major release
- * mm = minor release
- * rrrr = bug-fix release
- */
- ver_t CC KAppVersion ( void )
- {
- return SRA_VERS;
- }
-
-
- const char UsageDefaultName[] = "sra";
-
- /* Usage
- * This function is called when the command line argument
- * handling sees -? -h or --help
- */
- rc_t CC UsageSummary ( const char *progname )
- {
- return KOutMsg (
- "\n"
- "Usage:\n"
- " %s [Options]\n"
- "\n"
- "Summary:\n"
- " Launch bash shell with appropriate environment configuration.\n"
- , progname );
- }
-
- rc_t CC Usage ( const Args *args )
- {
-#if 0
- uint32_t i;
- const char *progname, *fullpath;
- rc_t rc = ArgsProgram ( args, & fullpath, & progname );
- if ( rc != 0 )
- progname = fullpath = UsageDefaultName;
-
- UsageSummary ( progname );
-
- KOutMsg ( "Options:\n" );
-
- for ( i = 0; i < sizeof options / sizeof options [ 0 ]; ++ i )
- {
- HelpOptionLine ( options [ i ] . aliases, options [ i ] . name,
- option_params [ i ], options [ i ] . help );
- }
-
- HelpOptionsStandard ();
-
- HelpVersion ( fullpath, KAppVersion () );
-#endif
- return 0;
- }
-
- rc_t CC KMain ( int argc, char *argv [] )
- {
- rc_t rc = 0;
-
- try
- {
- sra :: run ( argc, argv );
- }
- catch ( sra :: Exception &x )
- {
- rc = x . return_code ();
-#if _DEBUGGING
- // TBD - check that this doesn't conflict with internal attempts
- // at printing filename and lineno. It's probably just going to repeat...
- pLogErr ( klogErr, rc, "$(file):$(lineno): $(msg)",
- "file=%s,lineno=%u,msg=%s"
- , x . file_name ()
- , x . line_number ()
- , x . what () );
-#else
- LogErr ( klogErr, rc, x . what () );
-#endif
- }
- catch ( std :: exception & x )
- {
- rc = RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown );
- LogErr ( klogFatal, rc, x . what () );
- }
- catch ( ... )
- {
- rc = RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown );
- rc = LogErr ( klogFatal, rc, "unknown error" );
- }
-
- return rc;
-
- }
-}
diff --git a/tools/workspace/sra.vers b/tools/workspace/sra.vers
deleted file mode 100644
index 8a9ecc2..0000000
--- a/tools/workspace/sra.vers
+++ /dev/null
@@ -1 +0,0 @@
-0.0.1
\ No newline at end of file
diff --git a/tools/workspace/sratoolkit-exception.cpp b/tools/workspace/sratoolkit-exception.cpp
deleted file mode 100644
index 4c1c5bc..0000000
--- a/tools/workspace/sratoolkit-exception.cpp
+++ /dev/null
@@ -1,126 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include "sratoolkit-exception.hpp"
-#include <klib/printf.h>
-#include <klib/rc.h>
-
-namespace sra
-{
-
- /* Exception
- * create with or without an RC
- * with or without an accompanying text string
- */
-
- /* what
- * implement std::exception::what
- */
- const char * Exception :: what () const
- throw ()
- {
- try
- {
- return error_msg . c_str ();
- }
- catch ( ... )
- {
- return "INTERNAL ERROR";
- }
- }
-
- /* with only an RC
- * just records rc, filename and lineno
- * initializes what with empty or unspecified or something
- */
- Exception :: Exception ( rc_t _rc, const char * _filename, uint32_t _lineno )
- throw ()
- : filename ( _filename )
- , lineno ( _lineno )
- , rc ( _rc )
- {
- try
- {
- error_msg = std :: string ( "unspecified" );
- }
- catch ( ... )
- {
- }
- }
-
- /* with a printf-style error message
- * records filename and lineno
- * uses string_vprintf to build error_msg from fmt + args
- * initializes rc with an unknown error code
- */
- Exception :: Exception ( const char * _filename, uint32_t _lineno, const char *fmt, ... )
- throw ()
- : filename ( _filename )
- , lineno ( _lineno )
- , rc ( SILENT_RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown ) )
- {
- }
-
- /* with an RC and a printf-style error message
- * records rc, filename and lineno
- * uses string_vprintf to build error_msg from fmt + args
- */
- Exception :: Exception ( rc_t _rc, const char * _filename, uint32_t _lineno, const char *fmt, ... )
- throw ()
- : filename ( _filename )
- , lineno ( _lineno )
- , rc ( _rc )
- {
- try
- {
- va_list args;
- va_start ( args, fmt );
-
- size_t msg_size;
- char msg [ 4096 ];
- rc_t print_rc = string_vprintf ( msg, sizeof msg, & msg_size, fmt, args );
-
- va_end ( args );
-
- if ( print_rc == 0 )
- error_msg = std :: string ( msg, msg_size );
- else
- error_msg = std :: string ( "INTERNAL ERROR" );
- }
- catch ( ... )
- {
- }
- }
-
- /* destructor
- * tears down string
- */
- Exception :: ~ Exception ()
- throw ()
- {
- }
-
-}
diff --git a/tools/workspace/sratoolkit-exception.hpp b/tools/workspace/sratoolkit-exception.hpp
deleted file mode 100644
index bfeb659..0000000
--- a/tools/workspace/sratoolkit-exception.hpp
+++ /dev/null
@@ -1,111 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#ifndef _hpp_sratoolkit_exception_
-#define _hpp_sratoolkit_exception_
-
-#ifndef _h_klib_defs_
-#include <klib/defs.h>
-#endif
-
-#include <exception>
-#include <string>
-
-namespace sra
-{
-
- /* Exception
- * create with or without an RC
- * with or without an accompanying text string
- */
- class Exception : std::exception
- {
- public:
-
- /* what
- * implement std::exception::what
- */
- virtual const char *what () const
- throw ();
-
- /* return_code
- * get the offending rc
- */
- inline rc_t return_code () const
- { return rc; }
-
- /* file_name
- * get the filename where error occurred
- */
- inline const char *file_name () const
- { return filename; }
-
- /* line_number
- * get the line number where error occurred
- */
- inline uint32_t line_number () const
- { return lineno; }
-
- /* CONSTRUCTORS */
-
- /* with only an RC
- * just records rc, filename and lineno
- * initializes what with empty or unspecified or something
- */
- Exception ( rc_t rc, const char * filename, uint32_t lineno )
- throw ();
-
- /* with a printf-style error message
- * records filename and lineno
- * uses string_vprintf to build error_msg from fmt + args
- * initializes rc with an unknown error code
- */
- Exception ( const char * filename, uint32_t lineno, const char *fmt, ... )
- throw ();
-
- /* with an RC and a printf-style error message
- * records rc, filename and lineno
- * uses string_vprintf to build error_msg from fmt + args
- */
- Exception ( rc_t rc, const char * filename, uint32_t lineno, const char *fmt, ... )
- throw ();
-
- /* destructor
- * tears down string
- */
- virtual ~ Exception ()
- throw ();
-
- protected:
-
- std :: string error_msg;
- const char *filename;
- uint32_t lineno;
- rc_t rc;
- };
-}
-
-#endif /* _hpp_sratoolkit_exception_ */
diff --git a/tools/workspace/token.c b/tools/workspace/token.c
deleted file mode 100644
index f20a001..0000000
--- a/tools/workspace/token.c
+++ /dev/null
@@ -1,228 +0,0 @@
-/*===========================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * ===========================================================================
- *
- */
-
-#include "token.h"
-#include "prefs-priv.h"
-
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <ctype.h>
-#include <assert.h>
-
-
-/* conversion functions
- * returns false if there were problems, e.g. integer overflow
- * returns a typed value in "val" on success
- * "text" and "len" are taken from a volatile text buffer
- */
-bool toolkit_dtoi ( uint64_t * val, const char *text, int len )
-{
- uint64_t num;
- int i, cnt = len - 1;
-
- /* skip over leading zeros, leaving at least one digit */
- for ( i = 0; i < cnt; ++ i )
- {
- if ( text [ i ] != '0' )
- break;
- }
-
- /* initialize to left-most non-zero digit or zero */
- num = text [ i ] - '0';
-
- /* process remaining digits */
- for ( ++ i; i < len; ++ i )
- {
- uint64_t dig = text [ i ] - '0';
-
- /* test for overflow when scaling by 10 */
- if ( num > ( UINT64_MAX / 10 ) )
- return false;
- num *= 10;
-
- /* test for overflow when adding in new digit */
- if ( num > ( UINT64_MAX - dig ) )
- return false;
- num += dig;
- }
-
- /* return 64-bit unsigned value */
- * val = num;
- return true;
-}
-
-#undef VERS
-#define VERS( M, m, r, b ) \
- ( ( ( ver_t ) ( M ) << 24 ) | \
- ( ( ver_t ) ( m ) << 16 ) | \
- ( ( ver_t ) ( r ) << 8 ) | \
- ( ( ver_t ) ( b ) << 0) )
-
-bool toolkit_atov ( ver_t *val, const char *text, int len )
-{
- int i, part;
-
- uint32_t num [ 4 ];
- num [ 0 ] = num [ 1 ] = num [ 2 ] = num [ 3 ] = 0;
-
- for ( i = part = 0; i < len; ++ i )
- {
- if ( text [ i ] == '.' )
- ++ part;
- else
- {
- num [ part ] *= 10;
- num [ part ] += text [ i ] - '0';
- if ( num [ part ] >= 256 )
- {
- * val = 0;
- return false;
- }
- }
- }
-
- * val = VERS ( num [ 0 ], num [ 1 ], num [ 2 ], num [ 3 ] );
- return true;
-}
-
-bool toolkit_atotm ( KTime_t *val, const char *text, int len )
-{
- char *end;
- struct tm tm;
-
- /* must fit format of YYYY-MM-DDThh:mm:ssZ
- or potentially with time zone */
- if ( len < 20 )
- return false;
-
- /* get year 1900..2999 */
- tm . tm_year = ( int ) strtoul ( & text [ 0 ], & end, 10 );
- assert ( end == text + 4 );
- assert ( end [ 0 ] == '-' );
- if ( tm . tm_year < 1900 || tm . tm_year >= 3000 )
- return false;
-
- /* get month 1..12 */
- tm . tm_mon = ( int ) strtoul ( & text [ 5 ], & end, 10 );
- assert ( end == text + 7 );
- assert ( end [ 0 ] == '-' );
- if ( tm . tm_mon < 1 || tm . tm_mon > 12 )
- return false;
-
- /* get day 1..31 */
- tm . tm_mday = ( int ) strtoul ( & text [ 8 ], & end, 10 );
- assert ( end == text + 10 );
- assert ( toupper ( end [ 0 ] ) == 'T' );
- if ( tm . tm_mday < 1 || tm . tm_mday > 31 )
- return false;
-
- /* get hour 0..23 */
- tm . tm_hour = ( int ) strtoul ( & text [ 11 ], & end, 10 );
- assert ( end == text + 13 );
- assert ( end [ 0 ] == ':' );
- if ( tm . tm_hour > 23 )
- return false;
-
- /* get min 0..59 */
- tm . tm_min = ( int ) strtoul ( & text [ 14 ], & end, 10 );
- assert ( end == text + 16 );
- assert ( end [ 0 ] == ':' );
- if ( tm . tm_min > 59 )
- return false;
-
- /* get sec 0..61 - for leap seconds */
- tm . tm_sec = ( int ) strtoul ( & text [ 17 ], & end, 10 );
- assert ( end == text + 19 );
- assert ( toupper ( end [ 0 ] ) == 'Z' );
- if ( tm . tm_sec > 61 )
- return false;
-
- /* adjust for Unix mktime */
- tm . tm_year -= 1900;
- tm . tm_mon -= 1;
- * val = mktime ( & tm );
- return * val != -1;
-}
-
-/* rewrite token
- * convert integer to ver_t
- * integer is in t -> val . u
- * and we can accept value 0 .. 255 anything else is an error
- * maj only
- */
-Token toolkit_itov ( const Token *i )
-{
- Token t = *i;
-
- if ( i -> val . u > 255 )
- {
- prefs_token_error ( i, "version component overflow" );
- t . val . u = 0;
- }
- else
- {
- t . val . v = VERS ( i -> val . u, 0, 0 , 0 );
- t . type = val_vers;
- }
-
- return t;
-}
-
-/* rewrite token
- * real number to ver_t
- * real number is a string in t -> val . c
- * we perform two integer conversions from ASCII
- * split on '.'
- * maj . min
- */
-Token toolkit_rtov ( const Token *r )
-{
- Token t = *r;
-
- char *dot;
- unsigned long maj, min;
-
- assert ( r -> type == val_txt );
-
- /* we know that the string being used in the transformation is '.'
- * in between integers because of the lex parser */
- maj = strtoul ( r -> val . c, & dot, 10 );
- min = strtoul ( dot + 1, NULL, 10 );
-
- if ( maj > 255 || min > 255 )
- {
- prefs_token_error ( r, "version component overflow" );
- t . val . u = 0;
- }
- else
- {
- t . val . v = VERS ( maj, min, 0, 0 );
- t . type = val_vers;
- }
-
- return t;
-}
diff --git a/tools/workspace/token.h b/tools/workspace/token.h
deleted file mode 100644
index 53bf0af..0000000
--- a/tools/workspace/token.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*===========================================================================
- *
- * PUBLIC DOMAIN NOTICE
- * National Center for Biotechnology Information
- *
- * This software/database is a "United States Government Work" under the
- * terms of the United States Copyright Act. It was written as part of
- * the author's official duties as a United States Government employee and
- * thus cannot be copyrighted. This software/database is freely available
- * to the public for use. The National Library of Medicine and the U.S.
- * Government have not placed any restriction on its use or reproduction.
- *
- * Although all reasonable efforts have been taken to ensure the accuracy
- * and reliability of the software and data, the NLM and the U.S.
- * Government do not and cannot warrant the performance or results that
- * may be obtained by using this software or data. The NLM and the U.S.
- * Government disclaim all warranties, express or implied, including
- * warranties of performance, merchantability or fitness for any particular
- * purpose.
- *
- * Please cite the author in any work or product based on this material.
- *
- * ===========================================================================
- *
- */
-
-#ifndef _h_token_
-#define _h_token_
-
-#ifndef _h_klib_defs_
-#include <klib/defs.h>
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct Token Token;
-struct Token
-{
- /* token value ( if any ) */
- union
- {
- /* yytext */
- const char *c;
-
- /* unsigned integer */
- uint64_t u;
-
- /* version number */
- ver_t v;
-
- /* datetime */
- KTime_t t;
-
- } val;
-
- /* text length */
- uint32_t len;
-
- /* location of token in source */
- uint32_t lineno;
-
- /* token type - defined by bison/yacc */
- uint16_t type;
-
- /* value variant */
- uint16_t var;
-};
-
-
-/* token value variant codes */
-enum
-{
- val_none, /* no value */
- val_u64, /* val.u is valid */
- val_vers, /* val.v is valid */
- val_time, /* val.t is valid */
- val_txt, /* val.c is valid and NUL terminated */
- val_quot, /* val.c has a fully quoted NUL terminated string */
- val_esc /* val.c has a fully quoted string with escape sequences */
-};
-
-/* conversion functions
- * returns false if there were problems, e.g. integer overflow
- * returns a typed value in "val" on success
- * "text" and "len" are taken from a volatile text buffer
- */
-bool toolkit_dtoi ( uint64_t *val, const char *text, int len );
-bool toolkit_atov ( ver_t *val, const char *text, int len );
-bool toolkit_atotm ( KTime_t *val, const char *text, int len );
-Token toolkit_itov ( const Token *i );
-Token toolkit_rtov ( const Token *r );
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _h_token_ */
diff --git a/tools/workspace/workspace.cpp b/tools/workspace/workspace.cpp
deleted file mode 100644
index 8c42236..0000000
--- a/tools/workspace/workspace.cpp
+++ /dev/null
@@ -1,1116 +0,0 @@
-/*===========================================================================
-*
-* PUBLIC DOMAIN NOTICE
-* National Center for Biotechnology Information
-*
-* This software/database is a "United States Government Work" under the
-* terms of the United States Copyright Act. It was written as part of
-* the author's official duties as a United States Government employee and
-* thus cannot be copyrighted. This software/database is freely available
-* to the public for use. The National Library of Medicine and the U.S.
-* Government have not placed any restriction on its use or reproduction.
-*
-* Although all reasonable efforts have been taken to ensure the accuracy
-* and reliability of the software and data, the NLM and the U.S.
-* Government do not and cannot warrant the performance or results that
-* may be obtained by using this software or data. The NLM and the U.S.
-* Government disclaim all warranties, express or implied, including
-* warranties of performance, merchantability or fitness for any particular
-* purpose.
-*
-* Please cite the author in any work or product based on this material.
-*
-* ===========================================================================
-*
-*/
-
-#include <kapp/main.h>
-#include <kapp/args.h>
-#include <klib/text.h>
-#include <klib/out.h>
-#include <klib/log.h>
-#include <klib/rc.h>
-#include <klib/printf.h>
-#include <klib/debug.h>
-#include <kfg/config.h>
-#include <kns/manager.h>
-#include <kns/http.h>
-#include <kns/stream.h>
-#include <kfs/file.h>
-#include <kfs/directory.h>
-
-#include "prefs-priv.h"
-#include "token.h"
-#include "sra.vers.h"
-#include "sratoolkit-exception.hpp"
-
-#include <cerrno>
-#include <cstdlib>
-#include <cstring>
-#include <cstdio>
-#include <ctime>
-#include <string>
-#include <iostream>
-#include <sys/wait.h>
-#include <unistd.h>
-#include <time.h>
-
-
-#define OPT_IGNORE_FAILURE "ignore-failure"
-#define OPT_FORCE "force"
-
-static const char *hlp_ignore_failure [] = { "ignore failure when ......", NULL };
-static const char *hlp_force [] = { "force overwrite of exisiting destination", NULL };
-
-static OptDef options [] =
-{
- /* 1. long-name
- 2. list of single character short names
- 3. help-gen function
- 4. list of help strings, NULL terminated
- 5. max count
- 6. option requires value
- 7. option is required
- */
- { OPT_IGNORE_FAILURE, "i", NULL, hlp_ignore_failure, 1, false, false }
- , { OPT_FORCE, "f", NULL, hlp_force, 1, true, false }
-};
-
-static const char *option_params [] =
-{
- NULL
- , NULL
-};
-
-namespace workspace
-{
- struct PrefsData
- {
- PrefsData ();
-
- KTime_t dl_date; // download date
- KTime_t last_check_date;
- ver_t l_vers; // latest version available from NCBI
- ver_t c_vers; // current version on user's system
- std :: string download_url;
- std :: string latest_vers_url;
- std :: string path_to_installation;
- bool auto_download_enabled;
- };
-
- PrefsData :: PrefsData ()
- : dl_date ( 0 )
- , last_check_date ( 0 )
- , l_vers ( 0 )
- , c_vers ( 0 )
- , download_url ( "http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk" )
- // NB - this is currently a static file
- // needs to be replaced with CGI
- , latest_vers_url ( "http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version" )
- , auto_download_enabled ( true )
- {
- }
-
- static PrefsData prefs_data;
- static bool dirty;
-
- static std :: string latest_vers_string; // TBD - record this from NCBI
- static std :: string path_to_prefs;
- static KDirectory * wd;
- static KNSManager * kns;
-
-
- /* Initialize to specified default settings
- * Latest version
- * ( last check date )
- * url for checking latest version
- * current version
- * download date
- * path to the installation
- */
- static
- void InitWithDefaultSettings ()
- {
- // "prefs_data is initialized via constructor
-
- // by default, we will look for prefs file in user's home directory
- // set the default path to prefs file
- const char * HOME = getenv ( "HOME" );
-#if WINDOWS
- if ( HOME == NULL )
- HOME = getenv ( "USERPROFILE" );
-#endif
- if ( HOME != NULL )
- {
- path_to_prefs = HOME;
- path_to_prefs += "/.ncbi/sratoolkit.prefs";
- }
-
- // some operations will require access to KFS
- rc_t rc = KDirectoryNativeDir ( & wd );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to create native KDirectory" );
-
- // other operations will require access to KNS
- rc = KNSManagerMake ( & kns );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to create KNS manager" );
- }
-
-
-
- static
- const KFile * FindUserPreferences ( const Args *args )
- {
- const KFile * prefs = NULL;
-
- // see if the user is giving us specific prefs
- uint32_t opt_count;
- const char *opt_string = "-f"; // TBD - what option are we counting / looking for here
- rc_t rc = ArgsOptionCount ( args, opt_string, & opt_count );
- if ( rc == 0 && opt_count != 0 )
- {
- // send in 0-based index to fetch last prefs option
- const char *opt_value = NULL;
- rc = ArgsOptionValue ( args, opt_string, opt_count - 1, & opt_value );
- if ( rc == 0 && opt_value != NULL )
- {
- rc = KDirectoryOpenFileRead ( wd, & prefs, "%s", opt_value );
- if ( rc == 0 )
- {
- path_to_prefs = std :: string ( opt_value );
- return prefs;
- }
- }
- }
-
- // check factory defaults
- // if no $HOME could be determined, we have a problem
- if ( path_to_prefs . size () == 0 )
- {
- rc = RC ( rcExe, rcFile, rcOpening, rcPath, rcUndefined );
- throw Exception ( rc, __FILE__, __LINE__, "path to preferences could not be established" );
- }
-
- // use factory defaults
- rc = KDirectoryOpenFileRead ( wd, & prefs, "%s", path_to_prefs.c_str() );
- if ( rc == 0 )
- return prefs;
-
- // no prefs could be found
- DBGMSG ( DBG_APP, -1, ("no prefs found ( rc = %R )\n", rc ) );
-
- // leave factory defaults in place
- return NULL;
-
- }
-
- /* Init with users previous settings,
- * always initialized with default settings */
- static
- void InitWithUserSettings ( const KFile *prefs )
- {
-#if 0
- // 1. initialize lexical scanner with "prefs" file.
- // it will then read the entire text of the file
- // 2. call parser function, pass in the address of "prefs_data"
- // the PARSER will call all of the functions below whenever
- // it sees a line trying to set the value
- // 3. call the function to cleanup the lexical scanner
-#endif
-#if 0
- if ( PrefsDataSetDownloadDate ( &prefs_data, prefs . download_date ) )
- dirty = true;
- if ( PrefsDataSetLastCheckDate ( &prefs_data, prefs . last_check_date ) )
- dirty = true;
- if ( PrefsDataSetLatestVersion ( &prefs_data, prefs . latest_version ) )
- dirty = true;
- if ( PrefsDataSetCurrentVersion ( &prefs_data, prefs . current_version ) )
- dirty = true;
- if ( PrefsDataSetDownloadURL ( &prefs_data, prefs . download_url ) )
- dirty = true;
- if ( PrefsDataSetLatestVersionURL ( &prefs_data, prefs . latest_version_url ) )
- dirty = true;
- if ( PrefsDataSetPathToInstallation ( &prefs_data, prefs . path_to_installation ) )
- dirty = true;
- if ( PrefsDataSetAutoDownloadEnabled ( &prefs_data, prefs . download_enabled ) )
- dirty = true;
-#endif
- }
-
- static
- void InitWithCmdLineSettings ( const Args *args, const uint32_t count )
- {
- const char *dst;
- rc_t rc = ArgsParamValue ( args, count - 1, &dst );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "ArgsParamValue [ %u ] failed", count - 1 );
-
- // TBD - switch to the "prefs" data and the "working" data or whatever
- // allow the command line to change working data, but not prefs
- }
-
- static
- void Init ( int argc, char *argv [] )
- {
- InitWithDefaultSettings ();
-
- // parse command line
- Args *args;
- rc_t rc = ArgsMakeAndHandle ( & args, argc, argv, 0 );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to parse args list" );
-
- // have to perform these steps within try/catch
- // in order to handle cleanup of "args"
- try
- {
- const KFile *prefs = FindUserPreferences ( args );
- if ( prefs != NULL )
- {
- try
- {
- InitWithUserSettings ( prefs );
- }
- catch ( ... )
- {
- KFileRelease ( prefs );
- throw;
- }
-
- KFileRelease ( prefs );
- }
-
- // Command line settings trump previous user settings
- uint32_t count;
- rc = ArgsParamCount ( args, &count );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "ArgsParamCount failed" );
- else if ( count < 2 )
- {
- rc = RC ( rcExe, rcArgv, rcParsing, rcParam, rcInsufficient );
- throw Exception ( rc, __FILE__, __LINE__, " expect source amd destination parameters" );
- }
- else
- {
- InitWithCmdLineSettings ( args, count );
- }
-
-
- }
- catch ( ... )
- {
- ArgsWhack ( args );
- throw;
- }
-
- ArgsWhack ( args );
- }
-
- static
- void Cleanup ()
- {
- KNSManagerRelease ( kns );
- kns = NULL;
-
- KDirectoryRelease ( wd );
- wd = NULL;
- }
-
-
- static
- bool ReadInstallationVersion ()
- {
- return false;
- }
-
- /* Contact NCBI to find latest version of the ToolKit
- * No failure if connection is not established
- * if connection established, download information will be
- * gathered here
- */
- static
- bool GetLatestToolkitVersion ()
- {
- bool success = false;
-
- // NB - since all functions underneath are C, they are "nothrow"
- // there is no need for try/catch blocks unless C++ functions are
- // later interleaved.
-
- // read the latest published version from NCBI
- // NB - this has to change to use CGI
- assert ( prefs_data . latest_vers_url . c_str () != 0 ); // assume the url has been properly set
-
- KHttpRequest *req;
- rc_t rc = KNSManagerMakeRequest ( kns, &req, 0x01010000, NULL,
- "%s", prefs_data . latest_vers_url . c_str () );
- if ( rc == 0 )
- {
- KHttpResult *rslt;
- rc = KHttpRequestGET ( req, &rslt );
- if ( rc == 0 )
- {
- printf ( " Success Making a request\n" );
-
- uint32_t code;
- size_t msg_size;
- char msg_buff [ 1024 ];
- rc = KHttpResultStatus ( rslt, &code, msg_buff, sizeof msg_buff, &msg_size );
- if ( rc != 0 )
- printf ( "Could not retrieve status code\n" );
- else if ( code != 200 )
- printf ( "Connection failed with status code:%d\n", code );
- else
- {
- // TBD - do this
- // read the version ( text to ver_t )
- KStream *vers;
- rc = KHttpResultGetInputStream ( rslt, &vers );
- if ( rc == 0 )
- {
- size_t num_read = 0;
- char buffer [ 256 ];
-
- rc = KStreamRead ( vers, buffer, sizeof buffer, &num_read );
- if ( rc != 0 || num_read == 0 )
- {
- printf ( "Error retrieving version number\n" );
- return false;
- }
- else
- {
- printf( "Version: %s\n", buffer );
-
- ver_t new_vers = 0;
- if ( prefs_data . l_vers != new_vers )
- {
- // detect if different from current l_vers, update "dirty" flag
- dirty = true;
- // store it in static variables
- prefs_data . l_vers = new_vers;
- success = true;
- }
- }
-
- KStreamRelease ( vers );
- }
- }
-
- KHttpResultRelease ( rslt );
- }
-
- KHttpRequestRelease ( req );
- }
-
- return success;
- }
-
- /* Locate the toolkit in the user's environment
- * If it is not found, prompt the user if they have an existing version elsewhere
- * YES - Have the user direct us to it, and specify their desired directory
- * update location in prefs if it has changed.
- * If newer version exists, prompt to download and install.
- * NO - Prompt user to download and install the toolkit and specify a directory
- */
- static
- bool LocateToolkit ()
- {
- rc_t rc;
- const KFile *file;
-
- // first, test value from prefs
- if ( prefs_data . path_to_installation . size () != 0 )
- {
- // test if this path still points at an installation
- rc = KDirectoryOpenFileRead ( wd, &file, "%s", prefs_data . path_to_installation . c_str () );
- if ( rc == 0 )
- {
- KFileRelease ( file );
- return true;
- }
- }
-
- // try to find toolkit in current $PATH
- // NB - can be dangerous, since the tools may not belong to user
- char *PATH = getenv ( "PATH" );
- for ( char *path = strtok ( PATH, ":" ) ; path ; path = strtok ( NULL, ":" ) )
- {
- size_t psize = strlen ( path );
- const char fname [] = "sratoolkit.exe";
- path = strncat ( path, fname, psize + sizeof fname );
- rc = KDirectoryOpenFileRead ( wd, &file, "%s", path );
- if ( rc == 0 )
- {
- KFileRelease ( file );
- return true;
- }
- }
-
- // prompt user for location of previous installation
- std :: string path;
- std :: cout << "A previous installation of SRAToolkit was not found." << std :: endl
- << "If you have previously installed SRAToolkit, please indicate"
- << "where it is located. Otherwise type NA." << std :: endl
- << "Location: ";
- while ( 1 )
- {
- std :: getline ( std :: cin, path );
- if ( path . compare ( "NA" ) == 0 )
- break; // at this point the user has indicated tha the installation is not located anywhere else or they are done searching
- // should we prompt the user to tell us where he wants it installed, or exit.
- else
- {
- rc = KDirectoryOpenFileRead ( wd, &file, "%s", path . c_str () );
- if ( rc != 0 )
- {
- std :: cout << "Unable to locate SRAToolkit at: '"
- << path
- << '\'' << std :: endl
- << "Please indicate another location. Otherwise type NA" << std :: endl
- << "Location: ";
- continue;
- }
- else
- {
- KFileRelease ( file );
-
- // if user gave a new place, update prefs_data and set dirty
- //if ( ! PrefsDataSetPathToInstallation ( path . c_str () ) )
- // throw Exception ( rc, __FILE__, __LINE__, "Failed to update path to installation" );
-
- dirty = true;
- return true;
- }
- }
- }
- return false;
- }
-
-#if 0 //
- static
- rc_t RunInstaller ()
- {
- // cases:
- // 1. linux & mac: *.Kar.gz
- // read first few bytes ( <= 256K )
- // detect gz file, wrap in gunzip file ( <kfs/gzip.h> KFileMakeGzipForRead )
- // read first few bytes again
- // detect kar file
- // convert to KDirectory ( <kfs/tar.h> KDirectoryOpenTarArchiveRead_silent_preopened ? )
- // extract directory to target location ( <kfs/directory.h> KDirectoryCopy )
- // 2. windows:
- // the downloaded file should be executable
- // you will run
- rc_t rc;
-
-#if WINDOWS
-#else
- const KFile *file = NULL;
- rc = KDirectoryOpenFileRead ( wd, &file
- , "/tmp/sratoolkit.%s-%s%u%s"
- , latest_vers_string . c_str ()
- , os
- , ( uint32_t ) _ARCH_BITS
- , ext
- );
- if ( rc == 0 )
- {
- char buffer [ 10 ];
- size_t num_read;
- rc = KFileRead ( file, 0, buffer, sizeof buffer, &num_read );
- if ( rc == 0 && num_read == 10 )
- {
- if ( buffer [ 0 ] == 0x1f && buffer [ 1 ] == 0x8b )// anything else to check?
- {
- // we have have a gzip file
- const KFile *gz = NULL;
- rc = KFileMakeGzipForRead ( &gz, file );
- if ( rc == 0 )
- {
- //somehow use kar code.
- // Make a directory from kar file
- KDirectory *kardir = NULL;
-
- rc = KDirectoryCopy ( kardir, wd, true, kardir, wd );
- if ( rc == 0 )
- {
- // success
- }
-
- KFileRelease ( gz );
- }
- }
- }
- KFileRelease ( file );
- }
-#endif
-
-
-#if 0
- /* fork to child process and run installer*/
- rc_t rc = 0;
- int lerrno, status;
-
- pid_t pid = fork ();
- switch ( pid )
- {
- case -1:
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- break;
- case 0: /* name of file to execute, name associated with file being executed, sratoolkit package ( buffer )*/
- status = execl ( "copycat", "copycat", buffer, NULL );
-
- /* having returned here, execl failed */
- lerrno = errno;
- printf("This print is after execl() and should not have been executed if execl were successful! \n\n");
- rc = RC ( rcExe, rcNoTarg, rcReading, rcNoObj, rcError );
- PLOGERR (klogErr,
- (klogErr, rc, "unknown system error '$(S)($(E))'",
- "S=%!,E=%d", lerrno, lerrno));
-
- /* MUST exit here to kill the forked child process */
- exit ( status );
- default:
- /* NOW - you need to wait until your child exits */
- pid = waitpid ( pid, & status, 0 );
-
- /* pid should be valid */
- if ( pid < 0 )
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- else if ( status != 0 )
- {
- /* you can display some info here */
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- }
- }
-#endif
- return rc;
- }
-
-#endif //
- static
- void SetupToolkitConfig ()
- {
- }
-
- static
- bool DownloadAndInstallLatestToolkit ()
- {
- bool success = false;
-
- // test for download location
- if ( prefs_data . path_to_installation . size () == 0 )
- {
- // ask user for preferred location
- // suggest something by default
- }
-
- const char *os, *ext;
-#if LINUX
- os = "centos_linux";
- ext = ".tar.gz";
-#elif MAC
- os = "mac";
- ext = ".tar.gz";
-#elif WINDOWS
- os = "win";
- ext = ".zip";
-#else
-#error "this OS is not supported"
-#endif
-
- // build a URL based upon:
- // 1. base URL
- // 2. version
- // 3. OS
- // 4. architecture
- char download_url [ 4096 ];
- rc_t rc = string_printf ( download_url, sizeof download_url, NULL,
- "%s/%s/sratoolkit.%s-%s%u%s"
- , prefs_data . download_url . c_str ()
- , latest_vers_string . c_str ()
- , latest_vers_string . c_str ()
- , os
- , ( uint32_t ) _ARCH_BITS
- , ext
- );
-
- // retrieve latest toolkit package by version
- // NB - on Linux and Mac, this is normally a *.tar.gz file
- // on Windows, it may be one, too, or it may become an installer...
- const KFile *in_file;
- rc = KNSManagerMakeReliableHttpFile ( kns, &in_file, NULL, 0x01010000, download_url );
- if ( rc == 0 )
- {
- // TBD - is path-to-installation the directory of the toolkit,
- // or is it the directory of its binaries? should be directory
- // of the toolkit itself, but this complicates browsing...
- // EXAMPLE: "$HOME/sratoolkit.2.3.4/bin/fastq-dump" is an executable
- // then "prefs_data.path_to_installation" should be "$HOME/sratoolkit.2.3.4"
- // which brings up another issue - do we want to recognize this pattern
- // and rename it to remove the version number? auto download wants to
- // either use a consistent directory, or introduce parallel directories.
- //
- // SO - either install a new directory altogether,
- // or use the same directory but erase prior contents,
- // or use the same directory and leave prior contents.
-
- // DOWNLOAD LOCATION - different by platform
- KFile *out_file;
- rc = KDirectoryCreateFile ( wd, &out_file, false, 0664, kcmInit | kcmParents,
-#if WINDOWS
- // TBD - this should involve the username and process id
- "%s/tmp/sratoolkit.%s-%s%u%s"
- , prefs_data . path_to_installation . c_str ()
- , latest_vers_string . c_str ()
- , os
- , ( uint32_t ) _ARCH_BITS
- , ext
-#else
- // TBD - this should involve the username and process id
- "/tmp/sratoolkit.%s-%s%u%s"
- , latest_vers_string . c_str ()
- , os
- , ( uint32_t ) _ARCH_BITS
- , ext
-#endif
- );
- if ( rc == 0 )
- {
- uint64_t in_fsize;
- rc = KFileSize ( in_file, &in_fsize );
- if ( rc == 0 )
- {
- // allocate a buffer
- size_t bsize = 1024 * 1024;
- void *buffer = malloc ( bsize );
- if ( buffer == NULL )
- rc = RC ( rcExe, rcNoTarg, rcAllocating, rcNoObj, rcNull );
- else
- {
- uint64_t pos;
- size_t num_read;
- for ( pos = 0, num_read = 0 ; rc == 0 && pos < in_fsize ; pos += num_read )
- {
- uint64_t to_read = in_fsize - pos;
- if ( to_read > ( uint64_t ) bsize )
- to_read = bsize;
- rc = KFileReadAll ( in_file, pos, buffer, ( size_t ) to_read, &num_read );
- if ( rc != 0 )
- printf ( "Error reading download file" );
- else
- {
- size_t num_writ = 0;
- rc = KFileWriteAll ( out_file, pos, buffer, num_read, &num_writ );
- if ( rc != 0 )
- printf ( "Error writing to tmp file" );
- }
- }
-
- free ( buffer );
- }
- }
-
- KFileRelease ( out_file );
- }
-
- KFileRelease ( in_file );
- }
-
- // NOW, you have to unpack and install the file!
- //rc = RunInstaller ();
-
- // setup toolkit config
- SetupToolkitConfig ();
- return success;
- }
-
-
- static
- bool EnsureEverythingIsThere ()
- {
- bool have_toolkit = false;
- if ( LocateToolkit () )
- {
- // run a special tool ( TBD )
- // capture its output as a vers
- // compare it to stored prefs_data.c_vers
- // if different, update static and set dirty
- have_toolkit = ReadInstallationVersion ();
- }
-
- if ( prefs_data . auto_download_enabled )
- {
- if ( GetLatestToolkitVersion () )
- {
- if ( prefs_data . l_vers != prefs_data . c_vers )
- {
- // download and install toolkit
- if ( DownloadAndInstallLatestToolkit () )
- have_toolkit = true;
- }
- }
- }
-
- return have_toolkit;
- }
-
-
- static
- void UpdateEnvironment ()
- {
- /* makre sure OS + SHELL environment is set up correctly */
- const char *env = getenv ( "PATH" );
- size_t env_size = strlen ( env );
-
- char path [ env_size + 4096 ];
- rc_t rc = string_printf ( path, sizeof path, NULL,
- "%s:%s"
- , prefs_data . path_to_installation . c_str ()
- , env
- );
- if ( rc != 0 )
- ;// some error throw
-
- if ( putenv ( path ) )
- ;// some error throw
-
- }
-
- static
- rc_t DiscoverShell ( char *buffer, size_t bsize )
- {
- rc_t rc = 0;
-
- if ( buffer == NULL )
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcParam, rcNull );
- else
- {
- const char *shell = getenv ( "SHELL" );
- if ( shell == NULL )
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcParam, rcNotFound );
- else
- {
- size_t ssize = strlen ( shell );
- if ( bsize <= ssize )
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcBuffer, rcInsufficient );
- else
- {
- size_t num_copied = string_copy ( buffer, bsize, shell, ssize );
- if ( num_copied == bsize )
- rc = RC ( rcExe, rcNoTarg, rcCopying, rcBuffer, rcInsufficient );
- }
- }
- }
-
- return rc;
- }
-
- static
- void WriteTimestamp ( FILE *f, const char *cat, KTime_t ts )
- {
- time_t time = ( time_t ) ts;
-
- struct tm tm = * gmtime ( &time );
-
- fprintf ( f, "%s %04d-%02d-%02dT%02d:%02d:%02dZ\n", cat
- , tm . tm_year + 1900
- , tm . tm_mon + 1
- , tm . tm_mday
- , tm . tm_hour
- , tm . tm_min
- , tm . tm_sec
- );
- }
-
- static
- void WriteVersion ( FILE *f, const char *cat, ver_t v )
- {
- size_t num_writ;
- char buffer [ 256 ];
- string_printf ( buffer, sizeof buffer, & num_writ, "%s %V\n", cat, v );
- fwrite ( buffer, sizeof buffer [ 0 ], num_writ, f );
- }
-
- static
- void WriteString ( FILE *f, const char *cat, const char *value )
- {
- fprintf ( f, "%s \"%s\"\n", cat, value );
- }
-
- static
- void WriteBOOL ( FILE *f, const char *cat, bool value )
- {
- fprintf ( f, "%s %s\n", cat, value ? "true" : "false" );
- }
-
- static
- void SavePrefs ()
- {
- if ( dirty )
- {
- std :: string name = path_to_prefs;
- name += ".tmp";
-
- // open temporary prefs file
- FILE *prefs = fopen ( name . c_str (), "w" );
- if ( prefs != NULL )
- {
- WriteTimestamp ( prefs, "download-date", prefs_data . dl_date );
- WriteTimestamp ( prefs, "last-check-date", prefs_data . last_check_date );
-
- WriteVersion ( prefs, "latest-version", prefs_data . l_vers );
- WriteVersion ( prefs, "current-version", prefs_data . c_vers );
-
- WriteString ( prefs, "download-url", prefs_data . download_url . c_str () );
- WriteString ( prefs, "latest-vers-url", prefs_data . latest_vers_url . c_str () );
- WriteString ( prefs, "path-to-installation", prefs_data . path_to_installation . c_str () );
-
- WriteBOOL ( prefs, "auto-download-enabled", prefs_data . auto_download_enabled );
-
- // close the temporary prefs file
- fclose ( prefs );
-
- // rename the temporary file to the real prefs name
- rc_t rc = KDirectoryRename ( wd, true, "prefs.tmp", "prefs.prefs" );
- if ( rc != 0 )
- throw Exception ( rc, __FILE__, __LINE__, "failed to rename file" );
- }
- dirty = false;
- }
- }
-
- static
- rc_t LaunchShell ()
- {
- int lerrno, status;
-
- char buffer [ 256 ];
- rc_t rc = DiscoverShell ( buffer, sizeof buffer );
- if ( rc == 0 )
- {
- /* fork to child process */
- pid_t pid = fork ();
- switch ( pid )
- {
- case -1:
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- break;
- case 0:
- status = execl ( buffer, buffer, "-i", NULL );
-
- /* having returned here, execl failed */
- lerrno = errno;
- printf("This print is after execl() and should not have been executed if execl were successful! \n\n");
- rc = RC ( rcExe, rcNoTarg, rcReading, rcNoObj, rcError );
- PLOGERR (klogErr,
- (klogErr, rc, "unknown system error '$(S)($(E))'",
- "S=%!,E=%d", lerrno, lerrno));
-
- /* MUST exit here to kill the forked child process */
- exit ( status );
- default:
- /* NOW - you need to wait until your child exits */
- pid = waitpid ( pid, & status, 0 );
-
- /* pid should be valid */
- if ( pid < 0 )
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- else if ( status != 0 )
- {
- /* you can display some info here */
- rc = RC ( rcExe, rcNoTarg, rcValidating, rcNoObj, rcError );
- }
- }
- }
-
- return rc;
- }
-
- static
- rc_t RunShell ()
- {
- rc_t rc;
- rc = LaunchShell ( );
- return rc;
- }
-
- static
- void run ( int argc, char *argv[] )
- {
- // put the tool into a known state
- // all of the information needed to start is known
- Init ( argc, argv );
-
- try
- {
- // contact NCBI, locate installation
- if ( EnsureEverythingIsThere () )
- {
- UpdateEnvironment ();
-
- RunShell ();
- }
- }
- catch ( ... )
- {
- Cleanup ();
- throw;
- }
-
- Cleanup ();
- }
-}
-
-extern "C"
-{
-
- void PrefsDataSetDownloadDate ( const Token *download_date )
- {
- if ( download_date -> val . t >= 0 /* TBD - some master date */
- || download_date -> val . t >= workspace :: prefs_data . dl_date ) // >= very small chance double
- // download in < time than registered interval
- workspace :: prefs_data . dl_date = download_date -> val . t;
- }
-
- void PrefsDataSetLastCheckDate ( const Token *last_check_date )
- {
- if ( last_check_date -> val . t >= workspace :: prefs_data . last_check_date ) // again slight chance that the time interval
- // between check dates wont register as different
- workspace :: prefs_data . last_check_date = last_check_date -> val . t;
- }
-
- void PrefsDataSetLatestVersion ( const Token *latest_version )
- {
- if ( latest_version -> val . v >= workspace :: prefs_data . l_vers )
- workspace :: prefs_data . l_vers = latest_version -> val . v;
- }
-
- void PrefsDataSetCurrentVersion ( const Token *current_version )
- {
- if ( current_version -> val . v >= workspace :: prefs_data . c_vers )
- workspace :: prefs_data . c_vers = current_version -> val . v;
- }
-
- void PrefsDataSetDownloadURL ( const Token *download_url )
- {
- if ( download_url -> val . c != NULL )
- workspace :: prefs_data . download_url = std :: string ( download_url -> val . c );
- }
-
- void PrefsDataSetLatestVersionURL ( const Token *latest_version_url )
- {
- if ( latest_version_url -> val .c != NULL )
- workspace :: prefs_data . latest_vers_url = std :: string ( latest_version_url -> val . c );
- }
-
- void PrefsDataSetPathToInstallation ( const Token *path_to_installation )
- {
- if ( path_to_installation -> val . c != NULL )
- workspace :: prefs_data . path_to_installation = std :: string ( path_to_installation -> val . c );
- }
-
- void PrefsDataSetAutoDownloadEnabled ( bool which )
- {
- workspace :: prefs_data . auto_download_enabled = which;
- }
-
- /* Version EXTERN
- * return 4-part version code: 0xMMmmrrrr, where
- * MM = major release
- * mm = minor release
- * rrrr = bug-fix release
- */
- ver_t CC KAppVersion ( void )
- {
- return SRA_VERS;
- }
-
-
- const char UsageDefaultName[] = "sra";
-
- /* Usage
- * This function is called when the command line argument
- * handling sees -? -h or --help
- */
- rc_t CC UsageSummary ( const char *progname )
- {
- return KOutMsg (
- "\n"
- "Usage:\n"
- " %s [Options]\n"
- "\n"
- "Summary:\n"
- " Launch bash shell with appropriate environment configuration.\n"
- , progname );
- }
-
- rc_t CC Usage ( const Args *args )
- {
- uint32_t i;
- const char *progname, *fullpath;
- rc_t rc = ArgsProgram ( args, & fullpath, & progname );
- if ( rc != 0 )
- progname = fullpath = UsageDefaultName;
-
- UsageSummary ( progname );
-
- KOutMsg ( "Options:\n" );
-
- for ( i = 0; i < sizeof options / sizeof options [ 0 ]; ++ i )
- {
- HelpOptionLine ( options [ i ] . aliases, options [ i ] . name,
- option_params [ i ], options [ i ] . help );
- }
-
- HelpOptionsStandard ();
-
- HelpVersion ( fullpath, KAppVersion () );
-
- return 0;
- }
-
- /* KMain
- * C entrypoint
- * call C++ function
- * prevent any exceptions from leaking out
- */
- rc_t CC KMain ( int argc, char *argv [] )
- {
- rc_t rc = 0;
-
- try
- {
- workspace :: run ( argc, argv );
- }
- catch ( workspace :: Exception & x )
- {
- rc = x . return_code ();
-#if _DEBUGGING
- // TBD - check that this doesn't conflict with internal attempts
- // at printing filename and lineno. It's probably just going to repeat...
- pLogErr ( klogErr, rc, "$(file):$(lineno): $(msg)",
- "file=%s,lineno=%u,msg=%s"
- , x . file_name ()
- , x . line_number ()
- , x . what () );
-#else
- LogErr ( klogErr, rc, x . what () );
-#endif
- }
- catch ( std :: exception & x )
- {
- rc = RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown );
- LogErr ( klogFatal, rc, x . what () );
- }
- catch ( ... )
- {
- rc = RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown );
- rc = LogErr ( klogFatal, rc, "unknown error" );
- }
-
- return rc;
- }
-
-}
diff --git a/win-configure.bat b/win-configure.bat
deleted file mode 100644
index 47a955e..0000000
--- a/win-configure.bat
+++ /dev/null
@@ -1 +0,0 @@
-perl -w ./konfigure.perl %*
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/sra-sdk.git
More information about the debian-med-commit
mailing list