[med-svn] [sra-sdk] 01/07: Imported Upstream version 2.3.4-2+dfsg
Andreas Tille
tille at debian.org
Tue Dec 24 07:59:15 UTC 2013
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository sra-sdk.
commit f14de0e4743fb578987963e6d6d0f2d47213b3b2
Author: Andreas Tille <tille at debian.org>
Date: Mon Dec 23 22:36:39 2013 +0100
Imported Upstream version 2.3.4-2+dfsg
---
build/Makefile.env | 10 +-
build/Makefile.mac | 4 +
build/ld.linux.exe.sh | 7 +-
build/ld.mac.exe.sh | 7 +-
build/win-cc.sh | 33 +-
configuration-assistant.perl | 19 +-
interfaces/kfg/kart.h | 3 +
interfaces/kfg/keystore.h | 2 -
interfaces/kfg/kfg-priv.h | 35 +-
interfaces/kfs/kfs-priv.h | 8 +
interfaces/klib/refcount.h | 9 +-
interfaces/klib/report.h | 5 +
.../klib/{report.h => sra-release-version.h} | 93 +-
interfaces/kns/KCurlRequest.h | 4 +
interfaces/kns/ascp.h | 9 +-
.../reref.h => interfaces/kns/manager-ext.h | 37 +-
interfaces/kns/manager.h | 13 +
interfaces/kns/socket.h | 67 +-
interfaces/krypto/extern.h | 33 +-
interfaces/ktst/unit_test_suite.hpp | 11 +-
interfaces/ncbi/wgs-contig.h | 25 +-
interfaces/ncbi/wgs-contig.vschema | 71 +-
interfaces/os/mac/os-native.h | 2 +-
interfaces/vdb/vdb-priv.h | 5 +
interfaces/vfs/manager-priv.h | 5 -
interfaces/vfs/path.h | 11 +
libs/align/Makefile | 30 +-
libs/align/bam.c | 21 +-
libs/align/libalign-access.vers | 1 -
libs/align/libalign-reader.vers | 1 -
libs/align/libncbi-bam.vers | 1 -
libs/align/reference.c | 26 +-
libs/align/writer-reference.c | 14 +-
libs/align/writer-refseq.c | 5 +-
libs/ascp/ascp.c | 71 +-
libs/ascp/win/ascp-path.c | 4 +-
libs/ascp/win/connect.c | 23 +-
libs/axf/libaxf.vers | 1 -
libs/axf/libwaxf.vers | 1 -
libs/ext/Makefile | 55 -
libs/ext/bzip2/Makefile | 143 --
libs/ext/curl/Makefile | 103 -
libs/ext/zlib/Makefile | 150 --
libs/kapp/args.c | 2 +-
libs/kapp/libkapp.vers | 1 -
libs/kapp/libload.vers | 1 -
libs/kapp/loader-file.c | 10 +-
libs/kapp/progressbar.c | 2 +-
libs/kapp/tokenizer.c | 4 +-
libs/kapp/unix/sysmain.c | 6 +
libs/kapp/win/sysmain.c | 1 +
libs/kdb/btree.c | 42 +-
libs/kdb/column-cc.c | 2 +-
libs/kdb/database.c | 4 +-
libs/kdb/libkdb.vers | 1 -
libs/kdb/libwkdb.vers | 1 -
libs/kdb/trieidx-v2.c | 70 +-
libs/kfg/Makefile | 10 +-
libs/kfg/config-grammar.c | 109 +-
libs/kfg/config-lex.c | 128 +-
libs/kfg/config-tokens.h | 4 +-
libs/kfg/config.c | 416 ++--
libs/kfg/kart.c | 131 +-
libs/kfg/keystore.c | 14 +-
libs/kfg/libkfg.vers | 1 -
libs/kfg/report-kfg.c | 9 +-
libs/kfg/repository.c | 10 +-
libs/kfs/bzip.c | 6 +-
libs/kfs/cacheteefile.c | 2 +-
libs/kfs/gzip.c | 6 +-
libs/kfs/libkdl.vers | 1 -
libs/kfs/libkff.vers | 1 -
libs/kfs/libkfs.vers | 1 -
libs/kfs/quickmount.c | 6 +-
libs/kfs/tocentry.c | 2 +-
libs/kfs/win/lnk_tools.c | 4 +-
libs/kfs/win/sysdir-priv.h | 7 -
libs/kfs/win/sysdir.c | 6 +-
libs/kfs/win/sysdll.c | 2 +-
libs/kfs/win/sysfile.c | 15 +-
libs/klib/Makefile | 3 +-
libs/klib/libkascii.vers | 1 -
libs/klib/libklib.vers | 1 -
libs/klib/libktlog.vers | 1 -
libs/klib/libkutf8.vers | 1 -
libs/klib/log.c | 4 +-
libs/klib/maybeswap.h | 34 +-
libs/klib/pbstree-impl.c | 198 +-
libs/klib/printf.c | 27 +-
libs/klib/refcount.c | 29 +-
.../sra-pileup/reref.h => libs/klib/release-vers.h | 31 +-
libs/klib/report.c | 45 +-
libs/klib/sra-release-version.c | 328 +++
libs/klib/win/syserrcode.c | 2 +-
libs/klib/win/systime.c | 1 -
libs/klib/win/syswriter.c | 6 +-
libs/kns/KCurlFile.c | 4 +-
libs/kns/Makefile | 1 +
libs/kns/http-priv.h | 2 +-
libs/kns/http-test.c | 8 +-
libs/kns/http.c | 131 +-
libs/kns/kns_manager-ext.c | 140 ++
libs/kns/kns_manager.c | 29 +-
libs/kns/kns_mgr_priv.h | 10 +-
libs/kns/libkns.vers | 1 -
libs/kns/libkurl.vers | 1 -
libs/kns/unix/sysendpoint.c | 87 +-
libs/kns/unix/syssock.c | 160 +-
libs/kns/win/sysendpoint.c | 101 +-
libs/kns/win/syssock.c | 105 +-
libs/kproc/libkproc.vers | 1 -
libs/kproc/libkq.vers | 1 -
libs/kproc/libksproc.vers | 1 -
libs/kproc/unix/systimeout.c | 4 +-
libs/kproc/win/syscond.c | 4 +-
libs/krypto/Makefile | 29 +-
libs/krypto/cipher-byte.c | 2 +-
libs/krypto/cipher.c | 4 +-
libs/krypto/encfile-priv.h | 2 +-
libs/krypto/encfile.c | 4 +-
libs/krypto/encfilev2.c | 2 +-
libs/krypto/libkrypto.vers | 1 -
libs/krypto/libkryptotest.vers | 1 -
libs/krypto/reencfile.c | 2 +-
libs/krypto/wgaencrypt.c | 10 +-
libs/kxml/libkxfs.vers | 1 -
libs/kxml/libkxml.vers | 1 -
libs/kxml/xtoc.c | 4 +-
libs/loader/common-writer.c | 8 +-
libs/ncbi-vdb/Makefile | 4 +-
libs/ncbi-vdb/libncbi-vdb.vers | 2 +-
libs/ncbi-vdb/libncbi-wvdb.vers | 2 +-
libs/search/libksrch.vers | 1 -
libs/search/libnucstrstr.vers | 1 -
libs/search/search.c | 18 +-
libs/sra/libsra-schema.vers | 1 -
libs/sra/libsradb.vers | 1 -
libs/sra/libsrareader.vers | 1 -
libs/sra/libsraschema.vers | 1 -
libs/sra/libwsra-schema.vers | 1 -
libs/sra/libwsradb.vers | 1 -
libs/sra/libwsraschema.vers | 1 -
libs/sra/sracache.c | 2 +-
libs/sraxf/libsraxf.vers | 1 -
libs/sraxf/libwsraxf.vers | 1 -
libs/vdb/column-priv.h | 3 +-
libs/vdb/cursor-cmn.c | 63 +-
libs/vdb/cursor-priv.h | 9 +-
libs/vdb/cursor.c | 2 +-
libs/vdb/database-cmn.c | 12 +
libs/vdb/libvdb.vers | 1 -
libs/vdb/libwvdb.vers | 1 -
libs/vdb/page-map.h | 4 +-
libs/vdb/prod-cmn.c | 32 +-
libs/vdb/prod-expr.c | 6 +-
libs/vdb/prod-expr.h | 3 +
libs/vdb/prod-func.c | 235 +-
libs/vdb/prod.c | 2 +-
libs/vdb/schema-eval.c | 80 +-
libs/vdb/schema-expr.c | 14 +-
libs/vdb/schema-expr.h | 6 +-
libs/vdb/schema-func.c | 437 ++--
libs/vdb/schema-parse.h | 8 +-
libs/vdb/schema-priv.h | 42 +-
libs/vdb/schema-prod.c | 16 +-
libs/vdb/schema-type.c | 21 +-
libs/vdb/schema.c | 1 +
libs/vdb/wcolumn.c | 6 +-
libs/vdb/wcursor.c | 23 +-
libs/vdb/wprod.c | 8 +-
libs/vfs/Makefile | 3 +-
libs/vfs/keyring.c | 189 +-
libs/vfs/libsrapath.vers | 1 -
libs/vfs/libvfs.vers | 1 -
libs/vfs/manager.c | 91 +-
libs/vfs/path-priv.h | 4 +-
libs/vfs/path.c | 6 +-
libs/vfs/resolver.c | 512 +++-
libs/vfs/unix/syspath.c | 2 +-
libs/vfs/win/syspath.c | 5 +-
libs/vxf/libvxf.vers | 1 -
libs/vxf/libwvxf.vers | 1 -
libs/vxf/simple-sub-select.c | 170 +-
libs/wgsxf/libwgsxf.vers | 1 -
libs/wgsxf/libwwgsxf.vers | 1 -
tools/Makefile | 12 -
tools/align-info/align-info.vers | 2 +-
tools/bam-loader/bam-load.3.vers | 1 +
tools/bam-loader/bam-load.vers | 2 +-
tools/bam-loader/loader-imp.c | 27 +-
tools/cache-mgr/Makefile | 23 +-
tools/cache-mgr/cache-mgr.c | 8 +-
tools/cache-mgr/cache-mgr.vers | 2 +-
tools/cg-load/Makefile | 19 +-
tools/cg-load/cg-load.c | 132 +-
tools/cg-load/cg-load.vers | 2 +-
tools/cg-load/defs.h | 4 +
tools/cg-load/f1_5.c | 10 +-
tools/cg-load/f2_0.c | 9 +-
tools/cg-load/f2_2.c | 9 +-
tools/cg-load/factory-cmn.h | 2 +-
tools/cg-load/factory-evidence-dnbs.c | 2 +
tools/cg-load/factory-evidence-intervals.c | 2 +
tools/cg-load/factory-mappings.c | 2 +
tools/cg-load/factory-reads.c | 1 +
tools/cg-load/file-version-factory.c | 1 +
tools/cg-load/file.c | 16 +
tools/cg-load/file.h | 3 +
tools/fastq-loader/Makefile | 8 -
tools/fastq-loader/fastq-lex.c | 56 +-
tools/fastq-loader/latf-load.vers | 2 +-
tools/kar/Makefile | 28 +-
tools/kar/kar.c | 60 +-
tools/kar/kar.vers | 2 +-
tools/kar/sra-kar.vers | 2 +-
tools/nenctool/nenctest.vers | 2 +-
tools/nenctool/nenctool.vers | 2 +-
tools/nencvalid/Makefile | 10 -
tools/nencvalid/nencvalid.vers | 2 +-
tools/prefetch/prefetch.c | 2634 +++++++++++++-------
tools/prefetch/prefetch.vers | 2 +-
tools/sam-dump/sam-dump.c | 2 +-
tools/sra-dbcc/Makefile | 15 +-
tools/sra-dbcc/sra-dbcc.vers | 2 +-
tools/sra-dump/Makefile | 19 -
tools/sra-dump/abi-dump.vers | 2 +-
tools/sra-dump/core.c | 83 +-
tools/sra-dump/fastq-dump.vers | 2 +-
tools/sra-dump/illumina-dump.vers | 2 +-
tools/sra-dump/sff-dump.vers | 2 +-
tools/sra-dump/sra-dump.vers | 2 +-
tools/sra-load/Makefile | 8 -
tools/sra-load/abi-load.vers | 2 +-
tools/sra-load/fastq-load.vers | 2 +-
tools/sra-load/helicos-load.vers | 2 +-
tools/sra-load/illumina-load.vers | 2 +-
tools/sra-load/pacbio-loadxml.vers | 1 +
tools/sra-load/sff-load.vers | 2 +-
tools/sra-load/srf-load.vers | 2 +-
tools/sra-pileup/Makefile | 2 +
tools/sra-pileup/cmdline_cmn.c | 382 +--
tools/sra-pileup/cmdline_cmn.h | 27 +-
tools/sra-pileup/ref_regions.c | 491 ++++
.../manager.h => tools/sra-pileup/ref_regions.h | 65 +-
tools/sra-pileup/ref_walker.c | 997 ++++++++
tools/sra-pileup/ref_walker.h | 128 +
tools/sra-pileup/reref.c | 22 +-
tools/sra-pileup/reref.h | 5 -
tools/sra-pileup/sam-aligned.c | 244 +-
tools/sra-pileup/sam-dump-opts.c | 10 +-
tools/sra-pileup/sam-dump-opts.h | 6 +-
tools/sra-pileup/sam-dump.c | 2 +-
tools/sra-pileup/sam-dump.vers | 2 +-
tools/sra-pileup/sam-dump3.c | 99 +-
tools/sra-pileup/sam-hdr.c | 8 +-
tools/sra-pileup/sam-unaligned.c | 4 +-
tools/sra-pileup/sra-pileup.c | 315 ++-
tools/sra-pileup/sra-pileup.vers | 2 +-
tools/sra-sort/sra-sort.vers | 2 +-
tools/sra-stat/sra-stat.vers | 2 +-
tools/srapath/Makefile | 8 +-
tools/srapath/srapath.vers | 2 +-
tools/util/Makefile | 20 +-
tools/util/kdbmeta.c | 47 +-
tools/util/kdbmeta.vers | 2 +-
tools/util/md5cp.vers | 1 +
tools/util/rcexplain.vers | 2 +-
tools/util/rowwritetest.vers | 2 +-
tools/util/schema-replace.vers | 1 +
tools/{sra-pileup/reref.h => util/test-sra-priv.h} | 26 +-
tools/util/test-sra.c | 977 ++++++--
tools/util/test-sra.vers | 2 +-
tools/util/testld.vers | 2 +-
tools/util/txt2kdb.vers | 2 +-
.../krypto/extern.h => tools/util/unix/PrintOS.c | 64 +-
tools/util/vdb-lock.vers | 2 +-
tools/util/vdb-passwd.vers | 2 +-
tools/util/vdb-unlock.vers | 2 +-
tools/{sra-pileup/reref.h => util/win/PrintOS.c} | 46 +-
tools/vdb-config/Makefile | 10 -
tools/vdb-config/vdb-config.c | 27 +-
tools/vdb-config/vdb-config.vers | 2 +-
tools/vdb-copy/vdb-copy.vers | 2 +-
tools/vdb-decrypt/Makefile | 16 +-
tools/vdb-decrypt/vdb-decrypt.c | 2 +-
tools/vdb-decrypt/vdb-decrypt.vers | 2 +-
tools/vdb-decrypt/vdb-encrypt.vers | 2 +-
tools/vdb-dump/Makefile | 2 +
tools/vdb-dump/vdb-dump-context.c | 89 +-
tools/vdb-dump/vdb-dump-context.h | 25 +-
tools/vdb-dump/vdb-dump-fastq.c | 498 ++++
.../reref.h => vdb-dump/vdb-dump-fastq.h} | 21 +-
tools/vdb-dump/vdb-dump-redir.c | 120 +
.../reref.h => vdb-dump/vdb-dump-redir.h} | 32 +-
tools/vdb-dump/vdb-dump.c | 274 +-
tools/vdb-dump/vdb-dump.vers | 2 +-
tools/vdb-validate/vdb-validate.vers | 2 +-
297 files changed, 9988 insertions(+), 4270 deletions(-)
diff --git a/build/Makefile.env b/build/Makefile.env
index 0fe1472..ac28d96 100644
--- a/build/Makefile.env
+++ b/build/Makefile.env
@@ -89,8 +89,8 @@ include $(TOP)/build/Makefile.rules
# external schema modules
EXT_SCHEMA_MODULES = axf sraxf wgsxf vxf
-READONLY_SCHEMA_LIBS = $(addprefix -d,$(EXT_SCHEMA_MODULES))
-UPDATE_SCHEMA_LIBS = $(addprefix -dw,$(EXT_SCHEMA_MODULES))
+UPDATE_SCHEMA_LIBS := $(addprefix -lw,$(EXT_SCHEMA_MODULES))
+READONLY_SCHEMA_LIBS = $(addprefix -l,$(EXT_SCHEMA_MODULES))
ALWAYS_STATIC_SCHEMA_LIBS = $(addprefix -s,$(EXT_SCHEMA_MODULES))
# full directory paths
@@ -301,9 +301,9 @@ DLLX ?= $(SHLX)
ARCHDEFS = -D_ARCH_BITS=$(BITS) -DLIBPREFIX=$(LPFX) -DSHLIBEXT=$(DLLX)
# default tool parameters
-CFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(LINKAGE) $(DEFINES) $(ARCHDEFS) $(INCDIRS)
-CPFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(LINKAGE) $(DEFINES) $(ARCHDEFS) $(INCDIRS)
-LDFLAGS = $(DBG) $(PROF) $(CARCH)
+CFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(LINKAGE) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+CPFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(LINKAGE) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT)
#-------------------------------------------------------------------------------
# runtests
diff --git a/build/Makefile.mac b/build/Makefile.mac
index 74e5160..4568550 100644
--- a/build/Makefile.mac
+++ b/build/Makefile.mac
@@ -67,6 +67,10 @@ DEFINES := -DMAC -DBSD -DUNIX -D_REENTRANT -D_FILE_OFFSET_BITS=64
OS_DAD = bsd
OS_GDAD = unix
+# minimum deployment os
+MIN_DEPLOY_OS = 10.6
+MIN_DEPLOY_OS_OPT = -mmacosx-version-min=$(MIN_DEPLOY_OS)
+
LIBXML_LPATH = /usr/lib
# flex+bison: on Mac, use source-controlled generated .c/.h files
diff --git a/build/ld.linux.exe.sh b/build/ld.linux.exe.sh
index afa1de6..ff5b83f 100755
--- a/build/ld.linux.exe.sh
+++ b/build/ld.linux.exe.sh
@@ -291,7 +291,12 @@ fi
# add in xml
if [ $HAVE_XML -ne 0 ]
then
- CMD="$CMD -lxml2"
+ if [ "$NCBI" != "" ] && [ -f "$NCBI/libxml/lib/libxml2-static.a" ]
+ then
+ CMD="$CMD -L$NCBI/libxml/lib -lxml2-static"
+ else
+ CMD="$CMD -lxml2"
+ fi
fi
# add in math library
diff --git a/build/ld.mac.exe.sh b/build/ld.mac.exe.sh
index 1b70a5f..b693034 100755
--- a/build/ld.mac.exe.sh
+++ b/build/ld.mac.exe.sh
@@ -319,7 +319,12 @@ fi
# add in xml
if [ $HAVE_XML -ne 0 ]
then
- CMD="$CMD -lxml2"
+# if [ "$NCBI" != "" ] && [ -f "$NCBI/libxml/lib/libxml2-static.a" ]
+# then
+# CMD="$CMD -L$NCBI/libxml/lib -lxml2-static"
+# else
+ CMD="$CMD -lxml2"
+# fi
fi
# add in math library
diff --git a/build/win-cc.sh b/build/win-cc.sh
index 42f2d74..9aee096 100755
--- a/build/win-cc.sh
+++ b/build/win-cc.sh
@@ -163,7 +163,11 @@ do
DEPENDENCIES=1
;;
- -fPIC|-std=c99|-ansi|-pedantic)
+ -fPIC|-std=c99|-ansi)
+ ;;
+
+ -pedantic)
+ ARGS="$ARGS /W3"
;;
-*)
@@ -228,29 +232,12 @@ then
for inc in $(cat $TARG.inc)
do
inc_low=$(echo ${inc} | tr '[A-Z]' '[a-z]' | tr '\\' '/')
- # vers.h files are now generated in the objdir
- if [ "$inc" != "${inc%.vers.h}" ]
+ if [ "${inc_low#$rhome_low}" != "$inc_low" ]
then
- # convert the "...vers.h" in the object directory into "...vers" in the source directory
- inc="${inc%.h}"
- if [ "${inc_low#$rhome_low}" != "$inc_low" ]
- then
- inc="${inc:$rhome_len}"
- elif [ "${inc_low#$routdir_low}" != "$inc_low" ]
- then
- inc="$SRCDIR/$inc"
- fi
- inc="${inc//\\//}"
- inc="$(basename $inc)"
- inc="$SRCDIR/$inc"
- else
- if [ "${inc_low#$rhome_low}" != "$inc_low" ]
- then
- inc="$LHOME${inc:$rhome_len}"
- elif [ "${inc_low#$routdir_low}" != "$inc_low" ]
- then
- inc="$OUTDIR${inc:$routdir_len}"
- fi
+ inc="$LHOME${inc:$rhome_len}"
+ elif [ "${inc_low#$routdir_low}" != "$inc_low" ]
+ then
+ inc="$OUTDIR${inc:$routdir_len}"
fi
echo -n " $inc" | tr '\\' '/' >> $TARG.d
done
diff --git a/configuration-assistant.perl b/configuration-assistant.perl
index 1ddd55b..599f291 100755
--- a/configuration-assistant.perl
+++ b/configuration-assistant.perl
@@ -4,7 +4,7 @@
# N.B. Run "perl configuration-assistant.perl" if you see a message like:
# configuration-assistant.perl: /usr/bin/perl: bad interpreter: No such file or directory
################################################################################
-my $VERSION = '2.3.3';
+my $VERSION = '2.3.4-2';
################################################################################
use strict;
@@ -42,6 +42,7 @@ Version() if ($options{version});
$options{references} = 1 if ($#ARGV >= 0);
+printOS();
println "cwd = '" . getcwd() . "'\n";
my $DECRYPTION_PKG;
@@ -236,6 +237,22 @@ sub Posixify {
return $_;
}
+sub printOS {
+ my $fail;
+ if ($^O eq 'MSWin32') {
+ $fail = system('ver');
+ }
+ else {
+ $fail = system('uname -a');
+ if ($fail) {
+ $fail = system('ver');
+ }
+ }
+ if ($fail) {
+ println "Operating system: '$^O'";
+ }
+}
+
sub Version {
my $bin = basename($0);
print << "END";
diff --git a/interfaces/kfg/kart.h b/interfaces/kfg/kart.h
index b670f6b..fa54fd4 100644
--- a/interfaces/kfg/kart.h
+++ b/interfaces/kfg/kart.h
@@ -77,9 +77,12 @@ KFG_EXTERN rc_t CC KartMakeText(const struct KDirectory *dir, const char *path,
#endif
KFG_EXTERN rc_t CC KartPrint(const Kart *self);
+KFG_EXTERN rc_t CC KartPrintNumbered(const Kart *self);
KFG_EXTERN rc_t CC KartMakeNextItem(Kart *self, const KartItem **item);
+KFG_EXTERN rc_t CC KartItemsProcessed(const Kart *self, uint16_t *number);
+
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/kfg/keystore.h b/interfaces/kfg/keystore.h
index 56d1536..92fd7b7 100644
--- a/interfaces/kfg/keystore.h
+++ b/interfaces/kfg/keystore.h
@@ -39,8 +39,6 @@
extern "C" {
#endif
-#define ENV_KRYPTO_PWFILE "VDB_PWFILE"
-
struct KFile;
struct KConfig;
struct String;
diff --git a/interfaces/kfg/kfg-priv.h b/interfaces/kfg/kfg-priv.h
index 59a4dcd..8a7320d 100644
--- a/interfaces/kfg/kfg-priv.h
+++ b/interfaces/kfg/kfg-priv.h
@@ -27,6 +27,10 @@
#ifndef _h_kfg_priv_
#define _h_kfg_priv_
+#ifndef _h_kfg_extern_
+#include <kfg/extern.h>
+#endif
+
#ifndef _h_kfg_config_
#include <kfg/config.h>
#endif
@@ -35,6 +39,25 @@
extern "C" {
#endif
+/*--------------------------------------------------------------------------
+ * forwards
+ */
+struct KDirectory;
+struct KNamelist;
+
+
+/*--------------------------------------------------------------------------
+ * Environment and configuration keys
+ */
+#define ENV_KRYPTO_PWFILE "VDB_PWFILE"
+#define KFG_KRYPTO_PWFILE "krypto/pwfile"
+#define KFG_KRYPTO_PWFD "krypto/pwfd"
+
+/*--------------------------------------------------------------------------
+ * KConfig
+ * configuration paramter manager
+ */
+
/* ListIncluded
* list all included files
*/
@@ -48,9 +71,17 @@ KFG_EXTERN rc_t CC KConfigListIncluded ( const KConfig *self,
KFG_EXTERN rc_t CC KConfigGetLoadPath ( const KConfig *self,
const char **path );
-/* call KConfigMake not initializing the static global KConfig pointer */
+/* MakeLocal
+ * make a KConfig object that avoids singleton
+ */
KFG_EXTERN rc_t CC KConfigMakeLocal ( KConfig **cfg,
- const struct KDirectory * cfgdir );
+ struct KDirectory const * cfgdir );
+
+/* PrintDebug
+ * print configuration including some internal nodes information
+ */
+KFG_EXTERN rc_t CC KConfigPrintDebug ( const KConfig *self,
+ const char *root_node_name );
#ifdef __cplusplus
}
diff --git a/interfaces/kfs/kfs-priv.h b/interfaces/kfs/kfs-priv.h
index 9a8bb4c..e7dc3de 100644
--- a/interfaces/kfs/kfs-priv.h
+++ b/interfaces/kfs/kfs-priv.h
@@ -44,6 +44,7 @@ extern "C" {
* struct/class forwards
*/
struct KDirectory;
+struct KSysDir;
struct KFile;
struct KPath;
struct KDlset;
@@ -87,6 +88,13 @@ KFS_EXTERN rc_t KDirectoryPosixStringToSystemString (
#endif
+/* OSPath
+ * returns a real OS path
+ */
+rc_t KSysDirOSPath ( struct KSysDir const *self,
+ wchar_t *real, size_t bsize, const char *path, va_list args );
+
+
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/klib/refcount.h b/interfaces/klib/refcount.h
index bc7cfd5..9db3bd9 100644
--- a/interfaces/klib/refcount.h
+++ b/interfaces/klib/refcount.h
@@ -35,9 +35,16 @@
#include <atomic32.h>
#endif
+#ifndef FORCE_TRACK_REFERENCES
+#define FORCE_TRACK_REFERENCES 0
+#endif
+
/* normally turned off */
-#ifndef TRACK_REFERENCES
+#if FORCE_TRACK_REFERENCES
+#undef TRACK_REFERENCES
#define TRACK_REFERENCES 1
+#elif ! defined TRACK_REFERENCES
+#define TRACK_REFERENCES 0
#endif
/* normally turned on */
diff --git a/interfaces/klib/report.h b/interfaces/klib/report.h
index 2c60e2f..4f00232 100644
--- a/interfaces/klib/report.h
+++ b/interfaces/klib/report.h
@@ -81,6 +81,11 @@ KLIB_EXTERN void CC ReportSilence ( void );
*/
KLIB_EXTERN rc_t CC ReportFinalize ( rc_t rc );
+/* ForceFinalize
+ * Finalize forcing report generation to stdout
+ */
+KLIB_EXTERN rc_t CC ReportForceFinalize ( void );
+
/* ResetObject
* Resets the name of the current object beeing processes(path or accession),
diff --git a/interfaces/klib/report.h b/interfaces/klib/sra-release-version.h
similarity index 50%
copy from interfaces/klib/report.h
copy to interfaces/klib/sra-release-version.h
index 2c60e2f..11a2bfa 100644
--- a/interfaces/klib/report.h
+++ b/interfaces/klib/sra-release-version.h
@@ -24,8 +24,9 @@
*
*/
-#ifndef _h_klib_report_
-#define _h_klib_report_
+#ifndef _h_klib_sra_release_version_
+#define _h_klib_sra_release_version_
+
#ifndef _h_klib_extern_
#include <klib/extern.h>
@@ -35,64 +36,64 @@
#include <klib/defs.h>
#endif
+
#ifdef __cplusplus
extern "C" {
#endif
-/*--------------------------------------------------------------------------
- * An unrecoverable error happened.
- * We can help to solve it
- * by reporting information about known application execution environment.
- */
-
-
-/* Init
- * initialize with common information
+/* major . minor . release [ - [ type - ] revision ]
*
- * "argc" [ IN ] and "argv" [ IN ] - intact command line
- *
- * "tool_version" [ IN ] - version of tool
+ * Examples:
+ * "2.3.4-a0"
+ * "2.3.4-b3"
+ * "2.3.4-rc1"
+ * "2.3.4"
+ * "2.3.4-2"
*/
-KLIB_EXTERN void CC ReportInit ( int argc, char *argv [],
- ver_t tool_version );
-
-
-/* BuildDate
- * set the build date of the tool
- *
- * "date" [ IN ] - pre-processor __DATE__
+typedef struct SraReleaseVersion SraReleaseVersion;
+struct SraReleaseVersion
+{
+ ver_t version; /* major . minor . release */
+ uint32_t revision;
+ enum {
+ eSraReleaseVersionTypeAlpha,
+ eSraReleaseVersionTypeBeta,
+ eSraReleaseVersionTypeRC, /* release candidate */
+ eSraReleaseVersionTypeFinal
+ } type;
+};
+
+
+/* Get
+ * Get release version of this build of SRA Toolkit */
+KLIB_EXTERN rc_t CC SraReleaseVersionGet ( SraReleaseVersion *version );
+
+/* Cmp
+ * Compare two release versions
+ * result return values:
+ * 0 - the versions are the same
+ * 1 - version2 is more recent than self
+ * -1 - self is more recent than version2
*/
-KLIB_EXTERN void CC ReportBuildDate ( const char *date );
+KLIB_EXTERN rc_t CC SraReleaseVersionCmp ( const SraReleaseVersion *self,
+ const SraReleaseVersion *version2, int32_t *result );
-
-/* Silence
- * tell report to be silent at exit
- * useful especially in response to ^C
- */
-KLIB_EXTERN void CC ReportSilence ( void );
-
-
-/* Finalize
- * perform cleanup
- * optionally report error condition
- *
- * "rc" [ IN ] - report environment information if not zero
+/* Parse
+ * Initialize SraReleaseVersion from char version[size]
*/
-KLIB_EXTERN rc_t CC ReportFinalize ( rc_t rc );
-
+KLIB_EXTERN rc_t CC SraReleaseVersionInit ( SraReleaseVersion *self,
+ const char *version, size_t size );
-/* ResetObject
- * Resets the name of the current object beeing processes(path or accession),
- * usually from command line.
- * Should be called each time
- * when starting next command line argument processing.
+/* Print
+ * Convert SraReleaseVersion to version[size]
*/
-KLIB_EXTERN rc_t CC ReportResetObject ( const char *path );
-
+KLIB_EXTERN rc_t CC SraReleaseVersionPrint ( const SraReleaseVersion *self,
+ char *version, size_t size, size_t *num_writ );
#ifdef __cplusplus
}
#endif
-#endif /* _h_klib_report_ */
+
+#endif /* _h_klib_sra_release_version_ */
diff --git a/interfaces/kns/KCurlRequest.h b/interfaces/kns/KCurlRequest.h
index 3956da3..abe0149 100644
--- a/interfaces/kns/KCurlRequest.h
+++ b/interfaces/kns/KCurlRequest.h
@@ -30,6 +30,10 @@
#include <kns/extern.h>
#endif
+#ifndef _h_klib_defs_
+#include <klib/defs.h>
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
diff --git a/interfaces/kns/ascp.h b/interfaces/kns/ascp.h
index b35111c..cf1bf88 100644
--- a/interfaces/kns/ascp.h
+++ b/interfaces/kns/ascp.h
@@ -70,9 +70,12 @@ typedef struct {
TProgress *callback;
TQuitting *quitting;
+
+ bool disabled; /* output parameter for aspera_options */
} AscpOptions;
-/** ascp_bin and private_file should be freed by the caller */
+/** status - whether to print STSMSG(1-2) - information messages
+ ascp_bin and private_file should be freed by the caller */
KNS_EXTERN rc_t CC ascp_locate(const char **ascp_bin, const char **private_file,
bool use_config, bool status);
@@ -80,6 +83,10 @@ KNS_EXTERN rc_t CC ascp_locate(const char **ascp_bin, const char **private_file,
KNS_EXTERN rc_t CC aspera_get(const char *ascp_bin, const char *private_file,
const char *src, const char *dest, AscpOptions *opt);
+/** Fill AscpOptions members initialized by ascp library */
+KNS_EXTERN rc_t CC aspera_options(AscpOptions *opt);
+
+
#ifdef __cplusplus
}
#endif
diff --git a/tools/sra-pileup/reref.h b/interfaces/kns/manager-ext.h
similarity index 68%
copy from tools/sra-pileup/reref.h
copy to interfaces/kns/manager-ext.h
index 13e3136..fc9e667 100644
--- a/tools/sra-pileup/reref.h
+++ b/interfaces/kns/manager-ext.h
@@ -23,26 +23,37 @@
* ===========================================================================
*
*/
-#ifndef _h_reref_
-#define _h_reref_
+#ifndef _h_kns_manager_ext_
+#define _h_kns_manager_ext_
+
+#ifndef _h_kns_manager_
+#include <kns/manager.h>
+#endif
#ifdef __cplusplus
extern "C" {
#endif
-#if 0
-}
-#endif
-#include <kapp/args.h>
-#include <klib/out.h>
-#include <klib/rc.h>
-#include <klib/log.h>
+/*--------------------------------------------------------------------------
+ * forwards
+ */
+struct SraReleaseVersion;
+
-#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
+/*--------------------------------------------------------------------------
+ * KNSManager
+ */
-rc_t report_on_reference( Args * args, bool extended );
+/* NewReleaseVersion
+ * Get release version of the new (latest available) public SRA Toolkit
+ */
+KNS_EXTERN rc_t CC KNSManagerNewReleaseVersion ( const KNSManager *self,
+ struct SraReleaseVersion *newVersion );
+
+#ifdef __cplusplus
+}
#endif
+
+#endif /* _h_kns_manager_ext_ */
diff --git a/interfaces/kns/manager.h b/interfaces/kns/manager.h
index 1c78d49..f07b6d7 100644
--- a/interfaces/kns/manager.h
+++ b/interfaces/kns/manager.h
@@ -78,6 +78,19 @@ KNS_EXTERN rc_t CC KNSManagerAvail ( const KNSManager *self );
KNS_EXTERN rc_t CC KNSManagerCurlVersion ( const KNSManager *self, const char ** version_string );
+/* SetVerbose
+ * set/clear verbosity flag of manager ( dflt is false )...
+ * the network-code has to request it
+ */
+KNS_EXTERN void CC KNSManagerSetVerbose ( struct KNSManager *self, bool verbosity );
+
+
+/* IsVerbose
+ * request the verbosity flag of manager ( dflt is false )...
+ */
+KNS_EXTERN bool CC KNSManagerIsVerbose ( const KNSManager *self );
+
+
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/kns/socket.h b/interfaces/kns/socket.h
index 16d2e5c..e83c752 100644
--- a/interfaces/kns/socket.h
+++ b/interfaces/kns/socket.h
@@ -49,46 +49,59 @@ struct KNSManager;
/*--------------------------------------------------------------------------
* KSocket
*/
-struct KSocket;
+typedef struct KSocket KSocket;
-/* KSocketAddRef
- *
- */
-KNS_EXTERN rc_t CC KSocketAddRef( struct KSocket *self );
-/* KSocketRelease
- *
- */
-KNS_EXTERN rc_t CC KSocketRelease( struct KSocket *self );
/* MakeConnection
- * create a connection-oriented stream connected to an Internet server
- * conn [ OUT ] - a stream for communication with the server
- * from [ IN ] - client endpoint
- * to [ IN ] - server endpoint
+ * create a connection-oriented stream
+ *
+ * "conn" [ OUT ] - a stream for communication with the server
+ *
+ * "from" [ IN ] - client endpoint
*
- * both endpoints have to be of type epIP; creates a TCP connection
+ * "to" [ IN ] - server endpoint
+ *
+ * both endpoints have to be of type epIP; creates a TCP connection
*/
-KNS_EXTERN rc_t CC KNSMakeConnection ( struct KStream **conn, struct KEndPoint const *from, struct KEndPoint const *to );
+KNS_EXTERN rc_t CC KNSManagerMakeConnection ( struct KNSManager const * self,
+ struct KStream **conn, struct KEndPoint const *from, struct KEndPoint const *to );
/* MakeIPCConnection
* create a connection-oriented stream connected to an IPC server
- * conn [ OUT ] - a stream for communication with the server
- * to [ IN ] - server endpoint (must be of type epIPC)
- * max_retries [ IN ] - number of retries to be made if connection is refused. Will sleep for 1 second between retries
+ *
+ * "conn" [ OUT ] - a stream for communication with the server
+ *
+ * "to" [ IN ] - server endpoint (must be of type epIPC)
+ *
+ * "max_retries" [ IN ] - number of retries to be made if connection is refused.
+ * Will sleep for 1 second between retries
*/
-KNS_EXTERN rc_t CC KNSMakeIPCConnection ( struct KStream **conn, struct KEndPoint const *to, uint8_t max_retries );
+KNS_EXTERN rc_t CC KNSManagerMakeIPCConnection ( struct KNSManager const *self,
+ struct KStream **conn, struct KEndPoint const *to, uint32_t max_retries );
-/* Create a listener socket for accepting incoming IPC connections
- * ep [ IN ] - a endpoint of type epIPC
- * listener [ IN ] - a listener socket, to be called KNSListen() on
+/* MakeListener
+ * create a listener socket for accepting incoming IPC connections
+ *
+ * "ep" [ IN ] - a endpoint of type epIPC
+ *
+ * "listener" [ IN ] - a listener socket, to be called KSocketListen() on
*/
-KNS_EXTERN rc_t CC KNSMakeListener( struct KSocket** listener, struct KEndPoint const * ep );
+KNS_EXTERN rc_t CC KNSManagerMakeListener ( struct KNSManager const *self,
+ KSocket **listener, struct KEndPoint const * ep );
-/* Wait for an incoming connection
- * listener [ IN ] - a listener socket
- * conn [ OUT ] - a stream for communication with the client
+/* AddRef
+ * Release
+ */
+KNS_EXTERN rc_t CC KSocketAddRef ( KSocket *self );
+KNS_EXTERN rc_t CC KSocketRelease ( KSocket *self );
+
+
+/* Listen
+ * wait for an incoming connection
+ *
+ * "conn" [ OUT ] - a stream for communication with the client
*/
-KNS_EXTERN rc_t CC KNSListen ( struct KSocket *listener, struct KStream **conn );
+KNS_EXTERN rc_t CC KSocketListen ( KSocket *self, struct KStream **conn );
#ifdef __cplusplus
diff --git a/interfaces/krypto/extern.h b/interfaces/krypto/extern.h
index 4663585..a567690 100644
--- a/interfaces/krypto/extern.h
+++ b/interfaces/krypto/extern.h
@@ -27,40 +27,17 @@
#ifndef _h_krypto_extern_
#define _h_krypto_extern_
-#ifndef _h_klib_callconv_
-#include <klib/callconv.h>
-#endif
-
-#if ! defined EXPORT_LATCH && _LIBRARY
-
+#if ! defined EXPORT_LATCH && defined _LIBRARY
#define KRYPTO_EXTERN LIB_EXPORT
-#define KRYPTO_EXTERN_DATA extern LIB_EXPORT
+#define KRYPTO_EXTERN_DATA LIB_EXPORT_DATA
#define EXPORT_LATCH 1
-
#else
-
#define KRYPTO_EXTERN LIB_IMPORT
-#define KRYPTO_EXTERN_DATA LIB_IMPORT
-
+#define KRYPTO_EXTERN_DATA LIB_IMPORT_DATA
#endif
-#if defined _MSC_VER && ! _STATIC
-
-/* __declspec ( dllimport ) will cause creation of
- function pointers rather than thunks, which makes
- the code that imports unable to link statically
- against a library. we leave this symbol defined as
- "extern" to use thunks instead. as a result, all
- function addresses resolve to the thunk and not
- the actual function. */
-#define LIB_IMPORT extern
-#define LIB_EXPORT __declspec ( dllexport )
-
-#else
-
-#define LIB_IMPORT extern
-#define LIB_EXPORT
-
+#ifndef _h_klib_extern_
+#include <klib/extern.h>
#endif
#endif /* _h_krypto_extern_ */
diff --git a/interfaces/ktst/unit_test_suite.hpp b/interfaces/ktst/unit_test_suite.hpp
index 0e12d68..8b3a3d7 100644
--- a/interfaces/ktst/unit_test_suite.hpp
+++ b/interfaces/ktst/unit_test_suite.hpp
@@ -43,7 +43,11 @@
// these macros are available outside of test cases' code
#define LOG(log_level, msg) \
- if (log_level >= ncbi::NK::TestEnv::verbosity) std::cerr << msg
+ (log_level >= ncbi::NK::TestEnv::verbosity ? (std::cerr << msg) : std::cerr)
+
+/*#define TESTMESSAGE(M) \
+{ ncbi::NK::saveLocation(__FILE__,__LINE__); \
+ LOG(ncbi::NK::LogLevel::e_message, M); }*/
#define TEST_MESSAGE(M) \
{ ncbi::NK::saveLocation(__FILE__,__LINE__); \
@@ -240,6 +244,9 @@ protected:
? report_passed((#exp), __FILE__, __LINE__) \
: report_error ((#exp), __FILE__, __LINE__) )
+#define REPORT_ERROR(exp) \
+ report_error (exp, __FILE__, __LINE__)
+
/* TODO #define CHECK_MESSAGE(exp, M) \
std::ostringstream s, s << M, \
( (exp) \
@@ -448,7 +455,7 @@ ncbi::NK::counter_t Main(int argc, char* argv[],
"Entering test suite \"" << suite_name << "\"\n");
ec = t->Run(&globalFixtute);
LOG(ncbi::NK::LogLevel::e_test_suite,
- "Leaving test suite \"" << suite_name << "\"\n";)
+ "Leaving test suite \"" << suite_name << "\"\n");
}
catch (std::exception& ex)
{
diff --git a/interfaces/ncbi/wgs-contig.h b/interfaces/ncbi/wgs-contig.h
index 7b8ab2d..cc547be 100644
--- a/interfaces/ncbi/wgs-contig.h
+++ b/interfaces/ncbi/wgs-contig.h
@@ -105,6 +105,11 @@ enum
NCBI_WGS_gap_unknown = 2 * -1
};
+/* gap_linkage
+ * Describes linkage and linkage evidence
+ */
+typedef int32_t NCBI_WGS_gap_linkage;
+
/* gap type
* scaffold a gap between two sequence contigs in a scaffold
* contig an unspanned gap between two sequence contigs
@@ -140,15 +145,17 @@ enum
*/
enum
{
- NCBI_WGS_gap_linkage_paired_ends = 1 * -64,
- NCBI_WGS_gap_linkage_align_genus = 2 * -64,
- NCBI_WGS_gap_linkage_align_xgenus = 3 * -64,
- NCBI_WGS_gap_linkage_align_trnscpt = 4 * -64,
- NCBI_WGS_gap_linkage_within_clone = 5 * -64,
- NCBI_WGS_gap_linkage_clone_contig = 6 * -64,
- NCBI_WGS_gap_linkage_map = 7 * -64,
- NCBI_WGS_gap_linkage_strobe = 8 * -64,
- NCBI_WGS_gap_linkage_unspecified = 9 * -64
+ NCBI_WGS_gap_linkage_linked = 1,
+ NCBI_WGS_gap_linkage_evidence_paired_ends = 2,
+ NCBI_WGS_gap_linkage_evidence_align_genus = 4,
+ NCBI_WGS_gap_linkage_evidence_align_xgenus = 8,
+ NCBI_WGS_gap_linkage_evidence_align_trnscpt = 16,
+ NCBI_WGS_gap_linkage_evidence_within_clone = 32,
+ NCBI_WGS_gap_linkage_evidence_clone_contig = 64,
+ NCBI_WGS_gap_linkage_evidence_map = 128,
+ NCBI_WGS_gap_linkage_evidence_strobe = 256,
+ NCBI_WGS_gap_linkage_evidence_unspecified = 512,
+ NCBI_WGS_gap_linkage_evidence_pcr = 1024
};
diff --git a/interfaces/ncbi/wgs-contig.vschema b/interfaces/ncbi/wgs-contig.vschema
index 2fa1210..2793aaa 100644
--- a/interfaces/ncbi/wgs-contig.vschema
+++ b/interfaces/ncbi/wgs-contig.vschema
@@ -42,9 +42,9 @@ include 'ncbi/stats.vschema';
*/
/* component_props
- * a signed value describing either contigs or gaps
- * where values >= 0 refer to contigs
- * and negatives describe gaps
+ * a signed value describing contig or gap components of scaffolds, or
+ * gaps in contig sequences.
+ * Positive values refer to contigs and negatives describe gaps
*/
typedef I16 NCBI:WGS:component_props;
@@ -108,7 +108,13 @@ const NCBI:WGS:component_props NCBI:WGS:gap:heterochromatin = -20;
const NCBI:WGS:component_props NCBI:WGS:gap:telomere = -24;
const NCBI:WGS:component_props NCBI:WGS:gap:repeat = -28;
-/* gap linkage evidence
+/* gap_linkage
+ */
+typedef I32 NCBI:WGS:gap_linkage;
+
+/* gap linkage and linkage evidence
+ * There can be multiple linkage evidences or linkage with no evidence
+ *
* paired-ends paired sequences from the two ends of a DNA fragment
* align_genus alignment to a reference genome within the same genus
* align_xgenus alignment to a reference genome within another genus
@@ -120,17 +126,19 @@ const NCBI:WGS:component_props NCBI:WGS:gap:repeat = -28;
* such as RH, linkage, fingerprint or optical
* strobe strobe sequencing (PacBio)
* unspecified
+ * pcr PCR
*/
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:paired_ends = -64;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:align_genus = -128;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:align_xgenus = -192;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:align_trnscpt = -256;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:within_clone = -320;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:clone_contig = -384;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:map = -448;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:strobe = -512;
-const NCBI:WGS:component_props NCBI:WGS:gap:linkage:unspecified = -576;
-
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage:linked = 1;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:paired_ends = 2;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:align_genus = 4;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:align_xgenus = 8;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:align_trnscpt = 16;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:within_clone = 32;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:clone_contig = 64;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:map = 128;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:strobe = 256;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:unspecified = 512;
+const NCBI:WGS:gap_linkage NCBI:WGS:gap:linkage_evidence:pcr = 1024;
/*--------------------------------------------------------------------------
* functions
@@ -309,6 +317,27 @@ table NCBI:WGS:tbl:nucleotide #1.1
*/
extern column < NCBI:asn:binary > zip_encoding ANNOT;
+ /* GAP_START
+ * Starting position of a gap
+ */
+ extern column < INSDC:coord:zero > izip_encoding GAP_START;
+
+ /* GAP_LEN
+ * Length of a gap
+ */
+ extern column < INSDC:coord:len > izip_encoding GAP_LEN;
+
+ /* GAP_PROPS
+ * See description of type
+ */
+ extern column < NCBI:WGS:component_props > zip_encoding GAP_PROPS;
+
+ /* GAP_LINKAGE
+ * See description of type
+ */
+ extern column < NCBI:WGS:gap_linkage> zip_encoding GAP_LINKAGE;
+
+
ascii out_seqid_gi = sprintf < "gi|%u" > ( .GI );
ascii out_seqid_gb = sprintf < "gb|%s.%u|" > ( out_accession, ACC_VERSION )
| sprintf < "gb|%s.1|" > ( out_accession );
@@ -487,9 +516,15 @@ table NCBI:WGS:tbl:scaffold #1
extern column < NCBI:WGS:component_props > zip_encoding COMPONENT_PROPS;
/* COMPONENT_ID
- * one row-id for each non-gap component in COMPONENT_TYPE
+ * one row-id for each non-gap component
*/
extern column < I64 > izip_encoding COMPONENT_ID;
+
+ /* COMPONENT_LINKAGE
+ * see description of type
+ * one row-id for each gap component
+ */
+ extern column < NCBI:WGS:gap_linkage > zip_encoding COMPONENT_LINKAGE;
}
table NCBI:WGS:view:scaffold #1 = NCBI:WGS:tbl:scaffold #1
@@ -506,7 +541,7 @@ table NCBI:WGS:view:scaffold #1 = NCBI:WGS:tbl:scaffold #1
ascii acc_prefix
= < ascii > simple_sub_select < 'SEQUENCE', 'ACC_PREFIX' > ( acc_row_id );
U32 acc_contig_len
- = < ascii > simple_sub_select < 'SEQUENCE', 'ACC_CONTIG_LEN' > ( acc_row_id );
+ = < U32 > simple_sub_select < 'SEQUENCE', 'ACC_CONTIG_LEN' > ( acc_row_id );
ascii out_accession
= sprintf < "%sS%0*d" > ( acc_prefix, acc_contig_len, scaffold_row_id );
@@ -638,7 +673,7 @@ table NCBI:WGS:view:scaffold #1 = NCBI:WGS:tbl:scaffold #1
readonly column INSDC:coord:zero READ_START
= out_read_start;
INSDC:coord:zero out_read_start
- = ( INSDC:coord:zero ) < U32 > integral_0 ( .COMPONENT_LEN );
+ = ( INSDC:coord:zero ) < U32 > integral ( .COMPONENT_LEN );
readonly column INSDC:coord:len READ_LEN
= .COMPONENT_LEN;
readonly column INSDC:SRA:xread_type READ_TYPE
@@ -656,5 +691,5 @@ database NCBI:WGS:db:contig #1.1
table NCBI:WGS:tbl:nucleotide SEQUENCE;
table NCBI:WGS:tbl:protein PROTEIN;
table NCBI:WGS:tbl:gi_idx GI_IDX;
- table NCBI:WGS:tbl:scaffold SCAFFOLD;
+ table NCBI:WGS:view:scaffold SCAFFOLD;
};
diff --git a/interfaces/os/mac/os-native.h b/interfaces/os/mac/os-native.h
index 079c12b..3e26ff5 100644
--- a/interfaces/os/mac/os-native.h
+++ b/interfaces/os/mac/os-native.h
@@ -45,7 +45,7 @@ extern "C" {
*/
char *strdup ( const char *str );
-#if !defined(__MAC_10_7) || (__MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_7)
+#if !defined(__MAC_10_6) || (__MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_6)
static __inline__
char *strndup ( const char *str, size_t n )
{
diff --git a/interfaces/vdb/vdb-priv.h b/interfaces/vdb/vdb-priv.h
index b5f479d..6ab1d62 100644
--- a/interfaces/vdb/vdb-priv.h
+++ b/interfaces/vdb/vdb-priv.h
@@ -129,6 +129,11 @@ VDB_EXTERN rc_t CC VDatabaseOpenKDatabaseRead ( struct VDatabase const *self, st
VDB_EXTERN rc_t CC VDatabaseOpenKDatabaseUpdate ( struct VDatabase *self, struct KDatabase **kdb );
+/* IsCSRA
+ * ask an open database if it conforms to cSRA schema
+ */
+VDB_EXTERN bool CC VDatabaseIsCSRA ( struct VDatabase const *self );
+
/*--------------------------------------------------------------------------
* VTable
diff --git a/interfaces/vfs/manager-priv.h b/interfaces/vfs/manager-priv.h
index 00116ed..9046572 100644
--- a/interfaces/vfs/manager-priv.h
+++ b/interfaces/vfs/manager-priv.h
@@ -43,11 +43,6 @@
extern "C" {
#endif
-#define ENV_KRYPTO_PWFILE "VDB_PWFILE"
-#define KFG_KRYPTO_PWFILE "krypto/pwfile"
-#define KFG_KRYPTO_PWFD "krypto/pwfd"
-
-
struct VFSManager;
struct KDirectory;
struct KFile;
diff --git a/interfaces/vfs/path.h b/interfaces/vfs/path.h
index d9fa953..70d81cf 100644
--- a/interfaces/vfs/path.h
+++ b/interfaces/vfs/path.h
@@ -235,6 +235,17 @@ VFS_EXTERN rc_t CC VPathGetFragment ( const VPath * self, struct String * str );
/* TEMPORARY */
VFS_EXTERN uint32_t CC VPathGetOid ( const VPath * self );
+/* legacy support */
+#define VPathMake LegacyVPathMake
+VFS_EXTERN rc_t VPathMake ( VPath ** new_path, const char * posix_path );
+#define VPathMakeFmt LegacyVPathMakeFmt
+rc_t VPathMakeFmt ( VPath ** new_path, const char * fmt, ... );
+#define VPathMakeVFmt LegacyVPathMakeVFmt
+rc_t VPathMakeVFmt ( VPath ** new_path, const char * fmt, va_list args );
+#define VPathMakeSysPath LegacyVPathMakeSysPath
+VFS_EXTERN rc_t VPathMakeSysPath ( VPath ** new_path, const char * sys_path );
+
+
#ifdef __cplusplus
}
#endif
diff --git a/libs/align/Makefile b/libs/align/Makefile
index d2c1d2b..b2d8808 100644
--- a/libs/align/Makefile
+++ b/libs/align/Makefile
@@ -76,11 +76,7 @@ BAM_SRC = \
BAM_OBJ = \
$(addsuffix .$(LOBX),$(BAM_SRC))
-BAM_LIB = \
- -lvfs \
- -lkfs \
- -lklib \
- -lz
+BAM_LIB =
$(ILIBDIR)/libncbi-bam.$(LIBX): $(BAM_OBJ)
$(LD) --slib -o $@ $^ $(BAM_LIB)
@@ -97,13 +93,7 @@ ALIGN_ACCESS_SRC = \
ALIGN_ACCESS_OBJ = \
$(addsuffix .$(LOBX),$(ALIGN_ACCESS_SRC))
-ALIGN_ACCESS_LIB = \
- -sncbi-bam \
- -lvfs \
- -lkfs \
- -dkproc \
- -lklib \
- -lz
+ALIGN_ACCESS_LIB =
$(ILIBDIR)/libalign-access.$(LIBX): $(ALIGN_ACCESS_OBJ)
$(LD) --slib -o $@ $^ $(ALIGN_ACCESS_LIB)
@@ -129,13 +119,7 @@ ALIGN_READER_SRC = \
ALIGN_READER_OBJ = \
$(addsuffix .$(LOBX),$(ALIGN_READER_SRC))
-ALIGN_READER_LIB = \
- -lvdb \
- -lkdb \
- -lkfg \
- -lkfs \
- -lklib \
- -lz
+ALIGN_READER_LIB =
$(ILIBDIR)/libalign-reader.$(LIBX): $(ALIGN_READER_OBJ)
$(LD) --slib -o $@ $^ $(ALIGN_READER_LIB)
@@ -161,13 +145,7 @@ ALIGN_WRITER_SRC = \
ALIGN_WRITER_OBJ = \
$(addsuffix .$(LOBX),$(ALIGN_WRITER_SRC))
-ALIGN_WRITER_LIB = \
- -lwvdb \
- -lwkdb \
- -lkfg \
- -lkfs \
- -lklib \
- -lz
+ALIGN_WRITER_LIB =
$(ILIBDIR)/libalign-writer.$(LIBX): $(ALIGN_WRITER_OBJ)
$(LD) --slib -o $@ $^ $(ALIGN_WRITER_LIB)
diff --git a/libs/align/bam.c b/libs/align/bam.c
index 2a86445..1d8af69 100644
--- a/libs/align/bam.c
+++ b/libs/align/bam.c
@@ -261,19 +261,20 @@ static float BGZFileProPos(BGZFile const *const self)
static rc_t BGZFileSetPos(BGZFile *const self, uint64_t const pos)
{
- unsigned const bpos = (unsigned)(pos - self->fpos); /* < 64k */
-
- if (self->fpos <= pos && pos < self->fpos + self->bcount) {
- self->bpos = bpos;
- self->zs.avail_in = (uInt)(self->bcount - bpos);
- self->zs.next_in = (Bytef *)&self->buf[bpos];
- }
- else {
- self->fpos = pos & ~((uint64_t)(MEM_ALIGN_SIZE - 1));
- self->bpos = bpos;
+ if (self->fpos > pos || pos >= self->fpos + self->bcount) {
+ /* desired position is outside of current buffer */
+ self->fpos = pos ^ (pos & ((uint64_t)(MEM_ALIGN_SIZE - 1)));
+ self->bpos = (unsigned)(pos - self->fpos);
self->bcount = 0; /* force re-read */
}
+ else {
+ /* desired position is inside of current buffer */
+ unsigned const bpos = (unsigned)(pos - self->fpos); /* < 64k */
+ self->bpos = bpos; /* pos - self->fpos; */
+ self->zs.avail_in = (uInt)(self->bcount - bpos);
+ self->zs.next_in = (Bytef *)&self->buf[bpos];
+ }
return 0;
}
diff --git a/libs/align/libalign-access.vers b/libs/align/libalign-access.vers
deleted file mode 100644
index a970716..0000000
--- a/libs/align/libalign-access.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.15
diff --git a/libs/align/libalign-reader.vers b/libs/align/libalign-reader.vers
deleted file mode 100644
index 2ac9634..0000000
--- a/libs/align/libalign-reader.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.13
diff --git a/libs/align/libncbi-bam.vers b/libs/align/libncbi-bam.vers
deleted file mode 100644
index 21e8796..0000000
--- a/libs/align/libncbi-bam.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.3
diff --git a/libs/align/reference.c b/libs/align/reference.c
index 8827072..8f9835d 100644
--- a/libs/align/reference.c
+++ b/libs/align/reference.c
@@ -1383,7 +1383,7 @@ LIB_EXPORT rc_t CC ReferenceObj_MakePlacementIterator ( const ReferenceObj* csel
o->last_ref_row_of_window_rel = ref_window_start;
o->last_ref_row_of_window_rel += ref_window_len;
o->last_ref_row_of_window_rel /= mgr->max_seq_len;
- o->rowcount_of_ref = ( cself->end_rowid - cself->start_rowid );
+ o->rowcount_of_ref = ( cself->end_rowid - cself->start_rowid ) + 1;
/* get effective starting offset based on overlap
from alignments which started before the requested pos */
@@ -1733,15 +1733,16 @@ static rc_t read_alignments( PlacementIterator *self )
/*ALIGN_DBG("align rows: %u", cself->ids_col->len);*/
for ( i = 0; rc == 0 && i < self->ids_col->len; i++ )
{
- rc = TableReader_ReadRow( self->align_reader, self->ids_col->base.i64[ i ] );
+ int64_t row_id = self->ids_col->base.i64[ i ];
+ rc = TableReader_ReadRow( self->align_reader, row_id );
if ( rc == 0 )
{
INSDC_coord_zero apos = self->align_cols[ eplacementiter_cn_REF_POS ].base.coord0[ 0 ];
INSDC_coord_len alen = self->align_cols[ eplacementiter_cn_REF_LEN ].base.coord_len[ 0 ];
- /*ALIGN_DBG("align row: {%li, %u, %u}", cself->ids_col->base.i64[i], apos, alen);*/
- if ( self->cur_ref_row_rel < 0 )
- apos -= self->obj->seq_len;
+#if 0
+ ALIGN_DBG( "alignment read: {row_id:%,li, apos:%,d, alen:%u}", row_id, apos, alen );
+#endif
/* at this point we have the position of the alignment.
we want it to intersect with the window */
@@ -1781,19 +1782,21 @@ static rc_t read_alignments( PlacementIterator *self )
continue;
}
- else if ( ( self->obj->circular )&&( apos + alen > self->obj->seq_len ) )
+ else if ( ( self->obj->circular )&&
+ ( apos + alen > self->obj->seq_len )&&
+ ( self->cur_ref_row_rel < 0 ) )
{
/* the end of the alignment sticks over the end of the reference!
---> we have the rare case of an alignment that wraps arround !
let as insert the alignment 2 times!
( one with neg. position, one at real position ) */
- rc = make_alignment( self, self->ids_col->base.i64[i], apos - self->obj->seq_len, alen );
+ rc = make_alignment( self, row_id, apos - self->obj->seq_len, alen );
}
/* having arrived here, we know the alignment intersects our window
apos MAY be < 0 if the alignment wrapped around */
- if ( rc == 0 )
- rc = make_alignment( self, self->ids_col->base.i64[i], apos, alen );
+ if ( rc == 0 && self->cur_ref_row_rel >= 0 )
+ rc = make_alignment( self, row_id, apos, alen );
}
}
return rc;
@@ -1819,7 +1822,7 @@ LIB_EXPORT rc_t CC PlacementIteratorNextAvailPos( const PlacementIterator *cself
self->cur_ref_row_rel++; /* increment row offset */
#if 0
- ALIGN_DBG( "ref row: ref-start-row-id:%li - curr-rel-row:%li - of:%li",
+ ALIGN_DBG( "ref row: ref-start-row-id:%,li - curr-rel-row:%,li - of:%,li",
self->obj->start_rowid, self->cur_ref_row_rel, self->last_ref_row_of_window_rel );
#endif
@@ -1858,8 +1861,9 @@ LIB_EXPORT rc_t CC PlacementIteratorNextAvailPos( const PlacementIterator *cself
if ( len != NULL ) { *len = r->len; }
#if 0
- ALIGN_DBG( "PlacementIteratorNextAvailPos( id=%,li, pos=%,u, n=%,u )", r->id, r->pos, count );
+ ALIGN_DBG( "PlacementIteratorNextAvailPos( id=%,li, pos=%,d, len=%,u, n=%,u )", r->id, r->pos, r->len, count );
#endif
+
if ( !( cself->obj->circular ) && ( r->pos >= ( cself->ref_window_start + cself->ref_window_len ) ) )
{
/* the alignment !starts! after the end of the of the requested window! */
diff --git a/libs/align/writer-reference.c b/libs/align/writer-reference.c
index cccfa14..47c5484 100644
--- a/libs/align/writer-reference.c
+++ b/libs/align/writer-reference.c
@@ -1699,25 +1699,25 @@ LIB_EXPORT rc_t CC ReferenceMgr_Release(const ReferenceMgr *cself,
if (cself != NULL) {
ReferenceMgr *const self = (ReferenceMgr *)cself;
- uint64_t rows;
+ uint64_t rows = 0;
unsigned i;
-
+
rc = TableWriterRef_Whack(self->writer, commit, &rows);
if (Rows) *Rows = rows;
KDirectoryRelease(self->dir);
-
+
for (i = 0; i != self->refSeqsById.elem_count; ++i)
free((void *)((key_id_t *)self->refSeqsById.base)[i].key);
-
+
for (i = 0; i != self->refSeqs.elem_count; ++i)
ReferenceSeq_Whack(&self->refSeq[i]);
-
+
KDataBufferWhack(&self->compress);
KDataBufferWhack(&self->seq);
KDataBufferWhack(&self->refSeqs);
KDataBufferWhack(&self->refSeqsById);
-
- if (rc == 0 && build_coverage && commit)
+
+ if (rc == 0 && build_coverage && commit && rows > 0)
rc = ReferenceMgr_ReCover(cself, rows, quitting);
#if 0
{
diff --git a/libs/align/writer-refseq.c b/libs/align/writer-refseq.c
index 5361a92..c180b8b 100644
--- a/libs/align/writer-refseq.c
+++ b/libs/align/writer-refseq.c
@@ -88,14 +88,15 @@ LIB_EXPORT rc_t CC TableWriterRefSeq_Make(const TableWriterRefSeq** cself, VDBMa
*cself = self;
ALIGN_DBG("table %s created", table_path);
} else {
- TableWriterRefSeq_Whack(self, false, NULL, 0, NULL, NULL, NULL, 0);
+ TableWriterRefSeq_Whack(self, false, NULL, 0, 0, NULL, NULL, 0);
ALIGN_DBGERRP("table %s", rc, table_path);
}
}
return rc;
}
-LIB_EXPORT rc_t CC TableWriterRefSeq_Whack(const TableWriterRefSeq* cself, bool commit, uint64_t* rows,
+LIB_EXPORT rc_t CC TableWriterRefSeq_Whack(const TableWriterRefSeq* cself,
+ bool commit, uint64_t* rows,
const char loader_name[],
const ver_t loader_version,
const char loader_date[],
diff --git a/libs/ascp/ascp.c b/libs/ascp/ascp.c
index d9fae95..133155b 100644
--- a/libs/ascp/ascp.c
+++ b/libs/ascp/ascp.c
@@ -59,7 +59,7 @@ static bool _StringStartsWith(const String *self, const char *buf) {
if (self->len < len) {
return false;
}
- return string_cmp(self->addr, self->len, buf, len, len) == 0;
+ return string_cmp(self->addr, self->len, buf, len, (uint32_t)len) == 0;
}
static
@@ -80,10 +80,12 @@ bool _StringHas(const String *self, const char *buf, String *res)
return false;
}
if (_StringStartsWith(res, buf)) {
- res->len = res->size = len;
+ res->len = (uint32_t)len;
+ res->size = len;
return true;
}
- res->len = res->size = res->len - 1;
+ res->size = res->len - 1;
+ res->len = res->len - 1;
++res->addr;
}
}
@@ -275,18 +277,20 @@ rc_t ascpParse(const char *buf, size_t len, const char *filename,
}
}
if (n != NULL) {
- StringInit(line, p, n - p, n - p);
+ StringInit(line, p, n - p, (uint32_t)(n - p));
l -= n - p + 1;
}
else {
- StringInit(line, p, l, l);
+ StringInit(line, p, l, (uint32_t)l);
}
if (line->addr && line->len && line->addr[line->len - 1] == '\r') {
- line->len = line->size = line->len - 1;
+ line->size = line->len - 1;
+ line->len = line->len - 1;
}
if (line->addr && line->len && line->addr[0] == '\r') {
++line->addr;
- line->len = line->size = line->len - 1;
+ line->size = line->len - 1;
+ line->len = line->len - 1;
}
if (line->len != 0) {
SAscpState full;
@@ -495,6 +499,28 @@ static bool _SystemHelp(const char *command, bool status) {
}
}
+static rc_t _KConfigGetAscpRate(const KConfig *self, const char **max_rate) {
+ String *s = NULL;
+ assert(self && max_rate);
+ *max_rate = NULL;
+ s = _KConfigAscpString(self, "tools/ascp/max_rate", "Aspera max rate");
+ if (s != NULL) {
+ if (s->size == 0) {
+ free(s);
+ }
+ else {
+ *max_rate = string_dup_measure(s->addr, NULL);
+ free(s);
+ if (*max_rate == NULL) {
+ free((void*)*max_rate);
+ return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
+ }
+ }
+ return 0;
+ }
+ return 0;
+}
+
static rc_t _KConfigGetAscp(const KConfig *self,
const char **ascp_bin, const char **private_file)
{
@@ -622,6 +648,17 @@ LIB_EXPORT rc_t CC aspera_get(
opt = &dummy;
}
+ if (opt->target_rate == NULL) {
+ KConfig *cfg = NULL;
+ rc_t rc = KConfigMake(&cfg, NULL);
+ DISP_RC(rc, "cannot KConfigMake");
+ if (rc == 0) {
+ rc = _KConfigGetAscpRate(cfg, &opt->target_rate);
+ DISP_RC(rc, "cannot get aspera max rate");
+ }
+ RELEASE(KConfig, cfg);
+ }
+
sStatus = status = opt->status;
quitting = opt->quitting;
@@ -683,10 +720,26 @@ STSMSG(0, ("KDirectoryFileSize after ascp run: "
return rc;
}
-rc_t mkAscpCmd(const char *ascp_bin, const char *private_file,
+LIB_EXPORT rc_t CC aspera_options(AscpOptions *opt) {
+ KConfig *cfg = NULL;
+ rc_t rc = 0;
+ if (opt == NULL) {
+ return RC(rcNS, rcFile, rcVisiting, rcParam, rcNull);
+ }
+ memset(opt, 0, sizeof *opt);
+ rc = KConfigMake(&cfg, NULL);
+ if (rc == 0) {
+ rc = _KConfigGetAscpRate(cfg, &opt->target_rate);
+ opt->disabled = _KConfigAscpDisabled(cfg, false);
+ }
+ RELEASE(KConfig, cfg);
+ return rc;
+}
+
+/*rc_t mkAscpCmd(const char *ascp_bin, const char *private_file,
const char *src, const char *dest, const AscpOptions *opt,
char *const argv[], size_t argvSz)
{
rc_t rc = 0;
return rc;
-}
+}*/
diff --git a/libs/ascp/win/ascp-path.c b/libs/ascp/win/ascp-path.c
index f3eb105..bb1e144 100644
--- a/libs/ascp/win/ascp-path.c
+++ b/libs/ascp/win/ascp-path.c
@@ -4,8 +4,8 @@
bool ascp_path(const char **cmd, const char **key) {
static int idx = 0;
static const char *c[] = {
-"\"C:\\Program Files (x86)\\Aspera\\Aspera Connect\\bin\\ascp.exe\"",
-"\"C:\\Program Files\\Aspera\\Aspera Connect\\bin\\ascp.exe\""
+ "C:\\Program Files (x86)\\Aspera\\Aspera Connect\\bin\\ascp.exe",
+ "C:\\Program Files\\Aspera\\Aspera Connect\\bin\\ascp.exe"
};
static const char *k[] = {
"C:\\Program Files (x86)\\Aspera\\Aspera Connect\\etc\\asperaweb_id_dsa.putty",
diff --git a/libs/ascp/win/connect.c b/libs/ascp/win/connect.c
index cbc2df9..d80f2df 100644
--- a/libs/ascp/win/connect.c
+++ b/libs/ascp/win/connect.c
@@ -26,11 +26,13 @@
#include "ascp-priv.h" /* STS_DBG */
-#include <kfs/directory.h> // KDirectory
-#include <kfs/impl.h> // KSysDir
+#include <kfs/directory.h> /* KDirectory */
+#include <kfs/impl.h> /* KSysDir */
+#include <kfs/kfs-priv.h> /* KSysDirOSPath */
#include <klib/log.h> /* LOGERR */
#include <klib/out.h> /* OUTMSG */
+#include <klib/printf.h> /* string_printf */
#include <klib/rc.h>
#include <klib/status.h> /* STSMSG */
@@ -48,7 +50,7 @@ static rc_t posixStringToSystemString(char *buffer, size_t len,
rc_t rc = KDirectoryNativeDir(&wd);
struct KSysDir *sysDir = KDirectoryGetSysDir(wd);
wchar_t wd_path[MAX_PATH];
- int ret = 0;
+ size_t ret = 0;
va_list args;
va_start(args, path);
rc = KSysDirOSPath(sysDir, wd_path, sizeof wd_path, path, args);
@@ -120,7 +122,7 @@ static rc_t mkAscpCommand(char *buffer, size_t len,
}
rc = string_printf(buffer, len, &num_writ,
- "%s -i \"%s\" -pQTk1%s%s%s%s%s%s %s %s",
+ "\"%s\" -i \"%s\" -pQTk1%s%s%s%s%s%s %s %s",
path, key,
maxRate == NULL ? "" : " -l", maxRate == NULL ? "" : maxRate,
host == NULL ? "" : " --host ", host == NULL ? "" : host,
@@ -136,8 +138,9 @@ static rc_t mkAscpCommand(char *buffer, size_t len,
return rc;
}
-static int execute(const char *command, uint64_t heartbeat, bool cacheKey,
- bool *writeFailed, bool *canceled, const char *name, TQuitting *quitting)
+static int execute(const char *command, uint64_t heartbeat,
+ bool cacheKey, bool *writeFailed, bool *canceled,
+ const char *name, TQuitting *quitting)
{
DWORD exitCode = STILL_ACTIVE;
HANDLE g_hChildStd_IN_Rd = NULL;
@@ -233,7 +236,7 @@ static int execute(const char *command, uint64_t heartbeat, bool cacheKey,
String line;
StringInit(&line, NULL, 0, 0);
if (heartbeat > 0) {
- dwMillisecondsRemains = heartbeat;
+ dwMillisecondsRemains = (DWORD)heartbeat;
}
// Write to the pipe that is the standard input for a child process.
// Data is written to the pipe's buffers, so it is not necessary to wait
@@ -315,9 +318,9 @@ static int execute(const char *command, uint64_t heartbeat, bool cacheKey,
l = line.len;
WriteFile(hParentStdOut, "\r", 1,
&dwWritten, NULL);
- bSuccess = WriteFile(hParentStdOut, p, l,
- &dwWritten, NULL);
- dwMillisecondsRemains = heartbeat;
+ bSuccess = WriteFile(hParentStdOut, p,
+ (DWORD)l, &dwWritten, NULL);
+ dwMillisecondsRemains = (DWORD)heartbeat;
progressing = true;
}
}
diff --git a/libs/axf/libaxf.vers b/libs/axf/libaxf.vers
deleted file mode 100644
index b0f3d96..0000000
--- a/libs/axf/libaxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.8
diff --git a/libs/axf/libwaxf.vers b/libs/axf/libwaxf.vers
deleted file mode 100644
index b0f3d96..0000000
--- a/libs/axf/libwaxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.8
diff --git a/libs/ext/Makefile b/libs/ext/Makefile
deleted file mode 100644
index 3fcd509..0000000
--- a/libs/ext/Makefile
+++ /dev/null
@@ -1,55 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-TOP ?= $(shell ../../build/abspath.sh ../..)
-MODULE = libs/ext
-
-include $(TOP)/build/Makefile.shell
-
-#-------------------------------------------------------------------------------
-# default
-#
-SUBDIRS = \
- zlib \
- bzip2 \
- curl
-
-include $(TOP)/build/Makefile.targets
-
-default: $(SUBDIRS)
-
-$(SUBDIRS):
- @ $(MAKE) -C $@
-
-.PHONY: default $(SUBDIRS)
-
-
-#-------------------------------------------------------------------------------
-# pass-through
-#
-out debug profile release GCC ICC VC++:
- @ $(MAKE) TOP=$(TOP) -f $(TOP)/build/Makefile.env $@
-
-.PHONY: out debug profile release GCC ICC VC++
diff --git a/libs/ext/bzip2/Makefile b/libs/ext/bzip2/Makefile
deleted file mode 100644
index 79ae1a8..0000000
--- a/libs/ext/bzip2/Makefile
+++ /dev/null
@@ -1,143 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-default: std
-
-TOP ?= $(shell ../../../build/abspath.sh ../../..)
-MODULE = libs/ext/bzip2
-
-include $(TOP)/build/Makefile.env
-
-INT_LIBS = \
- libbz2
-
-ALL_LIBS = \
- $(INT_LIBS)
-
-#-------------------------------------------------------------------------------
-# outer targets
-#
-std all: makedirs download
- @ $(MAKE_CMD) $(TARGDIR)/$@
-
-$(INT_LIBS): makedirs download
- @ $(MAKE_CMD) $(ILIBDIR)/$@
-
-.PHONY: all std $(ALL_LIBS)
-
-#-------------------------------------------------------------------------------
-# all
-#
-$(TARGDIR)/all: $(ILIBDIR)/libbz2
-
-.PHONY: $(TARGDIR)/all
-
-#-------------------------------------------------------------------------------
-# std
-#
-$(TARGDIR)/std: $(ILIBDIR)/libbz2
-
-.PHONY: $(TARGDIR)/std
-
-#-------------------------------------------------------------------------------
-# clean
-#
-clean: stdclean
- @ rm -f libbz2.vers
-
-.PHONY: clean
-
-#-------------------------------------------------------------------------------
-# libbz2
-#
-$(ILIBDIR)/libbz2: $(addprefix $(ILIBDIR)/libbz2.,$(LIBX))
-
-LIBBZ2_SRC = \
- blocksort \
- huffman \
- crctable \
- randtable \
- compress \
- decompress \
- bzlib
-
-LIBBZ2_OBJ = \
- $(addsuffix .$(LOBX),$(LIBBZ2_SRC))
-
-$(ILIBDIR)/libbz2.$(LIBX): $(LIBBZ2_OBJ)
- $(LD) --slib -o $@ $^ $(LIBBZ2_LIB)
-
-
-#-------------------------------------------------------------------------------
-# libbz2 download
-#
-BZIP2_VERS := \
- 1.0.6
-
-BZIP2_TAR = \
- bzip2-$(BZIP2_VERS).tar
-
-BZIP2_TAR_GZ = \
- $(BZIP2_TAR).gz
-
-ifeq (win,$(OS))
-ECHO_VERS = echo -ne $(BZIP2_VERS) "\n"
-else
-ECHO_VERS = echo $(BZIP2_VERS)
-endif
-
-ifeq (sun,$(OS))
- BZIP2_TAR_GET = $(BZIP2_TAR)
- TAR_CMD = tar -xf
-else
- BZIP2_TAR_GET = $(BZIP2_TAR_GZ)
- TAR_CMD = tar --no-same-owner -x -z -f
-endif
-
-ifeq (mac,$(OS))
-RETRIEVE = curl -O
-else
-RETRIEVE = wget
-endif
-
-$(BZIP2_TAR_GZ):
- @ echo downloading the sources
- $(RETRIEVE) http://bzip.org/$(BZIP2_VERS)/bzip2-$(BZIP2_VERS).tar.gz
- @ touch $@
-
-$(BZIP2_TAR): $(BZIP2_TAR_GZ)
- gzcat $^ > $@
-
-libbz2.vers: $(BZIP2_TAR_GET)
- @ echo unpacking the sources
- $(TAR_CMD) $^
- cp bzip2-$(BZIP2_VERS)/*.[ch] .
- cp bzip2-$(BZIP2_VERS)/*.def .
- #-rm -rf bzip2-$(BZIP2_VERS)
- @ $(ECHO_VERS) > $@
-
-download: libbz2.vers
-
-.PHONY: download
diff --git a/libs/ext/curl/Makefile b/libs/ext/curl/Makefile
deleted file mode 100644
index e5736a8..0000000
--- a/libs/ext/curl/Makefile
+++ /dev/null
@@ -1,103 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-default: std
-
-TOP ?= $(shell ../../../build/abspath.sh ../../..)
-MODULE = libs/ext/curl
-
-include $(TOP)/build/Makefile.env
-
-INT_LIBS = \
-
-EXT_LIBS = \
- libcurl
-
-ALL_LIBS = \
- $(INT_LIBS) \
- $(EXT_LIBS)
-
-#-------------------------------------------------------------------------------
-# outer targets
-#
-ifneq (rwin,$(BUILD_OS))
-
-all std clean $(ALL_LIBS):
- @true
-
-.PHONY: all std clean $(ALL_LIBS)
-
-else
-
-std: makedirs
- @ $(MAKE_CMD) $(TARGDIR)/$@
-
-all: makedirs
- @ $(MAKE_CMD) $(TARGDIR)/$@
-
-$(INT_LIBS): makedirs
- @ $(MAKE_CMD) $(ILIBDIR)/$@
-
-$(EXT_LIBS): makedirs
- @ $(MAKE_CMD) $(LIBDIR)/$@
-
-.PHONY: all std $(ALL_LIBS)
-
-#-------------------------------------------------------------------------------
-# all
-#
-$(TARGDIR)/all: $(BINDIR)/libcurl.$(DLLX)
-
-.PHONY: $(TARGDIR)/all
-
-#-------------------------------------------------------------------------------
-# std
-#
-$(TARGDIR)/std: $(BINDIR)/libcurl.$(DLLX)
-
-.PHONY: $(TARGDIR)/std
-
-#-------------------------------------------------------------------------------
-# clean
-#
-clean: stdclean
- rm -f $(BINDIR)/libcurl.$(DLLX)
-
-.PHONY: clean
-
-#-------------------------------------------------------------------------------
-# libcurl
-#
-LIBCURL_VERS := \
- 7.29.0
-
-ifeq (dbg,$(BUILD))
- DBG=-dbg
-endif
-$(BINDIR)/libcurl.$(DLLX): /panfs/pan1/sra-test/TOOLKIT/libcurl/$(LIBCURL_VERS)/win$(BITS)$(DBG)/libcurl.$(DLLX)
- cp $^ $@
-
-endif
-
diff --git a/libs/ext/zlib/Makefile b/libs/ext/zlib/Makefile
deleted file mode 100644
index 4ae581a..0000000
--- a/libs/ext/zlib/Makefile
+++ /dev/null
@@ -1,150 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-default: std
-
-TOP ?= $(shell ../../../build/abspath.sh ../../..)
-MODULE = libs/ext/zlib
-
-include $(TOP)/build/Makefile.env
-
-INT_LIBS = \
- libz
-
-ALL_LIBS = \
- $(INT_LIBS)
-
-#-------------------------------------------------------------------------------
-# outer targets
-#
-std all: makedirs download
- @ $(MAKE_CMD) $(TARGDIR)/$@
-
-$(INT_LIBS): makedirs download
- @ $(MAKE_CMD) $(ILIBDIR)/$@
-
-.PHONY: all std $(ALL_LIBS)
-
-#-------------------------------------------------------------------------------
-# all
-#
-$(TARGDIR)/all: $(ILIBDIR)/libz
-
-.PHONY: $(TARGDIR)/all
-
-#-------------------------------------------------------------------------------
-# std
-#
-$(TARGDIR)/std: $(ILIBDIR)/libz
-
-.PHONY: $(TARGDIR)/std
-
-#-------------------------------------------------------------------------------
-# clean
-#
-clean: stdclean
-
-.PHONY: clean
-
-#-------------------------------------------------------------------------------
-# libz
-#
-$(ILIBDIR)/libz: $(addprefix $(ILIBDIR)/libz.,$(LIBX))
-
-LIBZ_SRC = \
- adler32 \
- compress \
- crc32 \
- deflate \
- gzclose \
- gzlib \
- gzread \
- gzwrite \
- infback \
- inffast \
- inflate \
- inftrees \
- trees \
- uncompr \
- zutil
-
-
-LIBZ_OBJ = \
- $(addsuffix .$(LOBX),$(LIBZ_SRC))
-
-$(ILIBDIR)/libz.$(LIBX): $(LIBZ_OBJ)
- $(LD) --slib -o $@ $^ $(LIBZ_LIB)
-
-
-#-------------------------------------------------------------------------------
-# libz download
-#
-ZLIB_VERS := \
- 1.2.8
-
-ZLIB_TAR = \
- zlib-$(ZLIB_VERS).tar
-
-ZLIB_TAR_GZ = \
- $(ZLIB_TAR).gz
-
-ifeq (win,$(OS))
-ECHO_VERS = echo -ne $(ZLIB_VERS) "\n"
-else
-ECHO_VERS = echo $(ZLIB_VERS)
-endif
-
-ifeq (sun,$(OS))
- ZLIB_TAR_GET = $(ZLIB_TAR)
- TAR_CMD = tar -xf
-else
- ZLIB_TAR_GET = $(ZLIB_TAR_GZ)
- TAR_CMD = tar --no-same-owner -x -z -f
-endif
-
-ifeq (mac,$(OS))
-RETRIEVE = curl -O
-else
-RETRIEVE = wget
-endif
-
-$(ZLIB_TAR_GZ):
- @ echo downloading the sources
- $(RETRIEVE) http://zlib.net/$@
- @ touch $@
-
-$(ZLIB_TAR): $(ZLIB_TAR_GZ)
- gzcat $^ > $@
-
-libz.vers: $(ZLIB_TAR_GET)
- @ echo unpacking the sources
- $(TAR_CMD) $^
- cp zlib-$(ZLIB_VERS)/*.[ch] .
- cp zlib-$(ZLIB_VERS)/win32/zlib.def libz.def
- @ $(ECHO_VERS) > $@
-
-download: libz.vers
-
-.PHONY: download
diff --git a/libs/kapp/args.c b/libs/kapp/args.c
index 06ef91c..0a0d096 100644
--- a/libs/kapp/args.c
+++ b/libs/kapp/args.c
@@ -1854,7 +1854,7 @@ static void print_indented( const size_t first_indent, const size_t indent,
if ( space != NULL )
{
/* space found, can we print the word on the current line? */
- int wordlen = ( space - msg );
+ int wordlen = (int) ( space - msg );
if ( ( line_len + wordlen + 1 ) < max_line_len )
{
if ( wordlen > 1 )
diff --git a/libs/kapp/libkapp.vers b/libs/kapp/libkapp.vers
deleted file mode 100644
index 9084fa2..0000000
--- a/libs/kapp/libkapp.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.0
diff --git a/libs/kapp/libload.vers b/libs/kapp/libload.vers
deleted file mode 100644
index 66c4c22..0000000
--- a/libs/kapp/libload.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.9
diff --git a/libs/kapp/loader-file.c b/libs/kapp/loader-file.c
index 26fc1b6..f048815 100644
--- a/libs/kapp/loader-file.c
+++ b/libs/kapp/loader-file.c
@@ -268,7 +268,7 @@ rc_t KLoaderFile_Fill(KLoaderFile *self)
if( (rc = KFileRead(self->file, self->pos + self->avail,
&self->buffer[self->avail], to_read, &num_read)) == 0 ) {
self->eof = (num_read == 0);
- self->avail += num_read;
+ self->avail += (uint32_t) num_read;
to_read -= num_read;
DBG(("KLoaderFile read %s from %lu %u bytes%s\n",
self->filename, self->pos + self->avail - num_read, num_read, self->eof ? " EOF" : ""));
@@ -320,7 +320,7 @@ LIB_EXPORT rc_t CC KLoaderFile_Reset(const KLoaderFile* cself)
self->buffer[0] = 0;
self->eof = false;
} else {
- self->avail += self->buffer_pos - self->buffer;
+ self->avail += (uint32_t) ( self->buffer_pos - self->buffer );
}
self->pos = 0;
self->eol = 0;
@@ -474,7 +474,7 @@ LIB_EXPORT rc_t CC KLoaderFile_Readline(const KLoaderFile* cself, const void** b
}
} else {
*length = nl - (uint8_t*)*buffer;
- self->eol = nl - self->buffer_pos + 1;
+ self->eol = (uint32_t) ( nl - self->buffer_pos + 1 );
if( *nl == '\r' && nl < &self->buffer[self->buffer_size - 1] && *(nl + 1) == '\n' ) {
/* \r\n */
self->eol++;
@@ -501,8 +501,8 @@ LIB_EXPORT rc_t CC KLoaderFile_Read(const KLoaderFile* cself, size_t advance, si
self->eol = 0;
} else {
self->buffer_pos += advance;
- self->avail -= advance;
- self->eol = self->eol > advance ? self->eol - advance : 0;
+ self->avail -= (uint32_t) advance;
+ self->eol = (uint32_t) ( self->eol > advance ? self->eol - advance : 0 );
}
}
if( size > self->avail || self->avail == 0 ) {
diff --git a/libs/kapp/progressbar.c b/libs/kapp/progressbar.c
index 37bdd81..130efaf 100644
--- a/libs/kapp/progressbar.c
+++ b/libs/kapp/progressbar.c
@@ -109,7 +109,7 @@ void CC job_report(bool force_report, bool final)
memset(&d, 0, sizeof(d));
SLListForEach(&g_jobs.jobs, job_percent, &d);
- d.qty = d.qty ? d.percent / d.qty : (final ? 100 : 0);
+ d.qty = (uint64_t) (d.qty ? d.percent / d.qty : (final ? 100 : 0) );
if( force_report || d.qty != g_jobs.percent ) {
g_jobs.percent = d.qty;
PLOGMSG(klogInfo, (klogInfo, "processed $(percent)%",
diff --git a/libs/kapp/tokenizer.c b/libs/kapp/tokenizer.c
index b386f37..27cfdce 100644
--- a/libs/kapp/tokenizer.c
+++ b/libs/kapp/tokenizer.c
@@ -139,7 +139,7 @@ static rc_t expand_argv( tokenzr *t )
}
else
{
- if ( ( *(t->argc) ) >= t->nargs )
+ if ( (uint32_t) ( *(t->argc) ) >= t->nargs )
{
char ** pargv1 = realloc( pargv, ( ( t->nargs + ARGV_INC ) * ( sizeof pargv[ 0 ] ) ) );
if ( pargv1 != NULL )
@@ -445,7 +445,7 @@ static int sized_str_cmp( const char *a, const char *b )
{
size_t asize = string_size ( a );
size_t bsize = string_size ( b );
- return strcase_cmp ( a, asize, b, bsize, ( asize > bsize ) ? asize : bsize );
+ return strcase_cmp ( a, asize, b, bsize, (uint32_t) ( ( asize > bsize ) ? asize : bsize ) );
}
diff --git a/libs/kapp/unix/sysmain.c b/libs/kapp/unix/sysmain.c
index cbeb87e..8bb8925 100644
--- a/libs/kapp/unix/sysmain.c
+++ b/libs/kapp/unix/sysmain.c
@@ -47,6 +47,10 @@
#define CATCH_SIGSEGV 1
#endif
+#if ! defined CATCH_SIGHUP
+#define CATCH_SIGHUP 0
+#endif
+
/*--------------------------------------------------------------------------
* Main
*/
@@ -187,7 +191,9 @@ int main ( int argc, char *argv [] )
int sig;
} sigs [] =
{
+#if CATCH_SIGHUP
{ SigHupHandler, SIGHUP },
+#endif
{ SigQuitHandler, SIGINT },
{ SigQuitHandler, SIGQUIT },
#if CATCH_SIGSEGV
diff --git a/libs/kapp/win/sysmain.c b/libs/kapp/win/sysmain.c
index e8f3bcc..a6d1a39 100644
--- a/libs/kapp/win/sysmain.c
+++ b/libs/kapp/win/sysmain.c
@@ -35,6 +35,7 @@
#include <klib/debug.h>
#include <klib/rc.h>
#include <klib/text.h>
+#include <klib/report.h>
#include <atomic32.h>
/* #define _WIN32_WINNT 0x0500 */
diff --git a/libs/kdb/btree.c b/libs/kdb/btree.c
index ec8f159..600dec3 100644
--- a/libs/kdb/btree.c
+++ b/libs/kdb/btree.c
@@ -1316,11 +1316,11 @@ int compare_keys ( const KBTree *self, const void *query, size_t qsize, const vo
if ( self -> cmp != NULL )
return ( * self -> cmp ) ( query, qsize, key, key_size );
else {
- int csize = (qsize < key_size) ? qsize : key_size;
- int diff = memcmp ( query, key, csize);
- if(diff == 0)
- return qsize-key_size;
- return diff;
+ size_t csize = (qsize < key_size) ? qsize : key_size;
+ int diff = memcmp ( query, key, csize);
+ if (diff == 0)
+ return (int)qsize - (int)key_size;
+ return diff;
}
}
@@ -1702,8 +1702,8 @@ rc_t make_entry ( KBTree *self, KBTreeEntryData *pb, void *hdrp, void *ordp )
uint8_t *page = hdrp;
KBTreeLeafNode *hdr = hdrp;
KBTreeLeafEntry *ord = ordp;
- const uint8_t *key = pb -> key;
- uint16_t key_size = pb -> key_size - hdr->key_prefix_len;
+ const uint8_t *key = pb -> key;
+ uint16_t key_size = (uint16_t) ( pb -> key_size - hdr->key_prefix_len );
assert(memcmp(key,page + hdr->key_prefix, hdr -> key_prefix_len)==0);/*** validate in debug mode **/
@@ -1854,7 +1854,7 @@ rc_t split_leaf ( KBTree *self, KBTreeEntryData *pb,
{
ksize = left -> ord [ j ] . ksize + sizeof ( uint32_t );
right -> ord [ i ] . ksize = left -> ord [ j ] . ksize;
- right -> key_bytes += ksize;
+ right -> key_bytes += (uint16_t) ksize;
right -> ord [ i ] . key = ( uint16_t ) ( PGSIZE - right -> key_bytes );
memcpy (rpage + right -> ord [ i ] . key, lpage + left -> ord [ j ] . key, ksize );
if(i == 0 && left->key_prefix_len > 0){
@@ -1877,7 +1877,7 @@ rc_t split_leaf ( KBTree *self, KBTreeEntryData *pb,
{
j = ord [ i ];
ksize = left -> ord [ j ] . ksize + sizeof ( uint32_t );
- off = PGSIZE - ( left -> key_bytes += ksize );
+ off = PGSIZE - ( left -> key_bytes += (uint16_t) ksize );
if ( left -> ord [ j ] . key != off )
{
memmove ( & lpage [ off ], & lpage [ left -> ord [ j ] . key ], ksize );
@@ -1943,7 +1943,7 @@ rc_t split_leaf ( KBTree *self, KBTreeEntryData *pb,
#endif
memcpy ( split -> key, pb -> key , pb -> key_size );
memcpy ( & ( ( uint8_t* ) split -> key ) [ pb -> key_size ], pb -> id, sizeof * pb -> id );
- split -> ksize = pb -> key_size;
+ split -> ksize = (uint16_t) pb -> key_size;
return 0;
#else
/* insert value */
@@ -1964,7 +1964,7 @@ rc_t split_leaf ( KBTree *self, KBTreeEntryData *pb,
will succeed, unless there would be a provision for overflow. */
/* decide where to insert entry */
- if ( slot <= median )
+ if ( slot <= (uint32_t) median )
return leaf_insert ( self, pb, left, slot );
return leaf_insert ( self, pb, right, slot - median - 1 );
}
@@ -2291,16 +2291,16 @@ rc_t split_branch ( KBTree *self,
{
ksize = left -> ord [ j ] . ksize + sizeof ( uint32_t );
right -> ord [ i ] . ksize = left -> ord [ j ] . ksize;
- right -> key_bytes += ksize;
+ right -> key_bytes += (uint16_t) ksize;
right -> ord [ i ] . key = ( uint16_t ) ( PGSIZE - right -> key_bytes );
memcpy ( & rpage [ PGSIZE - right -> key_bytes ], & lpage [ left -> ord [ j ] . key ], ksize );
right -> ord [ i - 1 ] . trans = left -> ord [ j - 1 ] . trans;
- if(i == 0 && left->key_prefix_len > 0){
+ if(i == 0 && left->key_prefix_len > 0){
off = PGSIZE - right -> key_bytes - left -> key_prefix_len;
memcpy ( & rpage [ off ], lpage + left -> key_prefix, left -> key_prefix_len );
right -> key_bytes += left -> key_prefix_len;
right -> key_prefix_len = left -> key_prefix_len;
- right -> key_prefix = off;
+ right -> key_prefix = (uint16_t) off;
}
}
@@ -2319,7 +2319,7 @@ rc_t split_branch ( KBTree *self,
{
j = ord [ i ];
ksize = left -> ord [ j ] . ksize + sizeof ( uint32_t );
- off = PGSIZE - ( left -> key_bytes += ksize );
+ off = PGSIZE - ( left -> key_bytes += (uint16_t) ksize );
if ( left -> ord [ j ] . key != off )
{
memmove ( & lpage [ off ], & lpage [ left -> ord [ j ] . key ], ksize );
@@ -2329,7 +2329,7 @@ rc_t split_branch ( KBTree *self,
off = PGSIZE - left -> key_bytes - left -> key_prefix_len;
memcpy ( & lpage [ off ], rpage + right -> key_prefix, left -> key_prefix_len );
left -> key_bytes += left -> key_prefix_len;
- left -> key_prefix = off;
+ left -> key_prefix = (uint16_t) off;
}
}
/*** maintain search windows ****/
@@ -2389,7 +2389,7 @@ rc_t split_branch ( KBTree *self,
}
/* decide where to insert entry */
- if ( slot <= median )
+ if ( slot <= (uint32_t) median )
return branch_insert ( self, left, val, slot );
return branch_insert ( self, right, val, slot - median - 1 );
}
@@ -2436,7 +2436,7 @@ rc_t leaf_compact (KBTree *self, KPage *pg,uint16_t prefix_len)
uint16_t ksize = node->ord[j].ksize + sizeof ( uint32_t ) - prefix_len;
memmove(dst,src,ksize);
node->ord[j].ksize -= prefix_len;
- node->ord[j].key = dst - ((uint8_t*)node);
+ node->ord[j].key = (uint16_t) ( dst - ((uint8_t*)node) );
}
assert( node -> key_bytes > (node->count-1) * prefix_len);
node -> key_bytes -= (node->count-1) * prefix_len;
@@ -2514,7 +2514,7 @@ rc_t branch_compact (KBTree *self, KPage *pg,uint16_t prefix_len)
uint16_t ksize = node->ord[j].ksize + sizeof ( uint32_t ) - prefix_len;
memmove(dst,src,ksize);
node->ord[j].ksize -= prefix_len;
- node->ord[j].key = dst - ((uint8_t*)node);
+ node->ord[j].key = (uint16_t) ( dst - ((uint8_t*)node) );
}
assert( node -> key_bytes > (node->count-1) * prefix_len);
node -> key_bytes -= (node->count-1) * prefix_len;
@@ -2942,7 +2942,7 @@ rc_t leaf_foreach ( const KBTree *self, bool reverse, uint32_t nid,
if ( nid == 0 )
return 0;
- rc = KPageFileGet ( self -> pgfile, & page, nid >> 1 );
+ rc = KPageFileGet ( self -> pgfile, & page, (uint32_t) ( nid >> 1 ) );
if ( rc == 0 )
{
const KBTreeLeafNode *cnode;
@@ -2976,7 +2976,7 @@ rc_t branch_foreach ( const KBTree *self, bool reverse, uint32_t nid,
#endif
{
KPage *page;
- rc_t rc = KPageFileGet ( self -> pgfile, & page, nid >> 1 );
+ rc_t rc = KPageFileGet ( self -> pgfile, & page, (uint32_t) ( nid >> 1 ) );
if ( rc == 0 )
{
const KBTreeBranchNode *cnode;
diff --git a/libs/kdb/column-cc.c b/libs/kdb/column-cc.c
index c283d4b..d52d1e7 100644
--- a/libs/kdb/column-cc.c
+++ b/libs/kdb/column-cc.c
@@ -141,7 +141,7 @@ rc_t KColumnCheckBlobs(const KColumn *self,
CCReportFunc report, void *ctx)
{
int64_t start;
- int64_t row;
+ uint64_t row;
uint64_t rows;
rc_t rc;
diff --git a/libs/kdb/database.c b/libs/kdb/database.c
index d81aba2..7329631 100644
--- a/libs/kdb/database.c
+++ b/libs/kdb/database.c
@@ -343,7 +343,7 @@ LIB_EXPORT bool CC KDatabaseVExists ( const KDatabase *self, uint32_t type, cons
if ( self != NULL && name != NULL && name [ 0 ] != 0 )
{
rc_t rc;
- size_t len;
+ uint32_t len;
const char *ns;
char path [ 256 ];
@@ -417,7 +417,7 @@ LIB_EXPORT bool CC KDatabaseIsAlias ( const KDatabase *self, uint32_t type,
if ( self != NULL && name != NULL && name [ 0 ] != 0 )
{
rc_t rc;
- size_t len;
+ uint32_t len;
const char *ns;
char path [ 256 ];
diff --git a/libs/kdb/libkdb.vers b/libs/kdb/libkdb.vers
deleted file mode 100644
index 0b1f1ed..0000000
--- a/libs/kdb/libkdb.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.2.13
diff --git a/libs/kdb/libwkdb.vers b/libs/kdb/libwkdb.vers
deleted file mode 100644
index 0b1f1ed..0000000
--- a/libs/kdb/libwkdb.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.2.13
diff --git a/libs/kdb/trieidx-v2.c b/libs/kdb/trieidx-v2.c
index c0eeaa4..1a2811e 100644
--- a/libs/kdb/trieidx-v2.c
+++ b/libs/kdb/trieidx-v2.c
@@ -372,14 +372,27 @@ rc_t KPTrieIndexInit_v2 ( KPTrieIndex_v2 *self, const KMMap *mm, bool byteswap )
rc = KMMapAddrRead ( mm, ( const void** ) & hdr );
if ( rc == 0 )
{
+ uint16_t id_bits, span_bits;
/* recheck header size */
if ( size < sizeof * hdr )
return RC ( rcDB, rcIndex, rcConstructing, rcTrie, rcCorrupt );
- self -> first = hdr -> first;
- self -> last = self -> maxid = hdr -> last;
- self -> id_bits = ( uint8_t ) hdr -> id_bits;
- self -> span_bits = ( uint8_t ) hdr -> span_bits;
+ if ( self -> byteswap )
+ {
+ self -> first = bswap_64(hdr -> first);
+ self -> last = self -> maxid = bswap_64(hdr -> last);
+ id_bits = bswap_16(hdr -> id_bits);
+ span_bits = bswap_16(hdr -> span_bits);
+ }
+ else
+ {
+ self -> first = hdr -> first;
+ self -> last = self -> maxid = hdr -> last;
+ id_bits = hdr -> id_bits;
+ span_bits = hdr -> span_bits;
+ }
+ self -> id_bits = ( uint8_t ) id_bits;
+ self -> span_bits = ( uint8_t ) span_bits;
self -> byteswap = byteswap;
/* try to create the pttree */
@@ -434,14 +447,14 @@ rc_t KPTrieIndexInit_v2 ( KPTrieIndex_v2 *self, const KMMap *mm, bool byteswap )
size -= ( size_t ) self -> count << 2;
/* unpack id map */
- if ( hdr -> id_bits <= 8 )
- rc = KPTrieIndexInitID2Ord ( self, size, 1, hdr -> span_bits, 8 );
- else if ( hdr -> id_bits <= 16 )
- rc = KPTrieIndexInitID2Ord ( self, size, 2, hdr -> span_bits, 16 );
- else if ( hdr -> id_bits <= 32 )
- rc = KPTrieIndexInitID2Ord ( self, size, 3, hdr -> span_bits, 32 );
+ if ( id_bits <= 8 )
+ rc = KPTrieIndexInitID2Ord ( self, size, 1, span_bits, 8 );
+ else if ( id_bits <= 16 )
+ rc = KPTrieIndexInitID2Ord ( self, size, 2, span_bits, 16 );
+ else if ( id_bits <= 32 )
+ rc = KPTrieIndexInitID2Ord ( self, size, 3, span_bits, 32 );
else
- rc = KPTrieIndexInitID2Ord ( self, size, 4, hdr -> span_bits, 64 );
+ rc = KPTrieIndexInitID2Ord ( self, size, 4, span_bits, 64 );
/* done */
if ( rc == 0 )
@@ -496,14 +509,27 @@ rc_t KPTrieIndexInit_v3_v4 ( KPTrieIndex_v2 *self, const KMMap *mm, bool byteswa
rc = KMMapAddrRead ( mm, ( const void** ) & hdr );
if ( rc == 0 )
{
+ uint16_t id_bits, span_bits;
/* recheck header size */
if ( size < sizeof * hdr )
return RC ( rcDB, rcIndex, rcConstructing, rcTrie, rcCorrupt );
- self -> first = hdr -> first;
- self -> last = self -> maxid = hdr -> last;
- self -> id_bits = ( uint8_t ) hdr -> id_bits;
- self -> span_bits = ( uint8_t ) hdr -> span_bits;
+ if ( self -> byteswap )
+ {
+ self -> first = bswap_64(hdr -> first);
+ self -> last = self -> maxid = bswap_64(hdr -> last);
+ id_bits = bswap_16(hdr -> id_bits);
+ span_bits = bswap_16(hdr -> span_bits);
+ }
+ else
+ {
+ self -> first = hdr -> first;
+ self -> last = self -> maxid = hdr -> last;
+ id_bits = hdr -> id_bits;
+ span_bits = hdr -> span_bits;
+ }
+ self -> id_bits = ( uint8_t ) id_bits;
+ self -> span_bits = ( uint8_t ) span_bits;
self -> byteswap = byteswap;
/* try to create the pttree */
@@ -558,14 +584,14 @@ rc_t KPTrieIndexInit_v3_v4 ( KPTrieIndex_v2 *self, const KMMap *mm, bool byteswa
size -= ( size_t ) self -> count << 2;
/* unpack id map */
- if ( hdr -> id_bits <= 8 )
- rc = KPTrieIndexInitID2Ord ( self, size, 1, hdr -> span_bits, 8 );
- else if ( hdr -> id_bits <= 16 )
- rc = KPTrieIndexInitID2Ord ( self, size, 2, hdr -> span_bits, 16 );
- else if ( hdr -> id_bits <= 32 )
- rc = KPTrieIndexInitID2Ord ( self, size, 3, hdr -> span_bits, 32 );
+ if ( id_bits <= 8 )
+ rc = KPTrieIndexInitID2Ord ( self, size, 1, span_bits, 8 );
+ else if ( id_bits <= 16 )
+ rc = KPTrieIndexInitID2Ord ( self, size, 2, span_bits, 16 );
+ else if ( id_bits <= 32 )
+ rc = KPTrieIndexInitID2Ord ( self, size, 3, span_bits, 32 );
else
- rc = KPTrieIndexInitID2Ord ( self, size, 4, hdr -> span_bits, 64 );
+ rc = KPTrieIndexInitID2Ord ( self, size, 4, span_bits, 64 );
/* done */
if ( rc == 0 )
diff --git a/libs/kfg/Makefile b/libs/kfg/Makefile
index 7220a5a..f53b060 100644
--- a/libs/kfg/Makefile
+++ b/libs/kfg/Makefile
@@ -33,8 +33,6 @@ INT_LIBS = \
ALL_LIBS = \
$(INT_LIBS)
-TEST_TOOLS = \
-
include $(TOP)/build/Makefile.env
#-------------------------------------------------------------------------------
@@ -46,10 +44,7 @@ all std: makedirs
$(INT_LIBS): makedirs
@ $(MAKE_CMD) $(ILIBDIR)/$@
-$(TEST_TOOLS): makedirs
- @ $(MAKE_CMD) $(TEST_BINDIR)/$@
-
-.PHONY: all std $(ALL_LIBS) $(TEST_TOOLS)
+.PHONY: all std $(ALL_LIBS)
#-------------------------------------------------------------------------------
# std
@@ -63,7 +58,6 @@ $(TARGDIR)/std: \
# clean
#
clean: stdclean
- @ rm -f $(addsuffix *,$(addprefix $(TEST_BINDIR)/,$(TEST_TOOLS)))
.PHONY: clean
@@ -78,9 +72,11 @@ $(NCBIDIR)/default.kfg: $(SRCDIR)/default.kfg
rm -f $(NCBIDIR)/config.kfg
cp $^ $@
+ifneq ($(NCBIDIR),$(TEST_BINDIR)/ncbi)
$(TEST_BINDIR)/ncbi/default.kfg: $(SRCDIR)/default.kfg
mkdir -p $(TEST_BINDIR)/ncbi
cp $^ $@
+endif
KFG_SRC = \
config-grammar \
diff --git a/libs/kfg/config-grammar.c b/libs/kfg/config-grammar.c
index 60c9a8a..9f18035 100644
--- a/libs/kfg/config-grammar.c
+++ b/libs/kfg/config-grammar.c
@@ -74,20 +74,19 @@
/* Copy the first part of user declarations. */
- #include <sysalloc.h>
-
- #include "kfg-parse.h"
-
- #define YYSTYPE KFGSymbol
- #include "config-tokens.h"
-
- #define KFG_lex KFGScan_yylex
-
- /* required parameter to VNamelistMake */
- #define NAMELIST_ALLOC_BLKSIZE 10
-
- static void ReportRc(KFGParseBlock* pb, KFGScanBlock* sb, rc_t rc);
- static void AppendName(KFGScanBlock* sb, VNamelist*, const KFGParseBlock*);
+ #include "kfg-parse.h"
+ #include <sysalloc.h>
+
+ #define YYSTYPE KFGSymbol
+ #include "config-tokens.h"
+
+ #define KFG_lex KFGScan_yylex
+
+ /* required parameter to VNamelistMake */
+ #define NAMELIST_ALLOC_BLKSIZE 10
+
+ static void ReportRc(KFGParseBlock* pb, KFGScanBlock* sb, rc_t rc);
+ static void AppendName(KFGScanBlock* sb, VNamelist*, const KFGParseBlock*);
@@ -138,8 +137,8 @@ typedef union YYSTYPE
{
- KFGParseBlock pb;
- const VNamelist* namelist;
+ KFGParseBlock pb;
+ const VNamelist* namelist;
@@ -443,8 +442,8 @@ static const yytype_int8 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint8 yyrline[] =
{
- 0, 82, 82, 83, 87, 88, 92, 101, 102, 106,
- 107, 111, 115, 116, 117, 118, 122, 123
+ 0, 81, 81, 82, 86, 87, 91, 100, 101, 105,
+ 106, 110, 114, 115, 116, 117, 121, 122
};
#endif
@@ -1099,7 +1098,7 @@ yydestruct (yymsg, yytype, yyvaluep, pb, sb)
case 20: /* "value" */
{
- VNamelistRelease((yyvaluep->namelist));
+ VNamelistRelease((yyvaluep->namelist));
};
break;
@@ -1404,13 +1403,13 @@ yyreduce:
case 6:
{
- rc_t rc=sb->write_nvp(sb->self, (yyvsp[(1) - (4)].pb).tokenText, (yyvsp[(1) - (4)].pb).tokenLength, (yyvsp[(3) - (4)].namelist));
- if (rc != 0)
- {
- ReportRc(pb, sb, rc);
- }
- VNamelistRelease((yyvsp[(3) - (4)].namelist));
- }
+ rc_t rc=sb->write_nvp(sb->self, (yyvsp[(1) - (4)].pb).tokenText, (yyvsp[(1) - (4)].pb).tokenLength, (yyvsp[(3) - (4)].namelist));
+ if (rc != 0)
+ {
+ ReportRc(pb, sb, rc);
+ }
+ VNamelistRelease((yyvsp[(3) - (4)].namelist));
+ }
break;
case 12:
@@ -1678,48 +1677,48 @@ void KFG_error(KFGParseBlock* pb, KFGScanBlock* sb, const char* msg)
void ReportRc(KFGParseBlock* pb, KFGScanBlock* sb, rc_t rc)
{
- char buf[1025];
- size_t num_writ;
- RCExplain(rc, buf, 1024, &num_writ);
- buf[1024]=0;
- yyerror(0, sb, buf);
+ char buf[1025];
+ size_t num_writ;
+ RCExplain(rc, buf, 1024, &num_writ);
+ buf[1024]=0;
+ yyerror(0, sb, buf);
}
void AppendName(KFGScanBlock* sb, VNamelist* nl, const KFGParseBlock* pb)
-{ /* pb represents either a kfgSTRING or a kfgESCAPED_STRING with opening and closed quotes clipped */
+{ /* pb represents either a kfgSTRING or a kfgESCAPED_STRING with opening and closed quotes clipped */
rc_t rc;
KToken t;
size_t value_size;
char* buf;
- if (pb->tokenLength == 0)
- {
- return;
- }
-
+ if (pb->tokenLength == 0)
+ {
+ return;
+ }
+
t.id= pb->tokenId == kfgESCAPED_STRING ? eEscapedString : eString;
- StringInit(&t.str, pb->tokenText-1, pb->tokenLength+2, pb->tokenLength+2); /* compensate for clipped quotes in order to use KTokenToString */
+ StringInit(&t.str, pb->tokenText-1, pb->tokenLength+2, (uint32_t) (pb->tokenLength + 2) ); /* compensate for clipped quotes in order to use KTokenToString */
buf=(char*)malloc(t.str.size);
- /* KTokenToString removes opening and closing quotes and handles escapes if present */
- rc = KTokenToString (&t, buf, t.str.size, &value_size);
- if (rc != 0)
- {
- ReportRc(0, sb, rc);
- }
- else
- {
- assert(value_size < t.str.size);
+ /* KTokenToString removes opening and closing quotes and handles escapes if present */
+ rc = KTokenToString (&t, buf, t.str.size, &value_size);
+ if (rc != 0)
+ {
+ ReportRc(0, sb, rc);
+ }
+ else
+ {
+ assert(value_size < t.str.size);
buf[value_size]=0;
- rc = VNamelistAppend(nl, buf);
- if (rc != 0)
- {
- ReportRc(0, sb, rc);
- }
- }
- free(buf);
+ rc = VNamelistAppend(nl, buf);
+ if (rc != 0)
+ {
+ ReportRc(0, sb, rc);
+ }
+ }
+ free(buf);
}
-
+
diff --git a/libs/kfg/config-lex.c b/libs/kfg/config-lex.c
index 74d1c4b..299650f 100644
--- a/libs/kfg/config-lex.c
+++ b/libs/kfg/config-lex.c
@@ -1,5 +1,6 @@
+#define YY_BUF_SIZE 16777216
#define YY_INT_ALIGNED short int
/* A lexical scanner generated by flex */
@@ -666,28 +667,27 @@ static yyconst flex_int32_t yy_rule_can_match_eol[36] =
* ===========================================================================
*
*/
- #include <sysalloc.h>
+ #include "kfg-parse.h"
+ #include <sysalloc.h>
+ #include "config-tokens.h"
- #include "kfg-parse.h"
- #include "config-tokens.h"
-
#define YYSTYPE KFGParseBlock
/* code to populate token with internal values */
#define YY_USER_ACTION \
- yylval -> tokenText = yytext; \
- yylval -> tokenLength = yyleng; \
- yylval -> line_no = yylineno; \
- yylval -> column_no = yycolumn; \
+ yylval -> tokenText = yytext; \
+ yylval -> tokenLength = yyleng; \
+ yylval -> line_no = yylineno; \
+ yylval -> column_no = yycolumn; \
yycolumn += yyleng;
#define REM_LQUOTE \
- ++ yylval -> tokenText; \
- -- yylval -> tokenLength; \
- ++ yylval -> column_no;
-
+ ++ yylval -> tokenText; \
+ -- yylval -> tokenLength; \
+ ++ yylval -> column_no;
+
#define REM_RQUOTE \
- -- yylval -> tokenLength;
+ -- yylval -> tokenLength;
#define YY_NO_UNISTD_H 1
/* re-entrant scanner */
/* bison options */
@@ -847,7 +847,7 @@ static int input (yyscan_t yyscanner );
/* This used to be an fputs(), but since the string might contain NUL's,
* we now use fwrite().
*/
-#define ECHO do { if (fwrite( yytext, yyleng, 1, yyout )) {} } while (0)
+#define ECHO fwrite( yytext, yyleng, 1, yyout )
#endif
/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL,
@@ -858,7 +858,7 @@ static int input (yyscan_t yyscanner );
if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
{ \
int c = '*'; \
- unsigned n; \
+ int n; \
for ( n = 0; n < max_size && \
(c = getc( yyin )) != EOF && c != '\n'; ++n ) \
buf[n] = (char) c; \
@@ -2446,66 +2446,66 @@ void yyfree (void * ptr , yyscan_t yyscanner)
rc_t CC KFGScan_yylex_init(KFGScanBlock* sb, const char *str)
{
- if (yylex_init(&sb->scanner) != 0)
- {
- return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );
- }
- sb->buffer=yy_scan_string(str,sb->scanner);
-
- { /* initialize line/col counters */
- /* yylineno and yycolumn are macros using variable yyg, so we have to prepare it for them */
- struct yyguts_t * yyg = (struct yyguts_t*)sb->scanner;
- yylineno=1;
- yycolumn=1;
- }
-
- return 0;
+ if (yylex_init(&sb->scanner) != 0)
+ {
+ return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );
+ }
+ sb->buffer=yy_scan_string((yyconst char*)str,sb->scanner);
+
+ { /* initialize line/col counters */
+ /* yylineno and yycolumn are macros using variable yyg, so we have to prepare it for them */
+ struct yyguts_t * yyg = (struct yyguts_t*)sb->scanner;
+ yylineno=1;
+ yycolumn=1;
+ }
+
+ return 0;
}
void CC KFGScan_yylex_destroy(KFGScanBlock* sb)
{
- yy_delete_buffer(sb->buffer,sb->scanner);
- if (sb->scanner)
- {
- yylex_destroy(sb->scanner);
- }
- sb->scanner=0;
- sb->buffer=0;
+ yy_delete_buffer(sb->buffer,sb->scanner);
+ if (sb->scanner)
+ {
+ yylex_destroy(sb->scanner);
+ }
+ sb->scanner=0;
+ sb->buffer=0;
}
void CC KFGScan_set_debug(bool on, KFGScanBlock* sb)
{
- yyset_debug(on ? 1 : 0,sb->scanner);
+ yyset_debug(on ? 1 : 0,sb->scanner);
}
int CC KFGScan_yylex(KFGSymbol* sym, KFGScanBlock* sb)
{
- sym->pb.tokenId=yylex(&sym->pb,sb->scanner);
- sb->lastToken= &sym->pb;
-
- switch (sym->pb.tokenId)
- {
- case kfgVAR_REF:
- if (!sb->look_up_var(sb->self, &sym->pb))
- {
- /* undefined variable reference is handled silently, to behave like a shell script */
- /* KFG_error(&sym->pb, sb, "variable not found");*/
- sym->pb.tokenLength = 0;
- }
- sym->pb.tokenId = kfgSTRING;
- break;
- case kfgUNTERM_VAR_REF: /* convert bad var ref into a string */
- KFG_error(&sym->pb, sb, "unterminated variable reference inside a string (use '\\$(' to avoid this message)");
- sym->pb.tokenId = kfgSTRING;
- break;
- case kfgUNTERM_STRING:
- case kfgUNTERM_ESCAPED_STRING:
- KFG_error(&sym->pb, sb, "unterminated string");
- break;
- default:
- break;
- }
-
- return sym->pb.tokenId;
+ sym->pb.tokenId=yylex(&sym->pb,sb->scanner);
+ sb->lastToken= &sym->pb;
+
+ switch (sym->pb.tokenId)
+ {
+ case kfgVAR_REF:
+ if (!sb->look_up_var(sb->self, &sym->pb))
+ {
+ /* undefined variable reference is handled silently, to behave like a shell script */
+ /* KFG_error(&sym->pb, sb, "variable not found");*/
+ sym->pb.tokenLength = 0;
+ }
+ sym->pb.tokenId = kfgSTRING;
+ break;
+ case kfgUNTERM_VAR_REF: /* convert bad var ref into a string */
+ KFG_error(&sym->pb, sb, "unterminated variable reference inside a string (use '\\$(' to avoid this message)");
+ sym->pb.tokenId = kfgSTRING;
+ break;
+ case kfgUNTERM_STRING:
+ case kfgUNTERM_ESCAPED_STRING:
+ KFG_error(&sym->pb, sb, "unterminated string");
+ break;
+ default:
+ break;
+ }
+
+ return sym->pb.tokenId;
}
diff --git a/libs/kfg/config-tokens.h b/libs/kfg/config-tokens.h
index 2090822..81fa170 100644
--- a/libs/kfg/config-tokens.h
+++ b/libs/kfg/config-tokens.h
@@ -59,8 +59,8 @@ typedef union YYSTYPE
{
- KFGParseBlock pb;
- const VNamelist* namelist;
+ KFGParseBlock pb;
+ const VNamelist* namelist;
diff --git a/libs/kfg/config.c b/libs/kfg/config.c
index ae1cf47..faf2380 100644
--- a/libs/kfg/config.c
+++ b/libs/kfg/config.c
@@ -113,6 +113,15 @@ int CC KConfigIncludedSort ( const BSTNode *item, const BSTNode *n )
return strcmp ( a -> path, b -> path );
}
+enum {
+ eInternalFalse = false, /* internal = false: non internal nodes */
+ eInternalTrue = true, /* internal = true ; internal nodes: read-only */
+ eInternalTrueUpdatable
+ /* internal = true ; internal nodes, but can be updated :
+ "kfg/dir", "kfg/name" */
+} EInternal;
+typedef uint32_t TInternal;
+
/*--------------------------------------------------------------------------
* KConfigNode
* node within configuration tree
@@ -145,7 +154,7 @@ struct KConfigNode
KRefcount refcount;
- bool internal;
+ TInternal internal; /* EInternal */
bool read_only;
bool dirty;
};
@@ -805,7 +814,7 @@ rc_t KConfigNodeVOpenNodeUpdateInt ( KConfigNode *self, KConfig *mgr,
if ( rc == 0 )
{
/* check to see if internal */
- if ( self -> internal )
+ if ( self -> internal == eInternalTrue )
rc = RC ( rcKFG, rcNode, rcOpening, rcNode, rcReadonly );
else
{
@@ -969,7 +978,7 @@ LIB_EXPORT rc_t CC KConfigNodeWrite ( KConfigNode *self, const char *buffer, siz
if ( self == NULL )
rc = RC ( rcKFG, rcNode, rcWriting, rcSelf, rcNull );
- else if ( self -> read_only || self -> internal )
+ else if ( self -> read_only || self -> internal == eInternalTrue )
rc = RC ( rcKFG, rcNode, rcWriting, rcSelf, rcReadonly );
else if ( size == 0 )
{
@@ -1133,7 +1142,8 @@ LIB_EXPORT rc_t CC KConfigNodeRenameChild ( KConfigNode *self, const char *from,
static
rc_t
-update_node ( KConfig* self, const char* key, const char* value, bool internal )
+update_node ( KConfig* self, const char* key, const char* value,
+ TInternal internal )
{
KConfigNode * node;
rc_t rc = KConfigVOpenNodeUpdate ( self, &node, key, NULL);
@@ -1144,20 +1154,26 @@ update_node ( KConfig* self, const char* key, const char* value, bool internal )
key, value);*/
rc = KConfigNodeWrite (node, value, string_size(value));
node -> internal = internal;
+ if (self->current_file != NULL && self->current_file->is_magic_file) {
+ if (node->came_from == NULL || !node->came_from->is_magic_file) {
+ node->came_from = self->current_file;
+ }
+ }
KConfigNodeRelease ( node );
}
return rc;
}
static
-rc_t write_nvp(void * self, const char* name, size_t nameLen, VNamelist* values)
+rc_t write_nvp(void* pself, const char* name, size_t nameLen, VNamelist* values)
{ /* concatenate all values from the namelist and put the result into config under the given name */
uint32_t count;
- uint32_t size=0;
- uint32_t concatTo=0;
+ size_t size=0;
+ size_t concatTo=0;
uint32_t i;
char* buf;
+ KConfig *self = (KConfig *)pself;
rc_t rc=VNameListCount(values, &count);
if (rc != 0)
{
@@ -1201,22 +1217,22 @@ rc_t write_nvp(void * self, const char* name, size_t nameLen, VNamelist* values)
/* some old config files may have "dbGaP" in their repository keys misspelled as "dbGap" - fix if seen */
const char* oldGaPprefix = "/repository/user/protected/dbGap-";
size_t size = sizeof("/repository/user/protected/dbGap-") - 1;
- bool needsFix = string_cmp(name, string_measure(name, NULL), oldGaPprefix, size, size) == 0;
+ bool needsFix = string_cmp(name, string_measure(name, NULL), oldGaPprefix, size, (uint32_t)size) == 0;
String tmp;
- StringInit(&tmp, name, nameLen, nameLen);
+ StringInit(&tmp, name, nameLen, (uint32_t)nameLen);
StringCopy((const String**)&nameStr, &tmp);
if (needsFix)
((char*)(nameStr->addr)) [ size - 2 ] = 'P';
- rc = update_node((KConfig *)self, nameStr->addr, buf, false);
+ rc = update_node(self, nameStr->addr, buf, false);
if (needsFix)
{
KConfigNode * node;
rc = KConfigVOpenNodeUpdate ( self, &node, nameStr->addr, NULL);
if (rc == 0)
{ /* we are likely to be initializing, so have to set the dirty flags directly, not through KConfigNodeSetDirty() */
- ((KConfig *)self) -> dirty = true;
+ self -> dirty = true;
node -> dirty = true;
KConfigNodeRelease ( node );
}
@@ -1256,6 +1272,161 @@ void CC report_error(KFGScanBlock* sb, const char* msg)
msg);
}
+#define DISP_RC2(rc, name, msg) (void)((rc == 0) ? 0 : \
+ PLOGERR(klogInt, (klogInt, rc, \
+ "$(name): $(msg)", "name=%s,msg=%s", name, msg)))
+
+static rc_t printIndent(int indent) {
+ rc_t rc = 0;
+
+ int i = 0;
+ for (i = 0; i < indent * 2; ++i) {
+ rc_t rc2 = OUTMSG((" "));
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
+
+ return rc;
+}
+
+static rc_t KConfigNodeReadData(const KConfigNode* self,
+ char* buf, size_t blen, size_t* num_read)
+{
+ rc_t rc = 0;
+ size_t remaining = 0;
+
+ assert(buf && blen && num_read);
+
+ rc = KConfigNodeRead(self, 0, buf, blen, num_read, &remaining);
+
+ assert(remaining == 0); /* TODO overflow check */
+ assert(*num_read <= blen);
+
+ return rc;
+}
+
+static rc_t _printNodeData(const char *name, const char *data, uint32_t dlen) {
+ const char ticket[] = "download-ticket";
+ size_t l = sizeof ticket - 1;
+ if (string_cmp(name, string_measure(name, NULL),
+ ticket, l, (uint32_t)l) == 0)
+ {
+ const char *ellipsis = "";
+ const char replace[] =
+"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ if (dlen > 70) {
+ dlen = 70;
+ ellipsis = "...";
+ }
+ return OUTMSG(("%.*s%s", dlen, replace, ellipsis));
+ }
+ else {
+ return OUTMSG(("%.*s", dlen, data));
+ }
+}
+
+static
+rc_t KConfigNodePrint(const KConfigNode* self,
+ int indent, const char* root, bool debug)
+{
+ rc_t rc = 0;
+ KNamelist* names = NULL;
+ uint32_t count = 0;
+ uint32_t i = 0;
+ char data[4097] = "";
+ size_t num_data = 0;
+ assert(self);
+
+ printIndent(indent);
+ OUTMSG(("<%s>", root));
+
+ if (rc == 0) {
+ rc_t rc = KConfigNodeReadData(self, data, sizeof data, &num_data);
+ DISP_RC2(rc, "KConfigNodeReadData()", root);
+ if (rc == 0 && num_data > 0) {
+ _printNodeData(root, data, num_data);
+ }
+ if (debug && self->came_from) {
+ OUTMSG(("<came_from is_magic_file=\"%s\"/>",
+ self->came_from->is_magic_file ? "true" : "false"));
+ }
+ }
+
+ if (rc == 0) {
+ rc = KConfigNodeListChild(self, &names);
+ DISP_RC2(rc, "KConfigNodeListChild()", root);
+ }
+ if (rc == 0) {
+ rc = KNamelistCount(names, &count);
+ DISP_RC2(rc, "KNamelistCount()", root);
+ }
+
+ if (rc == 0) {
+ if (count > 0) {
+ OUTMSG(("\n"));
+ }
+ for (i = 0; i < count; ++i) {
+ const char* name = NULL;
+ const KConfigNode* node = NULL;
+ if (rc == 0) {
+ rc = KNamelistGet(names, i, &name);
+ DISP_RC2(rc, "KNamelistGet()", root);
+ }
+ if (rc == 0) {
+ rc = KConfigNodeOpenNodeRead(self, &node, name);
+ DISP_RC2(rc, "KConfigNodeOpenNodeRead()", name);
+ }
+ if (rc == 0) {
+ KConfigNodePrint(node, indent + 1, name, debug);
+ }
+ KConfigNodeRelease(node);
+ }
+ }
+
+ if (count > 0) {
+ printIndent(indent);
+ }
+ OUTMSG(("</%s>\n", root));
+
+ KNamelistRelease(names);
+
+ return rc;
+}
+
+static rc_t CC KConfigPrintImpl(const KConfig* self, int indent,
+ const char *root, bool debug)
+{
+ rc_t rc = 0;
+
+ if (root == NULL) {
+ root = "Config";
+ }
+
+ if (self == NULL) {
+ OUTMSG(("<%s>", root));
+ OUTMSG(("KConfigPrint(const KConfig* self = NULL)\n"));
+ OUTMSG(("</%s>\n", root));
+ }
+ else {
+ const KConfigNode* node = NULL;
+ if (rc == 0) {
+ rc = KConfigOpenNodeRead(self, &node, "/");
+ DISP_RC2(rc, "KConfigOpenNodeRead()", "/");
+ }
+ if (rc == 0) {
+ KConfigNodePrint(node, indent, root, debug);
+ }
+ KConfigNodeRelease(node);
+ }
+
+ return rc;
+}
+
+LIB_EXPORT rc_t CC KConfigPrintDebug(const KConfig* self, const char *path) {
+ return KConfigPrintImpl(self, 0, path, true);
+}
+
/*
* Set up the parameter block and start parsing lines
*/
@@ -1266,6 +1437,8 @@ rc_t parse_file ( KConfig * self, const char* path, const char * src )
KFGScanBlock sb;
rc_t rc;
+/* KConfigPrintDebug(self, NULL); */
+
pb.tokenLength = 0;
pb.line_no = 0;
pb.column_no = 0;
@@ -1283,6 +1456,8 @@ rc_t parse_file ( KConfig * self, const char* path, const char * src )
KFGScan_yylex_destroy(&sb);
}
+/* KConfigPrintDebug(self, path); */
+
return rc;
}
@@ -1305,9 +1480,9 @@ LIB_EXPORT rc_t CC KConfigLoadFile ( KConfig * self, const char * path, const KF
/* populate file-specific predefined nodes */
#define UPDATE_NODES(dir, file) \
- rc = update_node(self, "kfg/dir", dir, false ); \
+ rc = update_node(self, "kfg/dir", dir, eInternalTrueUpdatable ); \
if (rc == 0) \
- rc = update_node(self, "kfg/name", file, false )
+ rc = update_node(self, "kfg/name", file, eInternalTrueUpdatable )
if ( path == NULL || path [ 0 ] == 0)
{
@@ -1340,8 +1515,8 @@ LIB_EXPORT rc_t CC KConfigLoadFile ( KConfig * self, const char * path, const KF
}
else
{
- update_node(self, "kfg/dir", "", false);
- update_node(self, "kfg/name", "", false);
+ update_node(self, "kfg/dir", "", eInternalTrueUpdatable);
+ update_node(self, "kfg/name", "", eInternalTrueUpdatable);
}
}
#undef UPDATE_NODES
@@ -1369,8 +1544,8 @@ LIB_EXPORT rc_t CC KConfigLoadFile ( KConfig * self, const char * path, const KF
buf[size]=0;
/* Parse the path to populate: */
- /* update_node(self, "kfg/dir", dir, false);*/
- /* update_node(self, "kfg/name", name, false);*/
+ /* update_node(self, "kfg/dir", dir, eInternalTrueUpdatable);*/
+ /* update_node(self, "kfg/name", name, eInternalTrueUpdatable);*/
/* parse config file */
rc = parse_file ( self, path, buf );
@@ -1542,7 +1717,9 @@ bool CC WriteDirtyNode ( BSTNode *n, void *data )
KConfigNode *self = ( KConfigNode * ) n;
PrintBuff *pb = data;
- if ( self -> dirty || ( self -> came_from != NULL && self -> came_from -> is_magic_file ) )
+ if ( self -> dirty
+ || ( self -> came_from != NULL && self -> came_from -> is_magic_file
+ && ! self -> internal ) )
{
if ( KConfigNodePrintPath ( self, pb ) )
return true;
@@ -1911,6 +2088,65 @@ bool load_from_std_location ( KConfig *self, const KDirectory *dir )
}
static
+rc_t find_home_directory ( KDyld *dyld, const KDirectory **dir )
+{
+ static const KDirectory * cached_dir = NULL;
+ static rc_t cached_rc = 0;
+ rc_t rc;
+
+ if ( cached_dir != NULL )
+ {
+ rc = KDirectoryAddRef ( cached_dir );
+ if ( rc == 0 ) {
+ * dir = cached_dir;
+ }
+ return rc;
+ }
+ else if ( cached_rc != 0 )
+ {
+ return cached_rc;
+ }
+
+ rc = KDyldHomeDirectory ( dyld, dir, ( fptr_t ) KConfigMake );
+
+ if ( rc != 0
+ || (KDirectoryPathType ( * dir, "ncbi" ) & ~kptAlias) != kptDir )
+ {
+ KDylib * lib;
+ if ( rc == 0 )
+ {
+ /* Nominally succeeded, but got a useless directory
+ * (for a statically linked executable?); try again. */
+ KDirectoryRelease ( * dir );
+ }
+ rc = KDyldLoadLib ( dyld, & lib, LPFX "kfg-beacon" SHLX );
+ if ( rc == 0 )
+ {
+ KSymAddr * sym;
+ if ( ( rc = KDylibSymbol ( lib, & sym, "KConfigBeacon" ) ) == 0 )
+ {
+ fptr_t fp;
+ KSymAddrAsFunc ( sym, & fp );
+ rc = KDyldHomeDirectory ( dyld, dir, fp );
+ KSymAddrRelease ( sym );
+ }
+ KDylibRelease ( lib );
+ }
+ }
+
+ if ( rc == 0 && KDirectoryAddRef ( * dir ) == 0 )
+ {
+ cached_dir = * dir;
+ }
+ else
+ {
+ cached_rc = rc;
+ }
+
+ return rc;
+}
+
+static
rc_t load_from_fs_location ( KConfig *self )
{
KDyld *dyld;
@@ -1918,7 +2154,7 @@ rc_t load_from_fs_location ( KConfig *self )
if ( rc == 0 )
{
const KDirectory *dir;
- rc = KDyldHomeDirectory ( dyld, & dir, ( fptr_t ) KConfigMake );
+ rc = find_home_directory ( dyld, & dir );
if ( rc == 0 )
{
char resolved[PATH_MAX + 1];
@@ -2081,7 +2317,7 @@ void add_predefined_nodes ( KConfig * self, const char *appname )
rc_t rc = KDyldMake ( & dyld );
if ( rc == 0 )
{
- rc = KDyldHomeDirectory ( dyld, & dir, ( fptr_t ) KConfigMake );
+ rc = find_home_directory ( dyld, & dir );
if ( rc == 0 )
{
KDirectoryResolvePath ( dir, true, buf, sizeof buf, "." );
@@ -2755,7 +2991,7 @@ LIB_EXPORT rc_t CC KConfigNodeReadString ( const KConfigNode *self, String** res
/* TBD - this is broken for non-ascii strings
much better to be WITHIN the config.c implementation
and reach into the node value directly! */
- StringInit ( value, (char*)( value + 1 ), to_read, to_read + 1 );
+ StringInit ( value, (char*)( value + 1 ), to_read, (uint32_t)to_read + 1 );
rc = ReadNodeValueFixed(self, (char*)value->addr, to_read + 1);
if ( rc == 0 )
*result = value;
@@ -2816,128 +3052,9 @@ LIB_EXPORT rc_t CC KConfigReadString ( const KConfig* self, const char* path, st
}
#define DISP_RC(rc, msg) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, msg))
-#define DISP_RC2(rc, name, msg) (void)((rc == 0) ? 0 : \
- PLOGERR(klogInt, (klogInt, rc, \
- "$(name): $(msg)", "name=%s,msg=%s", name, msg)))
-
-static rc_t KConfigNodeReadData(const KConfigNode* self,
- char* buf, size_t blen, size_t* num_read)
-{
- rc_t rc = 0;
- size_t remaining = 0;
-
- assert(buf && blen && num_read);
-
- rc = KConfigNodeRead(self, 0, buf, blen, num_read, &remaining);
-
- assert(remaining == 0); /* TODO overflow check */
- assert(*num_read <= blen);
-
- return rc;
-}
-
-static rc_t printIndent(int indent) {
- rc_t rc = 0;
-
- int i = 0;
- for (i = 0; i < indent * 2; ++i) {
- rc_t rc2 = OUTMSG((" "));
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
- }
- }
-
- return rc;
-}
-
-static
-rc_t KConfigNodePrint(const KConfigNode* self,
- int indent, const char* root)
-{
- rc_t rc = 0;
- KNamelist* names = NULL;
- uint32_t count = 0;
- uint32_t i = 0;
- char data[512] = "";
- size_t num_data = 0;
- assert(self);
-
- printIndent(indent);
- OUTMSG(("<%s>", root));
-
- if (rc == 0) {
- rc_t rc = KConfigNodeReadData(self, data, sizeof data, &num_data);
- DISP_RC2(rc, "KConfigNodeReadData()", root);
- if (rc == 0 && num_data > 0) {
- OUTMSG(("%.*s", (int)num_data, data));
- }
- }
-
- if (rc == 0) {
- rc = KConfigNodeListChild(self, &names);
- DISP_RC2(rc, "KConfigNodeListChild()", root);
- }
- if (rc == 0) {
- rc = KNamelistCount(names, &count);
- DISP_RC2(rc, "KNamelistCount()", root);
- }
-
-
- if (rc == 0) {
- if (count > 0) {
- OUTMSG(("\n"));
- }
- for (i = 0; i < count; ++i) {
- const char* name = NULL;
- const KConfigNode* node = NULL;
- if (rc == 0) {
- rc = KNamelistGet(names, i, &name);
- DISP_RC2(rc, "KNamelistGet()", root);
- }
- if (rc == 0) {
- rc = KConfigNodeOpenNodeRead(self, &node, name);
- DISP_RC2(rc, "KConfigNodeOpenNodeRead()", name);
- }
- if (rc == 0) {
- KConfigNodePrint(node, indent + 1, name);
- }
- KConfigNodeRelease(node);
- }
- }
-
- if (count > 0) {
- printIndent(indent);
- }
- OUTMSG(("</%s>\n", root));
-
- KNamelistRelease(names);
-
- return rc;
-}
LIB_EXPORT rc_t CC KConfigPrint(const KConfig* self, int indent) {
- rc_t rc = 0;
-
- const char root[] = "Config";
-
- if (self == NULL) {
- OUTMSG(("<%s>", root));
- OUTMSG(("KConfigPrint(const KConfig* self = NULL)\n"));
- OUTMSG(("</%s>\n", root));
- }
- else {
- const KConfigNode* node = NULL;
- if (rc == 0) {
- rc = KConfigOpenNodeRead(self, &node, "/");
- DISP_RC2(rc, "KConfigOpenNodeRead()", "/");
- }
- if (rc == 0) {
- KConfigNodePrint(node, indent, "Config");
- }
- KConfigNodeRelease(node);
- }
-
- return rc;
+ return KConfigPrintImpl(self, indent, NULL, false);
}
LIB_EXPORT void CC KConfigDisableUserSettings ( void )
@@ -2961,15 +3078,17 @@ rc_t open_file ( const KFile **f, const char *path )
}
static
-rc_t decode_file ( KDataBuffer *mem, const KFile *orig )
+rc_t decode_ncbi_gap ( KDataBuffer *mem, const KFile *orig )
{
char hdr [ 8 ];
size_t num_read;
rc_t rc = KFileReadAll ( orig, 0, hdr, sizeof hdr, & num_read );
if ( rc == 0 && num_read == sizeof hdr )
{
- if ( memcmp ( hdr, "ncbi_gap", sizeof hdr ) == 0 )
- {
+ if (memcmp(hdr, "ncbi_gap", sizeof hdr) != 0) {
+ rc = RC(rcKFG, rcFile, rcReading, rcFile, rcWrongType);
+ }
+ else {
uint64_t eof;
rc = KFileSize ( orig, & eof );
if ( rc == 0 )
@@ -3050,7 +3169,7 @@ rc_t _KConfigNncToKGapConfig(const KConfig *self, char *text, KGapConfig *kgc)
{
const char version[] = "version ";
size_t l = sizeof version - 1;
- if (string_cmp(version, l, text, len, l) != 0) {
+ if (string_cmp(version, l, text, len, (uint32_t)l) != 0) {
return RC(rcKFG, rcMgr, rcUpdating, rcFormat, rcUnrecognized);
}
text += l;
@@ -3060,7 +3179,7 @@ rc_t _KConfigNncToKGapConfig(const KConfig *self, char *text, KGapConfig *kgc)
{
const char version[] = "1.0";
size_t l = sizeof version - 1;
- if (string_cmp(version, l, text, l, l) != 0) {
+ if (string_cmp(version, l, text, l, (uint32_t)l) != 0) {
return RC(rcKFG, rcMgr, rcUpdating, rcFormat, rcUnsupported);
}
text += l;
@@ -3292,6 +3411,9 @@ static rc_t _KConfigDBGapRepositoryNodes(KConfig *self,
}
if (rc == 0) {
+ rc = _KConfigNodeUpdateChild(rep, "apps/file/volumes/flat", "files");
+ }
+ if (rc == 0) {
rc = _KConfigNodeUpdateChild(rep, "apps/sra/volumes/sraFlat", "sra");
}
@@ -3400,7 +3522,7 @@ LIB_EXPORT rc_t CC KConfigImportNgc(KConfig *self,
KDataBuffer mem;
memset ( & mem, 0, sizeof mem );
- rc = decode_file ( & mem, orig );
+ rc = decode_ncbi_gap ( & mem, orig );
KFileRelease ( orig );
orig = NULL;
diff --git a/libs/kfg/kart.c b/libs/kfg/kart.c
index 976570c..4bffe17 100644
--- a/libs/kfg/kart.c
+++ b/libs/kfg/kart.c
@@ -226,6 +226,8 @@ struct Kart {
const char *text;
uint64_t len;
+
+ uint16_t itemsProcessed;
};
static void KartWhack(Kart *self) {
@@ -268,6 +270,22 @@ LIB_EXPORT rc_t CC KartRelease(const Kart *self) {
return 0;
}
+LIB_EXPORT rc_t CC KartItemsProcessed(const Kart *self, uint16_t *number) {
+ if (number == NULL) {
+ return RC(rcKFG, rcFile, rcLoading, rcParam, rcNull);
+ }
+
+ *number = 0;
+
+ if (self == NULL) {
+ return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
+ }
+
+ *number = self->itemsProcessed;
+
+ return 0;
+}
+
static rc_t KartItemInitFromKartRow(const Kart *self, const KartItem **item,
const char *line, size_t len)
{
@@ -285,7 +303,8 @@ static rc_t KartItemInitFromKartRow(const Kart *self, const KartItem **item,
const char *p = string_chr(line, len, '|');
if (p == NULL) {
if (i != 4) {
- return RC(rcKFG, rcFile, rcParsing, rcFile, rcInsufficient);
+ rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcInsufficient);
+ break;
}
l = len;
}
@@ -309,13 +328,13 @@ static rc_t KartItemInitFromKartRow(const Kart *self, const KartItem **item,
next = &obj->itemDesc;
break;
default:
- return RC(rcKFG, rcFile, rcParsing, rcFile, rcExcessive);
+ rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcExcessive);
break;
}
assert(next);
- StringInit(next, line, l, l);
+ StringInit(next, line, l, (uint32_t)l);
if (l > len) {
- return RC(rcKFG, rcFile, rcParsing, rcFile, rcInvalid);
+ rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcInvalid);
}
if (len == l) {
break;
@@ -324,22 +343,112 @@ static rc_t KartItemInitFromKartRow(const Kart *self, const KartItem **item,
line += l;
len -= l;
}
- rc = KartAddRef(self);
if (rc == 0) {
+ rc = KartAddRef(self);
+ }
+ if (rc == 0) {
+ ++((Kart*)self)->itemsProcessed;
obj->dad = self;
*item = obj;
}
+ else {
+ free(obj);
+ obj = NULL;
+ }
return rc;
}
LIB_EXPORT rc_t CC KartPrint(const Kart *self) {
- uint32_t len = self->mem.elem_count;
+ uint32_t len = 0;
+
if (self == NULL) {
return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
}
+
+ len = (uint32_t)self->mem.elem_count;;
return OUTMSG(("%.*s", len, self->mem.base));
}
+LIB_EXPORT rc_t CC KartPrintNumbered(const Kart *self) {
+ rc_t rc = 0;
+ rc_t rc2 = 0;
+ const char *start = NULL;
+ int32_t remaining = 0;
+ int32_t len = 0;
+ uint32_t i = 0;
+ const char *next = NULL;
+ bool done = false;
+
+ if (self == NULL) {
+ return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
+ }
+
+ remaining = (uint32_t)self->mem.elem_count;
+ start = self->mem.base;
+
+ {
+ const char version[] = "version ";
+ size_t l = sizeof version - 1;
+ if (string_cmp(version, l, start, remaining, (uint32_t)l) != 0) {
+ return RC(rcKFG, rcMgr, rcAccessing, rcFormat, rcUnrecognized);
+ }
+ }
+
+ next = string_chr(start, remaining, '\n');
+ if (next == NULL) {
+ len = remaining;
+ }
+ else {
+ ++next;
+ len = next - start;
+ }
+ remaining -= len;
+ rc2 = OUTMSG(("%.*s", len, start));
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+ start = next;
+
+ rc2 = OUTMSG(("row\tproj-id|item-id|accession|name|item-desc\n"));
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+
+ for (i = 1; remaining > 0; ++i) {
+ if (*start == '$') {
+ const char end[] = "$end";
+ size_t l = sizeof end - 1;
+ if (string_cmp(end, l, start, remaining, (uint32_t)l) != 0) {
+ return RC(rcKFG, rcMgr, rcAccessing, rcFormat, rcUnrecognized);
+ }
+ else {
+ done = true;
+ }
+ }
+ next = string_chr(start, remaining, '\n');
+ if (next == NULL) {
+ len = remaining;
+ }
+ else {
+ ++next;
+ len = next - start;
+ }
+ remaining -= len;
+ if (done) {
+ rc2 = OUTMSG(("%.*s", len, start));
+ }
+ else {
+ rc2 = OUTMSG(("%d\t%.*s", i, len, start));
+ }
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+ start = next;
+ }
+
+ return rc;
+}
+
LIB_EXPORT rc_t CC KartMakeNextItem(Kart *self, const KartItem **item) {
size_t len = 0;
const char *line = NULL;
@@ -371,7 +480,7 @@ LIB_EXPORT rc_t CC KartMakeNextItem(Kart *self, const KartItem **item) {
--len;
}
- if (self->len >= next - self->text + 1) {
+ if (self->len >= (uint64_t) (next - self->text + 1) ){
self->len -= next - self->text + 1;
}
else {
@@ -383,9 +492,7 @@ LIB_EXPORT rc_t CC KartMakeNextItem(Kart *self, const KartItem **item) {
{
const char end[] = "$end";
- if (string_cmp(line, string_size(line), end,
- sizeof end - 1, sizeof end - 1) == 0)
- {
+ if (string_cmp(line, len, end, sizeof end - 1, sizeof end - 1) == 0) {
return 0;
}
}
@@ -462,7 +569,7 @@ static rc_t KartProcessHeader(Kart *self) {
{
const char version[] = "version ";
size_t l = sizeof version - 1;
- if (string_cmp(version, l, self->text, self->len, l) != 0) {
+ if (string_cmp(version, l, self->text, self->len, (uint32_t)l) != 0) {
return RC(rcKFG, rcMgr, rcUpdating, rcFormat, rcUnrecognized);
}
@@ -473,7 +580,7 @@ static rc_t KartProcessHeader(Kart *self) {
{
const char version[] = "1.0";
size_t l = sizeof version - 1;
- if (string_cmp(version, l, self->text, l, l) != 0) {
+ if (string_cmp(version, l, self->text, l, (uint32_t)l) != 0) {
return RC(rcKFG, rcMgr, rcUpdating, rcFormat, rcUnsupported);
}
diff --git a/libs/kfg/keystore.c b/libs/kfg/keystore.c
index abf2384..fe67b90 100644
--- a/libs/kfg/keystore.c
+++ b/libs/kfg/keystore.c
@@ -35,6 +35,7 @@
#include <kfg/config.h>
#include <kfg/repository.h>
+#include <kfg/kfg-priv.h>
#include <kfs/directory.h>
#include <kfs/file.h>
@@ -48,7 +49,6 @@
#include <string.h>
#define PASSWORD_MAX_SIZE 4096
-#define KFG_KRYPTO_PWFILE "krypto/pwfile"
#define MAX_PATH_SIZE 4096
static char defaultBindingsFile[MAX_PATH_SIZE];
@@ -363,7 +363,7 @@ static rc_t KEncryptionKeyMakeInt(const char* value, KEncryptionKey** self)
}
string_copy(data, size + 1, value, size);
- StringInit( & ret -> value, data, size, size ); /* do not include the 0-terminator */
+ StringInit( & ret -> value, data, size, (uint32_t)size ); /* do not include the 0-terminator */
KRefcountInit ( & ret -> refcount, 1, "KEncryptionKey", "init", "" );
@@ -494,7 +494,7 @@ static rc_t LocateObjectId(const KFile* file, const char* key, char* value, size
break;
}
/* check the key */
- if (string_cmp(key, key_size, buf + start, key_size, num_read - start) == 0)
+ if (string_cmp(key, key_size, buf + start, key_size, (uint32_t) ( num_read - start )) == 0)
{
if (buf [ start + key_size ] == '=')
{
@@ -611,7 +611,7 @@ static rc_t LocateObject(const KFile* file, const char* value, const size_t valu
}
/* compare the value */
- if (string_cmp(value, value_size, buf + value_start, start - value_start, start - value_start) == 0)
+ if (string_cmp(value, value_size, buf + value_start, start - value_start, (uint32_t) ( start - value_start )) == 0)
{
*key_read = key_end - key_start;
if (string_copy(key, key_size, buf + key_start, *key_read) != *key_read)
@@ -708,7 +708,7 @@ LIB_EXPORT rc_t CC KKeyStoreRegisterObject(struct KKeyStore* self, uint32_t oid,
if (rc == 0)
{ /* see if already registered */
if (LocateObjectId(file, oidString, oldName, sizeof(oldName), &num_read) == 0 &&
- string_cmp(oldName, num_read, newName->addr, newName->size, newName->size) != 0)
+ string_cmp(oldName, num_read, newName->addr, newName->size, (uint32_t)newName->size) != 0)
rc = RC ( rcVFS, rcMgr, rcRegistering, rcId, rcExists );
rc2 = KFileRelease(file);
if (rc == 0)
@@ -730,7 +730,7 @@ LIB_EXPORT rc_t CC KKeyStoreRegisterObject(struct KKeyStore* self, uint32_t oid,
{
if (LocateObjectId(lockedFile, oidString, oldName, sizeof(oldName), &num_read) == 0)
{ /* somebody inserted the binding before we locked - make sure their binding was the same */
- if (string_cmp(oldName, num_read, newName->addr, newName->size, newName->size) != 0)
+ if (string_cmp(oldName, num_read, newName->addr, newName->size, (uint32_t)newName->size) != 0)
rc = RC ( rcVFS, rcMgr, rcRegistering, rcId, rcExists );
}
else
@@ -816,7 +816,7 @@ LIB_EXPORT rc_t CC KKeyStoreGetObjectName(const struct KKeyStore* self, uint32_t
if (res != NULL)
{
string_copy((char*)res + sizeof(String), num_read, name, num_read);
- StringInit(res, (char*)res + sizeof(String), num_read, num_read);
+ StringInit(res, (char*)res + sizeof(String), num_read, (uint32_t)num_read);
*result = res;
}
else
diff --git a/libs/kfg/libkfg.vers b/libs/kfg/libkfg.vers
deleted file mode 100644
index a970716..0000000
--- a/libs/kfg/libkfg.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.15
diff --git a/libs/kfg/report-kfg.c b/libs/kfg/report-kfg.c
index a0e45e0..5febcc6 100644
--- a/libs/kfg/report-kfg.c
+++ b/libs/kfg/report-kfg.c
@@ -98,7 +98,7 @@ static rc_t ReportKfgFiles(const ReportFuncs *f,
rc, "KNamelistCount", "origin", "KConfigListIncluded");
}
else {
- int i = 0;
+ uint32_t i = 0;
reportOpen(indent, "Files", 1, "count", 'd', count);
for (i = 0; i < count && rc == 0; ++i) {
const char* name = NULL;
@@ -180,13 +180,14 @@ rc_t ReportConfigNodeChildren(const ReportFuncs *f, int indent,
}
else {
if (count) {
- int i = 0;
+ uint32_t i = 0;
+ int j;
OUTMSG(("\n"));
for (i = 0; i < count && rc == 0; ++i) {
rc = ReportChildNode
(f, indent + 1, names, node, nodeName, i);
}
- for (i = 0; i < indent; ++i)
+ for (j = 0; j < indent; ++j)
{ OUTMSG((" ")); }
}
}
@@ -379,7 +380,7 @@ rc_t ReportKrypto(const ReportFuncs *f, int indent, const KConfig* cfg)
{
rc_t rc = 0;
- const char root[] = "krypto/pwfile";
+ const char root[] = KFG_KRYPTO_PWFILE;
const char name[] = "krypto";
const KConfigNode* node = NULL;
diff --git a/libs/kfg/repository.c b/libs/kfg/repository.c
index 278c7ac..cc2fedf 100644
--- a/libs/kfg/repository.c
+++ b/libs/kfg/repository.c
@@ -1084,9 +1084,13 @@ LIB_EXPORT rc_t CC KRepositoryMgrCurrentProtectedRepository ( const KRepositoryM
}
/* get its canonical path */
- rc = KSysDirRealPath ( sysDir, resolved, path_size, root );
- if ( rc != 0 )
- break;
+ rc2 = KSysDirRealPath ( sysDir,
+ resolved, path_size, root );
+ if ( rc2 != 0 ) {
+ /* VDB-1096:
+ Invalid cannot get repository root? Ignore and continue */
+ continue;
+ }
/* we know the current directory's canonical path size
and we know the repository's canonical path size.
diff --git a/libs/kfs/bzip.c b/libs/kfs/bzip.c
index 4fe4362..dcfc4be 100644
--- a/libs/kfs/bzip.c
+++ b/libs/kfs/bzip.c
@@ -231,7 +231,7 @@ rc_t KBZipFileReadInt (KBZipFile * self, void * buffer, size_t bsize, size_t * p
BZIP_DEBUG (("%s: loop start tot_read %zu\n", __func__, tot_read));
strm->next_out = this_out = (char*)buffer + tot_read;
- strm->avail_out = bsize - tot_read;
+ strm->avail_out = (unsigned int) (bsize - tot_read);
BZIP_DEBUG(("%s: call Decompress\n", __func__));
@@ -348,7 +348,7 @@ rc_t KBZipFileReadInt (KBZipFile * self, void * buffer, size_t bsize, size_t * p
rc = RC (rcFS, rcFile, rcReading, rcData, rcInsufficient);
goto done;
}
- strm->avail_in = src_read;
+ strm->avail_in = (unsigned int) src_read;
self->filePosition += src_read;
strm->next_in = self->buff;
@@ -620,7 +620,7 @@ rc_t CC KBZipFileWrite (struct KBZipFile *self,
strm = &self->strm;
strm->next_in = (char*) buffer;
- strm->avail_in = bsize;
+ strm->avail_in = (unsigned int) bsize;
rc = KBZipFileWriteInt (self, BZ_RUN, &num_writ);
if (rc == 0)
diff --git a/libs/kfs/cacheteefile.c b/libs/kfs/cacheteefile.c
index 791e7ca..0139940 100644
--- a/libs/kfs/cacheteefile.c
+++ b/libs/kfs/cacheteefile.c
@@ -1615,7 +1615,7 @@ LIB_EXPORT rc_t CC GetCacheCompleteness( const struct KFile * self, float * perc
}
if ( in_cache > 0 && block_count > 0 )
{
- float res = in_cache;
+ float res = (float) in_cache;
res *= 100;
res /= block_count;
* percent = res;
diff --git a/libs/kfs/gzip.c b/libs/kfs/gzip.c
index ebf088f..b8bd6bb 100644
--- a/libs/kfs/gzip.c
+++ b/libs/kfs/gzip.c
@@ -254,7 +254,7 @@ static rc_t z_read ( KGZipFile * self, void * buffer, size_t bsize, size_t * _nu
int zret;
strm->next_out = (uint8_t*)buffer + num_read;
- strm->avail_out = bleft;
+ strm->avail_out = (uInt) bleft;
GZIP_DEBUG(("%s: call inflate\n",__func__));
@@ -313,7 +313,7 @@ static rc_t z_read ( KGZipFile * self, void * buffer, size_t bsize, size_t * _nu
if (rc)
break;
- strm->avail_in = src_read;
+ strm->avail_in = (uInt) src_read;
self->filePosition += src_read;
strm->next_in = self->buff;
if (src_read)
@@ -523,7 +523,7 @@ static rc_t CC KGZipFile_OutWrite( struct KGZipFile *self,
strm = &self->strm;
strm->next_in = (Bytef*) buffer;
- strm->avail_in = bsize;
+ strm->avail_in = (uInt) bsize;
rc = 0;
ret = s_GzipAndWrite( self, Z_NO_FLUSH, num_writ, &rc );
diff --git a/libs/kfs/libkdl.vers b/libs/kfs/libkdl.vers
deleted file mode 100644
index 7dea76e..0000000
--- a/libs/kfs/libkdl.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.1
diff --git a/libs/kfs/libkff.vers b/libs/kfs/libkff.vers
deleted file mode 100644
index 21e8796..0000000
--- a/libs/kfs/libkff.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.3
diff --git a/libs/kfs/libkfs.vers b/libs/kfs/libkfs.vers
deleted file mode 100644
index 1892b92..0000000
--- a/libs/kfs/libkfs.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.3.2
diff --git a/libs/kfs/quickmount.c b/libs/kfs/quickmount.c
index 83e2297..4a27909 100644
--- a/libs/kfs/quickmount.c
+++ b/libs/kfs/quickmount.c
@@ -179,7 +179,7 @@ rc_t KQuickMountDirRelativePath (const KQuickMountDir *self, enum RCContext ctx,
psize = strlen ( p );
/* open up space if needed */
- if ( p - path < bsize )
+ if ( (size_t) (p - path) < bsize )
{
/* prevent overflow */
if ( bsize + psize >= path_max )
@@ -192,7 +192,7 @@ rc_t KQuickMountDirRelativePath (const KQuickMountDir *self, enum RCContext ctx,
memcpy ( & path [ bsize ], "../", 3 );
/* close gap */
- if ( p - path > bsize )
+ if ( (size_t) (p - path) > bsize )
memmove ( & path [ bsize ], p, strlen ( p ) + 1 );
return 0;
@@ -432,7 +432,7 @@ static rc_t CC KQuickMountDirResolvePath (const KQuickMountDir *self,
rc_t rc = KQuickMountDirVMakePath ( self, rcResolving, true, full, sizeof full, path, args );
if ( rc == 0 )
{
- uint32_t path_size = strlen ( full );
+ size_t path_size = strlen ( full );
if ( absolute )
{
diff --git a/libs/kfs/tocentry.c b/libs/kfs/tocentry.c
index b2de16b..fed3bfb 100644
--- a/libs/kfs/tocentry.c
+++ b/libs/kfs/tocentry.c
@@ -708,7 +708,7 @@ rc_t KTocEntryGetFileLocator ( const KTocEntry * self, uint64_t * locator )
if (self->u.chunked_file.num_chunks > 0)
{
uint64_t loc;
- int ix;
+ uint32_t ix;
*locator = self->u.chunked_file.chunks[0].source_position;
for (ix = 1; ix < self->u.chunked_file.num_chunks; ++ix)
diff --git a/libs/kfs/win/lnk_tools.c b/libs/kfs/win/lnk_tools.c
index 853d441..82a565f 100644
--- a/libs/kfs/win/lnk_tools.c
+++ b/libs/kfs/win/lnk_tools.c
@@ -140,7 +140,7 @@ static bool lnk_file_exists( const wchar_t *path )
}
else
return false;
-}
+}
static unsigned char * lnk_file_read( HANDLE hfile, const size_t n_bytes, DWORD * read )
@@ -152,7 +152,7 @@ static unsigned char * lnk_file_read( HANDLE hfile, const size_t n_bytes, DWORD
if ( res != NULL )
{
DWORD dwBytesRead;
- if ( FALSE == ReadFile( hfile, res, n_bytes, &dwBytesRead, NULL ) )
+ if ( FALSE == ReadFile( hfile, res, ( DWORD ) n_bytes, &dwBytesRead, NULL ) )
{
free( res );
res = NULL;
diff --git a/libs/kfs/win/sysdir-priv.h b/libs/kfs/win/sysdir-priv.h
index 1ae8cc7..7bfe65a 100644
--- a/libs/kfs/win/sysdir-priv.h
+++ b/libs/kfs/win/sysdir-priv.h
@@ -78,13 +78,6 @@ rc_t KSysDirMakePath ( const KSysDir* self, enum RCContext ctx, bool canon,
#endif
-/* OSPath
- * returns a real OS path
- */
-rc_t KSysDirOSPath ( const KSysDir *self,
- wchar_t *real, size_t bsize, const char *path, va_list args );
-
-
#ifdef __cplusplus
}
#endif
diff --git a/libs/kfs/win/sysdir.c b/libs/kfs/win/sysdir.c
index 2aa89cb..159c54c 100644
--- a/libs/kfs/win/sysdir.c
+++ b/libs/kfs/win/sysdir.c
@@ -76,7 +76,6 @@ static
int utf16_utf32 ( uint32_t *dst, const wchar_t *begin, const wchar_t *end )
{
uint32_t ch;
- const wchar_t *src, *stop;
if ( dst == NULL || begin == NULL || end == NULL )
return -1;
@@ -332,7 +331,7 @@ int KSysDirListingSort ( const void *a, const void *b )
/* close enough for max chars? */
M = (A>B) ? A : B;
- return wstrcase_cmp (a, A, b, B, M);
+ return wstrcase_cmp (a, A, b, B, ( uint32_t ) M);
}
static
@@ -1726,7 +1725,6 @@ static __inline
rc_t get_attributes ( const wchar_t * wpath, uint32_t * access, KTime_t * date )
{
WIN32_FIND_DATA fd;
- DWORD error;
rc_t rc;
if ( FindFirstFile ( wpath, &fd ))
@@ -1997,7 +1995,7 @@ rc_t KSysDirCreateParents ( const KSysDir *self, wchar_t *path, uint32_t access,
{
#if ! OLD_CREATE_PARENTS
rc_t rc;
- uint32_t len;
+ size_t len;
wchar_t *p, *par = path;
/* if directory is chroot'd, skip past root and slash */
diff --git a/libs/kfs/win/sysdll.c b/libs/kfs/win/sysdll.c
index 3ccf125..b439843 100644
--- a/libs/kfs/win/sysdll.c
+++ b/libs/kfs/win/sysdll.c
@@ -983,7 +983,7 @@ rc_t CC KDlsetTryLib ( const KDirectory *dir,
if ( sizeof SHLX >= 2 )
{
/* SHLX has at least 1 character plus NUL byte */
- uint32_t len = strlen ( name );
+ size_t len = strlen ( name );
/* name must be at least 1 character larger */
if ( len <= ( sizeof SHLX - 1 ) )
return 0;
diff --git a/libs/kfs/win/sysfile.c b/libs/kfs/win/sysfile.c
index 681b0cc..5948bd4 100644
--- a/libs/kfs/win/sysfile.c
+++ b/libs/kfs/win/sysfile.c
@@ -191,7 +191,7 @@ static bool set_not_sparse (KSysFile * self)
*/
static rc_t set_zero_region (KSysFile * self, uint64_t start, uint64_t size)
{
- FILE_ZERO_DATA_INFORMATION b = { start, start + size };
+ FILE_ZERO_DATA_INFORMATION b = { ( DWORD ) start, ( LONG ) ( start + size ) };
DWORD ret;
BOOL worked;
@@ -515,7 +515,6 @@ rc_t KSysFileReadCommon ( const KSysFile *cself,
bytes_read = 0;
if ( ReadFile ( self -> handle, buffer, to_read, & bytes_read, NULL ) == 0 )
{
- rc_t rc;
DWORD last_error;
switch ( last_error = GetLastError () )
@@ -524,6 +523,14 @@ rc_t KSysFileReadCommon ( const KSysFile *cself,
break;
case ERROR_IO_PENDING:
continue;
+ default:
+ {
+ rc_t rc = RC ( rcFS, rcFile, rcReading, rcNoObj, rcUnknown);
+ PLOGERR ( klogErr,
+ ( klogErr, rc, "error reading system file - $(E)($(C))",
+ "E=%!,C=%u", last_error, last_error ) );
+ return rc;
+ }
}
}
@@ -539,7 +546,6 @@ static
rc_t CC KSysDiskFileRead ( const KSysFile *cself, uint64_t pos,
void *buffer, size_t bsize, size_t *num_read )
{
- DWORD to_read, bytes_read;
KSysFile *self = ( KSysFile* ) cself;
if ( self -> pos != pos )
@@ -568,7 +574,7 @@ rc_t CC KSysDiskFileRead ( const KSysFile *cself, uint64_t pos,
}
/* if we try to read beyond the end of the file... */
- if ( pos >= p . QuadPart )
+ if ( ( LONGLONG ) pos >= p . QuadPart )
{ /* We've defined reading beyond EOF as return RC of 0 but bytes read as 0 */
/*return RC ( rcFS, rcFile, rcPositioning, rcFileDesc, rcInvalid );*/
return 0;
@@ -618,7 +624,6 @@ static
rc_t CC KSysFileRead ( const KSysFile *cself, uint64_t pos,
void *buffer, size_t bsize, size_t *num_read )
{
- DWORD to_read, bytes_read;
KSysFile *self = ( KSysFile* ) cself;
if ( self -> pos != pos )
diff --git a/libs/klib/Makefile b/libs/klib/Makefile
index 21f2b1d..f2ba1ea 100644
--- a/libs/klib/Makefile
+++ b/libs/klib/Makefile
@@ -119,7 +119,8 @@ KLIB_SRC = \
rc-tbl \
rc-idx-tbl \
judy-vector \
- report
+ report \
+ sra-release-version
ifeq (win,$(OS))
KLIB_SRC += sysalloc
diff --git a/libs/klib/libkascii.vers b/libs/klib/libkascii.vers
deleted file mode 100644
index 7dea76e..0000000
--- a/libs/klib/libkascii.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.1
diff --git a/libs/klib/libklib.vers b/libs/klib/libklib.vers
deleted file mode 100644
index 0b1f1ed..0000000
--- a/libs/klib/libklib.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.2.13
diff --git a/libs/klib/libktlog.vers b/libs/klib/libktlog.vers
deleted file mode 100644
index 7dea76e..0000000
--- a/libs/klib/libktlog.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.1
diff --git a/libs/klib/libkutf8.vers b/libs/klib/libkutf8.vers
deleted file mode 100644
index 7dea76e..0000000
--- a/libs/klib/libkutf8.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.1
diff --git a/libs/klib/log.c b/libs/klib/log.c
index 28aa7ea..1145058 100644
--- a/libs/klib/log.c
+++ b/libs/klib/log.c
@@ -159,7 +159,7 @@ rc_t CC KLogDefaultFormatter( void* self, KWrtHandler* writer,
{
rc_t rc = 0;
size_t num_writ, nsize, msize;
- uint32_t mlen;
+ uint64_t mlen;
char buffer[8192], *nbuffer;
const char* msg, *rc_msg;
@@ -471,7 +471,7 @@ rc_t prep_v_args( uint32_t* argc, wrt_nvp_t argv[], size_t max_argc,
if ( rc == 0 )
{
/* tokenize the parameters into name/value pairs */
- rc = logtokenizeparams(fmt, pbuffer, argc, argv, max_argc);
+ rc = logtokenizeparams(fmt, pbuffer, argc, argv, ( uint32_t ) max_argc);
}
return rc;
}
diff --git a/libs/klib/maybeswap.h b/libs/klib/maybeswap.h
index a6ba8d1..71696a0 100644
--- a/libs/klib/maybeswap.h
+++ b/libs/klib/maybeswap.h
@@ -27,21 +27,31 @@
#ifndef _h_maybeswap_
#define _h_maybeswap_
-#if SWAP_PERSISTED
-
-#include <byteswap.h>
-
-#define GET16( x ) bswap_16 ( x )
-#define GET32( x ) bswap_32 ( x )
-#define GET64( x ) bswap_64 ( x )
-
+/* technically, a whitelist would be safer */
+#ifdef __sparc
+# if SWAP_PERSISTED
+# define GET_UNALIGNED( type, swap, dest, src ) \
+ do { \
+ type tmp##__LINE__; \
+ memcpy ( & tmp##__LINE__, & ( src ), sizeof ( type ) ); \
+ dest = swap ( tmp##__LINE__ ); \
+ } while (0)
+# else
+# define GET_UNALIGNED( type, swap, dest, src ) \
+ memcpy ( & ( dest ), & ( src ), sizeof ( type ) )
+# endif
+#elif SWAP_PERSISTED
+# define GET_UNALIGNED( type, swap, dest, src ) dest = swap ( src )
#else
+# define GET_UNALIGNED( type, swap, dest, src ) \
+ dest = ( * ( const type* ) & ( src ) )
+#endif
-#define GET16( x ) ( * ( const uint16_t* ) & ( x ) )
-#define GET32( x ) ( * ( const uint32_t* ) & ( x ) )
-#define GET64( x ) ( * ( const uint64_t* ) & ( x ) )
+#include <byteswap.h>
-#endif
+#define GET16( dest, src ) GET_UNALIGNED ( uint16_t, bswap_16, dest, src )
+#define GET32( dest, src ) GET_UNALIGNED ( uint32_t, bswap_32, dest, src )
+#define GET64( dest, src ) GET_UNALIGNED ( uint64_t, bswap_64, dest, src )
#endif /* _h_maybeswap_ */
diff --git a/libs/klib/pbstree-impl.c b/libs/klib/pbstree-impl.c
index c6ebf19..7b21db3 100644
--- a/libs/klib/pbstree-impl.c
+++ b/libs/klib/pbstree-impl.c
@@ -47,7 +47,9 @@
static
uint32_t CC PBSTreeImplCount ( const PBSTree *self )
{
- return GET32 ( self -> pt -> num_nodes );
+ uint32_t n;
+ GET32 ( n, self -> pt -> num_nodes );
+ return n;
}
/* Depth
@@ -58,7 +60,7 @@ uint32_t CC PBSTreeImplDepth ( const PBSTree *self )
{
uint32_t depth, num_nodes;
- num_nodes = GET32 ( self -> pt -> num_nodes );
+ GET32 ( num_nodes, self -> pt -> num_nodes );
for ( depth = 0; num_nodes != 0; ++ depth )
num_nodes >>= 1;
@@ -75,21 +77,25 @@ size_t CC PBSTreeImplSize ( const PBSTree *oself )
{
size_t bytes;
const P_BSTree *self = oself -> pt;
+ uint32_t data_size, bytes32;
- bytes = GET32( self -> num_nodes );
- if (bytes == 0) /* empty tree */
+ GET32 ( bytes32, self -> num_nodes );
+ if ( bytes32 == 0 ) /* empty tree */
return sizeof self -> num_nodes;
+ else
+ bytes = bytes32;
- if ( GET32( self -> data_size ) > 256 )
+ GET32 ( data_size, self -> data_size );
+ if ( data_size > 256 )
{
- if ( GET32( self -> data_size ) <= 65536 )
+ if ( data_size <= 65536 )
bytes += bytes;
else
bytes <<= 2;
}
return sizeof * self - sizeof self -> data_idx +
- bytes + GET32( self -> data_size );
+ bytes + data_size;
}
/* GetNodeData
@@ -105,8 +111,8 @@ rc_t CC PBSTreeImplGetNodeData8 ( const PBSTree *oself,
const uint8_t *data_start;
uint32_t off, end;
- num_nodes = GET32( self -> num_nodes );
- end = GET32 ( self -> data_size );
+ GET32 ( num_nodes, self -> num_nodes );
+ GET32 ( end, self -> data_size );
if ( id == 0 )
return RC ( rcCont, rcTree, rcAccessing, rcId, rcNull );
@@ -130,12 +136,12 @@ rc_t CC PBSTreeImplGetNodeData16 ( const PBSTree *oself,
{
const P_BSTree *self = oself -> pt;
- uint32_t num_nodes;
+ uint32_t num_nodes, end;
const uint8_t *data_start;
- uint32_t off, end;
+ uint16_t off;
- num_nodes = GET32( self -> num_nodes );
- end = GET32 ( self -> data_size );
+ GET32 ( num_nodes, self -> num_nodes );
+ GET32 ( end, self -> data_size );
if ( id == 0 )
return RC ( rcCont, rcTree, rcAccessing, rcId, rcNull );
@@ -144,9 +150,13 @@ rc_t CC PBSTreeImplGetNodeData16 ( const PBSTree *oself,
data_start = ( const uint8_t* )
& self -> data_idx . v16 [ num_nodes ];
- off = GET16( self -> data_idx . v16 [ id - 1 ] );
+ GET16 ( off, self -> data_idx . v16 [ id - 1 ] );
if ( id != num_nodes )
- end = GET16( self -> data_idx . v16 [ id ] );
+ {
+ uint16_t end16;
+ GET16 ( end16, self -> data_idx . v16 [ id ] );
+ end = end16;
+ }
* addr = & data_start [ off ];
* size = end - off;
@@ -163,8 +173,8 @@ rc_t CC PBSTreeImplGetNodeData32 ( const PBSTree *oself,
const uint8_t *data_start;
uint32_t off, end;
- num_nodes = GET32( self -> num_nodes );
- end = GET32 ( self -> data_size );
+ GET32 ( num_nodes, self -> num_nodes );
+ GET32 ( end, self -> data_size );
if ( id == 0 )
return RC ( rcCont, rcTree, rcAccessing, rcId, rcNull );
@@ -173,9 +183,9 @@ rc_t CC PBSTreeImplGetNodeData32 ( const PBSTree *oself,
data_start = ( const uint8_t* )
& self -> data_idx . v32 [ num_nodes ];
- off = GET32( self -> data_idx . v32 [ id - 1 ] );
+ GET32 ( off, self -> data_idx . v32 [ id - 1 ] );
if ( id != num_nodes )
- end = GET32( self -> data_idx . v32 [ id ] );
+ GET32 ( end, self -> data_idx . v32 [ id ] );
* addr = & data_start [ off ];
* size = end - off;
@@ -191,15 +201,17 @@ uint32_t CC PBSTreeImplFind8 ( const PBSTree *self, PBSTNode *n, const void *ite
int ( CC * cmp ) ( const void *item, const PBSTNode *n, void * data ), void * data )
{
const P_BSTree *pt = self -> pt;
- uint32_t right = GET32( pt -> num_nodes );
- if ( pt -> num_nodes != 0 )
+ uint32_t right;
+ GET32 ( right, pt -> num_nodes );
+ if ( right != 0 )
{
int diff;
const uint8_t *data_start;
- uint32_t off, end = GET32( pt -> data_size );
+ uint32_t off, end;
uint32_t id, left = 1;
uint32_t max_right = right;
+ GET32 ( end, pt -> data_size );
data_start = & pt -> data_idx . v8 [ right ];
do
@@ -233,28 +245,35 @@ uint32_t CC PBSTreeImplFind16 ( const PBSTree *self, PBSTNode *n, const void *it
int ( CC * cmp ) ( const void *item, const PBSTNode *n, void * data ), void * data )
{
const P_BSTree *pt = self -> pt;
- uint32_t right = GET32( pt -> num_nodes );
- if ( pt -> num_nodes != 0 )
+ uint32_t right;
+ GET32 ( right, pt -> num_nodes );
+ if ( right != 0 )
{
int diff;
const uint8_t *data_start;
- uint32_t off, end = GET32( pt -> data_size );
+ uint32_t end;
uint32_t id, left = 1;
uint32_t max_right = right;
+ uint16_t off;
+ GET32 ( end, pt -> data_size );
data_start = ( const uint8_t* )
& pt -> data_idx . v16 [ right ];
do
{
id = ( left + right ) >> 1;
- off = GET16( pt -> data_idx . v16 [ id - 1 ] );
+ GET16 ( off, pt -> data_idx . v16 [ id - 1 ] );
n -> id = id;
n -> data . addr = & data_start [ off ];
if ( id == max_right )
n -> data . size = end - off;
else
- n -> data . size = GET16( pt -> data_idx . v16 [ id ] ) - off;
+ {
+ uint16_t size;
+ GET16 ( size, pt -> data_idx . v16 [ id ] );
+ n -> data . size = size - off;
+ }
diff = ( * cmp ) ( item, n, data );
if ( diff == 0 )
return id;
@@ -276,28 +295,33 @@ uint32_t CC PBSTreeImplFind32 ( const PBSTree *self, PBSTNode *n, const void *it
int ( CC * cmp ) ( const void *item, const PBSTNode *n, void * data ), void * data )
{
const P_BSTree *pt = self -> pt;
- uint32_t right = GET32( pt -> num_nodes );
- if ( pt -> num_nodes != 0 )
+ uint32_t right;
+ GET32 ( right, pt -> num_nodes );
+ if ( right != 0 )
{
int diff;
const uint8_t *data_start;
- uint32_t off, end = GET32( pt -> data_size );
+ uint32_t off, end;
uint32_t id, left = 1;
uint32_t max_right = right;
+ GET32 ( end, pt -> data_size );
data_start = ( const uint8_t* )
& pt -> data_idx . v32 [ right ];
do
{
id = ( left + right ) >> 1;
- off = GET32( pt -> data_idx . v32 [ id - 1 ] );
+ GET32 ( off, pt -> data_idx . v32 [ id - 1 ] );
n -> id = id;
n -> data . addr = & data_start [ off ];
if ( id == max_right )
n -> data . size = end - off;
else
- n -> data . size = GET32( pt -> data_idx . v32 [ id ] ) - off;
+ {
+ GET32 ( n -> data . size, pt -> data_idx . v32 [ id ] );
+ n -> data . size -= off;
+ }
diff = ( * cmp ) ( item, n, data );
if ( diff == 0 )
return id;
@@ -322,7 +346,8 @@ void CC PBSTreeImplForEach8 ( const PBSTree *self, bool reverse,
void ( CC * f ) ( PBSTNode *n, void *data ), void *data )
{
const P_BSTree *pt = self -> pt;
- uint32_t num_nodes = GET32( pt -> num_nodes );
+ uint32_t num_nodes;
+ GET32 ( num_nodes, pt -> num_nodes );
if ( num_nodes != 0 )
{
PBSTNode n;
@@ -330,7 +355,7 @@ void CC PBSTreeImplForEach8 ( const PBSTree *self, bool reverse,
uint32_t off, end;
const uint8_t *data_start;
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
data_start = & pt -> data_idx . v8 [ num_nodes ];
@@ -347,7 +372,7 @@ void CC PBSTreeImplForEach8 ( const PBSTree *self, bool reverse,
( * f ) ( & n, data );
}
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
}
off = pt -> data_idx . v8 [ num_nodes - 1 ];
@@ -377,25 +402,28 @@ void CC PBSTreeImplForEach16 ( const PBSTree *self, bool reverse,
void ( CC * f ) ( PBSTNode *n, void *data ), void *data )
{
const P_BSTree *pt = self -> pt;
- uint32_t num_nodes = GET32( pt -> num_nodes );
+ uint32_t num_nodes;
+ GET32 ( num_nodes, pt -> num_nodes );
if ( num_nodes != 0 )
{
PBSTNode n;
- uint32_t id;
- uint32_t off, end;
+ uint32_t id, end;
+ uint16_t off;
const uint8_t *data_start;
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
data_start = ( const uint8_t* )
& pt -> data_idx . v16 [ num_nodes ];
if ( ! reverse )
{
- for ( off = GET16( pt -> data_idx . v16 [ 0 ] ),
- id = 1; id < num_nodes; off = end, ++ id )
+ GET16 ( off, pt -> data_idx . v16 [ 0 ] );
+ for ( id = 1; id < num_nodes; off = end, ++ id )
{
- end = GET16( pt -> data_idx . v16 [ id ] );
+ uint16_t end16;
+ GET16 ( end16, pt -> data_idx . v16 [ id ] );
+ end = end16;
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -403,10 +431,10 @@ void CC PBSTreeImplForEach16 ( const PBSTree *self, bool reverse,
( * f ) ( & n, data );
}
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
}
- off = GET16( pt -> data_idx . v16 [ num_nodes - 1 ] );
+ GET16 ( off, pt -> data_idx . v16 [ num_nodes - 1 ] );
n . internal = pt;
n . id = num_nodes;
n . data . addr = & data_start [ off ];
@@ -417,7 +445,7 @@ void CC PBSTreeImplForEach16 ( const PBSTree *self, bool reverse,
{
for ( end = off, id = num_nodes - 1; id > 0; end = off, -- id )
{
- off = GET16( pt -> data_idx . v16 [ id - 1 ] );
+ GET16 ( off, pt -> data_idx . v16 [ id - 1 ] );
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -433,7 +461,8 @@ void CC PBSTreeImplForEach32 ( const PBSTree *self, bool reverse,
void ( CC * f ) ( PBSTNode *n, void *data ), void *data )
{
const P_BSTree *pt = self -> pt;
- uint32_t num_nodes = GET32( pt -> num_nodes );
+ uint32_t num_nodes;
+ GET32 ( num_nodes, pt -> num_nodes );
if ( num_nodes != 0 )
{
PBSTNode n;
@@ -441,17 +470,17 @@ void CC PBSTreeImplForEach32 ( const PBSTree *self, bool reverse,
uint32_t off, end;
const uint8_t *data_start;
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
data_start = ( const uint8_t* )
& pt -> data_idx . v32 [ num_nodes ];
if ( ! reverse )
{
- for ( off = GET32( pt -> data_idx . v32 [ 0 ] ),
- id = 1; id < num_nodes; off = end, ++ id )
+ GET32 ( off, pt -> data_idx . v32 [ 0 ] );
+ for ( id = 1; id < num_nodes; off = end, ++ id )
{
- end =GET32( pt -> data_idx . v32 [ id ] );
+ GET32( end, pt -> data_idx . v32 [ id ] );
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -459,10 +488,10 @@ void CC PBSTreeImplForEach32 ( const PBSTree *self, bool reverse,
( * f ) ( & n, data );
}
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
}
- off = GET32( pt -> data_idx . v32 [ num_nodes - 1 ] );
+ GET32 ( off, pt -> data_idx . v32 [ num_nodes - 1 ] );
n . internal = pt;
n . id = num_nodes;
n . data . addr = & data_start [ off ];
@@ -473,7 +502,7 @@ void CC PBSTreeImplForEach32 ( const PBSTree *self, bool reverse,
{
for ( end = off, id = num_nodes - 1; id > 0; end = off, -- id )
{
- off = GET32( pt -> data_idx . v32 [ id - 1 ] );
+ GET32 ( off, pt -> data_idx . v32 [ id - 1 ] );
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -493,15 +522,16 @@ bool CC PBSTreeImplDoUntil8 ( const PBSTree *self, bool reverse,
bool ( CC * f ) ( PBSTNode *n, void *data ), void *data )
{
const P_BSTree *pt = self -> pt;
- uint32_t num_nodes = GET32( pt -> num_nodes );
- if ( pt -> num_nodes != 0 )
+ uint32_t num_nodes;
+ GET32 ( num_nodes, pt -> num_nodes );
+ if ( num_nodes != 0 )
{
PBSTNode n;
uint32_t id;
uint32_t off, end;
const uint8_t *data_start;
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
data_start = & pt -> data_idx . v8 [ num_nodes ];
@@ -519,7 +549,7 @@ bool CC PBSTreeImplDoUntil8 ( const PBSTree *self, bool reverse,
return true;
}
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
}
off = pt -> data_idx . v8 [ num_nodes - 1 ];
@@ -553,25 +583,28 @@ bool CC PBSTreeImplDoUntil16 ( const PBSTree *self, bool reverse,
bool ( CC * f ) ( PBSTNode *n, void *data ), void *data )
{
const P_BSTree *pt = self -> pt;
- uint32_t num_nodes = GET32( pt -> num_nodes );
- if ( pt -> num_nodes != 0 )
+ uint32_t num_nodes;
+ GET32 ( num_nodes, pt -> num_nodes );
+ if ( num_nodes != 0 )
{
PBSTNode n;
- uint32_t id;
- uint32_t off, end;
+ uint32_t id, end;
+ uint16_t off;
const uint8_t *data_start;
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
data_start = ( const uint8_t* )
& pt -> data_idx . v16 [ num_nodes ];
if ( ! reverse )
{
- for ( off = GET16( pt -> data_idx . v16 [ 0 ] ),
- id = 1; id < num_nodes; off = end, ++ id )
+ GET16 ( off, pt -> data_idx . v16 [ 0 ] );
+ for ( id = 1; id < num_nodes; off = end, ++ id )
{
- end = GET16( pt -> data_idx . v16 [ id ] );
+ uint16_t end16;
+ GET16 ( end16, pt -> data_idx . v16 [ id ] );
+ end = end16;
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -580,10 +613,10 @@ bool CC PBSTreeImplDoUntil16 ( const PBSTree *self, bool reverse,
return true;
}
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
}
- off = GET16( pt -> data_idx . v16 [ num_nodes - 1 ] );
+ GET16 ( off, pt -> data_idx . v16 [ num_nodes - 1 ] );
n . internal = pt;
n . id = num_nodes;
n . data . addr = & data_start [ off ];
@@ -595,7 +628,7 @@ bool CC PBSTreeImplDoUntil16 ( const PBSTree *self, bool reverse,
{
for ( end = off, id = num_nodes - 1; id > 0; end = off, -- id )
{
- off = GET16( pt -> data_idx . v16 [ id - 1 ] );
+ GET16 ( off, pt -> data_idx . v16 [ id - 1 ] );
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -613,25 +646,26 @@ bool CC PBSTreeImplDoUntil32 ( const PBSTree *self, bool reverse,
bool ( CC * f ) ( PBSTNode *n, void *data ), void *data )
{
const P_BSTree *pt = self -> pt;
- uint32_t num_nodes = GET32( pt -> num_nodes );
- if ( pt -> num_nodes != 0 )
+ uint32_t num_nodes;
+ GET32 ( num_nodes, pt -> num_nodes );
+ if ( num_nodes != 0 )
{
PBSTNode n;
uint32_t id;
uint32_t off, end;
const uint8_t *data_start;
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
data_start = ( const uint8_t* )
& pt -> data_idx . v32 [ num_nodes ];
if ( ! reverse )
{
- for ( off = GET32( pt -> data_idx . v32 [ 0 ] ),
- id = 1; id < num_nodes; off = end, ++ id )
+ GET32 ( off, pt -> data_idx . v32 [ 0 ] );
+ for ( id = 1; id < num_nodes; off = end, ++ id )
{
- end = GET32( pt -> data_idx . v32 [ id ] );
+ GET32 ( end, pt -> data_idx . v32 [ id ] );
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -640,10 +674,10 @@ bool CC PBSTreeImplDoUntil32 ( const PBSTree *self, bool reverse,
return true;
}
- end = GET32( pt -> data_size );
+ GET32 ( end, pt -> data_size );
}
- off = GET32( pt -> data_idx . v32 [ num_nodes - 1 ] );
+ GET32 ( off, pt -> data_idx . v32 [ num_nodes - 1 ] );
n . internal = pt;
n . id = num_nodes;
n . data . addr = & data_start [ off ];
@@ -655,7 +689,7 @@ bool CC PBSTreeImplDoUntil32 ( const PBSTree *self, bool reverse,
{
for ( end = off, id = num_nodes - 1; id > 0; end = off, -- id )
{
- off = GET32( pt -> data_idx . v32 [ id - 1 ] );
+ GET32 ( off, pt -> data_idx . v32 [ id - 1 ] );
n . internal = pt;
n . id = id;
n . data . addr = & data_start [ off ];
@@ -734,7 +768,8 @@ static PBSTree_vt_v1 vtPBSTreeImpl32 =
static
PBSTree_vt * CC PBSTreeImplGetVTable ( const P_BSTree *pt )
{
- uint32_t data_size = GET32 ( pt -> data_size );
+ uint32_t data_size;
+ GET32 ( data_size, pt -> data_size );
if ( data_size <= 256 )
return (PBSTree_vt *) & vtPBSTreeImpl8;
if ( data_size <= 65536 )
@@ -758,19 +793,18 @@ rc_t CC PBSTreeImplCheckPersisted ( const P_BSTree *pt, size_t size )
if ( size < sizeof pt -> num_nodes )
return RC ( rcCont, rcTree, rcConstructing, rcData, rcInvalid );
- num_nodes = GET32( pt -> num_nodes );
+ GET32 ( num_nodes, pt -> num_nodes );
if ( num_nodes > 0 )
{
uint32_t data_size;
const uint8_t *end, *data_start;
- /* no need to byteswap a value of 0 */
- if ( size < sizeof * pt || pt -> data_size == 0 )
+ GET32 ( data_size, pt -> data_size );
+ if ( size < sizeof * pt || data_size == 0 )
return RC ( rcCont, rcTree, rcConstructing, rcData, rcIncomplete );
end = ( const uint8_t* ) pt + size;
- data_size = GET32 ( pt -> data_size );
if ( data_size <= 256 )
data_start = & pt -> data_idx . v8 [ num_nodes ];
else if ( data_size <= 65536 )
diff --git a/libs/klib/printf.c b/libs/klib/printf.c
index 2907575..44bba83 100644
--- a/libs/klib/printf.c
+++ b/libs/klib/printf.c
@@ -2298,8 +2298,8 @@ rc_t print_padding ( KBufferedWrtHandler *out, size_t count, char pad_char )
num_writ = count - total;
else
{
- uint32_t i = out -> cur;
- uint32_t lim = out -> cur + count - total;
+ size_t i = out -> cur;
+ size_t lim = out -> cur + count - total;
if ( lim > out -> bsize )
lim = out -> bsize;
for ( num_writ = lim - i; i < lim; ++ i )
@@ -2538,7 +2538,8 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
f . type = sptString;
if ( f . u . f . precision < S . len )
#if STDC_COMPATIBILITY && !defined(__GLIBC__)
- S . size = S . len = f . u . f . precision;
+ S . size = f . u . f . precision;
+ S . len = (uint32_t) f . u . f . precision;
#else
StringInit ( & S, "", 0, 0 );
#endif
@@ -2710,7 +2711,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
case sptBytesPrinted:
n = args [ arg_idx ] . n;
if ( n != NULL )
- * n = out -> cur + out -> flushed;
+ * n = (uint32_t) ( out -> cur + out -> flushed );
++ arg_idx;
continue;
#endif
@@ -2725,7 +2726,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
case sptSignedInt16Vect:
case sptSignedInt32Vect:
case sptSignedInt64Vect:
- f64 = i64;
+ f64 = (double) i64;
break;
case sptUnsignedInt:
@@ -2733,14 +2734,14 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
case sptUnsignedInt16Vect:
case sptUnsignedInt32Vect:
case sptUnsignedInt64Vect:
- f64 = u64;
+ f64 = (double) u64;
break;
case sptFloat:
case sptFloat32Vect:
case sptFloat64Vect:
case sptFloatLongVect:
- i64 = f64;
+ i64 = (int64_t) f64;
break;
case sptChar:
@@ -2755,10 +2756,10 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
return RC ( rcText, rcString, rcConverting, rcType, rcUnsupported );
case sptRowId:
- f64 = i64;
+ f64 = (double) i64;
break;
case sptRowLen:
- f64 = u64;
+ f64 = (double) u64;
break;
#if SUPPORT_PERCENT_N
@@ -3111,12 +3112,12 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
break;
case spfRC:
- dst_len = KWrtFmt_rc_t ( text, sizeof text, f . explain_rc ? "#" : "", ( rc_t ) u64 );
+ dst_len = (uint32_t) KWrtFmt_rc_t ( text, sizeof text, f . explain_rc ? "#" : "", ( rc_t ) u64 );
StringInit ( & S, text, dst_len, dst_len );
break;
case spfOSErr:
- dst_len = KWrtFmt_error_code ( text, sizeof text, ( int ) i64 );
+ dst_len = (uint32_t) KWrtFmt_error_code ( text, sizeof text, ( int ) i64 );
StringInit ( & S, text, dst_len, dst_len );
break;
@@ -3134,7 +3135,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
{
assert ( S . addr != NULL );
assert ( S . size != 0 || S . len == 0 );
- if ( StringSubstr ( & S, & S, text_start, text_lim ) == NULL )
+ if ( StringSubstr ( & S, & S, ( uint32_t ) text_start, ( uint32_t ) text_lim ) == NULL )
StringInit ( & S, "", 0, 0 );
}
@@ -3149,7 +3150,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out,
if ( f . left_fill != 0 && f . u . f . min_field_width > dst_len )
{
assert ( S . size != 0 || S . len == 0 );
- left_pad += f . u . f . min_field_width - dst_len;
+ left_pad += ( uint32_t ) ( f . u . f . min_field_width ) - dst_len;
dst_len = ( uint32_t ) f . u . f . min_field_width;
}
diff --git a/libs/klib/refcount.c b/libs/klib/refcount.c
index 55590d9..2a94629 100644
--- a/libs/klib/refcount.c
+++ b/libs/klib/refcount.c
@@ -24,20 +24,17 @@
*
*/
-#include <klib/extern.h>
-#include <sysalloc.h>
-
-#ifndef TRACK_REFERENCES
#define TRACK_REFERENCES 1
-#endif
+#define INLINE_REFCOUNT 0
-#undef INLINE_REFCOUNT
-/* #define INLINE_REFCOUNT 1 */
+#include <klib/extern.h>
#include <klib/refcount.h>
#include <klib/log.h>
#include <atomic32.h>
+#include <sysalloc.h>
+
#include <limits.h>
#include <stdlib.h>
#include <string.h>
@@ -101,8 +98,7 @@
*
* "name" [ IN ] - NUL-terminated name of instance
*/
-#if INLINE_REFCOUNT
-#else
+#if ! INLINE_REFCOUNT
LIB_EXPORT void CC KRefcountInit ( KRefcount *refcount, int value,
const char *clsname, const char *op, const char *name )
{
@@ -136,8 +132,7 @@ LIB_EXPORT int CC KDualRefInit ( KDualRef *refcount, int owned, int dep,
/* Whack
* tear down whatever was built up in object
*/
-#if INLINE_REFCOUNT
-#else
+#if ! INLINE_REFCOUNT
LIB_EXPORT void CC KRefcountWhack ( KRefcount *self, const char *clsname )
{
REFMSG ( clsname, "whack", self );
@@ -163,8 +158,7 @@ LIB_EXPORT void CC KDualRefWhack ( KDualRef *self, const char *clsname )
* krefLimit : reference was NOT added, too many references
* krefNegative : reference was NOT added, prior value was negative
*/
-#if INLINE_REFCOUNT
-#else
+#if ! INLINE_REFCOUNT
LIB_EXPORT int CC KRefcountAdd ( const KRefcount *self, const char *clsname )
{
#if DETECT_ZERO_STATE
@@ -288,8 +282,7 @@ LIB_EXPORT int CC KDualRefAdd ( const KDualRef *self, const char *clsname )
* krefZero : reference was dropped, and no further owned refernces exist
* krefNegative : reference was NOT dropped, as resultant count would be negative
*/
-#if INLINE_REFCOUNT
-#else
+#if ! INLINE_REFCOUNT
LIB_EXPORT int CC KRefcountDrop ( const KRefcount *self, const char *clsname )
{
#if DETECT_ZERO_STATE
@@ -346,8 +339,7 @@ LIB_EXPORT int CC KDualRefDrop ( const KDualRef *self, const char *clsname )
* krefZero : reference was added, but prior value was zero
* krefNegative : reference was NOT added, prior value was negative
*/
-#if INLINE_REFCOUNT
-#else
+#if ! INLINE_REFCOUNT
LIB_EXPORT int CC KRefcountAddDep ( const KRefcount *self, const char *clsname )
{
#if DETECT_ZERO_STATE
@@ -461,8 +453,7 @@ LIB_EXPORT int CC KDualRefAddDep ( const KDualRef *self, const char *clsname )
* krefZero : reference was dropped, and no further dependent refernces exist
* krefNegative : reference was NOT dropped, as resultant count would be negative
*/
-#if INLINE_REFCOUNT
-#else
+#if ! INLINE_REFCOUNT
LIB_EXPORT int CC KRefcountDropDep ( const KRefcount *self, const char *clsname )
{
#if DETECT_ZERO_STATE
diff --git a/tools/sra-pileup/reref.h b/libs/klib/release-vers.h
similarity index 76%
copy from tools/sra-pileup/reref.h
copy to libs/klib/release-vers.h
index 13e3136..04f64a3 100644
--- a/tools/sra-pileup/reref.h
+++ b/libs/klib/release-vers.h
@@ -23,26 +23,19 @@
* ===========================================================================
*
*/
-#ifndef _h_reref_
-#define _h_reref_
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
+/* THIS FILE IS NOT GENERATED AUTOMATICALLY! */
-#include <kapp/args.h>
+/* The Version of current SRA Toolkit Release */
+#define RELEASE_VERS 0x02030004
-#include <klib/out.h>
-#include <klib/rc.h>
-#include <klib/log.h>
+/* The Revision of Version of current SRA Toolkit Release */
+#define RELEASE_REVISION 2
-#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
-
-rc_t report_on_reference( Args * args, bool extended );
-
-#endif
+/* The Type of Version of current SRA Toolkit Release is one of:
+ * 'a' - alpha
+ * 'b' - beta
+ * 'c' - release candidate
+ * 'r' - final
+ */
+#define RELEASE_TYPE 'r'
diff --git a/libs/klib/report.c b/libs/klib/report.c
index 94d6078..c1207e6 100644
--- a/libs/klib/report.c
+++ b/libs/klib/report.c
@@ -527,13 +527,19 @@ static rc_t reportToStdErrSet(const Report* self, KWrtHandler* old_handler) {
/* Finalize:
* If (rc_in != 0) then report environment information.
* Then clean itself.
+ *
+ * When "--" OPTION_REPORT "always" command line argument is passed
+ * then the report [usually] goes to strerr.
+ *
+ * When aForce == truethen the report goes to strout:
+ * it is done to insert it into test-sra output.
*/
-LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
+static rc_t _ReportFinalize(rc_t rc_in, bool aForce) {
rc_t rc = 0;
const char* report_arg = NULL;
- bool force = rc_in != 0;
+ bool force = rc_in != 0;
Report* self = NULL;
ReportGet(&self);
@@ -555,6 +561,11 @@ LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
}
}
+ if (!force && aForce) {
+ force = true;
+ self -> silence = false;
+ }
+
if (self->argv) {
int i = 0;
for (i = 1; i < self->argc; ++i) {
@@ -584,7 +595,7 @@ LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
"type=%s", self->argv[i + 1]));*/
if (force) {
- if (self->hasZombies) {
+ if (self->hasZombies && !aForce) {
KOutHandlerSetStdErr();
if (self->object != NULL) {
OUTMSG(("\nThe archive '%s' may be truncated: "
@@ -603,7 +614,9 @@ LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
bool to_file = false;
KWrtHandler old_handler;
if (rc_in == 0) {
- reportToStdErrSet(self, &old_handler);
+ if (!aForce) {
+ reportToStdErrSet(self, &old_handler);
+ }
}
else {
rc_t rc2 = 0;
@@ -647,7 +660,8 @@ LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
if ( self -> report_config != NULL )
{
- rc_t rc2 = ( * self -> report_config ) ( & report_funcs, indent + 1);
+ rc_t rc2 = ( * self -> report_config )
+ ( & report_funcs, indent + 1);
if (rc == 0 && rc2 != 0)
{ rc = rc2; }
}
@@ -663,11 +677,18 @@ LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
if ( self -> report_software != NULL )
{
const char *argv_0 = self -> argv ? self -> argv [ 0 ] : "";
- rc_t rc2 = ( * self -> report_software ) ( & report_funcs, indent + 1, argv_0, self -> date, self -> tool_ver );
+ rc_t rc2 = ( * self -> report_software )
+ ( & report_funcs, indent + 1,
+ argv_0, self -> date, self -> tool_ver );
if (rc == 0 && rc2 != 0)
{ rc = rc2; }
}
+ if (self->hasZombies) {
+ OUTMSG(("\nOne of archives may be truncated: "
+ "It should be redownloaded.\n"));
+ }
+
reportClose(indent, tag);
reportData1("\n");
@@ -691,6 +712,18 @@ LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
return rc;
}
+/* Finalize:
+ * If (rc_in != 0) then report environment information.
+ * Then clean itself.
+ */
+LIB_EXPORT rc_t CC ReportFinalize(rc_t rc_in) {
+ return _ReportFinalize(rc_in, false);
+}
+
+LIB_EXPORT rc_t CC ReportForceFinalize(void) {
+ return _ReportFinalize(0, true);
+}
+
static
char **copy_argv ( int argc, char **argv )
{
diff --git a/libs/klib/sra-release-version.c b/libs/klib/sra-release-version.c
new file mode 100644
index 0000000..44dd4cb
--- /dev/null
+++ b/libs/klib/sra-release-version.c
@@ -0,0 +1,328 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include <klib/printf.h> /* string_printf */
+#include <klib/rc.h>
+#include <klib/sra-release-version.h>
+
+#include "release-vers.h"
+
+#include <ctype.h> /* isdigit */
+#include <string.h> /* memset */
+#include <assert.h>
+
+LIB_EXPORT rc_t CC SraReleaseVersionGet(SraReleaseVersion *version) {
+ if (version == NULL) {
+ return RC(rcRuntime, rcArgv, rcAccessing, rcParam, rcNull);
+ }
+
+ version->version = RELEASE_VERS;
+ version->revision = RELEASE_REVISION;
+
+ switch (RELEASE_TYPE) {
+ case 'a':
+ version->type = eSraReleaseVersionTypeAlpha;
+ break;
+ case 'b':
+ version->type = eSraReleaseVersionTypeBeta;
+ break;
+ case 'c':
+ version->type = eSraReleaseVersionTypeRC;
+ break;
+ default:
+ version->type = eSraReleaseVersionTypeFinal;
+ }
+
+ return 0;
+}
+
+LIB_EXPORT rc_t CC SraReleaseVersionCmp(const SraReleaseVersion *self,
+ const SraReleaseVersion *version2, int32_t *result)
+{
+ if (result == NULL) {
+ return RC(rcRuntime, rcArgv, rcAccessing, rcParam, rcNull);
+ }
+
+ *result = 0;
+
+ if (self == NULL) {
+ return RC(rcRuntime, rcArgv, rcAccessing, rcSelf, rcNull);
+ }
+
+ if (version2 == NULL) {
+ return RC(rcRuntime, rcArgv, rcAccessing, rcParam, rcNull);
+ }
+
+ if (self->version < version2->version) {
+ *result = 1;
+ }
+ else if (self->version > version2->version) {
+ *result = -1;
+ }
+ else if (self->type < version2->type) {
+ *result = 1;
+ }
+ else if (self->type > version2->type) {
+ *result = -1;
+ }
+ else if (self->revision < version2->revision) {
+ *result = 1;
+ }
+ else if (self->revision > version2->revision) {
+ *result = -1;
+ }
+ else {
+ *result = 0;
+ }
+
+ return 0;
+}
+
+static int32_t getdigit(const char **s, size_t *size, int32_t max) {
+ int32_t d = 0;
+ char c = 0;
+
+ assert(s && size);
+
+ if (*size == 0) {
+ return 0;
+ }
+
+ c = **s;
+ if (!isdigit(c)) {
+ if (c == '\n') {
+ return 0;
+ }
+ else {
+ return -1;
+ }
+ }
+
+ while (*size > 0) {
+ char c = **s;
+ if (isdigit(c)) {
+ d = d * 10 + c - '0';
+ }
+ else {
+ break;
+ }
+ ++(*s);
+ --*size;
+ }
+
+ if (max > 0 && d > max) {
+ return -2;
+ }
+
+ return d;
+}
+
+#define SET_RC(i) ( \
+ (i) == -1 ? RC(rcRuntime, rcFile, rcReading, rcFile, rcInvalid) \
+ : RC(rcRuntime, rcFile, rcReading, rcNumeral, rcExcessive) )
+
+/* Parse
+ * Initialize SraReleaseVersion from char version[size]
+ */
+LIB_EXPORT rc_t CC SraReleaseVersionInit ( SraReleaseVersion *self,
+ const char *version, size_t size )
+{
+ rc_t rc = 0;
+ int32_t major = 0, minor = 0, release = 0, revision = 0, i = 0,
+ type = eSraReleaseVersionTypeFinal;
+ const char *p = version;
+
+ if (self == NULL) {
+ return RC(rcRuntime, rcArgv, rcAccessing, rcSelf, rcNull);
+ }
+
+ assert(self);
+ memset(self, 0, sizeof *self);
+
+ self->type = eSraReleaseVersionTypeFinal;
+
+ if (size == 0) {
+ return 0;
+ }
+
+ /* major: 1. */
+ if (size > 0 && rc == 0) {
+ int32_t i = getdigit(&p, &size, 255);
+ if (i >= 0) {
+ major = i;
+ }
+ else {
+ rc = SET_RC(i);
+ }
+ }
+
+ /* minor: 1.2 */
+ if (size > 0 && rc == 0 && *p != '\n') {
+ if (*p != '.' && *p != '-') {
+ rc = RC(rcRuntime, rcFile, rcReading, rcFile, rcInvalid);
+ }
+ if (*p == '.') {
+ ++p;
+ --size;
+ i = getdigit(&p, &size, 255);
+ if (i >= 0) {
+ minor = i;
+ }
+ else {
+ rc = SET_RC(i);
+ }
+ }
+ }
+
+ /* release: 1.2.3 */
+ if (size > 0 && rc == 0 && *p != '\n') {
+ if (*p != '.' && *p != '-') {
+ rc = RC(rcRuntime, rcFile, rcReading, rcFile, rcInvalid);
+ }
+ if (*p == '.') {
+ ++p;
+ --size;
+ i = getdigit(&p, &size, 0xFFFF);
+ if (i >= 0) {
+ release = i;
+ }
+ else {
+ rc = SET_RC(i);
+ }
+ }
+ }
+
+ /* dash: 1.2.3- */
+ if (size > 0 && rc == 0 && *p != '\n') {
+ if (*p != '-') {
+ rc = RC(rcRuntime, rcFile, rcReading, rcFile, rcInvalid);
+ }
+ ++p;
+ --size;
+ }
+
+ /* type: 1.2.3-a 1.2.3-rc */
+ if (size > 0 && rc == 0 && *p != '\n') {
+ if (!isalpha(*p)) {
+ if (!isdigit(*p)) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcFile, rcInvalid);
+ }
+ }
+ else {
+ switch (*p) {
+ case 'a':
+ type = eSraReleaseVersionTypeAlpha;
+ break;
+ case 'b':
+ type = eSraReleaseVersionTypeBeta;
+ break;
+ case 'r':
+ type = eSraReleaseVersionTypeRC;
+ ++p;
+ --size;
+ if (size == 0) {
+ rc = RC(rcRuntime,
+ rcFile, rcReading, rcFile, rcInvalid);
+ }
+ else if (*p != 'c') {
+ rc = RC(rcRuntime,
+ rcFile, rcReading, rcFile, rcInvalid);
+ }
+ break;
+ }
+ ++p;
+ --size;
+ }
+ }
+
+ /* revision: 1.2.3-4 1.2.3-b4 1.2.3-rc4 */
+ if (size > 0 && rc == 0 && *p != '\n') {
+ i = getdigit(&p, &size, 0);
+ if (i >= 0) {
+ revision = i;
+ }
+ else {
+ rc = SET_RC(i);
+ }
+ }
+
+ if (size > 0 && rc == 0 && *p != '\n') {
+ rc = RC(rcRuntime, rcFile, rcReading, rcFile, rcInvalid);
+ }
+
+ if (rc == 0) {
+ self->version = (major << 24) + (minor << 16) + release;
+ self->revision = revision;
+ self->type = type;
+ }
+
+ return rc;
+}
+
+LIB_EXPORT rc_t CC SraReleaseVersionPrint ( const SraReleaseVersion *self,
+ char *version, size_t size, size_t *num_writ )
+{
+ rc_t rc = 0;
+ const char *type = "";
+
+ if (self == NULL) {
+ return RC(rcRuntime, rcArgv, rcAccessing, rcSelf, rcNull);
+ }
+
+ switch (self->type) {
+ case eSraReleaseVersionTypeAlpha:
+ type = "-a";
+ break;
+ case eSraReleaseVersionTypeBeta:
+ type = "-b";
+ break;
+ case eSraReleaseVersionTypeRC:
+ type = "-rc";
+ break;
+ case eSraReleaseVersionTypeFinal:
+ if (self->revision == 0) {
+ type = "";
+ }
+ else {
+ type = "-";
+ }
+ break;
+ default:
+ assert(0);
+ break;
+ }
+
+ if (self->revision == 0) {
+ rc = string_printf(version, size, num_writ,
+ "%V%s", self->version, type);
+ }
+ else {
+ rc = string_printf(version, size, num_writ,
+ "%V%s%d", self->version, type, self->revision);
+ }
+
+ return rc;
+}
diff --git a/libs/klib/win/syserrcode.c b/libs/klib/win/syserrcode.c
index 9176d92..0c64db3 100644
--- a/libs/klib/win/syserrcode.c
+++ b/libs/klib/win/syserrcode.c
@@ -73,7 +73,7 @@ size_t KWrtFmt_error_code ( char * buffer, size_t buffer_size, uint32_t error_co
{
if (*pc == '\r')
{
- tbuffer_size = pc - tbuffer;
+ tbuffer_size = ( DWORD ) ( pc - tbuffer );
break;
}
}
diff --git a/libs/klib/win/systime.c b/libs/klib/win/systime.c
index 30b7fc7..913fbaa 100644
--- a/libs/klib/win/systime.c
+++ b/libs/klib/win/systime.c
@@ -105,7 +105,6 @@ int SYSTEMTIME_compare ( const SYSTEMTIME *a, const SYSTEMTIME *b )
static
void SYSTEMTIME_from_half_baked_SYSTEMTIME ( const SYSTEMTIME *half_baked, SYSTEMTIME *proper, WORD year )
{
- int i;
FILETIME ft;
* proper = * half_baked;
diff --git a/libs/klib/win/syswriter.c b/libs/klib/win/syswriter.c
index b204afc..f365702 100644
--- a/libs/klib/win/syswriter.c
+++ b/libs/klib/win/syswriter.c
@@ -63,12 +63,12 @@ LIB_EXPORT int CC snprintf ( char * buffer, size_t bufsize, const char * format,
rc = string_vprintf ( buffer, bufsize, &size, format, args );
if ( rc == 0 )
{
- ret = size;
+ ret = ( int ) size;
}
else
{
if ( ( GetRCState( rc ) == rcInsufficient )&&( GetRCObject( rc ) == rcBuffer ) )
- ret = size;
+ ret = ( int ) size;
else
ret = -1;
}
@@ -94,7 +94,7 @@ rc_t CC KWrt_DefaultWriter( void * self, const char * buffer, size_t bufsize, si
rc = 0;
total = 0;
- for (remaining = bufsize; remaining > 0; remaining -= num_written)
+ for (remaining = ( DWORD ) bufsize; remaining > 0; remaining -= num_written)
{
if (WriteFile (handle, pbyte, remaining, &num_written, NULL) == 0)
{
diff --git a/libs/kns/KCurlFile.c b/libs/kns/KCurlFile.c
index 18ff995..243fd40 100644
--- a/libs/kns/KCurlFile.c
+++ b/libs/kns/KCurlFile.c
@@ -164,7 +164,7 @@ const char *curly_errstr ( CURLcode code )
CURL_CASE2 ( CURLE_SSH, 79 );
CURL_CASE2 ( CURLE_SSL_SHUTDOWN_FAILED, 80 );
CURL_CASE2 ( CURLE_AGAIN, 81 );
- CURL_CASE ( CURLE_SSL_CRL_BADFILE );
+ CURL_CASE2 ( CURLE_SSL_CRL_BADFILE, 82 );
CURL_CASE2 ( CURLE_SSL_ISSUER_ERROR, 83 );
}
@@ -379,7 +379,7 @@ static rc_t get_content_length( struct KNSManager *kns_mgr, CURL *curl, uint64_t
}
else
{
- *size = double_size;
+ *size = (uint64_t) double_size;
}
return rc;
}
diff --git a/libs/kns/Makefile b/libs/kns/Makefile
index 3f21b8c..d8969b2 100644
--- a/libs/kns/Makefile
+++ b/libs/kns/Makefile
@@ -78,6 +78,7 @@ $(ILIBDIR)/libkns: $(addprefix $(ILIBDIR)/libkns.,$(ILIBEXT))
KNS_NO_HTTP_SRC = \
kns_manager \
+ kns_manager-ext \
stream \
stream-from-files \
stream-from-streams \
diff --git a/libs/kns/http-priv.h b/libs/kns/http-priv.h
index 2cd0c44..bf5a218 100644
--- a/libs/kns/http-priv.h
+++ b/libs/kns/http-priv.h
@@ -78,7 +78,7 @@ struct KHttpHeader
};
extern void KHttpHeaderWhack ( BSTNode *n, void *ignore );
-extern rc_t KHttpGetHeaderLine ( struct KHttp *self, BSTree *hdrs, bool *blank );
+extern rc_t KHttpGetHeaderLine ( struct KHttp *self, BSTree *hdrs, bool *blank, bool *close_connection );
extern rc_t KHttpGetStatusLine ( struct KHttp *self, String *msg, uint32_t *status, ver_t *version );
/* exported private functions
diff --git a/libs/kns/http-test.c b/libs/kns/http-test.c
index 15e949a..87539a5 100644
--- a/libs/kns/http-test.c
+++ b/libs/kns/http-test.c
@@ -206,7 +206,7 @@ rc_t HttpTest ( const KFile *input )
struct KHttp *http;
String host;
- CONST_STRING ( & host, "http://www.ncbi.nlm.nih.gov/" );
+ CONST_STRING ( & host, "www.ncbi.nlm.nih.gov" );
rc = KNSManagerMakeHttp ( mgr, & http, sock, 0x01010000, & host, 80 );
if ( rc == 0 )
@@ -220,15 +220,15 @@ rc_t HttpTest ( const KFile *input )
OUTMSG (( "%s: KHttpGetStatusLine failed with rc=%R\n", __func__, rc ));
else
{
- bool blank;
+ bool blank, close_connection;
BSTree hdrs;
OUTMSG (( "%s: KHttpGetStatusLine returned msg='%S', status=%u, version=%V\n",
__func__, & msg, status, version ));
BSTreeInit ( & hdrs );
- for ( blank = false; ! blank && rc == 0; )
- rc = KHttpGetHeaderLine ( http, & hdrs, & blank );
+ for ( blank = close_connection = false; ! blank && rc == 0; )
+ rc = KHttpGetHeaderLine ( http, & hdrs, & blank, & close_connection );
if ( rc != 0 )
OUTMSG (( "%s: KHttpGetHeaderLine failed with rc=%R\n", __func__, rc ));
diff --git a/libs/kns/http.c b/libs/kns/http.c
index 1c2a4fd..cb963d8 100644
--- a/libs/kns/http.c
+++ b/libs/kns/http.c
@@ -41,6 +41,8 @@ typedef struct KHttpFile KHttpFile;
#include <kns/impl.h>
#include <kfs/file.h>
#include <kfs/directory.h>
+
+#include <klib/debug.h> /* DBGMSG */
#include <klib/text.h>
#include <klib/container.h>
#include <klib/out.h>
@@ -49,6 +51,7 @@ typedef struct KHttpFile KHttpFile;
#include <klib/rc.h>
#include <klib/printf.h>
#include <klib/vector.h>
+
#include <strtol.h>
#include <va_copy.h>
@@ -425,6 +428,9 @@ struct KHttp
ver_t vers;
KRefcount refcount;
+
+ KEndPoint ep;
+ bool ep_valid;
};
@@ -436,6 +442,13 @@ struct KHttp
#define KHttpLineBufferReset( self ) \
( ( void ) ( ( self ) -> line_valid = 0 ) )
+
+static
+void KHttpClose ( KHttp *self )
+{
+ KStreamRelease ( self -> sock );
+ self -> sock = NULL;
+}
/* used to be in whack function, but we needed the ability
@@ -443,8 +456,7 @@ struct KHttp
static
rc_t KHttpClear ( KHttp *self )
{
- KStreamRelease ( self -> sock );
- self -> sock = NULL;
+ KHttpClose ( self );
KHttpBlockBufferReset ( self );
KHttpLineBufferReset ( self );
@@ -468,6 +480,31 @@ rc_t KHttpWhack ( KHttp * self )
return 0;
}
+static
+rc_t KHttpOpen ( KHttp * self, const String * hostname, uint32_t port )
+{
+ rc_t rc;
+
+ if ( ! self -> ep_valid )
+ {
+ rc = KNSManagerInitDNSEndpoint ( self -> mgr, & self -> ep, hostname, port );
+ if ( rc != 0 )
+ return rc;
+
+ self -> ep_valid = true;
+ }
+
+ rc = KNSManagerMakeConnection ( self -> mgr, & self -> sock, NULL, & self -> ep );
+ if ( rc == 0 )
+ {
+ self -> port = port;
+ return 0;
+ }
+
+ self -> sock = NULL;
+ return rc;
+}
+
/* Initialize KHttp object */
static
rc_t KHttpInit ( KHttp * http, const KDataBuffer *hostname_buffer, KStream * conn, ver_t _vers, const String * _host, uint32_t port )
@@ -482,14 +519,7 @@ rc_t KHttpInit ( KHttp * http, const KDataBuffer *hostname_buffer, KStream * con
rc = KStreamAddRef ( conn );
else
{
- /* KEndPoint holds an address and port number */
- KEndPoint to;
-
- /* make a KEndPoint using a host name */
- rc = KNSManagerInitDNSEndpoint ( http -> mgr, &to, _host, port );
- if ( rc == 0 )
- /* create the connection using the system default for from */
- rc = KNSMakeConnection ( &conn, NULL, &to );
+ rc = KHttpOpen ( http, _host, port );
}
if ( rc == 0 )
@@ -791,8 +821,13 @@ rc_t KHttpGetLine ( KHttp *self )
/* get out of loop if end of line */
if ( ch == 0 )
+ {
+#if _DEBUGGING
+ if ( KNSManagerIsVerbose ( self->mgr ) )
+ KOutMsg( "RX:%s\n", buffer );
+#endif
break;
-
+ }
/* not end of line - increase num of valid bytes read */
++ self -> line_valid;
}
@@ -909,7 +944,7 @@ rc_t KHttpVAddHeader ( BSTree *hdrs, const char *_name, const char *_val, va_lis
size_t blen = string_len ( buf, bsize );
/* init value */
- StringInit ( & value, buf, bsize, blen );
+ StringInit ( & value, buf, bsize, ( uint32_t ) blen );
rc = KHttpAddHeaderString ( hdrs, & name, & value );
}
@@ -929,7 +964,7 @@ rc_t KHttpAddHeader ( BSTree *hdrs, const char *name, const char *val, ... )
}
/* Capture each header line to add to BSTree */
-rc_t KHttpGetHeaderLine ( KHttp *self, BSTree *hdrs, bool *blank )
+rc_t KHttpGetHeaderLine ( KHttp *self, BSTree *hdrs, bool *blank, bool *close_connection )
{
/* Starting from the second line of the response */
rc_t rc = KHttpGetLine ( self );
@@ -975,6 +1010,21 @@ rc_t KHttpGetHeaderLine ( KHttp *self, BSTree *hdrs, bool *blank )
/* assign the value data into the value string */
StringInit ( & value, buffer, last - buffer, ( uint32_t ) ( last - buffer ) );
+ /* check for Connection: close header */
+ if ( name . size == sizeof "Connection" -1 && value . size == sizeof "close" - 1 )
+ {
+ if ( tolower ( name . addr [ 0 ] ) == 'c' && tolower ( value . addr [ 0 ] ) == 'c' )
+ {
+ if ( strcase_cmp ( name . addr, name . size, "Connection", name . size, ( uint32_t ) name . size ) == 0 &&
+ strcase_cmp ( value . addr, value . size, "close", value . size, ( uint32_t ) value . size ) == 0 )
+ {
+ DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
+ ("*** seen connection close ***\n"));
+ * close_connection = true;
+ }
+ }
+ }
+
rc = KHttpAddHeaderString ( hdrs, & name, & value );
}
}
@@ -996,7 +1046,9 @@ rc_t KHttpFindHeader ( const BSTree *hdrs, const char *_name, char *buffer, size
/* find the header */
node = ( KHttpHeader * ) BSTreeFind ( hdrs, &name, KHttpHeaderCmp );
if ( node == NULL )
+ {
rc = RC ( rcNS, rcNoTarg, rcSearching, rcName, rcNull );
+ }
else
{
/* make sure buffer is large enough */
@@ -1028,7 +1080,9 @@ rc_t KHttpGetStatusLine ( KHttp *self, String *msg, uint32_t *status, ver_t *ver
expect HTTP/1.[01]<sp><digit>+<sp><msg>\r\n */
sep = string_chr ( buffer, end - buffer, '/' );
if ( sep == NULL )
+ {
rc = RC ( rcNS, rcNoTarg, rcParsing, rcNoObj, rcNotFound );
+ }
else
{
/* make sure it is http */
@@ -1396,12 +1450,19 @@ struct KHttpResult
ver_t version;
KRefcount refcount;
+ bool close_connection;
};
static
rc_t KHttpResultWhack ( KHttpResult * self )
{
BSTreeWhack ( & self -> hdrs, KHttpHeaderWhack, NULL );
+ if ( self -> close_connection )
+ {
+ DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
+ ("*** closing connection ***\n"));
+ KHttpClose ( self -> http );
+ }
KHttpRelease ( self -> http );
KRefcountWhack ( & self -> refcount, "KHttpResult" );
free ( self );
@@ -1414,15 +1475,25 @@ static
rc_t KHttpSendReceiveMsg ( KHttp *self, KHttpResult **rslt,
const char *buffer, size_t len, const KDataBuffer *body, const char *url )
{
- rc_t rc;
+ rc_t rc = 0;
size_t sent;
/* TBD - may want to assert that there is an empty line in "buffer" */
+#if _DEBUGGING
+ if ( KNSManagerIsVerbose ( self->mgr ) )
+ KOutMsg( "TX:%.*s", len, buffer );
+#endif
+
+ /* reopen connection if NULL */
+ if ( self -> sock == NULL )
+ rc = KHttpOpen ( self, & self -> hostname, self -> port );
/* ALWAYS want to use write all when sending */
- rc = KStreamWriteAll ( self -> sock, buffer, len, & sent );
+ if ( rc == 0 )
+ rc = KStreamWriteAll ( self -> sock, buffer, len, & sent );
+
/* check the data was completely sent */
if ( rc == 0 && sent != len )
rc = RC ( rcNS, rcNoTarg, rcWriting, rcTransfer, rcIncomplete );
@@ -1481,7 +1552,7 @@ rc_t KHttpSendReceiveMsg ( KHttp *self, KHttpResult **rslt,
/* receive and parse all header lines
blank = end of headers */
for ( blank = false; ! blank && rc == 0; )
- rc = KHttpGetHeaderLine ( self, & result -> hdrs, & blank );
+ rc = KHttpGetHeaderLine ( self, & result -> hdrs, & blank, & result -> close_connection );
if ( rc == 0 )
{
@@ -1677,7 +1748,7 @@ rc_t KHttpResultHandleContentRange ( const KHttpResult *self, uint64_t *pos, siz
/* look for separation of 'bytes' and first position */
sep = string_chr ( buf, end - buf, ' ' );
- if ( sep != NULL )
+ if ( sep == NULL )
rc = RC ( rcNS, rcNoTarg, rcParsing, rcNoObj, rcNotFound );
else
{
@@ -1746,8 +1817,8 @@ rc_t KHttpResultHandleContentRange ( const KHttpResult *self, uint64_t *pos, siz
else
{
/* finally check all the acquired information */
- if ( length != end_pos - start_pos ||
- length > total )
+ if ( ( length != ( ( end_pos - start_pos ) + 1 ) ) ||
+ ( length > total ) )
rc = RC ( rcNS, rcNoTarg, rcParsing, rcNoObj, rcNotFound );
else
{
@@ -2455,6 +2526,7 @@ rc_t KHttpRequestFormatMsg ( const KHttpRequest *self,
sending it in its own header */
/* TBD - should we include the port with the host name? */
+ #if 0
rc = string_printf ( buffer, bsize, len,
"%s %S://%S%S%s%S HTTP/%.2V\r\n"
, method
@@ -2464,6 +2536,16 @@ rc_t KHttpRequestFormatMsg ( const KHttpRequest *self,
, has_query
, & self -> url_block . query
, http -> vers );
+ #else
+ rc = string_printf ( buffer, bsize, len,
+ "%s %S%s%S HTTP/%.2V\r\nHost: %S\r\nAccept: */*\r\n"
+ , method
+ , & self -> url_block . path
+ , has_query
+ , & self -> url_block . query
+ , http -> vers
+ , & hostname );
+ #endif
/* print all headers remaining into buffer */
@@ -2529,6 +2611,9 @@ rc_t KHttpRequestHandleRedirection ( KHttpRequest *self, KHttpResult *rslt )
/* close the open http connection and clear out all data except for the manager */
KHttpClear ( http );
+ /* clear the previous endpoint */
+ http -> ep_valid = false;
+
/* reinitialize the http from uri */
rc = KHttpInit ( http, &uri, NULL, http -> vers , &b . host, b . port );
if ( rc == 0 )
@@ -2764,11 +2849,11 @@ struct KHttpFile
static
rc_t KHttpFileDestroy ( KHttpFile *self )
{
- KHttpWhack ( self -> http );
- KDataBufferWhack ( & self -> url_buffer );
- free ( self );
+ KHttpRelease ( self -> http );
+ KDataBufferWhack ( & self -> url_buffer );
+ free ( self );
- return 0;
+ return 0;
}
static
diff --git a/libs/kns/kns_manager-ext.c b/libs/kns/kns_manager-ext.c
new file mode 100644
index 0000000..cba8cb8
--- /dev/null
+++ b/libs/kns/kns_manager-ext.c
@@ -0,0 +1,140 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <kns/extern.h>
+
+#include <kns/manager-ext.h>
+#include <kns/http.h>
+#include <kns/stream.h>
+
+#include <klib/sra-release-version.h>
+#include <klib/data-buffer.h>
+#include <klib/debug.h>
+#include <klib/rc.h>
+
+#include <ctype.h>
+#include <string.h>
+#include <assert.h>
+
+
+/** These functions belong to Extended KNSManager **/
+
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
+
+LIB_EXPORT rc_t CC KNSManagerNewReleaseVersion(const struct KNSManager *self,
+ SraReleaseVersion *newVersion)
+{
+ rc_t rc = 0;
+ KDataBuffer result;
+ KHttpRequest *req = NULL;
+ KHttpResult *rslt = NULL;
+ if (newVersion == NULL) {
+ return RC(rcNS, rcArgv, rcAccessing, rcParam, rcNull);
+ }
+ memset(newVersion, 0, sizeof *newVersion);
+ if (self == NULL) {
+ return RC(rcNS, rcArgv, rcAccessing, rcSelf, rcNull);
+ }
+ memset(&result, 0, sizeof result);
+ if (rc == 0) {
+ rc = KNSManagerMakeRequest(self, &req, 0x01000000, NULL,
+ "http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
+ );
+ }
+ if (rc == 0) {
+ rc = KHttpRequestGET(req, &rslt);
+ }
+ if (rc == 0) {
+ uint32_t code = 0;
+ rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL);
+ if (rc == 0) {
+ if (code != 200) {
+ rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid);
+ }
+ }
+ }
+ if (rc == 0) {
+ size_t total = 0;
+ KStream *response = NULL;
+ rc = KHttpResultGetInputStream(rslt, &response);
+ if (rc == 0) {
+ rc = KDataBufferMakeBytes(&result, 1024);
+ }
+ while (rc == 0) {
+ size_t num_read = 0;
+ uint8_t *base = NULL;
+ uint64_t avail = result.elem_count - total;
+ if (avail < 256) {
+ rc = KDataBufferResize(&result, result.elem_count + 1024);
+ if (rc != 0) {
+ break;
+ }
+ }
+ base = result.base;
+ rc = KStreamRead(response, &base[total], result.elem_count - total,
+ &num_read);
+ if (num_read > 0 || rc != 0) {
+ DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("KStreamRead"
+ "(sratoolkit.current.version, %zu) = %R\n", num_read, rc));
+ }
+ if (rc != 0) {
+ /* TBD - look more closely at rc */
+ if (num_read > 0) {
+ rc = 0;
+ }
+ else {
+ break;
+ }
+ }
+ if (num_read == 0) {
+ break;
+ }
+ total += num_read;
+ }
+ RELEASE(KStream, response);
+ if (rc == 0) {
+ DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
+ ("sratoolkit.current.version (%zu)\n", total));
+ result.elem_count = total;
+ }
+ }
+
+ if (rc == 0) {
+ const char *start = (const void*)(result.base);
+ size_t size = KDataBufferBytes(&result);
+ DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
+ ("sratoolkit.current.version = '%.*s'\n", (uint32_t)size, start));
+ rc = SraReleaseVersionInit(newVersion, start, size);
+ }
+
+ KDataBufferWhack(&result);
+ RELEASE(KHttpResult, rslt);
+ RELEASE(KHttpRequest, req);
+
+ return rc;
+}
diff --git a/libs/kns/kns_manager.c b/libs/kns/kns_manager.c
index bfd1493..31ce2f0 100644
--- a/libs/kns/kns_manager.c
+++ b/libs/kns/kns_manager.c
@@ -97,6 +97,17 @@ static rc_t KNSManager_Make_DlCurl( KDyld ** dl )
static rc_t KNSManagerLoadLib( struct KNSManager *self )
{
+#ifdef HAVE_LIBCURL /* just link directly */
+ self->curl_easy_init_fkt = &curl_easy_init;
+ self->curl_easy_cleanup_fkt = &curl_easy_cleanup;
+ self->curl_easy_setopt_fkt = &curl_easy_setopt;
+ self->curl_easy_perform_fkt = &curl_easy_perform;
+ self->curl_easy_getinfo_fkt = &curl_easy_getinfo;
+ self->curl_slist_append_fkt = &curl_slist_append;
+ self->curl_slist_free_all_fkt = &curl_slist_free_all;
+
+ return 0;
+#else
KDyld *dl;
/* make a dynamic-library loader */
rc_t rc = KDyldMake ( &dl );
@@ -201,7 +212,7 @@ static rc_t KNSManagerLoadLib( struct KNSManager *self )
}
return rc;
-
+#endif
}
@@ -268,6 +279,22 @@ LIB_EXPORT rc_t CC KNSManagerCurlVersion( const struct KNSManager *self, const c
}
+LIB_EXPORT void KNSManagerSetVerbose ( struct KNSManager *self, bool verbosity )
+{
+ if ( self != NULL )
+ self->verbose = verbosity;
+}
+
+
+LIB_EXPORT bool KNSManagerIsVerbose ( struct KNSManager *self )
+{
+ if ( self != NULL )
+ return self->verbose;
+ else
+ return false;
+}
+
+
static rc_t KNSManagerDestroy( struct KNSManager *self )
{
if ( self == NULL )
diff --git a/libs/kns/kns_mgr_priv.h b/libs/kns/kns_mgr_priv.h
index d2165f3..4d4adc2 100644
--- a/libs/kns/kns_mgr_priv.h
+++ b/libs/kns/kns_mgr_priv.h
@@ -45,7 +45,7 @@ struct KNSManager
KRefcount refcount;
rc_t create_rc;
-
+
/* curl-easy-function-pointers... */
CURL* ( CC * curl_easy_init_fkt ) ( void );
void ( CC * curl_easy_cleanup_fkt ) ( CURL * handle );
@@ -55,10 +55,14 @@ struct KNSManager
char * ( CC * curl_version_fkt ) ( void );
struct curl_slist* ( CC * curl_slist_append_fkt ) ( struct curl_slist * list, const char * string );
void ( CC * curl_slist_free_all_fkt ) ( struct curl_slist * list );
+
+ bool verbose;
};
- rc_t KNSManagerInit ( struct KNSManager *self );
- void KNSManagerCleanup ( struct KNSManager *self );
+
+rc_t KNSManagerInit ( struct KNSManager *self ); /* in kns/unix/sysmgr.c or kns/win/sysmgr.c */
+void KNSManagerCleanup ( struct KNSManager *self );
+
#ifdef __cplusplus
}
diff --git a/libs/kns/libkns.vers b/libs/kns/libkns.vers
deleted file mode 100644
index 3eefcb9..0000000
--- a/libs/kns/libkns.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.0
diff --git a/libs/kns/libkurl.vers b/libs/kns/libkurl.vers
deleted file mode 100644
index 7ee7020..0000000
--- a/libs/kns/libkurl.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.10
diff --git a/libs/kns/unix/sysendpoint.c b/libs/kns/unix/sysendpoint.c
index 2902cc5..456f336 100644
--- a/libs/kns/unix/sysendpoint.c
+++ b/libs/kns/unix/sysendpoint.c
@@ -56,7 +56,7 @@ LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
KEndPoint *ep, struct String const *dns, uint16_t port )
{
- rc_t rc;
+ rc_t rc = 0;
if ( ep == NULL )
rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
@@ -70,28 +70,38 @@ rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
else
{
- size_t size;
- char hostname [ 4096 ];
- if ( dns -> size >= sizeof hostname )
+ KDataBuffer b;
+ char buffer [ 4096 ], * hostname = buffer;
+ size_t buff_size = sizeof buffer;
+
+ if ( dns -> size >= sizeof buffer )
{
- KDataBuffer b;
rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
if ( rc == 0 )
{
- struct hostent *remote;
- char *host = b . base;
+ hostname = b . base;
+ buff_size = ( size_t ) b . elem_count;
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ size_t size;
+ rc = string_printf ( hostname, buff_size, & size, "%S", dns );
- rc = string_printf ( host, ( size_t ) b . elem_count, & size, "%S", dns );
- assert ( size < ( size_t ) b . elem_count );
- assert ( host [ size ] == 0 );
- remote = gethostbyname ( host );
+ assert ( rc == 0 );
+ assert ( size < buff_size );
+ assert ( hostname [ size ] == 0 );
+
+ if ( rc == 0 )
+ {
+ struct hostent *remote = gethostbyname ( hostname );
if ( remote != NULL )
{
- ep -> type = epIPV4;
- memcpy ( &ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
+ ep -> type = epIPV4;
+ memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
+ ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
ep -> u . ipv4 . port = ( uint16_t ) port;
- KDataBufferWhack ( & b );
- return 0;
}
else switch ( h_errno )
{
@@ -115,55 +125,16 @@ rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
default :
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
}
-
}
}
- else
- {
- struct hostent *remote;
- rc = string_printf ( hostname, sizeof hostname, & size, "%S", dns );
- assert ( size < sizeof hostname );
- assert ( hostname [ size ] == 0 );
- remote = gethostbyname ( hostname );
- if ( remote != NULL )
- {
- ep -> type = epIPV4;
- memcpy ( &ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
- ep -> u . ipv4 . port = ( uint16_t ) port;
- return 0;
- }
- else switch ( h_errno )
- {
- case HOST_NOT_FOUND: /* The specified host is unknown */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
- break;
- case NO_ADDRESS: /* The requested names valid but does not have an IP address */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcInconsistent );
- break;
-#if ! defined NO_ADDRESS || ! defined NO_DATA || NO_ADDRESS != NO_DATA
- case NO_DATA: /* The requested name s valid but does not have an IP address */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
- break;
-#endif
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
- break;
- case NO_RECOVERY: /* A nonrecoverable name server error occured */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
- break;
- case TRY_AGAIN: /* A temporary error occured on an authoritative name server. Try again later */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
- break;
- default :
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
- }
- }
+ if ( hostname != buffer )
+ KDataBufferWhack ( & b );
}
- memset ( ep, 0, sizeof * ep );
+ if ( rc != 0 )
+ memset ( ep, 0, sizeof * ep );
}
return rc;
-
-
}
diff --git a/libs/kns/unix/syssock.c b/libs/kns/unix/syssock.c
index 9244ba5..037a8e6 100644
--- a/libs/kns/unix/syssock.c
+++ b/libs/kns/unix/syssock.c
@@ -29,7 +29,6 @@
* forwards
*/
#define KSTREAM_IMPL KSocket
-typedef struct KSocket KSocket;
#include <kns/extern.h>
#include <kns/manager.h>
@@ -40,6 +39,7 @@ typedef struct KSocket KSocket;
#include <klib/log.h>
#include <klib/text.h>
#include <klib/printf.h>
+#include <klib/out.h>
#include "stream-priv.h"
@@ -52,7 +52,10 @@ typedef struct KSocket KSocket;
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
+#include <netinet/tcp.h>
+#include <poll.h>
#include <unistd.h>
+#include <fcntl.h>
#include <errno.h>
#include <pwd.h>
@@ -97,7 +100,7 @@ rc_t CC KSocketWhack ( KSocket *self )
while ( 1 )
{
char buffer [ 1024 ];
- ssize_t result = recv ( self -> fd, buffer, sizeof buffer, 0 );
+ ssize_t result = recv ( self -> fd, buffer, sizeof buffer, MSG_DONTWAIT );
if ( result <= 0 )
break;
}
@@ -117,7 +120,7 @@ rc_t CC KSocketWhack ( KSocket *self )
}
static
-rc_t HandleErrno()
+rc_t HandleErrno ( const char *func_name, int lineno )
{
int lerrno;
rc_t rc = 0;
@@ -232,28 +235,95 @@ rc_t HandleErrno()
(klogErr, rc, "unknown system error '$(S)($(E))'",
"S=%!,E=%d", lerrno, lerrno));
}
+
+#if _DEBUGGING
+ if ( rc != 0 )
+ pLogMsg ( klogInfo, "$(RC)\n", "RC=%R", rc );
+#endif
return rc;
}
static
+int socket_wait ( int fd, int events, int timeout )
+{
+ int i;
+ struct pollfd fds [ 1 ];
+
+ /* poll for data with no delay */
+ for ( i = 0; i < 2; ++ i )
+ {
+ fds [ 0 ] . fd = fd;
+ fds [ 0 ] . events = events;
+ fds [ 0 ] . revents = 0;
+
+ if ( poll ( fds, sizeof fds / sizeof fds [ 0 ], 0 ) != 0 )
+ return fds [ 0 ] . revents;
+ }
+
+ /* poll for data */
+ if ( timeout != 0 )
+ {
+ fds [ 0 ] . fd = fd;
+ fds [ 0 ] . events = events;
+ fds [ 0 ] . revents = 0;
+
+ if ( poll ( fds, sizeof fds / sizeof fds [ 0 ], timeout ) != 0 )
+ {
+ /* experiment - try another poll with no timeout */
+ fds [ 0 ] . fd = fd;
+ fds [ 0 ] . events = events;
+ fds [ 0 ] . revents = 0;
+ poll ( fds, sizeof fds / sizeof fds [ 0 ], 0 );
+
+ return fds [ 0 ] . revents;
+ }
+ }
+
+ return 0;
+}
+
+static
rc_t CC KSocketRead ( const KSocket *self,
void *buffer, size_t bsize, size_t *num_read )
{
rc_t rc = 0;
+ int timeout = 1000;
+
+ size_t total = 0;
+ char *b = buffer;
+
assert ( self != NULL );
- while ( rc == 0 )
+ while ( 1 )
{
- ssize_t count = recv ( self -> fd, buffer, bsize, 0 );
-
- if ( count >= 0 )
+ ssize_t count = 0;
+
+ /* wait for socket to become readable */
+ int revents = socket_wait ( self -> fd, POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND, timeout );
+ if ( revents != 0 )
+ count = recv ( self -> fd, & b [ total ], bsize - total, 0 );
+ if ( count <= 0 )
{
- assert ( num_read != NULL );
- * num_read = ( size_t ) count;
- return 0;
+ if ( total != 0 || count == 0 )
+ {
+ assert ( num_read != NULL );
+ * num_read = total;
+ return 0;
+ }
+
+ if ( errno != EAGAIN )
+ {
+ rc = HandleErrno ( __func__, __LINE__ );
+ if ( rc != 0 )
+ break;
+ }
+ }
+ else
+ {
+ total += count;
+ timeout = 0;
}
- rc = HandleErrno();
}
return rc;
@@ -268,14 +338,20 @@ rc_t CC KSocketWrite ( KSocket *self,
while ( rc == 0 )
{
- ssize_t count = send ( self -> fd, buffer, bsize, 0 );
+ ssize_t count;
+
+ /* wait for socket to become writable */
+ socket_wait ( self -> fd, POLLOUT | POLLWRNORM | POLLWRBAND, 1000 );
+
+ count = send ( self -> fd, buffer, bsize, 0 );
if ( count >= 0 )
{
assert ( num_writ != NULL );
* num_writ = count;
return 0;
}
- rc = HandleErrno();
+
+ rc = HandleErrno ( __func__, __LINE__ );
}
return rc;
@@ -302,11 +378,14 @@ rc_t MakeSocketPath(const char* name, char* buf, size_t buf_size)
}
LIB_EXPORT
-rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoint *to )
+rc_t CC KNSManagerMakeConnection ( struct KNSManager const * self,KStream **out, const KEndPoint *from, const KEndPoint *to )
{
rc_t rc;
int fd;
+ if ( self == NULL )
+ return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
+
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
@@ -322,6 +401,7 @@ rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoin
fd = socket ( AF_INET, SOCK_STREAM, 0 );
if ( fd != -1 )
{
+ int flag;
struct sockaddr_in ss;
memset ( &ss, 0, sizeof ss );
ss . sin_family = AF_INET;
@@ -330,13 +410,19 @@ rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoin
ss . sin_port = htons ( from -> u . ipv4 . port );
ss . sin_addr . s_addr = htonl ( from -> u . ipv4 . addr );
}
- if ( bind ( fd, (struct sockaddr *)&ss, sizeof ss ) == 0)
+
+ /* disable nagle algorithm */
+ flag = 1;
+ setsockopt ( fd, IPPROTO_TCP, TCP_NODELAY, ( char* ) & flag, sizeof flag );
+
+
+ if ( from == NULL || bind ( fd, ( struct sockaddr *) & ss, sizeof ss ) == 0)
{
memset ( & ss, 0, sizeof ss );
ss . sin_family = AF_INET;
ss . sin_port = htons ( to -> u . ipv4 . port );
ss . sin_addr . s_addr = htonl ( to -> u . ipv4 . addr );
- if (connect ( fd, (struct sockaddr *)&ss, sizeof ss ) == 0 )
+ if ( connect ( fd, (struct sockaddr *)&ss, sizeof ss ) == 0 )
{
/* create the KSocket */
KSocket *ksock = calloc ( sizeof *ksock, 1 );
@@ -349,6 +435,10 @@ rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoin
"KSocket", "tcp", true, true );
if ( rc == 0 )
{
+ /* set non-blocking mode */
+ flag = fcntl ( fd, F_GETFL );
+ fcntl ( fd, F_SETFL, flag | O_NONBLOCK );
+
ksock -> fd = fd;
*out = & ksock -> dad;
return 0;
@@ -358,24 +448,27 @@ rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoin
}
}
else /* connect() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
}
else /* bind() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
close(fd);
}
else /* socket() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
return rc;
}
LIB_EXPORT
-rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_retries )
+rc_t CC KNSManagerMakeIPCConnection ( struct KNSManager const *self, KStream **out, const KEndPoint *to, uint32_t max_retries )
{
rc_t rc;
- uint8_t retry_count = 0;
+ uint32_t retry_count = 0;
struct sockaddr_un ss;
+ if ( self == NULL )
+ return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
+
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
@@ -416,7 +509,7 @@ rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_r
}
else
{
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
close(fd);
if ( retry_count < max_retries &&
@@ -430,18 +523,21 @@ rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_r
}
}
else /* socket() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
break;
}
return rc;
}
LIB_EXPORT
-rc_t CC KNSMakeListener( struct KSocket** out, struct KEndPoint const * ep )
+rc_t CC KNSManagerMakeListener( struct KNSManager const *self, struct KSocket** out, struct KEndPoint const * ep )
{
int fd;
rc_t rc;
+ if ( self == NULL )
+ return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
+
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
@@ -492,29 +588,29 @@ rc_t CC KNSMakeListener( struct KSocket** out, struct KEndPoint const * ep )
}
}
else /* bind() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
}
close(fd);
}
else /* socket() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
return rc;
}
LIB_EXPORT
-rc_t CC KNSListen ( struct KSocket *listener, struct KStream **out )
+rc_t CC KSocketListen ( struct KSocket *listener, struct KStream **out )
{
rc_t rc;
+ if ( listener == NULL )
+ return RC ( rcNS, rcNoTarg, rcValidating, rcSelf, rcNull);
+
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
* out = NULL;
- if ( listener == NULL )
- return RC ( rcNS, rcNoTarg, rcValidating, rcParam, rcNull);
-
if ( listen(listener->fd, 5) == 0)
{
struct sockaddr_un remote;
@@ -541,10 +637,10 @@ rc_t CC KNSListen ( struct KSocket *listener, struct KStream **out )
close(fd);
}
else /* accept() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
}
else /* listen() failed */
- rc = HandleErrno();
+ rc = HandleErrno( __func__, __LINE__ );
return rc;
}
diff --git a/libs/kns/win/sysendpoint.c b/libs/kns/win/sysendpoint.c
index 67935ac..e7c3374 100644
--- a/libs/kns/win/sysendpoint.c
+++ b/libs/kns/win/sysendpoint.c
@@ -53,7 +53,7 @@ LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
KEndPoint *ep, struct String const *dns, uint16_t port )
{
- rc_t rc;
+ rc_t rc = 0;
if ( ep == NULL )
rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
@@ -67,30 +67,39 @@ rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
else
{
- int lerrno;
- size_t size;
- char hostname [ 4096 ];
- if ( dns -> size >= sizeof hostname )
+ KDataBuffer b;
+ char buffer [ 4096 ], * hostname = buffer;
+ size_t buff_size = sizeof buffer;
+
+ if ( dns -> size >= sizeof buffer )
{
- KDataBuffer b;
-
rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
if ( rc == 0 )
{
- struct hostent *remote;
- char *host = b . base;
+ hostname = b . base;
+ buff_size = ( size_t ) b . elem_count;
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ size_t size;
+ rc = string_printf ( hostname, buff_size, & size, "%S", dns );
- rc = string_printf ( host, ( size_t ) b . elem_count, & size, "%S", dns );
- assert ( size < ( size_t ) b . elem_count );
- assert ( host [ size ] == 0 );
- remote = gethostbyname ( host );
+ assert ( rc == 0 );
+ assert ( size < buff_size );
+ assert ( hostname [ size ] == 0 );
+
+ if ( rc == 0 )
+ {
+ int lerrno;
+ struct hostent *remote = gethostbyname ( hostname );
if ( remote != NULL )
- {
- ep -> type = epIPV4;
- memcpy ( &ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
+ {
+ ep -> type = epIPV4;
+ memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
+ ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
ep -> u . ipv4 . port = ( uint16_t ) port;
- KDataBufferWhack ( & b );
- return 0;
}
else switch ( lerrno = WSAGetLastError () )
{
@@ -126,60 +135,14 @@ rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
}
}
}
- else
- {
- struct hostent *remote;
-
- rc = string_printf ( hostname, sizeof hostname, & size, "%S", dns );
- assert ( size < sizeof hostname );
- assert ( hostname [ size ] == 0 );
- remote = gethostbyname ( hostname );
- if ( remote != NULL )
- {
- ep -> type = epIPV4;
- memcpy ( &ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
- ep -> u . ipv4 . port = ( uint16_t ) port;
- return 0;
- }
- else switch ( lerrno = WSAGetLastError () )
- {
- case WSANOTINITIALISED: /* Must have WSAStartup call */
- rc = RC ( rcNS, rcNoTarg, rcInitializing, rcEnvironment, rcUndefined );
- break;
- case WSAENETDOWN:/* network subsystem failed */
- rc = RC ( rcNS, rcNoTarg, rcInitializing, rcNoObj, rcFailed );
- break;
- case WSAHOST_NOT_FOUND: /* Answer host not found */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
- break;
- case WSATRY_AGAIN: /* host not found or server failure */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
- break;
- case WSANO_RECOVERY: /* non-recoverable error */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
- break;
- case WSANO_DATA: /* name is valid but no data */
- rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
- break;
- case WSAEINPROGRESS: /* call is in progress */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined );
- break;
- case WSAEFAULT: /* name paremeter is not valid part of addr space */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcOutofrange );
- break;
- case WSAEINTR: /* socket call was calanceled */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
- break;
- default:
- rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcError );
- }
- }
+
+ if ( hostname != buffer )
+ KDataBufferWhack ( & b );
}
- memset ( ep, 0, sizeof * ep );
+ if ( rc != 0 )
+ memset ( ep, 0, sizeof * ep );
}
return rc;
-
-
}
diff --git a/libs/kns/win/syssock.c b/libs/kns/win/syssock.c
index e09df72..cc59a6d 100644
--- a/libs/kns/win/syssock.c
+++ b/libs/kns/win/syssock.c
@@ -38,9 +38,11 @@ typedef struct KSocket KSocket;
#include <kns/endpoint.h>
#include <klib/rc.h>
#include <klib/log.h>
+#include <klib/printf.h>
+#include <klib/text.h>
#include <sysalloc.h>
-#include "stream-priv.h"
+#include "../stream-priv.h"
#include <stdlib.h>
#include <assert.h>
@@ -83,7 +85,7 @@ struct KSocket
KStream dad;
union {
SOCKET fd;
- char pipename[256];
+ wchar_t pipename[256];
HANDLE pipe;
} u;
enum {isSocket, isIpcListener, isIpcPipeServer, isIpcPipeClient} type;
@@ -138,13 +140,13 @@ rc_t CC KSocketWhack ( KSocket *self )
if (self->listenerPipe != INVALID_HANDLE_VALUE)
{ /* !!! In case there is an active call to ConnectNamedPipe() on some thread, "wake" the synchronous named pipe,
otherwise DisconnectNamedPipe/CloseHandle will block forever */
- HANDLE hPipe = CreateFile(self->u.pipename,
- GENERIC_READ,
- FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
- NULL,
- OPEN_EXISTING,
- 0,
- NULL);
+ HANDLE hPipe = CreateFileW(self->u.pipename,
+ GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ NULL,
+ OPEN_EXISTING,
+ 0,
+ NULL);
if (hPipe != INVALID_HANDLE_VALUE)
CloseHandle(hPipe);
@@ -310,8 +312,8 @@ rc_t CC KSocketRead ( const KSocket *self,
{
ssize_t count;
if ( self -> type == isSocket)
- count = recv ( self -> u . fd, buffer, bsize, 0 );
- else if (!ReadFile( self->u.pipe, buffer, bsize, &count, NULL))
+ count = recv ( self -> u . fd, buffer, (int)bsize, 0 );
+ else if (!ReadFile( self->u.pipe, buffer, (DWORD)bsize, &count, NULL))
count = -1;
if ( count >= 0 )
@@ -339,8 +341,8 @@ rc_t CC KSocketWrite ( KSocket *self,
{
ssize_t count;
if ( self -> type == isSocket)
- count = send ( self -> u . fd, buffer, bsize, 0 );
- else if (!WriteFile( self->u.pipe, buffer, bsize, &count, NULL))
+ count = send ( self -> u . fd, buffer, (int)bsize, 0 );
+ else if (!WriteFile( self->u.pipe, buffer, (DWORD)bsize, &count, NULL))
count = -1;
if ( count >= 0 )
@@ -371,7 +373,7 @@ rc_t KSocketBind ( KSocket *self, const struct sockaddr *ss, size_t addrlen )
{
rc_t rc = 0;
- if ( bind ( self -> u.fd, ss, addrlen ) == SOCKET_ERROR )
+ if ( bind ( self -> u.fd, ss, (int)addrlen ) == SOCKET_ERROR )
rc = HandleErrno();
return rc;
@@ -383,17 +385,20 @@ rc_t KSocketConnect ( KSocket *self, struct sockaddr *ss, size_t addrlen )
{
rc_t rc = 0;
- if ( connect ( self -> u . fd, ss, addrlen ) == SOCKET_ERROR )
+ if ( connect ( self -> u . fd, ss, (int)addrlen ) == SOCKET_ERROR )
rc = HandleErrno();
return rc;
}
LIB_EXPORT
-rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoint *to )
+rc_t CC KNSManagerMakeConnection ( struct KNSManager const *self, KStream **out, const KEndPoint *from, const KEndPoint *to )
{
rc_t rc;
- int fd;
+ SOCKET fd;
+
+ if ( self == NULL )
+ return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
@@ -448,7 +453,7 @@ rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoin
}
else /* bind() failed */
rc = HandleErrno();
- close(fd);
+ closesocket(fd);
}
else /* socket() failed */
rc = HandleErrno();
@@ -456,12 +461,16 @@ rc_t CC KNSMakeConnection ( KStream **out, const KEndPoint *from, const KEndPoin
}
LIB_EXPORT
-rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_retries )
+rc_t CC KNSManagerMakeIPCConnection ( struct KNSManager const *self, KStream **out, const KEndPoint *to, uint8_t max_retries )
{
uint8_t retry_count = 0;
rc_t rc;
char pipename[256];
+ wchar_t pipenameW[256];
size_t num_writ;
+
+ if ( self == NULL )
+ return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
@@ -476,16 +485,19 @@ rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_r
/* use named pipes to implement unix domain socket - like behavior */
rc = string_printf(pipename, sizeof(pipename), &num_writ, "\\\\.\\pipe\\%s", to->u.ipc_name);
+ if (rc == 0)
+ string_cvt_wchar_copy(pipenameW, sizeof(pipenameW), pipename, num_writ);
+
while (rc == 0)
{
- HANDLE h = CreateFile(pipename, /* pipe name */
- GENERIC_READ | /* read and write access */
+ HANDLE h = CreateFileW(pipenameW, /* pipe name */
+ GENERIC_READ | /* read and write access */
GENERIC_WRITE,
- 0, /* no sharing */
- NULL, /* default security attributes */
- OPEN_EXISTING, /* opens existing pipe */
- 0, /* default attributes */
- NULL); /* no template file */
+ 0, /* no sharing */
+ NULL, /* default security attributes */
+ OPEN_EXISTING, /* opens existing pipe */
+ 0, /* default attributes */
+ NULL); /* no template file */
if ( h != INVALID_HANDLE_VALUE )
{ /* create the KSocket */
KSocket* ksock;
@@ -519,7 +531,7 @@ rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_r
}
else if (GetLastError() == ERROR_PIPE_BUSY)
{
- if (!WaitNamedPipe(pipename, NMPWAIT_USE_DEFAULT_WAIT))
+ if (!WaitNamedPipeW(pipenameW, NMPWAIT_USE_DEFAULT_WAIT))
{ // timeout, try again
if ( retry_count < max_retries )
{
@@ -551,11 +563,14 @@ rc_t CC KNSMakeIPCConnection ( KStream **out, const KEndPoint *to, uint8_t max_r
}
LIB_EXPORT
-rc_t CC KNSMakeListener( struct KSocket** out, struct KEndPoint const * ep )
+rc_t CC KNSManagerMakeListener( struct KNSManager const *self, struct KSocket** out, struct KEndPoint const * ep )
{
rc_t rc = 0;
KSocket* ksock;
+ if ( self == NULL )
+ return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
+
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
@@ -578,7 +593,11 @@ rc_t CC KNSMakeListener( struct KSocket** out, struct KEndPoint const * ep )
if ( rc == 0 )
{
size_t num_writ;
- rc = string_printf(ksock->u.pipename, sizeof(ksock->u.pipename), &num_writ, "\\\\.\\pipe\\%s", ep->u.ipc_name);
+ char pipename[256];
+ rc = string_printf(pipename, sizeof(pipename), &num_writ, "\\\\.\\pipe\\%s", ep->u.ipc_name);
+ if (rc == 0)
+ string_cvt_wchar_copy(ksock->u.pipename, sizeof(ksock->u.pipename), pipename, num_writ);
+
if (rc == 0)
{
ksock -> type = isIpcListener;
@@ -596,32 +615,32 @@ rc_t CC KNSMakeListener( struct KSocket** out, struct KEndPoint const * ep )
}
LIB_EXPORT
-rc_t CC KNSListen ( struct KSocket *listener, struct KStream **out )
+rc_t CC KSocketListen ( struct KSocket *listener, struct KStream **out )
{
rc_t rc = 0;
+ if ( listener == NULL )
+ return RC ( rcNS, rcNoTarg, rcValidating, rcSelf, rcNull);
+
if ( out == NULL )
return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
* out = NULL;
- if ( listener == NULL )
- return RC ( rcNS, rcNoTarg, rcValidating, rcParam, rcNull);
-
if (listener->type != isIpcListener)
return RC ( rcNS, rcNoTarg, rcValidating, rcParam, rcInvalid);
{
- listener->listenerPipe = CreateNamedPipe(listener->u.pipename, /* pipe name */
- PIPE_ACCESS_DUPLEX, /* read/write access */
- PIPE_TYPE_MESSAGE | /* message type pipe */
- PIPE_READMODE_MESSAGE | /* message-read mode */
- PIPE_WAIT, /* blocking mode */
- PIPE_UNLIMITED_INSTANCES,/* max. instances */
- 1024, /* output buffer size */
- 1024, /* input buffer size */
- 0, /* client time-out */
- NULL); /* default security attribute */
+ listener->listenerPipe = CreateNamedPipeW(listener->u.pipename, /* pipe name */
+ PIPE_ACCESS_DUPLEX, /* read/write access */
+ PIPE_TYPE_MESSAGE | /* message type pipe */
+ PIPE_READMODE_MESSAGE | /* message-read mode */
+ PIPE_WAIT, /* blocking mode */
+ PIPE_UNLIMITED_INSTANCES,/* max. instances */
+ 1024, /* output buffer size */
+ 1024, /* input buffer size */
+ 0, /* client time-out */
+ NULL); /* default security attribute */
if ( listener->listenerPipe != INVALID_HANDLE_VALUE )
{
while (1)
diff --git a/libs/kproc/libkproc.vers b/libs/kproc/libkproc.vers
deleted file mode 100644
index 66c4c22..0000000
--- a/libs/kproc/libkproc.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.9
diff --git a/libs/kproc/libkq.vers b/libs/kproc/libkq.vers
deleted file mode 100644
index 238d6e8..0000000
--- a/libs/kproc/libkq.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.7
diff --git a/libs/kproc/libksproc.vers b/libs/kproc/libksproc.vers
deleted file mode 100644
index 238d6e8..0000000
--- a/libs/kproc/libksproc.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.7
diff --git a/libs/kproc/unix/systimeout.c b/libs/kproc/unix/systimeout.c
index 9dbb97b..35295b7 100644
--- a/libs/kproc/unix/systimeout.c
+++ b/libs/kproc/unix/systimeout.c
@@ -41,7 +41,7 @@
/* Init
* initialize a timeout in milliseconds
*/
-rc_t TimeoutInit ( timeout_t *tm, uint32_t msec )
+LIB_EXPORT rc_t TimeoutInit ( timeout_t *tm, uint32_t msec )
{
if ( tm == NULL )
return RC ( rcPS, rcTimeout, rcConstructing, rcSelf, rcNull );
@@ -55,7 +55,7 @@ rc_t TimeoutInit ( timeout_t *tm, uint32_t msec )
/* Prepare
* ensures that a timeout is prepared with an absolute value
*/
-rc_t TimeoutPrepare ( timeout_t *self )
+LIB_EXPORT rc_t TimeoutPrepare ( timeout_t *self )
{
if ( self == NULL )
return RC ( rcPS, rcTimeout, rcUpdating, rcSelf, rcNull );
diff --git a/libs/kproc/win/syscond.c b/libs/kproc/win/syscond.c
index 8777f2f..9fb08a2 100644
--- a/libs/kproc/win/syscond.c
+++ b/libs/kproc/win/syscond.c
@@ -161,7 +161,7 @@ int pthread_cond_waitImpl (pthread_cond_t *cv,
// semaphore until <pthread_cond_signal> or <pthread_cond_broadcast>
// are called by another thread.
if (!infinite && abstime != NULL)
- { dwMilliseconds = abstime->tv_sec * 1000 + abstime->tv_nsec / 1000000; }
+ { dwMilliseconds = ( DWORD ) ( abstime->tv_sec * 1000 + abstime->tv_nsec / 1000000 ) ; }
SignalObjectAndWait (*external_mutex, cv->sema_, dwMilliseconds, FALSE);
// Reacquire lock to avoid race conditions.
@@ -433,7 +433,7 @@ LIB_EXPORT rc_t CC KConditionTimedWait ( KCondition *self, struct KLock *lock, t
memset(&ts, 0, sizeof ts);
ts.tv_sec = tm -> mS / 1000;
- ts.tv_nsec = (tm -> mS - ts.tv_sec * 1000) * 1000000;
+ ts.tv_nsec = ( long ) ( (tm -> mS - ts.tv_sec * 1000) * 1000000 );
status = pthread_cond_timedwait ( & self -> cond, & lock -> mutex, & ts );
switch ( status )
diff --git a/libs/krypto/Makefile b/libs/krypto/Makefile
index 4743e6f..d7586c8 100644
--- a/libs/krypto/Makefile
+++ b/libs/krypto/Makefile
@@ -212,7 +212,6 @@ LIB_SRC = \
no-null-ncbi
TEST_LIB_SRC = \
- null-ncbi \
testcipher
ifeq ($(VEC_SUPPORTED),true)
@@ -221,15 +220,13 @@ ALL_LIB_SRC += \
aes-ncbi.vec
LIB_SRC += \
no-null-ncbi.vec
-TEST_LIB_SRC += \
- null-ncbi.vec
+TEST_LIB_SRC +=
else
ALL_LIB_SRC += \
cipher-no-vec.vec
LIB_SRC += \
no-null-ncbi.vec
-TEST_LIB_SRC += \
- no-null-ncbi.vec
+TEST_LIB_SRC +=
endif
ifeq ($(VECREG_SUPPORTED),true)
@@ -238,15 +235,13 @@ ALL_LIB_SRC += \
aes-ncbi.vecreg
LIB_SRC += \
no-null-ncbi.vecreg
-TEST_LIB_SRC += \
- null-ncbi.vecreg
+TEST_LIB_SRC +=
else
ALL_LIB_SRC += \
cipher-no-vec.vecreg
LIB_SRC += \
no-null-ncbi.vecreg
-TEST_LIB_SRC += \
- no-null-ncbi.vecreg
+TEST_LIB_SRC +=
endif
ifeq ($(AESNI_SUPPORTED),true)
@@ -255,15 +250,13 @@ ALL_LIB_SRC += \
aes-ncbi.aes-ni
LIB_SRC += \
no-null-ncbi.aes-ni
-TEST_LIB_SRC += \
- no-null-ncbi.aes-ni
+TEST_LIB_SRC +=
else
ALL_LIB_SRC += \
cipher-no-vec.aes-ni
LIB_SRC += \
no-null-ncbi.aes-ni
-TEST_LIB_SRC += \
- no-null-ncbi.aes-ni
+TEST_LIB_SRC +=
endif
@@ -273,10 +266,6 @@ KRYPTO_OBJ = \
$(addsuffix .$(LOBX),$(KRYPTO_SRC))
KRYPTO_LIB = \
- -dkfg \
- -dkfs \
- -dkproc \
- -dklib
$(ILIBDIR)/libkrypto.$(LIBX): $(KRYPTO_OBJ)
$(LD) --slib -o $@ $^ $(KRYPTO_LIB)
@@ -288,16 +277,12 @@ $(ILIBDIR)/libkrypto.$(LIBX): $(KRYPTO_OBJ)
$(ILIBDIR)/libkryptotest: $(addprefix $(ILIBDIR)/libkryptotest.,$(ILIBEXT))
-KRYPTOTEST_SRC = $(ALL_LIB_SRC) $(TEST_LIB_SRC)
+KRYPTOTEST_SRC = $(TEST_LIB_SRC)
KRYPTOTEST_OBJ = \
$(addsuffix .$(LOBX),$(KRYPTOTEST_SRC))
KRYPTOTEST_LIB = \
- -dkfg \
- -dkfs \
- -dkproc \
- -dklib
$(ILIBDIR)/libkryptotest.$(LIBX): $(KRYPTOTEST_OBJ)
$(LD) --slib -o $@ $^ $(KRYPTOTEST_LIB)
diff --git a/libs/krypto/cipher-byte.c b/libs/krypto/cipher-byte.c
index feb05c2..49dcb07 100644
--- a/libs/krypto/cipher-byte.c
+++ b/libs/krypto/cipher-byte.c
@@ -463,7 +463,7 @@ static __inline__ rc_t MEMBER(Init)(KCipherByte * self,
size_t block_size = block_cipher->byte.v1.block_size();
size_t key_size = block_cipher->byte.v1.key_size();
- KCipherInit (dad, block_size,
+ KCipherInit (dad, (uint32_t) block_size,
(const KCipher_vt*)&MEMBER(_vt_v1),
MEMBER(ClassName));
diff --git a/libs/krypto/cipher.c b/libs/krypto/cipher.c
index fc6b6c2..bf7a4a3 100644
--- a/libs/krypto/cipher.c
+++ b/libs/krypto/cipher.c
@@ -169,7 +169,7 @@ KRYPTO_EXTERN rc_t CC KCipherSetEncryptKey (KCipher * self, const void * user_ke
switch (self->vt.version->maj)
{
case 1:
- return self->vt.v1->set_encrypt_key (self, user_key, user_key_size);
+ return self->vt.v1->set_encrypt_key (self, user_key, (uint32_t) user_key_size);
}
return RC (rcKrypto, rcCipher, rcUpdating, rcInterface, rcBadVersion);
}
@@ -190,7 +190,7 @@ KRYPTO_EXTERN rc_t CC KCipherSetDecryptKey (KCipher * self, const void * user_ke
switch (self->vt.version->maj)
{
case 1:
- return self->vt.v1->set_decrypt_key (self, user_key, user_key_size);
+ return self->vt.v1->set_decrypt_key (self, user_key, (uint32_t) user_key_size);
}
return RC (rcKrypto, rcCipher, rcUpdating, rcInterface, rcBadVersion);
}
diff --git a/libs/krypto/encfile-priv.h b/libs/krypto/encfile-priv.h
index 8061508..9456b5a 100644
--- a/libs/krypto/encfile-priv.h
+++ b/libs/krypto/encfile-priv.h
@@ -122,7 +122,7 @@ uint64_t EncryptedPos_to_BlockId (uint64_t enc_offset, uint32_t * poffset,
else
{
if (poffset)
- *poffset = offset;
+ *poffset = (uint32_t)offset;
if (in_block)
*in_block = true;
}
diff --git a/libs/krypto/encfile.c b/libs/krypto/encfile.c
index 1c0a970..2755fca 100644
--- a/libs/krypto/encfile.c
+++ b/libs/krypto/encfile.c
@@ -1340,7 +1340,7 @@ rc_t KEncFileV1WriteInt (KEncFileV1 *self, uint64_t block_id, uint32_t block_off
memmove (self->block.data + block_offset, buffer, bsize);
self->dirty = true;
if (block_offset + bsize > self->block.u.valid)
- self->block.u.valid = block_offset + bsize;
+ self->block.u.valid = (KEncFileBlockValid) ( block_offset + bsize );
*pnum_writ = bsize;
return 0;
}
@@ -1423,7 +1423,7 @@ rc_t KEncFileV1WriteInt (KEncFileV1 *self, uint64_t block_id, uint32_t block_off
memmove (self->block.data, pb, bsize);
if (bsize > self->block.u.valid)
- self->block.u.valid = bsize;
+ self->block.u.valid = (KEncFileBlockValid) bsize;
self->dirty = true;
tot_writ += bsize;
diff --git a/libs/krypto/encfilev2.c b/libs/krypto/encfilev2.c
index 25315ba..44d82bd 100644
--- a/libs/krypto/encfilev2.c
+++ b/libs/krypto/encfilev2.c
@@ -1801,7 +1801,7 @@ rc_t CC KEncFileWrite (KEncFile *self, uint64_t pos,
self->dirty = true;
*pnum_writ = to_copy;
- new_valid = offset + to_copy;
+ new_valid = (uint32_t) ( offset + to_copy );
if (new_valid > self->block.u.valid)
{
uint64_t new_size;
diff --git a/libs/krypto/libkrypto.vers b/libs/krypto/libkrypto.vers
deleted file mode 100644
index 5b09c67..0000000
--- a/libs/krypto/libkrypto.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.14
diff --git a/libs/krypto/libkryptotest.vers b/libs/krypto/libkryptotest.vers
deleted file mode 100644
index 3eefcb9..0000000
--- a/libs/krypto/libkryptotest.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.0
diff --git a/libs/krypto/reencfile.c b/libs/krypto/reencfile.c
index 00ed4e3..8ae0b05 100644
--- a/libs/krypto/reencfile.c
+++ b/libs/krypto/reencfile.c
@@ -672,7 +672,7 @@ rc_t KReencFileReadHandleBlock (KReencFile *self,
* if we are here we decrypted and re-encrypted the
* expected block
*/
- offset = pos - BlockId_to_EncryptedPos (block_id);
+ offset = (uint32_t) ( pos - BlockId_to_EncryptedPos (block_id) );
rc = KReencFileReadBlockOut (self, offset, buffer, bsize, num_read);
if (rc)
LOGERR (klogErr, rc, "re-enc error copying out from block");
diff --git a/libs/krypto/wgaencrypt.c b/libs/krypto/wgaencrypt.c
index 0efc901..b2858fb 100644
--- a/libs/krypto/wgaencrypt.c
+++ b/libs/krypto/wgaencrypt.c
@@ -195,8 +195,8 @@ KRYPTO_EXTERN rc_t CC KFileIsWGAEnc (const void * buffer, size_t buffer_size)
const uint8_t * ph;
const uint8_t * pt;
const uint8_t * pb;
- int ix;
- int lim;
+ size_t ix;
+ size_t lim;
if ((buffer == NULL) || (buffer_size == 0))
return RC (rcFS, rcFile, rcIdentifying, rcParam, rcNull);
@@ -312,7 +312,7 @@ rc_t KWGAEncFileKeyInit (KWGAEncFile * self, const char * key, size_t key_size)
if (key_size < 16) /* even though g_key and some cipher keys are larger than 16 */
{
- int jx, ix;
+ size_t jx, ix;
for ((jx = key_size),(ix = 0); jx < 16; ++jx, ++ix)
{
g_key[jx] = self->md51[ix] | g_key[ix%(jx?jx:1)]; /* cringe? */
@@ -460,7 +460,7 @@ rc_t KWGAEncFileReadInt (KWGAEncFile * self, uint64_t pos, size_t bsize)
&tot_read);
#endif
self->buffer.offset = pos;
- self->buffer.valid = tot_read;
+ self->buffer.valid = (uint32_t)tot_read;
if (tot_read & 15)
rc = RC (rcKrypto, rcFile, rcReading, rcSize, rcInsufficient);
@@ -470,7 +470,7 @@ rc_t KWGAEncFileReadInt (KWGAEncFile * self, uint64_t pos, size_t bsize)
#if RETAINED_COMPATIBILTY_WITH_ERROR
KCipherDecryptECB (self->cipher, self->buffer.data, self->buffer.data,
- (tot_read / ECB_BYTES));
+ (uint32_t)(tot_read / ECB_BYTES));
#else
/* Well this is wrong for even being wrong now */
KCipherDecryptECB (self->cipher, self->buffer.data, self->buffer.data,
diff --git a/libs/kxml/libkxfs.vers b/libs/kxml/libkxfs.vers
deleted file mode 100644
index 7ee7020..0000000
--- a/libs/kxml/libkxfs.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.10
diff --git a/libs/kxml/libkxml.vers b/libs/kxml/libkxml.vers
deleted file mode 100644
index 5b09c67..0000000
--- a/libs/kxml/libkxml.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.14
diff --git a/libs/kxml/xtoc.c b/libs/kxml/xtoc.c
index 5d2dc22..d1acd3a 100644
--- a/libs/kxml/xtoc.c
+++ b/libs/kxml/xtoc.c
@@ -943,7 +943,6 @@ rc_t KXTocFileDestroy ( KXTocFile *self )
assert (self);
KFileRelease (self->base);
XTocCacheRelease (self->cache);
- self->cache->file = NULL;
free (self);
return 0;
}
@@ -1782,7 +1781,6 @@ rc_t CC KXTocDirDestroy (KXTocDir *self)
assert (self);
XTocCacheRelease (self->cache);
- self->cache->file = NULL;
KDirectoryRelease (self->base);
free (self);
return 0;
@@ -2174,7 +2172,7 @@ static rc_t CC KXTocDirResolvePath (const KXTocDir *self,
if (rsize < (path_sz - 1))
rc = RC(rcFS, rcDirectory, rcResolving, rcBuffer, rcInsufficient);
else
- string_copy(resolved, rsize, path, rsize);
+ string_copy(resolved, rsize, path, path_sz);
}
if (path)
free ((void*)path);
diff --git a/libs/loader/common-writer.c b/libs/loader/common-writer.c
index 74ca451..59f4c1c 100644
--- a/libs/loader/common-writer.c
+++ b/libs/loader/common-writer.c
@@ -258,7 +258,6 @@ static unsigned SeqHashKey(void const *const key, size_t const keylen)
return (h1 << 8) | h2;
}
-
#define USE_ILLUMINA_NAMING_POUND_NUMBER_SLASH_HACK 1
static size_t GetFixedNameLength(char const name[], size_t const namelen)
@@ -583,6 +582,7 @@ rc_t WriteSoloFragments(const CommonWriterSettings* settings, SpotAssembler* ctx
srec.cskey[read] = fip->cskey;
memcpy(srec.seq + srec.readStart[read], src, srec.readLen[read]);
src += fip->readlen;
+
memcpy(srec.qual + srec.readStart[read], src, srec.readLen[read]);
src += fip->readlen;
srec.spotGroup = (char *)src;
@@ -1529,6 +1529,8 @@ rc_t ArchiveFile(const struct ReaderFile *const reader,
}
else if (squal != NULL)
memcpy(qual, squal, csSeqLen);
+ else
+ memset(qual, 0, csSeqLen);
readlen = csSeqLen;
}
}
@@ -1570,7 +1572,7 @@ rc_t ArchiveFile(const struct ReaderFile *const reader,
}
if (mated) {
if (isPrimary || !originally_aligned) {
- if (value->spotId != 0) {
+ if (CTX_VALUE_GET_S_ID(*value) != 0) {
(void)PLOGMSG(klogWarn, (klogWarn, "Spot '$(name)' has already been assigned a spot id", "name=%.*s", namelen, name));
}
else if (!value->has_a_read) {
@@ -1770,7 +1772,7 @@ rc_t ArchiveFile(const struct ReaderFile *const reader,
}
}
}
- else if (wasInserted & (isPrimary || !originally_aligned)) {
+ else if (CTX_VALUE_GET_S_ID(*value) == 0 && (isPrimary || !originally_aligned)) {
/* new unmated fragment - no spot assembly */
unsigned readLen[1];
diff --git a/libs/ncbi-vdb/Makefile b/libs/ncbi-vdb/Makefile
index 7b08def..f8acf19 100644
--- a/libs/ncbi-vdb/Makefile
+++ b/libs/ncbi-vdb/Makefile
@@ -77,17 +77,16 @@ LIBVDB_BLAST =
endif
ifeq (win,$(OS))
-LIBENTREZ =
LIBKQ =
LIBKXML =
else
-LIBENTREZ =
LIBKQ = kq
LIBKXML = kxml kxfs
endif
VDB_LIB_CMN = \
align-access \
+ ncbi-bam \
bz2 \
kfg \
kfs \
@@ -137,7 +136,6 @@ VDB_LIB_RDWR = \
wsradb \
wsraxf \
wvdb \
- $(LIBENTREZ) \
wvxf \
wwgsxf
diff --git a/libs/ncbi-vdb/libncbi-vdb.vers b/libs/ncbi-vdb/libncbi-vdb.vers
index 0bee604..3f684d2 100644
--- a/libs/ncbi-vdb/libncbi-vdb.vers
+++ b/libs/ncbi-vdb/libncbi-vdb.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/libs/ncbi-vdb/libncbi-wvdb.vers b/libs/ncbi-vdb/libncbi-wvdb.vers
index 0bee604..3f684d2 100644
--- a/libs/ncbi-vdb/libncbi-wvdb.vers
+++ b/libs/ncbi-vdb/libncbi-wvdb.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/libs/search/libksrch.vers b/libs/search/libksrch.vers
deleted file mode 100644
index 2ac9634..0000000
--- a/libs/search/libksrch.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.13
diff --git a/libs/search/libnucstrstr.vers b/libs/search/libnucstrstr.vers
deleted file mode 100644
index 6d7de6e..0000000
--- a/libs/search/libnucstrstr.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.2
diff --git a/libs/search/search.c b/libs/search/search.c
index a516e8f..4f6661f 100644
--- a/libs/search/search.c
+++ b/libs/search/search.c
@@ -31,15 +31,16 @@
#include <stdlib.h>
#include <string.h>
-const unsigned char* IUPAC_decode[256] = { (const unsigned char*)0xFFFF };
+const unsigned char * IUPAC_decode [ 256 ];
static
-void IUPAC_init(void)
+void IUPAC_init ( void )
{
- if( IUPAC_decode[0] != NULL ) {
- const char** t = (const char**)IUPAC_decode;
+ static bool initialized;
+ if ( ! initialized )
+ {
+ const char ** t = ( const char** ) IUPAC_decode;
- memset(t, 0, sizeof(t));
t['A'] = t['a'] = "Aa";
t['C'] = t['c'] = "Cc";
t['G'] = t['g'] = "Gg";
@@ -55,7 +56,8 @@ void IUPAC_init(void)
t['B'] = t['b'] = "CcGgTtUu";
t['D'] = t['d'] = "AaGgTtUu";
t['H'] = t['h'] = "AaCcTtUu";
- t['N'] = t['n'] = t['.'] = "AaCcGgTtUuNn.-";
+ t['N'] = t['n'] = t [ '.' ] = "AaCcGgTtUuNn.-";
+ initialized = true;
}
}
@@ -156,9 +158,7 @@ LIB_EXPORT rc_t CC AgrepMake( AgrepParams **self, AgrepFlags mode, const char *p
(*self)->mode = mode;
if( mode & AGREP_PATTERN_4NA ) {
size_t i, l = strlen(pattern);
- if( IUPAC_decode[0] != NULL ) {
- IUPAC_init();
- }
+ IUPAC_init();
if( l == 0 ) {
rc = RC(rcText, rcString, rcSearching, rcParam, rcOutofrange);
}
diff --git a/libs/sra/libsra-schema.vers b/libs/sra/libsra-schema.vers
deleted file mode 100644
index 9084fa2..0000000
--- a/libs/sra/libsra-schema.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.0
diff --git a/libs/sra/libsradb.vers b/libs/sra/libsradb.vers
deleted file mode 100644
index 55f20a1..0000000
--- a/libs/sra/libsradb.vers
+++ /dev/null
@@ -1 +0,0 @@
-3.1.13
diff --git a/libs/sra/libsrareader.vers b/libs/sra/libsrareader.vers
deleted file mode 100644
index 0664a8f..0000000
--- a/libs/sra/libsrareader.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.6
diff --git a/libs/sra/libsraschema.vers b/libs/sra/libsraschema.vers
deleted file mode 100644
index 9084fa2..0000000
--- a/libs/sra/libsraschema.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.0
diff --git a/libs/sra/libwsra-schema.vers b/libs/sra/libwsra-schema.vers
deleted file mode 100644
index ee90284..0000000
--- a/libs/sra/libwsra-schema.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.4
diff --git a/libs/sra/libwsradb.vers b/libs/sra/libwsradb.vers
deleted file mode 100644
index 55f20a1..0000000
--- a/libs/sra/libwsradb.vers
+++ /dev/null
@@ -1 +0,0 @@
-3.1.13
diff --git a/libs/sra/libwsraschema.vers b/libs/sra/libwsraschema.vers
deleted file mode 100644
index 9084fa2..0000000
--- a/libs/sra/libwsraschema.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.0
diff --git a/libs/sra/sracache.c b/libs/sra/sracache.c
index dd84389..b33db0e 100644
--- a/libs/sra/sracache.c
+++ b/libs/sra/sracache.c
@@ -479,7 +479,7 @@ LIB_EXPORT rc_t CC SRACacheGetTable(SRACache* self, const char* acc, const SRATa
rc = KVectorGetPtr ( index->body, key, (void**)&elem );
if (rc == 0 && elem != NULL)
{
- if (elem->object->refcount.counter == 1) /* onwed by cache, not used elsewhere */
+ if (atomic32_read(&elem->object->refcount) == 1) /* owned by cache, not used elsewhere */
{
++ self->hits;
diff --git a/libs/sraxf/libsraxf.vers b/libs/sraxf/libsraxf.vers
deleted file mode 100644
index e25d8d9..0000000
--- a/libs/sraxf/libsraxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.5
diff --git a/libs/sraxf/libwsraxf.vers b/libs/sraxf/libwsraxf.vers
deleted file mode 100644
index e25d8d9..0000000
--- a/libs/sraxf/libwsraxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.5
diff --git a/libs/vdb/column-priv.h b/libs/vdb/column-priv.h
index 2dd451f..2e26e5d 100644
--- a/libs/vdb/column-priv.h
+++ b/libs/vdb/column-priv.h
@@ -57,6 +57,7 @@ extern "C" {
/*--------------------------------------------------------------------------
* forwards
*/
+struct Vector;
struct VDBManager;
struct VSchema;
struct SColumn;
@@ -205,7 +206,7 @@ struct WColumnRowMap
rc_t WColumnMake ( VColumn **col, struct VSchema const *schema,
struct SColumn const *scol, struct SExpression const *dflt_limit,
- struct VDBManager *mgr );
+ struct VDBManager *mgr, struct Vector *cx_bind );
rc_t WColumnSetDefault ( VColumn *self,
bitsz_t elem_bits, const void *buffer, bitsz_t boff, uint64_t len );
diff --git a/libs/vdb/cursor-cmn.c b/libs/vdb/cursor-cmn.c
index 1ee550b..85e6bb8 100644
--- a/libs/vdb/cursor-cmn.c
+++ b/libs/vdb/cursor-cmn.c
@@ -346,7 +346,7 @@ rc_t VCursorMake ( VCursor **cursp, const VTable *tbl )
KRefcountInit ( & curs -> refcount, 1, "VCursor", "make", "vcurs" );
curs -> state = vcConstruct;
curs -> permit_add_column = true;
- curs -> suspend_triggers = false;
+ curs -> suspend_triggers = false;
* cursp = curs;
return 0;
}
@@ -543,8 +543,15 @@ static rc_t VTableCreateCachedCursorReadImpl ( const VTable *self,
rc = VCursorSupplementSchema ( curs );
#if 0
- if(create_pagemap_thread && capacity > 0 && rc == 0 )
+ if ( create_pagemap_thread && capacity > 0 && rc == 0 )
+ {
rc = VCursorLaunchPagemapThread ( curs );
+ if ( rc != 0 )
+ {
+ if ( GetRCState( rc ) == rcNotAvailable )
+ rc = 0;
+ }
+ }
#endif
if ( rc == 0 )
{
@@ -623,7 +630,8 @@ LIB_EXPORT rc_t CC VCursorSuspendTriggers ( const VCursor *cself )
/* AddSColumn
*/
static
-rc_t VCursorAddSColumn ( VCursor *self, uint32_t *idx, const SColumn *scol, const VTypedecl *cast )
+rc_t VCursorAddSColumn ( VCursor *self, uint32_t *idx,
+ const SColumn *scol, const VTypedecl *cast, Vector *cx_bind )
{
rc_t rc;
VColumn *col;
@@ -650,7 +658,7 @@ rc_t VCursorAddSColumn ( VCursor *self, uint32_t *idx, const SColumn *scol, cons
}
/* make object */
- rc = VCursorMakeColumn ( self, & col, scol );
+ rc = VCursorMakeColumn ( self, & col, scol, cx_bind );
if ( rc == 0 )
{
/* insert it into vectors */
@@ -713,7 +721,12 @@ rc_t VCursorAddColspec ( VCursor *self, uint32_t *idx, const char *colspec )
if ( scol == NULL || type != eColumn )
rc = RC ( rcVDB, rcCursor, rcUpdating, rcColumn, rcNotFound );
else
- rc = VCursorAddSColumn ( self, idx, scol, & cast );
+ {
+ Vector cx_bind;
+ VectorInit ( & cx_bind, 1, self -> schema -> num_indirect );
+ rc = VCursorAddSColumn ( self, idx, scol, & cast, & cx_bind );
+ VectorWhack ( & cx_bind, NULL, NULL );
+ }
return rc;
}
@@ -825,7 +838,7 @@ rc_t VCursorGetColspec ( const VCursor *self, uint32_t *idx, const char *colspec
VTypedecl cast;
const SNameOverload *name;
const SColumn *scol = STableFind ( self -> tbl -> stbl, self -> schema,
- & cast, & name, & type, colspec, "VCursorAddColspec", true );
+ & cast, & name, & type, colspec, "VCursorGetColspec", true );
if ( scol == NULL || type != eColumn )
rc = RC ( rcVDB, rcCursor, rcAccessing, rcColumn, rcNotFound );
else
@@ -1187,6 +1200,7 @@ rc_t VCursorOpenColumn ( const VCursor *cself, VColumn *col )
KDlset *libs;
VCursor *self = ( VCursor* ) cself;
+ Vector cx_bind;
VProdResolveData pb;
pb . pr . schema = self -> schema;
pb . pr . ld = self -> tbl -> linker;
@@ -1194,11 +1208,14 @@ rc_t VCursorOpenColumn ( const VCursor *cself, VColumn *col )
pb . pr . curs = self;
pb . pr . cache = & self -> prod;
pb . pr . owned = & self -> owned;
+ pb . pr . cx_bind = & cx_bind;
pb . pr . chain = chainDecoding;
pb . pr . blobbing = false;
pb . pr . ignore_column_errors = false;
pb . pr . discover_writable_columns = false;
+ VectorInit ( & cx_bind, 1, self -> schema -> num_indirect );
+
pb . rc = VLinkerOpen ( pb . pr . ld, & libs );
if ( pb . rc == 0 )
{
@@ -1207,6 +1224,8 @@ rc_t VCursorOpenColumn ( const VCursor *cself, VColumn *col )
KDlsetRelease ( libs );
}
+ VectorWhack ( & cx_bind, NULL, NULL );
+
return pb . rc;
}
@@ -1223,6 +1242,7 @@ static
rc_t VCursorResolveColumnProductions ( VCursor *self,
const KDlset *libs, bool ignore_failures )
{
+ Vector cx_bind;
VProdResolveData pb;
pb . pr . schema = self -> schema;
pb . pr . ld = self -> tbl -> linker;
@@ -1231,16 +1251,21 @@ rc_t VCursorResolveColumnProductions ( VCursor *self,
pb . pr . curs = self;
pb . pr . cache = & self -> prod;
pb . pr . owned = & self -> owned;
+ pb . pr . cx_bind = & cx_bind;
pb . pr . chain = chainDecoding;
pb . pr . blobbing = false;
pb . pr . ignore_column_errors = ignore_failures;
pb . pr . discover_writable_columns = false;
pb . rc = 0;
- if ( VectorDoUntil ( & self -> row, false, VCursorResolveColumn, & pb ) )
- return pb . rc;
+ VectorInit ( & cx_bind, 1, self -> schema -> num_indirect );
- return 0;
+ if ( ! VectorDoUntil ( & self -> row, false, VCursorResolveColumn, & pb ) )
+ pb . rc = 0;
+
+ VectorWhack ( & cx_bind, NULL, NULL );
+
+ return pb . rc;
}
rc_t VCursorOpenRead ( VCursor *self, const KDlset *libs )
@@ -1974,15 +1999,20 @@ LIB_EXPORT rc_t CC VCursorOpenParentRead ( const VCursor *self, const VTable **t
return rc;
}
+struct insert_overloaded_pb
+{
+ VCursor *curs;
+ Vector *cx_bind;
+};
static
void CC insert_overloaded_scolumns ( void *item, void *data )
{
- VCursor *curs = data;
+ struct insert_overloaded_pb *pb = data;
const SColumn *scol = ( const void* ) item;
uint32_t ignore;
- VCursorAddSColumn ( curs, & ignore, scol, NULL );
+ VCursorAddSColumn ( pb -> curs, & ignore, scol, NULL, pb -> cx_bind );
}
static
@@ -1990,13 +2020,22 @@ void VCursorListCol_walk_through_columns_and_add_to_cursor ( VCursor *self )
{
uint32_t idx = VectorStart ( & self -> stbl -> cname );
uint32_t end = VectorLength ( & self -> stbl -> cname );
+
+ Vector cx_bind;
+ struct insert_overloaded_pb pb;
+ pb . curs = self;
+ pb . cx_bind = & cx_bind;
+ VectorInit ( & cx_bind, 1, self -> schema -> num_indirect );
+
for ( end += idx; idx < end; ++idx )
{
/* look at the table column name guy */
const SNameOverload* ol_entry = ( const SNameOverload* ) VectorGet ( & self -> stbl -> cname, idx );
if ( ol_entry != NULL )
- VectorForEach ( & ol_entry -> items, false, insert_overloaded_scolumns, self );
+ VectorForEach ( & ol_entry -> items, false, insert_overloaded_scolumns, & pb );
}
+
+ VectorWhack ( & cx_bind, NULL, NULL );
}
static
diff --git a/libs/vdb/cursor-priv.h b/libs/vdb/cursor-priv.h
index ec91798..9ec8b13 100644
--- a/libs/vdb/cursor-priv.h
+++ b/libs/vdb/cursor-priv.h
@@ -64,7 +64,7 @@ extern "C" {
#define VCURSOR_WRITE_MODES_SUPPORTED 0
/* temporary - until the full kproc for Windows is operational */
-#if WINDOWS
+#if defined(WINDOWS) || defined(_WIN32) || defined(NCBI_WITHOUT_MT)
#define VCURSOR_FLUSH_THREAD 0
#else
#define VCURSOR_FLUSH_THREAD 1
@@ -167,7 +167,7 @@ struct VCursor
struct STable SKONST *stbl;
/* background flush thread objects */
- int64_t launch_cnt;
+ int64_t launch_cnt;
struct KThread *flush_thread;
struct KLock *flush_lock;
struct KCondition *flush_cond;
@@ -182,6 +182,7 @@ struct VCursor
/* external named cursor parameters */
BSTree named_params;
+
/* linked cursors */
BSTree linked_cursors;
@@ -196,7 +197,7 @@ struct VCursor
/* physical columns by cid ( owned ) */
VCursorCache phys;
- uint32_t phys_cnt;
+ uint32_t phys_cnt;
/* productions by cid ( not-owned ) */
VCursorCache prod;
@@ -250,7 +251,7 @@ rc_t VCursorSupplementSchema ( struct VCursor const *self );
/* MakeColumn
*/
rc_t VCursorMakeColumn ( struct VCursor *self,
- struct VColumn **col, struct SColumn const *scol );
+ struct VColumn **col, struct SColumn const *scol, Vector *cx_bind );
/* SetRowIdRead - PRIVATE
* seek to given row id
diff --git a/libs/vdb/cursor.c b/libs/vdb/cursor.c
index 4786fef..63adbc3 100644
--- a/libs/vdb/cursor.c
+++ b/libs/vdb/cursor.c
@@ -68,7 +68,7 @@ rc_t VCursorWhack ( VCursor *self )
/* MakeColumn
*/
-rc_t VCursorMakeColumn ( VCursor *self, VColumn **col, const SColumn *scol )
+rc_t VCursorMakeColumn ( VCursor *self, VColumn **col, const SColumn *scol, Vector *cx_bind )
{
return VColumnMake ( col, self -> schema, scol );
}
diff --git a/libs/vdb/database-cmn.c b/libs/vdb/database-cmn.c
index b8ce8d7..c877fea 100644
--- a/libs/vdb/database-cmn.c
+++ b/libs/vdb/database-cmn.c
@@ -710,3 +710,15 @@ LIB_EXPORT rc_t CC VDatabaseTypespec ( const VDatabase *self, char *ts_buff, siz
return rc;
}
+
+
+/* IsCSRA
+ * ask an open database if it conforms to cSRA schema
+ */
+LIB_EXPORT bool CC VDatabaseIsCSRA ( const VDatabase *self )
+{
+ if ( self != NULL )
+ return KDatabaseExists ( self -> kdb, kptTable, "PRIMARY_ALIGNMENT" );
+
+ return false;
+}
diff --git a/libs/vdb/libvdb.vers b/libs/vdb/libvdb.vers
deleted file mode 100644
index 2714f53..0000000
--- a/libs/vdb/libvdb.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.6.4
diff --git a/libs/vdb/libwvdb.vers b/libs/vdb/libwvdb.vers
deleted file mode 100644
index 2714f53..0000000
--- a/libs/vdb/libwvdb.vers
+++ /dev/null
@@ -1 +0,0 @@
-2.6.4
diff --git a/libs/vdb/page-map.h b/libs/vdb/page-map.h
index 427dc45..0fead2e 100644
--- a/libs/vdb/page-map.h
+++ b/libs/vdb/page-map.h
@@ -328,7 +328,7 @@ static __inline__ row_count_t PageMapIteratorRepeatCount(const PageMapIterator *
switch ((*self->rgns)[self->cur_rgn].type){
case PM_REGION_EXPAND_FULL:
if((*self->rgns)[self->cur_rgn].expanded){
- int i;
+ row_count_t i;
elem_count_t* base = (*self->exp_base) + (*self->rgns)[self->cur_rgn].data_offset;
self->repeat_count = 1;
for(i=self->cur_rgn_row+1;i<(*self->rgns)[self->cur_rgn].numrows;i++){
@@ -341,7 +341,7 @@ static __inline__ row_count_t PageMapIteratorRepeatCount(const PageMapIterator *
break;
case PM_REGION_EXPAND_SAMELEN:
if((*self->rgns)[self->cur_rgn].expanded){
- int i;
+ row_count_t i;
elem_count_t* base = (*self->exp_base) + (*self->rgns)[self->cur_rgn].data_offset;
self->repeat_count = 1;
for(i=self->cur_rgn_row+1;i<(*self->rgns)[self->cur_rgn].numrows;i++){
diff --git a/libs/vdb/prod-cmn.c b/libs/vdb/prod-cmn.c
index 466fe6f..eb74b68 100644
--- a/libs/vdb/prod-cmn.c
+++ b/libs/vdb/prod-cmn.c
@@ -27,7 +27,7 @@
#define USE_EUGENE 1
-#define TRACK_REFERENCES 1
+#define TRACK_REFERENCES 0
#include <vdb/extern.h>
@@ -65,7 +65,7 @@
#include <stdio.h>
#include <limits.h>
-#if ! WINDOWS
+#if !defined(WINDOWS) && !defined(_WIN32) && !defined(NCBI_WITHOUT_MT)
#define LAUNCH_PAGEMAP_THREAD 1
#endif
@@ -1824,17 +1824,27 @@ bool CC fetch_param_FixedRowLength ( void *item, void *data )
}
static
-uint32_t VFunctionProdFixedRowLength ( const VFunctionProd *self, int64_t row_id,bool ignore_self )
+uint32_t VFunctionProdFixedRowLength ( const VFunctionProd *self, int64_t row_id, bool ignore_self )
{
- if(ignore_self == false && (self->dad.sub == vftRow || self->dad.sub == vftNonDetRow || self->dad.sub == vftIdDepRow )){
- return 0;
- } else {
- fetch_param_FixedRowLength_data pb;
- pb.first_time = true;
- pb.length = 0;
- VectorDoUntil ( & self -> parms, false, fetch_param_FixedRowLength, & pb );
- return pb.length;
+ fetch_param_FixedRowLength_data pb;
+
+ if ( ! ignore_self )
+ {
+ switch ( self -> dad . sub )
+ {
+ case vftRow:
+ case vftNonDetRow:
+ case vftIdDepRow:
+ return 0;
+ }
}
+
+ pb.first_time = true;
+ pb.length = 0;
+
+ VectorDoUntil ( & self -> parms, false, fetch_param_FixedRowLength, & pb );
+
+ return pb.length;
}
diff --git a/libs/vdb/prod-expr.c b/libs/vdb/prod-expr.c
index 07a3080..6d84d4e 100644
--- a/libs/vdb/prod-expr.c
+++ b/libs/vdb/prod-expr.c
@@ -108,7 +108,7 @@ rc_t VProdResolveCastExpr ( const VProdResolve *self, VProduction **out, const S
/* extract cast type */
VFormatdecl cast;
rc_t rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) expr -> left, self -> schema, & cast );
+ ( ( const STypeExpr* ) expr -> left, self -> schema, & cast, self -> cx_bind );
if ( rc == 0 )
{
/* resolve rh expression */
@@ -183,7 +183,7 @@ rc_t VProdResolveSProduction ( const VProdResolve *self, VProduction **out, cons
else
{
rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd );
+ ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd, self -> cx_bind );
}
}
if ( rc == 0 )
@@ -762,7 +762,7 @@ rc_t VProdResolveColumnRead ( const VProdResolve *self,
/* create a simple prod to manage fd and desc */
name = scol -> name -> name . addr;
assert ( name [ scol -> name -> name . size ] == 0 );
- rc = VSimpleProdMake ( out, self -> owned, self->curs, prodSimpleCast,
+ rc = VSimpleProdMake ( out, self -> owned, self -> curs, prodSimpleCast,
name, & fd, & vcol -> desc, NULL, * out, self -> chain );
assert (rc != -1);
diff --git a/libs/vdb/prod-expr.h b/libs/vdb/prod-expr.h
index 9fbf04a..d35a839 100644
--- a/libs/vdb/prod-expr.h
+++ b/libs/vdb/prod-expr.h
@@ -104,6 +104,9 @@ struct VProdResolve
/* vector of owned productions */
Vector *owned;
+ /* vector of late-binding expressions */
+ Vector *cx_bind;
+
/* which chain is being resolved
chainEncoding - when going from input to physical
chainDecoding - when pulling from physical to output
diff --git a/libs/vdb/prod-func.c b/libs/vdb/prod-func.c
index 173bd09..8c3f8ef 100644
--- a/libs/vdb/prod-func.c
+++ b/libs/vdb/prod-func.c
@@ -61,17 +61,28 @@
/*--------------------------------------------------------------------------
* VProdResolve
*/
+typedef struct SFunctionPushParam SFunctionPushParam;
+struct SFunctionPushParam
+{
+ /* vectors to allow recursive evaluation of types and functions */
+ Vector schema_prior;
+ Vector fact_parms, fact_prior;
+
+ VFormatdecl fd;
+ VTypedesc desc;
+};
static
-rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
- const SFuncExpr *expr, Vector *schema_prior, Vector *fact_prior,
- const VFormatdecl *lval_fd, VFormatdecl *fd, VTypedesc *desc )
+rc_t SFunctionPush ( const SFunction *self, const VProdResolve *pr,
+ const VFormatdecl *lval_fd, const SFuncExpr *expr, SFunctionPushParam *pb )
{
rc_t rc;
+ const VSchema *schema = pr -> schema;
/* validate input parameter counts against formal list */
uint32_t count = VectorLength ( & expr -> pfunc );
- if ( count < self -> func . mand ) {
+ if ( count < self -> func . mand )
+ {
rc = RC ( rcVDB, rcFunction, rcResolving, rcParam, rcInsufficient );
PLOGERR ( klogWarn, ( klogWarn, rc,
"function '$(func)' requires $(mand) params but got $(count)",
@@ -84,7 +95,8 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
else
{
uint32_t end = VectorLength ( & self -> func . parms );
- if ( end < count && ! self -> func . vararg ) {
+ if ( end < count && ! self -> func . vararg )
+ {
rc = RC ( rcVDB, rcFunction, rcResolving, rcParam, rcExcessive );
PLOGERR ( klogWarn, ( klogWarn, rc,
"function '$(func)' requires $(mand) params but got $(count)",
@@ -100,33 +112,33 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
else
{
/* bind schema params */
- rc = SFunctionBindSchemaParms ( self, schema_prior, & expr -> schem );
+ rc = SFunctionBindSchemaParms ( self, & pb -> schema_prior, & expr -> schem, pr -> cx_bind );
if ( rc == 0 )
{
/* resolve return type */
if ( self -> validate )
{
/* set type to "void" */
- fd -> td . type_id = fd -> td . dim = 1;
- fd -> fmt = 0;
- memset ( desc, 0, sizeof * desc );
- desc -> intrinsic_bits = 1;
- desc -> intrinsic_dim = 1;
+ pb -> fd . td . type_id = pb -> fd . td . dim = 1;
+ pb -> fd . fmt = 0;
+ memset ( & pb -> desc, 0, sizeof pb -> desc );
+ pb -> desc . intrinsic_bits = 1;
+ pb -> desc . intrinsic_dim = 1;
}
else
{
rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) self -> rt, schema, fd );
+ ( ( const STypeExpr* ) self -> rt, schema, & pb -> fd, pr -> cx_bind );
if ( rc == 0 )
{
/* pick up cast from lval when return type is "any" or typeset */
- if ( fd -> td . type_id == 0 ||
- fd -> td . type_id >= 0x40000000 )
+ if ( pb -> fd . td . type_id == 0 ||
+ pb -> fd . td . type_id >= 0x40000000 )
{
VFormatdecl cast;
- if ( VFormatdeclCommonAncestor ( fd, schema, lval_fd, & cast, NULL ) )
- * fd = cast;
+ if ( VFormatdeclCommonAncestor ( & pb -> fd, schema, lval_fd, & cast, NULL ) )
+ pb -> fd = cast;
else
{
const KSymbol *fd_name = NULL;
@@ -136,12 +148,12 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
if ( dt != NULL )
lval_name = dt -> name;
- dt = VSchemaFindTypeid ( schema, fd -> td . type_id );
+ dt = VSchemaFindTypeid ( schema, pb -> fd . td . type_id );
if ( dt != NULL )
fd_name = dt -> name;
else
{
- const STypeset *ts = VSchemaFindTypesetid ( schema, fd -> td . type_id );
+ const STypeset *ts = VSchemaFindTypesetid ( schema, pb -> fd . td . type_id );
if ( ts != NULL )
fd_name = ts->name;
}
@@ -154,7 +166,7 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
self->name->name.len, self->name->name.addr,
lval_name->name.len, lval_name->name.addr,
fd_name->name.len, fd_name->name.addr
- ));
+ ));
}
else
{
@@ -168,7 +180,7 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
}
/* pick up cast from lval when return dimension is "*" */
- else if ( fd -> td . dim == 0 )
+ else if ( pb -> fd . td . dim == 0 )
{
VTypedesc lh_desc, rh_desc;
/* since rh type is T' and is cast to lh type T,
@@ -177,35 +189,35 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
if ( rc == 0 )
{
VTypedecl rh_td;
- rh_td . type_id = fd -> td . type_id;
+ rh_td . type_id = pb -> fd . td . type_id;
rh_td . dim = 1;
rc = VSchemaDescribeTypedecl ( schema, & rh_desc, & rh_td );
if ( rc == 0 )
{
- fd -> td . dim = VTypedescSizeof ( & lh_desc ) /
+ pb -> fd . td . dim = VTypedescSizeof ( & lh_desc ) /
VTypedescSizeof ( & rh_desc );
/* force dimension "*" to be at least 1 */
- if ( fd -> td . dim == 0 )
- fd -> td . dim = 1;
+ if ( pb -> fd . td . dim == 0 )
+ pb -> fd . td . dim = 1;
}
}
}
/* evaluate type description */
- rc = VSchemaDescribeTypedecl ( schema, desc, & fd -> td );
+ rc = VSchemaDescribeTypedecl ( schema, & pb -> desc, & pb -> fd . td );
}
}
if ( rc == 0 )
{
/* bind factory params */
- rc = SFunctionBindFactParms ( self, fact_prior, & expr -> pfact );
+ rc = SFunctionBindFactParms ( self, & pb -> fact_parms, & pb -> fact_prior, & expr -> pfact, pr -> cx_bind );
if ( rc == 0 )
return 0;
}
- SFunctionRestSchemaParms ( self, schema_prior );
+ SFunctionRestSchemaParms ( self, & pb -> schema_prior, pr -> cx_bind );
}
}
}
@@ -215,10 +227,12 @@ rc_t SFunctionPush ( const SFunction *self, const VSchema *schema,
}
static
-void SFunctionPop ( const SFunction *self, Vector *schema_prior, Vector *fact_prior )
+void SFunctionPop ( const SFunction *self,
+ const VProdResolve *pr, SFunctionPushParam *pb )
{
- SFunctionRestFactParms ( self, fact_prior );
- SFunctionRestSchemaParms ( self, schema_prior );
+ SFunctionRestFactParms ( self, & pb -> fact_prior, pr -> cx_bind );
+ SFunctionRestSchemaParms ( self, & pb -> schema_prior, pr -> cx_bind );
+ VectorWhack ( & pb -> fact_parms, NULL, NULL );
}
static
@@ -293,7 +307,7 @@ rc_t VProdResolveValidateFuncParams ( const VProdResolve *self, Vector *out,
/* resolve production type */
sprod = ( const void* ) VectorGet ( & formal -> parms, 0 );
rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd );
+ ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd, self -> cx_bind );
if ( rc == 0 )
{
/* bring in source parameter */
@@ -310,7 +324,7 @@ rc_t VProdResolveValidateFuncParams ( const VProdResolve *self, Vector *out,
/* comparison formal */
sprod = ( const void* ) VectorGet ( & formal -> parms, 1 );
rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd );
+ ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd, self -> cx_bind );
if ( rc == 0 )
{
/* bring in comparison parameter */
@@ -349,7 +363,7 @@ rc_t VProdResolveFuncParams ( const VProdResolve *self, Vector *out,
/* resolve production type */
const SProduction *sprod = ( const void* ) VectorGet ( & formal -> parms, i );
rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd );
+ ( ( const STypeExpr* ) sprod -> fd, self -> schema, & fd, self -> cx_bind );
if ( rc == 0 )
{
/* bring in parameter */
@@ -384,11 +398,8 @@ rc_t VProdResolveScriptExpr ( const VProdResolve *self,
{
const SFunction *func = expr -> func;
- VTypedesc desc;
- VFormatdecl fd;
- Vector schema_prior, fact_prior;
- rc_t rc = SFunctionPush ( func, self -> schema, expr,
- & schema_prior, & fact_prior, lval_fd, & fd, & desc );
+ SFunctionPushParam pb;
+ rc_t rc = SFunctionPush ( func, self, lval_fd, expr, & pb );
if ( rc == 0 )
{
Vector *inputs = malloc ( sizeof * inputs );
@@ -400,13 +411,14 @@ rc_t VProdResolveScriptExpr ( const VProdResolve *self,
& func -> func, & expr -> pfunc );
if ( rc == 0 )
{
- /* script name */
+ /* script name
+ heuristic - known to have been created with NUL term */
const char *name = expr -> func -> name -> name . addr;
/* by this time, we have bound all parameters */
VScriptProd *script;
- rc = VScriptProdMake ( & script, self -> owned, self->curs,
- prodScriptFunction, name, & fd, & desc, self -> chain );
+ rc = VScriptProdMake ( & script, self -> owned, self -> curs,
+ prodScriptFunction, name, & pb . fd, & pb . desc, self -> chain );
if ( rc == 0 )
{
VCursorCache local;
@@ -424,7 +436,7 @@ rc_t VProdResolveScriptExpr ( const VProdResolve *self,
inputs = NULL;
/* now evaluate the return expression */
- rc = VProdResolveExpr ( & spr, & script -> rtn, NULL, & fd,
+ rc = VProdResolveExpr ( & spr, & script -> rtn, NULL, & pb . fd,
func -> u . script . rtn, false );
if ( rc != 0 || script -> rtn == NULL )
VProductionWhack ( & script -> dad, self -> owned );
@@ -450,18 +462,19 @@ rc_t VProdResolveScriptExpr ( const VProdResolve *self,
}
}
- SFunctionPop ( func, & schema_prior, & fact_prior );
+ SFunctionPop ( func, self, & pb );
}
return rc;
}
static
-rc_t VFunctionProdMakeFactParms ( VFunctionProd *self, const VSchema *schema,
- const SFormParmlist *pfact, VFactoryParams *fp, const SConstExpr **cxp )
+rc_t VFunctionProdMakeFactParms ( VFunctionProd *self, const VProdResolve *pr,
+ const SFormParmlist *pfact, const Vector *pval, VFactoryParams *fp, const SConstExpr **cxp )
{
rc_t rc;
VTypedecl td;
+ const VSchema *schema = pr -> schema;
uint32_t i, last = 0, count = fp -> argc;
for ( rc = 0, i = 0; rc == 0 && i < count; ++ i )
@@ -469,30 +482,36 @@ rc_t VFunctionProdMakeFactParms ( VFunctionProd *self, const VSchema *schema,
/* fetch the formal parameter, which can be NULL
when the factory parameters allow varargs */
const SIndirectConst *ic = ( const void* ) VectorGet ( & pfact -> parms, i );
- if( ic == NULL ) {
- assert( i != 0 );
+ if ( ic != NULL )
+ last = i;
+ else
+ {
+ assert ( i != 0 );
+
/* just use the last parameter's type */
ic = ( const void* ) VectorGet ( & pfact -> parms, last );
- assert(ic != NULL);
- } else {
- last = i;
+ assert ( ic != NULL );
}
+
/* TBD - nobody is using function pointer parameters yet */
- if( ic -> td == NULL ) {
+ if ( ic -> td == NULL )
+ {
rc = RC ( rcVDB, rcFunction, rcConstructing, rcType, rcNull );
LOGERR ( klogFatal, rc, "function factory with function param" );
break;
}
- /* evaluate constant typedecl */
- rc = STypeExprResolveAsTypedecl ( ( const STypeExpr* ) ic -> td, schema, & td );
- if ( rc == 0 ) {
+ /* evaluate constant typedecl */
+ rc = STypeExprResolveAsTypedecl ( ( const STypeExpr* ) ic -> td, schema, & td, pr -> cx_bind );
+ if ( rc == 0 )
rc = VSchemaDescribeTypedecl ( schema, & fp -> argv [ i ] . desc, & td );
- }
+
if ( rc == 0 )
{
/* evaluate constant expression */
- rc = eval_const_expr ( schema, & td, ic -> expr, ( SExpression** ) & cxp [ i ] );
+ const SExpression *ic_expr = ( const void* ) VectorGet ( pval, i );
+ assert ( ic_expr != NULL );
+ rc = eval_const_expr ( schema, & td, ic_expr, ( SExpression** ) & cxp [ i ], pr -> cx_bind );
if ( rc == 0 )
{
/* catch undefined factory parameters from outer script */
@@ -534,10 +553,12 @@ rc_t VProdResolveInvokeFactory ( const VProdResolve *self, const VXfactInfo *inf
VTransDesc *td, bool external )
{
rc_t rc;
+
VFuncDesc desc;
memset ( & desc, 0, sizeof desc );
+
rc = ( * td -> factory ) ( td -> fself, info, & desc, cp, dp );
- assert (rc != -1);
+ assert ( rc != -1 );
/* clobber returned context */
if ( rc != 0 )
@@ -602,18 +623,33 @@ rc_t VProdResolveBuildFuncParms ( const VProdResolve *self, const VXfactInfo *in
static
rc_t VProdResolveBuildFactParms ( const VProdResolve *self, const VXfactInfo *info,
- VFunctionProd *fprod, const SFormParmlist *pfact, uint32_t count, VTransDesc *td, bool external )
+ VFunctionProd *fprod, const SFormParmlist *pfact, const Vector *pval,
+ VTransDesc *td, bool external )
{
rc_t rc;
uint32_t i;
+ /* count the number of parameters */
+ uint32_t count = VectorLength ( pval );
+
/* create a block of factory parameters
start with data on the stack */
VFactoryParams fpbuff, * fp = & fpbuff;
const SConstExpr *cxbuff [ 16 ], **cxp = cxbuff;
+ /* 9/11/13
+ VFactoryParams has a declared array of 16 elements.
+ cxbuff is also declared to have 16 elements. the idea
+ is that they stay in sync. */
+ assert ( sizeof fpbuff . argv / sizeof fpbuff . argv [ 0 ] ==
+ sizeof cxbuff / sizeof cxbuff [ 0 ] );
+
+ /* 16 is generally more than enough parameters.
+ but in cases where it may not be ( e.g. maps ),
+ malloc the memory according to count. */
if ( count > sizeof fpbuff . argv / sizeof fpbuff . argv [ 0 ] )
{
+ /* allocate both structures in a single block */
fp = malloc ( sizeof fpbuff -
sizeof fpbuff . argv +
count * ( sizeof fpbuff . argv [ 0 ] + sizeof cxbuff [ 0 ] ) );
@@ -621,20 +657,28 @@ rc_t VProdResolveBuildFactParms ( const VProdResolve *self, const VXfactInfo *in
if ( fp == NULL )
return RC ( rcVDB, rcFunction, rcResolving, rcMemory, rcExhausted );
+ /* reset expression array to point within block */
cxp = ( const SConstExpr** ) & fp -> argv [ count ];
}
fp -> argc = count;
fp -> align = 0;
- rc = VFunctionProdMakeFactParms ( fprod, self -> schema, pfact, fp, cxp );
- assert (rc != -1);
+ /* the purpose of this function is to bind type expressions to type ids
+ before the remainder of the function is evaluated. for this to work,
+ we need a mapping of parameterized type ids to expressions.
+
+ the purpose of the cx array is to enable recursion by saving previously
+ bound expressions upon entry ... */
+ rc = VFunctionProdMakeFactParms ( fprod, self, pfact, pval, fp, cxp );
+ assert ( rc != -1 );
if ( rc == 0 )
rc = VProdResolveBuildFuncParms ( self, info, fprod, fp, td, external );
- assert (rc != -1);
+ assert ( rc != -1 );
for ( i = 0; i < count; ++ i )
SExpressionWhack ( & cxp [ i ] -> dad );
+
if ( fp != & fpbuff )
free ( fp );
@@ -647,52 +691,68 @@ rc_t VProdResolveFuncExpr ( const VProdResolve *self,
const SFunction *sfunc = expr -> func;
const char *name = sfunc -> name -> name . addr;
- VXfactInfo info;
- Vector schema_prior, fact_prior;
- rc_t rc = SFunctionPush ( sfunc, self -> schema, expr,
- & schema_prior, & fact_prior, lval_fd, & info . fdesc . fd, & info . fdesc . desc );
- assert (rc != -1);
+ SFunctionPushParam pb;
+ rc_t rc = SFunctionPush ( sfunc, self, lval_fd, expr, & pb );
+
+ assert ( rc != -1 );
+
if ( rc == 0 )
{
- /* locate factory function */
bool external;
VTransDesc td;
+ VXfactInfo info;
+
+ /* pass function type information */
+ info . fdesc . fd = pb . fd;
+ info . fdesc . desc = pb . desc;
+
+ /* locate factory function */
rc = VLinkerFindFactory ( self -> ld, self -> libs, & td, sfunc, & external );
- assert (rc != -1);
if ( rc == 0 )
{
+ /* create a production representing function */
VFunctionProd *fprod;
rc = VFunctionProdMake ( & fprod, self -> owned,
self -> curs, vftInvalid, name, & info . fdesc . fd,
& info . fdesc . desc, self -> chain );
- assert (rc != -1);
if ( rc == 0 )
{
+ /* check for a validation function
+ these functions are generally compiler-generated */
if ( sfunc -> validate )
{
rc = VProdResolveValidateFuncParams ( self, & fprod -> parms,
& sfunc -> func, & expr -> pfunc );
- assert (rc != -1);
}
else
{
+ /* resolve normal function parameters */
rc = VProdResolveFuncParams ( self, & fprod -> parms,
& sfunc -> func, & expr -> pfunc );
- assert (rc != -1);
}
+
if ( rc == 0 )
{
+ /* prepare info block for function factory */
info . schema = self -> schema;
info . tbl = self -> curs -> tbl;
info . mgr = info . tbl -> mgr;
- info . parms = (struct VCursorParams *) self -> curs;
-
- rc = VProdResolveBuildFactParms ( self, & info,
- fprod, & sfunc -> fact, VectorLength ( & expr -> pfact ),
- & td, external );
- assert (rc != -1);
+ info . parms = ( struct VCursorParams * ) self -> curs;
+
+ /* by this point the schema and factory parameters
+ have been bound: named type and constant parameters
+ are entered into the "self -> cx_bind" Vector,
+ factory parameter constants are positionally recorded
+ in "pb . fact_parms", and function parameter productions
+ are recorded in "fprod" */
+
+ /* the following function will evaluate the parameters
+ and invoke the factory function to construct production */
+ rc = VProdResolveBuildFactParms ( self, & info, fprod,
+ & sfunc -> fact, & pb . fact_parms, & td, external );
if ( rc == 0 )
{
+ /* successfully created */
* out = & fprod -> dad;
}
}
@@ -715,12 +775,12 @@ rc_t VProdResolveFuncExpr ( const VProdResolve *self,
{
/* TBD - perhaps report what went wrong, but if function
could not be resolved, don't stop the whole process */
- VDB_DEBUG (("failed to find function '%S' rc %R",
- &sfunc->name->name, rc));
+ VDB_DEBUG (( "failed to find function '%S' rc %R",
+ & sfunc -> name -> name, rc ));
rc = 0;
}
- SFunctionPop ( sfunc, & schema_prior, & fact_prior );
+ SFunctionPop ( sfunc, self, & pb );
}
return rc;
@@ -735,11 +795,11 @@ rc_t VProdResolveEncodingExpr ( const VProdResolve *self,
/* bind schema and factory params */
Vector schema_prior;
- rc_t rc = SPhysicalBindSchemaParms ( sphys, & schema_prior, & expr -> schem );
+ rc_t rc = SPhysicalBindSchemaParms ( sphys, & schema_prior, & expr -> schem, self -> cx_bind );
if ( rc == 0 )
{
- Vector fact_prior;
- rc = SPhysicalBindFactParms ( sphys, & fact_prior, & expr -> pfact );
+ Vector fact_parms, fact_prior;
+ rc = SPhysicalBindFactParms ( sphys, & fact_parms, & fact_prior, & expr -> pfact, self -> cx_bind );
if ( rc == 0 )
{
/* create single input param vector */
@@ -759,14 +819,14 @@ rc_t VProdResolveEncodingExpr ( const VProdResolve *self,
else
{
rc = STypeExprResolveAsFormatdecl
- ( ( const STypeExpr* ) sphys -> td, self -> schema, & fd );
+ ( ( const STypeExpr* ) sphys -> td, self -> schema, & fd, self -> cx_bind );
}
if ( rc == 0 )
{
VScriptProd *script;
const char *name = sphys -> name -> name . addr;
- rc = VScriptProdMake ( & script, self -> owned, self->curs,
+ rc = VScriptProdMake ( & script, self -> owned, self -> curs,
prodScriptFunction, name, & fd, NULL, self -> chain );
if ( rc == 0 )
{
@@ -811,10 +871,11 @@ rc_t VProdResolveEncodingExpr ( const VProdResolve *self,
}
}
- SPhysicalRestFactParms ( sphys, & fact_prior );
+ SPhysicalRestFactParms ( sphys, & fact_prior, self -> cx_bind );
+ VectorWhack ( & fact_parms, NULL, NULL );
}
- SPhysicalRestSchemaParms ( sphys, & schema_prior );
+ SPhysicalRestSchemaParms ( sphys, & schema_prior, self -> cx_bind );
}
return rc;
diff --git a/libs/vdb/prod.c b/libs/vdb/prod.c
index eb9730d..2dce059 100644
--- a/libs/vdb/prod.c
+++ b/libs/vdb/prod.c
@@ -79,7 +79,7 @@ rc_t VProdResolveColumn ( const VProdResolve *self,
vcol = VCursorCacheGet ( & curs -> col, & scol -> cid );
if ( vcol == NULL )
{
- rc = VCursorMakeColumn ( curs, & vcol, scol );
+ rc = VCursorMakeColumn ( curs, & vcol, scol, self -> cx_bind );
if ( rc != 0 )
return rc;
diff --git a/libs/vdb/schema-eval.c b/libs/vdb/schema-eval.c
index 07a54a0..2c8c5b7 100644
--- a/libs/vdb/schema-eval.c
+++ b/libs/vdb/schema-eval.c
@@ -906,11 +906,11 @@ rc_t SConstExprResolveAsU16 ( const SConstExpr *self,
const VSchema *schema, uint16_t *u16, uint32_t capacity );
rc_t SConstExprResolveAsU32 ( const SConstExpr *self,
- const VSchema *schema, uint32_t *u32, uint32_t capacity )
+ const VSchema *schema, uint32_t *u32, uint32_t capacity, Vector *cx_bind )
{
if ( capacity != 1 )
return RC ( rcVDB, rcExpression, rcEvaluating, rcType, rcUnsupported );
- return eval_uint_expr ( schema, ( const SExpression* ) self, u32 );
+ return eval_uint_expr ( schema, ( const SExpression* ) self, u32, cx_bind );
}
rc_t SConstExprResolveAsU64 ( const SConstExpr *self,
@@ -937,7 +937,7 @@ rc_t SConstExprResolveAsUTF32 ( const SConstExpr *self,
* resolve type expression to either a VTypedecl or VFormatdecl
*/
rc_t STypeExprResolveAsFormatdecl ( const STypeExpr *self,
- const VSchema *schema, VFormatdecl *fd )
+ const VSchema *schema, VFormatdecl *fd, Vector *cx_bind )
{
rc_t rc;
uint32_t dim;
@@ -959,12 +959,12 @@ rc_t STypeExprResolveAsFormatdecl ( const STypeExpr *self,
if ( self -> id != NULL )
{
/* the type needs to be resolved */
- const STypeExpr *type = ( const STypeExpr* ) self -> id -> type;
+ const STypeExpr *type = ( const STypeExpr* ) VectorGet ( cx_bind, self -> id -> type_id );
if ( type == NULL )
return RC ( rcVDB, rcExpression, rcEvaluating, rcType, rcUndefined );
if ( type -> dad . var != eTypeExpr )
return RC ( rcVDB, rcExpression, rcEvaluating, rcType, rcIncorrect );
- rc = STypeExprResolveAsFormatdecl ( type, schema, fd );
+ rc = STypeExprResolveAsFormatdecl ( type, schema, fd, cx_bind );
if ( rc != 0 )
return rc;
@@ -992,7 +992,7 @@ rc_t STypeExprResolveAsFormatdecl ( const STypeExpr *self,
if ( dx -> var == eConstExpr )
{
- rc = SConstExprResolveAsU32 ( ( const SConstExpr* ) dx, schema, & dim, 1 );
+ rc = SConstExprResolveAsU32 ( ( const SConstExpr* ) dx, schema, & dim, 1, cx_bind );
if ( rc != 0 )
return rc;
break;
@@ -1004,7 +1004,7 @@ rc_t STypeExprResolveAsFormatdecl ( const STypeExpr *self,
ic = ( ( const SSymExpr* ) dx ) -> _sym -> u . obj;
assert ( ic != NULL );
- dx = ic -> expr;
+ dx = ( const SExpression* ) VectorGet ( cx_bind, ic -> expr_id );
}
/* must have non-zero dim */
@@ -1019,10 +1019,10 @@ rc_t STypeExprResolveAsFormatdecl ( const STypeExpr *self,
}
rc_t STypeExprResolveAsTypedecl ( const STypeExpr *self,
- const VSchema *schema, VTypedecl *td )
+ const VSchema *schema, VTypedecl *td, Vector *cx_bind )
{
VFormatdecl fd;
- rc_t rc = STypeExprResolveAsFormatdecl ( self, schema, & fd );
+ rc_t rc = STypeExprResolveAsFormatdecl ( self, schema, & fd, cx_bind );
if ( rc == 0 )
{
if ( fd . fmt != 0 )
@@ -1053,13 +1053,17 @@ rc_t eval_type_expr ( const VSchema *self, const VTypedecl *td,
*/
static
rc_t eval_indirect_expr ( const VSchema *self, const VTypedecl *td,
- const SSymExpr *expr, SExpression **xp )
+ const SSymExpr *expr, SExpression **xp, Vector *cx_bind )
{
const SIndirectConst *ic = expr -> _sym -> u . obj;
/* if the expression is there */
- if ( ic -> expr != NULL )
- return eval_const_expr ( self, td, ic -> expr, xp );
+ if ( ic -> expr_id != 0 )
+ {
+ const SExpression *ic_expr = ( const SExpression* ) VectorGet ( cx_bind, ic -> expr_id );
+ if ( ic_expr != NULL )
+ return eval_const_expr ( self, td, ic_expr, xp, cx_bind );
+ }
/* just return self */
* xp = & ( ( SSymExpr* ) expr ) -> dad;
@@ -1089,10 +1093,12 @@ rc_t eval_func_param_expr ( const VSchema *self, const VTypedecl *td,
return -1;
}
-struct eval_vector_param_expr_pb {
+struct eval_vector_param_expr_pb
+{
rc_t rc;
const VSchema *self;
const VTypedecl *td;
+ Vector *cx_bind;
Vector v;
};
@@ -1103,23 +1109,26 @@ void CC vector_free ( void *item, void *data )
}
static
-bool CC do_eval_vector_param_expr ( void *item, void *data ) {
- struct eval_vector_param_expr_pb *pb = (struct eval_vector_param_expr_pb *)data;
- SExpression *rslt = 0;
+bool CC do_eval_vector_param_expr ( void *item, void *data )
+{
+ struct eval_vector_param_expr_pb *pb = data;
+ SExpression *rslt = NULL;
- pb->rc = eval_const_expr(pb->self, pb->td, (struct SExpression const *)item, &rslt);
- if (pb->rc)
+ pb -> rc = eval_const_expr ( pb -> self, pb -> td, item, & rslt, pb -> cx_bind );
+ if ( pb -> rc != 0 )
return true;
- switch (rslt->var) {
+
+ switch ( rslt -> var )
+ {
case eConstExpr:
- VectorAppend(&pb->v, 0, rslt);
+ VectorAppend ( & pb -> v, NULL, rslt );
break;
case eVectorExpr:
- free(rslt);
+ free ( rslt );
break;
default:
- free(rslt);
- pb->rc = RC ( rcVDB, rcExpression, rcEvaluating, rcExpression, rcUnexpected );
+ free ( rslt );
+ pb -> rc = RC ( rcVDB, rcExpression, rcEvaluating, rcExpression, rcUnexpected );
return true;
}
@@ -1129,16 +1138,17 @@ bool CC do_eval_vector_param_expr ( void *item, void *data ) {
/* eval-vector-param-expr
*/
static
-rc_t eval_vector_param_expr ( const VSchema *self,
- const VTypedecl *td, const SVectExpr *expr, SExpression **xp )
+rc_t eval_vector_param_expr ( const VSchema *self, const VTypedecl *td,
+ const SVectExpr *expr, SExpression **xp, Vector *cx_bind )
{
struct eval_vector_param_expr_pb pb;
*xp = NULL;
- pb.rc = 0;
- pb.self = self;
- pb.td = td;
+ pb . rc = 0;
+ pb . self = self;
+ pb . td = td;
+ pb . cx_bind = cx_bind;
VectorInit ( & pb . v, 0, 32 );
VectorDoUntil ( & expr -> expr, 0, do_eval_vector_param_expr, & pb );
@@ -1198,7 +1208,7 @@ rc_t eval_vector_param_expr ( const VSchema *self,
* returns non-zero error code if failed
*/
rc_t eval_const_expr ( const VSchema *self, const VTypedecl *td,
- const SExpression *expr, SExpression **xp )
+ const SExpression *expr, SExpression **xp, Vector *cx_bind )
{
rc_t rc;
const SConstExpr *s;
@@ -1212,7 +1222,7 @@ rc_t eval_const_expr ( const VSchema *self, const VTypedecl *td,
break;
#if SLVL >= 3
case eIndirectExpr:
- return eval_indirect_expr ( self, td, ( const SSymExpr* ) expr, xp );
+ return eval_indirect_expr ( self, td, ( const SSymExpr* ) expr, xp, cx_bind );
#endif
#if SLVL >= 4
case eFuncParamExpr:
@@ -1221,7 +1231,7 @@ rc_t eval_const_expr ( const VSchema *self, const VTypedecl *td,
case eCastExpr:
return eval_const_cast_expr ( self, td, expr, xp );
case eVectorExpr:
- return eval_vector_param_expr(self, td, ( const SVectExpr* )expr, xp);
+ return eval_vector_param_expr(self, td, ( const SVectExpr* ) expr, xp, cx_bind );
default:
*xp = NULL;
return RC ( rcVDB, rcExpression, rcEvaluating, rcExpression, rcUnexpected );
@@ -1241,7 +1251,7 @@ rc_t eval_const_expr ( const VSchema *self, const VTypedecl *td,
* special const expression evaluator for uint32_t
*/
rc_t eval_uint_expr ( const VSchema *self,
- const SExpression *expr, uint32_t *value )
+ const SExpression *expr, uint32_t *value, Vector *cx_bind )
{
rc_t rc;
VTypedecl td;
@@ -1254,7 +1264,7 @@ rc_t eval_uint_expr ( const VSchema *self,
/* evaluate expression against type */
td . type_id = U32_id;
td . dim = 1;
- rc = eval_const_expr ( self, & td, expr, ( SExpression** ) & x );
+ rc = eval_const_expr ( self, & td, expr, ( SExpression** ) & x, cx_bind );
if ( rc != 0 )
return rc;
@@ -1274,7 +1284,7 @@ rc_t eval_uint_expr ( const VSchema *self,
* special const expression evaluator for uint32_t
*/
rc_t eval_uint64_expr ( const VSchema *self,
- const SExpression *expr, uint64_t *value )
+ const SExpression *expr, uint64_t *value, Vector *cx_bind )
{
rc_t rc;
VTypedecl td;
@@ -1287,7 +1297,7 @@ rc_t eval_uint64_expr ( const VSchema *self,
/* evaluate expression against type */
td . type_id = U64_id;
td . dim = 1;
- rc = eval_const_expr ( self, & td, expr, ( SExpression** ) & x );
+ rc = eval_const_expr ( self, & td, expr, ( SExpression** ) & x, cx_bind );
if ( rc != 0 )
return rc;
diff --git a/libs/vdb/schema-expr.c b/libs/vdb/schema-expr.c
index df73ba2..139cdee 100644
--- a/libs/vdb/schema-expr.c
+++ b/libs/vdb/schema-expr.c
@@ -632,10 +632,8 @@ void STypeExprMark ( const STypeExpr *self, const VSchema *schema )
SDatatypeMark ( self -> dt );
if ( self -> ts != NULL )
STypesetMark ( self -> ts, schema );
- if ( self -> id != NULL )
- SIndirectTypeMark ( ( void * )self -> id, ( void * )schema );
if ( self -> dim )
- SExpressionMark ( ( void * )self -> dim, ( void * )schema );
+ SExpressionMark ( ( void * ) self -> dim, ( void * ) schema );
}
@@ -2145,15 +2143,17 @@ rc_t phys_encoding_expr ( KSymTable *tbl, KTokenSource *src, KToken *t,
if ( td != NULL )
{
/* bind schema parameters */
- Vector prior;
- rc = SPhysicalBindSchemaParms ( x -> phys, & prior, & x -> schem );
+ Vector prior, cx_bind;
+ VectorInit ( & cx_bind, 1, self -> num_indirect ? self -> num_indirect : 16 );
+ rc = SPhysicalBindSchemaParms ( x -> phys, & prior, & x -> schem, & cx_bind );
if ( rc == 0 )
{
const SExpression *tx = x -> phys -> td;
assert ( tx != NULL );
- rc = STypeExprResolveAsTypedecl ( ( const STypeExpr* ) tx, self, td );
- SPhysicalRestSchemaParms ( x -> phys, & prior );
+ rc = STypeExprResolveAsTypedecl ( ( const STypeExpr* ) tx, self, td, & cx_bind );
+ SPhysicalRestSchemaParms ( x -> phys, & prior, & cx_bind );
}
+ VectorWhack ( & cx_bind, NULL, NULL );
}
if ( rc == 0 )
diff --git a/libs/vdb/schema-expr.h b/libs/vdb/schema-expr.h
index 40e9337..b815890 100644
--- a/libs/vdb/schema-expr.h
+++ b/libs/vdb/schema-expr.h
@@ -155,9 +155,9 @@ struct STypeExpr
* resolve type expression to either a VTypedecl or VFormatdecl
*/
rc_t STypeExprResolveAsTypedecl ( const STypeExpr *self,
- struct VSchema const *schema, VTypedecl *td );
+ struct VSchema const *schema, VTypedecl *td, Vector *cx_bind );
rc_t STypeExprResolveAsFormatdecl ( const STypeExpr *self,
- struct VSchema const *schema, VFormatdecl *fd );
+ struct VSchema const *schema, VFormatdecl *fd, Vector *cx_bind );
/* Mark
*/
@@ -232,7 +232,7 @@ rc_t SConstExprResolveAsU8 ( const SConstExpr *self,
rc_t SConstExprResolveAsU16 ( const SConstExpr *self,
struct VSchema const *schema, uint16_t *u16, uint32_t capacity );
rc_t SConstExprResolveAsU32 ( const SConstExpr *self,
- struct VSchema const *schema, uint32_t *u32, uint32_t capacity );
+ struct VSchema const *schema, uint32_t *u32, uint32_t capacity, Vector *cx_bind );
rc_t SConstExprResolveAsU64 ( const SConstExpr *self,
struct VSchema const *schema, uint32_t *u64, uint32_t capacity );
rc_t SConstExprResolveAsF32 ( const SConstExpr *self,
diff --git a/libs/vdb/schema-func.c b/libs/vdb/schema-func.c
index a29ee0f..43065f7 100644
--- a/libs/vdb/schema-func.c
+++ b/libs/vdb/schema-func.c
@@ -31,6 +31,7 @@
#include <klib/symbol.h>
#include <klib/symtab.h>
+#include <klib/out.h>
#include <klib/rc.h>
#include <sysalloc.h>
@@ -50,7 +51,6 @@
void CC SIndirectConstWhack ( void *item, void *ignore )
{
SIndirectConst * self = item;
- SExpressionWhack ( self -> expr );
SExpressionWhack ( self -> td );
free ( self );
}
@@ -61,10 +61,7 @@ void CC SIndirectConstMark ( void * item, void * data )
{
const SIndirectConst * self = item;
if ( self != NULL )
- {
- SExpressionMark ( ( void * )self -> td, data );
- SExpressionMark ( ( void * )self -> expr, data );
- }
+ SExpressionMark ( ( void* ) self -> td, data );
}
/* Dump
@@ -99,9 +96,7 @@ bool CC SIndirectConstDefDump ( void *item, void *data )
*/
void CC SIndirectTypeWhack ( void *item, void *ignore )
{
- SIndirectType *self = item;
- SExpressionWhack ( self -> type );
- free ( self );
+ free ( item );
}
@@ -121,16 +116,6 @@ SIndirectType *VSchemaFindITypeid ( const VSchema *self, uint32_t id )
}
-/* Mark
- */
-void CC SIndirectTypeMark ( void * item, void * data )
-{
- const SIndirectType * self = item;
- if ( self != NULL )
- SExpressionMark ( ( void * ) self -> type, data );
-}
-
-
/* Dump
*/
rc_t SIndirectTypeDump ( const SIndirectType *self, struct SDumper *d )
@@ -283,20 +268,44 @@ int CC SFunctionSort ( const void *item, const void *n )
return ( int ) ( a -> version >> 24 ) - ( int ) ( b -> version >> 24 );
}
+#if _DEBUGGING && 0
+static String no_name = { "<no-name>", sizeof "<no-name>" - 1, sizeof "<no-name>" - 1 };
+#define DBG_CXBIND1( op, name, id, val ) \
+ OUTMSG (( "%s:%d - cx_bind %s: name=%S, id=%u, val=%p\n", __func__, __LINE__, op, & name, id, val ))
+#define DBG_CXBIND2( op, name, id, old, new ) \
+ OUTMSG (( "%s:%d - cx_bind %s: name=%S, id=%u, old=%p, new=%p\n", __func__, __LINE__, op, & name, id, old, new ))
+#else
+#define DBG_CXBIND1( op, name, id, val ) \
+ ( ( void ) 0 )
+#define DBG_CXBIND2( op, name, id, old, new ) \
+ ( ( void ) 0 )
+#endif
+
/* Bind
* perform schema and factory param substitution
* returns prior param values
+ *
+ * 9/11/13
+ * "self" is a cursor-local cloned version of schema decl
*/
rc_t SFunctionBindSchemaParms ( const SFunction *self,
- Vector *prior, const Vector *subst )
+ Vector *prior, const Vector *subst, Vector *cx_bind )
{
rc_t rc = 0;
+
uint32_t i, count;
+ void *cx_old, *cx_new;
- /* count input params */
+ const SIndirectType *id;
+ const SIndirectConst *ic;
+
+ /* count input params
+ the first bunch are types
+ the remainder are constants */
count = VectorLength ( subst );
- /* initialize return value */
+ /* initialize return value
+ the prior values act as a stack for recursion */
VectorInit ( prior, 0, count );
/* determine total schema params */
@@ -317,74 +326,120 @@ rc_t SFunctionBindSchemaParms ( const SFunction *self,
return rc;
}
- /* save prior types */
+ /* bind types */
count = VectorLength ( & self -> type );
for ( i = 0; i < count; ++ i )
{
- const SIndirectType *id = VectorGet ( & self -> type, i );
+ /* get the indirect type object */
+ id = VectorGet ( & self -> type, i );
assert ( id != NULL );
- rc = VectorSet ( prior, id -> pos, id -> type );
+
+ /* get the new type expression */
+ cx_new = VectorGet ( subst, id -> pos );
+ assert ( cx_new != NULL );
+
+ /* update the binding vector */
+ rc = VectorSwap ( cx_bind, id -> type_id, cx_new, & cx_old );
if ( rc != 0 )
- {
- VectorWhack ( prior, NULL, NULL );
- return rc;
- }
- }
+ break;
+ DBG_CXBIND2 ( "bind schema type", id -> name -> name, id -> type_id, cx_old, cx_new );
- /* save prior constants */
- count = VectorLength ( & self -> schem );
- for ( i = 0 ; i < count; ++ i )
- {
- const SIndirectConst *ic = VectorGet ( & self -> schem, i );
- assert ( ic != NULL );
- rc = VectorSet ( prior, ic -> id, ic -> td ? ( void* ) ic -> expr : ( void* ) ic -> func );
+ /* save old value on stack for recursion */
+ rc = VectorSet ( prior, id -> pos, cx_old );
if ( rc != 0 )
{
- VectorWhack ( prior, NULL, NULL );
- return rc;
+ cx_new = cx_old;
+ VectorSwap ( cx_bind, id -> type_id, cx_new, & cx_old );
+ DBG_CXBIND2 ( "revert bind type", id -> name -> name, id -> type_id, cx_old, cx_new );
+ break;
}
}
- /* set new values */
- count = VectorLength ( & self -> type );
- for ( i = 0; i < count; ++ i )
- {
- SIndirectType *id = VectorGet ( & self -> type, i );
- id -> type = VectorGet ( subst, id -> pos );
- assert ( id -> type != NULL );
- }
-
- count = VectorLength ( & self -> schem );
- for ( i = 0; i < count; ++ i )
+ /* bind constants */
+ if ( rc == 0 )
{
- SIndirectConst *ic = VectorGet ( & self -> schem, i );
- if ( ic -> td == NULL )
+ count = VectorLength ( & self -> schem );
+ for ( i = 0 ; i < count; ++ i )
{
- ic -> func = VectorGet ( subst, ic -> id );
- assert ( ic -> func != NULL );
+ ic = VectorGet ( & self -> schem, i );
+ assert ( ic != NULL );
+
+ /* get the new constant value expression */
+ cx_new = VectorGet ( subst, ic -> pos );
+ assert ( cx_new != NULL );
+
+ /* update the binding vector */
+ rc = VectorSwap ( cx_bind, ic -> expr_id, cx_new, & cx_old );
+ if ( rc != 0 )
+ break;
+ DBG_CXBIND2 ( "bind schema const", ic -> name -> name, ic -> expr_id, cx_old, cx_new );
+
+ /* save old value on stack for recursion */
+ rc = VectorSet ( prior, ic -> pos, cx_old );
+ if ( rc != 0 )
+ {
+ cx_new = cx_old;
+ VectorSwap ( cx_bind, ic -> expr_id, cx_new, & cx_old );
+ DBG_CXBIND2 ( "revert bind const", ic -> name -> name, ic -> expr_id, cx_old, cx_new );
+ break;
+ }
}
- else
+
+ /* if there was no error, we're done */
+ if ( rc == 0 )
+ return 0;
+
+ /* reverse the damage done by binding constants */
+ while ( i -- > 0 )
{
- ic -> expr = VectorGet ( subst, ic -> id );
- assert ( ic -> expr != NULL );
+ ic = VectorGet ( & self -> schem, i );
+ assert ( ic != NULL );
+
+ /* get the old constant value expression */
+ cx_new = VectorGet ( prior, ic -> pos );
+ assert ( cx_new != NULL );
+
+ /* restore the binding vector */
+ VectorSwap ( cx_bind, ic -> expr_id, cx_new, & cx_old );
+ DBG_CXBIND2 ( "revert bind const", ic -> name -> name, ic -> expr_id, cx_old, cx_new );
}
+
+ /* reset i */
+ i = VectorLength ( & self -> type );
}
- return 0;
+ /* reverse the damage done by binding types */
+ while ( i -- > 0 )
+ {
+ /* get the indirect type object */
+ id = VectorGet ( & self -> type, i );
+ assert ( id != NULL );
+
+ /* get the old type expression */
+ cx_new = VectorGet ( prior, id -> pos );
+ assert ( cx_new != NULL );
+
+ /* restore the binding vector */
+ VectorSwap ( cx_bind, id -> type_id, cx_new, & cx_old );
+ DBG_CXBIND2 ( "revert bind type", id -> name -> name, id -> type_id, cx_old, cx_new );
+ }
+
+ /* a non-zero rc indicates the Vector is invalid */
+ VectorWhack ( prior, NULL, NULL );
+ return rc;
}
rc_t SFunctionBindFactParms ( const SFunction *self,
- Vector *prior, const Vector *subst )
+ Vector *parms, Vector *prior, const Vector *subst, Vector *cx_bind )
{
- rc_t rc;
+ rc_t rc = 0;
+ SIndirectConst *ic;
+ void *cx_old, *cx_new;
uint32_t i, count, act_count, form_count;
- SIndirectConst *ic, *last;
/* count input params */
count = act_count = VectorLength ( subst );
- VectorInit ( prior, 0, 0 );
-
/* must have minimum count */
if ( act_count < self -> fact . mand )
{
@@ -396,6 +451,7 @@ rc_t SFunctionBindFactParms ( const SFunction *self,
self -> fact . mand, act_count ));
return rc;
}
+
/* test against maximum count */
form_count = VectorLength ( & self -> fact . parms );
if ( act_count > form_count )
@@ -413,65 +469,117 @@ rc_t SFunctionBindFactParms ( const SFunction *self,
count = form_count;
}
- /* save prior constants */
+ /* initialize return values
+ the "parms" vector is a positional vector of expressions,
+ and is the only way to get at varargs.
+ the prior values act as a stack for recursion */
+ VectorInit ( parms, 0, act_count );
VectorInit ( prior, 0, form_count );
- for ( i = 0 ; i < form_count; ++ i )
+
+ /* bind actual formal parameter values */
+ for ( i = 0; i < form_count && i < act_count; ++ i )
{
+ /* get the indirect constant object */
ic = VectorGet ( & self -> fact . parms, i );
assert ( ic != NULL );
- rc = VectorSet ( prior, ic -> id, ic -> td ? ( void* ) ic -> expr : ( void* ) ic -> func );
+
+ /* get the new value expression */
+ assert ( ic -> pos == i );
+ cx_new = VectorGet ( subst, i );
+ assert ( cx_new != NULL );
+
+ /* update the positional vector */
+ rc = VectorAppend ( parms, NULL, cx_new );
+ if ( rc != 0 )
+ break;
+
+ /* update the binding vector */
+ rc = VectorSwap ( cx_bind, ic -> expr_id, cx_new, & cx_old );
+ if ( rc != 0 )
+ break;
+ DBG_CXBIND2 ( "bind fact const", ic -> name -> name, ic -> expr_id, cx_old, cx_new );
+
+ /* save the old value on stack for recursion */
+ rc = VectorSet ( prior, i, cx_old );
if ( rc != 0 )
{
- VectorWhack ( prior, NULL, NULL );
- return rc;
+ cx_new = cx_old;
+ VectorSwap ( cx_bind, ic -> expr_id, cx_new, & cx_old );
+ DBG_CXBIND2 ( "revert bind const", ic -> name -> name, ic -> expr_id, cx_old, cx_new );
+ break;
}
}
- /* set new values */
- for ( last = NULL, i = 0; i < count; last = ic, ++ i )
+ if ( rc == 0 )
{
- ic = VectorGet ( & self -> fact . parms, i );
- if ( ic -> td == NULL )
- {
- ic -> func = VectorGet ( subst, ic -> id );
- assert ( ic -> func != NULL );
- }
- else
+ /* this loop should only actually execute
+ to record missing optional formals */
+ assert ( i == form_count || ( i == act_count && act_count < form_count ) );
+
+ /* record optional formal parameter values */
+ for ( ; i < form_count; ++ i )
{
- ic -> expr = VectorGet ( subst, ic -> id );
- assert ( ic -> expr != NULL );
+ /* get the indirect constant object */
+ ic = VectorGet ( & self -> fact . parms, i );
+ assert ( ic != NULL );
+
+ /* get the new value expression */
+ assert ( ic -> pos == i );
+ cx_new = VectorGet ( subst, i );
+
+ /* save the same value on stack for recursion */
+ rc = VectorSet ( prior, i, cx_new );
+ if ( rc != 0 )
+ break;
+
+ DBG_CXBIND1 ( "ignore optional fact const", ic -> name -> name, ic -> expr_id, cx_new );
}
}
- /* set vararg values */
- for ( rc = 0; i < act_count; ++ i )
+ if ( rc == 0 )
{
- ic = malloc(sizeof(*ic));
- if (ic == NULL) {
- rc = RC ( rcVDB, rcFunction, rcEvaluating, rcMemory, rcExhausted );
- break;
- }
-
- /* not quite right - should have a temp name... */
- * ic = * last;
- ic->id = i;
+ /* this loop should only actually execute
+ to record vararg params beyond all formals */
+ assert ( i == form_count );
- if ( ic -> td == NULL )
+ /* set vararg values */
+ for ( ; i < act_count; ++ i )
{
- ic -> func = VectorGet ( subst, ic -> id );
- assert ( ic -> func != NULL );
- }
- else
- {
- ic -> expr = VectorGet ( subst, ic -> id );
- assert ( ic -> expr != NULL );
+ /* get the extra value expression */
+ cx_new = VectorGet ( subst, i );
+ assert ( cx_new != NULL );
+
+ /* update the positional vector */
+ rc = VectorAppend ( parms, NULL, cx_new );
+ if ( rc != 0 )
+ break;
+
+ DBG_CXBIND1 ( "vararg fact const", no_name, 0, cx_new );
}
- rc = VectorSet ( ( Vector* ) & self -> fact . parms, i, ic );
+ if ( rc == 0 )
+ return 0;
+
+ i = form_count;
}
- if (rc != 0)
- VectorWhack ( prior, NULL, NULL );
+ while ( i -- > 0 )
+ {
+ /* get the indirect type object */
+ ic = VectorGet ( & self -> fact . parms, i );
+ assert ( ic != NULL );
+
+ /* get the old type expression */
+ cx_new = VectorGet ( subst, ic -> pos );
+ assert ( cx_new != NULL );
+
+ /* restore the binding vector */
+ VectorSwap ( cx_bind, ic -> expr_id, cx_new, & cx_old );
+ DBG_CXBIND2 ( "revert bind const", ic -> name -> name, ic -> expr_id, cx_old, cx_new );
+ }
+
+ VectorWhack ( parms, NULL, NULL );
+ VectorWhack ( prior, NULL, NULL );
return rc;
}
@@ -480,56 +588,76 @@ rc_t SFunctionBindFactParms ( const SFunction *self,
* restore schema and factory param substitution
* destroys prior param vector
*/
-void SFunctionRestSchemaParms ( const SFunction *self, Vector *prior )
+void SFunctionRestSchemaParms ( const SFunction *self, Vector *prior, Vector *cx_bind )
{
+ rc_t rc;
uint32_t i, count;
+ void * cx_old, * ignore;
+
+ /* must have whole thing in prior */
+ assert ( VectorLength ( prior ) == VectorLength ( & self -> type ) + VectorLength ( & self -> schem ) );
/* restore prior values */
count = VectorLength ( & self -> type );
for ( i = 0; i < count; ++ i )
{
- SIndirectType *id = VectorGet ( & self -> type, i );
- id -> type = VectorGet ( prior, id -> pos );
+ /* get the indirect type object */
+ const SIndirectType *id = VectorGet ( & self -> type, i );
+ assert ( id != NULL );
+
+ /* get the old type expression */
+ cx_old = VectorGet ( prior, id -> pos );
+
+ /* update the binding vector */
+ rc = VectorSwap ( cx_bind, id -> type_id, cx_old, & ignore );
+ assert ( rc == 0 );
+
+ DBG_CXBIND2 ( "restore bind type", id -> name -> name, id -> type_id, ignore, cx_old );
}
count = VectorLength ( & self -> schem );
for ( i = 0; i < count; ++ i )
{
- SIndirectConst *ic = VectorGet ( & self -> schem, i );
- if ( ic -> td == NULL )
- ic -> func = VectorGet ( prior, ic -> id );
- else
- ic -> expr = VectorGet ( prior, ic -> id );
+ const SIndirectConst *ic = VectorGet ( & self -> schem, i );
+ assert ( ic != NULL );
+
+ /* get the old constant value expression */
+ cx_old = VectorGet ( prior, ic -> pos );
+
+ /* restore the binding vector */
+ rc = VectorSwap ( cx_bind, ic -> expr_id, cx_old, & ignore );
+ assert ( rc == 0 );
+
+ DBG_CXBIND2 ( "restore bind const", ic -> name -> name, ic -> expr_id, ignore, cx_old );
}
VectorWhack ( prior, NULL, NULL );
}
-void SFunctionRestFactParms ( const SFunction *self, Vector *prior )
+void SFunctionRestFactParms ( const SFunction *self, Vector *prior, Vector *cx_bind )
{
+ rc_t rc;
uint32_t i, count;
+ void * cx_old, * ignore;
- /* whack extra fake params created with varargs - see above */
- count = VectorLength ( &self->fact.parms );
- for (i = VectorLength ( prior ); i < count; ) {
- SIndirectConst *ic;
-
- assert ( i != 0 );
- VectorRemove ( ( Vector* ) & self -> fact . parms, -- count, ( void** ) & ic );
- free ( ic );
- }
-
- assert ( VectorLength ( & self -> fact . parms ) == VectorLength ( prior ) );
+ /* must have whole thing in prior */
+ assert ( VectorLength ( prior ) == VectorLength ( & self -> fact . parms ) );
/* restore prior values */
- assert ( count == i );
+ count = VectorLength ( & self -> fact . parms );
for ( i = 0; i < count; ++ i )
{
- SIndirectConst *ic = VectorGet ( & self -> fact . parms, i );
- if ( ic -> td == NULL )
- ic -> func = VectorGet ( prior, ic -> id );
- else
- ic -> expr = VectorGet ( prior, ic -> id );
+ const SIndirectConst *ic = VectorGet ( & self -> fact . parms, i );
+ assert ( ic != NULL );
+
+ /* get the old constant value expression */
+ cx_old = VectorGet ( prior, ic -> pos );
+
+ /* restore the binding vector */
+ rc = VectorSwap ( cx_bind, ic -> expr_id, cx_old, & ignore );
+ assert ( rc == 0 );
+
+ DBG_CXBIND2 ( "restore bind const", ic -> name -> name, ic -> expr_id, ignore, cx_old );
}
VectorWhack ( prior, NULL, NULL );
@@ -554,7 +682,6 @@ void CC SFunctionMark ( void * item, void * data )
SExpressionMark ( ( void * )self -> rt, data );
SFormParmlistMark ( & self -> fact, SIndirectConstMark, schema );
SFormParmlistMark ( & self -> func, SProductionMark, schema );
- VectorForEach ( & self -> type, false, SIndirectTypeMark, data );
VectorForEach ( & self -> schem, false, SIndirectConstMark, data );
if ( self -> script )
@@ -752,7 +879,7 @@ rc_t formal_symbol ( KSymTable *tbl, KTokenSource *src, KToken *t,
*/
static
rc_t param_formal ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, SFormParmlist *sig )
+ const SchemaEnv *env, VSchema *self, SFormParmlist *sig )
{
rc_t rc;
@@ -824,7 +951,7 @@ rc_t param_formal ( KSymTable *tbl, KTokenSource *src, KToken *t,
*/
static
rc_t fact_formal ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, SFormParmlist *sig )
+ const SchemaEnv *env, VSchema *self, SFormParmlist *sig )
{
rc_t rc;
SIndirectConst *param = malloc ( sizeof * param );
@@ -834,24 +961,27 @@ rc_t fact_formal ( KSymTable *tbl, KTokenSource *src, KToken *t,
return KTokenRCExplain ( t, klogInt, rc );
}
- param -> expr = NULL;
- param -> func = NULL;
+ param -> td = NULL;
+ param -> expr_id = 0;
+#if ACCEPT_FUNCTION_AS_FACT_PARAM
/* type could be 'function' */
if ( t -> id == kw_function )
- {
- param -> td = NULL;
next_token ( tbl, src, t );
- }
/* should be a typedecl */
else
+#endif
{
rc = type_expr ( tbl, src, t, env, self, & param -> td );
if ( rc != 0 )
{
free ( param );
- return KTokenFailure ( t, klogErr, rc, "function or data type" );
+ return KTokenFailure ( t, klogErr, rc,
+#if ACCEPT_FUNCTION_AS_FACT_PARAM
+ "function or "
+#endif
+ "data type" );
}
}
@@ -865,13 +995,16 @@ rc_t fact_formal ( KSymTable *tbl, KTokenSource *src, KToken *t,
param -> name = t -> sym;
/* store as a parameter */
- rc = VectorAppend ( & sig -> parms, & param -> id, param );
+ rc = VectorAppend ( & sig -> parms, & param -> pos, param );
if ( rc != 0 )
{
SIndirectConstWhack ( param, NULL );
return KTokenRCExplain ( t, klogInt, rc );
}
+ /* set binding constant */
+ param -> expr_id = ++ self -> num_indirect;
+
next_token ( tbl, src, t );
return 0;
}
@@ -882,9 +1015,9 @@ rc_t fact_formal ( KSymTable *tbl, KTokenSource *src, KToken *t,
*/
static
rc_t formal_params ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, SFormParmlist *sig,
+ const SchemaEnv *env, VSchema *self, SFormParmlist *sig,
rc_t ( * formal_param ) ( KSymTable*, KTokenSource*, KToken*,
- const SchemaEnv*, const VSchema*, SFormParmlist* ) )
+ const SchemaEnv*, VSchema*, SFormParmlist* ) )
{
while ( t -> sym != NULL || t -> id == eIdent )
{
@@ -909,9 +1042,9 @@ rc_t formal_params ( KSymTable *tbl, KTokenSource *src, KToken *t,
*/
static
rc_t formal_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, SFormParmlist *sig,
+ const SchemaEnv *env, VSchema *self, SFormParmlist *sig,
rc_t ( * formal_param ) ( KSymTable*, KTokenSource*, KToken*,
- const SchemaEnv*, const VSchema*, SFormParmlist* ) )
+ const SchemaEnv*, VSchema*, SFormParmlist* ) )
{
/* read mandatory parameters */
rc_t rc = formal_params ( tbl, src, t, env, self, sig, formal_param );
@@ -952,7 +1085,7 @@ rc_t formal_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
*/
static
rc_t parm_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, SFormParmlist *sig )
+ const SchemaEnv *env, VSchema *self, SFormParmlist *sig )
{
/* open list */
rc_t rc = expect ( tbl, src, t, eLeftParen, "(", true );
@@ -974,7 +1107,7 @@ rc_t parm_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
* | '...'
*/
rc_t fact_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, SFormParmlist *sig )
+ const SchemaEnv *env, VSchema *self, SFormParmlist *sig )
{
/* open list */
rc_t rc = expect ( tbl, src, t, eLeftAngle, "<", true );
@@ -1062,7 +1195,7 @@ rc_t schema_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
/* initialize to raw format,
undefined type, and no dimension */
- formal -> type = NULL;
+ formal -> type_id = 0;
/* create symbol */
rc = KSymTableCreateConstSymbol ( tbl, & formal -> name,
@@ -1088,6 +1221,8 @@ rc_t schema_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
free ( formal );
return KTokenRCExplain ( t, klogInt, rc );
}
+
+ formal -> type_id = ++ self -> num_indirect;
}
else
{
@@ -1099,9 +1234,8 @@ rc_t schema_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
return KTokenRCExplain ( t, klogInt, rc );
}
- /* initialize with no value or function */
- formal -> expr = NULL;
- formal -> func = NULL;
+ /* initialize with no value */
+ formal -> expr_id = 0;
formal -> td = td;
/* create symbol */
@@ -1110,15 +1244,17 @@ rc_t schema_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
if ( rc == 0 )
{
/* record formal */
- rc = VectorAppend ( & sig -> schem, & formal -> id, formal );
+ rc = VectorAppend ( & sig -> schem, & formal -> pos, formal );
if ( rc == 0 )
- formal -> id += VectorLength ( & sig -> type );
+ formal -> pos += VectorLength ( & sig -> type );
}
if ( rc != 0 )
{
SIndirectConstWhack ( formal, NULL );
return KTokenRCExplain ( t, klogInt, rc );
}
+
+ formal -> expr_id = ++ self -> num_indirect;
}
}
while ( next_token ( tbl, src, t ) -> id == eComma );
@@ -1310,8 +1446,9 @@ rc_t func_decl ( KSymTable *tbl, KTokenSource *src, KToken *t,
if ( t -> id == eLeftCurly )
{
#if SLVL >= 4
- /* if user already specified extern function */
- if ( type == eFunction )
+ /* if user already specified extern function
+ or if user specified vararg factory params */
+ if ( type == eFunction || f -> fact . vararg )
#endif
return KTokenExpected ( t, klogErr, "; or =" );
#if SLVL >= 4
diff --git a/libs/vdb/schema-parse.h b/libs/vdb/schema-parse.h
index 2c955dc..efd695f 100644
--- a/libs/vdb/schema-parse.h
+++ b/libs/vdb/schema-parse.h
@@ -418,7 +418,7 @@ rc_t schema_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
* | '...'
*/
rc_t fact_signature ( KSymTable *tbl, KTokenSource *src, KToken *t,
- const SchemaEnv *env, const VSchema *self, struct SFormParmlist *sig );
+ const SchemaEnv *env, VSchema *self, struct SFormParmlist *sig );
/*--------------------------------------------------------------------------
@@ -506,19 +506,19 @@ rc_t phys_encoding_expr ( KSymTable *tbl, KTokenSource *src, KToken *t,
* returns non-zero error code if failed
*/
rc_t eval_const_expr ( const VSchema *self, const VTypedecl *td,
- struct SExpression const *expr, struct SExpression **value );
+ struct SExpression const *expr, struct SExpression **value, Vector *cx_bind );
/* eval-uint-expr
* special const expression evaluator for uint32_t
*/
rc_t eval_uint_expr ( const VSchema *self,
- struct SExpression const *expr, uint32_t *value );
+ struct SExpression const *expr, uint32_t *value, Vector *cx_bind );
/* eval-uint64-expr
* special const expression evaluator for uint64_t
*/
rc_t eval_uint64_expr ( const VSchema *self,
- struct SExpression const *expr, uint64_t *value );
+ struct SExpression const *expr, uint64_t *value, Vector *cx_bind );
/* eval-expr-syntax
* examine expression syntax
diff --git a/libs/vdb/schema-priv.h b/libs/vdb/schema-priv.h
index dab6a6c..7e321ab 100644
--- a/libs/vdb/schema-priv.h
+++ b/libs/vdb/schema-priv.h
@@ -289,6 +289,12 @@ struct VSchema
KRefcount refcount;
uint32_t file_count;
+
+ /* number of indirect expressions
+ these are uniquely identified place-holders
+ for cursor-open binding of
+ type and constant expressions */
+ uint32_t num_indirect;
};
@@ -539,9 +545,8 @@ struct SIndirectType
/* symbolic name */
struct KSymbol const *name;
- /* MUTABLE substitution typedecl
- expression. may use indirect types */
- struct SExpression const *type;
+ /* index into binding vector */
+ uint32_t type_id;
/* formal type id */
uint32_t id;
@@ -558,10 +563,6 @@ void CC SIndirectTypeWhack ( void *item, void *ignore );
*/
SIndirectType *VSchemaFindITypeid ( const VSchema *self, uint32_t id );
-/* Mark
- */
-void CC SIndirectTypeMark ( void * item, void * data );
-
/* Dump
*/
rc_t SIndirectTypeDump ( const SIndirectType *self, struct SDumper *d );
@@ -616,15 +617,14 @@ struct SIndirectConst
/* symbolic name */
struct KSymbol const *name;
- /* MUTABLE constant value or function */
- struct SExpression const *expr;
- struct SFunction const *func;
-
/* formal typedecl or NULL if function */
struct SExpression const *td;
- /* formal type id */
- uint32_t id;
+ /* index into binding vector */
+ uint32_t expr_id;
+
+ /* offset position from # of indirect types */
+ uint32_t pos;
};
/* Whack
@@ -808,16 +808,16 @@ int CC SFunctionSort ( const void *item, const void *n );
* returns prior param values
*/
rc_t SFunctionBindSchemaParms ( const SFunction *self,
- Vector *prior, const Vector *subst );
+ Vector *prior, const Vector *subst, Vector *cx_bind );
rc_t SFunctionBindFactParms ( const SFunction *self,
- Vector *prior, const Vector *subst );
+ Vector *parms, Vector *prior, const Vector *subst, Vector *cx_bind );
/* Rest-ore
* restore schema and factory param substitution
* destroys prior param vector
*/
-void SFunctionRestSchemaParms ( const SFunction *self, Vector *prior );
-void SFunctionRestFactParms ( const SFunction *self, Vector *prior );
+void SFunctionRestSchemaParms ( const SFunction *self, Vector *prior, Vector *cx_bind );
+void SFunctionRestFactParms ( const SFunction *self, Vector *prior, Vector *cx_bind );
/* Mark
*/
@@ -884,16 +884,16 @@ int CC SPhysicalSort ( const void *item, const void *n );
* returns prior param values
*/
rc_t SPhysicalBindSchemaParms ( const SPhysical *self,
- Vector *prior, const Vector *subst );
+ Vector *prior, const Vector *subst, Vector *cx_bind );
rc_t SPhysicalBindFactParms ( const SPhysical *self,
- Vector *prior, const Vector *subst );
+ Vector *parms, Vector *prior, const Vector *subst, Vector *cx_bind );
/* Rest-ore
* restore schema and factory param substitution
* destroys prior param vector
*/
-void SPhysicalRestSchemaParms ( const SPhysical *self, Vector *prior );
-void SPhysicalRestFactParms ( const SPhysical *self, Vector *prior );
+void SPhysicalRestSchemaParms ( const SPhysical *self, Vector *prior, Vector *cx_bind );
+void SPhysicalRestFactParms ( const SPhysical *self, Vector *prior, Vector *cx_bind );
/* Mark
*/
diff --git a/libs/vdb/schema-prod.c b/libs/vdb/schema-prod.c
index ef507b1..c637b10 100644
--- a/libs/vdb/schema-prod.c
+++ b/libs/vdb/schema-prod.c
@@ -207,29 +207,29 @@ int CC SPhysicalSort ( const void *item, const void *n )
* returns prior param values
*/
rc_t SPhysicalBindSchemaParms ( const SPhysical *self,
- Vector *prior, const Vector *subst )
+ Vector *prior, const Vector *subst, Vector *cx_bind )
{
- return SFunctionBindSchemaParms ( & self -> decode, prior, subst );
+ return SFunctionBindSchemaParms ( & self -> decode, prior, subst, cx_bind );
}
rc_t SPhysicalBindFactParms ( const SPhysical *self,
- Vector *prior, const Vector *subst )
+ Vector *parms, Vector *prior, const Vector *subst, Vector *cx_bind )
{
- return SFunctionBindFactParms ( & self -> decode, prior, subst );
+ return SFunctionBindFactParms ( & self -> decode, parms, prior, subst, cx_bind );
}
/* Rest-ore
* restore schema and factory param substitution
* destroys prior param vector
*/
-void SPhysicalRestSchemaParms ( const SPhysical *self, Vector *prior )
+void SPhysicalRestSchemaParms ( const SPhysical *self, Vector *prior, Vector *cx_bind )
{
- SFunctionRestSchemaParms ( & self -> decode, prior );
+ SFunctionRestSchemaParms ( & self -> decode, prior, cx_bind );
}
-void SPhysicalRestFactParms ( const SPhysical *self, Vector *prior )
+void SPhysicalRestFactParms ( const SPhysical *self, Vector *prior, Vector *cx_bind )
{
- SFunctionRestFactParms ( & self -> decode, prior );
+ SFunctionRestFactParms ( & self -> decode, prior, cx_bind );
}
/* Mark
diff --git a/libs/vdb/schema-type.c b/libs/vdb/schema-type.c
index 0bb5186..dcd729d 100644
--- a/libs/vdb/schema-type.c
+++ b/libs/vdb/schema-type.c
@@ -1600,7 +1600,6 @@ void VSchemaTypeMark ( const VSchema *self, uint32_t type_id )
{
const SDatatype *dt;
const STypeset *ts;
- const SIndirectType *pt;
switch ( type_id & 0xC0000000 )
{
@@ -1629,19 +1628,6 @@ void VSchemaTypeMark ( const VSchema *self, uint32_t type_id )
if ( ts != NULL )
STypesetMark ( ts, self );
break;
-
- default:
- while ( type_id < VectorStart ( & self -> pt ) )
- {
- self = self -> dad;
- if ( self -> dad == NULL )
- return;
- }
-
- pt = VectorGet ( & self -> pt, type_id );
- if ( pt != NULL )
- SIndirectTypeMark ( ( void * )pt, ( void * )self );
- break;
}
}
@@ -1941,9 +1927,14 @@ rc_t dim ( const KSymTable *tbl, KTokenSource *src, KToken *t,
rc = const_expr ( tbl, src, t, env, self, & x );
if ( rc == 0 )
{
+ Vector dummy;
+ VectorInit ( & dummy, 1, 16 );
+
/* should have evaluated to a constant expression */
- rc = eval_uint_expr ( self, x, dim );
+ rc = eval_uint_expr ( self, x, dim, & dummy );
SExpressionWhack ( x );
+
+ VectorWhack ( & dummy, NULL, NULL );
}
if ( rc != 0 )
return rc;
diff --git a/libs/vdb/schema.c b/libs/vdb/schema.c
index ff38a0d..b848997 100644
--- a/libs/vdb/schema.c
+++ b/libs/vdb/schema.c
@@ -665,6 +665,7 @@ rc_t VSchemaMake ( VSchema **sp, const VSchema *dad )
KRefcountInit ( & schema -> refcount, 1, "VSchema", "make", "vschema" );
schema -> file_count = 0;
+ schema -> num_indirect = ( dad == NULL ) ? 0 : dad -> num_indirect;
/* copy the versioned name vectors and names */
if ( dad != NULL )
diff --git a/libs/vdb/wcolumn.c b/libs/vdb/wcolumn.c
index f454032..fa7e2c8 100644
--- a/libs/vdb/wcolumn.c
+++ b/libs/vdb/wcolumn.c
@@ -128,8 +128,8 @@ void CC VColumnWhack ( void *item, void *data )
/* Make - PRIVATE
* make a write column
*/
-rc_t WColumnMake ( VColumn **colp, const VSchema *schema,
- const SColumn *scol, const SExpression *blob_limit, VDBManager *mgr )
+rc_t WColumnMake ( VColumn **colp, const VSchema *schema, const SColumn *scol,
+ const SExpression *blob_limit, VDBManager *mgr, Vector *cx_bind )
{
rc_t rc;
WColumn *col;
@@ -175,7 +175,7 @@ rc_t WColumnMake ( VColumn **colp, const VSchema *schema,
{
/* evaluate column blob limit */
uint64_t trigger;
- rc = eval_uint64_expr ( schema, blob_limit, & trigger );
+ rc = eval_uint64_expr ( schema, blob_limit, & trigger, cx_bind );
col -> trigger = ( size_t ) trigger;
}
diff --git a/libs/vdb/wcursor.c b/libs/vdb/wcursor.c
index 6c88983..a9790b5 100644
--- a/libs/vdb/wcursor.c
+++ b/libs/vdb/wcursor.c
@@ -197,7 +197,7 @@ LIB_EXPORT rc_t CC VTableCreateCursorWrite ( VTable *self, VCursor **cursp, KCre
/* MakeColumn
*/
-rc_t VCursorMakeColumn ( VCursor *self, VColumn **col, const SColumn *scol )
+rc_t VCursorMakeColumn ( VCursor *self, VColumn **col, const SColumn *scol, Vector *cx_bind )
{
VTable *vtbl;
@@ -205,7 +205,7 @@ rc_t VCursorMakeColumn ( VCursor *self, VColumn **col, const SColumn *scol )
return VColumnMake ( col, self -> schema, scol );
vtbl = self -> tbl;
- return WColumnMake ( col, self -> schema, scol, vtbl -> stbl -> limit, vtbl -> mgr );
+ return WColumnMake ( col, self -> schema, scol, vtbl -> stbl -> limit, vtbl -> mgr, cx_bind );
}
@@ -293,6 +293,7 @@ LIB_EXPORT rc_t CC VCursorOpen ( const VCursor *cself )
{
if ( ! self -> read_only )
{
+ Vector cx_bind;
VProdResolve pr;
pr . schema = self -> schema;
pr . ld = ld;
@@ -301,14 +302,20 @@ LIB_EXPORT rc_t CC VCursorOpen ( const VCursor *cself )
pr . curs = self;
pr . cache = & self -> prod;
pr . owned = & self -> owned;
+ pr . cx_bind = & cx_bind;
pr . chain = chainEncoding;
pr . blobbing = false;
pr . ignore_column_errors = false;
pr . discover_writable_columns = false;
- if ( !self -> suspend_triggers )
- rc = VProdResolveAddTriggers ( & pr, self -> stbl );
+ VectorInit ( & cx_bind, 1, self -> schema -> num_indirect );
+
+ if ( ! self -> suspend_triggers )
+ rc = VProdResolveAddTriggers ( & pr, self -> stbl );
+
+ VectorWhack ( & cx_bind, NULL, NULL );
}
+
if ( rc == 0 )
{
/* TBD - warn if any input columns are unreferenced by schema */
@@ -441,6 +448,7 @@ rc_t VCursorListSeededWritableColumns ( VCursor *self, BSTree *columns, const KN
rc_t rc;
KDlset *libs;
+ Vector cx_bind;
struct resolve_phys_data pb;
pb . pr . schema = self -> schema;
pb . pr . ld = self -> tbl -> linker;
@@ -448,6 +456,7 @@ rc_t VCursorListSeededWritableColumns ( VCursor *self, BSTree *columns, const KN
pb . pr . curs = self;
pb . pr . cache = & self -> prod;
pb . pr . owned = & self -> owned;
+ pb . pr . cx_bind = & cx_bind;
pb . pr . chain = chainEncoding;
pb . pr . blobbing = false;
pb . pr . ignore_column_errors = true;
@@ -461,12 +470,16 @@ rc_t VCursorListSeededWritableColumns ( VCursor *self, BSTree *columns, const KN
return rc;
}
+ VectorInit ( & cx_bind, 1, self -> schema -> num_indirect );
+
/* open the dynamic linker libraries */
rc = VLinkerOpen ( pb . pr . ld, & libs );
if ( rc == 0 )
{
pb . pr . libs = libs;
- VProdResolveWritableColumns ( & pb , self->suspend_triggers );
+ VProdResolveWritableColumns ( & pb , self -> suspend_triggers );
+
+ VectorWhack ( & cx_bind, NULL, NULL );
KDlsetRelease ( libs );
if ( rc == 0 )
diff --git a/libs/vdb/wprod.c b/libs/vdb/wprod.c
index 41db5e9..dafb8bb 100644
--- a/libs/vdb/wprod.c
+++ b/libs/vdb/wprod.c
@@ -176,7 +176,7 @@ rc_t VProdResolveColumn ( const VProdResolve *self,
vcol = VCursorCacheGet ( & curs -> col, & scol -> cid );
if ( vcol == NULL )
{
- rc = VCursorMakeColumn ( curs, & vcol, scol );
+ rc = VCursorMakeColumn ( curs, & vcol, scol, self -> cx_bind );
if ( rc != 0 )
return rc;
@@ -227,7 +227,7 @@ rc_t VProdResolveColumn ( const VProdResolve *self,
/* auto-create writable column for purposes of discovery */
if ( scol -> read_only )
return 0;
- rc = VCursorMakeColumn ( curs, & vcol, scol );
+ rc = VCursorMakeColumn ( curs, & vcol, scol, self -> cx_bind );
if ( rc != 0 )
return rc;
@@ -345,7 +345,7 @@ rc_t VProdResolvePhysicalWrite ( const VProdResolve *self, VPhysical *phys )
/* build encoding schema in steps:
in <- page-to-blob
*/
- rc = VSimpleProdMake ( & prod, pr . owned, self->curs,
+ rc = VSimpleProdMake ( & prod, pr . owned, pr . curs,
prodSimplePage2Blob, name, & fd, & desc, NULL, phys -> in, chainEncoding );
if ( rc == 0 && enc != NULL )
{
@@ -361,7 +361,7 @@ rc_t VProdResolvePhysicalWrite ( const VProdResolve *self, VPhysical *phys )
}
if ( rc == 0 )
{
- rc = VSimpleProdMake ( & phys -> b2s, pr . owned, self->curs,
+ rc = VSimpleProdMake ( & phys -> b2s, pr . owned, pr . curs,
prodSimpleBlob2Serial, name, & fd, & desc, NULL, prod, chainEncoding );
}
diff --git a/libs/vfs/Makefile b/libs/vfs/Makefile
index 5fe9c6c..a3507de 100644
--- a/libs/vfs/Makefile
+++ b/libs/vfs/Makefile
@@ -78,8 +78,7 @@ VFS_SRC_CMN = \
syspath \
manager \
resolver \
- keyring \
- syskeyring
+
VFS_SRC = \
path \
diff --git a/libs/vfs/keyring.c b/libs/vfs/keyring.c
index b64e7c8..f73740a 100644
--- a/libs/vfs/keyring.c
+++ b/libs/vfs/keyring.c
@@ -28,45 +28,155 @@
#include <klib/rc.h>
#include <klib/refcount.h>
+#include <klib/text.h>
+#include <klib/data-buffer.h>
+#include <klib/log.h>
#include <kns/stream.h>
+#include <kns/endpoint.h>
+#include <kns/socket.h>
+#include <kns/manager.h>
-#include "keyring-priv.h"
+#include <vfs/keyring-priv.h>
#include <sysalloc.h>
#include <stdlib.h>
#include <string.h>
-
+#include <assert.h>
+
+/*TODO: define protocol messages in a central location */
+/*requests*/
+static const char* shutdownMsg = "\x01X";
+static const char* initMsg = "\x01I";
+/*responses*/
+static const char* initMsgSuccess = "IY";
+static const char* initMsgFailure = "IN";
+
#define rcTarget rcNoTarg
+#define KEYRING_MAX_PATH 4096
+
struct KKeyRing
{
KRefcount refcount;
+
+ char serverDataDir[KEYRING_MAX_PATH];
bool read_only;
+ bool shutting_down;
KStream* ipc;
};
static
-rc_t CC KKeyRingInit(KKeyRing* self)
+rc_t CC ShakeHands(KStream* ipc)
+{ /* Send Init message, wait for a response (give new server a chance to ask for password */
+ rc_t rc;
+ LogMsg ( klogInfo, "KeyringClient: sending Init");
+ rc = KStreamWrite(ipc, initMsg, string_size(initMsg), NULL);
+ if (rc == 0)
+ {
+ unsigned char msg_size;
+ size_t numRead = 0;
+ rc = KStreamReadAll(ipc, &msg_size, 1, &numRead);
+ if (rc == 0 && numRead == 1)
+ {
+ char resp[2];
+ if (msg_size == sizeof(resp))
+ {
+ rc = KStreamReadAll(ipc, resp, msg_size, &numRead);
+ if (rc == 0 && numRead == msg_size)
+ {
+ pLogMsg ( klogInfo, "KeyringClient: received '$(b1)$(b2)'", "b1=%c,b2=%c", resp[0], resp[1] );
+ if (string_cmp(resp, sizeof(resp), initMsgSuccess, sizeof(initMsgSuccess), msg_size) == 0)
+ rc = 0;
+ else if (string_cmp(resp, sizeof(resp), initMsgFailure, sizeof(initMsgFailure), msg_size) == 0)
+ rc = RC ( rcVFS, rcTarget, rcAttaching, rcParam, rcIncorrect);
+ else
+ rc = RC ( rcVFS, rcTarget, rcAttaching, rcMessage, rcUnexpected);
+ }
+ else
+ LogErr(klogErr, rc, "KeyringClient: KStreamReadAll(2) failed");
+ }
+ else
+ {
+ pLogMsg ( klogInfo, "KeyringClient: received '$(bytes)' bytes, expected 2", "bytes=%d", (int32_t)msg_size );
+ rc = RC ( rcVFS, rcTarget, rcAttaching, rcMessage, rcUnexpected);
+ }
+ }
+ else
+ {
+ pLogMsg ( klogInfo, "KeyringClient: received '$(bytes)' bytes, expected 1", "bytes=%d", (int32_t)numRead );
+ LogErr(klogErr, rc, "KeyringClient: KStreamReadAll(1) failed");
+ }
+ }
+
+ return rc;
+}
+
+static
+rc_t CC KKeyRingInit(KKeyRing* self, const char* path)
{
+ rc_t rc = 0;
memset(self, 0, sizeof(KKeyRing));
KRefcountInit ( & self -> refcount, 1, "KKeyRing", "init", "" );
+
+ LogMsg ( klogInfo, "KeyringClient: starting keyring");
+ rc = StartKeyRing(path);
+ LogMsg ( klogInfo, "KeyringClient: keyring started");
- return StartKeyRing(&self->ipc);
+ if (rc == 0)
+ {
+ KNSManager* kns;
+ rc = KNSManagerMake ( &kns );
+ if (rc == 0)
+ {
+ rc_t rc2;
+ KEndPoint ep;
+ String name;
+ CONST_STRING(&name, KEYRING_IPC_NAME);
+ rc = KNSManagerInitIPCEndpoint(kns, &ep, &name);
+ if (rc == 0)
+ {
+ LogMsg ( klogInfo, "KeyringClient: caling KNSMakeIPCConnection");
+ rc = KNSManagerMakeIPCConnection ( kns, &self->ipc, &ep, 5 );
+ if (rc == 0)
+ {
+ LogMsg ( klogInfo, "KeyringClient: KNSMakeIPCConnection successful");
+ rc = ShakeHands(self->ipc);
+ }
+ }
+ rc2 = KNSManagerRelease(kns);
+ if (rc == 0)
+ rc2 = rc;
+ }
+ }
+
+ return rc;
}
static
rc_t CC KKeyRingWhack(KKeyRing* self)
{
- rc_t rc = KStreamRelease(self->ipc);
+ rc_t rc = 0;
+ rc_t rc2;
+ if (self->ipc != NULL && self->shutting_down)
+ { /* send a shutdown message to the server */
+ LogMsg ( klogInfo, "KeyringClient: sending Shutdown");
+ rc = KStreamWrite(self->ipc, shutdownMsg, string_size(shutdownMsg), NULL);
+ }
+
+ rc2 = KStreamRelease(self->ipc);
+ if (rc == 0)
+ rc = rc2;
+
free(self);
+
return rc;
}
-extern
+LIB_EXPORT
rc_t CC KKeyRingAddRef ( const KKeyRing *self )
{
if ( self != NULL )
@@ -74,53 +184,56 @@ rc_t CC KKeyRingAddRef ( const KKeyRing *self )
switch ( KRefcountAdd ( & self -> refcount, "KKeyRing" ) )
{
case krefLimit:
- return RC ( rcKFG, rcTarget, rcAttaching, rcRange, rcExcessive );
+ return RC ( rcVFS, rcTarget, rcAttaching, rcRange, rcExcessive );
}
}
return 0;
}
-extern
+LIB_EXPORT
rc_t CC KKeyRingRelease ( KKeyRing *self, bool shutdown_server )
{
if ( self != NULL )
{
+ if (shutdown_server)
+ self->shutting_down = true; /* request shutdown when refcount reaches 0 */
+
switch ( KRefcountDrop ( & self -> refcount, "KKeyRing" ) )
{
case krefWhack:
return KKeyRingWhack ( ( KKeyRing* ) self );
break;
case krefLimit:
- return RC ( rcKFG, rcTarget, rcReleasing, rcRange, rcExcessive );
+ return RC ( rcVFS, rcTarget, rcReleasing, rcRange, rcExcessive );
}
}
return 0;
}
-LIB_EXPORT rc_t CC KKeyRingMakeRead( const KKeyRing** cself )
+LIB_EXPORT rc_t CC KKeyRingMakeRead( const KKeyRing** cself, const char* path )
{
KKeyRing** self = (KKeyRing**)cself;
- rc_t rc = KKeyRingMakeUpdate(self);
+ rc_t rc = KKeyRingMakeUpdate(self, path);
if (rc == 0)
(*self)->read_only = true;
return rc;
}
LIB_EXPORT
-rc_t CC KKeyRingMakeUpdate(KKeyRing** self)
+rc_t CC KKeyRingMakeUpdate(KKeyRing** self, const char* path)
{
KKeyRing* obj;
rc_t rc;
if ( self == NULL )
- rc = RC ( rcKFG, rcTarget, rcCreating, rcParam, rcNull );
+ rc = RC ( rcVFS, rcTarget, rcCreating, rcParam, rcNull );
else
{
obj = malloc(sizeof(KKeyRing));
if (obj == NULL)
- rc = RC ( rcKFG, rcTarget, rcCreating, rcMemory, rcExhausted );
+ rc = RC ( rcVFS, rcTarget, rcCreating, rcMemory, rcExhausted );
{
- rc = KKeyRingInit(obj);
+ rc = KKeyRingInit(obj, path);
if (rc == 0)
*self = obj;
else
@@ -131,3 +244,49 @@ rc_t CC KKeyRingMakeUpdate(KKeyRing** self)
return rc;
}
+rc_t CC KKeyRingAddProject(KKeyRing* self, const char* project_key, const char* dl_key, const char* enc_key)
+{
+ rc_t rc = 0;
+ if ( self == NULL )
+ rc = RC ( rcVFS, rcTarget, rcSending, rcSelf, rcNull );
+ else if (project_key == NULL || dl_key == NULL || enc_key == NULL)
+ rc = RC ( rcVFS, rcTarget, rcSending, rcParam, rcNull );
+ else
+ { /* Project Add: "<uint8 msg_length>PA<uint8 length>name<uint8 length>dl_ticket<uint8 length>enc_key" */
+ size_t pkSize = string_size(project_key);
+ size_t dkSize = string_size(dl_key);
+ size_t ekSize = string_size(enc_key);
+
+ char buf[256];
+ uint8_t msg_size = 2 + sizeof(uint8_t) + pkSize + sizeof(uint8_t) + dkSize + sizeof(uint8_t) + ekSize;
+ if ( 1 + msg_size > sizeof(buf) )
+ rc = RC ( rcVFS, rcTarget, rcSending, rcSize, rcExcessive );
+ else
+ {
+ size_t idx = 0;
+
+ buf[0] = msg_size; ++idx;
+
+ buf[idx]='P'; ++idx;
+ buf[idx]='A'; ++idx;
+
+ buf[idx] = (char)pkSize; ++idx;
+ string_copy(buf + idx, sizeof(buf)-idx, project_key, pkSize);
+ idx += pkSize;
+
+ buf[idx] = (char)dkSize; ++idx;
+ string_copy(buf + idx, sizeof(buf)-idx, dl_key, dkSize);
+ idx += dkSize;
+
+ buf[idx] = (char)ekSize; ++idx;
+ string_copy(buf + idx, sizeof(buf)-idx, enc_key, ekSize);
+ idx += ekSize;
+
+ assert(idx == msg_size + 1);
+ rc = KStreamWrite(self->ipc, buf, idx, NULL);
+ }
+ }
+ return rc;
+
+}
+
diff --git a/libs/vfs/libsrapath.vers b/libs/vfs/libsrapath.vers
deleted file mode 100644
index e9bc149..0000000
--- a/libs/vfs/libsrapath.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.14
diff --git a/libs/vfs/libvfs.vers b/libs/vfs/libvfs.vers
deleted file mode 100644
index 59e9e60..0000000
--- a/libs/vfs/libvfs.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.11
diff --git a/libs/vfs/manager.c b/libs/vfs/manager.c
index ec38743..227bcab 100644
--- a/libs/vfs/manager.c
+++ b/libs/vfs/manager.c
@@ -42,6 +42,7 @@
#include <kfg/repository.h>
#include <kfg/keystore.h>
#include <kfg/keystore-priv.h>
+#include <kfg/kfg-priv.h>
#include <vfs/resolver.h>
#include <sra/srapath.h>
@@ -84,8 +85,6 @@
#define DEFAULT_CACHE_BLOCKSIZE ( 32768 * 4 )
#define DEFAULT_CACHE_CLUSTER 1
-#define ENV_KRYPTO_PWFILE "VDB_PWFILE"
-
#define VFS_KRYPTO_PASSWORD_MAX_SIZE 4096
/*--------------------------------------------------------------------------
@@ -266,28 +265,50 @@ static rc_t CC VFSManagerGetConfigPWFile (const VFSManager * self, char * b, siz
return 0;
}
- rc = KConfigOpenNodeRead (self->cfg, &node, "krypto/pwfile");
- if (rc)
- {
- /* if not found, change object from path to encryption key */
+ { /* If we are in a protected area, there may be an area-specific key file */
+ const KRepositoryMgr *repoMgr;
+ rc = KConfigMakeRepositoryMgrRead ( self->cfg, &repoMgr );
+ if (rc == 0)
+ {
+ const KRepository* prot;
+ rc = KRepositoryMgrCurrentProtectedRepository ( repoMgr, &prot );
+ if (rc == 0)
+ {
+ rc = KRepositoryEncryptionKeyFile ( prot, b, bz, pz);
+ KRepositoryRelease(prot);
+ }
+ KRepositoryMgrRelease(repoMgr);
+ }
if (GetRCState(rc) == rcNotFound)
rc = RC (rcVFS, rcMgr, rcOpening, rcEncryptionKey, rcNotFound);
}
- else
- {
- rc = KConfigNodeRead (node, 0, b, bz-1, &z, &oopsy);
- if (rc == 0)
+
+ if (rc != 0)
+ { /* fall back on an old-style global password file*/
+ rc = KConfigOpenNodeRead (self->cfg, &node, KFG_KRYPTO_PWFILE);
+ if (rc)
{
- if (oopsy != 0)
- rc = RC (rcKrypto, rcMgr, rcReading, rcBuffer, rcInsufficient);
- else
+ /* if not found, change object from path to encryption key */
+ if (GetRCState(rc) == rcNotFound)
+ rc = RC (rcVFS, rcMgr, rcOpening, rcEncryptionKey, rcNotFound);
+ }
+ else
+ {
+ rc = KConfigNodeRead (node, 0, b, bz-1, &z, &oopsy);
+ if (rc == 0)
{
- b[z] = '\0';
- *pz = z;
+ if (oopsy != 0)
+ rc = RC (rcKrypto, rcMgr, rcReading, rcBuffer, rcInsufficient);
+ else
+ {
+ b[z] = '\0';
+ *pz = z;
+ }
}
+ KConfigNodeRelease (node);
}
- KConfigNodeRelease (node);
}
+
return rc;
}
@@ -1307,7 +1328,8 @@ rc_t TransformFileToDirectory(const KDirectory * dir,
else
{
/* we only use KAR/SRA or tar files as archives so try to identify
- * as our KAR/SRA file */
+ * as our KAR/SRA file.
+ IT IS NOT TRUE ANYMORE ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ */
if ( KFileIsSRA( tbuff, tz ) == 0 )
/* if it was open it as a directory */
rc = KDirectoryOpenSraArchiveReadUnbounded_silent_preopened( dir, d, false, file, path_str );
@@ -1318,21 +1340,30 @@ rc_t TransformFileToDirectory(const KDirectory * dir,
/*
* if RC here we did not have an SRA and did not have
- * a tar file we could use; assume the problem was decryption
- * if the file was encrypted
+ * a tar file we could use; assume the problem was:
+ * - decryption if the file was encrypted
+ * - or it is not an archive
*/
- if ( was_encrypted )
- {
- rc = RC( rcVFS, rcEncryptionKey, rcOpening, rcEncryption,
- rcIncorrect );
- PLOGERR (klogErr, (klogErr, rc, "could not use '$(P)' as an "
- "archive it was encrypted so the password"
- " was possibly wrong", "P=%s", path_str));
+ if (rc != 0) {
+ if ( was_encrypted ) {
+ /* the following RC update is not correct anymore but:
+ TODO: check tools/libraries
+ that expect this returned code and fix them
+ rc = RC(rcVFS, rcEncryptionKey, rcOpening, rcEncryption,
+ rcIncorrect ); */
+ PLOGERR (klogErr, (klogErr, rc,
+ "could not use '$(P)' as an "
+ "archive it was encrypted so the password"
+ " was possibly wrong or it is not SRA or"
+ " TAR file", "P=%s", path_str));
+ }
+ else {
+ PLOGERR (klogInfo, (klogInfo, rc,
+ "could not use '$(P)' as an "
+ "archive not identified as SRA or"
+ " TAR file", "P=%s", path_str));
+ }
}
- else
- PLOGERR (klogInfo, (klogInfo, rc, "could not use '$(P)' as an "
- "archive not identified as SRA or"
- " TAR file", "P=%s", path_str));
}
}
}
diff --git a/libs/vfs/path-priv.h b/libs/vfs/path-priv.h
index ee7f142..7724632 100644
--- a/libs/vfs/path-priv.h
+++ b/libs/vfs/path-priv.h
@@ -108,13 +108,13 @@ enum VHostVariant
/* legacy support */
#define VPathMake LegacyVPathMake
-rc_t VPathMake ( VPath ** new_path, const char * posix_path );
+VFS_EXTERN rc_t VPathMake ( VPath ** new_path, const char * posix_path );
#define VPathMakeFmt LegacyVPathMakeFmt
rc_t VPathMakeFmt ( VPath ** new_path, const char * fmt, ... );
#define VPathMakeVFmt LegacyVPathMakeVFmt
rc_t VPathMakeVFmt ( VPath ** new_path, const char * fmt, va_list args );
#define VPathMakeSysPath LegacyVPathMakeSysPath
-rc_t VPathMakeSysPath ( VPath ** new_path, const char * sys_path );
+VFS_EXTERN rc_t VPathMakeSysPath ( VPath ** new_path, const char * sys_path );
typedef enum eVPUri_t
{
diff --git a/libs/vfs/path.c b/libs/vfs/path.c
index ba010b5..35bffcd 100644
--- a/libs/vfs/path.c
+++ b/libs/vfs/path.c
@@ -1028,7 +1028,7 @@ rc_t VPathParse ( VPath * self, const char * uri, size_t uri_size )
{
state = vppOidRel;
oid = ch - '0';
- oid_anchor = i;
+ oid_anchor = (uint32_t)i;
}
else if ( ch != '/' )
@@ -1279,7 +1279,7 @@ rc_t VPathParse ( VPath * self, const char * uri, size_t uri_size )
else if ( isdigit ( ch ) )
{
if ( oid == 0 )
- oid_anchor = i;
+ oid_anchor = (uint32_t)i;
oid *= 10;
oid += ch - '0';
@@ -3256,7 +3256,7 @@ LIB_EXPORT rc_t CC VPathOption ( const VPath * self, VPOption_t option,
}
-rc_t LegacyVPathMake ( VPath ** new_path, const char * posix_path )
+LIB_EXPORT rc_t LegacyVPathMake ( VPath ** new_path, const char * posix_path )
{
return LegacyVPathMakeFmt ( new_path, posix_path );
}
diff --git a/libs/vfs/resolver.c b/libs/vfs/resolver.c
index 6c4c40b..9f5fb3a 100644
--- a/libs/vfs/resolver.c
+++ b/libs/vfs/resolver.c
@@ -46,7 +46,7 @@
#include <klib/namelist.h>
#include <klib/printf.h>
#include <klib/data-buffer.h>
-#include <klib/debug.h> /* DBGMSG */
+#include <klib/debug.h>
#include <klib/log.h>
#include <klib/rc.h>
@@ -62,12 +62,16 @@
/* to turn off CGI name resolution for
any refseq accessions */
-#define NO_REFSEQ_CGI 1
+#define NO_REFSEQ_CGI 0
/* to turn off CGI name resolution for
legacy WGS packages used by refseq */
#define NO_LEGACY_WGS_REFSEQ_CGI NO_REFSEQ_CGI
+/* do not allow fragments to be returned
+ unless we are expecting them for refseq */
+#define DISALLOW_FRAGMENT NO_LEGACY_WGS_REFSEQ_CGI
+
#define USE_CURL 1
#define NAME_SERVICE_MAJ_VERS 1
@@ -113,6 +117,16 @@ struct VResolverAccToken
String suffix;
};
+static
+void VResolverAccTokenInitFromOID ( VResolverAccToken *t, const String *acc )
+{
+ CONST_STRING ( & t -> prefix, "" );
+ t -> alpha = t -> prefix;
+ t -> acc = t -> digits = * acc;
+ t -> ext1 = t -> ext2 = t -> prefix;
+ t -> suffix = t -> prefix;
+}
+
/*--------------------------------------------------------------------------
* VResolverAlg
* represents a set of zero or more volumes
@@ -122,6 +136,7 @@ typedef enum
{
appUnknown,
appAny,
+ appFILE,
appREFSEQ,
appSRA,
appWGS,
@@ -133,7 +148,7 @@ typedef enum
typedef enum
{
algCGI,
- algLeafPath,
+ algFlat,
algSRAFlat,
algSRA1024,
algSRA1000,
@@ -188,7 +203,7 @@ struct VResolverAlg
bool protected;
/* whether the volumes are cache-capable
- in particularl, enabled if cache forced */
+ in particular, enabled if cache forced */
bool cache_capable;
/* whether the volumes are cache-enabled */
@@ -290,6 +305,19 @@ rc_t VResolverAlgMakeLocalPath ( const VResolverAlg *self,
return VPathMakeFmt ( ( VPath** ) path, "%S/%S/%S", self -> root, vol, exp );
}
+/* MakeLocalFilePath
+ * the path is known to exist in the local file system
+ * turn it into a VPath
+ */
+static
+rc_t VResolverAlgMakeLocalFilePath ( const VResolverAlg *self,
+ const String *vol, const String *exp, const char *krypto_ext, const VPath ** path )
+{
+ if ( self -> root == NULL )
+ return VPathMakeFmt ( ( VPath** ) path, "%S/%S%s", vol, exp, krypto_ext );
+ return VPathMakeFmt ( ( VPath** ) path, "%S/%S/%S%s", self -> root, vol, exp, krypto_ext );
+}
+
/* expand_accession
* expand accession according to algorithm
*/
@@ -304,7 +332,7 @@ rc_t expand_algorithm ( const VResolverAlg *self, const VResolverAccToken *tok,
{
case algCGI:
return RC ( rcVFS, rcResolver, rcResolving, rcType, rcIncorrect );
- case algLeafPath:
+ case algFlat:
rc = string_printf ( expanded, bsize, size, "%S", & tok -> acc );
break;
case algSRAFlat:
@@ -336,21 +364,21 @@ rc_t expand_algorithm ( const VResolverAlg *self, const VResolverAccToken *tok,
case algWGSFlat:
num = ( uint32_t ) ( tok -> alpha . size + 2 );
if ( tok -> prefix . size != 0 )
- num += tok -> prefix . size + 1;
+ num += (uint32_t) ( tok -> prefix . size + 1 );
rc = string_printf ( expanded, bsize, size,
"%.*S", num, & tok -> acc );
break;
case algWGS:
num = ( uint32_t ) ( tok -> alpha . size + 2 );
if ( tok -> prefix . size != 0 )
- num += tok -> prefix . size + 1;
+ num += (uint32_t) ( tok -> prefix . size + 1 );
rc = string_printf ( expanded, bsize, size,
"WGS/%.2s/%.2s/%.*S", tok -> alpha . addr, tok -> alpha . addr + 2, num, & tok -> acc );
break;
case algFuseWGS:
num = ( uint32_t ) ( tok -> alpha . size + 2 );
if ( tok -> prefix . size != 0 )
- num += tok -> prefix . size + 1;
+ num += (uint32_t) ( tok -> prefix . size + 1 );
rc = string_printf ( expanded, bsize, size,
"%.2s/%.2s/%.*S", tok -> alpha . addr, tok -> alpha . addr + 2, num, & tok -> acc );
break;
@@ -497,6 +525,121 @@ rc_t VResolverAlgLocalResolve ( const VResolverAlg *self,
return RC ( rcVFS, rcResolver, rcResolving, rcName, rcNotFound );
}
+/* LocalFile
+ * resolve an file name into a VPath or not found
+ */
+static
+rc_t VResolverAlgLocalFile ( const VResolverAlg *self,
+ const KDirectory *wd, const VPath *query,
+ const VPath ** path, bool for_cache )
+{
+ KPathType kpt;
+ uint32_t i, count;
+
+ /* in some cases, "root" is NULL */
+ const String *vol, *root = self -> root;
+
+ /* the file name */
+ String fname = query -> path;
+
+ /* cache extension */
+ const char *cache_ext = for_cache ? ".cache" : "";
+
+ /* encryption extension */
+ const char *krypto_ext = self -> protected ? ".ncbi_enc" : "";
+
+ /* now search all volumes */
+ count = VectorLength ( & self -> vols );
+ if ( root == NULL )
+ {
+ for ( i = 0; i < count; ++ i )
+ {
+ vol = VectorGet ( & self -> vols, i );
+ kpt = KDirectoryPathType ( wd
+ , "%.*s/%.*s%s%s"
+ , ( int ) vol -> size, vol -> addr
+ , ( int ) fname . size, fname . addr
+ , krypto_ext
+ , cache_ext
+ );
+
+ switch ( kpt & ~ kptAlias )
+ {
+ case kptFile:
+ case kptDir:
+ return VResolverAlgMakeLocalFilePath ( self, vol, & fname, krypto_ext, path );
+ default:
+ break;
+ }
+
+ if ( krypto_ext [ 0 ] != 0 )
+ {
+ kpt = KDirectoryPathType ( wd
+ , "%.*s/%.*s%s"
+ , ( int ) vol -> size, vol -> addr
+ , ( int ) fname . size, fname . addr
+ , cache_ext
+ );
+
+ switch ( kpt & ~ kptAlias )
+ {
+ case kptFile:
+ case kptDir:
+ return VResolverAlgMakeLocalFilePath ( self, vol, & fname, "", path );
+ default:
+ break;
+ }
+ }
+ }
+ }
+ else
+ {
+ for ( i = 0; i < count; ++ i )
+ {
+ vol = VectorGet ( & self -> vols, i );
+ kpt = KDirectoryPathType ( wd
+ , "%.*s/%.*s/%.*s%s%s"
+ , ( int ) root -> size, root -> addr
+ , ( int ) vol -> size, vol -> addr
+ , ( int ) fname . size, fname . addr
+ , krypto_ext
+ , cache_ext
+ );
+
+ switch ( kpt & ~ kptAlias )
+ {
+ case kptFile:
+ case kptDir:
+ return VResolverAlgMakeLocalFilePath ( self, vol, & fname, krypto_ext, path );
+ default:
+ break;
+ }
+
+ if ( krypto_ext [ 0 ] != 0 )
+ {
+ kpt = KDirectoryPathType ( wd
+ , "%.*s/%.*s/%.*s%s"
+ , ( int ) root -> size, root -> addr
+ , ( int ) vol -> size, vol -> addr
+ , ( int ) fname . size, fname . addr
+ , cache_ext
+ );
+
+ switch ( kpt & ~ kptAlias )
+ {
+ case kptFile:
+ case kptDir:
+ return VResolverAlgMakeLocalFilePath ( self, vol, & fname, "", path );
+ default:
+ break;
+ }
+ }
+ }
+ }
+
+ return RC ( rcVFS, rcResolver, rcResolving, rcName, rcNotFound );
+}
+
static
rc_t VPathCheckFromNamesCGI ( const VPath * path, const String *ticket, const VPath ** mapping )
{
@@ -534,6 +677,7 @@ rc_t VPathCheckFromNamesCGI ( const VPath * path, const String *ticket, const VP
{
case '.':
case '-':
+ case '_':
continue;
}
return RC ( rcVFS, rcResolver, rcResolving, rcMessage, rcCorrupt );
@@ -562,6 +706,7 @@ rc_t VPathCheckFromNamesCGI ( const VPath * path, const String *ticket, const VP
case '/':
case '.':
case '-':
+ case '_':
continue;
}
return RC ( rcVFS, rcResolver, rcResolving, rcMessage, rcCorrupt );
@@ -590,9 +735,11 @@ rc_t VPathCheckFromNamesCGI ( const VPath * path, const String *ticket, const VP
return RC ( rcVFS, rcResolver, rcResolving, rcMessage, rcCorrupt );
}
+#if DISALLOW_FRAGMENT
/* cannot have a fragment */
if ( path -> fragment . size != 0 )
return RC ( rcVFS, rcResolver, rcResolving, rcMessage, rcCorrupt );
+#endif
return 0;
}
@@ -1158,9 +1305,9 @@ rc_t VResolverAlgRemoteProtectedResolve ( const VResolverAlg *self,
default:
rc = RC ( rcVFS, rcResolver, rcResolving, rcParam, rcInvalid );
}
- if ( rc == 0 ) {
- DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
- (" %S = %S\n", &name, &val));
+ if ( rc == 0 )
+ {
+ DBGMSG(DBG_VFS, DBG_FLAG ( DBG_VFS ), (" %S = %S\n", & name, & val ) );
rc = KCurlRequestAddSField ( req, & name, & val );
}
}
@@ -1270,10 +1417,13 @@ rc_t VResolverAlgRemoteProtectedResolve ( const VResolverAlg *self,
}
if ( num_read == 0 )
break;
+ total += num_read;
}
-
+
if ( rc == 0 )
{
+ result.elem_count = total;
+
rc = VResolverAlgParseResolverCGIResponse ( & result, path, mapping, acc, self -> ticket );
KDataBufferWhack ( &result );
}
@@ -1415,6 +1565,25 @@ rc_t VResolverAlgCacheResolve ( const VResolverAlg *self,
}
+/* CacheFile
+ * try to resolve name for currently cached file
+ */
+static
+rc_t VResolverAlgCacheFile ( const VResolverAlg *self,
+ const KDirectory *wd, const VPath *query, const VPath ** path )
+{
+ /* see if the cache file already exists */
+ const bool for_cache = true;
+ rc_t rc = VResolverAlgLocalFile ( self, wd, query, path, for_cache );
+ if ( rc == 0 )
+ return 0;
+
+ /* TBD - see if any of these volumes is a good candidate for creating a file */
+
+ return RC ( rcVFS, rcResolver, rcResolving, rcName, rcNotFound );
+}
+
+
/* MakeCachePath
* we have an accession that matches this volume
@@ -1457,6 +1626,31 @@ rc_t VResolverAlgMakeCachePath ( const VResolverAlg *self,
}
+/* MakeCacheFilePath
+ * we have a filename that matches this volume
+ * create a path for it
+ */
+static
+rc_t VResolverAlgMakeCacheFilePath ( const VResolverAlg *self,
+ const VPath *query, const VPath ** path )
+{
+ String fname = query -> path;
+
+ /* needs proper extension for krypto */
+ const char * krypto_ext = self -> protected ? ".ncbi_enc" : "";
+
+ /* now search all volumes */
+ uint32_t i, count = VectorLength ( & self -> vols );
+ for ( i = 0; i < count; ++ i )
+ {
+ const String *vol = VectorGet ( & self -> vols, i );
+ return VResolverAlgMakeLocalFilePath ( self, vol, & fname, krypto_ext, path );
+ }
+
+ return RC ( rcVFS, rcResolver, rcResolving, rcPath, rcNotFound );
+}
+
+
/*--------------------------------------------------------------------------
* VResolver
@@ -1609,7 +1803,7 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
if ( i == size || i == 0 || i >= MAX_ACCESSION_LEN )
{
StringInit ( & tok -> prefix, acc, 0, 0 );
- StringInit ( & tok -> alpha, acc, i, i );
+ StringInit ( & tok -> alpha, acc, i, (uint32_t)i );
StringInit ( & tok -> digits, & acc [ i ], 0, 0 );
tok -> ext1 = tok -> ext2 = tok -> suffix = tok -> digits;
return 0;
@@ -1621,7 +1815,7 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
/* prefix
store only its presence, not length */
code = 1 << 4 * 4;
- StringInit ( & tok -> prefix, acc, i, i );
+ StringInit ( & tok -> prefix, acc, i, (uint32_t)i );
/* remove prefix */
acc += ++ i;
@@ -1636,19 +1830,19 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
if ( i == size || i >= MAX_ACCESSION_LEN )
{
- StringInit ( & tok -> alpha, acc, i, i );
+ StringInit ( & tok -> alpha, acc, i, (uint32_t)i );
StringInit ( & tok -> digits, & acc [ i ], 0, 0 );
tok -> ext1 = tok -> ext2 = tok -> suffix = tok -> digits;
return 0;
}
code |= i << 4 * 3;
- StringInit ( & tok -> alpha, acc, i, i );
+ StringInit ( & tok -> alpha, acc, i, (uint32_t)i );
}
else if ( ! isdigit ( acc [ i ] ) )
{
StringInit ( & tok -> prefix, acc, 0, 0 );
- StringInit ( & tok -> alpha, acc, i, i );
+ StringInit ( & tok -> alpha, acc, i, (uint32_t)i );
StringInit ( & tok -> digits, & acc [ i ], 0, 0 );
tok -> ext1 = tok -> ext2 = tok -> suffix = tok -> digits;
return 0;
@@ -1656,9 +1850,9 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
else
{
/* alpha */
- code = i << 4 * 3;
+ code = (uint32_t) ( i << 4 * 3 );
StringInit ( & tok -> prefix, acc, 0, 0 );
- StringInit ( & tok -> alpha, acc, i, i );
+ StringInit ( & tok -> alpha, acc, i, (uint32_t)i );
}
/* remove alpha */
@@ -1673,7 +1867,7 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
}
/* record digits */
- StringInit ( & tok -> digits, acc, i, i );
+ StringInit ( & tok -> digits, acc, i, (uint32_t)i );
StringInit ( & tok -> ext1, & acc [ i ], 0, 0 );
tok -> ext2 = tok -> suffix = tok -> ext1;
@@ -1705,7 +1899,7 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
return 0;
/* record the actual extension */
- StringInit ( & tok -> ext1, acc, i, i );
+ StringInit ( & tok -> ext1, acc, i, (uint32_t)i );
/* codify the extension simply as present, not by its length */
code |= 1 << 4 * 1;
@@ -1727,7 +1921,7 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
if ( i == 0 || i != size )
return 0;
- StringInit ( & tok -> suffix, acc, i, i );
+ StringInit ( & tok -> suffix, acc, i, (uint32_t)i );
/* NB - not incorporating suffix into code right now */
return code;
}
@@ -1750,7 +1944,7 @@ uint32_t get_accession_code ( const String * accession, VResolverAccToken *tok )
if ( i == 0 || i >= MAX_ACCESSION_LEN )
return 0;
- StringInit ( & tok -> ext2, acc, i, i );
+ StringInit ( & tok -> ext2, acc, i, (uint32_t)i );
code |= 1 << 4 * 0;
if ( i == size )
@@ -1867,7 +2061,6 @@ static
rc_t VResolverLocalResolve ( const VResolver *self,
const String * accession, const VPath ** path, bool refseq_ctx )
{
- rc_t rc;
uint32_t i, count;
VResolverAccToken tok;
@@ -1882,7 +2075,31 @@ rc_t VResolverLocalResolve ( const VResolver *self,
if ( alg -> app_id == app )
{
const bool for_cache = false;
- rc = VResolverAlgLocalResolve ( alg, self -> wd, & tok, path, legacy_wgs_refseq, for_cache );
+ rc_t rc = VResolverAlgLocalResolve ( alg, self -> wd, & tok, path, legacy_wgs_refseq, for_cache );
+ if ( rc == 0 )
+ return 0;
+ }
+ }
+
+ return RC ( rcVFS, rcResolver, rcResolving, rcName, rcNotFound );
+}
+
+/* LocalFile
+ * locate a locally stored file
+ */
+static
+rc_t VResolverLocalFile ( const VResolver *self, const VPath * query, const VPath ** path )
+{
+ uint32_t i, count;
+
+ count = VectorLength ( & self -> local );
+ for ( i = 0; i < count; ++ i )
+ {
+ const VResolverAlg *alg = VectorGet ( & self -> local, i );
+ if ( alg -> app_id == appFILE )
+ {
+ const bool for_cache = false;
+ rc_t rc = VResolverAlgLocalFile ( alg, self -> wd, query, path, for_cache );
if ( rc == 0 )
return 0;
}
@@ -1899,9 +2116,10 @@ bool VPathHasRefseqContext ( const VPath * accession )
rc_t rc = VPathOption ( accession, vpopt_vdb_ctx, option, sizeof option, & num_read );
if ( rc != 0 )
return false;
+
return ( num_read == sizeof "refseq" - 1 &&
strcase_cmp ( "refseq", sizeof "refseq" - 1,
- option, num_read, num_read ) == 0 );
+ option, num_read, (uint32_t)num_read ) == 0 );
}
@@ -1928,12 +2146,18 @@ rc_t CC VResolverLocal ( const VResolver * self,
case vpNameOrOID:
case vpNameOrAccession:
if ( * path != accession )
- break;
- default:
- VPathRelease ( * path );
- * path = NULL;
- rc = RC ( rcVFS, rcResolver, rcResolving, rcPath, rcNotFound );
+ return 0;
+ break;
+
+ case vpName:
+ if ( accession -> query . size != 0 && VPathHasRefseqContext ( accession ) )
+ return 0;
+ break;
}
+
+ VPathRelease ( * path );
+ * path = NULL;
+ rc = RC ( rcVFS, rcResolver, rcResolving, rcPath, rcNotFound );
}
return rc;
}
@@ -2074,7 +2298,7 @@ static
rc_t VResolverRemoteResolve ( const VResolver *self,
VRemoteProtocols protocols, const String * accession,
const VPath ** path, const VPath **mapping,
- const KFile ** opt_file_rtn, bool refseq_ctx )
+ const KFile ** opt_file_rtn, bool refseq_ctx, bool is_oid )
{
rc_t rc, try_rc;
uint32_t i, count;
@@ -2086,7 +2310,13 @@ rc_t VResolverRemoteResolve ( const VResolver *self,
VResolverEnableState remote_state = atomic32_read ( & enable_remote );
/* subject the accession to pattern recognition */
- app = get_accession_app ( accession, refseq_ctx, & tok, & legacy_wgs_refseq );
+ if ( ! is_oid )
+ app = get_accession_app ( accession, refseq_ctx, & tok, & legacy_wgs_refseq );
+ else
+ {
+ app = appAny;
+ VResolverAccTokenInitFromOID ( & tok, accession );
+ }
/* search all remote volumes by app and accession algorithm expansion */
count = VectorLength ( & self -> remote );
@@ -2360,6 +2590,77 @@ rc_t VResolverCacheResolve ( const VResolver *self,
return rc;
}
+static
+rc_t VResolverCacheFile ( const VResolver *self, const VPath * query, const VPath ** cache )
+{
+ rc_t rc = 0;
+
+ /* going to walk the local volumes, and remember
+ which one was best. actually, we have no algorithm
+ for determining it, so it's just the comment for TBD */
+ const VResolverAlg *alg, *best = NULL;
+
+ /* search the volumes for a cache-enabled place */
+ uint32_t i, count = VectorLength ( & self -> local );
+
+ /* check for protected status by presence of a download ticket */
+ bool protected = VPathHasDownloadTicket ( query );
+
+ VResolverEnableState cache_state = atomic32_read ( & enable_cache );
+
+ /* check for cache-enable override */
+ if ( cache_state == vrAlwaysEnable )
+ {
+ for ( i = 0; i < count; ++ i )
+ {
+ alg = VectorGet ( & self -> local, i );
+ if ( alg -> cache_capable && alg -> protected == protected && alg -> app_id == appFILE )
+ {
+ /* try to find an existing cache file
+ NB - race condition exists unless
+ we do something with lock files */
+ rc = VResolverAlgCacheFile ( alg, self -> wd, query, cache );
+ if ( rc == 0 )
+ return 0;
+
+ /* just remember the first as best for now */
+ if ( best == NULL )
+ best = alg;
+ }
+ }
+ }
+ else
+ {
+ for ( i = 0; i < count; ++ i )
+ {
+ alg = VectorGet ( & self -> local, i );
+ if ( alg -> cache_enabled && alg -> protected == protected && alg -> app_id == appFILE )
+ {
+ /* try to find an existing cache file
+ NB - race condition exists unless
+ we do something with lock files */
+ rc = VResolverAlgCacheFile ( alg, self -> wd, query, cache );
+ if ( rc == 0 )
+ return 0;
+
+ /* just remember the first as best for now */
+ if ( best == NULL )
+ best = alg;
+ }
+ }
+ }
+
+ /* no existing cache file was found,
+ so create a new one using the best
+ TBD - this should remember a volume path */
+ if ( best == NULL )
+ rc = RC ( rcVFS, rcResolver, rcResolving, rcPath, rcNotFound );
+ else
+ rc = VResolverAlgMakeCacheFilePath ( best, query, cache );
+
+ return rc;
+}
+
/* Cache
* Find a cache directory that might or might not contain a partially
@@ -2465,7 +2766,7 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
{
const VPath * remote2, * remote_mapping = NULL;
rc = VResolverRemoteResolve ( self, protocols, & accession,
- & remote2, & remote_mapping, NULL, refseq_ctx );
+ & remote2, & remote_mapping, NULL, refseq_ctx, true );
if ( rc == 0 )
{
/* got it. now enter into VFS manager's table */
@@ -2503,12 +2804,21 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
/* RESOLVE FOR LOCAL PATH */
if ( local != NULL && mapped_query != NULL )
{
- /* grab the path as accession */
- accession = mapped_query -> path;
+ if ( mapped_query -> path_type == vpName )
+ {
+ /* see if this is a file stored locally */
+ rc = VResolverLocalFile ( self, mapped_query, local );
+ }
+ else
+ {
+ /* grab the path as accession */
+ accession = mapped_query -> path;
+
+ /* resolve from accession to local path
+ will NOT find partial cache files */
+ rc = VResolverLocalResolve ( self, & accession, local, refseq_ctx );
+ }
- /* resolve from accession to local path
- will NOT find partial cache files */
- rc = VResolverLocalResolve ( self, & accession, local, refseq_ctx );
if ( rc == 0 && remote != NULL && * remote != NULL )
{
/* dump remote path used to map oid */
@@ -2530,7 +2840,7 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
const VPath * remote_mapping = NULL;
rc = VResolverRemoteResolve ( self, protocols, & accession, remote,
( mapped_query == NULL && cache != NULL ) ? & remote_mapping : NULL,
- NULL, refseq_ctx );
+ NULL, refseq_ctx, true );
if ( rc == 0 && mapped_query == NULL && cache != NULL && remote_mapping == NULL )
{
@@ -2564,8 +2874,17 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
/* RESOLVE FOR CACHE */
if ( ( remote == NULL || * remote != NULL ) && cache != NULL && mapped_query != NULL )
{
- /* resolve from accession to cache path */
- rc = VResolverCacheResolve ( self, mapped_query, has_fragment, cache, refseq_ctx );
+ /* test for file or accession */
+ if ( mapped_query -> path_type == vpName )
+ {
+ /* see if this is a file stored locally */
+ rc = VResolverCacheFile ( self, mapped_query, cache );
+ }
+ else
+ {
+ /* resolve from accession to cache path */
+ rc = VResolverCacheResolve ( self, mapped_query, has_fragment, cache, refseq_ctx );
+ }
if ( rc != 0 && remote != NULL )
{
assert ( * cache == NULL );
@@ -2619,7 +2938,7 @@ rc_t VResolverQueryAcc ( const VResolver * self, VRemoteProtocols protocols,
/* request remote resolution
this does not need to map the query to an accession */
rc = VResolverRemoteResolve ( self, protocols, accession,
- & remote2, mapped_ptr, NULL, refseq_ctx );
+ & remote2, mapped_ptr, NULL, refseq_ctx, false );
if ( rc == 0 )
{
@@ -2913,6 +3232,16 @@ rc_t CC VResolverQuery ( const VResolver * self, VRemoteProtocols protocols,
break;
case vpName:
+ /* try to handle a weird case with non-accessioned names in refseq area */
+ if ( query -> query . size != 0 )
+ {
+ if ( VPathHasRefseqContext ( query ) )
+ {
+ rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache );
+ if ( rc == 0 )
+ break;
+ }
+ }
try_name:
rc = VResolverQueryName ( self, protocols, query, local, remote, cache );
break;
@@ -3075,6 +3404,9 @@ rc_t VResolverLoadVolumes ( Vector *algs, const String *root, const String *tick
/* if using CGI for resolution */
if ( resolver_cgi || strcmp ( algname, "cgi" ) == 0 )
alg_id = algCGI;
+ /* stored in a flat directory as-is */
+ else if ( strcmp ( algname, "flat" ) == 0 )
+ alg_id = algFlat;
/* stored in a flat directory with ".sra" extension */
else if ( strcmp ( algname, "sraFlat" ) == 0 )
alg_id = algSRAFlat;
@@ -3253,16 +3585,18 @@ rc_t VResolverLoadApps ( VResolver *self, Vector *algs, const String *root,
if ( rc == 0 )
{
VResolverAppID app_id = appUnknown;
- if ( strcmp ( appname, "refseq" ) == 0 )
+ if ( strcmp ( appname, "file" ) == 0 )
+ app_id = appFILE;
+ else if ( strcmp ( appname, "nakmer" ) == 0 )
+ app_id = appNAKMER;
+ else if ( strcmp ( appname, "nannot" ) == 0 )
+ app_id = appNANNOT;
+ else if ( strcmp ( appname, "refseq" ) == 0 )
app_id = appREFSEQ;
else if ( strcmp ( appname, "sra" ) == 0 )
app_id = appSRA;
else if ( strcmp ( appname, "wgs" ) == 0 )
app_id = appWGS;
- else if ( strcmp ( appname, "nannot" ) == 0 )
- app_id = appNANNOT;
- else if ( strcmp ( appname, "nakmer" ) == 0 )
- app_id = appNAKMER;
rc = VResolverLoadApp ( self, algs, root, ticket, allow_cache, app_id,
& self -> num_app_vols [ app_id ], app, resolver_cgi, protected, disabled, caching );
@@ -3719,7 +4053,7 @@ rc_t VResolverDetectSRALeafPath ( VResolver *self )
/* create an algorithm for any application where the
spec is to be treated as a leaf path */
VResolverAlg *alg;
- rc = VResolverAlgMake ( & alg, root, appAny, algLeafPath, self -> ticket != NULL, false );
+ rc = VResolverAlgMake ( & alg, root, appAny, algFlat, self -> ticket != NULL, false );
if ( rc == 0 )
{
const String *vol;
@@ -3752,6 +4086,70 @@ rc_t VResolverDetectSRALeafPath ( VResolver *self )
}
static
+rc_t VResolverForceUserFilesVol ( VResolver *self, const VResolverAlg *sraAlg )
+{
+ /* create an algorithm for "file" application where the
+ spec is to be treated as a leaf path */
+ VResolverAlg *alg;
+ rc_t rc = VResolverAlgMake ( & alg, sraAlg -> root, appFILE, algFlat, sraAlg -> protected, sraAlg -> disabled );
+ if ( rc == 0 )
+ {
+ String vol_str;
+ const String *vol;
+
+ /* create a single volume - "files" */
+ CONST_STRING ( & vol_str, "files" );
+ rc = StringCopy ( & vol, & vol_str );
+ if ( rc == 0 )
+ {
+ rc = VectorAppend ( & alg -> vols, NULL, vol );
+ if ( rc != 0 )
+ free ( ( void* ) vol );
+ else
+ {
+ /* copy sra settings */
+ alg -> ticket = sraAlg -> ticket;
+ alg -> cache_capable = sraAlg -> cache_capable;
+ alg -> cache_enabled = sraAlg -> cache_enabled;
+
+ /* insert into local resolution path */
+ rc = VectorAppend ( & self -> local, NULL, alg );
+ if ( rc == 0 )
+ {
+ ++ self -> num_app_vols [ appFILE ];
+ return 0;
+ }
+ }
+ }
+
+ VResolverAlgWhack ( alg, NULL );
+ }
+
+ return rc;
+}
+
+static
+rc_t VResolverForceUserFiles ( VResolver *self )
+{
+ rc_t rc;
+ uint32_t i, count = VectorLength ( & self -> local );
+
+ for ( rc = 0, i = 0; i < count; ++ i )
+ {
+ const VResolverAlg *alg = VectorGet ( & self -> local, i );
+ if ( alg -> app_id == appSRA && alg -> cache_capable )
+ {
+ rc = VResolverForceUserFilesVol ( self, alg );
+ if ( rc != 0 )
+ break;
+ }
+ }
+
+ return rc;
+}
+
+
+static
rc_t VResolverLoad ( VResolver *self, const KRepository *protected, const KConfig *cfg )
{
bool have_remote_protected = false;
@@ -3771,8 +4169,11 @@ rc_t VResolverLoad ( VResolver *self, const KRepository *protected, const KConfi
/* if the user is inside of a protected workspace, load it now */
if ( rc == 0 && self -> ticket != NULL )
+ {
rc = VResolverLoadProtected ( self, kfg, buffer );
-
+ if ( rc == 0 && self -> num_app_vols [ appFILE ] == 0 )
+ rc = VResolverForceUserFiles ( self );
+ }
/* now load user public repositories */
if ( rc == 0 )
rc = VResolverLoadSubCategory ( self, & self -> local, kfg, NULL, "user/main", cacheAllow, false );
@@ -3841,6 +4242,9 @@ rc_t VResolverLoad ( VResolver *self, const KRepository *protected, const KConfi
}
}
+ if ( rc == 0 && self -> num_app_vols [ appFILE ] == 0 )
+ rc = VResolverForceUserFiles ( self );
+
if ( rc == 0 && self -> ticket != NULL && ! have_remote_protected )
rc = VResolverForceRemoteProtected ( self );
@@ -3849,24 +4253,20 @@ rc_t VResolverLoad ( VResolver *self, const KRepository *protected, const KConfi
return rc;
}
-#if 1
LIB_EXPORT
rc_t CC VResolverProtocols ( VResolver * self,
VRemoteProtocols protocols )
{
- if ( self == NULL ) {
+ if ( self == NULL )
return RC ( rcVFS, rcResolver, rcUpdating, rcSelf, rcNull );
- }
- if ( protocols >= eProtocolLastDefined ) {
+ if ( protocols >= eProtocolLastDefined )
return RC ( rcVFS, rcResolver, rcUpdating, rcParam, rcInvalid );
- }
self -> protocols = protocols;
return 0;
}
-#endif
/* Make
diff --git a/libs/vfs/unix/syspath.c b/libs/vfs/unix/syspath.c
index e128181..334b7a4 100644
--- a/libs/vfs/unix/syspath.c
+++ b/libs/vfs/unix/syspath.c
@@ -143,7 +143,7 @@ LIB_EXPORT rc_t CC VFSManagerWMakeSysPath ( const VFSManager * self,
HACK O' MATIC
*/
-rc_t LegacyVPathMakeSysPath ( VPath ** new_path, const char * sys_path )
+LIB_EXPORT rc_t LegacyVPathMakeSysPath ( VPath ** new_path, const char * sys_path )
{
return LegacyVPathMake ( new_path, sys_path );
}
diff --git a/libs/vfs/win/syspath.c b/libs/vfs/win/syspath.c
index 2a60ee5..2c9a849 100644
--- a/libs/vfs/win/syspath.c
+++ b/libs/vfs/win/syspath.c
@@ -32,8 +32,9 @@
#include <klib/rc.h>
#include <klib/log.h>
#include <klib/out.h>
+#include <klib/debug.h>
-#include "path-priv.h"
+#include "../path-priv.h"
#include <wchar.h>
#include <windows.h>
@@ -329,7 +330,7 @@ LIB_EXPORT rc_t CC VFSManagerWMakeSysPath ( const VFSManager * self,
HACK O' MATIC
*/
-rc_t LegacyVPathMakeSysPath ( VPath ** new_path, const char * sys_path )
+LIB_EXPORT rc_t LegacyVPathMakeSysPath ( VPath ** new_path, const char * sys_path )
{
VFSManager * vfs;
rc_t rc = VFSManagerMake ( & vfs );
diff --git a/libs/vxf/libvxf.vers b/libs/vxf/libvxf.vers
deleted file mode 100644
index 2bf1ca5..0000000
--- a/libs/vxf/libvxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.7
diff --git a/libs/vxf/libwvxf.vers b/libs/vxf/libwvxf.vers
deleted file mode 100644
index 0664a8f..0000000
--- a/libs/vxf/libwvxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.1.6
diff --git a/libs/vxf/simple-sub-select.c b/libs/vxf/simple-sub-select.c
index 044d0aa..527f111 100644
--- a/libs/vxf/simple-sub-select.c
+++ b/libs/vxf/simple-sub-select.c
@@ -74,42 +74,42 @@ rc_t CC simple_sub_select ( void *data, const VXformInfo *info,
if(argv [ 0 ] . u . data . elem_count == 0){/** alow empty returns ***/
rc = KDataBufferResize ( rslt -> data, 0 );
rslt -> elem_count = 0;
- return 0;
+ return 0;
}
assert ( argv [ 0 ] . u . data . elem_count == 1 );
if(argc > 1 ){
const int32_t *remote_idx = argv[1].u.data.base;
- idx=remote_idx[argv[1].u.data.first_elem];
+ idx=remote_idx[argv[1].u.data.first_elem];
}
if(self->native_curs && self->first_time){ /*** can we use native cursor for the data? ***/
- SubSelect *mself = (SubSelect *)self;
- uint32_t idx;
- rc = VCursorAddColumn ( mself->native_curs, & idx, "%.*s", mself->col_name_len, mself->col_name );
- if(rc == 0 || GetRCState(rc) == rcExists){
- VCursorRelease( mself->curs);
- mself->curs = mself->native_curs;
- mself->idx = idx;
- }
- mself->first_time = false;
- rc = 0; /** reset rc? **/
+ SubSelect *mself = (SubSelect *)self;
+ uint32_t idx;
+ rc = VCursorAddColumn ( mself->native_curs, & idx, "%.*s", mself->col_name_len, mself->col_name );
+ if(rc == 0 || GetRCState(rc) == rcExists){
+ VCursorRelease( mself->curs);
+ mself->curs = mself->native_curs;
+ mself->idx = idx;
+ }
+ mself->first_time = false;
+ rc = 0; /** reset rc? **/
}
/* sub-select */
rc = VCursorCellDataDirect ( self -> curs, * remote_row_id, self -> idx,
- & elem_bits, & base, & boff, & row_len );
+ & elem_bits, & base, & boff, & row_len );
if ( rc == 0 )
{
- uint8_t *cbase=(uint8_t*)base + (boff>>3);
+ uint8_t *cbase=(uint8_t*)base + (boff>>3);
rslt -> data -> elem_bits = elem_bits;
- if(idx < 0 || idx > row_len){ /** out of bounds **/
- rc = KDataBufferResize ( rslt -> data, 0 );
- rslt -> elem_count = 0;
- return rc;
- } else if (idx > 0){ /*** do subset ***/
- row_len=1;
- cbase += (elem_bits>>3)*(idx-1);
- boff += elem_bits * (idx-1);
- }
+ if(idx < 0 || idx > row_len){ /** out of bounds **/
+ rc = KDataBufferResize ( rslt -> data, 0 );
+ rslt -> elem_count = 0;
+ return rc;
+ } else if (idx > 0){ /*** do subset ***/
+ row_len=1;
+ cbase += (elem_bits>>3)*(idx-1);
+ boff += elem_bits * (idx-1);
+ }
rc = KDataBufferResize ( rslt -> data, row_len );
if ( rc == 0 )
@@ -119,9 +119,9 @@ rc_t CC simple_sub_select ( void *data, const VXformInfo *info,
if ( ( elem_bits & 7 ) != 0 )
bitcpy ( rslt -> data -> base, 0, base, boff, bits );
else{
- assert((boff&7)==0);
+ assert((boff&7)==0);
memcpy ( rslt -> data -> base, cbase, bits>>3 );
- }
+ }
rslt -> elem_count = row_len;
}
@@ -148,83 +148,83 @@ rc_t open_sub_cursor ( SubSelect **fself, const VXfactInfo *info, const VFactory
if ( cp -> argv [ 0 ] . count > 0 )
{
- sprintf(name,"%.*s",(int)cp->argv[0].count,cp->argv[0].data.ascii);
- rc = VCursorLinkedCursorGet(native_curs,name,&curs);
- if(rc == 0){
- rc = VCursorOpenParentRead(curs,&ftbl);
- if(rc != 0)
- return rc;
- VCursorAddRef(curs);
- tbl = ftbl;
- } else {
- const VDatabase *db;
- rc = VTableOpenParentRead ( info -> tbl, & db );
- if(rc != 0) return rc;
- rc = VDatabaseOpenTableRead ( db, & ftbl,name);
- if(rc != 0) return rc;
- VDatabaseRelease ( db );
- tbl = ftbl;
- rc = VTableCreateCachedCursorRead ( tbl, (const VCursor**)& curs, 256*1024*1024 ); /*** some random io is expected ***/
- if(rc != 0) return rc;
- rc = VCursorPermitPostOpenAdd( curs );
- if(rc != 0) return rc;
- rc = VCursorOpen( curs );
- if(rc != 0) return rc;
- rc = VCursorLinkedCursorSet(native_curs,name,curs);
- if(rc != 0) return rc;
- }
- native_curs = NULL;
+ sprintf(name,"%.*s",(int)cp->argv[0].count,cp->argv[0].data.ascii);
+ rc = VCursorLinkedCursorGet(native_curs,name,&curs);
+ if(rc == 0){
+ rc = VCursorOpenParentRead(curs,&ftbl);
+ if(rc != 0)
+ return rc;
+ VCursorAddRef(curs);
+ tbl = ftbl;
+ } else {
+ const VDatabase *db;
+ rc = VTableOpenParentRead ( info -> tbl, & db );
+ if(rc != 0) return rc;
+ rc = VDatabaseOpenTableRead ( db, & ftbl,name);
+ if(rc != 0) return rc;
+ VDatabaseRelease ( db );
+ tbl = ftbl;
+ rc = VTableCreateCachedCursorRead ( tbl, (const VCursor**)& curs, 256*1024*1024 ); /*** some random io is expected ***/
+ if(rc != 0) return rc;
+ rc = VCursorPermitPostOpenAdd( curs );
+ if(rc != 0) return rc;
+ rc = VCursorOpen( curs );
+ if(rc != 0) return rc;
+ rc = VCursorLinkedCursorSet(native_curs,name,curs);
+ if(rc != 0) return rc;
+ }
+ native_curs = NULL;
}
else /** we don't now if native_curs permits adding columns **/
{
tbl = info -> tbl;
- rc = VTableCreateCachedCursorRead ( tbl, & curs, 0 ); /*** some random io is expected ***/
+ rc = VTableCreateCachedCursorRead ( tbl, & curs, 0 ); /*** some random io is expected ***/
}
if ( rc == 0 )
{
- uint32_t idx;
- rc = VCursorAddColumn ( curs, & idx, "%.*s",
- ( int ) cp -> argv [ 1 ] . count, cp -> argv [ 1 ] . data . ascii );
- if ( rc == 0 || GetRCState(rc) == rcExists)
+ uint32_t idx;
+ rc = VCursorAddColumn ( curs, & idx, "%.*s",
+ ( int ) cp -> argv [ 1 ] . count, cp -> argv [ 1 ] . data . ascii );
+ if ( rc == 0 || GetRCState(rc) == rcExists)
+ {
+ rc = VCursorOpen ( curs );
+ if ( rc == 0 )
{
- rc = VCursorOpen ( curs );
+ VTypedesc src;
+ rc = VCursorDatatype ( curs, idx, NULL, & src );
if ( rc == 0 )
{
- VTypedesc src;
- rc = VCursorDatatype ( curs, idx, NULL, & src );
- if ( rc == 0 )
+ /* selected column should have same characteristics */
+ if ( src . domain != info -> fdesc . desc . domain ||
+ src . intrinsic_bits != info -> fdesc . desc . intrinsic_bits ||
+ src . intrinsic_dim != info -> fdesc . desc. intrinsic_dim )
{
- /* selected column should have same characteristics */
- if ( src . domain != info -> fdesc . desc . domain ||
- src . intrinsic_bits != info -> fdesc . desc . intrinsic_bits ||
- src . intrinsic_dim != info -> fdesc . desc. intrinsic_dim )
- {
- rc = RC ( rcXF, rcFunction, rcConstructing, rcType, rcInconsistent );
- }
+ rc = RC ( rcXF, rcFunction, rcConstructing, rcType, rcInconsistent );
+ }
+ else
+ {
+ SubSelect *self = malloc ( sizeof * self );
+ if ( self == NULL )
+ rc = RC ( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
else
{
- SubSelect *self = malloc ( sizeof * self );
- if ( self == NULL )
- rc = RC ( rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted );
- else
- {
- self -> curs = curs;
- self -> idx = idx;
- self->native_curs = native_curs;
- self->first_time = true;
- self->col_name_len = cp->argv[1].count;
- self->col_name = malloc(self->col_name_len);
- memcpy(self->col_name,cp->argv[1].data.ascii,self->col_name_len);
- * fself = self;
- if ( ftbl != NULL )
- VTableRelease ( ftbl );
- return 0;
- }
+ self -> curs = curs;
+ self -> idx = idx;
+ self->native_curs = native_curs;
+ self->first_time = true;
+ self->col_name_len = cp->argv[1].count;
+ self->col_name = malloc(self->col_name_len);
+ memcpy(self->col_name,cp->argv[1].data.ascii,self->col_name_len);
+ * fself = self;
+ if ( ftbl != NULL )
+ VTableRelease ( ftbl );
+ return 0;
}
}
}
}
- VCursorRelease ( curs );
+ }
+ VCursorRelease ( curs );
}
if ( ftbl != NULL )
VTableRelease ( ftbl );
diff --git a/libs/wgsxf/libwgsxf.vers b/libs/wgsxf/libwgsxf.vers
deleted file mode 100644
index 3eefcb9..0000000
--- a/libs/wgsxf/libwgsxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.0
diff --git a/libs/wgsxf/libwwgsxf.vers b/libs/wgsxf/libwwgsxf.vers
deleted file mode 100644
index 3eefcb9..0000000
--- a/libs/wgsxf/libwwgsxf.vers
+++ /dev/null
@@ -1 +0,0 @@
-1.0.0
diff --git a/tools/Makefile b/tools/Makefile
index 09776c5..5b4ba65 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -63,18 +63,6 @@ $(SUBDIRS):
.PHONY: default $(SUBDIRS)
#-------------------------------------------------------------------------------
-# tag
-#
-SUBDIRS_TAG = $(addsuffix _tag,$(SUBDIRS))
-
-tag: $(SUBDIRS_TAG)
-
-$(SUBDIRS_TAG):
- @ $(MAKE) -C $(subst _tag,,$@) tag
-
-.PHONY: tag $(SUBDIRS_TAG)
-
-#-------------------------------------------------------------------------------
# pass-through
#
out debug profile release GCC ICC VC++:
diff --git a/tools/align-info/align-info.vers b/tools/align-info/align-info.vers
index 0bee604..3f684d2 100644
--- a/tools/align-info/align-info.vers
+++ b/tools/align-info/align-info.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/bam-loader/bam-load.3.vers b/tools/bam-loader/bam-load.3.vers
new file mode 100644
index 0000000..4a36342
--- /dev/null
+++ b/tools/bam-loader/bam-load.3.vers
@@ -0,0 +1 @@
+3.0.0
diff --git a/tools/bam-loader/bam-load.vers b/tools/bam-loader/bam-load.vers
index 0bee604..3f684d2 100644
--- a/tools/bam-loader/bam-load.vers
+++ b/tools/bam-loader/bam-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/bam-loader/loader-imp.c b/tools/bam-loader/loader-imp.c
index 7883a76..9f4d0de 100644
--- a/tools/bam-loader/loader-imp.c
+++ b/tools/bam-loader/loader-imp.c
@@ -362,8 +362,30 @@ static unsigned HashKey(void const *key, unsigned keylen)
return h;
}
-static rc_t GetKeyID(context_t *const ctx, uint64_t *const rslt, bool *const wasInserted, char const key[], char const name[], unsigned const namelen)
+#define USE_ILLUMINA_NAMING_POUND_NUMBER_SLASH_HACK 1
+
+static size_t GetFixedNameLength(char const name[], size_t const namelen)
{
+#if USE_ILLUMINA_NAMING_POUND_NUMBER_SLASH_HACK
+ char const *const pound = string_chr(name, namelen, '#');
+
+ if (pound && pound + 2u < name + namelen && pound[1] >= '0' && pound[1] <= '9' && pound[2] == '/') {
+ return (size_t)(pound - name) + 2u;
+ }
+#endif
+ return namelen;
+}
+
+static
+rc_t GetKeyID(context_t *const ctx,
+ uint64_t *const rslt,
+ bool *const wasInserted,
+ char const key[],
+ char const name[],
+ size_t const o_namelen)
+{
+ size_t const namelen = GetFixedNameLength(name, o_namelen);
+
if (ctx->key2id_max == 1)
return GetKeyIDOld(ctx, rslt, wasInserted, key, name, namelen);
else {
@@ -462,6 +484,7 @@ static rc_t GetKeyID(context_t *const ctx, uint64_t *const rslt, bool *const was
tmpKey = ctx->idCount[f];
rc = KBTreeEntry(ctx->key2id[f], &tmpKey, wasInserted, name, namelen);
if (rc == 0) {
+ /* fprintf(stderr, "GetKeyID: { Key: '%s', Name: '%.*s', id: '%u:%x', new: %s }\n", key, (int)namelen, name, (unsigned)f, (unsigned)tmpKey, *wasInserted ? "true" : "false"); */
*rslt = (((uint64_t)f) << 32) | tmpKey;
if (*wasInserted)
++ctx->idCount[f];
@@ -1587,7 +1610,7 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
}
}
}
- else if (wasInserted & (isPrimary || !originally_aligned)) {
+ else if (CTX_VALUE_GET_S_ID(*value) == 0 && (isPrimary || !originally_aligned)) {
/* new unmated fragment - no spot assembly */
unsigned readLen[1];
diff --git a/tools/cache-mgr/Makefile b/tools/cache-mgr/Makefile
index b58dd2e..ea778c2 100644
--- a/tools/cache-mgr/Makefile
+++ b/tools/cache-mgr/Makefile
@@ -81,13 +81,6 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# tag
-#
-tag: $(addsuffix _tag,$(ALL_TOOLS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_TOOLS))
-
-#-------------------------------------------------------------------------------
# cache-mgr
#
TOOL_SRC = \
@@ -98,23 +91,9 @@ TOOL_OBJ = \
TOOL_LIB = \
-lkapp \
- -ssraschema \
- -dalign-reader \
- $(READONLY_SCHEMA_LIBS) \
- -lvdb \
- -lkdb \
- -lvfs \
- -lkurl \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lksrch \
- -lkproc \
- -lklib \
+ -lncbi-vdb \
-lm
$(BINDIR)/cache-mgr: $(TOOL_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(TOOL_LIB)
-cache-mgr_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) cache-mgr $(TOOL_OBJ)
diff --git a/tools/cache-mgr/cache-mgr.c b/tools/cache-mgr/cache-mgr.c
index d06e2dc..7c863dd 100644
--- a/tools/cache-mgr/cache-mgr.c
+++ b/tools/cache-mgr/cache-mgr.c
@@ -1117,14 +1117,14 @@ static rc_t perform_disable( visit_ctx * octx )
/***************************************************************************************************************/
-static rc_t perform( tool_options * options, void * data )
+static rc_t perform( tool_options * options, Args * args )
{
visit_ctx octx;
rc_t rc = KDirectoryNativeDir( &octx.dir );
if ( rc == 0 )
{
octx.options = options;
- octx.data = data;
+ octx.data = NULL;
switch( options->main_function )
{
case tf_report : rc = perform_report( &octx ); break;
@@ -1133,7 +1133,7 @@ static rc_t perform( tool_options * options, void * data )
case tf_clear : rc = perform_clear( &octx ); break;
case tf_enable : rc = perform_enable( &octx ); break;
case tf_disable : rc = perform_disable( &octx ); break;
- case tf_unknown : rc = KOutMsg( "unknown function requested\n" ); break;
+ case tf_unknown : rc = Usage( args ); break;
}
KDirectoryRelease( octx.dir );
}
@@ -1311,7 +1311,7 @@ rc_t CC KMain ( int argc, char *argv [] )
rc = explain_no_cache_found ();
}
else
- rc = perform( &options, NULL );
+ rc = perform( &options, args );
}
}
clear_tool_options( &options );
diff --git a/tools/cache-mgr/cache-mgr.vers b/tools/cache-mgr/cache-mgr.vers
index 0bee604..3f684d2 100644
--- a/tools/cache-mgr/cache-mgr.vers
+++ b/tools/cache-mgr/cache-mgr.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/cg-load/Makefile b/tools/cg-load/Makefile
index 9636568..38038fa 100644
--- a/tools/cg-load/Makefile
+++ b/tools/cg-load/Makefile
@@ -76,21 +76,22 @@ clean: stdclean
# cg-load
#
CGLOAD_SRC = \
- file-version-factory \
- file \
+ cg-load \
+ factory-evidence-intervals \
+ factory-evidence-dnbs \
+ factory-mappings \
+ factory-reads \
+ factory-tag-lfr \
f1_3 \
f1_5 \
f2_0 \
f2_2 \
- factory-reads \
- factory-mappings \
- factory-evidence-intervals \
- factory-evidence-dnbs \
- writer-seq \
+ file \
+ file-version-factory \
writer-algn \
- writer-evidence-intervals \
writer-evidence-dnbs \
- cg-load
+ writer-evidence-intervals \
+ writer-seq
CGLOAD_OBJ = \
$(addsuffix .$(OBJX),$(CGLOAD_SRC))
diff --git a/tools/cg-load/cg-load.c b/tools/cg-load/cg-load.c
index e40a4c5..19eac93 100644
--- a/tools/cg-load/cg-load.c
+++ b/tools/cg-load/cg-load.c
@@ -255,7 +255,8 @@ typedef struct FGroupKey_struct {
} FGroupKey;
static
-rc_t CC FGroupKey_Make(FGroupKey* key, const CGLoaderFile* file, const SParam* param)
+rc_t CC FGroupKey_Make(FGroupKey* key,
+ const CGLoaderFile* file, const SParam* param)
{
rc_t rc = 0;
CG_EFileType ftype;
@@ -264,11 +265,14 @@ rc_t CC FGroupKey_Make(FGroupKey* key, const CGLoaderFile* file, const SParam* p
switch(ftype) {
case cg_eFileType_READS:
case cg_eFileType_MAPPINGS:
+ case cg_eFileType_TAG_LFR:
key->type = cg_eFileType_READS;
- if( (rc = CGLoaderFile_GetAssemblyId(file, &key->assembly_id)) == 0 &&
+ if ((rc = CGLoaderFile_GetAssemblyId(file, &key->assembly_id)) == 0 &&
(rc = CGLoaderFile_GetSlide(file, &key->u.map.slide)) == 0 &&
- (rc = CGLoaderFile_GetLane(file, &key->u.map.lane)) == 0 ) {
- rc = CGLoaderFile_GetBatchFileNumber(file, &key->u.map.batch_file_number);
+ (rc = CGLoaderFile_GetLane(file, &key->u.map.lane)) == 0)
+ {
+ rc = CGLoaderFile_GetBatchFileNumber(file,
+ &key->u.map.batch_file_number);
}
break;
case cg_eFileType_EVIDENCE_INTERVALS:
@@ -320,6 +324,7 @@ typedef struct FGroupMAP_struct {
FGroupKey key;
const CGLoaderFile* seq;
const CGLoaderFile* align;
+ const CGLoaderFile* tagLfr;
} FGroupMAP;
static
@@ -416,6 +421,13 @@ rc_t CC FGroupMAP_Set(FGroupMAP* g, const CGLoaderFile* file)
} else {
rc = RC(rcExe, rcQueue, rcInserting, rcItem, rcDuplicate);
}
+ }
+ else if (ftype == cg_eFileType_TAG_LFR) {
+ if (g->tagLfr == NULL) {
+ g->tagLfr = file;
+ } else {
+ rc = RC(rcExe, rcQueue, rcInserting, rcItem, rcDuplicate);
+ }
} else {
rc = RC(rcExe, rcQueue, rcInserting, rcItem, rcWrongType);
}
@@ -460,6 +472,11 @@ void CC FGroupMAP_Validate( BSTNode *n, void *data )
mnm = mnm ? strrchr(mnm, '/') : mnm;
DEBUG_MSG(5, (" MAPPINGS(%s)", mnm));
}
+ if (g->tagLfr != NULL) {
+ CGLoaderFile_Filename(g->tagLfr, &mnm);
+ mnm = mnm ? strrchr(mnm, '/') : mnm;
+ DEBUG_MSG(5, (" TAG_LFR(%s)", mnm));
+ }
DEBUG_MSG(5, ("\n"));
if( rc == 0 && g->seq == NULL ) {
rc = RC(rcExe, rcQueue, rcValidating, rcItem, rcIncomplete);
@@ -496,7 +513,7 @@ rc_t CC DirVisitor(const KDirectory *dir, uint32_t type, const char *name, void
strcmp(&name[strlen(name) - 7], ".tsv.gz") == 0)
{
char buf[4096];
- const CGLoaderFile* file;
+ const CGLoaderFile* file = NULL;
FGroupKey key;
if( (rc = KDirectoryResolvePath(dir, true, buf, sizeof(buf), name)) == 0 &&
(rc = CGLoaderFile_Make(&file, d->dir, buf, NULL, !d->param->no_read_ahead)) == 0 &&
@@ -545,46 +562,105 @@ typedef struct FGroupMAP_LoadData_struct {
const BSTree* reads;
} FGroupMAP_LoadData;
+typedef enum {
+ eCtxRead,
+ eCtxLfr,
+ eCtxMapping
+} TCtx;
+static bool _FGroupMAPDone(FGroupMAP *self, TCtx ctx, FGroupMAP_LoadData* d) {
+ /* (rcData rcDone) is always set on reads file EOF */
+ bool eofLfr = true;
+ bool eofMapping = true;
+ assert(self && d);
+ if (d->rc == 0 ||
+ GetRCState(d->rc) != rcDone || GetRCObject(d->rc) != rcData)
+ {
+ return false;
+ }
+ d->rc = 0;
+ if (d->rc == 0 && self->tagLfr != NULL) {
+ d->rc = CGLoaderFile_IsEof(self->tagLfr, &eofLfr);
+ }
+ if (d->rc == 0 && self->align != NULL) {
+ d->rc = CGLoaderFile_IsEof(self->align, &eofMapping);
+ }
+ if (d->rc == 0) {
+ switch (ctx) {
+ case eCtxRead:
+ if (!eofLfr) {
+ /* not EOF */
+ d->rc = RC(rcExe, rcFile, rcReading, rcData, rcUnexpected);
+ CGLoaderFile_LOG(self->align, klogErr, d->rc,
+ "extra tag LFRs, possible that corresponding "
+ "reads file is truncated", NULL);
+ }
+ else if (!eofMapping) {
+ /* not EOF */
+ d->rc = RC(rcExe, rcFile, rcReading, rcData, rcUnexpected);
+ CGLoaderFile_LOG(self->align, klogErr, d->rc,
+ "extra mappings, possible that corresponding "
+ "reads file is truncated", NULL);
+ }
+ break;
+ case eCtxLfr:
+ case eCtxMapping:
+ d->rc = RC(rcExe, rcFile, rcReading, rcCondition, rcInvalid);
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ }
+ if (d->rc == 0) {
+ /* mappings and lfr file EOF detected ok */
+ DEBUG_MSG(5, (" done\n", FGroupKey_Validate(&self->key)));
+ }
+ return true;
+}
+
bool CC FGroupMAP_LoadReads( BSTNode *node, void *data )
{
+ TCtx ctx = eCtxRead;
FGroupMAP* n = (FGroupMAP*)node;
FGroupMAP_LoadData* d = (FGroupMAP_LoadData*)data;
+ bool done = false;
+
DEBUG_MSG(5, (" started\n", FGroupKey_Validate(&n->key)));
- while( d->rc == 0 )
- {
- if( (d->rc = CGLoaderFile_GetRead(n->seq, d->db.reads)) == 0 ) {
- if( (d->db.reads->flags & (cg_eLeftHalfDnbNoMatches | cg_eLeftHalfDnbMapOverflow)) &&
- (d->db.reads->flags & (cg_eRightHalfDnbNoMatches | cg_eRightHalfDnbMapOverflow)) ) {
+ while (!done && d->rc == 0) {
+ ctx = eCtxRead;
+ d->rc = CGLoaderFile_GetRead(n->seq, d->db.reads);
+ if (d->rc == 0 && n->tagLfr != NULL) {
+ ctx = eCtxLfr;
+ d->rc = CGLoaderFile_GetTagLfr(n->tagLfr, d->db.reads);
+ }
+ if (d->rc == 0) {
+ if ((d->db.reads->flags
+ & (cg_eLeftHalfDnbNoMatches | cg_eLeftHalfDnbMapOverflow))
+ &&
+ (d->db.reads->flags
+ & (cg_eRightHalfDnbNoMatches | cg_eRightHalfDnbMapOverflow)))
+ {
d->db.mappings->map_qty = 0;
} else {
+ ctx = eCtxMapping;
d->rc = CGLoaderFile_GetMapping(n->align, d->db.mappings);
}
- /* alignment written 1st than sequence -> primary_alignment_id must be set!! */
- if( d->rc == 0 && (d->rc = CGWriterAlgn_Write(d->db.walgn, d->db.reads)) == 0 ) {
+/* alignment written 1st than sequence -> primary_alignment_id must be set!! */
+ if (d->rc == 0 &&
+ (d->rc = CGWriterAlgn_Write(d->db.walgn, d->db.reads)) == 0)
+ {
d->rc = CGWriterSeq_Write(d->db.wseq);
}
}
- if( GetRCState(d->rc) == rcDone && GetRCObject(d->rc) == rcData ) {
- bool eof = false;
- d->rc = 0;
- if( n->align == NULL || ((d->rc = CGLoaderFile_IsEof(n->align, &eof)) == 0 && eof) ) {
- /* mappings file EOF detected ok */
- DEBUG_MSG(5, (" done\n", FGroupKey_Validate(&n->key)));
- break;
- } else if( d->rc == 0 ) {
- /* not EOF */
- d->rc = RC(rcExe, rcFile, rcReading, rcData, rcUnexpected);
- CGLoaderFile_LOG(n->align, klogErr, d->rc,
- "extra mappings, possible that corresponding reads file is truncated", NULL);
- }
- }
+ done = _FGroupMAPDone(n, ctx, d);
d->rc = d->rc ? d->rc : Quitting();
}
if( d->rc != 0 ) {
CGLoaderFile_LOG(n->seq, klogErr, d->rc, NULL, NULL);
CGLoaderFile_LOG(n->align, klogErr, d->rc, NULL, NULL);
}
+
FGroupMAP_CloseFiles(n);
return d->rc != 0;
}
@@ -1105,8 +1181,8 @@ rc_t CC KMain( int argc, char* argv[] )
}
else
{
- PLOGERR(klogErr, (klogErr, rc, "load failed: $(reason_short)",
- "severity=total,status=failure,accession=%s", refseq_chunk));
+ PLOGERR(klogErr, (klogErr, rc, "load failed: $(accession)",
+ "severity=total,status=failure,accession=%s", refseq_chunk));
}
}
else
diff --git a/tools/cg-load/cg-load.vers b/tools/cg-load/cg-load.vers
index 0bee604..3f684d2 100644
--- a/tools/cg-load/cg-load.vers
+++ b/tools/cg-load/cg-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/cg-load/defs.h b/tools/cg-load/defs.h
index 9e06969..d983d9b 100644
--- a/tools/cg-load/defs.h
+++ b/tools/cg-load/defs.h
@@ -42,6 +42,7 @@
#define CG_EVDNC_PLOIDY (3)
#define CG_EVDNC_INTERVALID_LEN (32)
#define CG_EVDNC_SPOT_LEN (10 * 1024)
+#define CG_TAG_LFR_DATA_LEN (10)
/* CG_EVDNC_ALLELE_NUM is 3 in v.1.5; 4 in v.2 */
#define CG_EVDNC_ALLELE_NUM (4)
@@ -73,6 +74,7 @@ typedef uint16_t CGFIELD_WINDOW_SHIFT_TYPE;
typedef uint16_t CGFIELD_WINDOW_WIDTH_TYPE;
typedef uint16_t CGFIELD_NUMBER_LEVELS_TYPE;
typedef uint16_t CGFIELD_MEAN_LEVEL_X_TYPE;
+typedef uint16_t CGFIELD_WELL_ID;
typedef enum CG_EFileType_enum {
cg_eFileType_Unknown = 0,
@@ -115,6 +117,8 @@ typedef enum CG_EFileType_enum {
cg_eFileType_COVERAGE_BY_GC_CODING,
cg_eFileType_LIB_SMALL_GAPS_ROLLUP,
cg_eFileType_DIPLOID_SOMATIC_CNV_DETAILSCORES,
+
+ cg_eFileType_TAG_LFR,
cg_eFileType_Last
} CG_EFileType;
diff --git a/tools/cg-load/f1_5.c b/tools/cg-load/f1_5.c
index dcb152c..f766a8b 100644
--- a/tools/cg-load/f1_5.c
+++ b/tools/cg-load/f1_5.c
@@ -23,10 +23,11 @@
* ===========================================================================
*/
-#include "factory-reads.h"
-#include "factory-mappings.h"
-#include "factory-evidence-intervals.h"
#include "factory-evidence-dnbs.h"
+#include "factory-evidence-intervals.h"
+#include "factory-mappings.h"
+#include "factory-reads.h"
+#include "factory-tag-lfr.h"
#include "file.h"
@@ -63,8 +64,9 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "JUNCTIONS", cg_eFileType_JUNCTIONS, NULL },
{ "JUNCTION-DNBS", cg_eFileType_JUNCTION_DNBS, NULL },
/* undocumented */
- { "VAR-OLPL", cg_eFileType_VAR_OLPL, NULL }
+ { "VAR-OLPL", cg_eFileType_VAR_OLPL, NULL },
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
};
rc_t CGFile15_Make(const CGFileType** self,
diff --git a/tools/cg-load/f2_0.c b/tools/cg-load/f2_0.c
index ba28f44..7af4181 100644
--- a/tools/cg-load/f2_0.c
+++ b/tools/cg-load/f2_0.c
@@ -23,10 +23,11 @@
* ===========================================================================
*/
-#include "factory-reads.h"
-#include "factory-mappings.h"
-#include "factory-evidence-intervals.h"
#include "factory-evidence-dnbs.h"
+#include "factory-evidence-intervals.h"
+#include "factory-mappings.h"
+#include "factory-reads.h"
+#include "factory-tag-lfr.h"
#include "file.h"
@@ -68,6 +69,8 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
{ "SV-EVENTS", cg_eFileType_SV_EVENTS, NULL },
{ "VAR-OLPL", cg_eFileType_VAR_OLPL, NULL },
{ "MEI", cg_eFileType_MEI, NULL },
+
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
};
rc_t CGFile20_Make(const CGFileType** self,
diff --git a/tools/cg-load/f2_2.c b/tools/cg-load/f2_2.c
index 1d4986d..02c84c1 100644
--- a/tools/cg-load/f2_2.c
+++ b/tools/cg-load/f2_2.c
@@ -23,10 +23,11 @@
* ===========================================================================
*/
-#include "factory-reads.h"
-#include "factory-mappings.h"
-#include "factory-evidence-intervals.h"
#include "factory-evidence-dnbs.h"
+#include "factory-evidence-intervals.h"
+#include "factory-mappings.h"
+#include "factory-reads.h"
+#include "factory-tag-lfr.h"
#include "file.h"
@@ -84,6 +85,8 @@ static const CGFileTypeFactory cg_ETypeXX_names[] = {
/* from Cancer Sequencing Service Data File Formats file test */
{ "DIPLOID-SOMATIC-CNV-DETAILSCORES",
cg_eFileType_DIPLOID_SOMATIC_CNV_DETAILSCORES, NULL },
+
+ { "TAG_LFR", cg_eFileType_TAG_LFR, CGTagLfr_Make },
};
rc_t CGFile22_Make(const CGFileType** self,
diff --git a/tools/cg-load/factory-cmn.h b/tools/cg-load/factory-cmn.h
index a2eaaaa..d86258e 100644
--- a/tools/cg-load/factory-cmn.h
+++ b/tools/cg-load/factory-cmn.h
@@ -34,7 +34,7 @@ typedef CGFIELD_SAMPLE_TYPE CGFIELD15_SAMPLE[32];
typedef CGFIELD_SLIDE_TYPE CGFIELD15_SLIDE[CG_SLIDE];
typedef CGFIELD_LANE_TYPE CGFIELD15_LANE[CG_LANE];
typedef CGFIELD_CHROMOSOME_TYPE CGFIELD15_CHROMOSOME[CG_CHROMOSOME_NAME];
-typedef CGFIELD_ASSEMBLY_ID_TYPE CGFIELD15_ASSEMBLY_ID[32];
+typedef CGFIELD_ASSEMBLY_ID_TYPE CGFIELD15_ASSEMBLY_ID[44];
/* VDB-951: make it process invalid #SOFTWARE_VERSION value
typedef CGFIELD_SOFTWARE_VERSION_TYPE CGFIELD15_SOFTWARE_VERSION[16];*/
diff --git a/tools/cg-load/factory-evidence-dnbs.c b/tools/cg-load/factory-evidence-dnbs.c
index a4af12b..c85c810 100644
--- a/tools/cg-load/factory-evidence-dnbs.c
+++ b/tools/cg-load/factory-evidence-dnbs.c
@@ -283,6 +283,7 @@ static const CGFileType_vt CGEvidenceDnbs15_vt =
NULL,
NULL,
CGEvidenceDnbs15_Read,
+ NULL, /* tag_lfr */
CGEvidenceDnbs15_GetAssemblyId,
NULL,
NULL,
@@ -300,6 +301,7 @@ static const CGFileType_vt CGEvidenceDnbs20_vt =
NULL,
NULL,
CGEvidenceDnbs20_Read,
+ NULL, /* tag_lfr */
CGEvidenceDnbs15_GetAssemblyId,
NULL,
NULL,
diff --git a/tools/cg-load/factory-evidence-intervals.c b/tools/cg-load/factory-evidence-intervals.c
index 1705bdc..f672084 100644
--- a/tools/cg-load/factory-evidence-intervals.c
+++ b/tools/cg-load/factory-evidence-intervals.c
@@ -234,6 +234,7 @@ static const CGFileType_vt CGEvidenceIntervals15_vt =
NULL,
CGEvidenceIntervals15_Read,
NULL,
+ NULL, /* tag_lfr */
CGEvidenceIntervals15_GetAssemblyId,
NULL,
NULL,
@@ -251,6 +252,7 @@ static const CGFileType_vt CGEvidenceIntervals20_vt =
NULL,
CGEvidenceIntervals20_Read,
NULL,
+ NULL, /* tag_lfr */
CGEvidenceIntervals15_GetAssemblyId,
NULL,
NULL,
diff --git a/tools/cg-load/factory-mappings.c b/tools/cg-load/factory-mappings.c
index 43537f2..6e00f08 100644
--- a/tools/cg-load/factory-mappings.c
+++ b/tools/cg-load/factory-mappings.c
@@ -272,6 +272,7 @@ static const CGFileType_vt CGMappings15_vt =
CGMappings15_Read,
NULL,
NULL,
+ NULL, /* tag_lfr */
CGMappings15_GetAssemblyId,
CGMappings15_GetSlide,
CGMappings15_GetLane,
@@ -289,6 +290,7 @@ static const CGFileType_vt CGMappings22_vt =
CGMappings22_Read,
NULL,
NULL,
+ NULL, /* tag_lfr */
CGMappings15_GetAssemblyId,
CGMappings15_GetSlide,
CGMappings15_GetLane,
diff --git a/tools/cg-load/factory-reads.c b/tools/cg-load/factory-reads.c
index 8b39f9a..e0fa92b 100644
--- a/tools/cg-load/factory-reads.c
+++ b/tools/cg-load/factory-reads.c
@@ -216,6 +216,7 @@ static const CGFileType_vt CGReads15_vt =
NULL,
NULL,
NULL,
+ NULL, /* tag_lfr */
CGReads15_GetAssemblyId,
CGReads15_GetSlide,
CGReads15_GetLane,
diff --git a/tools/cg-load/file-version-factory.c b/tools/cg-load/file-version-factory.c
index 55372f8..7304c0e 100644
--- a/tools/cg-load/file-version-factory.c
+++ b/tools/cg-load/file-version-factory.c
@@ -93,6 +93,7 @@ static const CGFileType_vt CGIgnored_vt =
NULL,
NULL,
NULL,
+ NULL, /* tag_lfr */
NULL,
NULL,
NULL,
diff --git a/tools/cg-load/file.c b/tools/cg-load/file.c
index 0e9a272..1294701 100644
--- a/tools/cg-load/file.c
+++ b/tools/cg-load/file.c
@@ -254,6 +254,22 @@ rc_t CGLoaderFile_GetRead(const CGLoaderFile* cself, TReadsData* data)
return rc;
}
+rc_t CGLoaderFile_GetTagLfr(const CGLoaderFile* cself, TReadsData* data)
+{
+ rc_t rc = 0;
+
+ if( cself == NULL || data == NULL ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcParam, rcNull);
+ } else if( cself->cg_file->type != cg_eFileType_TAG_LFR ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcInterface, rcUnsupported);
+ } else if( cself->cg_file->vt->tag_lfr == NULL ) {
+ rc = RC(rcRuntime, rcFile, rcReading, rcInterface, rcIncomplete);
+ } else if( (rc = CGLoaderFile_header(cself)) == 0 ) {
+ rc = cself->cg_file->vt->tag_lfr(cself->cg_file, data);
+ }
+ return rc;
+}
+
rc_t CGLoaderFile_GetStartRow(const CGLoaderFile* cself, int64_t* rowid)
{
rc_t rc = 0;
diff --git a/tools/cg-load/file.h b/tools/cg-load/file.h
index d9bc44a..1e750f3 100644
--- a/tools/cg-load/file.h
+++ b/tools/cg-load/file.h
@@ -238,6 +238,7 @@ typedef struct CGFileType_vt_struct {
rc_t ( CC *mappings ) (const CGFILETYPE_IMPL* cself, TMappingsData* data);
rc_t ( CC *evidence_intervals )(const CGFILETYPE_IMPL* cself, TEvidenceIntervalsData* data);
rc_t ( CC *evidence_dnbs )(const CGFILETYPE_IMPL* cself, const char* interval_id, TEvidenceDnbsData* data);
+ rc_t ( CC *tag_lfr )(const CGFILETYPE_IMPL* cself, TReadsData* data);
rc_t ( CC *assembly_id) (const CGFILETYPE_IMPL* self, const CGFIELD_ASSEMBLY_ID_TYPE** assembly_id);
rc_t ( CC *slide) (const CGFILETYPE_IMPL* self, const CGFIELD_SLIDE_TYPE** slide);
@@ -291,6 +292,8 @@ rc_t CGLoaderFile_GetType(const CGLoaderFile* cself, CG_EFileType* type);
rc_t CGLoaderFile_GetRead(const CGLoaderFile* cself, TReadsData* data);
rc_t CGLoaderFile_GetStartRow(const CGLoaderFile* cself, int64_t* rowid);
+rc_t CGLoaderFile_GetTagLfr(const CGLoaderFile* cself, TReadsData* data);
+
rc_t CGLoaderFile_GetMapping(const CGLoaderFile* cself, TMappingsData* data);
rc_t CGLoaderFile_GetEvidenceIntervals(const CGLoaderFile* cself, TEvidenceIntervalsData* data);
diff --git a/tools/fastq-loader/Makefile b/tools/fastq-loader/Makefile
index 8f7c286..31c50f0 100644
--- a/tools/fastq-loader/Makefile
+++ b/tools/fastq-loader/Makefile
@@ -93,14 +93,6 @@ clean: stdclean
.PRECIOUS: $(INTERM_SRC)
#-------------------------------------------------------------------------------
-# tag
-#
-tag: \
- $(addsuffix _tag,$(ALL_LIBS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_LIBS))
-
-#-------------------------------------------------------------------------------
# fastq
#
$(ILIBDIR)/libfastqloader: $(ILIBDIR)/libfastqloader.$(LIBX)
diff --git a/tools/fastq-loader/fastq-lex.c b/tools/fastq-loader/fastq-lex.c
index 970d1c6..e31d4bd 100644
--- a/tools/fastq-loader/fastq-lex.c
+++ b/tools/fastq-loader/fastq-lex.c
@@ -1,5 +1,6 @@
+#define YY_BUF_SIZE 16777216
#define YY_INT_ALIGNED short int
/* A lexical scanner generated by flex */
@@ -596,22 +597,23 @@ goto find_rule; \
* ===========================================================================
*
*/
- #include <sysalloc.h>
+ #include <sysalloc.h>
#include <ctype.h>
#include <klib/rc.h>
+ #include <klib/log.h>
- #include "fastq-parse.h"
- #include "fastq-tokens.h"
-
+ #include "fastq-parse.h"
+ #include "fastq-tokens.h"
+
#define YYSTYPE FASTQToken
/* code to populate token with internal values */
#define YY_USER_ACTION \
- yylval -> tokenText = yytext; \
- yylval -> tokenLength = yyleng; \
- yylval -> line_no = yylineno; \
- yylval -> column_no = yyextra -> column; \
+ yylval -> tokenText = yytext; \
+ yylval -> tokenLength = yyleng; \
+ yylval -> line_no = yylineno; \
+ yylval -> column_no = yyextra -> column; \
yyextra -> column += yyleng; \
yyextra -> length += yyleng; \
yyextra -> lastToken = yylval;
@@ -625,6 +627,9 @@ goto find_rule; \
#define ECHO
+ static void FASTQ_fatal_error(yyconst char* msg);
+ #define YY_FATAL_ERROR(msg) FASTQ_fatal_error(msg)
+
#define YY_NO_UNISTD_H 1
/*%option debug*/
@@ -1057,7 +1062,7 @@ YY_RULE_SETUP
case 10:
/* rule 10 can match eol */
YY_RULE_SETUP
-{ BEGIN IN_SEQUENCE; ENDLINE; }
+{ BEGIN IN_SEQUENCE; ENDLINE; }
YY_BREAK
case 11:
@@ -2356,10 +2361,10 @@ void FASTQ_free (void * ptr , yyscan_t yyscanner)
rc_t CC FASTQScan_yylex_init(FASTQParseBlock* sb, bool debug)
{
- if (FASTQ_lex_init_extra(sb,&sb->scanner) != 0)
- {
- return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );
- }
+ if (FASTQ_lex_init_extra(sb,&sb->scanner) != 0)
+ {
+ return RC ( rcApp, rcFile, rcParsing, rcMemory, rcExhausted );
+ }
sb->length = 0;
sb->lastToken = NULL;
@@ -2375,25 +2380,25 @@ rc_t CC FASTQScan_yylex_init(FASTQParseBlock* sb, bool debug)
KDataBufferMakeBytes( &sb->quality, 1024);
sb->expectedQualityLines = 0;
- FASTQ_set_debug(debug ? 1 : 0,sb->scanner);
+ FASTQ_set_debug(debug ? 1 : 0,sb->scanner);
yy_push_state(INITIAL, sb->scanner);
- return 0;
+ return 0;
}
void CC FASTQScan_yylex_destroy(FASTQParseBlock* sb)
{
KDataBufferWhack( & sb->tagLine);
KDataBufferWhack( & sb->quality);
- if (sb->scanner)
- {
- FASTQ_lex_destroy(sb->scanner);
- }
- sb->scanner=0;
+ if (sb->scanner)
+ {
+ FASTQ_lex_destroy(sb->scanner);
+ }
+ sb->scanner=0;
}
-void CC FASTQScan_inline_sequence(FASTQParseBlock* pb)
+void CC FASTQScan_inline_sequence(FASTQParseBlock* pb)
{
yy_pop_state(pb->scanner);
yy_push_state(INLINE_SEQUENCE, pb->scanner);
@@ -2417,3 +2422,12 @@ void CC FASTQ_unlex(FASTQParseBlock* pb, FASTQToken* token)
yyextra -> length -= token->tokenLength;
}
+void FASTQ_fatal_error(yyconst char* msg)
+{
+ rc_t rc = RC ( rcApp, rcFile, rcParsing, rcError, rcUnexpected );
+ if (strcmp(msg, "input buffer overflow, can't enlarge buffer because scanner uses REJECT") == 0)
+ pLogErr(klogErr, rc, "line is too long (maximum line length = $(B) bytes)", "B=%d", YY_BUF_SIZE);
+ else
+ LogErr(klogErr, rc, msg);
+ exit(rc);
+}
diff --git a/tools/fastq-loader/latf-load.vers b/tools/fastq-loader/latf-load.vers
index 0bee604..3f684d2 100644
--- a/tools/fastq-loader/latf-load.vers
+++ b/tools/fastq-loader/latf-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/kar/Makefile b/tools/kar/Makefile
index 5f2ea7a..04fc22f 100644
--- a/tools/kar/Makefile
+++ b/tools/kar/Makefile
@@ -73,15 +73,6 @@ clean: stdclean
.PHONY: clean
-#-------------------------------------------------------------------------------
-# tag
-#
-tag: \
- $(addsuffix _tag,$(ALL_TOOLS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_TOOLS))
-
-
#----------------------------------------------------------------
# kar
#
@@ -93,21 +84,13 @@ KAR_OBJ = \
$(addsuffix .$(OBJX),$(KAR_SRC))
KAR_LIB = \
- -skapp \
- -lvfs \
- -lkurl \
- -lkfg \
- -lkrypto \
- -lkfs \
- -lkproc \
- -lklib \
- -lm
+ -lkapp \
+ -lncbi-vdb \
+
$(BINDIR)/kar: $(KAR_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(KAR_LIB)
-kar_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) kar $(KAR_OBJ)
#----------------------------------------------------------------
# sra-kar
@@ -122,13 +105,8 @@ SRA_KAR_OBJ = \
SRA_KAR_LIB = \
-skapp \
-lncbi-vdb \
- -lxml2 \
- -lm
$(BINDIR)/sra-kar: $(SRA_KAR_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SRA_KAR_LIB)
-sra-kar_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) sra-kar $(SRA_KAR_OBJ)
-
diff --git a/tools/kar/kar.c b/tools/kar/kar.c
index 0fadca6..3cca677 100644
--- a/tools/kar/kar.c
+++ b/tools/kar/kar.c
@@ -797,16 +797,24 @@ rc_t derive_directory_name (char ** dirname, const char * arcname, const char *
}
if (rc == 0)
{
- *dirname = malloc (len + 1);
- if (*dirname == NULL)
+ if ( len == 0 )
{
- rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
- LOGERR (klogErr, rc, "Unable to allocate memory for directory name");
+ rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcInvalid);
+ * dirname = NULL;
}
else
{
- memcpy (*dirname, srcname, len);
- (*dirname)[len] = '\0';
+ *dirname = malloc (len + 1);
+ if (*dirname == NULL)
+ {
+ rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
+ LOGERR (klogErr, rc, "Unable to allocate memory for directory name");
+ }
+ else
+ {
+ memcpy (*dirname, srcname, len);
+ (*dirname)[len] = '\0';
+ }
}
}
return rc;
@@ -826,18 +834,48 @@ rc_t run_kar_create(const char * archive, const char * directory)
rc = derive_directory_name (&directorystr, archive, directory);
- assert (directorystr != NULL);
- assert (directorystr[0] != '\0');
-
if (rc != 0)
LOGERR (klogErr, rc,"failed to derive directory name");
else
{
+ assert (directorystr != NULL);
+ assert (directorystr[0] != '\0');
+
STSMSG (4, ("start creation of archive"));
- rc = open_dir_as_archive (directorystr, &fin);
+ /* Jira ticket: SRA-1876
+ Date: September 13, 2013
+ raw usage of "directorystr" causes tests to fail within libs/kfs/arc.c */
+ {
+ char full [ 4096 ];
+ rc = KDirectoryResolvePath ( kdir, true, full, sizeof full, "%s", directorystr );
+ if ( rc == 0 )
+ {
+ /* string should be non-empty based upon behavior of
+ "derive_directory_name" ( also fixed today ) */
+ assert ( full [ 0 ] != 0 );
+
+ /* eliminate double-slashes */
+ if ( full [ 1 ] != 0 )
+ {
+ uint32_t i, j;
+
+ /* ALLOW double slashes at the very start
+ set starting index to 2 for that reason */
+ for ( i = j = 2; full [ i ] != 0; ++ i )
+ {
+ if ( ( full [ j ] = full [ i ] ) != '/' || full [ j - 1 ] != '/' )
+ ++ j;
+ }
+
+ full [ j ] = 0;
+ }
+
+ rc = open_dir_as_archive ( full, & fin );
+ }
+ }
if (rc != 0)
- PLOGERR (klogErr, (klogErr, rc,"failed to open directory [$(D)] as archive",
+ PLOGERR (klogErr, (klogErr, rc,"failed to open directory '$(D)' as archive",
PLOG_S(D),directorystr));
else
{
diff --git a/tools/kar/kar.vers b/tools/kar/kar.vers
index 0bee604..3f684d2 100644
--- a/tools/kar/kar.vers
+++ b/tools/kar/kar.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/kar/sra-kar.vers b/tools/kar/sra-kar.vers
index 0bee604..3f684d2 100644
--- a/tools/kar/sra-kar.vers
+++ b/tools/kar/sra-kar.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/nenctool/nenctest.vers b/tools/nenctool/nenctest.vers
index 0bee604..3f684d2 100644
--- a/tools/nenctool/nenctest.vers
+++ b/tools/nenctool/nenctest.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/nenctool/nenctool.vers b/tools/nenctool/nenctool.vers
index 0bee604..3f684d2 100644
--- a/tools/nenctool/nenctool.vers
+++ b/tools/nenctool/nenctool.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/nencvalid/Makefile b/tools/nencvalid/Makefile
index 2628e5e..a64466d 100644
--- a/tools/nencvalid/Makefile
+++ b/tools/nencvalid/Makefile
@@ -73,14 +73,6 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# tag
-#
-tag: \
- $(addsuffix _tag,$(ALL_TOOLS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_LIBS)) $(addsuffix _tag,$(ALL_TOOLS))
-
-#-------------------------------------------------------------------------------
# nencvalid
# Copy files from one directory to another while cataloging the contents.
# XML files can be redirected as well.
@@ -104,8 +96,6 @@ NENCVALID_LIB = \
$(BINDIR)/nencvalid: $(NENCVALID_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(NENCVALID_LIB)
-nencvalid_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) nencvalid $(NENCVALID_OBJ)
diff --git a/tools/nencvalid/nencvalid.vers b/tools/nencvalid/nencvalid.vers
index 0bee604..3f684d2 100644
--- a/tools/nencvalid/nencvalid.vers
+++ b/tools/nencvalid/nencvalid.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/prefetch/prefetch.c b/tools/prefetch/prefetch.c
index 46d99f5..eb30e1a 100644
--- a/tools/prefetch/prefetch.c
+++ b/tools/prefetch/prefetch.c
@@ -24,6 +24,8 @@
*
*/
+/********** includes **********/
+
#include "prefetch.vers.h"
#include <kapp/main.h> /* KAppVersion */
@@ -36,6 +38,8 @@
#include <vdb/dependencies.h> /* VDBDependencies */
#include <vdb/manager.h> /* VDBManager */
+#include <kdb/manager.h> /* kptDatabase */
+
#include <vfs/manager.h> /* VFSManager */
#include <vfs/path.h> /* VPath */
#include <vfs/resolver.h> /* VResolver */
@@ -83,6 +87,8 @@
#define STS_DBG 2
#define STS_FIN 3
+#define USE_CURL 1
+
#define rcResolver rcTree
static bool NotFoundByResolver(rc_t rc) {
if (GetRCModule(rc) == rcVFS) {
@@ -97,24 +103,103 @@ static bool NotFoundByResolver(rc_t rc) {
return false;
}
+typedef enum {
+ eOrderSize,
+ eOrderOrig
+} EOrder;
+typedef enum {
+ eForceNo, /* do not download found and complete objects */
+ eForceYes,/* force download of found and complete objects */
+ eForceYES /* force download; ignore lockes */
+} EForce;
+typedef enum {
+ eRunTypeUnknown,
+ eRunTypeList, /* list sizes */
+ eRunTypeDownload,
+ eRunTypeGetSize /* download ordered by sizes */
+} ERunType;
typedef struct {
const VPath *path;
const String *str;
} VPathStr;
-static rc_t VPathStrFini(VPathStr *self) {
- rc_t rc = 0;
+typedef struct {
+ BSTNode n;
+ char *path;
+} TreeNode;
+typedef struct {
+ ERunType type;
+ char *name;
- assert(self);
+ VPathStr local;
+ const String *cache;
+ const String *remote;
- VPathRelease(self->path);
+ const KFile *file;
+ uint64_t remoteSz;
- free((void*)self->str);
+ bool undersized; /* remoteSz < min allowed size */
+ bool oversized; /* remoteSz >= max allowed size */
- memset(self, 0, sizeof *self);
+ bool existing;
- return rc;
-}
+ /* path to the resolved object : either local or cache:
+ should not be released */
+ const char *path;
+
+ VPath *accession;
+ uint64_t project;
+
+ const KartItem *kartItem;
+
+ VResolver *resolver;
+} Resolved;
+typedef struct {
+ Args *args;
+ bool check_all;
+
+ bool list_kart;
+ bool list_kart_numbered;
+ bool list_kart_sized;
+ EOrder order;
+
+ const char *rows;
+
+ EForce force;
+ KConfig *cfg;
+ KDirectory *dir;
+
+ const KRepositoryMgr *repoMgr;
+ const VDBManager *mgr;
+ VFSManager *vfsMgr;
+
+ VResolver *resolver;
+
+ void *buffer;
+ size_t bsize;
+
+ bool undersized; /* remoteSz < min allowed size */
+ bool oversized; /* remoteSz >= max allowed size */
+
+ BSTree downloaded;
+
+ size_t minSize;
+ size_t maxSize;
+ uint64_t heartbeat;
+
+ bool noAscp;
+ bool noHttp;
+ bool forceAscpFail;
+
+ bool ascpChecked;
+ const char *ascp;
+ const char *asperaKey;
+ String *ascpMaxRate;
+
+#ifdef _DEBUGGING
+ const char *textkart;
+#endif
+} Main;
typedef struct {
/* "plain" command line argument */
const char *desc;
@@ -124,10 +209,25 @@ typedef struct {
#ifdef _DEBUGGING
const char *textkart;
#endif
+
+ Resolved resolved;
+ int number;
+
+ Main *main; /* just a pointer, no refcount here, don't release it */
} Item;
-static rc_t ItemInit(Item *self, const char *obj) {
- assert(self);
- self->desc = obj;
+typedef struct {
+ const char *obj;
+ bool done;
+ Kart *kart;
+ bool isKart;
+} Iterator;
+typedef struct {
+ BSTNode n;
+ Item *i;
+} KartTreeNode;
+/********** String extension **********/
+static rc_t StringRelease(const String *self) {
+ free((String*)self);
return 0;
}
@@ -140,537 +240,137 @@ static char* StringCheck(const String *self, rc_t rc) {
return NULL;
}
-static char* ItemName(const Item *self) {
- char *c = NULL;
- assert(self);
- if (self->desc != NULL) {
- return string_dup_measure(self->desc, NULL);
+static
+bool _StringIsFasp(const String *self, const char **withoutScheme)
+{
+ const char fasp[] = "fasp://";
+ const char *dummy = NULL;
+
+ assert(self && self->addr);
+
+ if (withoutScheme == NULL) {
+ withoutScheme = &dummy;
}
- else {
- rc_t rc = 0;
- const String *elem = NULL;
- assert(self->item);
- rc = KartItemItemDesc(self->item, &elem);
- c = StringCheck(elem, rc);
- if (c != NULL) {
- return c;
- }
+ *withoutScheme = NULL;
- rc = KartItemAccession(self->item, &elem);
- c = StringCheck(elem, rc);
- if (c != NULL) {
- return c;
- }
+ if (memcmp(self->addr, fasp, sizeof fasp - 1) == 0) {
+ *withoutScheme = self->addr + sizeof fasp - 1;
+ return true;
+ }
+ return false;
+}
- rc = KartItemItemId(self->item, &elem);
- return StringCheck(elem, rc);
+/********** KFile extension **********/
+static rc_t _KFileOpenRemote(const KFile **self, const char *path) {
+ rc_t rc = 0;
+ assert(self);
+ if (*self != NULL) {
+ return 0;
}
+#if USE_CURL
+ rc = KCurlFileMake(self, path, false);
+#else
+ rc = KNSManagerMakeHttpFile(kns, &file, NULL, 0x01010000, "%S", s);
+#endif
+ return rc;
}
-static rc_t V_ResolverRemote ( const VResolver * self,
- VRemoteProtocols protocols, struct VPath const * accession,
- struct VPath const ** remote, struct KFile const ** opt_file_rtn,
- struct VPath const ** cache)
+/********** KDirectory extension **********/
+static rc_t _KDirectoryMkTmpPrefix(const KDirectory *self,
+ const String *prefix, char *out, size_t sz)
{
- return VResolverQuery(self, protocols, accession, NULL, remote, cache);
+ size_t num_writ = 0;
+ rc_t rc = string_printf(out, sz, &num_writ, "%S.tmp", prefix);
+ if (rc != 0) {
+ DISP_RC2(rc, "string_printf(tmp)", prefix->addr);
+ return rc;
+ }
+
+ if (num_writ > sz) {
+ rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
+ PLOGERR(klogInt, (klogInt, rc,
+ "bad string_printf($(s).tmp) result", "s=%s", prefix->addr));
+ return rc;
+ }
+
+ return rc;
}
-static rc_t _VResolverRemote(VResolver *self, VRemoteProtocols protocols,
- const char *name, const VPath *vaccession,
- const VPath **vremote, const String **remote,
- const String **cache)
+static rc_t _KDirectoryMkTmpName(const KDirectory *self,
+ const String *prefix, char *out, size_t sz)
{
rc_t rc = 0;
- const VPath *vcache = NULL;
- assert(vaccession && vremote && !*vremote);
- rc = V_ResolverRemote(self, protocols, vaccession, vremote, NULL, &vcache);
- if (rc == 0) {
- char path[PATH_MAX] = "";
- size_t len = 0;
- rc = VPathReadUri(*vremote, path, sizeof path, &len);
- DISP_RC2(rc, "VPathReadUri(VResolverRemote)", name);
- if (rc == 0) {
- String local_str;
- char *query = string_chr(path, len, '?');
- if (query != NULL) {
- *query = '\0';
- }
- StringInit(&local_str, path, len, len);
- rc = StringCopy(remote, &local_str);
- DISP_RC2(rc, "StringCopy(VResolverRemote)", name);
+ int i = 0;
+
+ assert(prefix);
+
+ while (rc == 0) {
+ size_t num_writ = 0;
+ rc = string_printf(out, sz, &num_writ,
+ "%S.tmp.%d.tmp", prefix, rand() % 100000);
+ if (rc != 0) {
+ DISP_RC2(rc, "string_printf(tmp.rand)", prefix->addr);
+ break;
}
- }
- else if (NotFoundByResolver(rc)) {
- PLOGERR(klogErr, (klogErr, rc, "$(acc) cannot be found.",
- "acc=%s", name));
- }
- else {
- DISP_RC2(rc, "Cannot resolve remote", name);
- }
- if (rc == 0 && cache != NULL) {
- String path_str;
- if (vcache == NULL) {
- rc = RC(rcExe, rcResolver, rcResolving, rcPath, rcNotFound);
- PLOGERR(klogInt, (klogInt, rc, "cannot get cache location "
- "for $(acc). Try to cd out of protected repository.",
- "acc=%s" , name));
+
+ if (num_writ > sz) {
+ rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
+ PLOGERR(klogInt, (klogInt, rc,
+ "bad string_printf($(s).tmp.rand) result",
+ "s=%s", prefix->addr));
+ return rc;
}
- if (rc == 0) {
- rc = VPathGetPath(vcache, &path_str);
- DISP_RC2(rc, "VPathGetPath(VResolverCache)", name);
+ if (KDirectoryPathType(self, out) == kptNotFound) {
+ break;
}
- if (rc == 0) {
- rc = StringCopy(cache, &path_str);
- DISP_RC2(rc, "StringCopy(VResolverCache)", name);
+ if (++i > 999) {
+ rc = RC(rcExe, rcFile, rcCopying, rcName, rcUndefined);
+ PLOGERR(klogInt, (klogInt, rc,
+ "cannot generate unique tmp file name for $(name)", "name=%s",
+ prefix->addr));
+ return rc;
}
}
- RELEASE(VPath, vcache);
+
return rc;
}
-static rc_t V_ResolverLocal ( const VResolver * self,
- struct VPath const * accession, struct VPath const ** path )
+static rc_t _KDirectoryMkLockName(const KDirectory *self,
+ const String *prefix, char *out, size_t sz)
{
- return VResolverQuery(self, eProtocolHttp, accession, path, NULL, NULL);
+ rc_t rc = 0;
+ size_t num_writ = 0;
+
+ assert(prefix);
+
+ rc = string_printf(out, sz, &num_writ, "%S.lock", prefix);
+ DISP_RC2(rc, "string_printf(lock)", prefix->addr);
+
+ if (rc == 0 && num_writ > sz) {
+ rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
+ PLOGERR(klogInt, (klogInt, rc,
+ "bad string_printf($(s).lock) result", "s=%s", prefix->addr));
+ return rc;
+ }
+
+ return rc;
}
-static rc_t _VResolverResolve(VResolver *self, VRemoteProtocols protocols,
- const Item *item, VPathStr *local, const String **remote,
- const String **cache, const KFile **file, VPath **vaccession,
- uint64_t *project, const KRepositoryMgr *repoMgr, const KConfig *cfg,
- const char *name)
+static
+rc_t _KDirectoryCleanCache(KDirectory *self, const String *local)
{
- VResolver *resolver = NULL;
- const char *accession = NULL;
rc_t rc = 0;
- rc_t rc2 = 0;
- VFSManager *vfs = NULL;
-
- const VPath **vlocal = NULL;
- const VPath *vremote = NULL;
- if (item == NULL) {
- return RC(rcExe, rcResolver, rcResolving, rcParam, rcNull);
- }
- if (self == NULL) {
- return RC(rcExe, rcResolver, rcResolving, rcSelf, rcNull);
- }
- if (local != NULL) {
- memset(local, 0, sizeof *local);
- vlocal = &local->path;
- }
+ char cache[PATH_MAX] = "";
+ size_t num_writ = 0;
- assert(vaccession && !*vaccession && project && !*project);
+ assert(self && local && local->addr);
if (rc == 0) {
- rc = VFSManagerMake(&vfs);
- }
-
- accession = item->desc;
- if (accession == NULL) {
- uint64_t oid = 0;
- rc = KartItemProjIdNumber(item->item, project);
- if (rc != 0) {
- DISP_RC(rc, "KartItemProjIdNumber");
- return rc;
- }
- rc = KartItemItemIdNumber(item->item, &oid);
- if (rc != 0) {
- DISP_RC(rc, "KartItemItemIdNumber");
- return rc;
- }
- {
- const KRepository *p_protected = NULL;
- rc = KRepositoryMgrGetProtectedRepository(repoMgr,
- *project, &p_protected);
- if (rc == 0) {
- rc = KRepositoryMakeResolver(p_protected, &resolver, cfg);
- if (rc == 0) {
- self = resolver;
- }
- }
- RELEASE(KRepository, p_protected);
- }
- rc = VFSManagerMakeOidPath(vfs, vaccession, oid);
- }
- else {
- rc = VFSManagerMakePath(vfs, vaccession, accession);
- DISP_RC2(rc, "VFSManagerMakePath", accession);
- }
-
- RELEASE(VFSManager, vfs);
-
- if (rc == 0 && local != NULL) {
- rc = V_ResolverLocal(self, *vaccession, vlocal);
- if (rc == 0) {
- rc = VPathMakeString(*vlocal, &local->str);
- DISP_RC2(rc, "VPathMakeString(VResolverLocal)", name);
- }
- else if (NotFoundByResolver(rc)) {
- rc = 0;
- }
- else {
- DISP_RC2(rc, "VResolverLocal", name);
- }
- }
-
- if (remote != NULL) {
- rc2 = _VResolverRemote(self, protocols, name,
- *vaccession, &vremote, remote, cache);
- if (rc2 != 0 && rc == 0) {
- rc = rc2;
- }
- }
-
- RELEASE(VPath, vremote);
- RELEASE(VResolver, resolver);
- return rc;
-}
-
-typedef enum {
- eForceNo, /* do not download found and complete objects */
- eForceYes,/* force download of found and complete objects */
- eForceYES /* force download; ignore lockes */
-} Force;
-typedef struct {
- BSTNode n;
- char *path;
-} TreeNode;
-typedef struct {
- Args *args;
- bool check_all;
- bool list_kart;
- Force force;
- KConfig *cfg;
- const VDBManager *mgr;
- KDirectory *dir;
- const KRepositoryMgr *repoMgr;
- VResolver *resolver;
-
- void *buffer;
- size_t bsize;
-
- BSTree downloaded;
-
- uint64_t heartbeat;
-
- bool noAscp;
- bool noHttp;
-
- bool forceAscpFail;
-
- bool ascpChecked;
- const char *ascp;
- const char *asperaKey;
- String *ascpMaxRate;
-
-#ifdef _DEBUGGING
- const char *textkart;
-#endif
-} Main;
-static rc_t StringRelease(const String *self) {
- free((String*)self);
- return 0;
-}
-
-typedef struct {
- VPathStr local;
- const String *cache;
- const String *remote;
-
- const KFile *file;
- uint64_t remoteSz;
-
- bool existing;
-
- /* path to the resolved object : either local or cache:
- should not be released */
- const char *path;
-
- char *name;
- VPath *accession;
- uint64_t project;
-} Resolved;
-static rc_t ResolvedFini(Resolved *self) {
- rc_t rc = 0;
-
- assert(self);
-
- rc = VPathStrFini(&self->local);
-
- RELEASE(KFile, self->file);
- RELEASE(VPath, self->accession);
-
- RELEASE(String, self->remote);
- RELEASE(String, self->cache);
-
- free(self->name);
-
- memset(self, 0, sizeof *self);
-
- return rc;
-}
-
-static rc_t ResolvedInit(Resolved *self,
- const Item *item, VResolver *resolver, KDirectory *dir, bool ascp,
- const KRepositoryMgr *repoMgr, const KConfig *cfg)
-{
- rc_t rc = 0;
- KPathType type = kptNotFound;
- VRemoteProtocols protocols = ascp ? eProtocolFaspHttp : eProtocolHttp;
-
- assert(self && item);
-
- memset(self, 0, sizeof *self);
-
- self->name = ItemName(item);
-
- type = KDirectoryPathType(dir, item->desc) & ~kptAlias;
- if (type == kptFile || type == kptDir) {
- self->path = item->desc;
- self->existing = true;
- return 0;
- }
-
- rc = _VResolverResolve(resolver, protocols, item, &self->local,
- &self->remote, &self->cache, &self->file, &self->accession,
- &self->project, repoMgr, cfg, self->name);
-
- if (rc == 0 && self->file != NULL) {
- rc_t rc2 = KFileSize(self->file, &self->remoteSz);
- DISP_RC2(rc2, "KFileSize(remote)", self->name);
- }
-
-
- STSMSG(STS_DBG, ("Resolve(%s) = %R:", self->name, rc));
- STSMSG(STS_DBG, ("local(%s)",
- self->local.str ? self->local.str->addr : "NULL"));
- STSMSG(STS_DBG, ("cache(%s)", self->cache ? self->cache->addr : "NULL"));
- STSMSG(STS_DBG, ("remote(%s:%,ld)",
- self->remote ? self->remote->addr : "NULL", self->remoteSz));
-
- if (rc == 0) {
- if (self->local.str == NULL
- && (self->cache == NULL || self->remote == NULL))
- {
- rc = RC(rcExe, rcPath, rcValidating, rcParam, rcNull);
- PLOGERR(klogInt, (klogInt, rc,
- "bad VResolverResolve($(acc)) result", "acc=%s", self->name));
- }
- }
-
- return rc;
-}
-
-static bool _StringIsFasp(const String *self, const char **withoutScheme) {
- const char fasp[] = "fasp://";
- const char *dummy = NULL;
-
- assert(self && self->addr);
-
- if (withoutScheme == NULL) {
- withoutScheme = &dummy;
- }
-
- *withoutScheme = NULL;
-
- if (memcmp(self->addr, fasp, sizeof fasp - 1) == 0) {
- *withoutScheme = self->addr + sizeof fasp - 1;
- return true;
- }
- return false;
-}
-
-/** isLocal is set to true when the object is found locally.
- i.e. does not need need not be [re]downloaded */
-static rc_t ResolvedLocal(const Resolved *self,
- const KDirectory *dir, bool *isLocal, Force force)
-{
- rc_t rc = 0;
- uint64_t sRemote = 0;
- uint64_t sLocal = 0;
- const KFile *local = NULL;
- char path[PATH_MAX] = "";
-
- assert(isLocal && self);
-
- *isLocal = false;
-
- if (self->local.str == NULL) {
- return 0;
- }
-
- rc = VPathReadPath(self->local.path, path, sizeof path, NULL);
- DISP_RC(rc, "VPathReadPath");
-
- if (rc == 0 && KDirectoryPathType(dir, path) != kptFile) {
- if (force == eForceNo) {
- STSMSG(STS_TOP,
- ("%s (not a file) is found locally: consider it complete",
- path));
- *isLocal = true;
- }
- else {
- STSMSG(STS_TOP,
- ("%s (not a file) is found locally and will be redownloaded",
- path));
- }
- return 0;
- }
-
- if (rc == 0 && ! _StringIsFasp(self->remote, NULL) && self->file != NULL) {
- rc = KFileSize(self->file, &sRemote);
- DISP_RC2(rc, "KFileSize(remote)", self->name);
- }
-
- if (rc == 0) {
- rc = KDirectoryOpenFileRead(dir, &local, path);
- DISP_RC2(rc, "KDirectoryOpenFileRead", path);
- }
-
- if (rc == 0) {
- rc = KFileSize(local, &sLocal);
- DISP_RC2(rc, "KFileSize", path);
- }
-
- if (rc == 0) {
- if (sRemote == 0) {
- if (sLocal != 0) {
- if (force == eForceNo) {
- *isLocal = true;
- STSMSG(STS_INFO, ("%s (%,lu) is found", path, sLocal));
- }
- else {
- STSMSG(STS_INFO,
- ("%s (%,lu) is found and will be redownloaded",
- path, sLocal));
- }
- }
- else if (sLocal == 0) {
- STSMSG(STS_INFO,
- ("an empty %s (%,lu) is found and will be redownloaded",
- path, sLocal));
- }
- }
- else if (sRemote == sLocal) {
- if (force == eForceNo) {
- *isLocal = true;
- STSMSG(STS_INFO, ("%s (%,lu) is found and is complete",
- path, sLocal));
- }
- else {
- STSMSG(STS_INFO, ("%s (%,lu) is found and will be redownloaded",
- path, sLocal));
- }
- }
- else {
- STSMSG(STS_TOP, ("%s (%,lu) is incomplete. Expected size is %,lu. "
- "It will be re-downloaded", path, sLocal, sRemote));
- }
- }
-
- RELEASE(KFile, local);
-
- return rc;
-}
-
-static rc_t _KDirectoryMkTmpPrefix(const KDirectory *self,
- const String *prefix, char *out, size_t sz)
-{
- size_t num_writ = 0;
- rc_t rc = string_printf(out, sz, &num_writ,
- "%.*s.tmp", prefix->size, prefix->addr);
- if (rc != 0) {
- DISP_RC2(rc, "string_printf(tmp)", prefix->addr);
- return rc;
- }
-
- if (num_writ > sz) {
- rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
- PLOGERR(klogInt, (klogInt, rc,
- "bad string_printf($(s).tmp) result", "s=%s", prefix->addr));
- return rc;
- }
-
- return rc;
-}
-
-static rc_t _KDirectoryMkTmpName(const KDirectory *self,
- const String *prefix, char *out, size_t sz)
-{
- rc_t rc = 0;
- int i = 0;
-
- assert(prefix);
-
- while (rc == 0) {
- size_t num_writ = 0;
- rc = string_printf(out, sz, &num_writ,
- "%.*s.tmp.%d.tmp", prefix->size, prefix->addr, rand() % 100000);
- if (rc != 0) {
- DISP_RC2(rc, "string_printf(tmp.rand)", prefix->addr);
- break;
- }
-
- if (num_writ > sz) {
- rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
- PLOGERR(klogInt, (klogInt, rc,
- "bad string_printf($(s).tmp.rand) result",
- "s=%s", prefix->addr));
- return rc;
- }
- if (KDirectoryPathType(self, out) == kptNotFound) {
- break;
- }
- if (++i > 999) {
- rc = RC(rcExe, rcFile, rcCopying, rcName, rcUndefined);
- PLOGERR(klogInt, (klogInt, rc,
- "cannot generate unique tmp file name for $(name)", "name=%s",
- prefix->addr));
- return rc;
- }
- }
-
- return rc;
-}
-
-static rc_t _KDirectoryMkLockName(const KDirectory *self,
- const String *prefix, char *out, size_t sz)
-{
- rc_t rc = 0;
- size_t num_writ = 0;
-
- assert(prefix);
-
- rc = string_printf(out, sz, &num_writ,
- "%.*s.lock", prefix->size, prefix->addr);
- DISP_RC2(rc, "string_printf(lock)", prefix->addr);
-
- if (rc == 0 && num_writ > sz) {
- rc = RC(rcExe, rcFile, rcCopying, rcBuffer, rcInsufficient);
- PLOGERR(klogInt, (klogInt, rc,
- "bad string_printf($(s).lock) result", "s=%s", prefix->addr));
- return rc;
- }
-
- return rc;
-}
-
-static
-rc_t _KDirectoryCleanCache(KDirectory *self, const String *local)
-{
- rc_t rc = 0;
-
- char cache[PATH_MAX] = "";
- size_t num_writ = 0;
-
- assert(self && local && local->addr);
-
- if (rc == 0) {
- rc = string_printf(cache, sizeof cache, &num_writ, "%s.cache",
- local->addr);
- DISP_RC2(rc, "string_printf(.cache)", local->addr);
+ rc = string_printf(cache, sizeof cache, &num_writ, "%s.cache",
+ local->addr);
+ DISP_RC2(rc, "string_printf(.cache)", local->addr);
}
if (rc == 0 && KDirectoryPathType(self, cache) != kptNotFound) {
@@ -786,6 +486,90 @@ static rc_t _KDirectoryClean(KDirectory *self, const String *cache,
return rc;
}
+/********** VResolver extension **********/
+static rc_t V_ResolverRemote(const VResolver *self,
+ VRemoteProtocols protocols, struct VPath const * accession,
+ struct VPath const ** remote, struct VPath const ** cache)
+{
+ return VResolverQuery(self, protocols, accession, NULL, remote, cache);
+}
+
+static rc_t V_ResolverLocal(const VResolver *self,
+ struct VPath const * accession, struct VPath const ** path )
+{
+ return VResolverQuery(self, eProtocolHttp, accession, path, NULL, NULL);
+}
+
+static rc_t _VResolverRemote(VResolver *self, VRemoteProtocols protocols,
+ const char *name, const VPath *vaccession,
+ const VPath **vremote, const String **remote,
+ const String **cache)
+{
+ rc_t rc = 0;
+ const VPath *vcache = NULL;
+ assert(vaccession && vremote && !*vremote);
+ rc = V_ResolverRemote(self, protocols, vaccession, vremote, &vcache);
+ if (rc == 0) {
+ char path[PATH_MAX] = "";
+ size_t len = 0;
+ rc = VPathReadUri(*vremote, path, sizeof path, &len);
+ DISP_RC2(rc, "VPathReadUri(VResolverRemote)", name);
+ if (rc == 0) {
+ String local_str;
+ char *query = string_chr(path, len, '?');
+ if (query != NULL) {
+ *query = '\0';
+ }
+ StringInit(&local_str, path, len, (uint32_t)len);
+ RELEASE(String, *remote);
+ rc = StringCopy(remote, &local_str);
+ DISP_RC2(rc, "StringCopy(VResolverRemote)", name);
+ }
+ }
+ else if (NotFoundByResolver(rc)) {
+ PLOGERR(klogErr, (klogErr, rc, "'$(acc)' cannot be found.",
+ "acc=%s", name));
+ }
+ else {
+ DISP_RC2(rc, "Cannot resolve remote", name);
+ }
+ if (rc == 0 && cache != NULL) {
+ String path_str;
+ if (vcache == NULL) {
+ rc = RC(rcExe, rcResolver, rcResolving, rcPath, rcNotFound);
+ PLOGERR(klogInt, (klogInt, rc, "cannot get cache location "
+ "for $(acc). Try to cd out of protected repository.",
+ "acc=%s" , name));
+ }
+ if (rc == 0) {
+ rc = VPathGetPath(vcache, &path_str);
+ DISP_RC2(rc, "VPathGetPath(VResolverCache)", name);
+ }
+ if (rc == 0) {
+ rc = StringCopy(cache, &path_str);
+ DISP_RC2(rc, "StringCopy(VResolverCache)", name);
+ }
+ }
+ RELEASE(VPath, vcache);
+ return rc;
+}
+
+/********** VPathStr **********/
+static rc_t VPathStrFini(VPathStr *self) {
+ rc_t rc = 0;
+
+ assert(self);
+
+ VPathRelease(self->path);
+
+ RELEASE(String, self->str);
+
+ memset(self, 0, sizeof *self);
+
+ return rc;
+}
+
+/********** TreeNode **********/
static int CC bstCmp(const void *item, const BSTNode *n) {
const char* path = item;
const TreeNode* sn = (const TreeNode*) n;
@@ -806,259 +590,346 @@ static void CC bstWhack(BSTNode* n, void* ignore) {
assert(sn);
-/* #if ! WINDOWS #endif */
free(sn->path);
memset(sn, 0, sizeof *sn);
free(sn);
}
-/*
-static bool _KConfigAscpDisabled(const KConfig *self) {
- bool disabled = false;
- const char path[] = "tools/ascp/disabled";
- rc_t rc = KConfigReadBool(self, path, &disabled);
- if (rc != 0) {
- if (rc != SILENT_RC(rcKFG, rcNode, rcOpening, rcPath, rcNotFound)) {
- DISP_RC(rc, path);
- }
- else {
- STSMSG(STS_DBG, ("'%s': not found in configuration", path));
- }
- disabled = false;
- }
- else {
- STSMSG(STS_DBG, ("'%s' = '%s'", path, disabled ? "true" : "false"));
- }
- return disabled;
-}
-static String* _KConfigAscpString(const KConfig *self,
- const char *path, const char *name)
-{
- String *ascp = NULL;
- rc_t rc = KConfigReadString(self, path, &ascp);
- if (rc == 0) {
- assert(ascp);
- STSMSG(STS_INFO, ("Using %s from configuration: '%s'",
- name, ascp->addr));
- return ascp;
- }
- else {
- if (rc != SILENT_RC(rcKFG, rcNode, rcOpening, rcPath, rcNotFound)) {
- DISP_RC(rc, path);
- }
- else {
- STSMSG(STS_DBG, ("'%s': not found in configuration", path));
- }
- free(ascp);
- return NULL;
- }
-}
+/********** NumIterator **********/
-static String* _StringMake(const char *s) {
- size_t len = string_size(s);
- size_t tmp = 0;
- String *str = malloc(sizeof *str + len + 1);
- if (str == NULL) {
- return NULL;
- }
- StringInit(str,
- (char*)str + sizeof *str, len, len + 1);
- tmp = string_copy((char*)str + sizeof *str, len + 1, s, len + 1);
- assert(tmp == len + 1);
- return str;
+typedef enum {
+ eNIBegin,
+ eNINumber,
+ eNIInterval,
+ eNIDash,
+ eNIComma,
+ eNIBad,
+ eNIForever,
+ eNIEnd
+} ENumIteratorState;
+typedef struct {
+ ENumIteratorState state;
+ bool skip;
+ const char *s;
+ int32_t crnt;
+ int32_t intEnd;
+} NumIterator;
+static void NumIteratorInit(NumIterator *self, const char *row) {
+ assert(self);
+ memset(self, 0, sizeof *self);
+ self->crnt = self->intEnd = -1;
+ self->s = row;
}
-static String* _SystemHelp(String* prev, const char *command) {
- int value = 0;
- if (prev != NULL) {
- return prev;
- }
- STSMSG(STS_DBG, ("Checking '%s'", command));
- value = _SilentSystem("%s -h", command);
- if (value == 0) {
- STSMSG(STS_INFO, ("Using '%s'", command));
- return _StringMake(command);
- }
- else {
- STSMSG(STS_DBG, ("'%s': not found", command));
- return NULL;
+static int32_t NumIteratorGetNum(NumIterator *self) {
+ int32_t n = 0;
+ for (n = 0; *(self->s) >= '0' && *(self->s) <= '9'; ++(self->s)) {
+ n = n * 10 + *(self->s) - '0';
}
+ return n;
}
-static String* _KDirectoryFileFound(const KDirectory *self,
- String* prev, const char *path)
-{
- KPathType type = kptNotFound;
- if (prev != NULL) {
- return prev;
- }
- STSMSG(STS_DBG, ("Checking '%s'", path));
- type = KDirectoryPathType(self, path);
- if ((type & ~kptAlias) == kptFile) {
- STSMSG(STS_DBG, ("'%s': found", path));
- return _StringMake(path);
- }
- else {
- STSMSG(STS_DBG, ("'%s': not found", path));
- return NULL;
+static bool NumIteratorNext(NumIterator *self, int64_t crnt) {
+ char c = '\0';
+ assert(self);
+ while (true) {
+ self->skip = false;
+ switch (self->state) {
+ case eNIBegin:
+ case eNIComma:
+ if (self->s == NULL || *(self->s) == '\0') {
+ if (self->state == eNIBegin) {
+ self->state = eNIForever;
+ continue;
+ }
+ else {
+ self->state = eNIEnd;
+ continue;
+ }
+ }
+ c = *(self->s);
+ ++(self->s);
+ if (c == ',') {
+ self->state = eNIComma;
+ continue;
+ }
+ else if (c == '-') {
+ self->state = eNIDash;
+ continue;
+ }
+ else if (c >= '0' && c <= '9') {
+ --(self->s);
+ self->crnt = NumIteratorGetNum(self);
+ self->state = eNINumber;
+ if (self->crnt < crnt) {
+ continue;
+ }
+ else {
+ if (self->crnt > crnt) {
+ self->skip = true;
+ }
+ return true;
+ }
+ continue;
+ }
+ else {
+ self->state = eNIBad;
+ continue;
+ }
+ case eNIInterval:
+ if (crnt <= self->intEnd) {
+ return true;
+ }
+ /* no break here */
+ case eNINumber:
+ if (self->crnt >= crnt) {
+ if (self->crnt > crnt) {
+ self->skip = true;
+ }
+ return true;
+ }
+ if (self->s == NULL || *(self->s) == '\0') {
+ self->state = eNIEnd;
+ continue;
+ }
+ c = *(self->s);
+ ++(self->s);
+ if (c == ',') {
+ self->state = eNIComma;
+ continue;
+ }
+ else if (c == '-') {
+ self->state = eNIDash;
+ continue;
+ }
+ else {
+ self->state = eNIBad;
+ continue;
+ }
+ case eNIDash:
+ if (self->s == NULL || *(self->s) == '\0') {
+ self->state = eNIForever;
+ continue;
+ }
+ c = *(self->s);
+ ++(self->s);
+ if (c == ',' || c == '-') {
+ self->state = eNIForever;
+ continue;
+ }
+ else if (c >= '0' && c <= '9') {
+ --(self->s);
+ self->intEnd = NumIteratorGetNum(self);
+ self->state = eNIInterval;
+ if (crnt <= self->intEnd) {
+ return true;
+ }
+ else {
+ continue;
+ }
+ }
+ else {
+ self->state = eNIBad;
+ continue;
+ }
+ case eNIForever:
+ return true;
+ case eNIBad:
+ case eNIEnd:
+ return false;
+ }
}
}
-*/
-static bool MainUseAscp(Main *self) {
+
+/********** Resolved **********/
+static rc_t ResolvedFini(Resolved *self) {
rc_t rc = 0;
assert(self);
- if (self->ascpChecked) {
- return self->ascp != NULL;
- }
-
- self->ascpChecked = true;
+ rc = VPathStrFini(&self->local);
- if (self->noAscp) {
- return false;
- }
+ RELEASE(KFile, self->file);
+ RELEASE(VPath, self->accession);
+ RELEASE(VResolver, self->resolver);
- rc = ascp_locate(&self->ascp, &self->asperaKey, true, true);
- return rc == 0 && self->ascp && self->asperaKey;
-}
+ RELEASE(KartItem, self->kartItem);
-static bool MainHasDownloaded(const Main *self, const char *local) {
- TreeNode *sn = NULL;
+ RELEASE(String, self->remote);
+ RELEASE(String, self->cache);
- assert(self);
+ free(self->name);
- sn = (TreeNode*) BSTreeFind(&self->downloaded, local, bstCmp);
+ memset(self, 0, sizeof *self);
- return sn != NULL;
+ return rc;
}
-static rc_t MainDownloaded(Main *self, const char *path) {
- TreeNode *sn = NULL;
-
+static void ResolvedReset(Resolved *self, ERunType type) {
assert(self);
- if (MainHasDownloaded(self, path)) {
- return 0;
- }
-
- sn = calloc(1, sizeof *sn);
- if (sn == NULL) {
- return RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
- }
-
- sn->path = string_dup_measure(path, NULL);
- if (sn->path == NULL) {
- bstWhack((BSTNode*) sn, NULL);
- sn = NULL;
- return RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
- }
-
- BSTreeInsert(&self->downloaded, (BSTNode*)sn, bstSort);
+ memset(self, 0, sizeof *self);
- return 0;
+ self->type = type;
}
-typedef struct {
- const char *obj;
- bool done;
- Kart *kart;
- bool isKart;
-} Iterator;
-static
-rc_t IteratorInit(Iterator *self, const char *obj, const Main *main)
+/** isLocal is set to true when the object is found locally.
+ i.e. does not need need not be [re]downloaded */
+static rc_t ResolvedLocal(const Resolved *self,
+ const KDirectory *dir, bool *isLocal, EForce force)
{
rc_t rc = 0;
+ uint64_t sRemote = 0;
+ uint64_t sLocal = 0;
+ const KFile *local = NULL;
+ char path[PATH_MAX] = "";
- KPathType type = kptNotFound;
+ assert(isLocal && self);
- assert(self && main);
- memset(self, 0, sizeof *self);
+ *isLocal = false;
-#ifdef _DEBUGGING
- if (obj == NULL && main->textkart) {
- type = KDirectoryPathType(main->dir, main->textkart);
- if ((type & ~kptAlias) != kptFile) {
- rc = RC(rcExe, rcFile, rcOpening, rcFile, rcNotFound);
- DISP_RC(rc, main->textkart);
- return rc;
+ if (self->local.str == NULL) {
+ return 0;
+ }
+
+ rc = VPathReadPath(self->local.path, path, sizeof path, NULL);
+ DISP_RC(rc, "VPathReadPath");
+
+ if (rc == 0 && KDirectoryPathType(dir, path) != kptFile) {
+ if (force == eForceNo) {
+ STSMSG(STS_TOP,
+ ("%s (not a file) is found locally: consider it complete",
+ path));
+ *isLocal = true;
}
- rc = KartMakeText(main->dir, main->textkart, &self->kart,
- &self->isKart);
- if (!self->isKart) {
- rc = 0;
+ else {
+ STSMSG(STS_TOP,
+ ("%s (not a file) is found locally and will be redownloaded",
+ path));
}
- return rc;
+ return 0;
}
-#endif
- assert(obj);
- type = KDirectoryPathType(main->dir, obj);
- if ((type & ~kptAlias) == kptFile) {
- type = VDBManagerPathType(main->mgr, obj);
- if ((type & ~kptAlias) == kptFile) {
- rc = KartMake(main->dir, obj, &self->kart, &self->isKart);
- if (!self->isKart) {
- rc = 0;
- }
+ if (rc == 0) {
+ if (! _StringIsFasp(self->remote, NULL) && self->file != NULL) {
+ rc = KFileSize(self->file, &sRemote);
+ DISP_RC2(rc, "KFileSize(remote)", self->name);
+ }
+ else {
+ sRemote = self->remoteSz;
}
}
- if (rc == 0 && !self->isKart) {
- self->obj = obj;
+ if (rc == 0) {
+ rc = KDirectoryOpenFileRead(dir, &local, path);
+ DISP_RC2(rc, "KDirectoryOpenFileRead", path);
+ }
+
+ if (rc == 0) {
+ rc = KFileSize(local, &sLocal);
+ DISP_RC2(rc, "KFileSize", path);
+ }
+
+ if (rc == 0) {
+ if (sRemote == 0) {
+ if (sLocal != 0) {
+ if (force == eForceNo) {
+ *isLocal = true;
+ STSMSG(STS_INFO, ("%s (%,lu) is found", path, sLocal));
+ }
+ else {
+ STSMSG(STS_INFO,
+ ("%s (%,lu) is found and will be redownloaded",
+ path, sLocal));
+ }
+ }
+ else if (sLocal == 0) {
+ STSMSG(STS_INFO,
+ ("an empty %s (%,lu) is found and will be redownloaded",
+ path, sLocal));
+ }
+ }
+ else if (sRemote == sLocal) {
+ if (force == eForceNo) {
+ *isLocal = true;
+ STSMSG(STS_INFO, ("%s (%,lu) is found and is complete",
+ path, sLocal));
+ }
+ else {
+ STSMSG(STS_INFO, ("%s (%,lu) is found and will be redownloaded",
+ path, sLocal));
+ }
+ }
+ else {
+ STSMSG(STS_TOP, ("%s (%,lu) is incomplete. Expected size is %,lu. "
+ "It will be re-downloaded", path, sLocal, sRemote));
+ }
}
+ RELEASE(KFile, local);
+
return rc;
}
-static rc_t IteratorNext(Iterator *self, Item *next, bool *done) {
+/********** Main **********/
+static bool MainUseAscp(Main *self) {
rc_t rc = 0;
- assert(self && next && done);
+ assert(self);
- memset(next, 0, sizeof *next);
+ if (self->ascpChecked) {
+ return self->ascp != NULL;
+ }
- if (self->done) {
- *done = true;
- return 0;
+ self->ascpChecked = true;
+
+ if (self->noAscp) {
+ return false;
}
- *done = false;
+ rc = ascp_locate(&self->ascp, &self->asperaKey, true, true);
+ return rc == 0 && self->ascp && self->asperaKey;
+}
- if (self->isKart) {
- RELEASE(KartItem, next->item);
- rc = KartMakeNextItem(self->kart, &next->item);
- if (rc != 0) {
- LOGERR(klogErr, rc, "Invalid kart file row");
- }
- else if (next->item == NULL) {
- *done = true;
- }
+static bool MainHasDownloaded(const Main *self, const char *local) {
+ TreeNode *sn = NULL;
+
+ assert(self);
+
+ sn = (TreeNode*) BSTreeFind(&self->downloaded, local, bstCmp);
+
+ return sn != NULL;
+}
+
+static rc_t MainDownloaded(Main *self, const char *path) {
+ TreeNode *sn = NULL;
- if (rc == 0 && *done) {
- self->done = true;
- }
- }
- else {
- rc = ItemInit(next, self->obj);
+ assert(self);
- self->done = true;
+ if (MainHasDownloaded(self, path)) {
+ return 0;
}
- return rc;
-}
+ sn = calloc(1, sizeof *sn);
+ if (sn == NULL) {
+ return RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
+ }
-static void IteratorFini(Iterator *self) {
- rc_t rc = 0;
+ sn->path = string_dup_measure(path, NULL);
+ if (sn->path == NULL) {
+ bstWhack((BSTNode*) sn, NULL);
+ sn = NULL;
+ return RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
+ }
- assert(self);
+ BSTreeInsert(&self->downloaded, (BSTNode*)sn, bstSort);
- RELEASE(Kart, self->kart);
+ return 0;
}
-static rc_t ResolvedDownloadFile(Resolved *self,
+static rc_t MainDownloadFile(Resolved *self,
Main *main, const char *to)
{
rc_t rc = 0;
@@ -1080,13 +951,7 @@ static rc_t ResolvedDownloadFile(Resolved *self,
assert(self->remote);
if (self->file == NULL) {
-#define USE_CURL 1
-#if USE_CURL
- rc = KCurlFileMake(&self->file, self->remote->addr, false);
-#else
- rc =
- KNSManagerMakeHttpFile(kns, &self->file, NULL, 0x01010000, "%S", s);
-#endif
+ rc = _KFileOpenRemote(&self->file, self->remote->addr);
if (rc != 0) {
PLOGERR(klogInt, (klogInt, rc, "failed to open file for $(path)",
"path=%s", self->remote->addr));
@@ -1131,7 +996,7 @@ static rc_t ResolvedDownloadFile(Resolved *self,
/* http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByR.../SRR125365.sra
anonftp at ftp-private.ncbi.nlm.nih.gov:/sra/sra-instant/reads/ByR.../SRR125365.sra
*/
-static rc_t ResolvedDownloadAscp(const Resolved *self, Main *main,
+static rc_t MainDownloadAscp(const Resolved *self, Main *main,
const char *to)
{
const char *src = NULL;
@@ -1151,7 +1016,7 @@ static rc_t ResolvedDownloadAscp(const Resolved *self, Main *main,
return aspera_get(main->ascp, main->asperaKey, src, to, &opt);
}
-static rc_t ResolvedDownload(Resolved *self, Main *main) {
+static rc_t MainDownload(Resolved *self, Main *main) {
bool canceled = false;
rc_t rc = 0;
KFile *flock = NULL;
@@ -1224,10 +1089,12 @@ static rc_t ResolvedDownload(Resolved *self, Main *main) {
bool ascp = _StringIsFasp(self->remote, NULL);
if (ascp) {
STSMSG(STS_TOP, (" Downloading via fasp..."));
- rc = ResolvedDownloadAscp(self, main, tmp);
- if (rc == 0 && main->forceAscpFail) {
+ if (main->forceAscpFail) {
rc = 1;
}
+ else {
+ rc = MainDownloadAscp(self, main, tmp);
+ }
if (rc == 0) {
STSMSG(STS_TOP, (" fasp download succeed"));
}
@@ -1246,65 +1113,431 @@ static rc_t ResolvedDownload(Resolved *self, Main *main) {
{
const VPath *vremote = NULL;
STSMSG(STS_TOP, (" Downloading via http..."));
- RELEASE(String, self->remote);
- RELEASE(KFile, self->file);
- rc = _VResolverRemote(main->resolver, eProtocolHttp, self->name,
- self->accession, &vremote, &self->remote, &self->cache);
+ if (ascp) {
+ assert(self->resolver);
+ RELEASE(String, self->remote);
+ RELEASE(KFile, self->file);
+ rc = _VResolverRemote(self->resolver, eProtocolHttp, self->name,
+ self->accession, &vremote, &self->remote, &self->cache);
+ }
if (rc == 0) {
- rc = ResolvedDownloadFile(self, main, tmp);
+ rc = MainDownloadFile(self, main, tmp);
}
RELEASE(VPath, vremote);
}
}
- RELEASE(KFile, flock);
-
- if (rc == 0) {
- STSMSG(STS_DBG, ("renaming %s -> %s", tmp, self->cache->addr));
- rc = KDirectoryRename(main->dir, true, tmp, self->cache->addr);
- if (rc != 0) {
- PLOGERR(klogInt, (klogInt, rc, "cannot rename $(from) to $(to)",
- "from=%s,to=%s", tmp, self->cache->addr));
- }
- }
+ RELEASE(KFile, flock);
+
+ if (rc == 0) {
+ STSMSG(STS_DBG, ("renaming %s -> %s", tmp, self->cache->addr));
+ rc = KDirectoryRename(main->dir, true, tmp, self->cache->addr);
+ if (rc != 0) {
+ PLOGERR(klogInt, (klogInt, rc, "cannot rename $(from) to $(to)",
+ "from=%s,to=%s", tmp, self->cache->addr));
+ }
+ }
+
+ if (rc == 0) {
+ rc = MainDownloaded(main, self->cache->addr);
+ }
+
+ if (rc == 0) {
+ rc_t rc2 = _KDirectoryCleanCache(main->dir, self->cache);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
+
+ {
+ rc_t rc2 = _KDirectoryClean(main->dir, self->cache, lock, tmp, rc != 0);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
+
+ return rc;
+}
+
+static rc_t MainDependenciesList(const Main *self,
+ const char *path, const VDBDependencies **deps)
+{
+ rc_t rc = 0;
+ bool isDb = true;
+ const VDatabase *db = NULL;
+
+ assert(self && path && deps);
+
+ if ((VDBManagerPathType(self->mgr, path) & ~kptAlias) != kptDatabase) {
+ return 0;
+ }
+
+ rc = VDBManagerOpenDBRead(self->mgr, &db, NULL, path);
+ if (rc != 0) {
+ if (rc == SILENT_RC(rcDB, rcMgr, rcOpening, rcDatabase, rcIncorrect)) {
+ isDb = false;
+ rc = 0;
+ }
+ else if (rc ==
+ SILENT_RC(rcKFG, rcEncryptionKey, rcRetrieving, rcItem, rcNotFound))
+ {
+ STSMSG(STS_TOP, ("Cannot open encrypted file '%s'", path));
+ isDb = false;
+ rc = 0;
+ }
+ DISP_RC2(rc, "Cannot open database", path);
+ }
+
+ if (rc == 0 && isDb) {
+ bool all = self->check_all || self->force != eForceNo;
+ rc = VDatabaseListDependencies(db, deps, !all);
+ DISP_RC2(rc, "VDatabaseListDependencies", path);
+ }
+
+ RELEASE(VDatabase, db);
+
+ return rc;
+}
+
+/********** Item **********/
+static rc_t ItemRelease(Item *self) {
+ rc_t rc = 0;
+
+ assert(self);
+
+ rc = ResolvedFini(&self->resolved);
+ RELEASE(KartItem, self->item);
+
+ memset(self, 0, sizeof *self);
+
+ return rc;
+}
+
+static rc_t ItemInit(Item *self, const char *obj) {
+ assert(self);
+ self->desc = obj;
+ return 0;
+}
+
+static char* ItemName(const Item *self) {
+ char *c = NULL;
+ assert(self);
+ if (self->desc != NULL) {
+ return string_dup_measure(self->desc, NULL);
+ }
+ else {
+ rc_t rc = 0;
+ const String *elem = NULL;
+ assert(self->item);
+
+ rc = KartItemItemDesc(self->item, &elem);
+ c = StringCheck(elem, rc);
+ if (c != NULL) {
+ return c;
+ }
+
+ rc = KartItemAccession(self->item, &elem);
+ c = StringCheck(elem, rc);
+ if (c != NULL) {
+ return c;
+ }
+
+ rc = KartItemItemId(self->item, &elem);
+ return StringCheck(elem, rc);
+ }
+}
+
+static rc_t _ItemSetResolverAndAssessionInResolved(Item *item,
+ VResolver *resolver, const KConfig *cfg, const KRepositoryMgr *repoMgr,
+ const VFSManager *vfs)
+{
+ Resolved *resolved = NULL;
+ rc_t rc = 0;
+
+ assert(item && resolver && cfg && repoMgr && vfs);
+
+ resolved = &item->resolved;
+
+ if (item->desc != NULL) {
+ rc = VFSManagerMakePath(vfs, &resolved->accession, item->desc);
+ DISP_RC2(rc, "VFSManagerMakePath", item->desc);
+ if (rc == 0) {
+ rc = VResolverAddRef(resolver);
+ }
+ if (rc == 0) {
+ resolved->resolver = resolver;
+ }
+ }
+ else {
+ uint64_t oid = 0;
+ rc = KartItemProjIdNumber(item->item, &resolved->project);
+ if (rc != 0) {
+ DISP_RC(rc, "KartItemProjIdNumber");
+ return rc;
+ }
+ rc = KartItemItemIdNumber(item->item, &oid);
+ if (rc != 0) {
+ DISP_RC(rc, "KartItemItemIdNumber");
+ return rc;
+ }
+ else {
+ const KRepository *p_protected = NULL;
+ rc = KRepositoryMgrGetProtectedRepository(repoMgr,
+ (uint32_t)resolved->project, &p_protected);
+ if (rc == 0) {
+ rc = KRepositoryMakeResolver(p_protected,
+ &resolved->resolver, cfg);
+ if (rc != 0) {
+ DISP_RC(rc, "KRepositoryMakeResolver");
+ return rc;
+ }
+ }
+ else {
+ PLOGERR(klogErr, (klogErr, rc,
+ "project '$(P)': cannot find protected repository", "P=%d",
+ resolved->project));
+ }
+ RELEASE(KRepository, p_protected);
+ }
+ if (rc == 0) {
+ rc =
+ VFSManagerMakeOidPath(vfs, &resolved->accession, (uint32_t)oid);
+ if (rc != 0) {
+ DISP_RC(rc, "VFSManagerMakeOidPath");
+ return rc;
+ }
+ }
+ }
+
+ return rc;
+}
+
+/* resolve locations */
+static rc_t _ItemResolveResolved(VResolver *resolver,
+ VRemoteProtocols protocols, Item *item, const KRepositoryMgr *repoMgr,
+ const KConfig *cfg, const VFSManager *vfs, size_t minSize, size_t maxSize)
+{
+ Resolved *resolved = NULL;
+ rc_t rc = 0;
+ rc_t rc2 = 0;
+
+ const VPath *vremote = NULL;
+
+ assert(resolver && item);
+
+ resolved = &item->resolved;
+
+ memset(&resolved->local, 0, sizeof resolved->local);
+
+ assert(resolved->accession == NULL);
+
+ rc = _ItemSetResolverAndAssessionInResolved(item,
+ resolver, cfg, repoMgr, vfs);
+
+ if (rc == 0) {
+ rc = V_ResolverLocal(resolved->resolver,
+ resolved->accession, &resolved->local.path);
+ if (rc == 0) {
+ rc = VPathMakeString(resolved->local.path, &resolved->local.str);
+ DISP_RC2(rc, "VPathMakeString(VResolverLocal)", resolved->name);
+ }
+ else if (NotFoundByResolver(rc)) {
+ rc = 0;
+ }
+ else {
+ DISP_RC2(rc, "VResolverLocal", resolved->name);
+ }
+ }
+
+ if (rc == 0) {
+ rc2 = 0;
+ resolved->remoteSz = 0;
+ assert(item->main);
+ if ((minSize > 0 || maxSize > 0 || item->main->order == eOrderSize)
+ && (protocols == eProtocolFasp ||
+ protocols == eProtocolFaspHttp))
+ {
+ rc2 = _VResolverRemote(resolved->resolver, eProtocolHttp,
+ resolved->name, resolved->accession,
+ &vremote, &resolved->remote, NULL);
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+ else {
+ rc_t rc3 = 0;
+ if (resolved->file == NULL) {
+ rc3 = _KFileOpenRemote(&resolved->file,
+ resolved->remote->addr);
+ DISP_RC2(rc3,
+ "cannot open remote file", resolved->remote->addr);
+ }
+
+ RELEASE(VPath, vremote);
+
+ if (rc3 == 0 && resolved->file != NULL) {
+ rc3 = KFileSize(resolved->file, &resolved->remoteSz);
+ if (rc3 != 0) {
+ DISP_RC2(rc3, "cannot get remote file size",
+ resolved->remote->addr);
+ }
+ else if (resolved->remoteSz >= maxSize) {
+ return rc;
+ }
+ else if (resolved->remoteSz < minSize) {
+ return rc;
+ }
+ }
+ }
+ }
+
+ if (rc2 == 0) {
+ rc2 = _VResolverRemote(resolved->resolver, protocols,
+ resolved->name, resolved->accession,
+ &vremote, &resolved->remote, &resolved->cache);
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+ }
+
+ if (rc == 0) {
+ rc2 = 0;
+ if (resolved->file == NULL) {
+ assert(resolved->remote);
+ if (!_StringIsFasp(resolved->remote, NULL)) {
+ rc2 = _KFileOpenRemote(
+ &resolved->file, resolved->remote->addr);
+ }
+ }
+ if (rc2 == 0 && resolved->file != NULL && resolved->remoteSz == 0) {
+ rc2 = KFileSize(resolved->file, &resolved->remoteSz);
+ DISP_RC2(rc2, "KFileSize(remote)", resolved->name);
+ }
+ }
+ }
+
+ RELEASE(VPath, vremote);
+ return rc;
+}
+
+/* Resolved: resolve locations */
+static rc_t ItemInitResolved(Item *self, VResolver *resolver,
+ KDirectory *dir, bool ascp, const KRepositoryMgr *repoMgr,
+ const KConfig *cfg, const VFSManager *vfs, size_t minSize, size_t maxSize)
+{
+ Resolved *resolved = NULL;
+ rc_t rc = 0;
+ KPathType type = kptNotFound;
+ VRemoteProtocols protocols = ascp ? eProtocolFaspHttp : eProtocolHttp;
+
+ assert(self);
+
+ resolved = &self->resolved;
+ resolved->name = ItemName(self);
+
+ assert(resolved->type != eRunTypeUnknown);
+
+ type = KDirectoryPathType(dir, self->desc) & ~kptAlias;
+ if (type == kptFile || type == kptDir) {
+ resolved->path = self->desc;
+ resolved->existing = true;
+ if (resolved->type != eRunTypeDownload) {
+ uint64_t s = -1;
+ const KFile *f = NULL;
+ rc = KDirectoryOpenFileRead(dir, &f, self->desc);
+ if (rc == 0) {
+ rc = KFileSize(f, &s);
+ }
+ if (s != -1) {
+/* OUTMSG(("%s\t%,zuB\n", self->desc, s)); */
+ resolved->remoteSz = s;
+ }
+ else {
+ OUTMSG(("%s\tunknown\n", self->desc));
+ }
+ RELEASE(KFile, f);
+ }
+ else {
+ STSMSG(STS_TOP, ("'%s' is a local non-kart file", self->desc));
+ }
+ return 0;
+ }
+
+ rc = _ItemResolveResolved(resolver, protocols, self,
+ repoMgr, cfg, vfs, minSize, maxSize);
- if (rc == 0) {
- rc = MainDownloaded(main, self->cache->addr);
- }
+ STSMSG(STS_DBG, ("Resolve(%s) = %R:", resolved->name, rc));
+ STSMSG(STS_DBG, ("local(%s)",
+ resolved->local.str ? resolved->local.str->addr : "NULL"));
+ STSMSG(STS_DBG, ("cache(%s)",
+ resolved->cache ? resolved->cache->addr : "NULL"));
+ STSMSG(STS_DBG, ("remote(%s:%,ld)",
+ resolved->remote ? resolved->remote->addr : "NULL",
+ resolved->remoteSz));
if (rc == 0) {
- rc_t rc2 = _KDirectoryCleanCache(main->dir, self->cache);
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
+ if (resolved->remoteSz >= maxSize) {
+ resolved->oversized = true;
+ return rc;
+ }
+ if (resolved->remoteSz < minSize) {
+ resolved->undersized = true;
+ return rc;
}
- }
- {
- rc_t rc2 = _KDirectoryClean(main->dir, self->cache, lock, tmp, rc != 0);
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
+ if (resolved->local.str == NULL
+ && (resolved->cache == NULL || resolved->remote == NULL))
+ {
+ rc = RC(rcExe, rcPath, rcValidating, rcParam, rcNull);
+ PLOGERR(klogInt, (klogInt, rc,
+ "bad VResolverResolve($(acc)) result",
+ "acc=%s", resolved->name));
}
}
return rc;
}
-static
-rc_t ResolvedResolve(Resolved *self, Main *main, const Item *item)
-{
+/* resolve: locate */
+static rc_t ItemResolve(Item *item, int32_t row) {
+ Resolved *self = NULL;
static int n = 0;
rc_t rc = 0;
- bool isLocal = false;
bool ascp = false;
- assert(self && main);
+ assert(item && item->main);
+
+ self = &item->resolved;
+ assert(self->type);
++n;
+ if (row > 0 && item->desc == NULL) {
+ n = row;
+ }
+
+ item->number = n;
+
+ ascp = MainUseAscp(item->main);
+ if (self->type == eRunTypeList) {
+ ascp = false;
+ }
- ascp = MainUseAscp(main);
+ rc = ItemInitResolved(item, item->main->resolver, item->main->dir, ascp,
+ item->main->repoMgr, item->main->cfg, item->main->vfsMgr,
+ item->main->minSize, item->main->maxSize);
- rc = ResolvedInit(self, item,
- main->resolver, main->dir, ascp, main->repoMgr, main->cfg);
+ return rc;
+}
+
+/* download if not found; obey size restriction */
+static rc_t ItemDownload(Item *item) {
+ bool isLocal = false;
+ int n = 0;
+ rc_t rc = 0;
+ Resolved *self = NULL;
+ assert(item && item->main);
+ n = item->number;
+ self = &item->resolved;
+ assert(self->type);
if (rc == 0) {
if (self->existing) {
@@ -1312,28 +1545,42 @@ rc_t ResolvedResolve(Resolved *self, Main *main, const Item *item)
return rc;
}
- rc = ResolvedLocal(self, main->dir, &isLocal, main->force);
+ if (self->undersized) {
+ STSMSG(STS_TOP,
+ ("%d) '%s' (%,zu KB) is smaller than minimum allowed: skipped\n",
+ n, self->name, self->remoteSz / 1024));
+ return rc;
+ }
+
+ if (self->oversized) {
+ STSMSG(STS_TOP,
+ ("%d) '%s' (%,zu KB) is larger than maximum allowed: skipped\n",
+ n, self->name, self->remoteSz / 1024));
+ return rc;
+ }
+
+ rc = ResolvedLocal(self, item->main->dir, &isLocal, item->main->force);
}
if (rc == 0) {
if (isLocal) {
- STSMSG(STS_TOP, ("%d) %s is found locally", n, self->name));
+ STSMSG(STS_TOP, ("%d) '%s' is found locally", n, self->name));
if (self->local.str != NULL) {
self->path = self->local.str->addr;
}
}
- else if (!_StringIsFasp(self->remote, NULL) && main->noHttp) {
+ else if (!_StringIsFasp(self->remote, NULL) && item->main->noHttp) {
rc = RC(rcExe, rcFile, rcCopying, rcFile, rcNotFound);
PLOGERR(klogErr, (klogErr, rc,
"cannot download '$(name)' using requested transport",
"name=%s", self->name));
}
else {
- STSMSG(STS_TOP, ("%d) Downloading %s...", n, self->name));
- rc = ResolvedDownload(self, main);
+ STSMSG(STS_TOP, ("%d) Downloading '%s'...", n, self->name));
+ rc = MainDownload(self, item->main);
if (rc == 0) {
STSMSG(STS_TOP,
- ("%d) %s was downloaded successfully", n, self->name));
+ ("%d) '%s' was downloaded successfully", n, self->name));
if (self->cache != NULL) {
self->path = self->cache->addr;
}
@@ -1345,59 +1592,93 @@ rc_t ResolvedResolve(Resolved *self, Main *main, const Item *item)
}
}
}
+ else {
+ STSMSG(STS_TOP, ("%d) cannot locate '%s'", n, self->name));
+ }
return rc;
}
-static rc_t MainDependenciesList(const Main *self,
- const char *path, const VDBDependencies **deps)
-{
+static rc_t ItemPrintSized(const Item *self, int32_t row, size_t size) {
rc_t rc = 0;
- bool isDb = true;
- const VDatabase *db = NULL;
+ const String *projId = NULL;
+ const String *itemId = NULL;
+ const String *accession = NULL;
+ const String *name = NULL;
+ const String *itemDesc = NULL;
+ assert(self && row);
+ if (self->desc != NULL) {
+ OUTMSG(("%d\t%s\t%,zuB\n", row, self->desc, size));
+ }
+ else {
+ if (rc == 0) {
+ rc = KartItemProjId(self->item, &projId);
+ }
+ if (rc == 0) {
+ rc = KartItemItemId(self->item, &itemId);
+ }
+ if (rc == 0) {
+ rc = KartItemAccession(self->item, &accession);
+ }
+ if (rc == 0) {
+ rc = KartItemName(self->item, &name);
+ }
+ if (rc == 0) {
+ rc = KartItemItemDesc(self->item, &itemDesc);
+ }
+ if (rc == 0) {
+ rc = OUTMSG(("%d\t%S|%S|%S|%S|%S\t%,zuB\n",
+ row, projId, itemId, accession, name, itemDesc, size));
+ }
+ }
- assert(self && path && deps);
+ return rc;
+}
- rc = VDBManagerOpenDBRead(self->mgr, &db, NULL, path);
+/* resolve: locate; download if not found */
+static rc_t ItemResolveResolvedAndDownloadOrProcess(Item *self, int32_t row) {
+ rc_t rc = ItemResolve(self, row);
if (rc != 0) {
- if (rc == SILENT_RC(rcDB, rcMgr, rcOpening, rcDatabase, rcIncorrect)) {
- isDb = false;
- rc = 0;
- }
- DISP_RC2(rc, "Cannot open database", path);
+ return rc;
}
- if (rc == 0 && isDb) {
- bool all = self->check_all || self->force != eForceNo;
- rc = VDatabaseListDependencies(db, deps, !all);
- DISP_RC2(rc, "VDatabaseListDependencies", path);
- }
+ assert(self);
- RELEASE(VDatabase, db);
+ if (self->resolved.type == eRunTypeList) {
+ return ItemPrintSized(self, row, self->resolved.remoteSz);
+ }
+ else if (self->resolved.type != eRunTypeDownload) {
+ return rc;
+ }
- return rc;
+ return ItemDownload(self);
}
-static rc_t MainExecute(Main *self, const Item *item) {
+static rc_t ItemPostDownload(Item *item, int32_t row) {
+ Resolved *resolved = NULL;
rc_t rc = 0;
rc_t rc2 = 0;
const VDBDependencies *deps = NULL;
uint32_t count = 0;
uint32_t i = 0;
- Resolved resolved;
-
- assert(self && item);
-
- if (item->desc == NULL) {
- OUTMSG(("Requested kart row download\n"));
- }
+ assert(item && item->main);
- /* resolve: locate; download if not found */
- rc = ResolvedResolve(&resolved, self, item);
+ resolved = &item->resolved;
if (rc == 0) {
- rc = MainDependenciesList(self, resolved.path, &deps);
+ if (resolved->type == eRunTypeList) {
+ return rc;
+ }
+ else if (resolved->oversized) {
+ item->main->oversized = true;
+ return rc;
+ }
+ else if (resolved->undersized) {
+ item->main->undersized = true;
+ return rc;
+ }
+ rc = MainDependenciesList(item->main, resolved->path, &deps);
}
/* resolve dependencies (refseqs) */
@@ -1405,7 +1686,7 @@ static rc_t MainExecute(Main *self, const Item *item) {
rc = VDBDependenciesCount(deps, &count);
if (rc == 0) {
STSMSG(STS_TOP, ("'%s' has %d%s dependenc%s",
- item->desc, count, self->check_all ? "" : " unresolved",
+ item->desc, count, item->main->check_all ? "" : " unresolved",
count == 1 ? "y" : "ies"));
}
else {
@@ -1447,11 +1728,20 @@ static rc_t MainExecute(Main *self, const Item *item) {
}
if (rc == 0) {
- Resolved resolved;
- Item item;
- item.desc = ncbiAcc;
- rc = ResolvedResolve(&resolved, self, &item);
- rc2 = ResolvedFini(&resolved);
+ Item *ditem = calloc(1, sizeof *ditem);
+ if (ditem == NULL) {
+ return RC(rcExe,
+ rcStorage, rcAllocating, rcMemory, rcExhausted);
+ }
+
+ ditem->desc = ncbiAcc;
+ ditem->main = item->main;
+
+ ResolvedReset(&ditem->resolved, eRunTypeDownload);
+
+ rc = ItemResolveResolvedAndDownloadOrProcess(ditem, 0);
+
+ RELEASE(Item, ditem);
}
}
}
@@ -1462,41 +1752,167 @@ static rc_t MainExecute(Main *self, const Item *item) {
RELEASE(VDBDependencies, deps);
- {
- rc_t rc2 = ResolvedFini(&resolved);
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
+ return rc;
+}
+
+static rc_t ItemProcess(Item *item, int32_t row) {
+ rc_t rc = 0;
+
+ assert(item);
+
+ /* resolve: locate; download if not found */
+ rc = ItemResolveResolvedAndDownloadOrProcess(item, row);
+
+ if (item->resolved.type != eRunTypeDownload) {
+ return rc;
+ }
+
+ if (rc == 0) {
+ rc = ItemPostDownload(item, row);
+ }
+
+ return rc;
+}
+
+/*********** Iterator **********/
+static
+rc_t IteratorInit(Iterator *self, const char *obj, const Main *main)
+{
+ rc_t rc = 0;
+
+ KPathType type = kptNotFound;
+
+ assert(self && main);
+ memset(self, 0, sizeof *self);
+
+#ifdef _DEBUGGING
+ if (obj == NULL && main->textkart) {
+ type = KDirectoryPathType(main->dir, main->textkart);
+ if ((type & ~kptAlias) != kptFile) {
+ rc = RC(rcExe, rcFile, rcOpening, rcFile, rcNotFound);
+ DISP_RC(rc, main->textkart);
+ return rc;
+ }
+ rc = KartMakeText(main->dir, main->textkart, &self->kart,
+ &self->isKart);
+ if (rc != 0) {
+ if (!self->isKart) {
+ rc = 0;
+ }
+ else {
+ PLOGERR(klogErr, (klogErr, rc, "'$(F)' is not a text kart file",
+ "F=%s", main->textkart));
+ }
+ }
+ return rc;
+ }
+#endif
+
+ assert(obj);
+ type = KDirectoryPathType(main->dir, obj);
+ if ((type & ~kptAlias) == kptFile) {
+ type = VDBManagerPathType(main->mgr, obj);
+ if ((type & ~kptAlias) == kptFile) {
+ rc = KartMake(main->dir, obj, &self->kart, &self->isKart);
+ if (!self->isKart) {
+ rc = 0;
+ }
}
}
+ if (rc == 0 && !self->isKart) {
+ self->obj = obj;
+ }
+
return rc;
}
-static rc_t MainFini(Main *self) {
+static rc_t IteratorNext(Iterator *self, Item **next, bool *done) {
rc_t rc = 0;
- assert(self);
+ assert(self && next && done);
- RELEASE(KConfig, self->cfg);
- RELEASE(VResolver, self->resolver);
- RELEASE(VDBManager, self->mgr);
- RELEASE(KDirectory, self->dir);
- RELEASE(KRepositoryMgr, self->repoMgr);
- RELEASE(Args, self->args);
+ *next = NULL;
- BSTreeWhack(&self->downloaded, bstWhack, NULL);
+ if (self->done) {
+ *done = true;
+ return 0;
+ }
- free(self->buffer);
+ *done = false;
- free((void*)self->ascp);
- free((void*)self->asperaKey);
- free(self->ascpMaxRate);
+ *next = calloc(1, sizeof **next);
+ if (*next == NULL) {
+ return RC(rcExe, rcStorage, rcAllocating, rcMemory, rcExhausted);
+ }
- memset(self, 0, sizeof *self);
+ if (self->isKart) {
+ rc = KartMakeNextItem(self->kart, &(*next)->item);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Invalid kart file: cannot read next row");
+ }
+ else if ((*next)->item == NULL) {
+ RELEASE(Item, *next);
+ *next = NULL;
+ *done = true;
+ }
+
+ if (rc == 0 && *done) {
+ self->done = true;
+ }
+ }
+ else {
+ rc = ItemInit(*next, self->obj);
+
+ self->done = true;
+ }
return rc;
}
+static void IteratorFini(Iterator *self) {
+ rc_t rc = 0;
+
+ assert(self);
+
+ RELEASE(Kart, self->kart);
+}
+
+/*********** Command line arguments **********/
+
+static size_t _sizeFromString(const char *val) {
+ size_t s = 0;
+
+ for (s = 0; *val != '\0'; ++val) {
+ if (*val < '0' || *val > '9') {
+ break;
+ }
+ s = s * 10 + *val - '0';
+ }
+
+ if (*val == '\0' || *val == 'k' || *val == 'K') {
+ s *= 1024L;
+ }
+ else if (*val == 'b' || *val == 'B') {
+ }
+ else if (*val == 'm' || *val == 'M') {
+ s *= 1024L * 1024;
+ }
+ else if (*val == 'g' || *val == 'G') {
+ s *= 1024L * 1024 * 1024;
+ }
+ else if (*val == 'u' || *val == 'U') { /* unlimited */
+ s = 0;
+ }
+
+ return s;
+}
+
+#define ASCP_OPTION "ascp-path"
+#define ASCP_ALIAS "a"
+static const char* ASCP_USAGE[] =
+{ "path to ascp program and private key file (asperaweb_id_dsa.putty)", NULL };
+
#define CHECK_ALL_OPTION "check-all"
#define CHECK_ALL_ALIAS "c"
static const char* CHECK_ALL_USAGE[] = { "double-check all refseqs", NULL };
@@ -1507,24 +1923,49 @@ static const char* FORCE_USAGE[] = {
"force object download - one of: no, yes, all.",
"no [default]: skip download if the object if found and complete;",
"yes: download it even if it is found and is complete;", "all: ignore lock "
- "files (stale locks or it is beeing downloaded by another process)", NULL };
+ "files (stale locks or it is beeing downloaded by another process: "
+ "use at your own risk!)", NULL };
#define FAIL_ASCP_OPTION "FAIL-ASCP"
#define FAIL_ASCP_ALIAS "F"
static const char* FAIL_ASCP_USAGE[] = {
"force ascp download fail to test ascp->http download combination" };
-
#define LIST_OPTION "list"
#define LIST_ALIAS "l"
static const char* LIST_USAGE[] = { "list the content of a kart file", NULL };
-#define HBEAT_ALIAS "p"
+#define NM_L_OPTION "numbered-list"
+#define NM_L_ALIAS "n"
+static const char* NM_L_USAGE[] =
+{ "list the content of a kart file with kart row numbers", NULL };
+
+#define MINSZ_OPTION "min-size"
+#define MINSZ_ALIAS "N"
+static const char* MINSZ_USAGE[] =
+{ "minimum file size to download in KB (inclusive).", NULL };
+
+#define ORDR_OPTION "order"
+#define ORDR_ALIAS "o"
+static const char* ORDR_USAGE[] = { "kart prefetch order: one of: kart, size.",
+ "(in kart order, by file size: smallest first), default: size", NULL };
+
#define HBEAT_OPTION "progress"
+#define HBEAT_ALIAS "p"
static const char* HBEAT_USAGE[] = {
"time period in minutes to display download progress",
"(0: no progress), default: 1", NULL };
+#define ROWS_OPTION "rows"
+#define ROWS_ALIAS "R"
+static const char* ROWS_USAGE[] =
+{ "kart rows (default all).", "row list should be ordered", NULL };
+
+#define SZ_L_OPTION "list-sizes"
+#define SZ_L_ALIAS "s"
+static const char* SZ_L_USAGE[] =
+{ "list the content of a kart file with target file sizes", NULL };
+
#define TRANS_OPTION "transport"
#define TRASN_ALIAS "t"
static const char* TRANS_USAGE[] = { "transport: one of: ascp; http; both.",
@@ -1532,6 +1973,13 @@ static const char* TRANS_USAGE[] = { "transport: one of: ascp; http; both.",
"use http if cannot download by ascp).",
"Default: both", NULL };
+#define DEFAULT_MAX_FILE_SIZE "20G"
+#define SIZE_OPTION "max-size"
+#define SIZE_ALIAS "X"
+static const char* SIZE_USAGE[] = {
+ "maximum file size to download in KB (exclusive).",
+ "Default: " DEFAULT_MAX_FILE_SIZE, NULL };
+
#ifdef _DEBUGGING
#define TEXTKART_OPTION "text-kart"
static const char* TEXTKART_USAGE[] =
@@ -1543,12 +1991,19 @@ static OptDef Options[] = {
{ FORCE_OPTION , FORCE_ALIAS , NULL, FORCE_USAGE , 1, true, false }
,{ TRANS_OPTION , TRASN_ALIAS , NULL, TRANS_USAGE , 1, true, false }
,{ LIST_OPTION , LIST_ALIAS , NULL, LIST_USAGE , 1, false,false }
+ ,{ NM_L_OPTION , NM_L_ALIAS , NULL, NM_L_USAGE , 1, false,false }
+ ,{ SZ_L_OPTION , SZ_L_ALIAS , NULL, SZ_L_USAGE , 1, false,false }
+ ,{ ROWS_OPTION , ROWS_ALIAS , NULL, ROWS_USAGE , 1, true, false }
+ ,{ MINSZ_OPTION , MINSZ_ALIAS , NULL, MINSZ_USAGE , 1, true ,false }
+ ,{ SIZE_OPTION , SIZE_ALIAS , NULL, SIZE_USAGE , 1, true ,false }
+ ,{ ORDR_OPTION , ORDR_ALIAS , NULL, ORDR_USAGE , 1, true ,false }
+ ,{ ASCP_OPTION , ASCP_ALIAS , NULL, ASCP_USAGE , 1, true ,false }
,{ HBEAT_OPTION , HBEAT_ALIAS , NULL, HBEAT_USAGE , 1, true, false }
- ,{ CHECK_ALL_OPTION, CHECK_ALL_ALIAS, NULL, CHECK_ALL_USAGE, 1, false, false}
,{ FAIL_ASCP_OPTION, FAIL_ASCP_ALIAS, NULL, FAIL_ASCP_USAGE, 1, false, false}
#ifdef _DEBUGGING
,{ TEXTKART_OPTION , NULL , NULL, TEXTKART_USAGE , 1, true , false}
#endif
+ ,{ CHECK_ALL_OPTION, CHECK_ALL_ALIAS, NULL, CHECK_ALL_USAGE, 1, false, false}
};
static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
@@ -1566,6 +2021,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
}
do {
+/* FORCE_OPTION goes first */
rc = ArgsOptionCount (self->args, FORCE_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr, rc, "Failure to get '" FORCE_OPTION "' argument");
@@ -1610,36 +2066,99 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
}
}
- rc = ArgsOptionCount(self->args, CHECK_ALL_OPTION, &pcount);
+/* CHECK_ALL_OPTION goes after FORCE_OPTION */
+ rc = ArgsOptionCount(self->args, CHECK_ALL_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" CHECK_ALL_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0 || self->force != eForceNo) {
+ self->check_all = true;
+ }
+
+/******* LIST OPTIONS BEGIN ********/
+/* LIST_OPTION */
+ rc = ArgsOptionCount(self->args, LIST_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr,
- rc, "Failure to get '" CHECK_ALL_OPTION "' argument");
+ rc, "Failure to get '" LIST_OPTION "' argument");
break;
}
- if (pcount > 0 || self->force != eForceNo) {
- self->check_all = true;
+ if (pcount > 0) {
+ self->list_kart = true;
}
- rc = ArgsOptionCount(self->args, FAIL_ASCP_OPTION, &pcount);
+/* NM_L_OPTION */
+ rc = ArgsOptionCount(self->args, NM_L_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr,
- rc, "Failure to get '" FAIL_ASCP_OPTION "' argument");
+ rc, "Failure to get '" NM_L_OPTION "' argument");
break;
}
if (pcount > 0) {
- self->forceAscpFail = true;
+ self->list_kart = self->list_kart_numbered = true;
}
- rc = ArgsOptionCount(self->args, LIST_OPTION, &pcount);
+/* SZ_L_OPTION */
+ rc = ArgsOptionCount(self->args, SZ_L_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr,
- rc, "Failure to get '" LIST_OPTION "' argument");
+ rc, "Failure to get '" SZ_L_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0) { /* self->list_kart is not set here! */
+ self->list_kart_sized = true;
+ }
+/******* LIST OPTIONS END ********/
+
+/* ASCP_OPTION */
+ rc = ArgsOptionCount(self->args, ASCP_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" ASCP_OPTION "' argument");
break;
}
if (pcount > 0) {
- self->list_kart = true;
+ const char *val = NULL;
+ rc = ArgsOptionValue(self->args, ASCP_OPTION, 0, &val);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" ASCP_OPTION "' argument value");
+ break;
+ }
+ if (val != NULL) {
+ char *sep = strchr(val, '|');
+ if (sep == NULL) {
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
+ LOGERR(klogErr, rc,
+ "ascp-path expected in the following format:\n"
+ "--" ASCP_OPTION " \"<ascp-binary|private-key-file>\"\n"
+ "Examples:\n"
+ "--" ASCP_OPTION " \"/usr/bin/ascp|/etc/asperaweb_id_dsa.putty\"\n"
+ "--" ASCP_OPTION " \"C:\\Program Files\\Aspera\\ascp.exe|C:\\Program Files\\Aspera\\etc\\asperaweb_id_dsa.putty\"\n");
+ break;
+ }
+ else {
+ self->ascp = string_dup(val, sep - val);
+ self->asperaKey = string_dup_measure(sep + 1, NULL);
+ self->ascpChecked = true;
+ }
+ }
+ }
+
+/* FAIL_ASCP_OPTION */
+ rc = ArgsOptionCount(self->args, FAIL_ASCP_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" FAIL_ASCP_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ self->forceAscpFail = true;
}
+/* HBEAT_OPTION */
rc = ArgsOptionCount(self->args, HBEAT_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr, rc, "Failure to get '" HBEAT_OPTION "' argument");
@@ -1656,9 +2175,100 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
break;
}
f = atof(val) * 60000;
- self->heartbeat = f;
+ self->heartbeat = (uint64_t)f;
+ }
+
+/* ORDR_OPTION */
+ rc = ArgsOptionCount(self->args, ORDR_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr, rc, "Failure to get '" ORDR_OPTION "' argument");
+ break;
+ }
+
+ if (pcount > 0) {
+ const char *val = NULL;
+ rc = ArgsOptionValue(self->args, ORDR_OPTION, 0, &val);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" ORDR_OPTION "' argument value");
+ break;
+ }
+ if (val != NULL && val[0] == 's') {
+ self->order = eOrderSize;
+ }
+ else {
+ self->order = eOrderOrig;
+ }
+ }
+
+/* ROWS_OPTION */
+ rc = ArgsOptionCount(self->args, ROWS_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" ROWS_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ rc = ArgsOptionValue(self->args, ROWS_OPTION, 0, &self->rows);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" ROWS_OPTION "' argument value");
+ break;
+ }
+ }
+
+/* MINSZ_OPTION */
+ {
+ const char *val = "0";
+ rc = ArgsOptionCount(self->args, MINSZ_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" MINSZ_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ rc = ArgsOptionValue(self->args, MINSZ_OPTION, 0, &val);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" MINSZ_OPTION "' argument value");
+ break;
+ }
+ }
+ self->minSize = _sizeFromString(val);
+ }
+
+/* SIZE_OPTION */
+ {
+ const char *val = DEFAULT_MAX_FILE_SIZE;
+ rc = ArgsOptionCount(self->args, SIZE_OPTION, &pcount);
+ if (rc != 0) {
+ LOGERR(klogErr,
+ rc, "Failure to get '" SIZE_OPTION "' argument");
+ break;
+ }
+ if (pcount > 0) {
+ rc = ArgsOptionValue(self->args, SIZE_OPTION, 0, &val);
+ if (rc != 0) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" SIZE_OPTION "' argument value");
+ break;
+ }
+ }
+ self->maxSize = _sizeFromString(val);
+ if (self->maxSize == 0) {
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
+ LOGERR(klogErr, rc, "Maximum requested file size is zero");
+ break;
+ }
+ }
+
+ if (self->maxSize > 0 && self->minSize > self->maxSize) {
+ rc = RC(rcExe, rcArgv, rcParsing, rcParam, rcInvalid);
+ LOGERR(klogErr, rc, "Minimum file size is larger than maximum");
+ break;
}
+/* TRANS_OPTION */
rc = ArgsOptionCount(self->args, TRANS_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr, rc, "Failure to get '" TRANS_OPTION "' argument");
@@ -1683,6 +2293,7 @@ static rc_t MainProcessArgs(Main *self, int argc, char *argv[]) {
}
#ifdef _DEBUGGING
+/* TEXTKART_OPTION */
rc = ArgsOptionCount(self->args, TEXTKART_OPTION, &pcount);
if (rc != 0) {
LOGERR(klogErr, rc,
@@ -1752,10 +2363,39 @@ rc_t CC Usage(const Args *args) {
if (strcmp(Options[i].aliases, FAIL_ASCP_ALIAS) == 0) {
continue; /* debug option */
}
- if (strcmp(Options[i].aliases, FORCE_ALIAS) == 0) {
+ if (strcmp(Options[i].aliases, ASCP_ALIAS) == 0) {
+ param = "ascp-binary|private-key-file";
+ }
+ else if (strcmp(Options[i].aliases, FORCE_ALIAS) == 0 ||
+ strcmp(Options[i].aliases, HBEAT_ALIAS) == 0 ||
+ strcmp(Options[i].aliases, HBEAT_ALIAS) == 0 ||
+ strcmp(Options[i].aliases, ORDR_ALIAS) == 0 ||
+ strcmp(Options[i].aliases, TRASN_ALIAS) == 0)
+ {
param = "value";
}
+ else if (strcmp(Options[i].aliases, ROWS_ALIAS) == 0) {
+ param = "rows";
+ }
+ else if (strcmp(Options[i].aliases, SIZE_ALIAS) == 0
+ || strcmp(Options[i].aliases, MINSZ_ALIAS) == 0)
+ {
+ param = "size";
+ }
+ }
+#ifdef _DEBUGGING
+ else if (strcmp(Options[i].name, TEXTKART_OPTION) == 0) {
+ param = "value";
}
+#endif
+
+ if (Options[i].aliases != NULL &&
+ (strcmp(Options[i].aliases, TRASN_ALIAS) == 0 ||
+ strcmp(Options[i].aliases, CHECK_ALL_ALIAS) == 0))
+ {
+ OUTMSG(("\n"));
+ }
+
HelpOptionLine(Options[i].aliases, Options[i].name,
param, Options[i].help);
}
@@ -1767,9 +2407,75 @@ rc_t CC Usage(const Args *args) {
return rc;
}
+ver_t CC KAppVersion(void) { return PREFETCH_VERS; }
+
+/******************************************************************************/
+
+/********** KartTreeNode **********/
+static void CC bstKrtWhack(BSTNode *n, void *ignore) {
+ KartTreeNode *sn = (KartTreeNode*) n;
+
+ assert(sn);
+
+ ItemRelease(sn->i);
+
+ memset(sn, 0, sizeof *sn);
+
+ free(sn);
+}
+
+static int CC bstKrtSort(const BSTNode *item, const BSTNode *n) {
+ const KartTreeNode *sn1 = (const KartTreeNode*)item;
+ const KartTreeNode *sn2 = (const KartTreeNode*)n;
+
+ assert(sn1 && sn2 && sn1->i && sn2->i);
+
+ return sn1->i->resolved.remoteSz > sn2->i->resolved.remoteSz;
+}
+
+static void CC bstKrtDownload(BSTNode *n, void *data) {
+ rc_t rc = 0;
+
+ const KartTreeNode *sn = (const KartTreeNode*) n;
+ assert(sn && sn->i);
+
+ rc = ItemDownload(sn->i);
+
+ if (rc == 0) {
+ rc = ItemPostDownload(sn->i, sn->i->number);
+ }
+}
+
+/*********** Finalize Main object **********/
+static rc_t MainFini(Main *self) {
+ rc_t rc = 0;
+
+ assert(self);
+
+ RELEASE(KConfig, self->cfg);
+ RELEASE(VResolver, self->resolver);
+ RELEASE(VDBManager, self->mgr);
+ RELEASE(KDirectory, self->dir);
+ RELEASE(KRepositoryMgr, self->repoMgr);
+ RELEASE(VFSManager, self->vfsMgr);
+ RELEASE(Args, self->args);
+
+ BSTreeWhack(&self->downloaded, bstWhack, NULL);
+
+ free(self->buffer);
+
+ free((void*)self->ascp);
+ free((void*)self->asperaKey);
+ free(self->ascpMaxRate);
+
+ memset(self, 0, sizeof *self);
+
+ return rc;
+}
+
+/*********** Initialize Main object **********/
static rc_t MainInit(int argc, char *argv[], Main *self) {
rc_t rc = 0;
- VFSManager *mgr = NULL;
assert(self);
memset(self, 0, sizeof *self);
@@ -1792,13 +2498,13 @@ static rc_t MainInit(int argc, char *argv[], Main *self) {
}
if (rc == 0) {
- rc = VFSManagerMake(&mgr);
+ rc = VFSManagerMake(&self->vfsMgr);
DISP_RC(rc, "VFSManagerMake");
}
if (rc == 0) {
VResolver *resolver = NULL;
- rc = VFSManagerGetResolver(mgr, &resolver);
+ rc = VFSManagerGetResolver(self->vfsMgr, &resolver);
DISP_RC(rc, "VFSManagerGetResolver");
VResolverRemoteEnable(resolver, vrAlwaysEnable);
RELEASE(VResolver, resolver);
@@ -1815,7 +2521,7 @@ static rc_t MainInit(int argc, char *argv[], Main *self) {
}
if (rc == 0) {
- rc = VFSManagerMakeResolver(mgr, &self->resolver, self->cfg);
+ rc = VFSManagerMakeResolver(self->vfsMgr, &self->resolver, self->cfg);
DISP_RC(rc, "VFSManagerMakeResolver");
}
@@ -1835,18 +2541,195 @@ static rc_t MainInit(int argc, char *argv[], Main *self) {
}
if (rc == 0) {
- srand(time(NULL));
+ srand((unsigned)time(NULL));
}
- RELEASE(VFSManager, mgr);
-
return rc;
}
-ver_t CC KAppVersion(void) { return PREFETCH_VERS; }
+/*********** Process one command line argument **********/
+static rc_t MainRun(Main *self, const char *arg, const char *realArg) {
+ ERunType type = eRunTypeDownload;
+ static bool maxSzPrntd = false;
+ rc_t rc = 0;
+ Iterator it;
+ assert(self && realArg);
+ memset(&it, 0, sizeof it);
-/******************************************************************************/
+ if (rc == 0) {
+ rc = IteratorInit(&it, arg, self);
+ }
+
+ if (self->list_kart_sized) {
+ type = eRunTypeList;
+ }
+ else if (self->order == eOrderSize) {
+ if (rc == 0 && it.kart == NULL) {
+ type = eRunTypeDownload;
+ }
+ else {
+ type = eRunTypeGetSize;
+ }
+ }
+ else {
+ type = eRunTypeDownload;
+ }
+
+ if (rc == 0) {
+ BSTree trKrt;
+ BSTreeInit(&trKrt);
+ if (self->list_kart) {
+ if (it.kart != NULL) {
+ if (self->list_kart_numbered) {
+ rc = KartPrintNumbered(it.kart);
+ }
+ else {
+ rc = KartPrint(it.kart);
+ }
+ }
+ else {
+ PLOGMSG(klogWarn, (klogWarn,
+ "'$(F)' is invalid or not a kart file",
+ "F=%s", realArg));
+ }
+ }
+ else {
+ size_t total = 0;
+ const char *row = self->rows;
+ int64_t n = 1;
+ NumIterator nit;
+ NumIteratorInit(&nit, row);
+ if (type == eRunTypeList) {
+ self->maxSize = ~0;
+ if (it.kart != NULL) {
+ OUTMSG((
+ "row\tproj-id|item-id|accession|name|item-desc\tsize\n"));
+ }
+ }
+ else {
+ if (!maxSzPrntd) {
+ maxSzPrntd = true;
+ if (self->maxSize == 0) {
+ OUTMSG((
+ "Maximum file size download limit is unlimited\n"));
+ }
+ else {
+ OUTMSG(("Maximum file size download limit is %,zuKB\n",
+ self->maxSize / 1024));
+ }
+ }
+ if (it.kart != NULL) {
+ OUTMSG(("Downloading kart file '%s'\n", realArg));
+ if (type == eRunTypeGetSize) {
+ OUTMSG(("Checking sizes of kart files...\n"));
+ }
+ }
+ OUTMSG(("\n"));
+ }
+
+ for (n = 1; ; ++n) {
+ rc_t rc3 = 0;
+ bool done = false;
+ Item *item = NULL;
+ rc = Quitting();
+ if (rc != 0) {
+ break;
+ }
+ rc = IteratorNext(&it, &item, &done);
+ if (rc != 0 || done) {
+ break;
+ }
+ done = ! NumIteratorNext(&nit, n);
+ if (done) {
+ break;
+ }
+ if (!nit.skip) {
+ item->main = self;
+ ResolvedReset(&item->resolved, type);
+
+ rc3 = ItemProcess(item, (int32_t)n);
+ if (rc3 != 0) {
+ if (rc == 0) {
+ rc = rc3;
+ }
+ }
+ else {
+ if (item->resolved.undersized &&
+ type == eRunTypeGetSize)
+ {
+ STSMSG(STS_TOP,
+ ("%d) '%s' (%,zu KB) is smaller than minimum allowed: skipped\n",
+ n, item->resolved.name, item->resolved.remoteSz / 1024));
+ }
+ else if (item->resolved.oversized) {
+ STSMSG(STS_TOP,
+ ("%d) '%s' (%,zu KB) is larger than maximum allowed: skipped\n",
+ n, item->resolved.name, item->resolved.remoteSz / 1024));
+ }
+ else {
+ total += item->resolved.remoteSz;
+
+ if (item != NULL) {
+ if (type == eRunTypeGetSize) {
+ KartTreeNode *sn = calloc(1, sizeof *sn);
+ if (sn == NULL) {
+ return RC(rcExe, rcStorage,
+ rcAllocating, rcMemory, rcExhausted);
+ }
+ if (item->resolved.remoteSz == 0) {
+ /* remoteSz is unknown:
+ add it to the end of download list preserving kart order */
+ item->resolved.remoteSz
+ = (~0ul >> 1) + n + 1;
+ }
+ sn->i = item;
+ item = NULL;
+ BSTreeInsert(&trKrt, (BSTNode*)sn,
+ bstKrtSort);
+ }
+ }
+ }
+ }
+ }
+ else {
+ RELEASE(Item, item);
+ }
+ }
+
+ if (type == eRunTypeList) {
+ if (it.kart != NULL && total > 0) {
+ OUTMSG(("--------------------\ntotal\t%,zuB\n\n", total));
+ }
+ }
+ else if (type == eRunTypeGetSize) {
+ OUTMSG(("\nDownloading the files\n\n", realArg));
+ BSTreeForEach(&trKrt, false, bstKrtDownload, NULL);
+ }
+ }
+ BSTreeWhack(&trKrt, bstKrtWhack, NULL);
+ }
+ if (it.isKart) {
+ if (self->list_kart) {
+ rc_t rc2 = OUTMSG(("\n"));
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+ }
+ else if (rc == 0) {
+ uint16_t number = 0;
+ rc = KartItemsProcessed(it.kart, &number);
+ if (rc == 0 && number == 0) {
+ PLOGMSG(klogWarn, (klogWarn,
+ "kart file '$(F)' is empty", "F=%s", realArg));
+ }
+ }
+ }
+ IteratorFini(&it);
+
+ return rc;
+}
+/*********** Main **********/
rc_t CC KMain(int argc, char *argv[]) {
rc_t rc = 0;
uint32_t pcount = 0;
@@ -1869,88 +2752,35 @@ rc_t CC KMain(int argc, char *argv[]) {
#ifdef _DEBUGGING
if (pars.textkart) {
- rc_t rc2 = 0;
- Iterator it;
- memset(&it, 0, sizeof it);
- if (rc2 == 0) {
- rc2 = IteratorInit(&it, NULL, &pars);
- }
- if (rc2 == 0) {
- if (pars.list_kart) {
- rc2 = KartPrint(it.kart);
- }
- else {
- while (true) {
- rc_t rc3 = 0;
- bool done = false;
- Item item;
- rc2 = IteratorNext(&it, &item, &done);
- if (rc2 != 0 || done == true) {
- break;
- }
- rc3 = MainExecute(&pars, &item);
- if (rc == 0) {
- if (rc2 != 0) {
- rc = rc2;
- }
- else if (rc3 != 0) {
- rc = rc3;
- }
- }
- }
- }
- }
+ rc = MainRun(&pars, NULL, pars.textkart);
}
#endif
for (i = 0; i < pcount; ++i) {
- rc_t rc2 = 0;
- Iterator it;
const char *obj = NULL;
-
- memset(&it, 0, sizeof it);
-
- rc2 = ArgsParamValue(pars.args, i, &obj);
+ rc_t rc2 = ArgsParamValue(pars.args, i, &obj);
DISP_RC(rc2, "ArgsParamValue");
-
if (rc2 == 0) {
- rc2 = IteratorInit(&it, obj, &pars);
+ rc = MainRun(&pars, obj, obj);
}
+ }
- if (rc2 == 0) {
- if (pars.list_kart) {
- rc2 = KartPrint(it.kart);
- }
- else {
- while (true) {
- rc_t rc3 = 0;
- bool done = false;
- Item item;
- rc2 = IteratorNext(&it, &item, &done);
- if (rc2 != 0 || done == true) {
- break;
- }
- rc3 = MainExecute(&pars, &item);
- if (rc == 0) {
- if (rc2 != 0) {
- rc = rc2;
- }
- else if (rc3 != 0) {
- rc = rc3;
- }
- }
- }
- }
+ if (pars.undersized || pars.oversized) {
+ OUTMSG(("\n"));
+ if (pars.undersized) {
+ OUTMSG((
+ "Download of some files was skipped because they are too small\n"
+ ));
}
-
- if (pars.list_kart && it.isKart) {
- rc_t rc2 = OUTMSG(("\n"));
- if (rc2 != 0 && rc == 0) {
- rc = rc2;
- }
+ if (pars.oversized) {
+ OUTMSG((
+ "Download of some files was skipped because they are too large\n"
+ ));
}
-
- IteratorFini(&it);
+ OUTMSG((
+ "You can change size download limit by setting\n"
+ "--" MINSZ_OPTION " and --" SIZE_OPTION " command line arguments\n"
+ ));
}
}
diff --git a/tools/prefetch/prefetch.vers b/tools/prefetch/prefetch.vers
index 0bee604..3f684d2 100644
--- a/tools/prefetch/prefetch.vers
+++ b/tools/prefetch/prefetch.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sam-dump/sam-dump.c b/tools/sam-dump/sam-dump.c
index 682f264..5c2bf2a 100644
--- a/tools/sam-dump/sam-dump.c
+++ b/tools/sam-dump/sam-dump.c
@@ -3975,7 +3975,7 @@ static rc_t ProcessPath( VDBManager const *mgr, char const Path[] )
ext = strrchr( basename, '.' );
if ( ext != NULL )
{
- if ( strcasecmp( ext, ".nenc" ) == 0 || strcasecmp( ext, ",ncbi_enc" ) == 0 )
+ if ( strcasecmp( ext, ".nenc" ) == 0 || strcasecmp( ext, ".ncbi_enc" ) == 0 )
{
*ext = '\0';
ext = strrchr( basename, '.' );
diff --git a/tools/sra-dbcc/Makefile b/tools/sra-dbcc/Makefile
index beeda09..cb3c92d 100644
--- a/tools/sra-dbcc/Makefile
+++ b/tools/sra-dbcc/Makefile
@@ -90,21 +90,8 @@ SRA_DBCC_OBJ = \
$(addsuffix .$(OBJX),$(SRA_DBCC_SRC))
SRA_DBCC_LIB = \
- -dalign-reader \
- $(READONLY_SCHEMA_LIBS) \
- -dksrch \
- -lsradb \
- -lvdb \
- -lkdb \
-lkapp \
- -lvfs \
- -lkurl \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lkproc \
- -lklib \
- -lm
+ -lncbi-vdb \
sra-dbcc.vers.h: sra-dbcc.vers
$(TOP)/build/make-vers-inc.sh SRA_DBCC_VERS $^ > $@
diff --git a/tools/sra-dbcc/sra-dbcc.vers b/tools/sra-dbcc/sra-dbcc.vers
index f90b1af..3f684d2 100644
--- a/tools/sra-dbcc/sra-dbcc.vers
+++ b/tools/sra-dbcc/sra-dbcc.vers
@@ -1 +1 @@
-2.3.2
+2.3.4
diff --git a/tools/sra-dump/Makefile b/tools/sra-dump/Makefile
index 51302a8..13c5713 100644
--- a/tools/sra-dump/Makefile
+++ b/tools/sra-dump/Makefile
@@ -76,14 +76,6 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# tag
-#
-tag: \
- $(addsuffix _tag,$(ALL_TOOLS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_TOOLS))
-
-#-------------------------------------------------------------------------------
# Common dumper definitions
#
DUMP_COMMON_SRC = \
@@ -110,9 +102,6 @@ FASTQ_DUMP_OBJ = \
$(BINDIR)/fastq-dump: $(FASTQ_DUMP_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(DUMP_COMMON_LIB)
-fastq-dump_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) fastq-dump $(FASTQ_DUMP_OBJ)
-
#-------------------------------------------------------------------------------
# sff-dump
#
@@ -126,9 +115,6 @@ SFF_DUMP_OBJ = \
$(BINDIR)/sff-dump: $(SFF_DUMP_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(DUMP_COMMON_LIB)
-sff-dump_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) sff-dump $(SFF_DUMP_OBJ)
-
#-------------------------------------------------------------------------------
# illumina-dump
#
@@ -142,9 +128,6 @@ ILLUMINA_DUMP_OBJ = \
$(BINDIR)/illumina-dump: $(ILLUMINA_DUMP_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(DUMP_COMMON_LIB)
-illumina-dump_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) illumina-dump $(ILLUMINA_DUMP_OBJ)
-
#-------------------------------------------------------------------------------
# abi-dump
#
@@ -158,5 +141,3 @@ ABI_DUMP_OBJ = \
$(BINDIR)/abi-dump: $(ABI_DUMP_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(DUMP_COMMON_LIB)
-abi-dump_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) abi-dump $(ABI_DUMP_OBJ)
diff --git a/tools/sra-dump/abi-dump.vers b/tools/sra-dump/abi-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-dump/abi-dump.vers
+++ b/tools/sra-dump/abi-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-dump/core.c b/tools/sra-dump/core.c
index 4ab00b8..63a512a 100644
--- a/tools/sra-dump/core.c
+++ b/tools/sra-dump/core.c
@@ -28,6 +28,7 @@
#include <kfg/config.h> /* KConfigDisableUserSettings */
#include <vdb/manager.h> /* VDBManagerRelease */
+#include <vdb/vdb-priv.h> /* VDBManagerDisablePagemapThread() */
#include <kdb/manager.h> /* for different path-types */
#include <vdb/dependencies.h> /* UIError */
#include <klib/report.h> /* ReportInit */
@@ -657,41 +658,41 @@ static rc_t SRADumper_DumpRun( const SRATable* table,
static const SRADumperFmt_Arg KMainArgs[] =
{
- { NULL, "no-user-settings", NULL, {"Internal Only", NULL}},
- { "A", "accession", "accession", {"Replaces accession derived from <path> in filename(s) and deflines (only for single table dump)", NULL}},
- { "O", "outdir", "path", {"Output directory, default is working directory ( '.' )", NULL}},
- { "Z", "stdout", NULL, {"Output to stdout, all split data become joined into single stream", NULL}},
- { NULL, "gzip", NULL, {"Compress output using gzip", NULL}},
- { NULL, "bzip2", NULL, {"Compress output using bzip2", NULL}},
- { "N", "minSpotId", "rowid", {"Minimum spot id", NULL}},
- { "X", "maxSpotId", "rowid", {"Maximum spot id", NULL}},
- { "G", "spot-group", NULL, {"Split into files by SPOT_GROUP (member name)", NULL}},
- { NULL, "spot-groups", "[list]", {"Filter by SPOT_GROUP (member): name[,...]", NULL}},
- { "R", "read-filter", "[filter]", {"Split into files by READ_FILTER value",
- "optionally filter by a value: pass|reject|criteria|redacted", NULL}},
- { "T", "group-in-dirs", NULL, {"Split into subdirectories instead of files", NULL}},
- { "K", "keep-empty-files", NULL, {"Do not delete empty files", NULL}},
- { NULL, "table", "table-name", {"Table name within cSRA object, default is \"SEQUENCE\"", NULL}},
-
- { "h", "help", NULL, {"Output a brief explanation of program usage", NULL}},
- { "V", "version", NULL, {"Display the version of the program", NULL}},
-
- { "L", "log-level", "level", {"Logging level as number or enum string",
- "One of (fatal|sys|int|err|warn|info) or (0-5)",
- "Current/default is warn", NULL}},
- { "v", "verbose", NULL, {"Increase the verbosity level of the program",
- "Use multiple times for more verbosity", NULL}},
- { NULL, OPTION_REPORT, NULL, {
-"Control program execution environment report generation (if implemented).",
-"One of (never|error|always). Default is error",
- NULL}},
+ { NULL, "no-user-settings", NULL, { "Internal Only", NULL } },
+ { "A", "accession", "accession", { "Replaces accession derived from <path> in filename(s) and deflines (only for single table dump)", NULL } },
+ { "O", "outdir", "path", { "Output directory, default is working directory ( '.' )", NULL } },
+ { "Z", "stdout", NULL, { "Output to stdout, all split data become joined into single stream", NULL } },
+ { NULL, "gzip", NULL, { "Compress output using gzip", NULL } },
+ { NULL, "bzip2", NULL, { "Compress output using bzip2", NULL } },
+ { "N", "minSpotId", "rowid", { "Minimum spot id", NULL } },
+ { "X", "maxSpotId", "rowid", { "Maximum spot id", NULL } },
+ { "G", "spot-group", NULL, { "Split into files by SPOT_GROUP (member name)", NULL } },
+ { NULL, "spot-groups", "[list]", { "Filter by SPOT_GROUP (member): name[,...]", NULL } },
+ { "R", "read-filter", "[filter]", { "Split into files by READ_FILTER value",
+ "optionally filter by a value: pass|reject|criteria|redacted", NULL } },
+ { "T", "group-in-dirs", NULL, { "Split into subdirectories instead of files", NULL } },
+ { "K", "keep-empty-files", NULL, { "Do not delete empty files", NULL } },
+ { NULL, "table", "table-name", { "Table name within cSRA object, default is \"SEQUENCE\"", NULL } },
+
+ { NULL, "disable-multithreading", NULL, { "disable multithreading", NULL } },
+
+ { "h", "help", NULL, { "Output a brief explanation of program usage", NULL } },
+ { "V", "version", NULL, { "Display the version of the program", NULL } },
+
+ { "L", "log-level", "level", { "Logging level as number or enum string",
+ "One of (fatal|sys|int|err|warn|info) or (0-5)",
+ "Current/default is warn", NULL } },
+ { "v", "verbose", NULL, { "Increase the verbosity level of the program",
+ "Use multiple times for more verbosity", NULL } },
+ { NULL, OPTION_REPORT, NULL, { "Control program execution environment report generation (if implemented).",
+ "One of (never|error|always). Default is error", NULL } },
#if _DEBUGGING
- {"+", "debug", "Module[-Flag]", {"Turn on debug output for module",
- "All flags if not specified", NULL}},
+ { "+", "debug", "Module[-Flag]",{ "Turn on debug output for module",
+ "All flags if not specified", NULL } },
#endif
- { NULL, "legacy-report", NULL, { "use legacy style 'Written N spots' for tool" } },
- { NULL, NULL, NULL, {NULL}} /* terminator */
+ { NULL, "legacy-report", NULL, { "use legacy style 'Written N spots' for tool" } },
+ { NULL, NULL, NULL, { NULL } } /* terminator */
};
@@ -986,6 +987,7 @@ rc_t CC KMain ( int argc, char* argv[] )
const char* table_name = NULL;
bool spot_group_on = false;
+ bool no_mt = false;
int spot_groups = 0;
char* spot_group[128] = {NULL};
bool read_filter_on = false;
@@ -1064,6 +1066,10 @@ rc_t CC KMain ( int argc, char* argv[] )
goto Catch;
}
}
+ else if ( SRADumper_GetArg( &fmt, NULL, "disable-multithreading", &i, argc, argv, NULL ) )
+ {
+ no_mt = true;
+ }
else if ( SRADumper_GetArg( &fmt, NULL, OPTION_REPORT, &i, argc, argv, &arg ) )
{
}
@@ -1316,6 +1322,17 @@ rc_t CC KMain ( int argc, char* argv[] )
{
LOGERR( klogErr, rc2, "while calling SRAMgrGetVDBManagerRead" );
}
+ else
+ {
+ if ( no_mt )
+ {
+ rc2 = VDBManagerDisablePagemapThread ( vmgr );
+ if ( rc2 != 0 )
+ {
+ LOGERR( klogErr, rc2, "disabling multithreading failed" );
+ }
+ }
+ }
rc2 = ReportSetVDBManager( vmgr );
}
@@ -1440,7 +1457,7 @@ rc_t CC KMain ( int argc, char* argv[] )
ext = strrchr( fmt.accession, '.' );
if ( ext != NULL )
{
- if ( strcasecmp( ext, ".nenc" ) == 0 || strcasecmp( ext, ",ncbi_enc" ) == 0 )
+ if ( strcasecmp( ext, ".nenc" ) == 0 || strcasecmp( ext, ".ncbi_enc" ) == 0 )
{
*ext = '\0';
ext = strrchr( fmt.accession, '.' );
diff --git a/tools/sra-dump/fastq-dump.vers b/tools/sra-dump/fastq-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-dump/fastq-dump.vers
+++ b/tools/sra-dump/fastq-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-dump/illumina-dump.vers b/tools/sra-dump/illumina-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-dump/illumina-dump.vers
+++ b/tools/sra-dump/illumina-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-dump/sff-dump.vers b/tools/sra-dump/sff-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-dump/sff-dump.vers
+++ b/tools/sra-dump/sff-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-dump/sra-dump.vers b/tools/sra-dump/sra-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-dump/sra-dump.vers
+++ b/tools/sra-dump/sra-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-load/Makefile b/tools/sra-load/Makefile
index 3743ab8..02805e8 100644
--- a/tools/sra-load/Makefile
+++ b/tools/sra-load/Makefile
@@ -83,14 +83,6 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# tag
-#
-tag: \
- $(addsuffix _tag,$(ALL_TOOLS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_TOOLS))
-
-#-------------------------------------------------------------------------------
# loader
#
LOADER_SRC = \
diff --git a/tools/sra-load/abi-load.vers b/tools/sra-load/abi-load.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-load/abi-load.vers
+++ b/tools/sra-load/abi-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-load/fastq-load.vers b/tools/sra-load/fastq-load.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-load/fastq-load.vers
+++ b/tools/sra-load/fastq-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-load/helicos-load.vers b/tools/sra-load/helicos-load.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-load/helicos-load.vers
+++ b/tools/sra-load/helicos-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-load/illumina-load.vers b/tools/sra-load/illumina-load.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-load/illumina-load.vers
+++ b/tools/sra-load/illumina-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-load/pacbio-loadxml.vers b/tools/sra-load/pacbio-loadxml.vers
new file mode 100644
index 0000000..3f684d2
--- /dev/null
+++ b/tools/sra-load/pacbio-loadxml.vers
@@ -0,0 +1 @@
+2.3.4
diff --git a/tools/sra-load/sff-load.vers b/tools/sra-load/sff-load.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-load/sff-load.vers
+++ b/tools/sra-load/sff-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-load/srf-load.vers b/tools/sra-load/srf-load.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-load/srf-load.vers
+++ b/tools/sra-load/srf-load.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-pileup/Makefile b/tools/sra-pileup/Makefile
index 0ce8dea..21129ec 100644
--- a/tools/sra-pileup/Makefile
+++ b/tools/sra-pileup/Makefile
@@ -87,6 +87,8 @@ clean: stdclean
TOOL_SRC = \
cmdline_cmn \
reref \
+ ref_regions \
+ ref_walker \
sra-pileup
TOOL_OBJ = \
diff --git a/tools/sra-pileup/cmdline_cmn.c b/tools/sra-pileup/cmdline_cmn.c
index 28b3be1..3a93b03 100644
--- a/tools/sra-pileup/cmdline_cmn.c
+++ b/tools/sra-pileup/cmdline_cmn.c
@@ -69,8 +69,8 @@ const char * inf_usage[] = { "File with all input-parameters / options", NULL };
const char * schema_usage[] = { "optional schema-file to be used", NULL };
-#define OPTION_REF "aligned-region"
-#define ALIAS_REF "r"
+const char * no_mt_usage[] = { "disable multithreading", NULL };
+
#define OPTION_OUTF "outfile"
#define ALIAS_OUTF "o"
@@ -90,6 +90,8 @@ const char * schema_usage[] = { "optional schema-file to be used", NULL };
#define OPTION_SCHEMA "schema"
#define ALIAS_SCHEMA "S"
+#define OPTION_NO_MT "disable-multithreading"
+
OptDef CommonOptions[] =
{
/*name, alias, hfkt, usage-help, maxcount, needs value, required */
@@ -99,7 +101,8 @@ OptDef CommonOptions[] =
{ OPTION_GZIP, ALIAS_GZIP, NULL, gzip_usage, 1, false, false },
{ OPTION_BZIP, ALIAS_BZIP, NULL, bzip_usage, 1, false, false },
{ OPTION_INF, ALIAS_INF, NULL, inf_usage, 0, true, false },
- { OPTION_SCHEMA, ALIAS_SCHEMA, NULL, schema_usage, 1, true, false }
+ { OPTION_SCHEMA, ALIAS_SCHEMA, NULL, schema_usage, 1, true, false },
+ { OPTION_NO_MT, NULL, NULL, no_mt_usage, 1, false, false }
};
@@ -160,6 +163,9 @@ rc_t get_common_options( Args * args, common_options *opts )
rc = get_bool_option( args, OPTION_BZIP, &opts->bzip_output, false );
if ( rc == 0 )
+ rc = get_bool_option( args, OPTION_NO_MT, &opts->no_mt, false );
+
+ if ( rc == 0 )
rc = get_str_option( args, OPTION_SCHEMA, &opts->schema_file );
if ( rc == 0 )
@@ -195,6 +201,7 @@ void print_common_helplines( void )
HelpOptionLine ( ALIAS_TABLE, OPTION_TABLE, "shortcut", table_usage );
HelpOptionLine ( ALIAS_BZIP, OPTION_BZIP, NULL, bzip_usage );
HelpOptionLine ( ALIAS_GZIP, OPTION_GZIP, NULL, gzip_usage );
+ HelpOptionLine ( NULL, OPTION_NO_MT, NULL, no_mt_usage );
}
@@ -228,303 +235,6 @@ static int cmp_pchar( const char * a, const char * b )
/* =========================================================================================== */
-typedef struct range
-{
- uint32_t start;
- uint32_t end;
-} range;
-
-
-static range * make_range( const uint64_t start, const uint64_t end )
-{
- range *res = calloc( sizeof *res, 1 );
- if ( res != NULL )
- {
- res->start = start;
- res->end = end;
- }
- return res;
-}
-
-
-static int cmp_range( const range * a, const range * b )
-{
-
- int res = ( a->start - b->start );
- if ( res == 0 )
- res = ( a->end - b->end );
- return res;
-}
-
-
-static bool range_overlapp( const range * a, const range * b )
-{
- return ( !( ( b->end < a->start ) || ( b->start > a->end ) ) );
-}
-
-
-/* =========================================================================================== */
-
-typedef struct reference_region
-{
- BSTNode node;
- const char * name;
- Vector ranges;
-} reference_region;
-
-
-static reference_region * make_reference_region( const char *name )
-{
- reference_region *res = calloc( sizeof *res, 1 );
- if ( res != NULL )
- {
- res->name = string_dup_measure ( name, NULL );
- VectorInit ( &res->ranges, 0, 5 );
- }
- return res;
-}
-
-
-static int CC cmp_range_wrapper( const void *item, const void *n )
-{ return cmp_range( item, n ); }
-
-
-static rc_t add_ref_region_range( reference_region * self, const uint64_t start, const uint64_t end )
-{
- rc_t rc = 0;
- range *r = make_range( start, end );
- if ( r == NULL )
- rc = RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- else
- {
- rc = VectorInsert ( &self->ranges, r, NULL, cmp_range_wrapper );
- if ( rc != 0 )
- free( r );
- }
- return rc;
-}
-
-
-#define RR_NAME 1
-#define RR_START 2
-#define RR_END 3
-
-
-static void put_c( char *s, size_t size, size_t *dst, char c )
-{
- if ( *dst < ( size - 1 ) )
- s[ *dst ] = c;
- (*dst)++;
-}
-
-static void finish_txt( char *s, size_t size, size_t *dst )
-{
- if ( *dst > size )
- s[ size - 1 ] = 0;
- else
- s[ *dst ] = 0;
- *dst = 0;
-}
-
-static uint64_t finish_num( char *s, size_t size, size_t *dst )
-{
- uint64_t res = 0;
- char *endp;
- finish_txt( s, size, dst );
- res = strtou64( s, &endp, 10 );
- return res;
-}
-
-
-/* s = refname:1000-2000 */
-static void parse_definition( const char *s, char * name, size_t len,
- uint64_t *start, uint64_t *end )
-{
- size_t n = string_size( s );
-
- *start = 0;
- *end = 0;
- name[ 0 ] = 0;
- if ( n > 0 )
- {
- size_t i, st, dst = 0;
- char tmp[ 32 ];
- st = RR_NAME;
- for ( i = 0; i < n; ++i )
- {
- char c = s[ i ];
- switch( st )
- {
- case RR_NAME : if ( c == ':' )
- {
- finish_txt( name, len, &dst );
- st = RR_START;
- }
- else
- {
- put_c( name, len, &dst, c );
- }
- break;
-
- case RR_START : if ( c == '-' )
- {
- *start = finish_num( tmp, sizeof tmp, &dst );
- st = RR_END;
- }
- else if ( ( c >= '0' )&&( c <= '9' ) )
- {
- put_c( tmp, sizeof tmp, &dst, c );
- }
- break;
-
- case RR_END : if ( ( c >= '0' )&&( c <= '9' ) )
- {
- put_c( tmp, sizeof tmp, &dst, c );
- }
- break;
- }
- }
- switch( st )
- {
- case RR_NAME : finish_txt( name, len, &dst );
- break;
-
- case RR_START : *start = finish_num( tmp, sizeof tmp, &dst );
- break;
-
- case RR_END : *end = finish_num( tmp, sizeof tmp, &dst );
- break;
- }
- }
-}
-
-
-static void CC release_range_wrapper( void * item, void * data )
-{
- free( item );
-}
-
-
-static void free_reference_region( reference_region * self )
-{
- free( (void*)self->name );
- VectorWhack ( &self->ranges, release_range_wrapper, NULL );
- free( self );
-}
-
-
-static void check_ref_region_ranges( reference_region * self )
-{
- uint32_t n = VectorLength( &self->ranges );
- uint32_t i = 0;
- range *a = NULL;
- while ( i < n )
- {
- range *b = VectorGet ( &self->ranges, i );
- bool remove = false;
- if ( a != NULL )
- {
- remove = range_overlapp( a, b );
- if ( remove )
- {
- range *r;
- a->end = b->end;
- VectorRemove ( &self->ranges, i, (void**)&r );
- free( r );
- n--;
- }
- }
- if ( !remove )
- {
- a = b;
- ++i;
- }
- }
-}
-
-
-/* =========================================================================================== */
-
-
-static int CC reference_vs_pchar_wrapper( const void *item, const BSTNode *n )
-{
- const reference_region * r = ( const reference_region * )n;
- return cmp_pchar( (const char *)item, r->name );
-}
-
-static reference_region * find_reference_region( BSTree * regions, const char * name )
-{
- return ( reference_region * ) BSTreeFind ( regions, name, reference_vs_pchar_wrapper );
-}
-
-static int CC ref_vs_ref_wrapper( const BSTNode *item, const BSTNode *n )
-{
- const reference_region * a = ( const reference_region * )item;
- const reference_region * b = ( const reference_region * )n;
- return cmp_pchar( a->name, b->name );
-}
-
-static rc_t add_refrange( BSTree * regions, const char * name, const uint64_t start, const uint64_t end )
-{
- rc_t rc;
-
- reference_region * r = find_reference_region( regions, name );
- if ( r == NULL )
- {
- r = make_reference_region( name );
- if ( r == NULL )
- rc = RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- else
- rc = add_ref_region_range( r, start, end );
- if ( rc == 0 )
- rc = BSTreeInsert ( regions, (BSTNode *)r, ref_vs_ref_wrapper );
- if ( rc != 0 )
- free_reference_region( r );
- }
- else
- {
- rc = add_ref_region_range( r, start, end );
- }
- return rc;
-}
-
-
-rc_t parse_and_add_region( BSTree * regions, const char * s )
-{
- uint64_t start, end;
- char name[ 64 ];
- parse_definition( s, name, sizeof name, &start, &end );
- if ( name[ 0 ] == 0 )
- return RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
- else
- return add_refrange( regions, name, start, end );
-}
-
-
-static void CC check_refrange_wrapper( BSTNode *n, void *data )
-{
- check_ref_region_ranges( ( reference_region * ) n );
-}
-
-
-void check_ref_regions( BSTree * regions )
-{
- BSTreeForEach ( regions, false, check_refrange_wrapper, NULL );
-}
-
-
-static void CC release_ref_region_wrapper( BSTNode *n, void * data )
-{
- free_reference_region( ( reference_region * ) n );
-}
-
-
-void free_ref_regions( BSTree * regions )
-{
- BSTreeWhack ( regions, release_ref_region_wrapper, NULL );
-}
-
-
rc_t init_ref_regions( BSTree * tree, Args * args )
{
uint32_t count;
@@ -553,70 +263,6 @@ rc_t init_ref_regions( BSTree * tree, Args * args )
}
-static void CC count_ref_region_wrapper( BSTNode *n, void *data )
-{
- reference_region * r = ( reference_region * ) n;
- uint32_t * count = ( uint32_t * ) data;
- *count += VectorLength( &(r->ranges) );
-}
-
-
-uint32_t count_ref_regions( BSTree * regions )
-{
- uint32_t res = 0;
- BSTreeForEach ( regions, false, count_ref_region_wrapper, &res );
- return res;
-}
-
-
-typedef struct foreach_ref_region_func
-{
- rc_t ( CC * on_region ) ( const char * name, uint32_t start, uint32_t end, void *data );
- const char * name;
- void * data;
- rc_t rc;
-} foreach_ref_region_func;
-
-
-static void CC foreach_range_vector_wrapper( void *item, void *data )
-{
- range * r = ( range * ) item;
- foreach_ref_region_func * func = ( foreach_ref_region_func * )data;
-
- if ( func->rc == 0 )
- {
- func->rc = func->on_region( func->name, r->start, r->end, func->data );
- }
-}
-
-
-static void CC foreach_ref_region_wrapper( BSTNode *n, void *data )
-{
- reference_region * r = ( reference_region * ) n;
- foreach_ref_region_func * func = ( foreach_ref_region_func * )data;
-
- if ( func->rc == 0 )
- {
- func->name = r->name;
- VectorForEach ( &(r->ranges), false, foreach_range_vector_wrapper, data );
- }
-}
-
-
-rc_t foreach_ref_region( BSTree * regions,
- rc_t ( CC * on_region ) ( const char * name, uint32_t start, uint32_t end, void *data ),
- void *data )
-{
- foreach_ref_region_func func;
-
- func.on_region = on_region;
- func.data = data;
- func.rc = 0;
- BSTreeForEach ( regions, false, foreach_ref_region_wrapper, &func );
- return func.rc;
-}
-
-
/* =========================================================================================== */
#if TOOLS_USE_SRAPATH != 0
@@ -966,8 +612,7 @@ static rc_t prepare_db_table( prepare_ctx *ctx,
rc = VDBManagerOpenTableRead ( vdb_mgr, &ctx->seq_tab, NULL, "%s", path );
if ( rc != 0 )
{
- PLOGERR(klogInt, (klogInt, rc, "failed to open '$(path)'",
- "path=%s", path));
+ PLOGERR( klogErr, ( klogErr, rc, "failed to open '$(path)'", "path=%s", path ) );
}
else {
ReportResetTable(path, ctx->seq_tab);
@@ -980,8 +625,9 @@ static rc_t prepare_db_table( prepare_ctx *ctx,
{
LOGERR( klogInt, rc, "VDatabaseOpenTableRead( SEQUENCE ) failed" );
}
- else {
- ReportResetDatabase(path, ctx->db);
+ else
+ {
+ ReportResetDatabase( path, ctx->db );
}
}
return rc;
diff --git a/tools/sra-pileup/cmdline_cmn.h b/tools/sra-pileup/cmdline_cmn.h
index b29c298..7dd71e2 100644
--- a/tools/sra-pileup/cmdline_cmn.h
+++ b/tools/sra-pileup/cmdline_cmn.h
@@ -30,16 +30,14 @@
#ifdef __cplusplus
extern "C" {
#endif
-#if 0
-}
-#endif
+
+#include "ref_regions.h"
#include <kapp/args.h>
#include <klib/rc.h>
#include <klib/log.h>
#include <klib/text.h>
-#include <klib/vector.h>
#include <klib/container.h>
#include <kfs/directory.h>
@@ -58,6 +56,9 @@ extern "C" {
#include <ctype.h>
#include <assert.h>
+#define OPTION_REF "aligned-region"
+#define ALIAS_REF "r"
+
typedef uint8_t align_tab_select;
enum { primary_ats = 1, secondary_ats = 2, evidence_ats = 4 };
@@ -65,6 +66,7 @@ typedef struct common_options
{
bool gzip_output;
bool bzip_output;
+ bool no_mt;
align_tab_select tab_select;
const char * output_file;
const char * input_file;
@@ -80,21 +82,11 @@ size_t CommonOptions_count( void );
/* get ref-ranges from the command-line and iterate them... */
rc_t init_ref_regions( BSTree * regions, Args * args );
-uint32_t count_ref_regions( BSTree * regions );
-
-rc_t parse_and_add_region( BSTree * regions, const char * s );
-
-rc_t foreach_ref_region( BSTree * regions,
- rc_t ( CC * on_range ) ( const char * name, uint32_t start, uint32_t end, void *data ),
- void *data );
-
-void check_ref_regions( BSTree * regions );
-void free_ref_regions( BSTree * regions );
rc_t foreach_argument( Args * args, KDirectory *dir, bool div_by_spotgrp, bool * empty,
rc_t ( CC * on_argument ) ( const char * path, const char * spot_group, void * data ), void * data );
-
+
typedef struct prepare_ctx
{
ReferenceIterator *ref_iter;
@@ -131,4 +123,9 @@ rc_t prepare_plset_iter( prepare_ctx *ctx,
rc_t parse_inf_file( Args * args );
+
+#ifdef __cplusplus
+}
#endif
+
+#endif /* _h_cmdline_cmn_ */
diff --git a/tools/sra-pileup/ref_regions.c b/tools/sra-pileup/ref_regions.c
new file mode 100644
index 0000000..d505760
--- /dev/null
+++ b/tools/sra-pileup/ref_regions.c
@@ -0,0 +1,491 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+#include "ref_regions.h"
+
+#include <klib/rc.h>
+#include <klib/text.h>
+#include <klib/vector.h>
+#include <klib/container.h>
+
+#include <stdlib.h>
+
+#include <os-native.h>
+#include <sysalloc.h>
+#include <strtol.h>
+
+/* =========================================================================================== */
+
+
+static int cmp_pchar( const char * a, const char * b )
+{
+ int res = 0;
+ if ( ( a != NULL )&&( b != NULL ) )
+ {
+ size_t len_a = string_size( a );
+ size_t len_b = string_size( b );
+ res = string_cmp ( a, len_a, b, len_b, ( len_a < len_b ) ? len_b : len_a );
+ }
+ return res;
+}
+
+
+/* =========================================================================================== */
+
+struct reference_range
+{
+ uint64_t start;
+ uint64_t end;
+} reference_range;
+
+
+static struct reference_range * make_range( const uint64_t start, const uint64_t end )
+{
+ struct reference_range *res = calloc( 1, sizeof *res );
+ if ( res != NULL )
+ {
+ res->start = start;
+ res->end = end;
+ }
+ return res;
+}
+
+
+static int cmp_range( const struct reference_range * a, const struct reference_range * b )
+{
+
+ int64_t res = ( a->start - b->start );
+ if ( res == 0 )
+ res = ( a->end - b->end );
+ if ( res < 0 )
+ return -1;
+ else if ( res > 0 )
+ return 1;
+ else return 0;
+}
+
+
+static bool range_overlapp( const struct reference_range * a, const struct reference_range * b )
+{
+ return ( !( ( b->end < a->start ) || ( b->start > a->end ) ) );
+}
+
+
+/* =========================================================================================== */
+
+struct reference_region
+{
+ BSTNode node;
+ const char * name;
+ Vector ranges;
+} reference_region;
+
+
+static struct reference_region * make_reference_region( const char *name )
+{
+ struct reference_region *res = calloc( sizeof *res, 1 );
+ if ( res != NULL )
+ {
+ res->name = string_dup_measure ( name, NULL );
+ VectorInit ( &res->ranges, 0, 5 );
+ }
+ return res;
+}
+
+
+static int CC cmp_range_wrapper( const void *item, const void *n )
+{ return cmp_range( item, n ); }
+
+
+static rc_t add_ref_region_range( struct reference_region * self, const uint64_t start, const uint64_t end )
+{
+ rc_t rc = 0;
+ struct reference_range *r = make_range( start, end );
+ if ( r == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ rc = VectorInsert ( &self->ranges, r, NULL, cmp_range_wrapper );
+ if ( rc != 0 )
+ free( r );
+ }
+ return rc;
+}
+
+
+#define RR_NAME 1
+#define RR_START 2
+#define RR_END 3
+
+
+static void put_c( char *s, size_t size, size_t *dst, char c )
+{
+ if ( *dst < ( size - 1 ) )
+ s[ *dst ] = c;
+ (*dst)++;
+}
+
+static void finish_txt( char *s, size_t size, size_t *dst )
+{
+ if ( *dst > size )
+ s[ size - 1 ] = 0;
+ else
+ s[ *dst ] = 0;
+ *dst = 0;
+}
+
+static uint64_t finish_num( char *s, size_t size, size_t *dst )
+{
+ uint64_t res = 0;
+ char *endp;
+ finish_txt( s, size, dst );
+ res = strtou64( s, &endp, 10 );
+ return res;
+}
+
+
+/* s = refname:1000-2000 */
+static void parse_definition( const char *s, char * name, size_t len,
+ uint64_t *start, uint64_t *end )
+{
+ size_t n = string_size( s );
+
+ *start = 0;
+ *end = 0;
+ name[ 0 ] = 0;
+ if ( n > 0 )
+ {
+ size_t i, st, dst = 0;
+ char tmp[ 32 ];
+ st = RR_NAME;
+ for ( i = 0; i < n; ++i )
+ {
+ char c = s[ i ];
+ switch( st )
+ {
+ case RR_NAME : if ( c == ':' )
+ {
+ finish_txt( name, len, &dst );
+ st = RR_START;
+ }
+ else
+ {
+ put_c( name, len, &dst, c );
+ }
+ break;
+
+ case RR_START : if ( c == '-' )
+ {
+ *start = finish_num( tmp, sizeof tmp, &dst );
+ st = RR_END;
+ }
+ else if ( ( c >= '0' )&&( c <= '9' ) )
+ {
+ put_c( tmp, sizeof tmp, &dst, c );
+ }
+ break;
+
+ case RR_END : if ( ( c >= '0' )&&( c <= '9' ) )
+ {
+ put_c( tmp, sizeof tmp, &dst, c );
+ }
+ break;
+ }
+ }
+ switch( st )
+ {
+ case RR_NAME : finish_txt( name, len, &dst );
+ break;
+
+ case RR_START : *start = finish_num( tmp, sizeof tmp, &dst );
+ break;
+
+ case RR_END : *end = finish_num( tmp, sizeof tmp, &dst );
+ break;
+ }
+ }
+}
+
+
+static void CC release_range_wrapper( void * item, void * data )
+{
+ free( item );
+}
+
+
+static void free_reference_region( struct reference_region * self )
+{
+ free( (void*)self->name );
+ VectorWhack ( &self->ranges, release_range_wrapper, NULL );
+ free( self );
+}
+
+
+static void check_ref_region_ranges( struct reference_region * self )
+{
+ uint32_t n = VectorLength( &self->ranges );
+ uint32_t i = 0;
+ struct reference_range * a = NULL;
+ while ( i < n )
+ {
+ struct reference_range * b = VectorGet ( &self->ranges, i );
+ bool remove = false;
+ if ( a != NULL )
+ {
+ remove = range_overlapp( a, b );
+ if ( remove )
+ {
+ struct reference_range * r;
+ a->end = b->end;
+ VectorRemove ( &self->ranges, i, (void**)&r );
+ free( r );
+ n--;
+ }
+ }
+ if ( !remove )
+ {
+ a = b;
+ ++i;
+ }
+ }
+}
+
+
+/* =========================================================================================== */
+
+static int CC reference_vs_pchar_wrapper( const void *item, const BSTNode *n )
+{
+ const struct reference_region * r = ( const struct reference_region * )n;
+ return cmp_pchar( (const char *)item, r->name );
+}
+
+static struct reference_region * find_reference_region( BSTree * regions, const char * name )
+{
+ return ( struct reference_region * ) BSTreeFind ( regions, name, reference_vs_pchar_wrapper );
+}
+
+static int CC ref_vs_ref_wrapper( const BSTNode *item, const BSTNode *n )
+{
+ const struct reference_region * a = ( const struct reference_region * )item;
+ const struct reference_region * b = ( const struct reference_region * )n;
+ return cmp_pchar( a->name, b->name );
+}
+
+
+rc_t add_region( BSTree * regions, const char * name, const uint64_t start, const uint64_t end )
+{
+ rc_t rc;
+
+ struct reference_region * r = find_reference_region( regions, name );
+ if ( r == NULL )
+ {
+ r = make_reference_region( name );
+ if ( r == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
+ else
+ rc = add_ref_region_range( r, start, end );
+ if ( rc == 0 )
+ rc = BSTreeInsert ( regions, (BSTNode *)r, ref_vs_ref_wrapper );
+ if ( rc != 0 )
+ free_reference_region( r );
+ }
+ else
+ {
+ rc = add_ref_region_range( r, start, end );
+ }
+ return rc;
+}
+
+
+rc_t parse_and_add_region( BSTree * regions, const char * s )
+{
+ uint64_t start, end;
+ char name[ 64 ];
+ parse_definition( s, name, sizeof name, &start, &end );
+ if ( name[ 0 ] == 0 )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
+ else
+ return add_region( regions, name, start, end );
+}
+
+
+/* =========================================================================================== */
+
+
+static void CC check_refrange_wrapper( BSTNode *n, void *data )
+{
+ check_ref_region_ranges( ( struct reference_region * ) n );
+}
+
+
+void check_ref_regions( BSTree * regions )
+{
+ BSTreeForEach ( regions, false, check_refrange_wrapper, NULL );
+}
+
+
+/* =========================================================================================== */
+
+
+static void CC release_ref_region_wrapper( BSTNode *n, void * data )
+{
+ free_reference_region( ( struct reference_region * ) n );
+}
+
+
+void free_ref_regions( BSTree * regions )
+{
+ BSTreeWhack ( regions, release_ref_region_wrapper, NULL );
+}
+
+
+/* =========================================================================================== */
+
+
+static void CC count_ref_region_wrapper( BSTNode *n, void *data )
+{
+ struct reference_region * r = ( struct reference_region * ) n;
+ uint32_t * count = ( uint32_t * ) data;
+ *count += VectorLength( &(r->ranges) );
+}
+
+
+uint32_t count_ref_regions( BSTree * regions )
+{
+ uint32_t res = 0;
+ BSTreeForEach ( regions, false, count_ref_region_wrapper, &res );
+ return res;
+}
+
+
+/* =========================================================================================== */
+
+
+typedef struct foreach_ref_region_func
+{
+ rc_t ( CC * on_region ) ( const char * name, uint32_t start, uint32_t end, void *data );
+ const char * name;
+ void * data;
+ rc_t rc;
+} foreach_ref_region_func;
+
+
+static void CC foreach_range_vector_wrapper( void *item, void *data )
+{
+ struct reference_range * r = ( struct reference_range * ) item;
+ foreach_ref_region_func * func = ( foreach_ref_region_func * )data;
+
+ if ( func->rc == 0 )
+ {
+ func->rc = func->on_region( func->name, r->start, r->end, func->data );
+ }
+}
+
+
+static void CC foreach_ref_region_wrapper( BSTNode *n, void *data )
+{
+ struct reference_region * r = ( struct reference_region * ) n;
+ foreach_ref_region_func * func = ( foreach_ref_region_func * )data;
+
+ if ( func->rc == 0 )
+ {
+ func->name = r->name;
+ VectorForEach ( &(r->ranges), false, foreach_range_vector_wrapper, data );
+ }
+}
+
+
+rc_t foreach_ref_region( BSTree * regions,
+ rc_t ( CC * on_region ) ( const char * name, uint32_t start, uint32_t end, void *data ),
+ void *data )
+{
+ foreach_ref_region_func func;
+
+ func.on_region = on_region;
+ func.data = data;
+ func.rc = 0;
+ BSTreeForEach ( regions, false, foreach_ref_region_wrapper, &func );
+ return func.rc;
+}
+
+
+/* =========================================================================================== */
+
+
+const struct reference_region * get_first_ref_node( const BSTree * regions )
+{
+ return ( const struct reference_region * ) BSTreeFirst ( regions );
+}
+
+
+const struct reference_region * get_next_ref_node( const struct reference_region * node )
+{
+ return ( const struct reference_region * ) BSTNodeNext( ( const BSTNode * ) node );
+}
+
+
+const char * get_ref_node_name( const struct reference_region * node )
+{
+ return ( node->name );
+}
+
+
+uint32_t get_ref_node_range_count( const struct reference_region * node )
+{
+ return VectorLength( &( node->ranges ) );
+}
+
+
+const struct reference_range * get_ref_range( const struct reference_region * node, uint32_t idx )
+{
+ const struct reference_range * res = NULL;
+ if ( node != NULL )
+ {
+ uint32_t n = VectorLength( &( node->ranges ) );
+ if ( ( idx >=0 ) && ( idx < n ) )
+ {
+ res = ( const struct reference_range * ) VectorGet ( &( node->ranges ), idx );
+ }
+ }
+ return res;
+}
+
+
+uint64_t get_ref_range_start( const struct reference_range * range )
+{
+ uint64_t res = 0;
+ if ( range != NULL )
+ res = range->start;
+ return res;
+}
+
+
+uint64_t get_ref_range_end( const struct reference_range * range )
+{
+ uint64_t res = 0;
+ if ( range != NULL )
+ res = range->end;
+ return res;
+}
diff --git a/interfaces/kns/manager.h b/tools/sra-pileup/ref_regions.h
similarity index 53%
copy from interfaces/kns/manager.h
copy to tools/sra-pileup/ref_regions.h
index 1c78d49..0081091 100644
--- a/interfaces/kns/manager.h
+++ b/tools/sra-pileup/ref_regions.h
@@ -23,63 +23,54 @@
* ===========================================================================
*
*/
-#ifndef _h_kns_manager_
-#define _h_kns_manager_
-#ifndef _h_kns_extern_
-#include <kns/extern.h>
-#endif
+#ifndef _h_ref_regions_
+#define _h_ref_regions_
-#ifndef _h_klib_defs_
-#include <klib/defs.h>
-#endif
+#include <klib/container.h>
#ifdef __cplusplus
extern "C" {
#endif
+rc_t parse_and_add_region( BSTree * regions, const char * s );
+
+rc_t add_region( BSTree * regions, const char * name, const uint64_t start, const uint64_t end );
+
+void check_ref_regions( BSTree * regions );
+
+void free_ref_regions( BSTree * regions );
+
+uint32_t count_ref_regions( BSTree * regions );
+
+rc_t foreach_ref_region( BSTree * regions,
+ rc_t ( CC * on_region ) ( const char * name, uint32_t start, uint32_t end, void *data ),
+ void *data );
+
-/*--------------------------------------------------------------------------
- * forwards
- */
+struct reference_region;
+const struct reference_region * get_first_ref_node( const BSTree * regions );
-/*--------------------------------------------------------------------------
- * KNSManager
- * manages the network system
- */
-typedef struct KNSManager KNSManager;
+const struct reference_region * get_next_ref_node( const struct reference_region * node );
+
+const char * get_ref_node_name( const struct reference_region * node );
+uint32_t get_ref_node_range_count( const struct reference_region * node );
-/* Make
- * create a manager instance
- */
-KNS_EXTERN rc_t CC KNSManagerMake ( KNSManager **mgr );
-/* AddRef
- * Release
- * ignores NULL references
- */
-KNS_EXTERN rc_t CC KNSManagerAddRef ( const KNSManager *self );
-KNS_EXTERN rc_t CC KNSManagerRelease ( const KNSManager *self );
+struct reference_range;
+const struct reference_range * get_ref_range( const struct reference_region * node, uint32_t idx );
-/* Avail - DEPRECATED
- * indicate to caller whether networking services are available
- * mainly here to support libcurl
- */
-KNS_EXTERN rc_t CC KNSManagerAvail ( const KNSManager *self );
+uint64_t get_ref_range_start( const struct reference_range * range );
-/* CurlVersion - DEPRECATED
- * indicate which version of curl is available
- * return this as a string, because this is how libcurl does it
- */
-KNS_EXTERN rc_t CC KNSManagerCurlVersion ( const KNSManager *self, const char ** version_string );
+uint64_t get_ref_range_end( const struct reference_range * range );
#ifdef __cplusplus
}
#endif
-#endif /* _h_kns_manager_ */
+#endif /* _h_ref_regions_ */
diff --git a/tools/sra-pileup/ref_walker.c b/tools/sra-pileup/ref_walker.c
new file mode 100644
index 0000000..1d74a9f
--- /dev/null
+++ b/tools/sra-pileup/ref_walker.c
@@ -0,0 +1,997 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "ref_regions.h"
+#include "ref_walker.h"
+
+#include <klib/rc.h>
+#include <klib/namelist.h>
+#include <klib/vector.h>
+#include <klib/log.h>
+#include <klib/out.h>
+
+#include <kfs/directory.h>
+#include <kfs/file.h>
+
+#include <vfs/manager.h>
+#include <vfs/path.h>
+
+#include <kdb/manager.h>
+
+#include <vdb/manager.h>
+#include <vdb/schema.h>
+
+#include <align/manager.h>
+#include <align/reference.h>
+#include <align/iterator.h>
+
+#include <sra/sraschema.h>
+
+#include <stdlib.h>
+#include <os-native.h>
+#include <sysalloc.h>
+
+#include <string.h>
+
+
+
+/* ================================================================================================ */
+
+
+/***************************************
+ N (0x4E) n (0x6E) <--> 0x0
+ A (0x41) a (0x61) <--> 0x1
+ C (0x43) c (0x63) <--> 0x2
+ M (0x4D) m (0x6D) <--> 0x3
+ G (0x47) g (0x67) <--> 0x4
+ R (0x52) r (0x72) <--> 0x5
+ S (0x53) s (0x73) <--> 0x6
+ V (0x56) v (0x76) <--> 0x7
+ T (0x54) t (0x74) <--> 0x8
+ W (0x57) w (0x77) <--> 0x9
+ Y (0x59) y (0x79) <--> 0xA
+ H (0x48) h (0x68) <--> 0xB
+ K (0x4B) k (0x6B) <--> 0xC
+ D (0x44) d (0x64) <--> 0xD
+ B (0x42) b (0x62) <--> 0xE
+ N (0x4E) n (0x6E) <--> 0xF
+***************************************/
+
+
+static char _4na_2_ascii_tab[] =
+{
+/* 0x0 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F */
+ 'N', 'A', 'C', 'M', 'G', 'R', 'S', 'V', 'T', 'W', 'Y', 'H', 'K', 'D', 'B', 'N',
+ 'n', 'a', 'c', 'm', 'g', 'r', 's', 'v', 't', 'w', 'y', 'h', 'k', 'd', 'b', 'n'
+};
+
+
+static char _4na_to_ascii( INSDC_4na_bin c, bool reverse )
+{
+ return _4na_2_ascii_tab[ ( c & 0x0F ) | ( reverse ? 0x10 : 0 ) ];
+}
+
+
+/* ================================================================================================ */
+
+
+struct ref_walker
+{
+ /* objects that stay alive during the livetime of the ref-walker */
+ KDirectory * dir;
+ const VDBManager * vmgr;
+ VSchema * vschema;
+ const AlignMgr * amgr;
+ VFSManager * vfs_mgr;
+ PlacementRecordExtendFuncs cb_block;
+
+ /* options for the Reference-Iterator */
+ int32_t min_mapq;
+ bool omit_quality;
+ bool prepared;
+ bool read_tlen;
+ bool process_dups;
+ bool use_seq_name;
+ bool no_skip;
+ bool primary_alignments;
+ bool secondary_alignments;
+ bool evidence_alignments;
+ char * spot_group;
+
+ /* manages the sources and regions requested */
+ VNamelist * sources;
+ BSTree regions;
+
+ /* enter/exit reference */
+ rc_t ( CC * on_enter_ref ) ( ref_walker_data * rwd );
+ rc_t ( CC * on_exit_ref ) ( ref_walker_data * rwd );
+
+ /* enter/exit reference-window */
+ rc_t ( CC * on_enter_ref_window ) ( ref_walker_data * rwd );
+ rc_t ( CC * on_exit_ref_window ) ( ref_walker_data * rwd );
+
+ /* enter/exit reference-pos */
+ rc_t ( CC * on_enter_ref_pos ) ( ref_walker_data * rwd );
+ rc_t ( CC * on_exit_ref_pos ) ( ref_walker_data * rwd );
+
+ /* enter/exit spot-group */
+ rc_t ( CC * on_enter_spot_group ) ( ref_walker_data * rwd );
+ rc_t ( CC * on_exit_spot_group ) ( ref_walker_data * rwd );
+
+ /* alignment */
+ rc_t ( CC * on_alignment ) ( ref_walker_data * rwd );
+
+ /* callbacks for different events */
+} ref_walker;
+
+
+static void ref_walker_release( struct ref_walker * self )
+{
+ KDirectoryRelease( self->dir );
+ VDBManagerRelease( self->vmgr );
+ VSchemaRelease( self->vschema );
+ AlignMgrRelease ( self->amgr );
+ VFSManagerRelease ( self->vfs_mgr );
+ VNamelistRelease ( self->sources );
+ free_ref_regions( &self->regions );
+ free( ( void * )self->spot_group );
+}
+
+
+/* ================================================================================================ */
+/* data/callbacks used by Reference-iterator */
+
+
+typedef struct walker_col_ids
+{
+ uint32_t idx_quality;
+ uint32_t idx_ref_orientation;
+ uint32_t idx_read_filter;
+ uint32_t idx_template_len;
+} walker_col_ids;
+
+
+typedef struct walker_rec
+{
+ bool reverse; /* orientation towards reference ( false...in ref-orientation / true...reverse) */
+ int32_t tlen; /* template-len, for statistical analysis */
+ uint8_t * quality; /* ptr to quality... ( for sam-output ) */
+} walker_rec;
+
+
+static rc_t read_base_and_len( struct VCursor const *curs,
+ uint32_t column_idx,
+ int64_t row_id,
+ const void ** base,
+ uint32_t * len )
+{
+ uint32_t elem_bits, boff, len_intern;
+ const void * ptr;
+ rc_t rc = VCursorCellDataDirect ( curs, row_id, column_idx, &elem_bits, &ptr, &boff, &len_intern );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VCursorCellDataDirect() failed" );
+ }
+ else
+ {
+ if ( len != NULL ) *len = len_intern;
+ if ( base != NULL ) *base = ptr;
+ }
+ return rc;
+}
+
+
+static rc_t CC populate_data( void *obj, const PlacementRecord *placement,
+ struct VCursor const *curs, INSDC_coord_zero ref_window_start, INSDC_coord_len ref_window_len,
+ void *data, void * placement_ctx )
+{
+ walker_rec * rec = obj;
+ struct ref_walker * walker = data;
+ walker_col_ids * col_ids = placement_ctx;
+ rc_t rc = 0;
+
+ rec->quality = NULL;
+ if ( !walker->process_dups )
+ {
+ const uint8_t * read_filter;
+ uint32_t read_filter_len;
+ rc = read_base_and_len( curs, col_ids->idx_read_filter, placement->id,
+ (const void **)&read_filter, &read_filter_len );
+ if ( rc == 0 )
+ {
+ if ( ( *read_filter == SRA_READ_FILTER_REJECT )||
+ ( *read_filter == SRA_READ_FILTER_CRITERIA ) )
+ {
+ rc = RC( rcAlign, rcType, rcAccessing, rcId, rcIgnored );
+ }
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ const bool * orientation;
+ rc = read_base_and_len( curs, col_ids->idx_ref_orientation, placement->id,
+ (const void **)&orientation, NULL );
+ if ( rc == 0 )
+ rec->reverse = *orientation;
+ }
+
+ if ( rc == 0 && !walker->omit_quality )
+ {
+ const uint8_t * quality;
+ uint32_t quality_len;
+
+ rc = read_base_and_len( curs, col_ids->idx_quality, placement->id,
+ (const void **)&quality, &quality_len );
+ if ( rc == 0 )
+ {
+ rec->quality = ( uint8_t * )rec;
+ rec->quality += sizeof ( * rec );
+ memcpy( rec->quality, quality, quality_len );
+ }
+ }
+
+ if ( rc == 0 && walker->read_tlen )
+ {
+ const int32_t * tlen;
+ uint32_t tlen_len;
+
+ rc = read_base_and_len( curs, col_ids->idx_template_len, placement->id,
+ (const void **)&tlen, &tlen_len );
+ if ( rc == 0 && tlen_len > 0 )
+ rec->tlen = *tlen;
+ else
+ rec->tlen = 0;
+ }
+ else
+ rec->tlen = 0;
+
+ return rc;
+}
+
+
+static rc_t CC alloc_size( struct VCursor const *curs, int64_t row_id, size_t * size,
+ void *data, void * placement_ctx )
+{
+ rc_t rc = 0;
+ walker_rec * rec;
+ struct ref_walker * walker = data;
+ walker_col_ids * col_ids = placement_ctx;
+ *size = ( sizeof *rec );
+
+ if ( !walker->omit_quality )
+ {
+ uint32_t q_len;
+ rc = read_base_and_len( curs, col_ids->idx_quality, row_id, NULL, &q_len );
+ if ( rc == 0 )
+ *size += q_len;
+ }
+ return rc;
+}
+
+
+/* ================================================================================================ */
+
+
+static rc_t ref_walker_init( struct ref_walker * self )
+{
+ rc_t rc = KDirectoryNativeDir( &self->dir );
+ if ( rc == 0 )
+ rc = VDBManagerMakeRead ( &self->vmgr, self->dir );
+ if ( rc == 0 )
+ rc = VDBManagerMakeSRASchema( self->vmgr, &self->vschema );
+ if ( rc == 0 )
+ rc = AlignMgrMakeRead ( &self->amgr );
+ if ( rc == 0 )
+ rc = VFSManagerMake ( &self->vfs_mgr );
+ if ( rc == 0 )
+ rc = VNamelistMake ( &self->sources, 10 );
+
+ self->cb_block.data = self;
+ self->cb_block.destroy = NULL;
+ self->cb_block.populate = populate_data;
+ self->cb_block.alloc_size = alloc_size;
+ self->cb_block.fixed_size = 0;
+
+ BSTreeInit( &self->regions );
+ self->primary_alignments = true;
+
+ if ( rc != 0 )
+ ref_walker_release( self );
+ return rc;
+}
+
+
+/* ================================================================================================ */
+
+
+rc_t ref_walker_create( struct ref_walker ** self )
+{
+ rc_t rc;
+ if ( self == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ else
+ {
+ struct ref_walker * o = calloc( 1, sizeof **self );
+ *self = NULL;
+ if ( o == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ rc = ref_walker_init( o );
+ if ( rc == 0 )
+ {
+ *self = o;
+ }
+ else
+ free( ( void * )o );
+ }
+ }
+ return rc;
+}
+
+
+/* ================================================================================================ */
+
+
+rc_t ref_walker_set_min_mapq( struct ref_walker * self, int32_t min_mapq )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->min_mapq = min_mapq;
+ return 0;
+}
+
+
+rc_t ref_walker_set_omit_quality( struct ref_walker * self, bool omit_quality )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->omit_quality = omit_quality;
+ return 0;
+}
+
+
+rc_t ref_walker_set_read_tlen( struct ref_walker * self, bool read_tlen )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->read_tlen = read_tlen;
+ return 0;
+}
+
+
+rc_t ref_walker_set_process_dups( struct ref_walker * self, bool process_dups )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->process_dups = process_dups;
+ return 0;
+}
+
+
+rc_t ref_walker_set_use_seq_name( struct ref_walker * self, bool use_seq_name )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->use_seq_name = use_seq_name;
+ return 0;
+}
+
+
+rc_t ref_walker_set_no_skip( struct ref_walker * self, bool no_skip )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->no_skip = no_skip;
+ return 0;
+}
+
+
+rc_t ref_walker_set_primary_alignments( struct ref_walker * self, bool enabled )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->primary_alignments = enabled;
+ return 0;
+}
+
+
+rc_t ref_walker_set_secondary_alignments( struct ref_walker * self, bool enabled )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->secondary_alignments = enabled;
+ return 0;
+}
+
+
+rc_t ref_walker_set_evidence_alignments( struct ref_walker * self, bool enabled )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->evidence_alignments = enabled;
+ return 0;
+}
+
+
+rc_t ref_walker_set_spot_group( struct ref_walker * self, const char * spot_group )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ self->spot_group = string_dup ( spot_group, string_size( spot_group ) );
+ return 0;
+}
+
+
+/* ================================================================================================ */
+
+
+rc_t ref_walker_set_callbacks( struct ref_walker * self, ref_walker_callbacks * callbacks )
+{
+ if ( self == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcSelf, rcNull );
+ if ( callbacks == NULL )
+ return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+
+ self->on_enter_ref = callbacks->on_enter_ref;
+ self->on_exit_ref = callbacks->on_exit_ref;
+ self->on_enter_ref_window = callbacks->on_enter_ref_window;
+ self->on_exit_ref_window = callbacks->on_exit_ref_window;
+ self->on_enter_ref_pos = callbacks->on_enter_ref_pos;
+ self->on_exit_ref_pos = callbacks->on_exit_ref_pos;
+ self->on_enter_spot_group = callbacks->on_enter_spot_group;
+ self->on_exit_spot_group = callbacks->on_exit_spot_group;
+ self->on_alignment = callbacks->on_alignment;
+
+ return 0;
+}
+
+
+/* ================================================================================================ */
+
+
+rc_t ref_walker_add_source( struct ref_walker * self, const char * src )
+{
+ rc_t rc = 0;
+ if ( self == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ else
+ rc = VNamelistAppend ( self->sources, src );
+ return rc;
+}
+
+
+rc_t ref_walker_parse_and_add_range( struct ref_walker * self, const char * range )
+{
+ rc_t rc = 0;
+ if ( self == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ else
+ rc = parse_and_add_region( &self->regions, range );
+ return rc;
+}
+
+
+rc_t ref_walker_add_range( struct ref_walker * self, const char * name, const uint64_t start, const uint64_t end )
+{
+ rc_t rc = 0;
+ if ( self == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ else
+ rc = add_region( &self->regions, name, start, end );
+ return rc;
+}
+
+
+static rc_t ref_walker_prepare_1_src( struct ref_walker * self, const char * name )
+{
+ VPath * path = NULL;
+ const VPath * local_cache = NULL;
+ const KFile * remote_file = NULL;
+ rc_t rc = VFSManagerResolveSpec ( self->vfs_mgr, name, &path, &remote_file, &local_cache, true );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogErr, ( klogErr, rc, "cannot resolve '$(n)' via VFSManager", "n=%s", name ) );
+ }
+ else
+ {
+ char buffer[ 4096 ];
+ size_t num_read;
+ rc = VPathReadPath ( path, buffer, sizeof buffer, &num_read );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogErr, ( klogErr, rc, "cannot read path from vpath for '$(n)'", "n=%s", name ) );
+ }
+ else
+ {
+ if ( rc == 0 )
+ {
+ int path_type = ( VDBManagerPathType ( self->vmgr, "%s", buffer ) & ~ kptAlias );
+ if ( path_type == kptDatabase )
+ {
+ const ReferenceList * reflist;
+ uint32_t options = ( ereferencelist_usePrimaryIds |
+ ereferencelist_useSecondaryIds |
+ ereferencelist_useEvidenceIds );
+ rc = ReferenceList_MakePath( &reflist, self->vmgr, name, options, 0, NULL, 0 );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogErr, ( klogErr, rc, "cannot create ReferenceList for '$(n)'", "n=%s", name ) );
+ }
+ else
+ {
+ uint32_t count;
+ rc = ReferenceList_Count( reflist, &count );
+ if ( rc != 0 )
+ {
+ PLOGERR( klogErr, ( klogErr, rc, "ReferenceList_Count() for '$(n)' failed", "n=%s", name ) );
+ }
+ else
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const ReferenceObj * refobj;
+ rc = ReferenceList_Get( reflist, &refobj, idx );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "ReferenceList_Get() failed" );
+ }
+ else
+ {
+ const char * seqid;
+ rc = ReferenceObj_SeqId( refobj, &seqid );
+ if ( rc == 0 )
+ {
+ INSDC_coord_len seqlen;
+ rc = ReferenceObj_SeqLength( refobj, &seqlen );
+ if ( rc == 0 )
+ {
+ rc = add_region( &self->regions, seqid, 0, seqlen - 1 );
+ }
+ }
+ ReferenceObj_Release( refobj );
+ }
+ }
+ }
+ ReferenceList_Release( reflist );
+ }
+ }
+ }
+ }
+ KFileRelease( remote_file );
+ VPathRelease ( local_cache );
+ VPathRelease ( path );
+ }
+ return rc;
+}
+
+
+static rc_t ref_walker_prepare( struct ref_walker * self )
+{
+ rc_t rc = 0;
+ if ( self == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ else
+ {
+ uint32_t s_count = 0;
+ rc = VNameListCount ( self->sources, &s_count );
+ if ( rc == 0 && s_count > 0 )
+ {
+ uint32_t r_count = count_ref_regions( &self->regions );
+ if ( r_count == 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; idx < s_count && rc == 0; ++idx )
+ {
+ const char * name = NULL;
+ rc = VNameListGet ( self->sources, idx, &name );
+ if ( rc == 0 && name != NULL )
+ rc = ref_walker_prepare_1_src( self, name );
+ }
+ }
+ }
+ check_ref_regions( &self->regions );
+ self->prepared = ( rc == 0 );
+ }
+ return rc;
+}
+
+
+static uint32_t ref_walker_make_reflist_options( struct ref_walker * self )
+{
+ uint32_t res = ereferencelist_4na;
+
+ if ( self->primary_alignments )
+ res |= ereferencelist_usePrimaryIds;
+
+ if ( self->secondary_alignments )
+ res |= ereferencelist_useSecondaryIds;
+
+ if ( self->evidence_alignments )
+ res |= ereferencelist_useEvidenceIds;
+
+ return res;
+}
+
+
+static rc_t make_cursor_ids( Vector * cursor_id_vector, walker_col_ids ** cursor_ids )
+{
+ rc_t rc;
+ walker_col_ids * ids = malloc( sizeof * ids );
+ if ( ids == NULL )
+ rc = RC ( rcApp, rcNoTarg, rcOpening, rcMemory, rcExhausted );
+ else
+ {
+ rc = VectorAppend ( cursor_id_vector, NULL, ids );
+ if ( rc != 0 )
+ free( ids );
+ else
+ *cursor_ids = ids;
+ }
+ return rc;
+}
+
+#define COL_QUALITY "QUALITY"
+#define COL_REF_ORIENTATION "REF_ORIENTATION"
+#define COL_READ_FILTER "READ_FILTER"
+#define COL_TEMPLATE_LEN "TEMPLATE_LEN"
+
+static rc_t add_required_columns( struct ref_walker * self, const VTable *tbl, const VCursor ** cursor,
+ walker_col_ids * cursor_ids )
+{
+ rc_t rc = VTableCreateCursorRead ( tbl, cursor );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VTableCreateCursorRead() failed" );
+ }
+
+ if ( rc == 0 && !self->omit_quality )
+ {
+ rc = VCursorAddColumn ( *cursor, &cursor_ids->idx_quality, COL_QUALITY );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VCursorAddColumn(QUALITY) failed" );
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ rc = VCursorAddColumn ( *cursor, &cursor_ids->idx_ref_orientation, COL_REF_ORIENTATION );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VCursorAddColumn(REF_ORIENTATION) failed" );
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ rc = VCursorAddColumn ( *cursor, &cursor_ids->idx_read_filter, COL_READ_FILTER );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VCursorAddColumn(READ_FILTER) failed" );
+ }
+ }
+
+ if ( rc == 0 && self->read_tlen )
+ {
+ rc = VCursorAddColumn ( *cursor, &cursor_ids->idx_template_len, COL_TEMPLATE_LEN );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VCursorAddColumn(TEMPLATE_LEN) failed" );
+ }
+ }
+ return rc;
+}
+
+
+static rc_t ref_walker_add_iterator( struct ref_walker * self, const char * ref_name,
+ uint64_t start, uint64_t end, const char * src_name, Vector * cur_id_vector,
+ const VDatabase *db, const ReferenceObj * ref_obj, ReferenceIterator * ref_iter,
+ const char * table_name, align_id_src id_selector )
+{
+ walker_col_ids * cursor_ids;
+ rc_t rc = make_cursor_ids( cur_id_vector, &cursor_ids );
+ if ( rc == 0 )
+ {
+ const VTable *tbl;
+ rc_t rc = VDatabaseOpenTableRead ( db, &tbl, table_name );
+ if ( rc == 0 )
+ {
+ const VCursor *cursor;
+ rc = add_required_columns( self, tbl, &cursor, cursor_ids );
+ if ( rc == 0 )
+ {
+ rc = ReferenceIteratorAddPlacements ( ref_iter, /* the outer ref-iter */
+ ref_obj, /* the ref-obj for this chromosome */
+ start - 1, /* start ( zero-based ) */
+ ( end - start ) + 1, /* length */
+ NULL, /* ref-cursor */
+ cursor, /* align-cursor */
+ id_selector, /* which id's */
+ self->spot_group, /* what read-group */
+ cursor_ids /* placement-context */
+ );
+ VCursorRelease( cursor );
+ }
+ VTableRelease ( tbl );
+ }
+ }
+ return rc;
+}
+
+
+#define TBL_PRIM "PRIMARY_ALIGNMENT"
+#define TBL_SEC "SECONDARY_ALIGNMENT"
+#define TBL_EV "EVIDENCE_ALIGNMENT"
+
+rc_t CC Quitting( void );
+
+static rc_t ref_walker_walk_alignment( struct ref_walker * self,
+ ReferenceIterator * ref_iter,
+ const PlacementRecord * rec,
+ ref_walker_data * rwd )
+{
+ walker_rec * xrec = PlacementRecordCast ( rec, placementRecordExtension1 );
+ rwd->state = ReferenceIteratorState ( ref_iter, &rwd->seq_pos );
+ rwd->valid = ( ( rwd->state & align_iter_invalid ) == 0 );
+ rwd->reverse = xrec->reverse;
+ rwd->first = ( ( rwd->state & align_iter_first ) == align_iter_first );
+ rwd->last = ( ( rwd->state & align_iter_last ) == align_iter_last );
+ rwd->skip = ( ( rwd->state & align_iter_skip ) == align_iter_skip );
+ rwd->match = ( ( rwd->state & align_iter_match ) == align_iter_match );
+ rwd->bin_alignment_base = ( rwd->state & 0x0F );
+ rwd->ascii_alignment_base = _4na_to_ascii( rwd->state, rwd->reverse );
+ if ( !self->omit_quality )
+ rwd->quality = xrec->quality[ rwd->seq_pos ];
+ rwd->mapq = rec->mapq;
+ return self->on_alignment( rwd );
+}
+
+
+static rc_t ref_walker_walk_ref_range( struct ref_walker * self, ref_walker_data * rwd )
+{
+ ReferenceIterator * ref_iter;
+ rc_t rc = AlignMgrMakeReferenceIterator ( self->amgr, &ref_iter, &self->cb_block, self->min_mapq ); /* align/iterator.h */
+ if ( rc == 0 )
+ {
+ /* construct the reference iterator */
+
+ uint32_t idx, count;
+ uint32_t reflist_options = ref_walker_make_reflist_options( self ); /* above */
+ Vector cur_id_vector;
+ VectorInit ( &cur_id_vector, 0, 12 );
+
+ rc = VNameListCount ( self->sources, &count );
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * src_name = NULL;
+ rc = VNameListGet ( self->sources, idx, &src_name );
+ if ( rc == 0 && src_name != NULL )
+ {
+ const VDatabase *db;
+ rc = VDBManagerOpenDBRead ( self->vmgr, &db, self->vschema, "%s", src_name );
+ if ( rc == 0 )
+ {
+ const ReferenceList * ref_list;
+ rc = ReferenceList_MakeDatabase( &ref_list, db, reflist_options, 0, NULL, 0 );
+ if ( rc == 0 )
+ {
+ const ReferenceObj * ref_obj;
+ rc = ReferenceList_Find( ref_list, &ref_obj, rwd->ref_name, string_size( rwd->ref_name ) );
+ if ( rc == 0 )
+ {
+ INSDC_coord_len len;
+ rc = ReferenceObj_SeqLength( ref_obj, &len );
+ if ( rc == 0 )
+ {
+ if ( rwd->ref_start == 0 )
+ rwd->ref_start = 1;
+ if ( ( rwd->ref_end == 0 )||( rwd->ref_end > len + 1 ) )
+ rwd->ref_end = ( len - rwd->ref_start ) + 1;
+
+ if ( self->primary_alignments )
+ rc = ref_walker_add_iterator( self, rwd->ref_name, rwd->ref_start, rwd->ref_end, src_name,
+ &cur_id_vector, db, ref_obj, ref_iter, TBL_PRIM, primary_align_ids );
+
+ if ( rc == 0 && self->secondary_alignments )
+ rc = ref_walker_add_iterator( self, rwd->ref_name, rwd->ref_start, rwd->ref_end, src_name,
+ &cur_id_vector, db, ref_obj, ref_iter, TBL_SEC, secondary_align_ids );
+
+ if ( rc == 0 && self->evidence_alignments )
+ rc = ref_walker_add_iterator( self, rwd->ref_name, rwd->ref_start, rwd->ref_end, src_name,
+ &cur_id_vector, db, ref_obj, ref_iter, TBL_EV, evidence_align_ids );
+
+ }
+ ReferenceObj_Release( ref_obj );
+ }
+ ReferenceList_Release( ref_list );
+ }
+ VDatabaseRelease( db );
+ }
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ /* walk the reference iterator */
+ struct ReferenceObj const * ref_obj;
+ rc = ReferenceIteratorNextReference( ref_iter, NULL, NULL, &ref_obj );
+ if ( rc == 0 && ref_obj != NULL )
+ {
+ if ( self->use_seq_name )
+ rc = ReferenceObj_Name( ref_obj, &rwd->ref_name );
+ else
+ rc = ReferenceObj_SeqId( ref_obj, &rwd->ref_name );
+ if ( rc == 0 )
+ {
+ INSDC_coord_zero first_pos;
+ INSDC_coord_len len;
+ rc_t rc_w = ReferenceIteratorNextWindow ( ref_iter, &first_pos, &len );
+ while ( rc == 0 && rc_w == 0 )
+ {
+ rc_t rc_p = ReferenceIteratorNextPos ( ref_iter, !self->no_skip );
+ if ( rc_p == 0 )
+ {
+ rc = ReferenceIteratorPosition ( ref_iter, &rwd->pos, &rwd->depth, &rwd->bin_ref_base );
+ if ( rwd->depth > 0 && rc == 0 )
+ {
+ rc_t rc_sg = 0;
+ rwd->ascii_ref_base = _4na_to_ascii( rwd->bin_ref_base, false );
+ if ( self->on_enter_ref_pos != NULL )
+ rc = self->on_enter_ref_pos( rwd );
+
+ while ( rc_sg == 0 && rc == 0 )
+ {
+ rc_sg = ReferenceIteratorNextSpotGroup ( ref_iter, &rwd->spot_group, &rwd->spot_group_len );
+ if ( rc_sg == 0 )
+ {
+ rc_t rc_pr = 0;
+ if ( self->on_enter_spot_group != NULL )
+ rc = self->on_enter_spot_group( rwd );
+
+ while ( rc == 0 && rc_pr == 0 )
+ {
+ const PlacementRecord * rec;
+ rc_pr = ReferenceIteratorNextPlacement ( ref_iter, &rec );
+ if ( rc_pr == 0 && self->on_alignment != NULL )
+ rc = ref_walker_walk_alignment( self, ref_iter, rec, rwd );
+ }
+
+ if ( self->on_exit_spot_group != NULL )
+ rc = self->on_exit_spot_group( rwd );
+ }
+ }
+ if ( self->on_exit_ref_pos != NULL )
+ rc = self->on_exit_ref_pos( rwd );
+ }
+ rc = Quitting();
+ }
+ }
+ }
+ }
+ }
+
+ /* free cur_id_vector */
+
+ ReferenceIteratorRelease ( ref_iter );
+ }
+ return rc;
+}
+
+
+static rc_t ref_walker_walk_ref_region( struct ref_walker * self,
+ const struct reference_region * region, ref_walker_data * rwd )
+{
+ rc_t rc = 0;
+ uint32_t idx, count = get_ref_node_range_count( region );
+ rwd->ref_name = get_ref_node_name( region );
+
+ rwd->ref_start = 0;
+ rwd->ref_end = 0;
+ rwd->pos = 0;
+ rwd->depth = 0;
+ rwd->bin_ref_base = 0;
+ rwd->ascii_ref_base = 0;
+ rwd->spot_group = NULL;
+ rwd->spot_group_len = 0;
+ rwd->state = 0;
+ rwd->reverse = 0;
+
+ if ( self->on_enter_ref != NULL )
+ rc = self->on_enter_ref( rwd );
+
+ if ( rc == 0 )
+ {
+ for ( idx = 0; idx < count; ++ idx )
+ {
+ const struct reference_range * range = get_ref_range( region, idx );
+ if ( range != NULL )
+ {
+ rwd->ref_start = get_ref_range_start( range );
+ rwd->ref_end = get_ref_range_end( range );
+ if ( self->on_enter_ref_window != NULL )
+ rc = self->on_enter_ref_window( rwd );
+ if ( rc == 0 )
+ {
+ rc = ref_walker_walk_ref_range( self, rwd );
+ if ( rc == 0 && self->on_exit_ref_window != NULL )
+ rc = self->on_exit_ref_window( rwd );
+ }
+ rwd->ref_start = 0;
+ rwd->ref_end = 0;
+ }
+ }
+
+ if ( self->on_exit_ref != NULL )
+ rc = self->on_exit_ref( rwd );
+ }
+ rwd->ref_name = NULL;
+
+ return rc;
+}
+
+
+rc_t ref_walker_walk( struct ref_walker * self, void * data )
+{
+ rc_t rc = 0;
+ if ( self == NULL )
+ rc = RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
+ else
+ {
+ if ( !self->prepared )
+ rc = ref_walker_prepare( self );
+
+ if ( rc == 0 && self->prepared )
+ {
+ const struct reference_region * region = get_first_ref_node( &self->regions );
+ while ( region != NULL && rc == 0 )
+ {
+ ref_walker_data rwd; /* this record will be passed to all the enter/exit callback's */
+ rwd.data = data;
+
+ rc = ref_walker_walk_ref_region( self, region, &rwd );
+ if ( rc == 0 )
+ region = get_next_ref_node( region );
+ }
+ }
+ }
+ return rc;
+}
+
+
+rc_t ref_walker_destroy( struct ref_walker * self )
+{
+ rc_t rc = 0;
+ if ( self != NULL )
+ {
+ ref_walker_release( self );
+ free( ( void * ) self );
+ }
+ return rc;
+}
+
diff --git a/tools/sra-pileup/ref_walker.h b/tools/sra-pileup/ref_walker.h
new file mode 100644
index 0000000..2067995
--- /dev/null
+++ b/tools/sra-pileup/ref_walker.h
@@ -0,0 +1,128 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _h_ref_walker_
+#define _h_ref_walker_
+
+#include <klib/container.h>
+#include <insdc/sra.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct ref_walker;
+
+
+/* create the ref-walker ( not ref-counted ) */
+rc_t ref_walker_create( struct ref_walker ** self );
+
+
+typedef struct ref_walker_data
+{
+ /* for the reference - level */
+ const char * ref_name;
+ uint64_t ref_start;
+ uint64_t ref_end;
+
+ /* for the position - level */
+ INSDC_coord_zero pos;
+ uint32_t depth;
+ INSDC_4na_bin bin_ref_base;
+ char ascii_ref_base;
+
+ /* for the spot-group - level */
+ const char * spot_group;
+ size_t spot_group_len;
+
+ /* for the alignment - level */
+ int32_t state, mapq;
+ INSDC_4na_bin bin_alignment_base;
+ char ascii_alignment_base;
+ char quality;
+ INSDC_coord_zero seq_pos;
+ bool reverse, first, last, skip, match, valid;
+
+ void * data;
+} ref_walker_data;
+
+
+typedef rc_t ( CC * ref_walker_callback )( ref_walker_data * rwd );
+
+
+typedef struct ref_walker_callbacks
+{
+ ref_walker_callback on_enter_ref;
+ ref_walker_callback on_exit_ref;
+
+ ref_walker_callback on_enter_ref_window;
+ ref_walker_callback on_exit_ref_window;
+
+ ref_walker_callback on_enter_ref_pos;
+ ref_walker_callback on_exit_ref_pos;
+
+ ref_walker_callback on_enter_spot_group;
+ ref_walker_callback on_exit_spot_group;
+
+ ref_walker_callback on_alignment;
+} ref_walker_callbacks;
+
+
+/* set boolean / numeric parameters */
+rc_t ref_walker_set_min_mapq( struct ref_walker * self, int32_t min_mapq );
+rc_t ref_walker_set_omit_quality( struct ref_walker * self, bool omit_quality );
+rc_t ref_walker_set_read_tlen( struct ref_walker * self, bool read_tlen );
+rc_t ref_walker_set_process_dups( struct ref_walker * self, bool process_dups );
+rc_t ref_walker_set_use_seq_name( struct ref_walker * self, bool use_seq_name );
+rc_t ref_walker_set_no_skip( struct ref_walker * self, bool no_skip );
+rc_t ref_walker_set_primary_alignments( struct ref_walker * self, bool enabled );
+rc_t ref_walker_set_secondary_alignments( struct ref_walker * self, bool enabled );
+rc_t ref_walker_set_evidence_alignments( struct ref_walker * self, bool enabled );
+rc_t ref_walker_set_spot_group( struct ref_walker * self, const char * spot_group );
+
+/* set callbacks */
+rc_t ref_walker_set_callbacks( struct ref_walker * self, ref_walker_callbacks * callbacks );
+
+/* add_sources and ranges */
+rc_t ref_walker_add_source( struct ref_walker * self, const char * src );
+
+rc_t ref_walker_parse_and_add_range( struct ref_walker * self, const char * range );
+
+rc_t ref_walker_add_range( struct ref_walker * self, const char * name, const uint64_t start, const uint64_t end );
+
+
+/* walk the sources/ranges by calling the supplied call-backs, passing data to the callbacks */
+rc_t ref_walker_walk( struct ref_walker * self, void * data );
+
+
+/* destroy the ref-walker */
+rc_t ref_walker_destroy( struct ref_walker * self );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_ref_walker_ */
diff --git a/tools/sra-pileup/reref.c b/tools/sra-pileup/reref.c
index b282f1a..a3eb225 100644
--- a/tools/sra-pileup/reref.c
+++ b/tools/sra-pileup/reref.c
@@ -27,12 +27,20 @@
#include "reref.h"
#include <klib/text.h>
+#include <klib/log.h>
+
#include <kfs/file.h>
+
#include <vfs/manager.h>
#include <vfs/path.h>
#include <vfs/resolver.h>
+
#include <align/reference.h>
+#include <vdb/manager.h>
+#include <vdb/vdb-priv.h>
+#include <kdb/manager.h>
+
#include <sysalloc.h>
#include <stdlib.h>
@@ -248,17 +256,23 @@ static rc_t resolve_accession( VFSManager * vfs_mgr, const char * accession, con
{
VResolver * resolver;
rc_t rc = VFSManagerGetResolver( vfs_mgr, &resolver );
- if ( rc == 0 )
+ if ( rc != 0 )
+ KOutMsg( "cannot get VResolver from VFSManger for '%s'\n", accession );
+ else
{
VPath * vpath;
- rc = VFSManagerMakePath( vfs_mgr, &vpath, accession );
- if ( rc == 0 )
+ rc = VFSManagerMakePath( vfs_mgr, &vpath, "ncbi-acc:%s?vdb-ctx=refseq", accession );
+ if ( rc != 0 )
+ KOutMsg( "cannot make VPath from VFSManger for '%s'\n", accession );
+ else
{
const VPath * local;
const VPath * remote;
rc = VResolverQuery ( resolver, eProtocolHttp, vpath, &local, &remote, NULL );
- if ( rc == 0 )
+ if ( rc != 0 )
+ KOutMsg( "cannot resolve '%s'\n", accession );
+ else
{
if ( local != NULL )
rc = VPathMakeString( local, path );
diff --git a/tools/sra-pileup/reref.h b/tools/sra-pileup/reref.h
index 13e3136..85bbd5d 100644
--- a/tools/sra-pileup/reref.h
+++ b/tools/sra-pileup/reref.h
@@ -37,11 +37,6 @@ extern "C" {
#include <klib/out.h>
#include <klib/rc.h>
-#include <klib/log.h>
-
-#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
rc_t report_on_reference( Args * args, bool extended );
diff --git a/tools/sra-pileup/sam-aligned.c b/tools/sra-pileup/sam-aligned.c
index d0b4667..92af47f 100644
--- a/tools/sra-pileup/sam-aligned.c
+++ b/tools/sra-pileup/sam-aligned.c
@@ -68,6 +68,8 @@ const char * EV_AL_TABLE = "EVIDENCE_ALIGNMENT";
ALIGNMENT_COUNT X X - X
SEQ_NAME X
MAPQ X
+ ALIGN_GROUP X - - -
+
-------------------------------------------------------------------------------------------*/
@@ -99,6 +101,7 @@ const char * EV_AL_TABLE = "EVIDENCE_ALIGNMENT";
#define COL_AL_COUNT "(U8)ALIGNMENT_COUNT"
#define COL_SEQ_NAME "(ascii)SEQ_NAME"
#define COL_MAPQ "(I32)MAPQ"
+#define COL_ALIGN_GROUP "(ascii)ALIGN_GROUP"
enum align_table_type
{
@@ -155,6 +158,9 @@ typedef struct align_table_context
uint32_t mate_ref_pos_idx;
uint32_t tlen_idx;
+ /* this is only in prim */
+ uint32_t al_group_idx;
+
/* this is specific to ev-interval/ev-alignmnet */
uint32_t ploidy_idx;
uint32_t ev_alignments_idx;
@@ -198,6 +204,7 @@ static void invalidate_all_column_idx( align_table_context * const atx )
atx->ref_ploidy_idx = COL_NOT_AVAILABLE;
atx->seq_name_idx = COL_NOT_AVAILABLE;
atx->mapq_idx = COL_NOT_AVAILABLE;
+ atx->al_group_idx = COL_NOT_AVAILABLE;
invalidate_all_cmn_column_idx( &atx->cmn );
invalidate_all_cmn_column_idx( &atx->eval );
}
@@ -265,75 +272,66 @@ static rc_t adjust_align_table_context_cig_op_buffer( align_table_context * atx,
/* src: 'P'rimary, 'S'econdary, Evidence_'I'nterval, Evidence_'A'lignment */
static rc_t prepare_cmn_table_rows( const samdump_opts * const opts, const VTable *tbl,
- align_cmn_context * const cmn, char src )
+ align_cmn_context * const cmn, char src,
+ struct KNamelist * available_columns )
{
- struct KNamelist * available_columns;
- rc_t rc = VTableListReadableColumns ( tbl, &available_columns );
- if ( rc != 0 )
- {
- (void)PLOGERR( klogInt, ( klogInt, rc,
- "VTableListReadableColumns( $(src) ) failed", "src=%c", src ) );
- }
- else
- {
- const VCursor * cursor = cmn->cursor;
+ rc_t rc = 0;
+ const VCursor * cursor = cmn->cursor;
- if ( src == 'P' || src == 'S' )
- rc = add_column( cursor, COL_SEQ_SPOT_ID, &cmn->seq_spot_id_idx );
+ if ( src == 'P' || src == 'S' )
+ rc = add_column( cursor, COL_SEQ_SPOT_ID, &cmn->seq_spot_id_idx );
- if ( rc == 0 )
+ if ( rc == 0 )
+ {
+ if ( opts->use_long_cigar )
{
- if ( opts->use_long_cigar )
- {
- rc = add_column( cursor, COL_LONG_CIGAR, &cmn->cigar_idx );
- if ( rc == 0 && ( src == 'I' || src == 'A' ) )
- rc = add_column( cursor, COL_CIGAR_LONG_LEN, &cmn->cigar_len_idx );
- }
- else
- {
- rc = add_column( cursor, COL_SHORT_CIGAR, &cmn->cigar_idx );
- if ( rc == 0 && ( src == 'I' || src == 'A' ) )
- rc = add_column( cursor, COL_CIGAR_SHORT_LEN, &cmn->cigar_len_idx );
- }
+ rc = add_column( cursor, COL_LONG_CIGAR, &cmn->cigar_idx );
+ if ( rc == 0 && ( src == 'I' || src == 'A' ) )
+ rc = add_column( cursor, COL_CIGAR_LONG_LEN, &cmn->cigar_len_idx );
}
-
- if ( rc == 0 )
+ else
{
- if ( opts->print_matches_as_equal_sign )
- rc = add_column( cursor, COL_MISMATCH_READ, &cmn->read_idx );
- else
- rc = add_column( cursor, COL_READ, &cmn->read_idx );
+ rc = add_column( cursor, COL_SHORT_CIGAR, &cmn->cigar_idx );
+ if ( rc == 0 && ( src == 'I' || src == 'A' ) )
+ rc = add_column( cursor, COL_CIGAR_SHORT_LEN, &cmn->cigar_len_idx );
}
+ }
- if ( rc == 0 )
- rc = add_column( cursor, COL_READ_LEN, &cmn->read_len_idx );
+ if ( rc == 0 )
+ {
+ if ( opts->print_matches_as_equal_sign )
+ rc = add_column( cursor, COL_MISMATCH_READ, &cmn->read_idx );
+ else
+ rc = add_column( cursor, COL_READ, &cmn->read_idx );
+ }
- if ( rc == 0 )
- rc = add_column( cursor, COL_SAM_QUALITY, &cmn->sam_quality_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_READ_LEN, &cmn->read_len_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_REF_ORIENTATION, &cmn->ref_orientation_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_SAM_QUALITY, &cmn->sam_quality_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_EDIT_DIST, &cmn->edit_dist_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_REF_ORIENTATION, &cmn->ref_orientation_idx );
- if ( rc == 0 && ( src == 'P' || src == 'S' || src == 'A' ) )
- rc = add_column( cursor, COL_SEQ_SPOT_GROUP, &cmn->seq_spot_group_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_EDIT_DIST, &cmn->edit_dist_idx );
- if ( rc == 0 && ( src == 'P' || src == 'S' || src == 'A' ) )
- rc = add_column( cursor, COL_SEQ_READ_ID, &cmn->seq_read_id_idx );
+ if ( rc == 0 && ( src == 'P' || src == 'S' || src == 'A' ) )
+ rc = add_column( cursor, COL_SEQ_SPOT_GROUP, &cmn->seq_spot_group_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_RAW_READ, &cmn->raw_read_idx );
+ if ( rc == 0 && ( src == 'P' || src == 'S' || src == 'A' ) )
+ rc = add_column( cursor, COL_SEQ_READ_ID, &cmn->seq_read_id_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_READ_FILTER, &cmn->read_filter_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_RAW_READ, &cmn->raw_read_idx );
- if ( rc == 0 && ( src == 'P' || src == 'S' || src == 'A' ) )
- add_opt_column( cursor, available_columns, COL_AL_COUNT, &cmn->al_count_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_READ_FILTER, &cmn->read_filter_idx );
+
+ if ( rc == 0 && ( src == 'P' || src == 'S' || src == 'A' ) )
+ add_opt_column( cursor, available_columns, COL_AL_COUNT, &cmn->al_count_idx );
- KNamelistRelease( available_columns );
- }
return rc;
}
@@ -359,27 +357,45 @@ static rc_t prepare_prim_sec_table_cursor( const samdump_opts * const opts, cons
}
else
{
+ struct KNamelist * available_columns;
+ char table_char = 'P';
const VCursor * cursor = atx->cmn.cursor;
- rc = prepare_cmn_table_rows( opts, tbl, &atx->cmn, 'P' );
+ if ( cmp_pchar( table_name, "SECONDARY_ALIGNMENT" ) == 0 )
+ table_char = 'S';
+
+ rc = VTableListReadableColumns ( tbl, &available_columns );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogInt, ( klogInt, rc,
+ "VTableListReadableColumns( $(src) ) failed", "src=%c", table_char ) );
+ }
+ else
+ {
+ rc = prepare_cmn_table_rows( opts, tbl, &atx->cmn, table_char, available_columns );
- if ( rc == 0 )
- rc = add_column( cursor, COL_SAM_FLAGS, &atx->sam_flags_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_SAM_FLAGS, &atx->sam_flags_idx );
- /* i don't have to add REF_NAME or REF_SEQ_ID, because i have it from the ref_obj later
- i don't have to add REF_POS, because i have it from the iterator later
- i don't have to add MAPQ, it is in the PlacementRecord later
- ... when walking the iterator ...
- */
- if ( rc == 0 )
- rc = add_column( cursor, COL_MATE_ALIGN_ID, &atx->mate_align_id_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_MATE_REF_NAME, &atx->mate_ref_name_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_MATE_REF_POS, &atx->mate_ref_pos_idx );
- if ( rc == 0 )
- rc = add_column( cursor, COL_TEMPLATE_LEN, &atx->tlen_idx );
+ /* i don't have to add REF_NAME or REF_SEQ_ID, because i have it from the ref_obj later
+ i don't have to add REF_POS, because i have it from the iterator later
+ i don't have to add MAPQ, it is in the PlacementRecord later
+ ... when walking the iterator ...
+ */
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_MATE_ALIGN_ID, &atx->mate_align_id_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_MATE_REF_NAME, &atx->mate_ref_name_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_MATE_REF_POS, &atx->mate_ref_pos_idx );
+ if ( rc == 0 )
+ rc = add_column( cursor, COL_TEMPLATE_LEN, &atx->tlen_idx );
+ if ( rc == 0 && ( table_char == 'P' ) )
+ add_opt_column( cursor, available_columns, COL_ALIGN_GROUP, &atx->al_group_idx );
+
+ KNamelistRelease( available_columns );
+ }
if ( rc != 0 )
VCursorRelease( cursor );
}
@@ -413,7 +429,21 @@ static rc_t prepare_sub_ev_alignment_table_cursor( const samdump_opts * const op
}
else
{
- rc = prepare_cmn_table_rows( opts, evidence_alignment_tbl, &atx->eval, 'A' ); /* common to prim/sec/ev-align */
+ struct KNamelist * available_columns;
+
+ rc = VTableListReadableColumns ( evidence_alignment_tbl, &available_columns );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogInt, ( klogInt, rc,
+ "VTableListReadableColumns( $(src) ) failed", "src=%c", 'A' ) );
+ }
+ else
+ {
+ rc = prepare_cmn_table_rows( opts, evidence_alignment_tbl, &atx->eval,
+ 'A', available_columns ); /* common to prim/sec/ev-align */
+ KNamelistRelease( available_columns );
+ }
+
if ( rc == 0 )
{
/* special to ev-align */
@@ -459,7 +489,21 @@ static rc_t prepare_evidence_table_cursor( const samdump_opts * const opts, cons
}
else
{
- rc = prepare_cmn_table_rows( opts, evidence_interval_tbl, &atx->cmn, 'I' );
+ struct KNamelist * available_columns;
+
+ rc = VTableListReadableColumns ( evidence_interval_tbl, &available_columns );
+ if ( rc != 0 )
+ {
+ (void)PLOGERR( klogInt, ( klogInt, rc,
+ "VTableListReadableColumns( $(src) ) failed", "src=%c", 'I' ) );
+ }
+ else
+ {
+ rc = prepare_cmn_table_rows( opts, evidence_interval_tbl, &atx->cmn,
+ 'I', available_columns ); /* common to prim/sec/ev-align */
+ KNamelistRelease( available_columns );
+ }
+
if ( rc == 0 )
rc = add_column( atx->cmn.cursor, COL_PLOIDY, &atx->ploidy_idx );
@@ -1123,9 +1167,25 @@ static rc_t print_alignment_sam_ev( const samdump_opts * const opts, const char
uint32_t quality_offset = 0;
const uint32_t *cigar_len_vector, *read_len_vector, *edit_dist_vector;
const char * cigar, *read, *quality;
-
+ char * transformed_cigar = NULL;
+ char * org_transformed_cigar = NULL;
+
rc = read_char_ptr( rec->id, cursor, atx->cmn.cigar_idx, &cigar, &cigar_str_len, "CIGAR" );
if ( rc == 0 )
+ {
+ org_transformed_cigar = string_dup ( cigar, cigar_str_len );
+ if ( org_transformed_cigar != NULL )
+ {
+ uint32_t i;
+ for ( i = 0; i < cigar_str_len; ++i )
+ {
+ if ( org_transformed_cigar[ i ] == 'S' ) org_transformed_cigar[ i ] = 'I';
+ }
+ transformed_cigar = org_transformed_cigar;
+ }
+ }
+
+ if ( rc == 0 )
rc = read_uint32_ptr( rec->id, cursor, atx->cmn.cigar_len_idx, &cigar_len_vector, &cigar_len_vector_len, "CIGAR_LEN" );
if ( rc == 0 )
rc = read_char_ptr( rec->id, cursor, atx->cmn.read_idx, &read, &read_len, "READ" );
@@ -1160,7 +1220,7 @@ static rc_t print_alignment_sam_ev( const samdump_opts * const opts, const char
/* SAM-FIELD: CIGAR SRA-column: CIGAR_SHORT / CIGAR_LONG sliced!!! */
if ( rc == 0 )
- rc = KOutMsg( "%.*s\t", cigar_slice_len, cigar );
+ rc = KOutMsg( "%.*s\t", cigar_slice_len, transformed_cigar );
/* SAM-FIELD: RNEXT SRA-column: MATE_REF_NAME ( !!! row_len can be zero !!! ) */
/* SAM-FIELD: PNEXT SRA-column: MATE_REF_POS + 1 ( !!! row_len can be zero !!! ) */
@@ -1226,8 +1286,12 @@ static rc_t print_alignment_sam_ev( const samdump_opts * const opts, const char
/* advance the cigar-slice... */
cigar += cigar_slice_len;
+ if ( transformed_cigar != NULL )
+ transformed_cigar += cigar_slice_len;
read += read_slice_len;
}
+ if ( org_transformed_cigar != NULL )
+ free( org_transformed_cigar );
}
return rc;
}
@@ -1387,9 +1451,11 @@ static rc_t print_alignment_sam_ps( const samdump_opts * const opts, const char
rc = matecache_insert_same_ref( mc, atx->db_idx, id, pos, mate_flags, -tlen );
}
- if ( mate_align_id == 0 && mate_ref_name_len == 0 && opts->print_half_unaligned_reads )
+ if ( mate_align_id == 0 && mate_ref_name_len == 0 && opts->print_half_unaligned_reads &&
+ atx->align_table_type == att_primary )
{
- rc = matecache_insert_unaligned( mc, atx->db_idx, id, pos, atx->ref_idx, *seq_spot_id );
+ int64_t key = id;
+ rc = matecache_insert_unaligned( mc, atx->db_idx, key, pos, atx->ref_idx, *seq_spot_id );
}
}
}
@@ -1451,8 +1517,10 @@ static rc_t print_alignment_sam_ps( const samdump_opts * const opts, const char
static char const *bogus_quality = "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!";
rc = read_char_ptr( id, cursor, atx->cmn.cigar_idx, &cgc_input.p_cigar.ptr, &cgc_input.p_cigar.len, "CIGAR" );
- if (rc == 0) {
- if (cgc_output.p_quality.len == 0) {
+ if ( rc == 0 )
+ {
+ if ( cgc_output.p_quality.len == 0 )
+ {
cgc_output.p_quality.ptr = bogus_quality;
cgc_output.p_quality.len = 35;
}
@@ -1516,9 +1584,9 @@ static rc_t print_alignment_sam_ps( const samdump_opts * const opts, const char
}
/* OPT SAM-FIELD: RG SRA-column: SPOT_GROUP */
- if ( rc == 0 && atx->cmn.seq_spot_group_idx != COL_NOT_AVAILABLE )
+ if ( rc == 0 && ( atx->cmn.seq_spot_group_idx != COL_NOT_AVAILABLE ) )
{
- const char * spot_grp;
+ const char * spot_grp = NULL;
uint32_t spot_grp_len;
rc = read_char_ptr( id, cursor, atx->cmn.seq_spot_group_idx, &spot_grp, &spot_grp_len, "SPOT_GROUP" );
if ( rc == 0 && spot_grp_len > 0 )
@@ -1532,6 +1600,26 @@ static rc_t print_alignment_sam_ps( const samdump_opts * const opts, const char
if ( rc == 0 && opts->print_alignment_id_in_column_xi )
rc = KOutMsg( "\tXI:i:%u", id );
+ /* to match sam-tools output: in case we are dumping this in CG-mode.... */
+ if ( rc == 0 && ( opts->cigar_treatment != ct_unchanged ) && ( atx->al_group_idx != COL_NOT_AVAILABLE ) )
+ {
+ const char * align_grp;
+ uint32_t align_grp_len;
+ rc = read_char_ptr( id, cursor, atx->al_group_idx, &align_grp, &align_grp_len, "ALIGN_GROUP" );
+ if ( rc == 0 && align_grp_len > 0 )
+ {
+ uint32_t i;
+ for ( i = 0; rc == 0 && i < align_grp_len - 1; ++i )
+ {
+ if ( align_grp[ i ] == '_' )
+ {
+ rc = KOutMsg( "\tZI:i:%.*s\tZA:i:%.1s", i, align_grp, align_grp + i + 1 );
+ break;
+ }
+ }
+ }
+ }
+
/* OPT SAM-FIELD: NH SRA-column: ALIGNMENT_COUNT */
if ( rc == 0 && atx->cmn.al_count_idx != COL_NOT_AVAILABLE )
{
diff --git a/tools/sra-pileup/sam-dump-opts.c b/tools/sra-pileup/sam-dump-opts.c
index c1f9567..073ad56 100644
--- a/tools/sra-pileup/sam-dump-opts.c
+++ b/tools/sra-pileup/sam-dump-opts.c
@@ -34,7 +34,7 @@
/* =========================================================================================== */
-static int cmp_pchar( const char * a, const char * b )
+int cmp_pchar( const char * a, const char * b )
{
int res = 0;
if ( ( a != NULL )&&( b != NULL ) )
@@ -796,7 +796,13 @@ static rc_t gather_flag_options( Args * args, samdump_opts * opts )
rc = get_bool_option( args, OPT_CIGAR_CG_M, &opts->merge_cg_cigar );
if ( rc != 0 ) return rc;
+ /* do we enable rna-splicing */
rc = get_bool_option( args, OPT_RNA_SPLICE, &opts->rna_splicing );
+ if ( rc != 0 ) return rc;
+
+ /* do we disable multi-threading */
+ rc = get_bool_option( args, OPT_NO_MT, &opts->no_mt );
+
return rc;
}
@@ -1266,6 +1272,8 @@ void report_options( const samdump_opts * opts )
KOutMsg( "use min-mapq : %s\n", opts->use_min_mapq ? "YES" : "NO" );
KOutMsg( "min-mapq : %i\n", opts->min_mapq );
KOutMsg( "rna-splicing : %s\n", opts->rna_splicing ? "YES" : "NO" );
+
+ KOutMsg( "multithreading : %s\n", opts->no_mt ? "NO" : "YES" );
}
diff --git a/tools/sra-pileup/sam-dump-opts.h b/tools/sra-pileup/sam-dump-opts.h
index e17c3c8..52a1aaa 100644
--- a/tools/sra-pileup/sam-dump-opts.h
+++ b/tools/sra-pileup/sam-dump-opts.h
@@ -92,7 +92,7 @@ extern "C" {
#define OPT_LEGACY "legacy"
#define OPT_NEW "new"
#define OPT_RNA_SPLICE "rna-splicing"
-
+#define OPT_NO_MT "disable-multithreading"
typedef struct range
{
@@ -241,6 +241,9 @@ typedef struct samdump_opts
bool print_cg_names;
bool rna_splicing;
+ /* option to disable multi-threading */
+ bool no_mt;
+
uint8_t qual_quant_matrix[ 256 ];
} samdump_opts;
@@ -258,6 +261,7 @@ rc_t foreach_reference( BSTree * regions,
rc_t ( CC * on_reference ) ( const char * name, Vector *ranges, void *data ),
void *data );
+int cmp_pchar( const char * a, const char * b );
rc_t gather_options( Args * args, samdump_opts * opts );
void report_options( const samdump_opts * opts );
diff --git a/tools/sra-pileup/sam-dump.c b/tools/sra-pileup/sam-dump.c
index 4c491b9..9be83a4 100644
--- a/tools/sra-pileup/sam-dump.c
+++ b/tools/sra-pileup/sam-dump.c
@@ -3882,7 +3882,7 @@ static rc_t ProcessPath( VDBManager const *mgr, char const Path[] )
size_t i;
/* no scheme found - must be simple fs path */
- old:
+ /* old: ---unused label */
readGroup = NULL;
path = NULL;
diff --git a/tools/sra-pileup/sam-dump.vers b/tools/sra-pileup/sam-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-pileup/sam-dump.vers
+++ b/tools/sra-pileup/sam-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-pileup/sam-dump3.c b/tools/sra-pileup/sam-dump3.c
index 0bea584..6794065 100644
--- a/tools/sra-pileup/sam-dump3.c
+++ b/tools/sra-pileup/sam-dump3.c
@@ -27,6 +27,7 @@
#include <kapp/main.h>
#include <vdb/report.h> /* ReportSetVDBManager */
+#include <vdb/vdb-priv.h> /* VDBManagerDisablePagemapThread() */
#include <klib/report.h>
#include <sysalloc.h>
@@ -150,6 +151,8 @@ char const *sd_no_mate_cache_usage[] = { "do not use a mate-cache, slower but l
char const *rna_splice_usage[] = { "modify cigar-string and output flags if rna-splicing detected",
NULL };
+char const *no_mt_usage[] = { "disable multithreading", NULL };
+
OptDef SamDumpArgs[] =
{
{ OPT_UNALIGNED, "u", NULL, sd_unaligned_usage, 0, false, false }, /* print unaligned reads */
@@ -189,10 +192,11 @@ OptDef SamDumpArgs[] =
{ OPT_MIN_MAPQ, NULL, NULL, sd_min_mapq_usage, 0, true, false }, /* minimal mapping quality */
{ OPT_NO_MATE_CACHE,NULL, NULL, sd_no_mate_cache_usage, 0, false, false }, /* do not use mate-cache */
{ OPT_RNA_SPLICE, NULL, NULL, rna_splice_usage, 0, false, false }, /* detect rna-splicing in sequence */
+ { OPT_NO_MT, NULL, NULL, no_mt_usage, 0, false, false }, /* force new code-path */
{ OPT_DUMP_MODE, NULL, NULL, NULL, 0, true, false }, /* how to produce aligned reads if no regions given */
{ OPT_CIGAR_TEST, NULL, NULL, NULL, 0, true, false }, /* test cg-treatment of cigar string */
{ OPT_LEGACY, NULL, NULL, NULL, 0, false, false }, /* force legacy code-path */
- { OPT_NEW, NULL, NULL, NULL, 0, false, false } /* force new code-path */
+ { OPT_NEW, NULL, NULL, NULL, 0, false, false }, /* force new code-path */
};
char const *sd_usage_params[] =
@@ -234,10 +238,11 @@ char const *sd_usage_params[] =
NULL, /* min_mapq */
NULL, /* no mate-cache */
NULL, /* detect rna-splicing in sequence */
+ NULL, /* no-mt */
NULL, /* dump_mode */
NULL, /* cigar test */
NULL, /* force legacy code path */
- NULL, /* force new code path */
+ NULL /* force new code path */
};
ver_t CC KAppVersion( void )
@@ -339,61 +344,73 @@ static rc_t print_samdump( const samdump_opts * const opts )
ReportSetVDBManager( mgr ); /**/
- rc = discover_input_files( &ifs, mgr, opts->input_files, reflist_opt ); /* inputfiles.c */
- if ( rc == 0 )
+ if ( opts->no_mt )
{
- if ( ifs->database_count == 0 && ifs->table_count == 0 )
+ rc = VDBManagerDisablePagemapThread ( mgr );
+ if ( rc != 0 )
{
- rc = RC( rcExe, rcFile, rcReading, rcItem, rcNotFound );
- (void)LOGERR( klogErr, rc, "input object(s) not found" );
+ LOGERR( klogInt, rc, "VDBManagerDisablePagemapThread() failed" );
}
- else
- {
- matecache * mc = NULL;
-
- if ( opts->use_mate_cache )
- rc = make_matecache( &mc, ifs->database_count );
+ }
- if ( rc == 0 )
+ if ( rc == 0 )
+ {
+ rc = discover_input_files( &ifs, mgr, opts->input_files, reflist_opt ); /* inputfiles.c */
+ if ( rc == 0 )
+ {
+ if ( ifs->database_count == 0 && ifs->table_count == 0 )
{
- uint64_t rows_so_far = 0;
+ rc = RC( rcExe, rcFile, rcReading, rcItem, rcNotFound );
+ (void)LOGERR( klogErr, rc, "input object(s) not found" );
+ }
+ else
+ {
+ matecache * mc = NULL;
- /* print output of header */
- if ( ( opts->output_format == of_sam ) &&
- ( ifs->database_count > 0 ) &&
- ( opts->header_mode != hm_none ) &&
- !opts->dump_unaligned_only )
- /* ------------------------------------------------------ */
- rc = print_headers( opts, ifs ); /* sam-hdr.c */
- /* ------------------------------------------------------ */
+ if ( opts->use_mate_cache )
+ rc = make_matecache( &mc, ifs->database_count );
+ if ( rc == 0 )
+ {
+ uint64_t rows_so_far = 0;
- /* print output of aligned reads */
- if ( rc == 0 &&
- ifs->database_count > 0 &&
- !opts->dump_unaligned_only )
- /* ------------------------------------------------------ */
- rc = print_aligned_spots( opts, ifs, mc, &rows_so_far ); /* sam-aligned.c */
- /* ------------------------------------------------------ */
+ /* print output of header */
+ if ( ( opts->output_format == of_sam ) &&
+ ( ifs->database_count > 0 ) &&
+ ( opts->header_mode != hm_none ) &&
+ !opts->dump_unaligned_only )
+ /* ------------------------------------------------------ */
+ rc = print_headers( opts, ifs ); /* sam-hdr.c */
+ /* ------------------------------------------------------ */
- /* print output of unaligned reads */
- if ( rc == 0 )
- {
+ /* print output of aligned reads */
+ if ( rc == 0 &&
+ ifs->database_count > 0 &&
+ !opts->dump_unaligned_only )
/* ------------------------------------------------------ */
- rc = print_unaligned_spots( opts, ifs, mc, &rows_so_far ); /* sam-unaligned.c */
+ rc = print_aligned_spots( opts, ifs, mc, &rows_so_far ); /* sam-aligned.c */
/* ------------------------------------------------------ */
- }
- if ( opts->use_mate_cache )
- {
- if ( opts->report_cache )
- rc = matecache_report( mc ); /* matecache.c */
- release_matecache( mc ); /* matecache.c */
+
+ /* print output of unaligned reads */
+ if ( rc == 0 )
+ {
+ /* ------------------------------------------------------ */
+ rc = print_unaligned_spots( opts, ifs, mc, &rows_so_far ); /* sam-unaligned.c */
+ /* ------------------------------------------------------ */
+ }
+
+ if ( opts->use_mate_cache )
+ {
+ if ( opts->report_cache )
+ rc = matecache_report( mc ); /* matecache.c */
+ release_matecache( mc ); /* matecache.c */
+ }
}
}
+ release_input_files( ifs ); /* inputfiles.c */
}
- release_input_files( ifs ); /* inputfiles.c */
}
VDBManagerRelease( mgr );
}
diff --git a/tools/sra-pileup/sam-hdr.c b/tools/sra-pileup/sam-hdr.c
index 0adc2fa..2173ad8 100644
--- a/tools/sra-pileup/sam-hdr.c
+++ b/tools/sra-pileup/sam-hdr.c
@@ -88,7 +88,7 @@ typedef struct seq_id_node
} seq_id_node;
-static int cmp_pchar( const char * a, const char * b )
+static int cmp_pchar_0( const char * a, const char * b )
{
int res = 0;
if ( ( a != NULL )&&( b != NULL ) )
@@ -142,7 +142,7 @@ static void free_seq_id_tree( BSTree * tree )
static int CC seq_id_node_vs_pchar_wrapper( const void *item, const BSTNode *n )
{
const seq_id_node * node = ( const seq_id_node * )n;
- return cmp_pchar( (const char *)item, node->seq_id );
+ return cmp_pchar_0( (const char *)item, node->seq_id );
}
@@ -156,7 +156,7 @@ static int CC node_vs_node_wrapper( const BSTNode *item, const BSTNode *n )
{
const seq_id_node * a = ( const seq_id_node * )item;
const seq_id_node * b = ( const seq_id_node * )n;
- return cmp_pchar( a->seq_id, b->seq_id );
+ return cmp_pchar_0( a->seq_id, b->seq_id );
}
@@ -245,7 +245,7 @@ static void CC print_header_callback( BSTNode *n, void *data )
hctx->rc = KOutMsg( "@SQ\tSN:%s\tLN:%u\n", node->seq_id, node->seq_len );
else
{
- if ( cmp_pchar( node->seq_id, node->name ) == 0 )
+ if ( cmp_pchar_0( node->seq_id, node->name ) == 0 )
hctx->rc = KOutMsg( "@SQ\tSN:%s\tLN:%u\n", node->name, node->seq_len );
else
hctx->rc = KOutMsg( "@SQ\tSN:%s\tAS:%s\tLN:%u\n", node->name, node->seq_id, node->seq_len );
diff --git a/tools/sra-pileup/sam-unaligned.c b/tools/sra-pileup/sam-unaligned.c
index 09e9583..53cf05c 100644
--- a/tools/sra-pileup/sam-unaligned.c
+++ b/tools/sra-pileup/sam-unaligned.c
@@ -651,7 +651,7 @@ static rc_t dump_seq_row_sam_filtered( const samdump_opts * const opts, const se
/* OPT SAM-FIIELD: SRA-column: SPOT_GROUP */
if ( rc == 0 && spot_group == NULL )
rc = read_char_ptr( row_id, stx->cursor, stx->spot_group_idx, &spot_group, &spot_group_len, "SPOT_GROUP" );
- if ( rc == 0 )
+ if ( rc == 0 && spot_group_len > 0 )
rc = KOutMsg( "\tRG:Z:%.*s", spot_group_len, spot_group );
if ( rc == 0 )
@@ -813,7 +813,7 @@ static rc_t dump_seq_prim_row_sam( const samdump_opts * const opts, const seq_ta
/* OPT SAM-FIIELD: SRA-column: SPOT_GROUP */
if ( rc == 0 && spot_group == NULL )
rc = read_char_ptr( row_id, stx->cursor, stx->spot_group_idx, &spot_group, &spot_group_len, "SPOT_GROUP" );
- if ( rc == 0 )
+ if ( rc == 0 && spot_group_len > 0 )
rc = KOutMsg( "\tRG:Z:%.*s", spot_group_len, spot_group );
if ( rc == 0 )
diff --git a/tools/sra-pileup/sra-pileup.c b/tools/sra-pileup/sra-pileup.c
index c0c4d70..82918ab 100644
--- a/tools/sra-pileup/sra-pileup.c
+++ b/tools/sra-pileup/sra-pileup.c
@@ -26,8 +26,10 @@
#include "sra-pileup.vers.h"
+#include "ref_regions.h"
#include "cmdline_cmn.h"
#include "reref.h"
+#include "ref_walker.h"
#include <kapp/main.h>
@@ -35,6 +37,7 @@
#include <klib/printf.h>
#include <klib/report.h>
#include <klib/sort.h>
+#include <klib/vector.h>
#include <kfs/file.h>
#include <kfs/buffile.h>
@@ -43,9 +46,12 @@
#include <insdc/sra.h>
+#include <kdb/manager.h>
+
#include <vdb/manager.h>
#include <vdb/schema.h>
-#include <vdb/report.h> /* ReportSetVDBManager */
+#include <vdb/report.h> /* ReportSetVDBManager() */
+#include <vdb/vdb-priv.h> /* VDBManagerDisablePagemapThread() */
#include <sra/sraschema.h>
#include <align/manager.h>
@@ -80,6 +86,8 @@
#define OPTION_SEQNAME "seqname"
#define ALIAS_SEQNAME "e"
+#define OPTION_MIN_M "minmismatch"
+
#define OPTION_FUNC "function"
#define ALIAS_FUNC NULL
@@ -88,6 +96,8 @@
#define FUNC_RE_REF "ref"
#define FUNC_RE_REF_EXT "ref-ex"
#define FUNC_DEBUG "debug"
+#define FUNC_MISMATCH "mismatch"
+#define FUNC_TEST "test"
enum
{
@@ -96,7 +106,9 @@ enum
sra_pileup_stat = 2,
sra_pileup_report_ref = 3,
sra_pileup_report_ref_ext = 4,
- sra_pileup_debug = 5
+ sra_pileup_debug = 5,
+ sra_pileup_mismatch = 6,
+ sra_pileup_test = 7
};
static const char * minmapq_usage[] = { "Minimum mapq-value, ",
@@ -115,10 +127,13 @@ static const char * spotgrp_usage[] = { "divide by spotgroups", NULL };
static const char * seqname_usage[] = { "use original seq-name", NULL };
+static const char * min_m_usage[] = { "min percent of mismatches used in function mismatch, def is 5%", NULL };
+
static const char * func_ref_usage[] = { "list references", NULL };
static const char * func_ref_ex_usage[] = { "list references + coverage", NULL };
static const char * func_count_usage[] = { "sort pileup with counters", NULL };
static const char * func_stat_usage[] = { "strand/tlen statistic", NULL };
+static const char * func_mismatch_usage[] = { "only lines with mismatch", NULL };
static const char * func_usage[] = { "alternative functionality", NULL };
OptDef MyOptions[] =
@@ -131,6 +146,7 @@ OptDef MyOptions[] =
{ OPTION_SHOWID, ALIAS_SHOWID, NULL, showid_usage, 1, false, false },
{ OPTION_SPOTGRP, ALIAS_SPOTGRP, NULL, spotgrp_usage, 1, false, false },
{ OPTION_SEQNAME, ALIAS_SEQNAME, NULL, seqname_usage, 1, false, false },
+ { OPTION_MIN_M, NULL, NULL, min_m_usage, 1, true, false },
{ OPTION_FUNC, ALIAS_FUNC, NULL, func_usage, 1, true, false }
};
@@ -147,6 +163,7 @@ typedef struct pileup_options
bool div_by_spotgrp;
bool use_seq_name;
uint32_t minmapq;
+ uint32_t min_mismatch;
uint32_t source_table;
uint32_t function; /* sra_pileup_samtools, sra_pileup_counters, sra_pileup_stat,
sra_pileup_report_ref, sra_pileup_report_ref_ext, sra_pileup_debug */
@@ -248,6 +265,9 @@ static rc_t get_pileup_options( Args * args, pileup_options *opts )
rc = get_uint32_option( args, OPTION_MINMAPQ, &opts->minmapq, 0 );
if ( rc == 0 )
+ rc = get_uint32_option( args, OPTION_MIN_M, &opts->min_mismatch, 5 );
+
+ if ( rc == 0 )
rc = get_bool_option( args, OPTION_DUPS, &opts->process_dups, false );
if ( rc == 0 )
@@ -281,6 +301,10 @@ static rc_t get_pileup_options( Args * args, pileup_options *opts )
opts->function = sra_pileup_report_ref_ext;
else if ( cmp_pchar( fkt, FUNC_DEBUG ) == 0 )
opts->function = sra_pileup_debug;
+ else if ( cmp_pchar( fkt, FUNC_MISMATCH ) == 0 )
+ opts->function = sra_pileup_mismatch;
+ else if ( cmp_pchar( fkt, FUNC_TEST ) == 0 )
+ opts->function = sra_pileup_test;
}
}
return rc;
@@ -326,12 +350,14 @@ rc_t CC Usage ( const Args * args )
HelpOptionLine ( ALIAS_DUPS, OPTION_DUPS, "dup-mode", dups_usage );
HelpOptionLine ( ALIAS_SPOTGRP, OPTION_SPOTGRP, "spotgroups-modes", spotgrp_usage );
HelpOptionLine ( ALIAS_SEQNAME, OPTION_SEQNAME, NULL, seqname_usage );
-
+ HelpOptionLine ( NULL, OPTION_MIN_M, NULL, min_m_usage );
+
HelpOptionLine ( NULL, "function ref", NULL, func_ref_usage );
HelpOptionLine ( NULL, "function ref-ex", NULL, func_ref_ex_usage );
HelpOptionLine ( NULL, "function count", NULL, func_count_usage );
HelpOptionLine ( NULL, "function stat", NULL, func_stat_usage );
-
+ HelpOptionLine ( NULL, "function mismatch", NULL, func_mismatch_usage );
+
HelpOptionsStandard ();
HelpVersion ( fullpath, KAppVersion() );
return rc;
@@ -369,7 +395,7 @@ ver_t CC KAppVersion ( void )
***************************************/
-char _4na_2_ascii_tab[] =
+static char _4na_2_ascii_tab[] =
{
/* 0x0 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F */
'N', 'A', 'C', 'M', 'G', 'R', 'S', 'V', 'T', 'W', 'Y', 'H', 'K', 'D', 'B', 'N',
@@ -823,7 +849,7 @@ static rc_t walk_ref_position( ReferenceIterator *ref_iter,
rc = add_char_2_dyn_string( line, '$' );
if ( options->show_id )
- rc = print_2_dyn_string( line, "(%lu:%u-%u/%u)",
+ rc = print_2_dyn_string( line, "(%,lu:%,d-%,d/%u)",
rec->id, rec->pos + 1, rec->pos + rec->len, seq_pos );
return rc;
@@ -1699,6 +1725,85 @@ static rc_t walk_counters( ReferenceIterator *ref_iter, pileup_options *options
/* =========================================================================================== */
+static rc_t print_mismatches_line( const char * ref_name,
+ INSDC_coord_zero ref_pos,
+ uint32_t depth,
+ uint32_t min_mismatch_percent,
+ pileup_counters * counters )
+{
+ rc_t rc = 0;
+ if ( depth > 0 )
+ {
+ uint32_t total_mismatches = counters->mismatches[ 0 ] +
+ counters->mismatches[ 1 ] +
+ counters->mismatches[ 2 ] +
+ counters->mismatches[ 3 ];
+ if ( total_mismatches * 100 >= min_mismatch_percent * depth)
+ {
+ rc = KOutMsg( "%s\t%u\t%u\t%u\n", ref_name, ref_pos + 1, depth, total_mismatches );
+ }
+ }
+
+ free_fragments( &(counters->insert_fragments) );
+ free_fragments( &(counters->delete_fragments) );
+
+ return rc;
+}
+
+
+/* ........................................................................................... */
+
+
+static rc_t CC walk_mismatches_enter_ref_pos( walk_data * data )
+{
+ clear_counters( data->data );
+ return 0;
+}
+
+static rc_t CC walk_mismatches_exit_ref_pos( walk_data * data )
+{
+ rc_t rc = print_mismatches_line( data->ref_name, data->ref_pos,
+ data->depth, data->options->min_mismatch, data->data );
+ return rc;
+}
+
+static rc_t CC walk_mismatches_placement( walk_data * data )
+{
+ walk_counter_state( data->ref_iter, data->state, data->xrec->reverse, data->data );
+ return 0;
+}
+
+static rc_t walk_mismatches( ReferenceIterator *ref_iter, pileup_options * options )
+{
+ walk_data data;
+ walk_funcs funcs;
+ pileup_counters counters;
+
+ data.ref_iter = ref_iter;
+ data.options = options;
+ data.data = &counters;
+
+ funcs.on_enter_ref = NULL;
+ funcs.on_exit_ref = NULL;
+
+ funcs.on_enter_ref_window = NULL;
+ funcs.on_exit_ref_window = NULL;
+
+ funcs.on_enter_ref_pos = walk_mismatches_enter_ref_pos;
+ funcs.on_exit_ref_pos = walk_mismatches_exit_ref_pos;
+
+ funcs.on_enter_spotgroup = NULL;
+ funcs.on_exit_spotgroup = NULL;
+
+ funcs.on_placement = walk_mismatches_placement;
+
+ return walk( &data, &funcs );
+}
+
+
+/* =========================================================================================== */
+
+
typedef struct tlen_array
{
uint32_t * values;
@@ -2064,6 +2169,7 @@ static rc_t walk_stat( ReferenceIterator *ref_iter, pileup_options *options )
return rc;
}
+
/* =========================================================================================== */
static rc_t add_quality_and_orientation( const VTable *tbl, const VCursor ** cursor,
@@ -2211,7 +2317,7 @@ static rc_t CC prepare_section_cb( prepare_ctx * ctx, uint32_t start, uint32_t e
{
rc = SILENT_RC ( rcApp, rcNoTarg, rcOpening, rcSelf, rcInvalid );
/* it is opened in prepare_db_table even if ctx->db == NULL */
- PLOGERR(klogInt, (klogInt, rc, "failed to process $(path)",
+ PLOGERR( klogErr, ( klogErr, rc, "failed to process $(path)",
"path=%s", ctx->path == NULL ? "input argument" : ctx->path));
ReportSilence();
}
@@ -2367,24 +2473,55 @@ static rc_t CC on_argument( const char * path, const char * spot_group, void * d
{
rc_t rc = 0;
foreach_arg_ctx * ctx = ( foreach_arg_ctx * )data;
- prepare_ctx prep; /* from cmdline_cmn.h */
- prep.omit_qualities = ctx->options->omit_qualities;
- prep.read_tlen = ctx->options->read_tlen;
- prep.use_primary_alignments = ( ( ctx->options->cmn.tab_select & primary_ats ) == primary_ats );
- prep.use_secondary_alignments = ( ( ctx->options->cmn.tab_select & secondary_ats ) == secondary_ats );
- prep.use_evidence_alignments = ( ( ctx->options->cmn.tab_select & evidence_ats ) == evidence_ats );
- prep.ref_iter = ctx->ref_iter;
- prep.spot_group = spot_group;
- prep.on_section = prepare_section_cb;
- prep.data = ctx->cursor_ids;
- prep.path = path;
-
- rc = prepare_ref_iter( &prep, ctx->vdb_mgr, ctx->vdb_schema, path, ctx->ranges ); /* cmdline_cmn.c */
- if ( rc == 0 && prep.db == NULL )
+ int path_type = ( VDBManagerPathType ( ctx->vdb_mgr, "%s", path ) & ~ kptAlias );
+ if ( path_type != kptDatabase )
{
- rc = RC ( rcApp, rcNoTarg, rcOpening, rcSelf, rcInvalid );
- LOGERR( klogInt, rc, "unsupported source" );
+ rc = RC ( rcApp, rcNoTarg, rcOpening, rcItem, rcUnsupported );
+ PLOGERR( klogErr, ( klogErr, rc, "failed to open '$(path)', it is not a vdb-database", "path=%s", path ) );
+ }
+ else
+ {
+ const VDatabase *db;
+ rc = VDBManagerOpenDBRead ( ctx->vdb_mgr, &db, ctx->vdb_schema, "%s", path );
+ if ( rc != 0 )
+ {
+ rc = RC ( rcApp, rcNoTarg, rcOpening, rcItem, rcUnsupported );
+ PLOGERR( klogErr, ( klogErr, rc, "failed to open '$(path)'", "path=%s", path ) );
+ }
+ else
+ {
+ bool is_csra = VDatabaseIsCSRA ( db );
+ VDatabaseRelease ( db );
+ if ( !is_csra )
+ {
+ rc = RC ( rcApp, rcNoTarg, rcOpening, rcItem, rcUnsupported );
+ PLOGERR( klogErr, ( klogErr, rc, "failed to open '$(path)', it is not a csra-database", "path=%s", path ) );
+ }
+ else
+ {
+ prepare_ctx prep; /* from cmdline_cmn.h */
+
+ prep.omit_qualities = ctx->options->omit_qualities;
+ prep.read_tlen = ctx->options->read_tlen;
+ prep.use_primary_alignments = ( ( ctx->options->cmn.tab_select & primary_ats ) == primary_ats );
+ prep.use_secondary_alignments = ( ( ctx->options->cmn.tab_select & secondary_ats ) == secondary_ats );
+ prep.use_evidence_alignments = ( ( ctx->options->cmn.tab_select & evidence_ats ) == evidence_ats );
+ prep.ref_iter = ctx->ref_iter;
+ prep.spot_group = spot_group;
+ prep.on_section = prepare_section_cb;
+ prep.data = ctx->cursor_ids;
+ prep.path = path;
+
+
+ rc = prepare_ref_iter( &prep, ctx->vdb_mgr, ctx->vdb_schema, path, ctx->ranges ); /* cmdline_cmn.c */
+ if ( rc == 0 && prep.db == NULL )
+ {
+ rc = RC ( rcApp, rcNoTarg, rcOpening, rcSelf, rcInvalid );
+ LOGERR( klogInt, rc, "unsupported source" );
+ }
+ }
+ }
}
return rc;
}
@@ -2454,12 +2591,22 @@ static rc_t pileup_main( Args * args, pileup_options *options )
{
LOGERR( klogInt, rc, "VDBManagerMakeRead() failed" );
}
- else {
- ReportSetVDBManager(arg_ctx.vdb_mgr);
+ else
+ {
+ ReportSetVDBManager( arg_ctx.vdb_mgr );
}
}
+ if ( rc == 0 && options->cmn.no_mt )
+ {
+ rc = VDBManagerDisablePagemapThread ( arg_ctx.vdb_mgr );
+ if ( rc != 0 )
+ {
+ LOGERR( klogInt, rc, "VDBManagerDisablePagemapThread() failed" );
+ }
+ }
+
/* (5) make a vdb-schema */
if ( rc == 0 )
{
@@ -2496,6 +2643,10 @@ static rc_t pileup_main( Args * args, pileup_options *options )
case sra_pileup_samtools : options->read_tlen = false;
break;
+
+ case sra_pileup_mismatch : options->omit_qualities = true;
+ options->read_tlen = false;
+ break;
}
}
@@ -2527,6 +2678,7 @@ static rc_t pileup_main( Args * args, pileup_options *options )
case sra_pileup_stat : rc = walk_stat( arg_ctx.ref_iter, options ); break;
case sra_pileup_counters : rc = walk_counters( arg_ctx.ref_iter, options ); break;
case sra_pileup_debug : rc = walk_debug( arg_ctx.ref_iter, options ); break;
+ case sra_pileup_mismatch : rc = walk_mismatches( arg_ctx.ref_iter, options ); break;
default : rc = walk_ref_iter( arg_ctx.ref_iter, options ); break;
}
/* ============================================== */
@@ -2545,6 +2697,115 @@ static rc_t pileup_main( Args * args, pileup_options *options )
/* =========================================================================================== */
+static rc_t CC pileup_test_enter_ref( ref_walker_data * rwd )
+{
+ return KOutMsg( "\nentering >%s<\n", rwd->ref_name );
+}
+
+static rc_t CC pileup_test_exit_ref( ref_walker_data * rwd )
+{
+ return KOutMsg( "exit >%s<\n", rwd->ref_name );
+}
+
+static rc_t CC pileup_test_enter_ref_window( ref_walker_data * rwd )
+{
+ return KOutMsg( " enter window >%s< [ %,lu ... %,lu ]\n", rwd->ref_name, rwd->ref_start, rwd->ref_end );
+}
+
+static rc_t CC pileup_test_exit_ref_window( ref_walker_data * rwd )
+{
+ return KOutMsg( " exit window >%s< [ %,lu ... %,lu ]\n", rwd->ref_name, rwd->ref_start, rwd->ref_end );
+}
+
+static rc_t CC pileup_test_enter_ref_pos( ref_walker_data * rwd )
+{
+ return KOutMsg( " enter pos [ %,lu ], d=%u\n", rwd->pos, rwd->depth );
+}
+
+static rc_t CC pileup_test_exit_ref_pos( ref_walker_data * rwd )
+{
+ return KOutMsg( " exit pos [ %,lu ], d=%u\n", rwd->pos, rwd->depth );
+}
+
+static rc_t CC pileup_test_enter_spot_group( ref_walker_data * rwd )
+{
+ return KOutMsg( " enter spot-group [ %,lu ], %.*s\n", rwd->pos, rwd->spot_group_len, rwd->spot_group );
+}
+
+static rc_t CC pileup_test_exit_spot_group( ref_walker_data * rwd )
+{
+ return KOutMsg( " exit spot-group [ %,lu ], %.*s\n", rwd->pos, rwd->spot_group_len, rwd->spot_group );
+}
+
+static rc_t CC pileup_test_alignment( ref_walker_data * rwd )
+{
+ return KOutMsg( " alignment\n" );
+}
+
+
+static rc_t pileup_test( Args * args, pileup_options *options )
+{
+ struct ref_walker * walker;
+
+ /* create walker */
+ rc_t rc = ref_walker_create( &walker );
+ if ( rc == 0 )
+ {
+ uint32_t idx, count;
+
+ /* add sources to walker */
+ rc = ArgsParamCount( args, &count );
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * src = NULL;
+ rc = ArgsParamValue( args, idx, &src );
+ if ( rc == 0 && src != NULL )
+ rc = ref_walker_add_source( walker, src );
+ }
+
+ /* add ranges to walker */
+ if ( rc == 0 )
+ {
+ rc = ArgsOptionCount( args, OPTION_REF, &count );
+ for ( idx = 0; idx < count && rc == 0; ++idx )
+ {
+ const char * s = NULL;
+ rc = ArgsOptionValue( args, OPTION_REF, idx, &s );
+ if ( rc == 0 && s != NULL )
+ rc = ref_walker_parse_and_add_range( walker, s );
+ }
+ }
+
+ /* set callbacks for walker */
+ if ( rc == 0 )
+ {
+ ref_walker_callbacks callbacks =
+ { pileup_test_enter_ref,
+ pileup_test_exit_ref,
+ pileup_test_enter_ref_window,
+ pileup_test_exit_ref_window,
+ pileup_test_enter_ref_pos,
+ pileup_test_exit_ref_pos,
+ pileup_test_enter_spot_group,
+ pileup_test_exit_spot_group,
+ pileup_test_alignment };
+ rc = ref_walker_set_callbacks( walker, &callbacks );
+ }
+
+ /* let the walker call the callbacks while iterating over the sources/ranges */
+ if ( rc == 0 )
+ rc = ref_walker_walk( walker, NULL );
+
+ /* destroy the walker */
+ ref_walker_destroy( walker );
+ }
+ return rc;
+}
+
+
+/* =========================================================================================== */
+
+
rc_t CC KMain( int argc, char *argv [] )
{
rc_t rc = KOutHandlerSet( write_to_FILE, stdout );
@@ -2585,6 +2846,10 @@ rc_t CC KMain( int argc, char *argv [] )
{
rc = report_on_reference( args, options.function == sra_pileup_report_ref_ext ); /* reref.c */
}
+ else if ( options.function == sra_pileup_test )
+ {
+ rc = pileup_test( args, &options ); /* see above */
+ }
else
{
/* ============================== */
diff --git a/tools/sra-pileup/sra-pileup.vers b/tools/sra-pileup/sra-pileup.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-pileup/sra-pileup.vers
+++ b/tools/sra-pileup/sra-pileup.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-sort/sra-sort.vers b/tools/sra-sort/sra-sort.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-sort/sra-sort.vers
+++ b/tools/sra-sort/sra-sort.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-stat/sra-stat.vers b/tools/sra-stat/sra-stat.vers
index 0bee604..3f684d2 100644
--- a/tools/sra-stat/sra-stat.vers
+++ b/tools/sra-stat/sra-stat.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/srapath/Makefile b/tools/srapath/Makefile
index 2e346aa..4c20189 100644
--- a/tools/srapath/Makefile
+++ b/tools/srapath/Makefile
@@ -84,13 +84,7 @@ SRAPATH_OBJ = \
SRAPATH_LIB += \
-lkapp \
- -lvfs \
- -lkurl \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lkproc \
- -lklib
+ -lncbi-vdb \
$(BINDIR)/srapath: $(SRAPATH_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(SRAPATH_LIB)
diff --git a/tools/srapath/srapath.vers b/tools/srapath/srapath.vers
index 0bee604..3f684d2 100644
--- a/tools/srapath/srapath.vers
+++ b/tools/srapath/srapath.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/util/Makefile b/tools/util/Makefile
index af77fbf..3a44760 100644
--- a/tools/util/Makefile
+++ b/tools/util/Makefile
@@ -96,13 +96,7 @@ RCEXPLAIN_OBJ = \
RCEXPLAIN_LIB = \
-skapp \
- -svfs \
- -skurl \
- -skrypto \
- -skfg \
- -skfs \
- -sksproc \
- -sklib
+ -lncbi-vdb \
$(BINDIR)/rcexplain: $(RCEXPLAIN_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(RCEXPLAIN_LIB)
@@ -171,7 +165,8 @@ $(BINDIR)/vdb-unlock: $(VDB_UNLOCK_OBJ)
#
T_SRC = \
- test-sra
+ PrintOS \
+ test-sra \
T_OBJ = \
$(addsuffix .$(OBJX),$(T_SRC))
@@ -197,13 +192,8 @@ VDB_PASSWD_OBJ = \
VDB_PASSWD_LIB = \
-skapp \
- -lvfs \
- -lkurl \
- -lkrypto \
- -lkfg \
- -lkfs \
- -sksproc \
- -lklib
+ -lncbi-vdb \
+
$(BINDIR)/vdb-passwd: $(VDB_PASSWD_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(VDB_PASSWD_LIB)
diff --git a/tools/util/kdbmeta.c b/tools/util/kdbmeta.c
index c760d98..d9a9b53 100644
--- a/tools/util/kdbmeta.c
+++ b/tools/util/kdbmeta.c
@@ -50,6 +50,10 @@
#include <kdb/column.h>
#include <kdb/meta.h>
#include <kdb/namelist.h>
+#include <kdb/kdb-priv.h>
+#include <vfs/manager.h>
+#include <vfs/resolver.h>
+#include <vfs/path.h>
#include <kapp/main.h>
#include <kapp/args.h>
#include <klib/text.h>
@@ -1163,26 +1167,43 @@ rc_t CC KMain ( int argc, char *argv [] )
#if ! WINDOWS /* TOOLS_USE_SRAPATH != 0 */
#warning fix kdbmanagerVPathType to understand accessions
-#if 0
{
- SRAPath * sra_path = NULL;
- rc = SRAPathMake (&sra_path, NULL);
+ const VFSManager * vfs;
+ rc = KDBManagerGetVFSManager ( mgr, & vfs );
if ( rc == 0 )
{
- if ( ! SRAPathTest (sra_path, pc))
+ VResolver * resolver;
+ rc = VFSManagerGetResolver ( vfs, & resolver );
+ if ( rc == 0 )
{
- rc = SRAPathFind (sra_path, pc, objpath, sizeof (objpath));
- if (rc == 0)
- found = true;
+ VPath * query;
+ rc = VFSManagerMakePath ( vfs, & query, "%s", pc );
+ if ( rc == 0 )
+ {
+ const VPath * local;
+ rc = VResolverQuery ( resolver, 0, query, & local, NULL, NULL );
+ if ( rc == 0 )
+ {
+ rc = VPathReadPath ( local, objpath, sizeof objpath, NULL );
+ if ( rc == 0 )
+ found = true;
+
+ VPathRelease ( local );
+ }
+ else if ( GetRCState ( rc ) == rcNotFound )
+ {
+ rc = 0;
+ }
+
+ VPathRelease ( query );
+ }
+
+ VResolverRelease ( resolver );
}
- SRAPathRelease (sra_path);
- }
- else if ( GetRCState ( rc ) == rcNotFound && GetRCTarget ( rc ) == rcDylib )
- {
- rc = 0;
+
+ VFSManagerRelease ( vfs );
}
}
-#endif
if ( ! found)
{
diff --git a/tools/util/kdbmeta.vers b/tools/util/kdbmeta.vers
index 0bee604..3f684d2 100644
--- a/tools/util/kdbmeta.vers
+++ b/tools/util/kdbmeta.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/util/md5cp.vers b/tools/util/md5cp.vers
new file mode 100644
index 0000000..3f684d2
--- /dev/null
+++ b/tools/util/md5cp.vers
@@ -0,0 +1 @@
+2.3.4
diff --git a/tools/util/rcexplain.vers b/tools/util/rcexplain.vers
index 0bee604..3f684d2 100644
--- a/tools/util/rcexplain.vers
+++ b/tools/util/rcexplain.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/util/rowwritetest.vers b/tools/util/rowwritetest.vers
index f90b1af..3f684d2 100644
--- a/tools/util/rowwritetest.vers
+++ b/tools/util/rowwritetest.vers
@@ -1 +1 @@
-2.3.2
+2.3.4
diff --git a/tools/util/schema-replace.vers b/tools/util/schema-replace.vers
new file mode 100644
index 0000000..3f684d2
--- /dev/null
+++ b/tools/util/schema-replace.vers
@@ -0,0 +1 @@
+2.3.4
diff --git a/tools/sra-pileup/reref.h b/tools/util/test-sra-priv.h
similarity index 83%
copy from tools/sra-pileup/reref.h
copy to tools/util/test-sra-priv.h
index 13e3136..924e5e2 100644
--- a/tools/sra-pileup/reref.h
+++ b/tools/util/test-sra-priv.h
@@ -23,26 +23,24 @@
* ===========================================================================
*
*/
-#ifndef _h_reref_
-#define _h_reref_
+
+#ifndef _h_util_test_sra_priv_
+#define _h_util_test_sra_priv_
+
+#ifndef _h_klib_defs_
+#include <klib/defs.h> /* rc_t */
+#endif
#ifdef __cplusplus
extern "C" {
#endif
-#if 0
-}
-#endif
-#include <kapp/args.h>
-#include <klib/out.h>
-#include <klib/rc.h>
-#include <klib/log.h>
+rc_t PrintOS(bool xml);
-#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
-
-rc_t report_on_reference( Args * args, bool extended );
+#ifdef __cplusplus
+}
#endif
+
+#endif /* _h_util_test_sra_priv_ */
diff --git a/tools/util/test-sra.c b/tools/util/test-sra.c
index ebdab16..c27895c 100644
--- a/tools/util/test-sra.c
+++ b/tools/util/test-sra.c
@@ -24,6 +24,8 @@
*
*/
+#include "test-sra-priv.h" /* PrintOS */
+
#include <kapp/main.h> /* KMain */
#include <kfg/config.h> /* KConfig */
@@ -34,22 +36,32 @@
#include <vfs/path.h> /* VPath */
#include <vfs/resolver.h> /* VResolver */
+#include <sra/sraschema.h> /* VDBManagerMakeSRASchema */
+
#include <vdb/database.h> /* VDatabase */
#include <vdb/dependencies.h> /* VDBDependencies */
#include <vdb/manager.h> /* VDBManager */
+#include <vdb/report.h> /* ReportSetVDBManager */
+#include <vdb/schema.h> /* VSchemaRelease */
+#include <vdb/table.h> /* VDBManagerOpenTableRead */
+#include <kns/ascp.h> /* ascp_locate */
#include <kns/curl-file.h> /* KCurlFileMake */
-#include <kns/manager.h> /* KNSManager */
+#include <kns/KCurlRequest.h> /* KCurlRequestRelease */
+#include <kns/manager-ext.h> /* KNSManagerNewReleaseVersion */
#include <kdb/manager.h> /* kptDatabase */
#include <kfs/directory.h> /* KDirectory */
#include <kfs/file.h> /* KFile */
+#include <klib/sra-release-version.h>
+#include <klib/data-buffer.h> /* KDataBuffer */
#include <klib/log.h> /* KLogHandlerSet */
#include <klib/out.h> /* KOutMsg */
#include <klib/printf.h> /* string_vprintf */
#include <klib/rc.h>
+#include <klib/report.h> /* ReportForceFinalize */
#include <klib/text.h> /* String */
#include <sysalloc.h>
@@ -76,23 +88,39 @@ typedef enum {
eDependMissing = 4,
eDependAll = 8,
eCurl = 16,
- eAll = 32
+ eVersion = 32,
+ eNewVersion = 64,
+ eOpenTable = 128,
+ eOpenDB = 256,
+ eType = 512,
+ eNcbiReport = 1024,
+ eOS = 2048,
+ eAscp = 4096,
+ eAscpVerbose = 8192,
+ eAll = 16384,
+ eNoTestArg = 32768
} Type;
+typedef uint16_t TTest;
+static const char TESTS[] = "crdDksSoOtnufFa";
typedef struct {
KConfig *cfg;
KDirectory *dir;
+ KNSManager *knsMgr;
const VDBManager *mgr;
+ VFSManager *vMgr;
const KRepositoryMgr *repoMgr;
VResolver *resolver;
- uint8_t tests;
+
+ TTest tests;
bool recursive;
bool noVDBManagerPathType;
bool noRfs;
+ bool full;
+ bool xml;
bool allowCaching;
VResolverEnableState cacheState;
} Main;
-
uint32_t CC KAppVersion(void) { return 0; }
const char UsageDefaultName[] = "test-sra";
@@ -100,8 +128,12 @@ const char UsageDefaultName[] = "test-sra";
rc_t CC UsageSummary(const char *prog_name) {
return KOutMsg(
"Usage:\n"
- " %s [+crdDa] [-crdDa] [-R] [-N] [-C] [options] name [ name... ]\n",
- prog_name);
+ " quick check mode:\n"
+ " %s -Q [ name... ]\n\n"
+ " full test mode:\n"
+ " %s [+crdDa] [-crdDa] [-R] [-N] [-C] [-X <type>] [options] "
+ "name [ name... ]\n"
+ , prog_name, prog_name);
}
rc_t CC Usage(const Args *args) {
@@ -118,20 +150,44 @@ rc_t CC Usage(const Args *args) {
rc = rc2;
}
-/* " t - test object's VDB type\n" */
rc2 = KOutMsg("\n"
"Test [SRA] object, resolve it, print dependencies, configuration\n\n"
"[+tests] - add tests\n"
- "[-tests] - remove tests\n\n"
+ "[-tests] - remove tests\n\n");
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ rc2 = KOutMsg(
"Tests:\n"
+ " s - print SRA software information\n"
+ " S - print SRA software information and last SRA toolkit versions\n"
+ " u - print operation system information\n"
" c - print configuration\n"
+ " n - print NCBI error report\n"
" k - print curl info\n"
+ " f - print ascp information\n"
+ " F - print verbose ascp information\n"
+ " t - print object types\n");
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ rc2 = KOutMsg(
" r - call VResolver\n"
" d - call ListDependencies(missing)\n"
" D - call ListDependencies(all)\n"
- " a - all tests\n\n"
- "If no tests were specified then all tests will be run\n\n"
- "-R - check object type recursively\n"
+ " o - call VDBManagerOpenTableRead(object)\n"
+ " O - call VDBManagerOpenDBRead(object)\n"
+ " a - all tests except VDBManagerOpen...Read and verbose ascp\n\n");
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ rc2 = KOutMsg(
+ "In quick check mode - the base checks are run;\n"
+ "in full test mode (default) all the tests are available.\n\n"
+ "In full mode, if no tests were specified then all tests will be run.\n"
+ "\n"
+ "-X < xml | text > - whether to generate well-formed XML\n"
+ "-R - check objects recursively\n"
"-N - do not call VDBManagerPathType\n"
"-C - do not disable caching (default: from configuration)\n\n"
"More options:\n");
@@ -144,12 +200,9 @@ rc_t CC Usage(const Args *args) {
return rc;
}
-static bool testArg(const char *arg, uint8_t *testOn, uint8_t *testOff) {
+static bool testArg(const char *arg, TTest *testOn, TTest *testOff) {
int j = 1;
- uint8_t *res = NULL;
-
-/* const char tests[] = "ctrdDa"; */
- const char tests[] = "crdDka";
+ TTest *res = NULL;
assert(arg && testOn && testOff);
if (arg[0] != '+' && arg[0] != '-') {
@@ -157,7 +210,7 @@ static bool testArg(const char *arg, uint8_t *testOn, uint8_t *testOff) {
}
if (arg[0] == '-' &&
- arg[1] != '\0' && strchr(tests, arg[1]) == NULL)
+ arg[1] != '\0' && strchr(TESTS, arg[1]) == NULL)
{
return false;
}
@@ -165,9 +218,9 @@ static bool testArg(const char *arg, uint8_t *testOn, uint8_t *testOff) {
res = arg[0] == '-' ? testOff : testOn;
for (j = 1; arg[j] != '\0'; ++j) {
- char *c = strchr(tests, arg[j]);
+ char *c = strchr(TESTS, arg[j]);
if (c != NULL) {
- int offset = c - tests;
+ int64_t offset = c - TESTS;
*res |= 1 << offset;
}
}
@@ -175,8 +228,8 @@ static bool testArg(const char *arg, uint8_t *testOn, uint8_t *testOff) {
return true;
}
-static uint8_t Turn(uint8_t in, uint8_t tests, bool on) {
- uint8_t c = 1;
+static TTest Turn(TTest in, TTest tests, bool on) {
+ TTest c = 1;
for (c = 1; c < eAll; c <<= 1) {
if (tests & c) {
if (on) {
@@ -190,8 +243,8 @@ static uint8_t Turn(uint8_t in, uint8_t tests, bool on) {
return in;
}
-static uint8_t processTests(uint8_t testsOn, uint8_t testsOff) {
- uint8_t tests = 0;
+static TTest processTests(TTest testsOn, TTest testsOff) {
+ TTest tests = 0;
bool allOn = false;
bool allOff = false;
@@ -210,6 +263,9 @@ static uint8_t processTests(uint8_t testsOn, uint8_t testsOff) {
if (allOn) {
tests = ~0;
tests = Turn(tests, testsOff, false);
+ tests = Turn(tests, eOpenTable, testsOn & eOpenTable);
+ tests = Turn(tests, eOpenDB, testsOn & eOpenDB);
+ tests = Turn(tests, eOpenDB, testsOn & eAscpVerbose);
}
else if (allOff) {
tests = Turn(tests, testsOn, true);
@@ -218,16 +274,36 @@ static uint8_t processTests(uint8_t testsOn, uint8_t testsOff) {
tests = Turn(tests, testsOff, false);
tests = Turn(tests, testsOn, true);
}
- else {
+ else /* no test argument provided */ {
tests = ~0;
+ tests = Turn(tests, eOpenTable, false);
+ tests = Turn(tests, eOpenDB, false);
+ tests = Turn(tests, eAscpVerbose, false);
+ }
+
+ if (tests & eAscpVerbose) {
+ tests = Turn(tests, eAscp, true);
+ }
+
+ if (tests & eNcbiReport) {
+ tests = Turn(tests, eCfg, false);
}
return tests;
}
-static bool MainHasTest(const Main *self, Type type) {
+static void MainMakeQuick(Main *self) {
+ assert(self);
+ self->tests = eCurl;
+}
+
+static void MainAddTest(Main *self, Type type) {
assert(self);
- return self->tests & type;
+ self->tests |= type;
+}
+
+static bool MainHasTest(const Main *self, Type type) {
+ return (self->tests & type) != 0;
}
static void MainPrint(const Main *self) {
@@ -239,10 +315,6 @@ static void MainPrint(const Main *self) {
KOutMsg("eCfg\n");
}
-/* if (MainHasTest(self, eType)) {
- KOutMsg("eType\n");
- }*/
-
if (MainHasTest(self, eResolve)) {
KOutMsg("eResolve\n");
}
@@ -254,12 +326,47 @@ static void MainPrint(const Main *self) {
if (MainHasTest(self, eDependAll)) {
KOutMsg("eDependAll\n");
}
+
+ if (MainHasTest(self, eCurl)) {
+ KOutMsg("eCurl\n");
+ }
+
+ if (MainHasTest(self, eVersion)) {
+ KOutMsg("eVersion\n");
+ }
+
+ if (MainHasTest(self, eNewVersion)) {
+ KOutMsg("eNewVersion\n");
+ }
+
+ if (MainHasTest(self, eOpenTable)) {
+ KOutMsg("eOpenTable\n");
+ }
+
+ if (MainHasTest(self, eOpenDB)) {
+ KOutMsg("eOpenDB\n");
+ }
+
+ if (MainHasTest(self, eType)) {
+ KOutMsg("eType\n");
+ }
+
+ if (MainHasTest(self, eNcbiReport)) {
+ KOutMsg("eNcbiReport\n");
+ }
+
+ if (MainHasTest(self, eAll)) {
+ KOutMsg("eAll\n");
+ }
+
+ if (MainHasTest(self, eNoTestArg)) {
+ KOutMsg("eNoTestArg\n");
+ }
}
static rc_t MainInitObjects(Main *self) {
rc_t rc = 0;
- VFSManager* mgr = NULL;
VResolver *resolver = NULL;
if (rc == 0) {
@@ -268,6 +375,7 @@ static rc_t MainInitObjects(Main *self) {
if (rc == 0) {
rc = VDBManagerMakeRead(&self->mgr, NULL);
+ ReportSetVDBManager(self->mgr);
}
if (rc == 0) {
@@ -275,15 +383,19 @@ static rc_t MainInitObjects(Main *self) {
}
if (rc == 0) {
+ rc = KNSManagerMake(&self->knsMgr);
+ }
+
+ if (rc == 0) {
rc = KConfigMakeRepositoryMgrRead(self->cfg, &self->repoMgr);
}
if (rc == 0) {
- rc = VFSManagerMake(&mgr);
+ rc = VFSManagerMake(&self->vMgr);
}
if (rc == 0) {
- rc = VFSManagerGetResolver(mgr, &resolver);
+ rc = VFSManagerGetResolver(self->vMgr, &resolver);
}
if (rc == 0) {
@@ -293,12 +405,11 @@ static rc_t MainInitObjects(Main *self) {
}
if (rc == 0) {
- rc = VFSManagerMakeResolver(mgr, &self->resolver, self->cfg);
+ rc = VFSManagerMakeResolver(self->vMgr, &self->resolver, self->cfg);
}
RELEASE(VResolver, resolver);
- RELEASE(VFSManager, mgr);
return rc;
}
@@ -308,8 +419,10 @@ void _MainInit(Main *self, int argc, char *argv[], int *argi, char **argv2)
{
int i = 0;
- uint8_t testsOn = 0;
- uint8_t testsOff = 0;
+ bool hasTestArg = false;
+
+ TTest testsOn = 0;
+ TTest testsOff = 0;
assert(self && argv && argi && argv2);
@@ -320,10 +433,20 @@ void _MainInit(Main *self, int argc, char *argv[], int *argi, char **argv2)
if (!testArg(argv[i], &testsOn, &testsOff)) {
argv2[(*argi)++] = argv[i];
}
+ else {
+ hasTestArg = true;
+ }
}
self->tests = processTests(testsOn, testsOff);
+ if (hasTestArg) {
+ self->tests &= ~eNoTestArg;
+ }
+ else {
+ self->tests |= eNoTestArg;
+ }
+
MainPrint(self);
}
@@ -341,6 +464,132 @@ static char** MainInit(Main *self, int argc, char *argv[], int *argi) {
return argv2;
}
+static rc_t MainCallCgi(const Main *self,
+ const KConfigNode *node, const char *acc)
+{
+ rc_t rc = 0;
+ String *url = NULL;
+ struct KCurlRequest *req = NULL;
+ KDataBuffer result;
+ memset(&result, 0, sizeof result);
+ assert(self && node && acc);
+ if (rc == 0) {
+ rc = KConfigNodeReadString(node, &url);
+ if (url == NULL) {
+ rc = RC(rcExe, rcNode, rcReading, rcString, rcNull);
+ }
+ }
+ if (rc == 0) {
+ rc = KNSManagerMakeCurlRequest(self->knsMgr, &req, url->addr, false);
+ }
+ if (rc == 0) {
+ size_t s = string_size(acc);
+ String name;
+ String value;
+ CONST_STRING(&name, "acc");
+ StringInit(&value, acc, s, (uint32_t)s);
+ rc = KCurlRequestAddSField(req, &name, &value);
+ }
+ if (rc == 0) {
+ rc = KCurlRequestPerform(req, &result);
+ }
+ if (rc == 0) {
+ const char *start = (const void*)(result.base);
+ size_t size = KDataBufferBytes(&result);
+ if (*(start + size) != '\0') {
+ rc = RC(rcExe, rcString, rcParsing, rcString, rcUnexpected);
+ }
+ if (strstr(start, "200|ok") == NULL) {
+ rc = RC(rcExe, rcString, rcParsing, rcParam, rcIncorrect);
+ }
+ }
+ KDataBufferWhack(&result);
+ RELEASE(KCurlRequest, req);
+ free(url);
+ if (rc == 0) {
+ OUTMSG(("NCBI access: ok\n"));
+ }
+ else {
+ OUTMSG(("ERROR: cannot access NCBI Website\n"));
+ }
+ return rc;
+}
+
+#define rcResolver rcTree
+static rc_t MainQuickResolveQuery(const Main *self, const char *acc) {
+ rc_t rc = 0;
+ VPath *query = NULL;
+ const VPath *remote = NULL;
+ const VPath *cache = NULL;
+ assert(self && acc);
+ rc = VFSManagerMakePath(self->vMgr, &query, acc);
+ if (rc == 0) {
+ if (!self->allowCaching) {
+ VResolverCacheEnable(self->resolver, self->cacheState);
+ }
+ rc = VResolverQuery(self->resolver, eProtocolHttp, query,
+ NULL, &remote, &cache);
+ if (!self->allowCaching) {
+ VResolverCacheEnable(self->resolver, vrAlwaysDisable);
+ }
+ }
+ if (rc == 0) {
+ if (remote != NULL) {
+ OUTMSG(("remote location: ok\n"));
+ }
+ else {
+ OUTMSG(("ERROR: cannot resolve remote location\n"));
+ rc = RC(rcExe, rcResolver, rcResolving, rcPath, rcNotFound);
+ }
+ if (cache != NULL) {
+ OUTMSG(("cache location: ok\n"));
+ }
+ else {
+ OUTMSG(("ERROR: cannot resolve cache location\n"));
+ rc = RC(rcExe, rcResolver, rcResolving, rcPath, rcNotFound);
+ }
+ }
+ else {
+ OUTMSG(("ERROR: cannot resolve public accession\n"));
+ }
+ RELEASE(VPath, cache);
+ RELEASE(VPath, remote);
+ RELEASE(VPath, query);
+ return rc;
+}
+
+static rc_t MainQuickCheck(const Main *self) {
+ rc_t rc = 0;
+ rc_t rc2 = 0;
+ const char acc[] = "SRR000001";
+ const char path[] = "/repository/remote/protected/CGI/resolver-cgi";
+ const KConfigNode *node = NULL;
+ assert(self);
+ rc = KConfigOpenNodeRead(self->cfg, &node, path);
+ if (rc == 0) {
+ OUTMSG(("configuration: found\n"));
+ }
+ else {
+ OUTMSG(("ERROR: configuration not found or incomplete\n"));
+ }
+ if (rc == 0) {
+ rc_t rc3 = MainCallCgi(self, node, acc);
+ if (rc3 != 0 && rc2 == 0) {
+ rc2 = rc3;
+ }
+
+ rc3 = MainQuickResolveQuery(self, acc);
+ if (rc3 != 0 && rc2 == 0) {
+ rc2 = rc3;
+ }
+ }
+ RELEASE(KConfigNode, node);
+ if (rc2 != 0 && rc == 0) {
+ rc = rc2;
+ }
+ return rc;
+}
+
static rc_t MainPrintConfig(const Main *self) {
rc_t rc = 0;
@@ -596,7 +845,7 @@ static rc_t MainReport(const Main *self,
rc = _KDirectoryReport(self->dir, name, size, type, alias);
- if (!self->noVDBManagerPathType) { /* && MainHasTest(self, eType)) { */
+ if (!self->noVDBManagerPathType) {
_VDBManagerReport(self->mgr, name, type);
}
@@ -607,6 +856,40 @@ static rc_t MainReport(const Main *self,
return rc;
}
+static rc_t MainOpenAs(const Main *self, const char *name, bool isDb) {
+ rc_t rc = 0;
+ const VTable *tbl = NULL;
+ const VDatabase *db = NULL;
+ VSchema *schema = NULL;
+ assert(self);
+
+ rc = VDBManagerMakeSRASchema(self->mgr, &schema);
+ if (rc != 0) {
+ OUTMSG(("VDBManagerMakeSRASchema() = %R\n", rc));
+ }
+
+ if (isDb) {
+ rc = VDBManagerOpenDBRead(self->mgr, &db, schema, name);
+ ReportResetDatabase(name, db);
+ OUTMSG(("VDBManagerOpenDBRead(%s) = ", name));
+ }
+ else {
+ rc = VDBManagerOpenTableRead(self->mgr, &tbl, schema, name);
+ ReportResetTable(name, tbl);
+ OUTMSG(("VDBManagerOpenTableRead(%s) = ", name));
+ }
+ if (rc == 0) {
+ OUTMSG(("OK\n"));
+ }
+ else {
+ OUTMSG(("%R\n", rc));
+ }
+ RELEASE(VDatabase, db);
+ RELEASE(VTable, tbl);
+ RELEASE(VSchema, schema);
+ return rc;
+}
+
#define rcResolver rcTree
static bool NotFoundByResolver(rc_t rc) {
if (GetRCModule(rc) == rcVFS) {
@@ -630,49 +913,67 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
EPathType type, const char *name, const VPath* remote, int64_t *size,
bool fasp, const KFile *fRemote)
{
+ const char *eol = "\n";
+
+ assert(self);
+
+ eol = self->xml ? "<br/>\n" : "\n";
+
switch (type) {
case ePathLocal:
- OUTMSG(("Local: "));
+ OUTMSG(("Local:\t\t "));
break;
case ePathRemote:
- OUTMSG(("Remote %s: ", fasp ? "fasp" : "http"));
+ OUTMSG(("Remote %s:\t ", fasp ? "fasp" : "http"));
break;
case ePathCache:
- OUTMSG(("Cache %s: ", fasp ? "fasp" : "http"));
+ OUTMSG(("Cache %s:\t ", fasp ? "fasp" : "http"));
if (remote == NULL) {
- OUTMSG(("skipped\n"));
+ OUTMSG(("skipped%s", eol));
return rc;
}
break;
}
if (rc != 0) {
if (NotFoundByResolver(rc)) {
- OUTMSG(("not found\n"));
+ OUTMSG(("not found%s", eol));
rc = 0;
}
else {
switch (type) {
case ePathLocal:
- OUTMSG(("VResolverLocal(%s) = %R\n", name, rc));
+ OUTMSG(("VResolverLocal(%s) = %R%s", name, rc, eol));
break;
case ePathRemote:
- OUTMSG(("VResolverRemote(%s) = %R\n", name, rc));
+ OUTMSG(("VResolverRemote(%s) = %R%s", name, rc, eol));
break;
case ePathCache:
- OUTMSG(("VResolverCache(%s) = %R\n", name, rc));
+ OUTMSG(("VResolverCache(%s) = %R%s", name, rc, eol));
break;
}
}
}
else {
+ const char ncbiFile[] = "ncbi-file:";
+ size_t sz = sizeof ncbiFile - 1;
const String *s = NULL;
+ char buffer[PATH_MAX] = "";
+ const char *fPath = buffer;
rc_t rc = VPathMakeString(path, &s);
+ if (rc != 0 || s == NULL || s->addr == NULL ||
+ string_cmp(s->addr, sz, ncbiFile, sz, (uint32_t)sz) == 0)
+ {
+ rc = VPathReadPath(path, buffer, sizeof buffer, NULL);
+ }
+ else {
+ fPath = s->addr;
+ }
if (rc == 0) {
- OUTMSG(("%S ", s));
+ OUTMSG(("%s ", fPath));
switch (type) {
case ePathLocal:
case ePathCache:
- rc = MainReport(self, s->addr, size, NULL, NULL);
+ rc = MainReport(self, fPath, size, NULL, NULL);
break;
case ePathRemote: {
uint64_t sz = 0;
@@ -689,7 +990,7 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
break;
}
}
- OUTMSG(("\n"));
+ OUTMSG(("%s", eol));
}
else {
const char *s = "";
@@ -707,15 +1008,15 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
assert(0);
break;
}
- OUTMSG(("VPathMakeUri(VResolver%s(%s)) = %R\n", s, name, rc));
+ OUTMSG(("VPathMakeUri(VResolver%s(%s)) = %R%s", s, name, rc, eol));
}
if (type == ePathCache) {
char cachecache[PATH_MAX] = "";
if (rc == 0) {
rc = string_printf(cachecache,
- sizeof cachecache, NULL, "%s.cache", s->addr);
+ sizeof cachecache, NULL, "%s.cache", fPath);
if (rc != 0) {
- OUTMSG(("string_printf(%s) = %R\n", s->addr, rc));
+ OUTMSG(("string_printf(%s) = %R%s", fPath, rc, eol));
}
}
@@ -723,7 +1024,7 @@ static rc_t MainPathReport(const Main *self, rc_t rc, const VPath *path,
OUTMSG(("Cache.cache %s: ", fasp ? "fasp" : "http"));
OUTMSG(("%s ", cachecache));
rc = MainReport(self, cachecache, NULL, NULL, NULL);
- OUTMSG(("\n"));
+ OUTMSG(("%s", eol));
}
}
free((void*)s);
@@ -822,6 +1123,8 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
const char *name, const VPath *query,
bool fasp, VRemoteProtocols protocols, EQueryType type)
{
+ const char *eol = "\n";
+
rc_t rc = 0;
rc_t rc2 = 0;
const VPath *local = NULL;
@@ -831,6 +1134,10 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
const VPath **pRemote = NULL;
const VPath **pCache = NULL;
+ assert(self);
+
+ eol = self->xml ? "<br/>\n" : "\n";
+
switch (type) {
case eQueryLocal:
case eQueryAll:
@@ -853,10 +1160,10 @@ static rc_t VResolverQueryByType(const Main *self, const VResolver *resolver,
}
rc = VResolverQuery(resolver, protocols, query, pLocal, pRemote, pCache);
- OUTMSG(("\nVResolverQuery(%s, %s, local%s, remote%s, cache%s)= %R\n",
+ OUTMSG(("\nVResolverQuery(%s, %s, local%s, remote%s, cache%s)= %R%s",
name, protocols == eProtocolHttp ? "Http" : "FaspHttp",
pLocal == NULL ? "=NULL" : "", pRemote == NULL ? "=NULL" : "",
- pCache == NULL ? "=NULL" : "", rc));
+ pCache == NULL ? "=NULL" : "", rc, eol));
if (rc == 0) {
if (local != NULL) {
rc2 = MainPathReport(self,
@@ -923,8 +1230,9 @@ static rc_t MainResolveQuery(const Main *self, const VResolver *resolver,
}
static rc_t MainResolve(const Main *self, const KartItem *item,
- const char *name, int64_t *localSz, int64_t *remoteSz)
+ const char *name, int64_t *localSz, int64_t *remoteSz, bool refseqCtx)
{
+ const char root[] = "Resolve";
rc_t rc = 0;
VPath* acc = NULL;
@@ -932,61 +1240,63 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
assert(self);
+ if (self->xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+
if (rc == 0) {
- VFSManager *mgr = NULL;
- rc = VFSManagerMake(& mgr);
- if (rc != 0) {
- OUTMSG(("VFSManagerMake = %R\n", rc));
+ if (item == NULL) {
+ if (refseqCtx) {
+ rc = VFSManagerMakePath(self->vMgr, &acc,
+ "ncbi-acc:%s?vdb-ctx=refseq", name);
+ }
+ else {
+ rc = VFSManagerMakePath(self->vMgr, &acc, name);
+ }
+ if (rc != 0) {
+ OUTMSG(("VFSManagerMakePath(%s) = %R\n", name, rc));
+ }
}
else {
- if (item == NULL) {
- rc = VFSManagerMakePath(mgr, &acc, name);
+ const KRepository *p_protected = NULL;
+ uint64_t oid = 0;
+ uint64_t project = 0;
+ if (rc == 0) {
+ rc = KartItemProjIdNumber(item, &project);
if (rc != 0) {
- OUTMSG(("VFSManagerMakePath(%s) = %R\n", name, rc));
+ OUTMSG(("KartItemProjectIdNumber = %R\n", rc));
}
}
- else {
- const KRepository *p_protected = NULL;
- uint64_t oid = 0;
- uint64_t project = 0;
- if (rc == 0) {
- rc = KartItemProjIdNumber(item, &project);
- if (rc != 0) {
- OUTMSG(("KartItemProjectIdNumber = %R\n", rc));
- }
- }
- if (rc == 0) {
- rc = KartItemItemIdNumber(item, &oid);
- if (rc != 0) {
- OUTMSG(("KartItemItemIdNumber = %R\n", rc));
- }
+ if (rc == 0) {
+ rc = KartItemItemIdNumber(item, &oid);
+ if (rc != 0) {
+ OUTMSG(("KartItemItemIdNumber = %R\n", rc));
}
- if (rc == 0) {
- rc = VFSManagerMakeOidPath(mgr, &acc, oid);
- if (rc != 0) {
- OUTMSG(("VFSManagerMakePath(%d) = %R\n", oid, rc));
- }
+ }
+ if (rc == 0) {
+ rc = VFSManagerMakeOidPath(self->vMgr, &acc, (uint32_t)oid);
+ if (rc != 0) {
+ OUTMSG(("VFSManagerMakePath(%d) = %R\n", oid, rc));
}
- if (rc == 0) {
- rc = KRepositoryMgrGetProtectedRepository(self->repoMgr,
- project, &p_protected);
- if (rc != 0) {
- OUTMSG((
- "KRepositoryMgrGetProtectedRepository(%d) = %R\n",
- project, rc));
- }
+ }
+ if (rc == 0) {
+ rc = KRepositoryMgrGetProtectedRepository(self->repoMgr,
+ (uint32_t)project, &p_protected);
+ if (rc != 0) {
+ OUTMSG((
+ "KRepositoryMgrGetProtectedRepository(%d) = %R\n",
+ project, rc));
}
- if (rc == 0) {
- rc = KRepositoryMakeResolver(p_protected, &resolver,
- self->cfg);
- if (rc != 0) {
- OUTMSG((
- "KRepositoryMakeResolver(%d) = %R\n", project, rc));
- }
+ }
+ if (rc == 0) {
+ rc = KRepositoryMakeResolver(p_protected, &resolver,
+ self->cfg);
+ if (rc != 0) {
+ OUTMSG((
+ "KRepositoryMakeResolver(%d) = %R\n", project, rc));
}
- RELEASE(KRepository, p_protected);
}
- RELEASE(VFSManager, mgr);
+ RELEASE(KRepository, p_protected);
}
}
@@ -1036,18 +1346,31 @@ static rc_t MainResolve(const Main *self, const KartItem *item,
RELEASE(VPath, acc);
RELEASE(VResolver, resolver);
+ if (self->xml) {
+ OUTMSG(("</%s>\n", root));
+ }
+
return rc;
}
static
rc_t MainDepend(const Main *self, const char *name, bool missing)
{
+ const char root[] = "Dependencies";
+ const char *eol = "\n";
rc_t rc = 0;
const VDatabase *db = NULL;
const VDBDependencies* dep = NULL;
uint32_t count = 0;
+ assert(self);
+ eol = self->xml ? "<br/>\n" : "\n";
+
+ if (self->xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+
if (rc == 0) {
rc = VDBManagerOpenDBRead(self->mgr, &db, NULL, name);
if (rc != 0) {
@@ -1065,8 +1388,8 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
rc = VDatabaseListDependenciesWithCaching(db,
&dep, missing, !self->allowCaching);
if (rc != 0) {
- OUTMSG(("VDatabaseListDependencies(%s, %s) = %R\n",
- name, missing ? "missing" : "all", rc));
+ OUTMSG(("VDatabaseListDependencies(%s, %s) = %R%s",
+ name, missing ? "missing" : "all", rc, eol));
}
if (!self->allowCaching) {
VResolverCacheEnable(self->resolver, vrAlwaysDisable);
@@ -1089,15 +1412,21 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
uint32_t i = 0;
rc_t rc2 = 0;
for (i = 0; i < count; ++i) {
+ const char root[] = "Dependency";
bool b = true;
const char *s = NULL;
+ const char *seqId = NULL;
KPathType type = kptNotFound;
+ if (self->xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+
OUTMSG((" %6d\t", i + 1));
- rc2 = VDBDependenciesSeqId(dep, &s, i);
+ rc2 = VDBDependenciesSeqId(dep, &seqId, i);
if (rc2 == 0) {
- OUTMSG(("seqId=%s,", s));
+ OUTMSG(("seqId=%s,", seqId));
}
else {
OUTMSG(("VDBDependenciesSeqId(%s, %s, %i)=%R ",
@@ -1160,7 +1489,7 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
rc2 = VDBDependenciesPath(dep, &s, i);
if (rc2 == 0) {
- OUTMSG(("\n\tpathLocal=%s,", s == NULL ? "notFound" : s));
+ OUTMSG(("%s\tpathLocal=%s,", eol, s == NULL ? "notFound" : s));
}
else {
OUTMSG(("VDBDependenciesPath(%s, %s, %i)=%R ",
@@ -1173,10 +1502,10 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
rc2 = VDBDependenciesPathRemote(dep, &s, i);
if (rc2 == 0) {
if (s == NULL) {
- OUTMSG(("\n\tpathRemote: notFound "));
+ OUTMSG(("%s\tpathRemote: notFound ", eol));
}
else {
- OUTMSG(("\n\tpathRemote: %s ", s));
+ OUTMSG(("%s\tpathRemote: %s ", eol, s));
if (!self->noRfs) {
const KFile *f = NULL;
rc2 = KCurlFileMake(&f, s, false);
@@ -1213,11 +1542,11 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
rc2 = VDBDependenciesPathCache(dep, &s, i);
if (rc2 == 0) {
- OUTMSG(("\n\tpathCache: %s ", s == NULL ? "notFound" : s));
+ OUTMSG(("%s\tpathCache: %s ", eol, s == NULL ? "notFound" : s));
if (s != NULL) {
char cachecache[PATH_MAX] = "";
rc2 = MainReport(self, s, NULL, NULL, NULL);
- OUTMSG(("\n"));
+ OUTMSG(("%s", eol));
if (rc == 0) {
rc = rc2;
}
@@ -1228,7 +1557,7 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
if (rc == 0) {
rc = rc2;
}
- OUTMSG(("string_printf(%s) = %R\n", s, rc2));
+ OUTMSG(("string_printf(%s) = %R%s", s, rc2, eol));
}
}
if (rc == 0) {
@@ -1245,39 +1574,142 @@ rc_t MainDepend(const Main *self, const char *name, bool missing)
rc = rc2;
}
}
+ if (MainHasTest(self, eResolve) && seqId != NULL) {
+ int64_t remoteSz = 0;
+ OUTMSG(("%s", eol));
+ rc2 = MainResolve(self, NULL, seqId, NULL, &remoteSz, true);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
- OUTMSG(("\n"));
+ if (self->xml) {
+ OUTMSG(("</%s>\n", root));
+ }
+ else {
+ OUTMSG(("%s", eol));
+ }
}
}
RELEASE(VDBDependencies, dep);
RELEASE(VDatabase, db);
+ if (self->xml) {
+ OUTMSG(("</%s>\n", root));
+ }
+
return rc;
}
-static rc_t PrintCurl() {
+static rc_t MainPrintAscp(const Main *self) {
+ rc_t rc = 0;
+ bool status = false;
+ const char *b = self->xml ? " <Ascp>\n" : "";
+ const char *e = self->xml ? " </Ascp>\n" : "\n";
+ const char *ascp_bin = NULL;
+ const char *private_file = NULL;
+ AscpOptions opt;
+ memset(&opt, 0, sizeof opt);
+ assert(self);
+ if (MainHasTest(self, eAscpVerbose)) {
+ status = true;
+ }
+ OUTMSG(("%s", b));
+ rc = ascp_locate(&ascp_bin, &private_file, true, status);
+ if (rc != 0) {
+ OUTMSG(("ascp_locate = %R\n", rc));
+ }
+ if (rc == 0) {
+ const char *b = self->xml ? " <Ascp>" : "ascp : ";
+ const char *e = self->xml ? "</Ascp>\n" : "\n";
+ OUTMSG(("%s%s%s", b, ascp_bin == NULL ? "NotFound" : ascp_bin, e));
+ }
+ if (rc == 0) {
+ const char *b = self->xml ? " <KeyFile>" : "key file: ";
+ const char *e = self->xml ? "</KeyFile>\n" : "\n";
+ OUTMSG(("%s%s%s", b,
+ private_file == NULL ? "NotFound" : private_file, e));
+ }
+
+ rc = aspera_options(&opt);
+ if (rc != 0) {
+ OUTMSG(("%saspera_options = %R%s", b, rc, e));
+ }
+ if (rc == 0) {
+ const char *b = self->xml ? " <MaxRate>" : "max rate: ";
+ const char *e = self->xml ? "</MaxRate>\n" : "\n";
+ OUTMSG(("%s%s%s", b,
+ opt.target_rate == NULL ? "NotFound" : opt.target_rate, e));
+ }
+ if (rc == 0) {
+ const char *b = self->xml ? " <Disabled>" : "disabled: ";
+ const char *e = self->xml ? "</Disabled>\n" : "\n";
+ OUTMSG(("%s%s%s", b, opt.disabled ? "true" : "false", e));
+ }
+
+ if (ascp_bin != NULL) {
+ size_t num_writ = 0;
+ char command[PATH_MAX] = "";
+ const char *b = self->xml ? "<Version>\n" : "";
+ const char *e = self->xml ? "</Version>\n" : "";
+ OUTMSG(("%s", b));
+ rc = string_printf(command, sizeof command, &num_writ,
+ "\"%s\" -A", ascp_bin);
+ if (rc != 0 || num_writ >= sizeof command) {
+ OUTMSG(("cannot generate ascp command: %R\n", rc));
+ }
+ else {
+ int s = system(command);
+ if (s != 0) {
+ OUTMSG(("system(%s) = %d\n", command, s));
+ }
+ }
+ OUTMSG(("%s", e));
+ }
+
+ free((void*)ascp_bin);
+ free((void*)private_file);
+
+ OUTMSG(("%s", e));
+ return 0;
+}
+
+static rc_t PrintCurl(bool full, bool xml) {
+ const char *b = xml ? " <Curl>" : "";
+ const char *e = xml ? "</Curl>" : "";
+
KNSManager *mgr = NULL;
rc_t rc = KNSManagerMake(&mgr);
if (rc != 0) {
- OUTMSG(("KNSManagerMake = %R\n", rc));
+ OUTMSG(("%sKNSManagerMake = %R%s\n", b, rc, e));
}
-
- if (rc == 0) {
+ else {
rc_t rc = KNSManagerAvail(mgr);
- OUTMSG(("KNSManagerAvail = %R", rc));
+ OUTMSG(("%s", b));
+
+ if (full) {
+ OUTMSG(("KNSManagerAvail = %R. ", rc));
+ }
if (rc == 0) {
const char *version_string = NULL;
rc = KNSManagerCurlVersion(mgr, &version_string);
if (rc == 0) {
- OUTMSG((". Curl Version = %s\n", version_string));
+ if (full) {
+ OUTMSG(("Curl Version = %s", version_string));
+ }
}
else {
- OUTMSG((". KNSManagerCurlVersion = %R\n", rc));
+ OUTMSG(("KNSManagerCurlVersion = %R", rc));
}
}
+
+ if (rc == 0 && !full) {
+ OUTMSG(("libcurl: found"));
+ }
+ OUTMSG(("%s\n", e));
}
RELEASE(KNSManager, mgr);
@@ -1290,6 +1722,8 @@ static rc_t PrintCurl() {
static
rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
{
+ const char root[] = "Object";
+
rc_t rc = 0;
rc_t rce = 0;
@@ -1306,14 +1740,20 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
assert(self);
+ if (self->xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+
if (item != NULL) {
type = kptKartITEM;
}
else {
+ const char *eol = self->xml ? "<br/>\n" : "\n";
+
rc = string_vprintf(arg, sizeof arg, &num_writ, aArg, args);
if (rc != 0) {
- OUTMSG(("string_vprintf(%s)=%R\n", aArg, rc));
+ OUTMSG(("string_vprintf(%s)=%R%s", aArg, rc, eol));
return rc;
}
assert(num_writ < sizeof arg);
@@ -1321,12 +1761,27 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
OUTMSG(("\n"));
rc = printString(arg);
if (rc != 0) {
- OUTMSG(("printString=%R\n", rc));
+ OUTMSG(("printString=%R%s", rc, eol));
return rc;
}
- OUTMSG((" "));
- rc = MainReport(self, arg, &directSz, &type, &alias);
- OUTMSG(("\n"));
+ if (MainHasTest(self, eType)) {
+ OUTMSG((" "));
+ rc = MainReport(self, arg, &directSz, &type, &alias);
+ }
+ OUTMSG(("%s", eol));
+
+ if (MainHasTest(self, eOpenTable)) {
+ rc_t rc2 = MainOpenAs(self, arg, false);
+ if (rce == 0 && rc2 != 0) {
+ rce = rc2;
+ }
+ }
+ if (MainHasTest(self, eOpenDB)) {
+ rc_t rc2 = MainOpenAs(self, arg, true);
+ if (rce == 0 && rc2 != 0) {
+ rce = rc2;
+ }
+ }
}
if (self->recursive && type == kptDir && !alias) {
@@ -1395,7 +1850,8 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
}
else {
if (MainHasTest(self, eResolve)) {
- rc_t rc2 = MainResolve(self, item, arg, &localSz, &remoteSz);
+ rc_t rc2 = MainResolve(self, item, arg, &localSz, &remoteSz,
+ false);
if (rc == 0 && rc2 != 0) {
rc = rc2;
}
@@ -1425,11 +1881,14 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
)
{
OUTMSG(("FILE SIZES DO NOT MATCH: "));
- if (directSz != -1 && localSz != -1 && directSz != remoteSz)
+ if (directSz != -1 && localSz != -1 &&
+ directSz != remoteSz)
{
OUTMSG(("direct=%ld != remote=%ld. ", directSz, remoteSz));
}
- if (remoteSz != -1 && localSz != -1 && localSz != remoteSz) {
+ if (remoteSz != -1 && localSz != -1 &&
+ localSz != remoteSz)
+ {
OUTMSG(("local=%ld != remote=%ld. ", localSz, remoteSz));
}
OUTMSG(("\n"));
@@ -1442,6 +1901,90 @@ rc_t MainExec(const Main *self, const KartItem *item, const char *aArg, ...)
if (rce != 0 && rc == 0) {
rc = rce;
}
+
+ if (self->xml) {
+ OUTMSG(("</%s>\n", root));
+ }
+
+ return rc;
+}
+
+static
+rc_t _SraReleaseVersionPrint(const SraReleaseVersion *self, rc_t rc, bool xml,
+ const char *error, const char *msg)
+{
+ assert(self && error && msg);
+
+ if (rc != 0) {
+ OUTMSG(("ERROR: %s.", error));
+ }
+ else {
+ char version[256] = "";
+ const char *eol = xml ? "<br/>\n" : "\n";
+ rc = SraReleaseVersionPrint(self, version, sizeof version, NULL);
+ if (rc == 0) {
+ rc = OUTMSG(("%s: %s.%s", msg, version, eol));
+ }
+ }
+
+ return rc;
+}
+
+static rc_t MainPrintVersion(Main *self) {
+ const char root[] = "Version";
+ rc_t rc = 0;
+ SraReleaseVersion version;
+ SraReleaseVersion newVersion;
+ assert(self);
+ if (MainHasTest(self, eNewVersion)) {
+ MainAddTest(self, eVersion);
+ }
+ if (!MainHasTest(self, eVersion)) {
+ return 0;
+ }
+ memset(&version, 0, sizeof version);
+ memset(&newVersion, 0, sizeof newVersion);
+ if (self->xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+ rc = SraReleaseVersionGet(&version);
+ rc = _SraReleaseVersionPrint(&version, rc, self->xml,
+ "cannot get SRA Toolkit release version",
+ "NCBI SRA Toolkit release version");
+
+ if (MainHasTest(self, eNewVersion)) {
+ int32_t isNew = 0;
+ rc = KNSManagerNewReleaseVersion(self->knsMgr, &newVersion);
+ rc = _SraReleaseVersionPrint(&newVersion, rc, self->xml,
+ "cannot get latest available SRA Toolkit release version",
+ "Latest available NCBI SRA Toolkit release version");
+ if (rc == 0) {
+ rc = SraReleaseVersionCmp(&version, &newVersion, &isNew);
+ }
+ if (rc == 0) {
+ if (isNew > 0) {
+ OUTMSG((
+ "New version of SRA Toolkit is available for download from\n"
+ "\"http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=software\".\n"
+ ));
+ }
+ else if (isNew == 0) {
+ OUTMSG(("You already have the latest version of "
+ "SRA Toolkit.\n"));
+ }
+ else {
+ OUTMSG(("Your version of "
+ "SRA Toolkit is more recent than the latest available.\n"));
+ }
+ }
+ }
+ if (self->xml) {
+ OUTMSG(("</%s>\n", root));
+ }
+ else if (self->full) {
+ OUTMSG(("\n"));
+ }
+
return rc;
}
@@ -1452,7 +1995,9 @@ static rc_t MainFini(Main *self) {
RELEASE(VResolver, self->resolver);
RELEASE(KConfig, self->cfg);
+ RELEASE(KNSManager, self->knsMgr);
RELEASE(KRepositoryMgr, self->repoMgr);
+ RELEASE(VFSManager, self->vMgr);
RELEASE(VDBManager, self->mgr);
RELEASE(KDirectory, self->dir);
@@ -1463,6 +2008,14 @@ static rc_t MainFini(Main *self) {
#define ALIAS_CACHE "C"
static const char* USAGE_CACHE[] = { "do not disable caching", NULL };
+#define OPTION_FULL "full"
+#define ALIAS_FULL NULL
+static const char* USAGE_FULL[] = { "full test mode", NULL };
+
+#define OPTION_QUICK "quick"
+#define ALIAS_QUICK "Q"
+static const char* USAGE_QUICK[] = { "quick test mode", NULL };
+
#define OPTION_NO_RFS "no-rfs"
static const char* USAGE_NO_RFS[]
= { "do not check remote file size for dependencies", NULL };
@@ -1475,10 +2028,17 @@ static const char* USAGE_NO_VDB[] = { "do not call VDBManagerPathType", NULL };
#define ALIAS_REC "R"
static const char* USAGE_REC[] = { "check object type recursively", NULL };
+#define OPTION_OUT "output"
+#define ALIAS_OUT "X"
+static const char* USAGE_OUT[] = { "output type: one of (xml text)", NULL };
+
OptDef Options[] = { /* needs_value, required */
{ OPTION_CACHE , ALIAS_CACHE , NULL, USAGE_CACHE , 1, false, false },
+ { OPTION_FULL , ALIAS_FULL , NULL, USAGE_FULL , 1, false, false },
{ OPTION_NO_RFS, NULL , NULL, USAGE_NO_RFS, 1, false, false },
{ OPTION_NO_VDB, ALIAS_NO_VDB, NULL, USAGE_NO_VDB, 1, false, false },
+ { OPTION_OUT , ALIAS_OUT , NULL, USAGE_OUT , 1, true , false },
+ { OPTION_QUICK , ALIAS_QUICK , NULL, USAGE_QUICK , 1, false, false },
{ OPTION_REC , ALIAS_REC , NULL, USAGE_REC , 1, false, false }
};
@@ -1510,73 +2070,168 @@ rc_t CC KMain(int argc, char *argv[]) {
}
}
- if (rc == 0) {
- rc = MainInitObjects(&prms);
- }
-
- if (MainHasTest(&prms, eCfg)) {
- rc_t rc2 = MainPrintConfig(&prms);
- if (rc == 0 && rc2 != 0) {
- rc = rc2;
- }
- }
-
- if (MainHasTest(&prms, eCurl)) {
- PrintCurl();
- }
+ prms.full = true;
if (rc == 0) {
- rc = ArgsOptionCount(args, OPTION_NO_RFS, &pcount);
+ rc = ArgsOptionCount(args, OPTION_QUICK, &pcount);
if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_NO_RFS "' argument");
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_QUICK "' argument");
}
else {
if (pcount > 0) {
- prms.noRfs = true;
+ prms.full = false;
}
}
}
if (rc == 0) {
- rc = ArgsOptionCount(args, OPTION_NO_VDB, &pcount);
+ rc = ArgsOptionCount(args, OPTION_FULL, &pcount);
if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_NO_VDB "' argument");
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_FULL "' argument");
}
else {
if (pcount > 0) {
- prms.noVDBManagerPathType = true;
+ prms.full = true;
}
}
}
+ if (!prms.full) {
+ MainMakeQuick(&prms);
+ }
+
+ if (rc == 0) {
+ rc = MainInitObjects(&prms);
+ }
+
if (rc == 0) {
- rc = ArgsOptionCount(args, OPTION_REC, &pcount);
+ const char root[] = "Test-sra";
+ rc = ArgsOptionCount(args, OPTION_OUT, &pcount);
if (rc) {
- LOGERR(klogErr, rc, "Failure to get '" OPTION_REC "' argument");
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_OUT "' argument");
}
else {
if (pcount > 0) {
- prms.recursive = true;
+ const char* dummy = NULL;
+ rc = ArgsOptionValue(args, OPTION_OUT, 0, &dummy);
+ if (rc) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" OPTION_OUT "' argument");
+ }
+ else if (strcmp(dummy, "x") == 0 || strcmp(dummy, "X") == 0) {
+ prms.xml = true;
+ }
+ else {
+ prms.xml = false;
+ }
+ }
+ else {
+ prms.xml = MainHasTest(&prms, eCfg)
+ || MainHasTest(&prms, eNcbiReport);
}
}
- }
- if (rc == 0) {
- rc = ArgsParamCount(args, &pcount);
- }
+ if (prms.xml) {
+ OUTMSG(("<%s>\n", root));
+ }
+
+ MainPrintVersion(&prms);
- for (i = 0; i < pcount; ++i) {
- const char *name = NULL;
- rc3 = ArgsParamValue(args, i, &name);
- if (rc3 == 0) {
- rc_t rc2 = MainExec(&prms, NULL, name);
+ if (MainHasTest(&prms, eCfg)) {
+ rc_t rc2 = MainPrintConfig(&prms);
if (rc == 0 && rc2 != 0) {
rc = rc2;
}
}
- }
- if (rc == 0 && rc3 != 0) {
- rc = rc3;
+
+ if (MainHasTest(&prms, eOS)) {
+ PrintOS(prms.xml);
+ }
+
+ if (MainHasTest(&prms, eAscp)) {
+ MainPrintAscp(&prms);
+ }
+
+ if (MainHasTest(&prms, eCurl)) {
+ PrintCurl(prms.full, prms.xml);
+ }
+
+ if (!prms.full) {
+ rc_t rc2 = MainQuickCheck(&prms);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
+
+ if (rc == 0) {
+ rc = ArgsOptionCount(args, OPTION_NO_RFS, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" OPTION_NO_RFS "' argument");
+ }
+ else {
+ if (pcount > 0) {
+ prms.noRfs = true;
+ }
+ }
+ }
+
+ if (rc == 0) {
+ rc = ArgsOptionCount(args, OPTION_NO_VDB, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc,
+ "Failure to get '" OPTION_NO_VDB "' argument");
+ }
+ else {
+ if (pcount > 0) {
+ prms.noVDBManagerPathType = true;
+ }
+ }
+ }
+
+ if (rc == 0) {
+ rc = ArgsOptionCount(args, OPTION_REC, &pcount);
+ if (rc) {
+ LOGERR(klogErr, rc, "Failure to get '" OPTION_REC "' argument");
+ }
+ else {
+ if (pcount > 0) {
+ prms.recursive = true;
+ }
+ }
+ }
+
+ if (rc == 0) {
+ rc = ArgsParamCount(args, &pcount);
+ }
+
+ for (i = 0; i < pcount; ++i) {
+ const char *name = NULL;
+ rc3 = ArgsParamValue(args, i, &name);
+ if (rc3 == 0) {
+ rc_t rc2 = 0;
+ ReportResetObject(name);
+ rc2 = MainExec(&prms, NULL, name);
+ if (rc == 0 && rc2 != 0) {
+ rc = rc2;
+ }
+ }
+ }
+ if (rc == 0 && rc3 != 0) {
+ rc = rc3;
+ }
+
+ if (MainHasTest(&prms, eNcbiReport)) {
+ ReportForceFinalize();
+ }
+
+ if (!prms.full) {
+ OUTMSG(("\nAdd -F option to try all the tests."));
+ }
+
+ if (prms.xml) {
+ OUTMSG(("</%s>\n", root));
+ }
}
RELEASE(Args, args);
diff --git a/tools/util/test-sra.vers b/tools/util/test-sra.vers
index 0bee604..3f684d2 100644
--- a/tools/util/test-sra.vers
+++ b/tools/util/test-sra.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/util/testld.vers b/tools/util/testld.vers
index f90b1af..3f684d2 100644
--- a/tools/util/testld.vers
+++ b/tools/util/testld.vers
@@ -1 +1 @@
-2.3.2
+2.3.4
diff --git a/tools/util/txt2kdb.vers b/tools/util/txt2kdb.vers
index f90b1af..3f684d2 100644
--- a/tools/util/txt2kdb.vers
+++ b/tools/util/txt2kdb.vers
@@ -1 +1 @@
-2.3.2
+2.3.4
diff --git a/interfaces/krypto/extern.h b/tools/util/unix/PrintOS.c
similarity index 56%
copy from interfaces/krypto/extern.h
copy to tools/util/unix/PrintOS.c
index 4663585..019ca9c 100644
--- a/interfaces/krypto/extern.h
+++ b/tools/util/unix/PrintOS.c
@@ -24,43 +24,43 @@
*
*/
-#ifndef _h_krypto_extern_
-#define _h_krypto_extern_
+#include "test-sra-priv.h" /* PrintOS */
-#ifndef _h_klib_callconv_
-#include <klib/callconv.h>
-#endif
+#include <klib/debug.h> /* DBGMSG */
+#include <klib/out.h> /* OUTMSG */
-#if ! defined EXPORT_LATCH && _LIBRARY
+#include <errno.h>
+#include <string.h> /* memset */
+#include <stdio.h> /* perror */
-#define KRYPTO_EXTERN LIB_EXPORT
-#define KRYPTO_EXTERN_DATA extern LIB_EXPORT
-#define EXPORT_LATCH 1
+#include <sys/utsname.h> /* uname */
-#else
+rc_t PrintOS(bool xml) {
+ int ret = 1;
-#define KRYPTO_EXTERN LIB_IMPORT
-#define KRYPTO_EXTERN_DATA LIB_IMPORT
+ struct utsname unameData;
+ memset(&unameData, 0, sizeof unameData);
-#endif
+ errno = 0;
+ ret = uname(&unameData);
+ if (ret != 0) {
+ if (xml) {
+ OUTMSG((" <Os>"));
+ perror("uname returned : ");
+ OUTMSG(("</Os>\n"));
+ }
+ else {
+ perror("uname returned : ");
+ }
-#if defined _MSC_VER && ! _STATIC
+ return 0;
+ }
+ else {
+ const char *b = xml ? " <Os>" : "Operating system: '";
+ const char *e = xml ? "</Os>" : "'\n";
-/* __declspec ( dllimport ) will cause creation of
- function pointers rather than thunks, which makes
- the code that imports unable to link statically
- against a library. we leave this symbol defined as
- "extern" to use thunks instead. as a result, all
- function addresses resolve to the thunk and not
- the actual function. */
-#define LIB_IMPORT extern
-#define LIB_EXPORT __declspec ( dllexport )
-
-#else
-
-#define LIB_IMPORT extern
-#define LIB_EXPORT
-
-#endif
-
-#endif /* _h_krypto_extern_ */
+ return OUTMSG(("%s%s %s %s %s %s%s\n", b,
+ unameData.sysname, unameData.nodename, unameData.release,
+ unameData.version, unameData.machine, e));
+ }
+}
diff --git a/tools/util/vdb-lock.vers b/tools/util/vdb-lock.vers
index 0bee604..3f684d2 100644
--- a/tools/util/vdb-lock.vers
+++ b/tools/util/vdb-lock.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/util/vdb-passwd.vers b/tools/util/vdb-passwd.vers
index 0bee604..3f684d2 100644
--- a/tools/util/vdb-passwd.vers
+++ b/tools/util/vdb-passwd.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/util/vdb-unlock.vers b/tools/util/vdb-unlock.vers
index 0bee604..3f684d2 100644
--- a/tools/util/vdb-unlock.vers
+++ b/tools/util/vdb-unlock.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/sra-pileup/reref.h b/tools/util/win/PrintOS.c
similarity index 53%
copy from tools/sra-pileup/reref.h
copy to tools/util/win/PrintOS.c
index 13e3136..3b8aae6 100644
--- a/tools/sra-pileup/reref.h
+++ b/tools/util/win/PrintOS.c
@@ -23,26 +23,38 @@
* ===========================================================================
*
*/
-#ifndef _h_reref_
-#define _h_reref_
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if 0
-}
-#endif
+#include "test-sra-priv.h" /* PrintOS */
-#include <kapp/args.h>
+#include <klib/out.h> /* OUTMSG */
-#include <klib/out.h>
-#include <klib/rc.h>
-#include <klib/log.h>
+#include <windows.h>
-#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
+rc_t PrintOS(bool xml) {
+ const char *b = xml ? " <Os>" : "Operating system: '";
+ const char *e = xml ? "</Os>" : "'\n";
-rc_t report_on_reference( Args * args, bool extended );
+ OSVERSIONINFO osvi;
+ ZeroMemory(&osvi, sizeof osvi);
+ osvi.dwOSVersionInfoSize = sizeof osvi;
-#endif
+ if (GetVersionEx(&osvi)) {
+ if (osvi.dwPlatformId == 2) {
+ return OUTMSG((
+ "%sMicrosoft Windows. Version %d.%d (Build %d: %s)%s\n", b,
+ osvi.dwMajorVersion, osvi.dwMinorVersion, osvi.dwBuildNumber,
+ osvi.szCSDVersion, e));
+ }
+ else {
+ return OUTMSG((
+ "%sMicrosoft Windows. Version %d.%d (Build %d: %s). PlatformId %d%s\n",
+ b,
+ osvi.dwMajorVersion, osvi.dwMinorVersion, osvi.dwBuildNumber,
+ osvi.szCSDVersion, osvi.dwPlatformId, e));
+ }
+ }
+ else {
+ return OUTMSG(("%sGetLastError(GetVersionEx()) = %d%s\n",
+ b, GetLastError(), e));
+ }
+}
diff --git a/tools/vdb-config/Makefile b/tools/vdb-config/Makefile
index 5b5a192..f5c70b8 100644
--- a/tools/vdb-config/Makefile
+++ b/tools/vdb-config/Makefile
@@ -73,14 +73,6 @@ clean: stdclean
.PHONY: clean
#-------------------------------------------------------------------------------
-# tag
-#
-tag: \
- $(addsuffix _tag,$(ALL_TOOLS))
-
-.PHONY: tag $(addsuffix _tag,$(ALL_TOOLS))
-
-#-------------------------------------------------------------------------------
# vdb-config
#
VDB_CONFIG_SRC = \
@@ -98,5 +90,3 @@ VDB_CONFIG_LIB = \
$(BINDIR)/vdb-config: $(VDB_CONFIG_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(VDB_CONFIG_LIB)
-vdb_config_tag:
- @ $(TOP)/build/tag-module.sh $(MODULE) vdb-config $(VDB_CONFIG_OBJ)
diff --git a/tools/vdb-config/vdb-config.c b/tools/vdb-config/vdb-config.c
index 7d4b8a4..5263c6d 100644
--- a/tools/vdb-config/vdb-config.c
+++ b/tools/vdb-config/vdb-config.c
@@ -194,6 +194,26 @@ static rc_t KConfigNodeReadData(const KConfigNode* self,
return rc;
}
+static rc_t _printNodeData(const char *name, const char *data, uint32_t dlen) {
+ const char ticket[] = "download-ticket";
+ size_t l = sizeof ticket - 1;
+ if (string_cmp(name, string_measure(name, NULL),
+ ticket, l, (uint32_t)l) == 0)
+ {
+ const char *ellipsis = "";
+ const char replace[] =
+"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ if (dlen > 70) {
+ dlen = 70;
+ ellipsis = "...";
+ }
+ return OUTMSG(("%.*s%s", dlen, replace, ellipsis));
+ }
+ else {
+ return OUTMSG(("%.*s", dlen, data));
+ }
+}
+
#define VDB_CONGIG_OUTMSG(args) do { if (xml) { OUTMSG(args); } } while (false)
static rc_t KConfigNodePrintChildNames(bool xml, const KConfigNode* self,
const char* name, int indent, const char* aFullpath)
@@ -233,10 +253,12 @@ static rc_t KConfigNodePrintChildNames(bool xml, const KConfigNode* self,
{ VDB_CONGIG_OUTMSG((">")); }
if (hasData) {
if (xml) {
- VDB_CONGIG_OUTMSG(("%.*s", (int)num_read, buffer));
+ _printNodeData(name, buffer, num_read);
}
else {
- OUTMSG(("%s = \"%.*s\"\n", aFullpath, (uint32_t)num_read, buffer));
+ OUTMSG(("%s = \"", aFullpath));
+ _printNodeData(name, buffer, num_read);
+ OUTMSG(("\"\n"));
}
}
if (hasChildren)
@@ -1064,6 +1086,7 @@ rc_t CC KMain(int argc, char* argv[]) {
if (prm.ngc) {
const char *newRepoParentPath = NULL;
rc = KConfigImportNgc(cfg, prm.ngc, NULL, &newRepoParentPath);
+ DISP_RC2(rc, "cannot import ngc file", prm.ngc);
if (rc == 0) {
rc = KConfigCommit(cfg);
}
diff --git a/tools/vdb-config/vdb-config.vers b/tools/vdb-config/vdb-config.vers
index 0bee604..3f684d2 100644
--- a/tools/vdb-config/vdb-config.vers
+++ b/tools/vdb-config/vdb-config.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/vdb-copy/vdb-copy.vers b/tools/vdb-copy/vdb-copy.vers
index 0bee604..3f684d2 100644
--- a/tools/vdb-copy/vdb-copy.vers
+++ b/tools/vdb-copy/vdb-copy.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/vdb-decrypt/Makefile b/tools/vdb-decrypt/Makefile
index cd0e1da..85c41e3 100644
--- a/tools/vdb-decrypt/Makefile
+++ b/tools/vdb-decrypt/Makefile
@@ -92,13 +92,7 @@ VDB_DECRYPT_OBJ = \
VDB_DECRYPT_LIB = \
-lkapp \
- -lvfs \
- -lkurl \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lkproc \
- -lklib
+ -lncbi-vdb
$(BINDIR)/vdb-decrypt: $(VDB_DECRYPT_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(VDB_DECRYPT_LIB)
@@ -117,13 +111,7 @@ VDB_ENCRYPT_OBJ = \
VDB_ENCRYPT_LIB = \
-lkapp \
- -lvfs \
- -lkurl \
- -lkrypto \
- -lkfg \
- -lkfs \
- -lkproc \
- -lklib
+ -lncbi-vdb
$(BINDIR)/vdb-encrypt: $(VDB_ENCRYPT_OBJ)
$(LD) --exe --vers $(SRCDIR) -o $@ $^ $(VDB_ENCRYPT_LIB)
diff --git a/tools/vdb-decrypt/vdb-decrypt.c b/tools/vdb-decrypt/vdb-decrypt.c
index 8803acc..f0afbe2 100644
--- a/tools/vdb-decrypt/vdb-decrypt.c
+++ b/tools/vdb-decrypt/vdb-decrypt.c
@@ -102,7 +102,7 @@ bool DoThisFile (const KFile * infile, EncScheme enc, ArcScheme * parc)
* this will apply to KEncFiles versions 1 and 2, maybe not 3
* but will hopefully become obsolete eventually.
*/
- rc = KEncFileMakeRead (&Infile, infile, &Key);
+ rc = KEncFileMakeRead (&Infile, infile, &Key); /* replace with VFSManagerOpenFileReadDirectoryRelativeInt */
if (rc)
return false;
break;
diff --git a/tools/vdb-decrypt/vdb-decrypt.vers b/tools/vdb-decrypt/vdb-decrypt.vers
index 0bee604..3f684d2 100644
--- a/tools/vdb-decrypt/vdb-decrypt.vers
+++ b/tools/vdb-decrypt/vdb-decrypt.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/vdb-decrypt/vdb-encrypt.vers b/tools/vdb-decrypt/vdb-encrypt.vers
index 0bee604..3f684d2 100644
--- a/tools/vdb-decrypt/vdb-encrypt.vers
+++ b/tools/vdb-decrypt/vdb-encrypt.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/vdb-dump/Makefile b/tools/vdb-dump/Makefile
index 2101d7e..145ada3 100644
--- a/tools/vdb-dump/Makefile
+++ b/tools/vdb-dump/Makefile
@@ -93,6 +93,8 @@ VDB_DUMP_SRC = \
vdb-dump-helper \
vdb-dump-filter \
vdb-dump-formats \
+ vdb-dump-redir \
+ vdb-dump-fastq \
vdb-dump
VDB_DUMP_OBJ = \
diff --git a/tools/vdb-dump/vdb-dump-context.c b/tools/vdb-dump/vdb-dump-context.c
index 6c3d4ef..a4232e1 100644
--- a/tools/vdb-dump/vdb-dump-context.c
+++ b/tools/vdb-dump/vdb-dump-context.c
@@ -35,6 +35,8 @@
#include <os-native.h>
#include <sysalloc.h>
+#include <strtol.h>
+
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
@@ -81,6 +83,8 @@ static void vdco_init_values( p_dump_context ctx )
ctx->columns = NULL;
ctx->excluded_columns = NULL;
ctx->filter = NULL;
+ ctx->idx_range = NULL;
+ ctx->output_file = NULL;
ctx->print_row_id = true;
ctx->print_in_hex = false;
@@ -104,6 +108,9 @@ static void vdco_init_values( p_dump_context ctx )
ctx->objver_requested = false;
ctx->objtype_requested = false;
ctx->check_curl = false;
+ ctx->idx_enum_requested = false;
+ ctx->idx_range_requested = false;
+ ctx->disable_multithreading = false;
}
rc_t vdco_init( dump_context **ctx )
@@ -162,6 +169,10 @@ rc_t vdco_destroy( p_dump_context ctx )
free( (void*)ctx->excluded_columns );
ctx->excluded_columns = NULL;
}
+
+ free( (void*)ctx->output_file );
+ ctx->output_file = NULL;
+
vdn_destroy( ctx->row_generator );
free( ctx );
}
@@ -270,6 +281,38 @@ static rc_t vdco_set_row_range( p_dump_context ctx, const char *src )
}
+static rc_t vdco_set_idx_range( p_dump_context ctx, const char *src )
+{
+ rc_t rc = 0;
+ if ( ( ctx == NULL )||( src == NULL ) )
+ {
+ rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
+ }
+ if ( rc == 0 )
+ {
+ rc = vdco_set_str( (char**)&(ctx->idx_range), src );
+ DISP_RC( rc, "dump_context_set_str() failed" );
+ }
+ return rc;
+}
+
+
+static rc_t vdco_set_output_file( p_dump_context ctx, const char *src )
+{
+ rc_t rc = 0;
+ if ( ( ctx == NULL )||( src == NULL ) )
+ {
+ rc = RC( rcVDB, rcNoTarg, rcWriting, rcParam, rcNull );
+ }
+ if ( rc == 0 )
+ {
+ rc = vdco_set_str( (char**)&(ctx->output_file), src );
+ DISP_RC( rc, "dump_context_set_str() failed" );
+ }
+ return rc;
+}
+
+
static bool vdco_set_format( p_dump_context ctx, const char *src )
{
if ( ctx == NULL ) return false;
@@ -284,6 +327,10 @@ static bool vdco_set_format( p_dump_context ctx, const char *src )
ctx->format = df_piped;
else if ( strcmp( src, "tab" ) == 0 )
ctx->format = df_tab;
+ else if ( strcmp( src, "fastq" ) == 0 )
+ ctx->format = df_fastq;
+ else if ( strcmp( src, "fasta" ) == 0 )
+ ctx->format = df_fasta;
else ctx->format = df_default;
return true;
}
@@ -346,6 +393,30 @@ static uint16_t vdco_get_uint16_option( const Args *my_args,
return res;
}
+
+static size_t vdco_get_size_t_option( const Args *my_args,
+ const char *name,
+ const size_t def )
+{
+ size_t res = def;
+ uint32_t count;
+ rc_t rc = ArgsOptionCount( my_args, name, &count );
+ DISP_RC( rc, "ArgsOptionCount() failed" );
+ if ( ( rc == 0 )&&( count > 0 ) )
+ {
+ const char *s;
+ rc = ArgsOptionValue( my_args, name, 0, &s );
+ DISP_RC( rc, "ArgsOptionValue() failed" );
+ if ( rc == 0 )
+ {
+ char *endp;
+ res = strtou64( s, &endp, 10 );
+ }
+ }
+ return res;
+}
+
+
static const char* vdco_get_str_option( const Args *my_args,
const char *name )
{
@@ -420,11 +491,27 @@ static void vdco_evaluate_options( const Args *my_args,
ctx->show_blobbing = vdco_get_bool_option( my_args, OPTION_SHOW_BLOBBING, false );
ctx->enum_phys = vdco_get_bool_option( my_args, OPTION_ENUM_PHYS, false );
ctx->check_curl = vdco_get_bool_option( my_args, OPTION_CHECK_CURL, false );
-
+ ctx->idx_enum_requested = vdco_get_bool_option( my_args, OPTION_IDX_ENUM, false );
+ ctx->disable_multithreading = vdco_get_bool_option( my_args, OPTION_NO_MULTITHREAD, false );
+
+ ctx->cur_cache_size = vdco_get_size_t_option( my_args, OPTION_CUR_CACHE, CURSOR_CACHE_SIZE );
+ ctx->output_buffer_size = vdco_get_size_t_option( my_args, OPTION_OUT_BUF_SIZE, DEF_OPTION_OUT_BUF_SIZE );
+
+ if ( vdco_get_bool_option( my_args, OPTION_GZIP, false ) )
+ ctx->compress_mode = orm_gzip;
+ else if ( vdco_get_bool_option( my_args, OPTION_BZIP2, false ) )
+ ctx->compress_mode = orm_bzip2;
+ else
+ ctx->compress_mode = orm_uncompressed;
+
vdco_set_table( ctx, vdco_get_str_option( my_args, OPTION_TABLE ) );
vdco_set_columns( ctx, vdco_get_str_option( my_args, OPTION_COLUMNS ) );
vdco_set_excluded_columns( ctx, vdco_get_str_option( my_args, OPTION_EXCLUDED_COLUMNS ) );
vdco_set_row_range( ctx, vdco_get_str_option( my_args, OPTION_ROWS ) );
+ vdco_set_idx_range( ctx, vdco_get_str_option( my_args, OPTION_IDX_RANGE ) );
+ vdco_set_output_file( ctx, vdco_get_str_option( my_args, OPTION_OUT_FILE ) );
+
+ ctx->idx_range_requested = ( ctx->idx_range != NULL );
vdco_set_schemas( my_args, ctx );
vdco_set_filter( ctx, vdco_get_str_option( my_args, OPTION_FILTER ) );
vdco_set_boolean_char( ctx, vdco_get_str_option( my_args, OPTION_BOOLEAN ) );
diff --git a/tools/vdb-dump/vdb-dump-context.h b/tools/vdb-dump/vdb-dump-context.h
index 19bcdb5..3d96e02 100644
--- a/tools/vdb-dump/vdb-dump-context.h
+++ b/tools/vdb-dump/vdb-dump-context.h
@@ -37,6 +37,7 @@ extern "C" {
#include <kapp/args.h>
#include <klib/vector.h>
#include "vdb-dump-num-gen.h"
+#include "vdb-dump-redir.h"
#define OPTION_ROW_ID_ON "row_id_on"
#define OPTION_LINE_FEED "line_feed"
@@ -67,6 +68,14 @@ extern "C" {
#define OPTION_SHOW_BLOBBING "blobbing"
#define OPTION_ENUM_PHYS "phys"
#define OPTION_CHECK_CURL "check-curl"
+#define OPTION_IDX_ENUM "idx-report"
+#define OPTION_IDX_RANGE "idx-range"
+#define OPTION_CUR_CACHE "cur-cache"
+#define OPTION_OUT_FILE "output-file"
+#define OPTION_GZIP "gzip"
+#define OPTION_BZIP2 "bzip2"
+#define OPTION_OUT_BUF_SIZE "output-buffer-size"
+#define OPTION_NO_MULTITHREAD "disable-multithreading"
#define ALIAS_ROW_ID_ON "I"
#define ALIAS_LINE_FEED "l"
@@ -95,6 +104,10 @@ extern "C" {
#define ALIAS_NUMELEM "u"
#define ALIAS_NUMELEMSUM "U"
+#define USE_PATHTYPE_TO_DETECT_DB_OR_TAB 1
+#define CURSOR_CACHE_SIZE 256*1024*1024
+#define DEF_OPTION_OUT_BUF_SIZE 1024*1024
+
typedef enum dump_format_t
{
df_default,
@@ -102,7 +115,9 @@ typedef enum dump_format_t
df_xml,
df_json,
df_piped,
- df_tab
+ df_tab,
+ df_fastq,
+ df_fasta
} dump_format_t;
/********************************************************************
@@ -116,13 +131,18 @@ typedef struct dump_context
const char *columns;
const char *excluded_columns;
const char *filter;
+ const char *idx_range;
+ const char *output_file;
num_gen *row_generator;
bool print_row_id;
uint16_t lf_after_row;
uint16_t max_line_len;
uint16_t indented_line_len;
uint32_t generic_idx;
+ size_t cur_cache_size;
+ size_t output_buffer_size;
dump_format_t format;
+ out_redir_mode_t compress_mode;
char c_boolean;
bool print_column_names;
@@ -145,6 +165,9 @@ typedef struct dump_context
bool show_blobbing;
bool enum_phys;
bool check_curl;
+ bool idx_enum_requested;
+ bool idx_range_requested;
+ bool disable_multithreading;
} dump_context;
typedef dump_context* p_dump_context;
diff --git a/tools/vdb-dump/vdb-dump-fastq.c b/tools/vdb-dump/vdb-dump-fastq.c
new file mode 100644
index 0000000..c3c660b
--- /dev/null
+++ b/tools/vdb-dump/vdb-dump-fastq.c
@@ -0,0 +1,498 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "vdb-dump-fastq.h"
+#include "vdb-dump-helper.h"
+#include "vdb-dump-num-gen.h"
+
+#include <stdlib.h>
+
+#include <kdb/manager.h>
+#include <klib/log.h>
+#include <klib/out.h>
+
+rc_t CC Quitting ( void );
+
+#define INVALID_COLUMN 0xFFFFFFFF
+#define DEF_FASTA_LEN 70
+
+typedef struct fastq_ctx
+{
+ const char *run_name;
+ const VCursor * cursor;
+ uint32_t idx_read;
+ uint32_t idx_qual;
+ uint32_t idx_name;
+} fastq_ctx;
+
+
+static void vdb_fastq_row_error( const char * fmt, rc_t rc, int64_t row_id )
+{
+ PLOGERR( klogInt, ( klogInt, rc, fmt, "row_nr=%li", row_id ) );
+}
+
+
+static rc_t vdb_fastq_loop_with_name( const p_dump_context ctx, const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ int64_t row_id;
+
+ vdn_start( ctx->row_generator );
+ while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
+ {
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ uint32_t elem_bits, boff, row_len, name_len;
+ const char * data;
+ const char * name;
+
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_name, &elem_bits,
+ (const void**)&name, &boff, &name_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), NAME ) failed", rc, row_id );
+ else
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
+ (const void**)&data, &boff, &row_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
+ else
+ {
+ rc = KOutMsg( "@%s.%li %.*s length=%u\n%.*s\n",
+ fctx->run_name, row_id, name_len, name, row_len, row_len, data );
+ if ( rc == 0 )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_qual, &elem_bits,
+ (const void**)&data, &boff, &row_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), QUALITY ) failed", rc, row_id );
+ else
+ rc = KOutMsg( "+%s.%li %.*s length=%u\n%.*s\n",
+ fctx->run_name, row_id, name_len, name, row_len, row_len, data );
+ }
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fasta_loop_with_name( const p_dump_context ctx, const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ int64_t row_id;
+
+ vdn_start( ctx->row_generator );
+ while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
+ {
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ uint32_t elem_bits, boff, row_len, name_len;
+ const char * data;
+ const char * name;
+
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_name, &elem_bits,
+ (const void**)&name, &boff, &name_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), NAME ) failed", rc, row_id );
+ else
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
+ (const void**)&data, &boff, &row_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
+ else
+ {
+ uint32_t idx = 0;
+ int32_t to_print = row_len;
+
+ rc = KOutMsg( ">%s.%li %.*s length=%u\n",
+ fctx->run_name, row_id, name_len, name, row_len );
+ if ( to_print > ctx->max_line_len )
+ to_print = ctx->max_line_len;
+ while ( rc == 0 && to_print > 0 )
+ {
+ rc = KOutMsg( "%.*s\n", to_print, &data[ idx ] );
+ if ( rc == 0 )
+ {
+ idx += ctx->max_line_len;
+ to_print = ( row_len - idx );
+ if ( to_print > ctx->max_line_len )
+ to_print = ctx->max_line_len;
+ }
+ }
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fastq_loop_without_name( const p_dump_context ctx, const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ int64_t row_id;
+
+ vdn_start( ctx->row_generator );
+ while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
+ {
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ uint32_t elem_bits, boff, row_len;
+ const char * data;
+
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
+ (const void**)&data, &boff, &row_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
+ else
+ {
+ rc = KOutMsg( "@%s.%li %li length=%u\n%.*s\n",
+ fctx->run_name, row_id, row_id, row_len, row_len, data );
+ if ( rc == 0 )
+ {
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_qual, &elem_bits,
+ (const void**)&data, &boff, &row_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), QUALITY ) failed", rc, row_id );
+ else
+ rc = KOutMsg( "+%s.%li %li length=%u\n%.*s\n",
+ fctx->run_name, row_id, row_id, row_len, row_len, data );
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fasta_loop_without_name( const p_dump_context ctx, const fastq_ctx * fctx )
+{
+ rc_t rc = 0;
+ int64_t row_id;
+
+ vdn_start( ctx->row_generator );
+ while ( vdn_next( ctx->row_generator, (uint64_t*)&row_id ) && rc == 0 )
+ {
+ rc = Quitting();
+ if ( rc == 0 )
+ {
+ uint32_t elem_bits, boff, row_len;
+ const char * data;
+
+ rc = VCursorCellDataDirect( fctx->cursor, row_id, fctx->idx_read, &elem_bits,
+ (const void**)&data, &boff, &row_len );
+ if ( rc != 0 )
+ vdb_fastq_row_error( "VCursorCellDataDirect( row#$(row_nr), READ ) failed", rc, row_id );
+ else
+ {
+ uint32_t idx = 0;
+ int32_t to_print = row_len;
+
+ rc = KOutMsg( ">%s.%li %li length=%u\n",
+ fctx->run_name, row_id, row_id, row_len );
+ if ( to_print > ctx->max_line_len )
+ to_print = ctx->max_line_len;
+ while ( rc == 0 && to_print > 0 )
+ {
+ rc = KOutMsg( "%.*s\n", to_print, &data[ idx ] );
+ if ( rc == 0 )
+ {
+ idx += ctx->max_line_len;
+ to_print = ( row_len - idx );
+ if ( to_print > ctx->max_line_len )
+ to_print = ctx->max_line_len;
+ }
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_prepare_cursor( const p_dump_context ctx, const VTable * tbl, fastq_ctx * fctx )
+{
+ rc_t rc;
+
+ /* first we try to open READ/QUALITY/NAME */
+ rc = VTableCreateCachedCursorRead( tbl, &fctx->cursor, ctx->cur_cache_size );
+ DISP_RC( rc, "VTableCreateCursorRead( 1st ) failed" );
+ if ( rc == 0 )
+ {
+ rc = VCursorAddColumn( fctx->cursor, &fctx->idx_read, "(INSDC:dna:text)READ" );
+ if ( rc == 0 && ctx->format == df_fastq )
+ rc = VCursorAddColumn( fctx->cursor, &fctx->idx_qual, "(INSDC:quality:text:phred_33)QUALITY" );
+ else
+ fctx->idx_qual = INVALID_COLUMN;
+ if ( rc == 0 )
+ rc = VCursorAddColumn( fctx->cursor, &fctx->idx_name, "(ascii)NAME" );
+ if ( rc == 0 )
+ rc = VCursorOpen ( fctx->cursor );
+
+ if ( rc != 0 )
+ {
+ VCursorRelease( fctx->cursor );
+ rc = VTableCreateCachedCursorRead( tbl, &fctx->cursor, ctx->cur_cache_size );
+ DISP_RC( rc, "VTableCreateCursorRead( 2nd ) failed" );
+ if ( rc == 0 )
+ {
+ rc = VCursorAddColumn( fctx->cursor, &fctx->idx_read, "(INSDC:dna:text)READ" );
+ if ( rc == 0 && ctx->format == df_fastq )
+ rc = VCursorAddColumn( fctx->cursor, &fctx->idx_qual, "(INSDC:quality:text:phred_33)QUALITY" );
+ else
+ fctx->idx_qual = INVALID_COLUMN;
+ if ( rc == 0 )
+ rc = VCursorOpen ( fctx->cursor );
+ fctx->idx_name = INVALID_COLUMN;
+ }
+ }
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fastq_tbl( const p_dump_context ctx,
+ const VTable * tbl,
+ fastq_ctx * fctx )
+{
+ rc_t rc = vdb_prepare_cursor( ctx, tbl, fctx );
+ DISP_RC( rc, "the table lacks READ and/or QUALITY column" );
+ if ( rc == 0 )
+ {
+ int64_t first;
+ uint64_t count;
+ rc = VCursorIdRange( fctx->cursor, fctx->idx_read, &first, &count );
+ DISP_RC( rc, "VCursorIdRange() failed" );
+ if ( rc == 0 )
+ {
+ /* if the user did not specify a row-range, take all rows */
+ if ( vdn_range_defined( ctx->row_generator ) == false )
+ {
+ vdn_set_range( ctx->row_generator, first, count );
+ }
+ /* if the user did specify a row-range, check the boundaries */
+ else
+ {
+ vdn_check_range( ctx->row_generator, first, count );
+ }
+
+ if ( vdn_range_defined( ctx->row_generator ) )
+ {
+ if ( ctx->format == df_fastq )
+ {
+ if ( fctx->idx_name == INVALID_COLUMN)
+ rc = vdb_fastq_loop_without_name( ctx, fctx ); /* <--- */
+ else
+ rc = vdb_fastq_loop_with_name( ctx, fctx ); /* <--- */
+ }
+ else if ( ctx->format == df_fasta )
+ {
+ if ( ctx->max_line_len == 0 )
+ ctx->max_line_len = DEF_FASTA_LEN;
+ if ( fctx->idx_name == INVALID_COLUMN)
+ rc = vdb_fasta_loop_without_name( ctx, fctx ); /* <--- */
+ else
+ rc = vdb_fasta_loop_with_name( ctx, fctx ); /* <--- */
+ }
+ }
+ else
+ {
+ rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcEmpty );
+ }
+ }
+ VCursorRelease( fctx->cursor );
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fastq_table( const p_dump_context ctx,
+ const VDBManager *mgr,
+ fastq_ctx * fctx )
+{
+ const VTable * tbl;
+ VSchema * schema = NULL;
+ rc_t rc;
+
+ vdh_parse_schema( mgr, &schema, &(ctx->schema_list) );
+
+ rc = VDBManagerOpenTableRead( mgr, &tbl, schema, ctx->path );
+ DISP_RC( rc, "VDBManagerOpenTableRead() failed" );
+ if ( rc == 0 )
+ {
+ rc = vdb_fastq_tbl( ctx, tbl, fctx );
+ VTableRelease( tbl );
+ }
+ VSchemaRelease( schema );
+
+ return rc;
+}
+
+
+static rc_t vdb_fastq_database( const p_dump_context ctx,
+ const VDBManager *mgr,
+ fastq_ctx * fctx )
+{
+ const VDatabase * db;
+ VSchema *schema = NULL;
+ rc_t rc;
+
+ vdh_parse_schema( mgr, &schema, &(ctx->schema_list) );
+
+ rc = VDBManagerOpenDBRead( mgr, &db, schema, ctx->path );
+ DISP_RC( rc, "VDBManagerOpenDBRead() failed" );
+ if ( rc == 0 )
+ {
+ bool table_defined = ( ctx->table != NULL );
+ if ( !table_defined )
+ table_defined = vdh_take_this_table_from_db( ctx, db, "SEQUENCE" );
+
+ if ( table_defined )
+ {
+ const VTable * tbl;
+
+ rc = VDatabaseOpenTableRead( db, &tbl, ctx->table );
+ DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ if ( rc == 0 )
+ {
+ rc = vdb_fastq_tbl( ctx, tbl, fctx );
+ VTableRelease( tbl );
+ }
+ }
+ else
+ {
+ LOGMSG( klogInfo, "opened as vdb-database, but no table found/defined" );
+ ctx->usage_requested = true;
+ }
+ VDatabaseRelease( db );
+ }
+
+ VSchemaRelease( schema );
+ return rc;
+}
+
+
+static rc_t vdb_fastq_by_pathtype( const p_dump_context ctx,
+ const VDBManager *mgr,
+ fastq_ctx * fctx )
+{
+ rc_t rc;
+ int path_type = ( VDBManagerPathType ( mgr, ctx->path ) & ~ kptAlias );
+ /* types defined in <kdb/manager.h> */
+ switch ( path_type )
+ {
+ case kptDatabase : rc = vdb_fastq_database( ctx, mgr, fctx );
+ DISP_RC( rc, "dump_database() failed" );
+ break;
+
+ case kptPrereleaseTbl:
+ case kptTable : rc = vdb_fastq_table( ctx, mgr, fctx );
+ DISP_RC( rc, "dump_table() failed" );
+ break;
+
+ default : rc = RC( rcVDB, rcNoTarg, rcConstructing, rcItem, rcNotFound );
+ PLOGERR( klogInt, ( klogInt, rc,
+ "the path '$(p)' cannot be opened as vdb-database or vdb-table",
+ "p=%s", ctx->path ) );
+ if ( vdco_schema_count( ctx ) == 0 )
+ {
+ LOGERR( klogInt, rc, "Maybe it is a legacy table. If so, specify a schema with the -S option" );
+ }
+ break;
+ }
+ return rc;
+}
+
+
+static rc_t vdb_fastq_by_probing( const p_dump_context ctx,
+ const VDBManager *mgr,
+ fastq_ctx * fctx )
+{
+ rc_t rc;
+ if ( vdh_is_path_database( mgr, ctx->path, &(ctx->schema_list) ) )
+ {
+ rc = vdb_fastq_database( ctx, mgr, fctx );
+ DISP_RC( rc, "dump_database() failed" );
+ }
+ else if ( vdh_is_path_table( mgr, ctx->path, &(ctx->schema_list) ) )
+ {
+ rc = vdb_fastq_table( ctx, mgr, fctx );
+ DISP_RC( rc, "dump_table() failed" );
+ }
+ else
+ {
+ rc = RC( rcVDB, rcNoTarg, rcConstructing, rcItem, rcNotFound );
+ PLOGERR( klogInt, ( klogInt, rc,
+ "the path '$(p)' cannot be opened as vdb-database or vdb-table",
+ "p=%s", ctx->path ) );
+ if ( vdco_schema_count( ctx ) == 0 )
+ {
+ LOGERR( klogInt, rc, "Maybe it is a legacy table. If so, specify a schema with the -S option" );
+ }
+ }
+ return rc;
+}
+
+
+static char * vdb_fastq_extract_run_name( const char * acc_or_path )
+{
+ char * delim = string_rchr ( acc_or_path, string_size( acc_or_path ), '/' );
+ if ( delim == NULL )
+ return string_dup_measure ( acc_or_path, NULL );
+ else
+ return string_dup_measure ( delim + 1, NULL );
+}
+
+
+rc_t vdf_main( const p_dump_context ctx, const VDBManager * mgr, const char * acc_or_path )
+{
+ rc_t rc = 0;
+ fastq_ctx fctx;
+ fctx.run_name = vdb_fastq_extract_run_name( acc_or_path );
+
+ ctx->path = string_dup_measure ( acc_or_path, NULL );
+
+ if ( USE_PATHTYPE_TO_DETECT_DB_OR_TAB ) /* in vdb-dump-context.h */
+ {
+ rc = vdb_fastq_by_pathtype( ctx, mgr, &fctx );
+ }
+ else
+ {
+ rc = vdb_fastq_by_probing( ctx, mgr, &fctx );
+ }
+
+ free( (char*)ctx->path );
+ free( (void*)fctx.run_name );
+ ctx->path = NULL;
+
+ return rc;
+}
diff --git a/tools/sra-pileup/reref.h b/tools/vdb-dump/vdb-dump-fastq.h
similarity index 85%
copy from tools/sra-pileup/reref.h
copy to tools/vdb-dump/vdb-dump-fastq.h
index 13e3136..652718c 100644
--- a/tools/sra-pileup/reref.h
+++ b/tools/vdb-dump/vdb-dump-fastq.h
@@ -23,8 +23,9 @@
* ===========================================================================
*
*/
-#ifndef _h_reref_
-#define _h_reref_
+
+#ifndef _h_vdb_dump_fastq_
+#define _h_vdb_dump_fastq_
#ifdef __cplusplus
extern "C" {
@@ -33,16 +34,14 @@ extern "C" {
}
#endif
-#include <kapp/args.h>
-
-#include <klib/out.h>
-#include <klib/rc.h>
-#include <klib/log.h>
-
+#include "vdb-dump-context.h"
#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
+#include <kfs/directory.h>
+
+rc_t vdf_main( const p_dump_context ctx, const VDBManager * mgr, const char * acc_or_path );
-rc_t report_on_reference( Args * args, bool extended );
+#ifdef __cplusplus
+}
+#endif
#endif
diff --git a/tools/vdb-dump/vdb-dump-redir.c b/tools/vdb-dump/vdb-dump-redir.c
new file mode 100644
index 0000000..83bd1b3
--- /dev/null
+++ b/tools/vdb-dump/vdb-dump-redir.c
@@ -0,0 +1,120 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "vdb-dump-redir.h"
+
+#include <kfs/directory.h>
+#include <kfs/buffile.h>
+#include <kfs/bzip.h>
+#include <kfs/gzip.h>
+#include <sysalloc.h>
+
+static rc_t CC out_redir_callback( void * self, const char * buffer, size_t bufsize, size_t * num_writ )
+{
+ out_redir * redir = ( out_redir * )self;
+ rc_t rc = KFileWriteAll( redir->kfile, redir->pos, buffer, bufsize, num_writ );
+ if ( rc == 0 )
+ redir->pos += *num_writ;
+ return rc;
+}
+
+
+rc_t init_out_redir( out_redir * self, out_redir_mode_t mode, const char * filename, size_t bufsize )
+{
+ rc_t rc;
+ KFile *output_file;
+
+ if ( filename != NULL )
+ {
+ KDirectory *dir;
+ rc = KDirectoryNativeDir( &dir );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "KDirectoryNativeDir() failed" );
+ else
+ {
+ rc = KDirectoryCreateFile ( dir, &output_file, false, 0664, kcmInit, "%s", filename );
+ KDirectoryRelease( dir );
+ }
+ }
+ else
+ rc = KFileMakeStdOut ( &output_file );
+
+ if ( rc == 0 )
+ {
+ KFile *temp_file;
+
+ /* wrap the output-file in compression, if requested */
+ switch ( mode )
+ {
+ case orm_gzip : rc = KFileMakeGzipForWrite( &temp_file, output_file ); break;
+ case orm_bzip2 : rc = KFileMakeBzip2ForWrite( &temp_file, output_file ); break;
+ case orm_uncompressed : break;
+ }
+ if ( rc == 0 )
+ {
+ if ( mode != orm_uncompressed )
+ {
+ KFileRelease( output_file );
+ output_file = temp_file;
+ }
+
+ /* wrap the output/compressed-file in buffering, if requested */
+ if ( bufsize != 0 )
+ {
+ rc = KBufFileMakeWrite( &temp_file, output_file, false, bufsize );
+ if ( rc == 0 )
+ {
+ KFileRelease( output_file );
+ output_file = temp_file;
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ self->kfile = output_file;
+ self->org_writer = KOutWriterGet();
+ self->org_data = KOutDataGet();
+ self->pos = 0;
+ rc = KOutHandlerSet( out_redir_callback, self );
+ if ( rc != 0 )
+ LOGERR( klogInt, rc, "KOutHandlerSet() failed" );
+ }
+ }
+ }
+ return rc;
+}
+
+
+void release_out_redir( out_redir * self )
+{
+ KFileRelease( self->kfile );
+ if( self->org_writer != NULL )
+ {
+ KOutHandlerSet( self->org_writer, self->org_data );
+ }
+ self->org_writer = NULL;
+}
+
diff --git a/tools/sra-pileup/reref.h b/tools/vdb-dump/vdb-dump-redir.h
similarity index 74%
copy from tools/sra-pileup/reref.h
copy to tools/vdb-dump/vdb-dump-redir.h
index 13e3136..ceb198f 100644
--- a/tools/sra-pileup/reref.h
+++ b/tools/vdb-dump/vdb-dump-redir.h
@@ -23,8 +23,8 @@
* ===========================================================================
*
*/
-#ifndef _h_reref_
-#define _h_reref_
+#ifndef _h_out_redir_
+#define _h_out_redir_
#ifdef __cplusplus
extern "C" {
@@ -33,16 +33,32 @@ extern "C" {
}
#endif
-#include <kapp/args.h>
-
#include <klib/out.h>
#include <klib/rc.h>
#include <klib/log.h>
-#include <vdb/manager.h>
-#include <vdb/vdb-priv.h>
-#include <kdb/manager.h>
+#include <kfs/file.h>
+
+typedef enum out_redir_mode
+{
+ orm_uncompressed = 0,
+ orm_gzip,
+ orm_bzip2
+} out_redir_mode_t;
+
+
+/* GLOBAL VARIABLES */
+typedef struct out_redir
+{
+ KWrtWriter org_writer;
+ void* org_data;
+ KFile* kfile;
+ uint64_t pos;
+} out_redir;
+
+
+rc_t init_out_redir( out_redir * self, out_redir_mode_t mode, const char * filename, size_t bufsize );
-rc_t report_on_reference( Args * args, bool extended );
+void release_out_redir( out_redir * self );
#endif
diff --git a/tools/vdb-dump/vdb-dump.c b/tools/vdb-dump/vdb-dump.c
index 880b5ca..e09a771 100644
--- a/tools/vdb-dump/vdb-dump.c
+++ b/tools/vdb-dump/vdb-dump.c
@@ -37,6 +37,7 @@
#include <kdb/table.h>
#include <kdb/column.h>
#include <kdb/manager.h>
+#include <kdb/namelist.h>
#include <kfs/directory.h>
#include <kns/manager.h>
@@ -68,9 +69,8 @@
#include "vdb-dump-helper.h"
#include "vdb-dump-row-context.h"
#include "vdb-dump-formats.h"
-
-
-#define CURSOR_CACHE_SIZE 256*1024*1024
+#include "vdb-dump-fastq.h"
+#include "vdb-dump-redir.h"
static const char * row_id_on_usage[] = { "print row id", NULL };
static const char * line_feed_usage[] = { "line-feed's inbetween rows", NULL };
@@ -88,7 +88,7 @@ static const char * dna_bases_usage[] = { "print dna-bases", NULL };
static const char * max_line_len_usage[] = { "limits line length", NULL };
static const char * line_indent_usage[] = { "indents the line", NULL };
static const char * filter_usage[] = { "filters lines", NULL };
-static const char * format_usage[] = { "dump format (csv,xml,json,piped,tab)", NULL };
+static const char * format_usage[] = { "dump format (csv,xml,json,piped,tab,fastq,fasta)", NULL };
static const char * id_range_usage[] = { "prints id-range", NULL };
static const char * without_sra_usage[] = { "without sra-type-translation", NULL };
static const char * without_accession_usage[] = { "without accession-test", NULL };
@@ -100,6 +100,14 @@ static const char * numelemsum_usage[] = { "sum element-count", NULL };
static const char * show_blobbing_usage[] = { "show blobbing", NULL };
static const char * enum_phys_usage[] = { "enumerate physical columns", NULL };
static const char * objtype_usage[] = { "report type of object", NULL };
+static const char * idx_enum_usage[] = { "enumerate available index", NULL };
+static const char * idx_range_usage[] = { "show row-range of idx entry", NULL };
+static const char * cur_cache_usage[] = { "size of cursor cache", NULL };
+static const char * out_file_usage[] = { "write output to this file", NULL };
+static const char * gzip_usage[] = { "compress output using gzip", NULL };
+static const char * bzip2_usage[] = { "compress output using bzip2", NULL };
+static const char * outbuf_size_usage[] = { "size of output-buffer, 0...none", NULL };
+static const char * disable_mt_usage[] = { "disable multithreading", NULL };
OptDef DumpOptions[] =
{
@@ -132,7 +140,15 @@ OptDef DumpOptions[] =
{ OPTION_ENUM_PHYS, NULL, NULL, enum_phys_usage, 1, false, false },
{ OPTION_CHECK_CURL, NULL, NULL, NULL, 1, false, false },
{ OPTION_OBJVER, ALIAS_OBJVER, NULL, objver_usage, 1, false, false },
- { OPTION_OBJTYPE, ALIAS_OBJTYPE, NULL, objtype_usage, 1, false, false }
+ { OPTION_OBJTYPE, ALIAS_OBJTYPE, NULL, objtype_usage, 1, false, false },
+ { OPTION_IDX_ENUM, NULL, NULL, idx_enum_usage, 1, false, false },
+ { OPTION_IDX_RANGE, NULL, NULL, idx_range_usage, 1, true, false },
+ { OPTION_CUR_CACHE, NULL, NULL, cur_cache_usage, 1, true, false },
+ { OPTION_OUT_FILE, NULL, NULL, out_file_usage, 1, true, false },
+ { OPTION_GZIP, NULL, NULL, gzip_usage, 1, false, false },
+ { OPTION_BZIP2, NULL, NULL, bzip2_usage, 1, false, false },
+ { OPTION_OUT_BUF_SIZE, NULL, NULL, outbuf_size_usage, 1, true, false },
+ { OPTION_NO_MULTITHREAD, NULL, NULL, disable_mt_usage, 1, false, false },
};
const char UsageDefaultName[] = "vdb-dump";
@@ -192,7 +208,15 @@ rc_t CC Usage ( const Args * args )
HelpOptionLine ( ALIAS_NUMELEMSUM, OPTION_NUMELEMSUM, NULL, numelemsum_usage );
HelpOptionLine ( NULL, OPTION_SHOW_BLOBBING, NULL, show_blobbing_usage );
HelpOptionLine ( NULL, OPTION_ENUM_PHYS, NULL, enum_phys_usage );
-
+ HelpOptionLine ( NULL, OPTION_IDX_ENUM, NULL, idx_enum_usage );
+ HelpOptionLine ( NULL, OPTION_IDX_RANGE, NULL, idx_range_usage );
+ HelpOptionLine ( NULL, OPTION_CUR_CACHE, NULL, cur_cache_usage );
+ HelpOptionLine ( NULL, OPTION_OUT_FILE, NULL, out_file_usage );
+ HelpOptionLine ( NULL, OPTION_GZIP, NULL, gzip_usage );
+ HelpOptionLine ( NULL, OPTION_BZIP2, NULL, bzip2_usage );
+ HelpOptionLine ( NULL, OPTION_OUT_BUF_SIZE, NULL, outbuf_size_usage );
+ HelpOptionLine ( NULL, OPTION_NO_MULTITHREAD, NULL, disable_mt_usage );
+
HelpOptionsStandard ();
HelpVersion ( fullpath, KAppVersion() );
@@ -388,15 +412,19 @@ static rc_t vdm_dump_rows( p_row_context r_ctx )
break;
r_ctx->rc = VCursorSetRowId( r_ctx->cursor, r_ctx->row_id );
if ( r_ctx->rc != 0 )
+ {
vdm_row_error( "VCursorSetRowId( row#$(row_nr) ) failed",
r_ctx->rc, r_ctx->row_id );
- if ( r_ctx->rc == 0 )
+ }
+ else
{
r_ctx->rc = VCursorOpenRow( r_ctx->cursor );
if ( r_ctx->rc != 0 )
+ {
vdm_row_error( "VCursorOpenRow( row#$(row_nr) ) failed",
r_ctx->rc, r_ctx->row_id );
- if ( r_ctx->rc == 0 )
+ }
+ else
{
/* first reset the string and valid-flag for every column */
vdcd_reset_content( r_ctx->col_defs );
@@ -504,7 +532,7 @@ my_table [IN] ... open table needed for vdb-calls
static rc_t vdm_dump_opened_table( const p_dump_context ctx, const VTable *my_table )
{
row_context r_ctx;
- rc_t rc = VTableCreateCachedCursorRead( my_table, &(r_ctx.cursor), CURSOR_CACHE_SIZE );
+ rc_t rc = VTableCreateCachedCursorRead( my_table, &(r_ctx.cursor), ctx->cur_cache_size );
DISP_RC( rc, "VTableCreateCursorRead() failed" );
if ( rc == 0 )
{
@@ -621,12 +649,13 @@ my_database [IN] ... open database needed for vdb-calls
static rc_t vdm_dump_tab_schema( const p_dump_context ctx,
const VTable *my_table )
{
- const VSchema *my_schema;
+ const VSchema * my_schema;
rc_t rc = VTableOpenSchema( my_table, &my_schema );
DISP_RC( rc, "VTableOpenSchema() failed" );
if ( rc == 0 )
{
- rc = VSchemaDump( my_schema, sdmPrint, NULL,
+ const char * decl = ctx->columns;
+ rc = VSchemaDump( my_schema, sdmPrint, decl,
vdm_schema_dump_flush, stdout );
DISP_RC( rc, "VSchemaDump() failed" );
VSchemaRelease( my_schema );
@@ -1042,6 +1071,167 @@ static rc_t vdm_print_db_id_range( const p_dump_context ctx, const VDatabase *my
return rc;
}
+
+
+/* ************************************************************************************ */
+
+static rc_t vdm_enum_index( const KTable * my_ktable, uint32_t idx_nr, const char * idx_name )
+{
+ rc_t rc = KOutMsg( "idx #%u: %s", idx_nr + 1, idx_name );
+ if ( rc == 0 )
+ {
+ const KIndex * my_idx;
+ rc = KTableOpenIndexRead ( my_ktable, &my_idx, idx_name );
+ if ( rc != 0 )
+ rc = KOutMsg( " (cannot open)" );
+ else
+ {
+ uint32_t idx_version;
+ rc = KIndexVersion ( my_idx, &idx_version );
+ if ( rc != 0 )
+ rc = KOutMsg( " V?.?.?" );
+ else
+ rc = KOutMsg( " V%V", idx_version );
+
+ if ( rc == 0 )
+ {
+ KIdxType idx_type;
+ rc = KIndexType ( my_idx, &idx_type );
+ if ( rc != 0 )
+ rc = KOutMsg( " type = ?" );
+ else
+ {
+ switch ( idx_type &~ kitProj )
+ {
+ case kitText : rc = KOutMsg( " type = Text" ); break;
+ case kitU64 : rc = KOutMsg( " type = U64" ); break;
+ default : rc = KOutMsg( " type = unknown" ); break;
+ }
+ if ( rc == 0 && ( ( idx_type & kitProj ) == kitProj ) )
+ rc = KOutMsg( " reverse" );
+ }
+ }
+
+ if ( rc == 0 )
+ {
+ bool locked = KIndexLocked ( my_idx );
+ if ( locked )
+ rc = KOutMsg( " locked" );
+ }
+ KIndexRelease( my_idx );
+ }
+ }
+ if ( rc == 0 )
+ rc = KOutMsg( "\n" );
+ return rc;
+}
+
+
+static rc_t vdm_enum_tab_index( const p_dump_context ctx, const VTable *my_table )
+{
+ const KTable * my_ktable;
+ rc_t rc = VTableOpenKTableRead( my_table, &my_ktable );
+ DISP_RC( rc, "VTableOpenKTableRead() failed" );
+ if ( rc == 0 )
+ {
+ KNamelist *idx_names;
+ rc = KTableListIdx ( my_ktable, &idx_names );
+ if ( rc == 0 )
+ {
+ uint32_t count;
+ rc = KNamelistCount( idx_names, &count );
+ if ( rc == 0 )
+ {
+ uint32_t i;
+ for ( i = 0; i < count && rc == 0; ++i )
+ {
+ const char * idx_name = NULL;
+ rc = KNamelistGet( idx_names, i, &idx_name );
+ if ( rc == 0 && idx_name != NULL )
+ rc = vdm_enum_index( my_ktable, i, idx_name );
+ }
+ }
+ KNamelistRelease( idx_names );
+ }
+ else
+ rc = KOutMsg( "no index available\n" );
+ KTableRelease( my_ktable );
+ }
+ return rc;
+}
+
+static rc_t vdm_enum_db_index( const p_dump_context ctx, const VDatabase *my_database )
+{
+ rc_t rc;
+ const VTable *my_table;
+
+ rc = VDatabaseOpenTableRead( my_database, &my_table, ctx->table );
+ DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ if ( rc == 0 )
+ {
+ rc = vdm_enum_tab_index( ctx, my_table );
+ VTableRelease( my_table );
+ }
+ return rc;
+}
+
+
+/* ************************************************************************************ */
+
+static rc_t CC on_index_range( int64_t start_id, uint64_t id_count, void * data )
+{
+ return KOutMsg( "from %lu to %lu\n", start_id, start_id + id_count - 1 );
+}
+
+static rc_t vdm_range_tab_index( const p_dump_context ctx, const VTable *my_table )
+{
+ const KTable * my_ktable;
+ rc_t rc = VTableOpenKTableRead( my_table, &my_ktable );
+ DISP_RC( rc, "VTableOpenKTableRead() failed" );
+ if ( rc == 0 )
+ {
+ size_t s = string_size( ctx->idx_range );
+ char * equal = string_chr ( ctx->idx_range, s, '=' );
+ if ( equal == NULL )
+ rc = KOutMsg( "usage: IDX_NAME=VALUE\n" );
+ else
+ {
+ const KIndex * my_idx;
+ char * idx_name = string_dup( ctx->idx_range, (size_t)( equal - ( ctx->idx_range ) ) );
+ rc = KTableOpenIndexRead ( my_ktable, &my_idx, idx_name );
+ if ( rc != 0 )
+ rc = KOutMsg( "cannot open index '%s'", idx_name );
+ else
+ {
+ KOutMsg( "index '%s' opened\n", idx_name );
+ rc = KIndexFindAllText ( my_idx, equal + 1, on_index_range, NULL );
+ KIndexRelease( my_idx );
+ }
+ free( idx_name );
+ }
+ KTableRelease( my_ktable );
+ }
+ return rc;
+}
+
+
+static rc_t vdm_range_db_index( const p_dump_context ctx, const VDatabase *my_database )
+{
+ rc_t rc;
+ const VTable *my_table;
+
+ rc = VDatabaseOpenTableRead( my_database, &my_table, ctx->table );
+ DISP_RC( rc, "VDatabaseOpenTableRead() failed" );
+ if ( rc == 0 )
+ {
+ rc = vdm_range_tab_index( ctx, my_table );
+ VTableRelease( my_table );
+ }
+ return rc;
+}
+
+
+
typedef rc_t (*db_tab_t)( const p_dump_context ctx, const VTable *a_tab );
/*************************************************************************************
@@ -1077,7 +1267,7 @@ static rc_t vdm_dump_tab_fkt( const p_dump_context ctx,
VTableRelease( my_table );
}
- if ( my_schema )
+ if ( my_schema != NULL )
VSchemaRelease( my_schema );
return rc;
@@ -1125,6 +1315,14 @@ static rc_t vdm_dump_table( const p_dump_context ctx, const VDBManager *my_manag
{
rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_print_tab_id_range );
}
+ else if ( ctx->idx_enum_requested )
+ {
+ rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_enum_tab_index );
+ }
+ else if ( ctx->idx_range_requested )
+ {
+ rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_range_tab_index );
+ }
else
{
rc = vdm_dump_tab_fkt( ctx, my_manager, vdm_dump_opened_table );
@@ -1226,6 +1424,14 @@ static rc_t vdm_dump_database( const p_dump_context ctx, const VDBManager *my_ma
{
rc = vdm_dump_db_fkt( ctx, my_manager, vdm_print_db_id_range );
}
+ else if ( ctx->idx_enum_requested )
+ {
+ rc = vdm_dump_db_fkt( ctx, my_manager, vdm_enum_db_index );
+ }
+ else if ( ctx->idx_range_requested )
+ {
+ rc = vdm_dump_db_fkt( ctx, my_manager, vdm_range_db_index );
+ }
else
{
rc = vdm_dump_db_fkt( ctx, my_manager, vdm_dump_opened_database );
@@ -1272,14 +1478,11 @@ static void vdm_print_objtype( const VDBManager *mgr, const char * acc_or_path )
}
-#define USE_PATHTYPE_TO_DETECT_DB_OR_TAB 1
-
static rc_t vdb_main_one_obj_by_pathtype( const p_dump_context ctx,
- const VDBManager *mgr,
- const char * acc_or_path )
+ const VDBManager *mgr )
{
rc_t rc;
- int path_type = ( VDBManagerPathType ( mgr, acc_or_path ) & ~ kptAlias );
+ int path_type = ( VDBManagerPathType ( mgr, ctx->path ) & ~ kptAlias );
/* types defined in <kdb/manager.h> */
switch ( path_type )
{
@@ -1307,8 +1510,7 @@ static rc_t vdb_main_one_obj_by_pathtype( const p_dump_context ctx,
static rc_t vdb_main_one_obj_by_probing( const p_dump_context ctx,
- const VDBManager *mgr,
- const char * acc_or_path )
+ const VDBManager *mgr )
{
rc_t rc;
if ( vdh_is_path_database( mgr, ctx->path, &(ctx->schema_list) ) )
@@ -1337,7 +1539,7 @@ static rc_t vdb_main_one_obj_by_probing( const p_dump_context ctx,
static rc_t vdm_main_one_obj( const p_dump_context ctx,
- KDirectory *dir, const VDBManager *mgr,
+ const VDBManager *mgr,
const char * acc_or_path )
{
rc_t rc = 0;
@@ -1354,13 +1556,13 @@ static rc_t vdm_main_one_obj( const p_dump_context ctx,
}
else
{
- if ( USE_PATHTYPE_TO_DETECT_DB_OR_TAB )
+ if ( USE_PATHTYPE_TO_DETECT_DB_OR_TAB ) /* in vdb-dump-context.h */
{
- rc = vdb_main_one_obj_by_pathtype( ctx, mgr, acc_or_path );
+ rc = vdb_main_one_obj_by_pathtype( ctx, mgr );
}
else
{
- rc = vdb_main_one_obj_by_probing( ctx, mgr, acc_or_path );
+ rc = vdb_main_one_obj_by_probing( ctx, mgr );
}
}
@@ -1428,6 +1630,13 @@ static rc_t vdm_main( const p_dump_context ctx, Args * args )
DISP_RC( rc, "VDBManagerMakeRead() failed" );
if ( rc == 0 )
{
+ if ( ctx->disable_multithreading )
+ {
+ rc = VDBManagerDisablePagemapThread ( mgr );
+ DISP_RC( rc, "VDBManagerDisablePagemapThread() failed" );
+ rc = 0;
+ }
+
/* show manager is independend form db or tab */
if ( ctx->version_requested )
{
@@ -1455,7 +1664,12 @@ static rc_t vdm_main( const p_dump_context ctx, Args * args )
DISP_RC( rc, "ArgsParamValue() failed" );
if ( rc == 0 )
{
- rc = vdm_main_one_obj( ctx, dir, mgr, value );
+ switch( ctx->format )
+ {
+ case df_fastq : ;
+ case df_fasta : vdf_main( ctx, mgr, value ); break;
+ default : rc = vdm_main_one_obj( ctx, mgr, value ); break;
+ }
}
}
}
@@ -1514,8 +1728,18 @@ rc_t CC KMain ( int argc, char *argv [] )
rc = vdco_capture_arguments_and_options( args, ctx );
if ( rc == 0 )
{
+ out_redir redir; /* vdb-dump-redir.h */
+
KLogHandlerSetStdErr();
- rc = vdm_main( ctx, args );
+ rc = init_out_redir( &redir,
+ ctx->compress_mode,
+ ctx->output_file,
+ ctx->output_buffer_size ); /* vdb-dump-redir.c */
+ if ( rc == 0 )
+ {
+ rc = vdm_main( ctx, args );
+ release_out_redir( &redir ); /* vdb-dump-redir.c */
+ }
}
vdco_destroy( ctx );
}
diff --git a/tools/vdb-dump/vdb-dump.vers b/tools/vdb-dump/vdb-dump.vers
index 0bee604..3f684d2 100644
--- a/tools/vdb-dump/vdb-dump.vers
+++ b/tools/vdb-dump/vdb-dump.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
diff --git a/tools/vdb-validate/vdb-validate.vers b/tools/vdb-validate/vdb-validate.vers
index 0bee604..3f684d2 100644
--- a/tools/vdb-validate/vdb-validate.vers
+++ b/tools/vdb-validate/vdb-validate.vers
@@ -1 +1 @@
-2.3.3
+2.3.4
--
Alioth's /git/debian-med/git-commit-notice on /srv/git.debian.org/git/debian-med/sra-sdk.git
More information about the debian-med-commit
mailing list